diff --git a/build.gradle b/build.gradle index 1aa2113946..989f43ea93 100644 --- a/build.gradle +++ b/build.gradle @@ -127,3 +127,9 @@ checkstyle { checkstyleMain.ignoreFailures = false checkstyleTest.ignoreFailures = true +configurations.all { + exclude group: "commons-logging", module: "commons-logging" + // enforce 1.1.3, https://www.whitesourcesoftware.com/vulnerability-database/WS-2019-0379 + resolutionStrategy.force 'commons-codec:commons-codec:1.13' + resolutionStrategy.force 'com.google.guava:guava:29.0-jre' +} \ No newline at end of file diff --git a/core/build.gradle b/core/build.gradle index f53c250869..f1ebf57496 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -7,6 +7,12 @@ plugins { repositories { mavenCentral() } +// +//configurations.all { +// resolutionStrategy.dependencySubstitution { +// substitute module('com.google.guava:guava:26.0-jre') with module('com.google.guava:guava:29.0-jre') +// } +//} dependencies { // https://github.com/google/guava/wiki/CVE-2018-10237 @@ -14,6 +20,7 @@ dependencies { compile group: 'org.springframework', name: 'spring-context', version: '5.2.5.RELEASE' compile group: 'org.springframework', name: 'spring-beans', version: '5.2.5.RELEASE' compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.10' + compile group: 'com.facebook.presto', name: 'presto-matching', version: '0.240' compile project(':common') testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementor.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementor.java index e7c68df204..2d792285b2 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementor.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementor.java @@ -21,6 +21,8 @@ import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalEval; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalFilter; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalHead; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalIndexScan; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalIndexScanAggregation; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanNodeVisitor; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalProject; @@ -132,6 +134,18 @@ public PhysicalPlan visitRelation(LogicalRelation node, C context) { + "implementing and optimizing logical plan with relation involved"); } + @Override + public PhysicalPlan visitIndexScan(LogicalIndexScan plan, C context) { + throw new UnsupportedOperationException("Storage engine is responsible for " + + "implementing and optimizing logical plan with relation involved"); + } + + @Override + public PhysicalPlan visitIndexScanAggregation(LogicalIndexScanAggregation plan, C context) { + throw new UnsupportedOperationException("Storage engine is responsible for " + + "implementing and optimizing logical plan with relation involved"); + } + protected PhysicalPlan visitChild(LogicalPlan node, C context) { // Logical operators visited here must have a single child return node.getChild().get(0).accept(this, context); diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/Planner.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/Planner.java index e47232bdb6..31066f1fc5 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/Planner.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/Planner.java @@ -21,6 +21,7 @@ import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanNodeVisitor; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRelation; +import com.amazon.opendistroforelasticsearch.sql.planner.optimizer.LogicalPlanOptimizer; import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; import com.amazon.opendistroforelasticsearch.sql.storage.Table; @@ -38,6 +39,8 @@ public class Planner { */ private final StorageEngine storageEngine; + private final LogicalPlanOptimizer logicalOptimizer; + /** * Generate optimal physical plan for logical plan. If no table involved, * translate logical plan to physical by default implementor. @@ -53,7 +56,7 @@ public PhysicalPlan plan(LogicalPlan plan) { } Table table = storageEngine.getTable(tableName); - return table.implement(plan); + return table.implement(optimize(plan)); } private String findTableName(LogicalPlan plan) { @@ -75,4 +78,7 @@ public String visitRelation(LogicalRelation node, Object context) { }, null); } + private LogicalPlan optimize(LogicalPlan plan) { + return logicalOptimizer.optimize(plan); + } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalAggregation.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalAggregation.java index e4d9b690ac..ddfdbd5a01 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalAggregation.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalAggregation.java @@ -28,18 +28,25 @@ * Logical Aggregation. */ @ToString -@EqualsAndHashCode -@RequiredArgsConstructor +@EqualsAndHashCode(callSuper = true) public class LogicalAggregation extends LogicalPlan { - private final LogicalPlan child; + @Getter private final List aggregatorList; + @Getter private final List groupByList; - @Override - public List getChild() { - return Collections.singletonList(child); + /** + * Constructor of LogicalAggregation. + */ + public LogicalAggregation( + LogicalPlan child, + List aggregatorList, + List groupByList) { + super(Collections.singletonList(child)); + this.aggregatorList = aggregatorList; + this.groupByList = groupByList; } @Override diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalDedupe.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalDedupe.java index 5276c7a0b6..66dc74ff3c 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalDedupe.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalDedupe.java @@ -28,18 +28,26 @@ */ @Getter @ToString -@EqualsAndHashCode(callSuper = false) -@RequiredArgsConstructor +@EqualsAndHashCode(callSuper = true) public class LogicalDedupe extends LogicalPlan { - private final LogicalPlan child; + private final List dedupeList; private final Integer allowedDuplication; private final Boolean keepEmpty; private final Boolean consecutive; - @Override - public List getChild() { - return Arrays.asList(child); + /** + * Constructor of LogicalDedupe. + */ + public LogicalDedupe( + LogicalPlan child, + List dedupeList, Integer allowedDuplication, Boolean keepEmpty, + Boolean consecutive) { + super(Arrays.asList(child)); + this.dedupeList = dedupeList; + this.allowedDuplication = allowedDuplication; + this.keepEmpty = keepEmpty; + this.consecutive = consecutive; } @Override diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalEval.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalEval.java index 5e3f117610..49133dbfe1 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalEval.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalEval.java @@ -31,16 +31,20 @@ * distance/speed). */ @ToString -@EqualsAndHashCode(callSuper = false) -@RequiredArgsConstructor +@EqualsAndHashCode(callSuper = true) public class LogicalEval extends LogicalPlan { - private final LogicalPlan child; + @Getter private final List> expressions; - @Override - public List getChild() { - return Collections.singletonList(child); + /** + * Constructor of LogicalEval. + */ + public LogicalEval( + LogicalPlan child, + List> expressions) { + super(Collections.singletonList(child)); + this.expressions = expressions; } @Override diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalFilter.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalFilter.java index 306104ca6f..2978856512 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalFilter.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalFilter.java @@ -17,6 +17,7 @@ import com.amazon.opendistroforelasticsearch.sql.expression.Expression; import java.util.Arrays; +import java.util.Collections; import java.util.List; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -27,16 +28,18 @@ * Logical Filter represent the filter relation. */ @ToString -@EqualsAndHashCode -@RequiredArgsConstructor +@EqualsAndHashCode(callSuper = true) public class LogicalFilter extends LogicalPlan { - private final LogicalPlan child; + @Getter private final Expression condition; - @Override - public List getChild() { - return Arrays.asList(child); + /** + * Constructor of LogicalFilter. + */ + public LogicalFilter(LogicalPlan child, Expression condition) { + super(Collections.singletonList(child)); + this.condition = condition; } @Override diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalHead.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalHead.java index bb66f688f6..51c614bbb5 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalHead.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalHead.java @@ -17,6 +17,7 @@ import com.amazon.opendistroforelasticsearch.sql.expression.Expression; import java.util.Arrays; +import java.util.Collections; import java.util.List; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -26,20 +27,26 @@ @Getter @ToString -@EqualsAndHashCode(callSuper = false) -@RequiredArgsConstructor +@EqualsAndHashCode(callSuper = true) public class LogicalHead extends LogicalPlan { - private final LogicalPlan child; private final Boolean keeplast; private final Expression whileExpr; private final Integer number; - @Override - public List getChild() { - return Arrays.asList(child); + /** + * Constructor of LogicalHead. + */ + public LogicalHead( + LogicalPlan child, Boolean keeplast, + Expression whileExpr, Integer number) { + super(Collections.singletonList(child)); + this.keeplast = keeplast; + this.whileExpr = whileExpr; + this.number = number; } + @Override public R accept(LogicalPlanNodeVisitor visitor, C context) { return visitor.visitHead(this, context); diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalIndexScan.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalIndexScan.java new file mode 100644 index 0000000000..222685e4c7 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalIndexScan.java @@ -0,0 +1,98 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.NamedExpression; +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; + +/** + * Logical Index Scan Operation which could include Filter conditions ans Project Lists. + */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +public class LogicalIndexScan extends LogicalPlan { + + /** + * Relation Name. + */ + private final String relationName; + + /** + * Filter Condition. + */ + private Expression filter; + + /** + * Select List. + */ + private List projectList; + + /** + * Construct the {@link LogicalIndexScan} with relationName and filter condition. + */ + public LogicalIndexScan(String relationName, Expression filter) { + super(ImmutableList.of()); + this.relationName = relationName; + this.filter = filter; + } + + /** + * Construct the {@link LogicalIndexScan} with relationName and project list. + */ + public LogicalIndexScan(String relationName, List projectList) { + super(ImmutableList.of()); + this.relationName = relationName; + this.projectList = projectList; + } + + /** + * Construct the {@link LogicalIndexScan} with relationName, filter condition and project list. + */ + public LogicalIndexScan(String relationName, Expression filter, + List projectList) { + super(ImmutableList.of()); + this.relationName = relationName; + this.filter = filter; + this.projectList = projectList; + } + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitIndexScan(this, context); + } + + /** + * Has Projects. + */ + public boolean hasProjects() { + return projectList != null && !projectList.isEmpty(); + } + + /** + * Has Filter. + */ + public boolean hasFilter() { + return filter != null; + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalIndexScanAggregation.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalIndexScanAggregation.java new file mode 100644 index 0000000000..a701b5cb7f --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalIndexScanAggregation.java @@ -0,0 +1,71 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.NamedExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.NamedAggregator; +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; + +/** + * Logical Index Scan Aggregation Operation. + */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +public class LogicalIndexScanAggregation extends LogicalPlan { + + private final String relationName; + + private Expression filter; + + private List aggregatorList; + + private List groupByList; + + /** + * Construct {@link LogicalIndexScanAggregation} with Filter and Aggregation. + */ + public LogicalIndexScanAggregation(String relationName, Expression filter, + List aggregatorList, + List groupByList) { + super(ImmutableList.of()); + this.filter = filter; + this.relationName = relationName; + this.aggregatorList = aggregatorList; + this.groupByList = groupByList; + } + + /** + * Construct {@link LogicalIndexScanAggregation} with Aggregation without Filter. + */ + public LogicalIndexScanAggregation(String relationName, + List aggregatorList, + List groupByList) { + this(relationName, null, aggregatorList, groupByList); + } + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitIndexScanAggregation(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlan.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlan.java index 035598d568..0ba3cc19a4 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlan.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlan.java @@ -16,11 +16,22 @@ package com.amazon.opendistroforelasticsearch.sql.planner.logical; import com.amazon.opendistroforelasticsearch.sql.planner.PlanNode; +import java.util.Collections; +import java.util.List; +import lombok.EqualsAndHashCode; /** * The abstract base class for all the Logical Plan node. */ +@EqualsAndHashCode(callSuper = false) public abstract class LogicalPlan implements PlanNode { + + private List childPlans; + + public LogicalPlan(List childPlans) { + this.childPlans = childPlans; + } + /** * Accept the {@link LogicalPlanNodeVisitor}. * @@ -31,4 +42,15 @@ public abstract class LogicalPlan implements PlanNode { * @return returned object. */ public abstract R accept(LogicalPlanNodeVisitor visitor, C context); + + public LogicalPlan replaceChildPlans(List childPlans) { + this.childPlans = childPlans; + return this; + } + + + @Override + public List getChild() { + return childPlans; + } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanDSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanDSL.java index a19ce69112..68cb2c8935 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanDSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanDSL.java @@ -100,6 +100,22 @@ public static LogicalPlan rareTopN(LogicalPlan input, CommandType commandType, i return new LogicalRareTopN(input, commandType, noOfResults, Arrays.asList(fields), groupByList); } + public static LogicalPlan indexScan(String tableName, Expression filter) { + return new LogicalIndexScan(tableName, filter); + } + + public static LogicalPlan indexScanAgg(String tableName, List aggregators, + List groupByList) { + return new LogicalIndexScanAggregation(tableName, aggregators, groupByList); + } + + public static LogicalPlan indexScanAgg(String tableName, + Expression filter, + List aggregators, + List groupByList) { + return new LogicalIndexScanAggregation(tableName, filter, aggregators, groupByList); + } + @SafeVarargs public LogicalPlan values(List... values) { return new LogicalValues(Arrays.asList(values)); diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitor.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitor.java index 601b466909..34c08e3bb6 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitor.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitor.java @@ -75,4 +75,11 @@ public R visitRareTopN(LogicalRareTopN plan, C context) { return visitNode(plan, context); } + public R visitIndexScan(LogicalIndexScan plan, C context) { + return visitNode(plan, context); + } + + public R visitIndexScanAggregation(LogicalIndexScanAggregation plan, C context) { + return visitNode(plan, context); + } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalProject.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalProject.java index a68b176d60..680a264b84 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalProject.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalProject.java @@ -16,27 +16,30 @@ package com.amazon.opendistroforelasticsearch.sql.planner.logical; import com.amazon.opendistroforelasticsearch.sql.expression.NamedExpression; -import java.util.Arrays; +import java.util.Collections; import java.util.List; import lombok.EqualsAndHashCode; import lombok.Getter; -import lombok.RequiredArgsConstructor; import lombok.ToString; /** * Project field specified by the {@link LogicalProject#projectList}. */ @ToString -@EqualsAndHashCode -@RequiredArgsConstructor +@EqualsAndHashCode(callSuper = true) public class LogicalProject extends LogicalPlan { - private final LogicalPlan child; + @Getter private final List projectList; - @Override - public List getChild() { - return Arrays.asList(child); + /** + * Constructor of LogicalProject. + */ + public LogicalProject( + LogicalPlan child, + List projectList) { + super(Collections.singletonList(child)); + this.projectList = projectList; } @Override diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRareTopN.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRareTopN.java index 89c75d774a..d60818f26a 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRareTopN.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRareTopN.java @@ -30,19 +30,27 @@ */ @Getter @ToString -@EqualsAndHashCode(callSuper = false) -@RequiredArgsConstructor +@EqualsAndHashCode(callSuper = true) public class LogicalRareTopN extends LogicalPlan { - private final LogicalPlan child; private final CommandType commandType; private final Integer noOfResults; private final List fieldList; private final List groupByList; - @Override - public List getChild() { - return Collections.singletonList(child); + /** + * Constructor of LogicalRareTopN. + */ + public LogicalRareTopN( + LogicalPlan child, + CommandType commandType, Integer noOfResults, + List fieldList, + List groupByList) { + super(Collections.singletonList(child)); + this.commandType = commandType; + this.noOfResults = noOfResults; + this.fieldList = fieldList; + this.groupByList = groupByList; } @Override diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRelation.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRelation.java index a836ce043b..6a99a6d74b 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRelation.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRelation.java @@ -26,15 +26,17 @@ * Logical Relation represent the data source. */ @ToString -@EqualsAndHashCode -@RequiredArgsConstructor +@EqualsAndHashCode(callSuper = true) public class LogicalRelation extends LogicalPlan { @Getter private final String relationName; - @Override - public List getChild() { - return ImmutableList.of(); + /** + * Constructor of LogicalRelation. + */ + public LogicalRelation(String relationName) { + super(ImmutableList.of()); + this.relationName = relationName; } @Override diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRemove.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRemove.java index 11d280cd9b..75dc143a56 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRemove.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRemove.java @@ -17,6 +17,7 @@ import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Set; import lombok.EqualsAndHashCode; @@ -28,16 +29,20 @@ * Remove field specified by the {@link LogicalRemove#removeList}. */ @ToString -@EqualsAndHashCode -@RequiredArgsConstructor +@EqualsAndHashCode(callSuper = true) public class LogicalRemove extends LogicalPlan { - private final LogicalPlan child; + @Getter private final Set removeList; - @Override - public List getChild() { - return Arrays.asList(child); + /** + * Constructor of LogicalRemove. + */ + public LogicalRemove( + LogicalPlan child, + Set removeList) { + super(Collections.singletonList(child)); + this.removeList = removeList; } @Override diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRename.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRename.java index 812ac329e1..7762e4b0ab 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRename.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRename.java @@ -29,16 +29,20 @@ * renameList is list of mapping of source and target. */ @ToString -@EqualsAndHashCode -@RequiredArgsConstructor +@EqualsAndHashCode(callSuper = true) public class LogicalRename extends LogicalPlan { - private final LogicalPlan child; + @Getter private final Map renameMap; - @Override - public List getChild() { - return Collections.singletonList(child); + /** + * Constructor of LogicalRename. + */ + public LogicalRename( + LogicalPlan child, + Map renameMap) { + super(Collections.singletonList(child)); + this.renameMap = renameMap; } @Override diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalSort.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalSort.java index 4cebc975d8..1a4efcb95d 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalSort.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalSort.java @@ -18,6 +18,7 @@ import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort.SortOption; import com.amazon.opendistroforelasticsearch.sql.expression.Expression; import java.util.Arrays; +import java.util.Collections; import java.util.List; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -30,16 +31,21 @@ */ @Getter @ToString -@EqualsAndHashCode(callSuper = false) -@RequiredArgsConstructor +@EqualsAndHashCode(callSuper = true) public class LogicalSort extends LogicalPlan { - private final LogicalPlan child; + private final Integer count; private final List> sortList; - @Override - public List getChild() { - return Arrays.asList(child); + /** + * Constructor of LogicalSort. + */ + public LogicalSort( + LogicalPlan child, Integer count, + List> sortList) { + super(Collections.singletonList(child)); + this.count = count; + this.sortList = sortList; } @Override diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalValues.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalValues.java index 8ad790da7d..844c8626fa 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalValues.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalValues.java @@ -41,20 +41,23 @@ */ @ToString @Getter -@EqualsAndHashCode(callSuper = false) -@RequiredArgsConstructor +@EqualsAndHashCode(callSuper = true) public class LogicalValues extends LogicalPlan { private final List> values; - @Override - public R accept(LogicalPlanNodeVisitor visitor, C context) { - return visitor.visitValues(this, context); + /** + * Constructor of LogicalValues. + */ + public LogicalValues( + List> values) { + super(ImmutableList.of()); + this.values = values; } @Override - public List getChild() { - return ImmutableList.of(); + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitValues(this, context); } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/LogicalPlanOptimizer.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/LogicalPlanOptimizer.java new file mode 100644 index 0000000000..63e06b2461 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/LogicalPlanOptimizer.java @@ -0,0 +1,86 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.optimizer; + +import static com.facebook.presto.matching.DefaultMatcher.DEFAULT_MATCHER; + +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.optimizer.rule.MergeAggAndIndexScan; +import com.amazon.opendistroforelasticsearch.sql.planner.optimizer.rule.MergeAggAndRelation; +import com.amazon.opendistroforelasticsearch.sql.planner.optimizer.rule.MergeFilterAndFilter; +import com.amazon.opendistroforelasticsearch.sql.planner.optimizer.rule.MergeFilterAndRelation; +import com.facebook.presto.matching.Match; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +/** + * {@link LogicalPlan} Optimizer. + * The Optimizer will run in the TopDown manner. + * 1> Optimize the current node with all the rules. + * 2> Optimize the all the child nodes with all the rules. + * 3) In case the child node could change, Optimize the current node again. + */ +public class LogicalPlanOptimizer { + + private final List> rules; + + private LogicalPlanOptimizer( + List> rules) { + this.rules = rules; + } + + /** + * Create {@link LogicalPlanOptimizer} with pre-defined rules. + */ + public static LogicalPlanOptimizer create(DSL dsl) { + return new LogicalPlanOptimizer(Arrays.asList( + new MergeFilterAndRelation(), + new MergeAggAndIndexScan(), + new MergeAggAndRelation(), + new MergeFilterAndFilter(dsl))); + } + + /** + * Optimize {@link LogicalPlan}. + */ + public LogicalPlan optimize(LogicalPlan plan) { + LogicalPlan optimized = internalOptimize(plan); + optimized.replaceChildPlans( + optimized.getChild().stream().map(this::optimize).collect( + Collectors.toList())); + return internalOptimize(plan); + } + + private LogicalPlan internalOptimize(LogicalPlan plan) { + LogicalPlan node = plan; + boolean done = false; + while (!done) { + done = true; + for (Rule rule : rules) { + Match match = DEFAULT_MATCHER.match(rule.pattern(), node); + if (match.isPresent()) { + node = rule.apply(match.value(), match.captures()); + done = false; + } + } + } + return node; + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/Rule.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/Rule.java new file mode 100644 index 0000000000..2a836e29db --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/Rule.java @@ -0,0 +1,42 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.optimizer; + +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; + +/** + * Optimization Rule. + * @param LogicalPlan. + */ +public interface Rule { + + /** + * Get the {@link Pattern}. + */ + Pattern pattern(); + + /** + * Apply the Rule to the LogicalPlan. + * @param plan LogicalPlan which match the Pattern. + * @param captures A list of LogicalPlan which are captured by the Pattern. + * @return the transfromed LogicalPlan. + */ + LogicalPlan apply(T plan, Captures captures); +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/pattern/Patterns.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/pattern/Patterns.java new file mode 100644 index 0000000000..6310d49e41 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/pattern/Patterns.java @@ -0,0 +1,39 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.optimizer.pattern; + +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.facebook.presto.matching.Property; +import java.util.Optional; +import lombok.experimental.UtilityClass; + +/** + * Pattern helper class. + */ +@UtilityClass +public class Patterns { + + /** + * LogicalPlan source {@link Property}. + */ + public static Property source() { + return Property.optionalProperty("source", plan -> plan.getChild().size() == 1 + ? Optional.of(plan.getChild().get(0)) + : Optional.empty()); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/rule/MergeAggAndIndexScan.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/rule/MergeAggAndIndexScan.java new file mode 100644 index 0000000000..717eaf0553 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/rule/MergeAggAndIndexScan.java @@ -0,0 +1,67 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.optimizer.rule; + +import static com.amazon.opendistroforelasticsearch.sql.planner.optimizer.pattern.Patterns.source; +import static com.facebook.presto.matching.Pattern.typeOf; + +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalAggregation; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalIndexScan; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalIndexScanAggregation; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.optimizer.Rule; +import com.facebook.presto.matching.Capture; +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import lombok.Getter; +import lombok.experimental.Accessors; + +/** + * Merge Aggregation -- Relation to IndexScanAggregation. + */ +public class MergeAggAndIndexScan implements Rule { + + private final Capture capture; + + @Accessors(fluent = true) + @Getter + private final Pattern pattern; + + /** + * Constructor of MergeAggAndIndexScan. + */ + public MergeAggAndIndexScan() { + this.capture = Capture.newCapture(); + this.pattern = typeOf(LogicalAggregation.class) + .with(source().matching(typeOf(LogicalIndexScan.class) + .capturedAs(capture) + .matching(indexScan -> !indexScan.hasProjects()))); + } + + @Override + public LogicalPlan apply(LogicalAggregation aggregation, + Captures captures) { + LogicalIndexScan indexScan = captures.get(capture); + return new LogicalIndexScanAggregation( + indexScan.getRelationName(), + indexScan.getFilter(), + aggregation.getAggregatorList(), + aggregation.getGroupByList() + ); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/rule/MergeAggAndRelation.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/rule/MergeAggAndRelation.java new file mode 100644 index 0000000000..0aa871f680 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/rule/MergeAggAndRelation.java @@ -0,0 +1,64 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.optimizer.rule; + +import static com.amazon.opendistroforelasticsearch.sql.planner.optimizer.pattern.Patterns.source; +import static com.facebook.presto.matching.Pattern.typeOf; + +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalAggregation; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalIndexScanAggregation; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRelation; +import com.amazon.opendistroforelasticsearch.sql.planner.optimizer.Rule; +import com.facebook.presto.matching.Capture; +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import lombok.Getter; +import lombok.experimental.Accessors; + +/** + * Merge Aggregation -- Relation to IndexScanAggregation. + */ +public class MergeAggAndRelation implements Rule { + + private final Capture relationCapture; + + @Accessors(fluent = true) + @Getter + private final Pattern pattern; + + /** + * Constructor of MergeAggAndRelation. + */ + public MergeAggAndRelation() { + this.relationCapture = Capture.newCapture(); + this.pattern = typeOf(LogicalAggregation.class) + .with(source().matching(typeOf(LogicalRelation.class).capturedAs(relationCapture))); + } + + @Override + public LogicalPlan apply(LogicalAggregation aggregation, + Captures captures) { + LogicalRelation relation = captures.get(relationCapture); + return new LogicalIndexScanAggregation( + relation.getRelationName(), + aggregation.getAggregatorList(), + aggregation.getGroupByList() + ); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/rule/MergeFilterAndFilter.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/rule/MergeFilterAndFilter.java new file mode 100644 index 0000000000..d5c6a9840f --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/rule/MergeFilterAndFilter.java @@ -0,0 +1,65 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.optimizer.rule; + +import static com.amazon.opendistroforelasticsearch.sql.planner.optimizer.pattern.Patterns.source; +import static com.facebook.presto.matching.Pattern.typeOf; + +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalFilter; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.optimizer.Rule; +import com.facebook.presto.matching.Capture; +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import lombok.Getter; +import lombok.experimental.Accessors; + +/** + * Merge Filter --> Filter to the single Filter condition. + */ +public class MergeFilterAndFilter implements Rule { + + private final Capture capture; + + private final DSL dsl; + + @Accessors(fluent = true) + @Getter + private final Pattern pattern; + + /** + * Constructor of MergeFilterAndFilter. + */ + public MergeFilterAndFilter(DSL dsl) { + this.dsl = dsl; + this.capture = Capture.newCapture(); + this.pattern = typeOf(LogicalFilter.class) + .with(source().matching(typeOf(LogicalFilter.class).capturedAs(capture))); + } + + @Override + public LogicalPlan apply(LogicalFilter filter, + Captures captures) { + LogicalFilter childFilter = captures.get(capture); + return new LogicalFilter( + childFilter.getChild().get(0), + dsl.and(filter.getCondition(), childFilter.getCondition()) + ); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/rule/MergeFilterAndRelation.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/rule/MergeFilterAndRelation.java new file mode 100644 index 0000000000..d0d3a3da3c --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/rule/MergeFilterAndRelation.java @@ -0,0 +1,63 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.optimizer.rule; + +import static com.amazon.opendistroforelasticsearch.sql.planner.optimizer.pattern.Patterns.source; +import static com.facebook.presto.matching.Pattern.typeOf; + +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalFilter; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalIndexScan; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRelation; +import com.amazon.opendistroforelasticsearch.sql.planner.optimizer.Rule; +import com.facebook.presto.matching.Capture; +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; + +/** + * Merge Filter -- Relation to LogicalIndexScan. + */ +public class MergeFilterAndRelation implements Rule { + + private final Capture relationCapture; + private final Pattern pattern; + + /** + * Constructor of MergeFilterAndRelation. + */ + public MergeFilterAndRelation() { + this.relationCapture = Capture.newCapture(); + this.pattern = typeOf(LogicalFilter.class) + .with(source().matching(typeOf(LogicalRelation.class).capturedAs(relationCapture))); + } + + @Override + public Pattern pattern() { + return pattern; + } + + @Override + public LogicalPlan apply(LogicalFilter filter, + Captures captures) { + LogicalRelation relation = captures.get(relationCapture); + return new LogicalIndexScan( + relation.getRelationName(), + filter.getCondition() + ); + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementorTest.java index 2db202c8df..f43940d29f 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementorTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementorTest.java @@ -45,6 +45,8 @@ import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.AvgAggregator; import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.NamedAggregator; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalIndexScan; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalIndexScanAggregation; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRelation; @@ -57,9 +59,22 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +@ExtendWith(MockitoExtension.class) class DefaultImplementorTest { + @Mock + private Expression filter; + + @Mock + private NamedAggregator aggregator; + + @Mock + private NamedExpression groupBy; + private final DefaultImplementor implementor = new DefaultImplementor<>(); @Test @@ -152,4 +167,17 @@ public void visitRelationShouldThrowException() { () -> new LogicalRelation("test").accept(implementor, null)); } + @Test + public void visitIndexScanShouldThrowException() { + assertThrows(UnsupportedOperationException.class, + () -> new LogicalIndexScan("test", filter).accept(implementor, null)); + } + + @Test + public void visitIndexScanAggShouldThrowException() { + assertThrows(UnsupportedOperationException.class, + () -> new LogicalIndexScanAggregation("test", Arrays.asList(aggregator), + Arrays.asList(groupBy)).accept(implementor, + null)); + } } \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/PlannerTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/PlannerTest.java index be783c7d42..4568001a90 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/PlannerTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/PlannerTest.java @@ -19,7 +19,9 @@ import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; import static java.util.Collections.emptyList; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.AdditionalAnswers.returnsFirstArg; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.when; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; @@ -31,6 +33,7 @@ import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanNodeVisitor; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRelation; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRename; +import com.amazon.opendistroforelasticsearch.sql.planner.optimizer.LogicalPlanOptimizer; import com.amazon.opendistroforelasticsearch.sql.planner.physical.AggregationOperator; import com.amazon.opendistroforelasticsearch.sql.planner.physical.FilterOperator; import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; @@ -57,6 +60,9 @@ public class PlannerTest extends PhysicalPlanTestBase { @Mock private StorageEngine storageEngine; + @Mock + private LogicalPlanOptimizer optimizer; + @BeforeEach public void setUp() { when(storageEngine.getTable(any())).thenReturn(new MockTable()); @@ -64,6 +70,7 @@ public void setUp() { @Test public void planner_test() { + doAnswer(returnsFirstArg()).when(optimizer).optimize(any()); assertPhysicalPlan( PhysicalPlanDSL.rename( PhysicalPlanDSL.agg( @@ -116,7 +123,7 @@ protected void assertPhysicalPlan(PhysicalPlan expected, LogicalPlan logicalPlan } protected PhysicalPlan analyze(LogicalPlan logicalPlan) { - return new Planner(storageEngine).plan(logicalPlan); + return new Planner(storageEngine, optimizer).plan(logicalPlan); } protected class MockTable extends LogicalPlanNodeVisitor implements Table { diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalIndexScanAggregationTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalIndexScanAggregationTest.java new file mode 100644 index 0000000000..d65d3c4d34 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalIndexScanAggregationTest.java @@ -0,0 +1,46 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.amazon.opendistroforelasticsearch.sql.expression.NamedExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.NamedAggregator; +import java.util.Arrays; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class LogicalIndexScanAggregationTest { + + @Mock + private NamedAggregator aggregator; + + @Mock + private NamedExpression groupBy; + + @Test + public void visitor_return_null() { + LogicalPlan indexScan = new LogicalIndexScanAggregation("index", Arrays.asList(aggregator), + Arrays.asList(groupBy)); + assertNull(indexScan.accept(new LogicalPlanNodeVisitor() { + }, null)); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalIndexScanTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalIndexScanTest.java new file mode 100644 index 0000000000..ad1a6d92b2 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalIndexScanTest.java @@ -0,0 +1,68 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.NamedExpression; +import java.util.Collections; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class LogicalIndexScanTest { + + @Mock + private Expression filter; + + @Mock + private NamedExpression project; + + @Test + public void has_filter() { + LogicalIndexScan indexScan = new LogicalIndexScan("index", filter); + assertTrue(indexScan.hasFilter()); + + indexScan = new LogicalIndexScan("index", Collections.singletonList(project)); + assertFalse(indexScan.hasFilter()); + } + + @Test + public void has_project() { + LogicalIndexScan indexScan = new LogicalIndexScan("index", Collections.EMPTY_LIST); + assertFalse(indexScan.hasProjects()); + + indexScan = new LogicalIndexScan("index", filter); + assertFalse(indexScan.hasProjects()); + + indexScan = new LogicalIndexScan("index", filter, Collections.singletonList(project)); + assertTrue(indexScan.hasProjects()); + } + + @Test + public void visitor_return_null() { + LogicalPlan indexScan = new LogicalIndexScan("index", filter); + assertNull(indexScan.accept(new LogicalPlanNodeVisitor() { + }, null)); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/LogicalPlanOptimizerTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/LogicalPlanOptimizerTest.java new file mode 100644 index 0000000000..0d4df56c54 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/LogicalPlanOptimizerTest.java @@ -0,0 +1,181 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.optimizer; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.LONG; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.aggregation; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.filter; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.indexScan; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.indexScanAgg; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.project; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.relation; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.amazon.opendistroforelasticsearch.sql.analysis.AnalyzerTestBase; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.expression.config.ExpressionConfig; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalIndexScan; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.google.common.collect.ImmutableList; +import java.util.Collections; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +@Configuration +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {ExpressionConfig.class, AnalyzerTestBase.class}) +class LogicalPlanOptimizerTest extends AnalyzerTestBase { + /** + * SELECT integer_value as i FROM schema WHERE integer_value = 1. + */ + @Test + void project_filter_merge_with_relation() { + assertEquals( + project( + indexScan("schema", + dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(integerValue(1)))), + DSL.named("i", DSL.ref("integer_value", INTEGER)) + ), + optimize( + project( + filter( + relation("schema"), + dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(integerValue(1))) + ), + DSL.named("i", DSL.ref("integer_value", INTEGER))) + ) + ); + } + + /** + * Filter - Filter --> Filter. + */ + @Test + void filter_merge_filter() { + assertEquals( + indexScan("schema", + dsl.and(dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(integerValue(2))), + dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(integerValue(1))))), + optimize( + filter( + filter( + relation("schema"), + dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(integerValue(1))) + ), + dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(integerValue(2))) + ) + ) + ); + } + + /** + * SELECT avg(integer_value) FROM schema GROUP BY string_value. + */ + @Test + void aggregation_merge_relation() { + assertEquals( + project( + indexScanAgg("schema", ImmutableList + .of(DSL.named("AVG(integer_value)", + dsl.avg(DSL.ref("integer_value", INTEGER)))), + ImmutableList.of(DSL.named("long_value", + dsl.abs(DSL.ref("long_value", LONG))))), + DSL.named("AVG(integer_value)", DSL.ref("AVG(integer_value)", DOUBLE))), + optimize( + project( + aggregation( + relation("schema"), + ImmutableList + .of(DSL.named("AVG(integer_value)", + dsl.avg(DSL.ref("integer_value", INTEGER)))), + ImmutableList.of(DSL.named("long_value", + dsl.abs(DSL.ref("long_value", LONG))))), + DSL.named("AVG(integer_value)", DSL.ref("AVG(integer_value)", DOUBLE))) + ) + ); + } + + /** + * SELECT avg(integer_value) FROM schema WHERE integer_value = 1 GROUP BY string_value. + */ + @Test + void aggregation_merge_filter_relation() { + assertEquals( + project( + indexScanAgg("schema", + dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(integerValue(1))), + ImmutableList + .of(DSL.named("AVG(integer_value)", + dsl.avg(DSL.ref("integer_value", INTEGER)))), + ImmutableList.of(DSL.named("long_value", + dsl.abs(DSL.ref("long_value", LONG))))), + DSL.named("AVG(integer_value)", DSL.ref("AVG(integer_value)", DOUBLE))), + optimize( + project( + aggregation( + filter( + relation("schema"), + dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(integerValue(1))) + ), + ImmutableList + .of(DSL.named("AVG(integer_value)", + dsl.avg(DSL.ref("integer_value", INTEGER)))), + ImmutableList.of(DSL.named("long_value", + dsl.abs(DSL.ref("long_value", LONG))))), + DSL.named("AVG(integer_value)", DSL.ref("AVG(integer_value)", DOUBLE))) + ) + ); + } + + @Test + void aggregation_cant_merge_indexScan_with_project() { + assertEquals( + aggregation( + new LogicalIndexScan("schema", + dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(integerValue(1))), + Collections.singletonList(DSL.named("i", DSL.ref("integer_value", INTEGER)))), + ImmutableList + .of(DSL.named("AVG(integer_value)", + dsl.avg(DSL.ref("integer_value", INTEGER)))), + ImmutableList.of(DSL.named("long_value", + dsl.abs(DSL.ref("long_value", LONG))))), + optimize( + aggregation( + new LogicalIndexScan("schema", + dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(integerValue(1))), + Collections.singletonList(DSL.named("i", DSL.ref("integer_value", INTEGER)))), + ImmutableList + .of(DSL.named("AVG(integer_value)", + dsl.avg(DSL.ref("integer_value", INTEGER)))), + ImmutableList.of(DSL.named("long_value", + dsl.abs(DSL.ref("long_value", LONG)))))) + ); + } + + private LogicalPlan optimize(LogicalPlan plan) { + final LogicalPlanOptimizer optimizer = LogicalPlanOptimizer.create(dsl); + final LogicalPlan optimize = optimizer.optimize(plan); + return optimize; + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/pattern/PatternsTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/pattern/PatternsTest.java new file mode 100644 index 0000000000..ab1e501bb2 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/optimizer/pattern/PatternsTest.java @@ -0,0 +1,41 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.optimizer.pattern; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import java.util.Collections; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class PatternsTest { + + @Mock + LogicalPlan plan; + + @Test + void source_is_empty() { + when(plan.getChild()).thenReturn(Collections.emptyList()); + assertFalse(Patterns.source().getFunction().apply(plan).isPresent()); + } +} \ No newline at end of file diff --git a/docs/experiment/ppl/index.rst b/docs/experiment/ppl/index.rst index 764eab6660..f0fcc5ce5a 100644 --- a/docs/experiment/ppl/index.rst +++ b/docs/experiment/ppl/index.rst @@ -5,11 +5,11 @@ OpenDistro PPL Reference Manual Overview --------- -Piped Processing Language (PPL), powered by Open Distro for Elasticsearch, enables Open Distro for Elasticsearch users with exploration and discovery of, and finding search patterns in data stored in Elasticsearch, using a set of commands delimited by pipes (|). These are essentially read-only requests to process data and return results. +Piped Processing Language (PPL), powered by Open Distro for Elasticsearch, enables Open Distro for Elasticsearch users with exploration and discovery of, and finding search patterns in data stored in Elasticsearch, using a set of commands delimited by pipes (|). These are essentially read-only requests to process data and return results. Currently, Open Distro for Elasticsearch users can query data using either Query DSL or SQL. Query DSL is powerful and fast. However, it has a steep learning curve, and was not designed as a human interface to easily create ad hoc queries and explore user data. SQL allows users to extract and analyze data in Elasticsearch in a declarative manner. Open Distro for Elasticsearch now makes its search and query engine robust by introducing Piped Processing Language (PPL). It enables users to extract insights from Elasticsearch with a sequence of commands delimited by pipes (|). It supports a comprehensive set of commands including search, where, fields, rename, dedup, sort, eval, head, top and rare, and functions, operators and expressions. Even new users who have recently adopted Open Distro for Elasticsearch, can be productive day one, if they are familiar with the pipe (|) syntax. It enables developers, DevOps engineers, support engineers, site reliability engineers (SREs), and IT managers to effectively discover and explore log, monitoring and observability data stored in Open Distro for Elasticsearch. -We expand the capabilities of our Workbench, a comprehensive and integrated visual query tool currently supporting only SQL, to run on-demand PPL commands, and view and save results as text and JSON. We also add a new interactive standalone command line tool, the PPL CLI, to run on-demand PPL commands, and view and save results as text and JSON. +We expand the capabilities of our Workbench, a comprehensive and integrated visual query tool currently supporting only SQL, to run on-demand PPL commands, and view and save results as text and JSON. We also add a new interactive standalone command line tool, the PPL CLI, to run on-demand PPL commands, and view and save results as text and JSON. The query start with search command and then flowing a set of command delimited by pipe (|). | for example, the following query retrieve firstname and lastname from accounts if age large than 18. @@ -56,5 +56,6 @@ The query start with search command and then flowing a set of command delimited - `top command `_ +* **Functions** - + - `PPL Functions <../../user/dql/functions.rst>`_ diff --git a/docs/experiment/ppl/interfaces/endpoint.rst b/docs/experiment/ppl/interfaces/endpoint.rst index dd4df5e0ad..94a3cea63b 100644 --- a/docs/experiment/ppl/interfaces/endpoint.rst +++ b/docs/experiment/ppl/interfaces/endpoint.rst @@ -65,3 +65,37 @@ PPL query:: "size": 4 } +Explain +======= + +Description +----------- + +You can send HTTP explain request to endpoint **/_opendistro/_ppl/_explain** with your query in request body to understand the execution plan for the PPL query. The explain endpoint is useful when user want to get insight how the query is executed in the engine. + +Example +------- + +The following PPL query demonstrated that where and stats command were pushed down to Elasticsearch DSL aggregation query:: + + sh$ curl -sS -H 'Content-Type: application/json' \ + ... -X POST localhost:9200/_opendistro/_ppl/_explain \ + ... -d '{"query" : "source=accounts | where age > 10 | stats avg(age)"}' + { + "root": { + "name": "ProjectOperator", + "description": { + "fields": "[avg(age)]" + }, + "children": [ + { + "name": "ElasticsearchIndexScan", + "description": { + "request": "ElasticsearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":10,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}],\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}, searchDone=false)" + }, + "children": [] + } + ] + } + } + diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index 7655668e20..2616b16911 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -1,6 +1,6 @@ -============= -SQL Functions -============= +========= +Functions +========= .. rubric:: Table of contents @@ -11,7 +11,7 @@ SQL Functions Introduction ============ -There is support for a wide variety of SQL functions. We are intend to generate this part of documentation automatically from our type system. However, the type system is missing descriptive information for now. So only formal specifications of all SQL functions supported are listed at the moment. More details will be added in future. +There is support for a wide variety of functions shared by SQL/PPL. We are intend to generate this part of documentation automatically from our type system. However, the type system is missing descriptive information for now. So only formal specifications of all functions supported are listed at the moment. More details will be added in future. Most of the specifications can be self explained just as a regular function with data type as argument. The only notation that needs elaboration is generic type ``T`` which binds to an actual type and can be used as return type. For example, ``ABS(NUMBER T) -> T`` means function ``ABS`` accepts an numerical argument of type ``T`` which could be any sub-type of ``NUMBER`` type and returns the actual type of ``T`` as return type. The actual type binds to generic type at runtime dynamically. diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java index df737f3f3f..5b8ed19357 100644 --- a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java @@ -27,8 +27,8 @@ import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.filter.FilterQueryBuilder; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.serialization.DefaultExpressionSerializer; import com.amazon.opendistroforelasticsearch.sql.planner.DefaultImplementor; -import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalAggregation; -import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalFilter; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalIndexScan; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalIndexScanAggregation; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRelation; import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; @@ -101,59 +101,51 @@ public PhysicalPlan implement(LogicalPlan plan) { * aggregation, filter, will accumulate (push down) Elasticsearch query and aggregation DSL on * index scan. */ - return plan.accept(new DefaultImplementor() { - @Override - public PhysicalPlan visitFilter(LogicalFilter node, ElasticsearchIndexScan context) { - // For now (without optimizer), only push down filter close to relation - if (!(node.getChild().get(0) instanceof LogicalRelation)) { - return super.visitFilter(node, context); - } - - FilterQueryBuilder queryBuilder = - new FilterQueryBuilder(new DefaultExpressionSerializer()); - - QueryBuilder query = queryBuilder.build(node.getCondition()); - - context.pushDown(query); - return visitChild(node, context); - } - - @Override - public PhysicalPlan visitAggregation(LogicalAggregation node, - ElasticsearchIndexScan context) { - // Todo, aggregation in the following pattern can be push down - // aggregation -> relation - // aggregation -> filter -> relation - if ((node.getChild().get(0) instanceof LogicalRelation) - || (node.getChild().get(0) instanceof LogicalFilter && node.getChild().get(0) - .getChild().get(0) instanceof LogicalRelation)) { - AggregationQueryBuilder builder = - new AggregationQueryBuilder(new DefaultExpressionSerializer()); - - List aggregationBuilder = - builder.buildAggregationBuilder(node.getAggregatorList(), - node.getGroupByList()); - - context.pushDownAggregation(aggregationBuilder); - context.pushTypeMapping( - builder.buildTypeMapping(node.getAggregatorList(), - node.getGroupByList())); - - return visitChild(node, context); - } else { - return super.visitAggregation(node, context); - } - } - - @Override - public PhysicalPlan visitRelation(LogicalRelation node, ElasticsearchIndexScan context) { - return indexScan; - } - }, - indexScan); + return plan.accept(new ElasticsearchDefaultImplementor(indexScan), indexScan); } private ExprType transformESTypeToExprType(String esType) { return ES_TYPE_TO_EXPR_TYPE_MAPPING.getOrDefault(esType, ExprCoreType.UNKNOWN); } + + @RequiredArgsConstructor + private static class ElasticsearchDefaultImplementor + extends DefaultImplementor { + private final ElasticsearchIndexScan indexScan; + + @Override + public PhysicalPlan visitIndexScan(LogicalIndexScan node, + ElasticsearchIndexScan context) { + FilterQueryBuilder queryBuilder = new FilterQueryBuilder(new DefaultExpressionSerializer()); + QueryBuilder query = queryBuilder.build(node.getFilter()); + context.pushDown(query); + return indexScan; + } + + @Override + public PhysicalPlan visitIndexScanAggregation(LogicalIndexScanAggregation node, + ElasticsearchIndexScan context) { + if (node.getFilter() != null) { + FilterQueryBuilder queryBuilder = new FilterQueryBuilder( + new DefaultExpressionSerializer()); + QueryBuilder query = queryBuilder.build(node.getFilter()); + context.pushDown(query); + } + AggregationQueryBuilder builder = + new AggregationQueryBuilder(new DefaultExpressionSerializer()); + List aggregationBuilder = + builder.buildAggregationBuilder(node.getAggregatorList(), + node.getGroupByList()); + context.pushDownAggregation(aggregationBuilder); + context.pushTypeMapping( + builder.buildTypeMapping(node.getAggregatorList(), + node.getGroupByList())); + return indexScan; + } + + @Override + public PhysicalPlan visitRelation(LogicalRelation node, ElasticsearchIndexScan context) { + return indexScan; + } + } } diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexTest.java index f204d6fba9..1b72189171 100644 --- a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexTest.java +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexTest.java @@ -25,6 +25,8 @@ import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.aggregation; import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.eval; import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.filter; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.indexScan; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.indexScanAgg; import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.project; import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.relation; import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.remove; @@ -195,7 +197,7 @@ void implementOtherLogicalOperators() { } @Test - void shouldDiscardPhysicalFilterIfConditionPushedDown() { + void shouldImplLogicalIndexScan() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); ReferenceExpression field = ref("name", STRING); @@ -206,8 +208,8 @@ void shouldDiscardPhysicalFilterIfConditionPushedDown() { ElasticsearchIndex index = new ElasticsearchIndex(client, settings, indexName); PhysicalPlan plan = index.implement( project( - filter( - relation(indexName), + indexScan( + indexName, filterExpr ), named)); @@ -242,7 +244,7 @@ void shouldNotPushDownFilterFarFromRelation() { } @Test - void shouldPushDownAggregation() { + void shouldImplLogicalIndexScanAgg() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); ReferenceExpression field = ref("name", STRING); @@ -254,10 +256,12 @@ void shouldPushDownAggregation() { String indexName = "test"; ElasticsearchIndex index = new ElasticsearchIndex(client, settings, indexName); + + // IndexScanAgg without Filter PhysicalPlan plan = index.implement( filter( - aggregation( - relation(indexName), + indexScanAgg( + indexName, aggregators, groupByExprs ), @@ -265,11 +269,11 @@ void shouldPushDownAggregation() { assertTrue(plan.getChild().get(0) instanceof ElasticsearchIndexScan); + // IndexScanAgg with Filter plan = index.implement( - aggregation( - filter( - relation(indexName), - filterExpr), + indexScanAgg( + indexName, + filterExpr, aggregators, groupByExprs)); assertTrue(plan instanceof ElasticsearchIndexScan); diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 2cc05f042c..5a46ccbcbc 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -26,6 +26,7 @@ configurations.all { exclude group: "commons-logging", module: "commons-logging" // enforce 1.1.3, https://www.whitesourcesoftware.com/vulnerability-database/WS-2019-0379 resolutionStrategy.force 'commons-codec:commons-codec:1.13' + resolutionStrategy.force 'com.google.guava:guava:29.0-jre' } dependencies { @@ -46,7 +47,7 @@ dependencies { } testCompile group: 'com.h2database', name: 'h2', version: '1.4.200' testCompile group: 'org.xerial', name: 'sqlite-jdbc', version: '3.28.0' - //testCompile group: 'org.apache.derby', name: 'derby', version: '10.15.1.3' + testCompile group: 'com.google.code.gson', name: 'gson', version: '2.8.6' } dependencyLicenses.enabled = false diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/ExplainIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/ExplainIT.java index 70729fe23a..4970ee1a63 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/ExplainIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/ExplainIT.java @@ -16,6 +16,8 @@ package com.amazon.opendistroforelasticsearch.sql.ppl; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.assertJsonEquals; + import com.google.common.io.Resources; import java.io.IOException; import java.net.URI; @@ -32,9 +34,8 @@ public void init() throws IOException { @Test public void testExplain() throws Exception { - URI uri = Resources.getResource("expectedOutput/ppl/explain_output.json").toURI(); - String expected = new String(Files.readAllBytes(Paths.get(uri))); - assertEquals( + String expected = loadFromFile("expectedOutput/ppl/explain_output.json"); + assertJsonEquals( expected, explainQueryToString( "source=elasticsearch-sql_test_index_account" @@ -48,4 +49,35 @@ public void testExplain() throws Exception { ); } + @Test + public void testFilterPushDownExplain() throws Exception { + String expected = loadFromFile("expectedOutput/ppl/explain_filter_push.json"); + + assertJsonEquals( + expected, + explainQueryToString( + "source=elasticsearch-sql_test_index_account" + + "| where age > 30 " + + "| where age < 40 " + + "| where balance > 10000 ") + ); + } + + @Test + public void testFilterAndAggPushDownExplain() throws Exception { + String expected = loadFromFile("expectedOutput/ppl/explain_filter_agg_push.json"); + + assertJsonEquals( + expected, + explainQueryToString( + "source=elasticsearch-sql_test_index_account" + + "| where age > 30 " + + "| stats avg(age) AS avg_age by state, city") + ); + } + + String loadFromFile(String filename) throws Exception { + URI uri = Resources.getResource(filename).toURI(); + return new String(Files.readAllBytes(Paths.get(uri))); + } } diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/MatcherUtils.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/MatcherUtils.java index ff3a2a6327..d1d5cf47b6 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/MatcherUtils.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/MatcherUtils.java @@ -29,6 +29,7 @@ import static org.junit.Assert.assertThat; import com.google.common.base.Strings; +import com.google.gson.JsonParser; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -353,4 +354,16 @@ public static Matcher equalToIgnoreCaseAndWhiteSpace(String expectedStri return new IsEqualIgnoreCaseAndWhiteSpace(expectedString); } } + + /** + * Compare two JSON string are equals. + * @param expected expected JSON string. + * @param actual actual JSON string. + */ + public static void assertJsonEquals(String expected, String actual) { + assertEquals( + JsonParser.parseString(expected), + JsonParser.parseString(actual) + ); + } } diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.json new file mode 100644 index 0000000000..e7e27d0f22 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.json @@ -0,0 +1,17 @@ +{ + "root": { + "name": "ProjectOperator", + "description": { + "fields": "[avg_age, city, state]" + }, + "children": [ + { + "name": "ElasticsearchIndexScan", + "description": { + "request": "ElasticsearchQueryRequest(indexName\u003delasticsearch-sql_test_index_account, sourceBuilder\u003d{\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, searchDone\u003dfalse)" + }, + "children": [] + } + ] + } +} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push.json new file mode 100644 index 0000000000..8a75c8b806 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push.json @@ -0,0 +1,17 @@ +{ + "root": { + "name": "ProjectOperator", + "description": { + "fields": "[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]" + }, + "children": [ + { + "name": "ElasticsearchIndexScan", + "description": { + "request": "ElasticsearchQueryRequest(indexName\u003delasticsearch-sql_test_index_account, sourceBuilder\u003d{\"from\":0,\"size\":200,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"bool\":{\"filter\":[{\"range\":{\"balance\":{\"from\":10000,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"age\":{\"from\":null,\"to\":40,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, searchDone\u003dfalse)" + }, + "children": [] + } + ] + } +} \ No newline at end of file diff --git a/plugin/build.gradle b/plugin/build.gradle index 4ce2e52e70..ed47226d45 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -33,6 +33,7 @@ configurations.all { exclude group: "commons-logging", module: "commons-logging" // enforce 1.1.3, https://www.whitesourcesoftware.com/vulnerability-database/WS-2019-0379 resolutionStrategy.force 'commons-codec:commons-codec:1.13' + resolutionStrategy.force 'com.google.guava:guava:29.0-jre' } dependencies { diff --git a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/PPLService.java b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/PPLService.java index 8451a053b2..18763c18ba 100644 --- a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/PPLService.java +++ b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/PPLService.java @@ -23,8 +23,11 @@ import com.amazon.opendistroforelasticsearch.sql.common.response.ResponseListener; import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine; import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.ExplainResponse; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository; import com.amazon.opendistroforelasticsearch.sql.planner.Planner; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.optimizer.LogicalPlanOptimizer; import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; import com.amazon.opendistroforelasticsearch.sql.ppl.antlr.PPLSyntaxParser; import com.amazon.opendistroforelasticsearch.sql.ppl.domain.PPLQueryRequest; @@ -45,9 +48,12 @@ public class PPLService { private final ExecutionEngine executionEngine; + private final BuiltinFunctionRepository repository; + /** * Execute the {@link PPLQueryRequest}, using {@link ResponseListener} to get response. - * @param request {@link PPLQueryRequest} + * + * @param request {@link PPLQueryRequest} * @param listener {@link ResponseListener} */ public void execute(PPLQueryRequest request, ResponseListener listener) { @@ -84,7 +90,8 @@ private PhysicalPlan plan(PPLQueryRequest request) { new AnalysisContext()); // 3.Generate optimal physical plan from logical plan - return new Planner(storageEngine).plan(logicalPlan); + return new Planner(storageEngine, LogicalPlanOptimizer.create(new DSL(repository))) + .plan(logicalPlan); } } diff --git a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/config/PPLServiceConfig.java b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/config/PPLServiceConfig.java index f1a6e3aae0..500fed5237 100644 --- a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/config/PPLServiceConfig.java +++ b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/config/PPLServiceConfig.java @@ -48,7 +48,8 @@ public Analyzer analyzer() { @Bean public PPLService pplService() { - return new PPLService(new PPLSyntaxParser(), analyzer(), storageEngine, executionEngine); + return new PPLService(new PPLSyntaxParser(), analyzer(), storageEngine, executionEngine, + functionRepository); } } diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java index aaaf80f9b7..0e9ed90177 100644 --- a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java @@ -23,18 +23,23 @@ import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine; import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.ExplainResponse; import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.QueryResponse; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository; import com.amazon.opendistroforelasticsearch.sql.planner.Planner; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.optimizer.LogicalPlanOptimizer; import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; import com.amazon.opendistroforelasticsearch.sql.sql.antlr.SQLSyntaxParser; import com.amazon.opendistroforelasticsearch.sql.sql.domain.SQLQueryRequest; import com.amazon.opendistroforelasticsearch.sql.sql.parser.AstBuilder; import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; +import lombok.RequiredArgsConstructor; import org.antlr.v4.runtime.tree.ParseTree; /** * SQL service. */ +@RequiredArgsConstructor public class SQLService { private final SQLSyntaxParser parser; @@ -45,20 +50,7 @@ public class SQLService { private final ExecutionEngine executionEngine; - /** - * Initialize SQL service. - * @param parser SQL syntax parser - * @param analyzer AST analyzer - * @param storageEngine storage engine - * @param executionEngine execution engine - */ - public SQLService(SQLSyntaxParser parser, Analyzer analyzer, - StorageEngine storageEngine, ExecutionEngine executionEngine) { - this.parser = parser; - this.analyzer = analyzer; - this.storageEngine = storageEngine; - this.executionEngine = executionEngine; - } + private final BuiltinFunctionRepository repository; /** * Parse, analyze, plan and execute the query. @@ -121,7 +113,8 @@ public LogicalPlan analyze(UnresolvedPlan ast) { * Generate optimal physical plan from logical plan. */ public PhysicalPlan plan(LogicalPlan logicalPlan) { - return new Planner(storageEngine).plan(logicalPlan); + return new Planner(storageEngine, LogicalPlanOptimizer.create(new DSL(repository))) + .plan(logicalPlan); } } diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/config/SQLServiceConfig.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/config/SQLServiceConfig.java index e7584eccc9..a901884881 100644 --- a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/config/SQLServiceConfig.java +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/config/SQLServiceConfig.java @@ -52,7 +52,8 @@ public Analyzer analyzer() { @Bean public SQLService sqlService() { - return new SQLService(new SQLSyntaxParser(), analyzer(), storageEngine, executionEngine); + return new SQLService(new SQLSyntaxParser(), analyzer(), storageEngine, executionEngine, + functionRepository); } }