Add LogicalPlan optimization (#763)

* init

* build pass

* update ExplainIT

* update doc

* correct the doc path

* correct the doc

* correct index name

* change the description in functions to be generic for SQL/PPL

* update JavaDoc

* add JavaDoc

build.gradle

@@ -127,3 +127,9 @@ checkstyle {
 checkstyleMain.ignoreFailures = false
 checkstyleTest.ignoreFailures = true
 
+configurations.all {
+    exclude group: "commons-logging", module: "commons-logging"
+    // enforce 1.1.3, https://www.whitesourcesoftware.com/vulnerability-database/WS-2019-0379
+    resolutionStrategy.force 'commons-codec:commons-codec:1.13'
+    resolutionStrategy.force 'com.google.guava:guava:29.0-jre'
+}

core/build.gradle

@@ -7,13 +7,20 @@ plugins {
 repositories {
     mavenCentral()
 }
+//
+//configurations.all {
+//    resolutionStrategy.dependencySubstitution {
+//        substitute module('com.google.guava:guava:26.0-jre') with module('com.google.guava:guava:29.0-jre')
+//    }
+//}
 
 dependencies {
     // https://github.com/google/guava/wiki/CVE-2018-10237
     compile group: 'com.google.guava', name: 'guava', version: '29.0-jre'
     compile group: 'org.springframework', name: 'spring-context', version: '5.2.5.RELEASE'
     compile group: 'org.springframework', name: 'spring-beans', version: '5.2.5.RELEASE'
     compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.10'
+    compile group: 'com.facebook.presto', name: 'presto-matching', version: '0.240'
     compile project(':common')
 
     testImplementation('org.junit.jupiter:junit-jupiter:5.6.2')

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementor.java

@@ -21,6 +21,8 @@
 import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalEval;
 import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalFilter;
 import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalHead;
+import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalIndexScan;
+import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalIndexScanAggregation;
 import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan;
 import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanNodeVisitor;
 import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalProject;
@@ -132,6 +134,18 @@ public PhysicalPlan visitRelation(LogicalRelation node, C context) {
         + "implementing and optimizing logical plan with relation involved");
   }
 
+  @Override
+  public PhysicalPlan visitIndexScan(LogicalIndexScan plan, C context) {
+    throw new UnsupportedOperationException("Storage engine is responsible for "
+        + "implementing and optimizing logical plan with relation involved");
+  }
+
+  @Override
+  public PhysicalPlan visitIndexScanAggregation(LogicalIndexScanAggregation plan, C context) {
+    throw new UnsupportedOperationException("Storage engine is responsible for "
+        + "implementing and optimizing logical plan with relation involved");
+  }
+
   protected PhysicalPlan visitChild(LogicalPlan node, C context) {
     // Logical operators visited here must have a single child
     return node.getChild().get(0).accept(this, context);

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/Planner.java

@@ -21,6 +21,7 @@
 import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan;
 import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanNodeVisitor;
 import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRelation;
+import com.amazon.opendistroforelasticsearch.sql.planner.optimizer.LogicalPlanOptimizer;
 import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan;
 import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine;
 import com.amazon.opendistroforelasticsearch.sql.storage.Table;
@@ -38,6 +39,8 @@ public class Planner {
    */
   private final StorageEngine storageEngine;
 
+  private final LogicalPlanOptimizer logicalOptimizer;
+
   /**
    * Generate optimal physical plan for logical plan. If no table involved,
    * translate logical plan to physical by default implementor.
@@ -53,7 +56,7 @@ public PhysicalPlan plan(LogicalPlan plan) {
     }
 
     Table table = storageEngine.getTable(tableName);
-    return table.implement(plan);
+    return table.implement(optimize(plan));
   }
 
   private String findTableName(LogicalPlan plan) {
@@ -75,4 +78,7 @@ public String visitRelation(LogicalRelation node, Object context) {
     }, null);
   }
 
+  private LogicalPlan optimize(LogicalPlan plan) {
+    return logicalOptimizer.optimize(plan);
+  }
 }

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalAggregation.java

@@ -28,18 +28,25 @@
  * Logical Aggregation.
  */
 @ToString
-@EqualsAndHashCode
-@RequiredArgsConstructor
+@EqualsAndHashCode(callSuper = true)
 public class LogicalAggregation extends LogicalPlan {
-  private final LogicalPlan child;
+
   @Getter
   private final List<NamedAggregator> aggregatorList;
+
   @Getter
   private final List<NamedExpression> groupByList;
 
-  @Override
-  public List<LogicalPlan> getChild() {
-    return Collections.singletonList(child);
+  /**
+   * Constructor of LogicalAggregation.
+   */
+  public LogicalAggregation(
+      LogicalPlan child,
+      List<NamedAggregator> aggregatorList,
+      List<NamedExpression> groupByList) {
+    super(Collections.singletonList(child));
+    this.aggregatorList = aggregatorList;
+    this.groupByList = groupByList;
   }
 
   @Override

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalDedupe.java

@@ -28,18 +28,26 @@
  */
 @Getter
 @ToString
-@EqualsAndHashCode(callSuper = false)
-@RequiredArgsConstructor
+@EqualsAndHashCode(callSuper = true)
 public class LogicalDedupe extends LogicalPlan {
-  private final LogicalPlan child;
+
   private final List<Expression> dedupeList;
   private final Integer allowedDuplication;
   private final Boolean keepEmpty;
   private final Boolean consecutive;
 
-  @Override
-  public List<LogicalPlan> getChild() {
-    return Arrays.asList(child);
+  /**
+   * Constructor of LogicalDedupe.
+   */
+  public LogicalDedupe(
+      LogicalPlan child,
+      List<Expression> dedupeList, Integer allowedDuplication, Boolean keepEmpty,
+      Boolean consecutive) {
+    super(Arrays.asList(child));
+    this.dedupeList = dedupeList;
+    this.allowedDuplication = allowedDuplication;
+    this.keepEmpty = keepEmpty;
+    this.consecutive = consecutive;
   }
 
   @Override

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalEval.java

@@ -31,16 +31,20 @@
  * distance/speed).
  */
 @ToString
-@EqualsAndHashCode(callSuper = false)
-@RequiredArgsConstructor
+@EqualsAndHashCode(callSuper = true)
 public class LogicalEval extends LogicalPlan {
-  private final LogicalPlan child;
+
   @Getter
   private final List<Pair<ReferenceExpression, Expression>> expressions;
 
-  @Override
-  public List<LogicalPlan> getChild() {
-    return Collections.singletonList(child);
+  /**
+   * Constructor of LogicalEval.
+   */
+  public LogicalEval(
+      LogicalPlan child,
+      List<Pair<ReferenceExpression, Expression>> expressions) {
+    super(Collections.singletonList(child));
+    this.expressions = expressions;
   }
 
   @Override

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalFilter.java

@@ -17,6 +17,7 @@
 
 import com.amazon.opendistroforelasticsearch.sql.expression.Expression;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
@@ -27,16 +28,18 @@
  * Logical Filter represent the filter relation.
  */
 @ToString
-@EqualsAndHashCode
-@RequiredArgsConstructor
+@EqualsAndHashCode(callSuper = true)
 public class LogicalFilter extends LogicalPlan {
-  private final LogicalPlan child;
+
   @Getter
   private final Expression condition;
 
-  @Override
-  public List<LogicalPlan> getChild() {
-    return Arrays.asList(child);
+  /**
+   * Constructor of LogicalFilter.
+   */
+  public LogicalFilter(LogicalPlan child, Expression condition) {
+    super(Collections.singletonList(child));
+    this.condition = condition;
   }
 
   @Override

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalHead.java

@@ -17,6 +17,7 @@
 
 import com.amazon.opendistroforelasticsearch.sql.expression.Expression;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
@@ -26,20 +27,26 @@
 
 @Getter
 @ToString
-@EqualsAndHashCode(callSuper = false)
-@RequiredArgsConstructor
+@EqualsAndHashCode(callSuper = true)
 public class LogicalHead extends LogicalPlan {
 
-  private final LogicalPlan child;
   private final Boolean keeplast;
   private final Expression whileExpr;
   private final Integer number;
 
-  @Override
-  public List<LogicalPlan> getChild() {
-    return Arrays.asList(child);
+  /**
+   * Constructor of LogicalHead.
+   */
+  public LogicalHead(
+      LogicalPlan child, Boolean keeplast,
+      Expression whileExpr, Integer number) {
+    super(Collections.singletonList(child));
+    this.keeplast = keeplast;
+    this.whileExpr = whileExpr;
+    this.number = number;
   }
 
+
   @Override
   public <R, C> R accept(LogicalPlanNodeVisitor<R, C> visitor, C context) {
     return visitor.visitHead(this, context);

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalIndexScan.java

@@ -0,0 +1,98 @@
+/*
+ *
+ *    Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License").
+ *    You may not use this file except in compliance with the License.
+ *    A copy of the License is located at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    or in the "license" file accompanying this file. This file is distributed
+ *    on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *    express or implied. See the License for the specific language governing
+ *    permissions and limitations under the License.
+ *
+ */
+
+package com.amazon.opendistroforelasticsearch.sql.planner.logical;
+
+import com.amazon.opendistroforelasticsearch.sql.expression.Expression;
+import com.amazon.opendistroforelasticsearch.sql.expression.NamedExpression;
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.ToString;
+
+/**
+ * Logical Index Scan Operation which could include Filter conditions ans Project Lists.
+ */
+@Getter
+@ToString
+@EqualsAndHashCode(callSuper = false)
+public class LogicalIndexScan extends LogicalPlan {
+
+  /**
+   * Relation Name.
+   */
+  private final String relationName;
+
+  /**
+   * Filter Condition.
+   */
+  private Expression filter;
+
+  /**
+   * Select List.
+   */
+  private List<NamedExpression> projectList;
+
+  /**
+   * Construct the {@link LogicalIndexScan} with relationName and filter condition.
+   */
+  public LogicalIndexScan(String relationName, Expression filter) {
+    super(ImmutableList.of());
+    this.relationName = relationName;
+    this.filter = filter;
+  }
+
+  /**
+   * Construct the {@link LogicalIndexScan} with relationName and project list.
+   */
+  public LogicalIndexScan(String relationName, List<NamedExpression> projectList) {
+    super(ImmutableList.of());
+    this.relationName = relationName;
+    this.projectList = projectList;
+  }
+
+  /**
+   * Construct the {@link LogicalIndexScan} with relationName, filter condition and project list.
+   */
+  public LogicalIndexScan(String relationName, Expression filter,
+                          List<NamedExpression> projectList) {
+    super(ImmutableList.of());
+    this.relationName = relationName;
+    this.filter = filter;
+    this.projectList = projectList;
+  }
+
+  @Override
+  public <R, C> R accept(LogicalPlanNodeVisitor<R, C> visitor, C context) {
+    return visitor.visitIndexScan(this, context);
+  }
+
+  /**
+   * Has Projects.
+   */
+  public boolean hasProjects() {
+    return projectList != null && !projectList.isEmpty();
+  }
+
+  /**
+   * Has Filter.
+   */
+  public boolean hasFilter() {
+    return filter != null;
+  }
+}