Add where stats rename command (#466)

* add aggregator * update * update * add sum aggregator * update license header * update bindingtuple * address comments
opendistro-for-elasticsearch · May 18, 2020 · 30703ea · 30703ea
1 parent ce2c08b
commit 30703ea
Show file tree

Hide file tree

Showing 69 changed files with 2,566 additions and 229 deletions.
diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/Analyzer.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/Analyzer.java
@@ -16,16 +16,27 @@
 package com.amazon.opendistroforelasticsearch.sql.analysis;
 
 import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor;
+import com.amazon.opendistroforelasticsearch.sql.ast.expression.Field;
+import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression;
+import com.amazon.opendistroforelasticsearch.sql.ast.tree.Aggregation;
 import com.amazon.opendistroforelasticsearch.sql.ast.tree.Filter;
 import com.amazon.opendistroforelasticsearch.sql.ast.tree.Relation;
+import com.amazon.opendistroforelasticsearch.sql.ast.tree.Rename;
 import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan;
+import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException;
 import com.amazon.opendistroforelasticsearch.sql.expression.DSL;
 import com.amazon.opendistroforelasticsearch.sql.expression.Expression;
+import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression;
+import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator;
+import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalAggregation;
 import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalFilter;
 import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan;
 import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRelation;
+import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRename;
 import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine;
 import com.amazon.opendistroforelasticsearch.sql.storage.Table;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import lombok.RequiredArgsConstructor;
 
 /**
@@ -53,6 +64,48 @@ public LogicalPlan visitRelation(Relation node, AnalysisContext context) {
     public LogicalPlan visitFilter(Filter node, AnalysisContext context) {
         LogicalPlan child = node.getChild().get(0).accept(this, context);
         Expression condition = expressionAnalyzer.analyze(node.getCondition(), context);
-        return new LogicalFilter(condition, child);
+        return new LogicalFilter(child, condition);
+    }
+
+    /**
+     * Build {@link LogicalRename}
+     */
+    @Override
+    public LogicalPlan visitRename(Rename node, AnalysisContext context) {
+        LogicalPlan child = node.getChild().get(0).accept(this, context);
+        ImmutableMap.Builder<ReferenceExpression, ReferenceExpression> renameMapBuilder = new ImmutableMap.Builder<>();
+        for (com.amazon.opendistroforelasticsearch.sql.ast.expression.Map renameMap : node.getRenameList()) {
+            Expression origin = expressionAnalyzer.analyze(renameMap.getOrigin(), context);
+            // We should define the new target field in the context instead of analyze it.
+            if (renameMap.getTarget() instanceof Field) {
+                ReferenceExpression target =
+                        new ReferenceExpression(((Field) renameMap.getTarget()).getField().toString());
+                context.peek().define(target, origin.type(context.peek()));
+                renameMapBuilder.put(DSL.ref(origin.toString()), target);
+            } else {
+                throw new SemanticCheckException(String.format("the target expected to be field, but is %s",
+                        renameMap.getTarget()));
+            }
+        }
+
+        return new LogicalRename(child, renameMapBuilder.build());
+    }
+
+    /**
+     * Build {@link LogicalAggregation}
+     */
+    @Override
+    public LogicalPlan visitAggregation(Aggregation node, AnalysisContext context) {
+        LogicalPlan child = node.getChild().get(0).accept(this, context);
+        ImmutableList.Builder<Aggregator> aggregatorBuilder = new ImmutableList.Builder<>();
+        for (UnresolvedExpression uExpr : node.getAggExprList()) {
+            aggregatorBuilder.add((Aggregator) expressionAnalyzer.analyze(uExpr, context));
+        }
+
+        ImmutableList.Builder<Expression> groupbyBuilder = new ImmutableList.Builder<>();
+        for (UnresolvedExpression uExpr : node.getGroupExprList()) {
+            groupbyBuilder.add(expressionAnalyzer.analyze(uExpr, context));
+        }
+        return new LogicalAggregation(child, aggregatorBuilder.build(), groupbyBuilder.build());
     }
 }
diff --git a/.../src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionAnalyzer.java b/.../src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionAnalyzer.java
@@ -16,23 +16,33 @@
 package com.amazon.opendistroforelasticsearch.sql.analysis;
 
 import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor;
+import com.amazon.opendistroforelasticsearch.sql.ast.expression.AggregateFunction;
 import com.amazon.opendistroforelasticsearch.sql.ast.expression.And;
 import com.amazon.opendistroforelasticsearch.sql.ast.expression.EqualTo;
+import com.amazon.opendistroforelasticsearch.sql.ast.expression.Field;
 import com.amazon.opendistroforelasticsearch.sql.ast.expression.Literal;
 import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedAttribute;
 import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils;
+import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException;
 import com.amazon.opendistroforelasticsearch.sql.expression.DSL;
 import com.amazon.opendistroforelasticsearch.sql.expression.Expression;
 import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression;
+import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator;
+import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName;
+import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository;
 import lombok.RequiredArgsConstructor;
 
+import java.util.Collections;
+import java.util.Optional;
+
 /**
  * Analyze the {@link UnresolvedExpression} in the {@link AnalysisContext} to construct the {@link Expression}
  */
 @RequiredArgsConstructor
 public class ExpressionAnalyzer extends AbstractNodeVisitor<Expression, AnalysisContext> {
     private final DSL dsl;
+    private final BuiltinFunctionRepository repository;
 
     public Expression analyze(UnresolvedExpression unresolved, AnalysisContext context) {
         return unresolved.accept(this, context);
@@ -66,4 +76,26 @@ public Expression visitAnd(And node, AnalysisContext context) {
 
         return dsl.and(context.peek(), left, right);
     }
+
+    @Override
+    public Expression visitAggregateFunction(AggregateFunction node, AnalysisContext context) {
+        Optional<BuiltinFunctionName> builtinFunctionName = BuiltinFunctionName.of(node.getFuncName());
+        if (builtinFunctionName.isPresent()) {
+            Expression arg = node.getField().accept(this, context);
+            return (Aggregator) repository.compile(builtinFunctionName.get().getName(),
+                    Collections.singletonList(arg),
+                    context.peek());
+        } else {
+            throw new SemanticCheckException("Unsupported aggregation function " + node.getFuncName());
+        }
+    }
+
+    @Override
+    public Expression visitField(Field node, AnalysisContext context) {
+        String attr = node.getField().toString();
+        TypeEnvironment typeEnv = context.peek();
+        ReferenceExpression ref = DSL.ref(attr);
+        typeEnv.resolve(ref);
+        return ref;
+    }
 }
diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/AbstractNodeVisitor.java
@@ -34,6 +34,7 @@
 import com.amazon.opendistroforelasticsearch.sql.ast.tree.Filter;
 import com.amazon.opendistroforelasticsearch.sql.ast.tree.Project;
 import com.amazon.opendistroforelasticsearch.sql.ast.tree.Relation;
+import com.amazon.opendistroforelasticsearch.sql.ast.tree.Rename;
 
 /**
  * AST nodes visitor
@@ -138,5 +139,9 @@ public T visitField(Field node, C context) {
     public T visitQualifiedName(QualifiedName node, C context) {
         return visitChildren(node, context);
     }
+
+    public T visitRename(Rename node, C context) {
+        return visitChildren(node, context);
+    }
 }
 
diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java
@@ -33,6 +33,7 @@
 import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedAttribute;
 import com.amazon.opendistroforelasticsearch.sql.ast.tree.Aggregation;
 import com.amazon.opendistroforelasticsearch.sql.ast.tree.Filter;
+import com.amazon.opendistroforelasticsearch.sql.ast.tree.Rename;
 import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan;
 import com.amazon.opendistroforelasticsearch.sql.ast.tree.Project;
 import com.amazon.opendistroforelasticsearch.sql.ast.tree.Relation;
@@ -65,6 +66,10 @@ public static UnresolvedPlan agg(UnresolvedPlan input, List<UnresolvedExpression
         return new Aggregation(aggList, sortList, groupList, argList).attach(input);
     }
 
+    public static UnresolvedPlan rename(UnresolvedPlan input, Map... maps) {
+        return new Rename(Arrays.asList(maps), input);
+    }
+
     public static UnresolvedExpression qualifiedName(String... parts) {
         return new QualifiedName(Arrays.asList(parts));
     }
@@ -105,7 +110,7 @@ public static UnresolvedExpression map(String origin, String target) {
         return new Map(new Field(origin), new Field(target));
     }
 
-    public static UnresolvedExpression map(UnresolvedExpression origin, UnresolvedExpression target) {
+    public static Map map(UnresolvedExpression origin, UnresolvedExpression target) {
         return new Map(origin, target);
     }
 

diff --git a/...main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/AggregateFunction.java b/...main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/AggregateFunction.java
@@ -16,20 +16,17 @@
 package com.amazon.opendistroforelasticsearch.sql.ast.expression;
 
 import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
-import lombok.ToString;
 
 /**
  * Expression node of aggregate functions
  * Params include aggregate function name (AVG, SUM, MAX etc.) and the field to aggregate
  */
 @Getter
-@ToString
 @EqualsAndHashCode(callSuper = false)
 @RequiredArgsConstructor
 public class AggregateFunction extends UnresolvedExpression {
@@ -45,11 +42,16 @@ public AggregateFunction(String funcName, UnresolvedExpression field) {
 
     @Override
     public List<UnresolvedExpression> getChild() {
-        return Arrays.asList(field);
+        return Collections.singletonList(field);
     }
 
     @Override
     public <R, C> R accept(AbstractNodeVisitor<R, C> nodeVisitor, C context) {
         return nodeVisitor.visitAggregateFunction(this, context);
     }
+
+    @Override
+    public String toString() {
+        return String.format("%s(%s)", funcName, field);
+    }
 }
diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Rename.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Rename.java
@@ -0,0 +1,58 @@
+/*
+ *   Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License").
+ *   You may not use this file except in compliance with the License.
+ *   A copy of the License is located at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file. This file is distributed
+ *   on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *   express or implied. See the License for the specific language governing
+ *   permissions and limitations under the License.
+ */
+
+package com.amazon.opendistroforelasticsearch.sql.ast.tree;
+
+import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor;
+import com.amazon.opendistroforelasticsearch.sql.ast.expression.Map;
+import com.google.common.collect.ImmutableList;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.ToString;
+
+import java.util.List;
+
+@ToString
+@EqualsAndHashCode(callSuper = false)
+@Getter
+public class Rename extends UnresolvedPlan {
+    private final List<Map> renameList;
+    private UnresolvedPlan child;
+
+    public Rename(List<Map> renameList, UnresolvedPlan child) {
+        this.renameList = renameList;
+        this.child = child;
+    }
+
+    @Override
+    public Rename attach(UnresolvedPlan child) {
+        if (null == this.child) {
+            this.child = child;
+        } else {
+            this.child.attach(child);
+        }
+        return this;
+    }
+
+    @Override
+    public List<UnresolvedPlan> getChild() {
+        return ImmutableList.of(child);
+    }
+
+    @Override
+    public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
+        return nodeVisitor.visitRename(this, context);
+    }
+}
diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTupleValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTupleValue.java
@@ -15,16 +15,24 @@
 
 package com.amazon.opendistroforelasticsearch.sql.data.model;
 
-import lombok.EqualsAndHashCode;
-import lombok.RequiredArgsConstructor;
-
+import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.BindingTuple;
+import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.LazyBindingTuple;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.stream.Collectors;
+import lombok.RequiredArgsConstructor;
 
-@EqualsAndHashCode
 @RequiredArgsConstructor
 public class ExprTupleValue implements ExprValue {
-    private final Map<String, ExprValue> valueMap;
+
+    private final LinkedHashMap<String, ExprValue> valueMap;
+
+    public static ExprTupleValue fromExprValueMap(Map<String, ExprValue> map) {
+        LinkedHashMap<String, ExprValue> linkedHashMap = new LinkedHashMap<>(map);
+        return new ExprTupleValue(linkedHashMap);
+    }
 
     @Override
     public Object value() {
@@ -43,4 +51,28 @@ public String toString() {
                 .map(entry -> String.format("%s:%s", entry.getKey(), entry.getValue()))
                 .collect(Collectors.joining(",", "{", "}"));
     }
+
+    @Override
+    public BindingTuple bindingTuples() {
+        return new LazyBindingTuple(bindingName -> valueMap.getOrDefault(bindingName, ExprMissingValue.of()));
+    }
+
+    public boolean equals(Object o) {
+        if (o == this) {
+            return true;
+        } else if (!(o instanceof ExprTupleValue)) {
+            return false;
+        } else {
+            ExprTupleValue other = (ExprTupleValue) o;
+            Iterator<Entry<String, ExprValue>> thisIterator = this.valueMap.entrySet().iterator();
+            Iterator<Entry<String, ExprValue>> otherIterator = other.valueMap.entrySet().iterator();
+            while (thisIterator.hasNext() && otherIterator.hasNext()) {
+                if (!thisIterator.next().equals(otherIterator.next())) {
+                    return false;
+                }
+            }
+            return !(thisIterator.hasNext() || otherIterator.hasNext());
+        }
+    }
+
 }
diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValue.java
@@ -15,6 +15,8 @@
 
 package com.amazon.opendistroforelasticsearch.sql.data.model;
 
+import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.BindingTuple;
+
 /**
  * The definition of the Expression Value.
  */
@@ -44,4 +46,11 @@ default boolean isNull() {
     default boolean isMissing() {
         return false;
     }
+
+    /**
+     * Get the {@link BindingTuple}
+     */
+    default BindingTuple bindingTuples() {
+        return BindingTuple.EMPTY;
+    }
 }
diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java
@@ -17,10 +17,10 @@
 
 import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException;
 import com.google.common.annotations.VisibleForTesting;
+import java.util.LinkedHashMap;
 import lombok.experimental.UtilityClass;
 
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -64,7 +64,7 @@ public static ExprValue stringValue(String value) {
     }
 
     public static ExprValue tupleValue(Map<String, Object> map) {
-        Map<String, ExprValue> valueMap = new HashMap<>();
+        LinkedHashMap<String, ExprValue> valueMap = new LinkedHashMap<>();
         map.forEach((k, v) -> valueMap.put(k, fromObjectValue(v)));
         return new ExprTupleValue(valueMap);
     }
@@ -84,6 +84,9 @@ public static ExprValue nullValue() {
     }
 
     public static ExprValue fromObjectValue(Object o) {
+        if ( null == o) {
+            return LITERAL_NULL;
+        }
         if (o instanceof Map) {
             return tupleValue((Map) o);
         } else if (o instanceof List) {