Merge pull request #945 from penghuo/opendistro-1.11-rc3

Backporting latest change from develop to opendistro-1.11

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/Analyzer.java

@@ -194,7 +194,7 @@ public LogicalPlan visitAggregation(Aggregation node, AnalysisContext context) {
     for (UnresolvedExpression expr : node.getAggExprList()) {
       NamedExpression aggExpr = namedExpressionAnalyzer.analyze(expr, context);
       aggregatorBuilder
-          .add(new NamedAggregator(aggExpr.getName(), (Aggregator) aggExpr.getDelegated()));
+          .add(new NamedAggregator(aggExpr.getNameOrAlias(), (Aggregator) aggExpr.getDelegated()));
     }
     ImmutableList<NamedAggregator> aggregators = aggregatorBuilder.build();
 
@@ -210,7 +210,7 @@ public LogicalPlan visitAggregation(Aggregation node, AnalysisContext context) {
     aggregators.forEach(aggregator -> newEnv.define(new Symbol(Namespace.FIELD_NAME,
         aggregator.getName()), aggregator.type()));
     groupBys.forEach(group -> newEnv.define(new Symbol(Namespace.FIELD_NAME,
-        group.getName()), group.type()));
+        group.getNameOrAlias()), group.type()));
     return new LogicalAggregation(child, aggregators, groupBys);
   }
 
@@ -291,7 +291,7 @@ public LogicalPlan visitProject(Project node, AnalysisContext context) {
     context.push();
     TypeEnvironment newEnv = context.peek();
     namedExpressions.forEach(expr -> newEnv.define(new Symbol(Namespace.FIELD_NAME,
-        expr.getName()), expr.type()));
+        expr.getNameOrAlias()), expr.type()));
     return new LogicalProject(child, namedExpressions);
   }
 

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionReferenceOptimizer.java

@@ -136,7 +136,8 @@ public Void visitAggregation(LogicalAggregation plan, Void context) {
               new ReferenceExpression(namedAggregator.getName(), namedAggregator.type())));
       // Create the mapping for all the group by.
       plan.getGroupByList().forEach(groupBy -> expressionMap
-          .put(groupBy.getDelegated(), new ReferenceExpression(groupBy.getName(), groupBy.type())));
+          .put(groupBy.getDelegated(),
+              new ReferenceExpression(groupBy.getNameOrAlias(), groupBy.type())));
       return null;
     }
 

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java

@@ -409,6 +409,10 @@ public FunctionExpression strcmp(Expression... expressions) {
     return function(BuiltinFunctionName.STRCMP, expressions);
   }
 
+  public FunctionExpression right(Expression... expressions) {
+    return function(BuiltinFunctionName.RIGHT, expressions);
+  }
+
   public FunctionExpression and(Expression... expressions) {
     return function(BuiltinFunctionName.AND, expressions);
   }

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/ExpressionNodeVisitor.java

@@ -62,7 +62,7 @@ public T visitLiteral(LiteralExpression node, C context) {
   }
 
   public T visitNamed(NamedExpression node, C context) {
-    return visitNode(node, context);
+    return node.getDelegated().accept(this, context);
   }
 
   public T visitReference(ReferenceExpression node, C context) {

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/NamedExpression.java

@@ -24,7 +24,6 @@
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
-import lombok.ToString;
 
 /**
  * Named expression that represents expression with name.
@@ -33,6 +32,7 @@
  */
 @AllArgsConstructor
 @EqualsAndHashCode
+@Getter
 @RequiredArgsConstructor
 public class NamedExpression implements Expression {
 
@@ -44,13 +44,11 @@ public class NamedExpression implements Expression {
   /**
    * Expression that being named.
    */
-  @Getter
   private final Expression delegated;
 
   /**
    * Optional alias.
    */
-  @Getter
   private String alias;
 
   @Override
@@ -67,7 +65,7 @@ public ExprType type() {
    * Get expression name using name or its alias (if it's present).
    * @return  expression name
    */
-  public String getName() {
+  public String getNameOrAlias() {
     return Strings.isNullOrEmpty(alias) ? name : alias;
   }
 
@@ -78,7 +76,7 @@ public <T, C> T accept(ExpressionNodeVisitor<T, C> visitor, C context) {
 
   @Override
   public String toString() {
-    return getName();
+    return getNameOrAlias();
   }
 
 }

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/AggregatorFunction.java

@@ -28,14 +28,22 @@
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIME;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIMESTAMP;
 
+import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType;
+import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionBuilder;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionResolver;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionSignature;
 import com.google.common.collect.ImmutableMap;
+
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.stream.Collectors;
+
 import lombok.experimental.UtilityClass;
 
 /**
@@ -73,27 +81,11 @@ private static FunctionResolver avg() {
 
   private static FunctionResolver count() {
     FunctionName functionName = BuiltinFunctionName.COUNT.getName();
-    return new FunctionResolver(
-        functionName,
-        new ImmutableMap.Builder<FunctionSignature, FunctionBuilder>()
-            .put(new FunctionSignature(functionName, Collections.singletonList(INTEGER)),
-                arguments -> new CountAggregator(arguments, INTEGER))
-            .put(new FunctionSignature(functionName, Collections.singletonList(LONG)),
-                arguments -> new CountAggregator(arguments, INTEGER))
-            .put(new FunctionSignature(functionName, Collections.singletonList(FLOAT)),
-                arguments -> new CountAggregator(arguments, INTEGER))
-            .put(new FunctionSignature(functionName, Collections.singletonList(DOUBLE)),
-                arguments -> new CountAggregator(arguments, INTEGER))
-            .put(new FunctionSignature(functionName, Collections.singletonList(STRING)),
-                arguments -> new CountAggregator(arguments, INTEGER))
-            .put(new FunctionSignature(functionName, Collections.singletonList(STRUCT)),
-                arguments -> new CountAggregator(arguments, INTEGER))
-            .put(new FunctionSignature(functionName, Collections.singletonList(ARRAY)),
-                arguments -> new CountAggregator(arguments, INTEGER))
-            .put(new FunctionSignature(functionName, Collections.singletonList(BOOLEAN)),
-                arguments -> new CountAggregator(arguments, INTEGER))
-            .build()
-    );
+    FunctionResolver functionResolver = new FunctionResolver(functionName,
+        ExprCoreType.coreTypes().stream().collect(Collectors.toMap(
+          type -> new FunctionSignature(functionName, Collections.singletonList(type)),
+          type -> arguments -> new CountAggregator(arguments, INTEGER))));
+    return functionResolver;
   }
 
   private static FunctionResolver sum() {

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java

@@ -131,6 +131,7 @@ public enum BuiltinFunctionName {
   CONCAT_WS(FunctionName.of("concat_ws")),
   LENGTH(FunctionName.of("length")),
   STRCMP(FunctionName.of("strcmp")),
+  RIGHT(FunctionName.of("right")),
 
   /**
    * NULL Test.

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/text/TextFunction.java

@@ -60,6 +60,7 @@ public void register(BuiltinFunctionRepository repository) {
     repository.register(concat_ws());
     repository.register(length());
     repository.register(strcmp());
+    repository.register(right());
   }
 
   /**
@@ -194,6 +195,16 @@ private FunctionResolver strcmp() {
             INTEGER, STRING, STRING));
   }
 
+  /**
+   * Returns the rightmost len characters from the string str, or NULL if any argument is NULL.
+   * Supports following signatures:
+   * (STRING, INTEGER) -> STRING
+   */
+  private FunctionResolver right() {
+    return define(BuiltinFunctionName.RIGHT.getName(),
+            impl(nullMissingHandling(TextFunction::exprRight), STRING, STRING, INTEGER));
+  }
+
   private static ExprValue exprSubstrStart(ExprValue exprValue, ExprValue start) {
     int startIdx = start.integerValue();
     if (startIdx == 0) {
@@ -225,5 +236,10 @@ private static ExprValue exprSubStr(String str, int start, int len) {
     }
     return new ExprStringValue(str.substring(start, start + len));
   }
+
+  private static ExprValue exprRight(ExprValue str, ExprValue len) {
+    return new ExprStringValue(str.stringValue().substring(
+            str.stringValue().length() - len.integerValue()));
+  }
 }
 

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/AggregationOperator.java

@@ -172,7 +172,7 @@ public GroupKey(ExprValue value) {
     public LinkedHashMap<String, ExprValue> groupKeyMap() {
       LinkedHashMap<String, ExprValue> map = new LinkedHashMap<>();
       for (int i = 0; i < groupByExprList.size(); i++) {
-        map.put(groupByExprList.get(i).getName(), groupByValueList.get(i));
+        map.put(groupByExprList.get(i).getNameOrAlias(), groupByValueList.get(i));
       }
       return map;
     }

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperator.java

@@ -62,7 +62,7 @@ public ExprValue next() {
     ImmutableMap.Builder<String, ExprValue> mapBuilder = new Builder<>();
     for (NamedExpression expr : projectList) {
       ExprValue exprValue = expr.valueOf(inputValue.bindingTuples());
-      mapBuilder.put(expr.getName(), exprValue);
+      mapBuilder.put(expr.getNameOrAlias(), exprValue);
     }
     return ExprTupleValue.fromExprValueMap(mapBuilder.build());
   }

core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/NamedExpressionAnalyzerTest.java

@@ -41,6 +41,6 @@ void visit_named_seleteitem() {
         new NamedExpressionAnalyzer(expressionAnalyzer);
 
     NamedExpression analyze = analyzer.analyze(alias, analysisContext);
-    assertEquals("integer_value", analyze.getName());
+    assertEquals("integer_value", analyze.getNameOrAlias());
   }
 }

core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/NamedExpressionTest.java

@@ -30,7 +30,7 @@ void name_an_expression() {
     LiteralExpression delegated = DSL.literal(10);
     NamedExpression namedExpression = DSL.named("10", delegated);
 
-    assertEquals("10", namedExpression.getName());
+    assertEquals("10", namedExpression.getNameOrAlias());
     assertEquals(delegated.type(), namedExpression.type());
     assertEquals(delegated.valueOf(valueEnv()), namedExpression.valueOf(valueEnv()));
   }
@@ -39,7 +39,7 @@ void name_an_expression() {
   void name_an_expression_with_alias() {
     LiteralExpression delegated = DSL.literal(10);
     NamedExpression namedExpression = DSL.named("10", delegated, "ten");
-    assertEquals("ten", namedExpression.getName());
+    assertEquals("ten", namedExpression.getNameOrAlias());
   }
 
   @Test
@@ -48,7 +48,7 @@ void name_an_named_expression() {
     Expression expression = DSL.named("10", delegated, "ten");
 
     NamedExpression namedExpression = DSL.named(expression);
-    assertEquals("ten", namedExpression.getName());
+    assertEquals("ten", namedExpression.getNameOrAlias());
   }
 
 }

core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/CountAggregatorTest.java

@@ -17,12 +17,16 @@
 
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.ARRAY;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN;
+import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATE;
+import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATETIME;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.LONG;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT;
+import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIMESTAMP;
+
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
@@ -58,6 +62,24 @@ public void count_double_field_expression() {
     assertEquals(4, result.value());
   }
 
+  @Test
+  public void count_date_field_expression() {
+    ExprValue result = aggregation(dsl.count(DSL.ref("date_value", DATE)), tuples);
+    assertEquals(4, result.value());
+  }
+
+  @Test
+  public void count_timestamp_field_expression() {
+    ExprValue result = aggregation(dsl.count(DSL.ref("timestamp_value", TIMESTAMP)), tuples);
+    assertEquals(4, result.value());
+  }
+
+  @Test
+  public void count_datetime_field_expression() {
+    ExprValue result = aggregation(dsl.count(DSL.ref("datetime_value", DATETIME)), tuples);
+    assertEquals(4, result.value());
+  }
+
   @Test
   public void count_arithmetic_expression() {
     ExprValue result = aggregation(dsl.count(

core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/text/TextFunctionTest.java

@@ -25,6 +25,7 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.mockito.Mockito.when;
 
+import com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprStringValue;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue;
 import com.amazon.opendistroforelasticsearch.sql.expression.DSL;
@@ -296,6 +297,23 @@ void strcmp() {
     assertEquals(missingValue(), eval(dsl.strcmp(missingRef, nullRef)));
   }
 
+  @Test
+  void right() {
+    FunctionExpression expression = dsl.right(
+            DSL.literal(new ExprStringValue("foobarbar")),
+            DSL.literal(new ExprIntegerValue(4)));
+    assertEquals(STRING, expression.type());
+    assertEquals("rbar", eval(expression).stringValue());
+
+    when(nullRef.type()).thenReturn(STRING);
+    when(missingRef.type()).thenReturn(INTEGER);
+    assertEquals(missingValue(), eval(dsl.right(nullRef, missingRef)));
+    assertEquals(nullValue(), eval(dsl.right(nullRef, DSL.literal(new ExprIntegerValue(1)))));
+
+    when(nullRef.type()).thenReturn(INTEGER);
+    assertEquals(nullValue(), eval(dsl.right(DSL.literal(new ExprStringValue("value")), nullRef)));
+  }
+
   void testConcatString(List<String> strings) {
     String expected = null;
     if (strings.stream().noneMatch(Objects::isNull)) {

core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperatorTest.java

@@ -104,11 +104,11 @@ public void project_keep_missing_value() {
   public void project_schema() {
     PhysicalPlan project = project(inputPlan,
         DSL.named("response", DSL.ref("response", INTEGER)),
-        DSL.named("action", DSL.ref("action", STRING)));
+        DSL.named("action", DSL.ref("action", STRING), "act"));
 
     assertThat(project.schema().getColumns(), contains(
         new ExecutionEngine.Schema.Column("response", null, INTEGER),
-        new ExecutionEngine.Schema.Column("action", null, STRING)
+        new ExecutionEngine.Schema.Column("action", "act", STRING)
     ));
   }
 }

docs/experiment/ppl/functions/string.rst

@@ -150,6 +150,29 @@ Example::
     +---------------------+---------------------+
 
 
+RIGHT
+-----
+
+Description
+>>>>>>>>>>>
+
+Usage: right(str, len) returns the rightmost len characters from the string str, or NULL if any argument is NULL.
+
+Argument type: STRING, INTEGER
+
+Return type: STRING
+
+Example::
+
+    od> source=people | eval `RIGHT('helloworld', 5)` = RIGHT('helloworld', 5), `RIGHT('HELLOWORLD', 0)` = RIGHT('HELLOWORLD', 0) | fields `RIGHT('helloworld', 5)`, `RIGHT('HELLOWORLD', 0)`
+    fetched rows / total rows = 1/1
+    +--------------------------+--------------------------+
+    | RIGHT('helloworld', 5)   | RIGHT('HELLOWORLD', 0)   |
+    |--------------------------+--------------------------|
+    | world                    |                          |
+    +--------------------------+--------------------------+
+
+
 RTRIM
 -----
 

docs/experiment/ppl/index.rst

@@ -77,3 +77,7 @@ The query start with search command and then flowing a set of command delimited
   - `Identifiers <general/identifiers.rst>`_
 
   - `Data Types <general/datatypes.rst>`_
+
+* **Limitations**
+
+  - `Limitations <limitations/limitations.rst>`_