Lucene query pushdown optimization (#671)

* Add lucene builder interface and term query impl

* Add lucene query interface and term query impl

* Add range query impl

* Add more UT for range query

* Add wildcard query impl

* Add exists query impl

* Pass jacoco test

* Prepare PR

* Only push down filter close to relation

* Prepare PR

* Add limitation doc

* Add limitation doc

* Add limitation doc

docs/user/index.rst

@@ -49,3 +49,7 @@ Open Distro for Elasticsearch SQL enables you to extract insights out of Elastic
 
   - `Troubleshooting <dql/troubleshooting.rst>`_
 
+* **Limitations**
+
+  - `Limitations <limitations/limitations.rst>`_
+

docs/user/limitations/limitations.rst

@@ -0,0 +1,98 @@
+
+===========
+Limitations
+===========
+
+.. rubric:: Table of contents
+
+.. contents::
+   :local:
+   :depth: 2
+
+
+Introduction
+============
+
+In this doc, the restrictions and limitations of SQL plugin is covered as follows.
+
+
+Limitations on Aggregations
+===========================
+
+Aggregation over expression is not supported for now. You can only apply aggregation on fields, aggregations can't accept an expression as a parameter. For example, `avg(log(age))` is not supported.
+
+Here's a link to the Github issue - [Issue #288](https://github.com/opendistro-for-elasticsearch/sql/issues/288).
+
+
+Limitations on Subqueries
+=========================
+
+Subqueries in the FROM clause
+-----------------------------
+
+Subquery in the `FROM` clause in this format: `SELECT outer FROM (SELECT inner)` is supported only when the query is merged into one query. For example, the following query is supported::
+
+    SELECT t.f, t.d
+    FROM (
+        SELECT FlightNum as f, DestCountry as d
+        FROM kibana_sample_data_flights
+        WHERE OriginCountry = 'US') t
+
+But, if the outer query has `GROUP BY` or `ORDER BY`, then it's not supported.
+
+
+Limitations on JOINs
+====================
+
+JOIN does not support aggregations on the joined result. The `join` query does not support aggregations on the joined result.
+For example, e.g. `SELECT depo.name, avg(empo.age) FROM empo JOIN depo WHERE empo.id == depo.id GROUP BY depo.name` is not supported.
+
+Here's a link to the Github issue - [Issue 110](https://github.com/opendistro-for-elasticsearch/sql/issues/110).
+
+
+Limitations on Pagination
+=========================
+
+Pagination only supports basic queries for now. The pagination query enables you to get back paginated responses.
+Currently, the pagination only supports basic queries. For example, the following query returns the data with cursor id::
+
+    POST _opendistro/_sql/
+    {
+      "fetch_size" : 5,
+      "query" : "SELECT OriginCountry, DestCountry FROM kibana_sample_data_flights ORDER BY OriginCountry ASC"
+    }
+
+The response in JDBC format with cursor id::
+
+    {
+      "schema": [
+        {
+          "name": "OriginCountry",
+          "type": "keyword"
+        },
+        {
+          "name": "DestCountry",
+          "type": "keyword"
+        }
+      ],
+      "cursor": "d:eyJhIjp7fSwicyI6IkRYRjFaWEo1UVc1a1JtVjBZMmdCQUFBQUFBQUFCSllXVTJKVU4yeExiWEJSUkhsNFVrdDVXVEZSYkVKSmR3PT0iLCJjIjpbeyJuYW1lIjoiT3JpZ2luQ291bnRyeSIsInR5cGUiOiJrZXl3b3JkIn0seyJuYW1lIjoiRGVzdENvdW50cnkiLCJ0eXBlIjoia2V5d29yZCJ9XSwiZiI6MSwiaSI6ImtpYmFuYV9zYW1wbGVfZGF0YV9mbGlnaHRzIiwibCI6MTMwNTh9",
+      "total": 13059,
+      "datarows": [[
+        "AE",
+        "CN"
+      ]],
+      "size": 1,
+      "status": 200
+    }
+
+The query with `aggregation` and `join` does not support pagination for now.
+
+
+Limitations on Query Optimizations
+==================================
+
+Multi-fields in WHERE Conditions
+--------------------------------
+
+The filter expressions in ``WHERE`` clause may be pushed down to Elasticsearch DSL queries to avoid large amounts of data retrieved. In this case, for Elasticsearch multi-field (a text field with another keyword field inside), assumption is made that the keyword field name is always "keyword" which is true by default.
+

elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java

@@ -97,6 +97,11 @@ public PhysicalPlan implement(LogicalPlan plan) {
     return plan.accept(new DefaultImplementor<ElasticsearchIndexScan>() {
           @Override
           public PhysicalPlan visitFilter(LogicalFilter node, ElasticsearchIndexScan context) {
+            // For now (without optimizer), only push down filter close to relation
+            if (!(node.getChild().get(0) instanceof LogicalRelation)) {
+              return super.visitFilter(node, context);
+            }
+
             FilterQueryBuilder queryBuilder =
                 new FilterQueryBuilder(new DefaultExpressionSerializer());
 

elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexScan.java

@@ -16,6 +16,9 @@
 
 package com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage;
 
+import static org.elasticsearch.search.sort.FieldSortBuilder.DOC_FIELD_NAME;
+import static org.elasticsearch.search.sort.SortOrder.ASC;
+
 import com.amazon.opendistroforelasticsearch.sql.common.setting.Settings;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue;
 import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient;
@@ -102,14 +105,18 @@ public void pushDown(QueryBuilder query) {
     if (current == null) {
       source.query(query);
     } else {
-      if (isBoolMustQuery(current)) {
-        ((BoolQueryBuilder) current).must(query);
+      if (isBoolFilterQuery(current)) {
+        ((BoolQueryBuilder) current).filter(query);
       } else {
         source.query(QueryBuilders.boolQuery()
-                                  .must(current)
-                                  .must(query));
+                                  .filter(current)
+                                  .filter(query));
       }
     }
+
+    if (source.sorts() == null) {
+      source.sort(DOC_FIELD_NAME, ASC); // Make sure consistent order
+    }
   }
 
   @Override
@@ -119,9 +126,9 @@ public void close() {
     client.cleanup(request);
   }
 
-  private boolean isBoolMustQuery(QueryBuilder current) {
+  private boolean isBoolFilterQuery(QueryBuilder current) {
     return (current instanceof BoolQueryBuilder)
-        && !((BoolQueryBuilder) current).must().isEmpty();
+        && !((BoolQueryBuilder) current).filter().isEmpty();
   }
 
 }

elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilder.java

@@ -20,10 +20,19 @@
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.script.Script.DEFAULT_SCRIPT_TYPE;
 
+import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.filter.lucene.LuceneQuery;
+import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.filter.lucene.RangeQuery;
+import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.filter.lucene.RangeQuery.Comparison;
+import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.filter.lucene.TermQuery;
+import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.filter.lucene.WildcardQuery;
 import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.serialization.ExpressionSerializer;
 import com.amazon.opendistroforelasticsearch.sql.expression.Expression;
 import com.amazon.opendistroforelasticsearch.sql.expression.ExpressionNodeVisitor;
 import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression;
+import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName;
+import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName;
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
 import java.util.function.BiFunction;
 import lombok.RequiredArgsConstructor;
 import org.elasticsearch.index.query.BoolQueryBuilder;
@@ -40,6 +49,19 @@ public class FilterQueryBuilder extends ExpressionNodeVisitor<QueryBuilder, Obje
    */
   private final ExpressionSerializer serializer;
 
+  /**
+   * Mapping from function name to lucene query builder.
+   */
+  private final Map<FunctionName, LuceneQuery> luceneQueries =
+      ImmutableMap.<FunctionName, LuceneQuery>builder()
+          .put(BuiltinFunctionName.EQUAL.getName(), new TermQuery())
+          .put(BuiltinFunctionName.LESS.getName(), new RangeQuery(Comparison.LT))
+          .put(BuiltinFunctionName.GREATER.getName(), new RangeQuery(Comparison.GT))
+          .put(BuiltinFunctionName.LTE.getName(), new RangeQuery(Comparison.LTE))
+          .put(BuiltinFunctionName.GTE.getName(), new RangeQuery(Comparison.GTE))
+          .put(BuiltinFunctionName.LIKE.getName(), new WildcardQuery())
+          .build();
+
   /**
    * Build Elasticsearch filter query from expression.
    * @param expr  expression
@@ -55,16 +77,22 @@ public QueryBuilder build(Expression expr) {
   }
 
   @Override
-  public QueryBuilder visitFunction(FunctionExpression node, Object context) {
-    switch (node.getFunctionName().getFunctionName()) {
+  public QueryBuilder visitFunction(FunctionExpression func, Object context) {
+    FunctionName name = func.getFunctionName();
+    switch (name.getFunctionName()) {
       case "and":
-        return buildBoolQuery(node, context, BoolQueryBuilder::must);
+        return buildBoolQuery(func, context, BoolQueryBuilder::filter);
       case "or":
-        return buildBoolQuery(node, context, BoolQueryBuilder::should);
+        return buildBoolQuery(func, context, BoolQueryBuilder::should);
       case "not":
-        return buildBoolQuery(node, context, BoolQueryBuilder::mustNot);
-      default:
-        return buildScriptQuery(node);
+        return buildBoolQuery(func, context, BoolQueryBuilder::mustNot);
+      default: {
+        LuceneQuery query = luceneQueries.get(name);
+        if (query != null && query.canSupport(func)) {
+          return query.build(func);
+        }
+        return buildScriptQuery(func);
+      }
     }
   }
 

elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/lucene/LuceneQuery.java

@@ -0,0 +1,72 @@
+/*
+ *    Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License").
+ *    You may not use this file except in compliance with the License.
+ *    A copy of the License is located at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    or in the "license" file accompanying this file. This file is distributed
+ *    on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *    express or implied. See the License for the specific language governing
+ *    permissions and limitations under the License.
+ *
+ */
+
+package com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.filter.lucene;
+
+import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue;
+import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType;
+import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression;
+import com.amazon.opendistroforelasticsearch.sql.expression.LiteralExpression;
+import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression;
+import org.elasticsearch.index.query.QueryBuilder;
+
+/**
+ * Lucene query abstraction that builds Lucene query from function expression.
+ */
+public abstract class LuceneQuery {
+
+  /**
+   * Check if function expression supported by current Lucene query.
+   * Default behavior is that report supported if:
+   *  1. Left is a reference
+   *  2. Right side is a literal
+   *
+   * @param func    function
+   * @return        return true if supported, otherwise false.
+   */
+  public boolean canSupport(FunctionExpression func) {
+    return (func.getArguments().size() == 2)
+        && (func.getArguments().get(0) instanceof ReferenceExpression)
+        && (func.getArguments().get(1) instanceof LiteralExpression);
+  }
+
+  /**
+   * Build Lucene query from function expression.
+   *
+   * @param func  function
+   * @return      query
+   */
+  public QueryBuilder build(FunctionExpression func) {
+    ReferenceExpression ref = (ReferenceExpression) func.getArguments().get(0);
+    LiteralExpression literal = (LiteralExpression) func.getArguments().get(1);
+    return doBuild(ref.getAttr(), ref.type(), literal.valueOf(null));
+  }
+
+  /**
+   * Build method that subclass implements by default which is to build query
+   * from reference and literal in function arguments.
+   *
+   * @param fieldName   field name
+   * @param fieldType        expr fieldType
+   * @param literal       expr literal
+   * @return            query
+   */
+  protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue literal) {
+    throw new UnsupportedOperationException(
+        "Subclass doesn't implement this and build method either");
+  }
+
+}

elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/lucene/RangeQuery.java

@@ -0,0 +1,60 @@
+/*
+ *    Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License").
+ *    You may not use this file except in compliance with the License.
+ *    A copy of the License is located at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    or in the "license" file accompanying this file. This file is distributed
+ *    on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *    express or implied. See the License for the specific language governing
+ *    permissions and limitations under the License.
+ *
+ */
+
+package com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.filter.lucene;
+
+import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue;
+import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType;
+import lombok.RequiredArgsConstructor;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.RangeQueryBuilder;
+
+/**
+ * Lucene query that builds range query for non-quality comparison.
+ */
+@RequiredArgsConstructor
+public class RangeQuery extends LuceneQuery {
+
+  public enum Comparison {
+    LT, GT, LTE, GTE, BETWEEN
+  }
+
+  /**
+   * Comparison that range query build for.
+   */
+  private final Comparison comparison;
+
+  @Override
+  protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue literal) {
+    Object value = literal.value();
+
+    RangeQueryBuilder query = QueryBuilders.rangeQuery(fieldName);
+    switch (comparison) {
+      case LT:
+        return query.lt(value);
+      case GT:
+        return query.gt(value);
+      case LTE:
+        return query.lte(value);
+      case GTE:
+        return query.gte(value);
+      default:
+        throw new IllegalStateException("Comparison is supported by range query: " + comparison);
+    }
+  }
+
+}