Add JDBC formatter in new SQL engine (#854)

* Add jdbc formatter

* Add error response format method

* Add java doc

* Convert type to lower case

* Enable branch coverage and pass jacoco

* Remove version for now

* Fix broken ITs

* Change error code to 503

* Add back legcy ITs

* Convert all legacy types

* Add IT

* Prepare PR

* Prepare PR

* Address PR comment

* Remove valueMap

integ-test/build.gradle

@@ -128,9 +128,6 @@ task integTestWithNewEngine(type: RestIntegTestTask) {
     // Skip old semantic analyzer IT because analyzer in new engine has different behavior
     exclude 'com/amazon/opendistroforelasticsearch/sql/legacy/QueryAnalysisIT.class'
 
-    // Skip this IT to avoid breaking tests due to inconsistency in JDBC schema
-    exclude 'com/amazon/opendistroforelasticsearch/sql/legacy/AggregationExpressionIT.class'
-
     // Skip this IT because all assertions are against explain output
     exclude 'com/amazon/opendistroforelasticsearch/sql/legacy/OrderIT.class'
 }

integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/QueryIT.java

@@ -187,7 +187,6 @@ public void selectAllWithFieldAndGroupByReverseOrder() throws IOException {
     checkSelectAllAndFieldAggregationResponseSize(response, "age");
   }
 
-  @Ignore("This failed because there is no alias field in schema of new engine default formatter")
   @Test
   public void selectFieldWithAliasAndGroupBy() {
     String response =

integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/DateTimeFunctionIT.java

@@ -136,11 +136,11 @@ public void testDay() throws IOException {
   @Test
   public void testDayName() throws IOException {
     JSONObject result = executeQuery("select dayname(date('2020-09-16'))");
-    verifySchema(result, schema("dayname(date('2020-09-16'))", null, "string"));
+    verifySchema(result, schema("dayname(date('2020-09-16'))", null, "keyword"));
     verifyDataRows(result, rows("Wednesday"));
 
     result = executeQuery("select dayname('2020-09-16')");
-    verifySchema(result, schema("dayname('2020-09-16')", null, "string"));
+    verifySchema(result, schema("dayname('2020-09-16')", null, "keyword"));
     verifyDataRows(result, rows("Wednesday"));
   }
 
@@ -256,11 +256,11 @@ public void testMonth() throws IOException {
   @Test
   public void testMonthName() throws IOException {
     JSONObject result = executeQuery("select monthname(date('2020-09-16'))");
-    verifySchema(result, schema("monthname(date('2020-09-16'))", null, "string"));
+    verifySchema(result, schema("monthname(date('2020-09-16'))", null, "keyword"));
     verifyDataRows(result, rows("September"));
 
     result = executeQuery("select monthname('2020-09-16')");
-    verifySchema(result, schema("monthname('2020-09-16')", null, "string"));
+    verifySchema(result, schema("monthname('2020-09-16')", null, "keyword"));
     verifyDataRows(result, rows("September"));
   }
 
@@ -378,12 +378,12 @@ public void testWeek() throws IOException {
   void verifyDateFormat(String date, String type, String format, String formatted) throws IOException {
     String query = String.format("date_format(%s('%s'), '%s')", type, date, format);
     JSONObject result = executeQuery("select " + query);
-    verifySchema(result, schema(query, null, "string"));
+    verifySchema(result, schema(query, null, "keyword"));
     verifyDataRows(result, rows(formatted));
 
     query = String.format("date_format('%s', '%s')", date, format);
     result = executeQuery("select " + query);
-    verifySchema(result, schema(query, null, "string"));
+    verifySchema(result, schema(query, null, "keyword"));
     verifyDataRows(result, rows(formatted));
   }
 

integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/JdbcFormatIT.java

@@ -0,0 +1,58 @@
+/*
+ *    Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License").
+ *    You may not use this file except in compliance with the License.
+ *    A copy of the License is located at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    or in the "license" file accompanying this file. This file is distributed
+ *    on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *    express or implied. See the License for the specific language governing
+ *    permissions and limitations under the License.
+ *
+ */
+
+package com.amazon.opendistroforelasticsearch.sql.sql;
+
+import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_BANK;
+import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.schema;
+import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifySchema;
+
+import com.amazon.opendistroforelasticsearch.sql.legacy.SQLIntegTestCase;
+import org.json.JSONObject;
+import org.junit.jupiter.api.Test;
+
+public class JdbcFormatIT extends SQLIntegTestCase {
+
+  @Override
+  protected void init() throws Exception {
+    loadIndex(Index.BANK);
+  }
+
+  @Test
+  public void testSimpleDataTypesInSchema() {
+    JSONObject response = new JSONObject(executeQuery(
+        "SELECT account_number, address, age, birthdate, city, male, state "
+            + "FROM " + TEST_INDEX_BANK, "jdbc"));
+
+    verifySchema(response,
+        schema("account_number", "long"),
+        schema("address", "text"),
+        schema("age", "integer"),
+        schema("birthdate", "timestamp"),
+        schema("city", "keyword"),
+        schema("male", "boolean"),
+        schema("state", "text"));
+  }
+
+  @Test
+  public void testAliasInSchema() {
+    JSONObject response = new JSONObject(executeQuery(
+        "SELECT account_number AS acc FROM " + TEST_INDEX_BANK, "jdbc"));
+
+    verifySchema(response, schema("acc", "acc", "long"));
+  }
+
+}

integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/MathematicalFunctionIT.java

@@ -43,11 +43,11 @@ public void init() throws Exception {
   @Test
   public void testConv() throws IOException {
     JSONObject result = executeQuery("select conv(11, 10, 16)");
-    verifySchema(result, schema("conv(11, 10, 16)", null, "string"));
+    verifySchema(result, schema("conv(11, 10, 16)", null, "keyword"));
     verifyDataRows(result, rows("b"));
 
     result = executeQuery("select conv(11, 16, 10)");
-    verifySchema(result, schema("conv(11, 16, 10)", null, "string"));
+    verifySchema(result, schema("conv(11, 16, 10)", null, "keyword"));
     verifyDataRows(result, rows("17"));
   }
 

integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/TextFunctionIT.java

@@ -60,59 +60,59 @@ public void testRegexp() throws IOException {
 
   @Test
   public void testSubstr() throws IOException {
-    verifyQuery("substr('hello', 2)", "string", "ello");
-    verifyQuery("substr('hello', 2, 2)", "string", "el");
+    verifyQuery("substr('hello', 2)", "keyword", "ello");
+    verifyQuery("substr('hello', 2, 2)", "keyword", "el");
   }
 
   @Test
   public void testSubstring() throws IOException {
-    verifyQuery("substring('hello', 2)", "string", "ello");
-    verifyQuery("substring('hello', 2, 2)", "string", "el");
+    verifyQuery("substring('hello', 2)", "keyword", "ello");
+    verifyQuery("substring('hello', 2, 2)", "keyword", "el");
   }
 
   @Test
   public void testUpper() throws IOException {
-    verifyQuery("upper('hello')", "string", "HELLO");
-    verifyQuery("upper('HELLO')", "string", "HELLO");
+    verifyQuery("upper('hello')", "keyword", "HELLO");
+    verifyQuery("upper('HELLO')", "keyword", "HELLO");
   }
 
   @Test
   public void testLower() throws IOException {
-    verifyQuery("lower('hello')", "string", "hello");
-    verifyQuery("lower('HELLO')", "string", "hello");
+    verifyQuery("lower('hello')", "keyword", "hello");
+    verifyQuery("lower('HELLO')", "keyword", "hello");
   }
 
   @Test
   public void testTrim() throws IOException {
-    verifyQuery("trim(' hello')", "string", "hello");
-    verifyQuery("trim('hello ')", "string", "hello");
-    verifyQuery("trim('  hello  ')", "string", "hello");
+    verifyQuery("trim(' hello')", "keyword", "hello");
+    verifyQuery("trim('hello ')", "keyword", "hello");
+    verifyQuery("trim('  hello  ')", "keyword", "hello");
   }
 
   @Test
   public void testRtrim() throws IOException {
-    verifyQuery("rtrim(' hello')", "string", " hello");
-    verifyQuery("rtrim('hello ')", "string", "hello");
-    verifyQuery("rtrim('  hello  ')", "string", "  hello");
+    verifyQuery("rtrim(' hello')", "keyword", " hello");
+    verifyQuery("rtrim('hello ')", "keyword", "hello");
+    verifyQuery("rtrim('  hello  ')", "keyword", "  hello");
   }
 
   @Test
   public void testLtrim() throws IOException {
-    verifyQuery("ltrim(' hello')", "string", "hello");
-    verifyQuery("ltrim('hello ')", "string", "hello ");
-    verifyQuery("ltrim('  hello  ')", "string", "hello  ");
+    verifyQuery("ltrim(' hello')", "keyword", "hello");
+    verifyQuery("ltrim('hello ')", "keyword", "hello ");
+    verifyQuery("ltrim('  hello  ')", "keyword", "hello  ");
   }
 
   @Test
   public void testConcat() throws IOException {
-    verifyQuery("concat('hello', 'world')", "string", "helloworld");
-    verifyQuery("concat('', 'hello')", "string", "hello");
+    verifyQuery("concat('hello', 'world')", "keyword", "helloworld");
+    verifyQuery("concat('', 'hello')", "keyword", "hello");
   }
 
   @Test
   public void testConcat_ws() throws IOException {
-    verifyQuery("concat_ws(',', 'hello', 'world')", "string", "hello,world");
-    verifyQuery("concat_ws(',', '', 'hello')", "string", ",hello");
+    verifyQuery("concat_ws(',', 'hello', 'world')", "keyword", "hello,world");
+    verifyQuery("concat_ws(',', '', 'hello')", "keyword", ",hello");
   }
 
   @Test

legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java

@@ -28,8 +28,8 @@
 import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.ExplainResponse;
 import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan;
 import com.amazon.opendistroforelasticsearch.sql.protocol.response.QueryResult;
+import com.amazon.opendistroforelasticsearch.sql.protocol.response.format.JdbcResponseFormatter;
 import com.amazon.opendistroforelasticsearch.sql.protocol.response.format.JsonResponseFormatter;
-import com.amazon.opendistroforelasticsearch.sql.protocol.response.format.SimpleJsonResponseFormatter;
 import com.amazon.opendistroforelasticsearch.sql.sql.SQLService;
 import com.amazon.opendistroforelasticsearch.sql.sql.config.SQLServiceConfig;
 import com.amazon.opendistroforelasticsearch.sql.sql.domain.SQLQueryRequest;
@@ -149,9 +149,8 @@ public void onFailure(Exception e) {
     };
   }
 
-  // TODO: duplicate code here as in RestPPLQueryAction
   private ResponseListener<QueryResponse> createQueryResponseListener(RestChannel channel) {
-    SimpleJsonResponseFormatter formatter = new SimpleJsonResponseFormatter(PRETTY);
+    JdbcResponseFormatter formatter = new JdbcResponseFormatter(PRETTY);
     return new ResponseListener<QueryResponse>() {
       @Override
       public void onResponse(QueryResponse response) {

protocol/build.gradle

@@ -15,6 +15,7 @@ dependencies {
     compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.10.4'
     implementation 'com.google.code.gson:gson:2.8.6'
     compile project(':core')
+    compile project(':elasticsearch')
 
     testImplementation('org.junit.jupiter:junit-jupiter:5.6.2')
     testCompile group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1'
@@ -49,9 +50,13 @@ jacocoTestCoverageVerification {
     violationRules {
         rule {
             limit {
+                counter = 'LINE'
+                minimum = 1.0
+            }
+            limit {
+                counter = 'BRANCH'
                 minimum = 1.0
             }
-
         }
     }
     afterEvaluate {

protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java

@@ -23,6 +23,7 @@
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.Map;
+import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 
 /**
@@ -31,6 +32,8 @@
  */
 @RequiredArgsConstructor
 public class QueryResult implements Iterable<Object[]> {
+
+  @Getter
   private final ExecutionEngine.Schema schema;
 
   /**