Support show and describe statement (opendistro-for-elasticsearch#907)

* add frontend support * update core * add elasticsearch * update * add debug * fix UT * update * add comments * add explain * update
harold-wang · Dec 11, 2020 · 6990732 · 6990732
1 parent fe05b55
commit 6990732
Show file tree

Hide file tree

Showing 49 changed files with 1,667 additions and 138 deletions.
diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/TypeEnvironment.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/TypeEnvironment.java
@@ -23,7 +23,7 @@
 import com.amazon.opendistroforelasticsearch.sql.expression.Expression;
 import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression;
 import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Optional;
 import lombok.Getter;
@@ -70,7 +70,7 @@ public ExprType resolve(Symbol symbol) {
    * @return              all symbols in the namespace
    */
   public Map<String, ExprType> lookupAllFields(Namespace namespace) {
-    Map<String, ExprType> result = new HashMap<>();
+    Map<String, ExprType> result = new LinkedHashMap<>();
     symbolTable.lookupAllFields(namespace).forEach(result::putIfAbsent);
     return result;
   }

diff --git a/.../src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/symbol/SymbolTable.java b/.../src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/symbol/SymbolTable.java
@@ -20,11 +20,11 @@
 
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType;
 import java.util.EnumMap;
+import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.NavigableMap;
 import java.util.Optional;
 import java.util.TreeMap;
-import java.util.stream.Collectors;
 
 /**
  * Symbol table for symbol definition and resolution.
@@ -37,6 +37,14 @@ public class SymbolTable {
   private Map<Namespace, NavigableMap<String, ExprType>> tableByNamespace =
       new EnumMap<>(Namespace.class);
 
+  /**
+   * Two-dimension hash table to manage symbols with type in different namespace.
+   * Comparing with tableByNamespace, orderedTable use the LinkedHashMap to keep the order of
+   * symbol.
+   */
+  private Map<Namespace, LinkedHashMap<String, ExprType>> orderedTable =
+      new EnumMap<>(Namespace.class);
+
   /**
    * Store symbol with the type. Create new map for namespace for the first time.
    *
@@ -48,6 +56,11 @@ public void store(Symbol symbol, ExprType type) {
         symbol.getNamespace(),
         ns -> new TreeMap<>()
     ).put(symbol.getName(), type);
+
+    orderedTable.computeIfAbsent(
+        symbol.getNamespace(),
+        ns -> new LinkedHashMap<>()
+    ).put(symbol.getName(), type);
   }
 
   /**
@@ -61,6 +74,13 @@ public void remove(Symbol symbol) {
           return v;
         }
     );
+    orderedTable.computeIfPresent(
+        symbol.getNamespace(),
+        (k, v) -> {
+          v.remove(symbol.getName());
+          return v;
+        }
+    );
   }
 
   /**
@@ -106,13 +126,15 @@ public Map<String, ExprType> lookupByPrefix(Symbol prefix) {
    * @return              all symbols in the namespace map
    */
   public Map<String, ExprType> lookupAllFields(Namespace namespace) {
-    final Map<String, ExprType> allSymbols =
-        tableByNamespace.getOrDefault(namespace, emptyNavigableMap());
-    return allSymbols.entrySet().stream().filter(entry -> {
+    final LinkedHashMap<String, ExprType> allSymbols =
+        orderedTable.getOrDefault(namespace, new LinkedHashMap<>());
+    final LinkedHashMap<String, ExprType> results = new LinkedHashMap<>();
+    allSymbols.entrySet().stream().filter(entry -> {
       String symbolName = entry.getKey();
       int lastDot = symbolName.lastIndexOf(".");
       return -1 == lastDot || !allSymbols.containsKey(symbolName.substring(0, lastDot));
-    }).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+    }).forEach(entry -> results.put(entry.getKey(), entry.getValue()));
+    return results;
   }
 
   /**

diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java
@@ -18,8 +18,10 @@
 package com.amazon.opendistroforelasticsearch.sql.data.type;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 /**
@@ -77,6 +79,16 @@ public enum ExprCoreType implements ExprType {
    */
   private ExprCoreType parent;
 
+  /**
+   * The mapping between Type and legacy JDBC type name.
+   */
+  private static final Map<ExprCoreType, String> LEGACY_TYPE_NAME_MAPPING =
+      new ImmutableMap.Builder<ExprCoreType, String>()
+          .put(STRUCT, "object")
+          .put(ARRAY, "nested")
+          .put(STRING, "keyword")
+          .build();
+
   ExprCoreType(ExprCoreType... compatibleTypes) {
     for (ExprCoreType subType : compatibleTypes) {
       subType.parent = this;
@@ -93,6 +105,11 @@ public String typeName() {
     return this.name();
   }
 
+  @Override
+  public String legacyTypeName() {
+    return LEGACY_TYPE_NAME_MAPPING.getOrDefault(this, this.name());
+  }
+
   /**
    * Return all the valid ExprCoreType.
    */

diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprType.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprType.java
@@ -58,4 +58,11 @@ default List<ExprType> getParent() {
    * Get the type name.
    */
   String typeName();
+
+  /**
+   * Get the legacy type name for old engine.
+   */
+  default String legacyTypeName() {
+    return typeName();
+  }
 }
diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/storage/TableScanOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/storage/TableScanOperator.java
@@ -37,4 +37,10 @@ public List<PhysicalPlan> getChild() {
     return Collections.emptyList();
   }
 
+  /**
+   * Explain the execution plan.
+   *
+   * @return execution plan.
+   */
+  public abstract String explain();
 }
diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SystemIndexUtils.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SystemIndexUtils.java
@@ -0,0 +1,127 @@
+/*
+ *
+ *    Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License").
+ *    You may not use this file except in compliance with the License.
+ *    A copy of the License is located at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    or in the "license" file accompanying this file. This file is distributed
+ *    on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *    express or implied. See the License for the specific language governing
+ *    permissions and limitations under the License.
+ *
+ */
+
+package com.amazon.opendistroforelasticsearch.sql.utils;
+
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.UtilityClass;
+
+/**
+ * System Index Utils.
+ * Todo. Find the better name for this class.
+ */
+@UtilityClass
+public class SystemIndexUtils {
+  /**
+   * The prefix of all the system tables.
+   */
+  private static final String SYS_TABLES_PREFIX = "_ODFE_SYS_TABLE";
+
+  /**
+   * The prefix of all the meta tables.
+   */
+  private static final String SYS_META_PREFIX = SYS_TABLES_PREFIX + "_META";
+
+  /**
+   * The prefix of all the table mappings.
+   */
+  private static final String SYS_MAPPINGS_PREFIX = SYS_TABLES_PREFIX + "_MAPPINGS";
+
+  /**
+   * The _ODFE_SYS_TABLE_META.ALL contain all the table info.
+   */
+  public static final String TABLE_INFO = SYS_META_PREFIX + ".ALL";
+
+
+  public static Boolean isSystemIndex(String indexName) {
+    return indexName.startsWith(SYS_TABLES_PREFIX);
+  }
+
+  /**
+   * Compose system mapping table.
+   *
+   * @return system mapping table.
+   */
+  public static String mappingTable(String indexName) {
+    return String.join(".", SYS_MAPPINGS_PREFIX, indexName);
+  }
+
+  /**
+   * Build the {@link SystemTable}.
+   *
+   * @return {@link SystemTable}
+   */
+  public static SystemTable systemTable(String indexName) {
+    final int lastDot = indexName.indexOf(".");
+    String prefix = indexName.substring(0, lastDot);
+    String tableName = indexName.substring(lastDot + 1)
+        .replace("%", "*");
+
+    if (prefix.equalsIgnoreCase(SYS_META_PREFIX)) {
+      return new SystemInfoTable(tableName);
+    } else if (prefix.equalsIgnoreCase(SYS_MAPPINGS_PREFIX)) {
+      return new MetaInfoTable(tableName);
+    } else {
+      throw new IllegalStateException("Invalid system index name: " + indexName);
+    }
+  }
+
+  /**
+   * System Table.
+   */
+  public interface SystemTable {
+
+    String getTableName();
+
+    default boolean isSystemInfoTable() {
+      return false;
+    }
+
+    default boolean isMetaInfoTable() {
+      return false;
+    }
+  }
+
+  /**
+   * System Info Table.
+   */
+  @Getter
+  @RequiredArgsConstructor
+  public static class SystemInfoTable implements SystemTable {
+
+    private final String tableName;
+
+    public boolean isSystemInfoTable() {
+      return true;
+    }
+  }
+
+  /**
+   * System Table.
+   */
+  @Getter
+  @RequiredArgsConstructor
+  public static class MetaInfoTable implements SystemTable {
+
+    private final String tableName;
+
+    public boolean isMetaInfoTable() {
+      return true;
+    }
+  }
+}
diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/SelectAnalyzeTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/SelectAnalyzeTest.java
@@ -61,9 +61,9 @@ public void project_all_from_source() {
             LogicalPlanDSL.relation("schema"),
             DSL.named("integer_value", DSL.ref("integer_value", INTEGER)),
             DSL.named("double_value", DSL.ref("double_value", DOUBLE)),
-            DSL.named("string_value", DSL.ref("string_value", STRING)),
             DSL.named("integer_value", DSL.ref("integer_value", INTEGER)),
-            DSL.named("double_value", DSL.ref("double_value", DOUBLE))
+            DSL.named("double_value", DSL.ref("double_value", DOUBLE)),
+            DSL.named("string_value", DSL.ref("string_value", STRING))
         ),
         AstDSL.projectWithArg(
             AstDSL.relation("schema"),
@@ -127,8 +127,8 @@ public void stats_and_project_all() {
                 ImmutableList.of(DSL
                     .named("avg(integer_value)", dsl.avg(DSL.ref("integer_value", INTEGER)))),
                 ImmutableList.of(DSL.named("string_value", DSL.ref("string_value", STRING)))),
-            DSL.named("string_value", DSL.ref("string_value", STRING)),
-            DSL.named("avg(integer_value)", DSL.ref("avg(integer_value)", DOUBLE))
+            DSL.named("avg(integer_value)", DSL.ref("avg(integer_value)", DOUBLE)),
+            DSL.named("string_value", DSL.ref("string_value", STRING))
         ),
         AstDSL.projectWithArg(
             AstDSL.agg(
@@ -148,9 +148,9 @@ public void rename_and_project_all() {
             LogicalPlanDSL.rename(
                 LogicalPlanDSL.relation("schema"),
                 ImmutableMap.of(DSL.ref("integer_value", INTEGER), DSL.ref("ivalue", INTEGER))),
-            DSL.named("ivalue", DSL.ref("ivalue", INTEGER)),
+            DSL.named("double_value", DSL.ref("double_value", DOUBLE)),
             DSL.named("string_value", DSL.ref("string_value", STRING)),
-            DSL.named("double_value", DSL.ref("double_value", DOUBLE))
+            DSL.named("ivalue", DSL.ref("ivalue", INTEGER))
         ),
         AstDSL.projectWithArg(
             AstDSL.rename(

diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprTypeTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprTypeTest.java
@@ -17,14 +17,17 @@
 
 package com.amazon.opendistroforelasticsearch.sql.data.type;
 
+import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.ARRAY;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.LONG;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.SHORT;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING;
+import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.UNKNOWN;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -52,4 +55,19 @@ public void isCompatible() {
   public void getParent() {
     assertThat(((ExprType) () -> "test").getParent(), Matchers.contains(UNKNOWN));
   }
+
+  @Test
+  void legacyName() {
+    assertEquals("keyword", STRING.legacyTypeName());
+    assertEquals("nested", ARRAY.legacyTypeName());
+    assertEquals("object", STRUCT.legacyTypeName());
+    assertEquals("integer", INTEGER.legacyTypeName().toLowerCase());
+  }
+
+  // for test coverage.
+  @Test
+  void defaultLegacyTypeName() {
+    final ExprType exprType = () -> "dummy";
+    assertEquals("dummy", exprType.legacyTypeName());
+  }
 }