From 6990732cd43d6896d2ee6282b7a3c3367777cba9 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Fri, 11 Dec 2020 11:19:23 -0800 Subject: [PATCH] Support show and describe statement (#907) * add frontend support * update core * add elasticsearch * update * add debug * fix UT * update * add comments * add explain * update --- .../sql/analysis/TypeEnvironment.java | 4 +- .../sql/analysis/symbol/SymbolTable.java | 32 +++- .../sql/data/type/ExprCoreType.java | 17 ++ .../sql/data/type/ExprType.java | 7 + .../sql/storage/TableScanOperator.java | 6 + .../sql/utils/SystemIndexUtils.java | 127 ++++++++++++++ .../sql/analysis/SelectAnalyzeTest.java | 12 +- .../sql/data/type/ExprTypeTest.java | 18 ++ .../sql/executor/ExplainTest.java | 16 +- .../sql/storage/TableScanOperatorTest.java | 5 + .../sql/utils/SystemIndexUtilsTest.java | 67 ++++++++ docs/experiment/ppl/cmd/search.rst | 28 ++-- docs/experiment/ppl/cmd/stats.rst | 36 ++-- docs/user/general/identifiers.rst | 32 ++-- .../client/ElasticsearchClient.java | 17 ++ .../client/ElasticsearchNodeClient.java | 43 ++++- .../client/ElasticsearchRestClient.java | 52 ++++++ .../data/type/ElasticsearchDataType.java | 16 ++ .../ElasticsearchExecutionEngine.java | 4 +- .../ElasticsearchCatIndicesRequest.java | 76 +++++++++ .../ElasticsearchDescribeIndexRequest.java | 155 ++++++++++++++++++ .../system/ElasticsearchSystemRequest.java | 34 ++++ .../storage/ElasticsearchIndex.java | 43 +---- .../storage/ElasticsearchIndexScan.java | 4 + .../storage/ElasticsearchStorageEngine.java | 9 +- .../system/ElasticsearchSystemIndex.java | 89 ++++++++++ .../system/ElasticsearchSystemIndexScan.java | 64 ++++++++ .../ElasticsearchSystemIndexSchema.java | 78 +++++++++ .../client/ElasticsearchNodeClientTest.java | 42 +++++ .../client/ElasticsearchRestClientTest.java | 47 ++++++ .../data/type/ElasticsearchDataTypeTest.java | 6 + .../ElasticsearchExecutionEngineTest.java | 5 + .../ElasticsearchCatIndicesRequestTest.java | 58 +++++++ ...ElasticsearchDescribeIndexRequestTest.java | 68 ++++++++ .../ElasticsearchStorageEngineTest.java | 12 +- .../ElasticsearchSystemIndexScanTest.java | 55 +++++++ .../system/ElasticsearchSystemIndexTest.java | 81 +++++++++ .../sql/legacy/MetaDataQueriesIT.java | 4 + .../sql/sql/AdminIT.java | 93 +++++++++++ .../ppl/explain_filter_agg_push.json | 2 +- .../ppl/explain_filter_push.json | 2 +- .../expectedOutput/ppl/explain_sort_push.json | 2 +- .../expectedOutput/sql/explain_show.json | 25 +++ .../format/JdbcResponseFormatter.java | 18 +- sql/src/main/antlr/OpenDistroSQLLexer.g4 | 4 +- sql/src/main/antlr/OpenDistroSQLParser.g4 | 30 +++- .../sql/sql/parser/AstBuilder.java | 26 +++ .../sql/sql/parser/AstExpressionBuilder.java | 32 ++++ .../sql/sql/parser/AstBuilderTest.java | 102 ++++++++++++ 49 files changed, 1667 insertions(+), 138 deletions(-) create mode 100644 core/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SystemIndexUtils.java create mode 100644 core/src/test/java/com/amazon/opendistroforelasticsearch/sql/utils/SystemIndexUtilsTest.java create mode 100644 elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/system/ElasticsearchCatIndicesRequest.java create mode 100644 elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/system/ElasticsearchDescribeIndexRequest.java create mode 100644 elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/system/ElasticsearchSystemRequest.java create mode 100644 elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/system/ElasticsearchSystemIndex.java create mode 100644 elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/system/ElasticsearchSystemIndexScan.java create mode 100644 elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/system/ElasticsearchSystemIndexSchema.java create mode 100644 elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/system/ElasticsearchCatIndicesRequestTest.java create mode 100644 elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/system/ElasticsearchDescribeIndexRequestTest.java create mode 100644 elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/system/ElasticsearchSystemIndexScanTest.java create mode 100644 elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/system/ElasticsearchSystemIndexTest.java create mode 100644 integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/AdminIT.java create mode 100644 integ-test/src/test/resources/expectedOutput/sql/explain_show.json diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/TypeEnvironment.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/TypeEnvironment.java index 07849f92d9..f54c168f35 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/TypeEnvironment.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/TypeEnvironment.java @@ -23,7 +23,7 @@ import com.amazon.opendistroforelasticsearch.sql.expression.Expression; import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; import lombok.Getter; @@ -70,7 +70,7 @@ public ExprType resolve(Symbol symbol) { * @return all symbols in the namespace */ public Map lookupAllFields(Namespace namespace) { - Map result = new HashMap<>(); + Map result = new LinkedHashMap<>(); symbolTable.lookupAllFields(namespace).forEach(result::putIfAbsent); return result; } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/symbol/SymbolTable.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/symbol/SymbolTable.java index 4615583e30..4b423b35ee 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/symbol/SymbolTable.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/symbol/SymbolTable.java @@ -20,11 +20,11 @@ import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; import java.util.EnumMap; +import java.util.LinkedHashMap; import java.util.Map; import java.util.NavigableMap; import java.util.Optional; import java.util.TreeMap; -import java.util.stream.Collectors; /** * Symbol table for symbol definition and resolution. @@ -37,6 +37,14 @@ public class SymbolTable { private Map> tableByNamespace = new EnumMap<>(Namespace.class); + /** + * Two-dimension hash table to manage symbols with type in different namespace. + * Comparing with tableByNamespace, orderedTable use the LinkedHashMap to keep the order of + * symbol. + */ + private Map> orderedTable = + new EnumMap<>(Namespace.class); + /** * Store symbol with the type. Create new map for namespace for the first time. * @@ -48,6 +56,11 @@ public void store(Symbol symbol, ExprType type) { symbol.getNamespace(), ns -> new TreeMap<>() ).put(symbol.getName(), type); + + orderedTable.computeIfAbsent( + symbol.getNamespace(), + ns -> new LinkedHashMap<>() + ).put(symbol.getName(), type); } /** @@ -61,6 +74,13 @@ public void remove(Symbol symbol) { return v; } ); + orderedTable.computeIfPresent( + symbol.getNamespace(), + (k, v) -> { + v.remove(symbol.getName()); + return v; + } + ); } /** @@ -106,13 +126,15 @@ public Map lookupByPrefix(Symbol prefix) { * @return all symbols in the namespace map */ public Map lookupAllFields(Namespace namespace) { - final Map allSymbols = - tableByNamespace.getOrDefault(namespace, emptyNavigableMap()); - return allSymbols.entrySet().stream().filter(entry -> { + final LinkedHashMap allSymbols = + orderedTable.getOrDefault(namespace, new LinkedHashMap<>()); + final LinkedHashMap results = new LinkedHashMap<>(); + allSymbols.entrySet().stream().filter(entry -> { String symbolName = entry.getKey(); int lastDot = symbolName.lastIndexOf("."); return -1 == lastDot || !allSymbols.containsKey(symbolName.substring(0, lastDot)); - }).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + }).forEach(entry -> results.put(entry.getKey(), entry.getValue())); + return results; } /** diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java index fe12d2735c..48202206a9 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java @@ -18,8 +18,10 @@ package com.amazon.opendistroforelasticsearch.sql.data.type; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; /** @@ -77,6 +79,16 @@ public enum ExprCoreType implements ExprType { */ private ExprCoreType parent; + /** + * The mapping between Type and legacy JDBC type name. + */ + private static final Map LEGACY_TYPE_NAME_MAPPING = + new ImmutableMap.Builder() + .put(STRUCT, "object") + .put(ARRAY, "nested") + .put(STRING, "keyword") + .build(); + ExprCoreType(ExprCoreType... compatibleTypes) { for (ExprCoreType subType : compatibleTypes) { subType.parent = this; @@ -93,6 +105,11 @@ public String typeName() { return this.name(); } + @Override + public String legacyTypeName() { + return LEGACY_TYPE_NAME_MAPPING.getOrDefault(this, this.name()); + } + /** * Return all the valid ExprCoreType. */ diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprType.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprType.java index 59aaae5040..a32ffd777e 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprType.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprType.java @@ -58,4 +58,11 @@ default List getParent() { * Get the type name. */ String typeName(); + + /** + * Get the legacy type name for old engine. + */ + default String legacyTypeName() { + return typeName(); + } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/storage/TableScanOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/storage/TableScanOperator.java index bcc5a8cab2..bff0a622d4 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/storage/TableScanOperator.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/storage/TableScanOperator.java @@ -37,4 +37,10 @@ public List getChild() { return Collections.emptyList(); } + /** + * Explain the execution plan. + * + * @return execution plan. + */ + public abstract String explain(); } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SystemIndexUtils.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SystemIndexUtils.java new file mode 100644 index 0000000000..db0ba48176 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SystemIndexUtils.java @@ -0,0 +1,127 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.utils; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.experimental.UtilityClass; + +/** + * System Index Utils. + * Todo. Find the better name for this class. + */ +@UtilityClass +public class SystemIndexUtils { + /** + * The prefix of all the system tables. + */ + private static final String SYS_TABLES_PREFIX = "_ODFE_SYS_TABLE"; + + /** + * The prefix of all the meta tables. + */ + private static final String SYS_META_PREFIX = SYS_TABLES_PREFIX + "_META"; + + /** + * The prefix of all the table mappings. + */ + private static final String SYS_MAPPINGS_PREFIX = SYS_TABLES_PREFIX + "_MAPPINGS"; + + /** + * The _ODFE_SYS_TABLE_META.ALL contain all the table info. + */ + public static final String TABLE_INFO = SYS_META_PREFIX + ".ALL"; + + + public static Boolean isSystemIndex(String indexName) { + return indexName.startsWith(SYS_TABLES_PREFIX); + } + + /** + * Compose system mapping table. + * + * @return system mapping table. + */ + public static String mappingTable(String indexName) { + return String.join(".", SYS_MAPPINGS_PREFIX, indexName); + } + + /** + * Build the {@link SystemTable}. + * + * @return {@link SystemTable} + */ + public static SystemTable systemTable(String indexName) { + final int lastDot = indexName.indexOf("."); + String prefix = indexName.substring(0, lastDot); + String tableName = indexName.substring(lastDot + 1) + .replace("%", "*"); + + if (prefix.equalsIgnoreCase(SYS_META_PREFIX)) { + return new SystemInfoTable(tableName); + } else if (prefix.equalsIgnoreCase(SYS_MAPPINGS_PREFIX)) { + return new MetaInfoTable(tableName); + } else { + throw new IllegalStateException("Invalid system index name: " + indexName); + } + } + + /** + * System Table. + */ + public interface SystemTable { + + String getTableName(); + + default boolean isSystemInfoTable() { + return false; + } + + default boolean isMetaInfoTable() { + return false; + } + } + + /** + * System Info Table. + */ + @Getter + @RequiredArgsConstructor + public static class SystemInfoTable implements SystemTable { + + private final String tableName; + + public boolean isSystemInfoTable() { + return true; + } + } + + /** + * System Table. + */ + @Getter + @RequiredArgsConstructor + public static class MetaInfoTable implements SystemTable { + + private final String tableName; + + public boolean isMetaInfoTable() { + return true; + } + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/SelectAnalyzeTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/SelectAnalyzeTest.java index 814fb63a83..4f37b48792 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/SelectAnalyzeTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/SelectAnalyzeTest.java @@ -61,9 +61,9 @@ public void project_all_from_source() { LogicalPlanDSL.relation("schema"), DSL.named("integer_value", DSL.ref("integer_value", INTEGER)), DSL.named("double_value", DSL.ref("double_value", DOUBLE)), - DSL.named("string_value", DSL.ref("string_value", STRING)), DSL.named("integer_value", DSL.ref("integer_value", INTEGER)), - DSL.named("double_value", DSL.ref("double_value", DOUBLE)) + DSL.named("double_value", DSL.ref("double_value", DOUBLE)), + DSL.named("string_value", DSL.ref("string_value", STRING)) ), AstDSL.projectWithArg( AstDSL.relation("schema"), @@ -127,8 +127,8 @@ public void stats_and_project_all() { ImmutableList.of(DSL .named("avg(integer_value)", dsl.avg(DSL.ref("integer_value", INTEGER)))), ImmutableList.of(DSL.named("string_value", DSL.ref("string_value", STRING)))), - DSL.named("string_value", DSL.ref("string_value", STRING)), - DSL.named("avg(integer_value)", DSL.ref("avg(integer_value)", DOUBLE)) + DSL.named("avg(integer_value)", DSL.ref("avg(integer_value)", DOUBLE)), + DSL.named("string_value", DSL.ref("string_value", STRING)) ), AstDSL.projectWithArg( AstDSL.agg( @@ -148,9 +148,9 @@ public void rename_and_project_all() { LogicalPlanDSL.rename( LogicalPlanDSL.relation("schema"), ImmutableMap.of(DSL.ref("integer_value", INTEGER), DSL.ref("ivalue", INTEGER))), - DSL.named("ivalue", DSL.ref("ivalue", INTEGER)), + DSL.named("double_value", DSL.ref("double_value", DOUBLE)), DSL.named("string_value", DSL.ref("string_value", STRING)), - DSL.named("double_value", DSL.ref("double_value", DOUBLE)) + DSL.named("ivalue", DSL.ref("ivalue", INTEGER)) ), AstDSL.projectWithArg( AstDSL.rename( diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprTypeTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprTypeTest.java index 7d75e79748..0e1b4f859e 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprTypeTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprTypeTest.java @@ -17,14 +17,17 @@ package com.amazon.opendistroforelasticsearch.sql.data.type; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.ARRAY; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.LONG; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.SHORT; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.UNKNOWN; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -52,4 +55,19 @@ public void isCompatible() { public void getParent() { assertThat(((ExprType) () -> "test").getParent(), Matchers.contains(UNKNOWN)); } + + @Test + void legacyName() { + assertEquals("keyword", STRING.legacyTypeName()); + assertEquals("nested", ARRAY.legacyTypeName()); + assertEquals("object", STRUCT.legacyTypeName()); + assertEquals("integer", INTEGER.legacyTypeName().toLowerCase()); + } + + // for test coverage. + @Test + void defaultLegacyTypeName() { + final ExprType exprType = () -> "dummy"; + assertEquals("dummy", exprType.legacyTypeName()); + } } \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/ExplainTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/ExplainTest.java index 23d233f49c..d8fc00b106 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/ExplainTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/ExplainTest.java @@ -97,7 +97,7 @@ void can_explain_project_filter_table_scan() { singletonList(new ExplainResponseNode( "FilterOperator", ImmutableMap.of("conditions", "and(=(balance, 10000), >(age, 30))"), - singletonList(tableScan.explain()))))), + singletonList(tableScan.explainNode()))))), explain.apply(plan)); } @@ -117,7 +117,7 @@ void can_explain_aggregations() { ImmutableMap.of( "aggregators", "[avg(balance)]", "groupBy", "[state]"), - singletonList(tableScan.explain()))), + singletonList(tableScan.explainNode()))), explain.apply(plan)); } @@ -135,7 +135,7 @@ void can_explain_rare_top_n() { "noOfResults", 10, "fields", "[state]", "groupBy", "[]"), - singletonList(tableScan.explain()))), + singletonList(tableScan.explainNode()))), explain.apply(plan)); } @@ -157,7 +157,7 @@ void can_explain_head() { "keepLast", false, "whileExpr", "and(=(balance, 10000), >(age, 30))", "number", 5), - singletonList(tableScan.explain()))), + singletonList(tableScan.explainNode()))), explain.apply(plan)); } @@ -182,7 +182,7 @@ void can_explain_window() { "age", ImmutableMap.of( "sortOrder", "ASC", "nullOrder", "NULL_FIRST")))), - singletonList(tableScan.explain()))), + singletonList(tableScan.explainNode()))), explain.apply(plan)); } @@ -262,12 +262,16 @@ public String toString() { } /** Used to ignore table scan which is duplicate but required for each operator test. */ - public ExplainResponseNode explain() { + public ExplainResponseNode explainNode() { return new ExplainResponseNode( "FakeTableScan", ImmutableMap.of("request", "Fake DSL request"), emptyList()); } + + public String explain() { + return "explain"; + } } } \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/storage/TableScanOperatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/storage/TableScanOperatorTest.java index 25869deee9..392b44e7ab 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/storage/TableScanOperatorTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/storage/TableScanOperatorTest.java @@ -26,6 +26,11 @@ class TableScanOperatorTest { private final TableScanOperator tableScan = new TableScanOperator() { + @Override + public String explain() { + return "explain"; + } + @Override public boolean hasNext() { return false; diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/utils/SystemIndexUtilsTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/utils/SystemIndexUtilsTest.java new file mode 100644 index 0000000000..7090510319 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/utils/SystemIndexUtilsTest.java @@ -0,0 +1,67 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.utils; + +import static com.amazon.opendistroforelasticsearch.sql.utils.SystemIndexUtils.isSystemIndex; +import static com.amazon.opendistroforelasticsearch.sql.utils.SystemIndexUtils.mappingTable; +import static com.amazon.opendistroforelasticsearch.sql.utils.SystemIndexUtils.systemTable; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class SystemIndexUtilsTest { + + @Test + void test_system_index() { + assertTrue(isSystemIndex("_ODFE_SYS_TABLE_META.ALL")); + assertFalse(isSystemIndex(".kibana")); + } + + @Test + void test_compose_mapping_table() { + assertEquals("_ODFE_SYS_TABLE_MAPPINGS.employee", mappingTable("employee")); + } + + @Test + void test_system_info_table() { + final SystemIndexUtils.SystemTable table = systemTable("_ODFE_SYS_TABLE_META.ALL"); + + assertTrue(table.isSystemInfoTable()); + assertFalse(table.isMetaInfoTable()); + assertEquals("ALL", table.getTableName()); + } + + @Test + void test_mapping_info_table() { + final SystemIndexUtils.SystemTable table = systemTable("_ODFE_SYS_TABLE_MAPPINGS.employee"); + + assertTrue(table.isMetaInfoTable()); + assertFalse(table.isSystemInfoTable()); + assertEquals("employee", table.getTableName()); + } + + @Test + void throw_exception_for_invalid_index() { + final IllegalStateException exception = + assertThrows(IllegalStateException.class, () -> systemTable("_ODFE_SYS_TABLE.employee")); + assertEquals("Invalid system index name: _ODFE_SYS_TABLE.employee", exception.getMessage()); + } +} \ No newline at end of file diff --git a/docs/experiment/ppl/cmd/search.rst b/docs/experiment/ppl/cmd/search.rst index d1a943f24f..5e3b825b5e 100644 --- a/docs/experiment/ppl/cmd/search.rst +++ b/docs/experiment/ppl/cmd/search.rst @@ -32,14 +32,14 @@ PPL query:: od> source=accounts; fetched rows / total rows = 4/4 - +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ - | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | - |------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------| - | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | - | 6 | Hattie | 671 Bristol Street | 5686 | M | Dante | Netagy | TN | 36 | hattiebond@netagy.com | Bond | - | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | - | 18 | Dale | 467 Hutchinson Court | 4180 | M | Orick | null | MD | 33 | daleadams@boink.com | Adams | - +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ + +------------------+-------------+----------------------+----------+--------+------------+-----------+------------+---------+-------+-----------------------+ + | account_number | firstname | address | gender | city | lastname | balance | employer | state | age | email | + |------------------+-------------+----------------------+----------+--------+------------+-----------+------------+---------+-------+-----------------------| + | 1 | Amber | 880 Holmes Lane | M | Brogan | Duke | 39225 | Pyrami | IL | 32 | amberduke@pyrami.com | + | 6 | Hattie | 671 Bristol Street | M | Dante | Bond | 5686 | Netagy | TN | 36 | hattiebond@netagy.com | + | 13 | Nanette | 789 Madison Street | F | Nogal | Bates | 32838 | Quility | VA | 28 | null | + | 18 | Dale | 467 Hutchinson Court | M | Orick | Adams | 4180 | null | MD | 33 | daleadams@boink.com | + +------------------+-------------+----------------------+----------+--------+------------+-----------+------------+---------+-------+-----------------------+ Example 2: Fetch data with condition ==================================== @@ -50,10 +50,10 @@ PPL query:: od> source=accounts account_number=1 or gender="F"; fetched rows / total rows = 2/2 - +------------------+-------------+--------------------+-----------+----------+--------+------------+---------+-------+----------------------+------------+ - | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | - |------------------+-------------+--------------------+-----------+----------+--------+------------+---------+-------+----------------------+------------| - | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | - | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | - +------------------+-------------+--------------------+-----------+----------+--------+------------+---------+-------+----------------------+------------+ + +------------------+-------------+--------------------+----------+--------+------------+-----------+------------+---------+-------+----------------------+ + | account_number | firstname | address | gender | city | lastname | balance | employer | state | age | email | + |------------------+-------------+--------------------+----------+--------+------------+-----------+------------+---------+-------+----------------------| + | 1 | Amber | 880 Holmes Lane | M | Brogan | Duke | 39225 | Pyrami | IL | 32 | amberduke@pyrami.com | + | 13 | Nanette | 789 Madison Street | F | Nogal | Bates | 32838 | Quility | VA | 28 | null | + +------------------+-------------+--------------------+----------+--------+------------+-----------+------------+---------+-------+----------------------+ diff --git a/docs/experiment/ppl/cmd/stats.rst b/docs/experiment/ppl/cmd/stats.rst index 52cc5f59f8..88e8d6ce94 100644 --- a/docs/experiment/ppl/cmd/stats.rst +++ b/docs/experiment/ppl/cmd/stats.rst @@ -79,12 +79,12 @@ PPL query:: od> source=accounts | stats avg(age) by gender; fetched rows / total rows = 2/2 - +----------+--------------------+ - | gender | avg(age) | - |----------+--------------------| - | F | 28.0 | - | M | 33.666666666666664 | - +----------+--------------------+ + +--------------------+----------+ + | avg(age) | gender | + |--------------------+----------| + | 28.0 | F | + | 33.666666666666664 | M | + +--------------------+----------+ Example 4: Calculate the average, sum and count of a field by group @@ -96,12 +96,12 @@ PPL query:: od> source=accounts | stats avg(age), sum(age), count() by gender; fetched rows / total rows = 2/2 - +----------+--------------------+------------+-----------+ - | gender | avg(age) | sum(age) | count() | - |----------+--------------------+------------+-----------| - | F | 28.0 | 28 | 1 | - | M | 33.666666666666664 | 101 | 3 | - +----------+--------------------+------------+-----------+ + +--------------------+------------+-----------+----------+ + | avg(age) | sum(age) | count() | gender | + |--------------------+------------+-----------+----------| + | 28.0 | 28 | 1 | F | + | 33.666666666666664 | 101 | 3 | M | + +--------------------+------------+-----------+----------+ Example 5: Calculate the maximum of a field =========================================== @@ -127,10 +127,10 @@ PPL query:: od> source=accounts | stats max(age), min(age) by gender; fetched rows / total rows = 2/2 - +----------+------------+------------+ - | gender | min(age) | max(age) | - |----------+------------+------------| - | F | 28 | 28 | - | M | 32 | 36 | - +----------+------------+------------+ + +------------+------------+----------+ + | max(age) | min(age) | gender | + |------------+------------+----------| + | 28 | 28 | F | + | 36 | 32 | M | + +------------+------------+----------+ diff --git a/docs/user/general/identifiers.rst b/docs/user/general/identifiers.rst index 7c7805726f..c3d2b70c66 100644 --- a/docs/user/general/identifiers.rst +++ b/docs/user/general/identifiers.rst @@ -40,14 +40,14 @@ Here are examples for using index pattern directly without quotes:: od> SELECT * FROM *cc*nt*; fetched rows / total rows = 4/4 - +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ - | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | - |------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------| - | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | - | 6 | Hattie | 671 Bristol Street | 5686 | M | Dante | Netagy | TN | 36 | hattiebond@netagy.com | Bond | - | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | - | 18 | Dale | 467 Hutchinson Court | 4180 | M | Orick | null | MD | 33 | daleadams@boink.com | Adams | - +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ + +------------------+-------------+----------------------+----------+--------+------------+-----------+------------+---------+-------+-----------------------+ + | account_number | firstname | address | gender | city | lastname | balance | employer | state | age | email | + |------------------+-------------+----------------------+----------+--------+------------+-----------+------------+---------+-------+-----------------------| + | 1 | Amber | 880 Holmes Lane | M | Brogan | Duke | 39225 | Pyrami | IL | 32 | amberduke@pyrami.com | + | 6 | Hattie | 671 Bristol Street | M | Dante | Bond | 5686 | Netagy | TN | 36 | hattiebond@netagy.com | + | 13 | Nanette | 789 Madison Street | F | Nogal | Bates | 32838 | Quility | VA | 28 | null | + | 18 | Dale | 467 Hutchinson Court | M | Orick | Adams | 4180 | null | MD | 33 | daleadams@boink.com | + +------------------+-------------+----------------------+----------+--------+------------+-----------+------------+---------+-------+-----------------------+ Delimited Identifiers @@ -76,14 +76,14 @@ Here are examples for quoting an index name by back ticks:: od> SELECT * FROM `accounts`; fetched rows / total rows = 4/4 - +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ - | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | - |------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------| - | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | - | 6 | Hattie | 671 Bristol Street | 5686 | M | Dante | Netagy | TN | 36 | hattiebond@netagy.com | Bond | - | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | - | 18 | Dale | 467 Hutchinson Court | 4180 | M | Orick | null | MD | 33 | daleadams@boink.com | Adams | - +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ + +------------------+-------------+----------------------+----------+--------+------------+-----------+------------+---------+-------+-----------------------+ + | account_number | firstname | address | gender | city | lastname | balance | employer | state | age | email | + |------------------+-------------+----------------------+----------+--------+------------+-----------+------------+---------+-------+-----------------------| + | 1 | Amber | 880 Holmes Lane | M | Brogan | Duke | 39225 | Pyrami | IL | 32 | amberduke@pyrami.com | + | 6 | Hattie | 671 Bristol Street | M | Dante | Bond | 5686 | Netagy | TN | 36 | hattiebond@netagy.com | + | 13 | Nanette | 789 Madison Street | F | Nogal | Bates | 32838 | Quility | VA | 28 | null | + | 18 | Dale | 467 Hutchinson Court | M | Orick | Adams | 4180 | null | MD | 33 | daleadams@boink.com | + +------------------+-------------+----------------------+----------+--------+------------+-----------+------------+---------+-------+-----------------------+ Case Sensitivity diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchClient.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchClient.java index 13a82ac751..eb1badb238 100644 --- a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchClient.java +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchClient.java @@ -19,6 +19,7 @@ import com.amazon.opendistroforelasticsearch.sql.elasticsearch.mapping.IndexMapping; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.request.ElasticsearchRequest; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.response.ElasticsearchResponse; +import java.util.List; import java.util.Map; /** @@ -27,6 +28,8 @@ */ public interface ElasticsearchClient { + String META_CLUSTER_NAME = "CLUSTER_NAME"; + /** * Fetch index mapping(s) according to index expression given. * @@ -43,6 +46,20 @@ public interface ElasticsearchClient { */ ElasticsearchResponse search(ElasticsearchRequest request); + /** + * Get the combination of the indices and the alias. + * + * @return the combination of the indices and the alias + */ + List indices(); + + /** + * Get meta info of the cluster. + * + * @return meta info of the cluster. + */ + Map meta(); + /** * Clean up resources related to the search request, for example scroll context. * diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchNodeClient.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchNodeClient.java index 65b6662ae4..955a3dd2b8 100644 --- a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchNodeClient.java +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchNodeClient.java @@ -20,16 +20,24 @@ import com.amazon.opendistroforelasticsearch.sql.elasticsearch.request.ElasticsearchRequest; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.response.ElasticsearchResponse; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.function.Function; import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.logging.log4j.ThreadContext; +import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.service.ClusterService; @@ -92,7 +100,9 @@ public Map getIndexMappings(String indexExpression) { } } - /** TODO: Scroll doesn't work for aggregation. Support aggregation later. */ + /** + * TODO: Scroll doesn't work for aggregation. Support aggregation later. + */ @Override public ElasticsearchResponse search(ElasticsearchRequest request) { return request.search( @@ -101,6 +111,37 @@ public ElasticsearchResponse search(ElasticsearchRequest request) { ); } + /** + * Get the combination of the indices and the alias. + * + * @return the combination of the indices and the alias + */ + @Override + public List indices() { + final GetIndexResponse indexResponse = client.admin().indices() + .prepareGetIndex() + .setLocal(true) + .get(); + final Stream aliasStream = + ImmutableList.copyOf(indexResponse.aliases().valuesIt()).stream() + .flatMap(Collection::stream).map(AliasMetadata::alias); + + return Stream.concat(Arrays.stream(indexResponse.getIndices()), aliasStream) + .collect(Collectors.toList()); + } + + /** + * Get meta info of the cluster. + * + * @return meta info of the cluster. + */ + @Override + public Map meta() { + final ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + builder.put(META_CLUSTER_NAME, clusterService.getClusterName().value()); + return builder.build(); + } + @Override public void cleanup(ElasticsearchRequest request) { request.clean(scrollId -> client.prepareClearScroll().addScrollId(scrollId).get()); diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchRestClient.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchRestClient.java index 7a068b38c0..53295293cd 100644 --- a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchRestClient.java +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchRestClient.java @@ -19,15 +19,26 @@ import com.amazon.opendistroforelasticsearch.sql.elasticsearch.mapping.IndexMapping; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.request.ElasticsearchRequest; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.response.ElasticsearchResponse; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import java.util.stream.Stream; import lombok.RequiredArgsConstructor; +import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsRequest; import org.elasticsearch.action.search.ClearScrollRequest; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.indices.GetIndexRequest; +import org.elasticsearch.client.indices.GetIndexResponse; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.GetMappingsResponse; +import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.common.settings.Settings; /** * Elasticsearch REST client to support standalone mode that runs entire engine from remote. @@ -74,6 +85,47 @@ public ElasticsearchResponse search(ElasticsearchRequest request) { ); } + /** + * Get the combination of the indices and the alias. + * + * @return the combination of the indices and the alias + */ + @Override + public List indices() { + try { + GetIndexResponse indexResponse = + client.indices().get(new GetIndexRequest(), RequestOptions.DEFAULT); + final Stream aliasStream = + ImmutableList.copyOf(indexResponse.getAliases().values()).stream() + .flatMap(Collection::stream).map(AliasMetadata::alias); + return Stream.concat(Arrays.stream(indexResponse.getIndices()), aliasStream) + .collect(Collectors.toList()); + } catch (IOException e) { + throw new IllegalStateException("Failed to get indices", e); + } + } + + /** + * Get meta info of the cluster. + * + * @return meta info of the cluster. + */ + @Override + public Map meta() { + try { + final ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + ClusterGetSettingsRequest request = new ClusterGetSettingsRequest(); + request.includeDefaults(true); + request.local(true); + final Settings defaultSettings = + client.cluster().getSettings(request, RequestOptions.DEFAULT).getDefaultSettings(); + builder.put(META_CLUSTER_NAME, defaultSettings.get("cluster.name", "elasticsearch")); + return builder.build(); + } catch (IOException e) { + throw new IllegalStateException("Failed to get cluster meta info", e); + } + } + @Override public void cleanup(ElasticsearchRequest request) { request.clean(scrollId -> { diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/type/ElasticsearchDataType.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/type/ElasticsearchDataType.java index e52e338dfe..a99bb6cf3c 100644 --- a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/type/ElasticsearchDataType.java +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/type/ElasticsearchDataType.java @@ -21,9 +21,11 @@ import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.UNKNOWN; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.google.common.collect.ImmutableMap; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import lombok.RequiredArgsConstructor; /** @@ -50,6 +52,15 @@ public enum ElasticsearchDataType implements ExprType { ES_BINARY(Arrays.asList(UNKNOWN), "binary"); + /** + * The mapping between Type and legacy JDBC type name. + */ + private static final Map LEGACY_TYPE_NAME_MAPPING = + new ImmutableMap.Builder() + .put(ES_TEXT, "text") + .put(ES_TEXT_KEYWORD, "text") + .build(); + /** * Parent of current type. */ @@ -68,4 +79,9 @@ public List getParent() { public String typeName() { return jdbcType; } + + @Override + public String legacyTypeName() { + return LEGACY_TYPE_NAME_MAPPING.getOrDefault(this, typeName()); + } } diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ElasticsearchExecutionEngine.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ElasticsearchExecutionEngine.java index 3f4cd2bf37..c93b8ee65f 100644 --- a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ElasticsearchExecutionEngine.java +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ElasticsearchExecutionEngine.java @@ -69,9 +69,7 @@ public void explain(PhysicalPlan plan, ResponseListener listene @Override public ExplainResponseNode visitTableScan(TableScanOperator node, Object context) { return explain(node, context, explainNode -> { - ElasticsearchIndexScan indexScan = (ElasticsearchIndexScan) node; - explainNode.setDescription(ImmutableMap.of( - "request", indexScan.getRequest().toString())); + explainNode.setDescription(ImmutableMap.of("request", node.explain())); }); } }; diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/system/ElasticsearchCatIndicesRequest.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/system/ElasticsearchCatIndicesRequest.java new file mode 100644 index 0000000000..da1429fd90 --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/system/ElasticsearchCatIndicesRequest.java @@ -0,0 +1,76 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.request.system; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.stringValue; +import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient.META_CLUSTER_NAME; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTupleValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import lombok.RequiredArgsConstructor; + +/** + * Cat indices request. + */ +@RequiredArgsConstructor +public class ElasticsearchCatIndicesRequest implements ElasticsearchSystemRequest { + + private static final String DEFAULT_TABLE_CAT = "elasticsearch"; + + private static final String DEFAULT_TABLE_TAPE = "BASE TABLE"; + + /** Elasticsearch client connection. */ + private final ElasticsearchClient client; + + /** + * search all the index in the data store. + * + * @return list of {@link ExprValue} + */ + @Override + public List search() { + List results = new ArrayList<>(); + final Map meta = client.meta(); + for (String index : client.indices()) { + results.add(row(index, clusterName(meta))); + } + return results; + } + + private ExprTupleValue row(String indexName, String clusterName) { + LinkedHashMap valueMap = new LinkedHashMap<>(); + valueMap.put("TABLE_CAT", stringValue(clusterName)); + valueMap.put("TABLE_NAME", stringValue(indexName)); + valueMap.put("TABLE_TYPE", stringValue(DEFAULT_TABLE_TAPE)); + return new ExprTupleValue(valueMap); + } + + private String clusterName(Map meta) { + return meta.getOrDefault(META_CLUSTER_NAME, DEFAULT_TABLE_CAT); + } + + @Override + public String toString() { + return "ElasticsearchCatIndicesRequest{}"; + } +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/system/ElasticsearchDescribeIndexRequest.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/system/ElasticsearchDescribeIndexRequest.java new file mode 100644 index 0000000000..9d8dce2be5 --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/system/ElasticsearchDescribeIndexRequest.java @@ -0,0 +1,155 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.request.system; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.stringValue; +import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient.META_CLUSTER_NAME; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTupleValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.mapping.IndexMapping; +import com.google.common.collect.ImmutableMap; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import lombok.RequiredArgsConstructor; + +/** + * Describe index meta data request. + */ +@RequiredArgsConstructor +public class ElasticsearchDescribeIndexRequest implements ElasticsearchSystemRequest { + + private static final String DEFAULT_TABLE_CAT = "elasticsearch"; + + private static final Integer DEFAULT_NUM_PREC_RADIX = 10; + + private static final Integer DEFAULT_NULLABLE = 2; + + private static final String DEFAULT_IS_AUTOINCREMENT = "NO"; + + /** + * Type mapping from Elasticsearch data type to expression type in our type system in query + * engine. TODO: geo, ip etc. + */ + private static final Map ES_TYPE_TO_EXPR_TYPE_MAPPING = + ImmutableMap.builder() + .put("text", ElasticsearchDataType.ES_TEXT) + .put("text_keyword", ElasticsearchDataType.ES_TEXT_KEYWORD) + .put("keyword", ExprCoreType.STRING) + .put("byte", ExprCoreType.BYTE) + .put("short", ExprCoreType.SHORT) + .put("integer", ExprCoreType.INTEGER) + .put("long", ExprCoreType.LONG) + .put("float", ExprCoreType.FLOAT) + .put("half_float", ExprCoreType.FLOAT) + .put("scaled_float", ExprCoreType.DOUBLE) + .put("double", ExprCoreType.DOUBLE) + .put("boolean", ExprCoreType.BOOLEAN) + .put("nested", ExprCoreType.ARRAY) + .put("object", ExprCoreType.STRUCT) + .put("date", ExprCoreType.TIMESTAMP) + .put("ip", ElasticsearchDataType.ES_IP) + .put("geo_point", ElasticsearchDataType.ES_GEO_POINT) + .put("binary", ElasticsearchDataType.ES_BINARY) + .build(); + + /** + * Elasticsearch client connection. + */ + private final ElasticsearchClient client; + + /** + * Elasticsearch index name. + */ + private final String indexName; + + /** + * search all the index in the data store. + * + * @return list of {@link ExprValue} + */ + @Override + public List search() { + List results = new ArrayList<>(); + Map meta = client.meta(); + int pos = 0; + for (Map.Entry entry : getFieldTypes().entrySet()) { + results.add( + row(entry.getKey(), entry.getValue().legacyTypeName().toLowerCase(), pos++, + clusterName(meta))); + } + return results; + } + + /** + * Get the mapping of field and type. + * + * @return mapping of field and type. + */ + public Map getFieldTypes() { + Map fieldTypes = new HashMap<>(); + Map indexMappings = client.getIndexMappings(indexName); + for (IndexMapping indexMapping : indexMappings.values()) { + fieldTypes.putAll(indexMapping.getAllFieldTypes(this::transformESTypeToExprType)); + } + return fieldTypes; + } + + private ExprType transformESTypeToExprType(String esType) { + return ES_TYPE_TO_EXPR_TYPE_MAPPING.getOrDefault(esType, ExprCoreType.UNKNOWN); + } + + private ExprTupleValue row(String fieldName, String fieldType, int position, String clusterName) { + LinkedHashMap valueMap = new LinkedHashMap<>(); + valueMap.put("TABLE_CAT", stringValue(clusterName)); + valueMap.put("TABLE_NAME", stringValue(indexName)); + valueMap.put("COLUMN_NAME", stringValue(fieldName)); + // todo + valueMap.put("TYPE_NAME", stringValue(fieldType)); + valueMap.put("NUM_PREC_RADIX", integerValue(DEFAULT_NUM_PREC_RADIX)); + valueMap.put("NULLABLE", integerValue(DEFAULT_NULLABLE)); + // There is no deterministic position of column in table + valueMap.put("ORDINAL_POSITION", integerValue(position)); + // TODO Defaulting to unknown, need to check this + valueMap.put("IS_NULLABLE", stringValue("")); + // Defaulting to "NO" + valueMap.put("IS_AUTOINCREMENT", stringValue(DEFAULT_IS_AUTOINCREMENT)); + // TODO Defaulting to unknown, need to check + valueMap.put("IS_GENERATEDCOLUMN", stringValue("")); + return new ExprTupleValue(valueMap); + } + + private String clusterName(Map meta) { + return meta.getOrDefault(META_CLUSTER_NAME, DEFAULT_TABLE_CAT); + } + + @Override + public String toString() { + return "ElasticsearchDescribeIndexRequest{" + + "indexName='" + indexName + '\'' + + '}'; + } +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/system/ElasticsearchSystemRequest.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/system/ElasticsearchSystemRequest.java new file mode 100644 index 0000000000..a51b66a8ea --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/system/ElasticsearchSystemRequest.java @@ -0,0 +1,34 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.request.system; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import java.util.List; + +/** + * Elasticsearch system request query against the system index. + */ +public interface ElasticsearchSystemRequest { + + /** + * Search. + * + * @return list of ExprValue. + */ + List search(); +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java index 75bcf9d86f..d9bab9b829 100644 --- a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java @@ -18,15 +18,13 @@ import com.amazon.opendistroforelasticsearch.sql.common.setting.Settings; import com.amazon.opendistroforelasticsearch.sql.common.utils.StringUtils; -import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; -import com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value.ElasticsearchExprValueFactory; -import com.amazon.opendistroforelasticsearch.sql.elasticsearch.mapping.IndexMapping; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.planner.logical.ElasticsearchLogicalIndexAgg; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.planner.logical.ElasticsearchLogicalIndexScan; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.planner.logical.ElasticsearchLogicalPlanOptimizerFactory; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.request.system.ElasticsearchDescribeIndexRequest; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.aggregation.AggregationQueryBuilder; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.filter.FilterQueryBuilder; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.script.sort.SortQueryBuilder; @@ -37,8 +35,6 @@ import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; import com.amazon.opendistroforelasticsearch.sql.storage.Table; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableMap; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -50,32 +46,6 @@ @RequiredArgsConstructor public class ElasticsearchIndex implements Table { - /** - * Type mapping from Elasticsearch data type to expression type in our type system in query - * engine. TODO: geo, ip etc. - */ - private static final Map ES_TYPE_TO_EXPR_TYPE_MAPPING = - ImmutableMap.builder() - .put("text", ElasticsearchDataType.ES_TEXT) - .put("text_keyword", ElasticsearchDataType.ES_TEXT_KEYWORD) - .put("keyword", ExprCoreType.STRING) - .put("byte", ExprCoreType.BYTE) - .put("short", ExprCoreType.SHORT) - .put("integer", ExprCoreType.INTEGER) - .put("long", ExprCoreType.LONG) - .put("float", ExprCoreType.FLOAT) - .put("half_float", ExprCoreType.FLOAT) - .put("scaled_float", ExprCoreType.DOUBLE) - .put("double", ExprCoreType.DOUBLE) - .put("boolean", ExprCoreType.BOOLEAN) - .put("nested", ExprCoreType.ARRAY) - .put("object", ExprCoreType.STRUCT) - .put("date", ExprCoreType.TIMESTAMP) - .put("ip", ElasticsearchDataType.ES_IP) - .put("geo_point", ElasticsearchDataType.ES_GEO_POINT) - .put("binary", ElasticsearchDataType.ES_BINARY) - .build(); - /** Elasticsearch client connection. */ private final ElasticsearchClient client; @@ -91,12 +61,7 @@ public class ElasticsearchIndex implements Table { */ @Override public Map getFieldTypes() { - Map fieldTypes = new HashMap<>(); - Map indexMappings = client.getIndexMappings(indexName); - for (IndexMapping indexMapping : indexMappings.values()) { - fieldTypes.putAll(indexMapping.getAllFieldTypes(this::transformESTypeToExprType)); - } - return fieldTypes; + return new ElasticsearchDescribeIndexRequest(client, indexName).getFieldTypes(); } /** @@ -120,10 +85,6 @@ public LogicalPlan optimize(LogicalPlan plan) { return ElasticsearchLogicalPlanOptimizerFactory.create().optimize(plan); } - private ExprType transformESTypeToExprType(String esType) { - return ES_TYPE_TO_EXPR_TYPE_MAPPING.getOrDefault(esType, ExprCoreType.UNKNOWN); - } - @VisibleForTesting @RequiredArgsConstructor public static class ElasticsearchDefaultImplementor diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexScan.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexScan.java index fe78e278a7..c909a90d28 100644 --- a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexScan.java +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexScan.java @@ -159,4 +159,8 @@ private boolean isBoolFilterQuery(QueryBuilder current) { return (current instanceof BoolQueryBuilder); } + @Override + public String explain() { + return getRequest().toString(); + } } diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchStorageEngine.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchStorageEngine.java index a82bc98cb8..8e0aabe4b1 100644 --- a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchStorageEngine.java +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchStorageEngine.java @@ -16,8 +16,11 @@ package com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage; +import static com.amazon.opendistroforelasticsearch.sql.utils.SystemIndexUtils.isSystemIndex; + import com.amazon.opendistroforelasticsearch.sql.common.setting.Settings; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.system.ElasticsearchSystemIndex; import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; import com.amazon.opendistroforelasticsearch.sql.storage.Table; import lombok.RequiredArgsConstructor; @@ -33,6 +36,10 @@ public class ElasticsearchStorageEngine implements StorageEngine { @Override public Table getTable(String name) { - return new ElasticsearchIndex(client, settings, name); + if (isSystemIndex(name)) { + return new ElasticsearchSystemIndex(client, name); + } else { + return new ElasticsearchIndex(client, settings, name); + } } } diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/system/ElasticsearchSystemIndex.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/system/ElasticsearchSystemIndex.java new file mode 100644 index 0000000000..ad703aeb3b --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/system/ElasticsearchSystemIndex.java @@ -0,0 +1,89 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.system; + +import static com.amazon.opendistroforelasticsearch.sql.utils.SystemIndexUtils.systemTable; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.request.system.ElasticsearchCatIndicesRequest; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.request.system.ElasticsearchDescribeIndexRequest; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.request.system.ElasticsearchSystemRequest; +import com.amazon.opendistroforelasticsearch.sql.planner.DefaultImplementor; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRelation; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import com.amazon.opendistroforelasticsearch.sql.storage.Table; +import com.amazon.opendistroforelasticsearch.sql.utils.SystemIndexUtils; +import com.google.common.annotations.VisibleForTesting; +import java.util.Map; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.tuple.Pair; + +/** + * Elasticsearch System Index Table Implementation. + */ +public class ElasticsearchSystemIndex implements Table { + /** + * System Index Name. + */ + private final Pair systemIndexBundle; + + public ElasticsearchSystemIndex( + ElasticsearchClient client, String indexName) { + this.systemIndexBundle = buildIndexBundle(client, indexName); + } + + @Override + public Map getFieldTypes() { + return systemIndexBundle.getLeft().getMapping(); + } + + @Override + public PhysicalPlan implement(LogicalPlan plan) { + return plan.accept(new ElasticsearchSystemIndexDefaultImplementor(), null); + } + + @VisibleForTesting + @RequiredArgsConstructor + public class ElasticsearchSystemIndexDefaultImplementor + extends DefaultImplementor { + + @Override + public PhysicalPlan visitRelation(LogicalRelation node, Object context) { + return new ElasticsearchSystemIndexScan(systemIndexBundle.getRight()); + } + } + + /** + * Constructor of ElasticsearchSystemIndexName. + * + * @param indexName index name; + */ + private Pair buildIndexBundle( + ElasticsearchClient client, String indexName) { + SystemIndexUtils.SystemTable systemTable = systemTable(indexName); + if (systemTable.isSystemInfoTable()) { + return Pair.of(ElasticsearchSystemIndexSchema.SYS_TABLE_TABLES, + new ElasticsearchCatIndicesRequest(client)); + } else { + return Pair.of(ElasticsearchSystemIndexSchema.SYS_TABLE_MAPPINGS, + new ElasticsearchDescribeIndexRequest(client, systemTable.getTableName())); + } + } +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/system/ElasticsearchSystemIndexScan.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/system/ElasticsearchSystemIndexScan.java new file mode 100644 index 0000000000..30f6f5f30b --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/system/ElasticsearchSystemIndexScan.java @@ -0,0 +1,64 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.system; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.request.system.ElasticsearchSystemRequest; +import com.amazon.opendistroforelasticsearch.sql.storage.TableScanOperator; +import java.util.Iterator; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Elasticsearch index scan operator. + */ +@RequiredArgsConstructor +@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = false) +@ToString(onlyExplicitlyIncluded = true) +public class ElasticsearchSystemIndexScan extends TableScanOperator { + /** + * Elasticsearch client. + */ + private final ElasticsearchSystemRequest request; + + /** + * Search response for current batch. + */ + private Iterator iterator; + + @Override + public void open() { + iterator = request.search().iterator(); + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public ExprValue next() { + return iterator.next(); + } + + @Override + public String explain() { + return request.toString(); + } +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/system/ElasticsearchSystemIndexSchema.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/system/ElasticsearchSystemIndexSchema.java new file mode 100644 index 0000000000..a7e56470d8 --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/system/ElasticsearchSystemIndexSchema.java @@ -0,0 +1,78 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.system; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.google.common.collect.ImmutableMap; +import java.util.LinkedHashMap; +import java.util.Map; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +/** + * Definition of the system table schema. + */ +@Getter +@RequiredArgsConstructor +public enum ElasticsearchSystemIndexSchema { + + SYS_TABLE_TABLES(new LinkedHashMap() {{ + put("TABLE_CAT", STRING); + put("TABLE_SCHEM", STRING); + put("TABLE_NAME", STRING); + put("TABLE_TYPE", STRING); + put("REMARKS", STRING); + put("TYPE_CAT", STRING); + put("TYPE_SCHEM", STRING); + put("TYPE_NAME", STRING); + put("SELF_REFERENCING_COL_NAME", STRING); + put("REF_GENERATION", STRING); + } + } + ), + SYS_TABLE_MAPPINGS(new ImmutableMap.Builder() + .put("TABLE_CAT", STRING) + .put("TABLE_SCHEM", STRING) + .put("TABLE_NAME", STRING) + .put("COLUMN_NAME", STRING) + .put("DATA_TYPE", STRING) + .put("TYPE_NAME", STRING) + .put("COLUMN_SIZE", STRING) + .put("BUFFER_LENGTH", STRING) + .put("DECIMAL_DIGITS", STRING) + .put("NUM_PREC_RADIX", STRING) + .put("NULLABLE", STRING) + .put("REMARKS", STRING) + .put("COLUMN_DEF", STRING) + .put("SQL_DATA_TYPE", STRING) + .put("SQL_DATETIME_SUB", STRING) + .put("CHAR_OCTET_LENGTH", STRING) + .put("ORDINAL_POSITION", STRING) + .put("IS_NULLABLE", STRING) + .put("SCOPE_CATALOG", STRING) + .put("SCOPE_SCHEMA", STRING) + .put("SCOPE_TABLE", STRING) + .put("SOURCE_DATA_TYPE", STRING) + .put("IS_AUTOINCREMENT", STRING) + .put("IS_GENERATEDCOLUMN", STRING) + .build()); + + private final Map mapping; +} diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchNodeClientTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchNodeClientTest.java index f5df45b26d..642c44e036 100644 --- a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchNodeClientTest.java +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchNodeClientTest.java @@ -16,6 +16,7 @@ package com.amazon.opendistroforelasticsearch.sql.elasticsearch.client; +import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient.META_CLUSTER_NAME; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -41,14 +42,19 @@ import com.google.common.io.Resources; import java.io.IOException; import java.net.URL; +import java.util.Arrays; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.lucene.search.TotalHits; +import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.search.ClearScrollRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; @@ -88,6 +94,9 @@ class ElasticsearchNodeClientTest { @Mock private ThreadContext threadContext; + @Mock + private GetIndexResponse indexResponse; + private ExprTupleValue exprTupleValue = ExprTupleValue.fromExprValueMap(ImmutableMap.of("id", new ExprIntegerValue(1))); @@ -250,6 +259,39 @@ void cleanupWithoutScrollId() { verify(nodeClient, never()).prepareClearScroll(); } + @Test + void getIndices() { + AliasMetadata aliasMetadata = mock(AliasMetadata.class); + ImmutableOpenMap.Builder> builder = ImmutableOpenMap.builder(); + builder.fPut("index",Arrays.asList(aliasMetadata)); + final ImmutableOpenMap> openMap = builder.build(); + when(aliasMetadata.alias()).thenReturn("index_alias"); + when(nodeClient.admin().indices() + .prepareGetIndex() + .setLocal(true) + .get()).thenReturn(indexResponse); + when(indexResponse.getIndices()).thenReturn(new String[] {"index"}); + when(indexResponse.aliases()).thenReturn(openMap); + + ElasticsearchNodeClient client = + new ElasticsearchNodeClient(mock(ClusterService.class), nodeClient); + final List indices = client.indices(); + assertEquals(2, indices.size()); + } + + @Test + void meta() { + ClusterName clusterName = mock(ClusterName.class); + ClusterService mockService = mock(ClusterService.class); + when(clusterName.value()).thenReturn("cluster-name"); + when(mockService.getClusterName()).thenReturn(clusterName); + + ElasticsearchNodeClient client = + new ElasticsearchNodeClient(mockService, nodeClient); + final Map meta = client.meta(); + assertEquals("cluster-name", meta.get(META_CLUSTER_NAME)); + } + private ElasticsearchNodeClient mockClient(String indexName, String mappings) { ClusterService clusterService = mockClusterService(indexName, mappings); return new ElasticsearchNodeClient(clusterService, nodeClient); diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchRestClientTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchRestClientTest.java index 66b30f74a7..60f0adeaef 100644 --- a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchRestClientTest.java +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchRestClientTest.java @@ -16,6 +16,7 @@ package com.amazon.opendistroforelasticsearch.sql.elasticsearch.client; +import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient.META_CLUSTER_NAME; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -40,15 +41,21 @@ import java.io.IOException; import java.net.URL; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.lucene.search.TotalHits; +import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsResponse; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.indices.GetIndexRequest; +import org.elasticsearch.client.indices.GetIndexResponse; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.GetMappingsResponse; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; @@ -77,6 +84,9 @@ class ElasticsearchRestClientTest { @Mock private SearchHit searchHit; + @Mock + private GetIndexResponse getIndexResponse; + private ExprTupleValue exprTupleValue = ExprTupleValue.fromExprValueMap(ImmutableMap.of("id", new ExprIntegerValue(1))); @@ -232,6 +242,43 @@ void cleanupWithIOException() throws IOException { assertThrows(IllegalStateException.class, () -> client.cleanup(request)); } + @Test + void getIndices() throws IOException { + when(restClient.indices().get(any(GetIndexRequest.class), any(RequestOptions.class))) + .thenReturn(getIndexResponse); + when(getIndexResponse.getIndices()).thenReturn(new String[] {"index"}); + + final List indices = client.indices(); + assertFalse(indices.isEmpty()); + } + + @Test + void getIndicesWithIOException() throws IOException { + when(restClient.indices().get(any(GetIndexRequest.class), any(RequestOptions.class))) + .thenThrow(new IOException()); + assertThrows(IllegalStateException.class, () -> client.indices()); + } + + @Test + void meta() throws IOException { + Settings defaultSettings = Settings.builder().build(); + ClusterGetSettingsResponse settingsResponse = mock(ClusterGetSettingsResponse.class); + when(restClient.cluster().getSettings(any(), any(RequestOptions.class))) + .thenReturn(settingsResponse); + when(settingsResponse.getDefaultSettings()).thenReturn(defaultSettings); + + final Map meta = client.meta(); + assertEquals("elasticsearch", meta.get(META_CLUSTER_NAME)); + } + + @Test + void metaWithIOException() throws IOException { + when(restClient.cluster().getSettings(any(), any(RequestOptions.class))) + .thenThrow(new IOException()); + + assertThrows(IllegalStateException.class, () -> client.meta()); + } + private Map mockFieldMappings(String indexName, String mappings) throws IOException { return ImmutableMap.of(indexName, IndexMetadata.fromXContent(createParser(mappings)).mapping()); diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/type/ElasticsearchDataTypeTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/type/ElasticsearchDataTypeTest.java index 7b04afbe56..9ad0e17fd6 100644 --- a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/type/ElasticsearchDataTypeTest.java +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/type/ElasticsearchDataTypeTest.java @@ -41,4 +41,10 @@ public void testTypeName() { assertEquals("string", ES_TEXT.typeName()); assertEquals("string", ES_TEXT_KEYWORD.typeName()); } + + @Test + public void legacyTypeName() { + assertEquals("text", ES_TEXT.legacyTypeName()); + assertEquals("text", ES_TEXT_KEYWORD.legacyTypeName()); + } } \ No newline at end of file diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ElasticsearchExecutionEngineTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ElasticsearchExecutionEngineTest.java index 828c8ea14a..8859464b13 100644 --- a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ElasticsearchExecutionEngineTest.java +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ElasticsearchExecutionEngineTest.java @@ -208,5 +208,10 @@ public ExprValue next() { public ExecutionEngine.Schema schema() { return schema; } + + @Override + public String explain() { + return "explain"; + } } } diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/system/ElasticsearchCatIndicesRequestTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/system/ElasticsearchCatIndicesRequestTest.java new file mode 100644 index 0000000000..588d92e204 --- /dev/null +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/system/ElasticsearchCatIndicesRequestTest.java @@ -0,0 +1,58 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.request.system; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.stringValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.hasEntry; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ElasticsearchCatIndicesRequestTest { + + @Mock + private ElasticsearchClient client; + + @Test + void testSearch() { + when(client.indices()).thenReturn(Arrays.asList("index")); + + final List results = new ElasticsearchCatIndicesRequest(client).search(); + assertEquals(1, results.size()); + assertThat(results.get(0).tupleValue(), anyOf( + hasEntry("TABLE_NAME", stringValue("index")) + )); + } + + @Test + void testToString() { + assertEquals("ElasticsearchCatIndicesRequest{}", + new ElasticsearchCatIndicesRequest(client).toString()); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/system/ElasticsearchDescribeIndexRequestTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/system/ElasticsearchDescribeIndexRequestTest.java new file mode 100644 index 0000000000..910fb055ca --- /dev/null +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/system/ElasticsearchDescribeIndexRequestTest.java @@ -0,0 +1,68 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.request.system; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.stringValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.hasEntry; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.mapping.IndexMapping; +import com.google.common.collect.ImmutableMap; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ElasticsearchDescribeIndexRequestTest { + + @Mock + private ElasticsearchClient client; + + @Test + void testSearch() { + when(client.getIndexMappings("index")) + .thenReturn( + ImmutableMap.of( + "test", + new IndexMapping( + ImmutableMap.builder() + .put("name", "keyword") + .build()))); + + final List results = new ElasticsearchDescribeIndexRequest(client, "index").search(); + assertEquals(1, results.size()); + assertThat(results.get(0).tupleValue(), anyOf( + hasEntry("TABLE_NAME", stringValue("index")), + hasEntry("COLUMN_NAME", stringValue("name")), + hasEntry("TYPE_NAME", stringValue("STRING")) + )); + } + + @Test + void testToString() { + assertEquals("ElasticsearchDescribeIndexRequest{indexName='index'}", + new ElasticsearchDescribeIndexRequest(client, "index").toString()); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchStorageEngineTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchStorageEngineTest.java index 51f62c2d29..8972f6ea2c 100644 --- a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchStorageEngineTest.java +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchStorageEngineTest.java @@ -16,11 +16,13 @@ package com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage; +import static com.amazon.opendistroforelasticsearch.sql.utils.SystemIndexUtils.TABLE_INFO; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.amazon.opendistroforelasticsearch.sql.common.setting.Settings; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; -import com.amazon.opendistroforelasticsearch.sql.elasticsearch.setting.ElasticsearchSettings; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.system.ElasticsearchSystemIndex; import com.amazon.opendistroforelasticsearch.sql.storage.Table; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -40,4 +42,12 @@ public void getTable() { Table table = engine.getTable("test"); assertNotNull(table); } + + @Test + public void getSystemTable() { + ElasticsearchStorageEngine engine = new ElasticsearchStorageEngine(client, settings); + Table table = engine.getTable(TABLE_INFO); + assertNotNull(table); + assertTrue(table instanceof ElasticsearchSystemIndex); + } } diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/system/ElasticsearchSystemIndexScanTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/system/ElasticsearchSystemIndexScanTest.java new file mode 100644 index 0000000000..77d0e167a0 --- /dev/null +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/system/ElasticsearchSystemIndexScanTest.java @@ -0,0 +1,55 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.system; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.stringValue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.request.system.ElasticsearchSystemRequest; +import java.util.Collections; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ElasticsearchSystemIndexScanTest { + + @Mock + private ElasticsearchSystemRequest request; + + @Test + public void queryData() { + when(request.search()).thenReturn(Collections.singletonList(stringValue("text"))); + final ElasticsearchSystemIndexScan systemIndexScan = new ElasticsearchSystemIndexScan(request); + + systemIndexScan.open(); + assertTrue(systemIndexScan.hasNext()); + assertEquals(stringValue("text"), systemIndexScan.next()); + } + + @Test + public void explain() { + when(request.toString()).thenReturn("request"); + final ElasticsearchSystemIndexScan systemIndexScan = new ElasticsearchSystemIndexScan(request); + + assertEquals("request", systemIndexScan.explain()); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/system/ElasticsearchSystemIndexTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/system/ElasticsearchSystemIndexTest.java new file mode 100644 index 0000000000..14814a5481 --- /dev/null +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/system/ElasticsearchSystemIndexTest.java @@ -0,0 +1,81 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.system; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.named; +import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.ref; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.project; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.relation; +import static com.amazon.opendistroforelasticsearch.sql.utils.SystemIndexUtils.TABLE_INFO; +import static com.amazon.opendistroforelasticsearch.sql.utils.SystemIndexUtils.mappingTable; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.hasEntry; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; +import com.amazon.opendistroforelasticsearch.sql.expression.NamedExpression; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.ProjectOperator; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ElasticsearchSystemIndexTest { + + @Mock + private ElasticsearchClient client; + + @Test + void testGetFieldTypesOfMetaTable() { + ElasticsearchSystemIndex systemIndex = new ElasticsearchSystemIndex(client, TABLE_INFO); + final Map fieldTypes = systemIndex.getFieldTypes(); + assertThat(fieldTypes, anyOf( + hasEntry("TABLE_CAT", STRING) + )); + } + + @Test + void testGetFieldTypesOfMappingTable() { + ElasticsearchSystemIndex systemIndex = new ElasticsearchSystemIndex(client, mappingTable( + "test_index")); + final Map fieldTypes = systemIndex.getFieldTypes(); + assertThat(fieldTypes, anyOf( + hasEntry("COLUMN_NAME", STRING) + )); + } + + @Test + void implement() { + ElasticsearchSystemIndex systemIndex = new ElasticsearchSystemIndex(client, TABLE_INFO); + NamedExpression projectExpr = named("TABLE_NAME", ref("TABLE_NAME", STRING)); + + final PhysicalPlan plan = systemIndex.implement( + project( + relation(TABLE_INFO), + projectExpr + )); + assertTrue(plan instanceof ProjectOperator); + assertTrue(plan.getChild().get(0) instanceof ElasticsearchSystemIndexScan); + } +} \ No newline at end of file diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MetaDataQueriesIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MetaDataQueriesIT.java index a15e70f6a7..8d4fcf690c 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MetaDataQueriesIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MetaDataQueriesIT.java @@ -33,7 +33,9 @@ import org.hamcrest.TypeSafeMatcher; import org.json.JSONArray; import org.json.JSONObject; +import org.junit.Ignore; import org.junit.Test; +import org.junit.jupiter.api.Disabled; /** @@ -300,6 +302,7 @@ public void describeSingleIndex() throws IOException { assertThat(row.get(5), not(equalTo(JSONObject.NULL))); } + @Ignore("Breaking change, the new engine will return alias instead of index name") @Test public void showSingleIndexAlias() throws IOException { client().performRequest(new Request("PUT", @@ -313,6 +316,7 @@ public void showSingleIndexAlias() throws IOException { expected.similar(actual)); } + @Ignore("Breaking change, the new engine will return alias instead of index name") @Test public void describeSingleIndexAlias() throws IOException { client().performRequest(new Request("PUT", diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/AdminIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/AdminIT.java new file mode 100644 index 0000000000..2b7101c2c7 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/AdminIT.java @@ -0,0 +1,93 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql; + +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.assertJsonEquals; +import static org.hamcrest.Matchers.equalTo; + +import com.amazon.opendistroforelasticsearch.sql.common.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.SQLIntegTestCase; +import com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants; +import com.amazon.opendistroforelasticsearch.sql.util.TestUtils; +import com.google.common.io.Resources; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Paths; +import org.elasticsearch.client.Request; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Test; + +public class AdminIT extends SQLIntegTestCase { + + @Override + public void init() throws Exception { + super.init(); + TestUtils.enableNewQueryEngine(client()); + loadIndex(Index.ACCOUNT); + } + + @Test + public void showSingleIndexAlias() throws IOException { + String alias = "acc"; + addAlias(TestsConstants.TEST_INDEX_ACCOUNT, alias); + JSONObject response = new JSONObject(executeQuery("SHOW TABLES LIKE acc", "jdbc")); + + /* + * Assumed indices of fields in dataRows based on "schema" output for SHOW given above: + * "TABLE_NAME" : 2 + */ + JSONArray row = response.getJSONArray("datarows").getJSONArray(0); + assertThat(row.get(2), equalTo(alias)); + } + + @Test + public void describeSingleIndexAlias() throws IOException { + String alias = "acc"; + addAlias(TestsConstants.TEST_INDEX_ACCOUNT, alias); + JSONObject response = new JSONObject(executeQuery("DESCRIBE TABLES LIKE acc", "jdbc")); + + /* + * Assumed indices of fields in dataRows based on "schema" output for DESCRIBE given above: + * "TABLE_NAME" : 2 + */ + JSONArray row = response.getJSONArray("datarows").getJSONArray(0); + assertThat(row.get(2), equalTo(alias)); + } + + @Test + public void explainShow() throws Exception { + String expected = loadFromFile("expectedOutput/sql/explain_show.json"); + + final String actual = explainQuery("SHOW TABLES LIKE %"); + assertJsonEquals( + expected, + explainQuery("SHOW TABLES LIKE %") + ); + } + + private void addAlias(String index, String alias) throws IOException { + client().performRequest(new Request("PUT", StringUtils.format("%s/_alias/%s", index, alias))); + } + + private String loadFromFile(String filename) throws Exception { + URI uri = Resources.getResource(filename).toURI(); + return new String(Files.readAllBytes(Paths.get(uri))); + } +} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.json index e7e27d0f22..5423154ad4 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.json @@ -2,7 +2,7 @@ "root": { "name": "ProjectOperator", "description": { - "fields": "[avg_age, city, state]" + "fields": "[avg_age, state, city]" }, "children": [ { diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push.json index 8a75c8b806..ca9840d44d 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push.json +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push.json @@ -2,7 +2,7 @@ "root": { "name": "ProjectOperator", "description": { - "fields": "[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]" + "fields": "[account_number, firstname, address, gender, city, lastname, balance, employer, state, age, email]" }, "children": [ { diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_push.json index 8b15ab493d..569a95f726 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_push.json +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_push.json @@ -2,7 +2,7 @@ "root": { "name": "ProjectOperator", "description": { - "fields": "[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]" + "fields": "[account_number, firstname, address, gender, city, lastname, balance, employer, state, age, email]" }, "children": [ { diff --git a/integ-test/src/test/resources/expectedOutput/sql/explain_show.json b/integ-test/src/test/resources/expectedOutput/sql/explain_show.json new file mode 100644 index 0000000000..34360352f8 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/sql/explain_show.json @@ -0,0 +1,25 @@ +{ + "root": { + "name": "ProjectOperator", + "description": { + "fields": "[TABLE_CAT, TABLE_SCHEM, TABLE_NAME, TABLE_TYPE, REMARKS, TYPE_CAT, TYPE_SCHEM, TYPE_NAME, SELF_REFERENCING_COL_NAME, REF_GENERATION]" + }, + "children": [ + { + "name": "FilterOperator", + "description": { + "conditions": "like(TABLE_NAME, \"%\")" + }, + "children": [ + { + "name": "ElasticsearchSystemIndexScan", + "description": { + "request": "ElasticsearchCatIndicesRequest{}" + }, + "children": [] + } + ] + } + ] + } +} \ No newline at end of file diff --git a/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatter.java b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatter.java index a7c798551d..b293e2db77 100644 --- a/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatter.java +++ b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JdbcResponseFormatter.java @@ -16,12 +16,6 @@ package com.amazon.opendistroforelasticsearch.sql.protocol.response.format; -import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.ARRAY; -import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; -import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT; -import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT; -import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT_KEYWORD; - import com.amazon.opendistroforelasticsearch.sql.common.antlr.SyntaxCheckException; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; import com.amazon.opendistroforelasticsearch.sql.exception.QueryEngineException; @@ -78,17 +72,7 @@ private Column fetchColumn(Schema.Column col) { * Return old type name to avoid breaking impact on client-side. */ private String convertToLegacyType(ExprType type) { - if (type == ES_TEXT || type == ES_TEXT_KEYWORD) { - return "text"; - } else if (type == STRING) { - return "keyword"; - } else if (type == STRUCT) { - return "object"; - } else if (type == ARRAY) { - return "nested"; - } else { - return type.typeName().toLowerCase(); - } + return type.legacyTypeName().toLowerCase(); } private Object[][] fetchDataRows(QueryResult response) { diff --git a/sql/src/main/antlr/OpenDistroSQLLexer.g4 b/sql/src/main/antlr/OpenDistroSQLLexer.g4 index 90514571b4..a14f0ad45a 100644 --- a/sql/src/main/antlr/OpenDistroSQLLexer.g4 +++ b/sql/src/main/antlr/OpenDistroSQLLexer.g4 @@ -51,6 +51,7 @@ BY: 'BY'; CASE: 'CASE'; CAST: 'CAST'; CROSS: 'CROSS'; +COLUMNS: 'COLUMNS'; DATETIME: 'DATETIME'; DELETE: 'DELETE'; DESC: 'DESC'; @@ -364,7 +365,6 @@ BACKTICK_QUOTE_ID: BQUOTA_STRING; // Fragments for Literal primitives - fragment EXPONENT_NUM_PART: 'E' [-+]? DEC_DIGIT+; fragment ID_LITERAL: [@*A-Z]+?[*A-Z_\-0-9]*; fragment DQUOTA_STRING: '"' ( '\\'. | '""' | ~('"'| '\\') )* '"'; @@ -374,8 +374,6 @@ fragment HEX_DIGIT: [0-9A-F]; fragment DEC_DIGIT: [0-9]; fragment BIT_STRING_L: 'B' '\'' [01]+ '\''; - - // Last tokens must generate Errors ERROR_RECOGNITION: . -> channel(ERRORCHANNEL); diff --git a/sql/src/main/antlr/OpenDistroSQLParser.g4 b/sql/src/main/antlr/OpenDistroSQLParser.g4 index 894b240011..5eb850dd50 100644 --- a/sql/src/main/antlr/OpenDistroSQLParser.g4 +++ b/sql/src/main/antlr/OpenDistroSQLParser.g4 @@ -39,7 +39,7 @@ root // Only SELECT sqlStatement - : dmlStatement + : dmlStatement | adminStatement ; dmlStatement @@ -55,6 +55,34 @@ selectStatement : querySpecification #simpleSelect ; +adminStatement + : showStatement + | describeStatement + ; + +showStatement + : SHOW TABLES tableFilter? + ; + +describeStatement + : DESCRIBE TABLES tableFilter columnFilter? + ; + +columnFilter + : COLUMNS LIKE showDescribePattern + ; + +tableFilter + : LIKE showDescribePattern + ; + +showDescribePattern + : oldID=compatibleID | stringLiteral + ; + +compatibleID + : (MODULE | ID)+? + ; // Select Statement's Details diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilder.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilder.java index db7e5b1374..3d05ced57b 100644 --- a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilder.java +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilder.java @@ -16,6 +16,7 @@ package com.amazon.opendistroforelasticsearch.sql.sql.parser; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.qualifiedName; import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.FromClauseContext; import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.HavingClauseContext; import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.SelectClauseContext; @@ -24,10 +25,13 @@ import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.TableAsRelationContext; import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.WhereClauseContext; import static com.amazon.opendistroforelasticsearch.sql.sql.parser.ParserUtils.getTextInQuery; +import static com.amazon.opendistroforelasticsearch.sql.utils.SystemIndexUtils.TABLE_INFO; +import static com.amazon.opendistroforelasticsearch.sql.utils.SystemIndexUtils.mappingTable; import static java.util.Collections.emptyList; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Alias; import com.amazon.opendistroforelasticsearch.sql.ast.expression.AllFields; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Function; import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; import com.amazon.opendistroforelasticsearch.sql.ast.tree.Filter; import com.amazon.opendistroforelasticsearch.sql.ast.tree.Project; @@ -37,10 +41,12 @@ import com.amazon.opendistroforelasticsearch.sql.ast.tree.Values; import com.amazon.opendistroforelasticsearch.sql.common.antlr.SyntaxCheckException; import com.amazon.opendistroforelasticsearch.sql.common.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser; import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.QuerySpecificationContext; import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParserBaseVisitor; import com.amazon.opendistroforelasticsearch.sql.sql.parser.context.ParsingContext; import com.google.common.collect.ImmutableList; +import java.util.Collections; import java.util.Optional; import lombok.RequiredArgsConstructor; import org.antlr.v4.runtime.tree.ParseTree; @@ -64,6 +70,26 @@ public class AstBuilder extends OpenDistroSQLParserBaseVisitor { */ private final String query; + @Override + public UnresolvedPlan visitShowStatement(OpenDistroSQLParser.ShowStatementContext ctx) { + final UnresolvedExpression tableFilter = visitAstExpression(ctx.tableFilter()); + return new Project(Collections.singletonList(AllFields.of())) + .attach(new Filter(tableFilter).attach(new Relation(qualifiedName(TABLE_INFO)))); + } + + @Override + public UnresolvedPlan visitDescribeStatement(OpenDistroSQLParser.DescribeStatementContext ctx) { + final Function tableFilter = (Function) visitAstExpression(ctx.tableFilter()); + final String tableName = tableFilter.getFuncArgs().get(1).toString(); + final Relation table = new Relation(qualifiedName(mappingTable(tableName.toString()))); + if (ctx.columnFilter() == null) { + return new Project(Collections.singletonList(AllFields.of())).attach(table); + } else { + return new Project(Collections.singletonList(AllFields.of())) + .attach(new Filter(visitAstExpression(ctx.columnFilter())).attach(table)); + } + } + @Override public UnresolvedPlan visitQuerySpecification(QuerySpecificationContext queryContext) { context.push(); diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java index f60c91302b..4f45da4fc1 100644 --- a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java @@ -16,6 +16,8 @@ package com.amazon.opendistroforelasticsearch.sql.sql.parser; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.qualifiedName; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.stringLiteral; import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.IS_NOT_NULL; import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.IS_NULL; import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.LIKE; @@ -61,6 +63,7 @@ import com.amazon.opendistroforelasticsearch.sql.ast.expression.When; import com.amazon.opendistroforelasticsearch.sql.ast.expression.WindowFunction; import com.amazon.opendistroforelasticsearch.sql.common.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser; import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.AndExpressionContext; import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.ColumnNameContext; import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.IdentContext; @@ -130,6 +133,30 @@ public UnresolvedExpression visitScalarFunctionCall(ScalarFunctionCallContext ct ); } + @Override + public UnresolvedExpression visitTableFilter(OpenDistroSQLParser.TableFilterContext ctx) { + return new Function( + LIKE.getName().getFunctionName(), + Arrays.asList(qualifiedName("TABLE_NAME"), visit(ctx.showDescribePattern()))); + } + + @Override + public UnresolvedExpression visitColumnFilter(OpenDistroSQLParser.ColumnFilterContext ctx) { + return new Function( + LIKE.getName().getFunctionName(), + Arrays.asList(qualifiedName("COLUMN_NAME"), visit(ctx.showDescribePattern()))); + } + + @Override + public UnresolvedExpression visitShowDescribePattern( + OpenDistroSQLParser.ShowDescribePatternContext ctx) { + if (ctx.compatibleID() != null) { + return stringLiteral(ctx.compatibleID().getText()); + } else { + return visit(ctx.stringLiteral()); + } + } + @Override public UnresolvedExpression visitWindowFunction(WindowFunctionContext ctx) { OverClauseContext overClause = ctx.overClause(); @@ -229,6 +256,11 @@ public UnresolvedExpression visitBoolean(BooleanContext ctx) { return AstDSL.booleanLiteral(Boolean.valueOf(ctx.getText())); } + @Override + public UnresolvedExpression visitStringLiteral(OpenDistroSQLParser.StringLiteralContext ctx) { + return AstDSL.stringLiteral(StringUtils.unquoteText(ctx.getText())); + } + @Override public UnresolvedExpression visitNullLiteral(NullLiteralContext ctx) { return AstDSL.nullLiteral(); diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilderTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilderTest.java index da92cce889..db581cc42f 100644 --- a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilderTest.java +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilderTest.java @@ -33,6 +33,8 @@ import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.sort; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.stringLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.values; +import static com.amazon.opendistroforelasticsearch.sql.utils.SystemIndexUtils.TABLE_INFO; +import static com.amazon.opendistroforelasticsearch.sql.utils.SystemIndexUtils.mappingTable; import static java.util.Collections.emptyList; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -524,6 +526,106 @@ public void can_build_from_subquery() { ); } + @Test + public void can_build_show_all_tables() { + assertEquals( + project( + filter( + relation(TABLE_INFO), + function("like", qualifiedName("TABLE_NAME"), stringLiteral("%")) + ), + AllFields.of() + ), + buildAST("SHOW TABLES LIKE '%'") + ); + } + + @Test + public void can_build_show_selected_tables() { + assertEquals( + project( + filter( + relation(TABLE_INFO), + function("like", qualifiedName("TABLE_NAME"), stringLiteral("a_c%")) + ), + AllFields.of() + ), + buildAST("SHOW TABLES LIKE 'a_c%'") + ); + } + + /** + * Todo, ideally the identifier (%) couldn't be used in LIKE operator, only the string literal + * is allowed. + */ + @Test + public void show_compatible_with_old_engine_syntax() { + assertEquals( + project( + filter( + relation(TABLE_INFO), + function("like", qualifiedName("TABLE_NAME"), stringLiteral("%")) + ), + AllFields.of() + ), + buildAST("SHOW TABLES LIKE %") + ); + } + + @Test + public void describe_compatible_with_old_engine_syntax() { + assertEquals( + project( + relation(mappingTable("a_c%")), + AllFields.of() + ), + buildAST("DESCRIBE TABLES LIKE a_c%") + ); + } + + @Test + public void can_build_describe_selected_tables() { + assertEquals( + project( + relation(mappingTable("a_c%")), + AllFields.of() + ), + buildAST("DESCRIBE TABLES LIKE 'a_c%'") + ); + } + + @Test + public void can_build_describe_selected_tables_field_filter() { + assertEquals( + project( + filter( + relation(mappingTable("a_c%")), + function("like", qualifiedName("COLUMN_NAME"), stringLiteral("name%")) + ), + AllFields.of() + ), + buildAST("DESCRIBE TABLES LIKE 'a_c%' COLUMNS LIKE 'name%'") + ); + } + + /** + * Todo, ideally the identifier (%) couldn't be used in LIKE operator, only the string literal + * is allowed. + */ + @Test + public void describe_and_column_compatible_with_old_engine_syntax() { + assertEquals( + project( + filter( + relation(mappingTable("a_c%")), + function("like", qualifiedName("COLUMN_NAME"), stringLiteral("name%")) + ), + AllFields.of() + ), + buildAST("DESCRIBE TABLES LIKE a_c% COLUMNS LIKE name%") + ); + } + @Test public void can_build_alias_by_keywords() { assertEquals(