From 564ab60ba1225dcdf30b129fbef5685979a3ab8a Mon Sep 17 00:00:00 2001 From: Aparajita Pandey <37636092+aparajita31pandey@users.noreply.github.com> Date: Tue, 8 Oct 2024 20:05:56 +0530 Subject: [PATCH 1/2] Resolve Alias Issues in Legacy SQL with Filters (#2960) * Fix: Pagination of index aliases is not supported Signed-off-by: Aparajita Pandey * fix: remove extra debug log Signed-off-by: Aparajita Pandey * Integration testadded Signed-off-by: Aparajita Pandey Signed-off-by: Aparajita Pandey * rollback change Signed-off-by: Aparajita Pandey Signed-off-by: Aparajita Pandey * Integration TestAdded Signed-off-by: Aparajita Pandey Signed-off-by: Aparajita Pandey * Integration TestAdded Signed-off-by: Aparajita Pandey Signed-off-by: Aparajita Pandey * SpotlessCheck Signed-off-by: Aparajita Pandey Signed-off-by: Aparajita Pandey --------- Signed-off-by: Aparajita Pandey --- .../sql/legacy/SQLIntegTestCase.java | 6 +++ .../org/opensearch/sql/sql/PaginationIT.java | 50 +++++++++++++++++++ .../executor/format/SelectResultSet.java | 8 ++- 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index c6d15a305d..eed4e29c9c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -462,6 +462,12 @@ protected String makeRequest(String query, int fetch_size) { return String.format("{ \"fetch_size\": \"%s\", \"query\": \"%s\" }", fetch_size, query); } + protected String makeRequest(String query, int fetch_size, String filterQuery) { + return String.format( + "{ \"fetch_size\": \"%s\", \"query\": \"%s\", \"filter\" : %s }", + fetch_size, query, filterQuery); + } + protected String makeFetchLessRequest(String query) { return String.format("{\n" + " \"query\": \"%s\"\n" + "}", query); } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationIT.java index 49ef7c583e..fbe1e378e2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationIT.java @@ -7,6 +7,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.opensearch.sql.legacy.TestUtils.getResponseBody; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CALCS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ONLINE; @@ -18,6 +19,7 @@ import org.junit.Test; import org.opensearch.client.Request; import org.opensearch.client.RequestOptions; +import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.SQLIntegTestCase; @@ -215,4 +217,52 @@ public void testQueryWithoutFrom() { assertEquals(1, response.getInt("total")); assertEquals(1, response.getJSONArray("datarows").getJSONArray(0).getInt(0)); } + + @Test + public void testAlias() throws Exception { + String indexName = Index.ONLINE.getName(); + String aliasName = "alias_ONLINE"; + String filterQuery = "{\n" + " \"term\": {\n" + " \"107\": 72 \n" + " }\n" + "}"; + + // Execute the SQL query with filter + String selectQuery = "SELECT * FROM " + TEST_INDEX_ONLINE; + JSONObject initialResponse = + new JSONObject(executeFetchQuery(selectQuery, 10, "jdbc", filterQuery)); + assertEquals(initialResponse.getInt("size"), 10); + + // Create an alias + String createAliasQuery = + String.format( + "{ \"actions\": [ { \"add\": { \"index\": \"%s\", \"alias\": \"%s\" } } ] }", + indexName, aliasName); + Request createAliasRequest = new Request("POST", "/_aliases"); + createAliasRequest.setJsonEntity(createAliasQuery); + JSONObject aliasResponse = new JSONObject(executeRequest(createAliasRequest)); + + // Assert that alias creation was acknowledged + assertTrue(aliasResponse.getBoolean("acknowledged")); + + // Query using the alias + String aliasSelectQuery = String.format("SELECT * FROM %s", aliasName); + JSONObject aliasQueryResponse = new JSONObject(executeFetchQuery(aliasSelectQuery, 4, "jdbc")); + assertEquals(4, aliasQueryResponse.getInt("size")); + + // Query using the alias with filter + JSONObject aliasFilteredResponse = + new JSONObject(executeFetchQuery(aliasSelectQuery, 4, "jdbc", filterQuery)); + assertEquals(aliasFilteredResponse.getInt("size"), 4); + } + + private String executeFetchQuery(String query, int fetchSize, String requestType, String filter) + throws IOException { + String endpoint = "/_plugins/_sql?format=" + requestType; + String requestBody = makeRequest(query, fetchSize, filter); + + Request sqlRequest = new Request("POST", endpoint); + sqlRequest.setJsonEntity(requestBody); + + Response response = client().performRequest(sqlRequest); + String responseString = getResponseBody(response, true); + return responseString; + } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java index 9d1862023c..bc5c1fb162 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java @@ -26,6 +26,8 @@ import java.util.stream.StreamSupport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.opensearch.action.admin.indices.alias.get.GetAliasesResponse; import org.opensearch.action.admin.indices.mapping.get.GetFieldMappingsRequest; import org.opensearch.action.admin.indices.mapping.get.GetFieldMappingsResponse; import org.opensearch.action.search.ClearScrollResponse; @@ -164,7 +166,11 @@ private void populateResultSetFromDefaultCursor(DefaultCursor cursor) { private void loadFromEsState(Query query) { String indexName = fetchIndexName(query); String[] fieldNames = fetchFieldsAsArray(query); - + GetAliasesResponse getAliasesResponse = + client.admin().indices().getAliases(new GetAliasesRequest(indexName)).actionGet(); + if (getAliasesResponse != null && !getAliasesResponse.getAliases().isEmpty()) { + indexName = getAliasesResponse.getAliases().keySet().iterator().next(); + } // Reset boolean in the case of JOIN query where multiple calls to loadFromEsState() are made selectAll = isSimpleQuerySelectAll(query) || isJoinQuerySelectAll(query, fieldNames); From ac8678c60c3e98c7e3791b2abb2b0e71c643256c Mon Sep 17 00:00:00 2001 From: Tomoyuki MORITA Date: Tue, 8 Oct 2024 08:50:48 -0700 Subject: [PATCH 2/2] Disable join types in validators (#3056) * Disable join types in validators Signed-off-by: Tomoyuki Morita * Removed methods from SQLQueryValidationVisitor due to grammar file change Signed-off-by: Tomoyuki Morita --------- Signed-off-by: Tomoyuki Morita --- .../S3GlueGrammarElementValidator.java | 10 +++++++ .../validator/SQLQueryValidationVisitor.java | 28 ------------------ .../SecurityLakeGrammarElementValidator.java | 10 +++++++ .../validator/SQLQueryValidatorTest.java | 29 ++++++++----------- 4 files changed, 32 insertions(+), 45 deletions(-) diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java index e7a0ce1b36..9ed1fd9e9e 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/S3GlueGrammarElementValidator.java @@ -9,20 +9,25 @@ import static org.opensearch.sql.spark.validator.GrammarElement.CLUSTER_BY; import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_FUNCTION; import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_VIEW; +import static org.opensearch.sql.spark.validator.GrammarElement.CROSS_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_FUNCTION; import static org.opensearch.sql.spark.validator.GrammarElement.DISTRIBUTE_BY; import static org.opensearch.sql.spark.validator.GrammarElement.DROP_FUNCTION; import static org.opensearch.sql.spark.validator.GrammarElement.DROP_VIEW; import static org.opensearch.sql.spark.validator.GrammarElement.FILE; +import static org.opensearch.sql.spark.validator.GrammarElement.FULL_OUTER_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.HINTS; import static org.opensearch.sql.spark.validator.GrammarElement.INLINE_TABLE; import static org.opensearch.sql.spark.validator.GrammarElement.INSERT; +import static org.opensearch.sql.spark.validator.GrammarElement.LEFT_ANTI_JOIN; +import static org.opensearch.sql.spark.validator.GrammarElement.LEFT_SEMI_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.LOAD; import static org.opensearch.sql.spark.validator.GrammarElement.MANAGE_RESOURCE; import static org.opensearch.sql.spark.validator.GrammarElement.MISC_FUNCTIONS; import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_FUNCTION; import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_RESOURCE; import static org.opensearch.sql.spark.validator.GrammarElement.RESET; +import static org.opensearch.sql.spark.validator.GrammarElement.RIGHT_OUTER_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.SET; import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_FUNCTIONS; import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_VIEWS; @@ -50,6 +55,11 @@ public class S3GlueGrammarElementValidator extends DenyListGrammarElementValidat HINTS, INLINE_TABLE, FILE, + CROSS_JOIN, + LEFT_SEMI_JOIN, + RIGHT_OUTER_JOIN, + FULL_OUTER_JOIN, + LEFT_ANTI_JOIN, TABLESAMPLE, TABLE_VALUED_FUNCTION, TRANSFORM, diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java index 9ec0fb0109..d50503418e 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java @@ -9,15 +9,12 @@ import org.opensearch.sql.spark.antlr.parser.SqlBaseParser; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AddTableColumnsContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AddTablePartitionContext; -import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AlterClusterByContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AlterTableAlterColumnContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AlterViewQueryContext; -import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AlterViewSchemaBindingContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AnalyzeContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.AnalyzeTablesContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CacheTableContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ClearCacheContext; -import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.ClusterBySpecContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CreateNamespaceContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CreateTableContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.CreateTableLikeContext; @@ -81,7 +78,6 @@ import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.TransformClauseContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.TruncateTableContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.UncacheTableContext; -import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.UnsetNamespacePropertiesContext; import org.opensearch.sql.spark.antlr.parser.SqlBaseParserBaseVisitor; /** This visitor validate grammar using GrammarElementValidator */ @@ -101,12 +97,6 @@ public Void visitSetNamespaceProperties(SetNamespacePropertiesContext ctx) { return super.visitSetNamespaceProperties(ctx); } - @Override - public Void visitUnsetNamespaceProperties(UnsetNamespacePropertiesContext ctx) { - validateAllowed(GrammarElement.ALTER_NAMESPACE); - return super.visitUnsetNamespaceProperties(ctx); - } - @Override public Void visitAddTableColumns(AddTableColumnsContext ctx) { validateAllowed(GrammarElement.ALTER_NAMESPACE); @@ -173,12 +163,6 @@ public Void visitRecoverPartitions(RecoverPartitionsContext ctx) { return super.visitRecoverPartitions(ctx); } - @Override - public Void visitAlterClusterBy(AlterClusterByContext ctx) { - validateAllowed(GrammarElement.ALTER_NAMESPACE); - return super.visitAlterClusterBy(ctx); - } - @Override public Void visitSetNamespaceLocation(SetNamespaceLocationContext ctx) { validateAllowed(GrammarElement.ALTER_NAMESPACE); @@ -191,12 +175,6 @@ public Void visitAlterViewQuery(AlterViewQueryContext ctx) { return super.visitAlterViewQuery(ctx); } - @Override - public Void visitAlterViewSchemaBinding(AlterViewSchemaBindingContext ctx) { - validateAllowed(GrammarElement.ALTER_VIEW); - return super.visitAlterViewSchemaBinding(ctx); - } - @Override public Void visitRenameTable(RenameTableContext ctx) { if (ctx.VIEW() != null) { @@ -337,12 +315,6 @@ public Void visitCtes(CtesContext ctx) { return super.visitCtes(ctx); } - @Override - public Void visitClusterBySpec(ClusterBySpecContext ctx) { - validateAllowed(GrammarElement.CLUSTER_BY); - return super.visitClusterBySpec(ctx); - } - @Override public Void visitQueryOrganization(QueryOrganizationContext ctx) { if (ctx.CLUSTER() != null) { diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java index ca8f2b5bdd..7dd2b0ee89 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/validator/SecurityLakeGrammarElementValidator.java @@ -14,6 +14,7 @@ import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_FUNCTION; import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_NAMESPACE; import static org.opensearch.sql.spark.validator.GrammarElement.CREATE_VIEW; +import static org.opensearch.sql.spark.validator.GrammarElement.CROSS_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.CSV_FUNCTIONS; import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_FUNCTION; import static org.opensearch.sql.spark.validator.GrammarElement.DESCRIBE_NAMESPACE; @@ -24,9 +25,12 @@ import static org.opensearch.sql.spark.validator.GrammarElement.DROP_NAMESPACE; import static org.opensearch.sql.spark.validator.GrammarElement.DROP_VIEW; import static org.opensearch.sql.spark.validator.GrammarElement.FILE; +import static org.opensearch.sql.spark.validator.GrammarElement.FULL_OUTER_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.HINTS; import static org.opensearch.sql.spark.validator.GrammarElement.INLINE_TABLE; import static org.opensearch.sql.spark.validator.GrammarElement.INSERT; +import static org.opensearch.sql.spark.validator.GrammarElement.LEFT_ANTI_JOIN; +import static org.opensearch.sql.spark.validator.GrammarElement.LEFT_SEMI_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.LOAD; import static org.opensearch.sql.spark.validator.GrammarElement.MANAGE_RESOURCE; import static org.opensearch.sql.spark.validator.GrammarElement.MISC_FUNCTIONS; @@ -35,6 +39,7 @@ import static org.opensearch.sql.spark.validator.GrammarElement.REFRESH_TABLE; import static org.opensearch.sql.spark.validator.GrammarElement.REPAIR_TABLE; import static org.opensearch.sql.spark.validator.GrammarElement.RESET; +import static org.opensearch.sql.spark.validator.GrammarElement.RIGHT_OUTER_JOIN; import static org.opensearch.sql.spark.validator.GrammarElement.SET; import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_COLUMNS; import static org.opensearch.sql.spark.validator.GrammarElement.SHOW_CREATE_TABLE; @@ -76,6 +81,11 @@ public class SecurityLakeGrammarElementValidator extends DenyListGrammarElementV HINTS, INLINE_TABLE, FILE, + CROSS_JOIN, + LEFT_SEMI_JOIN, + RIGHT_OUTER_JOIN, + FULL_OUTER_JOIN, + LEFT_ANTI_JOIN, TABLESAMPLE, TABLE_VALUED_FUNCTION, TRANSFORM, diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java index 6726b56994..695a083809 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java @@ -34,11 +34,9 @@ private enum TestElement { // DDL Statements ALTER_DATABASE( "ALTER DATABASE inventory SET DBPROPERTIES ('Edit-date' = '01/01/2001');", - "ALTER DATABASE dbx.tab1 UNSET PROPERTIES ('winner');", "ALTER DATABASE dbx.tab1 SET LOCATION '/path/to/part/ways';"), ALTER_TABLE( "ALTER TABLE default.StudentInfo PARTITION (age='10') RENAME TO PARTITION (age='15');", - "ALTER TABLE dbx.tab1 UNSET TBLPROPERTIES ('winner');", "ALTER TABLE StudentInfo ADD columns (LastName string, DOB timestamp);", "ALTER TABLE StudentInfo ADD IF NOT EXISTS PARTITION (age=18);", "ALTER TABLE StudentInfo RENAME COLUMN name TO FirstName;", @@ -50,12 +48,10 @@ private enum TestElement { "ALTER TABLE StudentInfo DROP IF EXISTS PARTITION (age=18);", "ALTER TABLE dbx.tab1 PARTITION (a='1', b='2') SET LOCATION '/path/to/part/ways';", "ALTER TABLE dbx.tab1 RECOVER PARTITIONS;", - "ALTER TABLE dbx.tab1 CLUSTER BY NONE;", "ALTER TABLE dbx.tab1 SET LOCATION '/path/to/part/ways';"), ALTER_VIEW( "ALTER VIEW tempdb1.v1 RENAME TO tempdb1.v2;", - "ALTER VIEW tempdb1.v2 AS SELECT * FROM tempdb1.v1;", - "ALTER VIEW tempdb1.v2 WITH SCHEMA BINDING"), + "ALTER VIEW tempdb1.v2 AS SELECT * FROM tempdb1.v1;"), CREATE_DATABASE("CREATE DATABASE IF NOT EXISTS customer_db;\n"), CREATE_FUNCTION("CREATE FUNCTION simple_udf AS 'SimpleUdf' USING JAR '/tmp/SimpleUdf.jar';"), CREATE_TABLE( @@ -94,8 +90,7 @@ private enum TestElement { EXPLAIN("EXPLAIN SELECT * FROM my_table;"), COMMON_TABLE_EXPRESSION( "WITH cte AS (SELECT * FROM my_table WHERE age > 30) SELECT * FROM cte;"), - CLUSTER_BY_CLAUSE( - "SELECT * FROM my_table CLUSTER BY age;", "ALTER TABLE testTable CLUSTER BY (age);"), + CLUSTER_BY_CLAUSE("SELECT * FROM my_table CLUSTER BY age;"), DISTRIBUTE_BY_CLAUSE("SELECT * FROM my_table DISTRIBUTE BY name;"), GROUP_BY_CLAUSE("SELECT name, count(*) FROM my_table GROUP BY name;"), HAVING_CLAUSE("SELECT name, count(*) FROM my_table GROUP BY name HAVING count(*) > 1;"), @@ -370,12 +365,12 @@ void testS3glueQueries() { v.ng(TestElement.INLINE_TABLE); v.ng(TestElement.FILE); v.ok(TestElement.INNER_JOIN); - v.ok(TestElement.CROSS_JOIN); + v.ng(TestElement.CROSS_JOIN); v.ok(TestElement.LEFT_OUTER_JOIN); - v.ok(TestElement.LEFT_SEMI_JOIN); - v.ok(TestElement.RIGHT_OUTER_JOIN); - v.ok(TestElement.FULL_OUTER_JOIN); - v.ok(TestElement.LEFT_ANTI_JOIN); + v.ng(TestElement.LEFT_SEMI_JOIN); + v.ng(TestElement.RIGHT_OUTER_JOIN); + v.ng(TestElement.FULL_OUTER_JOIN); + v.ng(TestElement.LEFT_ANTI_JOIN); v.ok(TestElement.LIKE_PREDICATE); v.ok(TestElement.LIMIT_CLAUSE); v.ok(TestElement.OFFSET_CLAUSE); @@ -487,12 +482,12 @@ void testSecurityLakeQueries() { v.ng(TestElement.INLINE_TABLE); v.ng(TestElement.FILE); v.ok(TestElement.INNER_JOIN); - v.ok(TestElement.CROSS_JOIN); + v.ng(TestElement.CROSS_JOIN); v.ok(TestElement.LEFT_OUTER_JOIN); - v.ok(TestElement.LEFT_SEMI_JOIN); - v.ok(TestElement.RIGHT_OUTER_JOIN); - v.ok(TestElement.FULL_OUTER_JOIN); - v.ok(TestElement.LEFT_ANTI_JOIN); + v.ng(TestElement.LEFT_SEMI_JOIN); + v.ng(TestElement.RIGHT_OUTER_JOIN); + v.ng(TestElement.FULL_OUTER_JOIN); + v.ng(TestElement.LEFT_ANTI_JOIN); v.ok(TestElement.LIKE_PREDICATE); v.ok(TestElement.LIMIT_CLAUSE); v.ok(TestElement.OFFSET_CLAUSE);