diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index b696b8c794..a28d13a4e5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -506,6 +506,10 @@ public enum Index { "nestedType", getNestedTypeIndexMapping(), "src/test/resources/nested_objects.json"), + NESTED_WITHOUT_ARRAYS(TestsConstants.TEST_INDEX_NESTED_TYPE_WITHOUT_ARRAYS, + "nestedTypeWithoutArrays", + getNestedTypeIndexMapping(), + "src/test/resources/nested_objects_without_arrays.json"), NESTED_WITH_QUOTES(TestsConstants.TEST_INDEX_NESTED_WITH_QUOTES, "nestedType", getNestedTypeIndexMapping(), @@ -594,9 +598,9 @@ public enum Index { "wildcard", getMappingFile("wildcard_index_mappings.json"), "src/test/resources/wildcard.json"), - MULTI_NESTED(TestsConstants.TEST_INDEX_MULTI_NESTED, + MULTI_NESTED(TestsConstants.TEST_INDEX_MULTI_NESTED_TYPE, "multi_nested", - getMappingFile("indexDefinitions/multi_nested.json"), + getMappingFile("multi_nested.json"), "src/test/resources/multi_nested_objects.json"); private final String name; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index 271511c19e..6695c2feab 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -31,6 +31,8 @@ public class TestsConstants { public final static String TEST_INDEX_LOCATION = TEST_INDEX + "_location"; public final static String TEST_INDEX_LOCATION2 = TEST_INDEX + "_location2"; public final static String TEST_INDEX_NESTED_TYPE = TEST_INDEX + "_nested_type"; + public final static String TEST_INDEX_NESTED_TYPE_WITHOUT_ARRAYS = + TEST_INDEX + "_nested_type_without_arrays"; public final static String TEST_INDEX_NESTED_SIMPLE = TEST_INDEX + "_nested_simple"; public final static String TEST_INDEX_NESTED_WITH_QUOTES = TEST_INDEX + "_nested_type_with_quotes"; @@ -55,7 +57,7 @@ public class TestsConstants { public final static String TEST_INDEX_NULL_MISSING = TEST_INDEX + "_null_missing"; public final static String TEST_INDEX_CALCS = TEST_INDEX + "_calcs"; public final static String TEST_INDEX_WILDCARD = TEST_INDEX + "_wildcard"; - public final static String TEST_INDEX_MULTI_NESTED= TEST_INDEX + "_multi_nested"; + public final static String TEST_INDEX_MULTI_NESTED_TYPE = TEST_INDEX + "_multi_nested"; public final static String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; public final static String TS_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java index a635c89857..f1e7899c64 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java @@ -5,24 +5,107 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_MULTI_NESTED_TYPE; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_TYPE; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_TYPE_WITHOUT_ARRAYS; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import java.io.IOException; +import java.util.List; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; -import java.io.IOException; - -import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_MULTI_NESTED; - public class NestedIT extends SQLIntegTestCase { @Override public void init() throws IOException { loadIndex(Index.MULTI_NESTED); + loadIndex(Index.NESTED); + loadIndex(Index.NESTED_WITHOUT_ARRAYS); + } + + @Test + public void nested_function_with_array_of_nested_field_test() { + String query = "SELECT nested(message.info), nested(comment.data) FROM " + TEST_INDEX_NESTED_TYPE; + JSONObject result = executeJdbcRequest(query); + + assertEquals(5, result.getInt("total")); + verifyDataRows(result, + rows("a", "ab"), + rows("b", "aa"), + rows("c", "aa"), + rows(new JSONArray(List.of("c","a")), "ab"), + rows(new JSONArray(List.of("zz")), new JSONArray(List.of("aa", "bb")))); + } + + @Test + public void nested_function_in_select_test() { + String query = "SELECT nested(message.info), nested(comment.data), " + + "nested(message.dayOfWeek) FROM " + + TEST_INDEX_NESTED_TYPE_WITHOUT_ARRAYS; + JSONObject result = executeJdbcRequest(query); + + assertEquals(5, result.getInt("total")); + verifySchema(result, + schema("nested(message.info)", null, "keyword"), + schema("nested(comment.data)", null, "keyword"), + schema("nested(message.dayOfWeek)", null, "long")); + verifyDataRows(result, + rows("a", "ab", 1), + rows("b", "aa", 2), + rows("c", "aa", 1), + rows("c", "ab", 4), + rows("zz", "bb", 6)); } + // Has to be tested with JSON format when https://github.com/opensearch-project/sql/issues/1317 + // gets resolved @Test - public void nested_string_subfield_test() { - String query = "SELECT nested(message.info) FROM " + TEST_INDEX_MULTI_NESTED; + public void nested_function_in_an_aggregate_function_in_select_test() { + String query = "SELECT sum(nested(message.dayOfWeek)) FROM " + + TEST_INDEX_NESTED_TYPE_WITHOUT_ARRAYS; JSONObject result = executeJdbcRequest(query); - assertEquals(6, result.getInt("total")); + verifyDataRows(result, rows(14)); + } + + @Test + public void nested_function_with_arrays_in_an_aggregate_function_in_select_test() { + String query = "SELECT sum(nested(message.dayOfWeek)) FROM " + + TEST_INDEX_NESTED_TYPE; + JSONObject result = executeJdbcRequest(query); + verifyDataRows(result, rows(19)); + } + + @Test + public void nested_function_in_a_function_in_select_test() { + String query = "SELECT upper(nested(message.info)) FROM " + + TEST_INDEX_NESTED_TYPE_WITHOUT_ARRAYS; + JSONObject result = executeJdbcRequest(query); + + verifyDataRows(result, + rows("A"), + rows("B"), + rows("C"), + rows("C"), + rows("ZZ")); + } + + + @Test + public void nested_function_with_array_of_multi_nested_field_test() { + String query = "SELECT nested(message.author.name) FROM " + TEST_INDEX_MULTI_NESTED_TYPE; + JSONObject result = executeJdbcRequest(query); + + assertEquals(5, result.getInt("total")); + verifyDataRows(result, + rows("e"), + rows("f"), + rows("g"), + rows(new JSONArray(List.of("h", "p"))), + rows(new JSONArray(List.of("yy")))); } } diff --git a/integ-test/src/test/resources/multi_nested_objects.json b/integ-test/src/test/resources/multi_nested_objects.json index bdee031357..f5838fe90b 100644 --- a/integ-test/src/test/resources/multi_nested_objects.json +++ b/integ-test/src/test/resources/multi_nested_objects.json @@ -1,9 +1,9 @@ {"index":{"_id":"1"}} -{"message":[{"info":"a","author":{"name": "e", "address": {"street": "bc", "number": 1}},"dayOfWeek":1}]} +{"message":{"info":"a","author":{"name": "e", "address": {"street": "bc", "number": 1}},"dayOfWeek":1}} {"index":{"_id":"2"}} -{"message":[{"info":"b","author":{"name": "f", "address": {"street": "ab", "number": 2}},"dayOfWeek":2}]} +{"message":{"info":"b","author":{"name": "f", "address": {"street": "ab", "number": 2}},"dayOfWeek":2}} {"index":{"_id":"3"}} -{"message":[{"info":"c","author":{"name": "g", "address": {"street": "sk", "number": 3}},"dayOfWeek":1}]} +{"message":{"info":"c","author":{"name": "g", "address": {"street": "sk", "number": 3}},"dayOfWeek":1}} {"index":{"_id":"4"}} {"message":[{"info":"d","author":{"name": "h", "address": {"street": "mb", "number": 4}},"dayOfWeek":4},{"info":"i","author":{"name": "p", "address": {"street": "on", "number": 5}},"dayOfWeek":5}]} {"index":{"_id":"5"}} diff --git a/integ-test/src/test/resources/nested_objects_without_arrays.json b/integ-test/src/test/resources/nested_objects_without_arrays.json new file mode 100644 index 0000000000..626e63e079 --- /dev/null +++ b/integ-test/src/test/resources/nested_objects_without_arrays.json @@ -0,0 +1,10 @@ +{"index":{"_id":"1"}} +{"message":{"info":"a","author":"e","dayOfWeek":1},"comment":{"data":"ab","likes":3},"myNum":1,"someField":"b"} +{"index":{"_id":"2"}} +{"message":{"info":"b","author":"f","dayOfWeek":2},"comment":{"data":"aa","likes":2},"myNum":2,"someField":"a"} +{"index":{"_id":"3"}} +{"message":{"info":"c","author":"g","dayOfWeek":1},"comment":{"data":"aa","likes":3},"myNum":3,"someField":"a"} +{"index":{"_id":"4"}} +{"message":{"info":"c","author":"h","dayOfWeek":4},"comment":{"data":"ab","likes":1},"myNum":4,"someField":"b"} +{"index":{"_id":"5"}} +{"message": {"info":"zz","author":"zz","dayOfWeek":6},"comment":{"data":"bb","likes":10},"myNum":3,"someField":"a"} diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 73e4cc0f17..52877ca7c4 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -317,7 +317,7 @@ nestedFunction ; nestedField - : ID DOT ID (DOT ID)* + : nestedIdent DOT nestedIdent (DOT nestedIdent)* ; highlightFunction @@ -624,6 +624,14 @@ ident | scalarFunctionName ; + +nestedIdent + : ID + | BACKTICK_QUOTE_ID + | keywordsCanBeId + | scalarFunctionName + ; + keywordsCanBeId : FULL | FIELD | D | T | TS // OD SQL and ODBC special diff --git a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java index 16118d2a32..1d59c8aee5 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -509,6 +509,24 @@ public void can_parse_wildcard_query_relevance_function() { + "boost=1.5, case_insensitive=true, rewrite=\"scoring_boolean\")")); } + @Test + public void can_parse_nested_function() { + assertNotNull( + parser.parse("SELECT NESTED(FIELD.DAYOFWEEK) FROM TEST")); + assertNotNull( + parser.parse("SELECT SUM(NESTED(FIELD.SUBFIELD)) FROM TEST")); + assertNotNull( + parser.parse("SELECT COUNT(*) FROM TEST GROUP BY NESTED(FIELD.SUBFIELD)")); + } + + @Test + public void can_not_parse_nested_function_without_dot() { + assertThrows(SyntaxCheckException.class, + () -> parser.parse("SELECT NESTED(FIELD) FROM TEST")); + assertThrows(SyntaxCheckException.class, + () -> parser.parse("SELECT COUNT(*) FROM TEST GROUP BY NESTED(FIELD)")); + } + @Test public void describe_request_accepts_only_quoted_string_literals() { assertAll(