diff --git a/docs/user/admin/settings.rst b/docs/user/admin/settings.rst index 94c164c5f4..d1dd0ffaa3 100644 --- a/docs/user/admin/settings.rst +++ b/docs/user/admin/settings.rst @@ -121,164 +121,6 @@ Result set:: } } -opensearch.sql.query.analysis.enabled -===================================== - -Description ------------ - -You can disable query analyzer to bypass strict syntactic and semantic analysis. - -1. The default value is true. -2. This setting is node scope. -3. This setting can be updated dynamically. - - -Example -------- - -You can update the setting with a new value like this. - -SQL query:: - - >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_sql/settings -d '{ - "transient" : { - "opensearch.sql.query.analysis.enabled" : "false" - } - }' - -Result set:: - - { - "acknowledged" : true, - "persistent" : { }, - "transient" : { - "opensearch" : { - "sql" : { - "query" : { - "analysis" : { - "enabled" : "false" - } - } - } - } - } - } - -opensearch.sql.query.analysis.semantic.suggestion -================================================= - -Description ------------ - -You can enable query analyzer to suggest correct field names for quick fix. - -1. The default value is false. -2. This setting is node scope. -3. This setting can be updated dynamically. - - -Example 1 ---------- - -You can update the setting with a new value like this. - -SQL query:: - - >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_sql/settings -d '{ - "transient" : { - "opensearch.sql.query.analysis.semantic.suggestion" : "true" - } - }' - -Result set:: - - { - "acknowledged" : true, - "persistent" : { }, - "transient" : { - "opensearch" : { - "sql" : { - "query" : { - "analysis" : { - "semantic" : { - "suggestion" : "true" - } - } - } - } - } - } - } - -Example 2 ---------- - -Query result after the setting updated is like: - -SQL query:: - - >> curl -H 'Content-Type: application/json' -X POST localhost:9200/_plugins/_sql -d '{ - "query" : "SELECT first FROM accounts" - }' - -Result set:: - - { - "error" : { - "reason" : "Invalid SQL query", - "details" : "Field [first] cannot be found or used here. Did you mean [firstname]?", - "type" : "SemanticAnalysisException" - }, - "status" : 400 - } - -opensearch.sql.query.analysis.semantic.threshold -================================================ - -Description ------------ - -Because query analysis needs to build semantic context in memory, index with large number of field would be skipped. You can update it to apply analysis to smaller or larger index as needed. - -1. The default value is 200. -2. This setting is node scope. -3. This setting can be updated dynamically. - - -Example -------- - -You can update the setting with a new value like this. - -SQL query:: - - >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_sql/settings -d '{ - "transient" : { - "opensearch.sql.query.analysis.semantic.threshold" : "50" - } - }' - -Result set:: - - { - "acknowledged" : true, - "persistent" : { }, - "transient" : { - "opensearch" : { - "sql" : { - "query" : { - "analysis" : { - "semantic" : { - "threshold" : "50" - } - } - } - } - } - } - } - opensearch.sql.query.response.format ==================================== diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/admin/PluginSettingIT.java b/integ-test/src/test/java/org/opensearch/sql/doctest/admin/PluginSettingIT.java index bae91029fa..38c22057af 100644 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/admin/PluginSettingIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/doctest/admin/PluginSettingIT.java @@ -39,9 +39,6 @@ import static org.opensearch.sql.legacy.plugin.SqlSettings.CURSOR_ENABLED; import static org.opensearch.sql.legacy.plugin.SqlSettings.CURSOR_FETCH_SIZE; import static org.opensearch.sql.legacy.plugin.SqlSettings.CURSOR_KEEPALIVE; -import static org.opensearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_ENABLED; -import static org.opensearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_SUGGESTION; -import static org.opensearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_THRESHOLD; import static org.opensearch.sql.legacy.plugin.SqlSettings.QUERY_RESPONSE_FORMAT; import static org.opensearch.sql.legacy.plugin.SqlSettings.QUERY_SLOWLOG; import static org.opensearch.sql.legacy.plugin.SqlSettings.SQL_ENABLED; @@ -87,35 +84,6 @@ public void slowLogSetting() { ); } - @Section(3) - public void queryAnalysisEnabledSetting() { - docSetting( - QUERY_ANALYSIS_ENABLED, - "You can disable query analyzer to bypass strict syntactic and semantic analysis.", - false - ); - } - - @Section(4) - public void semanticSuggestionSetting() { - docSetting( - QUERY_ANALYSIS_SEMANTIC_SUGGESTION, - "You can enable query analyzer to suggest correct field names for quick fix.", - true, - "SELECT first FROM accounts" - ); - } - - @Section(5) - public void semanticAnalysisThresholdSetting() { - docSetting( - QUERY_ANALYSIS_SEMANTIC_THRESHOLD, - "Because query analysis needs to build semantic context in memory, index with large number of field " + - "would be skipped. You can update it to apply analysis to smaller or larger index as needed.", - 50 - ); - } - @Section(6) public void responseFormatSetting() { docSetting( diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java index 703483bb56..c9e1a9f82c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java @@ -59,11 +59,11 @@ public void noGroupKeySingleFuncOverAggWithoutAliasShouldPass() { @Test public void noGroupKeyMaxAddMinShouldPass() { JSONObject response = executeJdbcRequest(String.format( - "SELECT MAX(age) + MIN(age) as add " + + "SELECT MAX(age) + MIN(age) as addValue " + "FROM %s", Index.ACCOUNT.getName())); - verifySchema(response, schema("add", "add", "long")); + verifySchema(response, schema("MAX(age) + MIN(age)", "addValue", "long")); verifyDataRows(response, rows(60)); } @@ -111,14 +111,14 @@ public void hasGroupKeyAvgOnIntegerShouldPass() { @Test public void hasGroupKeyMaxAddMinShouldPass() { JSONObject response = executeJdbcRequest(String.format( - "SELECT gender, MAX(age) + MIN(age) as add " + + "SELECT gender, MAX(age) + MIN(age) as addValue " + "FROM %s " + "GROUP BY gender", Index.ACCOUNT.getName())); verifySchema(response, schema("gender", null, "text"), - schema("add", "add", "long")); + schema("MAX(age) + MIN(age)", "addValue", "long")); verifyDataRows(response, rows("m", 60), rows("f", 60)); @@ -156,14 +156,14 @@ public void noGroupKeyLogMaxAddMinShouldPass() { @Test public void hasGroupKeyLogMaxAddMinShouldPass() { JSONObject response = executeJdbcRequest(String.format( - "SELECT gender, Log(MAX(age) + MIN(age)) as log " + + "SELECT gender, Log(MAX(age) + MIN(age)) as logValue " + "FROM %s " + "GROUP BY gender", Index.ACCOUNT.getName())); verifySchema(response, schema("gender", null, "text"), - schema("log", "log", "double")); + schema("Log(MAX(age) + MIN(age))", "logValue", "double")); verifyDataRows(response, rows("m", 4.0943445622221d), rows("f", 4.0943445622221d)); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java index f5f6f93bb4..4d1451f8c5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java @@ -78,7 +78,7 @@ public void countTest() throws IOException { Assert.assertThat(getIntAggregationValue(result, "COUNT(*)", "value"), equalTo(1000)); } - @Test + @Ignore("The distinct is not supported in new engine") public void countDistinctTest() { JSONObject response = executeJdbcRequest( String.format("SELECT COUNT(distinct gender) FROM %s", TEST_INDEX_ACCOUNT)); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/PluginIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/PluginIT.java index acbce65332..d5eae6ec06 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/PluginIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/PluginIT.java @@ -237,17 +237,16 @@ public void sqlPersistentOnlySettingTest() throws IOException { /** * Both transient and persistent settings are applied for same settings. - * This is similiar to _cluster/settings behavior + * This is similar to _cluster/settings behavior */ @Test public void sqlCombinedSettingTest() throws IOException { String settings = "{" + " \"transient\": {" + - " \"opensearch.sql.metrics.rollingwindow\": \"3700\"," + - " \"opensearch.sql.query.analysis.semantic.suggestion\" : \"false\"" + + " \"opensearch.sql.metrics.rollingwindow\": \"3700\"" + " }," + " \"persistent\": {" + - " \"opensearch.sql.query.analysis.semantic.suggestion\" : \"true\"" + + " \"opensearch.sql.query.slowlog\" : \"2\"" + " }" + "}"; JSONObject actual = updateViaSQLSettingsAPI(settings); @@ -257,11 +256,7 @@ public void sqlCombinedSettingTest() throws IOException { " \"opensearch\" : {" + " \"sql\" : {" + " \"query\" : {" + - " \"analysis\" : {" + - " \"semantic\" : {" + - " \"suggestion\" : \"true\"" + - " }" + - " }" + + " \"slowlog\" : \"2\"" + " }" + " }" + " }" + @@ -271,13 +266,6 @@ public void sqlCombinedSettingTest() throws IOException { " \"sql\" : {" + " \"metrics\" : {" + " \"rollingwindow\" : \"3700\"" + - " }," + - " \"query\" : {" + - " \"analysis\" : {" + - " \"semantic\" : {" + - " \"suggestion\" : \"false\"" + - " }" + - " }" + " }" + " }" + " }" + @@ -300,7 +288,7 @@ public void ignoreNonSQLSettingsTest() throws IOException { " \"search.max_keep_alive\": \"24h\"" + " }," + " \"persistent\": {" + - " \"opensearch.sql.query.analysis.semantic.suggestion\": \"true\"," + + " \"opensearch.sql.query.slowlog\": \"2\"," + " \"opensearch.alerting.metrics.rollingwindow\": \"3700\"," + " \"thread_pool.analyze.queue_size\": \"16\"" + " }" + @@ -312,11 +300,7 @@ public void ignoreNonSQLSettingsTest() throws IOException { " \"opensearch\" : {" + " \"sql\" : {" + " \"query\" : {" + - " \"analysis\" : {" + - " \"semantic\" : {" + - " \"suggestion\" : \"true\"" + - " }" + - " }" + + " \"slowlog\" : \"2\"" + " }" + " }" + " }" + diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/QueryAnalysisIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/QueryAnalysisIT.java index 55095f7ead..ee6f4da43e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/QueryAnalysisIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/QueryAnalysisIT.java @@ -32,9 +32,6 @@ import static org.opensearch.rest.RestStatus.BAD_REQUEST; import static org.opensearch.rest.RestStatus.OK; import static org.opensearch.rest.RestStatus.SERVICE_UNAVAILABLE; -import static org.opensearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_ENABLED; -import static org.opensearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_SUGGESTION; -import static org.opensearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_THRESHOLD; import java.io.IOException; import org.junit.Assert; @@ -46,7 +43,6 @@ import org.opensearch.sql.legacy.antlr.semantic.SemanticAnalysisException; import org.opensearch.sql.legacy.antlr.syntax.SyntaxAnalysisException; import org.opensearch.sql.legacy.exception.SqlFeatureNotImplementedException; -import org.opensearch.sql.legacy.exception.SqlParseException; import org.opensearch.sql.legacy.utils.StringUtils; /** @@ -71,57 +67,6 @@ public void unsupportedOperatorShouldThrowSyntaxException() { ); } - @Test - public void unsupportedOperatorShouldSkipAnalysisAndThrowOtherExceptionIfAnalyzerDisabled() { - runWithClusterSetting( - new ClusterSetting("transient", QUERY_ANALYSIS_ENABLED, "false"), - () -> queryShouldThrowException( - "SELECT * FROM opensearch-sql_test_index_bank WHERE age <=> 1", - SqlParseException.class - ) - ); - } - - @Test - public void suggestionForWrongFieldNameShouldBeProvidedIfSuggestionEnabled() { - runWithClusterSetting( - new ClusterSetting("transient", QUERY_ANALYSIS_SEMANTIC_SUGGESTION, "true"), - () -> queryShouldThrowSemanticException( - "SELECT * FROM opensearch-sql_test_index_bank b WHERE a.balance = 1000", - "Field [a.balance] cannot be found or used here.", - "Did you mean [b.balance]?" - ) - ); - } - - @Test - public void wrongFieldNameShouldPassIfIndexMappingIsVeryLarge() { - runWithClusterSetting( - new ClusterSetting("transient", QUERY_ANALYSIS_SEMANTIC_THRESHOLD, "5"), - () -> queryShouldPassAnalysis( - "SELECT * FROM opensearch-sql_test_index_bank WHERE age123 = 1") - ); - } - - /* - @Test - public void useNewAddedFieldShouldPass() throws Exception { - // 1.Make sure new add fields not there originally - String query = "SELECT salary FROM opensearch-sql_test_index_bank WHERE education = 'PhD'"; - queryShouldThrowSemanticException(query, "Field [education] cannot be found or used here."); - - // 2.Index an document with fields not present in mapping previously - String docWithNewFields = "{\"account_number\":12345,\"education\":\"PhD\",\"salary\": \"10000\"}"; - IndexResponse resp = client().index(new IndexRequest().index("opensearch-sql_test_index_bank"). - source(docWithNewFields, JSON)).get(); - - Assert.assertEquals(RestStatus.CREATED, resp.status()); - - // 3.Same query should pass - executeQuery(query); - } - */ - @Test public void nonExistingFieldNameShouldThrowSemanticException() { queryShouldThrowSemanticException( diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLFunctionsIT.java index 327c5c6e5d..c3e5029c53 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLFunctionsIT.java @@ -284,34 +284,37 @@ public void castIntFieldToDoubleWithAliasOrderByTest() throws IOException { public void castIntFieldToFloatWithoutAliasJdbcFormatGroupByTest() { JSONObject response = executeJdbcRequest( "SELECT CAST(balance AS FLOAT) FROM " + - TestsConstants.TEST_INDEX_ACCOUNT + " GROUP BY balance DESC LIMIT 5"); + TestsConstants.TEST_INDEX_ACCOUNT + " GROUP BY balance ORDER BY balance DESC LIMIT 5"); verifySchema(response, schema("CAST(balance AS FLOAT)", null, "float")); verifyDataRows(response, - rows(22026), - rows(23285), - rows(36038), - rows(39063), - rows(45493)); + rows(49989.0), + rows(49795.0), + rows(49741.0), + rows(49671.0), + rows(49587.0)); } @Test public void castIntFieldToFloatWithAliasJdbcFormatGroupByTest() { JSONObject response = executeJdbcRequest( - "SELECT CAST(balance AS FLOAT) AS jdbc_float_alias " + - "FROM " + TestsConstants.TEST_INDEX_ACCOUNT + " GROUP BY jdbc_float_alias ASC LIMIT 5"); + "SELECT CAST(balance AS FLOAT) AS jdbc_float_alias " + + " FROM " + TestsConstants.TEST_INDEX_ACCOUNT + + " GROUP BY jdbc_float_alias " + + " ORDER BY jdbc_float_alias ASC " + + " LIMIT 5"); verifySchema(response, - schema("jdbc_float_alias", "jdbc_float_alias", "float")); + schema("CAST(balance AS FLOAT)", "jdbc_float_alias", "float")); verifyDataRows(response, - rows("22026.0"), - rows("23285.0"), - rows("36038.0"), - rows("39063.0"), - rows("45493.0")); + rows(1011.0), + rows(10116.0), + rows(10138.0), + rows(10147.0), + rows(10178.0)); } @Test @@ -631,7 +634,8 @@ public void operatorReplace() { ); } - @Test + @Ignore("The LOCATE function is not implemented in new SQL engine. https://github" + + ".com/opensearch-project/sql/issues/74") public void operatorLocate() throws IOException { String query = "SELECT LOCATE('a', lastname, 0) FROM " + TEST_INDEX_ACCOUNT + @@ -666,7 +670,8 @@ public void ltrim() throws IOException { ); } - @Test + @Ignore("The ASCII function is not implemented in new SQL engine. https://github" + + ".com/opensearch-project/sql/issues/73") public void ascii() throws IOException { assertThat( executeQuery("SELECT ASCII(lastname) FROM " + TEST_INDEX_ACCOUNT diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TypeInformationIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TypeInformationIT.java index 60a49a5172..9a96de37b9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TypeInformationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TypeInformationIT.java @@ -157,11 +157,11 @@ public void testSubtractLongWithLongReturnsLong() { */ @Test public void testDayOfWeekWithKeywordReturnsText() { - JSONObject response = executeJdbcRequest("SELECT DAY_OF_WEEK(insert_time) FROM " + JSONObject response = executeJdbcRequest("SELECT DAYOFWEEK(insert_time) FROM " + TestsConstants.TEST_INDEX_ONLINE + " LIMIT 2"); verifySchema(response, - schema("DAY_OF_WEEK(insert_time)", null, "integer")); + schema("DAYOFWEEK(insert_time)", null, "integer")); } @Test diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java index 33b52b99a7..d69b95570a 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java @@ -30,9 +30,6 @@ import static org.opensearch.rest.RestStatus.OK; import static org.opensearch.rest.RestStatus.SERVICE_UNAVAILABLE; import static org.opensearch.sql.legacy.plugin.SqlSettings.CURSOR_ENABLED; -import static org.opensearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_ENABLED; -import static org.opensearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_SUGGESTION; -import static org.opensearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_THRESHOLD; import static org.opensearch.sql.legacy.plugin.SqlSettings.SQL_ENABLED; import com.alibaba.druid.sql.parser.ParserException; @@ -294,11 +291,7 @@ private boolean isCursorDisabled() { private static ColumnTypeProvider performAnalysis(String sql) { LocalClusterState clusterState = LocalClusterState.state(); - SqlAnalysisConfig config = new SqlAnalysisConfig( - clusterState.getSettingValue(QUERY_ANALYSIS_ENABLED), - clusterState.getSettingValue(QUERY_ANALYSIS_SEMANTIC_SUGGESTION), - clusterState.getSettingValue(QUERY_ANALYSIS_SEMANTIC_THRESHOLD) - ); + SqlAnalysisConfig config = new SqlAnalysisConfig(false, false, 200); OpenSearchLegacySqlAnalyzer analyzer = new OpenSearchLegacySqlAnalyzer(config); Optional outputColumnType = analyzer.analyze(sql, clusterState); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/SqlSettings.java b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/SqlSettings.java index 7db1706aea..068efd9ce8 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/SqlSettings.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/SqlSettings.java @@ -51,9 +51,6 @@ public class SqlSettings { public static final String SQL_ENABLED = "opensearch.sql.enabled"; public static final String QUERY_SLOWLOG = "opensearch.sql.query.slowlog"; public static final String QUERY_RESPONSE_FORMAT = "opensearch.sql.query.response.format"; - public static final String QUERY_ANALYSIS_ENABLED = "opensearch.sql.query.analysis.enabled"; - public static final String QUERY_ANALYSIS_SEMANTIC_SUGGESTION = "opensearch.sql.query.analysis.semantic.suggestion"; - public static final String QUERY_ANALYSIS_SEMANTIC_THRESHOLD = "opensearch.sql.query.analysis.semantic.threshold"; public static final String METRICS_ROLLING_WINDOW = "opensearch.sql.metrics.rollingwindow"; public static final String METRICS_ROLLING_INTERVAL = "opensearch.sql.metrics.rollinginterval"; @@ -70,14 +67,6 @@ public SqlSettings() { settings.put(QUERY_RESPONSE_FORMAT, Setting.simpleString(QUERY_RESPONSE_FORMAT, Format.JDBC.getFormatName(), NodeScope, Dynamic)); - // Settings for new ANTLR query analyzer - settings.put(QUERY_ANALYSIS_ENABLED, Setting.boolSetting( - QUERY_ANALYSIS_ENABLED, true, NodeScope, Dynamic)); - settings.put(QUERY_ANALYSIS_SEMANTIC_SUGGESTION, Setting.boolSetting( - QUERY_ANALYSIS_SEMANTIC_SUGGESTION, false, NodeScope, Dynamic)); - settings.put(QUERY_ANALYSIS_SEMANTIC_THRESHOLD, Setting.intSetting( - QUERY_ANALYSIS_SEMANTIC_THRESHOLD, 200, NodeScope, Dynamic)); - settings.put(METRICS_ROLLING_WINDOW, Setting.longSetting(METRICS_ROLLING_WINDOW, 3600L, 2L, NodeScope, Dynamic)); settings.put(METRICS_ROLLING_INTERVAL, Setting.longSetting(METRICS_ROLLING_INTERVAL, 60L, 1L,