diff --git a/docs/user/admin/settings.rst b/docs/user/admin/settings.rst index 82ba48a10f..2029549195 100644 --- a/docs/user/admin/settings.rst +++ b/docs/user/admin/settings.rst @@ -510,3 +510,47 @@ Result set:: } } +opendistro.sql.engine.new.enabled +================================= + +Description +----------- + +We are migrating existing functionalities to a new query engine under development. User can choose to enable the new engine if interested or disable if any issue found. + +1. The default value is false. +2. This setting is node scope. +3. This setting can be updated dynamically. + + +Example +------- + +You can update the setting with a new value like this. + +SQL query:: + + >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_opendistro/_sql/settings -d '{ + "transient" : { + "opendistro.sql.engine.new.enabled" : "true" + } + }' + +Result set:: + + { + "acknowledged" : true, + "persistent" : { }, + "transient" : { + "opendistro" : { + "sql" : { + "engine" : { + "new" : { + "enabled" : "true" + } + } + } + } + } + } + diff --git a/docs/user/dql/expressions.rst b/docs/user/dql/expressions.rst index 2e57cb9305..c81f6ee6f7 100644 --- a/docs/user/dql/expressions.rst +++ b/docs/user/dql/expressions.rst @@ -14,6 +14,8 @@ Introduction Expressions, particularly value expressions, are those which return a scalar value. Expressions have different types and forms. For example, there are literal values as atom expression and arithmetic, predicate and function expression built on top of them. And also expressions can be used in different clauses, such as using arithmetic expression in ``SELECT``, ``WHERE`` or ``HAVING`` clause. +Note that before you try out examples using the SQL features in this doc, you need to enable the new query engine by following the steps in ``opendistro.sql.engine.new.enabled`` section in `Plugin Settings `_. + Literal Values ============== diff --git a/docs/user/dql/newsql.rst b/docs/user/dql/newsql.rst new file mode 100644 index 0000000000..9c7eed42c7 --- /dev/null +++ b/docs/user/dql/newsql.rst @@ -0,0 +1,35 @@ +.. highlight:: sh + +============== +New SQL Engine +============== + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + +Introduction +============ + +To use the SQL features present in documentation correctly, you need to enable our new SQL query engine by the following command:: + + sh$ curl -sS -H 'Content-Type: application/json' \ + ... -X PUT localhost:9200/_opendistro/_sql/settings \ + ... -d '{"transient" : {"opendistro.sql.engine.new.enabled" : "true"}}' + { + "acknowledged": true, + "persistent": {}, + "transient": { + "opendistro": { + "sql": { + "engine": { + "new": { + "enabled": "true" + } + } + } + } + } + } diff --git a/doctest/test_docs.py b/doctest/test_docs.py index 57e10f9a21..05829addbb 100644 --- a/doctest/test_docs.py +++ b/doctest/test_docs.py @@ -136,6 +136,25 @@ def run(self, result, debug=False): super().run(result, debug) +def doc_suite(fn): + return docsuite( + fn, + parser=bash_parser, + setUp=set_up_accounts, + globs={ + 'sh': partial( + subprocess.run, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=60, + shell=True + ), + 'pretty_print': pretty_print + } + ) + + def load_tests(loader, suite, ignore): tests = [] # Load doctest docs by category @@ -148,24 +167,8 @@ def load_tests(loader, suite, ignore): # docs with bash-based examples for fn in doctest_files(bash_docs): - tests.append( - docsuite( - fn, - parser=bash_parser, - setUp=set_up_accounts, - globs={ - 'sh': partial( - subprocess.run, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - timeout=60, - shell=True - ), - 'pretty_print': pretty_print - } - ) - ) + tests.append(doc_suite(fn)) + # docs with sql-cli based examples # TODO: add until the migration to new architecture is done, then we have an artifact including ppl and sql both # for fn in doctest_files('sql/basics.rst'): @@ -191,4 +194,8 @@ def load_tests(loader, suite, ignore): # randomize order of tests to make sure they don't depend on each other random.shuffle(tests) + + # prepend a temporary doc to enable new engine so new SQL docs followed can pass + tests.insert(0, doc_suite('../docs/user/dql/newsql.rst')) + return DocTests(tests) diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/admin/PluginSettingIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/admin/PluginSettingIT.java index f97e2e3e48..2ee7e92ee0 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/admin/PluginSettingIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/admin/PluginSettingIT.java @@ -28,6 +28,7 @@ import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_RESPONSE_FORMAT; import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_SLOWLOG; import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.SQL_ENABLED; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.SQL_NEW_ENGINE_ENABLED; import static org.elasticsearch.common.settings.Setting.Property; import static org.elasticsearch.common.settings.Setting.Property.Dynamic; import static org.elasticsearch.common.settings.Setting.Property.Final; @@ -147,6 +148,16 @@ public void cursorDefaultContextKeepAliveSetting() { ); } + @Section(10) + public void sqlNewQueryEngineSetting() { + docSetting( + SQL_NEW_ENGINE_ENABLED, + "We are migrating existing functionalities to a new query engine under development. " + + "User can choose to enable the new engine if interested or disable if any issue found.", + true + ); + } + /** * Generate content for sample queries with setting changed to new value. * Finally setting will be reverted to avoid potential impact on other test cases. diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/ExpressionIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/ExpressionIT.java index 49331a574c..46a0a702a4 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/ExpressionIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/ExpressionIT.java @@ -21,6 +21,7 @@ import static org.hamcrest.Matchers.is; import com.amazon.opendistroforelasticsearch.sql.legacy.RestIntegTestCase; +import com.amazon.opendistroforelasticsearch.sql.util.TestUtils; import java.io.IOException; import java.util.Locale; import java.util.function.Function; @@ -42,6 +43,12 @@ public class ExpressionIT extends RestIntegTestCase { @Rule public ExpectedException exceptionRule = ExpectedException.none(); + @Override + protected void init() throws Exception { + super.init(); + TestUtils.enableNewQueryEngine(client()); + } + @Test public void testDivideZeroExpression() throws Exception { expectResponseException().hasStatusCode(500) //TODO: should be client error code 400? diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLCorrectnessIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLCorrectnessIT.java index c052d73cda..7ad193b985 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLCorrectnessIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLCorrectnessIT.java @@ -16,6 +16,7 @@ package com.amazon.opendistroforelasticsearch.sql.sql; +import com.amazon.opendistroforelasticsearch.sql.util.TestUtils; import com.google.common.io.Resources; import java.io.IOException; import java.nio.file.Files; @@ -33,6 +34,12 @@ public class SQLCorrectnessIT extends SQLIntegTestCase { private static final String[] EXPR_TEST_DIR = { "expressions" }; private static final String[] QUERY_TEST_DIR = { "queries"/*, "bugfixes"*/ }; //TODO: skip bugfixes folder for now since it fails + @Override + protected void init() throws Exception { + super.init(); + TestUtils.enableNewQueryEngine(client()); + } + @Test public void runAllTests() throws Exception { verifyQueries(EXPR_TEST_DIR, expr -> "SELECT " + expr); @@ -64,4 +71,6 @@ private void verifyQueries(Path file, Function converter) { } } + + } diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/TestUtils.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/TestUtils.java index 5d7ecbb2db..f68dd74456 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/TestUtils.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/TestUtils.java @@ -15,6 +15,7 @@ package com.amazon.opendistroforelasticsearch.sql.util; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlSettingsAction.SETTINGS_API_ENDPOINT; import static com.google.common.base.Strings.isNullOrEmpty; import java.io.BufferedReader; @@ -38,10 +39,13 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.client.Client; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.RestStatus; import org.json.JSONObject; +import org.junit.Assert; public class TestUtils { @@ -847,4 +851,21 @@ public static List> getPermutations(final List items) { return result; } + + /** + * Enable new query engine which is disabled by default for now. + */ + public static void enableNewQueryEngine(RestClient client) throws IOException { + Request request = new Request("PUT", SETTINGS_API_ENDPOINT); + request.setJsonEntity("{\"transient\" : {\"opendistro.sql.engine.new.enabled\" : \"true\"}}"); + + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + + Response response = client.performRequest(request); + Assert.assertEquals(RestStatus.OK, + RestStatus.fromCode(response.getStatusLine().getStatusCode())); + } + } diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java index 2af051a1dd..b715dc8364 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java @@ -71,6 +71,7 @@ import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_SUGGESTION; import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_THRESHOLD; import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.SQL_ENABLED; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.SQL_NEW_ENGINE_ENABLED; import static org.elasticsearch.rest.RestStatus.BAD_REQUEST; import static org.elasticsearch.rest.RestStatus.OK; import static org.elasticsearch.rest.RestStatus.SERVICE_UNAVAILABLE; @@ -144,19 +145,21 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli Format format = SqlRequestParam.getFormat(request.params()); - // Route request to new query engine if it's supported already - SQLQueryRequest newSqlRequest = new SQLQueryRequest(sqlRequest.getJsonContent(), - sqlRequest.getSql(), - request.path(), - format.getFormatName()); - RestChannelConsumer result = newSqlQueryHandler.prepareRequest(newSqlRequest, client); - if (result != RestSQLQueryAction.NOT_SUPPORTED_YET) { - LOG.info("[{}] Request {} is handled by new SQL query engine", + if (isNewEngineEnabled()) { + // Route request to new query engine if it's supported already + SQLQueryRequest newSqlRequest = new SQLQueryRequest(sqlRequest.getJsonContent(), + sqlRequest.getSql(), + request.path(), + format.getFormatName()); + RestChannelConsumer result = newSqlQueryHandler.prepareRequest(newSqlRequest, client); + if (result != RestSQLQueryAction.NOT_SUPPORTED_YET) { + LOG.info("[{}] Request {} is handled by new SQL query engine", + LogUtils.getRequestId(), newSqlRequest); + return result; + } + LOG.debug("[{}] Request {} is not supported and falling back to old SQL engine", LogUtils.getRequestId(), newSqlRequest); - return result; } - LOG.debug("[{}] Request {} is not supported and falling back to old SQL engine", - LogUtils.getRequestId(), newSqlRequest); final QueryAction queryAction = explainRequest(client, sqlRequest, format); return channel -> executeSqlRequest(request, queryAction, client, channel); @@ -259,6 +262,10 @@ private boolean isSQLFeatureEnabled() { return allowExplicitIndex && isSqlEnabled; } + private boolean isNewEngineEnabled() { + return LocalClusterState.state().getSettingValue(SQL_NEW_ENGINE_ENABLED); + } + private static ColumnTypeProvider performAnalysis(String sql) { LocalClusterState clusterState = LocalClusterState.state(); SqlAnalysisConfig config = new SqlAnalysisConfig( diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SqlSettings.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SqlSettings.java index 8c379f40f3..7bb9fe7261 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SqlSettings.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SqlSettings.java @@ -39,6 +39,7 @@ public class SqlSettings { * 2) It has separate setting for Query and Fetch phase which are all ES internal concepts. */ public static final String SQL_ENABLED = "opendistro.sql.enabled"; + public static final String SQL_NEW_ENGINE_ENABLED = "opendistro.sql.engine.new.enabled"; public static final String QUERY_SLOWLOG = "opendistro.sql.query.slowlog"; public static final String QUERY_RESPONSE_FORMAT = "opendistro.sql.query.response.format"; public static final String QUERY_ANALYSIS_ENABLED = "opendistro.sql.query.analysis.enabled"; @@ -56,6 +57,7 @@ public class SqlSettings { public SqlSettings() { Map> settings = new HashMap<>(); settings.put(SQL_ENABLED, Setting.boolSetting(SQL_ENABLED, true, NodeScope, Dynamic)); + settings.put(SQL_NEW_ENGINE_ENABLED, Setting.boolSetting(SQL_NEW_ENGINE_ENABLED, false, NodeScope, Dynamic)); settings.put(QUERY_SLOWLOG, Setting.intSetting(QUERY_SLOWLOG, 2, NodeScope, Dynamic)); settings.put(QUERY_RESPONSE_FORMAT, Setting.simpleString(QUERY_RESPONSE_FORMAT, Format.JDBC.getFormatName(), NodeScope, Dynamic));