diff --git a/docs/reference/sql/language/data-types.asciidoc b/docs/reference/sql/language/data-types.asciidoc index a3409e6b92c9b..b42620e0c5415 100644 --- a/docs/reference/sql/language/data-types.asciidoc +++ b/docs/reference/sql/language/data-types.asciidoc @@ -49,7 +49,7 @@ As one can see, all of {es} <> are mapped to the data name in {es-sql}, with the exception of **date** data type which is mapped to **datetime** in {es-sql}. This is to avoid confusion with the ANSI SQL **DATE** (date only) type, which is also supported by {es-sql} in queries (with the use of <>/<>), -but doesn't correspond to an actual mapping in {es} (see the <> below). +but doesn't correspond to an actual mapping in {es} (see the <> below). Obviously, not all types in {es} have an equivalent in SQL and vice-versa hence why, {es-sql} uses the data type _particularities_ of the former over the latter as ultimately {es} is the backing store. @@ -57,7 +57,7 @@ uses the data type _particularities_ of the former over the latter as ultimately In addition to the types above, {es-sql} also supports at _runtime_ SQL-specific types that do not have an equivalent in {es}. Such types cannot be loaded from {es} (as it does not know about them) however can be used inside {es-sql} in queries or their results. -[[es-sql-extra-types]] +[[es-sql-only-types]] The table below indicates these types: diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestChannel.java b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestChannel.java index ff6b99bdc4a51..b280642a502c6 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestChannel.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestChannel.java @@ -24,6 +24,7 @@ import org.elasticsearch.rest.RestStatus; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; public final class FakeRestChannel extends AbstractRestChannel { @@ -59,4 +60,8 @@ public AtomicInteger responses() { public AtomicInteger errors() { return errors; } + + public boolean await() throws InterruptedException { + return latch.await(10, TimeUnit.SECONDS); + } } diff --git a/x-pack/plugin/sql/qa/debug/build.gradle b/x-pack/plugin/sql/qa/debug/build.gradle new file mode 100644 index 0000000000000..c86b23e15b42b --- /dev/null +++ b/x-pack/plugin/sql/qa/debug/build.gradle @@ -0,0 +1,14 @@ +description = 'IDE Debugging for SQL' + +dependencies { + testCompile project(path: ":client:transport") + testCompile project(path: ":modules:aggs-matrix-stats") + testCompile project(path: xpackModule('sql')) + testCompile project(path: xpackModule('core')) +} + +forbiddenApisTest.enabled = false +namingConventions.enabled = false +integTest.enabled = false +testingConventions.enabled = false + diff --git a/x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/ClientReference.java b/x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/ClientReference.java new file mode 100644 index 0000000000000..c916ca36ac7f3 --- /dev/null +++ b/x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/ClientReference.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.sql.qa.jdbc; + +import org.elasticsearch.client.Client; +import org.elasticsearch.common.inject.Module; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.plugins.Plugin; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.util.Collection; + +import static java.util.Arrays.asList; + +public class ClientReference extends Plugin { + + static final ClientInvocationHandler HANDLER = new ClientInvocationHandler(); + private static final Client CLIENT = + (Client) Proxy.newProxyInstance(ClientReference.class.getClassLoader(), new Class[]{Client.class}, HANDLER); + private static final XPackLicenseState LICENSE = new XPackLicenseState(Settings.EMPTY); + + @Override + public Collection createGuiceModules() { + return asList(b -> b.bind(Client.class).toInstance(CLIENT), b -> b.bind(XPackLicenseState.class).toInstance(LICENSE)); + } + + static class ClientInvocationHandler implements InvocationHandler { + + volatile Client actualClient; + + @Override + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + return method.invoke(actualClient, args); + } + } +} diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugCsvSpec.java b/x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugCsvSpec.java similarity index 81% rename from x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugCsvSpec.java rename to x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugCsvSpec.java index d5a633e5ea388..905a6aa1c3db8 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugCsvSpec.java +++ b/x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugCsvSpec.java @@ -6,10 +6,10 @@ package org.elasticsearch.xpack.sql.qa.jdbc; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - import org.apache.logging.log4j.Logger; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.CsvTestCase; +import org.junit.ClassRule; import java.sql.Connection; import java.sql.ResultSet; @@ -20,31 +20,40 @@ import static org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.executeCsvQuery; import static org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.specParser; -@TestLogging("org.elasticsearch.xpack.sql:TRACE") -public abstract class DebugCsvSpec extends SpecBaseIntegrationTestCase { +@TestLogging(JdbcTestUtils.SQL_TRACE) +public class DebugCsvSpec extends SpecBaseIntegrationTestCase { + + @ClassRule + public static final EmbeddedSqlServer EMBEDDED_SERVER = new EmbeddedSqlServer(); + private final CsvTestCase testCase; + public DebugCsvSpec(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase) { + super(fileName, groupName, testName, lineNumber); + this.testCase = testCase; + } + @ParametersFactory(shuffle = false, argumentFormatting = SqlSpecTestCase.PARAM_FORMATTING) public static List readScriptSpec() throws Exception { Parser parser = specParser(); return readScriptSpec("/debug.csv-spec", parser); } - public DebugCsvSpec(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase) { - super(fileName, groupName, testName, lineNumber); - this.testCase = testCase; + @Override + protected final void doTest() throws Throwable { + try (Connection csv = csvConnection(testCase); Connection es = esJdbc()) { + // pass the testName as table for debugging purposes (in case the underlying reader is missing) + ResultSet expected = executeCsvQuery(csv, testName); + ResultSet elasticResults = executeJdbcQuery(es, testCase.query); + assertResults(expected, elasticResults); + } } @Override - protected void assertResults(ResultSet expected, ResultSet elastic) throws SQLException { - Logger log = logEsResultSet() ? logger : null; - - // - // uncomment this to printout the result set and create new CSV tests - // - JdbcTestUtils.logResultSetMetadata(elastic, log); - JdbcTestUtils.logResultSetData(elastic, log); - //JdbcAssert.assertResultSets(expected, elastic, log); + public Connection esJdbc() throws SQLException { + // use the same random path as the rest of the tests + randomBoolean(); + return EMBEDDED_SERVER.connection(connectionProperties()); } @Override @@ -53,12 +62,13 @@ protected boolean logEsResultSet() { } @Override - protected final void doTest() throws Throwable { - try (Connection csv = csvConnection(testCase); Connection es = esJdbc()) { - // pass the testName as table for debugging purposes (in case the underlying reader is missing) - ResultSet expected = executeCsvQuery(csv, testName); - ResultSet elasticResults = executeJdbcQuery(es, testCase.query); - assertResults(expected, elasticResults); - } + protected void assertResults(ResultSet expected, ResultSet elastic) throws SQLException { + Logger log = logEsResultSet() ? logger : null; + + // + // uncomment this to printout the result set and create new CSV tests + // + JdbcTestUtils.logLikeCLI(elastic, log); + //JdbcAssert.assertResultSets(expected, elastic, log); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugSqlSpec.java b/x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugSqlSpec.java similarity index 52% rename from x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugSqlSpec.java rename to x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugSqlSpec.java index b51d66ace2e26..309515dd0beac 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugSqlSpec.java +++ b/x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugSqlSpec.java @@ -6,25 +6,51 @@ package org.elasticsearch.xpack.sql.qa.jdbc; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - +import org.apache.logging.log4j.Logger; import org.elasticsearch.test.junit.annotations.TestLogging; +import org.junit.ClassRule; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.List; @TestLogging(JdbcTestUtils.SQL_TRACE) -public abstract class DebugSqlSpec extends SqlSpecTestCase { +public class DebugSqlSpec extends SqlSpecTestCase { + + @ClassRule + public static final EmbeddedSqlServer EMBEDDED_SERVER = new EmbeddedSqlServer(); + + public DebugSqlSpec(String fileName, String groupName, String testName, Integer lineNumber, String query) { + super(fileName, groupName, testName, lineNumber, query); + } + @ParametersFactory(shuffle = false, argumentFormatting = PARAM_FORMATTING) public static List readScriptSpec() throws Exception { Parser parser = specParser(); return readScriptSpec("/datetime.sql-spec", parser); } - public DebugSqlSpec(String fileName, String groupName, String testName, Integer lineNumber, String query) { - super(fileName, groupName, testName, lineNumber, query); + @Override + public Connection esJdbc() throws SQLException { + // use the same random path as the rest of the tests + randomBoolean(); + return EMBEDDED_SERVER.connection(connectionProperties()); } @Override protected boolean logEsResultSet() { return true; } -} \ No newline at end of file + + @Override + protected void assertResults(ResultSet expected, ResultSet elastic) throws SQLException { + Logger log = logEsResultSet() ? logger : null; + + // + // uncomment this to printout the result set and create new CSV tests + // + JdbcTestUtils.logLikeCLI(elastic, log); + //JdbcAssert.assertResultSets(expected, elastic, log); + } +} diff --git a/x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/EmbeddedSqlServer.java b/x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/EmbeddedSqlServer.java new file mode 100644 index 0000000000000..bf53a9a52b245 --- /dev/null +++ b/x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/EmbeddedSqlServer.java @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.sql.qa.jdbc; + +import org.apache.logging.log4j.LogManager; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.search.aggregations.matrix.MatrixAggregationPlugin; +import org.elasticsearch.transport.client.PreBuiltTransportClient; +import org.elasticsearch.xpack.sql.plugin.SqlPlugin; +import org.junit.rules.ExternalResource; + +import java.net.InetAddress; +import java.security.AccessControlException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Properties; + +import static org.junit.Assert.assertNotNull; + +/** + * Embedded JDBC server that uses the transport client to power + * the jdbc endpoints in the same JVM as the tests. + */ +@SuppressWarnings("deprecation") +public class EmbeddedSqlServer extends ExternalResource implements AutoCloseable { + + private final Properties properties; + private TransportClient client; + private SqlHttpServer server; + private String jdbcUrl; + + EmbeddedSqlServer() { + this(false); + } + + private EmbeddedSqlServer(boolean debug) { + properties = new Properties(); + if (debug) { + properties.setProperty("debug", "true"); + } + } + + @Override + @SuppressWarnings({"resource"}) + protected void before() throws Throwable { + try { + Settings settings = Settings.builder() + .put("client.transport.ignore_cluster_name", true) + .build(); + TransportAddress transportAddress = new TransportAddress(InetAddress.getLoopbackAddress(), 9300); + client = new PreBuiltTransportClient(settings, MatrixAggregationPlugin.class, ClientReference.class, SqlPlugin.class) + .addTransportAddress(transportAddress); + + // update static reference + ClientReference.HANDLER.actualClient = client; + } catch (ExceptionInInitializerError e) { + if (e.getCause() instanceof AccessControlException) { + throw new RuntimeException(getClass().getSimpleName() + " is not available with the security manager", e); + } else { + throw e; + } + } + + server = new SqlHttpServer(client); + server.start(0); + jdbcUrl = server.jdbcUrl(); + + LogManager.getLogger(EmbeddedSqlServer.class).info("Embedded SQL started at [{}]", server.url()); + } + + @Override + public void close() { + after(); + } + + @Override + protected void after() { + if (client != null) { + client.close(); + client = null; + } + if (server != null) { + server.stop(); + server = null; + } + } + + Connection connection(Properties props) throws SQLException { + assertNotNull("ES JDBC Server is null - make sure ES is properly run as a @ClassRule", client); + Properties p = new Properties(properties); + p.putAll(props); + return DriverManager.getConnection(jdbcUrl, p); + // JdbcDataSource dataSource = new JdbcDataSource(); + // dataSource.setProperties(properties); + // dataSource.setUrl(jdbcUrl); + // return dataSource.getConnection(); + } +} diff --git a/x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/RootHandler.java b/x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/RootHandler.java new file mode 100644 index 0000000000000..d1aaa0fe6f8cb --- /dev/null +++ b/x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/RootHandler.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.sql.qa.jdbc; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.rest.RestStatus; + +import java.io.IOException; + +@SuppressForbidden(reason = "use http server") +@SuppressWarnings("restriction") +class RootHandler implements HttpHandler { + + private static final Logger log = LogManager.getLogger(RootHandler.class.getName()); + + @Override + public void handle(HttpExchange http) throws IOException { + log.debug("Received query call..."); + + if ("HEAD".equals(http.getRequestMethod())) { + http.sendResponseHeaders(RestStatus.OK.getStatus(), 0); + http.close(); + return; + } + + fail(http, new UnsupportedOperationException("only HEAD allowed")); + } + + private void fail(HttpExchange http, Exception ex) { + log.error("Caught error while transmitting response", ex); + try { + // the error conversion has failed, halt + if (http.getResponseHeaders().isEmpty()) { + http.sendResponseHeaders(RestStatus.INTERNAL_SERVER_ERROR.getStatus(), -1); + } + } catch (IOException ioEx) { + log.error("Caught error while trying to catch error", ex); + } finally { + http.close(); + } + } +} diff --git a/x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/SqlHandler.java b/x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/SqlHandler.java new file mode 100644 index 0000000000000..a474bbf2c8ebe --- /dev/null +++ b/x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/SqlHandler.java @@ -0,0 +1,128 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.sql.qa.jdbc; + +import com.sun.net.httpserver.Headers; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import io.netty.handler.codec.http.HttpHeaderNames; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.BytesRestResponse; +import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.rest.FakeRestChannel; +import org.elasticsearch.test.rest.FakeRestRequest; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; + +import static java.util.Collections.singletonList; + +@SuppressForbidden(reason = "use http server") +@SuppressWarnings("restriction") +class SqlHandler implements HttpHandler, AutoCloseable { + + private static final Logger log = LogManager.getLogger(SqlHandler.class.getName()); + + private final RestHandler handler; + private final SqlNodeClient client; + + SqlHandler(SqlNodeClient client, RestHandler restHandler) { + this.client = client; + this.handler = restHandler; + } + + @Override + public void handle(HttpExchange http) throws IOException { + log.debug("Received query call..."); + + if ("HEAD".equals(http.getRequestMethod())) { + http.sendResponseHeaders(RestStatus.OK.getStatus(), 0); + http.close(); + return; + } + + + boolean closeEndpoint = http.getRequestURI().toString().contains("/close"); + + FakeRestRequest request = createRequest(http); + FakeRestChannel channel = new FakeRestChannel(request, true, 1); + + try { + handler.handleRequest(request, channel, client); + if (!closeEndpoint) { + if (channel.await()) { + sendHttpResponse(http, channel.capturedResponse()); + } else { + sendHttpResponse(http, new BytesRestResponse(channel, new IllegalStateException("Timed-out"))); + } + } else { + sendHttpResponse(http, new BytesRestResponse(RestStatus.OK, "")); + } + } catch (Exception e) { + sendHttpResponse(http, new BytesRestResponse(channel, e)); + } + } + + private FakeRestRequest createRequest(HttpExchange http) throws IOException { + Headers headers = http.getRequestHeaders(); + XContentType contentType = XContentType.fromMediaTypeOrFormat( + headers.getOrDefault(HttpHeaderNames.CONTENT_TYPE.toString(), singletonList(XContentType.JSON.mediaType())).get(0)); + + Map params = new LinkedHashMap<>(); + params.put("pretty", ""); + params.put("human", "true"); + params.put("error_trace", "true"); + + BytesStreamOutput bso = new BytesStreamOutput(); + Streams.copy(http.getRequestBody(), bso); + FakeRestRequest request = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY) + .withHeaders(headers) + .withParams(params) + .withRemoteAddress(http.getRemoteAddress()) + .withContent(bso.bytes(), contentType) + .build(); + + // consume error_trace (in server Netty does that) + request.param("error_trace"); + return request; + } + + private void sendHttpResponse(HttpExchange http, RestResponse response) { + try { + // first do the conversion in case an exception is triggered + if (http.getResponseHeaders().isEmpty()) { + Headers headers = http.getResponseHeaders(); + headers.put(HttpHeaderNames.CONTENT_TYPE.toString(), singletonList(response.contentType())); + if (response.getHeaders() != null) { + headers.putAll(response.getHeaders()); + } + + // NB: this needs to be called last otherwise any calls to the headers are silently ignored... + http.sendResponseHeaders(response.status().getStatus(), response.content().length()); + } + response.content().writeTo(http.getResponseBody()); + } catch (IOException ex) { + log.error("Caught error while trying to catch error", ex); + } finally { + http.close(); + } + } + + @Override + public void close() { + // no-op + } +} diff --git a/x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/SqlHttpServer.java b/x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/SqlHttpServer.java new file mode 100644 index 0000000000000..9788e713ecdb4 --- /dev/null +++ b/x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/SqlHttpServer.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.sql.qa.jdbc; + +import com.sun.net.httpserver.HttpServer; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.mock.orig.Mockito; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.xpack.sql.plugin.RestSqlClearCursorAction; +import org.elasticsearch.xpack.sql.plugin.RestSqlQueryAction; +import org.elasticsearch.xpack.sql.proto.Protocol; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +@SuppressForbidden(reason = "use http server") +@SuppressWarnings("restriction") +class SqlHttpServer { + + private final SqlNodeClient sqlClient; + + private HttpServer server; + private ExecutorService executor; + + SqlHttpServer(Client client) { + this.sqlClient = client instanceof SqlNodeClient ? (SqlNodeClient) client : new SqlNodeClient(client); + } + + + void start(int port) throws IOException { + // similar to Executors.newCached but with a smaller bound and much smaller keep-alive + executor = new ThreadPoolExecutor(1, 10, 250, TimeUnit.MILLISECONDS, new SynchronousQueue<>()); + + server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), port), 0); + server.createContext("/", new RootHandler()); + initialize(server); + server.setExecutor(executor); + server.start(); + } + + private void initialize(HttpServer server) { + // initialize cursor + initializeActions(server); + } + + + private void initializeActions(HttpServer server) { + RestController mock = Mockito.mock(RestController.class); + server.createContext(Protocol.SQL_QUERY_REST_ENDPOINT, + new SqlHandler(sqlClient, new RestSqlQueryAction(Settings.EMPTY, mock))); + server.createContext(Protocol.CLEAR_CURSOR_REST_ENDPOINT, + new SqlHandler(sqlClient, new RestSqlClearCursorAction(Settings.EMPTY, mock))); + } + + + void stop() { + server.stop(1); + server = null; + executor.shutdownNow(); + executor = null; + } + + private InetSocketAddress address() { + return server != null ? server.getAddress() : null; + } + + String url() { + InetSocketAddress address = address(); + return address != null ? "localhost:" + address.getPort() : ""; + } + + String jdbcUrl() { + return "jdbc:es://" + url(); + } +} diff --git a/x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/SqlNodeClient.java b/x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/SqlNodeClient.java new file mode 100644 index 0000000000000..533250916f8aa --- /dev/null +++ b/x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/SqlNodeClient.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.sql.qa.jdbc; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.aggregations.matrix.MatrixAggregationPlugin; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.xpack.sql.action.SqlClearCursorAction; +import org.elasticsearch.xpack.sql.action.SqlClearCursorRequest; +import org.elasticsearch.xpack.sql.action.SqlClearCursorResponse; +import org.elasticsearch.xpack.sql.action.SqlQueryAction; +import org.elasticsearch.xpack.sql.action.SqlQueryRequest; +import org.elasticsearch.xpack.sql.action.SqlQueryResponse; +import org.elasticsearch.xpack.sql.analysis.index.IndexResolver; +import org.elasticsearch.xpack.sql.execution.PlanExecutor; +import org.elasticsearch.xpack.sql.plugin.TransportSqlClearCursorAction; +import org.elasticsearch.xpack.sql.plugin.TransportSqlQueryAction; + +import java.util.Objects; + +import static java.util.Collections.singletonList; + +/** + * Implements embedded sql mode by intercepting requests to SQL APIs and executing them locally. + */ +public class SqlNodeClient extends NodeClient { + + private final Client in; + private final PlanExecutor planExecutor; + + SqlNodeClient(Client in) { + super(in.settings(), in.threadPool()); + this.in = in; + + SearchModule searchModule = new SearchModule(Settings.EMPTY, true, singletonList(new MatrixAggregationPlugin())); + NamedWriteableRegistry writeableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables()); + IndexResolver indexResolver = new IndexResolver(in, "elasticsearch"); + this.planExecutor = new PlanExecutor(in, indexResolver, writeableRegistry); + } + + @Override + public Task + executeLocally(Action action, Request request, ActionListener listener) { + doExecute(action, request, listener); + return null; + } + + @Override + @SuppressWarnings("unchecked") + public void + doExecute(Action action, Request request, ActionListener listener) { + Objects.requireNonNull(planExecutor, "plan executor not set on EmbeddedClient"); + + if (action == SqlQueryAction.INSTANCE) { + TransportSqlQueryAction.operation( + planExecutor, (SqlQueryRequest) request, (ActionListener) listener, "marios", "debug-elasticsearch"); + } else if (action == SqlClearCursorAction.INSTANCE) { + TransportSqlClearCursorAction.operation(planExecutor, (SqlClearCursorRequest) request, + (ActionListener) listener); + } else { + in.execute(action, request, listener); + } + } +} diff --git a/x-pack/plugin/sql/qa/debug/src/test/resources/debug.csv-spec b/x-pack/plugin/sql/qa/debug/src/test/resources/debug.csv-spec new file mode 100644 index 0000000000000..2e521e9023040 --- /dev/null +++ b/x-pack/plugin/sql/qa/debug/src/test/resources/debug.csv-spec @@ -0,0 +1,14 @@ +// +// Spec used for debugging a certain test (without having to alter the spec suite of which it might be part of) +// + +debug +SHOW TABLES "test*,-test_a*"; + + name | type +test_alias |ALIAS +test_alias_emp |ALIAS +test_emp |BASE TABLE +test_emp_copy |BASE TABLE +test_emp_with_nulls|BASE TABLE +; \ No newline at end of file diff --git a/x-pack/plugin/sql/qa/src/main/resources/debug.sql-spec b/x-pack/plugin/sql/qa/debug/src/test/resources/debug.sql-spec similarity index 60% rename from x-pack/plugin/sql/qa/src/main/resources/debug.sql-spec rename to x-pack/plugin/sql/qa/debug/src/test/resources/debug.sql-spec index cd03b4764b73d..21905e259e39e 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/debug.sql-spec +++ b/x-pack/plugin/sql/qa/debug/src/test/resources/debug.sql-spec @@ -3,4 +3,4 @@ // debug -SELECT 5 + 2 AS a; +SELECT languages AS l, COUNT(*) AS c FROM test_emp WHERE languages > 2 GROUP BY l; diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SpecBaseIntegrationTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SpecBaseIntegrationTestCase.java index e69d229b6f170..8c1f4a375db43 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SpecBaseIntegrationTestCase.java +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SpecBaseIntegrationTestCase.java @@ -197,4 +197,4 @@ public interface Parser { public static InputStream readFromJarUrl(URL source) throws IOException { return source.openStream(); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/sql/qa/src/main/resources/debug.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/debug.csv-spec deleted file mode 100644 index a76b656cfa07b..0000000000000 --- a/x-pack/plugin/sql/qa/src/main/resources/debug.csv-spec +++ /dev/null @@ -1,20 +0,0 @@ -// -// Spec used for debugging a certain test (without having to alter the spec suite of which it might be part of) -// - -debug -SELECT first_name f, last_name l, dep.from_date d FROM test_emp WHERE dep.dep_name = 'Production' ORDER BY f LIMIT 5; - -f:s | l:s | d:ts - -Alain | Chappelet | 589420800000 -Chirstian | Koblick | 533779200000 -Duangkaew | Piveteau | 848793600000 -Elvis | Demeyer | 761443200000 -Gino | Leonhardt | 607996800000 -; - -//SELECT YEAR(dep.from_date) start FROM test_emp WHERE dep.dep_name = 'Production' GROUP BY start LIMIT 5; -//table:s -//test_emp -//; \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java index 1e78191ffad75..61e1e6bc67ef0 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java @@ -96,8 +96,6 @@ public Object extract(Bucket bucket) { return object; } else if (object instanceof Long) { object = DateUtils.asDateTime(((Long) object).longValue(), zoneId); - } else if (object instanceof String) { // CAST( AS DATE) is used - object = DateUtils.asDateOnly(object.toString()); } else { throw new SqlIllegalArgumentException("Invalid date key returned: {}", object); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java index a6740928b6810..5be5e28718459 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic; -import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; import org.elasticsearch.xpack.sql.expression.Expression; import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.BinaryArithmeticProcessor.BinaryArithmeticOperation; import org.elasticsearch.xpack.sql.tree.Source; @@ -15,7 +14,6 @@ import org.elasticsearch.xpack.sql.type.DataTypes; import static org.elasticsearch.common.logging.LoggerMessageFormat.format; -import static org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.BinaryArithmeticProcessor.BinaryArithmeticOperation.SUB; abstract class DateTimeArithmeticOperation extends ArithmeticOperation { @@ -47,15 +45,15 @@ protected TypeResolution resolveType() { if (DataTypeConversion.commonType(l, r) == null) { return new TypeResolution(format("[{}] has arguments with incompatible types [{}] and [{}]", symbol(), l, r)); } else { - if (function() == SUB && right().dataType().isDateBased() && DataTypes.isInterval(left().dataType())) { - throw new SqlIllegalArgumentException("Cannot subtract a date from an interval; do you mean the reverse?"); - } - return TypeResolution.TYPE_RESOLVED; + return resolveWithIntervals(); } } // fall-back to default checks return super.resolveType(); } - + + protected TypeResolution resolveWithIntervals() { + return TypeResolution.TYPE_RESOLVED; + } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/Sub.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/Sub.java index 32acfa8ed685d..652cfb98bd503 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/Sub.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/Sub.java @@ -7,8 +7,11 @@ import org.elasticsearch.xpack.sql.expression.Expression; import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.BinaryArithmeticProcessor.BinaryArithmeticOperation; -import org.elasticsearch.xpack.sql.tree.Source; import org.elasticsearch.xpack.sql.tree.NodeInfo; +import org.elasticsearch.xpack.sql.tree.Source; +import org.elasticsearch.xpack.sql.type.DataTypes; + +import static org.elasticsearch.common.logging.LoggerMessageFormat.format; /** * Subtraction function ({@code a - b}). @@ -28,4 +31,13 @@ protected NodeInfo info() { protected Sub replaceChildren(Expression newLeft, Expression newRight) { return new Sub(source(), newLeft, newRight); } + + @Override + protected TypeResolution resolveWithIntervals() { + if (right().dataType().isDateBased() && DataTypes.isInterval(left().dataType())) { + return new TypeResolution(format(null, "Cannot subtract a {}[{}] from an interval[{}]; do you mean the reverse?", + right().dataType().esType, right().source().text(), left().source().text())); + } + return TypeResolution.TYPE_RESOLVED; + } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByKey.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByKey.java index ba4b448b583f3..6f26ee1dd960c 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByKey.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByKey.java @@ -39,7 +39,9 @@ public final CompositeValuesSourceBuilder asValueSource() { builder.valueType(ValueType.DOUBLE); } else if (script.outputType().isString()) { builder.valueType(ValueType.STRING); - } else if (script.outputType().isDateBased()) { + } else if (script.outputType() == DataType.DATE) { + builder.valueType(ValueType.LONG); + } else if (script.outputType() == DataType.DATETIME) { builder.valueType(ValueType.DATE); } else if (script.outputType() == DataType.BOOLEAN) { builder.valueType(ValueType.BOOLEAN); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java index d6454e378fb01..bdd455fe10f63 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java @@ -34,11 +34,7 @@ private DateUtils() {} * Creates an date for SQL DATE type from the millis since epoch. */ public static ZonedDateTime asDateOnly(long millis) { - return ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), UTC) - .withHour(0) - .withMinute(0) - .withSecond(0) - .withNano(0); + return ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), UTC).toLocalDate().atStartOfDay(UTC); } /** @@ -78,11 +74,7 @@ public static ZonedDateTime asDateOnly(DateTime dateTime) { } public static ZonedDateTime asDateOnly(ZonedDateTime zdt) { - return zdt - .withHour(0) - .withMinute(0) - .withSecond(0) - .withNano(0); + return zdt.toLocalDate().atStartOfDay(zdt.getZone()); } /** diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java index d7d5c29b8250e..946e8f93a7091 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java @@ -199,11 +199,9 @@ public void testExtractNonDateTime() { } public void testSubtractFromInterval() { - Analyzer analyzer = new Analyzer(TestUtils.TEST_CFG, new FunctionRegistry(), indexResolution, new Verifier(new Metrics())); - SqlIllegalArgumentException e = expectThrows(SqlIllegalArgumentException.class, () -> analyzer.analyze(parser.createStatement( - "SELECT INTERVAL 1 MONTH - CAST('2000-01-01' AS DATETIME)"), true)); - assertEquals("Cannot subtract a date from an interval; do you mean the reverse?", - e.getMessage()); + assertEquals("1:8: Cannot subtract a datetime[CAST('2000-01-01' AS DATETIME)] " + + "from an interval[INTERVAL 1 MONTH]; do you mean the reverse?", + error("SELECT INTERVAL 1 MONTH - CAST('2000-01-01' AS DATETIME)")); } public void testMultipleColumns() {