From 1bd0f35fbdf341c0bae7b1377ab18fd3d5a50435 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Thu, 15 Sep 2022 09:29:27 -0700 Subject: [PATCH 01/23] Change ApplicationContext lifecycle to per node level (#822) Signed-off-by: penghuo Signed-off-by: penghuo --- .../expression/config/ExpressionConfig.java | 4 ++ .../sql/legacy/plugin/RestSQLQueryAction.java | 47 ++------------- .../sql/legacy/plugin/RestSqlAction.java | 15 +++-- .../legacy/plugin/RestSQLQueryActionTest.java | 31 +++++----- .../opensearch/security/SecurityAccess.java | 6 +- .../org/opensearch/sql/plugin/SQLPlugin.java | 48 +++++++++------ .../plugin/catalog/CatalogServiceImpl.java | 13 +---- .../plugin/config/OpenSearchPluginConfig.java | 19 ++++-- .../plugin/rest/OpenSearchPluginConfig.java | 58 ------------------- .../sql/plugin/rest/RestPPLQueryAction.java | 1 - .../transport/TransportPPLQueryAction.java | 37 ++---------- ppl/build.gradle | 1 - .../sql/ppl/config/PPLServiceConfig.java | 3 + .../sql/sql/config/SQLServiceConfig.java | 14 ++--- 14 files changed, 99 insertions(+), 198 deletions(-) rename legacy/src/main/java/org/opensearch/sql/legacy/plugin/OpenSearchSQLPluginConfig.java => plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginConfig.java (73%) delete mode 100644 plugin/src/main/java/org/opensearch/sql/plugin/rest/OpenSearchPluginConfig.java diff --git a/core/src/main/java/org/opensearch/sql/expression/config/ExpressionConfig.java b/core/src/main/java/org/opensearch/sql/expression/config/ExpressionConfig.java index 76e0eb0326..886d7ae399 100644 --- a/core/src/main/java/org/opensearch/sql/expression/config/ExpressionConfig.java +++ b/core/src/main/java/org/opensearch/sql/expression/config/ExpressionConfig.java @@ -20,8 +20,10 @@ import org.opensearch.sql.expression.operator.predicate.UnaryPredicateOperator; import org.opensearch.sql.expression.text.TextFunction; import org.opensearch.sql.expression.window.WindowFunctions; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Scope; /** * Expression Config for Spring IoC. @@ -32,6 +34,7 @@ public class ExpressionConfig { * BuiltinFunctionRepository constructor. */ @Bean + @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) public BuiltinFunctionRepository functionRepository() { BuiltinFunctionRepository builtinFunctionRepository = new BuiltinFunctionRepository(new HashMap<>()); @@ -50,6 +53,7 @@ public BuiltinFunctionRepository functionRepository() { } @Bean + @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) public DSL dsl(BuiltinFunctionRepository repository) { return new DSL(repository); } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSQLQueryAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSQLQueryAction.java index 0db08398b8..6524ac2f0e 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSQLQueryAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSQLQueryAction.java @@ -11,23 +11,17 @@ import static org.opensearch.sql.executor.ExecutionEngine.QueryResponse; import static org.opensearch.sql.protocol.response.format.JsonResponseFormatter.Style.PRETTY; -import java.io.IOException; -import java.security.PrivilegedExceptionAction; import java.util.List; -import javax.xml.catalog.Catalog; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.client.node.NodeClient; -import org.opensearch.cluster.service.ClusterService; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestStatus; -import org.opensearch.sql.catalog.CatalogService; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.response.ResponseListener; -import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponse; import org.opensearch.sql.legacy.metrics.MetricName; import org.opensearch.sql.legacy.metrics.Metrics; @@ -41,7 +35,6 @@ import org.opensearch.sql.protocol.response.format.RawResponseFormatter; import org.opensearch.sql.protocol.response.format.ResponseFormatter; import org.opensearch.sql.sql.SQLService; -import org.opensearch.sql.sql.config.SQLServiceConfig; import org.opensearch.sql.sql.domain.SQLQueryRequest; import org.springframework.context.annotation.AnnotationConfigApplicationContext; @@ -56,23 +49,14 @@ public class RestSQLQueryAction extends BaseRestHandler { public static final RestChannelConsumer NOT_SUPPORTED_YET = null; - private final ClusterService clusterService; - - /** - * Settings required by been initialization. - */ - private final Settings pluginSettings; - - private final CatalogService catalogService; + private final AnnotationConfigApplicationContext applicationContext; /** * Constructor of RestSQLQueryAction. */ - public RestSQLQueryAction(ClusterService clusterService, Settings pluginSettings, CatalogService catalogService) { + public RestSQLQueryAction(AnnotationConfigApplicationContext applicationContext) { super(); - this.clusterService = clusterService; - this.pluginSettings = pluginSettings; - this.catalogService = catalogService; + this.applicationContext = applicationContext; } @Override @@ -101,7 +85,8 @@ public RestChannelConsumer prepareRequest(SQLQueryRequest request, NodeClient no return NOT_SUPPORTED_YET; } - SQLService sqlService = createSQLService(nodeClient); + SQLService sqlService = + SecurityAccess.doPrivileged(() -> applicationContext.getBean(SQLService.class)); PhysicalPlan plan; try { // For now analyzing and planning stage may throw syntax exception as well @@ -123,20 +108,6 @@ public RestChannelConsumer prepareRequest(SQLQueryRequest request, NodeClient no return channel -> sqlService.execute(plan, createQueryResponseListener(channel, request)); } - private SQLService createSQLService(NodeClient client) { - return doPrivileged(() -> { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - context.registerBean(ClusterService.class, () -> clusterService); - context.registerBean(NodeClient.class, () -> client); - context.registerBean(Settings.class, () -> pluginSettings); - context.registerBean(CatalogService.class, () -> catalogService); - context.register(OpenSearchSQLPluginConfig.class); - context.register(SQLServiceConfig.class); - context.refresh(); - return context.getBean(SQLService.class); - }); - } - private ResponseListener createExplainResponseListener(RestChannel channel) { return new ResponseListener() { @Override @@ -185,14 +156,6 @@ public void onFailure(Exception e) { }; } - private T doPrivileged(PrivilegedExceptionAction action) { - try { - return SecurityAccess.doPrivileged(action); - } catch (IOException e) { - throw new IllegalStateException("Failed to perform privileged action", e); - } - } - private void sendResponse(RestChannel channel, RestStatus status, String content) { channel.sendResponse(new BytesRestResponse( status, "application/json; charset=UTF-8", content)); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java index ab146404f8..d47dac9325 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java @@ -27,7 +27,6 @@ import org.apache.logging.log4j.Logger; import org.opensearch.client.Client; import org.opensearch.client.node.NodeClient; -import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexNotFoundException; import org.opensearch.rest.BaseRestHandler; @@ -35,7 +34,6 @@ import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestStatus; -import org.opensearch.sql.catalog.CatalogService; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.utils.QueryContext; import org.opensearch.sql.exception.ExpressionEvaluationException; @@ -65,6 +63,7 @@ import org.opensearch.sql.legacy.utils.JsonPrettyFormatter; import org.opensearch.sql.legacy.utils.QueryDataAnonymizer; import org.opensearch.sql.sql.domain.SQLQueryRequest; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; public class RestSqlAction extends BaseRestHandler { @@ -89,12 +88,16 @@ public class RestSqlAction extends BaseRestHandler { */ private final RestSQLQueryAction newSqlQueryHandler; - public RestSqlAction(Settings settings, ClusterService clusterService, - org.opensearch.sql.common.setting.Settings pluginSettings, - CatalogService catalogService) { + /** + * Application context used to create SQLService for each request. + */ + private final AnnotationConfigApplicationContext applicationContext; + + public RestSqlAction(Settings settings, AnnotationConfigApplicationContext applicationContext) { super(); this.allowExplicitIndex = MULTI_ALLOW_EXPLICIT_INDEX.get(settings); - this.newSqlQueryHandler = new RestSQLQueryAction(clusterService, pluginSettings, catalogService); + this.newSqlQueryHandler = new RestSQLQueryAction(applicationContext); + this.applicationContext = applicationContext; } @Override diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/plugin/RestSQLQueryActionTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/plugin/RestSQLQueryActionTest.java index 56d153eb9d..6257f0dd95 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/plugin/RestSQLQueryActionTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/plugin/RestSQLQueryActionTest.java @@ -8,7 +8,6 @@ import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertSame; -import static org.mockito.Mockito.when; import static org.opensearch.sql.legacy.plugin.RestSQLQueryAction.NOT_SUPPORTED_YET; import static org.opensearch.sql.legacy.plugin.RestSqlAction.EXPLAIN_API_ENDPOINT; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; @@ -18,36 +17,38 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import org.opensearch.client.node.NodeClient; -import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.sql.catalog.CatalogService; -import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.executor.ExecutionEngine; +import org.opensearch.sql.sql.config.SQLServiceConfig; import org.opensearch.sql.sql.domain.SQLQueryRequest; +import org.opensearch.sql.storage.StorageEngine; import org.opensearch.threadpool.ThreadPool; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; @RunWith(MockitoJUnitRunner.class) public class RestSQLQueryActionTest { - @Mock - private ClusterService clusterService; - private NodeClient nodeClient; @Mock private ThreadPool threadPool; - @Mock - private Settings settings; - - @Mock - private CatalogService catalogService; + private AnnotationConfigApplicationContext context; @Before public void setup() { nodeClient = new NodeClient(org.opensearch.common.settings.Settings.EMPTY, threadPool); - when(threadPool.getThreadContext()) + context = new AnnotationConfigApplicationContext(); + context.registerBean(StorageEngine.class, () -> Mockito.mock(StorageEngine.class)); + context.registerBean(ExecutionEngine.class, () -> Mockito.mock(ExecutionEngine.class)); + context.registerBean(CatalogService.class, () -> Mockito.mock(CatalogService.class)); + context.register(SQLServiceConfig.class); + context.refresh(); + Mockito.lenient().when(threadPool.getThreadContext()) .thenReturn(new ThreadContext(org.opensearch.common.settings.Settings.EMPTY)); } @@ -59,7 +60,7 @@ public void handleQueryThatCanSupport() { QUERY_API_ENDPOINT, ""); - RestSQLQueryAction queryAction = new RestSQLQueryAction(clusterService, settings, catalogService); + RestSQLQueryAction queryAction = new RestSQLQueryAction(context); assertNotSame(NOT_SUPPORTED_YET, queryAction.prepareRequest(request, nodeClient)); } @@ -71,7 +72,7 @@ public void handleExplainThatCanSupport() { EXPLAIN_API_ENDPOINT, ""); - RestSQLQueryAction queryAction = new RestSQLQueryAction(clusterService, settings, catalogService); + RestSQLQueryAction queryAction = new RestSQLQueryAction(context); assertNotSame(NOT_SUPPORTED_YET, queryAction.prepareRequest(request, nodeClient)); } @@ -84,7 +85,7 @@ public void skipQueryThatNotSupport() { QUERY_API_ENDPOINT, ""); - RestSQLQueryAction queryAction = new RestSQLQueryAction(clusterService, settings, catalogService); + RestSQLQueryAction queryAction = new RestSQLQueryAction(context); assertSame(NOT_SUPPORTED_YET, queryAction.prepareRequest(request, nodeClient)); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/security/SecurityAccess.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/security/SecurityAccess.java index cff219578f..0c1b2e58b1 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/security/SecurityAccess.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/security/SecurityAccess.java @@ -6,7 +6,6 @@ package org.opensearch.sql.opensearch.security; -import java.io.IOException; import java.security.AccessController; import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; @@ -21,13 +20,12 @@ public class SecurityAccess { /** * Execute the operation in privileged mode. */ - public static T doPrivileged(final PrivilegedExceptionAction operation) - throws IOException { + public static T doPrivileged(final PrivilegedExceptionAction operation) { SpecialPermission.check(); try { return AccessController.doPrivileged(operation); } catch (final PrivilegedActionException e) { - throw (IOException) e.getCause(); + throw new IllegalStateException("Failed to perform privileged action", e); } } } diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index 200364580b..a53c52eb41 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -40,30 +40,35 @@ import org.opensearch.script.ScriptContext; import org.opensearch.script.ScriptEngine; import org.opensearch.script.ScriptService; +import org.opensearch.sql.catalog.CatalogService; import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.executor.AsyncRestExecutor; import org.opensearch.sql.legacy.metrics.Metrics; import org.opensearch.sql.legacy.plugin.RestSqlAction; import org.opensearch.sql.legacy.plugin.RestSqlStatsAction; import org.opensearch.sql.opensearch.client.OpenSearchNodeClient; +import org.opensearch.sql.opensearch.security.SecurityAccess; import org.opensearch.sql.opensearch.setting.LegacyOpenDistroSettings; import org.opensearch.sql.opensearch.setting.OpenSearchSettings; import org.opensearch.sql.opensearch.storage.OpenSearchStorageEngine; import org.opensearch.sql.opensearch.storage.script.ExpressionScriptEngine; import org.opensearch.sql.opensearch.storage.serialization.DefaultExpressionSerializer; import org.opensearch.sql.plugin.catalog.CatalogServiceImpl; -import org.opensearch.sql.plugin.catalog.CatalogSettings; +import org.opensearch.sql.plugin.config.OpenSearchPluginConfig; import org.opensearch.sql.plugin.rest.RestPPLQueryAction; import org.opensearch.sql.plugin.rest.RestPPLStatsAction; import org.opensearch.sql.plugin.rest.RestQuerySettingsAction; import org.opensearch.sql.plugin.transport.PPLQueryAction; import org.opensearch.sql.plugin.transport.TransportPPLQueryAction; import org.opensearch.sql.plugin.transport.TransportPPLQueryResponse; +import org.opensearch.sql.ppl.config.PPLServiceConfig; +import org.opensearch.sql.sql.config.SQLServiceConfig; import org.opensearch.sql.storage.StorageEngine; import org.opensearch.threadpool.ExecutorBuilder; import org.opensearch.threadpool.FixedExecutorBuilder; import org.opensearch.threadpool.ThreadPool; import org.opensearch.watcher.ResourceWatcherService; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; public class SQLPlugin extends Plugin implements ActionPlugin, ScriptPlugin, ReloadablePlugin { @@ -76,6 +81,8 @@ public class SQLPlugin extends Plugin implements ActionPlugin, ScriptPlugin, Rel private NodeClient client; + private AnnotationConfigApplicationContext applicationContext; + public String name() { return "sql"; } @@ -101,8 +108,7 @@ public List getRestHandlers( return Arrays.asList( new RestPPLQueryAction(pluginSettings, settings), - new RestSqlAction(settings, clusterService, pluginSettings, - CatalogServiceImpl.getInstance()), + new RestSqlAction(settings, applicationContext), new RestSqlStatsAction(settings, restController), new RestPPLStatsAction(settings, restController), new RestQuerySettingsAction(settings, restController)); @@ -137,21 +143,29 @@ public Collection createComponents( this.client = (NodeClient) client; CatalogServiceImpl.getInstance().loadConnectors(clusterService.getSettings()); CatalogServiceImpl.getInstance().registerOpenSearchStorageEngine(openSearchStorageEngine()); + + this.applicationContext = new AnnotationConfigApplicationContext(); + SecurityAccess.doPrivileged( + () -> { + applicationContext.registerBean(ClusterService.class, () -> clusterService); + applicationContext.registerBean(NodeClient.class, () -> (NodeClient) client); + applicationContext.registerBean( + org.opensearch.sql.common.setting.Settings.class, () -> pluginSettings); + applicationContext.registerBean( + CatalogService.class, () -> CatalogServiceImpl.getInstance()); + applicationContext.register(OpenSearchPluginConfig.class); + applicationContext.register(PPLServiceConfig.class); + applicationContext.register(SQLServiceConfig.class); + applicationContext.refresh(); + return null; + }); + LocalClusterState.state().setClusterService(clusterService); LocalClusterState.state().setPluginSettings((OpenSearchSettings) pluginSettings); - return super.createComponents( - client, - clusterService, - threadPool, - resourceWatcherService, - scriptService, - contentRegistry, - environment, - nodeEnvironment, - namedWriteableRegistry, - indexNameResolver, - repositoriesServiceSupplier); + // return objects used by Guice to inject dependencies for e.g., + // transport action handler constructors + return ImmutableList.of(applicationContext); } @Override @@ -170,7 +184,6 @@ public List> getSettings() { return new ImmutableList.Builder>() .addAll(LegacyOpenDistroSettings.legacySettings()) .addAll(OpenSearchSettings.pluginSettings()) - .add(CatalogSettings.CATALOG_CONFIG) .build(); } @@ -186,8 +199,7 @@ public void reload(Settings settings) { } private StorageEngine openSearchStorageEngine() { - return new OpenSearchStorageEngine(new OpenSearchNodeClient(client), - pluginSettings); + return new OpenSearchStorageEngine(new OpenSearchNodeClient(client), pluginSettings); } } diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogServiceImpl.java b/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogServiceImpl.java index 5a77961d8b..4a922e65ca 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogServiceImpl.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogServiceImpl.java @@ -11,7 +11,6 @@ import java.io.IOException; import java.io.InputStream; import java.net.URISyntaxException; -import java.security.PrivilegedExceptionAction; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -54,7 +53,7 @@ private CatalogServiceImpl() { * @param settings settings. */ public void loadConnectors(Settings settings) { - doPrivileged(() -> { + SecurityAccess.doPrivileged(() -> { InputStream inputStream = CatalogSettings.CATALOG_CONFIG.get(settings); if (inputStream != null) { ObjectMapper objectMapper = new ObjectMapper(); @@ -96,14 +95,6 @@ public void registerOpenSearchStorageEngine(StorageEngine storageEngine) { storageEngineMap.put(OPEN_SEARCH, storageEngine); } - private T doPrivileged(PrivilegedExceptionAction action) { - try { - return SecurityAccess.doPrivileged(action); - } catch (IOException e) { - throw new IllegalStateException("Failed to perform privileged action", e); - } - } - private StorageEngine createStorageEngine(CatalogMetadata catalog) throws URISyntaxException { StorageEngine storageEngine; ConnectorType connector = catalog.getConnector(); @@ -165,4 +156,4 @@ private void validateCatalogs(List catalogs) { } -} \ No newline at end of file +} diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/OpenSearchSQLPluginConfig.java b/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginConfig.java similarity index 73% rename from legacy/src/main/java/org/opensearch/sql/legacy/plugin/OpenSearchSQLPluginConfig.java rename to plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginConfig.java index b396d896b0..a2169eb839 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/OpenSearchSQLPluginConfig.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginConfig.java @@ -1,10 +1,13 @@ /* - * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ -package org.opensearch.sql.legacy.plugin; +package org.opensearch.sql.plugin.config; import org.opensearch.client.node.NodeClient; import org.opensearch.sql.common.setting.Settings; @@ -23,16 +26,19 @@ import org.opensearch.sql.opensearch.storage.OpenSearchStorageEngine; import org.opensearch.sql.storage.StorageEngine; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Scope; /** - * OpenSearch Plugin Config for SQL. + * OpenSearch plugin config that injects cluster service and node client from plugin + * and initialize OpenSearch storage and execution engine. */ @Configuration @Import({ExpressionConfig.class}) -public class OpenSearchSQLPluginConfig { +public class OpenSearchPluginConfig { @Autowired private NodeClient nodeClient; @@ -44,27 +50,32 @@ public class OpenSearchSQLPluginConfig { private BuiltinFunctionRepository functionRepository; @Bean + @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) public OpenSearchClient client() { return new OpenSearchNodeClient(nodeClient); } @Bean + @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) public StorageEngine storageEngine() { return new OpenSearchStorageEngine(client(), settings); } @Bean + @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) public ExecutionEngine executionEngine() { OpenSearchFunctions.register(functionRepository); return new OpenSearchExecutionEngine(client(), protector()); } @Bean + @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) public ResourceMonitor resourceMonitor() { return new OpenSearchResourceMonitor(settings, new OpenSearchMemoryHealthy()); } @Bean + @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) public ExecutionProtector protector() { return new OpenSearchExecutionProtector(resourceMonitor()); } diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/OpenSearchPluginConfig.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/OpenSearchPluginConfig.java deleted file mode 100644 index 24d7e4e7f5..0000000000 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/OpenSearchPluginConfig.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.plugin.rest; - -import org.opensearch.client.node.NodeClient; -import org.opensearch.sql.common.setting.Settings; -import org.opensearch.sql.executor.ExecutionEngine; -import org.opensearch.sql.monitor.ResourceMonitor; -import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.client.OpenSearchNodeClient; -import org.opensearch.sql.opensearch.executor.OpenSearchExecutionEngine; -import org.opensearch.sql.opensearch.executor.protector.ExecutionProtector; -import org.opensearch.sql.opensearch.executor.protector.OpenSearchExecutionProtector; -import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; -import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; -import org.opensearch.sql.opensearch.storage.OpenSearchStorageEngine; -import org.opensearch.sql.storage.StorageEngine; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -/** - * OpenSearch plugin config that injects cluster service and node client from plugin - * and initialize OpenSearch storage and execution engine. - */ -@Configuration -public class OpenSearchPluginConfig { - - @Autowired - private NodeClient nodeClient; - - @Autowired - private Settings settings; - - @Bean - public OpenSearchClient client() { - return new OpenSearchNodeClient(nodeClient); - } - - @Bean - public ExecutionEngine executionEngine() { - return new OpenSearchExecutionEngine(client(), protector()); - } - - @Bean - public ResourceMonitor resourceMonitor() { - return new OpenSearchResourceMonitor(settings, new OpenSearchMemoryHealthy()); - } - - @Bean - public ExecutionProtector protector() { - return new OpenSearchExecutionProtector(resourceMonitor()); - } -} diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java index e9202d96e8..2bc3c8d72d 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java @@ -39,7 +39,6 @@ import org.opensearch.sql.plugin.transport.PPLQueryAction; import org.opensearch.sql.plugin.transport.TransportPPLQueryRequest; import org.opensearch.sql.plugin.transport.TransportPPLQueryResponse; -import org.opensearch.sql.ppl.domain.PPLQueryRequest; public class RestPPLQueryAction extends BaseRestHandler { public static final String QUERY_API_ENDPOINT = "/_plugins/_ppl"; diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java index eaad009216..af57c91e5c 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java @@ -7,8 +7,6 @@ import static org.opensearch.sql.protocol.response.format.JsonResponseFormatter.Style.PRETTY; -import java.io.IOException; -import java.security.PrivilegedExceptionAction; import java.util.Locale; import java.util.Optional; import org.opensearch.action.ActionListener; @@ -18,7 +16,6 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.sql.catalog.CatalogService; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.utils.QueryContext; @@ -27,10 +24,7 @@ import org.opensearch.sql.legacy.metrics.Metrics; import org.opensearch.sql.opensearch.security.SecurityAccess; import org.opensearch.sql.opensearch.setting.OpenSearchSettings; -import org.opensearch.sql.plugin.catalog.CatalogServiceImpl; -import org.opensearch.sql.plugin.rest.OpenSearchPluginConfig; import org.opensearch.sql.ppl.PPLService; -import org.opensearch.sql.ppl.config.PPLServiceConfig; import org.opensearch.sql.ppl.domain.PPLQueryRequest; import org.opensearch.sql.protocol.response.QueryResult; import org.opensearch.sql.protocol.response.format.CsvResponseFormatter; @@ -55,6 +49,8 @@ public class TransportPPLQueryAction /** Settings required by been initialization. */ private final Settings pluginSettings; + private final AnnotationConfigApplicationContext applicationContext; + /** Constructor of TransportPPLQueryAction. */ @Inject @@ -63,11 +59,12 @@ public TransportPPLQueryAction( ActionFilters actionFilters, NodeClient client, ClusterService clusterService, - org.opensearch.common.settings.Settings clusterSettings) { + AnnotationConfigApplicationContext applicationContext) { super(PPLQueryAction.NAME, transportService, actionFilters, TransportPPLQueryRequest::new); this.client = client; this.clusterService = clusterService; this.pluginSettings = new OpenSearchSettings(clusterService.getClusterSettings()); + this.applicationContext = applicationContext; } /** @@ -82,7 +79,8 @@ protected void doExecute( QueryContext.addRequestId(); - PPLService pplService = createPPLService(client); + PPLService pplService = + SecurityAccess.doPrivileged(() -> applicationContext.getBean(PPLService.class)); TransportPPLQueryRequest transportRequest = TransportPPLQueryRequest.fromActionRequest(request); // in order to use PPL service, we need to convert TransportPPLQueryRequest to PPLQueryRequest PPLQueryRequest transformedRequest = transportRequest.toPPLQueryRequest(); @@ -94,29 +92,6 @@ protected void doExecute( } } - private PPLService createPPLService(NodeClient client) { - return doPrivileged( - () -> { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - context.registerBean(ClusterService.class, () -> clusterService); - context.registerBean(NodeClient.class, () -> client); - context.registerBean(Settings.class, () -> pluginSettings); - context.registerBean(CatalogService.class, CatalogServiceImpl::getInstance); - context.register(OpenSearchPluginConfig.class); - context.register(PPLServiceConfig.class); - context.refresh(); - return context.getBean(PPLService.class); - }); - } - - private T doPrivileged(PrivilegedExceptionAction action) { - try { - return SecurityAccess.doPrivileged(action); - } catch (IOException e) { - throw new IllegalStateException("Failed to perform privileged action", e); - } - } - /** * TODO: need to extract an interface for both SQL and PPL action handler and move these common * methods to the interface. This is not easy to do now because SQL action handler is still in diff --git a/ppl/build.gradle b/ppl/build.gradle index 12b0787efc..2c3c648478 100644 --- a/ppl/build.gradle +++ b/ppl/build.gradle @@ -46,7 +46,6 @@ dependencies { implementation "org.antlr:antlr4-runtime:4.7.1" implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' - api group: 'org.opensearch', name: 'opensearch-x-content', version: "${opensearch_version}" api group: 'org.json', name: 'json', version: '20180813' implementation group: 'org.springframework', name: 'spring-context', version: "${spring_version}" implementation group: 'org.springframework', name: 'spring-beans', version: "${spring_version}" diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/config/PPLServiceConfig.java b/ppl/src/main/java/org/opensearch/sql/ppl/config/PPLServiceConfig.java index bd6c4e3937..1c01ccce89 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/config/PPLServiceConfig.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/config/PPLServiceConfig.java @@ -13,9 +13,11 @@ import org.opensearch.sql.ppl.PPLService; import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Scope; @Configuration @Import({ExpressionConfig.class}) @@ -37,6 +39,7 @@ public class PPLServiceConfig { * @return PPLService. */ @Bean + @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) public PPLService pplService() { return new PPLService(new PPLSyntaxParser(), executionEngine, functionRepository, catalogService); diff --git a/sql/src/main/java/org/opensearch/sql/sql/config/SQLServiceConfig.java b/sql/src/main/java/org/opensearch/sql/sql/config/SQLServiceConfig.java index 2d22d92081..f7845701d2 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/config/SQLServiceConfig.java +++ b/sql/src/main/java/org/opensearch/sql/sql/config/SQLServiceConfig.java @@ -14,11 +14,12 @@ import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.sql.SQLService; import org.opensearch.sql.sql.antlr.SQLSyntaxParser; -import org.opensearch.sql.storage.StorageEngine; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Scope; /** * SQL service configuration for Spring container initialization. @@ -36,11 +37,6 @@ public class SQLServiceConfig { @Autowired private BuiltinFunctionRepository functionRepository; - @Bean - public Analyzer analyzer() { - return new Analyzer(new ExpressionAnalyzer(functionRepository), catalogService); - } - /** * The registration of OpenSearch storage engine happens here because * OpenSearchStorageEngine is dependent on NodeClient. @@ -48,8 +44,12 @@ public Analyzer analyzer() { * @return SQLService. */ @Bean + @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) public SQLService sqlService() { - return new SQLService(new SQLSyntaxParser(), analyzer(), executionEngine, + return new SQLService( + new SQLSyntaxParser(), + new Analyzer(new ExpressionAnalyzer(functionRepository), catalogService), + executionEngine, functionRepository); } From 213b6ae3c285eef534997a55283ee176e74672f7 Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Wed, 21 Sep 2022 09:15:39 -0700 Subject: [PATCH 02/23] Add low-level create table and table exists API (#834) * Add create index function in opensearch node client Signed-off-by: Chen Dai * Add create index function in opensearch rest client Signed-off-by: Chen Dai * Add create index function in OpenSearchIndex table Signed-off-by: Chen Dai * Separate index exist API from create index function Signed-off-by: Chen Dai * Separate table exist API from create table function Signed-off-by: Chen Dai * Add UT and Todo comment for OpenSearch system index Signed-off-by: Chen Dai * Fix broken UT in core due to new API in Table interface Signed-off-by: Chen Dai * Add more UT and javadoc for OpenSearch data type mapping Signed-off-by: Chen Dai * Rename isExist to exists which looks more readable Signed-off-by: Chen Dai Signed-off-by: Chen Dai --- .../org/opensearch/sql/storage/Table.java | 12 ++ .../sql/analysis/AnalyzerTestBase.java | 10 ++ .../org/opensearch/sql/config/TestConfig.java | 10 ++ .../opensearch/sql/planner/PlannerTest.java | 10 ++ .../opensearch/client/OpenSearchClient.java | 14 +++ .../client/OpenSearchNodeClient.java | 25 ++++ .../client/OpenSearchRestClient.java | 21 ++++ .../data/type/OpenSearchDataType.java | 57 +++++++++ .../OpenSearchDescribeIndexRequest.java | 30 +---- .../opensearch/storage/OpenSearchIndex.java | 19 +++ .../storage/system/OpenSearchSystemIndex.java | 11 ++ .../client/OpenSearchNodeClientTest.java | 113 ++++++++++++------ .../client/OpenSearchRestClientTest.java | 49 +++++++- .../data/type/OpenSearchDataTypeTest.java | 22 ++++ .../storage/OpenSearchIndexTest.java | 62 +++++----- .../system/OpenSearchSystemIndexTest.java | 15 +++ 16 files changed, 385 insertions(+), 95 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/storage/Table.java b/core/src/main/java/org/opensearch/sql/storage/Table.java index 731cf878c6..7cdd757cbb 100644 --- a/core/src/main/java/org/opensearch/sql/storage/Table.java +++ b/core/src/main/java/org/opensearch/sql/storage/Table.java @@ -16,6 +16,18 @@ */ public interface Table { + /** + * Check if current table exists. + * @return true if exists, otherwise false + */ + boolean exists(); + + /** + * Create table given table schema. + * @param schema table schema + */ + void create(Map schema); + /** * Get the {@link ExprType} for each field in the table. */ diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java index 3f912b8fde..0a743ac765 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java @@ -51,6 +51,16 @@ public Table getTable(String name) { @Bean protected Table table() { return new Table() { + @Override + public boolean exists() { + return true; + } + + @Override + public void create(Map schema) { + throw new UnsupportedOperationException("Create table is not supported"); + } + @Override public Map getFieldTypes() { return typeMapping(); diff --git a/core/src/test/java/org/opensearch/sql/config/TestConfig.java b/core/src/test/java/org/opensearch/sql/config/TestConfig.java index ab78109aa2..449b40409f 100644 --- a/core/src/test/java/org/opensearch/sql/config/TestConfig.java +++ b/core/src/test/java/org/opensearch/sql/config/TestConfig.java @@ -63,6 +63,16 @@ protected StorageEngine storageEngine() { @Override public Table getTable(String name) { return new Table() { + @Override + public boolean exists() { + return true; + } + + @Override + public void create(Map schema) { + throw new UnsupportedOperationException("Create table is not supported"); + } + @Override public Map getFieldTypes() { return typeMapping; diff --git a/core/src/test/java/org/opensearch/sql/planner/PlannerTest.java b/core/src/test/java/org/opensearch/sql/planner/PlannerTest.java index 32e9d1b45b..e02231ca06 100644 --- a/core/src/test/java/org/opensearch/sql/planner/PlannerTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/PlannerTest.java @@ -119,6 +119,16 @@ protected PhysicalPlan analyze(LogicalPlan logicalPlan) { protected class MockTable extends LogicalPlanNodeVisitor implements Table { + @Override + public boolean exists() { + return true; + } + + @Override + public void create(Map schema) { + throw new UnsupportedOperationException("Create table is not supported"); + } + @Override public Map getFieldTypes() { throw new UnsupportedOperationException(); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java index 09a83f65a5..dc6e72bd91 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java @@ -22,6 +22,20 @@ public interface OpenSearchClient { String META_CLUSTER_NAME = "CLUSTER_NAME"; + /** + * Check if the given index exists. + * @param indexName index name + * @return true if exists, otherwise false + */ + boolean exists(String indexName); + + /** + * Create OpenSearch index based on the given mappings. + * @param indexName index name + * @param mappings index mappings + */ + void createIndex(String indexName, Map mappings); + /** * Fetch index mapping(s) according to index expression given. * diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index 80a2fb8604..13fb0f6391 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -18,6 +18,9 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.opensearch.action.admin.indices.get.GetIndexResponse; import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; @@ -50,6 +53,28 @@ public OpenSearchNodeClient(NodeClient client) { this.resolver = new IndexNameExpressionResolver(client.threadPool().getThreadContext()); } + @Override + public boolean exists(String indexName) { + try { + IndicesExistsResponse checkExistResponse = client.admin().indices() + .exists(new IndicesExistsRequest(indexName)).actionGet(); + return checkExistResponse.isExists(); + } catch (Exception e) { + throw new IllegalStateException("Failed to check if index [" + indexName + "] exists", e); + } + } + + @Override + public void createIndex(String indexName, Map mappings) { + try { + // TODO: 1.pass index settings (the number of primary shards, etc); 2.check response? + CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName).mapping(mappings); + client.admin().indices().create(createIndexRequest).actionGet(); + } catch (Exception e) { + throw new IllegalStateException("Failed to create index [" + indexName + "]", e); + } + } + /** * Get field mappings of index by an index expression. Majority is copied from legacy * LocalClusterState. diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java index f354215e05..d9f9dbbe5d 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java @@ -23,6 +23,7 @@ import org.opensearch.action.search.ClearScrollRequest; import org.opensearch.client.RequestOptions; import org.opensearch.client.RestHighLevelClient; +import org.opensearch.client.indices.CreateIndexRequest; import org.opensearch.client.indices.GetIndexRequest; import org.opensearch.client.indices.GetIndexResponse; import org.opensearch.client.indices.GetMappingsRequest; @@ -46,6 +47,26 @@ public class OpenSearchRestClient implements OpenSearchClient { /** OpenSearch high level REST client. */ private final RestHighLevelClient client; + @Override + public boolean exists(String indexName) { + try { + return client.indices().exists( + new GetIndexRequest(indexName), RequestOptions.DEFAULT); + } catch (IOException e) { + throw new IllegalStateException("Failed to check if index [" + indexName + "] exist", e); + } + } + + @Override + public void createIndex(String indexName, Map mappings) { + try { + client.indices().create( + new CreateIndexRequest(indexName).mapping(mappings), RequestOptions.DEFAULT); + } catch (IOException e) { + throw new IllegalStateException("Failed to create index [" + indexName + "]", e); + } + } + @Override public Map getIndexMappings(String... indexExpression) { GetMappingsRequest request = new GetMappingsRequest().indices(indexExpression); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java index f620ae3aaf..05b80bfa23 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java @@ -9,12 +9,15 @@ import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.UNKNOWN; +import com.google.common.collect.BiMap; +import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableMap; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import lombok.RequiredArgsConstructor; +import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; /** @@ -52,6 +55,39 @@ public boolean shouldCast(ExprType other) { OPENSEARCH_BINARY(Arrays.asList(UNKNOWN), "binary"); + /** + * Bidirectional mapping between OpenSearch type name and ExprType. + */ + private static final BiMap OPENSEARCH_TYPE_TO_EXPR_TYPE_MAPPING = + ImmutableBiMap.builder() + .put("text", OPENSEARCH_TEXT) + .put("text_keyword", OPENSEARCH_TEXT_KEYWORD) + .put("keyword", ExprCoreType.STRING) + .put("byte", ExprCoreType.BYTE) + .put("short", ExprCoreType.SHORT) + .put("integer", ExprCoreType.INTEGER) + .put("long", ExprCoreType.LONG) + .put("float", ExprCoreType.FLOAT) + .put("double", ExprCoreType.DOUBLE) + .put("boolean", ExprCoreType.BOOLEAN) + .put("nested", ExprCoreType.ARRAY) + .put("object", ExprCoreType.STRUCT) + .put("date", ExprCoreType.TIMESTAMP) + .put("ip", OPENSEARCH_IP) + .put("geo_point", OPENSEARCH_GEO_POINT) + .put("binary", OPENSEARCH_BINARY) + .build(); + + /** + * Mapping from extra OpenSearch type name which may map to same ExprType as above. + */ + private static final Map EXTRA_OPENSEARCH_TYPE_TO_EXPR_TYPE_MAPPING = + ImmutableMap.builder() + .put("half_float", ExprCoreType.FLOAT) + .put("scaled_float", ExprCoreType.DOUBLE) + .put("date_nanos", ExprCoreType.TIMESTAMP) + .build(); + /** * The mapping between Type and legacy JDBC type name. */ @@ -70,6 +106,27 @@ public boolean shouldCast(ExprType other) { */ private final String jdbcType; + /** + * Convert OpenSearch type string to ExprType. + * @param openSearchType OpenSearch type string + * @return expr type + */ + public static ExprType getExprType(String openSearchType) { + if (OPENSEARCH_TYPE_TO_EXPR_TYPE_MAPPING.containsKey(openSearchType)) { + return OPENSEARCH_TYPE_TO_EXPR_TYPE_MAPPING.get(openSearchType); + } + return EXTRA_OPENSEARCH_TYPE_TO_EXPR_TYPE_MAPPING.getOrDefault(openSearchType, UNKNOWN); + } + + /** + * Convert ExprType to OpenSearch type string. + * @param type expr type + * @return OpenSearch type string + */ + public static String getOpenSearchType(ExprType type) { + return OPENSEARCH_TYPE_TO_EXPR_TYPE_MAPPING.inverse().get(type); + } + @Override public List getParent() { return parents; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequest.java index f321497099..50402fc75b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequest.java @@ -10,7 +10,6 @@ import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; import static org.opensearch.sql.opensearch.client.OpenSearchClient.META_CLUSTER_NAME; -import com.google.common.collect.ImmutableMap; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; @@ -39,33 +38,6 @@ public class OpenSearchDescribeIndexRequest implements OpenSearchSystemRequest { private static final String DEFAULT_IS_AUTOINCREMENT = "NO"; - /** - * Type mapping from OpenSearch data type to expression type in our type system in query - * engine. TODO: geo, ip etc. - */ - private static final Map OPENSEARCH_TYPE_TO_EXPR_TYPE_MAPPING = - ImmutableMap.builder() - .put("text", OpenSearchDataType.OPENSEARCH_TEXT) - .put("text_keyword", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD) - .put("keyword", ExprCoreType.STRING) - .put("byte", ExprCoreType.BYTE) - .put("short", ExprCoreType.SHORT) - .put("integer", ExprCoreType.INTEGER) - .put("long", ExprCoreType.LONG) - .put("float", ExprCoreType.FLOAT) - .put("half_float", ExprCoreType.FLOAT) - .put("scaled_float", ExprCoreType.DOUBLE) - .put("double", ExprCoreType.DOUBLE) - .put("boolean", ExprCoreType.BOOLEAN) - .put("nested", ExprCoreType.ARRAY) - .put("object", ExprCoreType.STRUCT) - .put("date", ExprCoreType.TIMESTAMP) - .put("date_nanos", ExprCoreType.TIMESTAMP) - .put("ip", OpenSearchDataType.OPENSEARCH_IP) - .put("geo_point", OpenSearchDataType.OPENSEARCH_GEO_POINT) - .put("binary", OpenSearchDataType.OPENSEARCH_BINARY) - .build(); - /** * OpenSearch client connection. */ @@ -132,7 +104,7 @@ public Integer getMaxResultWindow() { } private ExprType transformESTypeToExprType(String openSearchType) { - return OPENSEARCH_TYPE_TO_EXPR_TYPE_MAPPING.getOrDefault(openSearchType, ExprCoreType.UNKNOWN); + return OpenSearchDataType.getExprType(openSearchType); } private ExprTupleValue row(String fieldName, String fieldType, int position, String clusterName) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java index ef6159020f..88b823dd1f 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java @@ -7,6 +7,7 @@ package org.opensearch.sql.opensearch.storage; import com.google.common.annotations.VisibleForTesting; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -18,6 +19,7 @@ import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.planner.logical.OpenSearchLogicalIndexAgg; import org.opensearch.sql.opensearch.planner.logical.OpenSearchLogicalIndexScan; @@ -72,6 +74,23 @@ public OpenSearchIndex(OpenSearchClient client, Settings settings, String indexN this.indexName = new OpenSearchRequest.IndexName(indexName); } + @Override + public boolean exists() { + return client.exists(indexName.toString()); + } + + @Override + public void create(Map schema) { + Map mappings = new HashMap<>(); + Map properties = new HashMap<>(); + mappings.put("properties", properties); + + for (Map.Entry colType : schema.entrySet()) { + properties.put(colType.getKey(), OpenSearchDataType.getOpenSearchType(colType.getValue())); + } + client.createIndex(indexName.toString(), mappings); + } + /* * TODO: Assume indexName doesn't have wildcard. * Need to either handle field name conflicts diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java index edd5593f4d..7b6efeeba4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java @@ -38,6 +38,17 @@ public OpenSearchSystemIndex( this.systemIndexBundle = buildIndexBundle(client, indexName); } + @Override + public boolean exists() { + return true; // TODO: implement for system index later + } + + @Override + public void create(Map schema) { + throw new UnsupportedOperationException( + "OpenSearch system index is predefined and cannot be created"); + } + @Override public Map getFieldTypes() { return systemIndexBundle.getLeft().getMapping(); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java index ad26d792ed..c3cd30a530 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java @@ -32,12 +32,17 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.lucene.search.TotalHits; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.opensearch.action.admin.indices.get.GetIndexResponse; import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; @@ -88,12 +93,66 @@ class OpenSearchNodeClientTest { private ExprTupleValue exprTupleValue = ExprTupleValue.fromExprValueMap(ImmutableMap.of("id", new ExprIntegerValue(1))); + private OpenSearchClient client; + + @BeforeEach + void setUp() { + this.client = new OpenSearchNodeClient(nodeClient); + } + + @Test + void isIndexExist() { + when(nodeClient.admin().indices() + .exists(any(IndicesExistsRequest.class)).actionGet()) + .thenReturn(new IndicesExistsResponse(true)); + + assertTrue(client.exists("test")); + } + + @Test + void isIndexNotExist() { + String indexName = "test"; + when(nodeClient.admin().indices() + .exists(any(IndicesExistsRequest.class)).actionGet()) + .thenReturn(new IndicesExistsResponse(false)); + + assertFalse(client.exists(indexName)); + } + + @Test + void isIndexExistWithException() { + when(nodeClient.admin().indices().exists(any())).thenThrow(RuntimeException.class); + + assertThrows(IllegalStateException.class, () -> client.exists("test")); + } + @Test - public void getIndexMappings() throws IOException { + void createIndex() { + String indexName = "test"; + Map mappings = ImmutableMap.of( + "properties", + ImmutableMap.of("name", "text")); + when(nodeClient.admin().indices() + .create(any(CreateIndexRequest.class)).actionGet()) + .thenReturn(new CreateIndexResponse(true, true, indexName)); + + client.createIndex(indexName, mappings); + } + + @Test + void createIndexWithException() { + when(nodeClient.admin().indices().create(any())).thenThrow(RuntimeException.class); + + assertThrows(IllegalStateException.class, + () -> client.createIndex("test", ImmutableMap.of())); + } + + @Test + void getIndexMappings() throws IOException { URL url = Resources.getResource(TEST_MAPPING_FILE); String mappings = Resources.toString(url, Charsets.UTF_8); String indexName = "test"; - OpenSearchNodeClient client = mockClient(indexName, mappings); + mockNodeClientIndicesMappings(indexName, mappings); Map indexMappings = client.getIndexMappings(indexName); assertEquals(1, indexMappings.size()); @@ -121,9 +180,9 @@ public void getIndexMappings() throws IOException { } @Test - public void getIndexMappingsWithEmptyMapping() { + void getIndexMappingsWithEmptyMapping() { String indexName = "test"; - OpenSearchNodeClient client = mockClient(indexName, ""); + mockNodeClientIndicesMappings(indexName, ""); Map indexMappings = client.getIndexMappings(indexName); assertEquals(1, indexMappings.size()); @@ -132,28 +191,25 @@ public void getIndexMappingsWithEmptyMapping() { } @Test - public void getIndexMappingsWithIOException() { + void getIndexMappingsWithIOException() { String indexName = "test"; when(nodeClient.admin().indices()).thenThrow(RuntimeException.class); - OpenSearchNodeClient client = new OpenSearchNodeClient(nodeClient); assertThrows(IllegalStateException.class, () -> client.getIndexMappings(indexName)); } @Test - public void getIndexMappingsWithNonExistIndex() { - OpenSearchNodeClient client = - new OpenSearchNodeClient(mockNodeClient("test")); + void getIndexMappingsWithNonExistIndex() { + mockNodeClient("test"); assertTrue(client.getIndexMappings("non_exist_index").isEmpty()); } @Test - public void getIndexMaxResultWindows() throws IOException { + void getIndexMaxResultWindows() throws IOException { URL url = Resources.getResource(TEST_MAPPING_SETTINGS_FILE); String indexMetadata = Resources.toString(url, Charsets.UTF_8); String indexName = "accounts"; - OpenSearchNodeClient client = - new OpenSearchNodeClient(mockNodeClientSettings(indexName, indexMetadata)); + mockNodeClientSettings(indexName, indexMetadata); Map indexMaxResultWindows = client.getIndexMaxResultWindows(indexName); assertEquals(1, indexMaxResultWindows.size()); @@ -163,12 +219,11 @@ public void getIndexMaxResultWindows() throws IOException { } @Test - public void getIndexMaxResultWindowsWithDefaultSettings() throws IOException { + void getIndexMaxResultWindowsWithDefaultSettings() throws IOException { URL url = Resources.getResource(TEST_MAPPING_FILE); String indexMetadata = Resources.toString(url, Charsets.UTF_8); String indexName = "accounts"; - OpenSearchNodeClient client = - new OpenSearchNodeClient(mockNodeClientSettings(indexName, indexMetadata)); + mockNodeClientSettings(indexName, indexMetadata); Map indexMaxResultWindows = client.getIndexMaxResultWindows(indexName); assertEquals(1, indexMaxResultWindows.size()); @@ -178,25 +233,21 @@ public void getIndexMaxResultWindowsWithDefaultSettings() throws IOException { } @Test - public void getIndexMaxResultWindowsWithIOException() { + void getIndexMaxResultWindowsWithIOException() { String indexName = "test"; when(nodeClient.admin().indices()).thenThrow(RuntimeException.class); - OpenSearchNodeClient client = new OpenSearchNodeClient(nodeClient); assertThrows(IllegalStateException.class, () -> client.getIndexMaxResultWindows(indexName)); } /** Jacoco enforce this constant lambda be tested. */ @Test - public void testAllFieldsPredicate() { + void testAllFieldsPredicate() { assertTrue(OpenSearchNodeClient.ALL_FIELDS.apply("any_index").test("any_field")); } @Test - public void search() { - OpenSearchNodeClient client = - new OpenSearchNodeClient(nodeClient); - + void search() { // Mock first scroll request SearchResponse searchResponse = mock(SearchResponse.class); when(nodeClient.search(any()).actionGet()).thenReturn(searchResponse); @@ -233,7 +284,6 @@ public void search() { @Test void schedule() { - OpenSearchNodeClient client = new OpenSearchNodeClient(nodeClient); AtomicBoolean isRun = new AtomicBoolean(false); client.schedule( () -> { @@ -249,7 +299,6 @@ void cleanup() { when(requestBuilder.addScrollId(any())).thenReturn(requestBuilder); when(requestBuilder.get()).thenReturn(null); - OpenSearchNodeClient client = new OpenSearchNodeClient(nodeClient); OpenSearchScrollRequest request = new OpenSearchScrollRequest("test", factory); request.setScrollId("scroll123"); client.cleanup(request); @@ -263,8 +312,6 @@ void cleanup() { @Test void cleanupWithoutScrollId() { - OpenSearchNodeClient client = new OpenSearchNodeClient(nodeClient); - OpenSearchScrollRequest request = new OpenSearchScrollRequest("test", factory); client.cleanup(request); verify(nodeClient, never()).prepareClearScroll(); @@ -284,7 +331,6 @@ void getIndices() { when(indexResponse.getIndices()).thenReturn(new String[] {"index"}); when(indexResponse.aliases()).thenReturn(openMap); - OpenSearchNodeClient client = new OpenSearchNodeClient(nodeClient); final List indices = client.indices(); assertEquals(2, indices.size()); } @@ -295,22 +341,15 @@ void meta() { when(nodeClient.settings()).thenReturn(settings); when(settings.get(anyString(), anyString())).thenReturn("cluster-name"); - OpenSearchNodeClient client = new OpenSearchNodeClient(nodeClient); final Map meta = client.meta(); assertEquals("cluster-name", meta.get(META_CLUSTER_NAME)); } @Test void ml() { - OpenSearchNodeClient client = new OpenSearchNodeClient(nodeClient); assertNotNull(client.getNodeClient()); } - private OpenSearchNodeClient mockClient(String indexName, String mappings) { - mockNodeClientIndicesMappings(indexName, mappings); - return new OpenSearchNodeClient(nodeClient); - } - public void mockNodeClientIndicesMappings(String indexName, String mappings) { GetMappingsResponse mockResponse = mock(GetMappingsResponse.class); MappingMetadata emptyMapping = mock(MappingMetadata.class); @@ -336,17 +375,16 @@ public void mockNodeClientIndicesMappings(String indexName, String mappings) { } } - public NodeClient mockNodeClient(String indexName) { + public void mockNodeClient(String indexName) { GetMappingsResponse mockResponse = mock(GetMappingsResponse.class); when(nodeClient.admin().indices() .prepareGetMappings(any()) .setLocal(anyBoolean()) .get()).thenReturn(mockResponse); when(mockResponse.mappings()).thenReturn(ImmutableOpenMap.of()); - return nodeClient; } - private NodeClient mockNodeClientSettings(String indexName, String indexMetadata) + private void mockNodeClientSettings(String indexName, String indexMetadata) throws IOException { GetSettingsResponse mockResponse = mock(GetSettingsResponse.class); when(nodeClient.admin().indices().prepareGetSettings(any()).setLocal(anyBoolean()).get()) @@ -357,7 +395,6 @@ private NodeClient mockNodeClientSettings(String indexName, String indexMetadata .build(); when(mockResponse.getIndexToSettings()).thenReturn(metadata); - return nodeClient; } private XContentParser createParser(String mappings) throws IOException { diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java index bc334aaf39..25cfd6b35c 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java @@ -39,6 +39,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.RequestOptions; import org.opensearch.client.RestHighLevelClient; +import org.opensearch.client.indices.CreateIndexResponse; import org.opensearch.client.indices.GetIndexRequest; import org.opensearch.client.indices.GetIndexResponse; import org.opensearch.client.indices.GetMappingsRequest; @@ -69,7 +70,7 @@ class OpenSearchRestClientTest { @Mock(answer = RETURNS_DEEP_STUBS) private RestHighLevelClient restClient; - private OpenSearchRestClient client; + private OpenSearchClient client; @Mock private OpenSearchExprValueFactory factory; @@ -88,6 +89,52 @@ void setUp() { client = new OpenSearchRestClient(restClient); } + @Test + void isIndexExist() throws IOException { + when(restClient.indices() + .exists(any(), any())) // use any() because missing equals() in GetIndexRequest + .thenReturn(true); + + assertTrue(client.exists("test")); + } + + @Test + void isIndexNotExist() throws IOException { + when(restClient.indices() + .exists(any(), any())) // use any() because missing equals() in GetIndexRequest + .thenReturn(false); + + assertFalse(client.exists("test")); + } + + @Test + void isIndexExistWithException() throws IOException { + when(restClient.indices().exists(any(), any())).thenThrow(IOException.class); + + assertThrows(IllegalStateException.class, () -> client.exists("test")); + } + + @Test + void createIndex() throws IOException { + String indexName = "test"; + Map mappings = ImmutableMap.of( + "properties", + ImmutableMap.of("name", "text")); + when(restClient.indices() + .create(any(), any())) + .thenReturn(new CreateIndexResponse(true, true, indexName)); + + client.createIndex(indexName, mappings); + } + + @Test + void createIndexWithIOException() throws IOException { + when(restClient.indices().create(any(), any())).thenThrow(IOException.class); + + assertThrows(IllegalStateException.class, + () -> client.createIndex("test", ImmutableMap.of())); + } + @Test void getIndexMappings() throws IOException { URL url = Resources.getResource(TEST_MAPPING_FILE); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java index e55fff0e33..57bfcd1ea8 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java @@ -9,7 +9,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; +import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT; import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD; @@ -42,4 +45,23 @@ public void testShouldCast() { assertFalse(OPENSEARCH_TEXT.shouldCast(STRING)); assertFalse(OPENSEARCH_TEXT_KEYWORD.shouldCast(STRING)); } + + @Test + public void testGetExprType() { + assertEquals(OPENSEARCH_TEXT, OpenSearchDataType.getExprType("text")); + assertEquals(FLOAT, OpenSearchDataType.getExprType("float")); + assertEquals(FLOAT, OpenSearchDataType.getExprType("half_float")); + assertEquals(DOUBLE, OpenSearchDataType.getExprType("double")); + assertEquals(DOUBLE, OpenSearchDataType.getExprType("scaled_float")); + assertEquals(TIMESTAMP, OpenSearchDataType.getExprType("date")); + assertEquals(TIMESTAMP, OpenSearchDataType.getExprType("date_nanos")); + } + + @Test + public void testGetOpenSearchType() { + assertEquals("text", OpenSearchDataType.getOpenSearchType(OPENSEARCH_TEXT)); + assertEquals("float", OpenSearchDataType.getOpenSearchType(FLOAT)); + assertEquals("double", OpenSearchDataType.getOpenSearchType(DOUBLE)); + assertEquals("date", OpenSearchDataType.getOpenSearchType(TIMESTAMP)); + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java index 82ac3991ac..f7860403c7 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java @@ -14,6 +14,7 @@ import static org.hamcrest.Matchers.hasEntry; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.when; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; @@ -21,6 +22,7 @@ import static org.opensearch.sql.expression.DSL.literal; import static org.opensearch.sql.expression.DSL.named; import static org.opensearch.sql.expression.DSL.ref; +import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD; import static org.opensearch.sql.opensearch.utils.Utils.indexScan; import static org.opensearch.sql.opensearch.utils.Utils.indexScanAgg; import static org.opensearch.sql.opensearch.utils.Utils.noProjects; @@ -37,10 +39,12 @@ import com.google.common.collect.ImmutableMap; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -77,6 +81,8 @@ class OpenSearchIndexTest { private final DSL dsl = new ExpressionConfig().dsl(new ExpressionConfig().functionRepository()); + private final String indexName = "test"; + @Mock private OpenSearchClient client; @@ -89,6 +95,35 @@ class OpenSearchIndexTest { @Mock private Table table; + private OpenSearchIndex index; + + @BeforeEach + void setUp() { + this.index = new OpenSearchIndex(client, settings, indexName); + } + + @Test + void isExist() { + when(client.exists(indexName)).thenReturn(true); + + assertTrue(index.exists()); + } + + @Test + void createIndex() { + Map mappings = ImmutableMap.of( + "properties", + ImmutableMap.of( + "name", "text_keyword", + "age", "integer")); + doNothing().when(client).createIndex(indexName, mappings); + + Map schema = new HashMap<>(); + schema.put("name", OPENSEARCH_TEXT_KEYWORD); + schema.put("age", INTEGER); + index.create(schema); + } + @Test void getFieldTypes() { when(client.getIndexMappings("test")) @@ -112,7 +147,6 @@ void getFieldTypes() { .put("blob", "binary") .build()))); - OpenSearchIndex index = new OpenSearchIndex(client, settings, "test"); Map fieldTypes = index.getFieldTypes(); assertThat( fieldTypes, @@ -139,9 +173,7 @@ void implementRelationOperatorOnly() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); when(client.getIndexMaxResultWindows("test")).thenReturn(Map.of("test", 10000)); - String indexName = "test"; LogicalPlan plan = relation(indexName, table); - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); Integer maxResultWindow = index.getMaxResultWindow(); assertEquals( new OpenSearchIndexScan(client, settings, indexName, maxResultWindow, exprValueFactory), @@ -153,9 +185,7 @@ void implementRelationOperatorWithOptimization() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); when(client.getIndexMaxResultWindows("test")).thenReturn(Map.of("test", 10000)); - String indexName = "test"; LogicalPlan plan = relation(indexName, table); - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); Integer maxResultWindow = index.getMaxResultWindow(); assertEquals( new OpenSearchIndexScan(client, settings, indexName, maxResultWindow, exprValueFactory), @@ -167,7 +197,6 @@ void implementOtherLogicalOperators() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); when(client.getIndexMaxResultWindows("test")).thenReturn(Map.of("test", 10000)); - String indexName = "test"; NamedExpression include = named("age", ref("age", INTEGER)); ReferenceExpression exclude = ref("name", STRING); ReferenceExpression dedupeField = ref("name", STRING); @@ -199,7 +228,6 @@ void implementOtherLogicalOperators() { dedupeField), include); - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); Integer maxResultWindow = index.getMaxResultWindow(); assertEquals( PhysicalPlanDSL.project( @@ -228,8 +256,6 @@ void shouldImplLogicalIndexScan() { NamedExpression named = named("n", field); Expression filterExpr = dsl.equal(field, literal("John")); - String indexName = "test"; - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); PhysicalPlan plan = index.implement( project( indexScan( @@ -254,8 +280,6 @@ void shouldNotPushDownFilterFarFromRelation() { Arrays.asList(named("avg(age)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), DOUBLE))); - String indexName = "test"; - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); PhysicalPlan plan = index.implement( filter( aggregation( @@ -280,9 +304,6 @@ void shouldImplLogicalIndexScanAgg() { Arrays.asList(named("avg(age)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), DOUBLE))); - String indexName = "test"; - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); - // IndexScanAgg without Filter PhysicalPlan plan = index.implement( filter( @@ -317,9 +338,6 @@ void shouldNotPushDownAggregationFarFromRelation() { Arrays.asList(named("avg(age)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), DOUBLE))); - String indexName = "test"; - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); - PhysicalPlan plan = index.implement( aggregation( filter(filter( @@ -339,8 +357,6 @@ void shouldImplIndexScanWithSort() { NamedExpression named = named("n", field); Expression sortExpr = ref("name", STRING); - String indexName = "test"; - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); PhysicalPlan plan = index.implement( project( indexScan( @@ -361,8 +377,6 @@ void shouldImplIndexScanWithLimit() { ReferenceExpression field = ref("name", STRING); NamedExpression named = named("n", field); - String indexName = "test"; - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); PhysicalPlan plan = index.implement( project( indexScan( @@ -384,8 +398,6 @@ void shouldImplIndexScanWithSortAndLimit() { NamedExpression named = named("n", field); Expression sortExpr = ref("name", STRING); - String indexName = "test"; - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); PhysicalPlan plan = index.implement( project( indexScan( @@ -405,8 +417,6 @@ void shouldNotPushDownLimitFarFromRelationButUpdateScanSize() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); when(client.getIndexMaxResultWindows("test")).thenReturn(Map.of("test", 10000)); - String indexName = "test"; - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); PhysicalPlan plan = index.implement(index.optimize( project( limit( @@ -430,8 +440,6 @@ void shouldPushDownProjects() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); when(client.getIndexMaxResultWindows("test")).thenReturn(Map.of("test", 10000)); - String indexName = "test"; - OpenSearchIndex index = new OpenSearchIndex(client, settings, indexName); PhysicalPlan plan = index.implement( project( indexScan( diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java index e2efff22cb..a483f2dad8 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java @@ -9,6 +9,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.hasEntry; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.DSL.named; @@ -18,6 +19,7 @@ import static org.opensearch.sql.utils.SystemIndexUtils.TABLE_INFO; import static org.opensearch.sql.utils.SystemIndexUtils.mappingTable; +import com.google.common.collect.ImmutableMap; import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -58,6 +60,19 @@ void testGetFieldTypesOfMappingTable() { )); } + @Test + void testIsExist() { + Table systemIndex = new OpenSearchSystemIndex(client, TABLE_INFO); + assertTrue(systemIndex.exists()); + } + + @Test + void testCreateTable() { + Table systemIndex = new OpenSearchSystemIndex(client, TABLE_INFO); + assertThrows(UnsupportedOperationException.class, + () -> systemIndex.create(ImmutableMap.of())); + } + @Test void implement() { OpenSearchSystemIndex systemIndex = new OpenSearchSystemIndex(client, TABLE_INFO); From 3a9d217efdf87670c1aea41dc60bba2bec26c078 Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Fri, 21 Oct 2022 14:00:49 -0700 Subject: [PATCH 03/23] Add missing API method in prometheus table Signed-off-by: Chen Dai --- .../prometheus/storage/PrometheusMetricTable.java | 12 ++++++++++++ .../storage/PrometheusMetricTableTest.java | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTable.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTable.java index c2a637a85a..772f40da78 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTable.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTable.java @@ -58,6 +58,18 @@ public PrometheusMetricTable(PrometheusClient prometheusService, this.prometheusQueryRequest = Optional.of(prometheusQueryRequest); } + @Override + public boolean exists() { + throw new UnsupportedOperationException( + "Prometheus metric exists operation is not supported"); + } + + @Override + public void create(Map schema) { + throw new UnsupportedOperationException( + "Prometheus metric create operation is not supported"); + } + @Override public Map getFieldTypes() { if (cachedFieldTypes == null) { diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java index 1a048a7b8e..dadcfc7c03 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -19,6 +20,7 @@ import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.VALUE; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -125,4 +127,14 @@ void testOptimize() { assertEquals(inputPlan, optimizedPlan); } + @Test + void testUnsupportedOperation() { + PrometheusQueryRequest prometheusQueryRequest = new PrometheusQueryRequest(); + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, prometheusQueryRequest); + + assertThrows(UnsupportedOperationException.class, prometheusMetricTable::exists); + assertThrows(UnsupportedOperationException.class, + () -> prometheusMetricTable.create(Collections.emptyMap())); + } } From 91baab12260064997191ffefc28f847f8a5522a6 Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Thu, 27 Oct 2022 11:34:29 -0700 Subject: [PATCH 04/23] Add time window and window assigner (#950) * Add window and tumbling window assigner Signed-off-by: Chen Dai * Add sliding window assigner Signed-off-by: Chen Dai * Address PR comments Signed-off-by: Chen Dai Signed-off-by: Chen Dai --- .../planner/streaming/windowing/Window.java | 28 +++++++++ .../assigner/SlidingWindowAssigner.java | 57 +++++++++++++++++++ .../assigner/TumblingWindowAssigner.java | 38 +++++++++++++ .../windowing/assigner/WindowAssigner.java | 24 ++++++++ .../opensearch/sql/utils/DateTimeUtils.java | 10 ++++ .../streaming/windowing/WindowTest.java | 21 +++++++ .../assigner/SlidingWindowAssignerTest.java | 52 +++++++++++++++++ .../assigner/TumblingWindowAssignerTest.java | 39 +++++++++++++ 8 files changed, 269 insertions(+) create mode 100644 core/src/main/java/org/opensearch/sql/planner/streaming/windowing/Window.java create mode 100644 core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/SlidingWindowAssigner.java create mode 100644 core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/TumblingWindowAssigner.java create mode 100644 core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/WindowAssigner.java create mode 100644 core/src/test/java/org/opensearch/sql/planner/streaming/windowing/WindowTest.java create mode 100644 core/src/test/java/org/opensearch/sql/planner/streaming/windowing/assigner/SlidingWindowAssignerTest.java create mode 100644 core/src/test/java/org/opensearch/sql/planner/streaming/windowing/assigner/TumblingWindowAssignerTest.java diff --git a/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/Window.java b/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/Window.java new file mode 100644 index 0000000000..2a85ea391c --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/Window.java @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.streaming.windowing; + +import lombok.Data; + +/** + * A time window is a window of time interval with inclusive start time and exclusive end time. + */ +@Data +public class Window { + + /** Start timestamp (inclusive) of the time window. */ + private final long startTime; + + /** End timestamp (exclusive) of the time window. */ + private final long endTime; + + /** + * Return the maximum timestamp (inclusive) of the window. + */ + public long maxTimestamp() { + return endTime - 1; + } +} diff --git a/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/SlidingWindowAssigner.java b/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/SlidingWindowAssigner.java new file mode 100644 index 0000000000..f0f47fd575 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/SlidingWindowAssigner.java @@ -0,0 +1,57 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.streaming.windowing.assigner; + +import com.google.common.base.Preconditions; +import java.util.LinkedList; +import java.util.List; +import org.opensearch.sql.planner.streaming.windowing.Window; +import org.opensearch.sql.utils.DateTimeUtils; + +/** + * A sliding window assigner assigns multiple overlapped window per event timestamp. + * The overlap size is determined by the given slide interval. + */ +public class SlidingWindowAssigner implements WindowAssigner { + + /** Window size in millisecond. */ + private final long windowSize; + + /** Slide size in millisecond. */ + private final long slideSize; + + /** + * Create sliding window assigner with the given window and slide size in millisecond. + * + * @param windowSize window size in millisecond + * @param slideSize slide size in millisecond + */ + public SlidingWindowAssigner(long windowSize, long slideSize) { + Preconditions.checkArgument(windowSize > 0, + "Window size [%s] must be positive number", windowSize); + Preconditions.checkArgument(slideSize > 0, + "Slide size [%s] must be positive number", slideSize); + this.windowSize = windowSize; + this.slideSize = slideSize; + } + + @Override + public List assign(long timestamp) { + LinkedList windows = new LinkedList<>(); + + // Assign window from the last start time to the first until timestamp outside current window + long startTime = DateTimeUtils.getWindowStartTime(timestamp, slideSize); + for (Window win = window(startTime); win.maxTimestamp() >= timestamp; win = window(startTime)) { + windows.addFirst(win); + startTime -= slideSize; + } + return windows; + } + + private Window window(long startTime) { + return new Window(startTime, startTime + windowSize); + } +} diff --git a/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/TumblingWindowAssigner.java b/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/TumblingWindowAssigner.java new file mode 100644 index 0000000000..192bb6c429 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/TumblingWindowAssigner.java @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.streaming.windowing.assigner; + +import com.google.common.base.Preconditions; +import java.util.Collections; +import java.util.List; +import org.opensearch.sql.planner.streaming.windowing.Window; +import org.opensearch.sql.utils.DateTimeUtils; + +/** + * A tumbling window assigner assigns a single window per event timestamp without overlap. + */ +public class TumblingWindowAssigner implements WindowAssigner { + + /** Window size in millisecond. */ + private final long windowSize; + + /** + * Create tumbling window assigner with the given window size. + * + * @param windowSize window size in millisecond + */ + public TumblingWindowAssigner(long windowSize) { + Preconditions.checkArgument(windowSize > 0, + "Window size [%s] must be positive number", windowSize); + this.windowSize = windowSize; + } + + @Override + public List assign(long timestamp) { + long startTime = DateTimeUtils.getWindowStartTime(timestamp, windowSize); + return Collections.singletonList(new Window(startTime, startTime + windowSize)); + } +} diff --git a/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/WindowAssigner.java b/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/WindowAssigner.java new file mode 100644 index 0000000000..dac882c5ff --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/WindowAssigner.java @@ -0,0 +1,24 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.streaming.windowing.assigner; + +import java.util.List; +import org.opensearch.sql.planner.streaming.windowing.Window; + +/** + * A window assigner assigns zero or more window to an event timestamp + * based on different windowing approach. + */ +public interface WindowAssigner { + + /** + * Return window(s) assigned to the timestamp. + * @param timestamp given event timestamp + * @return windows assigned + */ + List assign(long timestamp); + +} diff --git a/core/src/main/java/org/opensearch/sql/utils/DateTimeUtils.java b/core/src/main/java/org/opensearch/sql/utils/DateTimeUtils.java index fbcf7deca4..5a99af3f83 100644 --- a/core/src/main/java/org/opensearch/sql/utils/DateTimeUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/DateTimeUtils.java @@ -85,6 +85,16 @@ public static long roundYear(long utcMillis, int interval) { return initDateTime.plusYears(yearToAdd).toInstant().toEpochMilli(); } + /** + * Get window start time which aligns with the given size. + * + * @param timestamp event timestamp + * @param size defines a window's start time to align with + * @return start timestamp of the window + */ + public long getWindowStartTime(long timestamp, long size) { + return timestamp - timestamp % size; + } /** * isValidMySqlTimeZoneId for timezones which match timezone the range set by MySQL. diff --git a/core/src/test/java/org/opensearch/sql/planner/streaming/windowing/WindowTest.java b/core/src/test/java/org/opensearch/sql/planner/streaming/windowing/WindowTest.java new file mode 100644 index 0000000000..9b9aafa933 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/planner/streaming/windowing/WindowTest.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.streaming.windowing; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class WindowTest { + + @Test + void test() { + Window window = new Window(1000, 2000); + assertEquals(1000, window.getStartTime()); + assertEquals(2000, window.getEndTime()); + assertEquals(1999, window.maxTimestamp()); + } +} \ No newline at end of file diff --git a/core/src/test/java/org/opensearch/sql/planner/streaming/windowing/assigner/SlidingWindowAssignerTest.java b/core/src/test/java/org/opensearch/sql/planner/streaming/windowing/assigner/SlidingWindowAssignerTest.java new file mode 100644 index 0000000000..fd69065742 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/planner/streaming/windowing/assigner/SlidingWindowAssignerTest.java @@ -0,0 +1,52 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.streaming.windowing.assigner; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.List; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.planner.streaming.windowing.Window; + +class SlidingWindowAssignerTest { + + @Test + void testAssignWindows() { + long windowSize = 1000; + long slideSize = 500; + SlidingWindowAssigner assigner = new SlidingWindowAssigner(windowSize, slideSize); + + assertEquals( + List.of( + new Window(0, 1000), + new Window(500, 1500)), + assigner.assign(500)); + + assertEquals( + List.of( + new Window(0, 1000), + new Window(500, 1500)), + assigner.assign(999)); + + assertEquals( + List.of( + new Window(500, 1500), + new Window(1000, 2000)), + assigner.assign(1000)); + } + + @Test + void testConstructWithIllegalArguments() { + IllegalArgumentException error1 = assertThrows(IllegalArgumentException.class, + () -> new SlidingWindowAssigner(-1, 100)); + assertEquals("Window size [-1] must be positive number", error1.getMessage()); + + IllegalArgumentException error2 = assertThrows(IllegalArgumentException.class, + () -> new SlidingWindowAssigner(1000, 0)); + assertEquals("Slide size [0] must be positive number", error2.getMessage()); + } +} \ No newline at end of file diff --git a/core/src/test/java/org/opensearch/sql/planner/streaming/windowing/assigner/TumblingWindowAssignerTest.java b/core/src/test/java/org/opensearch/sql/planner/streaming/windowing/assigner/TumblingWindowAssignerTest.java new file mode 100644 index 0000000000..4c98c40f7a --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/planner/streaming/windowing/assigner/TumblingWindowAssignerTest.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.streaming.windowing.assigner; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.Collections; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.planner.streaming.windowing.Window; + +class TumblingWindowAssignerTest { + + @Test + void testAssignWindow() { + long windowSize = 1000; + TumblingWindowAssigner assigner = new TumblingWindowAssigner(windowSize); + + assertEquals( + Collections.singletonList(new Window(0, 1000)), + assigner.assign(500)); + assertEquals( + Collections.singletonList(new Window(1000, 2000)), + assigner.assign(1999)); + assertEquals( + Collections.singletonList(new Window(2000, 3000)), + assigner.assign(2000)); + } + + @Test + void testConstructWithIllegalWindowSize() { + IllegalArgumentException error = assertThrows(IllegalArgumentException.class, + () -> new TumblingWindowAssigner(-1)); + assertEquals("Window size [-1] must be positive number", error.getMessage()); + } +} \ No newline at end of file From b286742ab06fdaba3e3ed6d24ad0a935c2fabe04 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Thu, 27 Oct 2022 13:52:15 -0700 Subject: [PATCH 05/23] add table default impl of exist and create Signed-off-by: Peng Huo --- .../java/org/opensearch/sql/storage/Table.java | 10 ++++++++-- .../physical/catalog/CatalogTableTest.java | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/storage/Table.java b/core/src/main/java/org/opensearch/sql/storage/Table.java index 7cdd757cbb..34e6ece30b 100644 --- a/core/src/main/java/org/opensearch/sql/storage/Table.java +++ b/core/src/main/java/org/opensearch/sql/storage/Table.java @@ -18,15 +18,21 @@ public interface Table { /** * Check if current table exists. + * * @return true if exists, otherwise false */ - boolean exists(); + default boolean exists() { + throw new UnsupportedOperationException("Unsupported Operation"); + } /** * Create table given table schema. + * * @param schema table schema */ - void create(Map schema); + default void create(Map schema) { + throw new UnsupportedOperationException("Unsupported Operation"); + } /** * Get the {@link ExprType} for each field in the table. diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java index 59e57a97b3..a35c94a21c 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java @@ -8,6 +8,7 @@ package org.opensearch.sql.planner.physical.catalog; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.HashMap; @@ -46,4 +47,21 @@ void testImplement() { assertTrue(physicalPlan instanceof CatalogTableScan); } + // todo. temporary added for code coverage. remove if required. + @Test + void testExist() { + UnsupportedOperationException exception = + assertThrows(UnsupportedOperationException.class, + () -> new CatalogTable(catalogService).exists()); + assertEquals("Unsupported Operation", exception.getMessage()); + } + + // todo. temporary added for code coverage. remove if required. + @Test + void testCreateTable() { + UnsupportedOperationException exception = + assertThrows(UnsupportedOperationException.class, + () -> new CatalogTable(catalogService).create(new HashMap<>())); + assertEquals("Unsupported Operation", exception.getMessage()); + } } From 63f34494ccfa7e27cf9800c731830c8658128cf1 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Fri, 28 Oct 2022 08:59:48 -0700 Subject: [PATCH 06/23] Add metadatalog interface and default in memory implementation (#974) Signed-off-by: Peng Huo --- .../streaming/DefaultMetadataLog.java | 76 +++++++++++ .../sql/executor/streaming/MetadataLog.java | 61 +++++++++ .../streaming/DefaultMetadataLogTest.java | 126 ++++++++++++++++++ 3 files changed, 263 insertions(+) create mode 100644 core/src/main/java/org/opensearch/sql/executor/streaming/DefaultMetadataLog.java create mode 100644 core/src/main/java/org/opensearch/sql/executor/streaming/MetadataLog.java create mode 100644 core/src/test/java/org/opensearch/sql/executor/streaming/DefaultMetadataLogTest.java diff --git a/core/src/main/java/org/opensearch/sql/executor/streaming/DefaultMetadataLog.java b/core/src/main/java/org/opensearch/sql/executor/streaming/DefaultMetadataLog.java new file mode 100644 index 0000000000..e439d93f6c --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/streaming/DefaultMetadataLog.java @@ -0,0 +1,76 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.executor.streaming; + +import com.google.common.base.Preconditions; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.SortedMap; +import java.util.TreeMap; +import org.apache.commons.lang3.tuple.Pair; + +/** + * In memory implementation of {@link MetadataLog}. Todo. Current implementation does not guarantee + * thread safe. We will re-evaluate it when adding pipeline execution. + * + * @param type of metadata type. + */ +public class DefaultMetadataLog implements MetadataLog { + + private static final long MIN_ACCEPTABLE_ID = 0L; + + private SortedMap metadataMap = new TreeMap<>(); + + @Override + public boolean add(Long batchId, T metadata) { + Preconditions.checkArgument(batchId >= MIN_ACCEPTABLE_ID, "batch id must large or equal 0"); + + if (metadataMap.containsKey(batchId)) { + return false; + } + metadataMap.put(batchId, metadata); + return true; + } + + @Override + public Optional get(Long batchId) { + if (!metadataMap.containsKey(batchId)) { + return Optional.empty(); + } else { + return Optional.of(metadataMap.get(batchId)); + } + } + + @Override + public List get(Optional startBatchId, Optional endBatchId) { + if (startBatchId.isEmpty() && endBatchId.isEmpty()) { + return new ArrayList<>(metadataMap.values()); + } else { + Long s = startBatchId.orElse(MIN_ACCEPTABLE_ID); + Long e = endBatchId.map(i -> i + 1).orElse(Long.MAX_VALUE); + return new ArrayList<>(metadataMap.subMap(s, e).values()); + } + } + + @Override + public Optional> getLatest() { + if (metadataMap.isEmpty()) { + return Optional.empty(); + } else { + Long latestId = metadataMap.lastKey(); + return Optional.of(Pair.of(latestId, metadataMap.get(latestId))); + } + } + + @Override + public void purge(Long batchId) { + metadataMap.headMap(batchId).clear(); + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/streaming/MetadataLog.java b/core/src/main/java/org/opensearch/sql/executor/streaming/MetadataLog.java new file mode 100644 index 0000000000..d6bb9bacd6 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/streaming/MetadataLog.java @@ -0,0 +1,61 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.executor.streaming; + +import java.util.List; +import java.util.Optional; +import org.apache.commons.lang3.tuple.Pair; + +/** + * Write-ahead Log (WAL). Which allow client write metadata associate with id. + * + * @param type of metadata type. + */ +public interface MetadataLog { + + /** + * add metadata to WAL. + * + * @param id metadata index in WAL. + * @param metadata metadata. + * @return true if add success, otherwise return false. + */ + boolean add(Long id, T metadata); + + /** + * get metadata from WAL. + * + * @param id metadata index in WAL. + * @return metadata. + */ + Optional get(Long id); + + /** + * Return metadata for id between [startId, endId]. + * + * @param startId If startId is empty, return all metadata before endId (inclusive). + * @param endId If end is empty, return all batches after endId (inclusive). + * @return a list of metadata sorted by id (nature order). + */ + List get(Optional startId, Optional endId); + + /** + * Get latest batchId and metadata. + * + * @return pair of id and metadata if not empty. + */ + Optional> getLatest(); + + /** + * Remove all the metadata less then id (exclusive). + * + * @param id smallest batchId should keep. + */ + void purge(Long id); +} diff --git a/core/src/test/java/org/opensearch/sql/executor/streaming/DefaultMetadataLogTest.java b/core/src/test/java/org/opensearch/sql/executor/streaming/DefaultMetadataLogTest.java new file mode 100644 index 0000000000..4d8c4f3e93 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/executor/streaming/DefaultMetadataLogTest.java @@ -0,0 +1,126 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.executor.streaming; + + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Arrays; +import java.util.Optional; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class DefaultMetadataLogTest { + + private DefaultMetadataLog metadataLog; + + @BeforeEach + void setup() { + metadataLog = new DefaultMetadataLog<>(); + } + + @Test + void addMetadataShouldSuccess() { + assertTrue(metadataLog.add(0L, 0L)); + assertTrue(metadataLog.add(1L, 1L)); + } + + @Test + void addMetadataWithSameBatchIdShouldFail() { + assertTrue(metadataLog.add(0L, 0L)); + assertFalse(metadataLog.add(0L, 1L)); + } + + @Test + void addMetadataWithInvalidIdShouldThrowException() { + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> metadataLog.add(-1L, 0L)); + assertEquals("batch id must large or equal 0", exception.getMessage()); + } + + @Test + void getWithIdReturnMetadata() { + metadataLog.add(0L, 0L); + + assertTrue(metadataLog.get(0L).isPresent()); + assertEquals(0L, metadataLog.get(0L).get()); + } + + @Test + void getWithNotExistShouldReturnEmtpy() { + metadataLog.add(0L, 0L); + + assertTrue(metadataLog.get(1L).isEmpty()); + assertTrue(metadataLog.get(-1L).isEmpty()); + } + + @Test + void getWithIdInRangeShouldReturnMetadataList() { + metadataLog.add(0L, 0L); + metadataLog.add(1L, 1L); + metadataLog.add(2L, 2L); + + assertEquals(Arrays.asList(0L, 1L, 2L), metadataLog.get(Optional.of(0L), Optional.of(2L))); + assertEquals(Arrays.asList(0L, 1L, 2L), metadataLog.get(Optional.of(0L), Optional.of(4L))); + assertEquals(Arrays.asList(0L, 1L, 2L), metadataLog.get(Optional.of(-1L), Optional.of(4L))); + assertEquals(Arrays.asList(0L, 1L), metadataLog.get(Optional.of(0L), Optional.of(1L))); + assertEquals(Arrays.asList(1L, 2L), metadataLog.get(Optional.of(1L), Optional.empty())); + assertEquals(Arrays.asList(0L, 1L), metadataLog.get(Optional.empty(), Optional.of(1L))); + assertEquals(Arrays.asList(0L, 1L, 2L), metadataLog.get(Optional.empty(), Optional.empty())); + } + + @Test + void getWithIdOutOfRangeShouldReturnEmpty() { + metadataLog.add(0L, 0L); + metadataLog.add(1L, 1L); + metadataLog.add(2L, 2L); + + assertTrue(metadataLog.get(Optional.of(3L), Optional.of(5L)).isEmpty()); + } + + @Test + void getLatestShouldReturnMetadata() { + metadataLog.add(0L, 10L); + metadataLog.add(1L, 11L); + + Optional> latest = metadataLog.getLatest(); + assertTrue(latest.isPresent()); + assertEquals(1L, latest.get().getLeft()); + assertEquals(11L, latest.get().getRight()); + } + + @Test + void getLatestFromEmptyWALShouldReturnEmpty() { + Optional> latest = metadataLog.getLatest(); + assertTrue(latest.isEmpty()); + } + + @Test + void purgeLatestShouldOnlyKeepLatest() { + metadataLog.add(0L, 10L); + metadataLog.add(1L, 11L); + metadataLog.add(2L, 12L); + + Optional> latest = metadataLog.getLatest(); + assertTrue(latest.isPresent()); + metadataLog.purge(latest.get().getLeft()); + + latest = metadataLog.getLatest(); + assertTrue(latest.isPresent()); + assertEquals(2L, latest.get().getLeft()); + assertEquals(12L, latest.get().getRight()); + } +} From 30fcd79983710bbc8866507b0e46cce946cae936 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Fri, 28 Oct 2022 15:34:02 -0700 Subject: [PATCH 07/23] Add Statement, QueryExecution and QueryManager (#845) Signed-off-by: Peng Huo --- .../sql/ast/AbstractNodeVisitor.java | 15 ++ .../opensearch/sql/ast/statement/Explain.java | 26 +++ .../opensearch/sql/ast/statement/Query.java | 35 ++++ .../sql/ast/statement/Statement.java | 22 +++ .../sql/executor/DefaultQueryManager.java | 24 +++ .../org/opensearch/sql/executor/QueryId.java | 36 ++++ .../opensearch/sql/executor/QueryManager.java | 25 +++ .../opensearch/sql/executor/QueryService.java | 70 ++++++++ .../sql/executor/execution/AbstractPlan.java | 41 +++++ .../sql/executor/execution/ExplainPlan.java | 44 +++++ .../sql/executor/execution/QueryPlan.java | 57 ++++++ .../executor/execution/QueryPlanFactory.java | 100 +++++++++++ .../sql/executor/DefaultQueryManagerTest.java | 41 +++++ .../opensearch/sql/executor/QueryIdTest.java | 22 +++ .../sql/executor/QueryServiceTest.java | 164 ++++++++++++++++++ .../executor/execution/ExplainPlanTest.java | 59 +++++++ .../execution/QueryPlanFactoryTest.java | 107 ++++++++++++ .../sql/executor/execution/QueryPlanTest.java | 58 +++++++ .../org/opensearch/sql/ppl/StandaloneIT.java | 57 +++++- .../sql/legacy/plugin/RestSQLQueryAction.java | 90 ++++++---- .../sql/legacy/plugin/RestSqlAction.java | 45 ++--- .../legacy/plugin/RestSQLQueryActionTest.java | 96 ++++++++-- .../executor/OpenSearchQueryManager.java | 50 ++++++ .../sql/opensearch/executor/Scheduler.java | 33 ---- ...t.java => OpenSearchQueryManagerTest.java} | 42 ++++- .../plugin/config/OpenSearchPluginConfig.java | 45 ++++- .../sql/plugin/rest/RestPPLQueryAction.java | 66 ++++--- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 8 + .../org/opensearch/sql/ppl/PPLService.java | 63 ++++--- .../sql/ppl/config/PPLServiceConfig.java | 23 +-- .../sql/ppl/domain/PPLQueryRequest.java | 13 +- .../opensearch/sql/ppl/parser/AstBuilder.java | 3 +- .../sql/ppl/parser/AstStatementBuilder.java | 58 +++++++ .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 20 +++ .../opensearch/sql/ppl/PPLServiceTest.java | 83 +++------ .../ppl/parser/AstStatementBuilderTest.java | 72 ++++++++ .../ppl/utils/PPLQueryDataAnonymizerTest.java | 32 +++- .../org/opensearch/sql/sql/SQLService.java | 94 ++++------ .../sql/sql/config/SQLServiceConfig.java | 29 +--- .../sql/sql/parser/AstStatementBuilder.java | 43 +++++ .../opensearch/sql/sql/SQLServiceTest.java | 85 +++------ 41 files changed, 1684 insertions(+), 412 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/ast/statement/Explain.java create mode 100644 core/src/main/java/org/opensearch/sql/ast/statement/Query.java create mode 100644 core/src/main/java/org/opensearch/sql/ast/statement/Statement.java create mode 100644 core/src/main/java/org/opensearch/sql/executor/DefaultQueryManager.java create mode 100644 core/src/main/java/org/opensearch/sql/executor/QueryId.java create mode 100644 core/src/main/java/org/opensearch/sql/executor/QueryManager.java create mode 100644 core/src/main/java/org/opensearch/sql/executor/QueryService.java create mode 100644 core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java create mode 100644 core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java create mode 100644 core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java create mode 100644 core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java create mode 100644 core/src/test/java/org/opensearch/sql/executor/DefaultQueryManagerTest.java create mode 100644 core/src/test/java/org/opensearch/sql/executor/QueryIdTest.java create mode 100644 core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java create mode 100644 core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java create mode 100644 core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanFactoryTest.java create mode 100644 core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchQueryManager.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/executor/Scheduler.java rename opensearch/src/test/java/org/opensearch/sql/opensearch/executor/{SchedulerTest.java => OpenSearchQueryManagerTest.java} (51%) create mode 100644 ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java create mode 100644 ppl/src/test/java/org/opensearch/sql/ppl/parser/AstStatementBuilderTest.java create mode 100644 sql/src/main/java/org/opensearch/sql/sql/parser/AstStatementBuilder.java diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index 295db7680f..84e460e66d 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -35,6 +35,9 @@ import org.opensearch.sql.ast.expression.When; import org.opensearch.sql.ast.expression.WindowFunction; import org.opensearch.sql.ast.expression.Xor; +import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.Query; +import org.opensearch.sql.ast.statement.Statement; import org.opensearch.sql.ast.tree.AD; import org.opensearch.sql.ast.tree.Aggregation; import org.opensearch.sql.ast.tree.Dedupe; @@ -269,4 +272,16 @@ public T visitAD(AD node, C context) { public T visitHighlightFunction(HighlightFunction node, C context) { return visitChildren(node, context); } + + public T visitStatement(Statement node, C context) { + return visit(node, context); + } + + public T visitQuery(Query node, C context) { + return visitStatement(node, context); + } + + public T visitExplain(Explain node, C context) { + return visitStatement(node, context); + } } diff --git a/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java b/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java new file mode 100644 index 0000000000..d0f2e3b372 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.ast.statement; + +import lombok.Data; +import org.opensearch.sql.ast.AbstractNodeVisitor; + +/** + * Explain Statement. + */ +@Data +public class Explain extends Statement { + + private final Statement statement; + + @Override + public R accept(AbstractNodeVisitor visitor, C context) { + return visitor.visitExplain(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/ast/statement/Query.java b/core/src/main/java/org/opensearch/sql/ast/statement/Query.java new file mode 100644 index 0000000000..a7b547ed2a --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/statement/Query.java @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.ast.statement; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.tree.UnresolvedPlan; + +/** + * Query Statement. + */ +@Getter +@Setter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class Query extends Statement { + + private final UnresolvedPlan plan; + + @Override + public R accept(AbstractNodeVisitor visitor, C context) { + return visitor.visitQuery(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/ast/statement/Statement.java b/core/src/main/java/org/opensearch/sql/ast/statement/Statement.java new file mode 100644 index 0000000000..e32a8dbfd8 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/statement/Statement.java @@ -0,0 +1,22 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.ast.statement; + +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.Node; + +/** + * Statement is the high interface of core engine. + */ +public abstract class Statement extends Node { + @Override + public R accept(AbstractNodeVisitor visitor, C context) { + return visitor.visitStatement(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/DefaultQueryManager.java b/core/src/main/java/org/opensearch/sql/executor/DefaultQueryManager.java new file mode 100644 index 0000000000..9ab3bd7486 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/DefaultQueryManager.java @@ -0,0 +1,24 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.executor; + +import org.opensearch.sql.executor.execution.AbstractPlan; + +/** + * Default QueryManager implementation which execute {@link AbstractPlan} on caller thread. + */ +public class DefaultQueryManager implements QueryManager { + + @Override + public QueryId submit(AbstractPlan queryExecution) { + queryExecution.execute(); + + return queryExecution.getQueryId(); + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryId.java b/core/src/main/java/org/opensearch/sql/executor/QueryId.java new file mode 100644 index 0000000000..933cb5d82d --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/QueryId.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.executor; + +import lombok.Getter; +import org.apache.commons.lang3.RandomStringUtils; +import org.opensearch.sql.executor.execution.AbstractPlan; + +/** + * Query id of {@link AbstractPlan}. + */ +public class QueryId { + /** + * Query id. + */ + @Getter + private final String queryId; + + /** + * Generate {@link QueryId}. + * @return {@link QueryId}. + */ + public static QueryId queryId() { + return new QueryId(RandomStringUtils.random(10, true, true)); + } + + private QueryId(String queryId) { + this.queryId = queryId; + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryManager.java b/core/src/main/java/org/opensearch/sql/executor/QueryManager.java new file mode 100644 index 0000000000..3a32e4c7e9 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/QueryManager.java @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.executor; + +import org.opensearch.sql.executor.execution.AbstractPlan; + +/** + * QueryManager is the high-level interface of core engine. + * Frontend submit {@link AbstractPlan} to QueryManager. + */ +public interface QueryManager { + + /** + * Submit {@link AbstractPlan}. + * @param queryPlan {@link AbstractPlan}. + * @return {@link QueryId}. + */ + QueryId submit(AbstractPlan queryPlan); +} diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryService.java b/core/src/main/java/org/opensearch/sql/executor/QueryService.java new file mode 100644 index 0000000000..423d152e06 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/QueryService.java @@ -0,0 +1,70 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.executor; + +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.analysis.AnalysisContext; +import org.opensearch.sql.analysis.Analyzer; +import org.opensearch.sql.ast.tree.UnresolvedPlan; +import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.planner.Planner; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; + +/** + * The low level interface of core engine. + */ +@RequiredArgsConstructor +public class QueryService { + + private final Analyzer analyzer; + + private final ExecutionEngine executionEngine; + + private final Planner planner; + + /** + * Execute the {@link UnresolvedPlan}, using {@link ResponseListener} to get response. + * + * @param plan {@link UnresolvedPlan} + * @param listener {@link ResponseListener} + */ + public void execute(UnresolvedPlan plan, + ResponseListener listener) { + try { + executionEngine.execute(plan(plan), listener); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Explain the query in {@link UnresolvedPlan} using {@link ResponseListener} to + * get and format explain response. + * + * @param plan {@link UnresolvedPlan} + * @param listener {@link ResponseListener} for explain response + */ + public void explain(UnresolvedPlan plan, + ResponseListener listener) { + try { + executionEngine.explain(plan(plan), listener); + } catch (Exception e) { + listener.onFailure(e); + } + } + + private PhysicalPlan plan(UnresolvedPlan plan) { + // 1.Analyze abstract syntax to generate logical plan + LogicalPlan logicalPlan = analyzer.analyze(plan, new AnalysisContext()); + + // 2.Generate optimal physical plan from logical plan + return planner.plan(logicalPlan); + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java new file mode 100644 index 0000000000..1654293c04 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.executor.execution; + + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.executor.ExecutionEngine; +import org.opensearch.sql.executor.QueryId; + +/** + * AbstractPlan represent the execution entity of the Statement. + */ +@RequiredArgsConstructor +public abstract class AbstractPlan { + + /** + * Uniq query id. + */ + @Getter + private final QueryId queryId; + + /** + * Start query execution. + */ + public abstract void execute(); + + /** + * Explain query execution. + * + * @param listener query explain response listener. + */ + public abstract void explain(ResponseListener listener); +} diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java new file mode 100644 index 0000000000..8c784f82ed --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java @@ -0,0 +1,44 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.executor.execution; + +import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.executor.ExecutionEngine; +import org.opensearch.sql.executor.QueryId; + +/** + * Explain plan. + */ +public class ExplainPlan extends AbstractPlan { + + private final AbstractPlan plan; + + private final ResponseListener explainListener; + + /** + * Constructor. + */ + public ExplainPlan(QueryId queryId, + AbstractPlan plan, + ResponseListener explainListener) { + super(queryId); + this.plan = plan; + this.explainListener = explainListener; + } + + @Override + public void execute() { + plan.explain(explainListener); + } + + @Override + public void explain(ResponseListener listener) { + throw new UnsupportedOperationException("explain query can not been explained."); + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java new file mode 100644 index 0000000000..02cb701a0b --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java @@ -0,0 +1,57 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.executor.execution; + +import org.opensearch.sql.ast.tree.UnresolvedPlan; +import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.executor.ExecutionEngine; +import org.opensearch.sql.executor.QueryId; +import org.opensearch.sql.executor.QueryService; + +/** + * Query plan. Which includes. + * + *

select query. + */ +public class QueryPlan extends AbstractPlan { + + /** + * The query plan ast. + */ + private final UnresolvedPlan plan; + + /** + * Query service. + */ + private final QueryService queryService; + + private final ResponseListener listener; + + /** constructor. */ + public QueryPlan( + QueryId queryId, + UnresolvedPlan plan, + QueryService queryService, + ResponseListener listener) { + super(queryId); + this.plan = plan; + this.queryService = queryService; + this.listener = listener; + } + + @Override + public void execute() { + queryService.execute(plan, listener); + } + + @Override + public void explain(ResponseListener listener) { + queryService.explain(plan, listener); + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java new file mode 100644 index 0000000000..851381cc7a --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java @@ -0,0 +1,100 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.executor.execution; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import java.util.Optional; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.Query; +import org.opensearch.sql.ast.statement.Statement; +import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.executor.ExecutionEngine; +import org.opensearch.sql.executor.QueryId; +import org.opensearch.sql.executor.QueryService; + +/** + * QueryExecution Factory. + */ +@RequiredArgsConstructor +public class QueryPlanFactory + extends AbstractNodeVisitor< + AbstractPlan, + Pair< + Optional>, + Optional>>> { + + /** + * Query Service. + */ + private final QueryService queryService; + + /** + * NO_CONSUMER_RESPONSE_LISTENER should never been called. It is only used as constructor + * parameter of {@link QueryPlan}. + */ + @VisibleForTesting + protected static final ResponseListener + NO_CONSUMER_RESPONSE_LISTENER = + new ResponseListener<>() { + @Override + public void onResponse(ExecutionEngine.QueryResponse response) { + throw new IllegalStateException( + "[BUG] query response should not sent to unexpected channel"); + } + + @Override + public void onFailure(Exception e) { + throw new IllegalStateException( + "[BUG] exception response should not sent to unexpected channel"); + } + }; + + /** + * Create QueryExecution from Statement. + */ + public AbstractPlan create( + Statement statement, + Optional> queryListener, + Optional> explainListener) { + return statement.accept(this, Pair.of(queryListener, explainListener)); + } + + @Override + public AbstractPlan visitQuery( + Query node, + Pair< + Optional>, + Optional>> + context) { + Preconditions.checkArgument( + context.getLeft().isPresent(), "[BUG] query listener must be not null"); + + return new QueryPlan(QueryId.queryId(), node.getPlan(), queryService, context.getLeft().get()); + } + + @Override + public AbstractPlan visitExplain( + Explain node, + Pair< + Optional>, + Optional>> + context) { + Preconditions.checkArgument( + context.getRight().isPresent(), "[BUG] explain listener must be not null"); + + return new ExplainPlan( + QueryId.queryId(), + create(node.getStatement(), Optional.of(NO_CONSUMER_RESPONSE_LISTENER), Optional.empty()), + context.getRight().get()); + } +} diff --git a/core/src/test/java/org/opensearch/sql/executor/DefaultQueryManagerTest.java b/core/src/test/java/org/opensearch/sql/executor/DefaultQueryManagerTest.java new file mode 100644 index 0000000000..988b41657d --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/executor/DefaultQueryManagerTest.java @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.executor; + + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.executor.execution.AbstractPlan; + +@ExtendWith(MockitoExtension.class) +class DefaultQueryManagerTest { + + @Mock + private AbstractPlan plan; + + @Mock + private QueryId queryId; + + @Test + public void submitQuery() { + when(plan.getQueryId()).thenReturn(queryId); + + QueryId actualQueryId = new DefaultQueryManager().submit(plan); + + assertEquals(queryId, actualQueryId); + verify(plan, times(1)).execute(); + } +} diff --git a/core/src/test/java/org/opensearch/sql/executor/QueryIdTest.java b/core/src/test/java/org/opensearch/sql/executor/QueryIdTest.java new file mode 100644 index 0000000000..7d837c3e24 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/executor/QueryIdTest.java @@ -0,0 +1,22 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.executor; + +import static org.junit.jupiter.api.Assertions.assertFalse; + +import com.google.common.base.Strings; +import org.junit.jupiter.api.Test; + +class QueryIdTest { + + @Test + public void createQueryId() { + assertFalse(Strings.isNullOrEmpty(QueryId.queryId().getQueryId())); + } +} diff --git a/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java b/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java new file mode 100644 index 0000000000..2884544dd0 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java @@ -0,0 +1,164 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.executor; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.when; + +import java.util.Collections; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.analysis.Analyzer; +import org.opensearch.sql.ast.tree.UnresolvedPlan; +import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.planner.Planner; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +@ExtendWith(MockitoExtension.class) +class QueryServiceTest { + private AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); + + private QueryService queryService; + + @Mock + private ExecutionEngine executionEngine; + + @Mock + private Analyzer analyzer; + + @Mock + private Planner planner; + + @Mock + private UnresolvedPlan ast; + + @Mock + private LogicalPlan logicalPlan; + + @Mock + private PhysicalPlan plan; + + @Mock + private ExecutionEngine.Schema schema; + + @BeforeEach + public void setUp() { + when(analyzer.analyze(any(), any())).thenReturn(logicalPlan); + when(planner.plan(any())).thenReturn(plan); + + queryService = new QueryService(analyzer, executionEngine, planner); + } + + @Test + public void testExecuteShouldPass() { + doAnswer( + invocation -> { + ResponseListener listener = invocation.getArgument(1); + listener.onResponse( + new ExecutionEngine.QueryResponse(schema, Collections.emptyList())); + return null; + }) + .when(executionEngine) + .execute(any(), any()); + + queryService.execute( + ast, + new ResponseListener<>() { + @Override + public void onResponse(ExecutionEngine.QueryResponse pplQueryResponse) { + + } + + @Override + public void onFailure(Exception e) { + fail(); + } + }); + } + + @Test + public void testExplainShouldPass() { + doAnswer( + invocation -> { + ResponseListener listener = + invocation.getArgument(1); + listener.onResponse( + new ExecutionEngine.ExplainResponse( + new ExecutionEngine.ExplainResponseNode("test"))); + return null; + }) + .when(executionEngine) + .explain(any(), any()); + + queryService.explain( + ast, + new ResponseListener<>() { + @Override + public void onResponse(ExecutionEngine.ExplainResponse pplQueryResponse) { + + } + + @Override + public void onFailure(Exception e) { + fail(); + } + }); + } + + @Test + public void testExecuteWithExceptionShouldBeCaughtByHandler() { + doThrow(new IllegalStateException("illegal state exception")) + .when(executionEngine) + .execute(any(), any()); + + queryService.execute( + ast, + new ResponseListener<>() { + @Override + public void onResponse(ExecutionEngine.QueryResponse pplQueryResponse) { + fail(); + } + + @Override + public void onFailure(Exception e) { + assertTrue(e instanceof IllegalStateException); + } + }); + } + + @Test + public void testExecuteWithIllegalQueryShouldBeCaughtByHandler() { + doThrow(new IllegalStateException("illegal state exception")) + .when(executionEngine) + .explain(any(), any()); + + queryService.explain( + ast, + new ResponseListener<>() { + @Override + public void onResponse(ExecutionEngine.ExplainResponse pplQueryResponse) { + fail(); + } + + @Override + public void onFailure(Exception e) { + assertTrue(e instanceof IllegalStateException); + } + }); + } +} diff --git a/core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java b/core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java new file mode 100644 index 0000000000..54b4f24db0 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java @@ -0,0 +1,59 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.executor.execution; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.executor.ExecutionEngine; +import org.opensearch.sql.executor.QueryId; + +@ExtendWith(MockitoExtension.class) +public class ExplainPlanTest { + @Mock + private QueryId queryId; + + @Mock + private QueryPlan queryPlan; + + @Mock + private ResponseListener explainListener; + + @Test + public void execute() { + doNothing().when(queryPlan).explain(any()); + + ExplainPlan explainPlan = new ExplainPlan(queryId, queryPlan, explainListener); + explainPlan.execute(); + + verify(queryPlan, times(1)).explain(explainListener); + } + + @Test + public void explainThrowException() { + ExplainPlan explainPlan = new ExplainPlan(queryId, queryPlan, explainListener); + + UnsupportedOperationException unsupportedExplainException = + assertThrows( + UnsupportedOperationException.class, + () -> { + explainPlan.explain(explainListener); + }); + assertEquals("explain query can not been explained.", unsupportedExplainException.getMessage()); + } +} diff --git a/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanFactoryTest.java b/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanFactoryTest.java new file mode 100644 index 0000000000..cc4bf070fb --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanFactoryTest.java @@ -0,0 +1,107 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.executor.execution; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.executor.execution.QueryPlanFactory.NO_CONSUMER_RESPONSE_LISTENER; + +import java.util.Optional; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.Query; +import org.opensearch.sql.ast.statement.Statement; +import org.opensearch.sql.ast.tree.UnresolvedPlan; +import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.executor.ExecutionEngine; +import org.opensearch.sql.executor.QueryService; + +@ExtendWith(MockitoExtension.class) +class QueryPlanFactoryTest { + + @Mock + private UnresolvedPlan plan; + + @Mock + private QueryService queryService; + + @Mock + private ResponseListener queryListener; + + @Mock + private ResponseListener explainListener; + + @Mock + private ExecutionEngine.QueryResponse queryResponse; + + private QueryPlanFactory factory; + + @BeforeEach + void init() { + factory = new QueryPlanFactory(queryService); + } + + @Test + public void createFromQueryShouldSuccess() { + Statement query = new Query(plan); + AbstractPlan queryExecution = + factory.create(query, Optional.of(queryListener), Optional.empty()); + assertTrue(queryExecution instanceof QueryPlan); + } + + @Test + public void createFromExplainShouldSuccess() { + Statement query = new Explain(new Query(plan)); + AbstractPlan queryExecution = + factory.create(query, Optional.empty(), Optional.of(explainListener)); + assertTrue(queryExecution instanceof ExplainPlan); + } + + @Test + public void createFromQueryWithoutQueryListenerShouldThrowException() { + Statement query = new Query(plan); + + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> factory.create(query, + Optional.empty(), Optional.empty())); + assertEquals("[BUG] query listener must be not null", exception.getMessage()); + } + + @Test + public void createFromExplainWithoutExplainListenerShouldThrowException() { + Statement query = new Explain(new Query(plan)); + + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> factory.create(query, + Optional.empty(), Optional.empty())); + assertEquals("[BUG] explain listener must be not null", exception.getMessage()); + } + + @Test + public void noConsumerResponseChannel() { + IllegalStateException exception = + assertThrows( + IllegalStateException.class, + () -> NO_CONSUMER_RESPONSE_LISTENER.onResponse(queryResponse)); + assertEquals( + "[BUG] query response should not sent to unexpected channel", exception.getMessage()); + + exception = + assertThrows( + IllegalStateException.class, + () -> NO_CONSUMER_RESPONSE_LISTENER.onFailure(new RuntimeException())); + assertEquals( + "[BUG] exception response should not sent to unexpected channel", exception.getMessage()); + } +} diff --git a/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java b/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java new file mode 100644 index 0000000000..834db76996 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java @@ -0,0 +1,58 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.executor.execution; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.ast.tree.UnresolvedPlan; +import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.executor.ExecutionEngine; +import org.opensearch.sql.executor.QueryId; +import org.opensearch.sql.executor.QueryService; + +@ExtendWith(MockitoExtension.class) +class QueryPlanTest { + + @Mock + private QueryId queryId; + + @Mock + private UnresolvedPlan plan; + + @Mock + private QueryService queryService; + + @Mock + private ResponseListener explainListener; + + @Mock + private ResponseListener queryListener; + + @Test + public void execute() { + QueryPlan query = new QueryPlan(queryId, plan, queryService, queryListener); + query.execute(); + + verify(queryService, times(1)).execute(any(), any()); + } + + @Test + public void explain() { + QueryPlan query = new QueryPlan(queryId, plan, queryService, queryListener); + query.explain(explainListener); + + verify(queryService, times(1)).explain(plan, explainListener); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java index 94cafef35c..bcd0c0ffb8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java @@ -18,23 +18,40 @@ import org.opensearch.client.Request; import org.opensearch.client.RestClient; import org.opensearch.client.RestHighLevelClient; +import org.opensearch.sql.analysis.Analyzer; +import org.opensearch.sql.analysis.ExpressionAnalyzer; import org.opensearch.sql.catalog.CatalogService; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.executor.DefaultQueryManager; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.ExecutionEngine.QueryResponse; +import org.opensearch.sql.executor.QueryManager; +import org.opensearch.sql.executor.QueryService; +import org.opensearch.sql.executor.execution.QueryPlanFactory; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.config.ExpressionConfig; +import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.monitor.AlwaysHealthyMonitor; import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.client.OpenSearchRestClient; import org.opensearch.sql.opensearch.executor.OpenSearchExecutionEngine; import org.opensearch.sql.opensearch.executor.protector.OpenSearchExecutionProtector; import org.opensearch.sql.opensearch.storage.OpenSearchStorageEngine; +import org.opensearch.sql.planner.Planner; +import org.opensearch.sql.planner.optimizer.LogicalPlanOptimizer; import org.opensearch.sql.plugin.catalog.CatalogServiceImpl; import org.opensearch.sql.ppl.config.PPLServiceConfig; import org.opensearch.sql.ppl.domain.PPLQueryRequest; import org.opensearch.sql.protocol.response.QueryResult; import org.opensearch.sql.protocol.response.format.SimpleJsonResponseFormatter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Scope; /** * Run PPL with query engine outside OpenSearch cluster. This IT doesn't require our plugin @@ -56,10 +73,13 @@ public void init() { AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); context.registerBean(ExecutionEngine.class, () -> new OpenSearchExecutionEngine(client, new OpenSearchExecutionProtector(new AlwaysHealthyMonitor()))); - context.register(PPLServiceConfig.class); + context.registerBean(OpenSearchClient.class, () -> client); + context.registerBean(Settings.class, () -> defaultSettings()); OpenSearchStorageEngine openSearchStorageEngine = new OpenSearchStorageEngine(client, defaultSettings()); CatalogServiceImpl.getInstance().registerDefaultOpenSearchCatalog(openSearchStorageEngine); context.registerBean(CatalogService.class, CatalogServiceImpl::getInstance); + context.register(StandaloneConfig.class); + context.register(PPLServiceConfig.class); context.refresh(); pplService = context.getBean(PPLService.class); @@ -144,4 +164,39 @@ public InternalRestHighLevelClient(RestClient restClient) { super(restClient, RestClient::close, Collections.emptyList()); } } + + @Configuration + @Import({ExpressionConfig.class}) + static class StandaloneConfig { + @Autowired + private CatalogService catalogService; + + @Autowired + private ExecutionEngine executionEngine; + + @Bean + QueryManager queryManager() { + return new DefaultQueryManager(); + } + + @Bean + @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) + QueryPlanFactory queryExecutionFactory(BuiltinFunctionRepository functionRepository) { + catalogService + .getCatalogs() + .forEach( + catalog -> + catalog + .getStorageEngine() + .getFunctions() + .forEach( + functionResolver -> + functionRepository.register(catalog.getName(), functionResolver))); + Analyzer analyzer = new Analyzer(new ExpressionAnalyzer(functionRepository), + catalogService, functionRepository); + Planner planner = + new Planner(LogicalPlanOptimizer.create(new DSL(functionRepository))); + return new QueryPlanFactory(new QueryService(analyzer, executionEngine, planner)); + } + } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSQLQueryAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSQLQueryAction.java index 6524ac2f0e..a5a3ac5a4f 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSQLQueryAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSQLQueryAction.java @@ -6,12 +6,12 @@ package org.opensearch.sql.legacy.plugin; -import static org.opensearch.rest.RestStatus.INTERNAL_SERVER_ERROR; import static org.opensearch.rest.RestStatus.OK; import static org.opensearch.sql.executor.ExecutionEngine.QueryResponse; import static org.opensearch.sql.protocol.response.format.JsonResponseFormatter.Style.PRETTY; import java.util.List; +import java.util.function.BiConsumer; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.client.node.NodeClient; @@ -22,11 +22,11 @@ import org.opensearch.rest.RestStatus; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.common.utils.QueryContext; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponse; import org.opensearch.sql.legacy.metrics.MetricName; import org.opensearch.sql.legacy.metrics.Metrics; import org.opensearch.sql.opensearch.security.SecurityAccess; -import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.protocol.response.QueryResult; import org.opensearch.sql.protocol.response.format.CsvResponseFormatter; import org.opensearch.sql.protocol.response.format.Format; @@ -76,39 +76,67 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient nod /** * Prepare REST channel consumer for a SQL query request. - * @param request SQL request - * @param nodeClient node client - * @return channel consumer + * + * @param request SQL request + * @param fallbackHandler handle request fallback to legacy engine. + * @param executionErrorHandler handle error response during new engine execution. + * @return {@link RestChannelConsumer} */ - public RestChannelConsumer prepareRequest(SQLQueryRequest request, NodeClient nodeClient) { + public RestChannelConsumer prepareRequest( + SQLQueryRequest request, + BiConsumer fallbackHandler, + BiConsumer executionErrorHandler) { if (!request.isSupported()) { - return NOT_SUPPORTED_YET; + return channel -> fallbackHandler.accept(channel, new IllegalStateException("not supported")); } SQLService sqlService = SecurityAccess.doPrivileged(() -> applicationContext.getBean(SQLService.class)); - PhysicalPlan plan; - try { - // For now analyzing and planning stage may throw syntax exception as well - // which hints the fallback to legacy code is necessary here. - plan = sqlService.plan( - sqlService.analyze( - sqlService.parse(request.getQuery()))); - } catch (SyntaxCheckException e) { - // When explain, print info log for what unsupported syntax is causing fallback to old engine - if (request.isExplainRequest()) { - LOG.info("Request is falling back to old SQL engine due to: " + e.getMessage()); - } - return NOT_SUPPORTED_YET; - } if (request.isExplainRequest()) { - return channel -> sqlService.explain(plan, createExplainResponseListener(channel)); + return channel -> + sqlService.explain( + request, + fallBackListener( + channel, + createExplainResponseListener(channel, executionErrorHandler), + fallbackHandler)); + } else { + return channel -> + sqlService.execute( + request, + fallBackListener( + channel, + createQueryResponseListener(channel, request, executionErrorHandler), + fallbackHandler)); } - return channel -> sqlService.execute(plan, createQueryResponseListener(channel, request)); } - private ResponseListener createExplainResponseListener(RestChannel channel) { + private ResponseListener fallBackListener( + RestChannel channel, + ResponseListener next, + BiConsumer fallBackHandler) { + return new ResponseListener() { + @Override + public void onResponse(T response) { + LOG.error("[{}] Request is handled by new SQL query engine", + QueryContext.getRequestId()); + next.onResponse(response); + } + + @Override + public void onFailure(Exception e) { + if (e instanceof SyntaxCheckException) { + fallBackHandler.accept(channel, e); + } else { + next.onFailure(e); + } + } + }; + } + + private ResponseListener createExplainResponseListener( + RestChannel channel, BiConsumer errorHandler) { return new ResponseListener() { @Override public void onResponse(ExplainResponse response) { @@ -122,15 +150,15 @@ protected Object buildJsonObject(ExplainResponse response) { @Override public void onFailure(Exception e) { - LOG.error("Error happened during explain", e); - logAndPublishMetrics(e); - sendResponse(channel, INTERNAL_SERVER_ERROR, - "Failed to explain the query due to error: " + e.getMessage()); + errorHandler.accept(channel, e); } }; } - private ResponseListener createQueryResponseListener(RestChannel channel, SQLQueryRequest request) { + private ResponseListener createQueryResponseListener( + RestChannel channel, + SQLQueryRequest request, + BiConsumer errorHandler) { Format format = request.format(); ResponseFormatter formatter; if (format.equals(Format.CSV)) { @@ -149,9 +177,7 @@ public void onResponse(QueryResponse response) { @Override public void onFailure(Exception e) { - LOG.error("Error happened during query handling", e); - logAndPublishMetrics(e); - sendResponse(channel, INTERNAL_SERVER_ERROR, formatter.format(e)); + errorHandler.accept(channel, e); } }; } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java index d47dac9325..de09bcee1a 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java @@ -9,7 +9,6 @@ import static org.opensearch.rest.RestStatus.BAD_REQUEST; import static org.opensearch.rest.RestStatus.OK; import static org.opensearch.rest.RestStatus.SERVICE_UNAVAILABLE; -import static org.opensearch.sql.opensearch.executor.Scheduler.schedule; import com.alibaba.druid.sql.parser.ParserException; import com.google.common.collect.ImmutableList; @@ -153,27 +152,29 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli Format format = SqlRequestParam.getFormat(request.params()); - return channel -> schedule(client, () -> { - try { - // Route request to new query engine if it's supported already - SQLQueryRequest newSqlRequest = new SQLQueryRequest(sqlRequest.getJsonContent(), - sqlRequest.getSql(), request.path(), request.params()); - RestChannelConsumer result = newSqlQueryHandler.prepareRequest(newSqlRequest, client); - if (result != RestSQLQueryAction.NOT_SUPPORTED_YET) { - LOG.info("[{}] Request is handled by new SQL query engine", - QueryContext.getRequestId()); - result.accept(channel); - } else { - LOG.debug("[{}] Request {} is not supported and falling back to old SQL engine", - QueryContext.getRequestId(), newSqlRequest); - QueryAction queryAction = explainRequest(client, sqlRequest, format); - executeSqlRequest(request, queryAction, client, channel); - } - } catch (Exception e) { - logAndPublishMetrics(e); - reportError(channel, e, isClientError(e) ? BAD_REQUEST : SERVICE_UNAVAILABLE); - } - }); + // Route request to new query engine if it's supported already + SQLQueryRequest newSqlRequest = new SQLQueryRequest(sqlRequest.getJsonContent(), + sqlRequest.getSql(), request.path(), request.params()); + return newSqlQueryHandler.prepareRequest(newSqlRequest, + (restChannel, exception) -> { + try{ + if (newSqlRequest.isExplainRequest()) { + LOG.info("Request is falling back to old SQL engine due to: " + exception.getMessage()); + } + LOG.debug("[{}] Request {} is not supported and falling back to old SQL engine", + QueryContext.getRequestId(), newSqlRequest); + QueryAction queryAction = explainRequest(client, sqlRequest, format); + executeSqlRequest(request, queryAction, client, restChannel); + } catch (Exception e) { + logAndPublishMetrics(e); + reportError(restChannel, e, isClientError(e) ? BAD_REQUEST : SERVICE_UNAVAILABLE); + } + }, + (restChannel, exception) -> { + logAndPublishMetrics(exception); + reportError(restChannel, exception, isClientError(exception) ? + BAD_REQUEST : SERVICE_UNAVAILABLE); + }); } catch (Exception e) { logAndPublishMetrics(e); return channel -> reportError(channel, e, isClientError(e) ? BAD_REQUEST : SERVICE_UNAVAILABLE); diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/plugin/RestSQLQueryActionTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/plugin/RestSQLQueryActionTest.java index 6257f0dd95..4c9afe802e 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/plugin/RestSQLQueryActionTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/plugin/RestSQLQueryActionTest.java @@ -6,12 +6,15 @@ package org.opensearch.sql.legacy.plugin; -import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertSame; -import static org.opensearch.sql.legacy.plugin.RestSQLQueryAction.NOT_SUPPORTED_YET; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; import static org.opensearch.sql.legacy.plugin.RestSqlAction.EXPLAIN_API_ENDPOINT; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; +import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; import org.json.JSONObject; import org.junit.Before; import org.junit.Test; @@ -21,8 +24,14 @@ import org.mockito.junit.MockitoJUnitRunner; import org.opensearch.client.node.NodeClient; import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestRequest; import org.opensearch.sql.catalog.CatalogService; +import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.executor.ExecutionEngine; +import org.opensearch.sql.executor.QueryManager; +import org.opensearch.sql.executor.execution.QueryPlanFactory; import org.opensearch.sql.sql.config.SQLServiceConfig; import org.opensearch.sql.sql.domain.SQLQueryRequest; import org.opensearch.sql.storage.StorageEngine; @@ -30,13 +39,25 @@ import org.springframework.context.annotation.AnnotationConfigApplicationContext; @RunWith(MockitoJUnitRunner.class) -public class RestSQLQueryActionTest { +public class RestSQLQueryActionTest extends BaseRestHandler { private NodeClient nodeClient; @Mock private ThreadPool threadPool; + @Mock + private QueryManager queryManager; + + @Mock + private QueryPlanFactory factory; + + @Mock + private ExecutionEngine.Schema schema; + + @Mock + private RestChannel restChannel; + private AnnotationConfigApplicationContext context; @Before @@ -46,6 +67,8 @@ public void setup() { context.registerBean(StorageEngine.class, () -> Mockito.mock(StorageEngine.class)); context.registerBean(ExecutionEngine.class, () -> Mockito.mock(ExecutionEngine.class)); context.registerBean(CatalogService.class, () -> Mockito.mock(CatalogService.class)); + context.registerBean(QueryManager.class, () -> queryManager); + context.registerBean(QueryPlanFactory.class, () -> factory); context.register(SQLServiceConfig.class); context.refresh(); Mockito.lenient().when(threadPool.getThreadContext()) @@ -53,7 +76,7 @@ public void setup() { } @Test - public void handleQueryThatCanSupport() { + public void handleQueryThatCanSupport() throws Exception { SQLQueryRequest request = new SQLQueryRequest( new JSONObject("{\"query\": \"SELECT -123\"}"), "SELECT -123", @@ -61,11 +84,15 @@ public void handleQueryThatCanSupport() { ""); RestSQLQueryAction queryAction = new RestSQLQueryAction(context); - assertNotSame(NOT_SUPPORTED_YET, queryAction.prepareRequest(request, nodeClient)); + queryAction.prepareRequest(request, (channel, exception) -> { + fail(); + }, (channel, exception) -> { + fail(); + }).accept(restChannel); } @Test - public void handleExplainThatCanSupport() { + public void handleExplainThatCanSupport() throws Exception { SQLQueryRequest request = new SQLQueryRequest( new JSONObject("{\"query\": \"SELECT -123\"}"), "SELECT -123", @@ -73,11 +100,15 @@ public void handleExplainThatCanSupport() { ""); RestSQLQueryAction queryAction = new RestSQLQueryAction(context); - assertNotSame(NOT_SUPPORTED_YET, queryAction.prepareRequest(request, nodeClient)); + queryAction.prepareRequest(request, (channel, exception) -> { + fail(); + }, (channel, exception) -> { + fail(); + }).accept(restChannel); } @Test - public void skipQueryThatNotSupport() { + public void queryThatNotSupportIsHandledByFallbackHandler() throws Exception { SQLQueryRequest request = new SQLQueryRequest( new JSONObject( "{\"query\": \"SELECT name FROM test1 JOIN test2 ON test1.name = test2.name\"}"), @@ -85,8 +116,53 @@ public void skipQueryThatNotSupport() { QUERY_API_ENDPOINT, ""); + AtomicBoolean fallback = new AtomicBoolean(false); + RestSQLQueryAction queryAction = new RestSQLQueryAction(context); + queryAction.prepareRequest(request, (channel, exception) -> { + fallback.set(true); + assertTrue(exception instanceof SyntaxCheckException); + }, (channel, exception) -> { + fail(); + }).accept(restChannel); + + assertTrue(fallback.get()); + } + + @Test + public void queryExecutionFailedIsHandledByExecutionErrorHandler() throws Exception { + SQLQueryRequest request = new SQLQueryRequest( + new JSONObject( + "{\"query\": \"SELECT -123\"}"), + "SELECT -123", + QUERY_API_ENDPOINT, + ""); + + doThrow(new IllegalStateException("execution exception")) + .when(queryManager) + .submit(any()); + + AtomicBoolean executionErrorHandler = new AtomicBoolean(false); RestSQLQueryAction queryAction = new RestSQLQueryAction(context); - assertSame(NOT_SUPPORTED_YET, queryAction.prepareRequest(request, nodeClient)); + queryAction.prepareRequest(request, (channel, exception) -> { + assertTrue(exception instanceof SyntaxCheckException); + }, (channel, exception) -> { + executionErrorHandler.set(true); + assertTrue(exception instanceof IllegalStateException); + }).accept(restChannel); + + assertTrue(executionErrorHandler.get()); } + @Override + public String getName() { + // do nothing, RestChannelConsumer is protected which required to extend BaseRestHandler + return null; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient nodeClient) + throws IOException { + // do nothing, RestChannelConsumer is protected which required to extend BaseRestHandler + return null; + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchQueryManager.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchQueryManager.java new file mode 100644 index 0000000000..9c6fcdb825 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchQueryManager.java @@ -0,0 +1,50 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.opensearch.executor; + +import java.util.Map; +import lombok.RequiredArgsConstructor; +import org.apache.logging.log4j.ThreadContext; +import org.opensearch.client.node.NodeClient; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.sql.executor.QueryId; +import org.opensearch.sql.executor.QueryManager; +import org.opensearch.sql.executor.execution.AbstractPlan; +import org.opensearch.threadpool.ThreadPool; + +/** + * QueryManager implemented in OpenSearch cluster. + */ +@RequiredArgsConstructor +public class OpenSearchQueryManager implements QueryManager { + + private final NodeClient nodeClient; + + private static final String SQL_WORKER_THREAD_POOL_NAME = "sql-worker"; + + @Override + public QueryId submit(AbstractPlan queryPlan) { + schedule(nodeClient, () -> queryPlan.execute()); + + return queryPlan.getQueryId(); + } + + private void schedule(NodeClient client, Runnable task) { + ThreadPool threadPool = client.threadPool(); + threadPool.schedule(withCurrentContext(task), new TimeValue(0), SQL_WORKER_THREAD_POOL_NAME); + } + + private Runnable withCurrentContext(final Runnable task) { + final Map currentContext = ThreadContext.getImmutableContext(); + return () -> { + ThreadContext.putAll(currentContext); + task.run(); + }; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/Scheduler.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/Scheduler.java deleted file mode 100644 index 5567d1f9b2..0000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/Scheduler.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.executor; - -import java.util.Map; -import lombok.experimental.UtilityClass; -import org.apache.logging.log4j.ThreadContext; -import org.opensearch.client.node.NodeClient; -import org.opensearch.common.unit.TimeValue; -import org.opensearch.threadpool.ThreadPool; - -/** The scheduler which schedule the task run in sql-worker thread pool. */ -@UtilityClass -public class Scheduler { - - public static final String SQL_WORKER_THREAD_POOL_NAME = "sql-worker"; - - public static void schedule(NodeClient client, Runnable task) { - ThreadPool threadPool = client.threadPool(); - threadPool.schedule(withCurrentContext(task), new TimeValue(0), SQL_WORKER_THREAD_POOL_NAME); - } - - private static Runnable withCurrentContext(final Runnable task) { - final Map currentContext = ThreadContext.getImmutableContext(); - return () -> { - ThreadContext.putAll(currentContext); - task.run(); - }; - } -} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/SchedulerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchQueryManagerTest.java similarity index 51% rename from opensearch/src/test/java/org/opensearch/sql/opensearch/executor/SchedulerTest.java rename to opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchQueryManagerTest.java index f14bda7a95..6d2b9b13ce 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/SchedulerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchQueryManagerTest.java @@ -1,6 +1,9 @@ /* - * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ package org.opensearch.sql.opensearch.executor; @@ -14,18 +17,47 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.client.node.NodeClient; +import org.opensearch.sql.ast.tree.UnresolvedPlan; +import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.executor.ExecutionEngine; +import org.opensearch.sql.executor.QueryId; +import org.opensearch.sql.executor.QueryService; +import org.opensearch.sql.executor.execution.AbstractPlan; +import org.opensearch.sql.executor.execution.QueryPlan; import org.opensearch.threadpool.ThreadPool; @ExtendWith(MockitoExtension.class) -class SchedulerTest { +class OpenSearchQueryManagerTest { + + @Mock + private QueryId queryId; + + @Mock + private QueryService queryService; + + @Mock + private UnresolvedPlan plan; + + @Mock + private ResponseListener listener; + @Test - public void schedule() { + public void submitQuery() { NodeClient nodeClient = mock(NodeClient.class); ThreadPool threadPool = mock(ThreadPool.class); when(nodeClient.threadPool()).thenReturn(threadPool); + AtomicBoolean isRun = new AtomicBoolean(false); + AbstractPlan queryPlan = new QueryPlan(queryId, plan, queryService, listener) { + @Override + public void execute() { + isRun.set(true); + } + }; + doAnswer( invocation -> { Runnable task = invocation.getArgument(0); @@ -34,8 +66,8 @@ public void schedule() { }) .when(threadPool) .schedule(any(), any(), any()); - AtomicBoolean isRun = new AtomicBoolean(false); - Scheduler.schedule(nodeClient, () -> isRun.set(true)); + new OpenSearchQueryManager(nodeClient).submit(queryPlan); + assertTrue(isRun.get()); } } diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginConfig.java b/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginConfig.java index a2169eb839..596296522c 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginConfig.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginConfig.java @@ -10,20 +10,29 @@ package org.opensearch.sql.plugin.config; import org.opensearch.client.node.NodeClient; +import org.opensearch.sql.analysis.Analyzer; +import org.opensearch.sql.analysis.ExpressionAnalyzer; +import org.opensearch.sql.catalog.CatalogService; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.executor.ExecutionEngine; +import org.opensearch.sql.executor.QueryManager; +import org.opensearch.sql.executor.QueryService; +import org.opensearch.sql.executor.execution.QueryPlanFactory; +import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.config.ExpressionConfig; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; -import org.opensearch.sql.expression.function.OpenSearchFunctions; import org.opensearch.sql.monitor.ResourceMonitor; import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.client.OpenSearchNodeClient; import org.opensearch.sql.opensearch.executor.OpenSearchExecutionEngine; +import org.opensearch.sql.opensearch.executor.OpenSearchQueryManager; import org.opensearch.sql.opensearch.executor.protector.ExecutionProtector; import org.opensearch.sql.opensearch.executor.protector.OpenSearchExecutionProtector; import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; import org.opensearch.sql.opensearch.storage.OpenSearchStorageEngine; +import org.opensearch.sql.planner.Planner; +import org.opensearch.sql.planner.optimizer.LogicalPlanOptimizer; import org.opensearch.sql.storage.StorageEngine; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.config.ConfigurableBeanFactory; @@ -47,7 +56,7 @@ public class OpenSearchPluginConfig { private Settings settings; @Autowired - private BuiltinFunctionRepository functionRepository; + private CatalogService catalogService; @Bean @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) @@ -64,7 +73,6 @@ public StorageEngine storageEngine() { @Bean @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) public ExecutionEngine executionEngine() { - OpenSearchFunctions.register(functionRepository); return new OpenSearchExecutionEngine(client(), protector()); } @@ -79,4 +87,35 @@ public ResourceMonitor resourceMonitor() { public ExecutionProtector protector() { return new OpenSearchExecutionProtector(resourceMonitor()); } + + /** + * Per node singleton object. + */ + @Bean + public QueryManager queryManager() { + return new OpenSearchQueryManager(nodeClient); + } + + /** + * QueryPlanFactory. + */ + @Bean + @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) + public QueryPlanFactory queryExecutionFactory(BuiltinFunctionRepository functionRepository) { + catalogService + .getCatalogs() + .forEach( + catalog -> + catalog + .getStorageEngine() + .getFunctions() + .forEach( + functionResolver -> + functionRepository.register(catalog.getName(), functionResolver))); + Analyzer analyzer = new Analyzer(new ExpressionAnalyzer(functionRepository), + catalogService, functionRepository); + Planner planner = + new Planner(LogicalPlanOptimizer.create(new DSL(functionRepository))); + return new QueryPlanFactory(new QueryService(analyzer, executionEngine(), planner)); + } } diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java index 2bc3c8d72d..0f4d2f7d0c 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java @@ -9,7 +9,6 @@ import static org.opensearch.rest.RestStatus.INTERNAL_SERVER_ERROR; import static org.opensearch.rest.RestStatus.OK; import static org.opensearch.rest.RestStatus.SERVICE_UNAVAILABLE; -import static org.opensearch.sql.opensearch.executor.Scheduler.schedule; import com.google.common.collect.ImmutableList; import java.util.Arrays; @@ -112,43 +111,42 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient nod PPLQueryRequestFactory.getPPLRequest(request) ); - return channel -> schedule(nodeClient, () -> - nodeClient.execute( - PPLQueryAction.INSTANCE, - transportPPLQueryRequest, - new ActionListener<>() { - @Override - public void onResponse(TransportPPLQueryResponse response) { - sendResponse(channel, OK, response.getResult()); - } - - @Override - public void onFailure(Exception e) { - if (transportPPLQueryRequest.isExplainRequest()) { - LOG.error("Error happened during explain", e); - sendResponse( - channel, - INTERNAL_SERVER_ERROR, - "Failed to explain the query due to error: " + e.getMessage()); - } else if (e instanceof IllegalAccessException) { - reportError(channel, e, BAD_REQUEST); - } else { - LOG.error("Error happened during query handling", e); - if (isClientError(e)) { - Metrics.getInstance() - .getNumericalMetric(MetricName.PPL_FAILED_REQ_COUNT_CUS) - .increment(); + return channel -> + nodeClient.execute( + PPLQueryAction.INSTANCE, + transportPPLQueryRequest, + new ActionListener<>() { + @Override + public void onResponse(TransportPPLQueryResponse response) { + sendResponse(channel, OK, response.getResult()); + } + + @Override + public void onFailure(Exception e) { + if (transportPPLQueryRequest.isExplainRequest()) { + LOG.error("Error happened during explain", e); + sendResponse( + channel, + INTERNAL_SERVER_ERROR, + "Failed to explain the query due to error: " + e.getMessage()); + } else if (e instanceof IllegalAccessException) { reportError(channel, e, BAD_REQUEST); } else { - Metrics.getInstance() - .getNumericalMetric(MetricName.PPL_FAILED_REQ_COUNT_SYS) - .increment(); - reportError(channel, e, SERVICE_UNAVAILABLE); + LOG.error("Error happened during query handling", e); + if (isClientError(e)) { + Metrics.getInstance() + .getNumericalMetric(MetricName.PPL_FAILED_REQ_COUNT_CUS) + .increment(); + reportError(channel, e, BAD_REQUEST); + } else { + Metrics.getInstance() + .getNumericalMetric(MetricName.PPL_FAILED_REQ_COUNT_SYS) + .increment(); + reportError(channel, e, SERVICE_UNAVAILABLE); + } } } - } - } - )); + }); } private void sendResponse(RestChannel channel, RestStatus status, String content) { diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index a1061b0020..c281b9130d 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -14,6 +14,14 @@ root /** statement */ pplStatement + : dmlStatement + ; + +dmlStatement + : queryStatement + ; + +queryStatement : pplCommands (PIPE commands)* ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java b/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java index e6cbbb92f5..e11edc1646 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java @@ -8,41 +8,35 @@ import static org.opensearch.sql.executor.ExecutionEngine.QueryResponse; +import java.util.Optional; import lombok.RequiredArgsConstructor; import org.antlr.v4.runtime.tree.ParseTree; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.sql.analysis.AnalysisContext; -import org.opensearch.sql.analysis.Analyzer; -import org.opensearch.sql.analysis.ExpressionAnalyzer; -import org.opensearch.sql.ast.tree.UnresolvedPlan; -import org.opensearch.sql.catalog.CatalogService; +import org.opensearch.sql.ast.statement.Statement; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.common.utils.QueryContext; -import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponse; -import org.opensearch.sql.expression.DSL; -import org.opensearch.sql.expression.function.BuiltinFunctionRepository; -import org.opensearch.sql.planner.Planner; -import org.opensearch.sql.planner.logical.LogicalPlan; -import org.opensearch.sql.planner.optimizer.LogicalPlanOptimizer; -import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.executor.QueryManager; +import org.opensearch.sql.executor.execution.AbstractPlan; +import org.opensearch.sql.executor.execution.QueryPlanFactory; import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; import org.opensearch.sql.ppl.domain.PPLQueryRequest; import org.opensearch.sql.ppl.parser.AstBuilder; import org.opensearch.sql.ppl.parser.AstExpressionBuilder; +import org.opensearch.sql.ppl.parser.AstStatementBuilder; import org.opensearch.sql.ppl.utils.PPLQueryDataAnonymizer; -import org.opensearch.sql.ppl.utils.UnresolvedPlanHelper; +/** + * PPLService. + */ @RequiredArgsConstructor public class PPLService { private final PPLSyntaxParser parser; - private final ExecutionEngine openSearchExecutionEngine; - - private final BuiltinFunctionRepository repository; + private final QueryManager queryManager; - private final CatalogService catalogService; + private final QueryPlanFactory queryExecutionFactory; private final PPLQueryDataAnonymizer anonymizer = new PPLQueryDataAnonymizer(); @@ -56,7 +50,7 @@ public class PPLService { */ public void execute(PPLQueryRequest request, ResponseListener listener) { try { - openSearchExecutionEngine.execute(plan(request), listener); + queryManager.submit(plan(request, Optional.of(listener), Optional.empty())); } catch (Exception e) { listener.onFailure(e); } @@ -71,28 +65,31 @@ public void execute(PPLQueryRequest request, ResponseListener lis */ public void explain(PPLQueryRequest request, ResponseListener listener) { try { - openSearchExecutionEngine.explain(plan(request), listener); + queryManager.submit(plan(request, Optional.empty(), Optional.of(listener))); } catch (Exception e) { listener.onFailure(e); } } - private PhysicalPlan plan(PPLQueryRequest request) { + private AbstractPlan plan( + PPLQueryRequest request, + Optional> queryListener, + Optional> explainListener) { // 1.Parse query and convert parse tree (CST) to abstract syntax tree (AST) ParseTree cst = parser.parse(request.getRequest()); - UnresolvedPlan ast = cst.accept( - new AstBuilder(new AstExpressionBuilder(), request.getRequest())); - LOG.info("[{}] Incoming request {}", QueryContext.getRequestId(), - anonymizer.anonymizeData(ast)); - // 2.Analyze abstract syntax to generate logical plan - LogicalPlan logicalPlan = - new Analyzer(new ExpressionAnalyzer(repository), catalogService, repository).analyze( - UnresolvedPlanHelper.addSelectAll(ast), - new AnalysisContext()); + Statement statement = + cst.accept( + new AstStatementBuilder( + new AstBuilder(new AstExpressionBuilder(), request.getRequest()), + AstStatementBuilder.StatementBuilderContext.builder() + .isExplain(request.isExplainRequest()) + .build())); - // 3.Generate optimal physical plan from logical plan - return new Planner(LogicalPlanOptimizer.create(new DSL(repository))) - .plan(logicalPlan); - } + LOG.info( + "[{}] Incoming request {}", + QueryContext.getRequestId(), + anonymizer.anonymizeStatement(statement)); + return queryExecutionFactory.create(statement, queryListener, explainListener); + } } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/config/PPLServiceConfig.java b/ppl/src/main/java/org/opensearch/sql/ppl/config/PPLServiceConfig.java index 8e6c4f4f7b..1067bbaa6b 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/config/PPLServiceConfig.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/config/PPLServiceConfig.java @@ -6,32 +6,24 @@ package org.opensearch.sql.ppl.config; -import org.opensearch.sql.catalog.CatalogService; -import org.opensearch.sql.catalog.model.ConnectorType; -import org.opensearch.sql.executor.ExecutionEngine; -import org.opensearch.sql.expression.config.ExpressionConfig; -import org.opensearch.sql.expression.function.BuiltinFunctionRepository; +import org.opensearch.sql.executor.QueryManager; +import org.opensearch.sql.executor.execution.QueryPlanFactory; import org.opensearch.sql.ppl.PPLService; import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Scope; @Configuration -@Import({ExpressionConfig.class}) public class PPLServiceConfig { @Autowired - private ExecutionEngine executionEngine; + private QueryManager queryManager; @Autowired - private CatalogService catalogService; - - @Autowired - private BuiltinFunctionRepository functionRepository; + private QueryPlanFactory queryPlanFactory; /** * The registration of OpenSearch storage engine happens here because @@ -42,12 +34,7 @@ public class PPLServiceConfig { @Bean @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) public PPLService pplService() { - catalogService.getCatalogs() - .forEach(catalog -> catalog.getStorageEngine().getFunctions() - .forEach(functionResolver -> functionRepository - .register(catalog.getName(), functionResolver))); - return new PPLService(new PPLSyntaxParser(), executionEngine, - functionRepository, catalogService); + return new PPLService(new PPLSyntaxParser(), queryManager, queryPlanFactory); } } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java b/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java index 0d8a4c63d1..87532e01d0 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java @@ -9,16 +9,17 @@ import java.util.Locale; import java.util.Optional; import lombok.Getter; -import lombok.RequiredArgsConstructor; import lombok.Setter; import lombok.experimental.Accessors; import org.json.JSONObject; import org.opensearch.sql.protocol.response.format.Format; import org.opensearch.sql.protocol.response.format.JsonResponseFormatter; -@RequiredArgsConstructor public class PPLQueryRequest { - public static final PPLQueryRequest NULL = new PPLQueryRequest("", null, "", ""); + + private static final String DEFAULT_PPL_PATH = "/_plugins/_ppl"; + + public static final PPLQueryRequest NULL = new PPLQueryRequest("", null, DEFAULT_PPL_PATH, ""); private final String pplQuery; @Getter @@ -38,13 +39,17 @@ public class PPLQueryRequest { @Accessors(fluent = true) private JsonResponseFormatter.Style style = JsonResponseFormatter.Style.COMPACT; + public PPLQueryRequest(String pplQuery, JSONObject jsonContent, String path) { + this(pplQuery, jsonContent, path, ""); + } + /** * Constructor of PPLQueryRequest. */ public PPLQueryRequest(String pplQuery, JSONObject jsonContent, String path, String format) { this.pplQuery = pplQuery; this.jsonContent = jsonContent; - this.path = path; + this.path = Optional.ofNullable(path).orElse(DEFAULT_PPL_PATH); this.format = format; } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index b0d17940a4..d58cf9dad2 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -13,7 +13,6 @@ import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.FieldsCommandContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.FromClauseContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.HeadCommandContext; -import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.PplStatementContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.RareCommandContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.RenameCommandContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.SearchFilterFromContext; @@ -87,7 +86,7 @@ public class AstBuilder extends OpenSearchPPLParserBaseVisitor { private final String query; @Override - public UnresolvedPlan visitPplStatement(PplStatementContext ctx) { + public UnresolvedPlan visitQueryStatement(OpenSearchPPLParser.QueryStatementContext ctx) { UnresolvedPlan pplCommand = visit(ctx.pplCommands()); return ctx.commands() .stream() diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java new file mode 100644 index 0000000000..e4f40e9a11 --- /dev/null +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java @@ -0,0 +1,58 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.ppl.parser; + +import com.google.common.collect.ImmutableList; +import lombok.Builder; +import lombok.Data; +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.ast.expression.AllFields; +import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.Query; +import org.opensearch.sql.ast.statement.Statement; +import org.opensearch.sql.ast.tree.Project; +import org.opensearch.sql.ast.tree.UnresolvedPlan; +import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser; +import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParserBaseVisitor; + +/** + * Build {@link Statement} from PPL Query. + */ +@RequiredArgsConstructor +public class AstStatementBuilder extends OpenSearchPPLParserBaseVisitor { + + private final AstBuilder astBuilder; + + private final StatementBuilderContext context; + + @Override + public Statement visitDmlStatement(OpenSearchPPLParser.DmlStatementContext ctx) { + Query query = new Query(addSelectAll(astBuilder.visit(ctx))); + return context.isExplain ? new Explain(query) : query; + } + + @Override + protected Statement aggregateResult(Statement aggregate, Statement nextResult) { + return nextResult != null ? nextResult : aggregate; + } + + @Data + @Builder + public static class StatementBuilderContext { + private final boolean isExplain; + } + + private UnresolvedPlan addSelectAll(UnresolvedPlan plan) { + if ((plan instanceof Project) && !((Project) plan).isExcluded()) { + return plan; + } else { + return new Project(ImmutableList.of(AllFields.of())).attach(plan); + } + } +} diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 314d97009c..1f0e6f0d52 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -29,6 +29,9 @@ import org.opensearch.sql.ast.expression.Or; import org.opensearch.sql.ast.expression.UnresolvedExpression; import org.opensearch.sql.ast.expression.Xor; +import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.Query; +import org.opensearch.sql.ast.statement.Statement; import org.opensearch.sql.ast.tree.Aggregation; import org.opensearch.sql.ast.tree.Dedupe; import org.opensearch.sql.ast.tree.Eval; @@ -74,6 +77,23 @@ public String anonymizeData(UnresolvedPlan plan) { return plan.accept(this, null); } + public String anonymizeStatement(Statement plan) { + return plan.accept(this, null); + } + + /** + * Handle Query Statement. + */ + @Override + public String visitQuery(Query node, String context) { + return node.getPlan().accept(this, null); + } + + @Override + public String visitExplain(Explain node, String context) { + return node.getStatement().accept(this, null); + } + @Override public String visitRelation(Relation node, String context) { return StringUtils.format("source=%s", node.getTableName()); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java index 0ecebf160d..9a560e25b0 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java @@ -8,86 +8,51 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.when; -import com.google.common.collect.ImmutableMap; import java.util.Collections; -import java.util.Set; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; -import org.opensearch.sql.catalog.CatalogService; -import org.opensearch.sql.catalog.model.Catalog; -import org.opensearch.sql.catalog.model.ConnectorType; import org.opensearch.sql.common.response.ResponseListener; -import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.executor.DefaultQueryManager; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponse; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponseNode; import org.opensearch.sql.executor.ExecutionEngine.QueryResponse; -import org.opensearch.sql.expression.DSL; -import org.opensearch.sql.expression.function.BuiltinFunctionRepository; -import org.opensearch.sql.expression.function.FunctionResolver; -import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.executor.QueryManager; +import org.opensearch.sql.executor.QueryService; +import org.opensearch.sql.executor.execution.QueryPlanFactory; import org.opensearch.sql.ppl.config.PPLServiceConfig; import org.opensearch.sql.ppl.domain.PPLQueryRequest; -import org.opensearch.sql.storage.StorageEngine; -import org.opensearch.sql.storage.Table; import org.springframework.context.annotation.AnnotationConfigApplicationContext; @RunWith(MockitoJUnitRunner.class) public class PPLServiceTest { - private AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - - private PPLService pplService; - - @Mock - private StorageEngine storageEngine; - @Mock - private ExecutionEngine executionEngine; + private static String QUERY = "/_plugins/_ppl"; - @Mock - private CatalogService catalogService; + private static String EXPLAIN = "/_plugins/_ppl/_explain"; - @Mock - private BuiltinFunctionRepository functionRepository; - - @Mock - private DSL dsl; + private AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - @Mock - private Table table; + private PPLService pplService; @Mock - private PhysicalPlan plan; + private QueryService queryService; @Mock private ExecutionEngine.Schema schema; - @Mock - private FunctionResolver functionResolver; - /** * Setup the test context. */ @Before public void setUp() { - when(table.getFieldTypes()).thenReturn(ImmutableMap.of("a", ExprCoreType.INTEGER)); - when(table.implement(any())).thenReturn(plan); - when(storageEngine.getTable(any())).thenReturn(table); - when(catalogService.getCatalogs()) - .thenReturn(Set.of(new Catalog("prometheus", ConnectorType.PROMETHEUS, storageEngine))); - when(catalogService.getCatalog(any())) - .thenReturn(new Catalog("prometheus", ConnectorType.PROMETHEUS, storageEngine)); - when(storageEngine.getFunctions()).thenReturn(Collections.singleton(functionResolver)); - - context.registerBean(StorageEngine.class, () -> storageEngine); - context.registerBean(ExecutionEngine.class, () -> executionEngine); - context.registerBean(CatalogService.class, () -> catalogService); + context.registerBean(QueryManager.class, DefaultQueryManager::new); + context.registerBean(QueryPlanFactory.class, () -> new QueryPlanFactory(queryService)); context.register(PPLServiceConfig.class); context.refresh(); pplService = context.getBean(PPLService.class); @@ -99,9 +64,9 @@ public void testExecuteShouldPass() { ResponseListener listener = invocation.getArgument(1); listener.onResponse(new QueryResponse(schema, Collections.emptyList())); return null; - }).when(executionEngine).execute(any(), any()); + }).when(queryService).execute(any(), any()); - pplService.execute(new PPLQueryRequest("search source=t a=1", null, null), + pplService.execute(new PPLQueryRequest("search source=t a=1", null, QUERY), new ResponseListener() { @Override public void onResponse(QueryResponse pplQueryResponse) { @@ -121,9 +86,9 @@ public void testExecuteCsvFormatShouldPass() { ResponseListener listener = invocation.getArgument(1); listener.onResponse(new QueryResponse(schema, Collections.emptyList())); return null; - }).when(executionEngine).execute(any(), any()); + }).when(queryService).execute(any(), any()); - pplService.execute(new PPLQueryRequest("search source=t a=1", null, "/_plugins/_ppl", "csv"), + pplService.execute(new PPLQueryRequest("search source=t a=1", null, QUERY, "csv"), new ResponseListener() { @Override public void onResponse(QueryResponse pplQueryResponse) { @@ -138,15 +103,13 @@ public void onFailure(Exception e) { @Test public void testExplainShouldPass() { - when(catalogService.getCatalog(any())) - .thenReturn(new Catalog("prometheus", ConnectorType.PROMETHEUS, storageEngine)); doAnswer(invocation -> { ResponseListener listener = invocation.getArgument(1); listener.onResponse(new ExplainResponse(new ExplainResponseNode("test"))); return null; - }).when(executionEngine).explain(any(), any()); + }).when(queryService).explain(any(), any()); - pplService.explain(new PPLQueryRequest("search source=t a=1", null, null), + pplService.explain(new PPLQueryRequest("search source=t a=1", null, EXPLAIN), new ResponseListener() { @Override public void onResponse(ExplainResponse pplQueryResponse) { @@ -161,7 +124,7 @@ public void onFailure(Exception e) { @Test public void testExecuteWithIllegalQueryShouldBeCaughtByHandler() { - pplService.execute(new PPLQueryRequest("search", null, null), + pplService.execute(new PPLQueryRequest("search", null, QUERY), new ResponseListener() { @Override public void onResponse(QueryResponse pplQueryResponse) { @@ -177,7 +140,7 @@ public void onFailure(Exception e) { @Test public void testExplainWithIllegalQueryShouldBeCaughtByHandler() { - pplService.explain(new PPLQueryRequest("search", null, null), + pplService.explain(new PPLQueryRequest("search", null, QUERY), new ResponseListener<>() { @Override public void onResponse(ExplainResponse pplQueryResponse) { @@ -197,9 +160,9 @@ public void testPrometheusQuery() { ResponseListener listener = invocation.getArgument(1); listener.onResponse(new QueryResponse(schema, Collections.emptyList())); return null; - }).when(executionEngine).execute(any(), any()); + }).when(queryService).execute(any(), any()); - pplService.execute(new PPLQueryRequest("source = prometheus.http_requests_total", null, null), + pplService.execute(new PPLQueryRequest("source = prometheus.http_requests_total", null, QUERY), new ResponseListener<>() { @Override public void onResponse(QueryResponse pplQueryResponse) { @@ -214,8 +177,8 @@ public void onFailure(Exception e) { } @Test - public void test() { - pplService.execute(new PPLQueryRequest("search", null, null), + public void testInvalidPPLQuery() { + pplService.execute(new PPLQueryRequest("search", null, QUERY), new ResponseListener() { @Override public void onResponse(QueryResponse pplQueryResponse) { diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstStatementBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstStatementBuilderTest.java new file mode 100644 index 0000000000..4760024692 --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstStatementBuilderTest.java @@ -0,0 +1,72 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.ppl.parser; + +import static org.junit.Assert.assertEquals; +import static org.opensearch.sql.ast.dsl.AstDSL.compare; +import static org.opensearch.sql.ast.dsl.AstDSL.field; +import static org.opensearch.sql.ast.dsl.AstDSL.filter; +import static org.opensearch.sql.ast.dsl.AstDSL.intLiteral; +import static org.opensearch.sql.ast.dsl.AstDSL.project; +import static org.opensearch.sql.ast.dsl.AstDSL.relation; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.opensearch.sql.ast.Node; +import org.opensearch.sql.ast.expression.AllFields; +import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.Query; +import org.opensearch.sql.ast.statement.Statement; +import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; + +public class AstStatementBuilderTest { + + @Rule + public ExpectedException exceptionRule = ExpectedException.none(); + + private PPLSyntaxParser parser = new PPLSyntaxParser(); + + @Test + public void buildQueryStatement() { + assertEqual( + "search source=t a=1", + new Query( + project( + filter(relation("t"), compare("=", field("a"), intLiteral(1))), AllFields.of()))); + } + + @Test + public void buildExplainStatement() { + assertExplainEqual( + "search source=t a=1", + new Explain( + new Query( + project( + filter(relation("t"), compare("=", field("a"), intLiteral(1))), + AllFields.of())))); + } + + private void assertEqual(String query, Statement expectedStatement) { + Node actualPlan = plan(query, false); + assertEquals(expectedStatement, actualPlan); + } + + private void assertExplainEqual(String query, Statement expectedStatement) { + Node actualPlan = plan(query, true); + assertEquals(expectedStatement, actualPlan); + } + + private Node plan(String query, boolean isExplain) { + final AstStatementBuilder builder = + new AstStatementBuilder(new AstBuilder(new AstExpressionBuilder(), query), + AstStatementBuilder.StatementBuilderContext.builder().isExplain(isExplain).build()); + return builder.visit(parser.parse(query)); + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 52f2f18b72..1998647dba 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -7,24 +7,20 @@ package org.opensearch.sql.ppl.utils; import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.when; import static org.opensearch.sql.ast.dsl.AstDSL.field; import static org.opensearch.sql.ast.dsl.AstDSL.projectWithArg; import static org.opensearch.sql.ast.dsl.AstDSL.relation; -import com.google.common.collect.ImmutableSet; import java.util.Collections; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; +import org.opensearch.sql.ast.statement.Statement; import org.opensearch.sql.ast.tree.UnresolvedPlan; -import org.opensearch.sql.catalog.CatalogService; import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; import org.opensearch.sql.ppl.parser.AstBuilder; import org.opensearch.sql.ppl.parser.AstExpressionBuilder; +import org.opensearch.sql.ppl.parser.AstStatementBuilder; @RunWith(MockitoJUnitRunner.class) public class PPLQueryDataAnonymizerTest { @@ -174,6 +170,20 @@ public void testDateFunction() { ); } + @Test + public void testExplain() { + assertEquals("source=t | fields + a", + anonymizeStatement("source=t | fields a", true) + ); + } + + @Test + public void testQuery() { + assertEquals("source=t | fields + a", + anonymizeStatement("source=t | fields a", false) + ); + } + @Test public void anonymizeFieldsNoArg() { assertEquals("source=t | fields + f", @@ -190,4 +200,14 @@ private String anonymize(UnresolvedPlan plan) { final PPLQueryDataAnonymizer anonymize = new PPLQueryDataAnonymizer(); return anonymize.anonymizeData(plan); } + + private String anonymizeStatement(String query, boolean isExplain) { + AstStatementBuilder builder = + new AstStatementBuilder( + new AstBuilder(new AstExpressionBuilder(), query), + AstStatementBuilder.StatementBuilderContext.builder().isExplain(isExplain).build()); + Statement statement = builder.visit(parser.parse(query)); + PPLQueryDataAnonymizer anonymize = new PPLQueryDataAnonymizer(); + return anonymize.anonymizeStatement(statement); + } } diff --git a/sql/src/main/java/org/opensearch/sql/sql/SQLService.java b/sql/src/main/java/org/opensearch/sql/sql/SQLService.java index 76de0f6249..082a3e9581 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/SQLService.java +++ b/sql/src/main/java/org/opensearch/sql/sql/SQLService.java @@ -6,25 +6,20 @@ package org.opensearch.sql.sql; +import java.util.Optional; import lombok.RequiredArgsConstructor; import org.antlr.v4.runtime.tree.ParseTree; -import org.opensearch.sql.analysis.AnalysisContext; -import org.opensearch.sql.analysis.Analyzer; -import org.opensearch.sql.ast.tree.UnresolvedPlan; +import org.opensearch.sql.ast.statement.Statement; import org.opensearch.sql.common.response.ResponseListener; -import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponse; import org.opensearch.sql.executor.ExecutionEngine.QueryResponse; -import org.opensearch.sql.expression.DSL; -import org.opensearch.sql.expression.function.BuiltinFunctionRepository; -import org.opensearch.sql.planner.Planner; -import org.opensearch.sql.planner.logical.LogicalPlan; -import org.opensearch.sql.planner.optimizer.LogicalPlanOptimizer; -import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.executor.QueryManager; +import org.opensearch.sql.executor.execution.AbstractPlan; +import org.opensearch.sql.executor.execution.QueryPlanFactory; import org.opensearch.sql.sql.antlr.SQLSyntaxParser; import org.opensearch.sql.sql.domain.SQLQueryRequest; import org.opensearch.sql.sql.parser.AstBuilder; -import org.opensearch.sql.storage.StorageEngine; +import org.opensearch.sql.sql.parser.AstStatementBuilder; /** * SQL service. @@ -34,75 +29,52 @@ public class SQLService { private final SQLSyntaxParser parser; - private final Analyzer analyzer; + private final QueryManager queryManager; - private final ExecutionEngine executionEngine; - - private final BuiltinFunctionRepository repository; + private final QueryPlanFactory queryExecutionFactory; /** - * Parse, analyze, plan and execute the query. - * @param request SQL query request - * @param listener callback listener + * Given {@link SQLQueryRequest}, execute it. Using listener to listen result. + * + * @param request {@link SQLQueryRequest} + * @param listener callback listener */ public void execute(SQLQueryRequest request, ResponseListener listener) { try { - executionEngine.execute( - plan( - analyze( - parse(request.getQuery()))), listener); + queryManager.submit(plan(request, Optional.of(listener), Optional.empty())); } catch (Exception e) { listener.onFailure(e); } } /** - * Given physical plan, execute it and listen on response. - * @param plan physical plan - * @param listener callback listener + * Given {@link SQLQueryRequest}, explain it. Using listener to listen result. + * + * @param request {@link SQLQueryRequest} + * @param listener callback listener */ - public void execute(PhysicalPlan plan, ResponseListener listener) { + public void explain(SQLQueryRequest request, ResponseListener listener) { try { - executionEngine.execute(plan, listener); + queryManager.submit(plan(request, Optional.empty(), Optional.of(listener))); } catch (Exception e) { listener.onFailure(e); } } - /** - * Given physical plan, explain it. - * @param plan physical plan - * @param listener callback listener - */ - public void explain(PhysicalPlan plan, ResponseListener listener) { - try { - executionEngine.explain(plan, listener); - } catch (Exception e) { - listener.onFailure(e); - } - } + private AbstractPlan plan( + SQLQueryRequest request, + Optional> queryListener, + Optional> explainListener) { + // 1.Parse query and convert parse tree (CST) to abstract syntax tree (AST) + ParseTree cst = parser.parse(request.getQuery()); + Statement statement = + cst.accept( + new AstStatementBuilder( + new AstBuilder(request.getQuery()), + AstStatementBuilder.StatementBuilderContext.builder() + .isExplain(request.isExplainRequest()) + .build())); - /** - * Parse query and convert parse tree (CST) to abstract syntax tree (AST). - */ - public UnresolvedPlan parse(String query) { - ParseTree cst = parser.parse(query); - return cst.accept(new AstBuilder(query)); + return queryExecutionFactory.create(statement, queryListener, explainListener); } - - /** - * Analyze abstract syntax to generate logical plan. - */ - public LogicalPlan analyze(UnresolvedPlan ast) { - return analyzer.analyze(ast, new AnalysisContext()); - } - - /** - * Generate optimal physical plan from logical plan. - */ - public PhysicalPlan plan(LogicalPlan logicalPlan) { - return new Planner(LogicalPlanOptimizer.create(new DSL(repository))) - .plan(logicalPlan); - } - } diff --git a/sql/src/main/java/org/opensearch/sql/sql/config/SQLServiceConfig.java b/sql/src/main/java/org/opensearch/sql/sql/config/SQLServiceConfig.java index e5c9de3d2a..4287883c34 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/config/SQLServiceConfig.java +++ b/sql/src/main/java/org/opensearch/sql/sql/config/SQLServiceConfig.java @@ -6,42 +6,27 @@ package org.opensearch.sql.sql.config; -import org.opensearch.sql.analysis.Analyzer; -import org.opensearch.sql.analysis.ExpressionAnalyzer; -import org.opensearch.sql.catalog.CatalogService; -import org.opensearch.sql.executor.ExecutionEngine; -import org.opensearch.sql.expression.config.ExpressionConfig; -import org.opensearch.sql.expression.function.BuiltinFunctionRepository; +import org.opensearch.sql.executor.QueryManager; +import org.opensearch.sql.executor.execution.QueryPlanFactory; import org.opensearch.sql.sql.SQLService; import org.opensearch.sql.sql.antlr.SQLSyntaxParser; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Scope; /** * SQL service configuration for Spring container initialization. */ @Configuration -@Import({ExpressionConfig.class}) public class SQLServiceConfig { @Autowired - private ExecutionEngine executionEngine; + private QueryManager queryManager; @Autowired - private CatalogService catalogService; - - @Autowired - private BuiltinFunctionRepository functionRepository; - - @Bean - public Analyzer analyzer() { - return new Analyzer(new ExpressionAnalyzer(functionRepository), catalogService, - functionRepository); - } + private QueryPlanFactory queryExecutionFactory; /** * The registration of OpenSearch storage engine happens here because @@ -54,10 +39,8 @@ public Analyzer analyzer() { public SQLService sqlService() { return new SQLService( new SQLSyntaxParser(), - new Analyzer(new ExpressionAnalyzer(functionRepository), - catalogService, functionRepository), - executionEngine, - functionRepository); + queryManager, + queryExecutionFactory); } } diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstStatementBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstStatementBuilder.java new file mode 100644 index 0000000000..40d549764a --- /dev/null +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstStatementBuilder.java @@ -0,0 +1,43 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.sql.parser; + +import lombok.Builder; +import lombok.Data; +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.Query; +import org.opensearch.sql.ast.statement.Statement; +import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser; +import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParserBaseVisitor; + +@RequiredArgsConstructor +public class AstStatementBuilder extends OpenSearchSQLParserBaseVisitor { + + private final AstBuilder astBuilder; + + private final StatementBuilderContext context; + + @Override + public Statement visitSqlStatement(OpenSearchSQLParser.SqlStatementContext ctx) { + Query query = new Query(astBuilder.visit(ctx)); + return context.isExplain ? new Explain(query) : query; + } + + @Override + protected Statement aggregateResult(Statement aggregate, Statement nextResult) { + return nextResult != null ? nextResult : aggregate; + } + + @Data + @Builder + public static class StatementBuilderContext { + private final boolean isExplain; + } +} diff --git a/sql/src/test/java/org/opensearch/sql/sql/SQLServiceTest.java b/sql/src/test/java/org/opensearch/sql/sql/SQLServiceTest.java index 774c5e2d52..f1d2c5293d 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/SQLServiceTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/SQLServiceTest.java @@ -10,8 +10,6 @@ import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; import static org.opensearch.sql.executor.ExecutionEngine.QueryResponse; import java.util.Collections; @@ -21,41 +19,39 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.catalog.CatalogService; import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.executor.DefaultQueryManager; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponse; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponseNode; -import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.executor.QueryManager; +import org.opensearch.sql.executor.QueryService; +import org.opensearch.sql.executor.execution.QueryPlanFactory; import org.opensearch.sql.sql.config.SQLServiceConfig; import org.opensearch.sql.sql.domain.SQLQueryRequest; -import org.opensearch.sql.storage.StorageEngine; import org.springframework.context.annotation.AnnotationConfigApplicationContext; @ExtendWith(MockitoExtension.class) class SQLServiceTest { - private AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); + private static String QUERY = "/_plugins/_sql"; - private SQLService sqlService; + private static String EXPLAIN = "/_plugins/_sql/_explain"; - @Mock - private StorageEngine storageEngine; + private AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - @Mock - private ExecutionEngine executionEngine; + private SQLService sqlService; @Mock - private CatalogService catalogService; + private QueryService queryService; @Mock private ExecutionEngine.Schema schema; @BeforeEach public void setUp() { - context.registerBean(StorageEngine.class, () -> storageEngine); - context.registerBean(ExecutionEngine.class, () -> executionEngine); - context.registerBean(CatalogService.class, () -> catalogService); + context.registerBean(QueryManager.class, DefaultQueryManager::new); + context.registerBean(QueryPlanFactory.class, () -> new QueryPlanFactory(queryService)); context.register(SQLServiceConfig.class); context.refresh(); sqlService = context.getBean(SQLService.class); @@ -67,10 +63,10 @@ public void canExecuteSqlQuery() { ResponseListener listener = invocation.getArgument(1); listener.onResponse(new QueryResponse(schema, Collections.emptyList())); return null; - }).when(executionEngine).execute(any(), any()); + }).when(queryService).execute(any(), any()); sqlService.execute( - new SQLQueryRequest(new JSONObject(), "SELECT 123", "_plugins/_sql", "jdbc"), + new SQLQueryRequest(new JSONObject(), "SELECT 123", QUERY, "jdbc"), new ResponseListener() { @Override public void onResponse(QueryResponse response) { @@ -90,10 +86,10 @@ public void canExecuteCsvFormatRequest() { ResponseListener listener = invocation.getArgument(1); listener.onResponse(new QueryResponse(schema, Collections.emptyList())); return null; - }).when(executionEngine).execute(any(), any()); + }).when(queryService).execute(any(), any()); sqlService.execute( - new SQLQueryRequest(new JSONObject(), "SELECT 123", "_plugins/_sql", "csv"), + new SQLQueryRequest(new JSONObject(), "SELECT 123", QUERY, "csv"), new ResponseListener() { @Override public void onResponse(QueryResponse response) { @@ -113,9 +109,9 @@ public void canExplainSqlQuery() { ResponseListener listener = invocation.getArgument(1); listener.onResponse(new ExplainResponse(new ExplainResponseNode("Test"))); return null; - }).when(executionEngine).explain(any(), any()); + }).when(queryService).explain(any(), any()); - sqlService.explain(mock(PhysicalPlan.class), + sqlService.explain(new SQLQueryRequest(new JSONObject(), "SELECT 123", EXPLAIN, "csv"), new ResponseListener() { @Override public void onResponse(ExplainResponse response) { @@ -129,50 +125,10 @@ public void onFailure(Exception e) { }); } - @Test - public void canExecuteFromPhysicalPlan() { - doAnswer(invocation -> { - ResponseListener listener = invocation.getArgument(1); - listener.onResponse(new QueryResponse(schema, Collections.emptyList())); - return null; - }).when(executionEngine).execute(any(), any()); - - sqlService.execute(mock(PhysicalPlan.class), - new ResponseListener() { - @Override - public void onResponse(QueryResponse response) { - assertNotNull(response); - } - - @Override - public void onFailure(Exception e) { - fail(e); - } - }); - } - @Test public void canCaptureErrorDuringExecution() { sqlService.execute( - new SQLQueryRequest(new JSONObject(), "SELECT", "_plugins/_sql", ""), - new ResponseListener() { - @Override - public void onResponse(QueryResponse response) { - fail(); - } - - @Override - public void onFailure(Exception e) { - assertNotNull(e); - } - }); - } - - @Test - public void canCaptureErrorDuringExecutionFromPhysicalPlan() { - doThrow(new RuntimeException()).when(executionEngine).execute(any(), any()); - - sqlService.execute(mock(PhysicalPlan.class), + new SQLQueryRequest(new JSONObject(), "SELECT", QUERY, ""), new ResponseListener() { @Override public void onResponse(QueryResponse response) { @@ -188,9 +144,8 @@ public void onFailure(Exception e) { @Test public void canCaptureErrorDuringExplain() { - doThrow(new RuntimeException()).when(executionEngine).explain(any(), any()); - - sqlService.explain(mock(PhysicalPlan.class), + sqlService.explain( + new SQLQueryRequest(new JSONObject(), "SELECT", EXPLAIN, ""), new ResponseListener() { @Override public void onResponse(ExplainResponse response) { From 7b6a1e266f1c987a09ef0ef2b282e5881afa0953 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Tue, 1 Nov 2022 09:17:43 -0700 Subject: [PATCH 08/23] Add Streaming Source Impl Signed-off-by: Peng Huo --- .../sql/executor/streaming/Batch.java | 18 ++ .../sql/executor/streaming/Offset.java | 17 ++ .../executor/streaming/StreamingSource.java | 29 ++++ .../opensearch/sql/storage/split/Split.java | 21 +++ filesystem/build.gradle | 69 ++++++++ .../storage/split/FileSystemSplit.java | 24 +++ .../filesystem/streaming/FileMetaData.java | 21 +++ .../streaming/FileSystemStreamSource.java | 103 ++++++++++++ .../streaming/FileSystemStreamSourceTest.java | 155 ++++++++++++++++++ settings.gradle | 1 + 10 files changed, 458 insertions(+) create mode 100644 core/src/main/java/org/opensearch/sql/executor/streaming/Batch.java create mode 100644 core/src/main/java/org/opensearch/sql/executor/streaming/Offset.java create mode 100644 core/src/main/java/org/opensearch/sql/executor/streaming/StreamingSource.java create mode 100644 core/src/main/java/org/opensearch/sql/storage/split/Split.java create mode 100644 filesystem/build.gradle create mode 100644 filesystem/src/main/java/org/opensearch/sql/filesystem/storage/split/FileSystemSplit.java create mode 100644 filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileMetaData.java create mode 100644 filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSource.java create mode 100644 filesystem/src/test/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSourceTest.java diff --git a/core/src/main/java/org/opensearch/sql/executor/streaming/Batch.java b/core/src/main/java/org/opensearch/sql/executor/streaming/Batch.java new file mode 100644 index 0000000000..7c27ab4622 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/streaming/Batch.java @@ -0,0 +1,18 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.executor.streaming; + +import java.util.List; +import lombok.Data; +import org.opensearch.sql.storage.split.Split; + +/** + * A batch of streaming execution. + */ +@Data +public class Batch { + private final List splits; +} diff --git a/core/src/main/java/org/opensearch/sql/executor/streaming/Offset.java b/core/src/main/java/org/opensearch/sql/executor/streaming/Offset.java new file mode 100644 index 0000000000..00f040e437 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/streaming/Offset.java @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.executor.streaming; + +import lombok.Data; + +/** + * Offset. + */ +@Data +public class Offset { + + private final Long offset; +} diff --git a/core/src/main/java/org/opensearch/sql/executor/streaming/StreamingSource.java b/core/src/main/java/org/opensearch/sql/executor/streaming/StreamingSource.java new file mode 100644 index 0000000000..ebd3fa714b --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/streaming/StreamingSource.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.executor.streaming; + +import java.util.Optional; + +/** + * Streaming source. + */ +public interface StreamingSource { + /** + * Get current {@link Offset} of stream data. + * + * @return empty if the stream does not has new data. + */ + Optional getLatestOffset(); + + /** + * Get a {@link Batch} from source between (start, end]. + * + * @param start start offset. + * @param end end offset. + * @return @link Batch}. + */ + Batch getBatch(Optional start, Offset end); +} diff --git a/core/src/main/java/org/opensearch/sql/storage/split/Split.java b/core/src/main/java/org/opensearch/sql/storage/split/Split.java new file mode 100644 index 0000000000..e9e0c6fcc1 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/storage/split/Split.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.storage.split; + +import org.opensearch.sql.storage.StorageEngine; + +/** + * Split is a sections of a data set. Each {@link StorageEngine} should have specific + * implementation of Split. + */ +public interface Split { + + /** + * Get the split id. + * @return split id. + */ + String getSplitId(); +} diff --git a/filesystem/build.gradle b/filesystem/build.gradle new file mode 100644 index 0000000000..64659d85d3 --- /dev/null +++ b/filesystem/build.gradle @@ -0,0 +1,69 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +plugins { + id 'java-library' + id "io.freefair.lombok" + id 'jacoco' +} + +ext { + hadoop = "3.3.4" + aws = "1.12.330" +} + + +dependencies { + implementation project(':core') + + testImplementation "org.junit.jupiter:junit-jupiter:${junit_jupiter}" + testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' + testImplementation group: 'org.mockito', name: 'mockito-core', version: '3.12.4' + testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.12.4' +} + +test { + useJUnitPlatform() + testLogging { + events "passed", "skipped", "failed" + exceptionFormat "full" + } +} + +jacocoTestReport { + reports { + html.enabled true + xml.enabled true + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it) + })) + } +} +test.finalizedBy(project.tasks.jacocoTestReport) + +jacocoTestCoverageVerification { + violationRules { + rule { + element = 'CLASS' + limit { + counter = 'LINE' + minimum = 1.0 + } + limit { + counter = 'BRANCH' + minimum = 1.0 + } + } + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it) + })) + } +} +check.dependsOn jacocoTestCoverageVerification +jacocoTestCoverageVerification.dependsOn jacocoTestReport diff --git a/filesystem/src/main/java/org/opensearch/sql/filesystem/storage/split/FileSystemSplit.java b/filesystem/src/main/java/org/opensearch/sql/filesystem/storage/split/FileSystemSplit.java new file mode 100644 index 0000000000..695af94fe4 --- /dev/null +++ b/filesystem/src/main/java/org/opensearch/sql/filesystem/storage/split/FileSystemSplit.java @@ -0,0 +1,24 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.filesystem.storage.split; + +import java.nio.file.Path; +import java.util.Set; +import java.util.UUID; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.opensearch.sql.storage.split.Split; + +@Data +public class FileSystemSplit implements Split { + + @Getter + @EqualsAndHashCode.Exclude + private final String splitId = UUID.randomUUID().toString(); + + private final Set paths; +} diff --git a/filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileMetaData.java b/filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileMetaData.java new file mode 100644 index 0000000000..24d2a822cd --- /dev/null +++ b/filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileMetaData.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.filesystem.streaming; + +import java.nio.file.Path; +import java.util.Set; +import lombok.Data; + +/** + * File metadata. Batch id associate with the set of {@link Path}. + */ +@Data +public class FileMetaData { + + private final Long batchId; + + private final Set paths; +} diff --git a/filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSource.java b/filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSource.java new file mode 100644 index 0000000000..9207583c5b --- /dev/null +++ b/filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSource.java @@ -0,0 +1,103 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.filesystem.streaming; + +import com.google.common.collect.Sets; +import java.io.File; +import java.nio.file.FileSystem; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.sql.executor.streaming.Batch; +import org.opensearch.sql.executor.streaming.DefaultMetadataLog; +import org.opensearch.sql.executor.streaming.MetadataLog; +import org.opensearch.sql.executor.streaming.Offset; +import org.opensearch.sql.executor.streaming.StreamingSource; +import org.opensearch.sql.filesystem.storage.split.FileSystemSplit; + +/** + * FileSystem Streaming Source use Hadoop FileSystem. + */ +public class FileSystemStreamSource implements StreamingSource { + + private static final Logger log = LogManager.getLogger(FileSystemStreamSource.class); + + private final MetadataLog fileMetaDataLog; + + private Set seenFiles; + + private final FileSystem fs; + + private final String basePath; + + /** + * Constructor of FileSystemStreamSource. + */ + public FileSystemStreamSource(FileSystem fs, String basePath) { + this.fs = fs; + this.basePath = basePath; + // todo, need to add state recovery + this.fileMetaDataLog = new DefaultMetadataLog<>(); + // todo, need to add state recovery + this.seenFiles = new HashSet<>(); + } + + @Override + public Optional getLatestOffset() { + // list all files. todo. improvement list performance. + Set allFiles = + Arrays.stream(fs.getPath(basePath).toFile().listFiles()) + .filter(file -> !file.isDirectory()) + .map(File::toPath) + .collect(Collectors.toSet()); + + // find unread files. + log.debug("all files {}", allFiles); + Set unread = Sets.difference(allFiles, seenFiles); + + // update seenFiles. + seenFiles = allFiles; + log.debug("seen files {}", seenFiles); + + Optional latestBatchIdOptional = fileMetaDataLog.getLatest().map(Pair::getKey); + if (!unread.isEmpty()) { + long latestBatchId = latestBatchIdOptional.map(id -> id + 1).orElse(0L); + fileMetaDataLog.add(latestBatchId, new FileMetaData(latestBatchId, unread)); + log.debug("latestBatchId {}", latestBatchId); + return Optional.of(new Offset(latestBatchId)); + } else { + log.debug("no unread data"); + Optional offset = + latestBatchIdOptional.isEmpty() + ? Optional.empty() + : Optional.of(new Offset(latestBatchIdOptional.get())); + log.debug("return empty offset {}", offset); + return offset; + } + } + + @Override + public Batch getBatch(Optional start, Offset end) { + Long startBatchId = start.map(Offset::getOffset).map(id -> id + 1).orElse(0L); + Long endBatchId = end.getOffset(); + + Set paths = + fileMetaDataLog.get(Optional.of(startBatchId), Optional.of(endBatchId)).stream() + .map(FileMetaData::getPaths) + .flatMap(Set::stream) + .collect(Collectors.toSet()); + + log.debug("fetch files {} with id from: {} to: {}.", paths, start, end); + return new Batch(Collections.singletonList(new FileSystemSplit(paths))); + } +} diff --git a/filesystem/src/test/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSourceTest.java b/filesystem/src/test/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSourceTest.java new file mode 100644 index 0000000000..537fd10c9f --- /dev/null +++ b/filesystem/src/test/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSourceTest.java @@ -0,0 +1,155 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.filesystem.streaming; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import java.io.IOException; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.executor.streaming.Batch; +import org.opensearch.sql.executor.streaming.Offset; +import org.opensearch.sql.filesystem.storage.split.FileSystemSplit; + +@ExtendWith(MockitoExtension.class) +class FileSystemStreamSourceTest { + + @TempDir + Path perTestTempDir; + + FileSystemStreamSource streamSource; + + @BeforeEach + void setup() { + streamSource = + new FileSystemStreamSource( + FileSystems.getDefault(), + perTestTempDir.toString()); + } + + @Test + void getBatchFromFolder() throws IOException { + Path file = Files.createFile(perTestTempDir.resolve("log.2022.01.01")); + assertTrue(file.toFile().exists()); + + Optional latestOffset = streamSource.getLatestOffset(); + assertTrue(latestOffset.isPresent()); + assertEquals(new Offset(0L), latestOffset.get()); + + // fetch batch (empty, latestOffset] + assertEquals( + Collections.singletonList( + new FileSystemSplit(ImmutableSet.of(file))), + streamSource.getBatch(Optional.empty(), latestOffset.get()).getSplits()); + } + + @Test + void latestOffsetShouldIncreaseIfNoNewFileAdded() throws IOException { + Path file1 = Files.createFile(perTestTempDir.resolve("log.2022.01.01")); + assertTrue(file1.toFile().exists()); + + Optional latestOffset = streamSource.getLatestOffset(); + assertTrue(latestOffset.isPresent()); + assertEquals(new Offset(0L), latestOffset.get()); + + Path file2 = Files.createFile(perTestTempDir.resolve("log.2022.01.02")); + assertTrue(file2.toFile().exists()); + + latestOffset = streamSource.getLatestOffset(); + assertTrue(latestOffset.isPresent()); + assertEquals(new Offset(1L), latestOffset.get()); + + // fetch batch (empty, 1L] + assertBatchEquals( + ImmutableList.of(file1, file2), + streamSource.getBatch(Optional.empty(), latestOffset.get())); + + // fetch batch (empty, 0L] + assertBatchEquals( + ImmutableList.of(file1), streamSource.getBatch(Optional.empty(), new Offset(0L))); + + // fetch batch (0L, 1L] + assertBatchEquals( + ImmutableList.of(file2), + streamSource.getBatch(Optional.of(new Offset(0L)), new Offset(1L))); + } + + @Test + void latestOffsetShouldSameIfNoNewFileAdded() throws IOException { + Path file1 = Files.createFile(perTestTempDir.resolve("log.2022.01.01")); + assertTrue(file1.toFile().exists()); + + Optional latestOffset = streamSource.getLatestOffset(); + assertTrue(latestOffset.isPresent()); + assertEquals(new Offset(0L), latestOffset.get()); + + // no new files. + latestOffset = streamSource.getLatestOffset(); + assertTrue(latestOffset.isPresent()); + assertEquals(new Offset(0L), latestOffset.get()); + } + + @Test + void latestOffsetIsEmptyIfNoFilesInSource() { + Optional latestOffset = streamSource.getLatestOffset(); + assertTrue(latestOffset.isEmpty()); + } + + @Test + void getBatchOutOfRange() throws IOException { + Path file = Files.createFile(perTestTempDir.resolve("log.2022.01.01")); + assertTrue(file.toFile().exists()); + + Optional latestOffset = streamSource.getLatestOffset(); + assertTrue(latestOffset.isPresent()); + assertEquals(new Offset(0L), latestOffset.get()); + + assertEquals( + Collections.singletonList( + new FileSystemSplit(ImmutableSet.of(file))), + streamSource.getBatch(Optional.empty(), latestOffset.get()).getSplits()); + } + + @Test + void dirIsFiltered() throws IOException { + Path file = Files.createFile(perTestTempDir.resolve("log.2022.01.01")); + assertTrue(file.toFile().exists()); + + Path dir = Files.createDirectory(perTestTempDir.resolve("logDir")); + assertTrue(dir.toFile().isDirectory()); + + Optional latestOffset = streamSource.getLatestOffset(); + assertTrue(latestOffset.isPresent()); + assertEquals(new Offset(0L), latestOffset.get()); + + // fetch batch (empty, latestOffset] + assertEquals( + Collections.singletonList( + new FileSystemSplit(ImmutableSet.of(file))), + streamSource.getBatch(Optional.empty(), latestOffset.get()).getSplits()); + } + + void assertBatchEquals(List expectedFiles, Batch batch) { + assertEquals(1, batch.getSplits().size()); + assertThat( + ((FileSystemSplit) batch.getSplits().get(0)).getPaths(), + containsInAnyOrder(expectedFiles.toArray())); + } +} diff --git a/settings.gradle b/settings.gradle index 2f850f422b..7650959451 100644 --- a/settings.gradle +++ b/settings.gradle @@ -18,4 +18,5 @@ include 'doctest' include 'legacy' include 'sql' include 'prometheus' +include 'filesystem' From 3231dfe345adf8023005d2a7079532b403d8c830 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Tue, 1 Nov 2022 09:30:05 -0700 Subject: [PATCH 09/23] update build.gradle Signed-off-by: Peng Huo --- filesystem/build.gradle | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/filesystem/build.gradle b/filesystem/build.gradle index 64659d85d3..d7fc3e02a8 100644 --- a/filesystem/build.gradle +++ b/filesystem/build.gradle @@ -9,16 +9,10 @@ plugins { id 'jacoco' } -ext { - hadoop = "3.3.4" - aws = "1.12.330" -} - - dependencies { implementation project(':core') - testImplementation "org.junit.jupiter:junit-jupiter:${junit_jupiter}" + testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' testImplementation group: 'org.mockito', name: 'mockito-core', version: '3.12.4' testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.12.4' From e8f3205dafe737a5e3a172bec4a090080b2e58d2 Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Tue, 1 Nov 2022 16:00:04 -0700 Subject: [PATCH 10/23] Add watermark generator (#959) * Add watermark generator Signed-off-by: Chen Dai * Update Javadoc and UT for readability Signed-off-by: Chen Dai Signed-off-by: Chen Dai --- .../BoundedOutOfOrderWatermarkGenerator.java | 27 ++++++++ .../watermark/WatermarkGenerator.java | 22 +++++++ ...undedOutOfOrderWatermarkGeneratorTest.java | 61 +++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 core/src/main/java/org/opensearch/sql/planner/streaming/watermark/BoundedOutOfOrderWatermarkGenerator.java create mode 100644 core/src/main/java/org/opensearch/sql/planner/streaming/watermark/WatermarkGenerator.java create mode 100644 core/src/test/java/org/opensearch/sql/planner/streaming/watermark/BoundedOutOfOrderWatermarkGeneratorTest.java diff --git a/core/src/main/java/org/opensearch/sql/planner/streaming/watermark/BoundedOutOfOrderWatermarkGenerator.java b/core/src/main/java/org/opensearch/sql/planner/streaming/watermark/BoundedOutOfOrderWatermarkGenerator.java new file mode 100644 index 0000000000..63d6a5b163 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/streaming/watermark/BoundedOutOfOrderWatermarkGenerator.java @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.streaming.watermark; + +import lombok.RequiredArgsConstructor; + +/** + * Watermark generator that generates watermark with bounded out-of-order delay. + */ +@RequiredArgsConstructor +public class BoundedOutOfOrderWatermarkGenerator implements WatermarkGenerator { + + /** The maximum out-of-order allowed in millisecond. */ + private final long maxOutOfOrderAllowed; + + /** The maximum timestamp seen so far in millisecond. */ + private long maxTimestamp; + + @Override + public long generate(long timestamp) { + maxTimestamp = Math.max(maxTimestamp, timestamp); + return (maxTimestamp - maxOutOfOrderAllowed - 1); + } +} diff --git a/core/src/main/java/org/opensearch/sql/planner/streaming/watermark/WatermarkGenerator.java b/core/src/main/java/org/opensearch/sql/planner/streaming/watermark/WatermarkGenerator.java new file mode 100644 index 0000000000..4f4c9a8a00 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/streaming/watermark/WatermarkGenerator.java @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.streaming.watermark; + +/** + * A watermark generator generates watermark timestamp based on some strategy which is defined + * in implementation class. + */ +public interface WatermarkGenerator { + + /** + * Generate watermark timestamp on the given event timestamp. + * + * @param timestamp event timestamp in millisecond + * @return watermark timestamp in millisecond + */ + long generate(long timestamp); + +} diff --git a/core/src/test/java/org/opensearch/sql/planner/streaming/watermark/BoundedOutOfOrderWatermarkGeneratorTest.java b/core/src/test/java/org/opensearch/sql/planner/streaming/watermark/BoundedOutOfOrderWatermarkGeneratorTest.java new file mode 100644 index 0000000000..1d18a16f2a --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/planner/streaming/watermark/BoundedOutOfOrderWatermarkGeneratorTest.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.streaming.watermark; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class BoundedOutOfOrderWatermarkGeneratorTest { + + @Test + void shouldAdvanceWatermarkIfNewerEvent() { + assertWatermarkGenerator() + .thatAllowMaxDelay(100) + .afterSeenEventTime(1000) + .shouldGenerateWatermark(899) + .afterSeenEventTime(2000) + .shouldGenerateWatermark(1899); + } + + @Test + void shouldNotAdvanceWatermarkIfLateEvent() { + assertWatermarkGenerator() + .thatAllowMaxDelay(100) + .afterSeenEventTime(1000) + .shouldGenerateWatermark(899) + .afterSeenEventTime(500) + .shouldGenerateWatermark(899) + .afterSeenEventTime(999) + .shouldGenerateWatermark(899); + } + + private static AssertionHelper assertWatermarkGenerator() { + return new AssertionHelper(); + } + + private static class AssertionHelper { + + private WatermarkGenerator generator; + + private long actualResult; + + public AssertionHelper thatAllowMaxDelay(long delay) { + this.generator = new BoundedOutOfOrderWatermarkGenerator(delay); + return this; + } + + public AssertionHelper afterSeenEventTime(long timestamp) { + this.actualResult = generator.generate(timestamp); + return this; + } + + public AssertionHelper shouldGenerateWatermark(long expected) { + assertEquals(expected, actualResult); + return this; + } + } +} \ No newline at end of file From 73787b700f73bd686a96600a9ab688680485dc6a Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Tue, 1 Nov 2022 16:01:32 -0700 Subject: [PATCH 11/23] Add stream context and window trigger (#958) * Add stream context and window trigger Signed-off-by: Chen Dai * Update Javadoc with more details Signed-off-by: Chen Dai Signed-off-by: Chen Dai --- .../sql/planner/streaming/StreamContext.java | 19 +++++++++++ .../trigger/AfterWatermarkWindowTrigger.java | 30 ++++++++++++++++ .../windowing/trigger/TriggerResult.java | 29 ++++++++++++++++ .../windowing/trigger/WindowTrigger.java | 24 +++++++++++++ .../AfterWatermarkWindowTriggerTest.java | 34 +++++++++++++++++++ 5 files changed, 136 insertions(+) create mode 100644 core/src/main/java/org/opensearch/sql/planner/streaming/StreamContext.java create mode 100644 core/src/main/java/org/opensearch/sql/planner/streaming/windowing/trigger/AfterWatermarkWindowTrigger.java create mode 100644 core/src/main/java/org/opensearch/sql/planner/streaming/windowing/trigger/TriggerResult.java create mode 100644 core/src/main/java/org/opensearch/sql/planner/streaming/windowing/trigger/WindowTrigger.java create mode 100644 core/src/test/java/org/opensearch/sql/planner/streaming/windowing/trigger/AfterWatermarkWindowTriggerTest.java diff --git a/core/src/main/java/org/opensearch/sql/planner/streaming/StreamContext.java b/core/src/main/java/org/opensearch/sql/planner/streaming/StreamContext.java new file mode 100644 index 0000000000..18eb10f19d --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/streaming/StreamContext.java @@ -0,0 +1,19 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.streaming; + +import lombok.Data; + +/** + * Stream context required by stream processing components and can be + * stored and restored between executions. + */ +@Data +public class StreamContext { + + /** Current watermark timestamp. */ + private long watermark; +} diff --git a/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/trigger/AfterWatermarkWindowTrigger.java b/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/trigger/AfterWatermarkWindowTrigger.java new file mode 100644 index 0000000000..1801880961 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/trigger/AfterWatermarkWindowTrigger.java @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.streaming.windowing.trigger; + +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.planner.streaming.StreamContext; +import org.opensearch.sql.planner.streaming.windowing.Window; + +/** + * After watermark window trigger fires window state output once a window is below watermark. + * Precisely speaking, after watermark means the window boundary (max timestamp) is equal to + * or less than the current watermark timestamp. + */ +@RequiredArgsConstructor +public class AfterWatermarkWindowTrigger implements WindowTrigger { + + /** Stream context that contains the current watermark. */ + private final StreamContext context; + + @Override + public TriggerResult trigger(Window window) { + if (window.maxTimestamp() <= context.getWatermark()) { + return TriggerResult.FIRE; + } + return TriggerResult.CONTINUE; + } +} diff --git a/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/trigger/TriggerResult.java b/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/trigger/TriggerResult.java new file mode 100644 index 0000000000..465f0aa9eb --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/trigger/TriggerResult.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.streaming.windowing.trigger; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +/** + * Result determined by a trigger for what should happen to the window. + */ +@Getter +@RequiredArgsConstructor +public enum TriggerResult { + + /** Continue without any operation. */ + CONTINUE(false, false), + + /** Fire and purge window state by default. */ + FIRE(true, true); + + /** If window should be fired to output. */ + private final boolean fire; + + /** If the window state should be discarded. */ + private final boolean purge; +} diff --git a/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/trigger/WindowTrigger.java b/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/trigger/WindowTrigger.java new file mode 100644 index 0000000000..f6c2eba50f --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/trigger/WindowTrigger.java @@ -0,0 +1,24 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.streaming.windowing.trigger; + +import org.opensearch.sql.planner.streaming.windowing.Window; + +/** + * A window trigger determines if the current window state should be evaluated to emit output. + * Typically, trigger strategy works with downstream Sink operator together to meet the semantic + * requirements. For example, per-event trigger can work with Sink for materialized view semantic. + */ +public interface WindowTrigger { + + /** + * Return trigger result for a window. + * + * @param window given window + * @return trigger result + */ + TriggerResult trigger(Window window); +} diff --git a/core/src/test/java/org/opensearch/sql/planner/streaming/windowing/trigger/AfterWatermarkWindowTriggerTest.java b/core/src/test/java/org/opensearch/sql/planner/streaming/windowing/trigger/AfterWatermarkWindowTriggerTest.java new file mode 100644 index 0000000000..3ef6907c38 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/planner/streaming/windowing/trigger/AfterWatermarkWindowTriggerTest.java @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.streaming.windowing.trigger; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; +import org.opensearch.sql.planner.streaming.StreamContext; +import org.opensearch.sql.planner.streaming.windowing.Window; + +class AfterWatermarkWindowTriggerTest { + + private final StreamContext context = new StreamContext(); + + private final AfterWatermarkWindowTrigger trigger = new AfterWatermarkWindowTrigger(context); + + @Test + void shouldNotFireWindowAboveWatermark() { + context.setWatermark(999); + assertEquals(TriggerResult.CONTINUE, trigger.trigger(new Window(500, 1500))); + assertEquals(TriggerResult.CONTINUE, trigger.trigger(new Window(500, 1001))); + assertEquals(TriggerResult.CONTINUE, trigger.trigger(new Window(1000, 1500))); + } + + @Test + void shouldFireWindowBelowWatermark() { + context.setWatermark(999); + assertEquals(TriggerResult.FIRE, trigger.trigger(new Window(500, 800))); + assertEquals(TriggerResult.FIRE, trigger.trigger(new Window(500, 1000))); + } +} \ No newline at end of file From 752168ff68092d0d37a78bc70db22142031dceb4 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Thu, 3 Nov 2022 10:24:36 -0700 Subject: [PATCH 12/23] change to hadoop-fs Signed-off-by: Peng Huo --- .../workflows/sql-test-and-build-workflow.yml | 4 +- doctest/build.gradle | 2 +- filesystem/build.gradle | 29 +++ .../storage/split/FileSystemSplit.java | 2 +- .../filesystem/streaming/FileMetaData.java | 2 +- .../streaming/FileSystemStreamSource.java | 19 +- .../streaming/FileSystemStreamSourceTest.java | 203 ++++++++++-------- 7 files changed, 153 insertions(+), 108 deletions(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 3d063a2bfc..25e0387cf3 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -25,10 +25,10 @@ jobs: matrix: entry: - { os: ubuntu-latest, java: 11 } - - { os: windows-latest, java: 11, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc} + - { os: windows-latest, java: 11, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc -PbuildPlatform=windows } - { os: macos-latest, java: 11, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc } - { os: ubuntu-latest, java: 17 } - - { os: windows-latest, java: 17, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc } + - { os: windows-latest, java: 17, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc -PbuildPlatform=windows } - { os: macos-latest, java: 17, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc } runs-on: ${{ matrix.entry.os }} diff --git a/doctest/build.gradle b/doctest/build.gradle index 69fac44d95..cf2329d9d3 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -86,7 +86,7 @@ task stopOpenSearch(type: KillProcessTask) { doctest.dependsOn startOpenSearch startOpenSearch.dependsOn startPrometheus doctest.finalizedBy stopOpenSearch -build.dependsOn doctest +check.dependsOn doctest clean.dependsOn(cleanBootstrap) // 2.0.0-alpha1-SNAPSHOT -> 2.0.0.0-alpha1-SNAPSHOT diff --git a/filesystem/build.gradle b/filesystem/build.gradle index d7fc3e02a8..b4be876507 100644 --- a/filesystem/build.gradle +++ b/filesystem/build.gradle @@ -9,8 +9,29 @@ plugins { id 'jacoco' } +ext { + hadoop = "3.3.4" + aws = "1.12.330" +} + dependencies { implementation project(':core') + // required by hadoop filesystem https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/filesystem/index.html. + implementation("org.apache.hadoop:hadoop-common:${hadoop}") { + exclude group: 'org.apache.zookeeper', module: 'zookeeper' + exclude group: 'com.sun.jersey', module: 'jersey-json' + exclude group: 'com.google.protobuf', module: 'protobuf-java' + exclude group: 'org.apache.avro', module: 'avro' + exclude group: 'org.eclipse.jetty', module: 'jetty-server' + } + constraints { + implementation('com.fasterxml.woodstox:woodstox-core:6.4.0') { + because 'https://www.mend.io/vulnerability-database/CVE-2022-40156' + } + } + // required https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/index.html + implementation("org.apache.hadoop:hadoop-aws:${hadoop}") + implementation "com.amazonaws:aws-java-sdk-bundle:${aws}" testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' @@ -24,6 +45,14 @@ test { events "passed", "skipped", "failed" exceptionFormat "full" } + + // hadoop-fs depend on native library which is missing on windows. + // https://hadoop.apache.org/docs/r3.3.4/hadoop-project-dist/hadoop-common/NativeLibraries.html#Native_Hadoop_Library + if ('windows' == project.getProperties().getOrDefault('buildPlatform', 'linux')) { + excludes = [ + '**/FileSystemStreamSourceTest.class' + ] + } } jacocoTestReport { diff --git a/filesystem/src/main/java/org/opensearch/sql/filesystem/storage/split/FileSystemSplit.java b/filesystem/src/main/java/org/opensearch/sql/filesystem/storage/split/FileSystemSplit.java index 695af94fe4..7fefb11a85 100644 --- a/filesystem/src/main/java/org/opensearch/sql/filesystem/storage/split/FileSystemSplit.java +++ b/filesystem/src/main/java/org/opensearch/sql/filesystem/storage/split/FileSystemSplit.java @@ -5,12 +5,12 @@ package org.opensearch.sql.filesystem.storage.split; -import java.nio.file.Path; import java.util.Set; import java.util.UUID; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.Getter; +import org.apache.hadoop.fs.Path; import org.opensearch.sql.storage.split.Split; @Data diff --git a/filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileMetaData.java b/filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileMetaData.java index 24d2a822cd..6a8c90ee80 100644 --- a/filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileMetaData.java +++ b/filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileMetaData.java @@ -5,9 +5,9 @@ package org.opensearch.sql.filesystem.streaming; -import java.nio.file.Path; import java.util.Set; import lombok.Data; +import org.apache.hadoop.fs.Path; /** * File metadata. Batch id associate with the set of {@link Path}. diff --git a/filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSource.java b/filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSource.java index 9207583c5b..0a1d032c53 100644 --- a/filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSource.java +++ b/filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSource.java @@ -6,16 +6,18 @@ package org.opensearch.sql.filesystem.streaming; import com.google.common.collect.Sets; -import java.io.File; -import java.nio.file.FileSystem; -import java.nio.file.Path; +import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import lombok.SneakyThrows; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.sql.executor.streaming.Batch; @@ -38,12 +40,12 @@ public class FileSystemStreamSource implements StreamingSource { private final FileSystem fs; - private final String basePath; + private final Path basePath; /** * Constructor of FileSystemStreamSource. */ - public FileSystemStreamSource(FileSystem fs, String basePath) { + public FileSystemStreamSource(FileSystem fs, Path basePath) { this.fs = fs; this.basePath = basePath; // todo, need to add state recovery @@ -52,13 +54,14 @@ public FileSystemStreamSource(FileSystem fs, String basePath) { this.seenFiles = new HashSet<>(); } + @SneakyThrows(value = IOException.class) @Override public Optional getLatestOffset() { // list all files. todo. improvement list performance. Set allFiles = - Arrays.stream(fs.getPath(basePath).toFile().listFiles()) - .filter(file -> !file.isDirectory()) - .map(File::toPath) + Arrays.stream(fs.listStatus(basePath)) + .filter(status -> !status.isDirectory()) + .map(FileStatus::getPath) .collect(Collectors.toSet()); // find unread files. diff --git a/filesystem/src/test/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSourceTest.java b/filesystem/src/test/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSourceTest.java index 537fd10c9f..75c494ec8c 100644 --- a/filesystem/src/test/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSourceTest.java +++ b/filesystem/src/test/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSourceTest.java @@ -8,148 +8,161 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import java.io.IOException; -import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Collections; +import java.util.Arrays; import java.util.List; import java.util.Optional; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.executor.streaming.Batch; import org.opensearch.sql.executor.streaming.Offset; import org.opensearch.sql.filesystem.storage.split.FileSystemSplit; +import org.opensearch.sql.storage.split.Split; @ExtendWith(MockitoExtension.class) class FileSystemStreamSourceTest { - @TempDir - Path perTestTempDir; + @TempDir Path perTestTempDir; FileSystemStreamSource streamSource; + /** + * use hadoop default filesystem. it only works on unix-like system. for running on windows, it + * require native library. Reference. + * https://hadoop.apache.org/docs/r3.3.4/hadoop-project-dist/hadoop-common/NativeLibraries.html + */ @BeforeEach - void setup() { + void setup() throws IOException { streamSource = new FileSystemStreamSource( - FileSystems.getDefault(), - perTestTempDir.toString()); + FileSystem.get(new Configuration()), + new org.apache.hadoop.fs.Path(perTestTempDir.toUri())); } @Test - void getBatchFromFolder() throws IOException { - Path file = Files.createFile(perTestTempDir.resolve("log.2022.01.01")); - assertTrue(file.toFile().exists()); - - Optional latestOffset = streamSource.getLatestOffset(); - assertTrue(latestOffset.isPresent()); - assertEquals(new Offset(0L), latestOffset.get()); - - // fetch batch (empty, latestOffset] - assertEquals( - Collections.singletonList( - new FileSystemSplit(ImmutableSet.of(file))), - streamSource.getBatch(Optional.empty(), latestOffset.get()).getSplits()); + void addOneFileToSource() throws IOException { + emptySource().addFile("log1").latestOffsetShouldBe(0L).batchFromStart("log1"); } @Test - void latestOffsetShouldIncreaseIfNoNewFileAdded() throws IOException { - Path file1 = Files.createFile(perTestTempDir.resolve("log.2022.01.01")); - assertTrue(file1.toFile().exists()); - - Optional latestOffset = streamSource.getLatestOffset(); - assertTrue(latestOffset.isPresent()); - assertEquals(new Offset(0L), latestOffset.get()); - - Path file2 = Files.createFile(perTestTempDir.resolve("log.2022.01.02")); - assertTrue(file2.toFile().exists()); - - latestOffset = streamSource.getLatestOffset(); - assertTrue(latestOffset.isPresent()); - assertEquals(new Offset(1L), latestOffset.get()); - - // fetch batch (empty, 1L] - assertBatchEquals( - ImmutableList.of(file1, file2), - streamSource.getBatch(Optional.empty(), latestOffset.get())); - - // fetch batch (empty, 0L] - assertBatchEquals( - ImmutableList.of(file1), streamSource.getBatch(Optional.empty(), new Offset(0L))); - - // fetch batch (0L, 1L] - assertBatchEquals( - ImmutableList.of(file2), - streamSource.getBatch(Optional.of(new Offset(0L)), new Offset(1L))); + void addMultipleFileInSequence() throws IOException { + emptySource() + .addFile("log1") + .latestOffsetShouldBe(0L) + .batchFromStart("log1") + .addFile("log2") + .latestOffsetShouldBe(1L) + .batchFromStart("log1", "log2") + .batchInBetween(0L, 1L, "log2"); } @Test void latestOffsetShouldSameIfNoNewFileAdded() throws IOException { - Path file1 = Files.createFile(perTestTempDir.resolve("log.2022.01.01")); - assertTrue(file1.toFile().exists()); - - Optional latestOffset = streamSource.getLatestOffset(); - assertTrue(latestOffset.isPresent()); - assertEquals(new Offset(0L), latestOffset.get()); - - // no new files. - latestOffset = streamSource.getLatestOffset(); - assertTrue(latestOffset.isPresent()); - assertEquals(new Offset(0L), latestOffset.get()); + emptySource() + .addFile("log1") + .latestOffsetShouldBe(0L) + .batchFromStart("log1") + .latestOffsetShouldBe(0L) + .batchFromStart("log1"); } @Test void latestOffsetIsEmptyIfNoFilesInSource() { - Optional latestOffset = streamSource.getLatestOffset(); - assertTrue(latestOffset.isEmpty()); + emptySource().noOffset(); } @Test - void getBatchOutOfRange() throws IOException { - Path file = Files.createFile(perTestTempDir.resolve("log.2022.01.01")); - assertTrue(file.toFile().exists()); - - Optional latestOffset = streamSource.getLatestOffset(); - assertTrue(latestOffset.isPresent()); - assertEquals(new Offset(0L), latestOffset.get()); - - assertEquals( - Collections.singletonList( - new FileSystemSplit(ImmutableSet.of(file))), - streamSource.getBatch(Optional.empty(), latestOffset.get()).getSplits()); + void dirIsFiltered() throws IOException { + emptySource() + .addFile("log1") + .latestOffsetShouldBe(0L) + .addDir("dir1") + .latestOffsetShouldBe(0L) + .batchFromStart("log1"); } @Test - void dirIsFiltered() throws IOException { - Path file = Files.createFile(perTestTempDir.resolve("log.2022.01.01")); - assertTrue(file.toFile().exists()); - - Path dir = Files.createDirectory(perTestTempDir.resolve("logDir")); - assertTrue(dir.toFile().isDirectory()); + void sneakThrowException() throws IOException { + FileSystem fs = Mockito.mock(FileSystem.class); + doThrow(IOException.class).when(fs).listStatus(any(org.apache.hadoop.fs.Path.class)); - Optional latestOffset = streamSource.getLatestOffset(); - assertTrue(latestOffset.isPresent()); - assertEquals(new Offset(0L), latestOffset.get()); + streamSource = + new FileSystemStreamSource(fs, + new org.apache.hadoop.fs.Path(perTestTempDir.toUri())); + assertThrows(IOException.class, () -> streamSource.getLatestOffset()); + } - // fetch batch (empty, latestOffset] - assertEquals( - Collections.singletonList( - new FileSystemSplit(ImmutableSet.of(file))), - streamSource.getBatch(Optional.empty(), latestOffset.get()).getSplits()); + StreamSource emptySource() { + return new StreamSource(); } - void assertBatchEquals(List expectedFiles, Batch batch) { - assertEquals(1, batch.getSplits().size()); - assertThat( - ((FileSystemSplit) batch.getSplits().get(0)).getPaths(), - containsInAnyOrder(expectedFiles.toArray())); + private class StreamSource { + + StreamSource addFile(String filename) throws IOException { + Path file = Files.createFile(perTestTempDir.resolve(filename)); + assertTrue(file.toFile().exists()); + + return this; + } + + StreamSource addDir(String dirname) throws IOException { + Path dir = Files.createDirectory(perTestTempDir.resolve(dirname)); + assertTrue(dir.toFile().isDirectory()); + + return this; + } + + StreamSource noOffset() { + assertFalse(streamSource.getLatestOffset().isPresent()); + + return this; + } + + StreamSource latestOffsetShouldBe(Long offset) { + Optional latestOffset = streamSource.getLatestOffset(); + assertTrue(latestOffset.isPresent()); + assertEquals(new Offset(offset), latestOffset.get()); + + return this; + } + + StreamSource batchFromStart(String... uris) { + assertTrue(streamSource.getLatestOffset().isPresent()); + internalBatchInBetween(Optional.empty(), streamSource.getLatestOffset().get(), uris); + + return this; + } + + StreamSource batchInBetween(Long start, Long end, String... uris) { + internalBatchInBetween(Optional.of(new Offset(start)), new Offset(end), uris); + + return this; + } + + private StreamSource internalBatchInBetween( + Optional start, Offset end, String... uris) { + List splits = streamSource.getBatch(start, end).getSplits(); + assertEquals(1, splits.size()); + assertThat( + ((FileSystemSplit) splits.get(0)).getPaths(), + containsInAnyOrder( + Arrays.stream(uris) + .map(name -> new org.apache.hadoop.fs.Path(perTestTempDir.resolve(name).toUri())) + .toArray())); + return this; + } } } From 91ef9b36b024aa9f8a594e8352f2509024137353 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Thu, 3 Nov 2022 11:15:31 -0700 Subject: [PATCH 13/23] exclude FileSystemStreamSource from jacoco Signed-off-by: Peng Huo --- filesystem/build.gradle | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/filesystem/build.gradle b/filesystem/build.gradle index b4be876507..37baec959e 100644 --- a/filesystem/build.gradle +++ b/filesystem/build.gradle @@ -71,6 +71,13 @@ test.finalizedBy(project.tasks.jacocoTestReport) jacocoTestCoverageVerification { violationRules { rule { + // hadoop-fs depend on native library which is missing on windows. + // https://hadoop.apache.org/docs/r3.3.4/hadoop-project-dist/hadoop-common/NativeLibraries.html#Native_Hadoop_Library + if ('windows' == project.getProperties().getOrDefault('buildPlatform', 'linux')) { + excludes = [ + 'org.opensearch.sql.filesystem.streaming.FileSystemStreamSource' + ] + } element = 'CLASS' limit { counter = 'LINE' From 5fc1eb579101dc944e0abd77d1f2fced9dadbc91 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Thu, 3 Nov 2022 16:49:22 -0700 Subject: [PATCH 14/23] exclude unnecessary depedency Signed-off-by: Peng Huo --- filesystem/build.gradle | 42 +++++++++++++++++++++++++++++++++++------ plugin/build.gradle | 5 +++++ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/filesystem/build.gradle b/filesystem/build.gradle index 37baec959e..0571088132 100644 --- a/filesystem/build.gradle +++ b/filesystem/build.gradle @@ -14,24 +14,54 @@ ext { aws = "1.12.330" } +configurations.all { + resolutionStrategy.force "commons-io:commons-io:2.8.0" +} + dependencies { implementation project(':core') // required by hadoop filesystem https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/filesystem/index.html. implementation("org.apache.hadoop:hadoop-common:${hadoop}") { - exclude group: 'org.apache.zookeeper', module: 'zookeeper' - exclude group: 'com.sun.jersey', module: 'jersey-json' + exclude group: 'org.apache.zookeeper' + exclude group: 'org.eclipse.jetty' + exclude group: 'com.sun.jersey' + exclude group: 'javax.servlet.jsp' + exclude group: 'javax.servlet' + exclude group: 'org.apache.kerby' + exclude group: 'org.apache.curator' exclude group: 'com.google.protobuf', module: 'protobuf-java' exclude group: 'org.apache.avro', module: 'avro' - exclude group: 'org.eclipse.jetty', module: 'jetty-server' + exclude group: 'com.nimbusds', module: 'nimbus-jose-jwt' + // enforce version. + exclude group: 'com.fasterxml.woodstox', module: 'woodstox-core' + exclude group: 'commons-io', module: 'commons-io' + exclude group: 'ch.qos.reload4j', module: 'reload4j' + exclude group: 'org.apache.httpcomponents', module: 'httpcore' } + implementation('com.fasterxml.woodstox:woodstox-core') constraints { implementation('com.fasterxml.woodstox:woodstox-core:6.4.0') { because 'https://www.mend.io/vulnerability-database/CVE-2022-40156' } } - // required https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/index.html - implementation("org.apache.hadoop:hadoop-aws:${hadoop}") - implementation "com.amazonaws:aws-java-sdk-bundle:${aws}" + implementation('commons-io:commons-io') + constraints { + implementation('commons-io:commons-io:2.8.0') { + because 'between versions 2.8.0 and 2.5' + } + } + implementation('ch.qos.reload4j:reload4j') + constraints { + implementation('ch.qos.reload4j:reload4j:1.2.22') { + because 'between versions 1.2.22 and 1.2.19' + } + } + implementation('org.apache.httpcomponents:httpcore') + constraints { + implementation('org.apache.httpcomponents:httpcore:4.4.15') { + because 'between versions 4.4.15 and 4.4.13' + } + } testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' diff --git a/plugin/build.gradle b/plugin/build.gradle index d170b72a95..4295854e37 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -91,6 +91,10 @@ configurations.all { resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_version}" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.6.0" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.6.0" + resolutionStrategy.force "org.apache.commons:commons-math3:3.6.1" + resolutionStrategy.force "org.apache.commons:commons-lang3:3.12.0" + resolutionStrategy.force "joda-time:joda-time:2.10.12" + resolutionStrategy.force "org.slf4j:slf4j-api:1.7.36" } compileJava { options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) @@ -110,6 +114,7 @@ dependencies { api project(':legacy') api project(':opensearch') api project(':prometheus') + api project(':filesystem') } test { From c8b0118638633e2bb85f2132773c516aa18553a9 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Thu, 3 Nov 2022 20:02:09 -0700 Subject: [PATCH 15/23] Update integ-test depedency Signed-off-by: Peng Huo --- integ-test/build.gradle | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 5e0a53bf1a..66b3c1c94c 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -58,6 +58,10 @@ configurations.all { resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_version}" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.6.0" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.6.0" + resolutionStrategy.force "org.apache.commons:commons-math3:3.6.1" + resolutionStrategy.force "org.apache.commons:commons-lang3:3.12.0" + resolutionStrategy.force "joda-time:joda-time:2.10.12" + resolutionStrategy.force "org.slf4j:slf4j-api:1.7.36" } dependencies { From 40535d76d967d731a8f91268d7331d0664abed5e Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Mon, 7 Nov 2022 19:17:26 -0800 Subject: [PATCH 16/23] Add micro batch streaming execution Signed-off-by: Peng Huo --- .../opensearch/sql/executor/QueryService.java | 29 ++- .../MicroBatchStreamingExecution.java | 127 ++++++++++ .../sql/executor/QueryServiceTest.java | 30 ++- .../MicroBatchStreamingExecutionTest.java | 235 ++++++++++++++++++ 4 files changed, 414 insertions(+), 7 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecution.java create mode 100644 core/src/test/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecutionTest.java diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryService.java b/core/src/main/java/org/opensearch/sql/executor/QueryService.java index 423d152e06..0d1604efa7 100644 --- a/core/src/main/java/org/opensearch/sql/executor/QueryService.java +++ b/core/src/main/java/org/opensearch/sql/executor/QueryService.java @@ -37,6 +37,17 @@ public class QueryService { */ public void execute(UnresolvedPlan plan, ResponseListener listener) { + executePlan(analyze(plan), listener); + } + + /** + * Todo. + * + * @param plan {@link LogicalPlan} + * @param listener {@link ResponseListener} + */ + public void executePlan(LogicalPlan plan, + ResponseListener listener) { try { executionEngine.execute(plan(plan), listener); } catch (Exception e) { @@ -54,17 +65,23 @@ public void execute(UnresolvedPlan plan, public void explain(UnresolvedPlan plan, ResponseListener listener) { try { - executionEngine.explain(plan(plan), listener); + executionEngine.explain(plan(analyze(plan)), listener); } catch (Exception e) { listener.onFailure(e); } } - private PhysicalPlan plan(UnresolvedPlan plan) { - // 1.Analyze abstract syntax to generate logical plan - LogicalPlan logicalPlan = analyzer.analyze(plan, new AnalysisContext()); + /** + * Todo. + */ + public LogicalPlan analyze(UnresolvedPlan plan) { + return analyzer.analyze(plan, new AnalysisContext()); + } - // 2.Generate optimal physical plan from logical plan - return planner.plan(logicalPlan); + /** + * Todo. + */ + public PhysicalPlan plan(LogicalPlan plan) { + return planner.plan(plan); } } diff --git a/core/src/main/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecution.java b/core/src/main/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecution.java new file mode 100644 index 0000000000..6d398bfa7c --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecution.java @@ -0,0 +1,127 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.executor.streaming; + +import com.google.common.base.Preconditions; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.executor.ExecutionEngine; +import org.opensearch.sql.executor.QueryService; +import org.opensearch.sql.planner.logical.LogicalPlan; + +public class MicroBatchStreamingExecution { + + private static final Logger log = LogManager.getLogger(MicroBatchStreamingExecution.class); + + static final long INITIAL_LATEST_BATCH_ID = -1L; + + private final StreamingSource source; + + private final LogicalPlan batchPlan; + + private final QueryService queryService; + + /** + * A write-ahead-log that records the offsets that are present in each batch. In order to ensure + * that a given batch will always consist of the same data, we write to this log before any + * processing is done. Thus, the Nth record in this log indicated data that is currently being + * processed and the N-1th entry indicates which offsets have been durably committed to the sink. + */ + private final MetadataLog offsetLog; + + /** keep track the latest commit batchId. */ + private final MetadataLog committedLog; + + /** + * Constructor. + */ + public MicroBatchStreamingExecution( + StreamingSource source, + LogicalPlan batchPlan, + QueryService queryService, + MetadataLog offsetLog, + MetadataLog committedLog) { + this.source = source; + this.batchPlan = batchPlan; + this.queryService = queryService; + // todo. add offsetLog and committedLog offset recovery. + this.offsetLog = offsetLog; + this.committedLog = committedLog; + } + + /** + * Execute micro-batch streaming execution. + */ + public void execute() { + Long latestBatchId = offsetLog.getLatest().map(Pair::getKey).orElse(INITIAL_LATEST_BATCH_ID); + Long latestCommittedBatchId = + committedLog.getLatest().map(Pair::getKey).orElse(INITIAL_LATEST_BATCH_ID); + Optional committedOffset = offsetLog.get(latestCommittedBatchId); + AtomicLong currentBatchId = new AtomicLong(INITIAL_LATEST_BATCH_ID); + + if (latestBatchId.equals(latestCommittedBatchId)) { + // there are no unhandled Offset. + currentBatchId.set(latestCommittedBatchId + 1L); + } else { + Preconditions.checkArgument( + latestBatchId.equals(latestCommittedBatchId + 1L), + "[BUG] Expected latestBatchId - latestCommittedBatchId = 0 or 1, " + + "but latestBatchId=%d, latestCommittedBatchId=%d", + latestBatchId, + latestCommittedBatchId); + + // latestBatchId is not committed yet. + currentBatchId.set(latestBatchId); + } + + Optional availableOffsets = source.getLatestOffset(); + if (hasNewData(availableOffsets, committedOffset)) { + // todo, add batch to execution context. + Batch batch = source.getBatch(committedOffset, availableOffsets.get()); + offsetLog.add(currentBatchId.get(), availableOffsets.get()); + + queryService.executePlan( + batchPlan, + new ResponseListener<>() { + @Override + public void onResponse(ExecutionEngine.QueryResponse response) { + long finalBatchId = currentBatchId.get(); + Offset finalAvailableOffsets = availableOffsets.get(); + committedLog.add(finalBatchId, finalAvailableOffsets); + } + + @Override + public void onFailure(Exception e) { + log.error("streaming processing failed. source = {}", source); + } + }); + } + } + + private boolean hasNewData(Optional availableOffsets, Optional committedOffset) { + if (availableOffsets.equals(committedOffset)) { + log.debug("source does not have new data, exit. source = {}", source); + return false; + } else { + Preconditions.checkArgument( + availableOffsets.isPresent(), "[BUG] available offsets must be no empty"); + + log.debug( + "source has new data. source = {}, availableOffsets:{}, committedOffset:{}", + source, + availableOffsets, + committedOffset); + return true; + } + } +} diff --git a/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java b/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java index 2884544dd0..2c70cf9f92 100644 --- a/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java @@ -13,6 +13,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.when; import java.util.Collections; @@ -58,7 +59,7 @@ class QueryServiceTest { @BeforeEach public void setUp() { - when(analyzer.analyze(any(), any())).thenReturn(logicalPlan); + lenient().when(analyzer.analyze(any(), any())).thenReturn(logicalPlan); when(planner.plan(any())).thenReturn(plan); queryService = new QueryService(analyzer, executionEngine, planner); @@ -161,4 +162,31 @@ public void onFailure(Exception e) { } }); } + + @Test + public void testExecutePlanShouldPass() { + doAnswer( + invocation -> { + ResponseListener listener = invocation.getArgument(1); + listener.onResponse( + new ExecutionEngine.QueryResponse(schema, Collections.emptyList())); + return null; + }) + .when(executionEngine) + .execute(any(), any()); + + queryService.executePlan( + logicalPlan, + new ResponseListener<>() { + @Override + public void onResponse(ExecutionEngine.QueryResponse pplQueryResponse) { + + } + + @Override + public void onFailure(Exception e) { + fail(); + } + }); + } } diff --git a/core/src/test/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecutionTest.java b/core/src/test/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecutionTest.java new file mode 100644 index 0000000000..4c0ca308bd --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecutionTest.java @@ -0,0 +1,235 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.executor.streaming; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.lenient; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.executor.ExecutionEngine; +import org.opensearch.sql.executor.QueryService; +import org.opensearch.sql.planner.logical.LogicalPlan; + +@ExtendWith(MockitoExtension.class) +class MicroBatchStreamingExecutionTest { + + @Test + void executedSuccess() { + streamingQuery() + .addData() + .executeSuccess() + .latestOffsetLogShouldBe(0L) + .latestCommittedLogShouldBe(0L); + } + + @Test + void executedFailed() { + streamingQuery() + .addData() + .executeFailed() + .latestOffsetLogShouldBe(0L) + .noCommittedLog(); + } + + @Test + void noDataInSource() { + streamingQuery() + .executeSuccess() + .noOffsetLog() + .noCommittedLog(); + } + + @Test + void noNewDataInSource() { + streamingQuery() + .addData() + .executeSuccess() + .latestOffsetLogShouldBe(0L) + .latestCommittedLogShouldBe(0L) + .executeSuccess() + .latestOffsetLogShouldBe(0L) + .latestCommittedLogShouldBe(0L); + } + + @Test + void addNewDataInSequenceAllExecuteSuccess() { + streamingQuery() + .addData() + .executeSuccess() + .latestOffsetLogShouldBe(0L) + .latestCommittedLogShouldBe(0L) + .addData() + .executeSuccess() + .latestOffsetLogShouldBe(1L) + .latestCommittedLogShouldBe(1L); + } + + @Test + void addNewDataInSequenceExecuteFailedInBetween() { + streamingQuery() + .addData() + .executeSuccess() + .latestOffsetLogShouldBe(0L) + .latestCommittedLogShouldBe(0L) + .addData() + .executeFailed() + .latestOffsetLogShouldBe(1L) + .latestCommittedLogShouldBe(0L) + .executeSuccess() + .latestOffsetLogShouldBe(1L) + .latestCommittedLogShouldBe(1L); + } + + @Test + void addNewDataInSequenceExecuteFailed() { + streamingQuery() + .addData() + .executeSuccess() + .latestOffsetLogShouldBe(0L) + .latestCommittedLogShouldBe(0L) + .addData() + .executeFailed() + .latestOffsetLogShouldBe(1L) + .latestCommittedLogShouldBe(0L) + .executeFailed() + .latestOffsetLogShouldBe(1L) + .latestCommittedLogShouldBe(0L); + } + + Helper streamingQuery() { + return new Helper(); + } + + private static class Helper { + + private final MicroBatchStreamingExecution execution; + + private final MetadataLog offsetLog; + + private final MetadataLog committedLog; + + private final LogicalPlan batchPlan; + + private final QueryService queryService; + + private final TestStreamingSource source = new TestStreamingSource(); + + public Helper() { + this.offsetLog = new DefaultMetadataLog<>(); + this.committedLog = new DefaultMetadataLog<>(); + this.batchPlan = Mockito.mock(LogicalPlan.class); + this.queryService = Mockito.mock(QueryService.class); + this.execution = + new MicroBatchStreamingExecution( + source, batchPlan, queryService, offsetLog, committedLog); + } + + Helper addData() { + source.addData(); + return this; + } + + Helper executeSuccess() { + lenient().doAnswer( + invocation -> { + ResponseListener listener = + invocation.getArgument(1); + listener.onResponse( + new ExecutionEngine.QueryResponse(null, Collections.emptyList())); + return null; + }) + .when(queryService) + .executePlan(any(), any()); + execution.execute(); + + return this; + } + + Helper executeFailed() { + lenient().doAnswer( + invocation -> { + ResponseListener listener = invocation.getArgument(1); + listener.onFailure(new RuntimeException()); + return null; + }) + .when(queryService) + .executePlan(any(), any()); + execution.execute(); + + return this; + } + + Helper noCommittedLog() { + assertTrue(committedLog.getLatest().isEmpty()); + return this; + } + + Helper latestCommittedLogShouldBe(Long offsetId) { + assertTrue(committedLog.getLatest().isPresent()); + assertEquals(offsetId, committedLog.getLatest().get().getRight().getOffset()); + return this; + } + + Helper noOffsetLog() { + assertTrue(offsetLog.getLatest().isEmpty()); + return this; + } + + Helper latestOffsetLogShouldBe(Long offsetId) { + assertTrue(offsetLog.getLatest().isPresent()); + assertEquals(offsetId, offsetLog.getLatest().get().getRight().getOffset()); + return this; + } + } + + /** + * StreamingSource impl only for testing. + * + *

* initially, offset is -1, getLatestOffset() will return Optional.emtpy(). + *

* call addData() add offset by one. + */ + static class TestStreamingSource implements StreamingSource { + + private final AtomicLong offset = new AtomicLong(-1L); + + /** + * add offset by one. + */ + void addData() { + offset.incrementAndGet(); + } + + /** + * return offset if addData was called. + */ + @Override + public Optional getLatestOffset() { + if (offset.get() == -1) { + return Optional.empty(); + } else { + return Optional.of(new Offset(offset.get())); + } + } + + /** + * always return `empty` Batch regardless start and end offset. + */ + @Override + public Batch getBatch(Optional start, Offset end) { + return new Batch(new ArrayList<>()); + } + } +} From 48eeb0e7dc9d75b8136cb98334371dbd32b5afa1 Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Tue, 8 Nov 2022 10:56:21 -0800 Subject: [PATCH 17/23] Deprecate span collector (#990) * Make span collector not special Signed-off-by: Chen Dai * Add more UT for span Signed-off-by: Chen Dai * Prepare for removing span collector Signed-off-by: Chen Dai * Delete span collector Signed-off-by: Chen Dai * Delete unused allocate and locate code in Rounding class Signed-off-by: Chen Dai * Fix broken UT Signed-off-by: Chen Dai * Update javadoc Signed-off-by: Chen Dai * Fix broken UT after merge Signed-off-by: Chen Dai Signed-off-by: Chen Dai --- .../sql/expression/span/SpanExpression.java | 15 +- .../planner/physical/AggregationOperator.java | 21 +- .../physical/collector/BucketCollector.java | 7 +- .../planner/physical/collector/Collector.java | 11 +- .../planner/physical/collector/Rounding.java | 222 ------------------ .../physical/collector/SpanCollector.java | 69 ------ .../expression/span/SpanExpressionTest.java | 10 +- .../physical/AggregationOperatorTest.java | 91 +++++++ .../physical/collector/RoundingTest.java | 5 - 9 files changed, 115 insertions(+), 336 deletions(-) delete mode 100644 core/src/main/java/org/opensearch/sql/planner/physical/collector/SpanCollector.java diff --git a/core/src/main/java/org/opensearch/sql/expression/span/SpanExpression.java b/core/src/main/java/org/opensearch/sql/expression/span/SpanExpression.java index 715c911b13..aff114145e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/span/SpanExpression.java +++ b/core/src/main/java/org/opensearch/sql/expression/span/SpanExpression.java @@ -7,7 +7,6 @@ import lombok.EqualsAndHashCode; import lombok.Getter; -import lombok.RequiredArgsConstructor; import lombok.ToString; import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.data.model.ExprValue; @@ -15,8 +14,8 @@ import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.ExpressionNodeVisitor; import org.opensearch.sql.expression.env.Environment; +import org.opensearch.sql.planner.physical.collector.Rounding; -@RequiredArgsConstructor @Getter @ToString @EqualsAndHashCode @@ -25,9 +24,19 @@ public class SpanExpression implements Expression { private final Expression value; private final SpanUnit unit; + /** + * Construct a span expression by field and span interval expression. + */ + public SpanExpression(Expression field, Expression value, SpanUnit unit) { + this.field = field; + this.value = value; + this.unit = unit; + } + @Override public ExprValue valueOf(Environment valueEnv) { - return value.valueOf(valueEnv); + Rounding rounding = Rounding.createRounding(this); //TODO: will integrate with WindowAssigner + return rounding.round(field.valueOf(valueEnv)); } /** diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/AggregationOperator.java b/core/src/main/java/org/opensearch/sql/planner/physical/AggregationOperator.java index 3cf823d0e2..1d9523464b 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/AggregationOperator.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/AggregationOperator.java @@ -34,8 +34,7 @@ public class AggregationOperator extends PhysicalPlan { private final List aggregatorList; @Getter private final List groupByExprList; - @Getter - private final NamedExpression span; + /** * {@link BindingTuple} Collector. */ @@ -56,18 +55,7 @@ public AggregationOperator(PhysicalPlan input, List aggregatorL this.input = input; this.aggregatorList = aggregatorList; this.groupByExprList = groupByExprList; - if (hasSpan(groupByExprList)) { - // span expression is always the first expression in group list if exist. - this.span = groupByExprList.get(0); - this.collector = - Collector.Builder.build( - this.span, groupByExprList.subList(1, groupByExprList.size()), this.aggregatorList); - - } else { - this.span = null; - this.collector = - Collector.Builder.build(this.span, this.groupByExprList, this.aggregatorList); - } + this.collector = Collector.Builder.build(groupByExprList, this.aggregatorList); } @Override @@ -99,9 +87,4 @@ public void open() { } iterator = collector.results().iterator(); } - - private boolean hasSpan(List namedExpressionList) { - return !namedExpressionList.isEmpty() - && namedExpressionList.get(0).getDelegated() instanceof SpanExpression; - } } diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/collector/BucketCollector.java b/core/src/main/java/org/opensearch/sql/planner/physical/collector/BucketCollector.java index c3a803ca9e..a00fd3cdf3 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/collector/BucketCollector.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/collector/BucketCollector.java @@ -7,11 +7,11 @@ import com.google.common.collect.ImmutableList; import java.util.Arrays; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.TreeMap; import java.util.function.Supplier; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; @@ -38,9 +38,10 @@ public class BucketCollector implements Collector { private final Supplier supplier; /** - * Map between bucketKey and collector in the bucket. + * Map from bucketKey to nested collector sorted by key to make sure + * final result is in order after traversal. */ - private final Map collectorMap = new HashMap<>(); + private final Map collectorMap = new TreeMap<>(); /** * Bucket Index. diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/collector/Collector.java b/core/src/main/java/org/opensearch/sql/planner/physical/collector/Collector.java index 66eba7440b..a2b3a41a27 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/collector/Collector.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/collector/Collector.java @@ -40,17 +40,14 @@ class Builder { /** * build {@link Collector}. */ - public static Collector build( - NamedExpression span, List buckets, List aggregators) { - if (span == null && buckets.isEmpty()) { + public static Collector build(List buckets, + List aggregators) { + if (buckets.isEmpty()) { return new MetricCollector(aggregators); - } else if (span != null) { - return new SpanCollector(span, () -> build(null, buckets, aggregators)); } else { return new BucketCollector( buckets.get(0), - () -> - build(null, ImmutableList.copyOf(buckets.subList(1, buckets.size())), aggregators)); + () -> build(ImmutableList.copyOf(buckets.subList(1, buckets.size())), aggregators)); } } } diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java b/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java index 2934b44e95..4eaafea59b 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java @@ -17,7 +17,6 @@ import java.time.LocalDateTime; import java.time.LocalTime; import java.time.ZoneId; -import java.time.ZonedDateTime; import java.time.temporal.ChronoField; import java.util.Arrays; import java.util.concurrent.TimeUnit; @@ -40,10 +39,6 @@ */ @EqualsAndHashCode public abstract class Rounding { - @Getter - protected T maxRounded; - @Getter - protected T minRounded; /** * Create Rounding instance. @@ -75,10 +70,6 @@ public static Rounding createRounding(SpanExpression span) { public abstract ExprValue round(ExprValue value); - public abstract Integer locate(ExprValue value); - - public abstract ExprValue[] createBuckets(); - static class TimestampRounding extends Rounding { private final ExprValue interval; @@ -93,50 +84,8 @@ public TimestampRounding(ExprValue interval, String unit) { public ExprValue round(ExprValue var) { Instant instant = Instant.ofEpochMilli(dateTimeUnit.round(var.timestampValue() .toEpochMilli(), interval.integerValue())); - updateRounded(instant); return new ExprTimestampValue(instant); } - - @Override - public ExprValue[] createBuckets() { - if (dateTimeUnit.isMillisBased) { - int size = (int) ((maxRounded.toEpochMilli() - minRounded.toEpochMilli()) / (interval - .integerValue() * dateTimeUnit.ratio)) + 1; - return new ExprValue[size]; - } else { - ZonedDateTime maxZonedDateTime = maxRounded.atZone(ZoneId.of("UTC")); - ZonedDateTime minZonedDateTime = minRounded.atZone(ZoneId.of("UTC")); - int monthDiff = (maxZonedDateTime.getYear() - minZonedDateTime - .getYear()) * 12 + maxZonedDateTime.getMonthValue() - minZonedDateTime.getMonthValue(); - int size = monthDiff / ((int) dateTimeUnit.ratio * interval.integerValue()) + 1; - return new ExprValue[size]; - } - } - - @Override - public Integer locate(ExprValue value) { - if (dateTimeUnit.isMillisBased) { - long intervalInEpochMillis = dateTimeUnit.ratio; - return Long.valueOf((value.timestampValue() - .atZone(ZoneId.of("UTC")).toInstant().toEpochMilli() - minRounded - .atZone(ZoneId.of("UTC")).toInstant().toEpochMilli()) / (intervalInEpochMillis - * interval.integerValue())).intValue(); - } else { - int monthDiff = (value.dateValue().getYear() - minRounded.atZone(ZoneId.of("UTC")) - .getYear()) * 12 + value.dateValue().getMonthValue() - minRounded - .atZone(ZoneId.of("UTC")).getMonthValue(); - return (int) (monthDiff / (dateTimeUnit.ratio * interval.integerValue())); - } - } - - private void updateRounded(Instant value) { - if (maxRounded == null || value.isAfter(maxRounded)) { - maxRounded = value; - } - if (minRounded == null || value.isBefore(minRounded)) { - minRounded = value; - } - } } @@ -153,52 +102,8 @@ public DatetimeRounding(ExprValue interval, String unit) { public ExprValue round(ExprValue var) { Instant instant = Instant.ofEpochMilli(dateTimeUnit.round(var.datetimeValue() .atZone(ZoneId.of("UTC")).toInstant().toEpochMilli(), interval.integerValue())); - updateRounded(instant); return new ExprDatetimeValue(instant.atZone(ZoneId.of("UTC")).toLocalDateTime()); } - - @Override - public ExprValue[] createBuckets() { - if (dateTimeUnit.isMillisBased) { - int size = (int) ((maxRounded.atZone(ZoneId.of("UTC")).toInstant() - .toEpochMilli() - minRounded.atZone(ZoneId.of("UTC")).toInstant() - .toEpochMilli()) / (interval.integerValue() * dateTimeUnit.ratio)) + 1; - return new ExprValue[size]; - } else { - ZonedDateTime maxZonedDateTime = maxRounded.atZone(ZoneId.of("UTC")); - ZonedDateTime minZonedDateTime = minRounded.atZone(ZoneId.of("UTC")); - int monthDiff = (maxZonedDateTime.getYear() - minZonedDateTime - .getYear()) * 12 + maxZonedDateTime.getMonthValue() - minZonedDateTime.getMonthValue(); - int size = monthDiff / ((int) dateTimeUnit.ratio * interval.integerValue()) + 1; - return new ExprValue[size]; - } - } - - @Override - public Integer locate(ExprValue value) { - if (dateTimeUnit.isMillisBased) { - long intervalInEpochMillis = dateTimeUnit.ratio; - return Long.valueOf((value.datetimeValue() - .atZone(ZoneId.of("UTC")).toInstant().toEpochMilli() - minRounded - .atZone(ZoneId.of("UTC")).toInstant().toEpochMilli()) / (intervalInEpochMillis - * interval.integerValue())).intValue(); - } else { - int monthDiff = (value.datetimeValue().getYear() - minRounded.getYear()) * 12 - + value.dateValue().getMonthValue() - minRounded.getMonthValue(); - return (int) (monthDiff / (dateTimeUnit.ratio * interval.integerValue())); - } - } - - private void updateRounded(Instant value) { - if (maxRounded == null || value.isAfter(maxRounded - .atZone(ZoneId.of("UTC")).toInstant())) { - maxRounded = value.atZone(ZoneId.of("UTC")).toLocalDateTime(); - } - if (minRounded == null || value.isBefore(minRounded - .atZone(ZoneId.of("UTC")).toInstant())) { - minRounded = value.atZone(ZoneId.of("UTC")).toLocalDateTime(); - } - } } @@ -215,52 +120,8 @@ public DateRounding(ExprValue interval, String unit) { public ExprValue round(ExprValue var) { Instant instant = Instant.ofEpochMilli(dateTimeUnit.round(var.dateValue().atStartOfDay() .atZone(ZoneId.of("UTC")).toInstant().toEpochMilli(), interval.integerValue())); - updateRounded(instant); return new ExprDateValue(instant.atZone(ZoneId.of("UTC")).toLocalDate()); } - - @Override - public ExprValue[] createBuckets() { - if (dateTimeUnit.isMillisBased) { - int size = (int) ((maxRounded.atStartOfDay().atZone(ZoneId.of("UTC")).toInstant() - .toEpochMilli() - minRounded.atStartOfDay().atZone(ZoneId.of("UTC")).toInstant() - .toEpochMilli()) / (interval.integerValue() * dateTimeUnit.ratio)) + 1; - return new ExprValue[size]; - } else { - ZonedDateTime maxZonedDateTime = maxRounded.atStartOfDay().atZone(ZoneId.of("UTC")); - ZonedDateTime minZonedDateTime = minRounded.atStartOfDay().atZone(ZoneId.of("UTC")); - int monthDiff = (maxZonedDateTime.getYear() - minZonedDateTime - .getYear()) * 12 + maxZonedDateTime.getMonthValue() - minZonedDateTime.getMonthValue(); - int size = monthDiff / ((int) dateTimeUnit.ratio * interval.integerValue()) + 1; - return new ExprValue[size]; - } - } - - @Override - public Integer locate(ExprValue value) { - if (dateTimeUnit.isMillisBased) { - long intervalInEpochMillis = dateTimeUnit.ratio; - return Long.valueOf((value.dateValue().atStartOfDay() - .atZone(ZoneId.of("UTC")).toInstant().toEpochMilli() - minRounded.atStartOfDay() - .atZone(ZoneId.of("UTC")).toInstant().toEpochMilli()) / (intervalInEpochMillis - * interval.integerValue())).intValue(); - } else { - int monthDiff = (value.dateValue().getYear() - minRounded.getYear()) * 12 - + value.dateValue().getMonthValue() - minRounded.getMonthValue(); - return (int) (monthDiff / (dateTimeUnit.ratio * interval.integerValue())); - } - } - - private void updateRounded(Instant value) { - if (maxRounded == null || value.isAfter(maxRounded.atStartOfDay() - .atZone(ZoneId.of("UTC")).toInstant())) { - maxRounded = value.atZone(ZoneId.of("UTC")).toLocalDate(); - } - if (minRounded == null || value.isBefore(minRounded.atStartOfDay() - .atZone(ZoneId.of("UTC")).toInstant())) { - minRounded = value.atZone(ZoneId.of("UTC")).toLocalDate(); - } - } } static class TimeRounding extends Rounding { @@ -281,39 +142,8 @@ public ExprValue round(ExprValue var) { Instant instant = Instant.ofEpochMilli(dateTimeUnit.round(var.timeValue().getLong( ChronoField.MILLI_OF_DAY), interval.integerValue())); - updateRounded(instant); return new ExprTimeValue(instant.atZone(ZoneId.of("UTC")).toLocalTime()); } - - @Override - public ExprValue[] createBuckets() { - // local time is converted to timestamp on 1970-01-01 for aggregations - int size = (int) ((maxRounded.atDate(LocalDate.of(1970, 1, 1)) - .atZone(ZoneId.of("UTC")).toInstant().toEpochMilli() - minRounded - .atDate(LocalDate.of(1970, 1, 1)).atZone(ZoneId.of("UTC")).toInstant() - .toEpochMilli()) / (interval.integerValue() * dateTimeUnit.ratio)) + 1; - return new ExprValue[size]; - } - - @Override - public Integer locate(ExprValue value) { - long intervalInEpochMillis = dateTimeUnit.ratio; - return Long.valueOf((value.timeValue().atDate(LocalDate.of(1970, 1, 1)) - .atZone(ZoneId.of("UTC")).toInstant().toEpochMilli() - minRounded - .atDate(LocalDate.of(1970, 1, 1)) - .atZone(ZoneId.of("UTC")).toInstant().toEpochMilli()) / (intervalInEpochMillis * interval - .integerValue())).intValue(); - } - - private void updateRounded(Instant value) { - if (maxRounded == null || value.isAfter(maxRounded.atDate(LocalDate.of(1970, 1, 1)) - .atZone(ZoneId.of("UTC")).toInstant())) { - maxRounded = value.atZone(ZoneId.of("UTC")).toLocalTime(); - } - if (minRounded == null) { - minRounded = value.atZone(ZoneId.of("UTC")).toLocalTime(); - } - } } @@ -327,29 +157,8 @@ protected LongRounding(ExprValue interval) { @Override public ExprValue round(ExprValue value) { long rounded = Math.floorDiv(value.longValue(), longInterval) * longInterval; - updateRounded(rounded); return ExprValueUtils.longValue(rounded); } - - @Override - public Integer locate(ExprValue value) { - return Long.valueOf((value.longValue() - minRounded) / longInterval).intValue(); - } - - @Override - public ExprValue[] createBuckets() { - int size = Long.valueOf((maxRounded - minRounded) / longInterval).intValue() + 1; - return new ExprValue[size]; - } - - private void updateRounded(Long value) { - if (maxRounded == null || value > maxRounded) { - maxRounded = value; - } - if (minRounded == null || value < minRounded) { - minRounded = value; - } - } } @@ -364,29 +173,8 @@ protected DoubleRounding(ExprValue interval) { public ExprValue round(ExprValue value) { double rounded = Double .valueOf(value.doubleValue() / doubleInterval).intValue() * doubleInterval; - updateRounded(rounded); return ExprValueUtils.doubleValue(rounded); } - - @Override - public Integer locate(ExprValue value) { - return Double.valueOf((value.doubleValue() - minRounded) / doubleInterval).intValue(); - } - - @Override - public ExprValue[] createBuckets() { - int size = Double.valueOf((maxRounded - minRounded) / doubleInterval).intValue() + 1; - return new ExprValue[size]; - } - - private void updateRounded(Double value) { - if (maxRounded == null || value > maxRounded) { - maxRounded = value; - } - if (minRounded == null || value < minRounded) { - minRounded = value; - } - } } @@ -396,16 +184,6 @@ static class UnknownRounding extends Rounding { public ExprValue round(ExprValue var) { return null; } - - @Override - public Integer locate(ExprValue value) { - return null; - } - - @Override - public ExprValue[] createBuckets() { - return new ExprValue[0]; - } } diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/collector/SpanCollector.java b/core/src/main/java/org/opensearch/sql/planner/physical/collector/SpanCollector.java deleted file mode 100644 index 092d79bd81..0000000000 --- a/core/src/main/java/org/opensearch/sql/planner/physical/collector/SpanCollector.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.planner.physical.collector; - -import java.util.function.Supplier; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.expression.NamedExpression; -import org.opensearch.sql.expression.span.SpanExpression; -import org.opensearch.sql.storage.bindingtuple.BindingTuple; - -/** - * Span Collector. - */ -public class SpanCollector extends BucketCollector { - - /** - * Span Expression. - */ - private final SpanExpression spanExpr; - - /** - * Rounding. - */ - private final Rounding rounding; - - /** - * Constructor. - */ - public SpanCollector(NamedExpression bucketExpr, Supplier supplier) { - super(bucketExpr, supplier); - this.spanExpr = (SpanExpression) bucketExpr.getDelegated(); - this.rounding = Rounding.createRounding(spanExpr); - } - - /** - * Rounding bucket value. - * - * @param tuple {@link BindingTuple}. - * @return {@link ExprValue}. - */ - @Override - protected ExprValue bucketKey(BindingTuple tuple) { - return rounding.round(spanExpr.getField().valueOf(tuple)); - } - - /** - * Allocates Buckets for building results. - * - * @return buckets. - */ - @Override - protected ExprValue[] allocateBuckets() { - return rounding.createBuckets(); - } - - /** - * Current Bucket index in allocated buckets. - * - * @param value bucket key. - * @return index. - */ - @Override - protected int locateBucket(ExprValue value) { - return rounding.locate(value); - } -} diff --git a/core/src/test/java/org/opensearch/sql/expression/span/SpanExpressionTest.java b/core/src/test/java/org/opensearch/sql/expression/span/SpanExpressionTest.java index a554a93153..f311c0147b 100644 --- a/core/src/test/java/org/opensearch/sql/expression/span/SpanExpressionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/span/SpanExpressionTest.java @@ -8,7 +8,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; -import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import org.junit.jupiter.api.DisplayNameGeneration; import org.junit.jupiter.api.DisplayNameGenerator; @@ -20,22 +19,17 @@ @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) public class SpanExpressionTest extends ExpressionTestBase { @Test - void span() { + void testSpanByNumeric() { SpanExpression span = DSL.span(DSL.ref("integer_value", INTEGER), DSL.literal(1), ""); assertEquals(INTEGER, span.type()); assertEquals(ExprValueUtils.integerValue(1), span.valueOf(valueEnv())); span = DSL.span(DSL.ref("integer_value", INTEGER), DSL.literal(1.5), ""); assertEquals(DOUBLE, span.type()); - assertEquals(ExprValueUtils.doubleValue(1.5), span.valueOf(valueEnv())); + assertEquals(ExprValueUtils.doubleValue(0.0), span.valueOf(valueEnv())); span = DSL.span(DSL.ref("double_value", DOUBLE), DSL.literal(1), ""); assertEquals(DOUBLE, span.type()); assertEquals(ExprValueUtils.doubleValue(1.0), span.valueOf(valueEnv())); - - span = DSL.span(DSL.ref("timestamp_value", TIMESTAMP), DSL.literal(1), "d"); - assertEquals(TIMESTAMP, span.type()); - assertEquals(ExprValueUtils.integerValue(1), span.valueOf(valueEnv())); } - } diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/AggregationOperatorTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/AggregationOperatorTest.java index 318499c075..46b9821752 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/AggregationOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/AggregationOperatorTest.java @@ -28,6 +28,7 @@ import org.junit.jupiter.api.Test; import org.opensearch.sql.data.model.ExprDateValue; import org.opensearch.sql.data.model.ExprDatetimeValue; +import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTimeValue; import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprValue; @@ -496,6 +497,96 @@ public void twoBucketsSpanAndLong() { )); } + @Test + public void aggregate_with_two_groups_with_windowing() { + PhysicalPlan plan = new AggregationOperator(testScan(compoundInputs), + Collections.singletonList(DSL.named("sum", dsl.sum(DSL.ref("errors", INTEGER)))), + Arrays.asList( + DSL.named("host", DSL.ref("host", STRING)), + DSL.named("span", DSL.span(DSL.ref("day", DATE), DSL.literal(1), "d")))); + + List result = execute(plan); + assertEquals(7, result.size()); + assertThat(result, containsInRelativeOrder( + ExprValueUtils.tupleValue(ImmutableMap.of( + "host", new ExprStringValue("h1"), + "span", new ExprDateValue("2021-01-03"), + "sum", 2)), + ExprValueUtils.tupleValue(ImmutableMap.of( + "host", new ExprStringValue("h1"), + "span", new ExprDateValue("2021-01-04"), + "sum", 1)), + ExprValueUtils.tupleValue(ImmutableMap.of( + "host", new ExprStringValue("h1"), + "span", new ExprDateValue("2021-01-06"), + "sum", 1)), + ExprValueUtils.tupleValue(ImmutableMap.of( + "host", new ExprStringValue("h1"), + "span", new ExprDateValue("2021-01-07"), + "sum", 6)), + ExprValueUtils.tupleValue(ImmutableMap.of( + "host", new ExprStringValue("h2"), + "span", new ExprDateValue("2021-01-03"), + "sum", 3)), + ExprValueUtils.tupleValue(ImmutableMap.of( + "host", new ExprStringValue("h2"), + "span", new ExprDateValue("2021-01-04"), + "sum", 10)), + ExprValueUtils.tupleValue(ImmutableMap.of( + "host", new ExprStringValue("h2"), + "span", new ExprDateValue("2021-01-07"), + "sum", 8)))); + } + + @Test + public void aggregate_with_three_groups_with_windowing() { + PhysicalPlan plan = new AggregationOperator(testScan(compoundInputs), + Collections.singletonList(DSL.named("sum", dsl.sum(DSL.ref("errors", INTEGER)))), + Arrays.asList( + DSL.named("host", DSL.ref("host", STRING)), + DSL.named("span", DSL.span(DSL.ref("day", DATE), DSL.literal(1), "d")), + DSL.named("region", DSL.ref("region", STRING)))); + + List result = execute(plan); + assertEquals(7, result.size()); + assertThat(result, containsInRelativeOrder( + ExprValueUtils.tupleValue(ImmutableMap.of( + "host", new ExprStringValue("h1"), + "span", new ExprDateValue("2021-01-03"), + "region", new ExprStringValue("iad"), + "sum", 2)), + ExprValueUtils.tupleValue(ImmutableMap.of( + "host", new ExprStringValue("h1"), + "span", new ExprDateValue("2021-01-04"), + "region", new ExprStringValue("iad"), + "sum", 1)), + ExprValueUtils.tupleValue(ImmutableMap.of( + "host", new ExprStringValue("h1"), + "span", new ExprDateValue("2021-01-06"), + "region", new ExprStringValue("iad"), + "sum", 1)), + ExprValueUtils.tupleValue(ImmutableMap.of( + "host", new ExprStringValue("h1"), + "span", new ExprDateValue("2021-01-07"), + "region", new ExprStringValue("iad"), + "sum", 6)), + ExprValueUtils.tupleValue(ImmutableMap.of( + "host", new ExprStringValue("h2"), + "span", new ExprDateValue("2021-01-03"), + "region", new ExprStringValue("iad"), + "sum", 3)), + ExprValueUtils.tupleValue(ImmutableMap.of( + "host", new ExprStringValue("h2"), + "span", new ExprDateValue("2021-01-04"), + "region", new ExprStringValue("iad"), + "sum", 10)), + ExprValueUtils.tupleValue(ImmutableMap.of( + "host", new ExprStringValue("h2"), + "span", new ExprDateValue("2021-01-07"), + "region", new ExprStringValue("iad"), + "sum", 8)))); + } + @Test public void copyOfAggregationOperatorShouldSame() { AggregationOperator plan = new AggregationOperator(testScan(datetimeInputs), diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/collector/RoundingTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/collector/RoundingTest.java index 41b3ea5d6b..f40e5c058b 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/collector/RoundingTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/collector/RoundingTest.java @@ -5,16 +5,13 @@ package org.opensearch.sql.planner.physical.collector; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.TIME; -import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Test; import org.opensearch.sql.data.model.ExprTimeValue; -import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.expression.DSL; @@ -34,8 +31,6 @@ void round_unknown_type() { SpanExpression span = DSL.span(DSL.ref("unknown", STRING), DSL.literal(1), ""); Rounding rounding = Rounding.createRounding(span); assertNull(rounding.round(ExprValueUtils.integerValue(1))); - assertNull(rounding.locate(ExprValueUtils.integerValue(1))); - assertEquals(0, rounding.createBuckets().length); } @Test From 510502200e5efd2fa30d46979fe45217cd84c5bb Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Tue, 8 Nov 2022 13:03:21 -0800 Subject: [PATCH 18/23] Add Streaming source impl (#994) * Add Streaming Source Impl Signed-off-by: Peng Huo * update build.gradle Signed-off-by: Peng Huo * change to hadoop-fs Signed-off-by: Peng Huo * exclude FileSystemStreamSource from jacoco Signed-off-by: Peng Huo * exclude unnecessary depedency Signed-off-by: Peng Huo * Update integ-test depedency Signed-off-by: Peng Huo * change from splits to split in batch Signed-off-by: Peng Huo Signed-off-by: Peng Huo --- .../workflows/sql-test-and-build-workflow.yml | 4 +- .../sql/executor/streaming/Batch.java | 17 ++ .../sql/executor/streaming/Offset.java | 17 ++ .../executor/streaming/StreamingSource.java | 29 +++ .../opensearch/sql/storage/split/Split.java | 21 +++ doctest/build.gradle | 2 +- filesystem/build.gradle | 129 ++++++++++++++ .../storage/split/FileSystemSplit.java | 24 +++ .../filesystem/streaming/FileMetaData.java | 21 +++ .../streaming/FileSystemStreamSource.java | 105 +++++++++++ .../streaming/FileSystemStreamSourceTest.java | 166 ++++++++++++++++++ integ-test/build.gradle | 4 + plugin/build.gradle | 6 +- settings.gradle | 1 + 14 files changed, 542 insertions(+), 4 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/executor/streaming/Batch.java create mode 100644 core/src/main/java/org/opensearch/sql/executor/streaming/Offset.java create mode 100644 core/src/main/java/org/opensearch/sql/executor/streaming/StreamingSource.java create mode 100644 core/src/main/java/org/opensearch/sql/storage/split/Split.java create mode 100644 filesystem/build.gradle create mode 100644 filesystem/src/main/java/org/opensearch/sql/filesystem/storage/split/FileSystemSplit.java create mode 100644 filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileMetaData.java create mode 100644 filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSource.java create mode 100644 filesystem/src/test/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSourceTest.java diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 3d063a2bfc..25e0387cf3 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -25,10 +25,10 @@ jobs: matrix: entry: - { os: ubuntu-latest, java: 11 } - - { os: windows-latest, java: 11, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc} + - { os: windows-latest, java: 11, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc -PbuildPlatform=windows } - { os: macos-latest, java: 11, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc } - { os: ubuntu-latest, java: 17 } - - { os: windows-latest, java: 17, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc } + - { os: windows-latest, java: 17, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc -PbuildPlatform=windows } - { os: macos-latest, java: 17, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc } runs-on: ${{ matrix.entry.os }} diff --git a/core/src/main/java/org/opensearch/sql/executor/streaming/Batch.java b/core/src/main/java/org/opensearch/sql/executor/streaming/Batch.java new file mode 100644 index 0000000000..cd7d7dae5a --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/streaming/Batch.java @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.executor.streaming; + +import lombok.Data; +import org.opensearch.sql.storage.split.Split; + +/** + * A batch of streaming execution. + */ +@Data +public class Batch { + private final Split split; +} diff --git a/core/src/main/java/org/opensearch/sql/executor/streaming/Offset.java b/core/src/main/java/org/opensearch/sql/executor/streaming/Offset.java new file mode 100644 index 0000000000..00f040e437 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/streaming/Offset.java @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.executor.streaming; + +import lombok.Data; + +/** + * Offset. + */ +@Data +public class Offset { + + private final Long offset; +} diff --git a/core/src/main/java/org/opensearch/sql/executor/streaming/StreamingSource.java b/core/src/main/java/org/opensearch/sql/executor/streaming/StreamingSource.java new file mode 100644 index 0000000000..ebd3fa714b --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/streaming/StreamingSource.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.executor.streaming; + +import java.util.Optional; + +/** + * Streaming source. + */ +public interface StreamingSource { + /** + * Get current {@link Offset} of stream data. + * + * @return empty if the stream does not has new data. + */ + Optional getLatestOffset(); + + /** + * Get a {@link Batch} from source between (start, end]. + * + * @param start start offset. + * @param end end offset. + * @return @link Batch}. + */ + Batch getBatch(Optional start, Offset end); +} diff --git a/core/src/main/java/org/opensearch/sql/storage/split/Split.java b/core/src/main/java/org/opensearch/sql/storage/split/Split.java new file mode 100644 index 0000000000..e9e0c6fcc1 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/storage/split/Split.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.storage.split; + +import org.opensearch.sql.storage.StorageEngine; + +/** + * Split is a sections of a data set. Each {@link StorageEngine} should have specific + * implementation of Split. + */ +public interface Split { + + /** + * Get the split id. + * @return split id. + */ + String getSplitId(); +} diff --git a/doctest/build.gradle b/doctest/build.gradle index 8378d5ec00..c1d069e50b 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -89,7 +89,7 @@ doctest.dependsOn startOpenSearch startOpenSearch.dependsOn startPrometheus doctest.finalizedBy stopOpenSearch stopOpenSearch.finalizedBy stopPrometheus -build.dependsOn doctest +check.dependsOn doctest clean.dependsOn(cleanBootstrap) // 2.0.0-alpha1-SNAPSHOT -> 2.0.0.0-alpha1-SNAPSHOT diff --git a/filesystem/build.gradle b/filesystem/build.gradle new file mode 100644 index 0000000000..0571088132 --- /dev/null +++ b/filesystem/build.gradle @@ -0,0 +1,129 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +plugins { + id 'java-library' + id "io.freefair.lombok" + id 'jacoco' +} + +ext { + hadoop = "3.3.4" + aws = "1.12.330" +} + +configurations.all { + resolutionStrategy.force "commons-io:commons-io:2.8.0" +} + +dependencies { + implementation project(':core') + // required by hadoop filesystem https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/filesystem/index.html. + implementation("org.apache.hadoop:hadoop-common:${hadoop}") { + exclude group: 'org.apache.zookeeper' + exclude group: 'org.eclipse.jetty' + exclude group: 'com.sun.jersey' + exclude group: 'javax.servlet.jsp' + exclude group: 'javax.servlet' + exclude group: 'org.apache.kerby' + exclude group: 'org.apache.curator' + exclude group: 'com.google.protobuf', module: 'protobuf-java' + exclude group: 'org.apache.avro', module: 'avro' + exclude group: 'com.nimbusds', module: 'nimbus-jose-jwt' + // enforce version. + exclude group: 'com.fasterxml.woodstox', module: 'woodstox-core' + exclude group: 'commons-io', module: 'commons-io' + exclude group: 'ch.qos.reload4j', module: 'reload4j' + exclude group: 'org.apache.httpcomponents', module: 'httpcore' + } + implementation('com.fasterxml.woodstox:woodstox-core') + constraints { + implementation('com.fasterxml.woodstox:woodstox-core:6.4.0') { + because 'https://www.mend.io/vulnerability-database/CVE-2022-40156' + } + } + implementation('commons-io:commons-io') + constraints { + implementation('commons-io:commons-io:2.8.0') { + because 'between versions 2.8.0 and 2.5' + } + } + implementation('ch.qos.reload4j:reload4j') + constraints { + implementation('ch.qos.reload4j:reload4j:1.2.22') { + because 'between versions 1.2.22 and 1.2.19' + } + } + implementation('org.apache.httpcomponents:httpcore') + constraints { + implementation('org.apache.httpcomponents:httpcore:4.4.15') { + because 'between versions 4.4.15 and 4.4.13' + } + } + + testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') + testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' + testImplementation group: 'org.mockito', name: 'mockito-core', version: '3.12.4' + testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.12.4' +} + +test { + useJUnitPlatform() + testLogging { + events "passed", "skipped", "failed" + exceptionFormat "full" + } + + // hadoop-fs depend on native library which is missing on windows. + // https://hadoop.apache.org/docs/r3.3.4/hadoop-project-dist/hadoop-common/NativeLibraries.html#Native_Hadoop_Library + if ('windows' == project.getProperties().getOrDefault('buildPlatform', 'linux')) { + excludes = [ + '**/FileSystemStreamSourceTest.class' + ] + } +} + +jacocoTestReport { + reports { + html.enabled true + xml.enabled true + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it) + })) + } +} +test.finalizedBy(project.tasks.jacocoTestReport) + +jacocoTestCoverageVerification { + violationRules { + rule { + // hadoop-fs depend on native library which is missing on windows. + // https://hadoop.apache.org/docs/r3.3.4/hadoop-project-dist/hadoop-common/NativeLibraries.html#Native_Hadoop_Library + if ('windows' == project.getProperties().getOrDefault('buildPlatform', 'linux')) { + excludes = [ + 'org.opensearch.sql.filesystem.streaming.FileSystemStreamSource' + ] + } + element = 'CLASS' + limit { + counter = 'LINE' + minimum = 1.0 + } + limit { + counter = 'BRANCH' + minimum = 1.0 + } + } + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it) + })) + } +} +check.dependsOn jacocoTestCoverageVerification +jacocoTestCoverageVerification.dependsOn jacocoTestReport diff --git a/filesystem/src/main/java/org/opensearch/sql/filesystem/storage/split/FileSystemSplit.java b/filesystem/src/main/java/org/opensearch/sql/filesystem/storage/split/FileSystemSplit.java new file mode 100644 index 0000000000..7fefb11a85 --- /dev/null +++ b/filesystem/src/main/java/org/opensearch/sql/filesystem/storage/split/FileSystemSplit.java @@ -0,0 +1,24 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.filesystem.storage.split; + +import java.util.Set; +import java.util.UUID; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.apache.hadoop.fs.Path; +import org.opensearch.sql.storage.split.Split; + +@Data +public class FileSystemSplit implements Split { + + @Getter + @EqualsAndHashCode.Exclude + private final String splitId = UUID.randomUUID().toString(); + + private final Set paths; +} diff --git a/filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileMetaData.java b/filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileMetaData.java new file mode 100644 index 0000000000..6a8c90ee80 --- /dev/null +++ b/filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileMetaData.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.filesystem.streaming; + +import java.util.Set; +import lombok.Data; +import org.apache.hadoop.fs.Path; + +/** + * File metadata. Batch id associate with the set of {@link Path}. + */ +@Data +public class FileMetaData { + + private final Long batchId; + + private final Set paths; +} diff --git a/filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSource.java b/filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSource.java new file mode 100644 index 0000000000..6a9639bdcb --- /dev/null +++ b/filesystem/src/main/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSource.java @@ -0,0 +1,105 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.filesystem.streaming; + +import com.google.common.collect.Sets; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.SneakyThrows; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.sql.executor.streaming.Batch; +import org.opensearch.sql.executor.streaming.DefaultMetadataLog; +import org.opensearch.sql.executor.streaming.MetadataLog; +import org.opensearch.sql.executor.streaming.Offset; +import org.opensearch.sql.executor.streaming.StreamingSource; +import org.opensearch.sql.filesystem.storage.split.FileSystemSplit; + +/** + * FileSystem Streaming Source use Hadoop FileSystem. + */ +public class FileSystemStreamSource implements StreamingSource { + + private static final Logger log = LogManager.getLogger(FileSystemStreamSource.class); + + private final MetadataLog fileMetaDataLog; + + private Set seenFiles; + + private final FileSystem fs; + + private final Path basePath; + + /** + * Constructor of FileSystemStreamSource. + */ + public FileSystemStreamSource(FileSystem fs, Path basePath) { + this.fs = fs; + this.basePath = basePath; + // todo, need to add state recovery + this.fileMetaDataLog = new DefaultMetadataLog<>(); + // todo, need to add state recovery + this.seenFiles = new HashSet<>(); + } + + @SneakyThrows(value = IOException.class) + @Override + public Optional getLatestOffset() { + // list all files. todo. improvement list performance. + Set allFiles = + Arrays.stream(fs.listStatus(basePath)) + .filter(status -> !status.isDirectory()) + .map(FileStatus::getPath) + .collect(Collectors.toSet()); + + // find unread files. + log.debug("all files {}", allFiles); + Set unread = Sets.difference(allFiles, seenFiles); + + // update seenFiles. + seenFiles = allFiles; + log.debug("seen files {}", seenFiles); + + Optional latestBatchIdOptional = fileMetaDataLog.getLatest().map(Pair::getKey); + if (!unread.isEmpty()) { + long latestBatchId = latestBatchIdOptional.map(id -> id + 1).orElse(0L); + fileMetaDataLog.add(latestBatchId, new FileMetaData(latestBatchId, unread)); + log.debug("latestBatchId {}", latestBatchId); + return Optional.of(new Offset(latestBatchId)); + } else { + log.debug("no unread data"); + Optional offset = + latestBatchIdOptional.isEmpty() + ? Optional.empty() + : Optional.of(new Offset(latestBatchIdOptional.get())); + log.debug("return empty offset {}", offset); + return offset; + } + } + + @Override + public Batch getBatch(Optional start, Offset end) { + Long startBatchId = start.map(Offset::getOffset).map(id -> id + 1).orElse(0L); + Long endBatchId = end.getOffset(); + + Set paths = + fileMetaDataLog.get(Optional.of(startBatchId), Optional.of(endBatchId)).stream() + .map(FileMetaData::getPaths) + .flatMap(Set::stream) + .collect(Collectors.toSet()); + + log.debug("fetch files {} with id from: {} to: {}.", paths, start, end); + return new Batch(new FileSystemSplit(paths)); + } +} diff --git a/filesystem/src/test/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSourceTest.java b/filesystem/src/test/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSourceTest.java new file mode 100644 index 0000000000..fba038f6a3 --- /dev/null +++ b/filesystem/src/test/java/org/opensearch/sql/filesystem/streaming/FileSystemStreamSourceTest.java @@ -0,0 +1,166 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.filesystem.streaming; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Optional; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.executor.streaming.Offset; +import org.opensearch.sql.filesystem.storage.split.FileSystemSplit; +import org.opensearch.sql.storage.split.Split; + +@ExtendWith(MockitoExtension.class) +class FileSystemStreamSourceTest { + + @TempDir Path perTestTempDir; + + FileSystemStreamSource streamSource; + + /** + * use hadoop default filesystem. it only works on unix-like system. for running on windows, it + * require native library. Reference. + * https://hadoop.apache.org/docs/r3.3.4/hadoop-project-dist/hadoop-common/NativeLibraries.html + */ + @BeforeEach + void setup() throws IOException { + streamSource = + new FileSystemStreamSource( + FileSystem.get(new Configuration()), + new org.apache.hadoop.fs.Path(perTestTempDir.toUri())); + } + + @Test + void addOneFileToSource() throws IOException { + emptySource().addFile("log1").latestOffsetShouldBe(0L).batchFromStart("log1"); + } + + @Test + void addMultipleFileInSequence() throws IOException { + emptySource() + .addFile("log1") + .latestOffsetShouldBe(0L) + .batchFromStart("log1") + .addFile("log2") + .latestOffsetShouldBe(1L) + .batchFromStart("log1", "log2") + .batchInBetween(0L, 1L, "log2"); + } + + @Test + void latestOffsetShouldSameIfNoNewFileAdded() throws IOException { + emptySource() + .addFile("log1") + .latestOffsetShouldBe(0L) + .batchFromStart("log1") + .latestOffsetShouldBe(0L) + .batchFromStart("log1"); + } + + @Test + void latestOffsetIsEmptyIfNoFilesInSource() { + emptySource().noOffset(); + } + + @Test + void dirIsFiltered() throws IOException { + emptySource() + .addFile("log1") + .latestOffsetShouldBe(0L) + .addDir("dir1") + .latestOffsetShouldBe(0L) + .batchFromStart("log1"); + } + + @Test + void sneakThrowException() throws IOException { + FileSystem fs = Mockito.mock(FileSystem.class); + doThrow(IOException.class).when(fs).listStatus(any(org.apache.hadoop.fs.Path.class)); + + streamSource = + new FileSystemStreamSource(fs, + new org.apache.hadoop.fs.Path(perTestTempDir.toUri())); + assertThrows(IOException.class, () -> streamSource.getLatestOffset()); + } + + StreamSource emptySource() { + return new StreamSource(); + } + + private class StreamSource { + + StreamSource addFile(String filename) throws IOException { + Path file = Files.createFile(perTestTempDir.resolve(filename)); + assertTrue(file.toFile().exists()); + + return this; + } + + StreamSource addDir(String dirname) throws IOException { + Path dir = Files.createDirectory(perTestTempDir.resolve(dirname)); + assertTrue(dir.toFile().isDirectory()); + + return this; + } + + StreamSource noOffset() { + assertFalse(streamSource.getLatestOffset().isPresent()); + + return this; + } + + StreamSource latestOffsetShouldBe(Long offset) { + Optional latestOffset = streamSource.getLatestOffset(); + assertTrue(latestOffset.isPresent()); + assertEquals(new Offset(offset), latestOffset.get()); + + return this; + } + + StreamSource batchFromStart(String... uris) { + assertTrue(streamSource.getLatestOffset().isPresent()); + internalBatchInBetween(Optional.empty(), streamSource.getLatestOffset().get(), uris); + + return this; + } + + StreamSource batchInBetween(Long start, Long end, String... uris) { + internalBatchInBetween(Optional.of(new Offset(start)), new Offset(end), uris); + + return this; + } + + private StreamSource internalBatchInBetween( + Optional start, Offset end, String... uris) { + Split split = streamSource.getBatch(start, end).getSplit(); + assertThat( + ((FileSystemSplit) split).getPaths(), + containsInAnyOrder( + Arrays.stream(uris) + .map(name -> new org.apache.hadoop.fs.Path(perTestTempDir.resolve(name).toUri())) + .toArray())); + return this; + } + } +} diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 11ba5542fd..7a2e5cd406 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -68,6 +68,10 @@ configurations.all { resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.6.0" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.6.0" resolutionStrategy.force "com.squareup.okhttp3:okhttp:4.9.3" + resolutionStrategy.force "org.apache.commons:commons-math3:3.6.1" + resolutionStrategy.force "org.apache.commons:commons-lang3:3.12.0" + resolutionStrategy.force "joda-time:joda-time:2.10.12" + resolutionStrategy.force "org.slf4j:slf4j-api:1.7.36" } dependencies { diff --git a/plugin/build.gradle b/plugin/build.gradle index 6a0900c3cc..4754292216 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -93,6 +93,10 @@ configurations.all { resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.6.0" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.6.0" resolutionStrategy.force "com.squareup.okhttp3:okhttp:4.9.3" + resolutionStrategy.force "org.apache.commons:commons-math3:3.6.1" + resolutionStrategy.force "org.apache.commons:commons-lang3:3.12.0" + resolutionStrategy.force "joda-time:joda-time:2.10.12" + resolutionStrategy.force "org.slf4j:slf4j-api:1.7.36" } compileJava { options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) @@ -108,11 +112,11 @@ dependencies { api "com.fasterxml.jackson.core:jackson-databind:${jackson_databind_version}" api "com.fasterxml.jackson.core:jackson-annotations:${jackson_version}" - api project(":ppl") api project(':legacy') api project(':opensearch') api project(':prometheus') + api project(':filesystem') } test { diff --git a/settings.gradle b/settings.gradle index 2f850f422b..7650959451 100644 --- a/settings.gradle +++ b/settings.gradle @@ -18,4 +18,5 @@ include 'doctest' include 'legacy' include 'sql' include 'prometheus' +include 'filesystem' From d63345b46c72338eae3fe3a60741e8720870a4be Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Tue, 8 Nov 2022 17:04:44 -0800 Subject: [PATCH 19/23] fix compile issue Signed-off-by: Peng Huo --- .../MicroBatchStreamingExecution.java | 5 +- .../MicroBatchStreamingExecutionTest.java | 48 ++++++++----------- 2 files changed, 23 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecution.java b/core/src/main/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecution.java index 6d398bfa7c..190387676d 100644 --- a/core/src/main/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecution.java +++ b/core/src/main/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecution.java @@ -19,6 +19,9 @@ import org.opensearch.sql.executor.QueryService; import org.opensearch.sql.planner.logical.LogicalPlan; +/** + * Micro batch streaming execution. + */ public class MicroBatchStreamingExecution { private static final Logger log = LogManager.getLogger(MicroBatchStreamingExecution.class); @@ -60,7 +63,7 @@ public MicroBatchStreamingExecution( } /** - * Execute micro-batch streaming execution. + * Pull the {@link Batch} from {@link StreamingSource} and execute the {@link Batch}. */ public void execute() { Long latestBatchId = offsetLog.getLatest().map(Pair::getKey).orElse(INITIAL_LATEST_BATCH_ID); diff --git a/core/src/test/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecutionTest.java b/core/src/test/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecutionTest.java index 4c0ca308bd..7fa8f4d670 100644 --- a/core/src/test/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecutionTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecutionTest.java @@ -10,7 +10,6 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.lenient; -import java.util.ArrayList; import java.util.Collections; import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; @@ -37,19 +36,12 @@ void executedSuccess() { @Test void executedFailed() { - streamingQuery() - .addData() - .executeFailed() - .latestOffsetLogShouldBe(0L) - .noCommittedLog(); + streamingQuery().addData().executeFailed().latestOffsetLogShouldBe(0L).noCommittedLog(); } @Test void noDataInSource() { - streamingQuery() - .executeSuccess() - .noOffsetLog() - .noCommittedLog(); + streamingQuery().executeSuccess().noOffsetLog().noCommittedLog(); } @Test @@ -143,7 +135,8 @@ Helper addData() { } Helper executeSuccess() { - lenient().doAnswer( + lenient() + .doAnswer( invocation -> { ResponseListener listener = invocation.getArgument(1); @@ -159,12 +152,14 @@ Helper executeSuccess() { } Helper executeFailed() { - lenient().doAnswer( - invocation -> { - ResponseListener listener = invocation.getArgument(1); - listener.onFailure(new RuntimeException()); - return null; - }) + lenient() + .doAnswer( + invocation -> { + ResponseListener listener = + invocation.getArgument(1); + listener.onFailure(new RuntimeException()); + return null; + }) .when(queryService) .executePlan(any(), any()); execution.execute(); @@ -198,23 +193,20 @@ Helper latestOffsetLogShouldBe(Long offsetId) { /** * StreamingSource impl only for testing. * - *

* initially, offset is -1, getLatestOffset() will return Optional.emtpy(). - *

* call addData() add offset by one. + *

initially, offset is -1, getLatestOffset() will return Optional.emtpy(). + * + *

call addData() add offset by one. */ static class TestStreamingSource implements StreamingSource { private final AtomicLong offset = new AtomicLong(-1L); - /** - * add offset by one. - */ + /** add offset by one. */ void addData() { offset.incrementAndGet(); } - /** - * return offset if addData was called. - */ + /** return offset if addData was called. */ @Override public Optional getLatestOffset() { if (offset.get() == -1) { @@ -224,12 +216,10 @@ public Optional getLatestOffset() { } } - /** - * always return `empty` Batch regardless start and end offset. - */ + /** always return `empty` Batch regardless start and end offset. */ @Override public Batch getBatch(Optional start, Offset end) { - return new Batch(new ArrayList<>()); + return new Batch(() -> "id"); } } } From 6aee254892b92d50db57bc3131bcae9172a8573d Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Wed, 9 Nov 2022 10:31:18 -0800 Subject: [PATCH 20/23] add PlanContext Signed-off-by: Peng Huo --- .../opensearch/sql/executor/QueryService.java | 14 ++- .../MicroBatchStreamingExecution.java | 4 +- .../opensearch/sql/planner/PlanContext.java | 31 +++++++ .../sql/executor/QueryServiceTest.java | 5 + .../MicroBatchStreamingExecutionTest.java | 91 +++++++++++++++---- .../sql/planner/PlanContextTest.java | 31 +++++++ 6 files changed, 154 insertions(+), 22 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/planner/PlanContext.java create mode 100644 core/src/test/java/org/opensearch/sql/planner/PlanContextTest.java diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryService.java b/core/src/main/java/org/opensearch/sql/executor/QueryService.java index 0d1604efa7..32fda1ab2c 100644 --- a/core/src/main/java/org/opensearch/sql/executor/QueryService.java +++ b/core/src/main/java/org/opensearch/sql/executor/QueryService.java @@ -13,6 +13,7 @@ import org.opensearch.sql.analysis.Analyzer; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.planner.PlanContext; import org.opensearch.sql.planner.Planner; import org.opensearch.sql.planner.logical.LogicalPlan; import org.opensearch.sql.planner.physical.PhysicalPlan; @@ -31,22 +32,27 @@ public class QueryService { /** * Execute the {@link UnresolvedPlan}, using {@link ResponseListener} to get response. + * Todo. deprecated this interface after finalize {@link PlanContext}. * * @param plan {@link UnresolvedPlan} * @param listener {@link ResponseListener} */ public void execute(UnresolvedPlan plan, ResponseListener listener) { - executePlan(analyze(plan), listener); + executePlan(analyze(plan), PlanContext.emptyPlanContext(), listener); } /** - * Todo. + * Execute the {@link UnresolvedPlan}, with {@link PlanContext} and using {@link ResponseListener} + * to get response. + * Todo. Pass split from PlanContext to ExecutionEngine in following PR. * * @param plan {@link LogicalPlan} + * @param planContext {@link PlanContext} * @param listener {@link ResponseListener} */ public void executePlan(LogicalPlan plan, + PlanContext planContext, ResponseListener listener) { try { executionEngine.execute(plan(plan), listener); @@ -72,14 +78,14 @@ public void explain(UnresolvedPlan plan, } /** - * Todo. + * Analyze {@link UnresolvedPlan}. */ public LogicalPlan analyze(UnresolvedPlan plan) { return analyzer.analyze(plan, new AnalysisContext()); } /** - * Todo. + * Translate {@link LogicalPlan} to {@link PhysicalPlan}. */ public PhysicalPlan plan(LogicalPlan plan) { return planner.plan(plan); diff --git a/core/src/main/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecution.java b/core/src/main/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecution.java index 190387676d..4f25b9433f 100644 --- a/core/src/main/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecution.java +++ b/core/src/main/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecution.java @@ -17,6 +17,7 @@ import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.QueryService; +import org.opensearch.sql.planner.PlanContext; import org.opensearch.sql.planner.logical.LogicalPlan; /** @@ -89,12 +90,11 @@ public void execute() { Optional availableOffsets = source.getLatestOffset(); if (hasNewData(availableOffsets, committedOffset)) { - // todo, add batch to execution context. Batch batch = source.getBatch(committedOffset, availableOffsets.get()); offsetLog.add(currentBatchId.get(), availableOffsets.get()); - queryService.executePlan( batchPlan, + new PlanContext(batch.getSplit()), new ResponseListener<>() { @Override public void onResponse(ExecutionEngine.QueryResponse response) { diff --git a/core/src/main/java/org/opensearch/sql/planner/PlanContext.java b/core/src/main/java/org/opensearch/sql/planner/PlanContext.java new file mode 100644 index 0000000000..3d43c02d61 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/PlanContext.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner; + +import java.util.Optional; +import lombok.Getter; +import org.opensearch.sql.storage.split.Split; + +/** + * Plan context hold planning related information. + */ +public class PlanContext { + + @Getter + private final Optional split; + + public PlanContext(Split split) { + this.split = Optional.of(split); + } + + private PlanContext(Optional split) { + this.split = split; + } + + public static PlanContext emptyPlanContext() { + return new PlanContext(Optional.empty()); + } +} diff --git a/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java b/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java index 2c70cf9f92..c65210e97e 100644 --- a/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java @@ -25,6 +25,7 @@ import org.opensearch.sql.analysis.Analyzer; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.planner.PlanContext; import org.opensearch.sql.planner.Planner; import org.opensearch.sql.planner.logical.LogicalPlan; import org.opensearch.sql.planner.physical.PhysicalPlan; @@ -57,6 +58,9 @@ class QueryServiceTest { @Mock private ExecutionEngine.Schema schema; + @Mock + private PlanContext planContext; + @BeforeEach public void setUp() { lenient().when(analyzer.analyze(any(), any())).thenReturn(logicalPlan); @@ -177,6 +181,7 @@ public void testExecutePlanShouldPass() { queryService.executePlan( logicalPlan, + planContext, new ResponseListener<>() { @Override public void onResponse(ExecutionEngine.QueryResponse pplQueryResponse) { diff --git a/core/src/test/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecutionTest.java b/core/src/test/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecutionTest.java index 7fa8f4d670..1a2b6e3f2a 100644 --- a/core/src/test/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecutionTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecutionTest.java @@ -7,12 +7,18 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.lenient; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; +import lombok.EqualsAndHashCode; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; @@ -20,7 +26,9 @@ import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.QueryService; +import org.opensearch.sql.planner.PlanContext; import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.storage.split.Split; @ExtendWith(MockitoExtension.class) class MicroBatchStreamingExecutionTest { @@ -29,29 +37,36 @@ class MicroBatchStreamingExecutionTest { void executedSuccess() { streamingQuery() .addData() - .executeSuccess() + .executeSuccess(0L) .latestOffsetLogShouldBe(0L) .latestCommittedLogShouldBe(0L); } @Test void executedFailed() { - streamingQuery().addData().executeFailed().latestOffsetLogShouldBe(0L).noCommittedLog(); + streamingQuery() + .addData() + .executeFailed() + .latestOffsetLogShouldBe(0L) + .noCommittedLog(); } @Test void noDataInSource() { - streamingQuery().executeSuccess().noOffsetLog().noCommittedLog(); + streamingQuery() + .neverProcess() + .noOffsetLog() + .noCommittedLog(); } @Test void noNewDataInSource() { streamingQuery() .addData() - .executeSuccess() + .executeSuccess(0L) .latestOffsetLogShouldBe(0L) .latestCommittedLogShouldBe(0L) - .executeSuccess() + .neverProcess() .latestOffsetLogShouldBe(0L) .latestCommittedLogShouldBe(0L); } @@ -60,11 +75,11 @@ void noNewDataInSource() { void addNewDataInSequenceAllExecuteSuccess() { streamingQuery() .addData() - .executeSuccess() + .executeSuccess(0L) .latestOffsetLogShouldBe(0L) .latestCommittedLogShouldBe(0L) .addData() - .executeSuccess() + .executeSuccess(1L) .latestOffsetLogShouldBe(1L) .latestCommittedLogShouldBe(1L); } @@ -73,14 +88,14 @@ void addNewDataInSequenceAllExecuteSuccess() { void addNewDataInSequenceExecuteFailedInBetween() { streamingQuery() .addData() - .executeSuccess() + .executeSuccess(0L) .latestOffsetLogShouldBe(0L) .latestCommittedLogShouldBe(0L) .addData() .executeFailed() .latestOffsetLogShouldBe(1L) .latestCommittedLogShouldBe(0L) - .executeSuccess() + .executeSuccess(1L) .latestOffsetLogShouldBe(1L) .latestCommittedLogShouldBe(1L); } @@ -89,7 +104,7 @@ void addNewDataInSequenceExecuteFailedInBetween() { void addNewDataInSequenceExecuteFailed() { streamingQuery() .addData() - .executeSuccess() + .executeSuccess(0L) .latestOffsetLogShouldBe(0L) .latestCommittedLogShouldBe(0L) .addData() @@ -134,18 +149,36 @@ Helper addData() { return this; } - Helper executeSuccess() { + Helper neverProcess() { + lenient() + .doAnswer( + invocation -> { + fail(); + return null; + }) + .when(queryService) + .executePlan(any(), any(), any()); + execution.execute(); + return this; + } + + Helper executeSuccess(Long... offsets) { lenient() .doAnswer( invocation -> { ResponseListener listener = - invocation.getArgument(1); + invocation.getArgument(2); listener.onResponse( new ExecutionEngine.QueryResponse(null, Collections.emptyList())); + + PlanContext planContext = invocation.getArgument(1); + assertTrue(planContext.getSplit().isPresent()); + assertEquals(new TestOffsetSplit(offsets), planContext.getSplit().get()); + return null; }) .when(queryService) - .executePlan(any(), any()); + .executePlan(any(), any(), any()); execution.execute(); return this; @@ -156,12 +189,13 @@ Helper executeFailed() { .doAnswer( invocation -> { ResponseListener listener = - invocation.getArgument(1); + invocation.getArgument(2); listener.onFailure(new RuntimeException()); + return null; }) .when(queryService) - .executePlan(any(), any()); + .executePlan(any(), any(), any()); execution.execute(); return this; @@ -219,7 +253,32 @@ public Optional getLatestOffset() { /** always return `empty` Batch regardless start and end offset. */ @Override public Batch getBatch(Optional start, Offset end) { - return new Batch(() -> "id"); + return new Batch( + new TestOffsetSplit( + start.map(v -> v.getOffset() + 1).orElse(0L), Long.min(offset.get(), + end.getOffset()))); + } + } + + @EqualsAndHashCode + static class TestOffsetSplit implements Split { + + private final List offsets; + + public TestOffsetSplit(Long start, Long end) { + this.offsets = new ArrayList<>(); + for (long l = start; l <= end; l++) { + this.offsets.add(l); + } + } + + public TestOffsetSplit(Long... offsets) { + this.offsets = Arrays.stream(offsets).collect(Collectors.toList()); + } + + @Override + public String getSplitId() { + return "id"; } } } diff --git a/core/src/test/java/org/opensearch/sql/planner/PlanContextTest.java b/core/src/test/java/org/opensearch/sql/planner/PlanContextTest.java new file mode 100644 index 0000000000..77ae78f77e --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/planner/PlanContextTest.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.storage.split.Split; + +@ExtendWith(MockitoExtension.class) +class PlanContextTest { + + @Mock + private Split split; + + @Test + void createEmptyPlanContext() { + assertTrue(PlanContext.emptyPlanContext().getSplit().isEmpty()); + } + + @Test + void createPlanContextWithSplit() { + assertTrue(new PlanContext(split).getSplit().isPresent()); + } +} From 5b99c082e2a97af744b397f9538b37c9c792881b Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Wed, 9 Nov 2022 10:43:36 -0800 Subject: [PATCH 21/23] bump jackson version Signed-off-by: Peng Huo --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index b25c36f687..bbfc122005 100644 --- a/build.gradle +++ b/build.gradle @@ -8,8 +8,8 @@ buildscript { ext { opensearch_version = System.getProperty("opensearch.version", "2.4.0-SNAPSHOT") spring_version = "5.3.22" - jackson_version = "2.13.4" - jackson_databind_version = "2.13.4.2" + jackson_version = "2.14.0" + jackson_databind_version = "2.14.0" isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") version_tokens = opensearch_version.tokenize('-') From 3c6b37ae15e293abcd54660631aa69aa43523c31 Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Wed, 9 Nov 2022 13:23:19 -0800 Subject: [PATCH 22/23] Merge 2.x to Maximus feature branch (#1062) * Fix `FLOAT` -> `DOUBLE` cast. (#1025) Signed-off-by: Yury-Fridlyand * Fix error messaging from prometheus. (#1029) (#1037) Signed-off-by: vamsi-amazon (cherry picked from commit 4a9cef32ac6aa565d1e92faf72a472e727b205e6) Co-authored-by: vamsi-amazon * Add `query` function as alternate syntax to `query_string` function (#1010) This maintains backwards compatibility with the v1 engine. * Update DATE and TIME functions to parse string input as datetime (#991) Add option to accept datetime like string in both TIME and DATE (eg. accept "1999-01-02 12:12:12" for both TIME and DATE. Strict check on date for testing for valid dates (eg. Don't accept Feb 30th as a valid date) and throws a SemanticCheckException. Co-authored-by: Yury-Fridlyand Signed-off-by: MitchellGale-BitQuill * back quote fix (#1041) (#1050) Signed-off-by: vamsi-amazon (cherry picked from commit d3bb902f66b1232dd8565890452c58bd9ca4d529) Co-authored-by: vamsi-amazon * Catalog to Datasource changes (#1027) (#1049) Signed-off-by: vamsi-amazon (cherry picked from commit 3e30379903ba596b8565e680af69073a16cdf940) * Bump jackson to 2.14.0 (#1058) Signed-off-by: Joshua Li (cherry picked from commit 5a1adb236f5589dfd10aa17783f13684b70a3c51) * Add valueOf() to Expression (#1055) Signed-off-by: Joshua Li Signed-off-by: Yury-Fridlyand Signed-off-by: MitchellGale-BitQuill Signed-off-by: Joshua Li Signed-off-by: Chen Dai Co-authored-by: Yury-Fridlyand Co-authored-by: opensearch-trigger-bot[bot] <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Co-authored-by: vamsi-amazon Co-authored-by: GabeFernandez310 <64448015+GabeFernandez310@users.noreply.github.com> Co-authored-by: MitchellGale-BitQuill <104795536+MitchellGale-BitQuill@users.noreply.github.com> Co-authored-by: Joshua Li --- build.gradle | 4 +- .../sql/analysis/ExpressionAnalyzer.java | 2 +- .../sql/data/model/ExprDateValue.java | 4 +- .../sql/data/model/ExprTimeValue.java | 16 +- .../org/opensearch/sql/expression/DSL.java | 6 +- .../opensearch/sql/expression/Expression.java | 7 + .../aggregation/TakeAggregator.java | 2 +- .../expression/datetime/DateTimeFunction.java | 1 + .../function/OpenSearchFunctions.java | 6 + .../operator/convert/TypeCastOperator.java | 2 +- .../sql/expression/parse/GrokExpression.java | 2 +- .../sql/expression/parse/ParseExpression.java | 2 +- .../expression/parse/PatternsExpression.java | 2 +- .../sql/expression/parse/RegexExpression.java | 2 +- .../sql/planner/physical/ValuesOperator.java | 2 +- .../physical/catalog/CatalogTable.java | 2 +- .../physical/catalog/CatalogTableScan.java | 2 +- .../physical/catalog/CatalogTableSchema.java | 2 +- .../planner/physical/collector/Rounding.java | 2 +- .../sql/utils/DateTimeFormatters.java | 12 +- .../sql/analysis/ExpressionAnalyzerTest.java | 19 +- .../sql/expression/NamedExpressionTest.java | 2 +- .../datetime/DateTimeFunctionTest.java | 37 ++++ .../expression/datetime/DateTimeTestBase.java | 26 +-- .../expression/datetime/FromUnixTimeTest.java | 36 ++-- .../datetime/NowLikeFunctionTest.java | 8 +- .../datetime/PeriodFunctionsTest.java | 6 +- .../datetime/UnixTimeStampTest.java | 2 +- .../function/OpenSearchFunctionsTest.java | 7 + .../arthmetic/ArithmeticFunctionTest.java | 10 +- .../convert/TypeCastOperatorTest.java | 98 ++++----- .../system/SystemFunctionsTest.java | 2 +- .../catalog/CatalogTableScanTest.java | 2 +- .../physical/catalog/CatalogTableTest.java | 2 +- docs/category.json | 2 +- docs/user/dql/functions.rst | 85 ++++++-- docs/user/general/identifiers.rst | 30 +-- docs/user/ppl/admin/catalog.rst | 83 -------- docs/user/ppl/admin/datasources.rst | 83 ++++++++ docs/user/ppl/cmd/information_schema.rst | 14 +- docs/user/ppl/cmd/showcatalogs.rst | 36 ---- docs/user/ppl/cmd/showdatasources.rst | 36 ++++ docs/user/ppl/functions/datetime.rst | 73 +++++-- docs/user/ppl/index.rst | 4 +- doctest/build.gradle | 2 +- integ-test/build.gradle | 2 +- .../opensearch/sql/legacy/MethodQueryIT.java | 2 +- .../sql/ppl/PrometheusCatalogCommandsIT.java | 12 +- .../sql/ppl/ShowCatalogsCommandIT.java | 8 +- .../java/org/opensearch/sql/sql/QueryIT.java | 84 ++++++++ .../dsl/BucketAggregationBuilder.java | 2 +- .../dsl/MetricAggregationBuilder.java | 2 +- .../script/filter/FilterQueryBuilder.java | 3 +- .../script/filter/lucene/LuceneQuery.java | 64 +++--- .../lucene/relevance/MultiFieldQuery.java | 4 +- .../filter/lucene/relevance/NoFieldQuery.java | 72 +++++++ .../filter/lucene/relevance/QueryQuery.java | 40 ++++ .../lucene/relevance/RelevanceQuery.java | 53 +++-- .../lucene/relevance/SingleFieldQuery.java | 4 +- .../OpenSearchDataTypeRecognitionTest.java | 2 +- .../script/filter/FilterQueryBuilderTest.java | 188 +++++++++++++++--- .../script/filter/lucene/QueryTest.java | 150 ++++++++++++++ .../lucene/relevance/NoFieldQueryTest.java | 49 +++++ .../sql/plugin/catalog/CatalogSettings.java | 2 +- .../catalog/CatalogServiceImplTest.java | 2 +- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 +- .../sql/ppl/parser/AstBuilderTest.java | 2 +- .../client/PrometheusClientImpl.java | 3 +- .../QueryRangeFunctionImplementation.java | 2 +- .../response/PrometheusResponse.java | 20 +- .../PrometheusDefaultImplementor.java | 8 +- .../model/PrometheusResponseFieldNames.java | 3 + .../querybuilder/AggregationQueryBuilder.java | 8 +- .../TimeRangeParametersResolver.java | 2 +- .../QueryRangeFunctionImplementationTest.java | 2 +- .../storage/PrometheusMetricScanTest.java | 47 +++++ .../storage/PrometheusMetricTableTest.java | 27 +++ sql/src/main/antlr/OpenSearchSQLParser.g4 | 10 +- .../sql/sql/parser/AstExpressionBuilder.java | 36 +++- .../sql/sql/antlr/SQLSyntaxParserTest.java | 43 ++++ .../sql/parser/AstExpressionBuilderTest.java | 16 ++ 81 files changed, 1347 insertions(+), 414 deletions(-) delete mode 100644 docs/user/ppl/admin/catalog.rst create mode 100644 docs/user/ppl/admin/datasources.rst delete mode 100644 docs/user/ppl/cmd/showcatalogs.rst create mode 100644 docs/user/ppl/cmd/showdatasources.rst create mode 100644 integ-test/src/test/java/org/opensearch/sql/sql/QueryIT.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/NoFieldQuery.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/QueryQuery.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/QueryTest.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/NoFieldQueryTest.java diff --git a/build.gradle b/build.gradle index b25c36f687..bbfc122005 100644 --- a/build.gradle +++ b/build.gradle @@ -8,8 +8,8 @@ buildscript { ext { opensearch_version = System.getProperty("opensearch.version", "2.4.0-SNAPSHOT") spring_version = "5.3.22" - jackson_version = "2.13.4" - jackson_databind_version = "2.13.4.2" + jackson_version = "2.14.0" + jackson_databind_version = "2.14.0" isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") version_tokens = opensearch_version.tokenize('-') diff --git a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java index 061c4b505f..9885302a69 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java @@ -184,7 +184,7 @@ public Expression visitConstantFunction(ConstantFunction node, AnalysisContext c } var value = visitFunction(node, context); - value = DSL.literal(value.valueOf(null)); + value = DSL.literal(value.valueOf()); context.getConstantFunctionValues().put(valueName, value); return value; } diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java index 09b2e56b44..7617e156ba 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java @@ -6,6 +6,8 @@ package org.opensearch.sql.data.model; +import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL; + import com.google.common.base.Objects; import java.time.Instant; import java.time.LocalDate; @@ -33,7 +35,7 @@ public class ExprDateValue extends AbstractExprValue { */ public ExprDateValue(String date) { try { - this.date = LocalDate.parse(date); + this.date = LocalDate.parse(date, DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL); } catch (DateTimeParseException e) { throw new SemanticCheckException(String.format("date:%s in unsupported format, please use " + "yyyy-MM-dd", date)); diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java index 9c5f5c5d55..6cc4021d2e 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java @@ -6,10 +6,15 @@ package org.opensearch.sql.data.model; -import static org.opensearch.sql.utils.DateTimeFormatters.TIME_FORMATTER_VARIABLE_NANOS; +import static java.time.format.DateTimeFormatter.ISO_LOCAL_TIME; +import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; import java.time.LocalTime; -import java.time.format.DateTimeFormatter; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.time.format.DateTimeParseException; import java.util.Objects; import lombok.RequiredArgsConstructor; @@ -22,14 +27,15 @@ */ @RequiredArgsConstructor public class ExprTimeValue extends AbstractExprValue { + private final LocalTime time; /** - * Constructor. + * Constructor of ExprTimeValue. */ public ExprTimeValue(String time) { try { - this.time = LocalTime.parse(time, TIME_FORMATTER_VARIABLE_NANOS); + this.time = LocalTime.parse(time, DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL); } catch (DateTimeParseException e) { throw new SemanticCheckException(String.format("time:%s in unsupported format, please use " + "HH:mm:ss[.SSSSSSSSS]", time)); @@ -38,7 +44,7 @@ public ExprTimeValue(String time) { @Override public String value() { - return DateTimeFormatter.ISO_LOCAL_TIME.format(time); + return ISO_LOCAL_TIME.format(time); } @Override diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index 961608d104..e7c94bea7c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -102,7 +102,7 @@ public static NamedExpression named(Expression expression) { return (NamedExpression) expression; } if (expression instanceof ParseExpression) { - return named(((ParseExpression) expression).getIdentifier().valueOf(null).stringValue(), + return named(((ParseExpression) expression).getIdentifier().valueOf().stringValue(), expression); } return named(expression.toString(), expression); @@ -707,6 +707,10 @@ public FunctionExpression simple_query_string(Expression... args) { return compile(BuiltinFunctionName.SIMPLE_QUERY_STRING, args); } + public FunctionExpression query(Expression... args) { + return compile(BuiltinFunctionName.QUERY, args); + } + public FunctionExpression query_string(Expression... args) { return compile(BuiltinFunctionName.QUERY_STRING, args); } diff --git a/core/src/main/java/org/opensearch/sql/expression/Expression.java b/core/src/main/java/org/opensearch/sql/expression/Expression.java index e5efed200a..25a8173efa 100644 --- a/core/src/main/java/org/opensearch/sql/expression/Expression.java +++ b/core/src/main/java/org/opensearch/sql/expression/Expression.java @@ -16,6 +16,13 @@ */ public interface Expression extends Serializable { + /** + * Evaluate the value of expression that does not depend on value environment. + */ + default ExprValue valueOf() { + return valueOf(null); + } + /** * Evaluate the value of expression in the value environment. */ diff --git a/core/src/main/java/org/opensearch/sql/expression/aggregation/TakeAggregator.java b/core/src/main/java/org/opensearch/sql/expression/aggregation/TakeAggregator.java index 4ac0991979..cff08bb098 100644 --- a/core/src/main/java/org/opensearch/sql/expression/aggregation/TakeAggregator.java +++ b/core/src/main/java/org/opensearch/sql/expression/aggregation/TakeAggregator.java @@ -29,7 +29,7 @@ public TakeAggregator(List arguments, ExprCoreType returnType) { @Override public TakeState create() { - return new TakeState(getArguments().get(1).valueOf(null).integerValue()); + return new TakeState(getArguments().get(1).valueOf().integerValue()); } @Override diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index 43f5234d31..42274c0756 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -56,6 +56,7 @@ import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.exception.ExpressionEvaluationException; +import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java b/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java index 43a722b838..97afe3675e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java @@ -30,6 +30,7 @@ public void register(BuiltinFunctionRepository repository) { repository.register(match()); repository.register(multi_match()); repository.register(simple_query_string()); + repository.register(query()); repository.register(query_string()); // Register MATCHPHRASE as MATCH_PHRASE as well for backwards // compatibility. @@ -68,6 +69,11 @@ private static FunctionResolver simple_query_string() { return new RelevanceFunctionResolver(funcName, STRUCT); } + private static FunctionResolver query() { + FunctionName funcName = BuiltinFunctionName.QUERY.getName(); + return new RelevanceFunctionResolver(funcName, STRING); + } + private static FunctionResolver query_string() { FunctionName funcName = BuiltinFunctionName.QUERY_STRING.getName(); return new RelevanceFunctionResolver(funcName, STRUCT); diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperator.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperator.java index 23508406ac..8f904bfbf7 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperator.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperator.java @@ -125,7 +125,7 @@ private static DefaultFunctionResolver castToFloat() { impl(nullMissingHandling( (v) -> new ExprFloatValue(Float.valueOf(v.stringValue()))), FLOAT, STRING), impl(nullMissingHandling( - (v) -> new ExprFloatValue(v.longValue())), FLOAT, DOUBLE), + (v) -> new ExprFloatValue(v.floatValue())), FLOAT, DOUBLE), impl(nullMissingHandling( (v) -> new ExprFloatValue(v.booleanValue() ? 1f : 0f)), FLOAT, BOOLEAN) ); diff --git a/core/src/main/java/org/opensearch/sql/expression/parse/GrokExpression.java b/core/src/main/java/org/opensearch/sql/expression/parse/GrokExpression.java index 12c803dcbc..9797832f07 100644 --- a/core/src/main/java/org/opensearch/sql/expression/parse/GrokExpression.java +++ b/core/src/main/java/org/opensearch/sql/expression/parse/GrokExpression.java @@ -45,7 +45,7 @@ public class GrokExpression extends ParseExpression { */ public GrokExpression(Expression sourceField, Expression pattern, Expression identifier) { super("grok", sourceField, pattern, identifier); - this.grok = grokCompiler.compile(pattern.valueOf(null).stringValue()); + this.grok = grokCompiler.compile(pattern.valueOf().stringValue()); } @Override diff --git a/core/src/main/java/org/opensearch/sql/expression/parse/ParseExpression.java b/core/src/main/java/org/opensearch/sql/expression/parse/ParseExpression.java index 822651be20..d99242bdde 100644 --- a/core/src/main/java/org/opensearch/sql/expression/parse/ParseExpression.java +++ b/core/src/main/java/org/opensearch/sql/expression/parse/ParseExpression.java @@ -48,7 +48,7 @@ public ParseExpression(String functionName, Expression sourceField, Expression p this.sourceField = sourceField; this.pattern = pattern; this.identifier = identifier; - this.identifierStr = identifier.valueOf(null).stringValue(); + this.identifierStr = identifier.valueOf().stringValue(); } @Override diff --git a/core/src/main/java/org/opensearch/sql/expression/parse/PatternsExpression.java b/core/src/main/java/org/opensearch/sql/expression/parse/PatternsExpression.java index c60588c910..67160dad58 100644 --- a/core/src/main/java/org/opensearch/sql/expression/parse/PatternsExpression.java +++ b/core/src/main/java/org/opensearch/sql/expression/parse/PatternsExpression.java @@ -44,7 +44,7 @@ public class PatternsExpression extends ParseExpression { */ public PatternsExpression(Expression sourceField, Expression pattern, Expression identifier) { super("patterns", sourceField, pattern, identifier); - String patternStr = pattern.valueOf(null).stringValue(); + String patternStr = pattern.valueOf().stringValue(); useCustomPattern = !patternStr.isEmpty(); if (useCustomPattern) { this.pattern = Pattern.compile(patternStr); diff --git a/core/src/main/java/org/opensearch/sql/expression/parse/RegexExpression.java b/core/src/main/java/org/opensearch/sql/expression/parse/RegexExpression.java index 850f9f8d08..f3a3ff0b66 100644 --- a/core/src/main/java/org/opensearch/sql/expression/parse/RegexExpression.java +++ b/core/src/main/java/org/opensearch/sql/expression/parse/RegexExpression.java @@ -40,7 +40,7 @@ public class RegexExpression extends ParseExpression { */ public RegexExpression(Expression sourceField, Expression pattern, Expression identifier) { super("regex", sourceField, pattern, identifier); - this.regexPattern = Pattern.compile(pattern.valueOf(null).stringValue()); + this.regexPattern = Pattern.compile(pattern.valueOf().stringValue()); } @Override diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/ValuesOperator.java b/core/src/main/java/org/opensearch/sql/planner/physical/ValuesOperator.java index d0aeceecba..51d2850df7 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/ValuesOperator.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/ValuesOperator.java @@ -58,7 +58,7 @@ public boolean hasNext() { @Override public ExprValue next() { List values = valuesIterator.next().stream() - .map(expr -> expr.valueOf(null)) + .map(expr -> expr.valueOf()) .collect(Collectors.toList()); return new ExprCollectionValue(values); } diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTable.java b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTable.java index 26c9c3d04f..4e6a87e21b 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTable.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTable.java @@ -21,7 +21,7 @@ /** - * Table implementation to handle show catalogs command. + * Table implementation to handle show datasources command. * Since catalog information is not tied to any storage engine, this info * is handled via Catalog Table. * diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScan.java b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScan.java index 894ff9f216..efc59c97ec 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScan.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScan.java @@ -51,7 +51,7 @@ public void open() { for (Catalog catalog : catalogs) { exprValues.add( new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( - "CATALOG_NAME", ExprValueUtils.stringValue(catalog.getName()), + "DATASOURCE_NAME", ExprValueUtils.stringValue(catalog.getName()), "CONNECTOR_TYPE", ExprValueUtils.stringValue(catalog.getConnectorType().name()))))); } iterator = exprValues.iterator(); diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableSchema.java b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableSchema.java index 35fd63f098..b360eb87db 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableSchema.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableSchema.java @@ -22,7 +22,7 @@ public enum CatalogTableSchema { CATALOG_TABLE_SCHEMA(new LinkedHashMap<>() { { - put("CATALOG_NAME", STRING); + put("DATASOURCE_NAME", STRING); put("CONNECTOR_TYPE", STRING); } } diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java b/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java index 4eaafea59b..71ae5dc7c3 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java @@ -44,7 +44,7 @@ public abstract class Rounding { * Create Rounding instance. */ public static Rounding createRounding(SpanExpression span) { - ExprValue interval = span.getValue().valueOf(null); + ExprValue interval = span.getValue().valueOf(); ExprType type = span.type(); if (LONG.isCompatible(type)) { diff --git a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java index 2efdbb3755..2556aed8d8 100644 --- a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java +++ b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java @@ -113,23 +113,25 @@ public class DateTimeFormatters { public static final DateTimeFormatter DATE_TIME_FORMATTER_VARIABLE_NANOS = new DateTimeFormatterBuilder() - .appendPattern("yyyy-MM-dd HH:mm:ss") + .appendPattern("uuuu-MM-dd HH:mm:ss") .appendFraction( ChronoField.NANO_OF_SECOND, MIN_FRACTION_SECONDS, MAX_FRACTION_SECONDS, true) - .toFormatter(Locale.ROOT); + .toFormatter(Locale.ROOT) + .withResolverStyle(ResolverStyle.STRICT); - public static final DateTimeFormatter TIME_FORMATTER_VARIABLE_NANOS = + public static final DateTimeFormatter DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL = new DateTimeFormatterBuilder() - .appendPattern("HH:mm:ss") + .appendPattern("[uuuu-MM-dd HH:mm:ss][uuuu-MM-dd HH:mm][HH:mm:ss][HH:mm][uuuu-MM-dd]") .appendFraction( ChronoField.NANO_OF_SECOND, MIN_FRACTION_SECONDS, MAX_FRACTION_SECONDS, true) - .toFormatter(); + .toFormatter(Locale.ROOT) + .withResolverStyle(ResolverStyle.STRICT); // YYMMDD public static final DateTimeFormatter DATE_FORMATTER_SHORT_YEAR = diff --git a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java index d1cef1edb3..84c5005eff 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java @@ -487,6 +487,15 @@ void simple_query_string_expression_two_fields() { AstDSL.unresolvedArg("query", stringLiteral("sample query")))); } + @Test + void query_expression() { + assertAnalyzeEqual( + dsl.query( + dsl.namedArgument("query", DSL.literal("field:query"))), + AstDSL.function("query", + AstDSL.unresolvedArg("query", stringLiteral("field:query")))); + } + @Test void query_string_expression() { assertAnalyzeEqual( @@ -574,7 +583,7 @@ public void constant_function_returns_constant_cached_value() { var values = List.of(analyze(AstDSL.constantFunction("now")), analyze(AstDSL.constantFunction("now")), analyze(AstDSL.constantFunction("now"))); assertTrue(values.stream().allMatch(v -> - v.valueOf(null) == analyze(AstDSL.constantFunction("now")).valueOf(null))); + v.valueOf() == analyze(AstDSL.constantFunction("now")).valueOf())); } @Test @@ -584,8 +593,8 @@ public void function_returns_non_constant_value() { // different values var values = List.of(analyze(function("sysdate")), analyze(function("sysdate")), analyze(function("sysdate")), analyze(function("sysdate"))); - var referenceValue = analyze(function("sysdate")).valueOf(null); - assertTrue(values.stream().noneMatch(v -> v.valueOf(null) == referenceValue)); + var referenceValue = analyze(function("sysdate")).valueOf(); + assertTrue(values.stream().noneMatch(v -> v.valueOf() == referenceValue)); } @Test @@ -593,8 +602,8 @@ public void now_as_a_function_not_cached() { // // We can call `now()` as a function, in that case nothing should be cached var values = List.of(analyze(function("now")), analyze(function("now")), analyze(function("now")), analyze(function("now"))); - var referenceValue = analyze(function("now")).valueOf(null); - assertTrue(values.stream().noneMatch(v -> v.valueOf(null) == referenceValue)); + var referenceValue = analyze(function("now")).valueOf(); + assertTrue(values.stream().noneMatch(v -> v.valueOf() == referenceValue)); } protected Expression analyze(UnresolvedExpression unresolvedExpression) { diff --git a/core/src/test/java/org/opensearch/sql/expression/NamedExpressionTest.java b/core/src/test/java/org/opensearch/sql/expression/NamedExpressionTest.java index 7dc56fdf89..38f1ce3ca9 100644 --- a/core/src/test/java/org/opensearch/sql/expression/NamedExpressionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/NamedExpressionTest.java @@ -59,7 +59,7 @@ void name_a_parse_expression() { DSL.literal("group")); NamedExpression named = DSL.named(parse); assertEquals(parse, named.getDelegated()); - assertEquals(parse.getIdentifier().valueOf(null).stringValue(), named.getName()); + assertEquals(parse.getIdentifier().valueOf().stringValue(), named.getName()); } } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index 79efa2a015..29bfa3d91b 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -24,6 +24,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import com.google.common.collect.ImmutableList; +import java.time.LocalDate; import java.util.List; import lombok.AllArgsConstructor; import org.junit.jupiter.api.BeforeEach; @@ -240,6 +241,18 @@ public void date() { assertEquals(DATE, expr.type()); assertEquals(new ExprDateValue("2020-08-17"), eval(expr)); assertEquals("date(DATE '2020-08-17')", expr.toString()); + + expr = dsl.date(DSL.literal(new ExprDateValue("2020-08-17 12:12:00"))); + assertEquals(DATE, expr.type()); + assertEquals(new ExprDateValue("2020-08-17 12:12:00"), eval(expr)); + assertEquals("date(DATE '2020-08-17')", expr.toString()); + + expr = dsl.date(DSL.literal(new ExprDateValue("2020-08-17 12:12"))); + assertEquals(DATE, expr.type()); + assertEquals(new ExprDateValue("2020-08-17 12:12"), eval(expr)); + assertEquals("date(DATE '2020-08-17')", expr.toString()); + + } @Test @@ -795,6 +808,30 @@ public void time() { assertEquals(TIME, expr.type()); assertEquals(new ExprTimeValue("01:01:01"), eval(expr)); assertEquals("time(TIME '01:01:01')", expr.toString()); + + expr = dsl.time(DSL.literal(new ExprTimeValue("01:01"))); + assertEquals(TIME, expr.type()); + assertEquals(new ExprTimeValue("01:01"), eval(expr)); + assertEquals("time(TIME '01:01:00')", expr.toString()); + + expr = dsl.time(DSL.literal(new ExprTimeValue("2019-04-19 01:01:01"))); + assertEquals(TIME, expr.type()); + assertEquals(new ExprTimeValue("2019-04-19 01:01:01"), eval(expr)); + assertEquals("time(TIME '01:01:01')", expr.toString()); + + expr = dsl.time(DSL.literal(new ExprTimeValue("2019-04-19 01:01"))); + assertEquals(TIME, expr.type()); + assertEquals(new ExprTimeValue("2019-04-19 01:01"), eval(expr)); + assertEquals("time(TIME '01:01:00')", expr.toString()); + + expr = dsl.time(DSL.literal(new ExprTimeValue("01:01:01.0123"))); + assertEquals(TIME, expr.type()); + assertEquals(new ExprTimeValue("01:01:01.0123"), eval(expr)); + assertEquals("time(TIME '01:01:01.0123')", expr.toString()); + + expr = dsl.time(dsl.date(DSL.literal("2020-01-02"))); + assertEquals(TIME, expr.type()); + assertEquals(new ExprTimeValue("00:00:00"), expr.valueOf()); } @Test diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java index 91d1fc4b0f..bee16cb92a 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java @@ -66,19 +66,19 @@ protected FunctionExpression fromUnixTime(Expression value, Expression format) { } protected LocalDateTime fromUnixTime(Long value) { - return fromUnixTime(DSL.literal(value)).valueOf(null).datetimeValue(); + return fromUnixTime(DSL.literal(value)).valueOf().datetimeValue(); } protected LocalDateTime fromUnixTime(Double value) { - return fromUnixTime(DSL.literal(value)).valueOf(null).datetimeValue(); + return fromUnixTime(DSL.literal(value)).valueOf().datetimeValue(); } protected String fromUnixTime(Long value, String format) { - return fromUnixTime(DSL.literal(value), DSL.literal(format)).valueOf(null).stringValue(); + return fromUnixTime(DSL.literal(value), DSL.literal(format)).valueOf().stringValue(); } protected String fromUnixTime(Double value, String format) { - return fromUnixTime(DSL.literal(value), DSL.literal(format)).valueOf(null).stringValue(); + return fromUnixTime(DSL.literal(value), DSL.literal(format)).valueOf().stringValue(); } protected FunctionExpression makedate(Expression year, Expression dayOfYear) { @@ -89,7 +89,7 @@ protected FunctionExpression makedate(Expression year, Expression dayOfYear) { } protected LocalDate makedate(Double year, Double dayOfYear) { - return makedate(DSL.literal(year), DSL.literal(dayOfYear)).valueOf(null).dateValue(); + return makedate(DSL.literal(year), DSL.literal(dayOfYear)).valueOf().dateValue(); } protected FunctionExpression maketime(Expression hour, Expression minute, Expression second) { @@ -101,7 +101,7 @@ protected FunctionExpression maketime(Expression hour, Expression minute, Expres protected LocalTime maketime(Double hour, Double minute, Double second) { return maketime(DSL.literal(hour), DSL.literal(minute), DSL.literal(second)) - .valueOf(null).timeValue(); + .valueOf().timeValue(); } protected FunctionExpression period_add(Expression period, Expression months) { @@ -112,7 +112,7 @@ protected FunctionExpression period_add(Expression period, Expression months) { } protected Integer period_add(Integer period, Integer months) { - return period_add(DSL.literal(period), DSL.literal(months)).valueOf(null).integerValue(); + return period_add(DSL.literal(period), DSL.literal(months)).valueOf().integerValue(); } protected FunctionExpression period_diff(Expression first, Expression second) { @@ -123,7 +123,7 @@ protected FunctionExpression period_diff(Expression first, Expression second) { } protected Integer period_diff(Integer first, Integer second) { - return period_diff(DSL.literal(first), DSL.literal(second)).valueOf(null).integerValue(); + return period_diff(DSL.literal(first), DSL.literal(second)).valueOf().integerValue(); } protected FunctionExpression unixTimeStampExpr() { @@ -133,7 +133,7 @@ protected FunctionExpression unixTimeStampExpr() { } protected Long unixTimeStamp() { - return unixTimeStampExpr().valueOf(null).longValue(); + return unixTimeStampExpr().valueOf().longValue(); } protected FunctionExpression unixTimeStampOf(Expression value) { @@ -144,18 +144,18 @@ protected FunctionExpression unixTimeStampOf(Expression value) { } protected Double unixTimeStampOf(Double value) { - return unixTimeStampOf(DSL.literal(value)).valueOf(null).doubleValue(); + return unixTimeStampOf(DSL.literal(value)).valueOf().doubleValue(); } protected Double unixTimeStampOf(LocalDate value) { - return unixTimeStampOf(DSL.literal(new ExprDateValue(value))).valueOf(null).doubleValue(); + return unixTimeStampOf(DSL.literal(new ExprDateValue(value))).valueOf().doubleValue(); } protected Double unixTimeStampOf(LocalDateTime value) { - return unixTimeStampOf(DSL.literal(new ExprDatetimeValue(value))).valueOf(null).doubleValue(); + return unixTimeStampOf(DSL.literal(new ExprDatetimeValue(value))).valueOf().doubleValue(); } protected Double unixTimeStampOf(Instant value) { - return unixTimeStampOf(DSL.literal(new ExprTimestampValue(value))).valueOf(null).doubleValue(); + return unixTimeStampOf(DSL.literal(new ExprTimestampValue(value))).valueOf().doubleValue(); } } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java index b6d04cbc97..58387ef04f 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java @@ -132,54 +132,54 @@ public void checkOfDoubleWithFormat(Double value, String format, String expected @Test public void checkInvalidFormat() { assertEquals(new ExprStringValue("q"), - fromUnixTime(DSL.literal(0L), DSL.literal("%q")).valueOf(null)); + fromUnixTime(DSL.literal(0L), DSL.literal("%q")).valueOf()); assertEquals(new ExprStringValue(""), - fromUnixTime(DSL.literal(0L), DSL.literal("")).valueOf(null)); + fromUnixTime(DSL.literal(0L), DSL.literal("")).valueOf()); } @Test public void checkValueOutsideOfTheRangeWithoutFormat() { - assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(-1L)).valueOf(null)); - assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(-1.5d)).valueOf(null)); - assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771200L)).valueOf(null)); - assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771200d)).valueOf(null)); + assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(-1L)).valueOf()); + assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(-1.5d)).valueOf()); + assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771200L)).valueOf()); + assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771200d)).valueOf()); } @Test public void checkInsideTheRangeWithoutFormat() { - assertNotEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771199L)).valueOf(null)); - assertNotEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771199d)).valueOf(null)); + assertNotEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771199L)).valueOf()); + assertNotEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771199d)).valueOf()); } @Test public void checkValueOutsideOfTheRangeWithFormat() { assertEquals(ExprNullValue.of(), - fromUnixTime(DSL.literal(32536771200L), DSL.literal("%d")).valueOf(null)); + fromUnixTime(DSL.literal(32536771200L), DSL.literal("%d")).valueOf()); assertEquals(ExprNullValue.of(), - fromUnixTime(DSL.literal(32536771200d), DSL.literal("%d")).valueOf(null)); + fromUnixTime(DSL.literal(32536771200d), DSL.literal("%d")).valueOf()); } @Test public void checkInsideTheRangeWithFormat() { assertNotEquals(ExprNullValue.of(), - fromUnixTime(DSL.literal(32536771199L), DSL.literal("%d")).valueOf(null)); + fromUnixTime(DSL.literal(32536771199L), DSL.literal("%d")).valueOf()); assertNotEquals(ExprNullValue.of(), - fromUnixTime(DSL.literal(32536771199d), DSL.literal("%d")).valueOf(null)); + fromUnixTime(DSL.literal(32536771199d), DSL.literal("%d")).valueOf()); } @Test public void checkNullOrNegativeValues() { - assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(ExprNullValue.of())).valueOf(null)); + assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(ExprNullValue.of())).valueOf()); assertEquals(ExprNullValue.of(), - fromUnixTime(DSL.literal(-1L), DSL.literal("%d")).valueOf(null)); + fromUnixTime(DSL.literal(-1L), DSL.literal("%d")).valueOf()); assertEquals(ExprNullValue.of(), - fromUnixTime(DSL.literal(-1.5d), DSL.literal("%d")).valueOf(null)); + fromUnixTime(DSL.literal(-1.5d), DSL.literal("%d")).valueOf()); assertEquals(ExprNullValue.of(), - fromUnixTime(DSL.literal(42L), DSL.literal(ExprNullValue.of())).valueOf(null)); + fromUnixTime(DSL.literal(42L), DSL.literal(ExprNullValue.of())).valueOf()); assertEquals(ExprNullValue.of(), - fromUnixTime(DSL.literal(ExprNullValue.of()), DSL.literal("%d")).valueOf(null)); + fromUnixTime(DSL.literal(ExprNullValue.of()), DSL.literal("%d")).valueOf()); assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(ExprNullValue.of()), DSL.literal(ExprNullValue.of())) - .valueOf(null)); + .valueOf()); } } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/NowLikeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/NowLikeFunctionTest.java index e8f5c16025..b6b82fe8fb 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/NowLikeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/NowLikeFunctionTest.java @@ -60,9 +60,9 @@ private static Stream functionNames() { private Temporal extractValue(FunctionExpression func) { switch ((ExprCoreType)func.type()) { - case DATE: return func.valueOf(null).dateValue(); - case DATETIME: return func.valueOf(null).datetimeValue(); - case TIME: return func.valueOf(null).timeValue(); + case DATE: return func.valueOf().dateValue(); + case DATETIME: return func.valueOf().datetimeValue(); + case TIME: return func.valueOf().timeValue(); // unreachable code default: throw new IllegalArgumentException(String.format("%s", func.type())); } @@ -105,7 +105,7 @@ public void test_now_like_functions(Function f for (var wrongFspValue: List.of(-1, 10)) { var exception = assertThrows(IllegalArgumentException.class, - () -> function.apply(new Expression[]{DSL.literal(wrongFspValue)}).valueOf(null)); + () -> function.apply(new Expression[]{DSL.literal(wrongFspValue)}).valueOf()); assertEquals(String.format("Invalid `fsp` value: %d, allowed 0 to 6", wrongFspValue), exception.getMessage()); } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/PeriodFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/PeriodFunctionsTest.java index ff63cb6f0f..bf228dd509 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/PeriodFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/PeriodFunctionsTest.java @@ -96,8 +96,8 @@ public static Stream getInvalidTestData() { @ParameterizedTest @MethodSource("getInvalidTestData") public void period_add_returns_null_on_invalid_input(int period) { - assertNull(period_add(DSL.literal(period), DSL.literal(1)).valueOf(null).value()); - assertNull(period_diff(DSL.literal(period), DSL.literal(1)).valueOf(null).value()); - assertNull(period_diff(DSL.literal(1), DSL.literal(period)).valueOf(null).value()); + assertNull(period_add(DSL.literal(period), DSL.literal(1)).valueOf().value()); + assertNull(period_diff(DSL.literal(period), DSL.literal(1)).valueOf().value()); + assertNull(period_diff(DSL.literal(1), DSL.literal(period)).valueOf().value()); } } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java index 437e195f3e..4e7541177f 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java @@ -229,7 +229,7 @@ private static Stream getInvalidDoubleSamples() { @MethodSource("getInvalidDoubleSamples") public void checkInvalidDoubleCausesNull(Double value) { assertEquals(ExprNullValue.of(), - unixTimeStampOf(DSL.literal(new ExprDoubleValue(value))).valueOf(null), + unixTimeStampOf(DSL.literal(new ExprDoubleValue(value))).valueOf(), new DecimalFormat("0.#").format(value)); } } diff --git a/core/src/test/java/org/opensearch/sql/expression/function/OpenSearchFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/function/OpenSearchFunctionsTest.java index 741caa5f91..620bdbf1b4 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/OpenSearchFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/OpenSearchFunctionsTest.java @@ -183,6 +183,13 @@ void simple_query_string() { expr.toString()); } + @Test + void query() { + FunctionExpression expr = dsl.query(query); + assertEquals(String.format("query(query=%s)", query.getValue()), + expr.toString()); + } + @Test void query_string() { FunctionExpression expr = dsl.query_string(fields, query); diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/arthmetic/ArithmeticFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/arthmetic/ArithmeticFunctionTest.java index b90314f9c1..44af20a690 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/arthmetic/ArithmeticFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/arthmetic/ArithmeticFunctionTest.java @@ -71,7 +71,7 @@ public void add(ExprValue op1, ExprValue op2) { FunctionExpression expression = dsl.add(literal(op1), literal(op2)); ExprType expectedType = WideningTypeRule.max(op1.type(), op2.type()); assertEquals(expectedType, expression.type()); - assertValueEqual(BuiltinFunctionName.ADD, expectedType, op1, op2, expression.valueOf(null)); + assertValueEqual(BuiltinFunctionName.ADD, expectedType, op1, op2, expression.valueOf()); assertEquals(String.format("+(%s, %s)", op1.toString(), op2.toString()), expression.toString()); } @@ -147,7 +147,7 @@ public void subtract(ExprValue op1, ExprValue op2) { ExprType expectedType = WideningTypeRule.max(op1.type(), op2.type()); assertEquals(expectedType, expression.type()); assertValueEqual(BuiltinFunctionName.SUBTRACT, expectedType, op1, op2, - expression.valueOf(null)); + expression.valueOf()); assertEquals(String.format("-(%s, %s)", op1.toString(), op2.toString()), expression.toString()); } @@ -159,7 +159,7 @@ public void multiply(ExprValue op1, ExprValue op2) { ExprType expectedType = WideningTypeRule.max(op1.type(), op2.type()); assertEquals(expectedType, expression.type()); assertValueEqual(BuiltinFunctionName.MULTIPLY, expectedType, op1, op2, - expression.valueOf(null)); + expression.valueOf()); assertEquals(String.format("*(%s, %s)", op1.toString(), op2.toString()), expression.toString()); } @@ -170,7 +170,7 @@ public void divide(ExprValue op1, ExprValue op2) { FunctionExpression expression = dsl.divide(literal(op1), literal(op2)); ExprType expectedType = WideningTypeRule.max(op1.type(), op2.type()); assertEquals(expectedType, expression.type()); - assertValueEqual(BuiltinFunctionName.DIVIDE, expectedType, op1, op2, expression.valueOf(null)); + assertValueEqual(BuiltinFunctionName.DIVIDE, expectedType, op1, op2, expression.valueOf()); assertEquals(String.format("/(%s, %s)", op1.toString(), op2.toString()), expression.toString()); @@ -187,7 +187,7 @@ public void module(ExprValue op1, ExprValue op2) { FunctionExpression expression = dsl.module(literal(op1), literal(op2)); ExprType expectedType = WideningTypeRule.max(op1.type(), op2.type()); assertEquals(expectedType, expression.type()); - assertValueEqual(BuiltinFunctionName.MODULES, expectedType, op1, op2, expression.valueOf(null)); + assertValueEqual(BuiltinFunctionName.MODULES, expectedType, op1, op2, expression.valueOf()); assertEquals(String.format("%%(%s, %s)", op1.toString(), op2.toString()), expression.toString()); diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java index c2ca793e39..1f3748709e 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java @@ -48,8 +48,8 @@ class TypeCastOperatorTest { private static Stream numberData() { return Stream.of(new ExprByteValue(3), new ExprShortValue(3), - new ExprIntegerValue(3), new ExprLongValue(3L), new ExprFloatValue(3f), - new ExprDoubleValue(3D)); + new ExprIntegerValue(3), new ExprLongValue(3L), new ExprFloatValue(3.14f), + new ExprDoubleValue(3.1415D)); } private static Stream stringData() { @@ -81,7 +81,7 @@ private static Stream datetime() { void castToString(ExprValue value) { FunctionExpression expression = dsl.castString(DSL.literal(value)); assertEquals(STRING, expression.type()); - assertEquals(new ExprStringValue(value.value().toString()), expression.valueOf(null)); + assertEquals(new ExprStringValue(value.value().toString()), expression.valueOf()); } @ParameterizedTest(name = "castToByte({0})") @@ -89,7 +89,7 @@ void castToString(ExprValue value) { void castToByte(ExprValue value) { FunctionExpression expression = dsl.castByte(DSL.literal(value)); assertEquals(BYTE, expression.type()); - assertEquals(new ExprByteValue(value.byteValue()), expression.valueOf(null)); + assertEquals(new ExprByteValue(value.byteValue()), expression.valueOf()); } @ParameterizedTest(name = "castToShort({0})") @@ -97,7 +97,7 @@ void castToByte(ExprValue value) { void castToShort(ExprValue value) { FunctionExpression expression = dsl.castShort(DSL.literal(value)); assertEquals(SHORT, expression.type()); - assertEquals(new ExprShortValue(value.shortValue()), expression.valueOf(null)); + assertEquals(new ExprShortValue(value.shortValue()), expression.valueOf()); } @ParameterizedTest(name = "castToInt({0})") @@ -105,67 +105,67 @@ void castToShort(ExprValue value) { void castToInt(ExprValue value) { FunctionExpression expression = dsl.castInt(DSL.literal(value)); assertEquals(INTEGER, expression.type()); - assertEquals(new ExprIntegerValue(value.integerValue()), expression.valueOf(null)); + assertEquals(new ExprIntegerValue(value.integerValue()), expression.valueOf()); } @Test void castStringToByte() { FunctionExpression expression = dsl.castByte(DSL.literal("100")); assertEquals(BYTE, expression.type()); - assertEquals(new ExprByteValue(100), expression.valueOf(null)); + assertEquals(new ExprByteValue(100), expression.valueOf()); } @Test void castStringToShort() { FunctionExpression expression = dsl.castShort(DSL.literal("100")); assertEquals(SHORT, expression.type()); - assertEquals(new ExprShortValue(100), expression.valueOf(null)); + assertEquals(new ExprShortValue(100), expression.valueOf()); } @Test void castStringToInt() { FunctionExpression expression = dsl.castInt(DSL.literal("100")); assertEquals(INTEGER, expression.type()); - assertEquals(new ExprIntegerValue(100), expression.valueOf(null)); + assertEquals(new ExprIntegerValue(100), expression.valueOf()); } @Test void castStringToIntException() { FunctionExpression expression = dsl.castInt(DSL.literal("invalid")); - assertThrows(RuntimeException.class, () -> expression.valueOf(null)); + assertThrows(RuntimeException.class, () -> expression.valueOf()); } @Test void castBooleanToByte() { FunctionExpression expression = dsl.castByte(DSL.literal(true)); assertEquals(BYTE, expression.type()); - assertEquals(new ExprByteValue(1), expression.valueOf(null)); + assertEquals(new ExprByteValue(1), expression.valueOf()); expression = dsl.castByte(DSL.literal(false)); assertEquals(BYTE, expression.type()); - assertEquals(new ExprByteValue(0), expression.valueOf(null)); + assertEquals(new ExprByteValue(0), expression.valueOf()); } @Test void castBooleanToShort() { FunctionExpression expression = dsl.castShort(DSL.literal(true)); assertEquals(SHORT, expression.type()); - assertEquals(new ExprShortValue(1), expression.valueOf(null)); + assertEquals(new ExprShortValue(1), expression.valueOf()); expression = dsl.castShort(DSL.literal(false)); assertEquals(SHORT, expression.type()); - assertEquals(new ExprShortValue(0), expression.valueOf(null)); + assertEquals(new ExprShortValue(0), expression.valueOf()); } @Test void castBooleanToInt() { FunctionExpression expression = dsl.castInt(DSL.literal(true)); assertEquals(INTEGER, expression.type()); - assertEquals(new ExprIntegerValue(1), expression.valueOf(null)); + assertEquals(new ExprIntegerValue(1), expression.valueOf()); expression = dsl.castInt(DSL.literal(false)); assertEquals(INTEGER, expression.type()); - assertEquals(new ExprIntegerValue(0), expression.valueOf(null)); + assertEquals(new ExprIntegerValue(0), expression.valueOf()); } @ParameterizedTest(name = "castToLong({0})") @@ -173,31 +173,31 @@ void castBooleanToInt() { void castToLong(ExprValue value) { FunctionExpression expression = dsl.castLong(DSL.literal(value)); assertEquals(LONG, expression.type()); - assertEquals(new ExprLongValue(value.longValue()), expression.valueOf(null)); + assertEquals(new ExprLongValue(value.longValue()), expression.valueOf()); } @Test void castStringToLong() { FunctionExpression expression = dsl.castLong(DSL.literal("100")); assertEquals(LONG, expression.type()); - assertEquals(new ExprLongValue(100), expression.valueOf(null)); + assertEquals(new ExprLongValue(100), expression.valueOf()); } @Test void castStringToLongException() { FunctionExpression expression = dsl.castLong(DSL.literal("invalid")); - assertThrows(RuntimeException.class, () -> expression.valueOf(null)); + assertThrows(RuntimeException.class, () -> expression.valueOf()); } @Test void castBooleanToLong() { FunctionExpression expression = dsl.castLong(DSL.literal(true)); assertEquals(LONG, expression.type()); - assertEquals(new ExprLongValue(1), expression.valueOf(null)); + assertEquals(new ExprLongValue(1), expression.valueOf()); expression = dsl.castLong(DSL.literal(false)); assertEquals(LONG, expression.type()); - assertEquals(new ExprLongValue(0), expression.valueOf(null)); + assertEquals(new ExprLongValue(0), expression.valueOf()); } @ParameterizedTest(name = "castToFloat({0})") @@ -205,31 +205,31 @@ void castBooleanToLong() { void castToFloat(ExprValue value) { FunctionExpression expression = dsl.castFloat(DSL.literal(value)); assertEquals(FLOAT, expression.type()); - assertEquals(new ExprFloatValue(value.floatValue()), expression.valueOf(null)); + assertEquals(new ExprFloatValue(value.floatValue()), expression.valueOf()); } @Test void castStringToFloat() { FunctionExpression expression = dsl.castFloat(DSL.literal("100.0")); assertEquals(FLOAT, expression.type()); - assertEquals(new ExprFloatValue(100.0), expression.valueOf(null)); + assertEquals(new ExprFloatValue(100.0), expression.valueOf()); } @Test void castStringToFloatException() { FunctionExpression expression = dsl.castFloat(DSL.literal("invalid")); - assertThrows(RuntimeException.class, () -> expression.valueOf(null)); + assertThrows(RuntimeException.class, () -> expression.valueOf()); } @Test void castBooleanToFloat() { FunctionExpression expression = dsl.castFloat(DSL.literal(true)); assertEquals(FLOAT, expression.type()); - assertEquals(new ExprFloatValue(1), expression.valueOf(null)); + assertEquals(new ExprFloatValue(1), expression.valueOf()); expression = dsl.castFloat(DSL.literal(false)); assertEquals(FLOAT, expression.type()); - assertEquals(new ExprFloatValue(0), expression.valueOf(null)); + assertEquals(new ExprFloatValue(0), expression.valueOf()); } @ParameterizedTest(name = "castToDouble({0})") @@ -237,31 +237,31 @@ void castBooleanToFloat() { void castToDouble(ExprValue value) { FunctionExpression expression = dsl.castDouble(DSL.literal(value)); assertEquals(DOUBLE, expression.type()); - assertEquals(new ExprDoubleValue(value.doubleValue()), expression.valueOf(null)); + assertEquals(new ExprDoubleValue(value.doubleValue()), expression.valueOf()); } @Test void castStringToDouble() { FunctionExpression expression = dsl.castDouble(DSL.literal("100.0")); assertEquals(DOUBLE, expression.type()); - assertEquals(new ExprDoubleValue(100), expression.valueOf(null)); + assertEquals(new ExprDoubleValue(100), expression.valueOf()); } @Test void castStringToDoubleException() { FunctionExpression expression = dsl.castDouble(DSL.literal("invalid")); - assertThrows(RuntimeException.class, () -> expression.valueOf(null)); + assertThrows(RuntimeException.class, () -> expression.valueOf()); } @Test void castBooleanToDouble() { FunctionExpression expression = dsl.castDouble(DSL.literal(true)); assertEquals(DOUBLE, expression.type()); - assertEquals(new ExprDoubleValue(1), expression.valueOf(null)); + assertEquals(new ExprDoubleValue(1), expression.valueOf()); expression = dsl.castDouble(DSL.literal(false)); assertEquals(DOUBLE, expression.type()); - assertEquals(new ExprDoubleValue(0), expression.valueOf(null)); + assertEquals(new ExprDoubleValue(0), expression.valueOf()); } @ParameterizedTest(name = "castToBoolean({0})") @@ -269,96 +269,96 @@ void castBooleanToDouble() { void castToBoolean(ExprValue value) { FunctionExpression expression = dsl.castBoolean(DSL.literal(value)); assertEquals(BOOLEAN, expression.type()); - assertEquals(ExprBooleanValue.of(true), expression.valueOf(null)); + assertEquals(ExprBooleanValue.of(true), expression.valueOf()); } @Test void castZeroToBoolean() { FunctionExpression expression = dsl.castBoolean(DSL.literal(0)); assertEquals(BOOLEAN, expression.type()); - assertEquals(ExprBooleanValue.of(false), expression.valueOf(null)); + assertEquals(ExprBooleanValue.of(false), expression.valueOf()); } @Test void castStringToBoolean() { FunctionExpression expression = dsl.castBoolean(DSL.literal("True")); assertEquals(BOOLEAN, expression.type()); - assertEquals(ExprBooleanValue.of(true), expression.valueOf(null)); + assertEquals(ExprBooleanValue.of(true), expression.valueOf()); } @Test void castBooleanToBoolean() { FunctionExpression expression = dsl.castBoolean(DSL.literal(true)); assertEquals(BOOLEAN, expression.type()); - assertEquals(ExprBooleanValue.of(true), expression.valueOf(null)); + assertEquals(ExprBooleanValue.of(true), expression.valueOf()); } @Test void castToDate() { FunctionExpression expression = dsl.castDate(DSL.literal("2012-08-07")); assertEquals(DATE, expression.type()); - assertEquals(new ExprDateValue("2012-08-07"), expression.valueOf(null)); + assertEquals(new ExprDateValue("2012-08-07"), expression.valueOf()); expression = dsl.castDate(DSL.literal(new ExprDatetimeValue("2012-08-07 01:01:01"))); assertEquals(DATE, expression.type()); - assertEquals(new ExprDateValue("2012-08-07"), expression.valueOf(null)); + assertEquals(new ExprDateValue("2012-08-07"), expression.valueOf()); expression = dsl.castDate(DSL.literal(new ExprTimestampValue("2012-08-07 01:01:01"))); assertEquals(DATE, expression.type()); - assertEquals(new ExprDateValue("2012-08-07"), expression.valueOf(null)); + assertEquals(new ExprDateValue("2012-08-07"), expression.valueOf()); expression = dsl.castDate(DSL.literal(new ExprDateValue("2012-08-07"))); assertEquals(DATE, expression.type()); - assertEquals(new ExprDateValue("2012-08-07"), expression.valueOf(null)); + assertEquals(new ExprDateValue("2012-08-07"), expression.valueOf()); } @Test void castToTime() { FunctionExpression expression = dsl.castTime(DSL.literal("01:01:01")); assertEquals(TIME, expression.type()); - assertEquals(new ExprTimeValue("01:01:01"), expression.valueOf(null)); + assertEquals(new ExprTimeValue("01:01:01"), expression.valueOf()); expression = dsl.castTime(DSL.literal(new ExprDatetimeValue("2012-08-07 01:01:01"))); assertEquals(TIME, expression.type()); - assertEquals(new ExprTimeValue("01:01:01"), expression.valueOf(null)); + assertEquals(new ExprTimeValue("01:01:01"), expression.valueOf()); expression = dsl.castTime(DSL.literal(new ExprTimestampValue("2012-08-07 01:01:01"))); assertEquals(TIME, expression.type()); - assertEquals(new ExprTimeValue("01:01:01"), expression.valueOf(null)); + assertEquals(new ExprTimeValue("01:01:01"), expression.valueOf()); expression = dsl.castTime(DSL.literal(new ExprTimeValue("01:01:01"))); assertEquals(TIME, expression.type()); - assertEquals(new ExprTimeValue("01:01:01"), expression.valueOf(null)); + assertEquals(new ExprTimeValue("01:01:01"), expression.valueOf()); } @Test void castToTimestamp() { FunctionExpression expression = dsl.castTimestamp(DSL.literal("2012-08-07 01:01:01")); assertEquals(TIMESTAMP, expression.type()); - assertEquals(new ExprTimestampValue("2012-08-07 01:01:01"), expression.valueOf(null)); + assertEquals(new ExprTimestampValue("2012-08-07 01:01:01"), expression.valueOf()); expression = dsl.castTimestamp(DSL.literal(new ExprDatetimeValue("2012-08-07 01:01:01"))); assertEquals(TIMESTAMP, expression.type()); - assertEquals(new ExprTimestampValue("2012-08-07 01:01:01"), expression.valueOf(null)); + assertEquals(new ExprTimestampValue("2012-08-07 01:01:01"), expression.valueOf()); expression = dsl.castTimestamp(DSL.literal(new ExprTimestampValue("2012-08-07 01:01:01"))); assertEquals(TIMESTAMP, expression.type()); - assertEquals(new ExprTimestampValue("2012-08-07 01:01:01"), expression.valueOf(null)); + assertEquals(new ExprTimestampValue("2012-08-07 01:01:01"), expression.valueOf()); } @Test void castToDatetime() { FunctionExpression expression = dsl.castDatetime(DSL.literal("2012-08-07 01:01:01")); assertEquals(DATETIME, expression.type()); - assertEquals(new ExprDatetimeValue("2012-08-07 01:01:01"), expression.valueOf(null)); + assertEquals(new ExprDatetimeValue("2012-08-07 01:01:01"), expression.valueOf()); expression = dsl.castDatetime(DSL.literal(new ExprTimestampValue("2012-08-07 01:01:01"))); assertEquals(DATETIME, expression.type()); - assertEquals(new ExprDatetimeValue("2012-08-07 01:01:01"), expression.valueOf(null)); + assertEquals(new ExprDatetimeValue("2012-08-07 01:01:01"), expression.valueOf()); expression = dsl.castDatetime(DSL.literal(new ExprDateValue("2012-08-07"))); assertEquals(DATETIME, expression.type()); - assertEquals(new ExprDatetimeValue("2012-08-07 00:00:00"), expression.valueOf(null)); + assertEquals(new ExprDatetimeValue("2012-08-07 00:00:00"), expression.valueOf()); } } diff --git a/core/src/test/java/org/opensearch/sql/expression/system/SystemFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/system/SystemFunctionsTest.java index 453018a700..e6cf4ce585 100644 --- a/core/src/test/java/org/opensearch/sql/expression/system/SystemFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/system/SystemFunctionsTest.java @@ -88,6 +88,6 @@ public ExprType type() { } private String typeofGetValue(ExprValue input) { - return dsl.typeof(DSL.literal(input)).valueOf(null).stringValue(); + return dsl.typeof(DSL.literal(input)).valueOf().stringValue(); } } diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScanTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScanTest.java index 26374ff042..cf9b5fe016 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScanTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScanTest.java @@ -61,7 +61,7 @@ void testIterator() { assertTrue(catalogTableScan.hasNext()); for (Catalog catalog : catalogSet) { assertEquals(new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( - "CATALOG_NAME", ExprValueUtils.stringValue(catalog.getName()), + "DATASOURCE_NAME", ExprValueUtils.stringValue(catalog.getName()), "CONNECTOR_TYPE", ExprValueUtils.stringValue(catalog.getConnectorType().name())))), catalogTableScan.next()); } diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java index a35c94a21c..1c069005f0 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java @@ -34,7 +34,7 @@ void testGetFieldTypes() { CatalogTable catalogTable = new CatalogTable(catalogService); Map fieldTypes = catalogTable.getFieldTypes(); Map expectedTypes = new HashMap<>(); - expectedTypes.put("CATALOG_NAME", ExprCoreType.STRING); + expectedTypes.put("DATASOURCE_NAME", ExprCoreType.STRING); expectedTypes.put("CONNECTOR_TYPE", ExprCoreType.STRING); assertEquals(expectedTypes, fieldTypes); } diff --git a/docs/category.json b/docs/category.json index 0b11209111..b8feaa654a 100644 --- a/docs/category.json +++ b/docs/category.json @@ -10,7 +10,7 @@ "user/ppl/cmd/ad.rst", "user/ppl/cmd/dedup.rst", "user/ppl/cmd/describe.rst", - "user/ppl/cmd/showcatalogs.rst", + "user/ppl/cmd/showdatasources.rst", "user/ppl/cmd/information_schema.rst", "user/ppl/cmd/eval.rst", "user/ppl/cmd/fields.rst", diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index a04f01446e..788cac0433 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -1076,13 +1076,13 @@ Return type: DATE Example:: - >od SELECT DATE('2020-08-26'), DATE(TIMESTAMP('2020-08-26 13:49:00')) + os> SELECT DATE('2020-08-26'), DATE(TIMESTAMP('2020-08-26 13:49:00')), DATE('2020-08-26 13:49:00'), DATE('2020-08-26 13:49') fetched rows / total rows = 1/1 - +----------------------+------------------------------------------+ - | DATE('2020-08-26') | DATE(TIMESTAMP('2020-08-26 13:49:00')) | - |----------------------+------------------------------------------| - | DATE '2020-08-26' | DATE '2020-08-26' | - +----------------------+------------------------------------------+ + +----------------------+------------------------------------------+-------------------------------+----------------------------+ + | DATE('2020-08-26') | DATE(TIMESTAMP('2020-08-26 13:49:00')) | DATE('2020-08-26 13:49:00') | DATE('2020-08-26 13:49') | + |----------------------+------------------------------------------+-------------------------------+----------------------------| + | 2020-08-26 | 2020-08-26 | 2020-08-26 | 2020-08-26 | + +----------------------+------------------------------------------+-------------------------------+----------------------------+ DATETIME @@ -1926,13 +1926,13 @@ Return type: TIME Example:: - >od SELECT TIME('13:49:00'), TIME(TIMESTAMP('2020-08-26 13:49:00')) + os> SELECT TIME('13:49:00'), TIME('13:49'), TIME(TIMESTAMP('2020-08-26 13:49:00')), TIME('2020-08-26 13:49:00') fetched rows / total rows = 1/1 - +--------------------+------------------------------------------+ - | TIME('13:49:00') | TIME(TIMESTAMP('2020-08-26 13:49:00')) | - |--------------------+------------------------------------------| - | TIME '13:49:00' | TIME '13:49:00' | - +--------------------+------------------------------------------+ + +--------------------+-----------------+------------------------------------------+-------------------------------+ + | TIME('13:49:00') | TIME('13:49') | TIME(TIMESTAMP('2020-08-26 13:49:00')) | TIME('2020-08-26 13:49:00') | + |--------------------+-----------------+------------------------------------------+-------------------------------| + | 13:49:00 | 13:49:00 | 13:49:00 | 13:49:00 | + +--------------------+-----------------+------------------------------------------+-------------------------------+ TIME_TO_SEC @@ -3017,6 +3017,67 @@ Another example to show how to set custom values for the optional parameters:: +------+--------------------------+----------------------+ +QUERY +----- + +Description +>>>>>>>>>>> + +``query("query_expression" [, option=]*)`` + +The `query` function is an alternative syntax to the `query_string`_ function. It maps to the query_string query used in search engine, to return the documents that match a provided text, number, date or boolean value with a given query expression. +``query_expression`` must be a string provided in Lucene query string syntax. Please refer to examples below: + +| ``query('Tags:taste OR Body:taste', ...)`` +| ``query("Tags:taste AND Body:taste", ...)`` + +Available parameters include: + +- analyzer +- escape +- allow_leading_wildcard +- analyze_wildcard +- auto_generate_synonyms_phrase_query +- boost +- default_operator +- enable_position_increments +- fuzziness +- fuzzy_max_expansions +- fuzzy_prefix_length +- fuzzy_transpositions +- fuzzy_rewrite +- tie_breaker +- lenient +- type +- max_determinized_states +- minimum_should_match +- quote_analyzer +- phrase_slop +- quote_field_suffix +- rewrite +- time_zone + +Example with only ``query_expressions``, and all other parameters are set default values:: + + os> select * from books where query('title:Pooh House'); + fetched rows / total rows = 2/2 + +------+--------------------------+----------------------+ + | id | title | author | + |------+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + | 2 | Winnie-the-Pooh | Alan Alexander Milne | + +------+--------------------------+----------------------+ + +Another example to show how to set custom values for the optional parameters:: + + os> select * from books where query('title:Pooh House', default_operator='AND'); + fetched rows / total rows = 1/1 + +------+--------------------------+----------------------+ + | id | title | author | + |------+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + +------+--------------------------+----------------------+ + HIGHLIGHT ------------ diff --git a/docs/user/general/identifiers.rst b/docs/user/general/identifiers.rst index affd381d41..8bb42bb7e7 100644 --- a/docs/user/general/identifiers.rst +++ b/docs/user/general/identifiers.rst @@ -184,18 +184,18 @@ Fully Qualified Table Names Description ----------- -With the introduction of different datasource catalogs along with Opensearch, support for fully qualified table names became compulsory to resolve tables to a catalog. +With the introduction of different datasources along with Opensearch, support for fully qualified table names became compulsory to resolve tables to a datasource. Format for fully qualified table name. -``..`` +``..`` -* catalogName:[Mandatory] Catalog information is mandatory when querying over tables from catalogs other than opensearch connector. +* datasourceName:[Mandatory] Datasource information is mandatory when querying over tables from datasources other than opensearch connector. * schemaName:[Optional] Schema is a logical abstraction for a group of tables. In the current state, we only support ``default`` and ``information_schema``. Any schema mentioned in the fully qualified name other than these two will be resolved to be part of tableName. * tableName:[Mandatory] tableName is mandatory. -The current resolution algorithm works in such a way, the old queries on opensearch work without specifying any catalog name. +The current resolution algorithm works in such a way, the old queries on opensearch work without specifying any datasource name. So queries on opensearch indices doesn't need a fully qualified table name. Table Name Resolution Algorithm. @@ -205,24 +205,24 @@ Fully qualified Name is divided into parts based on ``.`` character. TableName resolution algorithm works in the following manner. -1. Take the first part of the qualified name and resolve it to a catalog from the list of catalogs configured. -If it doesn't resolve to any of the catalog names configured, catalog name will default to ``@opensearch`` catalog. +1. Take the first part of the qualified name and resolve it to a datasource from the list of datasources configured. +If it doesn't resolve to any of the datasource names configured, datasource name will default to ``@opensearch`` datasource. -2. Take the first part of the remaining qualified name after capturing the catalog name. -If this part represents any of the supported schemas under catalog, it will resolve to the same otherwise schema name will resolve to ``default`` schema. +2. Take the first part of the remaining qualified name after capturing the datasource name. +If this part represents any of the supported schemas under datasource, it will resolve to the same otherwise schema name will resolve to ``default`` schema. Currently ``default`` and ``information_schema`` are the only schemas supported. 3. Rest of the parts are combined to resolve tablename. -** Only table name identifiers are supported with fully qualified names, identifiers used for columns and other attributes doesn't require prefixing with catalog and schema information.** +** Only table name identifiers are supported with fully qualified names, identifiers used for columns and other attributes doesn't require prefixing with datasource and schema information.** Examples -------- -Assume [my_prometheus] is the only catalog configured other than default opensearch engine. +Assume [my_prometheus] is the only datasource configured other than default opensearch engine. 1. ``my_prometheus.default.http_requests_total`` -catalogName = ``my_prometheus`` [Is in the list of catalogs configured]. +datasourceName = ``my_prometheus`` [Is in the list of datasources configured]. schemaName = ``default`` [Is in the list of schemas supported]. @@ -231,7 +231,7 @@ tableName = ``http_requests_total``. 2. ``logs.12.13.1`` -catalogName = ``@opensearch`` [Resolves to default @opensearch connector since [my_prometheus] is the only catalog configured name.] +datasourceName = ``@opensearch`` [Resolves to default @opensearch connector since [my_prometheus] is the only catalog configured name.] schemaName = ``default`` [No supported schema found, so default to `default`]. @@ -241,7 +241,7 @@ tableName = ``logs.12.13.1``. 3. ``my_prometheus.http_requests_total`` -catalogName = ```my_prometheus`` [Is in the list of catalogs configured]. +datasourceName = ```my_prometheus`` [Is in the list of datasources configured]. schemaName = ``default`` [No supported schema found, so default to `default`]. @@ -249,7 +249,7 @@ tableName = ``http_requests_total``. 4. ``prometheus.http_requests_total`` -catalogName = ``@opensearch`` [Resolves to default @opensearch connector since [my_prometheus] is the only catalog configured name.] +datasourceName = ``@opensearch`` [Resolves to default @opensearch connector since [my_prometheus] is the only datasource configured name.] schemaName = ``default`` [No supported schema found, so default to `default`]. @@ -257,7 +257,7 @@ tableName = ``prometheus.http_requests_total``. 5. ``prometheus.default.http_requests_total.1.2.3`` -catalogName = ``@opensearch`` [Resolves to default @opensearch connector since [my_prometheus] is the only catalog configured name.] +datasourceName = ``@opensearch`` [Resolves to default @opensearch connector since [my_prometheus] is the only catalog configured name.] schemaName = ``default`` [No supported schema found, so default to `default`]. diff --git a/docs/user/ppl/admin/catalog.rst b/docs/user/ppl/admin/catalog.rst deleted file mode 100644 index ccaab342a5..0000000000 --- a/docs/user/ppl/admin/catalog.rst +++ /dev/null @@ -1,83 +0,0 @@ -.. highlight:: sh - -================= -Catalog Settings -================= - -.. rubric:: Table of contents - -.. contents:: - :local: - :depth: 1 - -Introduction -============ - -The concept of ``catalog`` is introduced to support the federation of SQL/PPL query engine to multiple data sources. -This helps PPL users to leverage data from multiple data sources and derive correlation and insights. -Catalog definition provides the information to connect to a datasource and also gives a name to them to refer in PPL commands. - - -Definitions of catalog and connector -==================================== -* Connector is a component that adapts the query engine to a datasource. For example, Prometheus connector would adapt and help execute the queries to run on Prometheus data source. connector name is enough in the catalog definition json. -* Catalog is a construct to define how to connect to a datasource and which connector to adapt by query engine. - -Example Prometheus Catalog Definition :: - - [{ - "name" : "my_prometheus", - "connector": "prometheus", - "properties" : { - "prometheus.uri" : "http://localhost:8080", - "prometheus.auth.type" : "basicauth", - "prometheus.auth.username" : "admin", - "prometheus.auth.password" : "admin" - } - }] -Catalog configuration Restrictions. - -* ``name``, ``connector``, ``properties`` are required fields in the catalog configuration. -* All the catalog names should be unique and match the following regex[``[@*A-Za-z]+?[*a-zA-Z_\-0-9]*``]. -* Allowed Connectors. - * ``prometheus`` [More details: `Prometheus Connector `_] -* All the allowed config parameters in ``properties`` are defined in individual connector pages mentioned above. - -Configuring catalog in OpenSearch -==================================== - -* Catalogs are configured in opensearch keystore as secure settings under ``plugins.query.federation.catalog.config`` key as they contain credential info. -* A json file containing array of catalog configurations should be injected into keystore with the above mentioned key. sample json file can be seen in the above section. - - -[**To be run on all the nodes in the cluster**] Command to add catalog.json file to OpenSearch Keystore :: - - >> bin/opensearch-keystore add-file plugins.query.federation.catalog.config catalog.json - -Catalogs can be configured during opensearch start up or can be updated while the opensearch is running. -If we update catalog configuration during runtime, the following api should be triggered to update the query engine with the latest changes. - -[**Required only if we update keystore settings during runtime**] Secure Settings refresh api:: - - >> curl --request POST \ - --url http://{{opensearch-domain}}:9200/_nodes/reload_secure_settings \ - --data '{"secure_settings_password":"{{keystore-password}}"}' - - -Using a catalog in PPL command -==================================== -Catalog is referred in source command as show in the code block below. -Based on the abstraction designed by the connector, -one can refer the corresponding entity as table in the source command. -For example in prometheus connector, each metric is abstracted as a table. -so we can refer a metric and apply stats over it in the following way. - -Example source command with prometheus catalog :: - - >> source = my_prometheus.prometheus_http_requests_total | stats avg(@value) by job; - - -Limitations of catalog -==================================== -Catalog settings are global and users with PPL access are allowed to fetch data from all the defined catalogs. -PPL access can be controlled using roles.(More details: `Security Settings `_) \ No newline at end of file diff --git a/docs/user/ppl/admin/datasources.rst b/docs/user/ppl/admin/datasources.rst new file mode 100644 index 0000000000..2974ac20ce --- /dev/null +++ b/docs/user/ppl/admin/datasources.rst @@ -0,0 +1,83 @@ +.. highlight:: sh + +=================== +Datasource Settings +=================== + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 1 + +Introduction +============ + +The concept of ``datasource`` is introduced to support the federation of SQL/PPL query engine to multiple data stores. +This helps PPL users to leverage data from multiple data stores and derive correlation and insights. +Datasource definition provides the information to connect to a data store and also gives a name to them to refer in PPL commands. + + +Definitions of datasource and connector +==================================== +* Connector is a component that adapts the query engine to a datastore. For example, Prometheus connector would adapt and help execute the queries to run on Prometheus datastore. connector name is enough in the datasource definition json. +* Datasource is a construct to define how to connect to a data store and which connector to adapt by query engine. + +Example Prometheus Datasource Definition :: + + [{ + "name" : "my_prometheus", + "connector": "prometheus", + "properties" : { + "prometheus.uri" : "http://localhost:8080", + "prometheus.auth.type" : "basicauth", + "prometheus.auth.username" : "admin", + "prometheus.auth.password" : "admin" + } + }] +Datasource configuration Restrictions. + +* ``name``, ``connector``, ``properties`` are required fields in the datasource configuration. +* All the datasource names should be unique and match the following regex[``[@*A-Za-z]+?[*a-zA-Z_\-0-9]*``]. +* Allowed Connectors. + * ``prometheus`` [More details: `Prometheus Connector `_] +* All the allowed config parameters in ``properties`` are defined in individual connector pages mentioned above. + +Configuring a datasource in OpenSearch +====================================== + +* Datasources are configured in opensearch keystore as secure settings under ``plugins.query.federation.datasources.config`` key as they contain credential info. +* A json file containing array of datasource configurations should be injected into keystore with the above mentioned key. sample json file can be seen in the above section. + + +[**To be run on all the nodes in the cluster**] Command to add datasources.json file to OpenSearch Keystore :: + + >> bin/opensearch-keystore add-file plugins.query.federation.datasource.config datasources.json + +Datasources can be configured during opensearch start up or can be updated while the opensearch is running. +If we update a datasource configuration during runtime, the following api should be triggered to update the query engine with the latest changes. + +[**Required only if we update keystore settings during runtime**] Secure Settings refresh api:: + + >> curl --request POST \ + --url http://{{opensearch-domain}}:9200/_nodes/reload_secure_settings \ + --data '{"secure_settings_password":"{{keystore-password}}"}' + + +Using a datasource in PPL command +==================================== +Datasource is referred in source command as show in the code block below. +Based on the abstraction designed by the connector, +one can refer the corresponding entity as table in the source command. +For example in prometheus connector, each metric is abstracted as a table. +so we can refer a metric and apply stats over it in the following way. + +Example source command with prometheus datasource :: + + >> source = my_prometheus.prometheus_http_requests_total | stats avg(@value) by job; + + +Limitations of datasource +==================================== +Datasource settings are global and users with PPL access are allowed to fetch data from all the defined datasources. +PPL access can be controlled using roles.(More details: `Security Settings `_) \ No newline at end of file diff --git a/docs/user/ppl/cmd/information_schema.rst b/docs/user/ppl/cmd/information_schema.rst index a756fb080e..26341d6972 100644 --- a/docs/user/ppl/cmd/information_schema.rst +++ b/docs/user/ppl/cmd/information_schema.rst @@ -11,19 +11,19 @@ Metadata queries using information_schema Description ============ -| Use ``information_schema`` in source command to query tables information under a catalog. +| Use ``information_schema`` in source command to query tables information under a datasource. In the current state, ``information_schema`` only support metadata of tables. This schema will be extended for views, columns and other metadata info in future. Syntax ============ -source = catalog.information_schema.tables; +source = datasource.information_schema.tables; -Example 1: Fetch tables in prometheus catalog. +Example 1: Fetch tables in prometheus datasource. ============================================== -The examples fetches tables in the prometheus catalog. +The examples fetches tables in the prometheus datasource. PPL query for fetching PROMETHEUS TABLES with where clause:: @@ -36,10 +36,10 @@ PPL query for fetching PROMETHEUS TABLES with where clause:: +-----------------+----------------+--------------------------------+--------------+--------+---------------------------+ -Example 2: Search tables in prometheus catalog. -=============================================== +Example 2: Search tables in prometheus datasource. +================================================= -The examples searches tables in the prometheus catalog. +The examples searches tables in the prometheus datasource. PPL query for searching PROMETHEUS TABLES:: diff --git a/docs/user/ppl/cmd/showcatalogs.rst b/docs/user/ppl/cmd/showcatalogs.rst deleted file mode 100644 index d304cba768..0000000000 --- a/docs/user/ppl/cmd/showcatalogs.rst +++ /dev/null @@ -1,36 +0,0 @@ -============= -show catalogs -============= - -.. rubric:: Table of contents - -.. contents:: - :local: - :depth: 2 - - -Description -============ -| Using ``show catalogs`` command to query catalogs configured in the PPL engine. ``show catalogs`` command could be only used as the first command in the PPL query. - - -Syntax -============ -show catalogs - - -Example 1: Fetch all PROMETHEUS catalogs -================================= - -The example fetches all the catalogs configured. - -PPL query for all PROMETHEUS CATALOGS:: - - os> show catalogs | where CONNECTOR_TYPE='PROMETHEUS'; - fetched rows / total rows = 1/1 - +----------------+------------------+ - | CATALOG_NAME | CONNECTOR_TYPE | - |----------------+------------------| - | my_prometheus | PROMETHEUS | - +----------------+------------------+ - diff --git a/docs/user/ppl/cmd/showdatasources.rst b/docs/user/ppl/cmd/showdatasources.rst new file mode 100644 index 0000000000..f7c6beb82f --- /dev/null +++ b/docs/user/ppl/cmd/showdatasources.rst @@ -0,0 +1,36 @@ +================ +show datasources +================ + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + + +Description +============ +| Using ``show datasources`` command to query datasources configured in the PPL engine. ``show datasources`` command could be only used as the first command in the PPL query. + + +Syntax +============ +show datasources + + +Example 1: Fetch all PROMETHEUS datasources +=========================================== + +The example fetches all the datasources of type prometheus. + +PPL query for all PROMETHEUS DATASOURCES:: + + os> show datasources | where CONNECTOR_TYPE='PROMETHEUS'; + fetched rows / total rows = 1/1 + +-------------------+------------------+ + | DATASOURCE_NAME | CONNECTOR_TYPE | + |-------------------+------------------| + | my_prometheus | PROMETHEUS | + +-------------------+------------------+ + diff --git a/docs/user/ppl/functions/datetime.rst b/docs/user/ppl/functions/datetime.rst index 223b3c5557..ca191e2426 100644 --- a/docs/user/ppl/functions/datetime.rst +++ b/docs/user/ppl/functions/datetime.rst @@ -291,13 +291,38 @@ Return type: DATE Example:: - >od source=people | eval `DATE('2020-08-26')` = DATE('2020-08-26'), `DATE(TIMESTAMP('2020-08-26 13:49:00'))` = DATE(TIMESTAMP('2020-08-26 13:49:00')) | fields `DATE('2020-08-26')`, `DATE(TIMESTAMP('2020-08-26 13:49:00'))` + os> source=people | eval `DATE('2020-08-26')` = DATE('2020-08-26') | fields `DATE('2020-08-26')` fetched rows / total rows = 1/1 - +----------------------+------------------------------------------+ - | DATE('2020-08-26') | DATE(TIMESTAMP('2020-08-26 13:49:00')) | - |----------------------+------------------------------------------| - | DATE '2020-08-26' | DATE '2020-08-26' | - +----------------------+------------------------------------------+ + +----------------------+ + | DATE('2020-08-26') | + |----------------------| + | 2020-08-26 | + +----------------------+ + + os> source=people | eval `DATE(TIMESTAMP('2020-08-26 13:49:00'))` = DATE(TIMESTAMP('2020-08-26 13:49:00')) | fields `DATE(TIMESTAMP('2020-08-26 13:49:00'))` + fetched rows / total rows = 1/1 + +------------------------------------------+ + | DATE(TIMESTAMP('2020-08-26 13:49:00')) | + |------------------------------------------| + | 2020-08-26 | + +------------------------------------------+ + + os> source=people | eval `DATE('2020-08-26 13:49')` = DATE('2020-08-26 13:49') | fields `DATE('2020-08-26 13:49')` + fetched rows / total rows = 1/1 + +----------------------------+ + | DATE('2020-08-26 13:49') | + |----------------------------| + | 2020-08-26 | + +----------------------------+ + + os> source=people | eval `DATE('2020-08-26 13:49')` = DATE('2020-08-26 13:49') | fields `DATE('2020-08-26 13:49')` + fetched rows / total rows = 1/1 + +----------------------------+ + | DATE('2020-08-26 13:49') | + |----------------------------| + | 2020-08-26 | + +----------------------------+ + DATE_ADD @@ -1098,13 +1123,37 @@ Return type: TIME Example:: - >od source=people | eval `TIME('13:49:00')` = TIME('13:49:00'), `TIME(TIMESTAMP('2020-08-26 13:49:00'))` = TIME(TIMESTAMP('2020-08-26 13:49:00')) | fields `TIME('13:49:00')`, `TIME(TIMESTAMP('2020-08-26 13:49:00'))` + os> source=people | eval `TIME('13:49:00')` = TIME('13:49:00') | fields `TIME('13:49:00')` fetched rows / total rows = 1/1 - +--------------------+------------------------------------------+ - | TIME('13:49:00') | TIME(TIMESTAMP('2020-08-26 13:49:00')) | - |--------------------+------------------------------------------| - | TIME '13:49:00' | TIME '13:49:00' | - +--------------------+------------------------------------------+ + +--------------------+ + | TIME('13:49:00') | + |--------------------| + | 13:49:00 | + +--------------------+ + + os> source=people | eval `TIME('13:49')` = TIME('13:49') | fields `TIME('13:49')` + fetched rows / total rows = 1/1 + +-----------------+ + | TIME('13:49') | + |-----------------| + | 13:49:00 | + +-----------------+ + + os> source=people | eval `TIME('2020-08-26 13:49:00')` = TIME('2020-08-26 13:49:00') | fields `TIME('2020-08-26 13:49:00')` + fetched rows / total rows = 1/1 + +-------------------------------+ + | TIME('2020-08-26 13:49:00') | + |-------------------------------| + | 13:49:00 | + +-------------------------------+ + + os> source=people | eval `TIME('2020-08-26 13:49')` = TIME('2020-08-26 13:49') | fields `TIME('2020-08-26 13:49')` + fetched rows / total rows = 1/1 + +----------------------------+ + | TIME('2020-08-26 13:49') | + |----------------------------| + | 13:49:00 | + +----------------------------+ TIME_TO_SEC diff --git a/docs/user/ppl/index.rst b/docs/user/ppl/index.rst index e09315b1c3..a69136bb19 100644 --- a/docs/user/ppl/index.rst +++ b/docs/user/ppl/index.rst @@ -34,7 +34,7 @@ The query start with search command and then flowing a set of command delimited - `Monitoring `_ - - `Catalog Settings `_ + - `Datasource Settings `_ - `Prometheus Connector `_ @@ -48,7 +48,7 @@ The query start with search command and then flowing a set of command delimited - `describe command `_ - - `show catalogs command `_ + - `show datasources command `_ - `eval command `_ diff --git a/doctest/build.gradle b/doctest/build.gradle index c1d069e50b..62efd29b18 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -104,7 +104,7 @@ String mlCommonsPlugin = 'opensearch-ml' testClusters { docTestCluster { - keystore 'plugins.query.federation.catalog.config', new File("$projectDir/catalog", 'catalog.json') + keystore 'plugins.query.federation.datasources.config', new File("$projectDir/catalog", 'catalog.json') // Disable loading of `ML-commons` plugin, because it might be unavailable (not released yet). /* plugin(provider(new Callable(){ diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 7a2e5cd406..58618cd67c 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -115,7 +115,7 @@ testClusters.all { testClusters.integTest { plugin ":opensearch-sql-plugin" - keystore 'plugins.query.federation.catalog.config', new File("$projectDir/src/test/resources/catalog/", 'catalog.json') + keystore 'plugins.query.federation.datasources.config', new File("$projectDir/src/test/resources/catalog/", 'catalog.json') } task startPrometheus(type: SpawnProcessTask) { mustRunAfter ':doctest:doctest' diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java index 0859a03784..b1b6b24ada 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java @@ -38,7 +38,7 @@ public void queryTest() throws IOException { "select address from %s where query('address:880 Holmes Lane') limit 3", TestsConstants.TEST_INDEX_ACCOUNT)); Assert.assertThat(result, - containsString("query_string\":{\"query\":\"address:880 Holmes Lane")); + containsString("query_string\\\":{\\\"query\\\":\\\"address:880 Holmes Lane")); } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusCatalogCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusCatalogCommandsIT.java index 9e197bbb27..10c1e911ab 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusCatalogCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusCatalogCommandsIT.java @@ -46,12 +46,12 @@ public void testSourceMetricCommand() { @SneakyThrows public void testMetricAvgAggregationCommand() { JSONObject response = - executeQuery("source=my_prometheus.prometheus_http_requests_total | stats avg(@value) by span(@timestamp, 15s), handler, job"); + executeQuery("source=`my_prometheus`.`prometheus_http_requests_total` | stats avg(@value) as `agg` by span(@timestamp, 15s), `handler`, `job`"); verifySchema(response, - schema("avg(@value)", "double"), + schema("agg", "double"), schema("span(@timestamp,15s)", "timestamp"), - schema("handler", "string"), - schema("job", "string")); + schema("`handler`", "string"), + schema("`job`", "string")); Assertions.assertTrue(response.getInt("size") > 0); Assertions.assertEquals(4, response.getJSONArray("datarows").getJSONArray(0).length()); JSONArray firstRow = response.getJSONArray("datarows").getJSONArray(0); @@ -65,11 +65,11 @@ public void testMetricAvgAggregationCommand() { @SneakyThrows public void testMetricAvgAggregationCommandWithAlias() { JSONObject response = - executeQuery("source=my_prometheus.prometheus_http_requests_total | stats avg(@value) as agg by span(@timestamp, 15s), handler, job"); + executeQuery("source=my_prometheus.prometheus_http_requests_total | stats avg(@value) as agg by span(@timestamp, 15s), `handler`, job"); verifySchema(response, schema("agg", "double"), schema("span(@timestamp,15s)", "timestamp"), - schema("handler", "string"), + schema("`handler`", "string"), schema("job", "string")); Assertions.assertTrue(response.getInt("size") > 0); Assertions.assertEquals(4, response.getJSONArray("datarows").getJSONArray(0).length()); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ShowCatalogsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ShowCatalogsCommandIT.java index 23418366be..e12aa040e3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ShowCatalogsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ShowCatalogsCommandIT.java @@ -20,25 +20,25 @@ public class ShowCatalogsCommandIT extends PPLIntegTestCase { @Test public void testShowCatalogsCommands() throws IOException { - JSONObject result = executeQuery("show catalogs"); + JSONObject result = executeQuery("show datasources"); verifyDataRows(result, rows("my_prometheus", "PROMETHEUS"), rows("@opensearch", "OPENSEARCH")); verifyColumn( result, - columnName("CATALOG_NAME"), + columnName("DATASOURCE_NAME"), columnName("CONNECTOR_TYPE") ); } @Test public void testShowCatalogsCommandsWithWhereClause() throws IOException { - JSONObject result = executeQuery("show catalogs | where CONNECTOR_TYPE='PROMETHEUS'"); + JSONObject result = executeQuery("show datasources | where CONNECTOR_TYPE='PROMETHEUS'"); verifyDataRows(result, rows("my_prometheus", "PROMETHEUS")); verifyColumn( result, - columnName("CATALOG_NAME"), + columnName("DATASOURCE_NAME"), columnName("CONNECTOR_TYPE") ); } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/QueryIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/QueryIT.java new file mode 100644 index 0000000000..e61593eb21 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/sql/QueryIT.java @@ -0,0 +1,84 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.sql; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BEER; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.sql.legacy.SQLIntegTestCase; + +public class QueryIT extends SQLIntegTestCase { + @Override + public void init() throws IOException { + loadIndex(Index.BEER); + } + + @Test + public void all_fields_test() throws IOException { + String query = "SELECT * FROM " + + TEST_INDEX_BEER + " WHERE query('*:taste')"; + JSONObject result = executeJdbcRequest(query); + assertEquals(16, result.getInt("total")); + } + + @Test + public void mandatory_params_test() throws IOException { + String query = "SELECT Id FROM " + + TEST_INDEX_BEER + " WHERE query('Tags:taste OR Body:taste')"; + JSONObject result = executeJdbcRequest(query); + assertEquals(16, result.getInt("total")); + } + + @Test + public void all_params_test() throws IOException { + String query = "SELECT Id FROM " + TEST_INDEX_BEER + + " WHERE query('Tags:taste', escape=false," + + "allow_leading_wildcard=true, enable_position_increments=true," + + "fuzziness= 1, fuzzy_rewrite='constant_score', max_determinized_states = 10000," + + "analyzer='standard', analyze_wildcard = false, quote_field_suffix = '.exact'," + + "auto_generate_synonyms_phrase_query=true, boost = 0.77," + + "quote_analyzer='standard', phrase_slop=0, rewrite='constant_score', type='best_fields'," + + "tie_breaker=0.3, time_zone='Canada/Pacific', default_operator='or'," + + "fuzzy_transpositions = false, lenient = true, fuzzy_max_expansions = 25," + + "minimum_should_match = '2<-25% 9<-3', fuzzy_prefix_length = 7);"; + JSONObject result = executeJdbcRequest(query); + assertEquals(8, result.getInt("total")); + } + + @Test + public void wildcard_test() throws IOException { + String query1 = "SELECT Id FROM " + + TEST_INDEX_BEER + " WHERE query('Tags:taste')"; + JSONObject result1 = executeJdbcRequest(query1); + String query2 = "SELECT Id FROM " + + TEST_INDEX_BEER + " WHERE query('*:taste')"; + JSONObject result2 = executeJdbcRequest(query2); + assertNotEquals(result2.getInt("total"), result1.getInt("total")); + + String query3 = "SELECT Id FROM " + TEST_INDEX_BEER + + " WHERE query('Tags:tas*');"; + JSONObject result3 = executeJdbcRequest(query3); + assertEquals(8, result3.getInt("total")); + + String query4 = "SELECT Id FROM " + TEST_INDEX_BEER + + " WHERE query('Tags:tas?e');"; + JSONObject result4 = executeJdbcRequest(query3); + assertEquals(8, result4.getInt("total")); + } + + @Test + public void query_string_and_query_return_the_same_results_test() throws IOException { + String query1 = "SELECT Id FROM " + + TEST_INDEX_BEER + " WHERE query('Tags:taste')"; + JSONObject result1 = executeJdbcRequest(query1); + String query2 = "SELECT Id FROM " + + TEST_INDEX_BEER + " WHERE query_string(['Tags'],'taste')"; + JSONObject result2 = executeJdbcRequest(query2); + assertEquals(result2.getInt("total"), result1.getInt("total")); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java index cf368c940d..8ef8a5e4a8 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java @@ -55,7 +55,7 @@ private CompositeValuesSourceBuilder buildCompositeValuesSourceBuilder( return buildHistogram( expr.getNameOrAlias(), spanExpr.getField().toString(), - spanExpr.getValue().valueOf(null).doubleValue(), + spanExpr.getValue().valueOf().doubleValue(), spanExpr.getUnit(), missingOrder); } else { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java index f4ff22dfbf..5e7d34abce 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java @@ -215,7 +215,7 @@ private Pair make(TopHitsAggregationBuilder bu MetricParser parser) { String fieldName = ((ReferenceExpression) expression).getAttr(); builder.fetchSource(fieldName, null); - builder.size(size.valueOf(null).integerValue()); + builder.size(size.valueOf().integerValue()); builder.from(0); if (condition != null) { return Pair.of( diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java index 68f8ec8c66..ab8fb562da 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java @@ -34,6 +34,7 @@ import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.MatchPhraseQuery; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.MatchQuery; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.MultiMatchQuery; +import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.QueryQuery; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.QueryStringQuery; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.SimpleQueryStringQuery; import org.opensearch.sql.opensearch.storage.serialization.ExpressionSerializer; @@ -60,7 +61,7 @@ public class FilterQueryBuilder extends ExpressionNodeVisitor>builder() .put(BuiltinFunctionName.CAST_TO_STRING.getName(), expr -> { if (!expr.type().equals(ExprCoreType.STRING)) { - return new ExprStringValue(String.valueOf(expr.valueOf(null).value())); + return new ExprStringValue(String.valueOf(expr.valueOf().value())); } else { - return expr.valueOf(null); + return expr.valueOf(); } }) .put(BuiltinFunctionName.CAST_TO_BYTE.getName(), expr -> { if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprByteValue(expr.valueOf(null).byteValue()); + return new ExprByteValue(expr.valueOf().byteValue()); } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprByteValue(expr.valueOf(null).booleanValue() ? 1 : 0); + return new ExprByteValue(expr.valueOf().booleanValue() ? 1 : 0); } else { - return new ExprByteValue(Byte.valueOf(expr.valueOf(null).stringValue())); + return new ExprByteValue(Byte.valueOf(expr.valueOf().stringValue())); } }) .put(BuiltinFunctionName.CAST_TO_SHORT.getName(), expr -> { if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprShortValue(expr.valueOf(null).shortValue()); + return new ExprShortValue(expr.valueOf().shortValue()); } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprShortValue(expr.valueOf(null).booleanValue() ? 1 : 0); + return new ExprShortValue(expr.valueOf().booleanValue() ? 1 : 0); } else { - return new ExprShortValue(Short.valueOf(expr.valueOf(null).stringValue())); + return new ExprShortValue(Short.valueOf(expr.valueOf().stringValue())); } }) .put(BuiltinFunctionName.CAST_TO_INT.getName(), expr -> { if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprIntegerValue(expr.valueOf(null).integerValue()); + return new ExprIntegerValue(expr.valueOf().integerValue()); } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprIntegerValue(expr.valueOf(null).booleanValue() ? 1 : 0); + return new ExprIntegerValue(expr.valueOf().booleanValue() ? 1 : 0); } else { - return new ExprIntegerValue(Integer.valueOf(expr.valueOf(null).stringValue())); + return new ExprIntegerValue(Integer.valueOf(expr.valueOf().stringValue())); } }) .put(BuiltinFunctionName.CAST_TO_LONG.getName(), expr -> { if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprLongValue(expr.valueOf(null).longValue()); + return new ExprLongValue(expr.valueOf().longValue()); } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprLongValue(expr.valueOf(null).booleanValue() ? 1 : 0); + return new ExprLongValue(expr.valueOf().booleanValue() ? 1 : 0); } else { - return new ExprLongValue(Long.valueOf(expr.valueOf(null).stringValue())); + return new ExprLongValue(Long.valueOf(expr.valueOf().stringValue())); } }) .put(BuiltinFunctionName.CAST_TO_FLOAT.getName(), expr -> { if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprFloatValue(expr.valueOf(null).floatValue()); + return new ExprFloatValue(expr.valueOf().floatValue()); } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprFloatValue(expr.valueOf(null).booleanValue() ? 1 : 0); + return new ExprFloatValue(expr.valueOf().booleanValue() ? 1 : 0); } else { - return new ExprFloatValue(Float.valueOf(expr.valueOf(null).stringValue())); + return new ExprFloatValue(Float.valueOf(expr.valueOf().stringValue())); } }) .put(BuiltinFunctionName.CAST_TO_DOUBLE.getName(), expr -> { if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprDoubleValue(expr.valueOf(null).doubleValue()); + return new ExprDoubleValue(expr.valueOf().doubleValue()); } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprDoubleValue(expr.valueOf(null).booleanValue() ? 1 : 0); + return new ExprDoubleValue(expr.valueOf().booleanValue() ? 1 : 0); } else { - return new ExprDoubleValue(Double.valueOf(expr.valueOf(null).stringValue())); + return new ExprDoubleValue(Double.valueOf(expr.valueOf().stringValue())); } }) .put(BuiltinFunctionName.CAST_TO_BOOLEAN.getName(), expr -> { if (ExprCoreType.numberTypes().contains(expr.type())) { - return expr.valueOf(null).doubleValue() == 1 + return expr.valueOf().doubleValue() != 0 ? ExprBooleanValue.of(true) : ExprBooleanValue.of(false); } else if (expr.type().equals(ExprCoreType.STRING)) { - return ExprBooleanValue.of(Boolean.valueOf(expr.valueOf(null).stringValue())); + return ExprBooleanValue.of(Boolean.valueOf(expr.valueOf().stringValue())); } else { - return expr.valueOf(null); + return expr.valueOf(); } }) .put(BuiltinFunctionName.CAST_TO_DATE.getName(), expr -> { if (expr.type().equals(ExprCoreType.STRING)) { - return new ExprDateValue(expr.valueOf(null).stringValue()); + return new ExprDateValue(expr.valueOf().stringValue()); } else { - return new ExprDateValue(expr.valueOf(null).dateValue()); + return new ExprDateValue(expr.valueOf().dateValue()); } }) .put(BuiltinFunctionName.CAST_TO_TIME.getName(), expr -> { if (expr.type().equals(ExprCoreType.STRING)) { - return new ExprTimeValue(expr.valueOf(null).stringValue()); + return new ExprTimeValue(expr.valueOf().stringValue()); } else { - return new ExprTimeValue(expr.valueOf(null).timeValue()); + return new ExprTimeValue(expr.valueOf().timeValue()); } }) .put(BuiltinFunctionName.CAST_TO_DATETIME.getName(), expr -> { if (expr.type().equals(ExprCoreType.STRING)) { - return new ExprDatetimeValue(expr.valueOf(null).stringValue()); + return new ExprDatetimeValue(expr.valueOf().stringValue()); } else { - return new ExprDatetimeValue(expr.valueOf(null).datetimeValue()); + return new ExprDatetimeValue(expr.valueOf().datetimeValue()); } }) .put(BuiltinFunctionName.CAST_TO_TIMESTAMP.getName(), expr -> { if (expr.type().equals(ExprCoreType.STRING)) { - return new ExprTimestampValue(expr.valueOf(null).stringValue()); + return new ExprTimestampValue(expr.valueOf().stringValue()); } else { - return new ExprTimestampValue(expr.valueOf(null).timestampValue()); + return new ExprTimestampValue(expr.valueOf().timestampValue()); } }) .build(); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQuery.java index 8390b5ef44..9f37951072 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQuery.java @@ -37,13 +37,13 @@ public T createQueryBuilder(List arguments) { var fieldsAndWeights = fields .getValue() - .valueOf(null) + .valueOf() .tupleValue() .entrySet() .stream() .collect(ImmutableMap.toImmutableMap(e -> e.getKey(), e -> e.getValue().floatValue())); - return createBuilder(fieldsAndWeights, query.getValue().valueOf(null).stringValue()); + return createBuilder(fieldsAndWeights, query.getValue().valueOf().stringValue()); } protected abstract T createBuilder(ImmutableMap fields, String query); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/NoFieldQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/NoFieldQuery.java new file mode 100644 index 0000000000..1467cf8e4b --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/NoFieldQuery.java @@ -0,0 +1,72 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.NamedArgumentExpression; + +/** + * Base class to represent relevance queries that have no 'fields' array as an argument. + * + * @param The builder class for the OpenSearch query. + */ +abstract class NoFieldQuery extends RelevanceQuery { + public NoFieldQuery(Map> queryBuildActions) { + super(queryBuildActions); + } + + @Override + protected void ignoreArguments(List arguments) { + arguments.removeIf(a -> a.getArgName().equalsIgnoreCase("query")); + } + + @Override + protected void checkValidArguments(String argNormalized, T queryBuilder) { + if (!getQueryBuildActions().containsKey(argNormalized)) { + throw new SemanticCheckException( + String.format("Parameter %s is invalid for %s function.", + argNormalized, getQueryName())); + } + } + /** + * Override build function because RelevanceQuery requires 2 fields, + * but NoFieldQuery must have no fields. + * + * @param func : Contains function name and passed in arguments. + * @return : QueryBuilder object + */ + + @Override + public QueryBuilder build(FunctionExpression func) { + var arguments = func.getArguments().stream().map( + a -> (NamedArgumentExpression) a).collect(Collectors.toList()); + if (arguments.size() < 1) { + throw new SyntaxCheckException(String.format( + "%s requires at least one parameter", func.getFunctionName())); + } + + return loadArguments(arguments); + } + + + @Override + public T createQueryBuilder(List arguments) { + // Extract 'query' + var query = arguments.stream().filter(a -> a.getArgName().equalsIgnoreCase("query")).findFirst() + .orElseThrow(() -> new SemanticCheckException("'query' parameter is missing")); + + return createBuilder(query.getValue().valueOf().stringValue()); + } + + protected abstract T createBuilder(String query); +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/QueryQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/QueryQuery.java new file mode 100644 index 0000000000..35d5a43a41 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/QueryQuery.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; + +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.QueryStringQueryBuilder; + +/** + * Class for Lucene query that builds the 'query' query. + */ +public class QueryQuery extends NoFieldQuery { + + private final String queryQueryName = "query"; + + /** + * Default constructor for QueryQuery configures how RelevanceQuery.build() handles + * named arguments by calling the constructor of QueryStringQuery. + */ + public QueryQuery() { + super(FunctionParameterRepository.QueryStringQueryBuildActions); + } + + /** + * Builds QueryBuilder with query value and other default parameter values set. + * + * @param query : Query value for query_string query + * @return : Builder for query query + */ + protected QueryStringQueryBuilder createBuilder(String query) { + return QueryBuilders.queryStringQuery(query); + } + + @Override + public String getQueryName() { + return queryQueryName; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQuery.java index 579f77d2cd..e07c0cd966 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQuery.java @@ -10,6 +10,7 @@ import java.util.Objects; import java.util.function.BiFunction; import java.util.stream.Collectors; +import lombok.Getter; import lombok.RequiredArgsConstructor; import org.opensearch.index.query.QueryBuilder; import org.opensearch.sql.common.antlr.SyntaxCheckException; @@ -24,17 +25,24 @@ */ @RequiredArgsConstructor public abstract class RelevanceQuery extends LuceneQuery { + @Getter private final Map> queryBuildActions; - @Override - public QueryBuilder build(FunctionExpression func) { - var arguments = func.getArguments().stream() - .map(a -> (NamedArgumentExpression)a).collect(Collectors.toList()); - if (arguments.size() < 2) { - throw new SyntaxCheckException( - String.format("%s requires at least two parameters", getQueryName())); + protected void ignoreArguments(List arguments) { + arguments.removeIf(a -> a.getArgName().equalsIgnoreCase("field") + || a.getArgName().equalsIgnoreCase("fields") + || a.getArgName().equalsIgnoreCase("query")); + } + + protected void checkValidArguments(String argNormalized, T queryBuilder) { + if (!queryBuildActions.containsKey(argNormalized)) { + throw new SemanticCheckException( + String.format("Parameter %s is invalid for %s function.", + argNormalized, queryBuilder.getWriteableName())); } + } + protected T loadArguments(List arguments) throws SemanticCheckException { // Aggregate parameters by name, so getting a Map arguments.stream().collect(Collectors.groupingBy(a -> a.getArgName().toLowerCase())) .forEach((k, v) -> { @@ -46,32 +54,45 @@ public QueryBuilder build(FunctionExpression func) { T queryBuilder = createQueryBuilder(arguments); - arguments.removeIf(a -> a.getArgName().equalsIgnoreCase("field") - || a.getArgName().equalsIgnoreCase("fields") - || a.getArgName().equalsIgnoreCase("query")); + ignoreArguments(arguments); var iterator = arguments.listIterator(); while (iterator.hasNext()) { NamedArgumentExpression arg = iterator.next(); String argNormalized = arg.getArgName().toLowerCase(); - if (!queryBuildActions.containsKey(argNormalized)) { - throw new SemanticCheckException( - String.format("Parameter %s is invalid for %s function.", - argNormalized, queryBuilder.getWriteableName())); - } + checkValidArguments(argNormalized, queryBuilder); + (Objects.requireNonNull( queryBuildActions .get(argNormalized))) - .apply(queryBuilder, arg.getValue().valueOf(null)); + .apply(queryBuilder, arg.getValue().valueOf()); } + return queryBuilder; } + @Override + public QueryBuilder build(FunctionExpression func) { + var arguments = func.getArguments().stream() + .map(a -> (NamedArgumentExpression)a).collect(Collectors.toList()); + if (arguments.size() < 2) { + throw new SyntaxCheckException( + String.format("%s requires at least two parameters", getQueryName())); + } + + return loadArguments(arguments); + + } + protected abstract T createQueryBuilder(List arguments); protected abstract String getQueryName(); + public Map> getQueryBuildActions() { + return queryBuildActions; + } + /** * Convenience interface for a function that updates a QueryBuilder * based on ExprValue. diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQuery.java index 15eda7f483..a7d7584d4f 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQuery.java @@ -36,8 +36,8 @@ protected T createQueryBuilder(List arguments) { .orElseThrow(() -> new SemanticCheckException("'query' parameter is missing")); return createBuilder( - field.getValue().valueOf(null).stringValue(), - query.getValue().valueOf(null).stringValue()); + field.getValue().valueOf().stringValue(), + query.getValue().valueOf().stringValue()); } protected abstract T createBuilder(String field, String query); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeRecognitionTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeRecognitionTest.java index 48121baad2..d7ec5bee5e 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeRecognitionTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeRecognitionTest.java @@ -42,6 +42,6 @@ private static Stream types() { } private String typeofGetValue(ExprValue input) { - return dsl.typeof(DSL.literal(input)).valueOf(null).stringValue(); + return dsl.typeof(DSL.literal(input)).valueOf().stringValue(); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java index ff80f3bcc0..bae4028986 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java @@ -48,7 +48,6 @@ import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValueUtils; -import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; @@ -64,18 +63,18 @@ class FilterQueryBuilderTest { private final DSL dsl = new ExpressionConfig().dsl(new ExpressionConfig().functionRepository()); private static Stream numericCastSource() { - return Stream.of(literal((byte) 1), literal((short) 1), literal( - 1), literal(1L), literal(1F), literal(1D), literal(true), literal("1")); + return Stream.of(literal((byte) 1), literal((short) -1), literal( + 1), literal(21L), literal(3.14F), literal(3.1415D), literal(true), literal("1")); } private static Stream booleanTrueCastSource() { - return Stream.of(literal((byte) 1), literal((short) 1), literal( - 1), literal(1L), literal(1F), literal(1D), literal(true), literal("true")); + return Stream.of(literal((byte) 1), literal((short) -1), literal( + 1), literal(42L), literal(3.14F), literal(3.1415D), literal(true), literal("true")); } private static Stream booleanFalseCastSource() { return Stream.of(literal((byte) 0), literal((short) 0), literal( - 0), literal(0L), literal(0F), literal(0D), literal(false), literal("false")); + 0), literal(0L), literal(0.0F), literal(0.0D), literal(false), literal("false")); } @Mock @@ -411,6 +410,15 @@ void match_missing_field() { assertEquals("'field' parameter is missing.", msg); } + @Test + void match_missing_query() { + FunctionExpression expr = dsl.match( + dsl.namedArgument("field", literal("field1")), + dsl.namedArgument("analyzer", literal("keyword"))); + var msg = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)).getMessage(); + assertEquals("'query' parameter is missing", msg); + } + @Test void should_build_match_phrase_query_with_default_parameters() { assertJsonEquals( @@ -626,6 +634,93 @@ void should_build_match_phrase_query_with_custom_parameters() { dsl.namedArgument("zero_terms_query", literal("ALL"))))); } + @Test + void query_invalid_parameter() { + FunctionExpression expr = dsl.query( + dsl.namedArgument("invalid_parameter", literal("invalid_value"))); + assertThrows(SemanticCheckException.class, () -> buildQuery(expr), + "Parameter invalid_parameter is invalid for query function."); + } + + @Test + void query_invalid_fields_parameter_exception_message() { + FunctionExpression expr = dsl.query( + dsl.namedArgument("fields", literal("field1")), + dsl.namedArgument("query", literal("search query"))); + + var exception = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)); + assertEquals("Parameter fields is invalid for query function.", exception.getMessage()); + } + + @Test + void should_build_query_query_with_default_parameters() { + var expected = "{\n" + + " \"query_string\" : {\n" + + " \"query\" : \"field1:query_value\",\n" + + " \"fields\" : [],\n" + + " \"type\" : \"best_fields\",\n" + + " \"default_operator\" : \"or\",\n" + + " \"max_determinized_states\" : 10000,\n" + + " \"enable_position_increments\" : true,\n" + + " \"fuzziness\" : \"AUTO\",\n" + + " \"fuzzy_prefix_length\" : 0,\n" + + " \"fuzzy_max_expansions\" : 50,\n" + + " \"phrase_slop\" : 0,\n" + + " \"escape\" : false,\n" + + " \"auto_generate_synonyms_phrase_query\" : true,\n" + + " \"fuzzy_transpositions\" : true,\n" + + " \"boost\" : 1.0\n" + + " }\n" + + "}"; + + assertJsonEquals(expected, buildQuery(dsl.query( + dsl.namedArgument("query", literal("field1:query_value"))))); + } + + @Test + void should_build_query_query_with_custom_parameters() { + var expected = "{\n" + + " \"query_string\" : {\n" + + " \"query\" : \"field1:query_value\",\n" + + " \"fields\" : [],\n" + + " \"type\" : \"cross_fields\",\n" + + " \"tie_breaker\" : 1.3,\n" + + " \"default_operator\" : \"and\",\n" + + " \"analyzer\" : \"keyword\",\n" + + " \"max_determinized_states\" : 10000,\n" + + " \"enable_position_increments\" : true,\n" + + " \"fuzziness\" : \"AUTO\",\n" + + " \"fuzzy_prefix_length\" : 2,\n" + + " \"fuzzy_max_expansions\" : 10,\n" + + " \"phrase_slop\" : 0,\n" + + " \"analyze_wildcard\" : true,\n" + + " \"minimum_should_match\" : \"3\",\n" + + " \"lenient\" : false,\n" + + " \"escape\" : false,\n" + + " \"auto_generate_synonyms_phrase_query\" : false,\n" + + " \"fuzzy_transpositions\" : false,\n" + + " \"boost\" : 2.0,\n" + + " }\n" + + "}"; + var actual = buildQuery( + dsl.query( + dsl.namedArgument("query", literal("field1:query_value")), + dsl.namedArgument("analyze_wildcard", literal("true")), + dsl.namedArgument("analyzer", literal("keyword")), + dsl.namedArgument("auto_generate_synonyms_phrase_query", literal("false")), + dsl.namedArgument("default_operator", literal("AND")), + dsl.namedArgument("fuzzy_max_expansions", literal("10")), + dsl.namedArgument("fuzzy_prefix_length", literal("2")), + dsl.namedArgument("fuzzy_transpositions", literal("false")), + dsl.namedArgument("lenient", literal("false")), + dsl.namedArgument("minimum_should_match", literal("3")), + dsl.namedArgument("tie_breaker", literal("1.3")), + dsl.namedArgument("type", literal("cross_fields")), + dsl.namedArgument("boost", literal("2.0")))); + + assertJsonEquals(expected, actual); + } + @Test void query_string_invalid_parameter() { FunctionExpression expr = dsl.query_string( @@ -981,6 +1076,18 @@ void multi_match_missing_fields_even_with_struct() { assertEquals("'fields' parameter is missing.", msg); } + @Test + void multi_match_missing_query_even_with_struct() { + FunctionExpression expr = dsl.multi_match( + dsl.namedArgument("fields", DSL.literal( + new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( + "field1", ExprValueUtils.floatValue(1.F), + "field2", ExprValueUtils.floatValue(.3F)))))), + dsl.namedArgument("analyzer", literal("keyword"))); + var msg = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)).getMessage(); + assertEquals("'query' parameter is missing", msg); + } + @Test void should_build_match_phrase_prefix_query_with_default_parameters() { assertJsonEquals( @@ -1042,93 +1149,126 @@ void cast_to_string_in_filter() { dsl.equal(ref("string_value", STRING), dsl.castString(literal("1"))))); } + private Float castToFloat(Object o) { + if (o instanceof Number) { + return ((Number)o).floatValue(); + } + if (o instanceof String) { + return Float.parseFloat((String) o); + } + if (o instanceof Boolean) { + return ((Boolean)o) ? 1F : 0F; + } + // unreachable code + throw new IllegalArgumentException(); + } + + private Integer castToInteger(Object o) { + if (o instanceof Number) { + return ((Number)o).intValue(); + } + if (o instanceof String) { + return Integer.parseInt((String) o); + } + if (o instanceof Boolean) { + return ((Boolean)o) ? 1 : 0; + } + // unreachable code + throw new IllegalArgumentException(); + } + @ParameterizedTest(name = "castByte({0})") @MethodSource({"numericCastSource"}) void cast_to_byte_in_filter(LiteralExpression expr) { - assertJsonEquals( + assertJsonEquals(String.format( "{\n" + " \"term\" : {\n" + " \"byte_value\" : {\n" - + " \"value\" : 1,\n" + + " \"value\" : %d,\n" + " \"boost\" : 1.0\n" + " }\n" + " }\n" - + "}", + + "}", castToInteger(expr.valueOf().value())), buildQuery(dsl.equal(ref("byte_value", BYTE), dsl.castByte(expr)))); } @ParameterizedTest(name = "castShort({0})") @MethodSource({"numericCastSource"}) void cast_to_short_in_filter(LiteralExpression expr) { - assertJsonEquals( + assertJsonEquals(String.format( "{\n" + " \"term\" : {\n" + " \"short_value\" : {\n" - + " \"value\" : 1,\n" + + " \"value\" : %d,\n" + " \"boost\" : 1.0\n" + " }\n" + " }\n" - + "}", + + "}", castToInteger(expr.valueOf().value())), buildQuery(dsl.equal(ref("short_value", SHORT), dsl.castShort(expr)))); } @ParameterizedTest(name = "castInt({0})") @MethodSource({"numericCastSource"}) void cast_to_int_in_filter(LiteralExpression expr) { - assertJsonEquals( + assertJsonEquals(String.format( "{\n" + " \"term\" : {\n" + " \"integer_value\" : {\n" - + " \"value\" : 1,\n" + + " \"value\" : %d,\n" + " \"boost\" : 1.0\n" + " }\n" + " }\n" - + "}", + + "}", castToInteger(expr.valueOf().value())), buildQuery(dsl.equal(ref("integer_value", INTEGER), dsl.castInt(expr)))); } @ParameterizedTest(name = "castLong({0})") @MethodSource({"numericCastSource"}) void cast_to_long_in_filter(LiteralExpression expr) { - assertJsonEquals( + assertJsonEquals(String.format( "{\n" + " \"term\" : {\n" + " \"long_value\" : {\n" - + " \"value\" : 1,\n" + + " \"value\" : %d,\n" + " \"boost\" : 1.0\n" + " }\n" + " }\n" - + "}", + + "}", castToInteger(expr.valueOf().value())), buildQuery(dsl.equal(ref("long_value", LONG), dsl.castLong(expr)))); } @ParameterizedTest(name = "castFloat({0})") @MethodSource({"numericCastSource"}) void cast_to_float_in_filter(LiteralExpression expr) { - assertJsonEquals( + assertJsonEquals(String.format( "{\n" + " \"term\" : {\n" + " \"float_value\" : {\n" - + " \"value\" : 1.0,\n" + + " \"value\" : %f,\n" + " \"boost\" : 1.0\n" + " }\n" + " }\n" - + "}", + + "}", castToFloat(expr.valueOf().value())), buildQuery(dsl.equal(ref("float_value", FLOAT), dsl.castFloat(expr)))); } @ParameterizedTest(name = "castDouble({0})") @MethodSource({"numericCastSource"}) void cast_to_double_in_filter(LiteralExpression expr) { - assertJsonEquals( + // double values affected by floating point imprecision, so we can't compare them in json + // (Double)(Float)3.14 -> 3.14000010490417 + assertEquals(castToFloat(expr.valueOf().value()), + dsl.castDouble(expr).valueOf().doubleValue(), 0.00001); + + assertJsonEquals(String.format( "{\n" + " \"term\" : {\n" + " \"double_value\" : {\n" - + " \"value\" : 1.0,\n" + + " \"value\" : %2.20f,\n" + " \"boost\" : 1.0\n" + " }\n" + " }\n" - + "}", + + "}", dsl.castDouble(expr).valueOf().doubleValue()), buildQuery(dsl.equal(ref("double_value", DOUBLE), dsl.castDouble(expr)))); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/QueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/QueryTest.java new file mode 100644 index 0000000000..e0681bceac --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/QueryTest.java @@ -0,0 +1,150 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.script.filter.lucene; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.DisplayNameGeneration; +import org.junit.jupiter.api.DisplayNameGenerator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.LiteralExpression; +import org.opensearch.sql.expression.NamedArgumentExpression; +import org.opensearch.sql.expression.config.ExpressionConfig; +import org.opensearch.sql.expression.env.Environment; +import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.QueryQuery; + +@DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) +class QueryTest { + private static final DSL dsl = new ExpressionConfig() + .dsl(new ExpressionConfig().functionRepository()); + private final QueryQuery queryQuery = new QueryQuery(); + private final FunctionName queryFunc = FunctionName.of("query"); + private static final LiteralExpression query_value = DSL.literal("title:query_value"); + + static Stream> generateValidData() { + Expression query = dsl.namedArgument("query", query_value); + return List.of( + dsl.namedArgument("analyzer", DSL.literal("standard")), + dsl.namedArgument("analyze_wildcard", DSL.literal("true")), + dsl.namedArgument("allow_leading_wildcard", DSL.literal("true")), + dsl.namedArgument("auto_generate_synonyms_phrase_query", DSL.literal("true")), + dsl.namedArgument("boost", DSL.literal("1")), + dsl.namedArgument("default_operator", DSL.literal("AND")), + dsl.namedArgument("default_operator", DSL.literal("and")), + dsl.namedArgument("enable_position_increments", DSL.literal("true")), + dsl.namedArgument("escape", DSL.literal("false")), + dsl.namedArgument("fuzziness", DSL.literal("1")), + dsl.namedArgument("fuzzy_rewrite", DSL.literal("constant_score")), + dsl.namedArgument("fuzzy_max_expansions", DSL.literal("42")), + dsl.namedArgument("fuzzy_prefix_length", DSL.literal("42")), + dsl.namedArgument("fuzzy_transpositions", DSL.literal("true")), + dsl.namedArgument("lenient", DSL.literal("true")), + dsl.namedArgument("max_determinized_states", DSL.literal("10000")), + dsl.namedArgument("minimum_should_match", DSL.literal("4")), + dsl.namedArgument("quote_analyzer", DSL.literal("standard")), + dsl.namedArgument("phrase_slop", DSL.literal("0")), + dsl.namedArgument("quote_field_suffix", DSL.literal(".exact")), + dsl.namedArgument("rewrite", DSL.literal("constant_score")), + dsl.namedArgument("type", DSL.literal("best_fields")), + dsl.namedArgument("tie_breaker", DSL.literal("0.3")), + dsl.namedArgument("time_zone", DSL.literal("Canada/Pacific")), + dsl.namedArgument("ANALYZER", DSL.literal("standard")), + dsl.namedArgument("ANALYZE_wildcard", DSL.literal("true")), + dsl.namedArgument("Allow_Leading_wildcard", DSL.literal("true")), + dsl.namedArgument("Auto_Generate_Synonyms_Phrase_Query", DSL.literal("true")), + dsl.namedArgument("Boost", DSL.literal("1")) + ).stream().map(arg -> List.of(query, arg)); + } + + @ParameterizedTest + @MethodSource("generateValidData") + public void test_valid_parameters(List validArgs) { + Assertions.assertNotNull(queryQuery.build( + new QueryExpression(validArgs))); + } + + @Test + public void test_SyntaxCheckException_when_no_arguments() { + List arguments = List.of(); + assertThrows(SyntaxCheckException.class, + () -> queryQuery.build(new QueryExpression(arguments))); + } + + @Test + public void test_SyntaxCheckException_when_field_argument() { + List arguments = List.of( + namedArgument("fields", "invalid argument"), + namedArgument("query", query_value)); + assertThrows(SemanticCheckException.class, + () -> queryQuery.build(new QueryExpression(arguments))); + } + + @Test + public void test_SemanticCheckException_when_invalid_parameter() { + List arguments = List.of( + namedArgument("query", query_value), + namedArgument("unsupported", "unsupported_value")); + Assertions.assertThrows(SemanticCheckException.class, + () -> queryQuery.build(new QueryExpression(arguments))); + } + + @Test + public void test_SemanticCheckException_when_sending_parameter_multiple_times() { + List arguments = List.of( + namedArgument("query", query_value), + namedArgument("allow_leading_wildcard", DSL.literal("true")), + namedArgument("allow_leading_wildcard", DSL.literal("true"))); + Assertions.assertThrows(SemanticCheckException.class, + () -> queryQuery.build(new QueryExpression(arguments))); + } + + private NamedArgumentExpression namedArgument(String name, String value) { + return dsl.namedArgument(name, DSL.literal(value)); + } + + private NamedArgumentExpression namedArgument(String name, LiteralExpression value) { + return dsl.namedArgument(name, value); + } + + private class QueryExpression extends FunctionExpression { + public QueryExpression(List arguments) { + super(QueryTest.this.queryFunc, arguments); + } + + @Override + public ExprValue valueOf(Environment valueEnv) { + throw new UnsupportedOperationException("Invalid function call, " + + "valueOf function need implementation only to support Expression interface"); + } + + @Override + public ExprType type() { + throw new UnsupportedOperationException("Invalid function call, " + + "type function need implementation only to support Expression interface"); + } + } + + @Test + public void test_can_get_query_name() { + List arguments = List.of(namedArgument("query", query_value)); + queryQuery.build(new QueryExpression(arguments)); + Assertions.assertEquals("query", + queryQuery.getQueryName()); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/NoFieldQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/NoFieldQueryTest.java new file mode 100644 index 0000000000..e06dd7d32a --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/NoFieldQueryTest.java @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; + +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.LiteralExpression; +import org.opensearch.sql.expression.config.ExpressionConfig; + +class NoFieldQueryTest { + NoFieldQuery query; + private final DSL dsl = new ExpressionConfig().dsl(new ExpressionConfig().functionRepository()); + private final String testQueryName = "test_query"; + private final Map actionMap + = ImmutableMap.of("paramA", (o, v) -> o); + + @BeforeEach + void setUp() { + query = mock(NoFieldQuery.class, + Mockito.withSettings().useConstructor(actionMap) + .defaultAnswer(Mockito.CALLS_REAL_METHODS)); + when(query.getQueryName()).thenReturn(testQueryName); + } + + @Test + void createQueryBuilderTest() { + String sampleQuery = "field:query"; + + query.createQueryBuilder(List.of( + dsl.namedArgument("query", + new LiteralExpression(ExprValueUtils.stringValue(sampleQuery))))); + + verify(query).createBuilder(eq(sampleQuery)); + } +} diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogSettings.java b/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogSettings.java index 20efce1b7a..558e7558ca 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogSettings.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogSettings.java @@ -12,6 +12,6 @@ public class CatalogSettings { public static final Setting CATALOG_CONFIG = SecureSetting.secureFile( - "plugins.query.federation.catalog.config", + "plugins.query.federation.datasources.config", null); } diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java index 07ee458e5c..cdbce55cb1 100644 --- a/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java +++ b/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java @@ -30,7 +30,7 @@ public class CatalogServiceImplTest { public static final String CATALOG_SETTING_METADATA_KEY = - "plugins.query.federation.catalog.config"; + "plugins.query.federation.datasources.config"; @Mock private StorageEngine storageEngine; diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 79c812949f..75b99f2daf 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -43,7 +43,7 @@ SOURCE: 'SOURCE'; INDEX: 'INDEX'; D: 'D'; DESC: 'DESC'; -CATALOGS: 'CATALOGS'; +CATALOGS: 'DATASOURCES'; // CLAUSE KEYWORDS SORTBY: 'SORTBY'; diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index 658bf1d295..1e6cc5a7b9 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -800,7 +800,7 @@ public void test_batchRCFADCommand() { @Test public void testShowCatalogsCommand() { - assertEqual("show catalogs", + assertEqual("show datasources", relation(".CATALOGS")); } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java index 4a469c7bbb..512c014564 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java @@ -106,7 +106,8 @@ private JSONObject readResponse(Response response) throws IOException { } } else { throw new RuntimeException( - String.format("Request to Prometheus is Unsuccessful with : %s", response.message())); + String.format("Request to Prometheus is Unsuccessful with : %s", Objects.requireNonNull( + response.body(), "Response body can't be null").string())); } } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/functions/implementation/QueryRangeFunctionImplementation.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/functions/implementation/QueryRangeFunctionImplementation.java index bccb9c3bff..2d3710037a 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/functions/implementation/QueryRangeFunctionImplementation.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/functions/implementation/QueryRangeFunctionImplementation.java @@ -83,7 +83,7 @@ private PrometheusQueryRequest buildQueryFromQueryRangeFunction(List arguments.forEach(arg -> { String argName = ((NamedArgumentExpression) arg).getArgName(); Expression argValue = ((NamedArgumentExpression) arg).getValue(); - ExprValue literalValue = argValue.valueOf(null); + ExprValue literalValue = argValue.valueOf(); switch (argName) { case QUERY: prometheusQueryRequest diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/response/PrometheusResponse.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/response/PrometheusResponse.java index e26e006403..ef7f19ba2f 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/response/PrometheusResponse.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/response/PrometheusResponse.java @@ -5,7 +5,6 @@ package org.opensearch.sql.prometheus.response; -import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.LONG; import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.LABELS; @@ -16,6 +15,7 @@ import java.util.LinkedHashMap; import java.util.List; import lombok.NonNull; +import org.apache.commons.lang3.StringUtils; import org.json.JSONArray; import org.json.JSONObject; import org.opensearch.sql.data.model.ExprDoubleValue; @@ -26,6 +26,8 @@ import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.expression.NamedExpression; +import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.prometheus.storage.model.PrometheusResponseFieldNames; public class PrometheusResponse implements Iterable { @@ -100,7 +102,7 @@ public Iterator iterator() { private void insertLabels(LinkedHashMap linkedHashMap, JSONObject metric) { for (String key : metric.keySet()) { - linkedHashMap.put(key, new ExprStringValue(metric.getString(key))); + linkedHashMap.put(getKey(key), new ExprStringValue(metric.getString(key))); } } @@ -113,4 +115,18 @@ private ExprValue getValue(JSONArray jsonArray, Integer index, ExprType exprType return new ExprDoubleValue(jsonArray.getDouble(index)); } + private String getKey(String key) { + if (this.prometheusResponseFieldNames.getGroupByList() == null) { + return key; + } else { + return this.prometheusResponseFieldNames.getGroupByList().stream() + .filter(expression -> expression.getDelegated() instanceof ReferenceExpression) + .filter(expression + -> ((ReferenceExpression) expression.getDelegated()).getAttr().equals(key)) + .findFirst() + .map(NamedExpression::getName) + .orElse(key); + } + } + } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/implementor/PrometheusDefaultImplementor.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/implementor/PrometheusDefaultImplementor.java index 071cd7ba8c..8cae250e5e 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/implementor/PrometheusDefaultImplementor.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/implementor/PrometheusDefaultImplementor.java @@ -7,10 +7,6 @@ package org.opensearch.sql.prometheus.storage.implementor; -import static org.opensearch.sql.data.type.ExprCoreType.STRING; -import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.LABELS; - -import java.util.ArrayList; import java.util.List; import java.util.Optional; import lombok.RequiredArgsConstructor; @@ -18,14 +14,11 @@ import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.NamedExpression; -import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.span.SpanExpression; import org.opensearch.sql.planner.DefaultImplementor; import org.opensearch.sql.planner.logical.LogicalPlan; -import org.opensearch.sql.planner.logical.LogicalProject; import org.opensearch.sql.planner.logical.LogicalRelation; import org.opensearch.sql.planner.physical.PhysicalPlan; -import org.opensearch.sql.planner.physical.ProjectOperator; import org.opensearch.sql.prometheus.planner.logical.PrometheusLogicalMetricAgg; import org.opensearch.sql.prometheus.planner.logical.PrometheusLogicalMetricScan; import org.opensearch.sql.prometheus.storage.PrometheusMetricScan; @@ -130,6 +123,7 @@ private void setPrometheusResponseFieldNames(PrometheusLogicalMetricAgg node, prometheusResponseFieldNames.setValueFieldName(node.getAggregatorList().get(0).getName()); prometheusResponseFieldNames.setValueType(node.getAggregatorList().get(0).type()); prometheusResponseFieldNames.setTimestampFieldName(spanExpression.get().getNameOrAlias()); + prometheusResponseFieldNames.setGroupByList(node.getGroupByList()); context.setPrometheusResponseFieldNames(prometheusResponseFieldNames); } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/model/PrometheusResponseFieldNames.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/model/PrometheusResponseFieldNames.java index 4276848aa2..d3a6ef184f 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/model/PrometheusResponseFieldNames.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/model/PrometheusResponseFieldNames.java @@ -11,9 +11,11 @@ import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.TIMESTAMP; import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.VALUE; +import java.util.List; import lombok.Getter; import lombok.Setter; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.expression.NamedExpression; @Getter @@ -23,5 +25,6 @@ public class PrometheusResponseFieldNames { private String valueFieldName = VALUE; private ExprType valueType = DOUBLE; private String timestampFieldName = TIMESTAMP; + private List groupByList; } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/AggregationQueryBuilder.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/AggregationQueryBuilder.java index 1aff9eca88..76c8c6872e 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/AggregationQueryBuilder.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/AggregationQueryBuilder.java @@ -7,11 +7,14 @@ package org.opensearch.sql.prometheus.storage.querybuilder; +import java.sql.Ref; import java.util.List; import java.util.Set; import java.util.stream.Collectors; import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; import org.opensearch.sql.expression.NamedExpression; +import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.aggregation.NamedAggregator; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.span.SpanExpression; @@ -63,7 +66,10 @@ public static String build(List namedAggregatorList, if (groupByList.size() > 0) { aggregateQuery.append("by("); aggregateQuery.append( - groupByList.stream().map(NamedExpression::getName).collect(Collectors.joining(", "))); + groupByList.stream() + .filter(expression -> expression.getDelegated() instanceof ReferenceExpression) + .map(expression -> ((ReferenceExpression) expression.getDelegated()).getAttr()) + .collect(Collectors.joining(", "))); aggregateQuery.append(")"); } } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/TimeRangeParametersResolver.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/TimeRangeParametersResolver.java index 6c338d61a6..810ed71379 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/TimeRangeParametersResolver.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/TimeRangeParametersResolver.java @@ -57,7 +57,7 @@ public Void visitFunction(FunctionExpression func, Object context) { ReferenceExpression ref = (ReferenceExpression) func.getArguments().get(0); Expression rightExpr = func.getArguments().get(1); if (ref.getAttr().equals("@timestamp")) { - ExprValue literalValue = rightExpr.valueOf(null); + ExprValue literalValue = rightExpr.valueOf(); if (func.getFunctionName().getFunctionName().contains(">")) { startTime = literalValue.timestampValue().toEpochMilli() / 1000; } diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeFunctionImplementationTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeFunctionImplementationTest.java index f2a54b7347..98195eecf7 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeFunctionImplementationTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeFunctionImplementationTest.java @@ -48,7 +48,7 @@ void testValueOfAndTypeAndToString() { QueryRangeFunctionImplementation queryRangeFunctionImplementation = new QueryRangeFunctionImplementation(functionName, namedArgumentExpressionList, client); UnsupportedOperationException exception = assertThrows(UnsupportedOperationException.class, - () -> queryRangeFunctionImplementation.valueOf(null)); + () -> queryRangeFunctionImplementation.valueOf()); assertEquals("Prometheus defined function [query_range] is only " + "supported in SOURCE clause with prometheus connector catalog", exception.getMessage()); assertEquals("query_range(query=\"http_latency\", starttime=12345, endtime=12345, step=14)", diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScanTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScanTest.java index ac99a996af..cb70e9e064 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScanTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScanTest.java @@ -11,6 +11,7 @@ import static org.mockito.Mockito.when; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.LONG; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.prometheus.constants.TestConstants.ENDTIME; import static org.opensearch.sql.prometheus.constants.TestConstants.QUERY; import static org.opensearch.sql.prometheus.constants.TestConstants.STARTTIME; @@ -22,6 +23,7 @@ import java.io.IOException; import java.time.Instant; +import java.util.Collections; import java.util.LinkedHashMap; import lombok.SneakyThrows; import org.json.JSONObject; @@ -36,6 +38,7 @@ import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.expression.DSL; import org.opensearch.sql.prometheus.client.PrometheusClient; import org.opensearch.sql.prometheus.storage.model.PrometheusResponseFieldNames; @@ -163,6 +166,49 @@ void testQueryResponseIteratorWithGivenPrometheusResponseWithLongInAggType() { Assertions.assertFalse(prometheusMetricScan.hasNext()); } + @Test + @SneakyThrows + void testQueryResponseIteratorWithGivenPrometheusResponseWithBackQuotedFieldNames() { + PrometheusResponseFieldNames prometheusResponseFieldNames + = new PrometheusResponseFieldNames(); + prometheusResponseFieldNames.setValueFieldName("testAgg"); + prometheusResponseFieldNames.setValueType(LONG); + prometheusResponseFieldNames.setTimestampFieldName(TIMESTAMP); + prometheusResponseFieldNames.setGroupByList( + Collections.singletonList(DSL.named("`instance`", DSL.ref("instance", STRING)))); + PrometheusMetricScan prometheusMetricScan = new PrometheusMetricScan(prometheusClient); + prometheusMetricScan.setPrometheusResponseFieldNames(prometheusResponseFieldNames); + prometheusMetricScan.getRequest().setPromQl(QUERY); + prometheusMetricScan.getRequest().setStartTime(STARTTIME); + prometheusMetricScan.getRequest().setEndTime(ENDTIME); + prometheusMetricScan.getRequest().setStep(STEP); + + when(prometheusClient.queryRange(any(), any(), any(), any())) + .thenReturn(new JSONObject(getJson("query_range_result.json"))); + prometheusMetricScan.open(); + Assertions.assertTrue(prometheusMetricScan.hasNext()); + ExprTupleValue firstRow = new ExprTupleValue(new LinkedHashMap<>() {{ + put(TIMESTAMP, new ExprTimestampValue(Instant.ofEpochMilli(1435781430781L))); + put("testAgg", new ExprLongValue(1)); + put("`instance`", new ExprStringValue("localhost:9090")); + put("__name__", new ExprStringValue("up")); + put("job", new ExprStringValue("prometheus")); + } + }); + assertEquals(firstRow, prometheusMetricScan.next()); + Assertions.assertTrue(prometheusMetricScan.hasNext()); + ExprTupleValue secondRow = new ExprTupleValue(new LinkedHashMap<>() {{ + put(TIMESTAMP, new ExprTimestampValue(Instant.ofEpochMilli(1435781430781L))); + put("testAgg", new ExprLongValue(0)); + put("`instance`", new ExprStringValue("localhost:9091")); + put("__name__", new ExprStringValue("up")); + put("job", new ExprStringValue("node")); + } + }); + assertEquals(secondRow, prometheusMetricScan.next()); + Assertions.assertFalse(prometheusMetricScan.hasNext()); + } + @Test @SneakyThrows void testQueryResponseIteratorForQueryRangeFunction() { @@ -247,6 +293,7 @@ void testEmptyQueryWithException() { runtimeException.getMessage()); } + @Test @SneakyThrows void testExplain() { diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java index 3acae0c493..e106fa225b 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java @@ -750,4 +750,31 @@ void testUnsupportedOperation() { assertThrows(UnsupportedOperationException.class, () -> prometheusMetricTable.create(Collections.emptyMap())); } + + @Test + void testImplementPrometheusQueryWithBackQuotedFieldNamesInStatsQuery() { + + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, "prometheus_http_total_requests"); + + + // IndexScanAgg with Filter + PhysicalPlan plan = prometheusMetricTable.implement( + indexScanAgg("prometheus_http_total_requests", + dsl.and(dsl.equal(DSL.ref("code", STRING), DSL.literal(stringValue("200"))), + dsl.equal(DSL.ref("handler", STRING), DSL.literal(stringValue("/ready/")))), + ImmutableList + .of(named("AVG(@value)", + dsl.avg(DSL.ref("@value", INTEGER)))), + ImmutableList.of(named("`job`", DSL.ref("job", STRING)), + named("span", DSL.span(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal(40), "s"))))); + assertTrue(plan instanceof PrometheusMetricScan); + PrometheusQueryRequest prometheusQueryRequest = ((PrometheusMetricScan) plan).getRequest(); + assertEquals( + "avg by(job) (avg_over_time" + + "(prometheus_http_total_requests{code=\"200\" , handler=\"/ready/\"}[40s]))", + prometheusQueryRequest.getPromQl()); + + } } diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 4dedaaf396..c803f2b5c3 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -329,7 +329,11 @@ specificFunction ; relevanceFunction - : singleFieldRelevanceFunction | multiFieldRelevanceFunction + : noFieldRelevanceFunction | singleFieldRelevanceFunction | multiFieldRelevanceFunction + ; + +noFieldRelevanceFunction + : noFieldRelevanceFunctionName LR_BRACKET query=relevanceQuery (COMMA relevanceArg)* RR_BRACKET ; // Field is a single column @@ -412,6 +416,10 @@ flowControlFunctionName : IF | IFNULL | NULLIF | ISNULL ; +noFieldRelevanceFunctionName + : QUERY + ; + systemFunctionName : TYPEOF ; diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java index 0d551e3f38..131b6d9116 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java @@ -380,6 +380,14 @@ public UnresolvedExpression visitConvertedDataType( return AstDSL.stringLiteral(ctx.getText()); } + @Override + public UnresolvedExpression visitNoFieldRelevanceFunction( + OpenSearchSQLParser.NoFieldRelevanceFunctionContext ctx) { + return new Function( + ctx.noFieldRelevanceFunctionName().getText().toLowerCase(), + noFieldRelevanceArguments(ctx)); + } + @Override public UnresolvedExpression visitSingleFieldRelevanceFunction( SingleFieldRelevanceFunctionContext ctx) { @@ -429,6 +437,24 @@ private QualifiedName visitIdentifiers(List identifiers) { ); } + private void fillRelevanceArgs(List args, + ImmutableList.Builder builder) { + args.forEach(v -> builder.add(new UnresolvedArgument( + v.relevanceArgName().getText().toLowerCase(), new Literal(StringUtils.unquoteText( + v.relevanceArgValue().getText()), DataType.STRING)))); + } + + private List noFieldRelevanceArguments( + OpenSearchSQLParser.NoFieldRelevanceFunctionContext ctx) { + // all the arguments are defaulted to string values + // to skip environment resolving and function signature resolving + ImmutableList.Builder builder = ImmutableList.builder(); + builder.add(new UnresolvedArgument("query", + new Literal(StringUtils.unquoteText(ctx.query.getText()), DataType.STRING))); + fillRelevanceArgs(ctx.relevanceArg(), builder); + return builder.build(); + } + private List singleFieldRelevanceArguments( OpenSearchSQLParser.SingleFieldRelevanceFunctionContext ctx) { // all the arguments are defaulted to string values @@ -438,12 +464,12 @@ private List singleFieldRelevanceArguments( new Literal(StringUtils.unquoteText(ctx.field.getText()), DataType.STRING))); builder.add(new UnresolvedArgument("query", new Literal(StringUtils.unquoteText(ctx.query.getText()), DataType.STRING))); - ctx.relevanceArg().forEach(v -> builder.add(new UnresolvedArgument( - v.relevanceArgName().getText().toLowerCase(), new Literal(StringUtils.unquoteText( - v.relevanceArgValue().getText()), DataType.STRING)))); + fillRelevanceArgs(ctx.relevanceArg(), builder); return builder.build(); } + + private List multiFieldRelevanceArguments( OpenSearchSQLParser.MultiFieldRelevanceFunctionContext ctx) { // all the arguments are defaulted to string values @@ -458,9 +484,7 @@ private List multiFieldRelevanceArguments( builder.add(new UnresolvedArgument("fields", fields)); builder.add(new UnresolvedArgument("query", new Literal(StringUtils.unquoteText(ctx.query.getText()), DataType.STRING))); - ctx.relevanceArg().forEach(v -> builder.add(new UnresolvedArgument( - v.relevanceArgName().getText().toLowerCase(), new Literal(StringUtils.unquoteText( - v.relevanceArgValue().getText()), DataType.STRING)))); + fillRelevanceArgs(ctx.relevanceArg(), builder); return builder.build(); } } diff --git a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java index af428bdc52..6b78376d45 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -325,6 +325,49 @@ public void can_parse_query_string_relevance_function() { + "operator='AND', tie_breaker=0.3, type = \"most_fields\", fuzziness = 4)")); } + + @Test + public void can_parse_query_relevance_function() { + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query('address:query')")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query('address:query OR notes:query')")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query(\"address:query\")")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query(\"address:query OR notes:query\")")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query(`address:query`)")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query(`address:query OR notes:query`)")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query('*:query')")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query(\"*:query\")")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query(`*:query`)")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query('address:*uery OR notes:?uery')")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query(\"address:*uery OR notes:?uery\")")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query(`address:*uery OR notes:?uery`)")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query('address:qu*ry OR notes:qu?ry')")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query(\"address:qu*ry OR notes:qu?ry\")")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query(`address:qu*ry OR notes:qu?ry`)")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query('address:query notes:query')")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query(\"address:query notes:query\")")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE " + + "query(\"Body:\'taste beer\' Tags:\'taste beer\' Title:\'taste beer\'\")")); + } + + @Test public void can_parse_match_relevance_function() { assertNotNull(parser.parse("SELECT * FROM test WHERE match(column, \"this is a test\")")); diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java index ec0a0dd0d3..cb00ea2f18 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java @@ -543,6 +543,22 @@ public void relevanceQuery_string() { + "analyzer='keyword', time_zone='Canada/Pacific', tie_breaker='1.3')")); } + @Test + public void relevanceQuery() { + assertEquals(AstDSL.function("query", + unresolvedArg("query", stringLiteral("field1:query OR field2:query"))), + buildExprAst("query('field1:query OR field2:query')") + ); + + assertEquals(AstDSL.function("query", + unresolvedArg("query", stringLiteral("search query")), + unresolvedArg("analyzer", stringLiteral("keyword")), + unresolvedArg("time_zone", stringLiteral("Canada/Pacific")), + unresolvedArg("tie_breaker", stringLiteral("1.3"))), + buildExprAst("query('search query'," + + "analyzer='keyword', time_zone='Canada/Pacific', tie_breaker='1.3')")); + } + @Test public void canBuildInClause() { assertEquals( From faff812bafe3a36ee868d48dc574c1079dc9b5cd Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Fri, 11 Nov 2022 19:31:54 -0800 Subject: [PATCH 23/23] fix bug Signed-off-by: Peng Huo --- .../opensearch/sql/executor/QueryService.java | 6 +++- .../sql/executor/QueryServiceTest.java | 28 ++++++++++++++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryService.java b/core/src/main/java/org/opensearch/sql/executor/QueryService.java index 32fda1ab2c..dcdf6bc010 100644 --- a/core/src/main/java/org/opensearch/sql/executor/QueryService.java +++ b/core/src/main/java/org/opensearch/sql/executor/QueryService.java @@ -39,7 +39,11 @@ public class QueryService { */ public void execute(UnresolvedPlan plan, ResponseListener listener) { - executePlan(analyze(plan), PlanContext.emptyPlanContext(), listener); + try { + executePlan(analyze(plan), PlanContext.emptyPlanContext(), listener); + } catch (Exception e) { + listener.onFailure(e); + } } /** diff --git a/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java b/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java index c65210e97e..d1ffa51fcc 100644 --- a/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java @@ -8,6 +8,7 @@ package org.opensearch.sql.executor; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; @@ -64,7 +65,7 @@ class QueryServiceTest { @BeforeEach public void setUp() { lenient().when(analyzer.analyze(any(), any())).thenReturn(logicalPlan); - when(planner.plan(any())).thenReturn(plan); + lenient().when(planner.plan(any())).thenReturn(plan); queryService = new QueryService(analyzer, executionEngine, planner); } @@ -86,7 +87,7 @@ public void testExecuteShouldPass() { new ResponseListener<>() { @Override public void onResponse(ExecutionEngine.QueryResponse pplQueryResponse) { - + assertNotNull(pplQueryResponse); } @Override @@ -115,7 +116,7 @@ public void testExplainShouldPass() { new ResponseListener<>() { @Override public void onResponse(ExecutionEngine.ExplainResponse pplQueryResponse) { - + assertNotNull(pplQueryResponse); } @Override @@ -185,7 +186,7 @@ public void testExecutePlanShouldPass() { new ResponseListener<>() { @Override public void onResponse(ExecutionEngine.QueryResponse pplQueryResponse) { - + assertNotNull(pplQueryResponse); } @Override @@ -194,4 +195,23 @@ public void onFailure(Exception e) { } }); } + + @Test + public void analyzeExceptionShouldBeCached() { + when(analyzer.analyze(any(), any())).thenThrow(IllegalStateException.class); + + queryService.execute( + ast, + new ResponseListener<>() { + @Override + public void onResponse(ExecutionEngine.QueryResponse pplQueryResponse) { + fail(); + } + + @Override + public void onFailure(Exception e) { + assertTrue(e instanceof IllegalStateException); + } + }); + } }