From e295d9c0bdabb919fd1610779b30fa7e6d7ef0f8 Mon Sep 17 00:00:00 2001 From: vamsi-amazon Date: Thu, 17 Nov 2022 09:25:05 -0800 Subject: [PATCH] Catalog to Datasource changes Signed-off-by: vamsi-amazon --- core/build.gradle | 4 +- ...emaName.java => DataSourceSchemaName.java} | 4 +- .../org/opensearch/sql/analysis/Analyzer.java | 67 +++---- ...taSourceSchemaIdentifierNameResolver.java} | 32 ++-- .../org/opensearch/sql/ast/tree/Relation.java | 2 +- .../sql/catalog/CatalogService.java | 40 ---- .../sql/datasource/DataSourceService.java | 40 ++++ .../model/ConnectorType.java | 2 +- .../model/DataSource.java} | 4 +- .../model/DataSourceMetadata.java} | 4 +- .../model/auth/AuthenticationType.java | 3 +- .../function/BuiltinFunctionRepository.java | 5 +- .../DataSourceTable.java} | 22 +-- .../DataSourceTableScan.java} | 32 ++-- .../DataSourceTableSchema.java} | 8 +- .../opensearch/sql/storage/StorageEngine.java | 8 +- .../sql/storage/StorageEngineFactory.java | 2 +- .../sql/utils/SystemIndexUtils.java | 2 +- .../opensearch/sql/analysis/AnalyzerTest.java | 33 ++-- .../sql/analysis/AnalyzerTestBase.java | 40 ++-- ...talogSchemaIdentifierNameResolverTest.java | 30 --- ...ourceSchemaIdentifierNameResolverTest.java | 30 +++ .../org/opensearch/sql/config/TestConfig.java | 4 +- .../BuiltinFunctionRepositoryTest.java | 4 +- .../opensearch/sql/planner/PlannerTest.java | 6 +- .../planner/logical/LogicalRelationTest.java | 2 +- .../catalog/CatalogTableScanTest.java | 70 ------- .../datasource/DataSourceTableScanTest.java | 70 +++++++ .../DataSourceTableTest.java} | 24 +-- .../sql/storage/StorageEngineTest.java | 2 +- docs/user/general/identifiers.rst | 4 +- docs/user/limitations/limitations.rst | 2 +- docs/user/ppl/admin/prometheus_connector.rst | 6 +- docs/user/ppl/cmd/describe.rst | 8 +- docs/user/ppl/cmd/stats.rst | 2 +- doctest/build.gradle | 2 +- .../datasources.json} | 0 integ-test/build.gradle | 3 +- ...va => PrometheusDataSourceCommandsIT.java} | 2 +- ...dIT.java => ShowDataSourcesCommandIT.java} | 6 +- .../org/opensearch/sql/ppl/StandaloneIT.java | 22 +-- .../datasources.json} | 0 .../legacy/plugin/RestSQLQueryActionTest.java | 4 +- .../storage/OpenSearchStorageEngine.java | 4 +- .../storage/OpenSearchStorageEngineTest.java | 9 +- .../org/opensearch/sql/plugin/SQLPlugin.java | 20 +- .../plugin/catalog/CatalogServiceImpl.java | 178 ----------------- .../plugin/config/OpenSearchPluginConfig.java | 10 +- .../datasource/DataSourceServiceImpl.java | 180 ++++++++++++++++++ .../DataSourceSettings.java} | 6 +- .../catalog/CatalogServiceImplTest.java | 139 -------------- .../datasource/DataSourceServiceImplTest.java | 141 ++++++++++++++ ...name.json => datasource_missing_name.json} | 0 .../{catalogs.json => datasources.json} | 0 ...s.json => duplicate_datasource_names.json} | 0 ...pty_catalog.json => empty_datasource.json} | 0 ...name.json => illegal_datasource_name.json} | 0 ...talogs.json => malformed_datasources.json} | 0 ...atalogs.json => multiple_datasources.json} | 0 ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 +- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 6 +- .../opensearch/sql/ppl/parser/AstBuilder.java | 12 +- .../opensearch/sql/ppl/PPLServiceTest.java | 6 +- .../sql/ppl/parser/AstBuilderTest.java | 7 +- .../PrometheusDescribeMetricRequest.java | 18 +- .../system/PrometheusListMetricsRequest.java | 6 +- .../storage/PrometheusStorageEngine.java | 16 +- .../storage/PrometheusStorageFactory.java | 4 +- .../storage/system/PrometheusSystemTable.java | 12 +- .../PrometheusDescribeMetricRequestTest.java | 12 +- .../PrometheusListMetricsRequestTest.java | 6 +- .../storage/PrometheusStorageEngineTest.java | 12 +- .../storage/PrometheusStorageFactoryTest.java | 2 +- .../system/PrometheusSystemTableTest.java | 8 +- ...elasticsearch-sql.release-notes-1.9.0.1.md | 2 +- .../opensearch-sql.release-notes-2.4.0.0.md | 8 +- 76 files changed, 744 insertions(+), 739 deletions(-) rename core/src/main/java/org/opensearch/sql/{CatalogSchemaName.java => DataSourceSchemaName.java} (77%) rename core/src/main/java/org/opensearch/sql/analysis/{CatalogSchemaIdentifierNameResolver.java => DataSourceSchemaIdentifierNameResolver.java} (56%) delete mode 100644 core/src/main/java/org/opensearch/sql/catalog/CatalogService.java create mode 100644 core/src/main/java/org/opensearch/sql/datasource/DataSourceService.java rename core/src/main/java/org/opensearch/sql/{catalog => datasource}/model/ConnectorType.java (75%) rename core/src/main/java/org/opensearch/sql/{catalog/model/Catalog.java => datasource/model/DataSource.java} (86%) rename core/src/main/java/org/opensearch/sql/{catalog/model/CatalogMetadata.java => datasource/model/DataSourceMetadata.java} (89%) rename core/src/main/java/org/opensearch/sql/{catalog => datasource}/model/auth/AuthenticationType.java (89%) rename core/src/main/java/org/opensearch/sql/planner/physical/{catalog/CatalogTable.java => datasource/DataSourceTable.java} (60%) rename core/src/main/java/org/opensearch/sql/planner/physical/{catalog/CatalogTableScan.java => datasource/DataSourceTableScan.java} (54%) rename core/src/main/java/org/opensearch/sql/planner/physical/{catalog/CatalogTableSchema.java => datasource/DataSourceTableSchema.java} (72%) delete mode 100644 core/src/test/java/org/opensearch/sql/analysis/model/CatalogSchemaIdentifierNameResolverTest.java create mode 100644 core/src/test/java/org/opensearch/sql/analysis/model/DataSourceSchemaIdentifierNameResolverTest.java delete mode 100644 core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScanTest.java create mode 100644 core/src/test/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScanTest.java rename core/src/test/java/org/opensearch/sql/planner/physical/{catalog/CatalogTableTest.java => datasource/DataSourceTableTest.java} (66%) rename doctest/{catalog/catalog.json => datasource/datasources.json} (100%) rename integ-test/src/test/java/org/opensearch/sql/ppl/{PrometheusCatalogCommandsIT.java => PrometheusDataSourceCommandsIT.java} (98%) rename integ-test/src/test/java/org/opensearch/sql/ppl/{ShowCatalogsCommandIT.java => ShowDataSourcesCommandIT.java} (83%) rename integ-test/src/test/resources/{catalog/catalog.json => datasource/datasources.json} (100%) delete mode 100644 plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogServiceImpl.java create mode 100644 plugin/src/main/java/org/opensearch/sql/plugin/datasource/DataSourceServiceImpl.java rename plugin/src/main/java/org/opensearch/sql/plugin/{catalog/CatalogSettings.java => datasource/DataSourceSettings.java} (62%) delete mode 100644 plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java create mode 100644 plugin/src/test/java/org/opensearch/sql/plugin/datasource/DataSourceServiceImplTest.java rename plugin/src/test/resources/{catalog_missing_name.json => datasource_missing_name.json} (100%) rename plugin/src/test/resources/{catalogs.json => datasources.json} (100%) rename plugin/src/test/resources/{duplicate_catalog_names.json => duplicate_datasource_names.json} (100%) rename plugin/src/test/resources/{empty_catalog.json => empty_datasource.json} (100%) rename plugin/src/test/resources/{illegal_catalog_name.json => illegal_datasource_name.json} (100%) rename plugin/src/test/resources/{malformed_catalogs.json => malformed_datasources.json} (100%) rename plugin/src/test/resources/{multiple_catalogs.json => multiple_datasources.json} (100%) diff --git a/core/build.gradle b/core/build.gradle index eb70f110d1..512d74c432 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -73,7 +73,7 @@ jacocoTestReport { afterEvaluate { classDirectories.setFrom(files(classDirectories.files.collect { fileTree(dir: it, - exclude: ['**/ast/**', '**/catalog/model/**']) + exclude: ['**/ast/**', '**/datasource/model/**']) })) } } @@ -85,7 +85,7 @@ jacocoTestCoverageVerification { excludes = [ 'org.opensearch.sql.utils.MLCommonsConstants', 'org.opensearch.sql.utils.Constants', - 'org.opensearch.sql.catalog.model.*' + 'org.opensearch.sql.datasource.model.*' ] limit { counter = 'LINE' diff --git a/core/src/main/java/org/opensearch/sql/CatalogSchemaName.java b/core/src/main/java/org/opensearch/sql/DataSourceSchemaName.java similarity index 77% rename from core/src/main/java/org/opensearch/sql/CatalogSchemaName.java rename to core/src/main/java/org/opensearch/sql/DataSourceSchemaName.java index 8dde03ca3d..47988097c3 100644 --- a/core/src/main/java/org/opensearch/sql/CatalogSchemaName.java +++ b/core/src/main/java/org/opensearch/sql/DataSourceSchemaName.java @@ -12,9 +12,9 @@ @Getter @RequiredArgsConstructor -public class CatalogSchemaName { +public class DataSourceSchemaName { - private final String catalogName; + private final String dataSourceName; private final String schemaName; diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index 7d0a452e1b..d463ed424d 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -11,19 +11,11 @@ import static org.opensearch.sql.ast.tree.Sort.SortOrder.ASC; import static org.opensearch.sql.ast.tree.Sort.SortOrder.DESC; import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; -import static org.opensearch.sql.utils.MLCommonsConstants.ACTION; -import static org.opensearch.sql.utils.MLCommonsConstants.MODELID; -import static org.opensearch.sql.utils.MLCommonsConstants.PREDICT; import static org.opensearch.sql.utils.MLCommonsConstants.RCF_ANOMALOUS; import static org.opensearch.sql.utils.MLCommonsConstants.RCF_ANOMALY_GRADE; import static org.opensearch.sql.utils.MLCommonsConstants.RCF_SCORE; -import static org.opensearch.sql.utils.MLCommonsConstants.RCF_TIMESTAMP; -import static org.opensearch.sql.utils.MLCommonsConstants.STATUS; -import static org.opensearch.sql.utils.MLCommonsConstants.TASKID; import static org.opensearch.sql.utils.MLCommonsConstants.TIME_FIELD; -import static org.opensearch.sql.utils.MLCommonsConstants.TRAIN; -import static org.opensearch.sql.utils.MLCommonsConstants.TRAINANDPREDICT; -import static org.opensearch.sql.utils.SystemIndexUtils.CATALOGS_TABLE_NAME; +import static org.opensearch.sql.utils.SystemIndexUtils.DATASOURCES_TABLE_NAME; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; @@ -37,7 +29,7 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; -import org.opensearch.sql.CatalogSchemaName; +import org.opensearch.sql.DataSourceSchemaName; import org.opensearch.sql.analysis.symbol.Namespace; import org.opensearch.sql.analysis.symbol.Symbol; import org.opensearch.sql.ast.AbstractNodeVisitor; @@ -69,10 +61,10 @@ import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; -import org.opensearch.sql.catalog.CatalogService; -import org.opensearch.sql.catalog.model.Catalog; import org.opensearch.sql.data.model.ExprMissingValue; import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.datasource.DataSourceService; +import org.opensearch.sql.datasource.model.DataSource; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; @@ -101,7 +93,7 @@ import org.opensearch.sql.planner.logical.LogicalRename; import org.opensearch.sql.planner.logical.LogicalSort; import org.opensearch.sql.planner.logical.LogicalValues; -import org.opensearch.sql.planner.physical.catalog.CatalogTable; +import org.opensearch.sql.planner.physical.datasource.DataSourceTable; import org.opensearch.sql.storage.Table; import org.opensearch.sql.utils.ParseUtils; @@ -117,7 +109,7 @@ public class Analyzer extends AbstractNodeVisitor private final NamedExpressionAnalyzer namedExpressionAnalyzer; - private final CatalogService catalogService; + private final DataSourceService dataSourceService; private final BuiltinFunctionRepository repository; @@ -126,10 +118,10 @@ public class Analyzer extends AbstractNodeVisitor */ public Analyzer( ExpressionAnalyzer expressionAnalyzer, - CatalogService catalogService, + DataSourceService dataSourceService, BuiltinFunctionRepository repository) { this.expressionAnalyzer = expressionAnalyzer; - this.catalogService = catalogService; + this.dataSourceService = dataSourceService; this.selectExpressionAnalyzer = new SelectExpressionAnalyzer(expressionAnalyzer); this.namedExpressionAnalyzer = new NamedExpressionAnalyzer(expressionAnalyzer); this.repository = repository; @@ -142,25 +134,27 @@ public LogicalPlan analyze(UnresolvedPlan unresolved, AnalysisContext context) { @Override public LogicalPlan visitRelation(Relation node, AnalysisContext context) { QualifiedName qualifiedName = node.getTableQualifiedName(); - Set allowedCatalogNames = catalogService.getCatalogs() + Set allowedDataSourceNames = dataSourceService.getDataSources() .stream() - .map(Catalog::getName) + .map(DataSource::getName) .collect(Collectors.toSet()); - CatalogSchemaIdentifierNameResolver catalogSchemaIdentifierNameResolver - = new CatalogSchemaIdentifierNameResolver(qualifiedName.getParts(), allowedCatalogNames); - String tableName = catalogSchemaIdentifierNameResolver.getIdentifierName(); + DataSourceSchemaIdentifierNameResolver dataSourceSchemaIdentifierNameResolver + = new DataSourceSchemaIdentifierNameResolver(qualifiedName.getParts(), + allowedDataSourceNames); + String tableName = dataSourceSchemaIdentifierNameResolver.getIdentifierName(); context.push(); TypeEnvironment curEnv = context.peek(); Table table; - if (CATALOGS_TABLE_NAME.equals(tableName)) { - table = new CatalogTable(catalogService); + if (DATASOURCES_TABLE_NAME.equals(tableName)) { + table = new DataSourceTable(dataSourceService); } else { - table = catalogService - .getCatalog(catalogSchemaIdentifierNameResolver.getCatalogName()) + table = dataSourceService + .getDataSource(dataSourceSchemaIdentifierNameResolver.getDataSourceName()) .getStorageEngine() - .getTable(new CatalogSchemaName(catalogSchemaIdentifierNameResolver.getCatalogName(), - catalogSchemaIdentifierNameResolver.getSchemaName()), - catalogSchemaIdentifierNameResolver.getIdentifierName()); + .getTable(new DataSourceSchemaName( + dataSourceSchemaIdentifierNameResolver.getDataSourceName(), + dataSourceSchemaIdentifierNameResolver.getSchemaName()), + dataSourceSchemaIdentifierNameResolver.getIdentifierName()); } table.getFieldTypes().forEach((k, v) -> curEnv.define(new Symbol(Namespace.FIELD_NAME, k), v)); @@ -188,28 +182,29 @@ public LogicalPlan visitRelationSubquery(RelationSubquery node, AnalysisContext @Override public LogicalPlan visitTableFunction(TableFunction node, AnalysisContext context) { QualifiedName qualifiedName = node.getFunctionName(); - Set allowedCatalogNames = catalogService.getCatalogs() + Set allowedDataSourceNames = dataSourceService.getDataSources() .stream() - .map(Catalog::getName) + .map(DataSource::getName) .collect(Collectors.toSet()); - CatalogSchemaIdentifierNameResolver catalogSchemaIdentifierNameResolver - = new CatalogSchemaIdentifierNameResolver(qualifiedName.getParts(), allowedCatalogNames); + DataSourceSchemaIdentifierNameResolver dataSourceSchemaIdentifierNameResolver + = new DataSourceSchemaIdentifierNameResolver(qualifiedName.getParts(), + allowedDataSourceNames); FunctionName functionName - = FunctionName.of(catalogSchemaIdentifierNameResolver.getIdentifierName()); + = FunctionName.of(dataSourceSchemaIdentifierNameResolver.getIdentifierName()); List arguments = node.getArguments().stream() .map(unresolvedExpression -> this.expressionAnalyzer.analyze(unresolvedExpression, context)) .collect(Collectors.toList()); TableFunctionImplementation tableFunctionImplementation = (TableFunctionImplementation) repository.compile( - catalogSchemaIdentifierNameResolver.getCatalogName(), functionName, arguments); + dataSourceSchemaIdentifierNameResolver.getDataSourceName(), functionName, arguments); context.push(); TypeEnvironment curEnv = context.peek(); Table table = tableFunctionImplementation.applyArguments(); table.getFieldTypes().forEach((k, v) -> curEnv.define(new Symbol(Namespace.FIELD_NAME, k), v)); curEnv.define(new Symbol(Namespace.INDEX_NAME, - catalogSchemaIdentifierNameResolver.getIdentifierName()), STRUCT); - return new LogicalRelation(catalogSchemaIdentifierNameResolver.getIdentifierName(), + dataSourceSchemaIdentifierNameResolver.getIdentifierName()), STRUCT); + return new LogicalRelation(dataSourceSchemaIdentifierNameResolver.getIdentifierName(), tableFunctionImplementation.applyArguments()); } diff --git a/core/src/main/java/org/opensearch/sql/analysis/CatalogSchemaIdentifierNameResolver.java b/core/src/main/java/org/opensearch/sql/analysis/DataSourceSchemaIdentifierNameResolver.java similarity index 56% rename from core/src/main/java/org/opensearch/sql/analysis/CatalogSchemaIdentifierNameResolver.java rename to core/src/main/java/org/opensearch/sql/analysis/DataSourceSchemaIdentifierNameResolver.java index 7e0d2af028..f3552b029b 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/CatalogSchemaIdentifierNameResolver.java +++ b/core/src/main/java/org/opensearch/sql/analysis/DataSourceSchemaIdentifierNameResolver.java @@ -10,29 +10,31 @@ import java.util.List; import java.util.Set; -public class CatalogSchemaIdentifierNameResolver { +public class DataSourceSchemaIdentifierNameResolver { - public static final String DEFAULT_CATALOG_NAME = "@opensearch"; + public static final String DEFAULT_DATASOURCE_NAME = "@opensearch"; public static final String DEFAULT_SCHEMA_NAME = "default"; public static final String INFORMATION_SCHEMA_NAME = "information_schema"; - private String catalogName = DEFAULT_CATALOG_NAME; + private String dataSourceName = DEFAULT_DATASOURCE_NAME; private String schemaName = DEFAULT_SCHEMA_NAME; private String identifierName; private static final String DOT = "."; /** - * Data model for capturing catalog, schema and identifier from + * Data model for capturing dataSourceName, schema and identifier from * fully qualifiedName. In the current state, it is used to capture - * CatalogSchemaTable name and CatalogSchemaFunction in case of table + * DataSourceSchemaTable name and DataSourceSchemaFunction in case of table * functions. * * @param parts parts of qualifiedName. - * @param allowedCatalogs allowedCatalogs. + * @param allowedDataSources allowedDataSources. */ - public CatalogSchemaIdentifierNameResolver(List parts, Set allowedCatalogs) { - List remainingParts = captureSchemaName(captureCatalogName(parts, allowedCatalogs)); + public DataSourceSchemaIdentifierNameResolver(List parts, + Set allowedDataSources) { + List remainingParts + = captureSchemaName(captureDataSourceName(parts, allowedDataSources)); identifierName = String.join(DOT, remainingParts); } @@ -40,8 +42,8 @@ public String getIdentifierName() { return identifierName; } - public String getCatalogName() { - return catalogName; + public String getDataSourceName() { + return dataSourceName; } public String getSchemaName() { @@ -49,12 +51,12 @@ public String getSchemaName() { } - // Capture catalog name and return remaining parts(schema name and table name) + // Capture datasource name and return remaining parts(schema name and table name) // from the fully qualified name. - private List captureCatalogName(List parts, Set allowedCatalogs) { - if (parts.size() > 1 && allowedCatalogs.contains(parts.get(0)) - || DEFAULT_CATALOG_NAME.equals(parts.get(0))) { - catalogName = parts.get(0); + private List captureDataSourceName(List parts, Set allowedDataSources) { + if (parts.size() > 1 && allowedDataSources.contains(parts.get(0)) + || DEFAULT_DATASOURCE_NAME.equals(parts.get(0))) { + dataSourceName = parts.get(0); return parts.subList(1, parts.size()); } else { return parts; diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Relation.java b/core/src/main/java/org/opensearch/sql/ast/tree/Relation.java index ef24597b90..8c3868329f 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/Relation.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Relation.java @@ -73,7 +73,7 @@ public String getAlias() { /** * Get Qualified name preservs parts of the user given identifiers. - * This can later be utilized to determine Catalog,Schema and Table Name during + * This can later be utilized to determine DataSource,Schema and Table Name during * Analyzer stage. So Passing QualifiedName directly to Analyzer Stage. * * @return TableQualifiedName. diff --git a/core/src/main/java/org/opensearch/sql/catalog/CatalogService.java b/core/src/main/java/org/opensearch/sql/catalog/CatalogService.java deleted file mode 100644 index 4c40920c7b..0000000000 --- a/core/src/main/java/org/opensearch/sql/catalog/CatalogService.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.catalog; - -import java.util.Set; -import org.opensearch.sql.catalog.model.Catalog; -import org.opensearch.sql.storage.StorageEngine; - -/** - * Catalog Service manages catalogs. - */ -public interface CatalogService { - - /** - * Returns all catalog objects. - * - * @return Catalog Catalogs. - */ - Set getCatalogs(); - - /** - * Returns Catalog with corresponding to the catalog name. - * - * @param catalogName Name of the catalog. - * @return Catalog catalog. - */ - Catalog getCatalog(String catalogName); - - /** - * Default opensearch engine is not defined in catalog.json. - * So the registration of default catalog happens separately. - * - * @param storageEngine StorageEngine. - */ - void registerDefaultOpenSearchCatalog(StorageEngine storageEngine); - -} diff --git a/core/src/main/java/org/opensearch/sql/datasource/DataSourceService.java b/core/src/main/java/org/opensearch/sql/datasource/DataSourceService.java new file mode 100644 index 0000000000..88ba8e508c --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/datasource/DataSourceService.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.datasource; + +import java.util.Set; +import org.opensearch.sql.datasource.model.DataSource; +import org.opensearch.sql.storage.StorageEngine; + +/** + * DataSource Service manages datasources. + */ +public interface DataSourceService { + + /** + * Returns all datasource objects. + * + * @return DataSource datasources. + */ + Set getDataSources(); + + /** + * Returns DataSource with corresponding to the datasource name. + * + * @param dataSourceName Name of the datasource. + * @return DataSource datasource. + */ + DataSource getDataSource(String dataSourceName); + + /** + * Default opensearch engine is not defined in datasources config. + * So the registration of default datasource happens separately. + * + * @param storageEngine StorageEngine. + */ + void registerDefaultOpenSearchDataSource(StorageEngine storageEngine); + +} diff --git a/core/src/main/java/org/opensearch/sql/catalog/model/ConnectorType.java b/core/src/main/java/org/opensearch/sql/datasource/model/ConnectorType.java similarity index 75% rename from core/src/main/java/org/opensearch/sql/catalog/model/ConnectorType.java rename to core/src/main/java/org/opensearch/sql/datasource/model/ConnectorType.java index b84c68adbf..b540d7d401 100644 --- a/core/src/main/java/org/opensearch/sql/catalog/model/ConnectorType.java +++ b/core/src/main/java/org/opensearch/sql/datasource/model/ConnectorType.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.catalog.model; +package org.opensearch.sql.datasource.model; public enum ConnectorType { PROMETHEUS,OPENSEARCH diff --git a/core/src/main/java/org/opensearch/sql/catalog/model/Catalog.java b/core/src/main/java/org/opensearch/sql/datasource/model/DataSource.java similarity index 86% rename from core/src/main/java/org/opensearch/sql/catalog/model/Catalog.java rename to core/src/main/java/org/opensearch/sql/datasource/model/DataSource.java index 5b7eaca523..a6ac9a6d66 100644 --- a/core/src/main/java/org/opensearch/sql/catalog/model/Catalog.java +++ b/core/src/main/java/org/opensearch/sql/datasource/model/DataSource.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.catalog.model; +package org.opensearch.sql.datasource.model; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -15,7 +15,7 @@ @Getter @RequiredArgsConstructor @EqualsAndHashCode -public class Catalog { +public class DataSource { private final String name; diff --git a/core/src/main/java/org/opensearch/sql/catalog/model/CatalogMetadata.java b/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceMetadata.java similarity index 89% rename from core/src/main/java/org/opensearch/sql/catalog/model/CatalogMetadata.java rename to core/src/main/java/org/opensearch/sql/datasource/model/DataSourceMetadata.java index a859090a5d..dbde5040e9 100644 --- a/core/src/main/java/org/opensearch/sql/catalog/model/CatalogMetadata.java +++ b/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceMetadata.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.catalog.model; +package org.opensearch.sql.datasource.model; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -15,7 +15,7 @@ @JsonIgnoreProperties(ignoreUnknown = true) @Getter @Setter -public class CatalogMetadata { +public class DataSourceMetadata { @JsonProperty(required = true) private String name; diff --git a/core/src/main/java/org/opensearch/sql/catalog/model/auth/AuthenticationType.java b/core/src/main/java/org/opensearch/sql/datasource/model/auth/AuthenticationType.java similarity index 89% rename from core/src/main/java/org/opensearch/sql/catalog/model/auth/AuthenticationType.java rename to core/src/main/java/org/opensearch/sql/datasource/model/auth/AuthenticationType.java index 1157d8e497..9cf3e01509 100644 --- a/core/src/main/java/org/opensearch/sql/catalog/model/auth/AuthenticationType.java +++ b/core/src/main/java/org/opensearch/sql/datasource/model/auth/AuthenticationType.java @@ -5,12 +5,11 @@ * */ -package org.opensearch.sql.catalog.model.auth; +package org.opensearch.sql.datasource.model.auth; import java.util.Collections; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; public enum AuthenticationType { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java index 33f652d534..b77c43b89a 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java @@ -30,9 +30,8 @@ /** * Builtin Function Repository. - * Repository registers catalog specific functions under catalog specific namespace and - * universal functions under default namespace. Catalog Specific Namespace carries their own - * namespace. + * Repository registers datasource specific functions under datasource namespace and + * universal functions under default namespace. * */ @RequiredArgsConstructor diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTable.java b/core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTable.java similarity index 60% rename from core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTable.java rename to core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTable.java index 4e6a87e21b..105ad5ed32 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTable.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTable.java @@ -5,14 +5,14 @@ * */ -package org.opensearch.sql.planner.physical.catalog; +package org.opensearch.sql.planner.physical.datasource; import com.google.common.annotations.VisibleForTesting; import java.util.Map; import lombok.EqualsAndHashCode; import lombok.RequiredArgsConstructor; -import org.opensearch.sql.catalog.CatalogService; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.planner.DefaultImplementor; import org.opensearch.sql.planner.logical.LogicalPlan; import org.opensearch.sql.planner.logical.LogicalRelation; @@ -22,36 +22,36 @@ /** * Table implementation to handle show datasources command. - * Since catalog information is not tied to any storage engine, this info - * is handled via Catalog Table. + * Since datasource information is not tied to any storage engine, this info + * is handled via DataSource Table. * */ @RequiredArgsConstructor @EqualsAndHashCode -public class CatalogTable implements Table { +public class DataSourceTable implements Table { - private final CatalogService catalogService; + private final DataSourceService dataSourceService; @Override public Map getFieldTypes() { - return CatalogTableSchema.CATALOG_TABLE_SCHEMA.getMapping(); + return DataSourceTableSchema.DATASOURCE_TABLE_SCHEMA.getMapping(); } @Override public PhysicalPlan implement(LogicalPlan plan) { - return plan.accept(new CatalogTableDefaultImplementor(catalogService), null); + return plan.accept(new DataSourceTableDefaultImplementor(dataSourceService), null); } @VisibleForTesting @RequiredArgsConstructor - public static class CatalogTableDefaultImplementor + public static class DataSourceTableDefaultImplementor extends DefaultImplementor { - private final CatalogService catalogService; + private final DataSourceService dataSourceService; @Override public PhysicalPlan visitRelation(LogicalRelation node, Object context) { - return new CatalogTableScan(catalogService); + return new DataSourceTableScan(dataSourceService); } } diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScan.java b/core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScan.java similarity index 54% rename from core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScan.java rename to core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScan.java index efc59c97ec..14cd09e162 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScan.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScan.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.planner.physical.catalog; +package org.opensearch.sql.planner.physical.datasource; import com.google.common.collect.ImmutableMap; import java.util.ArrayList; @@ -14,45 +14,47 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Set; -import org.opensearch.sql.catalog.CatalogService; -import org.opensearch.sql.catalog.model.Catalog; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.datasource.DataSourceService; +import org.opensearch.sql.datasource.model.DataSource; import org.opensearch.sql.storage.TableScanOperator; /** - * This class handles table scan of catalog table. - * Right now these are derived from catalogService thorough static fields. + * This class handles table scan of data source table. + * Right now these are derived from dataSourceService thorough static fields. * In future this might scan data from underlying datastore if we start - * persisting catalog info somewhere. + * persisting datasource info somewhere. * */ -public class CatalogTableScan extends TableScanOperator { +public class DataSourceTableScan extends TableScanOperator { - private final CatalogService catalogService; + private final DataSourceService dataSourceService; private Iterator iterator; - public CatalogTableScan(CatalogService catalogService) { - this.catalogService = catalogService; + public DataSourceTableScan(DataSourceService dataSourceService) { + this.dataSourceService = dataSourceService; this.iterator = Collections.emptyIterator(); } @Override public String explain() { - return "GetCatalogRequestRequest{}"; + return "GetDataSourcesInfoRequest{}"; } @Override public void open() { List exprValues = new ArrayList<>(); - Set catalogs = catalogService.getCatalogs(); - for (Catalog catalog : catalogs) { + Set dataSources = dataSourceService.getDataSources(); + for (DataSource dataSource : dataSources) { exprValues.add( new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( - "DATASOURCE_NAME", ExprValueUtils.stringValue(catalog.getName()), - "CONNECTOR_TYPE", ExprValueUtils.stringValue(catalog.getConnectorType().name()))))); + "DATASOURCE_NAME", + ExprValueUtils.stringValue(dataSource.getName()), + "CONNECTOR_TYPE", + ExprValueUtils.stringValue(dataSource.getConnectorType().name()))))); } iterator = exprValues.iterator(); } diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableSchema.java b/core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableSchema.java similarity index 72% rename from core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableSchema.java rename to core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableSchema.java index b360eb87db..dd959d9b56 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableSchema.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableSchema.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.planner.physical.catalog; +package org.opensearch.sql.planner.physical.datasource; import static org.opensearch.sql.data.type.ExprCoreType.STRING; @@ -14,13 +14,13 @@ import org.opensearch.sql.data.type.ExprType; /** - * Definition of the system table schema. + * Definition of the data source table schema. */ @Getter @RequiredArgsConstructor -public enum CatalogTableSchema { +public enum DataSourceTableSchema { - CATALOG_TABLE_SCHEMA(new LinkedHashMap<>() { + DATASOURCE_TABLE_SCHEMA(new LinkedHashMap<>() { { put("DATASOURCE_NAME", STRING); put("CONNECTOR_TYPE", STRING); diff --git a/core/src/main/java/org/opensearch/sql/storage/StorageEngine.java b/core/src/main/java/org/opensearch/sql/storage/StorageEngine.java index 609949578c..246a50ea09 100644 --- a/core/src/main/java/org/opensearch/sql/storage/StorageEngine.java +++ b/core/src/main/java/org/opensearch/sql/storage/StorageEngine.java @@ -8,7 +8,7 @@ import java.util.Collection; import java.util.Collections; -import org.opensearch.sql.CatalogSchemaName; +import org.opensearch.sql.DataSourceSchemaName; import org.opensearch.sql.expression.function.FunctionResolver; /** @@ -19,12 +19,12 @@ public interface StorageEngine { /** * Get {@link Table} from storage engine. */ - Table getTable(CatalogSchemaName catalogSchemaName, String tableName); + Table getTable(DataSourceSchemaName dataSourceSchemaName, String tableName); /** - * Get list of catalog related functions. + * Get list of datasource related functions. * - * @return FunctionResolvers of catalog functions. + * @return FunctionResolvers of datasource functions. */ default Collection getFunctions() { return Collections.emptyList(); diff --git a/core/src/main/java/org/opensearch/sql/storage/StorageEngineFactory.java b/core/src/main/java/org/opensearch/sql/storage/StorageEngineFactory.java index 4cc27f6fa0..85d29abf5c 100644 --- a/core/src/main/java/org/opensearch/sql/storage/StorageEngineFactory.java +++ b/core/src/main/java/org/opensearch/sql/storage/StorageEngineFactory.java @@ -8,7 +8,7 @@ package org.opensearch.sql.storage; import java.util.Map; -import org.opensearch.sql.catalog.model.ConnectorType; +import org.opensearch.sql.datasource.model.ConnectorType; public interface StorageEngineFactory { diff --git a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java index 9ba3a67847..5325ea371a 100644 --- a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java @@ -38,7 +38,7 @@ public class SystemIndexUtils { */ public static final String TABLE_INFO = "ALL." + SYS_META_SUFFIX; - public static final String CATALOGS_TABLE_NAME = ".CATALOGS"; + public static final String DATASOURCES_TABLE_NAME = ".DATASOURCES"; public static Boolean isSystemIndex(String indexName) { diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java index 97c560d505..fe534112ba 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java @@ -10,7 +10,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.opensearch.sql.analysis.CatalogSchemaIdentifierNameResolver.DEFAULT_CATALOG_NAME; +import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; import static org.opensearch.sql.ast.dsl.AstDSL.aggregate; import static org.opensearch.sql.ast.dsl.AstDSL.alias; import static org.opensearch.sql.ast.dsl.AstDSL.argument; @@ -52,6 +52,7 @@ import static org.opensearch.sql.utils.MLCommonsConstants.STATUS; import static org.opensearch.sql.utils.MLCommonsConstants.TASKID; import static org.opensearch.sql.utils.MLCommonsConstants.TRAIN; +import static org.opensearch.sql.utils.SystemIndexUtils.DATASOURCES_TABLE_NAME; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -88,7 +89,7 @@ import org.opensearch.sql.planner.logical.LogicalPlanDSL; import org.opensearch.sql.planner.logical.LogicalProject; import org.opensearch.sql.planner.logical.LogicalRelation; -import org.opensearch.sql.planner.physical.catalog.CatalogTable; +import org.opensearch.sql.planner.physical.datasource.DataSourceTable; import org.springframework.context.annotation.Configuration; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; @@ -123,7 +124,7 @@ public void filter_relation_with_alias() { } @Test - public void filter_relation_with_catalog() { + public void filter_relation_with_datasource() { assertAnalyzeEqual( LogicalPlanDSL.filter( LogicalPlanDSL.relation("http_total_requests", table), @@ -134,7 +135,7 @@ public void filter_relation_with_catalog() { } @Test - public void filter_relation_with_escaped_catalog() { + public void filter_relation_with_escaped_datasource() { assertAnalyzeEqual( LogicalPlanDSL.filter( LogicalPlanDSL.relation("prometheus.http_total_requests", table), @@ -145,7 +146,7 @@ public void filter_relation_with_escaped_catalog() { } @Test - public void filter_relation_with_information_schema_and_prom_catalog() { + public void filter_relation_with_information_schema_and_prom_datasource() { assertAnalyzeEqual( LogicalPlanDSL.filter( LogicalPlanDSL.relation("tables", table), @@ -156,7 +157,7 @@ public void filter_relation_with_information_schema_and_prom_catalog() { } @Test - public void filter_relation_with_default_schema_and_prom_catalog() { + public void filter_relation_with_default_schema_and_prom_datasource() { assertAnalyzeEqual( LogicalPlanDSL.filter( LogicalPlanDSL.relation("tables", table), @@ -167,14 +168,14 @@ public void filter_relation_with_default_schema_and_prom_catalog() { } @Test - public void filter_relation_with_information_schema_and_os_catalog() { + public void filter_relation_with_information_schema_and_os_datasource() { assertAnalyzeEqual( LogicalPlanDSL.filter( LogicalPlanDSL.relation("tables", table), dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(integerValue(1)))), AstDSL.filter( AstDSL.relation( - AstDSL.qualifiedName(DEFAULT_CATALOG_NAME, "information_schema", "tables")), + AstDSL.qualifiedName(DEFAULT_DATASOURCE_NAME, "information_schema", "tables")), AstDSL.equalTo(AstDSL.field("integer_value"), AstDSL.intLiteral(1)))); } @@ -190,7 +191,7 @@ public void filter_relation_with_information_schema() { } @Test - public void filter_relation_with_non_existing_catalog() { + public void filter_relation_with_non_existing_datasource() { assertAnalyzeEqual( LogicalPlanDSL.filter( LogicalPlanDSL.relation("test.http_total_requests", table), @@ -201,7 +202,7 @@ public void filter_relation_with_non_existing_catalog() { } @Test - public void filter_relation_with_non_existing_catalog_with_three_parts() { + public void filter_relation_with_non_existing_datasource_with_three_parts() { assertAnalyzeEqual( LogicalPlanDSL.filter( LogicalPlanDSL.relation("test.nonexisting_schema.http_total_requests", table), @@ -1043,7 +1044,7 @@ public void table_function() { } @Test - public void table_function_with_no_catalog() { + public void table_function_with_no_datasource() { ExpressionEvaluationException exception = assertThrows(ExpressionEvaluationException.class, () -> analyze(AstDSL.tableFunction(List.of("query_range"), unresolvedArg("query", stringLiteral("http_latency")), @@ -1055,7 +1056,7 @@ public void table_function_with_no_catalog() { } @Test - public void table_function_with_wrong_catalog() { + public void table_function_with_wrong_datasource() { ExpressionEvaluationException exception = assertThrows(ExpressionEvaluationException.class, () -> analyze(AstDSL.tableFunction(Arrays.asList("prome", "query_range"), unresolvedArg("query", stringLiteral("http_latency")), @@ -1077,10 +1078,10 @@ public void table_function_with_wrong_table_function() { } @Test - public void show_catalogs() { - assertAnalyzeEqual(new LogicalRelation(".CATALOGS", new CatalogTable(catalogService)), - AstDSL.relation(qualifiedName(".CATALOGS"))); - + public void show_datasources() { + assertAnalyzeEqual(new LogicalRelation(DATASOURCES_TABLE_NAME, + new DataSourceTable(dataSourceService)), + AstDSL.relation(qualifiedName(DATASOURCES_TABLE_NAME))); } @Test diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java index 447802c963..fa5530aaef 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java @@ -15,16 +15,15 @@ import java.util.Map; import java.util.Set; import org.apache.commons.lang3.tuple.Pair; -import org.opensearch.sql.CatalogSchemaName; import org.opensearch.sql.analysis.symbol.Namespace; import org.opensearch.sql.analysis.symbol.Symbol; import org.opensearch.sql.analysis.symbol.SymbolTable; import org.opensearch.sql.ast.tree.UnresolvedPlan; -import org.opensearch.sql.catalog.CatalogService; -import org.opensearch.sql.catalog.model.Catalog; -import org.opensearch.sql.catalog.model.ConnectorType; import org.opensearch.sql.config.TestConfig; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.datasource.DataSourceService; +import org.opensearch.sql.datasource.model.ConnectorType; +import org.opensearch.sql.datasource.model.DataSource; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; @@ -52,7 +51,7 @@ protected Map typeMapping() { @Bean protected StorageEngine storageEngine() { - return (catalogSchemaName, tableName) -> table; + return (dataSourceSchemaName, tableName) -> table; } @Bean @@ -81,7 +80,7 @@ public PhysicalPlan implement(LogicalPlan plan) { } @Bean - protected Table catalogTable() { + protected Table dataSourceTable() { return new Table() { @Override public Map getFieldTypes() { @@ -96,8 +95,8 @@ public PhysicalPlan implement(LogicalPlan plan) { } @Bean - protected CatalogService catalogService() { - return new DefaultCatalogService(); + protected DataSourceService dataSourceService() { + return new DefaultDataSourceService(); } @@ -143,16 +142,17 @@ protected Environment typeEnv() { protected Table table; @Autowired - protected CatalogService catalogService; + protected DataSourceService dataSourceService; @Autowired protected Environment typeEnv; @Bean - protected Analyzer analyzer(ExpressionAnalyzer expressionAnalyzer, CatalogService catalogService, + protected Analyzer analyzer(ExpressionAnalyzer expressionAnalyzer, + DataSourceService dataSourceService, StorageEngine storageEngine, BuiltinFunctionRepository functionRepository, Table table) { - catalogService.registerDefaultOpenSearchCatalog(storageEngine); + dataSourceService.registerDefaultOpenSearchDataSource(storageEngine); functionRepository.register("prometheus", new FunctionResolver() { @Override @@ -170,7 +170,7 @@ public FunctionName getFunctionName() { return FunctionName.of("query_range"); } }); - return new Analyzer(expressionAnalyzer, catalogService, functionRepository); + return new Analyzer(expressionAnalyzer, dataSourceService, functionRepository); } @Bean @@ -196,25 +196,25 @@ protected LogicalPlan analyze(UnresolvedPlan unresolvedPlan) { return analyzer.analyze(unresolvedPlan, analysisContext); } - private class DefaultCatalogService implements CatalogService { + private class DefaultDataSourceService implements DataSourceService { private StorageEngine storageEngine = storageEngine(); - private final Catalog catalog - = new Catalog("prometheus", ConnectorType.PROMETHEUS, storageEngine); + private final DataSource dataSource + = new DataSource("prometheus", ConnectorType.PROMETHEUS, storageEngine); @Override - public Set getCatalogs() { - return ImmutableSet.of(catalog); + public Set getDataSources() { + return ImmutableSet.of(dataSource); } @Override - public Catalog getCatalog(String catalogName) { - return catalog; + public DataSource getDataSource(String dataSourceName) { + return dataSource; } @Override - public void registerDefaultOpenSearchCatalog(StorageEngine storageEngine) { + public void registerDefaultOpenSearchDataSource(StorageEngine storageEngine) { this.storageEngine = storageEngine; } } diff --git a/core/src/test/java/org/opensearch/sql/analysis/model/CatalogSchemaIdentifierNameResolverTest.java b/core/src/test/java/org/opensearch/sql/analysis/model/CatalogSchemaIdentifierNameResolverTest.java deleted file mode 100644 index 069a1d814f..0000000000 --- a/core/src/test/java/org/opensearch/sql/analysis/model/CatalogSchemaIdentifierNameResolverTest.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * - */ - -package org.opensearch.sql.analysis.model; - - -import java.util.Arrays; -import java.util.Collections; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.opensearch.sql.analysis.CatalogSchemaIdentifierNameResolver; - -public class CatalogSchemaIdentifierNameResolverTest { - - @Test - void testFullyQualifiedName() { - CatalogSchemaIdentifierNameResolver - catalogSchemaIdentifierNameResolver = new CatalogSchemaIdentifierNameResolver( - Arrays.asList("prom", "information_schema", "tables"), Collections.singleton("prom")); - Assertions.assertEquals("information_schema", - catalogSchemaIdentifierNameResolver.getSchemaName()); - Assertions.assertEquals("prom", catalogSchemaIdentifierNameResolver.getCatalogName()); - Assertions.assertEquals("tables", catalogSchemaIdentifierNameResolver.getIdentifierName()); - } - -} diff --git a/core/src/test/java/org/opensearch/sql/analysis/model/DataSourceSchemaIdentifierNameResolverTest.java b/core/src/test/java/org/opensearch/sql/analysis/model/DataSourceSchemaIdentifierNameResolverTest.java new file mode 100644 index 0000000000..7d7c92f3ed --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/analysis/model/DataSourceSchemaIdentifierNameResolverTest.java @@ -0,0 +1,30 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.analysis.model; + + +import java.util.Arrays; +import java.util.Collections; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver; + +public class DataSourceSchemaIdentifierNameResolverTest { + + @Test + void testFullyQualifiedName() { + DataSourceSchemaIdentifierNameResolver + dataSourceSchemaIdentifierNameResolver = new DataSourceSchemaIdentifierNameResolver( + Arrays.asList("prom", "information_schema", "tables"), Collections.singleton("prom")); + Assertions.assertEquals("information_schema", + dataSourceSchemaIdentifierNameResolver.getSchemaName()); + Assertions.assertEquals("prom", dataSourceSchemaIdentifierNameResolver.getDataSourceName()); + Assertions.assertEquals("tables", dataSourceSchemaIdentifierNameResolver.getIdentifierName()); + } + +} diff --git a/core/src/test/java/org/opensearch/sql/config/TestConfig.java b/core/src/test/java/org/opensearch/sql/config/TestConfig.java index 7475f577a6..a0ef436162 100644 --- a/core/src/test/java/org/opensearch/sql/config/TestConfig.java +++ b/core/src/test/java/org/opensearch/sql/config/TestConfig.java @@ -8,7 +8,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; -import org.opensearch.sql.CatalogSchemaName; +import org.opensearch.sql.DataSourceSchemaName; import org.opensearch.sql.analysis.symbol.Namespace; import org.opensearch.sql.analysis.symbol.Symbol; import org.opensearch.sql.analysis.symbol.SymbolTable; @@ -63,7 +63,7 @@ public class TestConfig { protected StorageEngine storageEngine() { return new StorageEngine() { @Override - public Table getTable(CatalogSchemaName catalogSchemaName, String name) { + public Table getTable(DataSourceSchemaName dataSourceSchemaName, String name) { return new Table() { @Override public boolean exists() { diff --git a/core/src/test/java/org/opensearch/sql/expression/function/BuiltinFunctionRepositoryTest.java b/core/src/test/java/org/opensearch/sql/expression/function/BuiltinFunctionRepositoryTest.java index 5dd98dfedf..f63304e6b5 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/BuiltinFunctionRepositoryTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/BuiltinFunctionRepositoryTest.java @@ -87,7 +87,7 @@ void register() { } @Test - void register_under_catalog_namespace() { + void register_under_datasource_namespace() { when(mockNamespaceMap.containsKey(TEST_NAMESPACE)).thenReturn(false); when(mockNamespaceMap.put(eq(TEST_NAMESPACE), any())).thenReturn(null); when(mockNamespaceMap.get(TEST_NAMESPACE)).thenReturn(mockMap); @@ -120,7 +120,7 @@ void compile() { @Test - void compile_function_under_catalog_namespace() { + void compile_function_under_datasource_namespace() { when(mockExpression.type()).thenReturn(UNDEFINED); when(functionSignature.getParamTypeList()).thenReturn(Arrays.asList(UNDEFINED)); when(mockfunctionResolver.getFunctionName()).thenReturn(mockFunctionName); diff --git a/core/src/test/java/org/opensearch/sql/planner/PlannerTest.java b/core/src/test/java/org/opensearch/sql/planner/PlannerTest.java index 4207c7d31b..e08b590082 100644 --- a/core/src/test/java/org/opensearch/sql/planner/PlannerTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/PlannerTest.java @@ -12,7 +12,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.when; -import static org.opensearch.sql.analysis.CatalogSchemaIdentifierNameResolver.DEFAULT_CATALOG_NAME; +import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; @@ -25,7 +25,7 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.CatalogSchemaName; +import org.opensearch.sql.DataSourceSchemaName; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.planner.logical.LogicalAggregation; @@ -81,7 +81,7 @@ public void planner_test() { LogicalPlanDSL.filter( LogicalPlanDSL.relation("schema", storageEngine.getTable( - new CatalogSchemaName(DEFAULT_CATALOG_NAME, "default"), + new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), "schema")), dsl.equal(DSL.ref("response", INTEGER), DSL.literal(10)) ), diff --git a/core/src/test/java/org/opensearch/sql/planner/logical/LogicalRelationTest.java b/core/src/test/java/org/opensearch/sql/planner/logical/LogicalRelationTest.java index 93448185cd..dacd84e12b 100644 --- a/core/src/test/java/org/opensearch/sql/planner/logical/LogicalRelationTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/logical/LogicalRelationTest.java @@ -28,7 +28,7 @@ public void logicalRelationHasNoInput() { } @Test - public void logicalRelationWithCatalogHasNoInput() { + public void logicalRelationWithDataSourceHasNoInput() { LogicalPlan relation = LogicalPlanDSL.relation("prometheus.index", table); assertEquals(0, relation.getChild().size()); } diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScanTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScanTest.java deleted file mode 100644 index cf9b5fe016..0000000000 --- a/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScanTest.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * - */ - -package org.opensearch.sql.planner.physical.catalog; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.when; - -import com.google.common.collect.ImmutableMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.Set; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.catalog.CatalogService; -import org.opensearch.sql.catalog.model.Catalog; -import org.opensearch.sql.catalog.model.ConnectorType; -import org.opensearch.sql.data.model.ExprTupleValue; -import org.opensearch.sql.data.model.ExprValueUtils; -import org.opensearch.sql.storage.StorageEngine; - -@ExtendWith(MockitoExtension.class) -public class CatalogTableScanTest { - - @Mock - private CatalogService catalogService; - - @Mock - private StorageEngine storageEngine; - - private CatalogTableScan catalogTableScan; - - @BeforeEach - private void setUp() { - catalogTableScan = new CatalogTableScan(catalogService); - } - - @Test - void testExplain() { - assertEquals("GetCatalogRequestRequest{}", catalogTableScan.explain()); - } - - @Test - void testIterator() { - Set catalogSet = new HashSet<>(); - catalogSet.add(new Catalog("prometheus", ConnectorType.PROMETHEUS, storageEngine)); - catalogSet.add(new Catalog("opensearch", ConnectorType.OPENSEARCH, storageEngine)); - when(catalogService.getCatalogs()).thenReturn(catalogSet); - - assertFalse(catalogTableScan.hasNext()); - catalogTableScan.open(); - assertTrue(catalogTableScan.hasNext()); - for (Catalog catalog : catalogSet) { - assertEquals(new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( - "DATASOURCE_NAME", ExprValueUtils.stringValue(catalog.getName()), - "CONNECTOR_TYPE", ExprValueUtils.stringValue(catalog.getConnectorType().name())))), - catalogTableScan.next()); - } - } - -} diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScanTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScanTest.java new file mode 100644 index 0000000000..a57d1b1a89 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScanTest.java @@ -0,0 +1,70 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.planner.physical.datasource; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Set; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.datasource.DataSourceService; +import org.opensearch.sql.datasource.model.ConnectorType; +import org.opensearch.sql.datasource.model.DataSource; +import org.opensearch.sql.storage.StorageEngine; + +@ExtendWith(MockitoExtension.class) +public class DataSourceTableScanTest { + + @Mock + private DataSourceService dataSourceService; + + @Mock + private StorageEngine storageEngine; + + private DataSourceTableScan dataSourceTableScan; + + @BeforeEach + private void setUp() { + dataSourceTableScan = new DataSourceTableScan(dataSourceService); + } + + @Test + void testExplain() { + assertEquals("GetDataSourcesInfoRequest{}", dataSourceTableScan.explain()); + } + + @Test + void testIterator() { + Set dataSourceSet = new HashSet<>(); + dataSourceSet.add(new DataSource("prometheus", ConnectorType.PROMETHEUS, storageEngine)); + dataSourceSet.add(new DataSource("opensearch", ConnectorType.OPENSEARCH, storageEngine)); + when(dataSourceService.getDataSources()).thenReturn(dataSourceSet); + + assertFalse(dataSourceTableScan.hasNext()); + dataSourceTableScan.open(); + assertTrue(dataSourceTableScan.hasNext()); + for (DataSource dataSource : dataSourceSet) { + assertEquals(new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( + "DATASOURCE_NAME", ExprValueUtils.stringValue(dataSource.getName()), + "CONNECTOR_TYPE", ExprValueUtils.stringValue(dataSource.getConnectorType().name())))), + dataSourceTableScan.next()); + } + } + +} diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableTest.java similarity index 66% rename from core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java rename to core/src/test/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableTest.java index 1c069005f0..6a596ff6fb 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableTest.java @@ -5,11 +5,12 @@ * */ -package org.opensearch.sql.planner.physical.catalog; +package org.opensearch.sql.planner.physical.datasource; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.utils.SystemIndexUtils.DATASOURCES_TABLE_NAME; import java.util.HashMap; import java.util.Map; @@ -17,22 +18,22 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.catalog.CatalogService; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.planner.logical.LogicalPlanDSL; import org.opensearch.sql.planner.physical.PhysicalPlan; @ExtendWith(MockitoExtension.class) -public class CatalogTableTest { +public class DataSourceTableTest { @Mock - private CatalogService catalogService; + private DataSourceService dataSourceService; @Test void testGetFieldTypes() { - CatalogTable catalogTable = new CatalogTable(catalogService); - Map fieldTypes = catalogTable.getFieldTypes(); + DataSourceTable dataSourceTable = new DataSourceTable(dataSourceService); + Map fieldTypes = dataSourceTable.getFieldTypes(); Map expectedTypes = new HashMap<>(); expectedTypes.put("DATASOURCE_NAME", ExprCoreType.STRING); expectedTypes.put("CONNECTOR_TYPE", ExprCoreType.STRING); @@ -41,10 +42,11 @@ void testGetFieldTypes() { @Test void testImplement() { - CatalogTable catalogTable = new CatalogTable(catalogService); + DataSourceTable dataSourceTable = new DataSourceTable(dataSourceService); PhysicalPlan physicalPlan - = catalogTable.implement(LogicalPlanDSL.relation(".CATALOGS", catalogTable)); - assertTrue(physicalPlan instanceof CatalogTableScan); + = dataSourceTable.implement( + LogicalPlanDSL.relation(DATASOURCES_TABLE_NAME, dataSourceTable)); + assertTrue(physicalPlan instanceof DataSourceTableScan); } // todo. temporary added for code coverage. remove if required. @@ -52,7 +54,7 @@ void testImplement() { void testExist() { UnsupportedOperationException exception = assertThrows(UnsupportedOperationException.class, - () -> new CatalogTable(catalogService).exists()); + () -> new DataSourceTable(dataSourceService).exists()); assertEquals("Unsupported Operation", exception.getMessage()); } @@ -61,7 +63,7 @@ void testExist() { void testCreateTable() { UnsupportedOperationException exception = assertThrows(UnsupportedOperationException.class, - () -> new CatalogTable(catalogService).create(new HashMap<>())); + () -> new DataSourceTable(dataSourceService).create(new HashMap<>())); assertEquals("Unsupported Operation", exception.getMessage()); } } diff --git a/core/src/test/java/org/opensearch/sql/storage/StorageEngineTest.java b/core/src/test/java/org/opensearch/sql/storage/StorageEngineTest.java index b0da30212d..0e969c6dac 100644 --- a/core/src/test/java/org/opensearch/sql/storage/StorageEngineTest.java +++ b/core/src/test/java/org/opensearch/sql/storage/StorageEngineTest.java @@ -16,7 +16,7 @@ public class StorageEngineTest { @Test void testFunctionsMethod() { - StorageEngine k = (catalogSchemaName, tableName) -> null; + StorageEngine k = (dataSourceSchemaName, tableName) -> null; Assertions.assertEquals(Collections.emptyList(), k.getFunctions()); } diff --git a/docs/user/general/identifiers.rst b/docs/user/general/identifiers.rst index 8bb42bb7e7..9d0467a9c4 100644 --- a/docs/user/general/identifiers.rst +++ b/docs/user/general/identifiers.rst @@ -231,7 +231,7 @@ tableName = ``http_requests_total``. 2. ``logs.12.13.1`` -datasourceName = ``@opensearch`` [Resolves to default @opensearch connector since [my_prometheus] is the only catalog configured name.] +datasourceName = ``@opensearch`` [Resolves to default @opensearch connector since [my_prometheus] is the only dataSource configured name.] schemaName = ``default`` [No supported schema found, so default to `default`]. @@ -257,7 +257,7 @@ tableName = ``prometheus.http_requests_total``. 5. ``prometheus.default.http_requests_total.1.2.3`` -datasourceName = ``@opensearch`` [Resolves to default @opensearch connector since [my_prometheus] is the only catalog configured name.] +datasourceName = ``@opensearch`` [Resolves to default @opensearch connector since [my_prometheus] is the only dataSource configured name.] schemaName = ``default`` [No supported schema found, so default to `default`]. diff --git a/docs/user/limitations/limitations.rst b/docs/user/limitations/limitations.rst index c2163af23d..d6e763a73d 100644 --- a/docs/user/limitations/limitations.rst +++ b/docs/user/limitations/limitations.rst @@ -18,7 +18,7 @@ In this doc, the restrictions and limitations of SQL plugin is covered as follow Limitations on Identifiers ========================== -Using OpenSearch cluster name as catalog name to qualify an index name, such as ``my_cluster.my_index``, is not supported for now. +Using OpenSearch cluster name as dataSource name to qualify an index name, such as ``my_cluster.my_index``, is not supported for now. Limitations on Fields ===================== diff --git a/docs/user/ppl/admin/prometheus_connector.rst b/docs/user/ppl/admin/prometheus_connector.rst index aced79cbdb..fd84f8114c 100644 --- a/docs/user/ppl/admin/prometheus_connector.rst +++ b/docs/user/ppl/admin/prometheus_connector.rst @@ -14,11 +14,11 @@ Prometheus Connector Introduction ============ -This page covers prometheus connector properties for catalog configuration +This page covers prometheus connector properties for dataSource configuration and the nuances associated with prometheus connector. -Prometheus Connector Properties in Catalog Configuration +Prometheus Connector Properties in DataSource Configuration ======================================================== Prometheus Connector Properties. @@ -32,7 +32,7 @@ Prometheus Connector Properties. * If prometheus.auth.type is awssigv4, following are required parameters. * ``prometheus.auth.region``, ``prometheus.auth.access_key`` and ``prometheus.auth.secret_key`` -Example prometheus catalog configuration with different authentications +Example prometheus dataSource configuration with different authentications ======================================================================= No Auth :: diff --git a/docs/user/ppl/cmd/describe.rst b/docs/user/ppl/cmd/describe.rst index 12fcf35ded..a0ecbd3169 100644 --- a/docs/user/ppl/cmd/describe.rst +++ b/docs/user/ppl/cmd/describe.rst @@ -16,9 +16,9 @@ Description Syntax ============ -describe .. +describe .. -* catalog: optional. If catalog is not provided, it resolves to opensearch catalog. +* dataSource: optional. If dataSource is not provided, it resolves to opensearch dataSource. * schema: optional. If schema is not provided, it resolves to default schema. * tablename: mandatory. describe command must specify which tablename to query from. @@ -67,10 +67,10 @@ PPL query:: +----------------+ -Example 3: Fetch metadata for table in prometheus catalog +Example 3: Fetch metadata for table in prometheus dataSource ========================================================= -The example retrieves table info for ``prometheus_http_requests_total`` metric in prometheus catalog. +The example retrieves table info for ``prometheus_http_requests_total`` metric in prometheus dataSource. PPL query:: diff --git a/docs/user/ppl/cmd/stats.rst b/docs/user/ppl/cmd/stats.rst index 3a34e68a7d..d9cca9e314 100644 --- a/docs/user/ppl/cmd/stats.rst +++ b/docs/user/ppl/cmd/stats.rst @@ -13,7 +13,7 @@ Description ============ | Using ``stats`` command to calculate the aggregation from search result. -The following table catalogs the aggregation functions and also indicates how the NULL/MISSING values is handled: +The following table dataSources the aggregation functions and also indicates how the NULL/MISSING values is handled: +----------+-------------+-------------+ | Function | NULL | MISSING | diff --git a/doctest/build.gradle b/doctest/build.gradle index 8ab939cb42..a13375d469 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -108,7 +108,7 @@ String mlCommonsPlugin = 'opensearch-ml' testClusters { docTestCluster { - keystore 'plugins.query.federation.datasources.config', new File("$projectDir/catalog", 'catalog.json') + keystore 'plugins.query.federation.datasources.config', new File("$projectDir/datasource", 'datasources.json') // Disable loading of `ML-commons` plugin, because it might be unavailable (not released yet). /* plugin(provider(new Callable(){ diff --git a/doctest/catalog/catalog.json b/doctest/datasource/datasources.json similarity index 100% rename from doctest/catalog/catalog.json rename to doctest/datasource/datasources.json diff --git a/integ-test/build.gradle b/integ-test/build.gradle index d8d4ab3844..5cdf21ab96 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -27,7 +27,6 @@ import org.opensearch.gradle.test.RestIntegTestTask import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask import java.util.concurrent.Callable -import org.gradle.internal.os.OperatingSystem plugins { id "de.undercouch.download" version "5.3.0" @@ -119,7 +118,7 @@ testClusters.all { testClusters.integTest { plugin ":opensearch-sql-plugin" - keystore 'plugins.query.federation.datasources.config', new File("$projectDir/src/test/resources/catalog/", 'catalog.json') + keystore 'plugins.query.federation.datasources.config', new File("$projectDir/src/test/resources/datasource/", 'datasources.json') } task startPrometheus(type: SpawnProcessTask) { diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusCatalogCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusDataSourceCommandsIT.java similarity index 98% rename from integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusCatalogCommandsIT.java rename to integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusDataSourceCommandsIT.java index 10c1e911ab..292ea26fc1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusCatalogCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusDataSourceCommandsIT.java @@ -19,7 +19,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -public class PrometheusCatalogCommandsIT extends PPLIntegTestCase { +public class PrometheusDataSourceCommandsIT extends PPLIntegTestCase { @Test @SneakyThrows diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ShowCatalogsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ShowDataSourcesCommandIT.java similarity index 83% rename from integ-test/src/test/java/org/opensearch/sql/ppl/ShowCatalogsCommandIT.java rename to integ-test/src/test/java/org/opensearch/sql/ppl/ShowDataSourcesCommandIT.java index e12aa040e3..3614168ab8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ShowCatalogsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ShowDataSourcesCommandIT.java @@ -16,10 +16,10 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; -public class ShowCatalogsCommandIT extends PPLIntegTestCase { +public class ShowDataSourcesCommandIT extends PPLIntegTestCase { @Test - public void testShowCatalogsCommands() throws IOException { + public void testShowDataSourcesCommands() throws IOException { JSONObject result = executeQuery("show datasources"); verifyDataRows(result, rows("my_prometheus", "PROMETHEUS"), @@ -32,7 +32,7 @@ public void testShowCatalogsCommands() throws IOException { } @Test - public void testShowCatalogsCommandsWithWhereClause() throws IOException { + public void testShowDataSourcesCommandsWithWhereClause() throws IOException { JSONObject result = executeQuery("show datasources | where CONNECTOR_TYPE='PROMETHEUS'"); verifyDataRows(result, rows("my_prometheus", "PROMETHEUS")); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java index bcd0c0ffb8..a1b3589e17 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java @@ -20,7 +20,7 @@ import org.opensearch.client.RestHighLevelClient; import org.opensearch.sql.analysis.Analyzer; import org.opensearch.sql.analysis.ExpressionAnalyzer; -import org.opensearch.sql.catalog.CatalogService; +import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.executor.DefaultQueryManager; @@ -40,7 +40,7 @@ import org.opensearch.sql.opensearch.storage.OpenSearchStorageEngine; import org.opensearch.sql.planner.Planner; import org.opensearch.sql.planner.optimizer.LogicalPlanOptimizer; -import org.opensearch.sql.plugin.catalog.CatalogServiceImpl; +import org.opensearch.sql.plugin.datasource.DataSourceServiceImpl; import org.opensearch.sql.ppl.config.PPLServiceConfig; import org.opensearch.sql.ppl.domain.PPLQueryRequest; import org.opensearch.sql.protocol.response.QueryResult; @@ -76,8 +76,8 @@ public void init() { context.registerBean(OpenSearchClient.class, () -> client); context.registerBean(Settings.class, () -> defaultSettings()); OpenSearchStorageEngine openSearchStorageEngine = new OpenSearchStorageEngine(client, defaultSettings()); - CatalogServiceImpl.getInstance().registerDefaultOpenSearchCatalog(openSearchStorageEngine); - context.registerBean(CatalogService.class, CatalogServiceImpl::getInstance); + DataSourceServiceImpl.getInstance().registerDefaultOpenSearchDataSource(openSearchStorageEngine); + context.registerBean(DataSourceService.class, DataSourceServiceImpl::getInstance); context.register(StandaloneConfig.class); context.register(PPLServiceConfig.class); context.refresh(); @@ -169,7 +169,7 @@ public InternalRestHighLevelClient(RestClient restClient) { @Import({ExpressionConfig.class}) static class StandaloneConfig { @Autowired - private CatalogService catalogService; + private DataSourceService dataSourceService; @Autowired private ExecutionEngine executionEngine; @@ -182,18 +182,18 @@ QueryManager queryManager() { @Bean @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) QueryPlanFactory queryExecutionFactory(BuiltinFunctionRepository functionRepository) { - catalogService - .getCatalogs() + dataSourceService + .getDataSources() .forEach( - catalog -> - catalog + dataSource -> + dataSource .getStorageEngine() .getFunctions() .forEach( functionResolver -> - functionRepository.register(catalog.getName(), functionResolver))); + functionRepository.register(dataSource.getName(), functionResolver))); Analyzer analyzer = new Analyzer(new ExpressionAnalyzer(functionRepository), - catalogService, functionRepository); + dataSourceService, functionRepository); Planner planner = new Planner(LogicalPlanOptimizer.create(new DSL(functionRepository))); return new QueryPlanFactory(new QueryService(analyzer, executionEngine, planner)); diff --git a/integ-test/src/test/resources/catalog/catalog.json b/integ-test/src/test/resources/datasource/datasources.json similarity index 100% rename from integ-test/src/test/resources/catalog/catalog.json rename to integ-test/src/test/resources/datasource/datasources.json diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/plugin/RestSQLQueryActionTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/plugin/RestSQLQueryActionTest.java index 4c9afe802e..3eafbd32d9 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/plugin/RestSQLQueryActionTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/plugin/RestSQLQueryActionTest.java @@ -27,7 +27,7 @@ import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; -import org.opensearch.sql.catalog.CatalogService; +import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.QueryManager; @@ -66,7 +66,7 @@ public void setup() { context = new AnnotationConfigApplicationContext(); context.registerBean(StorageEngine.class, () -> Mockito.mock(StorageEngine.class)); context.registerBean(ExecutionEngine.class, () -> Mockito.mock(ExecutionEngine.class)); - context.registerBean(CatalogService.class, () -> Mockito.mock(CatalogService.class)); + context.registerBean(DataSourceService.class, () -> Mockito.mock(DataSourceService.class)); context.registerBean(QueryManager.class, () -> queryManager); context.registerBean(QueryPlanFactory.class, () -> factory); context.register(SQLServiceConfig.class); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java index 9a9847dd8c..4a3393abc9 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java @@ -9,7 +9,7 @@ import static org.opensearch.sql.utils.SystemIndexUtils.isSystemIndex; import lombok.RequiredArgsConstructor; -import org.opensearch.sql.CatalogSchemaName; +import org.opensearch.sql.DataSourceSchemaName; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex; @@ -26,7 +26,7 @@ public class OpenSearchStorageEngine implements StorageEngine { private final Settings settings; @Override - public Table getTable(CatalogSchemaName catalogSchemaName, String name) { + public Table getTable(DataSourceSchemaName dataSourceSchemaName, String name) { if (isSystemIndex(name)) { return new OpenSearchSystemIndex(client, name); } else { diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java index dd660d54a1..ab87f4531c 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java @@ -8,14 +8,14 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.opensearch.sql.analysis.CatalogSchemaIdentifierNameResolver.DEFAULT_CATALOG_NAME; +import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; import static org.opensearch.sql.utils.SystemIndexUtils.TABLE_INFO; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.CatalogSchemaName; +import org.opensearch.sql.DataSourceSchemaName; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex; @@ -33,14 +33,15 @@ class OpenSearchStorageEngineTest { @Test public void getTable() { OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); - Table table = engine.getTable(new CatalogSchemaName(DEFAULT_CATALOG_NAME, "default"), "test"); + Table table = engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), + "test"); assertNotNull(table); } @Test public void getSystemTable() { OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); - Table table = engine.getTable(new CatalogSchemaName(DEFAULT_CATALOG_NAME, "default"), + Table table = engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), TABLE_INFO); assertNotNull(table); assertTrue(table instanceof OpenSearchSystemIndex); diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index e1d29a5775..7f565046a0 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -40,7 +40,7 @@ import org.opensearch.script.ScriptContext; import org.opensearch.script.ScriptEngine; import org.opensearch.script.ScriptService; -import org.opensearch.sql.catalog.CatalogService; +import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.executor.AsyncRestExecutor; import org.opensearch.sql.legacy.metrics.Metrics; @@ -53,9 +53,9 @@ import org.opensearch.sql.opensearch.storage.OpenSearchStorageEngine; import org.opensearch.sql.opensearch.storage.script.ExpressionScriptEngine; import org.opensearch.sql.opensearch.storage.serialization.DefaultExpressionSerializer; -import org.opensearch.sql.plugin.catalog.CatalogServiceImpl; -import org.opensearch.sql.plugin.catalog.CatalogSettings; import org.opensearch.sql.plugin.config.OpenSearchPluginConfig; +import org.opensearch.sql.plugin.datasource.DataSourceServiceImpl; +import org.opensearch.sql.plugin.datasource.DataSourceSettings; import org.opensearch.sql.plugin.rest.RestPPLQueryAction; import org.opensearch.sql.plugin.rest.RestPPLStatsAction; import org.opensearch.sql.plugin.rest.RestQuerySettingsAction; @@ -142,8 +142,9 @@ public Collection createComponents( this.clusterService = clusterService; this.pluginSettings = new OpenSearchSettings(clusterService.getClusterSettings()); this.client = (NodeClient) client; - CatalogServiceImpl.getInstance().loadConnectors(clusterService.getSettings()); - CatalogServiceImpl.getInstance().registerDefaultOpenSearchCatalog(openSearchStorageEngine()); + DataSourceServiceImpl.getInstance().loadConnectors(clusterService.getSettings()); + DataSourceServiceImpl.getInstance() + .registerDefaultOpenSearchDataSource(openSearchStorageEngine()); LocalClusterState.state().setClusterService(clusterService); LocalClusterState.state().setPluginSettings((OpenSearchSettings) pluginSettings); @@ -155,7 +156,7 @@ public Collection createComponents( applicationContext.registerBean( org.opensearch.sql.common.setting.Settings.class, () -> pluginSettings); applicationContext.registerBean( - CatalogService.class, () -> CatalogServiceImpl.getInstance()); + DataSourceService.class, () -> DataSourceServiceImpl.getInstance()); applicationContext.register(OpenSearchPluginConfig.class); applicationContext.register(PPLServiceConfig.class); applicationContext.register(SQLServiceConfig.class); @@ -184,7 +185,7 @@ public List> getSettings() { return new ImmutableList.Builder>() .addAll(LegacyOpenDistroSettings.legacySettings()) .addAll(OpenSearchSettings.pluginSettings()) - .add(CatalogSettings.CATALOG_CONFIG) + .add(DataSourceSettings.DATASOURCES_CONFIG) .build(); } @@ -195,8 +196,9 @@ public ScriptEngine getScriptEngine(Settings settings, Collection catalogMap = new HashMap<>(); - - private final Map connectorTypeStorageEngineFactoryMap; - - public static CatalogServiceImpl getInstance() { - return INSTANCE; - } - - private CatalogServiceImpl() { - connectorTypeStorageEngineFactoryMap = new HashMap<>(); - PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(); - connectorTypeStorageEngineFactoryMap.put(prometheusStorageFactory.getConnectorType(), - prometheusStorageFactory); - } - - /** - * This function reads settings and loads connectors to the data stores. - * This will be invoked during start up and also when settings are updated. - * - * @param settings settings. - */ - public void loadConnectors(Settings settings) { - SecurityAccess.doPrivileged(() -> { - InputStream inputStream = CatalogSettings.CATALOG_CONFIG.get(settings); - if (inputStream != null) { - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - try { - List catalogs = - objectMapper.readValue(inputStream, new TypeReference<>() { - }); - validateCatalogs(catalogs); - constructConnectors(catalogs); - } catch (IOException e) { - LOG.error("Catalog Configuration File uploaded is malformed. Verify and re-upload.", e); - } catch (Throwable e) { - LOG.error("Catalog construction failed.", e); - } - } - return null; - }); - } - - @Override - public Set getCatalogs() { - return new HashSet<>(catalogMap.values()); - } - - @Override - public Catalog getCatalog(String catalogName) { - if (!catalogMap.containsKey(catalogName)) { - throw new IllegalArgumentException( - String.format("Catalog with name %s doesn't exist.", catalogName)); - } - return catalogMap.get(catalogName); - } - - - @Override - public void registerDefaultOpenSearchCatalog(StorageEngine storageEngine) { - if (storageEngine == null) { - throw new IllegalArgumentException("Default storage engine can't be null"); - } - catalogMap.put(DEFAULT_CATALOG_NAME, - new Catalog(DEFAULT_CATALOG_NAME, ConnectorType.OPENSEARCH, storageEngine)); - } - - private StorageEngine createStorageEngine(CatalogMetadata catalog) { - ConnectorType connector = catalog.getConnector(); - switch (connector) { - case PROMETHEUS: - return connectorTypeStorageEngineFactoryMap - .get(catalog.getConnector()) - .getStorageEngine(catalog.getName(), catalog.getProperties()); - default: - throw new IllegalStateException( - String.format("Unsupported Connector: %s", connector.name())); - } - } - - private void constructConnectors(List catalogs) { - catalogMap = new HashMap<>(); - for (CatalogMetadata catalog : catalogs) { - try { - String catalogName = catalog.getName(); - StorageEngine storageEngine = createStorageEngine(catalog); - catalogMap.put(catalogName, - new Catalog(catalog.getName(), catalog.getConnector(), storageEngine)); - } catch (Throwable e) { - LOG.error("Catalog : {} storage engine creation failed with the following message: {}", - catalog.getName(), e.getMessage(), e); - } - } - } - - /** - * This can be moved to a different validator class - * when we introduce more connectors. - * - * @param catalogs catalogs. - */ - private void validateCatalogs(List catalogs) { - - Set reviewedCatalogs = new HashSet<>(); - for (CatalogMetadata catalog : catalogs) { - - if (StringUtils.isEmpty(catalog.getName())) { - throw new IllegalArgumentException( - "Missing Name Field from a catalog. Name is a required parameter."); - } - - if (!catalog.getName().matches(CATALOG_NAME_REGEX)) { - throw new IllegalArgumentException( - String.format("Catalog Name: %s contains illegal characters." - + " Allowed characters: a-zA-Z0-9_-*@ ", catalog.getName())); - } - - String catalogName = catalog.getName(); - if (reviewedCatalogs.contains(catalogName)) { - throw new IllegalArgumentException("Catalogs with same name are not allowed."); - } else { - reviewedCatalogs.add(catalogName); - } - - if (Objects.isNull(catalog.getProperties())) { - throw new IllegalArgumentException("Missing properties field in catalog configuration. " - + "Properties are required parameters"); - } - - } - } - - -} diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginConfig.java b/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginConfig.java index 596296522c..b8e5efabf0 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginConfig.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginConfig.java @@ -12,8 +12,8 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.sql.analysis.Analyzer; import org.opensearch.sql.analysis.ExpressionAnalyzer; -import org.opensearch.sql.catalog.CatalogService; import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.QueryManager; import org.opensearch.sql.executor.QueryService; @@ -56,7 +56,7 @@ public class OpenSearchPluginConfig { private Settings settings; @Autowired - private CatalogService catalogService; + private DataSourceService dataSourceService; @Bean @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) @@ -102,8 +102,8 @@ public QueryManager queryManager() { @Bean @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) public QueryPlanFactory queryExecutionFactory(BuiltinFunctionRepository functionRepository) { - catalogService - .getCatalogs() + dataSourceService + .getDataSources() .forEach( catalog -> catalog @@ -113,7 +113,7 @@ public QueryPlanFactory queryExecutionFactory(BuiltinFunctionRepository function functionResolver -> functionRepository.register(catalog.getName(), functionResolver))); Analyzer analyzer = new Analyzer(new ExpressionAnalyzer(functionRepository), - catalogService, functionRepository); + dataSourceService, functionRepository); Planner planner = new Planner(LogicalPlanOptimizer.create(new DSL(functionRepository))); return new QueryPlanFactory(new QueryService(analyzer, executionEngine(), planner)); diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/datasource/DataSourceServiceImpl.java b/plugin/src/main/java/org/opensearch/sql/plugin/datasource/DataSourceServiceImpl.java new file mode 100644 index 0000000000..50e920753b --- /dev/null +++ b/plugin/src/main/java/org/opensearch/sql/plugin/datasource/DataSourceServiceImpl.java @@ -0,0 +1,180 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.plugin.datasource; + +import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.settings.Settings; +import org.opensearch.sql.datasource.DataSourceService; +import org.opensearch.sql.datasource.model.ConnectorType; +import org.opensearch.sql.datasource.model.DataSource; +import org.opensearch.sql.datasource.model.DataSourceMetadata; +import org.opensearch.sql.opensearch.security.SecurityAccess; +import org.opensearch.sql.prometheus.storage.PrometheusStorageFactory; +import org.opensearch.sql.storage.StorageEngine; +import org.opensearch.sql.storage.StorageEngineFactory; + +/** + * This class manages datasources and responsible for creating connectors to the datasources. + */ +public class DataSourceServiceImpl implements DataSourceService { + + private static final DataSourceServiceImpl INSTANCE = new DataSourceServiceImpl(); + + private static final String DATASOURCE_NAME_REGEX = "[@*A-Za-z]+?[*a-zA-Z_\\-0-9]*"; + + private static final Logger LOG = LogManager.getLogger(); + + private Map datasourceMap = new HashMap<>(); + + private final Map connectorTypeStorageEngineFactoryMap; + + public static DataSourceServiceImpl getInstance() { + return INSTANCE; + } + + private DataSourceServiceImpl() { + connectorTypeStorageEngineFactoryMap = new HashMap<>(); + PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(); + connectorTypeStorageEngineFactoryMap.put(prometheusStorageFactory.getConnectorType(), + prometheusStorageFactory); + } + + /** + * This function reads settings and loads connectors to the data stores. + * This will be invoked during start up and also when settings are updated. + * + * @param settings settings. + */ + public void loadConnectors(Settings settings) { + SecurityAccess.doPrivileged(() -> { + InputStream inputStream = DataSourceSettings.DATASOURCES_CONFIG.get(settings); + if (inputStream != null) { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + try { + List dataSourceMetadataList = + objectMapper.readValue(inputStream, new TypeReference<>() { + }); + validateDataSourceMetadata(dataSourceMetadataList); + constructConnectors(dataSourceMetadataList); + } catch (IOException e) { + LOG.error("DataSources Configuration File uploaded is malformed. Verify and re-upload.", + e); + } catch (Throwable e) { + LOG.error("DataSource construction failed.", e); + } + } + return null; + }); + } + + @Override + public Set getDataSources() { + return new HashSet<>(datasourceMap.values()); + } + + @Override + public DataSource getDataSource(String dataSourceName) { + if (!datasourceMap.containsKey(dataSourceName)) { + throw new IllegalArgumentException( + String.format("DataSource with name %s doesn't exist.", dataSourceName)); + } + return datasourceMap.get(dataSourceName); + } + + + @Override + public void registerDefaultOpenSearchDataSource(StorageEngine storageEngine) { + if (storageEngine == null) { + throw new IllegalArgumentException("Default storage engine can't be null"); + } + datasourceMap.put(DEFAULT_DATASOURCE_NAME, + new DataSource(DEFAULT_DATASOURCE_NAME, ConnectorType.OPENSEARCH, storageEngine)); + } + + private StorageEngine createStorageEngine(DataSourceMetadata dataSourceMetadata) { + ConnectorType connector = dataSourceMetadata.getConnector(); + switch (connector) { + case PROMETHEUS: + return connectorTypeStorageEngineFactoryMap + .get(dataSourceMetadata.getConnector()) + .getStorageEngine(dataSourceMetadata.getName(), dataSourceMetadata.getProperties()); + default: + throw new IllegalStateException( + String.format("Unsupported Connector: %s", connector.name())); + } + } + + private void constructConnectors(List dataSourceMetadataList) { + datasourceMap = new HashMap<>(); + for (DataSourceMetadata dataSourceMetadata : dataSourceMetadataList) { + try { + String dataSourceName = dataSourceMetadata.getName(); + StorageEngine storageEngine = createStorageEngine(dataSourceMetadata); + datasourceMap.put(dataSourceName, + new DataSource(dataSourceMetadata.getName(), dataSourceMetadata.getConnector(), + storageEngine)); + } catch (Throwable e) { + LOG.error("DataSource : {} storage engine creation failed with the following message: {}", + dataSourceMetadata.getName(), e.getMessage(), e); + } + } + } + + /** + * This can be moved to a different validator class + * when we introduce more connectors. + * + * @param dataSourceMetadataList dataSourceMetadataList. + */ + private void validateDataSourceMetadata(List dataSourceMetadataList) { + + Set reviewedDataSources = new HashSet<>(); + for (DataSourceMetadata dataSourceMetadata : dataSourceMetadataList) { + + if (StringUtils.isEmpty(dataSourceMetadata.getName())) { + throw new IllegalArgumentException( + "Missing Name Field for a dataSource. Name is a required parameter."); + } + + if (!dataSourceMetadata.getName().matches(DATASOURCE_NAME_REGEX)) { + throw new IllegalArgumentException( + String.format("DataSource Name: %s contains illegal characters." + + " Allowed characters: a-zA-Z0-9_-*@ ", dataSourceMetadata.getName())); + } + + String dataSourceName = dataSourceMetadata.getName(); + if (reviewedDataSources.contains(dataSourceName)) { + throw new IllegalArgumentException("DataSources with same name are not allowed."); + } else { + reviewedDataSources.add(dataSourceName); + } + + if (Objects.isNull(dataSourceMetadata.getProperties())) { + throw new IllegalArgumentException("Missing properties field in dataSource configuration." + + "Properties are required parameters"); + } + + } + } + + +} diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogSettings.java b/plugin/src/main/java/org/opensearch/sql/plugin/datasource/DataSourceSettings.java similarity index 62% rename from plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogSettings.java rename to plugin/src/main/java/org/opensearch/sql/plugin/datasource/DataSourceSettings.java index 558e7558ca..af5ef6325c 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogSettings.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/datasource/DataSourceSettings.java @@ -3,15 +3,15 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.plugin.catalog; +package org.opensearch.sql.plugin.datasource; import java.io.InputStream; import org.opensearch.common.settings.SecureSetting; import org.opensearch.common.settings.Setting; -public class CatalogSettings { +public class DataSourceSettings { - public static final Setting CATALOG_CONFIG = SecureSetting.secureFile( + public static final Setting DATASOURCES_CONFIG = SecureSetting.secureFile( "plugins.query.federation.datasources.config", null); } diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java deleted file mode 100644 index cdbce55cb1..0000000000 --- a/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.plugin.catalog; - -import static org.opensearch.sql.analysis.CatalogSchemaIdentifierNameResolver.DEFAULT_CATALOG_NAME; - -import java.io.IOException; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.HashSet; -import java.util.Set; -import lombok.SneakyThrows; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; -import org.opensearch.common.settings.MockSecureSettings; -import org.opensearch.common.settings.Settings; -import org.opensearch.sql.catalog.model.Catalog; -import org.opensearch.sql.catalog.model.ConnectorType; -import org.opensearch.sql.storage.StorageEngine; - -@RunWith(MockitoJUnitRunner.class) -public class CatalogServiceImplTest { - - public static final String CATALOG_SETTING_METADATA_KEY = - "plugins.query.federation.datasources.config"; - - @Mock - private StorageEngine storageEngine; - - @SneakyThrows - @Test - public void testLoadConnectors() { - Settings settings = getCatalogSettings("catalogs.json"); - CatalogServiceImpl.getInstance().loadConnectors(settings); - Set expected = new HashSet<>() {{ - add(new Catalog("prometheus", ConnectorType.PROMETHEUS, storageEngine)); - }}; - Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); - } - - - @SneakyThrows - @Test - public void testLoadConnectorsWithMultipleCatalogs() { - Settings settings = getCatalogSettings("multiple_catalogs.json"); - CatalogServiceImpl.getInstance().loadConnectors(settings); - Set expected = new HashSet<>() {{ - add(new Catalog("prometheus", ConnectorType.PROMETHEUS, storageEngine)); - add(new Catalog("prometheus-1", ConnectorType.PROMETHEUS, storageEngine)); - }}; - Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); - } - - @SneakyThrows - @Test - public void testLoadConnectorsWithMissingName() { - Settings settings = getCatalogSettings("catalog_missing_name.json"); - Set expected = CatalogServiceImpl.getInstance().getCatalogs(); - CatalogServiceImpl.getInstance().loadConnectors(settings); - Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); - } - - @SneakyThrows - @Test - public void testLoadConnectorsWithDuplicateCatalogNames() { - Settings settings = getCatalogSettings("duplicate_catalog_names.json"); - Set expected = CatalogServiceImpl.getInstance().getCatalogs(); - CatalogServiceImpl.getInstance().loadConnectors(settings); - Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); - } - - @SneakyThrows - @Test - public void testLoadConnectorsWithMalformedJson() { - Settings settings = getCatalogSettings("malformed_catalogs.json"); - Set expected = CatalogServiceImpl.getInstance().getCatalogs(); - CatalogServiceImpl.getInstance().loadConnectors(settings); - Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); - } - - @SneakyThrows - @Test - public void testGetStorageEngineAfterGetCatalogs() { - Settings settings = getCatalogSettings("empty_catalog.json"); - CatalogServiceImpl.getInstance().loadConnectors(settings); - CatalogServiceImpl.getInstance().registerDefaultOpenSearchCatalog(storageEngine); - Set expected = new HashSet<>(); - expected.add(new Catalog(DEFAULT_CATALOG_NAME, ConnectorType.OPENSEARCH, storageEngine)); - Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); - Assert.assertEquals(storageEngine, - CatalogServiceImpl.getInstance().getCatalog(DEFAULT_CATALOG_NAME).getStorageEngine()); - Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); - Assert.assertEquals(storageEngine, - CatalogServiceImpl.getInstance().getCatalog(DEFAULT_CATALOG_NAME).getStorageEngine()); - IllegalArgumentException illegalArgumentException - = Assert.assertThrows(IllegalArgumentException.class, - () -> CatalogServiceImpl.getInstance().getCatalog("test")); - Assert.assertEquals("Catalog with name test doesn't exist.", - illegalArgumentException.getMessage()); - } - - - @SneakyThrows - @Test - public void testGetStorageEngineAfterLoadingConnectors() { - Settings settings = getCatalogSettings("empty_catalog.json"); - CatalogServiceImpl.getInstance().registerDefaultOpenSearchCatalog(storageEngine); - //Load Connectors will empty the catalogMap.So OpenSearch Storage Engine - CatalogServiceImpl.getInstance().loadConnectors(settings); - Set expected = new HashSet<>(); - Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); - } - - @SneakyThrows - @Test - public void testLoadConnectorsWithIllegalCatalogNames() { - Settings settings = getCatalogSettings("illegal_catalog_name.json"); - Set expected = CatalogServiceImpl.getInstance().getCatalogs(); - CatalogServiceImpl.getInstance().loadConnectors(settings); - Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); - } - - private Settings getCatalogSettings(String filename) throws URISyntaxException, IOException { - MockSecureSettings mockSecureSettings = new MockSecureSettings(); - ClassLoader classLoader = getClass().getClassLoader(); - Path filepath = Paths.get(classLoader.getResource(filename).toURI()); - mockSecureSettings.setFile(CATALOG_SETTING_METADATA_KEY, Files.readAllBytes(filepath)); - return Settings.builder().setSecureSettings(mockSecureSettings).build(); - } - -} diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/datasource/DataSourceServiceImplTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/datasource/DataSourceServiceImplTest.java new file mode 100644 index 0000000000..e53cdd64c3 --- /dev/null +++ b/plugin/src/test/java/org/opensearch/sql/plugin/datasource/DataSourceServiceImplTest.java @@ -0,0 +1,141 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.plugin.datasource; + +import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashSet; +import java.util.Set; +import lombok.SneakyThrows; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; +import org.opensearch.common.settings.MockSecureSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.sql.datasource.model.ConnectorType; +import org.opensearch.sql.datasource.model.DataSource; +import org.opensearch.sql.storage.StorageEngine; + +@RunWith(MockitoJUnitRunner.class) +public class DataSourceServiceImplTest { + + public static final String DATASOURCE_SETTING_METADATA_KEY = + "plugins.query.federation.datasources.config"; + + @Mock + private StorageEngine storageEngine; + + @SneakyThrows + @Test + public void testLoadConnectors() { + Settings settings = getDataSourceSettings("datasources.json"); + DataSourceServiceImpl.getInstance().loadConnectors(settings); + Set expected = new HashSet<>() {{ + add(new DataSource("prometheus", ConnectorType.PROMETHEUS, storageEngine)); + }}; + Assert.assertEquals(expected, DataSourceServiceImpl.getInstance().getDataSources()); + } + + + @SneakyThrows + @Test + public void testLoadConnectorsWithMultipleDataSources() { + Settings settings = getDataSourceSettings("multiple_datasources.json"); + DataSourceServiceImpl.getInstance().loadConnectors(settings); + Set expected = new HashSet<>() {{ + add(new DataSource("prometheus", ConnectorType.PROMETHEUS, storageEngine)); + add(new DataSource("prometheus-1", ConnectorType.PROMETHEUS, storageEngine)); + }}; + Assert.assertEquals(expected, DataSourceServiceImpl.getInstance().getDataSources()); + } + + @SneakyThrows + @Test + public void testLoadConnectorsWithMissingName() { + Settings settings = getDataSourceSettings("datasource_missing_name.json"); + Set expected = DataSourceServiceImpl.getInstance().getDataSources(); + DataSourceServiceImpl.getInstance().loadConnectors(settings); + Assert.assertEquals(expected, DataSourceServiceImpl.getInstance().getDataSources()); + } + + @SneakyThrows + @Test + public void testLoadConnectorsWithDuplicateDataSourceNames() { + Settings settings = getDataSourceSettings("duplicate_datasource_names.json"); + Set expected = DataSourceServiceImpl.getInstance().getDataSources(); + DataSourceServiceImpl.getInstance().loadConnectors(settings); + Assert.assertEquals(expected, DataSourceServiceImpl.getInstance().getDataSources()); + } + + @SneakyThrows + @Test + public void testLoadConnectorsWithMalformedJson() { + Settings settings = getDataSourceSettings("malformed_datasources.json"); + Set expected = DataSourceServiceImpl.getInstance().getDataSources(); + DataSourceServiceImpl.getInstance().loadConnectors(settings); + Assert.assertEquals(expected, DataSourceServiceImpl.getInstance().getDataSources()); + } + + @SneakyThrows + @Test + public void testGetStorageEngineAfterGetDataSources() { + Settings settings = getDataSourceSettings("empty_datasource.json"); + DataSourceServiceImpl.getInstance().loadConnectors(settings); + DataSourceServiceImpl.getInstance().registerDefaultOpenSearchDataSource(storageEngine); + Set expected = new HashSet<>(); + expected.add(new DataSource(DEFAULT_DATASOURCE_NAME, ConnectorType.OPENSEARCH, storageEngine)); + Assert.assertEquals(expected, DataSourceServiceImpl.getInstance().getDataSources()); + Assert.assertEquals(storageEngine, + DataSourceServiceImpl.getInstance() + .getDataSource(DEFAULT_DATASOURCE_NAME).getStorageEngine()); + Assert.assertEquals(expected, DataSourceServiceImpl.getInstance().getDataSources()); + Assert.assertEquals(storageEngine, + DataSourceServiceImpl.getInstance() + .getDataSource(DEFAULT_DATASOURCE_NAME).getStorageEngine()); + IllegalArgumentException illegalArgumentException + = Assert.assertThrows(IllegalArgumentException.class, + () -> DataSourceServiceImpl.getInstance().getDataSource("test")); + Assert.assertEquals("DataSource with name test doesn't exist.", + illegalArgumentException.getMessage()); + } + + + @SneakyThrows + @Test + public void testGetStorageEngineAfterLoadingConnectors() { + Settings settings = getDataSourceSettings("empty_datasource.json"); + DataSourceServiceImpl.getInstance().registerDefaultOpenSearchDataSource(storageEngine); + //Load Connectors will empty the dataSourceMap.So OpenSearch Storage Engine + DataSourceServiceImpl.getInstance().loadConnectors(settings); + Set expected = new HashSet<>(); + Assert.assertEquals(expected, DataSourceServiceImpl.getInstance().getDataSources()); + } + + @SneakyThrows + @Test + public void testLoadConnectorsWithIllegalDataSourceNames() { + Settings settings = getDataSourceSettings("illegal_datasource_name.json"); + Set expected = DataSourceServiceImpl.getInstance().getDataSources(); + DataSourceServiceImpl.getInstance().loadConnectors(settings); + Assert.assertEquals(expected, DataSourceServiceImpl.getInstance().getDataSources()); + } + + private Settings getDataSourceSettings(String filename) throws URISyntaxException, IOException { + MockSecureSettings mockSecureSettings = new MockSecureSettings(); + ClassLoader classLoader = getClass().getClassLoader(); + Path filepath = Paths.get(classLoader.getResource(filename).toURI()); + mockSecureSettings.setFile(DATASOURCE_SETTING_METADATA_KEY, Files.readAllBytes(filepath)); + return Settings.builder().setSecureSettings(mockSecureSettings).build(); + } + +} diff --git a/plugin/src/test/resources/catalog_missing_name.json b/plugin/src/test/resources/datasource_missing_name.json similarity index 100% rename from plugin/src/test/resources/catalog_missing_name.json rename to plugin/src/test/resources/datasource_missing_name.json diff --git a/plugin/src/test/resources/catalogs.json b/plugin/src/test/resources/datasources.json similarity index 100% rename from plugin/src/test/resources/catalogs.json rename to plugin/src/test/resources/datasources.json diff --git a/plugin/src/test/resources/duplicate_catalog_names.json b/plugin/src/test/resources/duplicate_datasource_names.json similarity index 100% rename from plugin/src/test/resources/duplicate_catalog_names.json rename to plugin/src/test/resources/duplicate_datasource_names.json diff --git a/plugin/src/test/resources/empty_catalog.json b/plugin/src/test/resources/empty_datasource.json similarity index 100% rename from plugin/src/test/resources/empty_catalog.json rename to plugin/src/test/resources/empty_datasource.json diff --git a/plugin/src/test/resources/illegal_catalog_name.json b/plugin/src/test/resources/illegal_datasource_name.json similarity index 100% rename from plugin/src/test/resources/illegal_catalog_name.json rename to plugin/src/test/resources/illegal_datasource_name.json diff --git a/plugin/src/test/resources/malformed_catalogs.json b/plugin/src/test/resources/malformed_datasources.json similarity index 100% rename from plugin/src/test/resources/malformed_catalogs.json rename to plugin/src/test/resources/malformed_datasources.json diff --git a/plugin/src/test/resources/multiple_catalogs.json b/plugin/src/test/resources/multiple_datasources.json similarity index 100% rename from plugin/src/test/resources/multiple_catalogs.json rename to plugin/src/test/resources/multiple_datasources.json diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 75b99f2daf..e0aeb0ac47 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -43,7 +43,7 @@ SOURCE: 'SOURCE'; INDEX: 'INDEX'; D: 'D'; DESC: 'DESC'; -CATALOGS: 'DATASOURCES'; +DATASOURCES: 'DATASOURCES'; // CLAUSE KEYWORDS SORTBY: 'SORTBY'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 7de8e19aa0..0dd30ddcd7 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -29,7 +29,7 @@ queryStatement pplCommands : searchCommand | describeCommand - | showCatalogsCommand + | showDataSourcesCommand ; commands @@ -46,8 +46,8 @@ describeCommand : DESCRIBE tableSourceClause ; -showCatalogsCommand - : SHOW CATALOGS +showDataSourcesCommand + : SHOW DATASOURCES ; whereCommand diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index c72b638645..73c7238624 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -23,7 +23,7 @@ import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.TableSourceClauseContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.TopCommandContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.WhereCommandContext; -import static org.opensearch.sql.utils.SystemIndexUtils.CATALOGS_TABLE_NAME; +import static org.opensearch.sql.utils.SystemIndexUtils.DATASOURCES_TABLE_NAME; import static org.opensearch.sql.utils.SystemIndexUtils.mappingTable; import com.google.common.collect.ImmutableList; @@ -46,7 +46,6 @@ import org.opensearch.sql.ast.expression.Map; import org.opensearch.sql.ast.expression.ParseMethod; import org.opensearch.sql.ast.expression.QualifiedName; -import org.opensearch.sql.ast.expression.UnresolvedArgument; import org.opensearch.sql.ast.expression.UnresolvedExpression; import org.opensearch.sql.ast.tree.AD; import org.opensearch.sql.ast.tree.Aggregation; @@ -63,7 +62,6 @@ import org.opensearch.sql.ast.tree.Relation; import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Sort; -import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser; @@ -122,7 +120,7 @@ public UnresolvedPlan visitSearchFilterFrom(SearchFilterFromContext ctx) { * Describe command. * Current logic separates table and metadata info about table by adding * MAPPING_ODFE_SYS_TABLE as suffix. - * Even with the introduction of catalog and schema name in fully qualified table name, + * Even with the introduction of datasource and schema name in fully qualified table name, * we do the same thing by appending MAPPING_ODFE_SYS_TABLE as syffix to the last part * of qualified name. */ @@ -139,9 +137,9 @@ public UnresolvedPlan visitDescribeCommand(DescribeCommandContext ctx) { * Show command. */ @Override - public UnresolvedPlan visitShowCatalogsCommand( - OpenSearchPPLParser.ShowCatalogsCommandContext ctx) { - return new Relation(qualifiedName(CATALOGS_TABLE_NAME)); + public UnresolvedPlan visitShowDataSourcesCommand( + OpenSearchPPLParser.ShowDataSourcesCommandContext ctx) { + return new Relation(qualifiedName(DATASOURCES_TABLE_NAME)); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java index ef8ec25df8..d4afd71c83 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java @@ -16,8 +16,8 @@ import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; -import org.opensearch.sql.catalog.CatalogService; import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.executor.DefaultQueryManager; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponse; @@ -52,7 +52,7 @@ public class PPLServiceTest { private ExecutionEngine executionEngine; @Mock - private CatalogService catalogService; + private DataSourceService dataSourceService; @Mock private ExecutionEngine.Schema schema; @@ -66,7 +66,7 @@ public void setUp() { context.registerBean(QueryPlanFactory.class, () -> new QueryPlanFactory(queryService)); context.registerBean(StorageEngine.class, () -> storageEngine); context.registerBean(ExecutionEngine.class, () -> executionEngine); - context.registerBean(CatalogService.class, () -> catalogService); + context.registerBean(DataSourceService.class, () -> dataSourceService); context.register(PPLServiceConfig.class); context.refresh(); pplService = context.getBean(PPLService.class); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index 1e6cc5a7b9..533254a599 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -40,6 +40,7 @@ import static org.opensearch.sql.ast.dsl.AstDSL.stringLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.tableFunction; import static org.opensearch.sql.ast.dsl.AstDSL.unresolvedArg; +import static org.opensearch.sql.utils.SystemIndexUtils.DATASOURCES_TABLE_NAME; import static org.opensearch.sql.utils.SystemIndexUtils.mappingTable; import com.google.common.collect.ImmutableMap; @@ -84,7 +85,7 @@ public void testPrometheusSearchCommand() { } @Test - public void testSearchCommandWithCatalogEscape() { + public void testSearchCommandWithDataSourceEscape() { assertEqual("search source = `prometheus.http_requests_total`", relation("prometheus.http_requests_total") ); @@ -799,9 +800,9 @@ public void test_batchRCFADCommand() { } @Test - public void testShowCatalogsCommand() { + public void testShowDataSourcesCommand() { assertEqual("show datasources", - relation(".CATALOGS")); + relation(DATASOURCES_TABLE_NAME)); } protected void assertEqual(String query, Node expectedPlan) { diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/PrometheusDescribeMetricRequest.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/PrometheusDescribeMetricRequest.java index 9b76cbff27..2e0d46b3e8 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/PrometheusDescribeMetricRequest.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/PrometheusDescribeMetricRequest.java @@ -22,7 +22,7 @@ import lombok.ToString; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.sql.CatalogSchemaName; +import org.opensearch.sql.DataSourceSchemaName; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprCoreType; @@ -43,7 +43,7 @@ public class PrometheusDescribeMetricRequest implements PrometheusSystemRequest @ToString.Include private final String metricName; - private final CatalogSchemaName catalogSchemaName; + private final DataSourceSchemaName dataSourceSchemaName; private static final Logger LOG = LogManager.getLogger(); @@ -53,15 +53,15 @@ public class PrometheusDescribeMetricRequest implements PrometheusSystemRequest * metric names are optional. * * @param prometheusClient prometheusClient. - * @param catalogSchemaName catalogSchemaName. + * @param dataSourceSchemaName dataSourceSchemaName. * @param metricName metricName. */ public PrometheusDescribeMetricRequest(PrometheusClient prometheusClient, - CatalogSchemaName catalogSchemaName, + DataSourceSchemaName dataSourceSchemaName, @NonNull String metricName) { this.prometheusClient = prometheusClient; this.metricName = metricName; - this.catalogSchemaName = catalogSchemaName; + this.dataSourceSchemaName = dataSourceSchemaName; } @@ -94,16 +94,16 @@ public List search() { List results = new ArrayList<>(); for (Map.Entry entry : getFieldTypes().entrySet()) { results.add(row(entry.getKey(), entry.getValue().legacyTypeName().toLowerCase(), - catalogSchemaName)); + dataSourceSchemaName)); } return results; } private ExprTupleValue row(String fieldName, String fieldType, - CatalogSchemaName catalogSchemaName) { + DataSourceSchemaName dataSourceSchemaName) { LinkedHashMap valueMap = new LinkedHashMap<>(); - valueMap.put("TABLE_CATALOG", stringValue(catalogSchemaName.getCatalogName())); - valueMap.put("TABLE_SCHEMA", stringValue(catalogSchemaName.getSchemaName())); + valueMap.put("TABLE_CATALOG", stringValue(dataSourceSchemaName.getDataSourceName())); + valueMap.put("TABLE_SCHEMA", stringValue(dataSourceSchemaName.getSchemaName())); valueMap.put("TABLE_NAME", stringValue(metricName)); valueMap.put("COLUMN_NAME", stringValue(fieldName)); valueMap.put("DATA_TYPE", stringValue(fieldType)); diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/PrometheusListMetricsRequest.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/PrometheusListMetricsRequest.java index c4dbbebde1..f5d2a44340 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/PrometheusListMetricsRequest.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/PrometheusListMetricsRequest.java @@ -19,7 +19,7 @@ import lombok.RequiredArgsConstructor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.sql.CatalogSchemaName; +import org.opensearch.sql.DataSourceSchemaName; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.prometheus.client.PrometheusClient; @@ -30,7 +30,7 @@ public class PrometheusListMetricsRequest implements PrometheusSystemRequest { private final PrometheusClient prometheusClient; - private final CatalogSchemaName catalogSchemaName; + private final DataSourceSchemaName dataSourceSchemaName; private static final Logger LOG = LogManager.getLogger(); @@ -60,7 +60,7 @@ public List search() { private ExprTupleValue row(String metricName, String tableType, String unit, String help) { LinkedHashMap valueMap = new LinkedHashMap<>(); - valueMap.put("TABLE_CATALOG", stringValue(catalogSchemaName.getCatalogName())); + valueMap.put("TABLE_CATALOG", stringValue(dataSourceSchemaName.getDataSourceName())); valueMap.put("TABLE_SCHEMA", stringValue("default")); valueMap.put("TABLE_NAME", stringValue(metricName)); valueMap.put("TABLE_TYPE", stringValue(tableType)); diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngine.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngine.java index f8ae0936ee..4bda60b822 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngine.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngine.java @@ -7,13 +7,13 @@ package org.opensearch.sql.prometheus.storage; -import static org.opensearch.sql.analysis.CatalogSchemaIdentifierNameResolver.INFORMATION_SCHEMA_NAME; +import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.INFORMATION_SCHEMA_NAME; import static org.opensearch.sql.utils.SystemIndexUtils.isSystemIndex; import java.util.Collection; import java.util.Collections; import lombok.RequiredArgsConstructor; -import org.opensearch.sql.CatalogSchemaName; +import org.opensearch.sql.DataSourceSchemaName; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.prometheus.client.PrometheusClient; @@ -39,21 +39,21 @@ public Collection getFunctions() { } @Override - public Table getTable(CatalogSchemaName catalogSchemaName, String tableName) { + public Table getTable(DataSourceSchemaName dataSourceSchemaName, String tableName) { if (isSystemIndex(tableName)) { - return new PrometheusSystemTable(prometheusClient, catalogSchemaName, tableName); - } else if (INFORMATION_SCHEMA_NAME.equals(catalogSchemaName.getSchemaName())) { - return resolveInformationSchemaTable(catalogSchemaName, tableName); + return new PrometheusSystemTable(prometheusClient, dataSourceSchemaName, tableName); + } else if (INFORMATION_SCHEMA_NAME.equals(dataSourceSchemaName.getSchemaName())) { + return resolveInformationSchemaTable(dataSourceSchemaName, tableName); } else { return new PrometheusMetricTable(prometheusClient, tableName); } } - private Table resolveInformationSchemaTable(CatalogSchemaName catalogSchemaName, + private Table resolveInformationSchemaTable(DataSourceSchemaName dataSourceSchemaName, String tableName) { if (SystemIndexUtils.TABLE_NAME_FOR_TABLES_INFO.equals(tableName)) { return new PrometheusSystemTable(prometheusClient, - catalogSchemaName, SystemIndexUtils.TABLE_INFO); + dataSourceSchemaName, SystemIndexUtils.TABLE_INFO); } else { throw new SemanticCheckException( String.format("Information Schema doesn't contain %s table", tableName)); diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactory.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactory.java index 41cbf3748f..6aec951b3d 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactory.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactory.java @@ -14,8 +14,8 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import okhttp3.OkHttpClient; -import org.opensearch.sql.catalog.model.ConnectorType; -import org.opensearch.sql.catalog.model.auth.AuthenticationType; +import org.opensearch.sql.datasource.model.ConnectorType; +import org.opensearch.sql.datasource.model.auth.AuthenticationType; import org.opensearch.sql.prometheus.authinterceptors.AwsSigningInterceptor; import org.opensearch.sql.prometheus.authinterceptors.BasicAuthenticationInterceptor; import org.opensearch.sql.prometheus.client.PrometheusClient; diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTable.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTable.java index 2d185d4a5d..dca946da57 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTable.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTable.java @@ -12,7 +12,7 @@ import java.util.Map; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.tuple.Pair; -import org.opensearch.sql.CatalogSchemaName; +import org.opensearch.sql.DataSourceSchemaName; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.planner.DefaultImplementor; import org.opensearch.sql.planner.logical.LogicalPlan; @@ -34,11 +34,11 @@ public class PrometheusSystemTable implements Table { */ private final Pair systemIndexBundle; - private final CatalogSchemaName catalogSchemaName; + private final DataSourceSchemaName dataSourceSchemaName; public PrometheusSystemTable( - PrometheusClient client, CatalogSchemaName catalogSchemaName, String indexName) { - this.catalogSchemaName = catalogSchemaName; + PrometheusClient client, DataSourceSchemaName dataSourceSchemaName, String indexName) { + this.dataSourceSchemaName = dataSourceSchemaName; this.systemIndexBundle = buildIndexBundle(client, indexName); } @@ -68,11 +68,11 @@ private Pair buildIndexBun SystemIndexUtils.SystemTable systemTable = systemTable(indexName); if (systemTable.isSystemInfoTable()) { return Pair.of(PrometheusSystemTableSchema.SYS_TABLE_TABLES, - new PrometheusListMetricsRequest(client, catalogSchemaName)); + new PrometheusListMetricsRequest(client, dataSourceSchemaName)); } else { return Pair.of(PrometheusSystemTableSchema.SYS_TABLE_MAPPINGS, new PrometheusDescribeMetricRequest(client, - catalogSchemaName, systemTable.getTableName())); + dataSourceSchemaName, systemTable.getTableName())); } } } diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/request/PrometheusDescribeMetricRequestTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/request/PrometheusDescribeMetricRequestTest.java index a190abb6a1..dfc9aee7dc 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/request/PrometheusDescribeMetricRequestTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/request/PrometheusDescribeMetricRequestTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.CatalogSchemaName; +import org.opensearch.sql.DataSourceSchemaName; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprCoreType; @@ -56,7 +56,7 @@ void testGetFieldTypes() { }}; PrometheusDescribeMetricRequest prometheusDescribeMetricRequest = new PrometheusDescribeMetricRequest(prometheusClient, - new CatalogSchemaName("prometheus", "default"), METRIC_NAME); + new DataSourceSchemaName("prometheus", "default"), METRIC_NAME); assertEquals(expected, prometheusDescribeMetricRequest.getFieldTypes()); verify(prometheusClient, times(1)).getLabels(METRIC_NAME); } @@ -71,7 +71,7 @@ void testGetFieldTypesWithEmptyMetricName() { }}; assertThrows(NullPointerException.class, () -> new PrometheusDescribeMetricRequest(prometheusClient, - new CatalogSchemaName("prometheus", "default"), + new DataSourceSchemaName("prometheus", "default"), null)); } @@ -82,7 +82,7 @@ void testGetFieldTypesWhenException() { when(prometheusClient.getLabels(METRIC_NAME)).thenThrow(new RuntimeException("ERROR Message")); PrometheusDescribeMetricRequest prometheusDescribeMetricRequest = new PrometheusDescribeMetricRequest(prometheusClient, - new CatalogSchemaName("prometheus", "default"), METRIC_NAME); + new DataSourceSchemaName("prometheus", "default"), METRIC_NAME); RuntimeException exception = assertThrows(RuntimeException.class, prometheusDescribeMetricRequest::getFieldTypes); verify(prometheusClient, times(1)).getLabels(METRIC_NAME); @@ -95,7 +95,7 @@ void testGetFieldTypesWhenIOException() { when(prometheusClient.getLabels(METRIC_NAME)).thenThrow(new IOException("ERROR Message")); PrometheusDescribeMetricRequest prometheusDescribeMetricRequest = new PrometheusDescribeMetricRequest(prometheusClient, - new CatalogSchemaName("prometheus", "default"), METRIC_NAME); + new DataSourceSchemaName("prometheus", "default"), METRIC_NAME); RuntimeException exception = assertThrows(RuntimeException.class, prometheusDescribeMetricRequest::getFieldTypes); assertEquals("Error while fetching labels for http_requests_total" @@ -113,7 +113,7 @@ void testSearch() { }); PrometheusDescribeMetricRequest prometheusDescribeMetricRequest = new PrometheusDescribeMetricRequest(prometheusClient, - new CatalogSchemaName("test", "default"), METRIC_NAME); + new DataSourceSchemaName("test", "default"), METRIC_NAME); List result = prometheusDescribeMetricRequest.search(); assertEquals(3, result.size()); assertEquals(expectedRow(), result.get(0)); diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/request/PrometheusListMetricsRequestTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/request/PrometheusListMetricsRequestTest.java index 2d0bf3f1e9..bf5bb22e96 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/request/PrometheusListMetricsRequestTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/request/PrometheusListMetricsRequestTest.java @@ -25,7 +25,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.CatalogSchemaName; +import org.opensearch.sql.DataSourceSchemaName; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.prometheus.client.PrometheusClient; @@ -51,7 +51,7 @@ void testSearch() { when(prometheusClient.getAllMetrics()).thenReturn(metricsResult); PrometheusListMetricsRequest prometheusListMetricsRequest = new PrometheusListMetricsRequest(prometheusClient, - new CatalogSchemaName("prometheus", "information_schema")); + new DataSourceSchemaName("prometheus", "information_schema")); List result = prometheusListMetricsRequest.search(); assertEquals(expectedRow(), result.get(0)); assertEquals(2, result.size()); @@ -65,7 +65,7 @@ void testSearchWhenIOException() { when(prometheusClient.getAllMetrics()).thenThrow(new IOException("ERROR Message")); PrometheusListMetricsRequest prometheusListMetricsRequest = new PrometheusListMetricsRequest(prometheusClient, - new CatalogSchemaName("prometheus", "information_schema")); + new DataSourceSchemaName("prometheus", "information_schema")); RuntimeException exception = assertThrows(RuntimeException.class, prometheusListMetricsRequest::search); assertEquals("Error while fetching metric list for from prometheus: ERROR Message", diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngineTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngineTest.java index fadd061072..f4c9734e15 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngineTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngineTest.java @@ -7,7 +7,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.utils.SystemIndexUtils.TABLE_INFO; @@ -17,7 +16,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.CatalogSchemaName; +import org.opensearch.sql.DataSourceSchemaName; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.prometheus.client.PrometheusClient; @@ -34,7 +33,7 @@ class PrometheusStorageEngineTest { @Test public void getTable() { PrometheusStorageEngine engine = new PrometheusStorageEngine(client); - Table table = engine.getTable(new CatalogSchemaName("prometheus", "default"), "test"); + Table table = engine.getTable(new DataSourceSchemaName("prometheus", "default"), "test"); assertNotNull(table); assertTrue(table instanceof PrometheusMetricTable); } @@ -53,7 +52,7 @@ public void getFunctions() { @Test public void getSystemTable() { PrometheusStorageEngine engine = new PrometheusStorageEngine(client); - Table table = engine.getTable(new CatalogSchemaName("prometheus", "default"), TABLE_INFO); + Table table = engine.getTable(new DataSourceSchemaName("prometheus", "default"), TABLE_INFO); assertNotNull(table); assertTrue(table instanceof PrometheusSystemTable); } @@ -62,7 +61,7 @@ public void getSystemTable() { public void getSystemTableForAllTablesInfo() { PrometheusStorageEngine engine = new PrometheusStorageEngine(client); Table table - = engine.getTable(new CatalogSchemaName("prometheus", "information_schema"), "tables"); + = engine.getTable(new DataSourceSchemaName("prometheus", "information_schema"), "tables"); assertNotNull(table); assertTrue(table instanceof PrometheusSystemTable); } @@ -71,7 +70,8 @@ public void getSystemTableForAllTablesInfo() { public void getSystemTableWithWrongInformationSchemaTable() { PrometheusStorageEngine engine = new PrometheusStorageEngine(client); SemanticCheckException exception = assertThrows(SemanticCheckException.class, - () -> engine.getTable(new CatalogSchemaName("prometheus", "information_schema"), "test")); + () -> engine.getTable(new DataSourceSchemaName("prometheus", "information_schema"), + "test")); assertEquals("Information Schema doesn't contain test table", exception.getMessage()); } diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactoryTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactoryTest.java index 1b54cde5d9..d2f445a093 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactoryTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactoryTest.java @@ -13,7 +13,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.catalog.model.ConnectorType; +import org.opensearch.sql.datasource.model.ConnectorType; import org.opensearch.sql.storage.StorageEngine; @ExtendWith(MockitoExtension.class) diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTableTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTableTest.java index 960b5b1319..0721f82c07 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTableTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTableTest.java @@ -24,7 +24,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.CatalogSchemaName; +import org.opensearch.sql.DataSourceSchemaName; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.planner.physical.PhysicalPlan; @@ -44,7 +44,7 @@ public class PrometheusSystemTableTest { @Test void testGetFieldTypesOfMetaTable() { PrometheusSystemTable systemIndex = new PrometheusSystemTable(client, - new CatalogSchemaName("prometheus", "information_schema"), TABLE_INFO); + new DataSourceSchemaName("prometheus", "information_schema"), TABLE_INFO); final Map fieldTypes = systemIndex.getFieldTypes(); assertThat(fieldTypes, anyOf( hasEntry("TABLE_CATALOG", STRING) @@ -57,7 +57,7 @@ void testGetFieldTypesOfMetaTable() { @Test void testGetFieldTypesOfMappingTable() { PrometheusSystemTable systemIndex = new PrometheusSystemTable(client, - new CatalogSchemaName("prometheus", "information_schema"), mappingTable( + new DataSourceSchemaName("prometheus", "information_schema"), mappingTable( "test_metric")); final Map fieldTypes = systemIndex.getFieldTypes(); assertThat(fieldTypes, anyOf( @@ -70,7 +70,7 @@ void testGetFieldTypesOfMappingTable() { @Test void implement() { PrometheusSystemTable systemIndex = new PrometheusSystemTable(client, - new CatalogSchemaName("prometheus", "information_schema"), TABLE_INFO); + new DataSourceSchemaName("prometheus", "information_schema"), TABLE_INFO); NamedExpression projectExpr = named("TABLE_NAME", ref("TABLE_NAME", STRING)); final PhysicalPlan plan = systemIndex.implement( diff --git a/release-notes/opendistro-for-elasticsearch-sql.release-notes-1.9.0.1.md b/release-notes/opendistro-for-elasticsearch-sql.release-notes-1.9.0.1.md index 36a7d94be4..870f8919a2 100644 --- a/release-notes/opendistro-for-elasticsearch-sql.release-notes-1.9.0.1.md +++ b/release-notes/opendistro-for-elasticsearch-sql.release-notes-1.9.0.1.md @@ -18,7 +18,7 @@ * Using UTC asdefault timezone for date_format function if not provided ([#605](https://github.com/opendistro-for-elasticsearch/sql/pull/605)) * ODBC: AddingPower BI M connector ([#596](https://github.com/opendistro-for-elasticsearch/sql/pull/596)) * ODBC: add ODC 2.x functions called by Excel for Mac ([#592](https://github.com/opendistro-for-elasticsearch/sql/pull/592)) -* ODBC: Remove catalog support from driver ([#566](https://github.com/opendistro-for-elasticsearch/sql/pull/566)) +* ODBC: Remove dataSource support from driver ([#566](https://github.com/opendistro-for-elasticsearch/sql/pull/566)) * ODBC: Build driver files in parallel ([#570](https://github.com/opendistro-for-elasticsearch/sql/pull/570)) * Keep mismatch results when error occurs in comparison test ([#557](https://github.com/opendistro-for-elasticsearch/sql/pull/557)) diff --git a/release-notes/opensearch-sql.release-notes-2.4.0.0.md b/release-notes/opensearch-sql.release-notes-2.4.0.0.md index 95b5b3a100..886d33e9fb 100644 --- a/release-notes/opensearch-sql.release-notes-2.4.0.0.md +++ b/release-notes/opensearch-sql.release-notes-2.4.0.0.md @@ -10,7 +10,7 @@ Compatible with OpenSearch and OpenSearch Dashboards Version 2.4.0 #### Prometheus Support * Prometheus Connector Initial Code ([#878](https://github.com/opensearch-project/sql/pull/878)) -* Restricted catalog name to [a-zA-Z0-9_-] characters ([#876](https://github.com/opensearch-project/sql/pull/876)) +* Restricted dataSource name to [a-zA-Z0-9_-] characters ([#876](https://github.com/opensearch-project/sql/pull/876)) * Table function for supporting prometheus query_range function ([#875](https://github.com/opensearch-project/sql/pull/875)) * List tables/metrics using information_schema in source command. ([#914](https://github.com/opensearch-project/sql/pull/914)) * [Backport 2.4] Prometheus select metric and stats queries. ([#1020](https://github.com/opensearch-project/sql/pull/1020)) @@ -31,10 +31,10 @@ Compatible with OpenSearch and OpenSearch Dashboards Version 2.4.0 * Add Support for Highlight Wildcard in SQL ([#827](https://github.com/opensearch-project/sql/pull/827)) * Update SQL CLI to use AWS session token. ([#918](https://github.com/opensearch-project/sql/pull/918)) * Add `typeof` function. ([#867](https://github.com/opensearch-project/sql/pull/867)) -* Show catalogs ([#925](https://github.com/opensearch-project/sql/pull/925)) +* Show dataSources ([#925](https://github.com/opensearch-project/sql/pull/925)) * Add functions `PERIOD_ADD` and `PERIOD_DIFF`. ([#933](https://github.com/opensearch-project/sql/pull/933)) * Add take() aggregation function in PPL ([#949](https://github.com/opensearch-project/sql/pull/949)) -* Describe Table with catalog name. ([#989](https://github.com/opensearch-project/sql/pull/989)) +* Describe Table with dataSource name. ([#989](https://github.com/opensearch-project/sql/pull/989)) * Catalog Enhancements ([#988](https://github.com/opensearch-project/sql/pull/988)) * Rework on error reporting to make it more verbose and human-friendly. ([#839](https://github.com/opensearch-project/sql/pull/839)) @@ -76,7 +76,7 @@ Compatible with OpenSearch and OpenSearch Dashboards Version 2.4.0 * Fix indentation of patterns example ([#880](https://github.com/opensearch-project/sql/pull/880)) * Update docs - missing changes for #754. ([#884](https://github.com/opensearch-project/sql/pull/884)) * Fix broken links ([#911](https://github.com/opensearch-project/sql/pull/911)) -* Adding docs related to catalog. ([#963](https://github.com/opensearch-project/sql/pull/963)) +* Adding docs related to dataSource. ([#963](https://github.com/opensearch-project/sql/pull/963)) * SHOW CATALOGS documentation and integ tests ([#977](https://github.com/opensearch-project/sql/pull/977)) * [Backport 2.4] Add document for ml command. ([#1017](https://github.com/opensearch-project/sql/pull/1017))