diff --git a/build.gradle.kts b/build.gradle.kts index d04fe1a0dcd..eb7c10854dc 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -466,6 +466,7 @@ tasks.rat { "dev/docker/**/*.conf", "dev/docker/kerberos-hive/kadm5.acl", "**/*.log", + "**/testsets", "**/licenses/*.txt", "**/licenses/*.md", "integration-test/**/*.sql", diff --git a/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryITBase.java b/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryITBase.java index 5dce53e355c..9b06ddd800a 100644 --- a/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryITBase.java +++ b/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryITBase.java @@ -53,12 +53,13 @@ public class TrinoQueryITBase { protected static boolean started = false; - protected static String gravitinoUri = "http://127.0.0.1:8090"; - protected static String trinoUri = "http://127.0.0.1:8080"; - protected static String hiveMetastoreUri = "thrift://127.0.0.1:9083"; - protected static String hdfsUri = "hdfs://127.0.0.1:9000"; - protected static String mysqlUri = "jdbc:mysql://127.0.0.1"; - protected static String postgresqlUri = "jdbc:postgresql://127.0.0.1"; + public static String testHost = "127.0.0.1"; + public static String gravitinoUri = String.format("http://%s:8090", testHost); + public static String trinoUri = String.format("http://%s:8080", testHost); + public static String hiveMetastoreUri = String.format("thrift://%s:9083", testHost); + public static String hdfsUri = String.format("hdfs://%s:9000", testHost); + public static String mysqlUri = String.format("jdbc:mysql://%s", testHost); + public static String postgresqlUri = String.format("jdbc:postgresql://%s", testHost); protected static GravitinoAdminClient gravitinoClient; protected static TrinoITContainers trinoITContainers; diff --git a/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryTestTool.java b/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryTestTool.java index d30643ab8ea..c687a554007 100644 --- a/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryTestTool.java +++ b/integration-test/src/test/java/org/apache/gravitino/integration/test/trino/TrinoQueryTestTool.java @@ -47,6 +47,12 @@ public static void main(String[] args) throws Exception { false, "Generate the output file for the test set, the default value is 'false'"); + options.addOption( + "test_host", + true, + "Host address for all test services (include gravitino server, trino, hive, postgresql, mysql..., " + + "if the services are not running on the same host. set the arguments like --gravitino_uri=xxx and --trino_uri=xxx), " + + "all test services use 127.0.0.1 as default, if --auto is set to 'all', this option is ignored"); options.addOption( "gravitino_uri", true, @@ -129,6 +135,8 @@ public static void main(String[] args) throws Exception { TrinoQueryIT.ciTestsets.clear(); + String testHost = commandLine.getOptionValue("test_host"); + TrinoQueryIT.testHost = Strings.isBlank(testHost) ? TrinoQueryIT.testHost : testHost; String gravitinoUri = commandLine.getOptionValue("gravitino_uri"); TrinoQueryIT.gravitinoUri = Strings.isBlank(gravitinoUri) ? TrinoQueryIT.gravitinoUri : gravitinoUri; diff --git a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/GravitinoConfig.java b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/GravitinoConfig.java index ffb70a1e867..36fecb11b6f 100644 --- a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/GravitinoConfig.java +++ b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/GravitinoConfig.java @@ -69,6 +69,20 @@ public class GravitinoConfig { "true", false); + private static final ConfigEntry GRAVITINO_CLOUD_REGION_CODE = + new ConfigEntry( + "gravitino.cloud.region-code", + "The property to specify the region code of the cloud that the catalog is running on.", + "", + false); + + private static final ConfigEntry GRAVITINO_CATALOG_CONNECTOR_FACTORY_CLASS_NAME = + new ConfigEntry( + "gravitino.catalog.connector.factory.class.name", + "The class name for the custom CatalogConnectorFactory. The class must implement the CatalogConnectorFactory interface", + "", + false); + private static final ConfigEntry TRINO_JDBC_USER = new ConfigEntry("trino.jdbc.user", "The jdbc user name of Trino", "admin", false); @@ -138,6 +152,11 @@ public String getTrinoJdbcURI() { } } + public String getRegion() { + return config.getOrDefault( + GRAVITINO_CLOUD_REGION_CODE.key, GRAVITINO_CLOUD_REGION_CODE.defaultValue); + } + public String getCatalogConfigDirectory() { if (config.containsKey(TRINO_CATALOG_CONFIG_DIR)) { return config.get(TRINO_CATALOG_CONFIG_DIR); @@ -155,6 +174,12 @@ public String getTrinoPassword() { return config.getOrDefault(TRINO_JDBC_PASSWORD.key, TRINO_JDBC_PASSWORD.defaultValue); } + public String getCatalogConnectorFactoryClassName() { + return config.getOrDefault( + GRAVITINO_CATALOG_CONNECTOR_FACTORY_CLASS_NAME.key, + GRAVITINO_CATALOG_CONNECTOR_FACTORY_CLASS_NAME.defaultValue); + } + public String toCatalogConfig() { List stringList = new ArrayList<>(); for (Map.Entry entry : CONFIG_DEFINITIONS.entrySet()) { diff --git a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/GravitinoConnectorFactory.java b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/GravitinoConnectorFactory.java index 6f1e0cda888..b8a199d3367 100644 --- a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/GravitinoConnectorFactory.java +++ b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/GravitinoConnectorFactory.java @@ -18,6 +18,8 @@ */ package org.apache.gravitino.trino.connector; +import static org.apache.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_RUNTIME_ERROR; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -27,10 +29,12 @@ import io.trino.spi.connector.ConnectorFactory; import java.util.Map; import java.util.function.Supplier; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.client.GravitinoAdminClient; import org.apache.gravitino.trino.connector.catalog.CatalogConnectorFactory; import org.apache.gravitino.trino.connector.catalog.CatalogConnectorManager; import org.apache.gravitino.trino.connector.catalog.CatalogRegister; +import org.apache.gravitino.trino.connector.catalog.DefaultCatalogConnectorFactory; import org.apache.gravitino.trino.connector.system.GravitinoSystemConnector; import org.apache.gravitino.trino.connector.system.storedprocdure.GravitinoStoredProcedureFactory; import org.apache.gravitino.trino.connector.system.table.GravitinoSystemTableFactory; @@ -76,8 +80,8 @@ public Connector create( if (catalogConnectorManager == null) { try { CatalogRegister catalogRegister = new CatalogRegister(); - CatalogConnectorFactory catalogConnectorFactory = new CatalogConnectorFactory(); + CatalogConnectorFactory catalogConnectorFactory = createCatalogConnectorFactory(config); catalogConnectorManager = new CatalogConnectorManager(catalogRegister, catalogConnectorFactory); catalogConnectorManager.config(config, clientProvider().get()); @@ -87,7 +91,7 @@ public Connector create( } catch (Exception e) { String message = "Initialization of the GravitinoConnector failed" + e.getMessage(); LOG.error(message); - throw new TrinoException(GravitinoErrorCode.GRAVITINO_RUNTIME_ERROR, message, e); + throw new TrinoException(GRAVITINO_RUNTIME_ERROR, message, e); } } } @@ -120,4 +124,24 @@ public Connector create( Supplier clientProvider() { return () -> null; } + + private CatalogConnectorFactory createCatalogConnectorFactory(GravitinoConfig config) { + // Create a CatalogConnectorFactory. If we specify a customized class name for the + // CatalogConnectorFactory, + // it creates a user-customized CatalogConnectorFactory; otherwise, it creates a + // DefaultCatalogConnectorFactory. + String className = config.getCatalogConnectorFactoryClassName(); + if (StringUtils.isEmpty(className)) { + return new DefaultCatalogConnectorFactory(config); + } + + try { + Class clazz = Class.forName(className); + Object obj = clazz.getDeclaredConstructor(GravitinoConfig.class).newInstance(config); + return (CatalogConnectorFactory) obj; + } catch (Exception e) { + throw new TrinoException( + GRAVITINO_RUNTIME_ERROR, "Can not create CatalogConnectorFactory", e); + } + } } diff --git a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/GravitinoConnectorPluginManager.java b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/GravitinoConnectorPluginManager.java index cded5a5cc70..51bd5f09480 100644 --- a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/GravitinoConnectorPluginManager.java +++ b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/GravitinoConnectorPluginManager.java @@ -39,7 +39,6 @@ import java.util.List; import java.util.Map; import java.util.ServiceLoader; -import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonatype.aether.artifact.Artifact; @@ -51,12 +50,6 @@ public class GravitinoConnectorPluginManager { public static final String APP_CLASS_LOADER_NAME = "app"; - public static final String CONNECTOR_HIVE = "hive"; - public static final String CONNECTOR_ICEBERG = "iceberg"; - public static final String CONNECTOR_MYSQL = "mysql"; - public static final String CONNECTOR_POSTGRESQL = "postgresql"; - public static final String CONNECTOR_MEMORY = "memory"; - private static final String PLUGIN_NAME_PREFIX = "gravitino-"; private static final String PLUGIN_CLASSLOADER_CLASS_NAME = "io.trino.server.PluginClassLoader"; @@ -64,18 +57,10 @@ public class GravitinoConnectorPluginManager { private Class pluginLoaderClass; - private static final Set usePlugins = - Set.of( - CONNECTOR_HIVE, - CONNECTOR_ICEBERG, - CONNECTOR_MYSQL, - CONNECTOR_POSTGRESQL, - CONNECTOR_MEMORY); - private final Map connectorPlugins = new HashMap<>(); private final ClassLoader appClassloader; - public GravitinoConnectorPluginManager(ClassLoader classLoader) { + private GravitinoConnectorPluginManager(ClassLoader classLoader) { this.appClassloader = classLoader; try { @@ -132,11 +117,12 @@ private void loadPluginsFromFile() { .getPath(); String pluginDir = Paths.get(jarPath).getParent().getParent().toString(); - // Load all plugins - for (String pluginName : usePlugins) { - loadPlugin(pluginDir, pluginName); - LOG.info("Load plugin {}/{} successful", pluginDir, pluginName); - } + Arrays.stream(new File(pluginDir).listFiles()) + .forEach( + file -> { + loadPlugin(pluginDir, file.getName()); + LOG.info("Load plugin {}/{} successful", pluginDir, file.getName()); + }); } catch (Exception e) { throw new TrinoException( GravitinoErrorCode.GRAVITINO_RUNTIME_ERROR, "Error while loading plugins from file", e); @@ -146,16 +132,10 @@ private void loadPluginsFromFile() { private void loadPlugin(String pluginPath, String pluginName) { String dirName = pluginPath + "/" + pluginName; File directory = new File(dirName); - if (!directory.exists()) { - LOG.warn("Can not found plugin {} in directory {}", pluginName, dirName); - return; - } - File[] pluginFiles = directory.listFiles(); if (pluginFiles == null || pluginFiles.length == 0) { - throw new TrinoException( - GravitinoErrorCode.GRAVITINO_RUNTIME_ERROR, - "Can not found any files plugin directory " + dirName); + LOG.warn("Can not load plugin {} from empty directory {}", pluginName, dirName); + return; } List files = Arrays.stream(pluginFiles) @@ -196,24 +176,20 @@ private void loadPluginWithUrls(List urls, String pluginName) { ServiceLoader.load(Plugin.class, (ClassLoader) pluginClassLoader); List pluginList = ImmutableList.copyOf(serviceLoader); if (pluginList.isEmpty()) { - throw new TrinoException( - GravitinoErrorCode.GRAVITINO_CREATE_INTERNAL_CONNECTOR_ERROR, - String.format("The %s plugin does not found connector SIP interface", pluginName)); + LOG.warn("The {} plugin directory does not found connector SIP interface", pluginName); + return; } Plugin plugin = pluginList.get(0); if (plugin.getConnectorFactories() == null || !plugin.getConnectorFactories().iterator().hasNext()) { - throw new TrinoException( - GravitinoErrorCode.GRAVITINO_CREATE_INTERNAL_CONNECTOR_ERROR, - String.format("The %s plugin does not contains any ConnectorFactories", pluginName)); + LOG.warn("The {} plugin does not contains any ConnectorFactories ", pluginName); + return; } connectorPlugins.put(pluginName, pluginList.get(0)); } catch (Exception e) { throw new TrinoException( - GravitinoErrorCode.GRAVITINO_RUNTIME_ERROR, - "Failed to create Plugin class loader " + pluginName, - e); + GravitinoErrorCode.GRAVITINO_RUNTIME_ERROR, "Failed to load Plugin " + pluginName, e); } } @@ -228,7 +204,8 @@ private void loadPluginsFromBundle() { String value = GravitinoConfig.trinoConfig.getProperty(TRINO_PLUGIN_BUNDLES); Splitter splitter = Splitter.on(',').omitEmptyStrings().trimResults(); - splitter.splitToList(value).stream() + splitter + .splitToList(value) .forEach( v -> { int start = v.indexOf("trino-"); @@ -240,10 +217,11 @@ private void loadPluginsFromBundle() { return; } String key = v.substring(start, end).replace("trino-", ""); - if (!usePlugins.contains(key)) { - return; + try { + loadPluginByPom(artifactResolver.resolvePom(new File(v)), key); + } catch (Throwable t) { + LOG.error("Fatal error in load plugin by {}", v, t); } - loadPluginByPom(artifactResolver.resolvePom(new File(v)), key); }); } diff --git a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/CatalogConnectorFactory.java b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/CatalogConnectorFactory.java index 34aae1a3b2e..e297c226562 100644 --- a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/CatalogConnectorFactory.java +++ b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/CatalogConnectorFactory.java @@ -18,47 +18,17 @@ */ package org.apache.gravitino.trino.connector.catalog; -import io.trino.spi.TrinoException; -import java.util.HashMap; -import org.apache.gravitino.trino.connector.GravitinoErrorCode; -import org.apache.gravitino.trino.connector.catalog.hive.HiveConnectorAdapter; -import org.apache.gravitino.trino.connector.catalog.iceberg.IcebergConnectorAdapter; -import org.apache.gravitino.trino.connector.catalog.jdbc.mysql.MySQLConnectorAdapter; -import org.apache.gravitino.trino.connector.catalog.jdbc.postgresql.PostgreSQLConnectorAdapter; -import org.apache.gravitino.trino.connector.catalog.memory.MemoryConnectorAdapter; import org.apache.gravitino.trino.connector.metadata.GravitinoCatalog; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -/** This class use to create CatalogConnectorContext instance by given catalog. */ -public class CatalogConnectorFactory { - private static final Logger LOG = LoggerFactory.getLogger(CatalogConnectorFactory.class); - - private final HashMap catalogBuilders = new HashMap<>(); - - public CatalogConnectorFactory() { - catalogBuilders.put("hive", new CatalogConnectorContext.Builder(new HiveConnectorAdapter())); - catalogBuilders.put( - "memory", new CatalogConnectorContext.Builder(new MemoryConnectorAdapter())); - catalogBuilders.put( - "lakehouse-iceberg", new CatalogConnectorContext.Builder(new IcebergConnectorAdapter())); - catalogBuilders.put( - "jdbc-mysql", new CatalogConnectorContext.Builder(new MySQLConnectorAdapter())); - catalogBuilders.put( - "jdbc-postgresql", new CatalogConnectorContext.Builder(new PostgreSQLConnectorAdapter())); - } - - public CatalogConnectorContext.Builder createCatalogConnectorContextBuilder( - GravitinoCatalog catalog) { - String catalogProvider = catalog.getProvider(); - CatalogConnectorContext.Builder builder = catalogBuilders.get(catalogProvider); - if (builder == null) { - String message = String.format("Unsupported catalog provider %s.", catalogProvider); - LOG.error(message); - throw new TrinoException(GravitinoErrorCode.GRAVITINO_UNSUPPORTED_CATALOG_PROVIDER, message); - } - - // Avoid using the same builder object to prevent catalog creation errors. - return builder.clone(catalog); - } +/** + * This interface is used to create a CatalogConnectorContext builder connector by Gravitino catalog + */ +public interface CatalogConnectorFactory { + /** + * Create a CatalogConnectorContext builder by Gravitino catalog + * + * @param catalog Gravitino catalog + * @return CatalogConnectorContext builder + */ + CatalogConnectorContext.Builder createCatalogConnectorContextBuilder(GravitinoCatalog catalog); } diff --git a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/DefaultCatalogConnectorFactory.java b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/DefaultCatalogConnectorFactory.java new file mode 100644 index 00000000000..0f48c4efda4 --- /dev/null +++ b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/DefaultCatalogConnectorFactory.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.trino.connector.catalog; + +import io.trino.spi.TrinoException; +import java.util.HashMap; +import org.apache.gravitino.trino.connector.GravitinoConfig; +import org.apache.gravitino.trino.connector.GravitinoErrorCode; +import org.apache.gravitino.trino.connector.catalog.hive.HiveConnectorAdapter; +import org.apache.gravitino.trino.connector.catalog.iceberg.IcebergConnectorAdapter; +import org.apache.gravitino.trino.connector.catalog.jdbc.mysql.MySQLConnectorAdapter; +import org.apache.gravitino.trino.connector.catalog.jdbc.postgresql.PostgreSQLConnectorAdapter; +import org.apache.gravitino.trino.connector.catalog.memory.MemoryConnectorAdapter; +import org.apache.gravitino.trino.connector.metadata.GravitinoCatalog; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** This class use to create CatalogConnectorContext instance by given catalog. */ +public class DefaultCatalogConnectorFactory implements CatalogConnectorFactory { + private static final Logger LOG = LoggerFactory.getLogger(DefaultCatalogConnectorFactory.class); + + protected final HashMap catalogBuilders = + new HashMap<>(); + + public DefaultCatalogConnectorFactory(GravitinoConfig config) { + catalogBuilders.put("hive", new CatalogConnectorContext.Builder(new HiveConnectorAdapter())); + catalogBuilders.put( + "memory", new CatalogConnectorContext.Builder(new MemoryConnectorAdapter())); + catalogBuilders.put( + "lakehouse-iceberg", new CatalogConnectorContext.Builder(new IcebergConnectorAdapter())); + catalogBuilders.put( + "jdbc-mysql", new CatalogConnectorContext.Builder(new MySQLConnectorAdapter())); + catalogBuilders.put( + "jdbc-postgresql", new CatalogConnectorContext.Builder(new PostgreSQLConnectorAdapter())); + } + + public CatalogConnectorContext.Builder createCatalogConnectorContextBuilder( + GravitinoCatalog catalog) { + String catalogProvider = catalog.getProvider(); + CatalogConnectorContext.Builder builder = catalogBuilders.get(catalogProvider); + if (builder == null) { + String message = String.format("Unsupported catalog provider %s.", catalogProvider); + LOG.error(message); + throw new TrinoException(GravitinoErrorCode.GRAVITINO_UNSUPPORTED_CATALOG_PROVIDER, message); + } + + // Avoid using the same builder object to prevent catalog creation errors. + return builder.clone(catalog); + } +} diff --git a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java index 1f07a2412b3..2af7650b1bc 100644 --- a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java +++ b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java @@ -18,8 +18,6 @@ */ package org.apache.gravitino.trino.connector.catalog.hive; -import static org.apache.gravitino.trino.connector.GravitinoConnectorPluginManager.CONNECTOR_HIVE; - import io.trino.spi.session.PropertyMetadata; import java.util.Collections; import java.util.HashMap; @@ -36,6 +34,7 @@ */ public class HiveConnectorAdapter implements CatalogConnectorAdapter { + private static final String CONNECTOR_HIVE = "hive"; private final HasPropertyMeta propertyMetadata; private final PropertyConverter catalogConverter; diff --git a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergConnectorAdapter.java b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergConnectorAdapter.java index fe609b7f4d8..89b8f0198cf 100644 --- a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergConnectorAdapter.java +++ b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergConnectorAdapter.java @@ -24,7 +24,6 @@ import java.util.List; import java.util.Map; import org.apache.gravitino.catalog.property.PropertyConverter; -import org.apache.gravitino.trino.connector.GravitinoConnectorPluginManager; import org.apache.gravitino.trino.connector.catalog.CatalogConnectorAdapter; import org.apache.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter; import org.apache.gravitino.trino.connector.metadata.GravitinoCatalog; @@ -35,6 +34,7 @@ */ public class IcebergConnectorAdapter implements CatalogConnectorAdapter { + private static final String CONNECTOR_ICEBERG = "iceberg"; private final IcebergPropertyMeta propertyMetadata; private final PropertyConverter catalogConverter; @@ -51,7 +51,7 @@ public Map buildInternalConnectorConfig(GravitinoCatalog catalog @Override public String internalConnectorName() { - return GravitinoConnectorPluginManager.CONNECTOR_ICEBERG; + return CONNECTOR_ICEBERG; } @Override diff --git a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/JDBCCatalogPropertyConverter.java b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/JDBCCatalogPropertyConverter.java index 5c244ed5c76..23eeb2a64a3 100644 --- a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/JDBCCatalogPropertyConverter.java +++ b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/JDBCCatalogPropertyConverter.java @@ -28,9 +28,9 @@ public class JDBCCatalogPropertyConverter extends PropertyConverter { - static final String JDBC_CONNECTION_URL_KEY = "connection-url"; - static final String JDBC_CONNECTION_USER_KEY = "connection-user"; - static final String JDBC_CONNECTION_PASSWORD_KEY = "connection-password"; + public static final String JDBC_CONNECTION_URL_KEY = "connection-url"; + public static final String JDBC_CONNECTION_USER_KEY = "connection-user"; + public static final String JDBC_CONNECTION_PASSWORD_KEY = "connection-password"; private static final TreeBidiMap TRINO_KEY_TO_GRAVITINO_KEY = new TreeBidiMap<>( diff --git a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/mysql/MySQLConnectorAdapter.java b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/mysql/MySQLConnectorAdapter.java index c380fe8cc13..c4a31d18fa3 100644 --- a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/mysql/MySQLConnectorAdapter.java +++ b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/mysql/MySQLConnectorAdapter.java @@ -24,7 +24,6 @@ import java.util.List; import java.util.Map; import org.apache.gravitino.catalog.property.PropertyConverter; -import org.apache.gravitino.trino.connector.GravitinoConnectorPluginManager; import org.apache.gravitino.trino.connector.catalog.CatalogConnectorAdapter; import org.apache.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter; import org.apache.gravitino.trino.connector.catalog.HasPropertyMeta; @@ -34,6 +33,7 @@ /** Transforming MySQL connector configuration and components into Apache Gravitino connector. */ public class MySQLConnectorAdapter implements CatalogConnectorAdapter { + private static final String CONNECTOR_MYSQL = "mysql"; private final PropertyConverter catalogConverter; private final HasPropertyMeta propertyMetadata; @@ -50,7 +50,7 @@ public Map buildInternalConnectorConfig(GravitinoCatalog catalog @Override public String internalConnectorName() { - return GravitinoConnectorPluginManager.CONNECTOR_MYSQL; + return CONNECTOR_MYSQL; } @Override diff --git a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/postgresql/PostgreSQLConnectorAdapter.java b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/postgresql/PostgreSQLConnectorAdapter.java index 938ca69afed..c68d282b063 100644 --- a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/postgresql/PostgreSQLConnectorAdapter.java +++ b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/postgresql/PostgreSQLConnectorAdapter.java @@ -22,7 +22,6 @@ import java.util.Map; import org.apache.gravitino.catalog.property.PropertyConverter; -import org.apache.gravitino.trino.connector.GravitinoConnectorPluginManager; import org.apache.gravitino.trino.connector.catalog.CatalogConnectorAdapter; import org.apache.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter; import org.apache.gravitino.trino.connector.catalog.jdbc.JDBCCatalogPropertyConverter; @@ -32,6 +31,7 @@ * Transforming PostgreSQL connector configuration and components into Apache Gravitino connector. */ public class PostgreSQLConnectorAdapter implements CatalogConnectorAdapter { + private static final String CONNECTOR_POSTGRESQL = "postgresql"; private final PropertyConverter catalogConverter; public PostgreSQLConnectorAdapter() { @@ -46,7 +46,7 @@ public Map buildInternalConnectorConfig(GravitinoCatalog catalog @Override public String internalConnectorName() { - return GravitinoConnectorPluginManager.CONNECTOR_POSTGRESQL; + return CONNECTOR_POSTGRESQL; } @Override diff --git a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/memory/MemoryConnectorAdapter.java b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/memory/MemoryConnectorAdapter.java index b4d267c8f4b..544fe9e315b 100644 --- a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/memory/MemoryConnectorAdapter.java +++ b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/memory/MemoryConnectorAdapter.java @@ -18,8 +18,6 @@ */ package org.apache.gravitino.trino.connector.catalog.memory; -import static org.apache.gravitino.trino.connector.GravitinoConnectorPluginManager.CONNECTOR_MEMORY; - import io.trino.spi.session.PropertyMetadata; import java.util.Collections; import java.util.List; @@ -35,6 +33,7 @@ */ public class MemoryConnectorAdapter implements CatalogConnectorAdapter { + private static final String CONNECTOR_MEMORY = "memory"; private final HasPropertyMeta propertyMetadata; public MemoryConnectorAdapter() { diff --git a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/metadata/GravitinoCatalog.java b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/metadata/GravitinoCatalog.java index 5492c5cb1a1..d59e59e9f73 100644 --- a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/metadata/GravitinoCatalog.java +++ b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/metadata/GravitinoCatalog.java @@ -18,10 +18,14 @@ */ package org.apache.gravitino.trino.connector.metadata; +import static org.apache.gravitino.Catalog.CLOUD_REGION_CODE; + import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; import io.trino.spi.TrinoException; import java.time.Instant; import java.util.Map; @@ -41,6 +45,16 @@ public class GravitinoCatalog { private final Map properties; private final long lastModifiedTime; + static { + objectMapper = + JsonMapper.builder() + .disable(MapperFeature.AUTO_DETECT_CREATORS) + .disable(MapperFeature.AUTO_DETECT_FIELDS) + .disable(MapperFeature.AUTO_DETECT_SETTERS) + .disable(MapperFeature.AUTO_DETECT_GETTERS) + .build(); + } + public GravitinoCatalog(String metalake, Catalog catalog) { this.metalake = metalake; this.provider = catalog.provider(); @@ -116,4 +130,18 @@ public static String toJson(GravitinoCatalog catalog) throws JsonProcessingExcep public static GravitinoCatalog fromJson(String jsonString) throws JsonProcessingException { return objectMapper.readValue(jsonString, GravitinoCatalog.class); } + + public String getRegion() { + return properties.getOrDefault(CLOUD_REGION_CODE, ""); + } + + public boolean isSameRegion(String region) { + // When the Gravitino connector has not configured the cloud.region-code, + // or the catalog has not configured the cloud.region-code, + // or the catalog cluster name is equal to the connector-configured region code, + // the catalog is belong to the region + return StringUtils.isEmpty(region) + || StringUtils.isEmpty(getRegion()) + || region.equals(getRegion()); + } } diff --git a/trino-connector/src/test/java/org/apache/gravitino/trino/connector/metadata/TestGravitinoCatalog.java b/trino-connector/src/test/java/org/apache/gravitino/trino/connector/metadata/TestGravitinoCatalog.java index 80d8ca1029a..e7e6404e2d5 100644 --- a/trino-connector/src/test/java/org/apache/gravitino/trino/connector/metadata/TestGravitinoCatalog.java +++ b/trino-connector/src/test/java/org/apache/gravitino/trino/connector/metadata/TestGravitinoCatalog.java @@ -19,11 +19,14 @@ package org.apache.gravitino.trino.connector.metadata; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import java.time.Instant; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import org.apache.gravitino.Audit; import org.apache.gravitino.Catalog; @@ -41,6 +44,57 @@ public void testGravitinoCatalog() { GravitinoCatalog catalog = new GravitinoCatalog("test", mockCatalog); assertEquals(catalogName, catalog.getName()); assertEquals(provider, catalog.getProvider()); + assertEquals(catalog.getRegion(), ""); + } + + @Test + public void testCatalogWithClusterInfo() { + String catalogName = "mock"; + String provider = "hive"; + + HashMap properties = new HashMap<>(); + properties.put("cloud.region-code", "c1"); + properties.put("cloud.trino.connection-url", "jdbc:trino://gt01.orb.local:8080"); + properties.put("cloud.trino.connection-user", "admin"); + properties.put("cloud.trino.connection-password", "123"); + + Catalog mockCatalog = + mockCatalog(catalogName, provider, "test catalog", Catalog.Type.RELATIONAL, properties); + GravitinoCatalog catalog = new GravitinoCatalog("test", mockCatalog); + assertEquals(catalogName, catalog.getName()); + assertEquals(provider, catalog.getProvider()); + assertEquals(catalog.getRegion(), "c1"); + + assertEquals( + catalog.getProperty("cloud.trino.connection-url", ""), "jdbc:trino://gt01.orb.local:8080"); + assertEquals(catalog.getProperty("cloud.trino.connection-user", ""), "admin"); + assertEquals(catalog.getProperty("cloud.trino.connection-password", ""), "123"); + } + + @Test + public void testCatalogIsSameRegion() { + String catalogName = "mock"; + String provider = "hive"; + + // test with cluster info + HashMap properties = new HashMap<>(); + properties.put("cloud.region-code", "c1"); + Catalog mockCatalog = + mockCatalog(catalogName, provider, "test catalog", Catalog.Type.RELATIONAL, properties); + GravitinoCatalog catalog = new GravitinoCatalog("test", mockCatalog); + assertTrue(catalog.isSameRegion("")); + assertTrue(catalog.isSameRegion("c1")); + assertFalse(catalog.isSameRegion("c2")); + + // test with non cluster info + properties.put("cloud.region-code", ""); + mockCatalog = + mockCatalog(catalogName, provider, "test catalog", Catalog.Type.RELATIONAL, properties); + catalog = new GravitinoCatalog("test", mockCatalog); + + assertTrue(catalog.isSameRegion("")); + assertTrue(catalog.isSameRegion("c1")); + assertTrue(catalog.isSameRegion("c2")); } public static Catalog mockCatalog(