From 59dc495146126230094dcffb3ef4936ff81efc3c Mon Sep 17 00:00:00 2001 From: Yuhui Date: Mon, 22 Apr 2024 23:26:13 +0800 Subject: [PATCH] [#1785] improve (trino-connector): Gravitino Trino connector support origin Trino connector configuration in the catalog properties (#3054) ### What changes were proposed in this pull request? Gravitino Trino connector Support passing Trino connector configuration in the catalog properties. the configuration key should start with `trino.bypass`. For example using `trino.bypass.hive.config.resources` to passing the `hive.config.resources` to the Gravitino Hive catalog in Trino runtime. ### Why are the changes needed? Fix: #1785 ### Does this PR introduce _any_ user-facing change? Add documents about : Passing Trino connector properties in `trino-connector/supported-catalog.md` ### How was this patch tested? UT IT --------- Co-authored-by: Qi Yu --- .../catalog/property/PropertyConverter.java | 2 + docs/apache-hive-catalog.md | 2 + docs/jdbc-mysql-catalog.md | 2 + docs/jdbc-postgresql-catalog.md | 2 + docs/lakehouse-iceberg-catalog.md | 2 + docs/trino-connector/supported-catalog.md | 24 ++ .../test/trino/TrinoConnectorIT.java | 40 ++- .../jdbc-mysql/00008_alter_catalog.sql | 6 +- .../jdbc-mysql/00008_alter_catalog.txt | 6 +- .../jdbc-mysql/catalog_mysql_prepare.sql | 2 +- .../jdbc-postgresql/catalog_pg_prepare.sql | 2 +- .../catalog/CatalogConnectorManager.java | 3 +- .../hive/HiveCatalogPropertyConverter.java | 261 +++++++++----- .../catalog/hive/HiveConnectorAdapter.java | 14 +- .../IcebergCatalogPropertyConverter.java | 328 ++++++++++++------ .../jdbc/JDBCCatalogPropertyConverter.java | 77 ++-- .../connector/metadata/GravitinoCatalog.java | 2 +- .../trino/connector/GravitinoMockServer.java | 7 +- .../TestCreateGravitinoConnector.java | 4 +- .../connector/TestGravitinoConnector.java | 2 +- ...itinoConnectorWithMetalakeCatalogName.java | 2 +- .../TestHiveCatalogPropertyConverter.java | 47 ++- .../TestIcebergCatalogPropertyConverter.java | 88 +++++ .../TestJDBCCatalogPropertyConverter.java | 84 +++++ .../metadata/TestGravitinoCatalog.java | 2 +- 25 files changed, 737 insertions(+), 274 deletions(-) diff --git a/catalogs/bundled-catalog/src/main/java/com/datastrato/gravitino/catalog/property/PropertyConverter.java b/catalogs/bundled-catalog/src/main/java/com/datastrato/gravitino/catalog/property/PropertyConverter.java index 7125a044002..30619f832cf 100644 --- a/catalogs/bundled-catalog/src/main/java/com/datastrato/gravitino/catalog/property/PropertyConverter.java +++ b/catalogs/bundled-catalog/src/main/java/com/datastrato/gravitino/catalog/property/PropertyConverter.java @@ -15,6 +15,8 @@ /** Transforming between gravitino schema/table/column property and engine property. */ public abstract class PropertyConverter { + protected static final String TRINO_PROPERTIES_PREFIX = "trino.bypass."; + private static final Logger LOG = LoggerFactory.getLogger(PropertyConverter.class); /** * Mapping that maps engine properties to Gravitino properties. It will return a map that holds diff --git a/docs/apache-hive-catalog.md b/docs/apache-hive-catalog.md index 2fe3da52748..6743dad5b93 100644 --- a/docs/apache-hive-catalog.md +++ b/docs/apache-hive-catalog.md @@ -40,6 +40,8 @@ The Hive catalog supports creating, updating, and deleting databases and tables | `kerberos.check-interval-sec` | The interval to check validness of the principal | 60 | No | 0.4.0 | | `kerberos.keytab-fetch-timeout-sec` | The timeout to fetch key tab | 60 | No | 0.4.0 | +When you use the Gravitino with Trino. You can pass the Trino Hive connector configuration using prefix `trino.bypass.`. For example, using `trino.bypass.hive.config.resources` to pass the `hive.config.resources` to the Gravitino Hive catalog in Trino runtime. + ### Catalog operations Refer to [Manage Relational Metadata Using Gravitino](./manage-relational-metadata-using-gravitino.md#catalog-operations) for more details. diff --git a/docs/jdbc-mysql-catalog.md b/docs/jdbc-mysql-catalog.md index aba936b28c6..fa0569f4415 100644 --- a/docs/jdbc-mysql-catalog.md +++ b/docs/jdbc-mysql-catalog.md @@ -37,6 +37,8 @@ You can pass to a MySQL data source any property that isn't defined by Gravitino Check the relevant data source configuration in [data source properties](https://commons.apache.org/proper/commons-dbcp/configuration.html) +When you use the Gravitino with Trino. You can pass the Trino MySQL connector configuration using prefix `trino.bypass.`. For example, using `trino.bypass.join-pushdown.strategy` to pass the `join-pushdown.strategy` to the Gravitino MySQL catalog in Trino runtime. + If you use a JDBC catalog, you must provide `jdbc-url`, `jdbc-driver`, `jdbc-user` and `jdbc-password` to catalog properties. | Configuration item | Description | Default value | Required | Since Version | diff --git a/docs/jdbc-postgresql-catalog.md b/docs/jdbc-postgresql-catalog.md index 5f673ecac68..69b7acd0350 100644 --- a/docs/jdbc-postgresql-catalog.md +++ b/docs/jdbc-postgresql-catalog.md @@ -35,6 +35,8 @@ Gravitino saves some system information in schema and table comment, like `(From Any property that isn't defined by Gravitino can pass to MySQL data source by adding `gravitino.bypass.` prefix as a catalog property. For example, catalog property `gravitino.bypass.maxWaitMillis` will pass `maxWaitMillis` to the data source property. You can check the relevant data source configuration in [data source properties](https://commons.apache.org/proper/commons-dbcp/configuration.html) +When you use the Gravitino with Trino. You can pass the Trino PostgreSQL connector configuration using prefix `trino.bypass.`. For example, using `trino.bypass.join-pushdown.strategy` to pass the `join-pushdown.strategy` to the Gravitino PostgreSQL catalog in Trino runtime. + If you use JDBC catalog, you must provide `jdbc-url`, `jdbc-driver`, `jdbc-database`, `jdbc-user` and `jdbc-password` to catalog properties. | Configuration item | Description | Default value | Required | Since Version | diff --git a/docs/lakehouse-iceberg-catalog.md b/docs/lakehouse-iceberg-catalog.md index a5d1b67b4b2..f33f32b97eb 100644 --- a/docs/lakehouse-iceberg-catalog.md +++ b/docs/lakehouse-iceberg-catalog.md @@ -41,6 +41,8 @@ Builds with Hadoop 2.10.x, there may be compatibility issues when accessing Hado Any properties not defined by Gravitino with `gravitino.bypass.` prefix will pass to Iceberg catalog properties and HDFS configuration. For example, if specify `gravitino.bypass.list-all-tables`, `list-all-tables` will pass to Iceberg catalog properties. +When you use the Gravitino with Trino. You can pass the Trino Iceberg connector configuration using prefix `trino.bypass.`. For example, using `trino.bypass.iceberg.table-statistics-enabled` to pass the `iceberg.table-statistics-enabled` to the Gravitino Iceberg catalog in Trino runtime. + #### JDBC catalog If you are using JDBC catalog, you must provide `jdbc-user`, `jdbc-password` and `jdbc-driver` to catalog properties. diff --git a/docs/trino-connector/supported-catalog.md b/docs/trino-connector/supported-catalog.md index 158002c469f..5c9458e7449 100644 --- a/docs/trino-connector/supported-catalog.md +++ b/docs/trino-connector/supported-catalog.md @@ -122,6 +122,30 @@ call gravitino.system.alter_catalog( if you need more information about catalog, please refer to: [Create a Catalog](../manage-relational-metadata-using-gravitino.md#create-a-catalog). +## Passing Trino connector configuration +A Gravitino catalog is implemented by the Trino connector, so you can pass the Trino connector configuration to the Gravitino catalog. +For example, you want to set the `hive.config.resources` configuration for the Hive catalog, you can pass the configuration to the +Gravitino catalog like this: + +```sql +call gravitino.system.create_catalog( + 'gt_hive', + 'hive', + map( + array['metastore.uris', 'trino.bypass.hive.config.resources'], + array['thrift://trino-ci-hive:9083', "/tmp/hive-site.xml,/tmp/core-site.xml"] + ) +); +``` + +A prefix with `trino.bypass.` in the configuration key is used to indicate Gravitino connector to pass the Trino connector configuration to the Gravitino catalog in the Trino runtime. + +More trino connector configurations can refer to: +- [Hive catalog](https://trino.io/docs/current/connector/hive.html#hive-general-configuration-properties) +- [Iceberg catalog](https://trino.io/docs/current/connector/iceberg.html#general-configuration) +- [MySQL catalog](https://trino.io/docs/current/connector/mysql.html#general-configuration-properties) +- [PostgreSQL catalog](https://trino.io/docs/current/connector/postgresql.html#general-configuration-properties) + ## Data type mapping between Trino and Gravitino Gravitino connector supports the following data type conversions between Trino and Gravitino currently. Depending on the detailed catalog, Gravitino may not support some data types conversion for this specific catalog, for example, diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java index 1f8c90ce769..57f64493d1d 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java @@ -825,16 +825,20 @@ void testHiveCatalogCreatedByGravitino() throws InterruptedException { "thrift://%s:%s", containerSuite.getHiveContainer().getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT)) - .put("hive.immutable-partitions", "true") - .put("hive.target-max-file-size", "1GB") - .put("hive.create-empty-bucket-files", "true") - .put("hive.validate-bucketing", "true") + .put("trino.bypass.hive.immutable-partitions", "true") + .put("trino.bypass.hive.target-max-file-size", "1GB") + .put("trino.bypass.hive.create-empty-bucket-files", "true") + .put("trino.bypass.hive.validate-bucketing", "true") .build()); Catalog catalog = createdMetalake.loadCatalog(NameIdentifier.of(metalakeName, catalogName)); - Assertions.assertEquals("true", catalog.properties().get("hive.immutable-partitions")); - Assertions.assertEquals("1GB", catalog.properties().get("hive.target-max-file-size")); - Assertions.assertEquals("true", catalog.properties().get("hive.create-empty-bucket-files")); - Assertions.assertEquals("true", catalog.properties().get("hive.validate-bucketing")); + Assertions.assertEquals( + "true", catalog.properties().get("trino.bypass.hive.immutable-partitions")); + Assertions.assertEquals( + "1GB", catalog.properties().get("trino.bypass.hive.target-max-file-size")); + Assertions.assertEquals( + "true", catalog.properties().get("trino.bypass.hive.create-empty-bucket-files")); + Assertions.assertEquals( + "true", catalog.properties().get("trino.bypass.hive.validate-bucketing")); String sql = String.format("show catalogs like '%s'", catalogName); boolean success = checkTrinoHasLoaded(sql, 30); @@ -863,17 +867,21 @@ void testWrongHiveCatalogProperty() throws InterruptedException { "thrift://%s:%s", containerSuite.getHiveContainer().getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT)) - .put("hive.immutable-partitions", "true") + .put("trino.bypass.hive.immutable-partitions", "true") // it should be like '1GB, 1MB', we make it wrong purposely. - .put("hive.target-max-file-size", "xxxx") - .put("hive.create-empty-bucket-files", "true") - .put("hive.validate-bucketing", "true") + .put("trino.bypass.hive.target-max-file-size", "xxxx") + .put("trino.bypass.hive.create-empty-bucket-files", "true") + .put("trino.bypass.hive.validate-bucketing", "true") .build()); Catalog catalog = createdMetalake.loadCatalog(NameIdentifier.of(metalakeName, catalogName)); - Assertions.assertEquals("true", catalog.properties().get("hive.immutable-partitions")); - Assertions.assertEquals("xxxx", catalog.properties().get("hive.target-max-file-size")); - Assertions.assertEquals("true", catalog.properties().get("hive.create-empty-bucket-files")); - Assertions.assertEquals("true", catalog.properties().get("hive.validate-bucketing")); + Assertions.assertEquals( + "true", catalog.properties().get("trino.bypass.hive.immutable-partitions")); + Assertions.assertEquals( + "xxxx", catalog.properties().get("trino.bypass.hive.target-max-file-size")); + Assertions.assertEquals( + "true", catalog.properties().get("trino.bypass.hive.create-empty-bucket-files")); + Assertions.assertEquals( + "true", catalog.properties().get("trino.bypass.hive.validate-bucketing")); String sql = String.format("show catalogs like '%s'", catalogName); checkTrinoHasLoaded(sql, 6); diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.sql index 46d8b8c8034..7fe2ab1ec39 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.sql @@ -12,7 +12,7 @@ select * from gravitino.system.catalog where name = 'gt_mysql_xxx1'; call gravitino.system.alter_catalog( 'gt_mysql_xxx1', map( - array['join-pushdown.strategy', 'test_key'], + array['trino.bypass.join-pushdown.strategy', 'test_key'], array['EAGER', 'test_value'] ) ); @@ -22,7 +22,7 @@ select * from gravitino.system.catalog where name = 'gt_mysql_xxx1'; call gravitino.system.alter_catalog( 'gt_mysql_xxx1', map(), - array['join-pushdown.strategy'] + array['trino.bypass.join-pushdown.strategy'] ); select * from gravitino.system.catalog where name = 'gt_mysql_xxx1'; @@ -30,7 +30,7 @@ select * from gravitino.system.catalog where name = 'gt_mysql_xxx1'; call gravitino.system.alter_catalog( catalog => 'gt_mysql_xxx1', set_properties => map( - array['join-pushdown.strategy'], + array['trino.bypass.join-pushdown.strategy'], array['EAGER'] ), remove_properties => array['test_key'] diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.txt index b0e3aac0dcc..35dab129f61 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.txt @@ -4,7 +4,7 @@ CALL CALL -"gt_mysql_xxx1","jdbc-mysql","{""join-pushdown.strategy"":""EAGER"",""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""jdbc-password"":""ds123"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""test_key"":""test_value""}" +"gt_mysql_xxx1","jdbc-mysql","{""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""jdbc-password"":""ds123"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""test_key"":""test_value"",""trino.bypass.join-pushdown.strategy"":""EAGER""}" CALL @@ -12,6 +12,6 @@ CALL CALL -"gt_mysql_xxx1","jdbc-mysql","{""join-pushdown.strategy"":""EAGER"",""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""jdbc-password"":""ds123"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver""}" +"gt_mysql_xxx1","jdbc-mysql","{""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""jdbc-password"":""ds123"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""trino.bypass.join-pushdown.strategy"":""EAGER""}" -CALL \ No newline at end of file +CALL diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/catalog_mysql_prepare.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/catalog_mysql_prepare.sql index eacf543b4ab..19c42e58ed3 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/catalog_mysql_prepare.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/catalog_mysql_prepare.sql @@ -2,7 +2,7 @@ call gravitino.system.create_catalog( 'gt_mysql', 'jdbc-mysql', map( - array['jdbc-url', 'jdbc-user', 'jdbc-password', 'jdbc-driver', 'join-pushdown.strategy'], + array['jdbc-url', 'jdbc-user', 'jdbc-password', 'jdbc-driver', 'trino.bypass.join-pushdown.strategy'], array['${mysql_uri}/?useSSL=false', 'trino', 'ds123', 'com.mysql.cj.jdbc.Driver', 'EAGER'] ) ); \ No newline at end of file diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/catalog_pg_prepare.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/catalog_pg_prepare.sql index d99ff9d8081..4872e799bd4 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/catalog_pg_prepare.sql +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/catalog_pg_prepare.sql @@ -2,7 +2,7 @@ call gravitino.system.create_catalog( 'gt_postgresql', 'jdbc-postgresql', map( - array['jdbc-url', 'jdbc-user', 'jdbc-password', 'jdbc-database', 'jdbc-driver', 'join-pushdown.strategy'], + array['jdbc-url', 'jdbc-user', 'jdbc-password', 'jdbc-database', 'jdbc-driver', 'trino.bypass.join-pushdown.strategy'], array['${postgresql_uri}/gt_db', 'trino', 'ds123', 'gt_db', 'org.postgresql.Driver', 'EAGER'] ) ); \ No newline at end of file diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java index ac1da69bfbe..d5541c9aed2 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java @@ -167,8 +167,7 @@ public void loadCatalogs(GravitinoMetalake metalake) { (NameIdentifier nameIdentifier) -> { try { Catalog catalog = metalake.loadCatalog(nameIdentifier); - GravitinoCatalog gravitinoCatalog = - new GravitinoCatalog(metalake.name(), catalog, config.simplifyCatalogNames()); + GravitinoCatalog gravitinoCatalog = new GravitinoCatalog(metalake.name(), catalog); if (catalogConnectors.containsKey(getTrinoCatalogName(gravitinoCatalog))) { // Reload catalogs that have been updated in Gravitino server. reloadCatalog(metalake, gravitinoCatalog); diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveCatalogPropertyConverter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveCatalogPropertyConverter.java index fec7317f523..4d675b977df 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveCatalogPropertyConverter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveCatalogPropertyConverter.java @@ -23,117 +23,204 @@ public class HiveCatalogPropertyConverter extends PropertyConverter { new ImmutableMap.Builder() // Key is the Trino property, value is the Gravitino property // General configuration - .put("hive.config.resources", "hive.config.resources") - .put("hive.recursive-directories", "hive.recursive-directories") - .put("hive.ignore-absent-partitions", "hive.ignore-absent-partitions") - .put("hive.storage-format", "hive.storage-format") - .put("hive.compression-codec", "hive.compression-codec") - .put("hive.force-local-scheduling", "hive.force-local-scheduling") - .put("hive.respect-table-format", "hive.respect-table-format") - .put("hive.immutable-partitions", "hive.immutable-partitions") + .put("hive.config.resources", TRINO_PROPERTIES_PREFIX + "hive.config.resources") + .put( + "hive.recursive-directories", + TRINO_PROPERTIES_PREFIX + "hive.recursive-directories") + .put( + "hive.ignore-absent-partitions", + TRINO_PROPERTIES_PREFIX + "hive.ignore-absent-partitions") + .put("hive.storage-format", TRINO_PROPERTIES_PREFIX + "hive.storage-format") + .put("hive.compression-codec", TRINO_PROPERTIES_PREFIX + "hive.compression-codec") + .put( + "hive.force-local-scheduling", + TRINO_PROPERTIES_PREFIX + "hive.force-local-scheduling") + .put( + "hive.respect-table-format", + TRINO_PROPERTIES_PREFIX + "hive.respect-table-format") + .put( + "hive.immutable-partitions", + TRINO_PROPERTIES_PREFIX + "hive.immutable-partitions") .put( "hive.insert-existing-partitions-behavior", - "hive.insert-existing-partitions-behavior") - .put("hive.target-max-file-size", "hive.target-max-file-size") - .put("hive.create-empty-bucket-files", "hive.create-empty-bucket-files") - .put("hive.validate-bucketing", "hive.validate-bucketing") - .put("hive.partition-statistics-sample-size", "hive.partition-statistics-sample-size") - .put("hive.max-partitions-per-writers", "hive.max-partitions-per-writers") - .put("hive.max-partitions-for-eager-load", "hive.max-partitions-for-eager-load") - .put("hive.max-partitions-per-scan", "hive.max-partitions-per-scan") - .put("hive.dfs.replication", "hive.dfs.replication") - .put("hive.security", "hive.security") - .put("security.config-file", "security.config-file") - .put("hive.non-managed-table-writes-enabled", "hive.non-managed-table-writes-enabled") + TRINO_PROPERTIES_PREFIX + "hive.insert-existing-partitions-behavior") + .put( + "hive.target-max-file-size", + TRINO_PROPERTIES_PREFIX + "hive.target-max-file-size") + .put( + "hive.create-empty-bucket-files", + TRINO_PROPERTIES_PREFIX + "hive.create-empty-bucket-files") + .put("hive.validate-bucketing", TRINO_PROPERTIES_PREFIX + "hive.validate-bucketing") + .put( + "hive.partition-statistics-sample-size", + TRINO_PROPERTIES_PREFIX + "hive.partition-statistics-sample-size") + .put( + "hive.max-partitions-per-writers", + TRINO_PROPERTIES_PREFIX + "hive.max-partitions-per-writers") + .put( + "hive.max-partitions-for-eager-load", + TRINO_PROPERTIES_PREFIX + "hive.max-partitions-for-eager-load") + .put( + "hive.max-partitions-per-scan", + TRINO_PROPERTIES_PREFIX + "hive.max-partitions-per-scan") + .put("hive.dfs.replication", TRINO_PROPERTIES_PREFIX + "hive.dfs.replication") + .put("hive.security", TRINO_PROPERTIES_PREFIX + "hive.security") + .put("security.config-file", TRINO_PROPERTIES_PREFIX + "security.config-file") + .put( + "hive.non-managed-table-writes-enabled", + TRINO_PROPERTIES_PREFIX + "hive.non-managed-table-writes-enabled") .put( "hive.non-managed-table-creates-enabled", - "hive.non-managed-table-creates-enabled") + TRINO_PROPERTIES_PREFIX + "hive.non-managed-table-creates-enabled") .put( "hive.collect-column-statistics-on-write", - "hive.collect-column-statistics-on-write") - .put("hive.file-status-cache-tables", "hive.file-status-cache-tables") + TRINO_PROPERTIES_PREFIX + "hive.collect-column-statistics-on-write") + .put( + "hive.file-status-cache-tables", + TRINO_PROPERTIES_PREFIX + "hive.file-status-cache-tables") .put( "hive.file-status-cache.max-retained-size", - "hive.file-status-cache.max-retained-size") - .put("hive.file-status-cache-expire-time", "hive.file-status-cache-expire-time") + TRINO_PROPERTIES_PREFIX + "hive.file-status-cache.max-retained-size") + .put( + "hive.file-status-cache-expire-time", + TRINO_PROPERTIES_PREFIX + "hive.file-status-cache-expire-time") .put( "hive.per-transaction-file-status-cache.max-retained-size", - "hive.per-transaction-file-status-cache.max-retained-size") - .put("hive.timestamp-precision", "hive.timestamp-precision") + TRINO_PROPERTIES_PREFIX + + "hive.per-transaction-file-status-cache.max-retained-size") + .put("hive.timestamp-precision", TRINO_PROPERTIES_PREFIX + "hive.timestamp-precision") .put( "hive.temporary-staging-directory-enabled", - "hive.temporary-staging-directory-enabled") - .put("hive.temporary-staging-directory-path", "hive.temporary-staging-directory-path") - .put("hive.hive-views.enabled", "hive.hive-views.enabled") - .put("hive.hive-views.legacy-translation", "hive.hive-views.legacy-translation") + TRINO_PROPERTIES_PREFIX + "hive.temporary-staging-directory-enabled") + .put( + "hive.temporary-staging-directory-path", + TRINO_PROPERTIES_PREFIX + "hive.temporary-staging-directory-path") + .put("hive.hive-views.enabled", TRINO_PROPERTIES_PREFIX + "hive.hive-views.enabled") + .put( + "hive.hive-views.legacy-translation", + TRINO_PROPERTIES_PREFIX + "hive.hive-views.legacy-translation") .put( "hive.parallel-partitioned-bucketed-writes", - "hive.parallel-partitioned-bucketed-writes") - .put("hive.fs.new-directory-permissions", "hive.fs.new-directory-permissions") - .put("hive.fs.cache.max-size", "hive.fs.cache.max-size") - .put("hive.query-partition-filter-required", "hive.query-partition-filter-required") - .put("hive.table-statistics-enabled", "hive.table-statistics-enabled") - .put("hive.auto-purge", "hive.auto-purge") - .put("hive.partition-projection-enabled", "hive.partition-projection-enabled") - .put("hive.max-partition-drops-per-query", "hive.max-partition-drops-per-query") - .put("hive.single-statement-writes", "hive.single-statement-writes") + TRINO_PROPERTIES_PREFIX + "hive.parallel-partitioned-bucketed-writes") + .put( + "hive.fs.new-directory-permissions", + TRINO_PROPERTIES_PREFIX + "hive.fs.new-directory-permissions") + .put("hive.fs.cache.max-size", TRINO_PROPERTIES_PREFIX + "hive.fs.cache.max-size") + .put( + "hive.query-partition-filter-required", + TRINO_PROPERTIES_PREFIX + "hive.query-partition-filter-required") + .put( + "hive.table-statistics-enabled", + TRINO_PROPERTIES_PREFIX + "hive.table-statistics-enabled") + .put("hive.auto-purge", TRINO_PROPERTIES_PREFIX + "hive.auto-purge") + .put( + "hive.partition-projection-enabled", + TRINO_PROPERTIES_PREFIX + "hive.partition-projection-enabled") + .put( + "hive.max-partition-drops-per-query", + TRINO_PROPERTIES_PREFIX + "hive.max-partition-drops-per-query") + .put( + "hive.single-statement-writes", + TRINO_PROPERTIES_PREFIX + "hive.single-statement-writes") // Performance - .put("hive.max-outstanding-splits", "hive.max-outstanding-splits") - .put("hive.max-outstanding-splits-size", "hive.max-outstanding-splits-size") - .put("hive.max-splits-per-second", "hive.max-splits-per-second") - .put("hive.max-initial-splits", "hive.max-initial-splits") - .put("hive.max-initial-split-size", "hive.max-initial-split-size") - .put("hive.max-split-size", "hive.max-split-size") + .put( + "hive.max-outstanding-splits", + TRINO_PROPERTIES_PREFIX + "hive.max-outstanding-splits") + .put( + "hive.max-outstanding-splits-size", + TRINO_PROPERTIES_PREFIX + "hive.max-outstanding-splits-size") + .put( + "hive.max-splits-per-second", + TRINO_PROPERTIES_PREFIX + "hive.max-splits-per-second") + .put("hive.max-initial-splits", TRINO_PROPERTIES_PREFIX + "hive.max-initial-splits") + .put( + "hive.max-initial-split-size", + TRINO_PROPERTIES_PREFIX + "hive.max-initial-split-size") + .put("hive.max-split-size", TRINO_PROPERTIES_PREFIX + "hive.max-split-size") // S3 - .put("hive.s3.aws-access-key", "hive.s3.aws-access-key") - .put("hive.s3.aws-secret-key", "hive.s3.aws-secret-key") - .put("hive.s3.iam-role", "hive.s3.iam-role") - .put("hive.s3.external-id", "hive.s3.external-id") - .put("hive.s3.endpoint", "hive.s3.endpoint") - .put("hive.s3.region", "hive.s3.region") - .put("hive.s3.storage-class", "hive.s3.storage-class") - .put("hive.s3.signer-type", "hive.s3.signer-type") - .put("hive.s3.signer-class", "hive.s3.signer-class") - .put("hive.s3.path-style-access", "hive.s3.path-style-access") - .put("hive.s3.staging-directory", "hive.s3.staging-directory") - .put("hive.s3.pin-client-to-current-region", "hive.s3.pin-client-to-current-region") - .put("hive.s3.ssl.enabled", "hive.s3.ssl.enabled") - .put("hive.s3.sse.enabled", "hive.s3.sse.enabled") - .put("hive.s3.sse.type", "hive.s3.sse.type") - .put("hive.s3.sse.kms-key-id", "hive.s3.sse.kms-key-id") - .put("hive.s3.kms-key-id", "hive.s3.kms-key-id") - .put("hive.s3.encryption-materials-provider", "hive.s3.encryption-materials-provider") - .put("hive.s3.upload-acl-type", "hive.s3.upload-acl-type") - .put("hive.s3.skip-glacier-objects", "hive.s3.skip-glacier-objects") - .put("hive.s3.streaming.enabled", "hive.s3.streaming.enabled") - .put("hive.s3.streaming.part-size", "hive.s3.streaming.part-size") - .put("hive.s3.proxy.host", "hive.s3.proxy.host") - .put("hive.s3.proxy.port", "hive.s3.proxy.port") - .put("hive.s3.proxy.protocol", "hive.s3.proxy.protocol") - .put("hive.s3.proxy.non-proxy-hosts", "hive.s3.proxy.non-proxy-hosts") - .put("hive.s3.proxy.username", "hive.s3.proxy.username") - .put("hive.s3.proxy.password", "hive.s3.proxy.password") - .put("hive.s3.proxy.preemptive-basic-auth", "hive.s3.proxy.preemptive-basic-auth") - .put("hive.s3.sts.endpoint", "hive.s3.sts.endpoint") - .put("hive.s3.sts.region", "hive.s3.sts.region") + .put("hive.s3.aws-access-key", TRINO_PROPERTIES_PREFIX + "hive.s3.aws-access-key") + .put("hive.s3.aws-secret-key", TRINO_PROPERTIES_PREFIX + "hive.s3.aws-secret-key") + .put("hive.s3.iam-role", TRINO_PROPERTIES_PREFIX + "hive.s3.iam-role") + .put("hive.s3.external-id", TRINO_PROPERTIES_PREFIX + "hive.s3.external-id") + .put("hive.s3.endpoint", TRINO_PROPERTIES_PREFIX + "hive.s3.endpoint") + .put("hive.s3.region", TRINO_PROPERTIES_PREFIX + "hive.s3.region") + .put("hive.s3.storage-class", TRINO_PROPERTIES_PREFIX + "hive.s3.storage-class") + .put("hive.s3.signer-type", TRINO_PROPERTIES_PREFIX + "hive.s3.signer-type") + .put("hive.s3.signer-class", TRINO_PROPERTIES_PREFIX + "hive.s3.signer-class") + .put( + "hive.s3.path-style-access", + TRINO_PROPERTIES_PREFIX + "hive.s3.path-style-access") + .put( + "hive.s3.staging-directory", + TRINO_PROPERTIES_PREFIX + "hive.s3.staging-directory") + .put( + "hive.s3.pin-client-to-current-region", + TRINO_PROPERTIES_PREFIX + "hive.s3.pin-client-to-current-region") + .put("hive.s3.ssl.enabled", TRINO_PROPERTIES_PREFIX + "hive.s3.ssl.enabled") + .put("hive.s3.sse.enabled", TRINO_PROPERTIES_PREFIX + "hive.s3.sse.enabled") + .put("hive.s3.sse.type", TRINO_PROPERTIES_PREFIX + "hive.s3.sse.type") + .put("hive.s3.sse.kms-key-id", TRINO_PROPERTIES_PREFIX + "hive.s3.sse.kms-key-id") + .put("hive.s3.kms-key-id", TRINO_PROPERTIES_PREFIX + "hive.s3.kms-key-id") + .put( + "hive.s3.encryption-materials-provider", + TRINO_PROPERTIES_PREFIX + "hive.s3.encryption-materials-provider") + .put("hive.s3.upload-acl-type", TRINO_PROPERTIES_PREFIX + "hive.s3.upload-acl-type") + .put( + "hive.s3.skip-glacier-objects", + TRINO_PROPERTIES_PREFIX + "hive.s3.skip-glacier-objects") + .put( + "hive.s3.streaming.enabled", + TRINO_PROPERTIES_PREFIX + "hive.s3.streaming.enabled") + .put( + "hive.s3.streaming.part-size", + TRINO_PROPERTIES_PREFIX + "hive.s3.streaming.part-size") + .put("hive.s3.proxy.host", TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.host") + .put("hive.s3.proxy.port", TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.port") + .put("hive.s3.proxy.protocol", TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.protocol") + .put( + "hive.s3.proxy.non-proxy-hosts", + TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.non-proxy-hosts") + .put("hive.s3.proxy.username", TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.username") + .put("hive.s3.proxy.password", TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.password") + .put( + "hive.s3.proxy.preemptive-basic-auth", + TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.preemptive-basic-auth") + .put("hive.s3.sts.endpoint", TRINO_PROPERTIES_PREFIX + "hive.s3.sts.endpoint") + .put("hive.s3.sts.region", TRINO_PROPERTIES_PREFIX + "hive.s3.sts.region") // Hive metastore Thrift service authentication - .put("hive.metastore.authentication.type", "hive.metastore.authentication.type") + .put( + "hive.metastore.authentication.type", + TRINO_PROPERTIES_PREFIX + "hive.metastore.authentication.type") .put( "hive.metastore.thrift.impersonation.enabled", - "hive.metastore.thrift.impersonation.enabled") - .put("hive.metastore.service.principal", "hive.metastore.service.principal") - .put("hive.metastore.client.principal", "hive.metastore.client.principal") - .put("hive.metastore.client.keytab", "hive.metastore.client.keytab") + TRINO_PROPERTIES_PREFIX + "hive.metastore.thrift.impersonation.enabled") + .put( + "hive.metastore.service.principal", + TRINO_PROPERTIES_PREFIX + "hive.metastore.service.principal") + .put( + "hive.metastore.client.principal", + TRINO_PROPERTIES_PREFIX + "hive.metastore.client.principal") + .put( + "hive.metastore.client.keytab", + TRINO_PROPERTIES_PREFIX + "hive.metastore.client.keytab") // HDFS authentication - .put("hive.hdfs.authentication.type", "hive.hdfs.authentication.type") - .put("hive.hdfs.impersonation.enabled", "hive.hdfs.impersonation.enabled") - .put("hive.hdfs.trino.principal", "hive.hdfs.trino.principal") - .put("hive.hdfs.trino.keytab", "hive.hdfs.trino.keytab") - .put("hive.hdfs.wire-encryption.enabled", "hive.hdfs.wire-encryption.enabled") + .put( + "hive.hdfs.authentication.type", + TRINO_PROPERTIES_PREFIX + "hive.hdfs.authentication.type") + .put( + "hive.hdfs.impersonation.enabled", + TRINO_PROPERTIES_PREFIX + "hive.hdfs.impersonation.enabled") + .put( + "hive.hdfs.trino.principal", + TRINO_PROPERTIES_PREFIX + "hive.hdfs.trino.principal") + .put("hive.hdfs.trino.keytab", TRINO_PROPERTIES_PREFIX + "hive.hdfs.trino.keytab") + .put( + "hive.hdfs.wire-encryption.enabled", + TRINO_PROPERTIES_PREFIX + "hive.hdfs.wire-encryption.enabled") .build()); @Override diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java index 3ada2983b94..9d42ce80342 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java @@ -9,7 +9,6 @@ import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter; import com.datastrato.gravitino.trino.connector.catalog.HasPropertyMeta; import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; -import com.google.common.collect.Maps; import io.trino.spi.session.PropertyMetadata; import java.util.Collections; import java.util.HashMap; @@ -44,18 +43,7 @@ public Map buildInternalConnectorConfig(GravitinoCatalog catalog properties.put("hive.security", "allow-all"); Map trinoProperty = catalogConverter.gravitinoToEngineProperties(catalog.getProperties()); - - // Trino only supports properties that define in catalogPropertyMeta, the name of entries in - // catalogPropertyMeta is in the format of "catalogName_propertyName", so we need to replace - // '_' with '.' to align with the name in trino. - Map> catalogPropertyMeta = - Maps.uniqueIndex( - propertyMetadata.getCatalogPropertyMeta(), - propertyMetadata -> propertyMetadata.getName().replace("_", ".")); - - trinoProperty.entrySet().stream() - .filter(entry -> catalogPropertyMeta.containsKey(entry.getKey())) - .forEach(entry -> properties.put(entry.getKey(), entry.getValue())); + properties.putAll(trinoProperty); config.put("properties", properties); return config; diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java index 4be199546f9..74397799a40 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java @@ -22,152 +22,259 @@ public class IcebergCatalogPropertyConverter extends PropertyConverter { new TreeBidiMap<>( new ImmutableMap.Builder() // General configuration - .put("iceberg.catalog.type", "iceberg.catalog.type") - .put("iceberg.file-format", "iceberg.file-format") - .put("iceberg.compression-codec", "iceberg.compression-codec") - .put("iceberg.use-file-size-from-metadata", "iceberg.use-file-size-from-metadata") - .put("iceberg.max-partitions-per-writer", "iceberg.max-partitions-per-writer") - .put("iceberg.target-max-file-size", "iceberg.target-max-file-size") - .put("iceberg.unique-table-location", "iceberg.unique-table-location") + .put("iceberg.catalog.type", TRINO_PROPERTIES_PREFIX + "iceberg.catalog.type") + .put("iceberg.file-format", TRINO_PROPERTIES_PREFIX + "iceberg.file-format") + .put( + "iceberg.compression-codec", + TRINO_PROPERTIES_PREFIX + "iceberg.compression-codec") + .put( + "iceberg.use-file-size-from-metadata", + TRINO_PROPERTIES_PREFIX + "iceberg.use-file-size-from-metadata") + .put( + "iceberg.max-partitions-per-writer", + TRINO_PROPERTIES_PREFIX + "iceberg.max-partitions-per-writer") + .put( + "iceberg.target-max-file-size", + TRINO_PROPERTIES_PREFIX + "iceberg.target-max-file-size") + .put( + "iceberg.unique-table-location", + TRINO_PROPERTIES_PREFIX + "iceberg.unique-table-location") .put( "iceberg.dynamic-filtering.wait-timeout", - "iceberg.dynamic-filtering.wait-timeout") + TRINO_PROPERTIES_PREFIX + "iceberg.dynamic-filtering.wait-timeout") .put( "iceberg.delete-schema-locations-fallback", - "iceberg.delete-schema-locations-fallback") - .put("iceberg.minimum-assigned-split-weight", "iceberg.minimum-assigned-split-weight") - .put("iceberg.table-statistics-enabled", "iceberg.table-statistics-enabled") - .put("iceberg.extended-statistics.enabled", "iceberg.extended-statistics.enabled") + TRINO_PROPERTIES_PREFIX + "iceberg.delete-schema-locations-fallback") + .put( + "iceberg.minimum-assigned-split-weight", + TRINO_PROPERTIES_PREFIX + "iceberg.minimum-assigned-split-weight") + .put( + "iceberg.table-statistics-enabled", + TRINO_PROPERTIES_PREFIX + "iceberg.table-statistics-enabled") + .put( + "iceberg.extended-statistics.enabled", + TRINO_PROPERTIES_PREFIX + "iceberg.extended-statistics.enabled") .put( "iceberg.extended-statistics.collect-on-write", - "iceberg.extended-statistics.collect-on-write") - .put("iceberg.projection-pushdown-enabled", "iceberg.projection-pushdown-enabled") - .put("iceberg.hive-catalog-name", "iceberg.hive-catalog-name") + TRINO_PROPERTIES_PREFIX + "iceberg.extended-statistics.collect-on-write") + .put( + "iceberg.projection-pushdown-enabled", + TRINO_PROPERTIES_PREFIX + "iceberg.projection-pushdown-enabled") + .put( + "iceberg.hive-catalog-name", + TRINO_PROPERTIES_PREFIX + "iceberg.hive-catalog-name") .put( "iceberg.materialized-views.storage-schema", - "iceberg.materialized-views.storage-schema") + TRINO_PROPERTIES_PREFIX + "iceberg.materialized-views.storage-schema") .put( "iceberg.materialized-views.hide-storage-table", - "iceberg.materialized-views.hide-storage-table") + TRINO_PROPERTIES_PREFIX + "iceberg.materialized-views.hide-storage-table") .put( "iceberg.register-table-procedure.enabled", - "iceberg.register-table-procedure.enabled") + TRINO_PROPERTIES_PREFIX + "iceberg.register-table-procedure.enabled") .put( "iceberg.query-partition-filter-required", - "iceberg.query-partition-filter-required") + TRINO_PROPERTIES_PREFIX + "iceberg.query-partition-filter-required") // Hive - .put("hive.config.resources", "hive.config.resources") - .put("hive.recursive-directories", "hive.recursive-directories") - .put("hive.ignore-absent-partitions", "hive.ignore-absent-partitions") - .put("hive.storage-format", "hive.storage-format") - .put("hive.compression-codec", "hive.compression-codec") - .put("hive.force-local-scheduling", "hive.force-local-scheduling") - .put("hive.respect-table-format", "hive.respect-table-format") - .put("hive.immutable-partitions", "hive.immutable-partitions") + .put("hive.config.resources", TRINO_PROPERTIES_PREFIX + "hive.config.resources") + .put( + "hive.recursive-directories", + TRINO_PROPERTIES_PREFIX + "hive.recursive-directories") + .put( + "hive.ignore-absent-partitions", + TRINO_PROPERTIES_PREFIX + "hive.ignore-absent-partitions") + .put("hive.storage-format", TRINO_PROPERTIES_PREFIX + "hive.storage-format") + .put("hive.compression-codec", TRINO_PROPERTIES_PREFIX + "hive.compression-codec") + .put( + "hive.force-local-scheduling", + TRINO_PROPERTIES_PREFIX + "hive.force-local-scheduling") + .put( + "hive.respect-table-format", + TRINO_PROPERTIES_PREFIX + "hive.respect-table-format") + .put( + "hive.immutable-partitions", + TRINO_PROPERTIES_PREFIX + "hive.immutable-partitions") .put( "hive.insert-existing-partitions-behavior", - "hive.insert-existing-partitions-behavior") - .put("hive.target-max-file-size", "hive.target-max-file-size") - .put("hive.create-empty-bucket-files", "hive.create-empty-bucket-files") - .put("hive.validate-bucketing", "hive.validate-bucketing") - .put("hive.partition-statistics-sample-size", "hive.partition-statistics-sample-size") - .put("hive.max-partitions-per-writers", "hive.max-partitions-per-writers") - .put("hive.max-partitions-for-eager-load", "hive.max-partitions-for-eager-load") - .put("hive.max-partitions-per-scan", "hive.max-partitions-per-scan") - .put("hive.dfs.replication", "hive.dfs.replication") - .put("hive.security", "hive.security") - .put("security.config-file", "security.config-file") - .put("hive.non-managed-table-writes-enabled", "hive.non-managed-table-writes-enabled") + TRINO_PROPERTIES_PREFIX + "hive.insert-existing-partitions-behavior") + .put( + "hive.target-max-file-size", + TRINO_PROPERTIES_PREFIX + "hive.target-max-file-size") + .put( + "hive.create-empty-bucket-files", + TRINO_PROPERTIES_PREFIX + "hive.create-empty-bucket-files") + .put("hive.validate-bucketing", TRINO_PROPERTIES_PREFIX + "hive.validate-bucketing") + .put( + "hive.partition-statistics-sample-size", + TRINO_PROPERTIES_PREFIX + "hive.partition-statistics-sample-size") + .put( + "hive.max-partitions-per-writers", + TRINO_PROPERTIES_PREFIX + "hive.max-partitions-per-writers") + .put( + "hive.max-partitions-for-eager-load", + TRINO_PROPERTIES_PREFIX + "hive.max-partitions-for-eager-load") + .put( + "hive.max-partitions-per-scan", + TRINO_PROPERTIES_PREFIX + "hive.max-partitions-per-scan") + .put("hive.dfs.replication", TRINO_PROPERTIES_PREFIX + "hive.dfs.replication") + .put("hive.security", TRINO_PROPERTIES_PREFIX + "hive.security") + .put("security.config-file", TRINO_PROPERTIES_PREFIX + "security.config-file") + .put( + "hive.non-managed-table-writes-enabled", + TRINO_PROPERTIES_PREFIX + "hive.non-managed-table-writes-enabled") .put( "hive.non-managed-table-creates-enabled", - "hive.non-managed-table-creates-enabled") + TRINO_PROPERTIES_PREFIX + "hive.non-managed-table-creates-enabled") .put( "hive.collect-column-statistics-on-write", - "hive.collect-column-statistics-on-write") - .put("hive.file-status-cache-tables", "hive.file-status-cache-tables") + TRINO_PROPERTIES_PREFIX + "hive.collect-column-statistics-on-write") + .put( + "hive.file-status-cache-tables", + TRINO_PROPERTIES_PREFIX + "hive.file-status-cache-tables") .put( "hive.file-status-cache.max-retained-size", - "hive.file-status-cache.max-retained-size") - .put("hive.file-status-cache-expire-time", "hive.file-status-cache-expire-time") + TRINO_PROPERTIES_PREFIX + "hive.file-status-cache.max-retained-size") + .put( + "hive.file-status-cache-expire-time", + TRINO_PROPERTIES_PREFIX + "hive.file-status-cache-expire-time") .put( "hive.per-transaction-file-status-cache.max-retained-size", - "hive.per-transaction-file-status-cache.max-retained-size") - .put("hive.timestamp-precision", "hive.timestamp-precision") + TRINO_PROPERTIES_PREFIX + + "hive.per-transaction-file-status-cache.max-retained-size") + .put("hive.timestamp-precision", TRINO_PROPERTIES_PREFIX + "hive.timestamp-precision") .put( "hive.temporary-staging-directory-enabled", - "hive.temporary-staging-directory-enabled") - .put("hive.temporary-staging-directory-path", "hive.temporary-staging-directory-path") - .put("hive.hive-views.enabled", "hive.hive-views.enabled") - .put("hive.hive-views.legacy-translation", "hive.hive-views.legacy-translation") + TRINO_PROPERTIES_PREFIX + "hive.temporary-staging-directory-enabled") + .put( + "hive.temporary-staging-directory-path", + TRINO_PROPERTIES_PREFIX + "hive.temporary-staging-directory-path") + .put("hive.hive-views.enabled", TRINO_PROPERTIES_PREFIX + "hive.hive-views.enabled") + .put( + "hive.hive-views.legacy-translation", + TRINO_PROPERTIES_PREFIX + "hive.hive-views.legacy-translation") .put( "hive.parallel-partitioned-bucketed-writes", - "hive.parallel-partitioned-bucketed-writes") - .put("hive.fs.new-directory-permissions", "hive.fs.new-directory-permissions") - .put("hive.fs.cache.max-size", "hive.fs.cache.max-size") - .put("hive.query-partition-filter-required", "hive.query-partition-filter-required") - .put("hive.table-statistics-enabled", "hive.table-statistics-enabled") - .put("hive.auto-purge", "hive.auto-purge") - .put("hive.partition-projection-enabled", "hive.partition-projection-enabled") - .put("hive.max-partition-drops-per-query", "hive.max-partition-drops-per-query") - .put("hive.single-statement-writes", "hive.single-statement-writes") + TRINO_PROPERTIES_PREFIX + "hive.parallel-partitioned-bucketed-writes") + .put( + "hive.fs.new-directory-permissions", + TRINO_PROPERTIES_PREFIX + "hive.fs.new-directory-permissions") + .put("hive.fs.cache.max-size", TRINO_PROPERTIES_PREFIX + "hive.fs.cache.max-size") + .put( + "hive.query-partition-filter-required", + TRINO_PROPERTIES_PREFIX + "hive.query-partition-filter-required") + .put( + "hive.table-statistics-enabled", + TRINO_PROPERTIES_PREFIX + "hive.table-statistics-enabled") + .put("hive.auto-purge", TRINO_PROPERTIES_PREFIX + "hive.auto-purge") + .put( + "hive.partition-projection-enabled", + TRINO_PROPERTIES_PREFIX + "hive.partition-projection-enabled") + .put( + "hive.max-partition-drops-per-query", + TRINO_PROPERTIES_PREFIX + "hive.max-partition-drops-per-query") + .put( + "hive.single-statement-writes", + TRINO_PROPERTIES_PREFIX + "hive.single-statement-writes") // Hive performance - .put("hive.max-outstanding-splits", "hive.max-outstanding-splits") - .put("hive.max-outstanding-splits-size", "hive.max-outstanding-splits-size") - .put("hive.max-splits-per-second", "hive.max-splits-per-second") - .put("hive.max-initial-splits", "hive.max-initial-splits") - .put("hive.max-initial-split-size", "hive.max-initial-split-size") - .put("hive.max-split-size", "hive.max-split-size") + .put( + "hive.max-outstanding-splits", + TRINO_PROPERTIES_PREFIX + "hive.max-outstanding-splits") + .put( + "hive.max-outstanding-splits-size", + TRINO_PROPERTIES_PREFIX + "hive.max-outstanding-splits-size") + .put( + "hive.max-splits-per-second", + TRINO_PROPERTIES_PREFIX + "hive.max-splits-per-second") + .put("hive.max-initial-splits", TRINO_PROPERTIES_PREFIX + "hive.max-initial-splits") + .put( + "hive.max-initial-split-size", + TRINO_PROPERTIES_PREFIX + "hive.max-initial-split-size") + .put("hive.max-split-size", TRINO_PROPERTIES_PREFIX + "hive.max-split-size") // S3 - .put("hive.s3.aws-access-key", "hive.s3.aws-access-key") - .put("hive.s3.aws-secret-key", "hive.s3.aws-secret-key") - .put("hive.s3.iam-role", "hive.s3.iam-role") - .put("hive.s3.external-id", "hive.s3.external-id") - .put("hive.s3.endpoint", "hive.s3.endpoint") - .put("hive.s3.region", "hive.s3.region") - .put("hive.s3.storage-class", "hive.s3.storage-class") - .put("hive.s3.signer-type", "hive.s3.signer-type") - .put("hive.s3.signer-class", "hive.s3.signer-class") - .put("hive.s3.path-style-access", "hive.s3.path-style-access") - .put("hive.s3.staging-directory", "hive.s3.staging-directory") - .put("hive.s3.pin-client-to-current-region", "hive.s3.pin-client-to-current-region") - .put("hive.s3.ssl.enabled", "hive.s3.ssl.enabled") - .put("hive.s3.sse.enabled", "hive.s3.sse.enabled") - .put("hive.s3.sse.type", "hive.s3.sse.type") - .put("hive.s3.sse.kms-key-id", "hive.s3.sse.kms-key-id") - .put("hive.s3.kms-key-id", "hive.s3.kms-key-id") - .put("hive.s3.encryption-materials-provider", "hive.s3.encryption-materials-provider") - .put("hive.s3.upload-acl-type", "hive.s3.upload-acl-type") - .put("hive.s3.skip-glacier-objects", "hive.s3.skip-glacier-objects") - .put("hive.s3.streaming.enabled", "hive.s3.streaming.enabled") - .put("hive.s3.streaming.part-size", "hive.s3.streaming.part-size") - .put("hive.s3.proxy.host", "hive.s3.proxy.host") - .put("hive.s3.proxy.port", "hive.s3.proxy.port") - .put("hive.s3.proxy.protocol", "hive.s3.proxy.protocol") - .put("hive.s3.proxy.non-proxy-hosts", "hive.s3.proxy.non-proxy-hosts") - .put("hive.s3.proxy.username", "hive.s3.proxy.username") - .put("hive.s3.proxy.password", "hive.s3.proxy.password") - .put("hive.s3.proxy.preemptive-basic-auth", "hive.s3.proxy.preemptive-basic-auth") - .put("hive.s3.sts.endpoint", "hive.s3.sts.endpoint") - .put("hive.s3.sts.region", "hive.s3.sts.region") + .put("hive.s3.aws-access-key", TRINO_PROPERTIES_PREFIX + "hive.s3.aws-access-key") + .put("hive.s3.aws-secret-key", TRINO_PROPERTIES_PREFIX + "hive.s3.aws-secret-key") + .put("hive.s3.iam-role", TRINO_PROPERTIES_PREFIX + "hive.s3.iam-role") + .put("hive.s3.external-id", TRINO_PROPERTIES_PREFIX + "hive.s3.external-id") + .put("hive.s3.endpoint", TRINO_PROPERTIES_PREFIX + "hive.s3.endpoint") + .put("hive.s3.region", TRINO_PROPERTIES_PREFIX + "hive.s3.region") + .put("hive.s3.storage-class", TRINO_PROPERTIES_PREFIX + "hive.s3.storage-class") + .put("hive.s3.signer-type", TRINO_PROPERTIES_PREFIX + "hive.s3.signer-type") + .put("hive.s3.signer-class", TRINO_PROPERTIES_PREFIX + "hive.s3.signer-class") + .put( + "hive.s3.path-style-access", + TRINO_PROPERTIES_PREFIX + "hive.s3.path-style-access") + .put( + "hive.s3.staging-directory", + TRINO_PROPERTIES_PREFIX + "hive.s3.staging-directory") + .put( + "hive.s3.pin-client-to-current-region", + TRINO_PROPERTIES_PREFIX + "hive.s3.pin-client-to-current-region") + .put("hive.s3.ssl.enabled", TRINO_PROPERTIES_PREFIX + "hive.s3.ssl.enabled") + .put("hive.s3.sse.enabled", TRINO_PROPERTIES_PREFIX + "hive.s3.sse.enabled") + .put("hive.s3.sse.type", TRINO_PROPERTIES_PREFIX + "hive.s3.sse.type") + .put("hive.s3.sse.kms-key-id", TRINO_PROPERTIES_PREFIX + "hive.s3.sse.kms-key-id") + .put("hive.s3.kms-key-id", TRINO_PROPERTIES_PREFIX + "hive.s3.kms-key-id") + .put( + "hive.s3.encryption-materials-provider", + TRINO_PROPERTIES_PREFIX + "hive.s3.encryption-materials-provider") + .put("hive.s3.upload-acl-type", TRINO_PROPERTIES_PREFIX + "hive.s3.upload-acl-type") + .put( + "hive.s3.skip-glacier-objects", + TRINO_PROPERTIES_PREFIX + "hive.s3.skip-glacier-objects") + .put( + "hive.s3.streaming.enabled", + TRINO_PROPERTIES_PREFIX + "hive.s3.streaming.enabled") + .put( + "hive.s3.streaming.part-size", + TRINO_PROPERTIES_PREFIX + "hive.s3.streaming.part-size") + .put("hive.s3.proxy.host", TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.host") + .put("hive.s3.proxy.port", TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.port") + .put("hive.s3.proxy.protocol", TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.protocol") + .put( + "hive.s3.proxy.non-proxy-hosts", + TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.non-proxy-hosts") + .put("hive.s3.proxy.username", TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.username") + .put("hive.s3.proxy.password", TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.password") + .put( + "hive.s3.proxy.preemptive-basic-auth", + TRINO_PROPERTIES_PREFIX + "hive.s3.proxy.preemptive-basic-auth") + .put("hive.s3.sts.endpoint", TRINO_PROPERTIES_PREFIX + "hive.s3.sts.endpoint") + .put("hive.s3.sts.region", TRINO_PROPERTIES_PREFIX + "hive.s3.sts.region") // Hive metastore Thrift service authentication - .put("hive.metastore.authentication.type", "hive.metastore.authentication.type") + .put( + "hive.metastore.authentication.type", + TRINO_PROPERTIES_PREFIX + "hive.metastore.authentication.type") .put( "hive.metastore.thrift.impersonation.enabled", - "hive.metastore.thrift.impersonation.enabled") - .put("hive.metastore.service.principal", "hive.metastore.service.principal") - .put("hive.metastore.client.principal", "hive.metastore.client.principal") - .put("hive.metastore.client.keytab", "hive.metastore.client.keytab") + TRINO_PROPERTIES_PREFIX + "hive.metastore.thrift.impersonation.enabled") + .put( + "hive.metastore.service.principal", + TRINO_PROPERTIES_PREFIX + "hive.metastore.service.principal") + .put( + "hive.metastore.client.principal", + TRINO_PROPERTIES_PREFIX + "hive.metastore.client.principal") + .put( + "hive.metastore.client.keytab", + TRINO_PROPERTIES_PREFIX + "hive.metastore.client.keytab") // HDFS authentication - .put("hive.hdfs.authentication.type", "hive.hdfs.authentication.type") - .put("hive.hdfs.impersonation.enabled", "hive.hdfs.impersonation.enabled") - .put("hive.hdfs.trino.principal", "hive.hdfs.trino.principal") - .put("hive.hdfs.trino.keytab", "hive.hdfs.trino.keytab") - .put("hive.hdfs.wire-encryption.enabled", "hive.hdfs.wire-encryption.enabled") + .put( + "hive.hdfs.authentication.type", + TRINO_PROPERTIES_PREFIX + "hive.hdfs.authentication.type") + .put( + "hive.hdfs.impersonation.enabled", + TRINO_PROPERTIES_PREFIX + "hive.hdfs.impersonation.enabled") + .put( + "hive.hdfs.trino.principal", + TRINO_PROPERTIES_PREFIX + "hive.hdfs.trino.principal") + .put("hive.hdfs.trino.keytab", TRINO_PROPERTIES_PREFIX + "hive.hdfs.trino.keytab") + .put( + "hive.hdfs.wire-encryption.enabled", + TRINO_PROPERTIES_PREFIX + "hive.hdfs.wire-encryption.enabled") .build()); private static final Set JDBC_BACKEND_REQUIRED_PROPERTIES = @@ -182,15 +289,20 @@ public TreeBidiMap engineToGravitinoMapping() { @Override public Map gravitinoToEngineProperties(Map properties) { + Map stringStringMap; String backend = properties.get("catalog-backend"); switch (backend) { case "hive": - return buildHiveBackendProperties(properties); + stringStringMap = buildHiveBackendProperties(properties); + break; case "jdbc": - return buildJDBCBackendProperties(properties); + stringStringMap = buildJDBCBackendProperties(properties); + break; default: throw new UnsupportedOperationException("Unsupported backend type: " + backend); } + stringStringMap.putAll(super.gravitinoToEngineProperties(properties)); + return stringStringMap; } private Map buildHiveBackendProperties(Map properties) { diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/JDBCCatalogPropertyConverter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/JDBCCatalogPropertyConverter.java index 5cc46c5a57a..4ce02a73e6c 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/JDBCCatalogPropertyConverter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/JDBCCatalogPropertyConverter.java @@ -28,42 +28,67 @@ public class JDBCCatalogPropertyConverter extends PropertyConverter { // Data source authentication .put(JDBC_CONNECTION_USER_KEY, "jdbc-user") .put(JDBC_CONNECTION_PASSWORD_KEY, "jdbc-password") - .put("credential-provider.type", "credential-provider.type") - .put("user-credential-name", "user-credential-name") - .put("password-credential-name", "password-credential-name") - .put("connection-credential-file", "connection-credential-file") - .put("keystore-file-path", "keystore-file-path") - .put("keystore-type", "keystore-type") - .put("keystore-password", "keystore-password") - .put("keystore-user-credential-name", "keystore-user-credential-name") - .put("keystore-user-credential-password", "keystore-user-credential-password") - .put("keystore-password-credential-name", "keystore-password-credential-name") - .put("keystore-password-credential-password", "keystore-password-credential-password") + .put("credential-provider.type", TRINO_PROPERTIES_PREFIX + "credential-provider.type") + .put("user-credential-name", TRINO_PROPERTIES_PREFIX + "user-credential-name") + .put("password-credential-name", TRINO_PROPERTIES_PREFIX + "password-credential-name") + .put( + "connection-credential-file", + TRINO_PROPERTIES_PREFIX + "connection-credential-file") + .put("keystore-file-path", TRINO_PROPERTIES_PREFIX + "keystore-file-path") + .put("keystore-type", TRINO_PROPERTIES_PREFIX + "keystore-type") + .put("keystore-password", TRINO_PROPERTIES_PREFIX + "keystore-password") + .put( + "keystore-user-credential-name", + TRINO_PROPERTIES_PREFIX + "keystore-user-credential-name") + .put( + "keystore-user-credential-password", + TRINO_PROPERTIES_PREFIX + "keystore-user-credential-password") + .put( + "keystore-password-credential-name", + TRINO_PROPERTIES_PREFIX + "keystore-password-credential-name") + .put( + "keystore-password-credential-password", + TRINO_PROPERTIES_PREFIX + "keystore-password-credential-password") // General configuration properties - .put("case-insensitive-name-matching", "ase-insensitive-name-matching") + .put( + "case-insensitive-name-matching", + TRINO_PROPERTIES_PREFIX + "ase-insensitive-name-matching") .put( "case-insensitive-name-matching.cache-ttl", - "case-insensitive-name-matching.cache-ttl") + TRINO_PROPERTIES_PREFIX + "case-insensitive-name-matching.cache-ttl") .put( "case-insensitive-name-matching.config-file", - "case-insensitive-name-matching.config-file") + TRINO_PROPERTIES_PREFIX + "case-insensitive-name-matching.config-file") .put( "case-insensitive-name-matching.config-file.refresh-period", - "case-insensitive-name-matching.config-file.refresh-period") - .put("metadata.cache-ttl", "metadata.cache-ttl") - .put("metadata.cache-missing", "metadata.cache-missing") - .put("metadata.schemas.cache-ttl", "metadata.schemas.cache-ttl") - .put("metadata.tables.cache-ttl", "metadata.tables.cache-ttl") - .put("metadata.statistics.cache-ttl", "metadata.statistics.cache-ttl") - .put("metadata.cache-maximum-size", "metadata.cache-maximum-size") - .put("write.batch-size", "write.batch-size") - .put("dynamic-filtering.enabled", "dynamic-filtering.enabled") - .put("dynamic-filtering.wait-timeout", "dynamic-filtering.wait-timeout") + TRINO_PROPERTIES_PREFIX + + "case-insensitive-name-matching.config-file.refresh-period") + .put("metadata.cache-ttl", TRINO_PROPERTIES_PREFIX + "metadata.cache-ttl") + .put("metadata.cache-missing", TRINO_PROPERTIES_PREFIX + "metadata.cache-missing") + .put( + "metadata.schemas.cache-ttl", + TRINO_PROPERTIES_PREFIX + "metadata.schemas.cache-ttl") + .put( + "metadata.tables.cache-ttl", + TRINO_PROPERTIES_PREFIX + "metadata.tables.cache-ttl") + .put( + "metadata.statistics.cache-ttl", + TRINO_PROPERTIES_PREFIX + "metadata.statistics.cache-ttl") + .put( + "metadata.cache-maximum-size", + TRINO_PROPERTIES_PREFIX + "metadata.cache-maximum-size") + .put("write.batch-size", TRINO_PROPERTIES_PREFIX + "write.batch-size") + .put( + "dynamic-filtering.enabled", + TRINO_PROPERTIES_PREFIX + "dynamic-filtering.enabled") + .put( + "dynamic-filtering.wait-timeout", + TRINO_PROPERTIES_PREFIX + "dynamic-filtering.wait-timeout") // Performance - .put("join-pushdown.enabled", "join-pushdown.enabled") - .put("join-pushdown.strategy", "join-pushdown.strategy") + .put("join-pushdown.enabled", TRINO_PROPERTIES_PREFIX + "join-pushdown.enabled") + .put("join-pushdown.strategy", TRINO_PROPERTIES_PREFIX + "join-pushdown.strategy") .build()); public static final Set REQUIRED_PROPERTIES = diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoCatalog.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoCatalog.java index 4dd61b1c368..2157891d939 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoCatalog.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoCatalog.java @@ -19,7 +19,7 @@ public class GravitinoCatalog { private final String metalake; private final Catalog catalog; - public GravitinoCatalog(String metalake, Catalog catalog, boolean usingSimpleName) { + public GravitinoCatalog(String metalake, Catalog catalog) { this.metalake = metalake; this.catalog = catalog; } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java index 66d1fa9821b..86fc466389c 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java @@ -61,12 +61,10 @@ public class GravitinoMockServer implements AutoCloseable { private final Map metalakes = new HashMap<>(); private boolean start = true; - private boolean simpleCatalogName; CatalogConnectorManager catalogConnectorManager; private GeneralDataTypeTransformer dataTypeTransformer = new HiveDataTypeTransformer(); - public GravitinoMockServer(boolean simpleCatalogName) { - this.simpleCatalogName = simpleCatalogName; + public GravitinoMockServer() { createMetalake(NameIdentifier.ofMetalake(testMetalake)); createCatalog(NameIdentifier.ofCatalog(testMetalake, testCatalog)); } @@ -214,8 +212,7 @@ private Catalog createCatalog(NameIdentifier catalogName) { when(mockAudit.createTime()).thenReturn(Instant.now()); when(catalog.auditInfo()).thenReturn(mockAudit); - GravitinoCatalog gravitinoCatalog = - new GravitinoCatalog(testMetalake, catalog, simpleCatalogName); + GravitinoCatalog gravitinoCatalog = new GravitinoCatalog(testMetalake, catalog); when(catalog.asTableCatalog()).thenAnswer(answer -> createTableCatalog(gravitinoCatalog)); when(catalog.asSchemas()).thenAnswer(answer -> createSchemas(gravitinoCatalog)); diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestCreateGravitinoConnector.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestCreateGravitinoConnector.java index 5f6aa375830..bd71b021bac 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestCreateGravitinoConnector.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestCreateGravitinoConnector.java @@ -20,7 +20,7 @@ public class TestCreateGravitinoConnector { @Test public void testCreateConnectorsWithEnableSimpleCatalog() throws Exception { - server = new GravitinoMockServer(true); + server = new GravitinoMockServer(); Session session = testSessionBuilder().setCatalog("gravitino").build(); QueryRunner queryRunner = DistributedQueryRunner.builder(session).setNodeCount(1).build(); @@ -56,7 +56,7 @@ public void testCreateConnectorsWithEnableSimpleCatalog() throws Exception { @Test public void testCreateConnectorsWithDisableSimpleCatalog() throws Exception { - server = new GravitinoMockServer(false); + server = new GravitinoMockServer(); Session session = testSessionBuilder().setCatalog("gravitino").build(); QueryRunner queryRunner = DistributedQueryRunner.builder(session).setNodeCount(1).build(); diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java index 93fb75da8c7..00fa2004b32 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java @@ -35,7 +35,7 @@ public class TestGravitinoConnector extends AbstractTestQueryFramework { @Override protected QueryRunner createQueryRunner() throws Exception { - server = closeAfterClass(new GravitinoMockServer(true)); + server = closeAfterClass(new GravitinoMockServer()); GravitinoAdminClient gravitinoClient = server.createGravitinoClient(); Session session = testSessionBuilder().setCatalog("gravitino").build(); diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithMetalakeCatalogName.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithMetalakeCatalogName.java index ed9d4457a0e..a7691c65cd6 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithMetalakeCatalogName.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithMetalakeCatalogName.java @@ -32,7 +32,7 @@ public class TestGravitinoConnectorWithMetalakeCatalogName extends AbstractTestQ @Override protected QueryRunner createQueryRunner() throws Exception { - server = closeAfterClass(new GravitinoMockServer(false)); + server = closeAfterClass(new GravitinoMockServer()); GravitinoAdminClient gravitinoClient = server.createGravitinoClient(); Session session = testSessionBuilder().setCatalog("gravitino").build(); diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/hive/TestHiveCatalogPropertyConverter.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/hive/TestHiveCatalogPropertyConverter.java index d07f7661f9d..124762aefa5 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/hive/TestHiveCatalogPropertyConverter.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/hive/TestHiveCatalogPropertyConverter.java @@ -5,7 +5,10 @@ package com.datastrato.gravitino.trino.connector.catalog.hive; +import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata; +import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; +import com.datastrato.gravitino.trino.connector.metadata.TestGravitinoCatalog; import com.google.common.collect.Sets; import java.util.Map; import java.util.Set; @@ -21,9 +24,9 @@ public void testConverter() { HiveCatalogPropertyConverter hiveCatalogPropertyConverter = new HiveCatalogPropertyConverter(); Map map = ImmutableMap.builder() - .put("hive.immutable-partitions", "true") - .put("hive.compression-codec", "ZSTD") - .put("hive.unknown-key", "1") + .put("trino.bypass.hive.immutable-partitions", "true") + .put("trino.bypass.hive.compression-codec", "ZSTD") + .put("trino.bypass.hive.unknown-key", "1") .build(); Map re = hiveCatalogPropertyConverter.gravitinoToEngineProperties(map); @@ -44,4 +47,42 @@ public void testPropertyMetadata() { gravitinoHiveKeys.remove("external"); Assert.assertTrue(actualGravitinoKeys.containsAll(gravitinoHiveKeys)); } + + @Test + @SuppressWarnings("unchecked") + public void testBuildConnectorProperties() throws Exception { + String name = "test_catalog"; + Map properties = + ImmutableMap.builder() + .put("metastore.uris", "thrift://localhost:9083") + .put("hive.unknown-key", "1") + .put("trino.bypass.unknown-key", "1") + .put("trino.bypass.hive.config.resources", "/tmp/hive-site.xml, /tmp/core-site.xml") + .build(); + Catalog mockCatalog = + TestGravitinoCatalog.mockCatalog( + name, "hive", "test catalog", Catalog.Type.RELATIONAL, properties); + HiveConnectorAdapter adapter = new HiveConnectorAdapter(); + Map stringObjectMap = + adapter.buildInternalConnectorConfig(new GravitinoCatalog("test", mockCatalog)); + + // test connector attributes + Assert.assertEquals(stringObjectMap.get("connectorName"), "hive"); + + Map propertiesMap = (Map) stringObjectMap.get("properties"); + + // test converted properties + Assert.assertEquals(propertiesMap.get("hive.metastore.uri"), "thrift://localhost:9083"); + + // test fixed properties + Assert.assertEquals(propertiesMap.get("hive.security"), "allow-all"); + + // test trino passing properties + Assert.assertEquals( + propertiesMap.get("hive.config.resources"), "/tmp/hive-site.xml, /tmp/core-site.xml"); + + // test unknown properties + Assert.assertNull(propertiesMap.get("hive.unknown-key")); + Assert.assertNull(propertiesMap.get("trino.bypass.unknown-key")); + } } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/TestIcebergCatalogPropertyConverter.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/TestIcebergCatalogPropertyConverter.java index 52ba989616e..e4c072fef2b 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/TestIcebergCatalogPropertyConverter.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/TestIcebergCatalogPropertyConverter.java @@ -5,8 +5,11 @@ package com.datastrato.gravitino.trino.connector.catalog.iceberg; +import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergTablePropertiesMetadata; import com.datastrato.gravitino.catalog.property.PropertyConverter; +import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; +import com.datastrato.gravitino.trino.connector.metadata.TestGravitinoCatalog; import com.google.common.collect.Sets; import io.trino.spi.TrinoException; import java.util.Map; @@ -85,4 +88,89 @@ public void testPropertyMetadata() { Assert.assertTrue(actualGravitinoKeys.containsAll(gravitinoHiveKeys)); } + + @Test + @SuppressWarnings("unchecked") + public void testBuildConnectorPropertiesWithHiveBackend() throws Exception { + String name = "test_catalog"; + Map properties = + ImmutableMap.builder() + .put("uri", "thrift://localhost:9083") + .put("catalog-backend", "hive") + .put("warehouse", "hdfs://tmp/warehouse") + .put("unknown-key", "1") + .put("trino.bypass.unknown-key", "1") + .put("trino.bypass.iceberg.table-statistics-enabled", "true") + .build(); + Catalog mockCatalog = + TestGravitinoCatalog.mockCatalog( + name, "lakehouse-iceberg", "test catalog", Catalog.Type.RELATIONAL, properties); + IcebergConnectorAdapter adapter = new IcebergConnectorAdapter(); + + Map stringObjectMap = + adapter.buildInternalConnectorConfig(new GravitinoCatalog("test", mockCatalog)); + + // test connector attributes + Assert.assertEquals(stringObjectMap.get("connectorName"), "iceberg"); + + Map propertiesMap = (Map) stringObjectMap.get("properties"); + + // test converted properties + Assert.assertEquals(propertiesMap.get("hive.metastore.uri"), "thrift://localhost:9083"); + Assert.assertEquals(propertiesMap.get("iceberg.catalog.type"), "hive_metastore"); + + // test trino passing properties + Assert.assertEquals(propertiesMap.get("iceberg.table-statistics-enabled"), "true"); + + // test unknown properties + Assert.assertNull(propertiesMap.get("hive.unknown-key")); + Assert.assertNull(propertiesMap.get("trino.bypass.unknown-key")); + } + + @Test + @SuppressWarnings("unchecked") + public void testBuildConnectorPropertiesWithMySqlBackEnd() throws Exception { + String name = "test_catalog"; + Map properties = + ImmutableMap.builder() + .put("uri", "jdbc:mysql://%s:3306/metastore_db?createDatabaseIfNotExist=true") + .put("catalog-backend", "jdbc") + .put("warehouse", "://tmp/warehouse") + .put("jdbc-user", "root") + .put("jdbc-password", "ds123") + .put("jdbc-driver", "com.mysql.cj.jdbc.Driver") + .put("unknown-key", "1") + .put("trino.bypass.unknown-key", "1") + .put("trino.bypass.iceberg.table-statistics-enabled", "true") + .build(); + Catalog mockCatalog = + TestGravitinoCatalog.mockCatalog( + name, "lakehouse-iceberg", "test catalog", Catalog.Type.RELATIONAL, properties); + IcebergConnectorAdapter adapter = new IcebergConnectorAdapter(); + + Map stringObjectMap = + adapter.buildInternalConnectorConfig(new GravitinoCatalog("test", mockCatalog)); + + // test connector attributes + Assert.assertEquals(stringObjectMap.get("connectorName"), "iceberg"); + + Map propertiesMap = (Map) stringObjectMap.get("properties"); + + // test converted properties + Assert.assertEquals( + propertiesMap.get("iceberg.jdbc-catalog.connection-url"), + "jdbc:mysql://%s:3306/metastore_db?createDatabaseIfNotExist=true"); + Assert.assertEquals(propertiesMap.get("iceberg.jdbc-catalog.connection-user"), "root"); + Assert.assertEquals(propertiesMap.get("iceberg.jdbc-catalog.connection-password"), "ds123"); + Assert.assertEquals( + propertiesMap.get("iceberg.jdbc-catalog.driver-class"), "com.mysql.cj.jdbc.Driver"); + Assert.assertEquals(propertiesMap.get("iceberg.catalog.type"), "jdbc"); + + // test trino passing properties + Assert.assertEquals(propertiesMap.get("iceberg.table-statistics-enabled"), "true"); + + // test unknown properties + Assert.assertNull(propertiesMap.get("hive.unknown-key")); + Assert.assertNull(propertiesMap.get("trino.bypass.unknown-key")); + } } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/TestJDBCCatalogPropertyConverter.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/TestJDBCCatalogPropertyConverter.java index ee9f10446d5..02320854041 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/TestJDBCCatalogPropertyConverter.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/TestJDBCCatalogPropertyConverter.java @@ -9,7 +9,12 @@ import static com.datastrato.gravitino.trino.connector.catalog.jdbc.JDBCCatalogPropertyConverter.JDBC_CONNECTION_URL_KEY; import static com.datastrato.gravitino.trino.connector.catalog.jdbc.JDBCCatalogPropertyConverter.JDBC_CONNECTION_USER_KEY; +import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.catalog.property.PropertyConverter; +import com.datastrato.gravitino.trino.connector.catalog.jdbc.mysql.MySQLConnectorAdapter; +import com.datastrato.gravitino.trino.connector.catalog.jdbc.postgresql.PostgreSQLConnectorAdapter; +import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; +import com.datastrato.gravitino.trino.connector.metadata.TestGravitinoCatalog; import java.util.Map; import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; import org.testng.Assert; @@ -44,4 +49,83 @@ public void testTrinoPropertyKeyToGravitino() { propertyConverter.gravitinoToEngineProperties(gravitinoPropertiesWithoutPassword); }); } + + @Test + @SuppressWarnings("unchecked") + public void testBuildPostgreSqlConnectorProperties() throws Exception { + String name = "test_catalog"; + Map properties = + ImmutableMap.builder() + .put("jdbc-url", "jdbc:postgresql://localhost:5432/test") + .put("jdbc-user", "test") + .put("jdbc-password", "test") + .put("trino.bypass.join-pushdown.strategy", "EAGER") + .put("unknown-key", "1") + .put("trino.bypass.unknown-key", "1") + .build(); + Catalog mockCatalog = + TestGravitinoCatalog.mockCatalog( + name, "jdbc-postgresql", "test catalog", Catalog.Type.RELATIONAL, properties); + PostgreSQLConnectorAdapter adapter = new PostgreSQLConnectorAdapter(); + + Map stringObjectMap = + adapter.buildInternalConnectorConfig(new GravitinoCatalog("test", mockCatalog)); + + // test connector attributes + Assert.assertEquals(stringObjectMap.get("connectorName"), "postgresql"); + + Map propertiesMap = (Map) stringObjectMap.get("properties"); + + // test converted properties + Assert.assertEquals( + propertiesMap.get("connection-url"), "jdbc:postgresql://localhost:5432/test"); + Assert.assertEquals(propertiesMap.get("connection-user"), "test"); + Assert.assertEquals(propertiesMap.get("connection-password"), "test"); + + // test trino passing properties + Assert.assertEquals(propertiesMap.get("join-pushdown.strategy"), "EAGER"); + + // test unknown properties + Assert.assertNull(propertiesMap.get("hive.unknown-key")); + Assert.assertNull(propertiesMap.get("trino.bypass.unknown-key")); + } + + @Test + @SuppressWarnings("unchecked") + public void testBuildMySqlConnectorProperties() throws Exception { + String name = "test_catalog"; + Map properties = + ImmutableMap.builder() + .put("jdbc-url", "jdbc:mysql://localhost:5432/test") + .put("jdbc-user", "test") + .put("jdbc-password", "test") + .put("trino.bypass.join-pushdown.strategy", "EAGER") + .put("unknown-key", "1") + .put("trino.bypass.unknown-key", "1") + .build(); + Catalog mockCatalog = + TestGravitinoCatalog.mockCatalog( + name, "jdbc-postgresql", "test catalog", Catalog.Type.RELATIONAL, properties); + MySQLConnectorAdapter adapter = new MySQLConnectorAdapter(); + + Map stringObjectMap = + adapter.buildInternalConnectorConfig(new GravitinoCatalog("test", mockCatalog)); + + // test connector attributes + Assert.assertEquals(stringObjectMap.get("connectorName"), "mysql"); + + Map propertiesMap = (Map) stringObjectMap.get("properties"); + + // test converted properties + Assert.assertEquals(propertiesMap.get("connection-url"), "jdbc:mysql://localhost:5432/test"); + Assert.assertEquals(propertiesMap.get("connection-user"), "test"); + Assert.assertEquals(propertiesMap.get("connection-password"), "test"); + + // test trino passing properties + Assert.assertEquals(propertiesMap.get("join-pushdown.strategy"), "EAGER"); + + // test unknown properties + Assert.assertNull(propertiesMap.get("hive.unknown-key")); + Assert.assertNull(propertiesMap.get("trino.bypass.unknown-key")); + } } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoCatalog.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoCatalog.java index 6641943465a..58453993a63 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoCatalog.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoCatalog.java @@ -24,7 +24,7 @@ public void testGravitinoCatalog() { Catalog mockCatalog = mockCatalog( catalogName, provider, "test catalog", Catalog.Type.RELATIONAL, Collections.emptyMap()); - GravitinoCatalog catalog = new GravitinoCatalog("test", mockCatalog, false); + GravitinoCatalog catalog = new GravitinoCatalog("test", mockCatalog); assertEquals(catalogName, catalog.getName()); assertEquals(provider, catalog.getProvider()); }