From b93ec673fd6dd948bd9d35275592087319348ab0 Mon Sep 17 00:00:00 2001 From: fanng Date: Tue, 17 Oct 2023 19:29:47 +0800 Subject: [PATCH 1/3] add JdbcCatalog service IT --- .../build.gradle.kts | 1 + .../lakehouse/iceberg/IcebergConfig.java | 24 +++++++++++ gradle/libs.versions.toml | 2 + integration-test/build.gradle.kts | 1 + .../iceberg/IcebergRESTJdbcCatalogIT.java | 17 ++++++++ .../iceberg/IcebergRESTServiceBaseIT.java | 43 +++++++++++++++++++ 6 files changed, 88 insertions(+) create mode 100644 integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTJdbcCatalogIT.java diff --git a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts index b8ed8c6b4be..76b5a6d06d9 100644 --- a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts +++ b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts @@ -38,6 +38,7 @@ dependencies { exclude("org.slf4j") } implementation(libs.iceberg.hive.metastore) + implementation(libs.sqlite.jdbc) implementation(libs.hive2.metastore) { exclude("org.apache.avro", "avro") diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java index df97d407b93..49003fdf704 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java @@ -6,6 +6,9 @@ package com.datastrato.gravitino.catalog.lakehouse.iceberg; import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogPropertiesMetadata.CATALOG_BACKEND_NAME; +import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogPropertiesMetadata.ICEBERG_JDBC_INITIALIZE; +import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogPropertiesMetadata.ICEBERG_JDBC_PASSWORD; +import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogPropertiesMetadata.ICEBERG_JDBC_USER; import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogPropertiesMetadata.URI; import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogPropertiesMetadata.WAREHOUSE; @@ -36,6 +39,27 @@ public class IcebergConfig extends Config { .stringConf() .createWithDefault(null); + public static final ConfigEntry JDBC_USER = + new ConfigBuilder(ICEBERG_JDBC_USER) + .doc("The username of the Jdbc") + .version(DEFAULT_VERSION) + .stringConf() + .createWithDefault(null); + + public static final ConfigEntry JDBC_PASSWORD = + new ConfigBuilder(ICEBERG_JDBC_PASSWORD) + .doc("The password of the Jdbc") + .version(DEFAULT_VERSION) + .stringConf() + .createWithDefault(null); + + public static final ConfigEntry JDBC_INIT_TABLES = + new ConfigBuilder(ICEBERG_JDBC_INITIALIZE) + .doc("whether to initialize meta tables when create Jdbc catalog") + .version(DEFAULT_VERSION) + .booleanConf() + .createWithDefault(Boolean.TRUE); + public IcebergConfig() { super(false); } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 27ef2349d35..3c4f526048a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -29,6 +29,7 @@ iceberg = '1.3.1' trino = '426' spark = "3.4.1" scala-collection-compat = "2.7.0" +sqlite-jdbc = "3.42.0.0" protobuf-plugin = "0.9.2" @@ -93,6 +94,7 @@ trino-testing= { group = "io.trino", name = "trino-testing", version.ref = "trin iceberg-spark-runtime = { group = "org.apache.iceberg", name = "iceberg-spark-runtime-3.4_2.13", version.ref = "iceberg" } spark-sql = { group = "org.apache.spark", name = "spark-sql_2.13", version.ref = "spark" } scala-collection-compat = { group = "org.scala-lang.modules", name = "scala-collection-compat_2.13", version.ref = "scala-collection-compat" } +sqlite-jdbc = { group = "org.xerial", name = "sqlite-jdbc", version.ref = "sqlite-jdbc" } [bundles] diff --git a/integration-test/build.gradle.kts b/integration-test/build.gradle.kts index daf098b965d..59917dda581 100644 --- a/integration-test/build.gradle.kts +++ b/integration-test/build.gradle.kts @@ -107,6 +107,7 @@ dependencies { } testImplementation(libs.scala.collection.compat) testImplementation(libs.slf4j.jdk14) + testImplementation(libs.sqlite.jdbc) } /* Optimizing integration test execution conditions */ diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTJdbcCatalogIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTJdbcCatalogIT.java new file mode 100644 index 00000000000..dca629dca7b --- /dev/null +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTJdbcCatalogIT.java @@ -0,0 +1,17 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.integration.test.catalog.lakehouse.iceberg; + +import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogBackend; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; + +@TestInstance(Lifecycle.PER_CLASS) +public class IcebergRESTJdbcCatalogIT extends IcebergRESTServiceIT { + public IcebergRESTJdbcCatalogIT() { + catalogType = IcebergCatalogBackend.JDBC; + } +} diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTServiceBaseIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTServiceBaseIT.java index 57f2b4590cb..67aefe00f2a 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTServiceBaseIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTServiceBaseIT.java @@ -112,6 +112,49 @@ private static Map getIcebergMemoryCatalogConfigs() { private static Map getIcebergJdbcCatalogConfigs() { Map configMap = new HashMap<>(); + + configMap.put( + AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX + + IcebergRESTService.SERVICE_NAME + + "." + + IcebergConfig.CATALOG_BACKEND.getKey(), + IcebergCatalogBackend.JDBC.toString().toLowerCase()); + + configMap.put( + AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX + + IcebergRESTService.SERVICE_NAME + + "." + + IcebergConfig.CATALOG_URI.getKey(), + "jdbc:sqlite:/tmp/iceberg-rest-sqlite"); + + configMap.put( + AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX + + IcebergRESTService.SERVICE_NAME + + "." + + IcebergConfig.JDBC_USER.getKey(), + "iceberg"); + + configMap.put( + AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX + + IcebergRESTService.SERVICE_NAME + + "." + + IcebergConfig.JDBC_PASSWORD.getKey(), + "iceberg"); + + configMap.put( + AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX + + IcebergRESTService.SERVICE_NAME + + "." + + IcebergConfig.JDBC_INIT_TABLES.getKey(), + "true"); + + configMap.put( + AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX + + IcebergRESTService.SERVICE_NAME + + "." + + IcebergConfig.CATALOG_WAREHOUSE.getKey(), + "file:///tmp/user/hive/warehouse-mysql/"); + return configMap; } From 8ed7d981a0a525944dfe12842ed5d92d639c4ccb Mon Sep 17 00:00:00 2001 From: fanng Date: Tue, 17 Oct 2023 20:44:10 +0800 Subject: [PATCH 2/3] fix warehouse location --- .../catalog/lakehouse/iceberg/IcebergRESTServiceBaseIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTServiceBaseIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTServiceBaseIT.java index 67aefe00f2a..28b9c24ad6c 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTServiceBaseIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTServiceBaseIT.java @@ -153,7 +153,7 @@ private static Map getIcebergJdbcCatalogConfigs() { + IcebergRESTService.SERVICE_NAME + "." + IcebergConfig.CATALOG_WAREHOUSE.getKey(), - "file:///tmp/user/hive/warehouse-mysql/"); + "file:///tmp/user/hive/warehouse-jdbc-sqlite/"); return configMap; } From 684e7d2a993512eed104da056c186e640b1d0523 Mon Sep 17 00:00:00 2001 From: fanng Date: Tue, 17 Oct 2023 21:41:03 +0800 Subject: [PATCH 3/3] fix comment --- .../catalog/lakehouse/iceberg/IcebergConfig.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java index 49003fdf704..56a2f8232cc 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java @@ -41,24 +41,24 @@ public class IcebergConfig extends Config { public static final ConfigEntry JDBC_USER = new ConfigBuilder(ICEBERG_JDBC_USER) - .doc("The username of the Jdbc") + .doc("The username of the Jdbc connection") .version(DEFAULT_VERSION) .stringConf() .createWithDefault(null); public static final ConfigEntry JDBC_PASSWORD = new ConfigBuilder(ICEBERG_JDBC_PASSWORD) - .doc("The password of the Jdbc") + .doc("The password of the Jdbc connection") .version(DEFAULT_VERSION) .stringConf() .createWithDefault(null); public static final ConfigEntry JDBC_INIT_TABLES = new ConfigBuilder(ICEBERG_JDBC_INITIALIZE) - .doc("whether to initialize meta tables when create Jdbc catalog") + .doc("Whether to initialize meta tables when create Jdbc catalog") .version(DEFAULT_VERSION) .booleanConf() - .createWithDefault(Boolean.TRUE); + .createWithDefault(true); public IcebergConfig() { super(false);