From 843e9f47faa3b1d1365ac8eab1f89d015704586c Mon Sep 17 00:00:00 2001 From: fanng Date: Tue, 2 Jul 2024 16:52:43 +0800 Subject: [PATCH] support start with Gravitino --- build.gradle.kts | 16 +- .../iceberg/IcebergCatalogOperations.java | 12 +- .../lakehouse/iceberg/IcebergConfig.java | 138 ------------ .../iceberg/ops/IcebergTableOps.java | 202 ------------------ .../iceberg/utils/IcebergCatalogUtil.java | 2 +- ...vitino.auxiliary.GravitinoAuxiliaryService | 5 - .../lakehouse/iceberg/TestIcebergCatalog.java | 2 +- .../lakehouse/iceberg/TestIcebergConfig.java | 1 + .../test/CatalogIcebergBaseIT.java | 2 +- .../test/CatalogIcebergKerberosHiveIT.java | 2 +- .../test/TestMultipleJDBCLoad.java | 2 +- .../iceberg/ops/TestIcebergTableUpdate.java | 1 + .../iceberg/utils/TestIcebergCatalogUtil.java | 2 +- .../iceberg/common/ops/IcebergTableOps.java | 3 +- .../com.datastrato.gravitino.CatalogProvider | 5 - ...vitino.auxiliary.GravitinoAuxiliaryService | 5 - iceberg-rest-server/build.gradle.kts | 2 +- .../com.datastrato.gravitino.CatalogProvider | 5 - ...vitino.auxiliary.GravitinoAuxiliaryService | 2 +- 19 files changed, 24 insertions(+), 385 deletions(-) delete mode 100644 catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java delete mode 100644 catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOps.java delete mode 100644 catalogs/catalog-lakehouse-iceberg/src/main/resources/META-INF/services/com.datastrato.gravitino.auxiliary.GravitinoAuxiliaryService delete mode 100644 iceberg-common/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider delete mode 100644 iceberg-common/src/main/resources/META-INF/services/com.datastrato.gravitino.auxiliary.GravitinoAuxiliaryService delete mode 100644 iceberg-rest-server/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider diff --git a/build.gradle.kts b/build.gradle.kts index a4868805344..654c0c3e650 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -547,27 +547,17 @@ tasks { val compileIcebergRESTServer by registering { dependsOn("iceberg-rest-server:copyLibs") group = "Iceberg REST server distribution" - outputs.dir(projectDir.dir("distribution/${rootProject.name}-iceberg-rest-server")) + outputs.dir(projectDir.dir("distribution/package/iceberg-rest-server")) doLast { copy { - from(projectDir.dir("conf")) { into("${rootProject.name}-iceberg-rest-server/conf") } - from(projectDir.dir("bin")) { into("${rootProject.name}-iceberg-rest-server/bin") } + from(projectDir.dir("conf")) { into("package/iceberg-rest-server/conf") } + from(projectDir.dir("bin")) { into("package/iceberg-rest-server/bin") } into(outputDir) rename { fileName -> fileName.replace(".template", "") } fileMode = 0b111101101 } - copy { - from(projectDir.dir("licenses")) { into("${rootProject.name}-iceberg-rest-server/licenses") } - from(projectDir.file("LICENSE.bin")) { into("${rootProject.name}-iceberg-rest-server") } - from(projectDir.file("NOTICE.bin")) { into("${rootProject.name}-iceberg-rest-server") } - from(projectDir.file("README.md")) { into("${rootProject.name}-iceberg-rest-server") } - into(outputDir) - rename { fileName -> - fileName.replace(".bin", "") - } - } } } diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java index dea857dd59f..21f3a12692f 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java @@ -9,7 +9,6 @@ import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.SchemaChange; -import com.datastrato.gravitino.catalog.lakehouse.iceberg.ops.IcebergTableOps; import com.datastrato.gravitino.catalog.lakehouse.iceberg.ops.IcebergTableOpsHelper; import com.datastrato.gravitino.connector.CatalogInfo; import com.datastrato.gravitino.connector.CatalogOperations; @@ -21,6 +20,8 @@ import com.datastrato.gravitino.exceptions.NonEmptySchemaException; import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; import com.datastrato.gravitino.exceptions.TableAlreadyExistsException; +import com.datastrato.gravitino.iceberg.common.IcebergConfig; +import com.datastrato.gravitino.iceberg.common.ops.IcebergTableOps; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.Table; @@ -44,6 +45,7 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.iceberg.Transaction; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.NamespaceNotEmptyException; @@ -58,6 +60,7 @@ import org.slf4j.LoggerFactory; /** Operations for interacting with the Iceberg catalog in Gravitino. */ +@SuppressWarnings("UnusedVariable") public class IcebergCatalogOperations implements CatalogOperations, SupportsSchemas, TableCatalog { private static final String ICEBERG_TABLE_DOES_NOT_EXIST_MSG = "Iceberg table does not exist: %s"; @@ -96,7 +99,7 @@ public void initialize( IcebergConfig icebergConfig = new IcebergConfig(resultConf); this.icebergTableOps = new IcebergTableOps(icebergConfig); - this.icebergTableOpsHelper = icebergTableOps.createIcebergTableOpsHelper(); + this.icebergTableOpsHelper = new IcebergTableOpsHelper(icebergTableOps.getCatalog()); } /** Closes the Iceberg catalog and releases the associated client pool. */ @@ -392,7 +395,10 @@ private Table internalUpdateTable(NameIdentifier tableIdent, TableChange... chan IcebergTableOpsHelper.IcebergTableChange icebergTableChange = icebergTableOpsHelper.buildIcebergTableChanges( NameIdentifier.of(levels[levels.length - 1], tableIdent.name()), changes); - LoadTableResponse loadTableResponse = icebergTableOps.updateTable(icebergTableChange); + Transaction transaction = icebergTableChange.getTransaction(); + transaction.commitTransaction(); + LoadTableResponse loadTableResponse = + icebergTableOps.loadTable(icebergTableChange.getTableIdentifier()); loadTableResponse.validate(); return IcebergTable.fromIcebergTable(loadTableResponse.tableMetadata(), tableIdent.name()); } catch (org.apache.iceberg.exceptions.NoSuchTableException e) { diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java deleted file mode 100644 index 3ef22dd050c..00000000000 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ - -package com.datastrato.gravitino.catalog.lakehouse.iceberg; - -import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogPropertiesMetadata.GRAVITINO_JDBC_DRIVER; -import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogPropertiesMetadata.ICEBERG_JDBC_INITIALIZE; -import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogPropertiesMetadata.ICEBERG_JDBC_PASSWORD; -import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogPropertiesMetadata.ICEBERG_JDBC_USER; -import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogPropertiesMetadata.URI; -import static com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogPropertiesMetadata.WAREHOUSE; - -import com.datastrato.gravitino.Config; -import com.datastrato.gravitino.catalog.lakehouse.iceberg.web.metrics.IcebergMetricsManager; -import com.datastrato.gravitino.config.ConfigBuilder; -import com.datastrato.gravitino.config.ConfigConstants; -import com.datastrato.gravitino.config.ConfigEntry; -import com.datastrato.gravitino.server.web.JettyServerConfig; -import com.datastrato.gravitino.server.web.OverwriteDefaultConfig; -import com.google.common.collect.ImmutableMap; -import java.util.Map; -import java.util.Optional; -import org.apache.commons.lang3.StringUtils; - -public class IcebergConfig extends Config implements OverwriteDefaultConfig { - - public static final ConfigEntry CATALOG_BACKEND = - new ConfigBuilder(IcebergCatalogPropertiesMetadata.CATALOG_BACKEND) - .doc("Catalog backend of Gravitino Iceberg catalog") - .version(ConfigConstants.VERSION_0_2_0) - .stringConf() - .createWithDefault("memory"); - - public static final ConfigEntry CATALOG_WAREHOUSE = - new ConfigBuilder(WAREHOUSE) - .doc("Warehouse directory of catalog") - .version(ConfigConstants.VERSION_0_2_0) - .stringConf() - .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) - .create(); - - public static final ConfigEntry CATALOG_URI = - new ConfigBuilder(URI) - .doc("The uri config of the Iceberg catalog") - .version(ConfigConstants.VERSION_0_2_0) - .stringConf() - .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) - .create(); - - public static final ConfigEntry JDBC_USER = - new ConfigBuilder(ICEBERG_JDBC_USER) - .doc("The username of the Jdbc connection") - .version(ConfigConstants.VERSION_0_2_0) - .stringConf() - .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) - .create(); - - public static final ConfigEntry JDBC_PASSWORD = - new ConfigBuilder(ICEBERG_JDBC_PASSWORD) - .doc("The password of the Jdbc connection") - .version(ConfigConstants.VERSION_0_2_0) - .stringConf() - .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) - .create(); - - public static final ConfigEntry JDBC_DRIVER = - new ConfigBuilder(GRAVITINO_JDBC_DRIVER) - .doc("The driver of the Jdbc connection") - .version(ConfigConstants.VERSION_0_3_0) - .stringConf() - .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) - .create(); - - public static final ConfigEntry JDBC_INIT_TABLES = - new ConfigBuilder(ICEBERG_JDBC_INITIALIZE) - .doc("Whether to initialize meta tables when create Jdbc catalog") - .version(ConfigConstants.VERSION_0_2_0) - .booleanConf() - .createWithDefault(true); - - public static final ConfigEntry ICEBERG_METRICS_STORE = - new ConfigBuilder(IcebergMetricsManager.ICEBERG_METRICS_STORE) - .doc("The store to save Iceberg metrics") - .version(ConfigConstants.VERSION_0_4_0) - .stringConf() - .create(); - - public static final ConfigEntry ICEBERG_METRICS_STORE_RETAIN_DAYS = - new ConfigBuilder(IcebergMetricsManager.ICEBERG_METRICS_STORE_RETAIN_DAYS) - .doc( - "The retain days of Iceberg metrics, the value not greater than 0 means retain forever") - .version(ConfigConstants.VERSION_0_4_0) - .intConf() - .createWithDefault(-1); - - public static final ConfigEntry ICEBERG_METRICS_QUEUE_CAPACITY = - new ConfigBuilder(IcebergMetricsManager.ICEBERG_METRICS_QUEUE_CAPACITY) - .doc("The capacity for Iceberg metrics queues, should greater than 0") - .version(ConfigConstants.VERSION_0_4_0) - .intConf() - .checkValue(value -> value > 0, ConfigConstants.POSITIVE_NUMBER_ERROR_MSG) - .createWithDefault(1000); - - public static final ConfigEntry CATALOG_BACKEND_NAME = - new ConfigBuilder(IcebergCatalogPropertiesMetadata.CATALOG_BACKEND_NAME) - .doc("The catalog name for Iceberg catalog backend") - .version(ConfigConstants.VERSION_0_5_2) - .stringConf() - .create(); - - public String getJdbcDriver() { - return get(JDBC_DRIVER); - } - - public String getCatalogBackendName(String defaultCatalogBackendName) { - return Optional.ofNullable(get(CATALOG_BACKEND_NAME)).orElse(defaultCatalogBackendName); - } - - public IcebergConfig(Map properties) { - super(false); - loadFromMap(properties, k -> true); - } - - public IcebergConfig() { - super(false); - } - - @Override - public Map getOverwriteDefaultConfig() { - return ImmutableMap.of( - JettyServerConfig.WEBSERVER_HTTP_PORT.getKey(), - String.valueOf(JettyServerConfig.DEFAULT_ICEBERG_REST_SERVICE_HTTP_PORT), - JettyServerConfig.WEBSERVER_HTTPS_PORT.getKey(), - String.valueOf(JettyServerConfig.DEFAULT_ICEBERG_REST_SERVICE_HTTPS_PORT)); - } -} diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOps.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOps.java deleted file mode 100644 index 8288d615600..00000000000 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOps.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ -package com.datastrato.gravitino.catalog.lakehouse.iceberg.ops; - -import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogBackend; -import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergConfig; -import com.datastrato.gravitino.catalog.lakehouse.iceberg.ops.IcebergTableOpsHelper.IcebergTableChange; -import com.datastrato.gravitino.catalog.lakehouse.iceberg.utils.IcebergCatalogUtil; -import com.datastrato.gravitino.utils.IsolatedClassLoader; -import com.google.common.base.Preconditions; -import java.sql.Driver; -import java.sql.DriverManager; -import java.util.Collections; -import java.util.Optional; -import javax.ws.rs.NotSupportedException; -import org.apache.iceberg.Transaction; -import org.apache.iceberg.catalog.Catalog; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.SupportsNamespaces; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.rest.CatalogHandlers; -import org.apache.iceberg.rest.requests.CreateNamespaceRequest; -import org.apache.iceberg.rest.requests.CreateTableRequest; -import org.apache.iceberg.rest.requests.RenameTableRequest; -import org.apache.iceberg.rest.requests.UpdateNamespacePropertiesRequest; -import org.apache.iceberg.rest.requests.UpdateTableRequest; -import org.apache.iceberg.rest.responses.CreateNamespaceResponse; -import org.apache.iceberg.rest.responses.GetNamespaceResponse; -import org.apache.iceberg.rest.responses.ListNamespacesResponse; -import org.apache.iceberg.rest.responses.ListTablesResponse; -import org.apache.iceberg.rest.responses.LoadTableResponse; -import org.apache.iceberg.rest.responses.UpdateNamespacePropertiesResponse; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class IcebergTableOps implements AutoCloseable { - public static final Logger LOG = LoggerFactory.getLogger(IcebergTableOps.class); - - protected Catalog catalog; - private SupportsNamespaces asNamespaceCatalog; - private final String catalogType; - private String catalogUri = null; - - public IcebergTableOps(IcebergConfig icebergConfig) { - this.catalogType = icebergConfig.get(IcebergConfig.CATALOG_BACKEND); - if (!IcebergCatalogBackend.MEMORY.name().equalsIgnoreCase(catalogType)) { - icebergConfig.get(IcebergConfig.CATALOG_WAREHOUSE); - this.catalogUri = icebergConfig.get(IcebergConfig.CATALOG_URI); - } - catalog = IcebergCatalogUtil.loadCatalogBackend(catalogType, icebergConfig.getAllConfig()); - if (catalog instanceof SupportsNamespaces) { - asNamespaceCatalog = (SupportsNamespaces) catalog; - } - } - - public IcebergTableOps() { - this(new IcebergConfig(Collections.emptyMap())); - } - - public IcebergTableOpsHelper createIcebergTableOpsHelper() { - return new IcebergTableOpsHelper(catalog); - } - - private void validateNamespace(Optional namespace) { - namespace.ifPresent( - n -> - Preconditions.checkArgument( - n.toString().isEmpty() == false, "Namespace couldn't be empty")); - if (asNamespaceCatalog == null) { - throw new NotSupportedException("The underlying catalog doesn't support namespace operation"); - } - } - - public CreateNamespaceResponse createNamespace(CreateNamespaceRequest request) { - validateNamespace(Optional.of(request.namespace())); - return CatalogHandlers.createNamespace(asNamespaceCatalog, request); - } - - public void dropNamespace(Namespace namespace) { - validateNamespace(Optional.of(namespace)); - CatalogHandlers.dropNamespace(asNamespaceCatalog, namespace); - } - - public GetNamespaceResponse loadNamespace(Namespace namespace) { - validateNamespace(Optional.of(namespace)); - return CatalogHandlers.loadNamespace(asNamespaceCatalog, namespace); - } - - public ListNamespacesResponse listNamespace(Namespace parent) { - validateNamespace(Optional.empty()); - return CatalogHandlers.listNamespaces(asNamespaceCatalog, parent); - } - - public UpdateNamespacePropertiesResponse updateNamespaceProperties( - Namespace namespace, UpdateNamespacePropertiesRequest updateNamespacePropertiesRequest) { - validateNamespace(Optional.of(namespace)); - return CatalogHandlers.updateNamespaceProperties( - asNamespaceCatalog, namespace, updateNamespacePropertiesRequest); - } - - public LoadTableResponse createTable(Namespace namespace, CreateTableRequest request) { - request.validate(); - if (request.stageCreate()) { - return CatalogHandlers.stageTableCreate(catalog, namespace, request); - } - return CatalogHandlers.createTable(catalog, namespace, request); - } - - public void dropTable(TableIdentifier tableIdentifier) { - CatalogHandlers.dropTable(catalog, tableIdentifier); - } - - public void purgeTable(TableIdentifier tableIdentifier) { - CatalogHandlers.purgeTable(catalog, tableIdentifier); - } - - public LoadTableResponse loadTable(TableIdentifier tableIdentifier) { - return CatalogHandlers.loadTable(catalog, tableIdentifier); - } - - public boolean tableExists(TableIdentifier tableIdentifier) { - return catalog.tableExists(tableIdentifier); - } - - public ListTablesResponse listTable(Namespace namespace) { - return CatalogHandlers.listTables(catalog, namespace); - } - - public void renameTable(RenameTableRequest renameTableRequest) { - CatalogHandlers.renameTable(catalog, renameTableRequest); - } - - public LoadTableResponse updateTable( - TableIdentifier tableIdentifier, UpdateTableRequest updateTableRequest) { - return CatalogHandlers.updateTable(catalog, tableIdentifier, updateTableRequest); - } - - public LoadTableResponse updateTable(IcebergTableChange icebergTableChange) { - Transaction transaction = icebergTableChange.getTransaction(); - transaction.commitTransaction(); - return loadTable(icebergTableChange.getTableIdentifier()); - } - - @Override - public void close() throws Exception { - if (catalog instanceof AutoCloseable) { - // JdbcCatalog need close. - ((AutoCloseable) catalog).close(); - } - - // Because each catalog in Gravitino has its own classloader, after a catalog is no longer used - // for a long time or dropped, the instance of classloader needs to be released. In order to - // let JVM GC remove the classloader, we need to release the resources of the classloader. The - // resources include the driver of the catalog backend and the - // AbandonedConnectionCleanupThread of MySQL. For more information about - // AbandonedConnectionCleanupThread, please refer to the corresponding java doc of MySQL - // driver. - if (catalogUri != null && catalogUri.contains("mysql")) { - closeMySQLCatalogResource(); - } else if (catalogUri != null && catalogUri.contains("postgresql")) { - closePostgreSQLCatalogResource(); - } else if (catalogType.equalsIgnoreCase(IcebergCatalogBackend.HIVE.name())) { - // TODO(yuqi) add close for other catalog types such Hive catalog - } - } - - private void closeMySQLCatalogResource() { - try { - // Close thread AbandonedConnectionCleanupThread if we are using `com.mysql.cj.jdbc.Driver`, - // for driver `com.mysql.jdbc.Driver` (deprecated), the daemon thead maybe not this one. - Class.forName("com.mysql.cj.jdbc.AbandonedConnectionCleanupThread") - .getMethod("uncheckedShutdown") - .invoke(null); - LOG.info("AbandonedConnectionCleanupThread has been shutdown..."); - - // Unload the MySQL driver, only Unload the driver if it is loaded by - // IsolatedClassLoader. - closeDriverLoadedByIsolatedClassLoader(catalogUri); - } catch (Exception e) { - LOG.warn("Failed to shutdown AbandonedConnectionCleanupThread or deregister MySQL driver", e); - } - } - - private void closeDriverLoadedByIsolatedClassLoader(String uri) { - try { - Driver driver = DriverManager.getDriver(uri); - if (driver.getClass().getClassLoader().getClass() - == IsolatedClassLoader.CUSTOM_CLASS_LOADER_CLASS) { - DriverManager.deregisterDriver(driver); - LOG.info("Driver {} has been deregistered...", driver); - } - } catch (Exception e) { - LOG.warn("Failed to deregister driver", e); - } - } - - private void closePostgreSQLCatalogResource() { - closeDriverLoadedByIsolatedClassLoader(catalogUri); - } -} diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/IcebergCatalogUtil.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/IcebergCatalogUtil.java index cb8cace4b58..f8275b4b4af 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/IcebergCatalogUtil.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/IcebergCatalogUtil.java @@ -9,10 +9,10 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogBackend; -import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergConfig; import com.datastrato.gravitino.catalog.lakehouse.iceberg.authentication.AuthenticationConfig; import com.datastrato.gravitino.catalog.lakehouse.iceberg.authentication.kerberos.HiveBackendProxy; import com.datastrato.gravitino.catalog.lakehouse.iceberg.authentication.kerberos.KerberosClient; +import com.datastrato.gravitino.iceberg.common.IcebergConfig; import java.io.File; import java.io.IOException; import java.util.Collections; diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/resources/META-INF/services/com.datastrato.gravitino.auxiliary.GravitinoAuxiliaryService b/catalogs/catalog-lakehouse-iceberg/src/main/resources/META-INF/services/com.datastrato.gravitino.auxiliary.GravitinoAuxiliaryService deleted file mode 100644 index fa3ff960adb..00000000000 --- a/catalogs/catalog-lakehouse-iceberg/src/main/resources/META-INF/services/com.datastrato.gravitino.auxiliary.GravitinoAuxiliaryService +++ /dev/null @@ -1,5 +0,0 @@ -# -# Copyright 2023 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. -# -com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergRESTService diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalog.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalog.java index 68a9346cfac..a0cb6a88ba0 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalog.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergCatalog.java @@ -10,10 +10,10 @@ import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.catalog.PropertiesMetadataHelpers; -import com.datastrato.gravitino.catalog.lakehouse.iceberg.ops.IcebergTableOps; import com.datastrato.gravitino.connector.CatalogOperations; import com.datastrato.gravitino.connector.HasPropertyMetadata; import com.datastrato.gravitino.connector.PropertiesMetadata; +import com.datastrato.gravitino.iceberg.common.ops.IcebergTableOps; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.CatalogEntity; import com.google.common.collect.Maps; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergConfig.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergConfig.java index d6b26003220..2973aa5397a 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergConfig.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/TestIcebergConfig.java @@ -5,6 +5,7 @@ package com.datastrato.gravitino.catalog.lakehouse.iceberg; +import com.datastrato.gravitino.iceberg.common.IcebergConfig; import com.datastrato.gravitino.server.web.JettyServerConfig; import com.google.common.collect.ImmutableMap; import java.util.Map; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java index 83c669bacae..cb4b4e647cd 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java @@ -19,7 +19,6 @@ import com.datastrato.gravitino.SchemaChange; import com.datastrato.gravitino.SupportsSchemas; import com.datastrato.gravitino.auth.AuthConstants; -import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergConfig; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergSchemaPropertiesMetadata; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergTable; import com.datastrato.gravitino.catalog.lakehouse.iceberg.ops.IcebergTableOpsHelper; @@ -29,6 +28,7 @@ import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; import com.datastrato.gravitino.exceptions.TableAlreadyExistsException; +import com.datastrato.gravitino.iceberg.common.IcebergConfig; import com.datastrato.gravitino.integration.test.container.ContainerSuite; import com.datastrato.gravitino.integration.test.util.AbstractIT; import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java index 07389277c82..e99c20a39a1 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java @@ -12,10 +12,10 @@ import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.NameIdentifier; -import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergConfig; import com.datastrato.gravitino.client.GravitinoAdminClient; import com.datastrato.gravitino.client.GravitinoMetalake; import com.datastrato.gravitino.client.KerberosTokenProvider; +import com.datastrato.gravitino.iceberg.common.IcebergConfig; import com.datastrato.gravitino.integration.test.container.ContainerSuite; import com.datastrato.gravitino.integration.test.container.HiveContainer; import com.datastrato.gravitino.integration.test.util.AbstractIT; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/TestMultipleJDBCLoad.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/TestMultipleJDBCLoad.java index 592e3d988e9..4cdcf2455db 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/TestMultipleJDBCLoad.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/TestMultipleJDBCLoad.java @@ -10,8 +10,8 @@ import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.NameIdentifier; -import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergConfig; import com.datastrato.gravitino.client.GravitinoMetalake; +import com.datastrato.gravitino.iceberg.common.IcebergConfig; import com.datastrato.gravitino.integration.test.container.MySQLContainer; import com.datastrato.gravitino.integration.test.container.PostgreSQLContainer; import com.datastrato.gravitino.integration.test.util.AbstractIT; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/TestIcebergTableUpdate.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/TestIcebergTableUpdate.java index 631017ec92d..ea4d1fd508a 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/TestIcebergTableUpdate.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/TestIcebergTableUpdate.java @@ -6,6 +6,7 @@ import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.catalog.lakehouse.iceberg.ops.IcebergTableOpsHelper.IcebergTableChange; +import com.datastrato.gravitino.iceberg.common.ops.IcebergTableOps; import com.datastrato.gravitino.rel.TableChange; import com.datastrato.gravitino.rel.TableChange.ColumnPosition; import com.datastrato.gravitino.rel.types.Types; diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/TestIcebergCatalogUtil.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/TestIcebergCatalogUtil.java index 70bb1c70075..4a51af99d9d 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/TestIcebergCatalogUtil.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/TestIcebergCatalogUtil.java @@ -6,7 +6,7 @@ package com.datastrato.gravitino.catalog.lakehouse.iceberg.utils; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogPropertiesMetadata; -import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergConfig; +import com.datastrato.gravitino.iceberg.common.IcebergConfig; import java.util.HashMap; import java.util.Map; import org.apache.iceberg.CatalogProperties; diff --git a/iceberg-common/src/main/java/com/datastrato/gravitino/iceberg/common/ops/IcebergTableOps.java b/iceberg-common/src/main/java/com/datastrato/gravitino/iceberg/common/ops/IcebergTableOps.java index f3f42446847..89b4205fa83 100644 --- a/iceberg-common/src/main/java/com/datastrato/gravitino/iceberg/common/ops/IcebergTableOps.java +++ b/iceberg-common/src/main/java/com/datastrato/gravitino/iceberg/common/ops/IcebergTableOps.java @@ -13,6 +13,7 @@ import java.sql.DriverManager; import java.util.Collections; import java.util.Optional; +import lombok.Getter; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.SupportsNamespaces; @@ -35,7 +36,7 @@ public class IcebergTableOps implements AutoCloseable { public static final Logger LOG = LoggerFactory.getLogger(IcebergTableOps.class); - protected Catalog catalog; + @Getter protected Catalog catalog; private SupportsNamespaces asNamespaceCatalog; private final String catalogType; private String catalogUri = null; diff --git a/iceberg-common/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider b/iceberg-common/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider deleted file mode 100644 index 25be79ab2c3..00000000000 --- a/iceberg-common/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider +++ /dev/null @@ -1,5 +0,0 @@ -# -# Copyright 2023 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. -# -com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalog \ No newline at end of file diff --git a/iceberg-common/src/main/resources/META-INF/services/com.datastrato.gravitino.auxiliary.GravitinoAuxiliaryService b/iceberg-common/src/main/resources/META-INF/services/com.datastrato.gravitino.auxiliary.GravitinoAuxiliaryService deleted file mode 100644 index fa3ff960adb..00000000000 --- a/iceberg-common/src/main/resources/META-INF/services/com.datastrato.gravitino.auxiliary.GravitinoAuxiliaryService +++ /dev/null @@ -1,5 +0,0 @@ -# -# Copyright 2023 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. -# -com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergRESTService diff --git a/iceberg-rest-server/build.gradle.kts b/iceberg-rest-server/build.gradle.kts index 3c742c4d22e..ccc5872010e 100644 --- a/iceberg-rest-server/build.gradle.kts +++ b/iceberg-rest-server/build.gradle.kts @@ -149,6 +149,6 @@ tasks { register("copyLibs", Copy::class) { dependsOn(copyDepends, "build") from("build/libs") - into("$rootDir/distribution/${rootProject.name}-iceberg-rest-server/libs") + into("$rootDir/distribution/package/iceberg-rest-server/libs") } } diff --git a/iceberg-rest-server/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider b/iceberg-rest-server/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider deleted file mode 100644 index 25be79ab2c3..00000000000 --- a/iceberg-rest-server/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider +++ /dev/null @@ -1,5 +0,0 @@ -# -# Copyright 2023 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. -# -com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalog \ No newline at end of file diff --git a/iceberg-rest-server/src/main/resources/META-INF/services/com.datastrato.gravitino.auxiliary.GravitinoAuxiliaryService b/iceberg-rest-server/src/main/resources/META-INF/services/com.datastrato.gravitino.auxiliary.GravitinoAuxiliaryService index fa3ff960adb..9f03c931ebd 100644 --- a/iceberg-rest-server/src/main/resources/META-INF/services/com.datastrato.gravitino.auxiliary.GravitinoAuxiliaryService +++ b/iceberg-rest-server/src/main/resources/META-INF/services/com.datastrato.gravitino.auxiliary.GravitinoAuxiliaryService @@ -2,4 +2,4 @@ # Copyright 2023 Datastrato Pvt Ltd. # This software is licensed under the Apache License version 2. # -com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergRESTService +com.datastrato.gravitino.iceberg.RESTService