[apache#4176]feat(iceberg) support multiple catalogs in Iceberg REST …

…catalog server
theoryxu · Aug 13, 2024 · 71bd4c7 · 71bd4c7
1 parent 3e3406a
commit 71bd4c7
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 18 deletions.
diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md
@@ -155,9 +155,9 @@ The Gravitino Iceberg REST server supports multiple catalogs and offers a config
 |----------------------------------------------|------------------------------------------------------------------------------------------------------------------|-----------------------------|----------|---------------|
 | `gravitino.iceberg-rest.catalog-provider`    | The implementation of IcebergTableOpsProvider defines how the Iceberg REST catalog server gets Iceberg catalogs. | `config-based-provider`     | No       | 0.7.0         |
 
-When using a config-based catalog provider, you can configure the default catalog with gravitino.iceberg-rest.catalog.<param name>=<value>. For specific catalogs, use the format gravitino.iceberg-rest.catalog.<catalog name>.<param name>=<value>.
+When using a config-based catalog provider, you can configure the default catalog with `gravitino.iceberg-rest.catalog.<param name>=<value>`. For specific catalogs, use the format `gravitino.iceberg-rest.catalog.<catalog name>.<param name>=<value>`.
 
-For instance, you can configure three different catalogs, the default catalog and the specific hive_backend and jdbc_backend catalogs separately.
+For instance, you can configure three different catalogs, the default catalog and the specific `hive_backend` and `jdbc_backend` catalogs separately.
 
 ```text
 gravitino.iceberg-rest.catalog-backend = jdbc
@@ -174,7 +174,7 @@ gravitino.iceberg-rest.catalog.jdbc_backend.warehouse = hdfs://127.0.0.1:9000/us
 ...
 ```
 
-You can access different catalogs by setting the prefix to the specific catalog name in the Iceberg REST client configuration. The default catalog will be used if you do not specify a prefix. For instance, consider the case of SparkSQL.
+You can access different catalogs by setting the `prefix` to the specific catalog name in the Iceberg REST client configuration. The default catalog will be used if you do not specify a `prefix`. For instance, consider the case of SparkSQL.
 
 ```shell
 ./bin/spark-sql -v \

diff --git a/...-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergTableOpsManager.java b/...-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergTableOpsManager.java
@@ -20,7 +20,6 @@
 
 import com.github.benmanes.caffeine.cache.Cache;
 import com.github.benmanes.caffeine.cache.Caffeine;
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableMap;
 import java.util.Map;
@@ -40,7 +39,7 @@ public class IcebergTableOpsManager implements AutoCloseable {
 
   private final Cache<String, IcebergTableOps> icebergTableOpsCache;
 
-  private IcebergTableOpsProvider provider;
+  private final IcebergTableOpsProvider provider;
 
   public IcebergTableOpsManager(Map<String, String> properties) {
     this.icebergTableOpsCache = Caffeine.newBuilder().build();
@@ -58,11 +57,6 @@ public IcebergTableOps getOps(String rawPrefix) {
     return icebergTableOpsCache.get(catalogName, k -> provider.getIcebergTableOps(catalogName));
   }
 
-  @VisibleForTesting
-  public void setIcebergTableOpsProvider(IcebergTableOpsProvider provider) {
-    this.provider = provider;
-  }
-
   private String getCatalogName(String rawPrefix) {
     String prefix = shelling(rawPrefix);
     Preconditions.checkArgument(
@@ -77,13 +71,8 @@ private String getCatalogName(String rawPrefix) {
   private IcebergTableOpsProvider createProvider(Map<String, String> properties) {
     String providerName =
         (new IcebergConfig(properties)).get(IcebergConfig.ICEBERG_REST_CATALOG_PROVIDER);
-    String className = ICEBERG_TABLE_OPS_PROVIDER_NAMES.get(providerName);
-
-    Preconditions.checkArgument(
-        StringUtils.isNotEmpty(className),
-        String.format("%s can not match any provider", providerName));
+    String className = ICEBERG_TABLE_OPS_PROVIDER_NAMES.getOrDefault(providerName, providerName);
     LOG.info("Load Iceberg catalog provider: {}.", className);
-
     try {
       Class<?> providerClz = Class.forName(className);
       return (IcebergTableOpsProvider) providerClz.getDeclaredConstructor().newInstance();

diff --git a/...t-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java b/...t-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java
@@ -23,6 +23,7 @@
 import java.util.Map;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants;
 import org.apache.gravitino.iceberg.common.IcebergConfig;
 import org.apache.gravitino.iceberg.common.ops.IcebergTableOpsManager;
 import org.apache.gravitino.iceberg.service.IcebergExceptionMapper;
@@ -70,9 +71,10 @@ public static ResourceConfig getIcebergResourceConfig(Class c, boolean bindIcebe
     if (bindIcebergTableOps) {
       Map<String, String> catalogConf = Maps.newHashMap();
       catalogConf.put(String.format("catalog.%s.catalog-backend-name", PREFIX), PREFIX);
+      catalogConf.put(
+          IcebergConstants.ICEBERG_REST_CATALOG_PROVIDER,
+          ConfigBasedIcebergTableOpsProviderForTest.class.getName());
       IcebergTableOpsManager icebergTableOpsManager = new IcebergTableOpsManager(catalogConf);
-      icebergTableOpsManager.setIcebergTableOpsProvider(
-          new ConfigBasedIcebergTableOpsProviderForTest());
 
       IcebergMetricsManager icebergMetricsManager = new IcebergMetricsManager(new IcebergConfig());
       resourceConfig.register(