[apache#3463] improvement(lakehouse-iceberg): Support user authentica…

…tion for Iceberg Hive backend. (apache#3724) ### What changes were proposed in this pull request? - Add user authentication for Iceberg with Hive backend using kerberos - Add e2e test. ### Why are the changes needed? It's a must-have feature for iceberg. Fix: apache#3463 ### Does this PR introduce _any_ user-facing change? N/A. ### How was this patch tested? ITs.
diqiu50 · Jun 13, 2024 · 468eb06 · 468eb06
1 parent 26ea392
commit 468eb06
Show file tree

Hide file tree

Showing 28 changed files with 1,196 additions and 57 deletions.
diff --git a/LICENSE b/LICENSE
@@ -235,6 +235,7 @@
    ./catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/web/IcebergExceptionMapper.java
    ./catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/converter/DescribeIcebergSortOrderVisitor.java
    ./catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/IcebergTablePropertiesUtil.java
+   ./catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergHiveCachedClientPool.java
    ./clients/client-java/src/main/java/com/datastrato/gravitino/client/HTTPClient.java
    ./clients/client-java/src/main/java/com/datastrato/gravitino/client/RESTClient.java
    ./clients/client-java/src/test/java/com/datastrato/gravitino/client/TestHTTPClient.java

diff --git a/...s/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalog.java b/...s/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalog.java
@@ -4,7 +4,7 @@
  */
 package com.datastrato.gravitino.catalog.hadoop;
 
-import com.datastrato.gravitino.catalog.hadoop.kerberos.KerberosConfig;
+import com.datastrato.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig;
 import com.datastrato.gravitino.connector.BaseCatalog;
 import com.datastrato.gravitino.connector.CatalogOperations;
 import com.datastrato.gravitino.connector.PropertiesMetadata;

diff --git a/...hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/...hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogOperations.java
@@ -15,8 +15,8 @@
 import com.datastrato.gravitino.Schema;
 import com.datastrato.gravitino.SchemaChange;
 import com.datastrato.gravitino.StringIdentifier;
-import com.datastrato.gravitino.catalog.hadoop.kerberos.AuthenticationConfig;
-import com.datastrato.gravitino.catalog.hadoop.kerberos.KerberosClient;
+import com.datastrato.gravitino.catalog.hadoop.authentication.AuthenticationConfig;
+import com.datastrato.gravitino.catalog.hadoop.authentication.kerberos.KerberosClient;
 import com.datastrato.gravitino.connector.CatalogInfo;
 import com.datastrato.gravitino.connector.CatalogOperations;
 import com.datastrato.gravitino.connector.HasPropertyMetadata;

diff --git a/...rc/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java b/...rc/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java
@@ -4,8 +4,8 @@
  */
 package com.datastrato.gravitino.catalog.hadoop;
 
-import com.datastrato.gravitino.catalog.hadoop.kerberos.AuthenticationConfig;
-import com.datastrato.gravitino.catalog.hadoop.kerberos.KerberosConfig;
+import com.datastrato.gravitino.catalog.hadoop.authentication.AuthenticationConfig;
+import com.datastrato.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig;
 import com.datastrato.gravitino.connector.BaseCatalogPropertiesMetadata;
 import com.datastrato.gravitino.connector.PropertyEntry;
 import com.google.common.collect.ImmutableMap;

diff --git a/...hadoop/kerberos/AuthenticationConfig.java → .../authentication/AuthenticationConfig.java b/...hadoop/kerberos/AuthenticationConfig.java → .../authentication/AuthenticationConfig.java
@@ -3,11 +3,10 @@
  * This software is licensed under the Apache License version 2.
  */
 
-package com.datastrato.gravitino.catalog.hadoop.kerberos;
-
-import static com.datastrato.gravitino.catalog.hadoop.kerberos.KerberosConfig.DEFAULT_IMPERSONATION_ENABLE;
+package com.datastrato.gravitino.catalog.hadoop.authentication;
 
 import com.datastrato.gravitino.Config;
+import com.datastrato.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig;
 import com.datastrato.gravitino.config.ConfigBuilder;
 import com.datastrato.gravitino.config.ConfigConstants;
 import com.datastrato.gravitino.config.ConfigEntry;
@@ -29,17 +28,18 @@ public AuthenticationConfig(Map<String, String> properties) {
 
   public static final ConfigEntry<String> AUTH_TYPE_ENTRY =
       new ConfigBuilder(AUTH_TYPE_KEY)
-          .doc("The type of authentication for Hadoop catalog, currently we only support Kerberos")
+          .doc(
+              "The type of authentication for Hadoop catalog, currently we only support simple and Kerberos")
           .version(ConfigConstants.VERSION_0_5_1)
           .stringConf()
-          .create();
+          .createWithDefault("simple");
 
   public static final ConfigEntry<Boolean> ENABLE_IMPERSONATION_ENTRY =
       new ConfigBuilder(IMPERSONATION_ENABLE_KEY)
           .doc("Whether to enable impersonation for the Hadoop catalog")
           .version(ConfigConstants.VERSION_0_5_1)
           .booleanConf()
-          .createWithDefault(DEFAULT_IMPERSONATION_ENABLE);
+          .createWithDefault(KerberosConfig.DEFAULT_IMPERSONATION_ENABLE);
 
   public String getAuthType() {
     return get(AUTH_TYPE_ENTRY);
@@ -58,7 +58,7 @@ public boolean isImpersonationEnabled() {
                   "Whether to enable impersonation for the Hadoop catalog",
                   false,
                   true,
-                  DEFAULT_IMPERSONATION_ENABLE,
+                  KerberosConfig.DEFAULT_IMPERSONATION_ENABLE,
                   false,
                   false))
           .put(

diff --git a/...talog/hadoop/kerberos/FetchFileUtils.java → ...thentication/kerberos/FetchFileUtils.java b/...talog/hadoop/kerberos/FetchFileUtils.java → ...thentication/kerberos/FetchFileUtils.java
@@ -2,7 +2,7 @@
  * Copyright 2024 Datastrato Pvt Ltd.
  * This software is licensed under the Apache License version 2.
  */
-package com.datastrato.gravitino.catalog.hadoop.kerberos;
+package com.datastrato.gravitino.catalog.hadoop.authentication.kerberos;
 
 import java.io.File;
 import java.io.IOException;

diff --git a/...talog/hadoop/kerberos/KerberosClient.java → ...thentication/kerberos/KerberosClient.java b/...talog/hadoop/kerberos/KerberosClient.java → ...thentication/kerberos/KerberosClient.java
@@ -3,7 +3,7 @@
  * This software is licensed under the Apache License version 2.
  */
 
-package com.datastrato.gravitino.catalog.hadoop.kerberos;
+package com.datastrato.gravitino.catalog.hadoop.authentication.kerberos;
 
 import com.google.common.base.Preconditions;
 import com.google.common.base.Splitter;

diff --git a/...talog/hadoop/kerberos/KerberosConfig.java → ...thentication/kerberos/KerberosConfig.java b/...talog/hadoop/kerberos/KerberosConfig.java → ...thentication/kerberos/KerberosConfig.java
@@ -3,8 +3,9 @@
  * This software is licensed under the Apache License version 2.
  */
 
-package com.datastrato.gravitino.catalog.hadoop.kerberos;
+package com.datastrato.gravitino.catalog.hadoop.authentication.kerberos;
 
+import com.datastrato.gravitino.catalog.hadoop.authentication.AuthenticationConfig;
 import com.datastrato.gravitino.config.ConfigBuilder;
 import com.datastrato.gravitino.config.ConfigConstants;
 import com.datastrato.gravitino.config.ConfigEntry;

diff --git a/.../com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java b/.../com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java
@@ -5,10 +5,10 @@
 
 package com.datastrato.gravitino.catalog.hadoop.integration.test;
 
-import static com.datastrato.gravitino.catalog.hadoop.kerberos.AuthenticationConfig.AUTH_TYPE_KEY;
-import static com.datastrato.gravitino.catalog.hadoop.kerberos.KerberosConfig.IMPERSONATION_ENABLE_KEY;
-import static com.datastrato.gravitino.catalog.hadoop.kerberos.KerberosConfig.KEY_TAB_URI_KEY;
-import static com.datastrato.gravitino.catalog.hadoop.kerberos.KerberosConfig.PRINCIPAL_KEY;
+import static com.datastrato.gravitino.catalog.hadoop.authentication.AuthenticationConfig.AUTH_TYPE_KEY;
+import static com.datastrato.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig.IMPERSONATION_ENABLE_KEY;
+import static com.datastrato.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig.KEY_TAB_URI_KEY;
+import static com.datastrato.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig.PRINCIPAL_KEY;
 
 import com.datastrato.gravitino.Catalog;
 import com.datastrato.gravitino.NameIdentifier;

diff --git a/...a/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java b/...a/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java
@@ -5,10 +5,10 @@
 
 package com.datastrato.gravitino.catalog.hadoop.integration.test;
 
-import static com.datastrato.gravitino.catalog.hadoop.kerberos.AuthenticationConfig.AUTH_TYPE_KEY;
-import static com.datastrato.gravitino.catalog.hadoop.kerberos.KerberosConfig.IMPERSONATION_ENABLE_KEY;
-import static com.datastrato.gravitino.catalog.hadoop.kerberos.KerberosConfig.KEY_TAB_URI_KEY;
-import static com.datastrato.gravitino.catalog.hadoop.kerberos.KerberosConfig.PRINCIPAL_KEY;
+import static com.datastrato.gravitino.catalog.hadoop.authentication.AuthenticationConfig.AUTH_TYPE_KEY;
+import static com.datastrato.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig.IMPERSONATION_ENABLE_KEY;
+import static com.datastrato.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig.KEY_TAB_URI_KEY;
+import static com.datastrato.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig.PRINCIPAL_KEY;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_PROTECTION_KEY;
 
 import com.datastrato.gravitino.Catalog;

diff --git a/...e/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java b/...e/src/test/java/com/datastrato/gravitino/catalog/hive/integration/test/CatalogHiveIT.java
@@ -199,18 +199,18 @@ public static void startup() throws Exception {
 
   @AfterAll
   public static void stop() throws IOException {
-    Arrays.stream(catalog.asSchemas().listSchemas())
-        .filter(schema -> !schema.equals("default"))
-        .forEach(
-            (schema -> {
-              catalog.asSchemas().dropSchema(schema, true);
-            }));
-    Arrays.stream(metalake.listCatalogs())
-        .forEach(
-            (catalogName -> {
-              metalake.dropCatalog(catalogName);
-            }));
     if (client != null) {
+      Arrays.stream(catalog.asSchemas().listSchemas())
+          .filter(schema -> !schema.equals("default"))
+          .forEach(
+              (schema -> {
+                catalog.asSchemas().dropSchema(schema, true);
+              }));
+      Arrays.stream(metalake.listCatalogs())
+          .forEach(
+              (catalogName -> {
+                metalake.dropCatalog(catalogName);
+              }));
       client.dropMetalake(metalakeName);
     }
     if (hiveClientPool != null) {
@@ -230,7 +230,6 @@ public static void stop() throws IOException {
       LOG.error("Failed to close CloseableGroup", e);
     }
 
-    AbstractIT.customConfigs.clear();
     AbstractIT.client = null;
   }
 

diff --git a/...java/com/datastrato/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java b/...java/com/datastrato/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java
@@ -117,7 +117,6 @@ public static void stop() {
     System.clearProperty("java.security.krb5.conf");
     System.clearProperty("sun.security.krb5.debug");
 
-    AbstractIT.customConfigs.clear();
     AbstractIT.client = null;
   }
 

diff --git a/.../test/java/com/datastrato/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java b/.../test/java/com/datastrato/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java
@@ -119,7 +119,6 @@ public static void stop() {
     setEnv(HADOOP_USER_NAME, originHadoopUser);
     anotherClient.close();
 
-    AbstractIT.customConfigs.clear();
     AbstractIT.client = null;
   }
 

diff --git a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts
@@ -25,6 +25,8 @@ dependencies {
   implementation(libs.bundles.jetty)
   implementation(libs.bundles.jersey)
   implementation(libs.bundles.log4j)
+  implementation(libs.caffeine)
+  implementation(libs.cglib)
   implementation(libs.commons.collections4)
   implementation(libs.commons.io)
   implementation(libs.commons.lang3)
@@ -163,6 +165,7 @@ tasks.test {
 
     doFirst {
       environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.12")
+      environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-kerberos-hive:0.1.2")
     }
 
     val init = project.extra.get("initIntegrationTest") as (Test) -> Unit

diff --git a/...ain/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java b/...ain/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java
@@ -89,6 +89,7 @@ public void initialize(
 
     Map<String, String> resultConf = Maps.newHashMap(prefixMap);
     resultConf.putAll(gravitinoConfig);
+    resultConf.put("catalog_uuid", info.id().toString());
 
     IcebergConfig icebergConfig = new IcebergConfig(resultConf);
 

diff --git a/.../com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java b/.../com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java
@@ -7,6 +7,8 @@
 import static com.datastrato.gravitino.connector.PropertyEntry.enumImmutablePropertyEntry;
 import static com.datastrato.gravitino.connector.PropertyEntry.stringRequiredPropertyEntry;
 
+import com.datastrato.gravitino.catalog.lakehouse.iceberg.authentication.AuthenticationConfig;
+import com.datastrato.gravitino.catalog.lakehouse.iceberg.authentication.kerberos.KerberosConfig;
 import com.datastrato.gravitino.connector.BaseCatalogPropertiesMetadata;
 import com.datastrato.gravitino.connector.PropertyEntry;
 import com.google.common.collect.ImmutableList;
@@ -34,8 +36,7 @@ public class IcebergCatalogPropertiesMetadata extends BaseCatalogPropertiesMetad
 
   // Map that maintains the mapping of keys in Gravitino to that in Iceberg, for example, users
   // will only need to set the configuration 'catalog-backend' in Gravitino and Gravitino will
-  // change
-  // it to `catalogType` automatically and pass it to Iceberg.
+  // change it to `catalogType` automatically and pass it to Iceberg.
   public static final Map<String, String> GRAVITINO_CONFIG_TO_ICEBERG =
       ImmutableMap.of(
           CATALOG_BACKEND_NAME,
@@ -51,6 +52,21 @@ public class IcebergCatalogPropertiesMetadata extends BaseCatalogPropertiesMetad
           WAREHOUSE,
           WAREHOUSE);
 
+  public static final Map<String, String> KERBEROS_CONFIGURATION_FOR_HIVE_BACKEND =
+      ImmutableMap.of(
+          KerberosConfig.PRINCIPAL_KEY,
+          KerberosConfig.PRINCIPAL_KEY,
+          KerberosConfig.KET_TAB_URI_KEY,
+          KerberosConfig.KET_TAB_URI_KEY,
+          KerberosConfig.CHECK_INTERVAL_SEC_KEY,
+          KerberosConfig.CHECK_INTERVAL_SEC_KEY,
+          KerberosConfig.FETCH_TIMEOUT_SEC_KEY,
+          KerberosConfig.FETCH_TIMEOUT_SEC_KEY,
+          AuthenticationConfig.IMPERSONATION_ENABLE_KEY,
+          AuthenticationConfig.IMPERSONATION_ENABLE_KEY,
+          AuthenticationConfig.AUTH_TYPE_KEY,
+          AuthenticationConfig.AUTH_TYPE_KEY);
+
   static {
     List<PropertyEntry<?>> propertyEntries =
         ImmutableList.of(
@@ -67,6 +83,8 @@ public class IcebergCatalogPropertiesMetadata extends BaseCatalogPropertiesMetad
                 WAREHOUSE, "Iceberg catalog warehouse config", false, false));
     HashMap<String, PropertyEntry<?>> result = Maps.newHashMap(BASIC_CATALOG_PROPERTY_ENTRIES);
     result.putAll(Maps.uniqueIndex(propertyEntries, PropertyEntry::getName));
+    result.putAll(KerberosConfig.KERBEROS_PROPERTY_ENTRIES);
+    result.putAll(AuthenticationConfig.AUTHENTICATION_PROPERTY_ENTRIES);
     PROPERTIES_METADATA = ImmutableMap.copyOf(result);
   }
 
@@ -82,6 +100,10 @@ public Map<String, String> transformProperties(Map<String, String> properties) {
           if (GRAVITINO_CONFIG_TO_ICEBERG.containsKey(key)) {
             gravitinoConfig.put(GRAVITINO_CONFIG_TO_ICEBERG.get(key), value);
           }
+
+          if (KERBEROS_CONFIGURATION_FOR_HIVE_BACKEND.containsKey(key)) {
+            gravitinoConfig.put(KERBEROS_CONFIGURATION_FOR_HIVE_BACKEND.get(key), value);
+          }
         });
     return gravitinoConfig;
   }