Merge branch 'main' of github.com:datastrato/graviton into fix_bundle…

…s_jar_problem
apache · Dec 26, 2024 · a9c1a29 · a9c1a29
2 parents bbefd00 + 082bbdc
commit a9c1a29
Show file tree

Hide file tree

Showing 36 changed files with 1,646 additions and 157 deletions.
diff --git a/bundles/aws-bundle/build.gradle.kts b/bundles/aws-bundle/build.gradle.kts
@@ -39,10 +39,12 @@ dependencies {
   implementation(project(":catalogs:hadoop-common")) {
     exclude("*")
   }
+
   implementation(libs.aws.iam)
   implementation(libs.aws.policy)
   implementation(libs.aws.sts)
   implementation(libs.commons.lang3)
+  implementation(libs.hadoop3.aws)
   implementation(libs.guava)
 }
 

diff --git a/...ogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java b/...ogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java
@@ -21,6 +21,7 @@
 
 public class CredentialConstants {
   public static final String CREDENTIAL_PROVIDER_TYPE = "credential-provider-type";
+  public static final String CREDENTIAL_PROVIDERS = "credential-providers";
   public static final String S3_TOKEN_CREDENTIAL_PROVIDER = "s3-token";
   public static final String S3_TOKEN_EXPIRE_IN_SECS = "s3-token-expire-in-secs";
 

diff --git a/...op/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java b/...op/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java
@@ -27,6 +27,7 @@
 import org.apache.gravitino.catalog.hadoop.fs.LocalFileSystemProvider;
 import org.apache.gravitino.connector.BaseCatalogPropertiesMetadata;
 import org.apache.gravitino.connector.PropertyEntry;
+import org.apache.gravitino.credential.config.CredentialConfig;
 
 public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetadata {
 
@@ -84,6 +85,7 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada
           // The following two are about authentication.
           .putAll(KERBEROS_PROPERTY_ENTRIES)
           .putAll(AuthenticationConfig.AUTHENTICATION_PROPERTY_ENTRIES)
+          .putAll(CredentialConfig.CREDENTIAL_PROPERTY_ENTRIES)
           .build();
 
   @Override

diff --git a/...op/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetPropertiesMetadata.java b/...op/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetPropertiesMetadata.java
@@ -24,6 +24,7 @@
 import org.apache.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig;
 import org.apache.gravitino.connector.BasePropertiesMetadata;
 import org.apache.gravitino.connector.PropertyEntry;
+import org.apache.gravitino.credential.config.CredentialConfig;
 
 public class HadoopFilesetPropertiesMetadata extends BasePropertiesMetadata {
 
@@ -32,6 +33,7 @@ protected Map<String, PropertyEntry<?>> specificPropertyEntries() {
     ImmutableMap.Builder<String, PropertyEntry<?>> builder = ImmutableMap.builder();
     builder.putAll(KerberosConfig.KERBEROS_PROPERTY_ENTRIES);
     builder.putAll(AuthenticationConfig.AUTHENTICATION_PROPERTY_ENTRIES);
+    builder.putAll(CredentialConfig.CREDENTIAL_PROPERTY_ENTRIES);
     return builder.build();
   }
 }
diff --git a/...oop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopSchemaPropertiesMetadata.java b/...oop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopSchemaPropertiesMetadata.java
@@ -24,6 +24,7 @@
 import org.apache.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig;
 import org.apache.gravitino.connector.BasePropertiesMetadata;
 import org.apache.gravitino.connector.PropertyEntry;
+import org.apache.gravitino.credential.config.CredentialConfig;
 
 public class HadoopSchemaPropertiesMetadata extends BasePropertiesMetadata {
 
@@ -49,6 +50,7 @@ public class HadoopSchemaPropertiesMetadata extends BasePropertiesMetadata {
                   false /* hidden */))
           .putAll(KerberosConfig.KERBEROS_PROPERTY_ENTRIES)
           .putAll(AuthenticationConfig.AUTHENTICATION_PROPERTY_ENTRIES)
+          .putAll(CredentialConfig.CREDENTIAL_PROPERTY_ENTRIES)
           .build();
 
   @Override

diff --git a/...doop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java b/...doop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java
@@ -20,11 +20,16 @@
 package org.apache.gravitino.catalog.hadoop;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
 import java.io.IOException;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
 import javax.security.auth.Subject;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.gravitino.Catalog;
 import org.apache.gravitino.Entity;
 import org.apache.gravitino.EntityStore;
@@ -38,6 +43,9 @@
 import org.apache.gravitino.connector.CatalogOperations;
 import org.apache.gravitino.connector.HasPropertyMetadata;
 import org.apache.gravitino.connector.SupportsSchemas;
+import org.apache.gravitino.connector.credential.PathContext;
+import org.apache.gravitino.connector.credential.SupportsPathBasedCredentials;
+import org.apache.gravitino.credential.CredentialUtils;
 import org.apache.gravitino.exceptions.FilesetAlreadyExistsException;
 import org.apache.gravitino.exceptions.NoSuchCatalogException;
 import org.apache.gravitino.exceptions.NoSuchEntityException;
@@ -50,13 +58,14 @@
 import org.apache.gravitino.file.FilesetChange;
 import org.apache.gravitino.meta.FilesetEntity;
 import org.apache.gravitino.meta.SchemaEntity;
+import org.apache.gravitino.utils.NameIdentifierUtil;
 import org.apache.gravitino.utils.PrincipalUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 @SuppressWarnings("removal")
 public class SecureHadoopCatalogOperations
-    implements CatalogOperations, SupportsSchemas, FilesetCatalog {
+    implements CatalogOperations, SupportsSchemas, FilesetCatalog, SupportsPathBasedCredentials {
 
   public static final Logger LOG = LoggerFactory.getLogger(SecureHadoopCatalogOperations.class);
 
@@ -66,6 +75,8 @@ public class SecureHadoopCatalogOperations
 
   private UserContext catalogUserContext;
 
+  private Map<String, String> catalogProperties;
+
   public SecureHadoopCatalogOperations() {
     this.hadoopCatalogOperations = new HadoopCatalogOperations();
   }
@@ -74,6 +85,20 @@ public SecureHadoopCatalogOperations(EntityStore store) {
     this.hadoopCatalogOperations = new HadoopCatalogOperations(store);
   }
 
+  @Override
+  public void initialize(
+      Map<String, String> config, CatalogInfo info, HasPropertyMetadata propertiesMetadata)
+      throws RuntimeException {
+    hadoopCatalogOperations.initialize(config, info, propertiesMetadata);
+    this.catalogUserContext =
+        UserContext.getUserContext(
+            NameIdentifier.of(info.namespace(), info.name()),
+            config,
+            hadoopCatalogOperations.getHadoopConf(),
+            info);
+    this.catalogProperties = info.properties();
+  }
+
   @VisibleForTesting
   public HadoopCatalogOperations getBaseHadoopCatalogOperations() {
     return hadoopCatalogOperations;
@@ -163,19 +188,6 @@ public boolean dropSchema(NameIdentifier ident, boolean cascade) throws NonEmpty
     }
   }
 
-  @Override
-  public void initialize(
-      Map<String, String> config, CatalogInfo info, HasPropertyMetadata propertiesMetadata)
-      throws RuntimeException {
-    hadoopCatalogOperations.initialize(config, info, propertiesMetadata);
-    catalogUserContext =
-        UserContext.getUserContext(
-            NameIdentifier.of(info.namespace(), info.name()),
-            config,
-            hadoopCatalogOperations.getHadoopConf(),
-            info);
-  }
-
   @Override
   public Fileset alterFileset(NameIdentifier ident, FilesetChange... changes)
       throws NoSuchFilesetException, IllegalArgumentException {
@@ -245,6 +257,29 @@ public void testConnection(
     hadoopCatalogOperations.testConnection(catalogIdent, type, provider, comment, properties);
   }
 
+  @Override
+  public List<PathContext> getPathContext(NameIdentifier filesetIdentifier) {
+    Fileset fileset = loadFileset(filesetIdentifier);
+    String path = fileset.storageLocation();
+    Preconditions.checkState(
+        StringUtils.isNotBlank(path), "The location of fileset should not be empty.");
+
+    Set<String> providers =
+        CredentialUtils.getCredentialProvidersByOrder(
+            () -> fileset.properties(),
+            () -> {
+              Namespace namespace = filesetIdentifier.namespace();
+              NameIdentifier schemaIdentifier =
+                  NameIdentifierUtil.ofSchema(
+                      namespace.level(0), namespace.level(1), namespace.level(2));
+              return loadSchema(schemaIdentifier).properties();
+            },
+            () -> catalogProperties);
+    return providers.stream()
+        .map(provider -> new PathContext(path, provider))
+        .collect(Collectors.toList());
+  }
+
   /**
    * Add the user to the subject so that we can get the last user in the subject. Hadoop catalog
    * uses this method to pass api user from the client side, so that we can get the user in the

diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java b/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java
@@ -32,6 +32,7 @@ public class ErrorMessages {
   public static final String MISSING_NAME = "Missing --name option.";
   public static final String MISSING_GROUP = "Missing --group option.";
   public static final String MISSING_USER = "Missing --user option.";
+  public static final String MISSING_TAG = "Missing --tag option.";
   public static final String METALAKE_EXISTS = "Metalake already exists.";
   public static final String CATALOG_EXISTS = "Catalog already exists.";
   public static final String SCHEMA_EXISTS = "Schema already exists.";