Merge branch 'main' into bak-add-impl-for-get-file-location

apache · Sep 9, 2024 · edd58c6 · edd58c6
2 parents f3227ad + 60f4beb
commit edd58c6
Show file tree

Hide file tree

Showing 124 changed files with 7,173 additions and 2,932 deletions.
diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
@@ -61,12 +61,15 @@ jobs:
         # Integration test for AMD64 architecture
         architecture: [linux/amd64]
         java-version: [ 8, 11, 17 ]
+        backend: [ h2, mysql, postgresql ]
         test-mode: [ embedded, deploy ]
-        include:
+        exclude:
           - test-mode: 'embedded'
-            backend: 'h2'
-          - test-mode: 'deploy'
             backend: 'mysql'
+          - test-mode: 'embedded'
+            backend: 'postgresql'
+          - test-mode: 'deploy'
+            backend: 'h2'
 
     env:
       PLATFORM: ${{ matrix.architecture }}
@@ -132,11 +135,14 @@ jobs:
         architecture: [ linux/amd64 ]
         java-version: [ 17 ]
         test-mode: [ embedded, deploy ]
-        include:
+        backend: [ h2, mysql, postgresql ]
+        exclude:
           - test-mode: 'embedded'
-            backend: 'h2'
-          - test-mode: 'deploy'
             backend: 'mysql'
+          - test-mode: 'embedded'
+            backend: 'postgresql'
+          - test-mode: 'deploy'
+            backend: 'h2'
 
     env:
       PLATFORM: ${{ matrix.architecture }}

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -45,6 +45,7 @@ jobs:
               - trino-connector/**
               - web/**
               - docs/open-api/**
+              - docs/build.gradle.kts
               - build.gradle.kts
               - gradle.properties
               - gradlew

diff --git a/...er/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java b/...er/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java
@@ -194,8 +194,7 @@ public Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner n
           UserEntity.builder()
               .withId(1L)
               .withName(newOwner.name())
-              .withRoleNames(Collections.emptyList())
-              .withRoleIds(Collections.emptyList())
+              .withRoles(Collections.emptyList())
               .withAuditInfo(auditInfo)
               .build();
       onUserAdded(userEntity);
@@ -204,8 +203,7 @@ public Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner n
           GroupEntity.builder()
               .withId(1L)
               .withName(newOwner.name())
-              .withRoleNames(Collections.emptyList())
-              .withRoleIds(Collections.emptyList())
+              .withRoles(Collections.emptyList())
               .withAuditInfo(auditInfo)
               .build();
       onGroupAdded(groupEntity);

diff --git a/...rc/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java b/...rc/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java
@@ -483,8 +483,7 @@ public void testOnGrantedRolesToUser() {
         UserEntity.builder()
             .withId(1L)
             .withName(userName1)
-            .withRoleNames(Collections.emptyList())
-            .withRoleIds(Collections.emptyList())
+            .withRoles(Collections.emptyList())
             .withAuditInfo(auditInfo)
             .build();
     Assertions.assertTrue(
@@ -502,8 +501,7 @@ public void testOnGrantedRolesToUser() {
         UserEntity.builder()
             .withId(1L)
             .withName(userName2)
-            .withRoleNames(Collections.emptyList())
-            .withRoleIds(Collections.emptyList())
+            .withRoles(Collections.emptyList())
             .withAuditInfo(auditInfo)
             .build();
     Assertions.assertTrue(
@@ -525,8 +523,7 @@ public void testOnRevokedRolesFromUser() {
         UserEntity.builder()
             .withId(1L)
             .withName(userName1)
-            .withRoleNames(Collections.emptyList())
-            .withRoleIds(Collections.emptyList())
+            .withRoles(Collections.emptyList())
             .withAuditInfo(auditInfo)
             .build();
     Assertions.assertTrue(
@@ -555,8 +552,7 @@ public void testOnGrantedRolesToGroup() {
         GroupEntity.builder()
             .withId(1L)
             .withName(groupName1)
-            .withRoleNames(Collections.emptyList())
-            .withRoleIds(Collections.emptyList())
+            .withRoles(Collections.emptyList())
             .withAuditInfo(auditInfo)
             .build();
     Assertions.assertTrue(
@@ -574,8 +570,7 @@ public void testOnGrantedRolesToGroup() {
         GroupEntity.builder()
             .withId(1L)
             .withName(groupName2)
-            .withRoleNames(Collections.emptyList())
-            .withRoleIds(Collections.emptyList())
+            .withRoles(Collections.emptyList())
             .withAuditInfo(auditInfo)
             .build();
     Assertions.assertTrue(
@@ -598,8 +593,7 @@ public void testOnRevokedRolesFromGroup() {
         GroupEntity.builder()
             .withId(1L)
             .withName(groupName1)
-            .withRoleNames(Collections.emptyList())
-            .withRoleIds(Collections.emptyList())
+            .withRoles(Collections.emptyList())
             .withAuditInfo(auditInfo)
             .build();
     Assertions.assertTrue(
@@ -680,8 +674,7 @@ public void testCreateUser() {
             .withId(0L)
             .withName(currentFunName())
             .withAuditInfo(auditInfo)
-            .withRoleIds(null)
-            .withRoleNames(null)
+            .withRoles(Collections.emptyList())
             .build();
     Assertions.assertTrue(rangerAuthPlugin.onUserAdded(user));
     Assertions.assertTrue(rangerAuthPlugin.onUserAcquired(user));
@@ -696,8 +689,7 @@ public void testCreateGroup() {
             .withId(0L)
             .withName(currentFunName())
             .withAuditInfo(auditInfo)
-            .withRoleIds(null)
-            .withRoleNames(null)
+            .withRoles(Collections.emptyList())
             .build();
 
     Assertions.assertTrue(rangerAuthPlugin.onGroupAdded(group));
@@ -769,8 +761,7 @@ public void testCombinationOperation() {
         UserEntity.builder()
             .withId(1L)
             .withName(userName1)
-            .withRoleNames(Collections.emptyList())
-            .withRoleIds(Collections.emptyList())
+            .withRoles(Collections.emptyList())
             .withAuditInfo(auditInfo)
             .build();
     Assertions.assertTrue(
@@ -786,8 +777,7 @@ public void testCombinationOperation() {
         UserEntity.builder()
             .withId(1L)
             .withName(userName2)
-            .withRoleNames(Collections.emptyList())
-            .withRoleIds(Collections.emptyList())
+            .withRoles(Collections.emptyList())
             .withAuditInfo(auditInfo)
             .build();
     Assertions.assertTrue(
@@ -800,8 +790,7 @@ public void testCombinationOperation() {
         UserEntity.builder()
             .withId(1L)
             .withName(userName3)
-            .withRoleNames(Collections.emptyList())
-            .withRoleIds(Collections.emptyList())
+            .withRoles(Collections.emptyList())
             .withAuditInfo(auditInfo)
             .build();
     Assertions.assertTrue(
@@ -833,8 +822,7 @@ public void testCombinationOperation() {
         GroupEntity.builder()
             .withId(1L)
             .withName(groupName1)
-            .withRoleNames(Collections.emptyList())
-            .withRoleIds(Collections.emptyList())
+            .withRoles(Collections.emptyList())
             .withAuditInfo(auditInfo)
             .build();
     Assertions.assertTrue(
@@ -852,8 +840,7 @@ public void testCombinationOperation() {
         GroupEntity.builder()
             .withId(1L)
             .withName(groupName2)
-            .withRoleNames(Collections.emptyList())
-            .withRoleIds(Collections.emptyList())
+            .withRoles(Collections.emptyList())
             .withAuditInfo(auditInfo)
             .build();
     Assertions.assertTrue(
@@ -871,8 +858,7 @@ public void testCombinationOperation() {
         GroupEntity.builder()
             .withId(1L)
             .withName(groupName3)
-            .withRoleNames(Collections.emptyList())
-            .withRoleIds(Collections.emptyList())
+            .withRoles(Collections.emptyList())
             .withAuditInfo(auditInfo)
             .build();
     Assertions.assertTrue(

diff --git a/build.gradle.kts b/build.gradle.kts
@@ -431,9 +431,9 @@ subprojects {
     reports.html.outputLocation.set(file("${rootProject.projectDir}/build/reports/"))
     val skipTests = project.hasProperty("skipTests")
     if (!skipTests) {
-      jvmArgs = listOf("-Xmx4G")
+      val extraArgs = project.property("extraJvmArgs") as List<String>
+      jvmArgs = listOf("-Xmx4G") + extraArgs
       useJUnitPlatform()
-      jvmArgs(project.property("extraJvmArgs") as List<*>)
       finalizedBy(tasks.getByName("jacocoTestReport"))
     }
   }

diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts
@@ -79,6 +79,7 @@ dependencies {
   testImplementation(libs.mockito.core)
   testImplementation(libs.mockito.inline)
   testImplementation(libs.mysql.driver)
+  testImplementation(libs.postgresql.driver)
   testImplementation(libs.junit.jupiter.api)
   testImplementation(libs.junit.jupiter.params)
   testImplementation(libs.testcontainers)

diff --git a/...log-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/...log-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java
@@ -69,7 +69,6 @@ public class HadoopCatalogOperations implements CatalogOperations, SupportsSchem
 
   private static final String SCHEMA_DOES_NOT_EXIST_MSG = "Schema %s does not exist";
   private static final String FILESET_DOES_NOT_EXIST_MSG = "Fileset %s does not exist";
-  private static final String SLASH = "/";
 
   private static final Logger LOG = LoggerFactory.getLogger(HadoopCatalogOperations.class);
 
@@ -358,29 +357,7 @@ public boolean dropFileset(NameIdentifier ident) {
   @Override
   public String getFileLocation(NameIdentifier ident, String subPath)
       throws NoSuchFilesetException {
-    // TODO we need move some check logics in the Hadoop / Python GVFS to here.
-    Preconditions.checkArgument(subPath != null, "subPath must not be null");
-    String processedSubPath;
-    if (!subPath.trim().isEmpty() && !subPath.trim().startsWith(SLASH)) {
-      processedSubPath = SLASH + subPath.trim();
-    } else {
-      processedSubPath = subPath.trim();
-    }
-
-    Fileset fileset = loadFileset(ident);
-
-    String fileLocation;
-    // subPath cannot be null, so we only need check if it is blank
-    if (StringUtils.isBlank(processedSubPath)) {
-      fileLocation = fileset.storageLocation();
-    } else {
-      String storageLocation =
-          fileset.storageLocation().endsWith(SLASH)
-              ? fileset.storageLocation().substring(0, fileset.storageLocation().length() - 1)
-              : fileset.storageLocation();
-      fileLocation = String.format("%s%s", storageLocation, processedSubPath);
-    }
-    return fileLocation;
+    throw new UnsupportedOperationException("Not implemented");
   }
 
   @Override

diff --git a/...doop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java b/...doop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java
@@ -221,7 +221,7 @@ public Fileset loadFileset(NameIdentifier ident) throws NoSuchFilesetException {
   @Override
   public String getFileLocation(NameIdentifier ident, String subPath)
       throws NoSuchFilesetException {
-    return hadoopCatalogOperations.getFileLocation(ident, subPath);
+    throw new UnsupportedOperationException("Not implemented");
   }
 
   @Override

diff --git a/...hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java b/...hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java
@@ -762,46 +762,6 @@ public void testTestConnection() {
                 ImmutableMap.of()));
   }
 
-  @Test
-  public void testGetFileLocation() throws IOException {
-    String schemaName = "schema1024";
-    String comment = "comment1024";
-    String schemaPath = TEST_ROOT_PATH + "/" + schemaName;
-    createSchema(schemaName, comment, null, schemaPath);
-
-    String catalogName = "c1";
-    String name = "fileset1024";
-    String storageLocation = TEST_ROOT_PATH + "/" + catalogName + "/" + schemaName + "/" + name;
-    Fileset fileset =
-        createFileset(name, schemaName, comment, Fileset.Type.MANAGED, null, storageLocation);
-
-    try (SecureHadoopCatalogOperations ops = new SecureHadoopCatalogOperations(store)) {
-      ops.initialize(Maps.newHashMap(), randomCatalogInfo(), HADOOP_PROPERTIES_METADATA);
-      NameIdentifier filesetIdent = NameIdentifier.of("m1", "c1", schemaName, name);
-      // test sub path starts with "/"
-      String subPath1 = "/test/test.parquet";
-      String fileLocation1 = ops.getFileLocation(filesetIdent, subPath1);
-      Assertions.assertEquals(
-          String.format("%s%s", fileset.storageLocation(), subPath1), fileLocation1);
-
-      // test sub path not starts with "/"
-      String subPath2 = "test/test.parquet";
-      String fileLocation2 = ops.getFileLocation(filesetIdent, subPath2);
-      Assertions.assertEquals(
-          String.format("%s/%s", fileset.storageLocation(), subPath2), fileLocation2);
-
-      // test sub path is null
-      String subPath3 = null;
-      Assertions.assertThrows(
-          IllegalArgumentException.class, () -> ops.getFileLocation(filesetIdent, subPath3));
-
-      // test sub path is blank but not null
-      String subPath4 = "";
-      String fileLocation3 = ops.getFileLocation(filesetIdent, subPath4);
-      Assertions.assertEquals(fileset.storageLocation(), fileLocation3);
-    }
-  }
-
   private static Stream<Arguments> locationArguments() {
     return Stream.of(
         // Honor the catalog location

diff --git a/catalogs/catalog-hive/build.gradle.kts b/catalogs/catalog-hive/build.gradle.kts
@@ -132,6 +132,7 @@ dependencies {
   testImplementation(libs.junit.jupiter.api)
   testImplementation(libs.mockito.core)
   testImplementation(libs.mysql.driver)
+  testImplementation(libs.postgresql.driver)
 
   testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion") {
     exclude("org.apache.hadoop")

diff --git a/catalogs/catalog-jdbc-doris/build.gradle.kts b/catalogs/catalog-jdbc-doris/build.gradle.kts
@@ -52,6 +52,7 @@ dependencies {
   testImplementation(libs.junit.jupiter.api)
   testImplementation(libs.junit.jupiter.params)
   testImplementation(libs.mysql.driver)
+  testImplementation(libs.postgresql.driver)
   testImplementation(libs.testcontainers)
   testImplementation(libs.testcontainers.mysql)
 

diff --git a/catalogs/catalog-jdbc-mysql/build.gradle.kts b/catalogs/catalog-jdbc-mysql/build.gradle.kts
@@ -55,6 +55,7 @@ dependencies {
   testImplementation(libs.junit.jupiter.api)
   testImplementation(libs.junit.jupiter.params)
   testImplementation(libs.mysql.driver)
+  testImplementation(libs.postgresql.driver)
   testImplementation(libs.testcontainers)
   testImplementation(libs.testcontainers.mysql)
 

diff --git a/catalogs/catalog-kafka/build.gradle.kts b/catalogs/catalog-kafka/build.gradle.kts
@@ -51,6 +51,7 @@ dependencies {
   testImplementation(libs.mockito.core)
   testImplementation(libs.mockito.inline)
   testImplementation(libs.mysql.driver)
+  testImplementation(libs.postgresql.driver)
   testImplementation(libs.testcontainers)
   testImplementation(libs.testcontainers.mysql)
 

diff --git a/catalogs/catalog-lakehouse-paimon/build.gradle.kts b/catalogs/catalog-lakehouse-paimon/build.gradle.kts
@@ -78,6 +78,7 @@ dependencies {
   testImplementation(libs.slf4j.api)
   testImplementation(libs.junit.jupiter.api)
   testImplementation(libs.mysql.driver)
+  testImplementation(libs.postgresql.driver)
   testImplementation(libs.bundles.log4j)
   testImplementation(libs.junit.jupiter.params)
   testImplementation(libs.testcontainers)