fix

apache · Oct 17, 2024 · b9ef8f0 · b9ef8f0
1 parent 4f00a2f
commit b9ef8f0
Show file tree

Hide file tree

Showing 4 changed files with 50 additions and 35 deletions.
diff --git a/...rc/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCSCatalogIT.java b/...rc/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCSCatalogIT.java
@@ -42,8 +42,21 @@ public class HadoopGCSCatalogIT extends HadoopCatalogIT {
   public static final String BUCKET_NAME = "YOUR_BUCKET";
   public static final String SERVICE_ACCOUNT_FILE = "YOUR_KEY_FILE";
 
+  @Override
+  public void startIntegrationTest() throws Exception {
+    // Just overwrite super, do nothing.
+  }
+
   @BeforeAll
   public void setup() throws IOException {
+    copyBundleJarsToHadoop("gcp-bundle");
+
+    try {
+      super.startIntegrationTest();
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+
     metalakeName = GravitinoITUtils.genRandomName("CatalogFilesetIT_metalake");
     catalogName = GravitinoITUtils.genRandomName("CatalogFilesetIT_catalog");
     schemaName = GravitinoITUtils.genRandomName("CatalogFilesetIT_schema");

diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py
@@ -38,6 +38,8 @@
 
 @unittest.skip("This test require GCS service account key file")
 class TestGvfsWithGCS(TestGvfsWithHDFS):
+    # Before running this test, please set the make sure gcp-bundle-x.jar has been
+    # copy to the $GRAVITINO_HOME/catalogs/hadoop/libs/ directory
     key_file = "your_key_file.json"
     bucket_name = "your_bucket_name"
     metalake_name: str = "TestGvfsWithGCS_metalake" + str(randint(1, 10000))

diff --git a/.../apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java b/.../apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java
@@ -26,11 +26,8 @@
 import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
-import java.util.Objects;
 import org.apache.gravitino.Catalog;
-import org.apache.gravitino.integration.test.util.DownloaderUtils;
 import org.apache.gravitino.integration.test.util.GravitinoITUtils;
-import org.apache.gravitino.integration.test.util.ITUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Assertions;
@@ -55,7 +52,8 @@ public void startIntegrationTest() {
 
   @BeforeAll
   public void startUp() throws Exception {
-    copyGCPJars();
+    // Copy the GCP jars to the gravitino server if in deploy mode.
+    copyBundleJarsToHadoop("gcp-bundle");
     // Need to download jars to gravitino server
     super.startIntegrationTest();
 
@@ -133,37 +131,6 @@ protected String genStorageLocation(String fileset) {
     return String.format("gs://%s/%s", BUCKET_NAME, fileset);
   }
 
-  private static boolean isDeploy() {
-    String mode =
-        System.getProperty(ITUtils.TEST_MODE) == null
-            ? ITUtils.EMBEDDED_TEST_MODE
-            : System.getProperty(ITUtils.TEST_MODE);
-
-    return Objects.equals(mode, ITUtils.DEPLOY_TEST_MODE);
-  }
-
-  private void copyGCPJars() {
-    if (!isDeploy()) {
-      return;
-    }
-
-    String gravitinoHome = System.getenv("GRAVITINO_HOME");
-    String jarName = String.format("gravitino-gcp-bundle-%s.jar", System.getenv("PROJECT_VERSION"));
-    String gcsJars =
-        ITUtils.joinPath(
-            gravitinoHome, "..", "..", "bundles", "gcp-bundle", "build", "libs", jarName);
-    gcsJars = "file://" + gcsJars;
-    try {
-      if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) {
-        String hadoopLibDirs = ITUtils.joinPath(gravitinoHome, "catalogs", "hadoop", "libs");
-        DownloaderUtils.downloadFile(gcsJars, hadoopLibDirs);
-      }
-    } catch (Exception e) {
-      throw new RuntimeException(
-          String.format("Failed to copy the gcs dependency jars: %s", gcsJars), e);
-    }
-  }
-
   @Disabled(
       "GCS does not support append, java.io.IOException: The append operation is not supported")
   public void testAppend() throws IOException {}

diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java
@@ -38,6 +38,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.ArrayUtils;
@@ -386,4 +387,36 @@ protected String readGitCommitIdFromGitFile() {
       return "";
     }
   }
+
+  private static boolean isDeploy() {
+    String mode =
+        System.getProperty(ITUtils.TEST_MODE) == null
+            ? ITUtils.EMBEDDED_TEST_MODE
+            : System.getProperty(ITUtils.TEST_MODE);
+
+    return Objects.equals(mode, ITUtils.DEPLOY_TEST_MODE);
+  }
+
+  protected void copyBundleJarsToHadoop(String bundleName) {
+    if (!isDeploy()) {
+      return;
+    }
+
+    String gravitinoHome = System.getenv("GRAVITINO_HOME");
+    String jarName =
+        String.format("gravitino-%s-%s.jar", bundleName, System.getenv("PROJECT_VERSION"));
+    String gcsJars =
+        ITUtils.joinPath(
+            gravitinoHome, "..", "..", "bundles", bundleName, "build", "libs", jarName);
+    gcsJars = "file://" + gcsJars;
+    try {
+      if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) {
+        String hadoopLibDirs = ITUtils.joinPath(gravitinoHome, "catalogs", "hadoop", "libs");
+        DownloaderUtils.downloadFile(gcsJars, hadoopLibDirs);
+      }
+    } catch (Exception e) {
+      throw new RuntimeException(
+          String.format("Failed to copy the %s dependency jars: %s", bundleName, gcsJars), e);
+    }
+  }
 }