From b9ef8f0afb3730a458aaab848b93b6c070773db4 Mon Sep 17 00:00:00 2001 From: yuqi Date: Thu, 17 Oct 2024 21:42:23 +0800 Subject: [PATCH] fix --- .../integration/test/HadoopGCSCatalogIT.java | 13 +++++++ .../tests/integration/test_gvfs_with_gcs.py | 2 + .../test/GravitinoVirtualFileSystemGCSIT.java | 37 +------------------ .../integration/test/util/BaseIT.java | 33 +++++++++++++++++ 4 files changed, 50 insertions(+), 35 deletions(-) diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCSCatalogIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCSCatalogIT.java index 74ae2a77cdb..cca13b77047 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCSCatalogIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCSCatalogIT.java @@ -42,8 +42,21 @@ public class HadoopGCSCatalogIT extends HadoopCatalogIT { public static final String BUCKET_NAME = "YOUR_BUCKET"; public static final String SERVICE_ACCOUNT_FILE = "YOUR_KEY_FILE"; + @Override + public void startIntegrationTest() throws Exception { + // Just overwrite super, do nothing. + } + @BeforeAll public void setup() throws IOException { + copyBundleJarsToHadoop("gcp-bundle"); + + try { + super.startIntegrationTest(); + } catch (Exception e) { + throw new RuntimeException(e); + } + metalakeName = GravitinoITUtils.genRandomName("CatalogFilesetIT_metalake"); catalogName = GravitinoITUtils.genRandomName("CatalogFilesetIT_catalog"); schemaName = GravitinoITUtils.genRandomName("CatalogFilesetIT_schema"); diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py index cb88194951a..13c316c8496 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_gcs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs.py @@ -38,6 +38,8 @@ @unittest.skip("This test require GCS service account key file") class TestGvfsWithGCS(TestGvfsWithHDFS): + # Before running this test, please set the make sure gcp-bundle-x.jar has been + # copy to the $GRAVITINO_HOME/catalogs/hadoop/libs/ directory key_file = "your_key_file.json" bucket_name = "your_bucket_name" metalake_name: str = "TestGvfsWithGCS_metalake" + str(randint(1, 10000)) diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java index a42d1c4b7b3..312236fe5da 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java @@ -26,11 +26,8 @@ import java.io.IOException; import java.util.Collections; import java.util.Map; -import java.util.Objects; import org.apache.gravitino.Catalog; -import org.apache.gravitino.integration.test.util.DownloaderUtils; import org.apache.gravitino.integration.test.util.GravitinoITUtils; -import org.apache.gravitino.integration.test.util.ITUtils; import org.apache.hadoop.conf.Configuration; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; @@ -55,7 +52,8 @@ public void startIntegrationTest() { @BeforeAll public void startUp() throws Exception { - copyGCPJars(); + // Copy the GCP jars to the gravitino server if in deploy mode. + copyBundleJarsToHadoop("gcp-bundle"); // Need to download jars to gravitino server super.startIntegrationTest(); @@ -133,37 +131,6 @@ protected String genStorageLocation(String fileset) { return String.format("gs://%s/%s", BUCKET_NAME, fileset); } - private static boolean isDeploy() { - String mode = - System.getProperty(ITUtils.TEST_MODE) == null - ? ITUtils.EMBEDDED_TEST_MODE - : System.getProperty(ITUtils.TEST_MODE); - - return Objects.equals(mode, ITUtils.DEPLOY_TEST_MODE); - } - - private void copyGCPJars() { - if (!isDeploy()) { - return; - } - - String gravitinoHome = System.getenv("GRAVITINO_HOME"); - String jarName = String.format("gravitino-gcp-bundle-%s.jar", System.getenv("PROJECT_VERSION")); - String gcsJars = - ITUtils.joinPath( - gravitinoHome, "..", "..", "bundles", "gcp-bundle", "build", "libs", jarName); - gcsJars = "file://" + gcsJars; - try { - if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) { - String hadoopLibDirs = ITUtils.joinPath(gravitinoHome, "catalogs", "hadoop", "libs"); - DownloaderUtils.downloadFile(gcsJars, hadoopLibDirs); - } - } catch (Exception e) { - throw new RuntimeException( - String.format("Failed to copy the gcs dependency jars: %s", gcsJars), e); - } - } - @Disabled( "GCS does not support append, java.io.IOException: The append operation is not supported") public void testAppend() throws IOException {} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java index e8f688f96ea..8bbb5a3b23f 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java @@ -38,6 +38,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.TimeUnit; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.ArrayUtils; @@ -386,4 +387,36 @@ protected String readGitCommitIdFromGitFile() { return ""; } } + + private static boolean isDeploy() { + String mode = + System.getProperty(ITUtils.TEST_MODE) == null + ? ITUtils.EMBEDDED_TEST_MODE + : System.getProperty(ITUtils.TEST_MODE); + + return Objects.equals(mode, ITUtils.DEPLOY_TEST_MODE); + } + + protected void copyBundleJarsToHadoop(String bundleName) { + if (!isDeploy()) { + return; + } + + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + String jarName = + String.format("gravitino-%s-%s.jar", bundleName, System.getenv("PROJECT_VERSION")); + String gcsJars = + ITUtils.joinPath( + gravitinoHome, "..", "..", "bundles", bundleName, "build", "libs", jarName); + gcsJars = "file://" + gcsJars; + try { + if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) { + String hadoopLibDirs = ITUtils.joinPath(gravitinoHome, "catalogs", "hadoop", "libs"); + DownloaderUtils.downloadFile(gcsJars, hadoopLibDirs); + } + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to copy the %s dependency jars: %s", bundleName, gcsJars), e); + } + } }