From 1d9262642d35f41f58a5ce737396ad4a930a0a81 Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Mon, 16 Dec 2024 18:56:22 +0800 Subject: [PATCH] [#5582] improvement(hadoop3-filesystem): Remove configuration `fs.gvfs.filesystem.providers` from GVFS client. (#5634) ### What changes were proposed in this pull request? Configuration `fs.gvfs.filesystem.providers` is redundant, so we'd better remove this configuation. ### Why are the changes needed? This configuration is redundant. Fix: #5582 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? Existing tests. --- build.gradle.kts | 4 +- bundles/aliyun-bundle/build.gradle.kts | 3 ++ bundles/aws-bundle/build.gradle.kts | 3 ++ bundles/azure-bundle/build.gradle.kts | 3 ++ bundles/gcp-bundle/build.gradle.kts | 3 ++ catalogs/catalog-hadoop/build.gradle.kts | 41 +++++++++++-------- .../catalog/hadoop/fs/FileSystemUtils.java | 4 +- catalogs/hadoop-common/build.gradle.kts | 28 +++++++++++++ .../catalog/hadoop/fs/FileSystemProvider.java | 0 clients/filesystem-hadoop3/build.gradle.kts | 13 ++++-- .../hadoop/GravitinoVirtualFileSystem.java | 28 ++++++++++--- ...avitinoVirtualFileSystemConfiguration.java | 9 ---- .../test/GravitinoVirtualFileSystemABSIT.java | 1 - .../test/GravitinoVirtualFileSystemGCSIT.java | 6 +-- .../test/GravitinoVirtualFileSystemOSSIT.java | 2 - .../test/GravitinoVirtualFileSystemS3IT.java | 2 - docs/how-to-use-gvfs.md | 10 ----- settings.gradle.kts | 1 + 18 files changed, 103 insertions(+), 58 deletions(-) create mode 100644 catalogs/hadoop-common/build.gradle.kts rename catalogs/{catalog-hadoop => hadoop-common}/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java (100%) diff --git a/build.gradle.kts b/build.gradle.kts index 401ccba3dfd..4ebeec80476 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -779,7 +779,7 @@ tasks { !it.name.startsWith("client") && !it.name.startsWith("filesystem") && !it.name.startsWith("spark") && !it.name.startsWith("iceberg") && it.name != "trino-connector" && it.name != "integration-test" && it.name != "bundled-catalog" && !it.name.startsWith("flink") && it.name != "integration-test" && it.name != "hive-metastore-common" && !it.name.startsWith("flink") && - it.name != "gcp-bundle" && it.name != "aliyun-bundle" && it.name != "aws-bundle" && it.name != "azure-bundle" + it.name != "gcp-bundle" && it.name != "aliyun-bundle" && it.name != "aws-bundle" && it.name != "azure-bundle" && it.name != "hadoop-common" ) { from(it.configurations.runtimeClasspath) into("distribution/package/libs") @@ -802,7 +802,7 @@ tasks { it.name != "bundled-catalog" && it.name != "hive-metastore-common" && it.name != "gcp-bundle" && it.name != "aliyun-bundle" && it.name != "aws-bundle" && it.name != "azure-bundle" && - it.name != "docs" + it.name != "hadoop-common" && it.name != "docs" ) { dependsOn("${it.name}:build") from("${it.name}/build/libs") diff --git a/bundles/aliyun-bundle/build.gradle.kts b/bundles/aliyun-bundle/build.gradle.kts index e803c517b38..bc2d21a6851 100644 --- a/bundles/aliyun-bundle/build.gradle.kts +++ b/bundles/aliyun-bundle/build.gradle.kts @@ -29,6 +29,9 @@ dependencies { compileOnly(project(":core")) compileOnly(project(":catalogs:catalog-common")) compileOnly(project(":catalogs:catalog-hadoop")) + compileOnly(project(":catalogs:hadoop-common")) { + exclude("*") + } compileOnly(libs.hadoop3.common) implementation(libs.aliyun.credentials.sdk) diff --git a/bundles/aws-bundle/build.gradle.kts b/bundles/aws-bundle/build.gradle.kts index 0036b5eea96..94c7d1cb2ce 100644 --- a/bundles/aws-bundle/build.gradle.kts +++ b/bundles/aws-bundle/build.gradle.kts @@ -29,6 +29,9 @@ dependencies { compileOnly(project(":core")) compileOnly(project(":catalogs:catalog-common")) compileOnly(project(":catalogs:catalog-hadoop")) + compileOnly(project(":catalogs:hadoop-common")) { + exclude("*") + } compileOnly(libs.hadoop3.common) implementation(libs.aws.iam) diff --git a/bundles/azure-bundle/build.gradle.kts b/bundles/azure-bundle/build.gradle.kts index fa6a68d1af5..8580c672e13 100644 --- a/bundles/azure-bundle/build.gradle.kts +++ b/bundles/azure-bundle/build.gradle.kts @@ -28,6 +28,9 @@ dependencies { compileOnly(project(":api")) compileOnly(project(":core")) compileOnly(project(":catalogs:catalog-hadoop")) + compileOnly(project(":catalogs:hadoop-common")) { + exclude("*") + } compileOnly(libs.hadoop3.common) diff --git a/bundles/gcp-bundle/build.gradle.kts b/bundles/gcp-bundle/build.gradle.kts index 7d679f62752..bae7411c75e 100644 --- a/bundles/gcp-bundle/build.gradle.kts +++ b/bundles/gcp-bundle/build.gradle.kts @@ -29,6 +29,9 @@ dependencies { compileOnly(project(":core")) compileOnly(project(":catalogs:catalog-common")) compileOnly(project(":catalogs:catalog-hadoop")) + compileOnly(project(":catalogs:hadoop-common")) { + exclude("*") + } compileOnly(libs.hadoop3.common) diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index 84488efb0b3..8873b795046 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -32,6 +32,7 @@ dependencies { implementation(project(":core")) { exclude(group = "*") } + implementation(project(":common")) { exclude(group = "*") } @@ -40,7 +41,9 @@ dependencies { exclude(group = "*") } - compileOnly(libs.guava) + implementation(project(":catalogs:hadoop-common")) { + exclude(group = "*") + } implementation(libs.hadoop3.common) { exclude("com.sun.jersey") @@ -54,6 +57,14 @@ dependencies { exclude("com.sun.jersey", "jersey-servlet") } + implementation(libs.hadoop3.client) { + exclude("org.apache.hadoop", "hadoop-mapreduce-client-core") + exclude("org.apache.hadoop", "hadoop-mapreduce-client-jobclient") + exclude("org.apache.hadoop", "hadoop-yarn-api") + exclude("org.apache.hadoop", "hadoop-yarn-client") + exclude("com.squareup.okhttp", "okhttp") + } + implementation(libs.hadoop3.hdfs) { exclude("com.sun.jersey") exclude("javax.servlet", "servlet-api") @@ -63,38 +74,32 @@ dependencies { exclude("io.netty") exclude("org.fusesource.leveldbjni") } - implementation(libs.hadoop3.client) { - exclude("org.apache.hadoop", "hadoop-mapreduce-client-core") - exclude("org.apache.hadoop", "hadoop-mapreduce-client-jobclient") - exclude("org.apache.hadoop", "hadoop-yarn-api") - exclude("org.apache.hadoop", "hadoop-yarn-client") - exclude("com.squareup.okhttp", "okhttp") - } implementation(libs.slf4j.api) - testImplementation(project(":clients:client-java")) - testImplementation(project(":integration-test-common", "testArtifacts")) - testImplementation(project(":server")) - testImplementation(project(":server-common")) + compileOnly(libs.guava) + testImplementation(project(":bundles:aws-bundle")) testImplementation(project(":bundles:gcp-bundle")) testImplementation(project(":bundles:aliyun-bundle")) testImplementation(project(":bundles:azure-bundle")) - - testImplementation(libs.minikdc) - testImplementation(libs.hadoop3.minicluster) + testImplementation(project(":clients:client-java")) + testImplementation(project(":integration-test-common", "testArtifacts")) + testImplementation(project(":server")) + testImplementation(project(":server-common")) testImplementation(libs.bundles.log4j) + testImplementation(libs.hadoop3.gcs) + testImplementation(libs.hadoop3.minicluster) + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.junit.jupiter.params) + testImplementation(libs.minikdc) testImplementation(libs.mockito.core) testImplementation(libs.mockito.inline) testImplementation(libs.mysql.driver) testImplementation(libs.postgresql.driver) - testImplementation(libs.junit.jupiter.api) - testImplementation(libs.junit.jupiter.params) testImplementation(libs.testcontainers) testImplementation(libs.testcontainers.mysql) - testImplementation(libs.hadoop3.gcs) testRuntimeOnly(libs.junit.jupiter.engine) } diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java index 3ed307aa0ab..129a8e88274 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java @@ -63,8 +63,8 @@ public static Map getFileSystemProviders(String file if (resultMap.containsKey(fileSystemProvider.scheme())) { throw new UnsupportedOperationException( String.format( - "File system provider: '%s' with scheme '%s' already exists in the use provider list " - + "Please make sure the file system provider scheme is unique.", + "File system provider: '%s' with scheme '%s' already exists in the provider list," + + "please make sure the file system provider scheme is unique.", fileSystemProvider.getClass().getName(), fileSystemProvider.scheme())); } resultMap.put(fileSystemProvider.scheme(), fileSystemProvider); diff --git a/catalogs/hadoop-common/build.gradle.kts b/catalogs/hadoop-common/build.gradle.kts new file mode 100644 index 00000000000..ab768cb1f11 --- /dev/null +++ b/catalogs/hadoop-common/build.gradle.kts @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + id("java") +} + +// try to avoid adding extra dependencies because it is used by catalogs and connectors. +dependencies { + implementation(libs.commons.lang3) + implementation(libs.hadoop3.common) +} diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java similarity index 100% rename from catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java rename to catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java diff --git a/clients/filesystem-hadoop3/build.gradle.kts b/clients/filesystem-hadoop3/build.gradle.kts index 55c0f59a05d..d24eb4efdf2 100644 --- a/clients/filesystem-hadoop3/build.gradle.kts +++ b/clients/filesystem-hadoop3/build.gradle.kts @@ -26,10 +26,11 @@ plugins { dependencies { compileOnly(project(":clients:client-java-runtime", configuration = "shadow")) compileOnly(libs.hadoop3.common) - implementation(project(":catalogs:catalog-hadoop")) { + + implementation(project(":catalogs:catalog-common")) { exclude(group = "*") } - implementation(project(":catalogs:catalog-common")) { + implementation(project(":catalogs:hadoop-common")) { exclude(group = "*") } @@ -42,16 +43,19 @@ dependencies { testImplementation(project(":server-common")) testImplementation(project(":clients:client-java")) testImplementation(project(":integration-test-common", "testArtifacts")) + testImplementation(project(":catalogs:catalog-hadoop")) testImplementation(project(":bundles:gcp-bundle")) testImplementation(project(":bundles:aliyun-bundle")) testImplementation(project(":bundles:aws-bundle")) testImplementation(project(":bundles:azure-bundle")) + testImplementation(project(":bundles:gcp-bundle")) + testImplementation(libs.awaitility) testImplementation(libs.bundles.jetty) testImplementation(libs.bundles.jersey) testImplementation(libs.bundles.jwt) - testImplementation(libs.testcontainers) testImplementation(libs.guava) + testImplementation(libs.hadoop3.client) testImplementation(libs.hadoop3.common) { exclude("com.sun.jersey") @@ -75,6 +79,8 @@ dependencies { } testImplementation(libs.mysql.driver) testImplementation(libs.postgresql.driver) + testImplementation(libs.testcontainers) + testRuntimeOnly(libs.junit.jupiter.engine) } @@ -99,6 +105,7 @@ tasks.test { dependsOn(":bundles:aws-bundle:jar") dependsOn(":bundles:aliyun-bundle:jar") dependsOn(":bundles:gcp-bundle:jar") + dependsOn(":bundles:azure-bundle:jar") } tasks.javadoc { diff --git a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java index a1e7bb5d55a..e18e376b46c 100644 --- a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java +++ b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java @@ -18,8 +18,6 @@ */ package org.apache.gravitino.filesystem.hadoop; -import static org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.FS_FILESYSTEM_PROVIDERS; - import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.Scheduler; @@ -27,6 +25,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Streams; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.File; import java.io.IOException; @@ -34,6 +33,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.ServiceLoader; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; @@ -46,7 +46,6 @@ import org.apache.gravitino.audit.FilesetDataOperation; import org.apache.gravitino.audit.InternalClientType; import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider; -import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils; import org.apache.gravitino.client.DefaultOAuth2TokenProvider; import org.apache.gravitino.client.GravitinoClient; import org.apache.gravitino.client.KerberosTokenProvider; @@ -135,8 +134,7 @@ public void initialize(URI name, Configuration configuration) throws IOException initializeClient(configuration); // Register the default local and HDFS FileSystemProvider - String fileSystemProviders = configuration.get(FS_FILESYSTEM_PROVIDERS); - fileSystemProvidersMap.putAll(FileSystemUtils.getFileSystemProviders(fileSystemProviders)); + fileSystemProvidersMap.putAll(getFileSystemProviders()); this.workingDirectory = new Path(name); this.uri = URI.create(name.getScheme() + "://" + name.getAuthority()); @@ -618,4 +616,24 @@ public FileSystem getFileSystem() { return fileSystem; } } + + private static Map getFileSystemProviders() { + Map resultMap = Maps.newHashMap(); + ServiceLoader allFileSystemProviders = + ServiceLoader.load(FileSystemProvider.class); + + Streams.stream(allFileSystemProviders.iterator()) + .forEach( + fileSystemProvider -> { + if (resultMap.containsKey(fileSystemProvider.scheme())) { + throw new UnsupportedOperationException( + String.format( + "File system provider: '%s' with scheme '%s' already exists in the provider list, " + + "please make sure the file system provider scheme is unique.", + fileSystemProvider.getClass().getName(), fileSystemProvider.scheme())); + } + resultMap.put(fileSystemProvider.scheme(), fileSystemProvider); + }); + return resultMap; + } } diff --git a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java index 95ce4df2a8f..e2bce734531 100644 --- a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java +++ b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystemConfiguration.java @@ -18,8 +18,6 @@ */ package org.apache.gravitino.filesystem.hadoop; -import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider; - /** Configuration class for Gravitino Virtual File System. */ public class GravitinoVirtualFileSystemConfiguration { @@ -44,13 +42,6 @@ public class GravitinoVirtualFileSystemConfiguration { /** The configuration key for the Gravitino client auth type. */ public static final String FS_GRAVITINO_CLIENT_AUTH_TYPE_KEY = "fs.gravitino.client.authType"; - /** - * File system provider names configuration key. The value is a comma separated list of file - * system provider name which is defined in the service loader. Users can custom their own file - * system by implementing the {@link FileSystemProvider} interface. - */ - public static final String FS_FILESYSTEM_PROVIDERS = "fs.gvfs.filesystem.providers"; - /** The authentication type for simple authentication. */ public static final String SIMPLE_AUTH_TYPE = "simple"; /** The authentication type for oauth2 authentication. */ diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSIT.java index cc16ce920ae..11557417fe5 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSIT.java @@ -95,7 +95,6 @@ public void startUp() throws Exception { conf.set("fs.gravitino.server.uri", serverUri); conf.set("fs.gravitino.client.metalake", metalakeName); - conf.set("fs.gvfs.filesystem.providers", AzureFileSystemProvider.ABS_PROVIDER_NAME); // Pass this configuration to the real file system conf.set(ABSProperties.GRAVITINO_ABS_ACCOUNT_NAME, ABS_ACCOUNT_NAME); conf.set(ABSProperties.GRAVITINO_ABS_ACCOUNT_KEY, ABS_ACCOUNT_KEY); diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java index b66fb34df37..f273708810c 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java @@ -20,7 +20,6 @@ package org.apache.gravitino.filesystem.hadoop.integration.test; import static org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS; -import static org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.FS_FILESYSTEM_PROVIDERS; import com.google.common.collect.Maps; import java.io.IOException; @@ -92,15 +91,14 @@ public void startUp() throws Exception { // Pass this configuration to the real file system conf.set(GCSProperties.GCS_SERVICE_ACCOUNT_JSON_PATH, SERVICE_ACCOUNT_FILE); - conf.set(FS_FILESYSTEM_PROVIDERS, "gcs"); } @AfterAll public void tearDown() throws IOException { Catalog catalog = metalake.loadCatalog(catalogName); catalog.asSchemas().dropSchema(schemaName, true); - metalake.dropCatalog(catalogName); - client.dropMetalake(metalakeName); + metalake.dropCatalog(catalogName, true); + client.dropMetalake(metalakeName, true); if (client != null) { client.close(); diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSIT.java index e2218b5bc89..5cd02ef4ef9 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSIT.java @@ -20,7 +20,6 @@ package org.apache.gravitino.filesystem.hadoop.integration.test; import static org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS; -import static org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.FS_FILESYSTEM_PROVIDERS; import com.google.common.collect.Maps; import java.io.IOException; @@ -100,7 +99,6 @@ public void startUp() throws Exception { conf.set(OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET, OSS_SECRET_KEY); conf.set(OSSProperties.GRAVITINO_OSS_ENDPOINT, OSS_ENDPOINT); conf.set("fs.oss.impl", "org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem"); - conf.set(FS_FILESYSTEM_PROVIDERS, "oss"); } @AfterAll diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3IT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3IT.java index 22c4872884d..4bb6ad38dcd 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3IT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3IT.java @@ -20,7 +20,6 @@ package org.apache.gravitino.filesystem.hadoop.integration.test; import static org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS; -import static org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.FS_FILESYSTEM_PROVIDERS; import com.google.common.collect.Maps; import java.io.IOException; @@ -157,7 +156,6 @@ public void startUp() throws Exception { conf.set(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, accessKey); conf.set(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, secretKey); conf.set(S3Properties.GRAVITINO_S3_ENDPOINT, s3Endpoint); - conf.set(FS_FILESYSTEM_PROVIDERS, "s3"); } @AfterAll diff --git a/docs/how-to-use-gvfs.md b/docs/how-to-use-gvfs.md index 5b79a80f8f6..34835ec8dc8 100644 --- a/docs/how-to-use-gvfs.md +++ b/docs/how-to-use-gvfs.md @@ -73,7 +73,6 @@ Apart from the above properties, to access fileset like S3, GCS, OSS and custom | Configuration item | Description | Default value | Required | Since version | |--------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|--------------------------|------------------| -| `fs.gvfs.filesystem.providers` | The file system providers to add. Set it to `s3` if it's a S3 fileset, or a comma separated string that contains `s3` like `gs,s3` to support multiple kinds of fileset including `s3`.| (none) | Yes if it's a S3 fileset.| 0.7.0-incubating | | `s3-endpoint` | The endpoint of the AWS S3. | (none) | Yes if it's a S3 fileset.| 0.7.0-incubating | | `s3-access-key-id` | The access key of the AWS S3. | (none) | Yes if it's a S3 fileset.| 0.7.0-incubating | | `s3-secret-access-key` | The secret key of the AWS S3. | (none) | Yes if it's a S3 fileset.| 0.7.0-incubating | @@ -85,7 +84,6 @@ At the same time, you need to place the corresponding bundle jar [`gravitino-aws | Configuration item | Description | Default value | Required | Since version | |--------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|---------------------------|------------------| -| `fs.gvfs.filesystem.providers` | The file system providers to add. Set it to `gs` if it's a GCS fileset, or a comma separated string that contains `gs` like `gs,s3` to support multiple kinds of fileset including `gs`. | (none) | Yes if it's a GCS fileset.| 0.7.0-incubating | | `gcs-service-account-file` | The path of GCS service account JSON file. | (none) | Yes if it's a GCS fileset.| 0.7.0-incubating | In the meantime, you need to place the corresponding bundle jar [`gravitino-gcp-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gcp-bundle/) in the Hadoop environment(typically located in `${HADOOP_HOME}/share/hadoop/common/lib/`). @@ -95,7 +93,6 @@ In the meantime, you need to place the corresponding bundle jar [`gravitino-gcp- | Configuration item | Description | Default value | Required | Since version | |---------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|---------------------------|------------------| -| `fs.gvfs.filesystem.providers` | The file system providers to add. Set it to `oss` if it's a OSS fileset, or a comma separated string that contains `oss` like `oss,gs,s3` to support multiple kinds of fileset including `oss`.| (none) | Yes if it's a OSS fileset.| 0.7.0-incubating | | `oss-endpoint` | The endpoint of the Aliyun OSS. | (none) | Yes if it's a OSS fileset.| 0.7.0-incubating | | `oss-access-key-id` | The access key of the Aliyun OSS. | (none) | Yes if it's a OSS fileset.| 0.7.0-incubating | | `oss-secret-access-key` | The secret key of the Aliyun OSS. | (none) | Yes if it's a OSS fileset.| 0.7.0-incubating | @@ -106,7 +103,6 @@ In the meantime, you need to place the corresponding bundle jar [`gravitino-aliy | Configuration item | Description | Default value | Required | Since version | |--------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|-------------------------------------------|------------------| -| `fs.gvfs.filesystem.providers` | The file system providers to add. Set it to `abs` if it's a Azure Blob Storage fileset, or a comma separated string that contains `abs` like `oss,abs,s3` to support multiple kinds of fileset including `abs`. | (none) | Yes | 0.8.0-incubating | | `abs-account-name` | The account name of Azure Blob Storage. | (none) | Yes if it's a Azure Blob Storage fileset. | 0.8.0-incubating | | `abs-account-key` | The account key of Azure Blob Storage. | (none) | Yes if it's a Azure Blob Storage fileset. | 0.8.0-incubating | @@ -118,7 +114,6 @@ So, if you want to access the custom fileset through GVFS, you need to configure | Configuration item | Description | Default value | Required | Since version | |--------------------------------|---------------------------------------------------------------------------------------------------------|---------------|----------|------------------| -| `fs.gvfs.filesystem.providers` | The file system providers. please set it to the value of `YourCustomFileSystemProvider#name` | (none) | Yes | 0.7.0-incubating | | `your-custom-properties` | The properties will be used to create a FileSystem instance in `CustomFileSystemProvider#getFileSystem` | (none) | No | - | You can configure these properties in two ways: @@ -133,7 +128,6 @@ You can configure these properties in two ways: conf.set("fs.gravitino.client.metalake","test_metalake"); // Optional. It's only for S3 catalog. For GCS and OSS catalog, you should set the corresponding properties. - conf.set("fs.gvfs.filesystem.providers", "s3"); conf.set("s3-endpoint", "http://localhost:9000"); conf.set("s3-access-key-id", "minio"); conf.set("s3-secret-access-key", "minio123"); @@ -171,10 +165,6 @@ For example if you want to access the S3 fileset, you need to place the S3 bundl - - fs.gvfs.filesystem.providers - s3 - s3-endpoint http://localhost:9000 diff --git a/settings.gradle.kts b/settings.gradle.kts index 75dd967c4e9..5776d34fac7 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -81,3 +81,4 @@ include(":bundles:aws-bundle") include(":bundles:gcp-bundle") include(":bundles:aliyun-bundle") include("bundles:azure-bundle") +include("catalogs:hadoop-common")