feat(hadoop_filesystem): support gravitino gvfs

tensorflow · Apr 1, 2024 · 008999f · 008999f
1 parent 4d70341
commit 008999f
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 8 deletions.
diff --git a/WORKSPACE b/WORKSPACE
@@ -878,23 +878,42 @@ http_archive(
 )
 
 # Needed for llvm_toolchain and Golang
+# http_archive(
+#     name = "com_grail_bazel_toolchain",
+#     sha256 = "9e6065ded4b7453143e1586d6819729a63cd233114b72bf85ff3435367b02c90",
+#     strip_prefix = "bazel-toolchain-edd07e96a2ecaa131af9234d6582875d980c0ac7",
+#     urls = [
+#         "https://storage.googleapis.com/mirror.tensorflow.org/github.com/grailbio/bazel-toolchain/archive/edd07e96a2ecaa131af9234d6582875d980c0ac7.tar.gz",
+#         "https://github.com/grailbio/bazel-toolchain/archive/edd07e96a2ecaa131af9234d6582875d980c0ac7.tar.gz",
+#     ],
+# )
+
+# load("@com_grail_bazel_toolchain//toolchain:rules.bzl", "llvm_toolchain")
+
+# llvm_toolchain(
+#     name = "llvm_toolchain",
+#     llvm_version = "9.0.0",
+# )
+
 http_archive(
     name = "com_grail_bazel_toolchain",
-    sha256 = "9e6065ded4b7453143e1586d6819729a63cd233114b72bf85ff3435367b02c90",
-    strip_prefix = "bazel-toolchain-edd07e96a2ecaa131af9234d6582875d980c0ac7",
-    urls = [
-        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/grailbio/bazel-toolchain/archive/edd07e96a2ecaa131af9234d6582875d980c0ac7.tar.gz",
-        "https://github.com/grailbio/bazel-toolchain/archive/edd07e96a2ecaa131af9234d6582875d980c0ac7.tar.gz",
-    ],
+    urls = ["https://github.com/bazel-contrib/toolchains_llvm/archive/refs/tags/0.9.tar.gz"],
+    sha256 = "b2d168315dd0785f170b2b306b86e577c36e812b8f8b05568f9403141f2c24dd",
+    strip_prefix = "toolchains_llvm-0.9",
 )
 
-load("@com_grail_bazel_toolchain//toolchain:rules.bzl", "llvm_toolchain")
+load("@com_grail_bazel_toolchain//toolchain:deps.bzl", "bazel_toolchain_dependencies")
+
+bazel_toolchain_dependencies()
 
+load("@com_grail_bazel_toolchain//toolchain:rules.bzl", "llvm_toolchain")
 llvm_toolchain(
     name = "llvm_toolchain",
     llvm_version = "9.0.0",
 )
 
+
+
 # Golang related ruls, consider removal after switching to C++/C client for prometheus
 http_archive(
     name = "io_bazel_rules_go",

diff --git a/tensorflow_io/core/filesystems/filesystem_plugins.cc b/tensorflow_io/core/filesystems/filesystem_plugins.cc
@@ -29,10 +29,11 @@ limitations under the License.
 TFIO_PLUGIN_EXPORT void TF_InitPlugin(TF_FilesystemPluginInfo* info) {
   info->plugin_memory_allocate = tensorflow::io::plugin_memory_allocate;
   info->plugin_memory_free = tensorflow::io::plugin_memory_free;
-  info->num_schemes = 7;
+
 #if !defined(_MSC_VER)
   info->num_schemes = 8;
 #endif
+
   info->ops = static_cast<TF_FilesystemPluginOps*>(
       tensorflow::io::plugin_memory_allocate(info->num_schemes *
                                              sizeof(info->ops[0])));
@@ -43,7 +44,11 @@ TFIO_PLUGIN_EXPORT void TF_InitPlugin(TF_FilesystemPluginInfo* info) {
   tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[4], "hdfs");
   tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[5], "viewfs");
   tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[6], "har");
+<<<<<<< HEAD
 #if !defined(_MSC_VER)
   tensorflow::io::oss::ProvideFilesystemSupportFor(&info->ops[7], "oss");
 #endif
+=======
+  tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[7], "gvfs");
+>>>>>>> feat(gvfs): support gvfs
 }
diff --git a/tensorflow_io/core/filesystems/hdfs/hadoop_filesystem.cc b/tensorflow_io/core/filesystems/hdfs/hadoop_filesystem.cc
@@ -541,6 +541,11 @@ hdfsFS Connect(tf_hdfs_filesystem::HadoopFileSystemImplementation* hadoop_file,
     std::string path_har = path;
     SplitArchiveNameAndPath(&path_har, &namenode, status);
     if (TF_GetCode(status) != TF_OK) return nullptr;
+  } else if (scheme == "gvfs") {
+    std::string dfsPath = path;
+    size_t pos = dfsPath.find(hdfs_path);
+    dfsPath.replace(pos, dfsPath.length(), "");
+    namenode = dfsPath + "/";
   } else {
     if (namenode.empty()) {
       namenode = "default";