Switch to modular file system for hdfs (tensorflow#1309)

* Switch to modular file system for hdfs This PR is part of the effort to switch to modular file system for hdfs. When TF_ENABLE_LEGACY_FILESYSTEM=1 is provided, old behavior will be preserved. Signed-off-by: Yong Tang <[email protected]> * Build against tf-nightly Signed-off-by: Yong Tang <[email protected]> * Update tests Signed-off-by: Yong Tang <[email protected]> * Adjust the if else logic, follow review comment Signed-off-by: Yong Tang <[email protected]>
i-ony · Mar 8, 2021 · 8631261 · 8631261
1 parent d463a91
commit 8631261
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 7 deletions.
diff --git a/tensorflow_io/core/plugins/file_system_plugins.cc b/tensorflow_io/core/plugins/file_system_plugins.cc
@@ -15,7 +15,13 @@ limitations under the License.
 
 #include "tensorflow_io/core/plugins/file_system_plugins.h"
 
+#include "absl/strings/ascii.h"
+
 void TF_InitPlugin(TF_FilesystemPluginInfo* info) {
+  const char* enable_legacy_env = getenv("TF_ENABLE_LEGACY_FILESYSTEM");
+  std::string enable_legacy =
+      enable_legacy_env ? absl::AsciiStrToLower(enable_legacy_env) : "";
+
   info->plugin_memory_allocate = tensorflow::io::plugin_memory_allocate;
   info->plugin_memory_free = tensorflow::io::plugin_memory_free;
   info->num_schemes = 7;
@@ -25,8 +31,14 @@ void TF_InitPlugin(TF_FilesystemPluginInfo* info) {
   tensorflow::io::az::ProvideFilesystemSupportFor(&info->ops[0], "az");
   tensorflow::io::http::ProvideFilesystemSupportFor(&info->ops[1], "http");
   tensorflow::io::s3::ProvideFilesystemSupportFor(&info->ops[2], "s3e");
-  tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[3], "hdfse");
-  tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[4], "viewfse");
-  tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[5], "hare");
+  if (enable_legacy == "true" || enable_legacy == "1") {
+    tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[3], "hdfse");
+    tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[4], "viewfse");
+    tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[5], "hare");
+  } else {
+    tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[3], "hdfs");
+    tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[4], "viewfs");
+    tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[5], "har");
+  }
   tensorflow::io::gs::ProvideFilesystemSupportFor(&info->ops[6], "gse");
 }
diff --git a/tensorflow_io/core/python/ops/version_ops.py b/tensorflow_io/core/python/ops/version_ops.py
@@ -14,5 +14,5 @@
 # ==============================================================================
 """version_ops"""
 
-package = "tensorflow>=2.4.0,<2.5.0"
-version = "0.17.0"
+package = "tf-nightly"
+version = "0.18.0"
diff --git a/tests/test_hdfs.py b/tests/test_hdfs.py
@@ -35,8 +35,8 @@ def test_read_file():
     print("ADDRESS: {}".format(address))
 
     body = b"1234567"
-    tf.io.write_file("hdfse://{}:9000/file.txt".format(address), body)
+    tf.io.write_file("hdfs://{}:9000/file.txt".format(address), body)
 
-    content = tf.io.read_file("hdfse://{}:9000/file.txt".format(address))
+    content = tf.io.read_file("hdfs://{}:9000/file.txt".format(address))
     print("CONTENT: {}".format(content))
     assert content == body