feat: Support GCS filesystem for bytewax engine (feast-dev#3774)

* fix: Support param timeout when persisting Signed-off-by: Hai Nguyen <[email protected]> * fix: fix java integration test Signed-off-by: Hai Nguyen <[email protected]> --------- Signed-off-by: Hai Nguyen <[email protected]>
tokoko · Oct 20, 2023 · fb6b807 · fb6b807
1 parent f05a6e7
commit fb6b807
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 4 deletions.
diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py
@@ -2,7 +2,6 @@
 
 import pyarrow as pa
 import pyarrow.parquet as pq
-import s3fs
 from bytewax.dataflow import Dataflow  # type: ignore
 from bytewax.execution import cluster_main
 from bytewax.inputs import ManualInputConfig, distribute
@@ -29,8 +28,7 @@ def __init__(
         self._run_dataflow()
 
     def process_path(self, path):
-        fs = s3fs.S3FileSystem()
-        dataset = pq.ParquetDataset(path, filesystem=fs, use_legacy_dataset=False)
+        dataset = pq.ParquetDataset(path, use_legacy_dataset=False)
         batches = []
         for fragment in dataset.fragments:
             for batch in fragment.to_table().to_batches():

diff --git a/setup.py b/setup.py
@@ -91,6 +91,7 @@
     "google-cloud-datastore>=2.1.0,<3",
     "google-cloud-storage>=1.34.0,<3",
     "google-cloud-bigtable>=2.11.0,<3",
+    "gcsfs",
 ]
 
 REDIS_REQUIRED = [
@@ -158,7 +159,7 @@
         "moto",
         "mypy>=0.981,<0.990",
         "avro==1.10.0",
-        "gcsfs>=0.4.0,<=2022.01.0",
+        "gcsfs",
         "urllib3>=1.25.4,<2",
         "psutil==5.9.0",
         "py>=1.11.0",  # https://github.com/pytest-dev/pytest/issues/10420