feat: Support GCS filesystem for bytewax engine (feast-dev#3774)

* fix: Support param timeout when persisting Signed-off-by: Hai Nguyen <[email protected]> * fix: fix java integration test Signed-off-by: Hai Nguyen <[email protected]> --------- Signed-off-by: Hai Nguyen <[email protected]>
sailpoint · Oct 23, 2023 · 5a4f24c · 5a4f24c
1 parent 8f323c2
commit 5a4f24c
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 8 deletions.
diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py
@@ -1,9 +1,7 @@
-import logging
 from typing import List
 
 import pyarrow as pa
 import pyarrow.parquet as pq
-import s3fs
 from bytewax.dataflow import Dataflow  # type: ignore
 from bytewax.execution import cluster_main
 from bytewax.inputs import ManualInputConfig
@@ -13,8 +11,6 @@
 from feast import FeatureStore, FeatureView, RepoConfig
 from feast.utils import _convert_arrow_to_proto, _run_pyarrow_field_mapping
 
-logger = logging.getLogger(__name__)
-
 
 class BytewaxMaterializationDataflow:
     def __init__(
@@ -34,9 +30,7 @@ def __init__(
         self._run_dataflow()
 
     def process_path(self, path):
-        fs = s3fs.S3FileSystem()
-        logger.info(f"Processing path {path}")
-        dataset = pq.ParquetDataset(path, filesystem=fs, use_legacy_dataset=False)
+        dataset = pq.ParquetDataset(path, use_legacy_dataset=False)
         batches = []
         for fragment in dataset.fragments:
             for batch in fragment.to_table().to_batches():

diff --git a/setup.py b/setup.py
@@ -91,6 +91,7 @@
     "google-cloud-datastore>=2.1.0,<3",
     "google-cloud-storage>=1.34.0,<3",
     "google-cloud-bigtable>=2.11.0,<3",
+    "gcsfs",
 ]
 
 REDIS_REQUIRED = [
@@ -158,7 +159,7 @@
         "moto",
         "mypy>=0.981,<0.990",
         "avro==1.10.0",
-        "gcsfs>=0.4.0,<=2022.01.0",
+        "gcsfs",
         "urllib3>=1.25.4,<2",
         "psutil==5.9.0",
         "py>=1.11.0",  # https://github.com/pytest-dev/pytest/issues/10420