Skip to content

Commit

Permalink
Remove _create_write_task method
Browse files Browse the repository at this point in the history
  • Loading branch information
RobbeSneyders committed Mar 6, 2024
1 parent 6eff97e commit 885dae5
Showing 1 changed file with 0 additions and 30 deletions.
30 changes: 0 additions & 30 deletions src/fondant/component/data_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -235,33 +235,3 @@ def _write_dataframe(self, dataframe: dd.DataFrame) -> dd.core.Scalar:
for future in as_completed(futures):
future.result()
future.release()

@staticmethod
def _create_write_task(
dataframe: dd.DataFrame,
*,
location: str,
schema: t.Dict[str, str],
) -> dd.core.Scalar:
"""
Creates a delayed Dask task to upload the given DataFrame to the remote storage location
specified in the manifest.
Args:
dataframe: The DataFrame to be uploaded.
location: the location to write the subset to
schema: the schema of the dataframe to write
Returns:
A delayed Dask task that uploads the DataFrame to the remote storage location when
executed.
"""
write_task = dd.to_parquet(
dataframe,
location,
schema=schema,
overwrite=False,
compute=False,
)
logging.info(f"Creating write task for: {location}")
return write_task

0 comments on commit 885dae5

Please sign in to comment.