Skip to content

Commit

Permalink
Temporarily disable the deletion of the dask dataframe (#3814)
Browse files Browse the repository at this point in the history
temporarily disable the deletion of the dask dataframe

Authors:
  - Joseph Nke (https://github.com/jnke2016)
  - Naim (https://github.com/naimnv)

Approvers:
  - Brad Rees (https://github.com/BradReesWork)
  - Rick Ratzel (https://github.com/rlratzel)

URL: #3814
  • Loading branch information
jnke2016 authored Sep 28, 2023
1 parent f0d6333 commit f57119b
Showing 1 changed file with 3 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import gc
from typing import Union
import warnings
import random

import cudf
import cupy as cp
Expand Down Expand Up @@ -182,10 +181,7 @@ def __from_edgelist(
workers = _client.scheduler_info()["workers"]
# Repartition to 2 partitions per GPU for memory efficient process
input_ddf = input_ddf.repartition(npartitions=len(workers) * 2)
# FIXME: Make a copy of the input ddf before implicitly altering it.
input_ddf = input_ddf.map_partitions(
lambda df: df.copy(), token="custom-" + str(random.random())
)
input_ddf = input_ddf.map_partitions(lambda df: df.copy())
# The dataframe will be symmetrized iff the graph is undirected
# otherwise, the inital dataframe will be returned
if edge_attr is not None:
Expand Down Expand Up @@ -337,7 +333,7 @@ def __from_edgelist(
)
for w, edata in ddf.items()
}
del ddf
# FIXME: For now, don't delete the copied dataframe to avoid crash
self._plc_graph = {
w: _client.compute(delayed_task, workers=w, allow_other_workers=False)
for w, delayed_task in delayed_tasks_d.items()
Expand Down Expand Up @@ -1196,7 +1192,5 @@ def _get_column_from_ls_dfs(lst_df, col_name):
if len_df == 0:
return lst_df[0][col_name]
output_col = cudf.concat([df[col_name] for df in lst_df], ignore_index=True)
for df in lst_df:
df.drop(columns=[col_name], inplace=True)
gc.collect()
# FIXME: For now, don't delete the copied dataframe to avoid cras
return output_col

0 comments on commit f57119b

Please sign in to comment.