Skip to content

Commit

Permalink
Fix performance issues with MemoryFileSystem.rm (#1725)
Browse files Browse the repository at this point in the history
Closes #1724.
  • Loading branch information
maresb authored Oct 15, 2024
1 parent 176efbe commit 03e89cc
Showing 1 changed file with 7 additions and 3 deletions.
10 changes: 7 additions & 3 deletions fsspec/implementations/memory.py
Original file line number Diff line number Diff line change
Expand Up @@ -248,21 +248,25 @@ def created(self, path):
except KeyError as e:
raise FileNotFoundError(path) from e

def isfile(self, path):
path = self._strip_protocol(path)
return path in self.store

def rm(self, path, recursive=False, maxdepth=None):
if isinstance(path, str):
path = self._strip_protocol(path)
else:
path = [self._strip_protocol(p) for p in path]
paths = self.expand_path(path, recursive=recursive, maxdepth=maxdepth)
for p in reversed(paths):
if self.isfile(p):
self.rm_file(p)
# If the expanded path doesn't exist, it is only because the expanded
# path was a directory that does not exist in self.pseudo_dirs. This
# is possible if you directly create files without making the
# directories first.
if not self.exists(p):
elif not self.exists(p):
continue
if self.isfile(p):
self.rm_file(p)
else:
self.rmdir(p)

Expand Down

0 comments on commit 03e89cc

Please sign in to comment.