Skip to content

Commit

Permalink
Start to fill out some tests...
Browse files Browse the repository at this point in the history
  • Loading branch information
jsirois committed Nov 1, 2024
1 parent 6a7a8ae commit 63e416f
Show file tree
Hide file tree
Showing 4 changed files with 339 additions and 75 deletions.
8 changes: 8 additions & 0 deletions pex/cache/dirs.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,14 @@ def __init__(
# type: (...) -> None
self.path = path

def __eq__(self, other):
# type: (Any) -> bool
return type(self) == type(other) and self.path == other.path

def __hash__(self):
# type: () -> int
return hash(self.path)

def __repr__(self):
# type: () -> str
return "{clazz}(path={path})".format(clazz=self.__class__.__name__, path=self.path)
Expand Down
9 changes: 6 additions & 3 deletions pex/cache/prunable.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,12 +109,15 @@ def scan(cls, cutoff):
) # type: Container[Union[BootstrapDir, UserCodeDir, InstalledWheelDir]]
pips = attr.ib() # type: Pips

def iter_pex_deps(self):
def iter_pex_unused_deps(self):
# type: () -> Iterator[Union[BootstrapDir, UserCodeDir, InstalledWheelDir]]
return iter(self._pex_deps)
for dep in self._pex_deps:
if dep not in self._unprunable_deps:
yield dep

def iter_unused_deps(self):
def iter_other_unused_deps(self):
# type: () -> Iterator[Union[BootstrapDir, UserCodeDir, InstalledWheelDir]]

for bootstrap_dir in BootstrapDir.iter_all():
if bootstrap_dir not in self._pex_deps and bootstrap_dir not in self._unprunable_deps:
yield bootstrap_dir
Expand Down
112 changes: 60 additions & 52 deletions pex/cli/commands/cache/command.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,59 @@ def parse(cls, spec):
cutoff = attr.ib() # type: datetime


def _prune_cache_dir(
dry_run, # type: bool
additional_cache_dirs_by_project_name_and_version, # type: Mapping[Tuple[ProjectName, Version], Iterable[AtomicCacheDir]]
cache_dir, # type: AtomicCacheDir
):
# type: (...) -> DiskUsage
paths_to_prune = [] # type: List[str]

def prune_if_exists(path):
# type: (Optional[str]) -> None
if path and os.path.exists(path):
paths_to_prune.append(path)

if isinstance(cache_dir, InstalledWheelDir):
paths_to_prune.append(os.path.dirname(cache_dir.path))
prune_if_exists(CacheDir.PACKED_WHEELS.path(cache_dir.install_hash))
for additional_dir in additional_cache_dirs_by_project_name_and_version.get(
(cache_dir.project_name, cache_dir.version), ()
):
prune_if_exists(additional_dir)
elif isinstance(cache_dir, BootstrapDir):
paths_to_prune.append(cache_dir.path)
prune_if_exists(CacheDir.BOOTSTRAP_ZIPS.path(cache_dir.bootstrap_hash))
else:
paths_to_prune.append(cache_dir.path)

disk_usages = [DiskUsage.collect(path) for path in paths_to_prune]
if not dry_run:
for path in paths_to_prune:
safe_rmtree(path)
if isinstance(cache_dir, InstalledWheelDir) and cache_dir.symlink_dir:
safe_rmtree(cache_dir.symlink_dir)
elif isinstance(cache_dir, VenvDirs):
safe_rmtree(cache_dir.short_dir)

return (
disk_usages[0]
if len(disk_usages) == 1
else DiskUsage.aggregate(cache_dir.path, disk_usages)
)


def _prune_pip(
dry_run, # type: bool
pip_path_to_prune, # type: str
):
# type: (...) -> DiskUsage
du = DiskUsage.collect(pip_path_to_prune)
if not dry_run:
safe_rmtree(pip_path_to_prune)
return du


class Cache(OutputMixin, BuildTimeCommand):
"""Interact with the Pex cache."""

Expand Down Expand Up @@ -467,54 +520,6 @@ def _purge(self):

return Ok()

def _prune_cache_dir(
self,
additional_cache_dirs_by_project_name_and_version, # type: Mapping[Tuple[ProjectName, Version], Iterable[AtomicCacheDir]]
cache_dir, # type: AtomicCacheDir
):
# type: (...) -> DiskUsage
paths_to_prune = [] # type: List[str]

def prune_if_exists(path):
# type: (Optional[str]) -> None
if path and os.path.exists(path):
paths_to_prune.append(path)

if isinstance(cache_dir, InstalledWheelDir):
paths_to_prune.append(os.path.dirname(cache_dir.path))
prune_if_exists(CacheDir.PACKED_WHEELS.path(cache_dir.install_hash))
for additional_dir in additional_cache_dirs_by_project_name_and_version.get(
(cache_dir.project_name, cache_dir.version), ()
):
prune_if_exists(additional_dir)
elif isinstance(cache_dir, BootstrapDir):
paths_to_prune.append(cache_dir.path)
prune_if_exists(CacheDir.BOOTSTRAP_ZIPS.path(cache_dir.bootstrap_hash))
else:
paths_to_prune.append(cache_dir.path)

disk_usages = [DiskUsage.collect(path) for path in paths_to_prune]
if not self.options.dry_run:
for path in paths_to_prune:
safe_rmtree(path)
if isinstance(cache_dir, InstalledWheelDir) and cache_dir.symlink_dir:
safe_rmtree(cache_dir.symlink_dir)
elif isinstance(cache_dir, VenvDirs):
safe_rmtree(cache_dir.short_dir)

return (
disk_usages[0]
if len(disk_usages) == 1
else DiskUsage.aggregate(cache_dir.path, disk_usages)
)

def _prune_pip(self, pip_path_to_prune):
# type: (str) -> DiskUsage
du = DiskUsage.collect(pip_path_to_prune)
if not self.options.dry_run:
safe_rmtree(pip_path_to_prune)
return du

def _prune(self):
# type: () -> Result

Expand All @@ -534,7 +539,7 @@ def _prune(self):

cutoff = self.options.cutoff
prunable = Prunable.scan(cutoff.cutoff)
unused_deps = tuple(prunable.iter_unused_deps())
unused_deps = tuple(prunable.iter_other_unused_deps())
unused_wheels = tuple(dep for dep in unused_deps if isinstance(dep, InstalledWheelDir))

additional_cache_dirs_by_project_name_and_version = defaultdict(
Expand All @@ -549,8 +554,11 @@ def _prune(self):
].append(cache_dir)

prune_cache_dir = functools.partial(
self._prune_cache_dir, additional_cache_dirs_by_project_name_and_version
_prune_cache_dir,
self.options.dry_run,
additional_cache_dirs_by_project_name_and_version,
)
prune_pip = functools.partial(_prune_pip, self.options.dry_run)

def prune_unused_deps(additional=False):
# type: (bool) -> None
Expand Down Expand Up @@ -596,7 +604,7 @@ def prune_pips():
tuple(
iter_map_parallel(
prunable.pips.paths,
function=self._prune_pip,
function=prune_pip,
noun="Pip",
verb="prune",
verb_past="pruned",
Expand Down Expand Up @@ -773,7 +781,7 @@ def prune_interpreters():
)
print(file=fp)

deps = tuple(prunable.iter_pex_deps())
deps = tuple(prunable.iter_pex_unused_deps())
if self.options.dry_run:
print(
"Might have pruned up to {count} {cached_pex_dependency}.".format(
Expand Down
Loading

0 comments on commit 63e416f

Please sign in to comment.