Skip to content

Commit

Permalink
fix: make the cache package path shorter to solve the windows path pr…
Browse files Browse the repository at this point in the history
…oblem (#2737)
  • Loading branch information
frostming authored Mar 29, 2024
1 parent 8fb9b93 commit f70e5b4
Show file tree
Hide file tree
Showing 5 changed files with 41 additions and 24 deletions.
1 change: 1 addition & 0 deletions news/2730.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Make the cache package path shorter to solve the Windows path problem.
3 changes: 1 addition & 2 deletions src/pdm/installers/synchronizers.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import dataclasses
import functools
import multiprocessing
import traceback
from concurrent.futures import Future, ThreadPoolExecutor
from functools import cached_property
Expand Down Expand Up @@ -270,7 +269,7 @@ def synchronize(self) -> None:
class Synchronizer(BaseSynchronizer):
def create_executor(self) -> ThreadPoolExecutor | DummyExecutor:
if self.parallel:
return ThreadPoolExecutor(max_workers=min(multiprocessing.cpu_count(), 8))
return ThreadPoolExecutor()
else:
return DummyExecutor()

Expand Down
19 changes: 13 additions & 6 deletions src/pdm/models/cached_package.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,11 @@ class CachedPackage:
"""A package cached in the central package store.
The directory name is similar to wheel's filename:
$PACKAGE_ROOT/<hash_part[:5]>/<dist_name>-<version>-<impl>-<abi>-<plat>/
$PACKAGE_ROOT/<checksum[:2]>/<dist_name>-<version>-<impl>-<abi>-<plat>/
Under the directory there should be a text file named `referrers`.
The checksum is stored in a file named `.checksum` under the directory.
Under the directory there could be a text file named `.referrers`.
Each line of the file is a distribution path that refers to this package.
*Only wheel installations will be cached*
"""
Expand All @@ -23,6 +25,11 @@ def __init__(self, path: str | Path) -> None:
self.path = Path(os.path.normcase(os.path.expanduser(path))).resolve()
self._referrers: set[str] | None = None

@cached_property
def checksum(self) -> str:
"""The checksum of the path"""
return self.path.joinpath(".checksum").read_text().strip()

@cached_property
def dist_info(self) -> Path:
"""The dist-info directory of the wheel"""
Expand All @@ -37,7 +44,7 @@ def dist_info(self) -> Path:
def referrers(self) -> set[str]:
"""A set of entries in referrers file"""
if self._referrers is None:
filepath = self.path / "referrers"
filepath = self.path / ".referrers"
if not filepath.is_file():
return set()
self._referrers = {
Expand All @@ -51,16 +58,16 @@ def add_referrer(self, path: str) -> None:
"""Add a new referrer"""
path = os.path.normcase(os.path.expanduser(os.path.abspath(path)))
referrers = self.referrers | {path}
(self.path / "referrers").write_text("\n".join(sorted(referrers)) + "\n", "utf8")
(self.path / ".referrers").write_text("\n".join(sorted(referrers)) + "\n", "utf8")
self._referrers = None

def remove_referrer(self, path: str) -> None:
"""Remove a referrer"""
path = os.path.normcase(os.path.expanduser(os.path.abspath(path)))
referrers = self.referrers - {path}
(self.path / "referrers").write_text("\n".join(referrers) + "\n", "utf8")
(self.path / ".referrers").write_text("\n".join(referrers) + "\n", "utf8")
self._referrers = None

def cleanup(self) -> None:
logger.info("Clean up cached package %s since it is not used by any project.", self.path)
logger.info("Clean up cached package %s", self.path)
shutil.rmtree(self.path)
34 changes: 22 additions & 12 deletions src/pdm/models/caches.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
from packaging.tags import Tag
from unearth import Link, TargetPython

from pdm.environments import BaseEnvironment

KT = TypeVar("KT")
VT = TypeVar("VT")
Expand Down Expand Up @@ -276,34 +275,45 @@ class PackageCache:
def __init__(self, root: Path) -> None:
self.root = root

def cache_wheel(self, wheel: Path, environment: BaseEnvironment, checksum: str | None = None) -> CachedPackage:
def cache_wheel(self, wheel: Path, checksum: str | None = None) -> CachedPackage:
"""Create a CachedPackage instance from a wheel file"""
import zipfile

from installer.utils import make_file_executable

if checksum is None:
checksum = get_file_hash(wheel)
parts = (checksum[:2], checksum[2:4], checksum[4:6], checksum[6:8], checksum[8:])
parts = (checksum[:2],) # shard by the first two chars of the checksum
dest = self.root.joinpath(*parts, f"{wheel.name}.cache")
pkg = CachedPackage(dest)
if dest.exists():
return pkg
dest.mkdir(parents=True, exist_ok=True)
dest.joinpath(".checksum").write_text(checksum)
logger.info("Unpacking wheel into cached location %s", dest)
with zipfile.ZipFile(wheel) as zf:
for item in zf.infolist():
target_path = zf.extract(item, dest)
mode = item.external_attr >> 16
is_executable = bool(mode and stat.S_ISREG(mode) and mode & 0o111)
if is_executable:
make_file_executable(target_path)
try:
for item in zf.infolist():
target_path = zf.extract(item, dest)
mode = item.external_attr >> 16
is_executable = bool(mode and stat.S_ISREG(mode) and mode & 0o111)
if is_executable:
make_file_executable(target_path)
except Exception: # pragma: no cover
pkg.cleanup() # cleanup on any error
raise
return pkg

def iter_packages(self) -> Iterable[tuple[str, CachedPackage]]:
for path in self.root.rglob("*.cache"):
hash_parts = path.relative_to(self.root).parent.parts
yield "".join(hash_parts), CachedPackage(path)
for path in self.root.rglob("*.whl.cache"):
hash_parts = path.parent.relative_to(self.root).parts
p = CachedPackage(path)
if not path.joinpath(".checksum").exists():
checksum = "".join(hash_parts)
path.joinpath(".checksum").write_text(checksum)
else:
checksum = p.checksum
yield checksum, p

def cleanup(self) -> int:
"""Remove unused cached packages"""
Expand Down
8 changes: 4 additions & 4 deletions src/pdm/models/candidates.py
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,7 @@ def build(self) -> CachedPackage:
if not self.req.editable:
cached, checksum = self._get_build_cache()
if cached:
self._cached = self.environment.project.package_cache.cache_wheel(cached, self.environment, checksum)
self._cached = self.environment.project.package_cache.cache_wheel(cached, checksum)
return self._cached
assert self._source_dir, "Source directory isn't ready yet"
builder_cls = EditableBuilder if self.req.editable else WheelBuilder
Expand All @@ -421,7 +421,7 @@ def build(self) -> CachedPackage:
with open(f"{wheel}.sha256", "w") as f:
f.write(checksum)
self.reporter.report_build_end(self.link.filename) # type: ignore[union-attr]
return self.environment.project.package_cache.cache_wheel(wheel, self.environment, checksum)
return self.environment.project.package_cache.cache_wheel(wheel, checksum)

def obtain(self, allow_all: bool = False, unpack: bool = True) -> None:
"""Fetch the link of the candidate and unpack to local if necessary.
Expand Down Expand Up @@ -461,7 +461,7 @@ def obtain(self, allow_all: bool = False, unpack: bool = True) -> None:
if allow_all and not self.req.editable:
cached, checksum = self._get_build_cache()
if cached:
self._cached = self.environment.project.package_cache.cache_wheel(cached, self.environment, checksum)
self._cached = self.environment.project.package_cache.cache_wheel(cached, checksum)
return
# If not, download and unpack the link
if unpack:
Expand Down Expand Up @@ -489,7 +489,7 @@ def _unpack(self, validate_hashes: bool = False) -> None:
)
if self.link.is_wheel:
checksum = hashes["sha256"][0] if (hashes := self.link.hash_option) and "sha256" in hashes else None
self._cached = self.environment.project.package_cache.cache_wheel(result, self.environment, checksum)
self._cached = self.environment.project.package_cache.cache_wheel(result, checksum)
else:
self._source_dir = Path(build_dir)
self._unpacked_dir = result
Expand Down

0 comments on commit f70e5b4

Please sign in to comment.