diff --git a/pex/cache/access.py b/pex/cache/access.py index 847300de5..da32d8b9d 100644 --- a/pex/cache/access.py +++ b/pex/cache/access.py @@ -4,15 +4,20 @@ from __future__ import absolute_import, print_function import fcntl +import itertools import os +import time from contextlib import contextmanager +from datetime import datetime from pex.common import safe_mkdir from pex.typing import TYPE_CHECKING from pex.variables import ENV if TYPE_CHECKING: - from typing import Iterator, Optional, Tuple + from typing import Iterator, Optional, Tuple, Union + + from pex.cache.dirs import UnzipDir, VenvDir # N.B.: The lock file path is last in the lock state tuple to allow for a simple encoding scheme in @@ -99,3 +104,24 @@ def await_delete_lock(): lock_file = _lock(exclusive=False) yield lock_file _lock(exclusive=True) + + +def record_access(pex_dir): + # type: (Union[UnzipDir, VenvDir]) -> None + + # N.B.: We explicitly set atime and do not rely on the filesystem implicitly setting it when the + # directory is read since filesystems may be mounted noatime, nodiratime or relatime on Linux + # and similar toggles exist, at least in part, for some macOS file systems. + atime = time.time() + mtime = os.stat(pex_dir.path).st_mtime + os.utime(pex_dir.path, (atime, mtime)) + + +def last_access_before(cutoff): + # type: (datetime) -> Iterator[Union[UnzipDir, VenvDir]] + pex_dirs = itertools.chain( + UnzipDir.iter_all(), VenvDir.iter_all() + ) # type: Iterator[Union[UnzipDir, VenvDir]] + for pex_dir in pex_dirs: + if datetime.fromtimestamp(os.stat(pex_dir.path).st_atime) < cutoff: + yield pex_dir diff --git a/pex/cache/data.py b/pex/cache/data.py index b359ca80b..0d99112dc 100644 --- a/pex/cache/data.py +++ b/pex/cache/data.py @@ -5,17 +5,24 @@ import os.path import sqlite3 -from contextlib import contextmanager +from contextlib import closing, contextmanager from pex.atomic_directory import atomic_directory -from pex.cache.dirs import VenvDir +from pex.cache.dirs import ( + AtomicCacheDir, + BootstrapDir, + InstalledWheelDir, + UnzipDir, + UserCodeDir, + VenvDir, +) from pex.common import CopyMode from pex.dist_metadata import ProjectNameAndVersion -from pex.typing import TYPE_CHECKING +from pex.typing import TYPE_CHECKING, cast from pex.variables import ENV if TYPE_CHECKING: - from typing import Iterator, List, Tuple, Union + from typing import Dict, Iterator, List, Optional, Union from pex.pex_info import PexInfo @@ -25,48 +32,46 @@ CREATE TABLE wheels ( name TEXT NOT NULL, - hash TEXT NOT NULL, + install_hash TEXT NOT NULL, + wheel_hash TEXT, project_name TEXT NOT NULL, version TEXT NOT NULL, - PRIMARY KEY (name ASC, hash ASC) + PRIMARY KEY (name ASC, install_hash ASC) ) WITHOUT ROWID; -CREATE UNIQUE INDEX wheels_idx_hash ON wheels (hash ASC); -CREATE INDEX wheels_idx_project_name ON wheels (project_name ASC); -CREATE INDEX wheels_idx_version ON wheels (version ASC); +CREATE UNIQUE INDEX wheels_idx_install_hash ON wheels (install_hash ASC); +CREATE INDEX wheels_idx_project_name_version ON wheels (project_name ASC, version ASC); CREATE TABLE zipapps ( pex_hash TEXT PRIMARY KEY ASC, bootstrap_hash TEXT NOT NULL, code_hash TEXT NOT NULL ) WITHOUT ROWID; -CREATE INDEX zipapps_idx_bootstrap_hash ON zipapps (bootstrap_hash ASC); -CREATE INDEX zipapps_idx_code_hash ON zipapps (code_hash ASC); CREATE TABLE zipapp_deps ( pex_hash TEXT NOT NULL REFERENCES zipapps(pex_hash) ON DELETE CASCADE, - wheel_hash TEXT NOT NULL REFERENCES wheels(hash) ON DELETE CASCADE + wheel_install_hash TEXT NOT NULL REFERENCES wheels(install_hash) ON DELETE CASCADE ); CREATE INDEX zipapp_deps_idx_pex_hash ON zipapp_deps (pex_hash ASC); -CREATE INDEX zipapp_deps_idx_wheel_hash ON zipapp_deps (wheel_hash ASC); +CREATE INDEX zipapp_deps_idx_wheel_install_hash ON zipapp_deps (wheel_install_hash ASC); CREATE TABLE venvs ( short_hash TEXT PRIMARY KEY ASC, pex_hash TEXT NOT NULL, contents_hash TEXT NOT NULL ) WITHOUT ROWID; -CREATE INDEX venvs_idx_pex_hash ON venvs (pex_hash ASC); +CREATE UNIQUE INDEX venvs_idx_pex_hash_contents_hash ON venvs (pex_hash ASC, contents_hash ASC); CREATE TABLE venv_deps ( venv_hash TEXT NOT NULL REFERENCES venvs(short_hash) ON DELETE CASCADE, - wheel_hash TEXT NOT NULL REFERENCES wheels(hash) ON DELETE CASCADE + wheel_install_hash TEXT NOT NULL REFERENCES wheels(install_hash) ON DELETE CASCADE ); CREATE INDEX venv_deps_idx_venv_hash ON venv_deps (venv_hash ASC); -CREATE INDEX venv_deps_idx_wheel_hash ON venv_deps (wheel_hash ASC); +CREATE INDEX venv_deps_idx_wheel_hash ON venv_deps (wheel_install_hash ASC); """ @contextmanager -def db_connection(): +def _db_connection(): # type: () -> Iterator[sqlite3.Connection] db_dir = os.path.join(ENV.PEX_ROOT, "data") with atomic_directory(db_dir) as atomic_dir: @@ -87,27 +92,36 @@ def db_connection(): def _inserted_wheels(pex_info): # type: (PexInfo) -> Iterator[sqlite3.Cursor] - wheels = [] # type: List[Tuple[str, str, str, str]] - for wheel_name, wheel_hash in pex_info.distributions.items(): + wheels = [] # type: List[Dict[str, Optional[str]]] + for wheel_name, install_hash in pex_info.distributions.items(): + wheel_hash = None # type: Optional[str] + installed_wheel_dir = InstalledWheelDir.create(wheel_name, install_hash) + if os.path.islink(installed_wheel_dir): + wheel_hash_dir, _ = os.path.split(os.path.realpath(installed_wheel_dir)) + wheel_hash = os.path.basename(wheel_hash_dir) + pnav = ProjectNameAndVersion.from_filename(wheel_name) wheels.append( - ( - wheel_name, - wheel_hash, - str(pnav.canonicalized_project_name), - str(pnav.canonicalized_version), + dict( + name=wheel_name, + install_hash=install_hash, + wheel_hash=wheel_hash, + project_name=str(pnav.canonicalized_project_name), + version=str(pnav.canonicalized_version), ) ) - with db_connection() as conn: + with _db_connection() as conn: cursor = conn.executemany( """ - INSERT OR IGNORE INTO wheels ( + INSERT INTO wheels ( name, - hash, + install_hash, + wheel_hash, project_name, version - ) VALUES (?, ?, ?, ?) + ) VALUES (:name, :install_hash, :wheel_hash, :project_name, :version) + ON CONFLICT (name, install_hash) DO UPDATE SET wheel_hash = :wheel_hash """, wheels, ) @@ -115,7 +129,7 @@ def _inserted_wheels(pex_info): cursor.close() -def record_zipapp_dependencies(pex_info): +def record_zipapp_install(pex_info): # type: (PexInfo) -> None with _inserted_wheels(pex_info) as cursor: @@ -123,8 +137,8 @@ def record_zipapp_dependencies(pex_info): """ INSERT OR IGNORE INTO zipapps ( pex_hash, - code_hash, - bootstrap_hash + bootstrap_hash, + code_hash ) VALUES (?, ?, ?) """, (pex_info.pex_hash, pex_info.bootstrap_hash, pex_info.code_hash), @@ -132,21 +146,17 @@ def record_zipapp_dependencies(pex_info): """ INSERT OR IGNORE INTO zipapp_deps ( pex_hash, - wheel_hash + wheel_install_hash ) VALUES (?, ?) """, tuple( - (pex_info.pex_hash, wheel_hash) for wheel_hash in pex_info.distributions.values() + (pex_info.pex_hash, wheel_install_hash) + for wheel_install_hash in pex_info.distributions.values() ), ).close() -def record_zipapp_access(unzip_dir): - # type: (...) -> None - os.utime(unzip_dir, None) - - -def record_venv_dependencies( +def record_venv_install( copy_mode, # type: CopyMode.Value pex_info, # type: PexInfo venv_dir, # type: VenvDir @@ -173,16 +183,93 @@ def record_venv(coon_or_cursor): """ INSERT OR IGNORE INTO venv_deps ( venv_hash, - wheel_hash + wheel_install_hash ) VALUES (?, ?) """, - tuple((venv_hash, wheel_hash) for wheel_hash in pex_info.distributions.values()), + tuple( + (venv_hash, wheel_install_hash) + for wheel_install_hash in pex_info.distributions.values() + ), ).close() else: - with db_connection() as conn: + with _db_connection() as conn: record_venv(conn).close() -def record_venv_access(venv_dir): - # type: (VenvDir) -> None - os.utime(venv_dir.path, None) +def zipapp_deps(pex_dir): + # type: (UnzipDir) -> Iterator[Union[BootstrapDir, UserCodeDir, str, InstalledWheelDir]] + with _db_connection() as conn: + with closing( + conn.execute( + "SELECT bootstrap_hash, code_hash FROM zipapps WHERE pex_hash = ?", + [pex_dir.pex_hash], + ) + ) as cursor: + bootstrap_hash, code_hash = cursor.fetchone() + yield BootstrapDir.create(bootstrap_hash) + yield UserCodeDir.create(code_hash) + + with closing( + conn.execute( + """ + SELECT name, install_hash, wheel_hash FROM wheels + JOIN zipapp_deps ON zipapp_deps.wheel_install_hash = wheels.install_hash + JOIN zipapps ON zipapps.pex_hash = zipapp_deps.pex_hash + WHERE zipapps.pex_hash = ? + """, + [pex_dir.pex_hash], + ) + ) as cursor: + for wheel_name, wheel_install_hash, wheel_hash in cursor: + installed_wheel_dir = InstalledWheelDir.create( + wheel_name=wheel_name, wheel_hash=wheel_install_hash + ) + if wheel_hash: + yield InstalledWheelDir.create(wheel_name=wheel_name, wheel_hash=wheel_hash) + yield installed_wheel_dir.path + else: + yield installed_wheel_dir + + +def venv_deps(venv_dir): + # type: (VenvDir) -> Iterator[Union[str, InstalledWheelDir]] + with _db_connection() as conn: + with closing( + conn.execute( + "SELECT short_hash FROM venvs WHERE pex_hash = ? AND contents_hash = ?", + (venv_dir.pex_hash, venv_dir.contents_hash), + ) + ) as cursor: + short_hash = cast(str, cursor.fetchone()[0]) + yield VenvDir.short_path(short_hash, include_symlink=True) + + with closing( + conn.execute( + """ + SELECT name, install_hash, wheel_hash FROM wheels + JOIN venv_deps ON venv_deps.wheel_install_hash = wheels.install_hash + JOIN venvs ON venvs.short_hash = venv_deps.venv_hash + WHERE venvs.short_hash = ? + """, + [short_hash], + ) + ) as cursor: + for wheel_name, wheel_install_hash, wheel_hash in cursor: + installed_wheel_dir = InstalledWheelDir.create( + wheel_name=wheel_name, wheel_hash=wheel_install_hash + ) + if wheel_hash: + yield InstalledWheelDir.create(wheel_name=wheel_name, wheel_hash=wheel_hash) + yield installed_wheel_dir.path + else: + yield installed_wheel_dir + + +def dir_dependencies(pex_dir): + # type: (Union[UnzipDir, VenvDir]) -> Iterator[Union[str, AtomicCacheDir]] + return zipapp_deps(pex_dir) if isinstance(pex_dir, UnzipDir) else venv_deps(pex_dir) + + +def delete(pex_dir): + # type: (Union[UnzipDir, VenvDir]) -> None + pass diff --git a/pex/cache/dirs.py b/pex/cache/dirs.py index a5c5683d0..983c6d860 100644 --- a/pex/cache/dirs.py +++ b/pex/cache/dirs.py @@ -3,6 +3,7 @@ from __future__ import absolute_import +import glob import os from pex.enum import Enum @@ -10,31 +11,7 @@ from pex.variables import ENV, Variables if TYPE_CHECKING: - from typing import Iterable, Iterator, Union - - -class VenvDir(str): - @staticmethod - def __new__( - cls, - path, # type: str - pex_hash, # type: str - contents_hash, # type: str - ): - # type: (...) -> VenvDir - # MyPy incorrectly flags the call to super(VenvDir, cls).__new__(cls, path) for Python 2.7. - return cast(VenvDir, super(VenvDir, cls).__new__(cls, path)) # type: ignore[call-arg] - - def __init__( - self, - path, # type: str - pex_hash, # type: str - contents_hash, # type: str - ): - # type: (...) -> None - self.path = path - self.pex_hash = pex_hash - self.contents_hash = contents_hash + from typing import Any, Iterable, Iterator, Type, TypeVar, Union class CacheDir(Enum["CacheDir.Value"]): @@ -203,3 +180,176 @@ def iter_transitive_dependents(self): description="Virtual environments generated at runtime for `--venv` mode PEXes.", dependencies=[INSTALLED_WHEELS], ) + + +if TYPE_CHECKING: + _AtomicCacheDir = TypeVar("_AtomicCacheDir", bound="AtomicCacheDir") + + +class AtomicCacheDir(str): + @staticmethod + def __new__( + cls, # type: Type[_AtomicCacheDir] + path, # type: str + *args, # type: Any + **kwargs # type: Any + ): + # type: (...) -> _AtomicCacheDir + # MyPy incorrectly flags the call to str.__new__(cls, path) for Python 2.7. + return cast("_AtomicCacheDir", str.__new__(cls, path)) # type: ignore[call-arg] + + def __init__( + self, + path, # type: str + *args, # type: Any + **kwargs # type: Any + ): + # type: (...) -> None + self.path = path + + def __repr__(self): + # type: () -> str + return "{clazz}(path={path})".format(clazz=self.__class__.__name__, path=self.path) + + +class UnzipDir(AtomicCacheDir): + @classmethod + def create( + cls, + pex_hash, # type: str + pex_root=ENV, # type: Union[str, Variables] + ): + # type: (...) -> UnzipDir + unzip_dir = CacheDir.UNZIPPED_PEXES.path(pex_hash, pex_root=pex_root) + return cls(path=unzip_dir, pex_hash=pex_hash) + + @classmethod + def iter_all(cls): + # type: () -> Iterator[UnzipDir] + for unzip_dir in glob.glob(CacheDir.UNZIPPED_PEXES.path("*")): + if os.path.isdir(unzip_dir): + pex_hash = os.path.basename(unzip_dir) + yield UnzipDir(path=unzip_dir, pex_hash=pex_hash) + + def __init__( + self, + path, # type: str + pex_hash, # type: str + ): + # type: (...) -> None + super(UnzipDir, self).__init__(path) + self.pex_hash = pex_hash + + +class VenvDir(AtomicCacheDir): + SHORT_SYMLINK_NAME = "venv" + + @classmethod + def short_path( + cls, + short_hash, # type: str + include_symlink, # type: bool + pex_root=ENV, # type: Union[str, Variables] + ): + # type: (...) -> str + subdirs = ["s", short_hash] + if include_symlink: + subdirs.append(cls.SHORT_SYMLINK_NAME) + return CacheDir.VENVS.path(*subdirs, pex_root=pex_root) + + @classmethod + def create( + cls, + pex_hash, # type: str + contents_hash, # type: str + pex_root=ENV, # type: Union[str, Variables] + ): + # type: (...) -> VenvDir + venv_dir = CacheDir.VENVS.path(pex_hash, contents_hash, pex_root=pex_root) + return cls(path=venv_dir, pex_hash=pex_hash, contents_hash=contents_hash) + + @classmethod + def iter_all(cls): + # type: () -> Iterator[VenvDir] + for venv_dir in glob.glob(CacheDir.VENVS.path("*", "*")): + if os.path.isdir(venv_dir): + head, contents_hash = os.path.split(venv_dir) + pex_hash = os.path.basename(head) + yield VenvDir(path=venv_dir, pex_hash=pex_hash, contents_hash=contents_hash) + + def __init__( + self, + path, # type: str + pex_hash, # type: str + contents_hash, # type: str + ): + # type: (...) -> None + super(VenvDir, self).__init__(path) + self.pex_hash = pex_hash + self.contents_hash = contents_hash + + +class InstalledWheelDir(AtomicCacheDir): + @classmethod + def create( + cls, + wheel_name, # type: str + wheel_hash, # type: str + pex_root=ENV, # type: Union[str, Variables] + ): + # type: (...) -> InstalledWheelDir + wheel_dir = CacheDir.INSTALLED_WHEELS.path(wheel_hash, wheel_name, pex_root=pex_root) + return cls(path=wheel_dir, wheel_name=wheel_name, wheel_hash=wheel_hash) + + def __init__( + self, + path, # type: str + wheel_name, # type: str + wheel_hash, # type: str + ): + # type: (...) -> None + super(InstalledWheelDir, self).__init__(path) + self.wheel_name = wheel_name + self.wheel_hash = wheel_hash + + +class BootstrapDir(AtomicCacheDir): + @classmethod + def create( + cls, + bootstrap_hash, # type: str + pex_root=ENV, # type: Union[str, Variables] + ): + # type: (...) -> BootstrapDir + bootstrap_dir = CacheDir.BOOTSTRAPS.path(bootstrap_hash, pex_root=pex_root) + return cls(path=bootstrap_dir, bootstrap_hash=bootstrap_hash) + + def __init__( + self, + path, # type: str + bootstrap_hash, # type: str + ): + # type: (...) -> None + super(BootstrapDir, self).__init__(path) + self.bootstrap_hash = bootstrap_hash + + +class UserCodeDir(AtomicCacheDir): + @classmethod + def create( + cls, + code_hash, # type: str + pex_root=ENV, # type: Union[str, Variables] + ): + # type: (...) -> UserCodeDir + user_code_dir = CacheDir.USER_CODE.path(code_hash, pex_root=pex_root) + return cls(path=user_code_dir, code_hash=code_hash) + + def __init__( + self, + path, # type: str + code_hash, # type: str + ): + # type: (...) -> None + super(UserCodeDir, self).__init__(path) + self.code_hash = code_hash diff --git a/pex/layout.py b/pex/layout.py index fcf76208b..40de89a28 100644 --- a/pex/layout.py +++ b/pex/layout.py @@ -10,8 +10,7 @@ from pex.atomic_directory import atomic_directory from pex.cache import access as cache_access -from pex.cache.data import record_zipapp_access, record_zipapp_dependencies -from pex.cache.dirs import CacheDir +from pex.cache.dirs import BootstrapDir, InstalledWheelDir, UserCodeDir from pex.common import ZipFileEx, is_script, open_zip, safe_copy, safe_mkdir, safe_mkdtemp from pex.enum import Enum from pex.tracer import TRACER @@ -164,7 +163,9 @@ def _install_distribution( location, sha = distribution_info is_wheel_file = pex_info.deps_are_wheel_files - spread_dest = CacheDir.INSTALLED_WHEELS.path(sha, location, pex_root=pex_info.pex_root) + spread_dest = InstalledWheelDir.create( + wheel_name=location, wheel_hash=sha, pex_root=pex_info.pex_root + ) dist_relpath = os.path.join(DEPS_DIR, location) source = None if is_wheel_file else layout.dist_strip_prefix(location) symlink_src = os.path.relpath( @@ -315,12 +316,13 @@ def _ensure_installed( with ENV.patch(PEX_ROOT=pex_root): cache_access.read_write() else: - record_zipapp_access(unzip_dir=install_to) + cache_access.record_access(install_to) with atomic_directory(install_to) as chroot: if not chroot.is_finalized(): with ENV.patch(PEX_ROOT=pex_root), TRACER.timed( "Installing {} to {}".format(pex, install_to) ): + from pex.cache.data import record_zipapp_install from pex.pex_info import PexInfo pex_info = PexInfo.from_pex(pex) @@ -330,16 +332,14 @@ def _ensure_installed( raise AssertionError( "Expected bootstrap_cache to be populated for {}.".format(layout) ) - bootstrap_cache = CacheDir.BOOTSTRAPS.path( + bootstrap_cache = BootstrapDir.create( pex_info.bootstrap_hash, pex_root=pex_info.pex_root ) if pex_info.code_hash is None: raise AssertionError( "Expected code_hash to be populated for {}.".format(layout) ) - code_cache = CacheDir.USER_CODE.path( - pex_info.code_hash, pex_root=pex_info.pex_root - ) + code_cache = UserCodeDir.create(pex_info.code_hash, pex_root=pex_info.pex_root) with atomic_directory( bootstrap_cache, source=layout.bootstrap_strip_prefix() @@ -376,7 +376,7 @@ def _ensure_installed( pex=pex, hash=pex_info.pex_hash ) ): - record_zipapp_dependencies(pex_info) + record_zipapp_install(pex_info) return install_to diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index 3dc424fff..acf61ba8e 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -10,8 +10,7 @@ from pex import pex_warnings from pex.atomic_directory import atomic_directory from pex.cache import access as cache_access -from pex.cache.data import record_venv_access, record_venv_dependencies -from pex.cache.dirs import CacheDir +from pex.cache.dirs import VenvDir from pex.common import CopyMode, die, pluralize from pex.environment import ResolveError from pex.inherit_path import InheritPath @@ -525,9 +524,10 @@ def ensure_venv( with ENV.patch(PEX_ROOT=pex_info.pex_root): cache_access.read_write() else: - record_venv_access(venv_dir=venv_dir) + cache_access.record_access(venv_dir) with atomic_directory(venv_dir) as venv: if not venv.is_finalized(): + from pex.cache.data import record_venv_install from pex.venv.virtualenv import Virtualenv virtualenv = Virtualenv.create_atomic( @@ -548,7 +548,9 @@ def ensure_venv( collisions = [] for chars in range(8, len(venv_hash) + 1): entropy = venv_hash[:chars] - short_venv_dir = CacheDir.VENVS.path("s", entropy, pex_root=pex_info.pex_root) + short_venv_dir = VenvDir.short_path( + short_hash=entropy, include_symlink=False, pex_root=pex_info.pex_root + ) with atomic_directory(short_venv_dir) as short_venv: if short_venv.is_finalized(): collisions.append(short_venv_dir) @@ -570,7 +572,9 @@ def ensure_venv( ) continue - os.symlink(venv_dir, os.path.join(short_venv.work_dir, "venv")) + os.symlink( + venv_dir, os.path.join(short_venv.work_dir, VenvDir.SHORT_SYMLINK_NAME) + ) # Loose PEXes don't need to unpack themselves to the PEX_ROOT before running; # so we'll not have a stable base there to symlink from. As such, always copy @@ -602,7 +606,7 @@ def ensure_venv( pex=pex.path(), hash=pex_info.pex_hash ) ): - record_venv_dependencies( + record_venv_install( copy_mode=copy_mode, pex_info=pex_info, venv_dir=venv_dir, diff --git a/pex/scie/science.py b/pex/scie/science.py index 62be4207c..f0aaa1644 100644 --- a/pex/scie/science.py +++ b/pex/scie/science.py @@ -11,7 +11,7 @@ from subprocess import CalledProcessError from pex.atomic_directory import atomic_directory -from pex.cache.dirs import CacheDir +from pex.cache.dirs import CacheDir, UnzipDir from pex.common import chmod_plus_x, is_exe, pluralize, safe_mkdtemp, safe_open from pex.compatibility import shlex_quote from pex.dist_metadata import NamedEntryPoint, parse_entry_point @@ -131,7 +131,7 @@ def create_manifests( else: production_assert(pex_info.pex_hash is not None) pex_hash = cast(str, pex_info.pex_hash) - configure_binding_args.append(CacheDir.UNZIPPED_PEXES.path(pex_hash, pex_root=pex_root)) + configure_binding_args.append(UnzipDir.create(pex_hash, pex_root=pex_root).path) commands = [] # type: List[Dict[str, Any]] entrypoints = configuration.options.busybox_entrypoints diff --git a/pex/variables.py b/pex/variables.py index b7baf47a2..cff68fd66 100644 --- a/pex/variables.py +++ b/pex/variables.py @@ -29,7 +29,7 @@ # N.B.: This import is circular, and we import lazily below as a result, but we also need the # import eagerly for type checking. - from pex.cache.dirs import VenvDir # noqa + from pex.cache.dirs import UnzipDir, VenvDir # noqa # N.B.: This is an expensive import, and we only need it for type checking. from pex.interpreter import PythonInterpreter @@ -809,13 +809,13 @@ def unzip_dir( pex_hash, # type: str expand_pex_root=True, # type: bool ): - # type: (...) -> str + # type: (...) -> UnzipDir # N.B.: We need lazy import gymnastics here since CacheType uses Variables for PEX_ROOT. - from pex.cache.dirs import CacheDir + from pex.cache.dirs import UnzipDir pex_root = _expand_pex_root(pex_root) if expand_pex_root else pex_root - return CacheDir.UNZIPPED_PEXES.path(pex_hash, pex_root=pex_root) + return UnzipDir.create(pex_hash=pex_hash, pex_root=pex_root) def venv_dir( @@ -830,7 +830,7 @@ def venv_dir( # type: (...) -> VenvDir # N.B.: We need lazy import gymnastics here since CacheType uses Variables for PEX_ROOT. - from pex.cache.dirs import CacheDir, VenvDir + from pex.cache.dirs import VenvDir # The venv contents are affected by which PEX files are in play as well as which interpreter # is selected. The former is influenced via PEX_PATH and the latter is influenced by interpreter @@ -893,7 +893,7 @@ def add_pex_path_items(pexes): json.dumps(venv_contents, sort_keys=True).encode("utf-8") ).hexdigest() pex_root = _expand_pex_root(pex_root) if expand_pex_root else pex_root - venv_path = CacheDir.VENVS.path(pex_hash, venv_contents_hash, pex_root=pex_root) + venv_path = VenvDir.create(pex_hash, venv_contents_hash, pex_root=pex_root) def warn(message): # type: (str) -> None @@ -948,4 +948,4 @@ def warn(message): ) ) - return VenvDir(path=venv_path, pex_hash=pex_hash, contents_hash=venv_contents_hash) + return venv_path