From 3f912ad48e06915723619f6395074ee8bd6c88f5 Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Sat, 14 Dec 2019 11:10:36 -0500 Subject: [PATCH 1/4] Add function to install directly from wheel files This will help us avoid some complicated directory-changing logic in WheelBuilder. --- src/pip/_internal/operations/install/wheel.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/pip/_internal/operations/install/wheel.py b/src/pip/_internal/operations/install/wheel.py index 028ed3df7c6..5893ad64d2e 100644 --- a/src/pip/_internal/operations/install/wheel.py +++ b/src/pip/_internal/operations/install/wheel.py @@ -28,7 +28,9 @@ from pip._internal.exceptions import InstallationError, UnsupportedWheel from pip._internal.locations import get_major_minor_version from pip._internal.utils.misc import captured_stdout, ensure_dir, hash_file +from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.unpacking import unpack_file if MYPY_CHECK_RUNNING: from typing import ( @@ -618,6 +620,27 @@ def is_entrypoint_wrapper(name): shutil.move(temp_record, record) +def install_wheel( + name, # type: str + wheel_path, # type: str + scheme, # type: Scheme + req_description, # type: str + pycompile=True, # type: bool + warn_script_location=True, # type: bool +): + # type: (...) -> None + with TempDirectory(kind="unpacked-wheel") as unpacked_dir: + unpack_file(wheel_path, unpacked_dir) + install_unpacked_wheel( + name=name, + wheeldir=unpacked_dir, + scheme=scheme, + req_description=req_description, + pycompile=pycompile, + warn_script_location=warn_script_location, + ) + + def wheel_version(source_dir): # type: (Optional[str]) -> Optional[Tuple[int, ...]] """Return the Wheel-Version of an extracted wheel, if possible. From c565d7a1b2e74a229edfc11087d2b8f5f1a86f3a Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Sat, 14 Dec 2019 11:21:00 -0500 Subject: [PATCH 2/4] Switch to install_wheel in unit tests Since it tests install_unpacked_wheel, the coverage should be the same. --- src/pip/_internal/operations/install/wheel.py | 9 ++++++--- tests/unit/test_wheel.py | 14 +++++++------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/pip/_internal/operations/install/wheel.py b/src/pip/_internal/operations/install/wheel.py index 5893ad64d2e..627cc4a32a6 100644 --- a/src/pip/_internal/operations/install/wheel.py +++ b/src/pip/_internal/operations/install/wheel.py @@ -627,13 +627,16 @@ def install_wheel( req_description, # type: str pycompile=True, # type: bool warn_script_location=True, # type: bool + _temp_dir_for_testing=None, # type: Optional[str] ): # type: (...) -> None - with TempDirectory(kind="unpacked-wheel") as unpacked_dir: - unpack_file(wheel_path, unpacked_dir) + with TempDirectory( + path=_temp_dir_for_testing, kind="unpacked-wheel" + ) as unpacked_dir: + unpack_file(wheel_path, unpacked_dir.path) install_unpacked_wheel( name=name, - wheeldir=unpacked_dir, + wheeldir=unpacked_dir.path, scheme=scheme, req_description=req_description, pycompile=pycompile, diff --git a/tests/unit/test_wheel.py b/tests/unit/test_wheel.py index 4bcbcbee2a2..52e4e15aaed 100644 --- a/tests/unit/test_wheel.py +++ b/tests/unit/test_wheel.py @@ -260,7 +260,6 @@ def prep(self, data, tmpdir): self.req = Requirement('sample') self.src = os.path.join(tmpdir, 'src') self.dest = os.path.join(tmpdir, 'dest') - unpack_file(self.wheelpath, self.src) self.scheme = Scheme( purelib=os.path.join(self.dest, 'lib'), platlib=os.path.join(self.dest, 'lib'), @@ -290,9 +289,9 @@ def assert_installed(self): def test_std_install(self, data, tmpdir): self.prep(data, tmpdir) - wheel.install_unpacked_wheel( + wheel.install_wheel( self.name, - self.src, + self.wheelpath, scheme=self.scheme, req_description=str(self.req), ) @@ -309,9 +308,9 @@ def test_install_prefix(self, data, tmpdir): isolated=False, prefix=prefix, ) - wheel.install_unpacked_wheel( + wheel.install_wheel( self.name, - self.src, + self.wheelpath, scheme=scheme, req_description=str(self.req), ) @@ -330,11 +329,12 @@ def test_dist_info_contains_empty_dir(self, data, tmpdir): self.src_dist_info, 'empty_dir', 'empty_dir') os.makedirs(src_empty_dir) assert os.path.isdir(src_empty_dir) - wheel.install_unpacked_wheel( + wheel.install_wheel( self.name, - self.src, + self.wheelpath, scheme=self.scheme, req_description=str(self.req), + _temp_dir_for_testing=self.src, ) self.assert_installed() assert not os.path.isdir( From 327c295554568af4e897c096190623fc1c198f85 Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Sat, 14 Dec 2019 11:22:00 -0500 Subject: [PATCH 3/4] Keep path to downloaded archive on InstallRequirement Now we'll be able to transition other parts of the code to use pre-existing archives directly instead of relying on unpacked sources. --- src/pip/_internal/operations/prepare.py | 24 +++++++++++++++++------- src/pip/_internal/req/req_install.py | 4 ++++ src/pip/_internal/wheel_builder.py | 1 + 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 47c8717830e..f2fc26d58f6 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -143,7 +143,7 @@ def unpack_http_url( download_dir=None, # type: Optional[str] hashes=None, # type: Optional[Hashes] ): - # type: (...) -> None + # type: (...) -> str temp_dir = TempDirectory(kind="unpack", globally_managed=True) # If a download dir is specified, is the file already downloaded there? already_downloaded_path = None @@ -171,6 +171,8 @@ def unpack_http_url( ): _copy_file(from_path, download_dir, link) + return from_path + def _copy2_ignoring_special_files(src, dest): # type: (str, str) -> None @@ -219,7 +221,7 @@ def unpack_file_url( download_dir=None, # type: Optional[str] hashes=None # type: Optional[Hashes] ): - # type: (...) -> None + # type: (...) -> Optional[str] """Unpack link into location. If download_dir is provided and link points to a file, make a copy @@ -233,7 +235,7 @@ def unpack_file_url( _copy_source_tree(link_path, location) if download_dir: logger.info('Link is a directory, ignoring download_dir') - return + return None # If --require-hashes is off, `hashes` is either empty, the # link's embedded hash, or MissingHashes; it is required to @@ -267,6 +269,8 @@ def unpack_file_url( ): _copy_file(from_path, download_dir, link) + return from_path + def unpack_url( link, # type: Link @@ -275,7 +279,7 @@ def unpack_url( download_dir=None, # type: Optional[str] hashes=None, # type: Optional[Hashes] ): - # type: (...) -> None + # type: (...) -> Optional[str] """Unpack link. If link is a VCS link: if only_download, export into download_dir and ignore location @@ -293,14 +297,15 @@ def unpack_url( # non-editable vcs urls if link.is_vcs: unpack_vcs_link(link, location) + return None # file urls elif link.is_file: - unpack_file_url(link, location, download_dir, hashes=hashes) + return unpack_file_url(link, location, download_dir, hashes=hashes) # http urls else: - unpack_http_url( + return unpack_http_url( link, location, downloader, @@ -500,7 +505,7 @@ def prepare_linked_requirement( download_dir = self.wheel_download_dir try: - unpack_url( + local_path = unpack_url( link, req.source_dir, self.downloader, download_dir, hashes=hashes, ) @@ -515,6 +520,11 @@ def prepare_linked_requirement( 'error {} for URL {}'.format(req, exc, link) ) + # For use in later processing, preserve the file path on the + # requirement. + if local_path: + req.local_file_path = local_path + if link.is_wheel: if download_dir: # When downloading, we only unpack wheels to get diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 059dc04e881..0671d7eb8e4 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -137,6 +137,10 @@ def __init__( # PEP 508 URL requirement link = Link(req.url) self.link = self.original_link = link + # Path to any downloaded or already-existing package. + self.local_file_path = None # type: Optional[str] + if self.link and self.link.is_file: + self.local_file_path = self.link.file_path if extras: self.extras = extras diff --git a/src/pip/_internal/wheel_builder.py b/src/pip/_internal/wheel_builder.py index 4e191061bb3..8a51c727582 100644 --- a/src/pip/_internal/wheel_builder.py +++ b/src/pip/_internal/wheel_builder.py @@ -460,6 +460,7 @@ def build( ) # Update the link for this. req.link = Link(path_to_url(wheel_file)) + req.local_file_path = req.link.file_path assert req.link.is_wheel # extract the wheel into the dir unpack_file(req.link.file_path, req.source_dir) From 11472e1b12f8c883d9326fecd5c9d4e0373c9b7e Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Sat, 14 Dec 2019 11:23:19 -0500 Subject: [PATCH 4/4] Switch to install_wheel in InstallRequirement This removes one of usages of the overloaded `source_dir` member. We know the local_file_path must be set at this point because it is set in the only 3 places that wheels are added to InstallRequirement. --- src/pip/_internal/req/req_install.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 0671d7eb8e4..cd0563c529e 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -27,7 +27,7 @@ generate_metadata as generate_metadata_legacy from pip._internal.operations.install.editable_legacy import \ install as install_editable_legacy -from pip._internal.operations.install.wheel import install_unpacked_wheel +from pip._internal.operations.install.wheel import install_wheel from pip._internal.pyproject import load_pyproject_toml, make_pyproject_path from pip._internal.req.req_uninstall import UninstallPathSet from pip._internal.utils.deprecation import deprecated @@ -774,9 +774,10 @@ def install( return if self.is_wheel: - install_unpacked_wheel( + assert self.local_file_path + install_wheel( self.name, - self.source_dir, + self.local_file_path, scheme=scheme, req_description=str(self.req), pycompile=pycompile,