diff --git a/news/5a7a341b-ca90-476b-8d4b-c40cef58b444.trivial b/news/5a7a341b-ca90-476b-8d4b-c40cef58b444.trivial new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/pip/_internal/network/download.py b/src/pip/_internal/network/download.py index 44f9985a32b..38828216424 100644 --- a/src/pip/_internal/network/download.py +++ b/src/pip/_internal/network/download.py @@ -8,7 +8,7 @@ from pip._vendor.requests.models import CONTENT_CHUNK_SIZE from pip._internal.cli.progress_bars import DownloadProgressProvider -from pip._internal.exceptions import NetworkConnectionError +from pip._internal.exceptions import HashMismatch, NetworkConnectionError from pip._internal.models.index import PyPI from pip._internal.network.cache import is_from_cache from pip._internal.network.utils import ( @@ -21,15 +21,19 @@ redact_auth_from_url, splitext, ) +from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Iterable, Optional + from typing import Iterable, Optional, Tuple from pip._vendor.requests.models import Response from pip._internal.models.link import Link from pip._internal.network.session import PipSession + from pip._internal.utils.hashes import Hashes + + File = Tuple[str, Optional[str]] logger = logging.getLogger(__name__) @@ -133,25 +137,31 @@ def _get_http_response_filename(resp, link): return filename -def _http_get_download(session, link): - # type: (PipSession, Link) -> Response - target_url = link.url.split('#', 1)[0] - resp = session.get(target_url, headers=HEADERS, stream=True) - raise_for_status(resp) - return resp +def check_download_dir(link, location, hashes): + # type: (Link, str, Optional[Hashes]) -> Optional[str] + """Check location for previously downloaded file with correct hash. + If a correct file is found return its path else None. + """ + download_path = os.path.join(location, link.filename) -class Download(object): - def __init__( - self, - response, # type: Response - filename, # type: str - chunks, # type: Iterable[bytes] - ): - # type: (...) -> None - self.response = response - self.filename = filename - self.chunks = chunks + if not os.path.exists(download_path): + return None + + # If already downloaded, does its hash match? + logger.info('File was already downloaded %s', download_path) + if hashes: + try: + hashes.check_against_path(download_path) + except HashMismatch: + logger.warning( + 'Previously-downloaded file %s has bad hash. ' + 'Re-downloading.', + download_path + ) + os.unlink(download_path) + return None + return download_path class Downloader(object): @@ -163,11 +173,14 @@ def __init__( # type: (...) -> None self._session = session self._progress_bar = progress_bar + self._tmpdir = TempDirectory(kind='unpack', globally_managed=True) - def __call__(self, link): - # type: (Link) -> Download + def _download(self, link, location): + # type: (Link, str) -> File + url, sep, checksum = link.url.partition('#') + response = self._session.get(url, headers=HEADERS, stream=True) try: - resp = _http_get_download(self._session, link) + raise_for_status(response) except NetworkConnectionError as e: assert e.response is not None logger.critical( @@ -175,8 +188,25 @@ def __call__(self, link): ) raise - return Download( - resp, - _get_http_response_filename(resp, link), - _prepare_download(resp, link, self._progress_bar), - ) + chunks = _prepare_download(response, link, self._progress_bar) + filename = _get_http_response_filename(response, link) + file_path = os.path.join(location, filename) + + with open(file_path, 'wb') as content_file: + for chunk in chunks: + content_file.write(chunk) + return file_path, response.headers.get('content-type', '') + + def __call__(self, link, location=None, hashes=None): + # type: (Link, Optional[str], Optional[Hashes]) -> File + if location is None: + location = self._tmpdir.path + file_path = check_download_dir(link, location, hashes) + if file_path is not None: + content_type = mimetypes.guess_type(file_path)[0] + return file_path, content_type + + file_path, content_type = self._download(link, location) + if hashes: + hashes.check_against_path(file_path) + return file_path, content_type diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index a5455fcc8e7..e5dc2cee810 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -17,13 +17,13 @@ from pip._internal.distributions.installed import InstalledDistribution from pip._internal.exceptions import ( DirectoryUrlHashUnsupported, - HashMismatch, HashUnpinned, InstallationError, NetworkConnectionError, PreviousBuildDirError, VcsHashUnsupported, ) +from pip._internal.network.download import check_download_dir from pip._internal.utils.filesystem import copy2_fixed from pip._internal.utils.hashes import MissingHashes from pip._internal.utils.logging import indent_log @@ -33,15 +33,12 @@ path_to_display, rmtree, ) -from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.unpacking import unpack_file from pip._internal.vcs import vcs if MYPY_CHECK_RUNNING: - from typing import ( - Callable, List, Optional, Tuple, - ) + from typing import Callable, List, Optional from mypy_extensions import TypedDict @@ -101,7 +98,7 @@ def unpack_vcs_link(link, location): class File(object): def __init__(self, path, content_type): - # type: (str, str) -> None + # type: (str, Optional[str]) -> None self.path = path self.content_type = content_type @@ -113,11 +110,10 @@ def get_http_url( hashes=None, # type: Optional[Hashes] ): # type: (...) -> File - temp_dir = TempDirectory(kind="unpack", globally_managed=True) # If a download dir is specified, is the file already downloaded there? already_downloaded_path = None if download_dir: - already_downloaded_path = _check_download_dir( + already_downloaded_path = check_download_dir( link, download_dir, hashes ) @@ -126,9 +122,7 @@ def get_http_url( content_type = mimetypes.guess_type(from_path)[0] else: # let's download to a tmp dir - from_path, content_type = _download_http_url( - link, downloader, temp_dir.path, hashes - ) + from_path, content_type = downloader(link, hashes=hashes) return File(from_path, content_type) @@ -197,7 +191,7 @@ def get_file_url( # If a download dir is specified, is the file already there and valid? already_downloaded_path = None if download_dir: - already_downloaded_path = _check_download_dir( + already_downloaded_path = check_download_dir( link, download_dir, hashes ) @@ -229,91 +223,37 @@ def unpack_url( # type: (...) -> Optional[File] """Unpack link into location, downloading if required. - :param hashes: A Hashes object, one of whose embedded hashes must match, - or HashMismatch will be raised. If the Hashes is empty, no matches are - required, and unhashable types of requirements (like VCS ones, which - would ordinarily raise HashUnsupported) are allowed. + One of embedded hashes in the given Hashes object must match, + or HashMismatch will be raised. If the Hashes is empty, no matches + are required, and unhashable types of requirements (like VCS ones, + which would ordinarily raise HashUnsupported) are allowed. """ - # non-editable vcs urls + # Non-editable VCS URL if link.is_vcs: unpack_vcs_link(link, location) return None - # If it's a url to a local directory + # URL to a local directory if link.is_existing_dir(): if os.path.isdir(location): rmtree(location) _copy_source_tree(link.file_path, location) return None - # file urls if link.is_file: file = get_file_url(link, download_dir, hashes=hashes) - - # http urls else: - file = get_http_url( - link, - downloader, - download_dir, - hashes=hashes, - ) + file = get_http_url(link, downloader, download_dir, hashes=hashes) - # unpack the archive to the build dir location. even when only downloading - # archives, they have to be unpacked to parse dependencies, except wheels + # Unpack the archive to the build directory unless it is a wheel. + # Even if the command is download, archives still have to be + # unpacked to parse dependencies. if not link.is_wheel: unpack_file(file.path, location, file.content_type) return file -def _download_http_url( - link, # type: Link - downloader, # type: Downloader - temp_dir, # type: str - hashes, # type: Optional[Hashes] -): - # type: (...) -> Tuple[str, str] - """Download link url into temp_dir using provided session""" - download = downloader(link) - - file_path = os.path.join(temp_dir, download.filename) - with open(file_path, 'wb') as content_file: - for chunk in download.chunks: - content_file.write(chunk) - - if hashes: - hashes.check_against_path(file_path) - - return file_path, download.response.headers.get('content-type', '') - - -def _check_download_dir(link, download_dir, hashes): - # type: (Link, str, Optional[Hashes]) -> Optional[str] - """ Check download_dir for previously downloaded file with correct hash - If a correct file is found return its path else None - """ - download_path = os.path.join(download_dir, link.filename) - - if not os.path.exists(download_path): - return None - - # If already downloaded, does its hash match? - logger.info('File was already downloaded %s', download_path) - if hashes: - try: - hashes.check_against_path(download_path) - except HashMismatch: - logger.warning( - 'Previously-downloaded file %s has bad hash. ' - 'Re-downloading.', - download_path - ) - os.unlink(download_path) - return None - return download_path - - class RequirementPreparer(object): """Prepares a Requirement """ diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index c289bb5839c..57a4184974b 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -114,7 +114,7 @@ def make_install_req_from_dist(dist, template): return ireq -class _InstallRequirementBackedCandidate(Candidate): +class InstallRequirementBackedCandidate(Candidate): """A candidate backed by an ``InstallRequirement``. This represents a package request with the target not being already @@ -141,7 +141,7 @@ def __init__( version=None, # type: Optional[_BaseVersion] ): # type: (...) -> None - self._link = link + self.link = link self._source_link = source_link self._factory = factory self._ireq = ireq @@ -154,17 +154,17 @@ def __repr__(self): # type: () -> str return "{class_name}({link!r})".format( class_name=self.__class__.__name__, - link=str(self._link), + link=str(self.link), ) def __hash__(self): # type: () -> int - return hash((self.__class__, self._link)) + return hash((self.__class__, self.link)) def __eq__(self, other): # type: (Any) -> bool if isinstance(other, self.__class__): - return self._link == other._link + return self.link == other.link return False # Needed for Python 2, which does not implement this by default @@ -194,11 +194,8 @@ def version(self): def format_for_error(self): # type: () -> str - return "{} {} (from {})".format( - self.name, - self.version, - self._link.file_path if self._link.is_file else self._link - ) + origin = self.link.file_path if self.link.is_file else self.link + return "{} {} (from {})".format(self.name, self.version, origin) def _prepare_abstract_distribution(self): # type: () -> AbstractDistribution @@ -237,7 +234,7 @@ def _fetch_metadata(self): """Fetch metadata, using lazy wheel if possible.""" preparer = self._factory.preparer use_lazy_wheel = self._factory.use_lazy_wheel - remote_wheel = self._link.is_wheel and not self._link.is_file + remote_wheel = self.link.is_wheel and not self.link.is_file if use_lazy_wheel and remote_wheel and not preparer.require_hashes: assert self._name is not None logger.info('Collecting %s', self._ireq.req or self._ireq) @@ -247,7 +244,7 @@ def _fetch_metadata(self): 'Obtaining dependency information from %s %s', self._name, self._version, ) - url = self._link.url.split('#', 1)[0] + url, sep, checksum = self.link.url.partition('#') session = preparer.downloader._session self._dist = dist_from_wheel_url(self._name, url, session) self._check_metadata_consistency() @@ -286,12 +283,12 @@ def iter_dependencies(self): yield python_dep def get_install_requirement(self): - # type: () -> Optional[InstallRequirement] + # type: () -> InstallRequirement self._prepare() return self._ireq -class LinkCandidate(_InstallRequirementBackedCandidate): +class LinkCandidate(InstallRequirementBackedCandidate): is_editable = False def __init__( @@ -331,7 +328,7 @@ def _prepare_abstract_distribution(self): ) -class EditableCandidate(_InstallRequirementBackedCandidate): +class EditableCandidate(InstallRequirementBackedCandidate): is_editable = True def __init__( @@ -426,7 +423,7 @@ def iter_dependencies(self): yield self._factory.make_requirement_from_spec(str(r), self._ireq) def get_install_requirement(self): - # type: () -> Optional[InstallRequirement] + # type: () -> None return None @@ -547,7 +544,7 @@ def iter_dependencies(self): yield requirement def get_install_requirement(self): - # type: () -> Optional[InstallRequirement] + # type: () -> None # We don't return anything here, because we always # depend on the base candidate, and we'll get the # install requirement from that. @@ -590,5 +587,5 @@ def iter_dependencies(self): return () def get_install_requirement(self): - # type: () -> Optional[InstallRequirement] + # type: () -> None return None diff --git a/src/pip/_internal/resolution/resolvelib/resolver.py b/src/pip/_internal/resolution/resolvelib/resolver.py index 43ea248632d..4ddb52e0fdb 100644 --- a/src/pip/_internal/resolution/resolvelib/resolver.py +++ b/src/pip/_internal/resolution/resolvelib/resolver.py @@ -10,6 +10,9 @@ from pip._internal.req.req_install import check_invalid_constraint_type from pip._internal.req.req_set import RequirementSet from pip._internal.resolution.base import BaseResolver +from pip._internal.resolution.resolvelib.candidates import ( + InstallRequirementBackedCandidate, +) from pip._internal.resolution.resolvelib.provider import PipProvider from pip._internal.utils.misc import dist_is_editable from pip._internal.utils.typing import MYPY_CHECK_RUNNING @@ -120,16 +123,41 @@ def resolve(self, root_reqs, check_supported_wheels): self._result = resolver.resolve( requirements, max_rounds=try_to_avoid_resolution_too_deep, ) - except ResolutionImpossible as e: error = self.factory.get_installation_error(e) six.raise_from(error, e) + return self._make_req_set( + self._get_ireq_backed_candidates(), + check_supported_wheels, + ) + + def _get_ireq_backed_candidates(self): + # type: () -> List[InstallRequirementBackedCandidate] + """Return list of pinned, InstallRequirement-backed candidates. + + The candidates returned by this method + must have everything ready for installation. + """ + assert self._result is not None, "must call resolve() first" + candidates = [ + candidate for candidate in self._result.mapping.values() + if isinstance(candidate, InstallRequirementBackedCandidate) + ] + if self.factory.use_lazy_wheel: + for candidate in candidates: + self.factory.preparer.downloader(candidate.link) + return candidates + + def _make_req_set( + self, + candidates, # type: List[InstallRequirementBackedCandidate] + check_supported_wheels, # type: bool + ): + # type: (...) -> RequirementSet req_set = RequirementSet(check_supported_wheels=check_supported_wheels) - for candidate in self._result.mapping.values(): + for candidate in candidates: ireq = candidate.get_install_requirement() - if ireq is None: - continue # Check if there is already an installation under the same name, # and set a flag for later stages to uninstall it, if needed. diff --git a/tests/unit/test_operations_prepare.py b/tests/unit/test_operations_prepare.py index 41d8be260d3..d2e4d609107 100644 --- a/tests/unit/test_operations_prepare.py +++ b/tests/unit/test_operations_prepare.py @@ -10,11 +10,7 @@ from pip._internal.models.link import Link from pip._internal.network.download import Downloader from pip._internal.network.session import PipSession -from pip._internal.operations.prepare import ( - _copy_source_tree, - _download_http_url, - unpack_url, -) +from pip._internal.operations.prepare import _copy_source_tree, unpack_url from pip._internal.utils.hashes import Hashes from pip._internal.utils.urls import path_to_url from tests.lib.filesystem import ( @@ -83,12 +79,7 @@ def test_download_http_url__no_directory_traversal(mock_raise_for_status, download_dir = tmpdir.joinpath('download') os.mkdir(download_dir) - file_path, content_type = _download_http_url( - link, - downloader, - download_dir, - hashes=None, - ) + file_path, content_type = downloader(link, download_dir) # The file should be downloaded to download_dir. actual = os.listdir(download_dir) assert actual == ['out_dir_file']