Skip to content

Commit

Permalink
Merge pull request #8588 from McSinyx/fast-deps
Browse files Browse the repository at this point in the history
Use lazy wheel to obtain dep info for new resolver
  • Loading branch information
pradyunsg authored Jul 24, 2020
2 parents 43485f5 + 4efae5c commit 89a51a6
Show file tree
Hide file tree
Showing 9 changed files with 124 additions and 16 deletions.
3 changes: 3 additions & 0 deletions news/8588.feature
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Allow the new resolver to obtain dependency information through wheels
lazily downloaded using HTTP range requests. To enable this feature,
invoke ``pip`` with ``--use-feature=fast-deps``.
2 changes: 1 addition & 1 deletion src/pip/_internal/cli/cmdoptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -913,7 +913,7 @@ def check_list_path_option(options):
metavar='feature',
action='append',
default=[],
choices=['2020-resolver'],
choices=['2020-resolver', 'fast-deps'],
help='Enable new functionality, that may be backward incompatible.',
) # type: Callable[..., Option]

Expand Down
1 change: 1 addition & 0 deletions src/pip/_internal/cli/req_command.py
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,7 @@ def make_resolver(
force_reinstall=force_reinstall,
upgrade_strategy=upgrade_strategy,
py_version_info=py_version_info,
lazy_wheel='fast-deps' in options.features_enabled,
)
import pip._internal.resolution.legacy.resolver
return pip._internal.resolution.legacy.resolver.Resolver(
Expand Down
61 changes: 46 additions & 15 deletions src/pip/_internal/resolution/resolvelib/candidates.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,22 @@
import logging
import sys

from pip._vendor.contextlib2 import suppress
from pip._vendor.packaging.specifiers import InvalidSpecifier, SpecifierSet
from pip._vendor.packaging.utils import canonicalize_name
from pip._vendor.packaging.version import Version

from pip._internal.exceptions import HashError, MetadataInconsistent
from pip._internal.network.lazy_wheel import (
HTTPRangeRequestUnsupported,
dist_from_wheel_url,
)
from pip._internal.req.constructors import (
install_req_from_editable,
install_req_from_line,
)
from pip._internal.req.req_install import InstallRequirement
from pip._internal.utils.logging import indent_log
from pip._internal.utils.misc import dist_is_editable, normalize_version_info
from pip._internal.utils.packaging import get_requires_python
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
Expand Down Expand Up @@ -142,6 +148,7 @@ def __init__(
self._name = name
self._version = version
self._dist = None # type: Optional[Distribution]
self._prepared = False

def __repr__(self):
# type: () -> str
Expand Down Expand Up @@ -197,11 +204,23 @@ def _prepare_abstract_distribution(self):
# type: () -> AbstractDistribution
raise NotImplementedError("Override in subclass")

def _check_metadata_consistency(self):
# type: () -> None
"""Check for consistency of project name and version of dist."""
# TODO: (Longer term) Rather than abort, reject this candidate
# and backtrack. This would need resolvelib support.
dist = self._dist # type: Distribution
name = canonicalize_name(dist.project_name)
if self._name is not None and self._name != name:
raise MetadataInconsistent(self._ireq, "name", dist.project_name)
version = dist.parsed_version
if self._version is not None and self._version != version:
raise MetadataInconsistent(self._ireq, "version", dist.version)

def _prepare(self):
# type: () -> None
if self._dist is not None:
if self._prepared:
return

try:
abstract_dist = self._prepare_abstract_distribution()
except HashError as e:
Expand All @@ -210,24 +229,36 @@ def _prepare(self):

self._dist = abstract_dist.get_pkg_resources_distribution()
assert self._dist is not None, "Distribution already installed"
self._check_metadata_consistency()
self._prepared = True

# TODO: (Longer term) Rather than abort, reject this candidate
# and backtrack. This would need resolvelib support.
name = canonicalize_name(self._dist.project_name)
if self._name is not None and self._name != name:
raise MetadataInconsistent(
self._ireq, "name", self._dist.project_name,
)
version = self._dist.parsed_version
if self._version is not None and self._version != version:
raise MetadataInconsistent(
self._ireq, "version", self._dist.version,
)
def _fetch_metadata(self):
# type: () -> None
"""Fetch metadata, using lazy wheel if possible."""
preparer = self._factory.preparer
use_lazy_wheel = self._factory.use_lazy_wheel
remote_wheel = self._link.is_wheel and not self._link.is_file
if use_lazy_wheel and remote_wheel and not preparer.require_hashes:
assert self._name is not None
logger.info('Collecting %s', self._ireq.req or self._ireq)
# If HTTPRangeRequestUnsupported is raised, fallback silently.
with indent_log(), suppress(HTTPRangeRequestUnsupported):
logger.info(
'Obtaining dependency information from %s %s',
self._name, self._version,
)
url = self._link.url.split('#', 1)[0]
session = preparer.downloader._session
self._dist = dist_from_wheel_url(self._name, url, session)
self._check_metadata_consistency()
if self._dist is None:
self._prepare()

@property
def dist(self):
# type: () -> Distribution
self._prepare()
if self._dist is None:
self._fetch_metadata()
return self._dist

def _get_requires_python_specifier(self):
Expand Down
2 changes: 2 additions & 0 deletions src/pip/_internal/resolution/resolvelib/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ def __init__(
ignore_installed, # type: bool
ignore_requires_python, # type: bool
py_version_info=None, # type: Optional[Tuple[int, ...]]
lazy_wheel=False, # type: bool
):
# type: (...) -> None
self._finder = finder
Expand All @@ -92,6 +93,7 @@ def __init__(
self._use_user_site = use_user_site
self._force_reinstall = force_reinstall
self._ignore_requires_python = ignore_requires_python
self.use_lazy_wheel = lazy_wheel

self._link_candidate_cache = {} # type: Cache[LinkCandidate]
self._editable_candidate_cache = {} # type: Cache[EditableCandidate]
Expand Down
9 changes: 9 additions & 0 deletions src/pip/_internal/resolution/resolvelib/resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,16 @@ def __init__(
force_reinstall, # type: bool
upgrade_strategy, # type: str
py_version_info=None, # type: Optional[Tuple[int, ...]]
lazy_wheel=False, # type: bool
):
super(Resolver, self).__init__()
if lazy_wheel:
logger.warning(
'pip is using lazily downloaded wheels using HTTP '
'range requests to obtain dependency information. '
'This experimental feature is enabled through '
'--use-feature=fast-deps and it is not ready for production.'
)

assert upgrade_strategy in self._allowed_strategies

Expand All @@ -64,6 +72,7 @@ def __init__(
ignore_installed=ignore_installed,
ignore_requires_python=ignore_requires_python,
py_version_info=py_version_info,
lazy_wheel=lazy_wheel,
)
self.ignore_dependencies = ignore_dependencies
self.upgrade_strategy = upgrade_strategy
Expand Down
9 changes: 9 additions & 0 deletions tests/data/packages/requiresPaste/pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
[build-system]
requires = ['flit_core >=2,<4']
build-backend = 'flit_core.buildapi'

[tool.flit.metadata]
module = 'requiresPaste'
author = 'A. Random Developer'
author-email = '[email protected]'
requires = ['Paste==3.4.2']
3 changes: 3 additions & 0 deletions tests/data/packages/requiresPaste/requiresPaste.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
"""Module requiring Paste to test dependencies download of pip wheel."""

__version__ = '3.1.4'
50 changes: 50 additions & 0 deletions tests/functional/test_fast_deps.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import fnmatch
import json
from os.path import basename

from pip._vendor.packaging.utils import canonicalize_name
from pytest import mark


def pip(script, command, requirement):
return script.pip(
command, '--prefer-binary', '--no-cache-dir',
'--use-feature=fast-deps', requirement,
allow_stderr_warning=True,
)


def assert_installed(script, names):
list_output = json.loads(script.pip('list', '--format=json').stdout)
installed = {canonicalize_name(item['name']) for item in list_output}
assert installed.issuperset(map(canonicalize_name, names))


@mark.network
@mark.parametrize(('requirement', 'expected'), (
('Paste==3.4.2', ('Paste', 'six')),
('Paste[flup]==3.4.2', ('Paste', 'six', 'flup')),
))
def test_install_from_pypi(requirement, expected, script):
pip(script, 'install', requirement)
assert_installed(script, expected)


@mark.network
@mark.parametrize(('requirement', 'expected'), (
('Paste==3.4.2', ('Paste-3.4.2-*.whl', 'six-*.whl')),
('Paste[flup]==3.4.2', ('Paste-3.4.2-*.whl', 'six-*.whl', 'flup-*')),
))
def test_download_from_pypi(requirement, expected, script):
result = pip(script, 'download', requirement)
created = list(map(basename, result.files_created))
assert all(fnmatch.filter(created, f) for f in expected)


@mark.network
def test_build_wheel_with_deps(data, script):
result = pip(script, 'wheel', data.packages/'requiresPaste')
created = list(map(basename, result.files_created))
assert fnmatch.filter(created, 'requiresPaste-3.1.4-*.whl')
assert fnmatch.filter(created, 'Paste-3.4.2-*.whl')
assert fnmatch.filter(created, 'six-*.whl')

0 comments on commit 89a51a6

Please sign in to comment.