From 2253ed303735c8a68228c2053cffdde7dd97741b Mon Sep 17 00:00:00 2001 From: Damian Shaw Date: Sat, 4 Sep 2021 19:45:19 -0400 Subject: [PATCH 1/5] Prefer backtracking on dependencies involved in the most recent conflict This can significantly reduce the amount of backtracking required, by avoiding backtracking on unrelated packages in the dependency graph. Co-authored-by: Pradyun Gedam --- news/10479.feature.rst | 1 + .../resolution/resolvelib/provider.py | 20 ++++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 news/10479.feature.rst diff --git a/news/10479.feature.rst b/news/10479.feature.rst new file mode 100644 index 00000000000..23eaa6c8faa --- /dev/null +++ b/news/10479.feature.rst @@ -0,0 +1 @@ +When backtracking during dependency resolution, prefer the dependencies which are involved in the most recent conflict. This can significantly reduce the amount of backtracking required. diff --git a/src/pip/_internal/resolution/resolvelib/provider.py b/src/pip/_internal/resolution/resolvelib/provider.py index c2203933e40..85d3b315605 100644 --- a/src/pip/_internal/resolution/resolvelib/provider.py +++ b/src/pip/_internal/resolution/resolvelib/provider.py @@ -66,12 +66,13 @@ def __init__( def identify(self, requirement_or_candidate: Union[Requirement, Candidate]) -> str: return requirement_or_candidate.name - def get_preference( + def get_preference( # type: ignore self, identifier: str, resolutions: Mapping[str, Candidate], candidates: Mapping[str, Iterator[Candidate]], information: Mapping[str, Iterable["PreferenceInformation"]], + backtrack_causes: Sequence["PreferenceInformation"], ) -> "Preference": """Produce a sort key for given requirement based on preference. @@ -132,11 +133,17 @@ def get_preference( # while we work on "proper" branch pruning techniques. delay_this = identifier == "setuptools" + # Prefer the causes of backtracking on the assumption that the problem + # resolving the dependency tree is related to the failures that caused + # the backtracking + backtrack_cause = self.is_backtrack_cause(identifier, backtrack_causes) + return ( not requires_python, delay_this, not direct, not pinned, + not backtrack_cause, inferred_depth, requested_order, not unfree, @@ -195,3 +202,14 @@ def is_satisfied_by(self, requirement: Requirement, candidate: Candidate) -> boo def get_dependencies(self, candidate: Candidate) -> Sequence[Requirement]: with_requires = not self._ignore_dependencies return [r for r in candidate.iter_dependencies(with_requires) if r is not None] + + @staticmethod + def is_backtrack_cause( + identifier: str, backtrack_causes: Sequence["PreferenceInformation"] + ) -> bool: + for backtrack_cause in backtrack_causes: + if identifier == backtrack_cause.requirement.name: + return True + if backtrack_cause.parent and identifier == backtrack_cause.parent.name: + return True + return False From 394a24eb1a5f9af5da7d4d2452ed5fe952de5db2 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sat, 9 Oct 2021 09:26:37 +0100 Subject: [PATCH 2/5] Upgrade resolvelib to 0.8.0 --- news/resolvelib.vendor.rst | 1 + src/pip/_vendor/resolvelib/__init__.py | 2 +- src/pip/_vendor/resolvelib/providers.py | 11 ++++++++++- src/pip/_vendor/resolvelib/resolvers.py | 18 ++++++++++++++---- src/pip/_vendor/vendor.txt | 2 +- 5 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 news/resolvelib.vendor.rst diff --git a/news/resolvelib.vendor.rst b/news/resolvelib.vendor.rst new file mode 100644 index 00000000000..f1d800f7ba7 --- /dev/null +++ b/news/resolvelib.vendor.rst @@ -0,0 +1 @@ +Upgrade resolvelib to 0.8.0 diff --git a/src/pip/_vendor/resolvelib/__init__.py b/src/pip/_vendor/resolvelib/__init__.py index 1bddc2fd4e0..af18ddc9712 100644 --- a/src/pip/_vendor/resolvelib/__init__.py +++ b/src/pip/_vendor/resolvelib/__init__.py @@ -11,7 +11,7 @@ "ResolutionTooDeep", ] -__version__ = "0.7.1" +__version__ = "0.8.0" from .providers import AbstractProvider, AbstractResolver diff --git a/src/pip/_vendor/resolvelib/providers.py b/src/pip/_vendor/resolvelib/providers.py index 4822d166551..7d0a9c22a46 100644 --- a/src/pip/_vendor/resolvelib/providers.py +++ b/src/pip/_vendor/resolvelib/providers.py @@ -9,7 +9,14 @@ def identify(self, requirement_or_candidate): """ raise NotImplementedError - def get_preference(self, identifier, resolutions, candidates, information): + def get_preference( + self, + identifier, + resolutions, + candidates, + information, + backtrack_causes, + ): """Produce a sort key for given requirement based on preference. The preference is defined as "I think this requirement should be @@ -25,6 +32,8 @@ def get_preference(self, identifier, resolutions, candidates, information): Each value is an iterator of candidates. :param information: Mapping of requirement information of each package. Each value is an iterator of *requirement information*. + :param backtrack_causes: Sequence of requirement information that were + the requirements that caused the resolver to most recently backtrack. A *requirement information* instance is a named tuple with two members: diff --git a/src/pip/_vendor/resolvelib/resolvers.py b/src/pip/_vendor/resolvelib/resolvers.py index 42484423c9e..35e00fa90a1 100644 --- a/src/pip/_vendor/resolvelib/resolvers.py +++ b/src/pip/_vendor/resolvelib/resolvers.py @@ -99,7 +99,7 @@ def __init__(self, round_count): # Resolution state in a round. -State = collections.namedtuple("State", "mapping criteria") +State = collections.namedtuple("State", "mapping criteria backtrack_causes") class Resolution(object): @@ -131,6 +131,7 @@ def _push_new_state(self): state = State( mapping=base.mapping.copy(), criteria=base.criteria.copy(), + backtrack_causes=base.backtrack_causes[:], ) self._states.append(state) @@ -185,6 +186,7 @@ def _get_preference(self, name): self.state.criteria, operator.attrgetter("information"), ), + backtrack_causes=self.state.backtrack_causes, ) def _is_current_pin_satisfying(self, name, criterion): @@ -335,7 +337,13 @@ def resolve(self, requirements, max_rounds): self._r.starting() # Initialize the root state. - self._states = [State(mapping=collections.OrderedDict(), criteria={})] + self._states = [ + State( + mapping=collections.OrderedDict(), + criteria={}, + backtrack_causes=[], + ) + ] for r in requirements: try: self._add_to_criteria(self.state.criteria, r, parent=None) @@ -369,11 +377,13 @@ def resolve(self, requirements, max_rounds): # Backtrack if pinning fails. The backtrack process puts us in # an unpinned state, so we can work on it in the next round. success = self._backtrack() + self.state.backtrack_causes[:] = [ + i for c in failure_causes for i in c.information + ] # Dead ends everywhere. Give up. if not success: - causes = [i for c in failure_causes for i in c.information] - raise ResolutionImpossible(causes) + raise ResolutionImpossible(self.state.backtrack_causes) else: # Pinning was successful. Push a new state to do another pin. self._push_new_state() diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index 2260e86fdbb..0b74c2bacc2 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -14,7 +14,7 @@ requests==2.26.0 chardet==4.0.0 idna==3.2 urllib3==1.26.7 -resolvelib==0.7.1 +resolvelib==0.8.0 setuptools==44.0.0 six==1.16.0 tenacity==8.0.1 From c01b5c6d8a4858cf733408b4b020933f902dda9e Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sat, 9 Oct 2021 09:29:21 +0100 Subject: [PATCH 3/5] Update a test for resolvelib 0.8.0 --- tests/unit/resolution_resolvelib/test_provider.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/resolution_resolvelib/test_provider.py b/tests/unit/resolution_resolvelib/test_provider.py index 09b887f09b9..5b0d8d4df84 100644 --- a/tests/unit/resolution_resolvelib/test_provider.py +++ b/tests/unit/resolution_resolvelib/test_provider.py @@ -46,6 +46,7 @@ def test_provider_known_depths(factory: Factory) -> None: resolutions={}, candidates={}, information={root_requirement_name: root_requirement_information}, + backtrack_causes=[], ) assert provider._known_depths == {root_requirement_name: 1.0} @@ -69,6 +70,7 @@ def test_provider_known_depths(factory: Factory) -> None: root_requirement_name: root_requirement_information, transative_requirement_name: transative_package_information, }, + backtrack_causes=[], ) assert provider._known_depths == { transative_requirement_name: 2.0, From 610424f9f8ad1f99d0a48bf9a53e7a9df4242304 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sat, 9 Oct 2021 09:40:21 +0100 Subject: [PATCH 4/5] Quote "PreferenceInformation" to avoid runtime NameError --- tests/unit/resolution_resolvelib/test_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/resolution_resolvelib/test_provider.py b/tests/unit/resolution_resolvelib/test_provider.py index 5b0d8d4df84..a62808741d7 100644 --- a/tests/unit/resolution_resolvelib/test_provider.py +++ b/tests/unit/resolution_resolvelib/test_provider.py @@ -19,7 +19,7 @@ def build_requirement_information( install_requirement = install_req_from_req_string(name) # RequirementInformation is typed as a tuple, but it is a namedtupled. # https://github.com/sarugaku/resolvelib/blob/7bc025aa2a4e979597c438ad7b17d2e8a08a364e/src/resolvelib/resolvers.pyi#L20-L22 - requirement_information: PreferenceInformation = RequirementInformation( + requirement_information: "PreferenceInformation" = RequirementInformation( requirement=SpecifierRequirement(install_requirement), # type: ignore[call-arg] parent=parent, ) From 1e3c127d4a938643aca1bbc25e6581493e316476 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Sat, 9 Oct 2021 09:41:04 +0100 Subject: [PATCH 5/5] Avoid passing `.` to vendoring --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 910e61eb51b..df42af8b8f5 100644 --- a/noxfile.py +++ b/noxfile.py @@ -174,7 +174,7 @@ def vendoring(session: nox.Session) -> None: session.install("vendoring~=1.0.0") if "--upgrade" not in session.posargs: - session.run("vendoring", "sync", ".", "-v") + session.run("vendoring", "sync", "-v") return def pinned_requirements(path: Path) -> Iterator[Tuple[str, str]]: