Merge branch 'main' into joe/warm_cache_in_threadpool

.github/workflows/ci.yml

@@ -91,7 +91,7 @@ jobs:
       - run: git diff --exit-code
 
   tests-unix:
-    name: tests / ${{ matrix.python }} / ${{ matrix.os }}
+    name: tests / ${{ matrix.python.key || matrix.python }} / ${{ matrix.os }}
     runs-on: ${{ matrix.os }}-latest
 
     needs: [packaging, determine-changes]
@@ -109,12 +109,14 @@ jobs:
           - "3.9"
           - "3.10"
           - "3.11"
+          - "3.12"
 
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
+          allow-prereleases: true
 
       - name: Install Ubuntu dependencies
         if: matrix.os == 'Ubuntu'
@@ -129,12 +131,12 @@ jobs:
       # Main check
       - name: Run unit tests
         run: >-
-          nox -s test-${{ matrix.python }} --
+          nox -s test-${{ matrix.python.key || matrix.python }} --
           -m unit
           --verbose --numprocesses auto --showlocals
       - name: Run integration tests
         run: >-
-          nox -s test-${{ matrix.python }} --
+          nox -s test-${{ matrix.python.key || matrix.python }} --
           -m integration
           --verbose --numprocesses auto --showlocals
           --durations=5

.pre-commit-config.yaml

@@ -17,13 +17,13 @@ repos:
     exclude: .patch
 
 - repo: https://github.com/psf/black
-  rev: 23.1.0
+  rev: 23.7.0
   hooks:
   - id: black
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: v0.0.270
+  rev: v0.0.287
   hooks:
     - id: ruff
 

.readthedocs.yml

@@ -6,7 +6,7 @@ build:
     python: "3.11"
 
 sphinx:
-  builder: htmldir
+  builder: dirhtml
   configuration: docs/html/conf.py
 
 python:

README.rst

@@ -19,8 +19,6 @@ We release updates regularly, with a new version every 3 months. Find more detai
 * `Release notes`_
 * `Release process`_
 
-In pip 20.3, we've `made a big improvement to the heart of pip`_; `learn more`_. We want your input, so `sign up for our user experience research studies`_ to help us do it right.
-
 **Note**: pip 21.0, in January 2021, removed Python 2 support, per pip's `Python 2 support policy`_. Please migrate to Python 3.
 
 If you find bugs, need help, or want to talk to the developers, please use our mailing lists or chat rooms:
@@ -49,9 +47,6 @@ rooms, and mailing lists is expected to follow the `PSF Code of Conduct`_.
 .. _Release process: https://pip.pypa.io/en/latest/development/release-process/
 .. _GitHub page: https://github.com/pypa/pip
 .. _Development documentation: https://pip.pypa.io/en/latest/development
-.. _made a big improvement to the heart of pip: https://pyfound.blogspot.com/2020/11/pip-20-3-new-resolver.html
-.. _learn more: https://pip.pypa.io/en/latest/user_guide/#changes-to-the-pip-dependency-resolver-in-20-3-2020
-.. _sign up for our user experience research studies: https://pyfound.blogspot.com/2020/03/new-pip-resolver-to-roll-out-this-year.html
 .. _Python 2 support policy: https://pip.pypa.io/en/latest/development/release-process/#python-2-support
 .. _Issue tracking: https://github.com/pypa/pip/issues
 .. _Discourse channel: https://discuss.python.org/c/packaging

SECURITY.md

@@ -1,3 +1,10 @@
-# Security and Vulnerability Reporting
+# Security Policy
 
-If you find any security issues, please report to [[email protected]](mailto:[email protected])
+## Reporting a Vulnerability
+
+Please read the guidelines on reporting security issues [on the
+official website](https://www.python.org/dev/security/) for
+instructions on how to report a security-related problem to
+the Python Security Response Team responsibly.
+
+To reach the response team, email `security at python dot org`.

docs/html/installation.md

@@ -103,7 +103,7 @@ $ pip install --upgrade pip
 The current version of pip works on:
 
 - Windows, Linux and MacOS.
-- CPython 3.7, 3.8, 3.9, 3.10 and latest PyPy3.
+- CPython 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, and latest PyPy3.
 
 pip is tested to work on the latest patch version of the Python interpreter,
 for each of the minor versions listed above. Previous patch versions are

docs/html/reference/installation-report.md

@@ -56,6 +56,9 @@ package with the following properties:
   URL reference. `false` if the requirements was provided as a name and version
   specifier.
 
+- `is_yanked`: `true` if the requirement was yanked from the index, but was still
+  selected by pip conform to [PEP 592](https://peps.python.org/pep-0592/#installers).
+
 - `download_info`: Information about the artifact (to be) downloaded for installation,
   using the [direct URL data
   structure](https://packaging.python.org/en/latest/specifications/direct-url-data-structure/).
@@ -106,6 +109,7 @@ will produce an output similar to this (metadata abriged for brevity):
         }
       },
       "is_direct": false,
+      "is_yanked": false,
       "requested": true,
       "metadata": {
         "name": "pydantic",
@@ -133,6 +137,7 @@ will produce an output similar to this (metadata abriged for brevity):
         }
       },
       "is_direct": true,
+      "is_yanked": false,
       "requested": true,
       "metadata": {
         "name": "packaging",

docs/html/topics/authentication.md

@@ -68,7 +68,7 @@ man pages][netrc-docs].
 pip supports loading credentials stored in your keyring using the
 {pypi}`keyring` library, which can be enabled py passing `--keyring-provider`
 with a value of `auto`, `disabled`, `import`, or `subprocess`. The default
-value `auto` respects `--no-input` and not query keyring at all if the option
+value `auto` respects `--no-input` and does not query keyring at all if the option
 is used; otherwise it tries the `import`, `subprocess`, and `disabled`
 providers (in this order) and uses the first one that works.
 

docs/html/topics/more-dependency-resolution.md

@@ -8,7 +8,7 @@ and this article is intended to help readers understand what is happening
 ```{note}
 This document is a work in progress. The details included are accurate (at the
 time of writing), but there is additional information, in particular around
-pip's interface with resolvelib, which have not yet been included.
+pip's interface with resolvelib, which has not yet been included.
 
 Contributions to improve this document are welcome.
 ```
@@ -26,7 +26,7 @@ The practical implication of that is that there will always be some situations
 where pip cannot determine what to install in a reasonable length of time. We
 make every effort to ensure that such situations happen rarely, but eliminating
 them altogether isn't even theoretically possible. We'll discuss what options
-yopu have if you hit a problem situation like this a little later.
+you have if you hit a problem situation like this a little later.
 
 ## Python specific issues
 
@@ -136,7 +136,7 @@ operations:
   that satisfy them. This is essentially where the finder interacts with the
   resolver.
 * `is_satisfied_by` - checks if a candidate satisfies a requirement. This is
-  basically the implementation of what a requirement meams.
+  basically the implementation of what a requirement means.
 * `get_dependencies` - get the dependency metadata for a candidate. This is
   the implementation of the process of getting and reading package metadata.
 

docs/requirements.txt

@@ -1,4 +1,4 @@
-sphinx ~= 6.0
+sphinx ~= 7.0
 towncrier
 furo
 myst_parser

news/11394.bugfix.rst

@@ -0,0 +1 @@
+Ignore errors in temporary directory cleanup (show a warning instead).

news/12005.bugfix.rst

@@ -0,0 +1 @@
+Removed uses of ``datetime.datetime.utcnow`` from non-vendored code.

news/12155.process.rst

@@ -0,0 +1,6 @@
+The metadata-fetching log message is moved to the VERBOSE level and now hidden
+by default. The more significant information in this message to most users are
+already available in surrounding logs (the package name and version of the
+metadata being fetched), while the URL to the exact metadata file is generally
+too long and clutters the output. The message can be brought back with
+``--verbose``.

news/12183.trivial.rst

@@ -0,0 +1 @@
+Add test cases for some behaviors of ``install --dry-run`` and ``--use-feature=fast-deps``.

news/12187.bugfix.rst

@@ -0,0 +1 @@
+Fix improper handling of the new onexc argument of ``shutil.rmtree()`` in Python 3.12.

news/12191.bugfix.rst

@@ -0,0 +1 @@
+Prevent downloading sdists twice when PEP 658 metadata is present.

news/12194.trivial.rst

@@ -0,0 +1 @@
+Add lots of comments to the ``BuildTracker``.

news/12204.feature.rst

@@ -0,0 +1 @@
+Improve use of datastructures to make candidate selection 1.6x faster

news/12215.feature.rst

@@ -0,0 +1 @@
+Allow ``pip install --dry-run`` to use platform and ABI overriding options similar to ``--target``.

news/12224.feature.rst

@@ -0,0 +1 @@
+Add ``is_yanked`` boolean entry to the installation report (``--report``) to indicate whether the requirement was yanked from the index, but was still selected by pip conform to PEP 592.

news/12225.bugfix.rst

@@ -0,0 +1 @@
+Filter out yanked links from the available versions error message: "(from versions: 1.0, 2.0, 3.0)" will not contain yanked versions conform PEP 592. The yanked versions (if any) will be mentioned in a separate error message.

news/12254.process.rst

@@ -0,0 +1 @@
+Added reference to `vulnerability reporting guidelines <https://www.python.org/dev/security/>`_ to pip's security policy.

news/4A0C40FF-ABE1-48C7-954C-7C3EB229135F.trivial.rst

@@ -0,0 +1 @@
+Add ruff rules ASYNC,C4,C90,PERF,PLE,PLR for minor optimizations and to set upper limits on code complexity.

news/certifi.vendor.rst

@@ -0,0 +1 @@
+Upgrade certifi to 2023.7.22

noxfile.py

@@ -67,7 +67,7 @@ def should_update_common_wheels() -> bool:
 # -----------------------------------------------------------------------------
 # Development Commands
 # -----------------------------------------------------------------------------
-@nox.session(python=["3.7", "3.8", "3.9", "3.10", "3.11", "pypy3"])
+@nox.session(python=["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "pypy3"])
 def test(session: nox.Session) -> None:
     # Get the common wheels.
     if should_update_common_wheels():
@@ -89,6 +89,7 @@ def test(session: nox.Session) -> None:
         shutil.rmtree(sdist_dir, ignore_errors=True)
 
     # fmt: off
+    session.install("setuptools")
     session.run(
         "python", "setup.py", "sdist", "--formats=zip", "--dist-dir", sdist_dir,
         silent=True,
@@ -351,6 +352,7 @@ def build_dists(session: nox.Session) -> List[str]:
         )
 
     session.log("# Build distributions")
+    session.install("setuptools", "wheel")
     session.run("python", "setup.py", "sdist", "bdist_wheel", silent=True)
     produced_dists = glob.glob("dist/*")
 

pyproject.toml

@@ -74,9 +74,9 @@ webencodings = "https://github.com/SimonSapin/python-webencodings/raw/master/LIC
 
 [tool.ruff]
 extend-exclude = [
+    "_vendor",
     "./build",
     ".scratch",
-    "_vendor",
     "data",
 ]
 ignore = [
@@ -88,21 +88,38 @@ ignore = [
 ]
 line-length = 88
 select = [
+    "ASYNC",
     "B",
+    "C4",
+    "C90",
     "E",
     "F",
-    "W",
     "G",
-    "ISC",
     "I",
+    "ISC",
+    "PERF",
+    "PLE",
+    "PLR0",
+    "W",
 ]
 
-[tool.ruff.per-file-ignores]
-"noxfile.py" = ["G"]
-"tests/*" = ["B011"]
-
 [tool.ruff.isort]
 # We need to explicitly make pip "first party" as it's imported by code in
 # the docs and tests directories.
 known-first-party = ["pip"]
 known-third-party = ["pip._vendor"]
+
+[tool.ruff.mccabe]
+max-complexity = 33  # default is 10
+
+[tool.ruff.per-file-ignores]
+"noxfile.py" = ["G"]
+"src/pip/_internal/*" = ["PERF203"]
+"tests/*" = ["B011"]
+"tests/unit/test_finder.py" = ["C414"]
+
+[tool.ruff.pylint]
+max-args = 15  # default is 5
+max-branches = 28  # default is 12
+max-returns = 13  # default is 6
+max-statements = 134  # default is 50

src/pip/_internal/cache.py

@@ -78,12 +78,10 @@ def _get_candidates(self, link: Link, canonical_package_name: str) -> List[Any]:
         if can_not_cache:
             return []
 
-        candidates = []
         path = self.get_path_for_link(link)
         if os.path.isdir(path):
-            for candidate in os.listdir(path):
-                candidates.append((candidate, path))
-        return candidates
+            return [(candidate, path) for candidate in os.listdir(path)]
+        return []
 
     def get_path_for_link(self, link: Link) -> str:
         """Return a directory to store cached items in for link."""

src/pip/_internal/cli/autocompletion.py

@@ -71,8 +71,9 @@ def autocomplete() -> None:
 
         for opt in subcommand.parser.option_list_all:
             if opt.help != optparse.SUPPRESS_HELP:
-                for opt_str in opt._long_opts + opt._short_opts:
-                    options.append((opt_str, opt.nargs))
+                options += [
+                    (opt_str, opt.nargs) for opt_str in opt._long_opts + opt._short_opts
+                ]
 
         # filter out previously specified options from available options
         prev_opts = [x.split("=")[0] for x in cwords[1 : cword - 1]]

src/pip/_internal/cli/cmdoptions.py

@@ -92,10 +92,10 @@ def check_dist_restriction(options: Values, check_target: bool = False) -> None:
         )
 
     if check_target:
-        if dist_restriction_set and not options.target_dir:
+        if not options.dry_run and dist_restriction_set and not options.target_dir:
             raise CommandError(
                 "Can not use any platform or abi specific options unless "
-                "installing via '--target'"
+                "installing via '--target' or using '--dry-run'"
             )
 
 

src/pip/_internal/commands/cache.py

@@ -3,10 +3,10 @@
 from optparse import Values
 from typing import Any, List
 
-import pip._internal.utils.filesystem as filesystem
 from pip._internal.cli.base_command import Command
 from pip._internal.cli.status_codes import ERROR, SUCCESS
 from pip._internal.exceptions import CommandError, PipError
+from pip._internal.utils import filesystem
 from pip._internal.utils.logging import getLogger
 
 logger = getLogger(__name__)
@@ -151,14 +151,8 @@ def format_for_human(self, files: List[str]) -> None:
         logger.info("\n".join(sorted(results)))
 
     def format_for_abspath(self, files: List[str]) -> None:
-        if not files:
-            return
-
-        results = []
-        for filename in files:
-            results.append(filename)
-
-        logger.info("\n".join(sorted(results)))
+        if files:
+            logger.info("\n".join(sorted(files)))
 
     def remove_cache_items(self, options: Values, args: List[Any]) -> None:
         if len(args) > 1:

src/pip/_internal/commands/debug.py

@@ -105,7 +105,7 @@ def show_tags(options: Values) -> None:
     tag_limit = 10
 
     target_python = make_target_python(options)
-    tags = target_python.get_tags()
+    tags = target_python.get_sorted_tags()
 
     # Display the target options that were explicitly provided.
     formatted_target = target_python.format_given()
@@ -134,10 +134,9 @@ def show_tags(options: Values) -> None:
 
 
 def ca_bundle_info(config: Configuration) -> str:
-    levels = set()
-    for key, _ in config.items():
-        levels.add(key.split(".")[0])
-
+    # Ruff misidentifies config as a dict.
+    # Configuration does not have support the mapping interface.
+    levels = {key.split(".", 1)[0] for key, _ in config.items()}  # noqa: PERF102
     if not levels:
         return "Not specified"
 

src/pip/_internal/commands/list.py

@@ -297,7 +297,7 @@ def output_package_listing_columns(
 
         # Create and add a separator.
         if len(data) > 0:
-            pkg_strings.insert(1, " ".join(map(lambda x: "-" * x, sizes)))
+            pkg_strings.insert(1, " ".join("-" * x for x in sizes))
 
         for val in pkg_strings:
             write_output(val)