refactor: switch to httpx client (pdm-project#2709)

Signed-off-by: Frost Ming <[email protected]>

news/2709.refactor.md

@@ -0,0 +1 @@
+Switch to `httpx.Client` for HTTP requests, drop `requests` dependency.

pyproject.toml

@@ -21,20 +21,20 @@ dependencies = [
     "rich>=12.3.0",
     "virtualenv>=20",
     "pyproject-hooks",
-    "requests-toolbelt",
-    "unearth>=0.14.0",
+    "unearth>=0.15.0",
     "dep-logic>=0.2.0,<1.0",
     "findpython>=0.4.0,<1.0.0a0",
     "tomlkit>=0.11.1,<1",
     "shellingham>=1.3.2",
     "python-dotenv>=0.15",
     "resolvelib>=1.0.1",
     "installer<0.8,>=0.7",
-    "cachecontrol[filecache]>=0.13.0",
     "truststore; python_version >= \"3.10\"",
     "tomli>=1.1.0; python_version < \"3.11\"",
     "importlib-resources>=5; python_version < \"3.9\"",
     "importlib-metadata>=3.6; python_version < \"3.10\"",
+    "hishel>=0.0.24,<0.1.0",
+    "msgpack>=1.0",
 ]
 readme = "README.md"
 keywords = ["packaging", "dependency", "workflow"]

src/pdm/cli/commands/publish/__init__.py

@@ -14,7 +14,7 @@
 from pdm.termui import logger
 
 if TYPE_CHECKING:
-    from requests import Response
+    from httpx import Response
 
     from pdm.project import Project
 
@@ -90,7 +90,7 @@ def _make_package(filename: str, signatures: dict[str, str], options: argparse.N
     @staticmethod
     def _skip_upload(response: Response) -> bool:
         status = response.status_code
-        reason = response.reason.lower()
+        reason = response.reason_phrase.lower()
         text = response.text.lower()
 
         # Borrowed from https://github.com/pypa/twine/blob/main/twine/commands/upload.py#L149
@@ -109,21 +109,21 @@ def _skip_upload(response: Response) -> bool:
 
     @staticmethod
     def _check_response(response: Response) -> None:
-        import requests
+        import httpx
 
         message = ""
-        if response.status_code == 410 and "pypi.python.org" in response.url:
+        if response.status_code == 410 and "pypi.python.org" in str(response.url):
             message = (
                 "Uploading to these sites is deprecated. "
                 "Try using https://upload.pypi.org/legacy/ "
                 "(or https://test.pypi.org/legacy/) instead."
             )
-        elif response.status_code == 405 and "pypi.org" in response.url:
+        elif response.status_code == 405 and "pypi.org" in str(response.url):
             message = "It appears you're trying to upload to pypi.org but have an invalid URL."
         else:
             try:
                 response.raise_for_status()
-            except requests.HTTPError as err:
+            except httpx.HTTPStatusError as err:
                 message = str(err)
                 if response.text:
                     logger.debug(response.text)
@@ -149,7 +149,7 @@ def get_repository(project: Project, options: argparse.Namespace) -> Repository:
             config.ca_certs = ca_certs
         if options.verify_ssl is False:
             config.verify_ssl = options.verify_ssl
-        return Repository(project, config.url, config.username, config.password, config.ca_certs, config.verify_ssl)
+        return Repository(project, config)
 
     def handle(self, project: Project, options: argparse.Namespace) -> None:
         hooks = HookManager(project, options.skip)
@@ -172,7 +172,7 @@ def handle(self, project: Project, options: argparse.Namespace) -> None:
             )
             for package in packages:
                 resp = repository.upload(package)
-                logger.debug("Response from %s:\n%s %s", resp.url, resp.status_code, resp.reason)
+                logger.debug("Response from %s:\n%s %s", resp.url, resp.status_code, resp.reason_phrase)
 
                 if options.skip_existing and self._skip_upload(resp):
                     project.core.ui.warn(f"Skipping {package.base_filename} because it appears to already exist")

src/pdm/cli/commands/publish/repository.py

@@ -1,41 +1,50 @@
 from __future__ import annotations
 
 import os
-import pathlib
-from typing import TYPE_CHECKING, Any, Iterable
+from typing import TYPE_CHECKING, Any, Iterable, cast
 from urllib.parse import urlparse, urlunparse
 
+import httpx
 from rich.progress import BarColumn, DownloadColumn, TimeRemainingColumn, TransferSpeedColumn
 
 from pdm import termui
 from pdm.cli.commands.publish.package import PackageFile
 from pdm.exceptions import PdmUsageError
 from pdm.project import Project
 from pdm.project.config import DEFAULT_REPOSITORIES
+from pdm.utils import get_trusted_hosts
 
 if TYPE_CHECKING:
-    from requests import Response
+    from typing import Callable, Self
+
+    from httpx import Response
+    from httpx._multipart import MultipartStream
+
+    from pdm._types import RepositoryConfig
+
+
+class CallbackWrapperStream(httpx.SyncByteStream):
+    def __init__(self, stream: httpx.SyncByteStream, callback: Callable[[Self], Any]) -> None:
+        self._stream = stream
+        self._callback = callback
+        self.bytes_read = 0
+
+    def __iter__(self) -> Iterable[bytes]:
+        for chunk in self._stream:
+            self.bytes_read += len(chunk)
+            self._callback(self)
+            yield chunk
 
 
 class Repository:
-    def __init__(
-        self,
-        project: Project,
-        url: str,
-        username: str | None,
-        password: str | None,
-        ca_certs: str | None,
-        verify_ssl: bool | None = True,
-    ) -> None:
-        self.url = url
-        self.session = project.environment.session
-        if verify_ssl is False:
-            self.session.verify = verify_ssl
-        elif ca_certs is not None:
-            self.session.set_ca_certificates(pathlib.Path(ca_certs))
+    def __init__(self, project: Project, config: RepositoryConfig) -> None:
+        self.url = cast(str, config.url)
+        trusted_hosts = get_trusted_hosts([config])
+        self.session = project.environment._build_session(trusted_hosts, verify=config.ca_certs)
+
         self._credentials_to_save: tuple[str, str, str] | None = None
         self.ui = project.core.ui
-        username, password = self._ensure_credentials(username, password)
+        username, password = self._ensure_credentials(config.username, config.password)
         self.session.auth = (username, password)
 
     def _ensure_credentials(self, username: str | None, password: str | None) -> tuple[str, str]:
@@ -69,7 +78,7 @@ def _get_pypi_token_via_oidc(self) -> str | None:
         if not ACTIONS_ID_TOKEN_REQUEST_TOKEN or not ACTIONS_ID_TOKEN_REQUEST_URL:
             return None
         self.ui.echo("Getting PyPI token via GitHub Actions OIDC...")
-        import requests
+        import httpx
 
         try:
             parsed_url = urlparse(self.url)
@@ -89,7 +98,7 @@ def _get_pypi_token_via_oidc(self) -> str | None:
             resp = self.session.post(mint_token_url, json={"token": oidc_token})
             resp.raise_for_status()
             token = resp.json()["token"]
-        except requests.RequestException:
+        except httpx.HTTPError:
             self.ui.echo("Failed to get PyPI token via GitHub Actions OIDC", err=True)
             return None
         else:
@@ -137,16 +146,13 @@ def get_release_urls(self, packages: list[PackageFile]) -> Iterable[str]:
         return {f"{base}project/{package.metadata['name']}/{package.metadata['version']}/" for package in packages}
 
     def upload(self, package: PackageFile) -> Response:
-        import requests_toolbelt
-
-        payload = package.metadata_dict
-        payload.update(
+        data_fields = package.metadata_dict
+        data_fields.update(
             {
                 ":action": "file_upload",
                 "protocol_version": "1",
             }
         )
-        field_parts = self._convert_to_list_of_tuples(payload)
         with self.ui.make_progress(
             " [progress.percentage]{task.percentage:>3.0f}%",
             BarColumn(),
@@ -162,20 +168,18 @@ def upload(self, package: PackageFile) -> Response:
             progress.console.print(f"Uploading [success]{package.base_filename}")
 
             with open(package.filename, "rb") as fp:
-                field_parts.append(("content", (package.base_filename, fp, "application/octet-stream")))
+                file_fields = [("content", (package.base_filename, fp, "application/octet-stream"))]
 
-                def on_upload(monitor: requests_toolbelt.MultipartEncoderMonitor) -> None:
+                def on_upload(monitor: CallbackWrapperStream) -> None:
                     progress.update(job, completed=monitor.bytes_read)
 
-                monitor = requests_toolbelt.MultipartEncoderMonitor.from_fields(field_parts, callback=on_upload)
-                job = progress.add_task("", total=monitor.len)
-                resp = self.session.post(
-                    self.url,
-                    data=monitor,
-                    headers={"Content-Type": monitor.content_type},
-                    allow_redirects=False,
-                )
-                if resp.status_code < 400 and self._credentials_to_save is not None:
+                request = self.session.build_request("POST", self.url, data=data_fields, files=file_fields)
+                stream = cast("MultipartStream", request.stream)
+                request.stream = CallbackWrapperStream(stream, on_upload)
+
+                job = progress.add_task("", total=stream.get_content_length())
+                resp = self.session.send(request, follow_redirects=False)
+                if not resp.is_error and self._credentials_to_save is not None:
                     self._save_credentials(*self._credentials_to_save)
                     self._credentials_to_save = None
                 return resp

src/pdm/environments/base.py

@@ -12,8 +12,7 @@
 from contextlib import contextmanager
 from functools import cached_property, partial
 from pathlib import Path
-from threading import local
-from typing import TYPE_CHECKING, Generator, cast, no_type_check
+from typing import TYPE_CHECKING, Generator, Mapping, no_type_check
 
 from pdm.exceptions import BuildError, PdmUsageError
 from pdm.models.in_process import get_pep508_environment, get_python_abis, get_uname, sysconfig_get_platform
@@ -23,9 +22,11 @@
 
 if TYPE_CHECKING:
     import unearth
+    from httpx import BaseTransport
+    from httpx._types import CertTypes, VerifyTypes
 
     from pdm._types import RepositoryConfig
-    from pdm.models.session import PDMSession
+    from pdm.models.session import PDMPyPIClient
     from pdm.project import Project
 
 
@@ -72,8 +73,6 @@ def __init__(self, project: Project, *, python: str | None = None) -> None:
         else:
             self._interpreter = PythonInfo.from_path(python)
 
-        self._local_cache = local()
-
     @property
     def is_global(self) -> bool:
         """For backward compatibility, it is opposite to ``is_local``."""
@@ -112,34 +111,48 @@ def target_python(self) -> unearth.TargetPython:
             tp.supported_tags()
         return tp
 
-    def _build_session(self) -> PDMSession:
-        from pdm.models.session import PDMSession
+    def _build_session(
+        self,
+        trusted_hosts: list[str] | None = None,
+        verify: VerifyTypes | None = None,
+        cert: CertTypes | None = None,
+        mounts: Mapping[str, BaseTransport | None] | None = None,
+    ) -> PDMPyPIClient:
+        from pdm.models.session import PDMPyPIClient
 
-        ca_certs = self.project.config.get("pypi.ca_certs")
-        trusted_hosts = get_trusted_hosts(self.project.sources)
-        session = PDMSession(
-            cache_dir=self.project.cache("http"),
-            trusted_hosts=trusted_hosts,
-            ca_certificates=Path(ca_certs) if ca_certs is not None else None,
-            timeout=self.project.config["request_timeout"],
-        )
-        certfn = self.project.config.get("pypi.client_cert")
-        if certfn:
-            keyfn = self.project.config.get("pypi.client_key")
-            session.cert = (Path(certfn), Path(keyfn) if keyfn else None)
+        if trusted_hosts is None:
+            trusted_hosts = get_trusted_hosts(self.project.sources)
 
-        session.auth = self.auth
+        if verify is None:
+            verify = self.project.config.get("pypi.ca_certs")
+
+        if cert is None:
+            certfn = self.project.config.get("pypi.client_cert")
+            keyfn = self.project.config.get("pypi.client_key")
+            if certfn:
+                cert = (certfn, keyfn)
+
+        session_args = {
+            "cache_dir": self.project.cache("http"),
+            "trusted_hosts": trusted_hosts,
+            "timeout": self.project.config["request_timeout"],
+            "auth": self.auth,
+        }
+        if verify is not None:
+            session_args["verify"] = verify
+        if cert is not None:
+            session_args["cert"] = cert
+        if mounts:
+            session_args["mounts"] = mounts
+
+        session = PDMPyPIClient(**session_args)
         self.project.core.exit_stack.callback(session.close)
         return session
 
-    @property
-    def session(self) -> PDMSession:
-        """Build the session and cache it in the thread local storage."""
-        sess = getattr(self._local_cache, "session", None)
-        if sess is None:
-            sess = self._build_session()
-            self._local_cache.session = sess
-        return cast("PDMSession", sess)
+    @cached_property
+    def session(self) -> PDMPyPIClient:
+        """Build the session and cache it."""
+        return self._build_session()
 
     @contextmanager
     def _patch_target_python(self) -> Generator[None, None, None]: