diff --git a/src/poetry/config/config.py b/src/poetry/config/config.py index 3100fab7dee..cbd89ebf614 100644 --- a/src/poetry/config/config.py +++ b/src/poetry/config/config.py @@ -226,6 +226,22 @@ def virtualenvs_path(self) -> Path: path = Path(self.get("cache-dir")) / "virtualenvs" return Path(path).expanduser() + @property + def installer_max_workers(self) -> int: + # This should be directly handled by ThreadPoolExecutor + # however, on some systems the number of CPUs cannot be determined + # (it raises a NotImplementedError), so, in this case, we assume + # that the system only has one CPU. + try: + default_max_workers = (os.cpu_count() or 1) + 4 + except NotImplementedError: + default_max_workers = 5 + + desired_max_workers = self.get("installer.max-workers") + if desired_max_workers is None: + return default_max_workers + return min(default_max_workers, int(desired_max_workers)) + def get(self, setting_name: str, default: Any = None) -> Any: """ Retrieve a setting value. diff --git a/src/poetry/console/commands/init.py b/src/poetry/console/commands/init.py index d7dc3bbeea3..cf59517b3a6 100644 --- a/src/poetry/console/commands/init.py +++ b/src/poetry/console/commands/init.py @@ -489,6 +489,7 @@ def _get_pool(self) -> RepositoryPool: if self._pool is None: self._pool = RepositoryPool() - self._pool.add_repository(PyPiRepository()) + pool_size = self.poetry.config.installer_max_workers + self._pool.add_repository(PyPiRepository(pool_size=pool_size)) return self._pool diff --git a/src/poetry/factory.py b/src/poetry/factory.py index 828ce26bc24..15bd004e747 100644 --- a/src/poetry/factory.py +++ b/src/poetry/factory.py @@ -224,12 +224,14 @@ def create_package_source( except KeyError: raise InvalidSourceError("Missing [name] in source.") + pool_size = config.installer_max_workers + if name.lower() == "pypi": if "url" in source: raise InvalidSourceError( "The PyPI repository cannot be configured with a custom url." ) - return PyPiRepository(disable_cache=disable_cache) + return PyPiRepository(disable_cache=disable_cache, pool_size=pool_size) try: url = source["url"] @@ -246,6 +248,7 @@ def create_package_source( url, config=config, disable_cache=disable_cache, + pool_size=pool_size, ) @classmethod diff --git a/src/poetry/installation/executor.py b/src/poetry/installation/executor.py index a0ebcbcf910..fa1f986c083 100644 --- a/src/poetry/installation/executor.py +++ b/src/poetry/installation/executor.py @@ -4,7 +4,6 @@ import csv import itertools import json -import os import threading from concurrent.futures import ThreadPoolExecutor @@ -73,9 +72,7 @@ def __init__( parallel = config.get("installer.parallel", True) if parallel: - self._max_workers = self._get_max_workers( - desired_max_workers=config.get("installer.max-workers") - ) + self._max_workers = config.installer_max_workers else: self._max_workers = 1 @@ -223,21 +220,6 @@ def execute(self, operations: list[Operation]) -> int: return 1 if self._shutdown else 0 - @staticmethod - def _get_max_workers(desired_max_workers: int | None = None) -> int: - # This should be directly handled by ThreadPoolExecutor - # however, on some systems the number of CPUs cannot be determined - # (it raises a NotImplementedError), so, in this case, we assume - # that the system only has one CPU. - try: - default_max_workers = (os.cpu_count() or 1) + 4 - except NotImplementedError: - default_max_workers = 5 - - if desired_max_workers is None: - return default_max_workers - return min(default_max_workers, desired_max_workers) - def _write(self, operation: Operation, line: str) -> None: if not self.supports_fancy_output() or not self._should_write_operation( operation diff --git a/src/poetry/repositories/http_repository.py b/src/poetry/repositories/http_repository.py index ebf8b1a4d77..826703a3760 100644 --- a/src/poetry/repositories/http_repository.py +++ b/src/poetry/repositories/http_repository.py @@ -11,6 +11,7 @@ from typing import Iterator import requests +import requests.adapters from poetry.core.constraints.version import parse_constraint from poetry.core.packages.dependency import Dependency @@ -44,6 +45,7 @@ def __init__( url: str, config: Config | None = None, disable_cache: bool = False, + pool_size: int = requests.adapters.DEFAULT_POOLSIZE, ) -> None: super().__init__(name, disable_cache, config) self._url = url @@ -51,6 +53,7 @@ def __init__( config=config, cache_id=name, disable_cache=disable_cache, + pool_size=pool_size, ) self._authenticator.add_repository(name, url) self.get_page = functools.lru_cache(maxsize=None)(self._get_page) diff --git a/src/poetry/repositories/legacy_repository.py b/src/poetry/repositories/legacy_repository.py index 2c23a9e281d..a6557eeb7d8 100644 --- a/src/poetry/repositories/legacy_repository.py +++ b/src/poetry/repositories/legacy_repository.py @@ -3,6 +3,8 @@ from typing import TYPE_CHECKING from typing import Any +import requests.adapters + from poetry.core.packages.package import Package from poetry.inspection.info import PackageInfo @@ -27,11 +29,12 @@ def __init__( url: str, config: Config | None = None, disable_cache: bool = False, + pool_size: int = requests.adapters.DEFAULT_POOLSIZE, ) -> None: if name == "pypi": raise ValueError("The name [pypi] is reserved for repositories") - super().__init__(name, url.rstrip("/"), config, disable_cache) + super().__init__(name, url.rstrip("/"), config, disable_cache, pool_size) @property def packages(self) -> list[Package]: diff --git a/src/poetry/repositories/pypi_repository.py b/src/poetry/repositories/pypi_repository.py index e23a2f96976..66f6fb6d86c 100644 --- a/src/poetry/repositories/pypi_repository.py +++ b/src/poetry/repositories/pypi_repository.py @@ -7,6 +7,7 @@ from typing import Any import requests +import requests.adapters from cachecontrol.controller import logger as cache_control_logger from poetry.core.packages.package import Package @@ -38,9 +39,13 @@ def __init__( url: str = "https://pypi.org/", disable_cache: bool = False, fallback: bool = True, + pool_size: int = requests.adapters.DEFAULT_POOLSIZE, ) -> None: super().__init__( - "PyPI", url.rstrip("/") + "/simple/", disable_cache=disable_cache + "PyPI", + url.rstrip("/") + "/simple/", + disable_cache=disable_cache, + pool_size=pool_size, ) self._base_url = url diff --git a/src/poetry/utils/authenticator.py b/src/poetry/utils/authenticator.py index e181f2018ca..2628e5ba699 100644 --- a/src/poetry/utils/authenticator.py +++ b/src/poetry/utils/authenticator.py @@ -13,6 +13,7 @@ from typing import Any import requests +import requests.adapters import requests.auth import requests.exceptions @@ -110,7 +111,7 @@ def __init__( io: IO | None = None, cache_id: str | None = None, disable_cache: bool = False, - pool_size: int = 10, + pool_size: int = requests.adapters.DEFAULT_POOLSIZE, ) -> None: self._config = config or Config.create() self._io = io