Align connection pool size with installer.max-workers config option (…

…python-poetry#8559)

src/poetry/config/config.py

@@ -226,6 +226,22 @@ def virtualenvs_path(self) -> Path:
             path = Path(self.get("cache-dir")) / "virtualenvs"
         return Path(path).expanduser()
 
+    @property
+    def installer_max_workers(self) -> int:
+        # This should be directly handled by ThreadPoolExecutor
+        # however, on some systems the number of CPUs cannot be determined
+        # (it raises a NotImplementedError), so, in this case, we assume
+        # that the system only has one CPU.
+        try:
+            default_max_workers = (os.cpu_count() or 1) + 4
+        except NotImplementedError:
+            default_max_workers = 5
+
+        desired_max_workers = self.get("installer.max-workers")
+        if desired_max_workers is None:
+            return default_max_workers
+        return min(default_max_workers, int(desired_max_workers))
+
     def get(self, setting_name: str, default: Any = None) -> Any:
         """
         Retrieve a setting value.

src/poetry/console/commands/init.py

@@ -489,6 +489,7 @@ def _get_pool(self) -> RepositoryPool:
 
         if self._pool is None:
             self._pool = RepositoryPool()
-            self._pool.add_repository(PyPiRepository())
+            pool_size = self.poetry.config.installer_max_workers
+            self._pool.add_repository(PyPiRepository(pool_size=pool_size))
 
         return self._pool

src/poetry/factory.py

@@ -224,12 +224,14 @@ def create_package_source(
         except KeyError:
             raise InvalidSourceError("Missing [name] in source.")
 
+        pool_size = config.installer_max_workers
+
         if name.lower() == "pypi":
             if "url" in source:
                 raise InvalidSourceError(
                     "The PyPI repository cannot be configured with a custom url."
                 )
-            return PyPiRepository(disable_cache=disable_cache)
+            return PyPiRepository(disable_cache=disable_cache, pool_size=pool_size)
 
         try:
             url = source["url"]
@@ -246,6 +248,7 @@ def create_package_source(
             url,
             config=config,
             disable_cache=disable_cache,
+            pool_size=pool_size,
         )
 
     @classmethod

src/poetry/installation/executor.py

@@ -4,7 +4,6 @@
 import csv
 import itertools
 import json
-import os
 import threading
 
 from concurrent.futures import ThreadPoolExecutor
@@ -73,9 +72,7 @@ def __init__(
             parallel = config.get("installer.parallel", True)
 
         if parallel:
-            self._max_workers = self._get_max_workers(
-                desired_max_workers=config.get("installer.max-workers")
-            )
+            self._max_workers = config.installer_max_workers
         else:
             self._max_workers = 1
 
@@ -223,21 +220,6 @@ def execute(self, operations: list[Operation]) -> int:
 
         return 1 if self._shutdown else 0
 
-    @staticmethod
-    def _get_max_workers(desired_max_workers: int | None = None) -> int:
-        # This should be directly handled by ThreadPoolExecutor
-        # however, on some systems the number of CPUs cannot be determined
-        # (it raises a NotImplementedError), so, in this case, we assume
-        # that the system only has one CPU.
-        try:
-            default_max_workers = (os.cpu_count() or 1) + 4
-        except NotImplementedError:
-            default_max_workers = 5
-
-        if desired_max_workers is None:
-            return default_max_workers
-        return min(default_max_workers, desired_max_workers)
-
     def _write(self, operation: Operation, line: str) -> None:
         if not self.supports_fancy_output() or not self._should_write_operation(
             operation

src/poetry/repositories/http_repository.py

@@ -11,6 +11,7 @@
 from typing import Iterator
 
 import requests
+import requests.adapters
 
 from poetry.core.constraints.version import parse_constraint
 from poetry.core.packages.dependency import Dependency
@@ -44,13 +45,15 @@ def __init__(
         url: str,
         config: Config | None = None,
         disable_cache: bool = False,
+        pool_size: int = requests.adapters.DEFAULT_POOLSIZE,
     ) -> None:
         super().__init__(name, disable_cache, config)
         self._url = url
         self._authenticator = Authenticator(
             config=config,
             cache_id=name,
             disable_cache=disable_cache,
+            pool_size=pool_size,
         )
         self._authenticator.add_repository(name, url)
         self.get_page = functools.lru_cache(maxsize=None)(self._get_page)

src/poetry/repositories/legacy_repository.py

@@ -3,6 +3,8 @@
 from typing import TYPE_CHECKING
 from typing import Any
 
+import requests.adapters
+
 from poetry.core.packages.package import Package
 
 from poetry.inspection.info import PackageInfo
@@ -27,11 +29,12 @@ def __init__(
         url: str,
         config: Config | None = None,
         disable_cache: bool = False,
+        pool_size: int = requests.adapters.DEFAULT_POOLSIZE,
     ) -> None:
         if name == "pypi":
             raise ValueError("The name [pypi] is reserved for repositories")
 
-        super().__init__(name, url.rstrip("/"), config, disable_cache)
+        super().__init__(name, url.rstrip("/"), config, disable_cache, pool_size)
 
     @property
     def packages(self) -> list[Package]:

src/poetry/repositories/pypi_repository.py

@@ -7,6 +7,7 @@
 from typing import Any
 
 import requests
+import requests.adapters
 
 from cachecontrol.controller import logger as cache_control_logger
 from poetry.core.packages.package import Package
@@ -38,9 +39,13 @@ def __init__(
         url: str = "https://pypi.org/",
         disable_cache: bool = False,
         fallback: bool = True,
+        pool_size: int = requests.adapters.DEFAULT_POOLSIZE,
     ) -> None:
         super().__init__(
-            "PyPI", url.rstrip("/") + "/simple/", disable_cache=disable_cache
+            "PyPI",
+            url.rstrip("/") + "/simple/",
+            disable_cache=disable_cache,
+            pool_size=pool_size,
         )
 
         self._base_url = url

src/poetry/utils/authenticator.py

@@ -13,6 +13,7 @@
 from typing import Any
 
 import requests
+import requests.adapters
 import requests.auth
 import requests.exceptions
 
@@ -110,7 +111,7 @@ def __init__(
         io: IO | None = None,
         cache_id: str | None = None,
         disable_cache: bool = False,
-        pool_size: int = 10,
+        pool_size: int = requests.adapters.DEFAULT_POOLSIZE,
     ) -> None:
         self._config = config or Config.create()
         self._io = io