feat: configurable download timeout (#1124)

CS-SI · May 2, 2024 · a29e7a7 · a29e7a7
1 parent bc1953a
commit a29e7a7
Show file tree

Hide file tree

Showing 4 changed files with 37 additions and 21 deletions.
diff --git a/eodag/plugins/download/aws.py b/eodag/plugins/download/aws.py
@@ -440,6 +440,7 @@ def _configure_safe_build(self, build_safe: bool, product: EOProduct):
             product.product_type, {}
         )
         ssl_verify = getattr(self.config, "ssl_verify", True)
+        timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
 
         if build_safe and "fetch_metadata" in product_conf.keys():
             fetch_format = product_conf["fetch_metadata"]["fetch_format"]
@@ -452,11 +453,11 @@ def _configure_safe_build(self, build_safe: bool, product: EOProduct):
                 resp = requests.get(
                     fetch_url,
                     headers=USER_AGENT,
-                    timeout=HTTP_REQ_TIMEOUT,
+                    timeout=timeout,
                     verify=ssl_verify,
                 )
             except requests.exceptions.Timeout as exc:
-                raise TimeOutError(exc, timeout=HTTP_REQ_TIMEOUT) from exc
+                raise TimeOutError(exc, timeout=timeout) from exc
             update_metadata = mtd_cfg_as_conversion_and_querypath(update_metadata)
             if fetch_format == "json":
                 json_resp = resp.json()

diff --git a/eodag/plugins/download/http.py b/eodag/plugins/download/http.py
@@ -165,6 +165,7 @@ def orderDownload(
 
         order_method = getattr(self.config, "order_method", "GET").upper()
         ssl_verify = getattr(self.config, "ssl_verify", True)
+        timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
         OrderKwargs = TypedDict(
             "OrderKwargs", {"json": Dict[str, Union[Any, List[str]]]}, total=False
         )
@@ -188,7 +189,7 @@ def orderDownload(
                 method=order_method,
                 url=order_url,
                 auth=auth,
-                timeout=HTTP_REQ_TIMEOUT,
+                timeout=timeout,
                 headers=headers,
                 verify=ssl_verify,
                 **order_kwargs,
@@ -212,7 +213,7 @@ def orderDownload(
                     self._check_auth_exception(e)
                 return self.order_response_process(response, product)
         except requests.exceptions.Timeout as exc:
-            raise TimeOutError(exc, timeout=HTTP_REQ_TIMEOUT) from exc
+            raise TimeOutError(exc, timeout=timeout) from exc
 
     def order_response_process(
         self, response: Response, product: EOProduct
@@ -281,6 +282,8 @@ def orderDownloadStatus(
         status_config = getattr(self.config, "order_status", {})
         success_code: Optional[int] = status_config.get("success", {}).get("http_code")
 
+        timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
+
         def _request(
             url: str,
             method: str = "GET",
@@ -341,6 +344,7 @@ def _request(
                 status_request_method,
                 status_request.get("headers"),
                 json_data,
+                timeout,
             )
             json_response = response.json()
             if not isinstance(json_response, dict):
@@ -405,7 +409,7 @@ def _request(
         if config_on_success.get("need_search"):
             logger.debug(f"Search for new location: {product.properties['searchLink']}")
             try:
-                response = _request(product.properties["searchLink"])
+                response = _request(product.properties["searchLink"], timeout=timeout)
             except RequestException as e:
                 logger.warning(
                     "%s order status could not be checked, request returned %s",
@@ -1215,6 +1219,7 @@ def _get_asset_sizes(
     ) -> int:
         total_size = 0
 
+        timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
         ssl_verify = getattr(self.config, "ssl_verify", True)
         # loop for assets size & filename
         for asset in assets_values:
@@ -1224,7 +1229,7 @@ def _get_asset_sizes(
                     asset["href"],
                     auth=auth,
                     headers=USER_AGENT,
-                    timeout=HTTP_REQ_TIMEOUT,
+                    timeout=timeout,
                 ).headers
 
                 if not getattr(asset, "size", 0):

diff --git a/eodag/plugins/download/s3rest.py b/eodag/plugins/download/s3rest.py
@@ -177,12 +177,13 @@ def download_request(
             logger.debug("Retrieving product content from %s", nodes_list_url)
 
             ssl_verify = getattr(self.config, "ssl_verify", True)
+            timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
 
             bucket_contents = requests.get(
                 nodes_list_url,
                 auth=auth,
                 headers=USER_AGENT,
-                timeout=HTTP_REQ_TIMEOUT,
+                timeout=timeout,
                 verify=ssl_verify,
             )
             try:

diff --git a/tests/units/test_download_plugins.py b/tests/units/test_download_plugins.py
@@ -1105,20 +1105,29 @@ def test_plugins_download_http_order_get(self, mock_request):
         self.product.properties["orderLink"] = "http://somewhere/order"
         self.product.properties["storageStatus"] = OFFLINE_STATUS
 
-        auth_plugin = self.get_auth_plugin(self.product.provider)
-        auth_plugin.config.credentials = {"username": "foo", "password": "bar"}
-        auth = auth_plugin.authenticate()
-
-        plugin.orderDownload(self.product, auth=auth)
-
-        mock_request.assert_called_once_with(
-            method="GET",
-            url=self.product.properties["orderLink"],
-            auth=auth,
-            headers=USER_AGENT,
-            timeout=HTTP_REQ_TIMEOUT,
-            verify=True,
-        )
+        # customized timeout
+        timeout_backup = getattr(plugin.config, "timeout", None)
+        plugin.config.timeout = 10
+        try:
+            auth_plugin = self.get_auth_plugin(self.product.provider)
+            auth_plugin.config.credentials = {"username": "foo", "password": "bar"}
+            auth = auth_plugin.authenticate()
+
+            plugin.orderDownload(self.product, auth=auth)
+
+            mock_request.assert_called_once_with(
+                method="GET",
+                url=self.product.properties["orderLink"],
+                auth=auth,
+                headers=USER_AGENT,
+                timeout=10,
+                verify=True,
+            )
+        finally:
+            if timeout_backup:
+                plugin.config.timeout = timeout_backup
+            else:
+                del plugin.config.timeout
 
     @mock.patch("eodag.plugins.download.http.requests.request", autospec=True)
     def test_plugins_download_http_order_post(self, mock_request):