feat: generate record hash from product_type + id

eodag/plugins/apis/base.py

@@ -61,9 +61,9 @@ class Api(PluginTopic):
       (e.g. 'file:///tmp/product_folder' on Linux or
       'file:///C:/Users/username/AppData/LOcal/Temp' on Windows)
     - save a *record* file in the directory ``outputs_prefix/.downloaded`` whose name
-      is built on the MD5 hash of the product's ``remote_location`` attribute
-      (``hashlib.md5(remote_location.encode("utf-8")).hexdigest()``) and whose content is
-      the product's ``remote_location`` attribute itself.
+      is built on the MD5 hash of the product's ``product_type`` and ``properties['id']``
+      attributes (``hashlib.md5((product.product_type+"-"+product.properties['id']).encode("utf-8")).hexdigest()``)
+      and whose content is the product's ``remote_location`` attribute itself.
     - not try to download a product whose ``location`` attribute already points to an
       existing file/directory
     - not try to download a product if its *record* file exists as long as the expected

eodag/plugins/download/base.py

@@ -86,9 +86,9 @@ class Download(PluginTopic):
       (e.g. 'file:///tmp/product_folder' on Linux or
       'file:///C:/Users/username/AppData/LOcal/Temp' on Windows)
     - save a *record* file in the directory ``outputs_prefix/.downloaded`` whose name
-      is built on the MD5 hash of the product's ``remote_location`` attribute
-      (``hashlib.md5(remote_location.encode("utf-8")).hexdigest()``) and whose content is
-      the product's ``remote_location`` attribute itself.
+      is built on the MD5 hash of the product's ``product_type`` and ``properties['id']``
+      attributes (``hashlib.md5((product.product_type+"-"+product.properties['id']).encode("utf-8")).hexdigest()``)
+      and whose content is the product's ``remote_location`` attribute itself.
     - not try to download a product whose ``location`` attribute already points to an
       existing file/directory
     - not try to download a product if its *record* file exists as long as the expected
@@ -246,8 +246,9 @@ def _prepare_download(
                 logger.warning(
                     f"Unable to create records directory. Got:\n{tb.format_exc()}",
                 )
-        url_hash = hashlib.md5(url.encode("utf-8")).hexdigest()
-        record_filename = os.path.join(download_records_dir, url_hash)
+        record_filename = os.path.join(
+            download_records_dir, self.generate_record_hash(product)
+        )
         if os.path.isfile(record_filename) and os.path.isfile(fs_path):
             logger.info(
                 f"Product already downloaded: {fs_path}",
@@ -278,6 +279,21 @@ def _prepare_download(
 
         return fs_path, record_filename
 
+    def generate_record_hash(self, product: EOProduct) -> str:
+        """Generate the record hash of the given product.
+
+        The MD5 hash is built from the product's ``product_type`` and ``properties['id']`` attributes
+        (``hashlib.md5((product.product_type+"-"+product.properties['id']).encode("utf-8")).hexdigest()``)
+
+        :param product: The product to calculate the record hash
+        :type product: :class:`~eodag.api.product._product.EOProduct`
+        :returns: The MD5 hash
+        :rtype: str
+        """
+        # In some unit tests, `product.product_type` is `None` and `product.properties["id"]` is `ìnt`
+        product_hash = str(product.product_type) + "-" + str(product.properties["id"])
+        return hashlib.md5(product_hash.encode("utf-8")).hexdigest()
+
     def _resolve_archive_depth(self, product_path: str) -> str:
         """Update product_path using archive_depth from provider configuration.
 

eodag/plugins/download/s3rest.py

@@ -17,7 +17,6 @@
 # limitations under the License.
 from __future__ import annotations
 
-import hashlib
 import logging
 import os
 import os.path
@@ -252,8 +251,9 @@ def download_request(
                         "Unable to create records directory. Got:\n%s", tb.format_exc()
                     )
             # check if product has already been downloaded
-            url_hash = hashlib.md5(product.remote_location.encode("utf-8")).hexdigest()
-            record_filename = os.path.join(download_records_dir, url_hash)
+            record_filename = os.path.join(
+                download_records_dir, self.generate_record_hash(product)
+            )
             if os.path.isfile(record_filename) and os.path.exists(product_local_path):
                 product.location = path_to_uri(product_local_path)
                 return product_local_path

tests/test_end_to_end.py

@@ -834,8 +834,9 @@ def test_end_to_end_complete_peps(self):
         record_dir = os.path.join(self.tmp_download_path, ".downloaded")
         self.assertTrue(os.path.isdir(record_dir))
         # It must contain a file per product downloade, whose name is
-        # the MD5 hash of the product's remote location
-        expected_hash = hashlib.md5(product.remote_location.encode("utf-8")).hexdigest()
+        # the MD5 hash of the product's ``product_type`` and ``properties['id']``
+        expected_hash = product.product_type + "-" + product.properties["id"]
+        expected_hash = hashlib.md5(expected_hash.encode("utf-8")).hexdigest()
         record_file = os.path.join(record_dir, expected_hash)
         self.assertTrue(os.path.isfile(record_file))
         # Its content must be the product's remote location