feat(plugins): flatten_top_dirs true by default (#1220)

CS-SI · Jun 19, 2024 · 1526a6f · 1526a6f
1 parent 4cfc99b
commit 1526a6f
Show file tree

Hide file tree

Showing 10 changed files with 12 additions and 30 deletions.
diff --git a/docs/add_provider.rst b/docs/add_provider.rst
@@ -32,7 +32,6 @@ the `STAC client page <notebooks/tutos/tuto_stac_client.ipynb#stac-api>`_, shows
       download:
          type: AwsDownload
          base_uri: https://tamn.snapplanet.io
-         flatten_top_dirs: True
       auth:
          type: AwsAuth
          credentials:

diff --git a/docs/notebooks/api_user_guide/3_configuration.ipynb b/docs/notebooks/api_user_guide/3_configuration.ipynb
@@ -95,7 +95,6 @@
     "        download:\n",
     "            type: AwsDownload\n",
     "            base_uri: https://tamn.snapplanet.io\n",
-    "            flatten_top_dirs: True\n",
     "        auth:\n",
     "            type: AwsAuth\n",
     "            credentials:\n",

diff --git a/docs/notebooks/tutos/tuto_stac_client.ipynb b/docs/notebooks/tutos/tuto_stac_client.ipynb
@@ -110,7 +110,6 @@
     "        download:\n",
     "            type: AwsDownload\n",
     "            base_uri: https://tamn.snapplanet.io\n",
-    "            flatten_top_dirs: True\n",
     "        auth:\n",
     "            type: AwsAuth\n",
     "            credentials:\n",
@@ -375,7 +374,6 @@
     "    download:\n",
     "        type: HTTPDownload\n",
     "        base_uri: https://fake-endpoint\n",
-    "        flatten_top_dirs: True\n",
     "        outputs_prefix: %s\n",
     "\"\"\" % os.path.abspath(workspace))\n",
     "\n",

diff --git a/eodag/plugins/download/aws.py b/eodag/plugins/download/aws.py
@@ -299,7 +299,7 @@ def download(
 
         # product conf overrides provider conf for "flatten_top_dirs"
         flatten_top_dirs = product_conf.get(
-            "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", False)
+            "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", True)
         )
 
         # xtra metadata needed for SAFE product
@@ -808,7 +808,7 @@ def get_chunk_parts(
             product.product_type, {}
         )
         flatten_top_dirs = product_conf.get(
-            "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", False)
+            "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", True)
         )
         common_path = ""
         if flatten_top_dirs:

diff --git a/eodag/plugins/download/http.py b/eodag/plugins/download/http.py
@@ -1072,7 +1072,7 @@ def get_chunks(stream: Response) -> Any:
             product.product_type, {}
         )
         flatten_top_dirs = product_conf.get(
-            "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", False)
+            "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", True)
         )
         ssl_verify = getattr(self.config, "ssl_verify", True)
 
@@ -1184,7 +1184,7 @@ def _download_assets(
             product.product_type, {}
         )
         flatten_top_dirs = product_conf.get(
-            "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", False)
+            "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", True)
         )
 
         # count local assets
@@ -1231,13 +1231,17 @@ def _download_assets(
             shutil.rmtree(fs_dir_path)
             # and return assets_urls[0] path
             fs_dir_path = uri_to_path(assets_urls[0])
+            # do not flatten dir
+            flatten_top_dirs = False
         # several local assets
         elif local_assets_count == len(assets_urls) and local_assets_count > 0:
             common_path = os.path.commonpath([uri_to_path(uri) for uri in assets_urls])
             # remove empty {fs_dir_path}
             shutil.rmtree(fs_dir_path)
             # and return assets_urls common path
             fs_dir_path = common_path
+            # do not flatten dir
+            flatten_top_dirs = False
         # no assets downloaded but some should have been
         elif len(os.listdir(fs_dir_path)) == 0:
             raise NotAvailableError("No assets could be downloaded")

diff --git a/eodag/resources/providers.yml b/eodag/resources/providers.yml
@@ -1657,7 +1657,6 @@
   download: !plugin
     type: AwsDownload
     requester_pays: True
-    flatten_top_dirs: True
     ssl_verify: true
     products:
       S1_SAR_GRD:
@@ -1749,7 +1748,6 @@
   download: !plugin
     type: AwsDownload
     requester_pays: True
-    flatten_top_dirs: True
     ssl_verify: true
   auth: !plugin
     type: AwsAuth
@@ -1846,7 +1844,6 @@
   download: !plugin
     type: AwsDownload
     requester_pays: True
-    flatten_top_dirs: True
     ssl_verify: true
     products:
       S2_MSI_L1C:
@@ -4226,7 +4223,6 @@
   download: !plugin
     type: HTTPDownload
     base_uri: 'https://planetarycomputer.microsoft.com/api/stac/download'
-    flatten_top_dirs: True
     auth_error_code: 403
     ssl_verify: true
   auth: !plugin
@@ -4277,7 +4273,6 @@
   download: !plugin
     type: HTTPDownload
     base_uri: https://hydroweb.next.theia-land.fr
-    flatten_top_dirs: true
     auth_error_code: 401
     ssl_verify: true
   auth: !plugin
@@ -5919,7 +5914,6 @@
   download: !plugin
     type: HTTPDownload
     base_uri: https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess
-    flatten_top_dirs: true
     auth_error_code: 401
     order_enabled: true
     order_method: 'POST'
@@ -6038,7 +6032,6 @@
   download: !plugin
     type: HTTPDownload
     base_uri: https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess
-    flatten_top_dirs: true
     auth_error_code: 401
     order_enabled: true
     order_method: 'POST'
@@ -6222,7 +6215,6 @@
       productIdentifier: '$.S3Path'
   download: !plugin
     type: CreodiasS3Download
-    flatten_top_dirs: True
     base_uri: 'https://eodata.cloudferro.com'
     s3_bucket: 'eodata'
     ssl_verify: true
@@ -6544,7 +6536,6 @@
     type: HTTPDownload
     base_uri: https://polytope.lumi.apps.dte.destination-earth.eu/api/v1/
     ssl_verify: true
-    flatten_top_dirs: true
     auth_error_code: 401
     order_enabled: True
     order_method: POST
@@ -6646,7 +6637,6 @@
     timeout: 20
     ssl_verify: true
     extract: true
-    flatten_top_dirs: true
     order_enabled: True
     order_method: GET
     order_on_response:
@@ -7162,7 +7152,6 @@
     type: HTTPDownload
     base_uri: 'https://api.eumetsat.int/data/download/1.0.0'
     extract: true
-    flatten_top_dirs: True
     ignore_assets: True
     ssl_verify: true
   auth: !plugin
@@ -7301,6 +7290,5 @@
     type: AwsDownload
     base_uri: https://s3.waw3-1.cloudferro.com
     bucket_path_level: 0
-    flatten_top_dirs: True
   auth: !plugin
     type: AwsAuth
diff --git a/tests/integration/test_search_stac_static.py b/tests/integration/test_search_stac_static.py
@@ -81,7 +81,6 @@ def setUp(self):
                 download:
                     type: HTTPDownload
                     base_uri: https://fake-endpoint
-                    flatten_top_dirs: True
         """
         )
         self.dag.set_preferred_provider(self.static_stac_provider)

diff --git a/tests/resources/file_config_override.yml b/tests/resources/file_config_override.yml
@@ -36,7 +36,7 @@ my_new_provider:
     download:
         type: AwsDownload
         base_uri: https://api.my_new_provider
-        flatten_top_dirs: True
+        flatten_top_dirs: false
     auth:
         type: AwsAuth
         credentials:

diff --git a/tests/test_config.py b/tests/test_config.py
@@ -290,7 +290,6 @@ def test_override_config_from_str(self):
                 download:
                     type: AwsDownload
                     base_uri: https://api.my_new_provider
-                    flatten_top_dirs: True
                 auth:
                     type: AwsAuth
                     credentials:
@@ -354,7 +353,7 @@ def test_override_config_from_file(self):
             download:
                 type: AwsDownload
                 base_uri: https://api.my_new_provider
-                flatten_top_dirs: True
+                flatten_top_dirs: false
             auth:
                 type: AwsAuth
                 credentials:
@@ -410,7 +409,7 @@ def test_override_config_from_file(self):
         self.assertEqual(
             my_new_provider_conf.download.base_uri, "https://api.my_new_provider"
         )
-        self.assertTrue(my_new_provider_conf.download.flatten_top_dirs)
+        self.assertFalse(my_new_provider_conf.download.flatten_top_dirs)
         self.assertIsInstance(my_new_provider_conf.auth, config.PluginConfig)
         self.assertEqual(my_new_provider_conf.auth.type, "AwsAuth")
         self.assertEqual(

diff --git a/tests/units/test_download_plugins.py b/tests/units/test_download_plugins.py
@@ -633,11 +633,7 @@ def test_plugins_download_http_assets_filename_from_href(
         self.assertEqual(path, os.path.join(self.output_dir, "dummy_product"))
         self.assertTrue(os.path.isdir(path))
         self.assertTrue(
-            os.path.isfile(
-                os.path.join(
-                    self.output_dir, "dummy_product", "mal_for_matted", "something"
-                )
-            )
+            os.path.isfile(os.path.join(self.output_dir, "dummy_product", "something"))
         )
 
         # Check if the GET request has been called for both size request and download request