From 1526a6fedbb6d599c2537e30f66a25238089ed05 Mon Sep 17 00:00:00 2001 From: Sylvain Brunato <61419125+sbrunato@users.noreply.github.com> Date: Tue, 18 Jun 2024 14:16:30 +0200 Subject: [PATCH] feat(plugins): flatten_top_dirs true by default (#1220) --- docs/add_provider.rst | 1 - docs/notebooks/api_user_guide/3_configuration.ipynb | 1 - docs/notebooks/tutos/tuto_stac_client.ipynb | 2 -- eodag/plugins/download/aws.py | 4 ++-- eodag/plugins/download/http.py | 8 ++++++-- eodag/resources/providers.yml | 12 ------------ tests/integration/test_search_stac_static.py | 1 - tests/resources/file_config_override.yml | 2 +- tests/test_config.py | 5 ++--- tests/units/test_download_plugins.py | 6 +----- 10 files changed, 12 insertions(+), 30 deletions(-) diff --git a/docs/add_provider.rst b/docs/add_provider.rst index 9714290a40..df70738529 100644 --- a/docs/add_provider.rst +++ b/docs/add_provider.rst @@ -32,7 +32,6 @@ the `STAC client page `_, shows download: type: AwsDownload base_uri: https://tamn.snapplanet.io - flatten_top_dirs: True auth: type: AwsAuth credentials: diff --git a/docs/notebooks/api_user_guide/3_configuration.ipynb b/docs/notebooks/api_user_guide/3_configuration.ipynb index 67116834af..bd33e491dd 100644 --- a/docs/notebooks/api_user_guide/3_configuration.ipynb +++ b/docs/notebooks/api_user_guide/3_configuration.ipynb @@ -95,7 +95,6 @@ " download:\n", " type: AwsDownload\n", " base_uri: https://tamn.snapplanet.io\n", - " flatten_top_dirs: True\n", " auth:\n", " type: AwsAuth\n", " credentials:\n", diff --git a/docs/notebooks/tutos/tuto_stac_client.ipynb b/docs/notebooks/tutos/tuto_stac_client.ipynb index cf8e577517..dd6a33c6fb 100644 --- a/docs/notebooks/tutos/tuto_stac_client.ipynb +++ b/docs/notebooks/tutos/tuto_stac_client.ipynb @@ -110,7 +110,6 @@ " download:\n", " type: AwsDownload\n", " base_uri: https://tamn.snapplanet.io\n", - " flatten_top_dirs: True\n", " auth:\n", " type: AwsAuth\n", " credentials:\n", @@ -375,7 +374,6 @@ " download:\n", " type: HTTPDownload\n", " base_uri: https://fake-endpoint\n", - " flatten_top_dirs: True\n", " outputs_prefix: %s\n", "\"\"\" % os.path.abspath(workspace))\n", "\n", diff --git a/eodag/plugins/download/aws.py b/eodag/plugins/download/aws.py index b252ebb61f..ce1f4deffc 100644 --- a/eodag/plugins/download/aws.py +++ b/eodag/plugins/download/aws.py @@ -299,7 +299,7 @@ def download( # product conf overrides provider conf for "flatten_top_dirs" flatten_top_dirs = product_conf.get( - "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", False) + "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", True) ) # xtra metadata needed for SAFE product @@ -808,7 +808,7 @@ def get_chunk_parts( product.product_type, {} ) flatten_top_dirs = product_conf.get( - "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", False) + "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", True) ) common_path = "" if flatten_top_dirs: diff --git a/eodag/plugins/download/http.py b/eodag/plugins/download/http.py index 57a89abac6..efb59462dd 100644 --- a/eodag/plugins/download/http.py +++ b/eodag/plugins/download/http.py @@ -1072,7 +1072,7 @@ def get_chunks(stream: Response) -> Any: product.product_type, {} ) flatten_top_dirs = product_conf.get( - "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", False) + "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", True) ) ssl_verify = getattr(self.config, "ssl_verify", True) @@ -1184,7 +1184,7 @@ def _download_assets( product.product_type, {} ) flatten_top_dirs = product_conf.get( - "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", False) + "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", True) ) # count local assets @@ -1231,6 +1231,8 @@ def _download_assets( shutil.rmtree(fs_dir_path) # and return assets_urls[0] path fs_dir_path = uri_to_path(assets_urls[0]) + # do not flatten dir + flatten_top_dirs = False # several local assets elif local_assets_count == len(assets_urls) and local_assets_count > 0: common_path = os.path.commonpath([uri_to_path(uri) for uri in assets_urls]) @@ -1238,6 +1240,8 @@ def _download_assets( shutil.rmtree(fs_dir_path) # and return assets_urls common path fs_dir_path = common_path + # do not flatten dir + flatten_top_dirs = False # no assets downloaded but some should have been elif len(os.listdir(fs_dir_path)) == 0: raise NotAvailableError("No assets could be downloaded") diff --git a/eodag/resources/providers.yml b/eodag/resources/providers.yml index 45222977db..3e4ad5b0df 100644 --- a/eodag/resources/providers.yml +++ b/eodag/resources/providers.yml @@ -1657,7 +1657,6 @@ download: !plugin type: AwsDownload requester_pays: True - flatten_top_dirs: True ssl_verify: true products: S1_SAR_GRD: @@ -1749,7 +1748,6 @@ download: !plugin type: AwsDownload requester_pays: True - flatten_top_dirs: True ssl_verify: true auth: !plugin type: AwsAuth @@ -1846,7 +1844,6 @@ download: !plugin type: AwsDownload requester_pays: True - flatten_top_dirs: True ssl_verify: true products: S2_MSI_L1C: @@ -4226,7 +4223,6 @@ download: !plugin type: HTTPDownload base_uri: 'https://planetarycomputer.microsoft.com/api/stac/download' - flatten_top_dirs: True auth_error_code: 403 ssl_verify: true auth: !plugin @@ -4277,7 +4273,6 @@ download: !plugin type: HTTPDownload base_uri: https://hydroweb.next.theia-land.fr - flatten_top_dirs: true auth_error_code: 401 ssl_verify: true auth: !plugin @@ -5919,7 +5914,6 @@ download: !plugin type: HTTPDownload base_uri: https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess - flatten_top_dirs: true auth_error_code: 401 order_enabled: true order_method: 'POST' @@ -6038,7 +6032,6 @@ download: !plugin type: HTTPDownload base_uri: https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess - flatten_top_dirs: true auth_error_code: 401 order_enabled: true order_method: 'POST' @@ -6222,7 +6215,6 @@ productIdentifier: '$.S3Path' download: !plugin type: CreodiasS3Download - flatten_top_dirs: True base_uri: 'https://eodata.cloudferro.com' s3_bucket: 'eodata' ssl_verify: true @@ -6544,7 +6536,6 @@ type: HTTPDownload base_uri: https://polytope.lumi.apps.dte.destination-earth.eu/api/v1/ ssl_verify: true - flatten_top_dirs: true auth_error_code: 401 order_enabled: True order_method: POST @@ -6646,7 +6637,6 @@ timeout: 20 ssl_verify: true extract: true - flatten_top_dirs: true order_enabled: True order_method: GET order_on_response: @@ -7162,7 +7152,6 @@ type: HTTPDownload base_uri: 'https://api.eumetsat.int/data/download/1.0.0' extract: true - flatten_top_dirs: True ignore_assets: True ssl_verify: true auth: !plugin @@ -7301,6 +7290,5 @@ type: AwsDownload base_uri: https://s3.waw3-1.cloudferro.com bucket_path_level: 0 - flatten_top_dirs: True auth: !plugin type: AwsAuth diff --git a/tests/integration/test_search_stac_static.py b/tests/integration/test_search_stac_static.py index b66e116644..2aca84714e 100644 --- a/tests/integration/test_search_stac_static.py +++ b/tests/integration/test_search_stac_static.py @@ -81,7 +81,6 @@ def setUp(self): download: type: HTTPDownload base_uri: https://fake-endpoint - flatten_top_dirs: True """ ) self.dag.set_preferred_provider(self.static_stac_provider) diff --git a/tests/resources/file_config_override.yml b/tests/resources/file_config_override.yml index a7e9dd4d85..7116eec895 100644 --- a/tests/resources/file_config_override.yml +++ b/tests/resources/file_config_override.yml @@ -36,7 +36,7 @@ my_new_provider: download: type: AwsDownload base_uri: https://api.my_new_provider - flatten_top_dirs: True + flatten_top_dirs: false auth: type: AwsAuth credentials: diff --git a/tests/test_config.py b/tests/test_config.py index e819986ec8..baed054907 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -290,7 +290,6 @@ def test_override_config_from_str(self): download: type: AwsDownload base_uri: https://api.my_new_provider - flatten_top_dirs: True auth: type: AwsAuth credentials: @@ -354,7 +353,7 @@ def test_override_config_from_file(self): download: type: AwsDownload base_uri: https://api.my_new_provider - flatten_top_dirs: True + flatten_top_dirs: false auth: type: AwsAuth credentials: @@ -410,7 +409,7 @@ def test_override_config_from_file(self): self.assertEqual( my_new_provider_conf.download.base_uri, "https://api.my_new_provider" ) - self.assertTrue(my_new_provider_conf.download.flatten_top_dirs) + self.assertFalse(my_new_provider_conf.download.flatten_top_dirs) self.assertIsInstance(my_new_provider_conf.auth, config.PluginConfig) self.assertEqual(my_new_provider_conf.auth.type, "AwsAuth") self.assertEqual( diff --git a/tests/units/test_download_plugins.py b/tests/units/test_download_plugins.py index cafa794f50..2e994f4290 100644 --- a/tests/units/test_download_plugins.py +++ b/tests/units/test_download_plugins.py @@ -633,11 +633,7 @@ def test_plugins_download_http_assets_filename_from_href( self.assertEqual(path, os.path.join(self.output_dir, "dummy_product")) self.assertTrue(os.path.isdir(path)) self.assertTrue( - os.path.isfile( - os.path.join( - self.output_dir, "dummy_product", "mal_for_matted", "something" - ) - ) + os.path.isfile(os.path.join(self.output_dir, "dummy_product", "something")) ) # Check if the GET request has been called for both size request and download request