From 83e7a975d02ce8ab971ceb061c4becf75332d5e4 Mon Sep 17 00:00:00 2001 From: Augustin Nesson Date: Fri, 23 Jun 2023 09:16:25 +0200 Subject: [PATCH] feat: standarize output tree --- eodag/plugins/apis/cds.py | 22 +++++- eodag/plugins/apis/ecmwf.py | 16 +++- eodag/plugins/download/aws.py | 113 +++++++++------------------ eodag/plugins/download/base.py | 24 +++--- eodag/plugins/download/http.py | 16 ++++ eodag/resources/providers.yml | 12 ++- tests/test_end_to_end.py | 6 +- tests/units/test_apis_plugins.py | 40 +++++++--- tests/units/test_download_plugins.py | 29 +++++++ tests/units/test_safe_build.py | 12 +-- 10 files changed, 177 insertions(+), 113 deletions(-) diff --git a/eodag/plugins/apis/cds.py b/eodag/plugins/apis/cds.py index af10c8cbe..ec4055664 100644 --- a/eodag/plugins/apis/cds.py +++ b/eodag/plugins/apis/cds.py @@ -16,6 +16,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +import os +import shutil from datetime import datetime import cdsapi @@ -31,7 +33,13 @@ from eodag.plugins.search.base import Search from eodag.plugins.search.build_search_result import BuildPostSearchResult from eodag.rest.stac import DEFAULT_MISSION_START_DATE -from eodag.utils import datetime_range, get_geometry_from_various, path_to_uri, urlsplit +from eodag.utils import ( + datetime_range, + get_geometry_from_various, + path_to_uri, + sanitize, + urlsplit, +) from eodag.utils.exceptions import AuthenticationError, DownloadError, RequestError from eodag.utils.logging import get_logging_verbose @@ -215,7 +223,17 @@ def download(self, product, auth=None, progress_callback=None, **kwargs): fh.write(product.properties["downloadLink"]) logger.debug("Download recorded in %s", record_filename) - # do not try to extract or delete grib/netcdf + # Check the output configuration + if getattr(self.config, "outputs_in_folder", False): + new_fs_path = os.path.join( + os.path.dirname(fs_path), sanitize(product.properties["title"]) + ) + if not os.path.isdir(new_fs_path): + os.makedirs(new_fs_path) + shutil.move(fs_path, new_fs_path) + fs_path = new_fs_path + + # do not try to extract or delete grib/netcdf or a directory kwargs["extract"] = False product_path = self._finalize( diff --git a/eodag/plugins/apis/ecmwf.py b/eodag/plugins/apis/ecmwf.py index 3200c969c..0a4f5113c 100644 --- a/eodag/plugins/apis/ecmwf.py +++ b/eodag/plugins/apis/ecmwf.py @@ -16,6 +16,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +import os +import shutil from datetime import datetime import geojson @@ -31,7 +33,7 @@ from eodag.plugins.search.base import Search from eodag.plugins.search.build_search_result import BuildPostSearchResult from eodag.rest.stac import DEFAULT_MISSION_START_DATE -from eodag.utils import get_geometry_from_various, path_to_uri, urlsplit +from eodag.utils import get_geometry_from_various, path_to_uri, sanitize, urlsplit from eodag.utils.exceptions import AuthenticationError, DownloadError from eodag.utils.logging import get_logging_verbose @@ -203,7 +205,17 @@ def download(self, product, auth=None, progress_callback=None, **kwargs): fh.write(product.properties["downloadLink"]) logger.debug("Download recorded in %s", record_filename) - # do not try to extract or delete grib/netcdf + # Check the output configuration + if getattr(self.config, "outputs_in_folder", False): + new_fs_path = os.path.join( + os.path.dirname(fs_path), sanitize(product.properties["title"]) + ) + if not os.path.isdir(new_fs_path): + os.makedirs(new_fs_path) + shutil.move(fs_path, new_fs_path) + fs_path = new_fs_path + + # do not try to extract or delete grib/netcdf or a directory kwargs["extract"] = False product_path = self._finalize( diff --git a/eodag/plugins/download/aws.py b/eodag/plugins/download/aws.py index 3e440a02d..cfbfea745 100644 --- a/eodag/plugins/download/aws.py +++ b/eodag/plugins/download/aws.py @@ -733,32 +733,26 @@ def get_chunk_dest_path(self, product, chunk, dir_prefix=None, build_safe=False) # S2 L2A Tile files ----------------------------------------------- if S2L2A_TILE_IMG_REGEX.match(chunk.key): found_dict = S2L2A_TILE_IMG_REGEX.match(chunk.key).groupdict() - product_path = ( - "%s.SAFE/GRANULE/%s/IMG_DATA/R%s/T%s%s%s_%s_%s_%s.jp2" - % ( - product.properties["title"], - found_dict["num"], - found_dict["res"], - found_dict["tile1"], - found_dict["tile2"], - found_dict["tile3"], - title_date1, - found_dict["file"], - found_dict["res"], - ) + product_path = "GRANULE/%s/IMG_DATA/R%s/T%s%s%s_%s_%s_%s.jp2" % ( + found_dict["num"], + found_dict["res"], + found_dict["tile1"], + found_dict["tile2"], + found_dict["tile3"], + title_date1, + found_dict["file"], + found_dict["res"], ) elif S2L2A_TILE_AUX_DIR_REGEX.match(chunk.key): found_dict = S2L2A_TILE_AUX_DIR_REGEX.match(chunk.key).groupdict() - product_path = "%s.SAFE/GRANULE/%s/AUX_DATA/%s" % ( - product.properties["title"], + product_path = "GRANULE/%s/AUX_DATA/%s" % ( found_dict["num"], found_dict["file"], ) # S2 L2A QI Masks elif S2_TILE_QI_MSK_REGEX.match(chunk.key): found_dict = S2_TILE_QI_MSK_REGEX.match(chunk.key).groupdict() - product_path = "%s.SAFE/GRANULE/%s/QI_DATA/MSK_%sPRB_%s" % ( - product.properties["title"], + product_path = "GRANULE/%s/QI_DATA/MSK_%sPRB_%s" % ( found_dict["num"], found_dict["file_base"], found_dict["file_suffix"], @@ -766,8 +760,7 @@ def get_chunk_dest_path(self, product, chunk, dir_prefix=None, build_safe=False) # S2 L2A QI PVI elif S2_TILE_QI_PVI_REGEX.match(chunk.key): found_dict = S2_TILE_QI_PVI_REGEX.match(chunk.key).groupdict() - product_path = "%s.SAFE/GRANULE/%s/QI_DATA/%s_%s_PVI.jp2" % ( - product.properties["title"], + product_path = "GRANULE/%s/QI_DATA/%s_%s_PVI.jp2" % ( found_dict["num"], title_part3, title_date1, @@ -775,15 +768,13 @@ def get_chunk_dest_path(self, product, chunk, dir_prefix=None, build_safe=False) # S2 Tile files --------------------------------------------------- elif S2_TILE_PREVIEW_DIR_REGEX.match(chunk.key): found_dict = S2_TILE_PREVIEW_DIR_REGEX.match(chunk.key).groupdict() - product_path = "%s.SAFE/GRANULE/%s/preview/%s" % ( - product.properties["title"], + product_path = "GRANULE/%s/preview/%s" % ( found_dict["num"], found_dict["file"], ) elif S2_TILE_IMG_REGEX.match(chunk.key): found_dict = S2_TILE_IMG_REGEX.match(chunk.key).groupdict() - product_path = "%s.SAFE/GRANULE/%s/IMG_DATA/T%s%s%s_%s_%s" % ( - product.properties["title"], + product_path = "GRANULE/%s/IMG_DATA/T%s%s%s_%s_%s" % ( found_dict["num"], found_dict["tile1"], found_dict["tile2"], @@ -793,97 +784,74 @@ def get_chunk_dest_path(self, product, chunk, dir_prefix=None, build_safe=False) ) elif S2_TILE_THUMBNAIL_REGEX.match(chunk.key): found_dict = S2_TILE_THUMBNAIL_REGEX.match(chunk.key).groupdict() - product_path = "%s.SAFE/GRANULE/%s/%s" % ( - product.properties["title"], + product_path = "GRANULE/%s/%s" % ( found_dict["num"], found_dict["file"], ) elif S2_TILE_MTD_REGEX.match(chunk.key): found_dict = S2_TILE_MTD_REGEX.match(chunk.key).groupdict() - product_path = "%s.SAFE/GRANULE/%s/MTD_TL.xml" % ( - product.properties["title"], - found_dict["num"], - ) + product_path = "GRANULE/%s/MTD_TL.xml" % (found_dict["num"],) elif S2_TILE_AUX_DIR_REGEX.match(chunk.key): found_dict = S2_TILE_AUX_DIR_REGEX.match(chunk.key).groupdict() - product_path = "%s.SAFE/GRANULE/%s/AUX_DATA/AUX_%s" % ( - product.properties["title"], + product_path = "GRANULE/%s/AUX_DATA/AUX_%s" % ( found_dict["num"], found_dict["file"], ) elif S2_TILE_QI_DIR_REGEX.match(chunk.key): found_dict = S2_TILE_QI_DIR_REGEX.match(chunk.key).groupdict() - product_path = "%s.SAFE/GRANULE/%s/QI_DATA/%s" % ( - product.properties["title"], + product_path = "GRANULE/%s/QI_DATA/%s" % ( found_dict["num"], found_dict["file"], ) # S2 Tiles generic elif S2_TILE_REGEX.match(chunk.key): found_dict = S2_TILE_REGEX.match(chunk.key).groupdict() - product_path = "%s.SAFE/GRANULE/%s/%s" % ( - product.properties["title"], + product_path = "GRANULE/%s/%s" % ( found_dict["num"], found_dict["file"], ) # S2 Product files elif S2_PROD_DS_MTD_REGEX.match(chunk.key): found_dict = S2_PROD_DS_MTD_REGEX.match(chunk.key).groupdict() - product_path = "%s.SAFE/DATASTRIP/%s/MTD_DS.xml" % ( - product.properties["title"], - ds_dir, - ) + product_path = "DATASTRIP/%s/MTD_DS.xml" % (ds_dir,) elif S2_PROD_DS_QI_REPORT_REGEX.match(chunk.key): found_dict = S2_PROD_DS_QI_REPORT_REGEX.match(chunk.key).groupdict() - product_path = "%s.SAFE/DATASTRIP/%s/QI_DATA/%s.xml" % ( - product.properties["title"], + product_path = "DATASTRIP/%s/QI_DATA/%s.xml" % ( ds_dir, found_dict["filename"], ) elif S2_PROD_DS_QI_REGEX.match(chunk.key): found_dict = S2_PROD_DS_QI_REGEX.match(chunk.key).groupdict() - product_path = "%s.SAFE/DATASTRIP/%s/QI_DATA/%s" % ( - product.properties["title"], + product_path = "DATASTRIP/%s/QI_DATA/%s" % ( ds_dir, found_dict["file"], ) elif S2_PROD_INSPIRE_REGEX.match(chunk.key): found_dict = S2_PROD_INSPIRE_REGEX.match(chunk.key).groupdict() - product_path = "%s.SAFE/INSPIRE.xml" % (product.properties["title"],) + product_path = "INSPIRE.xml" elif S2_PROD_MTD_REGEX.match(chunk.key): found_dict = S2_PROD_MTD_REGEX.match(chunk.key).groupdict() - product_path = "%s.SAFE/MTD_MSI%s.xml" % ( - product.properties["title"], - s2_processing_level, - ) + product_path = "MTD_MSI%s.xml" % (s2_processing_level,) # S2 Product generic elif S2_PROD_REGEX.match(chunk.key): found_dict = S2_PROD_REGEX.match(chunk.key).groupdict() - product_path = "%s.SAFE/%s" % ( - product.properties["title"], - found_dict["file"], - ) + product_path = "%s" % (found_dict["file"],) # S1 -------------------------------------------------------------- elif S1_CALIB_REGEX.match(chunk.key): found_dict = S1_CALIB_REGEX.match(chunk.key).groupdict() - product_path = ( - "%s.SAFE/annotation/calibration/%s-%s-%s-grd-%s-%s-%03d.xml" - % ( - product.properties["title"], - found_dict["file_prefix"], - product.properties["platformSerialIdentifier"].lower(), - found_dict["file_beam"], - found_dict["file_pol"], - s1_title_suffix, - S1_IMG_NB_PER_POLAR.get( - product.properties["polarizationMode"], {} - ).get(found_dict["file_pol"].upper(), 1), - ) + product_path = "annotation/calibration/%s-%s-%s-grd-%s-%s-%03d.xml" % ( + found_dict["file_prefix"], + product.properties["platformSerialIdentifier"].lower(), + found_dict["file_beam"], + found_dict["file_pol"], + s1_title_suffix, + S1_IMG_NB_PER_POLAR.get( + product.properties["polarizationMode"], {} + ).get(found_dict["file_pol"].upper(), 1), ) elif S1_ANNOT_REGEX.match(chunk.key): found_dict = S1_ANNOT_REGEX.match(chunk.key).groupdict() - product_path = "%s.SAFE/annotation/%s-%s-grd-%s-%s-%03d.xml" % ( - product.properties["title"], + product_path = "annotation/%s-%s-grd-%s-%s-%03d.xml" % ( product.properties["platformSerialIdentifier"].lower(), found_dict["file_beam"], found_dict["file_pol"], @@ -894,8 +862,7 @@ def get_chunk_dest_path(self, product, chunk, dir_prefix=None, build_safe=False) ) elif S1_MEAS_REGEX.match(chunk.key): found_dict = S1_MEAS_REGEX.match(chunk.key).groupdict() - product_path = "%s.SAFE/measurement/%s-%s-grd-%s-%s-%03d.%s" % ( - product.properties["title"], + product_path = "measurement/%s-%s-grd-%s-%s-%03d.%s" % ( product.properties["platformSerialIdentifier"].lower(), found_dict["file_beam"], found_dict["file_pol"], @@ -907,18 +874,14 @@ def get_chunk_dest_path(self, product, chunk, dir_prefix=None, build_safe=False) ) elif S1_REPORT_REGEX.match(chunk.key): found_dict = S1_REPORT_REGEX.match(chunk.key).groupdict() - product_path = "%s.SAFE/%s.SAFE-%s" % ( - product.properties["title"], + product_path = "%s.SAFE-%s" % ( product.properties["title"], found_dict["file"], ) # S1 generic elif S1_REGEX.match(chunk.key): found_dict = S1_REGEX.match(chunk.key).groupdict() - product_path = "%s.SAFE/%s" % ( - product.properties["title"], - found_dict["file"], - ) + product_path = "%s" % (found_dict["file"],) # out of SAFE format else: raise NotAvailableError( diff --git a/eodag/plugins/download/base.py b/eodag/plugins/download/base.py index 8b0e6f2bb..2dab65448 100644 --- a/eodag/plugins/download/base.py +++ b/eodag/plugins/download/base.py @@ -265,6 +265,11 @@ def _finalize(self, fs_path, progress_callback=None, **kwargs): extract = ( extract if extract is not None else getattr(self.config, "extract", True) ) + if not extract: + logger.info("Extraction not activated. The product is available as is.") + progress_callback(1, total=1) + return fs_path + delete_archive = kwargs.pop("delete_archive", None) delete_archive = ( delete_archive @@ -273,11 +278,6 @@ def _finalize(self, fs_path, progress_callback=None, **kwargs): ) outputs_extension = kwargs.pop("outputs_extension", ".zip") - if not extract: - logger.info("Extraction not activated. The product is available as is.") - progress_callback(1, total=1) - return fs_path - product_path = ( fs_path[: fs_path.index(outputs_extension)] if outputs_extension in fs_path @@ -335,19 +335,27 @@ def _finalize(self, fs_path, progress_callback=None, **kwargs): path=extraction_dir, ) progress_callback(1) - shutil.move(extraction_dir, outputs_dir) + shutil.move(self._resolve_archive_depth(extraction_dir), outputs_dir) elif fs_path.endswith(".tar.gz"): with tarfile.open(fs_path, "r:gz") as zfile: progress_callback.reset(total=1) zfile.extractall(path=extraction_dir) progress_callback(1) - shutil.move(extraction_dir, outputs_dir) + shutil.move(self._resolve_archive_depth(extraction_dir), outputs_dir) else: progress_callback(1, total=1) tmp_dir.cleanup() + # in some cases, only a file is extracted without being in a directory + # we create a directory in which we place this file + if os.path.isfile(outputs_dir): + product_path = os.path.splitext(product_path)[0] + if not os.path.isdir(product_path): + os.makedirs(product_path) + shutil.move(outputs_dir, product_path) + if delete_archive: logger.info(f"Deleting archive {os.path.basename(fs_path)}") os.unlink(fs_path) @@ -362,8 +370,6 @@ def _finalize(self, fs_path, progress_callback=None, **kwargs): if close_progress_callback: progress_callback.close() - product_path = self._resolve_archive_depth(product_path) - return product_path def download_all( diff --git a/eodag/plugins/download/http.py b/eodag/plugins/download/http.py index 4a5a20abf..c8becf1af 100644 --- a/eodag/plugins/download/http.py +++ b/eodag/plugins/download/http.py @@ -492,6 +492,22 @@ def download_request( shutil.move(fs_path, new_fs_path) product.location = path_to_uri(new_fs_path) return new_fs_path + + # Check that the downloaded file is not a lone file and must not be placed in a directory + if os.path.isfile(fs_path) and getattr( + self.config, "outputs_in_folder", False + ): + new_fs_path = os.path.join( + os.path.dirname(fs_path), + sanitize(product.properties["title"]), + ) + if not os.path.isdir(new_fs_path): + os.makedirs(new_fs_path) + shutil.move(fs_path, new_fs_path) + fs_path = new_fs_path + + # do not try to extract or delete a directory + kwargs["extract"] = False product_path = self._finalize( fs_path, progress_callback=progress_callback, **kwargs ) diff --git a/eodag/resources/providers.yml b/eodag/resources/providers.yml index cfd19e153..a4187d918 100644 --- a/eodag/resources/providers.yml +++ b/eodag/resources/providers.yml @@ -704,6 +704,7 @@ type: HTTPDownload base_uri: 'https://theia.cnes.fr/atdistrib/resto2' extract: true + archive_depth: 2 order_enabled: true auth_error_code: 403 dl_url_params: @@ -970,7 +971,7 @@ - null - '$.Attributes.processingLevel' # INSPIRE obligated OpenSearch Parameters for Collection Search (Table 4) - title: '$.Name' + title: '{$.Name#remove_extension}' resolution: - null - '$.Attributes.spatialResolution' @@ -1661,7 +1662,7 @@ - '$.Metadata.sensorType' # INSPIRE obligated OpenSearch Parameters for Collection Search (Table 4) - title: '$.Metadata.filename' + title: '{$.Metadata.filename#remove_extension}' topicCategory: - null - '$.Metadata.topicCategory' @@ -1851,6 +1852,7 @@ type: HTTPDownload base_uri: 'https://catalogue.onda-dias.eu/dias-catalogue/Products' extract: true + archive_depth: 2 auth_error_code: 401 order_enabled: true order_method: 'POST' @@ -2213,6 +2215,7 @@ type: EcmwfApi api_endpoint: https://api.ecmwf.int/v1 extract: false + outputs_in_folder: true metadata_mapping: productType: '$.productType' title: '$.id' @@ -2435,6 +2438,7 @@ type: CdsApi api_endpoint: https://ads.atmosphere.copernicus.eu/api/v2 extract: false + outputs_in_folder: true metadata_mapping: productType: '$.productType' title: '$.id' @@ -2752,6 +2756,7 @@ type: CdsApi api_endpoint: https://cds.climate.copernicus.eu/api/v2 extract: false + outputs_in_folder: true metadata_mapping: productType: '$.productType' title: '$.id' @@ -3329,6 +3334,7 @@ order_status_error: status: error outputs_extension: .nc + outputs_in_folder: true auth: !plugin type: HttpQueryStringAuth auth_uri: 'http://my.meteoblue.com/dataset/meta?dataset=NEMSAUTO' @@ -3416,7 +3422,7 @@ - null - '$.Attributes.processingLevel' # INSPIRE obligated OpenSearch Parameters for Collection Search (Table 4) - title: '$.Name' + title: '{$.Name#remove_extension}' resolution: - null - '$.Attributes.spatialResolution' diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index d86f2cd21..cc3c354e9 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -498,7 +498,7 @@ def test_end_to_end_search_download_ecmwf(self): product = self.execute_search( *ECMWF_SEARCH_ARGS, search_kwargs_dict=ECMWF_SEARCH_KWARGS ) - expected_filename = "{}.grib".format(product.properties["title"]) + expected_filename = "{}".format(product.properties["title"]) self.execute_download(product, expected_filename) def test_end_to_end_search_download_cop_ads(self): @@ -512,7 +512,7 @@ def test_end_to_end_search_download_cop_cds(self): product = self.execute_search( *COP_CDS_SEARCH_ARGS, search_kwargs_dict=COP_CDS_SEARCH_KWARGS ) - expected_filename = "{}.grib".format(product.properties["title"]) + expected_filename = "{}".format(product.properties["title"]) self.execute_download(product, expected_filename) def test_end_to_end_search_download_sara(self): @@ -522,7 +522,7 @@ def test_end_to_end_search_download_sara(self): def test_end_to_end_search_download_meteoblue(self): product = self.execute_search(*METEOBLUE_SEARCH_ARGS) - expected_filename = "{}.nc".format(product.properties["title"]) + expected_filename = "{}".format(product.properties["title"]) self.execute_download(product, expected_filename) # @unittest.skip("service unavailable for the moment") diff --git a/tests/units/test_apis_plugins.py b/tests/units/test_apis_plugins.py index 8c4c6d3c9..4a32377d0 100644 --- a/tests/units/test_apis_plugins.py +++ b/tests/units/test_apis_plugins.py @@ -273,12 +273,20 @@ def test_plugins_apis_ecmwf_download( output_data_path = os.path.join(os.path.expanduser("~"), "data") # public dataset request + def create_empty_file_for_public_dataset(*args, **kwargs): + with open(args[1]["target"], "x"): + pass + + mock_ecmwfdataserver_retrieve.side_effect = create_empty_file_for_public_dataset results, _ = dag.search( **self.query_dates, **self.custom_query_params, ) eoproduct = results[0] expected_path = os.path.join( + output_data_path, "%s" % eoproduct.properties["title"] + ) + arg_path = os.path.join( output_data_path, "%s.grib" % eoproduct.properties["title"] ) path = eoproduct.download(outputs_prefix=output_data_path) @@ -286,21 +294,22 @@ def test_plugins_apis_ecmwf_download( mock_ecmwfdataserver_retrieve.assert_called_once_with( mock.ANY, # ECMWFDataServer instance dict( - target=expected_path, + target=arg_path, **geojson.loads(urlsplit(eoproduct.remote_location).query), ), ) + assert path == expected_path assert path_to_uri(expected_path) == eoproduct.location mock_ecmwfservice_execute.reset_mock() mock_ecmwfdataserver_retrieve.reset_mock() # operation archive request - def create_empty_file(*args, **kwargs): + def create_empty_file_for_operation_archive(*args, **kwargs): with open(args[2], "x"): pass - mock_ecmwfservice_execute.side_effect = create_empty_file + mock_ecmwfservice_execute.side_effect = create_empty_file_for_operation_archive operation_archive_custom_query_params = self.custom_query_params.copy() operation_archive_custom_query_params.pop("dataset") operation_archive_custom_query_params["format"] = "netcdf" @@ -310,6 +319,9 @@ def create_empty_file(*args, **kwargs): ) eoproduct = results[0] expected_path = os.path.join( + output_data_path, "%s" % eoproduct.properties["title"] + ) + arg_path = os.path.join( output_data_path, "%s.nc" % eoproduct.properties["title"] ) path = eoproduct.download(outputs_prefix=output_data_path) @@ -320,7 +332,7 @@ def create_empty_file(*args, **kwargs): dict( **download_request, ), - expected_path, + arg_path, ) mock_ecmwfdataserver_retrieve.assert_not_called() assert path == expected_path @@ -345,7 +357,7 @@ def test_plugins_apis_ecmwf_download_all( mock_ecmwfdataserver_retrieve, mock_fetch_product_types_list, ): - """EcmwfApi.download_all must call the appriate ecmwf api service""" + """EcmwfApi.download_all must call the appropriate ecmwf api service""" dag = EODataAccessGateway() dag.set_preferred_provider("ecmwf") @@ -356,13 +368,13 @@ def test_plugins_apis_ecmwf_download_all( results, _ = dag.search( **self.query_dates, **self.custom_query_params, - foo="bar", + accuracy="bar", ) eoproducts.extend(results) results, _ = dag.search( **self.query_dates, **self.custom_query_params, - foo="baz", + accuracy="baz", ) eoproducts.extend(results) assert len(eoproducts) == 2 @@ -841,6 +853,11 @@ def test_plugins_apis_cds_download( output_data_path = os.path.join(os.path.expanduser("~"), "data") # public dataset request + def create_empty_file(*args, **kwargs): + with open(kwargs["target"], "x"): + pass + + mock_client_retrieve.side_effect = create_empty_file results, _ = dag.search( **self.query_dates, **self.custom_query_params, @@ -851,6 +868,9 @@ def test_plugins_apis_cds_download( expected_download_request = geojson.loads(query_str) expected_dataset_name = expected_download_request.pop("dataset") expected_path = os.path.join( + output_data_path, "%s" % eoproduct.properties["title"] + ) + arg_path = os.path.join( output_data_path, "%s.grib" % eoproduct.properties["title"] ) @@ -859,7 +879,7 @@ def test_plugins_apis_cds_download( mock.ANY, # instance name=expected_dataset_name, request=expected_download_request, - target=expected_path, + target=arg_path, ) assert path == expected_path @@ -893,13 +913,13 @@ def test_plugins_apis_cds_download_all( results, _ = dag.search( **self.query_dates, **self.custom_query_params, - foo="bar", + accuracy="bar", ) eoproducts.extend(results) results, _ = dag.search( **self.query_dates, **self.custom_query_params, - foo="baz", + accuracy="baz", ) eoproducts.extend(results) assert len(eoproducts) == 2 diff --git a/tests/units/test_download_plugins.py b/tests/units/test_download_plugins.py index 0755ad55a..a4a0ead3c 100644 --- a/tests/units/test_download_plugins.py +++ b/tests/units/test_download_plugins.py @@ -153,6 +153,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest): def test_plugins_download_http_ok(self, mock_requests_get): """HTTPDownload.download() must create an outputfile""" + # download a lone file with a ".zip" extension plugin = self.get_download_plugin(self.product) self.product.location = self.product.remote_location = "http://somewhere" self.product.properties["id"] = "someproduct" @@ -171,6 +172,34 @@ def test_plugins_download_http_ok(self, mock_requests_get): timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT, ) + # download a lone file with different extension and results output configuration + mock_requests_get.reset_mock() + self.product = EOProduct( + "meteoblue", + dict( + geometry="POINT (0 0)", + title="dummy_product_2", + id="dummy_2", + ), + ) + plugin = self.get_download_plugin(self.product) + self.product.location = self.product.remote_location = "http://somewhereelse" + self.product.properties["id"] = "someotherproduct" + + path = plugin.download(self.product, outputs_prefix=self.output_dir) + + self.assertEqual(path, os.path.join(self.output_dir, "dummy_product_2")) + self.assertTrue(os.path.isfile(os.path.join(path, os.listdir(path)[0]))) + mock_requests_get.assert_called_once_with( + "post", + self.product.remote_location, + stream=True, + auth=None, + params={}, + headers=USER_AGENT, + timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT, + ) + @mock.patch("eodag.plugins.download.http.requests.head", autospec=True) @mock.patch("eodag.plugins.download.http.requests.get", autospec=True) def test_plugins_download_http_assets_filename_from_href( diff --git a/tests/units/test_safe_build.py b/tests/units/test_safe_build.py index c9113c046..c8d4ee534 100644 --- a/tests/units/test_safe_build.py +++ b/tests/units/test_safe_build.py @@ -123,9 +123,7 @@ def chunk(): copyfile( os.path.join(TEST_RESOURCES_PATH, "safe_build", "manifest.safe.S1_SAR_GRD"), - os.path.join( - product_path, "%s.SAFE" % prod.properties["title"], "manifest.safe" - ), + os.path.join(product_path, "manifest.safe"), ) with self.assertLogs(self.logger, logging.WARN) as cm: @@ -167,9 +165,7 @@ def chunk(): copyfile( os.path.join(TEST_RESOURCES_PATH, "safe_build", "manifest.safe.S2_MSI_L2A"), - os.path.join( - product_path, "%s.SAFE" % prod.properties["title"], "manifest.safe" - ), + os.path.join(product_path, "manifest.safe"), ) self.awsd.finalize_s2_safe_product(product_path) @@ -213,9 +209,7 @@ def chunk(): copyfile( os.path.join(TEST_RESOURCES_PATH, "safe_build", "manifest.safe.S2_MSI_L1C"), - os.path.join( - product_path, "%s.SAFE" % prod.properties["title"], "manifest.safe" - ), + os.path.join(product_path, "manifest.safe"), ) self.awsd.finalize_s2_safe_product(product_path)