From e8cf1371558ee9d0e29cd142a054359b590b18ee Mon Sep 17 00:00:00 2001 From: Ozgur Yurekten Date: Fri, 28 Jun 2024 10:57:10 +0100 Subject: [PATCH] fix: windows path and new line issues --- .gitignore | 2 +- README.md | 4 +- .../commands/public/public_remove.py | 2 +- .../submission/submission_create_assay.py | 5 +- .../submission/submission_delete_assay.py | 2 +- .../submission/submission_describe.py | 2 +- .../commands/submission/submission_login.py | 6 +- .../commands/submission/submission_upload.py | 10 +- .../submission/submission_validate.py | 2 +- .../commands/submission/utils.py | 2 +- .../provider/ftp/default_ftp_client.py | 17 +- .../provider/ftp/folder_metadata_collector.py | 21 ++- metabolights_utils/provider/ftp_repository.py | 30 ++-- .../provider/submission_repository.py | 33 ++-- metabolights_utils/tsv/tsv_file_updater.py | 4 +- metabolights_utils/utils/audit_utils.py | 7 +- metabolights_utils/utils/filename_utils.py | 11 +- metabolights_utils/utils/hash_utils.py | 4 +- metabolights_utils/utils/search_utils.py | 4 +- .../examples/test_read_investigation_file.py | 2 +- tests/examples/test_update_isa_table.py | 2 +- .../commands/public/test_public_download.py | 4 +- .../commands/public/test_public_remove.py | 4 +- .../commands/public/test_public_search.py | 6 +- .../submission/test_submission_login.py | 30 ++-- .../submission/test_submission_utils.py | 16 +- .../isatab/test_assay_file.py | 2 +- .../isatab/test_investigation_file.py | 2 +- .../isatab/test_isa_table_actions.py | 168 +++++++++--------- .../test_parse_investigation_from_fs.py | 2 +- .../provider/test_ftp_client.py | 14 +- .../provider/test_ftp_folder_metadata.py | 10 +- .../provider/test_submission_provider.py | 32 ++-- .../metabolights_utils/provider/test_utils.py | 4 +- tests/utils/test_audit_utils.py | 8 +- tests/utils/test_hash_utils.py | 6 +- 36 files changed, 255 insertions(+), 225 deletions(-) diff --git a/.gitignore b/.gitignore index 83dac70..b4216bb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ site/ .sandbox -.test-temp +test-temp # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/README.md b/README.md index a39f8fb..eae388e 100644 --- a/README.md +++ b/README.md @@ -435,7 +435,7 @@ def test_investigation_file_write_01(): ) tmp_file_name = uuid.uuid4().hex - tmp_path = pathlib.Path(f"/tmp/test_{tmp_file_name}.txt") + tmp_path = pathlib.Path(f"test-temp/test_{tmp_file_name}.txt") writer: InvestigationFileWriter = Writer.get_investigation_file_writer() writer.write(result.investigation, file_buffer_or_path=tmp_path) @@ -564,7 +564,7 @@ def test_assay_file_read_write(): "tests/test-data/MTBLS1/a_MTBLS1_metabolite_profiling_NMR_spectroscopy.txt" ) file_path = ( - ".test-temp/test-data/MTBLS1/a_MTBLS1_metabolite_profiling_NMR_spectroscopy.txt" + "test-temp/test-data/MTBLS1/a_MTBLS1_metabolite_profiling_NMR_spectroscopy.txt" ) os.makedirs(os.path.dirname(file_path), exist_ok=True) shutil.copy(path_original, file_path) diff --git a/metabolights_utils/commands/public/public_remove.py b/metabolights_utils/commands/public/public_remove.py index bd4e20b..30fc746 100644 --- a/metabolights_utils/commands/public/public_remove.py +++ b/metabolights_utils/commands/public/public_remove.py @@ -51,4 +51,4 @@ def public_remove( if __name__ == "__main__": - public_remove() + public_remove(["MTBLS1"]) diff --git a/metabolights_utils/commands/submission/submission_create_assay.py b/metabolights_utils/commands/submission/submission_create_assay.py index dbd518c..23bfaca 100644 --- a/metabolights_utils/commands/submission/submission_create_assay.py +++ b/metabolights_utils/commands/submission/submission_create_assay.py @@ -82,7 +82,10 @@ def submission_create_assay( study_id: MetaboLights study accession number (MTBLSxxxx). assay_technique: Valid assay techniques: NMR, LC-MS, LC-DAD, GC-MS, GCxGC-MS, GC-FID, DI-MS, FIA-MS, CE-MS, MALDI-MS, MSImaging - Acronyms: Diode array detection (LC-DAD), Tandem MS (GCxGC-MS), Flame ionisation detector (GC-FID), Direct infusion (DI-MS), Flow injection analysis (FIA-MS), Capillary electrophoresis (CE-MS), Matrix-assisted laser desorption-ionisation imaging mass spectrometry (MALDI-MS), Nuclear magnetic resonance (NMR), Mass spec spectrometry (MSImaging) + Acronyms: Diode array detection (LC-DAD), Tandem MS (GCxGC-MS), Flame ionisation detector (GC-FID), + Direct infusion (DI-MS), Flow injection analysis (FIA-MS), Capillary electrophoresis (CE-MS), + Matrix-assisted laser desorption-ionisation imaging mass spectrometry (MALDI-MS), + Nuclear magnetic resonance (NMR), Mass spec spectrometry (MSImaging) diff --git a/metabolights_utils/commands/submission/submission_delete_assay.py b/metabolights_utils/commands/submission/submission_delete_assay.py index 493af18..d11f553 100644 --- a/metabolights_utils/commands/submission/submission_delete_assay.py +++ b/metabolights_utils/commands/submission/submission_delete_assay.py @@ -99,5 +99,5 @@ def submission_delete_assay( if __name__ == "__main__": submission_delete_assay( - ["MTBLS5397", "a_MTBLS5397_LC-MS_positive_hilic_metabolite_profiling-1.txt"] + ["MTBLS9776", "a_MTBLS5397_LC-MS_positive_hilic_metabolite_profiling-1.txt"] ) diff --git a/metabolights_utils/commands/submission/submission_describe.py b/metabolights_utils/commands/submission/submission_describe.py index 5dcce8b..9163911 100644 --- a/metabolights_utils/commands/submission/submission_describe.py +++ b/metabolights_utils/commands/submission/submission_describe.py @@ -138,4 +138,4 @@ def submission_describe( if __name__ == "__main__": - submission_describe(["MTBLS1", "-o"]) + submission_describe(["MTBLS397", "-o"]) diff --git a/metabolights_utils/commands/submission/submission_login.py b/metabolights_utils/commands/submission/submission_login.py index f51aa4a..f93ec66 100644 --- a/metabolights_utils/commands/submission/submission_login.py +++ b/metabolights_utils/commands/submission/submission_login.py @@ -53,7 +53,7 @@ def submission_login( initial: Union[None, LoginCredentials] = None try: if os.path.exists(credentials_file_path): - with open(credentials_file_path, "r") as f: + with open(credentials_file_path, "r", encoding="utf-8") as f: data = json.load(f) credentials = LoginCredentials.model_validate(data) initial = LoginCredentials.model_validate(data) @@ -84,7 +84,7 @@ def submission_login( if api_token or ftp_user or ftp_user_password: json_data = credentials.model_dump() - with open(credentials_file_path, "w") as f: + with open(credentials_file_path, "w", encoding="utf-8") as f: json.dump(json_data, f, indent=4) click.echo("User credentials are updated succesfully.") else: @@ -94,7 +94,7 @@ def submission_login( if initial: os.makedirs(parent_directory, exist_ok=True) json_data = initial.model_dump_json(indent=4) - with open(credentials_file_path, "w") as f: + with open(credentials_file_path, "w", encoding="utf-8") as f: f.write(json_data) exit(1) diff --git a/metabolights_utils/commands/submission/submission_upload.py b/metabolights_utils/commands/submission/submission_upload.py index 2d41be0..7870ccc 100644 --- a/metabolights_utils/commands/submission/submission_upload.py +++ b/metabolights_utils/commands/submission/submission_upload.py @@ -66,13 +66,15 @@ def submission_upload( rest_api_base_url=rest_api_base_url, credentials_file_path=credentials_file_path, ) - success, error = client.upload_metadata_files( + success, message = client.upload_metadata_files( study_id=study_id, override_remote_files=override_remote_files, metadata_files=None, ) if success: - click.echo(f"Upload private study {study_id}: Success") + click.echo(f"Upload private study {study_id} metadata files: Success") + click.echo(message) + success, error = client.sync_private_ftp_metadata_files( study_id=study_id, pool_period=10, retry=10 ) @@ -82,8 +84,8 @@ def submission_upload( click.echo(f"Failure: Sync private study {study_id} folder: {error}") else: - click.echo(f"Failure: Upload private study {study_id}: {error}") + click.echo(f"Upload private study {study_id}: {message}") if __name__ == "__main__": - submission_upload(["MTBLS5397", "-o"]) + submission_upload(["MTBLS5397"]) diff --git a/metabolights_utils/commands/submission/submission_validate.py b/metabolights_utils/commands/submission/submission_validate.py index 00e848d..1b7219b 100644 --- a/metabolights_utils/commands/submission/submission_validate.py +++ b/metabolights_utils/commands/submission/submission_validate.py @@ -67,7 +67,7 @@ def submission_validate( ) if success: - with open(validation_file_path, "r") as f: + with open(validation_file_path, "r", encoding="utf-8") as f: validation_report = f.readlines() for line in validation_report: click.echo(line.strip()) diff --git a/metabolights_utils/commands/submission/utils.py b/metabolights_utils/commands/submission/utils.py index e161617..1d618e3 100644 --- a/metabolights_utils/commands/submission/utils.py +++ b/metabolights_utils/commands/submission/utils.py @@ -33,7 +33,7 @@ def get_submission_credentials(credentials_file_path: str): credentials: Union[None, LoginCredentials] = None try: if os.path.exists(credentials_file_path): - with open(credentials_file_path, "r") as f: + with open(credentials_file_path, "r", encoding="utf-8") as f: data = json.load(f) credentials = LoginCredentials.model_validate(data) except Exception as ex: diff --git a/metabolights_utils/provider/ftp/default_ftp_client.py b/metabolights_utils/provider/ftp/default_ftp_client.py index e1276aa..cb66ed8 100644 --- a/metabolights_utils/provider/ftp/default_ftp_client.py +++ b/metabolights_utils/provider/ftp/default_ftp_client.py @@ -6,6 +6,7 @@ from typing import List, Set, Union from metabolights_utils.provider.ftp.model import FtpFolderContent, LocalDirectory +from metabolights_utils.utils.filename_utils import join_path logger = logging.getLogger() @@ -22,7 +23,7 @@ def __init__( password: Union[str, None] = None, ) -> None: self.ftp_server_url = ftp_server_url - self.remote_repository_root_directory = remote_repository_root_directory + self.remote_repository_root_directory = remote_repository_root_directory.replace("\\","").rstrip("/") self.local_storage_root_path = local_storage_root_path self.username = username if username else "" self.password = password if password else "" @@ -74,9 +75,9 @@ def list_directory( directory: Union[str, None] = None, search_pattern: Union[str, None] = None, ) -> FtpFolderContent: - directory = directory.strip("/") if directory else "" + directory = directory.replace("\\", '/').strip("/") if directory else "" root_dir = self.remote_repository_root_directory - remote_directory = os.path.join(root_dir, directory) + remote_directory = f"{root_dir}/{directory}" self.connect() @@ -133,6 +134,7 @@ def download_file( keep_local_files: Union[Set[str], None] = None, ) -> LocalDirectory: local_path = local_path if local_path else self.local_storage_root_path + local_path = join_path(local_path) if local_files is None: response = LocalDirectory(root_path=local_path) else: @@ -156,10 +158,11 @@ def download_file( parent_path = os.path.dirname(target_path) filename = os.path.basename(target_path) - remote_directory = os.path.join(remote_root_dir, relative_file_path) + remote_directory = f"{remote_root_dir}/{relative_file_path}".replace("\\", "/") relative_parent_path = os.path.dirname(relative_file_path) - remote_parent_directory = os.path.dirname(remote_directory) + remote_parent_directory = os.path.dirname(os.path.join(remote_root_dir, relative_file_path)) + remote_parent_directory = remote_parent_directory.replace("\\", "/") logger.debug("List files within %s on FTP server ", relative_parent_path) result = self.list_directory(relative_parent_path, search_pattern=filename) is_directory = True @@ -218,7 +221,7 @@ def download_file( item_path: str = os.path.join(target_path, item) relative_item_path = item_path.replace( f"{local_path}", "" - ).strip("/") + ).strip("/").strip("\\") if keep_local_files and item in keep_local_files: actions[relative_item_path] = "SKIPPED" logger.debug( @@ -240,7 +243,7 @@ def download_file( response.local_folders.append(relative_file_path) for collection in (result.folders, result.files): for entry in collection: - new_relative_file_path = os.path.join(relative_file_path, entry) + new_relative_file_path = join_path(relative_file_path, entry).replace("\\", "/") self.download_file( relative_file_path=new_relative_file_path, local_path=local_path, diff --git a/metabolights_utils/provider/ftp/folder_metadata_collector.py b/metabolights_utils/provider/ftp/folder_metadata_collector.py index 5bf1e5c..63b629a 100644 --- a/metabolights_utils/provider/ftp/folder_metadata_collector.py +++ b/metabolights_utils/provider/ftp/folder_metadata_collector.py @@ -16,6 +16,7 @@ from metabolights_utils.provider import definitions from metabolights_utils.provider.ftp.default_ftp_client import DefaultFtpClient from metabolights_utils.provider.study_provider import AbstractFolderMetadataCollector +from metabolights_utils.utils.filename_utils import join_path logger = logging.getLogger() @@ -43,9 +44,9 @@ def __init__( self.study_id = study_id.upper().strip("/") self.folder_index_file_path = ( - folder_index_file_path + join_path(folder_index_file_path) if folder_index_file_path - else os.path.join( + else join_path( definitions.default_local_repority_cache_path, study_id, "mtbls_index.json", @@ -53,9 +54,10 @@ def __init__( ) logger.debug("Folder index path is %s", self.folder_index_file_path) self.rebuild_folder_index_file = rebuild_folder_index_file - self.remote_study_directory = os.path.join( + self.remote_study_directory = join_path( self.client.remote_repository_root_directory, self.study_id ) + self.remote_study_directory = self.remote_study_directory.replace("\\", "/").rstrip("/") def visit_folder( self, @@ -67,6 +69,7 @@ def visit_folder( try: prefix = f"{str(study_path).rstrip('/')}/" + directory = directory.replace("\\", "/") dir_relative_path = ( str(directory).replace(prefix, "") if study_path != directory else "" ) @@ -81,7 +84,7 @@ def visit_folder( messages.append(f"{dir_relative_path} is in ignore list. SKIPPED.") return directory_input = ( - os.path.join(self.study_id, dir_relative_path) + join_path(self.study_id, dir_relative_path) if dir_relative_path else self.study_id ) @@ -125,10 +128,10 @@ def visit_folder( ) for item in selected_descriptors: entry = item.base_name - full_path: str = os.path.join(directory, entry) - relative_path = os.path.join(dir_relative_path, entry) + full_path: str = join_path(directory, entry).replace("\\", "/") + relative_path = join_path(dir_relative_path, entry).replace("\\", "/") base_name = os.path.basename(relative_path) - parent_directory = os.path.dirname(relative_path) + parent_directory = os.path.dirname(relative_path).replace("\\", "/") in_ignore_list = False for pattern in definitions.ignore_file_patterns: if pattern.match(relative_path): @@ -182,7 +185,7 @@ def get_folder_metadata( ): logger.info("%s file exists, loading...", self.folder_index_file_path) try: - with open(self.folder_index_file_path, "r") as f: + with open(self.folder_index_file_path, "r", encoding="utf-8") as f: data = json.load(f) current_file_index = FolderIndex.model_validate( data, from_attributes=True @@ -238,7 +241,7 @@ def get_folder_metadata( now = datetime.datetime.now(datetime.timezone.utc) file_index = FolderIndex(update_time=now, content=study_folder_metadata) - with open(self.folder_index_file_path, "w") as fw: + with open(self.folder_index_file_path, "w", encoding="utf-8") as fw: fw.write(file_index.model_dump_json(indent=4)) msg = f"{self.folder_index_file_path} file is updated." logger.info(msg) diff --git a/metabolights_utils/provider/ftp_repository.py b/metabolights_utils/provider/ftp_repository.py index 9671357..205b7fc 100644 --- a/metabolights_utils/provider/ftp_repository.py +++ b/metabolights_utils/provider/ftp_repository.py @@ -36,6 +36,7 @@ is_metadata_file, is_metadata_filename_pattern, ) +from metabolights_utils.utils.filename_utils import join_path logger = logging.getLogger() @@ -50,9 +51,8 @@ def __init__( ) -> None: self.local_storage_cache_path = local_storage_cache_path if not local_storage_cache_path: - self.local_storage_cache_path = ( - definitions.default_local_repority_cache_path - ) + self.local_storage_cache_path = definitions.default_local_repority_cache_path + self.local_storage_cache_path = join_path(self.local_storage_cache_path) logger.debug( "local_storage_cache_path is set to path: %s", self.local_storage_cache_path, @@ -60,6 +60,7 @@ def __init__( self.local_storage_root_path = local_storage_root_path if not self.local_storage_root_path: self.local_storage_root_path = definitions.default_local_repority_root_path + self.local_storage_root_path = join_path(self.local_storage_root_path) logger.debug( "local_storage_root_path is set to path: %s", self.local_storage_root_path, @@ -76,6 +77,7 @@ def __init__( self.remote_repository_root_directory = ( definitions.default_remote_repository_root_directory ) + self.remote_repository_root_directory = self.remote_repository_root_directory.replace("\\", "/") logger.debug( "remote_repository_root_directory is set to directory: %s", self.remote_repository_root_directory, @@ -104,16 +106,17 @@ def load_study_model( study_id = study_id.upper().strip("/") if not local_path: local_path = self.local_storage_root_path - target_path = os.path.join(local_path, study_id) + + target_path = join_path(local_path, study_id) if not folder_index_file_path: - folder_index_file_path = os.path.join( + folder_index_file_path = join_path( self.local_storage_cache_path, study_id, "mtbls_index.json", ) model_cache_path = study_model_file_path if not study_model_file_path: - model_cache_path = os.path.join( + model_cache_path = join_path( self.local_storage_cache_path, study_id, "study_model.json", @@ -124,7 +127,7 @@ def load_study_model( if use_study_model_cache: try: if os.path.exists(model_cache_path): - with open(model_cache_path) as f: + with open(model_cache_path, encoding="utf-8") as f: data = json.load(f) model = MetabolightsStudyModel.model_validate(data) if model.investigation_file_path == "i_Investigation.txt": @@ -167,7 +170,7 @@ def load_study_model( if model: parent = os.path.dirname(model_cache_path) os.makedirs(parent, exist_ok=True) - with open(model_cache_path, "w") as fw: + with open(model_cache_path, "w", encoding="utf-8") as fw: fw.write(model.model_dump_json(indent=4)) return model, [InfoMessage(short="Loaded from local isa metadata files.")] @@ -228,7 +231,7 @@ def load_study_model( if model: parent = os.path.dirname(model_cache_path) os.makedirs(parent, exist_ok=True) - with open(model_cache_path, "w") as fw: + with open(model_cache_path, "w", encoding="utf-8") as fw: fw.write(model.model_dump_json(indent=4)) return model, messages @@ -244,6 +247,7 @@ def download_study_metadata_files( return LocalDirectory(root_path="", code=400, message="invalid study_id") if not local_path: local_path = f"{self.local_storage_root_path}/{study_id}" + local_path = join_path(local_path) response = LocalDirectory(root_path=local_path) study_id = study_id.upper().strip("/") @@ -277,7 +281,7 @@ def download_study_metadata_files( try: for filename in requested_files: # new_local_path = os.path.join(local_path, study_id) - new_relative_file_path = os.path.join(study_id, filename) + new_relative_file_path = f"{study_id}/{filename}".replace("\\", "/").rstrip("/") current_file = filename self.ftp_client.download_file( relative_file_path=new_relative_file_path, @@ -291,7 +295,7 @@ def download_study_metadata_files( if delete_unlisted_metadata_files: for filename in os.listdir(local_path): if filename not in listed_files: - file_path = os.path.join(local_path, filename) + file_path = join_path(local_path, filename) if is_metadata_file(file_path): response.actions[filename] = "DELETED" os.remove(file_path) @@ -334,7 +338,7 @@ def download_study_data_files( study_id = study_id.upper().strip("/") try: for file in selected_data_files: - new_relative_file_path = os.path.join(study_id, file) + new_relative_file_path = f"{study_id}/{file}".replace("\\", "/").rstrip("/") self.ftp_client.download_file( relative_file_path=new_relative_file_path, local_path=local_path, @@ -387,7 +391,7 @@ def list_study_directory( ) -> FtpFolderContent: study_id = study_id.upper().strip("/") if study_id else "" - directory = os.path.join(study_id, subdirectory) if subdirectory else study_id + directory = f"{study_id}/{subdirectory}" if subdirectory else study_id return self.ftp_client.list_directory(directory) def rebuild_study_folder_content( diff --git a/metabolights_utils/provider/submission_repository.py b/metabolights_utils/provider/submission_repository.py index e06a443..2d128d3 100644 --- a/metabolights_utils/provider/submission_repository.py +++ b/metabolights_utils/provider/submission_repository.py @@ -49,6 +49,7 @@ is_metadata_filename_pattern, rest_api_get, ) +from metabolights_utils.utils.filename_utils import join_path class MetabolightsSubmissionRepository: @@ -192,8 +193,10 @@ def upload_metadata_files( ) -> Tuple[bool, str]: if not local_path: local_path = self.local_storage_root_path + local_path = join_path(local_path) if not credentials_file_path: credentials_file_path = self.credentials_file_path + credentials_file_path = join_path(credentials_file_path) if not ftp_server_url: ftp_server_url = self.ftp_server_url @@ -235,12 +238,12 @@ def upload_metadata_files( file_path = os.path.join(study_path, file_name) if remote_modified_time: local_modified_time = int(os.path.getmtime(file_path)) - if remote_modified_time > local_modified_time: + if remote_modified_time < local_modified_time: new_requested_files.append(relative_path) else: new_requested_files.append(relative_path) - if not new_requested_files and override_remote_files: - return False, "No files to upload" + if not new_requested_files: + return False, "There is no metadata file to upload or local metadata files are up-to-date." username, password, error = self.get_ftp_credentials() if not error: ftp_client = DefaultFtpClient( @@ -254,7 +257,7 @@ def upload_metadata_files( try: ftp_client.upload_files(remote_folder_directory, input_files) - return True, None + return True, "Uploaded Files: " + ", ".join(new_requested_files) except Exception as ex: return False, str(ex) else: @@ -298,7 +301,7 @@ def validate_study( api_name = "validation task start" try: - url = os.path.join(self.rest_api_base_url.rstrip("/"), sub_path.lstrip("/")) + url = f"{self.rest_api_base_url.rstrip('/')}/{sub_path.lstrip('/')}" parameters = {} response = httpx.post( url=url, @@ -358,10 +361,7 @@ def validate_study( try: api_success = False report_sub_path = f"/studies/{study_id}/validation-report" - report_url = os.path.join( - self.rest_api_base_url.rstrip("/"), - report_sub_path.lstrip("/"), - ) + report_url = f"{self.rest_api_base_url.rstrip('/')}/{report_sub_path.lstrip('/')}" for _ in range(retry + 1): try: response = httpx.get( @@ -395,7 +395,7 @@ def validate_study( dir_name = os.path.dirname(validation_file_path) os.makedirs(dir_name, exist_ok=True) - with open(validation_file_path, "w") as f: + with open(validation_file_path, "w", encoding="utf-8") as f: f.write( "section\t" "status\t" "message\t" "description\t" "metadata_file\n" ) @@ -424,7 +424,7 @@ def sync_private_ftp_metadata_files( headers["Source-Staging-Area"] = "private-ftp" headers["Target-Staging-Area"] = "rw-study" - url = os.path.join(self.rest_api_base_url.rstrip("/"), sub_path.lstrip("/")) + url = f"{self.rest_api_base_url.rstrip('/')}/{sub_path.lstrip('/')}" parameters = {} response = httpx.post( url=url, @@ -479,7 +479,7 @@ def create_assay( return None, None, error sub_path = f"/studies/{study_id}/assays" - url = os.path.join(self.rest_api_base_url.rstrip("/"), sub_path.lstrip("/")) + url = f"{self.rest_api_base_url.rstrip('/')}/{sub_path.lstrip('/')}" body = {"assay": {"type": assay_technique, "columns": []}} if scan_polarity: body["assay"]["columns"].append( @@ -522,8 +522,7 @@ def delete_assay( return None, None, error sub_path = f"/studies/{study_id}/assays/{assay_filename}" - url = os.path.join(self.rest_api_base_url.rstrip("/"), sub_path.lstrip("/")) - + url = f"{self.rest_api_base_url.rstrip('/')}/{sub_path.lstrip('/')}" try: response = httpx.delete( url=url, @@ -564,6 +563,7 @@ def download_submission_metadata_files( return LocalDirectory(code=400, message="Invalid study_id") if not local_path: local_path = self.local_storage_root_path + local_path = join_path(local_path) response = LocalDirectory(root_path=local_path) study_id = study_id.upper().strip("/") @@ -609,6 +609,7 @@ def download_submission_metadata_files( remote_modified_time = int(modified) if remote_modified_time > local_modified_time: new_requested_files.append(filename) + response.actions[key] = "DOWNLOADED" else: response.actions[key] = "SKIPPED" else: @@ -703,7 +704,7 @@ def list_studies( return None, error sub_path = "/studies/user" - url = os.path.join(self.rest_api_base_url.rstrip("/"), sub_path.lstrip("/")) + url = f"{self.rest_api_base_url.rstrip('/')}/{sub_path.lstrip('/')}" response = httpx.get( url=url, timeout=timeout, @@ -776,7 +777,7 @@ def list_study_directory( if subdirectory: parameters["directory"] = subdirectory.strip("/") paths.append(subdirectory.strip("/")) - url = os.path.join(self.rest_api_base_url.rstrip("/"), sub_path.lstrip("/")) + url = f"{self.rest_api_base_url.rstrip('/')}/{sub_path.lstrip('/')}" data, error = rest_api_get( url, timeout=timeout, parameters=parameters, headers=headers ) diff --git a/metabolights_utils/tsv/tsv_file_updater.py b/metabolights_utils/tsv/tsv_file_updater.py index bed7830..164c653 100644 --- a/metabolights_utils/tsv/tsv_file_updater.py +++ b/metabolights_utils/tsv/tsv_file_updater.py @@ -74,7 +74,7 @@ def apply_actions( f"File '{str(file)}' does not exist or it is not a regular file." ) return report - sha256 = HashUtils.sha256sum(file) + sha256 = HashUtils.sha256sum(file, convert_to_linux_line_ending=True) if sha256 != file_sha256_hash: report.message = f"SH256 of '{str(file)}' does not match the input value." @@ -82,7 +82,7 @@ def apply_actions( task_id = str(uuid.uuid4().hex) timestamp = str(int(datetime.datetime.now(datetime.timezone.utc).timestamp())) - temp_folder = pathlib.Path(f"/tmp/isa_table_actions/{timestamp}/{task_id}") + temp_folder = pathlib.Path(f"test-temp/isa_table_actions/{timestamp}/{task_id}") os.makedirs(str(temp_folder)) file_copy_path = temp_folder / pathlib.Path(f"{file.name}_{task_id}") diff --git a/metabolights_utils/utils/audit_utils.py b/metabolights_utils/utils/audit_utils.py index fc9e356..c128027 100644 --- a/metabolights_utils/utils/audit_utils.py +++ b/metabolights_utils/utils/audit_utils.py @@ -3,6 +3,7 @@ import shutil from typing import Union +from metabolights_utils.utils.filename_utils import join_path from metabolights_utils.utils.search_utils import MetabolightsSearchUtils as SearchUtils @@ -30,13 +31,13 @@ def create_audit_folder( f"{folder_prefix}_{folder_name}" if folder_prefix else folder_name ) - target_folder_path = os.path.join(target_root_path, folder_name) + target_folder_path = join_path(target_root_path, folder_name) os.makedirs(target_folder_path, exist_ok=True) for file in metadata_files_list: basename = os.path.basename(file) - target_file = os.path.join(target_folder_path, basename) + target_file = join_path(target_folder_path, basename) shutil.copy2(file, target_file, follow_symlinks=False) return target_folder_path return None @@ -54,7 +55,7 @@ def copy_isa_metadata_files( os.makedirs(target_folder_path, exist_ok=True) for file in metadata_files_list: basename = os.path.basename(file) - target_file = os.path.join(target_folder_path, basename) + target_file = join_path(target_folder_path, basename) shutil.copy2(file, target_file, follow_symlinks=False) return target_folder_path return None diff --git a/metabolights_utils/utils/filename_utils.py b/metabolights_utils/utils/filename_utils.py index 8e28c58..f3aef44 100644 --- a/metabolights_utils/utils/filename_utils.py +++ b/metabolights_utils/utils/filename_utils.py @@ -1,5 +1,6 @@ +import os import re -from typing import Union +from typing import List, Tuple, Union import unidecode @@ -25,3 +26,11 @@ def sanitise_filename( filename = re.sub(replacement_chars_pattern, "_", filename) return filename + + +def join_path(*args: Tuple[str]): + target_sep = "/" if os.sep == "/" else "\\" + source_sep = "/" if os.sep == "\\" else "/" + inputs = list([x for x in args if x and x.strip()]) + _path = target_sep.join(inputs).replace(source_sep, target_sep) + return _path \ No newline at end of file diff --git a/metabolights_utils/utils/hash_utils.py b/metabolights_utils/utils/hash_utils.py index 67e3d55..8d06055 100644 --- a/metabolights_utils/utils/hash_utils.py +++ b/metabolights_utils/utils/hash_utils.py @@ -16,13 +16,15 @@ class IsaMetadataFolderHash(BaseModel): class MetabolightsHashUtils(object): @staticmethod - def sha256sum(filepath: str): + def sha256sum(filepath: str, convert_to_linux_line_ending: bool = True): if not filepath or not os.path.exists(filepath): return EMPTY_FILE_HASH sha256_hash = hashlib.sha256() with open(filepath, "rb") as f: for byte_block in iter(lambda: f.read(4096), b""): + if convert_to_linux_line_ending: + byte_block = byte_block.replace(b"\r\n", b"\n") sha256_hash.update(byte_block) return sha256_hash.hexdigest() diff --git a/metabolights_utils/utils/search_utils.py b/metabolights_utils/utils/search_utils.py index e2be773..25afb14 100644 --- a/metabolights_utils/utils/search_utils.py +++ b/metabolights_utils/utils/search_utils.py @@ -2,6 +2,8 @@ import os from typing import List +from metabolights_utils.utils.filename_utils import join_path + class MetabolightsSearchUtils(object): @staticmethod @@ -12,6 +14,6 @@ def get_isa_metadata_files(folder_path: str, recursive=False) -> List[str]: return metadata_files for pattern in patterns: - search_pattern = os.path.join(folder_path, pattern) + search_pattern = join_path(folder_path, pattern) metadata_files.extend(glob.glob(search_pattern, recursive=recursive)) return metadata_files diff --git a/tests/examples/test_read_investigation_file.py b/tests/examples/test_read_investigation_file.py index a99981f..a4302af 100644 --- a/tests/examples/test_read_investigation_file.py +++ b/tests/examples/test_read_investigation_file.py @@ -38,7 +38,7 @@ def test_investigation_file_write_01(): ) tmp_file_name = uuid.uuid4().hex - tmp_path = pathlib.Path(f"/tmp/test_{tmp_file_name}.txt") + tmp_path = pathlib.Path(f"test-temp/test_{tmp_file_name}.txt") writer: InvestigationFileWriter = Writer.get_investigation_file_writer() writer.write(result.investigation, file_buffer_or_path=tmp_path) diff --git a/tests/examples/test_update_isa_table.py b/tests/examples/test_update_isa_table.py index add2f37..ca9f804 100644 --- a/tests/examples/test_update_isa_table.py +++ b/tests/examples/test_update_isa_table.py @@ -15,7 +15,7 @@ def test_assay_metadata_file_read_write(): "tests/test-data/MTBLS1/a_MTBLS1_metabolite_profiling_NMR_spectroscopy.txt" ) file_path = ( - ".test-temp/test-data/MTBLS1/a_MTBLS1_metabolite_profiling_NMR_spectroscopy.txt" + "test-temp/test-data/MTBLS1/a_MTBLS1_metabolite_profiling_NMR_spectroscopy.txt" ) try: os.makedirs(os.path.dirname(file_path), exist_ok=True) diff --git a/tests/metabolights_utils/commands/public/test_public_download.py b/tests/metabolights_utils/commands/public/test_public_download.py index fcd1a55..d4c02ad 100644 --- a/tests/metabolights_utils/commands/public/test_public_download.py +++ b/tests/metabolights_utils/commands/public/test_public_download.py @@ -20,7 +20,7 @@ def test_public_download_01(mocker: MockerFixture, study_id: str): mock_local_directory.success = True mock_local_directory.actions = {f"s_{study_id}.txt": "DOWNLOAD"} mock_repository.download_study_metadata_files.return_value = mock_local_directory - mock_repository.local_storage_root_path = "/tmp/test" + mock_repository.local_storage_root_path = "test-temp/test" runner = CliRunner() result = runner.invoke(public_download, [study_id]) @@ -46,7 +46,7 @@ def test_public_download_02(mocker: MockerFixture, study_id: str): mock_local_directory = MagicMock() mock_repository.download_study_metadata_files.return_value = mock_local_directory - mock_repository.local_storage_root_path = "/tmp/test" + mock_repository.local_storage_root_path = "test-temp/test" mock_local_directory.actions = {} runner = CliRunner() diff --git a/tests/metabolights_utils/commands/public/test_public_remove.py b/tests/metabolights_utils/commands/public/test_public_remove.py index d315c32..1712118 100644 --- a/tests/metabolights_utils/commands/public/test_public_remove.py +++ b/tests/metabolights_utils/commands/public/test_public_remove.py @@ -80,8 +80,8 @@ def test_public_remove_04(study_id: str): def test_public_remove_05(study_id: str): runner = CliRunner() - local_path = f"/tmp/{random.randint(1000000, 9999999)}" - local_cache_path = f"/tmp/{random.randint(1000000, 9999999)}_tmp" + local_path = f"test-temp/{random.randint(1000000, 9999999)}" + local_cache_path = f"test-temp/{random.randint(1000000, 9999999)}_tmp" try: study_path = os.path.join(local_path, study_id) study_cache_path = os.path.join(local_cache_path, study_id) diff --git a/tests/metabolights_utils/commands/public/test_public_search.py b/tests/metabolights_utils/commands/public/test_public_search.py index 016f25d..082f414 100644 --- a/tests/metabolights_utils/commands/public/test_public_search.py +++ b/tests/metabolights_utils/commands/public/test_public_search.py @@ -47,7 +47,7 @@ def test_public_search_02(mocker: MockerFixture, study_id: str): @pytest.mark.parametrize("study_id", ["MTBLS1", "MTBLS60"]) def test_public_search_03(mocker: MockerFixture, study_id: str): - with open("tests/test-data/rest-api-test-data/study_search.json", "r") as f: + with open("tests/test-data/rest-api-test-data/study_search.json", "r", encoding="utf-8") as f: data = f.read() response = json.loads(data) mocker.patch( @@ -64,7 +64,7 @@ def test_public_search_03(mocker: MockerFixture, study_id: str): def test_public_search_04(mocker: MockerFixture): - with open("tests/test-data/rest-api-test-data/diet_search.json", "r") as f: + with open("tests/test-data/rest-api-test-data/diet_search.json", "r", encoding="utf-8") as f: data = f.read() response = json.loads(data) mocker.patch( @@ -180,7 +180,7 @@ def test_public_search_08(mocker: MockerFixture): def test_public_search_09(mocker: MockerFixture): with open( - "tests/test-data/rest-api-test-data/diet_aggregation_search.json", "r" + "tests/test-data/rest-api-test-data/diet_aggregation_search.json", "r", encoding="utf-8" ) as f: data = f.read() response = json.loads(data) diff --git a/tests/metabolights_utils/commands/submission/test_submission_login.py b/tests/metabolights_utils/commands/submission/test_submission_login.py index 7196ae7..a521b3b 100644 --- a/tests/metabolights_utils/commands/submission/test_submission_login.py +++ b/tests/metabolights_utils/commands/submission/test_submission_login.py @@ -15,7 +15,7 @@ def test_submission_login_01(): """ runner = CliRunner() credentials_file_path = ( - f"/tmp/mtbls_unit_test_login{random.randint(1000000, 9999999)}_tmp" + f"test-temp/mtbls_unit_test_login{random.randint(1000000, 9999999)}_tmp" ) try: @@ -41,11 +41,11 @@ def test_submission_login_02(): """ runner = CliRunner() credentials_file_path = ( - f"/tmp/mtbls_unit_test_login{random.randint(1000000, 9999999)}_tmp" + f"test-temp/mtbls_unit_test_login{random.randint(1000000, 9999999)}_tmp" ) try: - with open(credentials_file_path, "w") as f: + with open(credentials_file_path, "w", encoding="utf-8") as f: json.dump("invalid", f) assert os.path.exists(credentials_file_path) @@ -70,7 +70,7 @@ def test_submission_login_03(): """ runner = CliRunner() credentials_file_path = ( - f"/tmp/mtbls_unit_test_login{random.randint(1000000, 9999999)}_tmp" + f"test-temp/mtbls_unit_test_login{random.randint(1000000, 9999999)}_tmp" ) try: @@ -80,7 +80,7 @@ def test_submission_login_03(): "https://www.ebi.ac.uk/metabolights/ws": {"api_token": "c"} }, } - with open(credentials_file_path, "w") as f: + with open(credentials_file_path, "w", encoding="utf-8") as f: json.dump(test_data, f) assert os.path.exists(credentials_file_path) result = runner.invoke( @@ -91,7 +91,7 @@ def test_submission_login_03(): assert result.exit_code == 0 assert "User credentials are updated succesfully" in result.output assert os.path.exists(credentials_file_path) - with open(credentials_file_path, "r") as f: + with open(credentials_file_path, "r", encoding="utf-8") as f: data = json.load(f) credentials = LoginCredentials.model_validate(data) assert credentials.ftp_login["ftp-private.ebi.ac.uk"].user_name == "x" @@ -115,7 +115,7 @@ def test_submission_login_04(): """ runner = CliRunner() credentials_file_path = ( - f"/tmp/mtbls_unit_test_login{random.randint(1000000, 9999999)}_tmp" + f"test-temp/mtbls_unit_test_login{random.randint(1000000, 9999999)}_tmp" ) try: @@ -125,7 +125,7 @@ def test_submission_login_04(): "https://www.ebi.ac.uk/metabolights/ws": {"api_token": "z"} }, } - with open(credentials_file_path, "w") as f: + with open(credentials_file_path, "w", encoding="utf-8") as f: json.dump(test_data, f) assert os.path.exists(credentials_file_path) result = runner.invoke( @@ -136,7 +136,7 @@ def test_submission_login_04(): assert result.exit_code == 0 assert "User credentials are not updated." in result.output assert os.path.exists(credentials_file_path) - with open(credentials_file_path, "r") as f: + with open(credentials_file_path, "r", encoding="utf-8") as f: data = json.load(f) credentials = LoginCredentials.model_validate(data) assert credentials.ftp_login["ftp-private.ebi.ac.uk"].user_name == "x" @@ -160,7 +160,7 @@ def test_submission_login_05(mocker: MockerFixture): """ runner = CliRunner() credentials_file_path = ( - f"/tmp/mtbls_unit_test_login{random.randint(1000000, 9999999)}_tmp" + f"test-temp/mtbls_unit_test_login{random.randint(1000000, 9999999)}_tmp" ) try: @@ -170,7 +170,7 @@ def test_submission_login_05(mocker: MockerFixture): "https://www.ebi.ac.uk/metabolights/ws": {"api_token": "z"} }, } - with open(credentials_file_path, "w") as f: + with open(credentials_file_path, "w", encoding="utf-8") as f: json.dump(test_data, f) assert os.path.exists(credentials_file_path) mocker.patch( @@ -185,7 +185,7 @@ def test_submission_login_05(mocker: MockerFixture): assert result.exit_code == 1 assert "Error while login" in result.output assert os.path.exists(credentials_file_path) - with open(credentials_file_path, "r") as f: + with open(credentials_file_path, "r", encoding="utf-8") as f: data = json.load(f) credentials = LoginCredentials.model_validate(data) assert credentials.ftp_login["ftp-private.ebi.ac.uk"].user_name == "x" @@ -209,7 +209,7 @@ def test_submission_login_06(): """ runner = CliRunner() credentials_file_path = ( - f"/tmp/mtbls_unit_test_login{random.randint(1000000, 9999999)}_tmp" + f"test-temp/mtbls_unit_test_login{random.randint(1000000, 9999999)}_tmp" ) try: @@ -219,7 +219,7 @@ def test_submission_login_06(): "https://www.ebi.ac.uk/metabolights/ws": {"api_token": "z"} }, } - with open(credentials_file_path, "w") as f: + with open(credentials_file_path, "w", encoding="utf-8") as f: json.dump(test_data, f) assert os.path.exists(credentials_file_path) rest_api_base_url = "test-url" @@ -239,7 +239,7 @@ def test_submission_login_06(): assert result.exit_code == 0 assert "User credentials are updated succesfully" in result.output assert os.path.exists(credentials_file_path) - with open(credentials_file_path, "r") as f: + with open(credentials_file_path, "r", encoding="utf-8") as f: data = json.load(f) credentials = LoginCredentials.model_validate(data) assert credentials.ftp_login["ftp-private.ebi.ac.uk"].user_name == "x" diff --git a/tests/metabolights_utils/commands/submission/test_submission_utils.py b/tests/metabolights_utils/commands/submission/test_submission_utils.py index 7f40ea5..83fbe6c 100644 --- a/tests/metabolights_utils/commands/submission/test_submission_utils.py +++ b/tests/metabolights_utils/commands/submission/test_submission_utils.py @@ -17,11 +17,11 @@ def test_get_submission_credentials_01(): def test_get_submission_credentials_02(): test_data = "invalid" credentials_file_path = ( - f"/tmp/mtbls_unit_test_{random.randint(1000000, 9999999)}_tmp" + f"test-temp/mtbls_unit_test_{random.randint(1000000, 9999999)}_tmp" ) try: - with open(credentials_file_path, "w") as f: + with open(credentials_file_path, "w", encoding="utf-8") as f: json.dump(test_data, f) result = get_submission_credentials(credentials_file_path) assert result is None @@ -40,11 +40,11 @@ def test_get_submission_credentials_03(): }, } credentials_file_path = ( - f"/tmp/mtbls_unit_test_{random.randint(1000000, 9999999)}_tmp" + f"test-temp/mtbls_unit_test_{random.randint(1000000, 9999999)}_tmp" ) try: - with open(credentials_file_path, "w") as f: + with open(credentials_file_path, "w", encoding="utf-8") as f: json.dump(test_data, f) result = get_submission_credentials(credentials_file_path) assert result @@ -72,11 +72,11 @@ def test_get_submission_private_ftp_credentials_01(): }, } credentials_file_path = ( - f"/tmp/mtbls_unit_test_{random.randint(1000000, 9999999)}_tmp" + f"test-temp/mtbls_unit_test_{random.randint(1000000, 9999999)}_tmp" ) try: - with open(credentials_file_path, "w") as f: + with open(credentials_file_path, "w", encoding="utf-8") as f: json.dump(test_data, f) result = get_submission_private_ftp_credentials( credentials_file_path, "ftp-private.ebi.ac.uk" @@ -105,11 +105,11 @@ def test_get_submission_rest_api_credentials_01(): }, } credentials_file_path = ( - f"/tmp/mtbls_unit_test_{random.randint(1000000, 9999999)}_tmp" + f"test-temp/mtbls_unit_test_{random.randint(1000000, 9999999)}_tmp" ) try: - with open(credentials_file_path, "w") as f: + with open(credentials_file_path, "w", encoding="utf-8") as f: json.dump(test_data, f) result = get_submission_rest_api_credentials( credentials_file_path, "https://www.ebi.ac.uk/metabolights/ws" diff --git a/tests/metabolights_utils/isatab/test_assay_file.py b/tests/metabolights_utils/isatab/test_assay_file.py index 88d5c23..593cd01 100644 --- a/tests/metabolights_utils/isatab/test_assay_file.py +++ b/tests/metabolights_utils/isatab/test_assay_file.py @@ -184,7 +184,7 @@ def test_assay_metadata_file_read_write(): "tests/test-data/MTBLS1/a_MTBLS1_metabolite_profiling_NMR_spectroscopy.txt" ) file_path = ( - ".test-temp/test-data/MTBLS1/a_MTBLS1_metabolite_profiling_NMR_spectroscopy.txt" + "test-temp/test-data/MTBLS1/a_MTBLS1_metabolite_profiling_NMR_spectroscopy.txt" ) try: os.makedirs(os.path.dirname(file_path), exist_ok=True) diff --git a/tests/metabolights_utils/isatab/test_investigation_file.py b/tests/metabolights_utils/isatab/test_investigation_file.py index 2c2225a..6ff529b 100644 --- a/tests/metabolights_utils/isatab/test_investigation_file.py +++ b/tests/metabolights_utils/isatab/test_investigation_file.py @@ -22,7 +22,7 @@ def test_investigation_file_success_01(): assert len(result.investigation.studies) == 1 tmp_file_name = uuid.uuid4().hex - tmp_path = pathlib.Path(f"/tmp/test_{tmp_file_name}.txt") + tmp_path = pathlib.Path(f"test-temp/test_{tmp_file_name}.txt") writer: InvestigationFileWriter = Writer.get_investigation_file_writer() writer.write(result.investigation, file_buffer_or_path=tmp_path) diff --git a/tests/metabolights_utils/isatab/test_isa_table_actions.py b/tests/metabolights_utils/isatab/test_isa_table_actions.py index a6ff21a..ee1e275 100644 --- a/tests/metabolights_utils/isatab/test_isa_table_actions.py +++ b/tests/metabolights_utils/isatab/test_isa_table_actions.py @@ -21,6 +21,7 @@ TsvUpdateRowsAction, ) from metabolights_utils.tsv.tsv_file_updater import TsvFileUpdater +from metabolights_utils.utils.filename_utils import join_path def test_add_row_action_01(): @@ -35,13 +36,13 @@ def test_add_row_action_01(): action: TsvAddRowsAction = TsvAddRowsAction( new_row_indices=[2, 5, 7, 8170, 8171, 8172, 8173], row_data=row_data ) - path_original = "tests/test-data/test-data-01/s_MTBLS66_test_01.txt" - path = ".test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt" - os.makedirs(os.path.dirname(path), exist_ok=True) - shutil.copy(path_original, path) + path_original = os.path.realpath(join_path("tests/test-data/test-data-01/s_MTBLS66_test_01.txt")) + path_target =os.path.realpath(join_path("test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt")) + os.makedirs(os.path.dirname(path_target), exist_ok=True) + shutil.copy(path_original, path_target) isa_table_updater = TsvFileUpdater() result: TsvActionReport = isa_table_updater.apply_actions( - file_path=path, file_sha256_hash=sha, actions=[action] + file_path=path_target, file_sha256_hash=sha, actions=[action] ) assert result.success assert result.updated_file_sha256_hash @@ -50,7 +51,7 @@ def test_add_row_action_01(): current_row_indices=[8171, 8172] ) result: TsvActionReport = isa_table_updater.apply_actions( - file_path=path, + file_path=path_target, file_sha256_hash=result.updated_file_sha256_hash, actions=[delete_row_action], ) @@ -66,7 +67,7 @@ def test_add_row_action_01(): row_data[8171] = row update_row_action: TsvActionReport = TsvUpdateRowsAction(rows=row_data) result: TsvActionReport = isa_table_updater.apply_actions( - file_path=path, + file_path=path_target, file_sha256_hash=result.updated_file_sha256_hash, actions=[update_row_action], ) @@ -77,15 +78,15 @@ def test_add_row_action_01(): def test_move_row_action_01(): sha = "add85bc3770cda13450b6fd95fe1735fe6337553076ed7cd269fc8163e002fac" isa_table_updater = TsvFileUpdater() - path_original = "tests/test-data/test-data-01/s_MTBLS66_test_01.txt" - path = ".test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt" - os.makedirs(os.path.dirname(path), exist_ok=True) - shutil.copy(path_original, path) + path_original = os.path.realpath(join_path("tests/test-data/test-data-01/s_MTBLS66_test_01.txt")) + path_target =os.path.realpath(os.path.realpath(join_path("test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt"))) + os.makedirs(os.path.dirname(path_target), exist_ok=True) + shutil.copy(path_original, path_target) move_row_action: TsvActionReport = TsvMoveRowAction( source_row_index=4, new_row_index=1 ) result: TsvActionReport = isa_table_updater.apply_actions( - file_path=path, + file_path=path_target, file_sha256_hash=sha, actions=[move_row_action], ) @@ -96,15 +97,15 @@ def test_move_row_action_01(): def test_move_row_action_02(): sha = "add85bc3770cda13450b6fd95fe1735fe6337553076ed7cd269fc8163e002fac" isa_table_updater = TsvFileUpdater() - path_original = "tests/test-data/test-data-01/s_MTBLS66_test_01.txt" - path = ".test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt" - os.makedirs(os.path.dirname(path), exist_ok=True) - shutil.copy(path_original, path) + path_original = os.path.realpath(join_path("tests/test-data/test-data-01/s_MTBLS66_test_01.txt")) + path_target =os.path.realpath(os.path.realpath(join_path("test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt"))) + os.makedirs(os.path.dirname(path_target), exist_ok=True) + shutil.copy(path_original, path_target) move_row_action: TsvActionReport = TsvMoveRowAction( source_row_index=10, new_row_index=0 ) result: TsvActionReport = isa_table_updater.apply_actions( - file_path=path, + file_path=path_target, file_sha256_hash=sha, actions=[move_row_action], ) @@ -115,15 +116,15 @@ def test_move_row_action_02(): def test_move_row_action_03(): sha = "add85bc3770cda13450b6fd95fe1735fe6337553076ed7cd269fc8163e002fac" isa_table_updater = TsvFileUpdater() - path_original = "tests/test-data/test-data-01/s_MTBLS66_test_01.txt" - path = ".test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt" - os.makedirs(os.path.dirname(path), exist_ok=True) - shutil.copy(path_original, path) + path_original = os.path.realpath(join_path("tests/test-data/test-data-01/s_MTBLS66_test_01.txt")) + path_target =os.path.realpath(os.path.realpath(join_path("test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt"))) + os.makedirs(os.path.dirname(path_target), exist_ok=True) + shutil.copy(path_original, path_target) move_row_action: TsvActionReport = TsvMoveRowAction( source_row_index=2, new_row_index=8166 ) result: TsvActionReport = isa_table_updater.apply_actions( - file_path=path, + file_path=path_target, file_sha256_hash=sha, actions=[move_row_action], ) @@ -134,15 +135,15 @@ def test_move_row_action_03(): def test_move_row_action_04(): sha = "add85bc3770cda13450b6fd95fe1735fe6337553076ed7cd269fc8163e002fac" isa_table_updater = TsvFileUpdater() - path_original = "tests/test-data/test-data-01/s_MTBLS66_test_01.txt" - path = ".test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt" - os.makedirs(os.path.dirname(path), exist_ok=True) - shutil.copy(path_original, path) + path_original = os.path.realpath(join_path("tests/test-data/test-data-01/s_MTBLS66_test_01.txt")) + path_target =os.path.realpath(os.path.realpath(join_path("test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt"))) + os.makedirs(os.path.dirname(path_target), exist_ok=True) + shutil.copy(path_original, path_target) move_row_action: TsvActionReport = TsvMoveRowAction( source_row_index=8166, new_row_index=20 ) result: TsvActionReport = isa_table_updater.apply_actions( - file_path=path, + file_path=path_target, file_sha256_hash=sha, actions=[move_row_action], ) @@ -153,15 +154,15 @@ def test_move_row_action_04(): def test_move_row_action_05(): sha = "add85bc3770cda13450b6fd95fe1735fe6337553076ed7cd269fc8163e002fac" isa_table_updater = TsvFileUpdater() - path_original = "tests/test-data/test-data-01/s_MTBLS66_test_01.txt" - path = ".test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt" - os.makedirs(os.path.dirname(path), exist_ok=True) - shutil.copy(path_original, path) + path_original = os.path.realpath(join_path("tests/test-data/test-data-01/s_MTBLS66_test_01.txt")) + path_target =os.path.realpath(os.path.realpath(join_path("test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt"))) + os.makedirs(os.path.dirname(path_target), exist_ok=True) + shutil.copy(path_original, path_target) move_row_action: TsvActionReport = TsvMoveRowAction( source_row_index=20, new_row_index=21 ) result: TsvActionReport = isa_table_updater.apply_actions( - file_path=path, + file_path=path_target, file_sha256_hash=sha, actions=[move_row_action], ) @@ -172,15 +173,15 @@ def test_move_row_action_05(): def test_move_row_action_06(): sha = "add85bc3770cda13450b6fd95fe1735fe6337553076ed7cd269fc8163e002fac" isa_table_updater = TsvFileUpdater() - path_original = "tests/test-data/test-data-01/s_MTBLS66_test_01.txt" - path = ".test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt" - os.makedirs(os.path.dirname(path), exist_ok=True) - shutil.copy(path_original, path) + path_original = os.path.realpath(join_path("tests/test-data/test-data-01/s_MTBLS66_test_01.txt")) + path_target =os.path.realpath(os.path.realpath(join_path("test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt"))) + os.makedirs(os.path.dirname(path_target), exist_ok=True) + shutil.copy(path_original, path_target) move_row_action: TsvActionReport = TsvMoveRowAction( source_row_index=25, new_row_index=24 ) result: TsvActionReport = isa_table_updater.apply_actions( - file_path=path, + file_path=path_target, file_sha256_hash=sha, actions=[move_row_action], ) @@ -191,15 +192,15 @@ def test_move_row_action_06(): def test_move_row_action_07(): sha = "add85bc3770cda13450b6fd95fe1735fe6337553076ed7cd269fc8163e002fac" isa_table_updater = TsvFileUpdater() - path_original = "tests/test-data/test-data-01/s_MTBLS66_test_01.txt" - path = ".test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt" - os.makedirs(os.path.dirname(path), exist_ok=True) - shutil.copy(path_original, path) + path_original = os.path.realpath(join_path("tests/test-data/test-data-01/s_MTBLS66_test_01.txt")) + path_target =os.path.realpath(os.path.realpath(join_path("test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt"))) + os.makedirs(os.path.dirname(path_target), exist_ok=True) + shutil.copy(path_original, path_target) move_row_action: TsvActionReport = TsvMoveRowAction( source_row_index=11, new_row_index=11 ) result: TsvActionReport = isa_table_updater.apply_actions( - file_path=path, + file_path=path_target, file_sha256_hash=sha, actions=[move_row_action], ) @@ -210,15 +211,15 @@ def test_move_row_action_07(): def test_copy_row_action_01(): sha = "add85bc3770cda13450b6fd95fe1735fe6337553076ed7cd269fc8163e002fac" isa_table_updater = TsvFileUpdater() - path_original = "tests/test-data/test-data-01/s_MTBLS66_test_01.txt" - path = ".test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt" - os.makedirs(os.path.dirname(path), exist_ok=True) - shutil.copy(path_original, path) + path_original = os.path.realpath(join_path("tests/test-data/test-data-01/s_MTBLS66_test_01.txt")) + path_target =os.path.realpath(os.path.realpath(join_path("test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt"))) + os.makedirs(os.path.dirname(path_target), exist_ok=True) + shutil.copy(path_original, path_target) copy_row_action: TsvActionReport = TsvCopyRowAction( source_row_index=7, target_row_indices=[0, 1, 2, 3, 4, 5] ) result: TsvActionReport = isa_table_updater.apply_actions( - file_path=path, + file_path=path_target, file_sha256_hash=sha, actions=[copy_row_action], ) @@ -229,10 +230,10 @@ def test_copy_row_action_01(): def test_add_column_action_01(): sha = "add85bc3770cda13450b6fd95fe1735fe6337553076ed7cd269fc8163e002fac" isa_table_updater = TsvFileUpdater() - path_original = "tests/test-data/test-data-01/s_MTBLS66_test_01.txt" - path = ".test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt" - os.makedirs(os.path.dirname(path), exist_ok=True) - shutil.copy(path_original, path) + path_original = os.path.realpath(join_path("tests/test-data/test-data-01/s_MTBLS66_test_01.txt")) + path_target =os.path.realpath(os.path.realpath(join_path("test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt"))) + os.makedirs(os.path.dirname(path_target), exist_ok=True) + shutil.copy(path_original, path_target) columns: Dict[int, TsvColumnData] = {} cell_default_values: Dict[int, str] = {} column = TsvColumnData(header_name="Protocol REF", values={}) @@ -242,7 +243,7 @@ def test_add_column_action_01(): columns=columns, cell_default_values=cell_default_values ) result: TsvActionReport = isa_table_updater.apply_actions( - file_path=path, + file_path=path_target, file_sha256_hash=sha, actions=[add_column_action], ) @@ -253,10 +254,10 @@ def test_add_column_action_01(): def test_delete_column_action_01(): sha = "add85bc3770cda13450b6fd95fe1735fe6337553076ed7cd269fc8163e002fac" isa_table_updater = TsvFileUpdater() - path_original = "tests/test-data/test-data-01/s_MTBLS66_test_01.txt" - path = ".test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt" - os.makedirs(os.path.dirname(path), exist_ok=True) - shutil.copy(path_original, path) + path_original = os.path.realpath(join_path("tests/test-data/test-data-01/s_MTBLS66_test_01.txt")) + path_target =os.path.realpath(os.path.realpath(join_path("test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt"))) + os.makedirs(os.path.dirname(path_target), exist_ok=True) + shutil.copy(path_original, path_target) columns: Dict[int, TsvColumnData] = {} cell_default_values: Dict[int, str] = {} column = TsvColumnData(header_name="Protocol REF", values={}) @@ -266,7 +267,7 @@ def test_delete_column_action_01(): current_columns={1: "Characteristics[Organism]", 7: "Characteristics[Variant]"} ) result: TsvActionReport = isa_table_updater.apply_actions( - file_path=path, + file_path=path_target, file_sha256_hash=sha, actions=[delete_column_action], ) @@ -277,10 +278,10 @@ def test_delete_column_action_01(): def test_move_column_action_01(): sha = "add85bc3770cda13450b6fd95fe1735fe6337553076ed7cd269fc8163e002fac" isa_table_updater = TsvFileUpdater() - path_original = "tests/test-data/test-data-01/s_MTBLS66_test_01.txt" - path = ".test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt" - os.makedirs(os.path.dirname(path), exist_ok=True) - shutil.copy(path_original, path) + path_original = os.path.realpath(join_path("tests/test-data/test-data-01/s_MTBLS66_test_01.txt")) + path_target =os.path.realpath(os.path.realpath(join_path("test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt"))) + os.makedirs(os.path.dirname(path_target), exist_ok=True) + shutil.copy(path_original, path_target) columns: Dict[int, TsvColumnData] = {} cell_default_values: Dict[int, str] = {} column = TsvColumnData(header_name="Protocol REF", values={}) @@ -292,7 +293,7 @@ def test_move_column_action_01(): new_column_index=6, ) result: TsvActionReport = isa_table_updater.apply_actions( - file_path=path, + file_path=path_target, file_sha256_hash=sha, actions=[move_column_action], ) @@ -303,17 +304,17 @@ def test_move_column_action_01(): def test_copy_column_action_01(): sha = "add85bc3770cda13450b6fd95fe1735fe6337553076ed7cd269fc8163e002fac" isa_table_updater = TsvFileUpdater() - path_original = "tests/test-data/test-data-01/s_MTBLS66_test_01.txt" - path = ".test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt" - os.makedirs(os.path.dirname(path), exist_ok=True) - shutil.copy(path_original, path) + path_original = os.path.realpath(join_path("tests/test-data/test-data-01/s_MTBLS66_test_01.txt")) + path_target =os.path.realpath(os.path.realpath(join_path("test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt"))) + os.makedirs(os.path.dirname(path_target), exist_ok=True) + shutil.copy(path_original, path_target) copy_column_action: TsvActionReport = TsvCopyColumnAction( source_column_index=1, source_column_header="Characteristics[Organism]", target_columns={7: "Characteristics[Variant]", 8: "Term Source REF"}, ) result: TsvActionReport = isa_table_updater.apply_actions( - file_path=path, + file_path=path_target, file_sha256_hash=sha, actions=[copy_column_action], ) @@ -324,15 +325,15 @@ def test_copy_column_action_01(): def test_update_column_header_action_01(): sha = "add85bc3770cda13450b6fd95fe1735fe6337553076ed7cd269fc8163e002fac" isa_table_updater = TsvFileUpdater() - path_original = "tests/test-data/test-data-01/s_MTBLS66_test_01.txt" - path = ".test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt" - os.makedirs(os.path.dirname(path), exist_ok=True) - shutil.copy(path_original, path) + path_original = os.path.realpath(join_path("tests/test-data/test-data-01/s_MTBLS66_test_01.txt")) + path_target =os.path.realpath(os.path.realpath(join_path("test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt"))) + os.makedirs(os.path.dirname(path_target), exist_ok=True) + shutil.copy(path_original, path_target) update_column_header_action: TsvActionReport = TsvUpdateColumnHeaderAction( new_headers={7: "Characteristics[Variant].2", 8: "Term Source REF.2"}, ) result: TsvActionReport = isa_table_updater.apply_actions( - file_path=path, + file_path=path_target, file_sha256_hash=sha, actions=[update_column_header_action], ) @@ -343,10 +344,10 @@ def test_update_column_header_action_01(): def test_update_column_action_01(): sha = "add85bc3770cda13450b6fd95fe1735fe6337553076ed7cd269fc8163e002fac" isa_table_updater = TsvFileUpdater() - path_original = "tests/test-data/test-data-01/s_MTBLS66_test_01.txt" - path = ".test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt" - os.makedirs(os.path.dirname(path), exist_ok=True) - shutil.copy(path_original, path) + path_original = os.path.realpath(join_path("tests/test-data/test-data-01/s_MTBLS66_test_01.txt")) + path_target =os.path.realpath(os.path.realpath(join_path("test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt"))) + os.makedirs(os.path.dirname(path_target), exist_ok=True) + shutil.copy(path_original, path_target) columns: Dict[int, TsvColumnData] = {} column = TsvColumnData( header_name="Characteristics[Organism]", values={2: "test 2", 5: "test 5"} @@ -359,7 +360,7 @@ def test_update_column_action_01(): columns[4] = column update_columns_action: TsvActionReport = TsvUpdateColumnsAction(columns=columns) result: TsvActionReport = isa_table_updater.apply_actions( - file_path=path, + file_path=path_target, file_sha256_hash=sha, actions=[update_columns_action], ) @@ -370,22 +371,19 @@ def test_update_column_action_01(): def test_update_cells_action_01(): sha = "add85bc3770cda13450b6fd95fe1735fe6337553076ed7cd269fc8163e002fac" isa_table_updater = TsvFileUpdater() - path_original = "tests/test-data/test-data-01/s_MTBLS66_test_01.txt" - path = ".test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt" - os.makedirs(os.path.dirname(path), exist_ok=True) - shutil.copy(path_original, path) + path_original = os.path.realpath(join_path("tests/test-data/test-data-01/s_MTBLS66_test_01.txt")) + path_target =os.path.realpath(join_path("test-temp/test-data/test-data-01/s_MTBLS66_test_01_result.txt")) + os.makedirs(os.path.dirname(path_target), exist_ok=True) + shutil.copy(path_original, path_target) cell1 = TsvCellData(row_index=1, column_index=2, value="Cell Update") cell2 = TsvCellData(row_index=2, column_index=1, value="Cell Update2") cell3 = TsvCellData(row_index=3, column_index=10, value="Cell Update3") cells: List[TsvCellData] = [cell1, cell2, cell3] update_cells_action: TsvActionReport = TsvUpdateCellsAction(cells=cells) result: TsvActionReport = isa_table_updater.apply_actions( - file_path=path, + file_path=path_target, file_sha256_hash=sha, actions=[update_cells_action], ) assert result.success assert result.updated_file_sha256_hash - - -test_add_row_action_01() diff --git a/tests/metabolights_utils/parse_from_fs/test_parse_investigation_from_fs.py b/tests/metabolights_utils/parse_from_fs/test_parse_investigation_from_fs.py index d57c0cc..c983728 100644 --- a/tests/metabolights_utils/parse_from_fs/test_parse_investigation_from_fs.py +++ b/tests/metabolights_utils/parse_from_fs/test_parse_investigation_from_fs.py @@ -55,7 +55,7 @@ def test_investigation_file_read_write_success_1(): sha25_hash = result.sha256_hash tmp_file_name = uuid.uuid4().hex - tmp_path = pathlib.Path(f"/tmp/test_{tmp_file_name}.txt") + tmp_path = pathlib.Path(f"test-temp/test_{tmp_file_name}.txt") try: writer: InvestigationFileWriter = Writer.get_investigation_file_writer() writer.write(result.investigation, file_buffer_or_path=tmp_path) diff --git a/tests/metabolights_utils/provider/test_ftp_client.py b/tests/metabolights_utils/provider/test_ftp_client.py index a280972..f492f54 100644 --- a/tests/metabolights_utils/provider/test_ftp_client.py +++ b/tests/metabolights_utils/provider/test_ftp_client.py @@ -171,7 +171,7 @@ def test_load_study_02(mocker: MockerFixture, study_id: str): mock_ftp = MagicMock() mocker.patch("ftplib.FTP", return_value=mock_ftp) - local_path = ".test-temp/ftp_client/test02" + local_path = "test-temp/ftp_client/test02" model = None try: os.makedirs(local_path, exist_ok=True) @@ -227,7 +227,7 @@ def retrbinary(self, command, file_writer): file_path = os.path.join(self.current_path, file_name) if file_path and os.path.exists(file_path): - with open(file_path) as f: + with open(file_path, encoding="utf-8") as f: lines = f.readlines() for line in lines: file_writer(bytes(line, encoding="utf-8")) @@ -288,7 +288,7 @@ def test_download_study_data_files_01(mocker: MockerFixture): mock_ftp.cwd = mock_writer.cwd mocker.patch("ftplib.FTP", return_value=mock_ftp) - local_path = ".test-temp/ftp_client/download_data_files_01" + local_path = "test-temp/ftp_client/download_data_files_01" try: shutil.rmtree(local_path, ignore_errors=True) os.makedirs(local_path, exist_ok=True) @@ -324,7 +324,7 @@ def test_download_study_metadata_files_01(mocker: MockerFixture): mock_ftp.cwd = mock_writer.cwd mocker.patch("ftplib.FTP", return_value=mock_ftp) - local_path = ".test-temp/ftp_client/download_metadata_files_01" + local_path = "test-temp/ftp_client/download_metadata_files_01" try: shutil.rmtree(local_path, ignore_errors=True) os.makedirs(local_path, exist_ok=True) @@ -360,7 +360,7 @@ def test_download_study_metadata_files_02(mocker: MockerFixture): mock_ftp.cwd = mock_writer.cwd mocker.patch("ftplib.FTP", return_value=mock_ftp) - local_path = ".test-temp/ftp_client/download_metadata_files_01" + local_path = "test-temp/ftp_client/download_metadata_files_01" try: shutil.rmtree(local_path, ignore_errors=True) os.makedirs(local_path, exist_ok=True) @@ -468,7 +468,7 @@ def test_get_study_folder_content_01(mocker: MockerFixture): mock_ftp.cwd = mock_writer.cwd mocker.patch("ftplib.FTP", return_value=mock_ftp) - local_path = ".test-temp/ftp_client/test_get_study_folder_content_01" + local_path = "test-temp/ftp_client/test_get_study_folder_content_01" try: shutil.rmtree(local_path, ignore_errors=True) os.makedirs(local_path, exist_ok=True) @@ -494,7 +494,7 @@ def mock_connect(): assert result.files assert os.path.exists(folder_index_file_path) assert "FILES/test.raw" in result.files - with open(folder_index_file_path) as f: + with open(folder_index_file_path, encoding="utf-8") as f: data = json.load(f) index = FolderIndex.model_validate(data) assert "FILES/test.raw" in index.content.files diff --git a/tests/metabolights_utils/provider/test_ftp_folder_metadata.py b/tests/metabolights_utils/provider/test_ftp_folder_metadata.py index 0727804..a4b6760 100644 --- a/tests/metabolights_utils/provider/test_ftp_folder_metadata.py +++ b/tests/metabolights_utils/provider/test_ftp_folder_metadata.py @@ -23,7 +23,7 @@ def folder_index_file_path() -> ( Generator[Tuple[MetabolightsSubmissionRepository, Dict[str, Any]], None, None] ): folder_index_file_path = ( - f"/tmp/mtbls_folder_index_file_path_{random.randint(1000000, 9999999)}_tmp" + f"test-temp/mtbls_folder_index_file_path_{random.randint(1000000, 9999999)}_tmp" ) try: os.makedirs(os.path.dirname(folder_index_file_path), exist_ok=True) @@ -38,7 +38,7 @@ def folder_index_file_path() -> ( @pytest.mark.parametrize("study_id", valid_study_ids) def test_get_folder_metadata_01(folder_index_file_path: str, study_id: str): mock_client = MagicMock() - mock_client.remote_repository_root_directory.return_value = "/tmp/path" + mock_client.remote_repository_root_directory.return_value = "test-temp/path" folder_content = FtpFolderContent() folder_content.descriptors.append( FtpFileDescriptor(base_name="i_Investigation.txt", size_in_bytes=10, mode="755") @@ -51,6 +51,8 @@ def test_get_folder_metadata_01(folder_index_file_path: str, study_id: str): ) mock_client.list_directory.side_effect = [folder_content, FtpFolderContent()] + + mock_client.remote_repository_root_directory = "test-repo" collector = FtpFolderMetadataCollector( client=mock_client, study_id=study_id, @@ -66,14 +68,14 @@ def test_get_folder_metadata_01(folder_index_file_path: str, study_id: str): assert len(metadata.files) == 2 # reuse same file - with open(folder_index_file_path, "w") as f: + with open(folder_index_file_path, "w", encoding="utf-8") as f: f.write("invalid data") metadata, messages = collector.get_folder_metadata(None, False, False) assert len(metadata.files) == 0 assert len(messages) > 0 # reuse same file - with open(folder_index_file_path, "w") as f: + with open(folder_index_file_path, "w", encoding="utf-8") as f: f.write("{}") metadata, messages = collector.get_folder_metadata(None, False, False) assert len(metadata.files) == 0 diff --git a/tests/metabolights_utils/provider/test_submission_provider.py b/tests/metabolights_utils/provider/test_submission_provider.py index 224cb47..b4084c5 100644 --- a/tests/metabolights_utils/provider/test_submission_provider.py +++ b/tests/metabolights_utils/provider/test_submission_provider.py @@ -34,13 +34,13 @@ class MockHttpResponse(BaseModel): def test_create_assay_01(mocker: MockerFixture): credentials_file_path = ( - f"/tmp/mtbls_unit_test_login{random.randint(1000000, 9999999)}_tmp" + f"test-temp/mtbls_unit_test_login{random.randint(1000000, 9999999)}_tmp" ) local_storage_root_path = ( - f"/tmp/mtbls_unit_test_local{random.randint(1000000, 9999999)}_tmp" + f"test-temp/mtbls_unit_test_local{random.randint(1000000, 9999999)}_tmp" ) local_storage_cache_path = ( - f"/tmp/mtbls_unit_test_local_cache_{random.randint(1000000, 9999999)}_tmp" + f"test-temp/mtbls_unit_test_local_cache_{random.randint(1000000, 9999999)}_tmp" ) try: test_data = { @@ -49,7 +49,7 @@ def test_create_assay_01(mocker: MockerFixture): "https://www.ebi.ac.uk/metabolights/ws": {"api_token": "z"} }, } - with open(credentials_file_path, "w") as f: + with open(credentials_file_path, "w", encoding="utf-8") as f: json.dump(test_data, f) provider = MetabolightsSubmissionRepository( @@ -139,12 +139,12 @@ def submission_repository() -> ( Generator[Tuple[MetabolightsSubmissionRepository, Dict[str, Any]], None, None] ): credentials_file_path = ( - f"/tmp/mtbls_unit_test_load_study_{random.randint(1000000, 9999999)}_tmp" + f"test-temp/mtbls_unit_test_load_study_{random.randint(1000000, 9999999)}_tmp" ) local_storage_root_path = ( - f"/tmp/mtbls_unit_test_load_study_{random.randint(1000000, 9999999)}_tmp" + f"test-temp/mtbls_unit_test_load_study_{random.randint(1000000, 9999999)}_tmp" ) - local_storage_cache_path = f"/tmp/mtbls_unit_test_local_cache_load_study_{random.randint(1000000, 9999999)}_tmp" + local_storage_cache_path = f"test-temp/mtbls_unit_test_local_cache_load_study_{random.randint(1000000, 9999999)}_tmp" try: credentials = { "ftp_login": {"ftp-private.ebi.ac.uk": {"user_name": "x", "password": "y"}}, @@ -152,7 +152,7 @@ def submission_repository() -> ( "https://www.ebi.ac.uk/metabolights/ws": {"api_token": "z"} }, } - with open(credentials_file_path, "w") as f: + with open(credentials_file_path, "w", encoding="utf-8") as f: json.dump(credentials, f) repository = MetabolightsSubmissionRepository( @@ -422,7 +422,7 @@ def list_response(**argv): override_remote_files=False, ) assert success - assert not message + assert "Uploaded Files:" in message @pytest.mark.parametrize("study_id", ["MTBLS1"]) @@ -445,7 +445,7 @@ class HttpxResponse(BaseModel): return_value=HttpxResponse( status_code=201, text=open( - "tests/test-data/rest-api-test-data/validation_task_started_response.json" + "tests/test-data/rest-api-test-data/validation_task_started_response.json", encoding="utf-8" ).read(), ), ) @@ -455,13 +455,13 @@ class HttpxResponse(BaseModel): HttpxResponse( status_code=200, text=open( - "tests/test-data/rest-api-test-data/validation_task_success_response.json" + "tests/test-data/rest-api-test-data/validation_task_success_response.json", encoding="utf-8" ).read(), ), HttpxResponse( status_code=200, text=open( - "tests/test-data/rest-api-test-data/validation_report.json" + "tests/test-data/rest-api-test-data/validation_report.json", encoding="utf-8" ).read(), ), ], @@ -470,7 +470,7 @@ class HttpxResponse(BaseModel): repository, credentials = submission_repository validation_file_path = ( - f"/tmp/mtbls_unit_test_validation_{random.randint(1000000, 9999999)}_tmp.json" + f"test-temp/mtbls_unit_test_validation_{random.randint(1000000, 9999999)}_tmp.json" ) try: success, message = repository.validate_study( @@ -501,7 +501,7 @@ class HttpxResponse(BaseModel): get_response = HttpxResponse( status_code=201, text=open( - "tests/test-data/rest-api-test-data/ftp_sync_task_started_response.json" + "tests/test-data/rest-api-test-data/ftp_sync_task_started_response.json", encoding="utf-8" ).read(), ) mocker.patch( @@ -514,7 +514,7 @@ class HttpxResponse(BaseModel): return_value=HttpxResponse( status_code=200, text=open( - "tests/test-data/rest-api-test-data/ftp_sync_task_success_response.json" + "tests/test-data/rest-api-test-data/ftp_sync_task_success_response.json", encoding="utf-8" ).read(), ), ) @@ -593,7 +593,7 @@ def list_isa_metadata_files(*arg, **argv): os.makedirs(study_path, exist_ok=True) for file in list_isa_metadata_files()[0].study: file_path = os.path.join(study_path, file.file) - with open(file_path, "w") as fw: + with open(file_path, "w", encoding="utf-8") as fw: fw.write("test\n") modified = datetime.datetime.strptime( file.created_at, diff --git a/tests/metabolights_utils/provider/test_utils.py b/tests/metabolights_utils/provider/test_utils.py index effe3f7..ae6487f 100644 --- a/tests/metabolights_utils/provider/test_utils.py +++ b/tests/metabolights_utils/provider/test_utils.py @@ -251,8 +251,8 @@ def __exit__(self, exception_type, exception_value, exception_traceback): def test_download_file_from_rest_api_01(mocker: MockerFixture): file_path = "tests/test-data/rest-api-test-data/s_MTBLS1.txt.zip" - shutil.rmtree("/tmp/donwload_test", ignore_errors=True) - dir_path = "/tmp/donwload_test" + shutil.rmtree("test-temp/donwload_test", ignore_errors=True) + dir_path = "test-temp/donwload_test" local_file_path = ( f"{dir_path}/download_test_{random.randint(1000000, 9999999)}_tmp.zip" ) diff --git a/tests/utils/test_audit_utils.py b/tests/utils/test_audit_utils.py index 2f09a5d..1d86d01 100644 --- a/tests/utils/test_audit_utils.py +++ b/tests/utils/test_audit_utils.py @@ -7,7 +7,7 @@ def test_create_audit_folder_01(): - tmp_path = f"/tmp/test_{uuid.uuid4().hex}" + tmp_path = f"test-temp/test_{uuid.uuid4().hex}" try: audit_path = MetabolightsAuditUtils.create_audit_folder( src_root_path="tests/test-data/MTBLS1", target_root_path=tmp_path @@ -25,7 +25,7 @@ def test_create_audit_folder_01(): def test_create_audit_folder_non_exist_target_01(): - tmp_path = f"/tmp/test_{uuid.uuid4().hex}" + tmp_path = f"test-temp/test_{uuid.uuid4().hex}" try: audit_path = MetabolightsAuditUtils.create_audit_folder( src_root_path="tests/test-data/MTBLS1x", target_root_path=tmp_path @@ -38,7 +38,7 @@ def test_create_audit_folder_non_exist_target_01(): def test_copy_isa_metadata_files_01(): - tmp_path = f"/tmp/test_{uuid.uuid4().hex}" + tmp_path = f"test-temp/test_{uuid.uuid4().hex}" try: target_path = MetabolightsAuditUtils.copy_isa_metadata_files( src_folder_path="tests/test-data/MTBLS1", target_folder_path=tmp_path @@ -52,7 +52,7 @@ def test_copy_isa_metadata_files_01(): def test_copy_isa_metadata_files_02(): - tmp_path = f"/tmp/test_{uuid.uuid4().hex}" + tmp_path = f"test-temp/test_{uuid.uuid4().hex}" try: target_path = MetabolightsAuditUtils.copy_isa_metadata_files( src_folder_path="tests/test-data/MTBLS1x", target_folder_path=tmp_path diff --git a/tests/utils/test_hash_utils.py b/tests/utils/test_hash_utils.py index 5e79181..3453438 100644 --- a/tests/utils/test_hash_utils.py +++ b/tests/utils/test_hash_utils.py @@ -22,7 +22,7 @@ def test_get_sha256sum_01(): def test_get_sha256sum_02(): expected = "fa2f11ad5b556c57d8b8c03a6e8efe3e3b0ee62a7c0a0b725e640d1ce090d47f" hash_val = MetabolightsHashUtils.sha256sum( - "tests/test-data/MTBLS1/i_Investigation.txt" + "tests/test-data/MTBLS1/i_Investigation.txt", convert_to_linux_line_ending=True ) assert hash_val == expected @@ -53,7 +53,7 @@ def test_get_isa_metadata_folder_hash_03(): """ additional isa medata files cause different hash """ - tmp_path = f"/tmp/test_{uuid.uuid4().hex}" + tmp_path = f"test-temp/test_{uuid.uuid4().hex}" try: source_path = "tests/test-data/MTBLS1" target_path = MetabolightsAuditUtils.copy_isa_metadata_files( @@ -81,7 +81,7 @@ def test_get_isa_metadata_folder_hash_04(): """ renamed files cause different hash """ - tmp_path = f"/tmp/test_{uuid.uuid4().hex}" + tmp_path = f"test-temp/test_{uuid.uuid4().hex}" try: source_path = "tests/test-data/MTBLS1" target_path = MetabolightsAuditUtils.copy_isa_metadata_files(