fix: windows path and new line issues

.gitignore

@@ -1,6 +1,6 @@
 site/
 .sandbox
-.test-temp
+test-temp
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

README.md

@@ -435,7 +435,7 @@ def test_investigation_file_write_01():
     )
 
     tmp_file_name = uuid.uuid4().hex
-    tmp_path = pathlib.Path(f"/tmp/test_{tmp_file_name}.txt")
+    tmp_path = pathlib.Path(f"test-temp/test_{tmp_file_name}.txt")
     writer: InvestigationFileWriter = Writer.get_investigation_file_writer()
     writer.write(result.investigation, file_buffer_or_path=tmp_path)
 
@@ -564,7 +564,7 @@ def test_assay_file_read_write():
         "tests/test-data/MTBLS1/a_MTBLS1_metabolite_profiling_NMR_spectroscopy.txt"
     )
     file_path = (
-        ".test-temp/test-data/MTBLS1/a_MTBLS1_metabolite_profiling_NMR_spectroscopy.txt"
+        "test-temp/test-data/MTBLS1/a_MTBLS1_metabolite_profiling_NMR_spectroscopy.txt"
     )
     os.makedirs(os.path.dirname(file_path), exist_ok=True)
     shutil.copy(path_original, file_path)

metabolights_utils/commands/public/public_remove.py

@@ -51,4 +51,4 @@ def public_remove(
 
 
 if __name__ == "__main__":
-    public_remove()
+    public_remove(["MTBLS1"])

metabolights_utils/commands/submission/submission_create_assay.py

@@ -82,7 +82,10 @@ def submission_create_assay(
     study_id: MetaboLights study accession number (MTBLSxxxx).
     assay_technique: Valid assay techniques:  NMR, LC-MS, LC-DAD, GC-MS, GCxGC-MS, GC-FID, DI-MS, FIA-MS, CE-MS, MALDI-MS, MSImaging
 
-    Acronyms: Diode array detection (LC-DAD), Tandem MS (GCxGC-MS), Flame ionisation detector (GC-FID), Direct infusion (DI-MS), Flow injection analysis (FIA-MS), Capillary electrophoresis (CE-MS), Matrix-assisted laser desorption-ionisation imaging mass spectrometry (MALDI-MS), Nuclear magnetic resonance (NMR), Mass spec spectrometry (MSImaging)
+    Acronyms: Diode array detection (LC-DAD), Tandem MS (GCxGC-MS), Flame ionisation detector (GC-FID), 
+    Direct infusion (DI-MS), Flow injection analysis (FIA-MS), Capillary electrophoresis (CE-MS),
+    Matrix-assisted laser desorption-ionisation imaging mass spectrometry (MALDI-MS), 
+    Nuclear magnetic resonance (NMR), Mass spec spectrometry (MSImaging)
 
 
 

metabolights_utils/commands/submission/submission_delete_assay.py

@@ -99,5 +99,5 @@ def submission_delete_assay(
 
 if __name__ == "__main__":
     submission_delete_assay(
-        ["MTBLS5397", "a_MTBLS5397_LC-MS_positive_hilic_metabolite_profiling-1.txt"]
+        ["MTBLS9776", "a_MTBLS5397_LC-MS_positive_hilic_metabolite_profiling-1.txt"]
     )

metabolights_utils/commands/submission/submission_describe.py

@@ -138,4 +138,4 @@ def submission_describe(
 
 
 if __name__ == "__main__":
-    submission_describe(["MTBLS1", "-o"])
+    submission_describe(["MTBLS397", "-o"])

metabolights_utils/commands/submission/submission_login.py

@@ -53,7 +53,7 @@ def submission_login(
     initial: Union[None, LoginCredentials] = None
     try:
         if os.path.exists(credentials_file_path):
-            with open(credentials_file_path, "r") as f:
+            with open(credentials_file_path, "r", encoding="utf-8") as f:
                 data = json.load(f)
                 credentials = LoginCredentials.model_validate(data)
                 initial = LoginCredentials.model_validate(data)
@@ -84,7 +84,7 @@ def submission_login(
 
         if api_token or ftp_user or ftp_user_password:
             json_data = credentials.model_dump()
-            with open(credentials_file_path, "w") as f:
+            with open(credentials_file_path, "w", encoding="utf-8") as f:
                 json.dump(json_data, f, indent=4)
             click.echo("User credentials are updated succesfully.")
         else:
@@ -94,7 +94,7 @@ def submission_login(
         if initial:
             os.makedirs(parent_directory, exist_ok=True)
             json_data = initial.model_dump_json(indent=4)
-            with open(credentials_file_path, "w") as f:
+            with open(credentials_file_path, "w", encoding="utf-8") as f:
                 f.write(json_data)
         exit(1)
 

metabolights_utils/commands/submission/submission_upload.py

@@ -66,13 +66,15 @@ def submission_upload(
         rest_api_base_url=rest_api_base_url,
         credentials_file_path=credentials_file_path,
     )
-    success, error = client.upload_metadata_files(
+    success, message = client.upload_metadata_files(
         study_id=study_id,
         override_remote_files=override_remote_files,
         metadata_files=None,
     )
     if success:
-        click.echo(f"Upload private study {study_id}: Success")
+        click.echo(f"Upload private study {study_id} metadata files: Success")
+        click.echo(message)
+
         success, error = client.sync_private_ftp_metadata_files(
             study_id=study_id, pool_period=10, retry=10
         )
@@ -82,8 +84,8 @@ def submission_upload(
             click.echo(f"Failure: Sync private study {study_id} folder: {error}")
 
     else:
-        click.echo(f"Failure: Upload private study {study_id}: {error}")
+        click.echo(f"Upload private study {study_id}: {message}")
 
 
 if __name__ == "__main__":
-    submission_upload(["MTBLS5397", "-o"])
+    submission_upload(["MTBLS5397"])

metabolights_utils/commands/submission/submission_validate.py

@@ -67,7 +67,7 @@ def submission_validate(
     )
 
     if success:
-        with open(validation_file_path, "r") as f:
+        with open(validation_file_path, "r", encoding="utf-8") as f:
             validation_report = f.readlines()
             for line in validation_report:
                 click.echo(line.strip())

metabolights_utils/commands/submission/utils.py

@@ -33,7 +33,7 @@ def get_submission_credentials(credentials_file_path: str):
     credentials: Union[None, LoginCredentials] = None
     try:
         if os.path.exists(credentials_file_path):
-            with open(credentials_file_path, "r") as f:
+            with open(credentials_file_path, "r", encoding="utf-8") as f:
                 data = json.load(f)
                 credentials = LoginCredentials.model_validate(data)
     except Exception as ex:

metabolights_utils/provider/ftp/default_ftp_client.py

@@ -6,6 +6,7 @@
 from typing import List, Set, Union
 
 from metabolights_utils.provider.ftp.model import FtpFolderContent, LocalDirectory
+from metabolights_utils.utils.filename_utils import join_path
 
 logger = logging.getLogger()
 
@@ -22,7 +23,7 @@ def __init__(
         password: Union[str, None] = None,
     ) -> None:
         self.ftp_server_url = ftp_server_url
-        self.remote_repository_root_directory = remote_repository_root_directory
+        self.remote_repository_root_directory = remote_repository_root_directory.replace("\\","").rstrip("/")
         self.local_storage_root_path = local_storage_root_path
         self.username = username if username else ""
         self.password = password if password else ""
@@ -74,9 +75,9 @@ def list_directory(
         directory: Union[str, None] = None,
         search_pattern: Union[str, None] = None,
     ) -> FtpFolderContent:
-        directory = directory.strip("/") if directory else ""
+        directory = directory.replace("\\", '/').strip("/") if directory else ""
         root_dir = self.remote_repository_root_directory
-        remote_directory = os.path.join(root_dir, directory)
+        remote_directory = f"{root_dir}/{directory}"
 
         self.connect()
 
@@ -133,6 +134,7 @@ def download_file(
         keep_local_files: Union[Set[str], None] = None,
     ) -> LocalDirectory:
         local_path = local_path if local_path else self.local_storage_root_path
+        local_path = join_path(local_path)
         if local_files is None:
             response = LocalDirectory(root_path=local_path)
         else:
@@ -156,10 +158,11 @@ def download_file(
         parent_path = os.path.dirname(target_path)
         filename = os.path.basename(target_path)
 
-        remote_directory = os.path.join(remote_root_dir, relative_file_path)
+        remote_directory = f"{remote_root_dir}/{relative_file_path}".replace("\\", "/")
         relative_parent_path = os.path.dirname(relative_file_path)
 
-        remote_parent_directory = os.path.dirname(remote_directory)
+        remote_parent_directory = os.path.dirname(os.path.join(remote_root_dir, relative_file_path))
+        remote_parent_directory = remote_parent_directory.replace("\\", "/")
         logger.debug("List files within %s on FTP server ", relative_parent_path)
         result = self.list_directory(relative_parent_path, search_pattern=filename)
         is_directory = True
@@ -218,7 +221,7 @@ def download_file(
                                 item_path: str = os.path.join(target_path, item)
                                 relative_item_path = item_path.replace(
                                     f"{local_path}", ""
-                                ).strip("/")
+                                ).strip("/").strip("\\")
                                 if keep_local_files and item in keep_local_files:
                                     actions[relative_item_path] = "SKIPPED"
                                     logger.debug(
@@ -240,7 +243,7 @@ def download_file(
                 response.local_folders.append(relative_file_path)
                 for collection in (result.folders, result.files):
                     for entry in collection:
-                        new_relative_file_path = os.path.join(relative_file_path, entry)
+                        new_relative_file_path = join_path(relative_file_path, entry).replace("\\", "/")
                         self.download_file(
                             relative_file_path=new_relative_file_path,
                             local_path=local_path,

metabolights_utils/provider/ftp/folder_metadata_collector.py

@@ -16,6 +16,7 @@
 from metabolights_utils.provider import definitions
 from metabolights_utils.provider.ftp.default_ftp_client import DefaultFtpClient
 from metabolights_utils.provider.study_provider import AbstractFolderMetadataCollector
+from metabolights_utils.utils.filename_utils import join_path
 
 logger = logging.getLogger()
 
@@ -43,19 +44,20 @@ def __init__(
         self.study_id = study_id.upper().strip("/")
 
         self.folder_index_file_path = (
-            folder_index_file_path
+            join_path(folder_index_file_path)
             if folder_index_file_path
-            else os.path.join(
+            else join_path(
                 definitions.default_local_repority_cache_path,
                 study_id,
                 "mtbls_index.json",
             )
         )
         logger.debug("Folder index path is %s", self.folder_index_file_path)
         self.rebuild_folder_index_file = rebuild_folder_index_file
-        self.remote_study_directory = os.path.join(
+        self.remote_study_directory = join_path(
             self.client.remote_repository_root_directory, self.study_id
         )
+        self.remote_study_directory = self.remote_study_directory.replace("\\", "/").rstrip("/")
 
     def visit_folder(
         self,
@@ -67,6 +69,7 @@ def visit_folder(
         try:
 
             prefix = f"{str(study_path).rstrip('/')}/"
+            directory = directory.replace("\\", "/")
             dir_relative_path = (
                 str(directory).replace(prefix, "") if study_path != directory else ""
             )
@@ -81,7 +84,7 @@ def visit_folder(
                 messages.append(f"{dir_relative_path} is in ignore list. SKIPPED.")
                 return
             directory_input = (
-                os.path.join(self.study_id, dir_relative_path)
+                join_path(self.study_id, dir_relative_path)
                 if dir_relative_path
                 else self.study_id
             )
@@ -125,10 +128,10 @@ def visit_folder(
                 )
             for item in selected_descriptors:
                 entry = item.base_name
-                full_path: str = os.path.join(directory, entry)
-                relative_path = os.path.join(dir_relative_path, entry)
+                full_path: str = join_path(directory, entry).replace("\\", "/")
+                relative_path = join_path(dir_relative_path, entry).replace("\\", "/")
                 base_name = os.path.basename(relative_path)
-                parent_directory = os.path.dirname(relative_path)
+                parent_directory = os.path.dirname(relative_path).replace("\\", "/")
                 in_ignore_list = False
                 for pattern in definitions.ignore_file_patterns:
                     if pattern.match(relative_path):
@@ -182,7 +185,7 @@ def get_folder_metadata(
         ):
             logger.info("%s file exists, loading...", self.folder_index_file_path)
             try:
-                with open(self.folder_index_file_path, "r") as f:
+                with open(self.folder_index_file_path, "r", encoding="utf-8") as f:
                     data = json.load(f)
                     current_file_index = FolderIndex.model_validate(
                         data, from_attributes=True
@@ -238,7 +241,7 @@ def get_folder_metadata(
 
             now = datetime.datetime.now(datetime.timezone.utc)
             file_index = FolderIndex(update_time=now, content=study_folder_metadata)
-            with open(self.folder_index_file_path, "w") as fw:
+            with open(self.folder_index_file_path, "w", encoding="utf-8") as fw:
                 fw.write(file_index.model_dump_json(indent=4))
             msg = f"{self.folder_index_file_path} file is updated."
             logger.info(msg)