refactor: upload submission refactor and create study utility method

metabolights_utils/commands/submission/submission_download.py

@@ -65,7 +65,7 @@ def submission_download(
 
     study_id: MetaboLights study accession number (MTBLSxxxx).
 
-    files (optional): files will be donwloaded. If not specified, downloads all ISA metadata files.
+    files (optional): files will be downloaded. If not specified, downloads all ISA metadata files.
     """
     study_id = study_id.upper().strip()
     client = MetabolightsSubmissionRepository(
@@ -83,7 +83,3 @@ def submission_download(
     click.echo(f"Download submission study {study_id} on {study_path} status:")
     for item in result.actions:
         click.echo(f"  {result.actions[item]}\t{item}")
-
-
-if __name__ == "__main__":
-    submission_download(["MTBLS5397"])

metabolights_utils/commands/submission/submission_upload.py

@@ -1,4 +1,4 @@
-from typing import Union
+from typing import List, Union
 
 import click
 
@@ -40,24 +40,30 @@
     default=definitions.default_local_submission_credentials_file_path,
     help="Path to store cache files of study submission file indices, study models, etc.",
 )
+@click.option(
+    "--user_api_token",
+    "-a",
+    help="MetaboLights user API token.",
+)
 @click.argument("study_id")
 @click.argument("metadata_files", required=False)
 def submission_upload(
     study_id: str = "",
-    metadata_files: str = "",
+    metadata_files: Union[List[str], None] = None,
     use_only_local: bool = False,
     local_path: Union[None, str] = None,
     local_cache_path: Union[None, str] = None,
     rest_api_base_url: Union[None, str] = None,
     override_remote_files: bool = False,
     credentials_file_path: str = "",
+    user_api_token: Union[str, None] = None,
 ):
     """
-    Uploads local metadata files to private FTP and start sync task to update study folder.
+    Uploads local files to private FTP and start sync task to update study folder.
 
-    study_id: MetaboLights study accession number (MTBLSxxxx).
+    study_id: MetaboLights study accession number / submission id (MTBLSxxxx or REQXXXX).
 
-    files (optional): files will be donwloaded. If not specified, downloads all ISA metadata files.
+    files (optional): files will be downloaded. If not specified, downloads all ISA metadata files.
     """
     study_id = study_id.upper().strip()
     client = MetabolightsSubmissionRepository(
@@ -66,26 +72,66 @@ def submission_upload(
         rest_api_base_url=rest_api_base_url,
         credentials_file_path=credentials_file_path,
     )
+    ftp_details, error = client.get_ftp_upload_details(
+        study_id=study_id,
+        user_api_token=user_api_token,
+        rest_api_base_url=rest_api_base_url,
+    )
+    if not ftp_details:
+        click.echo(f"Upload private ftp details for {study_id}: Failed {str(error)}")
+        exit(1)
     success, message = client.upload_metadata_files(
         study_id=study_id,
         override_remote_files=override_remote_files,
-        metadata_files=None,
+        metadata_files=metadata_files,
+        ftp_server_url=ftp_details.ftp_host,
+        ftp_username=ftp_details.ftp_user,
+        ftp_password=ftp_details.ftp_password,
+        user_api_token=user_api_token,
     )
     if success:
         click.echo(f"Upload private study {study_id} metadata files: Success")
         click.echo(message)
-
+    else:
+        click.echo(f"Upload private study {study_id} metadata files: Failed {message}")
+        exit(1)
+    success, message = client.upload_data_files(
+        study_id=study_id,
+        local_path=local_path,
+        remote_folder_directory=ftp_details.ftp_folder,
+        ftp_server_url=ftp_details.ftp_host,
+        ftp_username=ftp_details.ftp_user,
+        ftp_password=ftp_details.ftp_password,
+    )
+    if success:
+        click.echo(f"Upload private study {study_id} data files: Success")
+        click.echo(message)
+        click.echo(f"Sync private FTP metadata files for {study_id}.")
         success, error = client.sync_private_ftp_metadata_files(
-            study_id=study_id, pool_period=10, retry=10
+            study_id=study_id, pool_period=10, retry=10, user_api_token=user_api_token
         )
         if success:
-            click.echo(f"Sync private study {study_id} folder: Success")
+            click.echo(f"Success: Sync private FTP metadata files for {study_id}.")
+            click.echo(f"Sync private FTP data files for {study_id}.")
+            success, error = client.sync_private_ftp_data_files(
+                study_id=study_id,
+                pool_period=10,
+                retry=10,
+                user_api_token=user_api_token,
+            )
+            if success:
+                click.echo(
+                    f"Sync private study date files for {study_id} folder: Success"
+                )
+            else:
+                click.echo(
+                    f"Failure: Sync private study metadata files for {study_id}: {error}"
+                )
+
         else:
-            click.echo(f"Failure: Sync private study {study_id} folder: {error}")
+            click.echo(
+                f"Failure: Sync private study metadata files for {study_id}. {error}"
+            )
 
     else:
         click.echo(f"Upload private study {study_id}: {message}")
-
-
-if __name__ == "__main__":
-    submission_upload(["MTBLS5397"])

metabolights_utils/provider/ftp/default_ftp_client.py

@@ -23,7 +23,9 @@ def __init__(
         password: Union[str, None] = None,
     ) -> None:
         self.ftp_server_url = ftp_server_url
-        self.remote_repository_root_directory = remote_repository_root_directory.replace("\\","").rstrip("/")
+        self.remote_repository_root_directory = (
+            remote_repository_root_directory.replace("\\", "").rstrip("/")
+        )
         self.local_storage_root_path = local_storage_root_path
         self.username = username if username else ""
         self.password = password if password else ""
@@ -75,7 +77,7 @@ def list_directory(
         directory: Union[str, None] = None,
         search_pattern: Union[str, None] = None,
     ) -> FtpFolderContent:
-        directory = directory.replace("\\", '/').strip("/") if directory else ""
+        directory = directory.replace("\\", "/").strip("/") if directory else ""
         root_dir = self.remote_repository_root_directory
         remote_directory = f"{root_dir}/{directory}"
 
@@ -118,8 +120,11 @@ def upload_files(
 
                 with open(file_path, "rb") as file:
                     self.ftp.storbinary(f"STOR {filename}", file)
+            return True, ""
         except Exception as ex:
-            logger.error("FTP directory %s upload error: %s", remote_directory, str(ex))
+            message = f"FTP directory {remote_directory} upload error: {str(ex)}"
+            logger.error(message)
+            return False, message
         finally:
             self.quit()
 
@@ -161,7 +166,9 @@ def download_file(
         remote_directory = f"{remote_root_dir}/{relative_file_path}".replace("\\", "/")
         relative_parent_path = os.path.dirname(relative_file_path)
 
-        remote_parent_directory = os.path.dirname(os.path.join(remote_root_dir, relative_file_path))
+        remote_parent_directory = os.path.dirname(
+            os.path.join(remote_root_dir, relative_file_path)
+        )
         remote_parent_directory = remote_parent_directory.replace("\\", "/")
         logger.debug("List files within %s on FTP server ", relative_parent_path)
         result = self.list_directory(relative_parent_path, search_pattern=filename)
@@ -219,9 +226,11 @@ def download_file(
                         for item in local_file_list:
                             if item not in listed_folders and item not in listed_files:
                                 item_path: str = os.path.join(target_path, item)
-                                relative_item_path = item_path.replace(
-                                    f"{local_path}", ""
-                                ).strip("/").strip("\\")
+                                relative_item_path = (
+                                    item_path.replace(f"{local_path}", "")
+                                    .strip("/")
+                                    .strip("\\")
+                                )
                                 if keep_local_files and item in keep_local_files:
                                     actions[relative_item_path] = "SKIPPED"
                                     logger.debug(
@@ -243,7 +252,9 @@ def download_file(
                 response.local_folders.append(relative_file_path)
                 for collection in (result.folders, result.files):
                     for entry in collection:
-                        new_relative_file_path = join_path(relative_file_path, entry).replace("\\", "/")
+                        new_relative_file_path = join_path(
+                            relative_file_path, entry
+                        ).replace("\\", "/")
                         self.download_file(
                             relative_file_path=new_relative_file_path,
                             local_path=local_path,

metabolights_utils/provider/submission_model.py

@@ -199,3 +199,11 @@ class PolicyResultResponse(WorkerTaskStatus):
             description="This field contains task id of the task.",
         ),
     ] = PolicySummaryResult()
+
+
+class FtpUploadDetails(BaseModel):
+    study_id: str
+    ftp_folder: str
+    ftp_host: str
+    ftp_user: str
+    ftp_password: str