DDS-1681: Retry database insert if failed files after upload

SPRINTLOG.md

@@ -302,3 +302,7 @@ _Nothing merged in CLI during this sprint_
 
 - GitHub Actions to generate the documentation fixed ([#1473])(https://scilifelab.atlassian.net/jira/software/projects/DDS/boards/13?selectedIssue=DDS-1473)
 - Print project information and ask user for confirmation when deleting or archiving projects ([#1401])(https://scilifelab.atlassian.net/jira/software/projects/DDS/boards/13?selectedIssue=DDS-1401)
+
+# 2023-10-02 - 2023-10-13
+
+- Make a single last request to update the database in the case of failed files during upload ([#656](https://github.com/ScilifelabDataCentre/dds_cli/pull/656)

dds_cli/__init__.py

@@ -94,6 +94,7 @@ class DDSEndpoint:
     FILE_INFO = BASE_ENDPOINT + "/file/info"
     FILE_INFO_ALL = BASE_ENDPOINT + "/file/all/info"
     FILE_UPDATE = BASE_ENDPOINT + "/file/update"
+    FILE_ADD_FAILED = BASE_ENDPOINT + "/file/failed/add"
 
     # Project specific urls
     PROJ_ACCESS = BASE_ENDPOINT + "/proj/access"

dds_cli/data_putter.py

@@ -9,6 +9,7 @@
 import itertools
 import logging
 import pathlib
+import json
 
 # Installed
 import boto3
@@ -169,6 +170,85 @@ def put(
                         for x in [y.id for y in progress.tasks if y.fields.get("step") != "put"]
                     ]
 
+        # Make a single database update for files that have failed
+        # Json file for failed files should only be created if there has been an error
+        if putter.failed_delivery_log.is_file():
+            LOG.warning(
+                "Some uploaded files could not be added to the database %s.\n",
+                str(putter.failed_delivery_log),
+            )
+            LOG.warning("Attempting to add the file to the database.")
+
+            try:
+                with putter.failed_delivery_log.open(mode="r", encoding="utf-8") as json_f:
+                    failed = json.load(json_f)
+            except Exception as e:
+                LOG.error(f"Failed to load failed delivery log: {e}")
+                failed = {}
+            # remove from log any files that  failed for other reasons
+            for file, values in failed.items():
+                if values.get("status", {}).get("failed_op") != "add_file_db":
+                    failed.pop(file)
+
+            # send failed to API endpoint
+            params = {"project": putter.project}
+            try:
+                response, _ = dds_cli.utils.perform_request(
+                    DDSEndpoint.FILE_ADD_FAILED,
+                    method="put",
+                    headers=putter.token,
+                    params=params,
+                    json=failed,
+                    error_message="Failed to add missing files",
+                )
+            except (
+                dds_cli.exceptions.ApiRequestError,
+                dds_cli.exceptions.ApiResponseError,
+                dds_cli.exceptions.DDSCLIException,
+            ) as err:
+                LOG.warning(str(err))
+            # if not response:
+            #     raise dds_cli.exceptions.ApiResponseError(
+            #         message="Failed to add missing files to database."
+            #     )
+            files_added = response.get("files_added")
+            message = response.get("message")
+
+            # get a list of files from the log
+            files = list(failed.keys())
+
+            if message:
+                # adding the files failed
+                # check why
+                # if the files already exist in the database, then everything is fine and we need to clean status
+                for file, error in message.items():
+                    if error["error"] == "File already in database":
+                        print(f"Error for {file}: {error['error']}")
+                # LOG.warning("Message from API: %s", message)
+            if files_added:
+                missing_files = [file for file in files if file not in files_added]
+                if missing_files:
+                    print("The following files are missing:", missing_files)
+                    # raise exceptions.NoDataError("Some files could not be added to the database.")
+                else:
+                    LOG.warning("All files were added to the database.")
+
+            LOG.warning("files: %s", files)
+            LOG.warning("files added: %s", files_added)
+
+            for file in files:
+                putter.status[file] = {
+                    "cancel": False,
+                    "started": False,
+                    "message": "",
+                    "failed_op": None,
+                    "put": {"started": False, "done": False},
+                    "add_file_db": {"started": False, "done": False},
+                }
+            dds_cli.utils.console.print(
+                "\nAll the uploaded files successfuly added to the database."
+            )
+
 
 ###############################################################################
 # CLASSES ########################################################### CLASSES #
@@ -250,7 +330,6 @@ def __init__(
 
             # Remove spinner
             progress.remove_task(wait_task)
-
         if not self.filehandler.data:
             if self.temporary_directory and self.temporary_directory.is_dir():
                 LOG.debug("Deleting temporary folder %s.", self.temporary_directory)
@@ -408,22 +487,23 @@ def add_file_db(self, file):
 
         # Send file info to API - post if new file, put if overwrite
         request_method = "put" if fileinfo["overwrite"] else "post"
-        try:
-            response_json, _ = dds_cli.utils.perform_request(
-                DDSEndpoint.FILE_NEW,
-                method=request_method,
-                params=params,
-                json=file_info,
-                headers=self.token,
-                error_message=f"Failed to add file '{file}' to database",
-            )
-            added_to_db, message = (True, response_json)
-        except (
-            dds_cli.exceptions.ApiRequestError,
-            dds_cli.exceptions.ApiResponseError,
-            dds_cli.exceptions.DDSCLIException,
-        ) as err:
-            message = str(err)
-            LOG.warning(message)
+        # try:
+        #     response_json, _ = dds_cli.utils.perform_request(
+        #         DDSEndpoint.FILE_NEW,
+        #         method=request_method,
+        #         params=params,
+        #         json=file_info,
+        #         headers=self.token,
+        #         error_message=f"Failed to add file '{file}' to database",
+        #     )
+        #     added_to_db, message = (True, response_json)
+        # except (
+        #     dds_cli.exceptions.ApiRequestError,
+        #     dds_cli.exceptions.ApiResponseError,
+        #     dds_cli.exceptions.DDSCLIException,
+        # ) as err:
+        #     message = str(err)
+        #     LOG.warning(message)
+        message = "test"
 
         return added_to_db, message