fix extract shorelines bugs with metadata

src/coastseg/common.py

@@ -39,7 +39,8 @@
 # Logger setup
 logger = logging.getLogger(__name__)
 
-def create_new_config(roi_ids:list,settings:dict,roi_settings:dict)->dict:
+
+def create_new_config(roi_ids: list, settings: dict, roi_settings: dict) -> dict:
     """
     Creates a new configuration dictionary by combining the given settings and ROI settings.
 
@@ -60,20 +61,20 @@ def create_new_config(roi_ids:list,settings:dict,roi_settings:dict)->dict:
         A dictionary containing the combined settings and ROI settings, as well as the ROI IDs.
     """
     new_config = {
-        'settings': {},
-        'roi_ids': [],
-
+        "settings": {},
+        "roi_ids": [],
     }
     if isinstance(roi_ids, str):
         roi_ids = [roi_ids]
     if not all(roi_id in roi_settings.keys() for roi_id in roi_ids):
-        raise ValueError(f'roi_ids {roi_ids} not in roi_settings {roi_settings.keys()}')
+        raise ValueError(f"roi_ids {roi_ids} not in roi_settings {roi_settings.keys()}")
     new_config = {**new_config, **roi_settings}
-    new_config['roi_ids'].extend(roi_ids)
-    new_config['settings'] =settings
+    new_config["roi_ids"].extend(roi_ids)
+    new_config["settings"] = settings
     return new_config
 
-def save_new_config(path:str,roi_ids:list, destination:str)->dict:
+
+def save_new_config(path: str, roi_ids: list, destination: str) -> dict:
     """Save a new config file to a path.
 
     Args:
@@ -83,19 +84,20 @@ def save_new_config(path:str,roi_ids:list, destination:str)->dict:
     """
     with open(path) as f:
         config = json.load(f)
-        
+
     if isinstance(roi_ids, str):
         roi_ids = [roi_ids]
-    
+
     roi_settings = {}
     for roi_id in roi_ids:
         if roi_id in config.keys():
             roi_settings[roi_id] = config[roi_id]
-    
-    new_config=create_new_config(roi_ids,config['settings'],roi_settings)
+
+    new_config = create_new_config(roi_ids, config["settings"], roi_settings)
     with open(destination, "w") as f:
         json.dump(new_config, f)
 
+
 def filter_images_by_roi(roi_settings: list[dict]):
     """
     Filters images in specified locations based on their Regions of Interest (ROI).

src/coastseg/extracted_shoreline.py

@@ -292,16 +292,16 @@ def combine_satellite_data(satellite_data: dict) -> dict:
                 merged_satellite_data[key] += [value]
             else:
                 merged_satellite_data[key] += value
-        # Add satellite name entries for each date
-        if "dates" in sat_data.keys():
-            merged_satellite_data["satname"] += [satname] * len(sat_data["dates"])
+            # Add the satellite name to the satellite name list
+        merged_satellite_data["satname"] += [
+            _ for _ in np.tile(satname, len(satellite_data[satname]["dates"]))
+        ]
     # Sort dates chronologically
     if "dates" in merged_satellite_data.keys():
         idx_sorted = sorted(
             range(len(merged_satellite_data["dates"])),
             key=lambda i: merged_satellite_data["dates"][i],
         )
-
         for key in merged_satellite_data.keys():
             merged_satellite_data[key] = [
                 merged_satellite_data[key][i] for i in idx_sorted
@@ -1275,7 +1275,7 @@ def extract_shorelines_with_dask(
     logger.info(f"edit_metadata metadata: {metadata}")
 
     result_dict = {}
-    for satname in metadata:
+    for satname in metadata.keys():
         satellite_dict = process_satellite(
             satname,
             settings,
@@ -1287,7 +1287,12 @@ def extract_shorelines_with_dask(
             batch_size=10,
             **kwargs,
         )
-        result_dict.update(satellite_dict)
+
+        logger.info(f"satellite_dict : {satellite_dict}")
+        logger.info(f"before result_dict : {result_dict}")
+        result_dict[satname] = satellite_dict[satname]
+        logger.info(f"result_dict.keys() : {result_dict.keys()}")
+        logger.info(f" after result_dict : {result_dict}")
 
         # combine the extracted shorelines for each satellite
     logger.info(f"Combining extracted shorelines for each satellite : {result_dict}")
@@ -1313,21 +1318,19 @@ def get_sorted_model_outputs_directory(
     bad_folder = os.path.join(session_path, "bad")
     satellites = get_satellites_in_directory(session_path)
     for satname in satellites:
-        if os.path.exists(good_folder) and os.listdir(good_folder)!= []:
-            return good_folder
-        else:
+        # get all the model_outputs that have the satellite in the filename
+        try:
             # get all the model_outputs that have the satellite in the filename
-            try:
-                # get all the model_outputs that have the satellite in the filename
-                files = file_utilities.find_files_recursively(
-                    session_path, f".*{re.escape(satname)}.*\\.npz$", raise_error=False
-                )
-            except Exception as e:
-                logger.error(f"Error finding files for satellite {satname}: {e}")
-                continue
-            logger.info(f"{session_path} contained {satname} files: {files} ")
-            if len(files) != 0:
-                filter_model_outputs(satname, files, good_folder, bad_folder)
+            files = file_utilities.find_files_recursively(
+                session_path, f".*{re.escape(satname)}.*\\.npz$", raise_error=False
+            )
+            logger.info(f"fetched files {files} for satellite {satname}")
+        except Exception as e:
+            logger.error(f"Error finding files for satellite {satname}: {e}")
+            continue
+        logger.info(f"{session_path} contained {satname} files: {files} ")
+        if len(files) != 0:
+            filter_model_outputs(satname, files, good_folder, bad_folder)
     return good_folder
 
 

src/coastseg/filters.py

@@ -21,7 +21,11 @@ def copy_files(files: list, dest_folder: str) -> None:
     Returns:
         None
     """
+    os.makedirs(dest_folder, exist_ok=True)
     for f in files:
+        dest_path = os.path.join(dest_folder, os.path.basename(f))
+        if os.path.exists(os.path.abspath(dest_path)):
+            continue
         shutil.copy(f, dest_folder)
 
 
@@ -155,8 +159,10 @@ def handle_files_and_directories(
     """
     os.makedirs(dest_folder_bad, exist_ok=True)
     os.makedirs(dest_folder_good, exist_ok=True)
-    logger.info(f"Copying {len(files_bad)} files to {dest_folder_bad}")
-    logger.info(f"Copying {len(files_good)} files to {dest_folder_good}")
+    logger.info(f"Copying {len(files_bad)} files to {dest_folder_bad} \n {files_bad}")
+    logger.info(
+        f"Copying {len(files_good)} files to {dest_folder_good} \n {files_good}"
+    )
     copy_files(files_bad, dest_folder_bad)
     copy_files(files_good, dest_folder_good)