From 773ed3e6a85f774a3a97276f60a8ba305d62f2df Mon Sep 17 00:00:00 2001 From: Maadhyam Rana <864mmr@gmail.com> Date: Wed, 25 Dec 2024 12:32:25 +0000 Subject: [PATCH 1/3] updated DeleteOldObservations to handle FramesFiles better --- RMS/ConfigReader.py | 12 ++--- RMS/DeleteOldObservations.py | 95 +++++++++++++++++++++--------------- 2 files changed, 62 insertions(+), 45 deletions(-) diff --git a/RMS/ConfigReader.py b/RMS/ConfigReader.py index b4c28bec6..c5548207e 100644 --- a/RMS/ConfigReader.py +++ b/RMS/ConfigReader.py @@ -361,12 +361,12 @@ def __init__(self): # Frame dirs to keep # Keep this many frame dirs (days) # Zero means keep them all - self.frame_dirs_to_keep = 4 + self.frame_days_to_keep = 4 # Video dirs to keep # Keep this many video dirs (days) # Zero means keep them all - self.video_dirs_to_keep = 2 + self.video_days_to_keep = 2 # Space quotas in GB @@ -964,11 +964,11 @@ def parseCapture(config, parser): if parser.has_option(section, "capt_dirs_to_keep"): config.capt_dirs_to_keep = int(parser.get(section, "capt_dirs_to_keep")) - if parser.has_option(section, "frame_dirs_to_keep"): - config.bz2_files_to_keep = int(parser.get(section, "frame_dirs_to_keep")) + if parser.has_option(section, "frame_days_to_keep"): + config.frame_days_to_keep = int(parser.get(section, "frame_days_to_keep")) - if parser.has_option(section, "video_dirs_to_keep"): - config.video_dirs_to_keep = int(parser.get(section, "video_dirs_to_keep")) + if parser.has_option(section, "video_days_to_keep"): + config.video_days_to_keep = int(parser.get(section, "video_days_to_keep")) if parser.has_option(section, "quota_management_disabled"): config.quota_management_disabled = parser.getboolean(section, "quota_management_disabled") diff --git a/RMS/DeleteOldObservations.py b/RMS/DeleteOldObservations.py index 2341c1f54..ccd882207 100644 --- a/RMS/DeleteOldObservations.py +++ b/RMS/DeleteOldObservations.py @@ -12,6 +12,7 @@ import glob import argparse import subprocess +import re import ephem @@ -308,26 +309,27 @@ def getNightDirs(dir_path, stationID): return dir_list -def getRawDirs(dir_path, in_frame_dir=False): - """ Returns a sorted list of directories / files from within the raw video/frames directories. +def getRawItems(dir_path, in_frame_dir=False, unique=False): + """ Returns a sorted list of video directories / frame files from within the raw video / frames directories respectively. In case of frames directory, only adds to the list the processed frames-(archive, timelapse, json) files and not directories as they may be unprocessed. Arguments: - dir_path: [str] Path to the raw frames/video directory. - in_frame_dir: [bool] Set to True when dir_path is frames directory. False by default. + dir_path: [str] Path to the raw video / frames directory. + in_frame_dir: [bool] Set this to True when dir_path is frames directory. False by default. + unique: [bool] Set this to True to get unique files by date Return: - dir_list: [list] A list of directories in the raw frame/video directory, each corresponding to one day of data + dir_list: [list] A list of directories / files in the raw video / frame directories, each corresponding to one day of data """ - # Helper function to check file conditions + # Helper function to check frames file conditions def isProcessedFrameFile(path): suffix = ['_frametimes.json', '_frames_timelapse.mp4', '_frames.tar.gz', '_frames.tar.bz2'] return (os.path.isfile(path) and any(path.endswith(end) for end in suffix)) - # Get a list of directories in the given directory + # Get a list of directories / files in the given directory if not os.path.exists(dir_path): return [] @@ -341,6 +343,20 @@ def isProcessedFrameFile(path): else: raw_list += [os.path.join(year_path, day_dir) for day_dir in os.listdir(year_path) if os.path.isdir(os.path.join(year_path, day_dir))] + + # Output files with unique dates + if unique: + unique_dates = [] + temp_path_list = [] + + for raw_item in raw_list: + date = os.path.basename(raw_item).split('_')[1] # YYYYMMDD-DoY + if date not in unique_dates: + temp_path_list += [raw_item] + unique_dates += [date] + + raw_list = temp_path_list + return sorted(raw_list) @@ -407,41 +423,42 @@ def deleteNightFolders(dir_path, config, delete_all=False): return getNightDirs(dir_path, config.stationID) -def deleteRawFolders(dir_path, delete_all=False, in_frame_dir=False): - """ Deletes raw frame/video data directories and files to free up disk space. In case of frames directory, +def deleteRawItems(dir_path, delete_all=False, in_frame_dir=False, unique=False): + """ Deletes raw video / frames data directories / files respectively, to free up disk space. In case of frames directory, it only will check for and delete old processed files (timelapses, archives, and json files) per day - and not directories. + and not directories as they may be unprocessed. Arguments: dir_path: [str] Path to the data directory Keyword arguments: - delete_all: [bool] If True, all raw video/frame folders will be deleted. False by default. - in_frame_dir: [bool] Set to True when dir_path is frames directory. False by default. + delete_all: [bool] If True, all raw video / frame folders will be deleted. False by default. + in_frame_dir: [bool] Set this to True when dir_path is frames directory. False by default. + unique: [bool] Set this to True to get unique files by date Return: - dir_list: [list] A list of remaining raw video/frame directories in the data directory. - + dir_list: [list] A list of remaining raw video/frame directories/files in the data directory. """ - # Get the list of night directories - del_list = getRawDirs(dir_path, in_frame_dir=in_frame_dir) + # Get the list of raw directories/files + del_list = getRawItems(dir_path, in_frame_dir=in_frame_dir) - # Delete the raw frame/video directories + # Delete the raw video / frames directories / files, respectively for item in del_list: - # Delete the next directory or file in the list, i.e. the oldest one + # Delete the next directory or file(s) in the list, i.e. the oldest ones try: if in_frame_dir: - # For frame files, each day has a corresponding archive (gz or bz2), timelapse and a json file - # We match file date prefix to remove this batch of files for one day, as the list is sorted - # File base names are of type STATIONID_YYYYMMDD-DoY_* - for item2 in del_list: - if (''.join(os.path.basename(item).split('_')[1]) == ''.join(os.path.basename(item2).split('_')[1])): - os.remove(item2) - else: - break + # For frame files, each day has a triplet of files: an archive (gz or bz2), a timelapse, and a json file + # We match file date for this batch of files for one day. The file base names are of type STATIONID_YYYYMMDD-DoY_* + date_str = os.path.basename(item).split('_')[1] + + # Delete all files with this target date + files_to_delete = [path for path in del_list if date_str in os.path.basename(path)] + + for file in files_to_delete: + os.remove(file) else: shutil.rmtree(item) @@ -454,7 +471,7 @@ def deleteRawFolders(dir_path, delete_all=False, in_frame_dir=False): break # Return the list of remaining raw frame/video directories - return getRawDirs(dir_path) + return getRawItems(dir_path, in_frame_dir=in_frame_dir, unique=unique) @@ -635,7 +652,7 @@ def deleteOldObservations(data_dir, captured_dir, archived_dir, config, duration while True: # Delete one day of video directory data - video_dirs_remaining = deleteRawFolders(video_dir) + video_dirs_remaining = deleteRawItems(video_dir) log.info("Deleted dir in video directory: {:s}".format(video_dir)) log.info("Free space: {:.2f} GB".format(availableSpace(data_dir)/1024/1024/1024)) @@ -647,9 +664,9 @@ def deleteOldObservations(data_dir, captured_dir, archived_dir, config, duration # Delete one day of frame directory data - frame_dirs_remaining = deleteRawFolders(frame_dir, in_frame_dir=True) + frame_dirs_remaining = deleteRawItems(frame_dir, in_frame_dir=True) - log.info("Deleted dir in frame directory: {:s}".format(frame_dir)) + log.info("Deleted files in frame directory: {:s}".format(frame_dir)) log.info("Free space: {:.2f} GB".format(availableSpace(data_dir)/1024/1024/1024)) # Break the there's enough space @@ -807,29 +824,29 @@ def deleteOldDirs(data_dir, config): orig_count = 0 final_count = 0 frame_dir = os.path.join(data_dir, config.frame_dir) - if config.frame_dirs_to_keep > 0: - framedir_list = getRawDirs(frame_dir) + if config.frame_days_to_keep > 0: + framedir_list = getRawItems(frame_dir, in_frame_dir=True, unique=True) orig_count = len(framedir_list) - while len(framedir_list) > config.frame_dirs_to_keep: + while len(framedir_list) > config.frame_days_to_keep: prev_length = len(framedir_list) - framedir_list = deleteRawFolders(frame_dir, in_frame_dir=True) + framedir_list = deleteRawItems(frame_dir, in_frame_dir=True, unique=True) if len(framedir_list) == prev_length: log.error("Failed to delete folder from FrameFiles. Exiting loop.") break final_count = len(framedir_list) - log.info('Purged {} old files from FrameFiles'.format(orig_count - final_count)) + log.info('Purged old files from {} days in FrameFiles'.format(orig_count - final_count)) # Deleting old video dirs orig_count = 0 final_count = 0 video_dir = os.path.join(data_dir, config.video_dir) - if config.video_dirs_to_keep > 0: - videodir_list = getRawDirs(video_dir) + if config.video_days_to_keep > 0: + videodir_list = getRawItems(video_dir) orig_count = len(videodir_list) - while len(videodir_list) > config.video_dirs_to_keep: + while len(videodir_list) > config.video_days_to_keep: prev_length = len(videodir_list) - videodir_list = deleteRawFolders(video_dir) + videodir_list = deleteRawItems(video_dir) if len(videodir_list) == prev_length: log.error("Failed to delete folder from VideoFiles. Exiting loop.") break From ddd6175bcf7ba2bed32cff7230411eb5612fca8d Mon Sep 17 00:00:00 2001 From: MaadhyamRana <68155554+MaadhyamRana@users.noreply.github.com> Date: Wed, 25 Dec 2024 15:42:11 +0300 Subject: [PATCH 2/3] Update .config --- .config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.config b/.config index e3ca9089a..df4433002 100644 --- a/.config +++ b/.config @@ -267,11 +267,11 @@ bz2_files_to_keep: 20 ; Number of CapturedFiles folders to keep. Default is 8 capt_dirs_to_keep: 8 -; Number of frame folders (days) to keep. Default is 8 -frame_dirs_to_keep: 4 +; Number of frame file days to keep. Default is 4 +frame_days_to_keep: 4 ; Number of raw video folders (days) to keep. Default is 2 -video_dirs_to_keep: 2 +video_days_to_keep: 2 ; the next settings configure the archive management by space used on disk From 6ce48331b4f2e61d0ae71e2912bbc380836e06a0 Mon Sep 17 00:00:00 2001 From: Denis Vida Date: Thu, 26 Dec 2024 11:59:20 -0500 Subject: [PATCH 3/3] Reset resume flag after capture execution to ensure correct behavior in subsequent runs --- RMS/StartCapture.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/RMS/StartCapture.py b/RMS/StartCapture.py index eee6b6682..0ed0f2318 100644 --- a/RMS/StartCapture.py +++ b/RMS/StartCapture.py @@ -217,9 +217,6 @@ def runCapture(config, duration=None, video_file=None, nodetect=False, detect_en else: log.info("Previous capture directory found: {:s}".format(night_data_dir)) - # Resume run is finished now, reset resume flag - cml_args.resume = False - # Make a name for the capture data directory if night_data_dir_name is None: @@ -983,6 +980,7 @@ def processIncompleteCaptures(config, upload_manager): # Run the capture for the given number of hours runCapture(config, duration=duration, nodetect=cml_args.nodetect, upload_manager=upload_manager, \ detect_end=cml_args.detectend, resume_capture=cml_args.resume) + cml_args.resume = False if upload_manager is not None: # Stop the upload manager @@ -1008,6 +1006,7 @@ def processIncompleteCaptures(config, upload_manager): # Capture the video frames from the video file runCapture(config, duration=None, video_file=video_file, nodetect=cml_args.nodetect, resume_capture=cml_args.resume) + cml_args.resume = False sys.exit() @@ -1332,6 +1331,7 @@ def processIncompleteCaptures(config, upload_manager): night_archive_dir = runCapture(config, duration=duration, nodetect=cml_args.nodetect, \ upload_manager=upload_manager, eventmonitor=eventmonitor, detect_end=(cml_args.detectend or config.postprocess_at_end), \ resume_capture=cml_args.resume, daytime_mode=daytime_mode) + cml_args.resume = False # Indicate that the capture was done once ran_once = True