From c3937af8006ef04fe689eab1bbd29014ad63c4c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Jos=C3=A9=20Pereira?= Date: Wed, 31 Jan 2024 05:28:07 -0300 Subject: [PATCH] core: services: log_zipper: Remove .gz files if available disk size is below limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Patrick José Pereira --- core/services/log_zipper/main.py | 35 +++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/core/services/log_zipper/main.py b/core/services/log_zipper/main.py index 548e2edb7e..b3f5863563 100755 --- a/core/services/log_zipper/main.py +++ b/core/services/log_zipper/main.py @@ -10,7 +10,11 @@ import time from typing import List -from commonwealth.utils.general import delete_everything, limit_ram_usage +from commonwealth.utils.general import ( + available_disk_space_mb, + delete_everything, + limit_ram_usage, +) from commonwealth.utils.logs import InterceptHandler, init_logger from loguru import logger @@ -33,10 +37,18 @@ def zip_files(files: List[str], output_path: str) -> None: logger.debug(f"Error deleting file: {file} - {e}") +# pylint: disable=too-many-locals def main() -> None: parser = argparse.ArgumentParser(description="Periodically scan a directory and zip files older than one hour") parser.add_argument("path", help="Directory path or glob to scan") parser.add_argument("-a", "--max-age-minutes", type=int, default=10, help="Maximum age for files in minutes") + parser.add_argument( + "-l", + "--free-disk-limit", + type=int, + default=30, + help="Minimum free disk (MB) allowed before starting deleting logs", + ) args = parser.parse_args() logging.basicConfig(handlers=[InterceptHandler()], level=0) @@ -49,12 +61,29 @@ def main() -> None: # We need to transform from minutes to seconds, since this is what time and st_mtime returns max_age_seconds = args.max_age_minutes * 60 + zip_extension = "gz" + while True: now = time.time() - logger.info(f"Scanning {args.path} for files older than {str(datetime.timedelta(seconds=max_age_seconds))}...") + + free_disk_space_mb = int(available_disk_space_mb()) + not_enough_space = free_disk_space_mb < args.free_disk_limit # Get the root directories of all files + if not_enough_space: + logger.warning( + f"Available disk space is lower than our limit: {free_disk_space_mb}MB < {args.free_disk_limit}MB" + ) + logger.warning(f"Going to delete all compressed files.. (*.{zip_extension})") + files = glob.glob(args.path.replace('.log', '.gz'), recursive=True) + pathlib_files = [pathlib.Path(file) for file in files] + gz_files = [file for file in pathlib_files if file.is_file() and file.suffix == f".{zip_extension}"] + for file in gz_files: + logger.warning(f"Deleting {file}: {int(file.stat().st_size / 2**20)} MB") + delete_everything(pathlib.Path(file)) + files = glob.glob(args.path, recursive=True) + logger.info(f"Scanning {args.path} for files older than {str(datetime.timedelta(seconds=max_age_seconds))}...") files = [file for file in files if os.path.isfile(file) and os.stat(file).st_mtime < now - max_age_seconds] root_dirs = list(set(os.path.dirname(file) for file in files)) root_dirs.sort() @@ -66,7 +95,7 @@ def main() -> None: continue folder_name = os.path.basename(folder) timestamp = time.strftime("%Y%m%d-%H%M%S") - zipped_file = f"{folder}/{folder_name}-{timestamp}.gz" + zipped_file = f"{folder}/{folder_name}-{timestamp}.{zip_extension}" zip_files(local_files, zipped_file) logger.info(f"Created zip archive {zipped_file} with {len(local_files)} files.")