From d794cf18bfaaa3e3dcffa0a2a903a4fa2f920831 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Jos=C3=A9=20Pereira?= Date: Wed, 31 Jan 2024 03:04:53 -0300 Subject: [PATCH 1/4] core: libs: commonwealth: setup: Add psutil MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Patrick José Pereira --- core/libs/commonwealth/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/libs/commonwealth/setup.py b/core/libs/commonwealth/setup.py index cca6da8109..66d3b1eed5 100644 --- a/core/libs/commonwealth/setup.py +++ b/core/libs/commonwealth/setup.py @@ -24,6 +24,7 @@ "appdirs == 1.4.4", "loguru == 0.5.3", "starlette == 0.27.0", + "psutil == 5.7.2", "pykson == 1.0.2", ], dependency_links=[ From df60887df1614f0b2b376013b17e5a02e84fcc4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Jos=C3=A9=20Pereira?= Date: Wed, 31 Jan 2024 03:04:02 -0300 Subject: [PATCH 2/4] core: libs: commonwealth: utils: general: Add available_disk_space_mb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Patrick José Pereira --- core/libs/commonwealth/commonwealth/utils/general.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/core/libs/commonwealth/commonwealth/utils/general.py b/core/libs/commonwealth/commonwealth/utils/general.py index e36d944308..29259deff8 100644 --- a/core/libs/commonwealth/commonwealth/utils/general.py +++ b/core/libs/commonwealth/commonwealth/utils/general.py @@ -5,8 +5,11 @@ from functools import cache from pathlib import Path +import psutil from loguru import logger +from commonwealth.utils.decorators import temporary_cache + @cache def blueos_version() -> str: @@ -108,3 +111,9 @@ def device_id() -> str: logger.exception(f"Could not get device's machine-id. {error}") raise ValueError("Could not get device id.") + + +@temporary_cache(timeout_seconds=600) # type: ignore +def available_disk_space_mb() -> float: + # Make mypy happy + return float(psutil.disk_usage("/").free / (2**20)) From 0394dafb3474d4ec946689e98efe7b10a523cf49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Jos=C3=A9=20Pereira?= Date: Wed, 31 Jan 2024 05:28:07 -0300 Subject: [PATCH 3/4] core: services: log_zipper: Remove .gz files if available disk size is below limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Patrick José Pereira --- core/services/log_zipper/main.py | 35 +++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/core/services/log_zipper/main.py b/core/services/log_zipper/main.py index 548e2edb7e..93a9eb9287 100755 --- a/core/services/log_zipper/main.py +++ b/core/services/log_zipper/main.py @@ -10,7 +10,11 @@ import time from typing import List -from commonwealth.utils.general import delete_everything, limit_ram_usage +from commonwealth.utils.general import ( + available_disk_space_mb, + delete_everything, + limit_ram_usage, +) from commonwealth.utils.logs import InterceptHandler, init_logger from loguru import logger @@ -33,10 +37,18 @@ def zip_files(files: List[str], output_path: str) -> None: logger.debug(f"Error deleting file: {file} - {e}") +# pylint: disable=too-many-locals def main() -> None: parser = argparse.ArgumentParser(description="Periodically scan a directory and zip files older than one hour") parser.add_argument("path", help="Directory path or glob to scan") parser.add_argument("-a", "--max-age-minutes", type=int, default=10, help="Maximum age for files in minutes") + parser.add_argument( + "-l", + "--free-disk-limit", + type=int, + default=30, + help="Minimum free disk (MB) allowed before starting deleting logs", + ) args = parser.parse_args() logging.basicConfig(handlers=[InterceptHandler()], level=0) @@ -49,12 +61,29 @@ def main() -> None: # We need to transform from minutes to seconds, since this is what time and st_mtime returns max_age_seconds = args.max_age_minutes * 60 + zip_extension = "gz" + while True: now = time.time() - logger.info(f"Scanning {args.path} for files older than {str(datetime.timedelta(seconds=max_age_seconds))}...") + + free_disk_space_mb = int(available_disk_space_mb()) + not_enough_space = free_disk_space_mb < args.free_disk_limit # Get the root directories of all files + if not_enough_space: + logger.warning( + f"Available disk space is lower than our limit: {free_disk_space_mb}MB < {args.free_disk_limit}MB" + ) + logger.warning(f"Going to delete all compressed files.. (*.{zip_extension})") + files = glob.glob(args.path.replace(".log", ".gz"), recursive=True) + pathlib_files = [pathlib.Path(file) for file in files] + gz_files = [file for file in pathlib_files if file.is_file() and file.suffix == f".{zip_extension}"] + for file in gz_files: + logger.warning(f"Deleting {file}: {int(file.stat().st_size / 2**20)} MB") + delete_everything(pathlib.Path(file)) + files = glob.glob(args.path, recursive=True) + logger.info(f"Scanning {args.path} for files older than {str(datetime.timedelta(seconds=max_age_seconds))}...") files = [file for file in files if os.path.isfile(file) and os.stat(file).st_mtime < now - max_age_seconds] root_dirs = list(set(os.path.dirname(file) for file in files)) root_dirs.sort() @@ -66,7 +95,7 @@ def main() -> None: continue folder_name = os.path.basename(folder) timestamp = time.strftime("%Y%m%d-%H%M%S") - zipped_file = f"{folder}/{folder_name}-{timestamp}.gz" + zipped_file = f"{folder}/{folder_name}-{timestamp}.{zip_extension}" zip_files(local_files, zipped_file) logger.info(f"Created zip archive {zipped_file} with {len(local_files)} files.") From f2e22684b71ebadc25260d4577915254b3e55dcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Jos=C3=A9=20Pereira?= Date: Wed, 31 Jan 2024 07:00:09 -0300 Subject: [PATCH 4/4] core: libs: install-libs: Install BUILD_PACKAGES in all platforms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Patrick José Pereira --- core/libs/install-libs.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/core/libs/install-libs.sh b/core/libs/install-libs.sh index 29ac3cdb34..a1e118302c 100755 --- a/core/libs/install-libs.sh +++ b/core/libs/install-libs.sh @@ -9,11 +9,10 @@ BUILD_PACKAGES=( echo "Target architecture: $TARGETARCH" echo "Target variant: $TARGETVARIANT" -# Install build packages if not on armv7 or amd64, which are the only platforms that have pre-built wheels -if ! { [ "$TARGETARCH" == "arm" ] && [ "$TARGETVARIANT" == "v7" ]; } && [ "$TARGETARCH" != "amd64" ]; then - apt update - apt install -y --no-install-recommends ${BUILD_PACKAGES[*]} -fi + +# psutil requires BUILD_PACKAGES to build to all platforms +apt update +apt install -y --no-install-recommends ${BUILD_PACKAGES[*]} # Piwheels is a Python package repository providing Arm platform wheels (pre-compiled binary Python packages) # specifically for the Raspberry Pi, making pip installations much faster.