From 0e8a4ef8b76ac5d22d6f3a298fcb3ace1846bf23 Mon Sep 17 00:00:00 2001 From: geirawsm Date: Tue, 9 Jul 2024 13:56:07 +0200 Subject: [PATCH] Added logging to file and `log_maintenance` --- dockerfile | 2 +- sausage_bot/__main__.py | 3 +- sausage_bot/cogs/log_maintenance.py | 181 ++++++++++++++++++++++++++++ sausage_bot/util/args.py | 7 +- sausage_bot/util/config.py | 2 + sausage_bot/util/envs.py | 2 + sausage_bot/util/file_io.py | 30 +++++ 7 files changed, 224 insertions(+), 3 deletions(-) create mode 100755 sausage_bot/cogs/log_maintenance.py diff --git a/dockerfile b/dockerfile index 31c005b..76005fa 100755 --- a/dockerfile +++ b/dockerfile @@ -23,4 +23,4 @@ RUN echo -e \ > /app/sausage_bot/version.json # Run bot -CMD ["python", "-m", "sausage_bot", "--log", "--verbose", "--log-print", "--log-database", "--debug", "--data-dir", "/data"] +CMD ["python", "-m", "sausage_bot", "--log", "--verbose", "--log-print", "--log-database", "--debug", "--log-file", "--data-dir", "/data"] diff --git a/sausage_bot/__main__.py b/sausage_bot/__main__.py index ff73b53..0b7b46c 100755 --- a/sausage_bot/__main__.py +++ b/sausage_bot/__main__.py @@ -15,7 +15,8 @@ # Create necessary folders before starting check_and_create_folders = [ - envs.DB_DIR + envs.DB_DIR, + envs.LOG_DIR ] for folder in check_and_create_folders: try: diff --git a/sausage_bot/cogs/log_maintenance.py b/sausage_bot/cogs/log_maintenance.py new file mode 100755 index 0000000..38f516b --- /dev/null +++ b/sausage_bot/cogs/log_maintenance.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- +import os +from discord.ext import commands, tasks +import discord + +from sausage_bot.util import envs, file_io, config +from sausage_bot.util import discord_commands, db_helper +from sausage_bot.util.log import log + + +class LogMaintenance(commands.Cog): + ''' + Start or stop the log maintenance + ''' + + def __init__(self, bot): + self.bot = bot + super().__init__() + + log_group = discord.app_commands.Group( + name="log_maintenance", description='Administer log task' + ) + + @log_group.command( + name='start', description='Start log maintenance' + ) + async def log_maintenance_start( + self, interaction: discord.Interaction + ): + await interaction.response.defer(ephemeral=True) + log.log('Task started') + LogMaintenance.log_maintenance.start() + await db_helper.update_fields( + template_info=envs.tasks_db_schema, + where=[ + ('cog', 'log'), + ('task', 'log_maintenance') + ], + updates=('status', 'started') + ) + await interaction.followup.send( + 'Log maintenance started' + ) + + @log_group.command( + name='stop', description='Stop log maintenance' + ) + async def rss_posting_stop( + self, interaction: discord.Interaction + ): + await interaction.response.defer(ephemeral=True) + log.log('Task stopped') + LogMaintenance.log_maintenance.cancel() + await db_helper.update_fields( + template_info=envs.tasks_db_schema, + where=[ + ('task', 'log_maintenance'), + ('cog', 'log'), + ], + updates=('status', 'stopped') + ) + await interaction.followup.send( + 'Log maintenance stopped' + ) + + # Tasks + @tasks.loop(hours=4) + async def log_maintenance(): + # maintenance logs folder + log_files = os.listdir(envs.LOG_DIR) + log.debug( + f'Log limit is {config.LOG_LIMIT} in {config.LOG_LIMIT_TYPE}' + ) + if config.LOG_LIMIT == 0: + log.verbose( + f'`LOG_LIMIT` is {config.LOG_LIMIT} {type(config.LOG_LIMIT)}' + ) + log_msg = 'I\'m not deleting logs, but have notified the bot '\ + 'channel about the situation' + if config.LOG_LIMIT_TYPE == 'size': + folder_size = file_io.folder_size( + str(envs.LOG_DIR), human=True + ) + discord_msg = '`LOG_LIMIT` is set to `0` in the .env file. '\ + f'The log folder\'s size as of now is {folder_size}. To '\ + 'disable these messages, run `/log_clean stop`' + elif config.LOG_LIMIT_TYPE in ['day', 'days']: + num_log_files = len(os.listdir(str(envs.LOG_DIR))) + discord_msg = '`LOG_LIMIT` is set to `0` in the .env file. '\ + f'The log folder has logs from {num_log_files} days '\ + 'back. To disable these messages, run `/log_clean stop`' + else: + log_msg = 'Wrong input in LOG_LIMIT_TYPE: '\ + f'{config.LOG_LIMIT_TYPE}' + discord_msg = log_msg + log.verbose(log_msg) + await discord_commands.log_to_bot_channel(discord_msg) + elif config.LOG_LIMIT > 0: + deleted_files = [] + status_msg = 'Log maintenance done. Deleted the following files:' + discord_msg_out = '' + log_msg_out = '' + folder_size = file_io.folder_size(str(envs.LOG_DIR)) + log.debug(f'`folder_size` is {folder_size}') + if folder_size < config.LOG_LIMIT: + log.verbose('`folder_size` is below threshold in `LOG_LIMIT`') + return + if config.LOG_LIMIT_TYPE == 'size': + # Check total size and get diff from wanted limit + size_diff = folder_size - config.LOG_LIMIT + log.debug(f'`size_diff` is {size_diff}') + log.debug(f'Checking these files: {log_files}') + for _file in log_files: + print(f'Checking file {_file}') + size_diff -= file_io.file_size(envs.LOG_DIR / _file) + deleted_files.append(_file) + print(f'`size_diff` reduced to {size_diff}') + if size_diff <= 0: + break + print(f'Got enough files to delete: {deleted_files}') + for _file in deleted_files: + os.remove(envs.LOG_DIR / _file) + new_folder_size = file_io.folder_size(str(envs.LOG_DIR)) + print(f'Folder went from {folder_size} to {new_folder_size}') + elif config.LOG_LIMIT_TYPE in ['day', 'days']: + if len(log_files) > 10: + for _file in log_files[0:-10]: + os.remove(envs.LOG_DIR / _file) + deleted_files.append(_file) + else: + log.error( + '`LOG_LIMIT_TYPE` empty or not parseable: ' + f'`{config.LOG_LIMIT_TYPE}`' + ) + return + if len(deleted_files) > 0: + discord_msg_out += status_msg + for _file in deleted_files: + discord_msg_out += f'\n- {_file}' + await discord_commands.log_to_bot_channel(discord_msg_out) + log_msg_out += status_msg + log_msg_out += ' ' + log_msg_out += ', '.join(deleted_files) + log.log(log_msg_out) + else: + _error_msg = 'LOG_LIMIT is less than 0' + log.error(_error_msg) + await discord_commands.log_to_bot_channel(_error_msg) + + +async def setup(bot): + # Create necessary databases before starting + cog_name = 'log_maintenance' + log.log(envs.COG_STARTING.format(cog_name)) + log.verbose('Checking db') + + log.verbose('Registering cog to bot') + await bot.add_cog(LogMaintenance(bot)) + + task_list = await db_helper.get_output( + template_info=envs.tasks_db_schema, + select=('task', 'status'), + where=('cog', 'log') + ) + if len(task_list) == 0: + log.debug('Could not find `log_maintenance` as a task, adding...') + await db_helper.insert_many_all( + template_info=envs.tasks_db_schema, + inserts=( + ('log', 'log_maintenance', 'stopped') + ) + ) + for task in task_list: + if task[0] == 'log_maintenance': + if task[1] == 'started': + log.debug(f'`{task[0]}` is set as `{task[1]}`, starting...') + LogMaintenance.log_maintenance.start() + elif task[1] == 'stopped': + log.debug(f'`{task[0]}` is set as `{task[1]}`') + LogMaintenance.log_maintenance.cancel() diff --git a/sausage_bot/util/args.py b/sausage_bot/util/args.py index 20cbed1..e899cf9 100755 --- a/sausage_bot/util/args.py +++ b/sausage_bot/util/args.py @@ -29,10 +29,15 @@ default=False, dest='log_error') logging_args.add_argument('--log-print', '-lp', - help='Print logging instead of writing to file', + help='Print logging to output', action='store_true', default=False, dest='log_print') +logging_args.add_argument('--log-file', '-lf', + help='Write log to files', + action='store_true', + default=False, + dest='log_file') logging_args.add_argument('--log-database', '-ld', help='Log database actions', action='store_true', diff --git a/sausage_bot/util/config.py b/sausage_bot/util/config.py index 6371d6a..ff8e277 100755 --- a/sausage_bot/util/config.py +++ b/sausage_bot/util/config.py @@ -33,6 +33,8 @@ def config(): ROLE_CHANNEL = env('ROLE_CHANNEL', default='roles') SPOTIFY_ID = env('SPOTIFY_ID', default=None) SPOTIFY_SECRET = env('SPOTIFY_SECRET', default=None) +LOG_LIMIT_TYPE = env('LOG_LIMIT_TYPE', default=envs.LOG_LIMIT_TYPE_DEFAULT) # Default is 'size' but can also be 'days' +LOG_LIMIT = env.int('LOG_LIMIT', default=envs.LOG_LIMIT_DEFAULT) # Default equals 1 GB try: DISCORD_GUILD = env('DISCORD_GUILD') diff --git a/sausage_bot/util/envs.py b/sausage_bot/util/envs.py index 9c0c74b..be84667 100755 --- a/sausage_bot/util/envs.py +++ b/sausage_bot/util/envs.py @@ -495,6 +495,8 @@ def log_extra_info(type): # VARIABLES input_split_regex = r'[\s\.\-_,;\\\/]+' roles_ensure_separator = ('><', '> <') +LOG_LIMIT_TYPE_DEFAULT = 'size' +LOG_LIMIT_DEFAULT = 1073741824 # Default is 1 GB ### DISCORD PERMISSIONS ### SELECT_PERMISSIONS = { diff --git a/sausage_bot/util/file_io.py b/sausage_bot/util/file_io.py index dac7c5d..fb09f50 100755 --- a/sausage_bot/util/file_io.py +++ b/sausage_bot/util/file_io.py @@ -94,6 +94,36 @@ def file_size(filename): return False +def folder_size(path_to_folder, human=False): + ''' + Checks the size of files in a folder. If it can't find the folder it will + return False + ''' + # Check if path exist + log.debug(f'Checking `path_to_folder`: {path_to_folder}') + if file_exist(str(path_to_folder)): + path_files = os.listdir(path_to_folder) + log.debug(f'Got files: {path_files}') + temp_size = 0 + for _file in path_files: + _size = os.stat( + f'{path_to_folder}/{_file}', follow_symlinks=True + )[stat.ST_SIZE] + temp_size += _size + if human: + return size_in_human(temp_size) + else: + return temp_size + + +def size_in_human(num, suffix="B"): + for unit in ("", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"): + if abs(num) < 1024.0: + return f"{num:3.1f}{unit}{suffix}" + num /= 1024.0 + return f"{num:.1f}Yi{suffix}" + + def file_exist(filename): ''' Checks if the file exist. If it can't find the file it will return