From 454c7267cc2aa7aff3148f261e4d392ee5b71c80 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 5 Jan 2024 13:35:53 +0530 Subject: [PATCH 001/100] remove zombie subprocess processes --- frappe_manager/docker_wrapper/utils.py | 7 ++++++- frappe_manager/logger/log.py | 21 ++++++++++++++++++--- frappe_manager/main.py | 6 +++++- frappe_manager/utils.py | 24 ++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 5 deletions(-) create mode 100644 frappe_manager/utils.py diff --git a/frappe_manager/docker_wrapper/utils.py b/frappe_manager/docker_wrapper/utils.py index a1e56920..96056a24 100644 --- a/frappe_manager/docker_wrapper/utils.py +++ b/frappe_manager/docker_wrapper/utils.py @@ -15,9 +15,10 @@ from rich import control from frappe_manager.docker_wrapper.DockerException import DockerException -logger = log.get_logger() +process_opened = [] def reader(pipe, pipe_name, queue): + logger = log.get_logger() try: with pipe: for line in iter(pipe.readline, b""): @@ -43,6 +44,10 @@ def stream_stdout_and_stderr( full_cmd = list(map(str, full_cmd)) process = Popen(full_cmd, stdout=PIPE, stderr=PIPE, env=subprocess_env) + + + process_opened.append(process.pid) + q = Queue() full_stderr = b"" # for the error message # we use deamon threads to avoid hanging if the user uses ctrl+c diff --git a/frappe_manager/logger/log.py b/frappe_manager/logger/log.py index 42e4f469..9bae5685 100644 --- a/frappe_manager/logger/log.py +++ b/frappe_manager/logger/log.py @@ -5,7 +5,7 @@ import shutil import gzip from typing import Dict, Optional, Union - +from frappe_manager.site_manager.Richprint import richprint def namer(name): return name + ".gz" @@ -19,18 +19,33 @@ def rotator(source, dest): loggers: Dict[str, logging.Logger] = {} log_directory = CLI_DIR / 'logs' +# Define MESSAGE log level +CLEANUP = 25 + +# "Register" new loggin level +logging.addLevelName(CLEANUP, 'CLEANUP') + +class FMLOGGER(logging.Logger): + def cleanup(self, msg, *args, **kwargs): + if self.isEnabledFor(CLEANUP): + self._log(CLEANUP, msg, args, **kwargs) + def get_logger(log_dir=log_directory, log_file_name='fm') -> logging.Logger: """ Creates a Log File and returns Logger object """ # Build Log File Full Path logPath = log_dir / f"{log_file_name}.log" - # if the directory doesn't exits then create it - log_dir.mkdir(parents=True, exist_ok=True) + + try: + log_dir.mkdir(parents=False, exist_ok=True) + except PermissionError as e: + richprint.exit(f"Logging not working. {e}",os_exit=True) # Create logger object and set the format for logging and other attributes if loggers.get(log_file_name): logger: Optional[logging.Logger] = loggers.get(log_file_name) else: + logging.setLoggerClass(FMLOGGER) logger: Optional[logging.Logger] = logging.getLogger(log_file_name) logger.setLevel(logging.DEBUG) diff --git a/frappe_manager/main.py b/frappe_manager/main.py index c33dba8d..c8321798 100644 --- a/frappe_manager/main.py +++ b/frappe_manager/main.py @@ -10,15 +10,20 @@ from frappe_manager.site_manager.Richprint import richprint from frappe_manager import CLI_DIR, default_extension, SiteServicesEnum from frappe_manager.logger import log +from frappe_manager.utils import remove_zombie_subprocess_process app = typer.Typer(no_args_is_help=True,rich_markup_mode='rich') +global_service = None +sites = None def exit_cleanup(): """ This function is used to perform cleanup at the exit. """ + remove_zombie_subprocess_process() richprint.stop() + def cli_entrypoint(): # logging logger = log.get_logger() @@ -340,4 +345,3 @@ def info( # pass # def config(): # pass - diff --git a/frappe_manager/utils.py b/frappe_manager/utils.py new file mode 100644 index 00000000..1ba7d7f6 --- /dev/null +++ b/frappe_manager/utils.py @@ -0,0 +1,24 @@ +from frappe_manager.logger import log +from frappe_manager.docker_wrapper.utils import process_opened + +def remove_zombie_subprocess_process(): + """ + Terminates any zombie process + """ + if process_opened: + logger = log.get_logger() + logger.cleanup("-" * 20) + logger.cleanup(f"PROCESS: USED PROCESS {process_opened}") + + # terminate zombie docker process + import psutil + for pid in process_opened: + try: + process = psutil.Process(pid) + process.terminate() + logger.cleanup(f"Terminated Process {process.cmdline}:{pid}") + except psutil.NoSuchProcess: + logger.cleanup(f"{pid} Process not found") + except psutil.AccessDenied: + logger.cleanup(f"{pid} Permission denied") + logger.cleanup("-" * 20) From 4ade60c72825a70d8fc022b2a87557e839a17022 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 5 Jan 2024 13:41:48 +0530 Subject: [PATCH 002/100] notify user for to update --- frappe_manager/main.py | 3 ++- frappe_manager/utils.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/frappe_manager/main.py b/frappe_manager/main.py index c8321798..557aaa5a 100644 --- a/frappe_manager/main.py +++ b/frappe_manager/main.py @@ -10,7 +10,7 @@ from frappe_manager.site_manager.Richprint import richprint from frappe_manager import CLI_DIR, default_extension, SiteServicesEnum from frappe_manager.logger import log -from frappe_manager.utils import remove_zombie_subprocess_process +from frappe_manager.utils import check_update, remove_zombie_subprocess_process app = typer.Typer(no_args_is_help=True,rich_markup_mode='rich') global_service = None @@ -21,6 +21,7 @@ def exit_cleanup(): This function is used to perform cleanup at the exit. """ remove_zombie_subprocess_process() + check_update() richprint.stop() diff --git a/frappe_manager/utils.py b/frappe_manager/utils.py index 1ba7d7f6..02343b6b 100644 --- a/frappe_manager/utils.py +++ b/frappe_manager/utils.py @@ -1,5 +1,9 @@ +import importlib +import requests +import json from frappe_manager.logger import log from frappe_manager.docker_wrapper.utils import process_opened +from frappe_manager.site_manager.Richprint import richprint def remove_zombie_subprocess_process(): """ @@ -22,3 +26,18 @@ def remove_zombie_subprocess_process(): except psutil.AccessDenied: logger.cleanup(f"{pid} Permission denied") logger.cleanup("-" * 20) + +def check_update(): + url = "https://pypi.org/pypi/frappe-manager/json" + try: + update_info = requests.get(url, timeout=0.1) + update_info = json.loads(update_info.text) + fm_version = importlib.metadata.version("frappe-manager") + latest_version = update_info["info"]["version"] + if not fm_version == latest_version: + richprint.warning( + f'Ready for an update? Run "pip install --upgrade frappe-manager" to update to the latest version {latest_version}.', + emoji_code=":arrows_counterclockwise:️", + ) + except Exception as e: + pass From 866701d2ecfa0a93765937c6a40a478b67519f91 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 5 Jan 2024 14:01:52 +0530 Subject: [PATCH 003/100] fix: docker on detection --- frappe_manager/docker_wrapper/DockerClient.py | 3 ++- frappe_manager/docker_wrapper/utils.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe_manager/docker_wrapper/DockerClient.py b/frappe_manager/docker_wrapper/DockerClient.py index e83659d0..7aae9ae1 100644 --- a/frappe_manager/docker_wrapper/DockerClient.py +++ b/frappe_manager/docker_wrapper/DockerClient.py @@ -36,7 +36,7 @@ def version(self) -> dict: for source ,line in iterator: if source == 'stdout': output = json.loads(line.decode()) - except Exception: + except Exception as e: return {} return output @@ -48,6 +48,7 @@ def server_running(self) -> bool: function returns True. Otherwise, it returns False. """ docker_info = self.version() + if 'Server' in docker_info: return True else: diff --git a/frappe_manager/docker_wrapper/utils.py b/frappe_manager/docker_wrapper/utils.py index 96056a24..0c4ed8fd 100644 --- a/frappe_manager/docker_wrapper/utils.py +++ b/frappe_manager/docker_wrapper/utils.py @@ -34,6 +34,7 @@ def stream_stdout_and_stderr( full_cmd: list, env: Dict[str, str] = None, ) -> Iterable[Tuple[str, bytes]]: + logger = log.get_logger() logger.debug('- -'*10) logger.debug(f"DOCKER COMMAND: {' '.join(full_cmd)}") if env is None: From a84694dd2a26c7114676a582b9b3ea18f1d8b2bb Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 5 Jan 2024 14:02:34 +0530 Subject: [PATCH 004/100] fix: log and shell command when using --service --- frappe_manager/main.py | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/frappe_manager/main.py b/frappe_manager/main.py index 557aaa5a..e23d2386 100644 --- a/frappe_manager/main.py +++ b/frappe_manager/main.py @@ -308,7 +308,7 @@ def logs( ): """Show frappe dev server logs or container logs for a given site. """ sites.init(sitename) - sites.logs(service,follow) + sites.logs(SiteServicesEnum(service).value,follow) @app.command(no_args_is_help=True) @@ -319,7 +319,7 @@ def shell( ): """Open shell for the give site. """ sites.init(sitename) - sites.shell(service, user) + sites.shell(SiteServicesEnum(service).value, user) @app.command(no_args_is_help=True) def info( @@ -328,21 +328,3 @@ def info( """Shows information about given site.""" sites.init(sitename) sites.info() - -# @app.command() -# def doctor(): -# # Runs the doctor script in the container. or commands defined in py file -# pass - -# def db_import(): -# pass -# def db_export(): -# pass -# def site_export(): -# # backup export () -# pass -# def site_import(): -# # backup import () -# pass -# def config(): -# pass From cb31f2b57c895a2d9b64e8c61f7cafc4e792bec6 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 15:16:37 +0530 Subject: [PATCH 005/100] check site after creation --- Docker/frappe/user-script.sh | 2 ++ frappe_manager/main.py | 10 ++++++++- frappe_manager/site_manager/Richprint.py | 5 +++-- frappe_manager/site_manager/manager.py | 28 +++++++++++++++++++++--- frappe_manager/site_manager/site.py | 21 ++++++++++++++++-- 5 files changed, 58 insertions(+), 8 deletions(-) diff --git a/Docker/frappe/user-script.sh b/Docker/frappe/user-script.sh index 0865f39a..3338c709 100755 --- a/Docker/frappe/user-script.sh +++ b/Docker/frappe/user-script.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash # This script creates bench and executes it. + +set -e emer() { echo "$@" exit 1 diff --git a/frappe_manager/main.py b/frappe_manager/main.py index e23d2386..8648b89b 100644 --- a/frappe_manager/main.py +++ b/frappe_manager/main.py @@ -15,6 +15,7 @@ app = typer.Typer(no_args_is_help=True,rich_markup_mode='rich') global_service = None sites = None +logger = None def exit_cleanup(): """ @@ -22,11 +23,12 @@ def exit_cleanup(): """ remove_zombie_subprocess_process() check_update() + print('') richprint.stop() - def cli_entrypoint(): # logging + global logger logger = log.get_logger() logger.info('') logger.info(f"{':'*20}FM Invoked{':'*20}") @@ -63,6 +65,8 @@ def app_callback( """ FrappeManager for creating frappe development envrionments. """ + ctx.obj = {} + richprint.start(f"Working") sitesdir = CLI_DIR / 'sites' @@ -112,6 +116,10 @@ def app_callback( if verbose: sites.set_verbose() + ctx.obj["services"] = global_service + ctx.obj["sites"] = sites + ctx.obj["logger"] = logger + def check_frappe_app_exists(appname: str, branchname: str | None = None): # check appname try: diff --git a/frappe_manager/site_manager/Richprint.py b/frappe_manager/site_manager/Richprint.py index 90b391e6..d8237c37 100644 --- a/frappe_manager/site_manager/Richprint.py +++ b/frappe_manager/site_manager/Richprint.py @@ -127,9 +127,10 @@ def update_live(self,renderable = None, padding: tuple = (0,0,0,0)): left)`. These values represent the amount of padding to be added to the `renderable` object on each :type padding: tuple """ - if padding: - renderable=Padding(renderable,padding) if renderable: + if padding: + renderable=Padding(renderable,padding) + group = Group(self.spinner,renderable) self.live.update(group) else: diff --git a/frappe_manager/site_manager/manager.py b/frappe_manager/site_manager/manager.py index 2ebe3a06..bef7adff 100644 --- a/frappe_manager/site_manager/manager.py +++ b/frappe_manager/site_manager/manager.py @@ -14,6 +14,7 @@ from rich.columns import Columns from rich.panel import Panel from rich.table import Table +from rich.text import Text from rich import box class SiteManager: @@ -152,8 +153,29 @@ def create_site(self, template_inputs: dict): richprint.change_head(f"Starting Site") self.site.start() self.site.frappe_logs_till_start() - richprint.change_head(f"Started site") - self.info() + richprint.update_live() + richprint.change_head(f"Checking site") + + # check if site is created + if self.site.is_site_created(): + richprint.print(f"Creating Site: Done") + self.typer_context.obj["logger"].info( + f"SITE_STATUS {self.site.name}: WORKING" + ) + richprint.print(f"Started site") + self.info() + else: + self.typer_context.obj["logger"].error(f"{self.site.name}: NOT WORKING") + richprint.stop() + richprint.error( + f"There has been some error creating/starting the site.\nPlease check the logs at {CLI_DIR/ 'logs'/'fm.log'}" + ) + # prompt if site not working to delete the site + if typer.confirm(f"Do you want to delete this site {self.site.name}?"): + richprint.start("Removing Site") + self.remove_site() + else: + self.info() def remove_site(self): """ @@ -410,5 +432,5 @@ def migrate_site(self): """ richprint.change_head("Migrating Environment") if not self.site.composefile.is_services_name_same_as_template(): - self.site.down() + self.site.down(volumes=False) self.site.migrate_site_compose() diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 78a5f094..4230d562 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -217,7 +217,7 @@ def running(self) -> bool: except DockerException as e: richprint.exit(f"{e.stdout}{e.stderr}") - def down(self) -> bool: + def down(self,remove_ophans=True,volumes=True,timeout=5) -> bool: """ The `down` function removes containers using Docker Compose and prints the status of the operation. """ @@ -225,7 +225,7 @@ def down(self) -> bool: status_text='Removing Containers' richprint.change_head(status_text) try: - output = self.docker.compose.down(remove_orphans=True,volumes=True,timeout=2,stream=self.quiet) + output = self.docker.compose.down(remove_orphans=remove_ophans,volumes=volumes,timeout=timeout,stream=self.quiet) if self.quiet: exit_code = richprint.live_lines(output,padding=(0,0,0,2)) richprint.print(f"Removing Containers: Done") @@ -317,3 +317,20 @@ def bench_dev_server_logs(self, follow = False): richprint.stdout.print("Detected CTRL+C. Exiting.") else: richprint.error(f"Log file not found: {bench_start_log_path}") + + def is_site_created(self, retry=30, interval=1) -> bool: + import requests + from time import sleep + i = 0 + while i < retry: + try: + response = requests.get(f"http://{self.name}") + except Exception: + return False + if response.status_code == 200: + return True + else: + sleep(interval) + i += 1 + continue + return False From edb3472ca59daec23b860fa48a2f225b9ebf78f4 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 15:18:41 +0530 Subject: [PATCH 006/100] fix: shell,logs command --service flag --- frappe_manager/main.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/frappe_manager/main.py b/frappe_manager/main.py index 8648b89b..a491277e 100644 --- a/frappe_manager/main.py +++ b/frappe_manager/main.py @@ -316,7 +316,10 @@ def logs( ): """Show frappe dev server logs or container logs for a given site. """ sites.init(sitename) - sites.logs(SiteServicesEnum(service).value,follow) + if service: + sites.logs(SiteServicesEnum(service).value,follow) + else: + sites.logs(follow) @app.command(no_args_is_help=True) @@ -327,7 +330,10 @@ def shell( ): """Open shell for the give site. """ sites.init(sitename) - sites.shell(SiteServicesEnum(service).value, user) + if service: + sites.shell(SiteServicesEnum(service).value,follow) + else: + sites.shell(follow) @app.command(no_args_is_help=True) def info( From 49dc657bd60c52a409f1c520bbf8b69f64743071 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 15:19:58 +0530 Subject: [PATCH 007/100] docker group checking when daemon is not running --- frappe_manager/docker_wrapper/DockerClient.py | 5 +++- frappe_manager/docker_wrapper/utils.py | 25 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/frappe_manager/docker_wrapper/DockerClient.py b/frappe_manager/docker_wrapper/DockerClient.py index 7aae9ae1..149ef1c5 100644 --- a/frappe_manager/docker_wrapper/DockerClient.py +++ b/frappe_manager/docker_wrapper/DockerClient.py @@ -2,7 +2,9 @@ import json from frappe_manager.docker_wrapper.DockerCompose import DockerComposeWrapper from pathlib import Path +from frappe_manager.site_manager.Richprint import richprint from frappe_manager.docker_wrapper.utils import ( + is_current_user_in_group, parameters_to_options, run_command_with_exit_code, ) @@ -48,8 +50,9 @@ def server_running(self) -> bool: function returns True. Otherwise, it returns False. """ docker_info = self.version() - if 'Server' in docker_info: return True else: + # check if the current user in the docker group and notify the user + is_current_user_in_group('docker') return False diff --git a/frappe_manager/docker_wrapper/utils.py b/frappe_manager/docker_wrapper/utils.py index 0c4ed8fd..13811b4f 100644 --- a/frappe_manager/docker_wrapper/utils.py +++ b/frappe_manager/docker_wrapper/utils.py @@ -134,3 +134,28 @@ def parameters_to_options(param: dict, exclude: list = []) -> list: params += value return params + +def is_current_user_in_group(group_name) -> bool: + """Check if the current user is in the given group""" + + from frappe_manager.site_manager.Richprint import richprint + + import platform + if platform.system() == 'Linux': + import grp + import pwd + import os + current_user = pwd.getpwuid(os.getuid()).pw_name + try: + docker_gid = grp.getgrnam(group_name).gr_gid + docker_group_members = grp.getgrgid(docker_gid).gr_mem + if current_user in docker_group_members: + return True + else: + richprint.error(f"Your current user [blue][b] {current_user} [/b][/blue] is not in the group 'docker'. Please add it to the group and restart your terminal.") + return False + except KeyError: + richprint.error(f"The group '{group_name}' does not exist. Please create it and add your current user [blue][b] {current_user} [/b][/blue] to it.") + return False + else: + return True From 2cd6f54a78abb19c6f2395f60b405d90e0c326b6 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 15:44:28 +0530 Subject: [PATCH 008/100] add and optmize helper functions --- frappe_manager/site_manager/SiteCompose.py | 220 ++++++++++++++++----- 1 file changed, 166 insertions(+), 54 deletions(-) diff --git a/frappe_manager/site_manager/SiteCompose.py b/frappe_manager/site_manager/SiteCompose.py index 74c388a4..d90ff85e 100644 --- a/frappe_manager/site_manager/SiteCompose.py +++ b/frappe_manager/site_manager/SiteCompose.py @@ -19,9 +19,11 @@ def represent_none(self, _): class SiteCompose: - def __init__(self,loadfile: Path): + def __init__(self,loadfile: Path, template_name:str = 'docker-compose.tmpl'): self.compose_path:Path = loadfile self.site_name:str = loadfile.parent.name + self.template_name = template_name + self.is_template_loaded = False self.yml: yaml | None = None self.init() @@ -35,8 +37,9 @@ def init(self): with open(self.compose_path,'r') as f: self.yml = yaml.safe_load(f) else: - template =self.__get_template('docker-compose.tmpl') + template =self.get_template(self.template_name) self.yml = yaml.safe_load(template) + self.is_template_loaded = True def exists(self): """ @@ -52,55 +55,35 @@ def get_compose_path(self): """ return self.compose_path - def migrate_compose(self,version) -> bool: + def get_template(self, file_name: str, template_directory = "templates")-> None | str: """ - The `migrate_compose` function migrates a Docker Compose file by updating the version, environment - variables, and extra hosts. - - :param version: current version of the fm. - """ - if self.exists(): - frappe_envs = self.get_envs('frappe') - nginx_envs = self.get_envs('nginx') - extra_hosts = self.get_extrahosts('frappe') - - template =self.__get_template('docker-compose.tmpl') - self.yml = yaml.safe_load(template) - - self.set_version(version) - self.set_envs('frappe',frappe_envs) - self.set_envs('nginx',nginx_envs) - self.set_extrahosts('frappe',extra_hosts) - self.set_container_names() - self.write_to_file() - return True - return False - - def __get_template(self,file_name: str)-> None | str: - """ - The function `__get_template` retrieves a template file and returns its contents as a string, or + The function `get_template` retrieves a template file and returns its contents as a string, or raises an error if the template file is not found. - + :param file_name: The `file_name` parameter is a string that represents the name of the template file. It is used to construct the file path by appending it to the "templates/" directory :type file_name: str :return: either None or a string. """ - file_name = f"templates/{file_name}" + file_name = f"{template_directory}/{file_name}" try: data = pkgutil.get_data(__name__,file_name) - except: - richprint.error(f"{file_name} template not found!") - raise typer.Exit(1) + except Exception as e: + richprint.exit(f"{file_name} template not found! Error:{e}") yml = data.decode() return yml - def set_container_names(self): + def load_template(self): + template = self.get_template(self.template_name) + self.yml = yaml.safe_load(template) + + def set_container_names(self,prefix): """ The function sets the container names for each service in a compose file based on the site name. """ + for service in self.yml['services'].keys(): - self.yml['services'][service]['container_name'] = self.site_name.replace('.','') + f'-{service}' + self.yml['services'][service]['container_name'] = prefix + f'-{service}' def get_container_names(self) -> dict: """ @@ -108,11 +91,17 @@ def get_container_names(self) -> dict: compose file. :return: a dictionary containing the names of the containers specified in the compose file. """ + container_names:dict = {} + + # site_name = self.compose_path.parent.name + if self.exists(): - container_names:dict = {} - for service in self.yml['services'].keys(): + services = self.get_services_list() + for service in services: container_names[service] = self.yml['services'][service]['container_name'] - return container_names + # container_names[service] = site_name.replace('.','') + f'-{service}' + + return container_names def get_services_list(self) -> list: """ @@ -136,6 +125,45 @@ def is_services_name_same_as_template(self): current_service_name_list.sort() return current_service_name_list == template_service_name_list + def set_user(self,service,uid,gid): + try: + self.yml['services'][service]['user'] = f'{uid}:{gid}' + except KeyError: + richprint.exit("Issue in docker template. Not able to set user.") + + def get_user(self, service): + try: + user = self.yml[service]['user'] + uid = user.split(":")[0] + uid = user.split(":")[1] + + except KeyError: + return None + return user + + def set_network_alias(self,service_name, network_name,alias: list = []): + if alias: + try: + all_networks= self.yml['services'][service_name]['networks'] + if network_name in all_networks: + self.yml['services'][service_name]['networks'][network_name] = {"aliases":alias} + return True + except KeyError as e: + return False + else: + return False + + def get_network_alias(self,service_name,network_name) -> list | None: + try: + all_networks= self.yml['services'][service_name]['networks'] + if network_name in all_networks: + aliases = self.yml['services'][service_name]['networks'][network_name]['aliases'] + return aliases + except KeyError as e: + return None + else: + return None + def get_version(self): """ The function `get_version` returns the value of the `x-version` key from composer file, or @@ -157,10 +185,70 @@ def set_version(self, version): """ self.yml['x-version'] = version - def set_envs(self,container:str,env:dict): + def get_all_envs(self): + """ + This functtion returns all the container environment variables + """ + envs = {} + for service in self.yml['services'].keys(): + try: + env = self.yml['services'][service]['environment'] + envs[service] = env + except KeyError: + pass + return envs + + def set_all_envs(self,environments:dict): + """ + This functtion returns all the container environment variables + """ + for container_name in environments.keys(): + self.set_envs(container_name,environments[container_name],append=True) + + def get_all_labels(self): + """ + This functtion returns all the container labels variables + """ + labels = {} + for service in self.yml['services'].keys(): + try: + label = self.yml['services'][service]['labels'] + labels[service] = label + except KeyError: + pass + return labels + + def set_all_labels(self,labels:dict): + """ + This functtion returns all the container environment variables + """ + for container_name in labels.keys(): + self.set_labels(container_name,labels[container_name]) + + def get_all_extrahosts(self): + """ + This functtion returns all the container labels variables + """ + extrahosts= {} + for service in self.yml['services'].keys(): + try: + extrahost = self.yml['services'][service]['extra_hosts'] + extrahosts[service] = extrahost + except KeyError: + pass + return extrahosts + + def set_all_extrahosts(self,extrahosts:dict,skip_not_found: bool = False): + """ + This functtion returns all the container environment variables + """ + for container_name in extrahosts.keys(): + self.set_extrahosts(container_name,extrahosts[container_name]) + + def set_envs(self,container:str,env:dict, append = False): """ The function `set_envs` sets environment variables for a given container in a compose file. - + :param container: A string representing the name of the container :type container: str :param env: The `env` parameter is a dictionary that contains environment variables. Each key-value @@ -168,23 +256,40 @@ def set_envs(self,container:str,env:dict): the value is the variable value :type env: dict """ - self.yml['services'][container]['environment'] = env + # change dict to list + if append and type(env) == dict: + prev_env = self.get_envs(container) + if prev_env: + new_env = prev_env | env + else: + new_env = env + else: + new_env = env + + try: + self.yml['services'][container]['environment'] = new_env + except KeyError as e: + pass def get_envs(self, container:str) -> dict: """ The function `get_envs` retrieves the environment variables from a specified container in a compose file. - + :param container: A string representing the name of the container :type container: str :return: a dictionary containing the environment variables of the specified container. """ - return self.yml['services'][container]['environment'] + try: + env = self.yml['services'][container]['environment'] + return env + except KeyError: + return None def set_labels(self,container:str, labels:dict): """ The function `set_labels` sets the labels for a specified container in a compose file. - + :param container: The `container` parameter is a string that represents the name of a container in a YAML file :type container: str @@ -194,39 +299,46 @@ def set_labels(self,container:str, labels:dict): parameter :type labels: dict """ - self.yml['services'][container]['labels'] = labels + try: + self.yml['services'][container]['labels'] = labels + except KeyError as e: + pass def get_labels(self,container:str) -> dict: """ The function `get_labels` takes a container name as input and returns the labels associated with that container from a compose file. - + :param container: The `container` parameter is a string that represents the name of a container :type container: str :return: a dictionary of labels. """ try: labels = self.yml['services'][container]['labels'] + return labels except KeyError: - return {} - return labels + return None - def set_extrahosts(self,container:str,extrahosts:list): + def set_extrahosts(self,container:str,extrahosts:list): """ The function `set_extrahosts` sets the `extra_hosts` property of a container in a compose file. - + :param container: The container parameter is a string that represents the name of the container :type container: str :param extrahosts: A list of additional hostnames to be added to the container's /etc/hosts file. Each item in the list should be in the format "hostname:IP_address" :type extrahosts: list """ - self.yml['services'][container]['extra_hosts'] = extrahosts + try: + self.yml['services'][container]['extra_hosts'] = extrahosts + except KeyError as e: + pass + def get_extrahosts(self,container:str) -> list: """ The function `get_extrahosts` returns a list of extra hosts for a given container. - + :param container: The `container` parameter is a string that represents the name of a container :type container: str :return: a list of extra hosts for a given container. If the container is not found or if there are @@ -234,9 +346,9 @@ def get_extrahosts(self,container:str) -> list: """ try: extra_hosts = self.yml['services'][container]['extra_hosts'] + return extra_hosts except KeyError: - return [] - return extra_hosts + return None def write_to_file(self): """ From b984427529698e2c32611bf7ff4eca1b044bfa72 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 15:53:38 +0530 Subject: [PATCH 009/100] refractor: compose file managment --- .../ComposeFile.py} | 159 +++++++++--------- frappe_manager/compose_manager/utils.py | 10 ++ 2 files changed, 89 insertions(+), 80 deletions(-) rename frappe_manager/{site_manager/SiteCompose.py => compose_manager/ComposeFile.py} (68%) create mode 100644 frappe_manager/compose_manager/utils.py diff --git a/frappe_manager/site_manager/SiteCompose.py b/frappe_manager/compose_manager/ComposeFile.py similarity index 68% rename from frappe_manager/site_manager/SiteCompose.py rename to frappe_manager/compose_manager/ComposeFile.py index d90ff85e..3981c106 100644 --- a/frappe_manager/site_manager/SiteCompose.py +++ b/frappe_manager/compose_manager/ComposeFile.py @@ -5,26 +5,17 @@ import typer from frappe_manager.site_manager.Richprint import richprint +from frappe_manager.compose_manager.utils import represent_none -def represent_none(self, _): - """ - The function `represent_none` represents the value `None` as a null scalar in YAML format. - - :param _: The underscore (_) parameter is a convention in Python to indicate that the parameter is - not going to be used in the function. - :return: a representation of `None` as a YAML scalar with the tag `tag:yaml.org,2002:null` and an - empty string as its value. - """ - return self.represent_scalar('tag:yaml.org,2002:null', '') - - -class SiteCompose: - def __init__(self,loadfile: Path, template_name:str = 'docker-compose.tmpl'): - self.compose_path:Path = loadfile - self.site_name:str = loadfile.parent.name +yaml.representer.ignore_aliases = lambda *args: True + + +class ComposeFile: + def __init__(self, loadfile: Path, template_name: str = "docker-compose.tmpl"): + self.compose_path: Path = loadfile self.template_name = template_name self.is_template_loaded = False - self.yml: yaml | None = None + self.yml = None self.init() def init(self): @@ -34,10 +25,10 @@ def init(self): """ # if the load file not found then the site not exits if self.exists(): - with open(self.compose_path,'r') as f: + with open(self.compose_path, "r") as f: self.yml = yaml.safe_load(f) else: - template =self.get_template(self.template_name) + template = self.get_template(self.template_name) self.yml = yaml.safe_load(template) self.is_template_loaded = True @@ -55,7 +46,9 @@ def get_compose_path(self): """ return self.compose_path - def get_template(self, file_name: str, template_directory = "templates")-> None | str: + def get_template( + self, file_name: str, template_directory="templates" + ) -> None | str: """ The function `get_template` retrieves a template file and returns its contents as a string, or raises an error if the template file is not found. @@ -67,7 +60,7 @@ def get_template(self, file_name: str, template_directory = "templates")-> None """ file_name = f"{template_directory}/{file_name}" try: - data = pkgutil.get_data(__name__,file_name) + data = pkgutil.get_data(__name__, file_name) except Exception as e: richprint.exit(f"{file_name} template not found! Error:{e}") yml = data.decode() @@ -77,13 +70,13 @@ def load_template(self): template = self.get_template(self.template_name) self.yml = yaml.safe_load(template) - def set_container_names(self,prefix): + def set_container_names(self, prefix): """ The function sets the container names for each service in a compose file based on the site name. """ - for service in self.yml['services'].keys(): - self.yml['services'][service]['container_name'] = prefix + f'-{service}' + for service in self.yml["services"].keys(): + self.yml["services"][service]["container_name"] = prefix + f"-{service}" def get_container_names(self) -> dict: """ @@ -91,14 +84,16 @@ def get_container_names(self) -> dict: compose file. :return: a dictionary containing the names of the containers specified in the compose file. """ - container_names:dict = {} + container_names: dict = {} # site_name = self.compose_path.parent.name if self.exists(): services = self.get_services_list() for service in services: - container_names[service] = self.yml['services'][service]['container_name'] + container_names[service] = self.yml["services"][service][ + "container_name" + ] # container_names[service] = site_name.replace('.','') + f'-{service}' return container_names @@ -108,7 +103,7 @@ def get_services_list(self) -> list: Getting for getting all docker compose services name as a list. :return: list of docker composer servicers. """ - return list(self.yml['services'].keys()) + return list(self.yml["services"].keys()) def is_services_name_same_as_template(self): """ @@ -117,48 +112,52 @@ def is_services_name_same_as_template(self): :return: a boolean value indicating whether the list of service names in the current YAML file is the same as the list of service names in the template YAML file. """ - template = self.__get_template('docker-compose.tmpl') + template = self.__get_template("docker-compose.tmpl") template_yml = yaml.safe_load(template) - template_service_name_list = list(template_yml['services'].keys()) + template_service_name_list = list(template_yml["services"].keys()) template_service_name_list.sort() - current_service_name_list = list(self.yml['services'].keys()) + current_service_name_list = list(self.yml["services"].keys()) current_service_name_list.sort() return current_service_name_list == template_service_name_list - def set_user(self,service,uid,gid): + def set_user(self, service, uid, gid): try: - self.yml['services'][service]['user'] = f'{uid}:{gid}' + self.yml["services"][service]["user"] = f"{uid}:{gid}" except KeyError: richprint.exit("Issue in docker template. Not able to set user.") def get_user(self, service): try: - user = self.yml[service]['user'] - uid = user.split(":")[0] - uid = user.split(":")[1] + user = self.yml[service]["user"] + uid = user.split(":")[0] + uid = user.split(":")[1] except KeyError: return None return user - def set_network_alias(self,service_name, network_name,alias: list = []): + def set_network_alias(self, service_name, network_name, alias: list = []): if alias: try: - all_networks= self.yml['services'][service_name]['networks'] + all_networks = self.yml["services"][service_name]["networks"] if network_name in all_networks: - self.yml['services'][service_name]['networks'][network_name] = {"aliases":alias} + self.yml["services"][service_name]["networks"][network_name] = { + "aliases": alias + } return True except KeyError as e: return False else: return False - def get_network_alias(self,service_name,network_name) -> list | None: + def get_network_alias(self, service_name, network_name) -> list | None: try: - all_networks= self.yml['services'][service_name]['networks'] - if network_name in all_networks: - aliases = self.yml['services'][service_name]['networks'][network_name]['aliases'] - return aliases + all_networks = self.yml["services"][service_name]["networks"] + if network_name in all_networks: + aliases = self.yml["services"][service_name]["networks"][network_name][ + "aliases" + ] + return aliases except KeyError as e: return None else: @@ -172,80 +171,80 @@ def get_version(self): it returns None. """ try: - compose_version = self.yml['x-version'] + compose_version = self.yml["x-version"] except KeyError: - return None + return 0 return compose_version def set_version(self, version): """ The function sets the value of the 'x-version' key in a YAML dictionary to the specified version. - + :param version: current fm version to set it to "x-version" key in the compose file. """ - self.yml['x-version'] = version + self.yml["x-version"] = version def get_all_envs(self): """ This functtion returns all the container environment variables """ envs = {} - for service in self.yml['services'].keys(): + for service in self.yml["services"].keys(): try: - env = self.yml['services'][service]['environment'] + env = self.yml["services"][service]["environment"] envs[service] = env except KeyError: pass return envs - def set_all_envs(self,environments:dict): + def set_all_envs(self, environments: dict): """ This functtion returns all the container environment variables """ for container_name in environments.keys(): - self.set_envs(container_name,environments[container_name],append=True) + self.set_envs(container_name, environments[container_name], append=True) def get_all_labels(self): """ This functtion returns all the container labels variables """ labels = {} - for service in self.yml['services'].keys(): + for service in self.yml["services"].keys(): try: - label = self.yml['services'][service]['labels'] + label = self.yml["services"][service]["labels"] labels[service] = label except KeyError: pass return labels - def set_all_labels(self,labels:dict): + def set_all_labels(self, labels: dict): """ This functtion returns all the container environment variables """ for container_name in labels.keys(): - self.set_labels(container_name,labels[container_name]) + self.set_labels(container_name, labels[container_name]) def get_all_extrahosts(self): """ This functtion returns all the container labels variables """ - extrahosts= {} - for service in self.yml['services'].keys(): + extrahosts = {} + for service in self.yml["services"].keys(): try: - extrahost = self.yml['services'][service]['extra_hosts'] + extrahost = self.yml["services"][service]["extra_hosts"] extrahosts[service] = extrahost except KeyError: pass return extrahosts - def set_all_extrahosts(self,extrahosts:dict,skip_not_found: bool = False): + def set_all_extrahosts(self, extrahosts: dict, skip_not_found: bool = False): """ This functtion returns all the container environment variables """ for container_name in extrahosts.keys(): - self.set_extrahosts(container_name,extrahosts[container_name]) + self.set_extrahosts(container_name, extrahosts[container_name]) - def set_envs(self,container:str,env:dict, append = False): + def set_envs(self, container: str, env: dict, append=False): """ The function `set_envs` sets environment variables for a given container in a compose file. @@ -258,20 +257,20 @@ def set_envs(self,container:str,env:dict, append = False): """ # change dict to list if append and type(env) == dict: - prev_env = self.get_envs(container) - if prev_env: - new_env = prev_env | env - else: - new_env = env + prev_env = self.get_envs(container) + if prev_env: + new_env = prev_env | env + else: + new_env = env else: new_env = env try: - self.yml['services'][container]['environment'] = new_env + self.yml["services"][container]["environment"] = new_env except KeyError as e: pass - def get_envs(self, container:str) -> dict: + def get_envs(self, container: str) -> dict: """ The function `get_envs` retrieves the environment variables from a specified container in a compose file. @@ -281,12 +280,12 @@ def get_envs(self, container:str) -> dict: :return: a dictionary containing the environment variables of the specified container. """ try: - env = self.yml['services'][container]['environment'] + env = self.yml["services"][container]["environment"] return env except KeyError: return None - def set_labels(self,container:str, labels:dict): + def set_labels(self, container: str, labels: dict): """ The function `set_labels` sets the labels for a specified container in a compose file. @@ -300,11 +299,11 @@ def set_labels(self,container:str, labels:dict): :type labels: dict """ try: - self.yml['services'][container]['labels'] = labels + self.yml["services"][container]["labels"] = labels except KeyError as e: pass - def get_labels(self,container:str) -> dict: + def get_labels(self, container: str) -> dict: """ The function `get_labels` takes a container name as input and returns the labels associated with that container from a compose file. @@ -314,12 +313,12 @@ def get_labels(self,container:str) -> dict: :return: a dictionary of labels. """ try: - labels = self.yml['services'][container]['labels'] + labels = self.yml["services"][container]["labels"] return labels except KeyError: return None - def set_extrahosts(self,container:str,extrahosts:list): + def set_extrahosts(self, container: str, extrahosts: list): """ The function `set_extrahosts` sets the `extra_hosts` property of a container in a compose file. @@ -330,12 +329,11 @@ def set_extrahosts(self,container:str,extrahosts:list): :type extrahosts: list """ try: - self.yml['services'][container]['extra_hosts'] = extrahosts + self.yml["services"][container]["extra_hosts"] = extrahosts except KeyError as e: pass - - def get_extrahosts(self,container:str) -> list: + def get_extrahosts(self, container: str) -> list: """ The function `get_extrahosts` returns a list of extra hosts for a given container. @@ -345,7 +343,7 @@ def get_extrahosts(self,container:str) -> list: no extra hosts defined for the container, an empty list is returned. """ try: - extra_hosts = self.yml['services'][container]['extra_hosts'] + extra_hosts = self.yml["services"][container]["extra_hosts"] return extra_hosts except KeyError: return None @@ -354,7 +352,8 @@ def write_to_file(self): """ The function writes the contents of a YAML object to a file. """ + # saving the docker compose to the directory - with open(self.compose_path,'w') as f: + with open(self.compose_path, "w") as f: yaml.add_representer(type(None), represent_none) - f.write(yaml.dump(self.yml)) + f.write(yaml.dump(self.yml, default_flow_style=False)) diff --git a/frappe_manager/compose_manager/utils.py b/frappe_manager/compose_manager/utils.py new file mode 100644 index 00000000..9f832354 --- /dev/null +++ b/frappe_manager/compose_manager/utils.py @@ -0,0 +1,10 @@ +def represent_none(self, _): + """ + The function `represent_none` represents the value `None` as a null scalar in YAML format. + + :param _: The underscore (_) parameter is a convention in Python to indicate that the parameter is + not going to be used in the function. + :return: a representation of `None` as a YAML scalar with the tag `tag:yaml.org,2002:null` and an + empty string as its value. + """ + return self.represent_scalar('tag:yaml.org,2002:null', '') From a689604b16c1f9a377b1f93e55ca167694115062 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 15:54:56 +0530 Subject: [PATCH 010/100] move ComposeFile default templates dir. --- .../templates/docker-compose.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename frappe_manager/{site_manager => compose_manager}/templates/docker-compose.tmpl (96%) diff --git a/frappe_manager/site_manager/templates/docker-compose.tmpl b/frappe_manager/compose_manager/templates/docker-compose.tmpl similarity index 96% rename from frappe_manager/site_manager/templates/docker-compose.tmpl rename to frappe_manager/compose_manager/templates/docker-compose.tmpl index 2d23f760..fba1aa61 100644 --- a/frappe_manager/site_manager/templates/docker-compose.tmpl +++ b/frappe_manager/compose_manager/templates/docker-compose.tmpl @@ -1,7 +1,7 @@ version: "3.9" services: frappe: - image: ghcr.io/rtcamp/frappe-manager-frappe:v0.9.0 + image: frappe-manager-frappe:dev environment: - SHELL=/bin/bash volumes: From 4cab05965d70a87d7103a35f30e327ff3bf8cbf2 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 16:08:37 +0530 Subject: [PATCH 011/100] refractor: logic to check site status --- frappe_manager/site_manager/site.py | 71 +++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 19 deletions(-) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 4230d562..5e36b9a5 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -198,25 +198,6 @@ def stop(self) -> bool: except DockerException as e: richprint.exit(f"{status_text}: Failed") - def running(self) -> bool: - """ - The `running` function checks if all the services defined in a Docker Compose file are running. - :return: a boolean value. If the number of running containers is greater than or equal to the number - of services listed in the compose file, it returns True. Otherwise, it returns False. - """ - try: - output = self.docker.compose.ps(format='json',filter='running',stream=True) - status: dict = {} - for source,line in output: - if source == 'stdout': - status = json.loads(line.decode()) - running_containers = len(status) - if running_containers >= len(self.composefile.get_services_list()): - return True - return False - except DockerException as e: - richprint.exit(f"{e.stdout}{e.stderr}") - def down(self,remove_ophans=True,volumes=True,timeout=5) -> bool: """ The `down` function removes containers using Docker Compose and prints the status of the operation. @@ -334,3 +315,55 @@ def is_site_created(self, retry=30, interval=1) -> bool: i += 1 continue return False + + def running(self) -> bool: + """ + The `running` function checks if all the services defined in a Docker Compose file are running. + :return: a boolean value. If the number of running containers is greater than or equal to the number + of services listed in the compose file, it returns True. Otherwise, it returns False. + """ + # try: + # output = self.docker.compose.ps(format='json',filter='running',stream=True) + # status: dict = {} + # for source,line in output: + # if source == 'stdout': + # status = json.loads(line.decode()) + # running_containers = len(status) + # if running_containers >= len(self.composefile.get_services_list()): + # return True + # return False + # except DockerException as e: + # richprint.exit(f"{e.stdout}{e.stderr}") + + services = self.composefile.get_services_list() + running_status = self.get_services_running_status() + + if running_status: + for service in services: + try: + if not running_status[service] == "running": + return False + except KeyError: + return False + else: + return False + return True + + def get_services_running_status(self)-> dict: + services = self.composefile.get_services_list() + containers = self.composefile.get_container_names().values() + services_status = {} + try: + output = self.docker.compose.ps(service=services,format="json",all=True,stream=True) + status: dict = {} + for source, line in output: + if source == "stdout": + status = json.loads(line.decode()) + + # this is done to exclude docker runs using docker compose run command + for container in status: + if container['Name'] in containers: + services_status[container['Service']] = container['State'] + return services_status + except DockerException as e: + richprint.exit(f"{e.stdout}{e.stderr}") From daa48649428ff5b410565c2f5286bcf8ca6d04a6 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 16:09:16 +0530 Subject: [PATCH 012/100] refractor: site.py --- frappe_manager/site_manager/site.py | 81 +++++++++++++++++++++++------ 1 file changed, 66 insertions(+), 15 deletions(-) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 5e36b9a5..179569b9 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -7,9 +7,9 @@ from frappe_manager.docker_wrapper import DockerClient, DockerException -from frappe_manager.site_manager.SiteCompose import SiteCompose +from frappe_manager.compose_manager.ComposeFile import ComposeFile from frappe_manager.site_manager.Richprint import richprint -from frappe_manager.site_manager.utils import log_file +from frappe_manager.site_manager.utils import log_file, get_container_name_prefix class Site: def __init__(self,path: Path , name:str, verbose: bool = False): @@ -22,7 +22,7 @@ def init(self): """ The function checks if the Docker daemon is running and exits with an error message if it is not. """ - self.composefile = SiteCompose(self.path / 'docker-compose.yml') + self.composefile = ComposeFile(self.path / "docker-compose.yml") self.docker = DockerClient(compose_file_path=self.composefile.compose_path) if not self.docker.server_running(): @@ -57,7 +57,7 @@ def get_frappe_container_hex(self) -> None | str: container_name = self.composefile.get_container_names() return container_name['frappe'].encode().hex() - def migrate_site_compose(self) : + def migrate_site_compose(self): """ The `migrate_site` function checks the environment version and migrates it if necessary. :return: a boolean value,`True` if the site migrated else `False`. @@ -65,32 +65,83 @@ def migrate_site_compose(self) : if self.composefile.exists(): richprint.change_head("Checking Environment Version") compose_version = self.composefile.get_version() - fm_version = importlib.metadata.version('frappe-manager') + fm_version = importlib.metadata.version("frappe-manager") if not compose_version == fm_version: - status = self.composefile.migrate_compose(fm_version) + status = False + if self.composefile.exists(): + envs = self.composefile.get_all_envs() + # extrahosts = self.composefile.get_all_extrahosts() + labels = self.composefile.get_all_labels() + + self.composefile.load_template() + self.composefile.set_version(fm_version) + + self.composefile.set_all_envs(envs) + # self.composefile.set_all_extrahosts(extrahosts) + self.composefile.set_all_labels(labels) + + self.composefile.set_container_names(get_container_name_prefix(self.name)) + self.set_site_network_name() + self.composefile.write_to_file() + status = True + if status: - richprint.print(f"Environment Migration Done: {compose_version} -> {fm_version}") + richprint.print( + f"Environment Migration Done: {compose_version} -> {fm_version}" + ) else: - richprint.print(f"Environment Migration Failed: {compose_version} -> {fm_version}") + richprint.print( + f"Environment Migration Failed: {compose_version} -> {fm_version}" + ) else: richprint.print("Already Latest Environment Version") - def generate_compose(self,inputs:dict) -> None: + def set_site_network_name(self): + self.composefile.yml["networks"]["site-network"]["name"] = ( + self.name.replace(".", "") + f"-network" + ) + + def generate_compose(self, inputs: dict) -> None: """ The function `generate_compose` sets environment variables, extra hosts, and version information in a compose file and writes it to a file. - + :param inputs: The `inputs` parameter is a dictionary that contains the values which will be used in compose file. :type inputs: dict """ - self.composefile.set_envs('frappe',inputs['frappe_env']) - self.composefile.set_envs('nginx',inputs['nginx_env']) - self.composefile.set_extrahosts('frappe',inputs['extra_hosts']) - self.composefile.set_container_names() - fm_version = importlib.metadata.version('frappe-manager') + try: + if "environment" in inputs.keys(): + environments: dict = inputs["environment"] + self.composefile.set_all_envs(environments) + + if "labels" in inputs.keys(): + labels: dict = inputs["labels"] + self.composefile.set_all_labels(labels) + + # handle user + if "user" in inputs.keys(): + user: dict = inputs["user"] + for container_name in user.keys(): + uid = user[container_name]["uid"] + gid = user[container_name]["gid"] + self.composefile.set_user(container_name, uid, gid) + + except Exception as e: + richprint.exit(f"Not able to generate site compose. Error: {e}") + + + self.composefile.set_network_alias('nginx','site-network',[self.name]) + self.composefile.set_container_names(get_container_name_prefix(self.name)) + fm_version = importlib.metadata.version("frappe-manager") self.composefile.set_version(fm_version) + self.set_site_network_name() self.composefile.write_to_file() + def set_site_network_name(self): + self.composefile.yml["networks"]["site-network"]["name"] = ( + self.name.replace(".", "") + f"-network" + ) + def create_dirs(self) -> bool: """ The function `create_dirs` creates two directories, `workspace` and `certs`, within a specified From 3494c0921a879aa37cd0bfe6aff2e78d097b3512 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 16:09:54 +0530 Subject: [PATCH 013/100] add func get_container_hex in utils --- frappe_manager/site_manager/utils.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/frappe_manager/site_manager/utils.py b/frappe_manager/site_manager/utils.py index 3a9cd5cd..e011a5c2 100644 --- a/frappe_manager/site_manager/utils.py +++ b/frappe_manager/site_manager/utils.py @@ -49,25 +49,23 @@ def check_ports(ports): return already_binded def log_file(file, refresh_time:float = 0.1, follow:bool =False): - '''generator function that yields new lines in a file ''' - # while True: - # line = file.readline() # sleep if file hasn't been updated - # yield line - # if not line: - # break - # seek the end of the file - # file.seek(0, os.SEEK_END) + Generator function that yields new lines in a file + ''' file.seek(0) # start infinite loop while True: # read last line of file - line = file.readline() # sleep if file hasn't been updated + line = file.readline() if not line: if not follow: break + # sleep if file hasn't been updated time.sleep(refresh_time) continue line = line.strip('\n') yield line + +def get_container_name_prefix(site_name): + return site_name.replace('.','') From 65a9b480735a3080c5c2a3baaa55c771df892173 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 16:13:58 +0530 Subject: [PATCH 014/100] chore: format --- frappe_manager/site_manager/site.py | 218 ++++++++++++++++------------ 1 file changed, 128 insertions(+), 90 deletions(-) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 179569b9..53cd6a7e 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -11,10 +11,11 @@ from frappe_manager.site_manager.Richprint import richprint from frappe_manager.site_manager.utils import log_file, get_container_name_prefix + class Site: - def __init__(self,path: Path , name:str, verbose: bool = False): - self.path= path - self.name= name + def __init__(self, path: Path, name: str, verbose: bool = False): + self.path = path + self.name = name self.quiet = not verbose self.init() @@ -43,9 +44,13 @@ def validate_sitename(self) -> bool: it returns False. """ sitename = self.name - match = re.search(r'^[A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])?.localhost$',sitename) + match = re.search( + r"^[A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])?.localhost$", sitename + ) if not match: - richprint.exit("The site name must follow a single-level subdomain Fully Qualified Domain Name (FQDN) format of localhost, such as 'subdomain.localhost'.") + richprint.exit( + "The site name must follow a single-level subdomain Fully Qualified Domain Name (FQDN) format of localhost, such as 'subdomain.localhost'." + ) def get_frappe_container_hex(self) -> None | str: """ @@ -55,7 +60,7 @@ def get_frappe_container_hex(self) -> None | str: Frappe container is found. """ container_name = self.composefile.get_container_names() - return container_name['frappe'].encode().hex() + return container_name["frappe"].encode().hex() def migrate_site_compose(self): """ @@ -80,7 +85,9 @@ def migrate_site_compose(self): # self.composefile.set_all_extrahosts(extrahosts) self.composefile.set_all_labels(labels) - self.composefile.set_container_names(get_container_name_prefix(self.name)) + self.composefile.set_container_names( + get_container_name_prefix(self.name) + ) self.set_site_network_name() self.composefile.write_to_file() status = True @@ -129,8 +136,7 @@ def generate_compose(self, inputs: dict) -> None: except Exception as e: richprint.exit(f"Not able to generate site compose. Error: {e}") - - self.composefile.set_network_alias('nginx','site-network',[self.name]) + self.composefile.set_network_alias("nginx", "site-network", [self.name]) self.composefile.set_container_names(get_container_name_prefix(self.name)) fm_version = importlib.metadata.version("frappe-manager") self.composefile.set_version(fm_version) @@ -150,21 +156,23 @@ def create_dirs(self) -> bool: # create site dir self.path.mkdir(parents=True, exist_ok=True) # create compose bind dirs -> workspace - workspace_path = self.path / 'workspace' + workspace_path = self.path / "workspace" workspace_path.mkdir(parents=True, exist_ok=True) - certs_path = self.path / 'certs' + certs_path = self.path / "certs" certs_path.mkdir(parents=True, exist_ok=True) def start(self) -> bool: """ The function starts Docker containers and prints the status of the operation. """ - status_text= 'Starting Docker Containers' + status_text = "Starting Docker Containers" richprint.change_head(status_text) try: - output = self.docker.compose.up(detach=True,pull='never',stream=self.quiet) + output = self.docker.compose.up( + detach=True, pull="never", stream=self.quiet + ) if self.quiet: - richprint.live_lines(output, padding=(0,0,0,2)) + richprint.live_lines(output, padding=(0, 0, 0, 2)) richprint.print(f"{status_text}: Done") except DockerException as e: richprint.exit(f"{status_text}: Failed") @@ -173,22 +181,22 @@ def pull(self): """ The function pulls Docker images and displays the status of the operation. """ - status_text= 'Pulling Docker Images' + status_text = "Pulling Docker Images" richprint.change_head(status_text) try: output = self.docker.compose.pull(stream=self.quiet) richprint.stdout.clear_live() if self.quiet: - richprint.live_lines(output, padding=(0,0,0,2)) + richprint.live_lines(output, padding=(0, 0, 0, 2)) richprint.print(f"{status_text}: Done") except DockerException as e: richprint.warning(f"{status_text}: Failed") - def logs(self,service:str, follow:bool=False): + def logs(self, service: str, follow: bool = False): """ The `logs` function prints the logs of a specified service, with the option to follow the logs in real-time. - + :param service: The "service" parameter is a string that specifies the name of the service whose logs you want to retrieve. It is used to filter the logs and only retrieve the logs for that specific service @@ -199,10 +207,12 @@ def logs(self,service:str, follow:bool=False): False :type follow: bool (optional) """ - output = self.docker.compose.logs(services=[service],no_log_prefix=True,follow=follow,stream=True) - for source , line in output: + output = self.docker.compose.logs( + services=[service], no_log_prefix=True, follow=follow, stream=True + ) + for source, line in output: line = line.decode() - if source == 'stdout': + if source == "stdout": if "[==".lower() in line.lower(): print(line) else: @@ -212,16 +222,24 @@ def frappe_logs_till_start(self): """ The function `frappe_logs_till_start` prints logs until a specific line is found and then stops. """ - status_text= 'Creating Site' + status_text = "Creating Site" richprint.change_head(status_text) try: - output = self.docker.compose.logs(services=['frappe'],no_log_prefix=True,follow=True,stream=True) + output = self.docker.compose.logs( + services=["frappe"], no_log_prefix=True, follow=True, stream=True + ) if self.quiet: - richprint.live_lines(output, padding=(0,0,0,2),stop_string="INFO spawned: 'bench-dev' with pid") + richprint.live_lines( + output, + padding=(0, 0, 0, 2), + stop_string="INFO spawned: 'bench-dev' with pid", + ) else: - for source , line in self.docker.compose.logs(services=['frappe'],no_log_prefix=True,follow=True,stream=True): - if not source == 'exit_code': + for source, line in self.docker.compose.logs( + services=["frappe"], no_log_prefix=True, follow=True, stream=True + ): + if not source == "exit_code": line = line.decode() if "[==".lower() in line.lower(): print(line) @@ -233,33 +251,37 @@ def frappe_logs_till_start(self): except DockerException as e: richprint.warning(f"{status_text}: Failed") - def stop(self) -> bool: """ The `stop` function stops containers and prints the status of the operation using the `richprint` module. """ - status_text= 'Stopping Containers' + status_text = "Stopping Containers" richprint.change_head(status_text) try: - output = self.docker.compose.stop(timeout=10,stream=self.quiet) + output = self.docker.compose.stop(timeout=10, stream=self.quiet) if self.quiet: - richprint.live_lines(output, padding=(0,0,0,2)) + richprint.live_lines(output, padding=(0, 0, 0, 2)) richprint.print(f"{status_text}: Done") except DockerException as e: richprint.exit(f"{status_text}: Failed") - def down(self,remove_ophans=True,volumes=True,timeout=5) -> bool: + def down(self, remove_ophans=True, volumes=True, timeout=5) -> bool: """ The `down` function removes containers using Docker Compose and prints the status of the operation. """ if self.composefile.exists(): - status_text='Removing Containers' + status_text = "Removing Containers" richprint.change_head(status_text) try: - output = self.docker.compose.down(remove_orphans=remove_ophans,volumes=volumes,timeout=timeout,stream=self.quiet) + output = self.docker.compose.down( + remove_orphans=remove_ophans, + volumes=volumes, + timeout=timeout, + stream=self.quiet, + ) if self.quiet: - exit_code = richprint.live_lines(output,padding=(0,0,0,2)) + exit_code = richprint.live_lines(output, padding=(0, 0, 0, 2)) richprint.print(f"Removing Containers: Done") except DockerException as e: richprint.exit(f"{status_text}: Failed") @@ -270,12 +292,14 @@ def remove(self) -> bool: """ # TODO handle low leverl error like read only, write only etc if self.composefile.exists(): - status_text = 'Removing Containers' + status_text = "Removing Containers" richprint.change_head(status_text) try: - output = self.docker.compose.down(remove_orphans=True,volumes=True,timeout=2,stream=self.quiet) + output = self.docker.compose.down( + remove_orphans=True, volumes=True, timeout=2, stream=self.quiet + ) if self.quiet: - exit_code = richprint.live_lines(output,padding=(0,0,0,2)) + exit_code = richprint.live_lines(output, padding=(0, 0, 0, 2)) richprint.print(f"Removing Containers: Done") except DockerException as e: richprint.exit(f"{status_text}: Failed") @@ -284,13 +308,13 @@ def remove(self) -> bool: shutil.rmtree(self.path) except Exception as e: richprint.error(e) - richprint.exit(f'Please remove {self.path} manually') + richprint.exit(f"Please remove {self.path} manually") richprint.change_head(f"Removing Dirs: Done") - def shell(self,container:str, user:str | None = None): + def shell(self, container: str, user: str | None = None): """ The `shell` function spawns a shell for a specified container and user. - + :param container: The `container` parameter is a string that specifies the name of the container in which the shell command will be executed :type container: str @@ -301,47 +325,58 @@ def shell(self,container:str, user:str | None = None): """ # TODO check user exists richprint.stop() - non_bash_supported = ['redis-cache','redis-cache','redis-socketio','redis-queue'] + non_bash_supported = [ + "redis-cache", + "redis-cache", + "redis-socketio", + "redis-queue", + ] try: if not container in non_bash_supported: - if container == 'frappe': - shell_path = '/usr/bin/zsh' + if container == "frappe": + shell_path = "/usr/bin/zsh" else: - shell_path = '/bin/bash' + shell_path = "/bin/bash" if user: - self.docker.compose.exec(container,user=user,command=shell_path) + self.docker.compose.exec(container, user=user, command=shell_path) else: - self.docker.compose.exec(container,command=shell_path) + self.docker.compose.exec(container, command=shell_path) else: if user: - self.docker.compose.exec(container,user=user,command='sh') + self.docker.compose.exec(container, user=user, command="sh") else: - self.docker.compose.exec(container,command='sh') + self.docker.compose.exec(container, command="sh") except DockerException as e: - richprint.warning(f"Shell exited with error code: {e.return_code}") + richprint.warning(f"Shell exited with error code: {e.return_code}") def get_site_installed_apps(self): """ The function executes a command to list the installed apps for a specific site and prints the output. """ - command = f'/opt/.pyenv/shims/bench --site {self.name} list-apps' + command = f"/opt/.pyenv/shims/bench --site {self.name} list-apps" # command = f'which bench' - output = self.docker.compose.exec('frappe',user='frappe',workdir='/workspace/frappe-bench',command=command,stream=True) - for source,line in output: + output = self.docker.compose.exec( + "frappe", + user="frappe", + workdir="/workspace/frappe-bench", + command=command, + stream=True, + ) + for source, line in output: line = line.decode() pass - def bench_dev_server_logs(self, follow = False): + def bench_dev_server_logs(self, follow=False): """ This function is used to tail logs found at /workspace/logs/bench-start.log. :param follow: Bool detemines whether to follow the log file for changes """ - bench_start_log_path = self.path / 'workspace' / 'logs' / 'bench-start.log' + bench_start_log_path = self.path / "workspace" / "logs" / "bench-start.log" if bench_start_log_path.exists() and bench_start_log_path.is_file(): - with open(bench_start_log_path,'r') as bench_start_log: - bench_start_log_data = log_file(bench_start_log,follow=follow) + with open(bench_start_log_path, "r") as bench_start_log: + bench_start_log_data = log_file(bench_start_log, follow=follow) try: for line in bench_start_log_data: richprint.stdout.print(line) @@ -353,6 +388,7 @@ def bench_dev_server_logs(self, follow = False): def is_site_created(self, retry=30, interval=1) -> bool: import requests from time import sleep + i = 0 while i < retry: try: @@ -368,44 +404,46 @@ def is_site_created(self, retry=30, interval=1) -> bool: return False def running(self) -> bool: - """ - The `running` function checks if all the services defined in a Docker Compose file are running. - :return: a boolean value. If the number of running containers is greater than or equal to the number - of services listed in the compose file, it returns True. Otherwise, it returns False. - """ - # try: - # output = self.docker.compose.ps(format='json',filter='running',stream=True) - # status: dict = {} - # for source,line in output: - # if source == 'stdout': - # status = json.loads(line.decode()) - # running_containers = len(status) - # if running_containers >= len(self.composefile.get_services_list()): - # return True - # return False - # except DockerException as e: - # richprint.exit(f"{e.stdout}{e.stderr}") - - services = self.composefile.get_services_list() - running_status = self.get_services_running_status() - - if running_status: - for service in services: - try: - if not running_status[service] == "running": - return False - except KeyError: + """ + The `running` function checks if all the services defined in a Docker Compose file are running. + :return: a boolean value. If the number of running containers is greater than or equal to the number + of services listed in the compose file, it returns True. Otherwise, it returns False. + """ + # try: + # output = self.docker.compose.ps(format='json',filter='running',stream=True) + # status: dict = {} + # for source,line in output: + # if source == 'stdout': + # status = json.loads(line.decode()) + # running_containers = len(status) + # if running_containers >= len(self.composefile.get_services_list()): + # return True + # return False + # except DockerException as e: + # richprint.exit(f"{e.stdout}{e.stderr}") + + services = self.composefile.get_services_list() + running_status = self.get_services_running_status() + + if running_status: + for service in services: + try: + if not running_status[service] == "running": return False - else: - return False - return True + except KeyError: + return False + else: + return False + return True - def get_services_running_status(self)-> dict: + def get_services_running_status(self) -> dict: services = self.composefile.get_services_list() containers = self.composefile.get_container_names().values() services_status = {} try: - output = self.docker.compose.ps(service=services,format="json",all=True,stream=True) + output = self.docker.compose.ps( + service=services, format="json", all=True, stream=True + ) status: dict = {} for source, line in output: if source == "stdout": @@ -413,8 +451,8 @@ def get_services_running_status(self)-> dict: # this is done to exclude docker runs using docker compose run command for container in status: - if container['Name'] in containers: - services_status[container['Service']] = container['State'] + if container["Name"] in containers: + services_status[container["Service"]] = container["State"] return services_status except DockerException as e: richprint.exit(f"{e.stdout}{e.stderr}") From 06be7bc4c653ce79734bbf5fd60ffd17cbdcc2aa Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 16:21:56 +0530 Subject: [PATCH 015/100] move check_ports functions --- frappe_manager/site_manager/utils.py | 45 ------------------ frappe_manager/utils.py | 68 ++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 45 deletions(-) diff --git a/frappe_manager/site_manager/utils.py b/frappe_manager/site_manager/utils.py index e011a5c2..77a2e8aa 100644 --- a/frappe_manager/site_manager/utils.py +++ b/frappe_manager/site_manager/utils.py @@ -3,51 +3,6 @@ import subprocess import os -def is_port_in_use(port): - """ - Check if port is in use or not. - - :param port: The port which will be checked if it's in use or not. - :return: Bool In use then True and False when not in use. - """ - import psutil - for conn in psutil.net_connections(): - if conn.laddr.port == port and conn.status == 'LISTEN': - return True - return False - -def check_ports(ports): - """ - This function checks if the ports is in use. - :param ports: list of ports to be checked - returns: list of binded port(can be empty) - """ - - # TODO handle if ports are open using docker - - current_system = platform.system() - - already_binded = [] - - for port in ports: - if current_system == 'Darwin': - - # Mac Os - # check port using lsof command - cmd = f"lsof -iTCP:{port} -sTCP:LISTEN -P -n" - try: - output = subprocess.run(cmd,check=True,shell=True,capture_output=True) - if output.returncode == 0: - already_binded.append(port) - except subprocess.CalledProcessError as e: - pass - else: - # Linux or any other machines - if is_port_in_use(port): - already_binded.append(port) - - return already_binded - def log_file(file, refresh_time:float = 0.1, follow:bool =False): ''' Generator function that yields new lines in a file diff --git a/frappe_manager/utils.py b/frappe_manager/utils.py index 02343b6b..05acfa29 100644 --- a/frappe_manager/utils.py +++ b/frappe_manager/utils.py @@ -1,6 +1,10 @@ import importlib +import sys import requests import json +import subprocess +import platform + from frappe_manager.logger import log from frappe_manager.docker_wrapper.utils import process_opened from frappe_manager.site_manager.Richprint import richprint @@ -41,3 +45,67 @@ def check_update(): ) except Exception as e: pass + +def is_port_in_use(port): + """ + Check if port is in use or not. + + :param port: The port which will be checked if it's in use or not. + :return: Bool In use then True and False when not in use. + """ + import psutil + + for conn in psutil.net_connections(): + if conn.laddr.port == port and conn.status == "LISTEN": + return True + return False + + +def check_ports(ports): + """ + This function checks if the ports is in use. + :param ports: list of ports to be checked + returns: list of binded port(can be empty) + """ + + # TODO handle if ports are open using docker + + current_system = platform.system() + already_binded = [] + for port in ports: + if current_system == "Darwin": + # Mac Os + # check port using lsof command + cmd = f"lsof -iTCP:{port} -sTCP:LISTEN -P -n" + try: + output = subprocess.run( + cmd, check=True, shell=True, capture_output=True + ) + if output.returncode == 0: + already_binded.append(port) + except subprocess.CalledProcessError as e: + pass + else: + # Linux or any other machines + if is_port_in_use(port): + already_binded.append(port) + + return already_binded + + +def check_ports_with_msg(ports_to_check: list, exclude=[]): + """ + The `check_ports` function checks if certain ports are already bound by another process using the + `lsof` command. + """ + richprint.change_head("Checking Ports") + if exclude: + # Removing elements present in remove_array from original_array + ports_to_check = [x for x in exclude if x not in ports_to_check] + if ports_to_check: + already_binded = check_ports(ports_to_check) + if already_binded: + richprint.exit( + f"Whoa there! Looks like the {' '.join([ str(x) for x in already_binded ])} { 'ports are' if len(already_binded) > 1 else 'port is' } having a party already! Can you do us a solid and free up those ports?" + ) + richprint.print("Ports Check : Passed") From 2c0305332ec0a5018a4b3361d20fc03a709509e7 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 16:28:00 +0530 Subject: [PATCH 016/100] optimize ports checking --- frappe_manager/site_manager/manager.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/frappe_manager/site_manager/manager.py b/frappe_manager/site_manager/manager.py index bef7adff..9a5d2f7b 100644 --- a/frappe_manager/site_manager/manager.py +++ b/frappe_manager/site_manager/manager.py @@ -8,8 +8,12 @@ import shutil from frappe_manager.site_manager.site import Site -from frappe_manager.site_manager.utils import check_ports from frappe_manager.site_manager.Richprint import richprint +from frappe_manager import CLI_DIR + +from frappe_manager.utils import ( + check_ports_with_msg, +) from rich.columns import Columns from rich.panel import Panel @@ -323,34 +327,26 @@ def logs(self,service:str,follow): richprint.change_head(f"Showing logs") if self.site.running(): - if service: - self.site.logs(service,follow) + self.site.logs(service, follow) else: self.site.bench_dev_server_logs(follow) else: - richprint.error( - f"Site {self.site.name} not running!" - ) + richprint.error(f"Site {self.site.name} not running!") def check_ports(self): """ The `check_ports` function checks if certain ports are already bound by another process using the `lsof` command. """ - richprint.change_head("Checking Ports") - to_check = [9000,80,443] - already_binded = check_ports(to_check) - if already_binded: - richprint.exit(f"Whoa there! Looks like the {' '.join([ str(x) for x in already_binded ])} { 'ports are' if len(already_binded) > 1 else 'port is' } having a party already! Can you do us a solid and free up those ports?") - richprint.print("Ports Check : Passed") + check_ports_with_msg([80, 443], exclude=self.site.get_host_port_binds()) - def shell(self,container:str, user:str | None): + def shell(self, container: str, user: str | None): """ The `shell` function checks if a site exists and is running, and then executes a shell command on the specified container with the specified user. - + :param container: The "container" parameter is a string that specifies the name of the container. :type container: str :param user: The `user` parameter in the `shell` method is an optional parameter that specifies the From cd3fb66f7dd8dd89b33bf56b9c0792b0c73e1563 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 16:31:28 +0530 Subject: [PATCH 017/100] chore: format code --- frappe_manager/site_manager/manager.py | 144 ++++++++++++++----------- 1 file changed, 81 insertions(+), 63 deletions(-) diff --git a/frappe_manager/site_manager/manager.py b/frappe_manager/site_manager/manager.py index 9a5d2f7b..ffb70135 100644 --- a/frappe_manager/site_manager/manager.py +++ b/frappe_manager/site_manager/manager.py @@ -29,11 +29,11 @@ def __init__(self, sitesdir: Path): self.verbose = False self.typer_context: Optional[typer.Context] = None - def init(self, sitename: str| None = None): + def init(self, sitename: str | None = None): """ The `init` function initializes a site by checking if the site directory exists, creating it if necessary, and setting the site name and path. - + :param sitename: The `sitename` parameter is a string that represents the name of the site. It is optional and can be set to `None`. If a value is provided, it will be used to create a site path by appending ".localhost" to the sitename @@ -41,13 +41,16 @@ def init(self, sitename: str| None = None): """ if sitename: - if not '.localhost' in sitename: + if not ".localhost" in sitename: sitename = sitename + ".localhost" sitepath: Path = self.sitesdir / sitename - site_directory_exits_check_for_commands = ['create'] + site_directory_exits_check_for_commands = ["create"] - if self.typer_context.invoked_subcommand in site_directory_exits_check_for_commands: + if ( + self.typer_context.invoked_subcommand + in site_directory_exits_check_for_commands + ): if sitepath.exists(): richprint.exit( f"The site '{sitename}' already exists at {sitepath}. Aborting operation." @@ -58,7 +61,7 @@ def init(self, sitename: str| None = None): f"The site '{sitename}' does not exist. Aborting operation." ) - self.site: Site = Site(sitepath, sitename, verbose= self.verbose) + self.site: Site = Site(sitepath, sitename, verbose=self.verbose) def set_verbose(self): """ @@ -66,7 +69,7 @@ def set_verbose(self): """ self.verbose = True - def set_typer_context(self,ctx: typer.Context): + def set_typer_context(self, ctx: typer.Context): """ The function sets the typer context from the :param typer context @@ -78,7 +81,7 @@ def __get_all_sites_path(self, exclude: List[str] = []): """ The function `__get_all_sites_path` returns a list of paths to all the `docker-compose.yml` files in the `sitesdir` directory, excluding any directories specified in the `exclude` list. - + :param exclude: The `exclude` parameter is a list of strings that contains the names of directories to be excluded from the list of sites paths :type exclude: List[str] @@ -116,7 +119,7 @@ def stop_sites(self): The `stop_sites` function stops all sites except the current site by halting their Docker containers. """ - status_text='Halting other sites' + status_text = "Halting other sites" richprint.change_head(status_text) if self.site: exclude = [self.site.name] @@ -127,9 +130,9 @@ def stop_sites(self): for site_compose_path in site_compose: docker = DockerClient(compose_file_path=site_compose_path) try: - output = docker.compose.stop(timeout=10,stream=not self.verbose) + output = docker.compose.stop(timeout=10, stream=not self.verbose) if not self.verbose: - richprint.live_lines(output, padding=(0,0,0,2)) + richprint.live_lines(output, padding=(0, 0, 0, 2)) except DockerException as e: richprint.exit(f"{status_text}: Failed") richprint.print(f"{status_text}: Done") @@ -231,7 +234,7 @@ def stop_site(self): indicating that the site has been stopped. """ richprint.change_head(f"Stopping site") - #self.stop_sites() + # self.stop_sites() self.site.stop() richprint.print(f"Stopped site") @@ -253,7 +256,7 @@ def attach_to_site(self, user: str, extensions: List[str]): """ The `attach_to_site` function attaches to a running site and opens it in Visual Studio Code with specified extensions. - + :param user: The `user` parameter is a string that represents the username of the user who wants to attach to the site :type user: str @@ -263,7 +266,7 @@ def attach_to_site(self, user: str, extensions: List[str]): """ if self.site.running(): # check if vscode is installed - vscode_path= shutil.which('code') + vscode_path = shutil.which("code") if not vscode_path: richprint.exit("vscode(excutable code) not accessible via cli.") @@ -277,21 +280,27 @@ def attach_to_site(self, user: str, extensions: List[str]): ) extensions.sort() labels = { - "devcontainer.metadata": json.dumps([ - { - "remoteUser": user, - "customizations": {"vscode": {"extensions": extensions}}, - } - ]) + "devcontainer.metadata": json.dumps( + [ + { + "remoteUser": user, + "customizations": {"vscode": {"extensions": extensions}}, + } + ] + ) } - labels_previous = self.site.composefile.get_labels('frappe') + labels_previous = self.site.composefile.get_labels("frappe") # check if the extension are the same if they are different then only update # check if customizations key available try: - extensions_previous = json.loads(labels_previous['devcontainer.metadata']) - extensions_previous = extensions_previous[0]['customizations']['vscode']['extensions'] + extensions_previous = json.loads( + labels_previous["devcontainer.metadata"] + ) + extensions_previous = extensions_previous[0]["customizations"][ + "vscode" + ]["extensions"] except KeyError: extensions_previous = [] @@ -299,24 +308,24 @@ def attach_to_site(self, user: str, extensions: List[str]): if not extensions_previous == extensions: richprint.print(f"Extensions are changed, Recreating containers..") - self.site.composefile.set_labels('frappe',labels) + self.site.composefile.set_labels("frappe", labels) self.site.composefile.write_to_file() self.site.start() richprint.print(f"Recreating Containers : Done") # TODO check if vscode exists richprint.change_head("Attaching to Container") - output = subprocess.run(vscode_cmd,shell=True) + output = subprocess.run(vscode_cmd, shell=True) if output.returncode != 0: richprint.exit(f"Attaching to Container : Failed") richprint.print(f"Attaching to Container : Done") else: richprint.print(f"Site: {self.site.name} is not running") - def logs(self,service:str,follow): + def logs(self, follow, service: Optional[str] = None): """ The `logs` function checks if a site exists, and if it does, it shows the logs for a specific service. If the site is not running, it displays an error message. - + :param service: The `service` parameter is a string that represents the specific service or component for which you want to view the logs. It could be the name of a specific container :type service: str @@ -356,14 +365,12 @@ def shell(self, container: str, user: str | None): """ richprint.change_head(f"Spawning shell") if self.site.running(): - if container == 'frappe': + if container == "frappe": if not user: - user = 'frappe' - self.site.shell(container,user) + user = "frappe" + self.site.shell(container, user) else: - richprint.exit( - f"Site {self.site.name} not running!" - ) + richprint.exit(f"Site {self.site.name} not running!") def info(self): """ @@ -371,54 +378,65 @@ def info(self): details, Frappe username and password, and a list of installed apps. """ richprint.change_head(f"Getting site info") - site_config_file = self.site.path / 'workspace' / 'frappe-bench' / 'sites' / self.site.name / 'site_config.json' + site_config_file = ( + self.site.path + / "workspace" + / "frappe-bench" + / "sites" + / self.site.name + / "site_config.json" + ) db_user = None db_pass = None if site_config_file.exists(): - with open(site_config_file,'r') as f: + with open(site_config_file, "r") as f: site_config = json.load(f) - db_user = site_config['db_name'] - db_pass= site_config['db_password'] - - frappe_password = self.site.composefile.get_envs('frappe')['ADMIN_PASS'] - root_db_password = self.site.composefile.get_envs('mariadb')['MYSQL_ROOT_PASSWORD'] - site_info_table = Table(box=box.ASCII2,show_lines=True,show_header=False,highlight=True) + db_user = site_config["db_name"] + db_pass = site_config["db_password"] + + frappe_password = self.site.composefile.get_envs("frappe")["ADMIN_PASS"] + root_db_password = self.site.composefile.get_envs("mariadb")[ + "MYSQL_ROOT_PASSWORD" + ] + site_info_table = Table(show_lines=True, show_header=False, highlight=True) data = { - "Site Url":f"http://{self.site.name}", - "Site Root":f"{self.site.path.absolute()}", - "Mailhog Url":f"http://{self.site.name}/mailhog", - "Adminer Url":f"http://{self.site.name}/adminer", - "Frappe Username" : "administrator", - "Frappe Password" : frappe_password, - "Root DB User" : 'root', - "Root DB Password" : root_db_password, - "DB Host" : "mariadb", - "DB Name" : db_user, - "DB User" : db_user, - "DB Password" : db_pass, - - } + "Site Url": f"http://{self.site.name}", + "Site Root": f"{self.site.path.absolute()}", + "Mailhog Url": f"http://{self.site.name}/mailhog", + "Adminer Url": f"http://{self.site.name}/adminer", + "Frappe Username": "administrator", + "Frappe Password": frappe_password, + "Root DB User": "root", + "Root DB Password": root_db_password, + "DB Host": "mariadb", + "DB Name": db_user, + "DB User": db_user, + "DB Password": db_pass, + } site_info_table.add_column() site_info_table.add_column() for key in data.keys(): - site_info_table.add_row(key,data[key]) + site_info_table.add_row(key, data[key]) # bench apps list - richprint.stdout.print('') + richprint.stdout.print("") # bench_apps_list_table=Table(title="Bench Apps",box=box.ASCII2,show_lines=True) - bench_apps_list_table=Table(box=box.ASCII2,show_lines=True,expand=True,show_edge=False,pad_edge=False) + bench_apps_list_table = Table( + show_lines=True, expand=True, show_edge=False, pad_edge=False + ) bench_apps_list_table.add_column("App") bench_apps_list_table.add_column("Version") - - apps_json_file = self.site.path / 'workspace' / 'frappe-bench' / 'sites' / 'apps.json' + apps_json_file = ( + self.site.path / "workspace" / "frappe-bench" / "sites" / "apps.json" + ) if apps_json_file.exists(): - with open(apps_json_file,'r') as f: + with open(apps_json_file, "r") as f: apps_json = json.load(f) for app in apps_json.keys(): - bench_apps_list_table.add_row(app,apps_json[app]['version']) + bench_apps_list_table.add_row(app, apps_json[app]["version"]) - site_info_table.add_row('Bench Apps',bench_apps_list_table) + site_info_table.add_row("Bench Apps", bench_apps_list_table) richprint.stdout.print(site_info_table) def migrate_site(self): From 9abc7a020319673fb11cca49fab1e62615cfc014 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 16:43:32 +0530 Subject: [PATCH 018/100] Optimize shell and logs commands --- frappe_manager/main.py | 4 ++-- frappe_manager/site_manager/manager.py | 27 +++++++++++++------------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/frappe_manager/main.py b/frappe_manager/main.py index a491277e..3b678732 100644 --- a/frappe_manager/main.py +++ b/frappe_manager/main.py @@ -331,9 +331,9 @@ def shell( """Open shell for the give site. """ sites.init(sitename) if service: - sites.shell(SiteServicesEnum(service).value,follow) + sites.shell(SiteServicesEnum(service).value,user=user) else: - sites.shell(follow) + sites.shell(user=user) @app.command(no_args_is_help=True) def info( diff --git a/frappe_manager/site_manager/manager.py b/frappe_manager/site_manager/manager.py index ffb70135..d9fefd05 100644 --- a/frappe_manager/site_manager/manager.py +++ b/frappe_manager/site_manager/manager.py @@ -335,13 +335,11 @@ def logs(self, follow, service: Optional[str] = None): """ richprint.change_head(f"Showing logs") - if self.site.running(): - if service: + if service: + if self.site.is_service_running(service): self.site.logs(service, follow) - else: - self.site.bench_dev_server_logs(follow) else: - richprint.error(f"Site {self.site.name} not running!") + self.site.bench_dev_server_logs(follow) def check_ports(self): """ @@ -351,26 +349,27 @@ def check_ports(self): check_ports_with_msg([80, 443], exclude=self.site.get_host_port_binds()) - def shell(self, container: str, user: str | None): + def shell(self, service: str, user: str | None): """ The `shell` function checks if a site exists and is running, and then executes a shell command on the specified container with the specified user. - :param container: The "container" parameter is a string that specifies the name of the container. - :type container: str + :param service: The "container" parameter is a string that specifies the name of the container. + :type service: str :param user: The `user` parameter in the `shell` method is an optional parameter that specifies the user for which the shell command should be executed. If no user is provided, the default user is set to 'frappe' :type user: str | None """ richprint.change_head(f"Spawning shell") - if self.site.running(): - if container == "frappe": - if not user: - user = "frappe" - self.site.shell(container, user) + + if service == "frappe": + if not user: + user = "frappe" + if self.site.is_service_running(service): + self.site.shell(service, user) else: - richprint.exit(f"Site {self.site.name} not running!") + richprint.exit(f"Cannot spawn shell. [blue]{self.site.name}[/blue]'s compose service '{service}' not running!") def info(self): """ From bc30d1bbcf69dfd8e04b71e5d3db24460e41ff8d Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 17:11:54 +0530 Subject: [PATCH 019/100] add docker run,rm and cp command funcetions --- frappe_manager/docker_wrapper/DockerClient.py | 126 ++++++++++++++++-- 1 file changed, 117 insertions(+), 9 deletions(-) diff --git a/frappe_manager/docker_wrapper/DockerClient.py b/frappe_manager/docker_wrapper/DockerClient.py index 149ef1c5..fb14c86e 100644 --- a/frappe_manager/docker_wrapper/DockerClient.py +++ b/frappe_manager/docker_wrapper/DockerClient.py @@ -9,9 +9,10 @@ run_command_with_exit_code, ) + class DockerClient: def __init__(self, compose_file_path: Optional[Path] = None): - self.docker_cmd= ['docker'] + self.docker_cmd = ["docker"] if compose_file_path: self.compose = DockerComposeWrapper(compose_file_path) @@ -23,20 +24,18 @@ def version(self) -> dict: """ parameters: dict = locals() - parameters['format'] = 'json' + parameters["format"] = "json" ver_cmd: list = ["version"] ver_cmd += parameters_to_options(parameters) - iterator = run_command_with_exit_code( - self.docker_cmd + ver_cmd, quiet=False - ) + iterator = run_command_with_exit_code(self.docker_cmd + ver_cmd, quiet=False) output: dict = {} try: - for source ,line in iterator: - if source == 'stdout': + for source, line in iterator: + if source == "stdout": output = json.loads(line.decode()) except Exception as e: return {} @@ -50,9 +49,118 @@ def server_running(self) -> bool: function returns True. Otherwise, it returns False. """ docker_info = self.version() - if 'Server' in docker_info: + if "Server" in docker_info: return True else: # check if the current user in the docker group and notify the user - is_current_user_in_group('docker') + is_current_user_in_group("docker") + return False + + def cp( + self, + source: str, + destination: str, + source_container: str = None, + destination_container: str = None, + archive: bool = False, + follow_link: bool = False, + quiet: bool = False, + stream: bool = False, + stream_only_exit_code: bool = False, + ): + parameters: dict = locals() + cp_cmd: list = ["cp"] + + remove_parameters = [ + "stream", + "stream_only_exit_code", + "source", + "destination", + "source_container", + "destination_container", + ] + + cp_cmd += parameters_to_options(parameters, exclude=remove_parameters) + + if source_container: + source = f"{source_container}:{source}" + + if destination_container: + destination = f"{destination_container}:{destination}" + + cp_cmd += [f"{source}"] + cp_cmd += [f"{destination}"] + + iterator = run_command_with_exit_code( + self.docker_cmd + cp_cmd, quiet=stream_only_exit_code, stream=stream + ) + return iterator + + def kill( + self, + container: str, + signal: Optional[str] = None, + stream: bool = False, + stream_only_exit_code: bool = False, + ): + parameters: dict = locals() + kill_cmd: list = ["kill"] + + remove_parameters = ["stream", "stream_only_exit_code", "container"] + + kill_cmd += parameters_to_options(parameters, exclude=remove_parameters) + kill_cmd += [f"{container}"] + + iterator = run_command_with_exit_code( + self.docker_cmd + kill_cmd, quiet=stream_only_exit_code, stream=stream + ) + return iterator + + def rm( + self, + container: str, + force: bool = False, + link: bool = False, + volumes: bool = False, + stream: bool = False, + stream_only_exit_code: bool = False, + ): + parameters: dict = locals() + rm_cmd: list = ["rm"] + + remove_parameters = ["stream", "stream_only_exit_code", "container"] + + rm_cmd += parameters_to_options(parameters, exclude=remove_parameters) + rm_cmd += [f"{container}"] + + iterator = run_command_with_exit_code( + self.docker_cmd + rm_cmd, quiet=stream_only_exit_code, stream=stream + ) + return iterator + + def run( + self, + command: str, + image: str, + name: Optional[str] = None, + detach: bool = False, + entrypoint: Optional[str] = None, + stream: bool = False, + stream_only_exit_code: bool = False, + ): + parameters: dict = locals() + run_cmd: list = ["run"] + + remove_parameters = ["stream", "stream_only_exit_code", "image", "command"] + + run_cmd += parameters_to_options(parameters, exclude=remove_parameters) + run_cmd += [f"{image}"] + + if command: + run_cmd += [f"{command}"] + + iterator = run_command_with_exit_code( + self.docker_cmd + run_cmd, quiet=stream_only_exit_code, stream=stream + ) + return iterator From 5b9153ca69ed3dbdae66cd624e941518d1fb5b93 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 17:14:08 +0530 Subject: [PATCH 020/100] fix: docker compose up,restart and exec commands --- frappe_manager/docker_wrapper/DockerCompose.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/frappe_manager/docker_wrapper/DockerCompose.py b/frappe_manager/docker_wrapper/DockerCompose.py index 732fb00e..fb7ab6dc 100644 --- a/frappe_manager/docker_wrapper/DockerCompose.py +++ b/frappe_manager/docker_wrapper/DockerCompose.py @@ -36,12 +36,12 @@ def __init__(self, path: Path, timeout: int = 100): def up( self, + services: list[str] = [], detach: bool = True, build: bool = False, remove_orphans: bool = False, no_recreate: bool = False, always_recreate_deps: bool = False, - services: list[str] = [], quiet_pull: bool = False, pull: Literal["missing", "never", "always"] = "missing", stream: bool = False, @@ -86,7 +86,7 @@ def up( """ parameters: dict = locals() - remove_parameters = ["stream", "stream_only_exit_code"] + remove_parameters = ["services","stream", "stream_only_exit_code"] up_cmd: list = ["up"] up_cmd += services @@ -215,7 +215,7 @@ def restart( restart_cmd: list[str] = ["restart"] - remove_parameters = ["service", "stream", "stream_only_exit_code"] + remove_parameters = ["services", "stream", "stream_only_exit_code"] restart_cmd += parameters_to_options(parameters, exclude=remove_parameters) @@ -286,11 +286,12 @@ def exec( workdir: Union[None, str] = None, stream: bool = False, stream_only_exit_code: bool = False, + use_shlex_split: bool = True, ): """ The `exec` function in Python executes a command in a Docker container and returns an iterator for the command's output. - + :param service: The `service` parameter is a string that represents the name of the service you want to execute the command on :type service: str @@ -348,7 +349,10 @@ def exec( exec_cmd += [service] - exec_cmd += shlex.split(command) + if use_shlex_split: + exec_cmd += shlex.split(command, posix=True) + else: + exec_cmd += command iterator = run_command_with_exit_code( self.docker_compose_cmd + exec_cmd, From b409d42a8f32ea135d6b34291f0cc1df5ec7cae7 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 21:31:22 +0530 Subject: [PATCH 021/100] supervisor.conf for running different services --- Docker/frappe/Dockerfile | 12 +++- Docker/frappe/bench-dev-server | 3 + Docker/frappe/bench-start.sh | 4 -- Docker/frappe/divide-supervisor-conf.py | 41 +++++++++++++ Docker/frappe/entrypoint.sh | 12 +++- Docker/frappe/frappe-dev.conf | 22 +++++++ Docker/frappe/launch.sh | 61 ++++++++++++++++++ Docker/frappe/supervisord.conf | 24 ++------ Docker/frappe/user-script.sh | 82 +++++++++++++++++-------- 9 files changed, 208 insertions(+), 53 deletions(-) create mode 100644 Docker/frappe/bench-dev-server delete mode 100644 Docker/frappe/bench-start.sh create mode 100755 Docker/frappe/divide-supervisor-conf.py create mode 100644 Docker/frappe/frappe-dev.conf create mode 100644 Docker/frappe/launch.sh diff --git a/Docker/frappe/Dockerfile b/Docker/frappe/Dockerfile index 1818eee1..2953e96c 100644 --- a/Docker/frappe/Dockerfile +++ b/Docker/frappe/Dockerfile @@ -148,11 +148,17 @@ ENV PATH /opt/user/.bin:${PATH} RUN echo PATH='/opt/user/.bin:$PATH' >> "$USERZSHRC" COPY ./supervisord.conf /opt/user/ -COPY ./bench-start.sh /opt/user/ +COPY ./bench-dev-server /opt/user/ +COPY ./frappe-dev.conf /opt/user/ COPY ./bench-wrapper.sh /opt/user/.bin/bench + COPY ./entrypoint.sh / -COPY ./user-script.sh / +COPY ./user-script.sh /scripts/ +COPY ./launch.sh /scripts/ +COPY ./divide-supervisor-conf.py /scripts/ + +RUN mkdir -p /scripts -RUN sudo chmod +x /entrypoint.sh /user-script.sh /opt/user/bench-start.sh /opt/user/.bin/bench +RUN sudo chmod +x /entrypoint.sh /scripts/user-script.sh /scripts/launch.sh /scripts/divide-supervisor-conf.py /opt/user/bench-dev-server /opt/user/.bin/bench ENTRYPOINT ["/bin/bash","/entrypoint.sh"] diff --git a/Docker/frappe/bench-dev-server b/Docker/frappe/bench-dev-server new file mode 100644 index 00000000..6c7904e7 --- /dev/null +++ b/Docker/frappe/bench-dev-server @@ -0,0 +1,3 @@ +#!/bin/bash +fuser -k 80/tcp +bench serve --port 80 diff --git a/Docker/frappe/bench-start.sh b/Docker/frappe/bench-start.sh deleted file mode 100644 index 90e476e9..00000000 --- a/Docker/frappe/bench-start.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -fuser -k 80/tcp -fuser -k 9000/tcp -bench start --procfile /workspace/frappe-bench/Procfile.local_setup diff --git a/Docker/frappe/divide-supervisor-conf.py b/Docker/frappe/divide-supervisor-conf.py new file mode 100755 index 00000000..87988141 --- /dev/null +++ b/Docker/frappe/divide-supervisor-conf.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +import sys +import configparser +from pathlib import Path + +print(sys.argv) +conf_file_path= Path(sys.argv[1]) +conf_file_absolute_path = conf_file_path.absolute() + +if not len(sys.argv) == 2: + print(f"Generates individual program conf from supervisor.conf.\nUSAGE: {sys.argv[0]} SUPERVISOR_CONF_PATH\n\n SUPERVISOR_CONF_PATH -> Absolute path to supervisor.conf") + sys.exit(0) + +config = configparser.ConfigParser(allow_no_value=True,strict=False,interpolation=None) + +superconf = open(conf_file_absolute_path,'r+') + +config.read_file(superconf) + +print(f"Divided {conf_file_absolute_path} into ") + +for section_name in config.sections(): + if not 'group:' in section_name: + + section_config = configparser.ConfigParser(interpolation=None) + section_config.add_section(section_name) + for key, value in config.items(section_name): + if 'frappe-bench-frappe-web' in section_name: + if key == 'command': + value = value.replace("127.0.0.1:80","0.0.0.0:80") + section_config.set(section_name, key, value) + + if 'worker' in section_name: + file_name = f"{section_name.replace('program:','')}.fm.workers.supervisor.conf" + else: + file_name = f"{section_name.replace('program:','')}.fm.supervisor.conf" + + with open(conf_file_path.parent / file_name, 'w') as section_file: + section_config.write(section_file) + print(f" - {section_name} => {file_name}") diff --git a/Docker/frappe/entrypoint.sh b/Docker/frappe/entrypoint.sh index 63328bf4..072e80cb 100755 --- a/Docker/frappe/entrypoint.sh +++ b/Docker/frappe/entrypoint.sh @@ -1,4 +1,5 @@ #!/bin/bash + emer() { echo "$1" exit 1 @@ -15,17 +16,26 @@ useradd --no-log-init -r -m -u "$USERID" -g "$USERGROUP" -G sudo -s /usr/bin/zsh usermod -a -G tty "$NAME" echo "$NAME ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers +mkdir -p /opt/user/conf.d + chown -R "$USERID":"$USERGROUP" /opt if [[ ! -d "/workspace/.oh-my-zsh" ]]; then cp -r /opt/user/.oh-my-zsh /workspace/.oh-my-zsh fi + if [[ ! -f "/workspace/.zshrc" ]]; then cat /opt/user/.zshrc > /workspace/.zshrc fi + if [[ ! -f "/workspace/.profile" ]]; then cat /opt/user/.profile > /workspace/.profile fi chown -R "$USERID":"$USERGROUP" /workspace -gosu "${USERID}":"${USERGROUP}" /user-script.sh + +if [ "$#" -gt 0 ]; then + gosu "$USERID":"$USERGROUP" "/scripts/$@" +else + gosu "${USERID}":"${USERGROUP}" /scripts/user-script.sh +fi diff --git a/Docker/frappe/frappe-dev.conf b/Docker/frappe/frappe-dev.conf new file mode 100644 index 00000000..7bcc92f6 --- /dev/null +++ b/Docker/frappe/frappe-dev.conf @@ -0,0 +1,22 @@ +[program:frappe-bench-frappe-dev] +command=/opt/user/bench-dev-server.sh +priority=4 +autostart=true +autorestart=false +stdout_logfile=/workspace/frappe-bench/logs/web.dev.log +redirect_stderr=true +user=frappe +directory=/workspace/frappe-bench + +[program:frappe-bench-frappe-watch] +command=bench watch +priority=4 +autostart=true +autorestart=false +stdout_logfile=/workspace/frappe-bench/logs/watch.dev.log +redirect_stderr=true +user=frappe +directory=/workspace/frappe-bench + +[group:frappe-bench-dev] +programs=frappe-bench-frappe-dev,frappe-bench-frappe-watch diff --git a/Docker/frappe/launch.sh b/Docker/frappe/launch.sh new file mode 100644 index 00000000..f9f017b3 --- /dev/null +++ b/Docker/frappe/launch.sh @@ -0,0 +1,61 @@ +#!/bin/zsh +emer() { + echo "$1" + exit 1 +} + +[[ "${WAIT_FOR:-}" ]] || emer "The WAIT_FOR env is not given." +[[ "${COMMAND:-}" ]] || emer "COMMAND is not given." + +if [[ ! "${TIMEOUT:-}" ]]; then + TIMEOUT=300 +fi +if [[ ! "${CHECK_ITERATION:-}" ]]; then + CHECK_ITERATION=10 +fi + +file_to_check="${WAIT_FOR}" + +timeout_seconds="$TIMEOUT" + +start_time=$(date +%s) +check_iteration="$CHECK_ITERATION" + +total_iteration=1 +iteration=1 +IFS=',' read -A files <<< "$file_to_check" +while true; do + all_files_exist=true + + for file in ${files[@]}; do + if [[ ! -s "$file" ]]; then + all_files_exist=false + break + fi + done + + if $all_files_exist || [[ $(( $(date +%s) - start_time )) -ge "$timeout_seconds" ]]; then + break + fi + + sleep 1 + ((total_iteration++)) + ((iteration++)) + if [ $((iteration % check_iteration)) -eq 0 ]; then + echo "Checked $iteration times..." + fi +done + +if [[ "${CHANGE_DIR:-}" ]];then + cd "$CHANGE_DIR" || true +fi + +source /opt/user/.zshrc + +if $all_files_exist; then + echo "$file_to_check populated within $total_iteration seconds." + echo "Running Command: $COMMAND" + eval "$COMMAND" +else + echo "$file_to_check did not populate within $timeout_seconds seconds. Giving Up" +fi diff --git a/Docker/frappe/supervisord.conf b/Docker/frappe/supervisord.conf index ec55f757..a3d413e7 100644 --- a/Docker/frappe/supervisord.conf +++ b/Docker/frappe/supervisord.conf @@ -8,27 +8,11 @@ serverurl = unix:///opt/user/supervisor.sock [supervisord] nodaemon = true -logfile=/tmp/supervisord.log -pidfile=/tmp/supervisord.pid +logfile=/workspace/logs/supervisord.log +pidfile=/workspace/logs/supervisord.pid [rpcinterface:supervisor] supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface -# [program:taill] -# command =/usr/bin/tail -f /opt/user/bench-start.log -# stdout_logfile = /dev/stdout -# stderr_logfile = /dev/stdout -# stdout_logfile_maxbytes = 0 -# stderr_logfile_maxbytes = 0 -# autostart=true - -[program:bench-dev] -command = /opt/user/bench-start.sh -stdout_logfile = /workspace/logs/bench-start.log -redirect_stderr = true -user = frappe -startsecs = 4 -killasgroup = true -stopasgroup= true -autostart = true -directory = /workspace/frappe-bench +[include] +files = /opt/user/conf.d/*.conf diff --git a/Docker/frappe/user-script.sh b/Docker/frappe/user-script.sh index 3338c709..463c8976 100755 --- a/Docker/frappe/user-script.sh +++ b/Docker/frappe/user-script.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # This script creates bench and executes it. set -e @@ -11,28 +11,40 @@ if [[ ! -d 'logs' ]]; then mkdir -p logs fi -REDIS_SOCKETIO_PORT=9000 +REDIS_SOCKETIO_PORT=80 WEB_PORT=80 -MARIADB_ROOT_PASS='root' + +if [[ ! "${MARIADB_HOST:-}" ]]; then + MARIADB_HOST='mariadb' +fi + +if [[ ! "${MARIADB_ROOT_PASS:-}" ]]; then + MARIADB_ROOT_PASS='root' +fi BENCH_COMMAND='/opt/.pyenv/shims/bench' +[[ "${ENVIRONMENT:-}" ]] || emer "[ERROR] ENVIRONMENT env not found. Please provide ENVIRONMENT env." +[[ "${WEB_PORT:-}" ]] || emer "[ERROR] WEB_PORT env not found. Please provide WEB_PORT env." +[[ "${SITENAME:-}" ]] || emer "[ERROR] SITENAME env not found. Please provide SITENAME env." + # if the bench doesn't exists if [[ ! -d "frappe-bench" ]]; then - [[ "${SITENAME:-}" ]] || emer "[ERROR] SITENAME env not found. Please provide SITENAME env." [[ "${REDIS_SOCKETIO_PORT:-}" ]] || emer "[ERROR] REDIS_SOCKETIO_PORT env not found. Please provide REDIS_SOCKETIO_PORT env." [[ "${DEVELOPER_MODE:-}" ]] || emer "[ERROR] DEVELOPER_MODE env not found. Please provide DEVELOPER_MODE env." - [[ "${WEB_PORT:-}" ]] || emer "[ERROR] WEB_PORT env not found. Please provide WEB_PORT env." [[ "${MARIADB_ROOT_PASS:-}" ]] || emer "[ERROR] MARIADB_ROOT_PASS env not found. Please provide MARIADB_ROOT_PASS env." + [[ "${MARIADB_HOST:-}" ]] || emer "[ERROR] MARIADB_HOST env not found. Please provide MARIADB_HOST env." [[ "${ADMIN_PASS:-}" ]] || emer "[ERROR] ADMIN_PASS env not found. Please provide ADMIN_PASS env." [[ "${DB_NAME:-}" ]] || emer "[ERROR] DB_NAME env not found. Please provide DB_NAME env." + [[ "${CONTAINER_NAME_PREFIX:-}" ]] || emer "[ERROR] CONTAINER_NAME_PREFIX env not found. Please provide CONTAINER_NAME_PREFIX env." + # create the bench $BENCH_COMMAND init --skip-assets --skip-redis-config-generation --frappe-branch "$FRAPPE_BRANCH" frappe-bench # setting configuration - wait-for-it -t 120 mariadb:3306 + wait-for-it -t 120 "$MARIADB_HOST":3306 wait-for-it -t 120 redis-cache:6379 wait-for-it -t 120 redis-queue:6379 wait-for-it -t 120 redis-socketio:6379 @@ -40,17 +52,17 @@ if [[ ! -d "frappe-bench" ]]; then cd frappe-bench $BENCH_COMMAND config dns_multitenant on - $BENCH_COMMAND set-config -g db_host 'mariadb' + $BENCH_COMMAND set-config -g db_host "$MARIADB_HOST" $BENCH_COMMAND set-config -g db_port 3306 - $BENCH_COMMAND set-config -g redis_cache 'redis://redis-cache:6379' - $BENCH_COMMAND set-config -g redis_queue 'redis://redis-queue:6379' - $BENCH_COMMAND set-config -g redis_socketio 'redis://redis-socketio:6379' - $BENCH_COMMAND set-config -g socketio_port "$REDIS_SOCKETIO_PORT" + $BENCH_COMMAND set-config -g redis_cache "redis://${CONTAINER_NAME_PREFIX}-redis-cache:6379" + $BENCH_COMMAND set-config -g redis_queue "redis://${CONTAINER_NAME_PREFIX}-redis-queue:6379" + $BENCH_COMMAND set-config -g redis_socketio "redis://${CONTAINER_NAME_PREFIX}-redis-socketio:6379" $BENCH_COMMAND set-config -g mail_port 1025 $BENCH_COMMAND set-config -g mail_server 'mailhog' $BENCH_COMMAND set-config -g disable_mail_smtp_authentication 1 $BENCH_COMMAND set-config -g webserver_port "$WEB_PORT" $BENCH_COMMAND set-config -g developer_mode "$DEVELOPER_MODE" + $BENCH_COMMAND set-config -g socketio_port "$REDIS_SOCKETIO_PORT" # HANDLE APPS # apps are taken as follows @@ -83,21 +95,33 @@ if [[ ! -d "frappe-bench" ]]; then bench_serve_help_output=$($BENCH_COMMAND serve --help) - host_changed=$(echo "$bench_serve_help_output" | grep -c 'host') + host_changed=$(echo "$bench_serve_help_output" | grep -c 'host' || true) + + SUPERVIOSRCONFIG_STATUS=$(bench setup supervisor --skip-redis --skip-supervisord --yes --user "$USER") + + /scripts/divide-supervisor-conf.py config/supervisor.conf if [[ "$host_changed" -ge 1 ]]; then - awk -v a="$WEB_PORT" '{sub(/--port [[:digit:]]+/,"--host 0.0.0.0 --port "a); print}' Procfile > Procfile.local_setup + awk -v a="$WEB_PORT" '{sub(/--port [[:digit:]]+/,"--host 0.0.0.0 --port "a); print}' /opt/user/bench-dev-server > file.tmp && mv file.tmp /opt/user/bench-dev-server.sh else - awk -v a="$WEB_PORT" '{sub(/--port [[:digit:]]+/,"--port "a); print}' Procfile > Procfile.local_setup + awk -v a="$WEB_PORT" '{sub(/--port [[:digit:]]+/,"--port "a); print}' /opt/user/bench-dev-server > file.tmp && mv file.tmp /opt/user/bench-dev-server.sh fi + chmod +x /opt/user/bench-dev-server.sh $BENCH_COMMAND build + $BENCH_COMMAND new-site --db-root-password "$MARIADB_ROOT_PASS" --db-name "$DB_NAME" --no-mariadb-socket --admin-password "$ADMIN_PASS" "$SITENAME" $BENCH_COMMAND --site "$SITENAME" scheduler enable wait + if [[ "${ENVIRONMENT}" = "dev" ]]; then + cp /opt/user/frappe-dev.conf /opt/user/conf.d/frappe-dev.conf + else + ln -sfn /workspace/frappe-bench/config/frappe-bench-frappe-web.fm.supervisor.conf /opt/user/conf.d/frappe-bench-frappe-web.fm.supervisor.conf + fi + if [[ -n "$BENCH_START_OFF" ]]; then tail -f /dev/null else @@ -106,32 +130,40 @@ if [[ ! -d "frappe-bench" ]]; then else - wait-for-it -t 120 mariadb:3306; - wait-for-it -t 120 redis-cache:6379; - wait-for-it -t 120 redis-queue:6379; - wait-for-it -t 120 redis-socketio:6379; + wait-for-it -t 120 "$MARIADB_HOST":3306; + wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-cache":6379; + wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-queue":6379; + wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-socketio":6379; cd frappe-bench bench_serve_help_output=$($BENCH_COMMAND serve --help) - host_changed=$(echo "$bench_serve_help_output" | grep -c 'host') + host_changed=$(echo "$bench_serve_help_output" | grep -c 'host' || true) - if [[ ! -f "Procfile" ]]; then - echo "Procfile doesn't exist. Please create it so bench start can work." - exit 1 - fi + SUPERVIOSRCONFIG_STATUS=$(bench setup supervisor --skip-redis --skip-supervisord --yes --user "$USER") + + /scripts/divide-supervisor-conf.py config/supervisor.conf if [[ "$host_changed" -ge 1 ]]; then - awk -v a="$WEB_PORT" '{sub(/--port [[:digit:]]+/,"--host 0.0.0.0 --port "a); print}' Procfile > Procfile.local_setup + awk -v a="$WEB_PORT" '{sub(/--port [[:digit:]]+/,"--host 0.0.0.0 --port "a); print}' /opt/user/bench-dev-server > file.tmp && mv file.tmp /opt/user/bench-dev-server.sh else - awk -v a="$WEB_PORT" '{sub(/--port [[:digit:]]+/,"--port "a); print}' Procfile > Procfile.local_setup + awk -v a="$WEB_PORT" '{sub(/--port [[:digit:]]+/,"--port "a); print}' /opt/user/bench-dev-server > file.tmp && mv file.tmp /opt/user/bench-dev-server.sh fi + chmod +x /opt/user/bench-dev-server.sh + if [[ ! "${WEB_PORT}" == 80 ]]; then $BENCH_COMMAND set-config -g webserver_port "$WEB_PORT"; fi + + if [[ "${ENVIRONMENT}" = "dev" ]]; then + cp /opt/user/frappe-dev.conf /opt/user/conf.d/frappe-dev.conf + else + ln -sfn /workspace/frappe-bench/config/frappe-bench-frappe-web.fm.supervisor.conf /opt/user/conf.d/frappe-bench-frappe-web.fm.supervisor.conf + fi + if [[ -n "$BENCH_START_OFF" ]]; then tail -f /dev/null else From 8a4fee974404f121972ec3992867e1b033915a5c Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 21:35:21 +0530 Subject: [PATCH 022/100] feat: dynamic workers --- .../templates/docker-compose.workers.tmpl | 21 +++ frappe_manager/site_manager/site.py | 108 ++++++++++-- .../workers_manager/SiteWorker.py | 165 ++++++++++++++++++ 3 files changed, 279 insertions(+), 15 deletions(-) create mode 100644 frappe_manager/compose_manager/templates/docker-compose.workers.tmpl create mode 100644 frappe_manager/site_manager/workers_manager/SiteWorker.py diff --git a/frappe_manager/compose_manager/templates/docker-compose.workers.tmpl b/frappe_manager/compose_manager/templates/docker-compose.workers.tmpl new file mode 100644 index 00000000..d507dd79 --- /dev/null +++ b/frappe_manager/compose_manager/templates/docker-compose.workers.tmpl @@ -0,0 +1,21 @@ +services: + worker-name: + image: frappe-manager-frappe:v0.9.3 + environment: + TIMEOUT: 6000 + CHANGE_DIR: /workspace/frappe-bench + WAIT_FOR: /workspace/frappe-bench/config/frappe-bench-frappe-{worker-name}.fm.workers.supervisor.conf + # here worker-name will be replace with the worker name + COMMAND: | + ln -sfn /workspace/frappe-bench/config/frappe-bench-frappe-{worker-name}.fm.workers.supervisor.conf /opt/user/conf.d/frappe-bench-frappe-{worker-name}.fm.workers.supervisor.conf + supervisord -c /opt/user/supervisord.conf + command: launch.sh + volumes: + - ./workspace:/workspace:cached + networks: + site-network: + +networks: + site-network: + name: REPLACE_ME_WITH_SITE_NAME_NETWORK + external: true diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 53cd6a7e..53bf0f5b 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -9,7 +9,9 @@ from frappe_manager.compose_manager.ComposeFile import ComposeFile from frappe_manager.site_manager.Richprint import richprint +from frappe_manager.site_manager.workers_manager.SiteWorker import SiteWorkers from frappe_manager.site_manager.utils import log_file, get_container_name_prefix +from frappe_manager.utils import host_run_cp class Site: @@ -25,10 +27,16 @@ def init(self): """ self.composefile = ComposeFile(self.path / "docker-compose.yml") self.docker = DockerClient(compose_file_path=self.composefile.compose_path) + self.workers = SiteWorkers(self.path,self.name,self.quiet) if not self.docker.server_running(): richprint.exit("Docker daemon not running. Please start docker service.") + if self.workers.exists(): + if not self.workers.running(): + if self.running(): + self.workers.start() + def exists(self): """ The `exists` function checks if a file or directory exists at the specified path. @@ -103,10 +111,6 @@ def migrate_site_compose(self): else: richprint.print("Already Latest Environment Version") - def set_site_network_name(self): - self.composefile.yml["networks"]["site-network"]["name"] = ( - self.name.replace(".", "") + f"-network" - ) def generate_compose(self, inputs: dict) -> None: """ @@ -140,26 +144,45 @@ def generate_compose(self, inputs: dict) -> None: self.composefile.set_container_names(get_container_name_prefix(self.name)) fm_version = importlib.metadata.version("frappe-manager") self.composefile.set_version(fm_version) - self.set_site_network_name() + self.composefile.set_top_networks_name("site-network",get_container_name_prefix(self.name)) self.composefile.write_to_file() - def set_site_network_name(self): - self.composefile.yml["networks"]["site-network"]["name"] = ( - self.name.replace(".", "") + f"-network" - ) + def create_site_dir(self): + # create site dir + self.path.mkdir(parents=True, exist_ok=True) - def create_dirs(self) -> bool: + def create_compose_dirs(self) -> bool: """ The function `create_dirs` creates two directories, `workspace` and `certs`, within a specified path. """ - # create site dir - self.path.mkdir(parents=True, exist_ok=True) + richprint.change_head("Creating Compose directories") + # create compose bind dirs -> workspace workspace_path = self.path / "workspace" workspace_path.mkdir(parents=True, exist_ok=True) - certs_path = self.path / "certs" - certs_path.mkdir(parents=True, exist_ok=True) + + configs_path = self.path / 'configs' + configs_path.mkdir(parents=True, exist_ok=True) + + # create nginx dirs + nginx_dir = configs_path / "nginx" + nginx_dir.mkdir(parents=True, exist_ok=True) + + nginx_poluate_dir = ['conf'] + nginx_image = self.composefile.yml['services']['nginx']['image'] + + for directory in nginx_poluate_dir: + new_dir = nginx_dir / directory + new_dir_abs = str(new_dir.absolute()) + host_run_cp(nginx_image,source="/etc/nginx",destination=new_dir_abs,docker=self.docker) + + nginx_subdirs = ['logs','cache','run'] + for directory in nginx_subdirs: + new_dir = nginx_dir / directory + new_dir.mkdir(parents=True, exist_ok=True) + + richprint.print("Creating Compose directories: Done") def start(self) -> bool: """ @@ -175,7 +198,11 @@ def start(self) -> bool: richprint.live_lines(output, padding=(0, 0, 0, 2)) richprint.print(f"{status_text}: Done") except DockerException as e: - richprint.exit(f"{status_text}: Failed") + richprint.exit(f"{status_text}: Failed",error_msg=e) + + # start workers if exits + if self.workers.exists(): + self.workers.start() def pull(self): """ @@ -266,6 +293,10 @@ def stop(self) -> bool: except DockerException as e: richprint.exit(f"{status_text}: Failed") + # stopping worker containers + if self.workers.exists(): + self.workers.stop() + def down(self, remove_ophans=True, volumes=True, timeout=5) -> bool: """ The `down` function removes containers using Docker Compose and prints the status of the operation. @@ -456,3 +487,50 @@ def get_services_running_status(self) -> dict: return services_status except DockerException as e: richprint.exit(f"{e.stdout}{e.stderr}") + + def get_host_port_binds(self): + try: + output = self.docker.compose.ps(all=True, format="json", stream=True) + status: dict = {} + for source, line in output: + if source == "stdout": + status = json.loads(line.decode()) + break + ports_info = [] + for container in status: + try: + port_info = container["Publishers"] + if port_info: + ports_info = ports_info + port_info + except KeyError as e: + pass + + published_ports = set() + for port in ports_info: + try: + published_port = port["PublishedPort"] + if published_port > 0: + published_ports.add(published_port) + except KeyError as e: + pass + + return list(published_ports) + + except DockerException as e: + return [] + + def is_service_running(self, service): + running_status = self.get_services_running_status() + if running_status[service] == "running": + return True + else: + return False + + def sync_workers_compose(self): + + are_workers_not_changed = self.workers.is_expected_worker_same_as_template() + if not are_workers_not_changed: + self.workers.generate_compose() + self.workers.start() + else: + richprint.print("Workers configuration remains unchanged.") diff --git a/frappe_manager/site_manager/workers_manager/SiteWorker.py b/frappe_manager/site_manager/workers_manager/SiteWorker.py new file mode 100644 index 00000000..de3b8561 --- /dev/null +++ b/frappe_manager/site_manager/workers_manager/SiteWorker.py @@ -0,0 +1,165 @@ +import importlib +import yaml +import json +from copy import deepcopy +from frappe_manager.compose_manager.ComposeFile import ComposeFile +#from frappe_manager.console_manager.Richprint import richprint +from frappe_manager.site_manager.Richprint import richprint +from frappe_manager.site_manager.utils import get_container_name_prefix, log_file +from frappe_manager.docker_wrapper import DockerClient, DockerException + +class SiteWorkers: + def __init__(self,site_path, site_name, quiet: bool = True): + self.compose_path = site_path / "docker-compose.workers.yml" + self.config_dir = site_path / 'workspace' / 'frappe-bench' / 'config' + self.site_name = site_name + self.quiet = quiet + self.init() + + def init(self): + self.composefile = ComposeFile( self.compose_path, template_name='docker-compose.workers.tmpl') + self.docker = DockerClient(compose_file_path=self.composefile.compose_path) + + if not self.docker.server_running(): + richprint.exit("Docker daemon not running. Please start docker service.") + + def exists(self): + return self.compose_path.exists() + + def get_expected_workers(self)-> list[str]: + + richprint.change_head("Getting Workers info") + workers_supervisor_conf_paths = [] + + for file_path in self.config_dir.iterdir(): + file_path_abs = str(file_path.absolute()) + if file_path.is_file(): + if 'fm.workers.supervisor.conf' in file_path_abs: + workers_supervisor_conf_paths.append(file_path) + + workers_expected_service_names = [] + + for worker_name in workers_supervisor_conf_paths: + worker_name = worker_name.name + worker_name = worker_name.replace("frappe-bench-frappe-","") + worker_name = worker_name.replace(".fm.workers.supervisor.conf","") + workers_expected_service_names.append(worker_name) + workers_expected_service_names.sort() + + richprint.print("Getting Workers info: Done") + + return workers_expected_service_names + + def is_expected_worker_same_as_template(self) -> bool: + + if not self.composefile.is_template_loaded: + prev_workers = self.composefile.get_services_list() + prev_workers.sort() + return prev_workers == self.get_expected_workers() + else: + return False + + def generate_compose(self): + richprint.change_head("Generating Workers configuration") + + if not self.compose_path.exists(): + richprint.print("Workers compose not present. Generating...") + else: + richprint.print("Workers configuration changed. Recreating compose...") + + # create compose file for workers + self.composefile.load_template() + + template_worker_config = self.composefile.yml['services']['worker-name'] + + del self.composefile.yml['services']['worker-name'] + + workers_expected_service_names = self.get_expected_workers() + + if len(workers_expected_service_names) > 0: + import os + for worker in workers_expected_service_names: + worker_config = deepcopy(template_worker_config) + + # setting environments + worker_config['environment']['WAIT_FOR'] = str(worker_config['environment']['WAIT_FOR']).replace("{worker-name}",worker) + worker_config['environment']['COMMAND'] = str(worker_config['environment']['COMMAND']).replace("{worker-name}",worker) + worker_config['environment']['USERID'] = os.getuid() + worker_config['environment']['USERGROUP'] = os.getgid() + + # setting extrahosts + worker_config['extra_hosts'] = [f'{self.site_name}:127.0.0.1'] + + self.composefile.yml['services'][worker] = worker_config + + self.composefile.set_container_names(get_container_name_prefix(self.site_name)) + fm_version = importlib.metadata.version("frappe-manager") + self.composefile.set_version(fm_version) + + # set network name + self.composefile.yml["networks"]["site-network"]["name"] = (self.site_name.replace(".", "") + f"-network") + self.composefile.write_to_file() + else: + richprint.error("Workers configuration not found.") + + def start(self): + status_text = "Starting Workers Containers" + richprint.change_head(status_text) + try: + output = self.docker.compose.up(detach=True, pull="never", stream=self.quiet) + if self.quiet: + richprint.live_lines(output, padding=(0, 0, 0, 2)) + richprint.print(f"{status_text}: Done") + except DockerException as e: + richprint.error (f"{status_text}: Failed Error: {e}") + + def stop(self) -> bool: + """ + The `stop` function stops containers and prints the status of the operation using the `richprint` + module. + """ + status_text = "Stopping Workers Containers" + richprint.change_head(status_text) + try: + output = self.docker.compose.stop(timeout=10, stream=self.quiet) + if self.quiet: + richprint.live_lines(output, padding=(0, 0, 0, 2)) + richprint.print(f"{status_text}: Done") + except DockerException as e: + richprint.exit(f"{status_text}: Failed") + + def get_services_running_status(self)-> dict: + services = self.composefile.get_services_list() + containers = self.composefile.get_container_names().values() + services_status = {} + try: + output = self.docker.compose.ps(service=services,format="json",all=True,stream=True) + status: dict = {} + for source, line in output: + if source == "stdout": + status = json.loads(line.decode()) + + # this is done to exclude docker runs using docker compose run command + for container in status: + if container['Name'] in containers: + services_status[container['Service']] = container['State'] + return services_status + except DockerException as e: + richprint.exit(f"{e.stdout}{e.stderr}") + + def running(self) -> bool: + """ + The `running` function checks if all the services defined in a Docker Compose file are running. + :return: a boolean value. If the number of running containers is greater than or equal to the number + of services listed in the compose file, it returns True. Otherwise, it returns False. + """ + services = self.composefile.get_services_list() + running_status = self.get_services_running_status() + + if running_status: + for service in services: + if not running_status[service] == 'running': + return False + else: + return False + return True From 4708ee25d3ff2074532646551e5d5aef7fd278a1 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 21:36:33 +0530 Subject: [PATCH 023/100] Optmize site creation logic --- frappe_manager/main.py | 56 +++++++++++++++++--------- frappe_manager/site_manager/manager.py | 4 +- frappe_manager/site_manager/site.py | 27 +++++++++---- 3 files changed, 59 insertions(+), 28 deletions(-) diff --git a/frappe_manager/main.py b/frappe_manager/main.py index 3b678732..b42f472b 100644 --- a/frappe_manager/main.py +++ b/frappe_manager/main.py @@ -221,32 +221,50 @@ def create( uid: int = os.getuid() gid: int = os.getgid() - frappe_env: dict = { - "USERID": uid, - "USERGROUP": gid, - "APPS_LIST": ",".join(apps) if apps else None, - "FRAPPE_BRANCH": frappe_branch, - "DEVELOPER_MODE": developer_mode, - "ADMIN_PASS": admin_pass, - "DB_NAME": sites.site.name.replace(".", "-"), - "SITENAME": sites.site.name, + environment = { + "frappe": { + "USERID": uid, + "USERGROUP": gid, + "APPS_LIST": ",".join(apps) if apps else None, + "FRAPPE_BRANCH": frappe_branch, + "DEVELOPER_MODE": developer_mode, + "ADMIN_PASS": admin_pass, + "DB_NAME": sites.site.name.replace(".", "-"), + "SITENAME": sites.site.name, + "MARIADB_ROOT_PASS": 'root', + "CONTAINER_NAME_PREFIX": get_container_name_prefix(sites.site.name), + "ENVIRONMENT": "dev", + }, + "nginx": { + "ENABLE_SSL": enable_ssl, + "SITENAME": sites.site.name, + "VIRTUAL_HOST": sites.site.name, + "VIRTUAL_PORT": 80, + }, + "worker": { + "USERID": uid, + "USERGROUP": gid, + }, + "schedule": { + "USERID": uid, + "USERGROUP": gid, + }, + "socketio": { + "USERID": uid, + "USERGROUP": gid, + }, } - nginx_env: dict = { - "ENABLE_SSL": enable_ssl, - "SITENAME": sites.site.name, - } - - # fix for macos - extra_hosts: List[str] = [f"{sites.site.name}:127.0.0.1"] + users: dict = {"nginx": {"uid": uid, "gid": gid}} template_inputs: dict = { - "frappe_env": frappe_env, - "nginx_env": nginx_env, - "extra_hosts": extra_hosts, + "environment": environment, + # "extra_hosts": extra_hosts, + "user": users, } # turn off all previous # start the docker compose + sites.create_site(template_inputs) diff --git a/frappe_manager/site_manager/manager.py b/frappe_manager/site_manager/manager.py index d9fefd05..ca73366c 100644 --- a/frappe_manager/site_manager/manager.py +++ b/frappe_manager/site_manager/manager.py @@ -153,13 +153,15 @@ def create_site(self, template_inputs: dict): # check if ports are available self.check_ports() richprint.change_head(f"Creating Site Directory") - self.site.create_dirs() + self.site.create_site_dir() richprint.change_head(f"Generating Compose") self.site.generate_compose(template_inputs) + self.site.create_compose_dirs() self.site.pull() richprint.change_head(f"Starting Site") self.site.start() self.site.frappe_logs_till_start() + self.site.sync_workers_compose() richprint.update_live() richprint.change_head(f"Checking site") diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 53bf0f5b..a44f9b56 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -86,19 +86,30 @@ def migrate_site_compose(self): # extrahosts = self.composefile.get_all_extrahosts() labels = self.composefile.get_all_labels() - self.composefile.load_template() - self.composefile.set_version(fm_version) + + # handle users, should be of the new invocation + import os + users = {"nginx":{ + "uid": os.getuid(), + "gid": os.getgid() + } + } + self.composefile.set_all_envs(envs) - # self.composefile.set_all_extrahosts(extrahosts) self.composefile.set_all_labels(labels) + self.composefile.set_all_users(users) + # self.composefile.set_all_extrahosts(extrahosts) - self.composefile.set_container_names( - get_container_name_prefix(self.name) - ) - self.set_site_network_name() + # handle network name + self.composefile.load_template() + + self.composefile.set_network_alias("nginx", "site-network", [self.name]) + self.composefile.set_container_names(get_container_name_prefix(self.name)) + fm_version = importlib.metadata.version("frappe-manager") + self.composefile.set_version(fm_version) + self.composefile.set_top_networks_name("site-network",get_container_name_prefix(self.name)) self.composefile.write_to_file() - status = True if status: richprint.print( From 882ef607d86fbdf5693e45fbe3c5460da215923a Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 21:37:37 +0530 Subject: [PATCH 024/100] update: logs changes with new dev server run method using supervisor --- frappe_manager/main.py | 1 + frappe_manager/site_manager/site.py | 13 +++++-------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/frappe_manager/main.py b/frappe_manager/main.py index b42f472b..4e7b4c60 100644 --- a/frappe_manager/main.py +++ b/frappe_manager/main.py @@ -11,6 +11,7 @@ from frappe_manager import CLI_DIR, default_extension, SiteServicesEnum from frappe_manager.logger import log from frappe_manager.utils import check_update, remove_zombie_subprocess_process +from frappe_manager.site_manager.utils import get_container_name_prefix app = typer.Typer(no_args_is_help=True,rich_markup_mode='rich') global_service = None diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index a44f9b56..99936662 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -268,24 +268,21 @@ def frappe_logs_till_start(self): ) if self.quiet: - richprint.live_lines( + exit_code = richprint.live_lines( output, padding=(0, 0, 0, 2), - stop_string="INFO spawned: 'bench-dev' with pid", + stop_string="INFO supervisord started with pid", ) else: - for source, line in self.docker.compose.logs( - services=["frappe"], no_log_prefix=True, follow=True, stream=True - ): + for source, line in output: if not source == "exit_code": line = line.decode() if "[==".lower() in line.lower(): print(line) else: richprint.stdout.print(line) - if "INFO spawned: 'bench-dev' with pid".lower() in line.lower(): + if "INFO supervisord started with pid".lower() in line.lower(): break - richprint.print(f"{status_text}: Done") except DockerException as e: richprint.warning(f"{status_text}: Failed") @@ -414,7 +411,7 @@ def bench_dev_server_logs(self, follow=False): This function is used to tail logs found at /workspace/logs/bench-start.log. :param follow: Bool detemines whether to follow the log file for changes """ - bench_start_log_path = self.path / "workspace" / "logs" / "bench-start.log" + bench_start_log_path = self.path / "workspace" / "frappe-bench" / 'logs' / "web.dev.log" if bench_start_log_path.exists() and bench_start_log_path.is_file(): with open(bench_start_log_path, "r") as bench_start_log: From cd02f249916bf78edc4468f430f6204b1d0cac1f Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 21:38:25 +0530 Subject: [PATCH 025/100] add util function docker_host_cp --- frappe_manager/utils.py | 82 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/frappe_manager/utils.py b/frappe_manager/utils.py index 05acfa29..28a55085 100644 --- a/frappe_manager/utils.py +++ b/frappe_manager/utils.py @@ -109,3 +109,85 @@ def check_ports_with_msg(ports_to_check: list, exclude=[]): f"Whoa there! Looks like the {' '.join([ str(x) for x in already_binded ])} { 'ports are' if len(already_binded) > 1 else 'port is' } having a party already! Can you do us a solid and free up those ports?" ) richprint.print("Ports Check : Passed") + +def generate_random_text(length=50): + import random + import string + + alphanumeric_chars = string.ascii_letters + string.digits + return "".join(random.choice(alphanumeric_chars) for _ in range(length)) + + +def host_run_cp(image: str, source: str, destination: str, docker, verbose=False): + status_text = "Copying files" + richprint.change_head(f"{status_text} {source} -> {destination}") + source_container_name = generate_random_text(10) + dest_path = Path(destination) + + failed: bool = False + # run the container + try: + output = docker.run( + image=image, + name=source_container_name, + detach=True, + stream=not verbose, + command="tail -f /dev/null", + ) + if not verbose: + richprint.live_lines(output, padding=(0, 0, 0, 2)) + except DockerException as e: + failed = 0 + + if not failed: + # cp from the container + try: + output = docker.cp( + source=source, + destination=destination, + source_container=source_container_name, + stream=not verbose, + ) + if not verbose: + richprint.live_lines(output, padding=(0, 0, 0, 2)) + except DockerException as e: + failed = 1 + + # # kill the container + # try: + # output = docker.kill(container=source_container_name,stream=True) + # richprint.live_lines(output, padding=(0,0,0,2)) + # except DockerException as e: + # richprint.exit(f"{status_text} failed. Error: {e}") + + if not failed: + # rm the container + try: + output = docker.rm( + container=source_container_name, force=True, stream=not verbose + ) + if not verbose: + richprint.live_lines(output, padding=(0, 0, 0, 2)) + except DockerException as e: + failed = 2 + + # check if the destination file exists + if not type(failed) == bool: + if failed > 1: + if dest_path.exists(): + import shutil + + shutil.rmtree(dest_path) + if failed == 2: + try: + output = docker.rm( + container=source_container_name, force=True, stream=not verbose + ) + if not verbose: + richprint.live_lines(output, padding=(0, 0, 0, 2)) + except DockerException as e: + pass + richprint.exit(f"{status_text} failed.") + + elif not Path(destination).exists(): + richprint.exit(f"{status_text} failed. Copied {destination} not found.") From 13916a0dbcfa3c6f81199f4f03231321165b6eac Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 21:39:42 +0530 Subject: [PATCH 026/100] chore: format code --- frappe_manager/utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/frappe_manager/utils.py b/frappe_manager/utils.py index 28a55085..94a1c605 100644 --- a/frappe_manager/utils.py +++ b/frappe_manager/utils.py @@ -5,10 +5,13 @@ import subprocess import platform +from pathlib import Path from frappe_manager.logger import log from frappe_manager.docker_wrapper.utils import process_opened +from frappe_manager.docker_wrapper.DockerException import DockerException from frappe_manager.site_manager.Richprint import richprint + def remove_zombie_subprocess_process(): """ Terminates any zombie process @@ -20,6 +23,7 @@ def remove_zombie_subprocess_process(): # terminate zombie docker process import psutil + for pid in process_opened: try: process = psutil.Process(pid) @@ -31,6 +35,7 @@ def remove_zombie_subprocess_process(): logger.cleanup(f"{pid} Permission denied") logger.cleanup("-" * 20) + def check_update(): url = "https://pypi.org/pypi/frappe-manager/json" try: @@ -46,6 +51,7 @@ def check_update(): except Exception as e: pass + def is_port_in_use(port): """ Check if port is in use or not. @@ -110,6 +116,7 @@ def check_ports_with_msg(ports_to_check: list, exclude=[]): ) richprint.print("Ports Check : Passed") + def generate_random_text(length=50): import random import string From 2ba885d8ce14b939e9b3d712ddcd0f4bf15d5aad Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 21:41:10 +0530 Subject: [PATCH 027/100] functions for managingin user and tol level networks attribute names --- frappe_manager/compose_manager/ComposeFile.py | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/frappe_manager/compose_manager/ComposeFile.py b/frappe_manager/compose_manager/ComposeFile.py index 3981c106..fec4161d 100644 --- a/frappe_manager/compose_manager/ComposeFile.py +++ b/frappe_manager/compose_manager/ComposeFile.py @@ -112,7 +112,7 @@ def is_services_name_same_as_template(self): :return: a boolean value indicating whether the list of service names in the current YAML file is the same as the list of service names in the template YAML file. """ - template = self.__get_template("docker-compose.tmpl") + template = self.get_template(self.template_name) template_yml = yaml.safe_load(template) template_service_name_list = list(template_yml["services"].keys()) template_service_name_list.sort() @@ -136,6 +136,17 @@ def get_user(self, service): return None return user + def set_top_networks_name(self, networks_name, prefix): + """ + The function sets the network names for each service in a compose file based on the site name. + """ + + if not self.yml["networks"][networks_name]: + self.yml["networks"][networks_name] = { "name" : prefix + f"-network" } + else: + self.yml["networks"][networks_name]["name"] = prefix + f"-network" + + def set_network_alias(self, service_name, network_name, alias: list = []): if alias: try: @@ -184,6 +195,33 @@ def set_version(self, version): """ self.yml["x-version"] = version + def get_all_users(self): + """ + The function `get_all_users` returns a dictionary of users for each service in a compose file. + :return: a dictionary containing the users of the containers specified in the compose file. + """ + users: dict = {} + + if self.exists(): + services = self.get_services_list() + for service in services: + if "user" in self.yml["services"][service]: + user_data = self.yml["services"][service]["user"] + uid = user_data.split(":")[0] + gid = user_data.split(":")[1] + users[service] = {"uid": uid, "gid": gid} + return users + + def set_all_users(self, users: dict): + """ + The function `set_all_users` sets the users for each service in a compose file. + + :param users: The `users` parameter is a dictionary that contains users for each service in a + compose file. + """ + for service in users.keys(): + self.set_user(service, users[service]["uid"], users[service]["gid"]) + def get_all_envs(self): """ This functtion returns all the container environment variables From 67a8a91f5bdb84b70ffd4dcf8b2af2ed4f428b0a Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 21:43:17 +0530 Subject: [PATCH 028/100] richprint.error support extra error msg printing --- frappe_manager/site_manager/Richprint.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/frappe_manager/site_manager/Richprint.py b/frappe_manager/site_manager/Richprint.py index d8237c37..8b877c40 100644 --- a/frappe_manager/site_manager/Richprint.py +++ b/frappe_manager/site_manager/Richprint.py @@ -6,8 +6,9 @@ from rich.text import Text from rich.padding import Padding from typer import Exit - from rich.table import Table + +import typer from collections import deque from typing import Optional @@ -59,11 +60,11 @@ def warning(self,text: str,emoji_code: str = ':warning: '): """ self.stdout.print(f"{emoji_code} {text}") - def exit(self,text: str,emoji_code: str = ':x:'): + def exit(self,text: str,emoji_code: str = ':x:',os_exit= False, error_msg= None): """ The `exit` function stops the program, prints a message with an emoji, and exits using `typer.Exit` exception. - + :param text: The `text` parameter is a string that represents the message or reason for exiting. It is the text that will be printed when the `exit` method is called :type text: str @@ -72,8 +73,14 @@ def exit(self,text: str,emoji_code: str = ':x:'): :type emoji_code: str (optional) """ self.stop() - self.stdout.print(f"{emoji_code} {text}") - raise Exit(1) + if error_msg: + to_print = f"{emoji_code} {text}\n Error : {error_msg}" + else: + to_print = f"{emoji_code} {text} " + self.stdout.print(to_print) + if os_exit: + exit(1) + raise typer.Exit(1) def print(self,text: str,emoji_code: str = ':white_check_mark:'): """ From 0a72bb9ae142e8b80b14ddc332e7dda5e3401739 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 21:44:02 +0530 Subject: [PATCH 029/100] remove ssl support from nginx --- Docker/nginx/template.conf | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/Docker/nginx/template.conf b/Docker/nginx/template.conf index d994d143..01b08e85 100644 --- a/Docker/nginx/template.conf +++ b/Docker/nginx/template.conf @@ -3,7 +3,7 @@ upstream frappe-bench-frappe { server frappe:80 fail_timeout=120; } upstream frappe-bench-socketio-server { - server frappe:9000 fail_timeout=120; + server socketio:80 fail_timeout=120; } upstream mailhog { server mailhog:8025 fail_timeout=120; @@ -13,27 +13,8 @@ upstream adminer { } server { - {%- if ENABLE_SSL == False %} listen 80; listen [::]:80; - {%- endif %} - - {%- if ENABLE_SSL == True %} - listen 443 ssl; - listen [::]:443 ssl; - - ssl_certificate /etc/nginx/certs/cert.pem; - ssl_certificate_key /etc/nginx/certs/key.pem; - ssl_session_timeout 5m; - ssl_session_cache shared:SSL:10m; - ssl_session_tickets off; - ssl_stapling on; - ssl_stapling_verify on; - ssl_protocols TLSv1.2 TLSv1.3; - ssl_ciphers EECDH+AESGCM:EDH+AESGCM; - ssl_ecdh_curve secp384r1; - ssl_prefer_server_ciphers on; - {%- endif %} server_name {{ ' '.join(SITENAME.split(',')) }}; root /workspace/frappe-bench/sites; From 210b45db22227d6dd1b78b2b817b3feb34d726bf Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 21:44:38 +0530 Subject: [PATCH 030/100] chore: format code --- .../docker_wrapper/DockerCompose.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/frappe_manager/docker_wrapper/DockerCompose.py b/frappe_manager/docker_wrapper/DockerCompose.py index fb7ab6dc..cc61abbe 100644 --- a/frappe_manager/docker_wrapper/DockerCompose.py +++ b/frappe_manager/docker_wrapper/DockerCompose.py @@ -1,3 +1,4 @@ + from subprocess import Popen, run, TimeoutExpired, CalledProcessError from pathlib import Path from typing import Union, Literal @@ -50,7 +51,7 @@ def up( """ The `up` function is a Python method that runs the `docker-compose up` command with various options and returns an iterator. - + :param detach: A boolean flag indicating whether to run containers in the background or not. If set to True, containers will be detached and run in the background. If set to False, containers will run in the foreground, defaults to True @@ -103,7 +104,7 @@ def up( return iterator # @handle_docker_error - def down( + def down( self, timeout: int = 100, remove_orphans: bool = False, @@ -143,7 +144,7 @@ def start( """ The `start` function is used to start Docker services specified in the `services` parameter, with options for dry run, streaming output, and checking only the exit code. - + :param services: A list of services to start. If None, all services will be started :type services: Union[None, list[str]] :param dry_run: A boolean flag indicating whether the start operation should be performed in dry run @@ -189,7 +190,7 @@ def restart( """ The `restart` function restarts specified services in a Docker Compose environment with various options and returns an iterator. - + :param services: A list of services to restart. If set to None, all services will be restarted :type services: Union[None, list[str]] :param dry_run: A boolean flag indicating whether the restart operation should be performed as a dry @@ -239,7 +240,7 @@ def stop( """ The `stop` function stops specified services in a Docker Compose environment, with options for timeout, streaming output, and checking for service existence. - + :param services: A list of service names to stop. If None, all services will be stopped :type services: Union[None, list[str]] :param timeout: The `timeout` parameter specifies the maximum time (in seconds) to wait for the @@ -339,6 +340,7 @@ def exec( "stream_only_exit_code", "command", "env", + "use_shlex_split", ] exec_cmd += parameters_to_options(parameters, exclude=remove_parameters) @@ -401,7 +403,7 @@ def ps( """ The `ps` function is a Python method that executes the `docker-compose ps` command with various parameters and returns an iterator. - + :param service: A list of service names to filter the results by. If None, all services are included :type service: Union[None, list[str]] :param dry_run: A boolean flag indicating whether the command should be executed in dry run mode, @@ -415,7 +417,7 @@ def ps( :type services: bool (optional) :param filter: The `filter` parameter is used to filter the list of containers based on their status. It accepts the following values: - :type filter: Union[j ,, + :type filter: Union[j ,, :param format: The `format` parameter specifies the output format for the `ps` command. It can be set to either "table" or "json" :type format: Union[None, Literal["table", "json"]] @@ -493,7 +495,7 @@ def logs( """ The `logs` function in Python takes in various parameters and returns an iterator that runs a Docker Compose command to retrieve logs from specified services. - + :param services: A list of services for which to retrieve logs. If None, logs for all services will be retrieved :type services: Union[None, list[str]] @@ -559,7 +561,7 @@ def ls( """ The `ls` function is a Python method that runs the `ls` command in Docker Compose and returns the output. - + :param all: A boolean flag indicating whether to show hidden files and directories, defaults to False :type all: bool (optional) @@ -601,7 +603,7 @@ def pull( ): """ The `pull` function is used to pull Docker images, with various options for customization. - + :param dry_run: A boolean flag indicating whether the pull operation should be performed as a dry run (without actually pulling the images), defaults to False :type dry_run: bool (optional) From a84858716ec41e774d5990f85dfb8033a701d808 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 8 Jan 2024 22:33:17 +0530 Subject: [PATCH 031/100] detect keyboard interrupt when loggging --- frappe_manager/main.py | 6 +++--- frappe_manager/site_manager/site.py | 7 ++----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/frappe_manager/main.py b/frappe_manager/main.py index 4e7b4c60..a719bb35 100644 --- a/frappe_manager/main.py +++ b/frappe_manager/main.py @@ -336,9 +336,9 @@ def logs( """Show frappe dev server logs or container logs for a given site. """ sites.init(sitename) if service: - sites.logs(SiteServicesEnum(service).value,follow) + sites.logs(service=SiteServicesEnum(service).value,follow=follow) else: - sites.logs(follow) + sites.logs(follow=follow) @app.command(no_args_is_help=True) @@ -350,7 +350,7 @@ def shell( """Open shell for the give site. """ sites.init(sitename) if service: - sites.shell(SiteServicesEnum(service).value,user=user) + sites.shell(service=SiteServicesEnum(service).value,user=user) else: sites.shell(user=user) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 99936662..66223455 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -416,11 +416,8 @@ def bench_dev_server_logs(self, follow=False): if bench_start_log_path.exists() and bench_start_log_path.is_file(): with open(bench_start_log_path, "r") as bench_start_log: bench_start_log_data = log_file(bench_start_log, follow=follow) - try: - for line in bench_start_log_data: - richprint.stdout.print(line) - except KeyboardInterrupt: - richprint.stdout.print("Detected CTRL+C. Exiting.") + for line in bench_start_log_data: + richprint.stdout.print(line) else: richprint.error(f"Log file not found: {bench_start_log_path}") From d4e59b42510794d112614c1c6f8da211e7cd6ebd Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 01:35:36 +0530 Subject: [PATCH 032/100] optmize: compose file migration --- frappe_manager/site_manager/site.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 66223455..6fb12ab1 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -82,10 +82,11 @@ def migrate_site_compose(self): if not compose_version == fm_version: status = False if self.composefile.exists(): + + # get all the payloads envs = self.composefile.get_all_envs() - # extrahosts = self.composefile.get_all_extrahosts() labels = self.composefile.get_all_labels() - + # extrahosts = self.composefile.get_all_extrahosts() # handle users, should be of the new invocation import os @@ -95,15 +96,15 @@ def migrate_site_compose(self): } } + # load template + self.composefile.load_template() + # set all the payload self.composefile.set_all_envs(envs) self.composefile.set_all_labels(labels) self.composefile.set_all_users(users) # self.composefile.set_all_extrahosts(extrahosts) - # handle network name - self.composefile.load_template() - self.composefile.set_network_alias("nginx", "site-network", [self.name]) self.composefile.set_container_names(get_container_name_prefix(self.name)) fm_version = importlib.metadata.version("frappe-manager") From 18958ee35a6114a3fb77969acee51aa096e3164f Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 02:39:44 +0530 Subject: [PATCH 033/100] don't invoke fm when help msg --- frappe_manager/main.py | 107 +++++++++++++++++++++------------------- frappe_manager/utils.py | 29 +++++++++++ 2 files changed, 84 insertions(+), 52 deletions(-) diff --git a/frappe_manager/main.py b/frappe_manager/main.py index a719bb35..84cbf212 100644 --- a/frappe_manager/main.py +++ b/frappe_manager/main.py @@ -68,58 +68,61 @@ def app_callback( """ ctx.obj = {} - richprint.start(f"Working") - - sitesdir = CLI_DIR / 'sites' - # Checks for cli directory - if not CLI_DIR.exists(): - # creating the sites dir - # TODO check if it's writeable and readable -> by writing a file to it and catching exception - CLI_DIR.mkdir(parents=True, exist_ok=True) - sitesdir.mkdir(parents=True, exist_ok=True) - richprint.print(f"fm directory doesn't exists! Created at -> {str(CLI_DIR)}") - else: - if not CLI_DIR.is_dir(): - richprint.exit("Sites directory is not a directory! Aborting!") - - # Migration for directory change from CLI_DIR to CLI_DIR/sites - # TODO remove when not required, introduced in 0.8.4 - if not sitesdir.exists(): - richprint.change_head("Site directory migration") - move_directory_list = [] - for site_dir in CLI_DIR.iterdir(): - if site_dir.is_dir(): - docker_compose_path = site_dir / "docker-compose.yml" - if docker_compose_path.exists(): - move_directory_list.append(site_dir) - - # stop all the sites - sitesdir.mkdir(parents=True, exist_ok=True) - sites_mananger = SiteManager(CLI_DIR) - sites_mananger.stop_sites() - # move all the directories - for site in move_directory_list: - site_name = site.parts[-1] - new_path = sitesdir / site_name - try: - shutil.move(site,new_path) - richprint.print(f"Directory migrated: {site_name}") - except: - logger.debug(f'Site Directory migration failed: {site}') - richprint.warning(f"Unable to site directory migration for {site}\nPlease manually move it to {new_path}") - richprint.print("Site directory migration: Done") - - global sites - sites = SiteManager(sitesdir) - - sites.set_typer_context(ctx) - - if verbose: - sites.set_verbose() - - ctx.obj["services"] = global_service - ctx.obj["sites"] = sites - ctx.obj["logger"] = logger + help_called = is_cli_help_called(ctx) + ctx.obj["is_help_called"] = help_called + + if not help_called: + richprint.start(f"Working") + + sitesdir = CLI_DIR / 'sites' + # Checks for cli directory + if not CLI_DIR.exists(): + # creating the sites dir + # TODO check if it's writeable and readable -> by writing a file to it and catching exception + CLI_DIR.mkdir(parents=True, exist_ok=True) + sitesdir.mkdir(parents=True, exist_ok=True) + richprint.print(f"fm directory doesn't exists! Created at -> {str(CLI_DIR)}") + else: + if not CLI_DIR.is_dir(): + richprint.exit("Sites directory is not a directory! Aborting!") + + # Migration for directory change from CLI_DIR to CLI_DIR/sites + # TODO remove when not required, introduced in 0.8.4 + if not sitesdir.exists(): + richprint.change_head("Site directory migration") + move_directory_list = [] + for site_dir in CLI_DIR.iterdir(): + if site_dir.is_dir(): + docker_compose_path = site_dir / "docker-compose.yml" + if docker_compose_path.exists(): + move_directory_list.append(site_dir) + + # stop all the sites + sitesdir.mkdir(parents=True, exist_ok=True) + sites_mananger = SiteManager(CLI_DIR) + sites_mananger.stop_sites() + # move all the directories + for site in move_directory_list: + site_name = site.parts[-1] + new_path = sitesdir / site_name + try: + shutil.move(site,new_path) + richprint.print(f"Directory migrated: {site_name}") + except: + logger.debug(f'Site Directory migration failed: {site}') + richprint.warning(f"Unable to site directory migration for {site}\nPlease manually move it to {new_path}") + richprint.print("Site directory migration: Done") + + global sites + sites = SiteManager(sitesdir) + + sites.set_typer_context(ctx) + + if verbose: + sites.set_verbose() + + ctx.obj["sites"] = sites + ctx.obj["logger"] = logger def check_frappe_app_exists(appname: str, branchname: str | None = None): # check appname diff --git a/frappe_manager/utils.py b/frappe_manager/utils.py index 94a1c605..be607d37 100644 --- a/frappe_manager/utils.py +++ b/frappe_manager/utils.py @@ -198,3 +198,32 @@ def host_run_cp(image: str, source: str, destination: str, docker, verbose=False elif not Path(destination).exists(): richprint.exit(f"{status_text} failed. Copied {destination} not found.") + +def is_cli_help_called(ctx): + help_called = False + + # is called command is sub command group + try: + for subtyper_command in ctx.command.commands[ + ctx.invoked_subcommand + ].commands.keys(): + check_command = " ".join(sys.argv[2:]) + if check_command == subtyper_command: + if ( + ctx.command.commands[ctx.invoked_subcommand] + .commands[subtyper_command] + .params + ): + help_called = True + except AttributeError: + help_called = False + + if not help_called: + # is called command is sub command + check_command = " ".join(sys.argv[1:]) + if check_command == ctx.invoked_subcommand: + # is called command supports arguments then help called + if ctx.command.commands[ctx.invoked_subcommand].params: + help_called = True + + return help_called From 8d0b45b93b9bfb259b8aaeddcc3b498472fd8a78 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 02:44:04 +0530 Subject: [PATCH 034/100] enhancement: site services status in fm info command --- .../{manager.py => SiteManager.py} | 76 ++++++++++++++++--- 1 file changed, 67 insertions(+), 9 deletions(-) rename frappe_manager/site_manager/{manager.py => SiteManager.py} (83%) diff --git a/frappe_manager/site_manager/manager.py b/frappe_manager/site_manager/SiteManager.py similarity index 83% rename from frappe_manager/site_manager/manager.py rename to frappe_manager/site_manager/SiteManager.py index ca73366c..2257e268 100644 --- a/frappe_manager/site_manager/manager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -17,10 +17,12 @@ from rich.columns import Columns from rich.panel import Panel -from rich.table import Table +from rich.table import Table, Row from rich.text import Text +from rich.console import Group from rich import box + class SiteManager: def __init__(self, sitesdir: Path): self.sitesdir = sitesdir @@ -399,10 +401,12 @@ def info(self): root_db_password = self.site.composefile.get_envs("mariadb")[ "MYSQL_ROOT_PASSWORD" ] + site_info_table = Table(show_lines=True, show_header=False, highlight=True) + data = { "Site Url": f"http://{self.site.name}", - "Site Root": f"{self.site.path.absolute()}", + "Site Root": f"[link=file://{self.site.path.absolute()}]{self.site.path.absolute()}[/link]", "Mailhog Url": f"http://{self.site.name}/mailhog", "Adminer Url": f"http://{self.site.name}/adminer", "Frappe Username": "administrator", @@ -414,17 +418,15 @@ def info(self): "DB User": db_user, "DB Password": db_pass, } - site_info_table.add_column() - site_info_table.add_column() + + site_info_table.add_column(no_wrap=True) + site_info_table.add_column(no_wrap=True) + for key in data.keys(): site_info_table.add_row(key, data[key]) # bench apps list - richprint.stdout.print("") - # bench_apps_list_table=Table(title="Bench Apps",box=box.ASCII2,show_lines=True) - bench_apps_list_table = Table( - show_lines=True, expand=True, show_edge=False, pad_edge=False - ) + bench_apps_list_table = Table(show_lines=True, show_edge=False, pad_edge=False, expand=True) bench_apps_list_table.add_column("App") bench_apps_list_table.add_column("Version") @@ -432,13 +434,69 @@ def info(self): self.site.path / "workspace" / "frappe-bench" / "sites" / "apps.json" ) if apps_json_file.exists(): + with open(apps_json_file, "r") as f: apps_json = json.load(f) for app in apps_json.keys(): bench_apps_list_table.add_row(app, apps_json[app]["version"]) site_info_table.add_row("Bench Apps", bench_apps_list_table) + + # running site services status + running_site_services = self.site.get_services_running_status() + if running_site_services: + site_services_table = Table(show_lines=False, show_edge=False, pad_edge=False, show_header=False,expand=True,box=None) + site_services_table.add_column("Service Status",ratio=1,no_wrap=True,width=None,min_width=20) + site_services_table.add_column("Service Status",ratio=1,no_wrap=True,width=None,min_width=20) + # site_services_table.add_column("Service Status",ratio=1,no_wrap=True) + + #half_of_running_site_services = len(running_site_services) // 2 + + index = 0 + while index < len(running_site_services): + first_service_table = None + second_service_table = None + + try: + first_service = list(running_site_services.keys())[index] + index += 1 + except IndexError: + pass + first_service= None + try: + second_service = list(running_site_services.keys())[index] + index += 1 + except IndexError: + second_service = None + + # Fist Coloumn + if first_service: + first_service_table = Table(show_lines=False, show_header=False, highlight=True, expand=True,box=None) + first_service_table.add_column("Service",justify="left",no_wrap=True) + first_service_table.add_column("Status",justify="right",no_wrap=True) + first_service_table.add_row(f"{first_service}", f"{':green_square:' if running_site_services[first_service] == 'running' else ':red_square:'}") + + # Fist Coloumn + if second_service: + second_service_table = Table(show_lines=False, show_header=False, highlight=True, expand=True,box=None) + second_service_table.add_column("Service",justify="left",no_wrap=True,) + second_service_table.add_column("Status",justify="right",no_wrap=True) + second_service_table.add_row(f"{second_service}", f"{':green_square:' if running_site_services[second_service] == 'running' else ':red_square:'}") + + site_services_table.add_row(first_service_table,second_service_table) + + hints_table = Table(show_lines=True, show_header=False, highlight=True, expand=True,show_edge=True, box=None,padding=(1,0,0,0)) + hints_table.add_column("First",justify="center",no_wrap=True) + hints_table.add_column("Second",justify="center",ratio=8,no_wrap=True) + hints_table.add_column("Third",justify="center",ratio=8,no_wrap=True) + hints_table.add_row(":light_bulb:",f":green_square: -> Active", f":red_square: -> Inactive") + + site_services_table_group = Group(site_services_table,hints_table) + + site_info_table.add_row("Site Services", site_services_table_group) + richprint.stdout.print(site_info_table) + richprint.print(f"Run 'fm list' to list all available sites.",emoji_code=':light_bulb:') def migrate_site(self): """ From cb910737cd79a416edea04b7ae2099138c8365d0 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 02:45:15 +0530 Subject: [PATCH 035/100] revamp fm list command --- frappe_manager/site_manager/SiteManager.py | 63 ++++++++++++++++------ 1 file changed, 47 insertions(+), 16 deletions(-) diff --git a/frappe_manager/site_manager/SiteManager.py b/frappe_manager/site_manager/SiteManager.py index 2257e268..20545488 100644 --- a/frappe_manager/site_manager/SiteManager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -143,7 +143,7 @@ def create_site(self, template_inputs: dict): """ The `create_site` function creates a new site directory, generates a compose file, pulls the necessary images, starts the site, and displays information about the site. - + :param template_inputs: The `template_inputs` parameter is a dictionary that contains the inputs or configuration values required to generate the compose file for the site. These inputs can be used to customize the site's configuration, such as database settings, domain name, etc @@ -206,8 +206,11 @@ def list_sites(self): """ # format -> name , status [ 'stale', 'running' ] # sites_list = self.__get_all_sites_path() - running = [] + active = [] + inactive = [] stale = [] + + sites_list = self.get_all_sites() if not sites_list: richprint.error("No sites available !") @@ -215,22 +218,50 @@ def list_sites(self): else: for name in sites_list.keys(): temppath = self.sitesdir / name - tempSite = Site(temppath,name) + tempSite = Site(temppath, name) + + # know if all services are running + tempsite_services_status = tempSite.get_services_running_status() + + inactive_status = False + for service in tempsite_services_status.keys(): + if tempsite_services_status[service] == "running": + inactive_status = True + if tempSite.running(): - running.append({'name': name,'path':temppath.absolute()}) + active.append({"name": name, "path": temppath.absolute()}) + elif inactive_status: + inactive.append({"name": name, "path": temppath.absolute()}) else: - stale.append({'name': name,'path':temppath.absolute()}) - richprint.stop() - - if running: - columns_data = [ f"[b]{x['name']}[/b]\n[dim]{x['path']}[/dim]" for x in running ] - panel = Panel(Columns(columns_data),title='Running',title_align='left',style='green') - richprint.stdout.print(panel) - - if stale: - columns_data = [ f"[b]{x['name']}[/b]\n[dim]{x['path']}[/dim]" for x in stale ] - panel = Panel(Columns(columns_data),title='Stale',title_align='left',style='dark_turquoise') - richprint.stdout.print(panel) + stale.append({"name": name, "path": temppath.absolute()}) + + richprint.stop() + + list_table = Table(show_lines=True, show_header=True, highlight=True) + list_table.add_column("Site") + list_table.add_column("Status", vertical="middle") + list_table.add_column("Path") + + for site in active: + row_data = f"[link=http://{site['name']}]{site['name']}[/link]" + path_data = f"[link=file://{site['path']}]{site['path']}[/link]" + status_data = "[green]Active[/green]" + list_table.add_row(row_data, status_data, path_data, style="green") + + for site in inactive: + row_data = f"[link=http://{site['name']}]{site['name']}[/link]" + path_data = f"[link=file://{site['path']}]{site['path']}[/link]" + status_data = "[red]Inactive[/red]" + list_table.add_row(row_data, status_data, path_data,style="red") + + for site in stale: + row_data = f"[link=http://{site['name']}]{site['name']}[/link]" + path_data = f"[link=file://{site['path']}]{site['path']}[/link]" + status_data = "[grey]Stale[/grey]" + list_table.add_row(row_data, status_data, path_data) + + richprint.stdout.print(list_table) + richprint.print(f"Run 'fm info ' to get detail information about a site.",emoji_code=':light_bulb:') def stop_site(self): """ From 98f9c21ee59caf8994c5abce11b83fc56fced452 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 02:45:45 +0530 Subject: [PATCH 036/100] exception handling in fm logs command --- frappe_manager/site_manager/SiteManager.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/frappe_manager/site_manager/SiteManager.py b/frappe_manager/site_manager/SiteManager.py index 20545488..1079a215 100644 --- a/frappe_manager/site_manager/SiteManager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -370,11 +370,16 @@ def logs(self, follow, service: Optional[str] = None): """ richprint.change_head(f"Showing logs") - if service: - if self.site.is_service_running(service): - self.site.logs(service, follow) - else: - self.site.bench_dev_server_logs(follow) + try: + if service: + if self.site.is_service_running(service): + self.site.logs(service, follow) + else: + richprint.exit(f"Cannot show logs. [blue]{self.site.name}[/blue]'s compose service '{service}' not running!") + else: + self.site.bench_dev_server_logs(follow) + except KeyboardInterrupt: + richprint.stdout.print("Detected CTRL+C. Exiting.") def check_ports(self): """ From 9ea50d62ef1a0313af457d1e8df39e7d0a6ffdac Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 02:47:16 +0530 Subject: [PATCH 037/100] move Richprint to dedicated DisplayManager module --- frappe_manager/compose_manager/ComposeFile.py | 2 +- .../Richprint.py => display_manager/DisplayManager.py} | 7 ++++--- frappe_manager/docker_wrapper/DockerClient.py | 2 +- frappe_manager/docker_wrapper/utils.py | 4 ++-- frappe_manager/logger/log.py | 2 +- frappe_manager/main.py | 7 +++---- frappe_manager/site_manager/SiteManager.py | 2 +- frappe_manager/site_manager/site.py | 5 +++-- frappe_manager/site_manager/workers_manager/SiteWorker.py | 2 +- frappe_manager/utils.py | 2 +- 10 files changed, 18 insertions(+), 17 deletions(-) rename frappe_manager/{site_manager/Richprint.py => display_manager/DisplayManager.py} (98%) diff --git a/frappe_manager/compose_manager/ComposeFile.py b/frappe_manager/compose_manager/ComposeFile.py index fec4161d..0687d43a 100644 --- a/frappe_manager/compose_manager/ComposeFile.py +++ b/frappe_manager/compose_manager/ComposeFile.py @@ -4,7 +4,7 @@ from typing import List import typer -from frappe_manager.site_manager.Richprint import richprint +from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.compose_manager.utils import represent_none yaml.representer.ignore_aliases = lambda *args: True diff --git a/frappe_manager/site_manager/Richprint.py b/frappe_manager/display_manager/DisplayManager.py similarity index 98% rename from frappe_manager/site_manager/Richprint.py rename to frappe_manager/display_manager/DisplayManager.py index 8b877c40..c1fed410 100644 --- a/frappe_manager/site_manager/Richprint.py +++ b/frappe_manager/display_manager/DisplayManager.py @@ -17,7 +17,7 @@ 'errors': error }) -class Richprint: +class DisplayManager: def __init__(self): self.stdout = Console() # self.stderr = Console(stderr=True) @@ -224,7 +224,8 @@ def live_lines( # except DockerException: # self.update_live() # self.stop() - # raise + except KeyboardInterrupt as e: + richprint.live.refresh() except StopIteration: break @@ -236,4 +237,4 @@ def stop(self): self.live.update(Text('',end='')) self.live.stop() -richprint = Richprint() +richprint = DisplayManager() diff --git a/frappe_manager/docker_wrapper/DockerClient.py b/frappe_manager/docker_wrapper/DockerClient.py index fb14c86e..583dd224 100644 --- a/frappe_manager/docker_wrapper/DockerClient.py +++ b/frappe_manager/docker_wrapper/DockerClient.py @@ -2,7 +2,7 @@ import json from frappe_manager.docker_wrapper.DockerCompose import DockerComposeWrapper from pathlib import Path -from frappe_manager.site_manager.Richprint import richprint +from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.docker_wrapper.utils import ( is_current_user_in_group, parameters_to_options, diff --git a/frappe_manager/docker_wrapper/utils.py b/frappe_manager/docker_wrapper/utils.py index 13811b4f..dce65153 100644 --- a/frappe_manager/docker_wrapper/utils.py +++ b/frappe_manager/docker_wrapper/utils.py @@ -92,7 +92,7 @@ def run_command_with_exit_code( else: return stream_stdout_and_stderr(full_cmd) else: - from frappe_manager.site_manager.Richprint import richprint + from frappe_manager.display_manager.DisplayManager import richprint output = run(full_cmd) exit_code = output.returncode if exit_code != 0: @@ -138,7 +138,7 @@ def parameters_to_options(param: dict, exclude: list = []) -> list: def is_current_user_in_group(group_name) -> bool: """Check if the current user is in the given group""" - from frappe_manager.site_manager.Richprint import richprint + from frappe_manager.display_manager.DisplayManager import richprint import platform if platform.system() == 'Linux': diff --git a/frappe_manager/logger/log.py b/frappe_manager/logger/log.py index 9bae5685..0c6f7c2d 100644 --- a/frappe_manager/logger/log.py +++ b/frappe_manager/logger/log.py @@ -5,7 +5,7 @@ import shutil import gzip from typing import Dict, Optional, Union -from frappe_manager.site_manager.Richprint import richprint +from frappe_manager.display_manager.DisplayManager import richprint def namer(name): return name + ".gz" diff --git a/frappe_manager/main.py b/frappe_manager/main.py index 84cbf212..3f1d6bca 100644 --- a/frappe_manager/main.py +++ b/frappe_manager/main.py @@ -6,15 +6,14 @@ import shutil import atexit from typing import Annotated, List, Literal, Optional, Set -from frappe_manager.site_manager.manager import SiteManager -from frappe_manager.site_manager.Richprint import richprint +from frappe_manager.site_manager.SiteManager import SiteManager +from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager import CLI_DIR, default_extension, SiteServicesEnum from frappe_manager.logger import log -from frappe_manager.utils import check_update, remove_zombie_subprocess_process +from frappe_manager.utils import check_update, is_cli_help_called, remove_zombie_subprocess_process from frappe_manager.site_manager.utils import get_container_name_prefix app = typer.Typer(no_args_is_help=True,rich_markup_mode='rich') -global_service = None sites = None logger = None diff --git a/frappe_manager/site_manager/SiteManager.py b/frappe_manager/site_manager/SiteManager.py index 1079a215..0beffeca 100644 --- a/frappe_manager/site_manager/SiteManager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -8,7 +8,7 @@ import shutil from frappe_manager.site_manager.site import Site -from frappe_manager.site_manager.Richprint import richprint +from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager import CLI_DIR from frappe_manager.utils import ( diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 6fb12ab1..3cc3378a 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -8,7 +8,7 @@ from frappe_manager.docker_wrapper import DockerClient, DockerException from frappe_manager.compose_manager.ComposeFile import ComposeFile -from frappe_manager.site_manager.Richprint import richprint +from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.site_manager.workers_manager.SiteWorker import SiteWorkers from frappe_manager.site_manager.utils import log_file, get_container_name_prefix from frappe_manager.utils import host_run_cp @@ -88,7 +88,7 @@ def migrate_site_compose(self): labels = self.composefile.get_all_labels() # extrahosts = self.composefile.get_all_extrahosts() - # handle users, should be of the new invocation + # overwrite user for each invocation import os users = {"nginx":{ "uid": os.getuid(), @@ -111,6 +111,7 @@ def migrate_site_compose(self): self.composefile.set_version(fm_version) self.composefile.set_top_networks_name("site-network",get_container_name_prefix(self.name)) self.composefile.write_to_file() + status = True if status: richprint.print( diff --git a/frappe_manager/site_manager/workers_manager/SiteWorker.py b/frappe_manager/site_manager/workers_manager/SiteWorker.py index de3b8561..49f1c52f 100644 --- a/frappe_manager/site_manager/workers_manager/SiteWorker.py +++ b/frappe_manager/site_manager/workers_manager/SiteWorker.py @@ -4,7 +4,7 @@ from copy import deepcopy from frappe_manager.compose_manager.ComposeFile import ComposeFile #from frappe_manager.console_manager.Richprint import richprint -from frappe_manager.site_manager.Richprint import richprint +from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.site_manager.utils import get_container_name_prefix, log_file from frappe_manager.docker_wrapper import DockerClient, DockerException diff --git a/frappe_manager/utils.py b/frappe_manager/utils.py index be607d37..aed2e940 100644 --- a/frappe_manager/utils.py +++ b/frappe_manager/utils.py @@ -9,7 +9,7 @@ from frappe_manager.logger import log from frappe_manager.docker_wrapper.utils import process_opened from frappe_manager.docker_wrapper.DockerException import DockerException -from frappe_manager.site_manager.Richprint import richprint +from frappe_manager.display_manager.DisplayManager import richprint def remove_zombie_subprocess_process(): From 832803dfd5a3c09ea0625cbb021450c55a3ece27 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 04:56:42 +0530 Subject: [PATCH 038/100] more detail --- .../templates/docker-compose.tmpl | 107 +++++++++++++++--- 1 file changed, 89 insertions(+), 18 deletions(-) diff --git a/frappe_manager/compose_manager/templates/docker-compose.tmpl b/frappe_manager/compose_manager/templates/docker-compose.tmpl index fba1aa61..24ba98b7 100644 --- a/frappe_manager/compose_manager/templates/docker-compose.tmpl +++ b/frappe_manager/compose_manager/templates/docker-compose.tmpl @@ -1,33 +1,50 @@ version: "3.9" services: frappe: - image: frappe-manager-frappe:dev + # image: ghcr.io/rtcamp/frappe-manager-frappe:v0.10.0 + image: frappe-manager-frappe:v0.10.0 environment: - - SHELL=/bin/bash + ADMIN_PASS: REPLACE_ME_WITH_FRAPPE_WEB_ADMIN_PASS + # apps are defined as :, if branch name not given then default github branch will be used. + APPS_LIST: REPLACE_ME_APPS_LIST + DB_NAME: REPLACE_ME_WITH_DB_NAME_TO_CREATE + # DEVERLOPER_MODE bool -> true/false + DEVELOPER_MODE: REPLACE_ME_WITH_DEVELOPER_MODE_TOGGLE + FRAPPE_BRANCH: REPLACE_ME_WITH_BRANCH_OF_FRAPPE + SITENAME: REPLACE_ME_WITH_THE_SITE_NAME + USERGROUP: REPLACE_ME_WITH_CURRENT_USER_GROUP + USERID: REPLACE_ME_WITH_CURRENT_USER + MARIADB_ROOT_PASS: REPLACE_ME_WITH_MARIADB_ROOT_PASS volumes: - ./workspace:/workspace:cached - ports: + expose: - 80 - - 9000:9000 labels: devcontainer.metadata: '[{ "remoteUser": "frappe"}]' - extra_hosts: + networks: + site-network: nginx: - image: ghcr.io/rtcamp/frappe-manager-nginx:v0.8.3 + # image: ghcr.io/rtcamp/frappe-manager-nginx:v0.10.0 + image: frappe-manager-nginx:v0.10.0 + user: REPLACE_ME_WITH_CURRENT_USER:REPLACE_ME_WITH_CURRENT_USER_GROUP environment: + # not implemented as of now + ENABLE_SSL: REPLACE_ME_WITH_TOGGLE_ENABLE_SSL + SITENAME: REPLACE_ME_WITH_THE_SITE_NAME + # for nginx-proxy + VIRTUAL_HOST: REPLACE_ME_WITH_SITE_NAME + VIRTUAL_PORT: 80 volumes: - ./workspace:/workspace:cached - - ./certs:/etc/nginx/certs + - ./configs/nginx/conf:/etc/nginx + - ./configs/nginx/logs:/var/log/nginx + - ./configs/nginx/cache:/var/cache/nginx + - ./configs/nginx/run:/var/run ports: - 80:80 - - 443:443 - - mailhog: - image: ghcr.io/rtcamp/frappe-manager-mailhog:v0.8.3 - ports: - - 1025 - - 8025 + networks: + site-network: mariadb: image: mariadb:10.6 @@ -40,37 +57,91 @@ services: MYSQL_ROOT_PASSWORD: root volumes: - mariadb-data:/var/lib/mysql + networks: + site-network: + + mailhog: + image: ghcr.io/rtcamp/frappe-manager-mailhog:v0.8.3 + expose: + - 1025 + - 8025 + networks: + site-network: adminer: image: adminer:latest environment: - ADMINER_DEFAULT_SERVER=mariadb - ports: + expose: - 8080 + networks: + site-network: + + socketio: + image: frappe-manager-frappe:v0.10.0 + environment: + TIMEOUT: 60000 + CHANGE_DIR: /workspace/frappe-bench/logs + WAIT_FOR: '/workspace/frappe-bench/config/frappe-bench-node-socketio.fm.supervisor.conf' + COMMAND: | + ln -sfn /workspace/frappe-bench/config/frappe-bench-node-socketio.fm.supervisor.conf /opt/user/conf.d/frappe-bench-node-socketio.fm.supervisor.conf + supervisord -c /opt/user/supervisord.conf + expose: + - 80 + command: launch.sh + volumes: + - ./workspace:/workspace:cached + networks: + site-network: + + schedule: + image: frappe-manager-frappe:v0.10.0 + environment: + TIMEOUT: 60000 + CHANGE_DIR: /workspace/frappe-bench + WAIT_FOR: '/workspace/frappe-bench/config/frappe-bench-frappe-schedule.fm.supervisor.conf' + COMMAND: | + ln -sfn /workspace/frappe-bench/config/frappe-bench-frappe-schedule.fm.supervisor.conf /opt/user/conf.d/frappe-bench-frappe-schedule.fm.supervisor.conf + supervisord -c /opt/user/supervisord.conf + command: launch.sh + volumes: + - ./workspace:/workspace:cached + networks: + site-network: redis-cache: image: redis:alpine volumes: - redis-cache-data:/data - ports: + expose: - 6379 + networks: + site-network: redis-queue: image: redis:alpine volumes: - redis-queue-data:/data - ports: + expose: - 6379 + networks: + site-network: redis-socketio: image: redis:alpine volumes: - redis-socketio-data:/data - ports: + expose: - 6379 + networks: + site-network: volumes: mariadb-data: redis-socketio-data: redis-queue-data: redis-cache-data: + +networks: + site-network: + name: REPLACE_ME_WITH_SITE_NAME_NETWORK From 3d6dcc42767e3661c4be6b44e7bcc0702aac0322 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 04:59:28 +0530 Subject: [PATCH 039/100] show error msg when exit in this function --- frappe_manager/utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/frappe_manager/utils.py b/frappe_manager/utils.py index aed2e940..f7e41b9f 100644 --- a/frappe_manager/utils.py +++ b/frappe_manager/utils.py @@ -130,6 +130,7 @@ def host_run_cp(image: str, source: str, destination: str, docker, verbose=False richprint.change_head(f"{status_text} {source} -> {destination}") source_container_name = generate_random_text(10) dest_path = Path(destination) + errror_exception = None failed: bool = False # run the container @@ -144,6 +145,7 @@ def host_run_cp(image: str, source: str, destination: str, docker, verbose=False if not verbose: richprint.live_lines(output, padding=(0, 0, 0, 2)) except DockerException as e: + errror_exception = e failed = 0 if not failed: @@ -158,6 +160,7 @@ def host_run_cp(image: str, source: str, destination: str, docker, verbose=False if not verbose: richprint.live_lines(output, padding=(0, 0, 0, 2)) except DockerException as e: + errror_exception = e failed = 1 # # kill the container @@ -176,6 +179,7 @@ def host_run_cp(image: str, source: str, destination: str, docker, verbose=False if not verbose: richprint.live_lines(output, padding=(0, 0, 0, 2)) except DockerException as e: + errror_exception = e failed = 2 # check if the destination file exists @@ -194,7 +198,8 @@ def host_run_cp(image: str, source: str, destination: str, docker, verbose=False richprint.live_lines(output, padding=(0, 0, 0, 2)) except DockerException as e: pass - richprint.exit(f"{status_text} failed.") + # TODO introuduce custom exception to handle this type of cases where if the flow is not completed then it should raise exception which is handled by caller and then site creation check is done + richprint.exit(f"{status_text} failed.",error_msg=errror_exception) elif not Path(destination).exists(): richprint.exit(f"{status_text} failed. Copied {destination} not found.") From b5a776001aecaf52cc95d1b71f53e38ea9d24c34 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 05:00:51 +0530 Subject: [PATCH 040/100] chore: clean code --- frappe_manager/site_manager/site.py | 13 ------------- .../site_manager/workers_manager/SiteWorker.py | 7 +++++-- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 3cc3378a..a9e4dd57 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -447,19 +447,6 @@ def running(self) -> bool: :return: a boolean value. If the number of running containers is greater than or equal to the number of services listed in the compose file, it returns True. Otherwise, it returns False. """ - # try: - # output = self.docker.compose.ps(format='json',filter='running',stream=True) - # status: dict = {} - # for source,line in output: - # if source == 'stdout': - # status = json.loads(line.decode()) - # running_containers = len(status) - # if running_containers >= len(self.composefile.get_services_list()): - # return True - # return False - # except DockerException as e: - # richprint.exit(f"{e.stdout}{e.stderr}") - services = self.composefile.get_services_list() running_status = self.get_services_running_status() diff --git a/frappe_manager/site_manager/workers_manager/SiteWorker.py b/frappe_manager/site_manager/workers_manager/SiteWorker.py index 49f1c52f..6e8a2ddf 100644 --- a/frappe_manager/site_manager/workers_manager/SiteWorker.py +++ b/frappe_manager/site_manager/workers_manager/SiteWorker.py @@ -1,5 +1,4 @@ import importlib -import yaml import json from copy import deepcopy from frappe_manager.compose_manager.ComposeFile import ComposeFile @@ -147,6 +146,7 @@ def get_services_running_status(self)-> dict: except DockerException as e: richprint.exit(f"{e.stdout}{e.stderr}") + def running(self) -> bool: """ The `running` function checks if all the services defined in a Docker Compose file are running. @@ -158,7 +158,10 @@ def running(self) -> bool: if running_status: for service in services: - if not running_status[service] == 'running': + try: + if not running_status[service] == "running": + return False + except KeyError: return False else: return False From b1089e108ac85030fdcdbf4a9a54ee6de5045bf6 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 05:01:45 +0530 Subject: [PATCH 041/100] dep change pyyaml to ruamel.yaml --- frappe_manager/compose_manager/ComposeFile.py | 38 +++-- poetry.lock | 130 +++++++++++------- pyproject.toml | 2 +- 3 files changed, 103 insertions(+), 67 deletions(-) diff --git a/frappe_manager/compose_manager/ComposeFile.py b/frappe_manager/compose_manager/ComposeFile.py index 0687d43a..fb21efe0 100644 --- a/frappe_manager/compose_manager/ComposeFile.py +++ b/frappe_manager/compose_manager/ComposeFile.py @@ -1,13 +1,15 @@ import pkgutil from pathlib import Path -import yaml +from ruamel.yaml import YAML from typing import List import typer from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.compose_manager.utils import represent_none +yaml = YAML(typ='safe',pure=True) yaml.representer.ignore_aliases = lambda *args: True +yaml.representer.add_representer(type(None), represent_none) class ComposeFile: @@ -26,10 +28,10 @@ def init(self): # if the load file not found then the site not exits if self.exists(): with open(self.compose_path, "r") as f: - self.yml = yaml.safe_load(f) + self.yml = yaml.load(f) + else: - template = self.get_template(self.template_name) - self.yml = yaml.safe_load(template) + self.yml = self.load_template() self.is_template_loaded = True def exists(self): @@ -60,15 +62,20 @@ def get_template( """ file_name = f"{template_directory}/{file_name}" try: - data = pkgutil.get_data(__name__, file_name) + import pkg_resources + file_path = pkg_resources.resource_filename(__name__, file_name) + return file_path + #data = pkgutil.get_data(__name__, file_name) except Exception as e: richprint.exit(f"{file_name} template not found! Error:{e}") - yml = data.decode() - return yml + #yml = data def load_template(self): - template = self.get_template(self.template_name) - self.yml = yaml.safe_load(template) + template_path = self.get_template(self.template_name) + if template_path: + with open(template_path, "r") as f: + yml = yaml.load(f) + return yml def set_container_names(self, prefix): """ @@ -112,8 +119,7 @@ def is_services_name_same_as_template(self): :return: a boolean value indicating whether the list of service names in the current YAML file is the same as the list of service names in the template YAML file. """ - template = self.get_template(self.template_name) - template_yml = yaml.safe_load(template) + template_yml = self.load_template() template_service_name_list = list(template_yml["services"].keys()) template_service_name_list.sort() current_service_name_list = list(self.yml["services"].keys()) @@ -391,7 +397,9 @@ def write_to_file(self): The function writes the contents of a YAML object to a file. """ - # saving the docker compose to the directory - with open(self.compose_path, "w") as f: - yaml.add_representer(type(None), represent_none) - f.write(yaml.dump(self.yml, default_flow_style=False)) + try: + # saving the docker compose to the directory + with open(self.compose_path, "w") as f: + yaml.dump(self.yml, f) + except Exception as e: + richprint.exit(f"Error in writing compose file.",error_msg=e) diff --git a/poetry.lock b/poetry.lock index 3367e373..e7299a08 100644 --- a/poetry.lock +++ b/poetry.lock @@ -224,55 +224,6 @@ files = [ plugins = ["importlib-metadata"] windows-terminal = ["colorama (>=0.4.6)"] -[[package]] -name = "pyyaml" -version = "6.0.1" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.6" -files = [ - {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, - {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, - {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, - {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, - {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, - {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, - {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, - {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, - {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, -] - [[package]] name = "requests" version = "2.31.0" @@ -312,6 +263,83 @@ pygments = ">=2.13.0,<3.0.0" [package.extras] jupyter = ["ipywidgets (>=7.5.1,<9)"] +[[package]] +name = "ruamel-yaml" +version = "0.18.5" +description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruamel.yaml-0.18.5-py3-none-any.whl", hash = "sha256:a013ac02f99a69cdd6277d9664689eb1acba07069f912823177c5eced21a6ada"}, + {file = "ruamel.yaml-0.18.5.tar.gz", hash = "sha256:61917e3a35a569c1133a8f772e1226961bf5a1198bea7e23f06a0841dea1ab0e"}, +] + +[package.dependencies] +"ruamel.yaml.clib" = {version = ">=0.2.7", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.13\""} + +[package.extras] +docs = ["mercurial (>5.7)", "ryd"] +jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] + +[[package]] +name = "ruamel-yaml-clib" +version = "0.2.8" +description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" +optional = false +python-versions = ">=3.6" +files = [ + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b42169467c42b692c19cf539c38d4602069d8c1505e97b86387fcf7afb766e1d"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:07238db9cbdf8fc1e9de2489a4f68474e70dffcb32232db7c08fa61ca0c7c462"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:fff3573c2db359f091e1589c3d7c5fc2f86f5bdb6f24252c2d8e539d4e45f412"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:aa2267c6a303eb483de8d02db2871afb5c5fc15618d894300b88958f729ad74f"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:840f0c7f194986a63d2c2465ca63af8ccbbc90ab1c6001b1978f05119b5e7334"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:024cfe1fc7c7f4e1aff4a81e718109e13409767e4f871443cbff3dba3578203d"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-win32.whl", hash = "sha256:c69212f63169ec1cfc9bb44723bf2917cbbd8f6191a00ef3410f5a7fe300722d"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-win_amd64.whl", hash = "sha256:cabddb8d8ead485e255fe80429f833172b4cadf99274db39abc080e068cbcc31"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:bef08cd86169d9eafb3ccb0a39edb11d8e25f3dae2b28f5c52fd997521133069"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:b16420e621d26fdfa949a8b4b47ade8810c56002f5389970db4ddda51dbff248"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:25c515e350e5b739842fc3228d662413ef28f295791af5e5110b543cf0b57d9b"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux_2_24_aarch64.whl", hash = "sha256:1707814f0d9791df063f8c19bb51b0d1278b8e9a2353abbb676c2f685dee6afe"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:46d378daaac94f454b3a0e3d8d78cafd78a026b1d71443f4966c696b48a6d899"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:09b055c05697b38ecacb7ac50bdab2240bfca1a0c4872b0fd309bb07dc9aa3a9"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-win32.whl", hash = "sha256:53a300ed9cea38cf5a2a9b069058137c2ca1ce658a874b79baceb8f892f915a7"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-win_amd64.whl", hash = "sha256:c2a72e9109ea74e511e29032f3b670835f8a59bbdc9ce692c5b4ed91ccf1eedb"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:ebc06178e8821efc9692ea7544aa5644217358490145629914d8020042c24aa1"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:edaef1c1200c4b4cb914583150dcaa3bc30e592e907c01117c08b13a07255ec2"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d176b57452ab5b7028ac47e7b3cf644bcfdc8cacfecf7e71759f7f51a59e5c92"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux_2_24_aarch64.whl", hash = "sha256:1dc67314e7e1086c9fdf2680b7b6c2be1c0d8e3a8279f2e993ca2a7545fecf62"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3213ece08ea033eb159ac52ae052a4899b56ecc124bb80020d9bbceeb50258e9"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aab7fd643f71d7946f2ee58cc88c9b7bfc97debd71dcc93e03e2d174628e7e2d"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-win32.whl", hash = "sha256:5c365d91c88390c8d0a8545df0b5857172824b1c604e867161e6b3d59a827eaa"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-win_amd64.whl", hash = "sha256:1758ce7d8e1a29d23de54a16ae867abd370f01b5a69e1a3ba75223eaa3ca1a1b"}, + {file = "ruamel.yaml.clib-0.2.8-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a5aa27bad2bb83670b71683aae140a1f52b0857a2deff56ad3f6c13a017a26ed"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c58ecd827313af6864893e7af0a3bb85fd529f862b6adbefe14643947cfe2942"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_12_0_arm64.whl", hash = "sha256:f481f16baec5290e45aebdc2a5168ebc6d35189ae6fea7a58787613a25f6e875"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux_2_24_aarch64.whl", hash = "sha256:77159f5d5b5c14f7c34073862a6b7d34944075d9f93e681638f6d753606c6ce6"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:7f67a1ee819dc4562d444bbafb135832b0b909f81cc90f7aa00260968c9ca1b3"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4ecbf9c3e19f9562c7fdd462e8d18dd902a47ca046a2e64dba80699f0b6c09b7"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:87ea5ff66d8064301a154b3933ae406b0863402a799b16e4a1d24d9fbbcbe0d3"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-win32.whl", hash = "sha256:75e1ed13e1f9de23c5607fe6bd1aeaae21e523b32d83bb33918245361e9cc51b"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-win_amd64.whl", hash = "sha256:3f215c5daf6a9d7bbed4a0a4f760f3113b10e82ff4c5c44bec20a68c8014f675"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1b617618914cb00bf5c34d4357c37aa15183fa229b24767259657746c9077615"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:a6a9ffd280b71ad062eae53ac1659ad86a17f59a0fdc7699fd9be40525153337"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux_2_24_aarch64.whl", hash = "sha256:305889baa4043a09e5b76f8e2a51d4ffba44259f6b4c72dec8ca56207d9c6fe1"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:700e4ebb569e59e16a976857c8798aee258dceac7c7d6b50cab63e080058df91"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e2b4c44b60eadec492926a7270abb100ef9f72798e18743939bdbf037aab8c28"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e79e5db08739731b0ce4850bed599235d601701d5694c36570a99a0c5ca41a9d"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-win32.whl", hash = "sha256:955eae71ac26c1ab35924203fda6220f84dce57d6d7884f189743e2abe3a9fbe"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-win_amd64.whl", hash = "sha256:56f4252222c067b4ce51ae12cbac231bce32aee1d33fbfc9d17e5b8d6966c312"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:03d1162b6d1df1caa3a4bd27aa51ce17c9afc2046c31b0ad60a0a96ec22f8001"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:bba64af9fa9cebe325a62fa398760f5c7206b215201b0ec825005f1b18b9bccf"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux_2_24_aarch64.whl", hash = "sha256:a1a45e0bb052edf6a1d3a93baef85319733a888363938e1fc9924cb00c8df24c"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:da09ad1c359a728e112d60116f626cc9f29730ff3e0e7db72b9a2dbc2e4beed5"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:184565012b60405d93838167f425713180b949e9d8dd0bbc7b49f074407c5a8b"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a75879bacf2c987c003368cf14bed0ffe99e8e85acfa6c0bfffc21a090f16880"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-win32.whl", hash = "sha256:84b554931e932c46f94ab306913ad7e11bba988104c5cff26d90d03f68258cd5"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-win_amd64.whl", hash = "sha256:25ac8c08322002b06fa1d49d1646181f0b2c72f5cbc15a85e80b4c30a544bb15"}, + {file = "ruamel.yaml.clib-0.2.8.tar.gz", hash = "sha256:beb2e0404003de9a4cab9753a8805a8fe9320ee6673136ed7f04255fe60bb512"}, +] + [[package]] name = "shellingham" version = "1.5.4" @@ -376,5 +404,5 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.0" -python-versions = "^3.11" -content-hash = "af0187b6dde08ac088ad9254616fb72bd89ecb9e8090e729b3edc69882519e97" +python-versions = "^3.10" +content-hash = "2c35b61a8ad02e0fb3b4327845b8f8ba44abd8b830d92e81c98a13832ce85cf3" diff --git a/pyproject.toml b/pyproject.toml index a9b0f4e6..93430a0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,9 +18,9 @@ fm = "frappe_manager.main:cli_entrypoint" [tool.poetry.dependencies] python = "^3.10" typer = {extras = ["all"], version = "^0.9.0"} -pyyaml = "^6.0.1" requests = "^2.31.0" psutil = "^5.9.6" +ruamel-yaml = "^0.18.5" [build-system] From fe2a5790cc8a5932ac00c7fe30fb38521cbc8cb2 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 05:03:11 +0530 Subject: [PATCH 042/100] update images tags to v0.10.0 --- .../compose_manager/templates/docker-compose.tmpl | 10 ++++------ .../templates/docker-compose.workers.tmpl | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/frappe_manager/compose_manager/templates/docker-compose.tmpl b/frappe_manager/compose_manager/templates/docker-compose.tmpl index 24ba98b7..dd341938 100644 --- a/frappe_manager/compose_manager/templates/docker-compose.tmpl +++ b/frappe_manager/compose_manager/templates/docker-compose.tmpl @@ -1,8 +1,7 @@ version: "3.9" services: frappe: - # image: ghcr.io/rtcamp/frappe-manager-frappe:v0.10.0 - image: frappe-manager-frappe:v0.10.0 + image: ghcr.io/rtcamp/frappe-manager-frappe:v0.10.0 environment: ADMIN_PASS: REPLACE_ME_WITH_FRAPPE_WEB_ADMIN_PASS # apps are defined as :, if branch name not given then default github branch will be used. @@ -25,8 +24,7 @@ services: site-network: nginx: - # image: ghcr.io/rtcamp/frappe-manager-nginx:v0.10.0 - image: frappe-manager-nginx:v0.10.0 + image: ghcr.io/rtcamp/frappe-manager-nginx:v0.10.0 user: REPLACE_ME_WITH_CURRENT_USER:REPLACE_ME_WITH_CURRENT_USER_GROUP environment: # not implemented as of now @@ -78,7 +76,7 @@ services: site-network: socketio: - image: frappe-manager-frappe:v0.10.0 + image: ghcr.io/rtcamp/frappe-manager-frappe:v0.10.0 environment: TIMEOUT: 60000 CHANGE_DIR: /workspace/frappe-bench/logs @@ -95,7 +93,7 @@ services: site-network: schedule: - image: frappe-manager-frappe:v0.10.0 + image: ghcr.io/rtcamp/frappe-manager-frappe:v0.10.0 environment: TIMEOUT: 60000 CHANGE_DIR: /workspace/frappe-bench diff --git a/frappe_manager/compose_manager/templates/docker-compose.workers.tmpl b/frappe_manager/compose_manager/templates/docker-compose.workers.tmpl index d507dd79..109d4525 100644 --- a/frappe_manager/compose_manager/templates/docker-compose.workers.tmpl +++ b/frappe_manager/compose_manager/templates/docker-compose.workers.tmpl @@ -1,6 +1,6 @@ services: worker-name: - image: frappe-manager-frappe:v0.9.3 + image: ghcr.io/rtcamp/frappe-manager-frappe:v0.10.0 environment: TIMEOUT: 6000 CHANGE_DIR: /workspace/frappe-bench From ff94721c8edab378d95776c9225768d0f8e37611 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 05:03:56 +0530 Subject: [PATCH 043/100] add owrker service status info in fm info command --- frappe_manager/site_manager/SiteManager.py | 60 ++++++++++++++++++---- 1 file changed, 51 insertions(+), 9 deletions(-) diff --git a/frappe_manager/site_manager/SiteManager.py b/frappe_manager/site_manager/SiteManager.py index 0beffeca..58642911 100644 --- a/frappe_manager/site_manager/SiteManager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -521,18 +521,60 @@ def info(self): site_services_table.add_row(first_service_table,second_service_table) - hints_table = Table(show_lines=True, show_header=False, highlight=True, expand=True,show_edge=True, box=None,padding=(1,0,0,0)) - hints_table.add_column("First",justify="center",no_wrap=True) - hints_table.add_column("Second",justify="center",ratio=8,no_wrap=True) - hints_table.add_column("Third",justify="center",ratio=8,no_wrap=True) - hints_table.add_row(":light_bulb:",f":green_square: -> Active", f":red_square: -> Inactive") + site_info_table.add_row("Site Services", site_services_table) - site_services_table_group = Group(site_services_table,hints_table) - site_info_table.add_row("Site Services", site_services_table_group) + running_site_services = self.site.workers.get_services_running_status() + if running_site_services: + worker_services_table = Table(show_lines=False, show_edge=False, pad_edge=False, show_header=False,expand=True,box=None) + worker_services_table.add_column("Service Status",ratio=1,no_wrap=True,width=None,min_width=20) + worker_services_table.add_column("Service Status",ratio=1,no_wrap=True,width=None,min_width=20) + + index = 0 + while index < len(running_site_services): + first_service_table = None + second_service_table = None + + try: + first_service = list(running_site_services.keys())[index] + index += 1 + except IndexError: + pass + first_service= None + try: + second_service = list(running_site_services.keys())[index] + index += 1 + except IndexError: + second_service = None + + # Fist Coloumn + if first_service: + first_service_table = Table(show_lines=False, show_header=False, highlight=True, expand=True,box=None) + first_service_table.add_column("Service",justify="left",no_wrap=True) + first_service_table.add_column("Status",justify="right",no_wrap=True) + first_service_table.add_row(f"{first_service}", f"{':green_square:' if running_site_services[first_service] == 'running' else ':red_square:'}") + + # Fist Coloumn + if second_service: + second_service_table = Table(show_lines=False, show_header=False, highlight=True, expand=True,box=None) + second_service_table.add_column("Service",justify="left",no_wrap=True,) + second_service_table.add_column("Status",justify="right",no_wrap=True) + second_service_table.add_row(f"{second_service}", f"{':green_square:' if running_site_services[second_service] == 'running' else ':red_square:'}") + + worker_services_table.add_row(first_service_table,second_service_table) + + # hints_table = Table(show_lines=True, show_header=False, highlight=True, expand=True,show_edge=True, box=None,padding=(1,0,0,0)) + # hints_table.add_column("First",justify="center",no_wrap=True) + # hints_table.add_column("Second",justify="center",ratio=8,no_wrap=True) + # hints_table.add_column("Third",justify="center",ratio=8,no_wrap=True) + # hints_table.add_row(":light_bulb:",f":green_square: -> Active", f":red_square: -> Inactive") + + # worker_services_table_group = Group(worker_services_table,hints_table) + site_info_table.add_row("Worker Services", worker_services_table) - richprint.stdout.print(site_info_table) - richprint.print(f"Run 'fm list' to list all available sites.",emoji_code=':light_bulb:') + richprint.stdout.print(site_info_table) + richprint.print(f":green_square: -> Active :red_square: -> Inactive",emoji_code=':information:') + richprint.print(f"Run 'fm list' to list all available sites.",emoji_code=':light_bulb:') def migrate_site(self): """ From 9fc19398c27a42919f6eb0e1bdb5a1de2cfea997 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 05:04:30 +0530 Subject: [PATCH 044/100] add __init__.py to workers_manager --- frappe_manager/site_manager/workers_manager/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 frappe_manager/site_manager/workers_manager/__init__.py diff --git a/frappe_manager/site_manager/workers_manager/__init__.py b/frappe_manager/site_manager/workers_manager/__init__.py new file mode 100644 index 00000000..e69de29b From f9371b94d9f1cb8535dabf2f33646474e791cf7c Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 05:27:24 +0530 Subject: [PATCH 045/100] update docker imag tag --- Docker/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/build.sh b/Docker/build.sh index 22cfc69c..0440b6ef 100755 --- a/Docker/build.sh +++ b/Docker/build.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -TAG='v0.9.0' +TAG='v0.10.0' ARCH=$(uname -m) # arm64 From 58de7c9ba5e3f15f7ff31b1f486309194d221945 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 05:27:59 +0530 Subject: [PATCH 046/100] create symlink for old frappe dev server log --- Docker/frappe/user-script.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/Docker/frappe/user-script.sh b/Docker/frappe/user-script.sh index 463c8976..c7ac6659 100755 --- a/Docker/frappe/user-script.sh +++ b/Docker/frappe/user-script.sh @@ -9,6 +9,7 @@ emer() { if [[ ! -d 'logs' ]]; then mkdir -p logs + ln -sfn ../frappe-bench/logs/web.dev.log logs/web.dev.log fi REDIS_SOCKETIO_PORT=80 From 286a8aa24b4a1a7bc6668b7607fce90daa0ab06e Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 13:08:23 +0530 Subject: [PATCH 047/100] fix dynamic worker generation --- Docker/frappe/user-script.sh | 5 +++-- frappe_manager/site_manager/SiteManager.py | 1 + frappe_manager/site_manager/site.py | 2 +- frappe_manager/site_manager/workers_manager/SiteWorker.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Docker/frappe/user-script.sh b/Docker/frappe/user-script.sh index c7ac6659..a9e82880 100755 --- a/Docker/frappe/user-script.sh +++ b/Docker/frappe/user-script.sh @@ -98,7 +98,8 @@ if [[ ! -d "frappe-bench" ]]; then host_changed=$(echo "$bench_serve_help_output" | grep -c 'host' || true) - SUPERVIOSRCONFIG_STATUS=$(bench setup supervisor --skip-redis --skip-supervisord --yes --user "$USER") + # SUPERVIOSRCONFIG_STATUS=$(bench setup supervisor --skip-redis --skip-supervisord --yes --user "$USER") + bench setup supervisor --skip-redis --skip-supervisord --yes --user "$USER" /scripts/divide-supervisor-conf.py config/supervisor.conf @@ -142,7 +143,7 @@ else host_changed=$(echo "$bench_serve_help_output" | grep -c 'host' || true) - SUPERVIOSRCONFIG_STATUS=$(bench setup supervisor --skip-redis --skip-supervisord --yes --user "$USER") + bench setup supervisor --skip-redis --skip-supervisord --yes --user "$USER" /scripts/divide-supervisor-conf.py config/supervisor.conf diff --git a/frappe_manager/site_manager/SiteManager.py b/frappe_manager/site_manager/SiteManager.py index 58642911..fcf80845 100644 --- a/frappe_manager/site_manager/SiteManager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -285,6 +285,7 @@ def start_site(self): # start the provided site self.migrate_site() self.site.pull() + self.site.sync_workers_compose() self.site.start() def attach_to_site(self, user: str, extensions: List[str]): diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index a9e4dd57..7fbf340b 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -97,7 +97,7 @@ def migrate_site_compose(self): } # load template - self.composefile.load_template() + self.composefile.yml = self.composefile.load_template() # set all the payload self.composefile.set_all_envs(envs) diff --git a/frappe_manager/site_manager/workers_manager/SiteWorker.py b/frappe_manager/site_manager/workers_manager/SiteWorker.py index 6e8a2ddf..c3b50657 100644 --- a/frappe_manager/site_manager/workers_manager/SiteWorker.py +++ b/frappe_manager/site_manager/workers_manager/SiteWorker.py @@ -67,7 +67,7 @@ def generate_compose(self): richprint.print("Workers configuration changed. Recreating compose...") # create compose file for workers - self.composefile.load_template() + self.composefile.yml = self.composefile.load_template() template_worker_config = self.composefile.yml['services']['worker-name'] From d41a13ff59b1541900507b3a267eadd0b173b953 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 13:08:50 +0530 Subject: [PATCH 048/100] stop extra_hosts setup in workers compose --- frappe_manager/site_manager/workers_manager/SiteWorker.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/frappe_manager/site_manager/workers_manager/SiteWorker.py b/frappe_manager/site_manager/workers_manager/SiteWorker.py index c3b50657..dd8532b9 100644 --- a/frappe_manager/site_manager/workers_manager/SiteWorker.py +++ b/frappe_manager/site_manager/workers_manager/SiteWorker.py @@ -86,9 +86,6 @@ def generate_compose(self): worker_config['environment']['USERID'] = os.getuid() worker_config['environment']['USERGROUP'] = os.getgid() - # setting extrahosts - worker_config['extra_hosts'] = [f'{self.site_name}:127.0.0.1'] - self.composefile.yml['services'][worker] = worker_config self.composefile.set_container_names(get_container_name_prefix(self.site_name)) From 4885fe176a50bdb43639e809a779d0b5da8bca92 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 13:25:36 +0530 Subject: [PATCH 049/100] don't log when help is called --- frappe_manager/main.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/frappe_manager/main.py b/frappe_manager/main.py index 3f1d6bca..cc906aa9 100644 --- a/frappe_manager/main.py +++ b/frappe_manager/main.py @@ -27,16 +27,6 @@ def exit_cleanup(): richprint.stop() def cli_entrypoint(): - # logging - global logger - logger = log.get_logger() - logger.info('') - logger.info(f"{':'*20}FM Invoked{':'*20}") - logger.info('') - - # logging command provided by user - logger.info(f"RUNNING COMMAND: {' '.join(sys.argv[1:])}") - logger.info('-'*20) try: app() except Exception as e: @@ -65,16 +55,19 @@ def app_callback( """ FrappeManager for creating frappe development envrionments. """ + + ctx.obj = {} help_called = is_cli_help_called(ctx) ctx.obj["is_help_called"] = help_called if not help_called: - richprint.start(f"Working") sitesdir = CLI_DIR / 'sites' - # Checks for cli directory + + richprint.start(f"Working") + if not CLI_DIR.exists(): # creating the sites dir # TODO check if it's writeable and readable -> by writing a file to it and catching exception @@ -85,6 +78,17 @@ def app_callback( if not CLI_DIR.is_dir(): richprint.exit("Sites directory is not a directory! Aborting!") + # logging + global logger + logger = log.get_logger() + logger.info('') + logger.info(f"{':'*20}FM Invoked{':'*20}") + logger.info('') + + # logging command provided by user + logger.info(f"RUNNING COMMAND: {' '.join(sys.argv[1:])}") + logger.info('-'*20) + # Migration for directory change from CLI_DIR to CLI_DIR/sites # TODO remove when not required, introduced in 0.8.4 if not sitesdir.exists(): From 4775be5445a0f90c34a3e1fbaee2db668541b8a2 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 13:26:20 +0530 Subject: [PATCH 050/100] update: docker group not found status msg --- frappe_manager/docker_wrapper/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe_manager/docker_wrapper/utils.py b/frappe_manager/docker_wrapper/utils.py index dce65153..af4197b9 100644 --- a/frappe_manager/docker_wrapper/utils.py +++ b/frappe_manager/docker_wrapper/utils.py @@ -152,7 +152,7 @@ def is_current_user_in_group(group_name) -> bool: if current_user in docker_group_members: return True else: - richprint.error(f"Your current user [blue][b] {current_user} [/b][/blue] is not in the group 'docker'. Please add it to the group and restart your terminal.") + richprint.error(f"Your current user [blue][b] {current_user} [/b][/blue] is not in the 'docker' group. Please add it and restart your terminal.") return False except KeyError: richprint.error(f"The group '{group_name}' does not exist. Please create it and add your current user [blue][b] {current_user} [/b][/blue] to it.") From 8b50cf98fcd70ce59ed00f3a0ea7d7a204b72a9c Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 16:45:42 +0530 Subject: [PATCH 051/100] fix: running services detection --- frappe_manager/site_manager/site.py | 8 ++++++-- .../site_manager/workers_manager/SiteWorker.py | 18 ++++++++++++------ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 7fbf340b..325734e7 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -469,10 +469,14 @@ def get_services_running_status(self) -> dict: output = self.docker.compose.ps( service=services, format="json", all=True, stream=True ) - status: dict = {} + status: list = [] for source, line in output: if source == "stdout": - status = json.loads(line.decode()) + current_status = json.loads(line.decode()) + if type(current_status) == dict: + status.append(current_status) + else: + status += current_status # this is done to exclude docker runs using docker compose run command for container in status: diff --git a/frappe_manager/site_manager/workers_manager/SiteWorker.py b/frappe_manager/site_manager/workers_manager/SiteWorker.py index dd8532b9..162ef849 100644 --- a/frappe_manager/site_manager/workers_manager/SiteWorker.py +++ b/frappe_manager/site_manager/workers_manager/SiteWorker.py @@ -124,21 +124,27 @@ def stop(self) -> bool: except DockerException as e: richprint.exit(f"{status_text}: Failed") - def get_services_running_status(self)-> dict: + def get_services_running_status(self) -> dict: services = self.composefile.get_services_list() containers = self.composefile.get_container_names().values() services_status = {} try: - output = self.docker.compose.ps(service=services,format="json",all=True,stream=True) - status: dict = {} + output = self.docker.compose.ps( + service=services, format="json", all=True, stream=True + ) + status: list = [] for source, line in output: if source == "stdout": - status = json.loads(line.decode()) + current_status = json.loads(line.decode()) + if type(current_status) == dict: + status.append(current_status) + else: + status += current_status # this is done to exclude docker runs using docker compose run command for container in status: - if container['Name'] in containers: - services_status[container['Service']] = container['State'] + if container["Name"] in containers: + services_status[container["Service"]] = container["State"] return services_status except DockerException as e: richprint.exit(f"{e.stdout}{e.stderr}") From d0957e481b5b252c1dc8ff0d16a793cece4050b5 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 16:46:33 +0530 Subject: [PATCH 052/100] site compose convert null to empty string --- frappe_manager/compose_manager/ComposeFile.py | 12 +++++++----- frappe_manager/compose_manager/utils.py | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/frappe_manager/compose_manager/ComposeFile.py b/frappe_manager/compose_manager/ComposeFile.py index fb21efe0..869c6dbf 100644 --- a/frappe_manager/compose_manager/ComposeFile.py +++ b/frappe_manager/compose_manager/ComposeFile.py @@ -5,11 +5,14 @@ import typer from frappe_manager.display_manager.DisplayManager import richprint -from frappe_manager.compose_manager.utils import represent_none +from frappe_manager.compose_manager.utils import represent_null_empty yaml = YAML(typ='safe',pure=True) yaml.representer.ignore_aliases = lambda *args: True -yaml.representer.add_representer(type(None), represent_none) +#yaml.representer.add_representer(type(None), represent_none) + +# Set the default flow style to None to preserve the null representation +yaml.default_flow_style = None class ComposeFile: @@ -68,7 +71,6 @@ def get_template( #data = pkgutil.get_data(__name__, file_name) except Exception as e: richprint.exit(f"{file_name} template not found! Error:{e}") - #yml = data def load_template(self): template_path = self.get_template(self.template_name) @@ -392,14 +394,14 @@ def get_extrahosts(self, container: str) -> list: except KeyError: return None + def write_to_file(self): """ The function writes the contents of a YAML object to a file. """ - try: # saving the docker compose to the directory with open(self.compose_path, "w") as f: - yaml.dump(self.yml, f) + yaml.dump(self.yml, f,transform=represent_null_empty) except Exception as e: richprint.exit(f"Error in writing compose file.",error_msg=e) diff --git a/frappe_manager/compose_manager/utils.py b/frappe_manager/compose_manager/utils.py index 9f832354..0275cc41 100644 --- a/frappe_manager/compose_manager/utils.py +++ b/frappe_manager/compose_manager/utils.py @@ -1,4 +1,4 @@ -def represent_none(self, _): +def represent_null_empty(self, s): """ The function `represent_none` represents the value `None` as a null scalar in YAML format. @@ -7,4 +7,4 @@ def represent_none(self, _): :return: a representation of `None` as a YAML scalar with the tag `tag:yaml.org,2002:null` and an empty string as its value. """ - return self.represent_scalar('tag:yaml.org,2002:null', '') + return s.replace("null","") From ba1086342e54405232eac6acc4c7146bca99b80e Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 19:00:16 +0530 Subject: [PATCH 053/100] fix: fm info generate workers info --- frappe_manager/site_manager/SiteManager.py | 98 +++++++++++----------- 1 file changed, 48 insertions(+), 50 deletions(-) diff --git a/frappe_manager/site_manager/SiteManager.py b/frappe_manager/site_manager/SiteManager.py index fcf80845..7996109d 100644 --- a/frappe_manager/site_manager/SiteManager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -485,9 +485,6 @@ def info(self): site_services_table = Table(show_lines=False, show_edge=False, pad_edge=False, show_header=False,expand=True,box=None) site_services_table.add_column("Service Status",ratio=1,no_wrap=True,width=None,min_width=20) site_services_table.add_column("Service Status",ratio=1,no_wrap=True,width=None,min_width=20) - # site_services_table.add_column("Service Status",ratio=1,no_wrap=True) - - #half_of_running_site_services = len(running_site_services) // 2 index = 0 while index < len(running_site_services): @@ -525,53 +522,54 @@ def info(self): site_info_table.add_row("Site Services", site_services_table) - running_site_services = self.site.workers.get_services_running_status() - if running_site_services: - worker_services_table = Table(show_lines=False, show_edge=False, pad_edge=False, show_header=False,expand=True,box=None) - worker_services_table.add_column("Service Status",ratio=1,no_wrap=True,width=None,min_width=20) - worker_services_table.add_column("Service Status",ratio=1,no_wrap=True,width=None,min_width=20) - - index = 0 - while index < len(running_site_services): - first_service_table = None - second_service_table = None - - try: - first_service = list(running_site_services.keys())[index] - index += 1 - except IndexError: - pass - first_service= None - try: - second_service = list(running_site_services.keys())[index] - index += 1 - except IndexError: - second_service = None - - # Fist Coloumn - if first_service: - first_service_table = Table(show_lines=False, show_header=False, highlight=True, expand=True,box=None) - first_service_table.add_column("Service",justify="left",no_wrap=True) - first_service_table.add_column("Status",justify="right",no_wrap=True) - first_service_table.add_row(f"{first_service}", f"{':green_square:' if running_site_services[first_service] == 'running' else ':red_square:'}") - - # Fist Coloumn - if second_service: - second_service_table = Table(show_lines=False, show_header=False, highlight=True, expand=True,box=None) - second_service_table.add_column("Service",justify="left",no_wrap=True,) - second_service_table.add_column("Status",justify="right",no_wrap=True) - second_service_table.add_row(f"{second_service}", f"{':green_square:' if running_site_services[second_service] == 'running' else ':red_square:'}") - - worker_services_table.add_row(first_service_table,second_service_table) - - # hints_table = Table(show_lines=True, show_header=False, highlight=True, expand=True,show_edge=True, box=None,padding=(1,0,0,0)) - # hints_table.add_column("First",justify="center",no_wrap=True) - # hints_table.add_column("Second",justify="center",ratio=8,no_wrap=True) - # hints_table.add_column("Third",justify="center",ratio=8,no_wrap=True) - # hints_table.add_row(":light_bulb:",f":green_square: -> Active", f":red_square: -> Inactive") - - # worker_services_table_group = Group(worker_services_table,hints_table) - site_info_table.add_row("Worker Services", worker_services_table) + if self.site.workers.exists(): + running_site_services = self.site.workers.get_services_running_status() + if running_site_services: + worker_services_table = Table(show_lines=False, show_edge=False, pad_edge=False, show_header=False,expand=True,box=None) + worker_services_table.add_column("Service Status",ratio=1,no_wrap=True,width=None,min_width=20) + worker_services_table.add_column("Service Status",ratio=1,no_wrap=True,width=None,min_width=20) + + index = 0 + while index < len(running_site_services): + first_service_table = None + second_service_table = None + + try: + first_service = list(running_site_services.keys())[index] + index += 1 + except IndexError: + pass + first_service= None + try: + second_service = list(running_site_services.keys())[index] + index += 1 + except IndexError: + second_service = None + + # Fist Coloumn + if first_service: + first_service_table = Table(show_lines=False, show_header=False, highlight=True, expand=True,box=None) + first_service_table.add_column("Service",justify="left",no_wrap=True) + first_service_table.add_column("Status",justify="right",no_wrap=True) + first_service_table.add_row(f"{first_service}", f"{':green_square:' if running_site_services[first_service] == 'running' else ':red_square:'}") + + # Fist Coloumn + if second_service: + second_service_table = Table(show_lines=False, show_header=False, highlight=True, expand=True,box=None) + second_service_table.add_column("Service",justify="left",no_wrap=True,) + second_service_table.add_column("Status",justify="right",no_wrap=True) + second_service_table.add_row(f"{second_service}", f"{':green_square:' if running_site_services[second_service] == 'running' else ':red_square:'}") + + worker_services_table.add_row(first_service_table,second_service_table) + + # hints_table = Table(show_lines=True, show_header=False, highlight=True, expand=True,show_edge=True, box=None,padding=(1,0,0,0)) + # hints_table.add_column("First",justify="center",no_wrap=True) + # hints_table.add_column("Second",justify="center",ratio=8,no_wrap=True) + # hints_table.add_column("Third",justify="center",ratio=8,no_wrap=True) + # hints_table.add_row(":light_bulb:",f":green_square: -> Active", f":red_square: -> Inactive") + + # worker_services_table_group = Group(worker_services_table,hints_table) + site_info_table.add_row("Worker Services", worker_services_table) richprint.stdout.print(site_info_table) richprint.print(f":green_square: -> Active :red_square: -> Inactive",emoji_code=':information:') From 958e310e1d370caef30fd95ae164b3c2ba277f28 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 19:09:28 +0530 Subject: [PATCH 054/100] fix: fm info not showing anything when workers not available --- frappe_manager/site_manager/SiteManager.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/frappe_manager/site_manager/SiteManager.py b/frappe_manager/site_manager/SiteManager.py index 7996109d..6f1f3965 100644 --- a/frappe_manager/site_manager/SiteManager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -481,6 +481,7 @@ def info(self): # running site services status running_site_services = self.site.get_services_running_status() + if running_site_services: site_services_table = Table(show_lines=False, show_edge=False, pad_edge=False, show_header=False,expand=True,box=None) site_services_table.add_column("Service Status",ratio=1,no_wrap=True,width=None,min_width=20) @@ -571,9 +572,9 @@ def info(self): # worker_services_table_group = Group(worker_services_table,hints_table) site_info_table.add_row("Worker Services", worker_services_table) - richprint.stdout.print(site_info_table) - richprint.print(f":green_square: -> Active :red_square: -> Inactive",emoji_code=':information:') - richprint.print(f"Run 'fm list' to list all available sites.",emoji_code=':light_bulb:') + richprint.stdout.print(site_info_table) + richprint.print(f":green_square: -> Active :red_square: -> Inactive",emoji_code=':information:') + richprint.print(f"Run 'fm list' to list all available sites.",emoji_code=':light_bulb:') def migrate_site(self): """ From 04915d12e51f74171107a622fb9e1d3fd24a03ec Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 19:12:56 +0530 Subject: [PATCH 055/100] fix: alignment --- frappe_manager/site_manager/SiteManager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe_manager/site_manager/SiteManager.py b/frappe_manager/site_manager/SiteManager.py index 6f1f3965..e18ab579 100644 --- a/frappe_manager/site_manager/SiteManager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -573,7 +573,7 @@ def info(self): site_info_table.add_row("Worker Services", worker_services_table) richprint.stdout.print(site_info_table) - richprint.print(f":green_square: -> Active :red_square: -> Inactive",emoji_code=':information:') + richprint.print(f":green_square: -> Active :red_square: -> Inactive",emoji_code=':information: ') richprint.print(f"Run 'fm list' to list all available sites.",emoji_code=':light_bulb:') def migrate_site(self): From b20e3583bb29fcd00a0d291b2d12b66bac8da9cc Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 19:15:51 +0530 Subject: [PATCH 056/100] fix: wrong parameter --- frappe_manager/compose_manager/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe_manager/compose_manager/utils.py b/frappe_manager/compose_manager/utils.py index 0275cc41..fbc16283 100644 --- a/frappe_manager/compose_manager/utils.py +++ b/frappe_manager/compose_manager/utils.py @@ -1,4 +1,4 @@ -def represent_null_empty(self, s): +def represent_null_empty(s): """ The function `represent_none` represents the value `None` as a null scalar in YAML format. From 8f6ea18cb73ae14642a11e8421d68fbbd770874b Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 20:34:19 +0530 Subject: [PATCH 057/100] add supervisor config regeneration function --- frappe_manager/site_manager/site.py | 52 ++++++++++++++++++- .../workers_manager/SiteWorker.py | 5 +- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 325734e7..d97e7fb6 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -525,10 +525,60 @@ def is_service_running(self, service): return False def sync_workers_compose(self): - are_workers_not_changed = self.workers.is_expected_worker_same_as_template() if not are_workers_not_changed: self.workers.generate_compose() self.workers.start() else: richprint.print("Workers configuration remains unchanged.") + + + def regenerate_supervisor_conf(self): + richprint.change_head("Regenerating supervisor.conf.") + backup = False + # take backup + if self.workers.supervisor_config_path.exists(): + shutil.copy(self.workers.supervisor_config_path, self.workers.supervisor_config_path.parent / "supervisor.conf.bak") + for file_path in self.workers.config_dir.iterdir(): + file_path_abs = str(file_path.absolute()) + if file_path.is_file(): + if '.workers.fm.supervisor.conf' in file_path_abs: + shutil.copy(file_path, file_path.parent / f"{file_path.name}.bak") + backup = True + + # generate the supervisor.conf + try: + bench_setup_supervisor_command = 'bench setup supervisor --skip-redis --skip-supervisord --yes --user frappe' + + output = self.docker.compose.exec( + service='frappe', + command=bench_setup_supervisor_command, + stream=True, + user='frappe', + workdir='/workspace/frappe-bench' + ) + richprint.live_lines(output, padding=(0, 0, 0, 2)) + + generate_split_config_command = '/scripts/divide-supervisor-conf.py config/supervisor.conf' + + output = self.docker.compose.exec( + service='frappe', + command=generate_split_config_command , + stream=True, + user='frappe', + workdir='/workspace/frappe-bench' + ) + richprint.live_lines(output, padding=(0, 0, 0, 2)) + return True + except DockerException as e: + richprint.error("Failure in generating, supervisor.conf file.") + if backup: + richprint.print("Rolling back to previous workers configuration.") + shutil.copy(self.workers.supervisor_config_path.parent / "supervisor.conf.bak", self.workers.supervisor_config_path) + + for file_path in self.workers.config_dir.iterdir(): + file_path_abs = str(file_path.absolute()) + if file_path.is_file(): + if '.workers.fm.supervisor.conf.bak' in file_path_abs: + shutil.copy(file_path, file_path.parent / f"{file_path.name}".replace(".back","")) + return False diff --git a/frappe_manager/site_manager/workers_manager/SiteWorker.py b/frappe_manager/site_manager/workers_manager/SiteWorker.py index 162ef849..3bc64cee 100644 --- a/frappe_manager/site_manager/workers_manager/SiteWorker.py +++ b/frappe_manager/site_manager/workers_manager/SiteWorker.py @@ -11,6 +11,7 @@ class SiteWorkers: def __init__(self,site_path, site_name, quiet: bool = True): self.compose_path = site_path / "docker-compose.workers.yml" self.config_dir = site_path / 'workspace' / 'frappe-bench' / 'config' + self.supervisor_config_path = self.config_dir / 'supervisor.conf' self.site_name = site_name self.quiet = quiet self.init() @@ -33,7 +34,7 @@ def get_expected_workers(self)-> list[str]: for file_path in self.config_dir.iterdir(): file_path_abs = str(file_path.absolute()) if file_path.is_file(): - if 'fm.workers.supervisor.conf' in file_path_abs: + if '.workers.fm.supervisor.conf' in file_path_abs: workers_supervisor_conf_paths.append(file_path) workers_expected_service_names = [] @@ -41,7 +42,7 @@ def get_expected_workers(self)-> list[str]: for worker_name in workers_supervisor_conf_paths: worker_name = worker_name.name worker_name = worker_name.replace("frappe-bench-frappe-","") - worker_name = worker_name.replace(".fm.workers.supervisor.conf","") + worker_name = worker_name.replace(".workers.fm.supervisor.conf","") workers_expected_service_names.append(worker_name) workers_expected_service_names.sort() From bf5566c01fc3a69503ddb3f5ffa5918720f7a289 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 20:35:13 +0530 Subject: [PATCH 058/100] fix: workers are not getting created when the site is migrated --- Docker/frappe/divide-supervisor-conf.py | 2 +- frappe_manager/compose_manager/ComposeFile.py | 7 ++----- frappe_manager/site_manager/SiteManager.py | 3 ++- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Docker/frappe/divide-supervisor-conf.py b/Docker/frappe/divide-supervisor-conf.py index 87988141..26086043 100755 --- a/Docker/frappe/divide-supervisor-conf.py +++ b/Docker/frappe/divide-supervisor-conf.py @@ -32,7 +32,7 @@ section_config.set(section_name, key, value) if 'worker' in section_name: - file_name = f"{section_name.replace('program:','')}.fm.workers.supervisor.conf" + file_name = f"{section_name.replace('program:','')}.workers.fm.supervisor.conf" else: file_name = f"{section_name.replace('program:','')}.fm.supervisor.conf" diff --git a/frappe_manager/compose_manager/ComposeFile.py b/frappe_manager/compose_manager/ComposeFile.py index 869c6dbf..d1db693d 100644 --- a/frappe_manager/compose_manager/ComposeFile.py +++ b/frappe_manager/compose_manager/ComposeFile.py @@ -1,18 +1,15 @@ -import pkgutil from pathlib import Path from ruamel.yaml import YAML -from typing import List -import typer from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.compose_manager.utils import represent_null_empty yaml = YAML(typ='safe',pure=True) yaml.representer.ignore_aliases = lambda *args: True -#yaml.representer.add_representer(type(None), represent_none) # Set the default flow style to None to preserve the null representation -yaml.default_flow_style = None +yaml.default_flow_style = False + class ComposeFile: diff --git a/frappe_manager/site_manager/SiteManager.py b/frappe_manager/site_manager/SiteManager.py index e18ab579..bbc99f1b 100644 --- a/frappe_manager/site_manager/SiteManager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -285,8 +285,9 @@ def start_site(self): # start the provided site self.migrate_site() self.site.pull() - self.site.sync_workers_compose() self.site.start() + self.site.frappe_logs_till_start() + self.site.sync_workers_compose() def attach_to_site(self, user: str, extensions: List[str]): """ From 52cf000a7b9c19286832f82ba3a7a8105bcbef44 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 20:40:05 +0530 Subject: [PATCH 059/100] fix: migrate compose set ENVRONMENT to dev --- frappe_manager/site_manager/site.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index d97e7fb6..86f89778 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -86,7 +86,8 @@ def migrate_site_compose(self): # get all the payloads envs = self.composefile.get_all_envs() labels = self.composefile.get_all_labels() - # extrahosts = self.composefile.get_all_extrahosts() + + envs['frappe']['ENVRIRONMENT'] = 'dev' # overwrite user for each invocation import os From 6da60bfe66fe17ba61948fd973564fdf1f611892 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 20:43:01 +0530 Subject: [PATCH 060/100] default dev env when compose migrate --- frappe_manager/site_manager/site.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 86f89778..cc5eaa63 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -87,7 +87,8 @@ def migrate_site_compose(self): envs = self.composefile.get_all_envs() labels = self.composefile.get_all_labels() - envs['frappe']['ENVRIRONMENT'] = 'dev' + if not 'ENVIRONMENT' in envs['frappe']: + envs['frappe']['ENVRIRONMENT'] = 'dev' # overwrite user for each invocation import os From bf464c37436580be9ef1f86fce39c009c6d48291 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 21:16:57 +0530 Subject: [PATCH 061/100] chore: typo --- frappe_manager/site_manager/site.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index cc5eaa63..2b184a89 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -88,7 +88,7 @@ def migrate_site_compose(self): labels = self.composefile.get_all_labels() if not 'ENVIRONMENT' in envs['frappe']: - envs['frappe']['ENVRIRONMENT'] = 'dev' + envs['frappe']['ENVIRONMENT'] = 'dev' # overwrite user for each invocation import os From 6585cce4c59cd6f9fe03c7f640e91c263b7bbfbb Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 21:22:30 +0530 Subject: [PATCH 062/100] add more env for frappe container in migrate --- frappe_manager/site_manager/site.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 2b184a89..665edc5e 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -87,9 +87,13 @@ def migrate_site_compose(self): envs = self.composefile.get_all_envs() labels = self.composefile.get_all_labels() + # introduced in v0.10.0 if not 'ENVIRONMENT' in envs['frappe']: envs['frappe']['ENVIRONMENT'] = 'dev' + envs['frappe']['CONTAINER_NAME_PREFIX'] = get_container_name_prefix(self.name), + + # overwrite user for each invocation import os users = {"nginx":{ From 44bcc59ff071f92e06bb91dac4649cfd1ea68499 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 21:28:23 +0530 Subject: [PATCH 063/100] set VIRTUAL_HOST in nginx in migrate --- frappe_manager/site_manager/site.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 665edc5e..20b86808 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -91,7 +91,10 @@ def migrate_site_compose(self): if not 'ENVIRONMENT' in envs['frappe']: envs['frappe']['ENVIRONMENT'] = 'dev' - envs['frappe']['CONTAINER_NAME_PREFIX'] = get_container_name_prefix(self.name), + envs['frappe']['CONTAINER_NAME_PREFIX'] = get_container_name_prefix(self.name) + envs['frappe']['MARIADB_ROOT_PASS'] = 'root' + + envs['nginx']['VIRTUAL_HOST'] = self.name # overwrite user for each invocation From 1e2cb82712c139c07a149442a5daec747ebb9e97 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 22:41:59 +0530 Subject: [PATCH 064/100] add: param status_msg --- frappe_manager/site_manager/site.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 20b86808..b401cd12 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -267,11 +267,15 @@ def logs(self, service: str, follow: bool = False): else: richprint.stdout.print(line) - def frappe_logs_till_start(self): + def frappe_logs_till_start(self,status_msg = None): """ The function `frappe_logs_till_start` prints logs until a specific line is found and then stops. """ status_text = "Creating Site" + + if status_msg: + status_text = status_msg + richprint.change_head(status_text) try: output = self.docker.compose.logs( From c3690e4b10a756fce4cd2e4e5a858883fe1e83d3 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 22:44:03 +0530 Subject: [PATCH 065/100] add status when starting site --- frappe_manager/site_manager/SiteManager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe_manager/site_manager/SiteManager.py b/frappe_manager/site_manager/SiteManager.py index bbc99f1b..5ae6cf17 100644 --- a/frappe_manager/site_manager/SiteManager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -286,7 +286,7 @@ def start_site(self): self.migrate_site() self.site.pull() self.site.start() - self.site.frappe_logs_till_start() + self.site.frappe_logs_till_start(status_msg='Starting Site') self.site.sync_workers_compose() def attach_to_site(self, user: str, extensions: List[str]): From e48ab9da01356ca12d4648199ad6a06232ba23bd Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 22:47:10 +0530 Subject: [PATCH 066/100] symlink for bench-start.log --- Docker/frappe/user-script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/frappe/user-script.sh b/Docker/frappe/user-script.sh index a9e82880..d45c7ca3 100755 --- a/Docker/frappe/user-script.sh +++ b/Docker/frappe/user-script.sh @@ -9,7 +9,7 @@ emer() { if [[ ! -d 'logs' ]]; then mkdir -p logs - ln -sfn ../frappe-bench/logs/web.dev.log logs/web.dev.log + ln -sfn ../frappe-bench/logs/web.dev.log logs/bench-start.log fi REDIS_SOCKETIO_PORT=80 From 8327dfac7019bda3f25be48c76f7a3a603caeac6 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 9 Jan 2024 23:22:57 +0530 Subject: [PATCH 067/100] fix: compose file migration --- frappe_manager/site_manager/site.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index b401cd12..6dab09c0 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -1,3 +1,4 @@ +from copy import deepcopy import importlib import shutil import re @@ -96,9 +97,16 @@ def migrate_site_compose(self): envs['nginx']['VIRTUAL_HOST'] = self.name + import os + envs_user_info = {} + userid_groupid:dict = {"USERID": os.getuid(), "USERGROUP": os.getgid() } + + env_user_info_container_list = ['frappe','schedule','socketio'] + + for env in env_user_info_container_list: + envs_user_info[env] = deepcopy(userid_groupid) # overwrite user for each invocation - import os users = {"nginx":{ "uid": os.getuid(), "gid": os.getgid() @@ -110,10 +118,12 @@ def migrate_site_compose(self): # set all the payload self.composefile.set_all_envs(envs) + self.composefile.set_all_envs(envs_user_info) self.composefile.set_all_labels(labels) self.composefile.set_all_users(users) # self.composefile.set_all_extrahosts(extrahosts) + self.create_compose_dirs() self.composefile.set_network_alias("nginx", "site-network", [self.name]) self.composefile.set_container_names(get_container_name_prefix(self.name)) fm_version = importlib.metadata.version("frappe-manager") @@ -196,8 +206,9 @@ def create_compose_dirs(self) -> bool: for directory in nginx_poluate_dir: new_dir = nginx_dir / directory - new_dir_abs = str(new_dir.absolute()) - host_run_cp(nginx_image,source="/etc/nginx",destination=new_dir_abs,docker=self.docker) + if not new_dir.exists(): + new_dir_abs = str(new_dir.absolute()) + host_run_cp(nginx_image,source="/etc/nginx",destination=new_dir_abs,docker=self.docker) nginx_subdirs = ['logs','cache','run'] for directory in nginx_subdirs: From 2257e78b21b9a84ed4eb51eb133bdfac431c3cf1 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 10 Jan 2024 13:21:27 +0530 Subject: [PATCH 068/100] fix: getting docker port binds --- frappe_manager/site_manager/site.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 6dab09c0..997851c8 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -513,11 +513,14 @@ def get_services_running_status(self) -> dict: def get_host_port_binds(self): try: output = self.docker.compose.ps(all=True, format="json", stream=True) - status: dict = {} + status: list = [] for source, line in output: if source == "stdout": - status = json.loads(line.decode()) - break + current_status = json.loads(line.decode()) + if type(current_status) == dict: + status.append(current_status) + else: + status += current_status ports_info = [] for container in status: try: From d7ed47a87fa94b5bff99e4004b005dde4c52d221 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 10 Jan 2024 13:22:26 +0530 Subject: [PATCH 069/100] add: missing containers from enum --- frappe_manager/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe_manager/__init__.py b/frappe_manager/__init__.py index 5f1b9ff0..ba78eb2a 100644 --- a/frappe_manager/__init__.py +++ b/frappe_manager/__init__.py @@ -24,3 +24,5 @@ class SiteServicesEnum(str, Enum): redis_queue = "redis-queue" redis_cache = "redis-cache" redis_socketio = "redis-socketio" + scheduler = "scheduler" + socketio = "socketio" From b44c13c0f67274230bbad9fca58ffbeb89ef73b7 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Thu, 11 Jan 2024 16:32:58 +0530 Subject: [PATCH 070/100] fix: symlink bench-start.log with web.dev.log --- Docker/frappe/user-script.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Docker/frappe/user-script.sh b/Docker/frappe/user-script.sh index d45c7ca3..a01984f5 100755 --- a/Docker/frappe/user-script.sh +++ b/Docker/frappe/user-script.sh @@ -7,11 +7,14 @@ emer() { exit 1 } -if [[ ! -d 'logs' ]]; then - mkdir -p logs +if [[ -d 'logs' ]]; then + if [[ -f 'logs/bench-start.log' ]]; then + mv logs/bench-start.log logs/bench-start.log.bak + fi ln -sfn ../frappe-bench/logs/web.dev.log logs/bench-start.log fi + REDIS_SOCKETIO_PORT=80 WEB_PORT=80 From b63bc64bd7b2dc355c78a0bd36dd213a4d4f33d4 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Thu, 11 Jan 2024 17:25:48 +0530 Subject: [PATCH 071/100] fix: bench restart in frappe container --- Docker/frappe/bench-wrapper.sh | 2 +- Docker/frappe/entrypoint.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Docker/frappe/bench-wrapper.sh b/Docker/frappe/bench-wrapper.sh index 40b8d0e0..c4cdbff2 100644 --- a/Docker/frappe/bench-wrapper.sh +++ b/Docker/frappe/bench-wrapper.sh @@ -1,6 +1,6 @@ #!/bin/bash after_command() { - supervisorctl -c /opt/user/supervisord.conf restart bench-dev + supervisorctl -c /opt/user/supervisord.conf restart frappe-bench-dev: } if [[ "$@" =~ ^restart[[:space:]]* ]]; then after_command diff --git a/Docker/frappe/entrypoint.sh b/Docker/frappe/entrypoint.sh index 072e80cb..8f38c875 100755 --- a/Docker/frappe/entrypoint.sh +++ b/Docker/frappe/entrypoint.sh @@ -32,7 +32,7 @@ if [[ ! -f "/workspace/.profile" ]]; then cat /opt/user/.profile > /workspace/.profile fi -chown -R "$USERID":"$USERGROUP" /workspace +#chown -R "$USERID":"$USERGROUP" /workspace if [ "$#" -gt 0 ]; then gosu "$USERID":"$USERGROUP" "/scripts/$@" From 34aa6e47c3493d6e67be1890ad85f5e8df721367 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Thu, 11 Jan 2024 17:26:41 +0530 Subject: [PATCH 072/100] fix: typo --- frappe_manager/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe_manager/__init__.py b/frappe_manager/__init__.py index ba78eb2a..48c9fa13 100644 --- a/frappe_manager/__init__.py +++ b/frappe_manager/__init__.py @@ -24,5 +24,5 @@ class SiteServicesEnum(str, Enum): redis_queue = "redis-queue" redis_cache = "redis-cache" redis_socketio = "redis-socketio" - scheduler = "scheduler" + schedule = "schedule" socketio = "socketio" From b2d82933ab2a23f7c5185829df3b400a8052b467 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Thu, 11 Jan 2024 18:40:28 +0530 Subject: [PATCH 073/100] update supervisord log location and optmizations --- Docker/frappe/entrypoint.sh | 2 +- Docker/frappe/supervisord.conf | 4 ++-- Docker/frappe/user-script.sh | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Docker/frappe/entrypoint.sh b/Docker/frappe/entrypoint.sh index 8f38c875..072e80cb 100755 --- a/Docker/frappe/entrypoint.sh +++ b/Docker/frappe/entrypoint.sh @@ -32,7 +32,7 @@ if [[ ! -f "/workspace/.profile" ]]; then cat /opt/user/.profile > /workspace/.profile fi -#chown -R "$USERID":"$USERGROUP" /workspace +chown -R "$USERID":"$USERGROUP" /workspace if [ "$#" -gt 0 ]; then gosu "$USERID":"$USERGROUP" "/scripts/$@" diff --git a/Docker/frappe/supervisord.conf b/Docker/frappe/supervisord.conf index a3d413e7..33bf4643 100644 --- a/Docker/frappe/supervisord.conf +++ b/Docker/frappe/supervisord.conf @@ -8,8 +8,8 @@ serverurl = unix:///opt/user/supervisor.sock [supervisord] nodaemon = true -logfile=/workspace/logs/supervisord.log -pidfile=/workspace/logs/supervisord.pid +logfile=/workspace/frappe-bench/logs/supervisord.log +pidfile=/workspace/frappe-bench/logs/supervisord.pid [rpcinterface:supervisor] supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface diff --git a/Docker/frappe/user-script.sh b/Docker/frappe/user-script.sh index a01984f5..83537411 100755 --- a/Docker/frappe/user-script.sh +++ b/Docker/frappe/user-script.sh @@ -48,10 +48,10 @@ if [[ ! -d "frappe-bench" ]]; then $BENCH_COMMAND init --skip-assets --skip-redis-config-generation --frappe-branch "$FRAPPE_BRANCH" frappe-bench # setting configuration - wait-for-it -t 120 "$MARIADB_HOST":3306 - wait-for-it -t 120 redis-cache:6379 - wait-for-it -t 120 redis-queue:6379 - wait-for-it -t 120 redis-socketio:6379 + wait-for-it -t 120 "$MARIADB_HOST":3306; + wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-cache":6379; + wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-queue":6379; + wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-socketio":6379; cd frappe-bench From 2efb8e63cc6422b933dcb43ba51c3eeab1f38e2d Mon Sep 17 00:00:00 2001 From: Xieyt Date: Thu, 11 Jan 2024 19:18:28 +0530 Subject: [PATCH 074/100] force common_site_config update on each start --- Docker/frappe/user-script.sh | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/Docker/frappe/user-script.sh b/Docker/frappe/user-script.sh index 83537411..06a76540 100755 --- a/Docker/frappe/user-script.sh +++ b/Docker/frappe/user-script.sh @@ -102,7 +102,7 @@ if [[ ! -d "frappe-bench" ]]; then host_changed=$(echo "$bench_serve_help_output" | grep -c 'host' || true) # SUPERVIOSRCONFIG_STATUS=$(bench setup supervisor --skip-redis --skip-supervisord --yes --user "$USER") - bench setup supervisor --skip-redis --skip-supervisord --yes --user "$USER" + $BENCH_COMMAND setup supervisor --skip-redis --skip-supervisord --yes --user "$USER" /scripts/divide-supervisor-conf.py config/supervisor.conf @@ -146,7 +146,7 @@ else host_changed=$(echo "$bench_serve_help_output" | grep -c 'host' || true) - bench setup supervisor --skip-redis --skip-supervisord --yes --user "$USER" + $BENCH_COMMAND setup supervisor --skip-redis --skip-supervisord --yes --user "$USER" /scripts/divide-supervisor-conf.py config/supervisor.conf @@ -158,15 +158,29 @@ else chmod +x /opt/user/bench-dev-server.sh - if [[ ! "${WEB_PORT}" == 80 ]]; then - $BENCH_COMMAND set-config -g webserver_port "$WEB_PORT"; - fi - + $BENCH_COMMAND config dns_multitenant on + $BENCH_COMMAND set-config -g db_host "$MARIADB_HOST" + $BENCH_COMMAND set-config -g db_port 3306 + $BENCH_COMMAND set-config -g redis_cache "redis://${CONTAINER_NAME_PREFIX}-redis-cache:6379" + $BENCH_COMMAND set-config -g redis_queue "redis://${CONTAINER_NAME_PREFIX}-redis-queue:6379" + $BENCH_COMMAND set-config -g redis_socketio "redis://${CONTAINER_NAME_PREFIX}-redis-socketio:6379" + $BENCH_COMMAND set-config -g webserver_port "$WEB_PORT" + $BENCH_COMMAND set-config -g socketio_port "$REDIS_SOCKETIO_PORT" if [[ "${ENVIRONMENT}" = "dev" ]]; then + + $BENCH_COMMAND set-config -g mail_port 1025 + $BENCH_COMMAND set-config -g mail_server 'mailhog' + $BENCH_COMMAND set-config -g disable_mail_smtp_authentication 1 + $BENCH_COMMAND set-config -g developer_mode "$DEVELOPER_MODE" + cp /opt/user/frappe-dev.conf /opt/user/conf.d/frappe-dev.conf else - ln -sfn /workspace/frappe-bench/config/frappe-bench-frappe-web.fm.supervisor.conf /opt/user/conf.d/frappe-bench-frappe-web.fm.supervisor.conf + if [[ -f '/opt/user/conf.d/frappe-bench-frappe-web.fm.supervisor.conf' ]]; then + ln -sfn /workspace/frappe-bench/config/frappe-bench-frappe-web.fm.supervisor.conf /opt/user/conf.d/frappe-bench-frappe-web.fm.supervisor.conf + else + emer 'Not able to start the server. /opt/user/conf.d/frappe-bench-frappe-web.fm.supervisor.conf not available.' + fi fi if [[ -n "$BENCH_START_OFF" ]]; then From 19179c80e91e2b952de3cbe12dc83f6fb8a02677 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Thu, 11 Jan 2024 19:57:38 +0530 Subject: [PATCH 075/100] fix: workers not starting --- .../compose_manager/templates/docker-compose.workers.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe_manager/compose_manager/templates/docker-compose.workers.tmpl b/frappe_manager/compose_manager/templates/docker-compose.workers.tmpl index 109d4525..e9037bec 100644 --- a/frappe_manager/compose_manager/templates/docker-compose.workers.tmpl +++ b/frappe_manager/compose_manager/templates/docker-compose.workers.tmpl @@ -4,10 +4,10 @@ services: environment: TIMEOUT: 6000 CHANGE_DIR: /workspace/frappe-bench - WAIT_FOR: /workspace/frappe-bench/config/frappe-bench-frappe-{worker-name}.fm.workers.supervisor.conf + WAIT_FOR: /workspace/frappe-bench/config/frappe-bench-frappe-{worker-name}.workers.fm.supervisor.conf # here worker-name will be replace with the worker name COMMAND: | - ln -sfn /workspace/frappe-bench/config/frappe-bench-frappe-{worker-name}.fm.workers.supervisor.conf /opt/user/conf.d/frappe-bench-frappe-{worker-name}.fm.workers.supervisor.conf + ln -sfn /workspace/frappe-bench/config/frappe-bench-frappe-{worker-name}.workers.fm.supervisor.conf /opt/user/conf.d/frappe-bench-frappe-{worker-name}.workers.fm.supervisor.conf supervisord -c /opt/user/supervisord.conf command: launch.sh volumes: From 13964be1cbf6461440dec0ceb05bee23f9e03edd Mon Sep 17 00:00:00 2001 From: Xieyt Date: Thu, 11 Jan 2024 19:58:41 +0530 Subject: [PATCH 076/100] optimze: chown more faster --- Docker/frappe/entrypoint.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Docker/frappe/entrypoint.sh b/Docker/frappe/entrypoint.sh index 072e80cb..31e59e61 100755 --- a/Docker/frappe/entrypoint.sh +++ b/Docker/frappe/entrypoint.sh @@ -32,7 +32,8 @@ if [[ ! -f "/workspace/.profile" ]]; then cat /opt/user/.profile > /workspace/.profile fi -chown -R "$USERID":"$USERGROUP" /workspace +find /workspace -type d -print0 | xargs -0 -n 200 -P "$(nproc)" chown "$USERID":"$USERGROUP" +find /workspace -type f -print0 | xargs -0 -n 200 -P "$(nproc)" chown "$USERID":"$USERGROUP" if [ "$#" -gt 0 ]; then gosu "$USERID":"$USERGROUP" "/scripts/$@" From 4a62124ff8949f433dcc8a38072d73403e6c0a28 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 12 Jan 2024 17:10:35 +0530 Subject: [PATCH 077/100] fix: chown only when site is being created --- Docker/frappe/entrypoint.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Docker/frappe/entrypoint.sh b/Docker/frappe/entrypoint.sh index 31e59e61..430aca0f 100755 --- a/Docker/frappe/entrypoint.sh +++ b/Docker/frappe/entrypoint.sh @@ -32,8 +32,10 @@ if [[ ! -f "/workspace/.profile" ]]; then cat /opt/user/.profile > /workspace/.profile fi -find /workspace -type d -print0 | xargs -0 -n 200 -P "$(nproc)" chown "$USERID":"$USERGROUP" -find /workspace -type f -print0 | xargs -0 -n 200 -P "$(nproc)" chown "$USERID":"$USERGROUP" +if [[ ! -d '/workspace/frappe-bench' ]]; then + find /workspace -type d -print0 | xargs -0 -n 200 -P "$(nproc)" chown "$USERID":"$USERGROUP" + find /workspace -type f -print0 | xargs -0 -n 200 -P "$(nproc)" chown "$USERID":"$USERGROUP" +fi if [ "$#" -gt 0 ]; then gosu "$USERID":"$USERGROUP" "/scripts/$@" From 9b5c9735d86e92a0715c8436d8037d2e26f37e72 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 12 Jan 2024 17:12:34 +0530 Subject: [PATCH 078/100] fix: workers generation in migration --- frappe_manager/site_manager/site.py | 107 +++++++++++------- .../workers_manager/SiteWorker.py | 5 +- 2 files changed, 70 insertions(+), 42 deletions(-) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 997851c8..85ec1ac1 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -130,6 +130,9 @@ def migrate_site_compose(self): self.composefile.set_version(fm_version) self.composefile.set_top_networks_name("site-network",get_container_name_prefix(self.name)) self.composefile.write_to_file() + + # change the node socketio port + self.common_site_config_set('socketio_port','80') status = True if status: @@ -143,6 +146,22 @@ def migrate_site_compose(self): else: richprint.print("Already Latest Environment Version") + def common_site_config_set(self,key, value): + common_site_config_path = self.path / 'workspace/frappe-bench/sites/common_site_config.json' + common_site_config = {} + + with open(common_site_config_path,'r') as f: + common_site_config = json.load(f) + + try: + common_site_config[key] = value + with open(common_site_config_path,'w') as f: + json.dump(common_site_config,f) + return True + except KeyError as e: + # log error that not able to change common site config + return False + def generate_compose(self, inputs: dict) -> None: """ @@ -552,6 +571,7 @@ def is_service_running(self, service): return False def sync_workers_compose(self): + self.regenerate_supervisor_conf() are_workers_not_changed = self.workers.is_expected_worker_same_as_template() if not are_workers_not_changed: self.workers.generate_compose() @@ -561,51 +581,58 @@ def sync_workers_compose(self): def regenerate_supervisor_conf(self): - richprint.change_head("Regenerating supervisor.conf.") - backup = False - # take backup - if self.workers.supervisor_config_path.exists(): - shutil.copy(self.workers.supervisor_config_path, self.workers.supervisor_config_path.parent / "supervisor.conf.bak") - for file_path in self.workers.config_dir.iterdir(): - file_path_abs = str(file_path.absolute()) - if file_path.is_file(): - if '.workers.fm.supervisor.conf' in file_path_abs: - shutil.copy(file_path, file_path.parent / f"{file_path.name}.bak") - backup = True - - # generate the supervisor.conf - try: - bench_setup_supervisor_command = 'bench setup supervisor --skip-redis --skip-supervisord --yes --user frappe' + if self.name: + richprint.change_head("Regenerating supervisor.conf.") + backup = False + backup_list = [] + + # take backup + if self.workers.supervisor_config_path.exists(): + shutil.copy(self.workers.supervisor_config_path, self.workers.supervisor_config_path.parent / "supervisor.conf.bak") + for file_path in self.workers.config_dir.iterdir(): + file_path_abs = str(file_path.absolute()) + if file_path.is_file(): + if file_path_abs.endswith('.fm.supervisor.conf'): + from_path = file_path + to_path = file_path.parent / f"{file_path.name}.bak" + + shutil.copy(from_path, to_path) + + backup_list.append((from_path, to_path)) + backup = True + + # generate the supervisor.conf + try: + bench_setup_supervisor_command = 'bench setup supervisor --skip-redis --skip-supervisord --yes --user frappe' + + output = self.docker.compose.exec( + service='frappe', + command=bench_setup_supervisor_command, + stream=True, + user='frappe', + workdir='/workspace/frappe-bench' + ) + richprint.live_lines(output, padding=(0, 0, 0, 2)) + + generate_split_config_command = '/scripts/divide-supervisor-conf.py config/supervisor.conf' - output = self.docker.compose.exec( + output = self.docker.compose.exec( service='frappe', - command=bench_setup_supervisor_command, + command=generate_split_config_command , stream=True, user='frappe', workdir='/workspace/frappe-bench' - ) - richprint.live_lines(output, padding=(0, 0, 0, 2)) + ) - generate_split_config_command = '/scripts/divide-supervisor-conf.py config/supervisor.conf' + richprint.live_lines(output, padding=(0, 0, 0, 2)) - output = self.docker.compose.exec( - service='frappe', - command=generate_split_config_command , - stream=True, - user='frappe', - workdir='/workspace/frappe-bench' - ) - richprint.live_lines(output, padding=(0, 0, 0, 2)) - return True - except DockerException as e: - richprint.error("Failure in generating, supervisor.conf file.") - if backup: - richprint.print("Rolling back to previous workers configuration.") - shutil.copy(self.workers.supervisor_config_path.parent / "supervisor.conf.bak", self.workers.supervisor_config_path) + return True + except DockerException as e: + richprint.error(f"Failure in generating, supervisor.conf file.{e}") + if backup: + richprint.print("Rolling back to previous workers configuration.") + shutil.copy(self.workers.supervisor_config_path.parent / "supervisor.conf.bak", self.workers.supervisor_config_path) - for file_path in self.workers.config_dir.iterdir(): - file_path_abs = str(file_path.absolute()) - if file_path.is_file(): - if '.workers.fm.supervisor.conf.bak' in file_path_abs: - shutil.copy(file_path, file_path.parent / f"{file_path.name}".replace(".back","")) - return False + for from_path ,to_path in backup_list: + shutil.copy(to_path, from_path) + return False diff --git a/frappe_manager/site_manager/workers_manager/SiteWorker.py b/frappe_manager/site_manager/workers_manager/SiteWorker.py index 3bc64cee..21327f91 100644 --- a/frappe_manager/site_manager/workers_manager/SiteWorker.py +++ b/frappe_manager/site_manager/workers_manager/SiteWorker.py @@ -34,7 +34,7 @@ def get_expected_workers(self)-> list[str]: for file_path in self.config_dir.iterdir(): file_path_abs = str(file_path.absolute()) if file_path.is_file(): - if '.workers.fm.supervisor.conf' in file_path_abs: + if file_path_abs.endswith('.workers.fm.supervisor.conf'): workers_supervisor_conf_paths.append(file_path) workers_expected_service_names = [] @@ -55,7 +55,8 @@ def is_expected_worker_same_as_template(self) -> bool: if not self.composefile.is_template_loaded: prev_workers = self.composefile.get_services_list() prev_workers.sort() - return prev_workers == self.get_expected_workers() + expected_workers = self.get_expected_workers() + return prev_workers == expected_workers else: return False From b45e4bd23aa3b5d765af385b02d9f960e57dea68 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 31 Jan 2024 14:01:31 +0530 Subject: [PATCH 079/100] Move templates and utils --- frappe_manager/compose_manager/utils.py | 10 - frappe_manager/docker_wrapper/utils.py | 161 --------- frappe_manager/site_manager/utils.py | 26 -- frappe_manager/templates/__init__.py | 0 .../templates/docker-compose.services.tmpl | 69 ++++ .../templates/docker-compose.tmpl | 51 +-- .../templates/docker-compose.workers.tmpl | 4 + frappe_manager/templates/fm_metadata.py | 5 + frappe_manager/utils.py | 234 ------------- frappe_manager/utils/callbacks.py | 74 ++++ frappe_manager/utils/docker.py | 326 ++++++++++++++++++ frappe_manager/utils/helpers.py | 312 +++++++++++++++++ frappe_manager/utils/site.py | 77 +++++ 13 files changed, 898 insertions(+), 451 deletions(-) delete mode 100644 frappe_manager/compose_manager/utils.py delete mode 100644 frappe_manager/docker_wrapper/utils.py delete mode 100644 frappe_manager/site_manager/utils.py create mode 100644 frappe_manager/templates/__init__.py create mode 100644 frappe_manager/templates/docker-compose.services.tmpl rename frappe_manager/{compose_manager => }/templates/docker-compose.tmpl (76%) rename frappe_manager/{compose_manager => }/templates/docker-compose.workers.tmpl (87%) create mode 100644 frappe_manager/templates/fm_metadata.py delete mode 100644 frappe_manager/utils.py create mode 100644 frappe_manager/utils/callbacks.py create mode 100644 frappe_manager/utils/docker.py create mode 100644 frappe_manager/utils/helpers.py create mode 100644 frappe_manager/utils/site.py diff --git a/frappe_manager/compose_manager/utils.py b/frappe_manager/compose_manager/utils.py deleted file mode 100644 index fbc16283..00000000 --- a/frappe_manager/compose_manager/utils.py +++ /dev/null @@ -1,10 +0,0 @@ -def represent_null_empty(s): - """ - The function `represent_none` represents the value `None` as a null scalar in YAML format. - - :param _: The underscore (_) parameter is a convention in Python to indicate that the parameter is - not going to be used in the function. - :return: a representation of `None` as a YAML scalar with the tag `tag:yaml.org,2002:null` and an - empty string as its value. - """ - return s.replace("null","") diff --git a/frappe_manager/docker_wrapper/utils.py b/frappe_manager/docker_wrapper/utils.py deleted file mode 100644 index af4197b9..00000000 --- a/frappe_manager/docker_wrapper/utils.py +++ /dev/null @@ -1,161 +0,0 @@ -import os -import shlex -import shutil -import signal -import subprocess -import sys -from datetime import datetime, timedelta -from importlib.metadata import version -from pathlib import Path -from queue import Queue -from subprocess import PIPE, Popen, run -from threading import Thread -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union, overload -from frappe_manager.logger import log -from rich import control -from frappe_manager.docker_wrapper.DockerException import DockerException - -process_opened = [] - -def reader(pipe, pipe_name, queue): - logger = log.get_logger() - try: - with pipe: - for line in iter(pipe.readline, b""): - queue_line = line.decode().strip('\n') - logger.debug(queue_line) - queue.put((pipe_name, str(queue_line).encode())) - finally: - queue.put(None) - - - -def stream_stdout_and_stderr( - full_cmd: list, - env: Dict[str, str] = None, -) -> Iterable[Tuple[str, bytes]]: - logger = log.get_logger() - logger.debug('- -'*10) - logger.debug(f"DOCKER COMMAND: {' '.join(full_cmd)}") - if env is None: - subprocess_env = None - else: - subprocess_env = dict(os.environ) - subprocess_env.update(env) - - full_cmd = list(map(str, full_cmd)) - process = Popen(full_cmd, stdout=PIPE, stderr=PIPE, env=subprocess_env) - - - process_opened.append(process.pid) - - q = Queue() - full_stderr = b"" # for the error message - # we use deamon threads to avoid hanging if the user uses ctrl+c - th = Thread(target=reader, args=[process.stdout, "stdout", q]) - th.daemon = True - th.start() - th = Thread(target=reader, args=[process.stderr, "stderr", q]) - th.daemon = True - th.start() - - for _ in range(2): - for source, line in iter(q.get, None): - yield source, line - if source == "stderr": - full_stderr += line - - exit_code = process.wait() - - logger.debug(f"RETURN CODE: {exit_code}") - logger.debug('- -'*10) - if exit_code != 0: - raise DockerException(full_cmd, exit_code, stderr=full_stderr) - - yield ("exit_code", str(exit_code).encode()) - -def run_command_with_exit_code( - full_cmd: list, - env: Dict[str, str] = None, - stream: bool = True, - quiet: bool = False -): - if stream: - if quiet: - try: - for source ,line in stream_stdout_and_stderr(full_cmd): - if source == 'exit_code': - exit_code: int = int(line.decode()) - return(exit_code) - except Exception as e: - pass - else: - return stream_stdout_and_stderr(full_cmd) - else: - from frappe_manager.display_manager.DisplayManager import richprint - output = run(full_cmd) - exit_code = output.returncode - if exit_code != 0: - raise DockerException(full_cmd,exit_code) - -def parameter_to_option(param: str) -> str: - """changes parameter's to option""" - option = "--" + param.replace("_", "-") - return option - -def parameters_to_options(param: dict, exclude: list = []) -> list: - # remove the self parameter - temp_param: dict = dict(param) - - del temp_param["self"] - - for key in exclude: - del temp_param[key] - - # remove all parameters which are not booleans - params: list = [] - - for key in temp_param.keys(): - value = temp_param[key] - key = "--" + key.replace("_","-") - if type(value) == bool: - if value: - params.append(key) - if type(value) == int: - params.append(key) - params.append(value) - if type(value) == str: - if value: - params.append(key) - params.append(value) - if type(value) == list: - if value: - params.append(key) - params += value - - return params - -def is_current_user_in_group(group_name) -> bool: - """Check if the current user is in the given group""" - - from frappe_manager.display_manager.DisplayManager import richprint - - import platform - if platform.system() == 'Linux': - import grp - import pwd - import os - current_user = pwd.getpwuid(os.getuid()).pw_name - try: - docker_gid = grp.getgrnam(group_name).gr_gid - docker_group_members = grp.getgrgid(docker_gid).gr_mem - if current_user in docker_group_members: - return True - else: - richprint.error(f"Your current user [blue][b] {current_user} [/b][/blue] is not in the 'docker' group. Please add it and restart your terminal.") - return False - except KeyError: - richprint.error(f"The group '{group_name}' does not exist. Please create it and add your current user [blue][b] {current_user} [/b][/blue] to it.") - return False - else: - return True diff --git a/frappe_manager/site_manager/utils.py b/frappe_manager/site_manager/utils.py deleted file mode 100644 index 77a2e8aa..00000000 --- a/frappe_manager/site_manager/utils.py +++ /dev/null @@ -1,26 +0,0 @@ -import time -import platform -import subprocess -import os - -def log_file(file, refresh_time:float = 0.1, follow:bool =False): - ''' - Generator function that yields new lines in a file - ''' - file.seek(0) - - # start infinite loop - while True: - # read last line of file - line = file.readline() - if not line: - if not follow: - break - # sleep if file hasn't been updated - time.sleep(refresh_time) - continue - line = line.strip('\n') - yield line - -def get_container_name_prefix(site_name): - return site_name.replace('.','') diff --git a/frappe_manager/templates/__init__.py b/frappe_manager/templates/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/frappe_manager/templates/docker-compose.services.tmpl b/frappe_manager/templates/docker-compose.services.tmpl new file mode 100644 index 00000000..6e429d9a --- /dev/null +++ b/frappe_manager/templates/docker-compose.services.tmpl @@ -0,0 +1,69 @@ +version: "3.9" +services: + global-db: + container_name: fm_global-db + image: mariadb:10.6 + user: REPLACE_WITH_CURRENT_USER:REPLACE_WITH_CURRENT_USER_GROUP + restart: always + command: + - --character-set-server=utf8mb4 + - --collation-server=utf8mb4_unicode_ci + - --skip-character-set-client-handshake + - --skip-innodb-read-only-compressed + environment: + MYSQL_ROOT_PASSWORD_FILE: REPLACE_WITH_DB_ROOT_PASSWORD_SECRET_FILE + MYSQL_DATABASE: REPLACE_WITH_DB_NAME + MYSQL_USER: REPLACE_WITH_DB_USER + MYSQL_PASSWORD_FILE: REPLACE_WITH_DB_PASSWORD_SECRET_FILE + volumes: + - fm-global-db-data:/var/lib/mysql + - ./mariadb/conf:/etc/mysql + - ./mariadb/logs:/var/log/mysql + networks: + - global-backend-network + secrets: + - db_password + - db_root_password + + global-nginx-proxy: + container_name: fm_global-nginx-proxy + user: REPLACE_WITH_CURRENT_USER:REPLACE_WITH_CURRENT_USER_GROUP + image: jwilder/nginx-proxy + ports: + - "80:80" + - "443:443" + restart: always + volumes: + - "./nginx-proxy/certs:/etc/nginx/certs" + - "./nginx-proxy/dhparam:/etc/nginx/dhparam" + - "./nginx-proxy/confd:/etc/nginx/conf.d" + - "./nginx-proxy/htpasswd:/etc/nginx/htpasswd" + - "./nginx-proxy/vhostd:/etc/nginx/vhost.d" + - "./nginx-proxy/html:/usr/share/nginx/html" + - "./nginx-proxy/logs:/var/log/nginx" + - "./nginx-proxy/run:/var/run" + - "./nginx-proxy/cache:/var/cache/nginx" + - "/var/run/docker.sock:/tmp/docker.sock:ro" + networks: + - global-frontend-network + +networks: + global-frontend-network: + name: fm-global-frontend-network + ipam: + config: + - subnet: '10.1.0.0/16' + global-backend-network: + name: fm-global-backend-network + ipam: + config: + - subnet: '10.2.0.0/16' + +secrets: + db_password: + file: REPLACE_ME_WITH_DB_PASSWORD_TXT_PATH + db_root_password: + file: REPLACE_ME_WITH_DB_ROOT_PASSWORD_TXT_PATH + +volumes: + fm-global-db-data: diff --git a/frappe_manager/compose_manager/templates/docker-compose.tmpl b/frappe_manager/templates/docker-compose.tmpl similarity index 76% rename from frappe_manager/compose_manager/templates/docker-compose.tmpl rename to frappe_manager/templates/docker-compose.tmpl index dd341938..52293a30 100644 --- a/frappe_manager/compose_manager/templates/docker-compose.tmpl +++ b/frappe_manager/templates/docker-compose.tmpl @@ -2,18 +2,20 @@ version: "3.9" services: frappe: image: ghcr.io/rtcamp/frappe-manager-frappe:v0.10.0 + container_name: REPLACE_ME_WITH_CONTAINER_NAME environment: - ADMIN_PASS: REPLACE_ME_WITH_FRAPPE_WEB_ADMIN_PASS + ADMIN_PASS: REPLACE_me_with_frappe_web_admin_pass # apps are defined as :, if branch name not given then default github branch will be used. APPS_LIST: REPLACE_ME_APPS_LIST DB_NAME: REPLACE_ME_WITH_DB_NAME_TO_CREATE # DEVERLOPER_MODE bool -> true/false DEVELOPER_MODE: REPLACE_ME_WITH_DEVELOPER_MODE_TOGGLE FRAPPE_BRANCH: REPLACE_ME_WITH_BRANCH_OF_FRAPPE + MARIADB_ROOT_PASS: REPLACE_ME_WITH_DB_ROOT_PASSWORD SITENAME: REPLACE_ME_WITH_THE_SITE_NAME USERGROUP: REPLACE_ME_WITH_CURRENT_USER_GROUP USERID: REPLACE_ME_WITH_CURRENT_USER - MARIADB_ROOT_PASS: REPLACE_ME_WITH_MARIADB_ROOT_PASS + MARIADB_HOST: REPLACE_ME_WITH_DB_HOST volumes: - ./workspace:/workspace:cached expose: @@ -22,9 +24,13 @@ services: devcontainer.metadata: '[{ "remoteUser": "frappe"}]' networks: site-network: + global-backend-network: + secrets: + - db_root_password nginx: image: ghcr.io/rtcamp/frappe-manager-nginx:v0.10.0 + container_name: REPLACE_ME_WITH_CONTAINER_NAME user: REPLACE_ME_WITH_CURRENT_USER:REPLACE_ME_WITH_CURRENT_USER_GROUP environment: # not implemented as of now @@ -39,27 +45,15 @@ services: - ./configs/nginx/logs:/var/log/nginx - ./configs/nginx/cache:/var/cache/nginx - ./configs/nginx/run:/var/run - ports: - - 80:80 - networks: - site-network: - - mariadb: - image: mariadb:10.6 - command: - - --character-set-server=utf8mb4 - - --collation-server=utf8mb4_unicode_ci - - --skip-character-set-client-handshake - - --skip-innodb-read-only-compressed # Temporary fix for MariaDB 10.6 - environment: - MYSQL_ROOT_PASSWORD: root - volumes: - - mariadb-data:/var/lib/mysql + expose: + - 80 networks: site-network: + global-frontend-network: mailhog: image: ghcr.io/rtcamp/frappe-manager-mailhog:v0.8.3 + container_name: REPLACE_ME_WITH_CONTAINER_NAME expose: - 1025 - 8025 @@ -68,15 +62,18 @@ services: adminer: image: adminer:latest + container_name: REPLACE_ME_WITH_CONTAINER_NAME environment: - - ADMINER_DEFAULT_SERVER=mariadb + ADMINER_DEFAULT_SERVER: global-db expose: - 8080 networks: site-network: + global-backend-network: socketio: image: ghcr.io/rtcamp/frappe-manager-frappe:v0.10.0 + container_name: REPLACE_ME_WITH_CONTAINER_NAME environment: TIMEOUT: 60000 CHANGE_DIR: /workspace/frappe-bench/logs @@ -94,6 +91,7 @@ services: schedule: image: ghcr.io/rtcamp/frappe-manager-frappe:v0.10.0 + container_name: REPLACE_ME_WITH_CONTAINER_NAME environment: TIMEOUT: 60000 CHANGE_DIR: /workspace/frappe-bench @@ -106,9 +104,11 @@ services: - ./workspace:/workspace:cached networks: site-network: + global-backend-network: redis-cache: image: redis:alpine + container_name: REPLACE_ME_WITH_CONTAINER_NAME volumes: - redis-cache-data:/data expose: @@ -118,6 +118,7 @@ services: redis-queue: image: redis:alpine + container_name: REPLACE_ME_WITH_CONTAINER_NAME volumes: - redis-queue-data:/data expose: @@ -127,6 +128,7 @@ services: redis-socketio: image: redis:alpine + container_name: REPLACE_ME_WITH_CONTAINER_NAME volumes: - redis-socketio-data:/data expose: @@ -135,7 +137,6 @@ services: site-network: volumes: - mariadb-data: redis-socketio-data: redis-queue-data: redis-cache-data: @@ -143,3 +144,13 @@ volumes: networks: site-network: name: REPLACE_ME_WITH_SITE_NAME_NETWORK + global-frontend-network: + name: fm-global-frontend-network + external: true + global-backend-network: + name: fm-global-backend-network + external: true + +secrets: + db_root_password: + file: REPLACE_ME_WITH_SECERETS_PATH diff --git a/frappe_manager/compose_manager/templates/docker-compose.workers.tmpl b/frappe_manager/templates/docker-compose.workers.tmpl similarity index 87% rename from frappe_manager/compose_manager/templates/docker-compose.workers.tmpl rename to frappe_manager/templates/docker-compose.workers.tmpl index e9037bec..4eed9b67 100644 --- a/frappe_manager/compose_manager/templates/docker-compose.workers.tmpl +++ b/frappe_manager/templates/docker-compose.workers.tmpl @@ -14,8 +14,12 @@ services: - ./workspace:/workspace:cached networks: site-network: + global-backend-network: networks: site-network: name: REPLACE_ME_WITH_SITE_NAME_NETWORK external: true + global-backend-network: + name: fm-global-backend-network + external: true diff --git a/frappe_manager/templates/fm_metadata.py b/frappe_manager/templates/fm_metadata.py new file mode 100644 index 00000000..02b544af --- /dev/null +++ b/frappe_manager/templates/fm_metadata.py @@ -0,0 +1,5 @@ +from tomlkit import comment, document + +metadata = document() +metadata.add(comment("don't modify this file")) +metadata.add('version', '0.8.3') diff --git a/frappe_manager/utils.py b/frappe_manager/utils.py deleted file mode 100644 index f7e41b9f..00000000 --- a/frappe_manager/utils.py +++ /dev/null @@ -1,234 +0,0 @@ -import importlib -import sys -import requests -import json -import subprocess -import platform - -from pathlib import Path -from frappe_manager.logger import log -from frappe_manager.docker_wrapper.utils import process_opened -from frappe_manager.docker_wrapper.DockerException import DockerException -from frappe_manager.display_manager.DisplayManager import richprint - - -def remove_zombie_subprocess_process(): - """ - Terminates any zombie process - """ - if process_opened: - logger = log.get_logger() - logger.cleanup("-" * 20) - logger.cleanup(f"PROCESS: USED PROCESS {process_opened}") - - # terminate zombie docker process - import psutil - - for pid in process_opened: - try: - process = psutil.Process(pid) - process.terminate() - logger.cleanup(f"Terminated Process {process.cmdline}:{pid}") - except psutil.NoSuchProcess: - logger.cleanup(f"{pid} Process not found") - except psutil.AccessDenied: - logger.cleanup(f"{pid} Permission denied") - logger.cleanup("-" * 20) - - -def check_update(): - url = "https://pypi.org/pypi/frappe-manager/json" - try: - update_info = requests.get(url, timeout=0.1) - update_info = json.loads(update_info.text) - fm_version = importlib.metadata.version("frappe-manager") - latest_version = update_info["info"]["version"] - if not fm_version == latest_version: - richprint.warning( - f'Ready for an update? Run "pip install --upgrade frappe-manager" to update to the latest version {latest_version}.', - emoji_code=":arrows_counterclockwise:️", - ) - except Exception as e: - pass - - -def is_port_in_use(port): - """ - Check if port is in use or not. - - :param port: The port which will be checked if it's in use or not. - :return: Bool In use then True and False when not in use. - """ - import psutil - - for conn in psutil.net_connections(): - if conn.laddr.port == port and conn.status == "LISTEN": - return True - return False - - -def check_ports(ports): - """ - This function checks if the ports is in use. - :param ports: list of ports to be checked - returns: list of binded port(can be empty) - """ - - # TODO handle if ports are open using docker - - current_system = platform.system() - already_binded = [] - for port in ports: - if current_system == "Darwin": - # Mac Os - # check port using lsof command - cmd = f"lsof -iTCP:{port} -sTCP:LISTEN -P -n" - try: - output = subprocess.run( - cmd, check=True, shell=True, capture_output=True - ) - if output.returncode == 0: - already_binded.append(port) - except subprocess.CalledProcessError as e: - pass - else: - # Linux or any other machines - if is_port_in_use(port): - already_binded.append(port) - - return already_binded - - -def check_ports_with_msg(ports_to_check: list, exclude=[]): - """ - The `check_ports` function checks if certain ports are already bound by another process using the - `lsof` command. - """ - richprint.change_head("Checking Ports") - if exclude: - # Removing elements present in remove_array from original_array - ports_to_check = [x for x in exclude if x not in ports_to_check] - if ports_to_check: - already_binded = check_ports(ports_to_check) - if already_binded: - richprint.exit( - f"Whoa there! Looks like the {' '.join([ str(x) for x in already_binded ])} { 'ports are' if len(already_binded) > 1 else 'port is' } having a party already! Can you do us a solid and free up those ports?" - ) - richprint.print("Ports Check : Passed") - - -def generate_random_text(length=50): - import random - import string - - alphanumeric_chars = string.ascii_letters + string.digits - return "".join(random.choice(alphanumeric_chars) for _ in range(length)) - - -def host_run_cp(image: str, source: str, destination: str, docker, verbose=False): - status_text = "Copying files" - richprint.change_head(f"{status_text} {source} -> {destination}") - source_container_name = generate_random_text(10) - dest_path = Path(destination) - errror_exception = None - - failed: bool = False - # run the container - try: - output = docker.run( - image=image, - name=source_container_name, - detach=True, - stream=not verbose, - command="tail -f /dev/null", - ) - if not verbose: - richprint.live_lines(output, padding=(0, 0, 0, 2)) - except DockerException as e: - errror_exception = e - failed = 0 - - if not failed: - # cp from the container - try: - output = docker.cp( - source=source, - destination=destination, - source_container=source_container_name, - stream=not verbose, - ) - if not verbose: - richprint.live_lines(output, padding=(0, 0, 0, 2)) - except DockerException as e: - errror_exception = e - failed = 1 - - # # kill the container - # try: - # output = docker.kill(container=source_container_name,stream=True) - # richprint.live_lines(output, padding=(0,0,0,2)) - # except DockerException as e: - # richprint.exit(f"{status_text} failed. Error: {e}") - - if not failed: - # rm the container - try: - output = docker.rm( - container=source_container_name, force=True, stream=not verbose - ) - if not verbose: - richprint.live_lines(output, padding=(0, 0, 0, 2)) - except DockerException as e: - errror_exception = e - failed = 2 - - # check if the destination file exists - if not type(failed) == bool: - if failed > 1: - if dest_path.exists(): - import shutil - - shutil.rmtree(dest_path) - if failed == 2: - try: - output = docker.rm( - container=source_container_name, force=True, stream=not verbose - ) - if not verbose: - richprint.live_lines(output, padding=(0, 0, 0, 2)) - except DockerException as e: - pass - # TODO introuduce custom exception to handle this type of cases where if the flow is not completed then it should raise exception which is handled by caller and then site creation check is done - richprint.exit(f"{status_text} failed.",error_msg=errror_exception) - - elif not Path(destination).exists(): - richprint.exit(f"{status_text} failed. Copied {destination} not found.") - -def is_cli_help_called(ctx): - help_called = False - - # is called command is sub command group - try: - for subtyper_command in ctx.command.commands[ - ctx.invoked_subcommand - ].commands.keys(): - check_command = " ".join(sys.argv[2:]) - if check_command == subtyper_command: - if ( - ctx.command.commands[ctx.invoked_subcommand] - .commands[subtyper_command] - .params - ): - help_called = True - except AttributeError: - help_called = False - - if not help_called: - # is called command is sub command - check_command = " ".join(sys.argv[1:]) - if check_command == ctx.invoked_subcommand: - # is called command supports arguments then help called - if ctx.command.commands[ctx.invoked_subcommand].params: - help_called = True - - return help_called diff --git a/frappe_manager/utils/callbacks.py b/frappe_manager/utils/callbacks.py new file mode 100644 index 00000000..0d9e4216 --- /dev/null +++ b/frappe_manager/utils/callbacks.py @@ -0,0 +1,74 @@ +import typer +from typing import List, Optional +from frappe_manager.utils.helpers import check_frappe_app_exists, get_current_fm_version +from frappe_manager.display_manager.DisplayManager import richprint + + +def apps_list_validation_callback(value: List[str] | None): + """ + Validate the list of apps provided. + + Args: + value (List[str] | None): The list of apps to validate. + + Raises: + typer.BadParameter: If the list contains the 'frappe' app, or if any app is invalid or has an invalid branch. + + Returns: + List[str] | None: The validated list of apps. + """ + if value: + for app in value: + appx = app.split(":") + if appx == "frappe": + raise typer.BadParameter("Frappe should not be included here.") + if len(appx) == 1: + exists = check_frappe_app_exists(appx[0]) + if not exists["app"]: + raise typer.BadParameter(f"{app} is not a valid FrappeVerse app!") + if len(appx) == 2: + exists = check_frappe_app_exists(appx[0], appx[1]) + if not exists["app"]: + raise typer.BadParameter(f"{app} is not a valid FrappeVerse app!") + if not exists["branch"]: + raise typer.BadParameter( + f"{appx[1]} is not a valid branch of {appx[0]}!" + ) + if len(appx) > 2: + raise typer.BadParameter( + "App should be specified in format : or " + ) + return value + + +def frappe_branch_validation_callback(value: str): + """ + Validate the given Frappe branch. + + Args: + value (str): The Frappe branch to validate. + + Returns: + str: The validated Frappe branch. + + Raises: + typer.BadParameter: If the Frappe branch is not valid. + """ + if value: + exists = check_frappe_app_exists("frappe", value) + if exists['branch']: + return value + else: + raise typer.BadParameter(f"Frappe branch -> {value} is not valid!! ") + +def version_callback(version: Optional[bool] = None): + """ + Callback function to handle version option. + + Args: + version (bool, optional): If True, prints the current FM version and exits. Defaults to None. + """ + if version: + fm_version = get_current_fm_version() + richprint.print(fm_version, emoji_code='') + raise typer.Exit() \ No newline at end of file diff --git a/frappe_manager/utils/docker.py b/frappe_manager/utils/docker.py new file mode 100644 index 00000000..8fe0d0b4 --- /dev/null +++ b/frappe_manager/utils/docker.py @@ -0,0 +1,326 @@ +import os +import shlex +import shutil +import signal +import subprocess +import sys + +from datetime import datetime, timedelta +from importlib.metadata import version +from pathlib import Path +from queue import Queue +from subprocess import PIPE, Popen, run +from threading import Thread +from rich import control + +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union, overload + +from frappe_manager.logger import log +from frappe_manager.docker_wrapper.DockerException import DockerException +from frappe_manager.display_manager.DisplayManager import richprint + +process_opened = [] + +def reader(pipe, pipe_name, queue): + """ + Reads lines from a pipe and puts them into a queue. + + Args: + pipe (file-like object): The pipe to read from. + pipe_name (str): The name of the pipe. + queue (Queue): The queue to put the lines into. + """ + logger = log.get_logger() + try: + with pipe: + for line in iter(pipe.readline, b""): + queue_line = line.decode().strip('\n') + logger.debug(queue_line) + queue.put((pipe_name, str(queue_line).encode())) + finally: + queue.put(None) + + + +def stream_stdout_and_stderr( + full_cmd: list, + env: Dict[str, str] = None, +) -> Iterable[Tuple[str, bytes]]: + """ + Executes a command in Docker and streams the stdout and stderr outputs. + + Args: + full_cmd (list): The command to be executed in Docker. + env (Dict[str, str], optional): Environment variables to be passed to the Docker container. Defaults to None. + + Yields: + Tuple[str, bytes]: A tuple containing the source ("stdout" or "stderr") and the output line. + + Raises: + DockerException: If the Docker command returns a non-zero exit code. + + Returns: + Iterable[Tuple[str, bytes]]: An iterable of tuples containing the source and output line. + """ + logger = log.get_logger() + logger.debug('- -'*10) + logger.debug(f"DOCKER COMMAND: {' '.join(full_cmd)}") + if env is None: + subprocess_env = None + else: + subprocess_env = dict(os.environ) + subprocess_env.update(env) + + full_cmd = list(map(str, full_cmd)) + process = Popen(full_cmd, stdout=PIPE, stderr=PIPE, env=subprocess_env) + + process_opened.append(process.pid) + + q = Queue() + full_stderr = b"" # for the error message + # we use deamon threads to avoid hanging if the user uses ctrl+c + th = Thread(target=reader, args=[process.stdout, "stdout", q]) + th.daemon = True + th.start() + th = Thread(target=reader, args=[process.stderr, "stderr", q]) + th.daemon = True + th.start() + + for _ in range(2): + for source, line in iter(q.get, None): + yield source, line + if source == "stderr": + full_stderr += line + + exit_code = process.wait() + + logger.debug(f"RETURN CODE: {exit_code}") + logger.debug('- -'*10) + if exit_code != 0: + raise DockerException(full_cmd, exit_code, stderr=full_stderr) + + yield ("exit_code", str(exit_code).encode()) + +def run_command_with_exit_code( + full_cmd: list, + env: Dict[str, str] = None, + stream: bool = True, + quiet: bool = False +): + """ + Run a command and return the exit code. + + Args: + full_cmd (list): The command to be executed as a list of strings. + env (Dict[str, str], optional): Environment variables to be set for the command. Defaults to None. + stream (bool, optional): Flag indicating whether to stream the command output. Defaults to True. + quiet (bool, optional): Flag indicating whether to suppress the command output. Defaults to False. + + Raises: + DockerException: If the command execution returns a non-zero exit code. + """ + if stream: + if quiet: + for source ,line in stream_stdout_and_stderr(full_cmd): + if source == 'exit_code': + exit_code: int = int(line.decode()) + return(exit_code) + else: + return stream_stdout_and_stderr(full_cmd) + else: + from frappe_manager.display_manager.DisplayManager import richprint + output = run(full_cmd) + exit_code = output.returncode + if exit_code != 0: + raise DockerException(full_cmd,exit_code) + +def parameter_to_option(param: str) -> str: + """Converts a parameter to an option. + + Args: + param (str): The parameter to be converted. + + Returns: + str: The converted option. + """ + option = "--" + param.replace("_", "-") + return option + +def parameters_to_options(param: dict, exclude: list = []) -> list: + """ + Convert a dictionary of parameters to a list of options for a command. + + Args: + param (dict): The dictionary of parameters. + exclude (list, optional): A list of keys to exclude from the options. Defaults to []. + + Returns: + list: The list of options for the command. + """ + # remove the self parameter + temp_param: dict = dict(param) + + del temp_param["self"] + + for key in exclude: + del temp_param[key] + + # remove all parameters which are not booleans + params: list = [] + + for key in temp_param.keys(): + value = temp_param[key] + key = "--" + key.replace("_","-") + if type(value) == bool: + if value: + params.append(key) + if type(value) == int: + params.append(key) + params.append(value) + if type(value) == str: + if value: + params.append(key) + params.append(value) + if type(value) == list: + if value: + params.append(key) + params += value + + return params + +def is_current_user_in_group(group_name) -> bool: + """Check if the current user is in the given group. + + Args: + group_name (str): The name of the group to check. + + Returns: + bool: True if the current user is in the group, False otherwise. + """ + + from frappe_manager.display_manager.DisplayManager import richprint + + import platform + if platform.system() == 'Linux': + import grp + import pwd + import os + current_user = pwd.getpwuid(os.getuid()).pw_name + try: + docker_gid = grp.getgrnam(group_name).gr_gid + docker_group_members = grp.getgrgid(docker_gid).gr_mem + if current_user in docker_group_members: + return True + else: + richprint.error(f"Your current user [blue][b] {current_user} [/b][/blue] is not in the 'docker' group. Please add it and restart your terminal.") + return False + except KeyError: + richprint.error(f"The group '{group_name}' does not exist. Please create it and add your current user [blue][b] {current_user} [/b][/blue] to it.") + return False + else: + return True + +def generate_random_text(length=50): + """ + Generate a random text of specified length. + + Parameters: + length (int): The length of the random text to be generated. Default is 50. + + Returns: + str: The randomly generated text. + """ + import random, string + alphanumeric_chars = string.ascii_letters + string.digits + return "".join(random.choice(alphanumeric_chars) for _ in range(length)) + +def host_run_cp(image: str, source: str, destination: str, docker, verbose=False): + """Copy files from source to destination using Docker. + + Args: + image (str): The Docker image to run. + source (str): The source file or directory path. + destination (str): The destination file or directory path. + docker: The Docker client object. + verbose (bool, optional): Whether to display verbose output. Defaults to False. + """ + status_text = "Copying files" + richprint.change_head(f"{status_text} {source} -> {destination}") + source_container_name = generate_random_text(10) + dest_path = Path(destination) + errror_exception = None + + failed: bool = False + # run the container + try: + output = docker.run( + image=image, + name=source_container_name, + detach=True, + stream=not verbose, + command="tail -f /dev/null", + ) + if not verbose: + richprint.live_lines(output, padding=(0, 0, 0, 2)) + + except DockerException as e: + errror_exception = e + failed = 0 + + if not failed: + # cp from the container + try: + output = docker.cp( + source=source, + destination=destination, + source_container=source_container_name, + stream=not verbose, + ) + if not verbose: + richprint.live_lines(output, padding=(0, 0, 0, 2)) + except DockerException as e: + errror_exception = e + failed = 1 + + # # kill the container + # try: + # output = docker.kill(container=source_container_name,stream=True) + # richprint.live_lines(output, padding=(0,0,0,2)) + # except DockerException as e: + # richprint.exit(f"{status_text} failed. Error: {e}") + + if not failed: + # rm the container + try: + output = docker.rm( + container=source_container_name, force=True, stream=not verbose + ) + if not verbose: + richprint.live_lines(output, padding=(0, 0, 0, 2)) + except DockerException as e: + errror_exception = e + failed = 2 + + # check if the destination file exists + if not type(failed) == bool: + if failed > 1: + if dest_path.exists(): + import shutil + + shutil.rmtree(dest_path) + if failed == 2: + try: + output = docker.rm( + container=source_container_name, force=True, stream=not verbose + ) + if not verbose: + richprint.live_lines(output, padding=(0, 0, 0, 2)) + except DockerException as e: + pass + # TODO introuduce custom exception to handle this type of cases where if the flow is not completed then it should raise exception which is handled by caller and then site creation check is done + richprint.exit(f"{status_text} failed.", error_msg=errror_exception) + + elif not Path(destination).exists(): + richprint.exit(f"{status_text} failed. Copied {destination} not found.") + + diff --git a/frappe_manager/utils/helpers.py b/frappe_manager/utils/helpers.py new file mode 100644 index 00000000..98f4c1e1 --- /dev/null +++ b/frappe_manager/utils/helpers.py @@ -0,0 +1,312 @@ +import importlib +import sys +import requests +import json +import subprocess +import platform +import time +import secrets +import grp + +from pathlib import Path +from frappe_manager.logger import log +from frappe_manager.docker_wrapper.DockerException import DockerException +from frappe_manager.display_manager.DisplayManager import richprint + + +def remove_zombie_subprocess_process(process): + """ + This function iterates over a list of process IDs and terminates each process. + + Args: + process (list): A list of process IDs to be terminated. + + Returns: + None + """ + if process: + logger = log.get_logger() + logger.cleanup("-" * 20) + logger.cleanup(f"PROCESS: USED PROCESS {process}") + + import psutil + for pid in process: + try: + process = psutil.Process(pid) + process.terminate() + logger.cleanup(f"Terminated Process {process.cmdline}:{pid}") + except psutil.NoSuchProcess: + logger.cleanup(f"{pid} Process not found") + except psutil.AccessDenied: + logger.cleanup(f"{pid} Permission denied") + logger.cleanup("-" * 20) + + +def check_update(): + """ + Retrieves the latest version of the frappe-manager package from PyPI and compares it with the currently installed version. + If a newer version is available, it displays a warning message suggesting to update the package. + """ + url = "https://pypi.org/pypi/frappe-manager/json" + try: + update_info = requests.get(url, timeout=0.1) + update_info = json.loads(update_info.text) + fm_version = importlib.metadata.version("frappe-manager") + latest_version = update_info["info"]["version"] + if not fm_version == latest_version: + richprint.warning( + f'Ready for an update? Run "pip install --upgrade frappe-manager" to update to the latest version {latest_version}.', + emoji_code=":arrows_counterclockwise:️", + ) + except Exception as e: + pass + + +def is_port_in_use(port): + """ + Check if a port is in use or not. + + Args: + port (int): The port number to check. + + Returns: + bool: True if the port is in use, False otherwise. + """ + import psutil + + for conn in psutil.net_connections(): + if conn.laddr.port == port and conn.status == "LISTEN": + return True + return False + + +def check_ports(ports): + """ + Checks if the ports are in use. + + Args: + ports (list): List of ports to be checked. + + Returns: + list: List of binded ports (can be empty). + """ + # TODO handle if ports are open using docker + current_system = platform.system() + already_binded = [] + for port in ports: + if current_system == "Darwin": + # Mac Os + # check port using lsof command + cmd = f"lsof -iTCP:{port} -sTCP:LISTEN -P -n" + try: + output = subprocess.run( + cmd, check=True, shell=True, capture_output=True + ) + if output.returncode == 0: + already_binded.append(port) + except subprocess.CalledProcessError as e: + pass + else: + # Linux or any other machines + if is_port_in_use(port): + already_binded.append(port) + + return already_binded + + +def check_and_display_port_status(ports_to_check: list, exclude=[]): + """ + Check if the specified ports are already binded and display a message if they are. + + Args: + ports_to_check (list): List of ports to check. + exclude (list, optional): List of ports to exclude from checking. Defaults to []. + """ + richprint.change_head("Checking Ports") + if exclude: + # Removing elements present in remove_array from original_array + ports_to_check = [x for x in exclude if x not in ports_to_check] + + if ports_to_check: + already_binded = check_ports(ports_to_check) + if already_binded: + richprint.exit( + f"Whoa there! Looks like the {' '.join([ str(x) for x in already_binded ])} { 'ports are' if len(already_binded) > 1 else 'port is' } having a party already! Can you do us a solid and free up those ports?" + ) + richprint.print("Ports Check : Passed") + + +def generate_random_text(length=50): + """ + Generate a random text of specified length. + + Parameters: + length (int): The length of the random text to be generated. Default is 50. + + Returns: + str: The randomly generated text. + """ + import random, string + + alphanumeric_chars = string.ascii_letters + string.digits + return "".join(random.choice(alphanumeric_chars) for _ in range(length)) + + + +def is_cli_help_called(ctx): + """ + Checks if the help is called for the CLI command. + + Args: + ctx (object): The context object representing the CLI command. + + Returns: + bool: True if the help command is called, False otherwise. + """ + help_called = False + # is called command is sub command group + try: + for subtyper_command in ctx.command.commands[ + ctx.invoked_subcommand + ].commands.keys(): + check_command = " ".join(sys.argv[2:]) + if check_command == subtyper_command: + if ( + ctx.command.commands[ctx.invoked_subcommand] + .commands[subtyper_command] + .params + ): + help_called = True + except AttributeError: + help_called = False + + if not help_called: + # is called command is sub command + check_command = " ".join(sys.argv[1:]) + + if check_command == ctx.invoked_subcommand: + # is called command supports arguments then help called + + if ctx.command.commands[ctx.invoked_subcommand].params: + help_called = True + + return help_called + +def get_current_fm_version(): + """ + Get the current version of the frappe-manager package. + + Returns: + str: The current version of the frappe-manager package. + """ + return importlib.metadata.version("frappe-manager") + + +def check_frappe_app_exists(appname: str, branchname: str | None = None): + """ + Check if a Frappe app exists on GitHub. + + Args: + appname (str): The name of the Frappe app. + branchname (str | None, optional): The name of the branch to check. Defaults to None. + + Returns: + dict: A dictionary containing the existence status of the app and branch (if provided). + """ + try: + app_url = f"https://github.com/frappe/{appname}" + app = requests.get(app_url).status_code + + if branchname: + branch_url = f"https://github.com/frappe/{appname}/tree/{branchname}" + branch = requests.get(branch_url).status_code + return { + "app": True if app == 200 else False, + "branch": True if branch == 200 else False, + } + return {"app": True if app == 200 else False} + except Exception: + richprint.exit("Not able to connect to github.com.") + + +def represent_null_empty(string_null): + """ + Replaces the string "null" with an empty string. + + Args: + string_null (str): The input string. + + Returns: + str: The modified string with "null" replaced by an empty string. + """ + return string_null.replace("null","") + + +def log_file(file, refresh_time:float = 0.1, follow:bool =False): + ''' + Generator function that yields new lines in a file + + Parameters: + - file: The file object to read from + - refresh_time: The time interval (in seconds) to wait before checking for new lines in the file (default: 0.1) + - follow: If True, the function will continue to yield new lines as they are added to the file (default: False) + + Returns: + - A generator that yields each new line in the file + ''' + file.seek(0) + + # start infinite loop + while True: + # read last line of file + line = file.readline() + if not line: + if not follow: + break + # sleep if file hasn't been updated + time.sleep(refresh_time) + continue + line = line.strip('\n') + yield line + +def get_container_name_prefix(site_name): + """ + Returns the container name prefix by removing dots from the site name. + + Args: + site_name (str): The name of the site. + + Returns: + str: The container name prefix. + """ + return site_name.replace('.', '') + +def random_password_generate(password_length=13, symbols=False): + # Define the character set to include symbols + # symbols = "!@#$%^&*()_-+=[]{}|;:,.<>?`~" + symbols = "!@%_-+?" + + # Generate a password without symbols using token_urlsafe + + generated_password = secrets.token_urlsafe(password_length) + + # Replace some characters with symbols in the generated password + if symbols: + password = "".join( + c if secrets.choice([True, False]) else secrets.choice(symbols) + for c in generated_password + ) + return password + + return generated_password + +# Retrieve Unix groups and their corresponding integer mappings +def get_unix_groups(): + groups = {} + for group_entry in grp.getgrall(): + group_name = group_entry.gr_name + groups[group_name] = group_entry.gr_gid + return groups + +def downgrade_package(package_name, version): + subprocess.check_call([sys.executable, '-m', 'pip', 'install', f'{package_name}=={version}']) diff --git a/frappe_manager/utils/site.py b/frappe_manager/utils/site.py new file mode 100644 index 00000000..2ba92ed9 --- /dev/null +++ b/frappe_manager/utils/site.py @@ -0,0 +1,77 @@ +from rich.table import Table + + +def generate_services_table(services_status: dict): + # running site services status + services_table = Table( + show_lines=False, + show_edge=False, + pad_edge=False, + show_header=False, + expand=True, + box=None, + ) + + services_table.add_column( + "Service Status", ratio=1, no_wrap=True, width=None, min_width=20 + ) + services_table.add_column( + "Service Status", ratio=1, no_wrap=True, width=None, min_width=20 + ) + + for index in range(0,len(services_status),2): + first_service_table = None + second_service_table = None + + try: + first_service = list(services_status.keys())[index] + first_service_table = create_service_element(first_service, services_status[first_service]) + except IndexError: + pass + + try: + second_service = list(services_status.keys())[index+1] + second_service_table = create_service_element(second_service, services_status[second_service]) + except IndexError: + pass + + services_table.add_row(first_service_table, second_service_table) + + return services_table + +def create_service_element(service, running_status): + service_table = Table( + show_lines=False, + show_header=False, + highlight=True, + expand=True, + box=None, + ) + service_table.add_column("Service", justify="left", no_wrap=True) + service_table.add_column("Status", justify="right", no_wrap=True) + service_status = ":green_square:" if running_status == "running" else ":red_square:" + service_table.add_row( + f"{service}", + f"{service_status}", + ) + return service_table + +def parse_docker_volume(volume_string): + + string_parts = volume_string.split(':') + + if len(string_parts) > 1: + + volume = {"src": string_parts[0], "dest": string_parts[0]} + + is_bind_mount = string_parts[0].startswith('./') + + if len(string_parts) > 2: + volume = {"src": string_parts[0], "dest": string_parts[1]} + + volume['type'] = 'bind' + + if not is_bind_mount: + volume['type'] = 'volume' + + return volume From 7f3811d89b9f7ac75d80ce4b487c425c9ed09031 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 31 Jan 2024 14:14:03 +0530 Subject: [PATCH 080/100] feat: global services --- Docker/frappe/user-script.sh | 22 +- frappe_manager/logger/log.py | 1 - frappe_manager/services_manager/__init__.py | 5 + frappe_manager/services_manager/commands.py | 81 +++ frappe_manager/services_manager/services.py | 538 +++++++++++++++++ .../services_manager/services_exceptions.py | 15 + frappe_manager/site_manager/SiteManager.py | 560 +++++++----------- frappe_manager/site_manager/site.py | 373 +++++++++--- .../site_manager/site_exceptions.py | 46 ++ .../workers_manager/SiteWorker.py | 15 +- .../workers_manager/siteworker_exceptions.py | 0 11 files changed, 1205 insertions(+), 451 deletions(-) create mode 100644 frappe_manager/services_manager/__init__.py create mode 100644 frappe_manager/services_manager/commands.py create mode 100644 frappe_manager/services_manager/services.py create mode 100644 frappe_manager/services_manager/services_exceptions.py create mode 100644 frappe_manager/site_manager/site_exceptions.py create mode 100644 frappe_manager/site_manager/workers_manager/siteworker_exceptions.py diff --git a/Docker/frappe/user-script.sh b/Docker/frappe/user-script.sh index 06a76540..f233c5c1 100755 --- a/Docker/frappe/user-script.sh +++ b/Docker/frappe/user-script.sh @@ -1,6 +1,5 @@ #!/bin/bash # This script creates bench and executes it. - set -e emer() { echo "$@" @@ -19,7 +18,7 @@ REDIS_SOCKETIO_PORT=80 WEB_PORT=80 if [[ ! "${MARIADB_HOST:-}" ]]; then - MARIADB_HOST='mariadb' + MARIADB_HOST='global-db' fi if [[ ! "${MARIADB_ROOT_PASS:-}" ]]; then @@ -43,7 +42,6 @@ if [[ ! -d "frappe-bench" ]]; then [[ "${DB_NAME:-}" ]] || emer "[ERROR] DB_NAME env not found. Please provide DB_NAME env." [[ "${CONTAINER_NAME_PREFIX:-}" ]] || emer "[ERROR] CONTAINER_NAME_PREFIX env not found. Please provide CONTAINER_NAME_PREFIX env." - # create the bench $BENCH_COMMAND init --skip-assets --skip-redis-config-generation --frappe-branch "$FRAPPE_BRANCH" frappe-bench @@ -115,8 +113,7 @@ if [[ ! -d "frappe-bench" ]]; then chmod +x /opt/user/bench-dev-server.sh $BENCH_COMMAND build - - $BENCH_COMMAND new-site --db-root-password "$MARIADB_ROOT_PASS" --db-name "$DB_NAME" --no-mariadb-socket --admin-password "$ADMIN_PASS" "$SITENAME" + $BENCH_COMMAND new-site --db-root-password $(cat $MARIADB_ROOT_PASS) --db-name "$DB_NAME" --db-host "$MARIADB_HOST" --admin-password "$ADMIN_PASS" --no-mariadb-socket "$SITENAME" $BENCH_COMMAND --site "$SITENAME" scheduler enable wait @@ -135,6 +132,7 @@ if [[ ! -d "frappe-bench" ]]; then else + set -x wait-for-it -t 120 "$MARIADB_HOST":3306; wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-cache":6379; wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-queue":6379; @@ -158,22 +156,8 @@ else chmod +x /opt/user/bench-dev-server.sh - $BENCH_COMMAND config dns_multitenant on - $BENCH_COMMAND set-config -g db_host "$MARIADB_HOST" - $BENCH_COMMAND set-config -g db_port 3306 - $BENCH_COMMAND set-config -g redis_cache "redis://${CONTAINER_NAME_PREFIX}-redis-cache:6379" - $BENCH_COMMAND set-config -g redis_queue "redis://${CONTAINER_NAME_PREFIX}-redis-queue:6379" - $BENCH_COMMAND set-config -g redis_socketio "redis://${CONTAINER_NAME_PREFIX}-redis-socketio:6379" - $BENCH_COMMAND set-config -g webserver_port "$WEB_PORT" - $BENCH_COMMAND set-config -g socketio_port "$REDIS_SOCKETIO_PORT" - if [[ "${ENVIRONMENT}" = "dev" ]]; then - $BENCH_COMMAND set-config -g mail_port 1025 - $BENCH_COMMAND set-config -g mail_server 'mailhog' - $BENCH_COMMAND set-config -g disable_mail_smtp_authentication 1 - $BENCH_COMMAND set-config -g developer_mode "$DEVELOPER_MODE" - cp /opt/user/frappe-dev.conf /opt/user/conf.d/frappe-dev.conf else if [[ -f '/opt/user/conf.d/frappe-bench-frappe-web.fm.supervisor.conf' ]]; then diff --git a/frappe_manager/logger/log.py b/frappe_manager/logger/log.py index 0c6f7c2d..e92cda0e 100644 --- a/frappe_manager/logger/log.py +++ b/frappe_manager/logger/log.py @@ -35,7 +35,6 @@ def get_logger(log_dir=log_directory, log_file_name='fm') -> logging.Logger: # Build Log File Full Path logPath = log_dir / f"{log_file_name}.log" - try: log_dir.mkdir(parents=False, exist_ok=True) except PermissionError as e: diff --git a/frappe_manager/services_manager/__init__.py b/frappe_manager/services_manager/__init__.py new file mode 100644 index 00000000..fc0a03ed --- /dev/null +++ b/frappe_manager/services_manager/__init__.py @@ -0,0 +1,5 @@ +from enum import Enum + +class ServicesEnum(str, Enum): + global_db= "global-db" + global_nginx_proxy="global-nginx-proxy" diff --git a/frappe_manager/services_manager/commands.py b/frappe_manager/services_manager/commands.py new file mode 100644 index 00000000..6a116b3a --- /dev/null +++ b/frappe_manager/services_manager/commands.py @@ -0,0 +1,81 @@ +from typing import Annotated +from psutil import users +import typer + +from frappe_manager.services_manager.services import ServicesManager +from frappe_manager.display_manager.DisplayManager import richprint +from frappe_manager.services_manager import ServicesEnum + +services_app = typer.Typer(no_args_is_help=True, rich_markup_mode="rich") + +services_manager = None + +@services_app.callback() +def global_services_callback( + ctx: typer.Context, +): + global services_manager + + if not ctx.obj['is_help_called']: + services_manager = ctx.obj["services"] + services_manager.set_typer_context(ctx) + +def validate_servicename( + service_name: Annotated[str, typer.Argument(help="Name of the services_manager.")] +): + services = services_manager.composefile.get_services_list() + if not service_name in services: + richprint.exit(f"{service_name} is not a valid service name.") + else: + return service_name + +@services_app.command(no_args_is_help=True) +def stop( + service_name: Annotated[ + ServicesEnum, + typer.Argument(help="Name of the services_manager.") + ], +): + """Stops global services.""" + if services_manager.is_service_running(service_name.value): + services_manager.stop(service_name.value) + else: + richprint.exit(f"{service_name.value} is not running.") + + +@services_app.command(no_args_is_help=True) +def start( + service_name: Annotated[ + ServicesEnum, + typer.Argument(help="Name of the services_manager.") + ], +): + """Starts global services.""" + if not services_manager.is_service_running(service_name.value): + services_manager.start(service_name.value) + else: + richprint.exit(f"{service_name.value} is already running.") + +@services_app.command(no_args_is_help=True) +def restart( + service_name: Annotated[ + ServicesEnum, + typer.Argument(help="Name of the services_manager.") + ], +): + """Restarts global services.""" + services_manager.restart(service_name.value) + + +@services_app.command(no_args_is_help=True) +def shell( + service_name: Annotated[ + ServicesEnum, + typer.Argument(help="Name of the services_manager.") + ], + user: Annotated[str, typer.Option(help="Connect as this user.")] = None, +): + """ + Open shell for the specificed global services_manager. + """ + services_manager.shell(service_name.value, users) diff --git a/frappe_manager/services_manager/services.py b/frappe_manager/services_manager/services.py new file mode 100644 index 00000000..6a6eebbb --- /dev/null +++ b/frappe_manager/services_manager/services.py @@ -0,0 +1,538 @@ +import shutil +import platform +import os +import json +import typer +from datetime import datetime +from pathlib import Path +from typing import Optional + +from frappe_manager import CLI_DIR +from frappe_manager.services_manager.services_exceptions import ServicesComposeNotExist, ServicesDBNotStart +from frappe_manager.display_manager.DisplayManager import richprint +from frappe_manager.compose_manager.ComposeFile import ComposeFile +from frappe_manager.utils.helpers import ( + random_password_generate, + check_and_display_port_status, + get_unix_groups, + # check_ports_with_msg, + +) +from frappe_manager.utils.docker import host_run_cp +from frappe_manager.docker_wrapper import DockerClient, DockerException + + +class ServicesManager: + _instance = None + + def __new__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __init__(self, verbose: bool = False) -> None: + self.path = CLI_DIR / "services" + self.quiet = not verbose + self.typer_context: Optional[typer.Context] = None + self.compose_path = self.path / "docker-compose.yml" + + def set_typer_context(self, ctx: typer.Context): + """ + The function sets the typer context from the + :param typer context + :type ctx: typer.Context + """ + self.typer_context = ctx + + def entrypoint_checks(self, start = False): + + if not self.path.exists(): + richprint.print(f"Creating services",emoji_code=":construction:") + self.path.mkdir(parents=True, exist_ok=True) + self.create() + self.pull() + richprint.print(f"Creating services: Done") + if start: + self.start() + + if not self.compose_path.exists(): + raise ServicesComposeNotExist("Seems like services has taken a down. Please recreate services.") + # richprint.exit( + # "Seems like global services has taken a down. Please recreate global services." + # ) + + if start: + if not self.typer_context.invoked_subcommand == "service": + if not self.running(): + richprint.warning("services are not running. Starting it") + self.start() + + def init(self): + # check if the global services exits if not then create + + self.composefile = ComposeFile( + self.compose_path, template_name="docker-compose.services.tmpl" + ) + + self.docker = DockerClient(compose_file_path=self.composefile.compose_path) + + + def create(self, backup=False): + envs = { + "global-db": { + "MYSQL_ROOT_PASSWORD_FILE": '/run/secrets/db_root_password', + "MYSQL_DATABASE": "root", + "MYSQL_USER": "admin", + "MYSQL_PASSWORD_FILE": '/run/secrets/db_password', + } + } + + current_system = platform.system() + + + try: + user = { + "global-db": { + "uid": os.getuid(), + "gid": os.getgid(), + }} + if not current_system == "Darwin": + user["global-nginx-proxy"]: { + "uid": os.getuid(), + "gid": get_unix_groups()["docker"], + } + + except KeyError: + richprint.exit("docker group not found in system.") + + inputs = {"environment": envs, "user": user} + + if backup: + if self.path.exists(): + backup = CLI_DIR / "backups" + backup.mkdir(parents=True, exist_ok=True) + current_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + backup_dir_name = f"services_{current_time}" + self.path.rename(backup / backup_dir_name) + + shutil.rmtree(self.path) + + # create required directories + # this list of directores can be automated + dirs_to_create = [ + "mariadb/conf", + "mariadb/logs", + "nginx-proxy/dhparam", + "nginx-proxy/certs", + "nginx-proxy/confd", + "nginx-proxy/htpasswd", + "nginx-proxy/vhostd", + "nginx-proxy/html", + "nginx-proxy/logs", + "nginx-proxy/run", + "nginx-proxy/cache", + "secrets" + ] + #"mariadb/data", + + # create dirs + for folder in dirs_to_create: + temp_dir = self.path / folder + try: + temp_dir.mkdir(parents=True, exist_ok=True) + except Exception as e: + richprint.exit( + f"Failed to create global services bind mount directories. Error: {e}" + ) + + # populate secrets for db + db_password_path = self.path/ 'secrets'/ 'db_password.txt' + db_root_password_path = self.path/ 'secrets'/ 'db_root_password.txt' + + with open(db_password_path,'w') as f: + f.write(random_password_generate(password_length=16,symbols=True)) + + with open(db_root_password_path,'w') as f: + f.write(random_password_generate(password_length=24,symbols=True)) + + + # populate mariadb config + mariadb_conf = self.path / "mariadb/conf" + mariadb_conf = str(mariadb_conf.absolute()) + host_run_cp( + image="mariadb:10.6", + source="/etc/mysql/.", + destination=mariadb_conf, + docker=self.docker, + ) + # set secrets in compose + self.generate_compose(inputs) + + if current_system == "Darwin": + self.composefile.remove_container_user('global-nginx-proxy') + self.composefile.remove_container_user('global-db') + + self.composefile.set_secret_file_path('db_password',str(db_password_path.absolute())) + self.composefile.set_secret_file_path('db_root_password',str(db_root_password_path.absolute())) + self.composefile.write_to_file() + + def get_database_info(self): + """ + Provides info about databse + """ + info: dict = {} + try: + password_path = self.composefile.get_secret_file_path('db_root_password') + with open(Path(password_path),'r') as f: + password = f.read() + info["password"] = password + info["user"] = "root" + info["host"] = "global-db" + info["port"] = 3306 + return info + except KeyError as e: + # TODO secrets not exists + info["password"] = None + info["user"] = "root" + info["host"] = "global-db" + info["port"] = 3306 + return info + + def exists(self): + return (self.path / "docker-compose.yml").exists() + + def generate_compose(self, inputs: dict): + """ + This can get a file like + inputs = { + "environment" : {'key': 'value'}, + "extrahosts" : {'key': 'value'}, + "user" : {'uid': 'value','gid': 'value'}, + "labels" : {'key': 'value'}, + } + """ + try: + # handle envrionment + if "environment" in inputs.keys(): + environments: dict = inputs["environment"] + self.composefile.set_all_envs(environments) + + # handle lablels + if "labels" in inputs.keys(): + labels: dict = inputs["labels"] + self.composefile.set_all_labels(labels) + + # handle user + if "user" in inputs.keys(): + user: dict = inputs["user"] + for container_name in user.keys(): + uid = user[container_name]["uid"] + gid = user[container_name]["gid"] + self.composefile.set_user(container_name, uid, gid) + + except Exception as e: + richprint.exit(f"Not able to generate global site compose. Error: {e}") + + def pull(self): + """ + The function pulls Docker images and displays the status of the operation. + """ + status_text = "Pulling services images" + richprint.change_head(status_text) + try: + output = self.docker.compose.pull(stream=self.quiet) + richprint.stdout.clear_live() + if self.quiet: + richprint.live_lines(output, padding=(0, 0, 0, 2)) + richprint.print(f"{status_text}: Done") + except DockerException as e: + richprint.warning(f"{status_text}: Failed") + + def get_services_running_status(self) -> dict: + services = self.composefile.get_services_list() + containers = self.composefile.get_container_names().values() + services_status = {} + try: + output = self.docker.compose.ps( + service=services, format="json", all=True, stream=True + ) + status: list = [] + for source, line in output: + if source == "stdout": + current_status = json.loads(line.decode()) + if type(current_status) == dict: + status.append(current_status) + else: + status += current_status + + # this is done to exclude docker runs using docker compose run command + for container in status: + if container["Name"] in containers: + services_status[container["Service"]] = container["State"] + return services_status + except DockerException as e: + richprint.exit(f"{e.stdout}{e.stderr}") + + def is_service_running(self, service): + running_status = self.get_services_running_status() + try: + if running_status[service] == "running": + return True + else: + return False + except KeyError: + return False + + def running(self) -> bool: + """ + The `running` function checks if all the services defined in a Docker Compose file are running. + :return: a boolean value. If the number of running containers is greater than or equal to the number + of services listed in the compose file, it returns True. Otherwise, it returns False. + """ + services = self.composefile.get_services_list() + running_status = self.get_services_running_status() + + if running_status: + for service in services: + try: + if not running_status[service] == "running": + return False + except KeyError: + return False + else: + return False + return True + + def get_host_port_binds(self): + try: + output = self.docker.compose.ps(all=True, format="json", stream=True) + status_list: list = [] + + for source, line in output: + if source == "stdout": + status = json.loads(line.decode()) + if type(status) == list: + status_list += status + else: + status_list.append(status) + + ports_info = [] + + for container in status_list: + try: + port_info = container["Publishers"] + if port_info: + ports_info = ports_info + port_info + except KeyError as e: + pass + + published_ports = set() + + for port in ports_info: + try: + published_port = port["PublishedPort"] + if published_port > 0: + published_ports.add(published_port) + + except KeyError as e: + pass + + return list(published_ports) + + except Exception as e: + return [] + # richprint.exit(f"{e.stdout}{e.stderr}") + + def start(self, service=None): + if not self.running(): + docker_used_ports = self.get_host_port_binds() + check_and_display_port_status([80, 443], exclude=docker_used_ports) + + status_text = "Starting services" + richprint.change_head(status_text) + + if service: + if self.is_service_running(service): + richprint.warning(f"{service} is already in running state.") + + try: + if service: + output = self.docker.compose.up( + services=[service], detach=True, pull="missing", stream=self.quiet + ) + else: + output = self.docker.compose.up( + detach=True, pull="missing", stream=self.quiet + ) + + if self.quiet: + richprint.live_lines(output, padding=(0, 0, 0, 2)) + richprint.print(f"{status_text}: Done") + except DockerException as e: + richprint.exit(f"{status_text}: Failed", error_msg=e) + + def restart(self, service=None): + status_text = f"Restarting service {service}" + richprint.change_head(status_text) + try: + if service: + output = self.docker.compose.restart( + services=[service], stream=self.quiet + ) + else: + output = self.docker.compose.restart(stream=self.quiet) + + if self.quiet: + richprint.live_lines(output, padding=(0, 0, 0, 2)) + richprint.print(f"{status_text}: Done") + except DockerException as e: + richprint.exit(f"{status_text}: Failed", error_msg=e) + + def stop(self, service=None): + status_text = "Stopping global services" + richprint.change_head(status_text) + try: + if service: + status_text = f"Stopping global service {service}" + output = self.docker.compose.stop( + [service], stream=self.quiet + ) + else: + output = self.docker.compose.stop(stream=self.quiet) + + if self.quiet: + richprint.live_lines(output, padding=(0, 0, 0, 2)) + richprint.print(f"{status_text}: Done") + except DockerException as e: + richprint.exit(f"{status_text}: Failed", error_msg=e) + + def shell(self, container: str, user: str | None = None): + """ + The `shell` function spawns a shell for a specified container and user. + + :param container: The `container` parameter is a string that specifies the name of the container in + which the shell command will be executed + :type container: str + :param user: The `user` parameter is an optional argument that specifies the user under which the + shell command should be executed. If a user is provided, the shell command will be executed as that + user. If no user is provided, the shell command will be executed as the default user + :type user: str | None + """ + # TODO check user exists + richprint.stop() + shell_path = "/bin/bash" + try: + if user: + self.docker.compose.exec(container, user=user, command=shell_path) + else: + self.docker.compose.exec(container, command=shell_path) + except DockerException as e: + richprint.warning(f"Shell exited with error code: {e.return_code}") + + def remove_db_user(self,user_name): + global_db_info = self.get_database_info() + db_user = global_db_info["user"] + db_password = global_db_info["password"] + + remove_db_user = f"/usr/bin/mariadb -u{db_user} -p'{db_password}' -e 'DROP USER `{user_name}`@`%`;'" + # show_db_user= f"/usr/bin/mariadb -h{global_db_info['host']} -u{global_db_info['user']} -p'{global_db_info['password']}' -e 'SELECT User, Host FROM mysql.user;'" + # output = self.docker.compose.exec('frappe',command=show_db_user) + try: + output = self.docker.compose.exec( + "global-db", command=remove_db_user, stream=self.quiet + ) + if self.quiet: + exit_code = richprint.live_lines(output, padding=(0, 0, 0, 2)) + richprint.print(f"Removed {user_name} DB User: Done") + except DockerException as e: + richprint.warning(f"Remove DB User: Failed") + + def remove_db(self,db_name): + + global_db_info = self.get_database_info() + db_user = global_db_info["user"] + db_password = global_db_info["password"] + + # remove database + remove_db_command = f"/usr/bin/mariadb -u{db_user} -p'{db_password}' -e 'DROP DATABASE `{db_name}`;'" + # show_db_command = f"/usr/bin/mariadb -h{global_db_info['host']} -u{global_db_info['user']} -p'{global_db_info['password']}' -e 'show databases;'" + # output = self.docker.compose.exec('frappe',command=show_db_command) + try: + output = self.docker.compose.exec( + "global-db", command=remove_db_command, stream=self.quiet + ) + if self.quiet: + exit_code = richprint.live_lines(output, padding=(0, 0, 0, 2)) + richprint.print(f"Removed {db_name} DB: Done") + except DockerException as e: + richprint.warning(f"Remove DB: Failed") + + def down(self, remove_ophans=True, volumes=True) -> bool: + """ + The `down` function removes containers using Docker Compose and prints the status of the operation. + """ + if self.composefile.exists(): + status_text = "Removing Containers" + richprint.change_head(status_text) + try: + output = self.docker.compose.down( + remove_orphans=remove_ophans, + volumes=volumes, + stream=self.quiet, + ) + if self.quiet: + exit_code = richprint.live_lines(output, padding=(0, 0, 0, 2)) + richprint.print(f"Removing Containers: Done") + except DockerException as e: + richprint.exit(f"{status_text}: Failed") + + def add_user(self, site_db_name, site_db_user, site_db_pass, timeout = 25): + + db_host = '127.0.0.1' + global_db_info = self.get_database_info() + db_user = global_db_info["user"] + db_password = global_db_info["password"] + + + remove_db_user = f"/usr/bin/mariadb -P3306 -h{db_host} -u{db_user} -p'{db_password}' -e 'DROP USER `{site_db_user}`@`%`;'" + add_db_user = f"/usr/bin/mariadb -h{db_host} -P3306 -u{db_user} -p'{db_password}' -e 'CREATE USER `{site_db_user}`@`%` IDENTIFIED BY \"{site_db_pass}\";'" + grant_user = f"/usr/bin/mariadb -h{db_host} -P3306 -u{db_user} -p'{db_password}' -e 'GRANT ALL PRIVILEGES ON `{site_db_name}`.* TO `{site_db_user}`@`%`;'" + # SHOW_db_user= f"/usr/bin/mariadb -P3306-h{db_host} -u{db_user} -p'{db_password}' -e 'SELECT User, Host FROM mysql.user;'" +# + # import time; + # check_connection_command = f"/usr/bin/mariadb -h{db_host} -u{db_user} -p'{db_password}' -e 'SHOW DATABASES;'" + + # i = 0 + # connected = False + + # error = None + # while i < timeout: + # try: + # time.sleep(5) + # output = self.docker.compose.exec('global-db', command=check_connection_command, stream=self.quiet, stream_only_exit_code=True) + # if next(output) == 0: + # connected = True + # except DockerException as e: + # error = e + # pass + + # i += 1 + + # if not connected: + # raise ServicesDBNotStart(f"DB did not start: {error}") + + removed = True + try: + output = self.docker.compose.exec('global-db', command=remove_db_user, stream=self.quiet,stream_only_exit_code=True) + except DockerException as e: + removed = False + if 'error 1396' in str(e.stderr).lower(): + removed = True + + if removed: + try: + output = self.docker.compose.exec("global-db", command=add_db_user, stream=self.quiet,stream_only_exit_code=True) + output = self.docker.compose.exec("global-db", command=grant_user, stream=self.quiet,stream_only_exit_code=True) + richprint.print(f"Recreated user {site_db_user}") + except DockerException as e: + raise ServicesDBNotStart(f"Database user creation failed: {e}") diff --git a/frappe_manager/services_manager/services_exceptions.py b/frappe_manager/services_manager/services_exceptions.py new file mode 100644 index 00000000..93b84bb1 --- /dev/null +++ b/frappe_manager/services_manager/services_exceptions.py @@ -0,0 +1,15 @@ + +class ServicesComposeNotExist(Exception): + def __init__(self, message): + message = message + super().__init__(message) + +class ServicesSecretsDBRootPassNotExist(Exception): + def __init__(self, message): + message = message + super().__init__(message) + +class ServicesDBNotStart(Exception): + def __init__(self, message): + message = message + super().__init__(message) diff --git a/frappe_manager/site_manager/SiteManager.py b/frappe_manager/site_manager/SiteManager.py index 5ae6cf17..149ed00d 100644 --- a/frappe_manager/site_manager/SiteManager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -1,47 +1,38 @@ -from frappe_manager.docker_wrapper import DockerClient, DockerException -from typing import List, Optional -from pathlib import Path import subprocess import json import shlex +from ruamel.yaml import serialize import typer import shutil +from typing import List, Optional +from pathlib import Path from frappe_manager.site_manager.site import Site from frappe_manager.display_manager.DisplayManager import richprint +from frappe_manager.docker_wrapper import DockerClient, DockerException from frappe_manager import CLI_DIR +from rich.table import Table -from frappe_manager.utils import ( - check_ports_with_msg, -) - -from rich.columns import Columns -from rich.panel import Panel -from rich.table import Table, Row -from rich.text import Text -from rich.console import Group -from rich import box +from frappe_manager.utils.helpers import check_and_display_port_status +from frappe_manager.utils.site import generate_services_table class SiteManager: - def __init__(self, sitesdir: Path): + def __init__(self, sitesdir: Path, services = None): self.sitesdir = sitesdir self.site = None self.sitepath = None self.verbose = False + self.services = services self.typer_context: Optional[typer.Context] = None def init(self, sitename: str | None = None): """ - The `init` function initializes a site by checking if the site directory exists, creating it if - necessary, and setting the site name and path. + Initializes the SiteManager object. - :param sitename: The `sitename` parameter is a string that represents the name of the site. It is - optional and can be set to `None`. If a value is provided, it will be used to create a site path by - appending ".localhost" to the sitename - :type sitename: str| None + Args: + sitename (str | None): The name of the site. If None, the default site will be used. """ - if sitename: if not ".localhost" in sitename: sitename = sitename + ".localhost" @@ -63,7 +54,7 @@ def init(self, sitename: str | None = None): f"The site '{sitename}' does not exist. Aborting operation." ) - self.site: Site = Site(sitepath, sitename, verbose=self.verbose) + self.site: Site = Site(sitepath, sitename, verbose=self.verbose,services=self.services) def set_verbose(self): """ @@ -73,61 +64,37 @@ def set_verbose(self): def set_typer_context(self, ctx: typer.Context): """ - The function sets the typer context from the - :param typer context - :type ctx: typer.Context + Sets the Typer context for the SiteManager. + + Parameters: + - ctx (typer.Context): The Typer context to be set. """ self.typer_context = ctx - def __get_all_sites_path(self, exclude: List[str] = []): - """ - The function `__get_all_sites_path` returns a list of paths to all the `docker-compose.yml` files in - the `sitesdir` directory, excluding any directories specified in the `exclude` list. - - :param exclude: The `exclude` parameter is a list of strings that contains the names of directories - to be excluded from the list of sites paths - :type exclude: List[str] - :return: a list of paths to `docker-compose.yml` files within directories in `self.sitesdir`, - excluding any directories specified in the `exclude` list. - """ - sites_path = [] - for d in self.sitesdir.iterdir(): - if d.is_dir(): - if not d.parts[-1] in exclude: - d = d / "docker-compose.yml" - if d.exists(): - sites_path.append(d) - return sites_path - - def get_all_sites(self): - """ - The function `get_all_sites` returns a dictionary of site names and their corresponding - docker-compose.yml file paths from a given directory. - :return: a dictionary where the keys are the names of directories within the `sitesdir` directory - and the values are the paths to the corresponding `docker-compose.yml` files within those - directories. - """ + def get_all_sites(self, exclude: List[str] = []): sites = {} for dir in self.sitesdir.iterdir(): - if dir.is_dir(): + if dir.is_dir() and dir.parts[-1] not in exclude: name = dir.parts[-1] dir = dir / "docker-compose.yml" if dir.exists(): - sites[name] = str(dir) + sites[name] = dir return sites def stop_sites(self): """ - The `stop_sites` function stops all sites except the current site by halting their Docker - containers. + Stops all the sites except the current site. """ status_text = "Halting other sites" richprint.change_head(status_text) + + exclude = [] + if self.site: exclude = [self.site.name] - site_compose: list = self.__get_all_sites_path(exclude) - else: - site_compose: list = self.__get_all_sites_path() + + site_compose: list = list(self.get_all_sites(exclude).values()) + if site_compose: for site_compose_path in site_compose: docker = DockerClient(compose_file_path=site_compose_path) @@ -141,46 +108,57 @@ def stop_sites(self): def create_site(self, template_inputs: dict): """ - The `create_site` function creates a new site directory, generates a compose file, pulls the - necessary images, starts the site, and displays information about the site. + Creates a new site using the provided template inputs. - :param template_inputs: The `template_inputs` parameter is a dictionary that contains the inputs or - configuration values required to generate the compose file for the site. These inputs can be used to - customize the site's configuration, such as database settings, domain name, etc - :type template_inputs: dict + Args: + template_inputs (dict): A dictionary containing the template inputs. + + Returns: + None """ - # check if provided sitename is valid and only one level subdom of localhost self.site.validate_sitename() - self.stop_sites() - # check if ports are available - self.check_ports() + richprint.change_head(f"Creating Site Directory") self.site.create_site_dir() + richprint.change_head(f"Generating Compose") self.site.generate_compose(template_inputs) self.site.create_compose_dirs() self.site.pull() + richprint.change_head(f"Starting Site") self.site.start() self.site.frappe_logs_till_start() self.site.sync_workers_compose() richprint.update_live() + richprint.change_head(f"Checking site") # check if site is created if self.site.is_site_created(): richprint.print(f"Creating Site: Done") + self.site.remove_secrets() self.typer_context.obj["logger"].info( f"SITE_STATUS {self.site.name}: WORKING" ) + richprint.print(f"Started site") + self.info() else: self.typer_context.obj["logger"].error(f"{self.site.name}: NOT WORKING") + richprint.stop() - richprint.error( - f"There has been some error creating/starting the site.\nPlease check the logs at {CLI_DIR/ 'logs'/'fm.log'}" + + error_message = ( + "There has been some error creating/starting the site.\n" + "Please check the logs at {}" ) + + log_path = CLI_DIR / "logs" / "fm.log" + + richprint.error(error_message.format(log_path)) + # prompt if site not working to delete the site if typer.confirm(f"Do you want to delete this site {self.site.name}?"): richprint.start("Removing Site") @@ -190,234 +168,198 @@ def create_site(self, template_inputs: dict): def remove_site(self): """ - The `remove_site` function checks if a site exists, stops it if it is running, and then removes the - site directory. + Removes the site. """ - # TODO maybe the site is running and folder has been delted and all the containers are there. We need to clean it. richprint.change_head(f"Removing Site") - # check if running -> stop it - # remove dir + self.site.remove_database_and_user() self.site.remove() def list_sites(self): """ - The `list_sites` function retrieves a list of sites, categorizes them as either running or stale, - and displays them in separate panels using the Rich library. + Lists all the sites and their status. """ - # format -> name , status [ 'stale', 'running' ] - # sites_list = self.__get_all_sites_path() - active = [] - inactive = [] - stale = [] - + richprint.change_head("Generating site list") sites_list = self.get_all_sites() - if not sites_list: - richprint.error("No sites available !") - typer.Exit(2) - else: - for name in sites_list.keys(): - temppath = self.sitesdir / name - tempSite = Site(temppath, name) - # know if all services are running - tempsite_services_status = tempSite.get_services_running_status() + if not sites_list: + richprint.exit( + "Seems like you haven't created any sites yet. To create a site, use the command: 'fm create '." + ) - inactive_status = False - for service in tempsite_services_status.keys(): - if tempsite_services_status[service] == "running": - inactive_status = True + list_table = Table(show_lines=True, show_header=True, highlight=True) + list_table.add_column("Site") + list_table.add_column("Status", vertical="middle") + list_table.add_column("Path") - if tempSite.running(): - active.append({"name": name, "path": temppath.absolute()}) - elif inactive_status: - inactive.append({"name": name, "path": temppath.absolute()}) - else: - stale.append({"name": name, "path": temppath.absolute()}) + for site_name in sites_list.keys(): + site_path = self.sitesdir / site_name + temp_site = Site(site_path, site_name) - richprint.stop() + row_data = f"[link=http://{temp_site.name}]{temp_site.name}[/link]" + path_data = f"[link=file://{temp_site.path}]{temp_site.path}[/link]" - list_table = Table(show_lines=True, show_header=True, highlight=True) - list_table.add_column("Site") - list_table.add_column("Status", vertical="middle") - list_table.add_column("Path") + status_color = "white" + status_msg = "Inactive" - for site in active: - row_data = f"[link=http://{site['name']}]{site['name']}[/link]" - path_data = f"[link=file://{site['path']}]{site['path']}[/link]" - status_data = "[green]Active[/green]" - list_table.add_row(row_data, status_data, path_data, style="green") + if temp_site.running(): + status_color = "green" + status_msg = "Active" - for site in inactive: - row_data = f"[link=http://{site['name']}]{site['name']}[/link]" - path_data = f"[link=file://{site['path']}]{site['path']}[/link]" - status_data = "[red]Inactive[/red]" - list_table.add_row(row_data, status_data, path_data,style="red") + status_data = f"[{status_color}]{status_msg}[/{status_color}]" - for site in stale: - row_data = f"[link=http://{site['name']}]{site['name']}[/link]" - path_data = f"[link=file://{site['path']}]{site['path']}[/link]" - status_data = "[grey]Stale[/grey]" - list_table.add_row(row_data, status_data, path_data) + list_table.add_row( + row_data, status_data, path_data, style=f"{status_color}" + ) + richprint.update_live(list_table, padding=(0, 0, 0, 0)) - richprint.stdout.print(list_table) - richprint.print(f"Run 'fm info ' to get detail information about a site.",emoji_code=':light_bulb:') + richprint.stop() + richprint.stdout.print(list_table) + richprint.print( + f"Run 'fm info ' to get detail information about a site.", + emoji_code=":light_bulb:", + ) def stop_site(self): """ - The function `stop_site` checks if a site exists, stops it if it does, and prints a message - indicating that the site has been stopped. + Stops the site. """ richprint.change_head(f"Stopping site") - # self.stop_sites() self.site.stop() richprint.print(f"Stopped site") def start_site(self): """ - The function `start_site` checks if a site exists, stops all sites, checks ports, pulls the site, - and starts it. + Starts the site. """ - # stop all sites - self.stop_sites() - if not self.site.running(): - self.check_ports() - # start the provided site - self.migrate_site() + #self.migrate_site() self.site.pull() + self.site.sync_site_common_site_config() self.site.start() - self.site.frappe_logs_till_start(status_msg='Starting Site') + self.site.frappe_logs_till_start(status_msg="Starting Site") self.site.sync_workers_compose() def attach_to_site(self, user: str, extensions: List[str]): """ - The `attach_to_site` function attaches to a running site and opens it in Visual Studio Code with - specified extensions. - - :param user: The `user` parameter is a string that represents the username of the user who wants to - attach to the site - :type user: str - :param extensions: The `extensions` parameter is a list of strings that represents the extensions to - be installed in Visual Studio Code - :type extensions: List[str] + Attaches to a running site's container using Visual Studio Code Remote Containers extension. + + Args: + user (str): The username to be used in the container. + extensions (List[str]): List of extensions to be installed in the container. """ - if self.site.running(): - # check if vscode is installed - vscode_path = shutil.which("code") - - if not vscode_path: - richprint.exit("vscode(excutable code) not accessible via cli.") - - container_hex = self.site.get_frappe_container_hex() - vscode_cmd = shlex.join( - [ - vscode_path, - f"--folder-uri=vscode-remote://attached-container+{container_hex}+/workspace", - ] + + if not self.site.running(): + richprint.print(f"Site: {self.site.name} is not running") + + # check if vscode is installed + vscode_path = shutil.which("code") + + if not vscode_path: + richprint.exit( + "Visual Studio Code excutable 'code' nott accessible via cli." ) - extensions.sort() - labels = { - "devcontainer.metadata": json.dumps( - [ - { - "remoteUser": user, - "customizations": {"vscode": {"extensions": extensions}}, - } - ] - ) + + container_hex = self.site.get_frappe_container_hex() + + vscode_cmd = shlex.join( + [ + vscode_path, + f"--folder-uri=vscode-remote://attached-container+{container_hex}+/workspace", + ] + ) + extensions.sort() + + vscode_config_json = [ + { + "remoteUser": user, + "customizations": {"vscode": {"extensions": extensions}}, } + ] - labels_previous = self.site.composefile.get_labels("frappe") + labels = {"devcontainer.metadata": json.dumps(vscode_config_json)} - # check if the extension are the same if they are different then only update - # check if customizations key available - try: - extensions_previous = json.loads( - labels_previous["devcontainer.metadata"] - ) - extensions_previous = extensions_previous[0]["customizations"][ - "vscode" - ]["extensions"] - except KeyError: - extensions_previous = [] - - extensions_previous.sort() - - if not extensions_previous == extensions: - richprint.print(f"Extensions are changed, Recreating containers..") - self.site.composefile.set_labels("frappe", labels) - self.site.composefile.write_to_file() - self.site.start() - richprint.print(f"Recreating Containers : Done") - # TODO check if vscode exists - richprint.change_head("Attaching to Container") - output = subprocess.run(vscode_cmd, shell=True) - if output.returncode != 0: - richprint.exit(f"Attaching to Container : Failed") - richprint.print(f"Attaching to Container : Done") - else: - richprint.print(f"Site: {self.site.name} is not running") + labels_previous = self.site.composefile.get_labels("frappe") + + # check if the extension are the same if they are different then only update + # check if customizations key available + try: + extensions_previous = json.loads(labels_previous["devcontainer.metadata"]) + extensions_previous = extensions_previous[0]["customizations"]["vscode"][ + "extensions" + ] + + except KeyError: + extensions_previous = [] + + extensions_previous.sort() + + if not extensions_previous == extensions: + richprint.print(f"Extensions are changed, Recreating containers..") + self.site.composefile.set_labels("frappe", labels) + self.site.composefile.write_to_file() + self.site.start() + richprint.print(f"Recreating Containers : Done") + + richprint.change_head("Attaching to Container") + output = subprocess.run(vscode_cmd, shell=True) + + if output.returncode != 0: + richprint.exit(f"Attaching to Container : Failed") + richprint.print(f"Attaching to Container : Done") def logs(self, follow, service: Optional[str] = None): """ - The `logs` function checks if a site exists, and if it does, it shows the logs for a specific - service. If the site is not running, it displays an error message. - - :param service: The `service` parameter is a string that represents the specific service or - component for which you want to view the logs. It could be the name of a specific container - :type service: str - :param follow: The "follow" parameter is a boolean value that determines whether to continuously - follow the logs or not. If "follow" is set to True, the logs will be continuously displayed as they - are generated. If "follow" is set to False, only the existing logs will be displayed + Display logs for the site or a specific service. + + Args: + follow (bool): Whether to continuously follow the logs or not. + service (str, optional): The name of the service to display logs for. If not provided, logs for the entire site will be displayed. """ richprint.change_head(f"Showing logs") - try: - if service: - if self.site.is_service_running(service): - self.site.logs(service, follow) - else: - richprint.exit(f"Cannot show logs. [blue]{self.site.name}[/blue]'s compose service '{service}' not running!") - else: - self.site.bench_dev_server_logs(follow) + if not service: + return self.site.bench_dev_server_logs(follow) + + if not self.site.is_service_running(service): + richprint.exit( + f"Cannot show logs. [blue]{self.site.name}[/blue]'s compose service '{service}' not running!" + ) + + self.site.logs(service, follow) + except KeyboardInterrupt: richprint.stdout.print("Detected CTRL+C. Exiting.") - def check_ports(self): - """ - The `check_ports` function checks if certain ports are already bound by another process using the - `lsof` command. + def shell(self, service: str, user: str | None): """ + Spawns a shell for the specified service and user. - check_ports_with_msg([80, 443], exclude=self.site.get_host_port_binds()) + Args: + service (str): The name of the service. + user (str | None): The name of the user. If None, defaults to "frappe". - def shell(self, service: str, user: str | None): - """ - The `shell` function checks if a site exists and is running, and then executes a shell command on - the specified container with the specified user. - - :param service: The "container" parameter is a string that specifies the name of the container. - :type service: str - :param user: The `user` parameter in the `shell` method is an optional parameter that specifies the - user for which the shell command should be executed. If no user is provided, the default user is set - to 'frappe' - :type user: str | None """ richprint.change_head(f"Spawning shell") - if service == "frappe": - if not user: - user = "frappe" - if self.site.is_service_running(service): - self.site.shell(service, user) - else: - richprint.exit(f"Cannot spawn shell. [blue]{self.site.name}[/blue]'s compose service '{service}' not running!") + if service == "frappe" and not user: + user = "frappe" + + if not self.site.is_service_running(service): + richprint.exit( + f"Cannot spawn shell. [blue]{self.site.name}[/blue]'s compose service '{service}' not running!" + ) + + self.site.shell(service, user) def info(self): """ - The `info` function retrieves information about a site, including its URL, root path, database - details, Frappe username and password, and a list of installed apps. + Retrieves and displays information about the site. + + This method retrieves various information about the site, such as site URL, site root, database details, + Frappe username and password, root database user and password, and more. It then formats and displays + this information using the richprint library. """ + richprint.change_head(f"Getting site info") site_config_file = ( self.site.path @@ -427,8 +369,10 @@ def info(self): / self.site.name / "site_config.json" ) + db_user = None db_pass = None + if site_config_file.exists(): with open(site_config_file, "r") as f: site_config = json.load(f) @@ -436,9 +380,8 @@ def info(self): db_pass = site_config["db_password"] frappe_password = self.site.composefile.get_envs("frappe")["ADMIN_PASS"] - root_db_password = self.site.composefile.get_envs("mariadb")[ - "MYSQL_ROOT_PASSWORD" - ] + services_db_info = self.services.get_database_info() + root_db_password = services_db_info['password'] site_info_table = Table(show_lines=True, show_header=False, highlight=True) @@ -463,126 +406,49 @@ def info(self): for key in data.keys(): site_info_table.add_row(key, data[key]) - # bench apps list - bench_apps_list_table = Table(show_lines=True, show_edge=False, pad_edge=False, expand=True) - bench_apps_list_table.add_column("App") - bench_apps_list_table.add_column("Version") + # get bench apps data + apps_json = self.site.get_bench_installed_apps_list() - apps_json_file = ( - self.site.path / "workspace" / "frappe-bench" / "sites" / "apps.json" - ) - if apps_json_file.exists(): + if apps_json: + bench_apps_list_table = Table( + show_lines=True, show_edge=False, pad_edge=False, expand=True + ) - with open(apps_json_file, "r") as f: - apps_json = json.load(f) - for app in apps_json.keys(): - bench_apps_list_table.add_row(app, apps_json[app]["version"]) + bench_apps_list_table.add_column("App") + bench_apps_list_table.add_column("Version") + + for app in apps_json.keys(): + bench_apps_list_table.add_row(app, apps_json[app]["version"]) site_info_table.add_row("Bench Apps", bench_apps_list_table) - # running site services status running_site_services = self.site.get_services_running_status() + running_site_workers = self.site.workers.get_services_running_status() if running_site_services: - site_services_table = Table(show_lines=False, show_edge=False, pad_edge=False, show_header=False,expand=True,box=None) - site_services_table.add_column("Service Status",ratio=1,no_wrap=True,width=None,min_width=20) - site_services_table.add_column("Service Status",ratio=1,no_wrap=True,width=None,min_width=20) - - index = 0 - while index < len(running_site_services): - first_service_table = None - second_service_table = None - - try: - first_service = list(running_site_services.keys())[index] - index += 1 - except IndexError: - pass - first_service= None - try: - second_service = list(running_site_services.keys())[index] - index += 1 - except IndexError: - second_service = None - - # Fist Coloumn - if first_service: - first_service_table = Table(show_lines=False, show_header=False, highlight=True, expand=True,box=None) - first_service_table.add_column("Service",justify="left",no_wrap=True) - first_service_table.add_column("Status",justify="right",no_wrap=True) - first_service_table.add_row(f"{first_service}", f"{':green_square:' if running_site_services[first_service] == 'running' else ':red_square:'}") - - # Fist Coloumn - if second_service: - second_service_table = Table(show_lines=False, show_header=False, highlight=True, expand=True,box=None) - second_service_table.add_column("Service",justify="left",no_wrap=True,) - second_service_table.add_column("Status",justify="right",no_wrap=True) - second_service_table.add_row(f"{second_service}", f"{':green_square:' if running_site_services[second_service] == 'running' else ':red_square:'}") - - site_services_table.add_row(first_service_table,second_service_table) - + site_services_table = generate_services_table(running_site_services) site_info_table.add_row("Site Services", site_services_table) - - if self.site.workers.exists(): - running_site_services = self.site.workers.get_services_running_status() - if running_site_services: - worker_services_table = Table(show_lines=False, show_edge=False, pad_edge=False, show_header=False,expand=True,box=None) - worker_services_table.add_column("Service Status",ratio=1,no_wrap=True,width=None,min_width=20) - worker_services_table.add_column("Service Status",ratio=1,no_wrap=True,width=None,min_width=20) - - index = 0 - while index < len(running_site_services): - first_service_table = None - second_service_table = None - - try: - first_service = list(running_site_services.keys())[index] - index += 1 - except IndexError: - pass - first_service= None - try: - second_service = list(running_site_services.keys())[index] - index += 1 - except IndexError: - second_service = None - - # Fist Coloumn - if first_service: - first_service_table = Table(show_lines=False, show_header=False, highlight=True, expand=True,box=None) - first_service_table.add_column("Service",justify="left",no_wrap=True) - first_service_table.add_column("Status",justify="right",no_wrap=True) - first_service_table.add_row(f"{first_service}", f"{':green_square:' if running_site_services[first_service] == 'running' else ':red_square:'}") - - # Fist Coloumn - if second_service: - second_service_table = Table(show_lines=False, show_header=False, highlight=True, expand=True,box=None) - second_service_table.add_column("Service",justify="left",no_wrap=True,) - second_service_table.add_column("Status",justify="right",no_wrap=True) - second_service_table.add_row(f"{second_service}", f"{':green_square:' if running_site_services[second_service] == 'running' else ':red_square:'}") - - worker_services_table.add_row(first_service_table,second_service_table) - - # hints_table = Table(show_lines=True, show_header=False, highlight=True, expand=True,show_edge=True, box=None,padding=(1,0,0,0)) - # hints_table.add_column("First",justify="center",no_wrap=True) - # hints_table.add_column("Second",justify="center",ratio=8,no_wrap=True) - # hints_table.add_column("Third",justify="center",ratio=8,no_wrap=True) - # hints_table.add_row(":light_bulb:",f":green_square: -> Active", f":red_square: -> Inactive") - - # worker_services_table_group = Group(worker_services_table,hints_table) - site_info_table.add_row("Worker Services", worker_services_table) + if running_site_workers: + site_workers_table = generate_services_table(running_site_workers) + site_info_table.add_row("Site Workers", site_workers_table) richprint.stdout.print(site_info_table) - richprint.print(f":green_square: -> Active :red_square: -> Inactive",emoji_code=':information: ') - richprint.print(f"Run 'fm list' to list all available sites.",emoji_code=':light_bulb:') + richprint.print( + f":green_square: -> Active :red_square: -> Inactive", + emoji_code=":information: ", + ) + richprint.print( + f"Run 'fm list' to list all available sites.", emoji_code=":light_bulb:" + ) def migrate_site(self): """ - The function `migrate_site` checks if the services name is the same as the template, if not, it - brings down the site, migrates the site, and starts it. + Migrates the site to a new environment. """ richprint.change_head("Migrating Environment") + if not self.site.composefile.is_services_name_same_as_template(): self.site.down(volumes=False) + self.site.migrate_site_compose() diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 85ec1ac1..79b5a9be 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -5,21 +5,29 @@ import json from typing import List, Type from pathlib import Path +from rich import inspect +from rich.table import Table from frappe_manager.docker_wrapper import DockerClient, DockerException from frappe_manager.compose_manager.ComposeFile import ComposeFile from frappe_manager.display_manager.DisplayManager import richprint +from frappe_manager.site_manager.site_exceptions import ( + SiteDatabaseAddUserException, + SiteException, +) from frappe_manager.site_manager.workers_manager.SiteWorker import SiteWorkers -from frappe_manager.site_manager.utils import log_file, get_container_name_prefix -from frappe_manager.utils import host_run_cp +from frappe_manager.utils.helpers import log_file, get_container_name_prefix +from frappe_manager.utils.docker import host_run_cp class Site: - def __init__(self, path: Path, name: str, verbose: bool = False): + def __init__(self, path: Path, name: str, verbose: bool = False, services=None): self.path = path self.name = name self.quiet = not verbose + self.services = services + # self.logger = log.get_logger() self.init() def init(self): @@ -28,11 +36,9 @@ def init(self): """ self.composefile = ComposeFile(self.path / "docker-compose.yml") self.docker = DockerClient(compose_file_path=self.composefile.compose_path) - self.workers = SiteWorkers(self.path,self.name,self.quiet) - - if not self.docker.server_running(): - richprint.exit("Docker daemon not running. Please start docker service.") + self.workers = SiteWorkers(self.path, self.name, self.quiet) + # remove this from init if self.workers.exists(): if not self.workers.running(): if self.running(): @@ -83,35 +89,36 @@ def migrate_site_compose(self): if not compose_version == fm_version: status = False if self.composefile.exists(): - # get all the payloads envs = self.composefile.get_all_envs() labels = self.composefile.get_all_labels() # introduced in v0.10.0 - if not 'ENVIRONMENT' in envs['frappe']: - envs['frappe']['ENVIRONMENT'] = 'dev' + if not "ENVIRONMENT" in envs["frappe"]: + envs["frappe"]["ENVIRONMENT"] = "dev" - envs['frappe']['CONTAINER_NAME_PREFIX'] = get_container_name_prefix(self.name) - envs['frappe']['MARIADB_ROOT_PASS'] = 'root' + envs["frappe"]["CONTAINER_NAME_PREFIX"] = get_container_name_prefix( + self.name + ) + envs["frappe"]["MARIADB_ROOT_PASS"] = "root" - envs['nginx']['VIRTUAL_HOST'] = self.name + envs["nginx"]["VIRTUAL_HOST"] = self.name import os + envs_user_info = {} - userid_groupid:dict = {"USERID": os.getuid(), "USERGROUP": os.getgid() } + userid_groupid: dict = { + "USERID": os.getuid(), + "USERGROUP": os.getgid(), + } - env_user_info_container_list = ['frappe','schedule','socketio'] + env_user_info_container_list = ["frappe", "schedule", "socketio"] for env in env_user_info_container_list: envs_user_info[env] = deepcopy(userid_groupid) # overwrite user for each invocation - users = {"nginx":{ - "uid": os.getuid(), - "gid": os.getgid() - } - } + users = {"nginx": {"uid": os.getuid(), "gid": os.getgid()}} # load template self.composefile.yml = self.composefile.load_template() @@ -124,15 +131,29 @@ def migrate_site_compose(self): # self.composefile.set_all_extrahosts(extrahosts) self.create_compose_dirs() - self.composefile.set_network_alias("nginx", "site-network", [self.name]) - self.composefile.set_container_names(get_container_name_prefix(self.name)) + self.composefile.set_network_alias( + "nginx", "site-network", [self.name] + ) + + self.composefile.set_secret_file_path( + "db_root_password", + self.services.composefile.get_secret_file_path( + "db_root_password" + ), + ) + + self.composefile.set_container_names( + get_container_name_prefix(self.name) + ) fm_version = importlib.metadata.version("frappe-manager") self.composefile.set_version(fm_version) - self.composefile.set_top_networks_name("site-network",get_container_name_prefix(self.name)) + self.composefile.set_top_networks_name( + "site-network", get_container_name_prefix(self.name) + ) self.composefile.write_to_file() # change the node socketio port - self.common_site_config_set('socketio_port','80') + # self.common_site_config_set('socketio_port','80') status = True if status: @@ -146,23 +167,28 @@ def migrate_site_compose(self): else: richprint.print("Already Latest Environment Version") - def common_site_config_set(self,key, value): - common_site_config_path = self.path / 'workspace/frappe-bench/sites/common_site_config.json' - common_site_config = {} + def common_site_config_set(self, config: dict): + common_site_config_path = ( + self.path / "workspace/frappe-bench/sites/common_site_config.json" + ) + if common_site_config_path.exists(): + common_site_config = {} - with open(common_site_config_path,'r') as f: - common_site_config = json.load(f) + with open(common_site_config_path, "r") as f: + common_site_config = json.load(f) - try: - common_site_config[key] = value - with open(common_site_config_path,'w') as f: - json.dump(common_site_config,f) - return True - except KeyError as e: - # log error that not able to change common site config + try: + for key, value in config.items(): + common_site_config[key] = value + with open(common_site_config_path, "w") as f: + json.dump(common_site_config, f) + return True + except KeyError as e: + # log error that not able to change common site config + return False + else: return False - def generate_compose(self, inputs: dict) -> None: """ The function `generate_compose` sets environment variables, extra hosts, and version information in @@ -193,15 +219,37 @@ def generate_compose(self, inputs: dict) -> None: self.composefile.set_network_alias("nginx", "site-network", [self.name]) self.composefile.set_container_names(get_container_name_prefix(self.name)) + self.composefile.set_secret_file_path( + "db_root_password", + self.services.composefile.get_secret_file_path("db_root_password"), + ) fm_version = importlib.metadata.version("frappe-manager") self.composefile.set_version(fm_version) - self.composefile.set_top_networks_name("site-network",get_container_name_prefix(self.name)) + self.composefile.set_top_networks_name( + "site-network", get_container_name_prefix(self.name) + ) self.composefile.write_to_file() def create_site_dir(self): # create site dir self.path.mkdir(parents=True, exist_ok=True) + def sync_site_common_site_config(self): + global_db_info = self.services.get_database_info() + global_db_host = global_db_info["host"] + global_db_port = global_db_info["port"] + + # set common site config + common_site_config_data = { + "socketio_port": "80", + "db_host": global_db_host, + "db_port": global_db_port, + "redis_cache": f"redis://{get_container_name_prefix(self.name)}-redis-cache:6379", + "redis_queue": f"redis://{get_container_name_prefix(self.name)}-redis-queue:6379", + "redis_socketio": f"redis://{get_container_name_prefix(self.name)}-redis-cache:6379", + } + self.common_site_config_set(common_site_config_data) + def create_compose_dirs(self) -> bool: """ The function `create_dirs` creates two directories, `workspace` and `certs`, within a specified @@ -213,23 +261,28 @@ def create_compose_dirs(self) -> bool: workspace_path = self.path / "workspace" workspace_path.mkdir(parents=True, exist_ok=True) - configs_path = self.path / 'configs' + configs_path = self.path / "configs" configs_path.mkdir(parents=True, exist_ok=True) # create nginx dirs nginx_dir = configs_path / "nginx" nginx_dir.mkdir(parents=True, exist_ok=True) - nginx_poluate_dir = ['conf'] - nginx_image = self.composefile.yml['services']['nginx']['image'] + nginx_poluate_dir = ["conf"] + nginx_image = self.composefile.yml["services"]["nginx"]["image"] for directory in nginx_poluate_dir: new_dir = nginx_dir / directory if not new_dir.exists(): new_dir_abs = str(new_dir.absolute()) - host_run_cp(nginx_image,source="/etc/nginx",destination=new_dir_abs,docker=self.docker) + host_run_cp( + nginx_image, + source="/etc/nginx", + destination=new_dir_abs, + docker=self.docker, + ) - nginx_subdirs = ['logs','cache','run'] + nginx_subdirs = ["logs", "cache", "run"] for directory in nginx_subdirs: new_dir = nginx_dir / directory new_dir.mkdir(parents=True, exist_ok=True) @@ -250,7 +303,7 @@ def start(self) -> bool: richprint.live_lines(output, padding=(0, 0, 0, 2)) richprint.print(f"{status_text}: Done") except DockerException as e: - richprint.exit(f"{status_text}: Failed",error_msg=e) + richprint.exit(f"{status_text}: Failed", error_msg=e) # start workers if exits if self.workers.exists(): @@ -297,7 +350,7 @@ def logs(self, service: str, follow: bool = False): else: richprint.stdout.print(line) - def frappe_logs_till_start(self,status_msg = None): + def frappe_logs_till_start(self, status_msg=None): """ The function `frappe_logs_till_start` prints logs until a specific line is found and then stops. """ @@ -322,10 +375,13 @@ def frappe_logs_till_start(self,status_msg = None): for source, line in output: if not source == "exit_code": line = line.decode() + + if "Updating files:".lower() in line.lower(): + continue if "[==".lower() in line.lower(): print(line) - else: - richprint.stdout.print(line) + continue + richprint.stdout.print(line) if "INFO supervisord started with pid".lower() in line.lower(): break except DockerException as e: @@ -456,7 +512,9 @@ def bench_dev_server_logs(self, follow=False): This function is used to tail logs found at /workspace/logs/bench-start.log. :param follow: Bool detemines whether to follow the log file for changes """ - bench_start_log_path = self.path / "workspace" / "frappe-bench" / 'logs' / "web.dev.log" + bench_start_log_path = ( + self.path / "workspace" / "frappe-bench" / "logs" / "web.dev.log" + ) if bench_start_log_path.exists() and bench_start_log_path.is_file(): with open(bench_start_log_path, "r") as bench_start_log: @@ -466,22 +524,24 @@ def bench_dev_server_logs(self, follow=False): else: richprint.error(f"Log file not found: {bench_start_log_path}") - def is_site_created(self, retry=30, interval=1) -> bool: + def is_site_created(self, retry=60, interval=1) -> bool: import requests from time import sleep i = 0 while i < retry: try: - response = requests.get(f"http://{self.name}") - except Exception: - return False - if response.status_code == 200: - return True - else: + host_header = {"Host": f"{self.name}"} + response = requests.get(url=f"http://127.0.0.1", headers=host_header) + if response.status_code == 200: + return True + else: + raise Exception("Site not working.") + except Exception as e: sleep(interval) i += 1 continue + return False def running(self) -> bool: @@ -573,12 +633,12 @@ def is_service_running(self, service): def sync_workers_compose(self): self.regenerate_supervisor_conf() are_workers_not_changed = self.workers.is_expected_worker_same_as_template() - if not are_workers_not_changed: - self.workers.generate_compose() - self.workers.start() - else: + if are_workers_not_changed: richprint.print("Workers configuration remains unchanged.") + return + self.workers.generate_compose() + self.workers.start() def regenerate_supervisor_conf(self): if self.name: @@ -588,40 +648,47 @@ def regenerate_supervisor_conf(self): # take backup if self.workers.supervisor_config_path.exists(): - shutil.copy(self.workers.supervisor_config_path, self.workers.supervisor_config_path.parent / "supervisor.conf.bak") + shutil.copy( + self.workers.supervisor_config_path, + self.workers.supervisor_config_path.parent / "supervisor.conf.bak", + ) for file_path in self.workers.config_dir.iterdir(): file_path_abs = str(file_path.absolute()) - if file_path.is_file(): - if file_path_abs.endswith('.fm.supervisor.conf'): - from_path = file_path - to_path = file_path.parent / f"{file_path.name}.bak" - shutil.copy(from_path, to_path) + if not file_path.is_file(): + continue + + if file_path_abs.endswith(".fm.supervisor.conf"): + from_path = file_path + to_path = file_path.parent / f"{file_path.name}.bak" + shutil.copy(from_path, to_path) + backup_list.append((from_path, to_path)) - backup_list.append((from_path, to_path)) backup = True # generate the supervisor.conf try: - bench_setup_supervisor_command = 'bench setup supervisor --skip-redis --skip-supervisord --yes --user frappe' + bench_setup_supervisor_command = "bench setup supervisor --skip-redis --skip-supervisord --yes --user frappe" output = self.docker.compose.exec( - service='frappe', + service="frappe", command=bench_setup_supervisor_command, stream=True, - user='frappe', - workdir='/workspace/frappe-bench' + user="frappe", + workdir="/workspace/frappe-bench", ) richprint.live_lines(output, padding=(0, 0, 0, 2)) - generate_split_config_command = '/scripts/divide-supervisor-conf.py config/supervisor.conf' + generate_split_config_command = ( + "/scripts/divide-supervisor-conf.py config/supervisor.conf" + ) output = self.docker.compose.exec( - service='frappe', - command=generate_split_config_command , - stream=True, - user='frappe', - workdir='/workspace/frappe-bench' + service="frappe", + command=generate_split_config_command, + stream=True, + user="frappe", + workdir="/workspace/frappe-bench", ) richprint.live_lines(output, padding=(0, 0, 0, 2)) @@ -629,10 +696,160 @@ def regenerate_supervisor_conf(self): return True except DockerException as e: richprint.error(f"Failure in generating, supervisor.conf file.{e}") + if backup: richprint.print("Rolling back to previous workers configuration.") - shutil.copy(self.workers.supervisor_config_path.parent / "supervisor.conf.bak", self.workers.supervisor_config_path) + shutil.copy( + self.workers.supervisor_config_path.parent + / "supervisor.conf.bak", + self.workers.supervisor_config_path, + ) - for from_path ,to_path in backup_list: + for from_path, to_path in backup_list: shutil.copy(to_path, from_path) + return False + + def get_bench_installed_apps_list(self): + apps_json_file = ( + self.path / "workspace" / "frappe-bench" / "sites" / "apps.json" + ) + + apps_data: dict = {} + + if not apps_json_file.exists(): + return {} + + with open(apps_json_file, "r") as f: + apps_data = json.load(f) + + return apps_data + + def get_site_db_info(self): + db_info = {} + + site_config_file = ( + self.path + / "workspace" + / "frappe-bench" + / "sites" + / self.name + / "site_config.json" + ) + + if site_config_file.exists(): + with open(site_config_file, "r") as f: + site_config = json.load(f) + db_info["name"] = site_config["db_name"] + db_info["user"] = site_config["db_name"] + db_info["password"] = site_config["db_password"] + else: + db_info["name"] = str(self.name).replace(".", "-") + db_info["user"] = str(self.name).replace(".", "-") + db_info["password"] = None + + return db_info + + def add_user(self, service, db_user, db_password, force=False, timeout=5): + db_host = "127.0.0.1" + + site_db_info = self.get_site_db_info() + site_db_name = site_db_info["name"] + site_db_user = site_db_info["user"] + site_db_pass = site_db_info["password"] + + remove_db_user = f"/usr/bin/mariadb -P3306 -h{db_host} -u{db_user} -p'{db_password}' -e 'DROP USER `{site_db_user}`@`%`;'" + add_db_user = f"/usr/bin/mariadb -h{db_host} -P3306 -u{db_user} -p'{db_password}' -e 'CREATE USER `{site_db_user}`@`%` IDENTIFIED BY \"{site_db_pass}\";'" + grant_user = f"/usr/bin/mariadb -h{db_host} -P3306 -u{db_user} -p'{db_password}' -e 'GRANT ALL PRIVILEGES ON `{site_db_name}`.* TO `{site_db_user}`@`%`;'" + SHOW_db_user = f"/usr/bin/mariadb -P3306-h{db_host} -u{db_user} -p'{db_password}' -e 'SELECT User, Host FROM mysql.user;'" + # + import time + + check_connection_command = f"/usr/bin/mariadb -h{db_host} -u{db_user} -p'{db_password}' -e 'SHOW DATABASES;'" + + i = 0 + connected = False + + error = None + while i < timeout: + try: + time.sleep(5) + output = self.docker.compose.exec( + service, + command=check_connection_command, + stream=self.quiet, + stream_only_exit_code=True, + ) + if output == 0: + connected = True + except DockerException as e: + error = e + pass + + i += 1 + + if not connected: + raise SiteDatabaseAddUserException( + self.name, f"Not able to start db: {error}" + ) + + removed = True + try: + output = self.docker.compose.exec( + service, + command=remove_db_user, + stream=self.quiet, + stream_only_exit_code=True, + ) + except DockerException as e: + removed = False + if "error 1396" in str(e.stderr).lower(): + removed = True + + if removed: + try: + output = self.docker.compose.exec( + service, + command=add_db_user, + stream=self.quiet, + stream_only_exit_code=True, + ) + output = self.docker.compose.exec( + service, + command=grant_user, + stream=self.quiet, + stream_only_exit_code=True, + ) + richprint.print(f"Recreated user {site_db_user}") + except DockerException as e: + raise SiteDatabaseAddUserException( + self.name, f"Database user creation failed: {e}" + ) + + def remove_secrets(self): + richprint.print(f"Removing Secrets", emoji_code=":construction:") + richprint.change_head(f"Removing Secrets") + + running = False + if self.running(): + running = True + self.stop() + + self.composefile.remove_secrets_from_container("frappe") + self.composefile.remove_root_secrets_compose() + self.composefile.write_to_file() + + if running: + self.start() + richprint.print(f"Removing Secrets: Done") + + def remove_database_and_user(self): + """ + This function is used to remove db and user of the site at self.name and path at self.path. + """ + site_db_info = self.get_site_db_info() + if "name" in site_db_info: + db_name = site_db_info["name"] + db_user = site_db_info["user"] + self.services.remove_db_user(db_name) + self.services.remove_db(db_user) diff --git a/frappe_manager/site_manager/site_exceptions.py b/frappe_manager/site_manager/site_exceptions.py new file mode 100644 index 00000000..dd468bad --- /dev/null +++ b/frappe_manager/site_manager/site_exceptions.py @@ -0,0 +1,46 @@ +from typing import List, Optional + +class SiteException(Exception): + def __init__( + self, + site, + error_msg: str, + ): + error_msg = f"{site.name}: {error_msg}" + super().__init__(error_msg) + +class SiteWorkerNotStart(Exception): + def __init__( + self, + error_msg: str, + ): + error_msg = f"{error_msg}" + super().__init__(error_msg) + +class SiteDatabaseAddUserException(Exception): + def __init__( + self, + site_name, + error_msg: str, + ): + error_msg = f"{site_name}: {error_msg}" + super().__init__(error_msg) + + +class SiteDatabaseStartTimeout(Exception): + def __init__( + self, + site_name, + error_msg: str, + ): + error_msg = f"{site_name}: {error_msg}" + super().__init__(error_msg) + +class SiteDatabaseExport(Exception): + def __init__( + self, + site_name, + error_msg: str, + ): + error_msg = f"{site_name}: {error_msg}" + super().__init__(error_msg) diff --git a/frappe_manager/site_manager/workers_manager/SiteWorker.py b/frappe_manager/site_manager/workers_manager/SiteWorker.py index 21327f91..741d7463 100644 --- a/frappe_manager/site_manager/workers_manager/SiteWorker.py +++ b/frappe_manager/site_manager/workers_manager/SiteWorker.py @@ -1,10 +1,12 @@ import importlib import json from copy import deepcopy + +from frappe_manager.site_manager.site_exceptions import SiteWorkerNotStart +from rich import inspect from frappe_manager.compose_manager.ComposeFile import ComposeFile -#from frappe_manager.console_manager.Richprint import richprint from frappe_manager.display_manager.DisplayManager import richprint -from frappe_manager.site_manager.utils import get_container_name_prefix, log_file +from frappe_manager.utils.helpers import get_container_name_prefix from frappe_manager.docker_wrapper import DockerClient, DockerException class SiteWorkers: @@ -20,15 +22,13 @@ def init(self): self.composefile = ComposeFile( self.compose_path, template_name='docker-compose.workers.tmpl') self.docker = DockerClient(compose_file_path=self.composefile.compose_path) - if not self.docker.server_running(): - richprint.exit("Docker daemon not running. Please start docker service.") - def exists(self): return self.compose_path.exists() def get_expected_workers(self)-> list[str]: richprint.change_head("Getting Workers info") + workers_supervisor_conf_paths = [] for file_path in self.config_dir.iterdir(): @@ -44,6 +44,7 @@ def get_expected_workers(self)-> list[str]: worker_name = worker_name.replace("frappe-bench-frappe-","") worker_name = worker_name.replace(".workers.fm.supervisor.conf","") workers_expected_service_names.append(worker_name) + workers_expected_service_names.sort() richprint.print("Getting Workers info: Done") @@ -91,6 +92,7 @@ def generate_compose(self): self.composefile.yml['services'][worker] = worker_config self.composefile.set_container_names(get_container_name_prefix(self.site_name)) + fm_version = importlib.metadata.version("frappe-manager") self.composefile.set_version(fm_version) @@ -110,6 +112,7 @@ def start(self): richprint.print(f"{status_text}: Done") except DockerException as e: richprint.error (f"{status_text}: Failed Error: {e}") + raise e def stop(self) -> bool: """ @@ -124,7 +127,7 @@ def stop(self) -> bool: richprint.live_lines(output, padding=(0, 0, 0, 2)) richprint.print(f"{status_text}: Done") except DockerException as e: - richprint.exit(f"{status_text}: Failed") + richprint.warning(f"{status_text}: Failed") def get_services_running_status(self) -> dict: services = self.composefile.get_services_list() diff --git a/frappe_manager/site_manager/workers_manager/siteworker_exceptions.py b/frappe_manager/site_manager/workers_manager/siteworker_exceptions.py new file mode 100644 index 00000000..e69de29b From 416aff603a63ca7b71691721e60d1840d40cceac Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 31 Jan 2024 14:14:31 +0530 Subject: [PATCH 081/100] optimize frappe docker image --- Docker/frappe/Dockerfile | 12 ++++-------- Docker/frappe/entrypoint.sh | 7 +++++-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Docker/frappe/Dockerfile b/Docker/frappe/Dockerfile index 2953e96c..fa1f9354 100644 --- a/Docker/frappe/Dockerfile +++ b/Docker/frappe/Dockerfile @@ -69,9 +69,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install --no-instal jq \ gosu \ fonts-powerline \ - zsh - - #&& rm -rf /var/lib/apt/lists/* + zsh \ + && rm -rf /var/lib/apt/lists/* RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \ && dpkg-reconfigure --frontend=noninteractive locales @@ -112,11 +111,7 @@ RUN git clone --depth 1 https://github.com/pyenv/pyenv.git .pyenv \ && pyenv global $PYTHON_VERSION \ && echo 'export PYENV_ROOT="/opt/.pyenv"' >> "$USERZSHRC" \ && echo 'command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"' >> "$USERZSHRC" \ - && echo 'eval "$(pyenv init -)"' >>"$USERZSHRC" - - #&& echo 'eval "$(pyenv init -)"' >>"$USERPROFILE" - #&& sed -Ei -e '/^([^#]|$)/ {a export PYENV_ROOT="/opt/.pyenv" a export PATH="$PYENV_ROOT/bin:$PATH" a ' -e ':a' -e '$!{n;ba};}' "$USERPROFILE" \ - #&& echo 'eval "$(pyenv init --path)"' >>"$USERPROFILE" \ + && echo 'eval "$(pyenv init --path)"' >>"$USERZSHRC" RUN pip install frappe-bench @@ -138,6 +133,7 @@ RUN mkdir -p /opt/.nvm \ && echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm' >> "$USERZSHRC" \ && echo '[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion' >> "$USERZSHRC" + RUN mkdir -p /workspace WORKDIR /workspace diff --git a/Docker/frappe/entrypoint.sh b/Docker/frappe/entrypoint.sh index 430aca0f..14760e62 100755 --- a/Docker/frappe/entrypoint.sh +++ b/Docker/frappe/entrypoint.sh @@ -32,11 +32,14 @@ if [[ ! -f "/workspace/.profile" ]]; then cat /opt/user/.profile > /workspace/.profile fi + if [[ ! -d '/workspace/frappe-bench' ]]; then - find /workspace -type d -print0 | xargs -0 -n 200 -P "$(nproc)" chown "$USERID":"$USERGROUP" - find /workspace -type f -print0 | xargs -0 -n 200 -P "$(nproc)" chown "$USERID":"$USERGROUP" + chown -R "$USERID":"$USERGROUP" /workspace + # find /workspace -type d -print0 | xargs -0 -n 200 -P "$(nproc)" chown "$USERID":"$USERGROUP" + # find /workspace -type f -print0 | xargs -0 -n 200 -P "$(nproc)" chown "$USERID":"$USERGROUP" fi + if [ "$#" -gt 0 ]; then gosu "$USERID":"$USERGROUP" "/scripts/$@" else From ed5a21e36d9fb28561ef6495fd631234e5a6ba5b Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 31 Jan 2024 14:15:14 +0530 Subject: [PATCH 082/100] fix docker compose file formatting --- frappe_manager/compose_manager/ComposeFile.py | 361 ++++++++++++------ 1 file changed, 246 insertions(+), 115 deletions(-) diff --git a/frappe_manager/compose_manager/ComposeFile.py b/frappe_manager/compose_manager/ComposeFile.py index d1db693d..8cc8d071 100644 --- a/frappe_manager/compose_manager/ComposeFile.py +++ b/frappe_manager/compose_manager/ComposeFile.py @@ -1,15 +1,19 @@ from pathlib import Path +from rich import inspect from ruamel.yaml import YAML - +import platform +from ruamel.yaml.comments import CommentedMap as OrderedDict, CommentedSeq as OrderedList from frappe_manager.display_manager.DisplayManager import richprint -from frappe_manager.compose_manager.utils import represent_null_empty +from frappe_manager.utils.site import parse_docker_volume +from frappe_manager.utils.helpers import represent_null_empty +import importlib.resources as pkg_resources -yaml = YAML(typ='safe',pure=True) +yaml = YAML(typ="rt") yaml.representer.ignore_aliases = lambda *args: True # Set the default flow style to None to preserve the null representation yaml.default_flow_style = False - +yaml.default_style = None class ComposeFile: @@ -22,54 +26,57 @@ def __init__(self, loadfile: Path, template_name: str = "docker-compose.tmpl"): def init(self): """ - The function initializes a YAML object by loading data from a file if it exists, otherwise it - creates a new YAML object using a template. + Initializes the ComposeFile object. """ - # if the load file not found then the site not exits if self.exists(): with open(self.compose_path, "r") as f: self.yml = yaml.load(f) - else: self.yml = self.load_template() self.is_template_loaded = True def exists(self): """ - The function checks if a file or directory exists at the specified path. - :return: a boolean value, if compose file exits `True` else `False`. + Check if the compose file exists. + + Returns: + bool: True if the compose file exists, False otherwise. """ return self.compose_path.exists() def get_compose_path(self): """ - Getter for getting compose file path. - :return: The returns compose file path. + Returns the path of the compose file. """ return self.compose_path def get_template( - self, file_name: str, template_directory="templates" - ) -> None | str: + self, file_name: str + ): """ - The function `get_template` retrieves a template file and returns its contents as a string, or - raises an error if the template file is not found. + Get the file path of a template. + + Args: + file_name (str): The name of the template file. + template_directory (str, optional): The directory where the templates are located. Defaults to "templates". - :param file_name: The `file_name` parameter is a string that represents the name of the template - file. It is used to construct the file path by appending it to the "templates/" directory - :type file_name: str - :return: either None or a string. + Returns: + Optional[str]: The file path of the template, or None if the template is not found. """ - file_name = f"{template_directory}/{file_name}" + try: - import pkg_resources - file_path = pkg_resources.resource_filename(__name__, file_name) - return file_path - #data = pkgutil.get_data(__name__, file_name) - except Exception as e: - richprint.exit(f"{file_name} template not found! Error:{e}") + template_path = f"templates/{file_name}" + return Path(str(pkg_resources.files('frappe_manager').joinpath(template_path))) + except FileNotFoundError as e: + richprint.exit(f"{file_name} template not found.",error_msg=e) def load_template(self): + """ + Load the template file and return its contents as a YAML object. + + Returns: + dict: The contents of the template file as a YAML object. + """ template_path = self.get_template(self.template_name) if template_path: with open(template_path, "r") as f: @@ -78,45 +85,45 @@ def load_template(self): def set_container_names(self, prefix): """ - The function sets the container names for each service in a compose file based on the site name. - """ + Sets the container names for each service in the Compose file. + Args: + prefix (str): The prefix to be added to the container names. + """ for service in self.yml["services"].keys(): self.yml["services"][service]["container_name"] = prefix + f"-{service}" def get_container_names(self) -> dict: """ - The function `get_container_names` returns a dictionary of container names for each service in a - compose file. - :return: a dictionary containing the names of the containers specified in the compose file. + Returns a dictionary of container names for each service defined in the Compose file. + + Returns: + dict: A dictionary where the keys are service names and the values are container names. """ container_names: dict = {} - - # site_name = self.compose_path.parent.name - if self.exists(): services = self.get_services_list() for service in services: container_names[service] = self.yml["services"][service][ "container_name" ] - # container_names[service] = site_name.replace('.','') + f'-{service}' - return container_names def get_services_list(self) -> list: """ - Getting for getting all docker compose services name as a list. - :return: list of docker composer servicers. + Returns a list of services defined in the Compose file. + + Returns: + list: A list of service names. """ return list(self.yml["services"].keys()) def is_services_name_same_as_template(self): """ - The function checks if the service names in the current YAML file are the same as the service names - in the template YAML file. - :return: a boolean value indicating whether the list of service names in the current YAML file is - the same as the list of service names in the template YAML file. + Checks if the service names in the current Compose file are the same as the template file. + + Returns: + bool: True if the service names are the same, False otherwise. """ template_yml = self.load_template() template_service_name_list = list(template_yml["services"].keys()) @@ -126,12 +133,29 @@ def is_services_name_same_as_template(self): return current_service_name_list == template_service_name_list def set_user(self, service, uid, gid): + """ + Set the user for a specific service in the Compose file. + + Args: + service (str): The name of the service. + uid (str): The user ID. + gid (str): The group ID. + """ try: self.yml["services"][service]["user"] = f"{uid}:{gid}" except KeyError: richprint.exit("Issue in docker template. Not able to set user.") def get_user(self, service): + """ + Get the user associated with the specified service. + + Args: + service (str): The name of the service. + + Returns: + str or None: The user associated with the service, or None if not found. + """ try: user = self.yml[service]["user"] uid = user.split(":")[0] @@ -143,16 +167,29 @@ def get_user(self, service): def set_top_networks_name(self, networks_name, prefix): """ - The function sets the network names for each service in a compose file based on the site name. - """ + Sets the name of the top-level network in the Compose file. + Args: + networks_name (str): The name of the network. + prefix (str): The prefix to be added to the network name. + """ if not self.yml["networks"][networks_name]: - self.yml["networks"][networks_name] = { "name" : prefix + f"-network" } + self.yml["networks"][networks_name] = {"name": prefix + f"-network"} else: self.yml["networks"][networks_name]["name"] = prefix + f"-network" - def set_network_alias(self, service_name, network_name, alias: list = []): + """ + Sets the network alias for a given service in the Compose file. + + Args: + service_name (str): The name of the service. + network_name (str): The name of the network. + alias (list, optional): List of network aliases to be set. Defaults to []. + + Returns: + bool: True if the network alias is set successfully, False otherwise. + """ if alias: try: all_networks = self.yml["services"][service_name]["networks"] @@ -167,6 +204,16 @@ def set_network_alias(self, service_name, network_name, alias: list = []): return False def get_network_alias(self, service_name, network_name) -> list | None: + """ + Retrieves the network aliases for a given service and network name. + + Args: + service_name (str): The name of the service. + network_name (str): The name of the network. + + Returns: + list | None: A list of network aliases if found, otherwise None. + """ try: all_networks = self.yml["services"][service_name]["networks"] if network_name in all_networks: @@ -176,34 +223,39 @@ def get_network_alias(self, service_name, network_name) -> list | None: return aliases except KeyError as e: return None - else: - return None def get_version(self): """ - The function `get_version` returns the value of the `x-version` key from composer file, or - `None` if the key is not present. - :return: the value of the 'x-version', if not found then - it returns None. + Get the version of the compose file. + + Returns: + int: The version of the compose file, or 0 if the version is not specified. """ try: compose_version = self.yml["x-version"] + return compose_version except KeyError: - return 0 - return compose_version + return '0.0.0' def set_version(self, version): """ - The function sets the value of the 'x-version' key in a YAML dictionary to the specified version. + Sets the version of the Compose file. - :param version: current fm version to set it to "x-version" key in the compose file. + Args: + version (str): The version to set. + + Returns: + None """ self.yml["x-version"] = version def get_all_users(self): """ - The function `get_all_users` returns a dictionary of users for each service in a compose file. - :return: a dictionary containing the users of the containers specified in the compose file. + Retrieves a dictionary of all users defined in the Compose file. + + Returns: + dict: A dictionary where the keys are service names and the values are dictionaries + containing the user's UID and GID. """ users: dict = {} @@ -219,17 +271,20 @@ def get_all_users(self): def set_all_users(self, users: dict): """ - The function `set_all_users` sets the users for each service in a compose file. + Sets the UID and GID for all services in the ComposeFile. - :param users: The `users` parameter is a dictionary that contains users for each service in a - compose file. + Args: + users (dict): A dictionary containing the service names as keys and the UID and GID as values. """ for service in users.keys(): self.set_user(service, users[service]["uid"], users[service]["gid"]) def get_all_envs(self): """ - This functtion returns all the container environment variables + Retrieves all the environment variables for each service in the Compose file. + + Returns: + dict: A dictionary containing the service names as keys and their respective environment variables as values. """ envs = {} for service in self.yml["services"].keys(): @@ -242,14 +297,21 @@ def get_all_envs(self): def set_all_envs(self, environments: dict): """ - This functtion returns all the container environment variables + Sets environment variables for all containers in the Compose file. + + Args: + environments (dict): A dictionary containing container names as keys and environment variables as values. + """ for container_name in environments.keys(): self.set_envs(container_name, environments[container_name], append=True) def get_all_labels(self): """ - This functtion returns all the container labels variables + Retrieves all the labels for each service in the Compose file. + + Returns: + dict: A dictionary containing the service names as keys and their respective labels as values. """ labels = {} for service in self.yml["services"].keys(): @@ -262,14 +324,20 @@ def get_all_labels(self): def set_all_labels(self, labels: dict): """ - This functtion returns all the container environment variables + Sets labels for all containers in the ComposeFile. + + Args: + labels (dict): A dictionary containing container names as keys and labels as values. """ for container_name in labels.keys(): self.set_labels(container_name, labels[container_name]) def get_all_extrahosts(self): """ - This functtion returns all the container labels variables + Returns a dictionary of all the extra hosts for each service in the Compose file. + + Returns: + dict: A dictionary where the keys are the service names and the values are the extra hosts. """ extrahosts = {} for service in self.yml["services"].keys(): @@ -282,31 +350,32 @@ def get_all_extrahosts(self): def set_all_extrahosts(self, extrahosts: dict, skip_not_found: bool = False): """ - This functtion returns all the container environment variables + Sets the extrahosts for all containers in the ComposeFile. + + Args: + extrahosts (dict): A dictionary containing container names as keys and their corresponding extrahosts as values. + skip_not_found (bool, optional): If True, skips setting extrahosts for containers that are not found. Defaults to False. """ for container_name in extrahosts.keys(): self.set_extrahosts(container_name, extrahosts[container_name]) def set_envs(self, container: str, env: dict, append=False): """ - The function `set_envs` sets environment variables for a given container in a compose file. + Sets the environment variables for a specific container in the Compose file. - :param container: A string representing the name of the container - :type container: str - :param env: The `env` parameter is a dictionary that contains environment variables. Each key-value - pair in the dictionary represents an environment variable, where the key is the variable name and - the value is the variable value - :type env: dict + Args: + container (str): The name of the container. + env (dict): A dictionary containing the environment variables to be set. + append (bool, optional): If True, appends the new environment variables to the existing ones. """ - # change dict to list + new_env = OrderedDict(env) + if append and type(env) == dict: prev_env = self.get_envs(container) if prev_env: - new_env = prev_env | env - else: - new_env = env - else: - new_env = env + if not type(prev_env) == OrderedList: + env = OrderedDict(env) + new_env = prev_env | env try: self.yml["services"][container]["environment"] = new_env @@ -315,12 +384,14 @@ def set_envs(self, container: str, env: dict, append=False): def get_envs(self, container: str) -> dict: """ - The function `get_envs` retrieves the environment variables from a specified container in a compose - file. + Get the environment variables for a specific container. - :param container: A string representing the name of the container - :type container: str - :return: a dictionary containing the environment variables of the specified container. + Args: + container (str): The name of the container. + + Returns: + dict: A dictionary containing the environment variables for the container. + Returns None if the container or environment variables are not found. """ try: env = self.yml["services"][container]["environment"] @@ -330,16 +401,12 @@ def get_envs(self, container: str) -> dict: def set_labels(self, container: str, labels: dict): """ - The function `set_labels` sets the labels for a specified container in a compose file. + Sets the labels for a specific container in the Compose file. + + Args: + container (str): The name of the container. + labels (dict): A dictionary containing the labels to be set. - :param container: The `container` parameter is a string that represents the name of a container in a - YAML file - :type container: str - :param labels: The `labels` parameter is a dictionary that contains key-value pairs. Each key - represents a label name, and the corresponding value represents the label value. These labels can be - used to provide metadata or configuration information to the container specified by the `container` - parameter - :type labels: dict """ try: self.yml["services"][container]["labels"] = labels @@ -348,12 +415,13 @@ def set_labels(self, container: str, labels: dict): def get_labels(self, container: str) -> dict: """ - The function `get_labels` takes a container name as input and returns the labels associated with - that container from a compose file. + Get the labels of a specific container. + + Args: + container (str): The name of the container. - :param container: The `container` parameter is a string that represents the name of a container - :type container: str - :return: a dictionary of labels. + Returns: + dict: The labels of the container, or None if the container or labels are not found. """ try: labels = self.yml["services"][container]["labels"] @@ -363,13 +431,12 @@ def get_labels(self, container: str) -> dict: def set_extrahosts(self, container: str, extrahosts: list): """ - The function `set_extrahosts` sets the `extra_hosts` property of a container in a compose file. + Set the extra hosts for a specific container in the Compose file. + + Args: + container (str): The name of the container. + extrahosts (list): A list of extra hosts to be added. - :param container: The container parameter is a string that represents the name of the container - :type container: str - :param extrahosts: A list of additional hostnames to be added to the container's /etc/hosts file. - Each item in the list should be in the format "hostname:IP_address" - :type extrahosts: list """ try: self.yml["services"][container]["extra_hosts"] = extrahosts @@ -378,12 +445,13 @@ def set_extrahosts(self, container: str, extrahosts: list): def get_extrahosts(self, container: str) -> list: """ - The function `get_extrahosts` returns a list of extra hosts for a given container. + Get the extra hosts for a specific container. + + Args: + container (str): The name of the container. - :param container: The `container` parameter is a string that represents the name of a container - :type container: str - :return: a list of extra hosts for a given container. If the container is not found or if there are - no extra hosts defined for the container, an empty list is returned. + Returns: + list: A list of extra hosts for the container, or None if not found. """ try: extra_hosts = self.yml["services"][container]["extra_hosts"] @@ -391,14 +459,77 @@ def get_extrahosts(self, container: str) -> list: except KeyError: return None - def write_to_file(self): """ - The function writes the contents of a YAML object to a file. + Writes the Docker Compose file to the specified path. """ try: # saving the docker compose to the directory with open(self.compose_path, "w") as f: - yaml.dump(self.yml, f,transform=represent_null_empty) + yaml.dump(self.yml, f, transform=represent_null_empty) except Exception as e: - richprint.exit(f"Error in writing compose file.",error_msg=e) + richprint.exit(f"Error in writing compose file.", error_msg=e) + + def get_all_volumes(self): + """ + Get all the root volumes. + """ + + volumes = self.yml['volumes'] + + return volumes + + def get_all_services_volumes(self): + """ + Get all the volume mounts. + """ + volumes_set = set() + + services = self.get_services_list() + + for service in services: + try: + volumes_list = self.yml["services"][service]['volumes'] + for volume in volumes_list: + volumes_set.add(volume) + except KeyError as e: + continue + + volumes_list = [] + + for volume in volumes_set: + volumes_list.append((parse_docker_volume(volume))) + + return volumes_list + + def set_secret_file_path(self,secret_name,file_path): + try: + self.yml['secrets'][secret_name]['file'] = file_path + except KeyError: + richprint.warning("Not able to set secrets in compose.") + + def get_secret_file_path(self,secret_name): + try: + file_path = self.yml['secrets'][secret_name]['file'] + return file_path + except KeyError: + richprint.warning("Not able to set secrets in compose.") + + def remove_secrets_from_container(self,container): + try: + del self.yml['services'][container]['secrets'] + except KeyError: + richprint.warning(f"Not able to remove secrets from {container}.") + + def remove_root_secrets_compose(self): + try: + del self.yml['secrets'] + except KeyError: + richprint.warning(f"root level secrets not present.") + + + def remove_container_user(self, container): + try: + del self.yml['services'][container]['user'] + except KeyError: + richprint.warning(f"user not present.") From 7db9ca82be208b002653411bf49f8143125e6cb7 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 31 Jan 2024 14:16:34 +0530 Subject: [PATCH 083/100] refractor main.py to split commands --- frappe_manager/commands.py | 288 +++++++++++++++++++++++++++++ frappe_manager/main.py | 362 +------------------------------------ 2 files changed, 298 insertions(+), 352 deletions(-) create mode 100644 frappe_manager/commands.py diff --git a/frappe_manager/commands.py b/frappe_manager/commands.py new file mode 100644 index 00000000..2216d300 --- /dev/null +++ b/frappe_manager/commands.py @@ -0,0 +1,288 @@ +from ruamel.yaml import serialize +import typer +import os +import requests +import sys +import shutil +from typing import Annotated, List, Optional, Set +from frappe_manager.site_manager.SiteManager import SiteManager +from frappe_manager.display_manager.DisplayManager import richprint +from frappe_manager import CLI_DIR, default_extension, SiteServicesEnum, services_manager +from frappe_manager.logger import log +from frappe_manager.docker_wrapper import DockerClient +from frappe_manager.services_manager.services import ServicesManager +from frappe_manager.migration_manager.migration_executor import MigrationExecutor +from frappe_manager.utils.callbacks import apps_list_validation_callback, frappe_branch_validation_callback, version_callback +from frappe_manager.utils.helpers import get_container_name_prefix, is_cli_help_called +from frappe_manager.services_manager.commands import services_app + +app = typer.Typer(no_args_is_help=True,rich_markup_mode='rich') +app.add_typer(services_app, name="services", help="Handle global services.") + +# this will be initiated later in the app_callback +sites: Optional[SiteManager] = None + +@app.callback() +def app_callback( + ctx: typer.Context, + verbose: Annotated[Optional[bool], typer.Option('--verbose','-v',help="Enable verbose output.")] = None, + version: Annotated[ + Optional[bool], typer.Option("--version",help="Show Version.",callback=version_callback) + ] = None, +): + """ + Frappe-Manager for creating frappe development envrionments. + """ + + ctx.obj = {} + + help_called = is_cli_help_called(ctx) + ctx.obj["is_help_called"] = help_called + + if not help_called: + + sitesdir = CLI_DIR / 'sites' + + richprint.start(f"Working") + + if not CLI_DIR.exists(): + # creating the sites dir + # TODO check if it's writeable and readable -> by writing a file to it and catching exception + CLI_DIR.mkdir(parents=True, exist_ok=True) + sitesdir.mkdir(parents=True, exist_ok=True) + richprint.print(f"fm directory doesn't exists! Created at -> {str(CLI_DIR)}") + else: + if not CLI_DIR.is_dir(): + richprint.exit("Sites directory is not a directory! Aborting!") + + # logging + global logger + logger = log.get_logger() + logger.info('') + logger.info(f"{':'*20}FM Invoked{':'*20}") + logger.info('') + + # logging command provided by user + logger.info(f"RUNNING COMMAND: {' '.join(sys.argv[1:])}") + logger.info('-'*20) + + + # check docker daemon service + if not DockerClient().server_running(): + richprint.exit("Docker daemon not running. Please start docker service.") + + migrations = MigrationExecutor() + migration_status = migrations.execute() + if not migration_status: + richprint.exit(f"Rollbacked to previous version of fm {migrations.prev_version}.") + + global services_manager + services_manager = ServicesManager(verbose=verbose) + services_manager.init() + services_manager.entrypoint_checks() + + if not services_manager.running(): + services_manager.start() + + global sites + sites = SiteManager(sitesdir, services=services_manager) + + sites.set_typer_context(ctx) + + if verbose: + sites.set_verbose() + + ctx.obj["sites"] = sites + ctx.obj["logger"] = logger + ctx.obj["services"] = services_manager + + + +@app.command(no_args_is_help=True) +def create( + sitename: Annotated[str, typer.Argument(help="Name of the site")], + apps: Annotated[ + Optional[List[str]], + typer.Option( + "--apps", "-a", help="FrappeVerse apps to install. App should be specified in format : or .", callback=apps_list_validation_callback, + show_default=False + ), + ] = None, + developer_mode: Annotated[bool, typer.Option(help="Enable developer mode")] = True, + frappe_branch: Annotated[ + str, typer.Option(help="Specify the branch name for frappe app",callback=frappe_branch_validation_callback) + ] = "version-15", + admin_pass: Annotated[ + str, + typer.Option( + help="Default Password for the standard 'Administrator' User. This will be used as the password for the Administrator User for all new sites" + ), + ] = "admin", + enable_ssl: Annotated[bool, typer.Option(help="Enable https")] = False, +): + # TODO Create markdown table for the below help + """ + Create a new site. + + Frappe\[version-14] will be installed by default. + + [bold white on black]Examples:[/bold white on black] + + [bold]# Install frappe\[version-14][/bold] + $ [blue]fm create example[/blue] + + [bold]# Install frappe\[version-15-beta][/bold] + $ [blue]fm create example --frappe-branch version-15-beta[/blue] + + [bold]# Install frappe\[version-14], erpnext\[version-14] and hrms\[version-14][/bold] + $ [blue]fm create example --apps erpnext:version-14 --apps hrms:version-14[/blue] + + [bold]# Install frappe\[version-15-beta], erpnext\[version-15-beta] and hrms\[version-15-beta][/bold] + $ [blue]fm create example --frappe-branch version-15-beta --apps erpnext:version-15-beta --apps hrms:version-15-beta[/blue] + """ + + sites.init(sitename) + + uid: int = os.getuid() + gid: int = os.getgid() + + environment = { + "frappe": { + "USERID": uid, + "USERGROUP": gid, + "APPS_LIST": ",".join(apps) if apps else None, + "FRAPPE_BRANCH": frappe_branch, + "DEVELOPER_MODE": developer_mode, + "ADMIN_PASS": admin_pass, + "DB_NAME": sites.site.name.replace(".", "-"), + "SITENAME": sites.site.name, + "MARIADB_HOST" : 'global-db', + "MARIADB_ROOT_PASS": '/run/secrets/db_root_password', + "CONTAINER_NAME_PREFIX": get_container_name_prefix(sites.site.name), + "ENVIRONMENT": "dev", + }, + "nginx": { + "ENABLE_SSL": enable_ssl, + "SITENAME": sites.site.name, + "VIRTUAL_HOST": sites.site.name, + "VIRTUAL_PORT": 80, + }, + "worker": { + "USERID": uid, + "USERGROUP": gid, + }, + "schedule": { + "USERID": uid, + "USERGROUP": gid, + }, + "socketio": { + "USERID": uid, + "USERGROUP": gid, + }, + } + + users: dict = {"nginx": {"uid": uid, "gid": gid}} + + template_inputs: dict = { + "environment": environment, + # "extra_hosts": extra_hosts, + "user": users, + } + # turn off all previous + # start the docker compose + + sites.create_site(template_inputs) + + +@app.command(no_args_is_help=True) +def delete(sitename: Annotated[str, typer.Argument(help="Name of the site")]): + """Delete a site. """ + sites.init(sitename) + # turn off the site + sites.remove_site() + + +@app.command() +def list(): + """Lists all of the available sites. """ + sites.init() + sites.list_sites() + + +@app.command(no_args_is_help=True) +def start(sitename: Annotated[str, typer.Argument(help="Name of the site")]): + """Start a site. """ + sites.init(sitename) + sites.start_site() + + +@app.command(no_args_is_help=True) +def stop(sitename: Annotated[str, typer.Argument(help="Name of the site")]): + """Stop a site. """ + sites.init(sitename) + sites.stop_site() + + +def code_command_callback(extensions: List[str]) -> List[str]: + extx = extensions + default_extension + unique_ext: Set = set(extx) + unique_ext_list: List[str] = [x for x in unique_ext] + return unique_ext_list + + +@app.command(no_args_is_help=True) +def code( + sitename: Annotated[str, typer.Argument(help="Name of the site.")], + user: Annotated[str, typer.Option(help="Connect as this user.")] = "frappe", + extensions: Annotated[ + Optional[List[str]], + typer.Option( + "--extension", + "-e", + help="List of extensions to install in vscode at startup.Provide extension id eg: ms-python.python", + callback=code_command_callback, + ), + ] = default_extension, + force_start: Annotated[bool , typer.Option('--force-start','-f',help="Force start the site before attaching to container.")] = False +): + """Open site in vscode. """ + sites.init(sitename) + if force_start: + sites.start_site() + sites.attach_to_site(user, extensions) + + +@app.command(no_args_is_help=True) +def logs( + sitename: Annotated[str, typer.Argument(help="Name of the site.")], + service: Annotated[Optional[SiteServicesEnum], typer.Option(help="Specify service name to show container logs.")] = None, + follow: Annotated[bool, typer.Option('--follow','-f',help="Follow logs.")] = False, +): + """Show frappe dev server logs or container logs for a given site. """ + sites.init(sitename) + if service: + sites.logs(service=SiteServicesEnum(service).value,follow=follow) + else: + sites.logs(follow=follow) + + +@app.command(no_args_is_help=True) +def shell( + sitename: Annotated[str, typer.Argument(help="Name of the site.")], + user: Annotated[str, typer.Option(help="Connect as this user.")] = None, + service: Annotated[SiteServicesEnum, typer.Option(help="Specify Service")] = 'frappe', +): + """Open shell for the give site. """ + sites.init(sitename) + if service: + sites.shell(service=SiteServicesEnum(service).value,user=user) + else: + sites.shell(user=user) + +@app.command(no_args_is_help=True) +def info( + sitename: Annotated[str, typer.Argument(help="Name of the site.")], +): + """Shows information about given site.""" + sites.init(sitename) + sites.info() diff --git a/frappe_manager/main.py b/frappe_manager/main.py index cc906aa9..abc96a37 100644 --- a/frappe_manager/main.py +++ b/frappe_manager/main.py @@ -1,369 +1,27 @@ -import typer -import importlib -import os -import requests -import sys -import shutil import atexit -from typing import Annotated, List, Literal, Optional, Set -from frappe_manager.site_manager.SiteManager import SiteManager from frappe_manager.display_manager.DisplayManager import richprint -from frappe_manager import CLI_DIR, default_extension, SiteServicesEnum from frappe_manager.logger import log -from frappe_manager.utils import check_update, is_cli_help_called, remove_zombie_subprocess_process -from frappe_manager.site_manager.utils import get_container_name_prefix +from frappe_manager.utils.helpers import check_update, remove_zombie_subprocess_process +from frappe_manager.utils.docker import process_opened +from frappe_manager.commands import app -app = typer.Typer(no_args_is_help=True,rich_markup_mode='rich') -sites = None -logger = None - -def exit_cleanup(): - """ - This function is used to perform cleanup at the exit. - """ - remove_zombie_subprocess_process() - check_update() - print('') - richprint.stop() def cli_entrypoint(): try: app() except Exception as e: + logger = log.get_logger() logger.exception(f"Exception: : {e}") raise e finally: atexit.register(exit_cleanup) -# this will be initiated later in the app_callback -sites: Optional[SiteManager] = None - -def version_callback(version: Optional[bool] = None): - if version: - fm_version = importlib.metadata.version('frappe_manager') - richprint.print(fm_version,emoji_code='') - raise typer.Exit() - -@app.callback() -def app_callback( - ctx: typer.Context, - verbose: Annotated[Optional[bool], typer.Option('--verbose','-v',help="Enable verbose output.")] = None, - version: Annotated[ - Optional[bool], typer.Option("--version",help="Show Version.",callback=version_callback) - ] = None, -): - """ - FrappeManager for creating frappe development envrionments. - """ - - - ctx.obj = {} - - help_called = is_cli_help_called(ctx) - ctx.obj["is_help_called"] = help_called - - if not help_called: - - sitesdir = CLI_DIR / 'sites' - - richprint.start(f"Working") - - if not CLI_DIR.exists(): - # creating the sites dir - # TODO check if it's writeable and readable -> by writing a file to it and catching exception - CLI_DIR.mkdir(parents=True, exist_ok=True) - sitesdir.mkdir(parents=True, exist_ok=True) - richprint.print(f"fm directory doesn't exists! Created at -> {str(CLI_DIR)}") - else: - if not CLI_DIR.is_dir(): - richprint.exit("Sites directory is not a directory! Aborting!") - - # logging - global logger - logger = log.get_logger() - logger.info('') - logger.info(f"{':'*20}FM Invoked{':'*20}") - logger.info('') - - # logging command provided by user - logger.info(f"RUNNING COMMAND: {' '.join(sys.argv[1:])}") - logger.info('-'*20) - - # Migration for directory change from CLI_DIR to CLI_DIR/sites - # TODO remove when not required, introduced in 0.8.4 - if not sitesdir.exists(): - richprint.change_head("Site directory migration") - move_directory_list = [] - for site_dir in CLI_DIR.iterdir(): - if site_dir.is_dir(): - docker_compose_path = site_dir / "docker-compose.yml" - if docker_compose_path.exists(): - move_directory_list.append(site_dir) - - # stop all the sites - sitesdir.mkdir(parents=True, exist_ok=True) - sites_mananger = SiteManager(CLI_DIR) - sites_mananger.stop_sites() - # move all the directories - for site in move_directory_list: - site_name = site.parts[-1] - new_path = sitesdir / site_name - try: - shutil.move(site,new_path) - richprint.print(f"Directory migrated: {site_name}") - except: - logger.debug(f'Site Directory migration failed: {site}') - richprint.warning(f"Unable to site directory migration for {site}\nPlease manually move it to {new_path}") - richprint.print("Site directory migration: Done") - - global sites - sites = SiteManager(sitesdir) - - sites.set_typer_context(ctx) - - if verbose: - sites.set_verbose() - - ctx.obj["sites"] = sites - ctx.obj["logger"] = logger - -def check_frappe_app_exists(appname: str, branchname: str | None = None): - # check appname - try: - app_url = f"https://github.com/frappe/{appname}" - app = requests.get(app_url).status_code - - if branchname: - branch_url = f"https://github.com/frappe/{appname}/tree/{branchname}" - # check branch - branch = requests.get(branch_url).status_code - return { - "app": True if app == 200 else False, - "branch": True if branch == 200 else False, - } - return {"app": True if app == 200 else False} - except Exception: - richprint.exit("Not able to connect to github.com.") - - -def apps_validation(value: List[str] | None): - # don't allow frappe the be included throw error - if value: - for app in value: - appx = app.split(":") - if appx == "frappe": - raise typer.BadParameter("Frappe should not be included here.") - if len(appx) == 1: - exists = check_frappe_app_exists(appx[0]) - if not exists["app"]: - raise typer.BadParameter(f"{app} is not a valid FrappeVerse app!") - if len(appx) == 2: - exists = check_frappe_app_exists(appx[0], appx[1]) - if not exists["app"]: - raise typer.BadParameter(f"{app} is not a valid FrappeVerse app!") - if not exists["branch"]: - raise typer.BadParameter( - f"{appx[1]} is not a valid branch of {appx[0]}!" - ) - if len(appx) > 2: - raise typer.BadParameter( - f"App should be specified in format : or " - ) - return value - - -def frappe_branch_validation_callback(value: str): - if value: - exists = check_frappe_app_exists("frappe", value) - if exists['branch']: - return value - else: - raise typer.BadParameter(f"Frappe branch -> {value} is not valid!! ") - -@app.command(no_args_is_help=True) -def create( - sitename: Annotated[str, typer.Argument(help="Name of the site")], - apps: Annotated[ - Optional[List[str]], - typer.Option( - "--apps", "-a", help="FrappeVerse apps to install. App should be specified in format : or .", callback=apps_validation, - show_default=False - ), - ] = None, - developer_mode: Annotated[bool, typer.Option(help="Enable developer mode")] = True, - frappe_branch: Annotated[ - str, typer.Option(help="Specify the branch name for frappe app",callback=frappe_branch_validation_callback) - ] = "version-15", - admin_pass: Annotated[ - str, - typer.Option( - help="Default Password for the standard 'Administrator' User. This will be used as the password for the Administrator User for all new sites" - ), - ] = "admin", - enable_ssl: Annotated[bool, typer.Option(help="Enable https")] = False, -): - # TODO Create markdown table for the below help +def exit_cleanup(): """ - Create a new site. - - Frappe\[version-14] will be installed by default. - - [bold white on black]Examples:[/bold white on black] - - [bold]# Install frappe\[version-14][/bold] - $ [blue]fm create example[/blue] - - [bold]# Install frappe\[version-15-beta][/bold] - $ [blue]fm create example --frappe-branch version-15-beta[/blue] - - [bold]# Install frappe\[version-14], erpnext\[version-14] and hrms\[version-14][/bold] - $ [blue]fm create example --apps erpnext:version-14 --apps hrms:version-14[/blue] - - [bold]# Install frappe\[version-15-beta], erpnext\[version-15-beta] and hrms\[version-15-beta][/bold] - $ [blue]fm create example --frappe-branch version-15-beta --apps erpnext:version-15-beta --apps hrms:version-15-beta[/blue] + This function is used to perform cleanup at the exit. """ + remove_zombie_subprocess_process(process_opened) + check_update() + print('') + richprint.stop() - sites.init(sitename) - - uid: int = os.getuid() - gid: int = os.getgid() - - environment = { - "frappe": { - "USERID": uid, - "USERGROUP": gid, - "APPS_LIST": ",".join(apps) if apps else None, - "FRAPPE_BRANCH": frappe_branch, - "DEVELOPER_MODE": developer_mode, - "ADMIN_PASS": admin_pass, - "DB_NAME": sites.site.name.replace(".", "-"), - "SITENAME": sites.site.name, - "MARIADB_ROOT_PASS": 'root', - "CONTAINER_NAME_PREFIX": get_container_name_prefix(sites.site.name), - "ENVIRONMENT": "dev", - }, - "nginx": { - "ENABLE_SSL": enable_ssl, - "SITENAME": sites.site.name, - "VIRTUAL_HOST": sites.site.name, - "VIRTUAL_PORT": 80, - }, - "worker": { - "USERID": uid, - "USERGROUP": gid, - }, - "schedule": { - "USERID": uid, - "USERGROUP": gid, - }, - "socketio": { - "USERID": uid, - "USERGROUP": gid, - }, - } - - users: dict = {"nginx": {"uid": uid, "gid": gid}} - - template_inputs: dict = { - "environment": environment, - # "extra_hosts": extra_hosts, - "user": users, - } - # turn off all previous - # start the docker compose - - sites.create_site(template_inputs) - - -@app.command(no_args_is_help=True) -def delete(sitename: Annotated[str, typer.Argument(help="Name of the site")]): - """Delete a site. """ - sites.init(sitename) - # turn off the site - sites.remove_site() - - -@app.command() -def list(): - """Lists all of the available sites. """ - sites.init() - sites.list_sites() - - -@app.command(no_args_is_help=True) -def start(sitename: Annotated[str, typer.Argument(help="Name of the site")]): - """Start a site. """ - sites.init(sitename) - sites.start_site() - - -@app.command(no_args_is_help=True) -def stop(sitename: Annotated[str, typer.Argument(help="Name of the site")]): - """Stop a site. """ - sites.init(sitename) - sites.stop_site() - - -def code_command_callback(extensions: List[str]) -> List[str]: - extx = extensions + default_extension - unique_ext: Set = set(extx) - unique_ext_list: List[str] = [x for x in unique_ext] - return unique_ext_list - - -@app.command(no_args_is_help=True) -def code( - sitename: Annotated[str, typer.Argument(help="Name of the site.")], - user: Annotated[str, typer.Option(help="Connect as this user.")] = "frappe", - extensions: Annotated[ - Optional[List[str]], - typer.Option( - "--extension", - "-e", - help="List of extensions to install in vscode at startup.Provide extension id eg: ms-python.python", - callback=code_command_callback, - ), - ] = default_extension, - force_start: Annotated[bool , typer.Option('--force-start','-f',help="Force start the site before attaching to container.")] = False -): - """Open site in vscode. """ - sites.init(sitename) - if force_start: - sites.start_site() - sites.attach_to_site(user, extensions) - - -@app.command(no_args_is_help=True) -def logs( - sitename: Annotated[str, typer.Argument(help="Name of the site.")], - service: Annotated[Optional[SiteServicesEnum], typer.Option(help="Specify service name to show container logs.")] = None, - follow: Annotated[bool, typer.Option('--follow','-f',help="Follow logs.")] = False, -): - """Show frappe dev server logs or container logs for a given site. """ - sites.init(sitename) - if service: - sites.logs(service=SiteServicesEnum(service).value,follow=follow) - else: - sites.logs(follow=follow) - - -@app.command(no_args_is_help=True) -def shell( - sitename: Annotated[str, typer.Argument(help="Name of the site.")], - user: Annotated[str, typer.Option(help="Connect as this user.")] = None, - service: Annotated[SiteServicesEnum, typer.Option(help="Specify Service")] = 'frappe', -): - """Open shell for the give site. """ - sites.init(sitename) - if service: - sites.shell(service=SiteServicesEnum(service).value,user=user) - else: - sites.shell(user=user) - -@app.command(no_args_is_help=True) -def info( - sitename: Annotated[str, typer.Argument(help="Name of the site.")], -): - """Shows information about given site.""" - sites.init(sitename) - sites.info() From db788d251c4d9d9463408a64294b863c3a9e7904 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 31 Jan 2024 14:17:17 +0530 Subject: [PATCH 084/100] optmize docker wrapper --- .../display_manager/DisplayManager.py | 256 ++++++------ frappe_manager/docker_wrapper/DockerClient.py | 51 ++- .../docker_wrapper/DockerCompose.py | 380 ++++-------------- 3 files changed, 243 insertions(+), 444 deletions(-) diff --git a/frappe_manager/display_manager/DisplayManager.py b/frappe_manager/display_manager/DisplayManager.py index c1fed410..1d7b8bda 100644 --- a/frappe_manager/display_manager/DisplayManager.py +++ b/frappe_manager/display_manager/DisplayManager.py @@ -13,9 +13,8 @@ from typing import Optional error = Style() -theme = Theme({ - 'errors': error -}) +theme = Theme({"errors": error}) + class DisplayManager: def __init__(self): @@ -26,215 +25,196 @@ def __init__(self): self.spinner = Spinner(text=self.current_head, name="dots2", speed=1) self.live = Live(self.spinner, console=self.stdout, transient=True) - def start(self,text: str): + def start(self, text: str): """ - The `start` function updates the text of a spinner with a new value and starts a live update. - - :param text: The `text` parameter is a string that represents the text that will be displayed in the spinner. - :type text: str + Starts the display manager with the given text. + + Args: + text (str): The text to be displayed. + + Returns: + None """ - self.current_head = self.previous_head = Text(text=text,style='bold blue') + self.current_head = self.previous_head = Text(text=text, style="bold blue") self.spinner.update(text=self.current_head) self.live.start() - def error(self,text: str,emoji_code: str = ':x:'): + def error(self, text: str, emoji_code: str = ":x:"): """ - The function `error` prints a given text with an optional emoji code. - - :param text: A string parameter that represents the error message or text that you want to display - :type text: str - :param emoji_code: The `emoji_code` parameter is a string that represents an emoji code, defaults to `:x:`. - :type emoji_code: str (optional) + Display an error message with an optional emoji code. + + Args: + text (str): The error message to display. + emoji_code (str, optional): The emoji code to display before the error message. Defaults to ':x:'. """ self.stdout.print(f"{emoji_code} {text}") - def warning(self,text: str,emoji_code: str = ':warning: '): + def warning(self, text: str, emoji_code: str = ":warning: "): """ - The function "warning" prints a warning message with an optional emoji code. - - :param text: A string that represents the warning message to be displayed - :type text: str - :param emoji_code: The `emoji_code` parameter is a string that represents an emoji code, - defaults to `:warning: `. - :type emoji_code: str (optional) + Display a warning message with an optional emoji code. + + Args: + text (str): The warning message to display. + emoji_code (str, optional): The emoji code to prepend to the message. Defaults to ":warning: ". + + Returns: + None """ self.stdout.print(f"{emoji_code} {text}") - def exit(self,text: str,emoji_code: str = ':x:',os_exit= False, error_msg= None): + def exit(self, text: str, emoji_code: str = ":x:", os_exit=False, error_msg=None): """ - The `exit` function stops the program, prints a message with an emoji, and exits using `typer.Exit` - exception. + Exits the display manager and prints the given text with an optional emoji code and error message. - :param text: The `text` parameter is a string that represents the message or reason for exiting. It - is the text that will be printed when the `exit` method is called - :type text: str - :param emoji_code: The `emoji_code` parameter is a string that represents an emoji code, - defaults to `:x: `. - :type emoji_code: str (optional) + Args: + text (str): The text to be printed. + emoji_code (str, optional): The emoji code to be displayed before the text. Default is ":x:". + os_exit (bool, optional): If True, the program will exit with status code 1. Default is False. + error_msg (str, optional): The error message to be displayed after the text. Default is None. """ self.stop() + + to_print = f"{emoji_code} {text}" if error_msg: to_print = f"{emoji_code} {text}\n Error : {error_msg}" - else: - to_print = f"{emoji_code} {text} " + self.stdout.print(to_print) + if os_exit: exit(1) + raise typer.Exit(1) - def print(self,text: str,emoji_code: str = ':white_check_mark:'): + def print(self, text: str, emoji_code: str = ":white_check_mark:", prefix: Optional[str] = None): """ - The function `print` takes in a string `text` and an optional string `emoji_code` and prints the - `text` with an emoji. - - :param text: A string that represents the text you want to print - :type text: str - :param emoji_code: The `emoji_code` parameter is a string that represents an emoji code, - defaults to :white_check_mark: - :type emoji_code: str (optional) + Prints the given text with an optional emoji code. + + Args: + text (str): The text to be printed. + emoji_code (str, optional): The emoji code to be displayed before the text. Defaults to ":white_check_mark:". """ - self.stdout.print(f"{emoji_code} {text}") + msg = f"{emoji_code} {text}" + + if prefix: + msg = f"{emoji_code} {prefix} {text}" + + self.stdout.print(msg) def update_head(self, text: str): """ - The `update_head` function updates text of spinner and print out the prvious text of the - spinner. + Update the head of the display manager with the given text. - :param text: The `text` parameter is a string that represents the new value for the head of an - object - :type text: str + Args: + text (str): The new head text. + + Returns: + None """ self.previous_head = self.current_head self.current_head = text - self.live.console.print(self.previous_head,style='blue') - self.spinner.update(text=Text(self.current_head,style='blue bold'),style='bold blue') + self.live.console.print(self.previous_head, style="blue") + self.spinner.update( + text=Text(self.current_head, style="blue bold"), style="bold blue" + ) def change_head(self, text: str): """ - The `change_head` function updates the head of a spinner with the provided text and refreshes the - display. - - :param text: The `text` parameter is a string that represents the new head text that you want to set - :type text: str + Change the head text and update the spinner and live display. + + Args: + text (str): The new head text. + + Returns: + None """ self.previous_head = self.current_head self.current_head = text - self.spinner.update(text=Text(self.current_head,style='blue bold')) + self.spinner.update(text=Text(self.current_head, style="blue bold")) self.live.refresh() - def update_live(self,renderable = None, padding: tuple = (0,0,0,0)): + def update_live(self, renderable=None, padding: tuple = (0, 0, 0, 0)): """ - The `update_live` function updates the live display with a renderable object, applying padding if - specified. - - :param renderable: The `renderable` parameter is an rich renderable object that can be rendered on the screen by rich. It - could be an image, text, or any other visual element that you want to display - :param padding: The `padding` parameter is a tuple that specifies the padding values for the - `renderable` object. The tuple should contain four values in the order of `(top, right, bottom, - left)`. These values represent the amount of padding to be added to the `renderable` object on each - :type padding: tuple + Update the live display with the given renderable object and padding. + + Args: + renderable: The object to be rendered on the live display. + padding: The padding values for the renderable object (top, right, bottom, left). """ if renderable: if padding: - renderable=Padding(renderable,padding) + renderable = Padding(renderable, padding) - group = Group(self.spinner,renderable) + group = Group(self.spinner, renderable) self.live.update(group) else: self.live.update(self.spinner) self.live.refresh() def live_lines( - self, - data, - stdout:bool = True, - stderr:bool = True, - lines: int = 4, - padding:tuple = (0,0,0,0), - return_exit_code:bool = False, - exit_on_faliure:bool = False, - stop_string: Optional[str] = None, - log_prefix: str = '=>', + self, + data, + stdout: bool = True, + stderr: bool = True, + lines: int = 4, + padding: tuple = (0, 0, 0, 0), + stop_string: Optional[str] = None, + log_prefix: str = "=>", + return_exit_code: bool = False, + exit_on_failure: bool = False, ): """ - The `live_lines` function takes in various parameters and continuously reads lines from a data - source, displaying them in a table format with a specified number of lines and optional padding, and - stops when a specified stop string is encountered. - - :param data: The `data` parameter is an iterator that yields tuples of two elements: the first - element is a string indicating the source of the line (either "stdout" or "stderr"), and the second - element is the line itself - :param stdout: A boolean indicating whether to display lines from stdout or not. If set to True, - lines from stdout will be displayed, defaults to True - :type stdout: bool (optional) - :param stderr: A boolean indicating whether to display lines from stderr, defaults to True - :type stderr: bool (optional) - :param lines: The `lines` parameter specifies the maximum number of lines to display in the output. - Only the most recent `lines` lines will be shown, defaults to 4 - :type lines: int (optional) - :param padding: The `padding` parameter is a tuple that specifies the padding (in characters) to be - added to the left, right, top, and bottom of the displayed lines in the table. The tuple should have - four values in the order (left, right, top, bottom). For example, if you - :type padding: tuple - :param return_exit_code: The `return_exit_code` parameter is a boolean flag that determines whether - the `live_lines` function should return the exit code of the process being monitored. If set to - `True`, the function will return the exit code as an integer value. If set to `False`, the function - will not return, defaults to False - :type return_exit_code: bool (optional) - :param exit_on_faliure: The `exit_on_faliure` parameter is a boolean flag that determines whether - the function should exit if there is a failure. If set to `True`, the function will exit when a - failure occurs. If set to `False`, the function will continue running even if there is a failure, - defaults to False - :type exit_on_faliure: bool (optional) - :param stop_string: The `stop_string` parameter is an optional string that can be provided to the - `live_lines` function. If this string is specified, the function will stop iterating through the - `data` generator when it encounters a line that contains the `stop_string` - :type stop_string: Optional[str] - :param log_prefix: The `log_prefix` parameter is a string that is used as a prefix for each line of - output displayed in the live view. It is added before the actual line of output and is typically - used to indicate the source or type of the output (e.g., "=> stdout: This is a line, defaults to => - :type log_prefix: str (optional) - :return: The function does not explicitly return anything. However, it has a parameter - `return_exit_code` which, if set to `True`, will cause the function to return an exit code of 0 when - the `stop_string` is found in the output. Otherwise, the function will not return anything. + Display live lines from the given data source. + + Args: + data: An iterator that yields tuples of (source, line) where source is either "stdout" or "stderr" and line is a string. + stdout: Whether to display lines from the stdout source. Default is True. + stderr: Whether to display lines from the stderr source. Default is True. + lines: The maximum number of lines to display. Default is 4. + padding: A tuple of four integers representing the padding (top, right, bottom, left) around the displayed lines. Default is (0, 0, 0, 0). + stop_string: A string that, if found in a line, will stop the display and return 0. Default is None. + log_prefix: The prefix to add to each displayed line. Default is "=>". + return_exit_code: Whether to return the exit code when stop_string is found. Default is False. + exit_on_failure: Whether to exit the program when stop_string is found. Default is False. """ max_height = lines displayed_lines = deque(maxlen=max_height) + while True: try: source, line = next(data) line = line.decode() - if "[==".lower() in line.lower(): + + if "[==".lower() in line.lower() or 'Updating files:'.lower() in line.lower(): continue - if source == 'stdout' and stdout: + + if source == "stdout" and stdout: displayed_lines.append(line) - if source == 'stderr' and stderr: + + if source == "stderr" and stderr: displayed_lines.append(line) - if stop_string: - if stop_string.lower() in line.lower(): - return 0 - table = Table(show_header=False,box=None) + + if stop_string and stop_string.lower() in line.lower(): + return 0 + + table = Table(show_header=False, box=None) table.add_column() + for linex in list(displayed_lines): - table.add_row( - Text(f"{log_prefix} {linex.strip()}",style='grey') - ) - self.update_live(table,padding=padding) + table.add_row(Text(f"{log_prefix} {linex.strip()}", style="grey")) + + self.update_live(table, padding=padding) self.live.refresh() - # except DockerException: - # self.update_live() - # self.stop() + except KeyboardInterrupt as e: richprint.live.refresh() + except StopIteration: break def stop(self): - """ - The function `stop` updates the spinner and live output, and then stops the live output. - """ self.spinner.update() - self.live.update(Text('',end='')) + self.live.update(Text("", end="")) self.live.stop() + richprint = DisplayManager() diff --git a/frappe_manager/docker_wrapper/DockerClient.py b/frappe_manager/docker_wrapper/DockerClient.py index 583dd224..8eb7914c 100644 --- a/frappe_manager/docker_wrapper/DockerClient.py +++ b/frappe_manager/docker_wrapper/DockerClient.py @@ -1,9 +1,11 @@ -from typing import Literal, Optional import json -from frappe_manager.docker_wrapper.DockerCompose import DockerComposeWrapper +import shlex + +from typing import Literal, Optional from pathlib import Path +from frappe_manager.docker_wrapper.DockerCompose import DockerComposeWrapper from frappe_manager.display_manager.DisplayManager import richprint -from frappe_manager.docker_wrapper.utils import ( +from frappe_manager.utils.docker import ( is_current_user_in_group, parameters_to_options, run_command_with_exit_code, @@ -11,16 +13,33 @@ class DockerClient: + """ + This class provide one to one mapping to the docker command. + + Only this args have are different use case. + stream (bool, optional): A boolean flag indicating whether to stream the output of the command as it runs. + If set to True, the output will be displayed in real-time. If set to False, the output will be + displayed after the command completes. Defaults to False. + stream_only_exit_code (bool, optional): A boolean flag indicating whether to only stream the exit code of the + command. Defaults to False. + """ + def __init__(self, compose_file_path: Optional[Path] = None): + """ + Initializes a DockerClient object. + Args: + compose_file_path (Optional[Path]): The path to the Docker Compose file. Defaults to None. + """ self.docker_cmd = ["docker"] if compose_file_path: self.compose = DockerComposeWrapper(compose_file_path) def version(self) -> dict: """ - The `version` function retrieves the version information of a Docker container and returns it as a - JSON object. - :return: a dictionary object named "output". + Retrieves the version information of the Docker client. + + Returns: + A dictionary containing the version information. """ parameters: dict = locals() @@ -33,20 +52,22 @@ def version(self) -> dict: iterator = run_command_with_exit_code(self.docker_cmd + ver_cmd, quiet=False) output: dict = {} + try: for source, line in iterator: if source == "stdout": output = json.loads(line.decode()) except Exception as e: return {} + return output def server_running(self) -> bool: """ - The function `server_running` checks if the Docker server is running and returns a boolean value - indicating its status. - :return: a boolean value. If the 'Server' key in the 'docker_info' dictionary is truthy, then the - function returns True. Otherwise, it returns False. + Checks if the Docker server is running. + + Returns: + bool: True if the Docker server is running, False otherwise. """ docker_info = self.version() if "Server" in docker_info: @@ -146,19 +167,25 @@ def run( name: Optional[str] = None, detach: bool = False, entrypoint: Optional[str] = None, + pull: Literal["missing", "never", "always"] = "missing", + use_shlex_split: bool = True, stream: bool = False, stream_only_exit_code: bool = False, ): parameters: dict = locals() run_cmd: list = ["run"] - remove_parameters = ["stream", "stream_only_exit_code", "image", "command"] + remove_parameters = ["stream", "stream_only_exit_code", "command", "image","use_shlex_split"] run_cmd += parameters_to_options(parameters, exclude=remove_parameters) + run_cmd += [f"{image}"] if command: - run_cmd += [f"{command}"] + if use_shlex_split: + run_cmd += shlex.split(command, posix=True) + else: + run_cmd += [command] iterator = run_command_with_exit_code( self.docker_cmd + run_cmd, quiet=stream_only_exit_code, stream=stream diff --git a/frappe_manager/docker_wrapper/DockerCompose.py b/frappe_manager/docker_wrapper/DockerCompose.py index cc61abbe..c6b9e68d 100644 --- a/frappe_manager/docker_wrapper/DockerCompose.py +++ b/frappe_manager/docker_wrapper/DockerCompose.py @@ -1,10 +1,10 @@ - from subprocess import Popen, run, TimeoutExpired, CalledProcessError from pathlib import Path -from typing import Union, Literal +from typing import Union, Literal, Optional import shlex -from frappe_manager.docker_wrapper.utils import ( + +from frappe_manager.utils.docker import ( parameters_to_options, run_command_with_exit_code, ) @@ -13,15 +13,14 @@ class DockerComposeWrapper: """ This class provides one to one mapping between docker compose cli each function. - - There are two parameter which are different: - :param stream: A boolean flag indicating whether to stream the output of the command as it runs. If - set to True, the output will be displayed in real-time. If set to False, the output will be - displayed after the command completes, defaults to False - :type stream: bool (optional) - :param stream_only_exit_code: A boolean flag indicating whether to only stream the exit code of the - command, defaults to False - :type stream_only_exit_code: bool (optional) + Only this args have are different use case. + + Args: + stream (bool, optional): A boolean flag indicating whether to stream the output of the command as it runs. + If set to True, the output will be displayed in real-time. If set to False, the output will be + displayed after the command completes. Defaults to False. + stream_only_exit_code (bool, optional): A boolean flag indicating whether to only stream the exit code of the + command. Defaults to False. """ def __init__(self, path: Path, timeout: int = 100): # requires valid path directory @@ -48,43 +47,6 @@ def up( stream: bool = False, stream_only_exit_code: bool = False, ): - """ - The `up` function is a Python method that runs the `docker-compose up` command with various options - and returns an iterator. - - :param detach: A boolean flag indicating whether to run containers in the background or not. If set - to True, containers will be detached and run in the background. If set to False, containers will run - in the foreground, defaults to True - :type detach: bool (optional) - :param build: A boolean flag indicating whether to build images before starting containers, defaults - to False - :type build: bool (optional) - :param remove_orphans: A boolean flag indicating whether to remove containers for services that are - no longer defined in the Compose file, defaults to False - :type remove_orphans: bool (optional) - :param no_recreate: A boolean flag indicating whether to recreate containers that already exist, - defaults to False - :type no_recreate: bool (optional) - :param always_recreate_deps: A boolean flag indicating whether to always recreate dependencies, - defaults to False - :type always_recreate_deps: bool (optional) - :param services: A list of services to be started. These services are defined in the Docker Compose - file and represent different components of your application - :type services: list[str] - :param quiet_pull: A boolean flag indicating whether to suppress the output of the pull command - during the "up" operation, defaults to False - :type quiet_pull: bool (optional) - :param pull: The `pull` parameter determines when to pull new images. It can have one of three - values: "missing", "never", or "always", defaults to missing - :type pull: Literal["missing", "never", "always"] (optional) - :param stream: A boolean flag indicating whether to stream the output of the command as it runs, - defaults to False - :type stream: bool (optional) - :param stream_only_exit_code: A boolean flag indicating whether to only stream the exit code of the - command, defaults to False - :type stream_only_exit_code: bool (optional) - :return: an iterator. - """ parameters: dict = locals() remove_parameters = ["services","stream", "stream_only_exit_code"] @@ -103,7 +65,6 @@ def up( ) return iterator - # @handle_docker_error def down( self, timeout: int = 100, @@ -141,25 +102,6 @@ def start( stream: bool = False, stream_only_exit_code: bool = False, ): - """ - The `start` function is used to start Docker services specified in the `services` parameter, with - options for dry run, streaming output, and checking only the exit code. - - :param services: A list of services to start. If None, all services will be started - :type services: Union[None, list[str]] - :param dry_run: A boolean flag indicating whether the start operation should be performed in dry run - mode or not. If set to True, the start operation will not actually be executed, but the command and - options will be printed. If set to False, the start operation will be executed, defaults to False - :type dry_run: bool (optional) - :param stream: A boolean flag indicating whether to stream the output of the command in real-time or - not. If set to True, the output will be streamed as it is generated. If set to False, the output - will be returned as a single string after the command completes, defaults to False - :type stream: bool (optional) - :param stream_only_exit_code: A boolean flag indicating whether only the exit code should be - streamed, defaults to False - :type stream_only_exit_code: bool (optional) - :return: The `start` method returns an iterator. - """ parameters: dict = locals() start_cmd: list[str] = ["start"] @@ -187,30 +129,6 @@ def restart( stream: bool = False, stream_only_exit_code: bool = False, ): - """ - The `restart` function restarts specified services in a Docker Compose environment with various - options and returns an iterator. - - :param services: A list of services to restart. If set to None, all services will be restarted - :type services: Union[None, list[str]] - :param dry_run: A boolean flag indicating whether the restart operation should be performed as a dry - run (without actually restarting the services), defaults to False - :type dry_run: bool (optional) - :param timeout: The `timeout` parameter specifies the maximum time (in seconds) to wait for the - services to restart before timing out, defaults to 100 - :type timeout: int (optional) - :param no_deps: A boolean flag indicating whether to restart the services without recreating their - dependent services, defaults to False - :type no_deps: bool (optional) - :param stream: A boolean flag indicating whether to stream the output of the restart command or not. - If set to True, the output will be streamed in real-time. If set to False, the output will be - returned as an iterator, defaults to False - :type stream: bool (optional) - :param stream_only_exit_code: A boolean flag indicating whether to only stream the exit code of the - restart command, defaults to False - :type stream_only_exit_code: bool (optional) - :return: The `restart` method returns an iterator. - """ parameters: dict = locals() parameters["timeout"] = str(timeout) @@ -237,25 +155,6 @@ def stop( stream: bool = False, stream_only_exit_code: bool = False, ): - """ - The `stop` function stops specified services in a Docker Compose environment, with options for - timeout, streaming output, and checking for service existence. - - :param services: A list of service names to stop. If None, all services will be stopped - :type services: Union[None, list[str]] - :param timeout: The `timeout` parameter specifies the maximum time (in seconds) to wait for the - services to stop before forcefully terminating them, defaults to 100 - :type timeout: int (optional) - :param stream: A boolean flag indicating whether to stream the output of the command as it is - executed, defaults to False - :type stream: bool (optional) - :param stream_only_exit_code: The `stream_only_exit_code` parameter is a boolean flag that - determines whether only the exit code of the command should be streamed or not. If set to `True`, - only the exit code will be streamed, otherwise, the full output of the command will be streamed, - defaults to False - :type stream_only_exit_code: bool (optional) - :return: The `stop` method returns an iterator. - """ parameters: dict = locals() parameters["timeout"] = str(timeout) @@ -265,9 +164,8 @@ def stop( stop_cmd += parameters_to_options(parameters, exclude=remove_parameters) - # doesn't checks if service exists or not if type(services) == list: - stop_cmd += services + stop_cmd.extend(services) iterator = run_command_with_exit_code( self.docker_compose_cmd + stop_cmd, quiet=stream_only_exit_code, @@ -289,47 +187,6 @@ def exec( stream_only_exit_code: bool = False, use_shlex_split: bool = True, ): - """ - The `exec` function in Python executes a command in a Docker container and returns an iterator for - the command's output. - - :param service: The `service` parameter is a string that represents the name of the service you want - to execute the command on - :type service: str - :param command: The `command` parameter is a string that represents the command to be executed - within the specified service - :type command: str - :param detach: A boolean flag indicating whether the command should be detached from the terminal, - defaults to False - :type detach: bool (optional) - :param env: The `env` parameter is a list of environment variables that you can pass to the command - being executed. Each element in the list should be a string in the format "KEY=VALUE". These - environment variables will be set for the duration of the command execution - :type env: Union[None, list[str]] - :param no_tty: A boolean flag indicating whether to allocate a pseudo-TTY for the executed command, - defaults to False - :type no_tty: bool (optional) - :param privileged: A boolean flag indicating whether the command should be executed with elevated - privileges, defaults to False - :type privileged: bool (optional) - :param user: The `user` parameter is used to specify the username or UID (User Identifier) to run - the command as within the container. If `user` is set to `None`, the command will be executed as the - default user in the container - :type user: Union[None, str] - :param workdir: The `workdir` parameter specifies the working directory for the command to be - executed within the container. If `workdir` is set to `None`, the command will be executed in the - default working directory of the container - :type workdir: Union[None, str] - :param stream: A boolean flag indicating whether to stream the output of the command execution, - defaults to False - :type stream: bool (optional) - :param stream_only_exit_code: The `stream_only_exit_code` parameter is a boolean flag that - determines whether only the exit code of the command should be streamed or the entire output. If - `stream_only_exit_code` is set to `True`, only the exit code will be streamed. If it is set to - `False`, the, defaults to False - :type stream_only_exit_code: bool (optional) - :return: an iterator. - """ parameters: dict = locals() exec_cmd: list[str] = ["exec"] @@ -354,7 +211,7 @@ def exec( if use_shlex_split: exec_cmd += shlex.split(command, posix=True) else: - exec_cmd += command + exec_cmd += [command] iterator = run_command_with_exit_code( self.docker_compose_cmd + exec_cmd, @@ -400,55 +257,6 @@ def ps( stream: bool = False, stream_only_exit_code: bool = False, ): - """ - The `ps` function is a Python method that executes the `docker-compose ps` command with various - parameters and returns an iterator. - - :param service: A list of service names to filter the results by. If None, all services are included - :type service: Union[None, list[str]] - :param dry_run: A boolean flag indicating whether the command should be executed in dry run mode, - where no changes are actually made, defaults to False - :type dry_run: bool (optional) - :param all: A boolean flag indicating whether to show all containers, including stopped ones, - defaults to False - :type all: bool (optional) - :param services: A boolean flag indicating whether to display only service names or all container - information, defaults to False - :type services: bool (optional) - :param filter: The `filter` parameter is used to filter the list of containers based on their - status. It accepts the following values: - :type filter: Union[j ,, - :param format: The `format` parameter specifies the output format for the `ps` command. It can be - set to either "table" or "json" - :type format: Union[None, Literal["table", "json"]] - :param status: The `status` parameter is used to filter the output of the `ps` command based on the - status of the services. It can be a list of status values such as "paused", "restarting", - "removing", "running", "dead", "created", or "exited" - :type status: Union[ - None, - list[ - Literal[ - "paused", - "restarting", - "removing", - "running", - "dead", - "created", - "exited", - ] - ], - ] - :param quiet: A boolean flag indicating whether to suppress output and only display essential - information, defaults to False - :type quiet: bool (optional) - :param stream: A boolean flag indicating whether to stream the output of the command in real-time, - defaults to False - :type stream: bool (optional) - :param stream_only_exit_code: A boolean flag indicating whether to only stream the exit code of the - command, defaults to False - :type stream_only_exit_code: bool (optional) - :return: The `ps` method returns an iterator. - """ parameters: dict = locals() ps_cmd: list[str] = ["ps"] @@ -492,47 +300,6 @@ def logs( stream: bool = False, stream_only_exit_code: bool = False, ): - """ - The `logs` function in Python takes in various parameters and returns an iterator that runs a Docker - Compose command to retrieve logs from specified services. - - :param services: A list of services for which to retrieve logs. If None, logs for all services will - be retrieved - :type services: Union[None, list[str]] - :param dry_run: A boolean flag indicating whether the command should be executed in dry run mode, - where no changes are actually made, defaults to False - :type dry_run: bool (optional) - :param follow: A boolean flag indicating whether to follow the logs in real-time, defaults to False - :type follow: bool (optional) - :param no_color: A boolean flag indicating whether to disable color output in the logs. If set to - True, the logs will be displayed without any color formatting, defaults to False - :type no_color: bool (optional) - :param no_log_prefix: A boolean flag indicating whether to exclude the log prefix in the output. If - set to True, the log prefix will be omitted, defaults to False - :type no_log_prefix: bool (optional) - :param since: The `since` parameter is used to specify the start time for retrieving logs. It - accepts a string value representing a time duration. For example, "10s" means logs from the last 10 - seconds, "1h" means logs from the last 1 hour, and so on. If - :type since: Union[None, str] - :param tail: The `tail` parameter specifies the number of lines to show from the end of the logs. If - set to `None`, it will show all the logs - :type tail: Union[None, int] - :param until: The `until` parameter specifies a timestamp or duration to limit the logs output - until. It can be either a timestamp in the format `YYYY-MM-DDTHH:MM:SS` or a duration in the format - `10s`, `5m`, `2h`, etc - :type until: Union[None, int] - :param timestamps: A boolean flag indicating whether to show timestamps in the log output, defaults - to False - :type timestamps: bool (optional) - :param stream: A boolean flag indicating whether to stream the logs in real-time or not. If set to - True, the logs will be continuously streamed as they are generated. If set to False, the logs will - be displayed once and the function will return, defaults to False - :type stream: bool (optional) - :param stream_only_exit_code: A boolean flag indicating whether to only stream the exit code of the - logs command, defaults to False - :type stream_only_exit_code: bool (optional) - :return: The function `logs` returns an iterator. - """ parameters: dict = locals() logs_cmd: list[str] = ["logs"] @@ -558,25 +325,6 @@ def ls( format: Literal["table", "json"] = "table", quiet: bool = False, ): - """ - The `ls` function is a Python method that runs the `ls` command in Docker Compose and returns the - output. - - :param all: A boolean flag indicating whether to show hidden files and directories, defaults to - False - :type all: bool (optional) - :param dry_run: The `dry_run` parameter is a boolean flag that indicates whether the `ls` command - should be executed as a dry run. A dry run means that the command will be simulated and no actual - changes will be made, defaults to False - :type dry_run: bool (optional) - :param format: The `format` parameter specifies the output format for the `ls` command. It can be - either "table" or "json", defaults to table - :type format: Literal["table", "json"] (optional) - :param quiet: A boolean flag indicating whether to suppress all output except for errors. If set to - True, the command will not display any output except for error messages, defaults to False - :type quiet: bool (optional) - :return: the output of the `ls` command as a string. - """ parameters: dict = locals() ls_cmd: list[str] = ["ls"] @@ -601,35 +349,6 @@ def pull( stream: bool = False, stream_only_exit_code: bool = False, ): - """ - The `pull` function is used to pull Docker images, with various options for customization. - - :param dry_run: A boolean flag indicating whether the pull operation should be performed as a dry - run (without actually pulling the images), defaults to False - :type dry_run: bool (optional) - :param ignore_buildable: A boolean flag indicating whether to ignore services that are marked as - "buildable" in the docker-compose file, defaults to False - :type ignore_buildable: bool (optional) - :param ignore_pull_failures: A boolean flag indicating whether to ignore failures when pulling - images. If set to True, any failures encountered during the pull process will be ignored and the - pull operation will continue. If set to False, any failures will cause the pull operation to stop - and an error will be raised, defaults to False - :type ignore_pull_failures: bool (optional) - :param include_deps: A boolean flag indicating whether to include dependencies when pulling images. - If set to True, it will pull images for all services and their dependencies. If set to False, it - will only pull images for the specified services, defaults to False - :type include_deps: bool (optional) - :param quiet: A boolean flag indicating whether to suppress output from the command. If set to True, - the command will be executed quietly without printing any output, defaults to False - :type quiet: bool (optional) - :param stream: A boolean flag indicating whether to stream the output of the pull command, defaults - to False - :type stream: bool (optional) - :param stream_only_exit_code: A boolean flag indicating whether to only return the exit code of the - command when streaming, defaults to False - :type stream_only_exit_code: bool (optional) - :return: an iterator. - """ parameters: dict = locals() pull_cmd: list[str] = ["pull"] @@ -644,3 +363,76 @@ def pull( stream=stream, ) return iterator + + def run( + self, + service: str, + command: Optional[str] = None, + name: Optional[str] = None, + detach: bool = False, + rm: bool = False, + entrypoint: Optional[str] = None, + use_shlex_split: bool = True, + stream: bool = False, + stream_only_exit_code: bool = False, + ): + parameters: dict = locals() + run_cmd: list = ["run"] + + remove_parameters = ["stream", "stream_only_exit_code", "command", "service","use_shlex_split"] + + run_cmd += parameters_to_options(parameters, exclude=remove_parameters) + + run_cmd += [service] + + if command: + if use_shlex_split: + run_cmd += shlex.split(command, posix=True) + else: + run_cmd += [command] + + + iterator = run_command_with_exit_code( + self.docker_compose_cmd + run_cmd, quiet=stream_only_exit_code, stream=stream + ) + return iterator + + def cp( + self, + source: str, + destination: str, + source_container: str = None, + destination_container: str = None, + archive: bool = False, + follow_link: bool = False, + quiet: bool = False, + stream: bool = False, + stream_only_exit_code: bool = False, + ): + parameters: dict = locals() + cp_cmd: list = ["cp"] + + remove_parameters = [ + "stream", + "stream_only_exit_code", + "source", + "destination", + "source_container", + "destination_container", + ] + + cp_cmd += parameters_to_options(parameters, exclude=remove_parameters) + + if source_container: + source = f"{source_container}:{source}" + + if destination_container: + destination = f"{destination_container}:{destination}" + + cp_cmd += [f"{source}"] + cp_cmd += [f"{destination}"] + + iterator = run_command_with_exit_code( + self.docker_compose_cmd + cp_cmd, quiet=stream_only_exit_code, stream=stream + ) + return iterator From 62916b7f572112e93f03396bbbf4f4203ec646ee Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 31 Jan 2024 14:18:22 +0530 Subject: [PATCH 085/100] feat: migration module for handling migrations --- frappe_manager/__init__.py | 3 + frappe_manager/metadata_manager.py | 17 + frappe_manager/migration_manager/__init__.py | 0 .../migration_manager/backup_manager.py | 150 +++++ .../migration_manager/migrate_compose.py | 31 ++ .../migration_manager/migration_base.py | 24 + .../migration_manager/migration_exections.py | 7 + .../migration_manager/migration_executor.py | 178 ++++++ .../migration_manager/migrations/\\" | 61 +++ .../migration_manager/migrations/__init__.py | 1 + .../migrations/migrate_0_10_0.py | 514 ++++++++++++++++++ .../migrations/migrate_0_9_0.py | 91 ++++ frappe_manager/migration_manager/version.py | 28 + frappe_manager/toml_manager.py | 37 ++ poetry.lock | 13 +- pyproject.toml | 1 + 16 files changed, 1155 insertions(+), 1 deletion(-) create mode 100644 frappe_manager/metadata_manager.py create mode 100644 frappe_manager/migration_manager/__init__.py create mode 100644 frappe_manager/migration_manager/backup_manager.py create mode 100644 frappe_manager/migration_manager/migrate_compose.py create mode 100644 frappe_manager/migration_manager/migration_base.py create mode 100644 frappe_manager/migration_manager/migration_exections.py create mode 100644 frappe_manager/migration_manager/migration_executor.py create mode 100644 "frappe_manager/migration_manager/migrations/\\" create mode 100644 frappe_manager/migration_manager/migrations/__init__.py create mode 100644 frappe_manager/migration_manager/migrations/migrate_0_10_0.py create mode 100644 frappe_manager/migration_manager/migrations/migrate_0_9_0.py create mode 100644 frappe_manager/migration_manager/version.py create mode 100644 frappe_manager/toml_manager.py diff --git a/frappe_manager/__init__.py b/frappe_manager/__init__.py index 48c9fa13..14d37fac 100644 --- a/frappe_manager/__init__.py +++ b/frappe_manager/__init__.py @@ -4,6 +4,9 @@ # TODO configure this using config #sites_dir = Path().home() / __name__.split(".")[0] CLI_DIR = Path.home() / 'frappe' +CLI_METADATA_PATH = CLI_DIR / '.fm.toml' +CLI_SITES_ARCHIVE = CLI_DIR / 'archived' + default_extension = [ "dbaeumer.vscode-eslint", diff --git a/frappe_manager/metadata_manager.py b/frappe_manager/metadata_manager.py new file mode 100644 index 00000000..71f328ce --- /dev/null +++ b/frappe_manager/metadata_manager.py @@ -0,0 +1,17 @@ +from pathlib import Path +from frappe_manager.toml_manager import TomlManager +from frappe_manager.templates.fm_metadata import metadata +from frappe_manager.migration_manager.version import Version +from tomlkit import comment, document, dumps, loads, table, toml_document +from frappe_manager import CLI_METADATA_PATH + +class MetadataManager(TomlManager): + def __init__(self, metadata_file: Path = CLI_METADATA_PATH, template: toml_document.TOMLDocument = metadata): + super().__init__(metadata_file, template) + self.load() + + def get_version(self) -> Version: + return Version(str(self.get('version'))) + + def set_version(self, version: Version): + self.set('version', str(version)) diff --git a/frappe_manager/migration_manager/__init__.py b/frappe_manager/migration_manager/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/frappe_manager/migration_manager/backup_manager.py b/frappe_manager/migration_manager/backup_manager.py new file mode 100644 index 00000000..cdaf53dd --- /dev/null +++ b/frappe_manager/migration_manager/backup_manager.py @@ -0,0 +1,150 @@ +import shutil +from datetime import datetime +from pathlib import Path +import string + +from rich import inspect +from frappe_manager import CLI_DIR +from dataclasses import dataclass, field +from typing import Optional +from frappe_manager.logger import log + +random_strings = [] + +@dataclass +class BackupData(): + src: Path + dest: Path + site: Optional[str] = None + allow_restore: bool = True + _is_restored: bool = False + prefix_timestamp = False + _prefix_length: int = 5 + + @property + def is_restored(self) -> bool: + return self._is_restored + + @is_restored.setter + def is_restored(self, v: bool) -> None: + self._is_restored = v + + def __post_init__(self): + + file_name = self.dest.name + + if self.prefix_timestamp: + while True: + # Get current date and time + now = datetime.now() + # Format date and time + current_time = now.strftime('%d-%b-%y--%H-%M-%S') + + if current_time not in random_strings: + random_strings.append(current_time) + file_name = f"{self.dest.name}-{current_time}" + break + + self.real_dest = self.dest.parent + + if self.site: + self.real_dest = self.real_dest / self.site + + self.real_dest: Path = self.real_dest / file_name + + def exists(self): + return self.dest.exists() + + +CLI_MIGARATIONS_DIR = CLI_DIR / 'migrations'/ f"{datetime.now().strftime('%d-%b-%y--%H-%M-%S')}" / 'backups' + +class BackupManager(): + + def __init__(self,name, base_dir: Path = CLI_MIGARATIONS_DIR): + self.root_backup_dir = base_dir + self.backup_dir = self.root_backup_dir / name + self.backups = [] + self.logger = log.get_logger() + + # create backup dir if not exists + self.backup_dir.mkdir(parents=True,exist_ok=True) + + def backup(self, src: Path, dest: Optional[Path] = None, site_name: Optional[str] = None, allow_restore: bool = True ): + + if not src.exists(): + return None + + if not dest: + dest = self.backup_dir / src.name + + backup_data = BackupData(src, dest, allow_restore=allow_restore) + + if site_name: + backup_data = BackupData(src, dest,site=site_name) + if not backup_data.real_dest.parent.exists(): + backup_data.real_dest.parent.mkdir(parents=True,exist_ok=True) + + + self.logger.debug(f"Backup: {backup_data.src} => {backup_data.real_dest} ") + + if src.is_dir(): + # Copy directory + shutil.copytree(backup_data.src, backup_data.real_dest) + else: + # Copy file + shutil.copy2(backup_data.src, backup_data.real_dest) + + self.backups.append(backup_data) + + return backup_data + + def restore(self, backup_data, force = False): + """ + Restore a file from a backup. + """ + if not backup_data.allow_restore: + return None + + if not backup_data.real_dest.exists(): + # print(f"No backup found at {backup_data.real_dest}") + return None + + if force: + self.logger.debug(f"Restore: {backup_data.real_dest} => {backup_data.src} ") + if backup_data.src.exists(): + if backup_data.src.is_dir(): + shutil.rmtree(backup_data.src) + else: + backup_data.src.unlink() + + dest = shutil.copy(backup_data.real_dest, backup_data.src) + + backup_data.is_restored = True + + return dest + # print(f"Restored {backup_data.src} from backup") + + def delete(self, backup_data): + """ + Delete a specific backup. + """ + if not backup_data.real_dest.exists(): + # print(f"No backup found at {backup_data.real_dest}") + return None + + shutil.rmtree(backup_data.real_dest) + + self.backups.remove(backup_data) + # print(f"Deleted backup at {backup_data.real_dest}") + + def delete_all(self): + """ + Delete all backups. + """ + for backup_data in self.backups: + if backup_data.real_dest.exists(): + shutil.rmtree(backup_data.real_dest) + # print(f"Deleted backup at {backup_data.real_dest}") + + self.backups.clear() + # print("Deleted all backups") diff --git a/frappe_manager/migration_manager/migrate_compose.py b/frappe_manager/migration_manager/migrate_compose.py new file mode 100644 index 00000000..7928a965 --- /dev/null +++ b/frappe_manager/migration_manager/migrate_compose.py @@ -0,0 +1,31 @@ + +from frappe_manager.migration_manager.migration_base import MigrationBase +from frappe_manager.site_manager.SiteManager import SiteManager +from frappe_manager.site_manager.site import Site + + +class MigrateCompose(MigrationBase): + def __init__(self,sites_dir): + self.sites_dir = sites_dir + # super().__init__() + + def up(self): + # take backup of each of the site docker compose + sites_manager = SiteManager(self.sites_dir) + sites = sites_manager.get_all_sites() + + # migrate each site + for site_name,site_path in sites.items(): + site = Site(site_name,site_path) + # take backup of the docker compose.yml + self.backup_manager.backup(site.path) + site.migrate_site_compose() + + def backup_site(self,site_name): + site_path = self.sites_manager.get_site_path(site_name) + self.backup_manager.backup(site_patjh + + def down(self): + + for backup in self.backup_manager.backups: + self.backup_manager.restore(backup) diff --git a/frappe_manager/migration_manager/migration_base.py b/frappe_manager/migration_manager/migration_base.py new file mode 100644 index 00000000..db77242e --- /dev/null +++ b/frappe_manager/migration_manager/migration_base.py @@ -0,0 +1,24 @@ +from typing import Protocol, runtime_checkable + +from frappe_manager.migration_manager.backup_manager import BackupManager +from frappe_manager.migration_manager.version import Version +from frappe_manager import CLI_DIR +from frappe_manager.logger import log + +@runtime_checkable +class MigrationBase(Protocol): + + version: Version = Version("0.0.0") + skip: bool = False + migration_executor = None + backup_manager: BackupManager # Declare the backup_manager variable + logger = log.get_logger() + + def __init__(self): + self.backup_manager = BackupManager(str(self.version)) # Assign the value to backup_manager + + def up(self): + pass + + def down(self): + pass diff --git a/frappe_manager/migration_manager/migration_exections.py b/frappe_manager/migration_manager/migration_exections.py new file mode 100644 index 00000000..19c1fad1 --- /dev/null +++ b/frappe_manager/migration_manager/migration_exections.py @@ -0,0 +1,7 @@ + +class MigrationExceptionInSite(Exception): + def __init__( + self, + error_msg: str, + ): + super().__init__(error_msg) diff --git a/frappe_manager/migration_manager/migration_executor.py b/frappe_manager/migration_manager/migration_executor.py new file mode 100644 index 00000000..0ae50fa2 --- /dev/null +++ b/frappe_manager/migration_manager/migration_executor.py @@ -0,0 +1,178 @@ +import shutil +import typer +import os +import importlib +import configparser +import pkgutil +from pathlib import Path + +import rich + +from rich import inspect +from frappe_manager import CLI_DIR , CLI_SITES_ARCHIVE +from frappe_manager.compose_manager.ComposeFile import ComposeFile +from frappe_manager.metadata_manager import MetadataManager +from frappe_manager.migration_manager.migration_exections import MigrationExceptionInSite +from frappe_manager.utils.helpers import downgrade_package, get_current_fm_version +from frappe_manager.logger import log +from frappe_manager.migration_manager.version import Version +from frappe_manager.migration_manager.migration_base import MigrationBase +from frappe_manager.display_manager.DisplayManager import richprint + +class MigrationExecutor(): + """ + Migration executor class. + + This class is responsible for executing migrations. + """ + + def __init__(self): + self.metadata_manager = MetadataManager() + self.prev_version = self.metadata_manager.get_version() + self.current_version = Version(get_current_fm_version()) + self.migrations_path = Path(__file__).parent / 'migrations' + self.logger = log.get_logger() + self.migrations = [] + self.undo_stack = [] + self.migrate_sites = {} + + def execute(self): + """ + Execute the migration. + This method will execute the migration and return the number of + executed statements. + """ + + if not self.prev_version < self.current_version: + return True + + current_migration = None + + # Dynamically import all modules in the 'migrations' subfolder + for (_, name, _) in pkgutil.iter_modules([str(self.migrations_path)]): + try: + module = importlib.import_module(f'.migrations.{name}', __package__) + for attr_name in dir(module): + attr = getattr(module, attr_name) + if isinstance(attr, type) and hasattr(attr, 'up') and hasattr(attr, 'down') and hasattr(attr, 'set_migration_executor'): + migration = attr() + migration.set_migration_executor(migration_executor = self) + current_migration = migration + if migration.version > self.prev_version and migration.version <= self.current_version: + # if not migration.skip: + self.migrations.append(migration) + # else: + + + except Exception as e: + print(f"Failed to register migration {name}: {e}") + + #self.migrations.sort() + self.migrations = sorted(self.migrations, key=lambda x: x.version) + + # print info to user + if self.migrations: + richprint.print("Pending Migrations...") + + for migration in self.migrations: + richprint.print(f"[bold]MIGRATION:[/bold] v{migration.version}") + + rollback = False + archive = False + + try: + # run all the migrations + for migration in self.migrations: + richprint.change_head(f"Running migration introduced in v{migration.version}") + self.logger.info(f"[{migration.version}] : Migration starting") + try: + self.undo_stack.append(migration) + migration.up() + self.prev_version = migration.version + + except Exception as e: + self.logger.error(f"[{migration.version}] : Migration Failed\n{e}") + raise e + + except MigrationExceptionInSite as e: + richprint.stop() + if self.migrate_sites: + richprint.print("[green]Migration was successfull on these sites.[/green]") + + for site, exception in self.migrate_sites.items(): + if not exception: + richprint.print(f"[bold][green]SITE:[/green][/bold] {site.name}") + + richprint.print("[red]Migration failed on these sites[/red]") + + for site, exception in self.migrate_sites.items(): + if exception: + richprint.print(f"[bold][red]SITE[/red]:[/bold] {site.name}") + richprint.print(f"[bold][red]EXCEPTION[/red]:[/bold] {exception}") + + archive_msg =( + f"\nIF [y]: Sites that have failed will be rolled back and stored in {CLI_SITES_ARCHIVE}." + "\nIF [N]: Revert the entire migration to the previous fm version." + "\nDo you wish to archive all sites that failed during migration?" + ) + + from rich.text import Text + archive = typer.confirm(archive_msg) + + if not archive: + rollback = True + + except Exception as e: + richprint.print(f"Migration failed: {e}") + rollback = True + + + if archive: + for site, exception in self.migrate_sites.items(): + if exception: + archive_site_path = CLI_SITES_ARCHIVE / site.name + CLI_SITES_ARCHIVE.mkdir(exist_ok=True, parents=True) + shutil.move(site.path,archive_site_path ) + richprint.print(f"[bold]Archived site:[/bold] {site.name}") + + if rollback: + richprint.start('Rollback') + self.rollback() + richprint.stop() + self.metadata_manager.set_version(self.prev_version) + self.metadata_manager.save() + richprint.print(f"Installing [bold][blue]Frappe-Manager[/blue][/bold] version: v{str(self.prev_version.version)}") + downgrade_package('frappe-manager',str(self.prev_version.version)) + richprint.exit("Rollback complete.") + + self.metadata_manager.set_version(self.prev_version) + self.metadata_manager.save() + + return True + + def set_site_data(self,site,data = None): + self.migrate_sites[site] = data + + def get_site_data(self,site): + try: + data = self.migrate_sites[site] + except KeyError as e: + return None + + def rollback(self): + """ + Rollback the migration. + This method will rollback the migration and return the number of + rolled back statements. + """ + + # run all the migrations + for migration in reversed(self.undo_stack): + if migration.version > self.prev_version: + richprint.change_head(f"Rolling back migration introduced in v{migration.version}") + self.logger.info(f"[{migration.version}] : Rollback starting") + try: + migration.down() + except Exception as e: + self.logger.error(f"[{migration.version}] : Rollback Failed\n{e}") + raise e diff --git "a/frappe_manager/migration_manager/migrations/\\" "b/frappe_manager/migration_manager/migrations/\\" new file mode 100644 index 00000000..73873cee --- /dev/null +++ "b/frappe_manager/migration_manager/migrations/\\" @@ -0,0 +1,61 @@ +import shutil +from frappe_manager.migration_manager.migration_base import MigrationBase + +from frappe_manager import CLI_DIR +from frappe_manager.site_manager.SiteManager import SiteManager +#from frappe_manager.display_manager.DisplayManager import richprint + +class MigrationV090(MigrationBase): + + version = "0.9.0" + + def __init__(self): + self.sitesdir = CLI_DIR / "sites" + + def up(self): + if not self.sitesdir.exists(): + + move_directory_list = [] + for site_dir in CLI_DIR.iterdir(): + + if site_dir.is_dir(): + docker_compose_path = site_dir / "docker-compose.yml" + + if docker_compose_path.exists(): + move_directory_list.append(site_dir) + + # stop all the sites + self.sitesdir.mkdir(parents=True, exist_ok=True) + sites_mananger = SiteManager(CLI_DIR) + sites_mananger.stop_sites() + + # move all the directories + for site in move_directory_list: + site_name = site.parts[-1] + new_path = self.sitesdir / site_name + shutil.move(site, new_path) + + def down(self): + + # check if any dirs is available in sites dir + if self.sitesdir.exists(): + move_directory_list = [] + for site_dir in CLI_DIR.iterdir(): + if site_dir.is_dir(): + docker_compose_path = site_dir / "docker-compose.yml" + + if docker_compose_path.exists(): + move_directory_list.append(site_dir) + + # stop all the sites + sites_mananger = SiteManager(CLI_DIR) + sites_mananger.stop_sites() + + # move all the directories + for site in move_directory_list: + site_name = site.parts[-1] + new_path = self.sitesdir.parent / site_name + shutil.move(site, new_path) + + # delete the sitedir + shutil.rmtree(self.sitesdir) diff --git a/frappe_manager/migration_manager/migrations/__init__.py b/frappe_manager/migration_manager/migrations/__init__.py new file mode 100644 index 00000000..6ef18f7e --- /dev/null +++ b/frappe_manager/migration_manager/migrations/__init__.py @@ -0,0 +1 @@ +# from .migrate_0_8_4 import MigrationV084 diff --git a/frappe_manager/migration_manager/migrations/migrate_0_10_0.py b/frappe_manager/migration_manager/migrations/migrate_0_10_0.py new file mode 100644 index 00000000..e36589eb --- /dev/null +++ b/frappe_manager/migration_manager/migrations/migrate_0_10_0.py @@ -0,0 +1,514 @@ +from sys import exception +from typing import Optional +from dataclasses import dataclass +import shutil + +from copy import deepcopy + +import importlib +from frappe_manager.docker_wrapper import DockerException +from frappe_manager.migration_manager.backup_manager import BackupData +from frappe_manager.migration_manager.migration_base import MigrationBase +from frappe_manager import CLI_DIR +from frappe_manager.migration_manager.migration_exections import MigrationExceptionInSite +from frappe_manager.migration_manager.migration_executor import MigrationExecutor +from frappe_manager.services_manager.services import ServicesManager +from frappe_manager.site_manager.site_exceptions import ( + SiteDatabaseAddUserException, + SiteDatabaseExport, + SiteDatabaseStartTimeout, +) +from frappe_manager.utils.docker import host_run_cp +from frappe_manager.site_manager.SiteManager import Site +from frappe_manager.site_manager.SiteManager import SiteManager +from frappe_manager.display_manager.DisplayManager import richprint +from frappe_manager.utils.helpers import get_container_name_prefix +from frappe_manager.migration_manager.version import Version +from pathlib import Path + +# from frappe_manager.display_manager.DisplayManager import richprint + +# @dataclass +# class MigrateSite: +# site_name : str +# exception: Optional[Exception] = None + +class MigrationV0100(MigrationBase): + version = Version("0.10.0") + + def __init__(self): + super().__init__() + self.sites_dir = CLI_DIR / "sites" + self.services_manager = ServicesManager(verbose=False) + + def set_migration_executor(self, migration_executor: MigrationExecutor): + self.migration_executor = migration_executor + + def up(self): + # richprint.print(f"Started",prefix=f"[ Migration v{str(self.version)} ] : ") + richprint.print(f"Started", prefix=f"[bold]v{str(self.version)}:[/bold] ") + self.logger.info("-" * 40) + + # take backup of each of the site docker compose + sites_manager = SiteManager(self.sites_dir) + sites_manager.stop_sites() + sites = sites_manager.get_all_sites() + + # create services + self.services_manager.init() + self.services_manager.entrypoint_checks() + self.services_manager.down(volumes=False) + self.services_manager.start(service="global-db") + + # migrate each site + main_error = False + + for site_name, site_path in sites.items(): + + site = Site(name=site_name, path=site_path.parent) + self.migration_executor.set_site_data(site) + try: + self.migrate_site(site) + except Exception as e: + import traceback + traceback_str = traceback.format_exc() + self.logger.error(f"[ EXCEPTION TRACEBACK ]:\n {traceback_str}") + richprint.update_live() + main_error = True + self.migration_executor.set_site_data(site,e) + self.undo_site_migrate(site) + site.down(volumes=False,timeout=5) + + if main_error: + raise MigrationExceptionInSite('') + + # new bind mount is introudced so create it + richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)}:[/bold] ") + self.logger.info("-" * 40) + + def migrate_site(self,site): + richprint.print(f"Migrating site {site.name}", prefix=f"[bold]v{str(self.version)}:[/bold] ") + + # backup docker compose.yml + self.backup_manager.backup( + site.path / "docker-compose.yml", site_name=site.name + ) + + # backup common_site_config.json + self.backup_manager.backup( + site.path + / "workspace" + / "frappe-bench" + / "sites" + / "common_site_config.json", + site_name=site.name, + ) + + site.down(volumes=False) + + self.migrate_site_compose(site) + + #recreate_env_cmd = 'python -m venv env && env/bin/python -m pip install --quiet --upgrade pip && env/bin/python -m pip install --quiet wheel' + # recreat env + # output = site.docker.compose.run( + # 'frappe', + # entrypoint=recreate_env_cmd, + # stream=True, + # stream_only_exit_code=True, + # ) + + site.down(volumes=False) + + site_db_info = site.get_site_db_info() + site_db_name = site_db_info['name'] + site_db_user = site_db_info['user'] + site_db_pass = site_db_info['password'] + + self.services_manager.add_user(site_db_name,site_db_user,site_db_pass) + + # service = 'mariadb' + # services_list = [] + # services_list.append(service) + # output = site.docker.compose.up(services=services_list,stream=True) + # richprint.live_lines(output, padding=(0, 0, 0, 2)) + + # site.add_user('mariadb','root','root',force=True) + + def down(self): + # richprint.print(f"Started",prefix=f"[ Migration v{str(self.version)} ][ROLLBACK] : ") + richprint.print( + f"Started", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] " + ) + self.logger.info("-" * 40) + + self.services_manager.down() + + if self.services_manager.exists(): + shutil.rmtree(self.services_manager.path) + + sites_manager = SiteManager(self.sites_dir) + sites = sites_manager.get_all_sites() + + # undo each site + for site, exception in self.migration_executor.migrate_sites.items(): + if not exception: + self.undo_site_migrate(site) + + for backup in self.backup_manager.backups: + self.backup_manager.restore(backup, force=True) + + richprint.print( + f"Successfull", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] " + ) + self.logger.info("-" * 40) + + def undo_site_migrate(self,site): + + for backup in self.backup_manager.backups: + if backup.site == site.name: + self.backup_manager.restore(backup, force=True) + + configs_backup = site.path / "configs.bak" + + configs_path = site.path / "configs" + + if configs_path.exists(): + shutil.rmtree(configs_path) + + if configs_backup.exists(): + shutil.copytree(configs_backup, configs_path) + + service = "mariadb" + services_list = [] + services_list.append(service) + + # start each site and forcefully add user to the site + try: + output = site.docker.compose.up(services=services_list, stream=True) + # output = site.docker.run(ser) + richprint.live_lines(output, padding=(0, 0, 0, 2)) + try: + site.add_user("mariadb", "root", "root", force=True) + except SiteDatabaseAddUserException as e: + pass + site.down(volumes=False) + except DockerException as e: + pass + + self.logger.info(f'Undo successfull for site: {site.name}') + + def migrate_site_compose(self, site: Site): + + richprint.change_head('Migrating database') + compose_version = site.composefile.get_version() + fm_version = importlib.metadata.version("frappe-manager") + + if not site.composefile.exists(): + richprint.print( + f"{status_msg} {compose_version} -> {fm_version}: Failed " + ) + return + + # export db + db_backup_file = self.db_migration_export(site) + + # backup site_db + db_backup = self.backup_manager.backup(db_backup_file, site_name=site.name,allow_restore=False) + + self.db_migration_import(site=site, db_backup_file=db_backup) + + status_msg = 'Migrating site compose' + richprint.change_head(status_msg) + + # get all the payloads + envs = site.composefile.get_all_envs() + labels = site.composefile.get_all_labels() + + # introduced in v0.10.0 + if not "ENVIRONMENT" in envs["frappe"]: + envs["frappe"]["ENVIRONMENT"] = "dev" + + envs["frappe"]["CONTAINER_NAME_PREFIX"] = get_container_name_prefix(site.name) + envs["frappe"]["MARIADB_ROOT_PASS"] = "root" + envs["frappe"]["MARIADB_HOST"] = "global-db" + + envs["nginx"]["VIRTUAL_HOST"] = site.name + + envs["adminer"] ={"ADMINER_DEFAULT_SERVER":"global-db"} + + import os + + envs_user_info = {} + userid_groupid: dict = {"USERID": os.getuid(), "USERGROUP": os.getgid()} + + env_user_info_container_list = ["frappe", "schedule", "socketio"] + + for env in env_user_info_container_list: + envs_user_info[env] = deepcopy(userid_groupid) + + # overwrite user for each invocation + users = {"nginx": {"uid": os.getuid(), "gid": os.getgid()}} + + self.create_compose_dirs(site) + + # load template + site.composefile.yml = site.composefile.load_template() + + # set all the payload + site.composefile.set_all_envs(envs) + site.composefile.set_all_envs(envs_user_info) + site.composefile.set_all_labels(labels) + site.composefile.set_all_users(users) + # site.composefile.set_all_extrahosts(extrahosts) + + site.composefile.remove_secrets_from_container("frappe") + site.composefile.remove_root_secrets_compose() + site.composefile.set_network_alias("nginx", "site-network", [site.name]) + site.composefile.set_container_names(get_container_name_prefix(site.name)) + + fm_version = importlib.metadata.version("frappe-manager") + + site.composefile.set_version(fm_version) + site.composefile.set_top_networks_name( + "site-network", get_container_name_prefix(site.name) + ) + site.composefile.write_to_file() + + # change the node socketio port + site.common_site_config_set({"socketio_port":"80"}) + + richprint.print( + f"{status_msg} {compose_version} -> {fm_version}: Done" + ) + + return db_backup + + def create_compose_dirs(self, site): + #### directory creation + configs_path = site.path / "configs" + + # custom config directory found moving it + # check if config directory exits if exists then move it + if configs_path.exists(): + shutil.move(configs_path, configs_path.parent / f"{configs_path.name}.bak") + + configs_path.mkdir(parents=True, exist_ok=True) + + # create nginx dirs + nginx_dir = configs_path / "nginx" + nginx_dir.mkdir(parents=True, exist_ok=True) + + nginx_poluate_dir = ["conf"] + + nginx_image = site.composefile.yml["services"]["nginx"]["image"] + + for directory in nginx_poluate_dir: + new_dir = nginx_dir / directory + if not new_dir.exists(): + new_dir_abs = str(new_dir.absolute()) + host_run_cp( + nginx_image, + source="/etc/nginx", + destination=new_dir_abs, + docker=site.docker, + ) + + # raise Exception("Migration not implemented") + + nginx_subdirs = ["logs", "cache", "run"] + + for directory in nginx_subdirs: + new_dir = nginx_dir / directory + new_dir.mkdir(parents=True, exist_ok=True) + + def is_site_database_started(self, site, service="mariadb", interval=5, timeout=60): + import time + + i = 0 + db_host = "127.0.0.1" + db_user = "root" + db_password = "root" + + check_connection_command = f"/usr/bin/mariadb -h{db_host} -u{db_user} -p'{db_password}' -e 'SHOW DATABASES;'" + connected = False + error = None + while i < timeout: + try: + time.sleep(interval) + output = site.docker.compose.exec( + service, + command=check_connection_command, + stream=True, + stream_only_exit_code=True, + ) + connected = True + break + except DockerException as e: + self.logger.error(f"[db start check] try: {i} got exception {e}") + error = e + pass + i += 1 + + if not connected: + raise SiteDatabaseStartTimeout(site.name, f"Not able to start db: {error}") + + def db_migration_export(self, site) -> Path: + self.logger.debug("[db export] site: %s", site.name) + try: + # if site.composefile.exists(): + + # DB MIGRATION if version < 1 and site.config exits + # start the site + output = site.docker.compose.up( + services=["mariadb", "frappe"], detach=True, pull="missing", stream=True + ) + richprint.live_lines(output, padding=(0, 0, 0, 2)) + + self.logger.debug("[db export] checking if mariadb started") + + self.is_site_database_started(site) + + # create dir to store migration + db_migration_dir_path = site.path / "workspace" / "migrations" + db_migration_dir_path.mkdir(exist_ok=True) + + from datetime import datetime + + current_datetime = datetime.now() + formatted_date = current_datetime.strftime("%d-%m-%Y--%H-%M-%S") + + db_migration_file_path = ( + f"/workspace/migrations/db-{site.name}-{formatted_date}.sql" + ) + + site_db_info = site.get_site_db_info() + site_db_name = site_db_info["name"] + # site_db_user = site_db_info['user'] + # site_db_pass = site_db_info['password'] + + db_backup_command = f"mysqldump -uroot -proot -h'mariadb' -P3306 {site_db_name} --result-file={db_migration_file_path}" # db_backup_command = f"mysqldump -uroot -proot -h'mariadb' -p3306 {site_db_name} {db_migration_file_path}" + # db_backup_command = f"/opt/.pyenv/shims/bench --site {site.name} backup --backup-path-db {db_migration_file_path}" + + # backup the db + output_backup_db = site.docker.compose.exec( + "frappe", + command=db_backup_command, + stream=True, + workdir="/workspace/frappe-bench", + user="frappe", + stream_only_exit_code=True, + ) + + # gunzip_command = f"gunzip {db_migration_file_path}" + + # output_gunzip = site.docker.compose.exec( + # "frappe", + # command=gunzip_command, + # stream=True, + # workdir='/workspace/migrations', + # user='frappe', + # stream_only_exit_code=True + # ) + + output_stop = site.docker.compose.stop(timeout=10, stream=True) + + site_db_migration_file_path = Path(site.path / db_migration_file_path[1:]) + + return site_db_migration_file_path + + except Exception as e: + raise SiteDatabaseExport(site.name, f"Error while exporting db: {e}") + + def db_migration_import(self, site: Site, db_backup_file: BackupData): + # cp into the global contianer + self.services_manager.docker.compose.cp( + source=str(db_backup_file.src.absolute()), + destination=f"global-db:/tmp/{db_backup_file.src.name}", + stream=True, + stream_only_exit_code=True, + ) + + services_db_info = self.services_manager.get_database_info() + services_db_user = services_db_info["user"] + services_db_pass = services_db_info["password"] + services_db_host = "127.0.0.1" + + site_db_info = site.get_site_db_info() + site_db_name = site_db_info["name"] + site_db_user = site_db_info["user"] + site_db_pass = site_db_info["password"] + + mariadb_command = f"/usr/bin/mariadb -u{services_db_user} -p'{services_db_pass}' -h'{services_db_host}' -P3306 -e " + mariadb = f"/usr/bin/mariadb -u{services_db_user} -p'{services_db_pass}' -h'{services_db_host}' -P3306" + + + + db_add_database = mariadb_command + f"'CREATE DATABASE IF NOT EXISTS `{site_db_name}`';" + + output_add_db = self.services_manager.docker.compose.exec( + "global-db", + command=db_add_database, + stream=True, + stream_only_exit_code=True, + ) + + + db_remove_user = mariadb_command + f"'DROP USER `{site_db_user}`@`%`;'" + + error = None + removed = False + try: + output = self.services_manager.docker.compose.exec( + "global-db", + command=db_remove_user, + stream=True, + stream_only_exit_code=True, + ) + except DockerException as e: + error = e + removed = False + + if "error 1396" in str(e.stderr).lower(): + removed = True + + if removed: + + db_add_user = mariadb_command + f"'CREATE USER `{site_db_user}`@`%` IDENTIFIED BY \"{site_db_pass}\";'" + + output_add_user_db = self.services_manager.docker.compose.exec( + "global-db", + command=db_add_user, + stream=True, + stream_only_exit_code=True, + ) + + db_grant_user = mariadb_command + f"'GRANT ALL PRIVILEGES ON `{site_db_name}`.* TO `{site_db_user}`@`%`;'" + + output_grant_user_db = self.services_manager.docker.compose.exec( + "global-db", + command=db_grant_user, + stream=True, + stream_only_exit_code=True, + ) + + db_import_command = mariadb + f" {site_db_name} -e 'source /tmp/{db_backup_file.src.name}'" + + output_import_db = self.services_manager.docker.compose.exec( + "global-db", + command=db_import_command, + stream=True, + stream_only_exit_code=True, + ) + + check_connection_command = mariadb_command + f"'SHOW DATABASES;'" + + output_check_db = self.services_manager.docker.compose.exec( + "global-db", + command=check_connection_command, + stream=True, + stream_only_exit_code=True, + ) + else: + raise SiteDatabaseAddUserException( + site.name, f"Database user creation failed: {error}" + ) diff --git a/frappe_manager/migration_manager/migrations/migrate_0_9_0.py b/frappe_manager/migration_manager/migrations/migrate_0_9_0.py new file mode 100644 index 00000000..ad2a0dab --- /dev/null +++ b/frappe_manager/migration_manager/migrations/migrate_0_9_0.py @@ -0,0 +1,91 @@ +import shutil +from frappe_manager.migration_manager.migration_base import MigrationBase + +from frappe_manager import CLI_DIR +from frappe_manager.site_manager.SiteManager import SiteManager +from frappe_manager.migration_manager.version import Version +from frappe_manager.display_manager.DisplayManager import richprint +from frappe_manager.migration_manager.migration_executor import MigrationExecutor + +class MigrationV090(MigrationBase): + + version = Version("0.9.0") + + def __init__(self): + super().__init__() + self.sitesdir = CLI_DIR / "sites" + + if self.sitesdir.exists(): + self.skip = True + + def set_migration_executor(self, migration_executor: MigrationExecutor): + self.migration_executor = migration_executor + + def up(self): + if self.skip: + return True + + richprint.print(f"Started",prefix=f"[bold]v{str(self.version)}:[/bold] ") + self.logger.info("-" * 40) + + move_directory_list = [] + for site_dir in CLI_DIR.iterdir(): + + if site_dir.is_dir(): + docker_compose_path = site_dir / "docker-compose.yml" + if docker_compose_path.exists(): + move_directory_list.append(site_dir) + + # stop all the sites + self.sitesdir.mkdir(parents=True, exist_ok=True) + sites_mananger = SiteManager(CLI_DIR) + sites_mananger.stop_sites() + + # move all the directories + richprint.print(f"Moving sites from {CLI_DIR} to {self.sitesdir}",prefix=f"[bold]v{str(self.version)}:[/bold] ") + + for site in move_directory_list: + site_name = site.parts[-1] + new_path = self.sitesdir / site_name + shutil.move(site, new_path) + self.logger.debug(f"Moved:{site.exists()}") + + richprint.print(f"Successfull",prefix=f"[bold]v{str(self.version)}:[/bold] ") + self.logger.info(f"[{self.version}] : Migration starting") + self.logger.info("-" * 40) + + def down(self): + if self.skip: + return True + + # richprint.print(f"Started",prefix=f"[ Migration v{str(self.version)} ][ROLLBACK] : ") + richprint.print(f"Started",prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] ") + self.logger.info("-" * 40) + + if self.sitesdir.exists(): + richprint.print(f"Found sites directory change.",prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] ") + + move_directory_list = [] + for site_dir in self.sitesdir.iterdir(): + + if site_dir.is_dir(): + docker_compose_path = site_dir / "docker-compose.yml" + + if docker_compose_path.exists(): + move_directory_list.append(site_dir) + + # stop all the sites + sites_mananger = SiteManager(self.sitesdir) + sites_mananger.stop_sites() + + # move all the directories + for site in move_directory_list: + site_name = site.parts[-1] + new_path = self.sitesdir.parent / site_name + shutil.move(site, new_path) + + # delete the sitedir + shutil.rmtree(self.sitesdir) + + richprint.print(f"Successfull",prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] ") + self.logger.info("-" * 40) diff --git a/frappe_manager/migration_manager/version.py b/frappe_manager/migration_manager/version.py new file mode 100644 index 00000000..b848713e --- /dev/null +++ b/frappe_manager/migration_manager/version.py @@ -0,0 +1,28 @@ +from dataclasses import dataclass +from functools import total_ordering + +@total_ordering +@dataclass +class Version: + version: str + + def __post_init__(self): + self.version_parts = list(map(int, self.version.split('.'))) + + def __lt__(self, other): + if not isinstance(other, Version): + return NotImplemented + return self.version_parts < other.version_parts + + def __eq__(self, other): + if not isinstance(other, Version): + return NotImplemented + return self.version_parts == other.version_parts + + def __gt__(self, other): + if not isinstance(other, Version): + return NotImplemented + return self.version_parts > other.version_parts + + def __str__(self): + return self.version diff --git a/frappe_manager/toml_manager.py b/frappe_manager/toml_manager.py new file mode 100644 index 00000000..7c864bb6 --- /dev/null +++ b/frappe_manager/toml_manager.py @@ -0,0 +1,37 @@ +import importlib.resources as pkg_resources + +from pathlib import Path +from typing import Optional, Union +from tomlkit import comment, document, dumps, loads, table, toml_document + + +class TomlManager: + def __init__(self, config_file: Path, template: toml_document.TOMLDocument = document()): + self.toml_file = config_file + self.template = template + self.toml: Union[Optional[toml_document.TOMLDocument],dict] = None + + def load(self): + + if not self.toml_file.exists(): + self.toml = self.template + return + + with open(self.toml_file, 'r') as f: + self.toml = loads(f.read()) + + def save(self): + with open(self.toml_file, 'w') as f: + f.write(dumps(self.toml)) + + def get(self, key): + return self.toml[key] + + def set(self, key, value): + self.toml[key] = value + + def get_all(self): + return self.toml + + def set_all(self, toml): + self.toml = toml diff --git a/poetry.lock b/poetry.lock index e7299a08..a9dac53d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -351,6 +351,17 @@ files = [ {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, ] +[[package]] +name = "tomlkit" +version = "0.12.3" +description = "Style preserving TOML library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomlkit-0.12.3-py3-none-any.whl", hash = "sha256:b0a645a9156dc7cb5d3a1f0d4bab66db287fcb8e0430bdd4664a095ea16414ba"}, + {file = "tomlkit-0.12.3.tar.gz", hash = "sha256:75baf5012d06501f07bee5bf8e801b9f343e7aac5a92581f20f80ce632e6b5a4"}, +] + [[package]] name = "typer" version = "0.9.0" @@ -405,4 +416,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "2c35b61a8ad02e0fb3b4327845b8f8ba44abd8b830d92e81c98a13832ce85cf3" +content-hash = "3a5b6caeec0c15d7cb7be9f540ffa9b2dcf3b25e6ebbc6bda1dabb029006a54c" diff --git a/pyproject.toml b/pyproject.toml index 93430a0b..54cba801 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ typer = {extras = ["all"], version = "^0.9.0"} requests = "^2.31.0" psutil = "^5.9.6" ruamel-yaml = "^0.18.5" +tomlkit = "^0.12.3" [build-system] From 3381a25c4741955118d40b2fc70074c9e9ba0c5a Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 31 Jan 2024 14:20:53 +0530 Subject: [PATCH 086/100] misc: poetry inproject venv --- poetry.toml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 poetry.toml diff --git a/poetry.toml b/poetry.toml new file mode 100644 index 00000000..ab1033bd --- /dev/null +++ b/poetry.toml @@ -0,0 +1,2 @@ +[virtualenvs] +in-project = true From 04842db7909db21965acd00bbd812716fcc8b010 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 31 Jan 2024 14:28:39 +0530 Subject: [PATCH 087/100] bump: v0.10.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 54cba801..67554faf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "frappe-manager" -version = "0.9.0" +version = "0.10.0" license = "MIT" repository = "https://github.com/rtcamp/frappe-manager" description = "A CLI tool based on Docker Compose to easily manage Frappe based projects. As of now, only suitable for development in local machines running on Mac and Linux based OS." From c93bdc962018dbf8165e0d28b895b5b3a954772a Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 31 Jan 2024 19:17:40 +0530 Subject: [PATCH 088/100] fix bench restart in frappe container --- Docker/frappe/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/frappe/Dockerfile b/Docker/frappe/Dockerfile index fa1f9354..ad6d3d7a 100644 --- a/Docker/frappe/Dockerfile +++ b/Docker/frappe/Dockerfile @@ -141,7 +141,7 @@ WORKDIR /workspace RUN mkdir -p /opt/user/.bin ENV PATH /opt/user/.bin:${PATH} -RUN echo PATH='/opt/user/.bin:$PATH' >> "$USERZSHRC" +RUN echo 'export PATH="/opt/user/.bin:$PATH"' >> "$USERZSHRC" COPY ./supervisord.conf /opt/user/ COPY ./bench-dev-server /opt/user/ From fdeb3a827975806ac953628858133d5d5d10a3e8 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 31 Jan 2024 19:20:14 +0530 Subject: [PATCH 089/100] turn off fm pypi update check --- frappe_manager/main.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frappe_manager/main.py b/frappe_manager/main.py index abc96a37..3e812c95 100644 --- a/frappe_manager/main.py +++ b/frappe_manager/main.py @@ -21,7 +21,5 @@ def exit_cleanup(): This function is used to perform cleanup at the exit. """ remove_zombie_subprocess_process(process_opened) - check_update() - print('') + #check_update() richprint.stop() - From d04faa4f91693e97d30529986c6e9c0925223f57 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 31 Jan 2024 19:22:28 +0530 Subject: [PATCH 090/100] services rollback if failure --- frappe_manager/commands.py | 8 +- .../migrations/migrate_0_10_0.py | 23 +++--- frappe_manager/services_manager/commands.py | 2 +- frappe_manager/services_manager/services.py | 78 +++++++++---------- .../services_manager/services_exceptions.py | 10 +++ frappe_manager/site_manager/site.py | 3 +- .../workers_manager/SiteWorker.py | 4 +- .../templates/docker-compose.services.tmpl | 5 +- 8 files changed, 71 insertions(+), 62 deletions(-) diff --git a/frappe_manager/commands.py b/frappe_manager/commands.py index 2216d300..b858c46d 100644 --- a/frappe_manager/commands.py +++ b/frappe_manager/commands.py @@ -5,6 +5,7 @@ import sys import shutil from typing import Annotated, List, Optional, Set +from frappe_manager.services_manager.services_exceptions import ServicesNotCreated from frappe_manager.site_manager.SiteManager import SiteManager from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager import CLI_DIR, default_extension, SiteServicesEnum, services_manager @@ -12,6 +13,7 @@ from frappe_manager.docker_wrapper import DockerClient from frappe_manager.services_manager.services import ServicesManager from frappe_manager.migration_manager.migration_executor import MigrationExecutor +from frappe_manager.site_manager.site_exceptions import SiteException from frappe_manager.utils.callbacks import apps_list_validation_callback, frappe_branch_validation_callback, version_callback from frappe_manager.utils.helpers import get_container_name_prefix, is_cli_help_called from frappe_manager.services_manager.commands import services_app @@ -79,7 +81,11 @@ def app_callback( global services_manager services_manager = ServicesManager(verbose=verbose) services_manager.init() - services_manager.entrypoint_checks() + try: + services_manager.entrypoint_checks() + except ServicesNotCreated as e: + services_manager.remove_itself() + richprint.exit(f"Not able to create services. {e}") if not services_manager.running(): services_manager.start() diff --git a/frappe_manager/migration_manager/migrations/migrate_0_10_0.py b/frappe_manager/migration_manager/migrations/migrate_0_10_0.py index e36589eb..4bbe90ad 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_10_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_10_0.py @@ -9,7 +9,7 @@ from frappe_manager.docker_wrapper import DockerException from frappe_manager.migration_manager.backup_manager import BackupData from frappe_manager.migration_manager.migration_base import MigrationBase -from frappe_manager import CLI_DIR +from frappe_manager import CLI_DIR, services_manager from frappe_manager.migration_manager.migration_exections import MigrationExceptionInSite from frappe_manager.migration_manager.migration_executor import MigrationExecutor from frappe_manager.services_manager.services import ServicesManager @@ -143,8 +143,8 @@ def down(self): self.services_manager.down() - if self.services_manager.exists(): - shutil.rmtree(self.services_manager.path) + if self.services_manager.path.exists(): + self.services_manager.remove_itself() sites_manager = SiteManager(self.sites_dir) sites = sites_manager.get_all_sites() @@ -321,21 +321,18 @@ def create_compose_dirs(self, site): new_dir = nginx_dir / directory new_dir.mkdir(parents=True, exist_ok=True) - def is_site_database_started(self, site, service="mariadb", interval=5, timeout=60): + def is_database_started(self, docker_object, db_user='root', db_password='root',db_host='127.0.0.1', service="mariadb", interval=5, timeout=30): import time - i = 0 - db_host = "127.0.0.1" - db_user = "root" - db_password = "root" check_connection_command = f"/usr/bin/mariadb -h{db_host} -u{db_user} -p'{db_password}' -e 'SHOW DATABASES;'" connected = False error = None + while i < timeout: try: time.sleep(interval) - output = site.docker.compose.exec( + output = docker_object.compose.exec( service, command=check_connection_command, stream=True, @@ -366,7 +363,7 @@ def db_migration_export(self, site) -> Path: self.logger.debug("[db export] checking if mariadb started") - self.is_site_database_started(site) + self.is_database_started(site.docker) # create dir to store migration db_migration_dir_path = site.path / "workspace" / "migrations" @@ -420,6 +417,9 @@ def db_migration_export(self, site) -> Path: raise SiteDatabaseExport(site.name, f"Error while exporting db: {e}") def db_migration_import(self, site: Site, db_backup_file: BackupData): + + self.logger.info(f"[database import: global-db] {site.name} -> {db_backup_file}") + # cp into the global contianer self.services_manager.docker.compose.cp( source=str(db_backup_file.src.absolute()), @@ -441,7 +441,7 @@ def db_migration_import(self, site: Site, db_backup_file: BackupData): mariadb_command = f"/usr/bin/mariadb -u{services_db_user} -p'{services_db_pass}' -h'{services_db_host}' -P3306 -e " mariadb = f"/usr/bin/mariadb -u{services_db_user} -p'{services_db_pass}' -h'{services_db_host}' -P3306" - + self.is_database_started(self.services_manager.docker, service='global-db',db_user=services_db_user,db_password=services_db_pass) db_add_database = mariadb_command + f"'CREATE DATABASE IF NOT EXISTS `{site_db_name}`';" @@ -512,3 +512,4 @@ def db_migration_import(self, site: Site, db_backup_file: BackupData): raise SiteDatabaseAddUserException( site.name, f"Database user creation failed: {error}" ) + diff --git a/frappe_manager/services_manager/commands.py b/frappe_manager/services_manager/commands.py index 6a116b3a..cf1ec1eb 100644 --- a/frappe_manager/services_manager/commands.py +++ b/frappe_manager/services_manager/commands.py @@ -76,6 +76,6 @@ def shell( user: Annotated[str, typer.Option(help="Connect as this user.")] = None, ): """ - Open shell for the specificed global services_manager. + Open shell for the specificed global service. """ services_manager.shell(service_name.value, users) diff --git a/frappe_manager/services_manager/services.py b/frappe_manager/services_manager/services.py index 6a6eebbb..ed24eddd 100644 --- a/frappe_manager/services_manager/services.py +++ b/frappe_manager/services_manager/services.py @@ -8,7 +8,7 @@ from typing import Optional from frappe_manager import CLI_DIR -from frappe_manager.services_manager.services_exceptions import ServicesComposeNotExist, ServicesDBNotStart +from frappe_manager.services_manager.services_exceptions import ServicesComposeNotExist, ServicesDBNotStart, ServicesException, ServicesNotCreated from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.compose_manager.ComposeFile import ComposeFile from frappe_manager.utils.helpers import ( @@ -47,19 +47,20 @@ def set_typer_context(self, ctx: typer.Context): def entrypoint_checks(self, start = False): if not self.path.exists(): - richprint.print(f"Creating services",emoji_code=":construction:") - self.path.mkdir(parents=True, exist_ok=True) - self.create() + try: + richprint.print(f"Creating services",emoji_code=":construction:") + self.path.mkdir(parents=True, exist_ok=True) + self.create() + except Exception as e: + raise ServicesNotCreated(f'Error Caused: {e}') + self.pull() richprint.print(f"Creating services: Done") if start: self.start() if not self.compose_path.exists(): - raise ServicesComposeNotExist("Seems like services has taken a down. Please recreate services.") - # richprint.exit( - # "Seems like global services has taken a down. Please recreate global services." - # ) + raise ServicesComposeNotExist(f"Seems like services has taken a down. Compose file not found at -> {self.compose_path}. Please recreate services.") if start: if not self.typer_context.invoked_subcommand == "service": @@ -70,10 +71,16 @@ def entrypoint_checks(self, start = False): def init(self): # check if the global services exits if not then create + current_system = platform.system() self.composefile = ComposeFile( self.compose_path, template_name="docker-compose.services.tmpl" ) + if current_system == "Darwin": + self.composefile = ComposeFile( + self.compose_path, template_name="docker-compose.services.osx.tmpl" + ) + self.docker = DockerClient(compose_file_path=self.composefile.compose_path) @@ -89,6 +96,7 @@ def create(self, backup=False): current_system = platform.system() + inputs = {"environment": envs} try: user = { @@ -96,16 +104,17 @@ def create(self, backup=False): "uid": os.getuid(), "gid": os.getgid(), }} + if not current_system == "Darwin": - user["global-nginx-proxy"]: { + user["global-nginx-proxy"] = { "uid": os.getuid(), "gid": get_unix_groups()["docker"], } + inputs["user"]= user except KeyError: - richprint.exit("docker group not found in system.") + raise ServicesException("docker group not found in system. Please add docker group to the system and current user to the docker group.") - inputs = {"environment": envs, "user": user} if backup: if self.path.exists(): @@ -135,6 +144,12 @@ def create(self, backup=False): ] #"mariadb/data", + if current_system == "Darwin": + self.composefile.remove_container_user('global-nginx-proxy') + self.composefile.remove_container_user('global-db') + else: + dirs_to_create.append("mariadb/data") + # create dirs for folder in dirs_to_create: temp_dir = self.path / folder @@ -168,9 +183,6 @@ def create(self, backup=False): # set secrets in compose self.generate_compose(inputs) - if current_system == "Darwin": - self.composefile.remove_container_user('global-nginx-proxy') - self.composefile.remove_container_user('global-db') self.composefile.set_secret_file_path('db_password',str(db_password_path.absolute())) self.composefile.set_secret_file_path('db_root_password',str(db_root_password_path.absolute())) @@ -181,21 +193,18 @@ def get_database_info(self): Provides info about databse """ info: dict = {} + info["user"] = "root" + info["host"] = "global-db" + info["port"] = 3306 try: password_path = self.composefile.get_secret_file_path('db_root_password') with open(Path(password_path),'r') as f: password = f.read() info["password"] = password - info["user"] = "root" - info["host"] = "global-db" - info["port"] = 3306 return info except KeyError as e: # TODO secrets not exists info["password"] = None - info["user"] = "root" - info["host"] = "global-db" - info["port"] = 3306 return info def exists(self): @@ -204,12 +213,14 @@ def exists(self): def generate_compose(self, inputs: dict): """ This can get a file like + inputs = { "environment" : {'key': 'value'}, "extrahosts" : {'key': 'value'}, "user" : {'uid': 'value','gid': 'value'}, "labels" : {'key': 'value'}, } + """ try: # handle envrionment @@ -497,31 +508,9 @@ def add_user(self, site_db_name, site_db_user, site_db_pass, timeout = 25): remove_db_user = f"/usr/bin/mariadb -P3306 -h{db_host} -u{db_user} -p'{db_password}' -e 'DROP USER `{site_db_user}`@`%`;'" add_db_user = f"/usr/bin/mariadb -h{db_host} -P3306 -u{db_user} -p'{db_password}' -e 'CREATE USER `{site_db_user}`@`%` IDENTIFIED BY \"{site_db_pass}\";'" grant_user = f"/usr/bin/mariadb -h{db_host} -P3306 -u{db_user} -p'{db_password}' -e 'GRANT ALL PRIVILEGES ON `{site_db_name}`.* TO `{site_db_user}`@`%`;'" - # SHOW_db_user= f"/usr/bin/mariadb -P3306-h{db_host} -u{db_user} -p'{db_password}' -e 'SELECT User, Host FROM mysql.user;'" -# - # import time; - # check_connection_command = f"/usr/bin/mariadb -h{db_host} -u{db_user} -p'{db_password}' -e 'SHOW DATABASES;'" - - # i = 0 - # connected = False - - # error = None - # while i < timeout: - # try: - # time.sleep(5) - # output = self.docker.compose.exec('global-db', command=check_connection_command, stream=self.quiet, stream_only_exit_code=True) - # if next(output) == 0: - # connected = True - # except DockerException as e: - # error = e - # pass - - # i += 1 - - # if not connected: - # raise ServicesDBNotStart(f"DB did not start: {error}") removed = True + try: output = self.docker.compose.exec('global-db', command=remove_db_user, stream=self.quiet,stream_only_exit_code=True) except DockerException as e: @@ -536,3 +525,6 @@ def add_user(self, site_db_name, site_db_user, site_db_pass, timeout = 25): richprint.print(f"Recreated user {site_db_user}") except DockerException as e: raise ServicesDBNotStart(f"Database user creation failed: {e}") + + def remove_itself(self): + shutil.rmtree(self.path) diff --git a/frappe_manager/services_manager/services_exceptions.py b/frappe_manager/services_manager/services_exceptions.py index 93b84bb1..c0f2f6ce 100644 --- a/frappe_manager/services_manager/services_exceptions.py +++ b/frappe_manager/services_manager/services_exceptions.py @@ -13,3 +13,13 @@ class ServicesDBNotStart(Exception): def __init__(self, message): message = message super().__init__(message) + +class ServicesException(Exception): + def __init__(self, message): + message = message + super().__init__(message) + +class ServicesNotCreated(ServicesException): + def __init__(self, message): + message = message + super().__init__(message) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 79b5a9be..941779b0 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -565,6 +565,7 @@ def running(self) -> bool: return True def get_services_running_status(self) -> dict: + services = self.composefile.get_services_list() containers = self.composefile.get_container_names().values() services_status = {} @@ -587,7 +588,7 @@ def get_services_running_status(self) -> dict: services_status[container["Service"]] = container["State"] return services_status except DockerException as e: - richprint.exit(f"{e.stdout}{e.stderr}") + return {} def get_host_port_binds(self): try: diff --git a/frappe_manager/site_manager/workers_manager/SiteWorker.py b/frappe_manager/site_manager/workers_manager/SiteWorker.py index 741d7463..d19d16c3 100644 --- a/frappe_manager/site_manager/workers_manager/SiteWorker.py +++ b/frappe_manager/site_manager/workers_manager/SiteWorker.py @@ -130,8 +130,10 @@ def stop(self) -> bool: richprint.warning(f"{status_text}: Failed") def get_services_running_status(self) -> dict: + services = self.composefile.get_services_list() containers = self.composefile.get_container_names().values() + services_status = {} try: output = self.docker.compose.ps( @@ -152,7 +154,7 @@ def get_services_running_status(self) -> dict: services_status[container["Service"]] = container["State"] return services_status except DockerException as e: - richprint.exit(f"{e.stdout}{e.stderr}") + return {} def running(self) -> bool: diff --git a/frappe_manager/templates/docker-compose.services.tmpl b/frappe_manager/templates/docker-compose.services.tmpl index 6e429d9a..04c09388 100644 --- a/frappe_manager/templates/docker-compose.services.tmpl +++ b/frappe_manager/templates/docker-compose.services.tmpl @@ -16,7 +16,7 @@ services: MYSQL_USER: REPLACE_WITH_DB_USER MYSQL_PASSWORD_FILE: REPLACE_WITH_DB_PASSWORD_SECRET_FILE volumes: - - fm-global-db-data:/var/lib/mysql + - ./mariadb/data:/var/lib/mysql - ./mariadb/conf:/etc/mysql - ./mariadb/logs:/var/log/mysql networks: @@ -64,6 +64,3 @@ secrets: file: REPLACE_ME_WITH_DB_PASSWORD_TXT_PATH db_root_password: file: REPLACE_ME_WITH_DB_ROOT_PASSWORD_TXT_PATH - -volumes: - fm-global-db-data: From 57b0670d003f6bfd187752528e657590d069998e Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 31 Jan 2024 19:22:48 +0530 Subject: [PATCH 091/100] emoji change for service runnning status --- frappe_manager/site_manager/SiteManager.py | 9 +++++---- frappe_manager/utils/site.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/frappe_manager/site_manager/SiteManager.py b/frappe_manager/site_manager/SiteManager.py index 149ed00d..bf3cf14d 100644 --- a/frappe_manager/site_manager/SiteManager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -423,6 +423,7 @@ def info(self): site_info_table.add_row("Bench Apps", bench_apps_list_table) running_site_services = self.site.get_services_running_status() + running_site_workers = self.site.workers.get_services_running_status() if running_site_services: @@ -434,10 +435,10 @@ def info(self): site_info_table.add_row("Site Workers", site_workers_table) richprint.stdout.print(site_info_table) - richprint.print( - f":green_square: -> Active :red_square: -> Inactive", - emoji_code=":information: ", - ) + # richprint.print( + # f":green_square: -> Active :red_square: -> Inactive", + # emoji_code=":information: ", + # ) richprint.print( f"Run 'fm list' to list all available sites.", emoji_code=":light_bulb:" ) diff --git a/frappe_manager/utils/site.py b/frappe_manager/utils/site.py index 2ba92ed9..99f8d7c7 100644 --- a/frappe_manager/utils/site.py +++ b/frappe_manager/utils/site.py @@ -49,7 +49,7 @@ def create_service_element(service, running_status): ) service_table.add_column("Service", justify="left", no_wrap=True) service_table.add_column("Status", justify="right", no_wrap=True) - service_status = ":green_square:" if running_status == "running" else ":red_square:" + service_status = "\u2713" if running_status == "running" else "\u2718" service_table.add_row( f"{service}", f"{service_status}", From e7ce5c8932190e02531dc41e79a88bf281becfe8 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 31 Jan 2024 19:26:00 +0530 Subject: [PATCH 092/100] fix services not working in linux --- .../docker-compose.services.osx.tmpl | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 frappe_manager/templates/docker-compose.services.osx.tmpl diff --git a/frappe_manager/templates/docker-compose.services.osx.tmpl b/frappe_manager/templates/docker-compose.services.osx.tmpl new file mode 100644 index 00000000..6e429d9a --- /dev/null +++ b/frappe_manager/templates/docker-compose.services.osx.tmpl @@ -0,0 +1,69 @@ +version: "3.9" +services: + global-db: + container_name: fm_global-db + image: mariadb:10.6 + user: REPLACE_WITH_CURRENT_USER:REPLACE_WITH_CURRENT_USER_GROUP + restart: always + command: + - --character-set-server=utf8mb4 + - --collation-server=utf8mb4_unicode_ci + - --skip-character-set-client-handshake + - --skip-innodb-read-only-compressed + environment: + MYSQL_ROOT_PASSWORD_FILE: REPLACE_WITH_DB_ROOT_PASSWORD_SECRET_FILE + MYSQL_DATABASE: REPLACE_WITH_DB_NAME + MYSQL_USER: REPLACE_WITH_DB_USER + MYSQL_PASSWORD_FILE: REPLACE_WITH_DB_PASSWORD_SECRET_FILE + volumes: + - fm-global-db-data:/var/lib/mysql + - ./mariadb/conf:/etc/mysql + - ./mariadb/logs:/var/log/mysql + networks: + - global-backend-network + secrets: + - db_password + - db_root_password + + global-nginx-proxy: + container_name: fm_global-nginx-proxy + user: REPLACE_WITH_CURRENT_USER:REPLACE_WITH_CURRENT_USER_GROUP + image: jwilder/nginx-proxy + ports: + - "80:80" + - "443:443" + restart: always + volumes: + - "./nginx-proxy/certs:/etc/nginx/certs" + - "./nginx-proxy/dhparam:/etc/nginx/dhparam" + - "./nginx-proxy/confd:/etc/nginx/conf.d" + - "./nginx-proxy/htpasswd:/etc/nginx/htpasswd" + - "./nginx-proxy/vhostd:/etc/nginx/vhost.d" + - "./nginx-proxy/html:/usr/share/nginx/html" + - "./nginx-proxy/logs:/var/log/nginx" + - "./nginx-proxy/run:/var/run" + - "./nginx-proxy/cache:/var/cache/nginx" + - "/var/run/docker.sock:/tmp/docker.sock:ro" + networks: + - global-frontend-network + +networks: + global-frontend-network: + name: fm-global-frontend-network + ipam: + config: + - subnet: '10.1.0.0/16' + global-backend-network: + name: fm-global-backend-network + ipam: + config: + - subnet: '10.2.0.0/16' + +secrets: + db_password: + file: REPLACE_ME_WITH_DB_PASSWORD_TXT_PATH + db_root_password: + file: REPLACE_ME_WITH_DB_ROOT_PASSWORD_TXT_PATH + +volumes: + fm-global-db-data: From fe5b3221fa782b7a2077373b4743fb462f5cc0b5 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Thu, 1 Feb 2024 20:12:17 +0530 Subject: [PATCH 093/100] fix: after migration choose archive, version update --- frappe_manager/migration_manager/migration_executor.py | 3 +-- frappe_manager/migration_manager/migrations/migrate_0_10_0.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/frappe_manager/migration_manager/migration_executor.py b/frappe_manager/migration_manager/migration_executor.py index 0ae50fa2..46687190 100644 --- a/frappe_manager/migration_manager/migration_executor.py +++ b/frappe_manager/migration_manager/migration_executor.py @@ -67,10 +67,8 @@ def execute(self): except Exception as e: print(f"Failed to register migration {name}: {e}") - #self.migrations.sort() self.migrations = sorted(self.migrations, key=lambda x: x.version) - # print info to user if self.migrations: richprint.print("Pending Migrations...") @@ -128,6 +126,7 @@ def execute(self): if archive: + self.prev_version = self.undo_stack[-1].version for site, exception in self.migrate_sites.items(): if exception: archive_site_path = CLI_SITES_ARCHIVE / site.name diff --git a/frappe_manager/migration_manager/migrations/migrate_0_10_0.py b/frappe_manager/migration_manager/migrations/migrate_0_10_0.py index 4bbe90ad..1d413f67 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_10_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_10_0.py @@ -347,7 +347,7 @@ def is_database_started(self, docker_object, db_user='root', db_password='root', i += 1 if not connected: - raise SiteDatabaseStartTimeout(site.name, f"Not able to start db: {error}") + raise SiteDatabaseStartTimeout(f"Not able to start db: {error}") def db_migration_export(self, site) -> Path: self.logger.debug("[db export] site: %s", site.name) From ec74dad5061a61ce69471ced2267974f4b3661c6 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Thu, 1 Feb 2024 20:13:03 +0530 Subject: [PATCH 094/100] fix: services not starting in osx --- frappe_manager/services_manager/services.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe_manager/services_manager/services.py b/frappe_manager/services_manager/services.py index ed24eddd..ec2405e7 100644 --- a/frappe_manager/services_manager/services.py +++ b/frappe_manager/services_manager/services.py @@ -72,10 +72,10 @@ def init(self): # check if the global services exits if not then create current_system = platform.system() + self.composefile = ComposeFile( self.compose_path, template_name="docker-compose.services.tmpl" ) - if current_system == "Darwin": self.composefile = ComposeFile( self.compose_path, template_name="docker-compose.services.osx.tmpl" @@ -83,7 +83,6 @@ def init(self): self.docker = DockerClient(compose_file_path=self.composefile.compose_path) - def create(self, backup=False): envs = { "global-db": { @@ -110,6 +109,7 @@ def create(self, backup=False): "uid": os.getuid(), "gid": get_unix_groups()["docker"], } + inputs["user"]= user except KeyError: @@ -144,6 +144,9 @@ def create(self, backup=False): ] #"mariadb/data", + # set secrets in compose + self.generate_compose(inputs) + if current_system == "Darwin": self.composefile.remove_container_user('global-nginx-proxy') self.composefile.remove_container_user('global-db') @@ -180,9 +183,6 @@ def create(self, backup=False): destination=mariadb_conf, docker=self.docker, ) - # set secrets in compose - self.generate_compose(inputs) - self.composefile.set_secret_file_path('db_password',str(db_password_path.absolute())) self.composefile.set_secret_file_path('db_root_password',str(db_root_password_path.absolute())) From 9ba59ca9d7f2f5d77b2d04623ba7a8672d2f655a Mon Sep 17 00:00:00 2001 From: Xieyt Date: Thu, 1 Feb 2024 20:13:17 +0530 Subject: [PATCH 095/100] add logs for site start in fm create command --- frappe_manager/site_manager/SiteManager.py | 1 - frappe_manager/site_manager/site.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe_manager/site_manager/SiteManager.py b/frappe_manager/site_manager/SiteManager.py index bf3cf14d..e027d9fc 100644 --- a/frappe_manager/site_manager/SiteManager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -141,7 +141,6 @@ def create_site(self, template_inputs: dict): self.typer_context.obj["logger"].info( f"SITE_STATUS {self.site.name}: WORKING" ) - richprint.print(f"Started site") self.info() diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 941779b0..67ca32c6 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -842,6 +842,8 @@ def remove_secrets(self): if running: self.start() + self.frappe_logs_till_start(status_msg='Starting Site') + richprint.print(f"Removing Secrets: Done") def remove_database_and_user(self): From c034b35892f8150eab78913d7bcf4f713e88ac42 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 5 Feb 2024 14:23:28 +0530 Subject: [PATCH 096/100] add ask for migration prompt --- .../migration_manager/migration_executor.py | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/frappe_manager/migration_manager/migration_executor.py b/frappe_manager/migration_manager/migration_executor.py index 46687190..d5ea14fb 100644 --- a/frappe_manager/migration_manager/migration_executor.py +++ b/frappe_manager/migration_manager/migration_executor.py @@ -72,8 +72,27 @@ def execute(self): if self.migrations: richprint.print("Pending Migrations...") - for migration in self.migrations: - richprint.print(f"[bold]MIGRATION:[/bold] v{migration.version}") + for migration in self.migrations: + richprint.print(f"[bold]MIGRATION:[/bold] v{migration.version}") + + richprint.print( + f"This may take some time.", emoji_code=":light_bulb:" + ) + + migrate_msg =( + "\nIF [y]: Start Migration." + "\nIF [N]: Don't migrate and revert to previous fm version." + "\nDo you want to migrate ?" + ) + # prompt + richprint.stop() + continue_migration = typer.confirm(migrate_msg) + + if not continue_migration: + downgrade_package('frappe-manager',str(self.prev_version.version)) + richprint.exit(f'Successfully installed [bold][blue]Frappe-Manager[/blue][/bold] version: v{str(self.prev_version.version)}',emoji_code=':white_check_mark:') + + richprint.start('Working') rollback = False archive = False From 8c90508d2e4bec4e92f1c14e80bacd570228bf3a Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 5 Feb 2024 14:24:05 +0530 Subject: [PATCH 097/100] remove commands hints --- frappe_manager/site_manager/SiteManager.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/frappe_manager/site_manager/SiteManager.py b/frappe_manager/site_manager/SiteManager.py index e027d9fc..34781d15 100644 --- a/frappe_manager/site_manager/SiteManager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -214,10 +214,6 @@ def list_sites(self): richprint.stop() richprint.stdout.print(list_table) - richprint.print( - f"Run 'fm info ' to get detail information about a site.", - emoji_code=":light_bulb:", - ) def stop_site(self): """ @@ -438,9 +434,6 @@ def info(self): # f":green_square: -> Active :red_square: -> Inactive", # emoji_code=":information: ", # ) - richprint.print( - f"Run 'fm list' to list all available sites.", emoji_code=":light_bulb:" - ) def migrate_site(self): """ From 542e1ad78b9af0f897666c4eb3a633d4a3c6a736 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 5 Feb 2024 14:24:21 +0530 Subject: [PATCH 098/100] enable restart always on all sites --- frappe_manager/templates/docker-compose.tmpl | 9 +++++++++ frappe_manager/templates/docker-compose.workers.tmpl | 1 + 2 files changed, 10 insertions(+) diff --git a/frappe_manager/templates/docker-compose.tmpl b/frappe_manager/templates/docker-compose.tmpl index 52293a30..03e16bcf 100644 --- a/frappe_manager/templates/docker-compose.tmpl +++ b/frappe_manager/templates/docker-compose.tmpl @@ -3,6 +3,7 @@ services: frappe: image: ghcr.io/rtcamp/frappe-manager-frappe:v0.10.0 container_name: REPLACE_ME_WITH_CONTAINER_NAME + restart: always environment: ADMIN_PASS: REPLACE_me_with_frappe_web_admin_pass # apps are defined as :, if branch name not given then default github branch will be used. @@ -32,6 +33,7 @@ services: image: ghcr.io/rtcamp/frappe-manager-nginx:v0.10.0 container_name: REPLACE_ME_WITH_CONTAINER_NAME user: REPLACE_ME_WITH_CURRENT_USER:REPLACE_ME_WITH_CURRENT_USER_GROUP + restart: always environment: # not implemented as of now ENABLE_SSL: REPLACE_ME_WITH_TOGGLE_ENABLE_SSL @@ -54,6 +56,7 @@ services: mailhog: image: ghcr.io/rtcamp/frappe-manager-mailhog:v0.8.3 container_name: REPLACE_ME_WITH_CONTAINER_NAME + restart: always expose: - 1025 - 8025 @@ -63,6 +66,7 @@ services: adminer: image: adminer:latest container_name: REPLACE_ME_WITH_CONTAINER_NAME + restart: always environment: ADMINER_DEFAULT_SERVER: global-db expose: @@ -74,6 +78,7 @@ services: socketio: image: ghcr.io/rtcamp/frappe-manager-frappe:v0.10.0 container_name: REPLACE_ME_WITH_CONTAINER_NAME + restart: always environment: TIMEOUT: 60000 CHANGE_DIR: /workspace/frappe-bench/logs @@ -92,6 +97,7 @@ services: schedule: image: ghcr.io/rtcamp/frappe-manager-frappe:v0.10.0 container_name: REPLACE_ME_WITH_CONTAINER_NAME + restart: always environment: TIMEOUT: 60000 CHANGE_DIR: /workspace/frappe-bench @@ -109,6 +115,7 @@ services: redis-cache: image: redis:alpine container_name: REPLACE_ME_WITH_CONTAINER_NAME + restart: always volumes: - redis-cache-data:/data expose: @@ -119,6 +126,7 @@ services: redis-queue: image: redis:alpine container_name: REPLACE_ME_WITH_CONTAINER_NAME + restart: always volumes: - redis-queue-data:/data expose: @@ -129,6 +137,7 @@ services: redis-socketio: image: redis:alpine container_name: REPLACE_ME_WITH_CONTAINER_NAME + restart: always volumes: - redis-socketio-data:/data expose: diff --git a/frappe_manager/templates/docker-compose.workers.tmpl b/frappe_manager/templates/docker-compose.workers.tmpl index 4eed9b67..4cdeaf16 100644 --- a/frappe_manager/templates/docker-compose.workers.tmpl +++ b/frappe_manager/templates/docker-compose.workers.tmpl @@ -1,6 +1,7 @@ services: worker-name: image: ghcr.io/rtcamp/frappe-manager-frappe:v0.10.0 + restart: always environment: TIMEOUT: 6000 CHANGE_DIR: /workspace/frappe-bench From 5e987650dbcc42432f73d2581b23d94c9febdafe Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 5 Feb 2024 17:17:35 +0530 Subject: [PATCH 099/100] more info for user when migration --- frappe_manager/migration_manager/migration_executor.py | 8 +++++++- frappe_manager/site_manager/site.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/frappe_manager/migration_manager/migration_executor.py b/frappe_manager/migration_manager/migration_executor.py index d5ea14fb..953f35e8 100644 --- a/frappe_manager/migration_manager/migration_executor.py +++ b/frappe_manager/migration_manager/migration_executor.py @@ -76,7 +76,11 @@ def execute(self): richprint.print(f"[bold]MIGRATION:[/bold] v{migration.version}") richprint.print( - f"This may take some time.", emoji_code=":light_bulb:" + "This may take some time.", emoji_code=":light_bulb:" + ) + + richprint.print( + "Manual migration guide can be found here -> https://github.com/rtCamp/Frappe-Manager/wiki/Migrations#manual-migration-procedure", emoji_code=":light_bulb:" ) migrate_msg =( @@ -127,6 +131,8 @@ def execute(self): richprint.print(f"[bold][red]SITE[/red]:[/bold] {site.name}") richprint.print(f"[bold][red]EXCEPTION[/red]:[/bold] {exception}") + richprint.print(f"More details about the error can be found in the log -> `~/frappe/logs/fm.log`") + archive_msg =( f"\nIF [y]: Sites that have failed will be rolled back and stored in {CLI_SITES_ARCHIVE}." "\nIF [N]: Revert the entire migration to the previous fm version." diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 67ca32c6..f03ce605 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -422,7 +422,7 @@ def down(self, remove_ophans=True, volumes=True, timeout=5) -> bool: ) if self.quiet: exit_code = richprint.live_lines(output, padding=(0, 0, 0, 2)) - richprint.print(f"Removing Containers: Done") + richprint.print(f"{status_text}: Done") except DockerException as e: richprint.exit(f"{status_text}: Failed") From 41a8db432f491c69766075b309f67c743b771c64 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 5 Feb 2024 17:17:57 +0530 Subject: [PATCH 100/100] update create command to include template flag --- frappe_manager/commands.py | 19 ++++++++++--------- frappe_manager/site_manager/SiteManager.py | 12 +++++++++--- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/frappe_manager/commands.py b/frappe_manager/commands.py index b858c46d..78c0009c 100644 --- a/frappe_manager/commands.py +++ b/frappe_manager/commands.py @@ -118,6 +118,7 @@ def create( frappe_branch: Annotated[ str, typer.Option(help="Specify the branch name for frappe app",callback=frappe_branch_validation_callback) ] = "version-15", + template: Annotated[bool, typer.Option(help="Create template site.")] = False, admin_pass: Annotated[ str, typer.Option( @@ -130,21 +131,21 @@ def create( """ Create a new site. - Frappe\[version-14] will be installed by default. + Frappe\[version-15] will be installed by default. [bold white on black]Examples:[/bold white on black] - [bold]# Install frappe\[version-14][/bold] + [bold]# Install frappe\[version-15][/bold] $ [blue]fm create example[/blue] - [bold]# Install frappe\[version-15-beta][/bold] - $ [blue]fm create example --frappe-branch version-15-beta[/blue] + [bold]# Install frappe\[develop][/bold] + $ [blue]fm create example --frappe-branch develop[/blue] - [bold]# Install frappe\[version-14], erpnext\[version-14] and hrms\[version-14][/bold] - $ [blue]fm create example --apps erpnext:version-14 --apps hrms:version-14[/blue] + [bold]# Install frappe\[version-15], erpnext\[version-15] and hrms\[version-15][/bold] + $ [blue]fm create example --apps erpnext:version-15 --apps hrms:version-15[/blue] - [bold]# Install frappe\[version-15-beta], erpnext\[version-15-beta] and hrms\[version-15-beta][/bold] - $ [blue]fm create example --frappe-branch version-15-beta --apps erpnext:version-15-beta --apps hrms:version-15-beta[/blue] + [bold]# Install frappe\[version-15], erpnext\[version-14] and hrms\[version-14][/bold] + $ [blue]fm create example --frappe-branch version-14 --apps erpnext:version-14 --apps hrms:version-14[/blue] """ sites.init(sitename) @@ -197,7 +198,7 @@ def create( # turn off all previous # start the docker compose - sites.create_site(template_inputs) + sites.create_site(template_inputs,template_site=template) @app.command(no_args_is_help=True) diff --git a/frappe_manager/site_manager/SiteManager.py b/frappe_manager/site_manager/SiteManager.py index 34781d15..24e88557 100644 --- a/frappe_manager/site_manager/SiteManager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -106,7 +106,7 @@ def stop_sites(self): richprint.exit(f"{status_text}: Failed") richprint.print(f"{status_text}: Done") - def create_site(self, template_inputs: dict): + def create_site(self, template_inputs: dict,template_site: bool = False): """ Creates a new site using the provided template inputs. @@ -126,6 +126,10 @@ def create_site(self, template_inputs: dict): self.site.create_compose_dirs() self.site.pull() + if template_site: + self.site.remove_secrets() + richprint.exit(f"Created template site: {self.site.name}",emoji_code=":white_check_mark:") + richprint.change_head(f"Starting Site") self.site.start() self.site.frappe_logs_till_start() @@ -377,6 +381,8 @@ def info(self): frappe_password = self.site.composefile.get_envs("frappe")["ADMIN_PASS"] services_db_info = self.services.get_database_info() root_db_password = services_db_info['password'] + root_db_host = services_db_info['host'] + root_db_user = services_db_info['user'] site_info_table = Table(show_lines=True, show_header=False, highlight=True) @@ -387,9 +393,9 @@ def info(self): "Adminer Url": f"http://{self.site.name}/adminer", "Frappe Username": "administrator", "Frappe Password": frappe_password, - "Root DB User": "root", + "Root DB User": root_db_user, "Root DB Password": root_db_password, - "DB Host": "mariadb", + "Root DB Host": root_db_host, "DB Name": db_user, "DB User": db_user, "DB Password": db_pass,