diff --git a/.gitignore b/.gitignore index 32f4367d356..ba2740de7ff 100644 --- a/.gitignore +++ b/.gitignore @@ -177,3 +177,6 @@ Untitled* services/**/settings-schema.json tests/public-api/osparc_python_wheels/* + +# osparc-config repo files +repo.config diff --git a/scripts/release/Makefile b/scripts/release/Makefile new file mode 100644 index 00000000000..897974db3c9 --- /dev/null +++ b/scripts/release/Makefile @@ -0,0 +1,15 @@ +.DEFAULT_GOAL := help + +SHELL := /bin/bash + + +help: ## help on rule's targets + @awk --posix 'BEGIN {FS = ":.*?## "} /^[[:alpha:][:space:]_-]+:.*?## / {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) + + +.PHONY: install install-dev +install install-dev: ## installation and check + # installation + @pip install $(if $(findstring -dev, $@),-e,) . + # check executable runs + monitor-release --help diff --git a/scripts/release/monitor/README.md b/scripts/release/README.md similarity index 63% rename from scripts/release/monitor/README.md rename to scripts/release/README.md index 6b039a10c92..fee981b98cc 100644 --- a/scripts/release/monitor/README.md +++ b/scripts/release/README.md @@ -1,14 +1,16 @@ -# Helper for monitoring of release -`pip install .` -`monitor-release --help` +# release-monitor -Check current status of containers -`monitor-release master containers` -Check running sidecars: -`monitor-release master sidecars` -# Create .env file -``` +Helper for monitoring of release. Here some one-liner examples: + +- Installation: `pip install .; monitor-release --help` or simply `make install-ci` or `make install-dev` +- Check current status of containers in `master` deployment: `monitor-release containers master` containers. +- Check running sidecars in `master` deployment: `monitor-release sidecar master` + + +## Create .env file + +```.env MASTER_PORTAINER_URL= MASTER_PORTAINER_USERNAME= MASTER_PORTAINER_PASSWORD= diff --git a/scripts/release/monitor/Makefile b/scripts/release/monitor/Makefile deleted file mode 100644 index 25427db8b97..00000000000 --- a/scripts/release/monitor/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -.PHONY: install-dev -install-dev: - pip install . diff --git a/scripts/release/monitor/monitor_release/cli.py b/scripts/release/monitor/monitor_release/cli.py deleted file mode 100644 index 9820b4cd613..00000000000 --- a/scripts/release/monitor/monitor_release/cli.py +++ /dev/null @@ -1,31 +0,0 @@ -from enum import Enum - -import typer -from monitor_release.models import Deployment -from monitor_release.portainer import check_containers_deploys, check_running_sidecars -from monitor_release.settings import get_settings -from rich.console import Console - -app = typer.Typer() -console = Console() - - -class Action(str, Enum): - containers = "containers" - sidecars = "sidecars" - - -@app.command() -def main( - deployment: Deployment, - action: Action, - env_file: str = typer.Option(".env", help="Path to .env file"), -): - settings = get_settings(env_file, deployment) - console.print(f"Deployment: {deployment}") - console.print(f"Action: {action}") - - if action == Action.containers: - check_containers_deploys(settings, deployment) - if action == Action.sidecars: - check_running_sidecars(settings, deployment) diff --git a/scripts/release/monitor/monitor_release/settings.py b/scripts/release/monitor/monitor_release/settings.py deleted file mode 100644 index 2351be306e5..00000000000 --- a/scripts/release/monitor/monitor_release/settings.py +++ /dev/null @@ -1,112 +0,0 @@ -import os - -from dotenv import load_dotenv -from pydantic import BaseModel - - -class Settings(BaseModel): - portainer_url: str - portainer_username: str - portainer_password: str - starts_with: str - swarm_stack_name: str - portainer_endpoint_version: int - - -def get_settings(env_file, deployment): - # pylint: disable=too-many-return-statements - load_dotenv(env_file) - - if deployment == "master": - portainer_url = os.getenv("MASTER_PORTAINER_URL") - portainer_username = os.getenv("MASTER_PORTAINER_USERNAME") - portainer_password = os.getenv("MASTER_PORTAINER_PASSWORD") - - return Settings( - portainer_url=portainer_url, - portainer_username=portainer_username, - portainer_password=portainer_password, - starts_with="master-simcore_master", - swarm_stack_name="master-simcore", - portainer_endpoint_version=1, - ) - if deployment == "dalco-staging": - portainer_url = os.getenv("DALCO_STAGING_PORTAINER_URL") - portainer_username = os.getenv("DALCO_STAGING_PORTAINER_USERNAME") - portainer_password = os.getenv("DALCO_STAGING_PORTAINER_PASSWORD") - - return Settings( - portainer_url=portainer_url, - portainer_username=portainer_username, - portainer_password=portainer_password, - starts_with="staging-simcore_staging", - swarm_stack_name="staging-simcore", - portainer_endpoint_version=1, - ) - if deployment == "dalco-production": - portainer_url = os.getenv("DALCO_PRODUCTION_PORTAINER_URL") - portainer_username = os.getenv("DALCO_PRODUCTION_PORTAINER_USERNAME") - portainer_password = os.getenv("DALCO_PRODUCTION_PORTAINER_PASSWORD") - - return Settings( - portainer_url=portainer_url, - portainer_username=portainer_username, - portainer_password=portainer_password, - starts_with="production-simcore_production", - swarm_stack_name="production-simcore", - portainer_endpoint_version=1, - ) - if deployment == "tip-production": - portainer_url = os.getenv("TIP_PRODUCTION_PORTAINER_URL") - portainer_username = os.getenv("TIP_PRODUCTION_PORTAINER_USERNAME") - portainer_password = os.getenv("TIP_PRODUCTION_PORTAINER_PASSWORD") - - return Settings( - portainer_url=portainer_url, - portainer_username=portainer_username, - portainer_password=portainer_password, - starts_with="production-simcore_production", - swarm_stack_name="production-simcore", - portainer_endpoint_version=2, - ) - if deployment == "aws-staging": - portainer_url = os.getenv("AWS_STAGING_PORTAINER_URL") - portainer_username = os.getenv("AWS_STAGING_PORTAINER_USERNAME") - portainer_password = os.getenv("AWS_STAGING_PORTAINER_PASSWORD") - - return Settings( - portainer_url=portainer_url, - portainer_username=portainer_username, - portainer_password=portainer_password, - starts_with="staging-simcore_staging", - swarm_stack_name="staging-simcore", - portainer_endpoint_version=2, - ) - if deployment == "aws-nih-production": - portainer_url = os.getenv("AWS_NIH_PRODUCTION_PORTAINER_URL") - portainer_username = os.getenv("AWS_NIH_PRODUCTION_PORTAINER_USERNAME") - portainer_password = os.getenv("AWS_NIH_PRODUCTION_PORTAINER_PASSWORD") - - return Settings( - portainer_url=portainer_url, - portainer_username=portainer_username, - portainer_password=portainer_password, - starts_with="production-simcore_production", - swarm_stack_name="production-simcore", - portainer_endpoint_version=2, - ) - if deployment == "aws-zmt-production": - portainer_url = os.getenv("AWS_ZMT_PRODUCTION_PORTAINER_URL") - portainer_username = os.getenv("AWS_ZMT_PRODUCTION_PORTAINER_USERNAME") - portainer_password = os.getenv("AWS_ZMT_PRODUCTION_PORTAINER_PASSWORD") - - return Settings( - portainer_url=portainer_url, - portainer_username=portainer_username, - portainer_password=portainer_password, - starts_with="staging-simcore_staging", - swarm_stack_name="staging-simcore", - portainer_endpoint_version=1, - ) - else: - raise ValueError("Invalid environment type provided.") diff --git a/scripts/release/monitor/monitor_release/__init__.py b/scripts/release/monitor_release/__init__.py similarity index 100% rename from scripts/release/monitor/monitor_release/__init__.py rename to scripts/release/monitor_release/__init__.py diff --git a/scripts/release/monitor_release/cli.py b/scripts/release/monitor_release/cli.py new file mode 100644 index 00000000000..06a41782a7d --- /dev/null +++ b/scripts/release/monitor_release/cli.py @@ -0,0 +1,54 @@ +from pathlib import Path +from typing import Annotated + +import typer +from monitor_release.models import Deployment +from monitor_release.portainer import check_containers_deploys, check_running_sidecars +from monitor_release.settings import get_legacy_settings, get_release_settings +from rich.console import Console + +app = typer.Typer() +console = Console() + + +EnvFileOption = typer.Option( + exists=True, + file_okay=True, + dir_okay=False, + writable=False, + readable=True, + resolve_path=True, + help="Path to .env file", +) + + +@app.command() +def settings( + env_file: Annotated[Path, EnvFileOption] = Path("repo.config"), +): + settings_ = get_release_settings(env_file) + console.print(settings_.model_dump_json(indent=1)) + + +@app.command() +def containers( + deployment: Deployment, + env_file: Annotated[Path, EnvFileOption] = Path(".env"), +): + settings_ = get_legacy_settings(f"{env_file}", deployment) + console.print(f"Deployment: {deployment}") + console.print("Action: containers") + + check_containers_deploys(settings_, deployment) + + +@app.command() +def sidecars( + deployment: Deployment, + env_file: Annotated[Path, EnvFileOption] = Path(".env"), +): + settings_ = get_legacy_settings(f"{env_file}", deployment) + console.print(f"Deployment: {deployment}") + console.print("Action: sidecars") + + check_running_sidecars(settings_, deployment) diff --git a/scripts/release/monitor/monitor_release/gitlab.py b/scripts/release/monitor_release/gitlab.py similarity index 100% rename from scripts/release/monitor/monitor_release/gitlab.py rename to scripts/release/monitor_release/gitlab.py diff --git a/scripts/release/monitor/monitor_release/models.py b/scripts/release/monitor_release/models.py similarity index 100% rename from scripts/release/monitor/monitor_release/models.py rename to scripts/release/monitor_release/models.py diff --git a/scripts/release/monitor/monitor_release/portainer.py b/scripts/release/monitor_release/portainer.py similarity index 82% rename from scripts/release/monitor/monitor_release/portainer.py rename to scripts/release/monitor_release/portainer.py index 08864eda7a6..34a4a03589b 100644 --- a/scripts/release/monitor/monitor_release/portainer.py +++ b/scripts/release/monitor_release/portainer.py @@ -35,16 +35,15 @@ def check_containers_deploys(settings, deployment): container_git_sha = None for task in item["tasks"]: oldest_running_task_timestamp = None - if task["status"] == "running": - if ( - oldest_running_task_timestamp is None - or oldest_running_task_timestamp > task["timestamp"] - ): - container_status = f"[green]{task['status']}[/green]" - container_timestamp = f"{task['timestamp']}" - container_git_sha = task["git_sha"] + if task["status"] == "running" and ( + oldest_running_task_timestamp is None + or oldest_running_task_timestamp > task["timestamp"] + ): + container_status = f"[green]{task['status']}[/green]" + container_timestamp = f"{task['timestamp']}" + container_git_sha = task["git_sha"] - oldest_running_task_timestamp = task["timestamp"] + oldest_running_task_timestamp = task["timestamp"] if task["status"] == "starting": container_status = f"[blue]{task['status']}[/blue]" container_timestamp = f"{task['timestamp']}" diff --git a/scripts/release/monitor/monitor_release/portainer_utils.py b/scripts/release/monitor_release/portainer_utils.py similarity index 86% rename from scripts/release/monitor/monitor_release/portainer_utils.py rename to scripts/release/monitor_release/portainer_utils.py index 23b99883326..5a63a62f998 100644 --- a/scripts/release/monitor/monitor_release/portainer_utils.py +++ b/scripts/release/monitor_release/portainer_utils.py @@ -3,10 +3,10 @@ import arrow import requests from monitor_release.models import RunningSidecar -from monitor_release.settings import Settings +from monitor_release.settings import LegacySettings -def get_bearer_token(settings: Settings): +def get_bearer_token(settings: LegacySettings): headers = {"accept": "application/json", "Content-Type": "application/json"} payload = json.dumps( { @@ -19,11 +19,10 @@ def get_bearer_token(settings: Settings): headers=headers, data=payload, ) - bearer_token = response.json()["jwt"] - return bearer_token + return response.json()["jwt"] -def get_services(settings: Settings, bearer_token): +def get_services(settings: LegacySettings, bearer_token): services_url = f"{settings.portainer_url}/portainer/api/endpoints/{settings.portainer_endpoint_version}/docker/services" response = requests.get( services_url, @@ -32,11 +31,10 @@ def get_services(settings: Settings, bearer_token): "Content-Type": "application/json", }, ) - services = response.json() - return services + return response.json() -def get_tasks(settings: Settings, bearer_token): +def get_tasks(settings: LegacySettings, bearer_token): tasks_url = f"{settings.portainer_url}/portainer/api/endpoints/{settings.portainer_endpoint_version}/docker/tasks" response = requests.get( tasks_url, @@ -45,11 +43,10 @@ def get_tasks(settings: Settings, bearer_token): "Content-Type": "application/json", }, ) - tasks = response.json() - return tasks + return response.json() -def get_containers(settings: Settings, bearer_token): +def get_containers(settings: LegacySettings, bearer_token): bearer_token = get_bearer_token(settings) containers_url = f"{settings.portainer_url}/portainer/api/endpoints/{settings.portainer_endpoint_version}/docker/containers/json?all=true" @@ -60,11 +57,10 @@ def get_containers(settings: Settings, bearer_token): "Content-Type": "application/json", }, ) - containers = response.json() - return containers + return response.json() -def check_simcore_running_sidecars(settings: Settings, services): +def check_simcore_running_sidecars(settings: LegacySettings, services): running_sidecars: list[RunningSidecar] = [] for service in services: if ( @@ -106,7 +102,9 @@ def _generate_containers_map(containers): return container_map -def check_simcore_deployed_services(settings: Settings, services, tasks, containers): +def check_simcore_deployed_services( + settings: LegacySettings, services, tasks, containers +): container_map = _generate_containers_map(containers) service_task_map = {} for service in services: diff --git a/scripts/release/monitor/monitor_release/postgres.py b/scripts/release/monitor_release/postgres.py similarity index 100% rename from scripts/release/monitor/monitor_release/postgres.py rename to scripts/release/monitor_release/postgres.py diff --git a/scripts/release/monitor_release/settings.py b/scripts/release/monitor_release/settings.py new file mode 100644 index 00000000000..ba4f9c7e1d2 --- /dev/null +++ b/scripts/release/monitor_release/settings.py @@ -0,0 +1,231 @@ +import os +from pathlib import Path +from typing import Final + +from dotenv import load_dotenv +from pydantic import BaseModel, Field, HttpUrl, TypeAdapter, model_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + +from .models import Deployment + +# +_DEPLOYMENTS_MAP = { + Deployment.master: "osparc-master.speag.com", + Deployment.aws_staging: "osparc-staging.io", + Deployment.dalco_staging: "osparc-staging.speag.com", + Deployment.aws_nih_production: "osparc.io", + Deployment.dalco_production: "osparc.speag.com", + Deployment.tip_production: "tip.itis.swiss", + Deployment.aws_zmt_production: "sim4life.io", +} +_DEPLOYMENTS_IMAP = {v: k for k, v in _DEPLOYMENTS_MAP.items()} + +SECRETS_CONFIG_FILE_NAME: Final[str] = "repo.config" + + +def get_repo_configs_paths(top_folder: Path) -> list[Path]: + return list(top_folder.rglob(SECRETS_CONFIG_FILE_NAME)) + + +def get_deployment_name_or_none(repo_config: Path) -> str | None: + if repo_config.name == "repo.config": + return repo_config.resolve().parent.name + return None + + +class ReleaseSettings(BaseSettings): + OSPARC_DEPLOYMENT_TARGET: str + PORTAINER_DOMAIN: str + + portainer_username: str = Field(..., validation_alias="PORTAINER_USER") + portainer_password: str = Field(..., validation_alias="PORTAINER_PASSWORD") + swarm_stack_name: str = Field(..., validation_alias="SWARM_STACK_NAME") + portainer_endpoint_version: int + starts_with: str + portainer_url: HttpUrl | None = None + + model_config = SettingsConfigDict(extra="ignore") + + @model_validator(mode="after") + def deduce_portainer_url(self): + self.portainer_url = TypeAdapter(HttpUrl).validate_python( + f"https://{self.PORTAINER_DOMAIN}" + ) + return self + + +def get_release_settings(env_file_path: Path): + + # NOTE: these conversions and checks are done to keep + deployment_name = get_deployment_name_or_none(env_file_path) + if deployment_name is None: + msg = f"{env_file_path=} cannot be matched to any deployment" + raise ValueError(msg) + + deployment = _DEPLOYMENTS_IMAP.get(deployment_name) + if deployment is None: + msg = f"{deployment_name=} cannot be matched to any known deployment {set(_DEPLOYMENTS_IMAP.keys())}" + raise ValueError(msg) + + match deployment_name: + # NOTE: `portainer_endpoint_version` and `starts_with` cannot be deduced from the + # information in the `repo.config`. For that reason we have to set + # those values in the code. + # + + case "osparc-master.speag.com": + settings = ReleaseSettings( + _env_file=env_file_path, # type: ignore + portainer_endpoint_version=1, + starts_with="master-simcore_master", + ) + case "osparc-staging.speag.com": + settings = ReleaseSettings( + _env_file=env_file_path, # type: ignore + portainer_endpoint_version=1, + starts_with="staging-simcore_staging", + ) + case "osparc.speag.com": + settings = ReleaseSettings( + _env_file=env_file_path, # type: ignore + portainer_endpoint_version=1, + starts_with="production-simcore_production", + ) + case "tip.itis.swiss": + settings = ReleaseSettings( + _env_file=env_file_path, # type: ignore + portainer_endpoint_version=2, + starts_with="production-simcore_production", + ) + case "osparc-staging.io": + settings = ReleaseSettings( + _env_file=env_file_path, # type: ignore + portainer_endpoint_version=2, + starts_with="staging-simcore_staging", + ) + case "osparc.io": + settings = ReleaseSettings( + _env_file=env_file_path, # type: ignore + portainer_endpoint_version=2, + starts_with="production-simcore_production", + ) + case "sim4life.io": + settings = ReleaseSettings( + _env_file=env_file_path, # type: ignore + portainer_endpoint_version=1, + starts_with="staging-simcore_staging", + ) + case _: + msg = f"Unkown {deployment=}. Please setupa a new ReleaseSettings for this configuration" + raise ValueError(msg) + + return settings + + +class LegacySettings(BaseModel): + portainer_url: str + portainer_username: str + portainer_password: str + starts_with: str + swarm_stack_name: str + portainer_endpoint_version: int + + +def get_legacy_settings(env_file, deployment: str) -> LegacySettings: + # pylint: disable=too-many-return-statements + load_dotenv(env_file) + + if deployment == "master": + portainer_url = os.getenv("MASTER_PORTAINER_URL") + portainer_username = os.getenv("MASTER_PORTAINER_USERNAME") + portainer_password = os.getenv("MASTER_PORTAINER_PASSWORD") + + return LegacySettings( + portainer_url=portainer_url, + portainer_username=portainer_username, + portainer_password=portainer_password, + starts_with="master-simcore_master", + swarm_stack_name="master-simcore", + portainer_endpoint_version=1, + ) + if deployment == "dalco-staging": + portainer_url = os.getenv("DALCO_STAGING_PORTAINER_URL") + portainer_username = os.getenv("DALCO_STAGING_PORTAINER_USERNAME") + portainer_password = os.getenv("DALCO_STAGING_PORTAINER_PASSWORD") + + return LegacySettings( + portainer_url=portainer_url, + portainer_username=portainer_username, + portainer_password=portainer_password, + starts_with="staging-simcore_staging", + swarm_stack_name="staging-simcore", + portainer_endpoint_version=1, + ) + if deployment == "dalco-production": + portainer_url = os.getenv("DALCO_PRODUCTION_PORTAINER_URL") + portainer_username = os.getenv("DALCO_PRODUCTION_PORTAINER_USERNAME") + portainer_password = os.getenv("DALCO_PRODUCTION_PORTAINER_PASSWORD") + + return LegacySettings( + portainer_url=portainer_url, + portainer_username=portainer_username, + portainer_password=portainer_password, + starts_with="production-simcore_production", + swarm_stack_name="production-simcore", + portainer_endpoint_version=1, + ) + if deployment == "tip-production": + portainer_url = os.getenv("TIP_PRODUCTION_PORTAINER_URL") + portainer_username = os.getenv("TIP_PRODUCTION_PORTAINER_USERNAME") + portainer_password = os.getenv("TIP_PRODUCTION_PORTAINER_PASSWORD") + + return LegacySettings( + portainer_url=portainer_url, + portainer_username=portainer_username, + portainer_password=portainer_password, + starts_with="production-simcore_production", + swarm_stack_name="production-simcore", + portainer_endpoint_version=2, + ) + if deployment == "aws-staging": + portainer_url = os.getenv("AWS_STAGING_PORTAINER_URL") + portainer_username = os.getenv("AWS_STAGING_PORTAINER_USERNAME") + portainer_password = os.getenv("AWS_STAGING_PORTAINER_PASSWORD") + + return LegacySettings( + portainer_url=portainer_url, + portainer_username=portainer_username, + portainer_password=portainer_password, + starts_with="staging-simcore_staging", + swarm_stack_name="staging-simcore", + portainer_endpoint_version=2, + ) + if deployment == "aws-nih-production": + portainer_url = os.getenv("AWS_NIH_PRODUCTION_PORTAINER_URL") + portainer_username = os.getenv("AWS_NIH_PRODUCTION_PORTAINER_USERNAME") + portainer_password = os.getenv("AWS_NIH_PRODUCTION_PORTAINER_PASSWORD") + + return LegacySettings( + portainer_url=portainer_url, + portainer_username=portainer_username, + portainer_password=portainer_password, + starts_with="production-simcore_production", + swarm_stack_name="production-simcore", + portainer_endpoint_version=2, + ) + if deployment == "aws-zmt-production": + portainer_url = os.getenv("AWS_ZMT_PRODUCTION_PORTAINER_URL") + portainer_username = os.getenv("AWS_ZMT_PRODUCTION_PORTAINER_USERNAME") + portainer_password = os.getenv("AWS_ZMT_PRODUCTION_PORTAINER_PASSWORD") + + return LegacySettings( + portainer_url=portainer_url, + portainer_username=portainer_username, + portainer_password=portainer_password, + starts_with="staging-simcore_staging", + swarm_stack_name="staging-simcore", + portainer_endpoint_version=1, + ) + else: + msg = "Invalid environment type provided." + raise ValueError(msg) diff --git a/scripts/release/monitor/pyproject.toml b/scripts/release/pyproject.toml similarity index 76% rename from scripts/release/monitor/pyproject.toml rename to scripts/release/pyproject.toml index b60275b6eaf..e6ab2020205 100644 --- a/scripts/release/monitor/pyproject.toml +++ b/scripts/release/pyproject.toml @@ -8,7 +8,7 @@ version = "1.2.3" authors = [{name="Matus Drobuliak", email="drobuliak@itis.swiss" }] description = "Helper script for monitoring releases" readme = "README.md" -dependencies = ["python-dotenv","pydantic", "typer[all]", "rich", "requests", "arrow"] +dependencies = ["arrow", "python-dotenv","pydantic", "pydantic-settings", "typer[all]>=0.9", "rich", "requests"] requires-python = ">=3.10" [project.scripts]