From e859e195d9051cc4c7711c12ff50a95a4f661de0 Mon Sep 17 00:00:00 2001 From: Dmitrii Ovsyannikov Date: Sat, 13 Apr 2024 22:30:19 +0200 Subject: [PATCH] docs: add basic READMEs (#36) --- README.md | 37 ++- backend/.gitignore | 4 - backend/README.md | 250 ++++++++++++++++-- .../docs/configs/actions/telegram_webhook.md | 27 ++ backend/docs/configs/secrets/README.md | 7 + backend/docs/configs/secrets/env.md | 9 + backend/docs/configs/triggers/github.md | 64 +++++ backend/lib/app/app.py | 6 +- backend/lib/app/settings.py | 42 +-- 9 files changed, 402 insertions(+), 44 deletions(-) create mode 100644 backend/docs/configs/actions/telegram_webhook.md create mode 100644 backend/docs/configs/secrets/README.md create mode 100644 backend/docs/configs/secrets/env.md create mode 100644 backend/docs/configs/triggers/github.md diff --git a/README.md b/README.md index 3403cbb..052d29b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,41 @@ # Github Watcher -Github Watcher +GitHub Watcher is an easy-to-use extendable framework for setting up custom reactions on various triggers. + +### Supported triggers + +- Github: new PR, new Issue, failed workflow run + +### Supported reactions + +- Telegram message + +## Usage + +It can be easily setup in CI by [github-watcher-action](https://github.com/ovsds/github-watcher-action), example: + +- [workflow](.github/workflows/github-watcher.yaml) +- [config](.github/github-watcher-config.yaml) + +For any advanced use cases, it can be used as a standalone service by running the backend service in docker. Example: + +```shell +docker run \ + --rm \ + --volume ./backend/example/settings.yaml:/settings.yaml \ # provide settings file + --volume ./backend/example/config.yaml:/config.yaml \ # provide config file for yaml_file config backend + --volume ./backend/example/state:/state \ # provide state directory for local_dir state backend + --env GITHUB_WATCHER_SETTINGS_YAML=/settings.yaml \ + --env GITHUB_TOKEN \ + --env TELEGRAM_CHAT_ID \ + --env TELEGRAM_BOT_TOKEN \ + ghcr.io/ovsds/github-watcher:0.3.0 +``` + +### Backend + +The backend service is responsible for handling triggers and reactions. +More information can be found in [backend/README.md](backend/README.md). ## Development diff --git a/backend/.gitignore b/backend/.gitignore index 7138ccf..73aaba0 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -10,9 +10,5 @@ __pycache__/ /.coverage htmlcov/ -# Environment variables -.env -.secrets - # Local app state example/state diff --git a/backend/README.md b/backend/README.md index 9a297d8..9ddfa80 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1,27 +1,245 @@ -# "Github Watcher Backend" +# Github Watcher Backend -"Github Watcher Backend" +Asynchronous application for checking triggers and performing reactions. -## Development +## Usage -### Global dependencies +### Settings -- poetry +Can be set by both yaml file and environment variables, [example](example/settings.yaml). +File path can be set by `GITHUB_WATCHER_SETTINGS_YAML` environment variable. +Settings consist of next sections: -### Taskfile commands +- app - general application settings +- logs - logging settings +- tasks - task processing settings -For all commands see [Taskfile](Taskfile.yaml) or `task --list-all`. +#### App + +`app.env` - application environment, used mainly for logging. Default is `production`. + +```yaml +app: + env: development +``` + +Can be set by `GITHUB_WATCHER_APP__ENV` environment variable. + +--- + +`app.debug` - debug mode, enables more verbose logging. Default is `false`. + +```yaml +app: + debug: true +``` + +Can be set by `GITHUB_WATCHER_APP__DEBUG` environment variable. + +#### Logs + +`logs.level` logging level, can be one of `debug`, `info`, `warning`, `error`. Default is `info`. + +```yaml +logs: + level: debug +``` + +Can be set by `GITHUB_WATCHER_LOGS__LEVEL` environment variable. + +--- + +`logs.format` - logging format string, passed to python logging formatter. Default is `%(asctime)s - %(name)s - %(levelname)s - %(message)s`. + +```yaml +logs: + format: "%(message)s" +``` + +Can be set by `GITHUB_WATCHER_LOGS__FORMAT` environment variable. + +#### Tasks + +`tasks.config_backend` - backend configuration, used for setting up triggers and reactions. +Can select among different types. Currently, only `yaml_file` is supported. + +```yaml +tasks: + config_backend: + type: yaml_file + path: example/config.yaml +``` + +Can be set by environment variables: + +```shell +GITHUB_WATCHER_TASKS__CONFIG_BACKEND__TYPE=yaml_file +GITHUB_WATCHER_TASKS__CONFIG_BACKEND__PATH=example/config.yaml +``` + +--- + +`tasks.queue_backend` - queue configuration, used as a message broker for task processing. +Can select among different types. Currently, only `memory` is supported. + +```yaml +tasks: + queue_backend: + type: memory +``` + +Can be set by environment variables: + +```shell +GITHUB_WATCHER_TASKS__QUEUE_BACKEND__TYPE=memory +``` + +--- + +`tasks.state_backend` - state backend configuration, used for storing task and queue state. +Can select among different types. Currently, only `local_dir` is supported. + +```yaml +tasks: + state_backend: + type: local_dir + path: example/state +``` + +Can be set by environment variables: + +```shell +GITHUB_WATCHER_TASKS__STATE_BACKEND__TYPE=local_dir +GITHUB_WATCHER_TASKS__STATE_BACKEND__PATH=example/state +``` + +--- + +`tasks.scheduler.limit` - maximum number of parallel jobs. Default is `100`. + +```yaml +tasks: + scheduler: + limit: 10 +``` + +Can be set by `GITHUB_WATCHER_TASKS__SCHEDULER__LIMIT` environment variable. -### Environment variables +--- -Application: +`tasks.scheduler.pending_limit` - maximum number of pending jobs. `0` means no limit. Default is `0`. -- `APP_ENV` - Application environment (`development` or `production`) -- `APP_NAME` - Application name -- `APP_VERSION` - Application version -- `APP_DEBUG` - Application debug mode +```yaml +tasks: + scheduler: + pending_limit: 10 +``` -Logging: +Can be set by `GITHUB_WATCHER_TASKS__SCHEDULER__PENDING_LIMIT` environment variable. -- `LOGS_MIN_LEVEL` - Minimum log level (`DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`) -- `LOGS_FORMAT` - Log format +--- + +`tasks.scheduler.timeout` - maximum time for all jobs to finish in seconds. `0` means no timeout. +Default is `600`. + +```yaml +tasks: + scheduler: + timeout: 10 +``` + +Can be set by `GITHUB_WATCHER_TASKS__SCHEDULER__TIMEOUT` environment variable. + +--- + +`tasks.scheduler.close_timeout` - maximum time for scheduler to wait for all jobs to finish in seconds. + +```yaml +tasks: + scheduler: + close_timeout: 10 +``` + +Can be set by `GITHUB_WATCHER_TASKS__SCHEDULER__CLOSE_TIMEOUT` environment variable. + +### Task Processors + +Task (`tasks.task_processor`), trigger(`tasks.trigger_processor`) and +event (`tasks.event_processor`) processors can be set by same block with the same structure. + +--- + +`tasks.[...]_processor.count` - number of job processors. Default is `5`. + +```yaml +tasks: + [...]_processor: + count: 10 +``` + +Can be set by `GITHUB_WATCHER_TASKS__[...]_PROCESSOR__COUNT` environment variable. + +--- + +`tasks.[...]_processor.max_retries` - maximum number of retries for failed jobs. Default is `3`. + +```yaml +tasks: + [...]_processor: + max_retries: 5 +``` + +Can be set by `GITHUB_WATCHER_TASKS__[...]_PROCESSOR__MAX_RETRIES` environment variable. + +--- + +`tasks.[...]_processor.queue_state_mode` - queue state mode, sets queue state handling mode. +Can be one of `preserve`, `restart`, `ignore`. Default is `preserve`. + +```yaml +tasks: + [...]_processor: + queue_state_mode: restart +``` + +Can be set by `GITHUB_WATCHER_TASKS__[...]_PROCESSOR__QUEUE_STATE_MODE` environment variable. + +--- + +`tasks.[...]_processor.failed_queue_state_mode` - failed queue state mode, +sets queue state handling mode for failed jobs. Can be one of `preserve`, `restart`, `ignore`. Default is `preserve`. + +```yaml +tasks: + [...]_processor: + failed_queue_state_mode: ignore +``` + +### Config + +Config can be set by yaml file, [example](example/config.yaml) when using `yaml_file` config backend. + +```yaml +tasks: + - id: + triggers: ... + actions: ... +``` + +Task config consists of two main sections: + +- id - task id, used for task identification +- triggers - list of triggers. + Currently, only [github](docs/configs/triggers/github.md) trigger is supported. +- actions - list of actions. + Currently, only [telegram_webhook](docs/configs/actions/telegram_webhook.md) action is supported. + +## Development + +### Global dependencies + +- poetry + +### Taskfile commands + +For all commands see [Taskfile](Taskfile.yaml) or `task --list-all`. diff --git a/backend/docs/configs/actions/telegram_webhook.md b/backend/docs/configs/actions/telegram_webhook.md new file mode 100644 index 0000000..6886bed --- /dev/null +++ b/backend/docs/configs/actions/telegram_webhook.md @@ -0,0 +1,27 @@ +# Telegram webhook action + +Action to send a message to a Telegram chat using a webhook on event. + +## Configuration + +- `id` - action id, used for action identification. +- `type` - action type, should be `telegram_webhook`. +- `chat_id_secret` - [secret](../secrets/README.md) configuration to provide chat id. +- `token_secret` - [secret](../secrets/README.md) configuration to provide bot token. +- `max_message_title_length` - maximum message title length. Default is `100`. +- `max_message_body_length` - maximum message body length. Default is `500`. + +## Example + +```yaml +id: telegram_webhook +type: telegram_webhook +chat_id_secret: + type: env + key: TELEGRAM_CHAT_ID +token_secret: + type: env + key: TELEGRAM_BOT_TOKEN +max_message_title_length: 50 +max_message_body_length: 200 +``` diff --git a/backend/docs/configs/secrets/README.md b/backend/docs/configs/secrets/README.md new file mode 100644 index 0000000..7e7157f --- /dev/null +++ b/backend/docs/configs/secrets/README.md @@ -0,0 +1,7 @@ +# Secret config + +Special dynamic field type, used in the configuration to provide secrets. + +Supported types: + +- [env](env.md) - environment variable diff --git a/backend/docs/configs/secrets/env.md b/backend/docs/configs/secrets/env.md new file mode 100644 index 0000000..46d5971 --- /dev/null +++ b/backend/docs/configs/secrets/env.md @@ -0,0 +1,9 @@ +# Env secret + +The `env` secret type is used to provide configuration values using environment variables. + +```yaml +field_secret: + type: env + key: ENV_KEY +``` diff --git a/backend/docs/configs/triggers/github.md b/backend/docs/configs/triggers/github.md new file mode 100644 index 0000000..79371ff --- /dev/null +++ b/backend/docs/configs/triggers/github.md @@ -0,0 +1,64 @@ +# Github trigger + +Composite trigger for GitHub events, currently supported sub-triggers: + +- Created PR +- Created Issue +- Workflow run failed + +## Configuration + +- `id` - trigger id, used for trigger identification. +- `type` - trigger type, should be `github`. +- `owner` - repository owner. +- `include_repos` - list of repositories to include. +- `exclude_repos` - list of repositories to exclude. +- `default_timedelta_seconds` - default timedelta in seconds for events. Default is `86400` (1 day). +- `sub_triggers` - list of sub-triggers, currently supported. + +## Sub-triggers + +### Repository Issue created + +Triggered when a new Issue is created in repository from trigger. + +- `id` - sub-trigger id, used for sub-trigger identification. Optional, `type` is used if not provided. +- `type` - sub-trigger type, should be `repository_issue_created`. +- `include_author` - list of authors to include. +- `exclude_author` - list of authors to exclude. + +### Repository PR created + +Triggered when a new PR is created in repository from trigger. + +- `id` - sub-trigger id, used for sub-trigger identification. Optional, `type` is used if not provided. +- `type` - sub-trigger type, should be `repository_pr_created`. +- `include_author` - list of authors to include. +- `exclude_author` - list of authors to exclude. + +### Workflow run failed + +Triggered when a workflow run failed in repository from trigger. + +- `id` - sub-trigger id, used for sub-trigger identification. Optional, `type` is used if not provided. +- `type` - sub-trigger type, should be `repository_failed_workflow_run`. +- `include` - list of workflow names to include. +- `exclude` - list of workflow names to exclude. + +## Example + +```yaml +id: github_trigger +type: github +owner: ovsds +include_repos: + - github-watcher + - github-watcher-actions +default_timedelta_seconds: 3600 +sub_triggers: + - type: repository_pr_created + - type: repository_issue_created + - type: repository_failed_workflow_run + exclude: + - Check PR +``` diff --git a/backend/lib/app/app.py b/backend/lib/app/app.py index 13504a7..814cf45 100644 --- a/backend/lib/app/app.py +++ b/backend/lib/app/app.py @@ -51,7 +51,9 @@ def from_settings(cls, settings: app_settings.Settings) -> typing.Self: logger.info("Initializing application") - aiojobs_scheduler = aiojobs_utils.Scheduler.from_settings(settings=settings.tasks.aiojobs_scheduler_settings) + aiojobs_scheduler = aiojobs_utils.Scheduler.from_settings( + settings=settings.tasks.scheduler.aiojobs_scheduler_settings + ) # Clients @@ -244,7 +246,7 @@ async def start(self) -> None: raise app_errors.ServerRuntimeError("Application runtime error") from unexpected_error async def _start(self) -> None: - timer = asyncio_utils.TimeoutTimer(timeout=self._settings.tasks.timeout) + timer = asyncio_utils.TimeoutTimer(timeout=self._settings.tasks.scheduler.timeout) while not timer.is_expired: all_topics_finished = all( diff --git a/backend/lib/app/settings.py b/backend/lib/app/settings.py index 38d272f..f6a09c5 100644 --- a/backend/lib/app/settings.py +++ b/backend/lib/app/settings.py @@ -12,9 +12,7 @@ class AppSettings(pydantic_settings.BaseSettings): - env: str = "development" - name: str = "github-watcher-backend" - version: str = "0.0.1" + env: str = "production" debug: bool = False @property @@ -34,11 +32,26 @@ class LoggingSettings(pydantic_settings.BaseSettings): format: str = "%(asctime)s | %(name)s | %(levelname)s | %(message)s" +class SchedulerSettings(pydantic_settings.BaseSettings): + limit: int = 100 + pending_limit: int = 0 # 0 means no limit + timeout: int = 10 * 60 # 10 minutes, 0 means no timeout + close_timeout: int = 10 + + @property + def aiojobs_scheduler_settings(self) -> aiojobs_utils.Settings: + return aiojobs_utils.Settings( + limit=self.limit, + pending_limit=self.pending_limit, + close_timeout=self.close_timeout, + ) + + class JobProcessorSettings(pydantic_settings.BaseSettings): count: int = 5 max_retries: int = 3 queue_state_mode: task_services.JobProcessorQueueStateMode = pydantic.Field( - default=task_services.JobProcessorQueueStateMode.RESTART + default=task_services.JobProcessorQueueStateMode.PRESERVE ) failed_queue_state_mode: task_services.JobProcessorQueueStateMode = pydantic.Field( default=task_services.JobProcessorQueueStateMode.PRESERVE @@ -46,16 +59,6 @@ class JobProcessorSettings(pydantic_settings.BaseSettings): class TasksSettings(pydantic_settings.BaseSettings): - scheduler_limit: int = 100 - scheduler_pending_limit: int = 0 # 0 means no limit - - task_processor: JobProcessorSettings = pydantic.Field(default_factory=JobProcessorSettings) - trigger_processor: JobProcessorSettings = pydantic.Field(default_factory=JobProcessorSettings) - event_processor: JobProcessorSettings = pydantic.Field(default_factory=JobProcessorSettings) - - timeout: int = 10 * 60 # 10 minutes - close_timeout: int = 10 - config_backend: typing.Annotated[ task_repositories.BaseConfigSettings, pydantic.BeforeValidator(task_repositories.BaseConfigSettings.factory), @@ -69,13 +72,10 @@ class TasksSettings(pydantic_settings.BaseSettings): pydantic.BeforeValidator(task_repositories.BaseStateSettings.factory), ] = NotImplemented - @property - def aiojobs_scheduler_settings(self) -> aiojobs_utils.Settings: - return aiojobs_utils.Settings( - limit=self.scheduler_limit, - pending_limit=self.scheduler_pending_limit, - close_timeout=self.close_timeout, - ) + scheduler: SchedulerSettings = pydantic.Field(default_factory=SchedulerSettings) + task_processor: JobProcessorSettings = pydantic.Field(default_factory=JobProcessorSettings) + trigger_processor: JobProcessorSettings = pydantic.Field(default_factory=JobProcessorSettings) + event_processor: JobProcessorSettings = pydantic.Field(default_factory=JobProcessorSettings) class Settings(pydantic_settings.BaseSettings):