From 593cee1502c83bb091f3073067bbd44266453d6d Mon Sep 17 00:00:00 2001 From: ovsds Date: Sun, 30 Jun 2024 12:32:41 +0200 Subject: [PATCH 1/2] refactor: task protocols --- backend/lib/github/triggers.py | 8 ++- backend/lib/task/base/trigger.py | 6 +- backend/lib/task/jobs/task_spawner.py | 3 +- backend/lib/task/jobs/trigger_processor.py | 3 +- backend/lib/task/protocols.py | 36 ++++++++++ backend/lib/task/repositories/state/base.py | 49 +++----------- .../lib/task/repositories/state/local/file.py | 7 +- backend/lib/task/services/queue_state.py | 3 +- backend/tests/unit/github/__init__.py | 0 .../tests/unit/github/triggers/__init__.py | 0 ...ository_issue_created_subtrigger_config.py | 67 +++++++++++++++++++ backend/tests/unit/test_plug.py | 2 - 12 files changed, 131 insertions(+), 53 deletions(-) create mode 100644 backend/lib/task/protocols.py create mode 100644 backend/tests/unit/github/__init__.py create mode 100644 backend/tests/unit/github/triggers/__init__.py create mode 100644 backend/tests/unit/github/triggers/test_repository_issue_created_subtrigger_config.py delete mode 100644 backend/tests/unit/test_plug.py diff --git a/backend/lib/github/triggers.py b/backend/lib/github/triggers.py index f730650..628cc89 100644 --- a/backend/lib/github/triggers.py +++ b/backend/lib/github/triggers.py @@ -9,7 +9,7 @@ import lib.github.clients as github_clients import lib.github.models as github_models import lib.task.base as task_base -import lib.task.repositories as task_repositories +import lib.task.protocols import lib.utils.asyncio as asyncio_utils import lib.utils.pydantic as pydantic_utils @@ -42,6 +42,8 @@ def factory(cls, v: typing.Any, info: pydantic.ValidationInfo) -> "SubtriggerCon class RepositoryIssueCreatedSubtriggerConfig(SubtriggerConfig): + type: str = "repository_issue_created" + include_author: list[str] = pydantic.Field(default_factory=list) exclude_author: list[str] = pydantic.Field(default_factory=list) @@ -165,7 +167,7 @@ class GithubTriggerState(pydantic_utils.BaseModel): class GithubTriggerProcessor(task_base.TriggerProcessor[GithubTriggerConfig]): def __init__( self, - raw_state: task_repositories.StateProtocol, + raw_state: lib.task.protocols.StateProtocol, gql_github_client: github_clients.GqlGithubClient, rest_github_client: github_clients.RestGithubClient, config: GithubTriggerConfig, @@ -179,7 +181,7 @@ def __init__( def from_config( cls, config: GithubTriggerConfig, - state: task_repositories.StateProtocol, + state: lib.task.protocols.StateProtocol, ) -> typing.Self: gql_github_client = github_clients.GqlGithubClient(token=config.token_secret.value) rest_github_client = github_clients.RestGithubClient.from_token(token=config.token_secret.value) diff --git a/backend/lib/task/base/trigger.py b/backend/lib/task/base/trigger.py index e69a851..525cb0e 100644 --- a/backend/lib/task/base/trigger.py +++ b/backend/lib/task/base/trigger.py @@ -5,7 +5,7 @@ import pydantic import lib.task.base as task_base -import lib.task.repositories.state as task_state_repositories +import lib.task.protocols as task_protocols import lib.utils.pydantic as pydantic_utils @@ -43,7 +43,7 @@ class TriggerProcessor(typing.Generic[ConfigT], abc.ABC): def from_config( cls, config: ConfigT, - state: task_state_repositories.StateProtocol, + state: task_protocols.StateProtocol, ) -> typing.Self: ... async def produce_events(self) -> typing.AsyncGenerator[task_base.Event, None]: ... @@ -81,7 +81,7 @@ def trigger_config_factory(data: typing.Any) -> BaseTriggerConfig: def trigger_processor_factory( config: BaseTriggerConfig, - state: task_state_repositories.StateProtocol, + state: task_protocols.StateProtocol, ) -> TriggerProcessorProtocol: processor_class = _REGISTRY[config.type].processor_class return processor_class.from_config(config=config, state=state) diff --git a/backend/lib/task/jobs/task_spawner.py b/backend/lib/task/jobs/task_spawner.py index 1d1f685..6418ef5 100644 --- a/backend/lib/task/jobs/task_spawner.py +++ b/backend/lib/task/jobs/task_spawner.py @@ -5,6 +5,7 @@ import lib.task.base as task_base import lib.task.jobs.models as task_jobs_models +import lib.task.protocols as task_protocols import lib.task.repositories as task_repositories import lib.utils.aiojobs as aiojobs_utils import lib.utils.json as json_utils @@ -41,7 +42,7 @@ def __init__( self, config_repository: task_repositories.ConfigRepositoryProtocol, queue_repository: task_repositories.QueueRepositoryProtocol, - state_repository: task_repositories.StateRepositoryProtocol, + state_repository: task_protocols.StateRepositoryProtocol, ) -> None: self._config_repository = config_repository self._queue_repository = queue_repository diff --git a/backend/lib/task/jobs/trigger_processor.py b/backend/lib/task/jobs/trigger_processor.py index 0570944..bdf75c2 100644 --- a/backend/lib/task/jobs/trigger_processor.py +++ b/backend/lib/task/jobs/trigger_processor.py @@ -2,6 +2,7 @@ import lib.task.base as task_base import lib.task.jobs.models as task_job_models +import lib.task.protocols as task_protocols import lib.task.repositories as task_repositories import lib.utils.aiojobs as aiojobs_utils @@ -17,7 +18,7 @@ def __init__( job_id: int, max_retries: int, queue_repository: task_repositories.QueueRepositoryProtocol, - state_repository: task_repositories.StateRepositoryProtocol, + state_repository: task_protocols.StateRepositoryProtocol, ): self._id = job_id self._max_retries = max_retries diff --git a/backend/lib/task/protocols.py b/backend/lib/task/protocols.py new file mode 100644 index 0000000..c1732a6 --- /dev/null +++ b/backend/lib/task/protocols.py @@ -0,0 +1,36 @@ +import typing + +import lib.utils.json as json_utils + +StateData = json_utils.JsonSerializableDict + + +class StateProtocol(typing.Protocol): + async def get(self) -> StateData | None: ... + + async def set(self, value: StateData) -> None: ... + + async def clear(self) -> None: ... + + def acquire(self) -> typing.AsyncContextManager[StateData | None]: ... + + +class StateRepositoryProtocol(typing.Protocol): + async def dispose(self) -> None: ... + + async def get(self, path: str) -> StateData | None: ... + + async def set(self, path: str, value: StateData) -> None: ... + + async def clear(self, path: str) -> None: ... + + def acquire(self, path: str) -> typing.AsyncContextManager[StateData | None]: ... + + async def get_state(self, path: str) -> StateProtocol: ... + + +__all__ = [ + "StateData", + "StateProtocol", + "StateRepositoryProtocol", +] diff --git a/backend/lib/task/repositories/state/base.py b/backend/lib/task/repositories/state/base.py index fa45369..aae29e3 100644 --- a/backend/lib/task/repositories/state/base.py +++ b/backend/lib/task/repositories/state/base.py @@ -6,33 +6,7 @@ import pydantic import pydantic_settings -import lib.utils.json as json_utils - -StateData = json_utils.JsonSerializableDict - - -class StateProtocol(typing.Protocol): - async def get(self) -> StateData | None: ... - - async def set(self, value: StateData) -> None: ... - - async def clear(self) -> None: ... - - def acquire(self) -> typing.AsyncContextManager[StateData | None]: ... - - -class StateRepositoryProtocol(typing.Protocol): - async def dispose(self) -> None: ... - - async def get(self, path: str) -> StateData | None: ... - - async def set(self, path: str, value: StateData) -> None: ... - - async def clear(self, path: str) -> None: ... - - def acquire(self, path: str) -> typing.AsyncContextManager[StateData | None]: ... - - async def get_state(self, path: str) -> StateProtocol: ... +import lib.task.protocols as task_protocols class BaseStateSettings(pydantic_settings.BaseSettings): @@ -47,21 +21,21 @@ def factory(cls, v: typing.Any, info: pydantic.ValidationInfo) -> "BaseStateSett class State: - def __init__(self, repository: StateRepositoryProtocol, path: str): + def __init__(self, repository: task_protocols.StateRepositoryProtocol, path: str): self._repository = repository self._path = path - async def get(self) -> StateData | None: + async def get(self) -> task_protocols.StateData | None: return await self._repository.get(self._path) - async def set(self, value: StateData) -> None: + async def set(self, value: task_protocols.StateData) -> None: await self._repository.set(self._path, value) async def clear(self) -> None: await self._repository.clear(self._path) @contextlib.asynccontextmanager - async def acquire(self) -> typing.AsyncIterator[StateData | None]: + async def acquire(self) -> typing.AsyncIterator[task_protocols.StateData | None]: async with self._repository.acquire(self._path) as state: yield state @@ -74,17 +48,17 @@ def from_settings(cls, settings: SettingsT) -> typing.Self: ... async def dispose(self) -> None: ... @abc.abstractmethod - async def get(self, path: str) -> StateData | None: ... + async def get(self, path: str) -> task_protocols.StateData | None: ... @abc.abstractmethod - async def set(self, path: str, value: StateData) -> None: ... + async def set(self, path: str, value: task_protocols.StateData) -> None: ... async def clear(self, path: str) -> None: ... @abc.abstractmethod - def acquire(self, path: str) -> typing.AsyncContextManager[StateData | None]: ... + def acquire(self, path: str) -> typing.AsyncContextManager[task_protocols.StateData | None]: ... - async def get_state(self, path: str) -> StateProtocol: + async def get_state(self, path: str) -> task_protocols.StateProtocol: return State(repository=self, path=path) @@ -117,7 +91,7 @@ def state_settings_factory(data: typing.Any) -> BaseStateSettings: return settings_class.model_validate(data) -def state_repository_factory(settings: BaseStateSettings) -> StateRepositoryProtocol: +def state_repository_factory(settings: BaseStateSettings) -> task_protocols.StateRepositoryProtocol: repository_class = _REGISTRY[settings.type].repository_class return repository_class.from_settings(settings) @@ -125,9 +99,6 @@ def state_repository_factory(settings: BaseStateSettings) -> StateRepositoryProt __all__ = [ "BaseStateRepository", "BaseStateSettings", - "StateData", - "StateProtocol", - "StateRepositoryProtocol", "register_state_backend", "state_repository_factory", "state_settings_factory", diff --git a/backend/lib/task/repositories/state/local/file.py b/backend/lib/task/repositories/state/local/file.py index 6754b13..977534f 100644 --- a/backend/lib/task/repositories/state/local/file.py +++ b/backend/lib/task/repositories/state/local/file.py @@ -5,6 +5,7 @@ import aiofile +import lib.task.protocols as task_protocols import lib.task.repositories.state.base as state_base import lib.utils.asyncio as asyncio_utils import lib.utils.json as json_utils @@ -25,7 +26,7 @@ def __init__(self, root_path: str): def from_settings(cls, settings: LocalDirStateSettings) -> typing.Self: return cls(root_path=settings.path) - async def get(self, path: str) -> state_base.StateData | None: + async def get(self, path: str) -> task_protocols.StateData | None: logger.debug("Loading State(%s)", path) try: async with aiofile.async_open(f"{self._root_path}/{path}.json", "r") as file: @@ -39,7 +40,7 @@ async def get(self, path: str) -> state_base.StateData | None: logger.debug("No State(%s) was found", path) return None - async def set(self, path: str, value: state_base.StateData) -> None: + async def set(self, path: str, value: task_protocols.StateData) -> None: logger.debug("Saving State(%s)", path) self._root_path.joinpath(path).parent.mkdir(parents=True, exist_ok=True) @@ -54,7 +55,7 @@ async def clear(self, path: str) -> None: pass @contextlib.asynccontextmanager - async def acquire(self, path: str) -> typing.AsyncIterator[state_base.StateData | None]: + async def acquire(self, path: str) -> typing.AsyncIterator[task_protocols.StateData | None]: async with asyncio_utils.acquire_file_lock(f"{self._root_path}/{path}.lock"): logger.debug("Acquired lock for State(%s)", path) yield await self.get(path) diff --git a/backend/lib/task/services/queue_state.py b/backend/lib/task/services/queue_state.py index d05f5b9..7a1040d 100644 --- a/backend/lib/task/services/queue_state.py +++ b/backend/lib/task/services/queue_state.py @@ -3,6 +3,7 @@ import typing import lib.task.jobs as task_jobs +import lib.task.protocols as task_protocols import lib.task.repositories as task_repositories import lib.utils.json as json_utils import lib.utils.pydantic as pydantic_utils @@ -51,7 +52,7 @@ class QueueStateService: def __init__( self, queue_repository: task_repositories.QueueRepositoryProtocol, - state_repository: task_repositories.StateRepositoryProtocol, + state_repository: task_protocols.StateRepositoryProtocol, job_topic: task_repositories.JobTopic, failed_job_topic: task_repositories.JobTopic, job_model: type[task_jobs.BaseJob], diff --git a/backend/tests/unit/github/__init__.py b/backend/tests/unit/github/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/unit/github/triggers/__init__.py b/backend/tests/unit/github/triggers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/unit/github/triggers/test_repository_issue_created_subtrigger_config.py b/backend/tests/unit/github/triggers/test_repository_issue_created_subtrigger_config.py new file mode 100644 index 0000000..d0bef5b --- /dev/null +++ b/backend/tests/unit/github/triggers/test_repository_issue_created_subtrigger_config.py @@ -0,0 +1,67 @@ +import datetime + +import pytest + +import lib.github.models as github_models +import lib.github.triggers as github_triggers + + +@pytest.fixture(name="issue") +def issue_fixture() -> github_models.Issue: + return github_models.Issue( + id="test_id", + author="test_author", + url="test_url", + title="test_title", + body="test_body", + created_at=datetime.datetime.now(), + ) + + +def test_is_applicable_default(issue: github_models.Issue): + config = github_triggers.RepositoryIssueCreatedSubtriggerConfig( + id="test_id", + ) + + assert config.is_applicable(issue=issue) + + +def test_is_applicable_include_author(issue: github_models.Issue): + config = github_triggers.RepositoryIssueCreatedSubtriggerConfig( + id="test_id", + include_author=["test_author"], + ) + assert config.is_applicable(issue=issue) + + +def test_is_applicable_include_author_not_matching(issue: github_models.Issue): + config = github_triggers.RepositoryIssueCreatedSubtriggerConfig( + id="test_id", + include_author=["other_author"], + ) + assert not config.is_applicable(issue=issue) + + +def test_is_applicable_exclude_author(issue: github_models.Issue): + config = github_triggers.RepositoryIssueCreatedSubtriggerConfig( + id="test_id", + exclude_author=["other_author"], + ) + assert config.is_applicable(issue=issue) + + +def test_is_applicable_exclude_author_matching(issue: github_models.Issue): + config = github_triggers.RepositoryIssueCreatedSubtriggerConfig( + id="test_id", + exclude_author=["test_author"], + ) + assert not config.is_applicable(issue=issue) + + +def test_is_applicable_include_and_exclude_author(issue: github_models.Issue): + config = github_triggers.RepositoryIssueCreatedSubtriggerConfig( + id="test_id", + include_author=["test_author"], + exclude_author=["test_author"], + ) + assert not config.is_applicable(issue=issue) diff --git a/backend/tests/unit/test_plug.py b/backend/tests/unit/test_plug.py deleted file mode 100644 index 1faa777..0000000 --- a/backend/tests/unit/test_plug.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_plug(): - assert True From ddb71fac0b741feda754d3b31b85c4406d556e56 Mon Sep 17 00:00:00 2001 From: ovsds Date: Sun, 30 Jun 2024 12:34:06 +0200 Subject: [PATCH 2/2] feat: add RepositoryIssueCreatedSubtrigger by title filtration --- backend/lib/github/triggers.py | 76 ++++++++++++++-- ...ository_issue_created_subtrigger_config.py | 91 +++++++++++++++++++ 2 files changed, 159 insertions(+), 8 deletions(-) diff --git a/backend/lib/github/triggers.py b/backend/lib/github/triggers.py index 628cc89..3b5ade6 100644 --- a/backend/lib/github/triggers.py +++ b/backend/lib/github/triggers.py @@ -1,6 +1,7 @@ import contextlib import datetime import logging +import re import typing import warnings @@ -41,16 +42,70 @@ def factory(cls, v: typing.Any, info: pydantic.ValidationInfo) -> "SubtriggerCon raise ValueError(f"Unknown subtrigger type: {v['type']}") +def _check_match(string: str, items: list[str] | None = None) -> bool: + return items is not None and len(items) != 0 and string in items + + +def _check_regex_match(string: str, regex_items: list[str] | None = None) -> bool: + return regex_items is not None and len(regex_items) != 0 and any(re.match(regex, string) for regex in regex_items) + + +def _check_included( + string: str, + include: list[str] | None = None, + include_regex: list[str] | None = None, +) -> bool: + return ( + (not include and not include_regex) + or _check_match(string, include) + or _check_regex_match(string, include_regex) + ) + + +def _check_excluded( + string: str, + exclude: list[str] | None = None, + exclude_regex: list[str] | None = None, +) -> bool: + return _check_match(string, exclude) or _check_regex_match(string, exclude_regex) + + +def _check_applicable( + string: str, + include: list[str] | None = None, + exclude: list[str] | None = None, + include_regex: list[str] | None = None, + exclude_regex: list[str] | None = None, +) -> bool: + return _check_included(string, include=include, include_regex=include_regex) and not _check_excluded( + string, exclude=exclude, exclude_regex=exclude_regex + ) + + class RepositoryIssueCreatedSubtriggerConfig(SubtriggerConfig): type: str = "repository_issue_created" include_author: list[str] = pydantic.Field(default_factory=list) exclude_author: list[str] = pydantic.Field(default_factory=list) + include_title: list[str] = pydantic.Field(default_factory=list) + exclude_title: list[str] = pydantic.Field(default_factory=list) + include_title_regex: list[str] = pydantic.Field(default_factory=list) + exclude_title_regex: list[str] = pydantic.Field(default_factory=list) def is_applicable(self, issue: github_models.Issue) -> bool: - if self.include_author and issue.author not in self.include_author: + if issue.author is not None and not _check_applicable( + issue.author, + include=self.include_author, + exclude=self.exclude_author, + ): return False - if issue.author in self.exclude_author: + if not _check_applicable( + issue.title, + include=self.include_title, + exclude=self.exclude_title, + include_regex=self.include_title_regex, + exclude_regex=self.exclude_title_regex, + ): return False return True @@ -61,9 +116,11 @@ class RepositoryPRCreatedSubtriggerConfig(SubtriggerConfig): exclude_author: list[str] = pydantic.Field(default_factory=list) def is_applicable(self, pr: github_models.PullRequest) -> bool: - if self.include_author and pr.author not in self.include_author: - return False - if pr.author in self.exclude_author: + if pr.author is not None and _check_applicable( + pr.author, + include=self.include_author, + exclude=self.exclude_author, + ): return False return True @@ -78,9 +135,11 @@ def is_applicable(self, workflow_run: github_models.WorkflowRun) -> bool: return False if workflow_run.conclusion != "failure": return False - if self.include and workflow_run.name not in self.include: - return False - if workflow_run.name in self.exclude: + if not _check_applicable( + workflow_run.name, + include=self.include, + exclude=self.exclude, + ): return False return True @@ -429,4 +488,5 @@ async def _process_repository_failed_workflow_run( __all__ = [ "GithubTriggerConfig", "GithubTriggerProcessor", + "RepositoryIssueCreatedSubtriggerConfig", ] diff --git a/backend/tests/unit/github/triggers/test_repository_issue_created_subtrigger_config.py b/backend/tests/unit/github/triggers/test_repository_issue_created_subtrigger_config.py index d0bef5b..4e271f2 100644 --- a/backend/tests/unit/github/triggers/test_repository_issue_created_subtrigger_config.py +++ b/backend/tests/unit/github/triggers/test_repository_issue_created_subtrigger_config.py @@ -65,3 +65,94 @@ def test_is_applicable_include_and_exclude_author(issue: github_models.Issue): exclude_author=["test_author"], ) assert not config.is_applicable(issue=issue) + + +def test_is_applicable_include_title(issue: github_models.Issue): + config = github_triggers.RepositoryIssueCreatedSubtriggerConfig( + id="test_id", + include_title=["test_title"], + ) + assert config.is_applicable(issue=issue) + + +def test_is_applicable_include_title_not_matching(issue: github_models.Issue): + config = github_triggers.RepositoryIssueCreatedSubtriggerConfig( + id="test_id", + include_title=["other_title"], + ) + assert not config.is_applicable(issue=issue) + + +def test_is_applicable_exclude_title(issue: github_models.Issue): + config = github_triggers.RepositoryIssueCreatedSubtriggerConfig( + id="test_id", + exclude_title=["other_title"], + ) + assert config.is_applicable(issue=issue) + + +def test_is_applicable_exclude_title_matching(issue: github_models.Issue): + config = github_triggers.RepositoryIssueCreatedSubtriggerConfig( + id="test_id", + exclude_title=["test_title"], + ) + assert not config.is_applicable(issue=issue) + + +def test_is_applicable_include_and_exclude_title(issue: github_models.Issue): + config = github_triggers.RepositoryIssueCreatedSubtriggerConfig( + id="test_id", + include_title=["test_title"], + exclude_title=["test_title"], + ) + assert not config.is_applicable(issue=issue) + + +def test_is_applicable_include_regex_title(issue: github_models.Issue): + config = github_triggers.RepositoryIssueCreatedSubtriggerConfig( + id="test_id", + include_title_regex=["test_.*"], + ) + assert config.is_applicable(issue=issue) + + +def test_is_applicable_include_regex_title_not_matching(issue: github_models.Issue): + config = github_triggers.RepositoryIssueCreatedSubtriggerConfig( + id="test_id", + include_title_regex=["other_.*"], + ) + assert not config.is_applicable(issue=issue) + + +def test_is_applicable_exclude_regex_title(issue: github_models.Issue): + config = github_triggers.RepositoryIssueCreatedSubtriggerConfig( + id="test_id", + exclude_title_regex=["other_.*"], + ) + assert config.is_applicable(issue=issue) + + +def test_is_applicable_exclude_regex_title_matching(issue: github_models.Issue): + config = github_triggers.RepositoryIssueCreatedSubtriggerConfig( + id="test_id", + exclude_title_regex=["test_.*"], + ) + assert not config.is_applicable(issue=issue) + + +def test_is_applicable_include_and_exclude_regex_title(issue: github_models.Issue): + config = github_triggers.RepositoryIssueCreatedSubtriggerConfig( + id="test_id", + include_title_regex=["test_.*"], + exclude_title_regex=["test_.*"], + ) + assert not config.is_applicable(issue=issue) + + +def test_is_applicable_include_and_include_regex_title(issue: github_models.Issue): + config = github_triggers.RepositoryIssueCreatedSubtriggerConfig( + id="test_id", + include_title=["other_title"], + include_title_regex=["test_.*"], + ) + assert config.is_applicable(issue=issue)