Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add Github RepositoryIssueCreatedSubtrigger by title filtration #51

Merged
merged 2 commits into from
Jun 30, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 73 additions & 11 deletions backend/lib/github/triggers.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import contextlib
import datetime
import logging
import re
import typing
import warnings

Expand All @@ -9,7 +10,7 @@
import lib.github.clients as github_clients
import lib.github.models as github_models
import lib.task.base as task_base
import lib.task.repositories as task_repositories
import lib.task.protocols
import lib.utils.asyncio as asyncio_utils
import lib.utils.pydantic as pydantic_utils

Expand Down Expand Up @@ -41,14 +42,70 @@ def factory(cls, v: typing.Any, info: pydantic.ValidationInfo) -> "SubtriggerCon
raise ValueError(f"Unknown subtrigger type: {v['type']}")


def _check_match(string: str, items: list[str] | None = None) -> bool:
return items is not None and len(items) != 0 and string in items


def _check_regex_match(string: str, regex_items: list[str] | None = None) -> bool:
return regex_items is not None and len(regex_items) != 0 and any(re.match(regex, string) for regex in regex_items)


def _check_included(
string: str,
include: list[str] | None = None,
include_regex: list[str] | None = None,
) -> bool:
return (
(not include and not include_regex)
or _check_match(string, include)
or _check_regex_match(string, include_regex)
)


def _check_excluded(
string: str,
exclude: list[str] | None = None,
exclude_regex: list[str] | None = None,
) -> bool:
return _check_match(string, exclude) or _check_regex_match(string, exclude_regex)


def _check_applicable(
string: str,
include: list[str] | None = None,
exclude: list[str] | None = None,
include_regex: list[str] | None = None,
exclude_regex: list[str] | None = None,
) -> bool:
return _check_included(string, include=include, include_regex=include_regex) and not _check_excluded(
string, exclude=exclude, exclude_regex=exclude_regex
)


class RepositoryIssueCreatedSubtriggerConfig(SubtriggerConfig):
type: str = "repository_issue_created"

include_author: list[str] = pydantic.Field(default_factory=list)
exclude_author: list[str] = pydantic.Field(default_factory=list)
include_title: list[str] = pydantic.Field(default_factory=list)
exclude_title: list[str] = pydantic.Field(default_factory=list)
include_title_regex: list[str] = pydantic.Field(default_factory=list)
exclude_title_regex: list[str] = pydantic.Field(default_factory=list)

def is_applicable(self, issue: github_models.Issue) -> bool:
if self.include_author and issue.author not in self.include_author:
if issue.author is not None and not _check_applicable(
issue.author,
include=self.include_author,
exclude=self.exclude_author,
):
return False
if issue.author in self.exclude_author:
if not _check_applicable(
issue.title,
include=self.include_title,
exclude=self.exclude_title,
include_regex=self.include_title_regex,
exclude_regex=self.exclude_title_regex,
):
return False

return True
Expand All @@ -59,9 +116,11 @@ class RepositoryPRCreatedSubtriggerConfig(SubtriggerConfig):
exclude_author: list[str] = pydantic.Field(default_factory=list)

def is_applicable(self, pr: github_models.PullRequest) -> bool:
if self.include_author and pr.author not in self.include_author:
return False
if pr.author in self.exclude_author:
if pr.author is not None and _check_applicable(
pr.author,
include=self.include_author,
exclude=self.exclude_author,
):
return False

return True
Expand All @@ -76,9 +135,11 @@ def is_applicable(self, workflow_run: github_models.WorkflowRun) -> bool:
return False
if workflow_run.conclusion != "failure":
return False
if self.include and workflow_run.name not in self.include:
return False
if workflow_run.name in self.exclude:
if not _check_applicable(
workflow_run.name,
include=self.include,
exclude=self.exclude,
):
return False

return True
Expand Down Expand Up @@ -165,7 +226,7 @@ class GithubTriggerState(pydantic_utils.BaseModel):
class GithubTriggerProcessor(task_base.TriggerProcessor[GithubTriggerConfig]):
def __init__(
self,
raw_state: task_repositories.StateProtocol,
raw_state: lib.task.protocols.StateProtocol,
gql_github_client: github_clients.GqlGithubClient,
rest_github_client: github_clients.RestGithubClient,
config: GithubTriggerConfig,
Expand All @@ -179,7 +240,7 @@ def __init__(
def from_config(
cls,
config: GithubTriggerConfig,
state: task_repositories.StateProtocol,
state: lib.task.protocols.StateProtocol,
) -> typing.Self:
gql_github_client = github_clients.GqlGithubClient(token=config.token_secret.value)
rest_github_client = github_clients.RestGithubClient.from_token(token=config.token_secret.value)
Expand Down Expand Up @@ -427,4 +488,5 @@ async def _process_repository_failed_workflow_run(
__all__ = [
"GithubTriggerConfig",
"GithubTriggerProcessor",
"RepositoryIssueCreatedSubtriggerConfig",
]
6 changes: 3 additions & 3 deletions backend/lib/task/base/trigger.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import pydantic

import lib.task.base as task_base
import lib.task.repositories.state as task_state_repositories
import lib.task.protocols as task_protocols
import lib.utils.pydantic as pydantic_utils


Expand Down Expand Up @@ -43,7 +43,7 @@ class TriggerProcessor(typing.Generic[ConfigT], abc.ABC):
def from_config(
cls,
config: ConfigT,
state: task_state_repositories.StateProtocol,
state: task_protocols.StateProtocol,
) -> typing.Self: ...

async def produce_events(self) -> typing.AsyncGenerator[task_base.Event, None]: ...
Expand Down Expand Up @@ -81,7 +81,7 @@ def trigger_config_factory(data: typing.Any) -> BaseTriggerConfig:

def trigger_processor_factory(
config: BaseTriggerConfig,
state: task_state_repositories.StateProtocol,
state: task_protocols.StateProtocol,
) -> TriggerProcessorProtocol:
processor_class = _REGISTRY[config.type].processor_class
return processor_class.from_config(config=config, state=state)
Expand Down
3 changes: 2 additions & 1 deletion backend/lib/task/jobs/task_spawner.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import lib.task.base as task_base
import lib.task.jobs.models as task_jobs_models
import lib.task.protocols as task_protocols
import lib.task.repositories as task_repositories
import lib.utils.aiojobs as aiojobs_utils
import lib.utils.json as json_utils
Expand Down Expand Up @@ -41,7 +42,7 @@ def __init__(
self,
config_repository: task_repositories.ConfigRepositoryProtocol,
queue_repository: task_repositories.QueueRepositoryProtocol,
state_repository: task_repositories.StateRepositoryProtocol,
state_repository: task_protocols.StateRepositoryProtocol,
) -> None:
self._config_repository = config_repository
self._queue_repository = queue_repository
Expand Down
3 changes: 2 additions & 1 deletion backend/lib/task/jobs/trigger_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import lib.task.base as task_base
import lib.task.jobs.models as task_job_models
import lib.task.protocols as task_protocols
import lib.task.repositories as task_repositories
import lib.utils.aiojobs as aiojobs_utils

Expand All @@ -17,7 +18,7 @@ def __init__(
job_id: int,
max_retries: int,
queue_repository: task_repositories.QueueRepositoryProtocol,
state_repository: task_repositories.StateRepositoryProtocol,
state_repository: task_protocols.StateRepositoryProtocol,
):
self._id = job_id
self._max_retries = max_retries
Expand Down
36 changes: 36 additions & 0 deletions backend/lib/task/protocols.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
import typing

import lib.utils.json as json_utils

StateData = json_utils.JsonSerializableDict


class StateProtocol(typing.Protocol):
async def get(self) -> StateData | None: ...

async def set(self, value: StateData) -> None: ...

async def clear(self) -> None: ...

def acquire(self) -> typing.AsyncContextManager[StateData | None]: ...


class StateRepositoryProtocol(typing.Protocol):
async def dispose(self) -> None: ...

async def get(self, path: str) -> StateData | None: ...

async def set(self, path: str, value: StateData) -> None: ...

async def clear(self, path: str) -> None: ...

def acquire(self, path: str) -> typing.AsyncContextManager[StateData | None]: ...

async def get_state(self, path: str) -> StateProtocol: ...


__all__ = [
"StateData",
"StateProtocol",
"StateRepositoryProtocol",
]
49 changes: 10 additions & 39 deletions backend/lib/task/repositories/state/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,33 +6,7 @@
import pydantic
import pydantic_settings

import lib.utils.json as json_utils

StateData = json_utils.JsonSerializableDict


class StateProtocol(typing.Protocol):
async def get(self) -> StateData | None: ...

async def set(self, value: StateData) -> None: ...

async def clear(self) -> None: ...

def acquire(self) -> typing.AsyncContextManager[StateData | None]: ...


class StateRepositoryProtocol(typing.Protocol):
async def dispose(self) -> None: ...

async def get(self, path: str) -> StateData | None: ...

async def set(self, path: str, value: StateData) -> None: ...

async def clear(self, path: str) -> None: ...

def acquire(self, path: str) -> typing.AsyncContextManager[StateData | None]: ...

async def get_state(self, path: str) -> StateProtocol: ...
import lib.task.protocols as task_protocols


class BaseStateSettings(pydantic_settings.BaseSettings):
Expand All @@ -47,21 +21,21 @@ def factory(cls, v: typing.Any, info: pydantic.ValidationInfo) -> "BaseStateSett


class State:
def __init__(self, repository: StateRepositoryProtocol, path: str):
def __init__(self, repository: task_protocols.StateRepositoryProtocol, path: str):
self._repository = repository
self._path = path

async def get(self) -> StateData | None:
async def get(self) -> task_protocols.StateData | None:
return await self._repository.get(self._path)

async def set(self, value: StateData) -> None:
async def set(self, value: task_protocols.StateData) -> None:
await self._repository.set(self._path, value)

async def clear(self) -> None:
await self._repository.clear(self._path)

@contextlib.asynccontextmanager
async def acquire(self) -> typing.AsyncIterator[StateData | None]:
async def acquire(self) -> typing.AsyncIterator[task_protocols.StateData | None]:
async with self._repository.acquire(self._path) as state:
yield state

Expand All @@ -74,17 +48,17 @@ def from_settings(cls, settings: SettingsT) -> typing.Self: ...
async def dispose(self) -> None: ...

@abc.abstractmethod
async def get(self, path: str) -> StateData | None: ...
async def get(self, path: str) -> task_protocols.StateData | None: ...

@abc.abstractmethod
async def set(self, path: str, value: StateData) -> None: ...
async def set(self, path: str, value: task_protocols.StateData) -> None: ...

async def clear(self, path: str) -> None: ...

@abc.abstractmethod
def acquire(self, path: str) -> typing.AsyncContextManager[StateData | None]: ...
def acquire(self, path: str) -> typing.AsyncContextManager[task_protocols.StateData | None]: ...

async def get_state(self, path: str) -> StateProtocol:
async def get_state(self, path: str) -> task_protocols.StateProtocol:
return State(repository=self, path=path)


Expand Down Expand Up @@ -117,17 +91,14 @@ def state_settings_factory(data: typing.Any) -> BaseStateSettings:
return settings_class.model_validate(data)


def state_repository_factory(settings: BaseStateSettings) -> StateRepositoryProtocol:
def state_repository_factory(settings: BaseStateSettings) -> task_protocols.StateRepositoryProtocol:
repository_class = _REGISTRY[settings.type].repository_class
return repository_class.from_settings(settings)


__all__ = [
"BaseStateRepository",
"BaseStateSettings",
"StateData",
"StateProtocol",
"StateRepositoryProtocol",
"register_state_backend",
"state_repository_factory",
"state_settings_factory",
Expand Down
7 changes: 4 additions & 3 deletions backend/lib/task/repositories/state/local/file.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import aiofile

import lib.task.protocols as task_protocols
import lib.task.repositories.state.base as state_base
import lib.utils.asyncio as asyncio_utils
import lib.utils.json as json_utils
Expand All @@ -25,7 +26,7 @@ def __init__(self, root_path: str):
def from_settings(cls, settings: LocalDirStateSettings) -> typing.Self:
return cls(root_path=settings.path)

async def get(self, path: str) -> state_base.StateData | None:
async def get(self, path: str) -> task_protocols.StateData | None:
logger.debug("Loading State(%s)", path)
try:
async with aiofile.async_open(f"{self._root_path}/{path}.json", "r") as file:
Expand All @@ -39,7 +40,7 @@ async def get(self, path: str) -> state_base.StateData | None:
logger.debug("No State(%s) was found", path)
return None

async def set(self, path: str, value: state_base.StateData) -> None:
async def set(self, path: str, value: task_protocols.StateData) -> None:
logger.debug("Saving State(%s)", path)
self._root_path.joinpath(path).parent.mkdir(parents=True, exist_ok=True)

Expand All @@ -54,7 +55,7 @@ async def clear(self, path: str) -> None:
pass

@contextlib.asynccontextmanager
async def acquire(self, path: str) -> typing.AsyncIterator[state_base.StateData | None]:
async def acquire(self, path: str) -> typing.AsyncIterator[task_protocols.StateData | None]:
async with asyncio_utils.acquire_file_lock(f"{self._root_path}/{path}.lock"):
logger.debug("Acquired lock for State(%s)", path)
yield await self.get(path)
Expand Down
3 changes: 2 additions & 1 deletion backend/lib/task/services/queue_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import typing

import lib.task.jobs as task_jobs
import lib.task.protocols as task_protocols
import lib.task.repositories as task_repositories
import lib.utils.json as json_utils
import lib.utils.pydantic as pydantic_utils
Expand Down Expand Up @@ -51,7 +52,7 @@ class QueueStateService:
def __init__(
self,
queue_repository: task_repositories.QueueRepositoryProtocol,
state_repository: task_repositories.StateRepositoryProtocol,
state_repository: task_protocols.StateRepositoryProtocol,
job_topic: task_repositories.JobTopic,
failed_job_topic: task_repositories.JobTopic,
job_model: type[task_jobs.BaseJob],
Expand Down
Empty file.
Empty file.
Loading
Loading