From d88901faba9b8c2211277a5a0a951ea164f87cd4 Mon Sep 17 00:00:00 2001 From: hellcassius Date: Tue, 5 Mar 2024 11:02:07 -0300 Subject: [PATCH 01/10] add test error flows --- pipelines/flows.py | 1 + pipelines/test/__init__.py | 0 pipelines/test/flows.py | 25 +++++++++++++++++++++++++ pipelines/test/tasks.py | 22 ++++++++++++++++++++++ 4 files changed, 48 insertions(+) create mode 100644 pipelines/test/__init__.py create mode 100644 pipelines/test/flows.py create mode 100644 pipelines/test/tasks.py diff --git a/pipelines/flows.py b/pipelines/flows.py index f949fbbd..05d70ff7 100644 --- a/pipelines/flows.py +++ b/pipelines/flows.py @@ -7,3 +7,4 @@ from pipelines.capture.templates.flows import * # noqa from pipelines.exemplo import * # noqa from pipelines.treatment.bilhetagem.flows import * # noqa +from pipelines.test.flows import * # noqa diff --git a/pipelines/test/__init__.py b/pipelines/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pipelines/test/flows.py b/pipelines/test/flows.py new file mode 100644 index 00000000..5c8bc67c --- /dev/null +++ b/pipelines/test/flows.py @@ -0,0 +1,25 @@ +from prefect.run_configs import KubernetesRun +from prefect.storage import GCS + +# isort: off +# EMD Imports # + +from prefeitura_rio.pipelines_utils.custom import Flow +from prefeitura_rio.pipelines_utils.state_handlers import handler_initialize_sentry + +from pipelines.constants import constants as emd_constants +from pipelines.schedules import every_minute +from pipelines.test.tasks import test_raise_errors +from pipelines.utils.backup.tasks import get_current_timestamp + +with Flow('SMTR - Teste de Erros do Glitch Tip') as raise_flow: + datetime = get_current_timestamp() + test_raise_errors(datetime=datetime) + +raise_flow.storage = GCS(emd_constants.GCS_FLOWS_BUCKET.value) +raise_flow.run_config = KubernetesRun( + image=emd_constants.DOCKER_IMAGE.value, + labels=[emd_constants.RJ_SMTR_DEV_AGENT_LABEL.value], +) +raise_flow.state_handlers = [handler_initialize_sentry] +raise_flow.schedule = every_minute \ No newline at end of file diff --git a/pipelines/test/tasks.py b/pipelines/test/tasks.py new file mode 100644 index 00000000..631d1be7 --- /dev/null +++ b/pipelines/test/tasks.py @@ -0,0 +1,22 @@ +from datetime import datetime + +from prefect import task + +# EMD Imports # + +from prefeitura_rio.pipelines_utils.logging import log + +# SMTR Imports # + +from pipelines.constants import constants +from pipelines.utils.backup.utils import log_critical, map_dict_keys + + +@task +def test_raise_errors(datetime: datetime): + if datetime.minute % 5 == 0: + raise ValueError(f'{datetime} % 5 is equal to zero') + if datetime.minute % 3 == 0: + raise ValueError(f'{datetime} % 3 is equal to zero') + else: + return datetime.minute / 0 \ No newline at end of file From b83b3abedde8518d053d9e51f3e9a2d0dc0cb1bd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 5 Mar 2024 14:03:23 +0000 Subject: [PATCH 02/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pipelines/flows.py | 2 +- pipelines/test/flows.py | 5 +++-- pipelines/test/tasks.py | 14 ++++++++------ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/pipelines/flows.py b/pipelines/flows.py index 05d70ff7..732cbb39 100644 --- a/pipelines/flows.py +++ b/pipelines/flows.py @@ -6,5 +6,5 @@ from pipelines.capture.jae.flows import * # noqa from pipelines.capture.templates.flows import * # noqa from pipelines.exemplo import * # noqa -from pipelines.treatment.bilhetagem.flows import * # noqa from pipelines.test.flows import * # noqa +from pipelines.treatment.bilhetagem.flows import * # noqa diff --git a/pipelines/test/flows.py b/pipelines/test/flows.py index 5c8bc67c..38362829 100644 --- a/pipelines/test/flows.py +++ b/pipelines/test/flows.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from prefect.run_configs import KubernetesRun from prefect.storage import GCS @@ -12,7 +13,7 @@ from pipelines.test.tasks import test_raise_errors from pipelines.utils.backup.tasks import get_current_timestamp -with Flow('SMTR - Teste de Erros do Glitch Tip') as raise_flow: +with Flow("SMTR - Teste de Erros do Glitch Tip") as raise_flow: datetime = get_current_timestamp() test_raise_errors(datetime=datetime) @@ -22,4 +23,4 @@ labels=[emd_constants.RJ_SMTR_DEV_AGENT_LABEL.value], ) raise_flow.state_handlers = [handler_initialize_sentry] -raise_flow.schedule = every_minute \ No newline at end of file +raise_flow.schedule = every_minute diff --git a/pipelines/test/tasks.py b/pipelines/test/tasks.py index 631d1be7..7821a7c4 100644 --- a/pipelines/test/tasks.py +++ b/pipelines/test/tasks.py @@ -1,22 +1,24 @@ +# -*- coding: utf-8 -*- from datetime import datetime from prefect import task +from prefeitura_rio.pipelines_utils.logging import log + +from pipelines.constants import constants +from pipelines.utils.backup.utils import log_critical, map_dict_keys # EMD Imports # -from prefeitura_rio.pipelines_utils.logging import log # SMTR Imports # -from pipelines.constants import constants -from pipelines.utils.backup.utils import log_critical, map_dict_keys @task def test_raise_errors(datetime: datetime): if datetime.minute % 5 == 0: - raise ValueError(f'{datetime} % 5 is equal to zero') + raise ValueError(f"{datetime} % 5 is equal to zero") if datetime.minute % 3 == 0: - raise ValueError(f'{datetime} % 3 is equal to zero') + raise ValueError(f"{datetime} % 3 is equal to zero") else: - return datetime.minute / 0 \ No newline at end of file + return datetime.minute / 0 From faa9c3537a8d954db9323da24007e467efdc98e3 Mon Sep 17 00:00:00 2001 From: hellcassius Date: Tue, 5 Mar 2024 11:22:07 -0300 Subject: [PATCH 03/10] add bd_credentials --- pipelines/test/flows.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipelines/test/flows.py b/pipelines/test/flows.py index 5c8bc67c..3b2e8370 100644 --- a/pipelines/test/flows.py +++ b/pipelines/test/flows.py @@ -5,7 +5,7 @@ # EMD Imports # from prefeitura_rio.pipelines_utils.custom import Flow -from prefeitura_rio.pipelines_utils.state_handlers import handler_initialize_sentry +from prefeitura_rio.pipelines_utils.state_handlers import handler_initialize_sentry, handler_inject_bd_credentials from pipelines.constants import constants as emd_constants from pipelines.schedules import every_minute @@ -21,5 +21,5 @@ image=emd_constants.DOCKER_IMAGE.value, labels=[emd_constants.RJ_SMTR_DEV_AGENT_LABEL.value], ) -raise_flow.state_handlers = [handler_initialize_sentry] +raise_flow.state_handlers = [handler_initialize_sentry, handler_inject_bd_credentials] raise_flow.schedule = every_minute \ No newline at end of file From 762431698ac0c1bd1b860157426b1189310e7bfe Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 5 Mar 2024 14:26:23 +0000 Subject: [PATCH 04/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pipelines/test/flows.py | 5 ++++- pipelines/test/tasks.py | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pipelines/test/flows.py b/pipelines/test/flows.py index d2fc4a78..2b76af70 100644 --- a/pipelines/test/flows.py +++ b/pipelines/test/flows.py @@ -6,7 +6,10 @@ # EMD Imports # from prefeitura_rio.pipelines_utils.custom import Flow -from prefeitura_rio.pipelines_utils.state_handlers import handler_initialize_sentry, handler_inject_bd_credentials +from prefeitura_rio.pipelines_utils.state_handlers import ( + handler_initialize_sentry, + handler_inject_bd_credentials, +) from pipelines.constants import constants as emd_constants from pipelines.schedules import every_minute diff --git a/pipelines/test/tasks.py b/pipelines/test/tasks.py index 7821a7c4..b2d5f6f6 100644 --- a/pipelines/test/tasks.py +++ b/pipelines/test/tasks.py @@ -13,7 +13,6 @@ # SMTR Imports # - @task def test_raise_errors(datetime: datetime): if datetime.minute % 5 == 0: From e8ca1b16b101fe98d45690c057c0aa64e2ffa0c5 Mon Sep 17 00:00:00 2001 From: hellcassius Date: Wed, 13 Mar 2024 10:20:10 -0300 Subject: [PATCH 05/10] change flow module --- pipelines/flows.py | 2 +- pipelines/{test => glitchtip}/__init__.py | 0 pipelines/{test => glitchtip}/flows.py | 2 +- pipelines/{test => glitchtip}/tasks.py | 0 4 files changed, 2 insertions(+), 2 deletions(-) rename pipelines/{test => glitchtip}/__init__.py (100%) rename pipelines/{test => glitchtip}/flows.py (94%) rename pipelines/{test => glitchtip}/tasks.py (100%) diff --git a/pipelines/flows.py b/pipelines/flows.py index 732cbb39..5429dca4 100644 --- a/pipelines/flows.py +++ b/pipelines/flows.py @@ -6,5 +6,5 @@ from pipelines.capture.jae.flows import * # noqa from pipelines.capture.templates.flows import * # noqa from pipelines.exemplo import * # noqa -from pipelines.test.flows import * # noqa +from pipelines.glitchtip.flows import * # noqa from pipelines.treatment.bilhetagem.flows import * # noqa diff --git a/pipelines/test/__init__.py b/pipelines/glitchtip/__init__.py similarity index 100% rename from pipelines/test/__init__.py rename to pipelines/glitchtip/__init__.py diff --git a/pipelines/test/flows.py b/pipelines/glitchtip/flows.py similarity index 94% rename from pipelines/test/flows.py rename to pipelines/glitchtip/flows.py index 2b76af70..e4478f1d 100644 --- a/pipelines/test/flows.py +++ b/pipelines/glitchtip/flows.py @@ -13,7 +13,7 @@ from pipelines.constants import constants as emd_constants from pipelines.schedules import every_minute -from pipelines.test.tasks import test_raise_errors +from pipelines.glitchtip.tasks import test_raise_errors from pipelines.utils.backup.tasks import get_current_timestamp with Flow("SMTR - Teste de Erros do Glitch Tip") as raise_flow: diff --git a/pipelines/test/tasks.py b/pipelines/glitchtip/tasks.py similarity index 100% rename from pipelines/test/tasks.py rename to pipelines/glitchtip/tasks.py From 21a415eafab254e6492fa61c92b4fd8b1bfd6c15 Mon Sep 17 00:00:00 2001 From: hellcassius Date: Wed, 13 Mar 2024 14:51:58 -0300 Subject: [PATCH 06/10] add report tasks --- pipelines/constants.py | 7 ++++- pipelines/glitchtip/tasks.py | 57 ++++++++++++++++++++++++++++++++++-- 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/pipelines/constants.py b/pipelines/constants.py index fdc543f5..2258439f 100644 --- a/pipelines/constants.py +++ b/pipelines/constants.py @@ -31,7 +31,7 @@ class constants(Enum): # pylint: disable=c0103 # WEBHOOK # CRITICAL_SECRET_PATH = "critical_webhook" - + WEBHOOKS_SECRET_PATH = 'webhooks' # RETRY POLICY # MAX_TIMEOUT_SECONDS = 60 MAX_RETRIES = 3 @@ -47,6 +47,11 @@ class constants(Enum): # pylint: disable=c0103 MODE_INCR = "incr" FLOW_RUN_URL_PATTERN = "https://pipelines.dados.rio/smtr/flow-run/{run_id}" + GLITCH_API_ENDPOINT = 'https://glitch.dados.rio/api/0/projects/smtr/pipelines/issues/' + GLITCH_URL = 'https://glitch.dados.rio/' + GLITCH_AUTH = 'glitch_auth' + GLITCH_WEBHOOK = 'glitch' + # GPS STPL # # GPS_STPL_API_BASE_URL = "http://zn4.m2mcontrol.com.br/api/integracao/veiculos" # GPS_STPL_API_SECRET_PATH = "stpl_api" diff --git a/pipelines/glitchtip/tasks.py b/pipelines/glitchtip/tasks.py index b2d5f6f6..5d0fbe32 100644 --- a/pipelines/glitchtip/tasks.py +++ b/pipelines/glitchtip/tasks.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- from datetime import datetime +import requests from prefect import task from prefeitura_rio.pipelines_utils.logging import log from pipelines.constants import constants -from pipelines.utils.backup.utils import log_critical, map_dict_keys - +from pipelines.utils.secret import get_secret # EMD Imports # @@ -21,3 +21,56 @@ def test_raise_errors(datetime: datetime): raise ValueError(f"{datetime} % 3 is equal to zero") else: return datetime.minute / 0 + +@task +def glitch_api_get(query='is:unresolved'): + base_url = constants.GLITCH_API_ENDPOINT.value + + headers = get_secret(secret_path=constants.GLITCH_AUTH.value) + print(headers) + if query: + base_url += f'?query={query}' + print(base_url) + issues = requests.get(base_url, headers=headers) + + return issues.json() + +@task +def format_glitch_issue_messages(issues): + messages = [] + issue_count = len(issues) + base_message = f"**Issues não resolvidos: {issue_count}**" + messages.append(base_message) + for issue in issues: + msg = f"""**Fonte: {issue['culprit']}** +**Erro**: {issue['title']} +**Event Count:** {issue['count']} +**Criado:** {issue['firstSeen'].split('.')[0]} +**Link:** {constants.GLITCH_URL.value}/smtr/issues/{issue['id']} +""" + messages.append({'name':f"Issue {issue['id']}", 'value':msg}) + + return messages + +@task +def send_issue_report(messages): + timestamp = datetime.now().isoformat() + webhook = get_secret( + secret_path=constants.WEBHOOKS_SECRET_PATH.value, + secret_name=constants.GLITCH_WEBHOOK.value + )['glitch'] + headers = {"Content-Type": "application/json"} + message = { + "content": messages[0], + "embeds": [{ + "color": 16515072, + "timestamp": timestamp, + "author": {'name':"Glitch Tip Issues", 'url':constants.GLITCH_URL.value}, + "fields": messages[1:] + }] + } + response = requests.post(url=webhook, headers=headers, json=message) + + log(response.text) + log(response.status_code) + From cd4b167d91f5941b22262068f83032a568f89a77 Mon Sep 17 00:00:00 2001 From: hellcassius Date: Wed, 13 Mar 2024 15:37:07 -0300 Subject: [PATCH 07/10] add report flow --- pipelines/glitchtip/flows.py | 22 ++++++++++++++++++++-- pipelines/glitchtip/tasks.py | 5 ++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/pipelines/glitchtip/flows.py b/pipelines/glitchtip/flows.py index e4478f1d..f8206851 100644 --- a/pipelines/glitchtip/flows.py +++ b/pipelines/glitchtip/flows.py @@ -12,8 +12,13 @@ ) from pipelines.constants import constants as emd_constants -from pipelines.schedules import every_minute -from pipelines.glitchtip.tasks import test_raise_errors +from pipelines.schedules import every_minute, every_day_noon +from pipelines.glitchtip.tasks import ( + test_raise_errors, + glitch_api_get_issues, + format_glitch_issue_messages, + send_issue_report +) from pipelines.utils.backup.tasks import get_current_timestamp with Flow("SMTR - Teste de Erros do Glitch Tip") as raise_flow: @@ -27,3 +32,16 @@ ) raise_flow.state_handlers = [handler_initialize_sentry, handler_inject_bd_credentials] raise_flow.schedule = every_minute + +with Flow('SMTR - Report de Issues do Glitch Tip') as glitch_flow: + issues = glitch_api_get_issues() + messages = format_glitch_issue_messages(issues) + send_issue_report(messages) + +glitch_flow.storage = GCS(emd_constants.GCS_FLOWS_BUCKET.value) +glitch_flow.run_config = KubernetesRun( + image=emd_constants.DOCKER_IMAGE.value, + labels=[emd_constants.RJ_SMTR_DEV_AGENT_LABEL.value], +) +glitch_flow.state_handlers = [handler_initialize_sentry] +glitch_flow.schedule = every_day_noon \ No newline at end of file diff --git a/pipelines/glitchtip/tasks.py b/pipelines/glitchtip/tasks.py index 5d0fbe32..2c9ebb66 100644 --- a/pipelines/glitchtip/tasks.py +++ b/pipelines/glitchtip/tasks.py @@ -23,14 +23,13 @@ def test_raise_errors(datetime: datetime): return datetime.minute / 0 @task -def glitch_api_get(query='is:unresolved'): +def glitch_api_get_issues(query='is:unresolved'): base_url = constants.GLITCH_API_ENDPOINT.value headers = get_secret(secret_path=constants.GLITCH_AUTH.value) - print(headers) if query: base_url += f'?query={query}' - print(base_url) + log(f'Will query glitch tip api at: {base_url}') issues = requests.get(base_url, headers=headers) return issues.json() From 39cb4b551710930829e8e582fbd973c07958512d Mon Sep 17 00:00:00 2001 From: hellcassius Date: Wed, 13 Mar 2024 15:37:46 -0300 Subject: [PATCH 08/10] add schedule every_day_noon, add state_handler to templates --- pipelines/capture/templates/flows.py | 2 ++ pipelines/schedules.py | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/pipelines/capture/templates/flows.py b/pipelines/capture/templates/flows.py index 341982e4..43168a9f 100644 --- a/pipelines/capture/templates/flows.py +++ b/pipelines/capture/templates/flows.py @@ -12,6 +12,7 @@ from prefeitura_rio.pipelines_utils.custom import Flow from prefeitura_rio.pipelines_utils.state_handlers import ( handler_inject_bd_credentials, + handler_initialize_sentry, handler_skip_if_running, ) @@ -241,6 +242,7 @@ def create_default_capture_flow( ) capture_flow.state_handlers = [ handler_inject_bd_credentials, + handler_initialize_sentry ] if skip_if_running: diff --git a/pipelines/schedules.py b/pipelines/schedules.py index f87fcf1f..d0e112da 100644 --- a/pipelines/schedules.py +++ b/pipelines/schedules.py @@ -114,6 +114,18 @@ def generate_interval_schedule( ] ) +every_day_noon = Schedule( + clocks=[ + IntervalClock( + interval=timedelta(days=1), + start_date=datetime(2021, 1, 1, 12, 0, tzinfo=timezone(constants.TIMEZONE.value)), + labels=[ + emd_constants.RJ_SMTR_AGENT_LABEL.value, + ], + ), + ] +) + ftp_clocks = generate_ftp_schedules( interval_minutes=60, label=emd_constants.RJ_SMTR_DEV_AGENT_LABEL ) From 021813ac2b2bbda4c115e9aa0f97dbf62b47dc7a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 18:38:36 +0000 Subject: [PATCH 09/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pipelines/capture/templates/flows.py | 7 ++---- pipelines/constants.py | 10 ++++---- pipelines/glitchtip/flows.py | 10 ++++---- pipelines/glitchtip/tasks.py | 36 +++++++++++++++------------- 4 files changed, 32 insertions(+), 31 deletions(-) diff --git a/pipelines/capture/templates/flows.py b/pipelines/capture/templates/flows.py index 43168a9f..a16701c8 100644 --- a/pipelines/capture/templates/flows.py +++ b/pipelines/capture/templates/flows.py @@ -11,8 +11,8 @@ from prefect.tasks.core.function import FunctionTask from prefeitura_rio.pipelines_utils.custom import Flow from prefeitura_rio.pipelines_utils.state_handlers import ( - handler_inject_bd_credentials, handler_initialize_sentry, + handler_inject_bd_credentials, handler_skip_if_running, ) @@ -240,10 +240,7 @@ def create_default_capture_flow( image=constants.DOCKER_IMAGE.value, labels=[agent_label], ) - capture_flow.state_handlers = [ - handler_inject_bd_credentials, - handler_initialize_sentry - ] + capture_flow.state_handlers = [handler_inject_bd_credentials, handler_initialize_sentry] if skip_if_running: capture_flow.state_handlers.append(handler_skip_if_running) diff --git a/pipelines/constants.py b/pipelines/constants.py index 2258439f..ec54de17 100644 --- a/pipelines/constants.py +++ b/pipelines/constants.py @@ -31,7 +31,7 @@ class constants(Enum): # pylint: disable=c0103 # WEBHOOK # CRITICAL_SECRET_PATH = "critical_webhook" - WEBHOOKS_SECRET_PATH = 'webhooks' + WEBHOOKS_SECRET_PATH = "webhooks" # RETRY POLICY # MAX_TIMEOUT_SECONDS = 60 MAX_RETRIES = 3 @@ -47,10 +47,10 @@ class constants(Enum): # pylint: disable=c0103 MODE_INCR = "incr" FLOW_RUN_URL_PATTERN = "https://pipelines.dados.rio/smtr/flow-run/{run_id}" - GLITCH_API_ENDPOINT = 'https://glitch.dados.rio/api/0/projects/smtr/pipelines/issues/' - GLITCH_URL = 'https://glitch.dados.rio/' - GLITCH_AUTH = 'glitch_auth' - GLITCH_WEBHOOK = 'glitch' + GLITCH_API_ENDPOINT = "https://glitch.dados.rio/api/0/projects/smtr/pipelines/issues/" + GLITCH_URL = "https://glitch.dados.rio/" + GLITCH_AUTH = "glitch_auth" + GLITCH_WEBHOOK = "glitch" # GPS STPL # # GPS_STPL_API_BASE_URL = "http://zn4.m2mcontrol.com.br/api/integracao/veiculos" diff --git a/pipelines/glitchtip/flows.py b/pipelines/glitchtip/flows.py index f8206851..4da1db3f 100644 --- a/pipelines/glitchtip/flows.py +++ b/pipelines/glitchtip/flows.py @@ -14,10 +14,10 @@ from pipelines.constants import constants as emd_constants from pipelines.schedules import every_minute, every_day_noon from pipelines.glitchtip.tasks import ( - test_raise_errors, - glitch_api_get_issues, + test_raise_errors, + glitch_api_get_issues, format_glitch_issue_messages, - send_issue_report + send_issue_report, ) from pipelines.utils.backup.tasks import get_current_timestamp @@ -33,7 +33,7 @@ raise_flow.state_handlers = [handler_initialize_sentry, handler_inject_bd_credentials] raise_flow.schedule = every_minute -with Flow('SMTR - Report de Issues do Glitch Tip') as glitch_flow: +with Flow("SMTR - Report de Issues do Glitch Tip") as glitch_flow: issues = glitch_api_get_issues() messages = format_glitch_issue_messages(issues) send_issue_report(messages) @@ -44,4 +44,4 @@ labels=[emd_constants.RJ_SMTR_DEV_AGENT_LABEL.value], ) glitch_flow.state_handlers = [handler_initialize_sentry] -glitch_flow.schedule = every_day_noon \ No newline at end of file +glitch_flow.schedule = every_day_noon diff --git a/pipelines/glitchtip/tasks.py b/pipelines/glitchtip/tasks.py index 2c9ebb66..0a69c231 100644 --- a/pipelines/glitchtip/tasks.py +++ b/pipelines/glitchtip/tasks.py @@ -1,12 +1,13 @@ # -*- coding: utf-8 -*- from datetime import datetime -import requests +import requests from prefect import task from prefeitura_rio.pipelines_utils.logging import log from pipelines.constants import constants from pipelines.utils.secret import get_secret + # EMD Imports # @@ -21,19 +22,21 @@ def test_raise_errors(datetime: datetime): raise ValueError(f"{datetime} % 3 is equal to zero") else: return datetime.minute / 0 - + + @task -def glitch_api_get_issues(query='is:unresolved'): +def glitch_api_get_issues(query="is:unresolved"): base_url = constants.GLITCH_API_ENDPOINT.value headers = get_secret(secret_path=constants.GLITCH_AUTH.value) if query: - base_url += f'?query={query}' - log(f'Will query glitch tip api at: {base_url}') + base_url += f"?query={query}" + log(f"Will query glitch tip api at: {base_url}") issues = requests.get(base_url, headers=headers) return issues.json() + @task def format_glitch_issue_messages(issues): messages = [] @@ -47,29 +50,30 @@ def format_glitch_issue_messages(issues): **Criado:** {issue['firstSeen'].split('.')[0]} **Link:** {constants.GLITCH_URL.value}/smtr/issues/{issue['id']} """ - messages.append({'name':f"Issue {issue['id']}", 'value':msg}) - + messages.append({"name": f"Issue {issue['id']}", "value": msg}) + return messages + @task def send_issue_report(messages): timestamp = datetime.now().isoformat() webhook = get_secret( - secret_path=constants.WEBHOOKS_SECRET_PATH.value, - secret_name=constants.GLITCH_WEBHOOK.value - )['glitch'] + secret_path=constants.WEBHOOKS_SECRET_PATH.value, secret_name=constants.GLITCH_WEBHOOK.value + )["glitch"] headers = {"Content-Type": "application/json"} message = { - "content": messages[0], - "embeds": [{ + "content": messages[0], + "embeds": [ + { "color": 16515072, "timestamp": timestamp, - "author": {'name':"Glitch Tip Issues", 'url':constants.GLITCH_URL.value}, - "fields": messages[1:] - }] + "author": {"name": "Glitch Tip Issues", "url": constants.GLITCH_URL.value}, + "fields": messages[1:], } + ], + } response = requests.post(url=webhook, headers=headers, json=message) log(response.text) log(response.status_code) - From 143f3f65bf703673327d4019f6ea4b34b1d3f11a Mon Sep 17 00:00:00 2001 From: hellcassius Date: Wed, 10 Apr 2024 13:59:17 -0300 Subject: [PATCH 10/10] merge main --- pipelines/constants.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pipelines/constants.py b/pipelines/constants.py index 21fcf8a4..07506a9f 100644 --- a/pipelines/constants.py +++ b/pipelines/constants.py @@ -32,11 +32,8 @@ class constants(Enum): # pylint: disable=c0103 # WEBHOOK # CRITICAL_SECRET_PATH = "critical_webhook" WEBHOOKS_SECRET_PATH = "webhooks" -<<<<<<< HEAD -======= DATAPLEX_WEBHOOK = "dataplex" ->>>>>>> a876e811bdeff6c543c25335bf3dceae2b469e25 # RETRY POLICY # MAX_TIMEOUT_SECONDS = 60 MAX_RETRIES = 3