diff --git a/lambdas/indexer/app.py b/lambdas/indexer/app.py index 16421c7f97..94a01a83a5 100644 --- a/lambdas/indexer/app.py +++ b/lambdas/indexer/app.py @@ -34,6 +34,9 @@ from azul.indexer.log_forwarding_controller import ( LogForwardingController, ) +from azul.indexer.notify_controller import ( + NotifyController, +) from azul.logging import ( configure_app_logging, ) @@ -74,6 +77,10 @@ class IndexerApp(AzulChaliceApp, SignatureHelper): def health_controller(self): return self._controller(HealthController, lambda_name='indexer') + @cached_property + def notify_controller(self): + return self._controller(NotifyController) + @cached_property def index_controller(self) -> IndexController: return self._controller(IndexController) @@ -97,6 +104,13 @@ def log_forwarder(self, prefix: str): else: return lambda func: func + @property + def monitoring(self): + if config.enable_monitoring: + return self.on_sns_message(topic=config.qualified_resource_name('monitoring')) + else: + return lambda func: func + def _authenticate(self) -> Optional[HMACAuthentication]: return self.auth_from_request(self.current_request) @@ -291,3 +305,8 @@ def forward_alb_logs(event: chalice.app.S3Event): @app.log_forwarder(config.s3_access_log_path_prefix(deployment=None)) def forward_s3_logs(event: chalice.app.S3Event): app.log_controller.forward_s3_access_logs(event) + + +@app.monitoring +def notify(event: chalice.app.SNSEvent): + app.notify_controller.digest(event) diff --git a/src/azul/deployment.py b/src/azul/deployment.py index 663bbde610..12c53b47a7 100644 --- a/src/azul/deployment.py +++ b/src/azul/deployment.py @@ -139,6 +139,10 @@ def securityhub(self): def sns(self): return self.client('sns') + @property + def ses(self): + return self.client('ses') + @property def sts(self): return self.client('sts') diff --git a/src/azul/indexer/notify_controller.py b/src/azul/indexer/notify_controller.py new file mode 100644 index 0000000000..8cc8d906be --- /dev/null +++ b/src/azul/indexer/notify_controller.py @@ -0,0 +1,52 @@ +import json + +import chalice.app +from more_itertools import ( + one, +) + +from azul import ( + cached_property, +) +from azul.chalice import ( + AppController, +) +from azul.indexer.notify_service import ( + AzulEmailNotificationService, +) + + +class NotifyController(AppController): + + @cached_property + def email(self): + return AzulEmailNotificationService() + + def digest(self, event: chalice.app.SNSEvent) -> None: + email_body = self._format_to_html_body(json.loads(event.message)) + self.email.notify_group(subject=event.subject, html=email_body) + + def _format_to_html_body(self, body: dict) -> str: + trigger: dict = body['Trigger'] + metric: dict = one([m for m in trigger.pop('Metrics') + if 'MetricStat' in m])['MetricStat']['Metric'] + alarm_name = body['AlarmName'] + trigger = { + k: v if type(v) is str else str(v) + for k, v in trigger.items() + } + return f''' + +

+ The alarm {alarm_name!r} is in {body['NewStateValue']!r} state.
{body['NewStateReason']} +

+ {''.join(['

' + h[0] + '
' + h[1] + '

' + for h in (('Alarm State', body['NewStateValue']), + ('Alarm Name', alarm_name), + ('Timestamp', body['StateChangeTime']), + ('Namespace', metric.get('Namespace')), + ('Metric Name', metric.get('MetricName')), + *(trigger.items()), + )])} + + ''' diff --git a/src/azul/indexer/notify_service.py b/src/azul/indexer/notify_service.py new file mode 100644 index 0000000000..5f241f2c8f --- /dev/null +++ b/src/azul/indexer/notify_service.py @@ -0,0 +1,31 @@ +from azul import ( + JSON, + config, +) +from azul.deployment import ( + aws, +) + + +class AzulEmailNotificationService: + + def notify_group(self, subject: str, html: str) -> None: + aws.ses.send_email( + Source=f'NOTIFY@{config.indexer_endpoint.host}', + Destination={ + 'ToAddresses': [config.monitoring_email] + }, + Message=self._format_message(subject, html) + ) + + def _format_message(self, subject: str, html: str) -> JSON: + return { + 'Subject': { + 'Data': subject + }, + 'Body': { + 'Html': { + 'Data': html + } + } + } diff --git a/terraform/api_gateway.tf.json.template.py b/terraform/api_gateway.tf.json.template.py index 8cd604f61a..7d6ed3bbd3 100644 --- a/terraform/api_gateway.tf.json.template.py +++ b/terraform/api_gateway.tf.json.template.py @@ -317,7 +317,52 @@ def for_domain(cls, domain): 'sampled_requests_enabled': True, } } - } + }, + **( + ( + { + 'aws_route53_record': { + 'notify_amazonses_record': { + 'zone_id': '${data.aws_route53_zone.dev_singlecell_gi_ucsc_edu.id}', + 'name': f'_amazonses.{config.api_lambda_domain("indexer")}', + 'type': 'TXT', + 'ttl': '600', + 'records': [ + '${aws_ses_domain_identity.notify.verification_token}' + ] + } + }, + 'aws_ses_domain_identity': { + 'notify': { + 'domain': config.api_lambda_domain('indexer') + } + }, + 'aws_ses_identity_policy': { + 'notify': { + 'identity': '${aws_ses_domain_identity.notify.arn}', + 'name': config.qualified_resource_name('notify'), + 'policy': json.dumps({ + "Version": "2012-10-17", + "Statement": [ + { + 'Effect': 'Allow', + 'Principal': { + 'AWS': f'arn:aws:sts::{aws.account}:assumed-role/{config.indexer_name}/' + + config.indexer_function_name('notify') + }, + 'Action': [ + 'ses:SendEmail', + 'ses:SendRawEmail' + ], + 'Resource': '${aws_ses_domain_identity.notify.arn}', + } + ] + }) + } + } + } + ) if config.enable_monitoring else {} + ) }, *([ { @@ -327,13 +372,21 @@ def for_domain(cls, domain): 'maximum_retry_attempts': 0 } for function_name in ( - 'forward_alb_logs', - 'forward_s3_logs', + *( + ('forward_alb_logs', 'forward_s3_logs') + if config.enable_log_forwarding else + () + ), + *( + ('notify',) + if config.enable_monitoring else + () + ) ) } } ] - if config.enable_log_forwarding else + if config.enable_log_forwarding or config.enable_monitoring else []), *( { diff --git a/terraform/shared/shared.tf.json.template.py b/terraform/shared/shared.tf.json.template.py index 2712983ea9..756b42a9a8 100644 --- a/terraform/shared/shared.tf.json.template.py +++ b/terraform/shared/shared.tf.json.template.py @@ -160,6 +160,12 @@ def paren(s: str) -> str: vpc.default_vpc_name: { 'default': True } + }, + 'aws_route53_zone': { + 'gitlab': { + 'name': config.domain_name + '.', + 'private_zone': False + } } }, 'resource': {