From 9e96c499c0b15b5fed971537c45417f49beaa755 Mon Sep 17 00:00:00 2001 From: Abraham Chavez Date: Thu, 24 Aug 2023 19:03:33 -0700 Subject: [PATCH] [u] Route SNS notifications through a Lambda function (#5246) --- UPGRADING.rst | 10 +++++ lambdas/indexer/app.py | 19 +++++++++ src/azul/deployment.py | 4 ++ src/azul/indexer/notify_controller.py | 21 ++++++++++ src/azul/indexer/notify_service.py | 46 +++++++++++++++++++++ terraform/api_gateway.tf.json.template.py | 14 +++++-- terraform/gitlab/gitlab.tf.json.template.py | 12 ++++++ terraform/shared/shared.tf.json.template.py | 28 +++++++++++++ 8 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 src/azul/indexer/notify_controller.py create mode 100644 src/azul/indexer/notify_service.py diff --git a/UPGRADING.rst b/UPGRADING.rst index 0817bce79e..83b66b3c6f 100644 --- a/UPGRADING.rst +++ b/UPGRADING.rst @@ -19,6 +19,16 @@ branch that does not have the listed changes, the steps would need to be reverted. This is all fairly informal and loosely defined. Hopefully we won't have too many entries in this file. +#5246 Route SNS notifications through a Lambda function +======================================================= + +Operator +~~~~~~~~ + +Manually deploy the ``shared`` & ``gitlab`` components (in that order) +of any main deployment just before pushing the merge commit to the +GitLab instance in that deployment. + DataBiosphere/azul-private#95 Resolve vulnerabilities in AMI for GitLab ======================================================================= diff --git a/lambdas/indexer/app.py b/lambdas/indexer/app.py index 16421c7f97..e83a63c0b8 100644 --- a/lambdas/indexer/app.py +++ b/lambdas/indexer/app.py @@ -34,6 +34,9 @@ from azul.indexer.log_forwarding_controller import ( LogForwardingController, ) +from azul.indexer.notify_controller import ( + NotifyController, +) from azul.logging import ( configure_app_logging, ) @@ -74,6 +77,10 @@ class IndexerApp(AzulChaliceApp, SignatureHelper): def health_controller(self): return self._controller(HealthController, lambda_name='indexer') + @cached_property + def notify_controller(self): + return self._controller(NotifyController) + @cached_property def index_controller(self) -> IndexController: return self._controller(IndexController) @@ -97,6 +104,13 @@ def log_forwarder(self, prefix: str): else: return lambda func: func + @property + def monitoring(self): + if config.enable_monitoring: + return self.on_sns_message(topic=config.qualified_resource_name('monitoring')) + else: + return lambda func: func + def _authenticate(self) -> Optional[HMACAuthentication]: return self.auth_from_request(self.current_request) @@ -291,3 +305,8 @@ def forward_alb_logs(event: chalice.app.S3Event): @app.log_forwarder(config.s3_access_log_path_prefix(deployment=None)) def forward_s3_logs(event: chalice.app.S3Event): app.log_controller.forward_s3_access_logs(event) + + +@app.monitoring +def notify(event: chalice.app.SNSEvent): + app.notify_controller.notify_group(event) diff --git a/src/azul/deployment.py b/src/azul/deployment.py index 663bbde610..19be057983 100644 --- a/src/azul/deployment.py +++ b/src/azul/deployment.py @@ -139,6 +139,10 @@ def securityhub(self): def sns(self): return self.client('sns') + @property + def ses(self): + return self.client('sesv2') + @property def sts(self): return self.client('sts') diff --git a/src/azul/indexer/notify_controller.py b/src/azul/indexer/notify_controller.py new file mode 100644 index 0000000000..9437bc6bca --- /dev/null +++ b/src/azul/indexer/notify_controller.py @@ -0,0 +1,21 @@ +import chalice.app + +from azul import ( + cached_property, +) +from azul.chalice import ( + AppController, +) +from azul.indexer.notify_service import ( + AzulEmailNotificationService, +) + + +class NotifyController(AppController): + + @cached_property + def service(self): + return AzulEmailNotificationService() + + def notify_group(self, event: chalice.app.SNSEvent) -> None: + self.service.notify_group(event.subject, event.message) diff --git a/src/azul/indexer/notify_service.py b/src/azul/indexer/notify_service.py new file mode 100644 index 0000000000..0f49f5f25c --- /dev/null +++ b/src/azul/indexer/notify_service.py @@ -0,0 +1,46 @@ +import json +import logging + +from azul import ( + JSON, + config, +) +from azul.deployment import ( + aws, +) +from azul.strings import ( + trunc_ellipses, +) + +log = logging.getLogger(__name__) + + +class AzulEmailNotificationService: + + def notify_group(self, subject: str, message: str) -> None: + log.info('Notifying group of event %r', trunc_ellipses(message, 256)) + # `message` is deserialized to a Python object to be re-serialized as a + # printable JSON + body = json.loads(message) + response = aws.ses.send_email( + FromEmailAddress=f'notify@{config.domain_name}', + Destination={ + 'ToAddresses': [config.monitoring_email] + }, + Content=self._format_content(subject, json.dumps(body, indent=4)) + ) + log.info('Notification sent, %r', response['MessageId']) + + def _format_content(self, subject: str, body: str) -> JSON: + return { + 'Simple': { + 'Subject': { + 'Data': subject + }, + 'Body': { + 'Text': { + 'Data': body + } + } + } + } diff --git a/terraform/api_gateway.tf.json.template.py b/terraform/api_gateway.tf.json.template.py index 8cd604f61a..3a1e8c936a 100644 --- a/terraform/api_gateway.tf.json.template.py +++ b/terraform/api_gateway.tf.json.template.py @@ -327,13 +327,21 @@ def for_domain(cls, domain): 'maximum_retry_attempts': 0 } for function_name in ( - 'forward_alb_logs', - 'forward_s3_logs', + *( + ('forward_alb_logs', 'forward_s3_logs') + if config.enable_log_forwarding else + () + ), + *( + ('notify',) + if config.enable_monitoring else + () + ) ) } } ] - if config.enable_log_forwarding else + if config.enable_log_forwarding or config.enable_monitoring else []), *( { diff --git a/terraform/gitlab/gitlab.tf.json.template.py b/terraform/gitlab/gitlab.tf.json.template.py index f999cf8f41..2022f3573f 100644 --- a/terraform/gitlab/gitlab.tf.json.template.py +++ b/terraform/gitlab/gitlab.tf.json.template.py @@ -385,6 +385,11 @@ def qq(*words): 'private_zone': False } }, + 'aws_ses_domain_identity': { + 'notify': { + 'domain': config.domain_name + } + }, 'aws_s3_bucket': { 'logs': { 'bucket': aws.logs_bucket, @@ -1141,6 +1146,13 @@ def qq(*words): 'zone_id': '${aws_lb.gitlab_nlb.zone_id}', 'evaluate_target_health': False } + }, + 'notify_ses': { + 'zone_id': '${data.aws_route53_zone.gitlab.id}', + 'name': f'_amazonses.{config.domain_name}', + 'type': 'TXT', + 'ttl': '600', + 'records': ['${data.aws_ses_domain_identity.notify.verification_token}'] } }, 'aws_network_interface': { diff --git a/terraform/shared/shared.tf.json.template.py b/terraform/shared/shared.tf.json.template.py index 2712983ea9..1e2f525934 100644 --- a/terraform/shared/shared.tf.json.template.py +++ b/terraform/shared/shared.tf.json.template.py @@ -871,6 +871,34 @@ def paren(s: str) -> str: 'name': aws.monitoring_topic_name } }, + 'aws_ses_domain_identity': { + 'notify': { + 'domain': config.domain_name + } + }, + 'aws_ses_identity_policy': { + 'notify': { + 'identity': '${aws_ses_domain_identity.notify.arn}', + 'name': config.qualified_resource_name('notify'), + 'policy': json.dumps({ + "Version": "2012-10-17", + "Statement": [ + { + 'Effect': 'Allow', + 'Principal': { + 'AWS': f'arn:aws:sts::{aws.account}:assumed-role/{config.indexer_name}/' + + config.indexer_function_name('notify') + }, + 'Action': [ + 'ses:SendEmail', + 'ses:SendRawEmail' + ], + 'Resource': '${aws_ses_domain_identity.notify.arn}', + } + ] + }) + } + }, 'aws_sns_topic_subscription': { 'monitoring': { 'topic_arn': '${aws_sns_topic.monitoring.arn}',