[u] Route SNS notifications through a Lambda function (#5246)

UPGRADING.rst

@@ -19,6 +19,16 @@ branch that does not have the listed changes, the steps would need to be
 reverted. This is all fairly informal and loosely defined. Hopefully we won't
 have too many entries in this file.
 
+#5246 Route SNS notifications through a Lambda function
+=======================================================
+
+Operator
+~~~~~~~~
+
+Manually deploy the ``shared`` & ``gitlab`` components (in that order)
+of any main deployment just before pushing the merge commit to the
+GitLab instance in that deployment.
+
 
 DataBiosphere/azul-private#95 Resolve vulnerabilities in AMI for GitLab
 =======================================================================

lambdas/indexer/app.py

@@ -34,6 +34,9 @@
 from azul.indexer.log_forwarding_controller import (
     LogForwardingController,
 )
+from azul.indexer.notify_controller import (
+    NotifyController,
+)
 from azul.logging import (
     configure_app_logging,
 )
@@ -74,6 +77,10 @@ class IndexerApp(AzulChaliceApp, SignatureHelper):
     def health_controller(self):
         return self._controller(HealthController, lambda_name='indexer')
 
+    @cached_property
+    def notify_controller(self):
+        return self._controller(NotifyController)
+
     @cached_property
     def index_controller(self) -> IndexController:
         return self._controller(IndexController)
@@ -97,6 +104,13 @@ def log_forwarder(self, prefix: str):
         else:
             return lambda func: func
 
+    @property
+    def monitoring(self):
+        if config.enable_monitoring:
+            return self.on_sns_message(topic=config.qualified_resource_name('monitoring'))
+        else:
+            return lambda func: func
+
     def _authenticate(self) -> Optional[HMACAuthentication]:
         return self.auth_from_request(self.current_request)
 
@@ -291,3 +305,8 @@ def forward_alb_logs(event: chalice.app.S3Event):
 @app.log_forwarder(config.s3_access_log_path_prefix(deployment=None))
 def forward_s3_logs(event: chalice.app.S3Event):
     app.log_controller.forward_s3_access_logs(event)
+
+
+@app.monitoring
+def notify(event: chalice.app.SNSEvent):
+    app.notify_controller.notify_group(event)

src/azul/deployment.py

@@ -139,6 +139,10 @@ def securityhub(self):
     def sns(self):
         return self.client('sns')
 
+    @property
+    def ses(self):
+        return self.client('sesv2')
+
     @property
     def sts(self):
         return self.client('sts')

src/azul/indexer/notify_controller.py

@@ -0,0 +1,21 @@
+import chalice.app
+
+from azul import (
+    cached_property,
+)
+from azul.chalice import (
+    AppController,
+)
+from azul.indexer.notify_service import (
+    AzulEmailNotificationService,
+)
+
+
+class NotifyController(AppController):
+
+    @cached_property
+    def service(self):
+        return AzulEmailNotificationService()
+
+    def notify_group(self, event: chalice.app.SNSEvent) -> None:
+        self.service.notify_group(event.subject, event.message)

src/azul/indexer/notify_service.py

@@ -0,0 +1,46 @@
+import json
+import logging
+
+from azul import (
+    JSON,
+    config,
+)
+from azul.deployment import (
+    aws,
+)
+from azul.strings import (
+    trunc_ellipses,
+)
+
+log = logging.getLogger(__name__)
+
+
+class AzulEmailNotificationService:
+
+    def notify_group(self, subject: str, message: str) -> None:
+        log.info('Notifying group of event %r', trunc_ellipses(message, 256))
+        # `message` is deserialized to a Python object to be re-serialized as a
+        # printable JSON
+        body = json.loads(message)
+        response = aws.ses.send_email(
+            FromEmailAddress=f'notify@{config.domain_name}',
+            Destination={
+                'ToAddresses': [config.monitoring_email]
+            },
+            Content=self._format_content(subject, json.dumps(body, indent=4))
+        )
+        log.info('Notification sent, %r', response['MessageId'])
+
+    def _format_content(self, subject: str, body: str) -> JSON:
+        return {
+            'Simple': {
+                'Subject': {
+                    'Data': subject
+                },
+                'Body': {
+                    'Text': {
+                        'Data': body
+                    }
+                }
+            }
+        }

terraform/api_gateway.tf.json.template.py

@@ -327,13 +327,21 @@ def for_domain(cls, domain):
                           'maximum_retry_attempts': 0
                       }
                       for function_name in (
-                          'forward_alb_logs',
-                          'forward_s3_logs',
+                          *(
+                              ('forward_alb_logs', 'forward_s3_logs')
+                              if config.enable_log_forwarding else
+                              ()
+                          ),
+                          *(
+                              ('notify',)
+                              if config.enable_monitoring else
+                              ()
+                          )
                       )
                   }
               }
           ]
-          if config.enable_log_forwarding else
+          if config.enable_log_forwarding or config.enable_monitoring else
           []),
         *(
             {

terraform/gitlab/gitlab.tf.json.template.py

@@ -385,6 +385,11 @@ def qq(*words):
                 'private_zone': False
             }
         },
+        'aws_ses_domain_identity': {
+            'notify': {
+                'domain': config.domain_name
+            }
+        },
         'aws_s3_bucket': {
             'logs': {
                 'bucket': aws.logs_bucket,
@@ -1141,6 +1146,13 @@ def qq(*words):
                     'zone_id': '${aws_lb.gitlab_nlb.zone_id}',
                     'evaluate_target_health': False
                 }
+            },
+            'notify_ses': {
+                'zone_id': '${data.aws_route53_zone.gitlab.id}',
+                'name': f'_amazonses.{config.domain_name}',
+                'type': 'TXT',
+                'ttl': '600',
+                'records': ['${data.aws_ses_domain_identity.notify.verification_token}']
             }
         },
         'aws_network_interface': {

terraform/shared/shared.tf.json.template.py

@@ -871,6 +871,34 @@ def paren(s: str) -> str:
                 'name': aws.monitoring_topic_name
             }
         },
+        'aws_ses_domain_identity': {
+            'notify': {
+                'domain': config.domain_name
+            }
+        },
+        'aws_ses_identity_policy': {
+            'notify': {
+                'identity': '${aws_ses_domain_identity.notify.arn}',
+                'name': config.qualified_resource_name('notify'),
+                'policy': json.dumps({
+                    "Version": "2012-10-17",
+                    "Statement": [
+                        {
+                            'Effect': 'Allow',
+                            'Principal': {
+                                'AWS': f'arn:aws:sts::{aws.account}:assumed-role/{config.indexer_name}/'
+                                       + config.indexer_function_name('notify')
+                            },
+                            'Action': [
+                                'ses:SendEmail',
+                                'ses:SendRawEmail'
+                            ],
+                            'Resource': '${aws_ses_domain_identity.notify.arn}',
+                        }
+                    ]
+                })
+            }
+        },
         'aws_sns_topic_subscription': {
             'monitoring': {
                 'topic_arn': '${aws_sns_topic.monitoring.arn}',