Route SNS notifications through a Lambda function (#5246)

lambdas/indexer/app.py

@@ -34,6 +34,9 @@
 from azul.indexer.log_forwarding_controller import (
     LogForwardingController,
 )
+from azul.indexer.notify_controller import (
+    NotifyController,
+)
 from azul.logging import (
     configure_app_logging,
 )
@@ -74,6 +77,10 @@ class IndexerApp(AzulChaliceApp, SignatureHelper):
     def health_controller(self):
         return self._controller(HealthController, lambda_name='indexer')
 
+    @cached_property
+    def notify_controller(self):
+        return self._controller(NotifyController)
+
     @cached_property
     def index_controller(self) -> IndexController:
         return self._controller(IndexController)
@@ -97,6 +104,13 @@ def log_forwarder(self, prefix: str):
         else:
             return lambda func: func
 
+    @property
+    def monitoring(self):
+        if config.enable_monitoring:
+            return self.on_sns_message(topic=config.qualified_resource_name('monitoring'))
+        else:
+            return lambda func: func
+
     def _authenticate(self) -> Optional[HMACAuthentication]:
         return self.auth_from_request(self.current_request)
 
@@ -291,3 +305,8 @@ def forward_alb_logs(event: chalice.app.S3Event):
 @app.log_forwarder(config.s3_access_log_path_prefix(deployment=None))
 def forward_s3_logs(event: chalice.app.S3Event):
     app.log_controller.forward_s3_access_logs(event)
+
+
+@app.monitoring
+def notify(event: chalice.app.SNSEvent):
+    app.notify_controller.digest(event)

src/azul/deployment.py

@@ -139,6 +139,10 @@ def securityhub(self):
     def sns(self):
         return self.client('sns')
 
+    @property
+    def ses(self):
+        return self.client('ses')
+
     @property
     def sts(self):
         return self.client('sts')

src/azul/indexer/notify_controller.py

@@ -0,0 +1,21 @@
+import chalice.app
+
+from azul import (
+    cached_property,
+)
+from azul.chalice import (
+    AppController,
+)
+from azul.indexer.notify_service import (
+    AzulEmailNotificationService,
+)
+
+
+class NotifyController(AppController):
+
+    @cached_property
+    def email(self):
+        return AzulEmailNotificationService()
+
+    def digest(self, event: chalice.app.SNSEvent) -> None:
+        self.email.notify_group(subject=event.subject, body=event.message)

src/azul/indexer/notify_service.py

@@ -0,0 +1,26 @@
+from azul import config, JSON
+from azul.deployment import aws
+
+
+class AzulEmailNotificationService:
+
+    def notify_group(self, subject: str, body: str) -> None:
+        aws.ses.send_email(
+            Source=f'notify@{config.indexer_endpoint.host}',
+            Destination={
+                'ToAddresses': [config.monitoring_email]
+            },
+            Message=self._format_message(subject, body)
+        )
+
+    def _format_message(self, subject: str, body: str) -> JSON:
+        return {
+            'Subject': {
+                'Data': subject
+            },
+            'Body': {
+                'Text': {
+                    'Data': body
+                }
+            }
+        }

terraform/api_gateway.tf.json.template.py

@@ -317,7 +317,52 @@ def for_domain(cls, domain):
                         'sampled_requests_enabled': True,
                     }
                 }
-            }
+            },
+            **(
+                (
+                    {
+                        'aws_route53_record': {
+                            'notify_amazonses_record': {
+                                'zone_id': '${data.aws_route53_zone.dev_singlecell_gi_ucsc_edu.id}',
+                                'name': f'_amazonses.{config.api_lambda_domain("indexer")}',
+                                'type': 'TXT',
+                                'ttl': '600',
+                                'records': [
+                                    '${aws_ses_domain_identity.notify.verification_token}'
+                                ]
+                            }
+                        },
+                        'aws_ses_domain_identity': {
+                            'notify': {
+                                'domain': config.api_lambda_domain('indexer')
+                            }
+                        },
+                        'aws_ses_identity_policy': {
+                            'notify': {
+                                'identity': '${aws_ses_domain_identity.notify.arn}',
+                                'name': config.qualified_resource_name('notify'),
+                                'policy': json.dumps({
+                                    "Version": "2012-10-17",
+                                    "Statement": [
+                                        {
+                                            'Effect': 'Allow',
+                                            'Principal': {
+                                                'AWS': f'arn:aws:sts::{aws.account}:assumed-role/{config.indexer_name}/'
+                                                       + config.indexer_function_name('notify')
+                                            },
+                                            'Action': [
+                                                'ses:SendEmail',
+                                                'ses:SendRawEmail'
+                                            ],
+                                            'Resource': '${aws_ses_domain_identity.notify.arn}',
+                                        }
+                                    ]
+                                })
+                            }
+                        }
+                    }
+                ) if config.enable_monitoring else {}
+            )
         },
         *([
               {
@@ -327,13 +372,21 @@ def for_domain(cls, domain):
                           'maximum_retry_attempts': 0
                       }
                       for function_name in (
-                          'forward_alb_logs',
-                          'forward_s3_logs',
+                          *(
+                              ('forward_alb_logs', 'forward_s3_logs')
+                              if config.enable_log_forwarding else
+                              ()
+                          ),
+                          *(
+                              ('notify',)
+                              if config.enable_monitoring else
+                              ()
+                          )
                       )
                   }
               }
           ]
-          if config.enable_log_forwarding else
+          if config.enable_log_forwarding or config.enable_monitoring else
           []),
         *(
             {

terraform/shared/shared.tf.json.template.py

@@ -160,6 +160,12 @@ def paren(s: str) -> str:
             vpc.default_vpc_name: {
                 'default': True
             }
+        },
+        'aws_route53_zone': {
+            'gitlab': {
+                'name': config.domain_name + '.',
+                'private_zone': False
+            }
         }
     },
     'resource': {