[R] Route SNS notifications through a Lambda function (#5246)

DataBiosphere · Nov 30, 2023 · d411ff3 · d411ff3
1 parent e9ce6c9
commit d411ff3
Show file tree

Hide file tree

Showing 11 changed files with 176 additions and 22 deletions.
diff --git a/...eels/runtime/cryptography-41.0.5-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/...eels/runtime/cryptography-41.0.5-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
diff --git a/...eels/runtime/cryptography-41.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/...eels/runtime/cryptography-41.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
diff --git a/lambdas/indexer/app.py b/lambdas/indexer/app.py
@@ -34,6 +34,9 @@
 from azul.indexer.log_forwarding_controller import (
     LogForwardingController,
 )
+from azul.indexer.notification_controller import (
+    NotificationController,
+)
 from azul.logging import (
     configure_app_logging,
 )
@@ -74,6 +77,10 @@ class IndexerApp(AzulChaliceApp, SignatureHelper):
     def health_controller(self):
         return self._controller(HealthController, lambda_name='indexer')
 
+    @cached_property
+    def notification_controller(self):
+        return self._controller(NotificationController)
+
     @cached_property
     def index_controller(self) -> IndexController:
         return self._controller(IndexController)
@@ -97,6 +104,13 @@ def log_forwarder(self, prefix: str):
         else:
             return lambda func: func
 
+    @property
+    def monitoring(self):
+        if config.enable_monitoring:
+            return self.on_sns_message(topic=config.qualified_resource_name('monitoring'))
+        else:
+            return lambda func: func
+
     def _authenticate(self) -> Optional[HMACAuthentication]:
         return self.auth_from_request(self.current_request)
 
@@ -349,3 +363,8 @@ def forward_alb_logs(event: chalice.app.S3Event):
 )
 def forward_s3_logs(event: chalice.app.S3Event):
     app.log_controller.forward_s3_access_logs(event)
+
+
+@app.monitoring
+def notify(event: chalice.app.SNSEvent):
+    app.notification_controller.notify_group(event)
diff --git a/requirements.all.txt b/requirements.all.txt
@@ -11,7 +11,7 @@ blinker==1.7.0
 boto3==1.28.63
 boto3-stubs==1.28.63
 botocore==1.31.63
-botocore-stubs==1.32.6
+botocore-stubs==1.33.0
 brotli==1.1.0
 cachetools==5.3.2
 certifi==2023.11.17
@@ -23,7 +23,7 @@ click==8.1.7
 colorama==0.4.4
 configargparse==1.7
 coverage==6.5.0
-cryptography==41.0.5
+cryptography==41.0.7
 deprecated==1.2.13
 docker==6.1.2
 docutils==0.16
@@ -80,6 +80,7 @@ mypy-boto3-iam==1.28.79
 mypy-boto3-kms==1.28.37
 mypy-boto3-lambda==1.28.83
 mypy-boto3-s3==1.28.55
+mypy-boto3-sesv2==1.28.64
 mypy-boto3-sqs==1.28.82
 mypy-boto3-stepfunctions==1.28.36
 openapi-schema-validator==0.3.4

diff --git a/requirements.dev.trans.txt b/requirements.dev.trans.txt
@@ -1,7 +1,7 @@
 arrow==1.3.0
 blessed==1.20.0
 blinker==1.7.0
-botocore-stubs==1.32.6
+botocore-stubs==1.33.0
 brotli==1.1.0
 click==8.1.7
 colorama==0.4.4
@@ -31,6 +31,7 @@ mypy-boto3-iam==1.28.79
 mypy-boto3-kms==1.28.37
 mypy-boto3-lambda==1.28.83
 mypy-boto3-s3==1.28.55
+mypy-boto3-sesv2==1.28.64
 mypy-boto3-sqs==1.28.82
 mypy-boto3-stepfunctions==1.28.36
 openapi-schema-validator==0.3.4

diff --git a/requirements.dev.txt b/requirements.dev.txt
@@ -12,7 +12,7 @@ google-cloud-bigquery-reservation==1.11.3
 jq==1.3.0
 locust==2.12.2
 moto[s3,sqs,sns,dynamodb,iam]==4.1.13  # match the extras with the backends listed in AzulUnitTestCase._reset_moto
-boto3-stubs[s3,sqs,lambda,dynamodb,iam,ecr,stepfunctions,kms]==1.28.63  # match this with the version of the `boto3` runtime dependency
+boto3-stubs[s3,sqs,lambda,dynamodb,iam,ecr,stepfunctions,kms,sesv2]==1.28.63  # match this with the version of the `boto3` runtime dependency
 openapi-spec-validator==0.5.1
 openpyxl==3.0.6
 posix-ipc==1.1.1

diff --git a/requirements.trans.txt b/requirements.trans.txt
@@ -4,7 +4,7 @@ cachetools==5.3.2
 certifi==2023.11.17
 cffi==1.16.0
 charset-normalizer==3.3.2
-cryptography==41.0.5
+cryptography==41.0.7
 google-cloud-core==2.3.3
 google-crc32c==1.5.0
 google-resumable-media==2.6.0

diff --git a/src/azul/deployment.py b/src/azul/deployment.py
@@ -68,6 +68,9 @@
     from mypy_boto3_s3 import (
         S3Client,
     )
+    from mypy_boto3_sesv2 import (
+        SESV2Client,
+    )
     from mypy_boto3_stepfunctions import (
         SFNClient,
     )
@@ -155,6 +158,10 @@ def securityhub(self):
     def sns(self):
         return self.client('sns')
 
+    @property
+    def ses(self) -> 'SESV2Client':
+        return self.client('sesv2')
+
     @property
     def sts(self):
         return self.client('sts')

diff --git a/src/azul/indexer/notification_controller.py b/src/azul/indexer/notification_controller.py
@@ -0,0 +1,21 @@
+import chalice.app
+
+from azul import (
+    cached_property,
+)
+from azul.chalice import (
+    AppController,
+)
+from azul.indexer.notify_service import (
+    AzulEmailNotificationService,
+)
+
+
+class NotificationController(AppController):
+
+    @cached_property
+    def service(self):
+        return AzulEmailNotificationService()
+
+    def notify_group(self, event: chalice.app.SNSEvent) -> None:
+        self.service.notify_group(event.subject, event.message)
diff --git a/src/azul/indexer/notify_service.py b/src/azul/indexer/notify_service.py
@@ -0,0 +1,56 @@
+import json
+import logging
+
+from azul import (
+    JSON,
+    config,
+)
+from azul.deployment import (
+    aws,
+)
+from azul.strings import (
+    trunc_ellipses,
+)
+
+log = logging.getLogger(__name__)
+
+
+class AzulEmailNotificationService:
+
+    def notify_group(self, subject: str, message: str) -> None:
+        log.info('Notifying group of event %r', trunc_ellipses(message, 256))
+        # Try to improve readability by adding indent
+        try:
+            body = json.loads(message)
+        except json.decoder.JSONDecodeError:
+            log.warning('Not a JSON serializable event, sending as received.')
+            body = message
+        else:
+            body = json.dumps(body, indent=4)
+        response = aws.ses.send_email(
+            FromEmailAddress=' '.join([
+                'Azul',
+                config.deployment_stage,
+                'Monitoring',
+                '<monitoring@' + config.api_lambda_domain('indexer') + '>'
+            ]),
+            Destination={
+                'ToAddresses': [config.monitoring_email]
+            },
+            Content=self._content(subject, body)
+        )
+        log.info('Sent notification %r', response['MessageId'])
+
+    def _content(self, subject: str, body: str) -> JSON:
+        return {
+            'Simple': {
+                'Subject': {
+                    'Data': subject
+                },
+                'Body': {
+                    'Text': {
+                        'Data': body
+                    }
+                }
+            }
+        }
diff --git a/terraform/api_gateway.tf.json.template.py b/terraform/api_gateway.tf.json.template.py
@@ -345,24 +345,26 @@ def for_domain(cls, domain):
                         }
                     }
                 }
+            },
+            'aws_lambda_function_event_invoke_config': {
+                function_name: {
+                    'function_name': '${aws_lambda_function.indexer_%s.function_name}' % function_name,
+                    'maximum_retry_attempts': 0
+                }
+                for function_name in [
+                    *(
+                        ('forward_alb_logs', 'forward_s3_logs')
+                        if config.enable_log_forwarding else
+                        ()
+                    ),
+                    *(
+                        ('notify',)
+                        if config.enable_monitoring else
+                        ()
+                    )
+                ]
             }
         },
-        *([
-              {
-                  'aws_lambda_function_event_invoke_config': {
-                      function_name: {
-                          'function_name': '${aws_lambda_function.indexer_%s.function_name}' % function_name,
-                          'maximum_retry_attempts': 0
-                      }
-                      for function_name in (
-                          'forward_alb_logs',
-                          'forward_s3_logs',
-                      )
-                  }
-              }
-          ]
-          if config.enable_log_forwarding else
-          []),
         *(
             {
                 **chalice.tf_config(app.name)['resource'],
@@ -497,7 +499,19 @@ def for_domain(cls, domain):
                                    }
                                })
                         } for i, domain in enumerate(app.domains)
-                    }
+                    },
+                    **(
+                        {
+                            'notify_ses': {
+                                'zone_id': '${data.aws_route53_zone.%s.id}' % zones_by_domain[app.domains[0]].slug,
+                                'name': '_amazonses.' + config.api_lambda_domain(app.name),
+                                'type': 'TXT',
+                                'ttl': '600',
+                                'records': ['${aws_ses_domain_identity.notify.verification_token}']
+                            }
+                        } if app.name == 'indexer' and config.enable_monitoring else
+                        {}
+                    )
                 },
                 'aws_cloudwatch_log_group': {
                     app.name: {
@@ -619,6 +633,41 @@ def for_domain(cls, domain):
                         }
                     )
                 },
+                **(
+                    {
+                        'aws_ses_domain_identity': {
+                            'notify': {
+                                'domain': config.api_lambda_domain(app.name)
+                            }
+                        },
+                        'aws_ses_identity_policy': {
+                            'notify': {
+                                'identity': '${aws_ses_domain_identity.notify.arn}',
+                                'name': config.qualified_resource_name('notify'),
+                                'policy': json.dumps({
+                                    'Version': '2012-10-17',
+                                    'Statement': [
+                                        {
+                                            'Effect': 'Allow',
+                                            'Principal': {
+                                                'AWS': 'arn:aws:sts::'
+                                                       + aws.account
+                                                       + ':assumed-role/'
+                                                       + config.qualified_resource_name(app.name) + '/'
+                                                       + config.qualified_resource_name(app.name, '-notify')
+                                            },
+                                            'Action': [
+                                                'ses:SendEmail',
+                                                'ses:SendRawEmail'
+                                            ],
+                                            'Resource': '${aws_ses_domain_identity.notify.arn}',
+                                        }
+                                    ]
+                                })
+                            }
+                        }
+                    } if app.name == 'indexer' and config.enable_monitoring else {}
+                ),
                 **(
                     {
                         'aws_lb': {