Route SNS notifications through a Lambda function (#5246)

.github/workflows/ci.yml

@@ -28,6 +28,8 @@ jobs:
           _link dev
           _refresh
 
+          export azul_aws_account_name=test-hca-dev
+
           make virtualenv
           source .venv/bin/activate
           make requirements

README.md

@@ -670,6 +670,23 @@ template files. Changes to either the templates or anything in the `lambdas`
 directory requires running `make deploy` again in order to update cloud
 infrastructure for the selected deployment.
 
+### 3.3.1 Move Amazon SES out of sandbox for deployments with monitoring enabled
+
+Before proceeding, ensure that the SES identity provisioned by Terraform is
+listed in the Verified identities tab, from the AWS SES console. The identity
+listed should be the deployments' domain name.
+
+Once the previous requirement is satisfied, run the following AWS CLI command to
+request for the AWS SES Identity to be granted production access::
+
+    aws sesv2 put-account-details \
+    --contact-language EN \
+    --mail-type TRANSACTIONAL \
+    --production-access-enabled \
+    --website-url $(python -m azul 'config.api_lambda_domain("notify")') \
+    --use-case-description \
+        'Use a lambda function invoked by an SNS topic to forward the SNS notification via email to a single recipient.'
+
 
 ## 3.4 Creating the Elasticsearch indices
 

UPGRADING.rst

@@ -20,6 +20,18 @@ reverted. This is all fairly informal and loosely defined. Hopefully we won't
 have too many entries in this file.
 
 
+#5246 Route SNS notifications through a Lambda function
+=======================================================
+
+Operator
+~~~~~~~~
+
+After the merge commit builds on GitLab, follow the instructions in the
+README.md, for section 3.3.1 titled `Move Amazon SES out of sandbox for
+deployments with monitoring enabled`. This needs to be done for all main
+deployments, except for ``dev``.
+
+
 #6446 Base image of Azul image is not pinned to digest
 ======================================================
 

environment.py

@@ -154,6 +154,10 @@ def env() -> Mapping[str, Optional[str]]:
         #
         'AZUL_AWS_ACCOUNT_ID': None,
 
+        # Description
+        #
+        # Not set explicitly, do not modify for test and unittest usage only
+        'azul_aws_account_name': None,
         # The region of the Azul deployment. This variable is primarily used by
         # the AWS CLI, by TerraForm, botocore and boto3 but Azul references it
         # too. This variable is typically set in deployment-specific

lambdas/indexer/app.py

@@ -35,6 +35,9 @@
 from azul.indexer.log_forwarding_controller import (
     LogForwardingController,
 )
+from azul.indexer.notification_controller import (
+    MonitoringController,
+)
 from azul.logging import (
     configure_app_logging,
 )
@@ -75,6 +78,10 @@ class IndexerApp(AzulChaliceApp, SignatureHelper):
     def health_controller(self):
         return self._controller(HealthController, lambda_name='indexer')
 
+    @cached_property
+    def monitoring_controller(self):
+        return self._controller(MonitoringController)
+
     @cached_property
     def index_controller(self) -> IndexController:
         return self._controller(IndexController)
@@ -108,6 +115,13 @@ def decorator(f):
         else:
             return lambda func: func
 
+    @property
+    def monitoring_sns_handler(self):
+        if config.deployment.is_main:
+            return self.on_sns_message(topic=aws.monitoring_topic_name)
+        else:
+            return lambda func: func
+
     def _authenticate(self) -> Optional[HMACAuthentication]:
         return self.auth_from_request(self.current_request)
 
@@ -367,3 +381,8 @@ def forward_alb_logs(event: chalice.app.S3Event):
 )
 def forward_s3_logs(event: chalice.app.S3Event):
     app.log_controller.forward_s3_access_logs(event)
+
+
+@app.monitoring_sns_handler
+def notify(event: chalice.app.SNSEvent):
+    app.monitoring_controller.notify_group(event)

requirements.all.txt

@@ -80,6 +80,7 @@ mypy-boto3-iam==1.28.79
 mypy-boto3-kms==1.28.37
 mypy-boto3-lambda==1.28.83
 mypy-boto3-s3==1.28.55
+mypy-boto3-sesv2==1.28.64
 mypy-boto3-sqs==1.28.82
 mypy-boto3-stepfunctions==1.28.36
 openapi-schema-validator==0.3.4

requirements.dev.trans.txt

@@ -31,6 +31,7 @@ mypy-boto3-iam==1.28.79
 mypy-boto3-kms==1.28.37
 mypy-boto3-lambda==1.28.83
 mypy-boto3-s3==1.28.55
+mypy-boto3-sesv2==1.28.64
 mypy-boto3-sqs==1.28.82
 mypy-boto3-stepfunctions==1.28.36
 openapi-schema-validator==0.3.4

requirements.dev.txt

@@ -12,7 +12,7 @@ google-cloud-bigquery-reservation==1.11.3
 jq==1.3.0
 locust==2.12.2
 moto[s3,sqs,sns,dynamodb,iam]==4.1.13  # match the extras with the backends listed in AzulUnitTestCase._reset_moto
-boto3-stubs[s3,sqs,lambda,dynamodb,iam,ecr,stepfunctions,kms]==1.28.63  # match this with the version of the `boto3` runtime dependency
+boto3-stubs[s3,sqs,lambda,dynamodb,iam,ecr,stepfunctions,kms,sesv2]==1.28.63  # match this with the version of the `boto3` runtime dependency
 openapi-spec-validator==0.5.1
 openpyxl==3.0.6
 posix_ipc==1.1.1

src/azul/deployment.py

@@ -33,6 +33,9 @@
     AWSResponse,
 )
 import botocore.credentials
+from botocore.exceptions import (
+    NoCredentialsError,
+)
 import botocore.session
 import botocore.utils
 from more_itertools import (
@@ -71,6 +74,9 @@
     from mypy_boto3_s3 import (
         S3Client,
     )
+    from mypy_boto3_sesv2 import (
+        SESV2Client,
+    )
     from mypy_boto3_stepfunctions import (
         SFNClient,
     )
@@ -158,6 +164,10 @@ def securityhub(self):
     def sns(self):
         return self.client('sns')
 
+    @property
+    def ses(self) -> 'SESV2Client':
+        return self.client('sesv2')
+
     @property
     def sts(self):
         return self.client('sts')
@@ -667,8 +677,15 @@ def _validate_bucket_path_prefix(self, path_prefix):
 
     @property
     def monitoring_topic_name(self):
+        try:
+            stage = config.main_deployment_stage
+        except NoCredentialsError:
+            # Running `make openapi` in GitHub fails since retrieving the main
+            # deployment stage requires making a roundtrip to IAM. The
+            # monitoring topic is used by the monitoring Lambda.
+            stage = config.deployment_stage
         return config.qualified_resource_name('monitoring',
-                                              stage=config.main_deployment_stage)
+                                              stage=stage)
 
 
 aws = AWS()

src/azul/indexer/lambda_iam_policy.py

@@ -83,6 +83,46 @@
                 f"arn:aws:secretsmanager:{aws.region_name}:{aws.account}:secret:*"
             ]
         },
+        {
+            "Effect": "Allow",
+            "Action": [
+                'ses:SendEmail',
+                'ses:SendRawEmail'
+            ],
+            "Resource": [
+                # TODO; compose the arn of resource
+                f"arn:aws:ses:{aws.region_name}:{aws.account}:*"
+            ],
+            'aws_ses_identity_policy': {  # TODO: Move pol to lambda_policy
+                'notify': {
+                    'identity': '${aws_ses_domain_identity.notify.arn}',
+                    'name': config.qualified_resource_name('notify'),
+                    'policy': {
+                        'Version': '2012-10-17',
+                        'Statement': [
+                            {
+                                'Effect': 'Allow',
+                                'Principal': {
+                                    'AWS': 'arn:aws:sts::'
+                                           + aws.account
+                                           + ':assumed-role/'
+                                           + '${aws_lambda_function.' + 'app.name' + '.function_name}/'
+                                           # The following is the role-session-name of the principal
+                                           # assuming the role via an AWS STS AssumeRole operation.
+                                           + '${aws_lambda_function.' + '_'.join(['app.name', 'notify'])
+                                           + '.function_name}'
+                                },
+                                'Action': [
+                                    'ses:SendEmail',
+                                    'ses:SendRawEmail'
+                                ],
+                                'Resource': '${aws_ses_domain_identity.notify.arn}',
+                            }
+                        ]
+                    }
+                }
+            }
+        },
         *(
             [
                 {

src/azul/indexer/notification_controller.py

@@ -0,0 +1,21 @@
+import chalice.app
+
+from azul import (
+    cached_property,
+)
+from azul.chalice import (
+    AppController,
+)
+from azul.indexer.notify_service import (
+    EmailService,
+)
+
+
+class MonitoringController(AppController):
+
+    @cached_property
+    def email_service(self):
+        return EmailService()
+
+    def notify_group(self, event: chalice.app.SNSEvent) -> None:
+        self.email_service.send_message(event.subject, event.message)

src/azul/indexer/notify_service.py

@@ -0,0 +1,57 @@
+import json
+import logging
+
+from azul import (
+    config,
+)
+from azul.deployment import (
+    aws,
+)
+from azul.strings import (
+    trunc_ellipses,
+)
+
+log = logging.getLogger(__name__)
+
+
+class EmailService:
+
+    @property
+    def to_email(self):
+        return config.monitoring_email
+
+    @property
+    def from_email(self):
+        return ' '.join([
+            'Azul',
+            config.deployment_stage,
+            'Monitoring',
+            '<monitoring@' + config.api_lambda_domain('indexer') + '>'
+        ])
+
+    def send_message(self, subject: str, body: str) -> None:
+        log.info('Sending message %r with body %r',
+                 subject, trunc_ellipses(body, 256))
+        try:
+            body = json.loads(body)
+        except json.decoder.JSONDecodeError:
+            log.warning('Not a JSON serializable event, sending as is')
+        else:
+            body = json.dumps(body, indent=4)
+        content = {
+            'Simple': {
+                'Subject': {
+                    'Data': subject
+                },
+                'Body': {
+                    'Text': {
+                        'Data': body
+                    }
+                }
+            }
+        }
+        response = aws.ses.send_email(FromEmailAddress=self.from_email,
+                                      Destination=dict(ToAddresses=[self.to_email]),
+                                      Content=content)
+        log.info('Successfully sent message %r, message ID is %r',
+                 subject, response['MessageId'])

terraform/api_gateway.tf.json.template.py

@@ -545,7 +545,20 @@ def for_domain(cls, domain):
                                    }
                                })
                         } for i, domain in enumerate(app.domains)
-                    }
+                    },
+                    **(
+                        {
+                            'notify': {
+                                'zone_id': '${data.aws_route53_zone.%s.id}' % zones_by_domain[app.domains[0]].slug,
+                                'name': '_amazonses.' + app.domains[0],
+                                'type': 'TXT',
+                                'ttl': '600',
+                                'records': ['${aws_ses_domain_identity.notify.verification_token}']
+                            }
+                        }
+                        if app.name == 'indexer' and config.enable_monitoring else
+                        {}
+                    )
                 },
                 'aws_cloudwatch_log_group': {
                     app.name: {
@@ -667,6 +680,15 @@ def for_domain(cls, domain):
                         }
                     )
                 },
+                **(
+                    {
+                        'aws_ses_domain_identity': {
+                            'notify': {
+                                'domain': app.domains[0]
+                            }
+                        }
+                    } if app.name == 'indexer' and config.enable_monitoring else {}
+                ),
                 **(
                     {
                         'aws_lb': {

terraform/shared/shared.tf.json.template.py

@@ -1,5 +1,4 @@
 import json
-import shlex
 from typing import (
     NamedTuple,
 )
@@ -1004,25 +1003,6 @@ def conformance_pack(name: str) -> str:
                 })
             }
         },
-        'aws_sns_topic_subscription': {
-            'monitoring': {
-                'topic_arn': '${aws_sns_topic.monitoring.arn}',
-                # The `email` protocol is only partially supported. Since
-                # Terraform cannot confirm or delete pending subscriptions
-                # (see link below), we use a separate script for this purpose.
-                # https://registry.terraform.io/providers/hashicorp/aws/4.3.0/docs/resources/sns_topic_subscription#protocol-support
-                'protocol': 'email',
-                'endpoint': config.monitoring_email,
-                'provisioner': {
-                    'local-exec': {
-                        'command': ' '.join(map(shlex.quote, [
-                            'python',
-                            config.project_root + '/scripts/confirm_sns_subscription.py'
-                        ]))
-                    }
-                }
-            }
-        },
         'aws_wafv2_ip_set': {
             # FIXME: Remove once no deployments reference this
             #        https://github.com/DataBiosphere/azul/issues/6244