feat(backup): Relocation failed email

In any case where a relocation definitively fails (not merely retrying a
failed task, but actually marking the relocation as a failure), we send
the owner and/or creator of the relocation an email to this effect.

Issue: getsentry/team-ospo#203

src/sentry/tasks/relocation.py

@@ -11,11 +11,9 @@
 import yaml
 from cryptography.fernet import Fernet
 from django.db import router, transaction
-from django.utils import timezone as get_timezone
 from google.cloud.devtools.cloudbuild_v1 import Build
 from google.cloud.devtools.cloudbuild_v1 import CloudBuildClient as CloudBuildClient
 
-from sentry import options
 from sentry.api.serializers.rest_framework.base import camel_to_snake_case, convert_dict_key_case
 from sentry.backup.dependencies import NormalizedModelName, get_model
 from sentry.backup.exports import export_in_config_scope, export_in_user_scope
@@ -28,7 +26,6 @@
 )
 from sentry.backup.imports import import_in_organization_scope
 from sentry.filestore.gcs import GoogleCloudStorage
-from sentry.http import get_server_hostname
 from sentry.models.files.file import File
 from sentry.models.files.utils import get_storage
 from sentry.models.importchunk import RegionImportChunk
@@ -42,21 +39,21 @@
 )
 from sentry.models.user import User
 from sentry.services.hybrid_cloud.organization import organization_service
-from sentry.services.hybrid_cloud.user.service import user_service
 from sentry.silo import SiloMode
 from sentry.tasks.base import instrumented_task
 from sentry.utils import json
 from sentry.utils.db import atomic_transaction
-from sentry.utils.email.message_builder import MessageBuilder
 from sentry.utils.env import gcp_project_id
 from sentry.utils.relocation import (
     RELOCATION_BLOB_SIZE,
     RELOCATION_FILE_TYPE,
+    EmailKind,
     OrderedTask,
     create_cloudbuild_yaml,
     fail_relocation,
     get_bucket_name,
     retry_task_or_fail_relocation,
+    send_relocation_update_email,
     start_relocation_task,
 )
 
@@ -317,29 +314,14 @@ def preprocessing_scan(uuid: str) -> None:
             # The user's import data looks basically okay - we can use this opportunity to send a
             # "your relocation request has been accepted and is in flight, please give it a few
             # hours" email.
-            msg = MessageBuilder(
-                subject=f"{options.get('mail.subject-prefix')} Your Relocation has Started",
-                template="sentry/emails/relocation-started.txt",
-                html_template="sentry/emails/relocation-started.html",
-                type="relocation.started",
-                context={
-                    "domain": get_server_hostname(),
-                    "datetime": get_timezone.now(),
+            send_relocation_update_email(
+                relocation,
+                EmailKind.STARTED,
+                {
                     "uuid": str(relocation.uuid),
                     "orgs": relocation.want_org_slugs,
                 },
             )
-            email_to = []
-            owner = user_service.get_user(user_id=relocation.owner_id)
-            if owner is not None:
-                email_to.append(owner.email)
-
-            if relocation.owner_id != relocation.creator_id:
-                creator = user_service.get_user(user_id=relocation.creator_id)
-                if creator is not None:
-                    email_to.append(creator.email)
-
-            msg.send_async(to=email_to)
 
 
 @instrumented_task(
@@ -628,9 +610,14 @@ def _update_relocation_validation_attempt(
             if relocation_validation.status < status.value:
                 relocation_validation.status = status.value
                 relocation_validation_attempt.save()
-            return fail_relocation(
-                relocation, task, "Validation could not be completed. Please contact support."
+
+            transaction.on_commit(
+                lambda: fail_relocation(
+                    relocation, task, "Validation could not be completed. Please contact support."
+                ),
+                using=router.db_for_write(Relocation),
             )
+            return
 
         # All remaining statuses are final, so we can update the owning `RelocationValidation` now.
         assert status in {ValidationStatus.INVALID, ValidationStatus.VALID}
@@ -646,9 +633,15 @@ def _update_relocation_validation_attempt(
                 "Validation result: invalid",
                 extra={"uuid": relocation.uuid, "task": task.name},
             )
-            return fail_relocation(
-                relocation, task, "The data you provided failed validation. Please contact support."
+            transaction.on_commit(
+                lambda: fail_relocation(
+                    relocation,
+                    task,
+                    "The data you provided failed validation. Please contact support.",
+                ),
+                using=router.db_for_write(Relocation),
             )
+            return
 
         assert status == ValidationStatus.VALID
         relocation.step = Relocation.Step.IMPORTING.value

src/sentry/templates/sentry/debug/mail/preview.html

@@ -55,6 +55,7 @@
       <optgroup label="Relocation">
         <option value="mail/relocate-account/">Relocate Account</option>
         <option value="mail/relocation-started/">Relocation Started</option>
+        <option value="mail/relocation-failed/">Relocation Failed</option>
       </optgroup>
       <optgroup label="Reports">
         <option value="mail/weekly-reports/">Weekly Report</option>

src/sentry/templates/sentry/emails/relocation_failed.html

@@ -0,0 +1,17 @@
+{% extends "sentry/emails/base.html" %}
+
+{% load i18n %}
+
+{% block main %}
+    <h3>Relocation Failed</h3>
+    <p>Your relocation has failed for the following reason:</p>
+    {%if reason != "" %}
+    <code>
+        {{ reason }}
+    </code>
+    <br>
+    <br>
+    {% endif %}
+    <p>Please <a href="https://help.sentry.io">contact support</a> for further assistance if necessary.</p>
+    <p><small><i>ID: {{ uuid }}</i></small></p>
+{% endblock %}

src/sentry/templates/sentry/emails/relocation_failed.txt

@@ -0,0 +1,9 @@
+Your relocation has failed for the following reason:
+
+{%if reason != "" %}
+{{ reason }}
+{% endif %}
+
+Please contact support at https://help.sentry.io for further assistance if necessary.
+
+ID: {{ uuid }}

src/sentry/utils/relocation.py

@@ -5,13 +5,19 @@
 from enum import Enum, unique
 from functools import lru_cache
 from string import Template
-from typing import Generator, Optional, Tuple
+from typing import Any, Generator, Optional, Tuple
 
+from django.utils import timezone
+
+from sentry import options
 from sentry.backup.dependencies import dependencies, get_model_name, sorted_dependencies
 from sentry.backup.scopes import RelocationScope
+from sentry.http import get_server_hostname
 from sentry.models.files.utils import get_storage
 from sentry.models.relocation import Relocation, RelocationFile
 from sentry.models.user import User
+from sentry.services.hybrid_cloud.user.service import user_service
+from sentry.utils.email.message_builder import MessageBuilder as MessageBuilder
 
 logger = logging.getLogger("sentry.relocation.tasks")
 
@@ -267,6 +273,37 @@ class OrderedTask(Enum):
 )
 
 
+class EmailKind(Enum):
+    STARTED = 0
+    FAILED = 1
+    SUCCEEDED = 2
+
+
+def send_relocation_update_email(
+    relocation: Relocation, email_kind: EmailKind, args: dict[str, Any]
+):
+    name = str(email_kind.name)
+    name_lower = name.lower()
+    msg = MessageBuilder(
+        subject=f"{options.get('mail.subject-prefix')} Your Relocation has {name.capitalize()}",
+        template=f"sentry/emails/relocation-{name_lower}.txt",
+        html_template=f"sentry/emails/relocation-{name_lower}.html",
+        type=f"relocation.{name_lower}",
+        context={"domain": get_server_hostname(), "datetime": timezone.now(), **args},
+    )
+    email_to = []
+    owner = user_service.get_user(user_id=relocation.owner_id)
+    if owner is not None:
+        email_to.append(owner.email)
+
+    if relocation.owner_id != relocation.creator_id:
+        creator = user_service.get_user(user_id=relocation.creator_id)
+        if creator is not None:
+            email_to.append(creator.email)
+
+    msg.send_async(to=email_to)
+
+
 def start_relocation_task(
     uuid: str, step: Relocation.Step, task: OrderedTask, allowed_task_attempts: int
 ) -> Tuple[Optional[Relocation], int]:
@@ -341,7 +378,14 @@ def fail_relocation(relocation: Relocation, task: OrderedTask, reason: str = "")
     relocation.save()
 
     logger.info("Task failed", extra={"uuid": relocation.uuid, "task": task.name, "reason": reason})
-    return
+    send_relocation_update_email(
+        relocation,
+        EmailKind.FAILED,
+        {
+            "uuid": str(relocation.uuid),
+            "reason": reason,
+        },
+    )
 
 
 @contextmanager

src/sentry/web/debug_urls.py

@@ -123,6 +123,7 @@
     re_path(r"^debug/mail/confirm-email/$", sentry.web.frontend.debug.mail.confirm_email),
     re_path(r"^debug/mail/recover-account/$", sentry.web.frontend.debug.mail.recover_account),
     re_path(r"^debug/mail/relocate-account/$", sentry.web.frontend.debug.mail.relocate_account),
+    re_path(r"^debug/mail/relocation-failed/$", sentry.web.frontend.debug.mail.relocation_failed),
     re_path(r"^debug/mail/relocation-started/$", sentry.web.frontend.debug.mail.relocation_started),
     re_path(r"^debug/mail/unable-to-delete-repo/$", DebugUnableToDeleteRepository.as_view()),
     re_path(r"^debug/mail/unable-to-fetch-commits/$", DebugUnableToFetchCommitsEmailView.as_view()),

src/sentry/web/frontend/debug/mail.py

@@ -738,6 +738,20 @@ def relocate_account(request):
     ).render(request)
 
 
+@login_required
+def relocation_failed(request):
+    return MailPreview(
+        html_template="sentry/emails/relocation_failed.html",
+        text_template="sentry/emails/relocation_failed.txt",
+        context={
+            "domain": get_server_hostname(),
+            "datetime": timezone.now(),
+            "uuid": str(uuid.uuid4().hex),
+            "reason": "This is a sample failure reason",
+        },
+    ).render(request)
+
+
 @login_required
 def relocation_started(request):
     return MailPreview(