Skip to content

Commit

Permalink
test(backup): Create backup version snapshot tests (#58173)
Browse files Browse the repository at this point in the history
The goal for the export.json feature is to support two minor versions
(since we're using CalVer, roughly equivalent to two months) of releases
when importing. That is to say, if the latest release is 23.10.0, and
the previous two releases were 23.9.0 and 23.8.0, then a self-hosted
release running code from [email protected] or self-hosted@latest (aka
nightly) should be able to successfully import JSON backup data
generated by [email protected] and [email protected], but not
necessarily [email protected].

To support this, we add a new file `test_releases.py` that provides a
snapshot test of an "exhaustive" (aka, has at least one of every
exportable model) export. Every time we cut a new release, we will add a
new commit directly after the release commit that renames the
`test_at_head` snapshot to `test_at_<RELEASED_VERSION>`, and creates a
test case for that version (see `test_at_23_9_1` in this PR for one such
example). Test cases older than the 2 version support window will
simultaneously be removed.

In the future, we plan to automate this "release test rotation" process
via GitHub actions, but for now the ospo team will take care to do this
manually for every release.

Some other notes of interest:
- Exports cannot be compared for equality using simple string
comparisons, since they have special comparators for cases like
passwords, dates that may or may not have been updated, etc (see
comparators.py for greater detail). To accommodate this, we extend the
`insta_snapshot` fixture with an extra argument,
`inequality_comparator`, that allows a custom comparison lambda to be
passed in for such cases.
- The `test_at_23_9_1` snapshot was generated by taking an early version
of this PR, cherry-picking it onto the `23.9.1` release tag, generating
a new snapshot there, then copying the result back into the main commit.

Closes getsentry/team-ospo#197
  • Loading branch information
azaslavsky authored Oct 16, 2023
1 parent 9b30768 commit ceae88a
Show file tree
Hide file tree
Showing 5 changed files with 2,768 additions and 17 deletions.
67 changes: 50 additions & 17 deletions src/sentry/testutils/pytest/fixtures.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@
import sys
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
from typing import Optional
from string import Template
from typing import Optional, Tuple

import pytest
import requests
Expand Down Expand Up @@ -331,9 +332,24 @@ def ignore_aliases(self, data):
return True


def read_snapshot_file(reference_file: str) -> Tuple[str, str]:
with open(reference_file, encoding="utf-8") as f:
match = _yaml_snap_re.match(f.read())
if match is None:
raise OSError()

header, refval = match.groups()
return (header, refval)


@pytest.fixture
def insta_snapshot(request, log):
def inner(output, reference_file=None, subname=None):
def inner(
output,
reference_file=None,
subname=None,
inequality_comparator=lambda refval, output: refval != output,
):
if reference_file is None:
name = request.node.name
for c in UNSAFE_PATH_CHARS:
Expand Down Expand Up @@ -362,18 +378,15 @@ def inner(output, reference_file=None, subname=None):
)

try:
with open(reference_file, encoding="utf-8") as f:
match = _yaml_snap_re.match(f.read())
if match is None:
raise OSError()
_header, refval = match.groups()
_, refval = read_snapshot_file(reference_file)
except OSError:
refval = ""

refval = refval.rstrip()
output = output.rstrip()
is_unequal = inequality_comparator(refval, output)

if _snapshot_writeback is not None and refval != output:
if _snapshot_writeback is not None and is_unequal:
if not os.path.isdir(os.path.dirname(reference_file)):
os.makedirs(os.path.dirname(reference_file))
source = os.path.realpath(str(request.node.fspath))
Expand All @@ -397,27 +410,47 @@ def inner(output, reference_file=None, subname=None):
output,
)
)
elif refval != output:
elif is_unequal:
__tracebackhide__ = True
_print_insta_diff(reference_file, refval, output)
if isinstance(is_unequal, str):
_print_custom_insta_diff(reference_file, is_unequal)
else:
_print_insta_diff(reference_file, refval, output)

yield inner


INSTA_DIFF_TEMPLATE = Template(
"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Snapshot $reference_file changed!
Re-run pytest with SENTRY_SNAPSHOTS_WRITEBACK=new and then use 'make review-python-snapshots' to review.
Or: Use SENTRY_SNAPSHOTS_WRITEBACK=1 to update snapshots directly.
$diff_text
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
"""
)


def _print_insta_diff(reference_file, a, b):
__tracebackhide__ = True
pytest.fail(
"~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"
"Snapshot {} changed!\n\n"
"Re-run pytest with SENTRY_SNAPSHOTS_WRITEBACK=new and then use 'make review-python-snapshots' to review.\n"
"Or: Use SENTRY_SNAPSHOTS_WRITEBACK=1 to update snapshots directly.\n\n"
"{}\n"
"~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n".format(
reference_file, "\n".join(difflib.unified_diff(a.splitlines(), b.splitlines()))
INSTA_DIFF_TEMPLATE.substitute(
reference_file=reference_file,
diff_text="\n".join(difflib.unified_diff(a.splitlines(), b.splitlines())),
)
)


def _print_custom_insta_diff(reference_file, diff_text):
__tracebackhide__ = True
pytest.fail(INSTA_DIFF_TEMPLATE.substitute(reference_file=reference_file, diff_text=diff_text))


@pytest.fixture
def call_snuba(settings):
def inner(endpoint):
Expand Down
Loading

0 comments on commit ceae88a

Please sign in to comment.