Skip to content

Commit

Permalink
Fix leaky usage of time machine (#832)
Browse files Browse the repository at this point in the history
  • Loading branch information
@michelletran-codecov
michelletran-codecov authored Oct 29, 2024
1 parent 19cbe90 commit bcb07ef
Show file tree
Hide file tree
Showing 2 changed files with 177 additions and 181 deletions.
83 changes: 38 additions & 45 deletions tasks/tests/unit/test_process_flakes.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -443,57 +443,50 @@ def test_it_creates_flakes_expires(transactional_db):


def test_it_creates_rollups(transactional_db):
traveller = time_machine.travel("1970-1-1T00:00:00Z")
traveller.start()
rs = RepoSimulator()
commits = []
c1 = rs.create_commit()
rs.merge(c1)
rs.add_test_instance(c1, outcome=TestInstance.Outcome.FAILURE.value)
rs.add_test_instance(c1, outcome=TestInstance.Outcome.FAILURE.value)

ProcessFlakesTask().run_impl(
None,
repo_id=rs.repo.repoid,
commit_id_list=[c1.commitid],
branch=rs.repo.branch,
)

traveller.stop()

traveller = time_machine.travel("1970-1-2T00:00:00Z")
traveller.start()
with time_machine.travel("1970-1-1T00:00:00Z"):
rs = RepoSimulator()
commits = []
c1 = rs.create_commit()
rs.merge(c1)
rs.add_test_instance(c1, outcome=TestInstance.Outcome.FAILURE.value)
rs.add_test_instance(c1, outcome=TestInstance.Outcome.FAILURE.value)

c2 = rs.create_commit()
rs.merge(c2)
rs.add_test_instance(c2, outcome=TestInstance.Outcome.FAILURE.value)
rs.add_test_instance(c2, outcome=TestInstance.Outcome.FAILURE.value)
ProcessFlakesTask().run_impl(
None,
repo_id=rs.repo.repoid,
commit_id_list=[c1.commitid],
branch=rs.repo.branch,
)

ProcessFlakesTask().run_impl(
None,
repo_id=rs.repo.repoid,
commit_id_list=[c2.commitid],
branch=rs.repo.branch,
)
with time_machine.travel("1970-1-2T00:00:00Z"):
c2 = rs.create_commit()
rs.merge(c2)
rs.add_test_instance(c2, outcome=TestInstance.Outcome.FAILURE.value)
rs.add_test_instance(c2, outcome=TestInstance.Outcome.FAILURE.value)

rollups = DailyTestRollup.objects.all()
ProcessFlakesTask().run_impl(
None,
repo_id=rs.repo.repoid,
commit_id_list=[c2.commitid],
branch=rs.repo.branch,
)

assert len(rollups) == 4
rollups = DailyTestRollup.objects.all()

assert rollups[0].fail_count == 1
assert rollups[0].flaky_fail_count == 1
assert rollups[0].date == dt.date.today() - dt.timedelta(days=1)
assert len(rollups) == 4

assert rollups[1].fail_count == 1
assert rollups[1].flaky_fail_count == 1
assert rollups[1].date == dt.date.today() - dt.timedelta(days=1)
assert rollups[0].fail_count == 1
assert rollups[0].flaky_fail_count == 1
assert rollups[0].date == dt.date.today() - dt.timedelta(days=1)

assert rollups[2].fail_count == 1
assert rollups[2].flaky_fail_count == 1
assert rollups[2].date == dt.date.today()
assert rollups[1].fail_count == 1
assert rollups[1].flaky_fail_count == 1
assert rollups[1].date == dt.date.today() - dt.timedelta(days=1)

assert rollups[3].fail_count == 1
assert rollups[3].flaky_fail_count == 1
assert rollups[3].date == dt.date.today()
assert rollups[2].fail_count == 1
assert rollups[2].flaky_fail_count == 1
assert rollups[2].date == dt.date.today()

traveller.stop()
assert rollups[3].fail_count == 1
assert rollups[3].flaky_fail_count == 1
assert rollups[3].date == dt.date.today()
275 changes: 139 additions & 136 deletions tasks/tests/unit/test_test_results_processor_task.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -560,152 +560,155 @@ def test_upload_processor_task_call_daily_test_totals(
mock_redis,
celery_app,
):
traveller = travel("1970-1-1T00:00:00Z", tick=False)
traveller.start()
first_url = "v4/raw/2019-05-22/C3C4715CA57C910D11D5EB899FC86A7E/4c4e4654ac25037ae869caeb3619d485970b6304/a84d445c-9c1e-434f-8275-f18f1f320f81.txt"
with open(
here.parent.parent / "samples" / "sample_multi_test_part_1.json"
) as f:
content = f.read()
mock_storage.write_file("archive", first_url, content)

first_commit = CommitFactory.create(
message="hello world",
commitid="cd76b0821854a780b60012aed85af0a8263004ad",
repository__owner__unencrypted_oauth_token="test7lk5ndmtqzxlx06rip65nac9c7epqopclnoy",
repository__owner__username="joseph-sentry",
repository__owner__service="github",
repository__name="codecov-demo",
branch="first_branch",
)
dbsession.add(first_commit)
dbsession.flush()

first_report_row = CommitReport(commit_id=first_commit.id_)
dbsession.add(first_report_row)
dbsession.flush()

upload = UploadFactory.create(storage_path=first_url, report=first_report_row)
dbsession.add(upload)
dbsession.flush()

repoid = upload.report.commit.repoid
redis_queue = [{"url": first_url, "upload_pk": upload.id_}]
mocker.patch.object(TestResultsProcessorTask, "app", celery_app)

result = TestResultsProcessorTask().run_impl(
dbsession,
repoid=repoid,
commitid=first_commit.commitid,
commit_yaml={"codecov": {"max_report_age": False}},
arguments_list=redis_queue,
)
expected_result = [
{
"successful": True,
}
]

rollups = dbsession.query(DailyTestRollup).all()

assert [r.branch for r in rollups] == [
"first_branch",
"first_branch",
]

assert [r.date for r in rollups] == [
date.today(),
date.today(),
]

traveller.stop()

traveller = travel("1970-1-2T00:00:00Z", tick=False)
traveller.start()

second_commit = CommitFactory.create(
message="hello world 2",
commitid="bd76b0821854a780b60012aed85af0a8263004ad",
repository=first_commit.repository,
branch="second_branch",
)
dbsession.add(second_commit)
dbsession.flush()

second_report_row = CommitReport(commit_id=second_commit.id_)
dbsession.add(second_report_row)
dbsession.flush()

second_url = "v4/raw/2019-05-22/C3C4715CA57C910D11D5EB899FC86A7E/4c4e4654ac25037ae869caeb3619d485970b6304/b84d445c-9c1e-434f-8275-f18f1f320f81.txt"
with open(
here.parent.parent / "samples" / "sample_multi_test_part_2.json"
) as f:
content = f.read()
mock_storage.write_file("archive", second_url, content)
upload = UploadFactory.create(storage_path=second_url, report=second_report_row)
dbsession.add(upload)
dbsession.flush()

tests = dbsession.query(Test).all()
for test in tests:
flake = FlakeFactory.create(test=test)
dbsession.add(flake)
with travel("1970-1-1T00:00:00Z", tick=False):
first_url = "v4/raw/2019-05-22/C3C4715CA57C910D11D5EB899FC86A7E/4c4e4654ac25037ae869caeb3619d485970b6304/a84d445c-9c1e-434f-8275-f18f1f320f81.txt"
with open(
here.parent.parent / "samples" / "sample_multi_test_part_1.json"
) as f:
content = f.read()
mock_storage.write_file("archive", first_url, content)

first_commit = CommitFactory.create(
message="hello world",
commitid="cd76b0821854a780b60012aed85af0a8263004ad",
repository__owner__unencrypted_oauth_token="test7lk5ndmtqzxlx06rip65nac9c7epqopclnoy",
repository__owner__username="joseph-sentry",
repository__owner__service="github",
repository__name="codecov-demo",
branch="first_branch",
)
dbsession.add(first_commit)
dbsession.flush()

redis_queue = [{"url": second_url, "upload_pk": upload.id_}]
first_report_row = CommitReport(commit_id=first_commit.id_)
dbsession.add(first_report_row)
dbsession.flush()

result = TestResultsProcessorTask().run_impl(
dbsession,
repoid=repoid,
commitid=second_commit.commitid,
commit_yaml={"codecov": {"max_report_age": False}},
arguments_list=redis_queue,
)
expected_result = [
{
"successful": True,
}
]
upload = UploadFactory.create(
storage_path=first_url, report=first_report_row
)
dbsession.add(upload)
dbsession.flush()

rollups: list[DailyTestRollup] = dbsession.query(DailyTestRollup).all()
repoid = upload.report.commit.repoid
redis_queue = [{"url": first_url, "upload_pk": upload.id_}]
mocker.patch.object(TestResultsProcessorTask, "app", celery_app)

assert result == expected_result
result = TestResultsProcessorTask().run_impl(
dbsession,
repoid=repoid,
commitid=first_commit.commitid,
commit_yaml={"codecov": {"max_report_age": False}},
arguments_list=redis_queue,
)
expected_result = [
{
"successful": True,
}
]

rollups = dbsession.query(DailyTestRollup).all()

assert [r.branch for r in rollups] == [
"first_branch",
"first_branch",
]

assert [r.date for r in rollups] == [
date.today(),
date.today(),
]

with travel("1970-1-2T00:00:00Z", tick=False):
second_commit = CommitFactory.create(
message="hello world 2",
commitid="bd76b0821854a780b60012aed85af0a8263004ad",
repository=first_commit.repository,
branch="second_branch",
)
dbsession.add(second_commit)
dbsession.flush()

assert [r.branch for r in rollups] == [
"first_branch",
"first_branch",
"second_branch",
"second_branch",
]
second_report_row = CommitReport(commit_id=second_commit.id_)
dbsession.add(second_report_row)
dbsession.flush()

assert [r.date for r in rollups] == [
date.today() - timedelta(days=1),
date.today() - timedelta(days=1),
date.today(),
date.today(),
]
second_url = "v4/raw/2019-05-22/C3C4715CA57C910D11D5EB899FC86A7E/4c4e4654ac25037ae869caeb3619d485970b6304/b84d445c-9c1e-434f-8275-f18f1f320f81.txt"
with open(
here.parent.parent / "samples" / "sample_multi_test_part_2.json"
) as f:
content = f.read()
mock_storage.write_file("archive", second_url, content)
upload = UploadFactory.create(
storage_path=second_url, report=second_report_row
)
dbsession.add(upload)
dbsession.flush()

assert [r.fail_count for r in rollups] == [1, 0, 0, 1]
assert [r.pass_count for r in rollups] == [1, 1, 2, 0]
assert [r.skip_count for r in rollups] == [0, 0, 0, 0]
assert [r.flaky_fail_count for r in rollups] == [0, 0, 0, 1]
tests = dbsession.query(Test).all()
for test in tests:
flake = FlakeFactory.create(test=test)
dbsession.add(flake)
dbsession.flush()

assert [r.commits_where_fail for r in rollups] == [
["cd76b0821854a780b60012aed85af0a8263004ad"],
[],
[],
["bd76b0821854a780b60012aed85af0a8263004ad"],
]
redis_queue = [{"url": second_url, "upload_pk": upload.id_}]

assert [r.latest_run for r in rollups] == [
datetime(1970, 1, 1, 0, 0),
datetime(1970, 1, 1, 0, 0),
datetime(1970, 1, 2, 0, 0),
datetime(1970, 1, 2, 0, 0),
]
assert [r.avg_duration_seconds for r in rollups] == [0.001, 7.2, 0.002, 3.6]
assert [r.last_duration_seconds for r in rollups] == [0.001, 7.2, 0.002, 3.6]
traveller.stop()
result = TestResultsProcessorTask().run_impl(
dbsession,
repoid=repoid,
commitid=second_commit.commitid,
commit_yaml={"codecov": {"max_report_age": False}},
arguments_list=redis_queue,
)
expected_result = [
{
"successful": True,
}
]

rollups: list[DailyTestRollup] = dbsession.query(DailyTestRollup).all()

assert result == expected_result

assert [r.branch for r in rollups] == [
"first_branch",
"first_branch",
"second_branch",
"second_branch",
]

assert [r.date for r in rollups] == [
date.today() - timedelta(days=1),
date.today() - timedelta(days=1),
date.today(),
date.today(),
]

assert [r.fail_count for r in rollups] == [1, 0, 0, 1]
assert [r.pass_count for r in rollups] == [1, 1, 2, 0]
assert [r.skip_count for r in rollups] == [0, 0, 0, 0]
assert [r.flaky_fail_count for r in rollups] == [0, 0, 0, 1]

assert [r.commits_where_fail for r in rollups] == [
["cd76b0821854a780b60012aed85af0a8263004ad"],
[],
[],
["bd76b0821854a780b60012aed85af0a8263004ad"],
]

assert [r.latest_run for r in rollups] == [
datetime(1970, 1, 1, 0, 0),
datetime(1970, 1, 1, 0, 0),
datetime(1970, 1, 2, 0, 0),
datetime(1970, 1, 2, 0, 0),
]
assert [r.avg_duration_seconds for r in rollups] == [0.001, 7.2, 0.002, 3.6]
assert [r.last_duration_seconds for r in rollups] == [
0.001,
7.2,
0.002,
3.6,
]

@pytest.mark.integration
@pytest.mark.parametrize(
Expand Down

0 comments on commit bcb07ef

Please sign in to comment.