Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add parent field to records to represent repeat and retry information. #905

Merged
merged 3 commits into from
Dec 29, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions mobly/base_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -720,6 +720,7 @@ def should_retry(record):
retry_name = f'{test_name}_retry_{i+1}'
xpconanfan marked this conversation as resolved.
Show resolved Hide resolved
new_record = records.TestResultRecord(retry_name, self.TAG)
new_record.retry_parent = previous_record
new_record.parent = (previous_record, records.TestParentType.RETRY)
previous_record = self.exec_one_test(retry_name, test_method, new_record)
if not should_retry(previous_record):
break
Expand Down Expand Up @@ -749,16 +750,23 @@ def _exec_one_test_with_repeat(
if max_consecutive_error == 0:
max_consecutive_error = repeat_count

previous_record = None
for i in range(repeat_count):
new_test_name = f'{test_name}_{i}'
record = self.exec_one_test(new_test_name, test_method)
if record.result in [
new_record = records.TestResultRecord(new_test_name, self.TAG)
if i > 0:
new_record.parent = (previous_record, records.TestParentType.REPEAT)
previous_record = self.exec_one_test(
new_test_name, test_method, new_record
)
if previous_record.result in [
records.TestResultEnums.TEST_RESULT_FAIL,
records.TestResultEnums.TEST_RESULT_ERROR,
]:
consecutive_error_count += 1
else:
consecutive_error_count = 0

if consecutive_error_count == max_consecutive_error:
logging.error(
'Repeated test case "%s" has consecutively failed %d iterations, '
Expand Down
28 changes: 23 additions & 5 deletions mobly/records.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,13 @@ class Error(Exception):
"""Raised for errors in record module members."""


class TestParentType(enum.Enum):
"""The type of parent in a chain of executions of the same test."""

REPEAT = 'repeat'
RETRY = 'retry'


def uid(uid):
"""Decorator specifying the unique identifier (UID) of a test case.

Expand Down Expand Up @@ -182,6 +189,7 @@ class TestResultEnums:
RECORD_STACKTRACE = 'Stacktrace'
RECORD_SIGNATURE = 'Signature'
RECORD_RETRY_PARENT = 'Retry Parent'
RECORD_PARENT = 'Parent'
RECORD_POSITION = 'Position'
TEST_RESULT_PASS = 'PASS'
TEST_RESULT_FAIL = 'FAIL'
Expand Down Expand Up @@ -317,9 +325,10 @@ class TestResultRecord:
uid: User-defined unique identifier of the test.
signature: string, unique identifier of a test record, the value is
generated by Mobly.
retry_parent: TestResultRecord, only set for retry iterations. This is the
test result record of the previous retry iteration. Parsers can use this
field to construct the chain of execution for each retried test.
retry_parent: [DEPRECATED] Use the `parent` field instead.
parent: tuple[TestResultRecord, TestParentType], set for multiple iterations
of a test. This is the test result record of the previous iteration.
Parsers can use this field to construct the chain of execution for each test.
termination_signal: ExceptionRecord, the main exception of the test.
extra_errors: OrderedDict, all exceptions occurred during the entire
test lifecycle. The order of occurrence is preserved.
Expand All @@ -334,6 +343,7 @@ def __init__(self, t_name, t_class=None):
self.uid = None
self.signature = None
self.retry_parent = None
self.parent = None
self.termination_signal = None
self.extra_errors = collections.OrderedDict()
self.result = None
Expand Down Expand Up @@ -506,6 +516,14 @@ def to_dict(self):
d[TestResultEnums.RECORD_RETRY_PARENT] = (
self.retry_parent.signature if self.retry_parent else None
)
d[TestResultEnums.RECORD_PARENT] = (
{
'parent': self.parent[0].signature,
'type': self.parent[1].value,
}
if self.parent
else None
)
d[TestResultEnums.RECORD_EXTRAS] = self.extras
d[TestResultEnums.RECORD_DETAILS] = self.details
d[TestResultEnums.RECORD_TERMINATION_SIGNAL_TYPE] = (
Expand Down Expand Up @@ -644,9 +662,9 @@ def _count_eventually_passing_retries(self):
count = 0
for record in self.passed:
r = record
while r.retry_parent:
while r.parent is not None:
count += 1
r = r.retry_parent
r = r.parent[0]
return count

@property
Expand Down
26 changes: 26 additions & 0 deletions tests/mobly/base_test_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -2615,7 +2615,13 @@ def test_something(self):
self.assertEqual(error_record_1.test_name, 'test_something')
self.assertEqual(error_record_2.test_name, 'test_something_retry_1')
self.assertIs(error_record_1, error_record_2.retry_parent)
self.assertEqual(
(error_record_1, records.TestParentType.RETRY), error_record_2.parent
)
self.assertIs(error_record_2, pass_record.retry_parent)
self.assertEqual(
(error_record_2, records.TestParentType.RETRY), pass_record.parent
)

def test_retry_generated_test_last_pass(self):
max_count = 3
Expand Down Expand Up @@ -2650,7 +2656,13 @@ def pre_run(self):
self.assertEqual(error_record_1.test_name, 'test_generated_1')
self.assertEqual(error_record_2.test_name, 'test_generated_1_retry_1')
self.assertIs(error_record_1, error_record_2.retry_parent)
self.assertEqual(
(error_record_1, records.TestParentType.RETRY), error_record_2.parent
)
self.assertIs(error_record_2, pass_record.retry_parent)
self.assertEqual(
(error_record_2, records.TestParentType.RETRY), pass_record.parent
)

def test_retry_all_fail(self):
max_count = 3
Expand Down Expand Up @@ -2679,7 +2691,13 @@ def test_something(self):
self.assertEqual(error_record_2.test_name, 'test_something_retry_1')
self.assertEqual(error_record_3.test_name, 'test_something_retry_2')
self.assertIs(error_record_1, error_record_2.retry_parent)
self.assertEqual(
(error_record_1, records.TestParentType.RETRY), error_record_2.parent
)
self.assertIs(error_record_2, error_record_3.retry_parent)
self.assertEqual(
(error_record_2, records.TestParentType.RETRY), error_record_3.parent
)

def test_uid(self):
class MockBaseTest(base_test.BaseTestClass):
Expand Down Expand Up @@ -2726,9 +2744,17 @@ def test_something(self):
bt_cls = MockBaseTest(self.mock_test_cls_configs)
bt_cls.run()
self.assertEqual(repeat_count, len(bt_cls.results.passed))
previous_record = None
for i, record in enumerate(bt_cls.results.passed):
self.assertEqual(record.test_name, f'test_something_{i}')
self.assertEqual(record.uid, 'some-uid')
if i == 0:
self.assertIsNone(record.parent)
else:
self.assertEqual(
record.parent, (previous_record, records.TestParentType.REPEAT)
)
previous_record = record

def test_uid_with_repeat(self):
repeat_count = 3
Expand Down
1 change: 1 addition & 0 deletions tests/mobly/records_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ def verify_record(
)
d[records.TestResultEnums.RECORD_UID] = None
d[records.TestResultEnums.RECORD_RETRY_PARENT] = None
d[records.TestResultEnums.RECORD_PARENT] = None
d[records.TestResultEnums.RECORD_CLASS] = None
d[records.TestResultEnums.RECORD_EXTRA_ERRORS] = {}
d[records.TestResultEnums.RECORD_STACKTRACE] = stacktrace
Expand Down
Loading