From 86803fe0f812527a89dc2e02d58cd72144a29ec1 Mon Sep 17 00:00:00 2001 From: Boon Eng Oh Date: Fri, 22 Dec 2023 07:43:32 +0000 Subject: [PATCH 1/3] Add parent field to records to represent repeat and retry information. --- mobly/base_test.py | 12 ++++++++++-- mobly/records.py | 24 ++++++++++++++++++++++-- tests/mobly/base_test_test.py | 26 ++++++++++++++++++++++++++ tests/mobly/records_test.py | 1 + 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/mobly/base_test.py b/mobly/base_test.py index ae8d0232..e5060af7 100644 --- a/mobly/base_test.py +++ b/mobly/base_test.py @@ -720,6 +720,7 @@ def should_retry(record): retry_name = f'{test_name}_retry_{i+1}' new_record = records.TestResultRecord(retry_name, self.TAG) new_record.retry_parent = previous_record + new_record.parent = (previous_record, records.TestParentType.RETRY) previous_record = self.exec_one_test(retry_name, test_method, new_record) if not should_retry(previous_record): break @@ -749,16 +750,23 @@ def _exec_one_test_with_repeat( if max_consecutive_error == 0: max_consecutive_error = repeat_count + previous_record = None for i in range(repeat_count): new_test_name = f'{test_name}_{i}' - record = self.exec_one_test(new_test_name, test_method) - if record.result in [ + new_record = records.TestResultRecord(new_test_name, self.TAG) + if i > 0: + new_record.parent = (previous_record, records.TestParentType.REPEAT) + previous_record = self.exec_one_test( + new_test_name, test_method, new_record + ) + if previous_record.result in [ records.TestResultEnums.TEST_RESULT_FAIL, records.TestResultEnums.TEST_RESULT_ERROR, ]: consecutive_error_count += 1 else: consecutive_error_count = 0 + if consecutive_error_count == max_consecutive_error: logging.error( 'Repeated test case "%s" has consecutively failed %d iterations, ' diff --git a/mobly/records.py b/mobly/records.py index c5f41ef8..e4464388 100644 --- a/mobly/records.py +++ b/mobly/records.py @@ -37,6 +37,13 @@ class Error(Exception): """Raised for errors in record module members.""" +class TestParentType(enum.Enum): + """The type of parent in a chain of executions of the same test.""" + + REPEAT = 'repeat' + RETRY = 'retry' + + def uid(uid): """Decorator specifying the unique identifier (UID) of a test case. @@ -182,6 +189,7 @@ class TestResultEnums: RECORD_STACKTRACE = 'Stacktrace' RECORD_SIGNATURE = 'Signature' RECORD_RETRY_PARENT = 'Retry Parent' + RECORD_PARENT = 'Parent' RECORD_POSITION = 'Position' TEST_RESULT_PASS = 'PASS' TEST_RESULT_FAIL = 'FAIL' @@ -320,6 +328,9 @@ class TestResultRecord: retry_parent: TestResultRecord, only set for retry iterations. This is the test result record of the previous retry iteration. Parsers can use this field to construct the chain of execution for each retried test. + parent: tuple[TestResultRecord, TestParentType], set for multiple iterations + of a test. This is the test result record of the previous iteration. + Parsers can use this field to construct the chain of execution for each test. termination_signal: ExceptionRecord, the main exception of the test. extra_errors: OrderedDict, all exceptions occurred during the entire test lifecycle. The order of occurrence is preserved. @@ -334,6 +345,7 @@ def __init__(self, t_name, t_class=None): self.uid = None self.signature = None self.retry_parent = None + self.parent = None self.termination_signal = None self.extra_errors = collections.OrderedDict() self.result = None @@ -506,6 +518,14 @@ def to_dict(self): d[TestResultEnums.RECORD_RETRY_PARENT] = ( self.retry_parent.signature if self.retry_parent else None ) + d[TestResultEnums.RECORD_PARENT] = ( + { + 'parent': self.parent[0].signature, + 'type': self.parent[1].value, + } + if self.parent + else None + ) d[TestResultEnums.RECORD_EXTRAS] = self.extras d[TestResultEnums.RECORD_DETAILS] = self.details d[TestResultEnums.RECORD_TERMINATION_SIGNAL_TYPE] = ( @@ -644,9 +664,9 @@ def _count_eventually_passing_retries(self): count = 0 for record in self.passed: r = record - while r.retry_parent: + while r.parent is not None: count += 1 - r = r.retry_parent + r = r.parent[0] return count @property diff --git a/tests/mobly/base_test_test.py b/tests/mobly/base_test_test.py index 630bd820..bfc7b756 100755 --- a/tests/mobly/base_test_test.py +++ b/tests/mobly/base_test_test.py @@ -2615,7 +2615,13 @@ def test_something(self): self.assertEqual(error_record_1.test_name, 'test_something') self.assertEqual(error_record_2.test_name, 'test_something_retry_1') self.assertIs(error_record_1, error_record_2.retry_parent) + self.assertEqual( + (error_record_1, records.TestParentType.RETRY), error_record_2.parent + ) self.assertIs(error_record_2, pass_record.retry_parent) + self.assertEqual( + (error_record_2, records.TestParentType.RETRY), pass_record.parent + ) def test_retry_generated_test_last_pass(self): max_count = 3 @@ -2650,7 +2656,13 @@ def pre_run(self): self.assertEqual(error_record_1.test_name, 'test_generated_1') self.assertEqual(error_record_2.test_name, 'test_generated_1_retry_1') self.assertIs(error_record_1, error_record_2.retry_parent) + self.assertEqual( + (error_record_1, records.TestParentType.RETRY), error_record_2.parent + ) self.assertIs(error_record_2, pass_record.retry_parent) + self.assertEqual( + (error_record_2, records.TestParentType.RETRY), pass_record.parent + ) def test_retry_all_fail(self): max_count = 3 @@ -2679,7 +2691,13 @@ def test_something(self): self.assertEqual(error_record_2.test_name, 'test_something_retry_1') self.assertEqual(error_record_3.test_name, 'test_something_retry_2') self.assertIs(error_record_1, error_record_2.retry_parent) + self.assertEqual( + (error_record_1, records.TestParentType.RETRY), error_record_2.parent + ) self.assertIs(error_record_2, error_record_3.retry_parent) + self.assertEqual( + (error_record_2, records.TestParentType.RETRY), error_record_3.parent + ) def test_uid(self): class MockBaseTest(base_test.BaseTestClass): @@ -2726,9 +2744,17 @@ def test_something(self): bt_cls = MockBaseTest(self.mock_test_cls_configs) bt_cls.run() self.assertEqual(repeat_count, len(bt_cls.results.passed)) + previous_record = None for i, record in enumerate(bt_cls.results.passed): self.assertEqual(record.test_name, f'test_something_{i}') self.assertEqual(record.uid, 'some-uid') + if i == 0: + self.assertIsNone(record.parent) + else: + self.assertEqual( + record.parent, (previous_record, records.TestParentType.REPEAT) + ) + previous_record = record def test_uid_with_repeat(self): repeat_count = 3 diff --git a/tests/mobly/records_test.py b/tests/mobly/records_test.py index bcee67a1..f3140a97 100755 --- a/tests/mobly/records_test.py +++ b/tests/mobly/records_test.py @@ -98,6 +98,7 @@ def verify_record( ) d[records.TestResultEnums.RECORD_UID] = None d[records.TestResultEnums.RECORD_RETRY_PARENT] = None + d[records.TestResultEnums.RECORD_PARENT] = None d[records.TestResultEnums.RECORD_CLASS] = None d[records.TestResultEnums.RECORD_EXTRA_ERRORS] = {} d[records.TestResultEnums.RECORD_STACKTRACE] = stacktrace From c20abb0fac9f385f2db4b307c632cf22c8e1f913 Mon Sep 17 00:00:00 2001 From: Boon Eng Oh Date: Tue, 26 Dec 2023 07:32:33 +0000 Subject: [PATCH 2/3] Deprecate retry parent argument. --- mobly/records.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mobly/records.py b/mobly/records.py index e4464388..b7b9d2e2 100644 --- a/mobly/records.py +++ b/mobly/records.py @@ -325,9 +325,8 @@ class TestResultRecord: uid: User-defined unique identifier of the test. signature: string, unique identifier of a test record, the value is generated by Mobly. - retry_parent: TestResultRecord, only set for retry iterations. This is the - test result record of the previous retry iteration. Parsers can use this - field to construct the chain of execution for each retried test. + retry_parent: TestResultRecord, only set for retry iterations. This + argument is deprecated, use `parent` instead. parent: tuple[TestResultRecord, TestParentType], set for multiple iterations of a test. This is the test result record of the previous iteration. Parsers can use this field to construct the chain of execution for each test. From 1f7eb3de86124a1219ce33a58c39ed0bcd8c5f0a Mon Sep 17 00:00:00 2001 From: Boon Eng Oh Date: Wed, 27 Dec 2023 06:11:57 +0000 Subject: [PATCH 3/3] Fix deprecated message. --- mobly/records.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mobly/records.py b/mobly/records.py index b7b9d2e2..216f3f72 100644 --- a/mobly/records.py +++ b/mobly/records.py @@ -325,8 +325,7 @@ class TestResultRecord: uid: User-defined unique identifier of the test. signature: string, unique identifier of a test record, the value is generated by Mobly. - retry_parent: TestResultRecord, only set for retry iterations. This - argument is deprecated, use `parent` instead. + retry_parent: [DEPRECATED] Use the `parent` field instead. parent: tuple[TestResultRecord, TestParentType], set for multiple iterations of a test. This is the test result record of the previous iteration. Parsers can use this field to construct the chain of execution for each test.