diff --git a/pytest.ini b/pytest.ini index aea2fd8..b21d992 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,7 @@ [pytest] -norecursedirs = .git .github example* traceback-styles* -cache_dir = /tmp/python_cache_dir +norecursedirs = + .git .github example* traceback-styles* +cache_dir = + /tmp/python_cache_dir +markers = + task: A concept exercise task. \ No newline at end of file diff --git a/runner/__init__.py b/runner/__init__.py index 76b9dc9..2ae0720 100644 --- a/runner/__init__.py +++ b/runner/__init__.py @@ -23,12 +23,20 @@ def __init__(self): self.config = None def pytest_configure(self, config): + config.addinivalue_line("markers", "task(taskno): this marks the exercise task number.") self.config = config def pytest_collection_modifyitems(self, session, config, items): """ - Sorts the tests in definition order. + Sorts the tests in definition order & extracts task_id """ + for item in items: + test_id = Hierarchy(item.nodeid) + name = '.'.join(test_id.split("::")[1:]) + + for mark in item.iter_markers(name='task'): + self.tests[name] = Test(name=name, task_id=mark.kwargs['taskno']) + def _sort_by_lineno(item): test_id = Hierarchy(item.nodeid) @@ -44,9 +52,11 @@ def pytest_runtest_logreport(self, report): name = report.head_line if report.head_line else ".".join(report.nodeid.split("::")[1:]) + if name not in self.tests: self.tests[name] = Test(name) + state = self.tests[name] # ignore succesful setup and teardown stages @@ -81,6 +91,19 @@ def pytest_runtest_logreport(self, report): source = Path(self.config.rootdir) / report.fspath state.test_code = TestOrder.function_source(test_id, source) + # Looks up tast_ids from parent when the test is a subtest. + if state.task_id == 0 and 'variation' in state.name: + parent_test_name = state.name.split(' ')[0] + parent_task_id = self.tests[parent_test_name].task_id + state.task_id = parent_task_id + + # Changes status of parent to fail if any of the subtests fail. + if state.fail: + self.tests[parent_test_name].fail( + message="One or more subtests for this test failed. Details can be found under each variant." + ) + self.tests[parent_test_name].test_code = state.test_code + def pytest_sessionfinish(self, session, exitstatus): """ Processes the results into a report. diff --git a/runner/data.py b/runner/data.py index b7a2982..f324254 100644 --- a/runner/data.py +++ b/runner/data.py @@ -49,6 +49,7 @@ class Test: status: Status = Status.PASS message: Message = None test_code: str = "" + task_id: int = 0 # for an explanation of why both of these are necessary see # https://florimond.dev/blog/articles/2018/10/reconciling-dataclasses-and-properties-in-python/ @@ -57,6 +58,7 @@ class Test: def _update(self, status: Status, message: Message = None) -> None: self.status = status + if message: self.message = message @@ -93,6 +95,7 @@ def error(self, message: Message = None) -> None: """ self._update(Status.ERROR, message) + def is_passing(self): """ Check if the test is currently passing. @@ -106,7 +109,7 @@ class Results: Overall results of a test run. """ - version: int = 2 + version: int = 3 status: Status = Status.PASS message: Message = None tests: List[Test] = field(default_factory=list) @@ -117,6 +120,7 @@ def add(self, test: Test) -> None: """ if test.status is Status.FAIL: self.fail() + self.tests.append(test) def fail(self) -> None: diff --git a/runner/sort.py b/runner/sort.py index e0441cc..c678fea 100644 --- a/runner/sort.py +++ b/runner/sort.py @@ -57,6 +57,7 @@ def _visit_definition(self, node): while isinstance(last_body, (For, While, If)): last_body = last_body.body[-1] + testinfo = TestInfo(node.lineno, last_body.lineno, 1) self._cache[self.get_hierarchy(Hierarchy(node.name))] = testinfo diff --git a/test/example-all-fail/results.json b/test/example-all-fail/results.json index ccde9dd..8cd586d 100644 --- a/test/example-all-fail/results.json +++ b/test/example-all-fail/results.json @@ -1,30 +1,34 @@ { - "version": 2, + "version": 3, "status": "fail", "tests": [ { "name": "ExampleAllFailTest.test_hello", "status": "fail", "message": "AssertionError: 'Goodbye!' != 'Hello, World!'\n- Goodbye!\n+ Hello, World!", - "test_code": "self.assertEqual(hello(), \"Hello, World!\")" + "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0 }, { "name": "ExampleAllFailTest.test_abc", "status": "fail", "message": "AssertionError: 'Goodbye!' != 'Hello, World!'\n- Goodbye!\n+ Hello, World!", - "test_code": "self.assertEqual(hello(), \"Hello, World!\")" + "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0 }, { "name": "ExampleAllFailOtherTest.test_dummy", "status": "fail", "message": "AssertionError: 'Goodbye!' != 'Hello, World!'\n- Goodbye!\n+ Hello, World!", - "test_code": "self.assertEqual(hello(), \"Hello, World!\")" + "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0 }, { "name": "ExampleAllFailOtherTest.test_hello", "status": "fail", "message": "AssertionError: 'Goodbye!' != 'Hello, World!'\n- Goodbye!\n+ Hello, World!", - "test_code": "self.assertEqual(hello(), \"Hello, World!\")" + "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0 } ] } \ No newline at end of file diff --git a/test/example-empty-file/results.json b/test/example-empty-file/results.json index 3c1756f..0819ef0 100644 --- a/test/example-empty-file/results.json +++ b/test/example-empty-file/results.json @@ -1,5 +1,5 @@ { - "version": 2, + "version": 3, "status": "error", "message": "ImportError: cannot import name 'hello' from 'example_empty_file' (./test/example-empty-file/example_empty_file.py)", "tests": [] diff --git a/test/example-has-stdout-and-tasks/example_has_stdout_and_tasks.py b/test/example-has-stdout-and-tasks/example_has_stdout_and_tasks.py new file mode 100644 index 0000000..c11070f --- /dev/null +++ b/test/example-has-stdout-and-tasks/example_has_stdout_and_tasks.py @@ -0,0 +1,21 @@ +"""Example Exercism/Python solution file""" + + +def hello(): + print("Hello, World!") + + +def must_truncate(): + print( + """ + Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Vulputate ut pharetra sit amet aliquam. Amet dictum sit amet justo donec enim diam vulputate ut. Consequat nisl vel pretium lectus quam id leo. Maecenas accumsan lacus vel facilisis volutpat est velit egestas dui. Faucibus et molestie ac feugiat sed. Fringilla phasellus faucibus scelerisque eleifend donec pretium vulputate sapien. Nibh venenatis cras sed felis. Tortor at risus viverra adipiscing at. Orci dapibus ultrices in iaculis nunc. In fermentum et sollicitudin ac orci phasellus egestas tellus. Tincidunt lobortis feugiat vivamus at augue eget arcu dictum varius. Volutpat est velit egestas dui id. Non nisi est sit amet facilisis magna etiam tempor. Tincidunt id aliquet risus feugiat in ante metus dictum. Viverra aliquet eget sit amet tellus cras adipiscing. Arcu dictum varius duis at. Aliquet lectus proin nibh nisl. + + Urna molestie at elementum eu. Morbi quis commodo odio aenean. Commodo elit at imperdiet dui accumsan sit amet nulla. Faucibus a pellentesque sit amet porttitor. Donec pretium vulputate sapien nec. Felis eget velit aliquet sagittis id consectetur. Nulla malesuada pellentesque elit eget gravida cum. Mauris augue neque gravida in fermentum et sollicitudin. At quis risus sed vulputate odio ut enim blandit volutpat. Enim blandit volutpat maecenas volutpat blandit. Diam in arcu cursus euismod. Congue nisi vitae suscipit tellus mauris a diam. + + Adipiscing enim eu turpis egestas pretium aenean pharetra magna ac. Et odio pellentesque diam volutpat commodo sed egestas. Nulla porttitor massa id neque aliquam vestibulum morbi blandit cursus. Nisi porta lorem mollis aliquam ut porttitor. Morbi leo urna molestie at elementum eu facilisis. Vel elit scelerisque mauris pellentesque pulvinar. Fames ac turpis egestas maecenas pharetra convallis. Ornare arcu dui vivamus arcu felis bibendum ut tristique et. Blandit massa enim nec dui nunc mattis enim. Elit ullamcorper dignissim cras tincidunt lobortis feugiat vivamus at augue. + + Scelerisque varius morbi enim nunc faucibus a pellentesque sit. Enim diam vulputate ut pharetra. Tempor orci eu lobortis elementum nibh tellus molestie nunc non. Cras pulvinar mattis nunc sed. Ac turpis egestas maecenas pharetra convallis posuere morbi leo. Platea dictumst quisque sagittis purus sit amet. Vitae tortor condimentum lacinia quis vel eros donec ac odio. Viverra nibh cras pulvinar mattis nunc sed blandit. Tincidunt lobortis feugiat vivamus at augue. Duis at consectetur lorem donec massa sapien faucibus. Magna ac placerat vestibulum lectus mauris ultrices. Convallis posuere morbi leo urna molestie at. + + Porta non pulvinar neque laoreet suspendisse interdum consectetur libero. Id faucibus nisl tincidunt eget nullam. Ultricies lacus sed turpis tincidunt id. Hendrerit dolor magna eget est lorem ipsum. Enim ut sem viverra aliquet. Eget nulla facilisi etiam dignissim diam quis enim lobortis scelerisque. Ac tortor dignissim convallis aenean et tortor at. Non tellus orci ac auctor augue. Nec dui nunc mattis enim ut tellus. Eget nunc lobortis mattis aliquam faucibus purus in massa tempor. Elementum nibh tellus molestie nunc. Ornare lectus sit amet est placerat in. Nec feugiat in fermentum posuere urna nec tincidunt praesent. Vestibulum rhoncus est pellentesque elit. Mollis nunc sed id semper risus in. Vitae elementum curabitur vitae nunc sed velit. Duis tristique sollicitudin nibh sit amet commodo nulla facilisi.""" + ) + return "Goodbye!" diff --git a/test/example-has-stdout-and-tasks/example_has_stdout_and_tasks_test.py b/test/example-has-stdout-and-tasks/example_has_stdout_and_tasks_test.py new file mode 100644 index 0000000..a7dfea0 --- /dev/null +++ b/test/example-has-stdout-and-tasks/example_has_stdout_and_tasks_test.py @@ -0,0 +1,35 @@ +import unittest +import pytest + + +from example_has_stdout_and_tasks import hello, must_truncate + + +class ExampleHasStdoutTest(unittest.TestCase): + + @pytest.mark.task(taskno=1) + def test_hello(self): + self.assertEqual(hello(), "Hello, World!") + + @pytest.mark.task(taskno=2) + def test_abc(self): + self.assertEqual(hello(), "Hello, World!") + + @pytest.mark.task(taskno=3) + def test_trancation(self): + self.assertEqual(must_truncate(), "Hello, World!") + + +class ExampleHasStdoutOtherTest(unittest.TestCase): + + @pytest.mark.task(taskno=4) + def test_dummy(self): + self.assertEqual(hello(), "Hello, World!") + + @pytest.mark.task(taskno=5) + def test_hello(self): + self.assertEqual(hello(), "Hello, World!") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/example-has-stdout-and-tasks/results.json b/test/example-has-stdout-and-tasks/results.json new file mode 100644 index 0000000..b06dfc3 --- /dev/null +++ b/test/example-has-stdout-and-tasks/results.json @@ -0,0 +1,46 @@ +{ + "version": 3, + "status": "fail", + "tests": [ + { + "name": "ExampleHasStdoutTest.test_abc", + "status": "fail", + "message": "AssertionError: None != 'Hello, World!'", + "test_code": "def test_abc(self):\n self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 2, + "output": "Hello, World!" + }, + { + "name": "ExampleHasStdoutTest.test_hello", + "status": "fail", + "message": "AssertionError: None != 'Hello, World!'", + "test_code": "def test_hello(self):\n self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 1, + "output": "Hello, World!" + }, + { + "name": "ExampleHasStdoutTest.test_trancation", + "status": "fail", + "message": "AssertionError: 'Goodbye!' != 'Hello, World!'\n- Goodbye!\n+ Hello, World!", + "test_code": "def test_trancation(self):\n self.assertEqual(must_truncate(), \"Hello, World!\")", + "task_id": 3, + "output": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Vulputate ut pharetra sit amet aliquam. Amet dictum sit amet justo donec enim diam vulputate ut. Consequat nisl vel pretium lectus quam id leo. Maecenas accumsan lacus vel facilisis volutpat est velit egestas dui. Faucibus et molestie ac feugiat sed. Fringilla phasellus faucibus scelerisque eleifend donec pretium vulputate s [Output was truncated. Please limit to 500 chars]" + }, + { + "name": "ExampleHasStdoutOtherTest.test_dummy", + "status": "fail", + "message": "AssertionError: None != 'Hello, World!'", + "test_code": "def test_dummy(self):\n self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 4, + "output": "Hello, World!" + }, + { + "name": "ExampleHasStdoutOtherTest.test_hello", + "status": "fail", + "message": "AssertionError: None != 'Hello, World!'", + "test_code": "def test_hello(self):\n self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 5, + "output": "Hello, World!" + } + ] +} \ No newline at end of file diff --git a/test/example-has-stdout/results.json b/test/example-has-stdout/results.json index 492f98b..492b00c 100644 --- a/test/example-has-stdout/results.json +++ b/test/example-has-stdout/results.json @@ -1,5 +1,5 @@ { - "version": 2, + "version": 3, "status": "fail", "tests": [ { @@ -7,6 +7,7 @@ "status": "fail", "message": "AssertionError: None != 'Hello, World!'", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0, "output": "Hello, World!" }, { @@ -14,6 +15,7 @@ "status": "fail", "message": "AssertionError: None != 'Hello, World!'", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0, "output": "Hello, World!" }, { @@ -21,6 +23,7 @@ "status": "fail", "message": "AssertionError: 'Goodbye!' != 'Hello, World!'\n- Goodbye!\n+ Hello, World!", "test_code": "self.assertEqual(must_truncate(), \"Hello, World!\")", + "task_id": 0, "output": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Vulputate ut pharetra sit amet aliquam. Amet dictum sit amet justo donec enim diam vulputate ut. Consequat nisl vel pretium lectus quam id leo. Maecenas accumsan lacus vel facilisis volutpat est velit egestas dui. Faucibus et molestie ac feugiat sed. Fringilla phasellus faucibus scelerisque eleifend donec pretium vulputate s [Output was truncated. Please limit to 500 chars]" }, { @@ -28,6 +31,7 @@ "status": "fail", "message": "AssertionError: None != 'Hello, World!'", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0, "output": "Hello, World!" }, { @@ -35,6 +39,7 @@ "status": "fail", "message": "AssertionError: None != 'Hello, World!'", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0, "output": "Hello, World!" } ] diff --git a/test/example-partial-fail/results.json b/test/example-partial-fail/results.json index 27828b8..90763ec 100644 --- a/test/example-partial-fail/results.json +++ b/test/example-partial-fail/results.json @@ -1,28 +1,32 @@ { - "version": 2, + "version": 3, "status": "fail", "tests": [ { "name": "ExamplePartialFailTest.test_hello", "status": "fail", "message": "AssertionError: 'Hello, World!' != 'Goodbye'\n- Hello, World!\n+ Goodbye", - "test_code": "self.assertEqual(hello(), \"Goodbye\")" + "test_code": "self.assertEqual(hello(), \"Goodbye\")", + "task_id": 0 }, { "name": "ExamplePartialFailTest.test_abc", "status": "pass", - "test_code": "self.assertEqual(hello(), \"Hello, World!\")" + "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0 }, { "name": "ExamplePartialFailOtherTest.test_dummy", "status": "fail", "message": "AssertionError: 'Hello, World!' != 'Goodbye'\n- Hello, World!\n+ Goodbye", - "test_code": "self.assertEqual(hello(), \"Goodbye\")" + "test_code": "self.assertEqual(hello(), \"Goodbye\")", + "task_id": 0 }, { "name": "ExamplePartialFailOtherTest.test_hello", "status": "pass", - "test_code": "self.assertEqual(hello(), \"Hello, World!\")" + "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0 } ] } \ No newline at end of file diff --git a/test/example-partial-failure-with-subtests/example_partial_failure_with_subtests.py b/test/example-partial-failure-with-subtests/example_partial_failure_with_subtests.py new file mode 100644 index 0000000..57a8bce --- /dev/null +++ b/test/example-partial-failure-with-subtests/example_partial_failure_with_subtests.py @@ -0,0 +1,9 @@ +"""Example Exercism/Python solution file""" + + +def hello(param): + if isinstance(param, int): + return ("Hello, World!") + else: + return ("Hello, World!", param) + diff --git a/test/example-partial-failure-with-subtests/example_partial_failure_with_subtests_test.py b/test/example-partial-failure-with-subtests/example_partial_failure_with_subtests_test.py new file mode 100644 index 0000000..66bd5b7 --- /dev/null +++ b/test/example-partial-failure-with-subtests/example_partial_failure_with_subtests_test.py @@ -0,0 +1,59 @@ +import unittest +import pytest + + +from example_partial_failure_with_subtests import hello + + +class ExampleSuccessTest(unittest.TestCase): + + @pytest.mark.task(taskno=1) + def test_hello(self): + input_data = [1, 2, 5, 10] + result_data = [("Hello, World!", param) for param in input_data] + number_of_variants = range(1, len(input_data) + 1) + + for variant, param, result in zip(number_of_variants, input_data, result_data): + with self.subTest(f"variation #{variant}", param=param, result=result): + self.assertEqual(hello(param), result, + msg=f'Expected: {result} but got something else instead.') + + @pytest.mark.task(taskno=1) + def test_abc(self): + input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree'] + result_data = [("Hello, World!", param) for param in input_data] + number_of_variants = range(1, len(input_data) + 1) + + for variant, param, result in zip(number_of_variants, input_data, result_data): + with self.subTest(f"variation #{variant}", param=param, result=result): + self.assertEqual(hello(param), result, + msg=f'Expected: {result} but got something else instead.') + +class ExampleSuccessOtherTest(unittest.TestCase): + + @pytest.mark.task(taskno=2) + def test_dummy(self): + input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree'] + result_data = [("Hello, World!", param) for param in input_data] + number_of_variants = range(1, len(input_data) + 1) + + for variant, param, result in zip(number_of_variants, input_data, result_data): + with self.subTest(f"variation #{variant}", param=param, result=result): + self.assertEqual(hello(param), result, + msg=f'Expected: {result} but got something else instead.') + + + @pytest.mark.task(taskno=2) + def test_hello(self): + input_data = [15, 23, 33, 39] + result_data = [("Hello, World!", param) for param in input_data] + number_of_variants = range(1, len(input_data) + 1) + + for variant, param, result in zip(number_of_variants, input_data, result_data): + with self.subTest(f"variation #{variant}", param=param, result=result): + self.assertEqual(hello(param), result, + msg=f'Expected: {result} but got something else instead.') + + +if __name__ == "__main__": + unittest.main() diff --git a/test/example-partial-failure-with-subtests/results.json b/test/example-partial-failure-with-subtests/results.json new file mode 100644 index 0000000..7f0420c --- /dev/null +++ b/test/example-partial-failure-with-subtests/results.json @@ -0,0 +1,88 @@ +{ + "version": 3, + "status": "fail", + "tests": [ + { + "name": "ExampleSuccessTest.test_abc", + "status": "pass", + "test_code": "def test_abc(self):\n input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\n result_data = [(\"Hello, World!\", param) for param in input_data]\n number_of_variants = range(1, len(input_data) + 1)\n\n for variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,", + "task_id": 1 + }, + { + "name": "ExampleSuccessTest.test_hello", + "status": "fail", + "message": "One or more subtests for this test failed. Details can be found under each variant.", + "test_code": "def test_hello(self):\n input_data = [1, 2, 5, 10]\n result_data = [(\"Hello, World!\", param) for param in input_data]\n number_of_variants = range(1, len(input_data) + 1)\n\n for variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,", + "task_id": 1 + }, + { + "name": "ExampleSuccessOtherTest.test_dummy", + "status": "pass", + "test_code": "def test_dummy(self):\n input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\n result_data = [(\"Hello, World!\", param) for param in input_data]\n number_of_variants = range(1, len(input_data) + 1)\n\n for variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,", + "task_id": 2 + }, + { + "name": "ExampleSuccessOtherTest.test_hello", + "status": "fail", + "message": "One or more subtests for this test failed. Details can be found under each variant.", + "test_code": "def test_hello(self):\n input_data = [15, 23, 33, 39]\n result_data = [(\"Hello, World!\", param) for param in input_data]\n number_of_variants = range(1, len(input_data) + 1)\n\n for variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,", + "task_id": 2 + }, + { + "name": "ExampleSuccessTest.test_hello [variation #1] (param=1, result=('Hello, World!', 1))", + "status": "fail", + "message": "AssertionError: 'Hello, World!' != ('Hello, World!', 1) : Expected: ('Hello, World!', 1) but got something else instead.", + "test_code": "def test_hello(self):\n input_data = [1, 2, 5, 10]\n result_data = [(\"Hello, World!\", param) for param in input_data]\n number_of_variants = range(1, len(input_data) + 1)\n\n for variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,", + "task_id": 1 + }, + { + "name": "ExampleSuccessTest.test_hello [variation #2] (param=2, result=('Hello, World!', 2))", + "status": "fail", + "message": "AssertionError: 'Hello, World!' != ('Hello, World!', 2) : Expected: ('Hello, World!', 2) but got something else instead.", + "test_code": "def test_hello(self):\n input_data = [1, 2, 5, 10]\n result_data = [(\"Hello, World!\", param) for param in input_data]\n number_of_variants = range(1, len(input_data) + 1)\n\n for variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,", + "task_id": 1 + }, + { + "name": "ExampleSuccessTest.test_hello [variation #3] (param=5, result=('Hello, World!', 5))", + "status": "fail", + "message": "AssertionError: 'Hello, World!' != ('Hello, World!', 5) : Expected: ('Hello, World!', 5) but got something else instead.", + "test_code": "def test_hello(self):\n input_data = [1, 2, 5, 10]\n result_data = [(\"Hello, World!\", param) for param in input_data]\n number_of_variants = range(1, len(input_data) + 1)\n\n for variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,", + "task_id": 1 + }, + { + "name": "ExampleSuccessTest.test_hello [variation #4] (param=10, result=('Hello, World!', 10))", + "status": "fail", + "message": "AssertionError: 'Hello, World!' != ('Hello, World!', 10) : Expected: ('Hello, World!', 10) but got something else instead.", + "test_code": "def test_hello(self):\n input_data = [1, 2, 5, 10]\n result_data = [(\"Hello, World!\", param) for param in input_data]\n number_of_variants = range(1, len(input_data) + 1)\n\n for variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,", + "task_id": 1 + }, + { + "name": "ExampleSuccessOtherTest.test_hello [variation #1] (param=15, result=('Hello, World!', 15))", + "status": "fail", + "message": "AssertionError: 'Hello, World!' != ('Hello, World!', 15) : Expected: ('Hello, World!', 15) but got something else instead.", + "test_code": "def test_hello(self):\n input_data = [15, 23, 33, 39]\n result_data = [(\"Hello, World!\", param) for param in input_data]\n number_of_variants = range(1, len(input_data) + 1)\n\n for variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,", + "task_id": 2 + }, + { + "name": "ExampleSuccessOtherTest.test_hello [variation #2] (param=23, result=('Hello, World!', 23))", + "status": "fail", + "message": "AssertionError: 'Hello, World!' != ('Hello, World!', 23) : Expected: ('Hello, World!', 23) but got something else instead.", + "test_code": "def test_hello(self):\n input_data = [15, 23, 33, 39]\n result_data = [(\"Hello, World!\", param) for param in input_data]\n number_of_variants = range(1, len(input_data) + 1)\n\n for variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,", + "task_id": 2 + }, + { + "name": "ExampleSuccessOtherTest.test_hello [variation #3] (param=33, result=('Hello, World!', 33))", + "status": "fail", + "message": "AssertionError: 'Hello, World!' != ('Hello, World!', 33) : Expected: ('Hello, World!', 33) but got something else instead.", + "test_code": "def test_hello(self):\n input_data = [15, 23, 33, 39]\n result_data = [(\"Hello, World!\", param) for param in input_data]\n number_of_variants = range(1, len(input_data) + 1)\n\n for variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,", + "task_id": 2 + }, + { + "name": "ExampleSuccessOtherTest.test_hello [variation #4] (param=39, result=('Hello, World!', 39))", + "status": "fail", + "message": "AssertionError: 'Hello, World!' != ('Hello, World!', 39) : Expected: ('Hello, World!', 39) but got something else instead.", + "test_code": "def test_hello(self):\n input_data = [15, 23, 33, 39]\n result_data = [(\"Hello, World!\", param) for param in input_data]\n number_of_variants = range(1, len(input_data) + 1)\n\n for variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,", + "task_id": 2 + } + ] +} \ No newline at end of file diff --git a/test/example-success-with-subtests/example_success_with_subtests.py b/test/example-success-with-subtests/example_success_with_subtests.py new file mode 100644 index 0000000..f781f6d --- /dev/null +++ b/test/example-success-with-subtests/example_success_with_subtests.py @@ -0,0 +1,6 @@ +"""Example Exercism/Python solution file""" + + +def hello(param): + param = param + return ("Hello, World!", param) diff --git a/test/example-success-with-subtests/example_success_with_subtests_test.py b/test/example-success-with-subtests/example_success_with_subtests_test.py new file mode 100644 index 0000000..0e3a65c --- /dev/null +++ b/test/example-success-with-subtests/example_success_with_subtests_test.py @@ -0,0 +1,59 @@ +import unittest +import pytest + + +from example_success_with_subtests import hello + + +class ExampleSuccessTest(unittest.TestCase): + + @pytest.mark.task(taskno=1) + def test_hello(self): + input_data = [1, 2, 5, 10, 15, 23, 33, 39] + result_data = [("Hello, World!", param) for param in input_data] + number_of_variants = range(1, len(input_data) + 1) + + for variant, param, result in zip(number_of_variants, input_data, result_data): + with self.subTest(f"variation #{variant}", param=param, result=result): + self.assertEqual(hello(param), result, + msg=f'Expected: {result} but got something else instead.') + + @pytest.mark.task(taskno=1) + def test_abc(self): + input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree'] + result_data = [("Hello, World!", param) for param in input_data] + number_of_variants = range(1, len(input_data) + 1) + + for variant, param, result in zip(number_of_variants, input_data, result_data): + with self.subTest(f"variation #{variant}", param=param, result=result): + self.assertEqual(hello(param), result, + msg=f'Expected: {result} but got something else instead.') + +class ExampleSuccessOtherTest(unittest.TestCase): + + @pytest.mark.task(taskno=2) + def test_dummy(self): + input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree'] + result_data = [("Hello, World!", param) for param in input_data] + number_of_variants = range(1, len(input_data) + 1) + + for variant, param, result in zip(number_of_variants, input_data, result_data): + with self.subTest(f"variation #{variant}", param=param, result=result): + self.assertEqual(hello(param), result, + msg=f'Expected: {result} but got something else instead.') + + + @pytest.mark.task(taskno=2) + def test_hello(self): + input_data = [1, 2, 5, 10, 15, 23, 33, 39] + result_data = [("Hello, World!", param) for param in input_data] + number_of_variants = range(1, len(input_data) + 1) + + for variant, param, result in zip(number_of_variants, input_data, result_data): + with self.subTest(f"variation #{variant}", param=param, result=result): + self.assertEqual(hello(param), result, + msg=f'Expected: {result} but got something else instead.') + + +if __name__ == "__main__": + unittest.main() diff --git a/test/example-success-with-subtests/results.json b/test/example-success-with-subtests/results.json new file mode 100644 index 0000000..2d018aa --- /dev/null +++ b/test/example-success-with-subtests/results.json @@ -0,0 +1,30 @@ +{ + "version": 3, + "status": "pass", + "tests": [ + { + "name": "ExampleSuccessTest.test_abc", + "status": "pass", + "test_code": "def test_abc(self):\n input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\n result_data = [(\"Hello, World!\", param) for param in input_data]\n number_of_variants = range(1, len(input_data) + 1)\n\n for variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,", + "task_id": 1 + }, + { + "name": "ExampleSuccessTest.test_hello", + "status": "pass", + "test_code": "def test_hello(self):\n input_data = [1, 2, 5, 10, 15, 23, 33, 39]\n result_data = [(\"Hello, World!\", param) for param in input_data]\n number_of_variants = range(1, len(input_data) + 1)\n\n for variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,", + "task_id": 1 + }, + { + "name": "ExampleSuccessOtherTest.test_dummy", + "status": "pass", + "test_code": "def test_dummy(self):\n input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\n result_data = [(\"Hello, World!\", param) for param in input_data]\n number_of_variants = range(1, len(input_data) + 1)\n\n for variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,", + "task_id": 2 + }, + { + "name": "ExampleSuccessOtherTest.test_hello", + "status": "pass", + "test_code": "def test_hello(self):\n input_data = [1, 2, 5, 10, 15, 23, 33, 39]\n result_data = [(\"Hello, World!\", param) for param in input_data]\n number_of_variants = range(1, len(input_data) + 1)\n\n for variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,", + "task_id": 2 + } + ] +} \ No newline at end of file diff --git a/test/example-success/results.json b/test/example-success/results.json index 5ba60d8..c2bc453 100644 --- a/test/example-success/results.json +++ b/test/example-success/results.json @@ -1,30 +1,34 @@ { - "version": 2, + "version": 3, "status": "pass", "tests": [ { "name": "ExampleSuccessTest.test_hello", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0, "output": "User output is captured!" }, { "name": "ExampleSuccessTest.test_abc", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0, "output": "User output is captured!" }, { "name": "ExampleSuccessOtherTest.test_dummy", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0, "output": "User output is captured!" }, { "name": "ExampleSuccessOtherTest.test_hello", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0, "output": "User output is captured!" } ] -} +} \ No newline at end of file diff --git a/test/example-syntax-error/results.json b/test/example-syntax-error/results.json index 6fc17dc..46b00cc 100644 --- a/test/example-syntax-error/results.json +++ b/test/example-syntax-error/results.json @@ -1,5 +1,5 @@ { - "version": 2, + "version": 3, "status": "error", "message": " File \"./test/example-syntax-error/example_syntax_error.py\", line 3\n def hello();\n ^\nSyntaxError: invalid syntax", "tests": [] diff --git a/test/example-with-config-multiple-files/results.json b/test/example-with-config-multiple-files/results.json index cdd89c7..87f937a 100644 --- a/test/example-with-config-multiple-files/results.json +++ b/test/example-with-config-multiple-files/results.json @@ -1,54 +1,62 @@ { - "version": 2, + "version": 3, "status": "pass", "tests": [ { "name": "ExampleFirstTest.test_hello", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0, "output": "User output is captured!" }, { "name": "ExampleSecondTest.test_hello", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0, "output": "User output is captured!" }, { "name": "ExampleFirstTest.test_abc", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0, "output": "User output is captured!" }, { "name": "ExampleSecondTest.test_abc", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0, "output": "User output is captured!" }, { "name": "ExampleFirstOtherTest.test_dummy", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0, "output": "User output is captured!" }, { "name": "ExampleSecondOtherTest.test_dummy", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0, "output": "User output is captured!" }, { "name": "ExampleFirstOtherTest.test_hello", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0, "output": "User output is captured!" }, { "name": "ExampleSecondOtherTest.test_hello", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0, "output": "User output is captured!" } ] -} +} \ No newline at end of file diff --git a/test/example-with-config/results.json b/test/example-with-config/results.json index 50ae1f6..ccdc691 100644 --- a/test/example-with-config/results.json +++ b/test/example-with-config/results.json @@ -1,30 +1,34 @@ { - "version": 2, + "version": 3, "status": "pass", "tests": [ { "name": "ExampleWithConfigTest.test_hello", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0, "output": "User output is captured!" }, { "name": "ExampleWithConfigTest.test_abc", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0, "output": "User output is captured!" }, { "name": "ExampleWithConfigOtherTest.test_dummy", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0, "output": "User output is captured!" }, { "name": "ExampleWithConfigOtherTest.test_hello", "status": "pass", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", + "task_id": 0, "output": "User output is captured!" } ] -} +} \ No newline at end of file