From 1b6a7acca1d829ece9f32514e8ff3275a13d9550 Mon Sep 17 00:00:00 2001 From: Daniel Weindl Date: Tue, 16 Feb 2021 22:12:15 +0100 Subject: [PATCH 1/6] CI: SBML test suite: show tag summary for passed tests --- tests/conftest.py | 45 +++++++++++++++++++++++++++++++++++++++++- tests/testSBMLSuite.py | 16 +++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 2fee55564d..844adcf521 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,10 @@ """Pytest configuration for SBML test suite""" -from typing import List import re import sys +from typing import List + +import pytest def parse_selection(selection_str: str) -> List[int]: @@ -50,3 +52,44 @@ def pytest_generate_tests(metafunc): test_numbers -= {1395} metafunc.parametrize("test_number", test_numbers) + + +def pytest_sessionstart(session): + """Initialize result collection""" + session.results = dict() + + +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_makereport(item, call): + """Collect test results""" + outcome = yield + result = outcome.get_result() + + if result.when == 'call': + item.session.results[item] = result + + +def pytest_sessionfinish(session, exitstatus): + """Process test results""" + terminalreporter = session.config.pluginmanager.get_plugin( + 'terminalreporter') + terminalreporter.ensure_newline() + # parse test names to get passed case IDs (don't know any better way to + # access fixture values) + from testSBMLSuite import format_test_id + passed_ids = [re.sub(r'^.*\[(\d+)].*$', r'\1', item.name) + for item, result in session.results.items() + if result.outcome == 'passed'] + passed_ids = [format_test_id(_) for _ in passed_ids] + write_passed_tags(passed_ids, terminalreporter) + terminalreporter.ensure_newline() + + +def write_passed_tags(passed_ids, out=sys.stdout): + passed_tags = set() + from testSBMLSuite import get_tags_for_test + for test_id in passed_ids: + passed_tags |= get_tags_for_test(test_id) + + out.write("At least one test with the following tags has passed:\n") + out.write(' ' + '\n '.join(passed_tags)) diff --git a/tests/testSBMLSuite.py b/tests/testSBMLSuite.py index 8600da4007..d0c72b19fb 100755 --- a/tests/testSBMLSuite.py +++ b/tests/testSBMLSuite.py @@ -16,6 +16,7 @@ import copy import importlib import os +import re import shutil import sys @@ -303,3 +304,18 @@ def format_test_id(test_id) -> str: test_str = str(test_id) test_str = '0'*(5-len(test_str)) + test_str return test_str + + +def get_tags_for_test(test_id): + """Get sbml test suite tags for the given test ID""" + + current_test_path = os.path.join(TEST_PATH, test_id) + info_file = os.path.join(current_test_path, f'{test_id}-model.m') + with open(info_file) as f: + for line in f: + if line.startswith('testTags:'): + res = set(re.split(r'[ ,:]', line[len('testTags:'):].strip())) + res.discard('') + return res + print(f"No testTags found for test case {test_id}.") + return set() From 13ab7c81f4d13a26b01843238e22cfcb08036872 Mon Sep 17 00:00:00 2001 From: Daniel Weindl Date: Tue, 16 Feb 2021 22:31:20 +0100 Subject: [PATCH 2/6] filter relevant tests --- tests/conftest.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 844adcf521..4eaca5766e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -79,9 +79,11 @@ def pytest_sessionfinish(session, exitstatus): from testSBMLSuite import format_test_id passed_ids = [re.sub(r'^.*\[(\d+)].*$', r'\1', item.name) for item, result in session.results.items() - if result.outcome == 'passed'] + if item.name.startswith('test_sbml_testsuite_case[') + and result.outcome == 'passed'] passed_ids = [format_test_id(_) for _ in passed_ids] - write_passed_tags(passed_ids, terminalreporter) + if passed_ids: + write_passed_tags(passed_ids, terminalreporter) terminalreporter.ensure_newline() From 28ba509673dae6569b0d6d131e47e1ec7a4bca52 Mon Sep 17 00:00:00 2001 From: Daniel Weindl Date: Wed, 17 Feb 2021 17:04:51 +0100 Subject: [PATCH 3/6] pytest-xdist breaks pytest hooks - remove --- scripts/run-SBMLTestsuite.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/run-SBMLTestsuite.sh b/scripts/run-SBMLTestsuite.sh index 9906442b57..5c015d4528 100755 --- a/scripts/run-SBMLTestsuite.sh +++ b/scripts/run-SBMLTestsuite.sh @@ -10,7 +10,6 @@ if [[ ! -d "tests/sbml-test-suite" ]]; then fi source build/venv/bin/activate -pip show pytest-xdist > /dev/null 2>&1 || pip install pytest-xdist pip install coverage pytest-cov if [[ -z "$*" ]]; then @@ -26,5 +25,5 @@ if [[ -d "${RESULT_DIR}" ]]; then fi mkdir "${RESULT_DIR}" -pytest ./tests/testSBMLSuite.py --cases="${args}" -rfsE -s -n auto \ +pytest ./tests/testSBMLSuite.py --cases="${args}" -rfsE -s \ --cov=amici --cov-report=xml:"coverage_SBMLSuite.xml" --cov-append From d4460ef336e8033e7587b0725fa5718607164683 Mon Sep 17 00:00:00 2001 From: Daniel Weindl Date: Wed, 17 Feb 2021 17:30:52 +0100 Subject: [PATCH 4/6] Revert "pytest-xdist breaks pytest hooks - remove" This reverts commit 28ba509673dae6569b0d6d131e47e1ec7a4bca52. --- scripts/run-SBMLTestsuite.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/run-SBMLTestsuite.sh b/scripts/run-SBMLTestsuite.sh index 5c015d4528..9906442b57 100755 --- a/scripts/run-SBMLTestsuite.sh +++ b/scripts/run-SBMLTestsuite.sh @@ -10,6 +10,7 @@ if [[ ! -d "tests/sbml-test-suite" ]]; then fi source build/venv/bin/activate +pip show pytest-xdist > /dev/null 2>&1 || pip install pytest-xdist pip install coverage pytest-cov if [[ -z "$*" ]]; then @@ -25,5 +26,5 @@ if [[ -d "${RESULT_DIR}" ]]; then fi mkdir "${RESULT_DIR}" -pytest ./tests/testSBMLSuite.py --cases="${args}" -rfsE -s \ +pytest ./tests/testSBMLSuite.py --cases="${args}" -rfsE -s -n auto \ --cov=amici --cov-report=xml:"coverage_SBMLSuite.xml" --cov-append From 7f08a9d9934122c62123fec8dc693f8eb06509d0 Mon Sep 17 00:00:00 2001 From: Daniel Weindl Date: Wed, 17 Feb 2021 17:31:10 +0100 Subject: [PATCH 5/6] Use pytest-xdist compatible hooks --- tests/conftest.py | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 4eaca5766e..3e9a812164 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,6 +7,10 @@ import pytest +# stores passed SBML semantic test suite IDs +passed_ids = [] + + def parse_selection(selection_str: str) -> List[int]: """ Parse comma-separated list of integer ranges, return selected indices as @@ -54,33 +58,15 @@ def pytest_generate_tests(metafunc): metafunc.parametrize("test_number", test_numbers) -def pytest_sessionstart(session): - """Initialize result collection""" - session.results = dict() - - -@pytest.hookimpl(tryfirst=True, hookwrapper=True) -def pytest_runtest_makereport(item, call): - """Collect test results""" - outcome = yield - result = outcome.get_result() - - if result.when == 'call': - item.session.results[item] = result - - def pytest_sessionfinish(session, exitstatus): """Process test results""" + global passed_ids terminalreporter = session.config.pluginmanager.get_plugin( 'terminalreporter') terminalreporter.ensure_newline() # parse test names to get passed case IDs (don't know any better way to # access fixture values) from testSBMLSuite import format_test_id - passed_ids = [re.sub(r'^.*\[(\d+)].*$', r'\1', item.name) - for item, result in session.results.items() - if item.name.startswith('test_sbml_testsuite_case[') - and result.outcome == 'passed'] passed_ids = [format_test_id(_) for _ in passed_ids] if passed_ids: write_passed_tags(passed_ids, terminalreporter) @@ -88,6 +74,7 @@ def pytest_sessionfinish(session, exitstatus): def write_passed_tags(passed_ids, out=sys.stdout): + """Write tags of passed SBML semantic test cases""" passed_tags = set() from testSBMLSuite import get_tags_for_test for test_id in passed_ids: @@ -95,3 +82,13 @@ def write_passed_tags(passed_ids, out=sys.stdout): out.write("At least one test with the following tags has passed:\n") out.write(' ' + '\n '.join(passed_tags)) + + +def pytest_runtest_logreport(report: "TestReport") -> None: + """Collect test case IDs of passed SBML semantic test suite cases""" + if report.when == 'teardown'\ + and report.outcome == 'passed'\ + and '::test_sbml_testsuite_case[' in report.nodeid: + test_case_id = re.sub(r'^.*::test_sbml_testsuite_case\[(\d+)].*$', + r'\1', report.nodeid) + passed_ids.append(test_case_id) From 796e569285d8594fb8b90eb2a2a74316448fc55d Mon Sep 17 00:00:00 2001 From: Daniel Weindl Date: Wed, 17 Feb 2021 21:50:34 +0100 Subject: [PATCH 6/6] call --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3e9a812164..e6e1865354 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -86,7 +86,7 @@ def write_passed_tags(passed_ids, out=sys.stdout): def pytest_runtest_logreport(report: "TestReport") -> None: """Collect test case IDs of passed SBML semantic test suite cases""" - if report.when == 'teardown'\ + if report.when == 'call'\ and report.outcome == 'passed'\ and '::test_sbml_testsuite_case[' in report.nodeid: test_case_id = re.sub(r'^.*::test_sbml_testsuite_case\[(\d+)].*$',