diff --git a/CHANGELOG.md b/CHANGELOG.md index abfcb8e..4abf03a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,11 @@ # Change Log -## 0.0.10 - -- constrain Python to ">=3.10,<3.12" (discovered problem with 3.12 for some code runtime situations) - ## 0.0.9 +- constrain Python to ">=3.10,<3.12" (discovered problem with 3.12 for some code runtime situations) - update to reasoner-validator 4.0.2 (which updates BMT to 1.4.2 and adds Bioregistry 0.11.0) +- pulling in update to TranslatorTestingModel +- formatted test status messages converted from 2-tuple to a dictionary with "status" and "messages" ## 0.0.8 diff --git a/graph_validation_test/__init__.py b/graph_validation_test/__init__.py index 4c45dab..ac022c3 100644 --- a/graph_validation_test/__init__.py +++ b/graph_validation_test/__init__.py @@ -273,12 +273,12 @@ async def run_test_cases(test_cases: List[TestCaseRun]): def compute_status(self, tcr: TestCaseRun) -> Tuple[str, TestCaseResultEnum, Dict]: """ - Method to construct components for a test case result based on - the failure mode assessment of non-empty validation messages. + Method to construct components for a test case result based on the failure mode + assessment of non-empty validation messages (which are also returned). :param tcr: TestCaseRun containing reasoner-validator style validation message from test execution. :return: Tuple[str, TestCaseResultEnum, Dict], where position 0 is the target, - position 1 is the testcase status (passed/failed/skipped) and + position 1 is the testcase TestCaseResultEnum status (PASSED|FAILED|SKIPPED) and position 2 is a (possible empty) dictionary of non-empty validation messages """ target: str = tcr.default_target @@ -309,14 +309,15 @@ def format_results(self, test_cases: List[TestCaseRun]) -> Dict: results for the upstream consumer of TestRunner results. :param test_cases: List[TestCaseRun], list of Test Case runs with results. - - :return: Dict, of structured test message results for all TestCases, + :return: Dict, structured test PASSED/FAILED/SKIPPED status and message + results for all TestCases, indexed by a "test_case_id" + of format "-" "test_name" as specified by TRAPI generators of a given test run. """ # Originally, the results == [tc.get_all_messages() for tc in test_cases], which gives # [ - # { - # 'molepro': { + # { # components are "ars", ARA or KP infores object references + # '': { # 'by_subject': { # 'info': {}, # 'skipped': {}, @@ -327,7 +328,7 @@ def format_results(self, test_cases: List[TestCaseRun]) -> Dict: # } # }, # { - # 'molepro': { + # '': { # 'inverse_by_new_subject': { # 'info': {}, # 'skipped': {}, @@ -355,11 +356,18 @@ def format_results(self, test_cases: List[TestCaseRun]) -> Dict: # # results == { # "": { - # "molepro": , + # "": { + # "status: , # for result 1 + # "messages": + # } + # "": { + # "status: , # for 'test_case_id_1' from 'component_2' + # "messages": + # } # etc... # } # "": { - # "molepro": , + # "": , # etc... # } # etc... @@ -377,8 +385,14 @@ def format_results(self, test_cases: List[TestCaseRun]) -> Dict: # Are “Warnings” to also be taken as an indication of a test failure? Could/should we give TestRunner # “stringency” indications which affect whether “skipped” and “warnings” are returned as “PASSED”? # - # The above could be a 2-Tuple of (, ) where the message - # catelog is a Python dictionary pruned of all empty reasoner-validator validation message partitions, + # The above could be a dictionary of format: + # + # { + # "status": "", + # "messages": "" + # } + # + # where the message catalog is a Python dictionary pruned of all empty reasoner-validator message partitions, # where the status of the test is determined by the aforementioned stringency rules, however coded. # results: Dict = dict() @@ -389,12 +403,17 @@ def format_results(self, test_cases: List[TestCaseRun]) -> Dict: test_case_id: str = f"{test_asset_id}-{test_name}" if test_case_id not in results: results[test_case_id] = dict() - target: str + component: str status: TestCaseResultEnum messages: MESSAGE_CATALOG - target, status, messages = self.compute_status(tcr) - # TODO: we blissfully assume that targets only come up once for a given test_case_id - results[test_case_id][target] = (status, messages) + component, status, messages = self.compute_status(tcr) + # TODO: sanity check? we blissfully assume that a 'component' + # is only reported once for a given 'test_case_id' + assert component not in results[test_case_id] + results[test_case_id][component] = { + "status": status, + "messages": messages + } return results diff --git a/poetry.lock b/poetry.lock index d7a3b07..92deab1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "annotated-types" @@ -1725,8 +1725,8 @@ docs = ["linkml (>=1.7.8,<2.0.0)"] [package.source] type = "git" url = "https://github.com/TranslatorSRI/TranslatorTestingModel.git" -reference = "main" -resolved_reference = "1e11527e6918b78b7515774ddad63078e8274836" +reference = "bmt-and-linkml-updates" +resolved_reference = "a8fc81e2d41f120f5a741821bb4c172113e00873" [[package]] name = "typing-extensions" @@ -1848,4 +1848,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "5b61e1ab0880d99d4bf0ae8c292c313df8459f606ca0b7fcf11ef1a78634d189" +content-hash = "66e8b446eabbafc6fba6acb2f5f37b7851f922587f5ed1d05d80ae6f031078e5" diff --git a/pyproject.toml b/pyproject.toml index 6326c24..77e514d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "graph-validation-tests" -version = "0.0.10" +version = "0.0.9" description = "Validation of Translator Knowledge Graphs - TRAPI, Biolink Model and One Hop navigation" authors = [ "Richard Bruskiewich ", @@ -54,7 +54,7 @@ deepdiff = "^6.7.1" reasoner-validator = "^4.0.2" #translator-testing-model = { git = "https://github.com/TranslatorSRI/TranslatorTestingModel.git", rev = "ef5d68e" } -translator-testing-model = { git = "https://github.com/TranslatorSRI/TranslatorTestingModel.git", branch = "main" } +translator-testing-model = { git = "https://github.com/TranslatorSRI/TranslatorTestingModel.git", branch = "bmt-and-linkml-updates" } #translator-testing-model = "^0.2.7" pytest = "^7.4.2" diff --git a/tests/graph_validation_test/test_graph_validation_test.py b/tests/graph_validation_test/test_graph_validation_test.py index 9b2109d..5a141d8 100644 --- a/tests/graph_validation_test/test_graph_validation_test.py +++ b/tests/graph_validation_test/test_graph_validation_test.py @@ -1,6 +1,7 @@ """ Unit tests for pieces of the GraphValidationTests code """ +from typing import List, Dict from translator_testing_model.datamodel.pydanticmodel import TestAsset from graph_validation_test import TestCaseRun, GraphValidationTest from graph_validation_test.utils.unit_test_templates import by_subject, by_object @@ -99,3 +100,34 @@ def test_test_case_run_report_messages(): skipped = tcr.get_skipped() assert len(skipped) == 1 assert "skipped.test" in skipped + + +def test_format_results(): + test_asset_id: str = "TestAsset_1" + test_asset: TestAsset = TestAsset(id=test_asset_id) + gvt: GraphValidationTest = GraphValidationTest( + test_asset=test_asset + ) + tcr_1: TestCaseRun = TestCaseRun( + test_run=gvt, + test=by_subject + ) + tcr_2: TestCaseRun = TestCaseRun( + test_run=gvt, + test=by_object + ) + test_cases: List[TestCaseRun] = [ + tcr_1, + tcr_2 + ] + formatted_output: Dict = gvt.format_results(test_cases) + assert formatted_output + by_subject_test_case_id: str = f"{test_asset_id}-by_subject" + assert formatted_output[by_subject_test_case_id] + by_object_test_case_id: str = f"{test_asset_id}-by_object" + assert formatted_output[by_object_test_case_id] + assert "ars" in formatted_output[by_object_test_case_id] + assert formatted_output[by_object_test_case_id]["ars"] + assert "status" in formatted_output[by_object_test_case_id]["ars"] + assert formatted_output[by_object_test_case_id]["ars"]["status"] == "SKIPPED" + assert not formatted_output[by_object_test_case_id]["ars"]["messages"]