formatted test status messages converted from 2-tuple to a dictionary…

… with "status" and "messages"; using 'latest' TranslatorTestingModel code (still in PR); unti testing for format

CHANGELOG.md

@@ -1,12 +1,11 @@
 # Change Log
 
-## 0.0.10
-
-- constrain Python to ">=3.10,<3.12" (discovered problem with 3.12 for some code runtime situations)
-
 ## 0.0.9
 
+- constrain Python to ">=3.10,<3.12" (discovered problem with 3.12 for some code runtime situations)
 - update to reasoner-validator 4.0.2 (which updates BMT to 1.4.2 and adds Bioregistry 0.11.0)
+- pulling in update to TranslatorTestingModel
+- formatted test status messages converted from 2-tuple to a dictionary with "status" and "messages"
 
 ## 0.0.8
 

graph_validation_test/__init__.py

@@ -273,12 +273,12 @@ async def run_test_cases(test_cases: List[TestCaseRun]):
 
     def compute_status(self, tcr: TestCaseRun) -> Tuple[str, TestCaseResultEnum, Dict]:
         """
-        Method to construct components for a test case result based on
-        the failure mode assessment of non-empty validation messages.
+        Method to construct components for a test case result based on the failure mode
+        assessment of non-empty validation messages (which are also returned).
 
         :param tcr: TestCaseRun containing reasoner-validator style validation message from test execution.
         :return: Tuple[str, TestCaseResultEnum, Dict], where position 0 is the target,
-                 position 1 is the testcase status (passed/failed/skipped) and
+                 position 1 is the testcase TestCaseResultEnum status (PASSED|FAILED|SKIPPED) and
                  position 2 is a (possible empty) dictionary of non-empty validation messages
         """
         target: str = tcr.default_target
@@ -309,14 +309,15 @@ def format_results(self, test_cases: List[TestCaseRun]) -> Dict:
         results for the upstream consumer of TestRunner results.
 
         :param test_cases: List[TestCaseRun], list of Test Case runs with results.
-
-        :return: Dict, of structured test message results for all TestCases,
+        :return: Dict, structured test PASSED/FAILED/SKIPPED status and message
+                       results for all TestCases, indexed by a "test_case_id"
+                       of format "<test_asset_id>-<test_name>" "test_name" as
                        specified by TRAPI generators of a given test run.
         """
         # Originally, the results == [tc.get_all_messages() for tc in test_cases], which gives
         # [
-        #    {
-        #        'molepro': {
+        #    {   # components are "ars", ARA or KP infores object references
+        #        '<component_id>': {
         #            'by_subject': {
         #                'info': {},
         #                'skipped': {},
@@ -327,7 +328,7 @@ def format_results(self, test_cases: List[TestCaseRun]) -> Dict:
         #        }
         #    },
         #    {
-        #        'molepro': {
+        #        '<component_id>': {
         #            'inverse_by_new_subject': {
         #                'info': {},
         #                'skipped': {},
@@ -355,11 +356,18 @@ def format_results(self, test_cases: List[TestCaseRun]) -> Dict:
         #
         # results == {
         #     "<test_case_id_1>": {
-        #         "molepro": <result_1>,
+        #         "<component_1>": {
+        #            "status: <PASSED|FAILED|SKIPPED>, # for result 1
+        #            "messages": <reasoner-validator formatted message partition>
+        #         }
+        #         "<component_2>": {
+        #            "status: <PASSED|FAILED|SKIPPED>, # for 'test_case_id_1' from 'component_2'
+        #            "messages": <reasoner-validator formatted message partition>
+        #         }
         #         etc...
         #     }
         #     "<test case id 2>": {
-        #         "molepro": <result 2>,
+        #         "<component_1>": <result 2>,
         #         etc...
         #     }
         #     etc...
@@ -377,8 +385,14 @@ def format_results(self, test_cases: List[TestCaseRun]) -> Dict:
         # Are “Warnings” to also be taken as an indication of a test failure? Could/should we give TestRunner
         # “stringency” indications which affect whether “skipped” and “warnings” are returned as “PASSED”?
         #
-        # The above <result_#> could be a 2-Tuple of (<PASSED|FAILED>, <pruned message catalog>) where the message
-        # catelog is a Python dictionary pruned of all empty reasoner-validator validation message partitions,
+        # The above <result_#> could be a dictionary of format:
+        #
+        # {
+        #     "status": "<PASSED|FAILED|SKIPPED>",
+        #     "messages": "<pruned message catalog>"
+        # }
+        #
+        # where the message catalog is a Python dictionary pruned of all empty reasoner-validator message partitions,
         # where the status of the test is determined by the aforementioned stringency rules, however coded.
         #
         results: Dict = dict()
@@ -389,12 +403,17 @@ def format_results(self, test_cases: List[TestCaseRun]) -> Dict:
             test_case_id: str = f"{test_asset_id}-{test_name}"
             if test_case_id not in results:
                 results[test_case_id] = dict()
-            target: str
+            component: str
             status: TestCaseResultEnum
             messages: MESSAGE_CATALOG
-            target, status, messages = self.compute_status(tcr)
-            # TODO: we blissfully assume that targets only come up once for a given test_case_id
-            results[test_case_id][target] = (status, messages)
+            component, status, messages = self.compute_status(tcr)
+            # TODO: sanity check? we blissfully assume that a 'component'
+            #       is only reported once for a given 'test_case_id'
+            assert component not in results[test_case_id]
+            results[test_case_id][component] = {
+                "status": status,
+                "messages": messages
+            }
 
         return results
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "graph-validation-tests"
-version = "0.0.10"
+version = "0.0.9"
 description = "Validation of Translator Knowledge Graphs - TRAPI, Biolink Model and One Hop navigation"
 authors = [
     "Richard Bruskiewich <[email protected]>",
@@ -54,7 +54,7 @@ deepdiff = "^6.7.1"
 reasoner-validator = "^4.0.2"
 
 #translator-testing-model = { git = "https://github.com/TranslatorSRI/TranslatorTestingModel.git", rev = "ef5d68e" }
-translator-testing-model = { git = "https://github.com/TranslatorSRI/TranslatorTestingModel.git", branch = "main" }
+translator-testing-model = { git = "https://github.com/TranslatorSRI/TranslatorTestingModel.git", branch = "bmt-and-linkml-updates" }
 #translator-testing-model = "^0.2.7"
 
 pytest = "^7.4.2"

tests/graph_validation_test/test_graph_validation_test.py

@@ -1,6 +1,7 @@
 """
 Unit tests for pieces of the GraphValidationTests code
 """
+from typing import List, Dict
 from translator_testing_model.datamodel.pydanticmodel import TestAsset
 from graph_validation_test import TestCaseRun, GraphValidationTest
 from graph_validation_test.utils.unit_test_templates import by_subject, by_object
@@ -99,3 +100,34 @@ def test_test_case_run_report_messages():
     skipped = tcr.get_skipped()
     assert len(skipped) == 1
     assert "skipped.test" in skipped
+
+
+def test_format_results():
+    test_asset_id: str = "TestAsset_1"
+    test_asset: TestAsset = TestAsset(id=test_asset_id)
+    gvt: GraphValidationTest = GraphValidationTest(
+        test_asset=test_asset
+    )
+    tcr_1: TestCaseRun = TestCaseRun(
+        test_run=gvt,
+        test=by_subject
+    )
+    tcr_2: TestCaseRun = TestCaseRun(
+        test_run=gvt,
+        test=by_object
+    )
+    test_cases: List[TestCaseRun] = [
+        tcr_1,
+        tcr_2
+    ]
+    formatted_output: Dict = gvt.format_results(test_cases)
+    assert formatted_output
+    by_subject_test_case_id: str = f"{test_asset_id}-by_subject"
+    assert formatted_output[by_subject_test_case_id]
+    by_object_test_case_id: str = f"{test_asset_id}-by_object"
+    assert formatted_output[by_object_test_case_id]
+    assert "ars" in formatted_output[by_object_test_case_id]
+    assert formatted_output[by_object_test_case_id]["ars"]
+    assert "status" in formatted_output[by_object_test_case_id]["ars"]
+    assert formatted_output[by_object_test_case_id]["ars"]["status"] == "SKIPPED"
+    assert not formatted_output[by_object_test_case_id]["ars"]["messages"]