Merge pull request #474 from OP-TED/feature/TED-1224

XPATHs coverage to support Mapping Remarks
OP-TED · Apr 13, 2023 · 39584da · 39584da
2 parents 8e50c6c + 30a517d
commit 39584da
Show file tree

Hide file tree

Showing 4 changed files with 64 additions and 13 deletions.
diff --git a/ted_sws/core/model/manifestation.py b/ted_sws/core/model/manifestation.py
@@ -155,6 +155,7 @@ class XPATHCoverageValidationResultBase(PropertyBaseModel):
     xpath_covered: Optional[List[str]] = []
     xpath_not_covered: Optional[List[str]] = []
     xpath_extra: Optional[List[str]] = []
+    remarked_xpaths: Optional[List[str]] = []
     coverage: Optional[float]
     conceptual_coverage: Optional[float]
 

diff --git a/ted_sws/mapping_suite_processor/adapters/conceptual_mapping_reader.py b/ted_sws/mapping_suite_processor/adapters/conceptual_mapping_reader.py
@@ -147,7 +147,7 @@ def _read_conceptual_mapping_rules(cls, df: pd.DataFrame) -> List[ConceptualMapp
         return rules
 
     @classmethod
-    def _read_conceptual_mapping_remarks(cls, df: pd.DataFrame) -> List[ConceptualMappingRemark]:
+    def _read_conceptual_mapping_remarks(cls, df: pd.DataFrame, base_xpath: str) -> List[ConceptualMappingRemark]:
         """
 
         :param df:
@@ -161,7 +161,11 @@ def _read_conceptual_mapping_remarks(cls, df: pd.DataFrame) -> List[ConceptualMa
             remark = ConceptualMappingRemark()
             remark.standard_form_field_id = cls._read_pd_value(row[RULES_SF_FIELD_ID])
             remark.standard_form_field_name = cls._read_pd_value(row[RULES_SF_FIELD_NAME])
-            remark.field_xpath = cls._read_list_from_pd_multiline_value(row[RULES_FIELD_XPATH])
+            remarked_xpaths = cls._read_list_from_pd_multiline_value(row[RULES_FIELD_XPATH])
+            if remarked_xpaths:
+                remark.field_xpath = []
+                for remarked_xpath in remarked_xpaths:
+                    remark.field_xpath.append(cls.xpath_with_base(remarked_xpath, base_xpath))
             remarks.append(remark)
         return remarks
 
@@ -273,7 +277,7 @@ def mapping_suite_read_conceptual_mapping(cls, conceptual_mappings_file_path: Pa
             conceptual_mapping.metadata = metadata
             conceptual_mapping.rules = cls._read_conceptual_mapping_rules(dfs[CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME])
             conceptual_mapping.mapping_remarks = cls._read_conceptual_mapping_remarks(
-                dfs[CONCEPTUAL_MAPPINGS_REMARKS_SHEET_NAME])
+                dfs[CONCEPTUAL_MAPPINGS_REMARKS_SHEET_NAME], base_xpath=metadata.base_xpath)
             conceptual_mapping.resources = cls._read_conceptual_mapping_resources(
                 dfs[CONCEPTUAL_MAPPINGS_RESOURCES_SHEET_NAME])
             conceptual_mapping.rml_modules = cls._read_conceptual_mapping_rml_modules(

diff --git a/ted_sws/notice_validator/adapters/xpath_coverage_runner.py b/ted_sws/notice_validator/adapters/xpath_coverage_runner.py
@@ -25,6 +25,7 @@ class CoverageRunner:
     mapping_suite: MappingSuite
     mapping_suite_id: str
     conceptual_xpaths: Set[str] = set()
+    conceptual_remarked_xpaths: Set[str] = set()
     conceptual_xpath_data: Dict[str, ConceptualMappingXPATH] = {}
     base_xpath: str
 
@@ -41,6 +42,13 @@ def notice_xpaths(self, notice: Notice) -> List[str]:
         return notice.xml_metadata.unique_xpaths
 
     def init_xpath_data(self, conceptual_mapping: ConceptualMapping):
+        for cm_xpath in conceptual_mapping.mapping_remarks:
+            for xpath in cm_xpath.field_xpath:
+                self.conceptual_remarked_xpaths.add(xpath)
+                self.conceptual_xpath_data[xpath] = ConceptualMappingXPATH(
+                    xpath=xpath,
+                    form_field=f"{cm_xpath.standard_form_field_id} - {cm_xpath.standard_form_field_name}"
+                )
         for cm_xpath in conceptual_mapping.xpaths:
             self.conceptual_xpaths.add(cm_xpath.xpath)
             self.conceptual_xpath_data[cm_xpath.xpath] = cm_xpath
@@ -63,10 +71,13 @@ def find_notice_by_xpath(cls, notice_xpaths: XPathDict, xpath: str) -> Dict[str,
         notice_hit: Dict[str, int] = {k: v.count(xpath) for k, v in sorted(notice_xpaths.items()) if xpath in v}
         return notice_hit
 
+    def get_all_conceptual_xpaths(self) -> Set[str]:
+        return self.conceptual_remarked_xpaths | self.conceptual_xpaths
+
     def xpath_assertions(self, notice_xpaths: XPathDict,
                          xpaths_list: List[str]) -> List[XPATHCoverageValidationAssertion]:
         xpath_assertions = []
-        for xpath in self.conceptual_xpaths:
+        for xpath in self.get_all_conceptual_xpaths():
             xpath_assertion = XPATHCoverageValidationAssertion()
             xpath_data = self.conceptual_xpath_data[xpath]
             form_field = xpath_data.form_field
@@ -85,8 +96,10 @@ def validate_xpath_coverage_report(self, report: XPATHCoverageValidationReport,
         validation_result: XPATHCoverageValidationResult = XPATHCoverageValidationResult()
         validation_result.xpath_assertions = self.xpath_assertions(notice_xpaths, xpaths_list)
         validation_result.xpath_covered = sorted(list(self.conceptual_xpaths & unique_notice_xpaths))
-        validation_result.xpath_not_covered = sorted(list(unique_notice_xpaths - self.conceptual_xpaths))
-        validation_result.xpath_extra = sorted(list(self.conceptual_xpaths - unique_notice_xpaths))
+        all_conceptual_xpaths = self.get_all_conceptual_xpaths()
+        validation_result.xpath_not_covered = sorted(list(unique_notice_xpaths - all_conceptual_xpaths))
+        validation_result.xpath_extra = sorted(list(all_conceptual_xpaths - unique_notice_xpaths))
+        validation_result.remarked_xpaths = sorted(list(self.conceptual_remarked_xpaths))
         unique_notice_xpaths_len = len(unique_notice_xpaths)
         xpath_covered_len = len(validation_result.xpath_covered)
         conceptual_xpaths_len = len(self.conceptual_xpaths)
@@ -139,4 +152,3 @@ def html_report(cls, report: XPATHCoverageValidationReport, metadata: dict = Non
         data[TEMPLATE_METADATA_KEY] = metadata
         html_report = TEMPLATES.get_template(XPATH_COVERAGE_REPORT_TEMPLATE).render(data)
         return html_report
-
diff --git a/ted_sws/notice_validator/resources/templates/xpath_coverage_report.jinja2 b/ted_sws/notice_validator/resources/templates/xpath_coverage_report.jinja2
@@ -69,6 +69,13 @@
         [data-role=collapsible] .ui-collapsible-content {
             padding: 10px;
         }
+
+        hr {
+            margin: 12px 0;
+            height: 5PX;
+            background: #ccc;
+            border: 0;
+        }
     </style>
 </head>
 <body>
@@ -113,7 +120,7 @@
 {% if validation_result.xpath_assertions|length > 0 %}
 <hr>
 <h2>XPATH Assertions</h2>
-<table class="display" data-order='[[0, "asc"]]'>
+<table class="display results" data-order='[[0, "asc"]]'>
     <thead>
     <tr>
         <th>Form Field</th>
@@ -145,8 +152,8 @@
 {% endif %}
 {% if validation_result.xpath_covered|length > 0 %}
 <hr>
-<h2>XPATHs covered by Conceptual Mapping</h2>
-<table class="display" data-order='[[0, "asc"]]'>
+<h2>XPATHs covered in the "Rules" of Conceptual Mapping</h2>
+<table class="display summary" data-order='[[0, "asc"]]'>
     <thead>
     <tr>
         <th>XPATH</th>
@@ -161,10 +168,28 @@
     </tbody>
 </table>
 {% endif %}
+{% if validation_result.remarked_xpaths|length > 0 %}
+    <hr>
+    <h2>XPATHs covered in the "Mapping Remarks" of Conceptual Mapping</h2>
+    <table class="display summary" data-order='[[0, "asc"]]'>
+        <thead>
+        <tr>
+            <th>XPATH</th>
+        </tr>
+        </thead>
+        <tbody>
+        {% for xpath in validation_result.remarked_xpaths %}
+            <tr>
+                <td class="break-word">{{ xpath }}</td>
+            </tr>
+        {% endfor %}
+        </tbody>
+    </table>
+{% endif %}
 {% if validation_result.xpath_not_covered|length > 0 %}
 <hr>
 <h2>XPATHs not covered by Conceptual Mapping</h2>
-<table class="display" data-order='[[0, "asc"]]'>
+<table class="display summary" data-order='[[0, "asc"]]'>
     <thead>
     <tr>
         <th>XPATH</th>
@@ -182,7 +207,7 @@
 {% if validation_result.xpath_extra|length > 0 %}
 <hr>
 <h2>Extra XPATHs in Conceptual Mapping</h2>
-<table class="display" data-order='[[0, "asc"]]'>
+<table class="display summary" data-order='[[0, "asc"]]'>
     <thead>
     <tr>
         <th>XPATH</th>
@@ -214,7 +239,7 @@
             $c.attr("data-state", $c.attr("data-state") == "collapsed" ? "expanded" : "collapsed");
             return false;
         });
-        $("table.display").DataTable({
+        $("table.display.results").DataTable({
             dom: 'B<"clear">lfiprtip',
             buttons: [],
             "lengthMenu": [[5, 15, 30, -1], [5, 15, 30, "All"]],
@@ -227,6 +252,15 @@
             ]
         });
 
+        $("table.display.summary").DataTable({
+            dom: 'B<"clear">lfiprtip',
+            buttons: [],
+            "lengthMenu": [[5, 15, 30, -1], [5, 15, 30, "All"]],
+            "pageLength": 15,
+            responsive: {
+                details: true
+            }
+        });
     });
 
 </script>