From 30a517daea5e31fcfd65e685d0f29d9a45b54c48 Mon Sep 17 00:00:00 2001 From: Kolea PLESCO Date: Thu, 13 Apr 2023 19:53:59 +0300 Subject: [PATCH] XPATHs coverage to support Mapping Remarks --- ted_sws/core/model/manifestation.py | 1 + .../adapters/conceptual_mapping_reader.py | 10 ++-- .../adapters/xpath_coverage_runner.py | 20 ++++++-- .../templates/xpath_coverage_report.jinja2 | 46 ++++++++++++++++--- 4 files changed, 64 insertions(+), 13 deletions(-) diff --git a/ted_sws/core/model/manifestation.py b/ted_sws/core/model/manifestation.py index 7d81764d..160aec4c 100644 --- a/ted_sws/core/model/manifestation.py +++ b/ted_sws/core/model/manifestation.py @@ -155,6 +155,7 @@ class XPATHCoverageValidationResultBase(PropertyBaseModel): xpath_covered: Optional[List[str]] = [] xpath_not_covered: Optional[List[str]] = [] xpath_extra: Optional[List[str]] = [] + remarked_xpaths: Optional[List[str]] = [] coverage: Optional[float] conceptual_coverage: Optional[float] diff --git a/ted_sws/mapping_suite_processor/adapters/conceptual_mapping_reader.py b/ted_sws/mapping_suite_processor/adapters/conceptual_mapping_reader.py index 8346a9c5..ef60cb4c 100644 --- a/ted_sws/mapping_suite_processor/adapters/conceptual_mapping_reader.py +++ b/ted_sws/mapping_suite_processor/adapters/conceptual_mapping_reader.py @@ -147,7 +147,7 @@ def _read_conceptual_mapping_rules(cls, df: pd.DataFrame) -> List[ConceptualMapp return rules @classmethod - def _read_conceptual_mapping_remarks(cls, df: pd.DataFrame) -> List[ConceptualMappingRemark]: + def _read_conceptual_mapping_remarks(cls, df: pd.DataFrame, base_xpath: str) -> List[ConceptualMappingRemark]: """ :param df: @@ -161,7 +161,11 @@ def _read_conceptual_mapping_remarks(cls, df: pd.DataFrame) -> List[ConceptualMa remark = ConceptualMappingRemark() remark.standard_form_field_id = cls._read_pd_value(row[RULES_SF_FIELD_ID]) remark.standard_form_field_name = cls._read_pd_value(row[RULES_SF_FIELD_NAME]) - remark.field_xpath = cls._read_list_from_pd_multiline_value(row[RULES_FIELD_XPATH]) + remarked_xpaths = cls._read_list_from_pd_multiline_value(row[RULES_FIELD_XPATH]) + if remarked_xpaths: + remark.field_xpath = [] + for remarked_xpath in remarked_xpaths: + remark.field_xpath.append(cls.xpath_with_base(remarked_xpath, base_xpath)) remarks.append(remark) return remarks @@ -273,7 +277,7 @@ def mapping_suite_read_conceptual_mapping(cls, conceptual_mappings_file_path: Pa conceptual_mapping.metadata = metadata conceptual_mapping.rules = cls._read_conceptual_mapping_rules(dfs[CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME]) conceptual_mapping.mapping_remarks = cls._read_conceptual_mapping_remarks( - dfs[CONCEPTUAL_MAPPINGS_REMARKS_SHEET_NAME]) + dfs[CONCEPTUAL_MAPPINGS_REMARKS_SHEET_NAME], base_xpath=metadata.base_xpath) conceptual_mapping.resources = cls._read_conceptual_mapping_resources( dfs[CONCEPTUAL_MAPPINGS_RESOURCES_SHEET_NAME]) conceptual_mapping.rml_modules = cls._read_conceptual_mapping_rml_modules( diff --git a/ted_sws/notice_validator/adapters/xpath_coverage_runner.py b/ted_sws/notice_validator/adapters/xpath_coverage_runner.py index dbae286d..7786bb68 100644 --- a/ted_sws/notice_validator/adapters/xpath_coverage_runner.py +++ b/ted_sws/notice_validator/adapters/xpath_coverage_runner.py @@ -25,6 +25,7 @@ class CoverageRunner: mapping_suite: MappingSuite mapping_suite_id: str conceptual_xpaths: Set[str] = set() + conceptual_remarked_xpaths: Set[str] = set() conceptual_xpath_data: Dict[str, ConceptualMappingXPATH] = {} base_xpath: str @@ -41,6 +42,13 @@ def notice_xpaths(self, notice: Notice) -> List[str]: return notice.xml_metadata.unique_xpaths def init_xpath_data(self, conceptual_mapping: ConceptualMapping): + for cm_xpath in conceptual_mapping.mapping_remarks: + for xpath in cm_xpath.field_xpath: + self.conceptual_remarked_xpaths.add(xpath) + self.conceptual_xpath_data[xpath] = ConceptualMappingXPATH( + xpath=xpath, + form_field=f"{cm_xpath.standard_form_field_id} - {cm_xpath.standard_form_field_name}" + ) for cm_xpath in conceptual_mapping.xpaths: self.conceptual_xpaths.add(cm_xpath.xpath) self.conceptual_xpath_data[cm_xpath.xpath] = cm_xpath @@ -63,10 +71,13 @@ def find_notice_by_xpath(cls, notice_xpaths: XPathDict, xpath: str) -> Dict[str, notice_hit: Dict[str, int] = {k: v.count(xpath) for k, v in sorted(notice_xpaths.items()) if xpath in v} return notice_hit + def get_all_conceptual_xpaths(self) -> Set[str]: + return self.conceptual_remarked_xpaths | self.conceptual_xpaths + def xpath_assertions(self, notice_xpaths: XPathDict, xpaths_list: List[str]) -> List[XPATHCoverageValidationAssertion]: xpath_assertions = [] - for xpath in self.conceptual_xpaths: + for xpath in self.get_all_conceptual_xpaths(): xpath_assertion = XPATHCoverageValidationAssertion() xpath_data = self.conceptual_xpath_data[xpath] form_field = xpath_data.form_field @@ -85,8 +96,10 @@ def validate_xpath_coverage_report(self, report: XPATHCoverageValidationReport, validation_result: XPATHCoverageValidationResult = XPATHCoverageValidationResult() validation_result.xpath_assertions = self.xpath_assertions(notice_xpaths, xpaths_list) validation_result.xpath_covered = sorted(list(self.conceptual_xpaths & unique_notice_xpaths)) - validation_result.xpath_not_covered = sorted(list(unique_notice_xpaths - self.conceptual_xpaths)) - validation_result.xpath_extra = sorted(list(self.conceptual_xpaths - unique_notice_xpaths)) + all_conceptual_xpaths = self.get_all_conceptual_xpaths() + validation_result.xpath_not_covered = sorted(list(unique_notice_xpaths - all_conceptual_xpaths)) + validation_result.xpath_extra = sorted(list(all_conceptual_xpaths - unique_notice_xpaths)) + validation_result.remarked_xpaths = sorted(list(self.conceptual_remarked_xpaths)) unique_notice_xpaths_len = len(unique_notice_xpaths) xpath_covered_len = len(validation_result.xpath_covered) conceptual_xpaths_len = len(self.conceptual_xpaths) @@ -139,4 +152,3 @@ def html_report(cls, report: XPATHCoverageValidationReport, metadata: dict = Non data[TEMPLATE_METADATA_KEY] = metadata html_report = TEMPLATES.get_template(XPATH_COVERAGE_REPORT_TEMPLATE).render(data) return html_report - diff --git a/ted_sws/notice_validator/resources/templates/xpath_coverage_report.jinja2 b/ted_sws/notice_validator/resources/templates/xpath_coverage_report.jinja2 index 8fdc5fdf..8f74a98d 100644 --- a/ted_sws/notice_validator/resources/templates/xpath_coverage_report.jinja2 +++ b/ted_sws/notice_validator/resources/templates/xpath_coverage_report.jinja2 @@ -69,6 +69,13 @@ [data-role=collapsible] .ui-collapsible-content { padding: 10px; } + + hr { + margin: 12px 0; + height: 5PX; + background: #ccc; + border: 0; + } @@ -113,7 +120,7 @@ {% if validation_result.xpath_assertions|length > 0 %}

XPATH Assertions

- +
@@ -145,8 +152,8 @@ {% endif %} {% if validation_result.xpath_covered|length > 0 %}
-

XPATHs covered by Conceptual Mapping

-
Form Field
+

XPATHs covered in the "Rules" of Conceptual Mapping

+
@@ -161,10 +168,28 @@
XPATH
{% endif %} +{% if validation_result.remarked_xpaths|length > 0 %} +
+

XPATHs covered in the "Mapping Remarks" of Conceptual Mapping

+ + + + + + + + {% for xpath in validation_result.remarked_xpaths %} + + + + {% endfor %} + +
XPATH
{{ xpath }}
+{% endif %} {% if validation_result.xpath_not_covered|length > 0 %}

XPATHs not covered by Conceptual Mapping

- +
@@ -182,7 +207,7 @@ {% if validation_result.xpath_extra|length > 0 %}

Extra XPATHs in Conceptual Mapping

-
XPATH
+
@@ -214,7 +239,7 @@ $c.attr("data-state", $c.attr("data-state") == "collapsed" ? "expanded" : "collapsed"); return false; }); - $("table.display").DataTable({ + $("table.display.results").DataTable({ dom: 'B<"clear">lfiprtip', buttons: [], "lengthMenu": [[5, 15, 30, -1], [5, 15, 30, "All"]], @@ -227,6 +252,15 @@ ] }); + $("table.display.summary").DataTable({ + dom: 'B<"clear">lfiprtip', + buttons: [], + "lengthMenu": [[5, 15, 30, -1], [5, 15, 30, "All"]], + "pageLength": 15, + responsive: { + details: true + } + }); }); \ No newline at end of file
XPATH