Feature/ted 1058

ted_sws/data_manager/adapters/mapping_suite_repository.py

@@ -208,11 +208,13 @@ def _write_file_resources(self, file_resources: List[FileResource], path: pathli
                 f.write(file_resource.file_content)
 
     @classmethod
-    def read_flat_file_resources(cls, path: pathlib.Path, file_resources=None, extension=None) -> List[FileResource]:
+    def read_flat_file_resources(cls, path: pathlib.Path, file_resources=None, extension=None, with_content=True) -> \
+            List[FileResource]:
         """
         This method reads a folder (with nested-tree structure) of resources and returns a flat list of file-type
         resources from all beyond levels.
         Used for folders that contains files with unique names, but grouped into sub-folders.
+        :param with_content:
         :param extension:
         :param path:
         :param file_resources:
@@ -230,7 +232,8 @@ def read_flat_file_resources(cls, path: pathlib.Path, file_resources=None, exten
                     continue
                 file_path = pathlib.Path(os.path.join(root, f))
                 file_resource = FileResource(file_name=file_path.name,
-                                             file_content=file_path.read_text(encoding="utf-8"),
+                                             file_content=file_path.read_text(
+                                                 encoding="utf-8") if with_content else "",
                                              original_name=file_path.name,
                                              parents=file_parents)
                 file_resources.append(file_resource)
@@ -350,6 +353,10 @@ def _read_mapping_suite_package(self, mapping_suite_identifier: str) -> Optional
             return MappingSuite(**package_metadata)
         return None
 
+    @classmethod
+    def mapping_suite_notice_path_by_group_depth(cls, path: pathlib.Path, group_depth: int = 0) -> pathlib.Path:
+        return pathlib.Path(*path.parts[:(-group_depth if group_depth else None)]) if path else None
+
     def add(self, mapping_suite: MappingSuite):
         """
             This method allows you to add MappingSuite objects to the repository.

ted_sws/data_manager/services/mapping_suite_resource_manager.py

@@ -31,18 +31,29 @@ def mapping_suite_skipped_notice(notice_id: str, notice_ids: List[str]) -> bool:
     return notice_ids and notice_id not in notice_ids
 
 
-def mapping_suite_notices_grouped_by_path(mapping_suite: MappingSuite, notice_ids: List[str] = None) -> \
-        Dict[Path, List[ReportNotice]]:
+def mapping_suite_notice_path_by_group_depth(path: Path, group_depth: int = 0) -> Path:
+    return MappingSuiteRepositoryInFileSystem.mapping_suite_notice_path_by_group_depth(path, group_depth=group_depth)
+
+
+def mapping_suite_notices_grouped_by_path(mapping_suite: MappingSuite = None, with_content=True,
+                                          file_resources: List[FileResource] = None, group_depth: int = 0,
+                                          notice_ids: List[str] = None) -> Dict[Path, List[ReportNotice]]:
     grouped_notices: Dict[Path, List[ReportNotice]] = {}
-    for data in mapping_suite.transformation_test_data.test_data:
+    if file_resources is None:
+        file_resources = mapping_suite.transformation_test_data.test_data
+    for data in file_resources:
         notice_id = Path(data.file_name).stem
         if mapping_suite_skipped_notice(notice_id, notice_ids):
             continue
         notice = Notice(ted_id=notice_id)
-        notice.set_xml_manifestation(XMLManifestation(object_data=data.file_content))
+        if with_content:
+            notice.set_xml_manifestation(XMLManifestation(object_data=data.file_content))
         report_notice = ReportNotice(notice=notice,
                                      metadata=ReportNoticeMetadata(path=file_resource_path(data)))
-        group = file_resource_output_path(data)
+        group = mapping_suite_notice_path_by_group_depth(
+            path=file_resource_output_path(data),
+            group_depth=group_depth
+        )
         grouped_notices.setdefault(group, []).append(report_notice)
 
     return grouped_notices
@@ -58,9 +69,11 @@ def mapping_suite_files_grouped_by_path(file_resources: List[FileResource], grou
     return grouped_files
 
 
-def read_flat_file_resources(path: pathlib.Path, file_resources=None, extension=None) -> List[FileResource]:
+def read_flat_file_resources(path: pathlib.Path, file_resources=None, extension=None, with_content=True) -> \
+        List[FileResource]:
     return MappingSuiteRepositoryInFileSystem.read_flat_file_resources(
         path=path,
         file_resources=file_resources,
-        extension=extension
+        extension=extension,
+        with_content=with_content
     )

ted_sws/mapping_suite_processor/resources/templates/conceptual_mappings_diff_report.jinja2

@@ -1,10 +1,20 @@
+{% set mapping_suite1 = metadata['mapping_suite_ids'][0] or metadata['mapping_suite_ids'][1] %}
+{% set mapping_suite2 = metadata['mapping_suite_ids'][1] or metadata['mapping_suite_ids'][0] %}
+
+{% set ms1_parts = mapping_suite1.split("_") %}
+{% set ms1_acronym = ms1_parts[ms1_parts|length - 1] if ms1_parts else "" %}
+{% set ms2_parts = mapping_suite2.split("_") %}
+{% set ms2_acronym = ms2_parts[ms2_parts|length - 1] if ms2_parts else "" %}
+
+{% set ms_acronyms_title = ms1_acronym + ("-" + ms2_acronym if ms2_acronym else "") %}
+
 <!DOCTYPE html>
 <html lang="en">
   <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta http-equiv="X-UA-Compatible" content="ie=edge">
-    <title>HTML report for Conceptual Mappings Diff</title>
+    <title>CM {% if ms_acronyms_title %}({{ ms_acronyms_title }}) {% endif %}Diff Report</title>
     <link rel="stylesheet" href="https://code.jquery.com/ui/1.13.2/themes/base/jquery-ui.css">
     <link href="https://cdn.datatables.net/1.10.22/css/jquery.dataTables.min.css" rel="stylesheet" type="text/css">
     <style>
@@ -50,7 +60,7 @@
     </style>
   </head>
   <body>
-	<h1>Conceptual Mappings Diff HTML report </h1>
+	<h1>Conceptual Mappings Diff Report</h1>
 	<hr>
     <ul>
         <li>Created at: {{ created_at }}</li>
@@ -75,8 +85,6 @@
                 <td>{{ branch1 }}</td>
                 <td>{{ branch2 }}</td>
             </tr>
-            {% set mapping_suite1 = metadata['mapping_suite_ids'][0] or metadata['mapping_suite_ids'][1] %}
-            {% set mapping_suite2 = metadata['mapping_suite_ids'][1] or metadata['mapping_suite_ids'][0] %}
             <tr>
                 <th>Mapping Suite</th>
                 <td>{{ mapping_suite1 }}</td>

ted_sws/notice_transformer/adapters/notice_transformer.py

@@ -0,0 +1,40 @@
+from typing import List
+
+from ted_sws.core.model.notice import Notice
+from ted_sws.core.model.validation_report import ReportNotice
+from ted_sws.core.model.validation_report_data import ReportNoticeData
+from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryInFileSystem
+
+
+class NoticeTransformer:
+    @classmethod
+    def transform_report_notice(cls, report_notice: ReportNotice, group_depth: int = 0) -> ReportNoticeData:
+        report_path = str(MappingSuiteRepositoryInFileSystem.mapping_suite_notice_path_by_group_depth(
+            report_notice.metadata.path,
+            group_depth=group_depth)) if report_notice.metadata else None
+        return ReportNoticeData(
+            notice_id=report_notice.notice.ted_id,
+            path=report_path
+        )
+
+    @classmethod
+    def transform_report_notices(cls, report_notices: List[ReportNotice]) -> List[ReportNoticeData]:
+        report_datas = []
+        for report_notice in report_notices:
+            report_datas.append(cls.transform_report_notice(report_notice))
+        return report_datas
+
+    @classmethod
+    def transform_validation_report_notices(cls, report_notices: List[ReportNotice], group_depth: int = 0) \
+            -> List[ReportNoticeData]:
+        report_datas = []
+        for report_notice in report_notices:
+            report_datas.append(cls.transform_report_notice(report_notice, group_depth=group_depth))
+        return report_datas
+
+    @classmethod
+    def map_report_notices_to_notices(cls, report_notices: List[ReportNotice]) -> List[Notice]:
+        notices: List[Notice] = []
+        for report_notice in report_notices:
+            notices.append(report_notice.notice)
+        return notices

ted_sws/notice_transformer/services/notice_transformer.py

@@ -8,10 +8,11 @@
 from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryInFileSystem
 from ted_sws.data_manager.adapters.repository_abc import NoticeRepositoryABC, MappingSuiteRepositoryABC
 from ted_sws.data_manager.services.mapping_suite_resource_manager import file_resource_output_path, \
-    mapping_suite_skipped_notice
+    mapping_suite_skipped_notice, mapping_suite_notice_path_by_group_depth
 from ted_sws.event_manager.adapters.event_logger import EventLogger, EventMessageLogSettings
 from ted_sws.event_manager.model.event_message import NoticeEventMessage
 from ted_sws.event_manager.services.logger_from_context import get_env_logger
+from ted_sws.notice_transformer.adapters.notice_transformer import NoticeTransformer
 from ted_sws.notice_transformer.adapters.rml_mapper import RMLMapperABC
 from ted_sws.notice_transformer.services import DEFAULT_TRANSFORMATION_FILE_EXTENSION
 from ted_sws.core.model.validation_report import ReportNotice
@@ -121,19 +122,5 @@ def transform_test_data(mapping_suite: MappingSuite, rml_mapper: RMLMapperABC, o
             logger.info(event_message, settings=EventMessageLogSettings(briefly=True))
 
 
-def transform_report_notice(report_notice: ReportNotice) -> ReportNoticeData:
-    return ReportNoticeData(notice_id=report_notice.notice.ted_id, path=str(report_notice.metadata.path))
-
-
 def transform_report_notices(report_notices: List[ReportNotice]) -> List[ReportNoticeData]:
-    report_datas = []
-    for report_notice in report_notices:
-        report_datas.append(transform_report_notice(report_notice))
-    return report_datas
-
-
-def transform_validation_report_notices(report_notices: List[Notice]) -> List[ReportNoticeData]:
-    report_datas = []
-    for report_notice in report_notices:
-        report_datas.append(ReportNoticeData(notice_id=report_notice.ted_id))
-    return report_datas
+    return NoticeTransformer.transform_report_notices(report_notices)

ted_sws/notice_validator/adapters/validation_summary_runner.py

@@ -8,7 +8,9 @@
     XPATHCoverageSummaryResult, SPARQLSummaryCountReport, SHACLSummarySeverityCountReport, SPARQLSummaryResult, \
     SHACLSummaryResult, ValidationSummaryReport
 from ted_sws.core.model.notice import Notice
-from ted_sws.notice_transformer.services.notice_transformer import transform_validation_report_notices
+from ted_sws.core.model.validation_report import ReportNotice
+from ted_sws.notice_transformer.adapters.notice_transformer import NoticeTransformer
+from ted_sws.notice_validator.resources.templates import TEMPLATE_METADATA_KEY
 
 TEMPLATES = Environment(loader=PackageLoader("ted_sws.notice_validator.resources", "templates"))
 VALIDATION_SUMMARY_REPORT_TEMPLATE = "validation_summary_report.jinja2"
@@ -206,9 +208,11 @@ def validation_summary_for_notice(cls, notice: Notice) -> ValidationSummaryRepor
         return cls.validation_summary([notice])
 
     @classmethod
-    def validation_summary_for_notices(cls, notices: List[Notice]) -> ValidationSummaryReport:
-        report: ValidationSummaryReport = cls.validation_summary(notices)
-        report.notices = sorted(transform_validation_report_notices(notices),
+    def validation_summary_for_notices(cls, notices: List[ReportNotice]) -> ValidationSummaryReport:
+        report: ValidationSummaryReport = cls.validation_summary(
+            sorted(NoticeTransformer.map_report_notices_to_notices(notices),
+                   key=lambda notice: notice.ted_id))
+        report.notices = sorted(NoticeTransformer.transform_validation_report_notices(notices, group_depth=1),
                                 key=lambda report_data: report_data.notice_id)
 
         return report
@@ -218,6 +222,7 @@ def json_report(cls, report) -> dict:
         return report.dict()
 
     @classmethod
-    def html_report(cls, report) -> str:
+    def html_report(cls, report, metadata: dict = None) -> str:
         data: dict = cls.json_report(report)
+        data[TEMPLATE_METADATA_KEY] = metadata
         return TEMPLATES.get_template(VALIDATION_SUMMARY_REPORT_TEMPLATE).render(data)

ted_sws/notice_validator/resources/templates/_ms_meta_title_pre.jinja2

@@ -0,0 +1,12 @@
+{% set notice_in_title = "" %}
+
+{% if notice_ids %}
+    {% set notice_in_title = notice_ids[0] if notice_ids|length == 1 else "" %}
+{% elif notices %}
+    {% set notice_in_title = notices[0].notice_id if notices|length == 1 else "" %}
+{%  endif %}
+
+{% set ms_parts = mapping_suite_identifier.split("_") %}
+{% set ms_acronym = ms_parts[1] if ms_parts and ms_parts|length >= 2 else ms_parts[0] %}
+{% set report_meta_title = ms_acronym + ("/" + notice_in_title if notice_in_title else "") %}
+{% if report_meta_title %}{{ report_meta_title }} {% endif %}

ted_sws/notice_validator/resources/templates/shacl_shape_validation_results_report.jinja2

@@ -4,7 +4,7 @@
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta http-equiv="X-UA-Compatible" content="ie=edge">
-    <title>HTML report for SHACL Validation</title>
+    <title>{% include '_ms_meta_title_pre.jinja2' %}SHACL Validation Report</title>
     <link href="https://cdn.datatables.net/1.10.22/css/jquery.dataTables.min.css" rel="stylesheet" type="text/css">
     <style type="text/css">
         body {
@@ -56,26 +56,28 @@
     </style>
 </head>
 <body>
-<h1>SHACL Validation HTML report </h1>
+<h1>SHACL Validation Report</h1>
 <hr>
 <h2>Report details: </h2>
 <ul>
     <li>Date created: {{ created }}</li>
     <li>SHACL test suite identifier: {{ test_suite_identifier }}</li>
     <li>Mapping suite identifier: {{ mapping_suite_identifier }}</li>
     {% if notice_ids %}
-    <li><div><hr></div>Notice identifier(s):
-        <div data-role="collapsible" data-state="{% if notice_ids|length > 1 %}collapsed{% endif %}"
+    <li><div><hr></div>
+        {%  set nr_notices = notice_ids|length %}
+        {% if nr_notices > 1 %}
+        Notice identifiers ({{ nr_notices }}):
+        <div data-role="collapsible" data-state="collapsed"
             class="collapsible-wrapper">
             <h4><a href="#"></a></h4>
             <div class="ui-collapsible-content">
-            {% if notice_ids|length == 1 %}
-                {{ notice_ids[0] }}
-            {% else %}
                 {{ notice_ids | tojson(indent=2) | replace('\n', '<br>') }}
-            {% endif %}
             </div>
         </div>
+        {% elif nr_notices == 1 %}
+            <b>Notice identifier: {{ notice_ids[0] }}</b>
+        {% endif %}
     </li>
     {% endif %}
 </ul>

ted_sws/notice_validator/resources/templates/sparql_query_results_report.jinja2

@@ -1,14 +1,15 @@
 {% macro increment(dictionary, key, increment_by=1) %}
     {% if dictionary.update({key: dictionary[key] + increment_by}) %} {% endif %}
 {% endmacro %}
+
 {% set results_count = validation_results|length %}
 <!DOCTYPE html>
 <html lang="en">
   <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta http-equiv="X-UA-Compatible" content="ie=edge">
-    <title>HTML report for SPARQL Validation</title>
+    <title>{% include '_ms_meta_title_pre.jinja2' %}SPARQL Validation Report</title>
     <link href="https://cdn.datatables.net/1.10.22/css/jquery.dataTables.min.css" rel="stylesheet" type="text/css">
     <link rel="stylesheet" href="https://code.jquery.com/ui/1.13.2/themes/base/jquery-ui.css">
     <style type="text/css">
@@ -87,26 +88,28 @@
     </style>
   </head>
   <body>
-	<h1>SPARQL Validation HTML report </h1>
+	<h1>SPARQL Validation Report</h1>
 	<hr>
     <h2>Report details: </h2>
     <ul>
         <li>Date created: {{ created }}</li>
         <li>SPARQL test suite identifier: {{ test_suite_identifier }}</li>
         <li>Mapping suite identifier: {{ mapping_suite_identifier }}</li>
         {% if notice_ids %}
-        <li><div><hr></div>Notice identifier(s):
-            <div data-role="collapsible" data-state="{% if notice_ids|length > 1 %}collapsed{% endif %}"
+        <li><div><hr></div>
+            {%  set nr_notices = notice_ids|length %}
+            {% if nr_notices > 1 %}
+            Notice identifiers ({{ nr_notices }}):
+            <div data-role="collapsible" data-state="collapsed"
                 class="collapsible-wrapper">
                 <h4><a href="#"></a></h4>
                 <div class="ui-collapsible-content">
-                {% if notice_ids|length == 1 %}
-                    {{ notice_ids[0] }}
-                {% else %}
                     {{ notice_ids | tojson(indent=2) | replace('\n', '<br>') }}
-                {% endif %}
                 </div>
             </div>
+            {% elif nr_notices == 1 %}
+                <b>Notice identifier: {{ notice_ids[0] }}</b>
+            {% endif %}
         </li>
         {% endif %}
     </ul>