Merge pull request #220 from OP-TED/feature/TED-617

Feature/ted 617
OP-TED · Aug 30, 2022 · 28b2639 · 28b2639
2 parents 5ecd456 + 4ddb0ff
commit 28b2639
Show file tree

Hide file tree

Showing 42 changed files with 32,600 additions and 48 deletions.
diff --git a/dags/old_worker_single_notice_process_orchestrator.py b/dags/old_worker_single_notice_process_orchestrator.py
@@ -209,16 +209,17 @@ def _validate_transformed_rdf_manifestation(**context_args):
         mongodb_client = MongoClient(config.MONGO_DB_AUTH_URL)
         notice_repository = NoticeRepository(mongodb_client=mongodb_client)
         mapping_suite_repository = MappingSuiteRepositoryMongoDB(mongodb_client=mongodb_client)
+        validate_xpath_coverage_notice_by_id(notice_id=notice_id, mapping_suite_identifier=mapping_suite_id,
+                                             mapping_suite_repository=mapping_suite_repository,
+                                             mongodb_client=mongodb_client)
         validate_notice_by_id_with_sparql_suite(notice_id=notice_id, mapping_suite_identifier=mapping_suite_id,
                                                 notice_repository=notice_repository,
                                                 mapping_suite_repository=mapping_suite_repository)
         validate_notice_by_id_with_shacl_suite(notice_id=notice_id, mapping_suite_identifier=mapping_suite_id,
                                                notice_repository=notice_repository,
                                                mapping_suite_repository=mapping_suite_repository
                                                )
-        validate_xpath_coverage_notice_by_id(notice_id=notice_id, mapping_suite_identifier=mapping_suite_id,
-                                             mapping_suite_repository=mapping_suite_repository,
-                                             mongodb_client=mongodb_client)
+
         push_dag_downstream(NOTICE_ID, notice_id)
         push_dag_downstream(MAPPING_SUITE_ID, mapping_suite_id)
 

diff --git a/dags/worker_single_notice_process_orchestrator.py b/dags/worker_single_notice_process_orchestrator.py
@@ -200,9 +200,9 @@ def _validate_transformed_rdf_manifestation(**context_args):
         mapping_suite = pull_dag_upstream(MAPPING_SUITE_OBJECT)
         mongodb_client = MongoClient(config.MONGO_DB_AUTH_URL)
 
+        validate_xpath_coverage_notice(notice=notice, mapping_suite=mapping_suite, mongodb_client=mongodb_client)
         validate_notice_with_sparql_suite(notice=notice, mapping_suite_package=mapping_suite)
         validate_notice_with_shacl_suite(notice=notice, mapping_suite_package=mapping_suite)
-        validate_xpath_coverage_notice(notice=notice, mapping_suite=mapping_suite, mongodb_client=mongodb_client)
         push_dag_downstream(NOTICE_OBJECT, notice)
         context = get_current_context()
         handle_event_message_metadata_dag_context(event_message, context)

diff --git a/ted_sws/core/model/manifestation.py b/ted_sws/core/model/manifestation.py
@@ -127,6 +127,7 @@ class SPARQLQuery(PropertyBaseModel):
     """
     title: Optional[str]
     description: Optional[str]
+    xpath: Optional[List[str]] = []
     query: str
 
 
@@ -136,6 +137,9 @@ class SPARQLQueryResult(PropertyBaseModel):
     """
     query: SPARQLQuery
     result: Optional[str]
+    query_result: Optional[str]
+    fields_covered: Optional[bool] = True
+    missing_fields: Optional[List[str]] = []
     error: Optional[str]
     identifier: Optional[str]
 

diff --git a/ted_sws/event_manager/services/logger_from_context.py b/ted_sws/event_manager/services/logger_from_context.py
@@ -1,8 +1,7 @@
 from typing import Dict, Any, MutableMapping, Union
 
-from ted_sws.event_manager.adapters.event_handler_config import CLILoggerConfig, NULLLoggerConfig
+from ted_sws.event_manager.adapters.event_handler_config import NULLLoggerConfig, ConsoleLoggerConfig
 from ted_sws.event_manager.adapters.event_logger import EventLogger
-from ted_sws.event_manager.adapters.log import ConfigHandlerType
 from ted_sws.event_manager.adapters.log import EVENT_LOGGER_CONTEXT_KEY
 from ted_sws.event_manager.adapters.log import is_env_logging_enabled
 from ted_sws.event_manager.model.event_message import EventMessage, EventMessageProcessType, EventMessageMetadata
@@ -28,7 +27,7 @@ def get_env_logger(logger: EventLogger, is_cli: bool = False) -> EventLogger:
     if is_env_logging_enabled():
         return logger
     elif is_cli:
-        logger_config = CLILoggerConfig(config_handlers=ConfigHandlerType.ConsoleHandler.value)
+        logger_config = ConsoleLoggerConfig()
         logger_config.get_console_handler().logger.propagate = True
         return EventLogger(logger_config)
     else:

diff --git a/ted_sws/mapping_suite_processor/services/conceptual_mapping_generate_sparql_queries.py b/ted_sws/mapping_suite_processor/services/conceptual_mapping_generate_sparql_queries.py
@@ -53,7 +53,8 @@ def sparql_validation_generator(data: pd.DataFrame, base_xpath: str) -> Iterator
               f"#description: “{sf_field_id} - {sf_field_name}” in SF corresponds to “{e_form_bt_id} " \
               f"{e_form_bt_name}” in eForms. The corresponding XML element is " \
               f"{concat_field_xpath(base_xpath, field_xpath)}. " \
-              f"The expected ontology instances are epo: {class_path} ." \
+              f"The expected ontology instances are epo: {class_path} .\n" \
+              f"#xpath: {concat_field_xpath(base_xpath, field_xpath, separator=',')}" \
               "\n" + "\n" + "\n".join(prefixes) + "\n\n" \
                                                   f"ASK WHERE {{ {property_path} }}"
 

diff --git a/ted_sws/notice_validator/entrypoints/cli/cmd_sparql_runner.py b/ted_sws/notice_validator/entrypoints/cli/cmd_sparql_runner.py
@@ -7,12 +7,15 @@
 import click
 
 from ted_sws.core.adapters.cmd_runner import CmdRunner as BaseCmdRunner, DEFAULT_MAPPINGS_PATH
-from ted_sws.core.model.manifestation import RDFManifestation
+from ted_sws.core.model.manifestation import RDFManifestation, XMLManifestation
 from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryInFileSystem
 from ted_sws.event_manager.adapters.log import LOG_INFO_TEXT
 from ted_sws.notice_validator.entrypoints.cli import DEFAULT_RDF_FOLDER, DEFAULT_TEST_SUITE_REPORT_FOLDER
 from ted_sws.notice_validator.services.sparql_test_suite_runner import SPARQLTestSuiteRunner, SPARQLReportBuilder, \
     SPARQLTestSuiteValidationReport
+from ted_sws.notice_validator.entrypoints.cli.cmd_xpath_coverage_runner import JSON_REPORT_FILE as XPATH_JSON_FILE, \
+    DEFAULT_TEST_SUITE_REPORT_FOLDER
+from ted_sws.core.model.manifestation import XPATHCoverageValidationReport
 
 JSON_VALIDATIONS_REPORT = "sparql_validations.json"
 HTML_REPORT = "sparql_{id}.html"
@@ -49,9 +52,14 @@ def save_report(cls, report_path, report_name, report_id, content):
         with open(report_path / report_name, "w+") as f:
             f.write(content)
 
-    def validate(self, rdf_file, base_report_path):
+    def validate(self, rdf_file, xpath_report, base_report_path):
         self.log("Validating " + LOG_INFO_TEXT.format(rdf_file.name) + " ... ")
         rdf_manifestation = RDFManifestation(object_data=rdf_file.read_text(encoding="utf-8"))
+        xml_manifestation = None
+        if xpath_report:
+            xml_manifestation = XMLManifestation(object_data="",
+                                                 xpath_coverage_validation=XPATHCoverageValidationReport(
+                                                     **xpath_report))
 
         report_path = base_report_path / DEFAULT_TEST_SUITE_REPORT_FOLDER
         report_path.mkdir(parents=True, exist_ok=True)
@@ -60,6 +68,7 @@ def validate(self, rdf_file, base_report_path):
         sparql_test_suites = self.mapping_suite.sparql_test_suites
         for sparql_test_suite in sparql_test_suites:
             test_suite_execution = SPARQLTestSuiteRunner(rdf_manifestation=rdf_manifestation,
+                                                         xml_manifestation=xml_manifestation,
                                                          sparql_test_suite=sparql_test_suite,
                                                          mapping_suite=self.mapping_suite).execute_test_suite()
 
@@ -84,7 +93,9 @@ def run_cmd(self):
                     base_report_path = rdf_path / d.name
                     for f in d.iterdir():
                         if f.is_file():
-                            self.validate(rdf_file=f, base_report_path=base_report_path)
+                            xpath_file = f.parent / DEFAULT_TEST_SUITE_REPORT_FOLDER / XPATH_JSON_FILE
+                            xpath_report = json.load(open(xpath_file, "r")) if xpath_file.exists() else None
+                            self.validate(rdf_file=f, xpath_report=xpath_report, base_report_path=base_report_path)
         except Exception as e:
             error = e
 

diff --git a/ted_sws/notice_validator/entrypoints/cli/cmd_validation_summary_runner.py b/ted_sws/notice_validator/entrypoints/cli/cmd_validation_summary_runner.py
@@ -10,13 +10,13 @@
 from ted_sws.core.model.manifestation import XMLManifestation, RDFManifestation, XPATHCoverageValidationReport, \
     SPARQLTestSuiteValidationReport, SHACLTestSuiteValidationReport, ValidationSummaryReport
 from ted_sws.core.model.notice import Notice
+from ted_sws.notice_validator.entrypoints.cli import DEFAULT_TEST_SUITE_REPORT_FOLDER
 from ted_sws.notice_validator.entrypoints.cli.cmd_shacl_runner import JSON_VALIDATIONS_REPORT as JSON_SHACL_REPORT
 from ted_sws.notice_validator.entrypoints.cli.cmd_sparql_runner import JSON_VALIDATIONS_REPORT as JSON_SPARQL_REPORT
 from ted_sws.notice_validator.entrypoints.cli.cmd_xpath_coverage_runner import JSON_REPORT_FILE as XPATH_COV_REPORT
 from ted_sws.notice_validator.services.validation_summary_runner import generate_validation_summary_report_notices
 
 OUTPUT_FOLDER = '{mappings_path}/{mapping_suite_id}/' + DEFAULT_OUTPUT_PATH
-DEFAULT_TEST_SUITE_REPORT_FOLDER = "test_suite_report"
 
 REPORT_FILE = "validation_summary_report"
 CMD_NAME = "CMD_VALIDATION_SUMMARY_RUNNER"

diff --git a/ted_sws/notice_validator/entrypoints/cli/cmd_xpath_coverage_runner.py b/ted_sws/notice_validator/entrypoints/cli/cmd_xpath_coverage_runner.py
@@ -4,9 +4,8 @@
 import os
 from pathlib import Path
 from typing import List
-from pymongo import MongoClient
+
 import click
-from ted_sws import config
 
 from ted_sws.core.adapters.cmd_runner import CmdRunner as BaseCmdRunner, DEFAULT_MAPPINGS_PATH, DEFAULT_OUTPUT_PATH
 from ted_sws.core.model.manifestation import XMLManifestation
@@ -15,11 +14,11 @@
 from ted_sws.event_manager.adapters.log import LOG_INFO_TEXT
 from ted_sws.mapping_suite_processor.entrypoints.cli import CONCEPTUAL_MAPPINGS_FILE_TEMPLATE
 from ted_sws.notice_validator.adapters.xpath_coverage_runner import CoverageRunner
+from ted_sws.notice_validator.entrypoints.cli import DEFAULT_TEST_SUITE_REPORT_FOLDER
 from ted_sws.notice_validator.services.xpath_coverage_runner import coverage_notice_xpath_report, \
     xpath_coverage_html_report, xpath_coverage_json_report
 
 OUTPUT_FOLDER = '{mappings_path}/{mapping_suite_id}/' + DEFAULT_OUTPUT_PATH
-DEFAULT_TEST_SUITE_REPORT_FOLDER = "test_suite_report"
 REPORT_FILE = "xpath_coverage_validation"
 JSON_REPORT_FILE = REPORT_FILE + ".json"
 CMD_NAME = "CMD_XPATH_COVERAGE_RUNNER"

diff --git a/ted_sws/notice_validator/resources/templates/sparql_query_results_report.jinja2 b/ted_sws/notice_validator/resources/templates/sparql_query_results_report.jinja2
@@ -10,21 +10,47 @@
     <meta http-equiv="X-UA-Compatible" content="ie=edge">
     <title>HTML report for SPARQL Validation</title>
     <link href="https://cdn.datatables.net/1.10.22/css/jquery.dataTables.min.css" rel="stylesheet" type="text/css">
+    <link rel="stylesheet" href="//code.jquery.com/ui/1.13.2/themes/base/jquery-ui.css">
     <style type="text/css">
         body {
             font-family: Arial, sans-serif;
                 margin: 25px auto;
                 width: 80%;
         }
-        .error {
-            color: red;
-        }
-        .success {
-            color: green;
+        table th, table td {
+            vertical-align: top;
         }
         table thead th {
             text-align: left;
         }
+        .success {
+          color: #3c763d;
+        }
+        .info {
+          color: #31708f;
+        }
+        .warning {
+          color: #8a6d3b;
+        }
+        .danger {
+          color: #a94442;
+        }
+        .strong {
+            font-weight: bold;
+        }
+        .dialog h4 {
+            margin: 0 0 10px;
+        }
+        .dialog ul, .dialog ol {
+            margin-top: 0;
+        }
+        .dialog {
+            display: none;
+        }
+        .dialog li {
+            margin-bottom: 5px;
+        }
+
     </style>
   </head>
   <body>
@@ -45,6 +71,7 @@
             <th>Description</th>
             <th>Query content</th>
             <th>Result</th>
+            <th>Details</th>
             <th>Error</th>
         </tr>
      </thead>
@@ -57,9 +84,46 @@
         {% for result in validation_results %}
             <tr>
                 <td>{{ result.query.title }}</td>
-                <td>{{ result.query.description }}</td>
+                <td>
+                    <button class="open-dialog">Description</button>
+                    <div class="dialog" title="{{ result.query.title }}">
+                        {{ result.query.description }}
+                    </div>
+                </td>
                 <td>{{ result.query.query | e | replace('\n', '<br>') }}</td>
-                <td{% if result.result == "True" %} class="success" {% else %} class="error" {% endif %}>{{ result.result }}</td>
+                <td{% if result.result == "True" and result.fields_covered %} class="strong success" {% elif result.result == "True" and not result.fields_covered %} class="strong warning" {% else %} class="strong danger" {% endif %}>{{ result.result }}</td>
+                <td>
+                    <button class="open-dialog">Details</button>
+                    <div class="dialog" title="{{ result.query.title }}">
+                    <ul>
+                    <li>Query result: {{ result.query_result }}</li>
+                    {% if result.query.xpath|length == 0 %}
+                        <li>No XPATHs provided</li>
+                    {% endif %}
+                    <li>Fields are covered: {{ result.fields_covered }}</li>
+                    {% if result.query.xpath|length > 0 %}
+                        <li>
+                        <h4 class="info">Fields:</h4>
+                        <ol>
+                        {% for field in result.query.xpath %}
+                            <li>{{ field }}</li>
+                        {% endfor %}
+                        </ol>
+                        </li>
+                    {% endif %}
+                    {% if result.missing_fields|length > 0 %}
+                        <li>
+                        <h4 class="danger">Missing fields:</h4>
+                        <ol>
+                        {% for field in result.missing_fields %}
+                            <li>{{ field }}</li>
+                        {% endfor %}
+                        </ol>
+                        </li>
+                    {% endif %}
+                    </ul>
+                    </div>
+                </td>
                 <td>{{ result.error }}</td>
             </tr>
             {% if result.result == "True" %}
@@ -132,6 +196,23 @@
             }
         });
 
+        $(document).on('click', '.open-dialog', function () {
+            $dialog = $(this).next('.dialog');
+            $dialog.dialog({
+                        width: 600,
+                        modal: true,
+                        buttons: {
+                                        "Close": function(event, ui) {
+                                          $dialog.dialog('destroy');
+                                        }
+                                     },
+                        close: function(event, ui) {
+                                    $dialog.dialog('destroy');
+                                }
+                        })
+            $dialog.dialog('open');
+            return false;
+        });
     });
 
 </script>