From 432926c66738286c86953269ec31fdaddc56394d Mon Sep 17 00:00:00 2001 From: Bogdan Kostov Date: Thu, 11 Jul 2024 14:36:14 +0200 Subject: [PATCH 1/2] [Fix partially kbss-cvut/23ava-distribution#120] Optimize records export - optimize where clause - wrap where clause record and institution graphs --- .../persistence/dao/PatientRecordDao.java | 11 +- .../resources/query/find-raw-records.sparql | 313 +++++++++--------- 2 files changed, 164 insertions(+), 160 deletions(-) diff --git a/src/main/java/cz/cvut/kbss/study/persistence/dao/PatientRecordDao.java b/src/main/java/cz/cvut/kbss/study/persistence/dao/PatientRecordDao.java index a64d6e19..79d2c9b8 100644 --- a/src/main/java/cz/cvut/kbss/study/persistence/dao/PatientRecordDao.java +++ b/src/main/java/cz/cvut/kbss/study/persistence/dao/PatientRecordDao.java @@ -284,23 +284,26 @@ private void setQueryParameters(Query query, Map queryParams) { URI.create(Vocabulary.s_p_was_treated_at)) .setParameter("hasKey", URI.create(Vocabulary.s_p_key)) .setParameter("hasCreatedDate", URI.create(Vocabulary.s_p_created)) - .setParameter("hasLastModified", URI.create(Vocabulary.s_p_modified)); + .setParameter("hasLastModified", URI.create(Vocabulary.s_p_modified)) + .setParameter("institutionGraph", URI.create(Vocabulary.s_c_institution + "s")); queryParams.forEach(query::setParameter); } private static String constructWhereClause(RecordFilterParams filters, Map queryParams) { // Could not use Criteria API because it does not support OPTIONAL - String whereClause = "{" + + String whereClause = "{GRAPH ?r{" + "?r a ?type ; " + "?hasCreatedDate ?created ; " + "?hasInstitution ?institution . " + - "?institution ?hasKey ?institutionKey ." + "OPTIONAL { ?r ?hasPhase ?phase . } " + "OPTIONAL { ?r ?hasFormTemplate ?formTemplate . } " + "OPTIONAL { ?r ?hasLastModified ?lastModified . } " + "BIND (COALESCE(?lastModified, ?created) AS ?date) "; whereClause += mapParamsToQuery(filters, queryParams); - whereClause += "}"; + whereClause += "}" + + "GRAPH ?institutionGraph{" + + "?institution ?hasKey ?institutionKey ." + + "}}"; return whereClause; } diff --git a/src/main/resources/query/find-raw-records.sparql b/src/main/resources/query/find-raw-records.sparql index 902cf11a..0c9ccb0a 100644 --- a/src/main/resources/query/find-raw-records.sparql +++ b/src/main/resources/query/find-raw-records.sparql @@ -21,183 +21,184 @@ SELECT ?r (?r as ?uri) ?flightHours ?numberOfAirframeOverhauls ?classificationOfOccurrence ?failureAscertainmentCircumstances (?repeatedFailureCode as ?repeatedFailure) ?failureCause ?consequence ?mission ?repair (str(?repairDurationStr) as ?repairDuration) ?averageNumberOfMenDuringRepairment ?failureDescription ?descriptionOfCorrectiveAction ?yearOfProductionOfDefectiveEquipment ?numberOfOverhaulsOfDefectiveEquipment ?serialNoOf ?notes ?fhaEvent { ###RECORD_CLAUSE### + GRAPH ?r { + ?r rm:has-question ?f. + ?f doc:has_related_question ?s1. + ?f doc:has_related_question ?s2. + FILTER(?s2 != ?s1) + ?s2 doc:has_related_question ?fhaEventQ. + ?fhaEventQ form:has-question-origin avamod:fha-event. - ?r rm:has-question ?f. - ?f doc:has_related_question ?s1. - ?f doc:has_related_question ?s2. - FILTER(?s2 != ?s1) - ?s2 doc:has_related_question ?fhaEventQ. - ?fhaEventQ form:has-question-origin avamod:fha-event. + ?s1 doc:has_related_question ?FUSq. + ?FUSq form:has-question-origin avamod:fuselage-no. - ?s1 doc:has_related_question ?FUSq. - ?FUSq form:has-question-origin avamod:fuselage-no. + ?s1 doc:has_related_question ?Cq. + ?Cq form:has-question-origin ?componentQuestionOrigin. + FILTER(contains(str(?componentQuestionOrigin), "http://vfn.cz/ontologies/ava-study/model/system-equipment-block-part-")) - ?s1 doc:has_related_question ?Cq. - ?Cq form:has-question-origin ?componentQuestionOrigin. - FILTER(contains(str(?componentQuestionOrigin), "http://vfn.cz/ontologies/ava-study/model/system-equipment-block-part-")) + ?s1 doc:has_related_question ?failDateq. + ?failDateq form:has-question-origin avamod:date-of-failure-ascertainment. - ?s1 doc:has_related_question ?failDateq. - ?failDateq form:has-question-origin avamod:date-of-failure-ascertainment. + ?s1 doc:has_related_question ?FHq. + ?FHq form:has-question-origin avamod:flight-hours-of-airframe-since-the-service-beginning. - ?s1 doc:has_related_question ?FHq. - ?FHq form:has-question-origin avamod:flight-hours-of-airframe-since-the-service-beginning. + ?s1 doc:has_related_question ?classificationOfOccurrenceQ. + ?classificationOfOccurrenceQ form:has-question-origin avamod:classification-of-occurrence. - ?s1 doc:has_related_question ?classificationOfOccurrenceQ. - ?classificationOfOccurrenceQ form:has-question-origin avamod:classification-of-occurrence. + ?s1 doc:has_related_question ?failureAscertainmentCircumstancesQ. + ?failureAscertainmentCircumstancesQ form:has-question-origin avamod:failure-ascertainment-circumstances. - ?s1 doc:has_related_question ?failureAscertainmentCircumstancesQ. - ?failureAscertainmentCircumstancesQ form:has-question-origin avamod:failure-ascertainment-circumstances. + ?s1 doc:has_related_question ?repeatedFailureQ. + ?repeatedFailureQ form:has-question-origin avamod:repeated-failure. - ?s1 doc:has_related_question ?repeatedFailureQ. - ?repeatedFailureQ form:has-question-origin avamod:repeated-failure. + ?s1 doc:has_related_question ?failureCauseQ. + ?failureCauseQ form:has-question-origin avamod:failure-cause . - ?s1 doc:has_related_question ?failureCauseQ. - ?failureCauseQ form:has-question-origin avamod:failure-cause . + ?s1 doc:has_related_question ?consequenceQ. + ?consequenceQ form:has-question-origin avamod:consequence. - ?s1 doc:has_related_question ?consequenceQ. - ?consequenceQ form:has-question-origin avamod:consequence. + ?s1 doc:has_related_question ?missionQ. + ?missionQ form:has-question-origin avamod:mission. - ?s1 doc:has_related_question ?missionQ. - ?missionQ form:has-question-origin avamod:mission. + ?s1 doc:has_related_question ?repairQ. + ?repairQ form:has-question-origin avamod:repair. - ?s1 doc:has_related_question ?repairQ. - ?repairQ form:has-question-origin avamod:repair. + ?s1 doc:has_related_question ?repairDurationQ. + ?repairDurationQ form:has-question-origin avamod:repair-duration. - ?s1 doc:has_related_question ?repairDurationQ. - ?repairDurationQ form:has-question-origin avamod:repair-duration. + ?s1 doc:has_related_question ?averageNumberOfMenDuringRepairmentQ. + ?averageNumberOfMenDuringRepairmentQ form:has-question-origin avamod:average-number-of-men-during-repairment. - ?s1 doc:has_related_question ?averageNumberOfMenDuringRepairmentQ. - ?averageNumberOfMenDuringRepairmentQ form:has-question-origin avamod:average-number-of-men-during-repairment. + ?s1 doc:has_related_question ?failureDescriptionQ. + ?failureDescriptionQ form:has-question-origin avamod:failure-description. - ?s1 doc:has_related_question ?failureDescriptionQ. - ?failureDescriptionQ form:has-question-origin avamod:failure-description. + ?s1 doc:has_related_question ?descriptionOfCorrectiveActionQ. + ?descriptionOfCorrectiveActionQ form:has-question-origin avamod:description-of-corrective-action. - ?s1 doc:has_related_question ?descriptionOfCorrectiveActionQ. - ?descriptionOfCorrectiveActionQ form:has-question-origin avamod:description-of-corrective-action. + ?s1 doc:has_related_question ?numberOfAirframeOverhaulsQ. + ?numberOfAirframeOverhaulsQ form:has-question-origin avamod:number-of-airframe-overhauls. - ?s1 doc:has_related_question ?numberOfAirframeOverhaulsQ. - ?numberOfAirframeOverhaulsQ form:has-question-origin avamod:number-of-airframe-overhauls. + ?s1 doc:has_related_question ?yearOfProductionOfDefectiveEquipmentQ. + ?yearOfProductionOfDefectiveEquipmentQ form:has-question-origin avamod:year-of-production-of-defective-equipment. - ?s1 doc:has_related_question ?yearOfProductionOfDefectiveEquipmentQ. - ?yearOfProductionOfDefectiveEquipmentQ form:has-question-origin avamod:year-of-production-of-defective-equipment. + ?s1 doc:has_related_question ?numberOfOverhaulsOfDefectiveEquipmentQ. + ?numberOfOverhaulsOfDefectiveEquipmentQ form:has-question-origin avamod:number-of-overhauls-of-defective-equipment. - ?s1 doc:has_related_question ?numberOfOverhaulsOfDefectiveEquipmentQ. - ?numberOfOverhaulsOfDefectiveEquipmentQ form:has-question-origin avamod:number-of-overhauls-of-defective-equipment. - -# ?s1 doc:has_related_question ?serialNoOfQ. -# ?serialNoOfQ form:has-question-origin avamod:serial-no-of. - - ?s1 doc:has_related_question ?notesQ. - ?notesQ form:has-question-origin avamod:notes. - OPTIONAL { - # avamod:fuselage-no - ?FUSq doc:has_answer ?FUSa. - ?FUSa doc:has_data_value ?fus. - - #avamod:system-equipment-block-part-l-39ng.a1 - # ?comp - label of general component/system - # ?comp_iri - iri of component type specific to the aircraft - ?Cq doc:has_answer ?Ca. - ?Ca doc:has_object_value ?ac_comp. - - #avamod:date-of-failure-ascertainment - # ?failDate - creation date filter - ?failDateq doc:has_answer ?failDatea. - ?failDatea doc:has_data_value ?failDateStr. - # BIND(spif:parseDate(?failDateStr, "DD-MM-yyyy") as ?failDate) - # FILTER((!BOUND(?createdBefore) || ?createdBefore < ?failDate ) && (!BOUND(?createdAfter) || ?createdAfter >= ?failDate )) - - #avamod:flight-hours-of-airframe-since-the-service-beginning - - ?FHq doc:has_answer ?FHa. - ?FHa doc:has_data_value ?flightHours. - - #avamod:classification-of-occurrence - ?classificationOfOccurrenceQ doc:has_answer ?classificationOfOccurrenceA. - ?classificationOfOccurrenceA doc:has_object_value ?classificationOfOccurrence. - - #avamod:failure-ascertainment-circumstances - - ?failureAscertainmentCircumstancesQ doc:has_answer ?failureAscertainmentCircumstancesA. - ?failureAscertainmentCircumstancesA doc:has_object_value ?failureAscertainmentCircumstances. - - # during flight - # BIND(IF(BOUND(?failureAscertainmentCircumstances) && ?failureAscertainmentCircumstances = avadom:during-flight, 1, 0) as ?duringFlight) - - #avamod:repeated-failure - ?repeatedFailureQ doc:has_answer ?repeatedFailureA. - ?repeatedFailureA doc:has_object_value ?repeatedFailureCode. - - OPTIONAL{ - #avamod:failure-cause - ?failureCauseQ doc:has_answer ?failureCauseA. - ?failureCauseA doc:has_object_value ?failureCause. - } - #avamod:consequence - - ?consequenceQ doc:has_answer ?consequenceA. - ?consequenceA doc:has_object_value ?consequence. - - #avamod:mission - ?missionQ doc:has_answer ?missionA. - ?missionA doc:has_object_value ?mission. - - #avamod:repair - ?repairQ doc:has_answer ?repairA. - ?repairA doc:has_object_value ?repair. - - #avamod:repair-duration - ?repairDurationQ doc:has_answer ?repairDurationA. - ?repairDurationA doc:has_data_value ?repairDurationStr. - - #avamod:average-number-of-men-during-repairment - ?averageNumberOfMenDuringRepairmentQ doc:has_answer ?averageNumberOfMenDuringRepairmentA. - ?averageNumberOfMenDuringRepairmentA doc:has_data_value ?averageNumberOfMenDuringRepairmentStr. - BIND(xsd:decimal(str(?averageNumberOfMenDuringRepairmentStr)) as ?averageNumberOfMenDuringRepairment) - - #avamod:failure-description - ?failureDescriptionQ doc:has_answer ?failureDescriptionA. - ?failureDescriptionA doc:has_data_value ?failureDescription. - - #avamod:description-of-corrective-action - ?descriptionOfCorrectiveActionQ doc:has_answer ?descriptionOfCorrectiveActionA. - ?descriptionOfCorrectiveActionA doc:has_data_value ?descriptionOfCorrectiveAction. - - OPTIONAL { - #avamod:number-of-airframe-overhauls - ?numberOfAirframeOverhaulsQ doc:has_answer ?numberOfAirframeOverhaulsA. - ?numberOfAirframeOverhaulsA doc:has_data_value ?numberOfAirframeOverhaulsStr. - BIND(xsd:integer(str(?numberOfAirframeOverhaulsStr)) as ?numberOfAirframeOverhauls) - } + # ?s1 doc:has_related_question ?serialNoOfQ. + # ?serialNoOfQ form:has-question-origin avamod:serial-no-of. + ?s1 doc:has_related_question ?notesQ. + ?notesQ form:has-question-origin avamod:notes. OPTIONAL { - #avamod:year-of-production-of-defective-equipment - ?yearOfProductionOfDefectiveEquipmentQ doc:has_answer ?yearOfProductionOfDefectiveEquipmentA. - ?yearOfProductionOfDefectiveEquipmentA doc:has_data_value ?yearOfProductionOfDefectiveEquipment.# TODO - transform to more suitable datatype - FILTER(str(spif:trim(?yearOfProductionOfDefectiveEquipment)) != "" ) - } - - OPTIONAL { - #avamod:number-of-overhauls-of-defective-equipment - ?numberOfOverhaulsOfDefectiveEquipmentQ doc:has_answer ?numberOfOverhaulsOfDefectiveEquipmentA. - ?numberOfOverhaulsOfDefectiveEquipmentA doc:has_data_value ?numberOfOverhaulsOfDefectiveEquipment.# TODO - transform to more suitable datatype - FILTER(str(spif:trim(?numberOfOverhaulsOfDefectiveEquipment)) != "" ) - # } - # OPTIONAL{ - # #avamod:serial-no-of - # ?serialNoOfQ doc:has_answer ?serialNoOfA. - # ?serialNoOfA doc:has_data_value ?serialNoOf. - } - OPTIONAL{ - #avamod:notes - ?notesQ doc:has_answer ?notesA. - ?notesA doc:has_data_value ?notes. - FILTER(str(spif:trim(?notes)) != "" ) - } - OPTIONAL{ - #avamod:fha-event - ?fhaEventQ doc:has_answer ?fhaEventA. - ?fhaEventA doc:has_object_value ?fhaEvent. + # avamod:fuselage-no + ?FUSq doc:has_answer ?FUSa. + ?FUSa doc:has_data_value ?fus. + + #avamod:system-equipment-block-part-l-39ng.a1 + # ?comp - label of general component/system + # ?comp_iri - iri of component type specific to the aircraft + ?Cq doc:has_answer ?Ca. + ?Ca doc:has_object_value ?ac_comp. + + #avamod:date-of-failure-ascertainment + # ?failDate - creation date filter + ?failDateq doc:has_answer ?failDatea. + ?failDatea doc:has_data_value ?failDateStr. + # BIND(spif:parseDate(?failDateStr, "DD-MM-yyyy") as ?failDate) + # FILTER((!BOUND(?createdBefore) || ?createdBefore < ?failDate ) && (!BOUND(?createdAfter) || ?createdAfter >= ?failDate )) + + #avamod:flight-hours-of-airframe-since-the-service-beginning + + ?FHq doc:has_answer ?FHa. + ?FHa doc:has_data_value ?flightHours. + + #avamod:classification-of-occurrence + ?classificationOfOccurrenceQ doc:has_answer ?classificationOfOccurrenceA. + ?classificationOfOccurrenceA doc:has_object_value ?classificationOfOccurrence. + + #avamod:failure-ascertainment-circumstances + + ?failureAscertainmentCircumstancesQ doc:has_answer ?failureAscertainmentCircumstancesA. + ?failureAscertainmentCircumstancesA doc:has_object_value ?failureAscertainmentCircumstances. + + # during flight + # BIND(IF(BOUND(?failureAscertainmentCircumstances) && ?failureAscertainmentCircumstances = avadom:during-flight, 1, 0) as ?duringFlight) + + #avamod:repeated-failure + ?repeatedFailureQ doc:has_answer ?repeatedFailureA. + ?repeatedFailureA doc:has_object_value ?repeatedFailureCode. + + OPTIONAL{ + #avamod:failure-cause + ?failureCauseQ doc:has_answer ?failureCauseA. + ?failureCauseA doc:has_object_value ?failureCause. + } + #avamod:consequence + + ?consequenceQ doc:has_answer ?consequenceA. + ?consequenceA doc:has_object_value ?consequence. + + #avamod:mission + ?missionQ doc:has_answer ?missionA. + ?missionA doc:has_object_value ?mission. + + #avamod:repair + ?repairQ doc:has_answer ?repairA. + ?repairA doc:has_object_value ?repair. + + #avamod:repair-duration + ?repairDurationQ doc:has_answer ?repairDurationA. + ?repairDurationA doc:has_data_value ?repairDurationStr. + + #avamod:average-number-of-men-during-repairment + ?averageNumberOfMenDuringRepairmentQ doc:has_answer ?averageNumberOfMenDuringRepairmentA. + ?averageNumberOfMenDuringRepairmentA doc:has_data_value ?averageNumberOfMenDuringRepairmentStr. + BIND(xsd:decimal(str(?averageNumberOfMenDuringRepairmentStr)) as ?averageNumberOfMenDuringRepairment) + + #avamod:failure-description + ?failureDescriptionQ doc:has_answer ?failureDescriptionA. + ?failureDescriptionA doc:has_data_value ?failureDescription. + + #avamod:description-of-corrective-action + ?descriptionOfCorrectiveActionQ doc:has_answer ?descriptionOfCorrectiveActionA. + ?descriptionOfCorrectiveActionA doc:has_data_value ?descriptionOfCorrectiveAction. + + OPTIONAL { + #avamod:number-of-airframe-overhauls + ?numberOfAirframeOverhaulsQ doc:has_answer ?numberOfAirframeOverhaulsA. + ?numberOfAirframeOverhaulsA doc:has_data_value ?numberOfAirframeOverhaulsStr. + BIND(xsd:integer(str(?numberOfAirframeOverhaulsStr)) as ?numberOfAirframeOverhauls) + } + + OPTIONAL { + #avamod:year-of-production-of-defective-equipment + ?yearOfProductionOfDefectiveEquipmentQ doc:has_answer ?yearOfProductionOfDefectiveEquipmentA. + ?yearOfProductionOfDefectiveEquipmentA doc:has_data_value ?yearOfProductionOfDefectiveEquipment.# TODO - transform to more suitable datatype + FILTER(str(spif:trim(?yearOfProductionOfDefectiveEquipment)) != "" ) + } + + OPTIONAL { + #avamod:number-of-overhauls-of-defective-equipment + ?numberOfOverhaulsOfDefectiveEquipmentQ doc:has_answer ?numberOfOverhaulsOfDefectiveEquipmentA. + ?numberOfOverhaulsOfDefectiveEquipmentA doc:has_data_value ?numberOfOverhaulsOfDefectiveEquipment.# TODO - transform to more suitable datatype + FILTER(str(spif:trim(?numberOfOverhaulsOfDefectiveEquipment)) != "" ) + # } + # OPTIONAL{ + # #avamod:serial-no-of + # ?serialNoOfQ doc:has_answer ?serialNoOfA. + # ?serialNoOfA doc:has_data_value ?serialNoOf. + } + OPTIONAL{ + #avamod:notes + ?notesQ doc:has_answer ?notesA. + ?notesA doc:has_data_value ?notes. + FILTER(str(spif:trim(?notes)) != "" ) + } + OPTIONAL{ + #avamod:fha-event + ?fhaEventQ doc:has_answer ?fhaEventA. + ?fhaEventA doc:has_object_value ?fhaEvent. + } } } } From 49643ab7763ded2f2140fe0be0f595992a26ed7f Mon Sep 17 00:00:00 2001 From: Bogdan Kostov Date: Thu, 11 Jul 2024 15:53:53 +0200 Subject: [PATCH 2/2] [Fix partially kbss-cvut/23ava-distribution#120] Fix failing tests by reverting optimization, wrapping where clause in named graphs, for record export as JSON --- .../persistence/dao/PatientRecordDao.java | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/main/java/cz/cvut/kbss/study/persistence/dao/PatientRecordDao.java b/src/main/java/cz/cvut/kbss/study/persistence/dao/PatientRecordDao.java index 79d2c9b8..d7015067 100644 --- a/src/main/java/cz/cvut/kbss/study/persistence/dao/PatientRecordDao.java +++ b/src/main/java/cz/cvut/kbss/study/persistence/dao/PatientRecordDao.java @@ -245,7 +245,7 @@ private Page findRecords(RecordFilterParams filters, Pageable pageSpec, C public Page findAllRecordsRaw(RecordFilterParams filters, Pageable pageSpec){ final Map queryParams = new HashMap<>(); - final String whereClause = constructWhereClause(filters, queryParams); + final String whereClause = constructWhereClauseWithGraphs(filters, queryParams); final String queryStringNoPaging = Utils.loadQuery(FIND_ALL_RAW_PATIENT_RECORDS) .replaceFirst(RECORDS_CLAUSE_TEMPLATE_VAR, whereClause); @@ -284,12 +284,27 @@ private void setQueryParameters(Query query, Map queryParams) { URI.create(Vocabulary.s_p_was_treated_at)) .setParameter("hasKey", URI.create(Vocabulary.s_p_key)) .setParameter("hasCreatedDate", URI.create(Vocabulary.s_p_created)) - .setParameter("hasLastModified", URI.create(Vocabulary.s_p_modified)) - .setParameter("institutionGraph", URI.create(Vocabulary.s_c_institution + "s")); + .setParameter("hasLastModified", URI.create(Vocabulary.s_p_modified)); queryParams.forEach(query::setParameter); } private static String constructWhereClause(RecordFilterParams filters, Map queryParams) { + // Could not use Criteria API because it does not support OPTIONAL + String whereClause = "{" + + "?r a ?type ; " + + "?hasCreatedDate ?created ; " + + "?hasInstitution ?institution . " + + "?institution ?hasKey ?institutionKey ." + + "OPTIONAL { ?r ?hasPhase ?phase . } " + + "OPTIONAL { ?r ?hasFormTemplate ?formTemplate . } " + + "OPTIONAL { ?r ?hasLastModified ?lastModified . } " + + "BIND (COALESCE(?lastModified, ?created) AS ?date) "; + whereClause += mapParamsToQuery(filters, queryParams); + whereClause += "}"; + return whereClause; + } + + private static String constructWhereClauseWithGraphs(RecordFilterParams filters, Map queryParams) { // Could not use Criteria API because it does not support OPTIONAL String whereClause = "{GRAPH ?r{" + "?r a ?type ; " + @@ -304,6 +319,9 @@ private static String constructWhereClause(RecordFilterParams filters, Map