From 4f14d5e1e7f334e2ce8c2d0b35ce04ed9323309e Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Thu, 2 Nov 2023 09:57:55 +0100 Subject: [PATCH 01/42] some methods --- Cargo.toml | 1 + src/ast.rs | 22 ++--- src/cql.rs | 245 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 6 +- src/util.rs | 5 +- 5 files changed, 262 insertions(+), 17 deletions(-) create mode 100644 src/cql.rs diff --git a/Cargo.toml b/Cargo.toml index a37ceb6..656d97f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ once_cell = "1.18" # Command Line Interface clap = { version = "4.0", default_features = false, features = ["std", "env", "derive", "help"] } rand = { default-features = false, version = "0.8.5" } +chrono = "0.4.31" [dev-dependencies] tokio-test = "0.4.2" diff --git a/src/ast.rs b/src/ast.rs index 7f64c24..3cf99d1 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -41,34 +41,34 @@ pub enum ConditionValue { #[derive(Serialize, Deserialize, Debug, Clone)] pub struct NumRange { - min: f64, - max: f64, + pub min: f64, + pub max: f64, } #[derive(Serialize, Deserialize, Debug, Clone)] pub struct DateRange { - min: String, //we don't parse dates yet - max: String, + pub min: String, //we don't parse dates yet + pub max: String, } #[derive(Serialize, Deserialize, Debug, Clone)] pub struct Operation { - operand: Operand, - children: Vec, + pub operand: Operand, + pub children: Vec, } #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] pub struct Condition { - key: String, - type_: ConditionType, - value: ConditionValue, + pub key: String, + pub type_: ConditionType, + pub value: ConditionValue, } #[derive(Serialize, Deserialize, Debug, Clone)] pub struct Ast { - ast: Operation, - id: String, + pub ast: Operation, + pub id: String, } diff --git a/src/cql.rs b/src/cql.rs new file mode 100644 index 0000000..c5272c6 --- /dev/null +++ b/src/cql.rs @@ -0,0 +1,245 @@ +use crate::ast; + +use once_cell::sync::Lazy; +use std::collections::HashMap; + +static ALIAS: Lazy> = Lazy::new(|| { + let map = [ + ("icd10", "http://hl7.org/fhir/sid/icd-10"), + ("icd10gm", "http://fhir.de/CodeSystem/dimdi/icd-10-gm"), + ("loinc", "http://loinc.org"), + ( + "SampleMaterialType", + "https://fhir.bbmri.de/CodeSystem/SampleMaterialType", + ), + ( + "StorageTemperature", + "https://fhir.bbmri.de/CodeSystem/StorageTemperature", + ), + ( + "FastingStatus", + "http://terminology.hl7.org/CodeSystem/v2-0916", + ), + ( + "SmokingStatus", + "http://hl7.org/fhir/uv/ips/ValueSet/current-smoking-status-uv-ips", + ), + ] + .into(); + + map +}); + + +pub fn bbmri(ast: ast::Ast) -> String { + + let mut query: String = "(".to_string(); + + let mut filter: String = "".to_string(); + + let mut lists: String = "".to_string(); + + match ast.ast.operand { + ast::Operand::And => { + query += " and "; + + }, + ast::Operand::Or => { + query += " or "; + }, + } + + for grandchild in ast.ast.children { + + process(grandchild, &mut query, &mut filter, &mut lists); + + } + + query += ")"; + + + query + filter.as_str() + lists.as_str() + +} + +pub fn process(child: ast::Child, query: &mut String, filter: &mut String, lists: &mut String ) { + + let mut query_cond: String = "(".to_string(); + let mut filter_cond: String = "(".to_string(); + + match child { + + ast::Child::Condition(condition) => { + + query_cond += condition.key.as_str(); + filter_cond += condition.key.as_str(); + + + match condition.type_ { + ast::ConditionType::Between => { + query_cond += " between "; + }, + ast::ConditionType::In => { + query_cond += " in "; + }, + ast::ConditionType::Equals => { + query_cond += " equals "; + }, + ast::ConditionType::NotEquals => { + query_cond += " not_equals "; + }, + ast::ConditionType::Contains => { + query_cond += " contains "; + }, + ast::ConditionType::GreaterThan => { + query_cond += " greater than "; + }, + ast::ConditionType::LowerThan => { + query_cond += " lower than "; + } + + } + + query_cond += " "; + + match condition.value { + ast::ConditionValue::Boolean(value) => { + query_cond += value.to_string().as_str(); + }, + ast::ConditionValue::DateRange(date_range) => { + query_cond += date_range.min.as_str(); + query_cond += ","; + query_cond += date_range.max.as_str(); + }, + ast::ConditionValue::NumRange(num_range) => { + query_cond += num_range.min.to_string().as_str(); + query_cond += ","; + query_cond += num_range.max.to_string().as_str(); + }, + ast::ConditionValue::Number(value) => { + query_cond += value.to_string().as_str(); + }, + ast::ConditionValue::String(value) => { + query_cond += value.as_str(); + }, + ast::ConditionValue::StringArray(string_array) => { + for value in &string_array { + query_cond += value; + query_cond += ","; + } + query_cond += " greater than "; + } + + } + + + query_cond += " "; + }, + + + ast::Child::Operation(operation) => { + match operation.operand { + ast::Operand::And => { + query_cond += " and "; + + }, + ast::Operand::Or => { + query_cond += " or "; + }, + } + + for grandchild in operation.children { + process(grandchild, &mut query_cond, &mut filter_cond, lists); + + } + + }, + + } + + query_cond += ")"; + + *query += query_cond.as_str(); + *filter += filter_cond.as_str(); + +} + + + +#[cfg(test)] +mod test { + use super::*; + const AST: &str = r#"{"ast":{"operand":"AND","children":[{"key":"age","type":"EQUALS","value":5.0}]},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + + const MALE_OR_FEMALE: &str = r#"{"ast":{"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"gender","type":"EQUALS","system":"","value":"male"},{"key":"gender","type":"EQUALS","system":"","value":"female"}]}]}]},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + + const ALL_GLIOMS: &str = r#"{"ast": {"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis","type":"EQUALS","system":"","value":"D43.%"}]},{"operand":"OR","children":[{"key":"59847-4","type":"EQUALS","system":"","value":"9383/1"},{"key":"59847-4","type":"EQUALS","system":"","value":"9384/1"},{"key":"59847-4","type":"EQUALS","system":"","value":"9394/1"},{"key":"59847-4","type":"EQUALS","system":"","value":"9421/1"}]}]},{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis","type":"EQUALS","system":"","value":"C71.%"},{"key":"diagnosis","type":"EQUALS","system":"","value":"C72.%"}]},{"operand":"OR","children":[{"key":"59847-4","type":"EQUALS","system":"","value":"9382/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9391/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9400/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9424/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9425/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9450/3"}]}]},{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis","type":"EQUALS","system":"","value":"C71.%"},{"key":"diagnosis","type":"EQUALS","system":"","value":"C72.%"}]},{"operand":"OR","children":[{"key":"59847-4","type":"EQUALS","system":"","value":"9440/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9441/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9442/3"}]}]},{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis","type":"EQUALS","system":"","value":"C71.%"},{"key":"diagnosis","type":"EQUALS","system":"","value":"C72.%"}]},{"operand":"OR","children":[{"key":"59847-4","type":"EQUALS","system":"","value":"9381/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9382/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9401/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9451/3"}]}]}]}]}]},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + + const AGE_AT_DIAGNOSIS_30_TO_70: &str = r#"{"ast": {"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"age_at_primary_diagnosis","type":"BETWEEN","system":"","value":{"min":30,"max":70}}]}]}]}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + + const AGE_AT_DIAGNOSIS_LOWER_THAN_70: &str = r#"{"ast": {"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"age_at_primary_diagnosis","type":"BETWEEN","system":"","value":{"min":0,"max":70}}]}]}]}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + + const C61_OR_MALE: &str = r#"{"ast": {"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis","type":"EQUALS","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","value":"C61"}]},{"operand":"OR","children":[{"key":"gender","type":"EQUALS","system":"","value":"male"}]}]}]}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + + const ALL_GBN: &str = r#"{"ast":{"children":[{"key":"gender","system":"","type":"IN","value":["male","other"]},{"children":[{"key":"diagnosis","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","type":"EQUALS","value":"C25"},{"key":"diagnosis","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","type":"EQUALS","value":"C56"}],"de":"Diagnose ICD-10","en":"Diagnosis ICD-10","key":"diagnosis","operand":"OR"},{"key":"diagnosis_age_donor","system":"","type":"BETWEEN","value":{"max":100,"min":10}},{"key":"date_of_diagnosis","system":"","type":"BETWEEN","value":{"max":"2023-10-29T23:00:00.000Z","min":"2023-09-30T22:00:00.000Z"}},{"key":"BMI","system":"","type":"BETWEEN","value":{"max":100,"min":10}},{"key":"Body weight","system":"","type":"BETWEEN","value":{"max":1100,"min":10}},{"key":"fasting_status","system":"","type":"IN","value":["Sober","Other fasting status"]},{"key":"72166-2","system":"","type":"IN","value":["Smoker","Never smoked"]},{"key":"donor_age","system":"","type":"BETWEEN","value":{"max":10000,"min":100}},{"key":"sample_kind","system":"","type":"IN","value":["blood-serum","blood-plasma","buffy-coat"]},{"key":"sampling_date","system":"","type":"BETWEEN","value":{"max":"2023-10-29T23:00:00.000Z","min":"2023-10-03T22:00:00.000Z"}},{"key":"storage_temperature","system":"","type":"IN","value":["temperature-18to-35","temperature-60to-85"]}],"de":"haupt","en":"main","key":"main","operand":"AND"},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + + const SOME_GBN: &str = r#"{"ast":{"children":[{"key":"gender","system":"","type":"IN","value":["other","male"]},{"key":"diagnosis","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","type":"EQUALS","value":"C24"},{"key":"diagnosis_age_donor","system":"","type":"BETWEEN","value":{"max":11,"min":1}},{"key":"date_of_diagnosis","system":"","type":"BETWEEN","value":{"max":"2023-10-30T23:00:00.000Z","min":"2023-10-29T23:00:00.000Z"}},{"key":"bmi","system":"","type":"BETWEEN","value":{"max":111,"min":1}},{"key":"body_weight","system":"","type":"BETWEEN","value":{"max":1111,"min":110}},{"key":"fasting_status","system":"","type":"IN","value":["Sober","Not sober"]},{"key":"smoking_status","system":"","type":"IN","value":["Smoker","Never smoked"]},{"key":"donor_age","system":"","type":"BETWEEN","value":{"max":123,"min":1}},{"key":"sample_kind","system":"","type":"IN","value":["blood-serum","tissue-other"]},{"key":"sampling_date","system":"","type":"BETWEEN","value":{"max":"2023-10-30T23:00:00.000Z","min":"2023-10-29T23:00:00.000Z"}},{"key":"storage_temperature","system":"","type":"IN","value":["temperature2to10","temperatureGN"]}],"de":"haupt","en":"main","key":"main","operand":"AND"},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + + const LENS2: &str = r#"{"ast":{"children":[{"children":[{"children":[{"key":"gender","system":"","type":"EQUALS","value":"male"},{"key":"gender","system":"","type":"EQUALS","value":"female"}],"operand":"OR"},{"children":[{"key":"diagnosis","system":"","type":"EQUALS","value":"C41"},{"key":"diagnosis","system":"","type":"EQUALS","value":"C50"}],"operand":"OR"},{"children":[{"key":"sample_kind","system":"","type":"EQUALS","value":"tissue-frozen"},{"key":"sample_kind","system":"","type":"EQUALS","value":"blood-serum"}],"operand":"OR"}],"operand":"AND"},{"children":[{"children":[{"key":"gender","system":"","type":"EQUALS","value":"male"}],"operand":"OR"},{"children":[{"key":"diagnosis","system":"","type":"EQUALS","value":"C41"},{"key":"diagnosis","system":"","type":"EQUALS","value":"C50"}],"operand":"OR"},{"children":[{"key":"sample_kind","system":"","type":"EQUALS","value":"liquid-other"},{"key":"sample_kind","system":"","type":"EQUALS","value":"rna"},{"key":"sample_kind","system":"","type":"EQUALS","value":"urine"}],"operand":"OR"},{"children":[{"key":"storage_temperature","system":"","type":"EQUALS","value":"temperatureRoom"},{"key":"storage_temperature","system":"","type":"EQUALS","value":"four_degrees"}],"operand":"OR"}],"operand":"AND"}],"operand":"OR"},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + + #[test] + fn test_just_print() { + // println!( + // "{:?}", + // bbmri(serde_json::from_str(AST).expect("Failed to deserialize JSON")) + // ); + + // println!( + // "{:?}", + // bbmri(serde_json::from_str(MALE_OR_FEMALE).expect("Failed to deserialize JSON")) + // ); + + // println!( + // "{:?}", + // bbmri(serde_json::from_str(ALL_GLIOMS).expect("Failed to deserialize JSON")) + // ); + + // println!( + // "{:?}", + // bbmri(serde_json::from_str(AGE_AT_DIAGNOSIS_30_TO_70).expect("Failed to deserialize JSON")) + // ); + + // println!( + // "{:?}", + // bbmri(serde_json::from_str(AGE_AT_DIAGNOSIS_LOWER_THAN_70).expect("Failed to deserialize JSON")) + // ); + + // println!( + // "{:?}", + // bbmri(serde_json::from_str(C61_OR_MALE).expect("Failed to deserialize JSON")) + // ); + + println!( + "{:?}", + bbmri(serde_json::from_str(ALL_GBN).expect("Failed to deserialize JSON")) + ); + + println!(); + + println!( + "{:?}", + bbmri(serde_json::from_str(SOME_GBN).expect("Failed to deserialize JSON")) + ); + + println!(); + + println!( + "{:?}", + bbmri(serde_json::from_str(LENS2).expect("Failed to deserialize JSON")) + ); + + //println!("{:?}", CRITERION_MAP.get("gender")); + + //println!("{:?}",CRITERION_MAP); + } +} diff --git a/src/main.rs b/src/main.rs index 545bab0..f47ea35 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,6 +6,9 @@ mod logger; mod omop; mod util; mod ast; +mod errors; +mod graceful_shutdown; +mod cql; use std::collections::HashMap; use std::process::ExitCode; @@ -25,8 +28,7 @@ use crate::{config::CONFIG, errors::FocusError}; use laplace_rs::ObfCache; -mod errors; -mod graceful_shutdown; + // result cache type SearchQuery = String; diff --git a/src/util.rs b/src/util.rs index c41e343..ff49a9a 100644 --- a/src/util.rs +++ b/src/util.rs @@ -90,10 +90,7 @@ pub(crate) fn read_lines(filename: String) -> Result>, pub(crate) fn replace_cql(decoded_library: impl Into) -> String { let replace_map: HashMap<&str, &str> = [ - ("BBMRI_STRAT_GENDER_STRATIFIER", - "define Gender: - if (Patient.gender is null) then 'unknown' else Patient.gender" - ), + ("BBMRI_STRAT_GENDER_STRATIFIER", "define Gender:\nif (Patient.gender is null) then 'unknown' else Patient.gender"), ("BBMRI_STRAT_SAMPLE_TYPE_STRATIFIER", "define function SampleType(specimen FHIR.Specimen):\n case FHIRHelpers.ToCode(specimen.type.coding.where(system = 'https://fhir.bbmri.de/CodeSystem/SampleMaterialType').first())\n when Code 'plasma-edta' from SampleMaterialType then 'blood-plasma' \n when Code 'plasma-citrat' from SampleMaterialType then 'blood-plasma' \n when Code 'plasma-heparin' from SampleMaterialType then 'blood-plasma' \n when Code 'plasma-cell-free' from SampleMaterialType then 'blood-plasma' \n when Code 'plasma-other' from SampleMaterialType then 'blood-plasma' \n when Code 'plasma' from SampleMaterialType then 'blood-plasma' \n when Code 'tissue-formalin' from SampleMaterialType then 'tissue-ffpe' \n when Code 'tumor-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' \n when Code 'normal-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' \n when Code 'other-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' \n when Code 'tumor-tissue-frozen' from SampleMaterialType then 'tissue-frozen' \n when Code 'normal-tissue-frozen' from SampleMaterialType then 'tissue-frozen' \n when Code 'other-tissue-frozen' from SampleMaterialType then 'tissue-frozen' \n when Code 'tissue-paxgene-or-else' from SampleMaterialType then 'tissue-other' \n when Code 'derivative' from SampleMaterialType then 'derivative-other' \n when Code 'liquid' from SampleMaterialType then 'liquid-other' \n when Code 'tissue' from SampleMaterialType then 'tissue-other' \n when Code 'serum' from SampleMaterialType then 'blood-serum' \n when Code 'cf-dna' from SampleMaterialType then 'dna' \n when Code 'g-dna' from SampleMaterialType then 'dna' \n when Code 'blood-plasma' from SampleMaterialType then 'blood-plasma' \n when Code 'tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' \n when Code 'tissue-frozen' from SampleMaterialType then 'tissue-frozen' \n when Code 'tissue-other' from SampleMaterialType then 'tissue-other' \n when Code 'derivative-other' from SampleMaterialType then 'derivative-other' \n when Code 'liquid-other' from SampleMaterialType then 'liquid-other' \n when Code 'blood-serum' from SampleMaterialType then 'blood-serum' \n when Code 'dna' from SampleMaterialType then 'dna' \n when Code 'buffy-coat' from SampleMaterialType then 'buffy-coat' \n when Code 'urine' from SampleMaterialType then 'urine' \n when Code 'ascites' from SampleMaterialType then 'ascites' \n when Code 'saliva' from SampleMaterialType then 'saliva' \n when Code 'csf-liquor' from SampleMaterialType then 'csf-liquor' \n when Code 'bone-marrow' from SampleMaterialType then 'bone-marrow' \n when Code 'peripheral-blood-cells-vital' from SampleMaterialType then 'peripheral-blood-cells-vital' \n when Code 'stool-faeces' from SampleMaterialType then 'stool-faeces' \n when Code 'rna' from SampleMaterialType then 'rna' \n when Code 'whole-blood' from SampleMaterialType then 'whole-blood' \n when Code 'swab' from SampleMaterialType then 'swab' \n when Code 'dried-whole-blood' from SampleMaterialType then 'dried-whole-blood' \n when null then 'Unknown'\n else 'Unknown'\n end"), ("BBMRI_STRAT_CUSTODIAN_STRATIFIER", "define Custodian:\n First(from Specimen.extension E\n where E.url = 'https://fhir.bbmri.de/StructureDefinition/Custodian'\n return (E.value as Reference).identifier.value)"), ("BBMRI_STRAT_DIAGNOSIS_STRATIFIER", "define Diagnosis:\n if InInitialPopulation then [Condition] else {} as List \n define function DiagnosisCode(condition FHIR.Condition, specimen FHIR.Specimen):\n Coalesce(condition.code.coding.where(system = 'http://hl7.org/fhir/sid/icd-10').code.first(), condition.code.coding.where(system = 'http://fhir.de/CodeSystem/dimdi/icd-10-gm').code.first(), specimen.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code.first(), condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first())\n"), From c1f2acd3d004983231a6b0a1af3e369d82d5bae3 Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Fri, 10 Nov 2023 11:21:30 +0100 Subject: [PATCH 02/42] codesystems, WIP --- src/cql.rs | 73 ++++++++++++++++++++++++++++------------------------- src/util.rs | 2 +- 2 files changed, 39 insertions(+), 36 deletions(-) diff --git a/src/cql.rs b/src/cql.rs index c5272c6..ccc2364 100644 --- a/src/cql.rs +++ b/src/cql.rs @@ -2,8 +2,9 @@ use crate::ast; use once_cell::sync::Lazy; use std::collections::HashMap; +use std::collections::HashSet; -static ALIAS: Lazy> = Lazy::new(|| { +static ALIAS_BBMRI: Lazy> = Lazy::new(|| { let map = [ ("icd10", "http://hl7.org/fhir/sid/icd-10"), ("icd10gm", "http://fhir.de/CodeSystem/dimdi/icd-10-gm"), @@ -35,7 +36,9 @@ pub fn bbmri(ast: ast::Ast) -> String { let mut query: String = "(".to_string(); - let mut filter: String = "".to_string(); + let mut filter: String = "(".to_string(); + + let mut code_systems: HashSet = HashSet::new();; let mut lists: String = "".to_string(); @@ -51,18 +54,24 @@ pub fn bbmri(ast: ast::Ast) -> String { for grandchild in ast.ast.children { - process(grandchild, &mut query, &mut filter, &mut lists); + process(grandchild, &mut query, &mut filter, &mut code_systems); } query += ")"; + + + for code_system in code_systems { + lists = lists + format!("codesystem {}: '{}'", code_system.as_str(), ALIAS_BBMRI.get(code_system.as_str()).unwrap_or(&(""))).as_str(); + } + query + filter.as_str() + lists.as_str() } -pub fn process(child: ast::Child, query: &mut String, filter: &mut String, lists: &mut String ) { +pub fn process(child: ast::Child, query: &mut String, filter: &mut String, code_systems: &mut HashSet ) { let mut query_cond: String = "(".to_string(); let mut filter_cond: String = "(".to_string(); @@ -71,36 +80,29 @@ pub fn process(child: ast::Child, query: &mut String, filter: &mut String, lists ast::Child::Condition(condition) => { - query_cond += condition.key.as_str(); - filter_cond += condition.key.as_str(); + let condition_key_trans = condition.key.as_str(); + query_cond += condition_key_trans; - match condition.type_ { - ast::ConditionType::Between => { - query_cond += " between "; - }, - ast::ConditionType::In => { - query_cond += " in "; - }, - ast::ConditionType::Equals => { - query_cond += " equals "; - }, - ast::ConditionType::NotEquals => { - query_cond += " not_equals "; - }, - ast::ConditionType::Contains => { - query_cond += " contains "; - }, - ast::ConditionType::GreaterThan => { - query_cond += " greater than "; - }, - ast::ConditionType::LowerThan => { - query_cond += " lower than "; - } + match condition_key_trans { - } + _ => {} + } - query_cond += " "; + filter_cond += condition.key.as_str(); + + + let condition_type_trans = match condition.type_ { + ast::ConditionType::Between => "between", + ast::ConditionType::In => "in", + ast::ConditionType::Equals => "equals", + ast::ConditionType::NotEquals => "not_equals", + ast::ConditionType::Contains => "contains", + ast::ConditionType::GreaterThan => "greater than", + ast::ConditionType::LowerThan => "lower than" + }; + + query_cond = query_cond + " " + condition_type_trans + " "; match condition.value { ast::ConditionValue::Boolean(value) => { @@ -108,12 +110,12 @@ pub fn process(child: ast::Child, query: &mut String, filter: &mut String, lists }, ast::ConditionValue::DateRange(date_range) => { query_cond += date_range.min.as_str(); - query_cond += ","; + query_cond += ", "; query_cond += date_range.max.as_str(); }, ast::ConditionValue::NumRange(num_range) => { query_cond += num_range.min.to_string().as_str(); - query_cond += ","; + query_cond += ", "; query_cond += num_range.max.to_string().as_str(); }, ast::ConditionValue::Number(value) => { @@ -125,9 +127,9 @@ pub fn process(child: ast::Child, query: &mut String, filter: &mut String, lists ast::ConditionValue::StringArray(string_array) => { for value in &string_array { query_cond += value; - query_cond += ","; + query_cond += ", "; } - query_cond += " greater than "; + } } @@ -149,7 +151,7 @@ pub fn process(child: ast::Child, query: &mut String, filter: &mut String, lists } for grandchild in operation.children { - process(grandchild, &mut query_cond, &mut filter_cond, lists); + process(grandchild, &mut query_cond, &mut filter_cond, code_systems); } @@ -158,6 +160,7 @@ pub fn process(child: ast::Child, query: &mut String, filter: &mut String, lists } query_cond += ")"; + filter_cond += ")"; *query += query_cond.as_str(); *filter += filter_cond.as_str(); diff --git a/src/util.rs b/src/util.rs index ff49a9a..48bbafe 100644 --- a/src/util.rs +++ b/src/util.rs @@ -387,7 +387,7 @@ mod test { #[test] fn test_replace_cql() { let decoded_library = "BBMRI_STRAT_GENDER_STRATIFIER"; - let expected_result = "define Gender:\n if (Patient.gender is null) then 'unknown' else Patient.gender\n"; + let expected_result = "define Gender:\nif (Patient.gender is null) then 'unknown' else Patient.gender\n"; assert_eq!(replace_cql(decoded_library), expected_result); let decoded_library = "BBMRI_STRAT_CUSTODIAN_STRATIFIER"; From 9e322f31a55bb0eb0a579ae789369ce6abd7bce9 Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Tue, 14 Nov 2023 15:31:37 +0100 Subject: [PATCH 03/42] order --- src/cql.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cql.rs b/src/cql.rs index ccc2364..6cc318c 100644 --- a/src/cql.rs +++ b/src/cql.rs @@ -38,7 +38,7 @@ pub fn bbmri(ast: ast::Ast) -> String { let mut filter: String = "(".to_string(); - let mut code_systems: HashSet = HashSet::new();; + let mut code_systems: HashSet = HashSet::new(); let mut lists: String = "".to_string(); @@ -67,7 +67,7 @@ pub fn bbmri(ast: ast::Ast) -> String { } - query + filter.as_str() + lists.as_str() + lists + query.as_str() + filter.as_str() } From ecf667249c173919fb6fad548255117929c1103d Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Tue, 21 Nov 2023 15:39:30 +0100 Subject: [PATCH 04/42] style etc. --- src/beam.rs | 2 +- src/cql.rs | 160 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 2 +- 3 files changed, 162 insertions(+), 2 deletions(-) diff --git a/src/beam.rs b/src/beam.rs index de17cec..900ac1d 100644 --- a/src/beam.rs +++ b/src/beam.rs @@ -2,7 +2,7 @@ use std::time::Duration; use beam_lib::{TaskResult, BeamClient, BlockingOptions, MsgId, TaskRequest, RawString}; use once_cell::sync::Lazy; -use serde::{de::DeserializeOwned, Serialize}; +use serde::Serialize; use tracing::{debug, warn}; use crate::{config::CONFIG, errors::FocusError}; diff --git a/src/cql.rs b/src/cql.rs index 6cc318c..c9cb8db 100644 --- a/src/cql.rs +++ b/src/cql.rs @@ -4,6 +4,14 @@ use once_cell::sync::Lazy; use std::collections::HashMap; use std::collections::HashSet; +enum CriterionRole { + Query, + Filter, +} + +const OBSERVATION_BMI: &str = "39156-5"; +const OBSERVATION_BODY_WEIGHT: &str = "29463-7"; + static ALIAS_BBMRI: Lazy> = Lazy::new(|| { let map = [ ("icd10", "http://hl7.org/fhir/sid/icd-10"), @@ -31,6 +39,158 @@ static ALIAS_BBMRI: Lazy> = Lazy::new(|| { map }); +static CQL_TEMPLATE_BBMRI: Lazy> = Lazy::new(|| { + let map = [ + ("gender", "Patient.gender"), + ( + "conditionSampleDiagnosis", + "((exists[Condition: Code '{{C}}' from {{A1}}]) or (exists[Condition: Code '{{C}}' from {{A2}}])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains '{{C}}'))", + ), + ("conditionValue", "exists [Condition: Code '{{C}}' from {{A1}}]"), + ( + "conditionRangeDate", + "exists from [Condition] C\nwhere FHIRHelpers.ToDateTime(C.onset) between {{D1}} and {{D2}}", + ), + ( + "conditionRangeAge", + "exists from [Condition] C\nwhere AgeInYearsAt(FHIRHelpers.ToDateTime(C.onset)) between Ceiling({{D1}}) and Ceiling({{D2}})", + ), + ("age", "AgeInYears() between Ceiling({{D1}}) and Ceiling({{D2}})"), + ( + "observation", + "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere O.value.coding.code contains '{{C}}'", + ), + ( + "observationRange", + "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere O.value between {{D1}} and {{D2}}", + ), + ( + "observationBodyWeight", + "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere ((O.value as Quantity) < {{D1}} 'kg' and (O.value as Quantity) > {{D2}} 'kg')", + ), + ( + "observationBMI", + "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere ((O.value as Quantity) < {{D1}} 'kg/m2' and (O.value as Quantity) > {{D2}} 'kg/m2')", + ), + ("hasSpecimen", "exists [Specimen]"), + ("specimen", "exists [Specimen: Code '{{C}}' from {{A1}}]"), + ("retrieveSpecimenByType", "(S.type.coding.code contains '{{C}}')"), + ( + "retrieveSpecimenByTemperature", + "(S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding.code contains '{{C}}')", + ), + ( + "retrieveSpecimenBySamplingDate", + "(FHIRHelpers.ToDateTime(S.collection.collected) between {{D1}} and {{D2}})", + ), + ( + "retrieveSpecimenByFastingStatus", + "(S.collection.fastingStatus.coding.code contains '{{C}}')", + ), + ( + "samplingDate", + "exists from [Specimen] S\nwhere FHIRHelpers.ToDateTime(S.collection.collected) between {{D1}} and {{D2}}", + ), + ( + "fastingStatus", + "exists from [Specimen] S\nwhere S.collection.fastingStatus.coding.code contains '{{C}}'", + ), + ( + "storageTemperature", + "exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding contains Code '{{C}}' from {{A1}})", + ), + ] + .into(); + + map +}); + + +static CRITERION_MAP: Lazy>>>> = Lazy::new(|| { + let map = [ + ("gender", Some([("type", vec!["gender"])].into())), + ( + "diagnosis", + Some( + [ + ("type", vec!["conditionSampleDiagnosis"]), + ("alias", vec!["icd10", "icd10gm"]), + ] + .into(), + ), + ), + ( + "29463-7", + Some( + [ + ("type", vec!["observationBodyWeight"]), + ("alias", vec!["loinc"]), + ] + .into(), + ), + ), + ( + "39156-5", + Some([("type", vec!["observationBMI"]), ("alias", vec!["loinc"])].into()), + ), + ( + "72166-2", + Some([("type", vec!["observation"]), ("alias", vec!["loinc"])].into()), + ), + ("donor_age", Some([("type", vec!["age"])].into())), + ( + "date_of_diagnosis", + Some([("type", vec!["conditionRangeDate"])].into()), + ), + ( + "sample_kind", + Some( + [ + ("type", vec!["specimen"]), + ("alias", vec!["SampleMaterialType"]), + ] + .into(), + ), + ), + ( + "storage_temperature", + Some( + [ + ("type", vec!["storageTemperature"]), + ("alias", vec!["StorageTemperature"]), + ] + .into(), + ), + ), + ( + "pat_with_samples", + Some([("type", vec!["hasSpecimen"])].into()), + ), + ( + "diagnosis_age_donor", + Some([("type", vec!["conditionRangeAge"])].into()), + ), + ( + "fasting_status", + Some( + [ + ("type", vec!["fastingStatus"]), + ("alias", vec!["FastingStatus"]), + ] + .into(), + ), + ), + ( + "sampling_date", + Some([("type", vec!["samplingDate"])].into()), + ), + ] + .into(); + + map +}); + + pub fn bbmri(ast: ast::Ast) -> String { diff --git a/src/main.rs b/src/main.rs index 1bff9c9..8f9628e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -28,7 +28,7 @@ use base64::{engine::general_purpose, Engine as _}; use serde_json::from_slice; use serde::{Deserialize, Serialize}; -use tracing::{debug, error, info, warn}; +use tracing::{debug, error, warn}; From 38b2f0132a25184049659f9b45a84e474d7ab070 Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Fri, 1 Dec 2023 23:33:47 +0100 Subject: [PATCH 05/42] CQL generation part 2 --- Cargo.toml | 1 + .../{measure.json => measure_bbmri.json} | 0 resources/template_bbmri.cql | 83 +++ resources/{ => test}/library.json | 0 .../{ => test}/measure_report_bbmri.json | 0 resources/{ => test}/measure_report_dktk.json | 0 src/ast.rs | 2 +- src/cql.rs | 481 ++++++++++-------- src/errors.rs | 9 + src/util.rs | 4 +- 10 files changed, 352 insertions(+), 228 deletions(-) rename resources/{measure.json => measure_bbmri.json} (100%) create mode 100644 resources/template_bbmri.cql rename resources/{ => test}/library.json (100%) rename resources/{ => test}/measure_report_bbmri.json (100%) rename resources/{ => test}/measure_report_dktk.json (100%) diff --git a/Cargo.toml b/Cargo.toml index 656d97f..75dc099 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ reqwest = { version = "0.11", default_features = false, features = ["json", "def serde = { version = "1.0.152", features = ["serde_derive"] } serde_json = "1.0" thiserror = "1.0.38" +dateparser = "0.2.1" tokio = { version = "1.25.0", default_features = false, features = ["signal", "rt-multi-thread", "macros"] } beam-lib = { git = "https://github.com/samply/beam", branch = "develop", features = ["http-util"] } laplace_rs = {version = "0.2.0", git = "https://github.com/samply/laplace-rs.git", branch = "main" } diff --git a/resources/measure.json b/resources/measure_bbmri.json similarity index 100% rename from resources/measure.json rename to resources/measure_bbmri.json diff --git a/resources/template_bbmri.cql b/resources/template_bbmri.cql new file mode 100644 index 0000000..322aa36 --- /dev/null +++ b/resources/template_bbmri.cql @@ -0,0 +1,83 @@ +library Retrieve +using FHIR version '4.0.0' +include FHIRHelpers version '4.0.0' + +{{lists}} + +context Patient + +define AgeClass: +if (Patient.birthDate is null) then 'unknown' else ToString((AgeInYears() div 10) * 10) + +define Gender: +if (Patient.gender is null) then 'unknown' else Patient.gender + +define Custodian: + First(from Specimen.extension E + where E.url = 'https://fhir.bbmri.de/StructureDefinition/Custodian' + return (E.value as Reference).identifier.value) + +define function SampleType(specimen FHIR.Specimen): + case FHIRHelpers.ToCode(specimen.type.coding.where(system = 'https://fhir.bbmri.de/CodeSystem/SampleMaterialType').first()) + when Code 'plasma-edta' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-citrat' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-heparin' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-cell-free' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-other' from SampleMaterialType then 'blood-plasma' + when Code 'plasma' from SampleMaterialType then 'blood-plasma' + when Code 'tissue-formalin' from SampleMaterialType then 'tissue-ffpe' + when Code 'tumor-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'normal-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'other-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'tumor-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'normal-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'other-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'tissue-paxgene-or-else' from SampleMaterialType then 'tissue-other' + when Code 'derivative' from SampleMaterialType then 'derivative-other' + when Code 'liquid' from SampleMaterialType then 'liquid-other' + when Code 'tissue' from SampleMaterialType then 'tissue-other' + when Code 'serum' from SampleMaterialType then 'blood-serum' + when Code 'cf-dna' from SampleMaterialType then 'dna' + when Code 'g-dna' from SampleMaterialType then 'dna' + when Code 'blood-plasma' from SampleMaterialType then 'blood-plasma' + when Code 'tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'tissue-other' from SampleMaterialType then 'tissue-other' + when Code 'derivative-other' from SampleMaterialType then 'derivative-other' + when Code 'liquid-other' from SampleMaterialType then 'liquid-other' + when Code 'blood-serum' from SampleMaterialType then 'blood-serum' + when Code 'dna' from SampleMaterialType then 'dna' + when Code 'buffy-coat' from SampleMaterialType then 'buffy-coat' + when Code 'urine' from SampleMaterialType then 'urine' + when Code 'ascites' from SampleMaterialType then 'ascites' + when Code 'saliva' from SampleMaterialType then 'saliva' + when Code 'csf-liquor' from SampleMaterialType then 'csf-liquor' + when Code 'bone-marrow' from SampleMaterialType then 'bone-marrow' + when Code 'peripheral-blood-cells-vital' from SampleMaterialType then 'peripheral-blood-cells-vital' + when Code 'stool-faeces' from SampleMaterialType then 'stool-faeces' + when Code 'rna' from SampleMaterialType then 'rna' + when Code 'whole-blood' from SampleMaterialType then 'whole-blood' + when Code 'swab' from SampleMaterialType then 'swab' + when Code 'dried-whole-blood' from SampleMaterialType then 'dried-whole-blood' + when null then 'Unknown' + else 'Unknown' + end +define Specimen: + if InInitialPopulation then [Specimen] S {{filter_criteria}} else {} as List + +define Diagnosis: +if InInitialPopulation then [Condition] else {} as List + +define function DiagnosisCode(condition FHIR.Condition): +condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first() + +define function DiagnosisCode(condition FHIR.Condition, specimen FHIR.Specimen): +Coalesce( + condition.code.coding.where(system = 'http://hl7.org/fhir/sid/icd-10').code.first(), + condition.code.coding.where(system = 'http://fhir.de/CodeSystem/dimdi/icd-10-gm').code.first(), + specimen.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code.first(), + condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first() + ) + +define InInitialPopulation: +{{retrieval_criteria}} \ No newline at end of file diff --git a/resources/library.json b/resources/test/library.json similarity index 100% rename from resources/library.json rename to resources/test/library.json diff --git a/resources/measure_report_bbmri.json b/resources/test/measure_report_bbmri.json similarity index 100% rename from resources/measure_report_bbmri.json rename to resources/test/measure_report_bbmri.json diff --git a/resources/measure_report_dktk.json b/resources/test/measure_report_dktk.json similarity index 100% rename from resources/measure_report_dktk.json rename to resources/test/measure_report_dktk.json diff --git a/src/ast.rs b/src/ast.rs index 3cf99d1..76b8dae 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -47,7 +47,7 @@ pub struct NumRange { #[derive(Serialize, Deserialize, Debug, Clone)] pub struct DateRange { - pub min: String, //we don't parse dates yet + pub min: String, // we don't parse dates yet pub max: String, } diff --git a/src/cql.rs b/src/cql.rs index c9cb8db..bc5c390 100644 --- a/src/cql.rs +++ b/src/cql.rs @@ -1,18 +1,27 @@ use crate::ast; +use crate::errors::FocusError; +use chrono::offset::Utc; +use chrono::DateTime; +use dateparser::DateTimeUtc; use once_cell::sync::Lazy; use std::collections::HashMap; use std::collections::HashSet; +#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Hash)] enum CriterionRole { Query, Filter, } -const OBSERVATION_BMI: &str = "39156-5"; -const OBSERVATION_BODY_WEIGHT: &str = "29463-7"; +#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Hash)] +pub enum Project { + BBMRI, + DKTK, +} -static ALIAS_BBMRI: Lazy> = Lazy::new(|| { +static CODE_LISTS: Lazy> = Lazy::new(|| { + //code lists with their names let map = [ ("icd10", "http://hl7.org/fhir/sid/icd-10"), ("icd10gm", "http://fhir.de/CodeSystem/dimdi/icd-10-gm"), @@ -39,150 +48,96 @@ static ALIAS_BBMRI: Lazy> = Lazy::new(|| { map }); -static CQL_TEMPLATE_BBMRI: Lazy> = Lazy::new(|| { +static OBSERVATION_LOINC_CODE: Lazy> = Lazy::new(|| { let map = [ - ("gender", "Patient.gender"), - ( - "conditionSampleDiagnosis", - "((exists[Condition: Code '{{C}}' from {{A1}}]) or (exists[Condition: Code '{{C}}' from {{A2}}])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains '{{C}}'))", - ), - ("conditionValue", "exists [Condition: Code '{{C}}' from {{A1}}]"), - ( - "conditionRangeDate", - "exists from [Condition] C\nwhere FHIRHelpers.ToDateTime(C.onset) between {{D1}} and {{D2}}", - ), - ( - "conditionRangeAge", - "exists from [Condition] C\nwhere AgeInYearsAt(FHIRHelpers.ToDateTime(C.onset)) between Ceiling({{D1}}) and Ceiling({{D2}})", - ), - ("age", "AgeInYears() between Ceiling({{D1}}) and Ceiling({{D2}})"), - ( - "observation", - "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere O.value.coding.code contains '{{C}}'", - ), - ( - "observationRange", - "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere O.value between {{D1}} and {{D2}}", - ), - ( - "observationBodyWeight", - "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere ((O.value as Quantity) < {{D1}} 'kg' and (O.value as Quantity) > {{D2}} 'kg')", - ), - ( - "observationBMI", - "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere ((O.value as Quantity) < {{D1}} 'kg/m2' and (O.value as Quantity) > {{D2}} 'kg/m2')", - ), - ("hasSpecimen", "exists [Specimen]"), - ("specimen", "exists [Specimen: Code '{{C}}' from {{A1}}]"), - ("retrieveSpecimenByType", "(S.type.coding.code contains '{{C}}')"), - ( - "retrieveSpecimenByTemperature", - "(S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding.code contains '{{C}}')", - ), - ( - "retrieveSpecimenBySamplingDate", - "(FHIRHelpers.ToDateTime(S.collection.collected) between {{D1}} and {{D2}})", - ), - ( - "retrieveSpecimenByFastingStatus", - "(S.collection.fastingStatus.coding.code contains '{{C}}')", - ), - ( - "samplingDate", - "exists from [Specimen] S\nwhere FHIRHelpers.ToDateTime(S.collection.collected) between {{D1}} and {{D2}}", - ), - ( - "fastingStatus", - "exists from [Specimen] S\nwhere S.collection.fastingStatus.coding.code contains '{{C}}'", - ), + ("body_weight", "29463-7"), + ("bmi", "39156-5"), + ("smoking_status", "72166-2"), + ] + .into(); + + map +}); + +static CRITERION_CODE_LISTS: Lazy>> = Lazy::new(|| { + // code lists needed depending on the criteria selected + let map = [ + (("diagnosis", Project::BBMRI), vec!["icd10", "icd10gm"]), + (("body_weight", Project::BBMRI), vec!["loinc"]), + (("bmi", Project::BBMRI), vec!["loinc"]), + (("smoking_status", Project::BBMRI), vec!["loinc"]), + (("sample_kind", Project::BBMRI), vec!["SampleMaterialType"]), ( - "storageTemperature", - "exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding contains Code '{{C}}' from {{A1}})", + ("storage_temperature", Project::BBMRI), + vec!["StorageTemperature"], ), + (("fasting_status", Project::BBMRI), vec!["FastingStatus"]), ] .into(); map }); - -static CRITERION_MAP: Lazy>>>> = Lazy::new(|| { +static CQL_SNIPPETS_BBMRI: Lazy> = Lazy::new(|| { + // CQL snippets depending on the criteria let map = [ - ("gender", Some([("type", vec!["gender"])].into())), + (("gender", CriterionRole::Query, Project::BBMRI), "Patient.gender = '{{D1}}'"), + ( + ("conditionSampleDiagnosis", CriterionRole::Query, Project::BBMRI), + " ((exists[Condition: Code '{{C}}' from {{A1}}]) or (exists[Condition: Code '{{C}}' from {{A2}}])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains '{{C}}')) ", + ), + (("diagnosis", CriterionRole::Query, Project::BBMRI), " exists [Condition: Code '{{C}}' from {{A1}}] "), ( - "diagnosis", - Some( - [ - ("type", vec!["conditionSampleDiagnosis"]), - ("alias", vec!["icd10", "icd10gm"]), - ] - .into(), - ), + ("date_of_diagnosis", CriterionRole::Query, Project::BBMRI), + " exists from [Condition] C\nwhere FHIRHelpers.ToDateTime(C.onset) between {{D1}} and {{D2}} ", ), ( - "29463-7", - Some( - [ - ("type", vec!["observationBodyWeight"]), - ("alias", vec!["loinc"]), - ] - .into(), - ), + ("diagnosis_age_donor", CriterionRole::Query, Project::BBMRI), + " exists from [Condition] C\nwhere AgeInYearsAt(FHIRHelpers.ToDateTime(C.onset)) between Ceiling({{D1}}) and Ceiling({{D2}}) ", ), + (("donor_age", CriterionRole::Query, Project::BBMRI), " AgeInYears() between Ceiling({{D1}}) and Ceiling({{D2}}) "), ( - "39156-5", - Some([("type", vec!["observationBMI"]), ("alias", vec!["loinc"])].into()), + ("observationRange", CriterionRole::Query, Project::BBMRI), + " exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere O.value between {{D1}} and {{D2}} ", ), ( - "72166-2", - Some([("type", vec!["observation"]), ("alias", vec!["loinc"])].into()), + ("body_weight", CriterionRole::Query, Project::BBMRI), + " exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere ((O.value as Quantity) < {{D1}} 'kg' and (O.value as Quantity) > {{D2}} 'kg') ", ), - ("donor_age", Some([("type", vec!["age"])].into())), ( - "date_of_diagnosis", - Some([("type", vec!["conditionRangeDate"])].into()), + ("bmi", CriterionRole::Query, Project::BBMRI), + " exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere ((O.value as Quantity) < {{D1}} 'kg/m2' and (O.value as Quantity) > {{D2}} 'kg/m2') ", ), + (("pat_with_samples", CriterionRole::Query, Project::BBMRI), " exists [Specimen] "), + (("sample_kind", CriterionRole::Query, Project::BBMRI), " exists [Specimen: Code '{{C}}' from {{A1}}] "), + (("retrieveSpecimenByType", CriterionRole::Query, Project::BBMRI), " (S.type.coding.code contains '{{C}}') "), ( - "sample_kind", - Some( - [ - ("type", vec!["specimen"]), - ("alias", vec!["SampleMaterialType"]), - ] - .into(), - ), + ("retrieveSpecimenByTemperature", CriterionRole::Query, Project::BBMRI), + " (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding.code contains '{{C}}') ", ), ( - "storage_temperature", - Some( - [ - ("type", vec!["storageTemperature"]), - ("alias", vec!["StorageTemperature"]), - ] - .into(), - ), + ("retrieveSpecimenBySamplingDate", CriterionRole::Query, Project::BBMRI), + " (FHIRHelpers.ToDateTime(S.collection.collected) between {{D1}} and {{D2}}) ", ), ( - "pat_with_samples", - Some([("type", vec!["hasSpecimen"])].into()), + ("retrieveSpecimenByFastingStatus", CriterionRole::Query, Project::BBMRI), + " (S.collection.fastingStatus.coding.code contains '{{C}}') ", ), ( - "diagnosis_age_donor", - Some([("type", vec!["conditionRangeAge"])].into()), + ("sampling_date", CriterionRole::Query, Project::BBMRI), + " exists from [Specimen] S\nwhere FHIRHelpers.ToDateTime(S.collection.collected) between {{D1}} and {{D2}} ", ), ( - "fasting_status", - Some( - [ - ("type", vec!["fastingStatus"]), - ("alias", vec!["FastingStatus"]), - ] - .into(), - ), + ("fasting_status", CriterionRole::Query, Project::BBMRI), + " exists from [Specimen] S\nwhere S.collection.fastingStatus.coding.code contains '{{C}}' ", ), ( - "sampling_date", - Some([("type", vec!["samplingDate"])].into()), + ("storage_temperature", CriterionRole::Query, Project::BBMRI), + " exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding contains Code '{{C}}' from {{A1}}) ", + ), + ( + ("smoking_status", CriterionRole::Query, Project::BBMRI), + " exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere O.value.coding.code contains '{{C}}' ", ), ] .into(); @@ -190,148 +145,228 @@ static CRITERION_MAP: Lazy>>>> = La map }); +pub fn bbmri(ast: ast::Ast) -> Result { + let mut retrieval_criteria: String = "(".to_string(); // main selection criteria (Patient) + let mut filter_criteria: String = "(".to_string(); // criteria for filtering specimens -pub fn bbmri(ast: ast::Ast) -> String { - - let mut query: String = "(".to_string(); + let mut code_systems: HashSet<&str> = HashSet::new(); // code lists needed depending on the criteria + code_systems.insert("icd10"); //for diagnosis stratifier + code_systems.insert("SampleMaterialType"); //for sample type stratifier - let mut filter: String = "(".to_string(); + let mut lists: String = "".to_string(); // needed code lists, defined - let mut code_systems: HashSet = HashSet::new(); + let mut cql: String = include_str!("../resources/template_bbmri.cql").to_string(); - let mut lists: String = "".to_string(); + let operator_str = match ast.ast.operand { + ast::Operand::And => " and ", + ast::Operand::Or => " or ", + }; - match ast.ast.operand { - ast::Operand::And => { - query += " and "; + for (index, grandchild) in ast.ast.children.iter().enumerate() { + process( + grandchild.clone(), + &mut retrieval_criteria, + &mut filter_criteria, + &mut code_systems, + Project::BBMRI, + )?; - }, - ast::Operand::Or => { - query += " or "; - }, + // Only concatenate operator if it's not the last element + if index < ast.ast.children.len() - 1 { + retrieval_criteria += operator_str; + } } - for grandchild in ast.ast.children { - - process(grandchild, &mut query, &mut filter, &mut code_systems); - - } - - query += ")"; + retrieval_criteria += ")"; - - for code_system in code_systems { - lists = lists + format!("codesystem {}: '{}'", code_system.as_str(), ALIAS_BBMRI.get(code_system.as_str()).unwrap_or(&(""))).as_str(); - } + lists = lists + + format!( + "codesystem {}: '{}' \n", + code_system, + CODE_LISTS.get(code_system).unwrap_or(&("")) + ) + .as_str(); + } + cql = cql + .replace("{[lists}}", lists.as_str()) + .replace("{{filter_criteria}}", filter_criteria.as_str()) + .replace("{{retrieval_criteria}}", retrieval_criteria.as_str()); - lists + query.as_str() + filter.as_str() - + Ok(retrieval_criteria) } -pub fn process(child: ast::Child, query: &mut String, filter: &mut String, code_systems: &mut HashSet ) { - - let mut query_cond: String = "(".to_string(); +pub fn process( + child: ast::Child, + retrieval_criteria: &mut String, + filter_criteria: &mut String, + code_systems: &mut HashSet<&str>, + project: Project, +) -> Result<(), FocusError> { + let mut retrieval_cond: String = "(".to_string(); let mut filter_cond: String = "(".to_string(); match child { - ast::Child::Condition(condition) => { - let condition_key_trans = condition.key.as_str(); - query_cond += condition_key_trans; - - match condition_key_trans { + let condition_snippet = CQL_SNIPPETS_BBMRI.get(&( + condition_key_trans, + CriterionRole::Query, + project.clone(), + )); + + if let Some(snippet) = condition_snippet { + let mut condition_string = (*snippet).to_string(); + + let code_lists_option = CRITERION_CODE_LISTS.get(&(condition_key_trans, project)); + if let Some(code_lists_vec) = code_lists_option { + for (index, code_list) in code_lists_vec.iter().enumerate() { + code_systems.insert(code_list); + let placeholder = + "{{A".to_string() + (index + 1).to_string().as_str() + "}}"; //to keep compatibility with snippets in typescript + condition_string = + condition_string.replace(placeholder.as_str(), code_list); + } + } + + if condition_string.contains("{{K}}") { + //observation loinc code + let observation_code_option = OBSERVATION_LOINC_CODE.get(&condition_key_trans); + + if let Some(observation_code) = observation_code_option { + condition_string = condition_string.replace("{{K}}", observation_code); + } else { + return Err(FocusError::AstUnknownOption( + condition_key_trans.to_string(), + )); + } + } - _ => {} + match condition.type_ { + ast::ConditionType::Between => { + //it has both min and max values stated + match condition.value { + ast::ConditionValue::DateRange(date_range) => { + let datetime_str_min = date_range.min.as_str(); + let datetime_result_min: Result, _> = + datetime_str_min.parse(); + + if let Ok(datetime_min) = datetime_result_min { + let date_str_min = + format!("@{}", datetime_min.format("%Y-%m-%d")); + + condition_string = + condition_string.replace("{{D1}}", date_str_min.as_str()); + } else { + return Err(FocusError::AstInvalidDateFormat(date_range.min)); + } + + let datetime_str_max = date_range.max.as_str(); + let datetime_result_max: Result, _> = + datetime_str_max.parse(); + if let Ok(datetime_max) = datetime_result_max { + let date_str_max = + format!("@{}", datetime_max.format("%Y-%m-%d")); + + condition_string = + condition_string.replace("{{D2}}", date_str_max.as_str()); + } else { + return Err(FocusError::AstInvalidDateFormat(date_range.max)); + } + }, + ast::ConditionValue::NumRange(num_range) => { + condition_string = condition_string + .replace("{{D1}}", num_range.min.to_string().as_str()); + condition_string = condition_string + .replace("{{D2}}", num_range.max.to_string().as_str()); + }, + _ => { + return Err(FocusError::AstOperatorValueMismatch()); + } + } + + } // deal with no lower or no upper value + ast::ConditionType::In => { + " in "; + } // this becomes or of all - deal with clones + ast::ConditionType::Equals => { + match condition.value { + ast::ConditionValue::String(string) => { + condition_string = condition_string + .replace("{{C}}", string.as_str()); + + }, + _ => { + return Err(FocusError::AstOperatorValueMismatch()); + } + + } + } + ast::ConditionType::NotEquals => { //won't get it from Lens + + } + ast::ConditionType::Contains => { + "contains "; + } + ast::ConditionType::GreaterThan => { + " greater than "; + } // guess Lens won't send me this, convert between to it + ast::ConditionType::LowerThan => { + " lower than "; + } // guess Lens won't send me this, convert between to it + }; + + retrieval_cond += condition_string.as_str(); + } else { + return Err(FocusError::AstUnknownCriterion( + condition_key_trans.to_string(), + )); } filter_cond += condition.key.as_str(); + retrieval_cond += " "; + } - let condition_type_trans = match condition.type_ { - ast::ConditionType::Between => "between", - ast::ConditionType::In => "in", - ast::ConditionType::Equals => "equals", - ast::ConditionType::NotEquals => "not_equals", - ast::ConditionType::Contains => "contains", - ast::ConditionType::GreaterThan => "greater than", - ast::ConditionType::LowerThan => "lower than" + ast::Child::Operation(operation) => { + let operator_str = match operation.operand { + ast::Operand::And => " and ", + ast::Operand::Or => " or ", }; - query_cond = query_cond + " " + condition_type_trans + " "; - - match condition.value { - ast::ConditionValue::Boolean(value) => { - query_cond += value.to_string().as_str(); - }, - ast::ConditionValue::DateRange(date_range) => { - query_cond += date_range.min.as_str(); - query_cond += ", "; - query_cond += date_range.max.as_str(); - }, - ast::ConditionValue::NumRange(num_range) => { - query_cond += num_range.min.to_string().as_str(); - query_cond += ", "; - query_cond += num_range.max.to_string().as_str(); - }, - ast::ConditionValue::Number(value) => { - query_cond += value.to_string().as_str(); - }, - ast::ConditionValue::String(value) => { - query_cond += value.as_str(); - }, - ast::ConditionValue::StringArray(string_array) => { - for value in &string_array { - query_cond += value; - query_cond += ", "; - } - + for (index, grandchild) in operation.children.iter().enumerate() { + process( + grandchild.clone(), + &mut retrieval_cond, + &mut filter_cond, + code_systems, + project.clone(), + )?; + + // Only concatenate operator if it's not the last element + if index < operation.children.len() - 1 { + retrieval_cond += operator_str; } - - } - - - query_cond += " "; - }, - - - ast::Child::Operation(operation) => { - match operation.operand { - ast::Operand::And => { - query_cond += " and "; - - }, - ast::Operand::Or => { - query_cond += " or "; - }, - } - - for grandchild in operation.children { - process(grandchild, &mut query_cond, &mut filter_cond, code_systems); - } - - }, - + } } - - query_cond += ")"; + + retrieval_cond += ")"; filter_cond += ")"; - *query += query_cond.as_str(); - *filter += filter_cond.as_str(); + *retrieval_criteria += retrieval_cond.as_str(); + *filter_criteria += filter_cond.as_str(); + Ok(()) } - - #[cfg(test)] mod test { use super::*; + const AST: &str = r#"{"ast":{"operand":"AND","children":[{"key":"age","type":"EQUALS","value":5.0}]},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; const MALE_OR_FEMALE: &str = r#"{"ast":{"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"gender","type":"EQUALS","system":"","value":"male"},{"key":"gender","type":"EQUALS","system":"","value":"female"}]}]}]},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; @@ -344,11 +379,11 @@ mod test { const C61_OR_MALE: &str = r#"{"ast": {"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis","type":"EQUALS","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","value":"C61"}]},{"operand":"OR","children":[{"key":"gender","type":"EQUALS","system":"","value":"male"}]}]}]}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; - const ALL_GBN: &str = r#"{"ast":{"children":[{"key":"gender","system":"","type":"IN","value":["male","other"]},{"children":[{"key":"diagnosis","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","type":"EQUALS","value":"C25"},{"key":"diagnosis","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","type":"EQUALS","value":"C56"}],"de":"Diagnose ICD-10","en":"Diagnosis ICD-10","key":"diagnosis","operand":"OR"},{"key":"diagnosis_age_donor","system":"","type":"BETWEEN","value":{"max":100,"min":10}},{"key":"date_of_diagnosis","system":"","type":"BETWEEN","value":{"max":"2023-10-29T23:00:00.000Z","min":"2023-09-30T22:00:00.000Z"}},{"key":"BMI","system":"","type":"BETWEEN","value":{"max":100,"min":10}},{"key":"Body weight","system":"","type":"BETWEEN","value":{"max":1100,"min":10}},{"key":"fasting_status","system":"","type":"IN","value":["Sober","Other fasting status"]},{"key":"72166-2","system":"","type":"IN","value":["Smoker","Never smoked"]},{"key":"donor_age","system":"","type":"BETWEEN","value":{"max":10000,"min":100}},{"key":"sample_kind","system":"","type":"IN","value":["blood-serum","blood-plasma","buffy-coat"]},{"key":"sampling_date","system":"","type":"BETWEEN","value":{"max":"2023-10-29T23:00:00.000Z","min":"2023-10-03T22:00:00.000Z"}},{"key":"storage_temperature","system":"","type":"IN","value":["temperature-18to-35","temperature-60to-85"]}],"de":"haupt","en":"main","key":"main","operand":"AND"},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + const ALL_GBN: &str = r#"{"ast":{"children":[{"key":"gender","system":"","type":"IN","value":["male","other"]},{"children":[{"key":"diagnosis","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","type":"EQUALS","value":"C25"},{"key":"diagnosis","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","type":"EQUALS","value":"C56"}],"de":"Diagnose ICD-10","en":"Diagnosis ICD-10","key":"diagnosis","operand":"OR"},{"key":"diagnosis_age_donor","system":"","type":"BETWEEN","value":{"max":100,"min":10}},{"key":"date_of_diagnosis","system":"","type":"BETWEEN","value":{"max":"2023-10-29T23:00:00.000Z","min":"2023-09-30T22:00:00.000Z"}},{"key":"bmi","system":"","type":"BETWEEN","value":{"max":100,"min":10}},{"key":"body_weight","system":"","type":"BETWEEN","value":{"max":1100,"min":10}},{"key":"fasting_status","system":"","type":"IN","value":["Sober","Other fasting status"]},{"key":"smoking_status","system":"","type":"IN","value":["Smoker","Never smoked"]},{"key":"donor_age","system":"","type":"BETWEEN","value":{"max":10000,"min":100}},{"key":"sample_kind","system":"","type":"IN","value":["blood-serum","blood-plasma","buffy-coat"]},{"key":"sampling_date","system":"","type":"BETWEEN","value":{"max":"2023-10-29T23:00:00.000Z","min":"2023-10-03T22:00:00.000Z"}},{"key":"storage_temperature","system":"","type":"IN","value":["temperature-18to-35","temperature-60to-85"]}],"de":"haupt","en":"main","key":"main","operand":"AND"},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; const SOME_GBN: &str = r#"{"ast":{"children":[{"key":"gender","system":"","type":"IN","value":["other","male"]},{"key":"diagnosis","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","type":"EQUALS","value":"C24"},{"key":"diagnosis_age_donor","system":"","type":"BETWEEN","value":{"max":11,"min":1}},{"key":"date_of_diagnosis","system":"","type":"BETWEEN","value":{"max":"2023-10-30T23:00:00.000Z","min":"2023-10-29T23:00:00.000Z"}},{"key":"bmi","system":"","type":"BETWEEN","value":{"max":111,"min":1}},{"key":"body_weight","system":"","type":"BETWEEN","value":{"max":1111,"min":110}},{"key":"fasting_status","system":"","type":"IN","value":["Sober","Not sober"]},{"key":"smoking_status","system":"","type":"IN","value":["Smoker","Never smoked"]},{"key":"donor_age","system":"","type":"BETWEEN","value":{"max":123,"min":1}},{"key":"sample_kind","system":"","type":"IN","value":["blood-serum","tissue-other"]},{"key":"sampling_date","system":"","type":"BETWEEN","value":{"max":"2023-10-30T23:00:00.000Z","min":"2023-10-29T23:00:00.000Z"}},{"key":"storage_temperature","system":"","type":"IN","value":["temperature2to10","temperatureGN"]}],"de":"haupt","en":"main","key":"main","operand":"AND"},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; - - const LENS2: &str = r#"{"ast":{"children":[{"children":[{"children":[{"key":"gender","system":"","type":"EQUALS","value":"male"},{"key":"gender","system":"","type":"EQUALS","value":"female"}],"operand":"OR"},{"children":[{"key":"diagnosis","system":"","type":"EQUALS","value":"C41"},{"key":"diagnosis","system":"","type":"EQUALS","value":"C50"}],"operand":"OR"},{"children":[{"key":"sample_kind","system":"","type":"EQUALS","value":"tissue-frozen"},{"key":"sample_kind","system":"","type":"EQUALS","value":"blood-serum"}],"operand":"OR"}],"operand":"AND"},{"children":[{"children":[{"key":"gender","system":"","type":"EQUALS","value":"male"}],"operand":"OR"},{"children":[{"key":"diagnosis","system":"","type":"EQUALS","value":"C41"},{"key":"diagnosis","system":"","type":"EQUALS","value":"C50"}],"operand":"OR"},{"children":[{"key":"sample_kind","system":"","type":"EQUALS","value":"liquid-other"},{"key":"sample_kind","system":"","type":"EQUALS","value":"rna"},{"key":"sample_kind","system":"","type":"EQUALS","value":"urine"}],"operand":"OR"},{"children":[{"key":"storage_temperature","system":"","type":"EQUALS","value":"temperatureRoom"},{"key":"storage_temperature","system":"","type":"EQUALS","value":"four_degrees"}],"operand":"OR"}],"operand":"AND"}],"operand":"OR"},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + + const LENS2: &str = r#"{"ast":{"children":[{"children":[{"children":[{"key":"gender","system":"","type":"EQUALS","value":"male"},{"key":"gender","system":"","type":"EQUALS","value":"female"}],"operand":"OR"},{"children":[{"key":"diagnosis","system":"","type":"EQUALS","value":"C41"},{"key":"diagnosis","system":"","type":"EQUALS","value":"C50"}],"operand":"OR"},{"children":[{"key":"sample_kind","system":"","type":"EQUALS","value":"tissue-frozen"},{"key":"sample_kind","system":"","type":"EQUALS","value":"blood-serum"}],"operand":"OR"}],"operand":"AND"},{"children":[{"children":[{"key":"gender","system":"","type":"EQUALS","value":"male"}],"operand":"OR"},{"children":[{"key":"diagnosis","system":"","type":"EQUALS","value":"C41"},{"key":"diagnosis","system":"","type":"EQUALS","value":"C50"}],"operand":"OR"},{"children":[{"key":"sample_kind","system":"","type":"EQUALS","value":"liquid-other"},{"key":"sample_kind","system":"","type":"EQUALS","value":"rna"},{"key":"sample_kind","system":"","type":"EQUALS","value":"urine"}],"operand":"OR"},{"children":[{"key":"storage_temperature","system":"","type":"EQUALS","value":"temperatureRoom"},{"key":"storage_temperature","system":"","type":"EQUALS","value":"four_degrees"}],"operand":"OR"}],"operand":"AND"}],"operand":"OR"},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; #[test] fn test_just_print() { @@ -400,9 +435,5 @@ mod test { "{:?}", bbmri(serde_json::from_str(LENS2).expect("Failed to deserialize JSON")) ); - - //println!("{:?}", CRITERION_MAP.get("gender")); - - //println!("{:?}",CRITERION_MAP); } } diff --git a/src/errors.rs b/src/errors.rs index 8d3f988..cdd1b1e 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,4 +1,5 @@ use thiserror::Error; +use crate::ast; #[derive(Error, Debug)] pub enum FocusError { @@ -38,4 +39,12 @@ pub enum FocusError { UnableToPostAst(reqwest::Error), #[error("AST Posting error in Reqwest: {0}")] AstPostingErrorReqwest(String), + #[error("Unknown criterion in AST: {0}")] + AstUnknownCriterion(String), + #[error("Unknown option in AST: {0}")] + AstUnknownOption(String), + #[error("Mismatch between operator and value type")] + AstOperatorValueMismatch(), + #[error("Invalid date format: {0}")] + AstInvalidDateFormat(String), } diff --git a/src/util.rs b/src/util.rs index 90655ea..cb1aa65 100644 --- a/src/util.rs +++ b/src/util.rs @@ -354,8 +354,8 @@ mod test { use super::*; use serde_json::json; - const EXAMPLE_MEASURE_REPORT_BBMRI: &str = include_str!("../resources/measure_report_bbmri.json"); - const EXAMPLE_MEASURE_REPORT_DKTK: &str = include_str!("../resources/measure_report_dktk.json"); + const EXAMPLE_MEASURE_REPORT_BBMRI: &str = include_str!("../resources/test/measure_report_bbmri.json"); + const EXAMPLE_MEASURE_REPORT_DKTK: &str = include_str!("../resources/test/measure_report_dktk.json"); const DELTA_PATIENT: f64 = 1.; const DELTA_SPECIMEN: f64 = 20.; From 5b81d5789d872fde0d456569aa62e1eba1accf43 Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Sat, 2 Dec 2023 01:05:22 +0100 Subject: [PATCH 06/42] CQL retrieval criteria --- Cargo.toml | 1 - src/cql.rs | 150 ++++++++++++++++++++++++++++++-------------------- src/errors.rs | 1 - src/omop.rs | 4 +- 4 files changed, 91 insertions(+), 65 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 75dc099..656d97f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,6 @@ reqwest = { version = "0.11", default_features = false, features = ["json", "def serde = { version = "1.0.152", features = ["serde_derive"] } serde_json = "1.0" thiserror = "1.0.38" -dateparser = "0.2.1" tokio = { version = "1.25.0", default_features = false, features = ["signal", "rt-multi-thread", "macros"] } beam-lib = { git = "https://github.com/samply/beam", branch = "develop", features = ["http-util"] } laplace_rs = {version = "0.2.0", git = "https://github.com/samply/laplace-rs.git", branch = "main" } diff --git a/src/cql.rs b/src/cql.rs index bc5c390..db8a469 100644 --- a/src/cql.rs +++ b/src/cql.rs @@ -3,7 +3,6 @@ use crate::errors::FocusError; use chrono::offset::Utc; use chrono::DateTime; -use dateparser::DateTimeUtc; use once_cell::sync::Lazy; use std::collections::HashMap; use std::collections::HashSet; @@ -16,13 +15,13 @@ enum CriterionRole { #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Hash)] pub enum Project { - BBMRI, - DKTK, + Bbmri, + Dktk, } static CODE_LISTS: Lazy> = Lazy::new(|| { //code lists with their names - let map = [ + [ ("icd10", "http://hl7.org/fhir/sid/icd-10"), ("icd10gm", "http://fhir.de/CodeSystem/dimdi/icd-10-gm"), ("loinc", "http://loinc.org"), @@ -43,106 +42,98 @@ static CODE_LISTS: Lazy> = Lazy::new(|| { "http://hl7.org/fhir/uv/ips/ValueSet/current-smoking-status-uv-ips", ), ] - .into(); - - map + .into() }); static OBSERVATION_LOINC_CODE: Lazy> = Lazy::new(|| { - let map = [ + [ ("body_weight", "29463-7"), ("bmi", "39156-5"), ("smoking_status", "72166-2"), ] - .into(); - - map + .into() }); static CRITERION_CODE_LISTS: Lazy>> = Lazy::new(|| { // code lists needed depending on the criteria selected - let map = [ - (("diagnosis", Project::BBMRI), vec!["icd10", "icd10gm"]), - (("body_weight", Project::BBMRI), vec!["loinc"]), - (("bmi", Project::BBMRI), vec!["loinc"]), - (("smoking_status", Project::BBMRI), vec!["loinc"]), - (("sample_kind", Project::BBMRI), vec!["SampleMaterialType"]), + [ + (("diagnosis", Project::Bbmri), vec!["icd10", "icd10gm"]), + (("body_weight", Project::Bbmri), vec!["loinc"]), + (("bmi", Project::Bbmri), vec!["loinc"]), + (("smoking_status", Project::Bbmri), vec!["loinc"]), + (("sample_kind", Project::Bbmri), vec!["SampleMaterialType"]), ( - ("storage_temperature", Project::BBMRI), + ("storage_temperature", Project::Bbmri), vec!["StorageTemperature"], ), - (("fasting_status", Project::BBMRI), vec!["FastingStatus"]), + (("fasting_status", Project::Bbmri), vec!["FastingStatus"]), ] - .into(); - - map + .into() }); static CQL_SNIPPETS_BBMRI: Lazy> = Lazy::new(|| { // CQL snippets depending on the criteria - let map = [ - (("gender", CriterionRole::Query, Project::BBMRI), "Patient.gender = '{{D1}}'"), + [ + (("gender", CriterionRole::Query, Project::Bbmri), "Patient.gender = '{{C}}'"), ( - ("conditionSampleDiagnosis", CriterionRole::Query, Project::BBMRI), + ("diagnosis", CriterionRole::Query, Project::Bbmri), " ((exists[Condition: Code '{{C}}' from {{A1}}]) or (exists[Condition: Code '{{C}}' from {{A2}}])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains '{{C}}')) ", ), - (("diagnosis", CriterionRole::Query, Project::BBMRI), " exists [Condition: Code '{{C}}' from {{A1}}] "), + (("diagnosis_old", CriterionRole::Query, Project::Bbmri), " exists [Condition: Code '{{C}}' from {{A1}}] "), ( - ("date_of_diagnosis", CriterionRole::Query, Project::BBMRI), + ("date_of_diagnosis", CriterionRole::Query, Project::Bbmri), " exists from [Condition] C\nwhere FHIRHelpers.ToDateTime(C.onset) between {{D1}} and {{D2}} ", ), ( - ("diagnosis_age_donor", CriterionRole::Query, Project::BBMRI), + ("diagnosis_age_donor", CriterionRole::Query, Project::Bbmri), " exists from [Condition] C\nwhere AgeInYearsAt(FHIRHelpers.ToDateTime(C.onset)) between Ceiling({{D1}}) and Ceiling({{D2}}) ", ), - (("donor_age", CriterionRole::Query, Project::BBMRI), " AgeInYears() between Ceiling({{D1}}) and Ceiling({{D2}}) "), + (("donor_age", CriterionRole::Query, Project::Bbmri), " AgeInYears() between Ceiling({{D1}}) and Ceiling({{D2}}) "), ( - ("observationRange", CriterionRole::Query, Project::BBMRI), + ("observationRange", CriterionRole::Query, Project::Bbmri), " exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere O.value between {{D1}} and {{D2}} ", ), ( - ("body_weight", CriterionRole::Query, Project::BBMRI), + ("body_weight", CriterionRole::Query, Project::Bbmri), " exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere ((O.value as Quantity) < {{D1}} 'kg' and (O.value as Quantity) > {{D2}} 'kg') ", ), ( - ("bmi", CriterionRole::Query, Project::BBMRI), + ("bmi", CriterionRole::Query, Project::Bbmri), " exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere ((O.value as Quantity) < {{D1}} 'kg/m2' and (O.value as Quantity) > {{D2}} 'kg/m2') ", ), - (("pat_with_samples", CriterionRole::Query, Project::BBMRI), " exists [Specimen] "), - (("sample_kind", CriterionRole::Query, Project::BBMRI), " exists [Specimen: Code '{{C}}' from {{A1}}] "), - (("retrieveSpecimenByType", CriterionRole::Query, Project::BBMRI), " (S.type.coding.code contains '{{C}}') "), + (("sample_kind", CriterionRole::Query, Project::Bbmri), " exists [Specimen: Code '{{C}}' from {{A1}}] "), + (("sample_kind", CriterionRole::Filter, Project::Bbmri), " (S.type.coding.code contains '{{C}}') "), + ( - ("retrieveSpecimenByTemperature", CriterionRole::Query, Project::BBMRI), + ("storage_temperature", CriterionRole::Filter, Project::Bbmri), " (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding.code contains '{{C}}') ", ), ( - ("retrieveSpecimenBySamplingDate", CriterionRole::Query, Project::BBMRI), + ("sampling_date", CriterionRole::Filter, Project::Bbmri), " (FHIRHelpers.ToDateTime(S.collection.collected) between {{D1}} and {{D2}}) ", ), ( - ("retrieveSpecimenByFastingStatus", CriterionRole::Query, Project::BBMRI), + ("fasting_status", CriterionRole::Filter, Project::Bbmri), " (S.collection.fastingStatus.coding.code contains '{{C}}') ", ), ( - ("sampling_date", CriterionRole::Query, Project::BBMRI), + ("sampling_date", CriterionRole::Query, Project::Bbmri), " exists from [Specimen] S\nwhere FHIRHelpers.ToDateTime(S.collection.collected) between {{D1}} and {{D2}} ", ), ( - ("fasting_status", CriterionRole::Query, Project::BBMRI), + ("fasting_status", CriterionRole::Query, Project::Bbmri), " exists from [Specimen] S\nwhere S.collection.fastingStatus.coding.code contains '{{C}}' ", ), ( - ("storage_temperature", CriterionRole::Query, Project::BBMRI), + ("storage_temperature", CriterionRole::Query, Project::Bbmri), " exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding contains Code '{{C}}' from {{A1}}) ", ), ( - ("smoking_status", CriterionRole::Query, Project::BBMRI), + ("smoking_status", CriterionRole::Query, Project::Bbmri), " exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere O.value.coding.code contains '{{C}}' ", ), ] - .into(); - - map + .into() }); pub fn bbmri(ast: ast::Ast) -> Result { @@ -169,7 +160,7 @@ pub fn bbmri(ast: ast::Ast) -> Result { &mut retrieval_criteria, &mut filter_criteria, &mut code_systems, - Project::BBMRI, + Project::Bbmri, )?; // Only concatenate operator if it's not the last element @@ -181,8 +172,7 @@ pub fn bbmri(ast: ast::Ast) -> Result { retrieval_criteria += ")"; for code_system in code_systems { - lists = lists - + format!( + lists += format!( "codesystem {}: '{}' \n", code_system, CODE_LISTS.get(code_system).unwrap_or(&("")) @@ -191,11 +181,17 @@ pub fn bbmri(ast: ast::Ast) -> Result { } cql = cql - .replace("{[lists}}", lists.as_str()) - .replace("{{filter_criteria}}", filter_criteria.as_str()) - .replace("{{retrieval_criteria}}", retrieval_criteria.as_str()); + .replace("{{lists}}", lists.as_str()) + .replace("{{filter_criteria}}", filter_criteria.as_str()); + + if retrieval_criteria != *"()"{ // no criteria selected + cql = cql.replace("{{retrieval_criteria}}", retrieval_criteria.as_str()); + } else { + cql = cql.replace("{{retrieval_criteria}}", "true"); + } + - Ok(retrieval_criteria) + Ok(cql) } pub fn process( @@ -289,8 +285,32 @@ pub fn process( } } // deal with no lower or no upper value - ast::ConditionType::In => { - " in "; + ast::ConditionType::In => { // although in works in CQL, at least in some places, most of it is converted to multiple criteria with OR + let operator_str = " or "; + + match condition.value { + ast::ConditionValue::StringArray(string_array) => { + let mut condition_humongous_string = " (".to_string(); + for (index, string) in string_array.iter().enumerate() { + condition_humongous_string = condition_humongous_string + " (" + condition_string.as_str() + ") "; + condition_humongous_string = condition_humongous_string + .replace("{{C}}", string.as_str()); + + // Only concatenate operator if it's not the last element + if index < string_array.len() - 1 { + condition_humongous_string += operator_str; + } + + } + condition_string = condition_humongous_string + " )"; + + + }, + _ => { + return Err(FocusError::AstOperatorValueMismatch()); + } + } + } // this becomes or of all - deal with clones ast::ConditionType::Equals => { match condition.value { @@ -305,18 +325,16 @@ pub fn process( } } - ast::ConditionType::NotEquals => { //won't get it from Lens + ast::ConditionType::NotEquals => { // won't get it from Lens } - ast::ConditionType::Contains => { - "contains "; + ast::ConditionType::Contains => { // won't get it from Lens + } ast::ConditionType::GreaterThan => { - " greater than "; - } // guess Lens won't send me this, convert between to it + } // guess Lens won't send me this ast::ConditionType::LowerThan => { - " lower than "; - } // guess Lens won't send me this, convert between to it + } // guess Lens won't send me this }; retrieval_cond += condition_string.as_str(); @@ -384,6 +402,8 @@ mod test { const SOME_GBN: &str = r#"{"ast":{"children":[{"key":"gender","system":"","type":"IN","value":["other","male"]},{"key":"diagnosis","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","type":"EQUALS","value":"C24"},{"key":"diagnosis_age_donor","system":"","type":"BETWEEN","value":{"max":11,"min":1}},{"key":"date_of_diagnosis","system":"","type":"BETWEEN","value":{"max":"2023-10-30T23:00:00.000Z","min":"2023-10-29T23:00:00.000Z"}},{"key":"bmi","system":"","type":"BETWEEN","value":{"max":111,"min":1}},{"key":"body_weight","system":"","type":"BETWEEN","value":{"max":1111,"min":110}},{"key":"fasting_status","system":"","type":"IN","value":["Sober","Not sober"]},{"key":"smoking_status","system":"","type":"IN","value":["Smoker","Never smoked"]},{"key":"donor_age","system":"","type":"BETWEEN","value":{"max":123,"min":1}},{"key":"sample_kind","system":"","type":"IN","value":["blood-serum","tissue-other"]},{"key":"sampling_date","system":"","type":"BETWEEN","value":{"max":"2023-10-30T23:00:00.000Z","min":"2023-10-29T23:00:00.000Z"}},{"key":"storage_temperature","system":"","type":"IN","value":["temperature2to10","temperatureGN"]}],"de":"haupt","en":"main","key":"main","operand":"AND"},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; const LENS2: &str = r#"{"ast":{"children":[{"children":[{"children":[{"key":"gender","system":"","type":"EQUALS","value":"male"},{"key":"gender","system":"","type":"EQUALS","value":"female"}],"operand":"OR"},{"children":[{"key":"diagnosis","system":"","type":"EQUALS","value":"C41"},{"key":"diagnosis","system":"","type":"EQUALS","value":"C50"}],"operand":"OR"},{"children":[{"key":"sample_kind","system":"","type":"EQUALS","value":"tissue-frozen"},{"key":"sample_kind","system":"","type":"EQUALS","value":"blood-serum"}],"operand":"OR"}],"operand":"AND"},{"children":[{"children":[{"key":"gender","system":"","type":"EQUALS","value":"male"}],"operand":"OR"},{"children":[{"key":"diagnosis","system":"","type":"EQUALS","value":"C41"},{"key":"diagnosis","system":"","type":"EQUALS","value":"C50"}],"operand":"OR"},{"children":[{"key":"sample_kind","system":"","type":"EQUALS","value":"liquid-other"},{"key":"sample_kind","system":"","type":"EQUALS","value":"rna"},{"key":"sample_kind","system":"","type":"EQUALS","value":"urine"}],"operand":"OR"},{"children":[{"key":"storage_temperature","system":"","type":"EQUALS","value":"temperatureRoom"},{"key":"storage_temperature","system":"","type":"EQUALS","value":"four_degrees"}],"operand":"OR"}],"operand":"AND"}],"operand":"OR"},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + + const EMPTY: &str = r#"{"ast":{"children":[],"operand":"OR"}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; #[test] fn test_just_print() { @@ -435,5 +455,13 @@ mod test { "{:?}", bbmri(serde_json::from_str(LENS2).expect("Failed to deserialize JSON")) ); + + println!(); + + println!( + "{:?}", + bbmri(serde_json::from_str(EMPTY).expect("Failed to deserialize JSON")) + ); + } } diff --git a/src/errors.rs b/src/errors.rs index cdd1b1e..a11f00d 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,5 +1,4 @@ use thiserror::Error; -use crate::ast; #[derive(Error, Debug)] pub enum FocusError { diff --git a/src/omop.rs b/src/omop.rs index 58df307..e87a579 100644 --- a/src/omop.rs +++ b/src/omop.rs @@ -31,7 +31,7 @@ pub async fn post_ast(ast: ast::Ast) -> Result { .body(ast_string.clone()) .send() .await - .map_err(|e| FocusError::UnableToPostAst(e))?; + .map_err(FocusError::UnableToPostAst)?; debug!("Posted AST..."); @@ -40,7 +40,7 @@ pub async fn post_ast(ast: ast::Ast) -> Result { resp .text() .await - .map_err(|e| FocusError::UnableToPostAst(e))? + .map_err(FocusError::UnableToPostAst)? }, code => { warn!("Got unexpected code {code} while posting AST; reply was `{}`, debug info: {:?}", ast_string, resp); From 74cbe745324a47c6407e857e303148a8cc684b45 Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Sat, 2 Dec 2023 13:13:21 +0100 Subject: [PATCH 07/42] style --- src/main.rs | 87 ++++++++++++++++++++++++++--------------------------- 1 file changed, 42 insertions(+), 45 deletions(-) diff --git a/src/main.rs b/src/main.rs index fdeb66e..01ace44 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,14 +1,14 @@ +mod ast; mod banner; mod beam; mod blaze; mod config; +mod cql; +mod errors; +mod graceful_shutdown; mod logger; mod omop; mod util; -mod ast; -mod errors; -mod graceful_shutdown; -mod cql; use beam_lib::{TaskRequest, TaskResult}; use laplace_rs::ObfCache; @@ -25,13 +25,11 @@ use std::{process::exit, time::Duration}; use base64::{engine::general_purpose, Engine as _}; -use serde_json::from_slice; use serde::{Deserialize, Serialize}; +use serde_json::from_slice; use tracing::{debug, error, warn}; - - // result cache type SearchQuery = String; type Report = String; @@ -72,7 +70,6 @@ pub async fn main() -> ExitCode { } async fn main_loop() -> ExitCode { - let mut obf_cache: ObfCache = ObfCache { cache: HashMap::new(), }; @@ -80,15 +77,13 @@ async fn main_loop() -> ExitCode { let mut report_cache: ReportCache = ReportCache { cache: HashMap::new(), }; - if let Some(filename) = CONFIG.queries_to_cache_file_path.clone() { - let lines = util::read_lines(filename.clone().to_string()); match lines { Ok(ok_lines) => { for line in ok_lines { - let Ok(ok_line) = line else{ + let Ok(ok_line) = line else { warn!("A line in the file {} is not readable", filename); continue; }; @@ -145,16 +140,18 @@ async fn process_task( ) -> Result { debug!("Processing task {}", task.id); - let metadata: Metadata = serde_json::from_value(task.metadata.clone()).unwrap_or(Metadata {project: "default_obfuscation".to_string()}); + let metadata: Metadata = serde_json::from_value(task.metadata.clone()).unwrap_or(Metadata { + project: "default_obfuscation".to_string(), + }); if CONFIG.endpoint_type == config::EndpointType::Blaze { let query = parse_query(task)?; if query.lang == "cql" { // TODO: Change query.lang to an enum - return Ok(run_cql_query(task, &query, obf_cache, report_cache, metadata.project).await)?; + Ok(run_cql_query(task, &query, obf_cache, report_cache, metadata.project).await)? } else { warn!("Can't run queries with language {} in Blaze", query.lang); - return Ok(beam::beam_result::perm_failed( + Ok(beam::beam_result::perm_failed( CONFIG.beam_app_id_long.clone(), vec![task.from.clone()], task.id, @@ -162,20 +159,25 @@ async fn process_task( "Can't run queries with language {} and/or endpoint type {}", query.lang, CONFIG.endpoint_type ), - )); + )) } - } else if CONFIG.endpoint_type == config::EndpointType::Omop { - - let decoded = decode_body(task)?; - let omop_query: omop::OmopQuery = from_slice(&decoded).map_err(|e| FocusError::ParsingError(e.to_string()))?; - //TODO check that the language is ast - let query_decoded = general_purpose::STANDARD.decode(omop_query.query).map_err(|e| FocusError::DecodeError(e))?; - let ast: ast::Ast = from_slice(&query_decoded).map_err(|e| FocusError::ParsingError(e.to_string()))?; + } else if CONFIG.endpoint_type == config::EndpointType::Omop { + let decoded = decode_body(task)?; + let omop_query: omop::OmopQuery = + from_slice(&decoded).map_err(|e| FocusError::ParsingError(e.to_string()))?; + //TODO check that the language is ast + let query_decoded = general_purpose::STANDARD + .decode(omop_query.query) + .map_err(FocusError::DecodeError)?; + let ast: ast::Ast = + from_slice(&query_decoded).map_err(|e| FocusError::ParsingError(e.to_string()))?; return Ok(run_omop_query(task, ast).await)?; - } else { - warn!("Can't run queries with endpoint type {}", CONFIG.endpoint_type); + warn!( + "Can't run queries with endpoint type {}", + CONFIG.endpoint_type + ); return Ok(beam::beam_result::perm_failed( CONFIG.beam_app_id_long.clone(), vec![task.from.clone()], @@ -276,9 +278,8 @@ async fn run_cql_query( query: &Query, obf_cache: &mut ObfCache, report_cache: &mut ReportCache, - project: String + project: String, ) -> Result { - let encoded_query = query.lib["content"][0]["data"] .as_str() @@ -311,7 +312,7 @@ async fn run_cql_query( let cql_result_new: String = match CONFIG.obfuscate { config::Obfuscate::Yes => { - if !CONFIG.unobfuscated.contains(&project){ + if !CONFIG.unobfuscated.contains(&project) { obfuscate_counts_mr( &cql_result, obf_cache, @@ -328,7 +329,7 @@ async fn run_cql_query( } else { cql_result } - }, + } config::Obfuscate::No => cql_result, }; @@ -347,7 +348,7 @@ async fn run_cql_query( CONFIG.beam_app_id_long.clone(), vec![task.to_owned().from], task.to_owned().id, - e.to_string() + e.to_string(), ) }); @@ -364,23 +365,22 @@ async fn run_omop_query(task: &BeamTask, ast: ast::Ast) -> Result { - omop_result = omop_result.replacen("{", format!(r#"{{"provider_icon":"{}","#, provider_icon).as_str(), 1); - } - None => {} + if let Some(provider_icon) = CONFIG.provider_icon.clone() { + omop_result = omop_result.replacen( + '{', + format!(r#"{{"provider_icon":"{}","#, provider_icon).as_str(), + 1, + ); } - match CONFIG.provider.clone() { - Some(provider) => { - omop_result = omop_result.replacen("{", format!(r#"{{"provider":"{}","#, provider).as_str(), 1); - } - None => {} + if let Some(provider) = CONFIG.provider.clone() { + omop_result = + omop_result.replacen('{', format!(r#"{{"provider":"{}","#, provider).as_str(), 1); } let result = beam_result(task.to_owned(), omop_result).unwrap_or_else(|e| { err.body = beam_lib::RawString(e.to_string()); - return err; + err }); Ok(result) @@ -424,15 +424,12 @@ fn replace_cql_library(mut query: Query) -> Result { Ok(query) } -fn beam_result( - task: BeamTask, - measure_report: String, -) -> Result { +fn beam_result(task: BeamTask, measure_report: String) -> Result { let data = general_purpose::STANDARD.encode(measure_report.as_bytes()); Ok(beam::beam_result::succeeded( CONFIG.beam_app_id_long.clone(), vec![task.from], task.id, - data + data, )) } From d78b530ece0e088911b3560d03fcbae780287df9 Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Sat, 2 Dec 2023 21:54:29 +0100 Subject: [PATCH 08/42] style --- src/cql.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cql.rs b/src/cql.rs index db8a469..15ed534 100644 --- a/src/cql.rs +++ b/src/cql.rs @@ -71,7 +71,7 @@ static CRITERION_CODE_LISTS: Lazy>> = Lazy::n .into() }); -static CQL_SNIPPETS_BBMRI: Lazy> = Lazy::new(|| { +static CQL_SNIPPETS: Lazy> = Lazy::new(|| { // CQL snippets depending on the criteria [ (("gender", CriterionRole::Query, Project::Bbmri), "Patient.gender = '{{C}}'"), @@ -208,7 +208,7 @@ pub fn process( ast::Child::Condition(condition) => { let condition_key_trans = condition.key.as_str(); - let condition_snippet = CQL_SNIPPETS_BBMRI.get(&( + let condition_snippet = CQL_SNIPPETS.get(&( condition_key_trans, CriterionRole::Query, project.clone(), From 2f721d5f6ada1a96a8f416bffdb8552a4479672c Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Tue, 5 Dec 2023 14:08:08 +0100 Subject: [PATCH 09/42] filter --- src/cql.rs | 150 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 95 insertions(+), 55 deletions(-) diff --git a/src/cql.rs b/src/cql.rs index 15ed534..c3e07c9 100644 --- a/src/cql.rs +++ b/src/cql.rs @@ -139,7 +139,7 @@ static CQL_SNIPPETS: Lazy> = Lazy: pub fn bbmri(ast: ast::Ast) -> Result { let mut retrieval_criteria: String = "(".to_string(); // main selection criteria (Patient) - let mut filter_criteria: String = "(".to_string(); // criteria for filtering specimens + let mut filter_criteria: String = " where (".to_string(); // criteria for filtering specimens let mut code_systems: HashSet<&str> = HashSet::new(); // code lists needed depending on the criteria code_systems.insert("icd10"); //for diagnosis stratifier @@ -170,26 +170,34 @@ pub fn bbmri(ast: ast::Ast) -> Result { } retrieval_criteria += ")"; + filter_criteria += ")"; for code_system in code_systems { lists += format!( - "codesystem {}: '{}' \n", - code_system, - CODE_LISTS.get(code_system).unwrap_or(&("")) - ) - .as_str(); + "codesystem {}: '{}' \n", + code_system, + CODE_LISTS.get(code_system).unwrap_or(&("")) + ) + .as_str(); } cql = cql .replace("{{lists}}", lists.as_str()) .replace("{{filter_criteria}}", filter_criteria.as_str()); - if retrieval_criteria != *"()"{ // no criteria selected + if retrieval_criteria != *"()" { + // no criteria selected cql = cql.replace("{{retrieval_criteria}}", retrieval_criteria.as_str()); } else { cql = cql.replace("{{retrieval_criteria}}", "true"); } + if filter_criteria != *" where ()" { + // no criteria selected + cql = cql.replace("{{retrieval_criteria}}", filter_criteria.as_str()); + } else { + cql = cql.replace("{{retrieval_criteria}}", ""); + } Ok(cql) } @@ -202,20 +210,24 @@ pub fn process( project: Project, ) -> Result<(), FocusError> { let mut retrieval_cond: String = "(".to_string(); - let mut filter_cond: String = "(".to_string(); + let mut filter_cond: String = "".to_string(); match child { ast::Child::Condition(condition) => { let condition_key_trans = condition.key.as_str(); - let condition_snippet = CQL_SNIPPETS.get(&( - condition_key_trans, - CriterionRole::Query, - project.clone(), - )); + let condition_snippet = + CQL_SNIPPETS.get(&(condition_key_trans, CriterionRole::Query, project.clone())); if let Some(snippet) = condition_snippet { let mut condition_string = (*snippet).to_string(); + let mut filter_string: String = "".to_string(); + + let filter_snippet = CQL_SNIPPETS.get(&( + condition_key_trans, + CriterionRole::Filter, + project.clone(), + )); let code_lists_option = CRITERION_CODE_LISTS.get(&(condition_key_trans, project)); if let Some(code_lists_vec) = code_lists_option { @@ -226,10 +238,10 @@ pub fn process( condition_string = condition_string.replace(placeholder.as_str(), code_list); } - } + } - if condition_string.contains("{{K}}") { - //observation loinc code + if condition_string.contains("{{K}}") { + //observation loinc code, those only apply to query criteria, we don't filter specimens by observations let observation_code_option = OBSERVATION_LOINC_CODE.get(&condition_key_trans); if let Some(observation_code) = observation_code_option { @@ -241,9 +253,12 @@ pub fn process( } } + if let Some(filtret) = filter_snippet { + filter_string = (*filtret).to_string(); + } + match condition.type_ { - ast::ConditionType::Between => { - //it has both min and max values stated + ast::ConditionType::Between => { // both min and max values stated match condition.value { ast::ConditionValue::DateRange(date_range) => { let datetime_str_min = date_range.min.as_str(); @@ -256,6 +271,8 @@ pub fn process( condition_string = condition_string.replace("{{D1}}", date_str_min.as_str()); + filter_string = + filter_string.replace("{{D1}}", date_str_min.as_str()); // no condition needed, "" stays "" } else { return Err(FocusError::AstInvalidDateFormat(date_range.min)); } @@ -269,83 +286,95 @@ pub fn process( condition_string = condition_string.replace("{{D2}}", date_str_max.as_str()); + filter_string = + filter_string.replace("{{D2}}", date_str_max.as_str()); // no condition needed, "" stays "" } else { return Err(FocusError::AstInvalidDateFormat(date_range.max)); } - }, + } ast::ConditionValue::NumRange(num_range) => { condition_string = condition_string .replace("{{D1}}", num_range.min.to_string().as_str()); condition_string = condition_string .replace("{{D2}}", num_range.max.to_string().as_str()); - }, + filter_string = filter_string + .replace("{{D1}}", num_range.min.to_string().as_str()); // no condition needed, "" stays "" + filter_string = filter_string + .replace("{{D2}}", num_range.max.to_string().as_str()); // no condition needed, "" stays "" + } _ => { return Err(FocusError::AstOperatorValueMismatch()); } } - } // deal with no lower or no upper value - ast::ConditionType::In => { // although in works in CQL, at least in some places, most of it is converted to multiple criteria with OR + ast::ConditionType::In => { + // although in works in CQL, at least in some places, most of it is converted to multiple criteria with OR let operator_str = " or "; match condition.value { ast::ConditionValue::StringArray(string_array) => { let mut condition_humongous_string = " (".to_string(); + let mut filter_humongous_string = " (".to_string(); + for (index, string) in string_array.iter().enumerate() { - condition_humongous_string = condition_humongous_string + " (" + condition_string.as_str() + ") "; condition_humongous_string = condition_humongous_string - .replace("{{C}}", string.as_str()); + + " (" + + condition_string.as_str() + + ") "; + condition_humongous_string = condition_humongous_string + .replace("{{C}}", string.as_str()); + + filter_humongous_string = filter_humongous_string + + " (" + + filter_string.as_str() + + ") "; + filter_humongous_string = + filter_humongous_string.replace("{{C}}", string.as_str()); // Only concatenate operator if it's not the last element if index < string_array.len() - 1 { condition_humongous_string += operator_str; + filter_humongous_string += operator_str; } - } condition_string = condition_humongous_string + " )"; - - }, - _ => { - return Err(FocusError::AstOperatorValueMismatch()); + if filter_string != "" { + filter_string = filter_humongous_string + " )"; + } } - } - - } // this becomes or of all - deal with clones - ast::ConditionType::Equals => { - match condition.value { - ast::ConditionValue::String(string) => { - condition_string = condition_string - .replace("{{C}}", string.as_str()); - - }, _ => { return Err(FocusError::AstOperatorValueMismatch()); } - } - } + } // this becomes or of all + ast::ConditionType::Equals => match condition.value { + ast::ConditionValue::String(string) => { + condition_string = condition_string.replace("{{C}}", string.as_str()); + filter_string = filter_string.replace("{{C}}", string.as_str()); // no condition needed, "" stays "" + } + _ => { + return Err(FocusError::AstOperatorValueMismatch()); + } + }, ast::ConditionType::NotEquals => { // won't get it from Lens - } ast::ConditionType::Contains => { // won't get it from Lens - } - ast::ConditionType::GreaterThan => { - } // guess Lens won't send me this - ast::ConditionType::LowerThan => { - } // guess Lens won't send me this + ast::ConditionType::GreaterThan => {} // guess Lens won't send me this + ast::ConditionType::LowerThan => {} // guess Lens won't send me this }; retrieval_cond += condition_string.as_str(); + filter_cond += filter_string.as_str(); // no condition needed, "" can be added with no change } else { return Err(FocusError::AstUnknownCriterion( condition_key_trans.to_string(), )); } - - filter_cond += condition.key.as_str(); - + if filter_cond != "" { + filter_cond += " "; + } retrieval_cond += " "; } @@ -367,16 +396,27 @@ pub fn process( // Only concatenate operator if it's not the last element if index < operation.children.len() - 1 { retrieval_cond += operator_str; + if filter_cond != "" { + filter_cond += operator_str; + dbg!(filter_cond.clone()); + } } } } } retrieval_cond += ")"; - filter_cond += ")"; *retrieval_criteria += retrieval_cond.as_str(); - *filter_criteria += filter_cond.as_str(); + + if filter_cond != "" { + dbg!(filter_cond.clone()); + *filter_criteria += "("; + *filter_criteria += filter_cond.as_str(); + *filter_criteria += ")"; + + dbg!(filter_criteria.clone()); + } Ok(()) } @@ -402,8 +442,9 @@ mod test { const SOME_GBN: &str = r#"{"ast":{"children":[{"key":"gender","system":"","type":"IN","value":["other","male"]},{"key":"diagnosis","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","type":"EQUALS","value":"C24"},{"key":"diagnosis_age_donor","system":"","type":"BETWEEN","value":{"max":11,"min":1}},{"key":"date_of_diagnosis","system":"","type":"BETWEEN","value":{"max":"2023-10-30T23:00:00.000Z","min":"2023-10-29T23:00:00.000Z"}},{"key":"bmi","system":"","type":"BETWEEN","value":{"max":111,"min":1}},{"key":"body_weight","system":"","type":"BETWEEN","value":{"max":1111,"min":110}},{"key":"fasting_status","system":"","type":"IN","value":["Sober","Not sober"]},{"key":"smoking_status","system":"","type":"IN","value":["Smoker","Never smoked"]},{"key":"donor_age","system":"","type":"BETWEEN","value":{"max":123,"min":1}},{"key":"sample_kind","system":"","type":"IN","value":["blood-serum","tissue-other"]},{"key":"sampling_date","system":"","type":"BETWEEN","value":{"max":"2023-10-30T23:00:00.000Z","min":"2023-10-29T23:00:00.000Z"}},{"key":"storage_temperature","system":"","type":"IN","value":["temperature2to10","temperatureGN"]}],"de":"haupt","en":"main","key":"main","operand":"AND"},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; const LENS2: &str = r#"{"ast":{"children":[{"children":[{"children":[{"key":"gender","system":"","type":"EQUALS","value":"male"},{"key":"gender","system":"","type":"EQUALS","value":"female"}],"operand":"OR"},{"children":[{"key":"diagnosis","system":"","type":"EQUALS","value":"C41"},{"key":"diagnosis","system":"","type":"EQUALS","value":"C50"}],"operand":"OR"},{"children":[{"key":"sample_kind","system":"","type":"EQUALS","value":"tissue-frozen"},{"key":"sample_kind","system":"","type":"EQUALS","value":"blood-serum"}],"operand":"OR"}],"operand":"AND"},{"children":[{"children":[{"key":"gender","system":"","type":"EQUALS","value":"male"}],"operand":"OR"},{"children":[{"key":"diagnosis","system":"","type":"EQUALS","value":"C41"},{"key":"diagnosis","system":"","type":"EQUALS","value":"C50"}],"operand":"OR"},{"children":[{"key":"sample_kind","system":"","type":"EQUALS","value":"liquid-other"},{"key":"sample_kind","system":"","type":"EQUALS","value":"rna"},{"key":"sample_kind","system":"","type":"EQUALS","value":"urine"}],"operand":"OR"},{"children":[{"key":"storage_temperature","system":"","type":"EQUALS","value":"temperatureRoom"},{"key":"storage_temperature","system":"","type":"EQUALS","value":"four_degrees"}],"operand":"OR"}],"operand":"AND"}],"operand":"OR"},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; - - const EMPTY: &str = r#"{"ast":{"children":[],"operand":"OR"}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + + const EMPTY: &str = + r#"{"ast":{"children":[],"operand":"OR"}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; #[test] fn test_just_print() { @@ -462,6 +503,5 @@ mod test { "{:?}", bbmri(serde_json::from_str(EMPTY).expect("Failed to deserialize JSON")) ); - } } From 5860bdd5b091210b3b39b506ef9cdd3c229ccec5 Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Tue, 5 Dec 2023 16:14:21 +0100 Subject: [PATCH 10/42] hashset -> indexset --- Cargo.toml | 7 +- input/tests/results/result.txt | 0 resources/test/result_empty.cql | 85 +++++++++++++++++++++++ src/cql.rs | 115 +++++++++++++++++--------------- 4 files changed, 151 insertions(+), 56 deletions(-) create mode 100644 input/tests/results/result.txt create mode 100644 resources/test/result_empty.cql diff --git a/Cargo.toml b/Cargo.toml index 656d97f..4921e71 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,9 @@ reqwest = { version = "0.11", default_features = false, features = ["json", "def serde = { version = "1.0.152", features = ["serde_derive"] } serde_json = "1.0" thiserror = "1.0.38" +rand = { default-features = false, version = "0.8.5" } +chrono = "0.4.31" +indexmap = "2.1.0" tokio = { version = "1.25.0", default_features = false, features = ["signal", "rt-multi-thread", "macros"] } beam-lib = { git = "https://github.com/samply/beam", branch = "develop", features = ["http-util"] } laplace_rs = {version = "0.2.0", git = "https://github.com/samply/laplace-rs.git", branch = "main" } @@ -26,10 +29,10 @@ once_cell = "1.18" # Command Line Interface clap = { version = "4.0", default_features = false, features = ["std", "env", "derive", "help"] } -rand = { default-features = false, version = "0.8.5" } -chrono = "0.4.31" + [dev-dependencies] +pretty_assertions = "1.4.0" tokio-test = "0.4.2" [build-dependencies] diff --git a/input/tests/results/result.txt b/input/tests/results/result.txt new file mode 100644 index 0000000..e69de29 diff --git a/resources/test/result_empty.cql b/resources/test/result_empty.cql new file mode 100644 index 0000000..6fa43a7 --- /dev/null +++ b/resources/test/result_empty.cql @@ -0,0 +1,85 @@ +library Retrieve +using FHIR version '4.0.0' +include FHIRHelpers version '4.0.0' + +codesystem icd10: 'http://hl7.org/fhir/sid/icd-10' +codesystem SampleMaterialType: 'https://fhir.bbmri.de/CodeSystem/SampleMaterialType' + + +context Patient + +define AgeClass: +if (Patient.birthDate is null) then 'unknown' else ToString((AgeInYears() div 10) * 10) + +define Gender: +if (Patient.gender is null) then 'unknown' else Patient.gender + +define Custodian: + First(from Specimen.extension E + where E.url = 'https://fhir.bbmri.de/StructureDefinition/Custodian' + return (E.value as Reference).identifier.value) + +define function SampleType(specimen FHIR.Specimen): + case FHIRHelpers.ToCode(specimen.type.coding.where(system = 'https://fhir.bbmri.de/CodeSystem/SampleMaterialType').first()) + when Code 'plasma-edta' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-citrat' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-heparin' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-cell-free' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-other' from SampleMaterialType then 'blood-plasma' + when Code 'plasma' from SampleMaterialType then 'blood-plasma' + when Code 'tissue-formalin' from SampleMaterialType then 'tissue-ffpe' + when Code 'tumor-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'normal-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'other-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'tumor-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'normal-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'other-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'tissue-paxgene-or-else' from SampleMaterialType then 'tissue-other' + when Code 'derivative' from SampleMaterialType then 'derivative-other' + when Code 'liquid' from SampleMaterialType then 'liquid-other' + when Code 'tissue' from SampleMaterialType then 'tissue-other' + when Code 'serum' from SampleMaterialType then 'blood-serum' + when Code 'cf-dna' from SampleMaterialType then 'dna' + when Code 'g-dna' from SampleMaterialType then 'dna' + when Code 'blood-plasma' from SampleMaterialType then 'blood-plasma' + when Code 'tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'tissue-other' from SampleMaterialType then 'tissue-other' + when Code 'derivative-other' from SampleMaterialType then 'derivative-other' + when Code 'liquid-other' from SampleMaterialType then 'liquid-other' + when Code 'blood-serum' from SampleMaterialType then 'blood-serum' + when Code 'dna' from SampleMaterialType then 'dna' + when Code 'buffy-coat' from SampleMaterialType then 'buffy-coat' + when Code 'urine' from SampleMaterialType then 'urine' + when Code 'ascites' from SampleMaterialType then 'ascites' + when Code 'saliva' from SampleMaterialType then 'saliva' + when Code 'csf-liquor' from SampleMaterialType then 'csf-liquor' + when Code 'bone-marrow' from SampleMaterialType then 'bone-marrow' + when Code 'peripheral-blood-cells-vital' from SampleMaterialType then 'peripheral-blood-cells-vital' + when Code 'stool-faeces' from SampleMaterialType then 'stool-faeces' + when Code 'rna' from SampleMaterialType then 'rna' + when Code 'whole-blood' from SampleMaterialType then 'whole-blood' + when Code 'swab' from SampleMaterialType then 'swab' + when Code 'dried-whole-blood' from SampleMaterialType then 'dried-whole-blood' + when null then 'Unknown' + else 'Unknown' + end +define Specimen: + if InInitialPopulation then [Specimen] S where () else {} as List + +define Diagnosis: +if InInitialPopulation then [Condition] else {} as List + +define function DiagnosisCode(condition FHIR.Condition): +condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first() + +define function DiagnosisCode(condition FHIR.Condition, specimen FHIR.Specimen): +Coalesce( + condition.code.coding.where(system = 'http://hl7.org/fhir/sid/icd-10').code.first(), + condition.code.coding.where(system = 'http://fhir.de/CodeSystem/dimdi/icd-10-gm').code.first(), + specimen.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code.first(), + condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first() + ) + +define InInitialPopulation: +true \ No newline at end of file diff --git a/src/cql.rs b/src/cql.rs index c3e07c9..6683cbc 100644 --- a/src/cql.rs +++ b/src/cql.rs @@ -5,7 +5,7 @@ use chrono::offset::Utc; use chrono::DateTime; use once_cell::sync::Lazy; use std::collections::HashMap; -use std::collections::HashSet; +use indexmap::set::IndexSet; #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Hash)] enum CriterionRole { @@ -77,60 +77,60 @@ static CQL_SNIPPETS: Lazy> = Lazy: (("gender", CriterionRole::Query, Project::Bbmri), "Patient.gender = '{{C}}'"), ( ("diagnosis", CriterionRole::Query, Project::Bbmri), - " ((exists[Condition: Code '{{C}}' from {{A1}}]) or (exists[Condition: Code '{{C}}' from {{A2}}])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains '{{C}}')) ", + "((exists[Condition: Code '{{C}}' from {{A1}}]) or (exists[Condition: Code '{{C}}' from {{A2}}])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains '{{C}}'))", ), - (("diagnosis_old", CriterionRole::Query, Project::Bbmri), " exists [Condition: Code '{{C}}' from {{A1}}] "), + (("diagnosis_old", CriterionRole::Query, Project::Bbmri), " exists [Condition: Code '{{C}}' from {{A1}}]"), ( ("date_of_diagnosis", CriterionRole::Query, Project::Bbmri), - " exists from [Condition] C\nwhere FHIRHelpers.ToDateTime(C.onset) between {{D1}} and {{D2}} ", + "exists from [Condition] C\nwhere FHIRHelpers.ToDateTime(C.onset) between {{D1}} and {{D2}}", ), ( ("diagnosis_age_donor", CriterionRole::Query, Project::Bbmri), - " exists from [Condition] C\nwhere AgeInYearsAt(FHIRHelpers.ToDateTime(C.onset)) between Ceiling({{D1}}) and Ceiling({{D2}}) ", + "exists from [Condition] C\nwhere AgeInYearsAt(FHIRHelpers.ToDateTime(C.onset)) between Ceiling({{D1}}) and Ceiling({{D2}})", ), - (("donor_age", CriterionRole::Query, Project::Bbmri), " AgeInYears() between Ceiling({{D1}}) and Ceiling({{D2}}) "), + (("donor_age", CriterionRole::Query, Project::Bbmri), " AgeInYears() between Ceiling({{D1}}) and Ceiling({{D2}})"), ( ("observationRange", CriterionRole::Query, Project::Bbmri), - " exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere O.value between {{D1}} and {{D2}} ", + "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere O.value between {{D1}} and {{D2}}", ), ( ("body_weight", CriterionRole::Query, Project::Bbmri), - " exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere ((O.value as Quantity) < {{D1}} 'kg' and (O.value as Quantity) > {{D2}} 'kg') ", + "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere ((O.value as Quantity) < {{D1}} 'kg' and (O.value as Quantity) > {{D2}} 'kg')", ), ( ("bmi", CriterionRole::Query, Project::Bbmri), - " exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere ((O.value as Quantity) < {{D1}} 'kg/m2' and (O.value as Quantity) > {{D2}} 'kg/m2') ", + "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere ((O.value as Quantity) < {{D1}} 'kg/m2' and (O.value as Quantity) > {{D2}} 'kg/m2')", ), - (("sample_kind", CriterionRole::Query, Project::Bbmri), " exists [Specimen: Code '{{C}}' from {{A1}}] "), - (("sample_kind", CriterionRole::Filter, Project::Bbmri), " (S.type.coding.code contains '{{C}}') "), + (("sample_kind", CriterionRole::Query, Project::Bbmri), " exists [Specimen: Code '{{C}}' from {{A1}}]"), + (("sample_kind", CriterionRole::Filter, Project::Bbmri), " (S.type.coding.code contains '{{C}}')"), ( ("storage_temperature", CriterionRole::Filter, Project::Bbmri), - " (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding.code contains '{{C}}') ", + "(S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding.code contains '{{C}}')", ), ( ("sampling_date", CriterionRole::Filter, Project::Bbmri), - " (FHIRHelpers.ToDateTime(S.collection.collected) between {{D1}} and {{D2}}) ", + "(FHIRHelpers.ToDateTime(S.collection.collected) between {{D1}} and {{D2}}) ", ), ( ("fasting_status", CriterionRole::Filter, Project::Bbmri), - " (S.collection.fastingStatus.coding.code contains '{{C}}') ", + "(S.collection.fastingStatus.coding.code contains '{{C}}') ", ), ( ("sampling_date", CriterionRole::Query, Project::Bbmri), - " exists from [Specimen] S\nwhere FHIRHelpers.ToDateTime(S.collection.collected) between {{D1}} and {{D2}} ", + "exists from [Specimen] S\nwhere FHIRHelpers.ToDateTime(S.collection.collected) between {{D1}} and {{D2}} ", ), ( ("fasting_status", CriterionRole::Query, Project::Bbmri), - " exists from [Specimen] S\nwhere S.collection.fastingStatus.coding.code contains '{{C}}' ", + "exists from [Specimen] S\nwhere S.collection.fastingStatus.coding.code contains '{{C}}' ", ), ( ("storage_temperature", CriterionRole::Query, Project::Bbmri), - " exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding contains Code '{{C}}' from {{A1}}) ", + "exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding contains Code '{{C}}' from {{A1}}) ", ), ( ("smoking_status", CriterionRole::Query, Project::Bbmri), - " exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere O.value.coding.code contains '{{C}}' ", + "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere O.value.coding.code contains '{{C}}' ", ), ] .into() @@ -141,7 +141,7 @@ pub fn bbmri(ast: ast::Ast) -> Result { let mut filter_criteria: String = " where (".to_string(); // criteria for filtering specimens - let mut code_systems: HashSet<&str> = HashSet::new(); // code lists needed depending on the criteria + let mut code_systems: IndexSet<&str> = IndexSet::new(); // code lists needed depending on the criteria code_systems.insert("icd10"); //for diagnosis stratifier code_systems.insert("SampleMaterialType"); //for sample type stratifier @@ -174,7 +174,7 @@ pub fn bbmri(ast: ast::Ast) -> Result { for code_system in code_systems { lists += format!( - "codesystem {}: '{}' \n", + "codesystem {}: '{}'\n", code_system, CODE_LISTS.get(code_system).unwrap_or(&("")) ) @@ -206,7 +206,7 @@ pub fn process( child: ast::Child, retrieval_criteria: &mut String, filter_criteria: &mut String, - code_systems: &mut HashSet<&str>, + code_systems: &mut IndexSet<&str>, project: Project, ) -> Result<(), FocusError> { let mut retrieval_cond: String = "(".to_string(); @@ -313,21 +313,21 @@ pub fn process( match condition.value { ast::ConditionValue::StringArray(string_array) => { - let mut condition_humongous_string = " (".to_string(); - let mut filter_humongous_string = " (".to_string(); + let mut condition_humongous_string = "(".to_string(); + let mut filter_humongous_string = "(".to_string(); for (index, string) in string_array.iter().enumerate() { condition_humongous_string = condition_humongous_string - + " (" + + "(" + condition_string.as_str() - + ") "; + + ")"; condition_humongous_string = condition_humongous_string .replace("{{C}}", string.as_str()); filter_humongous_string = filter_humongous_string - + " (" + + "(" + filter_string.as_str() - + ") "; + + ")"; filter_humongous_string = filter_humongous_string.replace("{{C}}", string.as_str()); @@ -337,10 +337,10 @@ pub fn process( filter_humongous_string += operator_str; } } - condition_string = condition_humongous_string + " )"; + condition_string = condition_humongous_string + ")"; - if filter_string != "" { - filter_string = filter_humongous_string + " )"; + if !filter_string.is_empty() { + filter_string = filter_humongous_string + ")"; } } _ => { @@ -372,10 +372,10 @@ pub fn process( condition_key_trans.to_string(), )); } - if filter_cond != "" { - filter_cond += " "; + if !filter_cond.is_empty() { + // filter_cond += " "; } - retrieval_cond += " "; + //retrieval_cond += " "; } ast::Child::Operation(operation) => { @@ -396,7 +396,7 @@ pub fn process( // Only concatenate operator if it's not the last element if index < operation.children.len() - 1 { retrieval_cond += operator_str; - if filter_cond != "" { + if !filter_cond.is_empty() { filter_cond += operator_str; dbg!(filter_cond.clone()); } @@ -409,8 +409,11 @@ pub fn process( *retrieval_criteria += retrieval_cond.as_str(); - if filter_cond != "" { + if !filter_cond.is_empty() { dbg!(filter_cond.clone()); + if !filter_criteria.is_empty() { + *filter_criteria += " and "; + } *filter_criteria += "("; *filter_criteria += filter_cond.as_str(); *filter_criteria += ")"; @@ -424,6 +427,7 @@ pub fn process( #[cfg(test)] mod test { use super::*; + use pretty_assertions; const AST: &str = r#"{"ast":{"operand":"AND","children":[{"key":"age","type":"EQUALS","value":5.0}]},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; @@ -478,30 +482,33 @@ mod test { // bbmri(serde_json::from_str(C61_OR_MALE).expect("Failed to deserialize JSON")) // ); - println!( - "{:?}", - bbmri(serde_json::from_str(ALL_GBN).expect("Failed to deserialize JSON")) - ); + // println!( + // "{:?}", + // bbmri(serde_json::from_str(ALL_GBN).expect("Failed to deserialize JSON")) + // ); - println!(); + // println!(); - println!( - "{:?}", - bbmri(serde_json::from_str(SOME_GBN).expect("Failed to deserialize JSON")) - ); + // println!( + // "{:?}", + // bbmri(serde_json::from_str(SOME_GBN).expect("Failed to deserialize JSON")) + // ); - println!(); + // println!(); - println!( - "{:?}", - bbmri(serde_json::from_str(LENS2).expect("Failed to deserialize JSON")) - ); + // println!( + // "{:?}", + // bbmri(serde_json::from_str(LENS2).expect("Failed to deserialize JSON")) + // ); + + // println!(); + + // println!( + // "{:?}", + // bbmri(serde_json::from_str(EMPTY).expect("Failed to deserialize JSON")) + // ); - println!(); + pretty_assertions::assert_eq!(bbmri(serde_json::from_str(EMPTY).unwrap()).unwrap(), include_str!("../resources/test/result_empty.cql").to_string()); - println!( - "{:?}", - bbmri(serde_json::from_str(EMPTY).expect("Failed to deserialize JSON")) - ); } } From 91aeff7965eba97876d4601ffb46628638606f1f Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Wed, 6 Dec 2023 16:07:04 +0100 Subject: [PATCH 11/42] almost there --- resources/test/result.cql | 0 resources/test/result_empty.cql | 2 +- resources/test/result_lens2.cql | 87 +++++++++++++++++++++++++++++++++ src/cql.rs | 49 +++++++++---------- 4 files changed, 112 insertions(+), 26 deletions(-) create mode 100644 resources/test/result.cql create mode 100644 resources/test/result_lens2.cql diff --git a/resources/test/result.cql b/resources/test/result.cql new file mode 100644 index 0000000..e69de29 diff --git a/resources/test/result_empty.cql b/resources/test/result_empty.cql index 6fa43a7..735c0ad 100644 --- a/resources/test/result_empty.cql +++ b/resources/test/result_empty.cql @@ -65,7 +65,7 @@ define function SampleType(specimen FHIR.Specimen): else 'Unknown' end define Specimen: - if InInitialPopulation then [Specimen] S where () else {} as List + if InInitialPopulation then [Specimen] S else {} as List define Diagnosis: if InInitialPopulation then [Condition] else {} as List diff --git a/resources/test/result_lens2.cql b/resources/test/result_lens2.cql new file mode 100644 index 0000000..7733570 --- /dev/null +++ b/resources/test/result_lens2.cql @@ -0,0 +1,87 @@ +library Retrieve +using FHIR version '4.0.0' +include FHIRHelpers version '4.0.0' + +codesystem icd10: 'http://hl7.org/fhir/sid/icd-10' +codesystem SampleMaterialType: 'https://fhir.bbmri.de/CodeSystem/SampleMaterialType' +codesystem icd10gm: 'http://fhir.de/CodeSystem/dimdi/icd-10-gm' +codesystem StorageTemperature: 'https://fhir.bbmri.de/CodeSystem/StorageTemperature' + + +context Patient + +define AgeClass: +if (Patient.birthDate is null) then 'unknown' else ToString((AgeInYears() div 10) * 10) + +define Gender: +if (Patient.gender is null) then 'unknown' else Patient.gender + +define Custodian: + First(from Specimen.extension E + where E.url = 'https://fhir.bbmri.de/StructureDefinition/Custodian' + return (E.value as Reference).identifier.value) + +define function SampleType(specimen FHIR.Specimen): + case FHIRHelpers.ToCode(specimen.type.coding.where(system = 'https://fhir.bbmri.de/CodeSystem/SampleMaterialType').first()) + when Code 'plasma-edta' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-citrat' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-heparin' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-cell-free' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-other' from SampleMaterialType then 'blood-plasma' + when Code 'plasma' from SampleMaterialType then 'blood-plasma' + when Code 'tissue-formalin' from SampleMaterialType then 'tissue-ffpe' + when Code 'tumor-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'normal-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'other-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'tumor-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'normal-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'other-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'tissue-paxgene-or-else' from SampleMaterialType then 'tissue-other' + when Code 'derivative' from SampleMaterialType then 'derivative-other' + when Code 'liquid' from SampleMaterialType then 'liquid-other' + when Code 'tissue' from SampleMaterialType then 'tissue-other' + when Code 'serum' from SampleMaterialType then 'blood-serum' + when Code 'cf-dna' from SampleMaterialType then 'dna' + when Code 'g-dna' from SampleMaterialType then 'dna' + when Code 'blood-plasma' from SampleMaterialType then 'blood-plasma' + when Code 'tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'tissue-other' from SampleMaterialType then 'tissue-other' + when Code 'derivative-other' from SampleMaterialType then 'derivative-other' + when Code 'liquid-other' from SampleMaterialType then 'liquid-other' + when Code 'blood-serum' from SampleMaterialType then 'blood-serum' + when Code 'dna' from SampleMaterialType then 'dna' + when Code 'buffy-coat' from SampleMaterialType then 'buffy-coat' + when Code 'urine' from SampleMaterialType then 'urine' + when Code 'ascites' from SampleMaterialType then 'ascites' + when Code 'saliva' from SampleMaterialType then 'saliva' + when Code 'csf-liquor' from SampleMaterialType then 'csf-liquor' + when Code 'bone-marrow' from SampleMaterialType then 'bone-marrow' + when Code 'peripheral-blood-cells-vital' from SampleMaterialType then 'peripheral-blood-cells-vital' + when Code 'stool-faeces' from SampleMaterialType then 'stool-faeces' + when Code 'rna' from SampleMaterialType then 'rna' + when Code 'whole-blood' from SampleMaterialType then 'whole-blood' + when Code 'swab' from SampleMaterialType then 'swab' + when Code 'dried-whole-blood' from SampleMaterialType then 'dried-whole-blood' + when null then 'Unknown' + else 'Unknown' + end +define Specimen: + if InInitialPopulation then [Specimen] S where (((( (S.type.coding.code contains 'tissue-frozen')) or ( (S.type.coding.code contains 'blood-serum'))))((( (S.type.coding.code contains 'liquid-other')) or ( (S.type.coding.code contains 'rna')) or ( (S.type.coding.code contains 'urine'))) and (((S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding.code contains 'temperatureRoom')) or ((S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding.code contains 'four_degrees'))))) else {} as List + +define Diagnosis: +if InInitialPopulation then [Condition] else {} as List + +define function DiagnosisCode(condition FHIR.Condition): +condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first() + +define function DiagnosisCode(condition FHIR.Condition, specimen FHIR.Specimen): +Coalesce( + condition.code.coding.where(system = 'http://hl7.org/fhir/sid/icd-10').code.first(), + condition.code.coding.where(system = 'http://fhir.de/CodeSystem/dimdi/icd-10-gm').code.first(), + specimen.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code.first(), + condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first() + ) + +define InInitialPopulation: +((((Patient.gender = 'male') or (Patient.gender = 'female')) and ((((exists[Condition: Code 'C41' from icd10]) or (exists[Condition: Code 'C41' from icd10gm])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains 'C41'))) or (((exists[Condition: Code 'C50' from icd10]) or (exists[Condition: Code 'C50' from icd10gm])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains 'C50')))) and (( exists [Specimen: Code 'tissue-frozen' from SampleMaterialType]) or ( exists [Specimen: Code 'blood-serum' from SampleMaterialType]))) or (((Patient.gender = 'male')) and ((((exists[Condition: Code 'C41' from icd10]) or (exists[Condition: Code 'C41' from icd10gm])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains 'C41'))) or (((exists[Condition: Code 'C50' from icd10]) or (exists[Condition: Code 'C50' from icd10gm])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains 'C50')))) and (( exists [Specimen: Code 'liquid-other' from SampleMaterialType]) or ( exists [Specimen: Code 'rna' from SampleMaterialType]) or ( exists [Specimen: Code 'urine' from SampleMaterialType])) and ((exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding contains Code 'temperatureRoom' from StorageTemperature) ) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding contains Code 'four_degrees' from StorageTemperature) )))) \ No newline at end of file diff --git a/src/cql.rs b/src/cql.rs index 6683cbc..a825ba8 100644 --- a/src/cql.rs +++ b/src/cql.rs @@ -4,6 +4,7 @@ use crate::errors::FocusError; use chrono::offset::Utc; use chrono::DateTime; use once_cell::sync::Lazy; +use tracing_subscriber::filter; use std::collections::HashMap; use indexmap::set::IndexSet; @@ -137,9 +138,9 @@ static CQL_SNIPPETS: Lazy> = Lazy: }); pub fn bbmri(ast: ast::Ast) -> Result { - let mut retrieval_criteria: String = "(".to_string(); // main selection criteria (Patient) + let mut retrieval_criteria: String = "".to_string(); // main selection criteria (Patient) - let mut filter_criteria: String = " where (".to_string(); // criteria for filtering specimens + let mut filter_criteria: String = "".to_string(); // criteria for filtering specimens let mut code_systems: IndexSet<&str> = IndexSet::new(); // code lists needed depending on the criteria code_systems.insert("icd10"); //for diagnosis stratifier @@ -169,9 +170,6 @@ pub fn bbmri(ast: ast::Ast) -> Result { } } - retrieval_criteria += ")"; - filter_criteria += ")"; - for code_system in code_systems { lists += format!( "codesystem {}: '{}'\n", @@ -182,21 +180,22 @@ pub fn bbmri(ast: ast::Ast) -> Result { } cql = cql - .replace("{{lists}}", lists.as_str()) - .replace("{{filter_criteria}}", filter_criteria.as_str()); + .replace("{{lists}}", lists.as_str()); - if retrieval_criteria != *"()" { - // no criteria selected - cql = cql.replace("{{retrieval_criteria}}", retrieval_criteria.as_str()); + if retrieval_criteria.is_empty() { + cql = cql.replace("{{retrieval_criteria}}", "true"); //()? } else { - cql = cql.replace("{{retrieval_criteria}}", "true"); + let formatted_retrieval_criteria = format!("({})", retrieval_criteria); + cql = cql.replace("{{retrieval_criteria}}", formatted_retrieval_criteria.as_str()); } - if filter_criteria != *" where ()" { - // no criteria selected - cql = cql.replace("{{retrieval_criteria}}", filter_criteria.as_str()); + + if filter_criteria.is_empty() { + cql = cql.replace("{{filter_criteria}}", ""); } else { - cql = cql.replace("{{retrieval_criteria}}", ""); + let formatted_filter_criteria = format!("where ({})", filter_criteria); + dbg!(formatted_filter_criteria.clone()); + cql = cql.replace("{{filter_criteria}}", formatted_filter_criteria.as_str()); } Ok(cql) @@ -366,6 +365,11 @@ pub fn process( }; retrieval_cond += condition_string.as_str(); + + if !filter_cond.is_empty() && !filter_string.is_empty() { + filter_cond += " and "; + } + filter_cond += filter_string.as_str(); // no condition needed, "" can be added with no change } else { return Err(FocusError::AstUnknownCriterion( @@ -411,9 +415,6 @@ pub fn process( if !filter_cond.is_empty() { dbg!(filter_cond.clone()); - if !filter_criteria.is_empty() { - *filter_criteria += " and "; - } *filter_criteria += "("; *filter_criteria += filter_cond.as_str(); *filter_criteria += ")"; @@ -451,7 +452,7 @@ mod test { r#"{"ast":{"children":[],"operand":"OR"}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; #[test] - fn test_just_print() { + fn test_bbmri() { // println!( // "{:?}", // bbmri(serde_json::from_str(AST).expect("Failed to deserialize JSON")) @@ -496,12 +497,10 @@ mod test { // println!(); - // println!( - // "{:?}", - // bbmri(serde_json::from_str(LENS2).expect("Failed to deserialize JSON")) - // ); - - // println!(); + println!( + "{:?}", + bbmri(serde_json::from_str(LENS2).expect("Failed to deserialize JSON")) + ); // println!( // "{:?}", From 95a10a948dc2be16475a087e857d8a00224f6850 Mon Sep 17 00:00:00 2001 From: Sonat Suer Date: Thu, 7 Dec 2023 16:08:48 +0100 Subject: [PATCH 12/42] Notes of cql geneation --- Cargo.toml | 2 + src/ast.rs | 2 +- src/ast_alternative.rs | 199 +++++++++++++++++++++++++++++++++++++++++ src/main.rs | 1 + 4 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 src/ast_alternative.rs diff --git a/Cargo.toml b/Cargo.toml index 4921e71..f92e30e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,8 @@ indexmap = "2.1.0" tokio = { version = "1.25.0", default_features = false, features = ["signal", "rt-multi-thread", "macros"] } beam-lib = { git = "https://github.com/samply/beam", branch = "develop", features = ["http-util"] } laplace_rs = {version = "0.2.0", git = "https://github.com/samply/laplace-rs.git", branch = "main" } +validated = "0.4.0" +nonempty-collections = "0.1.4" # Logging tracing = { version = "0.1.37", default_features = false } diff --git a/src/ast.rs b/src/ast.rs index 76b8dae..1791a4c 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -78,7 +78,7 @@ mod test { use super::*; const EQUALS_AST: &str = r#"{"ast":{"operand":"AND","children":[{"key":"age","type":"EQUALS","value":5.0}]},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; - + #[test] fn test_deserialize_ast() { diff --git a/src/ast_alternative.rs b/src/ast_alternative.rs new file mode 100644 index 0000000..33149a7 --- /dev/null +++ b/src/ast_alternative.rs @@ -0,0 +1,199 @@ +use validated::Validated::{self, Good, Fail}; +use nonempty_collections::*; + + +// Here are a few design ideas to consider while implementing CQL generation. +// I am not sure how feasible/useful they are as I have a partial understanding +// of the spec and don't really know much about the constraints. Enola asked me to +// push it, so here it goes. + +// Caveat: Did not touch JSON side, I see that as a separate issue. Also +// I was sloppy with the borrow checker, so probably there is room for memory +// footprint optimization. + +// Some general remarks about safer Rust code. +// ------------------------------------------- + +// It is good practice to avoid using naked general purpose types like String. +// In the future we may want to restrict possible key values to, say, alphanumeric +// strings. Representing dates as naked strings is not kosher, either. The general +// idea is pushing preconditions upstream instead of implementing workarounds downstream. + +// As an example, this is how I would define the Date type. Something similar can be done +// for id and key fields which are naked Strings. +mod safe_date { + use chrono::NaiveDateTime; + + pub struct Date(String); + + impl Date { + // Type comes with its validator but strictly speaking this is not necessary + // as we do not process dates. If we start using optics in our Rust code + // we can cast this mechanisms as a prism. + fn new(str: String) -> Option { + match NaiveDateTime::parse_from_str(&str, "%Y-%m-%d") { + Ok(_) => Some(Date(str)), + Err(_) => None, + } + } + + // An un-wrapper + fn to_string(self) { + self.0; + } + + // Serialize/Deserialize would also be here. They can be implemented using new + // and to_string above. Ideally we would also have a roundtrip test and some unit + // tests. If we need to implement too many traits by hand we can use, for instance, + // https://docs.rs/newtype_derive/latest/newtype_derive/ + } +} + + +// Now specific comments on the implementation. +// -------------------------------------------- + +// I changed the name to AstWithId because that's what it is. +pub struct AstWithId { + pub ast: Ast, + pub id: String, // Better be a 'newtype' +} + +// Original AST definition was too complicated. An expression is +// either atomic or built from smaller expressions. No need for indirection. +pub enum Ast { + Atomic(Condition), + Compound(Operation, Vec) // we can disallow empty vectors here but we have sane defaults so it is not a big deal +} + +// Operand is the name of the inputs you give to +// to the operation in an expression. So changed this, too. +#[derive(Clone, Copy)] +pub enum Operation { + And, + Or, +} + +// Having all the operation related things in one place is good. +// CQL support Xor. If we decide to implement it we only change here +// and the rest of the code works. +impl Operation { + fn operation_name(self) -> &'static str { + match self { + Operation::And => " and ", + Operation::Or => " or ", + } + } + + // this is not needed if we disallow empty vectors. some people find + // this counterintuitive so maybe we should? + fn nullary_value(self) -> &'static str { + match self { + Operation::And => "true", //empty iterator returns true under all in Rust + Operation::Or => "false", //empty iterator returns false under any in Rust + } + } + + fn apply_to_group(self, group: Vec<&str>) -> String { + if group.is_empty() { + self.nullary_value().to_string() + } else { + group. // there should be a standard function for this somewhere + iter(). + map(|s| s.to_string()). + enumerate(). + map(|(index, s)| if index < group.len() - 1 {s + self.operation_name()} else {s}). + collect::>(). + concat() + } + } +} + + +// We can use some polymorphism here to avoid code duplication. +// and shine at cocktail parties. +pub struct AbstractRange { + pub min: T, + pub max: T, +} + +pub enum ConditionValue { + DateRange(AbstractRange), + NumberRange(AbstractRange), + Number(f64), + //etc. +} + +pub enum ConditionType { + Equals, + Between, + //etc. +} + +// We can have an enum of condition keys so we can reject unknown keys +// at json parsing level. +pub enum ConditionKey { + Gender, + Diagnosis, + DiagnosisOld, + // etc. +} + +// Condition keys may depend on the project but we can always +// define `pup struct Condition {...}`.. +pub struct Condition { + key: ConditionKey, + type_: ConditionType, + value: ConditionValue +} + +pub struct GeneratedCondition<'a> { + retrieval: &'a str, + filter: &'a str, + code_systems: Vec<&'a str>, // This should probably be a set as we don't want duplicates. +} + +// Specific errors about generation. At this level only incompatibility errors should be left. +// Everything else can be enforced by the type system so they can be pushed to the JSON parsing layer. +pub enum GenerationError { + IncompatibleBlah, + // etc. +} + +// Generating texts from a condition is a standalone operation. Having +// a separated function for this makes hings cleaner. +pub fn generate_condition<'a>(condition: &Condition) -> Result, GenerationError> { + unimplemented!("All the table lookups, compatibility checks etc. should happen here"); +} + +// If we are fine with recursion we can use this. If we want a stack based implementation later +// it would be much easier to refactor. Error handling is streamlined and it is accumulative. +// As a middle solution we can bound the recursion depth, say by ~10. +// Since we will be reporting errors to a UI it is better to collect errors. This is what +// Validated does. +pub fn generate_all<'a>(ast: &Ast) -> Validated, GenerationError> { + match ast { + Ast::Atomic(condition) => + match generate_condition(&condition){ + Ok(generated) => Good(generated), + Err(err) => Fail(nev![err]), + }, + Ast::Compound(op, vec_ast) => + { let recursive_step: Validated, _> = vec_ast.into_iter().map(generate_all).collect(); + match recursive_step { + Good(condition_vec) => { + let retrieval_vec: Vec<&str> = // i extracted the retrieval vec but you can generate all three needed vectors here in a single pass by a fold. + condition_vec. + into_iter(). + map(|g| g.retrieval). + collect(); + Good(GeneratedCondition + { retrieval: &format!("({})", op.apply_to_group(retrieval_vec)) + , filter: todo!("Combine filters") + , code_systems: todo!("Combine code systems") + })}, + Fail(errors) => + Fail(errors), + }}, + } +} diff --git a/src/main.rs b/src/main.rs index 01ace44..5a3c945 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,6 +9,7 @@ mod graceful_shutdown; mod logger; mod omop; mod util; +pub mod ast_alternative; use beam_lib::{TaskRequest, TaskResult}; use laplace_rs::ObfCache; From 175c751142fd96d615a1385321b2358bb74c60c8 Mon Sep 17 00:00:00 2001 From: Sonat Suer Date: Fri, 8 Dec 2023 11:35:46 +0100 Subject: [PATCH 13/42] Move the absence check for condition fields to type-level --- src/ast_alternative.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/ast_alternative.rs b/src/ast_alternative.rs index 33149a7..362fd80 100644 --- a/src/ast_alternative.rs +++ b/src/ast_alternative.rs @@ -148,8 +148,8 @@ pub struct Condition { } pub struct GeneratedCondition<'a> { - retrieval: &'a str, - filter: &'a str, + retrieval: Option<&'a str>, // Keep absence check for retrieval criteria at type level instead of inspecting the String later + filter: Option<&'a str>, // Same as above code_systems: Vec<&'a str>, // This should probably be a set as we don't want duplicates. } @@ -186,9 +186,13 @@ pub fn generate_all<'a>(ast: &Ast) -> Validated, Generati condition_vec. into_iter(). map(|g| g.retrieval). + flatten(). collect(); Good(GeneratedCondition - { retrieval: &format!("({})", op.apply_to_group(retrieval_vec)) + { retrieval: + if retrieval_vec.is_empty() + { None } else + { Some (&format!("({})", op.apply_to_group(retrieval_vec))) } , filter: todo!("Combine filters") , code_systems: todo!("Combine code systems") })}, From 2ff0b8c6552467a7b3937178a9a5324979695a99 Mon Sep 17 00:00:00 2001 From: Sonat Suer Date: Sat, 9 Dec 2023 02:03:52 +0100 Subject: [PATCH 14/42] Generate ts types for cql --- .gitignore | 1 + Cargo.toml | 4 + libs/focus_api/Cargo.toml | 20 ++++ libs/focus_api/src/lib.rs | 184 ++++++++++++++++++++++++++++++++++ src/ast_alternative.rs | 203 -------------------------------------- src/cql_alternative.rs | 51 ++++++++++ src/main.rs | 2 +- 7 files changed, 261 insertions(+), 204 deletions(-) create mode 100644 libs/focus_api/Cargo.toml create mode 100644 libs/focus_api/src/lib.rs delete mode 100644 src/ast_alternative.rs create mode 100644 src/cql_alternative.rs diff --git a/.gitignore b/.gitignore index 4595310..ec466b5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ dev/pki/* !dev/pki/pki Cargo.lock .vscode/* +pkg/ \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index f92e30e..a83bc14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,9 @@ license = "Apache-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[workspace] +members = ["libs/focus_api"] + [dependencies] base64 = { version = "0.21.0", default_features = false } http = "0.2" @@ -21,6 +24,7 @@ beam-lib = { git = "https://github.com/samply/beam", branch = "develop", feature laplace_rs = {version = "0.2.0", git = "https://github.com/samply/laplace-rs.git", branch = "main" } validated = "0.4.0" nonempty-collections = "0.1.4" +focus_api = { version = "0.1.0", path = "libs/focus_api" } # Logging tracing = { version = "0.1.37", default_features = false } diff --git a/libs/focus_api/Cargo.toml b/libs/focus_api/Cargo.toml new file mode 100644 index 0000000..7477291 --- /dev/null +++ b/libs/focus_api/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "focus_api" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +validated = "0.4.0" +nonempty-collections = "0.1.4" +tsify = "0.4.5" +wasm-bindgen = "0.2.89" +serde = { version = "1.0.152", features = ["serde_derive"] } +chrono = "0.4.31" + +[dev-dependencies] +pretty_assertions = "1.4.0" +tokio-test = "0.4.2" diff --git a/libs/focus_api/src/lib.rs b/libs/focus_api/src/lib.rs new file mode 100644 index 0000000..086afdf --- /dev/null +++ b/libs/focus_api/src/lib.rs @@ -0,0 +1,184 @@ +use validated::Validated::{self, Good, Fail}; +use nonempty_collections::*; +use serde::{Deserialize, Serialize}; +use tsify::Tsify; +use wasm_bindgen::prelude::*; + +// Here are a few design ideas to consider while implementing CQL generation. +// I am not sure how feasible/useful they are as I have a partial understanding +// of the spec and don't really know much about the constraints. Enola asked me to +// push it, so here it goes. + +// Caveat: Did not touch JSON side, I see that as a separate issue. Also +// I was sloppy with the borrow checker, so probably there is room for memory +// footprint optimization. + +// Some general remarks about safer Rust code. +// ------------------------------------------- + +// It is good practice to avoid using naked general purpose types like String. +// In the future we may want to restrict possible key values to, say, alphanumeric +// strings. Representing dates as naked strings is not kosher, either. The general +// idea is pushing preconditions upstream instead of implementing workarounds downstream. + +// As an example, this is how I would define the Date type. Something similar can be done +// for id and key fields which are naked Strings. +mod safe_date { + use chrono::NaiveDateTime; + + #[derive(super::Tsify, super::Serialize, super::Deserialize)] + #[tsify(into_wasm_abi, from_wasm_abi)] + pub struct Date(String); + + impl Date { + // Type comes with its validator but strictly speaking this is not necessary + // as we do not process dates. If we start using optics in our Rust code + // we can cast this mechanisms as a prism. + pub fn new(str: String) -> Option { + match NaiveDateTime::parse_from_str(&str, "%Y-%m-%d") { + Ok(_) => Some(Date(str)), + Err(_) => None, + } + } + + // An un-wrapper + pub fn to_string(self) { + self.0; + } + + // Serialize/Deserialize would also be here. They can be implemented using new + // and to_string above. Ideally we would also have a roundtrip test and some unit + // tests. If we need to implement too many traits by hand we can use, for instance, + // https://docs.rs/newtype_derive/latest/newtype_derive/ + } +} + + +// Now specific comments on the implementation. +// -------------------------------------------- + +// I changed the name to AstWithId because that's what it is. +#[derive(Tsify, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +#[tsify(into_wasm_abi, from_wasm_abi)] +pub struct AstWithId { + pub ast: Ast, + pub id: String, // Better be a 'newtype' +} + +// Original AST definition was too complicated. An expression is +// either atomic or built from smaller expressions. No need for indirection. +#[derive(Tsify, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +#[tsify(into_wasm_abi, from_wasm_abi)] +pub enum Ast { + Atomic(Condition), + Compound(Operation, Vec) // we can disallow empty vectors here but we have sane defaults so it is not a big deal +} + +// Operand is the name of the inputs you give to +// to the operation in an expression. So changed this, too. +#[derive(Tsify, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +#[tsify(into_wasm_abi, from_wasm_abi)] +#[derive(Clone, Copy)] +pub enum Operation { + And, + Or, +} + +// Having all the operation related things in one place is good. +// CQL support Xor. If we decide to implement it we only change here +// and the rest of the code works. +impl Operation { + pub fn operation_name(self) -> &'static str { + match self { + Operation::And => " and ", + Operation::Or => " or ", + } + } + + // this is not needed if we disallow empty vectors. some people find + // this counterintuitive so maybe we should? + pub fn nullary_value(self) -> &'static str { + match self { + Operation::And => "true", //empty iterator returns true under all in Rust + Operation::Or => "false", //empty iterator returns false under any in Rust + } + } + + pub fn apply_to_group(self, group: Vec<&str>) -> String { + if group.is_empty() { + self.nullary_value().to_string() + } else { + group. // there should be a standard function for this somewhere + iter(). + map(|s| s.to_string()). + enumerate(). + map(|(index, s)| if index < group.len() - 1 {s + self.operation_name()} else {s}). + collect::>(). + concat() + } + } +} + + +// We can use some polymorphism here to avoid code duplication. +// and shine at cocktail parties. +#[derive(Tsify, Serialize, Deserialize)] +#[tsify(into_wasm_abi, from_wasm_abi)] +pub struct AbstractRange { + pub min: T, + pub max: T, +} + +#[derive(Tsify, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[tsify(into_wasm_abi, from_wasm_abi)] +pub enum ConditionValue { + DateRange(AbstractRange), + NumberRange(AbstractRange), + Number(f64), + //etc. +} + +#[derive(Tsify, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +#[tsify(into_wasm_abi, from_wasm_abi)] +pub enum ConditionType { + Equals, + Between, + //etc. +} + +// We can have an enum of condition keys so we can reject unknown keys +// at json parsing level. +#[derive(Tsify, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +#[tsify(into_wasm_abi, from_wasm_abi)] +pub enum ConditionKey { + Gender, + Diagnosis, + DiagnosisOld, + // etc. +} + +// Condition keys may depend on the project but we can always +// define `pup struct Condition {...}`.. +#[derive(Tsify, Serialize, Deserialize)] +#[tsify(into_wasm_abi, from_wasm_abi)] +pub struct Condition { + pub key: ConditionKey, + pub type_: ConditionType, + pub value: ConditionValue +} + +// Specific errors about generation. At this level only incompatibility errors should be left. +// Everything else can be enforced by the type system so they can be pushed to the JSON parsing layer. +#[derive(Tsify, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +#[tsify(into_wasm_abi, from_wasm_abi)] +pub enum GenerationError { + IncompatibleBlah, + // etc. +} diff --git a/src/ast_alternative.rs b/src/ast_alternative.rs deleted file mode 100644 index 362fd80..0000000 --- a/src/ast_alternative.rs +++ /dev/null @@ -1,203 +0,0 @@ -use validated::Validated::{self, Good, Fail}; -use nonempty_collections::*; - - -// Here are a few design ideas to consider while implementing CQL generation. -// I am not sure how feasible/useful they are as I have a partial understanding -// of the spec and don't really know much about the constraints. Enola asked me to -// push it, so here it goes. - -// Caveat: Did not touch JSON side, I see that as a separate issue. Also -// I was sloppy with the borrow checker, so probably there is room for memory -// footprint optimization. - -// Some general remarks about safer Rust code. -// ------------------------------------------- - -// It is good practice to avoid using naked general purpose types like String. -// In the future we may want to restrict possible key values to, say, alphanumeric -// strings. Representing dates as naked strings is not kosher, either. The general -// idea is pushing preconditions upstream instead of implementing workarounds downstream. - -// As an example, this is how I would define the Date type. Something similar can be done -// for id and key fields which are naked Strings. -mod safe_date { - use chrono::NaiveDateTime; - - pub struct Date(String); - - impl Date { - // Type comes with its validator but strictly speaking this is not necessary - // as we do not process dates. If we start using optics in our Rust code - // we can cast this mechanisms as a prism. - fn new(str: String) -> Option { - match NaiveDateTime::parse_from_str(&str, "%Y-%m-%d") { - Ok(_) => Some(Date(str)), - Err(_) => None, - } - } - - // An un-wrapper - fn to_string(self) { - self.0; - } - - // Serialize/Deserialize would also be here. They can be implemented using new - // and to_string above. Ideally we would also have a roundtrip test and some unit - // tests. If we need to implement too many traits by hand we can use, for instance, - // https://docs.rs/newtype_derive/latest/newtype_derive/ - } -} - - -// Now specific comments on the implementation. -// -------------------------------------------- - -// I changed the name to AstWithId because that's what it is. -pub struct AstWithId { - pub ast: Ast, - pub id: String, // Better be a 'newtype' -} - -// Original AST definition was too complicated. An expression is -// either atomic or built from smaller expressions. No need for indirection. -pub enum Ast { - Atomic(Condition), - Compound(Operation, Vec) // we can disallow empty vectors here but we have sane defaults so it is not a big deal -} - -// Operand is the name of the inputs you give to -// to the operation in an expression. So changed this, too. -#[derive(Clone, Copy)] -pub enum Operation { - And, - Or, -} - -// Having all the operation related things in one place is good. -// CQL support Xor. If we decide to implement it we only change here -// and the rest of the code works. -impl Operation { - fn operation_name(self) -> &'static str { - match self { - Operation::And => " and ", - Operation::Or => " or ", - } - } - - // this is not needed if we disallow empty vectors. some people find - // this counterintuitive so maybe we should? - fn nullary_value(self) -> &'static str { - match self { - Operation::And => "true", //empty iterator returns true under all in Rust - Operation::Or => "false", //empty iterator returns false under any in Rust - } - } - - fn apply_to_group(self, group: Vec<&str>) -> String { - if group.is_empty() { - self.nullary_value().to_string() - } else { - group. // there should be a standard function for this somewhere - iter(). - map(|s| s.to_string()). - enumerate(). - map(|(index, s)| if index < group.len() - 1 {s + self.operation_name()} else {s}). - collect::>(). - concat() - } - } -} - - -// We can use some polymorphism here to avoid code duplication. -// and shine at cocktail parties. -pub struct AbstractRange { - pub min: T, - pub max: T, -} - -pub enum ConditionValue { - DateRange(AbstractRange), - NumberRange(AbstractRange), - Number(f64), - //etc. -} - -pub enum ConditionType { - Equals, - Between, - //etc. -} - -// We can have an enum of condition keys so we can reject unknown keys -// at json parsing level. -pub enum ConditionKey { - Gender, - Diagnosis, - DiagnosisOld, - // etc. -} - -// Condition keys may depend on the project but we can always -// define `pup struct Condition {...}`.. -pub struct Condition { - key: ConditionKey, - type_: ConditionType, - value: ConditionValue -} - -pub struct GeneratedCondition<'a> { - retrieval: Option<&'a str>, // Keep absence check for retrieval criteria at type level instead of inspecting the String later - filter: Option<&'a str>, // Same as above - code_systems: Vec<&'a str>, // This should probably be a set as we don't want duplicates. -} - -// Specific errors about generation. At this level only incompatibility errors should be left. -// Everything else can be enforced by the type system so they can be pushed to the JSON parsing layer. -pub enum GenerationError { - IncompatibleBlah, - // etc. -} - -// Generating texts from a condition is a standalone operation. Having -// a separated function for this makes hings cleaner. -pub fn generate_condition<'a>(condition: &Condition) -> Result, GenerationError> { - unimplemented!("All the table lookups, compatibility checks etc. should happen here"); -} - -// If we are fine with recursion we can use this. If we want a stack based implementation later -// it would be much easier to refactor. Error handling is streamlined and it is accumulative. -// As a middle solution we can bound the recursion depth, say by ~10. -// Since we will be reporting errors to a UI it is better to collect errors. This is what -// Validated does. -pub fn generate_all<'a>(ast: &Ast) -> Validated, GenerationError> { - match ast { - Ast::Atomic(condition) => - match generate_condition(&condition){ - Ok(generated) => Good(generated), - Err(err) => Fail(nev![err]), - }, - Ast::Compound(op, vec_ast) => - { let recursive_step: Validated, _> = vec_ast.into_iter().map(generate_all).collect(); - match recursive_step { - Good(condition_vec) => { - let retrieval_vec: Vec<&str> = // i extracted the retrieval vec but you can generate all three needed vectors here in a single pass by a fold. - condition_vec. - into_iter(). - map(|g| g.retrieval). - flatten(). - collect(); - Good(GeneratedCondition - { retrieval: - if retrieval_vec.is_empty() - { None } else - { Some (&format!("({})", op.apply_to_group(retrieval_vec))) } - , filter: todo!("Combine filters") - , code_systems: todo!("Combine code systems") - })}, - Fail(errors) => - Fail(errors), - }}, - } -} diff --git a/src/cql_alternative.rs b/src/cql_alternative.rs new file mode 100644 index 0000000..e5c3dfe --- /dev/null +++ b/src/cql_alternative.rs @@ -0,0 +1,51 @@ +use validated::Validated::{self, Good, Fail}; +use nonempty_collections::*; +use focus_api::*; + +pub struct GeneratedCondition<'a> { + pub retrieval: Option<&'a str>, // Keep absence check for retrieval criteria at type level instead of inspecting the String later + pub filter: Option<&'a str>, // Same as above + pub code_systems: Vec<&'a str>, // This should probably be a set as we don't want duplicates. +} + +// Generating texts from a condition is a standalone operation. Having +// a separated function for this makes hings cleaner. +pub fn generate_condition<'a>(condition: &Condition) -> Result, GenerationError> { + unimplemented!("All the table lookups, compatibility checks etc. should happen here"); +} + +// If we are fine with recursion we can use this. If we want a stack based implementation later +// it would be much easier to refactor. Error handling is streamlined and it is accumulative. +// As a middle solution we can bound the recursion depth, say by ~10. +// Since we will be reporting errors to a UI it is better to collect errors. This is what +// Validated does. +pub fn generate_all<'a>(ast: &Ast) -> Validated, GenerationError> { + match ast { + Ast::Atomic(condition) => + match generate_condition(&condition){ + Ok(generated) => Good(generated), + Err(err) => Fail(nev![err]), + }, + Ast::Compound(op, vec_ast) => + { let recursive_step: Validated, _> = vec_ast.into_iter().map(generate_all).collect(); + match recursive_step { + Good(condition_vec) => { + let retrieval_vec: Vec<&str> = // i extracted the retrieval vec but you can generate all three needed vectors here in a single pass by a fold. + condition_vec. + into_iter(). + map(|g| g.retrieval). + flatten(). + collect(); + Good(GeneratedCondition + { retrieval: + if retrieval_vec.is_empty() + { None } else + { Some (&format!("({})", op.apply_to_group(retrieval_vec))) } + , filter: todo!("Combine filters") + , code_systems: todo!("Combine code systems") + })}, + Fail(errors) => + Fail(errors), + }}, + } +} diff --git a/src/main.rs b/src/main.rs index 5a3c945..cbdb2db 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,7 +9,7 @@ mod graceful_shutdown; mod logger; mod omop; mod util; -pub mod ast_alternative; +mod cql_alternative; use beam_lib::{TaskRequest, TaskResult}; use laplace_rs::ObfCache; From 9167249c81d8bd9464a76ff632f3e7d3139292d3 Mon Sep 17 00:00:00 2001 From: lablans Date: Thu, 4 Jan 2024 14:33:42 +0000 Subject: [PATCH 15/42] Refactor project-specific code into individual .rs files --- Cargo.toml | 4 ++ src/cql.rs | 155 ++++------------------------------------- src/main.rs | 1 + src/projects/bbmri.rs | 110 +++++++++++++++++++++++++++++ src/projects/common.rs | 47 +++++++++++++ src/projects/dktk.rs | 27 +++++++ src/projects/mod.rs | 115 ++++++++++++++++++++++++++++++ 7 files changed, 316 insertions(+), 143 deletions(-) create mode 100644 src/projects/bbmri.rs create mode 100644 src/projects/common.rs create mode 100644 src/projects/dktk.rs create mode 100644 src/projects/mod.rs diff --git a/Cargo.toml b/Cargo.toml index a83bc14..68ec27a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,10 @@ once_cell = "1.18" # Command Line Interface clap = { version = "4.0", default_features = false, features = ["std", "env", "derive", "help"] } +[features] +default = ["bbmri", "dktk"] +bbmri = [] +dktk = [] [dev-dependencies] pretty_assertions = "1.4.0" diff --git a/src/cql.rs b/src/cql.rs index a825ba8..fcd1560 100644 --- a/src/cql.rs +++ b/src/cql.rs @@ -1,167 +1,36 @@ use crate::ast; use crate::errors::FocusError; +use crate::projects::{Project, CriterionRole, CQL_TEMPLATES, MANDATORY_CODE_SYSTEMS, CODE_LISTS, CQL_SNIPPETS, CRITERION_CODE_LISTS, OBSERVATION_LOINC_CODE}; use chrono::offset::Utc; use chrono::DateTime; -use once_cell::sync::Lazy; -use tracing_subscriber::filter; -use std::collections::HashMap; use indexmap::set::IndexSet; -#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Hash)] -enum CriterionRole { - Query, - Filter, -} - -#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Hash)] -pub enum Project { - Bbmri, - Dktk, -} - -static CODE_LISTS: Lazy> = Lazy::new(|| { - //code lists with their names - [ - ("icd10", "http://hl7.org/fhir/sid/icd-10"), - ("icd10gm", "http://fhir.de/CodeSystem/dimdi/icd-10-gm"), - ("loinc", "http://loinc.org"), - ( - "SampleMaterialType", - "https://fhir.bbmri.de/CodeSystem/SampleMaterialType", - ), - ( - "StorageTemperature", - "https://fhir.bbmri.de/CodeSystem/StorageTemperature", - ), - ( - "FastingStatus", - "http://terminology.hl7.org/CodeSystem/v2-0916", - ), - ( - "SmokingStatus", - "http://hl7.org/fhir/uv/ips/ValueSet/current-smoking-status-uv-ips", - ), - ] - .into() -}); - -static OBSERVATION_LOINC_CODE: Lazy> = Lazy::new(|| { - [ - ("body_weight", "29463-7"), - ("bmi", "39156-5"), - ("smoking_status", "72166-2"), - ] - .into() -}); - -static CRITERION_CODE_LISTS: Lazy>> = Lazy::new(|| { - // code lists needed depending on the criteria selected - [ - (("diagnosis", Project::Bbmri), vec!["icd10", "icd10gm"]), - (("body_weight", Project::Bbmri), vec!["loinc"]), - (("bmi", Project::Bbmri), vec!["loinc"]), - (("smoking_status", Project::Bbmri), vec!["loinc"]), - (("sample_kind", Project::Bbmri), vec!["SampleMaterialType"]), - ( - ("storage_temperature", Project::Bbmri), - vec!["StorageTemperature"], - ), - (("fasting_status", Project::Bbmri), vec!["FastingStatus"]), - ] - .into() -}); - -static CQL_SNIPPETS: Lazy> = Lazy::new(|| { - // CQL snippets depending on the criteria - [ - (("gender", CriterionRole::Query, Project::Bbmri), "Patient.gender = '{{C}}'"), - ( - ("diagnosis", CriterionRole::Query, Project::Bbmri), - "((exists[Condition: Code '{{C}}' from {{A1}}]) or (exists[Condition: Code '{{C}}' from {{A2}}])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains '{{C}}'))", - ), - (("diagnosis_old", CriterionRole::Query, Project::Bbmri), " exists [Condition: Code '{{C}}' from {{A1}}]"), - ( - ("date_of_diagnosis", CriterionRole::Query, Project::Bbmri), - "exists from [Condition] C\nwhere FHIRHelpers.ToDateTime(C.onset) between {{D1}} and {{D2}}", - ), - ( - ("diagnosis_age_donor", CriterionRole::Query, Project::Bbmri), - "exists from [Condition] C\nwhere AgeInYearsAt(FHIRHelpers.ToDateTime(C.onset)) between Ceiling({{D1}}) and Ceiling({{D2}})", - ), - (("donor_age", CriterionRole::Query, Project::Bbmri), " AgeInYears() between Ceiling({{D1}}) and Ceiling({{D2}})"), - ( - ("observationRange", CriterionRole::Query, Project::Bbmri), - "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere O.value between {{D1}} and {{D2}}", - ), - ( - ("body_weight", CriterionRole::Query, Project::Bbmri), - "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere ((O.value as Quantity) < {{D1}} 'kg' and (O.value as Quantity) > {{D2}} 'kg')", - ), - ( - ("bmi", CriterionRole::Query, Project::Bbmri), - "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere ((O.value as Quantity) < {{D1}} 'kg/m2' and (O.value as Quantity) > {{D2}} 'kg/m2')", - ), - (("sample_kind", CriterionRole::Query, Project::Bbmri), " exists [Specimen: Code '{{C}}' from {{A1}}]"), - (("sample_kind", CriterionRole::Filter, Project::Bbmri), " (S.type.coding.code contains '{{C}}')"), - - ( - ("storage_temperature", CriterionRole::Filter, Project::Bbmri), - "(S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding.code contains '{{C}}')", - ), - ( - ("sampling_date", CriterionRole::Filter, Project::Bbmri), - "(FHIRHelpers.ToDateTime(S.collection.collected) between {{D1}} and {{D2}}) ", - ), - ( - ("fasting_status", CriterionRole::Filter, Project::Bbmri), - "(S.collection.fastingStatus.coding.code contains '{{C}}') ", - ), - ( - ("sampling_date", CriterionRole::Query, Project::Bbmri), - "exists from [Specimen] S\nwhere FHIRHelpers.ToDateTime(S.collection.collected) between {{D1}} and {{D2}} ", - ), - ( - ("fasting_status", CriterionRole::Query, Project::Bbmri), - "exists from [Specimen] S\nwhere S.collection.fastingStatus.coding.code contains '{{C}}' ", - ), - ( - ("storage_temperature", CriterionRole::Query, Project::Bbmri), - "exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding contains Code '{{C}}' from {{A1}}) ", - ), - ( - ("smoking_status", CriterionRole::Query, Project::Bbmri), - "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere O.value.coding.code contains '{{C}}' ", - ), - ] - .into() -}); - -pub fn bbmri(ast: ast::Ast) -> Result { +pub fn generate_cql(ast: ast::Ast, project: Project) -> Result { let mut retrieval_criteria: String = "".to_string(); // main selection criteria (Patient) let mut filter_criteria: String = "".to_string(); // criteria for filtering specimens - let mut code_systems: IndexSet<&str> = IndexSet::new(); // code lists needed depending on the criteria - code_systems.insert("icd10"); //for diagnosis stratifier - code_systems.insert("SampleMaterialType"); //for sample type stratifier - let mut lists: String = "".to_string(); // needed code lists, defined - let mut cql: String = include_str!("../resources/template_bbmri.cql").to_string(); + let mut cql = CQL_TEMPLATES.get(&project).expect("missing project").to_string(); let operator_str = match ast.ast.operand { ast::Operand::And => " and ", ast::Operand::Or => " or ", }; + let mut mandatory_codes = MANDATORY_CODE_SYSTEMS.get(&project) + .expect("non-existent project") + .clone(); + for (index, grandchild) in ast.ast.children.iter().enumerate() { process( grandchild.clone(), &mut retrieval_criteria, &mut filter_criteria, - &mut code_systems, - Project::Bbmri, + &mut mandatory_codes, + project, )?; // Only concatenate operator if it's not the last element @@ -170,7 +39,7 @@ pub fn bbmri(ast: ast::Ast) -> Result { } } - for code_system in code_systems { + for code_system in mandatory_codes.iter() { lists += format!( "codesystem {}: '{}'\n", code_system, @@ -499,7 +368,7 @@ mod test { println!( "{:?}", - bbmri(serde_json::from_str(LENS2).expect("Failed to deserialize JSON")) + generate_cql(serde_json::from_str(LENS2).expect("Failed to deserialize JSON"), Project::Bbmri) ); // println!( @@ -507,7 +376,7 @@ mod test { // bbmri(serde_json::from_str(EMPTY).expect("Failed to deserialize JSON")) // ); - pretty_assertions::assert_eq!(bbmri(serde_json::from_str(EMPTY).unwrap()).unwrap(), include_str!("../resources/test/result_empty.cql").to_string()); + pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(EMPTY).unwrap(), Project::Bbmri).unwrap(), include_str!("../resources/test/result_empty.cql").to_string()); } } diff --git a/src/main.rs b/src/main.rs index cbdb2db..c92b2b9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,6 +10,7 @@ mod logger; mod omop; mod util; mod cql_alternative; +mod projects; use beam_lib::{TaskRequest, TaskResult}; use laplace_rs::ObfCache; diff --git a/src/projects/bbmri.rs b/src/projects/bbmri.rs new file mode 100644 index 0000000..b18d403 --- /dev/null +++ b/src/projects/bbmri.rs @@ -0,0 +1,110 @@ +use std::collections::HashMap; + +use indexmap::IndexSet; + +use super::{Project, CriterionRole}; + +const PROJECT: Project = Project::Bbmri; + +pub fn append_code_lists(_map: &mut HashMap<&'static str, &'static str>) { } + +pub fn append_observation_loinc_codes(_map: &mut HashMap<&'static str, &'static str>) { } + +pub fn append_criterion_code_lists(map: &mut HashMap<(&str, Project), Vec<&str>>) { + for (key, value) in + [ + ("diagnosis", vec!["icd10", "icd10gm"]), + ("body_weight", vec!["loinc"]), + ("bmi", vec!["loinc"]), + ("smoking_status", vec!["loinc"]), + ("sample_kind", vec!["SampleMaterialType"]), + ("storage_temperature", vec!["StorageTemperature"]), + ("fasting_status", vec!["FastingStatus"]), + ] { + map.insert( + (key, PROJECT), + value + ); + } +} + +pub fn append_cql_snippets(map: &mut HashMap<(&str, CriterionRole, Project), &str>) { + for (key, value) in + [ + (("gender", CriterionRole::Query), "Patient.gender = '{{C}}'"), + ( + ("diagnosis", CriterionRole::Query), + "((exists[Condition: Code '{{C}}' from {{A1}}]) or (exists[Condition: Code '{{C}}' from {{A2}}])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains '{{C}}'))", + ), + (("diagnosis_old", CriterionRole::Query), " exists [Condition: Code '{{C}}' from {{A1}}]"), + ( + ("date_of_diagnosis", CriterionRole::Query), + "exists from [Condition] C\nwhere FHIRHelpers.ToDateTime(C.onset) between {{D1}} and {{D2}}", + ), + ( + ("diagnosis_age_donor", CriterionRole::Query), + "exists from [Condition] C\nwhere AgeInYearsAt(FHIRHelpers.ToDateTime(C.onset)) between Ceiling({{D1}}) and Ceiling({{D2}})", + ), + (("donor_age", CriterionRole::Query), " AgeInYears() between Ceiling({{D1}}) and Ceiling({{D2}})"), + ( + ("observationRange", CriterionRole::Query), + "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere O.value between {{D1}} and {{D2}}", + ), + ( + ("body_weight", CriterionRole::Query), + "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere ((O.value as Quantity) < {{D1}} 'kg' and (O.value as Quantity) > {{D2}} 'kg')", + ), + ( + ("bmi", CriterionRole::Query), + "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere ((O.value as Quantity) < {{D1}} 'kg/m2' and (O.value as Quantity) > {{D2}} 'kg/m2')", + ), + (("sample_kind", CriterionRole::Query), " exists [Specimen: Code '{{C}}' from {{A1}}]"), + (("sample_kind", CriterionRole::Filter), " (S.type.coding.code contains '{{C}}')"), + + ( + ("storage_temperature", CriterionRole::Filter), + "(S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding.code contains '{{C}}')", + ), + ( + ("sampling_date", CriterionRole::Filter), + "(FHIRHelpers.ToDateTime(S.collection.collected) between {{D1}} and {{D2}}) ", + ), + ( + ("fasting_status", CriterionRole::Filter), + "(S.collection.fastingStatus.coding.code contains '{{C}}') ", + ), + ( + ("sampling_date", CriterionRole::Query), + "exists from [Specimen] S\nwhere FHIRHelpers.ToDateTime(S.collection.collected) between {{D1}} and {{D2}} ", + ), + ( + ("fasting_status", CriterionRole::Query), + "exists from [Specimen] S\nwhere S.collection.fastingStatus.coding.code contains '{{C}}' ", + ), + ( + ("storage_temperature", CriterionRole::Query), + "exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding contains Code '{{C}}' from {{A1}}) ", + ), + ( + ("smoking_status", CriterionRole::Query), + "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere O.value.coding.code contains '{{C}}' ", + ), + ] { + map.insert( + (key.0, key.1, PROJECT), + value + ); + } +} + +pub fn append_mandatory_code_lists(map: &mut HashMap>) { + let mut set = map.remove(&PROJECT).unwrap_or(IndexSet::new()); + for value in ["icd10", "SampleMaterialType"] { + set.insert(value); + } + map.insert(PROJECT, set); +} + +pub(crate) fn append_cql_templates(map: &mut HashMap) { + map.insert(PROJECT, include_str!("../../resources/template_bbmri.cql")); +} \ No newline at end of file diff --git a/src/projects/common.rs b/src/projects/common.rs new file mode 100644 index 0000000..13100b2 --- /dev/null +++ b/src/projects/common.rs @@ -0,0 +1,47 @@ +use std::collections::HashMap; + +use indexmap::IndexSet; + +use super::{Project, CriterionRole}; + +pub fn append_code_lists(map: &mut HashMap<&'static str, &'static str>) { + map.extend( + [ + ("icd10", "http://hl7.org/fhir/sid/icd-10"), + ("icd10gm", "http://fhir.de/CodeSystem/dimdi/icd-10-gm"), + ("loinc", "http://loinc.org"), + ( + "SampleMaterialType", + "https://fhir.bbmri.de/CodeSystem/SampleMaterialType", + ), + ( + "StorageTemperature", + "https://fhir.bbmri.de/CodeSystem/StorageTemperature", + ), + ( + "FastingStatus", + "http://terminology.hl7.org/CodeSystem/v2-0916", + ), + ( + "SmokingStatus", + "http://hl7.org/fhir/uv/ips/ValueSet/current-smoking-status-uv-ips", + ), + ]); +} + +pub fn append_observation_loinc_codes(map: &mut HashMap<&'static str, &'static str>) { + map.extend( + [ + ("body_weight", "29463-7"), + ("bmi", "39156-5"), + ("smoking_status", "72166-2"), + ]); +} + +pub fn append_criterion_code_lists(map: &mut HashMap<(&str, Project), Vec<&str>>) { } + +pub fn append_cql_snippets(map: &mut HashMap<(&str, CriterionRole, Project), &str>) { } + +pub fn append_mandatory_code_lists(map: &mut HashMap>) { } + +pub(crate) fn append_cql_templates(map: &mut HashMap) { } \ No newline at end of file diff --git a/src/projects/dktk.rs b/src/projects/dktk.rs new file mode 100644 index 0000000..0dc819f --- /dev/null +++ b/src/projects/dktk.rs @@ -0,0 +1,27 @@ +use std::collections::HashMap; + +use indexmap::IndexSet; + +use super::{Project, CriterionRole}; + +const PROJECT: Project = Project::Dktk; + +pub fn append_code_lists(_map: &mut HashMap<&'static str, &'static str>) { } + +pub fn append_observation_loinc_codes(_map: &mut HashMap<&'static str, &'static str>) { } + +pub fn append_criterion_code_lists(_map: &mut HashMap<(&str, Project), Vec<&str>>) { } + +pub fn append_cql_snippets(_map: &mut HashMap<(&str, CriterionRole, Project), &str>) { } + +pub fn append_mandatory_code_lists(map: &mut HashMap>) { + let mut set = map.remove(&PROJECT).unwrap_or(IndexSet::new()); + for value in ["icd10", "SampleMaterialType", "loinc"] { + set.insert(value); + } + map.insert(PROJECT, set); +} + +pub(crate) fn append_cql_templates(map: &mut HashMap) { + //map.insert(PROJECT, include_str!("../../resources/template_dktk.cql")); +} \ No newline at end of file diff --git a/src/projects/mod.rs b/src/projects/mod.rs new file mode 100644 index 0000000..34da35d --- /dev/null +++ b/src/projects/mod.rs @@ -0,0 +1,115 @@ +use std::{collections::HashMap, fmt::Display}; + +use indexmap::IndexSet; +use once_cell::sync::Lazy; + +mod common; +#[cfg(feature="bbmri")] +mod bbmri; +#[cfg(feature="dktk")] +mod dktk; + +#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Hash, Copy)] +pub enum Project { + #[cfg(feature="bbmri")] + Bbmri, + #[cfg(feature="dktk")] + Dktk, +} + +impl Display for Project { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let name = match self { + Project::Bbmri => "bbmri", + Project::Dktk => "dktk", + }; + write!(f, "{name}") + } +} + +#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Hash)] +pub enum CriterionRole { + Query, + Filter, +} + +// code lists with their names +pub static CODE_LISTS: Lazy> = Lazy::new(|| { + let mut map: HashMap<&'static str, &'static str> = HashMap::new(); + common::append_code_lists(&mut map); + + #[cfg(feature="bbmri")] + bbmri::append_code_lists(&mut map); + + #[cfg(feature="dktk")] + dktk::append_code_lists(&mut map); + + map +}); + +pub static OBSERVATION_LOINC_CODE: Lazy> = Lazy::new(|| { + let mut map: HashMap<&'static str, &'static str> = HashMap::new(); + common::append_observation_loinc_codes(&mut map); + + #[cfg(feature="bbmri")] + bbmri::append_observation_loinc_codes(&mut map); + + #[cfg(feature="dktk")] + dktk::append_observation_loinc_codes(&mut map); + + map +}); + +// code lists needed depending on the criteria selected +pub static CRITERION_CODE_LISTS: Lazy>> = Lazy::new(|| { + let mut map = HashMap::new(); + common::append_criterion_code_lists(&mut map); + + #[cfg(feature="bbmri")] + bbmri::append_criterion_code_lists(&mut map); + + #[cfg(feature="dktk")] + dktk::append_criterion_code_lists(&mut map); + + map +}); + +// CQL snippets depending on the criteria +pub static CQL_SNIPPETS: Lazy> = Lazy::new(|| { + let mut map = HashMap::new(); + common::append_cql_snippets(&mut map); + + #[cfg(feature="bbmri")] + bbmri::append_cql_snippets(&mut map); + + #[cfg(feature="dktk")] + dktk::append_cql_snippets(&mut map); + + map +}); + +pub static MANDATORY_CODE_SYSTEMS: Lazy>> = Lazy::new(|| { + let mut map = HashMap::new(); + common::append_mandatory_code_lists(&mut map); + + #[cfg(feature="bbmri")] + bbmri::append_mandatory_code_lists(&mut map); + + #[cfg(feature="dktk")] + dktk::append_mandatory_code_lists(&mut map); + + map +}); + +pub static CQL_TEMPLATES: Lazy> = Lazy::new(|| { + let mut map = HashMap::new(); + common::append_cql_templates(&mut map); + + #[cfg(feature="bbmri")] + bbmri::append_cql_templates(&mut map); + + #[cfg(feature="dktk")] + dktk::append_cql_templates(&mut map); + + map +}); From 5587bc6df908b4f309bb9e80a48c7bb511755032 Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Wed, 3 Apr 2024 13:45:08 +0200 Subject: [PATCH 16/42] prism age stratifier --- ...RI_STRAT_AGE_STRATIFIER => PRISM_STRAT_AGE_STRATIFIER_BBMRI} | 0 src/util.rs | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename resources/cql/{PRISM_BBMRI_STRAT_AGE_STRATIFIER => PRISM_STRAT_AGE_STRATIFIER_BBMRI} (100%) diff --git a/resources/cql/PRISM_BBMRI_STRAT_AGE_STRATIFIER b/resources/cql/PRISM_STRAT_AGE_STRATIFIER_BBMRI similarity index 100% rename from resources/cql/PRISM_BBMRI_STRAT_AGE_STRATIFIER rename to resources/cql/PRISM_STRAT_AGE_STRATIFIER_BBMRI diff --git a/src/util.rs b/src/util.rs index 83ab838..a7d3e0a 100644 --- a/src/util.rs +++ b/src/util.rs @@ -260,7 +260,7 @@ fn obfuscate_counts_recursive( obfuscate_below_10_mode: ObfuscateBelow10Mode, rounding_step: usize, ) -> Result<(), FocusError> { - let mut rng = thread_rng(); + let mut rng = thread_rng();// TODO evict match val { Value::Object(map) => { if let Some(count_val) = map.get_mut("count") { From 391fafd7b40bf669e1b2d6c10baa3f39426bd316 Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Thu, 25 Apr 2024 11:07:18 +0200 Subject: [PATCH 17/42] new ICD10GM --- resources/template_bbmri.cql | 2 +- src/projects/bbmri.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/template_bbmri.cql b/resources/template_bbmri.cql index 322aa36..e664efb 100644 --- a/resources/template_bbmri.cql +++ b/resources/template_bbmri.cql @@ -75,8 +75,8 @@ define function DiagnosisCode(condition FHIR.Condition, specimen FHIR.Specimen): Coalesce( condition.code.coding.where(system = 'http://hl7.org/fhir/sid/icd-10').code.first(), condition.code.coding.where(system = 'http://fhir.de/CodeSystem/dimdi/icd-10-gm').code.first(), - specimen.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code.first(), condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first() + specimen.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code.first(), ) define InInitialPopulation: diff --git a/src/projects/bbmri.rs b/src/projects/bbmri.rs index b18d403..7f84373 100644 --- a/src/projects/bbmri.rs +++ b/src/projects/bbmri.rs @@ -34,7 +34,7 @@ pub fn append_cql_snippets(map: &mut HashMap<(&str, CriterionRole, Project), &st (("gender", CriterionRole::Query), "Patient.gender = '{{C}}'"), ( ("diagnosis", CriterionRole::Query), - "((exists[Condition: Code '{{C}}' from {{A1}}]) or (exists[Condition: Code '{{C}}' from {{A2}}])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains '{{C}}'))", + "((exists[Condition: Code '{{C}}' from {{A1}}]) or (exists[Condition: Code '{{C}}' from {{A2}}]) or (exists[Condition: Code '{{C}}' from {{A3}}])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains '{{C}}'))", ), (("diagnosis_old", CriterionRole::Query), " exists [Condition: Code '{{C}}' from {{A1}}]"), ( From 1b956b33f1d41de9980dc7ca8d52af9cb0284478 Mon Sep 17 00:00:00 2001 From: lablans Date: Mon, 6 May 2024 12:59:37 +0000 Subject: [PATCH 18/42] Refactor projects into subdirs --- .github/workflows/rust.yml | 2 +- Cargo.toml | 2 +- src/cql.rs | 20 +-- src/projects/bbmri.rs | 110 ---------------- src/projects/bbmri/mod.rs | 122 ++++++++++++++++++ .../projects/bbmri/template.cql | 0 src/projects/common.rs | 47 ------- src/projects/dktk.rs | 27 ---- src/projects/dktk/mod.rs | 35 +++++ src/projects/mod.rs | 79 ++++++------ src/projects/shared/mod.rs | 64 +++++++++ 11 files changed, 277 insertions(+), 231 deletions(-) delete mode 100644 src/projects/bbmri.rs create mode 100644 src/projects/bbmri/mod.rs rename resources/template_bbmri.cql => src/projects/bbmri/template.cql (100%) delete mode 100644 src/projects/common.rs delete mode 100644 src/projects/dktk.rs create mode 100644 src/projects/dktk/mod.rs create mode 100644 src/projects/shared/mod.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a464e43..be8db06 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -17,7 +17,7 @@ jobs: #architectures: '[ "amd64", "arm64" ]' #profile: debug test-via-script: false - #features: '[ "" ]' + features: '[ "bbmri", "dktk" ]' push-to: ${{ (github.ref_protected == true || github.event_name == 'workflow_dispatch') && 'dockerhub' || 'ghcr' }} secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} diff --git a/Cargo.toml b/Cargo.toml index 17bf333..010f2ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,7 @@ once_cell = "1.18" clap = { version = "4.0", default_features = false, features = ["std", "env", "derive", "help"] } [features] -default = ["bbmri", "dktk"] +default = [] bbmri = [] dktk = [] diff --git a/src/cql.rs b/src/cql.rs index fcd1560..35d3483 100644 --- a/src/cql.rs +++ b/src/cql.rs @@ -6,21 +6,21 @@ use chrono::offset::Utc; use chrono::DateTime; use indexmap::set::IndexSet; -pub fn generate_cql(ast: ast::Ast, project: Project) -> Result { +pub fn generate_cql(ast: ast::Ast, project: impl Project) -> Result { let mut retrieval_criteria: String = "".to_string(); // main selection criteria (Patient) let mut filter_criteria: String = "".to_string(); // criteria for filtering specimens let mut lists: String = "".to_string(); // needed code lists, defined - let mut cql = CQL_TEMPLATES.get(&project).expect("missing project").to_string(); + let mut cql = CQL_TEMPLATES.get(project.name()).expect("missing project").to_string(); let operator_str = match ast.ast.operand { ast::Operand::And => " and ", ast::Operand::Or => " or ", }; - let mut mandatory_codes = MANDATORY_CODE_SYSTEMS.get(&project) + let mut mandatory_codes = MANDATORY_CODE_SYSTEMS.get(project.name()) .expect("non-existent project") .clone(); @@ -75,7 +75,7 @@ pub fn process( retrieval_criteria: &mut String, filter_criteria: &mut String, code_systems: &mut IndexSet<&str>, - project: Project, + project: impl Project, ) -> Result<(), FocusError> { let mut retrieval_cond: String = "(".to_string(); let mut filter_cond: String = "".to_string(); @@ -85,7 +85,7 @@ pub fn process( let condition_key_trans = condition.key.as_str(); let condition_snippet = - CQL_SNIPPETS.get(&(condition_key_trans, CriterionRole::Query, project.clone())); + CQL_SNIPPETS.get(&(condition_key_trans, CriterionRole::Query, project.name())); if let Some(snippet) = condition_snippet { let mut condition_string = (*snippet).to_string(); @@ -94,10 +94,10 @@ pub fn process( let filter_snippet = CQL_SNIPPETS.get(&( condition_key_trans, CriterionRole::Filter, - project.clone(), + project.name(), )); - let code_lists_option = CRITERION_CODE_LISTS.get(&(condition_key_trans, project)); + let code_lists_option = CRITERION_CODE_LISTS.get(&(condition_key_trans, project.name())); if let Some(code_lists_vec) = code_lists_option { for (index, code_list) in code_lists_vec.iter().enumerate() { code_systems.insert(code_list); @@ -320,8 +320,10 @@ mod test { const EMPTY: &str = r#"{"ast":{"children":[],"operand":"OR"}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + #[cfg(feature = "bbmri")] #[test] fn test_bbmri() { + use crate::projects::bbmri::Bbmri; // println!( // "{:?}", // bbmri(serde_json::from_str(AST).expect("Failed to deserialize JSON")) @@ -368,7 +370,7 @@ mod test { println!( "{:?}", - generate_cql(serde_json::from_str(LENS2).expect("Failed to deserialize JSON"), Project::Bbmri) + generate_cql(serde_json::from_str(LENS2).expect("Failed to deserialize JSON"), Bbmri) ); // println!( @@ -376,7 +378,7 @@ mod test { // bbmri(serde_json::from_str(EMPTY).expect("Failed to deserialize JSON")) // ); - pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(EMPTY).unwrap(), Project::Bbmri).unwrap(), include_str!("../resources/test/result_empty.cql").to_string()); + pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(EMPTY).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_empty.cql").to_string()); } } diff --git a/src/projects/bbmri.rs b/src/projects/bbmri.rs deleted file mode 100644 index 7f84373..0000000 --- a/src/projects/bbmri.rs +++ /dev/null @@ -1,110 +0,0 @@ -use std::collections::HashMap; - -use indexmap::IndexSet; - -use super::{Project, CriterionRole}; - -const PROJECT: Project = Project::Bbmri; - -pub fn append_code_lists(_map: &mut HashMap<&'static str, &'static str>) { } - -pub fn append_observation_loinc_codes(_map: &mut HashMap<&'static str, &'static str>) { } - -pub fn append_criterion_code_lists(map: &mut HashMap<(&str, Project), Vec<&str>>) { - for (key, value) in - [ - ("diagnosis", vec!["icd10", "icd10gm"]), - ("body_weight", vec!["loinc"]), - ("bmi", vec!["loinc"]), - ("smoking_status", vec!["loinc"]), - ("sample_kind", vec!["SampleMaterialType"]), - ("storage_temperature", vec!["StorageTemperature"]), - ("fasting_status", vec!["FastingStatus"]), - ] { - map.insert( - (key, PROJECT), - value - ); - } -} - -pub fn append_cql_snippets(map: &mut HashMap<(&str, CriterionRole, Project), &str>) { - for (key, value) in - [ - (("gender", CriterionRole::Query), "Patient.gender = '{{C}}'"), - ( - ("diagnosis", CriterionRole::Query), - "((exists[Condition: Code '{{C}}' from {{A1}}]) or (exists[Condition: Code '{{C}}' from {{A2}}]) or (exists[Condition: Code '{{C}}' from {{A3}}])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains '{{C}}'))", - ), - (("diagnosis_old", CriterionRole::Query), " exists [Condition: Code '{{C}}' from {{A1}}]"), - ( - ("date_of_diagnosis", CriterionRole::Query), - "exists from [Condition] C\nwhere FHIRHelpers.ToDateTime(C.onset) between {{D1}} and {{D2}}", - ), - ( - ("diagnosis_age_donor", CriterionRole::Query), - "exists from [Condition] C\nwhere AgeInYearsAt(FHIRHelpers.ToDateTime(C.onset)) between Ceiling({{D1}}) and Ceiling({{D2}})", - ), - (("donor_age", CriterionRole::Query), " AgeInYears() between Ceiling({{D1}}) and Ceiling({{D2}})"), - ( - ("observationRange", CriterionRole::Query), - "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere O.value between {{D1}} and {{D2}}", - ), - ( - ("body_weight", CriterionRole::Query), - "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere ((O.value as Quantity) < {{D1}} 'kg' and (O.value as Quantity) > {{D2}} 'kg')", - ), - ( - ("bmi", CriterionRole::Query), - "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere ((O.value as Quantity) < {{D1}} 'kg/m2' and (O.value as Quantity) > {{D2}} 'kg/m2')", - ), - (("sample_kind", CriterionRole::Query), " exists [Specimen: Code '{{C}}' from {{A1}}]"), - (("sample_kind", CriterionRole::Filter), " (S.type.coding.code contains '{{C}}')"), - - ( - ("storage_temperature", CriterionRole::Filter), - "(S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding.code contains '{{C}}')", - ), - ( - ("sampling_date", CriterionRole::Filter), - "(FHIRHelpers.ToDateTime(S.collection.collected) between {{D1}} and {{D2}}) ", - ), - ( - ("fasting_status", CriterionRole::Filter), - "(S.collection.fastingStatus.coding.code contains '{{C}}') ", - ), - ( - ("sampling_date", CriterionRole::Query), - "exists from [Specimen] S\nwhere FHIRHelpers.ToDateTime(S.collection.collected) between {{D1}} and {{D2}} ", - ), - ( - ("fasting_status", CriterionRole::Query), - "exists from [Specimen] S\nwhere S.collection.fastingStatus.coding.code contains '{{C}}' ", - ), - ( - ("storage_temperature", CriterionRole::Query), - "exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding contains Code '{{C}}' from {{A1}}) ", - ), - ( - ("smoking_status", CriterionRole::Query), - "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere O.value.coding.code contains '{{C}}' ", - ), - ] { - map.insert( - (key.0, key.1, PROJECT), - value - ); - } -} - -pub fn append_mandatory_code_lists(map: &mut HashMap>) { - let mut set = map.remove(&PROJECT).unwrap_or(IndexSet::new()); - for value in ["icd10", "SampleMaterialType"] { - set.insert(value); - } - map.insert(PROJECT, set); -} - -pub(crate) fn append_cql_templates(map: &mut HashMap) { - map.insert(PROJECT, include_str!("../../resources/template_bbmri.cql")); -} \ No newline at end of file diff --git a/src/projects/bbmri/mod.rs b/src/projects/bbmri/mod.rs new file mode 100644 index 0000000..79a4d12 --- /dev/null +++ b/src/projects/bbmri/mod.rs @@ -0,0 +1,122 @@ +use std::collections::HashMap; + +use indexmap::IndexSet; + +use super::{CriterionRole, Project, ProjectName}; + +#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Hash)] +pub(crate) struct Bbmri; + +// TODO: Include entries from shared +impl Project for Bbmri { + fn append_code_lists(&self, _map: &mut HashMap<&'static str, &'static str>) { + // none + } + + fn append_observation_loinc_codes(&self, _map: &mut HashMap<&'static str, &'static str>) { + // none + } + + fn append_criterion_code_lists(&self, map: &mut HashMap<(&str, &ProjectName), Vec<&str>>) { + for (key, value) in + [ + ("diagnosis", vec!["icd10", "icd10gm"]), + ("body_weight", vec!["loinc"]), + ("bmi", vec!["loinc"]), + ("smoking_status", vec!["loinc"]), + ("sample_kind", vec!["SampleMaterialType"]), + ("storage_temperature", vec!["StorageTemperature"]), + ("fasting_status", vec!["FastingStatus"]), + ] { + map.insert( + (key, self.name()), + value + ); + } + } + + fn append_cql_snippets(&self, map: &mut HashMap<(&str, CriterionRole, &ProjectName), &str>) { + for (key, value) in + [ + (("gender", CriterionRole::Query), "Patient.gender = '{{C}}'"), + ( + ("diagnosis", CriterionRole::Query), + "((exists[Condition: Code '{{C}}' from {{A1}}]) or (exists[Condition: Code '{{C}}' from {{A2}}]) or (exists[Condition: Code '{{C}}' from {{A3}}])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains '{{C}}'))", + ), + (("diagnosis_old", CriterionRole::Query), " exists [Condition: Code '{{C}}' from {{A1}}]"), + ( + ("date_of_diagnosis", CriterionRole::Query), + "exists from [Condition] C\nwhere FHIRHelpers.ToDateTime(C.onset) between {{D1}} and {{D2}}", + ), + ( + ("diagnosis_age_donor", CriterionRole::Query), + "exists from [Condition] C\nwhere AgeInYearsAt(FHIRHelpers.ToDateTime(C.onset)) between Ceiling({{D1}}) and Ceiling({{D2}})", + ), + (("donor_age", CriterionRole::Query), " AgeInYears() between Ceiling({{D1}}) and Ceiling({{D2}})"), + ( + ("observationRange", CriterionRole::Query), + "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere O.value between {{D1}} and {{D2}}", + ), + ( + ("body_weight", CriterionRole::Query), + "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere ((O.value as Quantity) < {{D1}} 'kg' and (O.value as Quantity) > {{D2}} 'kg')", + ), + ( + ("bmi", CriterionRole::Query), + "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere ((O.value as Quantity) < {{D1}} 'kg/m2' and (O.value as Quantity) > {{D2}} 'kg/m2')", + ), + (("sample_kind", CriterionRole::Query), " exists [Specimen: Code '{{C}}' from {{A1}}]"), + (("sample_kind", CriterionRole::Filter), " (S.type.coding.code contains '{{C}}')"), + + ( + ("storage_temperature", CriterionRole::Filter), + "(S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding.code contains '{{C}}')", + ), + ( + ("sampling_date", CriterionRole::Filter), + "(FHIRHelpers.ToDateTime(S.collection.collected) between {{D1}} and {{D2}}) ", + ), + ( + ("fasting_status", CriterionRole::Filter), + "(S.collection.fastingStatus.coding.code contains '{{C}}') ", + ), + ( + ("sampling_date", CriterionRole::Query), + "exists from [Specimen] S\nwhere FHIRHelpers.ToDateTime(S.collection.collected) between {{D1}} and {{D2}} ", + ), + ( + ("fasting_status", CriterionRole::Query), + "exists from [Specimen] S\nwhere S.collection.fastingStatus.coding.code contains '{{C}}' ", + ), + ( + ("storage_temperature", CriterionRole::Query), + "exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding contains Code '{{C}}' from {{A1}}) ", + ), + ( + ("smoking_status", CriterionRole::Query), + "exists from [Observation: Code '{{K}}' from {{A1}}] O\nwhere O.value.coding.code contains '{{C}}' ", + ), + ] { + map.insert( + (key.0, key.1, self.name()), + value + ); + } + } + + fn append_mandatory_code_lists(&self, map: &mut HashMap<&ProjectName, IndexSet<&str>>) { + let mut set = map.remove(self.name()).unwrap_or(IndexSet::new()); + for value in ["icd10", "SampleMaterialType"] { + set.insert(value); + } + map.insert(self.name(), set); + } + + fn append_cql_templates(&self, map: &mut HashMap<&ProjectName, &str>) { + map.insert(self.name(), include_str!("template.cql")); + } + + fn name(&self) -> &'static ProjectName { + &ProjectName::Bbmri + } +} \ No newline at end of file diff --git a/resources/template_bbmri.cql b/src/projects/bbmri/template.cql similarity index 100% rename from resources/template_bbmri.cql rename to src/projects/bbmri/template.cql diff --git a/src/projects/common.rs b/src/projects/common.rs deleted file mode 100644 index 13100b2..0000000 --- a/src/projects/common.rs +++ /dev/null @@ -1,47 +0,0 @@ -use std::collections::HashMap; - -use indexmap::IndexSet; - -use super::{Project, CriterionRole}; - -pub fn append_code_lists(map: &mut HashMap<&'static str, &'static str>) { - map.extend( - [ - ("icd10", "http://hl7.org/fhir/sid/icd-10"), - ("icd10gm", "http://fhir.de/CodeSystem/dimdi/icd-10-gm"), - ("loinc", "http://loinc.org"), - ( - "SampleMaterialType", - "https://fhir.bbmri.de/CodeSystem/SampleMaterialType", - ), - ( - "StorageTemperature", - "https://fhir.bbmri.de/CodeSystem/StorageTemperature", - ), - ( - "FastingStatus", - "http://terminology.hl7.org/CodeSystem/v2-0916", - ), - ( - "SmokingStatus", - "http://hl7.org/fhir/uv/ips/ValueSet/current-smoking-status-uv-ips", - ), - ]); -} - -pub fn append_observation_loinc_codes(map: &mut HashMap<&'static str, &'static str>) { - map.extend( - [ - ("body_weight", "29463-7"), - ("bmi", "39156-5"), - ("smoking_status", "72166-2"), - ]); -} - -pub fn append_criterion_code_lists(map: &mut HashMap<(&str, Project), Vec<&str>>) { } - -pub fn append_cql_snippets(map: &mut HashMap<(&str, CriterionRole, Project), &str>) { } - -pub fn append_mandatory_code_lists(map: &mut HashMap>) { } - -pub(crate) fn append_cql_templates(map: &mut HashMap) { } \ No newline at end of file diff --git a/src/projects/dktk.rs b/src/projects/dktk.rs deleted file mode 100644 index 0dc819f..0000000 --- a/src/projects/dktk.rs +++ /dev/null @@ -1,27 +0,0 @@ -use std::collections::HashMap; - -use indexmap::IndexSet; - -use super::{Project, CriterionRole}; - -const PROJECT: Project = Project::Dktk; - -pub fn append_code_lists(_map: &mut HashMap<&'static str, &'static str>) { } - -pub fn append_observation_loinc_codes(_map: &mut HashMap<&'static str, &'static str>) { } - -pub fn append_criterion_code_lists(_map: &mut HashMap<(&str, Project), Vec<&str>>) { } - -pub fn append_cql_snippets(_map: &mut HashMap<(&str, CriterionRole, Project), &str>) { } - -pub fn append_mandatory_code_lists(map: &mut HashMap>) { - let mut set = map.remove(&PROJECT).unwrap_or(IndexSet::new()); - for value in ["icd10", "SampleMaterialType", "loinc"] { - set.insert(value); - } - map.insert(PROJECT, set); -} - -pub(crate) fn append_cql_templates(map: &mut HashMap) { - //map.insert(PROJECT, include_str!("../../resources/template_dktk.cql")); -} \ No newline at end of file diff --git a/src/projects/dktk/mod.rs b/src/projects/dktk/mod.rs new file mode 100644 index 0000000..fd6e052 --- /dev/null +++ b/src/projects/dktk/mod.rs @@ -0,0 +1,35 @@ +use std::collections::HashMap; + +use indexmap::IndexSet; + +use super::{CriterionRole, Project, ProjectName}; + +#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Hash)] +pub(crate) struct Dktk; + +// TODO: Include entries from shared +impl Project for Dktk { + fn append_code_lists(&self, _map: &mut HashMap<&'static str, &'static str>) { } + + fn append_observation_loinc_codes(&self, _map: &mut HashMap<&'static str, &'static str>) { } + + fn append_criterion_code_lists(&self, _map: &mut HashMap<(&str, &ProjectName), Vec<&str>>) { } + + fn append_cql_snippets(&self, _map: &mut HashMap<(&str, CriterionRole, &ProjectName), &str>) { } + + fn append_mandatory_code_lists(&self, map: &mut HashMap<&ProjectName, IndexSet<&str>>) { + let mut set = map.remove(self.name()).unwrap_or(IndexSet::new()); + for value in ["icd10", "SampleMaterialType", "loinc"] { + set.insert(value); + } + map.insert(self.name(), set); + } + + fn append_cql_templates(&self, map: &mut HashMap<&ProjectName, &str>) { + //map.insert(&Self, include_str!("template.cql")); + } + + fn name(&self) -> &'static ProjectName { + &ProjectName::Dktk + } +} \ No newline at end of file diff --git a/src/projects/mod.rs b/src/projects/mod.rs index 34da35d..08a7e4b 100644 --- a/src/projects/mod.rs +++ b/src/projects/mod.rs @@ -1,31 +1,44 @@ -use std::{collections::HashMap, fmt::Display}; +use std::{collections::HashMap, hash::Hash}; use indexmap::IndexSet; use once_cell::sync::Lazy; -mod common; +mod shared; + #[cfg(feature="bbmri")] -mod bbmri; +pub(crate) mod bbmri; + #[cfg(feature="dktk")] -mod dktk; +pub(crate) mod dktk; + +pub(crate) trait Project: PartialEq + Eq + PartialOrd + Ord + Clone + Copy + Hash { + fn append_code_lists(&self, _map: &mut HashMap<&'static str, &'static str>); + fn append_observation_loinc_codes(&self, _map: &mut HashMap<&'static str, &'static str>); + fn append_criterion_code_lists(&self, _map: &mut HashMap<(&str, &ProjectName), Vec<&str>>); + fn append_cql_snippets(&self, _map: &mut HashMap<(&str, CriterionRole, &ProjectName), &str>); + fn append_mandatory_code_lists(&self, map: &mut HashMap<&ProjectName, IndexSet<&str>>); + fn append_cql_templates(&self, map: &mut HashMap<&ProjectName, &str>); + fn name(&self) -> &'static ProjectName; +} #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Hash, Copy)] -pub enum Project { +pub enum ProjectName { #[cfg(feature="bbmri")] Bbmri, #[cfg(feature="dktk")] Dktk, + NotSpecified } -impl Display for Project { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let name = match self { - Project::Bbmri => "bbmri", - Project::Dktk => "dktk", - }; - write!(f, "{name}") - } -} +// impl Display for ProjectName { +// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +// let name = match self { +// ProjectName::Bbmri => "bbmri", +// ProjectName::Dktk => "dktk" +// }; +// write!(f, "{name}") +// } +// } #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Hash)] pub enum CriterionRole { @@ -36,80 +49,74 @@ pub enum CriterionRole { // code lists with their names pub static CODE_LISTS: Lazy> = Lazy::new(|| { let mut map: HashMap<&'static str, &'static str> = HashMap::new(); - common::append_code_lists(&mut map); #[cfg(feature="bbmri")] - bbmri::append_code_lists(&mut map); + bbmri::Bbmri.append_code_lists(&mut map); #[cfg(feature="dktk")] - dktk::append_code_lists(&mut map); + dktk::Dktk.append_code_lists(&mut map); map }); pub static OBSERVATION_LOINC_CODE: Lazy> = Lazy::new(|| { let mut map: HashMap<&'static str, &'static str> = HashMap::new(); - common::append_observation_loinc_codes(&mut map); #[cfg(feature="bbmri")] - bbmri::append_observation_loinc_codes(&mut map); + bbmri::Bbmri.append_observation_loinc_codes(&mut map); #[cfg(feature="dktk")] - dktk::append_observation_loinc_codes(&mut map); + dktk::Dktk.append_observation_loinc_codes(&mut map); map }); // code lists needed depending on the criteria selected -pub static CRITERION_CODE_LISTS: Lazy>> = Lazy::new(|| { +pub static CRITERION_CODE_LISTS: Lazy>> = Lazy::new(|| { let mut map = HashMap::new(); - common::append_criterion_code_lists(&mut map); #[cfg(feature="bbmri")] - bbmri::append_criterion_code_lists(&mut map); + bbmri::Bbmri.append_criterion_code_lists(&mut map); #[cfg(feature="dktk")] - dktk::append_criterion_code_lists(&mut map); + dktk::Dktk.append_criterion_code_lists(&mut map); map }); // CQL snippets depending on the criteria -pub static CQL_SNIPPETS: Lazy> = Lazy::new(|| { +pub static CQL_SNIPPETS: Lazy> = Lazy::new(|| { let mut map = HashMap::new(); - common::append_cql_snippets(&mut map); #[cfg(feature="bbmri")] - bbmri::append_cql_snippets(&mut map); + bbmri::Bbmri.append_cql_snippets(&mut map); #[cfg(feature="dktk")] - dktk::append_cql_snippets(&mut map); + dktk::Dktk.append_cql_snippets(&mut map); map }); -pub static MANDATORY_CODE_SYSTEMS: Lazy>> = Lazy::new(|| { +pub static MANDATORY_CODE_SYSTEMS: Lazy>> = Lazy::new(|| { let mut map = HashMap::new(); - common::append_mandatory_code_lists(&mut map); #[cfg(feature="bbmri")] - bbmri::append_mandatory_code_lists(&mut map); + bbmri::Bbmri.append_mandatory_code_lists(&mut map); #[cfg(feature="dktk")] - dktk::append_mandatory_code_lists(&mut map); + dktk::Dktk.append_mandatory_code_lists(&mut map); map }); -pub static CQL_TEMPLATES: Lazy> = Lazy::new(|| { +pub static CQL_TEMPLATES: Lazy> = Lazy::new(|| { let mut map = HashMap::new(); - common::append_cql_templates(&mut map); #[cfg(feature="bbmri")] - bbmri::append_cql_templates(&mut map); + bbmri::Bbmri.append_cql_templates(&mut map); #[cfg(feature="dktk")] - dktk::append_cql_templates(&mut map); + dktk::Dktk.append_cql_templates(&mut map); map }); diff --git a/src/projects/shared/mod.rs b/src/projects/shared/mod.rs new file mode 100644 index 0000000..8423a93 --- /dev/null +++ b/src/projects/shared/mod.rs @@ -0,0 +1,64 @@ +use std::collections::HashMap; + +use indexmap::IndexSet; + +use super::{CriterionRole, Project, ProjectName}; + +#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Hash)] +pub(crate) struct Shared; + +impl Project for Shared { + fn append_code_lists(&self, map: &mut HashMap<&'static str, &'static str>) { + map.extend( + [ + ("icd10", "http://hl7.org/fhir/sid/icd-10"), + ("icd10gm", "http://fhir.de/CodeSystem/dimdi/icd-10-gm"), + ("loinc", "http://loinc.org"), + ( + "SampleMaterialType", + "https://fhir.bbmri.de/CodeSystem/SampleMaterialType", + ), + ( + "StorageTemperature", + "https://fhir.bbmri.de/CodeSystem/StorageTemperature", + ), + ( + "FastingStatus", + "http://terminology.hl7.org/CodeSystem/v2-0916", + ), + ( + "SmokingStatus", + "http://hl7.org/fhir/uv/ips/ValueSet/current-smoking-status-uv-ips", + ), + ]); + } + + fn append_observation_loinc_codes(&self, map: &mut HashMap<&'static str, &'static str>) { + map.extend( + [ + ("body_weight", "29463-7"), + ("bmi", "39156-5"), + ("smoking_status", "72166-2"), + ]); + } + + fn append_criterion_code_lists(&self, _map: &mut HashMap<(&str, &ProjectName), Vec<&str>>) { + // none + } + + fn append_cql_snippets(&self, _map: &mut HashMap<(&str, CriterionRole, &ProjectName), &str>) { + // none + } + + fn append_mandatory_code_lists(&self, _map: &mut HashMap<&ProjectName, IndexSet<&str>>) { + // none + } + + fn append_cql_templates(&self, _map: &mut HashMap<&ProjectName, &str>) { + // none + } + + fn name(&self) -> &'static ProjectName { + &ProjectName::NotSpecified + } +} \ No newline at end of file From 666e345317587f80370684f9851d9dbc9e0f34b7 Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Mon, 6 May 2024 18:02:10 +0200 Subject: [PATCH 19/42] everything I guess --- .github/workflows/rust.yml | 2 +- Cargo.toml | 2 +- input/tests/results/result.txt | 0 .../cql/BBMRI_STRAT_DIAGNOSIS_STRATIFIER | 2 +- resources/template_bbmri.cql | 4 +- resources/test/query_bbmri.cql | 2 +- resources/test/result_ast.cql | 85 ++++ resources/test/result_empty.cql | 4 +- resources/test/result_lens2.cql | 11 +- resources/test/result_male_or_female.cql | 85 ++++ src/ast.rs | 2 +- src/cql.rs | 90 +++- src/cql_new.rs | 410 ++++++++++++++++++ src/main.rs | 1 + src/projects/bbmri.rs | 21 +- src/projects/common.rs | 3 + src/projects/dktk.rs | 2 + src/projects/mod.rs | 19 + 18 files changed, 712 insertions(+), 33 deletions(-) delete mode 100644 input/tests/results/result.txt create mode 100644 resources/test/result_ast.cql create mode 100644 resources/test/result_male_or_female.cql create mode 100644 src/cql_new.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a464e43..be8db06 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -17,7 +17,7 @@ jobs: #architectures: '[ "amd64", "arm64" ]' #profile: debug test-via-script: false - #features: '[ "" ]' + features: '[ "bbmri", "dktk" ]' push-to: ${{ (github.ref_protected == true || github.event_name == 'workflow_dispatch') && 'dockerhub' || 'ghcr' }} secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} diff --git a/Cargo.toml b/Cargo.toml index 17bf333..010f2ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,7 @@ once_cell = "1.18" clap = { version = "4.0", default_features = false, features = ["std", "env", "derive", "help"] } [features] -default = ["bbmri", "dktk"] +default = [] bbmri = [] dktk = [] diff --git a/input/tests/results/result.txt b/input/tests/results/result.txt deleted file mode 100644 index e69de29..0000000 diff --git a/resources/cql/BBMRI_STRAT_DIAGNOSIS_STRATIFIER b/resources/cql/BBMRI_STRAT_DIAGNOSIS_STRATIFIER index 92e9d20..61fc693 100644 --- a/resources/cql/BBMRI_STRAT_DIAGNOSIS_STRATIFIER +++ b/resources/cql/BBMRI_STRAT_DIAGNOSIS_STRATIFIER @@ -4,5 +4,5 @@ define Diagnosis: Coalesce(condition.code.coding.where(system = 'http://hl7.org/fhir/sid/icd-10').code.first(), condition.code.coding.where(system = 'http://fhir.de/CodeSystem/dimdi/icd-10-gm').code.first(), condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first(), - specimen.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code.first(), condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first()) + specimen.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code.first()) diff --git a/resources/template_bbmri.cql b/resources/template_bbmri.cql index e664efb..e5f5f4e 100644 --- a/resources/template_bbmri.cql +++ b/resources/template_bbmri.cql @@ -75,8 +75,8 @@ define function DiagnosisCode(condition FHIR.Condition, specimen FHIR.Specimen): Coalesce( condition.code.coding.where(system = 'http://hl7.org/fhir/sid/icd-10').code.first(), condition.code.coding.where(system = 'http://fhir.de/CodeSystem/dimdi/icd-10-gm').code.first(), - condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first() - specimen.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code.first(), + condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first(), + specimen.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code.first() ) define InInitialPopulation: diff --git a/resources/test/query_bbmri.cql b/resources/test/query_bbmri.cql index b2550c4..90c0986 100644 --- a/resources/test/query_bbmri.cql +++ b/resources/test/query_bbmri.cql @@ -82,7 +82,7 @@ define Diagnosis: Coalesce(condition.code.coding.where(system = 'http://hl7.org/fhir/sid/icd-10').code.first(), condition.code.coding.where(system = 'http://fhir.de/CodeSystem/dimdi/icd-10-gm').code.first(), condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first(), - specimen.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code.first(), condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first()) + specimen.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code.first()) diff --git a/resources/test/result_ast.cql b/resources/test/result_ast.cql new file mode 100644 index 0000000..54d8c01 --- /dev/null +++ b/resources/test/result_ast.cql @@ -0,0 +1,85 @@ +library Retrieve +using FHIR version '4.0.0' +include FHIRHelpers version '4.0.0' + +codesystem icd10: 'http://hl7.org/fhir/sid/icd-10' +codesystem SampleMaterialType: 'https://fhir.bbmri.de/CodeSystem/SampleMaterialType' + + +context Patient + +define AgeClass: +if (Patient.birthDate is null) then 'unknown' else ToString((AgeInYears() div 10) * 10) + +define Gender: +if (Patient.gender is null) then 'unknown' else Patient.gender + +define Custodian: + First(from Specimen.extension E + where E.url = 'https://fhir.bbmri.de/StructureDefinition/Custodian' + return (E.value as Reference).identifier.value) + +define function SampleType(specimen FHIR.Specimen): + case FHIRHelpers.ToCode(specimen.type.coding.where(system = 'https://fhir.bbmri.de/CodeSystem/SampleMaterialType').first()) + when Code 'plasma-edta' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-citrat' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-heparin' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-cell-free' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-other' from SampleMaterialType then 'blood-plasma' + when Code 'plasma' from SampleMaterialType then 'blood-plasma' + when Code 'tissue-formalin' from SampleMaterialType then 'tissue-ffpe' + when Code 'tumor-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'normal-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'other-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'tumor-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'normal-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'other-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'tissue-paxgene-or-else' from SampleMaterialType then 'tissue-other' + when Code 'derivative' from SampleMaterialType then 'derivative-other' + when Code 'liquid' from SampleMaterialType then 'liquid-other' + when Code 'tissue' from SampleMaterialType then 'tissue-other' + when Code 'serum' from SampleMaterialType then 'blood-serum' + when Code 'cf-dna' from SampleMaterialType then 'dna' + when Code 'g-dna' from SampleMaterialType then 'dna' + when Code 'blood-plasma' from SampleMaterialType then 'blood-plasma' + when Code 'tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'tissue-other' from SampleMaterialType then 'tissue-other' + when Code 'derivative-other' from SampleMaterialType then 'derivative-other' + when Code 'liquid-other' from SampleMaterialType then 'liquid-other' + when Code 'blood-serum' from SampleMaterialType then 'blood-serum' + when Code 'dna' from SampleMaterialType then 'dna' + when Code 'buffy-coat' from SampleMaterialType then 'buffy-coat' + when Code 'urine' from SampleMaterialType then 'urine' + when Code 'ascites' from SampleMaterialType then 'ascites' + when Code 'saliva' from SampleMaterialType then 'saliva' + when Code 'csf-liquor' from SampleMaterialType then 'csf-liquor' + when Code 'bone-marrow' from SampleMaterialType then 'bone-marrow' + when Code 'peripheral-blood-cells-vital' from SampleMaterialType then 'peripheral-blood-cells-vital' + when Code 'stool-faeces' from SampleMaterialType then 'stool-faeces' + when Code 'rna' from SampleMaterialType then 'rna' + when Code 'whole-blood' from SampleMaterialType then 'whole-blood' + when Code 'swab' from SampleMaterialType then 'swab' + when Code 'dried-whole-blood' from SampleMaterialType then 'dried-whole-blood' + when null then 'Unknown' + else 'Unknown' + end +define Specimen: + if InInitialPopulation then [Specimen] S else {} as List + +define Diagnosis: +if InInitialPopulation then [Condition] else {} as List + +define function DiagnosisCode(condition FHIR.Condition): +condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first() + +define function DiagnosisCode(condition FHIR.Condition, specimen FHIR.Specimen): +Coalesce( + condition.code.coding.where(system = 'http://hl7.org/fhir/sid/icd-10').code.first(), + condition.code.coding.where(system = 'http://fhir.de/CodeSystem/dimdi/icd-10-gm').code.first(), + condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first(), + specimen.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code.first() + ) + +define InInitialPopulation: +true \ No newline at end of file diff --git a/resources/test/result_empty.cql b/resources/test/result_empty.cql index 735c0ad..54d8c01 100644 --- a/resources/test/result_empty.cql +++ b/resources/test/result_empty.cql @@ -77,8 +77,8 @@ define function DiagnosisCode(condition FHIR.Condition, specimen FHIR.Specimen): Coalesce( condition.code.coding.where(system = 'http://hl7.org/fhir/sid/icd-10').code.first(), condition.code.coding.where(system = 'http://fhir.de/CodeSystem/dimdi/icd-10-gm').code.first(), - specimen.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code.first(), - condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first() + condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first(), + specimen.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code.first() ) define InInitialPopulation: diff --git a/resources/test/result_lens2.cql b/resources/test/result_lens2.cql index 7733570..83ceede 100644 --- a/resources/test/result_lens2.cql +++ b/resources/test/result_lens2.cql @@ -5,6 +5,7 @@ include FHIRHelpers version '4.0.0' codesystem icd10: 'http://hl7.org/fhir/sid/icd-10' codesystem SampleMaterialType: 'https://fhir.bbmri.de/CodeSystem/SampleMaterialType' codesystem icd10gm: 'http://fhir.de/CodeSystem/dimdi/icd-10-gm' +codesystem icd10gmnew: 'http://fhir.de/CodeSystem/bfarm/icd-10-gm' codesystem StorageTemperature: 'https://fhir.bbmri.de/CodeSystem/StorageTemperature' @@ -67,7 +68,7 @@ define function SampleType(specimen FHIR.Specimen): else 'Unknown' end define Specimen: - if InInitialPopulation then [Specimen] S where (((( (S.type.coding.code contains 'tissue-frozen')) or ( (S.type.coding.code contains 'blood-serum'))))((( (S.type.coding.code contains 'liquid-other')) or ( (S.type.coding.code contains 'rna')) or ( (S.type.coding.code contains 'urine'))) and (((S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding.code contains 'temperatureRoom')) or ((S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding.code contains 'four_degrees'))))) else {} as List + if InInitialPopulation then [Specimen] S where (((((( (S.type.coding.code contains 'tissue-frozen')) or ( (S.type.coding.code contains 'tumor-tissue-frozen')) or ( (S.type.coding.code contains 'normal-tissue-frozen')) or ( (S.type.coding.code contains 'other-tissue-frozen')))) or ((( (S.type.coding.code contains 'blood-serum')) or ( (S.type.coding.code contains 'serum')))))) or ((((( (S.type.coding.code contains 'liquid-other')) or ( (S.type.coding.code contains 'liquid')))) or ((( (S.type.coding.code contains 'rna')))) or ((( (S.type.coding.code contains 'urine'))))) and (((((S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding.code contains 'temperatureRoom')))) or ((((S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding.code contains 'four_degrees'))))))) else {} as List define Diagnosis: if InInitialPopulation then [Condition] else {} as List @@ -77,11 +78,11 @@ condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm' define function DiagnosisCode(condition FHIR.Condition, specimen FHIR.Specimen): Coalesce( - condition.code.coding.where(system = 'http://hl7.org/fhir/sid/icd-10').code.first(), + condition.code.coding.where(system = 'http://hl7.org/fhir/sid/icd-10').code.first(), condition.code.coding.where(system = 'http://fhir.de/CodeSystem/dimdi/icd-10-gm').code.first(), - specimen.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code.first(), - condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first() + condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first(), + specimen.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code.first() ) define InInitialPopulation: -((((Patient.gender = 'male') or (Patient.gender = 'female')) and ((((exists[Condition: Code 'C41' from icd10]) or (exists[Condition: Code 'C41' from icd10gm])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains 'C41'))) or (((exists[Condition: Code 'C50' from icd10]) or (exists[Condition: Code 'C50' from icd10gm])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains 'C50')))) and (( exists [Specimen: Code 'tissue-frozen' from SampleMaterialType]) or ( exists [Specimen: Code 'blood-serum' from SampleMaterialType]))) or (((Patient.gender = 'male')) and ((((exists[Condition: Code 'C41' from icd10]) or (exists[Condition: Code 'C41' from icd10gm])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains 'C41'))) or (((exists[Condition: Code 'C50' from icd10]) or (exists[Condition: Code 'C50' from icd10gm])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains 'C50')))) and (( exists [Specimen: Code 'liquid-other' from SampleMaterialType]) or ( exists [Specimen: Code 'rna' from SampleMaterialType]) or ( exists [Specimen: Code 'urine' from SampleMaterialType])) and ((exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding contains Code 'temperatureRoom' from StorageTemperature) ) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding contains Code 'four_degrees' from StorageTemperature) )))) \ No newline at end of file +((((((Patient.gender = 'male'))) or (((Patient.gender = 'female')))) and ((((((exists[Condition: Code 'C41' from icd10]) or (exists[Condition: Code 'C41' from icd10gm]) or (exists[Condition: Code 'C41' from icd10gmnew])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains 'C41'))))) or (((((exists[Condition: Code 'C50' from icd10]) or (exists[Condition: Code 'C50' from icd10gm]) or (exists[Condition: Code 'C50' from icd10gmnew])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains 'C50')))))) and (((( exists [Specimen: Code 'tissue-frozen' from SampleMaterialType]) or ( exists [Specimen: Code 'tumor-tissue-frozen' from SampleMaterialType]) or ( exists [Specimen: Code 'normal-tissue-frozen' from SampleMaterialType]) or ( exists [Specimen: Code 'other-tissue-frozen' from SampleMaterialType]))) or ((( exists [Specimen: Code 'blood-serum' from SampleMaterialType]) or ( exists [Specimen: Code 'serum' from SampleMaterialType]))))) or (((((Patient.gender = 'male')))) and ((((((exists[Condition: Code 'C41' from icd10]) or (exists[Condition: Code 'C41' from icd10gm]) or (exists[Condition: Code 'C41' from icd10gmnew])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains 'C41'))))) or (((((exists[Condition: Code 'C50' from icd10]) or (exists[Condition: Code 'C50' from icd10gm]) or (exists[Condition: Code 'C50' from icd10gmnew])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains 'C50')))))) and (((( exists [Specimen: Code 'liquid-other' from SampleMaterialType]) or ( exists [Specimen: Code 'liquid' from SampleMaterialType]))) or ((( exists [Specimen: Code 'rna' from SampleMaterialType]))) or ((( exists [Specimen: Code 'urine' from SampleMaterialType])))) and ((((exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding contains Code 'temperatureRoom' from StorageTemperature) ))) or (((exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding contains Code 'four_degrees' from StorageTemperature) )))))) \ No newline at end of file diff --git a/resources/test/result_male_or_female.cql b/resources/test/result_male_or_female.cql new file mode 100644 index 0000000..54d8c01 --- /dev/null +++ b/resources/test/result_male_or_female.cql @@ -0,0 +1,85 @@ +library Retrieve +using FHIR version '4.0.0' +include FHIRHelpers version '4.0.0' + +codesystem icd10: 'http://hl7.org/fhir/sid/icd-10' +codesystem SampleMaterialType: 'https://fhir.bbmri.de/CodeSystem/SampleMaterialType' + + +context Patient + +define AgeClass: +if (Patient.birthDate is null) then 'unknown' else ToString((AgeInYears() div 10) * 10) + +define Gender: +if (Patient.gender is null) then 'unknown' else Patient.gender + +define Custodian: + First(from Specimen.extension E + where E.url = 'https://fhir.bbmri.de/StructureDefinition/Custodian' + return (E.value as Reference).identifier.value) + +define function SampleType(specimen FHIR.Specimen): + case FHIRHelpers.ToCode(specimen.type.coding.where(system = 'https://fhir.bbmri.de/CodeSystem/SampleMaterialType').first()) + when Code 'plasma-edta' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-citrat' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-heparin' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-cell-free' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-other' from SampleMaterialType then 'blood-plasma' + when Code 'plasma' from SampleMaterialType then 'blood-plasma' + when Code 'tissue-formalin' from SampleMaterialType then 'tissue-ffpe' + when Code 'tumor-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'normal-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'other-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'tumor-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'normal-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'other-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'tissue-paxgene-or-else' from SampleMaterialType then 'tissue-other' + when Code 'derivative' from SampleMaterialType then 'derivative-other' + when Code 'liquid' from SampleMaterialType then 'liquid-other' + when Code 'tissue' from SampleMaterialType then 'tissue-other' + when Code 'serum' from SampleMaterialType then 'blood-serum' + when Code 'cf-dna' from SampleMaterialType then 'dna' + when Code 'g-dna' from SampleMaterialType then 'dna' + when Code 'blood-plasma' from SampleMaterialType then 'blood-plasma' + when Code 'tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'tissue-other' from SampleMaterialType then 'tissue-other' + when Code 'derivative-other' from SampleMaterialType then 'derivative-other' + when Code 'liquid-other' from SampleMaterialType then 'liquid-other' + when Code 'blood-serum' from SampleMaterialType then 'blood-serum' + when Code 'dna' from SampleMaterialType then 'dna' + when Code 'buffy-coat' from SampleMaterialType then 'buffy-coat' + when Code 'urine' from SampleMaterialType then 'urine' + when Code 'ascites' from SampleMaterialType then 'ascites' + when Code 'saliva' from SampleMaterialType then 'saliva' + when Code 'csf-liquor' from SampleMaterialType then 'csf-liquor' + when Code 'bone-marrow' from SampleMaterialType then 'bone-marrow' + when Code 'peripheral-blood-cells-vital' from SampleMaterialType then 'peripheral-blood-cells-vital' + when Code 'stool-faeces' from SampleMaterialType then 'stool-faeces' + when Code 'rna' from SampleMaterialType then 'rna' + when Code 'whole-blood' from SampleMaterialType then 'whole-blood' + when Code 'swab' from SampleMaterialType then 'swab' + when Code 'dried-whole-blood' from SampleMaterialType then 'dried-whole-blood' + when null then 'Unknown' + else 'Unknown' + end +define Specimen: + if InInitialPopulation then [Specimen] S else {} as List + +define Diagnosis: +if InInitialPopulation then [Condition] else {} as List + +define function DiagnosisCode(condition FHIR.Condition): +condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first() + +define function DiagnosisCode(condition FHIR.Condition, specimen FHIR.Specimen): +Coalesce( + condition.code.coding.where(system = 'http://hl7.org/fhir/sid/icd-10').code.first(), + condition.code.coding.where(system = 'http://fhir.de/CodeSystem/dimdi/icd-10-gm').code.first(), + condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first(), + specimen.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code.first() + ) + +define InInitialPopulation: +true \ No newline at end of file diff --git a/src/ast.rs b/src/ast.rs index 1791a4c..e53ff58 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -11,7 +11,7 @@ pub enum Child { #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "UPPERCASE")] -pub enum Operand { +pub enum Operand { //this is operator, of course, but rename would need to be coordinated with all the Lenses, EUCAIM providers, etc And, Or, } diff --git a/src/cql.rs b/src/cql.rs index fcd1560..f92988b 100644 --- a/src/cql.rs +++ b/src/cql.rs @@ -1,6 +1,6 @@ use crate::ast; use crate::errors::FocusError; -use crate::projects::{Project, CriterionRole, CQL_TEMPLATES, MANDATORY_CODE_SYSTEMS, CODE_LISTS, CQL_SNIPPETS, CRITERION_CODE_LISTS, OBSERVATION_LOINC_CODE}; +use crate::projects::{Project, CriterionRole, CQL_TEMPLATES, MANDATORY_CODE_SYSTEMS, CODE_LISTS, CQL_SNIPPETS, CRITERION_CODE_LISTS, OBSERVATION_LOINC_CODE, SAMPLE_TYPE_WORKAROUNDS}; use chrono::offset::Utc; use chrono::DateTime; @@ -63,7 +63,7 @@ pub fn generate_cql(ast: ast::Ast, project: Project) -> Result { + let mut string_array_with_workarounds = string_array.clone(); + for value in string_array { + if let Some(additional_values) = SAMPLE_TYPE_WORKAROUNDS.get(value.as_str()){ + for additional_value in additional_values { + string_array_with_workarounds.push((*additional_value).into()); + } + } + } let mut condition_humongous_string = "(".to_string(); let mut filter_humongous_string = "(".to_string(); - for (index, string) in string_array.iter().enumerate() { + for (index, string) in string_array_with_workarounds.iter().enumerate() { condition_humongous_string = condition_humongous_string + "(" + condition_string.as_str() @@ -200,7 +208,7 @@ pub fn process( filter_humongous_string.replace("{{C}}", string.as_str()); // Only concatenate operator if it's not the last element - if index < string_array.len() - 1 { + if index < string_array_with_workarounds.len() - 1 { condition_humongous_string += operator_str; filter_humongous_string += operator_str; } @@ -218,8 +226,44 @@ pub fn process( } // this becomes or of all ast::ConditionType::Equals => match condition.value { ast::ConditionValue::String(string) => { - condition_string = condition_string.replace("{{C}}", string.as_str()); - filter_string = filter_string.replace("{{C}}", string.as_str()); // no condition needed, "" stays "" + let operator_str = " or "; + let mut string_array_with_workarounds = vec![string.clone()]; + if let Some(additional_values) = SAMPLE_TYPE_WORKAROUNDS.get(string.as_str()){ + for additional_value in additional_values { + string_array_with_workarounds.push((*additional_value).into()); + } + } + //condition_string = condition_string.replace("{{C}}", string.as_str()); + //filter_string = filter_string.replace("{{C}}", string.as_str()); // no condition needed, "" stays "" + let mut condition_humongous_string = "(".to_string(); + let mut filter_humongous_string = "(".to_string(); + + for (index, string) in string_array_with_workarounds.iter().enumerate() { + condition_humongous_string = condition_humongous_string + + "(" + + condition_string.as_str() + + ")"; + condition_humongous_string = condition_humongous_string + .replace("{{C}}", string.as_str()); + + filter_humongous_string = filter_humongous_string + + "(" + + filter_string.as_str() + + ")"; + filter_humongous_string = + filter_humongous_string.replace("{{C}}", string.as_str()); + + // Only concatenate operator if it's not the last element + if index < string_array_with_workarounds.len() - 1 { + condition_humongous_string += operator_str; + filter_humongous_string += operator_str; + } + } + condition_string = condition_humongous_string + ")"; + + if !filter_string.is_empty() { + filter_string = filter_humongous_string + ")"; + } } _ => { return Err(FocusError::AstOperatorValueMismatch()); @@ -271,7 +315,7 @@ pub fn process( retrieval_cond += operator_str; if !filter_cond.is_empty() { filter_cond += operator_str; - dbg!(filter_cond.clone()); + //dbg!(filter_cond.clone()); } } } @@ -283,12 +327,14 @@ pub fn process( *retrieval_criteria += retrieval_cond.as_str(); if !filter_cond.is_empty() { - dbg!(filter_cond.clone()); + //dbg!(filter_cond.clone()); *filter_criteria += "("; *filter_criteria += filter_cond.as_str(); *filter_criteria += ")"; - dbg!(filter_criteria.clone()); + *filter_criteria = filter_criteria.replace(")(", ") or ("); + + //dbg!(filter_criteria.clone()); } Ok(()) @@ -317,10 +363,15 @@ mod test { const LENS2: &str = r#"{"ast":{"children":[{"children":[{"children":[{"key":"gender","system":"","type":"EQUALS","value":"male"},{"key":"gender","system":"","type":"EQUALS","value":"female"}],"operand":"OR"},{"children":[{"key":"diagnosis","system":"","type":"EQUALS","value":"C41"},{"key":"diagnosis","system":"","type":"EQUALS","value":"C50"}],"operand":"OR"},{"children":[{"key":"sample_kind","system":"","type":"EQUALS","value":"tissue-frozen"},{"key":"sample_kind","system":"","type":"EQUALS","value":"blood-serum"}],"operand":"OR"}],"operand":"AND"},{"children":[{"children":[{"key":"gender","system":"","type":"EQUALS","value":"male"}],"operand":"OR"},{"children":[{"key":"diagnosis","system":"","type":"EQUALS","value":"C41"},{"key":"diagnosis","system":"","type":"EQUALS","value":"C50"}],"operand":"OR"},{"children":[{"key":"sample_kind","system":"","type":"EQUALS","value":"liquid-other"},{"key":"sample_kind","system":"","type":"EQUALS","value":"rna"},{"key":"sample_kind","system":"","type":"EQUALS","value":"urine"}],"operand":"OR"},{"children":[{"key":"storage_temperature","system":"","type":"EQUALS","value":"temperatureRoom"},{"key":"storage_temperature","system":"","type":"EQUALS","value":"four_degrees"}],"operand":"OR"}],"operand":"AND"}],"operand":"OR"},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; - const EMPTY: &str = - r#"{"ast":{"children":[],"operand":"OR"}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + const EMPTY: &str = r#"{"ast":{"children":[],"operand":"OR"}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; #[test] + fn test_common() { + // maybe nothing here + } + + #[test] + #[cfg(feature="bbmri")] fn test_bbmri() { // println!( // "{:?}", @@ -332,6 +383,7 @@ mod test { // bbmri(serde_json::from_str(MALE_OR_FEMALE).expect("Failed to deserialize JSON")) // ); + // println!( // "{:?}", // bbmri(serde_json::from_str(ALL_GLIOMS).expect("Failed to deserialize JSON")) @@ -366,17 +418,19 @@ mod test { // println!(); - println!( - "{:?}", - generate_cql(serde_json::from_str(LENS2).expect("Failed to deserialize JSON"), Project::Bbmri) - ); - // println!( // "{:?}", - // bbmri(serde_json::from_str(EMPTY).expect("Failed to deserialize JSON")) + // generate_cql(serde_json::from_str(LENS2).expect("Failed to deserialize JSON"), Project::Bbmri) // ); - pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(EMPTY).unwrap(), Project::Bbmri).unwrap(), include_str!("../resources/test/result_empty.cql").to_string()); + pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(LENS2).unwrap(), Project::Bbmri).unwrap(), include_str!("../resources/test/result_lens2.cql").to_string()); + + /* println!( + "{:?}", + generate_cql(serde_json::from_str(EMPTY).expect("Failed to deserialize JSON"), Project::Bbmri) + ); */ + + //pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(EMPTY).unwrap(), Project::Bbmri).unwrap(), include_str!("../resources/test/result_empty.cql").to_string()); } } diff --git a/src/cql_new.rs b/src/cql_new.rs new file mode 100644 index 0000000..8193058 --- /dev/null +++ b/src/cql_new.rs @@ -0,0 +1,410 @@ +use crate::ast; +use crate::errors::FocusError; +use crate::projects::{Project, CriterionRole, CQL_TEMPLATES, MANDATORY_CODE_SYSTEMS, CODE_LISTS, CQL_SNIPPETS, CRITERION_CODE_LISTS, OBSERVATION_LOINC_CODE}; + +use chrono::offset::Utc; +use chrono::DateTime; +use indexmap::set::IndexSet; + +pub struct GeneratedCondition<'a> { + pub retrieval: Option<&'a str>, // Keep absence check for retrieval criteria at type level instead of inspecting the String later + pub filter: Option<&'a str>, // Same as above + pub code_systems: Vec<&'a str>, // This should probably be a set as we don't want duplicates. + } + +// Generating texts from a condition is a standalone operation. Having +// a separated function for this makes hings cleaner. +pub fn generate_condition<'a>(condition: &ast::Condition, project: Project) -> Result, FocusError> { + + let mut code_systems: Vec<&str> = Vec::new(); + let mut filter: Option<&str> ; + let mut retrieval: Option<&str>; + + + //let generated_condition = GeneratedCondition::new(); + + let condition_key_trans = condition.key.as_str(); + + let condition_snippet = + CQL_SNIPPETS.get(&(condition_key_trans, CriterionRole::Query, project.clone())); + + if let Some(snippet) = condition_snippet { + let mut condition_string = (*snippet).to_string(); + let mut filter_string: String = "".to_string(); + + let filter_snippet = CQL_SNIPPETS.get(&( + condition_key_trans, + CriterionRole::Filter, + project.clone(), + )); + + let code_lists_option = CRITERION_CODE_LISTS.get(&(condition_key_trans, project)); + if let Some(code_lists_vec) = code_lists_option { + for (index, code_list) in code_lists_vec.iter().enumerate() { + code_systems.push(code_list); + let placeholder = + "{{A".to_string() + (index + 1).to_string().as_str() + "}}"; //to keep compatibility with snippets in typescript + condition_string = + condition_string.replace(placeholder.as_str(), code_list); + } + } + + if condition_string.contains("{{K}}") { + //observation loinc code, those only apply to query criteria, we don't filter specimens by observations + let observation_code_option = OBSERVATION_LOINC_CODE.get(&condition_key_trans); + + if let Some(observation_code) = observation_code_option { + condition_string = condition_string.replace("{{K}}", observation_code); + } else { + return Err(FocusError::AstUnknownOption( + condition_key_trans.to_string(), + )); + } + } + + if let Some(filtret) = filter_snippet { + filter_string = (*filtret).to_string(); + } + + match condition.type_ { + ast::ConditionType::Between => { // both min and max values stated + match condition.value.clone() { + ast::ConditionValue::DateRange(date_range) => { + let datetime_str_min = date_range.min.as_str(); + let datetime_result_min: Result, _> = + datetime_str_min.parse(); + + if let Ok(datetime_min) = datetime_result_min { + let date_str_min = + format!("@{}", datetime_min.format("%Y-%m-%d")); + + condition_string = + condition_string.replace("{{D1}}", date_str_min.as_str()); + filter_string = + filter_string.replace("{{D1}}", date_str_min.as_str()); // no condition needed, "" stays "" + } else { + return Err(FocusError::AstInvalidDateFormat(date_range.min)); + } + + let datetime_str_max = date_range.max.as_str(); + let datetime_result_max: Result, _> = + datetime_str_max.parse(); + if let Ok(datetime_max) = datetime_result_max { + let date_str_max = + format!("@{}", datetime_max.format("%Y-%m-%d")); + + condition_string = + condition_string.replace("{{D2}}", date_str_max.as_str()); + filter_string = + filter_string.replace("{{D2}}", date_str_max.as_str()); // no condition needed, "" stays "" + } else { + return Err(FocusError::AstInvalidDateFormat(date_range.max)); + } + } + ast::ConditionValue::NumRange(num_range) => { + condition_string = condition_string + .replace("{{D1}}", num_range.min.to_string().as_str()); + condition_string = condition_string + .replace("{{D2}}", num_range.max.to_string().as_str()); + filter_string = filter_string + .replace("{{D1}}", num_range.min.to_string().as_str()); // no condition needed, "" stays "" + filter_string = filter_string + .replace("{{D2}}", num_range.max.to_string().as_str()); // no condition needed, "" stays "" + } + _ => { + return Err(FocusError::AstOperatorValueMismatch()); + } + } + } // deal with no lower or no upper value + ast::ConditionType::In => { + // although in works in CQL, at least in some places, most of it is converted to multiple criteria with OR + let operator_str = " or "; + + match condition.value.clone() { + ast::ConditionValue::StringArray(string_array) => { + let mut condition_humongous_string = "(".to_string(); + let mut filter_humongous_string = "(".to_string(); + + for (index, string) in string_array.iter().enumerate() { + condition_humongous_string = condition_humongous_string + + "(" + + condition_string.as_str() + + ")"; + condition_humongous_string = condition_humongous_string + .replace("{{C}}", string.as_str()); + + filter_humongous_string = filter_humongous_string + + "(" + + filter_string.as_str() + + ")"; + filter_humongous_string = + filter_humongous_string.replace("{{C}}", string.as_str()); + + // Only concatenate operator if it's not the last element + if index < string_array.len() - 1 { + condition_humongous_string += operator_str; + filter_humongous_string += operator_str; + } + } + condition_string = condition_humongous_string + ")"; + + if !filter_string.is_empty() { + filter_string = filter_humongous_string + ")"; + } + } + _ => { + return Err(FocusError::AstOperatorValueMismatch()); + } + } + } // this becomes or of all + ast::ConditionType::Equals => match condition.value.clone() { + ast::ConditionValue::String(string) => { + condition_string = condition_string.replace("{{C}}", string.as_str()); + filter_string = filter_string.replace("{{C}}", string.as_str()); // no condition needed, "" stays "" + } + _ => { + return Err(FocusError::AstOperatorValueMismatch()); + } + }, + ast::ConditionType::NotEquals => { // won't get it from Lens + } + ast::ConditionType::Contains => { // won't get it from Lens + } + ast::ConditionType::GreaterThan => {} // guess Lens won't send me this + ast::ConditionType::LowerThan => {} // guess Lens won't send me this + }; + + retrieval = Some(condition_string.as_str()); + + // if !filter.is_() && !filter_string.is_empty() { + // filter_cond += " and "; + // } + + filter = Some(filter_string.as_str()); // no condition needed, "" can be added with no change + } else { + return Err(FocusError::AstUnknownCriterion( + condition_key_trans.to_string(), + )); + } + //if !filter_cond.is_empty() { + // filter_cond += " "; + //} + //retrieval_cond += " "; + + Ok( GeneratedCondition { + retrieval: retrieval.clone(), // Keep absence check for retrieval criteria at type level instead of inspecting the String later + filter: filter.clone(), // Same as above + code_systems: code_systems.clone(), + }) + + +} + + +pub fn generate_cql(ast: ast::Ast, project: Project) -> Result { + let mut retrieval_criteria: String = "".to_string(); // main selection criteria (Patient) + + let mut filter_criteria: String = "".to_string(); // criteria for filtering specimens + + let mut lists: String = "".to_string(); // needed code lists, defined + + let mut cql = CQL_TEMPLATES.get(&project).expect("missing project").to_string(); + + let operator_str = match ast.ast.operand { + ast::Operand::And => " and ", + ast::Operand::Or => " or ", + }; + + let mut mandatory_codes = MANDATORY_CODE_SYSTEMS.get(&project) + .expect("non-existent project") + .clone(); + + for (index, grandchild) in ast.ast.children.iter().enumerate() { + process( + grandchild.clone(), + &mut retrieval_criteria, + &mut filter_criteria, + &mut mandatory_codes, + project, + )?; + + // Only concatenate operator if it's not the last element + if index < ast.ast.children.len() - 1 { + retrieval_criteria += operator_str; + } + } + + for code_system in mandatory_codes.iter() { + lists += format!( + "codesystem {}: '{}'\n", + code_system, + CODE_LISTS.get(code_system).unwrap_or(&("")) + ) + .as_str(); + } + + cql = cql + .replace("{{lists}}", lists.as_str()); + + if retrieval_criteria.is_empty() { + cql = cql.replace("{{retrieval_criteria}}", "true"); //()? + } else { + let formatted_retrieval_criteria = format!("({})", retrieval_criteria); + cql = cql.replace("{{retrieval_criteria}}", formatted_retrieval_criteria.as_str()); + } + + + if filter_criteria.is_empty() { + cql = cql.replace("{{filter_criteria}}", ""); + } else { + let formatted_filter_criteria = format!("where ({})", filter_criteria); + dbg!(formatted_filter_criteria.clone()); + cql = cql.replace("{{filter_criteria}}", formatted_filter_criteria.as_str()); + } + + Ok(cql) +} + +pub fn process( + child: ast::Child, + retrieval_criteria: &mut String, + filter_criteria: &mut String, + code_systems: &mut IndexSet<&str>, + project: Project, +) -> Result<(), FocusError> { + let mut retrieval_cond: String = "(".to_string(); + let mut filter_cond: String = "".to_string(); + + match child { + ast::Child::Condition(condition) => { + } + + ast::Child::Operation(operation) => { + let operator_str = match operation.operand { + ast::Operand::And => " and ", + ast::Operand::Or => " or ", + }; + + for (index, grandchild) in operation.children.iter().enumerate() { + process( + grandchild.clone(), + &mut retrieval_cond, + &mut filter_cond, + code_systems, + project.clone(), + )?; + + // Only concatenate operator if it's not the last element + if index < operation.children.len() - 1 { + retrieval_cond += operator_str; + if !filter_cond.is_empty() { + filter_cond += operator_str; + dbg!(filter_cond.clone()); + } + } + } + } + } + + retrieval_cond += ")"; + + *retrieval_criteria += retrieval_cond.as_str(); + + if !filter_cond.is_empty() { + dbg!(filter_cond.clone()); + *filter_criteria += "("; + *filter_criteria += filter_cond.as_str(); + *filter_criteria += ")"; + + dbg!(filter_criteria.clone()); + } + + Ok(()) +} + +#[cfg(test)] +mod test { + use super::*; + use pretty_assertions; + + const AST: &str = r#"{"ast":{"operand":"AND","children":[{"key":"age","type":"EQUALS","value":5.0}]},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + + const MALE_OR_FEMALE: &str = r#"{"ast":{"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"gender","type":"EQUALS","system":"","value":"male"},{"key":"gender","type":"EQUALS","system":"","value":"female"}]}]}]},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + + const ALL_GLIOMS: &str = r#"{"ast": {"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis","type":"EQUALS","system":"","value":"D43.%"}]},{"operand":"OR","children":[{"key":"59847-4","type":"EQUALS","system":"","value":"9383/1"},{"key":"59847-4","type":"EQUALS","system":"","value":"9384/1"},{"key":"59847-4","type":"EQUALS","system":"","value":"9394/1"},{"key":"59847-4","type":"EQUALS","system":"","value":"9421/1"}]}]},{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis","type":"EQUALS","system":"","value":"C71.%"},{"key":"diagnosis","type":"EQUALS","system":"","value":"C72.%"}]},{"operand":"OR","children":[{"key":"59847-4","type":"EQUALS","system":"","value":"9382/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9391/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9400/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9424/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9425/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9450/3"}]}]},{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis","type":"EQUALS","system":"","value":"C71.%"},{"key":"diagnosis","type":"EQUALS","system":"","value":"C72.%"}]},{"operand":"OR","children":[{"key":"59847-4","type":"EQUALS","system":"","value":"9440/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9441/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9442/3"}]}]},{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis","type":"EQUALS","system":"","value":"C71.%"},{"key":"diagnosis","type":"EQUALS","system":"","value":"C72.%"}]},{"operand":"OR","children":[{"key":"59847-4","type":"EQUALS","system":"","value":"9381/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9382/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9401/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9451/3"}]}]}]}]}]},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + + const AGE_AT_DIAGNOSIS_30_TO_70: &str = r#"{"ast": {"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"age_at_primary_diagnosis","type":"BETWEEN","system":"","value":{"min":30,"max":70}}]}]}]}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + + const AGE_AT_DIAGNOSIS_LOWER_THAN_70: &str = r#"{"ast": {"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"age_at_primary_diagnosis","type":"BETWEEN","system":"","value":{"min":0,"max":70}}]}]}]}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + + const C61_OR_MALE: &str = r#"{"ast": {"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis","type":"EQUALS","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","value":"C61"}]},{"operand":"OR","children":[{"key":"gender","type":"EQUALS","system":"","value":"male"}]}]}]}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + + const ALL_GBN: &str = r#"{"ast":{"children":[{"key":"gender","system":"","type":"IN","value":["male","other"]},{"children":[{"key":"diagnosis","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","type":"EQUALS","value":"C25"},{"key":"diagnosis","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","type":"EQUALS","value":"C56"}],"de":"Diagnose ICD-10","en":"Diagnosis ICD-10","key":"diagnosis","operand":"OR"},{"key":"diagnosis_age_donor","system":"","type":"BETWEEN","value":{"max":100,"min":10}},{"key":"date_of_diagnosis","system":"","type":"BETWEEN","value":{"max":"2023-10-29T23:00:00.000Z","min":"2023-09-30T22:00:00.000Z"}},{"key":"bmi","system":"","type":"BETWEEN","value":{"max":100,"min":10}},{"key":"body_weight","system":"","type":"BETWEEN","value":{"max":1100,"min":10}},{"key":"fasting_status","system":"","type":"IN","value":["Sober","Other fasting status"]},{"key":"smoking_status","system":"","type":"IN","value":["Smoker","Never smoked"]},{"key":"donor_age","system":"","type":"BETWEEN","value":{"max":10000,"min":100}},{"key":"sample_kind","system":"","type":"IN","value":["blood-serum","blood-plasma","buffy-coat"]},{"key":"sampling_date","system":"","type":"BETWEEN","value":{"max":"2023-10-29T23:00:00.000Z","min":"2023-10-03T22:00:00.000Z"}},{"key":"storage_temperature","system":"","type":"IN","value":["temperature-18to-35","temperature-60to-85"]}],"de":"haupt","en":"main","key":"main","operand":"AND"},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + + const SOME_GBN: &str = r#"{"ast":{"children":[{"key":"gender","system":"","type":"IN","value":["other","male"]},{"key":"diagnosis","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","type":"EQUALS","value":"C24"},{"key":"diagnosis_age_donor","system":"","type":"BETWEEN","value":{"max":11,"min":1}},{"key":"date_of_diagnosis","system":"","type":"BETWEEN","value":{"max":"2023-10-30T23:00:00.000Z","min":"2023-10-29T23:00:00.000Z"}},{"key":"bmi","system":"","type":"BETWEEN","value":{"max":111,"min":1}},{"key":"body_weight","system":"","type":"BETWEEN","value":{"max":1111,"min":110}},{"key":"fasting_status","system":"","type":"IN","value":["Sober","Not sober"]},{"key":"smoking_status","system":"","type":"IN","value":["Smoker","Never smoked"]},{"key":"donor_age","system":"","type":"BETWEEN","value":{"max":123,"min":1}},{"key":"sample_kind","system":"","type":"IN","value":["blood-serum","tissue-other"]},{"key":"sampling_date","system":"","type":"BETWEEN","value":{"max":"2023-10-30T23:00:00.000Z","min":"2023-10-29T23:00:00.000Z"}},{"key":"storage_temperature","system":"","type":"IN","value":["temperature2to10","temperatureGN"]}],"de":"haupt","en":"main","key":"main","operand":"AND"},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + + const LENS2: &str = r#"{"ast":{"children":[{"children":[{"children":[{"key":"gender","system":"","type":"EQUALS","value":"male"},{"key":"gender","system":"","type":"EQUALS","value":"female"}],"operand":"OR"},{"children":[{"key":"diagnosis","system":"","type":"EQUALS","value":"C41"},{"key":"diagnosis","system":"","type":"EQUALS","value":"C50"}],"operand":"OR"},{"children":[{"key":"sample_kind","system":"","type":"EQUALS","value":"tissue-frozen"},{"key":"sample_kind","system":"","type":"EQUALS","value":"blood-serum"}],"operand":"OR"}],"operand":"AND"},{"children":[{"children":[{"key":"gender","system":"","type":"EQUALS","value":"male"}],"operand":"OR"},{"children":[{"key":"diagnosis","system":"","type":"EQUALS","value":"C41"},{"key":"diagnosis","system":"","type":"EQUALS","value":"C50"}],"operand":"OR"},{"children":[{"key":"sample_kind","system":"","type":"EQUALS","value":"liquid-other"},{"key":"sample_kind","system":"","type":"EQUALS","value":"rna"},{"key":"sample_kind","system":"","type":"EQUALS","value":"urine"}],"operand":"OR"},{"children":[{"key":"storage_temperature","system":"","type":"EQUALS","value":"temperatureRoom"},{"key":"storage_temperature","system":"","type":"EQUALS","value":"four_degrees"}],"operand":"OR"}],"operand":"AND"}],"operand":"OR"},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + + const EMPTY: &str = + r#"{"ast":{"children":[],"operand":"OR"}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + + #[test] + fn test_bbmri() { + // println!( + // "{:?}", + // bbmri(serde_json::from_str(AST).expect("Failed to deserialize JSON")) + // ); + + // println!( + // "{:?}", + // bbmri(serde_json::from_str(MALE_OR_FEMALE).expect("Failed to deserialize JSON")) + // ); + + // println!( + // "{:?}", + // bbmri(serde_json::from_str(ALL_GLIOMS).expect("Failed to deserialize JSON")) + // ); + + // println!( + // "{:?}", + // bbmri(serde_json::from_str(AGE_AT_DIAGNOSIS_30_TO_70).expect("Failed to deserialize JSON")) + // ); + + // println!( + // "{:?}", + // bbmri(serde_json::from_str(AGE_AT_DIAGNOSIS_LOWER_THAN_70).expect("Failed to deserialize JSON")) + // ); + + // println!( + // "{:?}", + // bbmri(serde_json::from_str(C61_OR_MALE).expect("Failed to deserialize JSON")) + // ); + + // println!( + // "{:?}", + // bbmri(serde_json::from_str(ALL_GBN).expect("Failed to deserialize JSON")) + // ); + + // println!(); + + // println!( + // "{:?}", + // bbmri(serde_json::from_str(SOME_GBN).expect("Failed to deserialize JSON")) + // ); + + // println!(); + + println!( + "{:?}", + generate_cql(serde_json::from_str(LENS2).expect("Failed to deserialize JSON"), Project::Bbmri) + ); + + // println!( + // "{:?}", + // bbmri(serde_json::from_str(EMPTY).expect("Failed to deserialize JSON")) + // ); + + pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(EMPTY).unwrap(), Project::Bbmri).unwrap(), include_str!("../resources/test/result_empty.cql").to_string()); + + } +} diff --git a/src/main.rs b/src/main.rs index f7cbf46..0269761 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,6 +4,7 @@ mod beam; mod blaze; mod config; mod cql; +//mod cql_new; mod errors; mod graceful_shutdown; mod logger; diff --git a/src/projects/bbmri.rs b/src/projects/bbmri.rs index 7f84373..5d8968d 100644 --- a/src/projects/bbmri.rs +++ b/src/projects/bbmri.rs @@ -13,7 +13,7 @@ pub fn append_observation_loinc_codes(_map: &mut HashMap<&'static str, &'static pub fn append_criterion_code_lists(map: &mut HashMap<(&str, Project), Vec<&str>>) { for (key, value) in [ - ("diagnosis", vec!["icd10", "icd10gm"]), + ("diagnosis", vec!["icd10", "icd10gm", "icd10gmnew"]), ("body_weight", vec!["loinc"]), ("bmi", vec!["loinc"]), ("smoking_status", vec!["loinc"]), @@ -107,4 +107,23 @@ pub fn append_mandatory_code_lists(map: &mut HashMap>) { pub(crate) fn append_cql_templates(map: &mut HashMap) { map.insert(PROJECT, include_str!("../../resources/template_bbmri.cql")); +} + +pub fn append_sample_type_workarounds(map: &mut HashMap<&str, Vec<&str>>) { + for (key, value) in + [ + ("blood-plasma", vec!["plasma-edta", "plasma-citrat", "plasma-heparin", "plasma-cell-free", "plasma-other", "plasma"]), + ("blood-serum", vec!["serum"]), + ("tissue-ffpe", vec!["tumor-tissue-ffpe", "normal-tissue-ffpe", "other-tissue-ffpe", "tissue-formalin"]), + ("tissue-frozen", vec!["tumor-tissue-frozen", "normal-tissue-frozen", "other-tissue-frozen"]), + ("dna", vec!["cf-dna", "g-dna"]), + ("tissue-other", vec!["tissue-paxgene-or-else", "tissue"]), + ("derivative-other", vec!["derivative"]), + ("liquid-other", vec!["liquid"]), + ] { + map.insert( + key, + value + ); + } } \ No newline at end of file diff --git a/src/projects/common.rs b/src/projects/common.rs index 13100b2..88c4cf0 100644 --- a/src/projects/common.rs +++ b/src/projects/common.rs @@ -9,6 +9,7 @@ pub fn append_code_lists(map: &mut HashMap<&'static str, &'static str>) { [ ("icd10", "http://hl7.org/fhir/sid/icd-10"), ("icd10gm", "http://fhir.de/CodeSystem/dimdi/icd-10-gm"), + ("icd10gmnew", "http://fhir.de/CodeSystem/bfarm/icd-10-gm"), ("loinc", "http://loinc.org"), ( "SampleMaterialType", @@ -38,6 +39,8 @@ pub fn append_observation_loinc_codes(map: &mut HashMap<&'static str, &'static s ]); } +pub fn append_sample_type_workarounds(map: &mut HashMap<&str, Vec<&str>>) {} + pub fn append_criterion_code_lists(map: &mut HashMap<(&str, Project), Vec<&str>>) { } pub fn append_cql_snippets(map: &mut HashMap<(&str, CriterionRole, Project), &str>) { } diff --git a/src/projects/dktk.rs b/src/projects/dktk.rs index 0dc819f..b5576ce 100644 --- a/src/projects/dktk.rs +++ b/src/projects/dktk.rs @@ -14,6 +14,8 @@ pub fn append_criterion_code_lists(_map: &mut HashMap<(&str, Project), Vec<&str> pub fn append_cql_snippets(_map: &mut HashMap<(&str, CriterionRole, Project), &str>) { } +pub fn append_sample_type_workarounds(_map: &mut HashMap<&str, Vec<&str>>) {} + pub fn append_mandatory_code_lists(map: &mut HashMap>) { let mut set = map.remove(&PROJECT).unwrap_or(IndexSet::new()); for value in ["icd10", "SampleMaterialType", "loinc"] { diff --git a/src/projects/mod.rs b/src/projects/mod.rs index 34da35d..361f900 100644 --- a/src/projects/mod.rs +++ b/src/projects/mod.rs @@ -15,13 +15,19 @@ pub enum Project { Bbmri, #[cfg(feature="dktk")] Dktk, + #[cfg(not(any(feature = "dktk", feature = "bbmri")))] + NoCql } impl Display for Project { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let name = match self { + #[cfg(feature="bbmri")] Project::Bbmri => "bbmri", + #[cfg(feature="dktk")] Project::Dktk => "dktk", + #[cfg(not(any(feature = "bbmri", feature = "dktk")))] + Project::NoCql => "nocql" }; write!(f, "{name}") } @@ -60,6 +66,19 @@ pub static OBSERVATION_LOINC_CODE: Lazy> = Lazy::new(|| { map }); +pub static SAMPLE_TYPE_WORKAROUNDS: Lazy>> = Lazy::new(|| { + let mut map: HashMap<&'static str, Vec<&'static str>> = HashMap::new(); + common::append_sample_type_workarounds(&mut map); + + #[cfg(feature="bbmri")] + bbmri::append_sample_type_workarounds(&mut map); + + #[cfg(feature="dktk")] + dktk::append_sample_type_workarounds(&mut map); + + map +}); + // code lists needed depending on the criteria selected pub static CRITERION_CODE_LISTS: Lazy>> = Lazy::new(|| { let mut map = HashMap::new(); From 3b83a8dca3c48f53766ad489630346ed43dc8b31 Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Tue, 7 May 2024 18:44:50 +0200 Subject: [PATCH 20/42] BBMRI CQL generation tests --- .../test/result_age_at_diagnosis_30_to_70.cql | 86 ++++++++++++++++ ...result_age_at_diagnosis_lower_than_70.cql} | 3 +- resources/test/result_all_gbn.cql | 99 +++++++++++++++++++ resources/test/result_c61_and_male.cql | 87 ++++++++++++++++ resources/test/result_male_or_female.cql | 2 +- resources/test/result_some_gbn.cql | 99 +++++++++++++++++++ src/cql.rs | 86 +++++----------- 7 files changed, 399 insertions(+), 63 deletions(-) create mode 100644 resources/test/result_age_at_diagnosis_30_to_70.cql rename resources/test/{result_ast.cql => result_age_at_diagnosis_lower_than_70.cql} (97%) create mode 100644 resources/test/result_all_gbn.cql create mode 100644 resources/test/result_c61_and_male.cql create mode 100644 resources/test/result_some_gbn.cql diff --git a/resources/test/result_age_at_diagnosis_30_to_70.cql b/resources/test/result_age_at_diagnosis_30_to_70.cql new file mode 100644 index 0000000..8d4ce81 --- /dev/null +++ b/resources/test/result_age_at_diagnosis_30_to_70.cql @@ -0,0 +1,86 @@ +library Retrieve +using FHIR version '4.0.0' +include FHIRHelpers version '4.0.0' + +codesystem icd10: 'http://hl7.org/fhir/sid/icd-10' +codesystem SampleMaterialType: 'https://fhir.bbmri.de/CodeSystem/SampleMaterialType' + + +context Patient + +define AgeClass: +if (Patient.birthDate is null) then 'unknown' else ToString((AgeInYears() div 10) * 10) + +define Gender: +if (Patient.gender is null) then 'unknown' else Patient.gender + +define Custodian: + First(from Specimen.extension E + where E.url = 'https://fhir.bbmri.de/StructureDefinition/Custodian' + return (E.value as Reference).identifier.value) + +define function SampleType(specimen FHIR.Specimen): + case FHIRHelpers.ToCode(specimen.type.coding.where(system = 'https://fhir.bbmri.de/CodeSystem/SampleMaterialType').first()) + when Code 'plasma-edta' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-citrat' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-heparin' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-cell-free' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-other' from SampleMaterialType then 'blood-plasma' + when Code 'plasma' from SampleMaterialType then 'blood-plasma' + when Code 'tissue-formalin' from SampleMaterialType then 'tissue-ffpe' + when Code 'tumor-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'normal-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'other-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'tumor-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'normal-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'other-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'tissue-paxgene-or-else' from SampleMaterialType then 'tissue-other' + when Code 'derivative' from SampleMaterialType then 'derivative-other' + when Code 'liquid' from SampleMaterialType then 'liquid-other' + when Code 'tissue' from SampleMaterialType then 'tissue-other' + when Code 'serum' from SampleMaterialType then 'blood-serum' + when Code 'cf-dna' from SampleMaterialType then 'dna' + when Code 'g-dna' from SampleMaterialType then 'dna' + when Code 'blood-plasma' from SampleMaterialType then 'blood-plasma' + when Code 'tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'tissue-other' from SampleMaterialType then 'tissue-other' + when Code 'derivative-other' from SampleMaterialType then 'derivative-other' + when Code 'liquid-other' from SampleMaterialType then 'liquid-other' + when Code 'blood-serum' from SampleMaterialType then 'blood-serum' + when Code 'dna' from SampleMaterialType then 'dna' + when Code 'buffy-coat' from SampleMaterialType then 'buffy-coat' + when Code 'urine' from SampleMaterialType then 'urine' + when Code 'ascites' from SampleMaterialType then 'ascites' + when Code 'saliva' from SampleMaterialType then 'saliva' + when Code 'csf-liquor' from SampleMaterialType then 'csf-liquor' + when Code 'bone-marrow' from SampleMaterialType then 'bone-marrow' + when Code 'peripheral-blood-cells-vital' from SampleMaterialType then 'peripheral-blood-cells-vital' + when Code 'stool-faeces' from SampleMaterialType then 'stool-faeces' + when Code 'rna' from SampleMaterialType then 'rna' + when Code 'whole-blood' from SampleMaterialType then 'whole-blood' + when Code 'swab' from SampleMaterialType then 'swab' + when Code 'dried-whole-blood' from SampleMaterialType then 'dried-whole-blood' + when null then 'Unknown' + else 'Unknown' + end +define Specimen: + if InInitialPopulation then [Specimen] S else {} as List + +define Diagnosis: +if InInitialPopulation then [Condition] else {} as List + +define function DiagnosisCode(condition FHIR.Condition): +condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first() + +define function DiagnosisCode(condition FHIR.Condition, specimen FHIR.Specimen): +Coalesce( + condition.code.coding.where(system = 'http://hl7.org/fhir/sid/icd-10').code.first(), + condition.code.coding.where(system = 'http://fhir.de/CodeSystem/dimdi/icd-10-gm').code.first(), + condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first(), + specimen.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code.first() + ) + +define InInitialPopulation: +((((exists from [Condition] C +where AgeInYearsAt(FHIRHelpers.ToDateTime(C.onset)) between Ceiling(30) and Ceiling(70))))) \ No newline at end of file diff --git a/resources/test/result_ast.cql b/resources/test/result_age_at_diagnosis_lower_than_70.cql similarity index 97% rename from resources/test/result_ast.cql rename to resources/test/result_age_at_diagnosis_lower_than_70.cql index 54d8c01..9230612 100644 --- a/resources/test/result_ast.cql +++ b/resources/test/result_age_at_diagnosis_lower_than_70.cql @@ -82,4 +82,5 @@ Coalesce( ) define InInitialPopulation: -true \ No newline at end of file +((((exists from [Condition] C +where AgeInYearsAt(FHIRHelpers.ToDateTime(C.onset)) between Ceiling(0) and Ceiling(70))))) \ No newline at end of file diff --git a/resources/test/result_all_gbn.cql b/resources/test/result_all_gbn.cql new file mode 100644 index 0000000..d167d93 --- /dev/null +++ b/resources/test/result_all_gbn.cql @@ -0,0 +1,99 @@ +library Retrieve +using FHIR version '4.0.0' +include FHIRHelpers version '4.0.0' + +codesystem icd10: 'http://hl7.org/fhir/sid/icd-10' +codesystem SampleMaterialType: 'https://fhir.bbmri.de/CodeSystem/SampleMaterialType' +codesystem icd10gm: 'http://fhir.de/CodeSystem/dimdi/icd-10-gm' +codesystem icd10gmnew: 'http://fhir.de/CodeSystem/bfarm/icd-10-gm' +codesystem loinc: 'http://loinc.org' +codesystem FastingStatus: 'http://terminology.hl7.org/CodeSystem/v2-0916' +codesystem StorageTemperature: 'https://fhir.bbmri.de/CodeSystem/StorageTemperature' + + +context Patient + +define AgeClass: +if (Patient.birthDate is null) then 'unknown' else ToString((AgeInYears() div 10) * 10) + +define Gender: +if (Patient.gender is null) then 'unknown' else Patient.gender + +define Custodian: + First(from Specimen.extension E + where E.url = 'https://fhir.bbmri.de/StructureDefinition/Custodian' + return (E.value as Reference).identifier.value) + +define function SampleType(specimen FHIR.Specimen): + case FHIRHelpers.ToCode(specimen.type.coding.where(system = 'https://fhir.bbmri.de/CodeSystem/SampleMaterialType').first()) + when Code 'plasma-edta' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-citrat' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-heparin' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-cell-free' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-other' from SampleMaterialType then 'blood-plasma' + when Code 'plasma' from SampleMaterialType then 'blood-plasma' + when Code 'tissue-formalin' from SampleMaterialType then 'tissue-ffpe' + when Code 'tumor-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'normal-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'other-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'tumor-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'normal-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'other-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'tissue-paxgene-or-else' from SampleMaterialType then 'tissue-other' + when Code 'derivative' from SampleMaterialType then 'derivative-other' + when Code 'liquid' from SampleMaterialType then 'liquid-other' + when Code 'tissue' from SampleMaterialType then 'tissue-other' + when Code 'serum' from SampleMaterialType then 'blood-serum' + when Code 'cf-dna' from SampleMaterialType then 'dna' + when Code 'g-dna' from SampleMaterialType then 'dna' + when Code 'blood-plasma' from SampleMaterialType then 'blood-plasma' + when Code 'tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'tissue-other' from SampleMaterialType then 'tissue-other' + when Code 'derivative-other' from SampleMaterialType then 'derivative-other' + when Code 'liquid-other' from SampleMaterialType then 'liquid-other' + when Code 'blood-serum' from SampleMaterialType then 'blood-serum' + when Code 'dna' from SampleMaterialType then 'dna' + when Code 'buffy-coat' from SampleMaterialType then 'buffy-coat' + when Code 'urine' from SampleMaterialType then 'urine' + when Code 'ascites' from SampleMaterialType then 'ascites' + when Code 'saliva' from SampleMaterialType then 'saliva' + when Code 'csf-liquor' from SampleMaterialType then 'csf-liquor' + when Code 'bone-marrow' from SampleMaterialType then 'bone-marrow' + when Code 'peripheral-blood-cells-vital' from SampleMaterialType then 'peripheral-blood-cells-vital' + when Code 'stool-faeces' from SampleMaterialType then 'stool-faeces' + when Code 'rna' from SampleMaterialType then 'rna' + when Code 'whole-blood' from SampleMaterialType then 'whole-blood' + when Code 'swab' from SampleMaterialType then 'swab' + when Code 'dried-whole-blood' from SampleMaterialType then 'dried-whole-blood' + when null then 'Unknown' + else 'Unknown' + end +define Specimen: + if InInitialPopulation then [Specimen] S where (((((S.collection.fastingStatus.coding.code contains 'Sober') ) or ((S.collection.fastingStatus.coding.code contains 'Other fasting status') ))) or ((( (S.type.coding.code contains 'blood-serum')) or ( (S.type.coding.code contains 'blood-plasma')) or ( (S.type.coding.code contains 'buffy-coat')) or ( (S.type.coding.code contains 'serum')) or ( (S.type.coding.code contains 'plasma-edta')) or ( (S.type.coding.code contains 'plasma-citrat')) or ( (S.type.coding.code contains 'plasma-heparin')) or ( (S.type.coding.code contains 'plasma-cell-free')) or ( (S.type.coding.code contains 'plasma-other')) or ( (S.type.coding.code contains 'plasma')))) or ((FHIRHelpers.ToDateTime(S.collection.collected) between @2023-10-03 and @2023-10-29) ) or ((((S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding.code contains 'temperature-18to-35')) or ((S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding.code contains 'temperature-60to-85'))))) else {} as List + +define Diagnosis: +if InInitialPopulation then [Condition] else {} as List + +define function DiagnosisCode(condition FHIR.Condition): +condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first() + +define function DiagnosisCode(condition FHIR.Condition, specimen FHIR.Specimen): +Coalesce( + condition.code.coding.where(system = 'http://hl7.org/fhir/sid/icd-10').code.first(), + condition.code.coding.where(system = 'http://fhir.de/CodeSystem/dimdi/icd-10-gm').code.first(), + condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first(), + specimen.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code.first() + ) + +define InInitialPopulation: +((((Patient.gender = 'male') or (Patient.gender = 'other'))) and ((((((exists[Condition: Code 'C25' from icd10]) or (exists[Condition: Code 'C25' from icd10gm]) or (exists[Condition: Code 'C25' from icd10gmnew])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains 'C25'))))) or (((((exists[Condition: Code 'C56' from icd10]) or (exists[Condition: Code 'C56' from icd10gm]) or (exists[Condition: Code 'C56' from icd10gmnew])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains 'C56')))))) and (exists from [Condition] C +where AgeInYearsAt(FHIRHelpers.ToDateTime(C.onset)) between Ceiling(10) and Ceiling(100)) and (exists from [Condition] C +where FHIRHelpers.ToDateTime(C.onset) between @2023-09-30 and @2023-10-29) and (exists from [Observation: Code '39156-5' from loinc] O +where ((O.value as Quantity) < 10 'kg/m2' and (O.value as Quantity) > 100 'kg/m2')) and (exists from [Observation: Code '29463-7' from loinc] O +where ((O.value as Quantity) < 10 'kg' and (O.value as Quantity) > 1100 'kg')) and (((exists from [Specimen] S +where S.collection.fastingStatus.coding.code contains 'Sober' ) or (exists from [Specimen] S +where S.collection.fastingStatus.coding.code contains 'Other fasting status' ))) and (((exists from [Observation: Code '72166-2' from loinc] O +where O.value.coding.code contains 'Smoker' ) or (exists from [Observation: Code '72166-2' from loinc] O +where O.value.coding.code contains 'Never smoked' ))) and ( AgeInYears() between Ceiling(100) and Ceiling(10000)) and ((( exists [Specimen: Code 'blood-serum' from SampleMaterialType]) or ( exists [Specimen: Code 'blood-plasma' from SampleMaterialType]) or ( exists [Specimen: Code 'buffy-coat' from SampleMaterialType]) or ( exists [Specimen: Code 'serum' from SampleMaterialType]) or ( exists [Specimen: Code 'plasma-edta' from SampleMaterialType]) or ( exists [Specimen: Code 'plasma-citrat' from SampleMaterialType]) or ( exists [Specimen: Code 'plasma-heparin' from SampleMaterialType]) or ( exists [Specimen: Code 'plasma-cell-free' from SampleMaterialType]) or ( exists [Specimen: Code 'plasma-other' from SampleMaterialType]) or ( exists [Specimen: Code 'plasma' from SampleMaterialType]))) and (exists from [Specimen] S +where FHIRHelpers.ToDateTime(S.collection.collected) between @2023-10-03 and @2023-10-29 ) and (((exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding contains Code 'temperature-18to-35' from StorageTemperature) ) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding contains Code 'temperature-60to-85' from StorageTemperature) )))) \ No newline at end of file diff --git a/resources/test/result_c61_and_male.cql b/resources/test/result_c61_and_male.cql new file mode 100644 index 0000000..3809a6f --- /dev/null +++ b/resources/test/result_c61_and_male.cql @@ -0,0 +1,87 @@ +library Retrieve +using FHIR version '4.0.0' +include FHIRHelpers version '4.0.0' + +codesystem icd10: 'http://hl7.org/fhir/sid/icd-10' +codesystem SampleMaterialType: 'https://fhir.bbmri.de/CodeSystem/SampleMaterialType' +codesystem icd10gm: 'http://fhir.de/CodeSystem/dimdi/icd-10-gm' +codesystem icd10gmnew: 'http://fhir.de/CodeSystem/bfarm/icd-10-gm' + + +context Patient + +define AgeClass: +if (Patient.birthDate is null) then 'unknown' else ToString((AgeInYears() div 10) * 10) + +define Gender: +if (Patient.gender is null) then 'unknown' else Patient.gender + +define Custodian: + First(from Specimen.extension E + where E.url = 'https://fhir.bbmri.de/StructureDefinition/Custodian' + return (E.value as Reference).identifier.value) + +define function SampleType(specimen FHIR.Specimen): + case FHIRHelpers.ToCode(specimen.type.coding.where(system = 'https://fhir.bbmri.de/CodeSystem/SampleMaterialType').first()) + when Code 'plasma-edta' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-citrat' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-heparin' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-cell-free' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-other' from SampleMaterialType then 'blood-plasma' + when Code 'plasma' from SampleMaterialType then 'blood-plasma' + when Code 'tissue-formalin' from SampleMaterialType then 'tissue-ffpe' + when Code 'tumor-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'normal-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'other-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'tumor-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'normal-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'other-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'tissue-paxgene-or-else' from SampleMaterialType then 'tissue-other' + when Code 'derivative' from SampleMaterialType then 'derivative-other' + when Code 'liquid' from SampleMaterialType then 'liquid-other' + when Code 'tissue' from SampleMaterialType then 'tissue-other' + when Code 'serum' from SampleMaterialType then 'blood-serum' + when Code 'cf-dna' from SampleMaterialType then 'dna' + when Code 'g-dna' from SampleMaterialType then 'dna' + when Code 'blood-plasma' from SampleMaterialType then 'blood-plasma' + when Code 'tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'tissue-other' from SampleMaterialType then 'tissue-other' + when Code 'derivative-other' from SampleMaterialType then 'derivative-other' + when Code 'liquid-other' from SampleMaterialType then 'liquid-other' + when Code 'blood-serum' from SampleMaterialType then 'blood-serum' + when Code 'dna' from SampleMaterialType then 'dna' + when Code 'buffy-coat' from SampleMaterialType then 'buffy-coat' + when Code 'urine' from SampleMaterialType then 'urine' + when Code 'ascites' from SampleMaterialType then 'ascites' + when Code 'saliva' from SampleMaterialType then 'saliva' + when Code 'csf-liquor' from SampleMaterialType then 'csf-liquor' + when Code 'bone-marrow' from SampleMaterialType then 'bone-marrow' + when Code 'peripheral-blood-cells-vital' from SampleMaterialType then 'peripheral-blood-cells-vital' + when Code 'stool-faeces' from SampleMaterialType then 'stool-faeces' + when Code 'rna' from SampleMaterialType then 'rna' + when Code 'whole-blood' from SampleMaterialType then 'whole-blood' + when Code 'swab' from SampleMaterialType then 'swab' + when Code 'dried-whole-blood' from SampleMaterialType then 'dried-whole-blood' + when null then 'Unknown' + else 'Unknown' + end +define Specimen: + if InInitialPopulation then [Specimen] S else {} as List + +define Diagnosis: +if InInitialPopulation then [Condition] else {} as List + +define function DiagnosisCode(condition FHIR.Condition): +condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first() + +define function DiagnosisCode(condition FHIR.Condition, specimen FHIR.Specimen): +Coalesce( + condition.code.coding.where(system = 'http://hl7.org/fhir/sid/icd-10').code.first(), + condition.code.coding.where(system = 'http://fhir.de/CodeSystem/dimdi/icd-10-gm').code.first(), + condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first(), + specimen.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code.first() + ) + +define InInitialPopulation: +((((((((exists[Condition: Code 'C61' from icd10]) or (exists[Condition: Code 'C61' from icd10gm]) or (exists[Condition: Code 'C61' from icd10gmnew])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains 'C61')))))) and ((((Patient.gender = 'male')))))) \ No newline at end of file diff --git a/resources/test/result_male_or_female.cql b/resources/test/result_male_or_female.cql index 54d8c01..b823b85 100644 --- a/resources/test/result_male_or_female.cql +++ b/resources/test/result_male_or_female.cql @@ -82,4 +82,4 @@ Coalesce( ) define InInitialPopulation: -true \ No newline at end of file +((((((Patient.gender = 'male'))) or (((Patient.gender = 'female')))))) \ No newline at end of file diff --git a/resources/test/result_some_gbn.cql b/resources/test/result_some_gbn.cql new file mode 100644 index 0000000..8a76a3d --- /dev/null +++ b/resources/test/result_some_gbn.cql @@ -0,0 +1,99 @@ +library Retrieve +using FHIR version '4.0.0' +include FHIRHelpers version '4.0.0' + +codesystem icd10: 'http://hl7.org/fhir/sid/icd-10' +codesystem SampleMaterialType: 'https://fhir.bbmri.de/CodeSystem/SampleMaterialType' +codesystem icd10gm: 'http://fhir.de/CodeSystem/dimdi/icd-10-gm' +codesystem icd10gmnew: 'http://fhir.de/CodeSystem/bfarm/icd-10-gm' +codesystem loinc: 'http://loinc.org' +codesystem FastingStatus: 'http://terminology.hl7.org/CodeSystem/v2-0916' +codesystem StorageTemperature: 'https://fhir.bbmri.de/CodeSystem/StorageTemperature' + + +context Patient + +define AgeClass: +if (Patient.birthDate is null) then 'unknown' else ToString((AgeInYears() div 10) * 10) + +define Gender: +if (Patient.gender is null) then 'unknown' else Patient.gender + +define Custodian: + First(from Specimen.extension E + where E.url = 'https://fhir.bbmri.de/StructureDefinition/Custodian' + return (E.value as Reference).identifier.value) + +define function SampleType(specimen FHIR.Specimen): + case FHIRHelpers.ToCode(specimen.type.coding.where(system = 'https://fhir.bbmri.de/CodeSystem/SampleMaterialType').first()) + when Code 'plasma-edta' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-citrat' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-heparin' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-cell-free' from SampleMaterialType then 'blood-plasma' + when Code 'plasma-other' from SampleMaterialType then 'blood-plasma' + when Code 'plasma' from SampleMaterialType then 'blood-plasma' + when Code 'tissue-formalin' from SampleMaterialType then 'tissue-ffpe' + when Code 'tumor-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'normal-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'other-tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'tumor-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'normal-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'other-tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'tissue-paxgene-or-else' from SampleMaterialType then 'tissue-other' + when Code 'derivative' from SampleMaterialType then 'derivative-other' + when Code 'liquid' from SampleMaterialType then 'liquid-other' + when Code 'tissue' from SampleMaterialType then 'tissue-other' + when Code 'serum' from SampleMaterialType then 'blood-serum' + when Code 'cf-dna' from SampleMaterialType then 'dna' + when Code 'g-dna' from SampleMaterialType then 'dna' + when Code 'blood-plasma' from SampleMaterialType then 'blood-plasma' + when Code 'tissue-ffpe' from SampleMaterialType then 'tissue-ffpe' + when Code 'tissue-frozen' from SampleMaterialType then 'tissue-frozen' + when Code 'tissue-other' from SampleMaterialType then 'tissue-other' + when Code 'derivative-other' from SampleMaterialType then 'derivative-other' + when Code 'liquid-other' from SampleMaterialType then 'liquid-other' + when Code 'blood-serum' from SampleMaterialType then 'blood-serum' + when Code 'dna' from SampleMaterialType then 'dna' + when Code 'buffy-coat' from SampleMaterialType then 'buffy-coat' + when Code 'urine' from SampleMaterialType then 'urine' + when Code 'ascites' from SampleMaterialType then 'ascites' + when Code 'saliva' from SampleMaterialType then 'saliva' + when Code 'csf-liquor' from SampleMaterialType then 'csf-liquor' + when Code 'bone-marrow' from SampleMaterialType then 'bone-marrow' + when Code 'peripheral-blood-cells-vital' from SampleMaterialType then 'peripheral-blood-cells-vital' + when Code 'stool-faeces' from SampleMaterialType then 'stool-faeces' + when Code 'rna' from SampleMaterialType then 'rna' + when Code 'whole-blood' from SampleMaterialType then 'whole-blood' + when Code 'swab' from SampleMaterialType then 'swab' + when Code 'dried-whole-blood' from SampleMaterialType then 'dried-whole-blood' + when null then 'Unknown' + else 'Unknown' + end +define Specimen: + if InInitialPopulation then [Specimen] S where (((((S.collection.fastingStatus.coding.code contains 'Sober') ) or ((S.collection.fastingStatus.coding.code contains 'Not sober') ))) or ((( (S.type.coding.code contains 'blood-serum')) or ( (S.type.coding.code contains 'tissue-other')) or ( (S.type.coding.code contains 'serum')) or ( (S.type.coding.code contains 'tissue-paxgene-or-else')) or ( (S.type.coding.code contains 'tissue')))) or ((FHIRHelpers.ToDateTime(S.collection.collected) between @2023-10-29 and @2023-10-30) ) or ((((S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding.code contains 'temperature2to10')) or ((S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding.code contains 'temperatureGN'))))) else {} as List + +define Diagnosis: +if InInitialPopulation then [Condition] else {} as List + +define function DiagnosisCode(condition FHIR.Condition): +condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first() + +define function DiagnosisCode(condition FHIR.Condition, specimen FHIR.Specimen): +Coalesce( + condition.code.coding.where(system = 'http://hl7.org/fhir/sid/icd-10').code.first(), + condition.code.coding.where(system = 'http://fhir.de/CodeSystem/dimdi/icd-10-gm').code.first(), + condition.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first(), + specimen.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code.first() + ) + +define InInitialPopulation: +((((Patient.gender = 'other') or (Patient.gender = 'male'))) and (((((exists[Condition: Code 'C24' from icd10]) or (exists[Condition: Code 'C24' from icd10gm]) or (exists[Condition: Code 'C24' from icd10gmnew])) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis').value.coding.code contains 'C24'))))) and (exists from [Condition] C +where AgeInYearsAt(FHIRHelpers.ToDateTime(C.onset)) between Ceiling(1) and Ceiling(11)) and (exists from [Condition] C +where FHIRHelpers.ToDateTime(C.onset) between @2023-10-29 and @2023-10-30) and (exists from [Observation: Code '39156-5' from loinc] O +where ((O.value as Quantity) < 1 'kg/m2' and (O.value as Quantity) > 111 'kg/m2')) and (exists from [Observation: Code '29463-7' from loinc] O +where ((O.value as Quantity) < 110 'kg' and (O.value as Quantity) > 1111 'kg')) and (((exists from [Specimen] S +where S.collection.fastingStatus.coding.code contains 'Sober' ) or (exists from [Specimen] S +where S.collection.fastingStatus.coding.code contains 'Not sober' ))) and (((exists from [Observation: Code '72166-2' from loinc] O +where O.value.coding.code contains 'Smoker' ) or (exists from [Observation: Code '72166-2' from loinc] O +where O.value.coding.code contains 'Never smoked' ))) and ( AgeInYears() between Ceiling(1) and Ceiling(123)) and ((( exists [Specimen: Code 'blood-serum' from SampleMaterialType]) or ( exists [Specimen: Code 'tissue-other' from SampleMaterialType]) or ( exists [Specimen: Code 'serum' from SampleMaterialType]) or ( exists [Specimen: Code 'tissue-paxgene-or-else' from SampleMaterialType]) or ( exists [Specimen: Code 'tissue' from SampleMaterialType]))) and (exists from [Specimen] S +where FHIRHelpers.ToDateTime(S.collection.collected) between @2023-10-29 and @2023-10-30 ) and (((exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding contains Code 'temperature2to10' from StorageTemperature) ) or (exists from [Specimen] S where (S.extension.where(url='https://fhir.bbmri.de/StructureDefinition/StorageTemperature').value.coding contains Code 'temperatureGN' from StorageTemperature) )))) \ No newline at end of file diff --git a/src/cql.rs b/src/cql.rs index d09a211..3c3c220 100644 --- a/src/cql.rs +++ b/src/cql.rs @@ -63,10 +63,8 @@ pub fn generate_cql(ast: ast::Ast, project: impl Project) -> Result { // won't get it from Lens + ast::ConditionType::NotEquals => { // won't get it from Lens yet } - ast::ConditionType::Contains => { // won't get it from Lens + ast::ConditionType::Contains => { // won't get it from Lens yet } - ast::ConditionType::GreaterThan => {} // guess Lens won't send me this - ast::ConditionType::LowerThan => {} // guess Lens won't send me this + ast::ConditionType::GreaterThan => {} // won't get it from Lens yet + ast::ConditionType::LowerThan => {} // won't get it from Lens yet }; retrieval_cond += condition_string.as_str(); @@ -290,9 +286,8 @@ pub fn process( )); } if !filter_cond.is_empty() { - // filter_cond += " "; + //for historical reasons } - //retrieval_cond += " "; } ast::Child::Operation(operation) => { @@ -315,7 +310,6 @@ pub fn process( retrieval_cond += operator_str; if !filter_cond.is_empty() { filter_cond += operator_str; - //dbg!(filter_cond.clone()); } } } @@ -327,14 +321,12 @@ pub fn process( *retrieval_criteria += retrieval_cond.as_str(); if !filter_cond.is_empty() { - //dbg!(filter_cond.clone()); *filter_criteria += "("; *filter_criteria += filter_cond.as_str(); *filter_criteria += ")"; *filter_criteria = filter_criteria.replace(")(", ") or ("); - //dbg!(filter_criteria.clone()); } Ok(()) @@ -351,11 +343,11 @@ mod test { const ALL_GLIOMS: &str = r#"{"ast": {"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis","type":"EQUALS","system":"","value":"D43.%"}]},{"operand":"OR","children":[{"key":"59847-4","type":"EQUALS","system":"","value":"9383/1"},{"key":"59847-4","type":"EQUALS","system":"","value":"9384/1"},{"key":"59847-4","type":"EQUALS","system":"","value":"9394/1"},{"key":"59847-4","type":"EQUALS","system":"","value":"9421/1"}]}]},{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis","type":"EQUALS","system":"","value":"C71.%"},{"key":"diagnosis","type":"EQUALS","system":"","value":"C72.%"}]},{"operand":"OR","children":[{"key":"59847-4","type":"EQUALS","system":"","value":"9382/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9391/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9400/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9424/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9425/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9450/3"}]}]},{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis","type":"EQUALS","system":"","value":"C71.%"},{"key":"diagnosis","type":"EQUALS","system":"","value":"C72.%"}]},{"operand":"OR","children":[{"key":"59847-4","type":"EQUALS","system":"","value":"9440/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9441/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9442/3"}]}]},{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis","type":"EQUALS","system":"","value":"C71.%"},{"key":"diagnosis","type":"EQUALS","system":"","value":"C72.%"}]},{"operand":"OR","children":[{"key":"59847-4","type":"EQUALS","system":"","value":"9381/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9382/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9401/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9451/3"}]}]}]}]}]},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; - const AGE_AT_DIAGNOSIS_30_TO_70: &str = r#"{"ast": {"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"age_at_primary_diagnosis","type":"BETWEEN","system":"","value":{"min":30,"max":70}}]}]}]}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + const AGE_AT_DIAGNOSIS_30_TO_70: &str = r#"{"ast": {"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis_age_donor","type":"BETWEEN","system":"","value":{"min":30,"max":70}}]}]}]}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; - const AGE_AT_DIAGNOSIS_LOWER_THAN_70: &str = r#"{"ast": {"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"age_at_primary_diagnosis","type":"BETWEEN","system":"","value":{"min":0,"max":70}}]}]}]}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + const AGE_AT_DIAGNOSIS_LOWER_THAN_70: &str = r#"{"ast": {"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis_age_donor","type":"BETWEEN","system":"","value":{"min":0,"max":70}}]}]}]}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; - const C61_OR_MALE: &str = r#"{"ast": {"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis","type":"EQUALS","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","value":"C61"}]},{"operand":"OR","children":[{"key":"gender","type":"EQUALS","system":"","value":"male"}]}]}]}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + const C61_AND_MALE: &str = r#"{"ast": {"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis","type":"EQUALS","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","value":"C61"}]},{"operand":"OR","children":[{"key":"gender","type":"EQUALS","system":"","value":"male"}]}]}]}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; const ALL_GBN: &str = r#"{"ast":{"children":[{"key":"gender","system":"","type":"IN","value":["male","other"]},{"children":[{"key":"diagnosis","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","type":"EQUALS","value":"C25"},{"key":"diagnosis","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","type":"EQUALS","value":"C56"}],"de":"Diagnose ICD-10","en":"Diagnosis ICD-10","key":"diagnosis","operand":"OR"},{"key":"diagnosis_age_donor","system":"","type":"BETWEEN","value":{"max":100,"min":10}},{"key":"date_of_diagnosis","system":"","type":"BETWEEN","value":{"max":"2023-10-29T23:00:00.000Z","min":"2023-09-30T22:00:00.000Z"}},{"key":"bmi","system":"","type":"BETWEEN","value":{"max":100,"min":10}},{"key":"body_weight","system":"","type":"BETWEEN","value":{"max":1100,"min":10}},{"key":"fasting_status","system":"","type":"IN","value":["Sober","Other fasting status"]},{"key":"smoking_status","system":"","type":"IN","value":["Smoker","Never smoked"]},{"key":"donor_age","system":"","type":"BETWEEN","value":{"max":10000,"min":100}},{"key":"sample_kind","system":"","type":"IN","value":["blood-serum","blood-plasma","buffy-coat"]},{"key":"sampling_date","system":"","type":"BETWEEN","value":{"max":"2023-10-29T23:00:00.000Z","min":"2023-10-03T22:00:00.000Z"}},{"key":"storage_temperature","system":"","type":"IN","value":["temperature-18to-35","temperature-60to-85"]}],"de":"haupt","en":"main","key":"main","operand":"AND"},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; @@ -375,64 +367,36 @@ mod test { #[cfg(feature="bbmri")] fn test_bbmri() { use crate::projects::{self, bbmri::Bbmri}; - // println!( - // "{:?}", - // bbmri(serde_json::from_str(AST).expect("Failed to deserialize JSON")) - // ); - - // println!( - // "{:?}", - // bbmri(serde_json::from_str(MALE_OR_FEMALE).expect("Failed to deserialize JSON")) - // ); + pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(MALE_OR_FEMALE).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_male_or_female.cql").to_string()); - // println!( - // "{:?}", - // bbmri(serde_json::from_str(ALL_GLIOMS).expect("Failed to deserialize JSON")) - // ); + pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(AGE_AT_DIAGNOSIS_30_TO_70).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_age_at_diagnosis_30_to_70.cql").to_string()); - // println!( - // "{:?}", - // bbmri(serde_json::from_str(AGE_AT_DIAGNOSIS_30_TO_70).expect("Failed to deserialize JSON")) - // ); + pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(AGE_AT_DIAGNOSIS_LOWER_THAN_70).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_age_at_diagnosis_lower_than_70.cql").to_string()); - // println!( - // "{:?}", - // bbmri(serde_json::from_str(AGE_AT_DIAGNOSIS_LOWER_THAN_70).expect("Failed to deserialize JSON")) - // ); + pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(C61_AND_MALE).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_c61_and_male.cql").to_string()); - // println!( - // "{:?}", - // bbmri(serde_json::from_str(C61_OR_MALE).expect("Failed to deserialize JSON")) - // ); + pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(ALL_GBN).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_all_gbn.cql").to_string()); - // println!( - // "{:?}", - // bbmri(serde_json::from_str(ALL_GBN).expect("Failed to deserialize JSON")) - // ); + pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(SOME_GBN).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_some_gbn.cql").to_string()); - // println!(); + pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(LENS2).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_lens2.cql").to_string()); - // println!( - // "{:?}", - // bbmri(serde_json::from_str(SOME_GBN).expect("Failed to deserialize JSON")) - // ); + pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(EMPTY).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_empty.cql").to_string()); - // println!(); + } - // println!( - // "{:?}", - // generate_cql(serde_json::from_str(LENS2).expect("Failed to deserialize JSON"), Project::Bbmri) - // ); + #[test] + #[cfg(feature="dktk")] + fn test_dktk() { + use crate::projects::{self, dktk::Dktk}; - pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(LENS2).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_lens2.cql").to_string()); + todo!("Implement DKTK CQL generation and create files with results"); - /* println!( - "{:?}", - generate_cql(serde_json::from_str(EMPTY).expect("Failed to deserialize JSON"), Project::Bbmri) - ); */ + pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(AST).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_ast.cql").to_string()); - //pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(EMPTY).unwrap(), Project::Bbmri).unwrap(), include_str!("../resources/test/result_empty.cql").to_string()); + pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(ALL_GLIOMS).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_all_glioms.cql").to_string()); } + } From 33cbc633a9dee80e4809d90a6b54399cc44234ac Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Tue, 7 May 2024 18:52:09 +0200 Subject: [PATCH 21/42] cleanup --- Cargo.toml | 6 - libs/focus_api/Cargo.toml | 20 -- libs/focus_api/src/lib.rs | 184 ----------------- src/cql_alternative.rs | 51 ----- src/cql_new.rs | 410 -------------------------------------- src/main.rs | 1 - 6 files changed, 672 deletions(-) delete mode 100644 libs/focus_api/Cargo.toml delete mode 100644 libs/focus_api/src/lib.rs delete mode 100644 src/cql_alternative.rs delete mode 100644 src/cql_new.rs diff --git a/Cargo.toml b/Cargo.toml index 010f2ab..f28c299 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,9 +6,6 @@ license = "Apache-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[workspace] -members = ["libs/focus_api"] - [dependencies] base64 = { version = "0.21.0", default_features = false } http = "0.2" @@ -22,9 +19,6 @@ indexmap = "2.1.0" tokio = { version = "1.25.0", default_features = false, features = ["signal", "rt-multi-thread", "macros"] } beam-lib = { git = "https://github.com/samply/beam", branch = "develop", features = ["http-util"] } laplace_rs = {version = "0.2.0", git = "https://github.com/samply/laplace-rs.git", branch = "main" } -validated = "0.4.0" -nonempty-collections = "0.1.4" -focus_api = { version = "0.1.0", path = "libs/focus_api" } # Logging tracing = { version = "0.1.37", default_features = false } diff --git a/libs/focus_api/Cargo.toml b/libs/focus_api/Cargo.toml deleted file mode 100644 index 7477291..0000000 --- a/libs/focus_api/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "focus_api" -version = "0.1.0" -edition = "2021" -license = "Apache-2.0" - -[lib] -crate-type = ["cdylib", "rlib"] - -[dependencies] -validated = "0.4.0" -nonempty-collections = "0.1.4" -tsify = "0.4.5" -wasm-bindgen = "0.2.89" -serde = { version = "1.0.152", features = ["serde_derive"] } -chrono = "0.4.31" - -[dev-dependencies] -pretty_assertions = "1.4.0" -tokio-test = "0.4.2" diff --git a/libs/focus_api/src/lib.rs b/libs/focus_api/src/lib.rs deleted file mode 100644 index 086afdf..0000000 --- a/libs/focus_api/src/lib.rs +++ /dev/null @@ -1,184 +0,0 @@ -use validated::Validated::{self, Good, Fail}; -use nonempty_collections::*; -use serde::{Deserialize, Serialize}; -use tsify::Tsify; -use wasm_bindgen::prelude::*; - -// Here are a few design ideas to consider while implementing CQL generation. -// I am not sure how feasible/useful they are as I have a partial understanding -// of the spec and don't really know much about the constraints. Enola asked me to -// push it, so here it goes. - -// Caveat: Did not touch JSON side, I see that as a separate issue. Also -// I was sloppy with the borrow checker, so probably there is room for memory -// footprint optimization. - -// Some general remarks about safer Rust code. -// ------------------------------------------- - -// It is good practice to avoid using naked general purpose types like String. -// In the future we may want to restrict possible key values to, say, alphanumeric -// strings. Representing dates as naked strings is not kosher, either. The general -// idea is pushing preconditions upstream instead of implementing workarounds downstream. - -// As an example, this is how I would define the Date type. Something similar can be done -// for id and key fields which are naked Strings. -mod safe_date { - use chrono::NaiveDateTime; - - #[derive(super::Tsify, super::Serialize, super::Deserialize)] - #[tsify(into_wasm_abi, from_wasm_abi)] - pub struct Date(String); - - impl Date { - // Type comes with its validator but strictly speaking this is not necessary - // as we do not process dates. If we start using optics in our Rust code - // we can cast this mechanisms as a prism. - pub fn new(str: String) -> Option { - match NaiveDateTime::parse_from_str(&str, "%Y-%m-%d") { - Ok(_) => Some(Date(str)), - Err(_) => None, - } - } - - // An un-wrapper - pub fn to_string(self) { - self.0; - } - - // Serialize/Deserialize would also be here. They can be implemented using new - // and to_string above. Ideally we would also have a roundtrip test and some unit - // tests. If we need to implement too many traits by hand we can use, for instance, - // https://docs.rs/newtype_derive/latest/newtype_derive/ - } -} - - -// Now specific comments on the implementation. -// -------------------------------------------- - -// I changed the name to AstWithId because that's what it is. -#[derive(Tsify, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -#[tsify(into_wasm_abi, from_wasm_abi)] -pub struct AstWithId { - pub ast: Ast, - pub id: String, // Better be a 'newtype' -} - -// Original AST definition was too complicated. An expression is -// either atomic or built from smaller expressions. No need for indirection. -#[derive(Tsify, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -#[tsify(into_wasm_abi, from_wasm_abi)] -pub enum Ast { - Atomic(Condition), - Compound(Operation, Vec) // we can disallow empty vectors here but we have sane defaults so it is not a big deal -} - -// Operand is the name of the inputs you give to -// to the operation in an expression. So changed this, too. -#[derive(Tsify, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -#[tsify(into_wasm_abi, from_wasm_abi)] -#[derive(Clone, Copy)] -pub enum Operation { - And, - Or, -} - -// Having all the operation related things in one place is good. -// CQL support Xor. If we decide to implement it we only change here -// and the rest of the code works. -impl Operation { - pub fn operation_name(self) -> &'static str { - match self { - Operation::And => " and ", - Operation::Or => " or ", - } - } - - // this is not needed if we disallow empty vectors. some people find - // this counterintuitive so maybe we should? - pub fn nullary_value(self) -> &'static str { - match self { - Operation::And => "true", //empty iterator returns true under all in Rust - Operation::Or => "false", //empty iterator returns false under any in Rust - } - } - - pub fn apply_to_group(self, group: Vec<&str>) -> String { - if group.is_empty() { - self.nullary_value().to_string() - } else { - group. // there should be a standard function for this somewhere - iter(). - map(|s| s.to_string()). - enumerate(). - map(|(index, s)| if index < group.len() - 1 {s + self.operation_name()} else {s}). - collect::>(). - concat() - } - } -} - - -// We can use some polymorphism here to avoid code duplication. -// and shine at cocktail parties. -#[derive(Tsify, Serialize, Deserialize)] -#[tsify(into_wasm_abi, from_wasm_abi)] -pub struct AbstractRange { - pub min: T, - pub max: T, -} - -#[derive(Tsify, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -#[tsify(into_wasm_abi, from_wasm_abi)] -pub enum ConditionValue { - DateRange(AbstractRange), - NumberRange(AbstractRange), - Number(f64), - //etc. -} - -#[derive(Tsify, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -#[tsify(into_wasm_abi, from_wasm_abi)] -pub enum ConditionType { - Equals, - Between, - //etc. -} - -// We can have an enum of condition keys so we can reject unknown keys -// at json parsing level. -#[derive(Tsify, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -#[tsify(into_wasm_abi, from_wasm_abi)] -pub enum ConditionKey { - Gender, - Diagnosis, - DiagnosisOld, - // etc. -} - -// Condition keys may depend on the project but we can always -// define `pup struct Condition {...}`.. -#[derive(Tsify, Serialize, Deserialize)] -#[tsify(into_wasm_abi, from_wasm_abi)] -pub struct Condition { - pub key: ConditionKey, - pub type_: ConditionType, - pub value: ConditionValue -} - -// Specific errors about generation. At this level only incompatibility errors should be left. -// Everything else can be enforced by the type system so they can be pushed to the JSON parsing layer. -#[derive(Tsify, Serialize, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -#[tsify(into_wasm_abi, from_wasm_abi)] -pub enum GenerationError { - IncompatibleBlah, - // etc. -} diff --git a/src/cql_alternative.rs b/src/cql_alternative.rs deleted file mode 100644 index e5c3dfe..0000000 --- a/src/cql_alternative.rs +++ /dev/null @@ -1,51 +0,0 @@ -use validated::Validated::{self, Good, Fail}; -use nonempty_collections::*; -use focus_api::*; - -pub struct GeneratedCondition<'a> { - pub retrieval: Option<&'a str>, // Keep absence check for retrieval criteria at type level instead of inspecting the String later - pub filter: Option<&'a str>, // Same as above - pub code_systems: Vec<&'a str>, // This should probably be a set as we don't want duplicates. -} - -// Generating texts from a condition is a standalone operation. Having -// a separated function for this makes hings cleaner. -pub fn generate_condition<'a>(condition: &Condition) -> Result, GenerationError> { - unimplemented!("All the table lookups, compatibility checks etc. should happen here"); -} - -// If we are fine with recursion we can use this. If we want a stack based implementation later -// it would be much easier to refactor. Error handling is streamlined and it is accumulative. -// As a middle solution we can bound the recursion depth, say by ~10. -// Since we will be reporting errors to a UI it is better to collect errors. This is what -// Validated does. -pub fn generate_all<'a>(ast: &Ast) -> Validated, GenerationError> { - match ast { - Ast::Atomic(condition) => - match generate_condition(&condition){ - Ok(generated) => Good(generated), - Err(err) => Fail(nev![err]), - }, - Ast::Compound(op, vec_ast) => - { let recursive_step: Validated, _> = vec_ast.into_iter().map(generate_all).collect(); - match recursive_step { - Good(condition_vec) => { - let retrieval_vec: Vec<&str> = // i extracted the retrieval vec but you can generate all three needed vectors here in a single pass by a fold. - condition_vec. - into_iter(). - map(|g| g.retrieval). - flatten(). - collect(); - Good(GeneratedCondition - { retrieval: - if retrieval_vec.is_empty() - { None } else - { Some (&format!("({})", op.apply_to_group(retrieval_vec))) } - , filter: todo!("Combine filters") - , code_systems: todo!("Combine code systems") - })}, - Fail(errors) => - Fail(errors), - }}, - } -} diff --git a/src/cql_new.rs b/src/cql_new.rs deleted file mode 100644 index 8193058..0000000 --- a/src/cql_new.rs +++ /dev/null @@ -1,410 +0,0 @@ -use crate::ast; -use crate::errors::FocusError; -use crate::projects::{Project, CriterionRole, CQL_TEMPLATES, MANDATORY_CODE_SYSTEMS, CODE_LISTS, CQL_SNIPPETS, CRITERION_CODE_LISTS, OBSERVATION_LOINC_CODE}; - -use chrono::offset::Utc; -use chrono::DateTime; -use indexmap::set::IndexSet; - -pub struct GeneratedCondition<'a> { - pub retrieval: Option<&'a str>, // Keep absence check for retrieval criteria at type level instead of inspecting the String later - pub filter: Option<&'a str>, // Same as above - pub code_systems: Vec<&'a str>, // This should probably be a set as we don't want duplicates. - } - -// Generating texts from a condition is a standalone operation. Having -// a separated function for this makes hings cleaner. -pub fn generate_condition<'a>(condition: &ast::Condition, project: Project) -> Result, FocusError> { - - let mut code_systems: Vec<&str> = Vec::new(); - let mut filter: Option<&str> ; - let mut retrieval: Option<&str>; - - - //let generated_condition = GeneratedCondition::new(); - - let condition_key_trans = condition.key.as_str(); - - let condition_snippet = - CQL_SNIPPETS.get(&(condition_key_trans, CriterionRole::Query, project.clone())); - - if let Some(snippet) = condition_snippet { - let mut condition_string = (*snippet).to_string(); - let mut filter_string: String = "".to_string(); - - let filter_snippet = CQL_SNIPPETS.get(&( - condition_key_trans, - CriterionRole::Filter, - project.clone(), - )); - - let code_lists_option = CRITERION_CODE_LISTS.get(&(condition_key_trans, project)); - if let Some(code_lists_vec) = code_lists_option { - for (index, code_list) in code_lists_vec.iter().enumerate() { - code_systems.push(code_list); - let placeholder = - "{{A".to_string() + (index + 1).to_string().as_str() + "}}"; //to keep compatibility with snippets in typescript - condition_string = - condition_string.replace(placeholder.as_str(), code_list); - } - } - - if condition_string.contains("{{K}}") { - //observation loinc code, those only apply to query criteria, we don't filter specimens by observations - let observation_code_option = OBSERVATION_LOINC_CODE.get(&condition_key_trans); - - if let Some(observation_code) = observation_code_option { - condition_string = condition_string.replace("{{K}}", observation_code); - } else { - return Err(FocusError::AstUnknownOption( - condition_key_trans.to_string(), - )); - } - } - - if let Some(filtret) = filter_snippet { - filter_string = (*filtret).to_string(); - } - - match condition.type_ { - ast::ConditionType::Between => { // both min and max values stated - match condition.value.clone() { - ast::ConditionValue::DateRange(date_range) => { - let datetime_str_min = date_range.min.as_str(); - let datetime_result_min: Result, _> = - datetime_str_min.parse(); - - if let Ok(datetime_min) = datetime_result_min { - let date_str_min = - format!("@{}", datetime_min.format("%Y-%m-%d")); - - condition_string = - condition_string.replace("{{D1}}", date_str_min.as_str()); - filter_string = - filter_string.replace("{{D1}}", date_str_min.as_str()); // no condition needed, "" stays "" - } else { - return Err(FocusError::AstInvalidDateFormat(date_range.min)); - } - - let datetime_str_max = date_range.max.as_str(); - let datetime_result_max: Result, _> = - datetime_str_max.parse(); - if let Ok(datetime_max) = datetime_result_max { - let date_str_max = - format!("@{}", datetime_max.format("%Y-%m-%d")); - - condition_string = - condition_string.replace("{{D2}}", date_str_max.as_str()); - filter_string = - filter_string.replace("{{D2}}", date_str_max.as_str()); // no condition needed, "" stays "" - } else { - return Err(FocusError::AstInvalidDateFormat(date_range.max)); - } - } - ast::ConditionValue::NumRange(num_range) => { - condition_string = condition_string - .replace("{{D1}}", num_range.min.to_string().as_str()); - condition_string = condition_string - .replace("{{D2}}", num_range.max.to_string().as_str()); - filter_string = filter_string - .replace("{{D1}}", num_range.min.to_string().as_str()); // no condition needed, "" stays "" - filter_string = filter_string - .replace("{{D2}}", num_range.max.to_string().as_str()); // no condition needed, "" stays "" - } - _ => { - return Err(FocusError::AstOperatorValueMismatch()); - } - } - } // deal with no lower or no upper value - ast::ConditionType::In => { - // although in works in CQL, at least in some places, most of it is converted to multiple criteria with OR - let operator_str = " or "; - - match condition.value.clone() { - ast::ConditionValue::StringArray(string_array) => { - let mut condition_humongous_string = "(".to_string(); - let mut filter_humongous_string = "(".to_string(); - - for (index, string) in string_array.iter().enumerate() { - condition_humongous_string = condition_humongous_string - + "(" - + condition_string.as_str() - + ")"; - condition_humongous_string = condition_humongous_string - .replace("{{C}}", string.as_str()); - - filter_humongous_string = filter_humongous_string - + "(" - + filter_string.as_str() - + ")"; - filter_humongous_string = - filter_humongous_string.replace("{{C}}", string.as_str()); - - // Only concatenate operator if it's not the last element - if index < string_array.len() - 1 { - condition_humongous_string += operator_str; - filter_humongous_string += operator_str; - } - } - condition_string = condition_humongous_string + ")"; - - if !filter_string.is_empty() { - filter_string = filter_humongous_string + ")"; - } - } - _ => { - return Err(FocusError::AstOperatorValueMismatch()); - } - } - } // this becomes or of all - ast::ConditionType::Equals => match condition.value.clone() { - ast::ConditionValue::String(string) => { - condition_string = condition_string.replace("{{C}}", string.as_str()); - filter_string = filter_string.replace("{{C}}", string.as_str()); // no condition needed, "" stays "" - } - _ => { - return Err(FocusError::AstOperatorValueMismatch()); - } - }, - ast::ConditionType::NotEquals => { // won't get it from Lens - } - ast::ConditionType::Contains => { // won't get it from Lens - } - ast::ConditionType::GreaterThan => {} // guess Lens won't send me this - ast::ConditionType::LowerThan => {} // guess Lens won't send me this - }; - - retrieval = Some(condition_string.as_str()); - - // if !filter.is_() && !filter_string.is_empty() { - // filter_cond += " and "; - // } - - filter = Some(filter_string.as_str()); // no condition needed, "" can be added with no change - } else { - return Err(FocusError::AstUnknownCriterion( - condition_key_trans.to_string(), - )); - } - //if !filter_cond.is_empty() { - // filter_cond += " "; - //} - //retrieval_cond += " "; - - Ok( GeneratedCondition { - retrieval: retrieval.clone(), // Keep absence check for retrieval criteria at type level instead of inspecting the String later - filter: filter.clone(), // Same as above - code_systems: code_systems.clone(), - }) - - -} - - -pub fn generate_cql(ast: ast::Ast, project: Project) -> Result { - let mut retrieval_criteria: String = "".to_string(); // main selection criteria (Patient) - - let mut filter_criteria: String = "".to_string(); // criteria for filtering specimens - - let mut lists: String = "".to_string(); // needed code lists, defined - - let mut cql = CQL_TEMPLATES.get(&project).expect("missing project").to_string(); - - let operator_str = match ast.ast.operand { - ast::Operand::And => " and ", - ast::Operand::Or => " or ", - }; - - let mut mandatory_codes = MANDATORY_CODE_SYSTEMS.get(&project) - .expect("non-existent project") - .clone(); - - for (index, grandchild) in ast.ast.children.iter().enumerate() { - process( - grandchild.clone(), - &mut retrieval_criteria, - &mut filter_criteria, - &mut mandatory_codes, - project, - )?; - - // Only concatenate operator if it's not the last element - if index < ast.ast.children.len() - 1 { - retrieval_criteria += operator_str; - } - } - - for code_system in mandatory_codes.iter() { - lists += format!( - "codesystem {}: '{}'\n", - code_system, - CODE_LISTS.get(code_system).unwrap_or(&("")) - ) - .as_str(); - } - - cql = cql - .replace("{{lists}}", lists.as_str()); - - if retrieval_criteria.is_empty() { - cql = cql.replace("{{retrieval_criteria}}", "true"); //()? - } else { - let formatted_retrieval_criteria = format!("({})", retrieval_criteria); - cql = cql.replace("{{retrieval_criteria}}", formatted_retrieval_criteria.as_str()); - } - - - if filter_criteria.is_empty() { - cql = cql.replace("{{filter_criteria}}", ""); - } else { - let formatted_filter_criteria = format!("where ({})", filter_criteria); - dbg!(formatted_filter_criteria.clone()); - cql = cql.replace("{{filter_criteria}}", formatted_filter_criteria.as_str()); - } - - Ok(cql) -} - -pub fn process( - child: ast::Child, - retrieval_criteria: &mut String, - filter_criteria: &mut String, - code_systems: &mut IndexSet<&str>, - project: Project, -) -> Result<(), FocusError> { - let mut retrieval_cond: String = "(".to_string(); - let mut filter_cond: String = "".to_string(); - - match child { - ast::Child::Condition(condition) => { - } - - ast::Child::Operation(operation) => { - let operator_str = match operation.operand { - ast::Operand::And => " and ", - ast::Operand::Or => " or ", - }; - - for (index, grandchild) in operation.children.iter().enumerate() { - process( - grandchild.clone(), - &mut retrieval_cond, - &mut filter_cond, - code_systems, - project.clone(), - )?; - - // Only concatenate operator if it's not the last element - if index < operation.children.len() - 1 { - retrieval_cond += operator_str; - if !filter_cond.is_empty() { - filter_cond += operator_str; - dbg!(filter_cond.clone()); - } - } - } - } - } - - retrieval_cond += ")"; - - *retrieval_criteria += retrieval_cond.as_str(); - - if !filter_cond.is_empty() { - dbg!(filter_cond.clone()); - *filter_criteria += "("; - *filter_criteria += filter_cond.as_str(); - *filter_criteria += ")"; - - dbg!(filter_criteria.clone()); - } - - Ok(()) -} - -#[cfg(test)] -mod test { - use super::*; - use pretty_assertions; - - const AST: &str = r#"{"ast":{"operand":"AND","children":[{"key":"age","type":"EQUALS","value":5.0}]},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; - - const MALE_OR_FEMALE: &str = r#"{"ast":{"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"gender","type":"EQUALS","system":"","value":"male"},{"key":"gender","type":"EQUALS","system":"","value":"female"}]}]}]},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; - - const ALL_GLIOMS: &str = r#"{"ast": {"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis","type":"EQUALS","system":"","value":"D43.%"}]},{"operand":"OR","children":[{"key":"59847-4","type":"EQUALS","system":"","value":"9383/1"},{"key":"59847-4","type":"EQUALS","system":"","value":"9384/1"},{"key":"59847-4","type":"EQUALS","system":"","value":"9394/1"},{"key":"59847-4","type":"EQUALS","system":"","value":"9421/1"}]}]},{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis","type":"EQUALS","system":"","value":"C71.%"},{"key":"diagnosis","type":"EQUALS","system":"","value":"C72.%"}]},{"operand":"OR","children":[{"key":"59847-4","type":"EQUALS","system":"","value":"9382/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9391/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9400/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9424/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9425/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9450/3"}]}]},{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis","type":"EQUALS","system":"","value":"C71.%"},{"key":"diagnosis","type":"EQUALS","system":"","value":"C72.%"}]},{"operand":"OR","children":[{"key":"59847-4","type":"EQUALS","system":"","value":"9440/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9441/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9442/3"}]}]},{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis","type":"EQUALS","system":"","value":"C71.%"},{"key":"diagnosis","type":"EQUALS","system":"","value":"C72.%"}]},{"operand":"OR","children":[{"key":"59847-4","type":"EQUALS","system":"","value":"9381/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9382/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9401/3"},{"key":"59847-4","type":"EQUALS","system":"","value":"9451/3"}]}]}]}]}]},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; - - const AGE_AT_DIAGNOSIS_30_TO_70: &str = r#"{"ast": {"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"age_at_primary_diagnosis","type":"BETWEEN","system":"","value":{"min":30,"max":70}}]}]}]}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; - - const AGE_AT_DIAGNOSIS_LOWER_THAN_70: &str = r#"{"ast": {"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"age_at_primary_diagnosis","type":"BETWEEN","system":"","value":{"min":0,"max":70}}]}]}]}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; - - const C61_OR_MALE: &str = r#"{"ast": {"operand":"OR","children":[{"operand":"AND","children":[{"operand":"OR","children":[{"key":"diagnosis","type":"EQUALS","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","value":"C61"}]},{"operand":"OR","children":[{"key":"gender","type":"EQUALS","system":"","value":"male"}]}]}]}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; - - const ALL_GBN: &str = r#"{"ast":{"children":[{"key":"gender","system":"","type":"IN","value":["male","other"]},{"children":[{"key":"diagnosis","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","type":"EQUALS","value":"C25"},{"key":"diagnosis","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","type":"EQUALS","value":"C56"}],"de":"Diagnose ICD-10","en":"Diagnosis ICD-10","key":"diagnosis","operand":"OR"},{"key":"diagnosis_age_donor","system":"","type":"BETWEEN","value":{"max":100,"min":10}},{"key":"date_of_diagnosis","system":"","type":"BETWEEN","value":{"max":"2023-10-29T23:00:00.000Z","min":"2023-09-30T22:00:00.000Z"}},{"key":"bmi","system":"","type":"BETWEEN","value":{"max":100,"min":10}},{"key":"body_weight","system":"","type":"BETWEEN","value":{"max":1100,"min":10}},{"key":"fasting_status","system":"","type":"IN","value":["Sober","Other fasting status"]},{"key":"smoking_status","system":"","type":"IN","value":["Smoker","Never smoked"]},{"key":"donor_age","system":"","type":"BETWEEN","value":{"max":10000,"min":100}},{"key":"sample_kind","system":"","type":"IN","value":["blood-serum","blood-plasma","buffy-coat"]},{"key":"sampling_date","system":"","type":"BETWEEN","value":{"max":"2023-10-29T23:00:00.000Z","min":"2023-10-03T22:00:00.000Z"}},{"key":"storage_temperature","system":"","type":"IN","value":["temperature-18to-35","temperature-60to-85"]}],"de":"haupt","en":"main","key":"main","operand":"AND"},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; - - const SOME_GBN: &str = r#"{"ast":{"children":[{"key":"gender","system":"","type":"IN","value":["other","male"]},{"key":"diagnosis","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","type":"EQUALS","value":"C24"},{"key":"diagnosis_age_donor","system":"","type":"BETWEEN","value":{"max":11,"min":1}},{"key":"date_of_diagnosis","system":"","type":"BETWEEN","value":{"max":"2023-10-30T23:00:00.000Z","min":"2023-10-29T23:00:00.000Z"}},{"key":"bmi","system":"","type":"BETWEEN","value":{"max":111,"min":1}},{"key":"body_weight","system":"","type":"BETWEEN","value":{"max":1111,"min":110}},{"key":"fasting_status","system":"","type":"IN","value":["Sober","Not sober"]},{"key":"smoking_status","system":"","type":"IN","value":["Smoker","Never smoked"]},{"key":"donor_age","system":"","type":"BETWEEN","value":{"max":123,"min":1}},{"key":"sample_kind","system":"","type":"IN","value":["blood-serum","tissue-other"]},{"key":"sampling_date","system":"","type":"BETWEEN","value":{"max":"2023-10-30T23:00:00.000Z","min":"2023-10-29T23:00:00.000Z"}},{"key":"storage_temperature","system":"","type":"IN","value":["temperature2to10","temperatureGN"]}],"de":"haupt","en":"main","key":"main","operand":"AND"},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; - - const LENS2: &str = r#"{"ast":{"children":[{"children":[{"children":[{"key":"gender","system":"","type":"EQUALS","value":"male"},{"key":"gender","system":"","type":"EQUALS","value":"female"}],"operand":"OR"},{"children":[{"key":"diagnosis","system":"","type":"EQUALS","value":"C41"},{"key":"diagnosis","system":"","type":"EQUALS","value":"C50"}],"operand":"OR"},{"children":[{"key":"sample_kind","system":"","type":"EQUALS","value":"tissue-frozen"},{"key":"sample_kind","system":"","type":"EQUALS","value":"blood-serum"}],"operand":"OR"}],"operand":"AND"},{"children":[{"children":[{"key":"gender","system":"","type":"EQUALS","value":"male"}],"operand":"OR"},{"children":[{"key":"diagnosis","system":"","type":"EQUALS","value":"C41"},{"key":"diagnosis","system":"","type":"EQUALS","value":"C50"}],"operand":"OR"},{"children":[{"key":"sample_kind","system":"","type":"EQUALS","value":"liquid-other"},{"key":"sample_kind","system":"","type":"EQUALS","value":"rna"},{"key":"sample_kind","system":"","type":"EQUALS","value":"urine"}],"operand":"OR"},{"children":[{"key":"storage_temperature","system":"","type":"EQUALS","value":"temperatureRoom"},{"key":"storage_temperature","system":"","type":"EQUALS","value":"four_degrees"}],"operand":"OR"}],"operand":"AND"}],"operand":"OR"},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; - - const EMPTY: &str = - r#"{"ast":{"children":[],"operand":"OR"}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; - - #[test] - fn test_bbmri() { - // println!( - // "{:?}", - // bbmri(serde_json::from_str(AST).expect("Failed to deserialize JSON")) - // ); - - // println!( - // "{:?}", - // bbmri(serde_json::from_str(MALE_OR_FEMALE).expect("Failed to deserialize JSON")) - // ); - - // println!( - // "{:?}", - // bbmri(serde_json::from_str(ALL_GLIOMS).expect("Failed to deserialize JSON")) - // ); - - // println!( - // "{:?}", - // bbmri(serde_json::from_str(AGE_AT_DIAGNOSIS_30_TO_70).expect("Failed to deserialize JSON")) - // ); - - // println!( - // "{:?}", - // bbmri(serde_json::from_str(AGE_AT_DIAGNOSIS_LOWER_THAN_70).expect("Failed to deserialize JSON")) - // ); - - // println!( - // "{:?}", - // bbmri(serde_json::from_str(C61_OR_MALE).expect("Failed to deserialize JSON")) - // ); - - // println!( - // "{:?}", - // bbmri(serde_json::from_str(ALL_GBN).expect("Failed to deserialize JSON")) - // ); - - // println!(); - - // println!( - // "{:?}", - // bbmri(serde_json::from_str(SOME_GBN).expect("Failed to deserialize JSON")) - // ); - - // println!(); - - println!( - "{:?}", - generate_cql(serde_json::from_str(LENS2).expect("Failed to deserialize JSON"), Project::Bbmri) - ); - - // println!( - // "{:?}", - // bbmri(serde_json::from_str(EMPTY).expect("Failed to deserialize JSON")) - // ); - - pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(EMPTY).unwrap(), Project::Bbmri).unwrap(), include_str!("../resources/test/result_empty.cql").to_string()); - - } -} diff --git a/src/main.rs b/src/main.rs index 0269761..f7cbf46 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,7 +4,6 @@ mod beam; mod blaze; mod config; mod cql; -//mod cql_new; mod errors; mod graceful_shutdown; mod logger; From a2c1cceca6cc1479492777ecf451ef1858df7ac5 Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Tue, 7 May 2024 18:58:03 +0200 Subject: [PATCH 22/42] commented DKTK tests for unimplemented --- src/cql.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cql.rs b/src/cql.rs index 3c3c220..86f7a6a 100644 --- a/src/cql.rs +++ b/src/cql.rs @@ -393,9 +393,9 @@ mod test { todo!("Implement DKTK CQL generation and create files with results"); - pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(AST).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_ast.cql").to_string()); + //pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(AST).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_ast.cql").to_string()); - pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(ALL_GLIOMS).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_all_glioms.cql").to_string()); + //pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(ALL_GLIOMS).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_all_glioms.cql").to_string()); } From 7404d086e7297f3adbfa3c496525ae8f476e8deb Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Tue, 7 May 2024 19:02:51 +0200 Subject: [PATCH 23/42] commented out todo! --- src/cql.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cql.rs b/src/cql.rs index 86f7a6a..1063be8 100644 --- a/src/cql.rs +++ b/src/cql.rs @@ -391,7 +391,7 @@ mod test { fn test_dktk() { use crate::projects::{self, dktk::Dktk}; - todo!("Implement DKTK CQL generation and create files with results"); + // TODO Implement DKTK CQL generation and create files with results //pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(AST).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_ast.cql").to_string()); From 9814339af9a424280a82ba66c8f60180437f6ad3 Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Mon, 13 May 2024 13:03:13 +0200 Subject: [PATCH 24/42] task for ast queries --- Cargo.toml | 1 + src/blaze.rs | 15 +- src/cql.rs | 177 ++++++++++++++-------- src/projects/bbmri/body.json | 172 +++++++++++++++++++++ src/projects/bbmri/mod.rs | 62 ++++---- src/projects/dktk/body.json | 286 +++++++++++++++++++++++++++++++++++ src/projects/dktk/mod.rs | 25 +-- src/projects/mod.rs | 61 ++++---- src/projects/shared/mod.rs | 18 ++- src/task_processing.rs | 49 +++--- 10 files changed, 712 insertions(+), 154 deletions(-) create mode 100644 src/projects/bbmri/body.json create mode 100644 src/projects/dktk/body.json diff --git a/Cargo.toml b/Cargo.toml index f28c299..8062f29 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ once_cell = "1.18" # Command Line Interface clap = { version = "4.0", default_features = false, features = ["std", "env", "derive", "help"] } +uuid = "1.8.0" [features] default = [] diff --git a/src/blaze.rs b/src/blaze.rs index 3d1c885..83d1380 100644 --- a/src/blaze.rs +++ b/src/blaze.rs @@ -9,6 +9,7 @@ use crate::errors::FocusError; use crate::util; use crate::util::get_json_field; use crate::config::CONFIG; +use crate::ast; #[derive(Serialize, Deserialize, Debug, Default, Clone)] pub struct CqlQuery { @@ -17,6 +18,12 @@ pub struct CqlQuery { pub measure: Value } +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct AstQuery { + pub lang: String, + pub payload: ast::Ast, +} + pub async fn check_availability() -> bool { debug!("Checking Blaze availability..."); @@ -124,7 +131,13 @@ pub async fn run_cql_query(library: &Value, measure: &Value) -> Result Result { +pub fn parse_blaze_query_cql(task: &BeamTask) -> Result { let decoded = util::base64_decode(&task.body)?; serde_json::from_slice(&decoded).map_err(|e| FocusError::ParsingError(e.to_string())) } + +// This could be part of an impl of Cqlquery +pub fn parse_blaze_query_ast(task: &BeamTask) -> Result { + let decoded = util::base64_decode(&task.body)?; + serde_json::from_slice(&decoded).map_err(|e| FocusError::ParsingError(e.to_string())) +} \ No newline at end of file diff --git a/src/cql.rs b/src/cql.rs index 1063be8..680cf5b 100644 --- a/src/cql.rs +++ b/src/cql.rs @@ -1,36 +1,51 @@ use crate::ast; use crate::errors::FocusError; -use crate::projects::{Project, CriterionRole, CQL_TEMPLATES, MANDATORY_CODE_SYSTEMS, CODE_LISTS, CQL_SNIPPETS, CRITERION_CODE_LISTS, OBSERVATION_LOINC_CODE, SAMPLE_TYPE_WORKAROUNDS}; +use crate::projects::{ + CriterionRole, BODY, CODE_LISTS, CQL_SNIPPETS, CQL_TEMPLATE, CRITERION_CODE_LISTS, + MANDATORY_CODE_SYSTEMS, OBSERVATION_LOINC_CODE, SAMPLE_TYPE_WORKAROUNDS, +}; +use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; use chrono::offset::Utc; use chrono::DateTime; use indexmap::set::IndexSet; +use uuid::Uuid; + +pub fn generate_body(ast: ast::Ast) -> Result { + Ok(BASE64.encode( + BODY.clone() + .to_string() + .replace("{{LIBRARY_UUID}}", Uuid::new_v4().to_string().as_str()) + .replace("{{MEASURE_UUID}}", Uuid::new_v4().to_string().as_str()) + .replace( + "{{LIBRARY_ENCODED}}", + BASE64.encode(generate_cql(ast)?).as_str(), + ), + )) +} -pub fn generate_cql(ast: ast::Ast, project: impl Project) -> Result { +fn generate_cql(ast: ast::Ast) -> Result { let mut retrieval_criteria: String = "".to_string(); // main selection criteria (Patient) let mut filter_criteria: String = "".to_string(); // criteria for filtering specimens let mut lists: String = "".to_string(); // needed code lists, defined - let mut cql = CQL_TEMPLATES.get(project.name()).expect("missing project").to_string(); + let mut cql = CQL_TEMPLATE.clone().to_string(); let operator_str = match ast.ast.operand { ast::Operand::And => " and ", ast::Operand::Or => " or ", }; - let mut mandatory_codes = MANDATORY_CODE_SYSTEMS.get(project.name()) - .expect("non-existent project") - .clone(); + let mut mandatory_codes = MANDATORY_CODE_SYSTEMS.clone(); for (index, grandchild) in ast.ast.children.iter().enumerate() { process( grandchild.clone(), &mut retrieval_criteria, &mut filter_criteria, - &mut mandatory_codes, - project, + &mut mandatory_codes )?; // Only concatenate operator if it's not the last element @@ -48,17 +63,18 @@ pub fn generate_cql(ast: ast::Ast, project: impl Project) -> Result, - project: impl Project, + code_systems: &mut IndexSet<&str> ) -> Result<(), FocusError> { let mut retrieval_cond: String = "(".to_string(); let mut filter_cond: String = "".to_string(); @@ -83,19 +98,17 @@ pub fn process( let condition_key_trans = condition.key.as_str(); let condition_snippet = - CQL_SNIPPETS.get(&(condition_key_trans, CriterionRole::Query, project.name())); + CQL_SNIPPETS.get(&(condition_key_trans, CriterionRole::Query)); if let Some(snippet) = condition_snippet { let mut condition_string = (*snippet).to_string(); let mut filter_string: String = "".to_string(); - let filter_snippet = CQL_SNIPPETS.get(&( - condition_key_trans, - CriterionRole::Filter, - project.name(), - )); + let filter_snippet = + CQL_SNIPPETS.get(&(condition_key_trans, CriterionRole::Filter)); - let code_lists_option = CRITERION_CODE_LISTS.get(&(condition_key_trans, project.name())); + let code_lists_option = + CRITERION_CODE_LISTS.get(&(condition_key_trans)); if let Some(code_lists_vec) = code_lists_option { for (index, code_list) in code_lists_vec.iter().enumerate() { code_systems.insert(code_list); @@ -106,7 +119,7 @@ pub fn process( } } - if condition_string.contains("{{K}}") { + if condition_string.contains("{{K}}") { //observation loinc code, those only apply to query criteria, we don't filter specimens by observations let observation_code_option = OBSERVATION_LOINC_CODE.get(&condition_key_trans); @@ -119,12 +132,13 @@ pub fn process( } } - if let Some(filtret) = filter_snippet { + if let Some(filtret) = filter_snippet { filter_string = (*filtret).to_string(); } match condition.type_ { - ast::ConditionType::Between => { // both min and max values stated + ast::ConditionType::Between => { + // both min and max values stated match condition.value { ast::ConditionValue::DateRange(date_range) => { let datetime_str_min = date_range.min.as_str(); @@ -138,7 +152,8 @@ pub fn process( condition_string = condition_string.replace("{{D1}}", date_str_min.as_str()); filter_string = - filter_string.replace("{{D1}}", date_str_min.as_str()); // no condition needed, "" stays "" + filter_string.replace("{{D1}}", date_str_min.as_str()); + // no condition needed, "" stays "" } else { return Err(FocusError::AstInvalidDateFormat(date_range.min)); } @@ -153,7 +168,8 @@ pub fn process( condition_string = condition_string.replace("{{D2}}", date_str_max.as_str()); filter_string = - filter_string.replace("{{D2}}", date_str_max.as_str()); // no condition needed, "" stays "" + filter_string.replace("{{D2}}", date_str_max.as_str()); + // no condition needed, "" stays "" } else { return Err(FocusError::AstInvalidDateFormat(date_range.max)); } @@ -166,7 +182,8 @@ pub fn process( filter_string = filter_string .replace("{{D1}}", num_range.min.to_string().as_str()); // no condition needed, "" stays "" filter_string = filter_string - .replace("{{D2}}", num_range.max.to_string().as_str()); // no condition needed, "" stays "" + .replace("{{D2}}", num_range.max.to_string().as_str()); + // no condition needed, "" stays "" } _ => { return Err(FocusError::AstOperatorValueMismatch()); @@ -180,17 +197,22 @@ pub fn process( match condition.value { ast::ConditionValue::StringArray(string_array) => { let mut string_array_with_workarounds = string_array.clone(); - for value in string_array { - if let Some(additional_values) = SAMPLE_TYPE_WORKAROUNDS.get(value.as_str()){ + for value in string_array { + if let Some(additional_values) = + SAMPLE_TYPE_WORKAROUNDS.get(value.as_str()) + { for additional_value in additional_values { - string_array_with_workarounds.push((*additional_value).into()); + string_array_with_workarounds + .push((*additional_value).into()); } } } let mut condition_humongous_string = "(".to_string(); let mut filter_humongous_string = "(".to_string(); - for (index, string) in string_array_with_workarounds.iter().enumerate() { + for (index, string) in + string_array_with_workarounds.iter().enumerate() + { condition_humongous_string = condition_humongous_string + "(" + condition_string.as_str() @@ -226,7 +248,9 @@ pub fn process( ast::ConditionValue::String(string) => { let operator_str = " or "; let mut string_array_with_workarounds = vec![string.clone()]; - if let Some(additional_values) = SAMPLE_TYPE_WORKAROUNDS.get(string.as_str()){ + if let Some(additional_values) = + SAMPLE_TYPE_WORKAROUNDS.get(string.as_str()) + { for additional_value in additional_values { string_array_with_workarounds.push((*additional_value).into()); } @@ -234,18 +258,17 @@ pub fn process( let mut condition_humongous_string = "(".to_string(); let mut filter_humongous_string = "(".to_string(); - for (index, string) in string_array_with_workarounds.iter().enumerate() { + for (index, string) in string_array_with_workarounds.iter().enumerate() + { condition_humongous_string = condition_humongous_string + "(" + condition_string.as_str() + ")"; - condition_humongous_string = condition_humongous_string - .replace("{{C}}", string.as_str()); + condition_humongous_string = + condition_humongous_string.replace("{{C}}", string.as_str()); - filter_humongous_string = filter_humongous_string - + "(" - + filter_string.as_str() - + ")"; + filter_humongous_string = + filter_humongous_string + "(" + filter_string.as_str() + ")"; filter_humongous_string = filter_humongous_string.replace("{{C}}", string.as_str()); @@ -301,8 +324,7 @@ pub fn process( grandchild.clone(), &mut retrieval_cond, &mut filter_cond, - code_systems, - project.clone(), + code_systems )?; // Only concatenate operator if it's not the last element @@ -320,13 +342,12 @@ pub fn process( *retrieval_criteria += retrieval_cond.as_str(); - if !filter_cond.is_empty() { + if !filter_cond.is_empty() { *filter_criteria += "("; *filter_criteria += filter_cond.as_str(); *filter_criteria += ")"; - *filter_criteria = filter_criteria.replace(")(", ") or ("); - + *filter_criteria = filter_criteria.replace(")(", ") or ("); } Ok(()) @@ -355,7 +376,8 @@ mod test { const LENS2: &str = r#"{"ast":{"children":[{"children":[{"children":[{"key":"gender","system":"","type":"EQUALS","value":"male"},{"key":"gender","system":"","type":"EQUALS","value":"female"}],"operand":"OR"},{"children":[{"key":"diagnosis","system":"","type":"EQUALS","value":"C41"},{"key":"diagnosis","system":"","type":"EQUALS","value":"C50"}],"operand":"OR"},{"children":[{"key":"sample_kind","system":"","type":"EQUALS","value":"tissue-frozen"},{"key":"sample_kind","system":"","type":"EQUALS","value":"blood-serum"}],"operand":"OR"}],"operand":"AND"},{"children":[{"children":[{"key":"gender","system":"","type":"EQUALS","value":"male"}],"operand":"OR"},{"children":[{"key":"diagnosis","system":"","type":"EQUALS","value":"C41"},{"key":"diagnosis","system":"","type":"EQUALS","value":"C50"}],"operand":"OR"},{"children":[{"key":"sample_kind","system":"","type":"EQUALS","value":"liquid-other"},{"key":"sample_kind","system":"","type":"EQUALS","value":"rna"},{"key":"sample_kind","system":"","type":"EQUALS","value":"urine"}],"operand":"OR"},{"children":[{"key":"storage_temperature","system":"","type":"EQUALS","value":"temperatureRoom"},{"key":"storage_temperature","system":"","type":"EQUALS","value":"four_degrees"}],"operand":"OR"}],"operand":"AND"}],"operand":"OR"},"id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; - const EMPTY: &str = r#"{"ast":{"children":[],"operand":"OR"}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; + const EMPTY: &str = + r#"{"ast":{"children":[],"operand":"OR"}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; #[cfg(feature = "bbmri")] #[test] @@ -364,30 +386,57 @@ mod test { } #[test] - #[cfg(feature="bbmri")] + #[cfg(feature = "bbmri")] fn test_bbmri() { use crate::projects::{self, bbmri::Bbmri}; - pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(MALE_OR_FEMALE).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_male_or_female.cql").to_string()); - - pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(AGE_AT_DIAGNOSIS_30_TO_70).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_age_at_diagnosis_30_to_70.cql").to_string()); - - pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(AGE_AT_DIAGNOSIS_LOWER_THAN_70).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_age_at_diagnosis_lower_than_70.cql").to_string()); - - pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(C61_AND_MALE).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_c61_and_male.cql").to_string()); - - pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(ALL_GBN).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_all_gbn.cql").to_string()); - - pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(SOME_GBN).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_some_gbn.cql").to_string()); - - pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(LENS2).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_lens2.cql").to_string()); - - pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(EMPTY).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_empty.cql").to_string()); - + pretty_assertions::assert_eq!( + generate_cql(serde_json::from_str(MALE_OR_FEMALE).unwrap()).unwrap(), + include_str!("../resources/test/result_male_or_female.cql").to_string() + ); + + pretty_assertions::assert_eq!( + generate_cql( + serde_json::from_str(AGE_AT_DIAGNOSIS_30_TO_70).unwrap()) + .unwrap(), + include_str!("../resources/test/result_age_at_diagnosis_30_to_70.cql").to_string() + ); + + pretty_assertions::assert_eq!( + generate_cql( + serde_json::from_str(AGE_AT_DIAGNOSIS_LOWER_THAN_70).unwrap()) + .unwrap(), + include_str!("../resources/test/result_age_at_diagnosis_lower_than_70.cql").to_string() + ); + + pretty_assertions::assert_eq!( + generate_cql(serde_json::from_str(C61_AND_MALE).unwrap()).unwrap(), + include_str!("../resources/test/result_c61_and_male.cql").to_string() + ); + + pretty_assertions::assert_eq!( + generate_cql(serde_json::from_str(ALL_GBN).unwrap()).unwrap(), + include_str!("../resources/test/result_all_gbn.cql").to_string() + ); + + pretty_assertions::assert_eq!( + generate_cql(serde_json::from_str(SOME_GBN).unwrap()).unwrap(), + include_str!("../resources/test/result_some_gbn.cql").to_string() + ); + + pretty_assertions::assert_eq!( + generate_cql(serde_json::from_str(LENS2).unwrap()).unwrap(), + include_str!("../resources/test/result_lens2.cql").to_string() + ); + + pretty_assertions::assert_eq!( + generate_cql(serde_json::from_str(EMPTY).unwrap()).unwrap(), + include_str!("../resources/test/result_empty.cql").to_string() + ); } #[test] - #[cfg(feature="dktk")] + #[cfg(feature = "dktk")] fn test_dktk() { use crate::projects::{self, dktk::Dktk}; @@ -396,7 +445,5 @@ mod test { //pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(AST).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_ast.cql").to_string()); //pretty_assertions::assert_eq!(generate_cql(serde_json::from_str(ALL_GLIOMS).unwrap(), Bbmri).unwrap(), include_str!("../resources/test/result_all_glioms.cql").to_string()); - } - } diff --git a/src/projects/bbmri/body.json b/src/projects/bbmri/body.json new file mode 100644 index 0000000..955c121 --- /dev/null +++ b/src/projects/bbmri/body.json @@ -0,0 +1,172 @@ +{ + "lang": "cql", + "lib": { + "content": [ + { + "contentType": "text/cql", + "data": "{{LIBRARY_ENCODED}}" + } + ], + "resourceType": "Library", + "status": "active", + "type": { + "coding": [ + { + "code": "logic-library", + "system": "http://terminology.hl7.org/CodeSystem/library-type" + } + ] + }, + "url": "{{LIBRARY_UUID}}" + }, + "measure": { + "group": [ + { + "code": { + "text": "patient" + }, + "population": [ + { + "code": { + "coding": [ + { + "code": "initial-population", + "system": "http://terminology.hl7.org/CodeSystem/measure-population" + } + ] + }, + "criteria": { + "expression": "InInitialPopulation", + "language": "text/cql-identifier" + } + } + ], + "stratifier": [ + { + "code": { + "text": "gender" + }, + "criteria": { + "expression": "Gender", + "language": "text/cql" + } + }, + { + "code": { + "text": "Age" + }, + "criteria": { + "expression": "AgeClass", + "language": "text/cql" + } + }, + { + "code": { + "text": "Custodian" + }, + "criteria": { + "expression": "Custodian", + "language": "text/cql" + } + } + ] + }, + { + "code": { + "text": "diagnosis" + }, + "extension": [ + { + "url": "http://hl7.org/fhir/us/cqfmeasures/StructureDefinition/cqfm-populationBasis", + "valueCode": "Condition" + } + ], + "population": [ + { + "code": { + "coding": [ + { + "code": "initial-population", + "system": "http://terminology.hl7.org/CodeSystem/measure-population" + } + ] + }, + "criteria": { + "expression": "Diagnosis", + "language": "text/cql-identifier" + } + } + ], + "stratifier": [ + { + "code": { + "text": "diagnosis" + }, + "criteria": { + "expression": "DiagnosisCode", + "language": "text/cql-identifier" + } + } + ] + }, + { + "code": { + "text": "specimen" + }, + "extension": [ + { + "url": "http://hl7.org/fhir/us/cqfmeasures/StructureDefinition/cqfm-populationBasis", + "valueCode": "Specimen" + } + ], + "population": [ + { + "code": { + "coding": [ + { + "code": "initial-population", + "system": "http://terminology.hl7.org/CodeSystem/measure-population" + } + ] + }, + "criteria": { + "expression": "Specimen", + "language": "text/cql-identifier" + } + } + ], + "stratifier": [ + { + "code": { + "text": "sample_kind" + }, + "criteria": { + "expression": "SampleType", + "language": "text/cql" + } + } + ] + } + ], + "library": "{{LIBRARY_UUID}}", + "resourceType": "Measure", + "scoring": { + "coding": [ + { + "code": "cohort", + "system": "http://terminology.hl7.org/CodeSystem/measure-scoring" + } + ] + }, + "status": "active", + "subjectCodeableConcept": { + "coding": [ + { + "code": "Patient", + "system": "http://hl7.org/fhir/resource-types" + } + ] + }, + "url": "{{MEASURE_UUID}}" + } +} \ No newline at end of file diff --git a/src/projects/bbmri/mod.rs b/src/projects/bbmri/mod.rs index ee47594..25e6989 100644 --- a/src/projects/bbmri/mod.rs +++ b/src/projects/bbmri/mod.rs @@ -19,7 +19,7 @@ impl Project for Bbmri { Shared::append_observation_loinc_codes(&Shared, map) } - fn append_criterion_code_lists(&self, map: &mut HashMap<(&str, &ProjectName), Vec<&str>>) { + fn append_criterion_code_lists(&self, map: &mut HashMap<&str, Vec<&str>>) { for (key, value) in [ ("diagnosis", vec!["icd10", "icd10gm", "icd10gmnew"]), @@ -30,14 +30,12 @@ impl Project for Bbmri { ("storage_temperature", vec!["StorageTemperature"]), ("fasting_status", vec!["FastingStatus"]), ] { - map.insert( - (key, self.name()), - value - ); + map.insert(key, value ); } } - fn append_cql_snippets(&self, map: &mut HashMap<(&str, CriterionRole, &ProjectName), &str>) { + fn append_cql_snippets(&self, map: &mut HashMap<(&str, CriterionRole), &str>) { + Shared::append_cql_snippets(&Shared, map); for (key, value) in [ (("gender", CriterionRole::Query), "Patient.gender = '{{C}}'"), @@ -100,44 +98,48 @@ impl Project for Bbmri { ), ] { map.insert( - (key.0, key.1, self.name()), + (key.0, key.1), value ); } } - fn append_mandatory_code_lists(&self, map: &mut HashMap<&ProjectName, IndexSet<&str>>) { - let mut set = map.remove(self.name()).unwrap_or(IndexSet::new()); + fn append_mandatory_code_lists(&self, set: &mut IndexSet<&str>) { + Shared::append_mandatory_code_lists(&Shared, set); for value in ["icd10", "SampleMaterialType"] { set.insert(value); } - map.insert(self.name(), set); } - fn append_cql_templates(&self, map: &mut HashMap<&ProjectName, &str>) { - map.insert(self.name(), include_str!("template.cql")); + fn append_cql_template(&self, template: &mut String) { + template.push_str(include_str!("template.cql")); } fn name(&self) -> &'static ProjectName { &ProjectName::Bbmri } -} -pub fn append_sample_type_workarounds(map: &mut HashMap<&str, Vec<&str>>) { - for (key, value) in - [ - ("blood-plasma", vec!["plasma-edta", "plasma-citrat", "plasma-heparin", "plasma-cell-free", "plasma-other", "plasma"]), - ("blood-serum", vec!["serum"]), - ("tissue-ffpe", vec!["tumor-tissue-ffpe", "normal-tissue-ffpe", "other-tissue-ffpe", "tissue-formalin"]), - ("tissue-frozen", vec!["tumor-tissue-frozen", "normal-tissue-frozen", "other-tissue-frozen"]), - ("dna", vec!["cf-dna", "g-dna"]), - ("tissue-other", vec!["tissue-paxgene-or-else", "tissue"]), - ("derivative-other", vec!["derivative"]), - ("liquid-other", vec!["liquid"]), - ] { - map.insert( - key, - value - ); - } + fn append_body(&self, body: &mut String) { + body.push_str(include_str!("body.json")); + } + + fn append_sample_type_workarounds(&self, map: &mut HashMap<&str, Vec<&str>>) { + for (key, value) in + [ + ("blood-plasma", vec!["plasma-edta", "plasma-citrat", "plasma-heparin", "plasma-cell-free", "plasma-other", "plasma"]), + ("blood-serum", vec!["serum"]), + ("tissue-ffpe", vec!["tumor-tissue-ffpe", "normal-tissue-ffpe", "other-tissue-ffpe", "tissue-formalin"]), + ("tissue-frozen", vec!["tumor-tissue-frozen", "normal-tissue-frozen", "other-tissue-frozen"]), + ("dna", vec!["cf-dna", "g-dna"]), + ("tissue-other", vec!["tissue-paxgene-or-else", "tissue"]), + ("derivative-other", vec!["derivative"]), + ("liquid-other", vec!["liquid"]), + ] { + map.insert( + key, + value + ); + } + } + } \ No newline at end of file diff --git a/src/projects/dktk/body.json b/src/projects/dktk/body.json new file mode 100644 index 0000000..e88b756 --- /dev/null +++ b/src/projects/dktk/body.json @@ -0,0 +1,286 @@ +{ + "lang": "cql", + "lib": { + "content": [ + { + "contentType": "text/cql", + "data": "{{LIBRARY_ENCODED}}" + } + ], + "resourceType": "Library", + "status": "active", + "type": { + "coding": [ + { + "code": "logic-library", + "system": "http://terminology.hl7.org/CodeSystem/library-type" + } + ] + }, + "url": "{{LIBRARY_UUID}}" + }, + "measure": { + "group": [ + { + "code": { + "text": "patients" + }, + "population": [ + { + "code": { + "coding": [ + { + "code": "initial-population", + "system": "http://terminology.hl7.org/CodeSystem/measure-population" + } + ] + }, + "criteria": { + "expression": "InInitialPopulation", + "language": "text/cql-identifier" + } + } + ], + "stratifier": [ + { + "code": { + "text": "gender" + }, + "criteria": { + "expression": "Gender", + "language": "text/cql" + } + }, + { + "code": { + "text": "75186-7" + }, + "criteria": { + "expression": "Deceased", + "language": "text/cql" + } + }, + { + "code": { + "text": "Age" + }, + "criteria": { + "expression": "AgeClass", + "language": "text/cql" + } + } + ] + }, + { + "code": { + "text": "diagnosis" + }, + "extension": [ + { + "url": "http://hl7.org/fhir/us/cqfmeasures/StructureDefinition/cqfm-populationBasis", + "valueCode": "Condition" + } + ], + "population": [ + { + "code": { + "coding": [ + { + "code": "initial-population", + "system": "http://terminology.hl7.org/CodeSystem/measure-population" + } + ] + }, + "criteria": { + "expression": "Diagnosis", + "language": "text/cql-identifier" + } + } + ], + "stratifier": [ + { + "code": { + "text": "diagnosis" + }, + "criteria": { + "expression": "DiagnosisCode", + "language": "text/cql-identifier" + } + } + ] + }, + { + "code": { + "text": "specimen" + }, + "extension": [ + { + "url": "http://hl7.org/fhir/us/cqfmeasures/StructureDefinition/cqfm-populationBasis", + "valueCode": "Specimen" + } + ], + "population": [ + { + "code": { + "coding": [ + { + "code": "initial-population", + "system": "http://terminology.hl7.org/CodeSystem/measure-population" + } + ] + }, + "criteria": { + "expression": "Specimen", + "language": "text/cql-identifier" + } + } + ], + "stratifier": [ + { + "code": { + "text": "sample_kind" + }, + "criteria": { + "expression": "SampleType", + "language": "text/cql" + } + } + ] + }, + { + "code": { + "text": "procedures" + }, + "extension": [ + { + "url": "http://hl7.org/fhir/us/cqfmeasures/StructureDefinition/cqfm-populationBasis", + "valueCode": "Procedure" + } + ], + "population": [ + { + "code": { + "coding": [ + { + "code": "initial-population", + "system": "http://terminology.hl7.org/CodeSystem/measure-population" + } + ] + }, + "criteria": { + "expression": "Procedure", + "language": "text/cql-identifier" + } + } + ], + "stratifier": [ + { + "code": { + "text": "ProcedureType" + }, + "criteria": { + "expression": "ProcedureType", + "language": "text/cql" + } + } + ] + }, + { + "code": { + "text": "medicationStatements" + }, + "extension": [ + { + "url": "http://hl7.org/fhir/us/cqfmeasures/StructureDefinition/cqfm-populationBasis", + "valueCode": "MedicationStatement" + } + ], + "population": [ + { + "code": { + "coding": [ + { + "code": "initial-population", + "system": "http://terminology.hl7.org/CodeSystem/measure-population" + } + ] + }, + "criteria": { + "expression": "MedicationStatement", + "language": "text/cql-identifier" + } + } + ], + "stratifier": [ + { + "code": { + "text": "MedicationType" + }, + "criteria": { + "expression": "ProcedureType", + "language": "text/cql" + } + } + ] + }, + { + "code": { + "text": "Histo" + }, + "extension": [ + { + "url": "http://hl7.org/fhir/us/cqfmeasures/StructureDefinition/cqfm-populationBasis", + "valueCode": "Observation" + } + ], + "population": [ + { + "code": { + "coding": [ + { + "code": "initial-population", + "system": "http://terminology.hl7.org/CodeSystem/measure-population" + } + ] + }, + "criteria": { + "expression": "Histo", + "language": "text/cql-identifier" + } + } + ], + "stratifier": [ + { + "code": { + "text": "Histlogoies" + }, + "criteria": { + "expression": "Histlogoy", + "language": "text/cql-identifier" + } + } + ] + } + ], + "library": "{{LIBRARY_UUID}}", + "resourceType": "Measure", + "scoring": { + "coding": [ + { + "code": "cohort", + "system": "http://terminology.hl7.org/CodeSystem/measure-scoring" + } + ] + }, + "status": "active", + "subjectCodeableConcept": { + "coding": [ + { + "code": "Patient", + "system": "http://hl7.org/fhir/resource-types" + } + ] + }, + "url": "{{MEASURE_UUID}}" + } +} \ No newline at end of file diff --git a/src/projects/dktk/mod.rs b/src/projects/dktk/mod.rs index b518ab1..f58842c 100644 --- a/src/projects/dktk/mod.rs +++ b/src/projects/dktk/mod.rs @@ -19,25 +19,32 @@ impl Project for Dktk { Shared::append_observation_loinc_codes(&Shared, map) } - fn append_criterion_code_lists(&self, _map: &mut HashMap<(&str, &ProjectName), Vec<&str>>) { } + fn append_criterion_code_lists(&self, _map: &mut HashMap<&str, Vec<&str>>) { } - fn append_cql_snippets(&self, _map: &mut HashMap<(&str, CriterionRole, &ProjectName), &str>) { } + fn append_cql_snippets(&self, _map: &mut HashMap<(&str, CriterionRole), &str>) { } - fn append_mandatory_code_lists(&self, map: &mut HashMap<&ProjectName, IndexSet<&str>>) { - let mut set = map.remove(self.name()).unwrap_or(IndexSet::new()); + fn append_mandatory_code_lists(&self, map: &mut IndexSet<&str>) { + //let mut set = map.remove(self.name()).unwrap_or(IndexSet::new()); for value in ["icd10", "SampleMaterialType", "loinc"] { - set.insert(value); + map.insert(value); } - map.insert(self.name(), set); + //map.insert(self.name(), set); } - fn append_cql_templates(&self, map: &mut HashMap<&ProjectName, &str>) { - //map.insert(&Self, include_str!("template.cql")); + fn append_cql_template(&self, _template: &mut String) { + //include_str!("template.cql") } fn name(&self) -> &'static ProjectName { &ProjectName::Dktk } + + fn append_body(&self, str: &mut String) { + str.push_str(include_str!("body.json")); + } + + fn append_sample_type_workarounds(&self, _map: &mut HashMap<&str, Vec<&str>>) { + //none + } } -pub fn append_sample_type_workarounds(map: &mut HashMap<&str, Vec<&str>>) {} \ No newline at end of file diff --git a/src/projects/mod.rs b/src/projects/mod.rs index 3ec5b40..7261ed8 100644 --- a/src/projects/mod.rs +++ b/src/projects/mod.rs @@ -14,11 +14,13 @@ pub(crate) mod dktk; pub(crate) trait Project: PartialEq + Eq + PartialOrd + Ord + Clone + Copy + Hash { fn append_code_lists(&self, _map: &mut HashMap<&'static str, &'static str>); fn append_observation_loinc_codes(&self, _map: &mut HashMap<&'static str, &'static str>); - fn append_criterion_code_lists(&self, _map: &mut HashMap<(&str, &ProjectName), Vec<&str>>); - fn append_cql_snippets(&self, _map: &mut HashMap<(&str, CriterionRole, &ProjectName), &str>); - fn append_mandatory_code_lists(&self, map: &mut HashMap<&ProjectName, IndexSet<&str>>); - fn append_cql_templates(&self, map: &mut HashMap<&ProjectName, &str>); + fn append_criterion_code_lists(&self, _map: &mut HashMap<&str, Vec<&str>>); + fn append_cql_snippets(&self, _map: &mut HashMap<(&str, CriterionRole), &str>); + fn append_mandatory_code_lists(&self, set: &mut IndexSet<&str>); + fn append_cql_template(&self, _template: &mut String); fn name(&self) -> &'static ProjectName; + fn append_body(&self, _body: &mut String); + fn append_sample_type_workarounds(&self, _map: &mut HashMap<&str, Vec<&str>>); } #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Hash, Copy)] @@ -30,16 +32,6 @@ pub enum ProjectName { NotSpecified } -// impl Display for ProjectName { -// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { -// let name = match self { -// ProjectName::Bbmri => "bbmri", -// ProjectName::Dktk => "dktk" -// }; -// write!(f, "{name}") -// } -// } - #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Hash)] pub enum CriterionRole { Query, @@ -76,16 +68,16 @@ pub static SAMPLE_TYPE_WORKAROUNDS: Lazy>> = Lazy::new(| let mut map: HashMap<&'static str, Vec<&'static str>> = HashMap::new(); #[cfg(feature="bbmri")] - bbmri::append_sample_type_workarounds(&mut map); + bbmri::Bbmri.append_sample_type_workarounds(&mut map); #[cfg(feature="dktk")] - dktk::append_sample_type_workarounds(&mut map); + dktk::Dktk.append_sample_type_workarounds(&mut map); map }); // code lists needed depending on the criteria selected -pub static CRITERION_CODE_LISTS: Lazy>> = Lazy::new(|| { +pub static CRITERION_CODE_LISTS: Lazy>> = Lazy::new(|| { let mut map = HashMap::new(); #[cfg(feature="bbmri")] @@ -98,7 +90,7 @@ pub static CRITERION_CODE_LISTS: Lazy>> }); // CQL snippets depending on the criteria -pub static CQL_SNIPPETS: Lazy> = Lazy::new(|| { +pub static CQL_SNIPPETS: Lazy> = Lazy::new(|| { let mut map = HashMap::new(); #[cfg(feature="bbmri")] @@ -110,26 +102,39 @@ pub static CQL_SNIPPETS: Lazy map }); -pub static MANDATORY_CODE_SYSTEMS: Lazy>> = Lazy::new(|| { - let mut map = HashMap::new(); +pub static MANDATORY_CODE_SYSTEMS: Lazy> = Lazy::new(|| { + let mut set = IndexSet::new(); #[cfg(feature="bbmri")] - bbmri::Bbmri.append_mandatory_code_lists(&mut map); + bbmri::Bbmri.append_mandatory_code_lists(&mut set); #[cfg(feature="dktk")] - dktk::Dktk.append_mandatory_code_lists(&mut map); + dktk::Dktk.append_mandatory_code_lists(&mut set); - map + set }); -pub static CQL_TEMPLATES: Lazy> = Lazy::new(|| { - let mut map = HashMap::new(); +pub static CQL_TEMPLATE: Lazy<&'static str> = Lazy::new(|| { + let mut template = String::new(); #[cfg(feature="bbmri")] - bbmri::Bbmri.append_cql_templates(&mut map); + bbmri::Bbmri.append_cql_template(&mut template); #[cfg(feature="dktk")] - dktk::Dktk.append_cql_templates(&mut map); + dktk::Dktk.append_cql_template(&mut template); - map + template.leak() }); + +pub static BODY: Lazy<&'static str> = Lazy::new(|| { + + let mut body = String::new(); + + #[cfg(feature="bbmri")] + bbmri::Bbmri.append_body(&mut body); + + #[cfg(feature="dktk")] + dktk::Dktk.append_body(&mut body); + + body.leak() +}); \ No newline at end of file diff --git a/src/projects/shared/mod.rs b/src/projects/shared/mod.rs index a6b4728..5b0a9b3 100644 --- a/src/projects/shared/mod.rs +++ b/src/projects/shared/mod.rs @@ -43,23 +43,33 @@ impl Project for Shared { ]); } - fn append_criterion_code_lists(&self, _map: &mut HashMap<(&str, &ProjectName), Vec<&str>>) { + fn append_criterion_code_lists(&self, _map: &mut HashMap<&str, Vec<&str>>) { // none } - fn append_cql_snippets(&self, _map: &mut HashMap<(&str, CriterionRole, &ProjectName), &str>) { + fn append_cql_snippets(&self, _map: &mut HashMap<(&str, CriterionRole), &str>) { // none } - fn append_mandatory_code_lists(&self, _map: &mut HashMap<&ProjectName, IndexSet<&str>>) { + fn append_mandatory_code_lists(&self, _set: &mut IndexSet<&str>) { // none } - fn append_cql_templates(&self, _map: &mut HashMap<&ProjectName, &str>) { + fn append_cql_template(&self, _template: &mut String) { + // none + } + + fn append_body(&self, _body: &mut String) { // none } fn name(&self) -> &'static ProjectName { &ProjectName::NotSpecified } + + fn append_sample_type_workarounds(&self, _map: &mut HashMap<&str, Vec<&str>>) { + //none + } + + } \ No newline at end of file diff --git a/src/task_processing.rs b/src/task_processing.rs index 27af77b..b41463f 100644 --- a/src/task_processing.rs +++ b/src/task_processing.rs @@ -1,11 +1,13 @@ -use std::{sync::Arc, collections::HashMap, time::Duration}; +use std::{collections::HashMap, sync::Arc, time::Duration}; use base64::{engine::general_purpose, Engine as _}; use laplace_rs::ObfCache; -use tokio::sync::{mpsc, Semaphore, Mutex}; -use tracing::{error, warn, debug, info, Instrument, info_span}; +use tokio::sync::{mpsc, Mutex, Semaphore}; +use tracing::{debug, error, info, info_span, warn, Instrument}; -use crate::{ReportCache, errors::FocusError, beam, BeamTask, BeamResult, run_exporter_query, config::{EndpointType, CONFIG}, run_cql_query, intermediate_rep, ast, run_intermediate_rep_query, Metadata, blaze::parse_blaze_query, util}; +use crate::{ + ast, beam, blaze::{parse_blaze_query_ast, parse_blaze_query_cql}, config::{EndpointType, CONFIG}, cql, errors::FocusError, intermediate_rep, run_cql_query, run_exporter_query, run_intermediate_rep_query, util, BeamResult, BeamTask, Metadata, ReportCache +}; const NUM_WORKERS: usize = 3; const WORKER_BUFFER: usize = 32; @@ -20,7 +22,7 @@ pub fn spawn_task_workers(report_cache: ReportCache) -> TaskQueue { })); let report_cache: Arc> = Arc::new(Mutex::new(report_cache)); - + tokio::spawn(async move { let semaphore = Arc::new(Semaphore::new(NUM_WORKERS)); while let Some(task) = rx.recv().await { @@ -29,7 +31,9 @@ pub fn spawn_task_workers(report_cache: ReportCache) -> TaskQueue { let local_obf_cache = obf_cache.clone(); tokio::spawn(async move { let span = info_span!("task handling", %task.id); - handle_beam_task(task, local_obf_cache, local_report_cache).instrument(span).await; + handle_beam_task(task, local_obf_cache, local_report_cache) + .instrument(span) + .await; drop(permit) }); } @@ -38,11 +42,16 @@ pub fn spawn_task_workers(report_cache: ReportCache) -> TaskQueue { tx } -async fn handle_beam_task(task: BeamTask, local_obf_cache: Arc>, local_report_cache: Arc>) { +async fn handle_beam_task( + task: BeamTask, + local_obf_cache: Arc>, + local_report_cache: Arc>, +) { let task_claiming = beam::claim_task(&task); - let mut task_processing = std::pin::pin!(process_task(&task, local_obf_cache, local_report_cache)); + let mut task_processing = + std::pin::pin!(process_task(&task, local_obf_cache, local_report_cache)); let task_result = tokio::select! { - // If task task processing happens before claiming is done drop the task claiming future + // If task task processing happens before claiming is done drop the task claiming future task_processed = &mut task_processing => { task_processed }, @@ -71,9 +80,7 @@ async fn handle_beam_task(task: BeamTask, local_obf_cache: Arc>, match beam::answer_task(&result).await { Ok(_) => break, Err(FocusError::ConfigurationError(s)) => { - error!( - "FATAL: Unable to report back to Beam due to a configuration issue: {s}" - ); + error!("FATAL: Unable to report back to Beam due to a configuration issue: {s}"); } Err(FocusError::UnableToAnswerTask(e)) => { warn!("Unable to report task result to Beam: {e}. Retrying (attempt {attempt}/{MAX_TRIES})."); @@ -103,7 +110,7 @@ async fn process_task( CONFIG.beam_app_id_long.clone(), vec![task.from.clone()], task.id, - "healthy".into() + "healthy".into(), )); } @@ -113,7 +120,14 @@ async fn process_task( } if CONFIG.endpoint_type == EndpointType::Blaze { - let query = parse_blaze_query(task)?; + let mut query_maybe = parse_blaze_query_cql(task); + if let Err(_e) = query_maybe { + let query_string = cql::generate_body(parse_blaze_query_ast(task)?.payload)?; + query_maybe = serde_json::from_str(&query_string).map_err(|e| FocusError::ParsingError(e.to_string())) + } + + let query = query_maybe.unwrap(); + if query.lang == "cql" { // TODO: Change query.lang to an enum @@ -133,13 +147,14 @@ async fn process_task( } else if CONFIG.endpoint_type == EndpointType::Omop { let decoded = util::base64_decode(&task.body)?; let intermediate_rep_query: intermediate_rep::IntermediateRepQuery = - serde_json::from_slice(&decoded).map_err(|e| FocusError::ParsingError(e.to_string()))?; + serde_json::from_slice(&decoded) + .map_err(|e| FocusError::ParsingError(e.to_string()))?; //TODO check that the language is ast let query_decoded = general_purpose::STANDARD .decode(intermediate_rep_query.query) .map_err(FocusError::DecodeError)?; - let ast: ast::Ast = - serde_json::from_slice(&query_decoded).map_err(|e| FocusError::ParsingError(e.to_string()))?; + let ast: ast::Ast = serde_json::from_slice(&query_decoded) + .map_err(|e| FocusError::ParsingError(e.to_string()))?; Ok(run_intermediate_rep_query(task, ast).await)? } else { From a00f9bff0ba5f3a5f02095759873003c1eeee4b4 Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Mon, 13 May 2024 13:51:59 +0200 Subject: [PATCH 25/42] removed some imports --- src/cql.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cql.rs b/src/cql.rs index 680cf5b..4892ae1 100644 --- a/src/cql.rs +++ b/src/cql.rs @@ -388,7 +388,7 @@ mod test { #[test] #[cfg(feature = "bbmri")] fn test_bbmri() { - use crate::projects::{self, bbmri::Bbmri}; + pretty_assertions::assert_eq!( generate_cql(serde_json::from_str(MALE_OR_FEMALE).unwrap()).unwrap(), @@ -438,7 +438,7 @@ mod test { #[test] #[cfg(feature = "dktk")] fn test_dktk() { - use crate::projects::{self, dktk::Dktk}; + //use crate::projects::{self, dktk::Dktk}; // TODO Implement DKTK CQL generation and create files with results From e00c964af7d4c8497a2a1a5c81cbb3ba70a81b4b Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Mon, 13 May 2024 13:57:22 +0200 Subject: [PATCH 26/42] put the input back in --- src/cql.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cql.rs b/src/cql.rs index 4892ae1..2427dae 100644 --- a/src/cql.rs +++ b/src/cql.rs @@ -388,7 +388,7 @@ mod test { #[test] #[cfg(feature = "bbmri")] fn test_bbmri() { - + use crate::projects::{self, bbmri::Bbmri}; pretty_assertions::assert_eq!( generate_cql(serde_json::from_str(MALE_OR_FEMALE).unwrap()).unwrap(), From ea6007298bf8483162157a5df78f0209d04e648c Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Wed, 22 May 2024 15:42:18 +0200 Subject: [PATCH 27/42] added sample exporter tasks --- README.md | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 10a72d9..8b5ef97 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Focus -Focus is a Samply component ran on the sites, which distributes tasks from Beam.Proxy to the applications on the site and re-transmits the results through Samply.Beam. +Focus is a Samply component ran on the sites, which distributes tasks from [Beam.Proxy](https://github.com/samply/beam/) to the applications on the site and re-transmits the results through [Samply.Beam](https://github.com/samply/beam/). -It is possible to specify Blaze queries whose results are to be cached to speed up retrieval. The cached results expire after 24 hours. +It is possible to specify [Blaze](https://github.com/samply/blaze) queries whose results are to be cached to speed up retrieval. The cached results expire after 24 hours. ## Installation @@ -66,12 +66,26 @@ Creating a sample focus healthcheck task using CURL (body can be any string and curl -v -X POST -H "Content-Type: application/json" --data '{"id":"7fffefff-ffef-fcff-feef-feffffffffff","from":"app1.proxy1.broker","to":["app1.proxy1.broker"],"ttl":"10s","failure_strategy":{"retry":{"backoff_millisecs":1000,"max_tries":5}},"metadata":{"project":"focus-healthcheck"},"body":"wie geht es"}' -H "Authorization: ApiKey app1.proxy1.broker App1Secret" http://localhost:8081/v1/tasks ``` -Creating a sample task containing a Blaze query using CURL: +Creating a sample task containing a [Blaze](https://github.com/samply/blaze) query using CURL: ```bash curl -v -X POST -H "Content-Type: application/json" --data '{"id":"7fffefff-ffef-fcff-feef-fefbffffeeff","from":"app1.proxy1.broker","to":["app1.proxy1.broker"],"ttl":"10s","failure_strategy":{"retry":{"backoff_millisecs":1000,"max_tries":5}},"metadata":{"project":"exliquid"},"body":"ewoJImxhbmciOiAiY3FsIiwKCSJsaWIiOiB7CgkJImNvbnRlbnQiOiBbCgkJCXsKCQkJCSJjb250ZW50VHlwZSI6ICJ0ZXh0L2NxbCIsCgkJCQkiZGF0YSI6ICJiR2xpY21GeWVTQlNaWFJ5YVdWMlpRcDFjMmx1WnlCR1NFbFNJSFpsY25OcGIyNGdKelF1TUM0d0p3cHBibU5zZFdSbElFWklTVkpJWld4d1pYSnpJSFpsY25OcGIyNGdKelF1TUM0d0p3b0tZMjlrWlhONWMzUmxiU0JzYjJsdVl6b2dKMmgwZEhBNkx5OXNiMmx1WXk1dmNtY25DbU52WkdWemVYTjBaVzBnYVdOa01UQTZJQ2RvZEhSd09pOHZhR3czTG05eVp5OW1hR2x5TDNOcFpDOXBZMlF0TVRBbkNtTnZaR1Z6ZVhOMFpXMGdVMkZ0Y0d4bFRXRjBaWEpwWVd4VWVYQmxPaUFuYUhSMGNITTZMeTltYUdseUxtSmliWEpwTG1SbEwwTnZaR1ZUZVhOMFpXMHZVMkZ0Y0d4bFRXRjBaWEpwWVd4VWVYQmxKd29LQ21OdmJuUmxlSFFnVUdGMGFXVnVkQW9LUWtKTlVrbGZVMVJTUVZSZlIwVk9SRVZTWDFOVVVrRlVTVVpKUlZJS0NrSkNUVkpKWDFOVVVrRlVYMFJGUmw5VFVFVkRTVTFGVGdwcFppQkpia2x1YVhScFlXeFFiM0IxYkdGMGFXOXVJSFJvWlc0Z1cxTndaV05wYldWdVhTQmxiSE5sSUh0OUlHRnpJRXhwYzNROFUzQmxZMmx0Wlc0K0NncENRazFTU1Y5VFZGSkJWRjlUUVUxUVRFVmZWRmxRUlY5VFZGSkJWRWxHU1VWU0NncENRazFTU1Y5VFZGSkJWRjlEVlZOVVQwUkpRVTVmVTFSU1FWUkpSa2xGVWdvS1FrSk5Va2xmVTFSU1FWUmZSRWxCUjA1UFUwbFRYMU5VVWtGVVNVWkpSVklLQ2tKQ1RWSkpYMU5VVWtGVVgwRkhSVjlUVkZKQlZFbEdTVVZTQ2dwQ1FrMVNTVjlUVkZKQlZGOUVSVVpmU1U1ZlNVNUpWRWxCVEY5UVQxQlZURUZVU1U5T0NuUnlkV1U9IgoJCQl9CgkJXSwKCQkicmVzb3VyY2VUeXBlIjogIkxpYnJhcnkiLAoJCSJzdGF0dXMiOiAiYWN0aXZlIiwKCQkidHlwZSI6IHsKCQkJImNvZGluZyI6IFsKCQkJCXsKCQkJCQkiY29kZSI6ICJsb2dpYy1saWJyYXJ5IiwKCQkJCQkic3lzdGVtIjogImh0dHA6Ly90ZXJtaW5vbG9neS5obDcub3JnL0NvZGVTeXN0ZW0vbGlicmFyeS10eXBlIgoJCQkJfQoJCQldCgkJfSwKCQkidXJsIjogInVybjp1dWlkOjdmZjUzMmFkLTY5ZTQtNDhlZC1hMmQzLTllZmFmYjYwOWY2MiIKCX0sCgkibWVhc3VyZSI6IHsKCQkiZ3JvdXAiOiBbCgkJCXsKCQkJCSJjb2RlIjogewoJCQkJCSJ0ZXh0IjogInBhdGllbnRzIgoJCQkJfSwKCQkJCSJwb3B1bGF0aW9uIjogWwoJCQkJCXsKCQkJCQkJImNvZGUiOiB7CgkJCQkJCQkiY29kaW5nIjogWwoJCQkJCQkJCXsKCQkJCQkJCQkJImNvZGUiOiAiaW5pdGlhbC1wb3B1bGF0aW9uIiwKCQkJCQkJCQkJInN5c3RlbSI6ICJodHRwOi8vdGVybWlub2xvZ3kuaGw3Lm9yZy9Db2RlU3lzdGVtL21lYXN1cmUtcG9wdWxhdGlvbiIKCQkJCQkJCQl9CgkJCQkJCQldCgkJCQkJCX0sCgkJCQkJCSJjcml0ZXJpYSI6IHsKCQkJCQkJCSJleHByZXNzaW9uIjogIkluSW5pdGlhbFBvcHVsYXRpb24iLAoJCQkJCQkJImxhbmd1YWdlIjogInRleHQvY3FsLWlkZW50aWZpZXIiCgkJCQkJCX0KCQkJCQl9CgkJCQldLAoJCQkJInN0cmF0aWZpZXIiOiBbCgkJCQkJewoJCQkJCQkiY29kZSI6IHsKCQkJCQkJCSJ0ZXh0IjogIkdlbmRlciIKCQkJCQkJfSwKCQkJCQkJImNyaXRlcmlhIjogewoJCQkJCQkJImV4cHJlc3Npb24iOiAiR2VuZGVyIiwKCQkJCQkJCSJsYW5ndWFnZSI6ICJ0ZXh0L2NxbCIKCQkJCQkJfQoJCQkJCX0sCgkJCQkJewoJCQkJCQkiY29kZSI6IHsKCQkJCQkJCSJ0ZXh0IjogIkFnZSIKCQkJCQkJfSwKCQkJCQkJImNyaXRlcmlhIjogewoJCQkJCQkJImV4cHJlc3Npb24iOiAiQWdlQ2xhc3MiLAoJCQkJCQkJImxhbmd1YWdlIjogInRleHQvY3FsIgoJCQkJCQl9CgkJCQkJfSwKCQkJCQl7CgkJCQkJCSJjb2RlIjogewoJCQkJCQkJInRleHQiOiAiQ3VzdG9kaWFuIgoJCQkJCQl9LAoJCQkJCQkiY3JpdGVyaWEiOiB7CgkJCQkJCQkiZXhwcmVzc2lvbiI6ICJDdXN0b2RpYW4iLAoJCQkJCQkJImxhbmd1YWdlIjogInRleHQvY3FsIgoJCQkJCQl9CgkJCQkJfQoJCQkJXQoJCQl9LAoJCQl7CgkJCQkiY29kZSI6IHsKCQkJCQkidGV4dCI6ICJkaWFnbm9zaXMiCgkJCQl9LAoJCQkJImV4dGVuc2lvbiI6IFsKCQkJCQl7CgkJCQkJCSJ1cmwiOiAiaHR0cDovL2hsNy5vcmcvZmhpci91cy9jcWZtZWFzdXJlcy9TdHJ1Y3R1cmVEZWZpbml0aW9uL2NxZm0tcG9wdWxhdGlvbkJhc2lzIiwKCQkJCQkJInZhbHVlQ29kZSI6ICJDb25kaXRpb24iCgkJCQkJfQoJCQkJXSwKCQkJCSJwb3B1bGF0aW9uIjogWwoJCQkJCXsKCQkJCQkJImNvZGUiOiB7CgkJCQkJCQkiY29kaW5nIjogWwoJCQkJCQkJCXsKCQkJCQkJCQkJImNvZGUiOiAiaW5pdGlhbC1wb3B1bGF0aW9uIiwKCQkJCQkJCQkJInN5c3RlbSI6ICJodHRwOi8vdGVybWlub2xvZ3kuaGw3Lm9yZy9Db2RlU3lzdGVtL21lYXN1cmUtcG9wdWxhdGlvbiIKCQkJCQkJCQl9CgkJCQkJCQldCgkJCQkJCX0sCgkJCQkJCSJjcml0ZXJpYSI6IHsKCQkJCQkJCSJleHByZXNzaW9uIjogIkRpYWdub3NpcyIsCgkJCQkJCQkibGFuZ3VhZ2UiOiAidGV4dC9jcWwtaWRlbnRpZmllciIKCQkJCQkJfQoJCQkJCX0KCQkJCV0sCgkJCQkic3RyYXRpZmllciI6IFsKCQkJCQl7CgkJCQkJCSJjb2RlIjogewoJCQkJCQkJInRleHQiOiAiZGlhZ25vc2lzIgoJCQkJCQl9LAoJCQkJCQkiY3JpdGVyaWEiOiB7CgkJCQkJCQkiZXhwcmVzc2lvbiI6ICJEaWFnbm9zaXNDb2RlIiwKCQkJCQkJCSJsYW5ndWFnZSI6ICJ0ZXh0L2NxbC1pZGVudGlmaWVyIgoJCQkJCQl9CgkJCQkJfQoJCQkJXQoJCQl9LAoJCQl7CgkJCQkiY29kZSI6IHsKCQkJCQkidGV4dCI6ICJzcGVjaW1lbiIKCQkJCX0sCgkJCQkiZXh0ZW5zaW9uIjogWwoJCQkJCXsKCQkJCQkJInVybCI6ICJodHRwOi8vaGw3Lm9yZy9maGlyL3VzL2NxZm1lYXN1cmVzL1N0cnVjdHVyZURlZmluaXRpb24vY3FmbS1wb3B1bGF0aW9uQmFzaXMiLAoJCQkJCQkidmFsdWVDb2RlIjogIlNwZWNpbWVuIgoJCQkJCX0KCQkJCV0sCgkJCQkicG9wdWxhdGlvbiI6IFsKCQkJCQl7CgkJCQkJCSJjb2RlIjogewoJCQkJCQkJImNvZGluZyI6IFsKCQkJCQkJCQl7CgkJCQkJCQkJCSJjb2RlIjogImluaXRpYWwtcG9wdWxhdGlvbiIsCgkJCQkJCQkJCSJzeXN0ZW0iOiAiaHR0cDovL3Rlcm1pbm9sb2d5LmhsNy5vcmcvQ29kZVN5c3RlbS9tZWFzdXJlLXBvcHVsYXRpb24iCgkJCQkJCQkJfQoJCQkJCQkJXQoJCQkJCQl9LAoJCQkJCQkiY3JpdGVyaWEiOiB7CgkJCQkJCQkiZXhwcmVzc2lvbiI6ICJTcGVjaW1lbiIsCgkJCQkJCQkibGFuZ3VhZ2UiOiAidGV4dC9jcWwtaWRlbnRpZmllciIKCQkJCQkJfQoJCQkJCX0KCQkJCV0sCgkJCQkic3RyYXRpZmllciI6IFsKCQkJCQl7CgkJCQkJCSJjb2RlIjogewoJCQkJCQkJInRleHQiOiAic2FtcGxlX2tpbmQiCgkJCQkJCX0sCgkJCQkJCSJjcml0ZXJpYSI6IHsKCQkJCQkJCSJleHByZXNzaW9uIjogIlNhbXBsZVR5cGUiLAoJCQkJCQkJImxhbmd1YWdlIjogInRleHQvY3FsIgoJCQkJCQl9CgkJCQkJfQoJCQkJXQoJCQl9CgkJXSwKCQkibGlicmFyeSI6ICJ1cm46dXVpZDo3ZmY1MzJhZC02OWU0LTQ4ZWQtYTJkMy05ZWZhZmI2MDlmNjIiLAoJCSJyZXNvdXJjZVR5cGUiOiAiTWVhc3VyZSIsCgkJInNjb3JpbmciOiB7CgkJCSJjb2RpbmciOiBbCgkJCQl7CgkJCQkJImNvZGUiOiAiY29ob3J0IiwKCQkJCQkic3lzdGVtIjogImh0dHA6Ly90ZXJtaW5vbG9neS5obDcub3JnL0NvZGVTeXN0ZW0vbWVhc3VyZS1zY29yaW5nIgoJCQkJfQoJCQldCgkJfSwKCQkic3RhdHVzIjogImFjdGl2ZSIsCgkJInN1YmplY3RDb2RlYWJsZUNvbmNlcHQiOiB7CgkJCSJjb2RpbmciOiBbCgkJCQl7CgkJCQkJImNvZGUiOiAiUGF0aWVudCIsCgkJCQkJInN5c3RlbSI6ICJodHRwOi8vaGw3Lm9yZy9maGlyL3Jlc291cmNlLXR5cGVzIgoJCQkJfQoJCQldCgkJfSwKCQkidXJsIjogInVybjp1dWlkOjVlZThkZTczLTM0N2UtNDdjYS1hMDE0LWYyZTcxNzY3YWRmYyIKCX0KfQ=="}' -H "Authorization: ApiKey app1.proxy1.broker App1Secret" http://localhost:8081/v1/tasks ``` +Creating a sample [Exporter](https://github.com/samply/exporter) "execute" task containing an Exporter query using CURL: + +```bash +curl -v -X POST -H "Content-Type: application/json" --data '{"body":"ew0KICAicXVlcnktY29udGV4dCIgOiAiVUZKUFNrVkRWQzFKUkQxa01qaGhZVEl5Wm1Wa01USTBNemM0T0RWallnPT0iLA0KICAicXVlcnktbGFiZWwiIDogIlRlc3QgMyIsDQogICJxdWVyeS1leGVjdXRpb24tY29udGFjdC1pZCIgOiAiYmstYWRtaW5AdGVzdC5kZmt6LmRlIiwNCiAgInF1ZXJ5LWRlc2NyaXB0aW9uIiA6ICJUaGlzIGlzIHRoZSB0ZXN0IDMiLA0KICAicXVlcnktZXhwaXJhdGlvbi1kYXRlIiA6ICIyMDI0LTA4LTE0IiwNCiAgIm91dHB1dC1mb3JtYXQiIDogIkVYQ0VMIiwNCiAgInF1ZXJ5IiA6ICJleUpzWVc1bklqb2lZM0ZzSWl3aWJHbGlJanA3SW5KbGMyOTFjbU5sVkhsd1pTSTZJa3hwWW5KaGNua2lMQ0oxY213aU9pSjFjbTQ2ZFhWcFpEcGpOelJrWmpJd05DMDFZalppTFRSaFpXUXRZakl5T0MwM1pqVXpNekE0TnpZME5UZ2lMQ0p6ZEdGMGRYTWlPaUpoWTNScGRtVWlMQ0owZVhCbElqcDdJbU52WkdsdVp5STZXM3NpYzNsemRHVnRJam9pYUhSMGNEb3ZMM1JsY20xcGJtOXNiMmQ1TG1oc055NXZjbWN2UTI5a1pWTjVjM1JsYlM5c2FXSnlZWEo1TFhSNWNHVWlMQ0pqYjJSbElqb2liRzluYVdNdGJHbGljbUZ5ZVNKOVhYMHNJbU52Ym5SbGJuUWlPbHQ3SW1OdmJuUmxiblJVZVhCbElqb2lkR1Y0ZEM5amNXd2lMQ0prWVhSaElqb2lZa2RzYVdOdFJubGxVMEpUV2xoU2VXRlhWakphVVhBeFl6SnNkVnA1UWtkVFJXeFRTVWhhYkdOdVRuQmlNalJuU25wUmRVMUROSGRLZDNCd1ltMU9jMlJYVW14SlJWcEpVMVpLU1ZwWGVIZGFXRXA2U1VoYWJHTnVUbkJpTWpSblNucFJkVTFETkhkS2QyOUxXVEk1YTFwWVRqVmpNMUpzWWxOQ2MySXliSFZaZW05blNqSm9NR1JJUVRaTWVUbHpZakpzZFZsNU5YWmpiV051UTJkd2FtSXlOVEJhV0dnd1NVWkNhR1JIYkd4aWJsRkxRMmR3UlZNeFVreFlNVTVWVld0R1ZWZ3daRVpVYTFKR1ZXdzVWRlpHU2tKV1JXeEhVMVZXVTBObmNFVlRNVkpNV0RGT1ZWVnJSbFZZTVVKVFUxVXhRbFZzYkdaU1JXeENVakExVUZVd2JGUllNVTVWVld0R1ZWTlZXa3BTVmtsTFVrVjBWVk14T1ZSV1JrcENWa1k1UWxJd1ZtWlJNSGhDVlRGT1psVXhVbE5SVmxKS1VtdHNSbFZuYjB0U1JYUlZVekU1VkZaR1NrSldSamxGVWxWT1JsRldUa1pTUmpsVVZrWktRbFpGYkVkVFZWWlRRMmR3UlZNeFVreFlNVTVWVld0R1ZWZ3dVa3BSVldSUFZERk9TbFV4T1ZSV1JrcENWa1ZzUjFOVlZsTkRaM0JGVXpGU1RGZ3hUbFZWYTBaVldERk9VVkpWVGtwVVZWWlBXREZPVlZWclJsVlRWVnBLVWxaSlMwTnJVa3hXUlhSbVZURlNVMUZXVW1aVlJrcFFVVEJXUlZaV1NrWllNVTVWVld0R1ZWTlZXa3BTVmtsTFEydFNURlpGZEdaVk1WSlRVVlpTWmxSVlZrVlRWVTVDVmtWc1VGUnNPVlJXUmtwQ1ZrVnNSMU5WVmxORGExSk1Wa1YwWmxVeFVsTlJWbEptVWtWV1IxZ3diRTlZTUd4UFUxWlNTbEZWZUdaVlJUbFJWbFY0UWxaRmJGQlViRUpvWkVkc2JHSnVVWFZhTWxaMVdrZFdlVWxFTUdkS01qRm9Za2RWYmlKOVhYMHNJbTFsWVhOMWNtVWlPbnNpY21WemIzVnlZMlZVZVhCbElqb2lUV1ZoYzNWeVpTSXNJblZ5YkNJNkluVnlianAxZFdsa09qaG1NMlV6WVRZeExXRXdPVGN0TkRoa05DMWlOMkZqTFRobE5ESTNZbVU0WVdNMFpDSXNJbk4wWVhSMWN5STZJbUZqZEdsMlpTSXNJbk4xWW1wbFkzUkRiMlJsWVdKc1pVTnZibU5sY0hRaU9uc2lZMjlrYVc1bklqcGJleUp6ZVhOMFpXMGlPaUpvZEhSd09pOHZhR3czTG05eVp5OW1hR2x5TDNKbGMyOTFjbU5sTFhSNWNHVnpJaXdpWTI5a1pTSTZJbEJoZEdsbGJuUWlmVjE5TENKc2FXSnlZWEo1SWpvaWRYSnVPblYxYVdRNll6YzBaR1l5TURRdE5XSTJZaTAwWVdWa0xXSXlNamd0TjJZMU16TXdPRGMyTkRVNElpd2ljMk52Y21sdVp5STZleUpqYjJScGJtY2lPbHQ3SW5ONWMzUmxiU0k2SW1oMGRIQTZMeTkwWlhKdGFXNXZiRzluZVM1b2JEY3ViM0puTDBOdlpHVlRlWE4wWlcwdmJXVmhjM1Z5WlMxelkyOXlhVzVuSWl3aVkyOWtaU0k2SW1OdmFHOXlkQ0o5WFgwc0ltZHliM1Z3SWpwYmV5SmpiMlJsSWpwN0luUmxlSFFpT2lKd1lYUnBaVzUwY3lKOUxDSndiM0IxYkdGMGFXOXVJanBiZXlKamIyUmxJanA3SW1OdlpHbHVaeUk2VzNzaWMzbHpkR1Z0SWpvaWFIUjBjRG92TDNSbGNtMXBibTlzYjJkNUxtaHNOeTV2Y21jdlEyOWtaVk41YzNSbGJTOXRaV0Z6ZFhKbExYQnZjSFZzWVhScGIyNGlMQ0pqYjJSbElqb2lhVzVwZEdsaGJDMXdiM0IxYkdGMGFXOXVJbjFkZlN3aVkzSnBkR1Z5YVdFaU9uc2liR0Z1WjNWaFoyVWlPaUowWlhoMEwyTnhiQzFwWkdWdWRHbG1hV1Z5SWl3aVpYaHdjbVZ6YzJsdmJpSTZJa2x1U1c1cGRHbGhiRkJ2Y0hWc1lYUnBiMjRpZlgxZExDSnpkSEpoZEdsbWFXVnlJanBiZXlKamIyUmxJanA3SW5SbGVIUWlPaUpIWlc1a1pYSWlmU3dpWTNKcGRHVnlhV0VpT25zaWJHRnVaM1ZoWjJVaU9pSjBaWGgwTDJOeGJDSXNJbVY0Y0hKbGMzTnBiMjRpT2lKSFpXNWtaWElpZlgwc2V5SmpiMlJsSWpwN0luUmxlSFFpT2lJM05URTROaTAzSW4wc0ltTnlhWFJsY21saElqcDdJbXhoYm1kMVlXZGxJam9pZEdWNGRDOWpjV3dpTENKbGVIQnlaWE56YVc5dUlqb2lSR1ZqWldGelpXUWlmWDBzZXlKamIyUmxJanA3SW5SbGVIUWlPaUpCWjJVaWZTd2lZM0pwZEdWeWFXRWlPbnNpYkdGdVozVmhaMlVpT2lKMFpYaDBMMk54YkNJc0ltVjRjSEpsYzNOcGIyNGlPaUpCWjJWRGJHRnpjeUo5ZlYxOUxIc2lZMjlrWlNJNmV5SjBaWGgwSWpvaVpHbGhaMjV2YzJsekluMHNJbVY0ZEdWdWMybHZiaUk2VzNzaWRYSnNJam9pYUhSMGNEb3ZMMmhzTnk1dmNtY3ZabWhwY2k5MWN5OWpjV1p0WldGemRYSmxjeTlUZEhKMVkzUjFjbVZFWldacGJtbDBhVzl1TDJOeFptMHRjRzl3ZFd4aGRHbHZia0poYzJseklpd2lkbUZzZFdWRGIyUmxJam9pUTI5dVpHbDBhVzl1SW4xZExDSndiM0IxYkdGMGFXOXVJanBiZXlKamIyUmxJanA3SW1OdlpHbHVaeUk2VzNzaWMzbHpkR1Z0SWpvaWFIUjBjRG92TDNSbGNtMXBibTlzYjJkNUxtaHNOeTV2Y21jdlEyOWtaVk41YzNSbGJTOXRaV0Z6ZFhKbExYQnZjSFZzWVhScGIyNGlMQ0pqYjJSbElqb2lhVzVwZEdsaGJDMXdiM0IxYkdGMGFXOXVJbjFkZlN3aVkzSnBkR1Z5YVdFaU9uc2liR0Z1WjNWaFoyVWlPaUowWlhoMEwyTnhiQzFwWkdWdWRHbG1hV1Z5SWl3aVpYaHdjbVZ6YzJsdmJpSTZJa1JwWVdkdWIzTnBjeUo5ZlYwc0luTjBjbUYwYVdacFpYSWlPbHQ3SW1OdlpHVWlPbnNpZEdWNGRDSTZJbVJwWVdkdWIzTnBjeUo5TENKamNtbDBaWEpwWVNJNmV5SnNZVzVuZFdGblpTSTZJblJsZUhRdlkzRnNMV2xrWlc1MGFXWnBaWElpTENKbGVIQnlaWE56YVc5dUlqb2lSR2xoWjI1dmMybHpRMjlrWlNKOWZWMTlMSHNpWTI5a1pTSTZleUowWlhoMElqb2ljM0JsWTJsdFpXNGlmU3dpWlhoMFpXNXphVzl1SWpwYmV5SjFjbXdpT2lKb2RIUndPaTh2YUd3M0xtOXlaeTltYUdseUwzVnpMMk54Wm0xbFlYTjFjbVZ6TDFOMGNuVmpkSFZ5WlVSbFptbHVhWFJwYjI0dlkzRm1iUzF3YjNCMWJHRjBhVzl1UW1GemFYTWlMQ0oyWVd4MVpVTnZaR1VpT2lKVGNHVmphVzFsYmlKOVhTd2ljRzl3ZFd4aGRHbHZiaUk2VzNzaVkyOWtaU0k2ZXlKamIyUnBibWNpT2x0N0luTjVjM1JsYlNJNkltaDBkSEE2THk5MFpYSnRhVzV2Ykc5bmVTNW9iRGN1YjNKbkwwTnZaR1ZUZVhOMFpXMHZiV1ZoYzNWeVpTMXdiM0IxYkdGMGFXOXVJaXdpWTI5a1pTSTZJbWx1YVhScFlXd3RjRzl3ZFd4aGRHbHZiaUo5WFgwc0ltTnlhWFJsY21saElqcDdJbXhoYm1kMVlXZGxJam9pZEdWNGRDOWpjV3d0YVdSbGJuUnBabWxsY2lJc0ltVjRjSEpsYzNOcGIyNGlPaUpUY0dWamFXMWxiaUo5ZlYwc0luTjBjbUYwYVdacFpYSWlPbHQ3SW1OdlpHVWlPbnNpZEdWNGRDSTZJbk5oYlhCc1pWOXJhVzVrSW4wc0ltTnlhWFJsY21saElqcDdJbXhoYm1kMVlXZGxJam9pZEdWNGRDOWpjV3dpTENKbGVIQnlaWE56YVc5dUlqb2lVMkZ0Y0d4bFZIbHdaU0o5ZlYxOUxIc2lZMjlrWlNJNmV5SjBaWGgwSWpvaWNISnZZMlZrZFhKbGN5SjlMQ0psZUhSbGJuTnBiMjRpT2x0N0luVnliQ0k2SW1oMGRIQTZMeTlvYkRjdWIzSm5MMlpvYVhJdmRYTXZZM0ZtYldWaGMzVnlaWE12VTNSeWRXTjBkWEpsUkdWbWFXNXBkR2x2Ymk5amNXWnRMWEJ2Y0hWc1lYUnBiMjVDWVhOcGN5SXNJblpoYkhWbFEyOWtaU0k2SWxCeWIyTmxaSFZ5WlNKOVhTd2ljRzl3ZFd4aGRHbHZiaUk2VzNzaVkyOWtaU0k2ZXlKamIyUnBibWNpT2x0N0luTjVjM1JsYlNJNkltaDBkSEE2THk5MFpYSnRhVzV2Ykc5bmVTNW9iRGN1YjNKbkwwTnZaR1ZUZVhOMFpXMHZiV1ZoYzNWeVpTMXdiM0IxYkdGMGFXOXVJaXdpWTI5a1pTSTZJbWx1YVhScFlXd3RjRzl3ZFd4aGRHbHZiaUo5WFgwc0ltTnlhWFJsY21saElqcDdJbXhoYm1kMVlXZGxJam9pZEdWNGRDOWpjV3d0YVdSbGJuUnBabWxsY2lJc0ltVjRjSEpsYzNOcGIyNGlPaUpRY205alpXUjFjbVVpZlgxZExDSnpkSEpoZEdsbWFXVnlJanBiZXlKamIyUmxJanA3SW5SbGVIUWlPaUpRY205alpXUjFjbVZVZVhCbEluMHNJbU55YVhSbGNtbGhJanA3SW14aGJtZDFZV2RsSWpvaWRHVjRkQzlqY1d3aUxDSmxlSEJ5WlhOemFXOXVJam9pVUhKdlkyVmtkWEpsVkhsd1pTSjlmVjE5TEhzaVkyOWtaU0k2ZXlKMFpYaDBJam9pYldWa2FXTmhkR2x2YmxOMFlYUmxiV1Z1ZEhNaWZTd2laWGgwWlc1emFXOXVJanBiZXlKMWNtd2lPaUpvZEhSd09pOHZhR3czTG05eVp5OW1hR2x5TDNWekwyTnhabTFsWVhOMWNtVnpMMU4wY25WamRIVnlaVVJsWm1sdWFYUnBiMjR2WTNGbWJTMXdiM0IxYkdGMGFXOXVRbUZ6YVhNaUxDSjJZV3gxWlVOdlpHVWlPaUpOWldScFkyRjBhVzl1VTNSaGRHVnRaVzUwSW4xZExDSndiM0IxYkdGMGFXOXVJanBiZXlKamIyUmxJanA3SW1OdlpHbHVaeUk2VzNzaWMzbHpkR1Z0SWpvaWFIUjBjRG92TDNSbGNtMXBibTlzYjJkNUxtaHNOeTV2Y21jdlEyOWtaVk41YzNSbGJTOXRaV0Z6ZFhKbExYQnZjSFZzWVhScGIyNGlMQ0pqYjJSbElqb2lhVzVwZEdsaGJDMXdiM0IxYkdGMGFXOXVJbjFkZlN3aVkzSnBkR1Z5YVdFaU9uc2liR0Z1WjNWaFoyVWlPaUowWlhoMEwyTnhiQzFwWkdWdWRHbG1hV1Z5SWl3aVpYaHdjbVZ6YzJsdmJpSTZJazFsWkdsallYUnBiMjVUZEdGMFpXMWxiblFpZlgxZExDSnpkSEpoZEdsbWFXVnlJanBiZXlKamIyUmxJanA3SW5SbGVIUWlPaUpOWldScFkyRjBhVzl1Vkhsd1pTSjlMQ0pqY21sMFpYSnBZU0k2ZXlKc1lXNW5kV0ZuWlNJNkluUmxlSFF2WTNGc0lpd2laWGh3Y21WemMybHZiaUk2SWxCeWIyTmxaSFZ5WlZSNWNHVWlmWDFkZlYxOWZRPT0iLA0KICAicXVlcnktY29udGFjdC1pZCIgOiAicmVzZWFyY2hlckB0ZXN0LmRrZnouZGUiLA0KICAicXVlcnktZm9ybWF0IiA6ICJDUUxfREFUQSIsDQogICJ0ZW1wbGF0ZS1pZCIgOiAiY2NwIg0KfQ==","failure_strategy":{"retry":{"backoff_millisecs":1000,"max_tries":5}},"from":"app1.proxy1.broker","id":"22e1ea3a-07f3-4592-a888-82f2226a44a2","metadata":{"project":"exporter","task_type":"EXECUTE"},"to":["app1.proxy1.broker"],"ttl":"10s","status":null,"task":null}' -H "Authorization: ApiKey app1.proxy1.broker App1Secret" http://localhost:8081/v1/tasks + +``` + +Creating a sample [Exporter](https://github.com/samply/exporter) "status" task using CURL: + +```bash +curl -v -X POST -H "Content-Type: application/json" --data '{"body":"ew0KICAicXVlcnktZXhlY3V0aW9uLWlkIiA6ICIxOSINCn0=","failure_strategy":{"retry":{"backoff_millisecs":1000,"max_tries":5}},"from":"app1.proxy1.broker","id":"22e1ea3a-07f3-4592-a888-82f2226a44a2","metadata":{"project":"exporter","task_type":"STATUS"},"to":["app1.proxy1.broker"],"ttl":"10s","status":null,"task":null}' -H "Authorization: ApiKey app1.proxy1.broker App1Secret" http://localhost:8081/v1/tasks + +``` + ## License This code is licensed under the Apache License 2.0. For details, please see [LICENSE](./LICENSE) From 337e6b22e4964805a72103be2803f3832ccbab4a Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Fri, 24 May 2024 16:54:38 +0200 Subject: [PATCH 28/42] handling both CQL and AST queries --- src/blaze.rs | 17 ++++----------- src/cql.rs | 19 ++++++++--------- src/errors.rs | 4 +++- src/main.rs | 57 +++++++++++++++++++++++++-------------------------- 4 files changed, 44 insertions(+), 53 deletions(-) diff --git a/src/blaze.rs b/src/blaze.rs index c613648..32325a4 100644 --- a/src/blaze.rs +++ b/src/blaze.rs @@ -13,15 +13,13 @@ use crate::ast; #[derive(Serialize, Deserialize, Debug, Default, Clone)] pub struct CqlQuery { - pub lang: String, pub lib: Value, pub measure: Value } #[derive(Serialize, Deserialize, Debug, Clone)] pub struct AstQuery { - pub lang: String, - pub payload: ast::Ast, + pub payload: String, } pub async fn check_availability() -> bool { @@ -130,14 +128,7 @@ pub async fn run_cql_query(library: &Value, measure: &Value) -> Result Result { - let decoded = util::base64_decode(&task.body)?; - serde_json::from_slice(&decoded).map_err(|e| FocusError::ParsingError(e.to_string())) -} - -// This could be part of an impl of Cqlquery -pub fn parse_blaze_query_ast(task: &BeamTask) -> Result { - let decoded = util::base64_decode(&task.body)?; - serde_json::from_slice(&decoded).map_err(|e| FocusError::ParsingError(e.to_string())) +pub fn parse_blaze_query_payload_ast(ast_query: &String) -> Result { + let decoded = util::base64_decode(ast_query)?; + Ok(serde_json::from_slice(&decoded)?) } \ No newline at end of file diff --git a/src/cql.rs b/src/cql.rs index 2427dae..e7a4cd4 100644 --- a/src/cql.rs +++ b/src/cql.rs @@ -12,16 +12,15 @@ use indexmap::set::IndexSet; use uuid::Uuid; pub fn generate_body(ast: ast::Ast) -> Result { - Ok(BASE64.encode( - BODY.clone() - .to_string() - .replace("{{LIBRARY_UUID}}", Uuid::new_v4().to_string().as_str()) - .replace("{{MEASURE_UUID}}", Uuid::new_v4().to_string().as_str()) - .replace( - "{{LIBRARY_ENCODED}}", - BASE64.encode(generate_cql(ast)?).as_str(), - ), - )) + Ok(BODY.clone() + .to_string() + .replace("{{LIBRARY_UUID}}", format!("urn:uuid:{}", Uuid::new_v4().to_string()).as_str()) + .replace("{{MEASURE_UUID}}", format!("urn:uuid:{}", Uuid::new_v4().to_string()).as_str()) + .replace( + "{{LIBRARY_ENCODED}}", + BASE64.encode(generate_cql(ast)?).as_str(), + ) +) } fn generate_cql(ast: ast::Ast) -> Result { diff --git a/src/errors.rs b/src/errors.rs index 70ba68e..f631a75 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -24,6 +24,8 @@ pub enum FocusError { ConfigurationError(String), #[error("Cannot open file: {0}")] FileOpeningError(String), + #[error("Serde parsing error: {0}")] + SerdeParsingError(#[from] serde_json::Error), #[error("Parsing error: {0}")] ParsingError(String), #[error("CQL tampered with: {0}")] @@ -66,7 +68,7 @@ impl FocusError { use FocusError::*; // TODO: Add more match arms match self { - DecodeError(_) | ParsingError(_) => "Cannot parse query.", + DecodeError(_) | SerdeParsingError(_) => "Cannot parse query.", LaplaceError(_) => "Cannot obfuscate result.", _ => "Failed to execute query." } diff --git a/src/main.rs b/src/main.rs index 8466d16..d1915a4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,9 +23,9 @@ use futures_util::FutureExt; use laplace_rs::ObfCache; use tokio::sync::Mutex; -use crate::blaze::{parse_blaze_query_ast, parse_blaze_query_cql}; +use crate::blaze::{parse_blaze_query_payload_ast, AstQuery}; use crate::config::EndpointType; -use crate::util::{is_cql_tampered_with, obfuscate_counts_mr}; +use crate::util::{base64_decode, is_cql_tampered_with, obfuscate_counts_mr}; use crate::{config::CONFIG, errors::FocusError}; use blaze::CqlQuery; @@ -167,46 +167,39 @@ async fn process_task( return Err(FocusError::MissingExporterTaskType); }; let body = &task.body; - return Ok(run_exporter_query(task, body, task_type).await)?; //we already made sure that it is not None + return Ok(run_exporter_query(task, body, task_type).await?); } if CONFIG.endpoint_type == EndpointType::Blaze { - let mut query_maybe = parse_blaze_query_cql(task); - if let Err(_e) = query_maybe { - let query_string = cql::generate_body(parse_blaze_query_ast(task)?.payload)?; - query_maybe = serde_json::from_str(&query_string).map_err(|e| FocusError::ParsingError(e.to_string())) + #[derive(Deserialize, Debug)] + #[serde(tag = "lang", rename_all = "lowercase")] + enum Language { + Cql(CqlQuery), + Ast(AstQuery) } + let mut generated_from_ast: bool = false; + let data = base64_decode(&task.body)?; + let query: CqlQuery = match serde_json::from_slice::(&data)? { + Language::Cql(cql_query) => cql_query, + Language::Ast(ast_query) => { + generated_from_ast = true; + serde_json::from_str(&cql::generate_body(parse_blaze_query_payload_ast(&ast_query.payload)?)?)? + } + }; + run_cql_query(task, &query, obf_cache, report_cache, metadata.project, generated_from_ast).await - let query = query_maybe.unwrap(); - - if query.lang == "cql" { - // TODO: Change query.lang to an enum - - Ok(run_cql_query(task, &query, obf_cache, report_cache, metadata.project).await)? - } else { - warn!("Can't run queries with language {} in Blaze", query.lang); - Ok(beam::beam_result::perm_failed( - CONFIG.beam_app_id_long.clone(), - vec![task.from.clone()], - task.id, - format!( - "Can't run queries with language {} and/or endpoint type {}", - query.lang, CONFIG.endpoint_type - ), - )) - } } else if CONFIG.endpoint_type == EndpointType::Omop { let decoded = util::base64_decode(&task.body)?; let intermediate_rep_query: intermediate_rep::IntermediateRepQuery = - serde_json::from_slice(&decoded).map_err(|e| FocusError::ParsingError(e.to_string()))?; + serde_json::from_slice(&decoded)?; //TODO check that the language is ast let query_decoded = general_purpose::STANDARD .decode(intermediate_rep_query.query) .map_err(FocusError::DecodeError)?; let ast: ast::Ast = - serde_json::from_slice(&query_decoded).map_err(|e| FocusError::ParsingError(e.to_string()))?; + serde_json::from_slice(&query_decoded)?; - Ok(run_intermediate_rep_query(task, ast).await)? + Ok(run_intermediate_rep_query(task, ast).await?) } else { warn!( "Can't run queries with endpoint type {}", @@ -230,6 +223,7 @@ async fn run_cql_query( obf_cache: Arc>, report_cache: Arc>, project: String, + generated_from_ast: bool ) -> Result { let encoded_query = query.lib["content"][0]["data"] @@ -264,7 +258,12 @@ async fn run_cql_query( let cql_result_new = match report_from_cache { Some(some_report_from_cache) => some_report_from_cache.to_string(), None => { - let query = replace_cql_library(query.clone())?; + let query = + if generated_from_ast { + query.clone() + } else { + replace_cql_library(query.clone())? + }; let cql_result = blaze::run_cql_query(&query.lib, &query.measure).await?; From 9dc6efed812f656c681992d1f67634c05bce4379 Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Fri, 24 May 2024 17:06:51 +0200 Subject: [PATCH 29/42] sample AST task in readme --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 8b5ef97..3b9e28c 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,12 @@ Creating a sample task containing a [Blaze](https://github.com/samply/blaze) que curl -v -X POST -H "Content-Type: application/json" --data '{"id":"7fffefff-ffef-fcff-feef-fefbffffeeff","from":"app1.proxy1.broker","to":["app1.proxy1.broker"],"ttl":"10s","failure_strategy":{"retry":{"backoff_millisecs":1000,"max_tries":5}},"metadata":{"project":"exliquid"},"body":"ewoJImxhbmciOiAiY3FsIiwKCSJsaWIiOiB7CgkJImNvbnRlbnQiOiBbCgkJCXsKCQkJCSJjb250ZW50VHlwZSI6ICJ0ZXh0L2NxbCIsCgkJCQkiZGF0YSI6ICJiR2xpY21GeWVTQlNaWFJ5YVdWMlpRcDFjMmx1WnlCR1NFbFNJSFpsY25OcGIyNGdKelF1TUM0d0p3cHBibU5zZFdSbElFWklTVkpJWld4d1pYSnpJSFpsY25OcGIyNGdKelF1TUM0d0p3b0tZMjlrWlhONWMzUmxiU0JzYjJsdVl6b2dKMmgwZEhBNkx5OXNiMmx1WXk1dmNtY25DbU52WkdWemVYTjBaVzBnYVdOa01UQTZJQ2RvZEhSd09pOHZhR3czTG05eVp5OW1hR2x5TDNOcFpDOXBZMlF0TVRBbkNtTnZaR1Z6ZVhOMFpXMGdVMkZ0Y0d4bFRXRjBaWEpwWVd4VWVYQmxPaUFuYUhSMGNITTZMeTltYUdseUxtSmliWEpwTG1SbEwwTnZaR1ZUZVhOMFpXMHZVMkZ0Y0d4bFRXRjBaWEpwWVd4VWVYQmxKd29LQ21OdmJuUmxlSFFnVUdGMGFXVnVkQW9LUWtKTlVrbGZVMVJTUVZSZlIwVk9SRVZTWDFOVVVrRlVTVVpKUlZJS0NrSkNUVkpKWDFOVVVrRlVYMFJGUmw5VFVFVkRTVTFGVGdwcFppQkpia2x1YVhScFlXeFFiM0IxYkdGMGFXOXVJSFJvWlc0Z1cxTndaV05wYldWdVhTQmxiSE5sSUh0OUlHRnpJRXhwYzNROFUzQmxZMmx0Wlc0K0NncENRazFTU1Y5VFZGSkJWRjlUUVUxUVRFVmZWRmxRUlY5VFZGSkJWRWxHU1VWU0NncENRazFTU1Y5VFZGSkJWRjlEVlZOVVQwUkpRVTVmVTFSU1FWUkpSa2xGVWdvS1FrSk5Va2xmVTFSU1FWUmZSRWxCUjA1UFUwbFRYMU5VVWtGVVNVWkpSVklLQ2tKQ1RWSkpYMU5VVWtGVVgwRkhSVjlUVkZKQlZFbEdTVVZTQ2dwQ1FrMVNTVjlUVkZKQlZGOUVSVVpmU1U1ZlNVNUpWRWxCVEY5UVQxQlZURUZVU1U5T0NuUnlkV1U9IgoJCQl9CgkJXSwKCQkicmVzb3VyY2VUeXBlIjogIkxpYnJhcnkiLAoJCSJzdGF0dXMiOiAiYWN0aXZlIiwKCQkidHlwZSI6IHsKCQkJImNvZGluZyI6IFsKCQkJCXsKCQkJCQkiY29kZSI6ICJsb2dpYy1saWJyYXJ5IiwKCQkJCQkic3lzdGVtIjogImh0dHA6Ly90ZXJtaW5vbG9neS5obDcub3JnL0NvZGVTeXN0ZW0vbGlicmFyeS10eXBlIgoJCQkJfQoJCQldCgkJfSwKCQkidXJsIjogInVybjp1dWlkOjdmZjUzMmFkLTY5ZTQtNDhlZC1hMmQzLTllZmFmYjYwOWY2MiIKCX0sCgkibWVhc3VyZSI6IHsKCQkiZ3JvdXAiOiBbCgkJCXsKCQkJCSJjb2RlIjogewoJCQkJCSJ0ZXh0IjogInBhdGllbnRzIgoJCQkJfSwKCQkJCSJwb3B1bGF0aW9uIjogWwoJCQkJCXsKCQkJCQkJImNvZGUiOiB7CgkJCQkJCQkiY29kaW5nIjogWwoJCQkJCQkJCXsKCQkJCQkJCQkJImNvZGUiOiAiaW5pdGlhbC1wb3B1bGF0aW9uIiwKCQkJCQkJCQkJInN5c3RlbSI6ICJodHRwOi8vdGVybWlub2xvZ3kuaGw3Lm9yZy9Db2RlU3lzdGVtL21lYXN1cmUtcG9wdWxhdGlvbiIKCQkJCQkJCQl9CgkJCQkJCQldCgkJCQkJCX0sCgkJCQkJCSJjcml0ZXJpYSI6IHsKCQkJCQkJCSJleHByZXNzaW9uIjogIkluSW5pdGlhbFBvcHVsYXRpb24iLAoJCQkJCQkJImxhbmd1YWdlIjogInRleHQvY3FsLWlkZW50aWZpZXIiCgkJCQkJCX0KCQkJCQl9CgkJCQldLAoJCQkJInN0cmF0aWZpZXIiOiBbCgkJCQkJewoJCQkJCQkiY29kZSI6IHsKCQkJCQkJCSJ0ZXh0IjogIkdlbmRlciIKCQkJCQkJfSwKCQkJCQkJImNyaXRlcmlhIjogewoJCQkJCQkJImV4cHJlc3Npb24iOiAiR2VuZGVyIiwKCQkJCQkJCSJsYW5ndWFnZSI6ICJ0ZXh0L2NxbCIKCQkJCQkJfQoJCQkJCX0sCgkJCQkJewoJCQkJCQkiY29kZSI6IHsKCQkJCQkJCSJ0ZXh0IjogIkFnZSIKCQkJCQkJfSwKCQkJCQkJImNyaXRlcmlhIjogewoJCQkJCQkJImV4cHJlc3Npb24iOiAiQWdlQ2xhc3MiLAoJCQkJCQkJImxhbmd1YWdlIjogInRleHQvY3FsIgoJCQkJCQl9CgkJCQkJfSwKCQkJCQl7CgkJCQkJCSJjb2RlIjogewoJCQkJCQkJInRleHQiOiAiQ3VzdG9kaWFuIgoJCQkJCQl9LAoJCQkJCQkiY3JpdGVyaWEiOiB7CgkJCQkJCQkiZXhwcmVzc2lvbiI6ICJDdXN0b2RpYW4iLAoJCQkJCQkJImxhbmd1YWdlIjogInRleHQvY3FsIgoJCQkJCQl9CgkJCQkJfQoJCQkJXQoJCQl9LAoJCQl7CgkJCQkiY29kZSI6IHsKCQkJCQkidGV4dCI6ICJkaWFnbm9zaXMiCgkJCQl9LAoJCQkJImV4dGVuc2lvbiI6IFsKCQkJCQl7CgkJCQkJCSJ1cmwiOiAiaHR0cDovL2hsNy5vcmcvZmhpci91cy9jcWZtZWFzdXJlcy9TdHJ1Y3R1cmVEZWZpbml0aW9uL2NxZm0tcG9wdWxhdGlvbkJhc2lzIiwKCQkJCQkJInZhbHVlQ29kZSI6ICJDb25kaXRpb24iCgkJCQkJfQoJCQkJXSwKCQkJCSJwb3B1bGF0aW9uIjogWwoJCQkJCXsKCQkJCQkJImNvZGUiOiB7CgkJCQkJCQkiY29kaW5nIjogWwoJCQkJCQkJCXsKCQkJCQkJCQkJImNvZGUiOiAiaW5pdGlhbC1wb3B1bGF0aW9uIiwKCQkJCQkJCQkJInN5c3RlbSI6ICJodHRwOi8vdGVybWlub2xvZ3kuaGw3Lm9yZy9Db2RlU3lzdGVtL21lYXN1cmUtcG9wdWxhdGlvbiIKCQkJCQkJCQl9CgkJCQkJCQldCgkJCQkJCX0sCgkJCQkJCSJjcml0ZXJpYSI6IHsKCQkJCQkJCSJleHByZXNzaW9uIjogIkRpYWdub3NpcyIsCgkJCQkJCQkibGFuZ3VhZ2UiOiAidGV4dC9jcWwtaWRlbnRpZmllciIKCQkJCQkJfQoJCQkJCX0KCQkJCV0sCgkJCQkic3RyYXRpZmllciI6IFsKCQkJCQl7CgkJCQkJCSJjb2RlIjogewoJCQkJCQkJInRleHQiOiAiZGlhZ25vc2lzIgoJCQkJCQl9LAoJCQkJCQkiY3JpdGVyaWEiOiB7CgkJCQkJCQkiZXhwcmVzc2lvbiI6ICJEaWFnbm9zaXNDb2RlIiwKCQkJCQkJCSJsYW5ndWFnZSI6ICJ0ZXh0L2NxbC1pZGVudGlmaWVyIgoJCQkJCQl9CgkJCQkJfQoJCQkJXQoJCQl9LAoJCQl7CgkJCQkiY29kZSI6IHsKCQkJCQkidGV4dCI6ICJzcGVjaW1lbiIKCQkJCX0sCgkJCQkiZXh0ZW5zaW9uIjogWwoJCQkJCXsKCQkJCQkJInVybCI6ICJodHRwOi8vaGw3Lm9yZy9maGlyL3VzL2NxZm1lYXN1cmVzL1N0cnVjdHVyZURlZmluaXRpb24vY3FmbS1wb3B1bGF0aW9uQmFzaXMiLAoJCQkJCQkidmFsdWVDb2RlIjogIlNwZWNpbWVuIgoJCQkJCX0KCQkJCV0sCgkJCQkicG9wdWxhdGlvbiI6IFsKCQkJCQl7CgkJCQkJCSJjb2RlIjogewoJCQkJCQkJImNvZGluZyI6IFsKCQkJCQkJCQl7CgkJCQkJCQkJCSJjb2RlIjogImluaXRpYWwtcG9wdWxhdGlvbiIsCgkJCQkJCQkJCSJzeXN0ZW0iOiAiaHR0cDovL3Rlcm1pbm9sb2d5LmhsNy5vcmcvQ29kZVN5c3RlbS9tZWFzdXJlLXBvcHVsYXRpb24iCgkJCQkJCQkJfQoJCQkJCQkJXQoJCQkJCQl9LAoJCQkJCQkiY3JpdGVyaWEiOiB7CgkJCQkJCQkiZXhwcmVzc2lvbiI6ICJTcGVjaW1lbiIsCgkJCQkJCQkibGFuZ3VhZ2UiOiAidGV4dC9jcWwtaWRlbnRpZmllciIKCQkJCQkJfQoJCQkJCX0KCQkJCV0sCgkJCQkic3RyYXRpZmllciI6IFsKCQkJCQl7CgkJCQkJCSJjb2RlIjogewoJCQkJCQkJInRleHQiOiAic2FtcGxlX2tpbmQiCgkJCQkJCX0sCgkJCQkJCSJjcml0ZXJpYSI6IHsKCQkJCQkJCSJleHByZXNzaW9uIjogIlNhbXBsZVR5cGUiLAoJCQkJCQkJImxhbmd1YWdlIjogInRleHQvY3FsIgoJCQkJCQl9CgkJCQkJfQoJCQkJXQoJCQl9CgkJXSwKCQkibGlicmFyeSI6ICJ1cm46dXVpZDo3ZmY1MzJhZC02OWU0LTQ4ZWQtYTJkMy05ZWZhZmI2MDlmNjIiLAoJCSJyZXNvdXJjZVR5cGUiOiAiTWVhc3VyZSIsCgkJInNjb3JpbmciOiB7CgkJCSJjb2RpbmciOiBbCgkJCQl7CgkJCQkJImNvZGUiOiAiY29ob3J0IiwKCQkJCQkic3lzdGVtIjogImh0dHA6Ly90ZXJtaW5vbG9neS5obDcub3JnL0NvZGVTeXN0ZW0vbWVhc3VyZS1zY29yaW5nIgoJCQkJfQoJCQldCgkJfSwKCQkic3RhdHVzIjogImFjdGl2ZSIsCgkJInN1YmplY3RDb2RlYWJsZUNvbmNlcHQiOiB7CgkJCSJjb2RpbmciOiBbCgkJCQl7CgkJCQkJImNvZGUiOiAiUGF0aWVudCIsCgkJCQkJInN5c3RlbSI6ICJodHRwOi8vaGw3Lm9yZy9maGlyL3Jlc291cmNlLXR5cGVzIgoJCQkJfQoJCQldCgkJfSwKCQkidXJsIjogInVybjp1dWlkOjVlZThkZTczLTM0N2UtNDdjYS1hMDE0LWYyZTcxNzY3YWRmYyIKCX0KfQ=="}' -H "Authorization: ApiKey app1.proxy1.broker App1Secret" http://localhost:8081/v1/tasks ``` +Creating a sample task containing an abstract syntax tree (AST) query using CURL: + +```bash +curl -v -X POST -H "Content-Type: application/json" --data '{"id":"7fffefff-ffef-fcff-feef-feffffffffff","from":"app1.proxy1.broker","to":["app1.proxy1.broker"],"ttl":"10s","failure_strategy":{"retry":{"backoff_millisecs":1000,"max_tries":5}},"metadata":{"project":"bbmri"},"body":"eyJsYW5nIjoiYXN0IiwicGF5bG9hZCI6ImV5SmhjM1FpT25zaWIzQmxjbUZ1WkNJNklrOVNJaXdpWTJocGJHUnlaVzRpT2x0N0ltOXdaWEpoYm1RaU9pSkJUa1FpTENKamFHbHNaSEpsYmlJNlczc2liM0JsY21GdVpDSTZJazlTSWl3aVkyaHBiR1J5Wlc0aU9sdDdJbXRsZVNJNkltZGxibVJsY2lJc0luUjVjR1VpT2lKRlVWVkJURk1pTENKemVYTjBaVzBpT2lJaUxDSjJZV3gxWlNJNkltMWhiR1VpZlN4N0ltdGxlU0k2SW1kbGJtUmxjaUlzSW5SNWNHVWlPaUpGVVZWQlRGTWlMQ0p6ZVhOMFpXMGlPaUlpTENKMllXeDFaU0k2SW1abGJXRnNaU0o5WFgxZGZWMTlMQ0pwWkNJNkltRTJaakZqWTJZekxXVmlaakV0TkRJMFppMDVaRFk1TFRSbE5XUXhNelZtTWpNME1DSjkifQ=="}' -H "Authorization: ApiKey app1.proxy1.broker App1Secret" http://localhost:8081/v1/tasks +``` + Creating a sample [Exporter](https://github.com/samply/exporter) "execute" task containing an Exporter query using CURL: ```bash From c1ab5a14319081fffdedf0c034f423642dbb4299 Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Fri, 24 May 2024 17:13:58 +0200 Subject: [PATCH 30/42] feature removed from (empty) common test --- src/cql.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cql.rs b/src/cql.rs index e7a4cd4..298ac29 100644 --- a/src/cql.rs +++ b/src/cql.rs @@ -378,7 +378,6 @@ mod test { const EMPTY: &str = r#"{"ast":{"children":[],"operand":"OR"}, "id":"a6f1ccf3-ebf1-424f-9d69-4e5d135f2340"}"#; - #[cfg(feature = "bbmri")] #[test] fn test_common() { // maybe nothing here From 277ed27f3c7ba5662e29728b171139df4eff022f Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Tue, 28 May 2024 14:50:15 +0200 Subject: [PATCH 31/42] changes requested by Reviewer #1 --- src/cql.rs | 103 ++++++++++++++++++++++---------------------------- src/errors.rs | 4 +- src/main.rs | 13 ++++--- 3 files changed, 55 insertions(+), 65 deletions(-) diff --git a/src/cql.rs b/src/cql.rs index 298ac29..3ef3972 100644 --- a/src/cql.rs +++ b/src/cql.rs @@ -10,17 +10,23 @@ use chrono::offset::Utc; use chrono::DateTime; use indexmap::set::IndexSet; use uuid::Uuid; +use tracing::info; pub fn generate_body(ast: ast::Ast) -> Result { - Ok(BODY.clone() + Ok(BODY .to_string() - .replace("{{LIBRARY_UUID}}", format!("urn:uuid:{}", Uuid::new_v4().to_string()).as_str()) - .replace("{{MEASURE_UUID}}", format!("urn:uuid:{}", Uuid::new_v4().to_string()).as_str()) + .replace( + "{{LIBRARY_UUID}}", + format!("urn:uuid:{}", Uuid::new_v4().to_string()).as_str(), + ) + .replace( + "{{MEASURE_UUID}}", + format!("urn:uuid:{}", Uuid::new_v4().to_string()).as_str(), + ) .replace( "{{LIBRARY_ENCODED}}", BASE64.encode(generate_cql(ast)?).as_str(), - ) -) + )) } fn generate_cql(ast: ast::Ast) -> Result { @@ -44,7 +50,7 @@ fn generate_cql(ast: ast::Ast) -> Result { grandchild.clone(), &mut retrieval_criteria, &mut filter_criteria, - &mut mandatory_codes + &mut mandatory_codes, )?; // Only concatenate operator if it's not the last element @@ -87,7 +93,7 @@ pub fn process( child: ast::Child, retrieval_criteria: &mut String, filter_criteria: &mut String, - code_systems: &mut IndexSet<&str> + code_systems: &mut IndexSet<&str>, ) -> Result<(), FocusError> { let mut retrieval_cond: String = "(".to_string(); let mut filter_cond: String = "".to_string(); @@ -96,8 +102,7 @@ pub fn process( ast::Child::Condition(condition) => { let condition_key_trans = condition.key.as_str(); - let condition_snippet = - CQL_SNIPPETS.get(&(condition_key_trans, CriterionRole::Query)); + let condition_snippet = CQL_SNIPPETS.get(&(condition_key_trans, CriterionRole::Query)); if let Some(snippet) = condition_snippet { let mut condition_string = (*snippet).to_string(); @@ -106,13 +111,11 @@ pub fn process( let filter_snippet = CQL_SNIPPETS.get(&(condition_key_trans, CriterionRole::Filter)); - let code_lists_option = - CRITERION_CODE_LISTS.get(&(condition_key_trans)); + let code_lists_option = CRITERION_CODE_LISTS.get(&(condition_key_trans)); if let Some(code_lists_vec) = code_lists_option { for (index, code_list) in code_lists_vec.iter().enumerate() { code_systems.insert(code_list); - let placeholder = - "{{A".to_string() + (index + 1).to_string().as_str() + "}}"; //to keep compatibility with snippets in typescript + let placeholder = format!("{{{{A{}}}}}", (index + 1).to_string()); //to keep compatibility with snippets in typescript condition_string = condition_string.replace(placeholder.as_str(), code_list); } @@ -142,36 +145,32 @@ pub fn process( ast::ConditionValue::DateRange(date_range) => { let datetime_str_min = date_range.min.as_str(); let datetime_result_min: Result, _> = - datetime_str_min.parse(); + datetime_str_min.parse().map_err(|_| { + FocusError::AstInvalidDateFormat(date_range.min) + }); - if let Ok(datetime_min) = datetime_result_min { - let date_str_min = - format!("@{}", datetime_min.format("%Y-%m-%d")); + let datetime_min = datetime_result_min.unwrap(); // we returned if Err + let date_str_min = format!("@{}", datetime_min.format("%Y-%m-%d")); - condition_string = - condition_string.replace("{{D1}}", date_str_min.as_str()); - filter_string = - filter_string.replace("{{D1}}", date_str_min.as_str()); + condition_string = + condition_string.replace("{{D1}}", date_str_min.as_str()); + filter_string = + filter_string.replace("{{D1}}", date_str_min.as_str()); // no condition needed, "" stays "" - } else { - return Err(FocusError::AstInvalidDateFormat(date_range.min)); - } let datetime_str_max = date_range.max.as_str(); let datetime_result_max: Result, _> = - datetime_str_max.parse(); - if let Ok(datetime_max) = datetime_result_max { - let date_str_max = - format!("@{}", datetime_max.format("%Y-%m-%d")); - - condition_string = - condition_string.replace("{{D2}}", date_str_max.as_str()); - filter_string = - filter_string.replace("{{D2}}", date_str_max.as_str()); + datetime_str_max.parse().map_err(|_| { + FocusError::AstInvalidDateFormat(date_range.max) + }); + let datetime_max = datetime_result_max.unwrap(); // we returned if Err + let date_str_max = format!("@{}", datetime_max.format("%Y-%m-%d")); + + condition_string = + condition_string.replace("{{D2}}", date_str_max.as_str()); + filter_string = + filter_string.replace("{{D2}}", date_str_max.as_str()); // no condition needed, "" stays "" - } else { - return Err(FocusError::AstInvalidDateFormat(date_range.max)); - } } ast::ConditionValue::NumRange(num_range) => { condition_string = condition_string @@ -184,8 +183,8 @@ pub fn process( .replace("{{D2}}", num_range.max.to_string().as_str()); // no condition needed, "" stays "" } - _ => { - return Err(FocusError::AstOperatorValueMismatch()); + other => { + return Err(FocusError::AstOperatorValueMismatch(format!("Operator BETWEEN can only be used for numerical and date values, not for {:?}", other))); } } } // deal with no lower or no upper value @@ -238,8 +237,8 @@ pub fn process( filter_string = filter_humongous_string + ")"; } } - _ => { - return Err(FocusError::AstOperatorValueMismatch()); + other => { + return Err(FocusError::AstOperatorValueMismatch(format!("Operator IN can only be used for string arrays, not for {:?}", other))); } } } // this becomes or of all @@ -283,16 +282,13 @@ pub fn process( filter_string = filter_humongous_string + ")"; } } - _ => { - return Err(FocusError::AstOperatorValueMismatch()); + other => { + return Err(FocusError::AstOperatorValueMismatch(format!("Operator EQUALS can only be used for string arrays, not for {:?}", other))); } }, - ast::ConditionType::NotEquals => { // won't get it from Lens yet + other => { // won't get it from Lens yet + info!("Got this condition type which Lens is not programmed to send, ignoring: {:?}", other); } - ast::ConditionType::Contains => { // won't get it from Lens yet - } - ast::ConditionType::GreaterThan => {} // won't get it from Lens yet - ast::ConditionType::LowerThan => {} // won't get it from Lens yet }; retrieval_cond += condition_string.as_str(); @@ -307,9 +303,6 @@ pub fn process( condition_key_trans.to_string(), )); } - if !filter_cond.is_empty() { - //for historical reasons - } } ast::Child::Operation(operation) => { @@ -323,7 +316,7 @@ pub fn process( grandchild.clone(), &mut retrieval_cond, &mut filter_cond, - code_systems + code_systems, )?; // Only concatenate operator if it's not the last element @@ -394,16 +387,12 @@ mod test { ); pretty_assertions::assert_eq!( - generate_cql( - serde_json::from_str(AGE_AT_DIAGNOSIS_30_TO_70).unwrap()) - .unwrap(), + generate_cql(serde_json::from_str(AGE_AT_DIAGNOSIS_30_TO_70).unwrap()).unwrap(), include_str!("../resources/test/result_age_at_diagnosis_30_to_70.cql").to_string() ); pretty_assertions::assert_eq!( - generate_cql( - serde_json::from_str(AGE_AT_DIAGNOSIS_LOWER_THAN_70).unwrap()) - .unwrap(), + generate_cql(serde_json::from_str(AGE_AT_DIAGNOSIS_LOWER_THAN_70).unwrap()).unwrap(), include_str!("../resources/test/result_age_at_diagnosis_lower_than_70.cql").to_string() ); diff --git a/src/errors.rs b/src/errors.rs index f631a75..c67c805 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -51,7 +51,7 @@ pub enum FocusError { #[error("Unknown option in AST: {0}")] AstUnknownOption(String), #[error("Mismatch between operator and value type")] - AstOperatorValueMismatch(), + AstOperatorValueMismatch(String), #[error("Invalid date format: {0}")] AstInvalidDateFormat(String), #[error("Invalid Header Value: {0}")] @@ -68,7 +68,7 @@ impl FocusError { use FocusError::*; // TODO: Add more match arms match self { - DecodeError(_) | SerdeParsingError(_) => "Cannot parse query.", + DecodeError(_) | ParsingError(_) | SerdeParsingError(_) => "Cannot parse query.", LaplaceError(_) => "Cannot obfuscate result.", _ => "Failed to execute query." } diff --git a/src/main.rs b/src/main.rs index d1915a4..c3f1763 100644 --- a/src/main.rs +++ b/src/main.rs @@ -48,6 +48,13 @@ type Created = std::time::SystemTime; //epoch type BeamTask = TaskRequest; type BeamResult = TaskResult; +#[derive(Deserialize, Debug)] +#[serde(tag = "lang", rename_all = "lowercase")] +enum Language { + Cql(CqlQuery), + Ast(AstQuery) +} + #[derive(Debug, Deserialize, Serialize, Clone)] struct Metadata { project: String, @@ -171,12 +178,6 @@ async fn process_task( } if CONFIG.endpoint_type == EndpointType::Blaze { - #[derive(Deserialize, Debug)] - #[serde(tag = "lang", rename_all = "lowercase")] - enum Language { - Cql(CqlQuery), - Ast(AstQuery) - } let mut generated_from_ast: bool = false; let data = base64_decode(&task.body)?; let query: CqlQuery = match serde_json::from_slice::(&data)? { From 27e7f057ba36c8358061e5a7f14d64d706931864 Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Tue, 11 Jun 2024 13:58:23 +0200 Subject: [PATCH 32/42] unicode license version change --- deny.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deny.toml b/deny.toml index 2bfd0b0..a144283 100644 --- a/deny.toml +++ b/deny.toml @@ -80,7 +80,7 @@ allow = [ "Apache-2.0", "ISC", "BSD-3-Clause", - "Unicode-DFS-2016", + "Unicode-3.0", #"Apache-2.0 WITH LLVM-exception", ] # List of explicitly disallowed licenses From 03c83b5064d7ccd9cc8afed8847464adc7ac9642 Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Tue, 11 Jun 2024 14:03:35 +0200 Subject: [PATCH 33/42] laplace version bump --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 56b3ee0..63eb7cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ chrono = "0.4.31" indexmap = "2.1.0" tokio = { version = "1.25.0", default_features = false, features = ["signal", "rt-multi-thread", "macros"] } beam-lib = { git = "https://github.com/samply/beam", branch = "develop", features = ["http-util"] } -laplace_rs = {version = "0.2.0", git = "https://github.com/samply/laplace-rs.git", branch = "main" } +laplace_rs = {version = "0.3.0", git = "https://github.com/samply/laplace-rs.git", branch = "main" } uuid = "1.8.0" rand = { default-features = false, version = "0.8.5" } futures-util = { version = "0.3", default-features = false, features = ["std"] } From b41e772032c7c85e9aedfeec2c0a82f3676373ae Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Tue, 11 Jun 2024 14:06:12 +0200 Subject: [PATCH 34/42] tag added to laplace crate --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 63eb7cd..de438f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ chrono = "0.4.31" indexmap = "2.1.0" tokio = { version = "1.25.0", default_features = false, features = ["signal", "rt-multi-thread", "macros"] } beam-lib = { git = "https://github.com/samply/beam", branch = "develop", features = ["http-util"] } -laplace_rs = {version = "0.3.0", git = "https://github.com/samply/laplace-rs.git", branch = "main" } +laplace_rs = {git = "https://github.com/samply/laplace-rs.git", tag = "v0.3.0" } uuid = "1.8.0" rand = { default-features = false, version = "0.8.5" } futures-util = { version = "0.3", default-features = false, features = ["std"] } From f6d66fac205e50fdbd306ad62622f6b1ab07a894 Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Tue, 11 Jun 2024 14:08:37 +0200 Subject: [PATCH 35/42] old unicode license in deny --- deny.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/deny.toml b/deny.toml index a144283..fbf4fae 100644 --- a/deny.toml +++ b/deny.toml @@ -80,6 +80,7 @@ allow = [ "Apache-2.0", "ISC", "BSD-3-Clause", + "Unicode-DFS-2016", "Unicode-3.0", #"Apache-2.0 WITH LLVM-exception", ] From 45dbc4c1870009207b2a01c1e0e23257b6c6ee47 Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Wed, 12 Jun 2024 10:46:30 +0200 Subject: [PATCH 36/42] renamed bbmri queries results of to cache file --- README.md | 2 +- .../{bbmri => bbmri_base64_encoded_queries_to_cache_results_of} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename resources/{bbmri => bbmri_base64_encoded_queries_to_cache_results_of} (100%) diff --git a/README.md b/README.md index 3b9e28c..84a22f6 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ DELTA_HISTO = "20." # Sensitivity parameter for obfuscating the counts in the Hi EPSILON = "0.1" # Privacy budget parameter for obfuscating the counts in the stratifiers, has no effect if OBFUSCATE = "no", default value: 0.1 ROUNDING_STEP = "10" # The granularity of the rounding of the obfuscated values, has no effect if OBFUSCATE = "no", default value: 10 PROJECTS_NO_OBFUSCATION = "exliquid;dktk_supervisors;exporter;ehds2" # Projects for which the results are not to be obfuscated, separated by ;, default value: "exliquid;dktk_supervisors;exporter;ehds2" -QUERIES_TO_CACHE_FILE_PATH = "resources/bbmri" # The path to the file containing BASE64 encoded queries whose results are to be cached, if not set, no results are cached +QUERIES_TO_CACHE_FILE_PATH = "resources/bbmri_base64_encoded_queries_to_cache_results" # The path to the file containing BASE64 encoded queries whose results are to be cached, if not set, no results are cached PROVIDER = "name" #OMOP provider name PROVIDER_ICON = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQMAAAAl21bKAAAAA1BMVEUAAACnej3aAAAAAXRSTlMAQObYZgAAAApJREFUCNdjYAAAAAIAAeIhvDMAAAAASUVORK5CYII=" #Base64 encoded OMOP provider icon AUTH_HEADER = "ApiKey XXXX" #Authorization header diff --git a/resources/bbmri b/resources/bbmri_base64_encoded_queries_to_cache_results_of similarity index 100% rename from resources/bbmri rename to resources/bbmri_base64_encoded_queries_to_cache_results_of From 4ae149319499e8d75d9d4b5e655aa03d3c3e445c Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Wed, 12 Jun 2024 10:50:31 +0200 Subject: [PATCH 37/42] added "of" --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 84a22f6..00e794f 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ DELTA_HISTO = "20." # Sensitivity parameter for obfuscating the counts in the Hi EPSILON = "0.1" # Privacy budget parameter for obfuscating the counts in the stratifiers, has no effect if OBFUSCATE = "no", default value: 0.1 ROUNDING_STEP = "10" # The granularity of the rounding of the obfuscated values, has no effect if OBFUSCATE = "no", default value: 10 PROJECTS_NO_OBFUSCATION = "exliquid;dktk_supervisors;exporter;ehds2" # Projects for which the results are not to be obfuscated, separated by ;, default value: "exliquid;dktk_supervisors;exporter;ehds2" -QUERIES_TO_CACHE_FILE_PATH = "resources/bbmri_base64_encoded_queries_to_cache_results" # The path to the file containing BASE64 encoded queries whose results are to be cached, if not set, no results are cached +QUERIES_TO_CACHE_FILE_PATH = "resources/bbmri_base64_encoded_queries_to_cache_results_of" # The path to the file containing BASE64 encoded queries whose results are to be cached, if not set, no results are cached PROVIDER = "name" #OMOP provider name PROVIDER_ICON = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQMAAAAl21bKAAAAA1BMVEUAAACnej3aAAAAAXRSTlMAQObYZgAAAApJREFUCNdjYAAAAAIAAeIhvDMAAAAASUVORK5CYII=" #Base64 encoded OMOP provider icon AUTH_HEADER = "ApiKey XXXX" #Authorization header From 3b50d27d0bc8fdb94fc655e8c0ffe50a844c2bc9 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Wed, 12 Jun 2024 16:43:20 +0200 Subject: [PATCH 38/42] Change config parameter to QUERIES_TO_CACHE --- README.md | 2 +- src/config.rs | 6 +++--- src/main.rs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 00e794f..ecf6abd 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ DELTA_HISTO = "20." # Sensitivity parameter for obfuscating the counts in the Hi EPSILON = "0.1" # Privacy budget parameter for obfuscating the counts in the stratifiers, has no effect if OBFUSCATE = "no", default value: 0.1 ROUNDING_STEP = "10" # The granularity of the rounding of the obfuscated values, has no effect if OBFUSCATE = "no", default value: 10 PROJECTS_NO_OBFUSCATION = "exliquid;dktk_supervisors;exporter;ehds2" # Projects for which the results are not to be obfuscated, separated by ;, default value: "exliquid;dktk_supervisors;exporter;ehds2" -QUERIES_TO_CACHE_FILE_PATH = "resources/bbmri_base64_encoded_queries_to_cache_results_of" # The path to the file containing BASE64 encoded queries whose results are to be cached, if not set, no results are cached +QUERIES_TO_CACHE = "queries_to_cache.conf" # The path to a file containing base64 encoded queries whose results are to be cached. If not set, no results are cached PROVIDER = "name" #OMOP provider name PROVIDER_ICON = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQMAAAAl21bKAAAAA1BMVEUAAACnej3aAAAAAXRSTlMAQObYZgAAAApJREFUCNdjYAAAAAIAAeIhvDMAAAAASUVORK5CYII=" #Base64 encoded OMOP provider icon AUTH_HEADER = "ApiKey XXXX" #Authorization header diff --git a/src/config.rs b/src/config.rs index 3a5acdc..6dd067d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -133,7 +133,7 @@ struct CliArgs { /// The path to the file containing BASE64 encoded queries whose results are to be cached #[clap(long, env, value_parser)] - queries_to_cache_file_path: Option, + queries_to_cache: Option, /// Outgoing HTTP proxy: Directory with CA certificates to trust for TLS connections (e.g. /etc/samply/cacerts/) #[clap(long, env, value_parser)] @@ -173,7 +173,7 @@ pub(crate) struct Config { pub epsilon: f64, pub rounding_step: usize, pub unobfuscated: Vec, - pub queries_to_cache_file_path: Option, + pub queries_to_cache: Option, tls_ca_certificates: Vec, pub client: Client, pub provider: Option, @@ -216,7 +216,7 @@ impl Config { epsilon: cli_args.epsilon, rounding_step: cli_args.rounding_step, unobfuscated: cli_args.projects_no_obfuscation.split(';').map(|s| s.to_string()).collect(), - queries_to_cache_file_path: cli_args.queries_to_cache_file_path, + queries_to_cache: cli_args.queries_to_cache, tls_ca_certificates, provider: cli_args.provider, provider_icon: cli_args.provider_icon, diff --git a/src/main.rs b/src/main.rs index c3f1763..63f8f92 100644 --- a/src/main.rs +++ b/src/main.rs @@ -70,7 +70,7 @@ impl ReportCache { pub fn new() -> Self { let mut cache = HashMap::new(); - if let Some(filename) = CONFIG.queries_to_cache_file_path.clone() { + if let Some(filename) = CONFIG.queries_to_cache.clone() { let lines = util::read_lines(filename.clone().to_string()); match lines { Ok(ok_lines) => { From 4be7c0073fe8c3986a1b20f1bfc449b37ea4b136 Mon Sep 17 00:00:00 2001 From: Martin Lablans Date: Wed, 12 Jun 2024 16:43:30 +0200 Subject: [PATCH 39/42] Update documentation --- README.md | 2 +- src/config.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ecf6abd..ee937b0 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ ROUNDING_STEP = "10" # The granularity of the rounding of the obfuscated values, PROJECTS_NO_OBFUSCATION = "exliquid;dktk_supervisors;exporter;ehds2" # Projects for which the results are not to be obfuscated, separated by ;, default value: "exliquid;dktk_supervisors;exporter;ehds2" QUERIES_TO_CACHE = "queries_to_cache.conf" # The path to a file containing base64 encoded queries whose results are to be cached. If not set, no results are cached PROVIDER = "name" #OMOP provider name -PROVIDER_ICON = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQMAAAAl21bKAAAAA1BMVEUAAACnej3aAAAAAXRSTlMAQObYZgAAAApJREFUCNdjYAAAAAIAAeIhvDMAAAAASUVORK5CYII=" #Base64 encoded OMOP provider icon +PROVIDER_ICON = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQMAAAAl21bKAAAAA1BMVEUAAACnej3aAAAAAXRSTlMAQObYZgAAAApJREFUCNdjYAAAAAIAAeIhvDMAAAAASUVORK5CYII=" # Base64 encoded OMOP provider icon AUTH_HEADER = "ApiKey XXXX" #Authorization header ``` diff --git a/src/config.rs b/src/config.rs index 6dd067d..c451a41 100644 --- a/src/config.rs +++ b/src/config.rs @@ -131,7 +131,7 @@ struct CliArgs { #[clap(long, env, value_parser, default_value = "exliquid;dktk_supervisors;exporter;ehds2")] projects_no_obfuscation: String, - /// The path to the file containing BASE64 encoded queries whose results are to be cached + /// Path to a file containing BASE64 encoded queries whose results are to be cached #[clap(long, env, value_parser)] queries_to_cache: Option, From b27ec5994e244c22788921dec6045d6193cf3aa5 Mon Sep 17 00:00:00 2001 From: lablans Date: Wed, 12 Jun 2024 14:54:36 +0000 Subject: [PATCH 40/42] Use PathBuf for path --- src/config.rs | 2 +- src/main.rs | 6 +++--- src/util.rs | 7 ++++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/config.rs b/src/config.rs index c451a41..3c783ec 100644 --- a/src/config.rs +++ b/src/config.rs @@ -173,7 +173,7 @@ pub(crate) struct Config { pub epsilon: f64, pub rounding_step: usize, pub unobfuscated: Vec, - pub queries_to_cache: Option, + pub queries_to_cache: Option, tls_ca_certificates: Vec, pub client: Client, pub provider: Option, diff --git a/src/main.rs b/src/main.rs index 63f8f92..5aa9f50 100644 --- a/src/main.rs +++ b/src/main.rs @@ -71,12 +71,12 @@ impl ReportCache { let mut cache = HashMap::new(); if let Some(filename) = CONFIG.queries_to_cache.clone() { - let lines = util::read_lines(filename.clone().to_string()); + let lines = util::read_lines(&filename); match lines { Ok(ok_lines) => { for line in ok_lines { let Ok(ok_line) = line else { - warn!("A line in the file {} is not readable", filename); + warn!("A line in the file {} is not readable", filename.display()); continue; }; cache.insert((ok_line.clone(), false), ("".into(), UNIX_EPOCH)); @@ -84,7 +84,7 @@ impl ReportCache { } } Err(_) => { - error!("The file {} cannot be opened", filename); //This shouldn't stop focus from running, it's just going to go to blaze every time, but that's not too slow + error!("The file {} cannot be opened", filename.display()); //This shouldn't stop focus from running, it's just going to go to blaze every time, but that's not too slow } } } diff --git a/src/util.rs b/src/util.rs index bc90047..7f6c359 100644 --- a/src/util.rs +++ b/src/util.rs @@ -8,6 +8,7 @@ use serde_json::{json, Value}; use std::collections::HashMap; use std::fs::File; use std::io::{self, BufRead, BufReader}; +use std::path::{Path, PathBuf}; use tracing::warn; #[derive(Debug, Deserialize, Serialize)] @@ -76,9 +77,9 @@ pub(crate) fn get_json_field(json_string: &str, field: &str) -> Result Result>, FocusError> { - let file = File::open(filename.clone()).map_err(|e| { - FocusError::FileOpeningError(format!("Cannot open file {}: {} ", filename, e)) +pub(crate) fn read_lines(filename: &Path) -> Result>, FocusError> { + let file = File::open(filename).map_err(|e| { + FocusError::FileOpeningError(format!("Cannot open file {}: {} ", filename.display(), e)) })?; Ok(io::BufReader::new(file).lines()) } From 814b72dcaf6363f0037d6292d687e0f8d0759c63 Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Thu, 13 Jun 2024 14:24:01 +0200 Subject: [PATCH 41/42] empty features --- .github/workflows/rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index be8db06..f7efcbb 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -17,7 +17,7 @@ jobs: #architectures: '[ "amd64", "arm64" ]' #profile: debug test-via-script: false - features: '[ "bbmri", "dktk" ]' + features: '[ "bbmri", "dktk", "" ]' push-to: ${{ (github.ref_protected == true || github.event_name == 'workflow_dispatch') && 'dockerhub' || 'ghcr' }} secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} From 28ed3d30ced832faec06f5f8ab559cf872607cf6 Mon Sep 17 00:00:00 2001 From: Enola Knezevic <115070135+enola-dkfz@users.noreply.github.com> Date: Wed, 19 Jun 2024 14:35:22 +0200 Subject: [PATCH 42/42] requested changes (#152) * requested changes * Update src/cql.rs Co-authored-by: Jan <59206115+Threated@users.noreply.github.com> * Update src/cql.rs Co-authored-by: Jan <59206115+Threated@users.noreply.github.com> * Update src/cql.rs Co-authored-by: Jan <59206115+Threated@users.noreply.github.com> * DateTime type added --------- Co-authored-by: Jan <59206115+Threated@users.noreply.github.com> --- Cargo.toml | 4 +- README.md | 10 +- resources/test/result.cql | 0 src/blaze.rs | 2 +- src/cql.rs | 301 ++++++++++++++++++-------------------- src/errors.rs | 2 +- src/util.rs | 2 +- 7 files changed, 155 insertions(+), 166 deletions(-) delete mode 100644 resources/test/result.cql diff --git a/Cargo.toml b/Cargo.toml index de438f7..168551a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "focus" -version = "0.5.2" +version = "0.6.0" edition = "2021" license = "Apache-2.0" @@ -32,8 +32,6 @@ once_cell = "1.18" # Command Line Interface clap = { version = "4", default_features = false, features = ["std", "env", "derive", "help", "color"] } - - [features] default = [] bbmri = [] diff --git a/README.md b/README.md index ee937b0..2de2579 100644 --- a/README.md +++ b/README.md @@ -60,32 +60,32 @@ Optionally, you can provide the `TLS_CA_CERTIFICATES_DIR` environment variable t ## Usage -Creating a sample focus healthcheck task using CURL (body can be any string and is ignored): +Creating a sample focus healthcheck task using curl (body can be any string and is ignored): ```bash curl -v -X POST -H "Content-Type: application/json" --data '{"id":"7fffefff-ffef-fcff-feef-feffffffffff","from":"app1.proxy1.broker","to":["app1.proxy1.broker"],"ttl":"10s","failure_strategy":{"retry":{"backoff_millisecs":1000,"max_tries":5}},"metadata":{"project":"focus-healthcheck"},"body":"wie geht es"}' -H "Authorization: ApiKey app1.proxy1.broker App1Secret" http://localhost:8081/v1/tasks ``` -Creating a sample task containing a [Blaze](https://github.com/samply/blaze) query using CURL: +Creating a sample task containing a [Blaze](https://github.com/samply/blaze) query using curl: ```bash curl -v -X POST -H "Content-Type: application/json" --data '{"id":"7fffefff-ffef-fcff-feef-fefbffffeeff","from":"app1.proxy1.broker","to":["app1.proxy1.broker"],"ttl":"10s","failure_strategy":{"retry":{"backoff_millisecs":1000,"max_tries":5}},"metadata":{"project":"exliquid"},"body":"ewoJImxhbmciOiAiY3FsIiwKCSJsaWIiOiB7CgkJImNvbnRlbnQiOiBbCgkJCXsKCQkJCSJjb250ZW50VHlwZSI6ICJ0ZXh0L2NxbCIsCgkJCQkiZGF0YSI6ICJiR2xpY21GeWVTQlNaWFJ5YVdWMlpRcDFjMmx1WnlCR1NFbFNJSFpsY25OcGIyNGdKelF1TUM0d0p3cHBibU5zZFdSbElFWklTVkpJWld4d1pYSnpJSFpsY25OcGIyNGdKelF1TUM0d0p3b0tZMjlrWlhONWMzUmxiU0JzYjJsdVl6b2dKMmgwZEhBNkx5OXNiMmx1WXk1dmNtY25DbU52WkdWemVYTjBaVzBnYVdOa01UQTZJQ2RvZEhSd09pOHZhR3czTG05eVp5OW1hR2x5TDNOcFpDOXBZMlF0TVRBbkNtTnZaR1Z6ZVhOMFpXMGdVMkZ0Y0d4bFRXRjBaWEpwWVd4VWVYQmxPaUFuYUhSMGNITTZMeTltYUdseUxtSmliWEpwTG1SbEwwTnZaR1ZUZVhOMFpXMHZVMkZ0Y0d4bFRXRjBaWEpwWVd4VWVYQmxKd29LQ21OdmJuUmxlSFFnVUdGMGFXVnVkQW9LUWtKTlVrbGZVMVJTUVZSZlIwVk9SRVZTWDFOVVVrRlVTVVpKUlZJS0NrSkNUVkpKWDFOVVVrRlVYMFJGUmw5VFVFVkRTVTFGVGdwcFppQkpia2x1YVhScFlXeFFiM0IxYkdGMGFXOXVJSFJvWlc0Z1cxTndaV05wYldWdVhTQmxiSE5sSUh0OUlHRnpJRXhwYzNROFUzQmxZMmx0Wlc0K0NncENRazFTU1Y5VFZGSkJWRjlUUVUxUVRFVmZWRmxRUlY5VFZGSkJWRWxHU1VWU0NncENRazFTU1Y5VFZGSkJWRjlEVlZOVVQwUkpRVTVmVTFSU1FWUkpSa2xGVWdvS1FrSk5Va2xmVTFSU1FWUmZSRWxCUjA1UFUwbFRYMU5VVWtGVVNVWkpSVklLQ2tKQ1RWSkpYMU5VVWtGVVgwRkhSVjlUVkZKQlZFbEdTVVZTQ2dwQ1FrMVNTVjlUVkZKQlZGOUVSVVpmU1U1ZlNVNUpWRWxCVEY5UVQxQlZURUZVU1U5T0NuUnlkV1U9IgoJCQl9CgkJXSwKCQkicmVzb3VyY2VUeXBlIjogIkxpYnJhcnkiLAoJCSJzdGF0dXMiOiAiYWN0aXZlIiwKCQkidHlwZSI6IHsKCQkJImNvZGluZyI6IFsKCQkJCXsKCQkJCQkiY29kZSI6ICJsb2dpYy1saWJyYXJ5IiwKCQkJCQkic3lzdGVtIjogImh0dHA6Ly90ZXJtaW5vbG9neS5obDcub3JnL0NvZGVTeXN0ZW0vbGlicmFyeS10eXBlIgoJCQkJfQoJCQldCgkJfSwKCQkidXJsIjogInVybjp1dWlkOjdmZjUzMmFkLTY5ZTQtNDhlZC1hMmQzLTllZmFmYjYwOWY2MiIKCX0sCgkibWVhc3VyZSI6IHsKCQkiZ3JvdXAiOiBbCgkJCXsKCQkJCSJjb2RlIjogewoJCQkJCSJ0ZXh0IjogInBhdGllbnRzIgoJCQkJfSwKCQkJCSJwb3B1bGF0aW9uIjogWwoJCQkJCXsKCQkJCQkJImNvZGUiOiB7CgkJCQkJCQkiY29kaW5nIjogWwoJCQkJCQkJCXsKCQkJCQkJCQkJImNvZGUiOiAiaW5pdGlhbC1wb3B1bGF0aW9uIiwKCQkJCQkJCQkJInN5c3RlbSI6ICJodHRwOi8vdGVybWlub2xvZ3kuaGw3Lm9yZy9Db2RlU3lzdGVtL21lYXN1cmUtcG9wdWxhdGlvbiIKCQkJCQkJCQl9CgkJCQkJCQldCgkJCQkJCX0sCgkJCQkJCSJjcml0ZXJpYSI6IHsKCQkJCQkJCSJleHByZXNzaW9uIjogIkluSW5pdGlhbFBvcHVsYXRpb24iLAoJCQkJCQkJImxhbmd1YWdlIjogInRleHQvY3FsLWlkZW50aWZpZXIiCgkJCQkJCX0KCQkJCQl9CgkJCQldLAoJCQkJInN0cmF0aWZpZXIiOiBbCgkJCQkJewoJCQkJCQkiY29kZSI6IHsKCQkJCQkJCSJ0ZXh0IjogIkdlbmRlciIKCQkJCQkJfSwKCQkJCQkJImNyaXRlcmlhIjogewoJCQkJCQkJImV4cHJlc3Npb24iOiAiR2VuZGVyIiwKCQkJCQkJCSJsYW5ndWFnZSI6ICJ0ZXh0L2NxbCIKCQkJCQkJfQoJCQkJCX0sCgkJCQkJewoJCQkJCQkiY29kZSI6IHsKCQkJCQkJCSJ0ZXh0IjogIkFnZSIKCQkJCQkJfSwKCQkJCQkJImNyaXRlcmlhIjogewoJCQkJCQkJImV4cHJlc3Npb24iOiAiQWdlQ2xhc3MiLAoJCQkJCQkJImxhbmd1YWdlIjogInRleHQvY3FsIgoJCQkJCQl9CgkJCQkJfSwKCQkJCQl7CgkJCQkJCSJjb2RlIjogewoJCQkJCQkJInRleHQiOiAiQ3VzdG9kaWFuIgoJCQkJCQl9LAoJCQkJCQkiY3JpdGVyaWEiOiB7CgkJCQkJCQkiZXhwcmVzc2lvbiI6ICJDdXN0b2RpYW4iLAoJCQkJCQkJImxhbmd1YWdlIjogInRleHQvY3FsIgoJCQkJCQl9CgkJCQkJfQoJCQkJXQoJCQl9LAoJCQl7CgkJCQkiY29kZSI6IHsKCQkJCQkidGV4dCI6ICJkaWFnbm9zaXMiCgkJCQl9LAoJCQkJImV4dGVuc2lvbiI6IFsKCQkJCQl7CgkJCQkJCSJ1cmwiOiAiaHR0cDovL2hsNy5vcmcvZmhpci91cy9jcWZtZWFzdXJlcy9TdHJ1Y3R1cmVEZWZpbml0aW9uL2NxZm0tcG9wdWxhdGlvbkJhc2lzIiwKCQkJCQkJInZhbHVlQ29kZSI6ICJDb25kaXRpb24iCgkJCQkJfQoJCQkJXSwKCQkJCSJwb3B1bGF0aW9uIjogWwoJCQkJCXsKCQkJCQkJImNvZGUiOiB7CgkJCQkJCQkiY29kaW5nIjogWwoJCQkJCQkJCXsKCQkJCQkJCQkJImNvZGUiOiAiaW5pdGlhbC1wb3B1bGF0aW9uIiwKCQkJCQkJCQkJInN5c3RlbSI6ICJodHRwOi8vdGVybWlub2xvZ3kuaGw3Lm9yZy9Db2RlU3lzdGVtL21lYXN1cmUtcG9wdWxhdGlvbiIKCQkJCQkJCQl9CgkJCQkJCQldCgkJCQkJCX0sCgkJCQkJCSJjcml0ZXJpYSI6IHsKCQkJCQkJCSJleHByZXNzaW9uIjogIkRpYWdub3NpcyIsCgkJCQkJCQkibGFuZ3VhZ2UiOiAidGV4dC9jcWwtaWRlbnRpZmllciIKCQkJCQkJfQoJCQkJCX0KCQkJCV0sCgkJCQkic3RyYXRpZmllciI6IFsKCQkJCQl7CgkJCQkJCSJjb2RlIjogewoJCQkJCQkJInRleHQiOiAiZGlhZ25vc2lzIgoJCQkJCQl9LAoJCQkJCQkiY3JpdGVyaWEiOiB7CgkJCQkJCQkiZXhwcmVzc2lvbiI6ICJEaWFnbm9zaXNDb2RlIiwKCQkJCQkJCSJsYW5ndWFnZSI6ICJ0ZXh0L2NxbC1pZGVudGlmaWVyIgoJCQkJCQl9CgkJCQkJfQoJCQkJXQoJCQl9LAoJCQl7CgkJCQkiY29kZSI6IHsKCQkJCQkidGV4dCI6ICJzcGVjaW1lbiIKCQkJCX0sCgkJCQkiZXh0ZW5zaW9uIjogWwoJCQkJCXsKCQkJCQkJInVybCI6ICJodHRwOi8vaGw3Lm9yZy9maGlyL3VzL2NxZm1lYXN1cmVzL1N0cnVjdHVyZURlZmluaXRpb24vY3FmbS1wb3B1bGF0aW9uQmFzaXMiLAoJCQkJCQkidmFsdWVDb2RlIjogIlNwZWNpbWVuIgoJCQkJCX0KCQkJCV0sCgkJCQkicG9wdWxhdGlvbiI6IFsKCQkJCQl7CgkJCQkJCSJjb2RlIjogewoJCQkJCQkJImNvZGluZyI6IFsKCQkJCQkJCQl7CgkJCQkJCQkJCSJjb2RlIjogImluaXRpYWwtcG9wdWxhdGlvbiIsCgkJCQkJCQkJCSJzeXN0ZW0iOiAiaHR0cDovL3Rlcm1pbm9sb2d5LmhsNy5vcmcvQ29kZVN5c3RlbS9tZWFzdXJlLXBvcHVsYXRpb24iCgkJCQkJCQkJfQoJCQkJCQkJXQoJCQkJCQl9LAoJCQkJCQkiY3JpdGVyaWEiOiB7CgkJCQkJCQkiZXhwcmVzc2lvbiI6ICJTcGVjaW1lbiIsCgkJCQkJCQkibGFuZ3VhZ2UiOiAidGV4dC9jcWwtaWRlbnRpZmllciIKCQkJCQkJfQoJCQkJCX0KCQkJCV0sCgkJCQkic3RyYXRpZmllciI6IFsKCQkJCQl7CgkJCQkJCSJjb2RlIjogewoJCQkJCQkJInRleHQiOiAic2FtcGxlX2tpbmQiCgkJCQkJCX0sCgkJCQkJCSJjcml0ZXJpYSI6IHsKCQkJCQkJCSJleHByZXNzaW9uIjogIlNhbXBsZVR5cGUiLAoJCQkJCQkJImxhbmd1YWdlIjogInRleHQvY3FsIgoJCQkJCQl9CgkJCQkJfQoJCQkJXQoJCQl9CgkJXSwKCQkibGlicmFyeSI6ICJ1cm46dXVpZDo3ZmY1MzJhZC02OWU0LTQ4ZWQtYTJkMy05ZWZhZmI2MDlmNjIiLAoJCSJyZXNvdXJjZVR5cGUiOiAiTWVhc3VyZSIsCgkJInNjb3JpbmciOiB7CgkJCSJjb2RpbmciOiBbCgkJCQl7CgkJCQkJImNvZGUiOiAiY29ob3J0IiwKCQkJCQkic3lzdGVtIjogImh0dHA6Ly90ZXJtaW5vbG9neS5obDcub3JnL0NvZGVTeXN0ZW0vbWVhc3VyZS1zY29yaW5nIgoJCQkJfQoJCQldCgkJfSwKCQkic3RhdHVzIjogImFjdGl2ZSIsCgkJInN1YmplY3RDb2RlYWJsZUNvbmNlcHQiOiB7CgkJCSJjb2RpbmciOiBbCgkJCQl7CgkJCQkJImNvZGUiOiAiUGF0aWVudCIsCgkJCQkJInN5c3RlbSI6ICJodHRwOi8vaGw3Lm9yZy9maGlyL3Jlc291cmNlLXR5cGVzIgoJCQkJfQoJCQldCgkJfSwKCQkidXJsIjogInVybjp1dWlkOjVlZThkZTczLTM0N2UtNDdjYS1hMDE0LWYyZTcxNzY3YWRmYyIKCX0KfQ=="}' -H "Authorization: ApiKey app1.proxy1.broker App1Secret" http://localhost:8081/v1/tasks ``` -Creating a sample task containing an abstract syntax tree (AST) query using CURL: +Creating a sample task containing an abstract syntax tree (AST) query using curl: ```bash curl -v -X POST -H "Content-Type: application/json" --data '{"id":"7fffefff-ffef-fcff-feef-feffffffffff","from":"app1.proxy1.broker","to":["app1.proxy1.broker"],"ttl":"10s","failure_strategy":{"retry":{"backoff_millisecs":1000,"max_tries":5}},"metadata":{"project":"bbmri"},"body":"eyJsYW5nIjoiYXN0IiwicGF5bG9hZCI6ImV5SmhjM1FpT25zaWIzQmxjbUZ1WkNJNklrOVNJaXdpWTJocGJHUnlaVzRpT2x0N0ltOXdaWEpoYm1RaU9pSkJUa1FpTENKamFHbHNaSEpsYmlJNlczc2liM0JsY21GdVpDSTZJazlTSWl3aVkyaHBiR1J5Wlc0aU9sdDdJbXRsZVNJNkltZGxibVJsY2lJc0luUjVjR1VpT2lKRlVWVkJURk1pTENKemVYTjBaVzBpT2lJaUxDSjJZV3gxWlNJNkltMWhiR1VpZlN4N0ltdGxlU0k2SW1kbGJtUmxjaUlzSW5SNWNHVWlPaUpGVVZWQlRGTWlMQ0p6ZVhOMFpXMGlPaUlpTENKMllXeDFaU0k2SW1abGJXRnNaU0o5WFgxZGZWMTlMQ0pwWkNJNkltRTJaakZqWTJZekxXVmlaakV0TkRJMFppMDVaRFk1TFRSbE5XUXhNelZtTWpNME1DSjkifQ=="}' -H "Authorization: ApiKey app1.proxy1.broker App1Secret" http://localhost:8081/v1/tasks ``` -Creating a sample [Exporter](https://github.com/samply/exporter) "execute" task containing an Exporter query using CURL: +Creating a sample [Exporter](https://github.com/samply/exporter) "execute" task containing an Exporter query using curl: ```bash curl -v -X POST -H "Content-Type: application/json" --data '{"body":"ew0KICAicXVlcnktY29udGV4dCIgOiAiVUZKUFNrVkRWQzFKUkQxa01qaGhZVEl5Wm1Wa01USTBNemM0T0RWallnPT0iLA0KICAicXVlcnktbGFiZWwiIDogIlRlc3QgMyIsDQogICJxdWVyeS1leGVjdXRpb24tY29udGFjdC1pZCIgOiAiYmstYWRtaW5AdGVzdC5kZmt6LmRlIiwNCiAgInF1ZXJ5LWRlc2NyaXB0aW9uIiA6ICJUaGlzIGlzIHRoZSB0ZXN0IDMiLA0KICAicXVlcnktZXhwaXJhdGlvbi1kYXRlIiA6ICIyMDI0LTA4LTE0IiwNCiAgIm91dHB1dC1mb3JtYXQiIDogIkVYQ0VMIiwNCiAgInF1ZXJ5IiA6ICJleUpzWVc1bklqb2lZM0ZzSWl3aWJHbGlJanA3SW5KbGMyOTFjbU5sVkhsd1pTSTZJa3hwWW5KaGNua2lMQ0oxY213aU9pSjFjbTQ2ZFhWcFpEcGpOelJrWmpJd05DMDFZalppTFRSaFpXUXRZakl5T0MwM1pqVXpNekE0TnpZME5UZ2lMQ0p6ZEdGMGRYTWlPaUpoWTNScGRtVWlMQ0owZVhCbElqcDdJbU52WkdsdVp5STZXM3NpYzNsemRHVnRJam9pYUhSMGNEb3ZMM1JsY20xcGJtOXNiMmQ1TG1oc055NXZjbWN2UTI5a1pWTjVjM1JsYlM5c2FXSnlZWEo1TFhSNWNHVWlMQ0pqYjJSbElqb2liRzluYVdNdGJHbGljbUZ5ZVNKOVhYMHNJbU52Ym5SbGJuUWlPbHQ3SW1OdmJuUmxiblJVZVhCbElqb2lkR1Y0ZEM5amNXd2lMQ0prWVhSaElqb2lZa2RzYVdOdFJubGxVMEpUV2xoU2VXRlhWakphVVhBeFl6SnNkVnA1UWtkVFJXeFRTVWhhYkdOdVRuQmlNalJuU25wUmRVMUROSGRLZDNCd1ltMU9jMlJYVW14SlJWcEpVMVpLU1ZwWGVIZGFXRXA2U1VoYWJHTnVUbkJpTWpSblNucFJkVTFETkhkS2QyOUxXVEk1YTFwWVRqVmpNMUpzWWxOQ2MySXliSFZaZW05blNqSm9NR1JJUVRaTWVUbHpZakpzZFZsNU5YWmpiV051UTJkd2FtSXlOVEJhV0dnd1NVWkNhR1JIYkd4aWJsRkxRMmR3UlZNeFVreFlNVTVWVld0R1ZWZ3daRVpVYTFKR1ZXdzVWRlpHU2tKV1JXeEhVMVZXVTBObmNFVlRNVkpNV0RGT1ZWVnJSbFZZTVVKVFUxVXhRbFZzYkdaU1JXeENVakExVUZVd2JGUllNVTVWVld0R1ZWTlZXa3BTVmtsTFVrVjBWVk14T1ZSV1JrcENWa1k1UWxJd1ZtWlJNSGhDVlRGT1psVXhVbE5SVmxKS1VtdHNSbFZuYjB0U1JYUlZVekU1VkZaR1NrSldSamxGVWxWT1JsRldUa1pTUmpsVVZrWktRbFpGYkVkVFZWWlRRMmR3UlZNeFVreFlNVTVWVld0R1ZWZ3dVa3BSVldSUFZERk9TbFV4T1ZSV1JrcENWa1ZzUjFOVlZsTkRaM0JGVXpGU1RGZ3hUbFZWYTBaVldERk9VVkpWVGtwVVZWWlBXREZPVlZWclJsVlRWVnBLVWxaSlMwTnJVa3hXUlhSbVZURlNVMUZXVW1aVlJrcFFVVEJXUlZaV1NrWllNVTVWVld0R1ZWTlZXa3BTVmtsTFEydFNURlpGZEdaVk1WSlRVVlpTWmxSVlZrVlRWVTVDVmtWc1VGUnNPVlJXUmtwQ1ZrVnNSMU5WVmxORGExSk1Wa1YwWmxVeFVsTlJWbEptVWtWV1IxZ3diRTlZTUd4UFUxWlNTbEZWZUdaVlJUbFJWbFY0UWxaRmJGQlViRUpvWkVkc2JHSnVVWFZhTWxaMVdrZFdlVWxFTUdkS01qRm9Za2RWYmlKOVhYMHNJbTFsWVhOMWNtVWlPbnNpY21WemIzVnlZMlZVZVhCbElqb2lUV1ZoYzNWeVpTSXNJblZ5YkNJNkluVnlianAxZFdsa09qaG1NMlV6WVRZeExXRXdPVGN0TkRoa05DMWlOMkZqTFRobE5ESTNZbVU0WVdNMFpDSXNJbk4wWVhSMWN5STZJbUZqZEdsMlpTSXNJbk4xWW1wbFkzUkRiMlJsWVdKc1pVTnZibU5sY0hRaU9uc2lZMjlrYVc1bklqcGJleUp6ZVhOMFpXMGlPaUpvZEhSd09pOHZhR3czTG05eVp5OW1hR2x5TDNKbGMyOTFjbU5sTFhSNWNHVnpJaXdpWTI5a1pTSTZJbEJoZEdsbGJuUWlmVjE5TENKc2FXSnlZWEo1SWpvaWRYSnVPblYxYVdRNll6YzBaR1l5TURRdE5XSTJZaTAwWVdWa0xXSXlNamd0TjJZMU16TXdPRGMyTkRVNElpd2ljMk52Y21sdVp5STZleUpqYjJScGJtY2lPbHQ3SW5ONWMzUmxiU0k2SW1oMGRIQTZMeTkwWlhKdGFXNXZiRzluZVM1b2JEY3ViM0puTDBOdlpHVlRlWE4wWlcwdmJXVmhjM1Z5WlMxelkyOXlhVzVuSWl3aVkyOWtaU0k2SW1OdmFHOXlkQ0o5WFgwc0ltZHliM1Z3SWpwYmV5SmpiMlJsSWpwN0luUmxlSFFpT2lKd1lYUnBaVzUwY3lKOUxDSndiM0IxYkdGMGFXOXVJanBiZXlKamIyUmxJanA3SW1OdlpHbHVaeUk2VzNzaWMzbHpkR1Z0SWpvaWFIUjBjRG92TDNSbGNtMXBibTlzYjJkNUxtaHNOeTV2Y21jdlEyOWtaVk41YzNSbGJTOXRaV0Z6ZFhKbExYQnZjSFZzWVhScGIyNGlMQ0pqYjJSbElqb2lhVzVwZEdsaGJDMXdiM0IxYkdGMGFXOXVJbjFkZlN3aVkzSnBkR1Z5YVdFaU9uc2liR0Z1WjNWaFoyVWlPaUowWlhoMEwyTnhiQzFwWkdWdWRHbG1hV1Z5SWl3aVpYaHdjbVZ6YzJsdmJpSTZJa2x1U1c1cGRHbGhiRkJ2Y0hWc1lYUnBiMjRpZlgxZExDSnpkSEpoZEdsbWFXVnlJanBiZXlKamIyUmxJanA3SW5SbGVIUWlPaUpIWlc1a1pYSWlmU3dpWTNKcGRHVnlhV0VpT25zaWJHRnVaM1ZoWjJVaU9pSjBaWGgwTDJOeGJDSXNJbVY0Y0hKbGMzTnBiMjRpT2lKSFpXNWtaWElpZlgwc2V5SmpiMlJsSWpwN0luUmxlSFFpT2lJM05URTROaTAzSW4wc0ltTnlhWFJsY21saElqcDdJbXhoYm1kMVlXZGxJam9pZEdWNGRDOWpjV3dpTENKbGVIQnlaWE56YVc5dUlqb2lSR1ZqWldGelpXUWlmWDBzZXlKamIyUmxJanA3SW5SbGVIUWlPaUpCWjJVaWZTd2lZM0pwZEdWeWFXRWlPbnNpYkdGdVozVmhaMlVpT2lKMFpYaDBMMk54YkNJc0ltVjRjSEpsYzNOcGIyNGlPaUpCWjJWRGJHRnpjeUo5ZlYxOUxIc2lZMjlrWlNJNmV5SjBaWGgwSWpvaVpHbGhaMjV2YzJsekluMHNJbVY0ZEdWdWMybHZiaUk2VzNzaWRYSnNJam9pYUhSMGNEb3ZMMmhzTnk1dmNtY3ZabWhwY2k5MWN5OWpjV1p0WldGemRYSmxjeTlUZEhKMVkzUjFjbVZFWldacGJtbDBhVzl1TDJOeFptMHRjRzl3ZFd4aGRHbHZia0poYzJseklpd2lkbUZzZFdWRGIyUmxJam9pUTI5dVpHbDBhVzl1SW4xZExDSndiM0IxYkdGMGFXOXVJanBiZXlKamIyUmxJanA3SW1OdlpHbHVaeUk2VzNzaWMzbHpkR1Z0SWpvaWFIUjBjRG92TDNSbGNtMXBibTlzYjJkNUxtaHNOeTV2Y21jdlEyOWtaVk41YzNSbGJTOXRaV0Z6ZFhKbExYQnZjSFZzWVhScGIyNGlMQ0pqYjJSbElqb2lhVzVwZEdsaGJDMXdiM0IxYkdGMGFXOXVJbjFkZlN3aVkzSnBkR1Z5YVdFaU9uc2liR0Z1WjNWaFoyVWlPaUowWlhoMEwyTnhiQzFwWkdWdWRHbG1hV1Z5SWl3aVpYaHdjbVZ6YzJsdmJpSTZJa1JwWVdkdWIzTnBjeUo5ZlYwc0luTjBjbUYwYVdacFpYSWlPbHQ3SW1OdlpHVWlPbnNpZEdWNGRDSTZJbVJwWVdkdWIzTnBjeUo5TENKamNtbDBaWEpwWVNJNmV5SnNZVzVuZFdGblpTSTZJblJsZUhRdlkzRnNMV2xrWlc1MGFXWnBaWElpTENKbGVIQnlaWE56YVc5dUlqb2lSR2xoWjI1dmMybHpRMjlrWlNKOWZWMTlMSHNpWTI5a1pTSTZleUowWlhoMElqb2ljM0JsWTJsdFpXNGlmU3dpWlhoMFpXNXphVzl1SWpwYmV5SjFjbXdpT2lKb2RIUndPaTh2YUd3M0xtOXlaeTltYUdseUwzVnpMMk54Wm0xbFlYTjFjbVZ6TDFOMGNuVmpkSFZ5WlVSbFptbHVhWFJwYjI0dlkzRm1iUzF3YjNCMWJHRjBhVzl1UW1GemFYTWlMQ0oyWVd4MVpVTnZaR1VpT2lKVGNHVmphVzFsYmlKOVhTd2ljRzl3ZFd4aGRHbHZiaUk2VzNzaVkyOWtaU0k2ZXlKamIyUnBibWNpT2x0N0luTjVjM1JsYlNJNkltaDBkSEE2THk5MFpYSnRhVzV2Ykc5bmVTNW9iRGN1YjNKbkwwTnZaR1ZUZVhOMFpXMHZiV1ZoYzNWeVpTMXdiM0IxYkdGMGFXOXVJaXdpWTI5a1pTSTZJbWx1YVhScFlXd3RjRzl3ZFd4aGRHbHZiaUo5WFgwc0ltTnlhWFJsY21saElqcDdJbXhoYm1kMVlXZGxJam9pZEdWNGRDOWpjV3d0YVdSbGJuUnBabWxsY2lJc0ltVjRjSEpsYzNOcGIyNGlPaUpUY0dWamFXMWxiaUo5ZlYwc0luTjBjbUYwYVdacFpYSWlPbHQ3SW1OdlpHVWlPbnNpZEdWNGRDSTZJbk5oYlhCc1pWOXJhVzVrSW4wc0ltTnlhWFJsY21saElqcDdJbXhoYm1kMVlXZGxJam9pZEdWNGRDOWpjV3dpTENKbGVIQnlaWE56YVc5dUlqb2lVMkZ0Y0d4bFZIbHdaU0o5ZlYxOUxIc2lZMjlrWlNJNmV5SjBaWGgwSWpvaWNISnZZMlZrZFhKbGN5SjlMQ0psZUhSbGJuTnBiMjRpT2x0N0luVnliQ0k2SW1oMGRIQTZMeTlvYkRjdWIzSm5MMlpvYVhJdmRYTXZZM0ZtYldWaGMzVnlaWE12VTNSeWRXTjBkWEpsUkdWbWFXNXBkR2x2Ymk5amNXWnRMWEJ2Y0hWc1lYUnBiMjVDWVhOcGN5SXNJblpoYkhWbFEyOWtaU0k2SWxCeWIyTmxaSFZ5WlNKOVhTd2ljRzl3ZFd4aGRHbHZiaUk2VzNzaVkyOWtaU0k2ZXlKamIyUnBibWNpT2x0N0luTjVjM1JsYlNJNkltaDBkSEE2THk5MFpYSnRhVzV2Ykc5bmVTNW9iRGN1YjNKbkwwTnZaR1ZUZVhOMFpXMHZiV1ZoYzNWeVpTMXdiM0IxYkdGMGFXOXVJaXdpWTI5a1pTSTZJbWx1YVhScFlXd3RjRzl3ZFd4aGRHbHZiaUo5WFgwc0ltTnlhWFJsY21saElqcDdJbXhoYm1kMVlXZGxJam9pZEdWNGRDOWpjV3d0YVdSbGJuUnBabWxsY2lJc0ltVjRjSEpsYzNOcGIyNGlPaUpRY205alpXUjFjbVVpZlgxZExDSnpkSEpoZEdsbWFXVnlJanBiZXlKamIyUmxJanA3SW5SbGVIUWlPaUpRY205alpXUjFjbVZVZVhCbEluMHNJbU55YVhSbGNtbGhJanA3SW14aGJtZDFZV2RsSWpvaWRHVjRkQzlqY1d3aUxDSmxlSEJ5WlhOemFXOXVJam9pVUhKdlkyVmtkWEpsVkhsd1pTSjlmVjE5TEhzaVkyOWtaU0k2ZXlKMFpYaDBJam9pYldWa2FXTmhkR2x2YmxOMFlYUmxiV1Z1ZEhNaWZTd2laWGgwWlc1emFXOXVJanBiZXlKMWNtd2lPaUpvZEhSd09pOHZhR3czTG05eVp5OW1hR2x5TDNWekwyTnhabTFsWVhOMWNtVnpMMU4wY25WamRIVnlaVVJsWm1sdWFYUnBiMjR2WTNGbWJTMXdiM0IxYkdGMGFXOXVRbUZ6YVhNaUxDSjJZV3gxWlVOdlpHVWlPaUpOWldScFkyRjBhVzl1VTNSaGRHVnRaVzUwSW4xZExDSndiM0IxYkdGMGFXOXVJanBiZXlKamIyUmxJanA3SW1OdlpHbHVaeUk2VzNzaWMzbHpkR1Z0SWpvaWFIUjBjRG92TDNSbGNtMXBibTlzYjJkNUxtaHNOeTV2Y21jdlEyOWtaVk41YzNSbGJTOXRaV0Z6ZFhKbExYQnZjSFZzWVhScGIyNGlMQ0pqYjJSbElqb2lhVzVwZEdsaGJDMXdiM0IxYkdGMGFXOXVJbjFkZlN3aVkzSnBkR1Z5YVdFaU9uc2liR0Z1WjNWaFoyVWlPaUowWlhoMEwyTnhiQzFwWkdWdWRHbG1hV1Z5SWl3aVpYaHdjbVZ6YzJsdmJpSTZJazFsWkdsallYUnBiMjVUZEdGMFpXMWxiblFpZlgxZExDSnpkSEpoZEdsbWFXVnlJanBiZXlKamIyUmxJanA3SW5SbGVIUWlPaUpOWldScFkyRjBhVzl1Vkhsd1pTSjlMQ0pqY21sMFpYSnBZU0k2ZXlKc1lXNW5kV0ZuWlNJNkluUmxlSFF2WTNGc0lpd2laWGh3Y21WemMybHZiaUk2SWxCeWIyTmxaSFZ5WlZSNWNHVWlmWDFkZlYxOWZRPT0iLA0KICAicXVlcnktY29udGFjdC1pZCIgOiAicmVzZWFyY2hlckB0ZXN0LmRrZnouZGUiLA0KICAicXVlcnktZm9ybWF0IiA6ICJDUUxfREFUQSIsDQogICJ0ZW1wbGF0ZS1pZCIgOiAiY2NwIg0KfQ==","failure_strategy":{"retry":{"backoff_millisecs":1000,"max_tries":5}},"from":"app1.proxy1.broker","id":"22e1ea3a-07f3-4592-a888-82f2226a44a2","metadata":{"project":"exporter","task_type":"EXECUTE"},"to":["app1.proxy1.broker"],"ttl":"10s","status":null,"task":null}' -H "Authorization: ApiKey app1.proxy1.broker App1Secret" http://localhost:8081/v1/tasks ``` -Creating a sample [Exporter](https://github.com/samply/exporter) "status" task using CURL: +Creating a sample [Exporter](https://github.com/samply/exporter) "status" task using curl: ```bash curl -v -X POST -H "Content-Type: application/json" --data '{"body":"ew0KICAicXVlcnktZXhlY3V0aW9uLWlkIiA6ICIxOSINCn0=","failure_strategy":{"retry":{"backoff_millisecs":1000,"max_tries":5}},"from":"app1.proxy1.broker","id":"22e1ea3a-07f3-4592-a888-82f2226a44a2","metadata":{"project":"exporter","task_type":"STATUS"},"to":["app1.proxy1.broker"],"ttl":"10s","status":null,"task":null}' -H "Authorization: ApiKey app1.proxy1.broker App1Secret" http://localhost:8081/v1/tasks diff --git a/resources/test/result.cql b/resources/test/result.cql deleted file mode 100644 index e69de29..0000000 diff --git a/src/blaze.rs b/src/blaze.rs index 32325a4..61d0ce4 100644 --- a/src/blaze.rs +++ b/src/blaze.rs @@ -128,7 +128,7 @@ pub async fn run_cql_query(library: &Value, measure: &Value) -> Result Result { +pub fn parse_blaze_query_payload_ast(ast_query: &str) -> Result { let decoded = util::base64_decode(ast_query)?; Ok(serde_json::from_slice(&decoded)?) } \ No newline at end of file diff --git a/src/cql.rs b/src/cql.rs index 3ef3972..eb93a2f 100644 --- a/src/cql.rs +++ b/src/cql.rs @@ -9,8 +9,8 @@ use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; use chrono::offset::Utc; use chrono::DateTime; use indexmap::set::IndexSet; -use uuid::Uuid; use tracing::info; +use uuid::Uuid; pub fn generate_body(ast: ast::Ast) -> Result { Ok(BODY @@ -30,11 +30,11 @@ pub fn generate_body(ast: ast::Ast) -> Result { } fn generate_cql(ast: ast::Ast) -> Result { - let mut retrieval_criteria: String = "".to_string(); // main selection criteria (Patient) + let mut retrieval_criteria: String = String::new(); // main selection criteria (Patient) - let mut filter_criteria: String = "".to_string(); // criteria for filtering specimens + let mut filter_criteria: String = String::new(); // criteria for filtering specimens - let mut lists: String = "".to_string(); // needed code lists, defined + let mut lists: String = String::new(); // needed code lists, defined let mut cql = CQL_TEMPLATE.clone().to_string(); @@ -96,7 +96,7 @@ pub fn process( code_systems: &mut IndexSet<&str>, ) -> Result<(), FocusError> { let mut retrieval_cond: String = "(".to_string(); - let mut filter_cond: String = "".to_string(); + let mut filter_cond: String = String::new(); match child { ast::Child::Condition(condition) => { @@ -104,154 +104,101 @@ pub fn process( let condition_snippet = CQL_SNIPPETS.get(&(condition_key_trans, CriterionRole::Query)); - if let Some(snippet) = condition_snippet { - let mut condition_string = (*snippet).to_string(); - let mut filter_string: String = "".to_string(); + let Some(snippet) = condition_snippet else { + return Err(FocusError::AstUnknownCriterion( + condition_key_trans.to_string(), + )); + }; + let mut condition_string = (*snippet).to_string(); + let mut filter_string: String = String::new(); - let filter_snippet = - CQL_SNIPPETS.get(&(condition_key_trans, CriterionRole::Filter)); + let filter_snippet = CQL_SNIPPETS.get(&(condition_key_trans, CriterionRole::Filter)); - let code_lists_option = CRITERION_CODE_LISTS.get(&(condition_key_trans)); - if let Some(code_lists_vec) = code_lists_option { - for (index, code_list) in code_lists_vec.iter().enumerate() { - code_systems.insert(code_list); - let placeholder = format!("{{{{A{}}}}}", (index + 1).to_string()); //to keep compatibility with snippets in typescript - condition_string = - condition_string.replace(placeholder.as_str(), code_list); - } + let code_lists_option = CRITERION_CODE_LISTS.get(&(condition_key_trans)); + if let Some(code_lists_vec) = code_lists_option { + for (index, code_list) in code_lists_vec.iter().enumerate() { + code_systems.insert(code_list); + let placeholder = format!("{{{{A{}}}}}", (index + 1).to_string()); //to keep compatibility with snippets in typescript + condition_string = condition_string.replace(placeholder.as_str(), code_list); } + } - if condition_string.contains("{{K}}") { - //observation loinc code, those only apply to query criteria, we don't filter specimens by observations - let observation_code_option = OBSERVATION_LOINC_CODE.get(&condition_key_trans); + if condition_string.contains("{{K}}") { + //observation loinc code, those only apply to query criteria, we don't filter specimens by observations + let observation_code_option = OBSERVATION_LOINC_CODE.get(&condition_key_trans); - if let Some(observation_code) = observation_code_option { - condition_string = condition_string.replace("{{K}}", observation_code); - } else { - return Err(FocusError::AstUnknownOption( - condition_key_trans.to_string(), - )); - } + if let Some(observation_code) = observation_code_option { + condition_string = condition_string.replace("{{K}}", observation_code); + } else { + return Err(FocusError::AstUnknownOption( + condition_key_trans.to_string(), + )); } + } - if let Some(filtret) = filter_snippet { - filter_string = (*filtret).to_string(); - } + if let Some(filtret) = filter_snippet { + filter_string = (*filtret).to_string(); + } - match condition.type_ { - ast::ConditionType::Between => { - // both min and max values stated - match condition.value { - ast::ConditionValue::DateRange(date_range) => { - let datetime_str_min = date_range.min.as_str(); - let datetime_result_min: Result, _> = - datetime_str_min.parse().map_err(|_| { - FocusError::AstInvalidDateFormat(date_range.min) - }); - - let datetime_min = datetime_result_min.unwrap(); // we returned if Err - let date_str_min = format!("@{}", datetime_min.format("%Y-%m-%d")); - - condition_string = - condition_string.replace("{{D1}}", date_str_min.as_str()); - filter_string = - filter_string.replace("{{D1}}", date_str_min.as_str()); - // no condition needed, "" stays "" - - let datetime_str_max = date_range.max.as_str(); - let datetime_result_max: Result, _> = - datetime_str_max.parse().map_err(|_| { - FocusError::AstInvalidDateFormat(date_range.max) - }); - let datetime_max = datetime_result_max.unwrap(); // we returned if Err - let date_str_max = format!("@{}", datetime_max.format("%Y-%m-%d")); - - condition_string = - condition_string.replace("{{D2}}", date_str_max.as_str()); - filter_string = - filter_string.replace("{{D2}}", date_str_max.as_str()); - // no condition needed, "" stays "" - } - ast::ConditionValue::NumRange(num_range) => { - condition_string = condition_string - .replace("{{D1}}", num_range.min.to_string().as_str()); - condition_string = condition_string - .replace("{{D2}}", num_range.max.to_string().as_str()); - filter_string = filter_string - .replace("{{D1}}", num_range.min.to_string().as_str()); // no condition needed, "" stays "" - filter_string = filter_string - .replace("{{D2}}", num_range.max.to_string().as_str()); - // no condition needed, "" stays "" - } - other => { - return Err(FocusError::AstOperatorValueMismatch(format!("Operator BETWEEN can only be used for numerical and date values, not for {:?}", other))); - } + match condition.type_ { + ast::ConditionType::Between => { + // both min and max values stated + match condition.value { + ast::ConditionValue::DateRange(date_range) => { + let datetime_str_min = date_range.min.as_str(); + let datetime_min: DateTime = datetime_str_min + .parse() + .map_err(|_| FocusError::AstInvalidDateFormat(date_range.min))?; + let date_str_min = format!("@{}", datetime_min.format("%Y-%m-%d")); + + condition_string = + condition_string.replace("{{D1}}", date_str_min.as_str()); + filter_string = filter_string.replace("{{D1}}", date_str_min.as_str()); + // no condition needed, "" stays "" + + let datetime_max: DateTime = date_range.max + .as_str() + .parse() + .map_err(|_| FocusError::AstInvalidDateFormat(date_range.max))?; + let date_str_max = format!("@{}", datetime_max.format("%Y-%m-%d")); + + condition_string = + condition_string.replace("{{D2}}", date_str_max.as_str()); + filter_string = filter_string.replace("{{D2}}", date_str_max.as_str()); + // no condition needed, "" stays "" } - } // deal with no lower or no upper value - ast::ConditionType::In => { - // although in works in CQL, at least in some places, most of it is converted to multiple criteria with OR - let operator_str = " or "; - - match condition.value { - ast::ConditionValue::StringArray(string_array) => { - let mut string_array_with_workarounds = string_array.clone(); - for value in string_array { - if let Some(additional_values) = - SAMPLE_TYPE_WORKAROUNDS.get(value.as_str()) - { - for additional_value in additional_values { - string_array_with_workarounds - .push((*additional_value).into()); - } - } - } - let mut condition_humongous_string = "(".to_string(); - let mut filter_humongous_string = "(".to_string(); - - for (index, string) in - string_array_with_workarounds.iter().enumerate() + ast::ConditionValue::NumRange(num_range) => { + condition_string = condition_string + .replace("{{D1}}", num_range.min.to_string().as_str()); + condition_string = condition_string + .replace("{{D2}}", num_range.max.to_string().as_str()); + filter_string = + filter_string.replace("{{D1}}", num_range.min.to_string().as_str()); // no condition needed, "" stays "" + filter_string = + filter_string.replace("{{D2}}", num_range.max.to_string().as_str()); + // no condition needed, "" stays "" + } + other => { + return Err(FocusError::AstOperatorValueMismatch(format!("Operator BETWEEN can only be used for numerical and date values, not for {:?}", other))); + } + } + } // deal with no lower or no upper value + ast::ConditionType::In => { + // although in works in CQL, at least in some places, most of it is converted to multiple criteria with OR + let operator_str = " or "; + + match condition.value { + ast::ConditionValue::StringArray(string_array) => { + let mut string_array_with_workarounds = string_array.clone(); + for value in string_array { + if let Some(additional_values) = + SAMPLE_TYPE_WORKAROUNDS.get(value.as_str()) { - condition_humongous_string = condition_humongous_string - + "(" - + condition_string.as_str() - + ")"; - condition_humongous_string = condition_humongous_string - .replace("{{C}}", string.as_str()); - - filter_humongous_string = filter_humongous_string - + "(" - + filter_string.as_str() - + ")"; - filter_humongous_string = - filter_humongous_string.replace("{{C}}", string.as_str()); - - // Only concatenate operator if it's not the last element - if index < string_array_with_workarounds.len() - 1 { - condition_humongous_string += operator_str; - filter_humongous_string += operator_str; + for additional_value in additional_values { + string_array_with_workarounds + .push((*additional_value).into()); } } - condition_string = condition_humongous_string + ")"; - - if !filter_string.is_empty() { - filter_string = filter_humongous_string + ")"; - } - } - other => { - return Err(FocusError::AstOperatorValueMismatch(format!("Operator IN can only be used for string arrays, not for {:?}", other))); - } - } - } // this becomes or of all - ast::ConditionType::Equals => match condition.value { - ast::ConditionValue::String(string) => { - let operator_str = " or "; - let mut string_array_with_workarounds = vec![string.clone()]; - if let Some(additional_values) = - SAMPLE_TYPE_WORKAROUNDS.get(string.as_str()) - { - for additional_value in additional_values { - string_array_with_workarounds.push((*additional_value).into()); - } } let mut condition_humongous_string = "(".to_string(); let mut filter_humongous_string = "(".to_string(); @@ -283,26 +230,70 @@ pub fn process( } } other => { - return Err(FocusError::AstOperatorValueMismatch(format!("Operator EQUALS can only be used for string arrays, not for {:?}", other))); + return Err(FocusError::AstOperatorValueMismatch(format!( + "Operator IN can only be used for string arrays, not for {:?}", + other + ))); } - }, - other => { // won't get it from Lens yet - info!("Got this condition type which Lens is not programmed to send, ignoring: {:?}", other); } - }; - - retrieval_cond += condition_string.as_str(); + } // this becomes or of all + ast::ConditionType::Equals => match condition.value { + ast::ConditionValue::String(string) => { + let operator_str = " or "; + let mut string_array_with_workarounds = vec![string.clone()]; + if let Some(additional_values) = + SAMPLE_TYPE_WORKAROUNDS.get(string.as_str()) + { + for additional_value in additional_values { + string_array_with_workarounds.push((*additional_value).into()); + } + } + let mut condition_humongous_string = "(".to_string(); + let mut filter_humongous_string = "(".to_string(); + + for (index, string) in string_array_with_workarounds.iter().enumerate() { + condition_humongous_string = + condition_humongous_string + "(" + condition_string.as_str() + ")"; + condition_humongous_string = + condition_humongous_string.replace("{{C}}", string.as_str()); + + filter_humongous_string = + filter_humongous_string + "(" + filter_string.as_str() + ")"; + filter_humongous_string = + filter_humongous_string.replace("{{C}}", string.as_str()); + + // Only concatenate operator if it's not the last element + if index < string_array_with_workarounds.len() - 1 { + condition_humongous_string += operator_str; + filter_humongous_string += operator_str; + } + } + condition_string = condition_humongous_string + ")"; - if !filter_cond.is_empty() && !filter_string.is_empty() { - filter_cond += " and "; + if !filter_string.is_empty() { + filter_string = filter_humongous_string + ")"; + } + } + other => { + return Err(FocusError::AstOperatorValueMismatch(format!( + "Operator EQUALS can only be used for string arrays, not for {:?}", + other + ))); + } + }, + other => { + // won't get it from Lens yet + info!("Got this condition type which Lens is not programmed to send, ignoring: {:?}", other); } + }; - filter_cond += filter_string.as_str(); // no condition needed, "" can be added with no change - } else { - return Err(FocusError::AstUnknownCriterion( - condition_key_trans.to_string(), - )); + retrieval_cond += condition_string.as_str(); + + if !filter_cond.is_empty() && !filter_string.is_empty() { + filter_cond += " and "; } + + filter_cond += filter_string.as_str(); // no condition needed, "" can be added with no change } ast::Child::Operation(operation) => { diff --git a/src/errors.rs b/src/errors.rs index c67c805..49acc63 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -50,7 +50,7 @@ pub enum FocusError { AstUnknownCriterion(String), #[error("Unknown option in AST: {0}")] AstUnknownOption(String), - #[error("Mismatch between operator and value type")] + #[error("Mismatch between operator and value type: {0}")] AstOperatorValueMismatch(String), #[error("Invalid date format: {0}")] AstInvalidDateFormat(String), diff --git a/src/util.rs b/src/util.rs index 7f6c359..843b789 100644 --- a/src/util.rs +++ b/src/util.rs @@ -284,7 +284,7 @@ fn obfuscate_counts_recursive( obfuscate_below_10_mode: ObfuscateBelow10Mode, rounding_step: usize, ) -> Result<(), FocusError> { - let mut rng = thread_rng();// TODO evict + let mut rng = thread_rng(); match val { Value::Object(map) => { if let Some(count_val) = map.get_mut("count") {