Merge pull request #121 from samply/feature/histo-obfuscation

Feature/histo obfuscation

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "focus"
-version = "0.5.0"
+version = "0.5.1"
 edition = "2021"
 license = "Apache-2.0"
 

README.md

@@ -44,6 +44,7 @@ DELTA_SPECIMEN = "20." # Sensitivity parameter for obfuscating the counts in the
 DELTA_DIAGNOSIS = "3." # Sensitivity parameter for obfuscating the counts in the Diagnosis stratifier, has no effect if OBFUSCATE = "no", default value: 3
 DELTA_PROCEDURES = "1.7" # Sensitivity parameter for obfuscating the counts in the Procedures stratifier, has no effect if OBFUSCATE = "no", default value: 1.7
 DELTA_MEDICATION_STATEMENTS = "2.1" # Sensitivity parameter for obfuscating the counts in the Medication Statements stratifier, has no effect if OBFUSCATE = "no", default value: 2.1
+DELTA_HISTO = "20." # Sensitivity parameter for obfuscating the counts in the Histo stratifier, has no effect if OBFUSCATE = "no", default value: 20
 EPSILON = "0.1" # Privacy budget parameter for obfuscating the counts in the stratifiers, has no effect if OBFUSCATE = "no", default value: 0.1
 ROUNDING_STEP = "10" # The granularity of the rounding of the obfuscated values, has no effect if OBFUSCATE = "no", default value: 10
 PROJECTS_NO_OBFUSCATION = "exliquid;dktk_supervisors;exporter;ehds2" # Projects for which the results are not to be obfuscated, separated by ;, default value: "exliquid;dktk_supervisors;exporter;ehds2"

resources/measure_report_dktk.json → resources/test/measure_report_dktk.json

@@ -5341,6 +5341,71 @@
                     ]
                 }
             ]
+        },
+        {
+          "code": {
+            "text": "Histo"
+          },
+          "population": [
+            {
+              "code": {
+                "coding": [
+                  {
+                    "code": "initial-population",
+                    "system": "http://terminology.hl7.org/CodeSystem/measure-population"
+                  }
+                ]
+              },
+              "count": 44000
+            }
+          ],
+          "stratifier": [
+            {
+              "code": [
+                {
+                  "text": "Histlogoies"
+                }
+              ],
+              "stratum": [
+                {
+                  "population": [
+                    {
+                      "code": {
+                        "coding": [
+                          {
+                            "code": "initial-population",
+                            "system": "http://terminology.hl7.org/CodeSystem/measure-population"
+                          }
+                        ]
+                      },
+                      "count": 33694
+                    }
+                  ],
+                  "value": {
+                    "text": "0"
+                  }
+                },
+                {
+                  "population": [
+                    {
+                      "code": {
+                        "coding": [
+                          {
+                            "code": "initial-population",
+                            "system": "http://terminology.hl7.org/CodeSystem/measure-population"
+                          }
+                        ]
+                      },
+                      "count": 10306
+                    }
+                  ],
+                  "value": {
+                    "text": "1"
+                  }
+                }
+              ]
+            }
+          ]
         }
     ],
     "resourceType": "MeasureReport"

src/config.rs

@@ -115,6 +115,10 @@ struct CliArgs {
     #[clap(long, env, value_parser, default_value = "2.1")]
     delta_medication_statements: f64,
 
+    /// Sensitivity parameter for obfuscating the counts in the Histo stratifier
+    #[clap(long, env, value_parser, default_value = "20.")]
+    delta_histo: f64,
+
     /// Privacy budget parameter for obfuscating the counts in the stratifiers
     #[clap(long, env, value_parser, default_value = "0.1")]
     epsilon: f64,
@@ -165,6 +169,7 @@ pub(crate) struct Config {
     pub delta_diagnosis: f64,
     pub delta_procedures: f64,
     pub delta_medication_statements: f64,
+    pub delta_histo: f64,
     pub epsilon: f64,
     pub rounding_step: usize,
     pub unobfuscated: Vec<String>,
@@ -207,6 +212,7 @@ impl Config {
             delta_diagnosis: cli_args.delta_diagnosis,
             delta_procedures: cli_args.delta_procedures,
             delta_medication_statements: cli_args.delta_medication_statements,
+            delta_histo: cli_args.delta_histo,
             epsilon: cli_args.epsilon,
             rounding_step: cli_args.rounding_step,
             unobfuscated: cli_args.projects_no_obfuscation.split(';').map(|s| s.to_string()).collect(),

src/main.rs

@@ -218,6 +218,7 @@ async fn run_cql_query(
                     CONFIG.delta_diagnosis,
                     CONFIG.delta_procedures,
                     CONFIG.delta_medication_statements,
+                    CONFIG.delta_histo,
                     CONFIG.epsilon,
                     CONFIG.rounding_step,
                 )?,

src/util.rs

@@ -116,6 +116,7 @@ pub fn obfuscate_counts_mr(
     delta_diagnosis: f64,
     delta_procedures: f64,
     delta_medication_statements: f64,
+    delta_histo: f64,
     epsilon: f64,
     rounding_step: usize,
 ) -> Result<String, FocusError> {
@@ -239,6 +240,28 @@ pub fn obfuscate_counts_mr(
                     rounding_step,
                 )?;
             }
+            "Histo" => {
+                obfuscate_counts_recursive(
+                    &mut g.population,
+                    delta_histo,
+                    epsilon,
+                    1,
+                    obf_cache,
+                    obfuscate_zero,
+                    obf_10.clone(),
+                    rounding_step,
+                )?;
+                obfuscate_counts_recursive(
+                    &mut g.stratifier,
+                    delta_histo,
+                    epsilon,
+                    2,
+                    obf_cache,
+                    obfuscate_zero,
+                    obf_10.clone(),
+                    rounding_step,
+                )?;
+            }
             _ => {
                 warn!("Focus is not aware of {} type of stratifier, therefore it will not obfuscate the values.", &g.code.text[..])
             }
@@ -326,8 +349,8 @@ mod test {
         include_str!("../resources/test/query_bbmri_placeholders.cql");
     const QUERY_BBMRI: &str = include_str!("../resources/test/query_bbmri.cql");
     const EXAMPLE_MEASURE_REPORT_BBMRI: &str =
-        include_str!("../resources/measure_report_bbmri.json");
-    const EXAMPLE_MEASURE_REPORT_DKTK: &str = include_str!("../resources/measure_report_dktk.json");
+        include_str!("../resources/test/measure_report_bbmri.json");
+    const EXAMPLE_MEASURE_REPORT_DKTK: &str = include_str!("../resources/test/measure_report_dktk.json");
     const EXAMPLE_MEASURE_REPORT_EXLIQUID: &str =
         include_str!("../resources/test/measure_report_exliquid.json");
 
@@ -336,6 +359,7 @@ mod test {
     const DELTA_DIAGNOSIS: f64 = 3.;
     const DELTA_PROCEDURES: f64 = 1.7;
     const DELTA_MEDICATION_STATEMENTS: f64 = 2.1;
+    const DELTA_HISTO: f64 = 20.;
     const EPSILON: f64 = 0.1;
     const ROUNDING_STEP: usize = 10;
 
@@ -411,13 +435,10 @@ mod test {
 
     #[test]
     fn test_replace_cql() {
-
-
         let decoded_library = QUERY_BBMRI_PLACEHOLDERS;
         let expected_result = QUERY_BBMRI;
         assert_eq!(replace_cql(decoded_library), expected_result);
 
-
         let decoded_library = "BBMRI_STRAT_GENDER_STRATIFIER";
         let expected_result = "define Gender:\n if (Patient.gender is null) then 'unknown'\n else if (Patient.gender != 'male' and Patient.gender != 'female' and Patient.gender != 'other' and Patient.gender != 'unknown') then 'other'\n else Patient.gender";
         assert_eq!(replace_cql(decoded_library), expected_result);
@@ -462,12 +483,10 @@ mod test {
         let expected_result = "define InInitialPopulation:\n  exists AnySpecimen\n      \ndefine AnySpecimen:\n  [Specimen] S\n\ndefine retrieveCondition:\n  First(from [Condition] C\n    return ('{\\\"subject_reference\\\": \\\"' + C.subject.reference \n    + '\\\", \\\"diagnosis_code\\\": \\\"' \n    + C.code.coding.where(system = 'http://fhir.de/CodeSystem/bfarm/icd-10-gm').code.first() \n    + '\\\"}'\n  ))\n  \ndefine Diagnosis:\n  if (retrieveCondition is null) then '{\\\"subject_reference\\\": \\\"\\\", \\\"diagnosis_code\\\": \\\"\\\"}' \n  else retrieveCondition\n\ndefine function getSampletype(specimen FHIR.Specimen):\n  if (not exists specimen.type.coding.where(system = 'https://fhir.bbmri.de/CodeSystem/SampleMaterialType').code) then 'null'\n  else specimen.type.coding.where(system = 'https://fhir.bbmri.de/CodeSystem/SampleMaterialType').code.first()\n\ndefine function getRestamount(specimen FHIR.Specimen):\n  if (not exists specimen.collection.quantity.value) then '0' else specimen.collection.quantity.value.toString()\n\ndefine function getParentReference(specimen FHIR.Specimen):  \n  if (not exists specimen.parent.reference) then 'null' else specimen.parent.reference\n\ndefine function getSubjectReference(specimen FHIR.Specimen):  \n  if (not exists specimen.subject.reference) then 'null' else specimen.subject.reference\n\ndefine function SingleStrat(specimen FHIR.Specimen):\n  '{\"specimen_id\": \"' + specimen.id + \n  '\", \"sampletype\": \"' +  getSampletype(specimen) +\n  '\", \"exliquid_tag\": ' + (specimen.identifier.system contains 'http://dktk.dkfz.de/fhir/sid/exliquid-specimen').toString() +\n  ', \"rest_amount\": \"' + getRestamount(specimen) +\n  '\", \"parent_reference\": \"' + getParentReference(specimen) +\n  '\", \"subject_reference\": \"' + getSubjectReference(specimen) +\n  '\"}'";
         assert_eq!(replace_cql(decoded_library), expected_result);
 
-
         let decoded_library = "EXLIQUID_STRAT_DEF_IN_INITIAL_POPULATION";
         let expected_result = "define InInitialPopulation:\n   exists ExliquidSpecimen and\n\n";
         assert_eq!(replace_cql(decoded_library), expected_result);
 
-
         let decoded_library = "MTBA_STRAT_GENETIC_VARIANT";
         let expected_result = "define GeneticVariantCode:\nFirst (from [Observation: Code '69548-6' from loinc] O return O.component.where(code.coding contains Code '48018-6' from loinc).value.coding.code.first())\n";
 
@@ -480,7 +499,6 @@ mod test {
         let decoded_library = "INVALID_KEY";
         let expected_result = "INVALID_KEY";
         assert_eq!(replace_cql(decoded_library), expected_result);
-
     }
 
     #[test]
@@ -498,6 +516,7 @@ mod test {
             DELTA_DIAGNOSIS,
             DELTA_PROCEDURES,
             DELTA_MEDICATION_STATEMENTS,
+            DELTA_HISTO,
             EPSILON,
             ROUNDING_STEP,
         )
@@ -520,6 +539,7 @@ mod test {
             DELTA_DIAGNOSIS,
             DELTA_PROCEDURES,
             DELTA_MEDICATION_STATEMENTS,
+            DELTA_HISTO,
             EPSILON,
             ROUNDING_STEP,
         )
@@ -542,6 +562,7 @@ mod test {
             DELTA_DIAGNOSIS,
             DELTA_PROCEDURES,
             DELTA_MEDICATION_STATEMENTS,
+            DELTA_HISTO,
             EPSILON,
             ROUNDING_STEP,
         )
@@ -564,6 +585,7 @@ mod test {
             DELTA_DIAGNOSIS,
             DELTA_PROCEDURES,
             DELTA_MEDICATION_STATEMENTS,
+            DELTA_HISTO,
             EPSILON,
             ROUNDING_STEP,
         )
@@ -586,6 +608,7 @@ mod test {
             DELTA_DIAGNOSIS,
             DELTA_PROCEDURES,
             DELTA_MEDICATION_STATEMENTS,
+            DELTA_HISTO,
             EPSILON,
             ROUNDING_STEP,
         );