Extending validation

Cargo.toml

@@ -24,7 +24,7 @@ heapless = "0.7"
 inline-c = "0.1"
 postcard = "0.7.0"
 serde = "1.0.126"
-criterion = "0.3"
+criterion = "0.3.6"
 
 [build-dependencies]
 cbindgen = { version = "0.18.0", optional = true }

convert_refdata/data/extract.m

@@ -0,0 +1,24 @@
+% Matlab script to extract checking values from reference dataset
+
+
+load gsw_data_v3_0.mat
+
+fields = fieldnames(gsw_cv)
+
+% varnames = {'f', 'grav', 'distance', 'lat_chck_cast', 'p_chck_cast', 'SA_chck_cast', 'CT_chck_cast', 't_chck_cast', 'SP_from_C', 'C_from_SP', 'SP_from_R', 'R_from_SP', 'SP_from_SK', 'SA_from_SP', 'Sstar_from_SP', 'CT_from_t', 'deltaSA_from_SP', 'SR_from_SP', 'SP_from_SA', 'pt_from_CT', 't_from_CT', 't90_from_t48', 't90_from_t68', 'z_from_p' 'p_from_z' 'z_from_depth', 'specvol', 'alpha', 'beta', 'specvol_anom', 'specvol_anom_standard', 'rho', 'sigma0', 'sigma1', 'sigma2', 'sigma3', 'sigma4', 'enthalpy', 'enthalpy_diff', 'dynamic_enthalpy', 'sound_speed', 'internal_energy', 'SA_from_rho', 'CT_from_rho', 'CT_maxdensity', 'specvol_t_exact'};
+
+% for v=varnames
+%     % specvol = gsw_cv.specvol
+%     eval([char(v), ' = gsw_cv.', char(v), ';'])
+% end
+
+
+for i=1:size(fields)
+    v = char(fields(i))
+    eval([char(v), ' = gsw_cv.', char(v), ';']);
+end
+
+command = 'save(''gsw_data_v3_0.chck_cast.mat''';
+% command = [command, ', ''', strjoin(varnames, ''', '''), ''', ''-v6'');']
+command = [command, ', ''version_number'', ''version_date'', ''', strjoin(fields, ''', '''), ''', ''-v6'');']
+eval(command)

convert_refdata/data/mods/conversions

@@ -0,0 +1,19 @@
+SA_chck_cast
+t_chck_cast
+p_chck_cast
+SP_chck_cast
+CT_chck_cast
+long_chck_cast
+lat_chck_cast
+deltaSA_from_SP
+SR_from_SP
+SP_from_SR
+CT_from_PT
+t90_from_t48
+t90_from_t68
+z_from_p
+p_from_z
+SP_from_SA
+t_from_CT
+entropy_from_CT
+CT_from_entropy

convert_refdata/data/mods/earth

@@ -0,0 +1,4 @@
+lat_chck_cast
+p_chck_cast
+f
+grav

convert_refdata/data/mods/volume

@@ -0,0 +1,47 @@
+p_chck_cast
+p_chck_cast_deep
+p_chck_cast_shallow
+SA_chck_cast
+CT_chck_cast
+rho_chck_cast
+specvol
+rho
+alpha
+beta
+alpha_on_beta
+rho_alpha_beta
+specvol_alpha_beta
+specvol_first_derivatives
+specvol_second_derivatives
+specvol_first_derivatives_wrt_enthalpy
+specvol_second_derivatives_wrt_enthalpy
+specvol_anom
+specvol_anom_standard
+rho_first_derivatives
+rho_second_derivatives
+rho_first_derivatives_wrt_enthalpy
+rho_second_derivatives_wrt_enthalpy
+sigma0
+sigma1
+sigma2
+sigma3
+sigma4
+cabbeling
+thermobaric
+enthalpy
+enthalpy_diff
+sound_speed
+kappa
+internal_energy
+internal_energy_first_derivatives
+internal_energy_second_derivatives
+CT_from_enthalpy
+dynamic_enthalpy
+SA_from_rho
+CT_from_rho
+CT_maxdensity
+specvol_ice
+specvol_diff
+specvol_from_pot_enthalpy_ice
+specvol_from_pot_enthalpy_ice_poly
+specvol_p_parts

convert_refdata/src/main.rs

@@ -1,72 +1,121 @@
 // use core::ops::Deref;
-use heapless::{FnvIndexMap, String, Vec};
-use postcard::{from_bytes, to_vec};
+use heapless::FnvIndexMap;
+use postcard::to_vec;
 use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
 use std::fs::File;
-use std::io::{BufWriter, Write};
+use std::io::{BufRead, BufWriter, Write};
 use std::ops::Deref;
+use std::path::PathBuf;
 
 // Structure of the validation dataset
 #[derive(Serialize, Deserialize, Debug)]
 struct DataRef {
-    version: String<8>,
-    src: String<32>,
-    src_md5: String<32>,
-    data_x: FnvIndexMap<String<24>, Vec<f64, 3>, 4>,
-    data2d: FnvIndexMap<String<24>, Vec<Vec<f64, 45>, 3>, 64>,
+    // created_at
+    version: heapless::String<8>,
+    src: heapless::String<32>,
+    src_md5: heapless::String<32>,
+    scalar: FnvIndexMap<heapless::String<64>, f64, 2>,
+    data_x: FnvIndexMap<heapless::String<64>, heapless::Vec<f64, 3>, 32>,
+    data2d: FnvIndexMap<heapless::String<64>, heapless::Vec<heapless::Vec<f64, 45>, 3>, 32>,
 }
 
 fn main() {
     let file = std::fs::File::open("data/gsw_data_v3_0.chck_cast.mat").unwrap();
     let mat_file = matfile::MatFile::parse(file).unwrap();
 
-    let mut data_x = FnvIndexMap::<heapless::String<24>, _, 4>::new();
-    let mut data2d = FnvIndexMap::<heapless::String<24>, _, 64>::new();
+    let paths: Vec<PathBuf> = std::fs::read_dir("data/mods/")
+        .unwrap()
+        .filter_map(|f| f.ok())
+        .map(|f| f.path())
+        .filter(|f| f.is_file())
+        .collect();
 
-    for variable in mat_file.arrays() {
-        // dbg!(variable.name());
-        // dbg!(variable.size());
-        if let Some(array) = mat_file.find_by_name(variable.name()) {
-            if let matfile::NumericData::Double { real, imag: _ } = array.data() {
-                let varname: String<24> = String::from(variable.name());
-                match array.size()[..] {
-                    [1, 3] => {
-                        // dbg!(array.name());
-                        let values: Vec<f64, 3> = Vec::from_slice(&real).unwrap();
-                        data_x.insert(varname, values).unwrap();
-                    }
-                    [45, 3] => {
-                        // dbg!(array.name());
-                        let values: Vec<_, 3> = real
-                            .chunks(45)
-                            .map(|x| Vec::<f64, 45>::from_slice(x).unwrap())
-                            .collect();
+    let mut groups: BTreeMap<String, Vec<String>> = BTreeMap::new();
 
-                        data2d.insert(varname, values).unwrap();
-                    }
-                    _ => {
-                        dbg!("MISSING");
-                        dbg!(varname);
-                        ()
-                    }
+    for p in paths.into_iter() {
+        let f = std::fs::File::open(&p).unwrap();
+        let reader = std::io::BufReader::new(f);
+        let varlist = reader.lines().map(|l| l.unwrap()).collect::<Vec<String>>();
+        groups.insert(p.file_name().unwrap().to_str().unwrap().into(), varlist);
+    }
+    dbg!(&groups);
+
+    for (group, varnames) in groups {
+        let mut scalar = FnvIndexMap::<heapless::String<64>, _, 2>::new();
+        let mut data_x = FnvIndexMap::<heapless::String<64>, _, 32>::new();
+        let mut data2d = FnvIndexMap::<heapless::String<64>, _, 32>::new();
+
+        let mut scalar_size: isize = 0;
+        let mut datax_size: isize = 0;
+        let mut datax2_size: isize = 0;
+        for variable in mat_file.arrays() {
+            //dbg!(variable.name());
+            //dbg!(variable.size());
+            if let Some(array) = mat_file.find_by_name(variable.name()) {
+                if let matfile::NumericData::Double { real, imag: _ } = array.data() {
+                    if varnames.contains(&variable.name().to_string()) {
+                        let varname: heapless::String<64> = heapless::String::from(variable.name());
+                        match array.size()[..] {
+                            [1, 1] => {
+                                scalar_size += 1;
+                                let value: f64 = real[0];
+                                scalar.insert(varname, value).unwrap();
+                            }
+                            [1, 3] => {
+                                datax_size += 1;
+                                let values: heapless::Vec<f64, 3> =
+                                    heapless::Vec::from_slice(&real).unwrap();
+                                data_x.insert(varname, values).unwrap();
+                            }
+                            [45, 3] => {
+                                datax2_size += 1;
+                                let values: heapless::Vec<_, 3> = real
+                                    .chunks(45)
+                                    .map(|x| heapless::Vec::<f64, 45>::from_slice(x).unwrap())
+                                    .collect();
+
+                                data2d.insert(varname, values).unwrap();
+                            }
+                            _ => {
+                                /*
+                                dbg!("MISSING");
+                                dbg!(varname);
+                                dbg!(array.size());
+                                */
+                                ()
+                            }
+                        }
+                    };
                 };
-            };
+            }
         }
-    }
+        dbg!("DATA SCALAR", &scalar_size);
+        dbg!("DATA 1D", &datax_size);
+        dbg!("DATA 2D", &datax2_size);
 
-    let dataset = DataRef {
-        version: String::from("3.06.12"),
-        src: String::from("gsw_data_v3_0.mat"),
-        src_md5: String::from("f9b65955407a8ed00246de4871ef0505"),
-        data_x,
-        data2d,
-    };
+        // let nd_arr: ndarray::ArrayD<f64> = specvol.try_into().expect("Fail converting into array");
+        /*
+        let gsw_cv = mat_file.find_by_name("gsw_cv");
+        println!("{:#?}", gsw_cv);
+        */
 
-    // dbg!("{:?}", &dataset);
+        let dataset = DataRef {
+            version: heapless::String::from("3.06.12"),
+            src: heapless::String::from("gsw_data_v3_0.mat"),
+            src_md5: heapless::String::from("25b840e6f8bdd72522e28d99eccd9e99"),
+            scalar,
+            data_x,
+            data2d,
+        };
 
-    let stream: Vec<u8, 100_000> = to_vec(&dataset).expect("Failed to vectorize dataset");
+        let stream: heapless::Vec<u8, 350_000> =
+            to_vec(&dataset).expect("Failed to vectorize dataset");
+        // let serialized = serde_json::to_string(&dataset).unwrap();
 
-    let f = File::create("gsw_validation.bin").expect("Unable to create file");
-    let mut f = BufWriter::new(f);
-    f.write_all(stream.deref()).expect("Unable to write data");
+        let outputfilename = format!("data/gsw_{}_validation.bin", group);
+        let f = File::create(dbg!(outputfilename)).expect("Unable to create file");
+        let mut f = BufWriter::new(f);
+        f.write_all(stream.deref()).expect("Unable to write data");
+    }
 }

convert_refdata/tests/check.rs

@@ -0,0 +1,52 @@
+use heapless::{FnvIndexMap, String, Vec};
+use postcard::from_bytes;
+use serde::{Deserialize, Serialize};
+use std::fs::File;
+use std::io::Read;
+
+// Structure of the validation dataset
+#[derive(Serialize, Deserialize, Debug)]
+struct DataRef {
+    version: String<8>,
+    src: String<32>,
+    src_md5: String<32>,
+    scalar: FnvIndexMap<String<64>, f64, 512>,
+    data_x: FnvIndexMap<String<64>, Vec<f64, 3>, 16>,
+    data2d: FnvIndexMap<String<64>, Vec<Vec<f64, 45>, 3>, 512>,
+}
+
+#[test]
+#[cfg(not(windows))]
+fn check_version() {
+    let mut input = File::open("data/gsw_validation.bin").expect("Unable to open file");
+    let mut contents = vec![];
+    input
+        .read_to_end(&mut contents)
+        .expect("Failed to read content");
+    let out: DataRef = from_bytes(&contents).unwrap();
+    assert_eq!(out.version, "3.06.12");
+    assert_eq!(out.src, "gsw_data_v3_0.mat");
+}
+
+#[test]
+#[cfg(not(windows))]
+fn check_1d() {
+    let mut input = File::open("data/gsw_validation.bin").expect("Unable to open file");
+    let mut contents = vec![];
+    input
+        .read_to_end(&mut contents)
+        .expect("Failed to read content");
+    let out: DataRef = from_bytes(&contents).unwrap();
+    for key in out.data_x.keys() {
+        dbg!(key);
+    }
+    for (key, val) in out.data_x.iter() {
+        dbg!("key: {} val: {}", key, val);
+    }
+    let neutral_density = out.data_x.get(&String::from("Neutral_Density")).unwrap();
+    dbg!(neutral_density);
+    let f = out.data_x.get(&heapless::String::from("f"));
+    dbg!(f);
+    let f = out.data_x.get(&String::from("f")).unwrap();
+    dbg!(f);
+}