From d6f8c7e4007d662df6a44d7e4ae274098de57538 Mon Sep 17 00:00:00 2001 From: Joel Natividad <1980690+jqnatividad@users.noreply.github.com> Date: Mon, 16 Dec 2024 10:44:01 -0500 Subject: [PATCH 1/4] feat: `template` add `--globals-json` option --- src/cmd/template.rs | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/src/cmd/template.rs b/src/cmd/template.rs index a727fd5b7..2929a1a1d 100644 --- a/src/cmd/template.rs +++ b/src/cmd/template.rs @@ -74,6 +74,7 @@ template arguments: template options: --template MiniJinja template string to use (alternative to --template-file) -t, --template-file MiniJinja template file to use + -j, --globals-json A JSON file to load in the MiniJinja context --outfilename MiniJinja template string to use to create the filename of the output files to write to . If set to just QSV_ROWNO, the filestem is set to the current rowno of the record, padded with leading @@ -116,6 +117,7 @@ Common options: use std::{ fs, io::{BufWriter, Write}, + path::PathBuf, sync::{ atomic::{AtomicBool, AtomicU16, Ordering}, OnceLock, RwLock, @@ -131,7 +133,7 @@ use rayon::{ prelude::IntoParallelRefIterator, }; use serde::Deserialize; -use simd_json::BorrowedValue; +use simd_json::{json, BorrowedValue}; use crate::{ config::{Config, Delimiter, DEFAULT_WTR_BUFFER_CAPACITY}, @@ -148,6 +150,7 @@ struct Args { arg_outdir: Option, flag_template: Option, flag_template_file: Option, + flag_globals_json: Option, flag_output: Option, flag_outfilename: String, flag_outsubdir_size: u16, @@ -218,6 +221,23 @@ pub fn run(argv: &[&str]) -> CliResult<()> { Ordering::Relaxed, ); + // setup globals JSON context if specified + let mut globals_flag = false; + let globals_ctx = if let Some(globals_json) = args.flag_globals_json { + globals_flag = true; + match std::fs::read(globals_json) { + Ok(mut bytes) => match simd_json::from_slice(&mut bytes) { + Ok(json) => json, + Err(e) => return fail_clierror!("Failed to parse globals JSON file: {e}"), + }, + Err(e) => return fail_clierror!("Failed to read globals JSON file: {e}"), + } + } else { + json!("") + }; + + let globals_ctx_borrowed: simd_json::BorrowedValue = globals_ctx.into(); + // Set up minijinja environment let mut env = Environment::new(); @@ -255,7 +275,7 @@ pub fn run(argv: &[&str]) -> CliResult<()> { } let width = rowcount.to_string().len(); - // read headers + // read headers - the headers are used as MiniJinja variables in the template let headers = if args.flag_no_headers { csv::StringRecord::new() } else { @@ -427,6 +447,13 @@ pub fn run(argv: &[&str]) -> CliResult<()> { let curr_record = record; let mut context = simd_json::borrowed::Object::default(); + // Add globals context to the record context + if globals_flag { + context.insert( + std::borrow::Cow::Borrowed("globals"), + globals_ctx_borrowed.clone(), + ); + } context.reserve(headers_len); let mut row_number = 0_u64; @@ -452,7 +479,7 @@ pub fn run(argv: &[&str]) -> CliResult<()> { } } } else { - // Use header names + // Use header names as template variables for (header, field) in headers.iter().zip(curr_record.iter()) { context.insert( std::borrow::Cow::Borrowed(header), From 4b26e43b3b055406aaad2e18448b9986c9e721bc Mon Sep 17 00:00:00 2001 From: Joel Natividad <1980690+jqnatividad@users.noreply.github.com> Date: Mon, 16 Dec 2024 10:44:28 -0500 Subject: [PATCH 2/4] test: add `template --global-json` tests --- tests/test_template.rs | 75 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/tests/test_template.rs b/tests/test_template.rs index cdc4dbde2..40779865e 100644 --- a/tests/test_template.rs +++ b/tests/test_template.rs @@ -1113,3 +1113,78 @@ fn template_round_banker_filter() { ); assert_eq!(got, expected); } + +#[test] +fn template_globals_json() { + let wrk = Workdir::new("template_globals"); + wrk.create( + "data.csv", + vec![ + svec!["name", "score"], + svec!["Alice", "85"], + svec!["Bob", "92"], + ], + ); + + wrk.create_from_string( + "globals.json", + r#"{ + "passing_score": 90, + "school_name": "Test Academy", + "year": 2023 + }"#, + ); + + let mut cmd = wrk.command("template"); + cmd.arg("--template") + .arg(concat!( + "School: {{globals.school_name}}\n", + "Year: {{globals.year}}\n", + "Student: {{name}}\n", + "Score: {{score}}\n", + "Status: {% if score|int >= globals.passing_score %}PASS{% else %}FAIL{% endif \ + %}\n\n\n" + )) + .arg("--globals-json") + .arg("globals.json") + .arg("data.csv"); + + let got: String = wrk.stdout(&mut cmd); + let expected = concat!( + "School: Test Academy\n", + "Year: 2023\n", + "Student: Alice\n", + "Score: 85\n", + "Status: FAIL\n\n", + "School: Test Academy\n", + "Year: 2023\n", + "Student: Bob\n", + "Score: 92\n", + "Status: PASS" + ); + assert_eq!(got, expected); +} + +#[test] +fn template_globals_json_invalid() { + let wrk = Workdir::new("template_globals_invalid"); + wrk.create("data.csv", vec![svec!["name"], svec!["test"]]); + + wrk.create_from_string( + "invalid.json", + r#"{ + "bad_json": "missing_comma" + "another_field": true + }"#, + ); + + let mut cmd = wrk.command("template"); + cmd.arg("--template") + .arg("{{name}}\n") + .arg("--globals-json") + .arg("invalid.json") + .arg("data.csv"); + + let got: String = wrk.output_stderr(&mut cmd); + assert!(got.contains("Failed to parse globals JSON file")); +} From 3a99f4338470ddf8641c968d24117ece1aa6ad11 Mon Sep 17 00:00:00 2001 From: Joel Natividad <1980690+jqnatividad@users.noreply.github.com> Date: Mon, 16 Dec 2024 10:49:12 -0500 Subject: [PATCH 3/4] docs: expanded explanation of the `--globals-json` option --- src/cmd/template.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cmd/template.rs b/src/cmd/template.rs index 2929a1a1d..5b2461078 100644 --- a/src/cmd/template.rs +++ b/src/cmd/template.rs @@ -74,7 +74,8 @@ template arguments: template options: --template MiniJinja template string to use (alternative to --template-file) -t, --template-file MiniJinja template file to use - -j, --globals-json A JSON file to load in the MiniJinja context + -j, --globals-json A JSON file to load in the MiniJinja context. + The JSON properties become MiniJinja variables in the "globals" namespace. --outfilename MiniJinja template string to use to create the filename of the output files to write to . If set to just QSV_ROWNO, the filestem is set to the current rowno of the record, padded with leading From 895efee85ccabf74f7122c5e1f138bb9d8c99548 Mon Sep 17 00:00:00 2001 From: Joel Natividad <1980690+jqnatividad@users.noreply.github.com> Date: Mon, 16 Dec 2024 10:52:28 -0500 Subject: [PATCH 4/4] doc: more expansive `--globals.json` option description --- src/cmd/template.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/cmd/template.rs b/src/cmd/template.rs index 5b2461078..1e5c5b400 100644 --- a/src/cmd/template.rs +++ b/src/cmd/template.rs @@ -74,8 +74,10 @@ template arguments: template options: --template MiniJinja template string to use (alternative to --template-file) -t, --template-file MiniJinja template file to use - -j, --globals-json A JSON file to load in the MiniJinja context. - The JSON properties become MiniJinja variables in the "globals" namespace. + -j, --globals-json A JSON file containing global variables to make available in templates. + The JSON properties can be accessed in templates using the "globals" + namespace (e.g. {{globals.school_name}}, {{globals.year}}). + This allows sharing common values across all template renders. --outfilename MiniJinja template string to use to create the filename of the output files to write to . If set to just QSV_ROWNO, the filestem is set to the current rowno of the record, padded with leading