From 8c0912bdc1aefb477048f78970a0343ca53d2325 Mon Sep 17 00:00:00 2001 From: matt rice Date: Wed, 30 Aug 2023 07:12:15 -0700 Subject: [PATCH] Add a cttests_macro crate to generate `#[test] fn` from `.test` files. --- Cargo.toml | 1 + lrpar/cttests/Cargo.toml | 5 + lrpar/cttests/build.rs | 170 +------------------------------ lrpar/cttests/src/cgen_helper.rs | 157 ++++++++++++++++++++++++++++ lrpar/cttests/src/lib.rs | 9 ++ lrpar/cttests_macro/Cargo.toml | 14 +++ lrpar/cttests_macro/src/lib.rs | 42 ++++++++ 7 files changed, 231 insertions(+), 167 deletions(-) create mode 100644 lrpar/cttests/src/cgen_helper.rs create mode 100644 lrpar/cttests_macro/Cargo.toml create mode 100644 lrpar/cttests_macro/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index bbd396f6f..4dddd30a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members=[ "lrlex/examples/calc_manual_lex", "lrpar", "lrpar/cttests", + "lrpar/cttests_macro", "lrpar/examples/calc_actions", "lrpar/examples/calc_ast", "lrpar/examples/calc_parsetree", diff --git a/lrpar/cttests/Cargo.toml b/lrpar/cttests/Cargo.toml index 4fcb34176..74c233977 100644 --- a/lrpar/cttests/Cargo.toml +++ b/lrpar/cttests/Cargo.toml @@ -17,3 +17,8 @@ yaml-rust = "0.4" cfgrammar = { path = "../../cfgrammar" } lrlex = { path = "../../lrlex" } lrpar = { path = "../" } +yaml-rust = "0.4" +glob = "0.3" + +[dev-dependencies] +cttests_macro = { path = "../cttests_macro" } diff --git a/lrpar/cttests/build.rs b/lrpar/cttests/build.rs index 2579203d9..197133e31 100644 --- a/lrpar/cttests/build.rs +++ b/lrpar/cttests/build.rs @@ -1,146 +1,8 @@ -use cfgrammar::yacc::{YaccKind, YaccOriginalActionKind}; use glob::glob; -use lrlex::{CTLexerBuilder, DefaultLexerTypes}; -use lrpar::CTParserBuilder; -use std::{ - env, fs, - path::{Path, PathBuf}, -}; -use yaml_rust::YamlLoader; +#[path = "src/cgen_helper.rs"] +mod cgen_helper; -fn run_test_path>(path: P) -> Result<(), Box> { - let out_dir = env::var("OUT_DIR").unwrap(); - let path = path.as_ref(); - if path.is_file() { - println!("cargo:rerun-if-changed={}", path.display()); - // Parse test file - let s = fs::read_to_string(&path).unwrap(); - let docs = YamlLoader::load_from_str(&s).unwrap(); - let grm = &docs[0]["grammar"].as_str().unwrap(); - let lex = &docs[0]["lexer"].as_str().unwrap(); - let yacckind = match docs[0]["yacckind"].as_str().unwrap() { - "Original(YaccOriginalActionKind::NoAction)" => { - YaccKind::Original(YaccOriginalActionKind::NoAction) - } - "Original(YaccOriginalActionKind::UserAction)" => { - YaccKind::Original(YaccOriginalActionKind::UserAction) - } - "Grmtools" => YaccKind::Grmtools, - "Original(YaccOriginalActionKind::GenericParseTree)" => { - YaccKind::Original(YaccOriginalActionKind::GenericParseTree) - } - s => panic!("YaccKind '{}' not supported", s), - }; - let (negative_yacc_flags, positive_yacc_flags) = &docs[0]["yacc_flags"] - .as_vec() - .map(|flags_vec| { - flags_vec - .iter() - .partition(|flag| flag.as_str().unwrap().starts_with('!')) - }) - .unwrap_or_else(|| (Vec::new(), Vec::new())); - let positive_yacc_flags = positive_yacc_flags - .iter() - .map(|flag| flag.as_str().unwrap()) - .collect::>(); - let negative_yacc_flags = negative_yacc_flags - .iter() - .map(|flag| { - let flag = flag.as_str().unwrap(); - flag.strip_prefix('!').unwrap() - }) - .collect::>(); - let yacc_flags = (&positive_yacc_flags, &negative_yacc_flags); - let (negative_lex_flags, positive_lex_flags) = &docs[0]["lex_flags"] - .as_vec() - .map(|flags_vec| { - flags_vec - .iter() - .partition(|flag| flag.as_str().unwrap().starts_with('!')) - }) - .unwrap_or_else(|| (Vec::new(), Vec::new())); - let negative_lex_flags = negative_lex_flags - .iter() - .map(|flag| { - let flag = flag.as_str().unwrap(); - flag.strip_prefix('!').unwrap() - }) - .collect::>(); - let positive_lex_flags = positive_lex_flags - .iter() - .map(|flag| flag.as_str().unwrap()) - .collect::>(); - let lex_flags = (&positive_lex_flags, &negative_lex_flags); - - // The code below, in essence, replicates lrlex and lrpar's internal / undocumented - // filename conventions. If those change, this code will also have to change. - - // Create grammar files - let base = path.file_stem().unwrap().to_str().unwrap(); - let mut pg = PathBuf::from(&out_dir); - pg.push(format!("{}.y.rs", base)); - fs::write(&pg, grm).unwrap(); - let mut pl = PathBuf::from(&out_dir); - pl.push(format!("{}.l.rs", base)); - fs::write(&pl, lex).unwrap(); - - // Build parser and lexer - let mut outp = PathBuf::from(&out_dir); - outp.push(format!("{}.y.rs", base)); - outp.set_extension("rs"); - let mut cp_build = CTParserBuilder::>::new() - .yacckind(yacckind) - .grammar_path(pg.to_str().unwrap()) - .output_path(&outp); - if let Some(flag) = check_flag(yacc_flags, "error_on_conflicts") { - cp_build = cp_build.error_on_conflicts(flag) - } - if let Some(flag) = check_flag(yacc_flags, "warnings_are_errors") { - cp_build = cp_build.warnings_are_errors(flag) - } - if let Some(flag) = check_flag(yacc_flags, "show_warnings") { - cp_build = cp_build.show_warnings(flag) - }; - let cp = cp_build.build()?; - - let mut outl = PathBuf::from(&out_dir); - outl.push(format!("{}.l.rs", base)); - outl.set_extension("rs"); - let mut cl_build = CTLexerBuilder::new() - .rule_ids_map(cp.token_map()) - .lexer_path(pl.to_str().unwrap()) - .output_path(&outl); - if let Some(flag) = check_flag(lex_flags, "allow_missing_terms_in_lexer") { - cl_build = cl_build.allow_missing_terms_in_lexer(flag) - } - if let Some(flag) = check_flag(lex_flags, "allow_missing_tokens_in_parser") { - cl_build = cl_build.allow_missing_tokens_in_parser(flag) - } - if let Some(flag) = check_flag(lex_flags, "dot_matches_new_line") { - cl_build = cl_build.dot_matches_new_line(flag) - } - if let Some(flag) = check_flag(lex_flags, "case_insensitive") { - cl_build = cl_build.case_insensitive(flag) - } - if let Some(flag) = check_flag(lex_flags, "multi_line") { - cl_build = cl_build.multi_line(flag) - } - if let Some(flag) = check_flag(lex_flags, "swap_greed") { - cl_build = cl_build.swap_greed(flag) - } - if let Some(flag) = check_flag(lex_flags, "ignore_whitespace") { - cl_build = cl_build.ignore_whitespace(flag) - } - if let Some(flag) = check_flag(lex_flags, "unicode") { - cl_build = cl_build.unicode(flag) - } - if let Some(flag) = check_flag(lex_flags, "octal") { - cl_build = cl_build.octal(flag) - } - cl_build.build()?; - } - Ok(()) -} +use cgen_helper::run_test_path; // Compiles the `*.test` files within `src`. Test files are written in Yaml syntax and have 4 // mandatory sections: name (describing what the test does), yacckind (defining the grammar type @@ -151,31 +13,5 @@ fn main() -> Result<(), Box> { for entry in glob("src/*.test")? { run_test_path(entry.unwrap())?; } - for entry in glob("src/ctfails/*.test")? { - let path = entry.unwrap(); - let result = - std::panic::catch_unwind(|| std::panic::AssertUnwindSafe(run_test_path(&path).is_ok())); - if !result.is_err() { - panic!( - "ctfails/{}: succeded unexpectedly with result {:?}", - path.display(), - result - ); - } - } Ok(()) } - -fn check_flag((positive, negative): (&Vec<&str>, &Vec<&str>), flag: &str) -> Option { - assert_eq!( - positive.contains(&flag) | negative.contains(&flag), - positive.contains(&flag) ^ negative.contains(&flag) - ); - if positive.contains(&flag) { - Some(true) - } else if negative.contains(&flag) { - Some(false) - } else { - None - } -} diff --git a/lrpar/cttests/src/cgen_helper.rs b/lrpar/cttests/src/cgen_helper.rs new file mode 100644 index 000000000..1206be9f4 --- /dev/null +++ b/lrpar/cttests/src/cgen_helper.rs @@ -0,0 +1,157 @@ +use cfgrammar::yacc::{YaccKind, YaccOriginalActionKind}; +use lrlex::{CTLexerBuilder, DefaultLexerTypes}; +use lrpar::CTParserBuilder; +use std::{ + env, fs, + path::{Path, PathBuf}, +}; +use yaml_rust::YamlLoader; + +#[allow(dead_code)] +pub(crate) fn run_test_path>(path: P) -> Result<(), Box> { + let out_dir = env::var("OUT_DIR").unwrap(); + let path = path.as_ref(); + if path.is_file() { + println!("cargo:rerun-if-changed={}", path.display()); + // Parse test file + let s = fs::read_to_string(path).unwrap(); + let docs = YamlLoader::load_from_str(&s).unwrap(); + let grm = &docs[0]["grammar"].as_str().unwrap(); + let lex = &docs[0]["lexer"].as_str().unwrap(); + let yacckind = match docs[0]["yacckind"].as_str().unwrap() { + "Original(YaccOriginalActionKind::NoAction)" => { + YaccKind::Original(YaccOriginalActionKind::NoAction) + } + "Original(YaccOriginalActionKind::UserAction)" => { + YaccKind::Original(YaccOriginalActionKind::UserAction) + } + "Grmtools" => YaccKind::Grmtools, + "Original(YaccOriginalActionKind::GenericParseTree)" => { + YaccKind::Original(YaccOriginalActionKind::GenericParseTree) + } + s => panic!("YaccKind '{}' not supported", s), + }; + let (negative_yacc_flags, positive_yacc_flags) = &docs[0]["yacc_flags"] + .as_vec() + .map(|flags_vec| { + flags_vec + .iter() + .partition(|flag| flag.as_str().unwrap().starts_with('!')) + }) + .unwrap_or_else(|| (Vec::new(), Vec::new())); + let positive_yacc_flags = positive_yacc_flags + .iter() + .map(|flag| flag.as_str().unwrap()) + .collect::>(); + let negative_yacc_flags = negative_yacc_flags + .iter() + .map(|flag| { + let flag = flag.as_str().unwrap(); + flag.strip_prefix('!').unwrap() + }) + .collect::>(); + let yacc_flags = (&positive_yacc_flags, &negative_yacc_flags); + let (negative_lex_flags, positive_lex_flags) = &docs[0]["lex_flags"] + .as_vec() + .map(|flags_vec| { + flags_vec + .iter() + .partition(|flag| flag.as_str().unwrap().starts_with('!')) + }) + .unwrap_or_else(|| (Vec::new(), Vec::new())); + let negative_lex_flags = negative_lex_flags + .iter() + .map(|flag| { + let flag = flag.as_str().unwrap(); + flag.strip_prefix('!').unwrap() + }) + .collect::>(); + let positive_lex_flags = positive_lex_flags + .iter() + .map(|flag| flag.as_str().unwrap()) + .collect::>(); + let lex_flags = (&positive_lex_flags, &negative_lex_flags); + + // The code below, in essence, replicates lrlex and lrpar's internal / undocumented + // filename conventions. If those change, this code will also have to change. + + // Create grammar files + let base = path.file_stem().unwrap().to_str().unwrap(); + let mut pg = PathBuf::from(&out_dir); + pg.push(format!("{}.y.rs", base)); + fs::write(&pg, grm).unwrap(); + let mut pl = PathBuf::from(&out_dir); + pl.push(format!("{}.l.rs", base)); + fs::write(&pl, lex).unwrap(); + + // Build parser and lexer + let mut outp = PathBuf::from(&out_dir); + outp.push(format!("{}.y.rs", base)); + outp.set_extension("rs"); + let mut cp_build = CTParserBuilder::>::new() + .yacckind(yacckind) + .grammar_path(pg.to_str().unwrap()) + .output_path(&outp); + if let Some(flag) = check_flag(yacc_flags, "error_on_conflicts") { + cp_build = cp_build.error_on_conflicts(flag) + } + if let Some(flag) = check_flag(yacc_flags, "warnings_are_errors") { + cp_build = cp_build.warnings_are_errors(flag) + } + if let Some(flag) = check_flag(yacc_flags, "show_warnings") { + cp_build = cp_build.show_warnings(flag) + }; + let cp = cp_build.build()?; + + let mut outl = PathBuf::from(&out_dir); + outl.push(format!("{}.l.rs", base)); + outl.set_extension("rs"); + let mut cl_build = CTLexerBuilder::new() + .rule_ids_map(cp.token_map()) + .lexer_path(pl.to_str().unwrap()) + .output_path(&outl); + if let Some(flag) = check_flag(lex_flags, "allow_missing_terms_in_lexer") { + cl_build = cl_build.allow_missing_terms_in_lexer(flag) + } + if let Some(flag) = check_flag(lex_flags, "allow_missing_tokens_in_parser") { + cl_build = cl_build.allow_missing_tokens_in_parser(flag) + } + if let Some(flag) = check_flag(lex_flags, "dot_matches_new_line") { + cl_build = cl_build.dot_matches_new_line(flag) + } + if let Some(flag) = check_flag(lex_flags, "case_insensitive") { + cl_build = cl_build.case_insensitive(flag) + } + if let Some(flag) = check_flag(lex_flags, "multi_line") { + cl_build = cl_build.multi_line(flag) + } + if let Some(flag) = check_flag(lex_flags, "swap_greed") { + cl_build = cl_build.swap_greed(flag) + } + if let Some(flag) = check_flag(lex_flags, "ignore_whitespace") { + cl_build = cl_build.ignore_whitespace(flag) + } + if let Some(flag) = check_flag(lex_flags, "unicode") { + cl_build = cl_build.unicode(flag) + } + if let Some(flag) = check_flag(lex_flags, "octal") { + cl_build = cl_build.octal(flag) + } + cl_build.build()?; + } + Ok(()) +} + +fn check_flag((positive, negative): (&Vec<&str>, &Vec<&str>), flag: &str) -> Option { + assert_eq!( + positive.contains(&flag) | negative.contains(&flag), + positive.contains(&flag) ^ negative.contains(&flag) + ); + if positive.contains(&flag) { + Some(true) + } else if negative.contains(&flag) { + Some(false) + } else { + None + } +} diff --git a/lrpar/cttests/src/lib.rs b/lrpar/cttests/src/lib.rs index 01ea48d71..e448c8837 100644 --- a/lrpar/cttests/src/lib.rs +++ b/lrpar/cttests/src/lib.rs @@ -1,5 +1,10 @@ #[cfg(test)] use cfgrammar::Span; +mod cgen_helper; +#[allow(unused)] +use cgen_helper::run_test_path; +#[cfg(test)] +use cttests_macro::generate_codegen_fail_tests; use lrlex::lrlex_mod; use lrpar::lrpar_mod; #[cfg(test)] @@ -256,3 +261,7 @@ fn test_passthrough() { fn test_expect() { // This test merely needs to compile in order to be successful. } + +// Codegen failure tests +#[cfg(test)] +generate_codegen_fail_tests!("src/ctfails/*.test"); diff --git a/lrpar/cttests_macro/Cargo.toml b/lrpar/cttests_macro/Cargo.toml new file mode 100644 index 000000000..337de62b7 --- /dev/null +++ b/lrpar/cttests_macro/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "cttests_macro" +version = "0.1.0" +edition = "2021" + +[lib] +proc_macro = true +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +glob = "0.3" +quote = "1.0" +proc-macro2 = { version = "1.0", features=["proc-macro"]} +syn = "2.0" diff --git a/lrpar/cttests_macro/src/lib.rs b/lrpar/cttests_macro/src/lib.rs new file mode 100644 index 000000000..a5fee5fa1 --- /dev/null +++ b/lrpar/cttests_macro/src/lib.rs @@ -0,0 +1,42 @@ +extern crate proc_macro; +use glob::glob; +use proc_macro::TokenStream; +use proc_macro2::Span; +use quote::quote; +use syn::{parse_macro_input, Ident, LitStr}; +#[proc_macro] +pub fn generate_codegen_fail_tests(item: TokenStream) -> TokenStream { + let mut out = Vec::new(); + let test_glob_str: LitStr = parse_macro_input!(item); + // Not env!("CARGO_MANIFEST_DIR"), which would be relative to the cttests_macro crate. + // An absolute path which may contain non-utf8 characters. + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); + let cwd = std::env::current_dir().unwrap(); + // We want a relative path to the glob from the working directory + // such as: lrpar/cttests/ with any potentially non-utf8 leading characters removed. + let manifest_dir = std::path::Path::new(&manifest_dir) + .strip_prefix(cwd) + .unwrap(); + let test_glob_path = manifest_dir.join(&test_glob_str.value()); + let test_glob_str = test_glob_path.into_os_string().into_string().unwrap(); + let test_files = glob(&test_glob_str).unwrap(); + for file in test_files { + let file = file.unwrap(); + // Remove potentially non-utf8 leading characters again. + // This time relative to the manifest dir e.g. `src/ctfails/foo.test` + let file = file.as_path().strip_prefix(manifest_dir).unwrap(); + // Need to convert to string, because `PathBuf` lacks + // an impl for `ToTokens` a bounds given by `quote!`. + let path = file.display().to_string(); + let stem = file.file_stem().unwrap().to_string_lossy(); + let ident = Ident::new(&format!("codegen_fail_{}", stem), Span::call_site()); + out.push(quote! { + #[should_panic] + #[test] + fn #ident(){ + run_test_path(#path).unwrap(); + } + }); + } + out.into_iter().collect::().into() +}