From 882ba95554ec226bedffc6348f1c5c5ecb387b9d Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 31 Oct 2023 11:06:42 -0700 Subject: [PATCH] Add tricky cse test series --- Cargo.lock | 148 +++++- Cargo.toml | 1 + src/compiler/compiler.rs | 18 +- src/tests/compiler/clvm.rs | 2 +- src/tests/compiler/optimizer/cse.rs | 671 +++++++++++++++++++++++++++- src/tests/util.rs | 55 ++- 6 files changed, 862 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 563a70902..96a8debce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -140,6 +140,7 @@ dependencies = [ "indoc 1.0.9", "js-sys", "lazy_static", + "lfsr", "linked-hash-map", "num", "num-bigint", @@ -242,8 +243,8 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.66", + "quote 1.0.32", "syn 1.0.109", ] @@ -288,6 +289,12 @@ dependencies = [ "spki", ] +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + [[package]] name = "elliptic-curve" version = "0.13.5" @@ -431,8 +438,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce046d161f000fffde5f432a0d034d0341dc152643b2598ed5bfce44c4f3a8f0" dependencies = [ "proc-macro-hack", - "proc-macro2", - "quote", + "proc-macro2 1.0.66", + "quote 1.0.32", "syn 1.0.109", "unindent", ] @@ -446,6 +453,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "itertools" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.9" @@ -481,6 +497,61 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lfsr" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1408c5559fc91f522a78aa46b85647f5bcbeee9e64e4b7d8e7b0d259d0047c57" +dependencies = [ + "lfsr-base", + "lfsr-instances", + "lfsr-macro-generate", + "lfsr-macro-lookup", +] + +[[package]] +name = "lfsr-base" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1355f8e4ef0c69b2f41f154716197b1027a75afb094dbc1b5d96cd124b2d946c" + +[[package]] +name = "lfsr-instances" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c2fd4e3b24c4c995c6d89cffb1f346b68bfa37466bfde5023975af945bfb75" +dependencies = [ + "lfsr-base", + "lfsr-macro-generate", +] + +[[package]] +name = "lfsr-macro-generate" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b797be749e5d7d013b3e18954049468c228841a5068ceb55dd9da23a300ecff6" +dependencies = [ + "itertools", + "lfsr-base", + "proc-macro2 0.4.30", + "quote 0.6.13", + "syn 0.15.44", +] + +[[package]] +name = "lfsr-macro-lookup" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3a3a6a85a7aac81c9f94adc14ab565eaaac8fd91460d06b89e4dd93462cdf8" +dependencies = [ + "itertools", + "lfsr-base", + "lfsr-instances", + "proc-macro2 0.4.30", + "quote 0.6.13", + "syn 0.15.44", +] + [[package]] name = "libc" version = "0.2.147" @@ -715,6 +786,15 @@ version = "0.5.20+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +dependencies = [ + "unicode-xid", +] + [[package]] name = "proc-macro2" version = "1.0.66" @@ -765,7 +845,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0bc5215d704824dfddddc03f93cb572e1155c68b6761c37005e1c288808ea8" dependencies = [ "pyo3-macros-backend", - "quote", + "quote 1.0.32", "syn 1.0.109", ] @@ -775,19 +855,28 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71623fc593224afaab918aa3afcaf86ed2f43d34f6afde7f3922608f253240df" dependencies = [ - "proc-macro2", + "proc-macro2 1.0.66", "pyo3-build-config 0.14.5", - "quote", + "quote 1.0.32", "syn 1.0.109", ] +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" +dependencies = [ + "proc-macro2 0.4.30", +] + [[package]] name = "quote" version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" dependencies = [ - "proc-macro2", + "proc-macro2 1.0.66", ] [[package]] @@ -943,8 +1032,8 @@ version = "1.0.180" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24e744d7782b686ab3b73267ef05697159cc0e5abbed3f47f9933165e5219036" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.66", + "quote 1.0.32", "syn 2.0.28", ] @@ -1015,14 +1104,25 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +[[package]] +name = "syn" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "unicode-xid", +] + [[package]] name = "syn" version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.66", + "quote 1.0.32", "unicode-ident", ] @@ -1032,8 +1132,8 @@ version = "2.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.66", + "quote 1.0.32", "unicode-ident", ] @@ -1074,6 +1174,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + [[package]] name = "unindent" version = "0.1.11" @@ -1113,8 +1219,8 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2", - "quote", + "proc-macro2 1.0.66", + "quote 1.0.32", "syn 2.0.28", "wasm-bindgen-shared", ] @@ -1137,7 +1243,7 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ - "quote", + "quote 1.0.32", "wasm-bindgen-macro-support", ] @@ -1147,8 +1253,8 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.66", + "quote 1.0.32", "syn 2.0.28", "wasm-bindgen-backend", "wasm-bindgen-shared", @@ -1180,8 +1286,8 @@ version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecb993dd8c836930ed130e020e77d9b2e65dd0fbab1b67c790b0f5d80b11a575" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.66", + "quote 1.0.32", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index a7d03a6ab..3b3af48cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,6 +47,7 @@ optional = true [dev-dependencies] rand = "0.8.5" rand_chacha = "0.3.1" +lfsr = "0.3.0" [lib] name = "clvm_tools_rs" diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index bc5cde0cd..e02c2f5cd 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -127,14 +127,11 @@ pub fn desugar_pre_forms( do_desugar(&p1) } -pub fn compile_pre_forms( +pub fn compile_from_compileform( context: &mut BasicCompileContext, opts: Rc, - pre_forms: &[Rc], + p2: CompileForm, ) -> Result { - // Resolve includes, convert program source to lexemes - let p2 = desugar_pre_forms(context, opts.clone(), pre_forms)?; - let p3 = context.post_desugar_optimization(opts.clone(), p2)?; // generate code from AST, optionally with optimization @@ -145,6 +142,17 @@ pub fn compile_pre_forms( Ok(g2) } +pub fn compile_pre_forms( + context: &mut BasicCompileContext, + opts: Rc, + pre_forms: &[Rc], +) -> Result { + // Resolve includes, convert program source to lexemes + let p2 = desugar_pre_forms(context, opts.clone(), pre_forms)?; + + compile_from_compileform(context, opts, p2) +} + pub fn compile_file( allocator: &mut Allocator, runner: Rc, diff --git a/src/tests/compiler/clvm.rs b/src/tests/compiler/clvm.rs index 1354c27c9..e7542ad57 100644 --- a/src/tests/compiler/clvm.rs +++ b/src/tests/compiler/clvm.rs @@ -25,7 +25,7 @@ use crate::tests::classic::run::RandomClvmNumber; use crate::util::Number; -const TEST_TIMEOUT: usize = 1000000; +pub const TEST_TIMEOUT: usize = 1000000; fn test_compiler_clvm(to_run: &String, args: &String) -> Result, RunFailure> { let mut allocator = Allocator::new(); diff --git a/src/tests/compiler/optimizer/cse.rs b/src/tests/compiler/optimizer/cse.rs index d000b92ea..7596d761b 100644 --- a/src/tests/compiler/optimizer/cse.rs +++ b/src/tests/compiler/optimizer/cse.rs @@ -1,15 +1,30 @@ +use num_bigint::ToBigInt; +use rand::prelude::*; +use regex::Regex; + +use std::borrow::Borrow; +use std::collections::{BTreeSet, BTreeMap, HashMap}; use std::rc::Rc; -use regex::Regex; +use clvmr::allocator::Allocator; -use crate::compiler::compiler::DefaultCompilerOpts; -use crate::compiler::comptypes::CompilerOpts; +use crate::classic::clvm::__type_compatibility__::bi_one; +use crate::classic::clvm_tools::stages::stage_0::DefaultProgramRunner; + +use crate::compiler::CompileContextWrapper; +use crate::compiler::clvm::run; +use crate::compiler::compiler::{compile_from_compileform, DefaultCompilerOpts}; +use crate::compiler::comptypes::{BodyForm, CompileForm, CompilerOpts, DefunData, HelperForm}; +use crate::compiler::dialect::AcceptedDialect; use crate::compiler::frontend::compile_bodyform; +use crate::compiler::optimize::get_optimizer; use crate::compiler::optimize::cse::cse_optimize_bodyform; -use crate::compiler::sexp::parse_sexp; +use crate::compiler::sexp::{enlist, parse_sexp, SExp}; use crate::compiler::srcloc::Srcloc; use crate::tests::classic::run::{do_basic_brun, do_basic_run}; +use crate::tests::compiler::clvm::TEST_TIMEOUT; +use crate::tests::util::RngLFSR; #[test] fn smoke_test_cse_optimization() { @@ -27,7 +42,6 @@ fn smoke_test_cse_optimization() { let cse_transformed = cse_optimize_bodyform(&srcloc, b"test", &bodyform).expect("should cse optimize"); let re_def = r"(let ((cse_[$]_[0-9]+ ([*] ([+] 1 Q) R))) (a (i Q (com (G (- Q 1) cse_[$]_[0-9]+)) (com cse_[$]_[0-9]+)) 1))".replace("(", r"\(").replace(")",r"\)"); - eprintln!("re_def {re_def}"); let re = Regex::new(&re_def).expect("should become a regex"); assert!(re.is_match(&cse_transformed.to_sexp().to_string())); } @@ -190,3 +204,650 @@ fn test_atomsort_bad_ref() { assert_eq!(run_result_three_items, "(0x100003 0x100002 0x100001)"); } + +// Produce trees of CSE eligible forms and challenge the CSE optimizer. +// +// Things that might trip up CSE: +// +// A cse occurrence is in a lambda +// A cse occurrence is in an assign or let binding +// A cse occurrence is in an assign or let body +// A cse occurrence is in an if condition +// A cse occurrence is in a conditional branch of an if +// A cse occurrence dominates another +// apply is weird + +// Generate expressions of the form +// +// Imagining a random tree structure, we'll generate a program that retrieves +// a few particular values in the tree based on the presence or absence of other +// values. +// +// If we evaluate it in the wrong order, it'll fail, because we'll ensure that +// earlier checks gate later checks at runtime. +// +// Imagine +// +// 50 +// 25 75 +// 12 37 62 87 +// 6 18 31 43 56 68 81 93 +// +// Each tree node is (v l . r) +// +// So we can write a function like: +// +// (if (select-tree tree "") +// (if (select-tree tree "l") +// (if (all (select-tree tree "ll") (select-tree "llr") (select-tree tree "r") (select-tree tree "rl")) +// (select-tree tree "llrl") +// (select-tree tree "ll") +// ) +// ... +// +// Choose a few nodes from the tree along with nodes to return when those nodes are +// present. +// +// ((37 . 18) (75 . 56) (87 . 18) (56 . 68)) +// +// We choose points along the paths to each node to insert checks +// +// paths: +// +// -- checks +// (50 25 37 31 . 50) +// (50 25 37 . 18) +// (50 75 . 56) +// (50 75 62 56 . 68) +// (50 75 87 . 18) +// -- returns +// (50) +// (50 25 12 18) +// (50 75 62 56) +// (50 75 62 68) +// -- set +// (12 18 25 37 50 56 62 68 75 87) +// -- choose gates (not leaves) +// ((12) 18 25 37 50 56 62 68 (75) 87) +// +// Then organize them by their relationships to the gates +// +// ((12 18) (75 56 62 68 87) 25 37 50) +// +// Find out which rows are gated by which gates: +// +// Must be sorted by dominance +// (50 25 37 31 . 50) -- not gated +// (50 25 37 . 18) -- gated by 12 +// (50 75 62 56 . 68) -- gated by 75 +// (50 75 87 . 18) -- gated by 12 and 75 +// (50 75 . 56) -- gated by 75 +// +// First write out a skeleton that mirrors these choices: +// (if +// (all +// ;; path to check +// (select-tree tree "") (select-tree tree "l") (select-tree tree "lr") (select-tree tree "lrl") +// ;; path to target (if not subsumed) +// ;; checks here if needed. +// ) +// +// (select-tree tree "") +// +// Next paths (gated by 12) +// (if +// (all (select-tree tree "") (select-tree tree "l") (select-tree tree "ll")) +// ;; Expressions gated by 12 +// (if (all (select-tree tree "") (select-tree tree "l") (select-tree tree "lr") (select-tree tree "lllr")) +// (select-tree tree "lllr") +// ;; also gated by 75 +// (if (all (select-tree tree "") (select-tree tree "r")) +// (if (all (select-tree tree "") (select-tree tree "r") (select-tree tree "rl") (select-tree tree "rll") (select-tree tree "rlr")) +// (select-tree tree "rlr") +// ;; Others downstream (50 75 . 56) is after gated by 12 and 75 +// (if (all (select-tree tree "") (select-tree tree "r") (select-tree tree "rl") (select-tree tree "rll")) +// (select-tree tree "rll") +// ... + +struct GenerateTrickyCSE { + pub numbers: BTreeSet, + pub tree: TreeNode, + pub choices: Vec, +} + +#[derive(Default, Clone, Debug)] +struct TreeNode { + pub number: u16, + pub left: Option>, + pub right: Option>, +} + +fn generate_option(srcloc: Srcloc, opt: Option<&Rc>, filter: &F) -> SExp +where + F: Fn(&TreeNode) -> bool +{ + if let Some(r) = opt { + if filter(r) { + return r.generate(srcloc, filter); + } + } + + SExp::Nil(srcloc) +} + +impl TreeNode { + fn generate(&self, srcloc: Srcloc, filter: &F) -> SExp + where + F: Fn(&TreeNode) -> bool + { + let n = SExp::Integer(srcloc.clone(), self.number.to_bigint().unwrap()); + if self.left.is_none() && self.right.is_none() { + n + } else { + let a = generate_option(srcloc.clone(), self.left.as_ref(), filter); + let b = generate_option(srcloc.clone(), self.right.as_ref(), filter); + enlist(srcloc, &[Rc::new(a), Rc::new(b), Rc::new(n)]) + } + } +} + +#[derive(Eq, PartialEq, Ord, PartialOrd, Clone, Debug)] +enum Direction { + L, + R, +} + +#[derive(Eq, PartialEq, Ord, PartialOrd, Clone, Debug)] +struct TreeChoice { + pub path: Vec, +} + +impl TreeChoice { + pub fn prerequisites(&self) -> BTreeSet { + self.path.iter().enumerate().map(|(i,_)| { + TreeChoice { path: self.path.iter().take(i).cloned().collect() } + }).collect() + } + + pub fn contains(&self, other: &TreeChoice) -> bool { + self.prerequisites().contains(other) + } + + pub fn condition(&self) -> Rc { + let srcloc = Srcloc::start("*tree-choice-condition*"); + let mut condition = Rc::new(BodyForm::Value(SExp::Atom(srcloc.clone(), b"arg".to_vec()))); + for p in self.path.iter() { + let op = match p { + Direction::L => b"f", + Direction::R => b"r", + }; + condition = Rc::new(BodyForm::Call( + srcloc.clone(), + vec![ + Rc::new(BodyForm::Value(SExp::Atom(srcloc.clone(), op.to_vec()))), + condition + ], + None, + )); + } + condition + } + + pub fn conditions(&self, myself: bool, prev: Option>) -> Vec> { + let mut condition_vec = + prev.map(|b| vec![b.clone()]).unwrap_or_else(|| vec![]); + let mut prerequisite_conditions = self.prerequisites().iter().map(|p| p.condition()).collect(); + condition_vec.append(&mut prerequisite_conditions); + if myself { + condition_vec.push(self.condition()); + } + condition_vec + } +} + +fn all_conditions(c: &[Rc]) -> Rc { + if c.is_empty() { + return Rc::new(BodyForm::Quoted(SExp::Integer(Srcloc::start("*all-conditions-empty*"), bi_one()))); + } + let mut copy_vec = c.to_vec(); + copy_vec.insert(0, Rc::new(BodyForm::Value(SExp::Atom(c[0].loc(), b"all".to_vec())))); + Rc::new(BodyForm::Call( + c[0].loc(), + copy_vec, + None, + )) +} + +fn if_expr(cond: Rc, then_clause: Rc, else_clause: Rc) -> Rc { + Rc::new(BodyForm::Call( + cond.loc(), + vec![ + Rc::new(BodyForm::Value(SExp::Atom(cond.loc(), b"a".to_vec()))), + Rc::new(BodyForm::Call( + cond.loc(), + vec![ + Rc::new(BodyForm::Value(SExp::Atom(cond.loc(), b"i".to_vec()))), + cond.clone(), + Rc::new(BodyForm::Call( + cond.loc(), + vec![ + Rc::new(BodyForm::Value(SExp::Atom(cond.loc(), b"com".to_vec()))), + then_clause, + ], + None, + )), + Rc::new(BodyForm::Call( + cond.loc(), + vec![ + Rc::new(BodyForm::Value(SExp::Atom(cond.loc(), b"com".to_vec()))), + else_clause, + ], + None, + )), + ], + None, + )), + Rc::new(BodyForm::Value(SExp::Atom(cond.loc(), b"@".to_vec()))), + ], + None, + )) +} + +#[test] +fn test_tree_choice_conditions() { + let tree = TreeChoice { path: vec![Direction::L, Direction::L, Direction::R] }; + let conditions_of = all_conditions(&tree.conditions(false, None)); + assert_eq!(conditions_of.to_sexp().to_string(), "(all arg (f arg) (f (f arg)))"); +} + +#[derive(Eq, PartialEq, Ord, PartialOrd, Clone, Debug)] +struct CheckAndRetrieve { + pub gate: Option, + pub retrieve: TreeChoice, + pub checks: Vec, +} + +impl CheckAndRetrieve { + // generate an if of the native checks and the check of the retrievable + // itself for this object, otherwise doing the else clause. + pub fn generate_body_if(&self, else_clause: Rc) -> Rc { + let check_conditions: Vec> = + self.checks.iter().map(|c| { + all_conditions(&c.conditions(true, None)) + }).collect(); + let retrieve_condition = all_conditions(&self.retrieve.conditions(false, Some(all_conditions(&check_conditions)))); + let what_to_do = self.retrieve.condition(); + + if_expr(retrieve_condition, what_to_do, else_clause) + } +} + +#[test] +fn test_check_and_retrieve_1() { + let car = CheckAndRetrieve { + gate: None, + retrieve: TreeChoice { path: vec![Direction::L, Direction::R] }, + checks: vec![] + }; + let else_clause = Rc::new(BodyForm::Value(SExp::Nil(Srcloc::start("*test*")))); + assert_eq!(car.generate_body_if(else_clause).to_sexp().to_string(), "(if (all (q . 1) arg (f arg) (r (f arg))) (r (f arg)) ())"); +} + +#[test] +fn test_check_and_retrieve_2() { + let car = CheckAndRetrieve { + gate: None, + retrieve: TreeChoice { path: vec![Direction::L, Direction::R] }, + checks: vec![TreeChoice { path: vec![Direction::R] }] + }; + let else_clause = Rc::new(BodyForm::Value(SExp::Nil(Srcloc::start("*test*")))); + assert_eq!(car.generate_body_if(else_clause).to_sexp().to_string(), "(if (all (all (all arg (r arg))) arg (f arg) (r (f arg))) (r (f arg)) ())"); +} + +fn create_number_tree(tree: &mut TreeNode, numbers: &[u16]) { + assert!(!numbers.is_empty()); + let len = numbers.len(); + if len == 1 { + tree.number = numbers[0]; + return; + } + + let mid = len / 2; + tree.number = numbers[mid]; + if mid > 0 { + let left_slice = &numbers[0..mid]; + let mut left_node = TreeNode::default(); + create_number_tree(&mut left_node, left_slice); + tree.left = Some(Rc::new(left_node)); + } + if len > mid+1 { + let right_slice = &numbers[mid+1..len]; + let mut right_node = TreeNode::default(); + create_number_tree(&mut right_node, right_slice); + tree.right = Some(Rc::new(right_node)); + } +} + +fn choose_in_tree(tree: &TreeNode, number: u16) -> TreeChoice { + let mut path: Vec = Vec::new(); + let mut tn: &TreeNode = tree; + while tn.number != number { + if number < tn.number { + path.push(Direction::L); + if let Some(t) = tn.left.as_ref().borrow() { + tn = t; + } else { + panic!("should have existed"); + } + } else { + path.push(Direction::R); + if let Some(t) = tn.right.as_ref().borrow() { + tn = t; + } else { + panic!("should have existed"); + } + } + } + TreeChoice { path } +} + +#[derive(Eq, PartialEq, Ord, PartialOrd, Clone, Debug)] +struct IfWithGate { + gate: TreeChoice, + in_gate: BTreeSet, + otherwise_reachable: BTreeSet, +} + +impl IfWithGate { + fn generate(&self) -> Rc { + let srcloc = Srcloc::start("*if-with-gate*"); + let gate_conditions = all_conditions(&self.gate.conditions(false, None)); + let mut maybe_also_these = Rc::new(BodyForm::Quoted(SExp::Nil(srcloc.clone()))); + for reachable in self.otherwise_reachable.iter().rev() { + maybe_also_these = reachable.generate_body_if(maybe_also_these); + } + + let mut gated = maybe_also_these.clone(); + for new_gated in self.in_gate.iter().rev() { + gated = new_gated.generate_body_if(gated); + } + + if_expr( + gate_conditions, + gated, + maybe_also_these, + ) + } +} + +#[test] +fn test_if_with_gate_generate_0() { + let mut in_gate_set = BTreeSet::new(); + in_gate_set.insert(CheckAndRetrieve { + gate: Some(TreeChoice { path: vec![Direction::R, Direction::L, Direction::R, Direction::L] }), + retrieve: TreeChoice { path: vec![Direction::R, Direction::R, Direction::R, Direction::R, Direction::L, Direction::L] }, + checks: vec![], + }); + let mut otherwise_set = BTreeSet::new(); + otherwise_set.insert(CheckAndRetrieve { + gate: None, + retrieve: TreeChoice { path: vec![Direction::R, Direction::R, Direction::R, Direction::R, Direction::L, Direction::R] }, + checks: vec![], + }); + let iwg = IfWithGate { + gate: TreeChoice { path: vec![Direction::R, Direction::R, Direction::R] }, + in_gate: in_gate_set, + otherwise_reachable: otherwise_set, + }; + assert_eq!(iwg.generate().to_sexp().to_string(), "(if (all arg (r arg) (r (r arg))) (if (all (q . 1) arg (r arg) (r (r arg)) (r (r (r arg))) (r (r (r (r arg)))) (f (r (r (r (r arg)))))) (f (f (r (r (r (r arg)))))) (if (all (q . 1) arg (r arg) (r (r arg)) (r (r (r arg))) (r (r (r (r arg)))) (f (r (r (r (r arg)))))) (r (f (r (r (r (r arg)))))) (q))) (if (all (q . 1) arg (r arg) (r (r arg)) (r (r (r arg))) (r (r (r (r arg)))) (f (r (r (r (r arg)))))) (r (f (r (r (r (r arg)))))) (q)))"); +} + +impl GenerateTrickyCSE { + fn new(rng: &mut R) -> Self { + // Generate number tree. + let number_of_numbers = 5 + rng.gen::() % 11; + let numbers_set: BTreeSet = (0..number_of_numbers).map(|_| rng.gen()).collect(); + let numbers: Vec = numbers_set.iter().copied().collect(); + let mut number_tree = TreeNode::default(); + create_number_tree(&mut number_tree, &numbers); + + // Now we have a number tree. Choose some random values and find their + // paths in the tree. + let num_choices = 3 + (rng.gen::() % 7); + let mut choices: Vec = (0..num_choices) + .map(|_| { + let retrieve = choose_in_tree( + &number_tree, + numbers[rng.gen::() % number_of_numbers], + ); + let gate = + if rng.gen() { + Some(choose_in_tree( + &number_tree, + numbers[rng.gen::() % number_of_numbers], + )) + } else { + None + }; + let num_checks = rng.gen::() % 3; + let checks: Vec = (0..num_checks).map(|_| { + choose_in_tree( + &number_tree, + numbers[rng.gen::() % number_of_numbers], + ) + }).collect(); + CheckAndRetrieve { + gate, + retrieve, + checks, + } + }) + .collect(); + + // Actually, we can just sort the choices since ones that dominate will + // come before ones that are dominated (longer path with same prefix). + choices.sort(); + + eprintln!("choices {choices:?}"); + GenerateTrickyCSE { choices, tree: number_tree.clone(), numbers: numbers_set } + } + + fn generate_expr(&self) -> Rc { + let srcloc = Srcloc::start("*tree-generate*"); + + // Generate if statements for each tier of gate + let mut choice_by_gate: BTreeMap = BTreeMap::new(); + + // Make a list of distinct gates and the directly gated choices. + for choice in self.choices.iter() { + if let Some(choice_gate) = choice.gate.as_ref() { + if let Some(gated_by_choice) = choice_by_gate.get_mut(choice_gate) { + gated_by_choice.in_gate.insert(choice.clone()); + } else { + let mut in_gate = BTreeSet::new(); + in_gate.insert(choice.clone()); + choice_by_gate.insert(choice_gate.clone(), IfWithGate { + gate: choice_gate.clone(), + in_gate, + otherwise_reachable: BTreeSet::default(), + }); + } + } + } + + // For each gate, filter all the remaining checks into in_gate or + // otherwise_reachable. + for (gate, ifwith) in choice_by_gate.iter_mut() { + let (in_gate, otherwise_reachable) = self.choices.iter().cloned().partition::, _>(|choice| { + choice.gate.as_ref().map(|g| gate.contains(g)) + .unwrap_or(false) + }); + for g in in_gate.iter() { + ifwith.in_gate.insert(g.clone()); + } + for g in otherwise_reachable.iter() { + ifwith.otherwise_reachable.insert(g.clone()); + } + } + + // Generate the full expression. + let mut final_value = Rc::new(BodyForm::Quoted(SExp::Nil(srcloc.clone()))); + for (gate, ifwith) in choice_by_gate.iter().rev() { + let this_gate = ifwith.generate(); + let gate_conditions = all_conditions(&gate.conditions(true, None)); + final_value = if_expr( + gate_conditions, + this_gate, + final_value, + ); + } + + final_value + } + + fn generate_helper(&self) -> HelperForm { + let srcloc = Srcloc::start("*helper*"); + let args = Rc::new(SExp::Cons( + srcloc.clone(), + Rc::new(SExp::Atom(srcloc.clone(), b"arg".to_vec())), + Rc::new(SExp::Nil(srcloc.clone())) + )); + HelperForm::Defun(false, Box::new(DefunData { + nl: srcloc.clone(), + loc: srcloc.clone(), + name: b"tricky-cse".to_vec(), + args: args.clone(), + orig_args: args.clone(), + body: self.generate_expr(), + kw: None, + synthetic: None, + })) + } + + fn generate_program(&self) -> CompileForm { + let helper = self.generate_helper(); + let args = Rc::new(SExp::Atom(helper.loc(), b"arg".to_vec())); + CompileForm { + loc: helper.loc(), + include_forms: vec![], + args, + exp: Rc::new(BodyForm::Call( + helper.loc(), + vec![ + Rc::new(BodyForm::Value(SExp::Atom(helper.loc(), helper.name().to_vec()))), + Rc::new(BodyForm::Value(SExp::Atom(helper.loc(), b"arg".to_vec()))), + ], + None, + )), + helpers: vec![helper], + } + } +} + +fn test_generated_cse(n: u32) { + eprintln!("=== SEED {n} ==="); + let mut rng = RngLFSR::new(n); + let tcse = GenerateTrickyCSE::new(&mut rng); + let generated = tcse.generate_program(); + eprintln!("{}", generated.to_sexp()); + let runner = Rc::new(DefaultProgramRunner::new()); + let opts: Rc = Rc::new(DefaultCompilerOpts::new("*tricky-cse*")); + let opts21 = opts.set_dialect(AcceptedDialect { + stepping: Some(21), + strict: true, + }); + let opts23 = opts.set_dialect(AcceptedDialect { + stepping: Some(23), + strict: true, + }).set_optimize(true); + let mut allocator = Allocator::new(); + let mut symbols = HashMap::new(); + let compiled21; + let compiled23; + + // Get strict cl21 compile + { + let mut wrapper21 = CompileContextWrapper::new( + &mut allocator, + runner.clone(), + &mut symbols, + get_optimizer(&generated.loc(), opts21.clone()).expect("should be ok dialect"), + ); + eprintln!("21 compile"); + compiled21 = Rc::new(compile_from_compileform( + &mut wrapper21.context, + opts21, + generated.clone(), + ).expect("compiled")) + } + + // Get cl23 compile + { + let mut wrapper23 = CompileContextWrapper::new( + &mut allocator, + runner.clone(), + &mut symbols, + get_optimizer(&generated.loc(), opts23.clone()).expect("should be ok dialect"), + ); + eprintln!("23 compile"); + compiled23 = Rc::new(compile_from_compileform( + &mut wrapper23.context, + opts23, + generated.clone(), + ).expect("compiled")) + } + + eprintln!("generate tree numbers"); + let check_numbers: BTreeSet = tcse.numbers.iter().filter(|_| { + let check: u8 = rng.gen(); + check & 15 >= 1 + }).copied().collect(); + eprintln!("generate tree"); + let tree = Rc::new(tcse.tree.generate(generated.loc(), &move |tn| { + check_numbers.contains(&tn.number) + })); + + eprintln!("compiled21 {compiled21}"); + eprintln!("compiled23 {compiled23}"); + eprintln!("tree {tree}"); + + let mut allocator = Allocator::new(); + let clvm21 = run( + &mut allocator, + runner.clone(), + opts.prim_map(), + compiled21.clone(), + tree.clone(), + None, + Some(TEST_TIMEOUT), + ); + let clvm23 = run( + &mut allocator, + runner, + opts.prim_map(), + compiled23.clone(), + tree.clone(), + None, + Some(TEST_TIMEOUT), + ); + if let Ok(res21) = clvm21.as_ref() { + eprintln!("cl21 {res21}"); + } + if let Ok(res23) = clvm23.as_ref() { + eprintln!("cl23 {res23}"); + } + if clvm21.is_err() || clvm23.is_err() { + // Precise errors might change due to differences in ordering and + // locations and such. + assert_eq!(clvm21.is_err(), clvm23.is_err()); + return; + } + assert_eq!(clvm21, clvm23); +} + +#[test] +fn test_generate_tricky_cse() { + for i in 0..16 { + test_generated_cse(10000 + i); + } +} diff --git a/src/tests/util.rs b/src/tests/util.rs index 5585a2ba0..b0bec8303 100644 --- a/src/tests/util.rs +++ b/src/tests/util.rs @@ -1,5 +1,8 @@ -use crate::util::toposort; use std::collections::HashSet; +use rand::prelude::*; +use rand::Error; + +use crate::util::toposort; #[derive(Debug, Clone)] struct TopoSortCheckItem { @@ -94,3 +97,53 @@ fn test_topo_sort_1() { assert!(result.is_err()); } + +// A simple pseudo RNG based on an lfsr. Good enough to generate interesting +// deterministic patterns. +pub struct RngLFSR { + generator: u32 +} + +impl RngLFSR { + pub fn new(state: u32) -> Self { RngLFSR { generator: state } } + fn next(&mut self) -> u32 { + self.generator = lfsr::galois::Galois16::up(self.generator); + self.generator + } +} + +impl RngCore for RngLFSR { + fn next_u32(&mut self) -> u32 { + let a = self.next(); + let b = self.next(); + (a << 16) | b + } + + fn next_u64(&mut self) -> u64 { + let a = self.next_u32() as u64; + let b = self.next_u32() as u64; + (a << 32) | b + } + + fn fill_bytes(&mut self, dest: &mut [u8]) { + if dest.is_empty() { + return; + } + + for i in 0..(dest.len() / 2) { + let a = self.next(); + dest[i * 2] = (a & 0xff) as u8; + dest[i * 2 + 1] = ((a >> 8) & 0xff) as u8; + } + + if (dest.len() & 1) != 0 { + let a = self.next(); + dest[dest.len() - 1] = (a & 0xff) as u8; + } + } + + fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), Error> { + self.fill_bytes(dest); + Ok(()) + } +}