From 50d7dec2c1d8fb20002d9fd6e1d07468dc777db8 Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 19 Dec 2022 20:33:56 -0800 Subject: [PATCH 001/117] WIP assign --- src/compiler/codegen.rs | 15 ++++++-- src/compiler/comptypes.rs | 12 ++++++- src/compiler/frontend.rs | 57 +++++++++++++++++++++++++++-- src/util/mod.rs | 76 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 154 insertions(+), 6 deletions(-) diff --git a/src/compiler/codegen.rs b/src/compiler/codegen.rs index d3bed4200..1eb3a3a65 100644 --- a/src/compiler/codegen.rs +++ b/src/compiler/codegen.rs @@ -13,7 +13,7 @@ use crate::classic::clvm::__type_compatibility__::bi_one; use crate::compiler::clvm::run; use crate::compiler::compiler::{is_at_capture, run_optimizer}; use crate::compiler::comptypes::{ - fold_m, join_vecs_to_string, list_to_cons, Binding, BodyForm, Callable, CompileErr, + fold_m, join_vecs_to_string, list_to_cons, Binding, BindingPattern, BodyForm, Callable, CompileErr, CompileForm, CompiledCode, CompilerOpts, DefunCall, DefunData, HelperForm, InlineFunction, LetData, LetFormKind, PrimaryCodegen, }; @@ -739,7 +739,16 @@ fn generate_let_defun( ) -> HelperForm { let new_arguments: Vec> = bindings .iter() - .map(|b| Rc::new(SExp::Atom(l.clone(), b.name.clone()))) + .map(|b| { + match b.pattern { + BindingPattern::Name(name) => { + Rc::new(SExp::Atom(l.clone(), name.clone())) + } + BindingPattern::Complex(sexp) => { + todo!(); + } + } + }) .collect(); let inner_function_args = SExp::Cons( @@ -827,7 +836,7 @@ fn hoist_body_let_binding( revised_bindings.push(Rc::new(Binding { loc: b.loc.clone(), nl: b.nl.clone(), - name: b.name.clone(), + pattern: b.pattern.clone(), body: new_binding, })); } diff --git a/src/compiler/comptypes.rs b/src/compiler/comptypes.rs index cbe4adb4c..b8d3fdcc1 100644 --- a/src/compiler/comptypes.rs +++ b/src/compiler/comptypes.rs @@ -57,11 +57,17 @@ pub fn list_to_cons(l: Srcloc, list: &[Rc]) -> SExp { result } +#[derive(Clone, Debug)] +pub enum BindingPattern { + Name(Vec), + Complex(Rc) +} + #[derive(Clone, Debug)] pub struct Binding { pub loc: Srcloc, pub nl: Srcloc, - pub name: Vec, + pub pattern: BindingPattern, pub body: Rc, } @@ -435,6 +441,10 @@ impl BodyForm { impl Binding { pub fn to_sexp(&self) -> Rc { + let pat = + match self.pattern { + BindingPattern:: + }; Rc::new(SExp::Cons( self.loc.clone(), Rc::new(SExp::atom_from_vec(self.loc.clone(), &self.name)), diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index be0038069..687cf4dd5 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -5,14 +5,14 @@ use std::rc::Rc; use crate::classic::clvm::__type_compatibility__::bi_one; use crate::compiler::comptypes::{ - list_to_cons, Binding, BodyForm, CompileErr, CompileForm, CompilerOpts, DefconstData, + list_to_cons, Binding, BindingPattern, BodyForm, CompileErr, CompileForm, CompilerOpts, DefconstData, DefmacData, DefunData, HelperForm, IncludeDesc, LetData, LetFormKind, ModAccum, }; use crate::compiler::preprocessor::preprocess; use crate::compiler::rename::rename_children_compileform; use crate::compiler::sexp::{enlist, SExp}; use crate::compiler::srcloc::Srcloc; -use crate::util::u8_from_number; +use crate::util::{toposort, u8_from_number}; fn collect_used_names_sexp(body: Rc) -> Vec> { match body.borrow() { @@ -304,6 +304,59 @@ pub fn compile_bodyform( body: Rc::new(compiled_body), }, )) + } else if *atom_name == "assign".as_bytes().to_vec() { + if v.len() % 1 == 0 { + return finish_err("assign"); + } + + let mut bindings = Vec::new(); + for idx in (0..(v.len() - 1) / 2).map(|idx| 1 + idx * 2) { + let destructure_pattern = v[idx].clone(); + let binding_body = compile_bodyform(opts.clone(), v[idx+1].clone())?; + bindings.push(Binding { + loc: v[idx].loc().ext(&v[idx+1].loc()), + nl: destructure_pattern.loc(), + pattern: BindingPattern::Complex(destructure_pattern), + body: Rc::new(binding_body), + }); + } + + // Topological sort of bindings. + let sorted_spec = toposort( + &bindings, + CompileErr(l.clone(), "deadlock resolving binding order".to_string()), + // Needs: What this binding relies on. + |possible, b| { + let mut need_set = HashSet::new(); + make_provides_set(&mut need_set, b.body.to_sexp()); + need_set.intersect(possible) + }, + // Has: What this binding provides. + |b| { + match b.pattern { + BindingPattern::Simple(name) => { + HashSet::from(vec![Rc::new(SExp::Atom(b.nl.clone(), name.clone()))]) + } + BindingPattern::Complex(sexp) => { + let mut result_set = HashSet::new(); + make_provides_set(&mut result_set, sexp.clone()); + result_set + } + } + }); + + let compiled_body = + compile_bodyform(opts, v[v.len()-1].clone())?; + + Ok(BodyForm::Let( + LetFormKind::Parallel, + LetData { + loc: l.clone(), + kw: Some(l.clone()), + bindings: bindings, + body: Rc::new(compiled_body) + } + )) } else if *atom_name == "quote".as_bytes().to_vec() { if v.len() != 1 { return finish_err("quote"); diff --git a/src/util/mod.rs b/src/util/mod.rs index 2f25a914b..96007056d 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,4 +1,6 @@ use num_bigint::BigInt; +use std::collections::HashSet; +use std::mem::swap; use unicode_segmentation::UnicodeSegmentation; pub type Number = BigInt; @@ -38,3 +40,77 @@ pub fn collapse(r: Result) -> A { Err(e) => e, } } + +#[derive(Debug, Clone)] +pub struct TopoSortItem { + index: usize, + needs: HashSet, + has: HashSet +} + +// F: tells whether t2 includes t1. +pub fn toposort(list: &Vec, deadlock: E, needs: Needs, has: Has) -> Result>, E> +where + Needs: Fn(&HashSet, &T) -> Result, E>, + Has: Fn(&T) -> HashSet, + K: std::cmp::Eq, + K: std::hash::Hash, + K: Clone +{ + let mut possible = HashSet::new(); + let mut done = HashSet::new(); + let mut items: Vec> = list.iter().enumerate().map(|(i,item)| { + TopoSortItem { + index: i, + needs: HashSet::new(), + has: has(item) + } + }).collect(); + let mut finished_idx = 0; + + // Determine what's defined in these bindings. + for (i,item) in items.iter().enumerate() { + for new_item in possible.union(&item.has) { + possible.insert(new_item.clone()); + } + } + + // Set needs based on possible. We may fail. + for i in 0..items.len() { + items[i].needs = needs(&possible, &list[i])?; + } + + while finished_idx < items.len() { + // Find things with no new dependencies. + let move_to_front: Vec<(usize, TopoSortItem)> = + items.iter().enumerate().skip(finished_idx).filter(|(i,item)| { + item.needs.is_subset(&done) + }).map(|(i,item)| (i, item.clone())).collect(); + + if move_to_front.is_empty() { + // Circular dependency somewhere. + return Err(deadlock); + } + + // Swap items into place earlier in the list. + for (idx,tomove) in move_to_front.iter() { + if *idx != finished_idx { + swap(&mut items[*idx], &mut items[finished_idx]); + } + + // Add new 'has' items to done. + let tmp = HashSet::new(); + for u in done.union(&items[finished_idx].has) { + tmp.insert(u.clone()); + } + let intersection = tmp.intersection(&possible); + done.clear(); + for i in intersection { + done.insert(i.clone()); + } + finished_idx += 1; + } + } + + return Ok(items); +} From 02cbf82457e66328b1cb6df39bd869fc224ea359 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 20 Dec 2022 10:12:13 -0800 Subject: [PATCH 002/117] Assign WIP part 2: patterns woven in without disturbing users of non-patterns --- src/compiler/codegen.rs | 4 +- src/compiler/comptypes.rs | 11 ++++-- src/compiler/evaluate.rs | 23 ++++++++++- src/compiler/frontend.rs | 40 ++++++++++++++------ src/compiler/rename.rs | 80 +++++++++++++++++++++++++++------------ src/util/mod.rs | 23 +++++------ 6 files changed, 127 insertions(+), 54 deletions(-) diff --git a/src/compiler/codegen.rs b/src/compiler/codegen.rs index 1eb3a3a65..974c9e913 100644 --- a/src/compiler/codegen.rs +++ b/src/compiler/codegen.rs @@ -740,11 +740,11 @@ fn generate_let_defun( let new_arguments: Vec> = bindings .iter() .map(|b| { - match b.pattern { + match b.pattern.borrow() { BindingPattern::Name(name) => { Rc::new(SExp::Atom(l.clone(), name.clone())) } - BindingPattern::Complex(sexp) => { + BindingPattern::Complex(_sexp) => { todo!(); } } diff --git a/src/compiler/comptypes.rs b/src/compiler/comptypes.rs index b8d3fdcc1..829efb488 100644 --- a/src/compiler/comptypes.rs +++ b/src/compiler/comptypes.rs @@ -442,12 +442,17 @@ impl BodyForm { impl Binding { pub fn to_sexp(&self) -> Rc { let pat = - match self.pattern { - BindingPattern:: + match &self.pattern { + BindingPattern::Name(name) => { + Rc::new(SExp::atom_from_vec(self.loc.clone(), &name)) + } + BindingPattern::Complex(sexp) => { + sexp.clone() + } }; Rc::new(SExp::Cons( self.loc.clone(), - Rc::new(SExp::atom_from_vec(self.loc.clone(), &self.name)), + pat, Rc::new(SExp::Cons( self.loc.clone(), self.body.to_sexp(), diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index 82d2eb5c8..b8d731ff5 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -13,7 +13,7 @@ use crate::compiler::clvm::run; use crate::compiler::codegen::codegen; use crate::compiler::compiler::is_at_capture; use crate::compiler::comptypes::{ - Binding, BodyForm, CompileErr, CompileForm, CompilerOpts, HelperForm, LetData, LetFormKind, + Binding, BindingPattern, BodyForm, CompileErr, CompileForm, CompilerOpts, HelperForm, LetData, LetFormKind, }; use crate::compiler::frontend::frontend; use crate::compiler::runtypes::RunFailure; @@ -48,13 +48,32 @@ fn select_helper(bindings: &[HelperForm], name: &[u8]) -> Option { None } +fn compute_paths_of_destructure(_bindings: &mut Vec<(Vec, Rc)>, _structure: Rc, _bodyform: Rc) { + todo!(); +} + fn update_parallel_bindings( bindings: &HashMap, Rc>, have_bindings: &[Rc], ) -> HashMap, Rc> { let mut new_bindings = bindings.clone(); for b in have_bindings.iter() { - new_bindings.insert(b.name.clone(), b.body.clone()); + match &b.pattern { + BindingPattern::Name(name) => { + new_bindings.insert(name.clone(), b.body.clone()); + } + BindingPattern::Complex(structure) => { + let mut computed_getters = Vec::new(); + compute_paths_of_destructure( + &mut computed_getters, + structure.clone(), + b.body.clone() + ); + for (name, p) in computed_getters.iter() { + new_bindings.insert(name.clone(), p.clone()); + } + } + } } new_bindings } diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index 687cf4dd5..b7a4fa209 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -223,7 +223,7 @@ fn make_let_bindings( result.push(Rc::new(Binding { loc: l.clone(), nl: l.clone(), - name: name.to_vec(), + pattern: BindingPattern::Name(name.to_vec()), body: Rc::new(compiled_body), })); result.append(&mut rest_bindings); @@ -236,6 +236,13 @@ fn make_let_bindings( } } +fn make_provides_set( + _need_set: &mut HashSet>, + _body_sexp: Rc +) { + todo!(); +} + pub fn compile_bodyform( opts: Rc, body: Rc, @@ -311,14 +318,14 @@ pub fn compile_bodyform( let mut bindings = Vec::new(); for idx in (0..(v.len() - 1) / 2).map(|idx| 1 + idx * 2) { - let destructure_pattern = v[idx].clone(); - let binding_body = compile_bodyform(opts.clone(), v[idx+1].clone())?; - bindings.push(Binding { + let destructure_pattern = Rc::new(v[idx].clone()); + let binding_body = compile_bodyform(opts.clone(), Rc::new(v[idx+1].clone()))?; + bindings.push(Rc::new(Binding { loc: v[idx].loc().ext(&v[idx+1].loc()), nl: destructure_pattern.loc(), pattern: BindingPattern::Complex(destructure_pattern), body: Rc::new(binding_body), - }); + })); } // Topological sort of bindings. @@ -329,13 +336,17 @@ pub fn compile_bodyform( |possible, b| { let mut need_set = HashSet::new(); make_provides_set(&mut need_set, b.body.to_sexp()); - need_set.intersect(possible) - }, + let mut need_set_thats_possible = HashSet::new(); + for need in need_set.intersection(possible) { + need_set_thats_possible.insert(need.clone()); + } + Ok(need_set_thats_possible) + }, // Has: What this binding provides. |b| { - match b.pattern { - BindingPattern::Simple(name) => { - HashSet::from(vec![Rc::new(SExp::Atom(b.nl.clone(), name.clone()))]) + match b.pattern.borrow() { + BindingPattern::Name(name) => { + HashSet::from([name.clone()]) } BindingPattern::Complex(sexp) => { let mut result_set = HashSet::new(); @@ -343,11 +354,16 @@ pub fn compile_bodyform( result_set } } - }); + })?; + + let sorted_bindings: Vec> = sorted_spec.iter().map(|item| { + bindings[item.index].clone() + }).collect(); let compiled_body = - compile_bodyform(opts, v[v.len()-1].clone())?; + compile_bodyform(opts, Rc::new(v[v.len()-1].clone()))?; + todo!(); Ok(BodyForm::Let( LetFormKind::Parallel, LetData { diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index c744d5cb8..d825ea6de 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use std::rc::Rc; use crate::compiler::comptypes::{ - Binding, BodyForm, CompileForm, DefconstData, DefmacData, DefunData, HelperForm, LetData, + Binding, BindingPattern, BodyForm, CompileForm, DefconstData, DefmacData, DefunData, HelperForm, LetData, LetFormKind, }; use crate::compiler::gensym::gensym; @@ -14,7 +14,7 @@ fn rename_in_qq(namemap: &HashMap, Vec>, body: Rc) -> Rc .and_then(|x| { if let [SExp::Atom(_, q), body] = &x[..] { if q == b"unquote" { - return Some(rename_in_cons(namemap, Rc::new(body.clone()))); + return Some(rename_in_cons(namemap, Rc::new(body.clone()), true)); } } @@ -31,7 +31,11 @@ fn rename_in_qq(namemap: &HashMap, Vec>, body: Rc) -> Rc } /* Given a cons cell, rename occurrences of oldname to newname */ -fn rename_in_cons(namemap: &HashMap, Vec>, body: Rc) -> Rc { +fn rename_in_cons( + namemap: &HashMap, Vec>, + body: Rc, + qq_handling: bool +) -> Rc { match body.borrow() { SExp::Atom(l, name) => match namemap.get(name) { Some(v) => Rc::new(SExp::Atom(l.clone(), v.to_vec())), @@ -61,7 +65,7 @@ fn rename_in_cons(namemap: &HashMap, Vec>, body: Rc) -> Rc body.clone(), }) .unwrap_or_else(|| body.clone()); - } else if *q == "qq".as_bytes().to_vec() { + } else if *q == "qq".as_bytes().to_vec() && qq_handling { return r .proper_list() .map(|x| match &x[..] { @@ -74,8 +78,8 @@ fn rename_in_cons(namemap: &HashMap, Vec>, body: Rc) -> Rc body.clone(), @@ -104,16 +108,42 @@ fn invent_new_names_sexp(body: Rc) -> Vec<(Vec, Vec)> { } } -fn make_binding_unique(b: &Binding) -> (Vec, Binding) { - ( - b.name.to_vec(), - Binding { - loc: b.loc.clone(), - nl: b.nl.clone(), - name: gensym(b.name.clone()), - body: b.body.clone(), - }, - ) +fn make_binding_unique(b: &Binding) -> (HashMap, Vec>, Binding) { + match b.pattern.borrow() { + BindingPattern::Name(name) => { + let mut single_name_map = HashMap::new(); + let new_name = gensym(name.clone()); + single_name_map.insert(name.to_vec(), new_name.clone()); + ( + single_name_map, + Binding { + loc: b.loc.clone(), + nl: b.nl.clone(), + pattern: BindingPattern::Name(new_name), + body: b.body.clone(), + } + ) + } + BindingPattern::Complex(pat) => { + let new_names_vec = invent_new_names_sexp(pat.clone()); + let mut new_names = HashMap::new(); + + for (n, v) in new_names_vec.iter() { + new_names.insert(n.clone(), v.clone()); + } + + let renamed_pattern = rename_in_cons(&new_names, pat.clone(), false); + ( + new_names, + Binding { + loc: b.loc.clone(), + nl: b.nl.clone(), + pattern: BindingPattern::Complex(renamed_pattern), + body: b.body.clone(), + } + ) + } + } } fn rename_in_bodyform(namemap: &HashMap, Vec>, b: Rc) -> BodyForm { @@ -126,7 +156,7 @@ fn rename_in_bodyform(namemap: &HashMap, Vec>, b: Rc) -> B Rc::new(Binding { loc: b.loc(), nl: b.nl.clone(), - name: b.name.clone(), + pattern: b.pattern.clone(), body: Rc::new(rename_in_bodyform(namemap, b.body.clone())), }) }) @@ -209,7 +239,7 @@ fn rename_args_bodyform(b: &BodyForm) -> BodyForm { } BodyForm::Let(LetFormKind::Parallel, letdata) => { - let renames: Vec<(Vec, Binding)> = letdata + let renames: Vec<(HashMap, Vec>, Binding)> = letdata .bindings .iter() .map(|x| make_binding_unique(x.borrow())) @@ -218,8 +248,10 @@ fn rename_args_bodyform(b: &BodyForm) -> BodyForm { renames.iter().map(|(_, x)| Rc::new(x.clone())).collect(); let mut local_namemap = HashMap::new(); for x in renames.iter() { - let (oldname, binding) = x; - local_namemap.insert(oldname.to_vec(), binding.name.clone()); + let (oldnames, _) = x; + for (oldname, binding_name) in oldnames.iter() { + local_namemap.insert(oldname.to_vec(), binding_name.clone()); + } } let new_bindings = new_renamed_bindings .iter() @@ -227,7 +259,7 @@ fn rename_args_bodyform(b: &BodyForm) -> BodyForm { Rc::new(Binding { loc: x.loc.clone(), nl: x.nl.clone(), - name: x.name.clone(), + pattern: x.pattern.clone(), body: Rc::new(rename_args_bodyform(&x.body)), }) }) @@ -307,7 +339,7 @@ fn rename_args_helperform(h: &HelperForm) -> HelperForm { for x in new_names.iter() { local_namemap.insert(x.0.to_vec(), x.1.to_vec()); } - let local_renamed_arg = rename_in_cons(&local_namemap, mac.args.clone()); + let local_renamed_arg = rename_in_cons(&local_namemap, mac.args.clone(), true); let local_renamed_body = rename_args_compileform(mac.program.borrow()); HelperForm::Defmacro(DefmacData { loc: mac.loc.clone(), @@ -327,7 +359,7 @@ fn rename_args_helperform(h: &HelperForm) -> HelperForm { for x in new_names.iter() { local_namemap.insert(x.0.clone(), x.1.clone()); } - let local_renamed_arg = rename_in_cons(&local_namemap, defun.args.clone()); + let local_renamed_arg = rename_in_cons(&local_namemap, defun.args.clone(), true); let local_renamed_body = rename_args_bodyform(defun.body.borrow()); HelperForm::Defun( *inline, @@ -379,7 +411,7 @@ pub fn rename_args_compileform(c: &CompileForm) -> CompileForm { for x in new_names.iter() { local_namemap.insert(x.0.clone(), x.1.clone()); } - let local_renamed_arg = rename_in_cons(&local_namemap, c.args.clone()); + let local_renamed_arg = rename_in_cons(&local_namemap, c.args.clone(), true); let local_renamed_helpers: Vec = c.helpers.iter().map(rename_args_helperform).collect(); let local_renamed_body = rename_args_bodyform(c.exp.borrow()); diff --git a/src/util/mod.rs b/src/util/mod.rs index 96007056d..ef44b97df 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -43,9 +43,9 @@ pub fn collapse(r: Result) -> A { #[derive(Debug, Clone)] pub struct TopoSortItem { - index: usize, - needs: HashSet, - has: HashSet + pub index: usize, + pub needs: HashSet, + pub has: HashSet } // F: tells whether t2 includes t1. @@ -69,8 +69,8 @@ where let mut finished_idx = 0; // Determine what's defined in these bindings. - for (i,item) in items.iter().enumerate() { - for new_item in possible.union(&item.has) { + for (_,item) in items.iter().enumerate() { + for new_item in item.has.iter() { possible.insert(new_item.clone()); } } @@ -79,11 +79,11 @@ where for i in 0..items.len() { items[i].needs = needs(&possible, &list[i])?; } - + while finished_idx < items.len() { // Find things with no new dependencies. - let move_to_front: Vec<(usize, TopoSortItem)> = - items.iter().enumerate().skip(finished_idx).filter(|(i,item)| { + let move_to_front: Vec<(usize, TopoSortItem)> = + items.iter().enumerate().skip(finished_idx).filter(|(_,item)| { item.needs.is_subset(&done) }).map(|(i,item)| (i, item.clone())).collect(); @@ -93,13 +93,14 @@ where } // Swap items into place earlier in the list. - for (idx,tomove) in move_to_front.iter() { + for (idx, _tomove) in move_to_front.iter() { if *idx != finished_idx { - swap(&mut items[*idx], &mut items[finished_idx]); + let mut tmp = items[*idx].clone(); + swap(&mut tmp, &mut items[finished_idx]); } // Add new 'has' items to done. - let tmp = HashSet::new(); + let mut tmp = HashSet::new(); for u in done.union(&items[finished_idx].has) { tmp.insert(u.clone()); } From 6cae84f52de0e441f9ea3dca383679b71c74d619 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 20 Dec 2022 11:35:51 -0800 Subject: [PATCH 003/117] A couple more things and it'll be good --- src/compiler/codegen.rs | 5 +- src/compiler/frontend.rs | 200 ++++++++++++++++++++++----------- src/tests/compiler/compiler.rs | 37 +++++- 3 files changed, 174 insertions(+), 68 deletions(-) diff --git a/src/compiler/codegen.rs b/src/compiler/codegen.rs index 974c9e913..985f68578 100644 --- a/src/compiler/codegen.rs +++ b/src/compiler/codegen.rs @@ -744,8 +744,8 @@ fn generate_let_defun( BindingPattern::Name(name) => { Rc::new(SExp::Atom(l.clone(), name.clone())) } - BindingPattern::Complex(_sexp) => { - todo!(); + BindingPattern::Complex(sexp) => { + sexp.clone() } } }) @@ -841,6 +841,7 @@ fn hoist_body_let_binding( })); } + eprintln!("generate let {}", body.to_sexp()); let generated_defun = generate_let_defun( compiler, letdata.loc.clone(), diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index b7a4fa209..2a8b57f6e 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -1,6 +1,7 @@ use std::borrow::Borrow; use std::collections::HashMap; use std::collections::HashSet; +use std::mem::swap; use std::rc::Rc; use crate::classic::clvm::__type_compatibility__::bi_one; @@ -236,11 +237,142 @@ fn make_let_bindings( } } +// Make a set of names in this sexp. fn make_provides_set( - _need_set: &mut HashSet>, - _body_sexp: Rc + provides_set: &mut HashSet>, + body_sexp: Rc ) { - todo!(); + match body_sexp.borrow() { + SExp::Cons(_, a, b) => { + make_provides_set(provides_set, a.clone()); + make_provides_set(provides_set, b.clone()); + }, + SExp::Atom(_, name) => { + provides_set.insert(name.clone()); + }, + _ => {} + } +} + +fn handle_assign_form( + opts: Rc, + l: Srcloc, + v: &[SExp] +) -> Result { + if v.len() % 1 == 1 { + return Err(CompileErr(l.clone(), "assign form should be in pairs of pattern value followed by an expression".to_string())); + } + + let mut bindings = Vec::new(); + for idx in (0..(v.len() - 1) / 2).map(|idx| idx * 2) { + let destructure_pattern = Rc::new(v[idx].clone()); + let binding_body = compile_bodyform(opts.clone(), Rc::new(v[idx+1].clone()))?; + bindings.push(Rc::new(Binding { + loc: v[idx].loc().ext(&v[idx+1].loc()), + nl: destructure_pattern.loc(), + pattern: BindingPattern::Complex(destructure_pattern), + body: Rc::new(binding_body), + })); + } + + // Topological sort of bindings. + let sorted_spec = toposort( + &bindings, + CompileErr(l.clone(), "deadlock resolving binding order".to_string()), + // Needs: What this binding relies on. + |possible, b| { + let mut need_set = HashSet::new(); + make_provides_set(&mut need_set, b.body.to_sexp()); + let mut need_set_thats_possible = HashSet::new(); + for need in need_set.intersection(possible) { + need_set_thats_possible.insert(need.clone()); + } + Ok(need_set_thats_possible) + }, + // Has: What this binding provides. + |b| { + match b.pattern.borrow() { + BindingPattern::Name(name) => { + HashSet::from([name.clone()]) + } + BindingPattern::Complex(sexp) => { + let mut result_set = HashSet::new(); + make_provides_set(&mut result_set, sexp.clone()); + result_set + } + } + })?; + + let compiled_body = + compile_bodyform(opts, Rc::new(v[v.len()-1].clone()))?; + // Break up into stages of parallel let forms. + // Track the needed bindings of this level. + // If this becomes broader in a way that doesn't + // match the existing provides, we need to break + // the let binding. + let mut current_provides = HashSet::new(); + let mut binding_lists = Vec::new(); + let mut this_round_bindings = Vec::new(); + let mut new_provides: HashSet> = HashSet::new(); + + for spec in sorted_spec.iter() { + let new_needs_vec: Vec> = + spec.needs.difference(¤t_provides).cloned().collect(); + if !new_needs_vec.is_empty() { + // Roll over the set we're accumulating to the finished version. + let mut empty_tmp = Vec::new(); + swap(&mut empty_tmp, &mut this_round_bindings); + binding_lists.push(empty_tmp); + for provided in new_provides.iter() { + current_provides.insert(provided.clone()); + } + new_provides.clear(); + } + // Record what we can provide to the next round. + for p in spec.has.iter() { + new_provides.insert(p.clone()); + } + this_round_bindings.push(bindings[spec.index].clone()); + } + + // Pick up the last ones that didn't add new needs. + if !this_round_bindings.is_empty() { + binding_lists.push(this_round_bindings); + } + + // We don't need to do much if there were no bindings. + if binding_lists.is_empty() { + return Ok(compiled_body); + } + + binding_lists.reverse(); + + // Spill let forms as parallel sets to get the best stack we can. + let mut end_bindings = Vec::new(); + swap(&mut end_bindings, &mut binding_lists[0]); + let mut output_let = BodyForm::Let( + LetFormKind::Parallel, + LetData { + loc: l.clone(), + kw: Some(l.clone()), + bindings: end_bindings, + body: Rc::new(compiled_body) + } + ); + + for binding_list in binding_lists.into_iter().skip(1) { + output_let = BodyForm::Let( + LetFormKind::Parallel, + LetData { + loc: l.clone(), + kw: Some(l.clone()), + bindings: binding_list, + body: Rc::new(output_let) + } + ) + } + + Ok(output_let) } pub fn compile_bodyform( @@ -312,67 +444,7 @@ pub fn compile_bodyform( }, )) } else if *atom_name == "assign".as_bytes().to_vec() { - if v.len() % 1 == 0 { - return finish_err("assign"); - } - - let mut bindings = Vec::new(); - for idx in (0..(v.len() - 1) / 2).map(|idx| 1 + idx * 2) { - let destructure_pattern = Rc::new(v[idx].clone()); - let binding_body = compile_bodyform(opts.clone(), Rc::new(v[idx+1].clone()))?; - bindings.push(Rc::new(Binding { - loc: v[idx].loc().ext(&v[idx+1].loc()), - nl: destructure_pattern.loc(), - pattern: BindingPattern::Complex(destructure_pattern), - body: Rc::new(binding_body), - })); - } - - // Topological sort of bindings. - let sorted_spec = toposort( - &bindings, - CompileErr(l.clone(), "deadlock resolving binding order".to_string()), - // Needs: What this binding relies on. - |possible, b| { - let mut need_set = HashSet::new(); - make_provides_set(&mut need_set, b.body.to_sexp()); - let mut need_set_thats_possible = HashSet::new(); - for need in need_set.intersection(possible) { - need_set_thats_possible.insert(need.clone()); - } - Ok(need_set_thats_possible) - }, - // Has: What this binding provides. - |b| { - match b.pattern.borrow() { - BindingPattern::Name(name) => { - HashSet::from([name.clone()]) - } - BindingPattern::Complex(sexp) => { - let mut result_set = HashSet::new(); - make_provides_set(&mut result_set, sexp.clone()); - result_set - } - } - })?; - - let sorted_bindings: Vec> = sorted_spec.iter().map(|item| { - bindings[item.index].clone() - }).collect(); - - let compiled_body = - compile_bodyform(opts, Rc::new(v[v.len()-1].clone()))?; - - todo!(); - Ok(BodyForm::Let( - LetFormKind::Parallel, - LetData { - loc: l.clone(), - kw: Some(l.clone()), - bindings: bindings, - body: Rc::new(compiled_body) - } - )) + handle_assign_form(opts.clone(), l.clone(), &v) } else if *atom_name == "quote".as_bytes().to_vec() { if v.len() != 1 { return finish_err("quote"); diff --git a/src/tests/compiler/compiler.rs b/src/tests/compiler/compiler.rs index ec1b342b4..798f90a61 100644 --- a/src/tests/compiler/compiler.rs +++ b/src/tests/compiler/compiler.rs @@ -1127,8 +1127,7 @@ fn arg_destructure_test_1() { (include *standard-cl-21*) delegated_puzzle_hash -)" - } +)"} .to_string(); let res = run_string(&prog, &"(1 2 3 . 4)".to_string()).unwrap(); assert_eq!(res.to_string(), "4"); @@ -1146,3 +1145,37 @@ fn test_defconstant_tree() { let res = run_string(&prog, &"()".to_string()).unwrap(); assert_eq!(res.to_string(), "((0x4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a . 0x9dcf97a184f32623d11a73124ceb99a5709b083721e878a16d78f596718ba7b2) 0x02a12871fee210fb8619291eaea194581cbd2531e4b23759d225f6806923f63222 . 0x02a8d5dd63fba471ebcb1f3e8f7c1e1879b7152a6e7298a91ce119a63400ade7c5)"); } + +#[test] +fn test_assign_form_0() { + let prog = indoc! {" +(mod (X) + (assign + X1 (+ X 1) ;; 14 + X1 + ) + )"} + .to_string(); + let res = run_string(&prog, &"(13)".to_string()).unwrap(); + assert_eq!(res.to_string(), "14"); +} + +#[test] +fn test_assign_form_1() { + let prog = indoc! {" +(mod (X) + (assign + X1 (+ X 1) ;; 14 + X2 (+ X1 1) ;; 15 + X3 (+ X2 1) ;; 16 + Y0 (+ X3 1) ;; 17 + X4 (+ X3 1) ;; 17 + X5 (+ Y0 1) ;; 18 + Y1 (+ X5 Y0) ;; 35 + Y1 + ) + )"} + .to_string(); + let res = run_string(&prog, &"(13)".to_string()).unwrap(); + assert_eq!(res.to_string(), "35"); +} From b5e6728774296e21ca5df9ed0a5f29f4cd1fc335 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 20 Dec 2022 12:35:43 -0800 Subject: [PATCH 004/117] Some tests (needs lots more) --- src/tests/compiler/compiler.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/tests/compiler/compiler.rs b/src/tests/compiler/compiler.rs index 798f90a61..4e949301e 100644 --- a/src/tests/compiler/compiler.rs +++ b/src/tests/compiler/compiler.rs @@ -1179,3 +1179,23 @@ fn test_assign_form_1() { let res = run_string(&prog, &"(13)".to_string()).unwrap(); assert_eq!(res.to_string(), "35"); } + +#[test] +fn test_assign_form_cplx_1() { + let prog = indoc! {" +(mod (X) + (defun-inline tup (X Y) (c X Y)) + (assign + (X1 . X2) (tup (+ X 1) (+ X 2)) ;; 14 + X2 (+ X1 1) ;; 15 + X3 (+ X2 1) ;; 16 + (Y0 . X4) (tup (+ X3 1) (+ X3 1)) ;; 17 + X5 (+ Y0 1) ;; 18 + Y1 (+ X5 Y0) ;; 35 + Y1 + ) + )"} + .to_string(); + let res = run_string(&prog, &"(13)".to_string()).unwrap(); + assert_eq!(res.to_string(), "35"); +} From 5e71ceb81bdd622b4fdfe63ae459954b656ac7f7 Mon Sep 17 00:00:00 2001 From: arty Date: Thu, 22 Dec 2022 14:13:06 -0800 Subject: [PATCH 005/117] Tests of assign --- src/compiler/codegen.rs | 22 +++--- src/compiler/frontend.rs | 7 +- src/compiler/sexp.rs | 8 +++ src/tests/compiler/compiler.rs | 126 +++++++++++++++++++++++++++++++++ 4 files changed, 149 insertions(+), 14 deletions(-) diff --git a/src/compiler/codegen.rs b/src/compiler/codegen.rs index 985f68578..41e459f06 100644 --- a/src/compiler/codegen.rs +++ b/src/compiler/codegen.rs @@ -382,15 +382,17 @@ fn compile_call( Rc::new(SExp::Atom(al.clone(), an.to_vec())), ) .and_then(|calltype| match calltype { - Callable::CallMacro(l, code) => process_macro_call( - allocator, - runner, - opts.clone(), - compiler, - l, - tl, - Rc::new(code), - ), + Callable::CallMacro(l, code) => { + process_macro_call( + allocator, + runner, + opts.clone(), + compiler, + l, + tl, + Rc::new(code), + ) + }, Callable::CallInline(l, inline) => replace_in_inline( allocator, @@ -840,8 +842,6 @@ fn hoist_body_let_binding( body: new_binding, })); } - - eprintln!("generate let {}", body.to_sexp()); let generated_defun = generate_let_defun( compiler, letdata.loc.clone(), diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index 2a8b57f6e..e44fbfb8a 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -216,8 +216,9 @@ fn make_let_bindings( SExp::Nil(_) => Ok(vec![]), SExp::Cons(_, head, tl) => head .proper_list() - .map(|x| match &x[..] { - [SExp::Atom(l, name), expr] => { + .filter(|x| x.len() == 2) + .map(|x| match (x[0].atomize(), &x[1]) { + (SExp::Atom(l, name), expr) => { let compiled_body = compile_bodyform(opts.clone(), Rc::new(expr.clone()))?; let mut result = Vec::new(); let mut rest_bindings = make_let_bindings(opts, tl.clone())?; @@ -242,7 +243,7 @@ fn make_provides_set( provides_set: &mut HashSet>, body_sexp: Rc ) { - match body_sexp.borrow() { + match body_sexp.atomize() { SExp::Cons(_, a, b) => { make_provides_set(provides_set, a.clone()); make_provides_set(provides_set, b.clone()); diff --git a/src/compiler/sexp.rs b/src/compiler/sexp.rs index d8ac5234c..fe1e991d9 100644 --- a/src/compiler/sexp.rs +++ b/src/compiler/sexp.rs @@ -462,6 +462,14 @@ impl SExp { } } + pub fn atomize(&self) -> SExp { + if let SExp::Integer(l, i) = self { + SExp::Atom(l.clone(), u8_from_number(i.clone())) + } else { + self.clone() + } + } + pub fn equal_to(&self, other: &SExp) -> bool { if self.nilp() && other.nilp() { true diff --git a/src/tests/compiler/compiler.rs b/src/tests/compiler/compiler.rs index 4e949301e..293d16b30 100644 --- a/src/tests/compiler/compiler.rs +++ b/src/tests/compiler/compiler.rs @@ -1199,3 +1199,129 @@ fn test_assign_form_cplx_1() { let res = run_string(&prog, &"(13)".to_string()).unwrap(); assert_eq!(res.to_string(), "35"); } + +#[test] +fn test_assign_form_in_let_binding() { + let prog = indoc! {" +(mod (X) + (defun-inline tup (X Y) (c X Y)) + (let + ((FOO + (assign + (X1 . X2) (tup (+ X 1) (+ X 2)) ;; 14 + X2 (+ X1 1) ;; 15 + X3 (+ X2 1) ;; 16 + (Y0 . X4) (tup (+ X3 1) (+ X3 1)) ;; 17 + X5 (+ Y0 1) ;; 18 + Y1 (+ X5 Y0) ;; 35 + Y1 + ))) + FOO + ) + )"} + .to_string(); + let res = run_string(&prog, &"(13)".to_string()).unwrap(); + assert_eq!(res.to_string(), "35"); +} + +#[test] +fn test_assign_form_in_function_argument() { + let prog = indoc! {" +(mod (X) + (defun-inline tup (X Y) (c X Y)) + (defun F (A B) (+ A B)) + (F + (assign + (X1 . X2) (tup (+ X 1) (+ X 2)) ;; 14 + X2 (+ X1 1) ;; 15 + X3 (+ X2 1) ;; 16 + (Y0 . X4) (tup (+ X3 1) (+ X3 1)) ;; 17 + X5 (+ Y0 1) ;; 18 + Y1 (+ X5 Y0) ;; 35 + Y1 + ) + 101 + ) + )"} + .to_string(); + let res = run_string(&prog, &"(13)".to_string()).unwrap(); + assert_eq!(res.to_string(), "136"); +} + +#[test] +fn test_assign_for_in_inline_argument() { + let prog = indoc! {" +(mod (X) + (defun-inline tup (X Y) (c X Y)) + (defun-inline F (A B) (+ A B)) + (F + (assign + (X1 . X2) (tup (+ X 1) (+ X 2)) ;; 14 + X2 (+ X1 1) ;; 15 + X3 (+ X2 1) ;; 16 + (Y0 . X4) (tup (+ X3 1) (+ X3 1)) ;; 17 + X5 (+ Y0 1) ;; 18 + Y1 (+ X5 Y0) ;; 35 + Y1 + ) + 101 + ) + )"} + .to_string(); + let res = run_string(&prog, &"(13)".to_string()).unwrap(); + assert_eq!(res.to_string(), "136"); +} + +#[test] +fn test_assign_in_if() { + let prog = indoc! {" +(mod (X) + (defun-inline tup (X Y) (c X Y)) + (defun-inline F (A B) (+ A B)) + (if X + (assign + (X1 . X2) (tup (+ X 1) (+ X 2)) ;; 14 + X2 (+ X1 1) ;; 15 + X3 (+ X2 1) ;; 16 + (Y0 . X4) (tup (+ X3 1) (+ X3 1)) ;; 17 + X5 (+ Y0 1) ;; 18 + Y1 (+ X5 Y0) ;; 35 + Y1 + ) + 101 + ) + )"} + .to_string(); + let res = run_string(&prog, &"(13)".to_string()).unwrap(); + assert_eq!(res.to_string(), "35"); +} + +#[test] +fn test_assign_fun_cplx_2() { + let prog = indoc! {" +(mod (X) + (defun-inline tup (X Y) (c X Y)) + (defun-inline F (A B) (+ A B)) + (if X + (let* + ((Z + (assign + (X1 . X2) (tup (+ X 1) (+ X 2)) ;; 14 + X2 (+ X1 1) ;; 15 + X3 (+ X2 1) ;; 16 + (Y0 . X4) (tup (+ X3 1) (+ X3 1)) ;; 17 + X5 (+ Y0 1) ;; 18 + Y1 (+ X5 Y0) ;; 35 + Y1 + )) + (Q (assign R (+ 3 2) (* R Z))) + ) + Q + ) + 101 + ) + )"} + .to_string(); + let res = run_string(&prog, &"(13)".to_string()).unwrap(); + assert_eq!(res.to_string(), "175"); +} From 52405bd3b9cbde72f337a5cd5394e0a572c92291 Mon Sep 17 00:00:00 2001 From: arty Date: Thu, 22 Dec 2022 16:52:04 -0800 Subject: [PATCH 006/117] Add more tests for assign form --- src/compiler/comptypes.rs | 2 +- src/compiler/evaluate.rs | 35 +++++++++++++++++++++++++++++++-- src/compiler/frontend.rs | 26 ++++++++++++++---------- src/compiler/rename.rs | 33 ++++++++++++++++++------------- src/tests/compiler/evaluate.rs | 36 +++++++++++++++++++++++++++++++++- src/util/mod.rs | 6 ++++-- 6 files changed, 108 insertions(+), 30 deletions(-) diff --git a/src/compiler/comptypes.rs b/src/compiler/comptypes.rs index 829efb488..c8029d86b 100644 --- a/src/compiler/comptypes.rs +++ b/src/compiler/comptypes.rs @@ -444,7 +444,7 @@ impl Binding { let pat = match &self.pattern { BindingPattern::Name(name) => { - Rc::new(SExp::atom_from_vec(self.loc.clone(), &name)) + Rc::new(SExp::atom_from_vec(self.loc.clone(), name)) } BindingPattern::Complex(sexp) => { sexp.clone() diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index b8d731ff5..5655bba55 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -48,8 +48,36 @@ fn select_helper(bindings: &[HelperForm], name: &[u8]) -> Option { None } -fn compute_paths_of_destructure(_bindings: &mut Vec<(Vec, Rc)>, _structure: Rc, _bodyform: Rc) { - todo!(); +fn compute_paths_of_destructure(bindings: &mut Vec<(Vec, Rc)>, structure: Rc, path: Number, mask: Number, bodyform: Rc) { + match structure.atomize() { + SExp::Cons(_, a, b) => { + let next_mask = mask.clone() * 2_u32.to_bigint().unwrap(); + let next_right_path = mask + path.clone(); + compute_paths_of_destructure(bindings, a.clone(), path, next_mask.clone(), bodyform.clone()); + compute_paths_of_destructure(bindings, b.clone(), next_right_path, next_mask, bodyform); + } + SExp::Atom(_, name) => { + let mut produce_path = path.clone() | mask; + let mut output_form = bodyform.clone(); + + while produce_path > bi_one() { + if path.clone() & produce_path.clone() != bi_zero() { + // Right path + output_form = + Rc::new(make_operator1(&bodyform.loc(), "r".to_string(), output_form)); + } else { + // Left path + output_form = + Rc::new(make_operator1(&bodyform.loc(), "f".to_string(), output_form)); + } + + produce_path /= 2_u32.to_bigint().unwrap(); + } + + bindings.push((name.clone(), output_form)); + } + _ => { } + } } fn update_parallel_bindings( @@ -67,6 +95,8 @@ fn update_parallel_bindings( compute_paths_of_destructure( &mut computed_getters, structure.clone(), + bi_zero(), + bi_one(), b.body.clone() ); for (name, p) in computed_getters.iter() { @@ -937,6 +967,7 @@ impl Evaluator { body: Rc, only_inline: bool, ) -> Result, CompileErr> { + eprintln!("evaluate {}", body.to_sexp()); match body.borrow() { BodyForm::Let(LetFormKind::Parallel, letdata) => { let updated_bindings = update_parallel_bindings(env, &letdata.bindings); diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index e44fbfb8a..e3ed9a4c4 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -224,7 +224,7 @@ fn make_let_bindings( let mut rest_bindings = make_let_bindings(opts, tl.clone())?; result.push(Rc::new(Binding { loc: l.clone(), - nl: l.clone(), + nl: l, pattern: BindingPattern::Name(name.to_vec()), body: Rc::new(compiled_body), })); @@ -245,11 +245,11 @@ fn make_provides_set( ) { match body_sexp.atomize() { SExp::Cons(_, a, b) => { - make_provides_set(provides_set, a.clone()); - make_provides_set(provides_set, b.clone()); + make_provides_set(provides_set, a); + make_provides_set(provides_set, b); }, SExp::Atom(_, name) => { - provides_set.insert(name.clone()); + provides_set.insert(name); }, _ => {} } @@ -260,8 +260,8 @@ fn handle_assign_form( l: Srcloc, v: &[SExp] ) -> Result { - if v.len() % 1 == 1 { - return Err(CompileErr(l.clone(), "assign form should be in pairs of pattern value followed by an expression".to_string())); + if v.len() % 2 == 0 { + return Err(CompileErr(l, "assign form should be in pairs of pattern value followed by an expression".to_string())); } let mut bindings = Vec::new(); @@ -317,12 +317,16 @@ fn handle_assign_form( let mut new_provides: HashSet> = HashSet::new(); for spec in sorted_spec.iter() { - let new_needs_vec: Vec> = - spec.needs.difference(¤t_provides).cloned().collect(); - if !new_needs_vec.is_empty() { + let mut new_needs = + spec.needs.difference(¤t_provides).cloned(); + if new_needs.next().is_some() { + eprintln!("bindings"); // Roll over the set we're accumulating to the finished version. - let mut empty_tmp = Vec::new(); + let mut empty_tmp: Vec> = Vec::new(); swap(&mut empty_tmp, &mut this_round_bindings); + for e in empty_tmp.iter() { + eprintln!("push bindings {}", e.to_sexp()); + } binding_lists.push(empty_tmp); for provided in new_provides.iter() { current_provides.insert(provided.clone()); @@ -351,6 +355,7 @@ fn handle_assign_form( // Spill let forms as parallel sets to get the best stack we can. let mut end_bindings = Vec::new(); swap(&mut end_bindings, &mut binding_lists[0]); + let mut output_let = BodyForm::Let( LetFormKind::Parallel, LetData { @@ -373,6 +378,7 @@ fn handle_assign_form( ) } + eprintln!("output let from assign {}", output_let.to_sexp()); Ok(output_let) } diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index d825ea6de..ab2ecab39 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -108,21 +108,27 @@ fn invent_new_names_sexp(body: Rc) -> Vec<(Vec, Vec)> { } } -fn make_binding_unique(b: &Binding) -> (HashMap, Vec>, Binding) { +#[derive(Debug, Clone)] +struct InnerRenameList { + bindings: HashMap, Vec>, + from_wing: Binding +} + +fn make_binding_unique(b: &Binding) -> InnerRenameList { match b.pattern.borrow() { BindingPattern::Name(name) => { let mut single_name_map = HashMap::new(); let new_name = gensym(name.clone()); single_name_map.insert(name.to_vec(), new_name.clone()); - ( - single_name_map, - Binding { + InnerRenameList { + bindings: single_name_map, + from_wing: Binding { loc: b.loc.clone(), nl: b.nl.clone(), pattern: BindingPattern::Name(new_name), body: b.body.clone(), } - ) + } } BindingPattern::Complex(pat) => { let new_names_vec = invent_new_names_sexp(pat.clone()); @@ -133,15 +139,15 @@ fn make_binding_unique(b: &Binding) -> (HashMap, Vec>, Binding) { } let renamed_pattern = rename_in_cons(&new_names, pat.clone(), false); - ( - new_names, - Binding { + InnerRenameList { + bindings: new_names, + from_wing: Binding { loc: b.loc.clone(), nl: b.nl.clone(), pattern: BindingPattern::Complex(renamed_pattern), body: b.body.clone(), } - ) + } } } } @@ -239,17 +245,16 @@ fn rename_args_bodyform(b: &BodyForm) -> BodyForm { } BodyForm::Let(LetFormKind::Parallel, letdata) => { - let renames: Vec<(HashMap, Vec>, Binding)> = letdata + let renames: Vec = letdata .bindings .iter() .map(|x| make_binding_unique(x.borrow())) .collect(); let new_renamed_bindings: Vec> = - renames.iter().map(|(_, x)| Rc::new(x.clone())).collect(); + renames.iter().map(|ir| Rc::new(ir.from_wing.clone())).collect(); let mut local_namemap = HashMap::new(); - for x in renames.iter() { - let (oldnames, _) = x; - for (oldname, binding_name) in oldnames.iter() { + for ir in renames.iter() { + for (oldname, binding_name) in ir.bindings.iter() { local_namemap.insert(oldname.to_vec(), binding_name.clone()); } } diff --git a/src/tests/compiler/evaluate.rs b/src/tests/compiler/evaluate.rs index 8fc33d5c9..f48350407 100644 --- a/src/tests/compiler/evaluate.rs +++ b/src/tests/compiler/evaluate.rs @@ -91,7 +91,7 @@ fn test_basic_expand_macro_4() { shrink_expr_from_string( "(mod () (defun torp (S A B) (if S (+ A B) (* A B))) (torp 0 2 3))".to_string() ) - .unwrap(), + .unwrap(), "(q . 6)" ); } @@ -120,3 +120,37 @@ fn test_simple_fe_opt_compile_1() { "(2 (1 1 . 99) (4 (1) 1))".to_string() ); } + +#[test] +fn test_assign_simple_form_0() { + assert_eq!( + shrink_expr_from_string( + "(assign A (* Z 3) X 3 Y 4 Z (+ X Y) A)".to_string() + ) + .unwrap(), + "(q . 21)" + ); +} + +#[test] +fn test_assign_simple_form_1() { + assert_eq!( + shrink_expr_from_string( + "(assign A (* Z 3) Z 2 A)".to_string() + ) + .unwrap(), + "(q . 6)" + ); + +} + +#[test] +fn test_assign_simple_form_2() { + assert_eq!( + shrink_expr_from_string( + "(assign Z 2 A (* Z 3) A)".to_string() + ) + .unwrap(), + "(q . 6)" + ); +} diff --git a/src/util/mod.rs b/src/util/mod.rs index ef44b97df..9a23b6dcc 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -49,7 +49,7 @@ pub struct TopoSortItem { } // F: tells whether t2 includes t1. -pub fn toposort(list: &Vec, deadlock: E, needs: Needs, has: Has) -> Result>, E> +pub fn toposort(list: &[T], deadlock: E, needs: Needs, has: Has) -> Result>, E> where Needs: Fn(&HashSet, &T) -> Result, E>, Has: Fn(&T) -> HashSet, @@ -97,6 +97,7 @@ where if *idx != finished_idx { let mut tmp = items[*idx].clone(); swap(&mut tmp, &mut items[finished_idx]); + items[*idx] = tmp; } // Add new 'has' items to done. @@ -109,9 +110,10 @@ where for i in intersection { done.insert(i.clone()); } + finished_idx += 1; } } - return Ok(items); + Ok(items) } From 23b12d8956c7e1db606b1f659b04bbe982c34f87 Mon Sep 17 00:00:00 2001 From: arty Date: Thu, 22 Dec 2022 17:19:45 -0800 Subject: [PATCH 007/117] Add some basic toposort tests --- src/tests/mod.rs | 1 + src/tests/util.rs | 95 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 src/tests/util.rs diff --git a/src/tests/mod.rs b/src/tests/mod.rs index 032619a9d..ed0d311e9 100644 --- a/src/tests/mod.rs +++ b/src/tests/mod.rs @@ -1,2 +1,3 @@ mod classic; mod compiler; +mod util; diff --git a/src/tests/util.rs b/src/tests/util.rs new file mode 100644 index 000000000..055314a7c --- /dev/null +++ b/src/tests/util.rs @@ -0,0 +1,95 @@ +use std::collections::HashSet; +use crate::util::toposort; + +#[derive(Debug, Clone)] +struct TopoSortCheckItem { + needs: HashSet, + has: HashSet +} + +impl TopoSortCheckItem { + pub fn new(needs_s: &[&str], has_s: &[&str]) -> Self { + let mut needs: HashSet = HashSet::new(); + let mut has: HashSet = HashSet::new(); + for n in needs_s.iter() { + needs.insert(n.to_string()); + } + for h in has_s.iter() { + has.insert(h.to_string()); + } + TopoSortCheckItem { needs, has } + } +} + +// Given +// +// A B -> X +// X Y -> Z +// A W -> Y +// [] -> A +// [] -> B +// B -> W +// +// We should get something like this: +// +// [] -> A +// [] -> B +// B -> W +// A B -> X +// A W -> Y +// X Y -> Z +// +#[test] +fn test_topo_sort_0() { + let t = |n, h| TopoSortCheckItem::new(n, h); + let items = vec![ + t(&["A", "B"], &["X"]), + t(&["X", "Y"], &["Z"]), + t(&["A", "W"], &["Y"]), + t(&[], &["A"]), + t(&[], &["B"]), + t(&["B"], &["W"]) + ]; + let result = toposort( + &items, + true, + |_p, n: &TopoSortCheckItem| Ok(n.needs.clone()), + |h| h.has.clone() + ).expect("no deadlocks in this data"); + + for (i, item) in result.iter().enumerate() { + let have_item = &items[item.index]; + for j in 0..i { + let item_to_check = &result[j]; + let item_to_check_for_dependencies_on_have = + &items[item_to_check.index]; + // item_to_check_for_dependencies is an item occurring prior to + // have_item in the sorted output. + // If its 'needs' has anything in have_item's 'has', then we failed. + let mut intersection = + item_to_check_for_dependencies_on_have.needs.intersection(&have_item.has); + assert!(intersection.next().is_none()); + } + } +} + +#[test] +fn test_topo_sort_1() { + let t = |n, h| TopoSortCheckItem::new(n, h); + let items = vec![ + t(&["A", "B"], &["X"]), + t(&["X", "Y"], &["Z"]), + t(&["A", "W"], &["Y"]), + t(&[], &["A"]), + t(&["Z"], &["B"]), + t(&["B"], &["W"]) + ]; + let result = toposort( + &items, + true, + |_p, n: &TopoSortCheckItem| Ok(n.needs.clone()), + |h| h.has.clone() + ); + + assert!(result.is_err()); +} From b0aa0bf1df51248efb15b816cf901f5c4ace2ead Mon Sep 17 00:00:00 2001 From: arty Date: Thu, 22 Dec 2022 17:22:57 -0800 Subject: [PATCH 008/117] fmt + clippy --- src/compiler/codegen.rs | 38 +++++++++-------------- src/compiler/comptypes.rs | 15 +++------ src/compiler/evaluate.rs | 41 +++++++++++++++++------- src/compiler/frontend.rs | 57 ++++++++++++++++------------------ src/compiler/rename.rs | 18 ++++++----- src/tests/compiler/evaluate.rs | 18 +++-------- src/tests/util.rs | 23 +++++++------- src/util/mod.rs | 44 ++++++++++++++++---------- 8 files changed, 129 insertions(+), 125 deletions(-) diff --git a/src/compiler/codegen.rs b/src/compiler/codegen.rs index 41e459f06..d112ce9d8 100644 --- a/src/compiler/codegen.rs +++ b/src/compiler/codegen.rs @@ -13,9 +13,9 @@ use crate::classic::clvm::__type_compatibility__::bi_one; use crate::compiler::clvm::run; use crate::compiler::compiler::{is_at_capture, run_optimizer}; use crate::compiler::comptypes::{ - fold_m, join_vecs_to_string, list_to_cons, Binding, BindingPattern, BodyForm, Callable, CompileErr, - CompileForm, CompiledCode, CompilerOpts, DefunCall, DefunData, HelperForm, InlineFunction, - LetData, LetFormKind, PrimaryCodegen, + fold_m, join_vecs_to_string, list_to_cons, Binding, BindingPattern, BodyForm, Callable, + CompileErr, CompileForm, CompiledCode, CompilerOpts, DefunCall, DefunData, HelperForm, + InlineFunction, LetData, LetFormKind, PrimaryCodegen, }; use crate::compiler::debug::{build_swap_table_mut, relabel}; use crate::compiler::frontend::compile_bodyform; @@ -382,17 +382,15 @@ fn compile_call( Rc::new(SExp::Atom(al.clone(), an.to_vec())), ) .and_then(|calltype| match calltype { - Callable::CallMacro(l, code) => { - process_macro_call( - allocator, - runner, - opts.clone(), - compiler, - l, - tl, - Rc::new(code), - ) - }, + Callable::CallMacro(l, code) => process_macro_call( + allocator, + runner, + opts.clone(), + compiler, + l, + tl, + Rc::new(code), + ), Callable::CallInline(l, inline) => replace_in_inline( allocator, @@ -741,15 +739,9 @@ fn generate_let_defun( ) -> HelperForm { let new_arguments: Vec> = bindings .iter() - .map(|b| { - match b.pattern.borrow() { - BindingPattern::Name(name) => { - Rc::new(SExp::Atom(l.clone(), name.clone())) - } - BindingPattern::Complex(sexp) => { - sexp.clone() - } - } + .map(|b| match b.pattern.borrow() { + BindingPattern::Name(name) => Rc::new(SExp::Atom(l.clone(), name.clone())), + BindingPattern::Complex(sexp) => sexp.clone(), }) .collect(); diff --git a/src/compiler/comptypes.rs b/src/compiler/comptypes.rs index c8029d86b..1780fa0a3 100644 --- a/src/compiler/comptypes.rs +++ b/src/compiler/comptypes.rs @@ -60,7 +60,7 @@ pub fn list_to_cons(l: Srcloc, list: &[Rc]) -> SExp { #[derive(Clone, Debug)] pub enum BindingPattern { Name(Vec), - Complex(Rc) + Complex(Rc), } #[derive(Clone, Debug)] @@ -441,15 +441,10 @@ impl BodyForm { impl Binding { pub fn to_sexp(&self) -> Rc { - let pat = - match &self.pattern { - BindingPattern::Name(name) => { - Rc::new(SExp::atom_from_vec(self.loc.clone(), name)) - } - BindingPattern::Complex(sexp) => { - sexp.clone() - } - }; + let pat = match &self.pattern { + BindingPattern::Name(name) => Rc::new(SExp::atom_from_vec(self.loc.clone(), name)), + BindingPattern::Complex(sexp) => sexp.clone(), + }; Rc::new(SExp::Cons( self.loc.clone(), pat, diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index 5655bba55..c2b77bc87 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -13,7 +13,8 @@ use crate::compiler::clvm::run; use crate::compiler::codegen::codegen; use crate::compiler::compiler::is_at_capture; use crate::compiler::comptypes::{ - Binding, BindingPattern, BodyForm, CompileErr, CompileForm, CompilerOpts, HelperForm, LetData, LetFormKind, + Binding, BindingPattern, BodyForm, CompileErr, CompileForm, CompilerOpts, HelperForm, LetData, + LetFormKind, }; use crate::compiler::frontend::frontend; use crate::compiler::runtypes::RunFailure; @@ -48,13 +49,25 @@ fn select_helper(bindings: &[HelperForm], name: &[u8]) -> Option { None } -fn compute_paths_of_destructure(bindings: &mut Vec<(Vec, Rc)>, structure: Rc, path: Number, mask: Number, bodyform: Rc) { +fn compute_paths_of_destructure( + bindings: &mut Vec<(Vec, Rc)>, + structure: Rc, + path: Number, + mask: Number, + bodyform: Rc, +) { match structure.atomize() { SExp::Cons(_, a, b) => { let next_mask = mask.clone() * 2_u32.to_bigint().unwrap(); let next_right_path = mask + path.clone(); - compute_paths_of_destructure(bindings, a.clone(), path, next_mask.clone(), bodyform.clone()); - compute_paths_of_destructure(bindings, b.clone(), next_right_path, next_mask, bodyform); + compute_paths_of_destructure( + bindings, + a, + path, + next_mask.clone(), + bodyform.clone(), + ); + compute_paths_of_destructure(bindings, b, next_right_path, next_mask, bodyform); } SExp::Atom(_, name) => { let mut produce_path = path.clone() | mask; @@ -63,20 +76,26 @@ fn compute_paths_of_destructure(bindings: &mut Vec<(Vec, Rc)>, str while produce_path > bi_one() { if path.clone() & produce_path.clone() != bi_zero() { // Right path - output_form = - Rc::new(make_operator1(&bodyform.loc(), "r".to_string(), output_form)); + output_form = Rc::new(make_operator1( + &bodyform.loc(), + "r".to_string(), + output_form, + )); } else { // Left path - output_form = - Rc::new(make_operator1(&bodyform.loc(), "f".to_string(), output_form)); + output_form = Rc::new(make_operator1( + &bodyform.loc(), + "f".to_string(), + output_form, + )); } produce_path /= 2_u32.to_bigint().unwrap(); } - bindings.push((name.clone(), output_form)); + bindings.push((name, output_form)); } - _ => { } + _ => {} } } @@ -97,7 +116,7 @@ fn update_parallel_bindings( structure.clone(), bi_zero(), bi_one(), - b.body.clone() + b.body.clone(), ); for (name, p) in computed_getters.iter() { new_bindings.insert(name.clone(), p.clone()); diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index e3ed9a4c4..8b6eacf14 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -6,8 +6,8 @@ use std::rc::Rc; use crate::classic::clvm::__type_compatibility__::bi_one; use crate::compiler::comptypes::{ - list_to_cons, Binding, BindingPattern, BodyForm, CompileErr, CompileForm, CompilerOpts, DefconstData, - DefmacData, DefunData, HelperForm, IncludeDesc, LetData, LetFormKind, ModAccum, + list_to_cons, Binding, BindingPattern, BodyForm, CompileErr, CompileForm, CompilerOpts, + DefconstData, DefmacData, DefunData, HelperForm, IncludeDesc, LetData, LetFormKind, ModAccum, }; use crate::compiler::preprocessor::preprocess; use crate::compiler::rename::rename_children_compileform; @@ -239,18 +239,15 @@ fn make_let_bindings( } // Make a set of names in this sexp. -fn make_provides_set( - provides_set: &mut HashSet>, - body_sexp: Rc -) { +fn make_provides_set(provides_set: &mut HashSet>, body_sexp: Rc) { match body_sexp.atomize() { SExp::Cons(_, a, b) => { make_provides_set(provides_set, a); make_provides_set(provides_set, b); - }, + } SExp::Atom(_, name) => { provides_set.insert(name); - }, + } _ => {} } } @@ -258,18 +255,21 @@ fn make_provides_set( fn handle_assign_form( opts: Rc, l: Srcloc, - v: &[SExp] + v: &[SExp], ) -> Result { if v.len() % 2 == 0 { - return Err(CompileErr(l, "assign form should be in pairs of pattern value followed by an expression".to_string())); + return Err(CompileErr( + l, + "assign form should be in pairs of pattern value followed by an expression".to_string(), + )); } let mut bindings = Vec::new(); for idx in (0..(v.len() - 1) / 2).map(|idx| idx * 2) { let destructure_pattern = Rc::new(v[idx].clone()); - let binding_body = compile_bodyform(opts.clone(), Rc::new(v[idx+1].clone()))?; + let binding_body = compile_bodyform(opts.clone(), Rc::new(v[idx + 1].clone()))?; bindings.push(Rc::new(Binding { - loc: v[idx].loc().ext(&v[idx+1].loc()), + loc: v[idx].loc().ext(&v[idx + 1].loc()), nl: destructure_pattern.loc(), pattern: BindingPattern::Complex(destructure_pattern), body: Rc::new(binding_body), @@ -291,21 +291,17 @@ fn handle_assign_form( Ok(need_set_thats_possible) }, // Has: What this binding provides. - |b| { - match b.pattern.borrow() { - BindingPattern::Name(name) => { - HashSet::from([name.clone()]) - } - BindingPattern::Complex(sexp) => { - let mut result_set = HashSet::new(); - make_provides_set(&mut result_set, sexp.clone()); - result_set - } + |b| match b.pattern.borrow() { + BindingPattern::Name(name) => HashSet::from([name.clone()]), + BindingPattern::Complex(sexp) => { + let mut result_set = HashSet::new(); + make_provides_set(&mut result_set, sexp.clone()); + result_set } - })?; + }, + )?; - let compiled_body = - compile_bodyform(opts, Rc::new(v[v.len()-1].clone()))?; + let compiled_body = compile_bodyform(opts, Rc::new(v[v.len() - 1].clone()))?; // Break up into stages of parallel let forms. // Track the needed bindings of this level. // If this becomes broader in a way that doesn't @@ -317,8 +313,7 @@ fn handle_assign_form( let mut new_provides: HashSet> = HashSet::new(); for spec in sorted_spec.iter() { - let mut new_needs = - spec.needs.difference(¤t_provides).cloned(); + let mut new_needs = spec.needs.difference(¤t_provides).cloned(); if new_needs.next().is_some() { eprintln!("bindings"); // Roll over the set we're accumulating to the finished version. @@ -362,8 +357,8 @@ fn handle_assign_form( loc: l.clone(), kw: Some(l.clone()), bindings: end_bindings, - body: Rc::new(compiled_body) - } + body: Rc::new(compiled_body), + }, ); for binding_list in binding_lists.into_iter().skip(1) { @@ -373,8 +368,8 @@ fn handle_assign_form( loc: l.clone(), kw: Some(l.clone()), bindings: binding_list, - body: Rc::new(output_let) - } + body: Rc::new(output_let), + }, ) } diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index ab2ecab39..7a1691eb6 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -3,8 +3,8 @@ use std::collections::HashMap; use std::rc::Rc; use crate::compiler::comptypes::{ - Binding, BindingPattern, BodyForm, CompileForm, DefconstData, DefmacData, DefunData, HelperForm, LetData, - LetFormKind, + Binding, BindingPattern, BodyForm, CompileForm, DefconstData, DefmacData, DefunData, + HelperForm, LetData, LetFormKind, }; use crate::compiler::gensym::gensym; use crate::compiler::sexp::SExp; @@ -34,7 +34,7 @@ fn rename_in_qq(namemap: &HashMap, Vec>, body: Rc) -> Rc fn rename_in_cons( namemap: &HashMap, Vec>, body: Rc, - qq_handling: bool + qq_handling: bool, ) -> Rc { match body.borrow() { SExp::Atom(l, name) => match namemap.get(name) { @@ -111,7 +111,7 @@ fn invent_new_names_sexp(body: Rc) -> Vec<(Vec, Vec)> { #[derive(Debug, Clone)] struct InnerRenameList { bindings: HashMap, Vec>, - from_wing: Binding + from_wing: Binding, } fn make_binding_unique(b: &Binding) -> InnerRenameList { @@ -127,7 +127,7 @@ fn make_binding_unique(b: &Binding) -> InnerRenameList { nl: b.nl.clone(), pattern: BindingPattern::Name(new_name), body: b.body.clone(), - } + }, } } BindingPattern::Complex(pat) => { @@ -146,7 +146,7 @@ fn make_binding_unique(b: &Binding) -> InnerRenameList { nl: b.nl.clone(), pattern: BindingPattern::Complex(renamed_pattern), body: b.body.clone(), - } + }, } } } @@ -250,8 +250,10 @@ fn rename_args_bodyform(b: &BodyForm) -> BodyForm { .iter() .map(|x| make_binding_unique(x.borrow())) .collect(); - let new_renamed_bindings: Vec> = - renames.iter().map(|ir| Rc::new(ir.from_wing.clone())).collect(); + let new_renamed_bindings: Vec> = renames + .iter() + .map(|ir| Rc::new(ir.from_wing.clone())) + .collect(); let mut local_namemap = HashMap::new(); for ir in renames.iter() { for (oldname, binding_name) in ir.bindings.iter() { diff --git a/src/tests/compiler/evaluate.rs b/src/tests/compiler/evaluate.rs index f48350407..feccb5a9d 100644 --- a/src/tests/compiler/evaluate.rs +++ b/src/tests/compiler/evaluate.rs @@ -91,7 +91,7 @@ fn test_basic_expand_macro_4() { shrink_expr_from_string( "(mod () (defun torp (S A B) (if S (+ A B) (* A B))) (torp 0 2 3))".to_string() ) - .unwrap(), + .unwrap(), "(q . 6)" ); } @@ -124,10 +124,7 @@ fn test_simple_fe_opt_compile_1() { #[test] fn test_assign_simple_form_0() { assert_eq!( - shrink_expr_from_string( - "(assign A (* Z 3) X 3 Y 4 Z (+ X Y) A)".to_string() - ) - .unwrap(), + shrink_expr_from_string("(assign A (* Z 3) X 3 Y 4 Z (+ X Y) A)".to_string()).unwrap(), "(q . 21)" ); } @@ -135,22 +132,15 @@ fn test_assign_simple_form_0() { #[test] fn test_assign_simple_form_1() { assert_eq!( - shrink_expr_from_string( - "(assign A (* Z 3) Z 2 A)".to_string() - ) - .unwrap(), + shrink_expr_from_string("(assign A (* Z 3) Z 2 A)".to_string()).unwrap(), "(q . 6)" ); - } #[test] fn test_assign_simple_form_2() { assert_eq!( - shrink_expr_from_string( - "(assign Z 2 A (* Z 3) A)".to_string() - ) - .unwrap(), + shrink_expr_from_string("(assign Z 2 A (* Z 3) A)".to_string()).unwrap(), "(q . 6)" ); } diff --git a/src/tests/util.rs b/src/tests/util.rs index 055314a7c..5585a2ba0 100644 --- a/src/tests/util.rs +++ b/src/tests/util.rs @@ -1,10 +1,10 @@ -use std::collections::HashSet; use crate::util::toposort; +use std::collections::HashSet; #[derive(Debug, Clone)] struct TopoSortCheckItem { needs: HashSet, - has: HashSet + has: HashSet, } impl TopoSortCheckItem { @@ -48,26 +48,27 @@ fn test_topo_sort_0() { t(&["A", "W"], &["Y"]), t(&[], &["A"]), t(&[], &["B"]), - t(&["B"], &["W"]) + t(&["B"], &["W"]), ]; let result = toposort( &items, true, |_p, n: &TopoSortCheckItem| Ok(n.needs.clone()), - |h| h.has.clone() - ).expect("no deadlocks in this data"); + |h| h.has.clone(), + ) + .expect("no deadlocks in this data"); for (i, item) in result.iter().enumerate() { let have_item = &items[item.index]; for j in 0..i { let item_to_check = &result[j]; - let item_to_check_for_dependencies_on_have = - &items[item_to_check.index]; + let item_to_check_for_dependencies_on_have = &items[item_to_check.index]; // item_to_check_for_dependencies is an item occurring prior to // have_item in the sorted output. // If its 'needs' has anything in have_item's 'has', then we failed. - let mut intersection = - item_to_check_for_dependencies_on_have.needs.intersection(&have_item.has); + let mut intersection = item_to_check_for_dependencies_on_have + .needs + .intersection(&have_item.has); assert!(intersection.next().is_none()); } } @@ -82,13 +83,13 @@ fn test_topo_sort_1() { t(&["A", "W"], &["Y"]), t(&[], &["A"]), t(&["Z"], &["B"]), - t(&["B"], &["W"]) + t(&["B"], &["W"]), ]; let result = toposort( &items, true, |_p, n: &TopoSortCheckItem| Ok(n.needs.clone()), - |h| h.has.clone() + |h| h.has.clone(), ); assert!(result.is_err()); diff --git a/src/util/mod.rs b/src/util/mod.rs index 9a23b6dcc..26d1a21f3 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -45,31 +45,38 @@ pub fn collapse(r: Result) -> A { pub struct TopoSortItem { pub index: usize, pub needs: HashSet, - pub has: HashSet + pub has: HashSet, } // F: tells whether t2 includes t1. -pub fn toposort(list: &[T], deadlock: E, needs: Needs, has: Has) -> Result>, E> +pub fn toposort( + list: &[T], + deadlock: E, + needs: Needs, + has: Has, +) -> Result>, E> where - Needs: Fn(&HashSet, &T) -> Result, E>, - Has: Fn(&T) -> HashSet, - K: std::cmp::Eq, - K: std::hash::Hash, - K: Clone + Needs: Fn(&HashSet, &T) -> Result, E>, + Has: Fn(&T) -> HashSet, + K: std::cmp::Eq, + K: std::hash::Hash, + K: Clone, { let mut possible = HashSet::new(); let mut done = HashSet::new(); - let mut items: Vec> = list.iter().enumerate().map(|(i,item)| { - TopoSortItem { + let mut items: Vec> = list + .iter() + .enumerate() + .map(|(i, item)| TopoSortItem { index: i, needs: HashSet::new(), - has: has(item) - } - }).collect(); + has: has(item), + }) + .collect(); let mut finished_idx = 0; // Determine what's defined in these bindings. - for (_,item) in items.iter().enumerate() { + for (_, item) in items.iter().enumerate() { for new_item in item.has.iter() { possible.insert(new_item.clone()); } @@ -82,10 +89,13 @@ where while finished_idx < items.len() { // Find things with no new dependencies. - let move_to_front: Vec<(usize, TopoSortItem)> = - items.iter().enumerate().skip(finished_idx).filter(|(_,item)| { - item.needs.is_subset(&done) - }).map(|(i,item)| (i, item.clone())).collect(); + let move_to_front: Vec<(usize, TopoSortItem)> = items + .iter() + .enumerate() + .skip(finished_idx) + .filter(|(_, item)| item.needs.is_subset(&done)) + .map(|(i, item)| (i, item.clone())) + .collect(); if move_to_front.is_empty() { // Circular dependency somewhere. From 04cc061fe1cc22404f4c463988c7eed70ebb02e4 Mon Sep 17 00:00:00 2001 From: arty Date: Thu, 22 Dec 2022 17:28:55 -0800 Subject: [PATCH 009/117] fmt --- src/compiler/evaluate.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index c2b77bc87..e23e1e624 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -60,13 +60,7 @@ fn compute_paths_of_destructure( SExp::Cons(_, a, b) => { let next_mask = mask.clone() * 2_u32.to_bigint().unwrap(); let next_right_path = mask + path.clone(); - compute_paths_of_destructure( - bindings, - a, - path, - next_mask.clone(), - bodyform.clone(), - ); + compute_paths_of_destructure(bindings, a, path, next_mask.clone(), bodyform.clone()); compute_paths_of_destructure(bindings, b, next_right_path, next_mask, bodyform); } SExp::Atom(_, name) => { From 227ec90cbe06b287389258d6541c15c1e5839159 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 3 Jan 2023 13:30:12 -0800 Subject: [PATCH 010/117] Clean up spam --- src/compiler/evaluate.rs | 1 - src/compiler/frontend.rs | 5 ----- 2 files changed, 6 deletions(-) diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index e23e1e624..76decea05 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -980,7 +980,6 @@ impl Evaluator { body: Rc, only_inline: bool, ) -> Result, CompileErr> { - eprintln!("evaluate {}", body.to_sexp()); match body.borrow() { BodyForm::Let(LetFormKind::Parallel, letdata) => { let updated_bindings = update_parallel_bindings(env, &letdata.bindings); diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index 8b6eacf14..2892d6d59 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -315,13 +315,9 @@ fn handle_assign_form( for spec in sorted_spec.iter() { let mut new_needs = spec.needs.difference(¤t_provides).cloned(); if new_needs.next().is_some() { - eprintln!("bindings"); // Roll over the set we're accumulating to the finished version. let mut empty_tmp: Vec> = Vec::new(); swap(&mut empty_tmp, &mut this_round_bindings); - for e in empty_tmp.iter() { - eprintln!("push bindings {}", e.to_sexp()); - } binding_lists.push(empty_tmp); for provided in new_provides.iter() { current_provides.insert(provided.clone()); @@ -373,7 +369,6 @@ fn handle_assign_form( ) } - eprintln!("output let from assign {}", output_let.to_sexp()); Ok(output_let) } From 84b1e70b45aff2dd2aa45fd2dc5ecd5c30c74c23 Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 4 Jan 2023 08:15:19 -0800 Subject: [PATCH 011/117] Add duplicate binding check --- src/compiler/frontend.rs | 20 +++++- src/tests/compiler/compiler.rs | 122 ++++++++++++++++++++++++--------- 2 files changed, 110 insertions(+), 32 deletions(-) diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index a17eba0c9..2d0279a10 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -11,7 +11,7 @@ use crate::compiler::comptypes::{ }; use crate::compiler::preprocessor::preprocess; use crate::compiler::rename::rename_children_compileform; -use crate::compiler::sexp::{enlist, SExp}; +use crate::compiler::sexp::{decode_string, enlist, SExp}; use crate::compiler::srcloc::Srcloc; use crate::util::{toposort, u8_from_number}; @@ -271,9 +271,27 @@ fn handle_assign_form( } let mut bindings = Vec::new(); + let mut check_duplicates = HashSet::new(); + for idx in (0..(v.len() - 1) / 2).map(|idx| idx * 2) { let destructure_pattern = Rc::new(v[idx].clone()); let binding_body = compile_bodyform(opts.clone(), Rc::new(v[idx + 1].clone()))?; + + // Ensure bindings aren't duplicated as we won't be able to guarantee their + // order during toposort. + let mut this_provides = HashSet::new(); + make_provides_set(&mut this_provides, destructure_pattern.clone()); + + for item in this_provides.iter() { + if check_duplicates.contains(item) { + return Err(CompileErr( + v[idx].loc(), + format!("Duplicate binding {}", decode_string(item)), + )); + } + check_duplicates.insert(item.clone()); + } + bindings.push(Rc::new(Binding { loc: v[idx].loc().ext(&v[idx + 1].loc()), nl: destructure_pattern.loc(), diff --git a/src/tests/compiler/compiler.rs b/src/tests/compiler/compiler.rs index c81cb698d..b21a6a655 100644 --- a/src/tests/compiler/compiler.rs +++ b/src/tests/compiler/compiler.rs @@ -1187,11 +1187,11 @@ fn test_assign_form_cplx_1() { (defun-inline tup (X Y) (c X Y)) (assign (X1 . X2) (tup (+ X 1) (+ X 2)) ;; 14 - X2 (+ X1 1) ;; 15 - X3 (+ X2 1) ;; 16 - (Y0 . X4) (tup (+ X3 1) (+ X3 1)) ;; 17 - X5 (+ Y0 1) ;; 18 - Y1 (+ X5 Y0) ;; 35 + X3 (+ X1 1) ;; 15 + X4 (+ X2 1) ;; 16 + (Y0 . X5) (tup (+ X4 1) (+ X4 1)) ;; 17 + X6 (+ Y0 1) ;; 18 + Y1 (+ X6 Y0) ;; 35 Y1 ) )"} @@ -1209,11 +1209,11 @@ fn test_assign_form_in_let_binding() { ((FOO (assign (X1 . X2) (tup (+ X 1) (+ X 2)) ;; 14 - X2 (+ X1 1) ;; 15 - X3 (+ X2 1) ;; 16 - (Y0 . X4) (tup (+ X3 1) (+ X3 1)) ;; 17 - X5 (+ Y0 1) ;; 18 - Y1 (+ X5 Y0) ;; 35 + X3 (+ X1 1) ;; 15 + X4 (+ X2 1) ;; 16 + (Y0 . X5) (tup (+ X4 1) (+ X4 1)) ;; 17 + X6 (+ Y0 1) ;; 18 + Y1 (+ X6 Y0) ;; 35 Y1 ))) FOO @@ -1233,11 +1233,11 @@ fn test_assign_form_in_function_argument() { (F (assign (X1 . X2) (tup (+ X 1) (+ X 2)) ;; 14 - X2 (+ X1 1) ;; 15 - X3 (+ X2 1) ;; 16 - (Y0 . X4) (tup (+ X3 1) (+ X3 1)) ;; 17 - X5 (+ Y0 1) ;; 18 - Y1 (+ X5 Y0) ;; 35 + X3 (+ X1 1) ;; 15 + X4 (+ X2 1) ;; 16 + (Y0 . X5) (tup (+ X4 1) (+ X4 1)) ;; 17 + X6 (+ Y0 1) ;; 18 + Y1 (+ X6 Y0) ;; 35 Y1 ) 101 @@ -1249,7 +1249,7 @@ fn test_assign_form_in_function_argument() { } #[test] -fn test_assign_for_in_inline_argument() { +fn test_assign_form_in_inline_argument() { let prog = indoc! {" (mod (X) (defun-inline tup (X Y) (c X Y)) @@ -1257,11 +1257,11 @@ fn test_assign_for_in_inline_argument() { (F (assign (X1 . X2) (tup (+ X 1) (+ X 2)) ;; 14 - X2 (+ X1 1) ;; 15 - X3 (+ X2 1) ;; 16 - (Y0 . X4) (tup (+ X3 1) (+ X3 1)) ;; 17 - X5 (+ Y0 1) ;; 18 - Y1 (+ X5 Y0) ;; 35 + X3 (+ X1 1) ;; 15 + X4 (+ X2 1) ;; 16 + (Y0 . X5) (tup (+ X4 1) (+ X4 1)) ;; 17 + X6 (+ Y0 1) ;; 18 + Y1 (+ X6 Y0) ;; 35 Y1 ) 101 @@ -1281,11 +1281,11 @@ fn test_assign_in_if() { (if X (assign (X1 . X2) (tup (+ X 1) (+ X 2)) ;; 14 - X2 (+ X1 1) ;; 15 - X3 (+ X2 1) ;; 16 - (Y0 . X4) (tup (+ X3 1) (+ X3 1)) ;; 17 - X5 (+ Y0 1) ;; 18 - Y1 (+ X5 Y0) ;; 35 + X3 (+ X1 1) ;; 15 + X4 (+ X3 1) ;; 16 + (Y0 . X5) (tup (+ X4 1) (+ X4 1)) ;; 17 + X6 (+ Y0 1) ;; 18 + Y1 (+ X6 Y0) ;; 35 Y1 ) 101 @@ -1307,11 +1307,11 @@ fn test_assign_fun_cplx_2() { ((Z (assign (X1 . X2) (tup (+ X 1) (+ X 2)) ;; 14 - X2 (+ X1 1) ;; 15 - X3 (+ X2 1) ;; 16 - (Y0 . X4) (tup (+ X3 1) (+ X3 1)) ;; 17 - X5 (+ Y0 1) ;; 18 - Y1 (+ X5 Y0) ;; 35 + X3 (+ X1 1) ;; 15 + X4 (+ X2 1) ;; 16 + (Y0 . X5) (tup (+ X4 1) (+ X4 1)) ;; 17 + X6 (+ Y0 1) ;; 18 + Y1 (+ X6 Y0) ;; 35 Y1 )) (Q (assign R (+ 3 2) (* R Z))) @@ -1326,6 +1326,66 @@ fn test_assign_fun_cplx_2() { assert_eq!(res.to_string(), "175"); } +#[test] +fn test_assign_simple_with_reodering() { + let prog = indoc! {" +(mod (A) ;; 11 + (include *standard-cl-21*) + (defun tup (a b) (c a b)) + (assign + ;; Tier 1 + (X0 . X1) (tup (+ A 1) (- A 1)) ;; 12 10 + + ;; Tier 4 + finish (+ x2_gtr_x3 (- X3 x2_minus_x3)) ;; 1 + (70 - 50) + + ;; Tier 3 + x2_gtr_x3 (> X2 X3) ;; 1 + x2_minus_x3 (- X2 X3) ;; 50 + + ;; Tier 2 + X2 (* X0 10) ;; 120 + X3 (* X1 7) ;; 70 + + finish + ))"} + .to_string(); + let res = run_string(&prog, &"(11)".to_string()).unwrap(); + assert_eq!(res.to_string(), "21"); +} + +#[test] +fn test_assign_detect_multiple_definition() { + let prog = indoc! {" +(mod (A) ;; 11 + (include *standard-cl-21*) + (defun tup (a b) (c a b)) + (assign + ;; Tier 1 + (X0 . X1) (tup (+ A 1) (- A 1)) ;; 12 10 + + ;; Tier 4 + finish (+ x2_gtr_x3 (- X3 x2_minus_x3)) ;; 1 + (70 - 50) + + ;; Tier 3 + x2_gtr_x3 (> X2 X3) ;; 1 + x2_minus_x3 (- X2 X3) ;; 50 + + ;; Tier 2 + X2 (* X0 10) ;; 120 + X2 (* X1 7) ;; 70 + + finish + ))"} + .to_string(); + if let Err(CompileErr(l, e)) = run_string(&prog, &"(11)".to_string()) { + assert_eq!(l.line, 17); + assert!(e.starts_with("Duplicate")); + } else { + assert!(false); + } +} + #[test] fn test_inline_out_of_bounds_diagnostic() { let prog = indoc! {" From 0f54e04ba69f4cd38f0e1b5dcaaade83f4d23372 Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 1 Feb 2023 11:55:21 -0800 Subject: [PATCH 012/117] merge up + fmt + clippy --- src/compiler/frontend.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index 37f63c041..0e66fe9dd 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -114,7 +114,6 @@ fn calculate_live_helpers( .collect(); needed_helpers = needed_helpers .union(&even_newer_names) - .into_iter() .map(|x| x.to_vec()) .collect(); } From aaacf4b47cbeae5bb754f4ce0a663917796f4839 Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 17 Mar 2023 03:18:10 -0700 Subject: [PATCH 013/117] Add selection for assign inlining --- src/compiler/codegen.rs | 18 ++++++++----- src/compiler/comptypes.rs | 11 ++++++++ src/compiler/evaluate.rs | 4 +-- src/compiler/frontend.rs | 27 +++++++++++++++++--- src/compiler/rename.rs | 7 +++-- src/tests/classic/run.rs | 54 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 104 insertions(+), 17 deletions(-) diff --git a/src/compiler/codegen.rs b/src/compiler/codegen.rs index 70ec6a337..17b465775 100644 --- a/src/compiler/codegen.rs +++ b/src/compiler/codegen.rs @@ -15,7 +15,7 @@ use crate::compiler::compiler::{is_at_capture, run_optimizer}; use crate::compiler::comptypes::{ fold_m, join_vecs_to_string, list_to_cons, Binding, BindingPattern, BodyForm, Callable, CompileErr, CompileForm, CompiledCode, CompilerOpts, ConstantKind, DefunCall, DefunData, - HelperForm, InlineFunction, LetData, LetFormKind, PrimaryCodegen, + HelperForm, InlineFunction, LetData, LetFormInlineHint, LetFormKind, PrimaryCodegen, }; use crate::compiler::debug::{build_swap_table_mut, relabel}; use crate::compiler::evaluate::{Evaluator, EVAL_STACK_LIMIT}; @@ -733,12 +733,18 @@ pub fn empty_compiler(prim_map: Rc, Rc>>, l: Srcloc) -> Pr } } +fn should_inline_let(inline_hint: &Option) -> bool { + matches!(inline_hint, None | Some(LetFormInlineHint::Inline(_))) +} + +#[allow(clippy::too_many_arguments)] fn generate_let_defun( _compiler: &PrimaryCodegen, l: Srcloc, kwl: Option, name: &[u8], args: Rc, + inline_hint: &Option, bindings: Vec>, body: Rc, ) -> HelperForm { @@ -757,7 +763,7 @@ fn generate_let_defun( )); HelperForm::Defun( - true, + should_inline_let(inline_hint), DefunData { loc: l.clone(), nl: l, @@ -797,10 +803,8 @@ fn hoist_body_let_binding( Rc::new(BodyForm::Let( LetFormKind::Sequential, LetData { - loc: letdata.loc.clone(), - kw: letdata.kw.clone(), bindings: sub_bindings, - body: letdata.body.clone(), + ..letdata.clone() }, )) }; @@ -812,10 +816,9 @@ fn hoist_body_let_binding( Rc::new(BodyForm::Let( LetFormKind::Parallel, LetData { - loc: letdata.loc.clone(), - kw: letdata.kw.clone(), bindings: vec![letdata.bindings[0].clone()], body: new_sub_expr, + ..letdata.clone() }, )), ) @@ -846,6 +849,7 @@ fn hoist_body_let_binding( None, &defun_name, args, + &letdata.inline_hint, revised_bindings.to_vec(), letdata.body.clone(), ); diff --git a/src/compiler/comptypes.rs b/src/compiler/comptypes.rs index 6ad40369c..1046c2888 100644 --- a/src/compiler/comptypes.rs +++ b/src/compiler/comptypes.rs @@ -90,6 +90,15 @@ pub enum BindingPattern { Complex(Rc), } +/// If present, states an intention for desugaring of this let form to favor +/// inlining or functions. +#[derive(Clone, Debug, Serialize)] +pub enum LetFormInlineHint { + NoChoice, + Inline(Srcloc), + NonInline(Srcloc), +} + /// A binding from a (let ...) form. Specifies the name of the bound variable /// the location of the whole binding form, the location of the name atom (nl) /// and the body as a BodyForm (which are chialisp expressions). @@ -126,6 +135,8 @@ pub struct LetData { pub loc: Srcloc, /// The location specifically of the let or let* keyword. pub kw: Option, + /// Inline hint. + pub inline_hint: Option, /// The bindings introduced. pub bindings: Vec>, /// The expression evaluated in the context of all the bindings. diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index 9486c046e..fa0207190 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -1068,10 +1068,8 @@ impl<'info> Evaluator { Rc::new(BodyForm::Let( LetFormKind::Sequential, LetData { - loc: letdata.loc.clone(), - kw: letdata.kw.clone(), bindings: rest_of_bindings, - body: letdata.body.clone(), + ..letdata.clone() }, )), only_inline, diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index 1a7187123..4204d055f 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -8,7 +8,7 @@ use crate::classic::clvm::__type_compatibility__::bi_one; use crate::compiler::comptypes::{ list_to_cons, Binding, BindingPattern, BodyForm, CompileErr, CompileForm, CompilerOpts, ConstantKind, DefconstData, DefmacData, DefunData, HelperForm, IncludeDesc, LetData, - LetFormKind, ModAccum, + LetFormInlineHint, LetFormKind, ModAccum, }; use crate::compiler::preprocessor::preprocess; use crate::compiler::rename::rename_children_compileform; @@ -262,6 +262,7 @@ fn handle_assign_form( opts: Rc, l: Srcloc, v: &[SExp], + inline_hint: Option, ) -> Result { if v.len() % 2 == 0 { return Err(CompileErr( @@ -377,6 +378,7 @@ fn handle_assign_form( loc: l.clone(), kw: Some(l.clone()), bindings: end_bindings, + inline_hint: inline_hint.clone(), body: Rc::new(compiled_body), }, ); @@ -388,6 +390,7 @@ fn handle_assign_form( loc: l.clone(), kw: Some(l.clone()), bindings: binding_list, + inline_hint: inline_hint.clone(), body: Rc::new(output_let), }, ) @@ -434,6 +437,9 @@ pub fn compile_bodyform( return Ok(BodyForm::Quoted(tail_copy.clone())); } + let assign_lambda = *atom_name == "assign-lambda".as_bytes().to_vec(); + let assign_inline = *atom_name == "assign-inline".as_bytes().to_vec(); + match tail.proper_list() { Some(v) => { if *atom_name == "let".as_bytes().to_vec() @@ -461,11 +467,26 @@ pub fn compile_bodyform( loc: l.clone(), kw: Some(l.clone()), bindings: let_bindings, + inline_hint: None, body: Rc::new(compiled_body), }, )) - } else if *atom_name == "assign".as_bytes().to_vec() { - handle_assign_form(opts.clone(), l.clone(), &v) + } else if assign_lambda + || assign_inline + || *atom_name == "assign".as_bytes().to_vec() + { + handle_assign_form( + opts.clone(), + l.clone(), + &v, + if assign_lambda { + Some(LetFormInlineHint::NonInline(l.clone())) + } else if assign_inline { + Some(LetFormInlineHint::Inline(l.clone())) + } else { + Some(LetFormInlineHint::NoChoice) + }, + ) } else if *atom_name == "quote".as_bytes().to_vec() { if v.len() != 1 { return finish_err("quote"); diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index 781ae26e4..f0e3837f1 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -173,10 +173,9 @@ fn rename_in_bodyform(namemap: &HashMap, Vec>, b: Rc) -> B BodyForm::Let( kind.clone(), LetData { - loc: letdata.loc.clone(), - kw: letdata.kw.clone(), bindings: new_bindings, body: Rc::new(new_body), + ..letdata.clone() }, ) } @@ -226,6 +225,7 @@ pub fn desugar_sequential_let_bindings( loc: want_binding.loc(), kw: None, bindings: vec![want_binding], + inline_hint: None, body: Rc::new(body.clone()), }, ), @@ -277,10 +277,9 @@ fn rename_args_bodyform(b: &BodyForm) -> BodyForm { BodyForm::Let( LetFormKind::Parallel, LetData { - loc: letdata.loc.clone(), - kw: letdata.kw.clone(), bindings: new_bindings, body: Rc::new(locally_renamed_body), + ..letdata.clone() }, ) } diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index 30bb07a74..4e54c289e 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -851,3 +851,57 @@ fn test_modern_sets_source_file_in_symbols() { Some("resources/tests/steprun/fact.cl".to_string()) ); } + +#[test] +fn test_assign_lambda_code_generation() { + let tname = "test_assign_lambda_code_generation.sym".to_string(); + do_basic_run(&vec![ + "run".to_string(), + "--extra-syms".to_string(), + "--symbol-output-file".to_string(), + tname.clone(), + "(mod (A) (include *standard-cl-21*) (defun F (X) (+ X 1)) (assign-lambda X (F A) X))" + .to_string(), + ]); + let read_in_file = fs::read_to_string(&tname).expect("should have dropped symbols"); + fs::remove_file(&tname).expect("should have existed"); + let decoded_symbol_file: HashMap = + serde_json::from_str(&read_in_file).expect("should decode"); + let found_wanted_symbols: Vec = decoded_symbol_file + .iter() + .filter(|(_, v)| *v == "F" || v.starts_with("letbinding")) + .map(|(k, _)| k.clone()) + .collect(); + assert_eq!(found_wanted_symbols.len(), 2); + // We should have these two functions. + assert!(found_wanted_symbols + .contains(&"ccd5be506752cebf01f9930b4c108fe18058c65e1ab57a72ca0a00d9788c7ca6".to_string())); + assert!(found_wanted_symbols + .contains(&"0a5af5ae61fae2e53cb309d4d9c2c64baf0261824823008b9cf2b21b09221e44".to_string())); +} + +#[test] +fn test_assign_lambda_code_generation_normally_inlines() { + let tname = "test_assign_inline_code_generation.sym".to_string(); + do_basic_run(&vec![ + "run".to_string(), + "--extra-syms".to_string(), + "--symbol-output-file".to_string(), + tname.clone(), + "(mod (A) (include *standard-cl-21*) (defun F (X) (+ X 1)) (assign-inline X (F A) X))" + .to_string(), + ]); + let read_in_file = fs::read_to_string(&tname).expect("should have dropped symbols"); + fs::remove_file(&tname).expect("should have existed"); + let decoded_symbol_file: HashMap = + serde_json::from_str(&read_in_file).expect("should decode"); + let found_wanted_symbols: Vec = decoded_symbol_file + .iter() + .filter(|(_, v)| *v == "F" || v.starts_with("letbinding")) + .map(|(k, _)| k.clone()) + .collect(); + assert_eq!(found_wanted_symbols.len(), 1); + // We should have these two functions. + assert!(found_wanted_symbols + .contains(&"ccd5be506752cebf01f9930b4c108fe18058c65e1ab57a72ca0a00d9788c7ca6".to_string())); +} From 9c3a7c79c62be35f5828bcd5eec5bd83d72ecc41 Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 17 Mar 2023 09:07:51 -0700 Subject: [PATCH 014/117] Move to box to tighten the stack up --- src/compiler/codegen.rs | 12 ++++++------ src/compiler/comptypes.rs | 2 +- src/compiler/evaluate.rs | 6 +++--- src/compiler/frontend.rs | 12 ++++++------ src/compiler/rename.rs | 16 ++++++++-------- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/compiler/codegen.rs b/src/compiler/codegen.rs index 17b465775..d05426eae 100644 --- a/src/compiler/codegen.rs +++ b/src/compiler/codegen.rs @@ -802,10 +802,10 @@ fn hoist_body_let_binding( let sub_bindings = letdata.bindings.iter().skip(1).cloned().collect(); Rc::new(BodyForm::Let( LetFormKind::Sequential, - LetData { + Box::new(LetData { bindings: sub_bindings, - ..letdata.clone() - }, + ..*letdata.clone() + }), )) }; @@ -815,11 +815,11 @@ fn hoist_body_let_binding( args, Rc::new(BodyForm::Let( LetFormKind::Parallel, - LetData { + Box::new(LetData { bindings: vec![letdata.bindings[0].clone()], body: new_sub_expr, - ..letdata.clone() - }, + ..*letdata.clone() + }), )), ) } diff --git a/src/compiler/comptypes.rs b/src/compiler/comptypes.rs index 1046c2888..24faec308 100644 --- a/src/compiler/comptypes.rs +++ b/src/compiler/comptypes.rs @@ -146,7 +146,7 @@ pub struct LetData { #[derive(Clone, Debug, Serialize)] pub enum BodyForm { /// A let or let* form (depending on LetFormKind). - Let(LetFormKind, LetData), + Let(LetFormKind, Box), /// An explicitly quoted constant of some kind. Quoted(SExp), /// An undiferentiated "value" of some kind in the source language. diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index fa0207190..7d5bd66be 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -1067,10 +1067,10 @@ impl<'info> Evaluator { &updated_bindings, Rc::new(BodyForm::Let( LetFormKind::Sequential, - LetData { + Box::new(LetData { bindings: rest_of_bindings, - ..letdata.clone() - }, + ..*letdata.clone() + }), )), only_inline, ) diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index 4204d055f..323f2da45 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -374,25 +374,25 @@ fn handle_assign_form( let mut output_let = BodyForm::Let( LetFormKind::Parallel, - LetData { + Box::new(LetData { loc: l.clone(), kw: Some(l.clone()), bindings: end_bindings, inline_hint: inline_hint.clone(), body: Rc::new(compiled_body), - }, + }), ); for binding_list in binding_lists.into_iter().skip(1) { output_let = BodyForm::Let( LetFormKind::Parallel, - LetData { + Box::new(LetData { loc: l.clone(), kw: Some(l.clone()), bindings: binding_list, inline_hint: inline_hint.clone(), body: Rc::new(output_let), - }, + }), ) } @@ -463,13 +463,13 @@ pub fn compile_bodyform( let compiled_body = compile_bodyform(opts, Rc::new(body))?; Ok(BodyForm::Let( kind, - LetData { + Box::new(LetData { loc: l.clone(), kw: Some(l.clone()), bindings: let_bindings, inline_hint: None, body: Rc::new(compiled_body), - }, + }), )) } else if assign_lambda || assign_inline diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index f0e3837f1..88362f775 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -172,11 +172,11 @@ fn rename_in_bodyform(namemap: &HashMap, Vec>, b: Rc) -> B let new_body = rename_in_bodyform(namemap, letdata.body.clone()); BodyForm::Let( kind.clone(), - LetData { + Box::new(LetData { bindings: new_bindings, body: Rc::new(new_body), - ..letdata.clone() - }, + ..*letdata.clone() + }), ) } @@ -221,13 +221,13 @@ pub fn desugar_sequential_let_bindings( bindings, &BodyForm::Let( LetFormKind::Parallel, - LetData { + Box::new(LetData { loc: want_binding.loc(), kw: None, bindings: vec![want_binding], inline_hint: None, body: Rc::new(body.clone()), - }, + }), ), n - 1, ) @@ -276,11 +276,11 @@ fn rename_args_bodyform(b: &BodyForm) -> BodyForm { let locally_renamed_body = rename_in_bodyform(&local_namemap, letdata.body.clone()); BodyForm::Let( LetFormKind::Parallel, - LetData { + Box::new(LetData { bindings: new_bindings, body: Rc::new(locally_renamed_body), - ..letdata.clone() - }, + ..*letdata.clone() + }), ) } From 1e342760c30651274ec345d1cc87c3d0ec3ed86f Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 17 Mar 2023 09:58:07 -0700 Subject: [PATCH 015/117] Improve a bit so we don't clobber non-inline assigns via the first stage of the frontend optimizer --- src/compiler/codegen.rs | 2 +- src/compiler/evaluate.rs | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/compiler/codegen.rs b/src/compiler/codegen.rs index d05426eae..29da83073 100644 --- a/src/compiler/codegen.rs +++ b/src/compiler/codegen.rs @@ -733,7 +733,7 @@ pub fn empty_compiler(prim_map: Rc, Rc>>, l: Srcloc) -> Pr } } -fn should_inline_let(inline_hint: &Option) -> bool { +pub fn should_inline_let(inline_hint: &Option) -> bool { matches!(inline_hint, None | Some(LetFormInlineHint::Inline(_))) } diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index 7d5bd66be..569a00e35 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -14,7 +14,7 @@ use crate::compiler::codegen::codegen; use crate::compiler::compiler::is_at_capture; use crate::compiler::comptypes::{ Binding, BindingPattern, BodyForm, CompileErr, CompileForm, CompilerOpts, HelperForm, LetData, - LetFormKind, + LetFormInlineHint, LetFormKind, }; use crate::compiler::frontend::frontend; use crate::compiler::runtypes::RunFailure; @@ -639,6 +639,10 @@ fn flatten_expression_to_names(expr: Rc) -> Rc { Rc::new(BodyForm::Call(expr.loc(), call_vec)) } +pub fn eval_dont_expand_let(inline_hint: &Option) -> bool { + matches!(inline_hint, Some(LetFormInlineHint::NonInline(_))) +} + impl<'info> Evaluator { pub fn new( opts: Rc, @@ -1033,6 +1037,10 @@ impl<'info> Evaluator { let mut visited = VisitedMarker::again(body.loc(), visited_)?; match body.borrow() { BodyForm::Let(LetFormKind::Parallel, letdata) => { + if eval_dont_expand_let(&letdata.inline_hint) && only_inline { + return Ok(body.clone()); + } + let updated_bindings = update_parallel_bindings(env, &letdata.bindings); self.shrink_bodyform_visited( allocator, @@ -1044,6 +1052,10 @@ impl<'info> Evaluator { ) } BodyForm::Let(LetFormKind::Sequential, letdata) => { + if eval_dont_expand_let(&letdata.inline_hint) && only_inline { + return Ok(body.clone()); + } + if letdata.bindings.is_empty() { self.shrink_bodyform_visited( allocator, From 17fbb55b9384d97d6fc592ab6415a84ca60e836c Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 19 Apr 2023 21:47:50 -0700 Subject: [PATCH 016/117] Change default to non-inline --- src/compiler/codegen.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/codegen.rs b/src/compiler/codegen.rs index 485f71a21..095f145dd 100644 --- a/src/compiler/codegen.rs +++ b/src/compiler/codegen.rs @@ -734,7 +734,7 @@ pub fn empty_compiler(prim_map: Rc, Rc>>, l: Srcloc) -> Pr } pub fn should_inline_let(inline_hint: &Option) -> bool { - matches!(inline_hint, None | Some(LetFormInlineHint::Inline(_))) + matches!(inline_hint, Some(LetFormInlineHint::Inline(_))) } #[allow(clippy::too_many_arguments)] From 8de55ad7de14073c6ed9c0fa80c083d41f398d41 Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 21 Apr 2023 10:54:25 -0700 Subject: [PATCH 017/117] Revert "Change default to non-inline" This reverts commit 17fbb55b9384d97d6fc592ab6415a84ca60e836c. --- src/compiler/codegen.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/codegen.rs b/src/compiler/codegen.rs index 095f145dd..485f71a21 100644 --- a/src/compiler/codegen.rs +++ b/src/compiler/codegen.rs @@ -734,7 +734,7 @@ pub fn empty_compiler(prim_map: Rc, Rc>>, l: Srcloc) -> Pr } pub fn should_inline_let(inline_hint: &Option) -> bool { - matches!(inline_hint, Some(LetFormInlineHint::Inline(_))) + matches!(inline_hint, None | Some(LetFormInlineHint::Inline(_))) } #[allow(clippy::too_many_arguments)] From 00d2e13bf22fd73bbce6f20bd9131efd9febe58f Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 26 Apr 2023 12:38:28 -0700 Subject: [PATCH 018/117] Try updating requests on behalf of twine re: error in ci --- .github/workflows/build-arm64-wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-arm64-wheels.yml b/.github/workflows/build-arm64-wheels.yml index 854b4f4f3..55fd89de2 100644 --- a/.github/workflows/build-arm64-wheels.yml +++ b/.github/workflows/build-arm64-wheels.yml @@ -69,6 +69,7 @@ jobs: python3 -m venv venv if [ ! -f "activate" ]; then ln -s venv/bin/activate; fi . ./activate + pip install --upgrade requests pip install setuptools_rust pip install twine From 3babcdb52ac06141fe25d83c6abb6f140e197ec5 Mon Sep 17 00:00:00 2001 From: arty Date: Sun, 21 May 2023 17:43:43 -0700 Subject: [PATCH 019/117] Update from assign refactor --- resources/tests/chia-gaming/last.clinc | 39 +++++++ resources/tests/chia-gaming/test-last.clsp | 6 + src/compiler/codegen.rs | 122 +++++++++++++++++++-- src/compiler/compiler.rs | 4 +- src/compiler/comptypes.rs | 79 +++++++++---- src/compiler/evaluate.rs | 16 ++- src/compiler/frontend.rs | 109 +++--------------- src/compiler/rename.rs | 4 + src/tests/classic/run.rs | 12 ++ 9 files changed, 259 insertions(+), 132 deletions(-) create mode 100644 resources/tests/chia-gaming/last.clinc create mode 100644 resources/tests/chia-gaming/test-last.clsp diff --git a/resources/tests/chia-gaming/last.clinc b/resources/tests/chia-gaming/last.clinc new file mode 100644 index 000000000..5a6ffd73e --- /dev/null +++ b/resources/tests/chia-gaming/last.clinc @@ -0,0 +1,39 @@ +( + (defun last_inner ((next . remainder)) + (if remainder + (last_inner remainder) + next + ) + ) + + (defmacro last ARGS + (defun snoc (L agg) + (if L + (if (r L) + (snoc (r L) (c (f L) agg)) + (c (f L) agg) + ) + (c () ()) + ) + ) + + (defun prefix (L P) + (if L + (c (f L) (prefix (r L) P)) + P + ) + ) + + (if ARGS + (if (r ARGS) + (assign + (final . rest) (snoc ARGS ()) + reversed (prefix rest (list final)) + (qq (last_inner (unquote (c list reversed)))) + ) + (qq (last_inner (unquote (f ARGS)))) + ) + (x "Last takes at least one argument") + ) + ) +) diff --git a/resources/tests/chia-gaming/test-last.clsp b/resources/tests/chia-gaming/test-last.clsp new file mode 100644 index 000000000..eb2076769 --- /dev/null +++ b/resources/tests/chia-gaming/test-last.clsp @@ -0,0 +1,6 @@ +(mod () + (include *standard-cl-21*) + (include last.clinc) + + (last 99 100 101) + ) diff --git a/src/compiler/codegen.rs b/src/compiler/codegen.rs index 58308fa0a..84982892f 100644 --- a/src/compiler/codegen.rs +++ b/src/compiler/codegen.rs @@ -1,6 +1,7 @@ use std::borrow::Borrow; use std::collections::HashMap; use std::collections::HashSet; +use std::mem::swap; use std::rc::Rc; use num_bigint::ToBigInt; @@ -19,7 +20,7 @@ use crate::compiler::comptypes::{ }; use crate::compiler::debug::{build_swap_table_mut, relabel}; use crate::compiler::evaluate::{Evaluator, EVAL_STACK_LIMIT}; -use crate::compiler::frontend::compile_bodyform; +use crate::compiler::frontend::{compile_bodyform, make_provides_set}; use crate::compiler::gensym::gensym; use crate::compiler::inline::{replace_in_inline, synthesize_args}; use crate::compiler::optimize::optimize_expr; @@ -27,7 +28,7 @@ use crate::compiler::prims::{primapply, primcons, primquote}; use crate::compiler::runtypes::RunFailure; use crate::compiler::sexp::{decode_string, SExp}; use crate::compiler::srcloc::Srcloc; -use crate::util::u8_from_number; +use crate::util::{toposort, u8_from_number}; const MACRO_TIME_LIMIT: usize = 1000000; const CONST_EVAL_LIMIT: usize = 1000000; @@ -779,15 +780,105 @@ fn generate_let_args(_l: Srcloc, blist: Vec>) -> Vec> { blist.iter().map(|b| b.body.clone()).collect() } +pub fn hoist_assign_form( + letdata: &LetData, +) -> Result { + // Topological sort of bindings. + let sorted_spec = toposort( + &letdata.bindings, + CompileErr(letdata.loc.clone(), "deadlock resolving binding order".to_string()), + // Needs: What this binding relies on. + |possible, b| { + let mut need_set = HashSet::new(); + make_provides_set(&mut need_set, b.body.to_sexp()); + let mut need_set_thats_possible = HashSet::new(); + for need in need_set.intersection(possible) { + need_set_thats_possible.insert(need.clone()); + } + Ok(need_set_thats_possible) + }, + // Has: What this binding provides. + |b| match b.pattern.borrow() { + BindingPattern::Name(name) => HashSet::from([name.clone()]), + BindingPattern::Complex(sexp) => { + let mut result_set = HashSet::new(); + make_provides_set(&mut result_set, sexp.clone()); + result_set + } + }, + )?; + + // Break up into stages of parallel let forms. + // Track the needed bindings of this level. + // If this becomes broader in a way that doesn't + // match the existing provides, we need to break + // the let binding. + let mut current_provides = HashSet::new(); + let mut binding_lists = Vec::new(); + let mut this_round_bindings = Vec::new(); + let mut new_provides: HashSet> = HashSet::new(); + + for spec in sorted_spec.iter() { + let mut new_needs = spec.needs.difference(¤t_provides).cloned(); + if new_needs.next().is_some() { + // Roll over the set we're accumulating to the finished version. + let mut empty_tmp: Vec> = Vec::new(); + swap(&mut empty_tmp, &mut this_round_bindings); + binding_lists.push(empty_tmp); + for provided in new_provides.iter() { + current_provides.insert(provided.clone()); + } + new_provides.clear(); + } + // Record what we can provide to the next round. + for p in spec.has.iter() { + new_provides.insert(p.clone()); + } + this_round_bindings.push(letdata.bindings[spec.index].clone()); + } + + // Pick up the last ones that didn't add new needs. + if !this_round_bindings.is_empty() { + binding_lists.push(this_round_bindings); + } + + binding_lists.reverse(); + + // Spill let forms as parallel sets to get the best stack we can. + let mut end_bindings = Vec::new(); + swap(&mut end_bindings, &mut binding_lists[0]); + + let mut output_let = BodyForm::Let( + LetFormKind::Parallel, + Box::new(LetData { + bindings: end_bindings, + .. letdata.clone() + }), + ); + + for binding_list in binding_lists.into_iter().skip(1) { + output_let = BodyForm::Let( + LetFormKind::Parallel, + Box::new(LetData { + bindings: binding_list, + body: Rc::new(output_let), + .. letdata.clone() + }), + ) + } + + Ok(output_let) +} + pub fn hoist_body_let_binding( outer_context: Option>, args: Rc, body: Rc, -) -> (Vec, Rc) { +) -> Result<(Vec, Rc), CompileErr> { match body.borrow() { BodyForm::Let(LetFormKind::Sequential, letdata) => { if letdata.bindings.is_empty() { - return (vec![], letdata.body.clone()); + return Ok((vec![], letdata.body.clone())); } // If we're here, we're in the middle of hoisting. @@ -827,7 +918,7 @@ pub fn hoist_body_let_binding( let mut revised_bindings = Vec::new(); for b in letdata.bindings.iter() { let (mut new_helpers, new_binding) = - hoist_body_let_binding(outer_context.clone(), args.clone(), b.body.clone()); + hoist_body_let_binding(outer_context.clone(), args.clone(), b.body.clone())?; out_defuns.append(&mut new_helpers); revised_bindings.push(Rc::new(Binding { loc: b.loc.clone(), @@ -873,24 +964,31 @@ pub fn hoist_body_let_binding( call_args.append(&mut let_args); let final_call = BodyForm::Call(letdata.loc.clone(), call_args); - (out_defuns, Rc::new(final_call)) + Ok((out_defuns, Rc::new(final_call))) + } + BodyForm::Let(LetFormKind::Assign, letdata) => { + hoist_body_let_binding( + outer_context, + args, + Rc::new(hoist_assign_form(letdata)?) + ) } BodyForm::Call(l, list) => { let mut vres = Vec::new(); let mut new_call_list = vec![list[0].clone()]; for i in list.iter().skip(1) { let (new_helper, new_arg) = - hoist_body_let_binding(outer_context.clone(), args.clone(), i.clone()); + hoist_body_let_binding(outer_context.clone(), args.clone(), i.clone())?; new_call_list.push(new_arg); vres.append(&mut new_helper.clone()); } - (vres, Rc::new(BodyForm::Call(l.clone(), new_call_list))) + Ok((vres, Rc::new(BodyForm::Call(l.clone(), new_call_list)))) } - _ => (Vec::new(), body.clone()), + _ => Ok((Vec::new(), body.clone())), } } -pub fn process_helper_let_bindings(helpers: &[HelperForm]) -> Vec { +pub fn process_helper_let_bindings(helpers: &[HelperForm]) -> Result, CompileErr> { let mut result = helpers.to_owned(); let mut i = 0; @@ -903,7 +1001,7 @@ pub fn process_helper_let_bindings(helpers: &[HelperForm]) -> Vec { None }; let helper_result = - hoist_body_let_binding(context, defun.args.clone(), defun.body.clone()); + hoist_body_let_binding(context, defun.args.clone(), defun.body.clone())?; let hoisted_helpers = helper_result.0; let hoisted_body = helper_result.1.clone(); @@ -932,7 +1030,7 @@ pub fn process_helper_let_bindings(helpers: &[HelperForm]) -> Vec { } } - result + Ok(result) } fn start_codegen( diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index 7639f88b5..5dc6c6735 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -166,14 +166,14 @@ pub fn compile_pre_forms( }; // Transform let bindings, merging nested let scopes with the top namespace - let hoisted_bindings = hoist_body_let_binding(None, p1.args.clone(), p1.exp.clone()); + let hoisted_bindings = hoist_body_let_binding(None, p1.args.clone(), p1.exp.clone())?; let mut new_helpers = hoisted_bindings.0; let expr = hoisted_bindings.1; // expr is the let-hoisted program // TODO: Distinguish the frontend_helpers and the hoisted_let helpers for later stages let mut combined_helpers = p1.helpers.clone(); combined_helpers.append(&mut new_helpers); - let combined_helpers = process_helper_let_bindings(&combined_helpers); + let combined_helpers = process_helper_let_bindings(&combined_helpers)?; let p2 = CompileForm { loc: p1.loc.clone(), diff --git a/src/compiler/comptypes.rs b/src/compiler/comptypes.rs index 17bc17eb5..9b5e12aa5 100644 --- a/src/compiler/comptypes.rs +++ b/src/compiler/comptypes.rs @@ -10,7 +10,7 @@ use crate::classic::clvm::__type_compatibility__::{Bytes, BytesFromType}; use crate::classic::clvm_tools::stages::stage_0::TRunProgram; use crate::compiler::clvm::sha256tree; -use crate::compiler::sexp::{decode_string, SExp}; +use crate::compiler::sexp::{decode_string, enlist, SExp}; use crate::compiler::srcloc::Srcloc; /// The basic error type. It contains a Srcloc identifying coordinates of the @@ -125,6 +125,7 @@ pub struct Binding { pub enum LetFormKind { Parallel, Sequential, + Assign, } /// Information about a let form. Encapsulates everything except whether it's @@ -585,6 +586,50 @@ impl HelperForm { } } +fn compose_let(marker: &[u8], letdata: &LetData) -> Rc { + let translated_bindings: Vec> = + letdata.bindings.iter().map(|x| x.to_sexp()).collect(); + let bindings_cons = list_to_cons(letdata.loc.clone(), &translated_bindings); + let translated_body = letdata.body.to_sexp(); + let kw_loc = letdata.kw.clone().unwrap_or_else(|| letdata.loc.clone()); + Rc::new(SExp::Cons( + letdata.loc.clone(), + Rc::new(SExp::Atom(kw_loc, marker.to_vec())), + Rc::new(SExp::Cons( + letdata.loc.clone(), + Rc::new(bindings_cons), + Rc::new(SExp::Cons( + letdata.loc.clone(), + translated_body, + Rc::new(SExp::Nil(letdata.loc.clone())), + )), + )), + )) +} + +fn compose_assign(letdata: &LetData) -> Rc { + let mut result = Vec::new(); + let kw_loc = letdata.kw.clone().unwrap_or_else(|| letdata.loc.clone()); + result.push(Rc::new(SExp::Atom(kw_loc, b"assign".to_vec()))); + for b in letdata.bindings.iter() { + // Binding pattern + match &b.pattern { + BindingPattern::Name(v) => { + result.push(Rc::new(SExp::Atom(b.nl.clone(), v.to_vec()))); + } + BindingPattern::Complex(c) => { + result.push(c.clone()); + } + } + + // Binding body. + result.push(b.body.to_sexp()); + } + + result.push(letdata.body.to_sexp()); + Rc::new(enlist(letdata.loc.clone(), result)) +} + impl BodyForm { /// Get the general location of the BodyForm. pub fn loc(&self) -> Srcloc { @@ -603,28 +648,16 @@ impl BodyForm { pub fn to_sexp(&self) -> Rc { match self { BodyForm::Let(kind, letdata) => { - let translated_bindings: Vec> = - letdata.bindings.iter().map(|x| x.to_sexp()).collect(); - let bindings_cons = list_to_cons(letdata.loc.clone(), &translated_bindings); - let translated_body = letdata.body.to_sexp(); - let marker = match kind { - LetFormKind::Parallel => "let", - LetFormKind::Sequential => "let*", - }; - let kw_loc = letdata.kw.clone().unwrap_or_else(|| letdata.loc.clone()); - Rc::new(SExp::Cons( - letdata.loc.clone(), - Rc::new(SExp::atom_from_string(kw_loc, marker)), - Rc::new(SExp::Cons( - letdata.loc.clone(), - Rc::new(bindings_cons), - Rc::new(SExp::Cons( - letdata.loc.clone(), - translated_body, - Rc::new(SExp::Nil(letdata.loc.clone())), - )), - )), - )) + if matches!(kind, LetFormKind::Assign) { + compose_assign(&letdata) + } else { + let marker = if matches!(kind, LetFormKind::Sequential) { + b"let*".to_vec() + } else { + b"let".to_vec() + }; + compose_let(&marker, &letdata) + } } BodyForm::Quoted(body) => Rc::new(SExp::Cons( body.loc(), diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index 569a00e35..fd54d10a9 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -10,7 +10,7 @@ use crate::classic::clvm::__type_compatibility__::{bi_one, bi_zero}; use crate::classic::clvm_tools::stages::stage_0::TRunProgram; use crate::compiler::clvm::run; -use crate::compiler::codegen::codegen; +use crate::compiler::codegen::{codegen, hoist_assign_form}; use crate::compiler::compiler::is_at_capture; use crate::compiler::comptypes::{ Binding, BindingPattern, BodyForm, CompileErr, CompileForm, CompilerOpts, HelperForm, LetData, @@ -1088,6 +1088,20 @@ impl<'info> Evaluator { ) } } + BodyForm::Let(LetFormKind::Assign, letdata) => { + if eval_dont_expand_let(&letdata.inline_hint) && only_inline { + return Ok(body.clone()); + } + + self.shrink_bodyform_visited( + allocator, + &mut visited, + prog_args, + env, + Rc::new(hoist_assign_form(&letdata)?), + only_inline + ) + } BodyForm::Quoted(_) => Ok(body.clone()), BodyForm::Value(SExp::Atom(l, name)) => { if name == &"@".as_bytes().to_vec() { diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index 323f2da45..f4020cc9f 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -1,7 +1,6 @@ use std::borrow::Borrow; use std::collections::HashMap; use std::collections::HashSet; -use std::mem::swap; use std::rc::Rc; use crate::classic::clvm::__type_compatibility__::bi_one; @@ -14,7 +13,7 @@ use crate::compiler::preprocessor::preprocess; use crate::compiler::rename::rename_children_compileform; use crate::compiler::sexp::{decode_string, enlist, SExp}; use crate::compiler::srcloc::Srcloc; -use crate::util::{toposort, u8_from_number}; +use crate::util::u8_from_number; fn collect_used_names_sexp(body: Rc) -> Vec> { match body.borrow() { @@ -237,15 +236,15 @@ fn make_let_bindings( result.append(&mut rest_bindings); Ok(result) } - _ => err.clone(), + _ => err.clone() }) .unwrap_or_else(|| err.clone()), - _ => err, + _ => err } } // Make a set of names in this sexp. -fn make_provides_set(provides_set: &mut HashSet>, body_sexp: Rc) { +pub fn make_provides_set(provides_set: &mut HashSet>, body_sexp: Rc) { match body_sexp.atomize() { SExp::Cons(_, a, b) => { make_provides_set(provides_set, a); @@ -278,15 +277,15 @@ fn handle_assign_form( let destructure_pattern = Rc::new(v[idx].clone()); let binding_body = compile_bodyform(opts.clone(), Rc::new(v[idx + 1].clone()))?; - // Ensure bindings aren't duplicated as we won't be able to guarantee their - // order during toposort. + // Ensure bindings aren't duplicated as we won't be able to + // guarantee their order during toposort. let mut this_provides = HashSet::new(); make_provides_set(&mut this_provides, destructure_pattern.clone()); for item in this_provides.iter() { if check_duplicates.contains(item) { return Err(CompileErr( - v[idx].loc(), + destructure_pattern.loc(), format!("Duplicate binding {}", decode_string(item)), )); } @@ -301,102 +300,24 @@ fn handle_assign_form( })); } - // Topological sort of bindings. - let sorted_spec = toposort( - &bindings, - CompileErr(l.clone(), "deadlock resolving binding order".to_string()), - // Needs: What this binding relies on. - |possible, b| { - let mut need_set = HashSet::new(); - make_provides_set(&mut need_set, b.body.to_sexp()); - let mut need_set_thats_possible = HashSet::new(); - for need in need_set.intersection(possible) { - need_set_thats_possible.insert(need.clone()); - } - Ok(need_set_thats_possible) - }, - // Has: What this binding provides. - |b| match b.pattern.borrow() { - BindingPattern::Name(name) => HashSet::from([name.clone()]), - BindingPattern::Complex(sexp) => { - let mut result_set = HashSet::new(); - make_provides_set(&mut result_set, sexp.clone()); - result_set - } - }, - )?; - let compiled_body = compile_bodyform(opts, Rc::new(v[v.len() - 1].clone()))?; - // Break up into stages of parallel let forms. - // Track the needed bindings of this level. - // If this becomes broader in a way that doesn't - // match the existing provides, we need to break - // the let binding. - let mut current_provides = HashSet::new(); - let mut binding_lists = Vec::new(); - let mut this_round_bindings = Vec::new(); - let mut new_provides: HashSet> = HashSet::new(); - - for spec in sorted_spec.iter() { - let mut new_needs = spec.needs.difference(¤t_provides).cloned(); - if new_needs.next().is_some() { - // Roll over the set we're accumulating to the finished version. - let mut empty_tmp: Vec> = Vec::new(); - swap(&mut empty_tmp, &mut this_round_bindings); - binding_lists.push(empty_tmp); - for provided in new_provides.iter() { - current_provides.insert(provided.clone()); - } - new_provides.clear(); - } - // Record what we can provide to the next round. - for p in spec.has.iter() { - new_provides.insert(p.clone()); - } - this_round_bindings.push(bindings[spec.index].clone()); - } - - // Pick up the last ones that didn't add new needs. - if !this_round_bindings.is_empty() { - binding_lists.push(this_round_bindings); - } - // We don't need to do much if there were no bindings. - if binding_lists.is_empty() { + if bindings.is_empty() { return Ok(compiled_body); } - binding_lists.reverse(); - - // Spill let forms as parallel sets to get the best stack we can. - let mut end_bindings = Vec::new(); - swap(&mut end_bindings, &mut binding_lists[0]); - - let mut output_let = BodyForm::Let( - LetFormKind::Parallel, + // Return a precise representation of this assign while storing up the work + // we did breaking it down. + Ok(BodyForm::Let( + LetFormKind::Assign, Box::new(LetData { loc: l.clone(), kw: Some(l.clone()), - bindings: end_bindings, + bindings: bindings, inline_hint: inline_hint.clone(), body: Rc::new(compiled_body), - }), - ); - - for binding_list in binding_lists.into_iter().skip(1) { - output_let = BodyForm::Let( - LetFormKind::Parallel, - Box::new(LetData { - loc: l.clone(), - kw: Some(l.clone()), - bindings: binding_list, - inline_hint: inline_hint.clone(), - body: Rc::new(output_let), - }), - ) - } - - Ok(output_let) + }) + )) } pub fn compile_bodyform( diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index 88362f775..f920b62dd 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -284,6 +284,10 @@ fn rename_args_bodyform(b: &BodyForm) -> BodyForm { ) } + BodyForm::Let(LetFormKind::Assign, _letdata) => { + b.clone() + } + BodyForm::Quoted(e) => BodyForm::Quoted(e.clone()), BodyForm::Value(v) => BodyForm::Value(v.clone()), diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index b8dab1bf8..49e4c8532 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -923,3 +923,15 @@ fn test_cost_reporting_0() { "cost = 1978\n0x6fcb06b1fe29d132bb37f3a21b86d7cf03d636bf6230aa206486bef5e68f9875" ); } + +#[test] +fn test_assign_fancy_final_dot_rest() { + let result_prog = do_basic_run(&vec![ + "run".to_string(), + "-i".to_string(), + "resources/tests/chia-gaming".to_string(), + "resources/tests/chia-gaming/test-last.clsp".to_string() + ]); + let result = do_basic_brun(&vec!["brun".to_string(), result_prog, "()".to_string()]).trim().to_string(); + assert_eq!(result, "101"); +} From cd03d6807fb68d9e5bcc07b198492968f13bf012 Mon Sep 17 00:00:00 2001 From: arty Date: Sun, 21 May 2023 17:46:18 -0700 Subject: [PATCH 020/117] fmt + clippy --- src/compiler/codegen.rs | 19 ++++++++----------- src/compiler/comptypes.rs | 7 +++---- src/compiler/evaluate.rs | 4 ++-- src/compiler/frontend.rs | 12 ++++++------ src/compiler/rename.rs | 4 +--- src/tests/classic/run.rs | 6 ++++-- 6 files changed, 24 insertions(+), 28 deletions(-) diff --git a/src/compiler/codegen.rs b/src/compiler/codegen.rs index 84982892f..33ae4e0c1 100644 --- a/src/compiler/codegen.rs +++ b/src/compiler/codegen.rs @@ -780,13 +780,14 @@ fn generate_let_args(_l: Srcloc, blist: Vec>) -> Vec> { blist.iter().map(|b| b.body.clone()).collect() } -pub fn hoist_assign_form( - letdata: &LetData, -) -> Result { +pub fn hoist_assign_form(letdata: &LetData) -> Result { // Topological sort of bindings. let sorted_spec = toposort( &letdata.bindings, - CompileErr(letdata.loc.clone(), "deadlock resolving binding order".to_string()), + CompileErr( + letdata.loc.clone(), + "deadlock resolving binding order".to_string(), + ), // Needs: What this binding relies on. |possible, b| { let mut need_set = HashSet::new(); @@ -852,7 +853,7 @@ pub fn hoist_assign_form( LetFormKind::Parallel, Box::new(LetData { bindings: end_bindings, - .. letdata.clone() + ..letdata.clone() }), ); @@ -862,7 +863,7 @@ pub fn hoist_assign_form( Box::new(LetData { bindings: binding_list, body: Rc::new(output_let), - .. letdata.clone() + ..letdata.clone() }), ) } @@ -967,11 +968,7 @@ pub fn hoist_body_let_binding( Ok((out_defuns, Rc::new(final_call))) } BodyForm::Let(LetFormKind::Assign, letdata) => { - hoist_body_let_binding( - outer_context, - args, - Rc::new(hoist_assign_form(letdata)?) - ) + hoist_body_let_binding(outer_context, args, Rc::new(hoist_assign_form(letdata)?)) } BodyForm::Call(l, list) => { let mut vres = Vec::new(); diff --git a/src/compiler/comptypes.rs b/src/compiler/comptypes.rs index 9b5e12aa5..bc657b0df 100644 --- a/src/compiler/comptypes.rs +++ b/src/compiler/comptypes.rs @@ -587,8 +587,7 @@ impl HelperForm { } fn compose_let(marker: &[u8], letdata: &LetData) -> Rc { - let translated_bindings: Vec> = - letdata.bindings.iter().map(|x| x.to_sexp()).collect(); + let translated_bindings: Vec> = letdata.bindings.iter().map(|x| x.to_sexp()).collect(); let bindings_cons = list_to_cons(letdata.loc.clone(), &translated_bindings); let translated_body = letdata.body.to_sexp(); let kw_loc = letdata.kw.clone().unwrap_or_else(|| letdata.loc.clone()); @@ -649,14 +648,14 @@ impl BodyForm { match self { BodyForm::Let(kind, letdata) => { if matches!(kind, LetFormKind::Assign) { - compose_assign(&letdata) + compose_assign(letdata) } else { let marker = if matches!(kind, LetFormKind::Sequential) { b"let*".to_vec() } else { b"let".to_vec() }; - compose_let(&marker, &letdata) + compose_let(&marker, letdata) } } BodyForm::Quoted(body) => Rc::new(SExp::Cons( diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index fd54d10a9..0ba8d68f1 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -1098,8 +1098,8 @@ impl<'info> Evaluator { &mut visited, prog_args, env, - Rc::new(hoist_assign_form(&letdata)?), - only_inline + Rc::new(hoist_assign_form(letdata)?), + only_inline, ) } BodyForm::Quoted(_) => Ok(body.clone()), diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index f4020cc9f..6d3473117 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -236,10 +236,10 @@ fn make_let_bindings( result.append(&mut rest_bindings); Ok(result) } - _ => err.clone() + _ => err.clone(), }) .unwrap_or_else(|| err.clone()), - _ => err + _ => err, } } @@ -312,11 +312,11 @@ fn handle_assign_form( LetFormKind::Assign, Box::new(LetData { loc: l.clone(), - kw: Some(l.clone()), - bindings: bindings, - inline_hint: inline_hint.clone(), + kw: Some(l), + bindings, + inline_hint, body: Rc::new(compiled_body), - }) + }), )) } diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index f920b62dd..8ef0b0fcf 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -284,9 +284,7 @@ fn rename_args_bodyform(b: &BodyForm) -> BodyForm { ) } - BodyForm::Let(LetFormKind::Assign, _letdata) => { - b.clone() - } + BodyForm::Let(LetFormKind::Assign, _letdata) => b.clone(), BodyForm::Quoted(e) => BodyForm::Quoted(e.clone()), BodyForm::Value(v) => BodyForm::Value(v.clone()), diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index 49e4c8532..986160dcf 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -930,8 +930,10 @@ fn test_assign_fancy_final_dot_rest() { "run".to_string(), "-i".to_string(), "resources/tests/chia-gaming".to_string(), - "resources/tests/chia-gaming/test-last.clsp".to_string() + "resources/tests/chia-gaming/test-last.clsp".to_string(), ]); - let result = do_basic_brun(&vec!["brun".to_string(), result_prog, "()".to_string()]).trim().to_string(); + let result = do_basic_brun(&vec!["brun".to_string(), result_prog, "()".to_string()]) + .trim() + .to_string(); assert_eq!(result, "101"); } From c65ff6d163925358e3cdaa803b8ead04534c05ef Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 23 May 2023 16:48:06 -0700 Subject: [PATCH 021/117] Add an object and a compiler opts setting for dialect. --- src/classic/clvm_tools/clvmc.rs | 28 ++++++++++++++++------------ src/classic/clvm_tools/cmds.rs | 6 +++--- src/compiler/compiler.rs | 12 +++++++++++- src/compiler/comptypes.rs | 10 ++++++++++ src/tests/classic/stage_2.rs | 8 +++++++- 5 files changed, 47 insertions(+), 17 deletions(-) diff --git a/src/classic/clvm_tools/clvmc.rs b/src/classic/clvm_tools/clvmc.rs index b2ac08bab..3f7112967 100644 --- a/src/classic/clvm_tools/clvmc.rs +++ b/src/classic/clvm_tools/clvmc.rs @@ -22,10 +22,8 @@ use crate::classic::platform::distutils::dep_util::newer; use crate::compiler::clvm::convert_to_clvm_rs; use crate::compiler::compiler::compile_file; -use crate::compiler::compiler::run_optimizer; -use crate::compiler::compiler::DefaultCompilerOpts; -use crate::compiler::comptypes::CompileErr; -use crate::compiler::comptypes::CompilerOpts; +use crate::compiler::compiler::{DefaultCompilerOpts, run_optimizer}; +use crate::compiler::comptypes::{AcceptedDialect, CompileErr, CompilerOpts}; use crate::compiler::runtypes::RunFailure; fn include_dialect( @@ -56,15 +54,19 @@ pub fn write_sym_output( .map(|_| ()) } -pub fn detect_modern(allocator: &mut Allocator, sexp: NodePtr) -> Option { +pub fn detect_modern(allocator: &mut Allocator, sexp: NodePtr) -> AcceptedDialect { let mut dialects = HashMap::new(); dialects.insert("*standard-cl-21*".as_bytes().to_vec(), 21); dialects.insert("*standard-cl-22*".as_bytes().to_vec(), 22); - proper_list(allocator, sexp, true).and_then(|l| { + let mut result = Default::default(); + + if let Some(l) = proper_list(allocator, sexp, true) { for elt in l.iter() { - if let Some(dialect) = detect_modern(allocator, *elt) { - return Some(dialect); + let detect_modern_result = detect_modern(allocator, *elt); + if detect_modern_result.stepping.is_some() { + result = detect_modern_result; + break; } match proper_list(allocator, *elt, true) { @@ -78,14 +80,15 @@ pub fn detect_modern(allocator: &mut Allocator, sexp: NodePtr) -> Option { } if let Some(dialect) = include_dialect(allocator, &dialects, &e) { - return Some(dialect); + result.stepping = Some(dialect); + break; } } } } + } - None - }) + result } pub fn compile_clvm_text( @@ -99,7 +102,8 @@ pub fn compile_clvm_text( let ir_src = read_ir(text).map_err(|s| EvalErr(allocator.null(), s.to_string()))?; let assembled_sexp = assemble_from_ir(allocator, Rc::new(ir_src))?; - if let Some(dialect) = detect_modern(allocator, assembled_sexp) { + let dialect = detect_modern(allocator, assembled_sexp); + if let Some(dialect) = dialect.stepping { let runner = Rc::new(DefaultProgramRunner::new()); let opts = opts.set_optimize(true).set_frontend_opt(dialect > 21); diff --git a/src/classic/clvm_tools/cmds.rs b/src/classic/clvm_tools/cmds.rs index 79eb746d7..dea8a5746 100644 --- a/src/classic/clvm_tools/cmds.rs +++ b/src/classic/clvm_tools/cmds.rs @@ -1145,9 +1145,9 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul .map(|a| matches!(a, ArgumentValue::ArgBool(true))) .unwrap_or(false); - let dialect = input_sexp.and_then(|i| detect_modern(&mut allocator, i)); + let dialect = input_sexp.map(|i| detect_modern(&mut allocator, i)); let mut stderr_output = |s: String| { - if dialect.is_some() { + if dialect.as_ref().and_then(|d| d.stepping).is_some() { eprintln!("{s}"); } else { stdout.write_str(&s); @@ -1183,7 +1183,7 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul .unwrap_or_else(|| "main.sym".to_string()); // In testing: short circuit for modern compilation. - if let Some(dialect) = dialect { + if let Some(dialect) = dialect.and_then(|d| d.stepping) { let do_optimize = parsed_args .get("optimize") .map(|x| matches!(x, ArgumentValue::ArgBool(true))) diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index 7639f88b5..0bab69b35 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -14,7 +14,7 @@ use crate::classic::clvm_tools::stages::stage_2::optimize::optimize_sexp; use crate::compiler::clvm::{convert_from_clvm_rs, convert_to_clvm_rs, sha256tree}; use crate::compiler::codegen::{codegen, hoist_body_let_binding, process_helper_let_bindings}; use crate::compiler::comptypes::{ - CompileErr, CompileForm, CompilerOpts, DefunData, HelperForm, PrimaryCodegen, + AcceptedDialect, CompileErr, CompileForm, CompilerOpts, DefunData, HelperForm, PrimaryCodegen, }; use crate::compiler::evaluate::{build_reflex_captures, Evaluator, EVAL_STACK_LIMIT}; use crate::compiler::frontend::frontend; @@ -74,6 +74,7 @@ pub struct DefaultCompilerOpts { pub frontend_check_live: bool, pub start_env: Option>, pub prim_map: Rc, Rc>>, + pub dialect: AcceptedDialect, known_dialects: Rc>, } @@ -228,6 +229,9 @@ impl CompilerOpts for DefaultCompilerOpts { fn code_generator(&self) -> Option { self.code_generator.clone() } + fn dialect(&self) -> AcceptedDialect { + self.dialect.clone() + } fn in_defun(&self) -> bool { self.in_defun } @@ -253,6 +257,11 @@ impl CompilerOpts for DefaultCompilerOpts { self.include_dirs.clone() } + fn set_dialect(&self, dialect: AcceptedDialect) -> Rc { + let mut copy = self.clone(); + copy.dialect = dialect; + Rc::new(copy) + } fn set_search_paths(&self, dirs: &[String]) -> Rc { let mut copy = self.clone(); copy.include_dirs = dirs.to_owned(); @@ -349,6 +358,7 @@ impl DefaultCompilerOpts { frontend_opt: false, frontend_check_live: true, start_env: None, + dialect: Default::default(), prim_map: create_prim_map(), known_dialects: Rc::new(KNOWN_DIALECTS.clone()), } diff --git a/src/compiler/comptypes.rs b/src/compiler/comptypes.rs index 0b7b89e1a..85851c647 100644 --- a/src/compiler/comptypes.rs +++ b/src/compiler/comptypes.rs @@ -30,6 +30,12 @@ impl From<(Srcloc, String)> for CompileErr { #[derive(Clone, Debug)] pub struct CompiledCode(pub Srcloc, pub Rc); +/// Specifying how the language is spoken. +#[derive(Clone, Debug, Default)] +pub struct AcceptedDialect { + pub stepping: Option, +} + /// A description of an inlined function for use during inline expansion. /// This is used only by PrimaryCodegen. #[derive(Clone, Debug)] @@ -309,6 +315,8 @@ pub trait CompilerOpts { /// complex constants, and into (com ...) forms. This allows the CompilerOpts /// to carry this info across boundaries into a new context. fn code_generator(&self) -> Option; + /// Get the dialect declared in the toplevel program. + fn dialect(&self) -> AcceptedDialect; /// Specifies whether code is being generated on behalf of an inner defun in /// the program. fn in_defun(&self) -> bool; @@ -333,6 +341,8 @@ pub trait CompilerOpts { /// Specifies the search paths we're carrying. fn get_search_paths(&self) -> Vec; + /// Set the dialect. + fn set_dialect(&self, dialect: AcceptedDialect) -> Rc; /// Set search paths. fn set_search_paths(&self, dirs: &[String]) -> Rc; /// Set whether we're compiling on behalf of a defun. diff --git a/src/tests/classic/stage_2.rs b/src/tests/classic/stage_2.rs index 0d80b005a..d7055c6d3 100644 --- a/src/tests/classic/stage_2.rs +++ b/src/tests/classic/stage_2.rs @@ -17,7 +17,7 @@ use crate::classic::clvm_tools::stages::stage_2::helpers::{brun, evaluate, quote use crate::classic::clvm_tools::stages::stage_2::operators::run_program_for_search_paths; use crate::classic::clvm_tools::stages::stage_2::reader::{process_embed_file, read_file}; -use crate::compiler::comptypes::{CompileErr, CompilerOpts, PrimaryCodegen}; +use crate::compiler::comptypes::{AcceptedDialect, CompileErr, CompilerOpts, PrimaryCodegen}; use crate::compiler::sexp::{decode_string, SExp}; use crate::compiler::srcloc::Srcloc; @@ -319,6 +319,9 @@ impl CompilerOpts for TestCompilerOptsPresentsOwnFiles { fn code_generator(&self) -> Option { None } + fn dialect(&self) -> AcceptedDialect { + Default::default() + } fn in_defun(&self) -> bool { false } @@ -343,6 +346,9 @@ impl CompilerOpts for TestCompilerOptsPresentsOwnFiles { fn get_search_paths(&self) -> Vec { vec![".".to_string()] } + fn set_dialect(&self, _dialect: AcceptedDialect) -> Rc { + Rc::new(self.clone()) + } fn set_search_paths(&self, _dirs: &[String]) -> Rc { Rc::new(self.clone()) } From 418824417fe73b9dd5949392e01824b3b2c5c139 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 23 May 2023 16:50:26 -0700 Subject: [PATCH 022/117] fmt + clippy --- src/classic/clvm_tools/clvmc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/classic/clvm_tools/clvmc.rs b/src/classic/clvm_tools/clvmc.rs index 3f7112967..e917ab9cb 100644 --- a/src/classic/clvm_tools/clvmc.rs +++ b/src/classic/clvm_tools/clvmc.rs @@ -22,7 +22,7 @@ use crate::classic::platform::distutils::dep_util::newer; use crate::compiler::clvm::convert_to_clvm_rs; use crate::compiler::compiler::compile_file; -use crate::compiler::compiler::{DefaultCompilerOpts, run_optimizer}; +use crate::compiler::compiler::{run_optimizer, DefaultCompilerOpts}; use crate::compiler::comptypes::{AcceptedDialect, CompileErr, CompilerOpts}; use crate::compiler::runtypes::RunFailure; From 5ad2e7d20bda8e88a0c64233bcf28f9b18f5ec7c Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 6 Jun 2023 11:08:14 -0700 Subject: [PATCH 023/117] Add new bls operators and an operator compatibility mode that ensures you can always get older versions of output from disassemble --- Cargo.lock | 121 +++++++++--------- Cargo.toml | 2 +- src/classic/clvm/mod.rs | 107 +++++++++++----- src/classic/clvm_tools/binutils.rs | 7 +- src/classic/clvm_tools/clvmc.rs | 4 +- src/classic/clvm_tools/cmds.rs | 79 ++++++++++-- src/classic/clvm_tools/debug.rs | 2 +- src/classic/clvm_tools/stages/stage_0.rs | 8 +- .../clvm_tools/stages/stage_2/compile.rs | 41 +++--- .../clvm_tools/stages/stage_2/module.rs | 2 +- .../clvm_tools/stages/stage_2/operators.rs | 43 +++++-- .../clvm_tools/stages/stage_2/optimize.rs | 26 ++-- src/compiler/compiler.rs | 10 ++ src/compiler/comptypes.rs | 4 + src/compiler/prims.rs | 50 +++++++- src/tests/classic/optimize.rs | 10 +- src/tests/classic/run.rs | 2 +- src/tests/classic/smoke.rs | 44 ++++--- src/tests/classic/stage_2.rs | 26 ++-- 19 files changed, 397 insertions(+), 191 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 182446923..ae1531e22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -42,23 +42,28 @@ dependencies = [ ] [[package]] -name = "block-buffer" -version = "0.10.3" +name = "bls12_381" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +checksum = "62250ece575fa9b22068b3a8d59586f01d426dd7785522efd97632959e71c986" dependencies = [ - "generic-array", + "ff 0.12.1", + "group 0.12.1", + "pairing 0.22.0", + "rand_core", + "subtle", ] [[package]] name = "bls12_381" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62250ece575fa9b22068b3a8d59586f01d426dd7785522efd97632959e71c986" +checksum = "d7bc6d6292be3a19e6379786dac800f551e5865a5bb51ebbe3064ab80433f403" dependencies = [ - "ff", - "group", - "pairing", + "digest", + "ff 0.13.0", + "group 0.13.0", + "pairing 0.23.0", "rand_core", "subtle", ] @@ -95,7 +100,7 @@ name = "clvm_tools_rs" version = "0.1.34" dependencies = [ "binascii", - "bls12_381", + "bls12_381 0.7.0", "bytestream", "clvmr", "derivative", @@ -116,7 +121,7 @@ dependencies = [ "rand_chacha", "serde", "serde_json", - "sha2 0.9.5", + "sha2", "tempfile", "unicode-segmentation", "wasm-bindgen", @@ -126,17 +131,18 @@ dependencies = [ [[package]] name = "clvmr" -version = "0.1.24" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5e907612d322d0d7def6b0ecb3ad681f6af2db106bcfabe4153746c60ef9e4" +checksum = "d234802ce73011e01f7019ef5701df1f9bffad9326d7d0a6dbca8d4b7591f083" dependencies = [ - "bls12_381", + "bls12_381 0.8.0", + "group 0.13.0", "hex", "lazy_static", "num-bigint", "num-integer", "num-traits", - "sha2 0.10.2", + "sha2", ] [[package]] @@ -149,15 +155,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "cpufeatures" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66c99696f6c9dd7f35d486b9d04d7e6e202aa3e8c40d553f2fdf5e7e0c6a71ef" -dependencies = [ - "libc", -] - [[package]] name = "cpufeatures" version = "0.2.5" @@ -167,16 +164,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - [[package]] name = "derivative" version = "2.2.0" @@ -197,16 +184,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "digest" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" -dependencies = [ - "block-buffer 0.10.3", - "crypto-common", -] - [[package]] name = "do-notation" version = "0.1.3" @@ -239,6 +216,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "ff" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded41244b729663b1e574f1b4fb731469f69f79c17667b5d776b16cda0479449" +dependencies = [ + "bitvec", + "rand_core", + "subtle", +] + [[package]] name = "funty" version = "2.0.0" @@ -274,7 +262,18 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" dependencies = [ - "ff", + "ff 0.12.1", + "rand_core", + "subtle", +] + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff 0.13.0", "rand_core", "subtle", ] @@ -470,7 +469,16 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135590d8bdba2b31346f9cd1fb2a912329f5135e832a4f422942eb6ead8b6b3b" dependencies = [ - "group", + "group 0.12.1", +] + +[[package]] +name = "pairing" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fec4625e73cf41ef4bb6846cafa6d44736525f442ba45e407c4a000a13996f" +dependencies = [ + "group 0.13.0", ] [[package]] @@ -709,28 +717,17 @@ dependencies = [ [[package]] name = "sha2" -version = "0.9.5" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12" +checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ - "block-buffer 0.9.0", + "block-buffer", "cfg-if", - "cpufeatures 0.1.5", - "digest 0.9.0", + "cpufeatures", + "digest", "opaque-debug", ] -[[package]] -name = "sha2" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.5", - "digest 0.10.6", -] - [[package]] name = "smallvec" version = "1.8.0" diff --git a/Cargo.toml b/Cargo.toml index 9aae5c82e..82387127e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ do-notation = "0.1.3" serde_json = "1.0" sha2 = "0.9.5" tempfile = "3.3.0" -clvmr = "0.1.24" +clvmr = { version = "0.2.5", features = ["pre-eval"] } binascii = "0.1.4" yaml-rust = "0.4" linked-hash-map = "0.5.6" diff --git a/src/classic/clvm/mod.rs b/src/classic/clvm/mod.rs index a0273739e..8a6a237f1 100644 --- a/src/classic/clvm/mod.rs +++ b/src/classic/clvm/mod.rs @@ -8,103 +8,152 @@ pub mod serialize; pub mod sexp; pub mod syntax_error; +pub const OPERATORS_LATEST_VERSION: usize = 1; + struct KwAtomPair { v: u8, n: &'static str, + version: usize } -const KW_PAIRS: [KwAtomPair; 32] = [ - KwAtomPair { v: 0x01, n: "q" }, - KwAtomPair { v: 0x02, n: "a" }, - KwAtomPair { v: 0x03, n: "i" }, - KwAtomPair { v: 0x04, n: "c" }, - KwAtomPair { v: 0x05, n: "f" }, - KwAtomPair { v: 0x06, n: "r" }, - KwAtomPair { v: 0x07, n: "l" }, - KwAtomPair { v: 0x08, n: "x" }, - KwAtomPair { v: 0x09, n: "=" }, - KwAtomPair { v: 0x0a, n: ">s" }, +const KW_PAIRS: [KwAtomPair; 44] = [ + KwAtomPair { v: 0x01, n: "q", version: 0 }, + KwAtomPair { v: 0x02, n: "a", version: 0 }, + KwAtomPair { v: 0x03, n: "i", version: 0 }, + KwAtomPair { v: 0x04, n: "c", version: 0 }, + KwAtomPair { v: 0x05, n: "f", version: 0 }, + KwAtomPair { v: 0x06, n: "r", version: 0 }, + KwAtomPair { v: 0x07, n: "l", version: 0 }, + KwAtomPair { v: 0x08, n: "x", version: 0 }, + KwAtomPair { v: 0x09, n: "=", version: 0 }, + KwAtomPair { v: 0x0a, n: ">s", version: 0 }, KwAtomPair { v: 0x0b, n: "sha256", + version: 0, }, KwAtomPair { v: 0x0c, n: "substr", + version: 0, }, KwAtomPair { v: 0x0d, n: "strlen", + version: 0, }, KwAtomPair { v: 0x0e, n: "concat", + version: 0, }, - KwAtomPair { v: 0x10, n: "+" }, - KwAtomPair { v: 0x11, n: "-" }, - KwAtomPair { v: 0x12, n: "*" }, - KwAtomPair { v: 0x13, n: "/" }, + KwAtomPair { v: 0x10, n: "+", version: 0 }, + KwAtomPair { v: 0x11, n: "-", version: 0 }, + KwAtomPair { v: 0x12, n: "*", version: 0 }, + KwAtomPair { v: 0x13, n: "/", version: 0 }, KwAtomPair { v: 0x14, n: "divmod", + version: 0, }, - KwAtomPair { v: 0x15, n: ">" }, - KwAtomPair { v: 0x16, n: "ash" }, - KwAtomPair { v: 0x17, n: "lsh" }, + KwAtomPair { v: 0x15, n: ">", version: 0 }, + KwAtomPair { v: 0x16, n: "ash", version: 0 }, + KwAtomPair { v: 0x17, n: "lsh", version: 0 }, KwAtomPair { v: 0x18, n: "logand", + version: 0, }, KwAtomPair { v: 0x19, n: "logior", + version: 0, }, KwAtomPair { v: 0x1a, n: "logxor", + version: 0, }, KwAtomPair { v: 0x1b, n: "lognot", + version: 0, }, KwAtomPair { v: 0x1d, n: "point_add", + version: 0, }, KwAtomPair { v: 0x1e, n: "pubkey_for_exp", + version: 0, }, - KwAtomPair { v: 0x20, n: "not" }, - KwAtomPair { v: 0x21, n: "any" }, - KwAtomPair { v: 0x22, n: "all" }, + KwAtomPair { v: 0x20, n: "not", version: 0 }, + KwAtomPair { v: 0x21, n: "any", version: 0 }, + KwAtomPair { v: 0x22, n: "all", version: 0 }, KwAtomPair { v: 0x24, n: "softfork", + version: 0, }, + KwAtomPair { v: 0x30, n: "coinid", version: 1 }, + KwAtomPair { v: 0x31, n: "bls_g1_subtract", version: 1 }, + KwAtomPair { v: 0x32, n: "bls_g1_multiply", version: 1 }, + KwAtomPair { v: 0x33, n: "bls_g1_negate", version: 1 }, + KwAtomPair { v: 0x34, n: "bls_g2_add", version: 1 }, + KwAtomPair { v: 0x35, n: "bls_g2_subtract", version: 1 }, + KwAtomPair { v: 0x36, n: "bls_g2_multiply", version: 1 }, + KwAtomPair { v: 0x37, n: "bls_g2_negate", version: 1 }, + KwAtomPair { v: 0x38, n: "bls_map_to_g1", version: 1 }, + KwAtomPair { v: 0x39, n: "bls_map_to_g2", version: 1 }, + KwAtomPair { v: 0x3a, n: "bls_pairing_identity", version: 1 }, + KwAtomPair { v: 0x3b, n: "bls_verify", version: 1 }, ]; lazy_static! { - pub static ref KEYWORD_FROM_ATOM_: HashMap, String> = { + pub static ref KEYWORD_FROM_ATOM_0: HashMap, String> = { + let mut result = HashMap::new(); + for pair in KW_PAIRS.iter().filter(|p| p.version == 0) { + result.insert(vec![pair.v], pair.n.to_string()); + } + result + }; + pub static ref KEYWORD_TO_ATOM_0: HashMap> = { + let mut result = HashMap::new(); + for pair in KW_PAIRS.iter().filter(|p| p.version == 0) { + result.insert(pair.n.to_string(), vec![pair.v]); + } + result + }; + pub static ref KEYWORD_FROM_ATOM_1: HashMap, String> = { let mut result = HashMap::new(); - for pair in KW_PAIRS { + for pair in KW_PAIRS.iter().filter(|p| p.version <= 1) { result.insert(vec![pair.v], pair.n.to_string()); } result }; - pub static ref KEYWORD_TO_ATOM_: HashMap> = { + pub static ref KEYWORD_TO_ATOM_1: HashMap> = { let mut result = HashMap::new(); - for pair in KW_PAIRS { + for pair in KW_PAIRS.iter().filter(|p| p.version <= 1) { result.insert(pair.n.to_string(), vec![pair.v]); } result }; } -pub fn keyword_from_atom() -> &'static HashMap, String> { - &KEYWORD_FROM_ATOM_ +pub fn keyword_from_atom(version: usize) -> &'static HashMap, String> { + if version == 0 { + &KEYWORD_FROM_ATOM_0 + } else { + &KEYWORD_FROM_ATOM_1 + } } -pub fn keyword_to_atom() -> &'static HashMap> { - &KEYWORD_TO_ATOM_ +pub fn keyword_to_atom(version: usize) -> &'static HashMap> { + if version == 0 { + &KEYWORD_TO_ATOM_0 + } else { + &KEYWORD_TO_ATOM_1 + } } diff --git a/src/classic/clvm_tools/binutils.rs b/src/classic/clvm_tools/binutils.rs index 9e5e160ff..a6e331a5f 100644 --- a/src/classic/clvm_tools/binutils.rs +++ b/src/classic/clvm_tools/binutils.rs @@ -7,6 +7,7 @@ use unicode_segmentation::UnicodeSegmentation; use clvm_rs::allocator::{Allocator, NodePtr, SExp}; use clvm_rs::reduction::EvalErr; +use crate::classic::clvm::OPERATORS_LATEST_VERSION; use crate::classic::clvm::__type_compatibility__::{Bytes, BytesFromType, Record, Stream}; use crate::classic::clvm::{keyword_from_atom, keyword_to_atom}; use crate::classic::clvm_tools::ir::r#type::IRRepr; @@ -39,7 +40,7 @@ pub fn assemble_from_ir( s_real_name = stripped.to_string(); } - match keyword_to_atom().get(&s_real_name) { + match keyword_to_atom(OPERATORS_LATEST_VERSION).get(&s_real_name) { Some(v) => allocator.new_atom(v), None => { let v: Vec = s_real_name.as_bytes().to_vec(); @@ -144,8 +145,8 @@ pub fn disassemble_with_kw( write_ir(Rc::new(symbols)) } -pub fn disassemble(allocator: &mut Allocator, sexp: NodePtr) -> String { - return disassemble_with_kw(allocator, sexp, keyword_from_atom()); +pub fn disassemble(allocator: &mut Allocator, sexp: NodePtr, version: Option) -> String { + return disassemble_with_kw(allocator, sexp, keyword_from_atom(version.unwrap_or(OPERATORS_LATEST_VERSION))); } pub fn assemble(allocator: &mut Allocator, s: &str) -> Result { diff --git a/src/classic/clvm_tools/clvmc.rs b/src/classic/clvm_tools/clvmc.rs index 3e91ae099..35c98449b 100644 --- a/src/classic/clvm_tools/clvmc.rs +++ b/src/classic/clvm_tools/clvmc.rs @@ -137,13 +137,13 @@ pub fn compile_clvm_inner( ) -> Result<(), String> { let result = compile_clvm_text( allocator, - opts, + opts.clone(), symbol_table, text, filename, classic_with_opts, ) - .map_err(|x| format!("error {} compiling {}", x.1, disassemble(allocator, x.0)))?; + .map_err(|x| format!("error {} compiling {}", x.1, disassemble(allocator, x.0, opts.disassembly_ver())))?; sexp_to_stream(allocator, result, result_stream); Ok(()) } diff --git a/src/classic/clvm_tools/cmds.rs b/src/classic/clvm_tools/cmds.rs index 79eb746d7..c490b8e70 100644 --- a/src/classic/clvm_tools/cmds.rs +++ b/src/classic/clvm_tools/cmds.rs @@ -20,6 +20,7 @@ use clvm_rs::allocator::{Allocator, NodePtr}; use clvm_rs::reduction::EvalErr; use clvm_rs::run_program::PreEval; +use crate::classic::clvm::OPERATORS_LATEST_VERSION; use crate::classic::clvm::__type_compatibility__::{ t, Bytes, BytesFromType, Stream, Tuple, UnvalidatedBytesFromType, }; @@ -77,7 +78,7 @@ fn get_tool_description(tool_name: &str) -> Option { } else if tool_name == "opd" { Some(ConversionDesc { desc: "Disassemble a compiled clvm script from hex.", - conv: Box::new(OpdConversion {}), + conv: Box::new(OpdConversion { op_version: None }), }) } else { None @@ -102,6 +103,11 @@ impl ArgumentValueConv for PathOrCodeConv { // } pub trait TConversion { + fn apply_args( + &mut self, + parsed_args: &HashMap + ); + fn invoke( &self, allocator: &mut Allocator, @@ -130,7 +136,7 @@ pub fn call_tool( tool_name: &str, input_args: &[String], ) -> Result<(), String> { - let task = + let mut task = get_tool_description(tool_name).ok_or_else(|| format!("unknown tool {tool_name}"))?; let props = TArgumentParserProps { description: task.desc.to_string(), @@ -150,6 +156,12 @@ pub fn call_tool( .set_action(TArgOptionAction::StoreTrue) .set_help("Show only sha256 tree hash of program".to_string()), ); + parser.add_argument( + vec!["--operators-version".to_string()], + Argument::new() + .set_type(Rc::new(OperatorsVersion {})) + .set_default(ArgumentValue::ArgInt(OPERATORS_LATEST_VERSION as i64)), + ); parser.add_argument( vec!["path_or_code".to_string()], Argument::new() @@ -168,6 +180,8 @@ pub fn call_tool( } }; + task.conv.apply_args(&args); + if args.contains_key("version") { let version = version(); println!("{version}"); @@ -214,6 +228,11 @@ pub fn call_tool( pub struct OpcConversion {} impl TConversion for OpcConversion { + fn apply_args( + &mut self, + _args: &HashMap + ) { } + fn invoke( &self, allocator: &mut Allocator, @@ -228,9 +247,21 @@ impl TConversion for OpcConversion { } } -pub struct OpdConversion {} +#[derive(Debug)] +pub struct OpdConversion { + pub op_version: Option +} impl TConversion for OpdConversion { + fn apply_args( + &mut self, + args: &HashMap + ) { + if let Some(ArgumentValue::ArgInt(i)) = args.get("operators_version") { + self.op_version = Some(*i as usize); + } + } + fn invoke( &self, allocator: &mut Allocator, @@ -246,7 +277,7 @@ impl TConversion for OpdConversion { sexp_from_stream(allocator, &mut stream, Box::new(SimpleCreateCLVMObject {})) .map_err(|e| e.1) .map(|sexp| { - let disassembled = disassemble(allocator, sexp.1); + let disassembled = disassemble(allocator, sexp.1, self.op_version); t(sexp.1, disassembled) }) } @@ -277,6 +308,15 @@ impl ArgumentValueConv for StageImport { } } +struct OperatorsVersion {} + +impl ArgumentValueConv for OperatorsVersion { + fn convert(&self, arg: &str) -> Result { + let ver = arg.parse::().map_err(|_| format!("expected number 0-1 found {arg}"))?; + Ok(ArgumentValue::ArgInt(ver)) + } +} + pub fn run(args: &[String]) { let mut s = Stream::new(None); launch_tool(&mut s, args, "run", 2); @@ -772,6 +812,14 @@ fn fix_log( } } +fn get_disassembly_ver(p: &HashMap) -> Option { + if let Some(ArgumentValue::ArgInt(x)) = p.get("operators_version") { + return Some(*x as usize); + } + + None +} + pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, default_stage: u32) { let props = TArgumentParserProps { description: "Execute a clvm script.".to_string(), @@ -910,6 +958,12 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul .set_type(Rc::new(PathJoin {})) .set_default(ArgumentValue::ArgString(None, "main.sym".to_string())), ); + parser.add_argument( + vec!["--operators-version".to_string()], + Argument::new() + .set_type(Rc::new(OperatorsVersion {})) + .set_default(ArgumentValue::ArgInt(OPERATORS_LATEST_VERSION as i64)), + ); if tool_name == "run" { parser.add_argument( @@ -939,9 +993,8 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul let empty_map = HashMap::new(); let keywords = match parsed_args.get("no_keywords") { - None => keyword_from_atom(), Some(ArgumentValue::ArgBool(_b)) => &empty_map, - _ => keyword_from_atom(), + _ => keyword_from_atom(get_disassembly_ver(&parsed_args).unwrap_or(OPERATORS_LATEST_VERSION)), }; // If extra symbol output is desired (not all keys are hashes, but there's @@ -1193,7 +1246,8 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul let opts = Rc::new(DefaultCompilerOpts::new(&use_filename)) .set_optimize(do_optimize) .set_search_paths(&search_paths) - .set_frontend_opt(dialect > 21); + .set_frontend_opt(dialect > 21) + .set_disassembly_ver(get_disassembly_ver(&parsed_args)); let mut symbol_table = HashMap::new(); let unopt_res = compile_file( @@ -1445,6 +1499,9 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul ) })); + // Get the disassembly ver we're using based on the user's request. + let disassembly_ver = get_disassembly_ver(&parsed_args); + let compile_sym_out = dpr.get_compiles(); if !compile_sym_out.is_empty() { write_sym_output(&compile_sym_out, &symbol_table_output).ok(); @@ -1477,7 +1534,9 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul only_exn, &log_content, symbol_table, - &disassemble, + &|allocator, p| { + disassemble(allocator, p, disassembly_ver) + } ); } else { stdout.write_str("\n"); @@ -1487,7 +1546,9 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul only_exn, &log_content, symbol_table, - &disassemble, + &|allocator, p| { + disassemble(allocator, p, disassembly_ver) + } ); } } diff --git a/src/classic/clvm_tools/debug.rs b/src/classic/clvm_tools/debug.rs index 761a5640f..20c088c93 100644 --- a/src/classic/clvm_tools/debug.rs +++ b/src/classic/clvm_tools/debug.rs @@ -126,7 +126,7 @@ pub fn build_symbol_dump( let args_name_atom = allocator.new_atom(&args_atom)?; let left_env_name_atom = allocator.new_atom(&left_env_atom)?; - let serialized_args = disassemble(allocator, extra.args); + let serialized_args = disassemble(allocator, extra.args, Some(0)); let serialized_args_atom = allocator.new_atom(serialized_args.as_bytes())?; let left_env_value = allocator.new_atom(&[extra.has_constants_tree as u8])?; diff --git a/src/classic/clvm_tools/stages/stage_0.rs b/src/classic/clvm_tools/stages/stage_0.rs index a447b90bb..80f6d6da6 100644 --- a/src/classic/clvm_tools/stages/stage_0.rs +++ b/src/classic/clvm_tools/stages/stage_0.rs @@ -1,9 +1,9 @@ use clvm_rs::allocator::{Allocator, NodePtr}; -use clvm_rs::chia_dialect::{ChiaDialect, NO_NEG_DIV, NO_UNKNOWN_OPS}; +use clvm_rs::chia_dialect::{ChiaDialect, NO_UNKNOWN_OPS, ENABLE_BLS_OPS}; use clvm_rs::cost::Cost; use clvm_rs::reduction::Response; -use clvm_rs::run_program::{run_program, PreEval}; +use clvm_rs::run_program::{run_program_with_pre_eval, PreEval}; pub struct RunProgramOption { pub max_cost: Option, @@ -45,9 +45,9 @@ impl TRunProgram for DefaultProgramRunner { ) -> Response { let max_cost = option.as_ref().and_then(|o| o.max_cost).unwrap_or(0); - run_program( + run_program_with_pre_eval( allocator, - &ChiaDialect::new(NO_NEG_DIV | NO_UNKNOWN_OPS), + &ChiaDialect::new(NO_UNKNOWN_OPS | ENABLE_BLS_OPS), program, args, max_cost, diff --git a/src/classic/clvm_tools/stages/stage_2/compile.rs b/src/classic/clvm_tools/stages/stage_2/compile.rs index a1866bbda..6501e5dec 100644 --- a/src/classic/clvm_tools/stages/stage_2/compile.rs +++ b/src/classic/clvm_tools/stages/stage_2/compile.rs @@ -4,6 +4,7 @@ use std::rc::Rc; use clvm_rs::allocator::{Allocator, AtomBuf, NodePtr, SExp}; use clvm_rs::reduction::{EvalErr, Reduction, Response}; +use crate::classic::clvm::OPERATORS_LATEST_VERSION; use crate::classic::clvm::__type_compatibility__::{Bytes, BytesFromType}; use crate::classic::clvm::sexp::{enlist, first, map_m, non_nil, proper_list, rest}; use crate::classic::clvm::{keyword_from_atom, keyword_to_atom}; @@ -20,10 +21,10 @@ const DIAG_OUTPUT: bool = false; lazy_static! { static ref PASS_THROUGH_OPERATORS: HashSet> = { let mut result = HashSet::new(); - for key in keyword_to_atom().keys() { + for key in keyword_to_atom(OPERATORS_LATEST_VERSION).keys() { result.insert(key.as_bytes().to_vec()); } - for key in keyword_from_atom().keys() { + for key in keyword_from_atom(OPERATORS_LATEST_VERSION).keys() { result.insert(key.to_vec()); } // added by optimize @@ -94,7 +95,7 @@ fn com_qq( sexp: NodePtr, ) -> Result { if DIAG_OUTPUT { - println!("com_qq {} {}", ident, disassemble(allocator, sexp)); + println!("com_qq {} {}", ident, disassemble(allocator, sexp, None)); } do_com_prog(allocator, 110, sexp, macro_lookup, symbol_table, runner).map(|x| x.1) } @@ -210,7 +211,7 @@ fn lower_quote_(allocator: &mut Allocator, prog: NodePtr) -> Result Result Result { let entry_name = allocator.new_atom("__chia__main_arguments".as_bytes())?; - let entry_value_string = disassemble(allocator, args); + let entry_value_string = disassemble(allocator, args, None); let entry_value = allocator.new_atom(entry_value_string.as_bytes())?; let entry_cons = allocator.new_pair(entry_name, entry_value)?; allocator.new_pair(entry_cons, symbols) diff --git a/src/classic/clvm_tools/stages/stage_2/operators.rs b/src/classic/clvm_tools/stages/stage_2/operators.rs index ae202bf67..11f97b704 100644 --- a/src/classic/clvm_tools/stages/stage_2/operators.rs +++ b/src/classic/clvm_tools/stages/stage_2/operators.rs @@ -5,12 +5,13 @@ use std::path::PathBuf; use std::rc::Rc; use clvm_rs::allocator::{Allocator, NodePtr, SExp}; -use clvm_rs::chia_dialect::{ChiaDialect, NO_NEG_DIV, NO_UNKNOWN_OPS}; +use clvm_rs::chia_dialect::{ChiaDialect, NO_UNKNOWN_OPS, ENABLE_BLS_OPS}; use clvm_rs::cost::Cost; -use clvm_rs::dialect::Dialect; +use clvm_rs::dialect::{Dialect, OperatorSet}; use clvm_rs::reduction::{EvalErr, Reduction, Response}; -use clvm_rs::run_program::run_program; +use clvm_rs::run_program::run_program_with_pre_eval; +use crate::classic::clvm::OPERATORS_LATEST_VERSION; use crate::classic::clvm::__type_compatibility__::{Bytes, BytesFromType, Stream}; use crate::classic::clvm::keyword_from_atom; @@ -118,7 +119,7 @@ impl Drop for CompilerOperators { impl CompilerOperatorsInternal { pub fn new(source_file: &str, search_paths: Vec, symbols_extra_info: bool) -> Self { - let base_dialect = Rc::new(ChiaDialect::new(NO_NEG_DIV | NO_UNKNOWN_OPS)); + let base_dialect = Rc::new(ChiaDialect::new(NO_UNKNOWN_OPS | ENABLE_BLS_OPS)); let base_runner = Rc::new(DefaultProgramRunner::new()); CompilerOperatorsInternal { base_dialect, @@ -213,7 +214,7 @@ impl CompilerOperatorsInternal { let filename_buf = allocator.buf(&filename_buf); let filename_bytes = Bytes::new(Some(BytesFromType::Raw(filename_buf.to_vec()))); - let ir = disassemble_to_ir_with_kw(allocator, data, keyword_from_atom(), true); + let ir = disassemble_to_ir_with_kw(allocator, data, keyword_from_atom(self.get_disassembly_ver()), true); let mut stream = Stream::new(None); write_ir_to_stream(Rc::new(ir), &mut stream); return fs::write(filename_bytes.decode(), stream.get_value().decode()) @@ -306,10 +307,16 @@ impl CompilerOperatorsInternal { Ok(Reduction(1, allocator.null())) } + + fn get_disassembly_ver(&self) -> usize { + self.get_compiler_opts() + .and_then(|o| o.disassembly_ver()) + .unwrap_or(OPERATORS_LATEST_VERSION) + } } impl Dialect for CompilerOperatorsInternal { - fn val_stack_limit(&self) -> usize { + fn stack_limit(&self) -> usize { 10000000 } fn quote_kw(&self) -> &[u8] { @@ -319,12 +326,28 @@ impl Dialect for CompilerOperatorsInternal { &[2] } + fn softfork_kw(&self) -> &[u8] { + &[36] + } + + // The softfork operator comes with an extension argument. + fn softfork_extension(&self, ext: u32) -> OperatorSet { + match ext { + 0 => { + OperatorSet::BLS + } + // new extensions go here + _ => OperatorSet::Default, + } + } + fn op( &self, allocator: &mut Allocator, op: NodePtr, sexp: NodePtr, max_cost: Cost, + _extension: OperatorSet ) -> Response { match allocator.sexp(op) { SExp::Atom(opname) => { @@ -350,12 +373,14 @@ impl Dialect for CompilerOperatorsInternal { } else if opbuf == "_get_source_file".as_bytes() { self.get_source_file(allocator) } else { - self.base_dialect.op(allocator, op, sexp, max_cost) + self.base_dialect.op(allocator, op, sexp, max_cost, OperatorSet::BLS) } } - _ => self.base_dialect.op(allocator, op, sexp, max_cost), + _ => self.base_dialect.op(allocator, op, sexp, max_cost, OperatorSet::BLS), } } + + fn allow_unknown_ops(&self) -> bool { false } } impl CompilerOperatorsInternal { @@ -383,7 +408,7 @@ impl TRunProgram for CompilerOperatorsInternal { option: Option, ) -> Response { let max_cost = option.as_ref().and_then(|o| o.max_cost).unwrap_or(0); - run_program( + run_program_with_pre_eval( allocator, self, program, diff --git a/src/classic/clvm_tools/stages/stage_2/optimize.rs b/src/classic/clvm_tools/stages/stage_2/optimize.rs index d723e279f..b1820661e 100644 --- a/src/classic/clvm_tools/stages/stage_2/optimize.rs +++ b/src/classic/clvm_tools/stages/stage_2/optimize.rs @@ -106,7 +106,7 @@ pub fn constant_optimizer( if DIAG_OPTIMIZATIONS { println!( "COPT {} SC_R {} NN_R {}", - disassemble(allocator, r), + disassemble(allocator, r, None), sc_r, nn_r ); @@ -123,8 +123,8 @@ pub fn constant_optimizer( let _ = if DIAG_OPTIMIZATIONS { println!( "CONSTANT_OPTIMIZER {} TO {}", - disassemble(allocator, r), - disassemble(allocator, r1) + disassemble(allocator, r, None), + disassemble(allocator, r1, None) ); }; quoted <- quote(allocator, r1); @@ -311,7 +311,7 @@ pub fn var_change_optimizer_cons_eval( if DIAG_OPTIMIZATIONS { println!( "XXX ORIGINAL_ARGS {}", - disassemble(allocator, *original_args) + disassemble(allocator, *original_args, None) ); }; let original_call = t1 @@ -321,7 +321,7 @@ pub fn var_change_optimizer_cons_eval( if DIAG_OPTIMIZATIONS { println!( "XXX ORIGINAL_CALL {}", - disassemble(allocator, *original_call) + disassemble(allocator, *original_call, None) ); }; @@ -330,8 +330,8 @@ pub fn var_change_optimizer_cons_eval( if DIAG_OPTIMIZATIONS { println!( "XXX new_eval_sexp_args {} ORIG {}", - disassemble(allocator, new_eval_sexp_args), - disassemble(allocator, *original_args) + disassemble(allocator, new_eval_sexp_args, None), + disassemble(allocator, *original_args, None) ); }; @@ -365,7 +365,7 @@ pub fn var_change_optimizer_cons_eval( println!( "XXX opt_operands {} {}", acc, - disassemble(allocator, val) + disassemble(allocator, val, None) ); } let increment = match allocator.sexp(val) { @@ -713,8 +713,8 @@ pub fn optimize_sexp_( println!( "OPT-{:?}[{}] => {}", name, - disassemble(allocator, start_r), - disassemble(allocator, r) + disassemble(allocator, start_r, None), + disassemble(allocator, r, None) ); } } @@ -730,14 +730,14 @@ pub fn optimize_sexp( let optimized = RefCell::new(HashMap::new()); if DIAG_OPTIMIZATIONS { - println!("START OPTIMIZE {}", disassemble(allocator, r)); + println!("START OPTIMIZE {}", disassemble(allocator, r, None)); } optimize_sexp_(allocator, &optimized, r, eval_f).map(|x| { if DIAG_OPTIMIZATIONS { println!( "OPTIMIZE_SEXP {} GIVING {}", - disassemble(allocator, r), - disassemble(allocator, x) + disassemble(allocator, r, None), + disassemble(allocator, x, None) ); } x diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index 7639f88b5..10aa3f38b 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -73,6 +73,7 @@ pub struct DefaultCompilerOpts { pub frontend_opt: bool, pub frontend_check_live: bool, pub start_env: Option>, + pub disassembly_ver: Option, pub prim_map: Rc, Rc>>, known_dialects: Rc>, @@ -249,6 +250,9 @@ impl CompilerOpts for DefaultCompilerOpts { fn prim_map(&self) -> Rc, Rc>> { self.prim_map.clone() } + fn disassembly_ver(&self) -> Option { + self.disassembly_ver + } fn get_search_paths(&self) -> Vec { self.include_dirs.clone() } @@ -258,6 +262,11 @@ impl CompilerOpts for DefaultCompilerOpts { copy.include_dirs = dirs.to_owned(); Rc::new(copy) } + fn set_disassembly_ver(&self, ver: Option) -> Rc { + let mut copy = self.clone(); + copy.disassembly_ver = ver; + Rc::new(copy) + } fn set_in_defun(&self, new_in_defun: bool) -> Rc { let mut copy = self.clone(); copy.in_defun = new_in_defun; @@ -350,6 +359,7 @@ impl DefaultCompilerOpts { frontend_check_live: true, start_env: None, prim_map: create_prim_map(), + disassembly_ver: None, known_dialects: Rc::new(KNOWN_DIALECTS.clone()), } } diff --git a/src/compiler/comptypes.rs b/src/compiler/comptypes.rs index 0b7b89e1a..ced4829c5 100644 --- a/src/compiler/comptypes.rs +++ b/src/compiler/comptypes.rs @@ -309,6 +309,8 @@ pub trait CompilerOpts { /// complex constants, and into (com ...) forms. This allows the CompilerOpts /// to carry this info across boundaries into a new context. fn code_generator(&self) -> Option; + /// Disassembly version (for disassembly style serialization) + fn disassembly_ver(&self) -> Option; /// Specifies whether code is being generated on behalf of an inner defun in /// the program. fn in_defun(&self) -> bool; @@ -335,6 +337,8 @@ pub trait CompilerOpts { /// Set search paths. fn set_search_paths(&self, dirs: &[String]) -> Rc; + /// Set disassembly version for. + fn set_disassembly_ver(&self, ver: Option) -> Rc; /// Set whether we're compiling on behalf of a defun. fn set_in_defun(&self, new_in_defun: bool) -> Rc; /// Set whether to inject the standard environment. diff --git a/src/compiler/prims.rs b/src/compiler/prims.rs index e8cc04f71..83f50e8ff 100644 --- a/src/compiler/prims.rs +++ b/src/compiler/prims.rs @@ -137,7 +137,55 @@ pub fn prims() -> Vec<(Vec, SExp)> { ), ( "softfork".as_bytes().to_vec(), - SExp::Integer(primloc, 36_u32.to_bigint().unwrap()), + SExp::Integer(primloc.clone(), 36_u32.to_bigint().unwrap()), + ), + ( + "coinid".as_bytes().to_vec(), + SExp::Integer(primloc.clone(), 48_u32.to_bigint().unwrap()), + ), + ( + "bls_g1_subtract".as_bytes().to_vec(), + SExp::Integer(primloc.clone(), 49_u32.to_bigint().unwrap()), + ), + ( + "bls_g1_multiply".as_bytes().to_vec(), + SExp::Integer(primloc.clone(), 50_u32.to_bigint().unwrap()), + ), + ( + "bls_g1_negate".as_bytes().to_vec(), + SExp::Integer(primloc.clone(), 51_u32.to_bigint().unwrap()), + ), + ( + "bls_g2_add".as_bytes().to_vec(), + SExp::Integer(primloc.clone(), 52_u32.to_bigint().unwrap()), + ), + ( + "bls_g2_subtract".as_bytes().to_vec(), + SExp::Integer(primloc.clone(), 53_u32.to_bigint().unwrap()), + ), + ( + "bls_g2_multiply".as_bytes().to_vec(), + SExp::Integer(primloc.clone(), 54_u32.to_bigint().unwrap()), + ), + ( + "bls_g2_negate".as_bytes().to_vec(), + SExp::Integer(primloc.clone(), 55_u32.to_bigint().unwrap()), + ), + ( + "bls_map_to_g1".as_bytes().to_vec(), + SExp::Integer(primloc.clone(), 56_u32.to_bigint().unwrap()), + ), + ( + "bls_map_to_g2".as_bytes().to_vec(), + SExp::Integer(primloc.clone(), 57_u32.to_bigint().unwrap()), + ), + ( + "bls_pairing_identity".as_bytes().to_vec(), + SExp::Integer(primloc.clone(), 58_u32.to_bigint().unwrap()), + ), + ( + "bls_verify".as_bytes().to_vec(), + SExp::Integer(primloc.clone(), 59_u32.to_bigint().unwrap()), ), ] } diff --git a/src/tests/classic/optimize.rs b/src/tests/classic/optimize.rs index 48608956e..b424ce1db 100644 --- a/src/tests/classic/optimize.rs +++ b/src/tests/classic/optimize.rs @@ -19,7 +19,7 @@ fn test_cons_q_a(src: String) -> String { let assembled = assemble_from_ir(&mut allocator, Rc::new(input_ir)).unwrap(); let runner = run_program_for_search_paths("*test*", &vec![".".to_string()], false); let optimized = cons_q_a_optimizer(&mut allocator, &memo, assembled, runner.clone()).unwrap(); - disassemble(&mut allocator, optimized) + disassemble(&mut allocator, optimized, Some(0)) } fn test_children_optimizer(src: String) -> String { @@ -29,7 +29,7 @@ fn test_children_optimizer(src: String) -> String { let assembled = assemble_from_ir(&mut allocator, Rc::new(input_ir)).unwrap(); let runner = run_program_for_search_paths("*test*", &vec![".".to_string()], false); let optimized = children_optimizer(&mut allocator, &memo, assembled, runner.clone()).unwrap(); - disassemble(&mut allocator, optimized) + disassemble(&mut allocator, optimized, Some(0)) } fn test_constant_optimizer(src: String) -> String { @@ -40,7 +40,7 @@ fn test_constant_optimizer(src: String) -> String { let runner = run_program_for_search_paths("*test*", &vec![".".to_string()], false); let optimized = constant_optimizer(&mut allocator, &memo, assembled, 0, runner.clone()).unwrap(); - disassemble(&mut allocator, optimized) + disassemble(&mut allocator, optimized, Some(0)) } fn test_optimizer(src: String) -> String { @@ -49,7 +49,7 @@ fn test_optimizer(src: String) -> String { let assembled = assemble_from_ir(&mut allocator, Rc::new(input_ir)).unwrap(); let runner = run_program_for_search_paths("*test*", &vec![".".to_string()], false); let optimized = optimize_sexp(&mut allocator, assembled, runner.clone()).unwrap(); - disassemble(&mut allocator, optimized) + disassemble(&mut allocator, optimized, Some(0)) } fn test_sub_args(src: String) -> String { @@ -59,7 +59,7 @@ fn test_sub_args(src: String) -> String { match allocator.sexp(assembled) { SExp::Pair(a, b) => { let optimized = sub_args(&mut allocator, a, b).unwrap(); - disassemble(&mut allocator, optimized) + disassemble(&mut allocator, optimized, Some(0)) } _ => { panic!("assembled a list got an atom"); diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index 6a849743a..a15814de5 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -746,7 +746,7 @@ fn test_check_tricky_arg_path_random() { Rc::new(sexp::SExp::Atom(random_tree.loc(), k.clone())), ) .unwrap(); - let disassembled = disassemble(&mut allocator, converted); + let disassembled = disassemble(&mut allocator, converted, Some(0)); eprintln!("run {} want {} have {}", program, disassembled, res); assert_eq!(disassembled, res); } diff --git a/src/tests/classic/smoke.rs b/src/tests/classic/smoke.rs index e7a7d6639..473ea9997 100644 --- a/src/tests/classic/smoke.rs +++ b/src/tests/classic/smoke.rs @@ -45,10 +45,12 @@ fn large_odd_sized_neg_opc() { assert_eq!(result.rest(), "ff8afde1e61f36454dc0000180"); } +fn opd_conversion() -> OpdConversion { OpdConversion { op_version: Some(0) } } + #[test] fn large_odd_sized_neg_opd() { let mut allocator = Allocator::new(); - let result = OpdConversion {} + let result = opd_conversion() .invoke(&mut allocator, &"ff8afde1e61f36454dc0000180".to_string()) .unwrap(); assert_eq!(result.rest(), "(0xfde1e61f36454dc00001)"); @@ -57,7 +59,7 @@ fn large_odd_sized_neg_opd() { #[test] fn mid_negative_value_opd_m1() { let mut allocator = Allocator::new(); - let result = OpdConversion {} + let result = opd_conversion() .invoke(&mut allocator, &"81ff".to_string()) .unwrap(); assert_eq!(result.rest(), "-1"); @@ -66,7 +68,7 @@ fn mid_negative_value_opd_m1() { #[test] fn mid_negative_value_opd_m2() { let mut allocator = Allocator::new(); - let result = OpdConversion {} + let result = opd_conversion() .invoke(&mut allocator, &"81fe".to_string()) .unwrap(); assert_eq!(result.rest(), "-2"); @@ -75,7 +77,7 @@ fn mid_negative_value_opd_m2() { #[test] fn mid_negative_value_opd_two_bytes() { let mut allocator = Allocator::new(); - let result = OpdConversion {} + let result = opd_conversion() .invoke(&mut allocator, &"82ffff".to_string()) .unwrap(); assert_eq!(result.rest(), "0xffff"); @@ -84,7 +86,7 @@ fn mid_negative_value_opd_two_bytes() { #[test] fn mid_negative_value_opd_three_bytes() { let mut allocator = Allocator::new(); - let result = OpdConversion {} + let result = opd_conversion() .invoke(&mut allocator, &"83ffffff".to_string()) .unwrap(); assert_eq!(result.rest(), "0xffffff"); @@ -93,7 +95,7 @@ fn mid_negative_value_opd_three_bytes() { #[test] fn mid_negative_value_opd_tricky_negative_2() { let mut allocator = Allocator::new(); - let result = OpdConversion {} + let result = opd_conversion() .invoke(&mut allocator, &"82ff00".to_string()) .unwrap(); assert_eq!(result.rest(), "-256"); @@ -102,7 +104,7 @@ fn mid_negative_value_opd_tricky_negative_2() { #[test] fn mid_negative_value_opd_tricky_positive_2() { let mut allocator = Allocator::new(); - let result = OpdConversion {} + let result = opd_conversion() .invoke(&mut allocator, &"8200ff".to_string()) .unwrap(); assert_eq!(result.rest(), "255"); @@ -111,7 +113,7 @@ fn mid_negative_value_opd_tricky_positive_2() { #[test] fn mid_negative_value_opd_tricky_negative_3() { let mut allocator = Allocator::new(); - let result = OpdConversion {} + let result = opd_conversion() .invoke(&mut allocator, &"83ff0000".to_string()) .unwrap(); assert_eq!(result.rest(), "0xff0000"); @@ -120,7 +122,7 @@ fn mid_negative_value_opd_tricky_negative_3() { #[test] fn mid_negative_value_opd_tricky_positive_3() { let mut allocator = Allocator::new(); - let result = OpdConversion {} + let result = opd_conversion() .invoke(&mut allocator, &"8300ffff".to_string()) .unwrap(); assert_eq!(result.rest(), "0x00ffff"); @@ -153,7 +155,7 @@ fn mid_negative_value_disassemble() { let nodeptr = allocator .new_atom(&[0xff, 0xff]) .expect("should be able to make an atom"); - assert_eq!(disassemble(&mut allocator, nodeptr), "0xffff"); + assert_eq!(disassemble(&mut allocator, nodeptr, Some(0)), "0xffff"); } #[test] @@ -177,7 +179,7 @@ fn small_test_opc() { #[test] fn large_odd_sized_pos_opd() { let mut allocator = Allocator::new(); - let result = OpdConversion {} + let result = opd_conversion() .invoke(&mut allocator, &"ff8700ffffffffffff80".to_string()) .unwrap(); assert_eq!(result.rest(), "(0x00ffffffffffff)"); @@ -186,7 +188,7 @@ fn large_odd_sized_pos_opd() { #[test] fn basic_opd() { let mut allocator = Allocator::new(); - let result = OpdConversion {} + let result = opd_conversion() .invoke(&mut allocator, &"80".to_string()) .unwrap(); assert_eq!(result.rest(), "()"); @@ -195,7 +197,7 @@ fn basic_opd() { #[test] fn nil_in_list_opd() { let mut allocator = Allocator::new(); - let result = OpdConversion {} + let result = opd_conversion() .invoke(&mut allocator, &"ff8080".to_string()) .unwrap(); assert_eq!(result.rest(), "(())"); @@ -217,7 +219,7 @@ fn big_decode_opd() { }) .unwrap(); - let result = OpdConversion {} + let result = opd_conversion() .invoke(&mut allocator, &expected.first()) .unwrap(); assert_eq!(expected.rest(), result.rest()); @@ -246,7 +248,7 @@ fn compile_program<'a>( let input_sexp = allocator.new_pair(input_program, allocator.null()).unwrap(); let res = runner.run_program(allocator, run_script, input_sexp, None); - return res.map(|x| disassemble(allocator, x.1)); + return res.map(|x| disassemble(allocator, x.1, Some(0))); } #[test] @@ -395,7 +397,7 @@ fn basic_opc_quoted_1() { #[test] fn test_simple_opd_conversion() { let mut allocator = Allocator::new(); - let result = OpdConversion {} + let result = opd_conversion() .invoke(&mut allocator, &"ff0183666f6f".to_string()) .unwrap(); assert_eq!(result.rest(), "(q . \"foo\")"); @@ -625,6 +627,8 @@ fn pool_member_innerpuz() { &mut s, &vec![ "run".to_string(), + "--operators-version".to_string(), + "0".to_string(), "-i".to_string(), testpath.into_os_string().into_string().unwrap(), program.to_string(), @@ -702,20 +706,20 @@ fn test_fancy_destructuring_type_language() { // Use first let First::Here(kw) = assert_node_not_error(First::Here(ThisNode::Here).select_nodes(&mut allocator, code)); - assert_eq!(disassemble(&mut allocator, kw), "\"defconst\""); + assert_eq!(disassemble(&mut allocator, kw, Some(0)), "\"defconst\""); // Use second of list let Rest::Here(First::Here(name)) = assert_node_not_error( Rest::Here(First::Here(ThisNode::Here)).select_nodes(&mut allocator, code), ); - assert_eq!(disassemble(&mut allocator, name), "88"); + assert_eq!(disassemble(&mut allocator, name, Some(0)), "88"); let NodeSel::Cons((), NodeSel::Cons(name_by_cons, rest)) = assert_node_not_error( NodeSel::Cons((), NodeSel::Cons(ThisNode::Here, ThisNode::Here)) .select_nodes(&mut allocator, code), ); - assert_eq!(disassemble(&mut allocator, name_by_cons), "88"); - assert_eq!(disassemble(&mut allocator, rest), "((+ 3 1))"); + assert_eq!(disassemble(&mut allocator, name_by_cons, Some(0)), "88"); + assert_eq!(disassemble(&mut allocator, rest, Some(0)), "((+ 3 1))"); } #[test] diff --git a/src/tests/classic/stage_2.rs b/src/tests/classic/stage_2.rs index 0d80b005a..cf2d44dd5 100644 --- a/src/tests/classic/stage_2.rs +++ b/src/tests/classic/stage_2.rs @@ -44,7 +44,7 @@ fn test_expand_macro( symbols_source, ) .unwrap(); - disassemble(allocator, exp_res.1) + disassemble(allocator, exp_res.1, Some(0)) } fn test_inner_expansion( @@ -57,7 +57,7 @@ fn test_inner_expansion( let prog_ir = read_ir(&prog_rest).unwrap(); let prog_source = assemble_from_ir(allocator, Rc::new(prog_ir)).unwrap(); let exp_res = brun(allocator, macro_source, prog_source).unwrap(); - disassemble(allocator, exp_res) + disassemble(allocator, exp_res, Some(0)) } fn test_do_com_prog( @@ -74,7 +74,7 @@ fn test_do_com_prog( let sym_ir = read_ir(&symbol_table_src).unwrap(); let symbol_table = assemble_from_ir(allocator, Rc::new(sym_ir)).unwrap(); let result = do_com_prog(allocator, 849, program, macro_lookup, symbol_table, runner).unwrap(); - disassemble(allocator, result.1) + disassemble(allocator, result.1, Some(0)) } #[test] @@ -151,7 +151,7 @@ fn test_stage_2_quote() { let mut allocator = Allocator::new(); let assembled = assemble(&mut allocator, "(1 2 3)").unwrap(); let quoted = quote(&mut allocator, assembled).unwrap(); - assert_eq!(disassemble(&mut allocator, quoted), "(q 1 2 3)"); + assert_eq!(disassemble(&mut allocator, quoted, Some(0)), "(q 1 2 3)"); } #[test] @@ -161,7 +161,7 @@ fn test_stage_2_evaluate() { let args = assemble(&mut allocator, "(q 9 15)").unwrap(); let to_eval = evaluate(&mut allocator, prog, args).unwrap(); assert_eq!( - disassemble(&mut allocator, to_eval), + disassemble(&mut allocator, to_eval, Some(0)), "(a (q 16 2 3) (q 9 15))" ); } @@ -173,7 +173,7 @@ fn test_stage_2_run() { let macro_lookup_throw = assemble(&mut allocator, "(q 9)").unwrap(); let to_eval = run(&mut allocator, prog, macro_lookup_throw).unwrap(); assert_eq!( - disassemble(&mut allocator, to_eval), + disassemble(&mut allocator, to_eval, Some(0)), "(a (\"com\" (q 16 2 3) (q 1 9)) 1)" ); } @@ -217,8 +217,8 @@ fn test_process_embed_file_as_sexp() { let (name, content) = process_embed_file(&mut allocator, runner, declaration_sexp).expect("should work"); assert_eq!( - disassemble(&mut allocator, want_exp), - disassemble(&mut allocator, content) + disassemble(&mut allocator, want_exp, Some(0)), + disassemble(&mut allocator, content, Some(0)) ); assert_eq!(name, b"test-embed"); } @@ -293,7 +293,7 @@ fn test_process_embed_file_as_hex() { ) .expect("should work"); assert_eq!( - disassemble(&mut allocator, matching_part_of_decl), + disassemble(&mut allocator, matching_part_of_decl, Some(0)), decode_string(outstream.get_value().data()) ); assert_eq!(name, b"test-embed-from-hex"); @@ -337,6 +337,9 @@ impl CompilerOpts for TestCompilerOptsPresentsOwnFiles { fn start_env(&self) -> Option> { None } + fn disassembly_ver(&self) -> Option { + None + } fn prim_map(&self) -> Rc, Rc>> { Rc::new(HashMap::new()) } @@ -367,6 +370,9 @@ impl CompilerOpts for TestCompilerOptsPresentsOwnFiles { fn set_start_env(&self, _start_env: Option>) -> Rc { Rc::new(self.clone()) } + fn set_disassembly_ver(&self, _ver: Option) -> Rc { + Rc::new(self.clone()) + } fn read_new_file( &self, inc_from: String, @@ -426,7 +432,7 @@ fn test_classic_compiler_with_compiler_opts() { ) .expect("should compile and find the content"); assert_eq!( - disassemble(&mut allocator, result), + disassemble(&mut allocator, result, Some(0)), "(a (q 2 2 (c 2 (c 5 ()))) (c (q 16 5 (q . 1)) 1))" ); // Verify lack of injection From 2accda3304203a806f0dee7b64099af566637cee Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 6 Jun 2023 11:10:57 -0700 Subject: [PATCH 024/117] fmt --- src/classic/clvm/mod.rs | 194 +++++++++++++++--- src/classic/clvm_tools/binutils.rs | 8 +- src/classic/clvm_tools/clvmc.rs | 8 +- src/classic/clvm_tools/cmds.rs | 35 ++-- src/classic/clvm_tools/stages/stage_0.rs | 2 +- .../clvm_tools/stages/stage_2/compile.rs | 2 +- .../clvm_tools/stages/stage_2/operators.rs | 28 ++- src/tests/classic/smoke.rs | 6 +- 8 files changed, 212 insertions(+), 71 deletions(-) diff --git a/src/classic/clvm/mod.rs b/src/classic/clvm/mod.rs index 8a6a237f1..c6de05d6c 100644 --- a/src/classic/clvm/mod.rs +++ b/src/classic/clvm/mod.rs @@ -13,20 +13,60 @@ pub const OPERATORS_LATEST_VERSION: usize = 1; struct KwAtomPair { v: u8, n: &'static str, - version: usize + version: usize, } const KW_PAIRS: [KwAtomPair; 44] = [ - KwAtomPair { v: 0x01, n: "q", version: 0 }, - KwAtomPair { v: 0x02, n: "a", version: 0 }, - KwAtomPair { v: 0x03, n: "i", version: 0 }, - KwAtomPair { v: 0x04, n: "c", version: 0 }, - KwAtomPair { v: 0x05, n: "f", version: 0 }, - KwAtomPair { v: 0x06, n: "r", version: 0 }, - KwAtomPair { v: 0x07, n: "l", version: 0 }, - KwAtomPair { v: 0x08, n: "x", version: 0 }, - KwAtomPair { v: 0x09, n: "=", version: 0 }, - KwAtomPair { v: 0x0a, n: ">s", version: 0 }, + KwAtomPair { + v: 0x01, + n: "q", + version: 0, + }, + KwAtomPair { + v: 0x02, + n: "a", + version: 0, + }, + KwAtomPair { + v: 0x03, + n: "i", + version: 0, + }, + KwAtomPair { + v: 0x04, + n: "c", + version: 0, + }, + KwAtomPair { + v: 0x05, + n: "f", + version: 0, + }, + KwAtomPair { + v: 0x06, + n: "r", + version: 0, + }, + KwAtomPair { + v: 0x07, + n: "l", + version: 0, + }, + KwAtomPair { + v: 0x08, + n: "x", + version: 0, + }, + KwAtomPair { + v: 0x09, + n: "=", + version: 0, + }, + KwAtomPair { + v: 0x0a, + n: ">s", + version: 0, + }, KwAtomPair { v: 0x0b, n: "sha256", @@ -47,18 +87,46 @@ const KW_PAIRS: [KwAtomPair; 44] = [ n: "concat", version: 0, }, - KwAtomPair { v: 0x10, n: "+", version: 0 }, - KwAtomPair { v: 0x11, n: "-", version: 0 }, - KwAtomPair { v: 0x12, n: "*", version: 0 }, - KwAtomPair { v: 0x13, n: "/", version: 0 }, + KwAtomPair { + v: 0x10, + n: "+", + version: 0, + }, + KwAtomPair { + v: 0x11, + n: "-", + version: 0, + }, + KwAtomPair { + v: 0x12, + n: "*", + version: 0, + }, + KwAtomPair { + v: 0x13, + n: "/", + version: 0, + }, KwAtomPair { v: 0x14, n: "divmod", version: 0, }, - KwAtomPair { v: 0x15, n: ">", version: 0 }, - KwAtomPair { v: 0x16, n: "ash", version: 0 }, - KwAtomPair { v: 0x17, n: "lsh", version: 0 }, + KwAtomPair { + v: 0x15, + n: ">", + version: 0, + }, + KwAtomPair { + v: 0x16, + n: "ash", + version: 0, + }, + KwAtomPair { + v: 0x17, + n: "lsh", + version: 0, + }, KwAtomPair { v: 0x18, n: "logand", @@ -89,26 +157,86 @@ const KW_PAIRS: [KwAtomPair; 44] = [ n: "pubkey_for_exp", version: 0, }, - KwAtomPair { v: 0x20, n: "not", version: 0 }, - KwAtomPair { v: 0x21, n: "any", version: 0 }, - KwAtomPair { v: 0x22, n: "all", version: 0 }, + KwAtomPair { + v: 0x20, + n: "not", + version: 0, + }, + KwAtomPair { + v: 0x21, + n: "any", + version: 0, + }, + KwAtomPair { + v: 0x22, + n: "all", + version: 0, + }, KwAtomPair { v: 0x24, n: "softfork", version: 0, }, - KwAtomPair { v: 0x30, n: "coinid", version: 1 }, - KwAtomPair { v: 0x31, n: "bls_g1_subtract", version: 1 }, - KwAtomPair { v: 0x32, n: "bls_g1_multiply", version: 1 }, - KwAtomPair { v: 0x33, n: "bls_g1_negate", version: 1 }, - KwAtomPair { v: 0x34, n: "bls_g2_add", version: 1 }, - KwAtomPair { v: 0x35, n: "bls_g2_subtract", version: 1 }, - KwAtomPair { v: 0x36, n: "bls_g2_multiply", version: 1 }, - KwAtomPair { v: 0x37, n: "bls_g2_negate", version: 1 }, - KwAtomPair { v: 0x38, n: "bls_map_to_g1", version: 1 }, - KwAtomPair { v: 0x39, n: "bls_map_to_g2", version: 1 }, - KwAtomPair { v: 0x3a, n: "bls_pairing_identity", version: 1 }, - KwAtomPair { v: 0x3b, n: "bls_verify", version: 1 }, + KwAtomPair { + v: 0x30, + n: "coinid", + version: 1, + }, + KwAtomPair { + v: 0x31, + n: "bls_g1_subtract", + version: 1, + }, + KwAtomPair { + v: 0x32, + n: "bls_g1_multiply", + version: 1, + }, + KwAtomPair { + v: 0x33, + n: "bls_g1_negate", + version: 1, + }, + KwAtomPair { + v: 0x34, + n: "bls_g2_add", + version: 1, + }, + KwAtomPair { + v: 0x35, + n: "bls_g2_subtract", + version: 1, + }, + KwAtomPair { + v: 0x36, + n: "bls_g2_multiply", + version: 1, + }, + KwAtomPair { + v: 0x37, + n: "bls_g2_negate", + version: 1, + }, + KwAtomPair { + v: 0x38, + n: "bls_map_to_g1", + version: 1, + }, + KwAtomPair { + v: 0x39, + n: "bls_map_to_g2", + version: 1, + }, + KwAtomPair { + v: 0x3a, + n: "bls_pairing_identity", + version: 1, + }, + KwAtomPair { + v: 0x3b, + n: "bls_verify", + version: 1, + }, ]; lazy_static! { diff --git a/src/classic/clvm_tools/binutils.rs b/src/classic/clvm_tools/binutils.rs index a6e331a5f..f09f01b62 100644 --- a/src/classic/clvm_tools/binutils.rs +++ b/src/classic/clvm_tools/binutils.rs @@ -7,8 +7,8 @@ use unicode_segmentation::UnicodeSegmentation; use clvm_rs::allocator::{Allocator, NodePtr, SExp}; use clvm_rs::reduction::EvalErr; -use crate::classic::clvm::OPERATORS_LATEST_VERSION; use crate::classic::clvm::__type_compatibility__::{Bytes, BytesFromType, Record, Stream}; +use crate::classic::clvm::OPERATORS_LATEST_VERSION; use crate::classic::clvm::{keyword_from_atom, keyword_to_atom}; use crate::classic::clvm_tools::ir::r#type::IRRepr; use crate::classic::clvm_tools::ir::reader::IRReader; @@ -146,7 +146,11 @@ pub fn disassemble_with_kw( } pub fn disassemble(allocator: &mut Allocator, sexp: NodePtr, version: Option) -> String { - return disassemble_with_kw(allocator, sexp, keyword_from_atom(version.unwrap_or(OPERATORS_LATEST_VERSION))); + return disassemble_with_kw( + allocator, + sexp, + keyword_from_atom(version.unwrap_or(OPERATORS_LATEST_VERSION)), + ); } pub fn assemble(allocator: &mut Allocator, s: &str) -> Result { diff --git a/src/classic/clvm_tools/clvmc.rs b/src/classic/clvm_tools/clvmc.rs index 35c98449b..d1a906986 100644 --- a/src/classic/clvm_tools/clvmc.rs +++ b/src/classic/clvm_tools/clvmc.rs @@ -143,7 +143,13 @@ pub fn compile_clvm_inner( filename, classic_with_opts, ) - .map_err(|x| format!("error {} compiling {}", x.1, disassemble(allocator, x.0, opts.disassembly_ver())))?; + .map_err(|x| { + format!( + "error {} compiling {}", + x.1, + disassemble(allocator, x.0, opts.disassembly_ver()) + ) + })?; sexp_to_stream(allocator, result, result_stream); Ok(()) } diff --git a/src/classic/clvm_tools/cmds.rs b/src/classic/clvm_tools/cmds.rs index c490b8e70..ff6785006 100644 --- a/src/classic/clvm_tools/cmds.rs +++ b/src/classic/clvm_tools/cmds.rs @@ -20,13 +20,13 @@ use clvm_rs::allocator::{Allocator, NodePtr}; use clvm_rs::reduction::EvalErr; use clvm_rs::run_program::PreEval; -use crate::classic::clvm::OPERATORS_LATEST_VERSION; use crate::classic::clvm::__type_compatibility__::{ t, Bytes, BytesFromType, Stream, Tuple, UnvalidatedBytesFromType, }; use crate::classic::clvm::keyword_from_atom; use crate::classic::clvm::serialize::{sexp_from_stream, sexp_to_stream, SimpleCreateCLVMObject}; use crate::classic::clvm::sexp::{enlist, proper_list, sexp_as_bin}; +use crate::classic::clvm::OPERATORS_LATEST_VERSION; use crate::classic::clvm_tools::binutils::{assemble_from_ir, disassemble, disassemble_with_kw}; use crate::classic::clvm_tools::clvmc::{detect_modern, write_sym_output}; use crate::classic::clvm_tools::debug::check_unused; @@ -103,10 +103,7 @@ impl ArgumentValueConv for PathOrCodeConv { // } pub trait TConversion { - fn apply_args( - &mut self, - parsed_args: &HashMap - ); + fn apply_args(&mut self, parsed_args: &HashMap); fn invoke( &self, @@ -228,10 +225,7 @@ pub fn call_tool( pub struct OpcConversion {} impl TConversion for OpcConversion { - fn apply_args( - &mut self, - _args: &HashMap - ) { } + fn apply_args(&mut self, _args: &HashMap) {} fn invoke( &self, @@ -249,14 +243,11 @@ impl TConversion for OpcConversion { #[derive(Debug)] pub struct OpdConversion { - pub op_version: Option + pub op_version: Option, } impl TConversion for OpdConversion { - fn apply_args( - &mut self, - args: &HashMap - ) { + fn apply_args(&mut self, args: &HashMap) { if let Some(ArgumentValue::ArgInt(i)) = args.get("operators_version") { self.op_version = Some(*i as usize); } @@ -312,7 +303,9 @@ struct OperatorsVersion {} impl ArgumentValueConv for OperatorsVersion { fn convert(&self, arg: &str) -> Result { - let ver = arg.parse::().map_err(|_| format!("expected number 0-1 found {arg}"))?; + let ver = arg + .parse::() + .map_err(|_| format!("expected number 0-1 found {arg}"))?; Ok(ArgumentValue::ArgInt(ver)) } } @@ -994,7 +987,9 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul let empty_map = HashMap::new(); let keywords = match parsed_args.get("no_keywords") { Some(ArgumentValue::ArgBool(_b)) => &empty_map, - _ => keyword_from_atom(get_disassembly_ver(&parsed_args).unwrap_or(OPERATORS_LATEST_VERSION)), + _ => { + keyword_from_atom(get_disassembly_ver(&parsed_args).unwrap_or(OPERATORS_LATEST_VERSION)) + } }; // If extra symbol output is desired (not all keys are hashes, but there's @@ -1534,9 +1529,7 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul only_exn, &log_content, symbol_table, - &|allocator, p| { - disassemble(allocator, p, disassembly_ver) - } + &|allocator, p| disassemble(allocator, p, disassembly_ver), ); } else { stdout.write_str("\n"); @@ -1546,9 +1539,7 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul only_exn, &log_content, symbol_table, - &|allocator, p| { - disassemble(allocator, p, disassembly_ver) - } + &|allocator, p| disassemble(allocator, p, disassembly_ver), ); } } diff --git a/src/classic/clvm_tools/stages/stage_0.rs b/src/classic/clvm_tools/stages/stage_0.rs index 80f6d6da6..eb57d7eb1 100644 --- a/src/classic/clvm_tools/stages/stage_0.rs +++ b/src/classic/clvm_tools/stages/stage_0.rs @@ -1,5 +1,5 @@ use clvm_rs::allocator::{Allocator, NodePtr}; -use clvm_rs::chia_dialect::{ChiaDialect, NO_UNKNOWN_OPS, ENABLE_BLS_OPS}; +use clvm_rs::chia_dialect::{ChiaDialect, ENABLE_BLS_OPS, NO_UNKNOWN_OPS}; use clvm_rs::cost::Cost; use clvm_rs::reduction::Response; diff --git a/src/classic/clvm_tools/stages/stage_2/compile.rs b/src/classic/clvm_tools/stages/stage_2/compile.rs index 6501e5dec..24ca19a46 100644 --- a/src/classic/clvm_tools/stages/stage_2/compile.rs +++ b/src/classic/clvm_tools/stages/stage_2/compile.rs @@ -4,9 +4,9 @@ use std::rc::Rc; use clvm_rs::allocator::{Allocator, AtomBuf, NodePtr, SExp}; use clvm_rs::reduction::{EvalErr, Reduction, Response}; -use crate::classic::clvm::OPERATORS_LATEST_VERSION; use crate::classic::clvm::__type_compatibility__::{Bytes, BytesFromType}; use crate::classic::clvm::sexp::{enlist, first, map_m, non_nil, proper_list, rest}; +use crate::classic::clvm::OPERATORS_LATEST_VERSION; use crate::classic::clvm::{keyword_from_atom, keyword_to_atom}; use crate::classic::clvm_tools::binutils::{assemble, disassemble}; diff --git a/src/classic/clvm_tools/stages/stage_2/operators.rs b/src/classic/clvm_tools/stages/stage_2/operators.rs index 11f97b704..169d7d92e 100644 --- a/src/classic/clvm_tools/stages/stage_2/operators.rs +++ b/src/classic/clvm_tools/stages/stage_2/operators.rs @@ -5,14 +5,14 @@ use std::path::PathBuf; use std::rc::Rc; use clvm_rs::allocator::{Allocator, NodePtr, SExp}; -use clvm_rs::chia_dialect::{ChiaDialect, NO_UNKNOWN_OPS, ENABLE_BLS_OPS}; +use clvm_rs::chia_dialect::{ChiaDialect, ENABLE_BLS_OPS, NO_UNKNOWN_OPS}; use clvm_rs::cost::Cost; use clvm_rs::dialect::{Dialect, OperatorSet}; use clvm_rs::reduction::{EvalErr, Reduction, Response}; use clvm_rs::run_program::run_program_with_pre_eval; -use crate::classic::clvm::OPERATORS_LATEST_VERSION; use crate::classic::clvm::__type_compatibility__::{Bytes, BytesFromType, Stream}; +use crate::classic::clvm::OPERATORS_LATEST_VERSION; use crate::classic::clvm::keyword_from_atom; use crate::classic::clvm::sexp::proper_list; @@ -214,7 +214,12 @@ impl CompilerOperatorsInternal { let filename_buf = allocator.buf(&filename_buf); let filename_bytes = Bytes::new(Some(BytesFromType::Raw(filename_buf.to_vec()))); - let ir = disassemble_to_ir_with_kw(allocator, data, keyword_from_atom(self.get_disassembly_ver()), true); + let ir = disassemble_to_ir_with_kw( + allocator, + data, + keyword_from_atom(self.get_disassembly_ver()), + true, + ); let mut stream = Stream::new(None); write_ir_to_stream(Rc::new(ir), &mut stream); return fs::write(filename_bytes.decode(), stream.get_value().decode()) @@ -333,9 +338,7 @@ impl Dialect for CompilerOperatorsInternal { // The softfork operator comes with an extension argument. fn softfork_extension(&self, ext: u32) -> OperatorSet { match ext { - 0 => { - OperatorSet::BLS - } + 0 => OperatorSet::BLS, // new extensions go here _ => OperatorSet::Default, } @@ -347,7 +350,7 @@ impl Dialect for CompilerOperatorsInternal { op: NodePtr, sexp: NodePtr, max_cost: Cost, - _extension: OperatorSet + _extension: OperatorSet, ) -> Response { match allocator.sexp(op) { SExp::Atom(opname) => { @@ -373,14 +376,19 @@ impl Dialect for CompilerOperatorsInternal { } else if opbuf == "_get_source_file".as_bytes() { self.get_source_file(allocator) } else { - self.base_dialect.op(allocator, op, sexp, max_cost, OperatorSet::BLS) + self.base_dialect + .op(allocator, op, sexp, max_cost, OperatorSet::BLS) } } - _ => self.base_dialect.op(allocator, op, sexp, max_cost, OperatorSet::BLS), + _ => self + .base_dialect + .op(allocator, op, sexp, max_cost, OperatorSet::BLS), } } - fn allow_unknown_ops(&self) -> bool { false } + fn allow_unknown_ops(&self) -> bool { + false + } } impl CompilerOperatorsInternal { diff --git a/src/tests/classic/smoke.rs b/src/tests/classic/smoke.rs index 473ea9997..37edd0961 100644 --- a/src/tests/classic/smoke.rs +++ b/src/tests/classic/smoke.rs @@ -45,7 +45,11 @@ fn large_odd_sized_neg_opc() { assert_eq!(result.rest(), "ff8afde1e61f36454dc0000180"); } -fn opd_conversion() -> OpdConversion { OpdConversion { op_version: Some(0) } } +fn opd_conversion() -> OpdConversion { + OpdConversion { + op_version: Some(0), + } +} #[test] fn large_odd_sized_neg_opd() { From 2f68498dd40a08bc6a018748946d38ca01f9aa30 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 6 Jun 2023 11:15:30 -0700 Subject: [PATCH 025/117] Clippy --- src/compiler/prims.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/prims.rs b/src/compiler/prims.rs index 83f50e8ff..c2e6d14bd 100644 --- a/src/compiler/prims.rs +++ b/src/compiler/prims.rs @@ -185,7 +185,7 @@ pub fn prims() -> Vec<(Vec, SExp)> { ), ( "bls_verify".as_bytes().to_vec(), - SExp::Integer(primloc.clone(), 59_u32.to_bigint().unwrap()), + SExp::Integer(primloc, 59_u32.to_bigint().unwrap()), ), ] } From bc3a2a29e18cddb99b5a680c352a7d21f0a2790d Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 6 Jun 2023 11:34:17 -0700 Subject: [PATCH 026/117] upgrade specification of clvmr in wasm --- wasm/Cargo.lock | 106 +++++++++++++++++++++++++----------------------- wasm/Cargo.toml | 2 +- 2 files changed, 57 insertions(+), 51 deletions(-) diff --git a/wasm/Cargo.lock b/wasm/Cargo.lock index 9c0b27d34..1b4343cb0 100644 --- a/wasm/Cargo.lock +++ b/wasm/Cargo.lock @@ -42,23 +42,28 @@ dependencies = [ ] [[package]] -name = "block-buffer" -version = "0.10.3" +name = "bls12_381" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +checksum = "62250ece575fa9b22068b3a8d59586f01d426dd7785522efd97632959e71c986" dependencies = [ - "generic-array", + "ff 0.12.1", + "group 0.12.1", + "pairing 0.22.0", + "rand_core", + "subtle", ] [[package]] name = "bls12_381" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62250ece575fa9b22068b3a8d59586f01d426dd7785522efd97632959e71c986" +checksum = "d7bc6d6292be3a19e6379786dac800f551e5865a5bb51ebbe3064ab80433f403" dependencies = [ - "ff", - "group", - "pairing", + "digest", + "ff 0.13.0", + "group 0.13.0", + "pairing 0.23.0", "rand_core", "subtle", ] @@ -95,7 +100,7 @@ name = "clvm_tools_rs" version = "0.1.34" dependencies = [ "binascii", - "bls12_381", + "bls12_381 0.7.0", "bytestream", "clvmr", "derivative", @@ -114,7 +119,7 @@ dependencies = [ "pyo3-build-config 0.15.2", "serde", "serde_json", - "sha2 0.9.9", + "sha2", "tempfile", "unicode-segmentation", "wasm-bindgen", @@ -135,17 +140,18 @@ dependencies = [ [[package]] name = "clvmr" -version = "0.1.24" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5e907612d322d0d7def6b0ecb3ad681f6af2db106bcfabe4153746c60ef9e4" +checksum = "d234802ce73011e01f7019ef5701df1f9bffad9326d7d0a6dbca8d4b7591f083" dependencies = [ - "bls12_381", + "bls12_381 0.8.0", + "group 0.13.0", "hex", "lazy_static", "num-bigint", "num-integer", "num-traits", - "sha2 0.10.2", + "sha2", ] [[package]] @@ -167,16 +173,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - [[package]] name = "derivative" version = "2.2.0" @@ -197,16 +193,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "digest" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" -dependencies = [ - "block-buffer 0.10.3", - "crypto-common", -] - [[package]] name = "do-notation" version = "0.1.3" @@ -239,6 +225,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "ff" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded41244b729663b1e574f1b4fb731469f69f79c17667b5d776b16cda0479449" +dependencies = [ + "bitvec", + "rand_core", + "subtle", +] + [[package]] name = "funty" version = "2.0.0" @@ -274,7 +271,18 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" dependencies = [ - "ff", + "ff 0.12.1", + "rand_core", + "subtle", +] + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff 0.13.0", "rand_core", "subtle", ] @@ -470,7 +478,16 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135590d8bdba2b31346f9cd1fb2a912329f5135e832a4f422942eb6ead8b6b3b" dependencies = [ - "group", + "group 0.12.1", +] + +[[package]] +name = "pairing" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fec4625e73cf41ef4bb6846cafa6d44736525f442ba45e407c4a000a13996f" +dependencies = [ + "group 0.13.0", ] [[package]] @@ -683,24 +700,13 @@ version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ - "block-buffer 0.9.0", + "block-buffer", "cfg-if", "cpufeatures", - "digest 0.9.0", + "digest", "opaque-debug", ] -[[package]] -name = "sha2" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest 0.10.5", -] - [[package]] name = "smallvec" version = "1.10.0" diff --git a/wasm/Cargo.toml b/wasm/Cargo.toml index 703831506..4388b2dc0 100644 --- a/wasm/Cargo.toml +++ b/wasm/Cargo.toml @@ -18,7 +18,7 @@ path = "src/mod.rs" [dependencies] clvm_tools_rs = { path= "..", features = [] } -clvmr = "0.1.24" +clvmr = { version = "0.2.5", features = ["pre-eval"] } wasm-bindgen = "=0.2.83" wasm-bindgen-test = "=0.3.25" js-sys = "0.3.60" From c3a007d17b5de85c39616daffa1bd7d03564ddb9 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 6 Jun 2023 14:47:29 -0700 Subject: [PATCH 027/117] Add first bls test --- Cargo.lock | 60 +++---------------- Cargo.toml | 2 +- .../tests/bls/classic-bls-op-test-1.clsp | 5 ++ src/tests/classic/bls.rs | 27 +++++++++ src/tests/classic/mod.rs | 1 + src/tests/classic/run.rs | 2 +- 6 files changed, 43 insertions(+), 54 deletions(-) create mode 100644 resources/tests/bls/classic-bls-op-test-1.clsp create mode 100644 src/tests/classic/bls.rs diff --git a/Cargo.lock b/Cargo.lock index ae1531e22..e1ba78191 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,19 +41,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "bls12_381" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62250ece575fa9b22068b3a8d59586f01d426dd7785522efd97632959e71c986" -dependencies = [ - "ff 0.12.1", - "group 0.12.1", - "pairing 0.22.0", - "rand_core", - "subtle", -] - [[package]] name = "bls12_381" version = "0.8.0" @@ -61,9 +48,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7bc6d6292be3a19e6379786dac800f551e5865a5bb51ebbe3064ab80433f403" dependencies = [ "digest", - "ff 0.13.0", - "group 0.13.0", - "pairing 0.23.0", + "ff", + "group", + "pairing", "rand_core", "subtle", ] @@ -100,7 +87,7 @@ name = "clvm_tools_rs" version = "0.1.34" dependencies = [ "binascii", - "bls12_381 0.7.0", + "bls12_381", "bytestream", "clvmr", "derivative", @@ -135,8 +122,8 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d234802ce73011e01f7019ef5701df1f9bffad9326d7d0a6dbca8d4b7591f083" dependencies = [ - "bls12_381 0.8.0", - "group 0.13.0", + "bls12_381", + "group", "hex", "lazy_static", "num-bigint", @@ -205,17 +192,6 @@ dependencies = [ "instant", ] -[[package]] -name = "ff" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" -dependencies = [ - "bitvec", - "rand_core", - "subtle", -] - [[package]] name = "ff" version = "0.13.0" @@ -256,24 +232,13 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "group" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" -dependencies = [ - "ff 0.12.1", - "rand_core", - "subtle", -] - [[package]] name = "group" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ - "ff 0.13.0", + "ff", "rand_core", "subtle", ] @@ -463,22 +428,13 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" -[[package]] -name = "pairing" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "135590d8bdba2b31346f9cd1fb2a912329f5135e832a4f422942eb6ead8b6b3b" -dependencies = [ - "group 0.12.1", -] - [[package]] name = "pairing" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81fec4625e73cf41ef4bb6846cafa6d44736525f442ba45e407c4a000a13996f" dependencies = [ - "group 0.13.0", + "group", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 82387127e..b7d0b705d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ python-source = "python" [dependencies] hex = "0.4.3" num-bigint = { version = "0.4.0", features = ["serde"] } -bls12_381 = "0.7.0" +bls12_381 = { version = "=0.8.0", features = ["experimental"] } bytestream = "0.4.1" num-traits = "0.2.14" lazy_static = "1.4.0" diff --git a/resources/tests/bls/classic-bls-op-test-1.clsp b/resources/tests/bls/classic-bls-op-test-1.clsp new file mode 100644 index 000000000..de796c4f3 --- /dev/null +++ b/resources/tests/bls/classic-bls-op-test-1.clsp @@ -0,0 +1,5 @@ +(mod () + (defconstant msg 0x9790635de8740e9a6a6b15fb6b72f3a16afa0973d971979b6ba54761d6e2502c50db76f4d26143f05459a42cfd520d44) + + (bls_map_to_g1 msg) + ) diff --git a/src/tests/classic/bls.rs b/src/tests/classic/bls.rs new file mode 100644 index 000000000..b82f32ec9 --- /dev/null +++ b/src/tests/classic/bls.rs @@ -0,0 +1,27 @@ +use bls12_381::hash_to_curve::{ExpandMsgXmd, HashToCurve}; +use bls12_381::{G1Affine, G1Projective}; + +use crate::classic::clvm::__type_compatibility__::{Bytes, BytesFromType}; +use crate::tests::classic::run::do_basic_run; + +#[test] +fn test_using_bls_operators_0() { + // Run a program which uses the map_to_g1 operator and check the output. + let msg: &[u8] = &[ + 0x97, 0x90, 0x63, 0x5d, 0xe8, 0x74, 0x0e, 0x9a, 0x6a, 0x6b, 0x15, 0xfb, 0x6b, 0x72, 0xf3, + 0xa1, 0x6a, 0xfa, 0x09, 0x73, 0xd9, 0x71, 0x97, 0x9b, 0x6b, 0xa5, 0x47, 0x61, 0xd6, 0xe2, + 0x50, 0x2c, 0x50, 0xdb, 0x76, 0xf4, 0xd2, 0x61, 0x43, 0xf0, 0x54, 0x59, 0xa4, 0x2c, 0xfd, + 0x52, 0x0d, 0x44, + ]; + let dst: &[u8] = b"BLS_SIG_BLS12381G1_XMD:SHA-256_SSWU_RO_AUG_"; + let point = >>::hash_to_curve(msg, dst); + let expected_output: [u8; 48] = G1Affine::from(point).to_compressed(); + let result = do_basic_run(&vec![ + "run".to_string(), + "resources/tests/bls/classic-bls-op-test-1.clsp".to_string(), + ]) + .trim() + .to_string(); + let hex = Bytes::new(Some(BytesFromType::Raw(expected_output.to_vec()))).hex(); + assert_eq!(result, format!("(q . 0x{hex})")); +} diff --git a/src/tests/classic/mod.rs b/src/tests/classic/mod.rs index 7b505ccf2..e9392c3de 100644 --- a/src/tests/classic/mod.rs +++ b/src/tests/classic/mod.rs @@ -1,3 +1,4 @@ +mod bls; mod clvmc; mod optimize; pub mod run; diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index a15814de5..a144f114e 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -35,7 +35,7 @@ fn do_basic_brun(args: &Vec) -> String { return s.get_value().decode(); } -fn do_basic_run(args: &Vec) -> String { +pub fn do_basic_run(args: &Vec) -> String { let mut s = Stream::new(None); launch_tool(&mut s, args, &"run".to_string(), 2); return s.get_value().decode(); From a23b30e8184876f307dd46d6cb46212ffdb7a00e Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 6 Jun 2023 15:11:47 -0700 Subject: [PATCH 028/117] Test bls operator in modern --- resources/tests/bls/modern-bls-op-test-1.clsp | 6 +++ src/tests/classic/bls.rs | 46 +++++++++++++++---- src/tests/classic/run.rs | 2 +- 3 files changed, 43 insertions(+), 11 deletions(-) create mode 100644 resources/tests/bls/modern-bls-op-test-1.clsp diff --git a/resources/tests/bls/modern-bls-op-test-1.clsp b/resources/tests/bls/modern-bls-op-test-1.clsp new file mode 100644 index 000000000..92be2ae10 --- /dev/null +++ b/resources/tests/bls/modern-bls-op-test-1.clsp @@ -0,0 +1,6 @@ +(mod () + (include *standard-cl-21*) + (defconstant msg 0x9790635de8740e9a6a6b15fb6b72f3a16afa0973d971979b6ba54761d6e2502c50db76f4d26143f05459a42cfd520d44) + + (bls_map_to_g1 msg) + ) diff --git a/src/tests/classic/bls.rs b/src/tests/classic/bls.rs index b82f32ec9..938eb8788 100644 --- a/src/tests/classic/bls.rs +++ b/src/tests/classic/bls.rs @@ -2,20 +2,27 @@ use bls12_381::hash_to_curve::{ExpandMsgXmd, HashToCurve}; use bls12_381::{G1Affine, G1Projective}; use crate::classic::clvm::__type_compatibility__::{Bytes, BytesFromType}; -use crate::tests::classic::run::do_basic_run; +use crate::tests::classic::run::{do_basic_brun, do_basic_run}; -#[test] -fn test_using_bls_operators_0() { - // Run a program which uses the map_to_g1 operator and check the output. - let msg: &[u8] = &[ - 0x97, 0x90, 0x63, 0x5d, 0xe8, 0x74, 0x0e, 0x9a, 0x6a, 0x6b, 0x15, 0xfb, 0x6b, 0x72, 0xf3, - 0xa1, 0x6a, 0xfa, 0x09, 0x73, 0xd9, 0x71, 0x97, 0x9b, 0x6b, 0xa5, 0x47, 0x61, 0xd6, 0xe2, - 0x50, 0x2c, 0x50, 0xdb, 0x76, 0xf4, 0xd2, 0x61, 0x43, 0xf0, 0x54, 0x59, 0xa4, 0x2c, 0xfd, - 0x52, 0x0d, 0x44, - ]; +const MSG1: &[u8] = &[ + 0x97, 0x90, 0x63, 0x5d, 0xe8, 0x74, 0x0e, 0x9a, 0x6a, 0x6b, 0x15, 0xfb, 0x6b, 0x72, 0xf3, + 0xa1, 0x6a, 0xfa, 0x09, 0x73, 0xd9, 0x71, 0x97, 0x9b, 0x6b, 0xa5, 0x47, 0x61, 0xd6, 0xe2, + 0x50, 0x2c, 0x50, 0xdb, 0x76, 0xf4, 0xd2, 0x61, 0x43, 0xf0, 0x54, 0x59, 0xa4, 0x2c, 0xfd, + 0x52, 0x0d, 0x44, +]; + + +fn bls_map_to_g1(msg: &[u8]) -> Vec { let dst: &[u8] = b"BLS_SIG_BLS12381G1_XMD:SHA-256_SSWU_RO_AUG_"; let point = >>::hash_to_curve(msg, dst); let expected_output: [u8; 48] = G1Affine::from(point).to_compressed(); + expected_output.to_vec() +} + +#[test] +fn test_using_bls_operators_0() { + let expected_output = bls_map_to_g1(&MSG1); + // Run a program which uses the map_to_g1 operator and check the output. let result = do_basic_run(&vec![ "run".to_string(), "resources/tests/bls/classic-bls-op-test-1.clsp".to_string(), @@ -25,3 +32,22 @@ fn test_using_bls_operators_0() { let hex = Bytes::new(Some(BytesFromType::Raw(expected_output.to_vec()))).hex(); assert_eq!(result, format!("(q . 0x{hex})")); } + +#[test] +fn test_using_bls_operators_1() { + let expected_output = bls_map_to_g1(&MSG1); + let prog = do_basic_run(&vec![ + "run".to_string(), + "resources/tests/bls/modern-bls-op-test-1.clsp".to_string(), + ]) + .trim() + .to_string(); + let result = do_basic_brun(&vec![ + "brun".to_string(), + prog + ]) + .trim() + .to_string(); + let hex = Bytes::new(Some(BytesFromType::Raw(expected_output.to_vec()))).hex(); + assert_eq!(result, format!("0x{hex}")); +} diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index a144f114e..0fbbf5ddf 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -29,7 +29,7 @@ use crate::util::{number_from_u8, Number}; const NUM_GEN_ATOMS: usize = 16; -fn do_basic_brun(args: &Vec) -> String { +pub fn do_basic_brun(args: &Vec) -> String { let mut s = Stream::new(None); launch_tool(&mut s, args, &"run".to_string(), 0); return s.get_value().decode(); From 80497f2034786f3853a4914ceaa912eb02773672 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 6 Jun 2023 15:12:11 -0700 Subject: [PATCH 029/117] fmt --- src/tests/classic/bls.rs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/tests/classic/bls.rs b/src/tests/classic/bls.rs index 938eb8788..548f59ed9 100644 --- a/src/tests/classic/bls.rs +++ b/src/tests/classic/bls.rs @@ -5,13 +5,11 @@ use crate::classic::clvm::__type_compatibility__::{Bytes, BytesFromType}; use crate::tests::classic::run::{do_basic_brun, do_basic_run}; const MSG1: &[u8] = &[ - 0x97, 0x90, 0x63, 0x5d, 0xe8, 0x74, 0x0e, 0x9a, 0x6a, 0x6b, 0x15, 0xfb, 0x6b, 0x72, 0xf3, - 0xa1, 0x6a, 0xfa, 0x09, 0x73, 0xd9, 0x71, 0x97, 0x9b, 0x6b, 0xa5, 0x47, 0x61, 0xd6, 0xe2, - 0x50, 0x2c, 0x50, 0xdb, 0x76, 0xf4, 0xd2, 0x61, 0x43, 0xf0, 0x54, 0x59, 0xa4, 0x2c, 0xfd, - 0x52, 0x0d, 0x44, + 0x97, 0x90, 0x63, 0x5d, 0xe8, 0x74, 0x0e, 0x9a, 0x6a, 0x6b, 0x15, 0xfb, 0x6b, 0x72, 0xf3, 0xa1, + 0x6a, 0xfa, 0x09, 0x73, 0xd9, 0x71, 0x97, 0x9b, 0x6b, 0xa5, 0x47, 0x61, 0xd6, 0xe2, 0x50, 0x2c, + 0x50, 0xdb, 0x76, 0xf4, 0xd2, 0x61, 0x43, 0xf0, 0x54, 0x59, 0xa4, 0x2c, 0xfd, 0x52, 0x0d, 0x44, ]; - fn bls_map_to_g1(msg: &[u8]) -> Vec { let dst: &[u8] = b"BLS_SIG_BLS12381G1_XMD:SHA-256_SSWU_RO_AUG_"; let point = >>::hash_to_curve(msg, dst); @@ -42,12 +40,9 @@ fn test_using_bls_operators_1() { ]) .trim() .to_string(); - let result = do_basic_brun(&vec![ - "brun".to_string(), - prog - ]) - .trim() - .to_string(); + let result = do_basic_brun(&vec!["brun".to_string(), prog]) + .trim() + .to_string(); let hex = Bytes::new(Some(BytesFromType::Raw(expected_output.to_vec()))).hex(); assert_eq!(result, format!("0x{hex}")); } From 1e0a14fa3bbe0bf7c6e0475ce968460b70a6d81f Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 7 Jun 2023 11:41:21 -0700 Subject: [PATCH 030/117] bls tests --- src/tests/classic/bls.rs | 65 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 6 deletions(-) diff --git a/src/tests/classic/bls.rs b/src/tests/classic/bls.rs index 548f59ed9..dafde5db9 100644 --- a/src/tests/classic/bls.rs +++ b/src/tests/classic/bls.rs @@ -4,6 +4,8 @@ use bls12_381::{G1Affine, G1Projective}; use crate::classic::clvm::__type_compatibility__::{Bytes, BytesFromType}; use crate::tests::classic::run::{do_basic_brun, do_basic_run}; +const HASH_LEN: usize = 32; + const MSG1: &[u8] = &[ 0x97, 0x90, 0x63, 0x5d, 0xe8, 0x74, 0x0e, 0x9a, 0x6a, 0x6b, 0x15, 0xfb, 0x6b, 0x72, 0xf3, 0xa1, 0x6a, 0xfa, 0x09, 0x73, 0xd9, 0x71, 0x97, 0x9b, 0x6b, 0xa5, 0x47, 0x61, 0xd6, 0xe2, 0x50, 0x2c, @@ -32,17 +34,68 @@ fn test_using_bls_operators_0() { } #[test] -fn test_using_bls_operators_1() { +fn test_using_bls_verify_signature_good_msg_classic() { + let right_msg = "(0x0102030405)"; + let expected_output = bls_map_to_g1(&MSG1); let prog = do_basic_run(&vec![ "run".to_string(), - "resources/tests/bls/modern-bls-op-test-1.clsp".to_string(), + "resources/tests/bls/classic-bls-verify-signature.clsp".to_string(), + ]) + .trim() + .to_string(); + let result = do_basic_brun(&vec!["brun".to_string(), prog, right_msg.to_string()]) + .trim() + .to_string(); + let hex = Bytes::new(Some(BytesFromType::Raw(expected_output.to_vec()))).hex(); + assert_eq!(result, format!("()")); +} + +#[test] +fn test_using_bls_verify_signature_bad_msg_classic() { + let wrong_msg = "(0x0102030415)"; + + let prog = do_basic_run(&vec![ + "run".to_string(), + "resources/tests/bls/classic-bls-verify-signature.clsp".to_string(), + ]) + .trim() + .to_string(); + let result = do_basic_brun(&vec!["brun".to_string(), prog, wrong_msg.to_string()]) + .trim() + .to_string(); + assert!(result.starts_with("FAIL")); +} +#[test] +fn test_using_bls_verify_signature_good_msg() { + let right_msg = "(0x0102030405)"; + + let expected_output = bls_map_to_g1(&MSG1); + let prog = do_basic_run(&vec![ + "run".to_string(), + "resources/tests/bls/modern-bls-verify-signature.clsp".to_string(), ]) - .trim() - .to_string(); - let result = do_basic_brun(&vec!["brun".to_string(), prog]) + .trim() + .to_string(); + let result = do_basic_brun(&vec!["brun".to_string(), prog, right_msg.to_string()]) .trim() .to_string(); let hex = Bytes::new(Some(BytesFromType::Raw(expected_output.to_vec()))).hex(); - assert_eq!(result, format!("0x{hex}")); + assert_eq!(result, format!("()")); +} + +#[test] +fn test_using_bls_verify_signature_bad_msg() { + let wrong_msg = "(0x0102030415)"; + + let prog = do_basic_run(&vec![ + "run".to_string(), + "resources/tests/bls/modern-bls-verify-signature.clsp".to_string(), + ]) + .trim() + .to_string(); + let result = do_basic_brun(&vec!["brun".to_string(), prog, wrong_msg.to_string()]) + .trim() + .to_string(); + assert!(result.starts_with("FAIL")); } From 8a316797124a4dc9a4802fe8e548c0cb0c780039 Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 7 Jun 2023 11:42:58 -0700 Subject: [PATCH 031/117] fmt --- src/tests/classic/bls.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/tests/classic/bls.rs b/src/tests/classic/bls.rs index dafde5db9..eee6413bf 100644 --- a/src/tests/classic/bls.rs +++ b/src/tests/classic/bls.rs @@ -42,8 +42,8 @@ fn test_using_bls_verify_signature_good_msg_classic() { "run".to_string(), "resources/tests/bls/classic-bls-verify-signature.clsp".to_string(), ]) - .trim() - .to_string(); + .trim() + .to_string(); let result = do_basic_brun(&vec!["brun".to_string(), prog, right_msg.to_string()]) .trim() .to_string(); @@ -59,8 +59,8 @@ fn test_using_bls_verify_signature_bad_msg_classic() { "run".to_string(), "resources/tests/bls/classic-bls-verify-signature.clsp".to_string(), ]) - .trim() - .to_string(); + .trim() + .to_string(); let result = do_basic_brun(&vec!["brun".to_string(), prog, wrong_msg.to_string()]) .trim() .to_string(); @@ -75,8 +75,8 @@ fn test_using_bls_verify_signature_good_msg() { "run".to_string(), "resources/tests/bls/modern-bls-verify-signature.clsp".to_string(), ]) - .trim() - .to_string(); + .trim() + .to_string(); let result = do_basic_brun(&vec!["brun".to_string(), prog, right_msg.to_string()]) .trim() .to_string(); @@ -92,8 +92,8 @@ fn test_using_bls_verify_signature_bad_msg() { "run".to_string(), "resources/tests/bls/modern-bls-verify-signature.clsp".to_string(), ]) - .trim() - .to_string(); + .trim() + .to_string(); let result = do_basic_brun(&vec!["brun".to_string(), prog, wrong_msg.to_string()]) .trim() .to_string(); From e35d2f467137a87cc979823aa554d80045c79143 Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 7 Jun 2023 12:11:58 -0700 Subject: [PATCH 032/117] opps, pull in the right test programs --- resources/tests/bls/classic-bls-verify-signature.clsp | 8 ++++++++ resources/tests/bls/modern-bls-verify-signature.clsp | 9 +++++++++ 2 files changed, 17 insertions(+) create mode 100644 resources/tests/bls/classic-bls-verify-signature.clsp create mode 100644 resources/tests/bls/modern-bls-verify-signature.clsp diff --git a/resources/tests/bls/classic-bls-verify-signature.clsp b/resources/tests/bls/classic-bls-verify-signature.clsp new file mode 100644 index 000000000..833d549d0 --- /dev/null +++ b/resources/tests/bls/classic-bls-verify-signature.clsp @@ -0,0 +1,8 @@ +(mod (message) + (defconstant pk_bytes 0x86243290bbcbfd9ae75bdece7981965350208eb5e99b04d5cd24e955ada961f8c0a162dee740be7bdc6c3c0613ba2eb1) + (defconstant signature_bytes 0xb00ab9a8af54804b43067531d96c176710c05980fccf8eee1ae12a4fd543df929cce860273af931fe4fdbc407d495f73114ab7d17ef08922e56625daada0497582340ecde841a9e997f2f557653c21c070119662dd2efa47e2d6c5e2de00eefa) + + ;; Seems like bls_verify should do it + ;; G2 G1 msg + (bls_verify signature_bytes pk_bytes message) + ) diff --git a/resources/tests/bls/modern-bls-verify-signature.clsp b/resources/tests/bls/modern-bls-verify-signature.clsp new file mode 100644 index 000000000..ccde5a1f2 --- /dev/null +++ b/resources/tests/bls/modern-bls-verify-signature.clsp @@ -0,0 +1,9 @@ +(mod (message) + (include *standard-cl-21*) + (defconstant pk_bytes 0x86243290bbcbfd9ae75bdece7981965350208eb5e99b04d5cd24e955ada961f8c0a162dee740be7bdc6c3c0613ba2eb1) + (defconstant signature_bytes 0xb00ab9a8af54804b43067531d96c176710c05980fccf8eee1ae12a4fd543df929cce860273af931fe4fdbc407d495f73114ab7d17ef08922e56625daada0497582340ecde841a9e997f2f557653c21c070119662dd2efa47e2d6c5e2de00eefa) + + ;; Seems like bls_verify should do it + ;; G2 G1 msg + (bls_verify signature_bytes pk_bytes message) + ) From 9eeceb32813ae0832e2c8513dc0d3d330872df8c Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 7 Jun 2023 13:48:40 -0700 Subject: [PATCH 033/117] Add a repl test that uses the operator literal syntax and softfork --- src/tests/classic/bls.rs | 2 -- src/tests/compiler/repl.rs | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/tests/classic/bls.rs b/src/tests/classic/bls.rs index eee6413bf..563d7b356 100644 --- a/src/tests/classic/bls.rs +++ b/src/tests/classic/bls.rs @@ -4,8 +4,6 @@ use bls12_381::{G1Affine, G1Projective}; use crate::classic::clvm::__type_compatibility__::{Bytes, BytesFromType}; use crate::tests::classic::run::{do_basic_brun, do_basic_run}; -const HASH_LEN: usize = 32; - const MSG1: &[u8] = &[ 0x97, 0x90, 0x63, 0x5d, 0xe8, 0x74, 0x0e, 0x9a, 0x6a, 0x6b, 0x15, 0xfb, 0x6b, 0x72, 0xf3, 0xa1, 0x6a, 0xfa, 0x09, 0x73, 0xd9, 0x71, 0x97, 0x9b, 0x6b, 0xa5, 0x47, 0x61, 0xd6, 0xe2, 0x50, 0x2c, diff --git a/src/tests/compiler/repl.rs b/src/tests/compiler/repl.rs index d614bef78..2dfea9d47 100644 --- a/src/tests/compiler/repl.rs +++ b/src/tests/compiler/repl.rs @@ -301,3 +301,17 @@ fn test_eval_list_partially_evaluated_xyz() { "(c x (c y (c z (q))))" ); } + +#[test] +fn test_eval_new_bls_operator() { + assert_eq!( + test_repl_outcome_with_stack_limit(vec![indoc!{ + "(softfork + (q . 196005) + (q . 0) + (q #bls_map_to_g1 (1 . 0x9790635de8740e9a6a6b15fb6b72f3a16afa0973d971979b6ba54761d6e2502c50db76f4d26143f05459a42cfd520d44)) () + )"}.to_string() + ], None).unwrap().unwrap(), + "(q)" + ); +} From ffe2884534c44e0eda71701a0e50b702be58ca58 Mon Sep 17 00:00:00 2001 From: arty Date: Thu, 8 Jun 2023 03:46:31 -0700 Subject: [PATCH 034/117] remove some unused stuff --- resources/tests/bls/coinid-fail.clsp | 5 +++++ resources/tests/bls/coinid-good.clsp | 5 +++++ src/tests/classic/bls.rs | 4 ---- 3 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 resources/tests/bls/coinid-fail.clsp create mode 100644 resources/tests/bls/coinid-good.clsp diff --git a/resources/tests/bls/coinid-fail.clsp b/resources/tests/bls/coinid-fail.clsp new file mode 100644 index 000000000..d7ef3b836 --- /dev/null +++ b/resources/tests/bls/coinid-fail.clsp @@ -0,0 +1,5 @@ +(mod () + (include *standard-cl-21*) + + (softfork (q . 1432) (q . 0) (q a (i (= (#coinid (q . 0x1234500000000000000000000000000000000000000000000000000000000000) (q . 0x6789abcdef000000000000000000000000000000000000000000000000000000) (q . 123456789)) (q . 0x69bfe81b052bfc6bd7f3fb9167fec61793175b897c16a35827f947d5cc98e4bc)) (q x) (q . 0)) (q . ())) (q . ())) + ) diff --git a/resources/tests/bls/coinid-good.clsp b/resources/tests/bls/coinid-good.clsp new file mode 100644 index 000000000..93f16b02c --- /dev/null +++ b/resources/tests/bls/coinid-good.clsp @@ -0,0 +1,5 @@ +(mod () + (include *standard-cl-21*) + + (softfork (q . 1432) (q . 0) (q a (i (= (#coinid (q . 0x1234500000000000000000000000000000000000000000000000000000000000) (q . 0x6789abcdef000000000000000000000000000000000000000000000000000000) (q . 123456789)) (q . 0x69bfe81b052bfc6bd7f3fb9167fec61793175b897c16a35827f947d5cc98e4bc)) (q . 0) (q x)) (q . ())) (q . ())) + ) diff --git a/src/tests/classic/bls.rs b/src/tests/classic/bls.rs index 563d7b356..88d017e7e 100644 --- a/src/tests/classic/bls.rs +++ b/src/tests/classic/bls.rs @@ -35,7 +35,6 @@ fn test_using_bls_operators_0() { fn test_using_bls_verify_signature_good_msg_classic() { let right_msg = "(0x0102030405)"; - let expected_output = bls_map_to_g1(&MSG1); let prog = do_basic_run(&vec![ "run".to_string(), "resources/tests/bls/classic-bls-verify-signature.clsp".to_string(), @@ -45,7 +44,6 @@ fn test_using_bls_verify_signature_good_msg_classic() { let result = do_basic_brun(&vec!["brun".to_string(), prog, right_msg.to_string()]) .trim() .to_string(); - let hex = Bytes::new(Some(BytesFromType::Raw(expected_output.to_vec()))).hex(); assert_eq!(result, format!("()")); } @@ -68,7 +66,6 @@ fn test_using_bls_verify_signature_bad_msg_classic() { fn test_using_bls_verify_signature_good_msg() { let right_msg = "(0x0102030405)"; - let expected_output = bls_map_to_g1(&MSG1); let prog = do_basic_run(&vec![ "run".to_string(), "resources/tests/bls/modern-bls-verify-signature.clsp".to_string(), @@ -78,7 +75,6 @@ fn test_using_bls_verify_signature_good_msg() { let result = do_basic_brun(&vec!["brun".to_string(), prog, right_msg.to_string()]) .trim() .to_string(); - let hex = Bytes::new(Some(BytesFromType::Raw(expected_output.to_vec()))).hex(); assert_eq!(result, format!("()")); } From 5c753ff389039d17ab4714f458d32779f288c412 Mon Sep 17 00:00:00 2001 From: arty Date: Thu, 8 Jun 2023 09:36:50 -0700 Subject: [PATCH 035/117] Fix up --- src/tests/classic/bls.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/tests/classic/bls.rs b/src/tests/classic/bls.rs index 88d017e7e..e5ffffcc3 100644 --- a/src/tests/classic/bls.rs +++ b/src/tests/classic/bls.rs @@ -93,3 +93,31 @@ fn test_using_bls_verify_signature_bad_msg() { .to_string(); assert!(result.starts_with("FAIL")); } + +#[test] +fn test_coinid_in_softfork_bad() { + let prog = do_basic_run(&vec![ + "run".to_string(), + "resources/tests/bls/coinid-fail.clsp".to_string(), + ]) + .trim() + .to_string(); + let result = do_basic_brun(&vec!["brun".to_string(), prog, "()".to_string()]) + .trim() + .to_string(); + assert!(result.starts_with("FAIL")); +} + +#[test] +fn test_coinid_in_softfork_good() { + let prog = do_basic_run(&vec![ + "run".to_string(), + "resources/tests/bls/coinid-good.clsp".to_string(), + ]) + .trim() + .to_string(); + let result = do_basic_brun(&vec!["brun".to_string(), prog, "()".to_string()]) + .trim() + .to_string(); + assert!(result.starts_with("()")); +} From cda115b76997dc33111d1881f329ab21e82fe776 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 27 Jun 2023 10:50:34 -0700 Subject: [PATCH 036/117] Add some more smoke tests of things that changed during the clvmr api change, remove bls operator prefixes and add tests for more operators, add secp256k1_verify and secp256kr1_verify and test ability to use the wide opcodes --- Cargo.lock | 268 ++++++++++++++++-- Cargo.toml | 2 +- .../tests/bls/classic-bls-op-test-1.clsp | 2 +- resources/tests/bls/modern-bls-op-test-1.clsp | 2 +- src/classic/clvm/mod.rs | 128 +++++---- src/classic/clvm/serialize.rs | 8 +- src/classic/clvm/sexp.rs | 22 +- src/classic/clvm_tools/binutils.rs | 7 +- src/classic/clvm_tools/clvmc.rs | 9 +- src/classic/clvm_tools/debug.rs | 2 +- src/classic/clvm_tools/pattern_match.rs | 43 ++- src/classic/clvm_tools/sha256tree.rs | 5 +- src/classic/clvm_tools/stages/stage_0.rs | 4 +- .../clvm_tools/stages/stage_2/compile.rs | 69 +++-- .../clvm_tools/stages/stage_2/inline.rs | 19 +- .../clvm_tools/stages/stage_2/module.rs | 23 +- .../clvm_tools/stages/stage_2/operators.rs | 32 +-- .../clvm_tools/stages/stage_2/optimize.rs | 46 +-- .../clvm_tools/stages/stage_2/reader.rs | 10 +- src/compiler/clvm.rs | 6 +- src/compiler/prims.rs | 28 +- src/tests/classic/run.rs | 266 +++++++++++++++++ src/tests/classic/smoke.rs | 58 +++- src/tests/compiler/repl.rs | 2 +- 24 files changed, 822 insertions(+), 239 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e1ba78191..740e20834 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,18 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "binascii" version = "0.1.4" @@ -41,13 +53,22 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bls12_381" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7bc6d6292be3a19e6379786dac800f551e5865a5bb51ebbe3064ab80433f403" dependencies = [ - "digest", + "digest 0.9.0", "ff", "group", "pairing", @@ -108,7 +129,7 @@ dependencies = [ "rand_chacha", "serde", "serde_json", - "sha2", + "sha2 0.9.9", "tempfile", "unicode-segmentation", "wasm-bindgen", @@ -118,18 +139,21 @@ dependencies = [ [[package]] name = "clvmr" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d234802ce73011e01f7019ef5701df1f9bffad9326d7d0a6dbca8d4b7591f083" +checksum = "83afaa6d5081706f202d31ed08ddb2425c902ca968d9832429bcdf9474ca6c9f" dependencies = [ "bls12_381", + "getrandom", "group", "hex", + "k256", "lazy_static", "num-bigint", "num-integer", "num-traits", - "sha2", + "p256", + "sha2 0.9.9", ] [[package]] @@ -142,6 +166,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "const-oid" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520fbf3c07483f94e3e3ca9d0cfd913d7718ef2483d2cfd91c0d9e91474ab913" + [[package]] name = "cpufeatures" version = "0.2.5" @@ -151,6 +181,39 @@ dependencies = [ "libc", ] +[[package]] +name = "crypto-bigint" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4c2f4e1afd912bc40bfd6fed5d9dc1f288e0ba01bfcc835cc5bc3eb13efe15" +dependencies = [ + "generic-array", + "rand_core", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "der" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56acb310e15652100da43d130af8d97b509e95af61aab1c5a7939ef24337ee17" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + [[package]] name = "derivative" version = "2.2.0" @@ -171,12 +234,58 @@ dependencies = [ "generic-array", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid", + "crypto-common", + "subtle", +] + [[package]] name = "do-notation" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e16a80c1dda2cf52fa07106427d3d798b6331dca8155fcb8c39f7fc78f6dd2" +[[package]] +name = "ecdsa" +version = "0.16.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0997c976637b606099b9985693efa3581e84e41f5c11ba5255f88711058ad428" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "elliptic-curve" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "968405c8fdc9b3bf4df0a6638858cc0b52462836ab6b1c87377785dd09cf1c0b" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pem-rfc7468", + "pkcs8", + "rand_core", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "encoding8" version = "0.3.2" @@ -211,19 +320,20 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "generic-array" -version = "0.14.5" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", + "zeroize", ] [[package]] name = "getrandom" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" dependencies = [ "cfg-if", "js-sys", @@ -249,6 +359,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "indoc" version = "0.3.6" @@ -302,6 +421,20 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "k256" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cadb76004ed8e97623117f3df85b17aaa6626ab0b0831e6573f104df16cd1bcc" +dependencies = [ + "cfg-if", + "ecdsa", + "elliptic-curve", + "once_cell", + "sha2 0.10.7", + "signature", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -310,9 +443,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.126" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "linked-hash-map" @@ -418,9 +551,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.12.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "opaque-debug" @@ -428,6 +561,18 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.7", +] + [[package]] name = "pairing" version = "0.23.0" @@ -481,12 +626,40 @@ dependencies = [ "proc-macro-hack", ] +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "ppv-lite86" version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +[[package]] +name = "primeorder" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c2fcef82c0ec6eefcc179b978446c399b3cdf73c392c35604e399eee6df1ee3" +dependencies = [ + "elliptic-curve", +] + [[package]] name = "proc-macro-hack" version = "0.5.19" @@ -597,9 +770,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ "getrandom", ] @@ -622,6 +795,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + [[package]] name = "ryu" version = "1.0.10" @@ -640,6 +823,20 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sec1" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0aec48e813d6b90b15f0b8948af3c63483992dee44c03e9930b3eebdabe046e" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + [[package]] name = "serde" version = "1.0.137" @@ -677,19 +874,50 @@ version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ - "block-buffer", + "block-buffer 0.9.0", "cfg-if", "cpufeatures", - "digest", + "digest 0.9.0", "opaque-debug", ] +[[package]] +name = "sha2" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.10.7", +] + +[[package]] +name = "signature" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e1788eed21689f9cf370582dfc467ef36ed9c707f073528ddafa8d83e3b8500" +dependencies = [ + "digest 0.10.7", + "rand_core", +] + [[package]] name = "smallvec" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" +[[package]] +name = "spki" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "subtle" version = "2.4.1" @@ -904,3 +1132,9 @@ checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" dependencies = [ "linked-hash-map", ] + +[[package]] +name = "zeroize" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" diff --git a/Cargo.toml b/Cargo.toml index b7d0b705d..1c5a933aa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ do-notation = "0.1.3" serde_json = "1.0" sha2 = "0.9.5" tempfile = "3.3.0" -clvmr = { version = "0.2.5", features = ["pre-eval"] } +clvmr = { version = "0.2.6", features = ["pre-eval"] } binascii = "0.1.4" yaml-rust = "0.4" linked-hash-map = "0.5.6" diff --git a/resources/tests/bls/classic-bls-op-test-1.clsp b/resources/tests/bls/classic-bls-op-test-1.clsp index de796c4f3..c8bbe6662 100644 --- a/resources/tests/bls/classic-bls-op-test-1.clsp +++ b/resources/tests/bls/classic-bls-op-test-1.clsp @@ -1,5 +1,5 @@ (mod () (defconstant msg 0x9790635de8740e9a6a6b15fb6b72f3a16afa0973d971979b6ba54761d6e2502c50db76f4d26143f05459a42cfd520d44) - (bls_map_to_g1 msg) + (g1_map msg) ) diff --git a/resources/tests/bls/modern-bls-op-test-1.clsp b/resources/tests/bls/modern-bls-op-test-1.clsp index 92be2ae10..d5f60d38a 100644 --- a/resources/tests/bls/modern-bls-op-test-1.clsp +++ b/resources/tests/bls/modern-bls-op-test-1.clsp @@ -2,5 +2,5 @@ (include *standard-cl-21*) (defconstant msg 0x9790635de8740e9a6a6b15fb6b72f3a16afa0973d971979b6ba54761d6e2502c50db76f4d26143f05459a42cfd520d44) - (bls_map_to_g1 msg) + (g1_map msg) ) diff --git a/src/classic/clvm/mod.rs b/src/classic/clvm/mod.rs index c6de05d6c..0254281b2 100644 --- a/src/classic/clvm/mod.rs +++ b/src/classic/clvm/mod.rs @@ -11,260 +11,270 @@ pub mod syntax_error; pub const OPERATORS_LATEST_VERSION: usize = 1; struct KwAtomPair { - v: u8, + v: &'static [u8], n: &'static str, version: usize, } -const KW_PAIRS: [KwAtomPair; 44] = [ +const KW_PAIRS: [KwAtomPair; 46] = [ KwAtomPair { - v: 0x01, + v: &[0x01], n: "q", version: 0, }, KwAtomPair { - v: 0x02, + v: &[0x02], n: "a", version: 0, }, KwAtomPair { - v: 0x03, + v: &[0x03], n: "i", version: 0, }, KwAtomPair { - v: 0x04, + v: &[0x04], n: "c", version: 0, }, KwAtomPair { - v: 0x05, + v: &[0x05], n: "f", version: 0, }, KwAtomPair { - v: 0x06, + v: &[0x06], n: "r", version: 0, }, KwAtomPair { - v: 0x07, + v: &[0x07], n: "l", version: 0, }, KwAtomPair { - v: 0x08, + v: &[0x08], n: "x", version: 0, }, KwAtomPair { - v: 0x09, + v: &[0x09], n: "=", version: 0, }, KwAtomPair { - v: 0x0a, + v: &[0x0a], n: ">s", version: 0, }, KwAtomPair { - v: 0x0b, + v: &[0x0b], n: "sha256", version: 0, }, KwAtomPair { - v: 0x0c, + v: &[0x0c], n: "substr", version: 0, }, KwAtomPair { - v: 0x0d, + v: &[0x0d], n: "strlen", version: 0, }, KwAtomPair { - v: 0x0e, + v: &[0x0e], n: "concat", version: 0, }, KwAtomPair { - v: 0x10, + v: &[0x10], n: "+", version: 0, }, KwAtomPair { - v: 0x11, + v: &[0x11], n: "-", version: 0, }, KwAtomPair { - v: 0x12, + v: &[0x12], n: "*", version: 0, }, KwAtomPair { - v: 0x13, + v: &[0x13], n: "/", version: 0, }, KwAtomPair { - v: 0x14, + v: &[0x14], n: "divmod", version: 0, }, KwAtomPair { - v: 0x15, + v: &[0x15], n: ">", version: 0, }, KwAtomPair { - v: 0x16, + v: &[0x16], n: "ash", version: 0, }, KwAtomPair { - v: 0x17, + v: &[0x17], n: "lsh", version: 0, }, KwAtomPair { - v: 0x18, + v: &[0x18], n: "logand", version: 0, }, KwAtomPair { - v: 0x19, + v: &[0x19], n: "logior", version: 0, }, KwAtomPair { - v: 0x1a, + v: &[0x1a], n: "logxor", version: 0, }, KwAtomPair { - v: 0x1b, + v: &[0x1b], n: "lognot", version: 0, }, KwAtomPair { - v: 0x1d, + v: &[0x1d], n: "point_add", version: 0, }, KwAtomPair { - v: 0x1e, + v: &[0x1e], n: "pubkey_for_exp", version: 0, }, KwAtomPair { - v: 0x20, + v: &[0x20], n: "not", version: 0, }, KwAtomPair { - v: 0x21, + v: &[0x21], n: "any", version: 0, }, KwAtomPair { - v: 0x22, + v: &[0x22], n: "all", version: 0, }, KwAtomPair { - v: 0x24, + v: &[0x24], n: "softfork", version: 0, }, KwAtomPair { - v: 0x30, + v: &[0x30], n: "coinid", version: 1, }, KwAtomPair { - v: 0x31, - n: "bls_g1_subtract", + v: &[0x31], + n: "g1_subtract", version: 1, }, KwAtomPair { - v: 0x32, - n: "bls_g1_multiply", + v: &[0x32], + n: "g1_multiply", version: 1, }, KwAtomPair { - v: 0x33, - n: "bls_g1_negate", + v: &[0x33], + n: "g1_negate", version: 1, }, KwAtomPair { - v: 0x34, - n: "bls_g2_add", + v: &[0x34], + n: "g2_add", version: 1, }, KwAtomPair { - v: 0x35, - n: "bls_g2_subtract", + v: &[0x35], + n: "g2_subtract", version: 1, }, KwAtomPair { - v: 0x36, - n: "bls_g2_multiply", + v: &[0x36], + n: "g2_multiply", version: 1, }, KwAtomPair { - v: 0x37, - n: "bls_g2_negate", + v: &[0x37], + n: "g2_negate", version: 1, }, KwAtomPair { - v: 0x38, - n: "bls_map_to_g1", + v: &[0x38], + n: "g1_map", version: 1, }, KwAtomPair { - v: 0x39, - n: "bls_map_to_g2", + v: &[0x39], + n: "g2_map", version: 1, }, KwAtomPair { - v: 0x3a, + v: &[0x3a], n: "bls_pairing_identity", version: 1, }, KwAtomPair { - v: 0x3b, + v: &[0x3b], n: "bls_verify", version: 1, }, + KwAtomPair { + v: &[0x13,0xd6,0x1f,0x00], + n: "secp256k1_verify", + version: 1, + }, + KwAtomPair { + v: &[0x1c,0x3a,0x8f,0x00], + n: "secp256r1_verify", + version: 1, + } ]; lazy_static! { pub static ref KEYWORD_FROM_ATOM_0: HashMap, String> = { let mut result = HashMap::new(); for pair in KW_PAIRS.iter().filter(|p| p.version == 0) { - result.insert(vec![pair.v], pair.n.to_string()); + result.insert(pair.v.to_vec(), pair.n.to_string()); } result }; pub static ref KEYWORD_TO_ATOM_0: HashMap> = { let mut result = HashMap::new(); for pair in KW_PAIRS.iter().filter(|p| p.version == 0) { - result.insert(pair.n.to_string(), vec![pair.v]); + result.insert(pair.n.to_string(), pair.v.to_vec()); } result }; pub static ref KEYWORD_FROM_ATOM_1: HashMap, String> = { let mut result = HashMap::new(); for pair in KW_PAIRS.iter().filter(|p| p.version <= 1) { - result.insert(vec![pair.v], pair.n.to_string()); + result.insert(pair.v.to_vec(), pair.n.to_string()); } result }; pub static ref KEYWORD_TO_ATOM_1: HashMap> = { let mut result = HashMap::new(); for pair in KW_PAIRS.iter().filter(|p| p.version <= 1) { - result.insert(pair.n.to_string(), vec![pair.v]); + result.insert(pair.n.to_string(), pair.v.to_vec()); } result }; diff --git a/src/classic/clvm/serialize.rs b/src/classic/clvm/serialize.rs index 95ab47c63..05246789a 100644 --- a/src/classic/clvm/serialize.rs +++ b/src/classic/clvm/serialize.rs @@ -98,9 +98,11 @@ impl<'a> Iterator for SExpToBytesIterator<'a> { fn next(&mut self) -> Option { self.state.pop().and_then(|step| match step { SExpToByteOp::Object(x) => match self.allocator.sexp(x) { - SExp::Atom(b) => { - let buf = self.allocator.buf(&b).to_vec(); - let bytes = Bytes::new(Some(BytesFromType::Raw(buf.to_vec()))); + SExp::Atom() => { + // The only node we have in scope is x, so this atom + // capture is trivial. + let buf = self.allocator.atom(x).to_vec(); + let bytes = Bytes::new(Some(BytesFromType::Raw(buf.clone()))); match atom_size_blob(&bytes) { Ok((original, b)) => { if original { diff --git a/src/classic/clvm/sexp.rs b/src/classic/clvm/sexp.rs index 2cc433912..ef730eedd 100644 --- a/src/classic/clvm/sexp.rs +++ b/src/classic/clvm/sexp.rs @@ -3,7 +3,7 @@ use std::fmt::Debug; use std::rc::Rc; use std::string::String; -use clvm_rs::allocator::{Allocator, AtomBuf, NodePtr, SExp}; +use clvm_rs::allocator::{Allocator, NodePtr, SExp}; use clvm_rs::reduction::EvalErr; use bls12_381::G1Affine; @@ -150,7 +150,7 @@ pub fn to_sexp_type(allocator: &mut Allocator, value: CastableType) -> Result { + SExp::Atom() => { return Err(EvalErr( *target_value, "attempt to set_pair in atom".to_string(), @@ -335,7 +335,8 @@ pub fn bool_sexp(allocator: &mut Allocator, b: bool) -> NodePtr { pub fn non_nil(allocator: &mut Allocator, sexp: NodePtr) -> bool { match allocator.sexp(sexp) { SExp::Pair(_, _) => true, - SExp::Atom(b) => !b.is_empty(), + // sexp is the only node in scope, was !is_empty + SExp::Atom() => allocator.atom_len(sexp) != 0, } } @@ -353,9 +354,9 @@ pub fn rest(allocator: &mut Allocator, sexp: NodePtr) -> Result Result { +pub fn atom(allocator: &mut Allocator, sexp: NodePtr) -> Result, EvalErr> { match allocator.sexp(sexp) { - SExp::Atom(abuf) => Ok(abuf), + SExp::Atom() => Ok(allocator.atom(sexp).to_vec()), // only sexp in scope _ => Err(EvalErr(sexp, "not an atom".to_string())), } } @@ -365,7 +366,7 @@ pub fn proper_list(allocator: &mut Allocator, sexp: NodePtr, store: bool) -> Opt let mut args_sexp = sexp; loop { match allocator.sexp(args_sexp) { - SExp::Atom(_) => { + SExp::Atom() => { if !non_nil(allocator, args_sexp) { return Some(args); } else { @@ -453,10 +454,9 @@ pub fn equal_to(allocator: &mut Allocator, first_: NodePtr, second_: NodePtr) -> return true; } match (allocator.sexp(first), allocator.sexp(second)) { - (SExp::Atom(fbuf), SExp::Atom(sbuf)) => { - let fvec = allocator.buf(&fbuf); - let svec = allocator.buf(&sbuf); - return fvec == svec; + (SExp::Atom(), SExp::Atom()) => { + // two atoms in scope, both are used + return allocator.atom(first) == allocator.atom(second); } (SExp::Pair(ff, fr), SExp::Pair(rf, rr)) => { if !equal_to(allocator, ff, rf) { @@ -477,7 +477,7 @@ pub fn flatten(allocator: &mut Allocator, tree_: NodePtr, res: &mut Vec loop { match allocator.sexp(tree) { - SExp::Atom(_) => { + SExp::Atom() => { if non_nil(allocator, tree) { res.push(tree); } diff --git a/src/classic/clvm_tools/binutils.rs b/src/classic/clvm_tools/binutils.rs index f09f01b62..2433da06c 100644 --- a/src/classic/clvm_tools/binutils.rs +++ b/src/classic/clvm_tools/binutils.rs @@ -128,8 +128,9 @@ pub fn disassemble_to_ir_with_kw( IRRepr::Cons(Rc::new(v0), Rc::new(v1)) } - SExp::Atom(a) => { - let bytes = Bytes::new(Some(BytesFromType::Raw(allocator.buf(&a).to_vec()))); + SExp::Atom() => { + // sexp is the only node in scope. + let bytes = Bytes::new(Some(BytesFromType::Raw(allocator.atom(sexp).to_vec()))); ir_for_atom(&bytes, allow_keyword, keyword_from_atom) } } @@ -140,7 +141,7 @@ pub fn disassemble_with_kw( sexp: NodePtr, keyword_from_atom: &Record, String>, ) -> String { - let with_keywords = !matches!(allocator.sexp(sexp), SExp::Atom(_)); + let with_keywords = !matches!(allocator.sexp(sexp), SExp::Atom()); let symbols = disassemble_to_ir_with_kw(allocator, sexp, keyword_from_atom, with_keywords); write_ir(Rc::new(symbols)) } diff --git a/src/classic/clvm_tools/clvmc.rs b/src/classic/clvm_tools/clvmc.rs index d1a906986..8acfbecf4 100644 --- a/src/classic/clvm_tools/clvmc.rs +++ b/src/classic/clvm_tools/clvmc.rs @@ -33,9 +33,12 @@ fn include_dialect( dialects: &HashMap, i32>, e: &[NodePtr], ) -> Option { - if let (SExp::Atom(inc), SExp::Atom(name)) = (allocator.sexp(e[0]), allocator.sexp(e[1])) { - if allocator.buf(&inc) == "include".as_bytes().to_vec() { - if let Some(dialect) = dialects.get(allocator.buf(&name)) { + // Propogated names from let capture to labeled nodes. + let inc_node = e[0]; + let name_node = e[1]; + if let (SExp::Atom(), SExp::Atom()) = (allocator.sexp(inc_node), allocator.sexp(name_node)) { + if allocator.atom(inc_node) == "include".as_bytes().to_vec() { + if let Some(dialect) = dialects.get(allocator.atom(name_node)) { return Some(*dialect); } } diff --git a/src/classic/clvm_tools/debug.rs b/src/classic/clvm_tools/debug.rs index 20c088c93..25767e05d 100644 --- a/src/classic/clvm_tools/debug.rs +++ b/src/classic/clvm_tools/debug.rs @@ -180,7 +180,7 @@ fn table_trace( ) { let (sexp, args) = match allocator.sexp(form) { SExp::Pair(sexp, args) => (sexp, args), - SExp::Atom(_) => (form, allocator.null()), + SExp::Atom() => (form, allocator.null()), }; stdout.write_str(&format!("exp: {}\n", disassemble_f(allocator, sexp))); diff --git a/src/classic/clvm_tools/pattern_match.rs b/src/classic/clvm_tools/pattern_match.rs index df24d3c77..75c6eba35 100644 --- a/src/classic/clvm_tools/pattern_match.rs +++ b/src/classic/clvm_tools/pattern_match.rs @@ -51,25 +51,22 @@ pub fn match_sexp( */ match (allocator.sexp(pattern), allocator.sexp(sexp)) { - (SExp::Atom(pat_buf), SExp::Atom(sexp_buf)) => { - let sexp_bytes = allocator.buf(&sexp_buf).to_vec(); - if allocator.buf(&pat_buf).to_vec() == sexp_bytes { + (SExp::Atom(), SExp::Atom()) => { + // Two nodes in scope, both used. + if allocator.atom(pattern) == allocator.atom(sexp) { Some(known_bindings) } else { None } } (SExp::Pair(pleft, pright), _) => match (allocator.sexp(pleft), allocator.sexp(pright)) { - (SExp::Atom(pat_left), SExp::Atom(pat_right)) => { - let pat_right_bytes = allocator.buf(&pat_right).to_vec(); - let pat_left_bytes = allocator.buf(&pat_left).to_vec(); - + (SExp::Atom(), SExp::Atom()) => { match allocator.sexp(sexp) { - SExp::Atom(sexp_buf) => { - let sexp_bytes = allocator.buf(&sexp_buf).to_vec(); - if pat_left_bytes == ATOM_MATCH.to_vec() { - if pat_right_bytes == ATOM_MATCH.to_vec() { - if sexp_bytes == ATOM_MATCH.to_vec() { + SExp::Atom() => { + // Expression is ($ . $), sexp is '$', result: no capture. + if allocator.atom(pleft) == ATOM_MATCH { + if allocator.atom(pright) == ATOM_MATCH { + if allocator.atom(sexp) == ATOM_MATCH { return Some(HashMap::new()); } return None; @@ -78,13 +75,13 @@ pub fn match_sexp( return unify_bindings( allocator, known_bindings, - &pat_right_bytes, + &allocator.atom(pright).to_vec(), sexp, ); } - if pat_left_bytes == SEXP_MATCH.to_vec() { - if pat_right_bytes == SEXP_MATCH.to_vec() - && sexp_bytes == SEXP_MATCH.to_vec() + if allocator.atom(pleft) == SEXP_MATCH { + if allocator.atom(pright) == SEXP_MATCH + && allocator.atom(sexp) == SEXP_MATCH { return Some(HashMap::new()); } @@ -92,7 +89,8 @@ pub fn match_sexp( return unify_bindings( allocator, known_bindings, - &pat_right_bytes, + // pat_right_bytes + &allocator.atom(pright).to_vec(), sexp, ); } @@ -100,13 +98,14 @@ pub fn match_sexp( None } SExp::Pair(sleft, sright) => { - if pat_left_bytes == SEXP_MATCH.to_vec() - && pat_right_bytes != SEXP_MATCH.to_vec() + if allocator.atom(pleft) == SEXP_MATCH + && allocator.atom(pright) != SEXP_MATCH { return unify_bindings( allocator, known_bindings, - &pat_right_bytes, + // pat_right_bytes + &allocator.atom(pright).to_vec(), sexp, ); } @@ -118,11 +117,11 @@ pub fn match_sexp( } } _ => match allocator.sexp(sexp) { - SExp::Atom(_) => None, + SExp::Atom() => None, SExp::Pair(sleft, sright) => match_sexp(allocator, pleft, sleft, known_bindings) .and_then(|new_bindings| match_sexp(allocator, pright, sright, new_bindings)), }, }, - (SExp::Atom(_), _) => None, + (SExp::Atom(), _) => None, } } diff --git a/src/classic/clvm_tools/sha256tree.rs b/src/classic/clvm_tools/sha256tree.rs index 3c9986cc3..0212d0c78 100644 --- a/src/classic/clvm_tools/sha256tree.rs +++ b/src/classic/clvm_tools/sha256tree.rs @@ -34,9 +34,10 @@ pub fn sha256tree(allocator: &mut Allocator, v: NodePtr) -> Bytes { .concat(&right), ) } - SExp::Atom(a) => sha256( + SExp::Atom() => sha256( Bytes::new(Some(BytesFromType::Raw(vec![1]))).concat(&Bytes::new(Some( - BytesFromType::Raw(allocator.buf(&a).to_vec()), + // only v in scope. + BytesFromType::Raw(allocator.atom(v).to_vec()), ))), ), } diff --git a/src/classic/clvm_tools/stages/stage_0.rs b/src/classic/clvm_tools/stages/stage_0.rs index eb57d7eb1..6292411b1 100644 --- a/src/classic/clvm_tools/stages/stage_0.rs +++ b/src/classic/clvm_tools/stages/stage_0.rs @@ -1,5 +1,5 @@ use clvm_rs::allocator::{Allocator, NodePtr}; -use clvm_rs::chia_dialect::{ChiaDialect, ENABLE_BLS_OPS, NO_UNKNOWN_OPS}; +use clvm_rs::chia_dialect::{ChiaDialect, ENABLE_BLS_OPS, ENABLE_SECP_OPS, NO_UNKNOWN_OPS}; use clvm_rs::cost::Cost; use clvm_rs::reduction::Response; @@ -47,7 +47,7 @@ impl TRunProgram for DefaultProgramRunner { run_program_with_pre_eval( allocator, - &ChiaDialect::new(NO_UNKNOWN_OPS | ENABLE_BLS_OPS), + &ChiaDialect::new(NO_UNKNOWN_OPS | ENABLE_BLS_OPS | ENABLE_SECP_OPS), program, args, max_cost, diff --git a/src/classic/clvm_tools/stages/stage_2/compile.rs b/src/classic/clvm_tools/stages/stage_2/compile.rs index 24ca19a46..0c0bae4e4 100644 --- a/src/classic/clvm_tools/stages/stage_2/compile.rs +++ b/src/classic/clvm_tools/stages/stage_2/compile.rs @@ -1,7 +1,7 @@ use std::collections::{HashMap, HashSet}; use std::rc::Rc; -use clvm_rs::allocator::{Allocator, AtomBuf, NodePtr, SExp}; +use clvm_rs::allocator::{Allocator, NodePtr, SExp}; use clvm_rs::reduction::{EvalErr, Reduction, Response}; use crate::classic::clvm::__type_compatibility__::{Bytes, BytesFromType}; @@ -122,13 +122,14 @@ pub fn compile_qq( }; match allocator.sexp(sexp) { - SExp::Atom(_) => { + SExp::Atom() => { // (qq ATOM) => (q . ATOM) quote(allocator, sexp) } SExp::Pair(op, sexp_rest) => { - if let SExp::Atom(opbuf) = allocator.sexp(op) { - if allocator.buf(&opbuf).to_vec() == qq_atom() { + if let SExp::Atom() = allocator.sexp(op) { + // opbuf => op + if allocator.atom(op).to_vec() == qq_atom() { return m! { cons_atom <- allocator.new_atom(&[4]); subexp <- @@ -138,7 +139,7 @@ pub fn compile_qq( run_list <- enlist(allocator, &[cons_atom, op, consed]); com_qq(allocator, "qq sexp pair".to_string(), macro_lookup, symbol_table, runner, run_list) }; - } else if allocator.buf(&opbuf).to_vec() == unquote_atom() { + } else if allocator.atom(op).to_vec() == unquote_atom() { // opbuf if level == 1 { // (qq (unquote X)) => X return m! { @@ -207,8 +208,10 @@ fn lower_quote_(allocator: &mut Allocator, prog: NodePtr) -> Result { - if allocator.buf(¯o_name).to_vec() == *operator { + SExp::Atom() => { + // was macro_name, but it's singular and probably + // not useful to rename. + if allocator.atom(mp_list[0]) == operator { return Ok(Some(value)); } } @@ -357,10 +362,10 @@ fn get_macro_program( fn transform_program_atom( allocator: &mut Allocator, prog: NodePtr, - a: &AtomBuf, + a: &[u8], symbol_table: NodePtr, ) -> Response { - if allocator.buf(a).to_vec() == "@".as_bytes().to_vec() { + if a == b"@" { return allocator .new_atom(NodePath::new(None).as_path().data()) .map(|x| Reduction(1, x)); @@ -379,8 +384,10 @@ fn transform_program_atom( let value = if v.len() > 1 { v[1] } else { allocator.null() }; match allocator.sexp(v[0]) { - SExp::Atom(s) => { - if allocator.buf(&s).to_vec() == allocator.buf(a).to_vec() { + SExp::Atom() => { + // v[0] is close by, and probably not useful to + // rename here. + if allocator.atom(v[0]) == a { return Ok(Reduction(1, value)); } } @@ -454,7 +461,7 @@ fn find_symbol_match( } match allocator.sexp(symdef[0]) { - SExp::Atom(symptr) => { + SExp::Atom() => { let symbol = symdef[0]; let value = if symdef.len() == 1 { allocator.null() @@ -462,10 +469,10 @@ fn find_symbol_match( symdef[1] }; - let symbuf = allocator.buf(&symptr).to_vec(); - if vec![b'*'] == symbuf { + let symbuf = allocator.atom(symdef[0]); + if b"*" == symbuf { return Ok(Some(SymbolResult::Direct(r))); - } else if *opname == symbuf { + } else if opname == symbuf { return Ok(Some(SymbolResult::Matched(symbol, value))); } } @@ -630,18 +637,21 @@ fn do_com_prog_( // quote atoms match allocator.sexp(prog) { - SExp::Atom(a) => { + SExp::Atom() => { + // Note: can't co-borrow with allocator below. + let prog_bytes = allocator.atom(prog).to_vec(); transform_program_atom( allocator, prog, - &a, + &prog_bytes, symbol_table ) }, SExp::Pair(operator,prog_rest) => { match allocator.sexp(operator) { - SExp::Atom(a) => { - let opbuf = allocator.buf(&a).to_vec(); + SExp::Atom() => { + // Note: can't co-borrow with allocator below. + let opbuf = allocator.atom(operator).to_vec(); get_macro_program(allocator, &opbuf, macro_lookup). and_then(|x| match x { Some(value) => { @@ -765,14 +775,15 @@ pub fn get_compile_filename( ) -> Result, EvalErr> { let cvt_prog = assemble(allocator, "(_get_compile_filename)")?; - let cvt_prog_result = runner.run_program(allocator, cvt_prog, allocator.null(), None)?; + let Reduction(_, cvt_prog_result) = runner.run_program(allocator, cvt_prog, allocator.null(), None)?; - if cvt_prog_result.1 == allocator.null() { + if cvt_prog_result == allocator.null() { return Ok(None); } - if let SExp::Atom(buf) = allocator.sexp(cvt_prog_result.1) { - let abuf = allocator.buf(&buf).to_vec(); + if let SExp::Atom() = allocator.sexp(cvt_prog_result) { + // only cvt_prog_result in scope. + let abuf = allocator.atom(cvt_prog_result).to_vec(); return Ok(Some(Bytes::new(Some(BytesFromType::Raw(abuf))).decode())); } @@ -792,10 +803,10 @@ pub fn get_search_paths( let mut res = Vec::new(); if let Some(l) = proper_list(allocator, search_path_result.1, true) { - for elt in l.iter() { - if let SExp::Atom(buf) = allocator.sexp(*elt) { - let abuf = allocator.buf(&buf).to_vec(); - res.push(Bytes::new(Some(BytesFromType::Raw(abuf))).decode()); + for elt in l.iter().copied() { + if let SExp::Atom() = allocator.sexp(elt) { + // Only elt in scope. + res.push(Bytes::new(Some(BytesFromType::Raw(allocator.atom(elt).to_vec()))).decode()); } } } diff --git a/src/classic/clvm_tools/stages/stage_2/inline.rs b/src/classic/clvm_tools/stages/stage_2/inline.rs index c989b6ad0..e47abbf73 100644 --- a/src/classic/clvm_tools/stages/stage_2/inline.rs +++ b/src/classic/clvm_tools/stages/stage_2/inline.rs @@ -15,11 +15,11 @@ pub fn is_at_capture( tree_first: NodePtr, tree_rest: NodePtr, ) -> Option<(NodePtr, NodePtr)> { - if let (SExp::Atom(a), Some(spec)) = ( + if let (SExp::Atom(), Some(spec)) = ( allocator.sexp(tree_first), proper_list(allocator, tree_rest, true), ) { - if allocator.buf(&a) == [b'@'] && spec.len() == 2 { + if allocator.atom(tree_first) == b"@" && spec.len() == 2 { return Some((spec[0], spec[1])); } } @@ -88,7 +88,7 @@ fn formulate_path_selections_for_destructuring_arg( SExp::Pair(a, b) => { let next_depth = arg_depth.clone() * 2_u32.to_bigint().unwrap(); if let Some((capture, substructure)) = is_at_capture(allocator, a, b) { - if let SExp::Atom(cbuf) = allocator.sexp(capture) { + if let SExp::Atom() = allocator.sexp(capture) { let (new_arg_path, new_arg_depth, tail) = if let Some(prev_ref) = referenced_from { (arg_path, arg_depth, prev_ref) @@ -99,7 +99,8 @@ fn formulate_path_selections_for_destructuring_arg( (bi_zero(), bi_one(), qtail) }; - selections.insert(allocator.buf(&cbuf).to_vec(), tail); + // Was cbuf from capture. + selections.insert(allocator.atom(capture).to_vec(), tail); return formulate_path_selections_for_destructuring_arg( allocator, @@ -146,8 +147,9 @@ fn formulate_path_selections_for_destructuring_arg( ) } } - SExp::Atom(b) => { - let buf = allocator.buf(&b).to_vec(); + SExp::Atom() => { + // Note: can't co-borrow with allocator below. + let buf = allocator.atom(arg_sexp).to_vec(); if !buf.is_empty() { if let Some(capture) = referenced_from { let tail = wrap_path_selection(allocator, arg_path + arg_depth, capture)?; @@ -223,10 +225,11 @@ pub fn formulate_path_selections_for_destructuring( ) -> Result { if let SExp::Pair(a, b) = allocator.sexp(args_sexp) { if let Some((capture, substructure)) = is_at_capture(allocator, a, b) { - if let SExp::Atom(cbuf) = allocator.sexp(capture) { + if let SExp::Atom() = allocator.sexp(capture) { let quoted_arg_list = wrap_in_unquote(allocator, capture)?; let tail = wrap_in_compile_time_list(allocator, quoted_arg_list)?; - let buf = allocator.buf(&cbuf); + // Was: cbuf from capture. + let buf = allocator.atom(capture); selections.insert(buf.to_vec(), tail); let newsub = formulate_path_selections_for_destructuring_arg( allocator, diff --git a/src/classic/clvm_tools/stages/stage_2/module.rs b/src/classic/clvm_tools/stages/stage_2/module.rs index 6f5f24925..6fb29f74e 100644 --- a/src/classic/clvm_tools/stages/stage_2/module.rs +++ b/src/classic/clvm_tools/stages/stage_2/module.rs @@ -144,8 +144,9 @@ fn build_used_constants_names( .collect::>(); let matching_names = matching_names_1.iter().filter_map(|v| { - if let SExp::Atom(b) = allocator.sexp(*v) { - Some(allocator.buf(&b).to_vec()) + // Only v usefully in scope. + if let SExp::Atom() = allocator.sexp(*v) { + Some(allocator.atom(*v).to_vec()) } else { None } @@ -222,8 +223,9 @@ fn unquote_args( matches: &HashMap, NodePtr>, ) -> Result { match allocator.sexp(code) { - SExp::Atom(code_buf) => { - let code_atom = allocator.buf(&code_buf); + SExp::Atom() => { + // Only code in scope. + let code_atom = allocator.atom(code); let matching_args = args .iter() .filter(|arg| *arg == code_atom) @@ -283,8 +285,9 @@ fn defun_inline_to_macro( let arg_name_list = arg_atom_list .iter() .filter_map(|x| { - if let SExp::Atom(a) = allocator.sexp(*x) { - Some(allocator.buf(&a)) + if let SExp::Atom() = allocator.sexp(*x) { + // only x usefully in scope. + Some(allocator.atom(*x)) } else { None } @@ -321,11 +324,13 @@ fn parse_mod_sexp( .select_nodes(allocator, declaration_sexp)?; let op = match allocator.sexp(op_node) { - SExp::Atom(b) => allocator.buf(&b).to_vec(), + // op_node in use. + SExp::Atom() => allocator.atom(op_node).to_vec(), _ => Vec::new(), }; let name = match allocator.sexp(name_node) { - SExp::Atom(b) => allocator.buf(&b).to_vec(), + // name_node in use. + SExp::Atom() => allocator.atom(name_node).to_vec(), _ => Vec::new(), }; @@ -540,7 +545,7 @@ fn symbol_table_for_tree( } match allocator.sexp(tree) { - SExp::Atom(_) => Ok(vec![(tree, root_node.as_path().data().to_vec())]), + SExp::Atom() => Ok(vec![(tree, root_node.as_path().data().to_vec())]), SExp::Pair(_, _) => { let left_bytes = NodePath::new(None).first(); let right_bytes = NodePath::new(None).rest(); diff --git a/src/classic/clvm_tools/stages/stage_2/operators.rs b/src/classic/clvm_tools/stages/stage_2/operators.rs index 169d7d92e..07927445f 100644 --- a/src/classic/clvm_tools/stages/stage_2/operators.rs +++ b/src/classic/clvm_tools/stages/stage_2/operators.rs @@ -5,7 +5,7 @@ use std::path::PathBuf; use std::rc::Rc; use clvm_rs::allocator::{Allocator, NodePtr, SExp}; -use clvm_rs::chia_dialect::{ChiaDialect, ENABLE_BLS_OPS, NO_UNKNOWN_OPS}; +use clvm_rs::chia_dialect::{ChiaDialect, ENABLE_BLS_OPS, ENABLE_SECP_OPS, NO_UNKNOWN_OPS}; use clvm_rs::cost::Cost; use clvm_rs::dialect::{Dialect, OperatorSet}; use clvm_rs::reduction::{EvalErr, Reduction, Response}; @@ -119,7 +119,7 @@ impl Drop for CompilerOperators { impl CompilerOperatorsInternal { pub fn new(source_file: &str, search_paths: Vec, symbols_extra_info: bool) -> Self { - let base_dialect = Rc::new(ChiaDialect::new(NO_UNKNOWN_OPS | ENABLE_BLS_OPS)); + let base_dialect = Rc::new(ChiaDialect::new(NO_UNKNOWN_OPS | ENABLE_BLS_OPS | ENABLE_SECP_OPS)); let base_runner = Rc::new(DefaultProgramRunner::new()); CompilerOperatorsInternal { base_dialect, @@ -177,9 +177,9 @@ impl CompilerOperatorsInternal { match allocator.sexp(sexp) { SExp::Pair(f, _) => match allocator.sexp(f) { - SExp::Atom(b) => { + SExp::Atom() => { let filename = - Bytes::new(Some(BytesFromType::Raw(allocator.buf(&b).to_vec()))).decode(); + Bytes::new(Some(BytesFromType::Raw(allocator.atom(f).to_vec()))).decode(); // Use the read interface in CompilerOpts if we have one. if let Some(opts) = self.get_compiler_opts() { if let Ok((_, content)) = @@ -210,8 +210,8 @@ impl CompilerOperatorsInternal { fn write(&self, allocator: &mut Allocator, sexp: NodePtr) -> Response { if let SExp::Pair(filename_sexp, r) = allocator.sexp(sexp) { if let SExp::Pair(data, _) = allocator.sexp(r) { - if let SExp::Atom(filename_buf) = allocator.sexp(filename_sexp) { - let filename_buf = allocator.buf(&filename_buf); + if let SExp::Atom() = allocator.sexp(filename_sexp) { + let filename_buf = allocator.atom(filename_sexp); let filename_bytes = Bytes::new(Some(BytesFromType::Raw(filename_buf.to_vec()))); let ir = disassemble_to_ir_with_kw( @@ -257,9 +257,10 @@ impl CompilerOperatorsInternal { }; if let SExp::Pair(l, _r) = allocator.sexp(sexp) { - if let SExp::Atom(b) = allocator.sexp(l) { + if let SExp::Atom() = allocator.sexp(l) { + // l most relevant in scope. let filename = - Bytes::new(Some(BytesFromType::Raw(allocator.buf(&b).to_vec()))).decode(); + Bytes::new(Some(BytesFromType::Raw(allocator.atom(l).to_vec()))).decode(); // If we have a compiler opts injected, let that handle reading // files. The name will bubble up to the _read function. if self.get_compiler_opts().is_some() { @@ -290,14 +291,15 @@ impl CompilerOperatorsInternal { { for kv in symtable.iter() { if let SExp::Pair(hash, name) = allocator.sexp(*kv) { - if let (SExp::Atom(hash), SExp::Atom(name)) = + if let (SExp::Atom(), SExp::Atom()) = (allocator.sexp(hash), allocator.sexp(name)) { + // hash and name in scope. let hash_text = - Bytes::new(Some(BytesFromType::Raw(allocator.buf(&hash).to_vec()))) + Bytes::new(Some(BytesFromType::Raw(allocator.atom(hash).to_vec()))) .decode(); let name_text = - Bytes::new(Some(BytesFromType::Raw(allocator.buf(&name).to_vec()))) + Bytes::new(Some(BytesFromType::Raw(allocator.atom(name).to_vec()))) .decode(); self.compile_outcomes.replace_with(|co| { @@ -321,9 +323,6 @@ impl CompilerOperatorsInternal { } impl Dialect for CompilerOperatorsInternal { - fn stack_limit(&self) -> usize { - 10000000 - } fn quote_kw(&self) -> &[u8] { &[1] } @@ -353,8 +352,9 @@ impl Dialect for CompilerOperatorsInternal { _extension: OperatorSet, ) -> Response { match allocator.sexp(op) { - SExp::Atom(opname) => { - let opbuf = allocator.buf(&opname); + SExp::Atom() => { + // use of op obvious. + let opbuf = allocator.atom(op); if opbuf == "_read".as_bytes() { self.read(allocator, sexp) } else if opbuf == "_write".as_bytes() { diff --git a/src/classic/clvm_tools/stages/stage_2/optimize.rs b/src/classic/clvm_tools/stages/stage_2/optimize.rs index b1820661e..224081a36 100644 --- a/src/classic/clvm_tools/stages/stage_2/optimize.rs +++ b/src/classic/clvm_tools/stages/stage_2/optimize.rs @@ -42,7 +42,7 @@ pub fn seems_constant_tail(allocator: &mut Allocator, sexp_: NodePtr) -> bool { sexp = r; } - SExp::Atom(_) => { + SExp::Atom() => { return sexp == allocator.null(); } } @@ -51,13 +51,14 @@ pub fn seems_constant_tail(allocator: &mut Allocator, sexp_: NodePtr) -> bool { pub fn seems_constant(allocator: &mut Allocator, sexp: NodePtr) -> bool { match allocator.sexp(sexp) { - SExp::Atom(_b) => { + SExp::Atom() => { return sexp == allocator.null(); } SExp::Pair(operator, r) => { match allocator.sexp(operator) { - SExp::Atom(b) => { - let atom = allocator.buf(&b); + SExp::Atom() => { + // Was buf of operator. + let atom = allocator.atom(operator); if atom.len() == 1 && atom[0] == 1 { return true; } else if atom.len() == 1 && atom[0] == 8 { @@ -92,8 +93,9 @@ pub fn constant_optimizer( * return the quoted result. */ if let SExp::Pair(first, _) = allocator.sexp(r) { - if let SExp::Atom(b) = allocator.sexp(first) { - let buf = allocator.buf(&b); + // first relevant in scope. + if let SExp::Atom() = allocator.sexp(first) { + let buf = allocator.atom(first); if buf.len() == 1 && buf[0] == 1 { // Short circuit already quoted expression. return Ok(r); @@ -136,8 +138,9 @@ pub fn constant_optimizer( } pub fn is_args_call(allocator: &mut Allocator, r: NodePtr) -> bool { - if let SExp::Atom(b) = allocator.sexp(r) { - let buf = allocator.buf(&b); + if let SExp::Atom() = allocator.sexp(r) { + // Only r in scope. + let buf = allocator.atom(r); buf.len() == 1 && buf[0] == 1 } else { false @@ -218,8 +221,9 @@ fn path_from_args( new_args: NodePtr, ) -> Result { match allocator.sexp(sexp) { - SExp::Atom(v_buf) => { - let v = number_from_u8(allocator.buf(&v_buf)); + SExp::Atom() => { + // Only sexp in scope. + let v = number_from_u8(allocator.atom(sexp)); if v <= bi_one() { Ok(new_args) } else { @@ -243,7 +247,7 @@ pub fn sub_args( new_args: NodePtr, ) -> Result { match allocator.sexp(sexp) { - SExp::Atom(_) => path_from_args(allocator, sexp, new_args), + SExp::Atom() => path_from_args(allocator, sexp, new_args), SExp::Pair(first_pre, rest) => { let first; @@ -251,8 +255,9 @@ pub fn sub_args( SExp::Pair(_, _) => { first = sub_args(allocator, first_pre, new_args)?; } - SExp::Atom(b) => { - let atom = allocator.buf(&b); + SExp::Atom() => { + // Atom is a reflection of first_pre. + let atom = allocator.atom(first_pre); if atom.len() == 1 && atom[0] == 1 { return Ok(sexp); } else { @@ -370,8 +375,9 @@ pub fn var_change_optimizer_cons_eval( } let increment = match allocator.sexp(val) { SExp::Pair(val_first, _) => match allocator.sexp(val_first) { - SExp::Atom(b) => { - let vf_buf = allocator.buf(&b); + SExp::Atom() => { + // Atom reflects val_first. + let vf_buf = allocator.atom(val_first); (vf_buf.len() != 1 || vf_buf[0] != 1) as i32 } _ => 0, @@ -414,8 +420,8 @@ pub fn children_optimizer( if list.is_empty() { return Ok(r); } - if let SExp::Atom(op_buf) = allocator.sexp(list[0]) { - if allocator.buf(&op_buf).to_vec() == vec![1] { + if let SExp::Atom() = allocator.sexp(list[0]) { + if allocator.atom(list[0]).to_vec() == vec![1] { return Ok(r); } } @@ -519,7 +525,7 @@ fn path_optimizer( match first. get("atom"). and_then(|a| atom(allocator, *a).ok()). - map(|atom| number_from_u8(allocator.buf(&atom))) + map(|atom| number_from_u8(&atom)) { Some(atom) => { let node = @@ -534,7 +540,7 @@ fn path_optimizer( match rest. get("atom"). and_then(|a| atom(allocator, *a).ok()). - map(|atom| number_from_u8(allocator.buf(&atom))) + map(|atom| number_from_u8(&atom)) { Some(atom) => { let node = @@ -678,7 +684,7 @@ pub fn optimize_sexp_( let mut name = "".to_string(); match allocator.sexp(r) { - SExp::Atom(_) => { + SExp::Atom() => { return Ok(r); } SExp::Pair(_, _) => { diff --git a/src/classic/clvm_tools/stages/stage_2/reader.rs b/src/classic/clvm_tools/stages/stage_2/reader.rs index 0ff21e906..bc91c89b4 100644 --- a/src/classic/clvm_tools/stages/stage_2/reader.rs +++ b/src/classic/clvm_tools/stages/stage_2/reader.rs @@ -90,16 +90,16 @@ pub fn process_embed_file( )); } - if let (SExp::Atom(name), SExp::Atom(kind), SExp::Atom(filename)) = ( + if let (SExp::Atom(), SExp::Atom(), SExp::Atom()) = ( allocator.sexp(l[0]), allocator.sexp(l[1]), allocator.sexp(l[2]), ) { // Note: we don't want to keep borrowing here because we // need the mutable borrow below. - let name_buf = allocator.buf(&name).to_vec(); - let kind_buf = allocator.buf(&kind).to_vec(); - let filename_buf = allocator.buf(&filename).to_vec(); + let name_buf = allocator.atom(l[0]).to_vec(); + let kind_buf = allocator.atom(l[1]); + let filename_buf = allocator.atom(l[2]).to_vec(); let file_data = if kind_buf == b"bin" { let file = read_file( runner, @@ -128,7 +128,7 @@ pub fn process_embed_file( return Err(EvalErr(declaration_sexp, "no such embed kind".to_string())); }; - Ok((name_buf, quote(allocator, file_data)?)) + Ok((name_buf.to_vec(), quote(allocator, file_data)?)) } else { Err(EvalErr( declaration_sexp, diff --git a/src/compiler/clvm.rs b/src/compiler/clvm.rs index eaa6e9a4e..dd4ad017c 100644 --- a/src/compiler/clvm.rs +++ b/src/compiler/clvm.rs @@ -242,11 +242,11 @@ pub fn convert_from_clvm_rs( head: NodePtr, ) -> Result, RunFailure> { match allocator.sexp(head) { - allocator::SExp::Atom(h) => { - if h.is_empty() { + allocator::SExp::Atom() => { + let atom_data = allocator.atom(head); + if atom_data.is_empty() { Ok(Rc::new(SExp::Nil(loc))) } else { - let atom_data = allocator.buf(&h); let integer = number_from_u8(atom_data); // Ensure that atom values that don't evaluate equal to integers // are represented faithfully as atoms. diff --git a/src/compiler/prims.rs b/src/compiler/prims.rs index c2e6d14bd..8a30023b2 100644 --- a/src/compiler/prims.rs +++ b/src/compiler/prims.rs @@ -144,39 +144,39 @@ pub fn prims() -> Vec<(Vec, SExp)> { SExp::Integer(primloc.clone(), 48_u32.to_bigint().unwrap()), ), ( - "bls_g1_subtract".as_bytes().to_vec(), + "g1_subtract".as_bytes().to_vec(), SExp::Integer(primloc.clone(), 49_u32.to_bigint().unwrap()), ), ( - "bls_g1_multiply".as_bytes().to_vec(), + "g1_multiply".as_bytes().to_vec(), SExp::Integer(primloc.clone(), 50_u32.to_bigint().unwrap()), ), ( - "bls_g1_negate".as_bytes().to_vec(), + "g1_negate".as_bytes().to_vec(), SExp::Integer(primloc.clone(), 51_u32.to_bigint().unwrap()), ), ( - "bls_g2_add".as_bytes().to_vec(), + "g2_add".as_bytes().to_vec(), SExp::Integer(primloc.clone(), 52_u32.to_bigint().unwrap()), ), ( - "bls_g2_subtract".as_bytes().to_vec(), + "g2_subtract".as_bytes().to_vec(), SExp::Integer(primloc.clone(), 53_u32.to_bigint().unwrap()), ), ( - "bls_g2_multiply".as_bytes().to_vec(), + "g2_multiply".as_bytes().to_vec(), SExp::Integer(primloc.clone(), 54_u32.to_bigint().unwrap()), ), ( - "bls_g2_negate".as_bytes().to_vec(), + "g2_negate".as_bytes().to_vec(), SExp::Integer(primloc.clone(), 55_u32.to_bigint().unwrap()), ), ( - "bls_map_to_g1".as_bytes().to_vec(), + "g1_map".as_bytes().to_vec(), SExp::Integer(primloc.clone(), 56_u32.to_bigint().unwrap()), ), ( - "bls_map_to_g2".as_bytes().to_vec(), + "g2_map".as_bytes().to_vec(), SExp::Integer(primloc.clone(), 57_u32.to_bigint().unwrap()), ), ( @@ -185,7 +185,15 @@ pub fn prims() -> Vec<(Vec, SExp)> { ), ( "bls_verify".as_bytes().to_vec(), - SExp::Integer(primloc, 59_u32.to_bigint().unwrap()), + SExp::Integer(primloc.clone(), 59_u32.to_bigint().unwrap()), + ), + ( + "secp256k1_verify".as_bytes().to_vec(), + SExp::Integer(primloc.clone(), 0x13d61f00.to_bigint().unwrap()), + ), + ( + "secp256r1_verify".as_bytes().to_vec(), + SExp::Integer(primloc, 0x1c3a8f00.to_bigint().unwrap()), ), ] } diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index 0fbbf5ddf..a23e0c21d 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -869,3 +869,269 @@ fn test_cost_reporting_0() { "cost = 1978\n0x6fcb06b1fe29d132bb37f3a21b86d7cf03d636bf6230aa206486bef5e68f9875" ); } + +#[test] +fn test_g1_map_op_modern() { + let program = "(mod (S) (include *standard-cl-21*) (g1_map S \"BLS_SIG_BLS12381G1_XMD:SHA-256_SSWU_RO_AUG_\"))"; + let compiled = do_basic_run(&vec![ + "run".to_string(), + program.to_string(), + ]); + let output = do_basic_brun(&vec![ + "brun".to_string(), + compiled, + "(abcdef0123456789)".to_string(), + ]) + .trim() + .to_string(); + assert_eq!( + output, + "0x88e7302bf1fa8fcdecfb96f6b81475c3564d3bcaf552ccb338b1c48b9ba18ab7195c5067fe94fb216478188c0a3bef4a" + ); +} + +#[test] +fn test_g1_map_op_classic() { + let program = "(mod (S) (g1_map S \"BLS_SIG_BLS12381G1_XMD:SHA-256_SSWU_RO_AUG_\"))"; + let compiled = do_basic_run(&vec![ + "run".to_string(), + program.to_string(), + ]); + let output = do_basic_brun(&vec![ + "brun".to_string(), + compiled, + "(abcdef0123456789)".to_string(), + ]) + .trim() + .to_string(); + assert_eq!( + output, + "0x88e7302bf1fa8fcdecfb96f6b81475c3564d3bcaf552ccb338b1c48b9ba18ab7195c5067fe94fb216478188c0a3bef4a" + ); +} + +#[test] +fn test_g2_map_op_modern() { + let program = "(mod (S) (include *standard-cl-21*) (g2_map S \"BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_AUG_\"))"; + let compiled = do_basic_run(&vec![ + "run".to_string(), + program.to_string(), + ]); + let output = do_basic_brun(&vec![ + "brun".to_string(), + compiled, + "(0x21473dab7ad0136f7488128d44247b04fa58a9c6b4fab6ef4d)".to_string() + ]) + .trim() + .to_string(); + assert_eq!( + output, + "0x879584f6c205b4492abca2be331fc2875596b08c7fbd958fb8d5e725a479d1794b85add1266fb5d410de5c416ce12305166b1c3e2e5d5ae2720a058169b057520d8f2a315f6097c774f659ce5619a070e1cbc8212fb460758e459498d0e598d6" + ); +} + +#[test] +fn test_g2_map_op_classic() { + let program = "(mod (S) (g2_map S \"BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_AUG_\"))"; + let compiled = do_basic_run(&vec![ + "run".to_string(), + program.to_string(), + ]); + let output = do_basic_brun(&vec![ + "brun".to_string(), + compiled, + "(0x21473dab7ad0136f7488128d44247b04fa58a9c6b4fab6ef4d)".to_string() + ]) + .trim() + .to_string(); + assert_eq!( + output, + "0x879584f6c205b4492abca2be331fc2875596b08c7fbd958fb8d5e725a479d1794b85add1266fb5d410de5c416ce12305166b1c3e2e5d5ae2720a058169b057520d8f2a315f6097c774f659ce5619a070e1cbc8212fb460758e459498d0e598d6" + ); +} + +#[test] +fn test_secp256k1_verify_modern_succeed() { + let program = "(mod (S) (include *standard-cl-21*) (secp256k1_verify S 0x85932e4d075615be881398cc765f9f78204033f0ef5f832ac37e732f5f0cbda2 0x481477e62a1d02268127ae89cc58929e09ad5d30229721965ae35965d098a5f630205a7e69f4cb8084f16c7407ed7312994ffbf87ba5eb1aee16682dd324943e))"; + let compiled = do_basic_run(&vec![ + "run".to_string(), + program.to_string(), + ]); + let output = do_basic_brun(&vec![ + "brun".to_string(), + compiled, + "(0x02390b19842e100324163334b16947f66125b76d4fa4a11b9ccdde9b7398e64076)".to_string() + ]) + .trim() + .to_string(); + assert_eq!( + output, + "()" + ); +} + +#[test] +fn test_secp256k1_verify_modern_fail() { + let program = "(mod (S) (include *standard-cl-21*) (secp256k1_verify S 0x935d863e2d28d8e5d399ea8af7393ef11fdffc7d862dcc6b5217a8ef15fb5442 0xbbf0712cc0a283a842011c19682629a5381c5f7ead576defcf12a9a19378e23b087cd0be730dbe78722dcfc81543fca17a30e41070ca2e5b3ae77ccec2cca935))"; + let compiled = do_basic_run(&vec![ + "run".to_string(), + program.to_string(), + ]); + let output = do_basic_brun(&vec![ + "brun".to_string(), + compiled, + "(0x0215043e969dcf616fabe8e8d6b61ddcf6e274c5b04fce957b086dbeb7e899ac63)".to_string() + ]) + .trim() + .to_string(); + assert!(output.starts_with("FAIL: secp256k1_verify failed")); +} + +#[test] +fn test_secp256k1_verify_classic_succeed() { + let program = "(mod (S) (secp256k1_verify S 0x85932e4d075615be881398cc765f9f78204033f0ef5f832ac37e732f5f0cbda2 0x481477e62a1d02268127ae89cc58929e09ad5d30229721965ae35965d098a5f630205a7e69f4cb8084f16c7407ed7312994ffbf87ba5eb1aee16682dd324943e))"; + let compiled = do_basic_run(&vec![ + "run".to_string(), + program.to_string(), + ]); + let output = do_basic_brun(&vec![ + "brun".to_string(), + compiled, + "(0x02390b19842e100324163334b16947f66125b76d4fa4a11b9ccdde9b7398e64076)".to_string() + ]) + .trim() + .to_string(); + assert_eq!( + output, + "()" + ); +} + +#[test] +fn test_secp256k1_verify_classic_fail() { + let program = "(mod (S) (secp256k1_verify S 0x935d863e2d28d8e5d399ea8af7393ef11fdffc7d862dcc6b5217a8ef15fb5442 0xbbf0712cc0a283a842011c19682629a5381c5f7ead576defcf12a9a19378e23b087cd0be730dbe78722dcfc81543fca17a30e41070ca2e5b3ae77ccec2cca935))"; + let compiled = do_basic_run(&vec![ + "run".to_string(), + program.to_string(), + ]); + let output = do_basic_brun(&vec![ + "brun".to_string(), + compiled, + "(0x0215043e969dcf616fabe8e8d6b61ddcf6e274c5b04fce957b086dbeb7e899ac63)".to_string() + ]) + .trim() + .to_string(); + assert!(output.starts_with("FAIL: secp256k1_verify failed")); +} + +#[test] +fn test_secp256k1_verify_modern_int_succeed() { + // Ensure that even if translated to integer (for example via classic unhygenic macro invocation), this works. + let program = "(mod (S) (if S (332799744 S 0x85932e4d075615be881398cc765f9f78204033f0ef5f832ac37e732f5f0cbda2 0x481477e62a1d02268127ae89cc58929e09ad5d30229721965ae35965d098a5f630205a7e69f4cb8084f16c7407ed7312994ffbf87ba5eb1aee16682dd324943e) \"empty-secp\"))"; + let compiled = do_basic_run(&vec![ + "run".to_string(), + program.to_string(), + ]); + let output = do_basic_brun(&vec![ + "brun".to_string(), + compiled, + "(0x02390b19842e100324163334b16947f66125b76d4fa4a11b9ccdde9b7398e64076)".to_string() + ]) + .trim() + .to_string(); + assert_eq!( + output, + "()" + ); +} + +#[test] +fn test_secp256k1_verify_modern_int_fail() { + let program = "(mod (S) (include *standard-cl-21*) (if S (332799744 S 0x935d863e2d28d8e5d399ea8af7393ef11fdffc7d862dcc6b5217a8ef15fb5442 0xbbf0712cc0a283a842011c19682629a5381c5f7ead576defcf12a9a19378e23b087cd0be730dbe78722dcfc81543fca17a30e41070ca2e5b3ae77ccec2cca935) \"empty-secp\"))"; + let compiled = do_basic_run(&vec![ + "run".to_string(), + program.to_string(), + ]); + let output = do_basic_brun(&vec![ + "brun".to_string(), + compiled, + "(0x0215043e969dcf616fabe8e8d6b61ddcf6e274c5b04fce957b086dbeb7e899ac63)".to_string() + ]) + .trim() + .to_string(); + assert!(output.starts_with("FAIL: secp256k1_verify failed")); +} + +#[test] +fn test_secp256r1_verify_modern_succeed() { + let program = "(mod (S) (include *standard-cl-21*) (secp256r1_verify S 0x85932e4d075615be881398cc765f9f78204033f0ef5f832ac37e732f5f0cbda2 0xeae2f488080919bd0a7069c24cdd9c6ce2db423861b0c9d4236cdadbd0005f6d8f3709e6eb19249fd9c8bea664aba35218e67ea4b0f2239488dc3147f336e1e6))"; + let compiled = do_basic_run(&vec![ + "run".to_string(), + program.to_string(), + ]); + let output = do_basic_brun(&vec![ + "brun".to_string(), + compiled, + "(0x033e1a1b2ccbc35883c60fdfc3f4a02175096ade6271fe85517ca5772594bbd0dc)".to_string() + ]) + .trim() + .to_string(); + assert_eq!( + output, + "()" + ); +} + +#[test] +fn test_secp256r1_verify_modern_fail() { + let program = "(mod (S) (include *standard-cl-21*) (secp256r1_verify S 0x935d863e2d28d8e5d399ea8af7393ef11fdffc7d862dcc6b5217a8ef15fb5442 0xecef274a7408e6cb0196eac64d2ae32fc54c2537f8a9efd5b75a4e8a53b0b156c64564306f38bade4adceac1073d464e4db3d0332141a7203dfd113ad36e393d))"; + let compiled = do_basic_run(&vec![ + "run".to_string(), + program.to_string(), + ]); + let output = do_basic_brun(&vec![ + "brun".to_string(), + compiled, + "(0x025195db74b1902d53758a62ccd9dd01837aa5ae755e878eb0aeccbe8fe477c543)".to_string() + ]) + .trim() + .to_string(); + assert!(output.starts_with("FAIL: secp256r1_verify failed")); +} + +#[test] +fn test_secp256r1_verify_classic_succeed() { + let program = "(mod (S) (secp256r1_verify S 0x85932e4d075615be881398cc765f9f78204033f0ef5f832ac37e732f5f0cbda2 0xeae2f488080919bd0a7069c24cdd9c6ce2db423861b0c9d4236cdadbd0005f6d8f3709e6eb19249fd9c8bea664aba35218e67ea4b0f2239488dc3147f336e1e6))"; + let compiled = do_basic_run(&vec![ + "run".to_string(), + program.to_string(), + ]); + let output = do_basic_brun(&vec![ + "brun".to_string(), + compiled, + "(0x033e1a1b2ccbc35883c60fdfc3f4a02175096ade6271fe85517ca5772594bbd0dc)".to_string() + ]) + .trim() + .to_string(); + assert_eq!( + output, + "()" + ); +} + +#[test] +fn test_secp256r1_verify_classic_fail() { + let program = "(mod (S) (secp256r1_verify S 0x935d863e2d28d8e5d399ea8af7393ef11fdffc7d862dcc6b5217a8ef15fb5442 0xecef274a7408e6cb0196eac64d2ae32fc54c2537f8a9efd5b75a4e8a53b0b156c64564306f38bade4adceac1073d464e4db3d0332141a7203dfd113ad36e393d))"; + let compiled = do_basic_run(&vec![ + "run".to_string(), + program.to_string(), + ]); + let output = do_basic_brun(&vec![ + "brun".to_string(), + compiled, + "(0x025195db74b1902d53758a62ccd9dd01837aa5ae755e878eb0aeccbe8fe477c543)".to_string() + ]) + .trim() + .to_string(); + assert!(output.starts_with("FAIL: secp256r1_verify failed")); +} diff --git a/src/tests/classic/smoke.rs b/src/tests/classic/smoke.rs index 37edd0961..b723bd869 100644 --- a/src/tests/classic/smoke.rs +++ b/src/tests/classic/smoke.rs @@ -1,5 +1,6 @@ use num_bigint::ToBigInt; +use std::collections::HashMap; use std::fs; use std::io; use std::path::PathBuf; @@ -20,9 +21,11 @@ use crate::classic::clvm_tools::binutils::{assemble, assemble_from_ir, disassemb use crate::classic::clvm_tools::ir::r#type::IRRepr; use crate::classic::clvm_tools::ir::reader::read_ir; use crate::classic::clvm_tools::node_path::NodePath; +use crate::classic::clvm_tools::pattern_match::match_sexp; use crate::classic::clvm_tools::stages; use crate::classic::clvm_tools::stages::stage_0::{DefaultProgramRunner, TRunProgram}; use crate::classic::clvm_tools::stages::stage_2::operators::run_program_for_search_paths; +use crate::classic::clvm_tools::stages::stage_2::optimize::sub_args; use crate::classic::platform::argparse::{ Argument, ArgumentParser, NArgsSpec, TArgumentParserProps, }; @@ -145,8 +148,8 @@ fn mid_negative_value_bin() { Box::new(SimpleCreateCLVMObject {}), ) .expect("should be able to make nodeptr"); - if let SExp::Atom(abuf) = allocator.sexp(atom.1) { - let res_bytes = allocator.buf(&abuf); + if let SExp::Atom() = allocator.sexp(atom.1) { + let res_bytes = allocator.atom(atom.1); assert_eq!(res_bytes, &[0xff, 0xff]); } else { assert!(false); @@ -273,8 +276,8 @@ fn can_run_from_source_nil() { let mut allocator = Allocator::new(); let res = run_from_source(&mut allocator, "()".to_string()); match allocator.sexp(res) { - SExp::Atom(b) => { - let res_bytes = allocator.buf(&b).to_vec(); + SExp::Atom() => { + let res_bytes = allocator.atom(res); assert_eq!(res_bytes.len(), 0); } _ => { @@ -288,8 +291,8 @@ fn can_echo_quoted_nil() { let mut allocator = Allocator::new(); let res = run_from_source(&mut allocator, "(1)".to_string()); match allocator.sexp(res) { - SExp::Atom(b) => { - let res_bytes = allocator.buf(&b).to_vec(); + SExp::Atom() => { + let res_bytes = allocator.atom(res); assert_eq!(res_bytes.len(), 0); } _ => { @@ -319,8 +322,8 @@ fn can_echo_quoted_atom() { let mut allocator = Allocator::new(); let res = run_from_source(&mut allocator, "(1 . 3)".to_string()); match allocator.sexp(res) { - SExp::Atom(b) => { - let res_bytes = allocator.buf(&b).to_vec(); + SExp::Atom() => { + let res_bytes = allocator.atom(res); assert_eq!(res_bytes.len(), 1); assert_eq!(res_bytes[0], 3); } @@ -335,8 +338,8 @@ fn can_do_operations() { let mut allocator = Allocator::new(); let res = run_from_source(&mut allocator, "(16 (1 . 3) (1 . 5))".to_string()); match allocator.sexp(res) { - SExp::Atom(b) => { - let res_bytes = allocator.buf(&b).to_vec(); + SExp::Atom() => { + let res_bytes = allocator.atom(res); assert_eq!(res_bytes.len(), 1); assert_eq!(res_bytes[0], 8); } @@ -351,8 +354,8 @@ fn can_do_operations_kw() { let mut allocator = Allocator::new(); let res = run_from_source(&mut allocator, "(+ (q . 3) (q . 5))".to_string()); match allocator.sexp(res) { - SExp::Atom(b) => { - let res_bytes = allocator.buf(&b).to_vec(); + SExp::Atom() => { + let res_bytes = allocator.atom(res); assert_eq!(res_bytes.len(), 1); assert_eq!(res_bytes[0], 8); } @@ -778,3 +781,34 @@ fn test_bytes_to_pybytes_repr_0() { "b'\\x11\\x01abc\\r\\ntest\\ttest\\r\\n'" ); } + +#[test] +fn test_pattern_match_dollar_for_dollar() { + let mut allocator = Allocator::new(); + let pattern = assemble(&mut allocator, "($ . $)").expect("should assemble"); + let target_expr = assemble(&mut allocator, "$").expect("should assemble"); + let empty_map = HashMap::new(); + let matched = match_sexp(&mut allocator, pattern, target_expr, empty_map.clone()); + // Returns empty map. + assert_eq!(Some(empty_map), matched); +} + +#[test] +fn test_pattern_match_colon_for_colon() { + let mut allocator = Allocator::new(); + let pattern = assemble(&mut allocator, "(: . :)").expect("should assemble"); + let target_expr = assemble(&mut allocator, ":").expect("should assemble"); + let empty_map = HashMap::new(); + let matched = match_sexp(&mut allocator, pattern, target_expr, empty_map.clone()); + // Returns empty map. + assert_eq!(Some(empty_map), matched); +} + +#[test] +fn test_sub_args() { + let mut allocator = Allocator::new(); + let expr_sexp = assemble(&mut allocator, "(body 2 5)").expect("should assemble"); + let new_args = assemble(&mut allocator, "(test1 test2)").expect("should assemble"); + let result = sub_args(&mut allocator, expr_sexp, new_args).expect("should run"); + assert_eq!(disassemble(&mut allocator, result, None), "(\"body\" (f (\"test1\" \"test2\")) (f (r (\"test1\" \"test2\"))))"); +} diff --git a/src/tests/compiler/repl.rs b/src/tests/compiler/repl.rs index 2dfea9d47..94ef84ff7 100644 --- a/src/tests/compiler/repl.rs +++ b/src/tests/compiler/repl.rs @@ -309,7 +309,7 @@ fn test_eval_new_bls_operator() { "(softfork (q . 196005) (q . 0) - (q #bls_map_to_g1 (1 . 0x9790635de8740e9a6a6b15fb6b72f3a16afa0973d971979b6ba54761d6e2502c50db76f4d26143f05459a42cfd520d44)) () + (q #g1_map (1 . 0x9790635de8740e9a6a6b15fb6b72f3a16afa0973d971979b6ba54761d6e2502c50db76f4d26143f05459a42cfd520d44)) () )"}.to_string() ], None).unwrap().unwrap(), "(q)" From c24da07a88af5b53a7bf5468cafba07250189e26 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 27 Jun 2023 10:55:56 -0700 Subject: [PATCH 037/117] fmt + clippy --- src/classic/clvm/mod.rs | 6 +- src/classic/clvm_tools/pattern_match.rs | 13 +- .../clvm_tools/stages/stage_2/compile.rs | 10 +- .../clvm_tools/stages/stage_2/operators.rs | 4 +- src/tests/classic/run.rs | 175 ++++++------------ src/tests/classic/smoke.rs | 5 +- 6 files changed, 81 insertions(+), 132 deletions(-) diff --git a/src/classic/clvm/mod.rs b/src/classic/clvm/mod.rs index 0254281b2..6c40d51d9 100644 --- a/src/classic/clvm/mod.rs +++ b/src/classic/clvm/mod.rs @@ -238,15 +238,15 @@ const KW_PAIRS: [KwAtomPair; 46] = [ version: 1, }, KwAtomPair { - v: &[0x13,0xd6,0x1f,0x00], + v: &[0x13, 0xd6, 0x1f, 0x00], n: "secp256k1_verify", version: 1, }, KwAtomPair { - v: &[0x1c,0x3a,0x8f,0x00], + v: &[0x1c, 0x3a, 0x8f, 0x00], n: "secp256r1_verify", version: 1, - } + }, ]; lazy_static! { diff --git a/src/classic/clvm_tools/pattern_match.rs b/src/classic/clvm_tools/pattern_match.rs index 75c6eba35..0df1345cd 100644 --- a/src/classic/clvm_tools/pattern_match.rs +++ b/src/classic/clvm_tools/pattern_match.rs @@ -61,9 +61,11 @@ pub fn match_sexp( } (SExp::Pair(pleft, pright), _) => match (allocator.sexp(pleft), allocator.sexp(pright)) { (SExp::Atom(), SExp::Atom()) => { + let pright_atom = allocator.atom(pright).to_vec(); match allocator.sexp(sexp) { SExp::Atom() => { // Expression is ($ . $), sexp is '$', result: no capture. + // Avoid double borrow. if allocator.atom(pleft) == ATOM_MATCH { if allocator.atom(pright) == ATOM_MATCH { if allocator.atom(sexp) == ATOM_MATCH { @@ -72,12 +74,7 @@ pub fn match_sexp( return None; } - return unify_bindings( - allocator, - known_bindings, - &allocator.atom(pright).to_vec(), - sexp, - ); + return unify_bindings(allocator, known_bindings, &pright_atom, sexp); } if allocator.atom(pleft) == SEXP_MATCH { if allocator.atom(pright) == SEXP_MATCH @@ -90,7 +87,7 @@ pub fn match_sexp( allocator, known_bindings, // pat_right_bytes - &allocator.atom(pright).to_vec(), + &pright_atom, sexp, ); } @@ -105,7 +102,7 @@ pub fn match_sexp( allocator, known_bindings, // pat_right_bytes - &allocator.atom(pright).to_vec(), + &pright_atom, sexp, ); } diff --git a/src/classic/clvm_tools/stages/stage_2/compile.rs b/src/classic/clvm_tools/stages/stage_2/compile.rs index 0c0bae4e4..74a9202df 100644 --- a/src/classic/clvm_tools/stages/stage_2/compile.rs +++ b/src/classic/clvm_tools/stages/stage_2/compile.rs @@ -139,7 +139,8 @@ pub fn compile_qq( run_list <- enlist(allocator, &[cons_atom, op, consed]); com_qq(allocator, "qq sexp pair".to_string(), macro_lookup, symbol_table, runner, run_list) }; - } else if allocator.atom(op).to_vec() == unquote_atom() { // opbuf + } else if allocator.atom(op).to_vec() == unquote_atom() { + // opbuf if level == 1 { // (qq (unquote X)) => X return m! { @@ -775,7 +776,8 @@ pub fn get_compile_filename( ) -> Result, EvalErr> { let cvt_prog = assemble(allocator, "(_get_compile_filename)")?; - let Reduction(_, cvt_prog_result) = runner.run_program(allocator, cvt_prog, allocator.null(), None)?; + let Reduction(_, cvt_prog_result) = + runner.run_program(allocator, cvt_prog, allocator.null(), None)?; if cvt_prog_result == allocator.null() { return Ok(None); @@ -806,7 +808,9 @@ pub fn get_search_paths( for elt in l.iter().copied() { if let SExp::Atom() = allocator.sexp(elt) { // Only elt in scope. - res.push(Bytes::new(Some(BytesFromType::Raw(allocator.atom(elt).to_vec()))).decode()); + res.push( + Bytes::new(Some(BytesFromType::Raw(allocator.atom(elt).to_vec()))).decode(), + ); } } } diff --git a/src/classic/clvm_tools/stages/stage_2/operators.rs b/src/classic/clvm_tools/stages/stage_2/operators.rs index 07927445f..4f33fadd7 100644 --- a/src/classic/clvm_tools/stages/stage_2/operators.rs +++ b/src/classic/clvm_tools/stages/stage_2/operators.rs @@ -119,7 +119,9 @@ impl Drop for CompilerOperators { impl CompilerOperatorsInternal { pub fn new(source_file: &str, search_paths: Vec, symbols_extra_info: bool) -> Self { - let base_dialect = Rc::new(ChiaDialect::new(NO_UNKNOWN_OPS | ENABLE_BLS_OPS | ENABLE_SECP_OPS)); + let base_dialect = Rc::new(ChiaDialect::new( + NO_UNKNOWN_OPS | ENABLE_BLS_OPS | ENABLE_SECP_OPS, + )); let base_runner = Rc::new(DefaultProgramRunner::new()); CompilerOperatorsInternal { base_dialect, diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index a23e0c21d..ae9d00e56 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -873,17 +873,14 @@ fn test_cost_reporting_0() { #[test] fn test_g1_map_op_modern() { let program = "(mod (S) (include *standard-cl-21*) (g1_map S \"BLS_SIG_BLS12381G1_XMD:SHA-256_SSWU_RO_AUG_\"))"; - let compiled = do_basic_run(&vec![ - "run".to_string(), - program.to_string(), - ]); + let compiled = do_basic_run(&vec!["run".to_string(), program.to_string()]); let output = do_basic_brun(&vec![ "brun".to_string(), compiled, "(abcdef0123456789)".to_string(), ]) - .trim() - .to_string(); + .trim() + .to_string(); assert_eq!( output, "0x88e7302bf1fa8fcdecfb96f6b81475c3564d3bcaf552ccb338b1c48b9ba18ab7195c5067fe94fb216478188c0a3bef4a" @@ -893,17 +890,14 @@ fn test_g1_map_op_modern() { #[test] fn test_g1_map_op_classic() { let program = "(mod (S) (g1_map S \"BLS_SIG_BLS12381G1_XMD:SHA-256_SSWU_RO_AUG_\"))"; - let compiled = do_basic_run(&vec![ - "run".to_string(), - program.to_string(), - ]); + let compiled = do_basic_run(&vec!["run".to_string(), program.to_string()]); let output = do_basic_brun(&vec![ "brun".to_string(), compiled, "(abcdef0123456789)".to_string(), ]) - .trim() - .to_string(); + .trim() + .to_string(); assert_eq!( output, "0x88e7302bf1fa8fcdecfb96f6b81475c3564d3bcaf552ccb338b1c48b9ba18ab7195c5067fe94fb216478188c0a3bef4a" @@ -913,17 +907,14 @@ fn test_g1_map_op_classic() { #[test] fn test_g2_map_op_modern() { let program = "(mod (S) (include *standard-cl-21*) (g2_map S \"BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_AUG_\"))"; - let compiled = do_basic_run(&vec![ - "run".to_string(), - program.to_string(), - ]); + let compiled = do_basic_run(&vec!["run".to_string(), program.to_string()]); let output = do_basic_brun(&vec![ "brun".to_string(), compiled, - "(0x21473dab7ad0136f7488128d44247b04fa58a9c6b4fab6ef4d)".to_string() + "(0x21473dab7ad0136f7488128d44247b04fa58a9c6b4fab6ef4d)".to_string(), ]) - .trim() - .to_string(); + .trim() + .to_string(); assert_eq!( output, "0x879584f6c205b4492abca2be331fc2875596b08c7fbd958fb8d5e725a479d1794b85add1266fb5d410de5c416ce12305166b1c3e2e5d5ae2720a058169b057520d8f2a315f6097c774f659ce5619a070e1cbc8212fb460758e459498d0e598d6" @@ -933,17 +924,14 @@ fn test_g2_map_op_modern() { #[test] fn test_g2_map_op_classic() { let program = "(mod (S) (g2_map S \"BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_AUG_\"))"; - let compiled = do_basic_run(&vec![ - "run".to_string(), - program.to_string(), - ]); + let compiled = do_basic_run(&vec!["run".to_string(), program.to_string()]); let output = do_basic_brun(&vec![ "brun".to_string(), compiled, - "(0x21473dab7ad0136f7488128d44247b04fa58a9c6b4fab6ef4d)".to_string() + "(0x21473dab7ad0136f7488128d44247b04fa58a9c6b4fab6ef4d)".to_string(), ]) - .trim() - .to_string(); + .trim() + .to_string(); assert_eq!( output, "0x879584f6c205b4492abca2be331fc2875596b08c7fbd958fb8d5e725a479d1794b85add1266fb5d410de5c416ce12305166b1c3e2e5d5ae2720a058169b057520d8f2a315f6097c774f659ce5619a070e1cbc8212fb460758e459498d0e598d6" @@ -953,74 +941,56 @@ fn test_g2_map_op_classic() { #[test] fn test_secp256k1_verify_modern_succeed() { let program = "(mod (S) (include *standard-cl-21*) (secp256k1_verify S 0x85932e4d075615be881398cc765f9f78204033f0ef5f832ac37e732f5f0cbda2 0x481477e62a1d02268127ae89cc58929e09ad5d30229721965ae35965d098a5f630205a7e69f4cb8084f16c7407ed7312994ffbf87ba5eb1aee16682dd324943e))"; - let compiled = do_basic_run(&vec![ - "run".to_string(), - program.to_string(), - ]); + let compiled = do_basic_run(&vec!["run".to_string(), program.to_string()]); let output = do_basic_brun(&vec![ "brun".to_string(), compiled, - "(0x02390b19842e100324163334b16947f66125b76d4fa4a11b9ccdde9b7398e64076)".to_string() + "(0x02390b19842e100324163334b16947f66125b76d4fa4a11b9ccdde9b7398e64076)".to_string(), ]) - .trim() - .to_string(); - assert_eq!( - output, - "()" - ); + .trim() + .to_string(); + assert_eq!(output, "()"); } #[test] fn test_secp256k1_verify_modern_fail() { let program = "(mod (S) (include *standard-cl-21*) (secp256k1_verify S 0x935d863e2d28d8e5d399ea8af7393ef11fdffc7d862dcc6b5217a8ef15fb5442 0xbbf0712cc0a283a842011c19682629a5381c5f7ead576defcf12a9a19378e23b087cd0be730dbe78722dcfc81543fca17a30e41070ca2e5b3ae77ccec2cca935))"; - let compiled = do_basic_run(&vec![ - "run".to_string(), - program.to_string(), - ]); + let compiled = do_basic_run(&vec!["run".to_string(), program.to_string()]); let output = do_basic_brun(&vec![ "brun".to_string(), compiled, - "(0x0215043e969dcf616fabe8e8d6b61ddcf6e274c5b04fce957b086dbeb7e899ac63)".to_string() + "(0x0215043e969dcf616fabe8e8d6b61ddcf6e274c5b04fce957b086dbeb7e899ac63)".to_string(), ]) - .trim() - .to_string(); + .trim() + .to_string(); assert!(output.starts_with("FAIL: secp256k1_verify failed")); } #[test] fn test_secp256k1_verify_classic_succeed() { let program = "(mod (S) (secp256k1_verify S 0x85932e4d075615be881398cc765f9f78204033f0ef5f832ac37e732f5f0cbda2 0x481477e62a1d02268127ae89cc58929e09ad5d30229721965ae35965d098a5f630205a7e69f4cb8084f16c7407ed7312994ffbf87ba5eb1aee16682dd324943e))"; - let compiled = do_basic_run(&vec![ - "run".to_string(), - program.to_string(), - ]); + let compiled = do_basic_run(&vec!["run".to_string(), program.to_string()]); let output = do_basic_brun(&vec![ "brun".to_string(), compiled, - "(0x02390b19842e100324163334b16947f66125b76d4fa4a11b9ccdde9b7398e64076)".to_string() + "(0x02390b19842e100324163334b16947f66125b76d4fa4a11b9ccdde9b7398e64076)".to_string(), ]) - .trim() - .to_string(); - assert_eq!( - output, - "()" - ); + .trim() + .to_string(); + assert_eq!(output, "()"); } #[test] fn test_secp256k1_verify_classic_fail() { let program = "(mod (S) (secp256k1_verify S 0x935d863e2d28d8e5d399ea8af7393ef11fdffc7d862dcc6b5217a8ef15fb5442 0xbbf0712cc0a283a842011c19682629a5381c5f7ead576defcf12a9a19378e23b087cd0be730dbe78722dcfc81543fca17a30e41070ca2e5b3ae77ccec2cca935))"; - let compiled = do_basic_run(&vec![ - "run".to_string(), - program.to_string(), - ]); + let compiled = do_basic_run(&vec!["run".to_string(), program.to_string()]); let output = do_basic_brun(&vec![ "brun".to_string(), compiled, - "(0x0215043e969dcf616fabe8e8d6b61ddcf6e274c5b04fce957b086dbeb7e899ac63)".to_string() + "(0x0215043e969dcf616fabe8e8d6b61ddcf6e274c5b04fce957b086dbeb7e899ac63)".to_string(), ]) - .trim() - .to_string(); + .trim() + .to_string(); assert!(output.starts_with("FAIL: secp256k1_verify failed")); } @@ -1028,110 +998,83 @@ fn test_secp256k1_verify_classic_fail() { fn test_secp256k1_verify_modern_int_succeed() { // Ensure that even if translated to integer (for example via classic unhygenic macro invocation), this works. let program = "(mod (S) (if S (332799744 S 0x85932e4d075615be881398cc765f9f78204033f0ef5f832ac37e732f5f0cbda2 0x481477e62a1d02268127ae89cc58929e09ad5d30229721965ae35965d098a5f630205a7e69f4cb8084f16c7407ed7312994ffbf87ba5eb1aee16682dd324943e) \"empty-secp\"))"; - let compiled = do_basic_run(&vec![ - "run".to_string(), - program.to_string(), - ]); + let compiled = do_basic_run(&vec!["run".to_string(), program.to_string()]); let output = do_basic_brun(&vec![ "brun".to_string(), compiled, - "(0x02390b19842e100324163334b16947f66125b76d4fa4a11b9ccdde9b7398e64076)".to_string() + "(0x02390b19842e100324163334b16947f66125b76d4fa4a11b9ccdde9b7398e64076)".to_string(), ]) - .trim() - .to_string(); - assert_eq!( - output, - "()" - ); + .trim() + .to_string(); + assert_eq!(output, "()"); } #[test] fn test_secp256k1_verify_modern_int_fail() { let program = "(mod (S) (include *standard-cl-21*) (if S (332799744 S 0x935d863e2d28d8e5d399ea8af7393ef11fdffc7d862dcc6b5217a8ef15fb5442 0xbbf0712cc0a283a842011c19682629a5381c5f7ead576defcf12a9a19378e23b087cd0be730dbe78722dcfc81543fca17a30e41070ca2e5b3ae77ccec2cca935) \"empty-secp\"))"; - let compiled = do_basic_run(&vec![ - "run".to_string(), - program.to_string(), - ]); + let compiled = do_basic_run(&vec!["run".to_string(), program.to_string()]); let output = do_basic_brun(&vec![ "brun".to_string(), compiled, - "(0x0215043e969dcf616fabe8e8d6b61ddcf6e274c5b04fce957b086dbeb7e899ac63)".to_string() + "(0x0215043e969dcf616fabe8e8d6b61ddcf6e274c5b04fce957b086dbeb7e899ac63)".to_string(), ]) - .trim() - .to_string(); + .trim() + .to_string(); assert!(output.starts_with("FAIL: secp256k1_verify failed")); } #[test] fn test_secp256r1_verify_modern_succeed() { let program = "(mod (S) (include *standard-cl-21*) (secp256r1_verify S 0x85932e4d075615be881398cc765f9f78204033f0ef5f832ac37e732f5f0cbda2 0xeae2f488080919bd0a7069c24cdd9c6ce2db423861b0c9d4236cdadbd0005f6d8f3709e6eb19249fd9c8bea664aba35218e67ea4b0f2239488dc3147f336e1e6))"; - let compiled = do_basic_run(&vec![ - "run".to_string(), - program.to_string(), - ]); + let compiled = do_basic_run(&vec!["run".to_string(), program.to_string()]); let output = do_basic_brun(&vec![ "brun".to_string(), compiled, - "(0x033e1a1b2ccbc35883c60fdfc3f4a02175096ade6271fe85517ca5772594bbd0dc)".to_string() + "(0x033e1a1b2ccbc35883c60fdfc3f4a02175096ade6271fe85517ca5772594bbd0dc)".to_string(), ]) - .trim() - .to_string(); - assert_eq!( - output, - "()" - ); + .trim() + .to_string(); + assert_eq!(output, "()"); } #[test] fn test_secp256r1_verify_modern_fail() { let program = "(mod (S) (include *standard-cl-21*) (secp256r1_verify S 0x935d863e2d28d8e5d399ea8af7393ef11fdffc7d862dcc6b5217a8ef15fb5442 0xecef274a7408e6cb0196eac64d2ae32fc54c2537f8a9efd5b75a4e8a53b0b156c64564306f38bade4adceac1073d464e4db3d0332141a7203dfd113ad36e393d))"; - let compiled = do_basic_run(&vec![ - "run".to_string(), - program.to_string(), - ]); + let compiled = do_basic_run(&vec!["run".to_string(), program.to_string()]); let output = do_basic_brun(&vec![ "brun".to_string(), compiled, - "(0x025195db74b1902d53758a62ccd9dd01837aa5ae755e878eb0aeccbe8fe477c543)".to_string() + "(0x025195db74b1902d53758a62ccd9dd01837aa5ae755e878eb0aeccbe8fe477c543)".to_string(), ]) - .trim() - .to_string(); + .trim() + .to_string(); assert!(output.starts_with("FAIL: secp256r1_verify failed")); } #[test] fn test_secp256r1_verify_classic_succeed() { let program = "(mod (S) (secp256r1_verify S 0x85932e4d075615be881398cc765f9f78204033f0ef5f832ac37e732f5f0cbda2 0xeae2f488080919bd0a7069c24cdd9c6ce2db423861b0c9d4236cdadbd0005f6d8f3709e6eb19249fd9c8bea664aba35218e67ea4b0f2239488dc3147f336e1e6))"; - let compiled = do_basic_run(&vec![ - "run".to_string(), - program.to_string(), - ]); + let compiled = do_basic_run(&vec!["run".to_string(), program.to_string()]); let output = do_basic_brun(&vec![ "brun".to_string(), compiled, - "(0x033e1a1b2ccbc35883c60fdfc3f4a02175096ade6271fe85517ca5772594bbd0dc)".to_string() + "(0x033e1a1b2ccbc35883c60fdfc3f4a02175096ade6271fe85517ca5772594bbd0dc)".to_string(), ]) - .trim() - .to_string(); - assert_eq!( - output, - "()" - ); + .trim() + .to_string(); + assert_eq!(output, "()"); } #[test] fn test_secp256r1_verify_classic_fail() { let program = "(mod (S) (secp256r1_verify S 0x935d863e2d28d8e5d399ea8af7393ef11fdffc7d862dcc6b5217a8ef15fb5442 0xecef274a7408e6cb0196eac64d2ae32fc54c2537f8a9efd5b75a4e8a53b0b156c64564306f38bade4adceac1073d464e4db3d0332141a7203dfd113ad36e393d))"; - let compiled = do_basic_run(&vec![ - "run".to_string(), - program.to_string(), - ]); + let compiled = do_basic_run(&vec!["run".to_string(), program.to_string()]); let output = do_basic_brun(&vec![ "brun".to_string(), compiled, - "(0x025195db74b1902d53758a62ccd9dd01837aa5ae755e878eb0aeccbe8fe477c543)".to_string() + "(0x025195db74b1902d53758a62ccd9dd01837aa5ae755e878eb0aeccbe8fe477c543)".to_string(), ]) - .trim() - .to_string(); + .trim() + .to_string(); assert!(output.starts_with("FAIL: secp256r1_verify failed")); } diff --git a/src/tests/classic/smoke.rs b/src/tests/classic/smoke.rs index b723bd869..c3d7ad72e 100644 --- a/src/tests/classic/smoke.rs +++ b/src/tests/classic/smoke.rs @@ -810,5 +810,8 @@ fn test_sub_args() { let expr_sexp = assemble(&mut allocator, "(body 2 5)").expect("should assemble"); let new_args = assemble(&mut allocator, "(test1 test2)").expect("should assemble"); let result = sub_args(&mut allocator, expr_sexp, new_args).expect("should run"); - assert_eq!(disassemble(&mut allocator, result, None), "(\"body\" (f (\"test1\" \"test2\")) (f (r (\"test1\" \"test2\"))))"); + assert_eq!( + disassemble(&mut allocator, result, None), + "(\"body\" (f (\"test1\" \"test2\")) (f (r (\"test1\" \"test2\"))))" + ); } From c28a5dabd9512d2798714599a39fa4f9c703a872 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 27 Jun 2023 11:05:27 -0700 Subject: [PATCH 038/117] secp requires a rust upgrade, try specifying 1.65 --- .github/workflows/build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 07c50967f..bf493929b 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -40,7 +40,7 @@ jobs: - name: Set up rust uses: actions-rs/toolchain@v1 with: - toolchain: stable + toolchain: 1.65 - name: Install dependencies run: | From 6c67b72e12b27c68e0c7da6a4325afe09ee20829 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 27 Jun 2023 11:25:23 -0700 Subject: [PATCH 039/117] It appears manylinux2010 is not used anymore and rust requires a newer version of glibc than would have been supported since 1.62. Since the new clvmr dependencies require rust 1.65, we need to bump to manylinux2014 --- .github/workflows/build-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index bf493929b..11f5ff146 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -40,7 +40,7 @@ jobs: - name: Set up rust uses: actions-rs/toolchain@v1 with: - toolchain: 1.65 + toolchain: stable - name: Install dependencies run: | @@ -80,7 +80,7 @@ jobs: . ./activate && \ pip install --upgrade pip ' - docker run --rm -v $(pwd):/io ghcr.io/pyo3/maturin:v0.13.1 build --release --strip --manylinux 2010 + docker run --rm -v $(pwd):/io ghcr.io/pyo3/maturin:v1.1.0 build --release --strip --manylinux 2014 # Refresh in case any ownerships changed. mv target target.docker && cp -r target.docker target # Ensure an empty .cargo-lock file exists. From 35027812f5b4e6f40594f291afe8903832208f8e Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 27 Jun 2023 11:58:45 -0700 Subject: [PATCH 040/117] Pull in wasm-pack update --- .github/workflows/npm-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index d8424af7c..395a02866 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -30,7 +30,7 @@ jobs: components: rustfmt, clippy - name: install wasm-pack - run: cargo install wasm-pack + run: cargo install --version 0.11.1 wasm-pack - name: wasm-pack build and pack run: wasm-pack build --release --target=nodejs wasm && wasm-pack pack wasm From c46a8ed26cad1407f2dc2ae407cdc955a6b09445 Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 28 Jul 2023 13:11:06 -0700 Subject: [PATCH 041/117] use operators_latest_version, improve wording --- src/classic/clvm_tools/cmds.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/classic/clvm_tools/cmds.rs b/src/classic/clvm_tools/cmds.rs index ff6785006..ec285b244 100644 --- a/src/classic/clvm_tools/cmds.rs +++ b/src/classic/clvm_tools/cmds.rs @@ -305,7 +305,7 @@ impl ArgumentValueConv for OperatorsVersion { fn convert(&self, arg: &str) -> Result { let ver = arg .parse::() - .map_err(|_| format!("expected number 0-1 found {arg}"))?; + .map_err(|_| format!("expected number 0-{OPERATORS_LATEST_VERSION} but found {arg}"))?; Ok(ArgumentValue::ArgInt(ver)) } } From cb3ebe34606fafc32703c074855c04f83a8257cb Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 28 Jul 2023 13:21:16 -0700 Subject: [PATCH 042/117] inc_node -> include_keyword_node --- src/classic/clvm_tools/clvmc.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/classic/clvm_tools/clvmc.rs b/src/classic/clvm_tools/clvmc.rs index 8acfbecf4..133021703 100644 --- a/src/classic/clvm_tools/clvmc.rs +++ b/src/classic/clvm_tools/clvmc.rs @@ -34,10 +34,10 @@ fn include_dialect( e: &[NodePtr], ) -> Option { // Propogated names from let capture to labeled nodes. - let inc_node = e[0]; + let include_keyword_node = e[0]; let name_node = e[1]; - if let (SExp::Atom(), SExp::Atom()) = (allocator.sexp(inc_node), allocator.sexp(name_node)) { - if allocator.atom(inc_node) == "include".as_bytes().to_vec() { + if let (SExp::Atom(), SExp::Atom()) = (allocator.sexp(include_keyword_node), allocator.sexp(name_node)) { + if allocator.atom(include_keyword_node) == "include".as_bytes().to_vec() { if let Some(dialect) = dialects.get(allocator.atom(name_node)) { return Some(*dialect); } From a4281fd3edcfe80eac6c7491f77d0b645566862c Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 28 Jul 2023 13:25:00 -0700 Subject: [PATCH 043/117] fmt + clippy --- src/classic/clvm_tools/clvmc.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/classic/clvm_tools/clvmc.rs b/src/classic/clvm_tools/clvmc.rs index 133021703..a5e8cea8a 100644 --- a/src/classic/clvm_tools/clvmc.rs +++ b/src/classic/clvm_tools/clvmc.rs @@ -36,7 +36,10 @@ fn include_dialect( // Propogated names from let capture to labeled nodes. let include_keyword_node = e[0]; let name_node = e[1]; - if let (SExp::Atom(), SExp::Atom()) = (allocator.sexp(include_keyword_node), allocator.sexp(name_node)) { + if let (SExp::Atom(), SExp::Atom()) = ( + allocator.sexp(include_keyword_node), + allocator.sexp(name_node), + ) { if allocator.atom(include_keyword_node) == "include".as_bytes().to_vec() { if let Some(dialect) = dialects.get(allocator.atom(name_node)) { return Some(*dialect); From 552edfa27ae3ba8f0c670104b57a217ad6a25573 Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 28 Jul 2023 14:13:37 -0700 Subject: [PATCH 044/117] Document OPERATORS_LATEST_VERSION --- src/classic/clvm/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/classic/clvm/mod.rs b/src/classic/clvm/mod.rs index 6c40d51d9..0c54c0087 100644 --- a/src/classic/clvm/mod.rs +++ b/src/classic/clvm/mod.rs @@ -8,6 +8,10 @@ pub mod serialize; pub mod sexp; pub mod syntax_error; +/// This specifies the latest "version" of the operator set. This will increase +/// each time a new set of operators is included in the primitive set in clvm. +/// We keep track of what was added when so users can specify what version of the +/// tools' output they're expecting when it matters. pub const OPERATORS_LATEST_VERSION: usize = 1; struct KwAtomPair { From dd7c8856f70ea119fe9ed365fb2a758550c72867 Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 2 Aug 2023 13:22:59 -0700 Subject: [PATCH 045/117] Route in the user's choice for operators extension when compiling code --- src/classic/clvm_tools/cmds.rs | 2 + .../clvm_tools/stages/stage_2/operators.rs | 41 +++++++++++++++++-- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/classic/clvm_tools/cmds.rs b/src/classic/clvm_tools/cmds.rs index ec285b244..1ef74c3b4 100644 --- a/src/classic/clvm_tools/cmds.rs +++ b/src/classic/clvm_tools/cmds.rs @@ -1072,6 +1072,8 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul let special_runner = run_program_for_search_paths(&reported_input_file, &search_paths, extra_symbol_info); + // Ensure we know the user's wishes about the disassembly version here. + special_runner.set_operators_version(get_disassembly_ver(&parsed_args)); let dpr = special_runner.clone(); let run_program = special_runner; diff --git a/src/classic/clvm_tools/stages/stage_2/operators.rs b/src/classic/clvm_tools/stages/stage_2/operators.rs index e1756aead..5a1486c9b 100644 --- a/src/classic/clvm_tools/stages/stage_2/operators.rs +++ b/src/classic/clvm_tools/stages/stage_2/operators.rs @@ -56,6 +56,8 @@ pub struct CompilerOperatorsInternal { // A compiler opts as in the modern compiler. If present, try using its // file system interface to read files. compiler_opts: RefCell>>, + // The version of the operators selected by the user. version 1 includes bls. + operators_version: RefCell>, } /// Given a list of search paths, find a full path to a file whose partial name @@ -132,6 +134,7 @@ impl CompilerOperatorsInternal { runner: RefCell::new(base_runner), opt_memo: RefCell::new(HashMap::new()), compiler_opts: RefCell::new(None), + operators_version: RefCell::new(None), } } @@ -166,6 +169,26 @@ impl CompilerOperatorsInternal { borrow.clone() } + fn get_operators_version(&self) -> Option { + let borrow: Ref<'_, Option> = self.operators_version.borrow(); + borrow.clone() + } + + // Return the extension operator system to use while compiling based on user + // preference. + fn get_operators_extension(&self) -> OperatorSet { + let ops_version = self.get_operators_version().unwrap_or(OPERATORS_LATEST_VERSION); + if ops_version == 0 { + OperatorSet::Default + } else { + OperatorSet::BLS + } + } + + fn set_operators_version(&self, ver: Option) { + self.operators_version.replace(ver); + } + fn read(&self, allocator: &mut Allocator, sexp: NodePtr) -> Response { // Given a string containing the data in the file to parse, parse it or // return EvalErr. @@ -352,8 +375,16 @@ impl Dialect for CompilerOperatorsInternal { op: NodePtr, sexp: NodePtr, max_cost: Cost, - extension: OperatorSet, + _extension: OperatorSet, ) -> Response { + // Ensure we have at least the bls extensions available. + // The extension passed in above is based on the state of whether + // we're approaching from within softfork... As the compiler author + // we're overriding this so the user can specify these in the compile + // context... Even when compiling code to go inside softfork, the + // compiler doesn't itself run in a softfork. + let extensions_to_clvmr_during_compile = self.get_operators_extension(); + match allocator.sexp(op) { SExp::Atom() => { // use of op obvious. @@ -380,12 +411,12 @@ impl Dialect for CompilerOperatorsInternal { self.get_source_file(allocator) } else { self.base_dialect - .op(allocator, op, sexp, max_cost, extension) + .op(allocator, op, sexp, max_cost, extensions_to_clvmr_during_compile) } } _ => self .base_dialect - .op(allocator, op, sexp, max_cost, extension), + .op(allocator, op, sexp, max_cost, extensions_to_clvmr_during_compile), } } @@ -408,6 +439,10 @@ impl CompilerOperators { pub fn set_compiler_opts(&self, opts: Option>) { self.parent.set_compiler_opts(opts); } + + pub fn set_operators_version(&self, ver: Option) { + self.parent.set_operators_version(ver); + } } impl TRunProgram for CompilerOperatorsInternal { From 4f920b75e512472561c97ea3e64b053316ff0c0a Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 2 Aug 2023 13:26:00 -0700 Subject: [PATCH 046/117] fmt + clippy --- .../clvm_tools/stages/stage_2/operators.rs | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/classic/clvm_tools/stages/stage_2/operators.rs b/src/classic/clvm_tools/stages/stage_2/operators.rs index 5a1486c9b..660dcfe13 100644 --- a/src/classic/clvm_tools/stages/stage_2/operators.rs +++ b/src/classic/clvm_tools/stages/stage_2/operators.rs @@ -171,13 +171,15 @@ impl CompilerOperatorsInternal { fn get_operators_version(&self) -> Option { let borrow: Ref<'_, Option> = self.operators_version.borrow(); - borrow.clone() + *borrow } // Return the extension operator system to use while compiling based on user // preference. fn get_operators_extension(&self) -> OperatorSet { - let ops_version = self.get_operators_version().unwrap_or(OPERATORS_LATEST_VERSION); + let ops_version = self + .get_operators_version() + .unwrap_or(OPERATORS_LATEST_VERSION); if ops_version == 0 { OperatorSet::Default } else { @@ -410,13 +412,22 @@ impl Dialect for CompilerOperatorsInternal { } else if opbuf == "_get_source_file".as_bytes() { self.get_source_file(allocator) } else { - self.base_dialect - .op(allocator, op, sexp, max_cost, extensions_to_clvmr_during_compile) + self.base_dialect.op( + allocator, + op, + sexp, + max_cost, + extensions_to_clvmr_during_compile, + ) } } - _ => self - .base_dialect - .op(allocator, op, sexp, max_cost, extensions_to_clvmr_during_compile), + _ => self.base_dialect.op( + allocator, + op, + sexp, + max_cost, + extensions_to_clvmr_during_compile, + ), } } From 99b4ead93dc187b6bc1772577b3386a356e9267e Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 4 Aug 2023 09:29:37 -0700 Subject: [PATCH 047/117] Add tests that exercise operator choice in classic run --- src/tests/classic/run.rs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index c4896b909..f44876c62 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -1199,3 +1199,34 @@ fn test_secp256r1_verify_classic_fail() { .to_string(); assert!(output.starts_with("FAIL: secp256r1_verify failed")); } + +#[test] +fn test_classic_obeys_operator_choice_at_compile_time_no_version() { + let compiled = do_basic_run(&vec![ + "run".to_string(), + "(mod () (coinid (sha256 99) (sha256 99) 1))".to_string() + ]).trim().to_string(); + assert_eq!(compiled, "(q . 0x97c3f14ced4dfc280611fd8d9b158163e8981b3bce4d1bb6dd0bcc679a2e2455)"); +} + +#[test] +fn test_classic_obeys_operator_choice_at_compile_time_version_1() { + let compiled = do_basic_run(&vec![ + "run".to_string(), + "--operators-version".to_string(), + "1".to_string(), + "(mod () (coinid (sha256 99) (sha256 99) 1))".to_string() + ]).trim().to_string(); + assert_eq!(compiled, "(q . 0x97c3f14ced4dfc280611fd8d9b158163e8981b3bce4d1bb6dd0bcc679a2e2455)"); +} + +#[test] +fn test_classic_obeys_operator_choice_at_compile_time_version_0() { + let compiled = do_basic_run(&vec![ + "run".to_string(), + "--operators-version".to_string(), + "0".to_string(), + "(mod () (coinid (sha256 99) (sha256 99) 1))".to_string() + ]).trim().to_string(); + assert_eq!(compiled, "FAIL: unimplemented operator 48"); +} From 5b693106a722bcbe620e0beb16b189ca5accf6db Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 4 Aug 2023 09:31:26 -0700 Subject: [PATCH 048/117] fmt --- src/tests/classic/run.rs | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index f44876c62..02c4974cd 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -1204,9 +1204,14 @@ fn test_secp256r1_verify_classic_fail() { fn test_classic_obeys_operator_choice_at_compile_time_no_version() { let compiled = do_basic_run(&vec![ "run".to_string(), - "(mod () (coinid (sha256 99) (sha256 99) 1))".to_string() - ]).trim().to_string(); - assert_eq!(compiled, "(q . 0x97c3f14ced4dfc280611fd8d9b158163e8981b3bce4d1bb6dd0bcc679a2e2455)"); + "(mod () (coinid (sha256 99) (sha256 99) 1))".to_string(), + ]) + .trim() + .to_string(); + assert_eq!( + compiled, + "(q . 0x97c3f14ced4dfc280611fd8d9b158163e8981b3bce4d1bb6dd0bcc679a2e2455)" + ); } #[test] @@ -1215,9 +1220,14 @@ fn test_classic_obeys_operator_choice_at_compile_time_version_1() { "run".to_string(), "--operators-version".to_string(), "1".to_string(), - "(mod () (coinid (sha256 99) (sha256 99) 1))".to_string() - ]).trim().to_string(); - assert_eq!(compiled, "(q . 0x97c3f14ced4dfc280611fd8d9b158163e8981b3bce4d1bb6dd0bcc679a2e2455)"); + "(mod () (coinid (sha256 99) (sha256 99) 1))".to_string(), + ]) + .trim() + .to_string(); + assert_eq!( + compiled, + "(q . 0x97c3f14ced4dfc280611fd8d9b158163e8981b3bce4d1bb6dd0bcc679a2e2455)" + ); } #[test] @@ -1226,7 +1236,9 @@ fn test_classic_obeys_operator_choice_at_compile_time_version_0() { "run".to_string(), "--operators-version".to_string(), "0".to_string(), - "(mod () (coinid (sha256 99) (sha256 99) 1))".to_string() - ]).trim().to_string(); + "(mod () (coinid (sha256 99) (sha256 99) 1))".to_string(), + ]) + .trim() + .to_string(); assert_eq!(compiled, "FAIL: unimplemented operator 48"); } From e48eb6e8f0aaa59c4c074eedef959c3b5cf88346 Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 4 Aug 2023 10:36:19 -0700 Subject: [PATCH 049/117] Recognize a list as proper when it has a weird but not non-nil tail --- src/compiler/clvm.rs | 19 +++++++++++++++---- src/tests/compiler/cldb.rs | 26 ++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/src/compiler/clvm.rs b/src/compiler/clvm.rs index dd4ad017c..4658e2d08 100644 --- a/src/compiler/clvm.rs +++ b/src/compiler/clvm.rs @@ -192,10 +192,21 @@ fn eval_args( sexp = b.clone(); } _ => { - return Err(RunFailure::RunErr( - sexp.loc(), - format!("bad argument list {sexp_} {context_}"), - )); + // A list of the following forms: + // (x y . 0) + // (x y . "") + // Are properly terminated lists and disassemble to (x y). + // + // This recognizes that our broader value space has more ways + // of expressing nil. + if !truthy(sexp.clone()) { + return Ok(RunStep::Op(head, context_, sexp, Some(eval_list), parent)); + } else { + return Err(RunFailure::RunErr( + sexp.loc(), + format!("bad argument list {sexp_} {context_}"), + )); + } } } } diff --git a/src/tests/compiler/cldb.rs b/src/tests/compiler/cldb.rs index 90c3da3ec..48ab13ad3 100644 --- a/src/tests/compiler/cldb.rs +++ b/src/tests/compiler/cldb.rs @@ -369,3 +369,29 @@ fn test_cldb_explicit_throw() { assert!(watcher.correct_result()); } + +#[test] +fn test_clvm_operator_with_weird_tail() { + let filename = "test-weird-tail.clvm"; + let loc = Srcloc::start(filename); + let program = "(+ (q . 3) (q . 5) . \"\")"; + let parsed = parse_sexp( + loc.clone(), + program.as_bytes().iter().copied(), + ) + .expect("should parse"); + let args = Rc::new(SExp::Nil(loc)); + let program_lines = Rc::new(vec![program.to_string()]); + + assert_eq!( + run_clvm_in_cldb( + filename, + program_lines, + parsed[0].clone(), + HashMap::new(), + args, + &mut DoesntWatchCldb {}, + ), + Some("8".to_string()) + ); +} From 29922239cc8db87b1fd800c1a1a6d1d3cdb67b73 Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 4 Aug 2023 10:44:35 -0700 Subject: [PATCH 050/117] fmt --- src/tests/compiler/cldb.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/tests/compiler/cldb.rs b/src/tests/compiler/cldb.rs index 48ab13ad3..47da4a32a 100644 --- a/src/tests/compiler/cldb.rs +++ b/src/tests/compiler/cldb.rs @@ -375,11 +375,7 @@ fn test_clvm_operator_with_weird_tail() { let filename = "test-weird-tail.clvm"; let loc = Srcloc::start(filename); let program = "(+ (q . 3) (q . 5) . \"\")"; - let parsed = parse_sexp( - loc.clone(), - program.as_bytes().iter().copied(), - ) - .expect("should parse"); + let parsed = parse_sexp(loc.clone(), program.as_bytes().iter().copied()).expect("should parse"); let args = Rc::new(SExp::Nil(loc)); let program_lines = Rc::new(vec![program.to_string()]); From 65fc3fbdb6eb036758ef7b153a1dac9543015142 Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 4 Aug 2023 11:11:53 -0700 Subject: [PATCH 051/117] Reorganize to be a bit nicer --- src/compiler/clvm.rs | 42 +++++++++++++++++------------------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/src/compiler/clvm.rs b/src/compiler/clvm.rs index 4658e2d08..f372f5019 100644 --- a/src/compiler/clvm.rs +++ b/src/compiler/clvm.rs @@ -183,31 +183,23 @@ fn eval_args( let mut eval_list: Vec> = Vec::new(); loop { - match sexp.borrow() { - SExp::Nil(_l) => { - return Ok(RunStep::Op(head, context_, sexp, Some(eval_list), parent)); - } - SExp::Cons(_l, a, b) => { - eval_list.push(a.clone()); - sexp = b.clone(); - } - _ => { - // A list of the following forms: - // (x y . 0) - // (x y . "") - // Are properly terminated lists and disassemble to (x y). - // - // This recognizes that our broader value space has more ways - // of expressing nil. - if !truthy(sexp.clone()) { - return Ok(RunStep::Op(head, context_, sexp, Some(eval_list), parent)); - } else { - return Err(RunFailure::RunErr( - sexp.loc(), - format!("bad argument list {sexp_} {context_}"), - )); - } - } + // A list of the following forms: + // (x y . 0) + // (x y . "") + // Are properly terminated lists and disassemble to (x y). + // + // This recognizes that our broader value space has more ways + // of expressing nil. + if let SExp::Cons(_l, a, b) = sexp.borrow() { + eval_list.push(a.clone()); + sexp = b.clone(); + } else if !truthy(sexp.clone()) { + return Ok(RunStep::Op(head, context_, sexp, Some(eval_list), parent)); + } else { + return Err(RunFailure::RunErr( + sexp.loc(), + format!("bad argument list {sexp_} {context_}"), + )); } } } From d383eb10249b9296dafd5652408856982b835a89 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Mon, 7 Aug 2023 14:42:58 +0100 Subject: [PATCH 052/117] add comments --- src/compiler/sexp.rs | 156 ++++++++++++++++++++++++------------------- 1 file changed, 87 insertions(+), 69 deletions(-) diff --git a/src/compiler/sexp.rs b/src/compiler/sexp.rs index 599eea3fe..0811d1026 100644 --- a/src/compiler/sexp.rs +++ b/src/compiler/sexp.rs @@ -203,9 +203,9 @@ enum TermListCommentState { } #[derive(Debug)] -enum SExpParseState { +enum SExpParseState { // The types of state that the Rust pre-forms can take Empty, - CommentText(Srcloc, Vec), + CommentText(Srcloc, Vec), //srcloc contains the file, line, column and length for the captured form Bareword(Srcloc, Vec), QuotedText(Srcloc, u8, Vec), QuotedEscaped(Srcloc, u8, Vec), @@ -221,7 +221,7 @@ enum SExpParseState { } #[derive(Debug)] -enum SExpParseResult { +enum SExpParseResult { // the result of a call to parse an SExp Resume(SExpParseState), Emit(Rc, SExpParseState), Error(Srcloc, String), @@ -318,16 +318,18 @@ pub fn enlist(l: Srcloc, v: Vec>) -> SExp { result } -fn emit(a: Rc, p: SExpParseState) -> SExpParseResult { - SExpParseResult::Emit(a, p) +// this function takes a ParseState and returns an Emit ParseResult which contains the ParseState +fn emit(a: Rc, current_state: SExpParseState) -> SExpParseResult { + SExpParseResult::Emit(a, current_state) } fn error(l: Srcloc, t: &str) -> SExpParseResult { SExpParseResult::Error(l, t.to_string()) } -fn resume(p: SExpParseState) -> SExpParseResult { - SExpParseResult::Resume(p) +// this function takes a ParseState and returns a Resume ParseResult which contains the ParseState +fn resume(current_state: SExpParseState) -> SExpParseResult { + SExpParseResult::Resume(current_state) } fn escape_quote(q: u8, s: &[u8]) -> String { @@ -538,127 +540,132 @@ impl SExp { } } -fn parse_sexp_step(loc: Srcloc, p: &SExpParseState, this_char: u8) -> SExpParseResult { - match p { - SExpParseState::Empty => match this_char as char { - '(' => resume(SExpParseState::OpenList(loc)), - '\n' => resume(SExpParseState::Empty), +fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) -> SExpParseResult { + // switch on our state + match current_state { + SExpParseState::Empty => match this_char as char { // we are not currently in a list + '(' => resume(SExpParseState::OpenList(loc)), // move to OpenList state + '\n' => resume(SExpParseState::Empty), // new line, same state ';' => resume(SExpParseState::CommentText(loc, Vec::new())), ')' => error(loc, "Too many close parens"), - '"' => resume(SExpParseState::QuotedText(loc, b'"', Vec::new())), - '\'' => resume(SExpParseState::QuotedText(loc, b'\'', Vec::new())), + '"' => resume(SExpParseState::QuotedText(loc, b'"', Vec::new())), // match on " + '\'' => resume(SExpParseState::QuotedText(loc, b'\'', Vec::new())), // match on ' ch => { if char::is_whitespace(ch) { resume(SExpParseState::Empty) } else { - resume(SExpParseState::Bareword(loc, vec![this_char])) + resume(SExpParseState::Bareword(loc, vec![this_char])) // start of a word - could be an atom or a keyword - the compiler will decide } } }, - SExpParseState::CommentText(pl, t) => match this_char as char { - '\r' => resume(SExpParseState::CommentText(pl.clone(), t.to_vec())), + // t is a Vec of the previous characters in this comment string + SExpParseState::CommentText(srcloc, t) => match this_char as char { + '\r' => resume(SExpParseState::CommentText(srcloc.clone(), t.to_vec())), '\n' => resume(SExpParseState::Empty), _ => { let mut tcopy = t.to_vec(); tcopy.push(this_char); - resume(SExpParseState::CommentText(pl.ext(&loc), tcopy)) + resume(SExpParseState::CommentText(srcloc.ext(&loc), tcopy)) } }, - SExpParseState::Bareword(pl, a) => { - if char::is_whitespace(this_char as char) { + // we currently processing a new word + SExpParseState::Bareword(srcloc, word_so_far) => { + if char::is_whitespace(this_char as char) { // we've found a space, so it's the end of a word emit( - Rc::new(make_atom(pl.clone(), a.to_vec())), + Rc::new(make_atom(srcloc.clone(), word_so_far.to_vec())), SExpParseState::Empty, ) - } else { - let mut acopy = a.to_vec(); - acopy.push(this_char); - resume(SExpParseState::Bareword(pl.ext(&loc), acopy)) + } else { // otherwise add letter to word + let mut word_copy = word_so_far.to_vec(); + word_copy.push(this_char); + resume(SExpParseState::Bareword(srcloc.ext(&loc), word_copy)) } } - SExpParseState::QuotedText(pl, term, t) => { - if this_char == b'\\' { - resume(SExpParseState::QuotedEscaped(pl.clone(), *term, t.to_vec())) - } else if this_char == *term { + SExpParseState::QuotedText(srcloc, term, t) => { + if this_char == b'\\' { // if we have a character escape then copy the character directly + resume(SExpParseState::QuotedEscaped(srcloc.clone(), *term, t.to_vec())) + } else if this_char == *term { // otherwise check if it's the terminating character (either ' or ") emit( - Rc::new(SExp::QuotedString(pl.ext(&loc), *term, t.to_vec())), + Rc::new(SExp::QuotedString(srcloc.ext(&loc), *term, t.to_vec())), // add quoted string to parent list SExpParseState::Empty, ) - } else { + } else { // otherwise copy the character let mut tcopy = t.to_vec(); tcopy.push(this_char); - resume(SExpParseState::QuotedText(pl.clone(), *term, tcopy)) + resume(SExpParseState::QuotedText(srcloc.clone(), *term, tcopy)) } } - SExpParseState::QuotedEscaped(pl, term, t) => { + // copy the character the quoted text because we have put the escape character first + SExpParseState::QuotedEscaped(srcloc, term, t) => { let mut tcopy = t.to_vec(); tcopy.push(this_char); - resume(SExpParseState::QuotedText(pl.clone(), *term, tcopy)) + resume(SExpParseState::QuotedText(srcloc.clone(), *term, tcopy)) } - SExpParseState::OpenList(pl) => match this_char as char { - ')' => emit(Rc::new(SExp::Nil(pl.ext(&loc))), SExpParseState::Empty), + SExpParseState::OpenList(srcloc) => match this_char as char { // we are beginning a new list + ')' => emit(Rc::new(SExp::Nil(srcloc.ext(&loc))), SExpParseState::Empty), // create a Nil object '.' => error(loc, "Dot can't appear directly after begin paren"), - _ => match parse_sexp_step(loc.clone(), &SExpParseState::Empty, this_char) { - SExpParseResult::Emit(o, p) => resume(SExpParseState::ParsingList( - pl.ext(&loc), - Rc::new(p), + _ => match parse_sexp_step(loc.clone(), &SExpParseState::Empty, this_char) { // fetch result of parsing as if we were in empty state + SExpParseResult::Emit(o, current_state) => resume(SExpParseState::ParsingList( // we found an object, resume processing + srcloc.ext(&loc), + Rc::new(current_state), // captured state from our pretend empty state vec![o], )), - SExpParseResult::Resume(p) => resume(SExpParseState::ParsingList( - pl.ext(&loc), - Rc::new(p), + SExpParseResult::Resume(current_state) => resume(SExpParseState::ParsingList( // we're still reading the object, resume processing + srcloc.ext(&loc), + Rc::new(current_state), // captured state from our pretend empty state Vec::new(), )), - SExpParseResult::Error(l, e) => SExpParseResult::Error(l, e), + SExpParseResult::Error(l, e) => SExpParseResult::Error(l, e), // propagate error }, }, - SExpParseState::ParsingList(pl, pp, list_content) => { + // We are in the middle of a list currently + SExpParseState::ParsingList(srcloc, pp, list_content) => { // pp is the captured inside-list state we received from OpenList match (this_char as char, pp.borrow()) { - ('.', SExpParseState::Empty) => resume(SExpParseState::TermList( - pl.ext(&loc), + ('.', SExpParseState::Empty) => resume(SExpParseState::TermList( // dot notation showing cons cell + srcloc.ext(&loc), TermListCommentState::Empty, None, Rc::new(SExpParseState::Empty), list_content.to_vec(), )), - (')', SExpParseState::Empty) => emit( - Rc::new(enlist(pl.ext(&loc), list_content.to_vec())), + (')', SExpParseState::Empty) => emit( // close list and emit it upwards as a complete entity + Rc::new(enlist(srcloc.ext(&loc), list_content.to_vec())), SExpParseState::Empty, ), - (')', SExpParseState::Bareword(l, t)) => { + (')', SExpParseState::Bareword(l, t)) => { // you've reached the end of the word AND the end of the list, close list and emit upwards let parsed_atom = make_atom(l.clone(), t.to_vec()); let mut updated_list = list_content.to_vec(); updated_list.push(Rc::new(parsed_atom)); emit( - Rc::new(enlist(pl.ext(&loc), updated_list)), + Rc::new(enlist(srcloc.ext(&loc), updated_list)), SExpParseState::Empty, ) } - (_, _) => match parse_sexp_step(loc.clone(), pp.borrow(), this_char) { - SExpParseResult::Emit(o, p) => { + (_, _) => match parse_sexp_step(loc.clone(), pp.borrow(), this_char) { // + SExpParseResult::Emit(o, current_state) => { // add result of nested call to our list let mut list_copy = list_content.clone(); list_copy.push(o); let result = - SExpParseState::ParsingList(pl.ext(&loc), Rc::new(p), list_copy); + SExpParseState::ParsingList(srcloc.ext(&loc), Rc::new(current_state), list_copy); resume(result) } - SExpParseResult::Resume(rp) => resume(SExpParseState::ParsingList( - pl.ext(&loc), + SExpParseResult::Resume(rp) => resume(SExpParseState::ParsingList( // + srcloc.ext(&loc), Rc::new(rp), list_content.to_vec(), )), - SExpParseResult::Error(l, e) => SExpParseResult::Error(l, e), + SExpParseResult::Error(l, e) => SExpParseResult::Error(l, e), // propagate error upwards }, } } - SExpParseState::TermList(pl, TermListCommentState::InComment, parsed, pp, list_content) => { + SExpParseState::TermList(srcloc, TermListCommentState::InComment, parsed, pp, list_content) => { // pp is the captured inside-list state we received from OpenList let end_comment = if this_char as char == '\n' || this_char as char == '\r' { TermListCommentState::Empty } else { TermListCommentState::InComment }; resume(SExpParseState::TermList( - pl.clone(), + srcloc.clone(), end_comment, parsed.clone(), pp.clone(), @@ -666,7 +673,7 @@ fn parse_sexp_step(loc: Srcloc, p: &SExpParseState, this_char: u8) -> SExpParseR )) } SExpParseState::TermList( - pl, + srcloc, TermListCommentState::Empty, Some(parsed), pp, @@ -674,7 +681,7 @@ fn parse_sexp_step(loc: Srcloc, p: &SExpParseState, this_char: u8) -> SExpParseR ) => { if this_char.is_ascii_whitespace() { resume(SExpParseState::TermList( - pl.ext(&loc), + srcloc.ext(&loc), TermListCommentState::Empty, Some(parsed.clone()), pp.clone(), @@ -699,7 +706,7 @@ fn parse_sexp_step(loc: Srcloc, p: &SExpParseState, this_char: u8) -> SExpParseR } } else if this_char == b';' { resume(SExpParseState::TermList( - pl.clone(), + srcloc.clone(), TermListCommentState::InComment, Some(parsed.clone()), pp.clone(), @@ -707,12 +714,12 @@ fn parse_sexp_step(loc: Srcloc, p: &SExpParseState, this_char: u8) -> SExpParseR )) } else { error( - pl.clone(), + srcloc.clone(), &format!("unexpected character {}", this_char as char), ) } } - SExpParseState::TermList(pl, TermListCommentState::Empty, None, pp, list_content) => { + SExpParseState::TermList(srcloc, TermListCommentState::Empty, None, pp, list_content) => { match (this_char as char, pp.borrow()) { ('.', SExpParseState::Empty) => { error(loc, "Multiple dots in list notation are illegal") @@ -722,7 +729,7 @@ fn parse_sexp_step(loc: Srcloc, p: &SExpParseState, this_char: u8) -> SExpParseR emit(list_content[0].clone(), SExpParseState::Empty) } else { emit( - Rc::new(enlist(pl.ext(&loc), list_content.to_vec())), + Rc::new(enlist(srcloc.ext(&loc), list_content.to_vec())), SExpParseState::Empty, ) } @@ -747,18 +754,18 @@ fn parse_sexp_step(loc: Srcloc, p: &SExpParseState, this_char: u8) -> SExpParseR } } (_, _) => match parse_sexp_step(loc.clone(), pp.borrow(), this_char) { - SExpParseResult::Emit(o, _p) => resume(SExpParseState::TermList( + SExpParseResult::Emit(o, _current_state) => resume(SExpParseState::TermList( loc, TermListCommentState::Empty, Some(o), pp.clone(), list_content.clone(), )), - SExpParseResult::Resume(p) => resume(SExpParseState::TermList( - pl.ext(&loc), + SExpParseResult::Resume(current_state) => resume(SExpParseState::TermList( + srcloc.ext(&loc), TermListCommentState::Empty, None, - Rc::new(p), + Rc::new(current_state), list_content.to_vec(), )), SExpParseResult::Error(l, e) => SExpParseResult::Error(l, e), @@ -776,19 +783,28 @@ fn parse_sexp_inner( where I: Iterator, { + // we support compiling multiple things at once, keep these in a Vec + // at the moment this will almost certainly only return 1 thing let mut res = Vec::new(); + // Loop through all the characters for this_char in s { + let next_location = start.clone().advance(this_char); + // call parse_sexp_step for current character + // it will return a ParseResult which contains the new ParseState match parse_sexp_step(start.clone(), parse_state.borrow(), this_char) { + // catch error and propagate it upwards SExpParseResult::Error(l, e) => { return Err((l, e)); } + // Keep parsing SExpParseResult::Resume(new_parse_state) => { start = next_location; parse_state = new_parse_state; } + // End of list (top level compile object), but not necessarily end of file SExpParseResult::Emit(o, new_parse_state) => { start = next_location; parse_state = new_parse_state; @@ -797,6 +813,7 @@ where } } + // depending on the state when we finished return Ok or Err enums match parse_state { SExpParseState::Empty => Ok(res), SExpParseState::Bareword(l, t) => Ok(vec![Rc::new(make_atom(l, t))]), @@ -813,6 +830,7 @@ where /// /// Entrypoint for parsing chialisp input. +/// Called from compiler.rs /// /// This produces Rc, where SExp is described above. /// From 8c2eaea8245e7e8fdcf5e79f4cf51fb55edac452 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Mon, 7 Aug 2023 15:36:03 +0100 Subject: [PATCH 053/117] fix merge --- src/compiler/sexp.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/compiler/sexp.rs b/src/compiler/sexp.rs index 9cb96ad23..5b5c4fd6c 100644 --- a/src/compiler/sexp.rs +++ b/src/compiler/sexp.rs @@ -636,6 +636,7 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - updated_list.push(Rc::new(parsed_atom)); emit( Rc::new(enlist(srcloc.clone(), &updated_list)), + SExpParseState::Empty, ) } (_, _) => match parse_sexp_step(loc.clone(), pp.borrow(), this_char) { // From 1a0ce403f59d027354957913642245d4e487ac1e Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 7 Aug 2023 10:05:49 -0700 Subject: [PATCH 054/117] Default::default -> AcceptedDialect::default for clarity --- src/classic/clvm_tools/clvmc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/classic/clvm_tools/clvmc.rs b/src/classic/clvm_tools/clvmc.rs index e4ffc688b..963828b39 100644 --- a/src/classic/clvm_tools/clvmc.rs +++ b/src/classic/clvm_tools/clvmc.rs @@ -73,7 +73,7 @@ pub fn detect_modern(allocator: &mut Allocator, sexp: NodePtr) -> AcceptedDialec dialects.insert("*standard-cl-22*".as_bytes().to_vec(), 22); // Start with an empty definition of the dialect (classic). - let mut result = Default::default(); + let mut result = AcceptedDialect::default(); // For each form in the source file, try to find a sigil at the top level of // the list it forms to find a sigil. From 058d010e5061b0e6402221a793dae3a9477928fc Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 7 Aug 2023 10:07:18 -0700 Subject: [PATCH 055/117] opps --- src/classic/clvm_tools/clvmc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/classic/clvm_tools/clvmc.rs b/src/classic/clvm_tools/clvmc.rs index 963828b39..3c20864aa 100644 --- a/src/classic/clvm_tools/clvmc.rs +++ b/src/classic/clvm_tools/clvmc.rs @@ -97,7 +97,7 @@ pub fn detect_modern(allocator: &mut Allocator, sexp: NodePtr) -> AcceptedDialec } if let Some(dialect) = include_dialect(allocator, &dialects, &e) { - // We found a sigl. + // We found a sigil. result.stepping = Some(dialect); break; } From 3909322cc756deddcd00a74228a1e240dd55afb9 Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 7 Aug 2023 10:17:36 -0700 Subject: [PATCH 056/117] Default::default -> AcceptedDialect::default --- src/compiler/compiler.rs | 2 +- src/tests/classic/stage_2.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index 3160cd505..3232a7ad3 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -367,7 +367,7 @@ impl DefaultCompilerOpts { frontend_opt: false, frontend_check_live: true, start_env: None, - dialect: Default::default(), + dialect: AcceptedDialect::default(), prim_map: create_prim_map(), disassembly_ver: None, known_dialects: Rc::new(KNOWN_DIALECTS.clone()), diff --git a/src/tests/classic/stage_2.rs b/src/tests/classic/stage_2.rs index e6dac793f..3ad0b61b0 100644 --- a/src/tests/classic/stage_2.rs +++ b/src/tests/classic/stage_2.rs @@ -320,7 +320,7 @@ impl CompilerOpts for TestCompilerOptsPresentsOwnFiles { None } fn dialect(&self) -> AcceptedDialect { - Default::default() + AcceptedDialect::default() } fn in_defun(&self) -> bool { false From a81957ee95df74523799b7d4e679e38bd78d8d41 Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 7 Aug 2023 17:26:13 -0700 Subject: [PATCH 057/117] Move dialect stuff to src/compiler/dialect.rs --- src/classic/clvm_tools/clvmc.rs | 78 +--------------------- src/classic/clvm_tools/cmds.rs | 3 +- src/compiler/compiler.rs | 3 +- src/compiler/comptypes.rs | 15 +---- src/compiler/dialect.rs | 113 ++++++++++++++++++++++++++++++++ src/compiler/mod.rs | 3 +- src/tests/classic/stage_2.rs | 3 +- 7 files changed, 125 insertions(+), 93 deletions(-) create mode 100644 src/compiler/dialect.rs diff --git a/src/classic/clvm_tools/clvmc.rs b/src/classic/clvm_tools/clvmc.rs index 3c20864aa..4a1beb076 100644 --- a/src/classic/clvm_tools/clvmc.rs +++ b/src/classic/clvm_tools/clvmc.rs @@ -6,12 +6,11 @@ use std::rc::Rc; use tempfile::NamedTempFile; -use clvm_rs::allocator::{Allocator, NodePtr, SExp}; +use clvm_rs::allocator::{Allocator, NodePtr}; use clvm_rs::reduction::EvalErr; use crate::classic::clvm::__type_compatibility__::Stream; use crate::classic::clvm::serialize::sexp_to_stream; -use crate::classic::clvm::sexp::proper_list; use crate::classic::clvm_tools::binutils::{assemble_from_ir, disassemble}; use crate::classic::clvm_tools::ir::reader::read_ir; use crate::classic::clvm_tools::stages::run; @@ -23,31 +22,10 @@ use crate::classic::platform::distutils::dep_util::newer; use crate::compiler::clvm::convert_to_clvm_rs; use crate::compiler::compiler::compile_file; use crate::compiler::compiler::{run_optimizer, DefaultCompilerOpts}; -use crate::compiler::comptypes::{AcceptedDialect, CompileErr, CompilerOpts}; +use crate::compiler::comptypes::{CompileErr, CompilerOpts}; +use crate::compiler::dialect::detect_modern; use crate::compiler::runtypes::RunFailure; -fn include_dialect( - allocator: &Allocator, - dialects: &HashMap, i32>, - e: &[NodePtr], -) -> Option { - // Propogated names from let capture to labeled nodes. - let include_keyword_node = e[0]; - let name_node = e[1]; - if let (SExp::Atom(), SExp::Atom()) = ( - allocator.sexp(include_keyword_node), - allocator.sexp(name_node), - ) { - if allocator.atom(include_keyword_node) == "include".as_bytes().to_vec() { - if let Some(dialect) = dialects.get(allocator.atom(name_node)) { - return Some(*dialect); - } - } - } - - None -} - pub fn write_sym_output( compiled_lookup: &HashMap, path: &str, @@ -60,56 +38,6 @@ pub fn write_sym_output( .map(|_| ()) } -// Now return more parameters about the "modern" dialect, including in the future, -// strictness. This will allow us to support the transition to modern macros which -// in turn allow us to turn on strictness in variable naming. Often multiple moves -// are needed to get from one point to another and there's a tension between -// unitary changes and smaller PRs which do fewer things by themselves. This is -// part of a broader narrative, which many requested that sets us on the path of -// being able to include more information in the dialect result. -pub fn detect_modern(allocator: &mut Allocator, sexp: NodePtr) -> AcceptedDialect { - let mut dialects = HashMap::new(); - dialects.insert("*standard-cl-21*".as_bytes().to_vec(), 21); - dialects.insert("*standard-cl-22*".as_bytes().to_vec(), 22); - - // Start with an empty definition of the dialect (classic). - let mut result = AcceptedDialect::default(); - - // For each form in the source file, try to find a sigil at the top level of - // the list it forms to find a sigil. - if let Some(l) = proper_list(allocator, sexp, true) { - for elt in l.iter() { - let detect_modern_result = detect_modern(allocator, *elt); - if detect_modern_result.stepping.is_some() { - // We found a dialect directive. - result = detect_modern_result; - break; - } - - match proper_list(allocator, *elt, true) { - None => { - continue; - } - - Some(e) => { - if e.len() != 2 { - continue; - } - - if let Some(dialect) = include_dialect(allocator, &dialects, &e) { - // We found a sigil. - result.stepping = Some(dialect); - break; - } - } - } - } - } - - // Return whatever we found or the default. - result -} - pub fn compile_clvm_text( allocator: &mut Allocator, opts: Rc, diff --git a/src/classic/clvm_tools/cmds.rs b/src/classic/clvm_tools/cmds.rs index d97ae8c6d..52d953d40 100644 --- a/src/classic/clvm_tools/cmds.rs +++ b/src/classic/clvm_tools/cmds.rs @@ -28,7 +28,7 @@ use crate::classic::clvm::serialize::{sexp_from_stream, sexp_to_stream, SimpleCr use crate::classic::clvm::sexp::{enlist, proper_list, sexp_as_bin}; use crate::classic::clvm::OPERATORS_LATEST_VERSION; use crate::classic::clvm_tools::binutils::{assemble_from_ir, disassemble, disassemble_with_kw}; -use crate::classic::clvm_tools::clvmc::{detect_modern, write_sym_output}; +use crate::classic::clvm_tools::clvmc::write_sym_output; use crate::classic::clvm_tools::debug::check_unused; use crate::classic::clvm_tools::debug::{ program_hash_from_program_env_cons, start_log_after, trace_pre_eval, trace_to_table, @@ -42,6 +42,7 @@ use crate::classic::clvm_tools::stages::stage_0::{ }; use crate::classic::clvm_tools::stages::stage_2::operators::run_program_for_search_paths; use crate::classic::platform::PathJoin; +use crate::compiler::dialect::detect_modern; use crate::classic::platform::argparse::{ Argument, ArgumentParser, ArgumentValue, ArgumentValueConv, IntConversion, NArgsSpec, diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index 3232a7ad3..91f564f91 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -14,8 +14,9 @@ use crate::classic::clvm_tools::stages::stage_2::optimize::optimize_sexp; use crate::compiler::clvm::{convert_from_clvm_rs, convert_to_clvm_rs, sha256tree}; use crate::compiler::codegen::{codegen, hoist_body_let_binding, process_helper_let_bindings}; use crate::compiler::comptypes::{ - AcceptedDialect, CompileErr, CompileForm, CompilerOpts, DefunData, HelperForm, PrimaryCodegen, + CompileErr, CompileForm, CompilerOpts, DefunData, HelperForm, PrimaryCodegen, }; +use crate::compiler::dialect::AcceptedDialect; use crate::compiler::evaluate::{build_reflex_captures, Evaluator, EVAL_STACK_LIMIT}; use crate::compiler::frontend::frontend; use crate::compiler::prims; diff --git a/src/compiler/comptypes.rs b/src/compiler/comptypes.rs index b02013598..f38ff8321 100644 --- a/src/compiler/comptypes.rs +++ b/src/compiler/comptypes.rs @@ -10,6 +10,7 @@ use crate::classic::clvm::__type_compatibility__::{Bytes, BytesFromType}; use crate::classic::clvm_tools::stages::stage_0::TRunProgram; use crate::compiler::clvm::sha256tree; +use crate::compiler::dialect::AcceptedDialect; use crate::compiler::sexp::{decode_string, SExp}; use crate::compiler::srcloc::Srcloc; @@ -30,20 +31,6 @@ impl From<(Srcloc, String)> for CompileErr { #[derive(Clone, Debug)] pub struct CompiledCode(pub Srcloc, pub Rc); -/// Specifying how the language is spoken. -/// -/// This object will eventually contain more information about the specifics of -/// the requested dialect. Initially, this includes a 'strict' setting in the -/// modern macros PR which allows us to begin with the *strict-cl-21* sigil to -/// include a more modern macro system and the ability to turn on strict variable -/// name use. This is a feature that's been widely requested and a first step -/// toward it is to make the object that specifies how chialisp is compiled be -/// able to carry more information. -#[derive(Clone, Debug, Default)] -pub struct AcceptedDialect { - pub stepping: Option, -} - /// A description of an inlined function for use during inline expansion. /// This is used only by PrimaryCodegen. #[derive(Clone, Debug)] diff --git a/src/compiler/dialect.rs b/src/compiler/dialect.rs new file mode 100644 index 000000000..4e1093da9 --- /dev/null +++ b/src/compiler/dialect.rs @@ -0,0 +1,113 @@ +use std::collections::HashMap; + +use clvmr::allocator::{Allocator, NodePtr, SExp}; + +use crate::classic::clvm::sexp::proper_list; + +use crate::compiler::sexp::decode_string; + +/// Specifying how the language is spoken. +#[derive(Clone, Debug, Default)] +pub struct AcceptedDialect { + pub stepping: Option, +} + +/// A package containing the content we should insert when a dialect include is +/// used, plus the compilation flags. +#[derive(Clone, Debug)] +pub struct DialectDescription { + pub accepted: AcceptedDialect, + pub content: String, +} + +lazy_static! { + pub static ref KNOWN_DIALECTS: HashMap = { + let mut dialects: HashMap = HashMap::new(); + let dialect_list = [ + ( + "*standard-cl-21*", + DialectDescription { + accepted: AcceptedDialect { + stepping: Some(21), + ..Default::default() + }, + content: indoc! {"( + (defconstant *chialisp-version* 21) + )"} + .to_string(), + }, + ), + ( + "*standard-cl-22*", + DialectDescription { + accepted: AcceptedDialect { stepping: Some(22) }, + content: indoc! {"( + (defconstant *chialisp-version* 22) + )"} + .to_string(), + }, + ), + ]; + for (n, v) in dialect_list.iter() { + dialects.insert(n.to_string(), v.clone()); + } + dialects + }; +} + +fn include_dialect(allocator: &Allocator, e: &[NodePtr]) -> Option { + let include_keyword_sexp = e[0]; + let name_sexp = e[1]; + if let (SExp::Atom(), SExp::Atom()) = ( + allocator.sexp(include_keyword_sexp), + allocator.sexp(name_sexp), + ) { + if allocator.atom(include_keyword_sexp) == "include".as_bytes().to_vec() { + if let Some(dialect) = KNOWN_DIALECTS.get(&decode_string(allocator.atom(name_sexp))) { + return Some(dialect.accepted.clone()); + } + } + } + + None +} + +// Now return more parameters about the "modern" dialect, including in the future, +// strictness. This will allow us to support the transition to modern macros which +// in turn allow us to turn on strictness in variable naming. Often multiple moves +// are needed to get from one point to another and there's a tension between +// unitary changes and smaller PRs which do fewer things by themselves. This is +// part of a broader narrative, which many requested that sets us on the path of +// being able to include more information in the dialect result. +pub fn detect_modern(allocator: &mut Allocator, sexp: NodePtr) -> AcceptedDialect { + let mut result = AcceptedDialect::default(); + + if let Some(l) = proper_list(allocator, sexp, true) { + for elt in l.iter() { + let detect_modern_result = detect_modern(allocator, *elt); + if detect_modern_result.stepping.is_some() { + result = detect_modern_result; + break; + } + + match proper_list(allocator, *elt, true) { + None => { + continue; + } + + Some(e) => { + if e.len() != 2 { + continue; + } + + if let Some(dialect) = include_dialect(allocator, &e) { + result = dialect; + break; + } + } + } + } + } + + result +} diff --git a/src/compiler/mod.rs b/src/compiler/mod.rs index 32e12ad78..af2c6ff15 100644 --- a/src/compiler/mod.rs +++ b/src/compiler/mod.rs @@ -13,8 +13,9 @@ pub mod compiler; /// - CompileForm - The type of finished (mod ) forms before code generation. /// - HelperForm - The type of declarations like macros, constants and functions. pub mod comptypes; -/// pub mod debug; +/// Utilities for chialisp dialect choice +pub mod dialect; pub mod evaluate; pub mod frontend; pub mod gensym; diff --git a/src/tests/classic/stage_2.rs b/src/tests/classic/stage_2.rs index 3ad0b61b0..8bf43064f 100644 --- a/src/tests/classic/stage_2.rs +++ b/src/tests/classic/stage_2.rs @@ -17,7 +17,8 @@ use crate::classic::clvm_tools::stages::stage_2::helpers::{brun, evaluate, quote use crate::classic::clvm_tools::stages::stage_2::operators::run_program_for_search_paths; use crate::classic::clvm_tools::stages::stage_2::reader::{process_embed_file, read_file}; -use crate::compiler::comptypes::{AcceptedDialect, CompileErr, CompilerOpts, PrimaryCodegen}; +use crate::compiler::comptypes::{CompileErr, CompilerOpts, PrimaryCodegen}; +use crate::compiler::dialect::AcceptedDialect; use crate::compiler::sexp::{decode_string, SExp}; use crate::compiler::srcloc::Srcloc; From ea68090c16257821a743dcd42862a94c89a1a66a Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 7 Aug 2023 17:31:59 -0700 Subject: [PATCH 058/117] Default -> AcceptedDialect --- src/compiler/dialect.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/dialect.rs b/src/compiler/dialect.rs index 4e1093da9..26f0a3629 100644 --- a/src/compiler/dialect.rs +++ b/src/compiler/dialect.rs @@ -29,7 +29,7 @@ lazy_static! { DialectDescription { accepted: AcceptedDialect { stepping: Some(21), - ..Default::default() + ..AcceptedDialect::default() }, content: indoc! {"( (defconstant *chialisp-version* 21) From 296cd2a7ba3c39ba4c14be243346b35f8ebdddaf Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 7 Aug 2023 17:34:00 -0700 Subject: [PATCH 059/117] Remove default --- src/compiler/dialect.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/compiler/dialect.rs b/src/compiler/dialect.rs index 26f0a3629..a01ac133d 100644 --- a/src/compiler/dialect.rs +++ b/src/compiler/dialect.rs @@ -29,7 +29,6 @@ lazy_static! { DialectDescription { accepted: AcceptedDialect { stepping: Some(21), - ..AcceptedDialect::default() }, content: indoc! {"( (defconstant *chialisp-version* 21) From f34cafd230ed34439c03c4e9e283779082831dc9 Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 7 Aug 2023 17:40:47 -0700 Subject: [PATCH 060/117] fmt --- src/compiler/dialect.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/compiler/dialect.rs b/src/compiler/dialect.rs index a01ac133d..58b561993 100644 --- a/src/compiler/dialect.rs +++ b/src/compiler/dialect.rs @@ -27,9 +27,7 @@ lazy_static! { ( "*standard-cl-21*", DialectDescription { - accepted: AcceptedDialect { - stepping: Some(21), - }, + accepted: AcceptedDialect { stepping: Some(21) }, content: indoc! {"( (defconstant *chialisp-version* 21) )"} From 804947f84da4be555387b298c784d0e87f15521f Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Tue, 8 Aug 2023 17:04:08 +0100 Subject: [PATCH 061/117] more comments --- src/compiler/sexp.rs | 46 +++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/src/compiler/sexp.rs b/src/compiler/sexp.rs index 5b5c4fd6c..1b1598560 100644 --- a/src/compiler/sexp.rs +++ b/src/compiler/sexp.rs @@ -213,7 +213,7 @@ enum SExpParseState { // The types of state that the Rust pre-forms can take Srcloc, TermListCommentState, Option>, - Rc, + Rc, // used for inner parsing Vec>, ), } @@ -621,9 +621,9 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - match (this_char as char, pp.borrow()) { ('.', SExpParseState::Empty) => resume(SExpParseState::TermList( // dot notation showing cons cell srcloc.ext(&loc), - TermListCommentState::Empty, + TermListCommentState::Empty, // we are not inside a comment None, - Rc::new(SExpParseState::Empty), + Rc::new(SExpParseState::Empty), // nested state is empty list_content.to_vec(), )), (')', SExpParseState::Empty) => emit( // close list and emit it upwards as a complete entity @@ -639,24 +639,26 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - SExpParseState::Empty, ) } + // analyze this character using the mock "inner state" stored in pp (_, _) => match parse_sexp_step(loc.clone(), pp.borrow(), this_char) { // - SExpParseResult::Emit(o, current_state) => { // add result of nested call to our list + SExpParseResult::Emit(o, current_state) => { // add result of parse_sexp_step to our list let mut list_copy = list_content.clone(); list_copy.push(o); let result = SExpParseState::ParsingList(srcloc.ext(&loc), Rc::new(current_state), list_copy); resume(result) } - SExpParseResult::Resume(rp) => resume(SExpParseState::ParsingList( // + SExpParseResult::Resume(rp) => resume(SExpParseState::ParsingList( // we aren't finished reading in our nested state srcloc.ext(&loc), - Rc::new(rp), + Rc::new(rp), // store the returned state from parse_sexp_step in pp list_content.to_vec(), )), SExpParseResult::Error(l, e) => SExpParseResult::Error(l, e), // propagate error upwards }, } } - SExpParseState::TermList(srcloc, TermListCommentState::InComment, parsed, pp, list_content) => { // pp is the captured inside-list state we received from OpenList + // if we're in a comment then just check for newline or carriage return otherwise stay in InComment state + SExpParseState::TermList(srcloc, TermListCommentState::InComment, parsed, pp, list_content) => { let end_comment = if this_char as char == '\n' || this_char as char == '\r' { TermListCommentState::Empty } else { @@ -664,12 +666,13 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - }; resume(SExpParseState::TermList( srcloc.clone(), - end_comment, + end_comment, // store the new commentstate parsed.clone(), pp.clone(), list_content.clone(), )) } + // if we're not in a comment and have already found a parsed second word for this dot expression SExpParseState::TermList( srcloc, TermListCommentState::Empty, @@ -677,7 +680,7 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - pp, list_content, ) => { - if this_char.is_ascii_whitespace() { + if this_char.is_ascii_whitespace() { // ignore whitespace after second word resume(SExpParseState::TermList( srcloc.ext(&loc), TermListCommentState::Empty, @@ -685,7 +688,7 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - pp.clone(), list_content.to_vec(), )) - } else if this_char == b')' { + } else if this_char == b')' { // if we see a `)` then we're ready to close this list let mut list_copy = list_content.to_vec(); match list_copy.pop() { Some(v) => { @@ -697,12 +700,12 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - for item in list_copy.iter().rev() { result_list = make_cons(item.clone(), Rc::new(result_list)); } - emit(Rc::new(result_list), SExpParseState::Empty) + emit(Rc::new(result_list), SExpParseState::Empty) // emit the resultant list } } None => error(loc, "Dot as first element of list?"), } - } else if this_char == b';' { + } else if this_char == b';' { // entering a comment resume(SExpParseState::TermList( srcloc.clone(), TermListCommentState::InComment, @@ -710,19 +713,20 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - pp.clone(), list_content.clone(), )) - } else { + } else { // we don't want to see any more characters after we've concluded a dot expression error( srcloc.clone(), &format!("unexpected character {}", this_char as char), ) } } - SExpParseState::TermList(srcloc, TermListCommentState::Empty, None, pp, list_content) => { - match (this_char as char, pp.borrow()) { - ('.', SExpParseState::Empty) => { + // we are passing a dot-expression (x . y) and not in a comment and don't have an object already discovered + SExpParseState::TermList(srcloc, TermListCommentState::Empty, None, pp, list_content) => { // pp is the inner parsestate inside the dot-expressions + match (this_char as char, pp.borrow()) { //match based on current character and inner state + ('.', SExpParseState::Empty) => { // if we aren't in a word and we see another dot that's illegal error(loc, "Multiple dots in list notation are illegal") } - (')', SExpParseState::Empty) => { + (')', SExpParseState::Empty) => { // attempt to close the list if list_content.len() == 1 { emit(list_content[0].clone(), SExpParseState::Empty) } else { @@ -751,19 +755,21 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - None => error(loc, "Dot as first element of list?"), } } + // if we see anything other than ')' or '.' parse it as if we were in empty state (_, _) => match parse_sexp_step(loc.clone(), pp.borrow(), this_char) { - SExpParseResult::Emit(o, _current_state) => resume(SExpParseState::TermList( + SExpParseResult::Emit(parsed_object, _current_state) => resume(SExpParseState::TermList( loc, TermListCommentState::Empty, - Some(o), + Some(parsed_object), // assert parsed_object is not None and then store it in parsed_list pp.clone(), list_content.clone(), )), + // resume means it didn't finish parsing yet, so store inner state and keep going SExpParseResult::Resume(current_state) => resume(SExpParseState::TermList( srcloc.ext(&loc), TermListCommentState::Empty, None, - Rc::new(current_state), + Rc::new(current_state), // store our partial inner parsestate in pp list_content.to_vec(), )), SExpParseResult::Error(l, e) => SExpParseResult::Error(l, e), From 8bb994877b31101acb54ad5331fb59b2e1618cbe Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Tue, 8 Aug 2023 17:07:03 +0100 Subject: [PATCH 062/117] explain an enum param --- src/compiler/sexp.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/sexp.rs b/src/compiler/sexp.rs index 1b1598560..e89dbb6e4 100644 --- a/src/compiler/sexp.rs +++ b/src/compiler/sexp.rs @@ -212,7 +212,7 @@ enum SExpParseState { // The types of state that the Rust pre-forms can take TermList( Srcloc, TermListCommentState, - Option>, + Option>, // this is the second value in the dot expression Rc, // used for inner parsing Vec>, ), From 59641cc810b569644c3f4cb20b0daed1da46af77 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Tue, 8 Aug 2023 17:47:22 +0100 Subject: [PATCH 063/117] cargo fmt --- src/compiler/sexp.rs | 151 +++++++++++++++++++++++++++---------------- 1 file changed, 96 insertions(+), 55 deletions(-) diff --git a/src/compiler/sexp.rs b/src/compiler/sexp.rs index e89dbb6e4..9759552ae 100644 --- a/src/compiler/sexp.rs +++ b/src/compiler/sexp.rs @@ -201,9 +201,10 @@ enum TermListCommentState { } #[derive(Debug)] -enum SExpParseState { // The types of state that the Rust pre-forms can take +enum SExpParseState { + // The types of state that the Rust pre-forms can take Empty, - CommentText(Srcloc, Vec), //srcloc contains the file, line, column and length for the captured form + CommentText(Srcloc, Vec), //srcloc contains the file, line, column and length for the captured form Bareword(Srcloc, Vec), QuotedText(Srcloc, u8, Vec), QuotedEscaped(Srcloc, u8, Vec), @@ -212,14 +213,15 @@ enum SExpParseState { // The types of state that the Rust pre-forms can take TermList( Srcloc, TermListCommentState, - Option>, // this is the second value in the dot expression - Rc, // used for inner parsing + Option>, // this is the second value in the dot expression + Rc, // used for inner parsing Vec>, ), } #[derive(Debug)] -enum SExpParseResult { // the result of a call to parse an SExp +enum SExpParseResult { + // the result of a call to parse an SExp Resume(SExpParseState), Emit(Rc, SExpParseState), Error(Srcloc, String), @@ -541,18 +543,19 @@ impl SExp { fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) -> SExpParseResult { // switch on our state match current_state { - SExpParseState::Empty => match this_char as char { // we are not currently in a list - '(' => resume(SExpParseState::OpenList(loc)), // move to OpenList state - '\n' => resume(SExpParseState::Empty), // new line, same state + SExpParseState::Empty => match this_char as char { + // we are not currently in a list + '(' => resume(SExpParseState::OpenList(loc)), // move to OpenList state + '\n' => resume(SExpParseState::Empty), // new line, same state ';' => resume(SExpParseState::CommentText(loc, Vec::new())), ')' => error(loc, "Too many close parens"), '"' => resume(SExpParseState::QuotedText(loc, b'"', Vec::new())), // match on " - '\'' => resume(SExpParseState::QuotedText(loc, b'\'', Vec::new())), // match on ' + '\'' => resume(SExpParseState::QuotedText(loc, b'\'', Vec::new())), // match on ' ch => { if char::is_whitespace(ch) { resume(SExpParseState::Empty) } else { - resume(SExpParseState::Bareword(loc, vec![this_char])) // start of a word - could be an atom or a keyword - the compiler will decide + resume(SExpParseState::Bareword(loc, vec![this_char])) // start of a word - could be an atom or a keyword - the compiler will decide } } }, @@ -568,26 +571,35 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - }, // we currently processing a new word SExpParseState::Bareword(srcloc, word_so_far) => { - if char::is_whitespace(this_char as char) { // we've found a space, so it's the end of a word + if char::is_whitespace(this_char as char) { + // we've found a space, so it's the end of a word emit( Rc::new(make_atom(srcloc.clone(), word_so_far.to_vec())), SExpParseState::Empty, ) - } else { // otherwise add letter to word + } else { + // otherwise add letter to word let mut word_copy = word_so_far.to_vec(); word_copy.push(this_char); resume(SExpParseState::Bareword(srcloc.ext(&loc), word_copy)) } } SExpParseState::QuotedText(srcloc, term, t) => { - if this_char == b'\\' { // if we have a character escape then copy the character directly - resume(SExpParseState::QuotedEscaped(srcloc.clone(), *term, t.to_vec())) - } else if this_char == *term { // otherwise check if it's the terminating character (either ' or ") + if this_char == b'\\' { + // if we have a character escape then copy the character directly + resume(SExpParseState::QuotedEscaped( + srcloc.clone(), + *term, + t.to_vec(), + )) + } else if this_char == *term { + // otherwise check if it's the terminating character (either ' or ") emit( Rc::new(SExp::QuotedString(srcloc.ext(&loc), *term, t.to_vec())), // add quoted string to parent list SExpParseState::Empty, ) - } else { // otherwise copy the character + } else { + // otherwise copy the character let mut tcopy = t.to_vec(); tcopy.push(this_char); resume(SExpParseState::QuotedText(srcloc.clone(), *term, tcopy)) @@ -599,38 +611,46 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - tcopy.push(this_char); resume(SExpParseState::QuotedText(srcloc.clone(), *term, tcopy)) } - SExpParseState::OpenList(srcloc) => match this_char as char { // we are beginning a new list - ')' => emit(Rc::new(SExp::Nil(srcloc.ext(&loc))), SExpParseState::Empty), // create a Nil object + SExpParseState::OpenList(srcloc) => match this_char as char { + // we are beginning a new list + ')' => emit(Rc::new(SExp::Nil(srcloc.ext(&loc))), SExpParseState::Empty), // create a Nil object '.' => error(loc, "Dot can't appear directly after begin paren"), - _ => match parse_sexp_step(loc.clone(), &SExpParseState::Empty, this_char) { // fetch result of parsing as if we were in empty state - SExpParseResult::Emit(o, current_state) => resume(SExpParseState::ParsingList( // we found an object, resume processing + _ => match parse_sexp_step(loc.clone(), &SExpParseState::Empty, this_char) { + // fetch result of parsing as if we were in empty state + SExpParseResult::Emit(o, current_state) => resume(SExpParseState::ParsingList( + // we found an object, resume processing srcloc.ext(&loc), - Rc::new(current_state), // captured state from our pretend empty state + Rc::new(current_state), // captured state from our pretend empty state vec![o], )), - SExpParseResult::Resume(current_state) => resume(SExpParseState::ParsingList( // we're still reading the object, resume processing + SExpParseResult::Resume(current_state) => resume(SExpParseState::ParsingList( + // we're still reading the object, resume processing srcloc.ext(&loc), - Rc::new(current_state), // captured state from our pretend empty state + Rc::new(current_state), // captured state from our pretend empty state Vec::new(), )), SExpParseResult::Error(l, e) => SExpParseResult::Error(l, e), // propagate error }, }, // We are in the middle of a list currently - SExpParseState::ParsingList(srcloc, pp, list_content) => { // pp is the captured inside-list state we received from OpenList + SExpParseState::ParsingList(srcloc, pp, list_content) => { + // pp is the captured inside-list state we received from OpenList match (this_char as char, pp.borrow()) { - ('.', SExpParseState::Empty) => resume(SExpParseState::TermList( // dot notation showing cons cell + ('.', SExpParseState::Empty) => resume(SExpParseState::TermList( + // dot notation showing cons cell srcloc.ext(&loc), - TermListCommentState::Empty, // we are not inside a comment + TermListCommentState::Empty, // we are not inside a comment None, - Rc::new(SExpParseState::Empty), // nested state is empty + Rc::new(SExpParseState::Empty), // nested state is empty list_content.to_vec(), )), - (')', SExpParseState::Empty) => emit( // close list and emit it upwards as a complete entity + (')', SExpParseState::Empty) => emit( + // close list and emit it upwards as a complete entity Rc::new(enlist(srcloc.clone(), list_content)), SExpParseState::Empty, ), - (')', SExpParseState::Bareword(l, t)) => { // you've reached the end of the word AND the end of the list, close list and emit upwards + (')', SExpParseState::Bareword(l, t)) => { + // you've reached the end of the word AND the end of the list, close list and emit upwards let parsed_atom = make_atom(l.clone(), t.to_vec()); let mut updated_list = list_content.to_vec(); updated_list.push(Rc::new(parsed_atom)); @@ -640,25 +660,37 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - ) } // analyze this character using the mock "inner state" stored in pp - (_, _) => match parse_sexp_step(loc.clone(), pp.borrow(), this_char) { // - SExpParseResult::Emit(o, current_state) => { // add result of parse_sexp_step to our list + (_, _) => match parse_sexp_step(loc.clone(), pp.borrow(), this_char) { + // + SExpParseResult::Emit(o, current_state) => { + // add result of parse_sexp_step to our list let mut list_copy = list_content.clone(); list_copy.push(o); - let result = - SExpParseState::ParsingList(srcloc.ext(&loc), Rc::new(current_state), list_copy); + let result = SExpParseState::ParsingList( + srcloc.ext(&loc), + Rc::new(current_state), + list_copy, + ); resume(result) } - SExpParseResult::Resume(rp) => resume(SExpParseState::ParsingList( // we aren't finished reading in our nested state + SExpParseResult::Resume(rp) => resume(SExpParseState::ParsingList( + // we aren't finished reading in our nested state srcloc.ext(&loc), - Rc::new(rp), // store the returned state from parse_sexp_step in pp + Rc::new(rp), // store the returned state from parse_sexp_step in pp list_content.to_vec(), )), - SExpParseResult::Error(l, e) => SExpParseResult::Error(l, e), // propagate error upwards + SExpParseResult::Error(l, e) => SExpParseResult::Error(l, e), // propagate error upwards }, } } // if we're in a comment then just check for newline or carriage return otherwise stay in InComment state - SExpParseState::TermList(srcloc, TermListCommentState::InComment, parsed, pp, list_content) => { + SExpParseState::TermList( + srcloc, + TermListCommentState::InComment, + parsed, + pp, + list_content, + ) => { let end_comment = if this_char as char == '\n' || this_char as char == '\r' { TermListCommentState::Empty } else { @@ -666,7 +698,7 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - }; resume(SExpParseState::TermList( srcloc.clone(), - end_comment, // store the new commentstate + end_comment, // store the new commentstate parsed.clone(), pp.clone(), list_content.clone(), @@ -680,7 +712,8 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - pp, list_content, ) => { - if this_char.is_ascii_whitespace() { // ignore whitespace after second word + if this_char.is_ascii_whitespace() { + // ignore whitespace after second word resume(SExpParseState::TermList( srcloc.ext(&loc), TermListCommentState::Empty, @@ -688,7 +721,8 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - pp.clone(), list_content.to_vec(), )) - } else if this_char == b')' { // if we see a `)` then we're ready to close this list + } else if this_char == b')' { + // if we see a `)` then we're ready to close this list let mut list_copy = list_content.to_vec(); match list_copy.pop() { Some(v) => { @@ -700,12 +734,13 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - for item in list_copy.iter().rev() { result_list = make_cons(item.clone(), Rc::new(result_list)); } - emit(Rc::new(result_list), SExpParseState::Empty) // emit the resultant list + emit(Rc::new(result_list), SExpParseState::Empty) // emit the resultant list } } None => error(loc, "Dot as first element of list?"), } - } else if this_char == b';' { // entering a comment + } else if this_char == b';' { + // entering a comment resume(SExpParseState::TermList( srcloc.clone(), TermListCommentState::InComment, @@ -713,7 +748,8 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - pp.clone(), list_content.clone(), )) - } else { // we don't want to see any more characters after we've concluded a dot expression + } else { + // we don't want to see any more characters after we've concluded a dot expression error( srcloc.clone(), &format!("unexpected character {}", this_char as char), @@ -721,12 +757,16 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - } } // we are passing a dot-expression (x . y) and not in a comment and don't have an object already discovered - SExpParseState::TermList(srcloc, TermListCommentState::Empty, None, pp, list_content) => { // pp is the inner parsestate inside the dot-expressions - match (this_char as char, pp.borrow()) { //match based on current character and inner state - ('.', SExpParseState::Empty) => { // if we aren't in a word and we see another dot that's illegal + SExpParseState::TermList(srcloc, TermListCommentState::Empty, None, pp, list_content) => { + // pp is the inner parsestate inside the dot-expressions + match (this_char as char, pp.borrow()) { + //match based on current character and inner state + ('.', SExpParseState::Empty) => { + // if we aren't in a word and we see another dot that's illegal error(loc, "Multiple dots in list notation are illegal") } - (')', SExpParseState::Empty) => { // attempt to close the list + (')', SExpParseState::Empty) => { + // attempt to close the list if list_content.len() == 1 { emit(list_content[0].clone(), SExpParseState::Empty) } else { @@ -757,13 +797,15 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - } // if we see anything other than ')' or '.' parse it as if we were in empty state (_, _) => match parse_sexp_step(loc.clone(), pp.borrow(), this_char) { - SExpParseResult::Emit(parsed_object, _current_state) => resume(SExpParseState::TermList( - loc, - TermListCommentState::Empty, - Some(parsed_object), // assert parsed_object is not None and then store it in parsed_list - pp.clone(), - list_content.clone(), - )), + SExpParseResult::Emit(parsed_object, _current_state) => { + resume(SExpParseState::TermList( + loc, + TermListCommentState::Empty, + Some(parsed_object), // assert parsed_object is not None and then store it in parsed_list + pp.clone(), + list_content.clone(), + )) + } // resume means it didn't finish parsing yet, so store inner state and keep going SExpParseResult::Resume(current_state) => resume(SExpParseState::TermList( srcloc.ext(&loc), @@ -793,7 +835,6 @@ where // Loop through all the characters for this_char in s { - let next_location = start.clone().advance(this_char); // call parse_sexp_step for current character From ba3fc5816db294641e358d4e0311a7d81f10d74f Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 8 Aug 2023 14:53:04 -0700 Subject: [PATCH 064/117] Version bump for new release --- Cargo.lock | 2 +- Cargo.toml | 2 +- wasm/Cargo.lock | 4 ++-- wasm/Cargo.toml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6cc688bc4..165533baa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -117,7 +117,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clvm_tools_rs" -version = "0.1.34" +version = "0.1.35" dependencies = [ "binascii", "bls12_381", diff --git a/Cargo.toml b/Cargo.toml index 63337780e..195c56297 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clvm_tools_rs" -version = "0.1.34" +version = "0.1.35" edition = "2018" authors = ["Art Yerkes "] description = "tools for working with chialisp language; compiler, repl, python and wasm bindings" diff --git a/wasm/Cargo.lock b/wasm/Cargo.lock index e6087b96a..c64baa9f3 100644 --- a/wasm/Cargo.lock +++ b/wasm/Cargo.lock @@ -117,7 +117,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clvm_tools_rs" -version = "0.1.34" +version = "0.1.35" dependencies = [ "binascii", "bls12_381", @@ -148,7 +148,7 @@ dependencies = [ [[package]] name = "clvm_tools_wasm" -version = "0.1.34" +version = "0.1.35" dependencies = [ "clvm_tools_rs", "clvmr", diff --git a/wasm/Cargo.toml b/wasm/Cargo.toml index 4388b2dc0..95c8cc9d2 100644 --- a/wasm/Cargo.toml +++ b/wasm/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clvm_tools_wasm" -version = "0.1.34" +version = "0.1.35" edition = "2018" authors = ["Art Yerkes "] description = "tools for working with chialisp language; compiler, repl, python and wasm bindings" From e3b35f0e3d1789dea2816d21bdf473e2b191586f Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 8 Aug 2023 15:00:15 -0700 Subject: [PATCH 065/117] Changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2bbf5116..e50a9fa5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,3 +24,9 @@ Skipped - hierarchial debug was added. - clvm command linetools: supported more command line features in both compiler front-ends. +## 0.1.35 + +- embed-file was added. +- &rest arguments. +- new bls and sec256 operators. + From 9f9ba67c0507e11f51d9496ecc310218da53544a Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Wed, 9 Aug 2023 15:56:10 +0100 Subject: [PATCH 066/117] remove TermListInComment enum --- src/compiler/sexp.rs | 91 ++++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 46 deletions(-) diff --git a/src/compiler/sexp.rs b/src/compiler/sexp.rs index 9759552ae..52bae6bef 100644 --- a/src/compiler/sexp.rs +++ b/src/compiler/sexp.rs @@ -194,12 +194,6 @@ fn make_cons(a: Rc, b: Rc) -> SExp { SExp::Cons(a.loc().ext(&b.loc()), a.clone(), b.clone()) } -#[derive(Debug, PartialEq, Eq)] -enum TermListCommentState { - InComment, - Empty, -} - #[derive(Debug)] enum SExpParseState { // The types of state that the Rust pre-forms can take @@ -212,10 +206,9 @@ enum SExpParseState { ParsingList(Srcloc, Rc, Vec>), TermList( Srcloc, - TermListCommentState, Option>, // this is the second value in the dot expression Rc, // used for inner parsing - Vec>, + Vec>, // list content ), } @@ -639,7 +632,6 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - ('.', SExpParseState::Empty) => resume(SExpParseState::TermList( // dot notation showing cons cell srcloc.ext(&loc), - TermListCommentState::Empty, // we are not inside a comment None, Rc::new(SExpParseState::Empty), // nested state is empty list_content.to_vec(), @@ -683,31 +675,10 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - }, } } - // if we're in a comment then just check for newline or carriage return otherwise stay in InComment state - SExpParseState::TermList( - srcloc, - TermListCommentState::InComment, - parsed, - pp, - list_content, - ) => { - let end_comment = if this_char as char == '\n' || this_char as char == '\r' { - TermListCommentState::Empty - } else { - TermListCommentState::InComment - }; - resume(SExpParseState::TermList( - srcloc.clone(), - end_comment, // store the new commentstate - parsed.clone(), - pp.clone(), - list_content.clone(), - )) - } + // if we're not in a comment and have already found a parsed second word for this dot expression SExpParseState::TermList( srcloc, - TermListCommentState::Empty, Some(parsed), pp, list_content, @@ -716,7 +687,6 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - // ignore whitespace after second word resume(SExpParseState::TermList( srcloc.ext(&loc), - TermListCommentState::Empty, Some(parsed.clone()), pp.clone(), list_content.to_vec(), @@ -741,23 +711,54 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - } } else if this_char == b';' { // entering a comment + // resume(SExpParseState::TermList( + // srcloc.clone(), + // TermListCommentState::InComment, + // Some(parsed.clone()), + // pp.clone(), + // list_content.clone(), + // )) resume(SExpParseState::TermList( - srcloc.clone(), - TermListCommentState::InComment, - Some(parsed.clone()), - pp.clone(), + loc.clone(), + Some(parsed.clone()), // assert parsed_object is not None and then store it in parsed_list + Rc::new(SExpParseState::CommentText(loc.clone(), Vec::new())), list_content.clone(), )) } else { - // we don't want to see any more characters after we've concluded a dot expression - error( - srcloc.clone(), - &format!("unexpected character {}", this_char as char), - ) + match pp.as_ref() { + SExpParseState::CommentText(comment_loc, comment_text) => { + match this_char as char { + '\r' => resume(SExpParseState::CommentText(comment_loc.clone(), comment_text.to_vec())), + '\n' => resume(SExpParseState::TermList( + loc.clone(), + Some(parsed.clone()), + Rc::new(SExpParseState::Empty), + list_content.clone(), + )), + _ => { + let mut tcopy = comment_text.to_vec(); + tcopy.push(this_char); + resume(SExpParseState::TermList( + loc.clone(), + Some(parsed.clone()), + Rc::new(SExpParseState::CommentText(srcloc.ext(&loc), tcopy)), + list_content.clone(), + )) + } + } + }, + _ => { + // we don't want to see any more characters after we've concluded a dot expression + error( + srcloc.clone(), + &format!("unexpected character {}", this_char as char), + ) + } + } } } // we are passing a dot-expression (x . y) and not in a comment and don't have an object already discovered - SExpParseState::TermList(srcloc, TermListCommentState::Empty, None, pp, list_content) => { + SExpParseState::TermList(srcloc, None, pp, list_content) => { // pp is the inner parsestate inside the dot-expressions match (this_char as char, pp.borrow()) { //match based on current character and inner state @@ -800,16 +801,14 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - SExpParseResult::Emit(parsed_object, _current_state) => { resume(SExpParseState::TermList( loc, - TermListCommentState::Empty, Some(parsed_object), // assert parsed_object is not None and then store it in parsed_list - pp.clone(), + Rc::new(SExpParseState::Empty), list_content.clone(), )) } // resume means it didn't finish parsing yet, so store inner state and keep going SExpParseResult::Resume(current_state) => resume(SExpParseState::TermList( srcloc.ext(&loc), - TermListCommentState::Empty, None, Rc::new(current_state), // store our partial inner parsestate in pp list_content.to_vec(), @@ -869,7 +868,7 @@ where } SExpParseState::OpenList(l) => Err((l, "Unterminated list (empty)".to_string())), SExpParseState::ParsingList(l, _, _) => Err((l, "Unterminated mid list".to_string())), - SExpParseState::TermList(l, _, _, _, _) => Err((l, "Unterminated tail list".to_string())), + SExpParseState::TermList(l, _, _, _) => Err((l, "Unterminated tail list".to_string())), } } From 1a9b6a7d3e455704e1fd0c6ceab30a4408429b0d Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Wed, 9 Aug 2023 17:26:17 +0100 Subject: [PATCH 067/117] simplify CommentText to not store the comment text --- src/compiler/sexp.rs | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/src/compiler/sexp.rs b/src/compiler/sexp.rs index 52bae6bef..443234b97 100644 --- a/src/compiler/sexp.rs +++ b/src/compiler/sexp.rs @@ -198,8 +198,8 @@ fn make_cons(a: Rc, b: Rc) -> SExp { enum SExpParseState { // The types of state that the Rust pre-forms can take Empty, - CommentText(Srcloc, Vec), //srcloc contains the file, line, column and length for the captured form - Bareword(Srcloc, Vec), + CommentText, + Bareword(Srcloc, Vec), //srcloc contains the file, line, column and length for the captured form QuotedText(Srcloc, u8, Vec), QuotedEscaped(Srcloc, u8, Vec), OpenList(Srcloc), @@ -540,7 +540,7 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - // we are not currently in a list '(' => resume(SExpParseState::OpenList(loc)), // move to OpenList state '\n' => resume(SExpParseState::Empty), // new line, same state - ';' => resume(SExpParseState::CommentText(loc, Vec::new())), + ';' => resume(SExpParseState::CommentText), ')' => error(loc, "Too many close parens"), '"' => resume(SExpParseState::QuotedText(loc, b'"', Vec::new())), // match on " '\'' => resume(SExpParseState::QuotedText(loc, b'\'', Vec::new())), // match on ' @@ -553,13 +553,10 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - } }, // t is a Vec of the previous characters in this comment string - SExpParseState::CommentText(srcloc, t) => match this_char as char { - '\r' => resume(SExpParseState::CommentText(srcloc.clone(), t.to_vec())), + SExpParseState::CommentText => match this_char as char { '\n' => resume(SExpParseState::Empty), _ => { - let mut tcopy = t.to_vec(); - tcopy.push(this_char); - resume(SExpParseState::CommentText(srcloc.ext(&loc), tcopy)) + resume(SExpParseState::CommentText) } }, // we currently processing a new word @@ -721,14 +718,13 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - resume(SExpParseState::TermList( loc.clone(), Some(parsed.clone()), // assert parsed_object is not None and then store it in parsed_list - Rc::new(SExpParseState::CommentText(loc.clone(), Vec::new())), + Rc::new(SExpParseState::CommentText), list_content.clone(), )) } else { match pp.as_ref() { - SExpParseState::CommentText(comment_loc, comment_text) => { + SExpParseState::CommentText => { match this_char as char { - '\r' => resume(SExpParseState::CommentText(comment_loc.clone(), comment_text.to_vec())), '\n' => resume(SExpParseState::TermList( loc.clone(), Some(parsed.clone()), @@ -736,12 +732,10 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - list_content.clone(), )), _ => { - let mut tcopy = comment_text.to_vec(); - tcopy.push(this_char); resume(SExpParseState::TermList( loc.clone(), Some(parsed.clone()), - Rc::new(SExpParseState::CommentText(srcloc.ext(&loc), tcopy)), + Rc::new(SExpParseState::CommentText), list_content.clone(), )) } @@ -861,7 +855,7 @@ where match parse_state { SExpParseState::Empty => Ok(res), SExpParseState::Bareword(l, t) => Ok(vec![Rc::new(make_atom(l, t))]), - SExpParseState::CommentText(_, _) => Ok(res), + SExpParseState::CommentText => Ok(res), SExpParseState::QuotedText(l, _, _) => Err((l, "unterminated quoted string".to_string())), SExpParseState::QuotedEscaped(l, _, _) => { Err((l, "unterminated quoted string with escape".to_string())) From a889d93916957c811155606dc4b4028f617a9426 Mon Sep 17 00:00:00 2001 From: Brandon Butler Date: Wed, 9 Aug 2023 11:22:28 -0700 Subject: [PATCH 068/117] Use ephemeral credentials from AWS OIDC provider --- .github/workflows/build-arm64-wheels.yml | 16 +++++++++++----- .github/workflows/build-m1-wheel.yml | 16 +++++++++++----- .github/workflows/build-test.yml | 17 +++++++++++------ 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-arm64-wheels.yml b/.github/workflows/build-arm64-wheels.yml index ba40fe845..0618633af 100644 --- a/.github/workflows/build-arm64-wheels.yml +++ b/.github/workflows/build-arm64-wheels.yml @@ -11,6 +11,10 @@ on: branches: - '**' +permissions: + id-token: write + contents: read + jobs: build_wheels: name: Build ARM64 Python Wheels @@ -84,7 +88,7 @@ jobs: echo HAS_AWS_SECRET=${HAS_AWS_SECRET} >>$GITHUB_OUTPUT env: SECRET: "${{ secrets.test_pypi_password }}" - AWS_SECRET: "${{ secrets.INSTALLER_UPLOAD_KEY }}" + AWS_SECRET: "${{ secrets.CHIA_AWS_ACCOUNT_ID }}" - name: publish (PyPi) if: startsWith(github.event.ref, 'refs/tags') && steps.check_secrets.outputs.HAS_SECRET @@ -106,12 +110,14 @@ jobs: . ./activate twine upload --non-interactive --skip-existing --verbose 'target/wheels/*' + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: arn:aws:iam::${{ secrets.CHIA_AWS_ACCOUNT_ID }}:role/installer-upload + aws-region: us-west-2 + - name: Publish Dev if: steps.check_secrets.outputs.HAS_AWS_SECRET && github.ref == 'refs/heads/dev' - env: - AWS_ACCESS_KEY_ID: "${{ secrets.INSTALLER_UPLOAD_KEY }}" - AWS_SECRET_ACCESS_KEY: "${{ secrets.INSTALLER_UPLOAD_SECRET }}" - AWS_REGION: us-west-2 run: | FILES=$(find ${{ github.workspace }}/target/wheels -type f -name '*.whl') while IFS= read -r file; do diff --git a/.github/workflows/build-m1-wheel.yml b/.github/workflows/build-m1-wheel.yml index 917451d4f..1fda47663 100644 --- a/.github/workflows/build-m1-wheel.yml +++ b/.github/workflows/build-m1-wheel.yml @@ -15,6 +15,10 @@ concurrency: group: ${{ github.ref }}-${{ github.workflow }}-${{ github.event_name }}--${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/') || startsWith(github.ref, 'refs/heads/long_lived/')) && github.sha || '' }} cancel-in-progress: true +permissions: + id-token: write + contents: read + jobs: build_wheels: name: Build wheel on Mac M1 @@ -121,7 +125,7 @@ jobs: echo HAS_AWS_SECRET=${HAS_AWS_SECRET} >>$GITHUB_OUTPUT env: SECRET: "${{ secrets.test_pypi_password }}" - AWS_SECRET: "${{ secrets.INSTALLER_UPLOAD_KEY }}" + AWS_SECRET: "${{ secrets.CHIA_AWS_ACCOUNT_ID }}" - name: Install twine run: arch -arm64 pip install twine @@ -143,12 +147,14 @@ jobs: TWINE_PASSWORD: ${{ secrets.test_pypi_password }} run: arch -arm64 twine upload --non-interactive --skip-existing --verbose 'target/wheels/*' + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: arn:aws:iam::${{ secrets.CHIA_AWS_ACCOUNT_ID }}:role/installer-upload + aws-region: us-west-2 + - name: Publish Dev if: steps.check_secrets.outputs.HAS_AWS_SECRET && github.ref == 'refs/heads/dev' - env: - AWS_ACCESS_KEY_ID: "${{ secrets.INSTALLER_UPLOAD_KEY }}" - AWS_SECRET_ACCESS_KEY: "${{ secrets.INSTALLER_UPLOAD_SECRET }}" - AWS_REGION: us-west-2 run: | FILES=$(find ${{ github.workspace }}/target/wheels -type f -name '*.whl') while IFS= read -r file; do diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index f56023f04..4633cde84 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -12,6 +12,10 @@ on: branches: - '**' +permissions: + id-token: write + contents: read + jobs: build_wheels: name: Wheel on ${{ matrix.os }} py-${{ matrix.python }} @@ -232,10 +236,9 @@ jobs: if [ -n "$AWS_SECRET" ]; then HAS_AWS_SECRET='true' ; fi echo HAS_AWS_SECRET=${HAS_AWS_SECRET} >>$GITHUB_OUTPUT - env: SECRET: "${{ secrets.test_pypi_password }}" - AWS_SECRET: "${{ secrets.INSTALLER_UPLOAD_KEY }}" + AWS_SECRET: "${{ secrets.CHIA_AWS_ACCOUNT_ID }}" - name: publish (PyPi) if: startsWith(github.event.ref, 'refs/tags') && steps.check_secrets.outputs.HAS_SECRET @@ -254,14 +257,16 @@ jobs: TWINE_PASSWORD: ${{ secrets.test_pypi_password }} run: twine upload --non-interactive --skip-existing --verbose 'target/wheels/*' + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: arn:aws:iam::${{ secrets.CHIA_AWS_ACCOUNT_ID }}:role/installer-upload + aws-region: us-west-2 + - name: Publish Dev if: steps.check_secrets.outputs.HAS_AWS_SECRET && github.ref == 'refs/heads/dev' shell: bash working-directory: ./target/wheels - env: - AWS_ACCESS_KEY_ID: "${{ secrets.INSTALLER_UPLOAD_KEY }}" - AWS_SECRET_ACCESS_KEY: "${{ secrets.INSTALLER_UPLOAD_SECRET }}" - AWS_REGION: us-west-2 run: | FILES=$(find . -type f -name '*.whl') while IFS= read -r file; do From 64b4c53d6984abedfcdcd5422436ec0de2fa8548 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Fri, 11 Aug 2023 12:29:46 +0100 Subject: [PATCH 069/117] further simplify termlist --- src/compiler/sexp.rs | 94 +++++++++++++++----------------------------- 1 file changed, 32 insertions(+), 62 deletions(-) diff --git a/src/compiler/sexp.rs b/src/compiler/sexp.rs index 443234b97..0104c3477 100644 --- a/src/compiler/sexp.rs +++ b/src/compiler/sexp.rs @@ -680,75 +680,45 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - pp, list_content, ) => { - if this_char.is_ascii_whitespace() { - // ignore whitespace after second word - resume(SExpParseState::TermList( - srcloc.ext(&loc), - Some(parsed.clone()), - pp.clone(), - list_content.to_vec(), - )) - } else if this_char == b')' { - // if we see a `)` then we're ready to close this list - let mut list_copy = list_content.to_vec(); - match list_copy.pop() { - Some(v) => { - let new_tail = make_cons(v, parsed.clone()); - if list_copy.is_empty() { - emit(Rc::new(new_tail), SExpParseState::Empty) - } else { - let mut result_list = new_tail; - for item in list_copy.iter().rev() { - result_list = make_cons(item.clone(), Rc::new(result_list)); + match (this_char as char, pp.borrow()) { + (')', SExpParseState::Empty) => { + // if we see a `)` then we're ready to close this list + let mut list_copy = list_content.to_vec(); + match list_copy.pop() { + Some(v) => { + let new_tail = make_cons(v, parsed.clone()); + if list_copy.is_empty() { + emit(Rc::new(new_tail), SExpParseState::Empty) + } else { + let mut result_list = new_tail; + for item in list_copy.iter().rev() { + result_list = make_cons(item.clone(), Rc::new(result_list)); + } + emit(Rc::new(result_list), SExpParseState::Empty) // emit the resultant list } - emit(Rc::new(result_list), SExpParseState::Empty) // emit the resultant list } + None => error(loc, "Dot as first element of list?"), } - None => error(loc, "Dot as first element of list?"), - } - } else if this_char == b';' { - // entering a comment - // resume(SExpParseState::TermList( - // srcloc.clone(), - // TermListCommentState::InComment, - // Some(parsed.clone()), - // pp.clone(), - // list_content.clone(), - // )) - resume(SExpParseState::TermList( - loc.clone(), - Some(parsed.clone()), // assert parsed_object is not None and then store it in parsed_list - Rc::new(SExpParseState::CommentText), - list_content.clone(), - )) - } else { - match pp.as_ref() { - SExpParseState::CommentText => { - match this_char as char { - '\n' => resume(SExpParseState::TermList( - loc.clone(), - Some(parsed.clone()), - Rc::new(SExpParseState::Empty), - list_content.clone(), - )), - _ => { + }, + _ => match parse_sexp_step(loc.clone(), pp.borrow(), this_char) { + // nothing should be emitted as we're a term list with an object found + SExpParseResult::Emit(parsed_object, _current_state) => error(loc, "found object during termlist"), + // resume means it didn't finish parsing yet, so store inner state and keep going + SExpParseResult::Resume(current_state) => { + match current_state { + SExpParseState::Empty | SExpParseState::CommentText => { resume(SExpParseState::TermList( - loc.clone(), - Some(parsed.clone()), - Rc::new(SExpParseState::CommentText), - list_content.clone(), + srcloc.ext(&loc), + None, + Rc::new(current_state), // store our partial inner parsestate in pp + list_content.to_vec(), )) - } + }, + _ => error(loc, "Illegal state during term list.") } }, - _ => { - // we don't want to see any more characters after we've concluded a dot expression - error( - srcloc.clone(), - &format!("unexpected character {}", this_char as char), - ) - } - } + SExpParseResult::Error(l, e) => SExpParseResult::Error(l, e), + }, } } // we are passing a dot-expression (x . y) and not in a comment and don't have an object already discovered From 43e6745179f91b6e73eb5101c1813efa8b28eafc Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 14 Aug 2023 11:20:14 -0700 Subject: [PATCH 070/117] WIP adding some test infra --- src/compiler/sexp.rs | 198 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 164 insertions(+), 34 deletions(-) diff --git a/src/compiler/sexp.rs b/src/compiler/sexp.rs index 52bae6bef..7f46e8cbd 100644 --- a/src/compiler/sexp.rs +++ b/src/compiler/sexp.rs @@ -17,6 +17,8 @@ use num_traits::{zero, Num}; use serde::Serialize; use crate::classic::clvm::__type_compatibility__::{bi_zero, Bytes, BytesFromType}; +#[cfg(test)] +use crate::classic::clvm::__type_compatibility__::bi_one; use crate::classic::clvm::casts::{bigint_from_bytes, bigint_to_bytes_clvm, TConvertOption}; use crate::compiler::prims::prims; use crate::compiler::srcloc::Srcloc; @@ -194,7 +196,7 @@ fn make_cons(a: Rc, b: Rc) -> SExp { SExp::Cons(a.loc().ext(&b.loc()), a.clone(), b.clone()) } -#[derive(Debug)] +#[derive(Debug, PartialEq, Eq)] enum SExpParseState { // The types of state that the Rust pre-forms can take Empty, @@ -212,7 +214,7 @@ enum SExpParseState { ), } -#[derive(Debug)] +#[derive(Debug, PartialEq, Eq)] enum SExpParseResult { // the result of a call to parse an SExp Resume(SExpParseState), @@ -726,9 +728,8 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - )) } else { match pp.as_ref() { - SExpParseState::CommentText(comment_loc, comment_text) => { + SExpParseState::CommentText(_, comment_text) => { match this_char as char { - '\r' => resume(SExpParseState::CommentText(comment_loc.clone(), comment_text.to_vec())), '\n' => resume(SExpParseState::TermList( loc.clone(), Some(parsed.clone()), @@ -820,58 +821,87 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - } } -fn parse_sexp_inner( - mut start: Srcloc, - mut parse_state: SExpParseState, - s: I, -) -> Result>, (Srcloc, String)> -where - I: Iterator, -{ +#[derive(Debug, PartialEq, Eq)] +pub struct ParsePartialResult { // we support compiling multiple things at once, keep these in a Vec // at the moment this will almost certainly only return 1 thing - let mut res = Vec::new(); + res: Vec>, + srcloc: Srcloc, + parse_state: SExpParseState, +} - // Loop through all the characters - for this_char in s { - let next_location = start.clone().advance(this_char); +impl ParsePartialResult { + pub fn new(srcloc: Srcloc) -> Self { + ParsePartialResult { + res: Default::default(), + srcloc: srcloc, + parse_state: SExpParseState::Empty + } + } + pub fn push( + &mut self, + this_char: u8 + ) -> Result<(), (Srcloc, String)> + { + let next_location = self.srcloc.clone().advance(this_char); // call parse_sexp_step for current character // it will return a ParseResult which contains the new ParseState - match parse_sexp_step(start.clone(), &parse_state, this_char) { + match parse_sexp_step(self.srcloc.clone(), &self.parse_state, this_char) { // catch error and propagate itupwards SExpParseResult::Error(l, e) => { return Err((l, e)); } // Keep parsing SExpParseResult::Resume(new_parse_state) => { - start = next_location; - parse_state = new_parse_state; + self.srcloc = next_location; + self.parse_state = new_parse_state; } // End of list (top level compile object), but not necessarily end of file SExpParseResult::Emit(o, new_parse_state) => { - start = next_location; - parse_state = new_parse_state; - res.push(o); + self.srcloc = next_location; + self.parse_state = new_parse_state; + self.res.push(o); } } + + Ok(()) } - // depending on the state when we finished return Ok or Err enums - match parse_state { - SExpParseState::Empty => Ok(res), - SExpParseState::Bareword(l, t) => Ok(vec![Rc::new(make_atom(l, t))]), - SExpParseState::CommentText(_, _) => Ok(res), - SExpParseState::QuotedText(l, _, _) => Err((l, "unterminated quoted string".to_string())), - SExpParseState::QuotedEscaped(l, _, _) => { - Err((l, "unterminated quoted string with escape".to_string())) + pub fn finalize(self) -> Result>, (Srcloc, String)> { + // depending on the state when we finished return Ok or Err enums + match self.parse_state { + SExpParseState::Empty => Ok(self.res), + SExpParseState::Bareword(l, t) => Ok(vec![Rc::new(make_atom(l, t))]), + SExpParseState::CommentText(_, _) => Ok(self.res), + SExpParseState::QuotedText(l, _, _) => Err((l, "unterminated quoted string".to_string())), + SExpParseState::QuotedEscaped(l, _, _) => { + Err((l, "unterminated quoted string with escape".to_string())) + } + SExpParseState::OpenList(l) => Err((l, "Unterminated list (empty)".to_string())), + SExpParseState::ParsingList(l, _, _) => Err((l, "Unterminated mid list".to_string())), + SExpParseState::TermList(l, _, _, _) => Err((l, "Unterminated tail list".to_string())), } - SExpParseState::OpenList(l) => Err((l, "Unterminated list (empty)".to_string())), - SExpParseState::ParsingList(l, _, _) => Err((l, "Unterminated mid list".to_string())), - SExpParseState::TermList(l, _, _, _) => Err((l, "Unterminated tail list".to_string())), } } +fn parse_sexp_inner( + start: Srcloc, + s: I, +) -> Result>, (Srcloc, String)> +where + I: Iterator, +{ + let mut partial_result = ParsePartialResult::new(start); + + // Loop through all the characters + for this_char in s { + partial_result.push(this_char)?; + } + + partial_result.finalize() +} + /// /// Entrypoint for parsing chialisp input. /// Called from compiler.rs @@ -882,5 +912,105 @@ pub fn parse_sexp(start: Srcloc, input: I) -> Result>, (Srcloc, where I: Iterator, { - parse_sexp_inner(start, SExpParseState::Empty, input) + parse_sexp_inner(start, input) +} + +#[cfg(test)] +fn check_parser_for_intermediate_result(parser: &mut ParsePartialResult, s: &str, desired: SExpParseState) { + for this_char in s.bytes() { + parser.push(this_char).unwrap(); + } + assert_eq!(parser.parse_state, desired); +} + +#[cfg(test)] +fn srcloc_range(name: &Rc, start: usize, end: usize) -> Srcloc { + Srcloc::new(name.clone(), 1, start).ext(&Srcloc::new(name.clone(), 1, end)) +} + +#[test] +fn test_tricky_parser_tail_01() { + let testname = Rc::new("*test*".to_string()); + let loc = Srcloc::start(&testname); + let mut parser = ParsePartialResult::new(loc.clone()); + check_parser_for_intermediate_result( + &mut parser, + "(1 . x", + SExpParseState::TermList( + srcloc_range(&testname, 1, 6), + None, + Rc::new(SExpParseState::Bareword(srcloc_range(&testname, 6, 6), vec![b'x'])), + vec![Rc::new(SExp::Integer(srcloc_range(&testname, 2, 3), bi_one()))] + ) + ); + + parser.push(b')').expect("should complete"); + assert_eq!( + parser.finalize(), + Ok(vec![ + Rc::new(SExp::Cons( + srcloc_range(&testname, 1, 7), + Rc::new(SExp::Integer(srcloc_range(&testname, 2, 3), bi_one())), + Rc::new(SExp::Atom(srcloc_range(&testname, 6, 7), b"x".to_vec())) + )) + ]) + ); +} + +#[test] +fn test_tricky_parser_tail_02() { + let testname = Rc::new("*test*".to_string()); + let loc = Srcloc::start(&testname); + let mut parser = ParsePartialResult::new(loc.clone()); + check_parser_for_intermediate_result( + &mut parser, + "(1 . ()", + SExpParseState::TermList( + srcloc_range(&testname, 7, 7), + Some(Rc::new(SExp::Nil(srcloc_range(&testname, 6, 7)))), + Rc::new(SExpParseState::Empty), + vec![Rc::new(SExp::Integer(srcloc_range(&testname, 2, 2), bi_one()))] + ) + ); + + parser.push(b')').expect("should complete"); + assert_eq!( + parser.finalize(), + Ok(vec![ + Rc::new(SExp::Cons( + srcloc_range(&testname, 1, 7), + Rc::new(SExp::Integer(srcloc_range(&testname, 2, 3), bi_one())), + Rc::new(SExp::Nil(srcloc_range(&testname, 6, 7))) + )) + ]) + ); +} + +#[test] +fn test_tricky_parser_tail_03() { + let testname = Rc::new("*test*".to_string()); + let loc = Srcloc::start(&testname); + let mut parser = ParsePartialResult::new(loc.clone()); + check_parser_for_intermediate_result( + &mut parser, + "(1 . () ;; Test\n", + SExpParseState::TermList( + srcloc_range(&testname, 7, 7), + Some(Rc::new(SExp::Nil(srcloc_range(&testname, 6, 7)))), + Rc::new(SExpParseState::Empty), + vec![Rc::new(SExp::Integer(srcloc_range(&testname, 2, 2), bi_one()))] + ) + ); + + parser.push(b')').expect("should complete"); + assert_eq!( + parser.finalize(), + Ok(vec![ + Rc::new(SExp::Cons( + srcloc_range(&testname, 1, 7), + Rc::new(SExp::Integer(srcloc_range(&testname, 2, 3), bi_one())), + Rc::new(SExp::Nil(srcloc_range(&testname, 6, 7))) + )) + ]) + ); } From d609b57c9dbcbc7e0639af48f7c3988f9d804c10 Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 14 Aug 2023 11:32:28 -0700 Subject: [PATCH 071/117] fmt + clippy --- src/compiler/sexp.rs | 117 ++++++++++++++++++++++--------------------- 1 file changed, 59 insertions(+), 58 deletions(-) diff --git a/src/compiler/sexp.rs b/src/compiler/sexp.rs index de3ea4392..3e0f4037c 100644 --- a/src/compiler/sexp.rs +++ b/src/compiler/sexp.rs @@ -16,9 +16,9 @@ use num_traits::{zero, Num}; use serde::Serialize; -use crate::classic::clvm::__type_compatibility__::{bi_zero, Bytes, BytesFromType}; #[cfg(test)] use crate::classic::clvm::__type_compatibility__::bi_one; +use crate::classic::clvm::__type_compatibility__::{bi_zero, Bytes, BytesFromType}; use crate::classic::clvm::casts::{bigint_from_bytes, bigint_to_bytes_clvm, TConvertOption}; use crate::compiler::prims::prims; use crate::compiler::srcloc::Srcloc; @@ -210,7 +210,7 @@ enum SExpParseState { Srcloc, Option>, // this is the second value in the dot expression Rc, // used for inner parsing - Vec>, // list content + Vec>, // list content ), } @@ -557,9 +557,7 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - // t is a Vec of the previous characters in this comment string SExpParseState::CommentText => match this_char as char { '\n' => resume(SExpParseState::Empty), - _ => { - resume(SExpParseState::CommentText) - } + _ => resume(SExpParseState::CommentText), }, // we currently processing a new word SExpParseState::Bareword(srcloc, word_so_far) => { @@ -676,12 +674,7 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - } // if we're not in a comment and have already found a parsed second word for this dot expression - SExpParseState::TermList( - srcloc, - Some(parsed), - pp, - list_content, - ) => { + SExpParseState::TermList(srcloc, Some(parsed), pp, list_content) => { match (this_char as char, pp.borrow()) { (')', SExpParseState::Empty) => { // if we see a `)` then we're ready to close this list @@ -696,15 +689,18 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - for item in list_copy.iter().rev() { result_list = make_cons(item.clone(), Rc::new(result_list)); } - emit(Rc::new(result_list), SExpParseState::Empty) // emit the resultant list + emit(Rc::new(result_list), SExpParseState::Empty) + // emit the resultant list } } None => error(loc, "Dot as first element of list?"), } - }, + } _ => match parse_sexp_step(loc.clone(), pp.borrow(), this_char) { // nothing should be emitted as we're a term list with an object found - SExpParseResult::Emit(_, _current_state) => error(loc, "found object during termlist"), + SExpParseResult::Emit(_, _current_state) => { + error(loc, "found object during termlist") + } // resume means it didn't finish parsing yet, so store inner state and keep going SExpParseResult::Resume(current_state) => { match current_state { @@ -715,10 +711,10 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - Rc::new(current_state), // store our partial inner parsestate in pp list_content.to_vec(), )) - }, - _ => error(loc, "Illegal state during term list.") + } + _ => error(loc, "Illegal state during term list."), } - }, + } SExpParseResult::Error(l, e) => SExpParseResult::Error(l, e), }, } @@ -799,15 +795,11 @@ impl ParsePartialResult { pub fn new(srcloc: Srcloc) -> Self { ParsePartialResult { res: Default::default(), - srcloc: srcloc, - parse_state: SExpParseState::Empty + srcloc, + parse_state: SExpParseState::Empty, } } - pub fn push( - &mut self, - this_char: u8 - ) -> Result<(), (Srcloc, String)> - { + pub fn push(&mut self, this_char: u8) -> Result<(), (Srcloc, String)> { let next_location = self.srcloc.clone().advance(this_char); // call parse_sexp_step for current character @@ -839,7 +831,9 @@ impl ParsePartialResult { SExpParseState::Empty => Ok(self.res), SExpParseState::Bareword(l, t) => Ok(vec![Rc::new(make_atom(l, t))]), SExpParseState::CommentText => Ok(self.res), - SExpParseState::QuotedText(l, _, _) => Err((l, "unterminated quoted string".to_string())), + SExpParseState::QuotedText(l, _, _) => { + Err((l, "unterminated quoted string".to_string())) + } SExpParseState::QuotedEscaped(l, _, _) => { Err((l, "unterminated quoted string with escape".to_string())) } @@ -850,10 +844,7 @@ impl ParsePartialResult { } } -fn parse_sexp_inner( - start: Srcloc, - s: I, -) -> Result>, (Srcloc, String)> +fn parse_sexp_inner(start: Srcloc, s: I) -> Result>, (Srcloc, String)> where I: Iterator, { @@ -881,7 +872,11 @@ where } #[cfg(test)] -fn check_parser_for_intermediate_result(parser: &mut ParsePartialResult, s: &str, desired: SExpParseState) { +fn check_parser_for_intermediate_result( + parser: &mut ParsePartialResult, + s: &str, + desired: SExpParseState, +) { for this_char in s.bytes() { parser.push(this_char).unwrap(); } @@ -904,21 +899,25 @@ fn test_tricky_parser_tail_01() { SExpParseState::TermList( srcloc_range(&testname, 1, 6), None, - Rc::new(SExpParseState::Bareword(srcloc_range(&testname, 6, 6), vec![b'x'])), - vec![Rc::new(SExp::Integer(srcloc_range(&testname, 2, 2), bi_one()))] - ) + Rc::new(SExpParseState::Bareword( + srcloc_range(&testname, 6, 6), + vec![b'x'], + )), + vec![Rc::new(SExp::Integer( + srcloc_range(&testname, 2, 2), + bi_one(), + ))], + ), ); parser.push(b')').expect("should complete"); assert_eq!( parser.finalize(), - Ok(vec![ - Rc::new(SExp::Cons( - srcloc_range(&testname, 1, 7), - Rc::new(SExp::Integer(srcloc_range(&testname, 2, 2), bi_one())), - Rc::new(SExp::Atom(srcloc_range(&testname, 6, 7), b"x".to_vec())) - )) - ]) + Ok(vec![Rc::new(SExp::Cons( + srcloc_range(&testname, 1, 7), + Rc::new(SExp::Integer(srcloc_range(&testname, 2, 2), bi_one())), + Rc::new(SExp::Atom(srcloc_range(&testname, 6, 7), b"x".to_vec())) + ))]) ); } @@ -934,20 +933,21 @@ fn test_tricky_parser_tail_02() { srcloc_range(&testname, 7, 7), Some(Rc::new(SExp::Nil(srcloc_range(&testname, 6, 7)))), Rc::new(SExpParseState::Empty), - vec![Rc::new(SExp::Integer(srcloc_range(&testname, 2, 2), bi_one()))] - ) + vec![Rc::new(SExp::Integer( + srcloc_range(&testname, 2, 2), + bi_one(), + ))], + ), ); parser.push(b')').expect("should complete"); assert_eq!( parser.finalize(), - Ok(vec![ - Rc::new(SExp::Cons( - srcloc_range(&testname, 1, 7), - Rc::new(SExp::Integer(srcloc_range(&testname, 2, 2), bi_one())), - Rc::new(SExp::Nil(srcloc_range(&testname, 6, 7))) - )) - ]) + Ok(vec![Rc::new(SExp::Cons( + srcloc_range(&testname, 1, 7), + Rc::new(SExp::Integer(srcloc_range(&testname, 2, 2), bi_one())), + Rc::new(SExp::Nil(srcloc_range(&testname, 6, 7))) + ))]) ); } @@ -963,19 +963,20 @@ fn test_tricky_parser_tail_03() { srcloc_range(&testname, 7, 16), Some(Rc::new(SExp::Nil(srcloc_range(&testname, 6, 7)))), Rc::new(SExpParseState::Empty), - vec![Rc::new(SExp::Integer(srcloc_range(&testname, 2, 2), bi_one()))] - ) + vec![Rc::new(SExp::Integer( + srcloc_range(&testname, 2, 2), + bi_one(), + ))], + ), ); parser.push(b')').expect("should complete"); assert_eq!( parser.finalize(), - Ok(vec![ - Rc::new(SExp::Cons( - srcloc_range(&testname, 1, 7), - Rc::new(SExp::Integer(srcloc_range(&testname, 2, 2), bi_one())), - Rc::new(SExp::Nil(srcloc_range(&testname, 6, 7))) - )) - ]) + Ok(vec![Rc::new(SExp::Cons( + srcloc_range(&testname, 1, 7), + Rc::new(SExp::Integer(srcloc_range(&testname, 2, 2), bi_one())), + Rc::new(SExp::Nil(srcloc_range(&testname, 6, 7))) + ))]) ); } From 14e22881790e04160f4d26921dbde0effcd64c33 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Mon, 14 Aug 2023 22:38:14 +0100 Subject: [PATCH 072/117] use rc::clone where appropriate --- src/compiler/sexp.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/compiler/sexp.rs b/src/compiler/sexp.rs index 3e0f4037c..a40a10586 100644 --- a/src/compiler/sexp.rs +++ b/src/compiler/sexp.rs @@ -193,7 +193,7 @@ impl Hash for SExp { } fn make_cons(a: Rc, b: Rc) -> SExp { - SExp::Cons(a.loc().ext(&b.loc()), a.clone(), b.clone()) + SExp::Cons(a.loc().ext(&b.loc()), Rc::clone(&a), Rc::clone(&b)) } #[derive(Debug, PartialEq, Eq)] @@ -308,7 +308,7 @@ pub fn enlist(l: Srcloc, v: &[Rc]) -> SExp { let mut result = SExp::Nil(l); for i_reverse in 0..v.len() { let i = v.len() - i_reverse - 1; - result = make_cons(v[i].clone(), Rc::new(result)); + result = make_cons(Rc::clone(&v[i]), Rc::new(result)); } result } @@ -416,14 +416,14 @@ impl SExp { pub fn cons_fst(&self) -> Rc { match self { - SExp::Cons(_, a, _) => a.clone(), + SExp::Cons(_, a, _) => Rc::clone(a), _ => Rc::new(SExp::Nil(self.loc())), } } pub fn cons_snd(&self) -> Rc { match self { - SExp::Cons(_, _, b) => b.clone(), + SExp::Cons(_, _, b) => Rc::clone(b), _ => Rc::new(SExp::Nil(self.loc())), } } @@ -671,7 +671,7 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - SExpParseResult::Error(l, e) => SExpParseResult::Error(l, e), // propagate error upwards }, } - } + }, // if we're not in a comment and have already found a parsed second word for this dot expression SExpParseState::TermList(srcloc, Some(parsed), pp, list_content) => { @@ -681,13 +681,13 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - let mut list_copy = list_content.to_vec(); match list_copy.pop() { Some(v) => { - let new_tail = make_cons(v, parsed.clone()); + let new_tail = make_cons(v, Rc::clone(&parsed)); if list_copy.is_empty() { emit(Rc::new(new_tail), SExpParseState::Empty) } else { let mut result_list = new_tail; for item in list_copy.iter().rev() { - result_list = make_cons(item.clone(), Rc::new(result_list)); + result_list = make_cons(Rc::clone(item), Rc::new(result_list)); } emit(Rc::new(result_list), SExpParseState::Empty) // emit the resultant list @@ -718,7 +718,7 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - SExpParseResult::Error(l, e) => SExpParseResult::Error(l, e), }, } - } + }, // we are passing a dot-expression (x . y) and not in a comment and don't have an object already discovered SExpParseState::TermList(srcloc, None, pp, list_content) => { // pp is the inner parsestate inside the dot-expressions @@ -731,7 +731,7 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - (')', SExpParseState::Empty) => { // attempt to close the list if list_content.len() == 1 { - emit(list_content[0].clone(), SExpParseState::Empty) + emit(Rc::clone(&list_content[0]), SExpParseState::Empty) } else { emit( Rc::new(enlist(srcloc.ext(&loc), list_content)), @@ -750,7 +750,7 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - } else { let mut result_list = new_tail; for item in list_copy.iter().rev() { - result_list = make_cons(item.clone(), Rc::new(result_list)); + result_list = make_cons(Rc::clone(item), Rc::new(result_list)); } emit(Rc::new(result_list), SExpParseState::Empty) } From 3a7bd43368cada5f62c8cd739d450699e931648d Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 14 Aug 2023 16:30:03 -0700 Subject: [PATCH 073/117] Pull in fixes from nightly --- src/compiler/codegen.rs | 22 +++++++++++++--------- src/compiler/frontend.rs | 15 +++++++++++++-- src/compiler/rename.rs | 37 ++++++++++++++++++++++++++++++++++++- 3 files changed, 62 insertions(+), 12 deletions(-) diff --git a/src/compiler/codegen.rs b/src/compiler/codegen.rs index aa69d20db..29c3cba8b 100644 --- a/src/compiler/codegen.rs +++ b/src/compiler/codegen.rs @@ -29,7 +29,7 @@ use crate::compiler::prims::{primapply, primcons, primquote}; use crate::compiler::runtypes::RunFailure; use crate::compiler::sexp::{decode_string, SExp}; use crate::compiler::srcloc::Srcloc; -use crate::util::{toposort, u8_from_number}; +use crate::util::{TopoSortItem, toposort, u8_from_number}; const MACRO_TIME_LIMIT: usize = 1000000; const CONST_EVAL_LIMIT: usize = 1000000; @@ -829,14 +829,14 @@ fn generate_let_args(_l: Srcloc, blist: Vec>) -> Vec> { blist.iter().map(|b| b.body.clone()).collect() } -pub fn hoist_assign_form(letdata: &LetData) -> Result { +pub fn toposort_assign_bindings( + loc: &Srcloc, + bindings: &[Rc], +) -> Result>>, CompileErr> { // Topological sort of bindings. - let sorted_spec = toposort( - &letdata.bindings, - CompileErr( - letdata.loc.clone(), - "deadlock resolving binding order".to_string(), - ), + toposort( + bindings, + CompileErr(loc.clone(), "deadlock resolving binding order".to_string()), // Needs: What this binding relies on. |possible, b| { let mut need_set = HashSet::new(); @@ -856,7 +856,11 @@ pub fn hoist_assign_form(letdata: &LetData) -> Result { result_set } }, - )?; + ) +} + +pub fn hoist_assign_form(letdata: &LetData) -> Result { + let sorted_spec = toposort_assign_bindings(&letdata.loc, &letdata.bindings)?; // Break up into stages of parallel let forms. // Track the needed bindings of this level. diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index a1526ea54..bf919e952 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -10,7 +10,7 @@ use crate::compiler::comptypes::{ LetData, LetFormInlineHint, LetFormKind, ModAccum, }; use crate::compiler::preprocessor::preprocess; -use crate::compiler::rename::rename_children_compileform; +use crate::compiler::rename::{rename_assign_bindings, rename_children_compileform}; use crate::compiler::sexp::{decode_string, enlist, SExp}; use crate::compiler::srcloc::Srcloc; use crate::util::u8_from_number; @@ -293,6 +293,10 @@ pub fn make_provides_set(provides_set: &mut HashSet>, body_sexp: Rc) -> bool { + opts.dialect().stepping.unwrap_or(0) > 22 +} + fn handle_assign_form( opts: Rc, l: Srcloc, @@ -336,12 +340,19 @@ fn handle_assign_form( })); } - let compiled_body = compile_bodyform(opts, Rc::new(v[v.len() - 1].clone()))?; + let mut compiled_body = compile_bodyform(opts.clone(), Rc::new(v[v.len() - 1].clone()))?; // We don't need to do much if there were no bindings. if bindings.is_empty() { return Ok(compiled_body); } + if at_or_above_23(opts.clone()) { + let (new_compiled_body, new_bindings) = + rename_assign_bindings(&l, &bindings, Rc::new(compiled_body))?; + compiled_body = new_compiled_body; + bindings = new_bindings; + }; + // Return a precise representation of this assign while storing up the work // we did breaking it down. Ok(BodyForm::Let( diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index 96433ee3f..d72fb0827 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -3,11 +3,13 @@ use std::collections::HashMap; use std::rc::Rc; use crate::compiler::comptypes::{ - Binding, BindingPattern, BodyForm, CompileForm, DefconstData, DefmacData, DefunData, + Binding, BindingPattern, BodyForm, CompileErr, CompileForm, DefconstData, DefmacData, DefunData, HelperForm, LetData, LetFormKind, }; +use crate::compiler::codegen::toposort_assign_bindings; use crate::compiler::gensym::gensym; use crate::compiler::sexp::SExp; +use crate::compiler::srcloc::Srcloc; /// Rename in a qq form. This searches for (unquote ...) forms inside and performs /// rename inside them, leaving the rest of the qq form as is. @@ -154,6 +156,39 @@ fn make_binding_unique(b: &Binding) -> InnerRenameList { } } +pub fn rename_assign_bindings( + l: &Srcloc, + bindings: &[Rc], + body: Rc, +) -> Result<(BodyForm, Vec>), CompileErr> { + // Order the bindings. + let sorted_bindings = toposort_assign_bindings(l, bindings)?; + let mut renames = HashMap::new(); + let renamed_bindings = sorted_bindings + .iter() + .rev() + .map(|item| { + let b: &Binding = bindings[item.index].borrow(); + if let BindingPattern::Complex(p) = &b.pattern { + let new_names = invent_new_names_sexp(p.clone()); + for (name, renamed) in new_names.iter() { + renames.insert(name.clone(), renamed.clone()); + } + Binding { + pattern: BindingPattern::Complex(rename_in_cons(&renames, p.clone(), false)), + body: Rc::new(rename_in_bodyform(&renames, b.body.clone())), + ..b.clone() + } + } else { + b.clone() + } + }) + .rev() + .map(Rc::new) + .collect(); + Ok((rename_in_bodyform(&renames, body), renamed_bindings)) +} + fn rename_in_bodyform(namemap: &HashMap, Vec>, b: Rc) -> BodyForm { match b.borrow() { BodyForm::Let(kind, letdata) => { From 2f2d70e34505304ef5a9b687f82c2196069ac3c3 Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 14 Aug 2023 16:43:11 -0700 Subject: [PATCH 074/117] fmt + clippy --- src/compiler/codegen.rs | 2 +- src/compiler/rename.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/compiler/codegen.rs b/src/compiler/codegen.rs index 29c3cba8b..a331a2ba2 100644 --- a/src/compiler/codegen.rs +++ b/src/compiler/codegen.rs @@ -29,7 +29,7 @@ use crate::compiler::prims::{primapply, primcons, primquote}; use crate::compiler::runtypes::RunFailure; use crate::compiler::sexp::{decode_string, SExp}; use crate::compiler::srcloc::Srcloc; -use crate::util::{TopoSortItem, toposort, u8_from_number}; +use crate::util::{toposort, u8_from_number, TopoSortItem}; const MACRO_TIME_LIMIT: usize = 1000000; const CONST_EVAL_LIMIT: usize = 1000000; diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index d72fb0827..ecbde85a1 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -2,11 +2,11 @@ use std::borrow::Borrow; use std::collections::HashMap; use std::rc::Rc; +use crate::compiler::codegen::toposort_assign_bindings; use crate::compiler::comptypes::{ - Binding, BindingPattern, BodyForm, CompileErr, CompileForm, DefconstData, DefmacData, DefunData, - HelperForm, LetData, LetFormKind, + Binding, BindingPattern, BodyForm, CompileErr, CompileForm, DefconstData, DefmacData, + DefunData, HelperForm, LetData, LetFormKind, }; -use crate::compiler::codegen::toposort_assign_bindings; use crate::compiler::gensym::gensym; use crate::compiler::sexp::SExp; use crate::compiler::srcloc::Srcloc; From f5005e9a044ff9d6d3917cdd1b437fc54f651ca1 Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 14 Aug 2023 19:46:55 -0700 Subject: [PATCH 075/117] Add commentary and do some light refactoring --- src/compiler/codegen.rs | 66 +++++++++++++++++++++++++++++++++++++++ src/compiler/comptypes.rs | 25 +++++++++------ 2 files changed, 81 insertions(+), 10 deletions(-) diff --git a/src/compiler/codegen.rs b/src/compiler/codegen.rs index a331a2ba2..fdf651b3e 100644 --- a/src/compiler/codegen.rs +++ b/src/compiler/codegen.rs @@ -793,6 +793,11 @@ fn generate_let_defun( kwl: Option, name: &[u8], args: Rc, + // Tells what the user's preference is for inlining. It can be set to None, + // which means use the form's default. + // Some(LetFormInlineHint::NoPreference), meaning the system should choose the + // best inlining strategy, + // Some(LetFormInlineHint::Inline(_)) or Some(LetFormInlineHint::NonInline(_)) inline_hint: &Option, bindings: Vec>, body: Rc, @@ -800,7 +805,10 @@ fn generate_let_defun( let new_arguments: Vec> = bindings .iter() .map(|b| match &b.pattern { + // This is the classic let form. It doesn't support destructuring. BindingPattern::Name(name) => Rc::new(SExp::Atom(l.clone(), name.clone())), + // The assign form, which supports destructuring and signals newer + // handling. BindingPattern::Complex(sexp) => sexp.clone(), }) .collect(); @@ -812,6 +820,9 @@ fn generate_let_defun( )); HelperForm::Defun( + // Some forms will be inlined and some as separate functions based on + // binary size, when permitted. Sometimes the user will signal a + // preference. should_inline_let(inline_hint), DefunData { loc: l.clone(), @@ -829,6 +840,38 @@ fn generate_let_args(_l: Srcloc, blist: Vec>) -> Vec> { blist.iter().map(|b| b.body.clone()).collect() } +/// Assign arranges its variable names via need and split into batches that don't +/// add additional dependencies. To illustrate: +/// +/// (assign +/// (X . Y) (F A W) +/// W (G A) +/// Z (H X Y W) +/// Next (H2 X Y W) +/// (doit Y Next) +/// +/// In this case, we have the following dependencies: +/// W depends on A (external) +/// X and Y depend on A (external) and W +/// Z depends on X Y and W +/// Next depends on X Y and W +/// The body depends on Y and Next. +/// +/// So we sort this: +/// W (G A) +/// --- X and Y add a dependency on W --- +/// (X . Y) (F A W) +/// --- Z and Next depend on X Y and W +/// Z (H X Y W) +/// Next (H2 X Y W) +/// --- done sorting, the body has access to all bindings --- +/// +/// We return TopoSortItem> (bytewise names), which is used in the +/// generic toposort function in util. +/// +/// This is used by facilities that need to know the order of the assignments. +/// +/// A good number of languages support reorderable assignment (haskell, elm). pub fn toposort_assign_bindings( loc: &Srcloc, bindings: &[Rc], @@ -859,6 +902,15 @@ pub fn toposort_assign_bindings( ) } +/// Let forms are "hoisted" (promoted) from being body forms to being functions +/// in the program (either defun or defun-inline). The arguments given are bound +/// in the downstream code, allowing the code generator to re-use functions to +/// allow the inner body forms to use the variable names defined in the assign. +/// This is isolated here from hoist_body_let_binding because it has its own +/// complexity that's separate from the original let features. +/// +/// In the future, things such as lambdas will also desugar along these same +/// routes. pub fn hoist_assign_form(letdata: &LetData) -> Result { let sorted_spec = toposort_assign_bindings(&letdata.loc, &letdata.bindings)?; @@ -902,6 +954,7 @@ pub fn hoist_assign_form(letdata: &LetData) -> Result { let mut end_bindings = Vec::new(); swap(&mut end_bindings, &mut binding_lists[0]); + // build a stack of let forms starting with the inner most bindings. let mut output_let = BodyForm::Let( LetFormKind::Parallel, Box::new(LetData { @@ -910,6 +963,7 @@ pub fn hoist_assign_form(letdata: &LetData) -> Result { }), ); + // build rest of the stack. for binding_list in binding_lists.into_iter().skip(1) { output_let = BodyForm::Let( LetFormKind::Parallel, @@ -924,6 +978,13 @@ pub fn hoist_assign_form(letdata: &LetData) -> Result { Ok(output_let) } +/// The main function that, when encountering something that needs to desugar to +/// a function, returns the functions that result (because things inside it may +/// also need to desugar) and rewrites the expression to incorporate that +/// function. +/// +/// We add result here in case something needs extra processing, such as assign +/// form sorting, which can fail if a workable order can't be solved. pub fn hoist_body_let_binding( outer_context: Option>, args: Rc, @@ -1021,6 +1082,7 @@ pub fn hoist_body_let_binding( let final_call = BodyForm::Call(letdata.loc.clone(), call_args, None); Ok((out_defuns, Rc::new(final_call))) } + // New alternative for assign forms. BodyForm::Let(LetFormKind::Assign, letdata) => { hoist_body_let_binding(outer_context, args, Rc::new(hoist_assign_form(letdata)?)) } @@ -1053,6 +1115,10 @@ pub fn hoist_body_let_binding( } } +/// Turn the helpers for a program into the fully desugared set of helpers for +/// that program. This expands and re-processes the helper set until all +/// desugarable body forms have been transformed to a state where no more +/// desugaring is needed. pub fn process_helper_let_bindings(helpers: &[HelperForm]) -> Result, CompileErr> { let mut result = helpers.to_owned(); let mut i = 0; diff --git a/src/compiler/comptypes.rs b/src/compiler/comptypes.rs index c27a5edc9..006ac843f 100644 --- a/src/compiler/comptypes.rs +++ b/src/compiler/comptypes.rs @@ -677,6 +677,16 @@ fn compose_assign(letdata: &LetData) -> Rc { Rc::new(enlist(letdata.loc.clone(), &result)) } +fn get_let_marker_text(kind: &LetFormKind, letdata: &LetData) -> Vec { + match (kind, letdata.inline_hint.as_ref()) { + (LetFormKind::Sequential, _) => b"let*".to_vec(), + (LetFormKind::Parallel, _) => b"let".to_vec(), + (LetFormKind::Assign, Some(LetFormInlineHint::Inline(_))) => b"assign-inline".to_vec(), + (LetFormKind::Assign, Some(LetFormInlineHint::NonInline(_))) => b"assign-lambda".to_vec(), + (LetFormKind::Assign, _) => b"assign".to_vec() + } +} + impl BodyForm { /// Get the general location of the BodyForm. pub fn loc(&self) -> Srcloc { @@ -694,17 +704,12 @@ impl BodyForm { /// afterward. pub fn to_sexp(&self) -> Rc { match self { + BodyForm::Let(LetFormKind::Assign, letdata) => { + compose_assign(letdata) + } BodyForm::Let(kind, letdata) => { - if matches!(kind, LetFormKind::Assign) { - compose_assign(letdata) - } else { - let marker = if matches!(kind, LetFormKind::Sequential) { - b"let*".to_vec() - } else { - b"let".to_vec() - }; - compose_let(&marker, letdata) - } + let marker = get_let_marker_text(kind, letdata); + compose_let(&marker, letdata) } BodyForm::Quoted(body) => Rc::new(SExp::Cons( body.loc(), From f7dac026e8d4ee5b306c0af5c3ee560987f84f14 Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 14 Aug 2023 19:49:30 -0700 Subject: [PATCH 076/117] fmt + clippy --- src/compiler/comptypes.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/compiler/comptypes.rs b/src/compiler/comptypes.rs index 006ac843f..f7dc35811 100644 --- a/src/compiler/comptypes.rs +++ b/src/compiler/comptypes.rs @@ -683,7 +683,7 @@ fn get_let_marker_text(kind: &LetFormKind, letdata: &LetData) -> Vec { (LetFormKind::Parallel, _) => b"let".to_vec(), (LetFormKind::Assign, Some(LetFormInlineHint::Inline(_))) => b"assign-inline".to_vec(), (LetFormKind::Assign, Some(LetFormInlineHint::NonInline(_))) => b"assign-lambda".to_vec(), - (LetFormKind::Assign, _) => b"assign".to_vec() + (LetFormKind::Assign, _) => b"assign".to_vec(), } } @@ -704,9 +704,7 @@ impl BodyForm { /// afterward. pub fn to_sexp(&self) -> Rc { match self { - BodyForm::Let(LetFormKind::Assign, letdata) => { - compose_assign(letdata) - } + BodyForm::Let(LetFormKind::Assign, letdata) => compose_assign(letdata), BodyForm::Let(kind, letdata) => { let marker = get_let_marker_text(kind, letdata); compose_let(&marker, letdata) From 012c212573ecccfbdb41e4c29f120905d5c06f87 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 15 Aug 2023 01:28:06 -0700 Subject: [PATCH 077/117] Add some matrix style assign tests --- .github/workflows/build-test.yml | 4 + src/tests/compiler/assign.rs | 368 +++++++++++++++++++++++++++++++ src/tests/compiler/mod.rs | 1 + src/tests/compiler/restargs.rs | 36 +++ 4 files changed, 409 insertions(+) create mode 100644 src/tests/compiler/assign.rs diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 4633cde84..906df34fb 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -208,6 +208,10 @@ jobs: if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') run: cargo test --no-default-features + - name: Exhaustive assign tests + if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') + run: cargo test -- --include-ignored assign + - name: Check coverage if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') run: | diff --git a/src/tests/compiler/assign.rs b/src/tests/compiler/assign.rs new file mode 100644 index 000000000..1f071c9f8 --- /dev/null +++ b/src/tests/compiler/assign.rs @@ -0,0 +1,368 @@ +use crate::tests::classic::run::{do_basic_brun, do_basic_run}; +use std::rc::Rc; + +trait TestCompute { + fn compute(&self, x: i64, y: i64) -> (i64, i64); +} + +#[derive(Default, Clone)] +struct ComposedComputation { + to_run: Vec>, +} +impl TestCompute for ComposedComputation { + fn compute(&self, mut x: i64, mut y: i64) -> (i64, i64) { + for entry in self.to_run.iter() { + let (new_x, new_y) = entry.compute(x, y); + x = new_x; + y = new_y; + } + (x, y) + } +} + +struct XPlus1 {} +impl TestCompute for XPlus1 { + fn compute(&self, x: i64, y: i64) -> (i64, i64) { + (x + 1, y) + } +} + +struct YPlus1 {} +impl TestCompute for YPlus1 { + fn compute(&self, x: i64, y: i64) -> (i64, i64) { + (x, y + 1) + } +} + +struct FFunc {} +impl TestCompute for FFunc { + fn compute(&self, x: i64, y: i64) -> (i64, i64) { + let divxy = x / y; + let modxy = x % y; + (divxy, modxy) + } +} + +struct GFunc {} +impl TestCompute for GFunc { + fn compute(&self, x: i64, _y: i64) -> (i64, i64) { + let v = 1 + (3 * x); + (v, v) + } +} + +struct HFunc {} +impl TestCompute for HFunc { + fn compute(&self, x: i64, y: i64) -> (i64, i64) { + let v = x * y; + (v, v) + } +} + +struct IFunc {} +impl TestCompute for IFunc { + fn compute(&self, x: i64, _y: i64) -> (i64, i64) { + (x - 1, x * 2) + } +} + +struct UseTestFunction { + name: String, + args: usize, + outputs: usize, + text: String, + compute: Rc, +} + +struct AssignTestMatrix { + functions: Vec, +} + +#[test] +#[ignore] +fn test_assign_matrix() { + let inline_kwds = vec!["assign", "assign-lambda", "assign-inline"]; + let matrix = AssignTestMatrix { + functions: vec![ + UseTestFunction { + name: "F".to_string(), + args: 2, + outputs: 2, + text: "(defun F (X Y) (divmod X Y))".to_string(), + compute: Rc::new(FFunc {}), + }, + UseTestFunction { + name: "F-inline".to_string(), + args: 2, + outputs: 2, + text: "(defun-inline F-inline (X Y) (divmod X Y))".to_string(), + compute: Rc::new(FFunc {}), + }, + UseTestFunction { + name: "G".to_string(), + args: 1, + outputs: 1, + text: "(defun G (X) (+ 1 (* 3 X)))".to_string(), + compute: Rc::new(GFunc {}), + }, + UseTestFunction { + name: "G-inline".to_string(), + args: 1, + outputs: 1, + text: "(defun-inline G-inline (X) (+ 1 (* 3 X)))".to_string(), + compute: Rc::new(GFunc {}), + }, + UseTestFunction { + name: "H".to_string(), + args: 2, + outputs: 1, + text: "(defun H (X Y) (* X Y))".to_string(), + compute: Rc::new(HFunc {}), + }, + UseTestFunction { + name: "H-inline".to_string(), + args: 2, + outputs: 1, + text: "(defun-inline H-inline (X Y) (* X Y))".to_string(), + compute: Rc::new(HFunc {}), + }, + UseTestFunction { + name: "I".to_string(), + args: 1, + outputs: 2, + text: "(defun I (X) (c (- X 1) (* X 2)))".to_string(), + compute: Rc::new(IFunc {}), + }, + UseTestFunction { + name: "I-inline".to_string(), + args: 1, + outputs: 2, + text: "(defun-inline I-inline (X) (c (- X 1) (* X 2)))".to_string(), + compute: Rc::new(IFunc {}), + }, + ], + }; + + // The program will take X, Y + + let mut starter_program = vec!["(mod (X Y) (include *standard-cl-21*) ".to_string()]; + for func in matrix.functions.iter() { + starter_program.push(func.text.clone()); + } + + let assert_program_worked = |program: &[String], to_compute: &ComposedComputation| { + let joined = program.join("\n").to_string(); + let compiled = do_basic_run(&vec!["run".to_string(), joined]); + let executed = do_basic_brun(&vec![ + "brun".to_string(), + "-n".to_string(), + compiled, + "(13 19)".to_string(), + ]); + let (ex, ey) = to_compute.compute(13, 19); + let expected = do_basic_brun(&vec![ + "brun".to_string(), + "-n".to_string(), + format!("(1 . ({ex} . {ey}))"), + ]); + assert_eq!(expected, executed); + }; + + let finish_program = |program: &mut Vec, main_or_function: usize, assign_expr: &str| { + if main_or_function == 0 { + program.push(assign_expr.to_string()); + } else { + if main_or_function == 1 { + program.push(format!("(defun Q (X Y) {})", assign_expr)); + } else { + program.push(format!("(defun-inline Q (X Y) {})", assign_expr)); + } + program.push("(Q X Y)".to_string()); + } + program.push(")".to_string()); + }; + + let test_triple_nesting = + |to_compute: &ComposedComputation, main_or_function: usize, assign_expr_list: &[String]| { + let assign_expr = assign_expr_list.join("\n").to_string(); + let mut program = starter_program.clone(); + finish_program(&mut program, main_or_function, &assign_expr); + assert_program_worked(&program, &to_compute); + }; + + let test_third_level_nestings = |to_compute_x: &ComposedComputation, + fourth_var: &str, + y: &UseTestFunction, + main_or_function: usize, + assign_expr: &str, + second_assign: &str, + end_parens: &str| { + // Put on a third level on the stack. + for z in matrix.functions.iter() { + let assign_call_z = if z.args == 1 { + format!("({} V2)", z.name) + } else { + format!("({} V2 (+ 1 {fourth_var}))", z.name) + }; + + let (third_assign, sixth_var) = if z.outputs == 1 { + (format!("V4 {assign_call_z}"), "V4") + } else { + (format!("(V4 . V5) {assign_call_z}"), "V5") + }; + + let final_expr = format!("(c V4 {sixth_var}))"); + let mut to_compute = to_compute_x.clone(); + to_compute.to_run.push(y.compute.clone()); + to_compute.to_run.push(Rc::new(YPlus1 {})); + to_compute.to_run.push(z.compute.clone()); + + // Try with z in the same assign. + test_triple_nesting( + &to_compute, + main_or_function, + &[ + assign_expr.to_string(), + second_assign.to_string(), + third_assign.clone(), + final_expr.clone(), + end_parens.to_string(), + ], + ); + + // Try with z nested. + test_triple_nesting( + &to_compute, + main_or_function, + &[ + assign_expr.to_string(), + second_assign.to_string(), + "(assign".to_string(), + third_assign, + final_expr, + ")".to_string(), + end_parens.to_string(), + ], + ); + } + }; + + for x in matrix.functions.iter() { + let main_expr = if x.args == 1 { + format!("({} X)", x.name) + } else { + format!("({} X Y)", x.name) + }; + + // Depth 1. + for inline_choice in inline_kwds.iter() { + let mut to_compute_x = ComposedComputation::default(); + to_compute_x.to_run.push(x.compute.clone()); + for main_or_function in 0..=2 { + let (assign_expr, second_var) = if x.outputs == 1 { + (format!("({inline_choice} V0 {main_expr}"), "V0") + } else { + (format!("({inline_choice} (V0 . V1) {main_expr}"), "V1") + }; + + { + let mut program = starter_program.clone(); + let finished_assign_expr = + vec![assign_expr.clone(), format!("(c V0 {second_var}))")] + .join("\n") + .to_string(); + finish_program(&mut program, main_or_function, &finished_assign_expr); + assert_program_worked(&program, &to_compute_x); + } + + // Second set of assignments. + for y in matrix.functions.iter() { + // Use both arguments in one more function call. + let second_var = if x.outputs == 1 { "V0" } else { "V1" }; + let assign_call_y = if y.args == 1 { + format!("({} V0)", y.name) + } else { + format!("({} V0 {second_var})", y.name) + }; + let (second_assign, fourth_var) = if y.outputs == 1 { + (format!("V2 {assign_call_y}"), "V2") + } else { + (format!("(V2 . V3) {assign_call_y}"), "V3") + }; + + { + let assign_expr = vec![ + assign_expr.clone(), + second_assign.clone(), + format!("(c V2 {fourth_var}))"), + ] + .join("\n") + .to_string(); + let mut program = starter_program.clone(); + let mut to_compute = to_compute_x.clone(); + to_compute.to_run.push(y.compute.clone()); + finish_program(&mut program, main_or_function, &assign_expr); + assert_program_worked(&program, &to_compute); + } + + test_third_level_nestings( + &to_compute_x, + fourth_var, + y, + main_or_function, + &assign_expr, + &second_assign, + "", + ); + } + + // Nested + for y in matrix.functions.iter() { + for inline_choice_y in inline_kwds.iter() { + // Use both arguments in one more function call. + let second_var = if x.outputs == 1 { "V0" } else { "V1" }; + let assign_call_y = if y.args == 1 { + format!("({} V0)", y.name) + } else { + format!("({} V0 {second_var})", y.name) + }; + let (second_assign, fourth_var) = if y.outputs == 1 { + (format!("({inline_choice_y} V2 {assign_call_y}"), "V2") + } else { + ( + format!("({inline_choice_y} (V2 . V3) {assign_call_y}"), + "V3", + ) + }; + + { + let assign_expr = vec![ + assign_expr.clone(), + second_assign.clone(), + format!("(c V2 {fourth_var})))"), + ] + .join("\n") + .to_string(); + + let mut program = starter_program.clone(); + let mut to_compute = to_compute_x.clone(); + to_compute.to_run.push(y.compute.clone()); + finish_program(&mut program, main_or_function, &assign_expr); + assert_program_worked(&program, &to_compute); + } + + test_third_level_nestings( + &to_compute_x, + fourth_var, + y, + main_or_function, + &assign_expr, + &second_assign, + ")", + ); + } + } + } + } + } +} diff --git a/src/tests/compiler/mod.rs b/src/tests/compiler/mod.rs index b0ea66ffb..74cb6aa23 100644 --- a/src/tests/compiler/mod.rs +++ b/src/tests/compiler/mod.rs @@ -4,6 +4,7 @@ use std::rc::Rc; use crate::compiler::sexp::{parse_sexp, SExp}; use crate::compiler::srcloc::{Srcloc, Until}; +mod assign; mod cldb; mod clvm; mod compiler; diff --git a/src/tests/compiler/restargs.rs b/src/tests/compiler/restargs.rs index 0bd66c842..15ea7e930 100644 --- a/src/tests/compiler/restargs.rs +++ b/src/tests/compiler/restargs.rs @@ -936,6 +936,42 @@ fn test_compiler_tail_let_ni() { assert_eq!(res.to_string(), "(5 7 8)"); } +#[test] +fn test_compiler_tail_assign_ni() { + let prog = indoc! {" +(mod (X Y) + (include *standard-cl-21*) + + (defun F (A B C) (list A B C)) + + (defun G (X Y) (F X &rest (assign Q (+ Y 1) (list Y Q)))) + + (G X Y) + )"} + .to_string(); + + let res = run_string(&prog, &"(5 7)".to_string()).expect("should compile and run"); + assert_eq!(res.to_string(), "(5 7 8)"); +} + +#[test] +fn test_compiler_tail_assign_inline() { + let prog = indoc! {" +(mod (X Y) + (include *standard-cl-21*) + + (defun F (A B C) (list A B C)) + + (defun-inline G (X Y) (F X &rest (assign Q (+ Y 1) (list Y Q)))) + + (G X Y) + )"} + .to_string(); + + let res = run_string(&prog, &"(5 7)".to_string()).expect("should compile and run"); + assert_eq!(res.to_string(), "(5 7 8)"); +} + #[test] fn test_repl_tail_let() { assert_eq!( From fc74dffb0d6939fcd3ecf29c5385606c3391f751 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 15 Aug 2023 02:15:53 -0700 Subject: [PATCH 078/117] Exclude tests from coverage now that we have an ignored test --- resources/coverage/run_coverage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/coverage/run_coverage.py b/resources/coverage/run_coverage.py index 5399cf0c7..4c6b48b06 100644 --- a/resources/coverage/run_coverage.py +++ b/resources/coverage/run_coverage.py @@ -62,7 +62,7 @@ def collect_coverage(): required_percentage = int(args.require_percent) has_required_pct = True - for file in result: + for file in filter(lambda f: '/tests/' not in f['name'], result): have_pct = int(file['coveragePercent']) if have_pct < required_percentage: print(f"{file['name']} lacks required coverage have {have_pct} want {required_percentage}") From ca2dca73db562fc44cc1ff4796e1deb38f7cb31d Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Tue, 15 Aug 2023 14:14:20 +0100 Subject: [PATCH 079/117] cargo fmt --- src/compiler/sexp.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/compiler/sexp.rs b/src/compiler/sexp.rs index a40a10586..6b6e05e61 100644 --- a/src/compiler/sexp.rs +++ b/src/compiler/sexp.rs @@ -671,7 +671,7 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - SExpParseResult::Error(l, e) => SExpParseResult::Error(l, e), // propagate error upwards }, } - }, + } // if we're not in a comment and have already found a parsed second word for this dot expression SExpParseState::TermList(srcloc, Some(parsed), pp, list_content) => { @@ -718,7 +718,7 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - SExpParseResult::Error(l, e) => SExpParseResult::Error(l, e), }, } - }, + } // we are passing a dot-expression (x . y) and not in a comment and don't have an object already discovered SExpParseState::TermList(srcloc, None, pp, list_content) => { // pp is the inner parsestate inside the dot-expressions From a42b510813ebebbd313c0e499e87b100257655f9 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Tue, 15 Aug 2023 14:29:41 +0100 Subject: [PATCH 080/117] fix build issue --- src/compiler/sexp.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/sexp.rs b/src/compiler/sexp.rs index 6b6e05e61..58d01ad5b 100644 --- a/src/compiler/sexp.rs +++ b/src/compiler/sexp.rs @@ -681,7 +681,7 @@ fn parse_sexp_step(loc: Srcloc, current_state: &SExpParseState, this_char: u8) - let mut list_copy = list_content.to_vec(); match list_copy.pop() { Some(v) => { - let new_tail = make_cons(v, Rc::clone(&parsed)); + let new_tail = make_cons(v, Rc::clone(parsed)); if list_copy.is_empty() { emit(Rc::new(new_tail), SExpParseState::Empty) } else { From c4840e44302b0121b61d1587bed8d47fd60f7c55 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 15 Aug 2023 13:50:55 -0700 Subject: [PATCH 081/117] Shrink the stack a bit by putting LetData in Box. Accomodates things needed by the assign and lambda prs. --- src/compiler/codegen.rs | 8 ++++---- src/compiler/comptypes.rs | 2 +- src/compiler/evaluate.rs | 4 ++-- src/compiler/frontend.rs | 4 ++-- src/compiler/rename.rs | 12 ++++++------ 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/compiler/codegen.rs b/src/compiler/codegen.rs index 9044f1031..a290b1558 100644 --- a/src/compiler/codegen.rs +++ b/src/compiler/codegen.rs @@ -839,12 +839,12 @@ pub fn hoist_body_let_binding( let sub_bindings = letdata.bindings.iter().skip(1).cloned().collect(); Rc::new(BodyForm::Let( LetFormKind::Sequential, - LetData { + Box::new(LetData { loc: letdata.loc.clone(), kw: letdata.kw.clone(), bindings: sub_bindings, body: letdata.body.clone(), - }, + }), )) }; @@ -853,12 +853,12 @@ pub fn hoist_body_let_binding( args, Rc::new(BodyForm::Let( LetFormKind::Parallel, - LetData { + Box::new(LetData { loc: letdata.loc.clone(), kw: letdata.kw.clone(), bindings: vec![letdata.bindings[0].clone()], body: new_sub_expr, - }, + }), )), ) } diff --git a/src/compiler/comptypes.rs b/src/compiler/comptypes.rs index f38ff8321..9f715270d 100644 --- a/src/compiler/comptypes.rs +++ b/src/compiler/comptypes.rs @@ -124,7 +124,7 @@ pub struct LetData { #[derive(Clone, Debug, Serialize)] pub enum BodyForm { /// A let or let* form (depending on LetFormKind). - Let(LetFormKind, LetData), + Let(LetFormKind, Box), /// An explicitly quoted constant of some kind. Quoted(SExp), /// An undiferentiated "value" of some kind in the source language. diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index 2eec38783..6f2800068 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -1025,12 +1025,12 @@ impl<'info> Evaluator { &updated_bindings, Rc::new(BodyForm::Let( LetFormKind::Sequential, - LetData { + Box::new(LetData { loc: letdata.loc.clone(), kw: letdata.kw.clone(), bindings: rest_of_bindings, body: letdata.body.clone(), - }, + }), )), only_inline, ) diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index 8452a7deb..efbbadc7d 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -342,12 +342,12 @@ pub fn compile_bodyform( let compiled_body = compile_bodyform(opts, Rc::new(body))?; Ok(BodyForm::Let( kind, - LetData { + Box::new(LetData { loc: l.clone(), kw: Some(l.clone()), bindings: let_bindings, body: Rc::new(compiled_body), - }, + }), )) } else if *atom_name == "quote".as_bytes().to_vec() { if v.len() != 1 { diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index 759e32f40..2c075cce8 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -136,12 +136,12 @@ fn rename_in_bodyform(namemap: &HashMap, Vec>, b: Rc) -> B let new_body = rename_in_bodyform(namemap, letdata.body.clone()); BodyForm::Let( kind.clone(), - LetData { + Box::new(LetData { loc: letdata.loc.clone(), kw: letdata.kw.clone(), bindings: new_bindings, body: Rc::new(new_body), - }, + }), ) } @@ -189,12 +189,12 @@ pub fn desugar_sequential_let_bindings( bindings, &BodyForm::Let( LetFormKind::Parallel, - LetData { + Box::new(LetData { loc: want_binding.loc(), kw: None, bindings: vec![want_binding], body: Rc::new(body.clone()), - }, + }), ), n - 1, ) @@ -240,12 +240,12 @@ fn rename_args_bodyform(b: &BodyForm) -> BodyForm { let locally_renamed_body = rename_in_bodyform(&local_namemap, letdata.body.clone()); BodyForm::Let( LetFormKind::Parallel, - LetData { + Box::new(LetData { loc: letdata.loc.clone(), kw: letdata.kw.clone(), bindings: new_bindings, body: Rc::new(locally_renamed_body), - }, + }), ) } From b07a31f1d44a0689d42ab55f28d9838e63d58ad4 Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 16 Aug 2023 10:24:54 -0700 Subject: [PATCH 082/117] Opps remove merge marks --- src/compiler/rename.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index 4c0cf9ee1..ecbde85a1 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -208,16 +208,9 @@ fn rename_in_bodyform(namemap: &HashMap, Vec>, b: Rc) -> B BodyForm::Let( kind.clone(), Box::new(LetData { -<<<<<<< HEAD bindings: new_bindings, body: Rc::new(new_body), ..*letdata.clone() -======= - loc: letdata.loc.clone(), - kw: letdata.kw.clone(), - bindings: new_bindings, - body: Rc::new(new_body), ->>>>>>> chia/base }), ) } From 42b199976c24d1d31b1252e8b2343cf8823af97d Mon Sep 17 00:00:00 2001 From: Brandon Butler Date: Thu, 17 Aug 2023 13:08:11 -0700 Subject: [PATCH 083/117] Use pypi trusted providers --- .github/workflows/build-arm64-wheels.yml | 43 ++++++------- .github/workflows/build-m1-wheel.yml | 78 ++++++++++++------------ .github/workflows/build-test.yml | 36 +++++------ 3 files changed, 81 insertions(+), 76 deletions(-) diff --git a/.github/workflows/build-arm64-wheels.yml b/.github/workflows/build-arm64-wheels.yml index 0618633af..b0830383b 100644 --- a/.github/workflows/build-arm64-wheels.yml +++ b/.github/workflows/build-arm64-wheels.yml @@ -65,7 +65,7 @@ jobs: name: wheels path: target/wheels/ - - name: Install Twine + - name: Install job deps run: | if [ ! -f "venv" ]; then rm -rf venv; fi sudo apt install python3 python3-pip -y @@ -73,7 +73,6 @@ jobs: if [ ! -f "activate" ]; then ln -s venv/bin/activate; fi . ./activate pip3 install setuptools_rust - pip3 install twine - name: Test for secrets access id: check_secrets shell: bash @@ -82,35 +81,37 @@ jobs: unset HAS_AWS_SECRET if [ -n "$SECRET" ]; then HAS_SECRET='true' ; fi - echo "HAS_SECRET=${HAS_SECRET}" >>$GITHUB_OUTPUT + echo HAS_SECRET=${HAS_SECRET} >>$GITHUB_OUTPUT if [ -n "$AWS_SECRET" ]; then HAS_AWS_SECRET='true' ; fi echo HAS_AWS_SECRET=${HAS_AWS_SECRET} >>$GITHUB_OUTPUT env: - SECRET: "${{ secrets.test_pypi_password }}" + SECRET: "${{ secrets.GITHUB_TOKEN }}" AWS_SECRET: "${{ secrets.CHIA_AWS_ACCOUNT_ID }}" - - name: publish (PyPi) - if: startsWith(github.event.ref, 'refs/tags') && steps.check_secrets.outputs.HAS_SECRET - env: - TWINE_USERNAME: __token__ - TWINE_NON_INTERACTIVE: 1 - TWINE_PASSWORD: ${{ secrets.pypi_password }} - run: | - . ./activate - twine upload --non-interactive --skip-existing --verbose 'target/wheels/*' - - name: publish (Test PyPi) + - name: Set Env if: steps.check_secrets.outputs.HAS_SECRET + uses: Chia-Network/actions/setjobenv@main env: - TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/ - TWINE_USERNAME: __token__ - TWINE_NON_INTERACTIVE: 1 - TWINE_PASSWORD: ${{ secrets.test_pypi_password }} - run: | - . ./activate - twine upload --non-interactive --skip-existing --verbose 'target/wheels/*' + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: publish (PyPi) + if: env.RELEASE == 'true' + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: target/wheels/ + skip-existing: true + + - name: publish (Test PyPi) + if: env.PRE_RELEASE == 'true' + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + packages-dir: target/wheels/ + skip-existing: true - name: Configure AWS credentials + if: steps.check_secrets.outputs.HAS_AWS_SECRET uses: aws-actions/configure-aws-credentials@v2 with: role-to-assume: arn:aws:iam::${{ secrets.CHIA_AWS_ACCOUNT_ID }}:role/installer-upload diff --git a/.github/workflows/build-m1-wheel.yml b/.github/workflows/build-m1-wheel.yml index 1fda47663..b8dc3fa0c 100644 --- a/.github/workflows/build-m1-wheel.yml +++ b/.github/workflows/build-m1-wheel.yml @@ -22,7 +22,7 @@ permissions: jobs: build_wheels: name: Build wheel on Mac M1 - runs-on: [m1] + runs-on: [MacOS, ARM64] strategy: fail-fast: false @@ -49,43 +49,43 @@ jobs: - name: Build m1 wheels run: | - arch -arm64 python3 -m venv venv + python3 -m venv venv . ./venv/bin/activate export PATH=~/.cargo/bin:$PATH - arch -arm64 pip install maturin==1.1.0 - arch -arm64 maturin build -i python --release --strip - arch -arm64 cargo test + pip install maturin==1.1.0 + maturin build -i python --release --strip + cargo test - name: Install clvm_tools_rs wheel run: | . ./venv/bin/activate ls target/wheels/ - arch -arm64 pip install ./target/wheels/clvm_tools_rs*.whl + pip install ./target/wheels/clvm_tools_rs*.whl - name: Install other wheels run: | . ./venv/bin/activate - arch -arm64 python -m pip install pytest - arch -arm64 python -m pip install blspy + python -m pip install pytest + python -m pip install blspy - name: install clvm & clvm_tools run: | . ./venv/bin/activate - arch -arm64 git clone https://github.com/Chia-Network/clvm.git --branch=main --single-branch - arch -arm64 python -m pip install ./clvm - arch -arm64 python -m pip install clvm_rs + git clone https://github.com/Chia-Network/clvm.git --branch=main --single-branch + python -m pip install ./clvm + python -m pip install clvm_rs - arch -arm64 git clone https://github.com/Chia-Network/clvm_tools.git --branch=main --single-branch - arch -arm64 python -m pip install ./clvm_tools + git clone https://github.com/Chia-Network/clvm_tools.git --branch=main --single-branch + python -m pip install ./clvm_tools - name: Ensure clvm, clvm_rs, clvm_tools are installed run: | . ./venv/bin/activate - arch -arm64 python -c 'import clvm' - arch -arm64 python -c 'import clvm; print(clvm.__file__)' - arch -arm64 python -c 'import clvm_rs; print(clvm_rs.__file__)' - arch -arm64 python -c 'import clvm_tools; print(clvm_tools.__file__)' - arch -arm64 python -c 'import clvm_tools_rs; print(clvm_tools_rs.__file__)' + python -c 'import clvm' + python -c 'import clvm; print(clvm.__file__)' + python -c 'import clvm_rs; print(clvm_rs.__file__)' + python -c 'import clvm_tools; print(clvm_tools.__file__)' + python -c 'import clvm_tools_rs; print(clvm_tools_rs.__file__)' - name: Install pytest run: | @@ -97,13 +97,13 @@ jobs: # run: | # . ./venv/bin/activate # cd clvm -# arch -arm64 python -m py.test tests +# python -m py.test tests - name: Run tests from clvm_tools run: | . ./venv/bin/activate cd clvm_tools - arch -arm64 pytest + pytest - name: Upload wheels uses: actions/upload-artifact@v3 @@ -124,30 +124,32 @@ jobs: if [ -n "$AWS_SECRET" ]; then HAS_AWS_SECRET='true' ; fi echo HAS_AWS_SECRET=${HAS_AWS_SECRET} >>$GITHUB_OUTPUT env: - SECRET: "${{ secrets.test_pypi_password }}" + SECRET: "${{ secrets.GITHUB_TOKEN }}" AWS_SECRET: "${{ secrets.CHIA_AWS_ACCOUNT_ID }}" - - name: Install twine - run: arch -arm64 pip install twine - - - name: Publish distribution to PyPI - if: startsWith(github.event.ref, 'refs/tags') && steps.check_secrets.outputs.HAS_SECRET - env: - TWINE_USERNAME: __token__ - TWINE_NON_INTERACTIVE: 1 - TWINE_PASSWORD: ${{ secrets.pypi_password }} - run: arch -arm64 twine upload --non-interactive --skip-existing --verbose 'target/wheels/*' - - - name: Publish distribution to Test PyPI + - name: Set Env if: steps.check_secrets.outputs.HAS_SECRET + uses: Chia-Network/actions/setjobenv@main env: - TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/ - TWINE_USERNAME: __token__ - TWINE_NON_INTERACTIVE: 1 - TWINE_PASSWORD: ${{ secrets.test_pypi_password }} - run: arch -arm64 twine upload --non-interactive --skip-existing --verbose 'target/wheels/*' + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: publish (PyPi) + if: env.RELEASE == 'true' + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: target/wheels/ + skip-existing: true + + - name: publish (Test PyPi) + if: env.PRE_RELEASE == 'true' + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + packages-dir: target/wheels/ + skip-existing: true - name: Configure AWS credentials + if: steps.check_secrets.outputs.HAS_AWS_SECRET uses: aws-actions/configure-aws-credentials@v2 with: role-to-assume: arn:aws:iam::${{ secrets.CHIA_AWS_ACCOUNT_ID }}:role/installer-upload diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 906df34fb..dd1be3b01 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -225,9 +225,6 @@ jobs: name: wheels path: ./target/wheels/ - - name: Install Twine - run: pip install twine - - name: Test for secrets access id: check_secrets shell: bash @@ -241,27 +238,32 @@ jobs: if [ -n "$AWS_SECRET" ]; then HAS_AWS_SECRET='true' ; fi echo HAS_AWS_SECRET=${HAS_AWS_SECRET} >>$GITHUB_OUTPUT env: - SECRET: "${{ secrets.test_pypi_password }}" + SECRET: "${{ secrets.GITHUB_TOKEN }}" AWS_SECRET: "${{ secrets.CHIA_AWS_ACCOUNT_ID }}" - - name: publish (PyPi) - if: startsWith(github.event.ref, 'refs/tags') && steps.check_secrets.outputs.HAS_SECRET + - name: Set Env + if: steps.check_secrets.outputs.HAS_SECRET + uses: Chia-Network/actions/setjobenv@main env: - TWINE_USERNAME: __token__ - TWINE_NON_INTERACTIVE: 1 - TWINE_PASSWORD: ${{ secrets.pypi_password }} - run: twine upload --non-interactive --skip-existing --verbose 'target/wheels/*' + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: publish (PyPi) + if: env.RELEASE == 'true' + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: target/wheels/ + skip-existing: true - name: publish (Test PyPi) - if: steps.check_secrets.outputs.HAS_SECRET - env: - TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/ - TWINE_USERNAME: __token__ - TWINE_NON_INTERACTIVE: 1 - TWINE_PASSWORD: ${{ secrets.test_pypi_password }} - run: twine upload --non-interactive --skip-existing --verbose 'target/wheels/*' + if: env.PRE_RELEASE == 'true' + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + packages-dir: target/wheels/ + skip-existing: true - name: Configure AWS credentials + if: steps.check_secrets.outputs.HAS_AWS_SECRET uses: aws-actions/configure-aws-credentials@v2 with: role-to-assume: arn:aws:iam::${{ secrets.CHIA_AWS_ACCOUNT_ID }}:role/installer-upload From 6a6f3909ec03f25170bf7f70335f0d4ea5cbb532 Mon Sep 17 00:00:00 2001 From: Brandon Butler Date: Thu, 17 Aug 2023 13:17:32 -0700 Subject: [PATCH 084/117] Remove checks for github_token, everyone gets one anyway --- .github/workflows/build-arm64-wheels.yml | 6 ------ .github/workflows/build-m1-wheel.yml | 6 ------ .github/workflows/build-test.yml | 6 ------ 3 files changed, 18 deletions(-) diff --git a/.github/workflows/build-arm64-wheels.yml b/.github/workflows/build-arm64-wheels.yml index b0830383b..9ee0dbe02 100644 --- a/.github/workflows/build-arm64-wheels.yml +++ b/.github/workflows/build-arm64-wheels.yml @@ -77,20 +77,14 @@ jobs: id: check_secrets shell: bash run: | - unset HAS_SECRET unset HAS_AWS_SECRET - if [ -n "$SECRET" ]; then HAS_SECRET='true' ; fi - echo HAS_SECRET=${HAS_SECRET} >>$GITHUB_OUTPUT - if [ -n "$AWS_SECRET" ]; then HAS_AWS_SECRET='true' ; fi echo HAS_AWS_SECRET=${HAS_AWS_SECRET} >>$GITHUB_OUTPUT env: - SECRET: "${{ secrets.GITHUB_TOKEN }}" AWS_SECRET: "${{ secrets.CHIA_AWS_ACCOUNT_ID }}" - name: Set Env - if: steps.check_secrets.outputs.HAS_SECRET uses: Chia-Network/actions/setjobenv@main env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-m1-wheel.yml b/.github/workflows/build-m1-wheel.yml index b8dc3fa0c..d9a6f7a5f 100644 --- a/.github/workflows/build-m1-wheel.yml +++ b/.github/workflows/build-m1-wheel.yml @@ -115,20 +115,14 @@ jobs: id: check_secrets shell: bash run: | - unset HAS_SECRET unset HAS_AWS_SECRET - if [ -n "$SECRET" ]; then HAS_SECRET='true' ; fi - echo "HAS_SECRET=${HAS_SECRET}" >>$GITHUB_OUTPUT - if [ -n "$AWS_SECRET" ]; then HAS_AWS_SECRET='true' ; fi echo HAS_AWS_SECRET=${HAS_AWS_SECRET} >>$GITHUB_OUTPUT env: - SECRET: "${{ secrets.GITHUB_TOKEN }}" AWS_SECRET: "${{ secrets.CHIA_AWS_ACCOUNT_ID }}" - name: Set Env - if: steps.check_secrets.outputs.HAS_SECRET uses: Chia-Network/actions/setjobenv@main env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index dd1be3b01..c0781cbef 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -229,20 +229,14 @@ jobs: id: check_secrets shell: bash run: | - unset HAS_SECRET unset HAS_AWS_SECRET - if [ -n "$SECRET" ]; then HAS_SECRET='true' ; fi - echo "HAS_SECRET=${HAS_SECRET}" >>$GITHUB_OUTPUT - if [ -n "$AWS_SECRET" ]; then HAS_AWS_SECRET='true' ; fi echo HAS_AWS_SECRET=${HAS_AWS_SECRET} >>$GITHUB_OUTPUT env: - SECRET: "${{ secrets.GITHUB_TOKEN }}" AWS_SECRET: "${{ secrets.CHIA_AWS_ACCOUNT_ID }}" - name: Set Env - if: steps.check_secrets.outputs.HAS_SECRET uses: Chia-Network/actions/setjobenv@main env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 86e25db5f3b591044a87a5011afec666f09bcf1d Mon Sep 17 00:00:00 2001 From: Adam Kelly <338792+aqk@users.noreply.github.com> Date: Wed, 23 Aug 2023 11:26:58 -0700 Subject: [PATCH 085/117] Use tools/manage_clvm.py and chia/wallet/puzzles/deployed_puzzle_hashes.json to keep the recompile checks up to date with chia-blockchain --- .github/workflows/build-test.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index c0781cbef..b6ecb1cd9 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -120,7 +120,7 @@ jobs: run: | . ./activate python -m pip install pytest - python -m pip install blspy + python -m pip install blspy - name: install clvm & clvm_tools run: | @@ -156,9 +156,8 @@ jobs: rm -rf chia-blockchain git clone https://github.com/Chia-Network/chia-blockchain - # Check recompiles - cp support/recompile_check.py chia-blockchain - (cd chia-blockchain && python recompile_check.py) + # Check that recompiling deployed puzzles match with their deployed hashes + (cd chia-blockchain && pip install .\[dev\] &&./activated.py python tools/manage_clvm.py check) - name: Test Classic command line tools with pytest if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') From 2d386c89b26e85f09f583fe47273097419734ff8 Mon Sep 17 00:00:00 2001 From: Adam Kelly <338792+aqk@users.noreply.github.com> Date: Wed, 23 Aug 2023 12:27:02 -0700 Subject: [PATCH 086/117] Make sure to use the correct (newer) version of clvm_tools_rs dependencies --- .github/workflows/build-test.yml | 7 ++++-- support/install_deps.sh | 23 +++++++++++++++++++ ...mpile_check.py.use-deployed-hash-list.diff | 15 ++++++++++++ support/verify_compiler_version.sh | 22 ++++++++++++++++++ 4 files changed, 65 insertions(+), 2 deletions(-) create mode 100755 support/install_deps.sh create mode 100644 support/recompile_check.py.use-deployed-hash-list.diff create mode 100755 support/verify_compiler_version.sh diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index b6ecb1cd9..1284e1cef 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -120,7 +120,7 @@ jobs: run: | . ./activate python -m pip install pytest - python -m pip install blspy + python -m pip install blspy - name: install clvm & clvm_tools run: | @@ -144,6 +144,8 @@ jobs: python -c 'import clvm; print(clvm.__file__)' python -c 'import clvm_rs; print(clvm_rs.__file__)' python -c 'import clvm_tools_rs; print(clvm_tools_rs.__file__)' + echo "CLVM_TOOLS_RS_VERSION=$(python -c 'import clvm_tools_rs; print(clvm_tools_rs.get_version())')" >> "$GITHUB_ENV" + - name: Verify recompilation of old sources match if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') @@ -157,7 +159,8 @@ jobs: git clone https://github.com/Chia-Network/chia-blockchain # Check that recompiling deployed puzzles match with their deployed hashes - (cd chia-blockchain && pip install .\[dev\] &&./activated.py python tools/manage_clvm.py check) + cp support/install_deps.sh support/verify_compiler_version.sh chia-blockchain + (cd chia-blockchain && pip install .\[dev\] && ./install_deps.sh && ./verify_compiler_version.sh ${CLVM_TOOLS_RS_VERSION} && ./activated.py python tools/manage_clvm.py check) - name: Test Classic command line tools with pytest if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') diff --git a/support/install_deps.sh b/support/install_deps.sh new file mode 100755 index 000000000..ccca448f6 --- /dev/null +++ b/support/install_deps.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# This script is called from $GIT_ROOT/.github/workflows/build-test.yml +# This script is called while in $GIT_ROOT/chia-blockchain of clvm_tools_rs + +. ./activate + +python -m pip uninstall clvm clvm_rs clvm_tools clvm_tools_rs + +git clone https://github.com/Chia-Network/clvm.git --branch=main --single-branch +python -m pip install ./clvm + +echo "installing clvm_rs via pip" +pip install clvm_rs + +echo "installing clvm_tools for clvm tests" + +# Ensure clvm_tools is installed from its own repo. +git clone https://github.com/Chia-Network/clvm_tools.git --branch=main --single-branch +python -m pip install ./clvm_tools + +# Install clvm_tools_rs from the directory above. +python -m pip install .. diff --git a/support/recompile_check.py.use-deployed-hash-list.diff b/support/recompile_check.py.use-deployed-hash-list.diff new file mode 100644 index 000000000..752262468 --- /dev/null +++ b/support/recompile_check.py.use-deployed-hash-list.diff @@ -0,0 +1,15 @@ +diff --git a/support/recompile_check.py b/support/recompile_check.py +index c991198e..999896b6 100644 +--- a/support/recompile_check.py ++++ b/support/recompile_check.py +@@ -66,6 +66,10 @@ recompile_list = [ + gentest('test_multiple_generator_input_arguments.clsp') + ] + ++here = Path(__file__).parent.resolve() ++root = here.parent ++hashes_path = root.joinpath("chia/wallet/puzzles/deployed_puzzle_hashes.json") ++ + for recompile_entry in recompile_list: + if 'dirname' in recompile_entry and 'fname' in recompile_entry: + dirname = recompile_entry['dirname'] diff --git a/support/verify_compiler_version.sh b/support/verify_compiler_version.sh new file mode 100755 index 000000000..693ebb641 --- /dev/null +++ b/support/verify_compiler_version.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +if [[ $# -ne 1 ]]; then + echo "Illegal number of parameters" >&2 + echo "Usage: $0 " >&2 + exit 2 +fi + +pip_installed_version=$(pip list | grep clvm_tools_rs | awk '{print $2}') +python_import_version=$(python -c 'import clvm_tools_rs; print(clvm_tools_rs.get_version())') + +expected_version="$1" + +if [ "$expected_version" == "$pip_installed_version" ] && [ "$expected_version" == "$python_import_version" ]; then + exit 0 +else + echo "clvm_tools_rs VERSIONS does not match expected version" + echo "PIP INSTALLED VERSION: $pip_installed_version" + echo "PYTHON IMPORTED VERSION: $python_import_version" + echo "EXPECTED VERSION: $expected_version" + exit 1 +fi From acfa43a009d0c94c733d7c44ef62ab6d7855377e Mon Sep 17 00:00:00 2001 From: Adam Kelly <338792+aqk@users.noreply.github.com> Date: Wed, 23 Aug 2023 12:35:39 -0700 Subject: [PATCH 087/117] Fix workflow YAML indent --- .github/workflows/build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 1284e1cef..55d59ac98 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -144,7 +144,7 @@ jobs: python -c 'import clvm; print(clvm.__file__)' python -c 'import clvm_rs; print(clvm_rs.__file__)' python -c 'import clvm_tools_rs; print(clvm_tools_rs.__file__)' - echo "CLVM_TOOLS_RS_VERSION=$(python -c 'import clvm_tools_rs; print(clvm_tools_rs.get_version())')" >> "$GITHUB_ENV" + echo "CLVM_TOOLS_RS_VERSION=$(python -c 'import clvm_tools_rs; print(clvm_tools_rs.get_version())')" >> "$GITHUB_ENV" - name: Verify recompilation of old sources match From 3ff04824392601c35ad0468e29a12d35e6dfac34 Mon Sep 17 00:00:00 2001 From: Adam Kelly <338792+aqk@users.noreply.github.com> Date: Wed, 23 Aug 2023 12:53:34 -0700 Subject: [PATCH 088/117] Build Alpine wheel last --- .github/workflows/build-test.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 55d59ac98..6ff9548d7 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -1,5 +1,5 @@ # Thanks: clvm_rs' github actions. -name: Build Mac, Linux, and Windows wheels +name: Build on: push: @@ -89,12 +89,6 @@ jobs: # Ensure an empty .cargo-lock file exists. touch target/release/.cargo-lock - - name: Build alpine wheel via docker - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') - run: | - cd resources/alpine && docker build -t clvm-tools-rs-alpine . - docker run -v ${GITHUB_WORKSPACE}:/root/clvm_tools_rs -t clvm-tools-rs-alpine sh /root/build-alpine.sh - - name: Build Windows with maturin on Python ${{ matrix.python }} if: startsWith(matrix.os, 'windows') run: | @@ -276,6 +270,12 @@ jobs: aws --no-progress s3 cp "$file" "s3://download.chia.net/simple-dev/clvm-tools-rs/$filename" done <<< "$FILES" + - name: Build alpine wheel via docker + if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') + run: | + cd resources/alpine && docker build -t clvm-tools-rs-alpine . + docker run -v ${GITHUB_WORKSPACE}:/root/clvm_tools_rs -t clvm-tools-rs-alpine sh /root/build-alpine.sh + fmt: runs-on: ubuntu-20.04 name: cargo fmt From 54b42c6cb407ebd51c8699f702a568ab753a66ed Mon Sep 17 00:00:00 2001 From: Adam Kelly <338792+aqk@users.noreply.github.com> Date: Wed, 23 Aug 2023 13:04:42 -0700 Subject: [PATCH 089/117] Activate secondary venv for recompile check --- .github/workflows/build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 6ff9548d7..2d8ecd805 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -154,7 +154,7 @@ jobs: # Check that recompiling deployed puzzles match with their deployed hashes cp support/install_deps.sh support/verify_compiler_version.sh chia-blockchain - (cd chia-blockchain && pip install .\[dev\] && ./install_deps.sh && ./verify_compiler_version.sh ${CLVM_TOOLS_RS_VERSION} && ./activated.py python tools/manage_clvm.py check) + (cd chia-blockchain && python -m venv venv && . venv/bin/activate && pip install .\[dev\] && python -m pip install maturin==1.1.0 && ./install_deps.sh && ./verify_compiler_version.sh ${CLVM_TOOLS_RS_VERSION} && ./activated.py python tools/manage_clvm.py check) - name: Test Classic command line tools with pytest if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') From 8ddc9e18abfb3112326458a34291f27a16755bf9 Mon Sep 17 00:00:00 2001 From: Adam Kelly <338792+aqk@users.noreply.github.com> Date: Wed, 23 Aug 2023 13:38:40 -0700 Subject: [PATCH 090/117] Use cannonical location of py venv activate --- support/install_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/support/install_deps.sh b/support/install_deps.sh index ccca448f6..9ce47d56b 100755 --- a/support/install_deps.sh +++ b/support/install_deps.sh @@ -3,7 +3,7 @@ # This script is called from $GIT_ROOT/.github/workflows/build-test.yml # This script is called while in $GIT_ROOT/chia-blockchain of clvm_tools_rs -. ./activate +. ./venv/bin/activate python -m pip uninstall clvm clvm_rs clvm_tools clvm_tools_rs From 96f8fef71cb001ed8c809c674e033e4d55c3f88d Mon Sep 17 00:00:00 2001 From: Adam Kelly <338792+aqk@users.noreply.github.com> Date: Wed, 23 Aug 2023 21:20:31 -0700 Subject: [PATCH 091/117] Install current versions first, before chia-blockchain installs its dependencies --- .github/workflows/build-test.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 2d8ecd805..959717b0e 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -154,7 +154,11 @@ jobs: # Check that recompiling deployed puzzles match with their deployed hashes cp support/install_deps.sh support/verify_compiler_version.sh chia-blockchain - (cd chia-blockchain && python -m venv venv && . venv/bin/activate && pip install .\[dev\] && python -m pip install maturin==1.1.0 && ./install_deps.sh && ./verify_compiler_version.sh ${CLVM_TOOLS_RS_VERSION} && ./activated.py python tools/manage_clvm.py check) + (cd chia-blockchain && python -m venv venv && . venv/bin/activate && \ + python -m pip install maturin==1.1.0 && \ + ./install_deps.sh && \ + pip install .\[dev\] && \ + ./verify_compiler_version.sh ${CLVM_TOOLS_RS_VERSION} && ./activated.py python tools/manage_clvm.py check) - name: Test Classic command line tools with pytest if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') From aa6c7bc0322eb8dd4bb9620d75437b2540e1e61b Mon Sep 17 00:00:00 2001 From: Adam Kelly <338792+aqk@users.noreply.github.com> Date: Thu, 24 Aug 2023 09:11:31 -0700 Subject: [PATCH 092/117] Install previously built clvm_tools_rs instead of rebuilding the wheel --- .github/workflows/build-test.yml | 6 +++--- support/install_deps.sh | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 959717b0e..ea5753a30 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -155,9 +155,9 @@ jobs: # Check that recompiling deployed puzzles match with their deployed hashes cp support/install_deps.sh support/verify_compiler_version.sh chia-blockchain (cd chia-blockchain && python -m venv venv && . venv/bin/activate && \ - python -m pip install maturin==1.1.0 && \ - ./install_deps.sh && \ - pip install .\[dev\] && \ + cd .. && python support/wheelname.py && \ + cd chia-blockchain && \ + pip install .\[dev\] && \ ./verify_compiler_version.sh ${CLVM_TOOLS_RS_VERSION} && ./activated.py python tools/manage_clvm.py check) - name: Test Classic command line tools with pytest diff --git a/support/install_deps.sh b/support/install_deps.sh index 9ce47d56b..ebc1a11a0 100755 --- a/support/install_deps.sh +++ b/support/install_deps.sh @@ -1,10 +1,11 @@ -#!/bin/bash +#!/bin/bash -x # This script is called from $GIT_ROOT/.github/workflows/build-test.yml # This script is called while in $GIT_ROOT/chia-blockchain of clvm_tools_rs . ./venv/bin/activate +python -m pip install --upgrade pip python -m pip uninstall clvm clvm_rs clvm_tools clvm_tools_rs git clone https://github.com/Chia-Network/clvm.git --branch=main --single-branch From 3e9ad89847437aceb793b13a4c9672cb70dc1707 Mon Sep 17 00:00:00 2001 From: Adam Kelly <338792+aqk@users.noreply.github.com> Date: Thu, 24 Aug 2023 11:06:48 -0700 Subject: [PATCH 093/117] Try recompile check without installing all chia-blockchain deps --- .github/workflows/build-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index ea5753a30..67ac870ed 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -141,7 +141,7 @@ jobs: echo "CLVM_TOOLS_RS_VERSION=$(python -c 'import clvm_tools_rs; print(clvm_tools_rs.get_version())')" >> "$GITHUB_ENV" - - name: Verify recompilation of old sources match + - name: Verify recompilation of old sources match with new compiler if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') run: | . ./activate @@ -157,7 +157,7 @@ jobs: (cd chia-blockchain && python -m venv venv && . venv/bin/activate && \ cd .. && python support/wheelname.py && \ cd chia-blockchain && \ - pip install .\[dev\] && \ + #pip install .\[dev\] && \ ./verify_compiler_version.sh ${CLVM_TOOLS_RS_VERSION} && ./activated.py python tools/manage_clvm.py check) - name: Test Classic command line tools with pytest From d9998129013e4a947b80be132c8939a17886caa8 Mon Sep 17 00:00:00 2001 From: Adam Kelly <338792+aqk@users.noreply.github.com> Date: Thu, 24 Aug 2023 11:31:15 -0700 Subject: [PATCH 094/117] Install deps for tools/manage_clvm.py --- .github/workflows/build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 67ac870ed..fd33686e8 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -157,7 +157,7 @@ jobs: (cd chia-blockchain && python -m venv venv && . venv/bin/activate && \ cd .. && python support/wheelname.py && \ cd chia-blockchain && \ - #pip install .\[dev\] && \ + pip install click typing_extensions && \ ./verify_compiler_version.sh ${CLVM_TOOLS_RS_VERSION} && ./activated.py python tools/manage_clvm.py check) - name: Test Classic command line tools with pytest From e451b32ab015d629d7586f14890712270c8b1e51 Mon Sep 17 00:00:00 2001 From: Adam Kelly <338792+aqk@users.noreply.github.com> Date: Thu, 24 Aug 2023 11:57:11 -0700 Subject: [PATCH 095/117] Remove pip annoyance --- .github/workflows/build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index fd33686e8..976b1ab6b 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -154,7 +154,7 @@ jobs: # Check that recompiling deployed puzzles match with their deployed hashes cp support/install_deps.sh support/verify_compiler_version.sh chia-blockchain - (cd chia-blockchain && python -m venv venv && . venv/bin/activate && \ + (cd chia-blockchain && python -m venv venv && . venv/bin/activate && pip install --upgrade pip \ cd .. && python support/wheelname.py && \ cd chia-blockchain && \ pip install click typing_extensions && \ From 280a1f6591f1f6ea14f5fa0704b30144ebb42b64 Mon Sep 17 00:00:00 2001 From: Adam Kelly <338792+aqk@users.noreply.github.com> Date: Thu, 24 Aug 2023 12:06:11 -0700 Subject: [PATCH 096/117] debugging --- .github/workflows/build-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 976b1ab6b..aa442ae6c 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -144,6 +144,7 @@ jobs: - name: Verify recompilation of old sources match with new compiler if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8') run: | + set -x . ./activate # Build cmd line tools PYO3_PYTHON=`which python` cargo build --no-default-features --release From 57df32667dcdf0f76058baee86d23d9aff5b8682 Mon Sep 17 00:00:00 2001 From: Adam Kelly <338792+aqk@users.noreply.github.com> Date: Thu, 24 Aug 2023 12:11:37 -0700 Subject: [PATCH 097/117] re-add maturin install --- .github/workflows/build-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index aa442ae6c..ae479e747 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -156,6 +156,7 @@ jobs: # Check that recompiling deployed puzzles match with their deployed hashes cp support/install_deps.sh support/verify_compiler_version.sh chia-blockchain (cd chia-blockchain && python -m venv venv && . venv/bin/activate && pip install --upgrade pip \ + python -m pip install maturin==1.1.0 && \ cd .. && python support/wheelname.py && \ cd chia-blockchain && \ pip install click typing_extensions && \ From 9238db711f4006d5346eaab2b7a4f349a53ef737 Mon Sep 17 00:00:00 2001 From: Adam Kelly <338792+aqk@users.noreply.github.com> Date: Thu, 24 Aug 2023 12:12:12 -0700 Subject: [PATCH 098/117] debugging --- support/verify_compiler_version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/support/verify_compiler_version.sh b/support/verify_compiler_version.sh index 693ebb641..7efbbdb75 100755 --- a/support/verify_compiler_version.sh +++ b/support/verify_compiler_version.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -x if [[ $# -ne 1 ]]; then echo "Illegal number of parameters" >&2 From 97cd553c5c23390cc83ed4ac43d0e88e9d14513c Mon Sep 17 00:00:00 2001 From: Adam Kelly <338792+aqk@users.noreply.github.com> Date: Thu, 24 Aug 2023 12:20:03 -0700 Subject: [PATCH 099/117] typo --- .github/workflows/build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index ae479e747..060cf52c3 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -155,7 +155,7 @@ jobs: # Check that recompiling deployed puzzles match with their deployed hashes cp support/install_deps.sh support/verify_compiler_version.sh chia-blockchain - (cd chia-blockchain && python -m venv venv && . venv/bin/activate && pip install --upgrade pip \ + (cd chia-blockchain && python -m venv venv && . venv/bin/activate && pip install --upgrade pip && \ python -m pip install maturin==1.1.0 && \ cd .. && python support/wheelname.py && \ cd chia-blockchain && \ From ce6cc644a463889e238289472c27fafab90967a3 Mon Sep 17 00:00:00 2001 From: Adam Kelly <338792+aqk@users.noreply.github.com> Date: Thu, 24 Aug 2023 12:33:59 -0700 Subject: [PATCH 100/117] Allow tools/manage_clvm.py to import from chia-blockchain --- .github/workflows/build-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 060cf52c3..cdaeba499 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -160,6 +160,7 @@ jobs: cd .. && python support/wheelname.py && \ cd chia-blockchain && \ pip install click typing_extensions && \ + export PYTHONPATH=${PYTHONPATH}:$(pwd) && \ ./verify_compiler_version.sh ${CLVM_TOOLS_RS_VERSION} && ./activated.py python tools/manage_clvm.py check) - name: Test Classic command line tools with pytest From 4695b95b7bada523b261c53d39a01d1a5a307109 Mon Sep 17 00:00:00 2001 From: Adam Kelly <338792+aqk@users.noreply.github.com> Date: Fri, 25 Aug 2023 08:37:37 -0700 Subject: [PATCH 101/117] add more deps for manage_clvm.py --- .github/workflows/build-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index cdaeba499..2508c4326 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -159,7 +159,8 @@ jobs: python -m pip install maturin==1.1.0 && \ cd .. && python support/wheelname.py && \ cd chia-blockchain && \ - pip install click typing_extensions && \ + # deps for manage_clvm.py + pip install click typing_extensions chia_rs clvm && \ export PYTHONPATH=${PYTHONPATH}:$(pwd) && \ ./verify_compiler_version.sh ${CLVM_TOOLS_RS_VERSION} && ./activated.py python tools/manage_clvm.py check) From 1a7b41a2aacfa0f064f5b5846fdba54e4ce2e736 Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 25 Aug 2023 10:40:58 -0700 Subject: [PATCH 102/117] Change clippy reporter action to garaffate's version --- .github/workflows/build-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index c0781cbef..7e98705ec 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -301,10 +301,10 @@ jobs: override: true - name: clippy run: cargo clippy --all -- -D warnings - - uses: actions-rs/clippy-check@v1 + - uses: giraffate/clippy-action@v1 with: - token: ${{ secrets.GITHUB_TOKEN }} - args: --all-features + reporter: 'github-pr-review' + github_token: ${{ secrets.GITHUB_TOKEN }} unit_tests: runs-on: ubuntu-20.04 From 0e5dab0777191294d51bc1382236fd560adc6502 Mon Sep 17 00:00:00 2001 From: Chris Marslender Date: Fri, 25 Aug 2023 14:11:49 -0500 Subject: [PATCH 103/117] Update npm publish workflow --- .github/workflows/npm-publish.yml | 37 +++++++++++++++---------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index 395a02866..d6fc15dc2 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -3,26 +3,33 @@ name: npm publish on: push: branches: - - main - - dev - tags: - - '**' + - base + release: + types: [published] pull_request: branches: - '**' +concurrency: + # SHA is added to the end if on `main` to let all main workflows run + group: ${{ github.ref }}-${{ github.workflow }}-${{ github.event_name }}-${{ (github.ref == 'refs/heads/main') && github.sha || '' }} + cancel-in-progress: true + jobs: build_npm: name: Npm runs-on: ubuntu-latest - strategy: - fail-fast: false steps: - uses: actions/checkout@v3 with: fetch-depth: 0 + - name: Set Env + uses: Chia-Network/actions/setjobenv@main + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Set up rusts uses: actions-rs/toolchain@v1 with: @@ -49,20 +56,12 @@ jobs: name: npm-pkg path: ./wasm/pkg/clvm_tools_wasm-*.tgz - - name: Test for secrets access - id: check_secrets - shell: bash - run: | - unset HAS_SECRET - if [ -n "$SECRET" ]; then HAS_SECRET='true' ; fi - echo "HAS_SECRET=${HAS_SECRET}" >>$GITHUB_OUTPUT - env: - SECRET: "${{ secrets.test_pypi_password }}" - - name: Publish wasm - if: steps.check_secrets.HAS_SECRET - shell: bash + if: env.FULL_RELEASE == 'true' + env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + working-directory: ${{ github.workspace }}/wasm/pkg run: | - cd wasm/pkg + echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > .npmrc rm -f clvm_tools_wasm-*.tgz npm publish --access public From 050b055814d10820eb0061742650b1452ebdc322 Mon Sep 17 00:00:00 2001 From: Chris Marslender Date: Fri, 25 Aug 2023 14:54:43 -0500 Subject: [PATCH 104/117] Update the package name before publishing to npm --- .github/workflows/npm-publish.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index d6fc15dc2..4b1aa23c9 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -47,6 +47,13 @@ jobs: with: node-version: '16.x' + # Cargo.toml won't allow an "@" in the name, so we just update the package name this way for NPM + - name: Update package name for npm + working-directory: ${{ github.workspace }}/wasm/pkg + run: | + cp package.json package.json.orig + jq '.name="@chia/chialisp"' package.json > temp.json && mv temp.json package.json + - name: Test wasm run: node wasm/tests/index.js @@ -56,8 +63,18 @@ jobs: name: npm-pkg path: ./wasm/pkg/clvm_tools_wasm-*.tgz + - name: Test for secrets access + id: check_secrets + shell: bash + run: | + unset HAS_SECRET + if [ -n "$SECRET" ]; then HAS_SECRET='true' ; fi + echo "HAS_SECRET=${HAS_SECRET}" >>$GITHUB_OUTPUT + env: + SECRET: "${{ secrets.NPM_TOKEN }}" + - name: Publish wasm - if: env.FULL_RELEASE == 'true' + if: env.FULL_RELEASE == 'true' && steps.check_secrets.HAS_SECRET env: NPM_TOKEN: ${{ secrets.NPM_TOKEN }} working-directory: ${{ github.workspace }}/wasm/pkg From e645feb3d187f559d1aa77593744f5c8dd13f11a Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 28 Aug 2023 01:04:08 -0700 Subject: [PATCH 105/117] Pull in maturin update which is needed to pip install from git ref --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bd9c56524..6049eb79b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.11.3,<0.12"] +requires = ["maturin>=1.1.0,<1.3.0"] build-backend = "maturin" [tool.maturin] From 044632ce76c04beac5a2f7ae2d57fd185fdf7df0 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 5 Sep 2023 11:19:41 -0700 Subject: [PATCH 106/117] Route in result in rename and improve map_m a bit --- src/compiler/comptypes.rs | 17 ++- src/compiler/frontend.rs | 2 +- src/compiler/rename.rs | 236 ++++++++++++++++++-------------------- 3 files changed, 128 insertions(+), 127 deletions(-) diff --git a/src/compiler/comptypes.rs b/src/compiler/comptypes.rs index f7dc35811..76894e6d3 100644 --- a/src/compiler/comptypes.rs +++ b/src/compiler/comptypes.rs @@ -838,7 +838,10 @@ pub fn cons_of_string_map( list_to_cons(l, &sorted_converted) } -pub fn map_m(f: &dyn Fn(&T) -> Result, list: &[T]) -> Result, E> { +pub fn map_m(mut f: F, list: &[T]) -> Result, E> +where + F: FnMut(&T) -> Result +{ let mut result = Vec::new(); for e in list { let val = f(e)?; @@ -847,6 +850,18 @@ pub fn map_m(f: &dyn Fn(&T) -> Result, list: &[T]) -> Result(mut f: F, list: &[T]) -> Result, E> +where + F: FnMut(&T) -> Result +{ + let mut result = Vec::new(); + for e in list { + let val = f(e)?; + result.push(val); + } + Ok(result.into_iter().rev().collect()) +} + pub fn fold_m(f: &dyn Fn(&R, &T) -> Result, start: R, list: &[T]) -> Result { let mut res: R = start; for elt in list.iter() { diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index bf919e952..341b1f7f7 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -916,7 +916,7 @@ pub fn frontend( } }; - let our_mod = rename_children_compileform(&compiled?); + let our_mod = rename_children_compileform(&compiled?)?; let expr_names: HashSet> = collect_used_names_bodyform(our_mod.exp.borrow()) .iter() diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index ecbde85a1..0e0e75e43 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -5,11 +5,12 @@ use std::rc::Rc; use crate::compiler::codegen::toposort_assign_bindings; use crate::compiler::comptypes::{ Binding, BindingPattern, BodyForm, CompileErr, CompileForm, DefconstData, DefmacData, - DefunData, HelperForm, LetData, LetFormKind, + DefunData, HelperForm, LetData, LetFormKind, map_m, map_m_reverse }; use crate::compiler::gensym::gensym; use crate::compiler::sexp::SExp; use crate::compiler::srcloc::Srcloc; +use crate::util::TopoSortItem; /// Rename in a qq form. This searches for (unquote ...) forms inside and performs /// rename inside them, leaving the rest of the qq form as is. @@ -156,6 +157,7 @@ fn make_binding_unique(b: &Binding) -> InnerRenameList { } } + pub fn rename_assign_bindings( l: &Srcloc, bindings: &[Rc], @@ -164,85 +166,80 @@ pub fn rename_assign_bindings( // Order the bindings. let sorted_bindings = toposort_assign_bindings(l, bindings)?; let mut renames = HashMap::new(); - let renamed_bindings = sorted_bindings + // Process in reverse order so we rename from inner to outer. + let bindings_to_rename: Vec> = sorted_bindings .iter() .rev() - .map(|item| { - let b: &Binding = bindings[item.index].borrow(); - if let BindingPattern::Complex(p) = &b.pattern { - let new_names = invent_new_names_sexp(p.clone()); - for (name, renamed) in new_names.iter() { - renames.insert(name.clone(), renamed.clone()); - } - Binding { - pattern: BindingPattern::Complex(rename_in_cons(&renames, p.clone(), false)), - body: Rc::new(rename_in_bodyform(&renames, b.body.clone())), - ..b.clone() - } - } else { - b.clone() - } - }) - .rev() - .map(Rc::new) + .cloned() .collect(); - Ok((rename_in_bodyform(&renames, body), renamed_bindings)) + let renamed_bindings = map_m_reverse(|item: &TopoSortItem<_>| -> Result, CompileErr> { + let b: &Binding = bindings[item.index].borrow(); + if let BindingPattern::Complex(p) = &b.pattern { + let new_names = invent_new_names_sexp(p.clone()); + for (name, renamed) in new_names.iter() { + renames.insert(name.clone(), renamed.clone()); + } + Ok(Rc::new(Binding { + pattern: BindingPattern::Complex(rename_in_cons(&renames, p.clone(), false)), + body: Rc::new(rename_in_bodyform(&renames, b.body.clone())?), + ..b.clone() + })) + } else { + Ok(bindings[item.index].clone()) + } + }, &bindings_to_rename)?; + Ok((rename_in_bodyform(&renames, body)?, renamed_bindings)) } -fn rename_in_bodyform(namemap: &HashMap, Vec>, b: Rc) -> BodyForm { +fn rename_in_bodyform(namemap: &HashMap, Vec>, b: Rc) -> Result { match b.borrow() { BodyForm::Let(kind, letdata) => { - let new_bindings = letdata - .bindings - .iter() - .map(|b| { - Rc::new(Binding { - loc: b.loc(), - nl: b.nl.clone(), - pattern: b.pattern.clone(), - body: Rc::new(rename_in_bodyform(namemap, b.body.clone())), - }) - }) - .collect(); - let new_body = rename_in_bodyform(namemap, letdata.body.clone()); - BodyForm::Let( + let new_bindings = map_m(&|b: &Rc| -> Result, CompileErr> { + Ok(Rc::new(Binding { + loc: b.loc(), + nl: b.nl.clone(), + pattern: b.pattern.clone(), + body: Rc::new(rename_in_bodyform(namemap, b.body.clone())?), + })) + }, &letdata.bindings)?; + let new_body = rename_in_bodyform(namemap, letdata.body.clone())?; + Ok(BodyForm::Let( kind.clone(), Box::new(LetData { bindings: new_bindings, body: Rc::new(new_body), ..*letdata.clone() }), - ) + )) } BodyForm::Quoted(atom) => match atom { SExp::Atom(l, n) => match namemap.get(n) { - Some(named) => BodyForm::Quoted(SExp::Atom(l.clone(), named.to_vec())), - None => BodyForm::Quoted(atom.clone()), + Some(named) => Ok(BodyForm::Quoted(SExp::Atom(l.clone(), named.to_vec()))), + None => Ok(BodyForm::Quoted(atom.clone())), }, - _ => BodyForm::Quoted(atom.clone()), + _ => Ok(BodyForm::Quoted(atom.clone())), }, BodyForm::Value(atom) => match atom { SExp::Atom(l, n) => match namemap.get(n) { - Some(named) => BodyForm::Value(SExp::Atom(l.clone(), named.to_vec())), - None => BodyForm::Value(atom.clone()), + Some(named) => Ok(BodyForm::Value(SExp::Atom(l.clone(), named.to_vec()))), + None => Ok(BodyForm::Value(atom.clone())), }, - _ => BodyForm::Value(atom.clone()), + _ => Ok(BodyForm::Value(atom.clone())), }, BodyForm::Call(l, vs, tail) => { - let new_vs = vs - .iter() - .map(|x| Rc::new(rename_in_bodyform(namemap, x.clone()))) - .collect(); - let new_tail = tail - .as_ref() - .map(|t| Rc::new(rename_in_bodyform(namemap, t.clone()))); - BodyForm::Call(l.clone(), new_vs, new_tail) + let new_vs = map_m(&|x: &Rc| -> Result, CompileErr> { Ok(Rc::new(rename_in_bodyform(namemap, x.clone())?)) }, &vs)?; + let new_tail = if let Some(t) = tail.as_ref() { + Some(Rc::new(rename_in_bodyform(namemap, t.clone())?)) + } else { + None + }; + Ok(BodyForm::Call(l.clone(), new_vs, new_tail)) } - BodyForm::Mod(l, prog) => BodyForm::Mod(l.clone(), prog.clone()), + BodyForm::Mod(l, prog) => Ok(BodyForm::Mod(l.clone(), prog.clone())), } } @@ -272,7 +269,7 @@ pub fn desugar_sequential_let_bindings( } } -fn rename_args_bodyform(b: &BodyForm) -> BodyForm { +fn rename_args_bodyform(b: &BodyForm) -> Result { match b { BodyForm::Let(LetFormKind::Sequential, letdata) => { // Renaming a sequential let is exactly as if the bindings were @@ -300,66 +297,62 @@ fn rename_args_bodyform(b: &BodyForm) -> BodyForm { local_namemap.insert(oldname.to_vec(), binding_name.clone()); } } - let new_bindings = new_renamed_bindings - .iter() - .map(|x| { - Rc::new(Binding { - loc: x.loc.clone(), - nl: x.nl.clone(), - pattern: x.pattern.clone(), - body: Rc::new(rename_args_bodyform(&x.body)), - }) - }) - .collect(); - let locally_renamed_body = rename_in_bodyform(&local_namemap, letdata.body.clone()); - BodyForm::Let( + let new_bindings = map_m(&|x: &Rc| -> Result, CompileErr> { + Ok(Rc::new(Binding { + loc: x.loc.clone(), + nl: x.nl.clone(), + pattern: x.pattern.clone(), + body: Rc::new(rename_args_bodyform(&x.body)?), + })) + }, &new_renamed_bindings)?; + let locally_renamed_body = rename_in_bodyform(&local_namemap, letdata.body.clone())?; + Ok(BodyForm::Let( LetFormKind::Parallel, Box::new(LetData { bindings: new_bindings, body: Rc::new(locally_renamed_body), ..*letdata.clone() }), - ) + )) } - BodyForm::Let(LetFormKind::Assign, _letdata) => b.clone(), + BodyForm::Let(LetFormKind::Assign, _letdata) => Ok(b.clone()), - BodyForm::Quoted(e) => BodyForm::Quoted(e.clone()), - BodyForm::Value(v) => BodyForm::Value(v.clone()), + BodyForm::Quoted(e) => Ok(BodyForm::Quoted(e.clone())), + BodyForm::Value(v) => Ok(BodyForm::Value(v.clone())), BodyForm::Call(l, vs, tail) => { - let new_vs = vs - .iter() - .map(|a| Rc::new(rename_args_bodyform(a))) - .collect(); - let new_tail = tail - .as_ref() - .map(|t| Rc::new(rename_args_bodyform(t.borrow()))); - BodyForm::Call(l.clone(), new_vs, new_tail) + let new_vs = map_m(&|a: &Rc| -> Result, CompileErr> { Ok(Rc::new(rename_args_bodyform(a)?)) }, &vs)?; + let new_tail = if let Some(t) = tail.as_ref() { + Some(Rc::new(rename_args_bodyform(t.borrow())?)) + } else { + None + }; + Ok(BodyForm::Call(l.clone(), new_vs, new_tail)) } - BodyForm::Mod(l, program) => BodyForm::Mod(l.clone(), program.clone()), + BodyForm::Mod(l, program) => Ok(BodyForm::Mod(l.clone(), program.clone())), } } -fn rename_in_helperform(namemap: &HashMap, Vec>, h: &HelperForm) -> HelperForm { +fn rename_in_helperform(namemap: &HashMap, Vec>, h: &HelperForm) -> Result { match h { - HelperForm::Defconstant(defc) => HelperForm::Defconstant(DefconstData { + HelperForm::Defconstant(defc) => Ok(HelperForm::Defconstant(DefconstData { loc: defc.loc.clone(), kind: defc.kind.clone(), name: defc.name.to_vec(), nl: defc.nl.clone(), kw: defc.kw.clone(), - body: Rc::new(rename_in_bodyform(namemap, defc.body.clone())), - }), - HelperForm::Defmacro(mac) => HelperForm::Defmacro(DefmacData { + body: Rc::new(rename_in_bodyform(namemap, defc.body.clone())?), + })), + HelperForm::Defmacro(mac) => Ok(HelperForm::Defmacro(DefmacData { loc: mac.loc.clone(), kw: mac.kw.clone(), nl: mac.nl.clone(), name: mac.name.to_vec(), args: mac.args.clone(), - program: Rc::new(rename_in_compileform(namemap, mac.program.clone())), - }), - HelperForm::Defun(inline, defun) => HelperForm::Defun( + program: Rc::new(rename_in_compileform(namemap, mac.program.clone())?), + })), + HelperForm::Defun(inline, defun) => Ok(HelperForm::Defun( *inline, DefunData { loc: defun.loc.clone(), @@ -368,22 +361,22 @@ fn rename_in_helperform(namemap: &HashMap, Vec>, h: &HelperForm) -> name: defun.name.to_vec(), orig_args: defun.orig_args.clone(), args: defun.args.clone(), - body: Rc::new(rename_in_bodyform(namemap, defun.body.clone())), + body: Rc::new(rename_in_bodyform(namemap, defun.body.clone())?), }, - ), + )), } } -fn rename_args_helperform(h: &HelperForm) -> HelperForm { +fn rename_args_helperform(h: &HelperForm) -> Result { match h { - HelperForm::Defconstant(defc) => HelperForm::Defconstant(DefconstData { + HelperForm::Defconstant(defc) => Ok(HelperForm::Defconstant(DefconstData { loc: defc.loc.clone(), kind: defc.kind.clone(), nl: defc.nl.clone(), kw: defc.kw.clone(), name: defc.name.clone(), - body: Rc::new(rename_args_bodyform(defc.body.borrow())), - }), + body: Rc::new(rename_args_bodyform(defc.body.borrow())?), + })), HelperForm::Defmacro(mac) => { let mut new_names: HashMap, Vec> = HashMap::new(); for x in invent_new_names_sexp(mac.args.clone()).iter() { @@ -394,8 +387,8 @@ fn rename_args_helperform(h: &HelperForm) -> HelperForm { local_namemap.insert(x.0.to_vec(), x.1.to_vec()); } let local_renamed_arg = rename_in_cons(&local_namemap, mac.args.clone(), true); - let local_renamed_body = rename_args_compileform(mac.program.borrow()); - HelperForm::Defmacro(DefmacData { + let local_renamed_body = rename_args_compileform(mac.program.borrow())?; + Ok(HelperForm::Defmacro(DefmacData { loc: mac.loc.clone(), kw: mac.kw.clone(), nl: mac.nl.clone(), @@ -404,8 +397,8 @@ fn rename_args_helperform(h: &HelperForm) -> HelperForm { program: Rc::new(rename_in_compileform( &local_namemap, Rc::new(local_renamed_body), - )), - }) + )?), + })) } HelperForm::Defun(inline, defun) => { let new_names = invent_new_names_sexp(defun.args.clone()); @@ -414,8 +407,8 @@ fn rename_args_helperform(h: &HelperForm) -> HelperForm { local_namemap.insert(x.0.clone(), x.1.clone()); } let local_renamed_arg = rename_in_cons(&local_namemap, defun.args.clone(), true); - let local_renamed_body = rename_args_bodyform(defun.body.borrow()); - HelperForm::Defun( + let local_renamed_body = rename_args_bodyform(defun.body.borrow())?; + Ok(HelperForm::Defun( *inline, DefunData { loc: defun.loc.clone(), @@ -427,40 +420,36 @@ fn rename_args_helperform(h: &HelperForm) -> HelperForm { body: Rc::new(rename_in_bodyform( &local_namemap, Rc::new(local_renamed_body), - )), + )?), }, - ) + )) } } } -fn rename_in_compileform(namemap: &HashMap, Vec>, c: Rc) -> CompileForm { - CompileForm { +fn rename_in_compileform(namemap: &HashMap, Vec>, c: Rc) -> Result { + Ok(CompileForm { loc: c.loc.clone(), args: c.args.clone(), include_forms: c.include_forms.clone(), - helpers: c - .helpers - .iter() - .map(|x| rename_in_helperform(namemap, x)) - .collect(), - exp: Rc::new(rename_in_bodyform(namemap, c.exp.clone())), - } + helpers: map_m(|x| rename_in_helperform(namemap, x), &c.helpers)?, + exp: Rc::new(rename_in_bodyform(namemap, c.exp.clone())?), + }) } -pub fn rename_children_compileform(c: &CompileForm) -> CompileForm { - let local_renamed_helpers = c.helpers.iter().map(rename_args_helperform).collect(); - let local_renamed_body = rename_args_bodyform(c.exp.borrow()); - CompileForm { +pub fn rename_children_compileform(c: &CompileForm) -> Result { + let local_renamed_helpers = map_m(&rename_args_helperform, &c.helpers)?; + let local_renamed_body = rename_args_bodyform(c.exp.borrow())?; + Ok(CompileForm { loc: c.loc.clone(), args: c.args.clone(), include_forms: c.include_forms.clone(), helpers: local_renamed_helpers, exp: Rc::new(local_renamed_body), - } + }) } -pub fn rename_args_compileform(c: &CompileForm) -> CompileForm { +pub fn rename_args_compileform(c: &CompileForm) -> Result { let new_names = invent_new_names_sexp(c.args.clone()); let mut local_namemap = HashMap::new(); for x in new_names.iter() { @@ -468,19 +457,16 @@ pub fn rename_args_compileform(c: &CompileForm) -> CompileForm { } let local_renamed_arg = rename_in_cons(&local_namemap, c.args.clone(), true); let local_renamed_helpers: Vec = - c.helpers.iter().map(rename_args_helperform).collect(); - let local_renamed_body = rename_args_bodyform(c.exp.borrow()); - CompileForm { + map_m(&rename_args_helperform, &c.helpers)?; + let local_renamed_body = rename_args_bodyform(c.exp.borrow())?; + Ok(CompileForm { loc: c.loc(), args: local_renamed_arg, include_forms: c.include_forms.clone(), - helpers: local_renamed_helpers - .iter() - .map(|x| rename_in_helperform(&local_namemap, x)) - .collect(), + helpers: map_m(|x| rename_in_helperform(&local_namemap, x), &local_renamed_helpers)?, exp: Rc::new(rename_in_bodyform( &local_namemap, Rc::new(local_renamed_body), - )), - } + )?), + }) } From 0cf1ce7815f3bc5e3321f5ca614ff89760122950 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 5 Sep 2023 11:25:20 -0700 Subject: [PATCH 107/117] fmt + clippy --- src/compiler/comptypes.rs | 4 +- src/compiler/rename.rs | 119 +++++++++++++++++++++++--------------- 2 files changed, 74 insertions(+), 49 deletions(-) diff --git a/src/compiler/comptypes.rs b/src/compiler/comptypes.rs index 76894e6d3..0d0c7a84e 100644 --- a/src/compiler/comptypes.rs +++ b/src/compiler/comptypes.rs @@ -840,7 +840,7 @@ pub fn cons_of_string_map( pub fn map_m(mut f: F, list: &[T]) -> Result, E> where - F: FnMut(&T) -> Result + F: FnMut(&T) -> Result, { let mut result = Vec::new(); for e in list { @@ -852,7 +852,7 @@ where pub fn map_m_reverse(mut f: F, list: &[T]) -> Result, E> where - F: FnMut(&T) -> Result + F: FnMut(&T) -> Result, { let mut result = Vec::new(); for e in list { diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index 0e0e75e43..e994e3be4 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -4,8 +4,8 @@ use std::rc::Rc; use crate::compiler::codegen::toposort_assign_bindings; use crate::compiler::comptypes::{ - Binding, BindingPattern, BodyForm, CompileErr, CompileForm, DefconstData, DefmacData, - DefunData, HelperForm, LetData, LetFormKind, map_m, map_m_reverse + map_m, map_m_reverse, Binding, BindingPattern, BodyForm, CompileErr, CompileForm, DefconstData, + DefmacData, DefunData, HelperForm, LetData, LetFormKind, }; use crate::compiler::gensym::gensym; use crate::compiler::sexp::SExp; @@ -157,7 +157,6 @@ fn make_binding_unique(b: &Binding) -> InnerRenameList { } } - pub fn rename_assign_bindings( l: &Srcloc, bindings: &[Rc], @@ -167,41 +166,46 @@ pub fn rename_assign_bindings( let sorted_bindings = toposort_assign_bindings(l, bindings)?; let mut renames = HashMap::new(); // Process in reverse order so we rename from inner to outer. - let bindings_to_rename: Vec> = sorted_bindings - .iter() - .rev() - .cloned() - .collect(); - let renamed_bindings = map_m_reverse(|item: &TopoSortItem<_>| -> Result, CompileErr> { - let b: &Binding = bindings[item.index].borrow(); - if let BindingPattern::Complex(p) = &b.pattern { - let new_names = invent_new_names_sexp(p.clone()); - for (name, renamed) in new_names.iter() { - renames.insert(name.clone(), renamed.clone()); + let bindings_to_rename: Vec> = sorted_bindings.iter().rev().cloned().collect(); + let renamed_bindings = map_m_reverse( + |item: &TopoSortItem<_>| -> Result, CompileErr> { + let b: &Binding = bindings[item.index].borrow(); + if let BindingPattern::Complex(p) = &b.pattern { + let new_names = invent_new_names_sexp(p.clone()); + for (name, renamed) in new_names.iter() { + renames.insert(name.clone(), renamed.clone()); + } + Ok(Rc::new(Binding { + pattern: BindingPattern::Complex(rename_in_cons(&renames, p.clone(), false)), + body: Rc::new(rename_in_bodyform(&renames, b.body.clone())?), + ..b.clone() + })) + } else { + Ok(bindings[item.index].clone()) } - Ok(Rc::new(Binding { - pattern: BindingPattern::Complex(rename_in_cons(&renames, p.clone(), false)), - body: Rc::new(rename_in_bodyform(&renames, b.body.clone())?), - ..b.clone() - })) - } else { - Ok(bindings[item.index].clone()) - } - }, &bindings_to_rename)?; + }, + &bindings_to_rename, + )?; Ok((rename_in_bodyform(&renames, body)?, renamed_bindings)) } -fn rename_in_bodyform(namemap: &HashMap, Vec>, b: Rc) -> Result { +fn rename_in_bodyform( + namemap: &HashMap, Vec>, + b: Rc, +) -> Result { match b.borrow() { BodyForm::Let(kind, letdata) => { - let new_bindings = map_m(&|b: &Rc| -> Result, CompileErr> { - Ok(Rc::new(Binding { - loc: b.loc(), - nl: b.nl.clone(), - pattern: b.pattern.clone(), - body: Rc::new(rename_in_bodyform(namemap, b.body.clone())?), - })) - }, &letdata.bindings)?; + let new_bindings = map_m( + &|b: &Rc| -> Result, CompileErr> { + Ok(Rc::new(Binding { + loc: b.loc(), + nl: b.nl.clone(), + pattern: b.pattern.clone(), + body: Rc::new(rename_in_bodyform(namemap, b.body.clone())?), + })) + }, + &letdata.bindings, + )?; let new_body = rename_in_bodyform(namemap, letdata.body.clone())?; Ok(BodyForm::Let( kind.clone(), @@ -230,7 +234,12 @@ fn rename_in_bodyform(namemap: &HashMap, Vec>, b: Rc) -> R }, BodyForm::Call(l, vs, tail) => { - let new_vs = map_m(&|x: &Rc| -> Result, CompileErr> { Ok(Rc::new(rename_in_bodyform(namemap, x.clone())?)) }, &vs)?; + let new_vs = map_m( + &|x: &Rc| -> Result, CompileErr> { + Ok(Rc::new(rename_in_bodyform(namemap, x.clone())?)) + }, + vs, + )?; let new_tail = if let Some(t) = tail.as_ref() { Some(Rc::new(rename_in_bodyform(namemap, t.clone())?)) } else { @@ -297,14 +306,17 @@ fn rename_args_bodyform(b: &BodyForm) -> Result { local_namemap.insert(oldname.to_vec(), binding_name.clone()); } } - let new_bindings = map_m(&|x: &Rc| -> Result, CompileErr> { - Ok(Rc::new(Binding { - loc: x.loc.clone(), - nl: x.nl.clone(), - pattern: x.pattern.clone(), - body: Rc::new(rename_args_bodyform(&x.body)?), - })) - }, &new_renamed_bindings)?; + let new_bindings = map_m( + &|x: &Rc| -> Result, CompileErr> { + Ok(Rc::new(Binding { + loc: x.loc.clone(), + nl: x.nl.clone(), + pattern: x.pattern.clone(), + body: Rc::new(rename_args_bodyform(&x.body)?), + })) + }, + &new_renamed_bindings, + )?; let locally_renamed_body = rename_in_bodyform(&local_namemap, letdata.body.clone())?; Ok(BodyForm::Let( LetFormKind::Parallel, @@ -322,7 +334,12 @@ fn rename_args_bodyform(b: &BodyForm) -> Result { BodyForm::Value(v) => Ok(BodyForm::Value(v.clone())), BodyForm::Call(l, vs, tail) => { - let new_vs = map_m(&|a: &Rc| -> Result, CompileErr> { Ok(Rc::new(rename_args_bodyform(a)?)) }, &vs)?; + let new_vs = map_m( + &|a: &Rc| -> Result, CompileErr> { + Ok(Rc::new(rename_args_bodyform(a)?)) + }, + vs, + )?; let new_tail = if let Some(t) = tail.as_ref() { Some(Rc::new(rename_args_bodyform(t.borrow())?)) } else { @@ -334,7 +351,10 @@ fn rename_args_bodyform(b: &BodyForm) -> Result { } } -fn rename_in_helperform(namemap: &HashMap, Vec>, h: &HelperForm) -> Result { +fn rename_in_helperform( + namemap: &HashMap, Vec>, + h: &HelperForm, +) -> Result { match h { HelperForm::Defconstant(defc) => Ok(HelperForm::Defconstant(DefconstData { loc: defc.loc.clone(), @@ -427,7 +447,10 @@ fn rename_args_helperform(h: &HelperForm) -> Result { } } -fn rename_in_compileform(namemap: &HashMap, Vec>, c: Rc) -> Result { +fn rename_in_compileform( + namemap: &HashMap, Vec>, + c: Rc, +) -> Result { Ok(CompileForm { loc: c.loc.clone(), args: c.args.clone(), @@ -456,14 +479,16 @@ pub fn rename_args_compileform(c: &CompileForm) -> Result = - map_m(&rename_args_helperform, &c.helpers)?; + let local_renamed_helpers: Vec = map_m(&rename_args_helperform, &c.helpers)?; let local_renamed_body = rename_args_bodyform(c.exp.borrow())?; Ok(CompileForm { loc: c.loc(), args: local_renamed_arg, include_forms: c.include_forms.clone(), - helpers: map_m(|x| rename_in_helperform(&local_namemap, x), &local_renamed_helpers)?, + helpers: map_m( + |x| rename_in_helperform(&local_namemap, x), + &local_renamed_helpers, + )?, exp: Rc::new(rename_in_bodyform( &local_namemap, Rc::new(local_renamed_body), From 041c2d4b5394182c176a77abc424779a376a01e2 Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 6 Sep 2023 09:55:45 -0700 Subject: [PATCH 108/117] Pull in full renaming for assign --- src/compiler/rename.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index e994e3be4..4caaf26e7 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -166,7 +166,7 @@ pub fn rename_assign_bindings( let sorted_bindings = toposort_assign_bindings(l, bindings)?; let mut renames = HashMap::new(); // Process in reverse order so we rename from inner to outer. - let bindings_to_rename: Vec> = sorted_bindings.iter().rev().cloned().collect(); + let bindings_to_rename: Vec> = sorted_bindings.iter().cloned().collect(); let renamed_bindings = map_m_reverse( |item: &TopoSortItem<_>| -> Result, CompileErr> { let b: &Binding = bindings[item.index].borrow(); @@ -328,7 +328,15 @@ fn rename_args_bodyform(b: &BodyForm) -> Result { )) } - BodyForm::Let(LetFormKind::Assign, _letdata) => Ok(b.clone()), + BodyForm::Let(LetFormKind::Assign, letdata) => { + let (new_compiled_body, new_bindings) = + rename_assign_bindings(&letdata.loc, &letdata.bindings, letdata.body.clone())?; + Ok(BodyForm::Let(LetFormKind::Assign, Box::new(LetData { + body: Rc::new(new_compiled_body), + bindings: new_bindings, + ..*letdata.clone() + }))) + } BodyForm::Quoted(e) => Ok(BodyForm::Quoted(e.clone())), BodyForm::Value(v) => Ok(BodyForm::Value(v.clone())), From ae94b0a60342025cb7d590c09286b9265ae781e7 Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 6 Sep 2023 09:57:37 -0700 Subject: [PATCH 109/117] fmt + clippy --- src/compiler/rename.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index 4caaf26e7..9bb6eb939 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -166,7 +166,7 @@ pub fn rename_assign_bindings( let sorted_bindings = toposort_assign_bindings(l, bindings)?; let mut renames = HashMap::new(); // Process in reverse order so we rename from inner to outer. - let bindings_to_rename: Vec> = sorted_bindings.iter().cloned().collect(); + let bindings_to_rename: Vec> = sorted_bindings.to_vec(); let renamed_bindings = map_m_reverse( |item: &TopoSortItem<_>| -> Result, CompileErr> { let b: &Binding = bindings[item.index].borrow(); @@ -331,11 +331,14 @@ fn rename_args_bodyform(b: &BodyForm) -> Result { BodyForm::Let(LetFormKind::Assign, letdata) => { let (new_compiled_body, new_bindings) = rename_assign_bindings(&letdata.loc, &letdata.bindings, letdata.body.clone())?; - Ok(BodyForm::Let(LetFormKind::Assign, Box::new(LetData { - body: Rc::new(new_compiled_body), - bindings: new_bindings, - ..*letdata.clone() - }))) + Ok(BodyForm::Let( + LetFormKind::Assign, + Box::new(LetData { + body: Rc::new(new_compiled_body), + bindings: new_bindings, + ..*letdata.clone() + }), + )) } BodyForm::Quoted(e) => Ok(BodyForm::Quoted(e.clone())), From 94e8537902a4698b74078a0f24dc5314b1716740 Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 6 Sep 2023 10:01:33 -0700 Subject: [PATCH 110/117] Smoothing it out --- src/compiler/rename.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index e994e3be4..c9d420c69 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -166,7 +166,7 @@ pub fn rename_assign_bindings( let sorted_bindings = toposort_assign_bindings(l, bindings)?; let mut renames = HashMap::new(); // Process in reverse order so we rename from inner to outer. - let bindings_to_rename: Vec> = sorted_bindings.iter().rev().cloned().collect(); + let bindings_to_rename: Vec> = sorted_bindings.to_vec(); let renamed_bindings = map_m_reverse( |item: &TopoSortItem<_>| -> Result, CompileErr> { let b: &Binding = bindings[item.index].borrow(); From ec1f93206160d7d815f045794b01d23826e862bd Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 6 Sep 2023 16:08:32 -0700 Subject: [PATCH 111/117] Also bug fix which ensures that inner desugars aren't captured by outer renaming --- src/compiler/rename.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index 9bb6eb939..1e82d6863 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -283,11 +283,13 @@ fn rename_args_bodyform(b: &BodyForm) -> Result { BodyForm::Let(LetFormKind::Sequential, letdata) => { // Renaming a sequential let is exactly as if the bindings were // nested in separate parallel lets. - rename_args_bodyform(&desugar_sequential_let_bindings( + let res = rename_args_bodyform(&desugar_sequential_let_bindings( &letdata.bindings, letdata.body.borrow(), letdata.bindings.len(), - )) + ))?; + eprintln!("sequentialized rename {}", res.to_sexp()); + Ok(res) } BodyForm::Let(LetFormKind::Parallel, letdata) => { @@ -317,15 +319,18 @@ fn rename_args_bodyform(b: &BodyForm) -> Result { }, &new_renamed_bindings, )?; - let locally_renamed_body = rename_in_bodyform(&local_namemap, letdata.body.clone())?; - Ok(BodyForm::Let( + let args_renamed = rename_args_bodyform(letdata.body.borrow())?; + let locally_renamed_body = rename_in_bodyform(&local_namemap, Rc::new(args_renamed))?; + let new_form = BodyForm::Let( LetFormKind::Parallel, Box::new(LetData { bindings: new_bindings, body: Rc::new(locally_renamed_body), ..*letdata.clone() }), - )) + ); + eprintln!("{} renamed to {}", b.to_sexp(), new_form.to_sexp()); + Ok(new_form) } BodyForm::Let(LetFormKind::Assign, letdata) => { From ea053b999a61fd1ffa717c7e6797332e36af63f0 Mon Sep 17 00:00:00 2001 From: Chris Marslender Date: Thu, 7 Sep 2023 11:46:39 -0500 Subject: [PATCH 112/117] change trigger to release published --- .github/workflows/build-arm64-wheels.yml | 4 ++-- .github/workflows/build-m1-wheel.yml | 4 ++-- .github/workflows/build-test.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-arm64-wheels.yml b/.github/workflows/build-arm64-wheels.yml index 9ee0dbe02..6d8a5f72c 100644 --- a/.github/workflows/build-arm64-wheels.yml +++ b/.github/workflows/build-arm64-wheels.yml @@ -5,8 +5,8 @@ on: branches: - main - dev - tags: - - '**' + release: + types: [published] pull_request: branches: - '**' diff --git a/.github/workflows/build-m1-wheel.yml b/.github/workflows/build-m1-wheel.yml index d9a6f7a5f..5831afae7 100644 --- a/.github/workflows/build-m1-wheel.yml +++ b/.github/workflows/build-m1-wheel.yml @@ -5,8 +5,8 @@ on: branches: - main - dev - tags: - - '**' + release: + types: [published] pull_request: branches: - '**' diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index cb9e698be..02c61ab73 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -6,8 +6,8 @@ on: branches: - base - dev - tags: - - '**' + release: + types: [published] pull_request: branches: - '**' From 290da3c9db3df173f818481d76a293a91d581fe4 Mon Sep 17 00:00:00 2001 From: arty Date: Thu, 7 Sep 2023 11:08:08 -0700 Subject: [PATCH 113/117] Remove spam --- src/compiler/rename.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index 1e82d6863..c21fa25b0 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -288,7 +288,6 @@ fn rename_args_bodyform(b: &BodyForm) -> Result { letdata.body.borrow(), letdata.bindings.len(), ))?; - eprintln!("sequentialized rename {}", res.to_sexp()); Ok(res) } @@ -329,7 +328,6 @@ fn rename_args_bodyform(b: &BodyForm) -> Result { ..*letdata.clone() }), ); - eprintln!("{} renamed to {}", b.to_sexp(), new_form.to_sexp()); Ok(new_form) } From 7270e0aaaee21a20f8547a4a72cc20ff8e44a13d Mon Sep 17 00:00:00 2001 From: arty Date: Thu, 7 Sep 2023 13:51:39 -0700 Subject: [PATCH 114/117] Add some tests along with this pr --- src/compiler/frontend.rs | 2 +- src/compiler/rename.rs | 2 +- src/tests/compiler/compiler.rs | 544 ++++++++++++++++++++++++++++++++- 3 files changed, 544 insertions(+), 4 deletions(-) diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index 341b1f7f7..ad24d0a24 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -15,7 +15,7 @@ use crate::compiler::sexp::{decode_string, enlist, SExp}; use crate::compiler::srcloc::Srcloc; use crate::util::u8_from_number; -fn collect_used_names_sexp(body: Rc) -> Vec> { +pub fn collect_used_names_sexp(body: Rc) -> Vec> { match body.borrow() { SExp::Atom(_, name) => vec![name.to_vec()], SExp::Cons(_, head, tail) => { diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index c9d420c69..f862372e7 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -36,7 +36,7 @@ fn rename_in_qq(namemap: &HashMap, Vec>, body: Rc) -> Rc } /* Given a cons cell, rename occurrences of oldname to newname */ -fn rename_in_cons( +pub fn rename_in_cons( namemap: &HashMap, Vec>, body: Rc, qq_handling: bool, diff --git a/src/tests/compiler/compiler.rs b/src/tests/compiler/compiler.rs index b013e8088..fe3284571 100644 --- a/src/tests/compiler/compiler.rs +++ b/src/tests/compiler/compiler.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use std::rc::Rc; use clvm_rs::allocator::Allocator; @@ -7,8 +7,10 @@ use crate::classic::clvm_tools::stages::stage_0::DefaultProgramRunner; use crate::compiler::clvm::run; use crate::compiler::compiler::{compile_file, DefaultCompilerOpts}; use crate::compiler::comptypes::{CompileErr, CompilerOpts}; +use crate::compiler::frontend::{collect_used_names_sexp, frontend}; +use crate::compiler::rename::rename_in_cons; use crate::compiler::runtypes::RunFailure; -use crate::compiler::sexp::{parse_sexp, SExp}; +use crate::compiler::sexp::{decode_string, parse_sexp, SExp}; use crate::compiler::srcloc::Srcloc; const TEST_TIMEOUT: usize = 1000000; @@ -62,6 +64,38 @@ pub fn run_string(content: &String, args: &String) -> Result, CompileEr run_string_maybe_opt(content, args, false) } +// Given some renaming that leaves behind gensym style names with _$_ in them, +// order them and use a locally predictable renaming scheme to give them a final +// test checkable value. +pub fn squash_name_differences(in_sexp: Rc) -> Result, String> { + let found_names_set: BTreeSet<_> = collect_used_names_sexp(in_sexp.clone()) + .into_iter() + .filter(|n| n.contains(&b'$')) + .collect(); + let mut found_names_progression = b'A'; + let mut replacement_map = HashMap::new(); + for found_name in found_names_set.iter() { + if let Some(located_dollar_part) = found_name.iter().position(|x| *x == b'$') { + let mut new_name: Vec = found_name + .iter() + .take(located_dollar_part + 2) + .copied() + .collect(); + new_name.push(found_names_progression); + found_names_progression += 1; + replacement_map.insert(found_name.clone(), new_name); + } else { + return Err(decode_string(&found_name)); + } + } + if replacement_map.len() != found_names_set.len() { + return Err(format!( + "mismatched lengths {replacement_map:?} vs {found_names_set:?}" + )); + } + Ok(rename_in_cons(&replacement_map, in_sexp, false)) +} + /* // Upcoming support for extra optimization (WIP) fn run_string_opt(content: &String, args: &String) -> Result, CompileErr> { run_string_maybe_opt(content, args, true) @@ -1437,6 +1471,393 @@ fn test_inline_out_of_bounds_diagnostic() { } } +#[test] +fn test_lambda_without_capture_from_function() { + let prog = indoc! {" +(mod (A B) + (include *standard-cl-21*) + (defun FOO () (lambda (X Y) (+ X Y))) + (a (FOO) (list A B)) + )"} + .to_string(); + let res = run_string(&prog, &"(3 4)".to_string()).unwrap(); + assert_eq!(res.to_string(), "7"); +} + +#[test] +fn test_lambda_without_capture() { + let prog = indoc! {" +(mod (A B) + (include *standard-cl-21*) + (a (lambda (X Y) (+ X Y)) (list A B)) + )"} + .to_string(); + let res = run_string(&prog, &"(3 4)".to_string()).unwrap(); + assert_eq!(res.to_string(), "7"); +} + +#[test] +fn test_lambda_with_capture_from_function() { + let prog = indoc! {" +(mod (A B) + (include *standard-cl-21*) + (defun FOO (Z) (lambda ((& Z) X) (- X Z))) + (a (FOO A) (list B)) + )"} + .to_string(); + let res = run_string(&prog, &"(5 19)".to_string()).unwrap(); + assert_eq!(res.to_string(), "14"); +} + +#[test] +fn test_lambda_with_capture() { + let prog = indoc! {" +(mod (A B) + (include *standard-cl-21*) + (a (lambda ((& A) Y) (- Y A)) (list B)) + )"} + .to_string(); + let res = run_string(&prog, &"(5 19)".to_string()).unwrap(); + assert_eq!(res.to_string(), "14"); +} + +#[test] +fn test_lambda_in_let_0() { + let prog = indoc! {" +(mod (A) + (include *standard-cl-21*) + (defun FOO (Z) + (let ((Q (* 2 Z))) + (lambda ((& Q)) (- 100 Q)) + ) + ) + (a (FOO A) ()) + )"} + .to_string(); + let res = run_string(&prog, &"(5)".to_string()).unwrap(); + assert_eq!(res.to_string(), "90"); +} + +#[test] +fn test_lambda_in_let_1() { + let prog = indoc! {" +(mod (A B) + (include *standard-cl-21*) + (defun FOO (Z) + (let ((Q (* 2 Z))) + (lambda ((& Q) X) (- X Q)) + ) + ) + (a (FOO A) (list B)) + )"} + .to_string(); + let res = run_string(&prog, &"(5 19)".to_string()).unwrap(); + assert_eq!(res.to_string(), "9"); +} + +#[test] +fn test_lambda_in_map() { + let prog = indoc! {" +(mod (add-number L) + + (include *standard-cl-21*) + + (defun map (F L) + (if L + (c (a F (list (f L))) (map F (r L))) + () + ) + ) + + (map + (lambda ((& add-number) number) (+ add-number number)) + L + ) + ) +"} + .to_string(); + let res = run_string(&prog, &"(5 (1 2 3 4))".to_string()).unwrap(); + assert_eq!(res.to_string(), "(6 7 8 9)"); +} + +#[test] +fn test_lambda_in_map_with_let_surrounding() { + let prog = indoc! {" +(mod (add-number L) + + (include *standard-cl-21*) + + (defun map (F L) + (if L + (c (a F (list (f L))) (map F (r L))) + () + ) + ) + + (map + (let ((A (* add-number 2))) + (lambda ((& A) number) (+ A number)) + ) + L + ) + ) +"} + .to_string(); + let res = run_string(&prog, &"(5 (1 2 3 4))".to_string()).unwrap(); + assert_eq!(res.to_string(), "(11 12 13 14)"); +} + +#[test] +fn test_map_with_lambda_function_from_env_and_bindings() { + let prog = indoc! {" + (mod (add-number L) + + (include *standard-cl-21*) + + (defun map (F L) + (if L + (c (a F (list (f L))) (map F (r L))) + () + ) + ) + + (defun add-twice (X Y) (+ (* 2 X) Y)) + + (map + (lambda ((& add-number) number) (add-twice add-number number)) + L + ) + )"} + .to_string(); + let res = run_string(&prog, &"(5 (1 2 3 4))".to_string()).unwrap(); + assert_eq!(res.to_string(), "(11 12 13 14)"); +} + +#[test] +fn test_map_with_lambda_function_from_env_no_bindings() { + let prog = indoc! {" + (mod (L) + + (include *standard-cl-21*) + + (defun map (F L) + (if L + (c (a F (list (f L))) (map F (r L))) + () + ) + ) + + (defun sum-list (L) + (if L + (+ (f L) (sum-list (r L))) + () + ) + ) + + (map + (lambda (lst) (sum-list lst)) + L + ) + )"} + .to_string(); + let res = run_string(&prog, &"(((5 10 15) (2 4 8) (3 6 9)))".to_string()).unwrap(); + assert_eq!(res.to_string(), "(30 14 18)"); +} + +#[test] +fn test_lambda_using_let() { + let prog = indoc! {" + (mod (P L) + + (include *standard-cl-21*) + + (defun map (F L) + (if L + (c (a F (list (f L))) (map F (r L))) + () + ) + ) + + (map + (lambda ((& P) item) (let ((composed (c P item))) composed)) + L + ) + )"} + .to_string(); + let res = run_string(&prog, &"(1 (10 20 30))".to_string()).unwrap(); + assert_eq!(res.to_string(), "((1 . 10) (1 . 20) (1 . 30))"); +} + +#[test] +fn test_lambda_using_macro() { + let prog = indoc! {" + (mod (P L) + + (include *standard-cl-21*) + + (defun map (F L) + (if L + (c (a F (list (f L))) (map F (r L))) + () + ) + ) + + (map + (lambda ((& P) item) (list P item)) + L + ) + )"} + .to_string(); + let res = run_string(&prog, &"(1 (10 20 30))".to_string()).unwrap(); + assert_eq!(res.to_string(), "((1 10) (1 20) (1 30))"); +} + +#[test] +fn test_lambda_reduce() { + let prog = indoc! {" + (mod (LST) + (include *standard-cl-21*) + (defun reduce (fun lst init) + (if lst + (reduce fun (r lst) (a fun (list (f lst) init))) + init + ) + ) + + (let + ((capture 100)) + (reduce (lambda ((& capture) (X Y) ACC) (+ (* X Y) ACC capture)) LST 0) + ) + ) + "} + .to_string(); + let res = run_string(&prog, &"(((2 3) (4 9)))".to_string()).unwrap(); + assert_eq!(res.to_string(), "242"); +} + +#[test] +fn test_lambda_as_let_binding() { + let prog = indoc! {" + (mod (P L) + (defun map (F L) + (if L (c (a F (list (f L))) (map F (r L))) ()) + ) + (defun x2 (N) (* 2 N)) + (defun x3p1 (N) (+ 1 (* 3 N))) + (let* ((H (lambda (N) (x2 N))) + (G (lambda (N) (x3p1 N))) + (F (if P G H))) + (map F L) + ) + ) + "} + .to_string(); + let res0 = run_string(&prog, &"(0 (1 2 3))".to_string()).unwrap(); + assert_eq!(res0.to_string(), "(2 4 6)"); + let res1 = run_string(&prog, &"(1 (1 2 3))".to_string()).unwrap(); + assert_eq!(res1.to_string(), "(4 7 10)"); +} + +#[test] +fn test_lambda_mixed_let_binding() { + let prog = indoc! {" + (mod (P L) + (defun map (F L) + (if L (c (a F (list (f L))) (map F (r L))) ()) + ) + (defun x2 (N) (* 2 N)) + (defun x3p1 (N) (+ 1 (* 3 N))) + (let* ((G (lambda (N) (x3p1 N))) + (F (if P G (lambda (N) (x2 N))))) + (map F L) + ) + ) + "} + .to_string(); + let res0 = run_string(&prog, &"(0 (1 2 3))".to_string()).unwrap(); + assert_eq!(res0.to_string(), "(2 4 6)"); + let res1 = run_string(&prog, &"(1 (1 2 3))".to_string()).unwrap(); + assert_eq!(res1.to_string(), "(4 7 10)"); +} + +#[test] +fn test_lambda_hof_1() { + let prog = indoc! {" + (mod (P) + (a (a (lambda ((& P) X) (lambda ((& P X)) (+ P X))) (list 3)) ()) + ) + "} + .to_string(); + let res = run_string(&prog, &"(1)".to_string()).unwrap(); + assert_eq!(res.to_string(), "4"); +} + +#[test] +fn test_lambda_as_argument_to_macro() { + let prog = indoc! {" + (mod (P) + (defun map-f (A L) + (if L (c (a (f L) A) (map-f A (r L))) ()) + ) + (let ((Fs (list (lambda (X) (- X 1)) (lambda (X) (+ X 1)) (lambda (X) (* 2 X)))) + (args (list P))) + (map-f args Fs) + ) + ) + "} + .to_string(); + let res = run_string(&prog, &"(10)".to_string()).unwrap(); + assert_eq!(res.to_string(), "(9 11 20)"); +} + +#[test] +fn test_lambda_as_argument_to_macro_with_inner_let() { + let prog = indoc! {" + (mod (P) + (defun map-f (A L) + (if L (c (a (f L) A) (map-f A (r L))) ()) + ) + (let ((Fs (list (lambda (X) (let ((N (* X 3))) N)) (lambda (X) (+ X 1)) (lambda (X) (* 2 X)))) + (args (list P))) + (map-f args Fs) + ) + ) + "} + .to_string(); + let res = run_string(&prog, &"(10)".to_string()).unwrap(); + assert_eq!(res.to_string(), "(30 11 20)"); +} + +#[test] +fn test_treat_function_name_as_value() { + let prog = indoc! {" +(mod (X) + (include *standard-cl-21*) + (defun G (X) (* 2 X)) + (defun F (X) (G (+ 1 X))) + (a F (list X)) +) + "} + .to_string(); + let res = run_string(&prog, &"(99)".to_string()).expect("should compile"); + assert_eq!(res.to_string(), "200"); +} + +#[test] +fn test_treat_function_name_as_value_filter() { + let prog = indoc! {" + (mod L + (include *standard-cl-21*) + (defun greater-than-3 (X) (> X 3)) + (defun filter (F L) (let ((rest (filter F (r L)))) (if L (if (a F (list (f L))) (c (f L) rest) rest) ()))) + (filter greater-than-3 L) + ) + "} + .to_string(); + let res = run_string(&prog, &"(1 2 3 4 5)".to_string()).expect("should compile"); + assert_eq!(res.to_string(), "(4 5)"); +} + #[test] fn test_inline_in_assign_not_actually_recursive() { let prog = indoc! {" @@ -1491,3 +1912,122 @@ fn test_simple_rest_call_inline() { let res = run_string(&prog, &"(13 99 144)".to_string()).expect("should compile and run"); assert_eq!(res.to_string(), "768"); } + +#[test] +fn test_simple_rest_lambda() { + let prog = indoc! {" +(mod (Z X) + (include *standard-cl-21*) + + (defun silly-lambda-consumer (Q F) (a F (list Q))) + + (silly-lambda-consumer &rest (list X (lambda ((& Z) X) (* Z X)))) + )"} + .to_string(); + let res = run_string(&prog, &"(13 51)".to_string()).expect("should compile and run"); + assert_eq!(res.to_string(), "663"); +} + +#[test] +fn test_lambda_in_lambda() { + let prog = indoc! {" +(mod (Z X) + (include *standard-cl-21*) + + (defun silly-lambda-consumer (Q F) (a F (list Q))) + + (a (silly-lambda-consumer X (lambda ((& Z) X) (lambda ((& Z X)) (* Z X)))) ()) + )"} + .to_string(); + let res = run_string(&prog, &"(13 51)".to_string()).expect("should compile and run"); + assert_eq!(res.to_string(), "663"); +} + +#[test] +fn test_let_in_rest_0() { + let prog = indoc! {" +(mod (Z X) + (include *standard-cl-21*) + + (defun F (X) (+ X 3)) + + (F &rest (list (let ((Q (* X Z))) (+ Q 99)))) + )"} + .to_string(); + let res = run_string(&prog, &"(3 2)".to_string()).expect("should compile and run"); + assert_eq!(res.to_string(), "108"); +} + +#[test] +fn test_let_in_rest_1() { + let prog = indoc! {" +(mod (Z X) + (include *standard-cl-21*) + + (defun F (X) (+ X 3)) + + (F &rest (let ((Q (* X Z))) (list (+ Q 99)))) + )"} + .to_string(); + let res = run_string(&prog, &"(3 2)".to_string()).expect("should compile and run"); + assert_eq!(res.to_string(), "108"); +} + +#[test] +fn test_rename_in_compileform_run() { + let prog = indoc! {" +(mod (X) + (include *standard-cl-21*) + + (defun F overridden + (let + ((overridden (* 3 (f overridden))) ;; overridden = 33 + (y (f (r overridden))) ;; y = 13 + (z (f (r (r overridden))))) ;; z = 17 + (+ overridden z y) ;; 33 + 13 + 17 = 63 + ) + ) + + (F X 13 17) + )"} + .to_string(); + + let res = run_string(&prog, &"(11)".to_string()).expect("should compile and run"); + assert_eq!(res.to_string(), "63"); +} + +#[test] +fn test_rename_in_compileform_simple() { + let prog = indoc! {" +(mod (X) + (include *standard-cl-21*) + + (defun F overridden + (let + ((overridden (* 3 (f overridden))) ;; overridden = 33 + (y (f (r overridden))) ;; y = 11 + (z (f (r (r overridden))))) ;; z = 17 + (+ overridden z y) ;; 33 11 17 + ) + ) + + (F X 13 17) + )"} + .to_string(); + // Note: renames use gensym so they're unique but not spot predictable. + // + // We'll rename them in detection order to a specific set of names and should + // get for F: + // + let desired_outcome = "(defun F overridden_$_A (let ((overridden_$_B (* 3 (f overridden_$_A))) (y_$_C (f (r overridden_$_A))) (z_$_D (f (r (r overridden_$_A))))) (+ overridden_$_B z_$_D y_$_C)))"; + let parsed = parse_sexp(Srcloc::start("*test*"), prog.bytes()).expect("should parse"); + let opts: Rc = Rc::new(DefaultCompilerOpts::new(&"*test*".to_string())); + let compiled = frontend(opts, &parsed).expect("should compile"); + let helper_f: Vec<_> = compiled + .helpers + .iter() + .filter(|f| f.name() == b"F") + .collect(); + let renamed_helperform = squash_name_differences(helper_f[0].to_sexp()).expect("should rename"); + assert_eq!(renamed_helperform.to_string(), desired_outcome); +} From 02db72c4de2207d64fc3966432bdc132ef3b51cc Mon Sep 17 00:00:00 2001 From: arty Date: Thu, 7 Sep 2023 14:03:34 -0700 Subject: [PATCH 115/117] Remove leftover lambda tests --- src/tests/compiler/compiler.rs | 417 --------------------------------- 1 file changed, 417 deletions(-) diff --git a/src/tests/compiler/compiler.rs b/src/tests/compiler/compiler.rs index fe3284571..ea916eb48 100644 --- a/src/tests/compiler/compiler.rs +++ b/src/tests/compiler/compiler.rs @@ -1471,393 +1471,6 @@ fn test_inline_out_of_bounds_diagnostic() { } } -#[test] -fn test_lambda_without_capture_from_function() { - let prog = indoc! {" -(mod (A B) - (include *standard-cl-21*) - (defun FOO () (lambda (X Y) (+ X Y))) - (a (FOO) (list A B)) - )"} - .to_string(); - let res = run_string(&prog, &"(3 4)".to_string()).unwrap(); - assert_eq!(res.to_string(), "7"); -} - -#[test] -fn test_lambda_without_capture() { - let prog = indoc! {" -(mod (A B) - (include *standard-cl-21*) - (a (lambda (X Y) (+ X Y)) (list A B)) - )"} - .to_string(); - let res = run_string(&prog, &"(3 4)".to_string()).unwrap(); - assert_eq!(res.to_string(), "7"); -} - -#[test] -fn test_lambda_with_capture_from_function() { - let prog = indoc! {" -(mod (A B) - (include *standard-cl-21*) - (defun FOO (Z) (lambda ((& Z) X) (- X Z))) - (a (FOO A) (list B)) - )"} - .to_string(); - let res = run_string(&prog, &"(5 19)".to_string()).unwrap(); - assert_eq!(res.to_string(), "14"); -} - -#[test] -fn test_lambda_with_capture() { - let prog = indoc! {" -(mod (A B) - (include *standard-cl-21*) - (a (lambda ((& A) Y) (- Y A)) (list B)) - )"} - .to_string(); - let res = run_string(&prog, &"(5 19)".to_string()).unwrap(); - assert_eq!(res.to_string(), "14"); -} - -#[test] -fn test_lambda_in_let_0() { - let prog = indoc! {" -(mod (A) - (include *standard-cl-21*) - (defun FOO (Z) - (let ((Q (* 2 Z))) - (lambda ((& Q)) (- 100 Q)) - ) - ) - (a (FOO A) ()) - )"} - .to_string(); - let res = run_string(&prog, &"(5)".to_string()).unwrap(); - assert_eq!(res.to_string(), "90"); -} - -#[test] -fn test_lambda_in_let_1() { - let prog = indoc! {" -(mod (A B) - (include *standard-cl-21*) - (defun FOO (Z) - (let ((Q (* 2 Z))) - (lambda ((& Q) X) (- X Q)) - ) - ) - (a (FOO A) (list B)) - )"} - .to_string(); - let res = run_string(&prog, &"(5 19)".to_string()).unwrap(); - assert_eq!(res.to_string(), "9"); -} - -#[test] -fn test_lambda_in_map() { - let prog = indoc! {" -(mod (add-number L) - - (include *standard-cl-21*) - - (defun map (F L) - (if L - (c (a F (list (f L))) (map F (r L))) - () - ) - ) - - (map - (lambda ((& add-number) number) (+ add-number number)) - L - ) - ) -"} - .to_string(); - let res = run_string(&prog, &"(5 (1 2 3 4))".to_string()).unwrap(); - assert_eq!(res.to_string(), "(6 7 8 9)"); -} - -#[test] -fn test_lambda_in_map_with_let_surrounding() { - let prog = indoc! {" -(mod (add-number L) - - (include *standard-cl-21*) - - (defun map (F L) - (if L - (c (a F (list (f L))) (map F (r L))) - () - ) - ) - - (map - (let ((A (* add-number 2))) - (lambda ((& A) number) (+ A number)) - ) - L - ) - ) -"} - .to_string(); - let res = run_string(&prog, &"(5 (1 2 3 4))".to_string()).unwrap(); - assert_eq!(res.to_string(), "(11 12 13 14)"); -} - -#[test] -fn test_map_with_lambda_function_from_env_and_bindings() { - let prog = indoc! {" - (mod (add-number L) - - (include *standard-cl-21*) - - (defun map (F L) - (if L - (c (a F (list (f L))) (map F (r L))) - () - ) - ) - - (defun add-twice (X Y) (+ (* 2 X) Y)) - - (map - (lambda ((& add-number) number) (add-twice add-number number)) - L - ) - )"} - .to_string(); - let res = run_string(&prog, &"(5 (1 2 3 4))".to_string()).unwrap(); - assert_eq!(res.to_string(), "(11 12 13 14)"); -} - -#[test] -fn test_map_with_lambda_function_from_env_no_bindings() { - let prog = indoc! {" - (mod (L) - - (include *standard-cl-21*) - - (defun map (F L) - (if L - (c (a F (list (f L))) (map F (r L))) - () - ) - ) - - (defun sum-list (L) - (if L - (+ (f L) (sum-list (r L))) - () - ) - ) - - (map - (lambda (lst) (sum-list lst)) - L - ) - )"} - .to_string(); - let res = run_string(&prog, &"(((5 10 15) (2 4 8) (3 6 9)))".to_string()).unwrap(); - assert_eq!(res.to_string(), "(30 14 18)"); -} - -#[test] -fn test_lambda_using_let() { - let prog = indoc! {" - (mod (P L) - - (include *standard-cl-21*) - - (defun map (F L) - (if L - (c (a F (list (f L))) (map F (r L))) - () - ) - ) - - (map - (lambda ((& P) item) (let ((composed (c P item))) composed)) - L - ) - )"} - .to_string(); - let res = run_string(&prog, &"(1 (10 20 30))".to_string()).unwrap(); - assert_eq!(res.to_string(), "((1 . 10) (1 . 20) (1 . 30))"); -} - -#[test] -fn test_lambda_using_macro() { - let prog = indoc! {" - (mod (P L) - - (include *standard-cl-21*) - - (defun map (F L) - (if L - (c (a F (list (f L))) (map F (r L))) - () - ) - ) - - (map - (lambda ((& P) item) (list P item)) - L - ) - )"} - .to_string(); - let res = run_string(&prog, &"(1 (10 20 30))".to_string()).unwrap(); - assert_eq!(res.to_string(), "((1 10) (1 20) (1 30))"); -} - -#[test] -fn test_lambda_reduce() { - let prog = indoc! {" - (mod (LST) - (include *standard-cl-21*) - (defun reduce (fun lst init) - (if lst - (reduce fun (r lst) (a fun (list (f lst) init))) - init - ) - ) - - (let - ((capture 100)) - (reduce (lambda ((& capture) (X Y) ACC) (+ (* X Y) ACC capture)) LST 0) - ) - ) - "} - .to_string(); - let res = run_string(&prog, &"(((2 3) (4 9)))".to_string()).unwrap(); - assert_eq!(res.to_string(), "242"); -} - -#[test] -fn test_lambda_as_let_binding() { - let prog = indoc! {" - (mod (P L) - (defun map (F L) - (if L (c (a F (list (f L))) (map F (r L))) ()) - ) - (defun x2 (N) (* 2 N)) - (defun x3p1 (N) (+ 1 (* 3 N))) - (let* ((H (lambda (N) (x2 N))) - (G (lambda (N) (x3p1 N))) - (F (if P G H))) - (map F L) - ) - ) - "} - .to_string(); - let res0 = run_string(&prog, &"(0 (1 2 3))".to_string()).unwrap(); - assert_eq!(res0.to_string(), "(2 4 6)"); - let res1 = run_string(&prog, &"(1 (1 2 3))".to_string()).unwrap(); - assert_eq!(res1.to_string(), "(4 7 10)"); -} - -#[test] -fn test_lambda_mixed_let_binding() { - let prog = indoc! {" - (mod (P L) - (defun map (F L) - (if L (c (a F (list (f L))) (map F (r L))) ()) - ) - (defun x2 (N) (* 2 N)) - (defun x3p1 (N) (+ 1 (* 3 N))) - (let* ((G (lambda (N) (x3p1 N))) - (F (if P G (lambda (N) (x2 N))))) - (map F L) - ) - ) - "} - .to_string(); - let res0 = run_string(&prog, &"(0 (1 2 3))".to_string()).unwrap(); - assert_eq!(res0.to_string(), "(2 4 6)"); - let res1 = run_string(&prog, &"(1 (1 2 3))".to_string()).unwrap(); - assert_eq!(res1.to_string(), "(4 7 10)"); -} - -#[test] -fn test_lambda_hof_1() { - let prog = indoc! {" - (mod (P) - (a (a (lambda ((& P) X) (lambda ((& P X)) (+ P X))) (list 3)) ()) - ) - "} - .to_string(); - let res = run_string(&prog, &"(1)".to_string()).unwrap(); - assert_eq!(res.to_string(), "4"); -} - -#[test] -fn test_lambda_as_argument_to_macro() { - let prog = indoc! {" - (mod (P) - (defun map-f (A L) - (if L (c (a (f L) A) (map-f A (r L))) ()) - ) - (let ((Fs (list (lambda (X) (- X 1)) (lambda (X) (+ X 1)) (lambda (X) (* 2 X)))) - (args (list P))) - (map-f args Fs) - ) - ) - "} - .to_string(); - let res = run_string(&prog, &"(10)".to_string()).unwrap(); - assert_eq!(res.to_string(), "(9 11 20)"); -} - -#[test] -fn test_lambda_as_argument_to_macro_with_inner_let() { - let prog = indoc! {" - (mod (P) - (defun map-f (A L) - (if L (c (a (f L) A) (map-f A (r L))) ()) - ) - (let ((Fs (list (lambda (X) (let ((N (* X 3))) N)) (lambda (X) (+ X 1)) (lambda (X) (* 2 X)))) - (args (list P))) - (map-f args Fs) - ) - ) - "} - .to_string(); - let res = run_string(&prog, &"(10)".to_string()).unwrap(); - assert_eq!(res.to_string(), "(30 11 20)"); -} - -#[test] -fn test_treat_function_name_as_value() { - let prog = indoc! {" -(mod (X) - (include *standard-cl-21*) - (defun G (X) (* 2 X)) - (defun F (X) (G (+ 1 X))) - (a F (list X)) -) - "} - .to_string(); - let res = run_string(&prog, &"(99)".to_string()).expect("should compile"); - assert_eq!(res.to_string(), "200"); -} - -#[test] -fn test_treat_function_name_as_value_filter() { - let prog = indoc! {" - (mod L - (include *standard-cl-21*) - (defun greater-than-3 (X) (> X 3)) - (defun filter (F L) (let ((rest (filter F (r L)))) (if L (if (a F (list (f L))) (c (f L) rest) rest) ()))) - (filter greater-than-3 L) - ) - "} - .to_string(); - let res = run_string(&prog, &"(1 2 3 4 5)".to_string()).expect("should compile"); - assert_eq!(res.to_string(), "(4 5)"); -} - #[test] fn test_inline_in_assign_not_actually_recursive() { let prog = indoc! {" @@ -1913,36 +1526,6 @@ fn test_simple_rest_call_inline() { assert_eq!(res.to_string(), "768"); } -#[test] -fn test_simple_rest_lambda() { - let prog = indoc! {" -(mod (Z X) - (include *standard-cl-21*) - - (defun silly-lambda-consumer (Q F) (a F (list Q))) - - (silly-lambda-consumer &rest (list X (lambda ((& Z) X) (* Z X)))) - )"} - .to_string(); - let res = run_string(&prog, &"(13 51)".to_string()).expect("should compile and run"); - assert_eq!(res.to_string(), "663"); -} - -#[test] -fn test_lambda_in_lambda() { - let prog = indoc! {" -(mod (Z X) - (include *standard-cl-21*) - - (defun silly-lambda-consumer (Q F) (a F (list Q))) - - (a (silly-lambda-consumer X (lambda ((& Z) X) (lambda ((& Z X)) (* Z X)))) ()) - )"} - .to_string(); - let res = run_string(&prog, &"(13 51)".to_string()).expect("should compile and run"); - assert_eq!(res.to_string(), "663"); -} - #[test] fn test_let_in_rest_0() { let prog = indoc! {" From 6ebcf6463634146b35ad4a311161aec52839b5a1 Mon Sep 17 00:00:00 2001 From: Rigidity Date: Sun, 10 Sep 2023 17:07:17 -0400 Subject: [PATCH 116/117] Bump clvmr to 0.3.0 --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- src/classic/clvm/serialize.rs | 2 +- src/classic/clvm/sexp.rs | 12 +++++----- src/classic/clvm_tools/binutils.rs | 4 ++-- src/classic/clvm_tools/debug.rs | 2 +- src/classic/clvm_tools/pattern_match.rs | 10 ++++----- src/classic/clvm_tools/sha256tree.rs | 2 +- .../clvm_tools/stages/stage_2/compile.rs | 20 ++++++++--------- .../clvm_tools/stages/stage_2/inline.rs | 8 +++---- .../clvm_tools/stages/stage_2/module.rs | 12 +++++----- .../clvm_tools/stages/stage_2/operators.rs | 10 ++++----- .../clvm_tools/stages/stage_2/optimize.rs | 22 +++++++++---------- .../clvm_tools/stages/stage_2/reader.rs | 2 +- src/compiler/clvm.rs | 6 ++--- src/compiler/dialect.rs | 2 +- src/tests/classic/smoke.rs | 12 +++++----- wasm/Cargo.toml | 2 +- 18 files changed, 67 insertions(+), 67 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 165533baa..2afc51d86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -151,9 +151,9 @@ dependencies = [ [[package]] name = "clvmr" -version = "0.2.7" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2890f01537f1be43d2767ae71bbba0d0b3543dbb1ee892092d0ed4d913227fc" +checksum = "9cd344b6dc76235f446025fe9ebe54aa6131e2e59acb49e16be48a3bb3492491" dependencies = [ "bls12_381", "getrandom", diff --git a/Cargo.toml b/Cargo.toml index 195c56297..43ad94367 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ do-notation = "0.1.3" serde_json = "1.0" sha2 = "0.9.5" tempfile = "3.3.0" -clvmr = { version = "0.2.6", features = ["pre-eval"] } +clvmr = { version = "0.3.0", features = ["pre-eval"] } binascii = "0.1.4" yaml-rust = "0.4" linked-hash-map = "0.5.6" diff --git a/src/classic/clvm/serialize.rs b/src/classic/clvm/serialize.rs index 05246789a..f863a42df 100644 --- a/src/classic/clvm/serialize.rs +++ b/src/classic/clvm/serialize.rs @@ -98,7 +98,7 @@ impl<'a> Iterator for SExpToBytesIterator<'a> { fn next(&mut self) -> Option { self.state.pop().and_then(|step| match step { SExpToByteOp::Object(x) => match self.allocator.sexp(x) { - SExp::Atom() => { + SExp::Atom => { // The only node we have in scope is x, so this atom // capture is trivial. let buf = self.allocator.atom(x).to_vec(); diff --git a/src/classic/clvm/sexp.rs b/src/classic/clvm/sexp.rs index 4757014b4..fd26de905 100644 --- a/src/classic/clvm/sexp.rs +++ b/src/classic/clvm/sexp.rs @@ -150,7 +150,7 @@ pub fn to_sexp_type(allocator: &mut Allocator, value: CastableType) -> Result { + SExp::Atom => { return Err(EvalErr( *target_value, "attempt to set_pair in atom".to_string(), @@ -336,7 +336,7 @@ pub fn non_nil(allocator: &Allocator, sexp: NodePtr) -> bool { match allocator.sexp(sexp) { SExp::Pair(_, _) => true, // sexp is the only node in scope, was !is_empty - SExp::Atom() => allocator.atom_len(sexp) != 0, + SExp::Atom => allocator.atom_len(sexp) != 0, } } @@ -356,7 +356,7 @@ pub fn rest(allocator: &Allocator, sexp: NodePtr) -> Result { pub fn atom(allocator: &Allocator, sexp: NodePtr) -> Result, EvalErr> { match allocator.sexp(sexp) { - SExp::Atom() => Ok(allocator.atom(sexp).to_vec()), // only sexp in scope + SExp::Atom => Ok(allocator.atom(sexp).to_vec()), // only sexp in scope _ => Err(EvalErr(sexp, "not an atom".to_string())), } } @@ -366,7 +366,7 @@ pub fn proper_list(allocator: &Allocator, sexp: NodePtr, store: bool) -> Option< let mut args_sexp = sexp; loop { match allocator.sexp(args_sexp) { - SExp::Atom() => { + SExp::Atom => { if !non_nil(allocator, args_sexp) { return Some(args); } else { @@ -454,7 +454,7 @@ pub fn equal_to(allocator: &mut Allocator, first_: NodePtr, second_: NodePtr) -> return true; } match (allocator.sexp(first), allocator.sexp(second)) { - (SExp::Atom(), SExp::Atom()) => { + (SExp::Atom, SExp::Atom) => { // two atoms in scope, both are used return allocator.atom(first) == allocator.atom(second); } @@ -477,7 +477,7 @@ pub fn flatten(allocator: &mut Allocator, tree_: NodePtr, res: &mut Vec loop { match allocator.sexp(tree) { - SExp::Atom() => { + SExp::Atom => { if non_nil(allocator, tree) { res.push(tree); } diff --git a/src/classic/clvm_tools/binutils.rs b/src/classic/clvm_tools/binutils.rs index 670d70e52..505123057 100644 --- a/src/classic/clvm_tools/binutils.rs +++ b/src/classic/clvm_tools/binutils.rs @@ -128,7 +128,7 @@ pub fn disassemble_to_ir_with_kw( IRRepr::Cons(Rc::new(v0), Rc::new(v1)) } - SExp::Atom() => { + SExp::Atom => { // sexp is the only node in scope. let bytes = Bytes::new(Some(BytesFromType::Raw(allocator.atom(sexp).to_vec()))); ir_for_atom(&bytes, allow_keyword, keyword_from_atom) @@ -141,7 +141,7 @@ pub fn disassemble_with_kw( sexp: NodePtr, keyword_from_atom: &Record, String>, ) -> String { - let with_keywords = !matches!(allocator.sexp(sexp), SExp::Atom()); + let with_keywords = !matches!(allocator.sexp(sexp), SExp::Atom); let symbols = disassemble_to_ir_with_kw(allocator, sexp, keyword_from_atom, with_keywords); write_ir(Rc::new(symbols)) } diff --git a/src/classic/clvm_tools/debug.rs b/src/classic/clvm_tools/debug.rs index 25767e05d..15bd5142a 100644 --- a/src/classic/clvm_tools/debug.rs +++ b/src/classic/clvm_tools/debug.rs @@ -180,7 +180,7 @@ fn table_trace( ) { let (sexp, args) = match allocator.sexp(form) { SExp::Pair(sexp, args) => (sexp, args), - SExp::Atom() => (form, allocator.null()), + SExp::Atom => (form, allocator.null()), }; stdout.write_str(&format!("exp: {}\n", disassemble_f(allocator, sexp))); diff --git a/src/classic/clvm_tools/pattern_match.rs b/src/classic/clvm_tools/pattern_match.rs index 0df1345cd..02e894b4e 100644 --- a/src/classic/clvm_tools/pattern_match.rs +++ b/src/classic/clvm_tools/pattern_match.rs @@ -51,7 +51,7 @@ pub fn match_sexp( */ match (allocator.sexp(pattern), allocator.sexp(sexp)) { - (SExp::Atom(), SExp::Atom()) => { + (SExp::Atom, SExp::Atom) => { // Two nodes in scope, both used. if allocator.atom(pattern) == allocator.atom(sexp) { Some(known_bindings) @@ -60,10 +60,10 @@ pub fn match_sexp( } } (SExp::Pair(pleft, pright), _) => match (allocator.sexp(pleft), allocator.sexp(pright)) { - (SExp::Atom(), SExp::Atom()) => { + (SExp::Atom, SExp::Atom) => { let pright_atom = allocator.atom(pright).to_vec(); match allocator.sexp(sexp) { - SExp::Atom() => { + SExp::Atom => { // Expression is ($ . $), sexp is '$', result: no capture. // Avoid double borrow. if allocator.atom(pleft) == ATOM_MATCH { @@ -114,11 +114,11 @@ pub fn match_sexp( } } _ => match allocator.sexp(sexp) { - SExp::Atom() => None, + SExp::Atom => None, SExp::Pair(sleft, sright) => match_sexp(allocator, pleft, sleft, known_bindings) .and_then(|new_bindings| match_sexp(allocator, pright, sright, new_bindings)), }, }, - (SExp::Atom(), _) => None, + (SExp::Atom, _) => None, } } diff --git a/src/classic/clvm_tools/sha256tree.rs b/src/classic/clvm_tools/sha256tree.rs index 0212d0c78..ebbc3197c 100644 --- a/src/classic/clvm_tools/sha256tree.rs +++ b/src/classic/clvm_tools/sha256tree.rs @@ -34,7 +34,7 @@ pub fn sha256tree(allocator: &mut Allocator, v: NodePtr) -> Bytes { .concat(&right), ) } - SExp::Atom() => sha256( + SExp::Atom => sha256( Bytes::new(Some(BytesFromType::Raw(vec![1]))).concat(&Bytes::new(Some( // only v in scope. BytesFromType::Raw(allocator.atom(v).to_vec()), diff --git a/src/classic/clvm_tools/stages/stage_2/compile.rs b/src/classic/clvm_tools/stages/stage_2/compile.rs index c71e18340..27d8c3f04 100644 --- a/src/classic/clvm_tools/stages/stage_2/compile.rs +++ b/src/classic/clvm_tools/stages/stage_2/compile.rs @@ -122,12 +122,12 @@ pub fn compile_qq( }; match allocator.sexp(sexp) { - SExp::Atom() => { + SExp::Atom => { // (qq ATOM) => (q . ATOM) quote(allocator, sexp) } SExp::Pair(op, sexp_rest) => { - if let SExp::Atom() = allocator.sexp(op) { + if let SExp::Atom = allocator.sexp(op) { // opbuf => op if allocator.atom(op).to_vec() == qq_atom() { return m! { @@ -211,7 +211,7 @@ fn lower_quote_(allocator: &mut Allocator, prog: NodePtr) -> Result { + SExp::Atom => { // was macro_name, but it's singular and probably // not useful to rename. if allocator.atom(mp_list[0]) == operator { @@ -385,7 +385,7 @@ fn transform_program_atom( let value = if v.len() > 1 { v[1] } else { allocator.null() }; match allocator.sexp(v[0]) { - SExp::Atom() => { + SExp::Atom => { // v[0] is close by, and probably not useful to // rename here. if allocator.atom(v[0]) == a { @@ -462,7 +462,7 @@ fn find_symbol_match( } match allocator.sexp(symdef[0]) { - SExp::Atom() => { + SExp::Atom => { let symbol = symdef[0]; let value = if symdef.len() == 1 { allocator.null() @@ -638,7 +638,7 @@ fn do_com_prog_( // quote atoms match allocator.sexp(prog) { - SExp::Atom() => { + SExp::Atom => { // Note: can't co-borrow with allocator below. let prog_bytes = allocator.atom(prog).to_vec(); transform_program_atom( @@ -650,7 +650,7 @@ fn do_com_prog_( }, SExp::Pair(operator,prog_rest) => { match allocator.sexp(operator) { - SExp::Atom() => { + SExp::Atom => { // Note: can't co-borrow with allocator below. let opbuf = allocator.atom(operator).to_vec(); get_macro_program(allocator, &opbuf, macro_lookup). @@ -783,7 +783,7 @@ pub fn get_compile_filename( return Ok(None); } - if let SExp::Atom() = allocator.sexp(cvt_prog_result) { + if let SExp::Atom = allocator.sexp(cvt_prog_result) { // only cvt_prog_result in scope. let abuf = allocator.atom(cvt_prog_result).to_vec(); return Ok(Some(Bytes::new(Some(BytesFromType::Raw(abuf))).decode())); @@ -806,7 +806,7 @@ pub fn get_search_paths( let mut res = Vec::new(); if let Some(l) = proper_list(allocator, search_path_result.1, true) { for elt in l.iter().copied() { - if let SExp::Atom() = allocator.sexp(elt) { + if let SExp::Atom = allocator.sexp(elt) { // Only elt in scope. res.push( Bytes::new(Some(BytesFromType::Raw(allocator.atom(elt).to_vec()))).decode(), diff --git a/src/classic/clvm_tools/stages/stage_2/inline.rs b/src/classic/clvm_tools/stages/stage_2/inline.rs index d0db520f1..215c3679d 100644 --- a/src/classic/clvm_tools/stages/stage_2/inline.rs +++ b/src/classic/clvm_tools/stages/stage_2/inline.rs @@ -15,7 +15,7 @@ pub fn is_at_capture( tree_first: NodePtr, tree_rest: NodePtr, ) -> Option<(NodePtr, NodePtr)> { - if let (SExp::Atom(), Some(spec)) = ( + if let (SExp::Atom, Some(spec)) = ( allocator.sexp(tree_first), proper_list(allocator, tree_rest, true), ) { @@ -88,7 +88,7 @@ fn formulate_path_selections_for_destructuring_arg( SExp::Pair(a, b) => { let next_depth = arg_depth.clone() * 2_u32.to_bigint().unwrap(); if let Some((capture, substructure)) = is_at_capture(allocator, a, b) { - if let SExp::Atom() = allocator.sexp(capture) { + if let SExp::Atom = allocator.sexp(capture) { let (new_arg_path, new_arg_depth, tail) = if let Some(prev_ref) = referenced_from { (arg_path, arg_depth, prev_ref) @@ -147,7 +147,7 @@ fn formulate_path_selections_for_destructuring_arg( ) } } - SExp::Atom() => { + SExp::Atom => { // Note: can't co-borrow with allocator below. let buf = allocator.atom(arg_sexp).to_vec(); if !buf.is_empty() { @@ -225,7 +225,7 @@ pub fn formulate_path_selections_for_destructuring( ) -> Result { if let SExp::Pair(a, b) = allocator.sexp(args_sexp) { if let Some((capture, substructure)) = is_at_capture(allocator, a, b) { - if let SExp::Atom() = allocator.sexp(capture) { + if let SExp::Atom = allocator.sexp(capture) { let quoted_arg_list = wrap_in_unquote(allocator, capture)?; let tail = wrap_in_compile_time_list(allocator, quoted_arg_list)?; // Was: cbuf from capture. diff --git a/src/classic/clvm_tools/stages/stage_2/module.rs b/src/classic/clvm_tools/stages/stage_2/module.rs index d905fad28..74083616a 100644 --- a/src/classic/clvm_tools/stages/stage_2/module.rs +++ b/src/classic/clvm_tools/stages/stage_2/module.rs @@ -146,7 +146,7 @@ fn build_used_constants_names( let matching_names = matching_names_1.iter().filter_map(|v| { // Only v usefully in scope. - if let SExp::Atom() = allocator.sexp(*v) { + if let SExp::Atom = allocator.sexp(*v) { Some(allocator.atom(*v).to_vec()) } else { None @@ -224,7 +224,7 @@ fn unquote_args( matches: &HashMap, NodePtr>, ) -> Result { match allocator.sexp(code) { - SExp::Atom() => { + SExp::Atom => { // Only code in scope. let code_atom = allocator.atom(code); let matching_args = args @@ -286,7 +286,7 @@ fn defun_inline_to_macro( let arg_name_list = arg_atom_list .iter() .filter_map(|x| { - if let SExp::Atom() = allocator.sexp(*x) { + if let SExp::Atom = allocator.sexp(*x) { // only x usefully in scope. Some(allocator.atom(*x)) } else { @@ -326,12 +326,12 @@ fn parse_mod_sexp( let op = match allocator.sexp(op_node) { // op_node in use. - SExp::Atom() => allocator.atom(op_node).to_vec(), + SExp::Atom => allocator.atom(op_node).to_vec(), _ => Vec::new(), }; let name = match allocator.sexp(name_node) { // name_node in use. - SExp::Atom() => allocator.atom(name_node).to_vec(), + SExp::Atom => allocator.atom(name_node).to_vec(), _ => Vec::new(), }; @@ -551,7 +551,7 @@ fn symbol_table_for_tree( } match allocator.sexp(tree) { - SExp::Atom() => Ok(vec![(tree, root_node.as_path().data().to_vec())]), + SExp::Atom => Ok(vec![(tree, root_node.as_path().data().to_vec())]), SExp::Pair(_, _) => { let left_bytes = NodePath::new(None).first(); let right_bytes = NodePath::new(None).rest(); diff --git a/src/classic/clvm_tools/stages/stage_2/operators.rs b/src/classic/clvm_tools/stages/stage_2/operators.rs index e95e48eb3..182147cb9 100644 --- a/src/classic/clvm_tools/stages/stage_2/operators.rs +++ b/src/classic/clvm_tools/stages/stage_2/operators.rs @@ -205,7 +205,7 @@ impl CompilerOperatorsInternal { match allocator.sexp(sexp) { SExp::Pair(f, _) => match allocator.sexp(f) { - SExp::Atom() => { + SExp::Atom => { let filename = Bytes::new(Some(BytesFromType::Raw(allocator.atom(f).to_vec()))).decode(); // Use the read interface in CompilerOpts if we have one. @@ -238,7 +238,7 @@ impl CompilerOperatorsInternal { fn write(&self, allocator: &Allocator, sexp: NodePtr) -> Response { if let SExp::Pair(filename_sexp, r) = allocator.sexp(sexp) { if let SExp::Pair(data, _) = allocator.sexp(r) { - if let SExp::Atom() = allocator.sexp(filename_sexp) { + if let SExp::Atom = allocator.sexp(filename_sexp) { let filename_buf = allocator.atom(filename_sexp); let filename_bytes = Bytes::new(Some(BytesFromType::Raw(filename_buf.to_vec()))); @@ -285,7 +285,7 @@ impl CompilerOperatorsInternal { }; if let SExp::Pair(l, _r) = allocator.sexp(sexp) { - if let SExp::Atom() = allocator.sexp(l) { + if let SExp::Atom = allocator.sexp(l) { // l most relevant in scope. let filename = Bytes::new(Some(BytesFromType::Raw(allocator.atom(l).to_vec()))).decode(); @@ -319,7 +319,7 @@ impl CompilerOperatorsInternal { { for kv in symtable.iter() { if let SExp::Pair(hash, name) = allocator.sexp(*kv) { - if let (SExp::Atom(), SExp::Atom()) = + if let (SExp::Atom, SExp::Atom) = (allocator.sexp(hash), allocator.sexp(name)) { // hash and name in scope. @@ -389,7 +389,7 @@ impl Dialect for CompilerOperatorsInternal { let extensions_to_clvmr_during_compile = self.get_operators_extension(); match allocator.sexp(op) { - SExp::Atom() => { + SExp::Atom => { // use of op obvious. let opbuf = allocator.atom(op); if opbuf == "_read".as_bytes() { diff --git a/src/classic/clvm_tools/stages/stage_2/optimize.rs b/src/classic/clvm_tools/stages/stage_2/optimize.rs index 64519c8c4..9aef8e77c 100644 --- a/src/classic/clvm_tools/stages/stage_2/optimize.rs +++ b/src/classic/clvm_tools/stages/stage_2/optimize.rs @@ -41,7 +41,7 @@ pub fn seems_constant_tail(allocator: &mut Allocator, sexp_: NodePtr) -> bool { sexp = r; } - SExp::Atom() => { + SExp::Atom => { return sexp == allocator.null(); } } @@ -50,12 +50,12 @@ pub fn seems_constant_tail(allocator: &mut Allocator, sexp_: NodePtr) -> bool { pub fn seems_constant(allocator: &mut Allocator, sexp: NodePtr) -> bool { match allocator.sexp(sexp) { - SExp::Atom() => { + SExp::Atom => { return sexp == allocator.null(); } SExp::Pair(operator, r) => { match allocator.sexp(operator) { - SExp::Atom() => { + SExp::Atom => { // Was buf of operator. let atom = allocator.atom(operator); if atom.len() == 1 && atom[0] == 1 { @@ -93,7 +93,7 @@ pub fn constant_optimizer( */ if let SExp::Pair(first, _) = allocator.sexp(r) { // first relevant in scope. - if let SExp::Atom() = allocator.sexp(first) { + if let SExp::Atom = allocator.sexp(first) { let buf = allocator.atom(first); if buf.len() == 1 && buf[0] == 1 { // Short circuit already quoted expression. @@ -137,7 +137,7 @@ pub fn constant_optimizer( } pub fn is_args_call(allocator: &Allocator, r: NodePtr) -> bool { - if let SExp::Atom() = allocator.sexp(r) { + if let SExp::Atom = allocator.sexp(r) { // Only r in scope. let buf = allocator.atom(r); buf.len() == 1 && buf[0] == 1 @@ -220,7 +220,7 @@ fn path_from_args( new_args: NodePtr, ) -> Result { match allocator.sexp(sexp) { - SExp::Atom() => { + SExp::Atom => { // Only sexp in scope. let v = number_from_u8(allocator.atom(sexp)); if v <= bi_one() { @@ -246,7 +246,7 @@ pub fn sub_args( new_args: NodePtr, ) -> Result { match allocator.sexp(sexp) { - SExp::Atom() => path_from_args(allocator, sexp, new_args), + SExp::Atom => path_from_args(allocator, sexp, new_args), SExp::Pair(first_pre, rest) => { let first; @@ -254,7 +254,7 @@ pub fn sub_args( SExp::Pair(_, _) => { first = sub_args(allocator, first_pre, new_args)?; } - SExp::Atom() => { + SExp::Atom => { // Atom is a reflection of first_pre. let atom = allocator.atom(first_pre); if atom.len() == 1 && atom[0] == 1 { @@ -374,7 +374,7 @@ pub fn var_change_optimizer_cons_eval( } let increment = match allocator.sexp(val) { SExp::Pair(val_first, _) => match allocator.sexp(val_first) { - SExp::Atom() => { + SExp::Atom => { // Atom reflects val_first. let vf_buf = allocator.atom(val_first); (vf_buf.len() != 1 || vf_buf[0] != 1) as i32 @@ -419,7 +419,7 @@ pub fn children_optimizer( if list.is_empty() { return Ok(r); } - if let SExp::Atom() = allocator.sexp(list[0]) { + if let SExp::Atom = allocator.sexp(list[0]) { if allocator.atom(list[0]).to_vec() == vec![1] { return Ok(r); } @@ -683,7 +683,7 @@ pub fn optimize_sexp_( let mut name = "".to_string(); match allocator.sexp(r) { - SExp::Atom() => { + SExp::Atom => { return Ok(r); } SExp::Pair(_, _) => { diff --git a/src/classic/clvm_tools/stages/stage_2/reader.rs b/src/classic/clvm_tools/stages/stage_2/reader.rs index bc91c89b4..716988f75 100644 --- a/src/classic/clvm_tools/stages/stage_2/reader.rs +++ b/src/classic/clvm_tools/stages/stage_2/reader.rs @@ -90,7 +90,7 @@ pub fn process_embed_file( )); } - if let (SExp::Atom(), SExp::Atom(), SExp::Atom()) = ( + if let (SExp::Atom, SExp::Atom, SExp::Atom) = ( allocator.sexp(l[0]), allocator.sexp(l[1]), allocator.sexp(l[2]), diff --git a/src/compiler/clvm.rs b/src/compiler/clvm.rs index f7d499678..f0020efab 100644 --- a/src/compiler/clvm.rs +++ b/src/compiler/clvm.rs @@ -228,9 +228,9 @@ pub fn convert_to_clvm_rs( }) } } - SExp::Cons(_, a, b) => convert_to_clvm_rs(allocator, a.clone()).and_then(|head| { + SExp::Cons(_, a, b) => convert_to_clvm_rs(allocator, a.clone()).and_then(|head_ptr| { convert_to_clvm_rs(allocator, b.clone()).and_then(|tail| { - allocator.new_pair(head, tail).map_err(|_e| { + allocator.new_pair(head_ptr, tail).map_err(|_e| { RunFailure::RunErr(a.loc(), format!("failed to alloc cons {head}")) }) }) @@ -245,7 +245,7 @@ pub fn convert_from_clvm_rs( head: NodePtr, ) -> Result, RunFailure> { match allocator.sexp(head) { - allocator::SExp::Atom() => { + allocator::SExp::Atom => { let atom_data = allocator.atom(head); if atom_data.is_empty() { Ok(Rc::new(SExp::Nil(loc))) diff --git a/src/compiler/dialect.rs b/src/compiler/dialect.rs index 58b561993..305a0048c 100644 --- a/src/compiler/dialect.rs +++ b/src/compiler/dialect.rs @@ -55,7 +55,7 @@ lazy_static! { fn include_dialect(allocator: &Allocator, e: &[NodePtr]) -> Option { let include_keyword_sexp = e[0]; let name_sexp = e[1]; - if let (SExp::Atom(), SExp::Atom()) = ( + if let (SExp::Atom, SExp::Atom) = ( allocator.sexp(include_keyword_sexp), allocator.sexp(name_sexp), ) { diff --git a/src/tests/classic/smoke.rs b/src/tests/classic/smoke.rs index c3d7ad72e..98e5187cd 100644 --- a/src/tests/classic/smoke.rs +++ b/src/tests/classic/smoke.rs @@ -148,7 +148,7 @@ fn mid_negative_value_bin() { Box::new(SimpleCreateCLVMObject {}), ) .expect("should be able to make nodeptr"); - if let SExp::Atom() = allocator.sexp(atom.1) { + if let SExp::Atom = allocator.sexp(atom.1) { let res_bytes = allocator.atom(atom.1); assert_eq!(res_bytes, &[0xff, 0xff]); } else { @@ -276,7 +276,7 @@ fn can_run_from_source_nil() { let mut allocator = Allocator::new(); let res = run_from_source(&mut allocator, "()".to_string()); match allocator.sexp(res) { - SExp::Atom() => { + SExp::Atom => { let res_bytes = allocator.atom(res); assert_eq!(res_bytes.len(), 0); } @@ -291,7 +291,7 @@ fn can_echo_quoted_nil() { let mut allocator = Allocator::new(); let res = run_from_source(&mut allocator, "(1)".to_string()); match allocator.sexp(res) { - SExp::Atom() => { + SExp::Atom => { let res_bytes = allocator.atom(res); assert_eq!(res_bytes.len(), 0); } @@ -322,7 +322,7 @@ fn can_echo_quoted_atom() { let mut allocator = Allocator::new(); let res = run_from_source(&mut allocator, "(1 . 3)".to_string()); match allocator.sexp(res) { - SExp::Atom() => { + SExp::Atom => { let res_bytes = allocator.atom(res); assert_eq!(res_bytes.len(), 1); assert_eq!(res_bytes[0], 3); @@ -338,7 +338,7 @@ fn can_do_operations() { let mut allocator = Allocator::new(); let res = run_from_source(&mut allocator, "(16 (1 . 3) (1 . 5))".to_string()); match allocator.sexp(res) { - SExp::Atom() => { + SExp::Atom => { let res_bytes = allocator.atom(res); assert_eq!(res_bytes.len(), 1); assert_eq!(res_bytes[0], 8); @@ -354,7 +354,7 @@ fn can_do_operations_kw() { let mut allocator = Allocator::new(); let res = run_from_source(&mut allocator, "(+ (q . 3) (q . 5))".to_string()); match allocator.sexp(res) { - SExp::Atom() => { + SExp::Atom => { let res_bytes = allocator.atom(res); assert_eq!(res_bytes.len(), 1); assert_eq!(res_bytes[0], 8); diff --git a/wasm/Cargo.toml b/wasm/Cargo.toml index 95c8cc9d2..25eca57c8 100644 --- a/wasm/Cargo.toml +++ b/wasm/Cargo.toml @@ -18,7 +18,7 @@ path = "src/mod.rs" [dependencies] clvm_tools_rs = { path= "..", features = [] } -clvmr = { version = "0.2.5", features = ["pre-eval"] } +clvmr = { version = "0.3.0", features = ["pre-eval"] } wasm-bindgen = "=0.2.83" wasm-bindgen-test = "=0.3.25" js-sys = "0.3.60" From 7ac4c8522db2de5dc4ff0b63e31b924d79d4aa25 Mon Sep 17 00:00:00 2001 From: Rigidity Date: Tue, 12 Sep 2023 10:28:24 -0400 Subject: [PATCH 117/117] Fmt --- src/classic/clvm_tools/stages/stage_2/operators.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/classic/clvm_tools/stages/stage_2/operators.rs b/src/classic/clvm_tools/stages/stage_2/operators.rs index 182147cb9..2e69564d3 100644 --- a/src/classic/clvm_tools/stages/stage_2/operators.rs +++ b/src/classic/clvm_tools/stages/stage_2/operators.rs @@ -319,9 +319,7 @@ impl CompilerOperatorsInternal { { for kv in symtable.iter() { if let SExp::Pair(hash, name) = allocator.sexp(*kv) { - if let (SExp::Atom, SExp::Atom) = - (allocator.sexp(hash), allocator.sexp(name)) - { + if let (SExp::Atom, SExp::Atom) = (allocator.sexp(hash), allocator.sexp(name)) { // hash and name in scope. let hash_text = Bytes::new(Some(BytesFromType::Raw(allocator.atom(hash).to_vec())))