From b17048d68ac6b5557589e8fd66fbc269167b0917 Mon Sep 17 00:00:00 2001 From: arty Date: Thu, 4 Jul 2024 18:30:47 -0700 Subject: [PATCH 01/14] More elegant parsing, make an object of the common outputs --- src/classic/clvm_tools/cmds.rs | 269 +++++++++++++++++---------------- 1 file changed, 140 insertions(+), 129 deletions(-) diff --git a/src/classic/clvm_tools/cmds.rs b/src/classic/clvm_tools/cmds.rs index 21a2dcc5d..b502c6ec1 100644 --- a/src/classic/clvm_tools/cmds.rs +++ b/src/classic/clvm_tools/cmds.rs @@ -496,6 +496,82 @@ pub fn cldb_hierarchy( result } +fn get_string_and_filename_with_default( + parsed_args: &HashMap, + argument: &str, + default: Option<&str>, +) -> Option<(Option, String)> { + if let Some(ArgumentValue::ArgString(path, content)) = parsed_args.get(argument) { + Some((path.clone(), content.to_string())) + } else { + default.map(|p| (None, p.to_string())) + } +} + +struct ParsedInputPathOrCode { + path: Option, + content: String, + parsed: NodePtr, +} + +fn parse_tool_input_sexp( + allocator: &mut Allocator, + argument: &str, + parsed_args: &HashMap, + default_hex: Option<&str>, + default_sexp: Option<&str>, +) -> Result { + match parsed_args.get("hex") { + Some(_) => { + let (path, use_sexp_text) = if let Some(r) = + get_string_and_filename_with_default(parsed_args, argument, default_hex) + { + r + } else { + return Err("missing argument {argument}".to_string()); + }; + + let sexp_serialized = + Bytes::new_validated(Some(UnvalidatedBytesFromType::Hex(use_sexp_text.clone()))) + .map_err(|e| format!("{e:?}"))?; + + let mut prog_stream = Stream::new(Some(sexp_serialized)); + + sexp_from_stream( + allocator, + &mut prog_stream, + Box::new(SimpleCreateCLVMObject {}), + ) + .map(|x| ParsedInputPathOrCode { + path, + content: use_sexp_text, + parsed: x.1, + }) + .map_err(|e| format!("{e:?}")) + } + _ => { + let (path, use_sexp_text) = if let Some(r) = + get_string_and_filename_with_default(parsed_args, argument, default_sexp) + { + r + } else { + return Err(format!("missing argument {argument}")); + }; + + read_ir(&use_sexp_text) + .map_err(|e| format!("{e:?}")) + .and_then(|v| { + Ok(ParsedInputPathOrCode { + path, + content: use_sexp_text, + parsed: assemble_from_ir(allocator, Rc::new(v)) + .map_err(|e| format!("{e:?}"))?, + }) + }) + } + } +} + pub fn cldb(args: &[String]) { let tool_name = "cldb".to_string(); let props = TArgumentParserProps { @@ -558,14 +634,10 @@ pub fn cldb(args: &[String]) { ); let arg_vec = args[1..].to_vec(); - let mut input_file = None; - let mut input_program = "()".to_string(); - let prog_srcloc = Srcloc::start("*program*"); let args_srcloc = Srcloc::start("*args*"); let mut args = Rc::new(sexp::SExp::atom_from_string(args_srcloc.clone(), "")); - let mut parsed_args_result: String = "".to_string(); let parsed_args: HashMap = match parser.parse_args(&arg_vec) { Err(e) => { @@ -583,16 +655,40 @@ pub fn cldb(args: &[String]) { } } - if let Some(ArgumentValue::ArgString(file, path_or_code)) = parsed_args.get("path_or_code") { - input_file.clone_from(file); - input_program = path_or_code.to_string(); - } + let mut allocator = Allocator::new(); + let mut output = Vec::new(); - if let Some(ArgumentValue::ArgString(_, s)) = parsed_args.get("env") { - parsed_args_result = s.to_string(); - } + let errorize = + |output: &mut Vec>, location: Option, c: &str| { + let mut parse_error = BTreeMap::new(); + if let Some(l) = location { + parse_error.insert( + "Error-Location".to_string(), + YamlElement::String(l.to_string()), + ); + }; + parse_error.insert("Error".to_string(), YamlElement::String(c.to_string())); + output.push(parse_error.clone()); + println!("{}", yamlette_string(output)); + }; - let mut allocator = Allocator::new(); + let parsed_program_result = + match parse_tool_input_sexp(&mut allocator, "path_or_code", &parsed_args, None, None) { + Ok(r) => r, + Err(e) => { + errorize(&mut output, None, &e.to_string()); + return; + } + }; + + let parsed_args_result = + match parse_tool_input_sexp(&mut allocator, "env", &parsed_args, Some("80"), Some("()")) { + Ok(r) => r, + Err(e) => { + errorize(&mut output, None, &e.to_string()); + return; + } + }; let symbol_table = parsed_args .get("symbol_table") @@ -612,7 +708,8 @@ pub fn cldb(args: &[String]) { .map(|x| matches!(x, ArgumentValue::ArgBool(true))) .unwrap_or_else(|| false); let runner = Rc::new(DefaultProgramRunner::new()); - let use_filename = input_file + let use_filename = parsed_program_result + .path .clone() .unwrap_or_else(|| "*command*".to_string()); let opts = Rc::new(DefaultCompilerOpts::new(&use_filename)) @@ -620,14 +717,13 @@ pub fn cldb(args: &[String]) { .set_search_paths(&search_paths); let mut use_symbol_table = symbol_table.unwrap_or_default(); - let mut output = Vec::new(); let res = match parsed_args.get("hex") { Some(ArgumentValue::ArgBool(true)) => hex_to_modern_sexp( &mut allocator, &use_symbol_table, prog_srcloc.clone(), - &input_program, + &parsed_program_result.content, ) .map_err(|_| CompileErr(prog_srcloc, "Failed to parse hex".to_string())), _ => { @@ -637,7 +733,7 @@ pub fn cldb(args: &[String]) { &mut allocator, runner.clone(), opts.clone(), - &input_program, + &parsed_program_result.content, &mut use_symbol_table, ); unopt_res.and_then(|x| { @@ -655,14 +751,7 @@ pub fn cldb(args: &[String]) { let program = match res { Ok(r) => r, Err(c) => { - let mut parse_error = BTreeMap::new(); - parse_error.insert( - "Error-Location".to_string(), - YamlElement::String(c.0.to_string()), - ); - parse_error.insert("Error".to_string(), YamlElement::String(c.1)); - output.push(parse_error.clone()); - println!("{}", yamlette_string(&output)); + errorize(&mut output, Some(c.0.clone()), &c.1); return; } }; @@ -673,7 +762,7 @@ pub fn cldb(args: &[String]) { &mut allocator, &HashMap::new(), args_srcloc, - &parsed_args_result, + &parsed_args_result.content, ) { Ok(r) => { args = r; @@ -687,7 +776,7 @@ pub fn cldb(args: &[String]) { } } } - _ => match parse_sexp(Srcloc::start("*arg*"), parsed_args_result.bytes()) { + _ => match parse_sexp(Srcloc::start("*arg*"), parsed_args_result.content.bytes()) { Ok(r) => { if !r.is_empty() { args = r[0].clone(); @@ -711,10 +800,15 @@ pub fn cldb(args: &[String]) { for p in prims::prims().iter() { prim_map.insert(p.0.clone(), Rc::new(p.1.clone())); } - let program_lines: Rc> = - Rc::new(input_program.lines().map(|x| x.to_string()).collect()); + let program_lines: Rc> = Rc::new( + parsed_program_result + .content + .lines() + .map(|x| x.to_string()) + .collect(), + ); let cldbenv = CldbRunEnv::new( - input_file.clone(), + parsed_program_result.path.clone(), program_lines.clone(), Box::new(CldbNoOverride::new_symbols(use_symbol_table.clone())), ); @@ -723,7 +817,7 @@ pub fn cldb(args: &[String]) { let result = cldb_hierarchy( runner, Rc::new(prim_map), - input_file, + parsed_program_result.path.clone(), program_lines, Rc::new(use_symbol_table), program, @@ -1142,14 +1236,8 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul let mut allocator = Allocator::new(); - let input_serialized = None; - let mut input_sexp: Option = None; - let time_start = SystemTime::now(); - let mut time_read_hex = SystemTime::now(); - let mut time_assemble = SystemTime::now(); - - let mut input_args = "()".to_string(); + let time_read_hex = SystemTime::now(); if let ( Some(ArgumentValue::ArgBool(true)), @@ -1178,11 +1266,6 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul return; } - if let Some(ArgumentValue::ArgString(file, path_or_code)) = parsed_args.get("env") { - input_file.clone_from(file); - input_args = path_or_code.to_string(); - } - let special_runner = run_program_for_search_paths(&reported_input_file, &search_paths, extra_symbol_info); // Ensure we know the user's wishes about the disassembly version here. @@ -1190,88 +1273,26 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul let dpr = special_runner.clone(); let run_program = special_runner; - match parsed_args.get("hex") { - Some(_) => { - let assembled_serialized = Bytes::new_validated(Some(UnvalidatedBytesFromType::Hex( - input_program.to_string(), - ))); - - let env_serialized = if input_args.is_empty() { - Bytes::new_validated(Some(UnvalidatedBytesFromType::Hex("80".to_string()))) - } else { - Bytes::new_validated(Some(UnvalidatedBytesFromType::Hex(input_args))) - }; - - let ee = match env_serialized { - Ok(x) => x, - Err(e) => { - stdout.write_str(&format!("FAIL: {e}\n")); - return; - } - }; - time_read_hex = SystemTime::now(); - - let mut prog_stream = Stream::new(Some(match assembled_serialized { - Ok(x) => x, - Err(e) => { - stdout.write_str(&format!("FAIL: {e}\n")); - return; - } - })); - - let input_prog_sexp = sexp_from_stream( - &mut allocator, - &mut prog_stream, - Box::new(SimpleCreateCLVMObject {}), - ) - .map(|x| Some(x.1)) - .unwrap(); - - let mut arg_stream = Stream::new(Some(ee)); - let input_arg_sexp = sexp_from_stream( - &mut allocator, - &mut arg_stream, - Box::new(SimpleCreateCLVMObject {}), - ) - .map(|x| Some(x.1)) - .unwrap(); - if let (Some(ip), Some(ia)) = (input_prog_sexp, input_arg_sexp) { - input_sexp = allocator.new_pair(ip, ia).ok(); - } - } - _ => { - let src_sexp; - if let Some(ArgumentValue::ArgString(f, content)) = parsed_args.get("path_or_code") { - match read_ir(content) { - Ok(s) => { - input_program.clone_from(content); - input_file.clone_from(f); - src_sexp = s; - } - Err(e) => { - stdout.write_str(&format!("FAIL: {e}\n")); - return; - } - } - } else { - stdout.write_str(&format!("FAIL: {}\n", "non-string argument")); + let assembled_sexp = + match parse_tool_input_sexp(&mut allocator, "path_or_code", &parsed_args, None, None) { + Ok(s) => s, + Err(e) => { + stdout.write_str(&format!("FAIL: {e}\n")); return; } + }; - let assembled_sexp = assemble_from_ir(&mut allocator, Rc::new(src_sexp)).unwrap(); - let mut parsed_args_result = "()".to_string(); - - if let Some(ArgumentValue::ArgString(_f, s)) = parsed_args.get("env") { - parsed_args_result = s.to_string(); - } + let env = match parse_tool_input_sexp(&mut allocator, "env", &parsed_args, None, None) { + Ok(s) => s, + Err(e) => { + stdout.write_str(&format!("FAIL: {e}\n")); + return; + } + }; - let env_ir = read_ir(&parsed_args_result).unwrap(); - let env = assemble_from_ir(&mut allocator, Rc::new(env_ir)).unwrap(); - time_assemble = SystemTime::now(); + let time_assemble = SystemTime::now(); - input_sexp = allocator.new_pair(assembled_sexp, env).ok(); - } - } + let input_sexp = allocator.new_pair(assembled_sexp.parsed, env.parsed).ok(); // Symbol table related checks: should one be loaded, should one be saved. // This code is confusingly woven due to 'run' and 'brun' serving many roles. @@ -1488,16 +1509,6 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul .unwrap_or_else(|| 0); let max_cost = max(0, max_cost); - if input_sexp.is_none() { - input_sexp = sexp_from_stream( - &mut allocator, - &mut Stream::new(input_serialized), - Box::new(SimpleCreateCLVMObject {}), - ) - .map(|x| Some(x.1)) - .unwrap(); - }; - // Part 2 of doing pre_eval: Have a thing that receives the messages and // performs some action. let log_entries_clone = log_entries.clone(); From 3d05a623c789f0ef50a09a40f2760a88ff6928e5 Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 5 Jul 2024 11:34:13 -0700 Subject: [PATCH 02/14] Break out all determination of dialect into something common between cldb and run --- src/classic/clvm_tools/cmds.rs | 136 +++++---------------------- src/classic/clvm_tools/comp_input.rs | 126 +++++++++++++++++++++++++ src/classic/clvm_tools/mod.rs | 1 + 3 files changed, 151 insertions(+), 112 deletions(-) create mode 100644 src/classic/clvm_tools/comp_input.rs diff --git a/src/classic/clvm_tools/cmds.rs b/src/classic/clvm_tools/cmds.rs index b502c6ec1..d2dd0c816 100644 --- a/src/classic/clvm_tools/cmds.rs +++ b/src/classic/clvm_tools/cmds.rs @@ -30,6 +30,7 @@ use crate::classic::clvm::sexp::{enlist, proper_list, sexp_as_bin}; use crate::classic::clvm::OPERATORS_LATEST_VERSION; use crate::classic::clvm_tools::binutils::{assemble_from_ir, disassemble, disassemble_with_kw}; use crate::classic::clvm_tools::clvmc::write_sym_output; +use crate::classic::clvm_tools::comp_input::RunAndCompileInputData; use crate::classic::clvm_tools::debug::check_unused; use crate::classic::clvm_tools::debug::{ program_hash_from_program_env_cons, start_log_after, trace_pre_eval, trace_to_table, @@ -55,7 +56,6 @@ use crate::compiler::clvm::start_step; use crate::compiler::compiler::{compile_file, DefaultCompilerOpts}; use crate::compiler::comptypes::{CompileErr, CompilerOpts}; use crate::compiler::debug::build_symbol_table_mut; -use crate::compiler::dialect::detect_modern; use crate::compiler::frontend::frontend; use crate::compiler::optimize::maybe_finalize_program_via_classic_optimizer; use crate::compiler::preprocessor::gather_dependencies; @@ -496,82 +496,6 @@ pub fn cldb_hierarchy( result } -fn get_string_and_filename_with_default( - parsed_args: &HashMap, - argument: &str, - default: Option<&str>, -) -> Option<(Option, String)> { - if let Some(ArgumentValue::ArgString(path, content)) = parsed_args.get(argument) { - Some((path.clone(), content.to_string())) - } else { - default.map(|p| (None, p.to_string())) - } -} - -struct ParsedInputPathOrCode { - path: Option, - content: String, - parsed: NodePtr, -} - -fn parse_tool_input_sexp( - allocator: &mut Allocator, - argument: &str, - parsed_args: &HashMap, - default_hex: Option<&str>, - default_sexp: Option<&str>, -) -> Result { - match parsed_args.get("hex") { - Some(_) => { - let (path, use_sexp_text) = if let Some(r) = - get_string_and_filename_with_default(parsed_args, argument, default_hex) - { - r - } else { - return Err("missing argument {argument}".to_string()); - }; - - let sexp_serialized = - Bytes::new_validated(Some(UnvalidatedBytesFromType::Hex(use_sexp_text.clone()))) - .map_err(|e| format!("{e:?}"))?; - - let mut prog_stream = Stream::new(Some(sexp_serialized)); - - sexp_from_stream( - allocator, - &mut prog_stream, - Box::new(SimpleCreateCLVMObject {}), - ) - .map(|x| ParsedInputPathOrCode { - path, - content: use_sexp_text, - parsed: x.1, - }) - .map_err(|e| format!("{e:?}")) - } - _ => { - let (path, use_sexp_text) = if let Some(r) = - get_string_and_filename_with_default(parsed_args, argument, default_sexp) - { - r - } else { - return Err(format!("missing argument {argument}")); - }; - - read_ir(&use_sexp_text) - .map_err(|e| format!("{e:?}")) - .and_then(|v| { - Ok(ParsedInputPathOrCode { - path, - content: use_sexp_text, - parsed: assemble_from_ir(allocator, Rc::new(v)) - .map_err(|e| format!("{e:?}"))?, - }) - }) - } - } -} - pub fn cldb(args: &[String]) { let tool_name = "cldb".to_string(); let props = TArgumentParserProps { @@ -672,17 +596,11 @@ pub fn cldb(args: &[String]) { println!("{}", yamlette_string(output)); }; - let parsed_program_result = - match parse_tool_input_sexp(&mut allocator, "path_or_code", &parsed_args, None, None) { - Ok(r) => r, - Err(e) => { - errorize(&mut output, None, &e.to_string()); - return; - } - }; - - let parsed_args_result = - match parse_tool_input_sexp(&mut allocator, "env", &parsed_args, Some("80"), Some("()")) { + let parsed = + match RunAndCompileInputData::new( + &mut allocator, + &parsed_args + ) { Ok(r) => r, Err(e) => { errorize(&mut output, None, &e.to_string()); @@ -708,7 +626,7 @@ pub fn cldb(args: &[String]) { .map(|x| matches!(x, ArgumentValue::ArgBool(true))) .unwrap_or_else(|| false); let runner = Rc::new(DefaultProgramRunner::new()); - let use_filename = parsed_program_result + let use_filename = parsed.program .path .clone() .unwrap_or_else(|| "*command*".to_string()); @@ -723,7 +641,7 @@ pub fn cldb(args: &[String]) { &mut allocator, &use_symbol_table, prog_srcloc.clone(), - &parsed_program_result.content, + &parsed.program.content, ) .map_err(|_| CompileErr(prog_srcloc, "Failed to parse hex".to_string())), _ => { @@ -733,7 +651,7 @@ pub fn cldb(args: &[String]) { &mut allocator, runner.clone(), opts.clone(), - &parsed_program_result.content, + &parsed.program.content, &mut use_symbol_table, ); unopt_res.and_then(|x| { @@ -762,7 +680,7 @@ pub fn cldb(args: &[String]) { &mut allocator, &HashMap::new(), args_srcloc, - &parsed_args_result.content, + &parsed.args.content, ) { Ok(r) => { args = r; @@ -776,7 +694,7 @@ pub fn cldb(args: &[String]) { } } } - _ => match parse_sexp(Srcloc::start("*arg*"), parsed_args_result.content.bytes()) { + _ => match parse_sexp(Srcloc::start("*arg*"), parsed.args.content.bytes()) { Ok(r) => { if !r.is_empty() { args = r[0].clone(); @@ -801,14 +719,14 @@ pub fn cldb(args: &[String]) { prim_map.insert(p.0.clone(), Rc::new(p.1.clone())); } let program_lines: Rc> = Rc::new( - parsed_program_result + parsed.program .content .lines() .map(|x| x.to_string()) .collect(), ); let cldbenv = CldbRunEnv::new( - parsed_program_result.path.clone(), + parsed.program.path.clone(), program_lines.clone(), Box::new(CldbNoOverride::new_symbols(use_symbol_table.clone())), ); @@ -817,7 +735,7 @@ pub fn cldb(args: &[String]) { let result = cldb_hierarchy( runner, Rc::new(prim_map), - parsed_program_result.path.clone(), + parsed.program.path.clone(), program_lines, Rc::new(use_symbol_table), program, @@ -1273,26 +1191,21 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul let dpr = special_runner.clone(); let run_program = special_runner; - let assembled_sexp = - match parse_tool_input_sexp(&mut allocator, "path_or_code", &parsed_args, None, None) { - Ok(s) => s, + let parsed = + match RunAndCompileInputData::new( + &mut allocator, + &parsed_args + ) { + Ok(r) => r, Err(e) => { stdout.write_str(&format!("FAIL: {e}\n")); return; } }; - let env = match parse_tool_input_sexp(&mut allocator, "env", &parsed_args, None, None) { - Ok(s) => s, - Err(e) => { - stdout.write_str(&format!("FAIL: {e}\n")); - return; - } - }; - let time_assemble = SystemTime::now(); - let input_sexp = allocator.new_pair(assembled_sexp.parsed, env.parsed).ok(); + let input_sexp = allocator.new_pair(parsed.program.parsed, parsed.args.parsed).ok(); // Symbol table related checks: should one be loaded, should one be saved. // This code is confusingly woven due to 'run' and 'brun' serving many roles. @@ -1330,9 +1243,8 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul .unwrap_or(false); // Dialect is now not overall optional. - let dialect = input_sexp.map(|i| detect_modern(&mut allocator, i)); let mut stderr_output = |s: String| { - if dialect.as_ref().and_then(|d| d.stepping).is_some() { + if parsed.dialect.stepping.is_some() { eprintln!("{s}"); } else { stdout.write_str(&s); @@ -1368,7 +1280,7 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul .unwrap_or_else(|| "main.sym".to_string()); // In testing: short circuit for modern compilation. - if let Some(stepping) = dialect.as_ref().and_then(|d| d.stepping) { + if let Some(stepping) = parsed.dialect.stepping { let do_optimize = parsed_args .get("optimize") .map(|x| matches!(x, ArgumentValue::ArgBool(true))) @@ -1376,7 +1288,7 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul let runner = Rc::new(DefaultProgramRunner::new()); let use_filename = input_file.unwrap_or_else(|| "*command*".to_string()); let opts = Rc::new(DefaultCompilerOpts::new(&use_filename)) - .set_dialect(dialect.unwrap_or_default()) + .set_dialect(parsed.dialect) .set_optimize(do_optimize || stepping > 22) .set_search_paths(&search_paths) .set_frontend_opt(stepping == 22) diff --git a/src/classic/clvm_tools/comp_input.rs b/src/classic/clvm_tools/comp_input.rs new file mode 100644 index 000000000..280b8765e --- /dev/null +++ b/src/classic/clvm_tools/comp_input.rs @@ -0,0 +1,126 @@ +use std::collections::HashMap; +use std::rc::Rc; + +use clvmr::{Allocator, NodePtr}; + +use crate::classic::platform::argparse::ArgumentValue; +use crate::classic::clvm::__type_compatibility__::{Bytes, Stream, UnvalidatedBytesFromType}; +use crate::classic::clvm::serialize::{SimpleCreateCLVMObject, sexp_from_stream}; + +use crate::classic::clvm_tools::binutils::assemble_from_ir; +use crate::classic::clvm_tools::ir::reader::read_ir; + +use crate::compiler::dialect::{AcceptedDialect, detect_modern}; + +fn get_string_and_filename_with_default( + parsed_args: &HashMap, + argument: &str, + default: Option<&str>, +) -> Option<(Option, String)> { + if let Some(ArgumentValue::ArgString(path, content)) = parsed_args.get(argument) { + Some((path.clone(), content.to_string())) + } else { + default.map(|p| (None, p.to_string())) + } +} + +pub struct ParsedInputPathOrCode { + pub path: Option, + pub content: String, + pub parsed: NodePtr, +} + +pub fn parse_tool_input_sexp( + allocator: &mut Allocator, + argument: &str, + parsed_args: &HashMap, + default_hex: Option<&str>, + default_sexp: Option<&str>, +) -> Result { + match parsed_args.get("hex") { + Some(_) => { + let (path, use_sexp_text) = if let Some(r) = + get_string_and_filename_with_default(parsed_args, argument, default_hex) + { + r + } else { + return Err("missing argument {argument}".to_string()); + }; + + let sexp_serialized = + Bytes::new_validated(Some(UnvalidatedBytesFromType::Hex(use_sexp_text.clone()))) + .map_err(|e| format!("{e:?}"))?; + + let mut prog_stream = Stream::new(Some(sexp_serialized)); + + sexp_from_stream( + allocator, + &mut prog_stream, + Box::new(SimpleCreateCLVMObject {}), + ) + .map(|x| ParsedInputPathOrCode { + path, + content: use_sexp_text, + parsed: x.1, + }) + .map_err(|e| format!("{e:?}")) + } + _ => { + let (path, use_sexp_text) = if let Some(r) = + get_string_and_filename_with_default(parsed_args, argument, default_sexp) + { + r + } else { + return Err(format!("missing argument {argument}")); + }; + + read_ir(&use_sexp_text) + .map_err(|e| format!("{e:?}")) + .and_then(|v| { + Ok(ParsedInputPathOrCode { + path, + content: use_sexp_text, + parsed: assemble_from_ir(allocator, Rc::new(v)) + .map_err(|e| format!("{e:?}"))? + }) + }) + } + } +} + +pub struct RunAndCompileInputData { + pub program: ParsedInputPathOrCode, + pub args: ParsedInputPathOrCode, + pub dialect: AcceptedDialect, +} + +impl RunAndCompileInputData { + pub fn new( + allocator: &mut Allocator, + parsed_args: &HashMap + ) -> Result { + let program = parse_tool_input_sexp( + allocator, + "path_or_code", + parsed_args, + None, + None + )?; + let args = parse_tool_input_sexp( + allocator, + "env", + parsed_args, + Some("80"), + Some("()"), + )?; + let dialect = detect_modern( + allocator, + program.parsed + ); + Ok(RunAndCompileInputData { + program, + args, + dialect, + }) + } +} diff --git a/src/classic/clvm_tools/mod.rs b/src/classic/clvm_tools/mod.rs index a5bd88464..9bbdfbe58 100644 --- a/src/classic/clvm_tools/mod.rs +++ b/src/classic/clvm_tools/mod.rs @@ -1,6 +1,7 @@ pub mod binutils; pub mod clvmc; pub mod cmds; +pub mod comp_input; pub mod curry; pub mod debug; pub mod ir; From f992752410b75eb84e0497b2084619e4c15d06cd Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 5 Jul 2024 11:35:17 -0700 Subject: [PATCH 03/14] fmt + clippy --- src/classic/clvm_tools/cmds.rs | 46 +++++++++++++--------------- src/classic/clvm_tools/comp_input.rs | 31 +++++-------------- 2 files changed, 29 insertions(+), 48 deletions(-) diff --git a/src/classic/clvm_tools/cmds.rs b/src/classic/clvm_tools/cmds.rs index d2dd0c816..536085a46 100644 --- a/src/classic/clvm_tools/cmds.rs +++ b/src/classic/clvm_tools/cmds.rs @@ -596,17 +596,13 @@ pub fn cldb(args: &[String]) { println!("{}", yamlette_string(output)); }; - let parsed = - match RunAndCompileInputData::new( - &mut allocator, - &parsed_args - ) { - Ok(r) => r, - Err(e) => { - errorize(&mut output, None, &e.to_string()); - return; - } - }; + let parsed = match RunAndCompileInputData::new(&mut allocator, &parsed_args) { + Ok(r) => r, + Err(e) => { + errorize(&mut output, None, &e.to_string()); + return; + } + }; let symbol_table = parsed_args .get("symbol_table") @@ -626,7 +622,8 @@ pub fn cldb(args: &[String]) { .map(|x| matches!(x, ArgumentValue::ArgBool(true))) .unwrap_or_else(|| false); let runner = Rc::new(DefaultProgramRunner::new()); - let use_filename = parsed.program + let use_filename = parsed + .program .path .clone() .unwrap_or_else(|| "*command*".to_string()); @@ -719,7 +716,8 @@ pub fn cldb(args: &[String]) { prim_map.insert(p.0.clone(), Rc::new(p.1.clone())); } let program_lines: Rc> = Rc::new( - parsed.program + parsed + .program .content .lines() .map(|x| x.to_string()) @@ -1191,21 +1189,19 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul let dpr = special_runner.clone(); let run_program = special_runner; - let parsed = - match RunAndCompileInputData::new( - &mut allocator, - &parsed_args - ) { - Ok(r) => r, - Err(e) => { - stdout.write_str(&format!("FAIL: {e}\n")); - return; - } - }; + let parsed = match RunAndCompileInputData::new(&mut allocator, &parsed_args) { + Ok(r) => r, + Err(e) => { + stdout.write_str(&format!("FAIL: {e}\n")); + return; + } + }; let time_assemble = SystemTime::now(); - let input_sexp = allocator.new_pair(parsed.program.parsed, parsed.args.parsed).ok(); + let input_sexp = allocator + .new_pair(parsed.program.parsed, parsed.args.parsed) + .ok(); // Symbol table related checks: should one be loaded, should one be saved. // This code is confusingly woven due to 'run' and 'brun' serving many roles. diff --git a/src/classic/clvm_tools/comp_input.rs b/src/classic/clvm_tools/comp_input.rs index 280b8765e..b2d4d138a 100644 --- a/src/classic/clvm_tools/comp_input.rs +++ b/src/classic/clvm_tools/comp_input.rs @@ -3,14 +3,14 @@ use std::rc::Rc; use clvmr::{Allocator, NodePtr}; -use crate::classic::platform::argparse::ArgumentValue; use crate::classic::clvm::__type_compatibility__::{Bytes, Stream, UnvalidatedBytesFromType}; -use crate::classic::clvm::serialize::{SimpleCreateCLVMObject, sexp_from_stream}; +use crate::classic::clvm::serialize::{sexp_from_stream, SimpleCreateCLVMObject}; +use crate::classic::platform::argparse::ArgumentValue; use crate::classic::clvm_tools::binutils::assemble_from_ir; use crate::classic::clvm_tools::ir::reader::read_ir; -use crate::compiler::dialect::{AcceptedDialect, detect_modern}; +use crate::compiler::dialect::{detect_modern, AcceptedDialect}; fn get_string_and_filename_with_default( parsed_args: &HashMap, @@ -81,7 +81,7 @@ pub fn parse_tool_input_sexp( path, content: use_sexp_text, parsed: assemble_from_ir(allocator, Rc::new(v)) - .map_err(|e| format!("{e:?}"))? + .map_err(|e| format!("{e:?}"))?, }) }) } @@ -97,26 +97,11 @@ pub struct RunAndCompileInputData { impl RunAndCompileInputData { pub fn new( allocator: &mut Allocator, - parsed_args: &HashMap + parsed_args: &HashMap, ) -> Result { - let program = parse_tool_input_sexp( - allocator, - "path_or_code", - parsed_args, - None, - None - )?; - let args = parse_tool_input_sexp( - allocator, - "env", - parsed_args, - Some("80"), - Some("()"), - )?; - let dialect = detect_modern( - allocator, - program.parsed - ); + let program = parse_tool_input_sexp(allocator, "path_or_code", parsed_args, None, None)?; + let args = parse_tool_input_sexp(allocator, "env", parsed_args, Some("80"), Some("()"))?; + let dialect = detect_modern(allocator, program.parsed); Ok(RunAndCompileInputData { program, args, From d40b2da54e9366fb908ad40fca14dd6d63ecdaa0 Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 5 Jul 2024 12:28:53 -0700 Subject: [PATCH 04/14] Make modern compilation equal between run and cldb and factor this code out from run into the new comp_input --- src/classic/clvm_tools/cmds.rs | 255 +++++++++------------------ src/classic/clvm_tools/comp_input.rs | 107 +++++++++++ 2 files changed, 186 insertions(+), 176 deletions(-) diff --git a/src/classic/clvm_tools/cmds.rs b/src/classic/clvm_tools/cmds.rs index 536085a46..a9b314587 100644 --- a/src/classic/clvm_tools/cmds.rs +++ b/src/classic/clvm_tools/cmds.rs @@ -30,7 +30,7 @@ use crate::classic::clvm::sexp::{enlist, proper_list, sexp_as_bin}; use crate::classic::clvm::OPERATORS_LATEST_VERSION; use crate::classic::clvm_tools::binutils::{assemble_from_ir, disassemble, disassemble_with_kw}; use crate::classic::clvm_tools::clvmc::write_sym_output; -use crate::classic::clvm_tools::comp_input::RunAndCompileInputData; +use crate::classic::clvm_tools::comp_input::{get_disassembly_ver, RunAndCompileInputData}; use crate::classic::clvm_tools::debug::check_unused; use crate::classic::clvm_tools::debug::{ program_hash_from_program_env_cons, start_log_after, trace_pre_eval, trace_to_table, @@ -53,11 +53,9 @@ use crate::classic::platform::argparse::{ use crate::compiler::cldb::{hex_to_modern_sexp, CldbNoOverride, CldbRun, CldbRunEnv}; use crate::compiler::cldb_hierarchy::{HierarchialRunner, HierarchialStepResult, RunPurpose}; use crate::compiler::clvm::start_step; -use crate::compiler::compiler::{compile_file, DefaultCompilerOpts}; +use crate::compiler::compiler::DefaultCompilerOpts; use crate::compiler::comptypes::{CompileErr, CompilerOpts}; -use crate::compiler::debug::build_symbol_table_mut; use crate::compiler::frontend::frontend; -use crate::compiler::optimize::maybe_finalize_program_via_classic_optimizer; use crate::compiler::preprocessor::gather_dependencies; use crate::compiler::prims; use crate::compiler::runtypes::RunFailure; @@ -497,13 +495,15 @@ pub fn cldb_hierarchy( } pub fn cldb(args: &[String]) { + let mut allocator = Allocator::new(); + let mut output = Vec::new(); + let tool_name = "cldb".to_string(); let props = TArgumentParserProps { description: "Execute a clvm script.".to_string(), prog: format!("clvm_tools {tool_name}"), }; - let mut search_paths = Vec::new(); let mut parser = ArgumentParser::new(Some(props)); parser.add_argument( vec!["-i".to_string(), "--include".to_string()], @@ -561,27 +561,6 @@ pub fn cldb(args: &[String]) { let prog_srcloc = Srcloc::start("*program*"); let args_srcloc = Srcloc::start("*args*"); - let mut args = Rc::new(sexp::SExp::atom_from_string(args_srcloc.clone(), "")); - - let parsed_args: HashMap = match parser.parse_args(&arg_vec) { - Err(e) => { - println!("FAIL: {e}"); - return; - } - Ok(pa) => pa, - }; - - if let Some(ArgumentValue::ArgArray(v)) = parsed_args.get("include") { - for p in v { - if let ArgumentValue::ArgString(_, s) = p { - search_paths.push(s.to_string()); - } - } - } - - let mut allocator = Allocator::new(); - let mut output = Vec::new(); - let errorize = |output: &mut Vec>, location: Option, c: &str| { let mut parse_error = BTreeMap::new(); @@ -596,12 +575,12 @@ pub fn cldb(args: &[String]) { println!("{}", yamlette_string(output)); }; - let parsed = match RunAndCompileInputData::new(&mut allocator, &parsed_args) { - Ok(r) => r, + let parsed_args: HashMap = match parser.parse_args(&arg_vec) { Err(e) => { - errorize(&mut output, None, &e.to_string()); + println!("FAIL: {e}"); return; } + Ok(pa) => pa, }; let symbol_table = parsed_args @@ -615,21 +594,20 @@ pub fn cldb(args: &[String]) { _ => None, }); + let parsed = match RunAndCompileInputData::new( + &mut allocator, + &parsed_args + ) { + Ok(r) => r, + Err(e) => { + println!("FAIL: {e}"); + return; + } + }; + let only_print = parsed_args.get("only_print").map(|_| true).unwrap_or(false); - let do_optimize = parsed_args - .get("optimize") - .map(|x| matches!(x, ArgumentValue::ArgBool(true))) - .unwrap_or_else(|| false); let runner = Rc::new(DefaultProgramRunner::new()); - let use_filename = parsed - .program - .path - .clone() - .unwrap_or_else(|| "*command*".to_string()); - let opts = Rc::new(DefaultCompilerOpts::new(&use_filename)) - .set_optimize(do_optimize) - .set_search_paths(&search_paths); let mut use_symbol_table = symbol_table.unwrap_or_default(); @@ -642,24 +620,7 @@ pub fn cldb(args: &[String]) { ) .map_err(|_| CompileErr(prog_srcloc, "Failed to parse hex".to_string())), _ => { - // don't clobber a symbol table brought in via -y unless we're - // compiling here. - let unopt_res = compile_file( - &mut allocator, - runner.clone(), - opts.clone(), - &parsed.program.content, - &mut use_symbol_table, - ); - unopt_res.and_then(|x| { - maybe_finalize_program_via_classic_optimizer( - &mut allocator, - runner.clone(), - opts, - false, - &x, - ) - }) + parsed.compile_modern(&mut allocator, &mut use_symbol_table) } }; @@ -671,45 +632,48 @@ pub fn cldb(args: &[String]) { } }; - match parsed_args.get("hex") { - Some(ArgumentValue::ArgBool(true)) => { - match hex_to_modern_sexp( - &mut allocator, - &HashMap::new(), - args_srcloc, - &parsed.args.content, - ) { + let env_loc = Srcloc::start("*args*"); + let env = + match parsed_args.get("hex") { + Some(ArgumentValue::ArgBool(true)) => { + match hex_to_modern_sexp( + &mut allocator, + &HashMap::new(), + args_srcloc, + &parsed.args.content, + ) { + Ok(r) => r, + Err(p) => { + let mut parse_error = BTreeMap::new(); + parse_error.insert("Error".to_string(), YamlElement::String(p.to_string())); + output.push(parse_error.clone()); + println!("{}", yamlette_string(&output)); + return; + } + } + } + + _ => match parse_sexp(env_loc.clone(), parsed.args.content.bytes()) { Ok(r) => { - args = r; + if !r.is_empty() { + r[0].clone() + } else { + Rc::new(sexp::SExp::Nil(env_loc)) + } } - Err(p) => { + Err(c) => { let mut parse_error = BTreeMap::new(); - parse_error.insert("Error".to_string(), YamlElement::String(p.to_string())); + parse_error.insert( + "Error-Location".to_string(), + YamlElement::String(c.0.to_string()), + ); + parse_error.insert("Error".to_string(), YamlElement::String(c.1)); output.push(parse_error.clone()); println!("{}", yamlette_string(&output)); return; } } - } - _ => match parse_sexp(Srcloc::start("*arg*"), parsed.args.content.bytes()) { - Ok(r) => { - if !r.is_empty() { - args = r[0].clone(); - } - } - Err(c) => { - let mut parse_error = BTreeMap::new(); - parse_error.insert( - "Error-Location".to_string(), - YamlElement::String(c.0.to_string()), - ); - parse_error.insert("Error".to_string(), YamlElement::String(c.1)); - output.push(parse_error.clone()); - println!("{}", yamlette_string(&output)); - return; - } - }, - }; + }; let mut prim_map = HashMap::new(); for p in prims::prims().iter() { @@ -737,7 +701,7 @@ pub fn cldb(args: &[String]) { program_lines, Rc::new(use_symbol_table), program, - args, + env, ); // Print the tree @@ -746,7 +710,7 @@ pub fn cldb(args: &[String]) { return; } - let step = start_step(program, args); + let step = start_step(program, env); let mut cldbrun = CldbRun::new(runner, Rc::new(prim_map), Box::new(cldbenv), step); let print_tree = |output: &mut Vec<_>, result: &BTreeMap| { let mut cvt_subtree = BTreeMap::new(); @@ -916,15 +880,9 @@ fn perform_preprocessing( Ok(()) } -fn get_disassembly_ver(p: &HashMap) -> Option { - if let Some(ArgumentValue::ArgInt(x)) = p.get("operators_version") { - return Some(*x as usize); - } - - None -} - pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, default_stage: u32) { + let mut allocator = Allocator::new(); + let props = TArgumentParserProps { description: "Execute a clvm script.".to_string(), prog: format!("clvm_tools {tool_name}"), @@ -1107,6 +1065,14 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul return; } + let parsed = match RunAndCompileInputData::new(&mut allocator, &parsed_args) { + Ok(r) => r, + Err(e) => { + stdout.write_str(&format!("FAIL: {e}\n")); + return; + } + }; + let empty_map = HashMap::new(); let keywords = match parsed_args.get("no_keywords") { Some(ArgumentValue::ArgBool(_b)) => &empty_map, @@ -1136,22 +1102,6 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul .cloned() .unwrap_or_else(|| "*command*".to_string()); - let search_paths = if let Some(ArgumentValue::ArgArray(v)) = parsed_args.get("include") { - v.iter() - .filter_map(|p| { - if let ArgumentValue::ArgString(_, s) = p { - Some(s.to_string()) - } else { - None - } - }) - .collect() - } else { - Vec::new() - }; - - let mut allocator = Allocator::new(); - let time_start = SystemTime::now(); let time_read_hex = SystemTime::now(); @@ -1163,7 +1113,7 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul parsed_args.get("path_or_code"), ) { if let Some(filename) = &file { - let opts = DefaultCompilerOpts::new(filename).set_search_paths(&search_paths); + let opts = DefaultCompilerOpts::new(filename).set_search_paths(&parsed.search_paths); match gather_dependencies(opts, filename, file_content) { Err(e) => { @@ -1183,20 +1133,12 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul } let special_runner = - run_program_for_search_paths(&reported_input_file, &search_paths, extra_symbol_info); + run_program_for_search_paths(&reported_input_file, &parsed.search_paths, extra_symbol_info); // Ensure we know the user's wishes about the disassembly version here. special_runner.set_operators_version(get_disassembly_ver(&parsed_args)); let dpr = special_runner.clone(); let run_program = special_runner; - let parsed = match RunAndCompileInputData::new(&mut allocator, &parsed_args) { - Ok(r) => r, - Err(e) => { - stdout.write_str(&format!("FAIL: {e}\n")); - return; - } - }; - let time_assemble = SystemTime::now(); let input_sexp = allocator @@ -1249,7 +1191,7 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul if do_check_unused { let opts = - Rc::new(DefaultCompilerOpts::new(&reported_input_file)).set_search_paths(&search_paths); + Rc::new(DefaultCompilerOpts::new(&reported_input_file)).set_search_paths(&parsed.search_paths); match check_unused(opts, &input_program) { Ok((success, output)) => { stderr_output(output); @@ -1264,70 +1206,31 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul } } - let symbol_table_output = parsed_args - .get("symbol_output_file") - .and_then(|s| { - if let ArgumentValue::ArgString(_, v) = s { - Some(v.clone()) - } else { - None - } - }) - .unwrap_or_else(|| "main.sym".to_string()); - // In testing: short circuit for modern compilation. - if let Some(stepping) = parsed.dialect.stepping { - let do_optimize = parsed_args - .get("optimize") - .map(|x| matches!(x, ArgumentValue::ArgBool(true))) - .unwrap_or_else(|| false); - let runner = Rc::new(DefaultProgramRunner::new()); - let use_filename = input_file.unwrap_or_else(|| "*command*".to_string()); - let opts = Rc::new(DefaultCompilerOpts::new(&use_filename)) - .set_dialect(parsed.dialect) - .set_optimize(do_optimize || stepping > 22) - .set_search_paths(&search_paths) - .set_frontend_opt(stepping == 22) - .set_disassembly_ver(get_disassembly_ver(&parsed_args)); - let mut symbol_table = HashMap::new(); - + if parsed.dialect.stepping.is_some() { // Short circuit preprocessing display. if parsed_args.contains_key("preprocess") { - if let Err(e) = perform_preprocessing(stdout, opts, &use_filename, &input_program) { + if let Err(e) = perform_preprocessing(stdout, parsed.opts.clone(), &parsed.use_filename(), &input_program) { stdout.write_str(&format!("{}: {}", e.0, e.1)); } return; } - let unopt_res = compile_file( - &mut allocator, - runner.clone(), - opts.clone(), - &input_program, - &mut symbol_table, - ); - let res = unopt_res.and_then(|x| { - maybe_finalize_program_via_classic_optimizer( - &mut allocator, - runner, - opts, - do_optimize, - &x, - ) - }); + let mut symbol_table = HashMap::new(); + match parsed.compile_modern(&mut allocator, &mut symbol_table).and_then(|r| { + write_sym_output(&symbol_table, &parsed.symbol_table_output).map_err(|e| { + CompileErr(Srcloc::start(&parsed.use_filename()), format!("writing symbols: {e:?}")) + })?; - match res { + Ok(r) + }) { Ok(r) => { stdout.write_str(&r.to_string()); - - build_symbol_table_mut(&mut symbol_table, &r); - write_sym_output(&symbol_table, &symbol_table_output).expect("writing symbols"); } Err(c) => { stdout.write_str(&format!("{}: {}", c.0, c.1)); } } - return; } @@ -1547,7 +1450,7 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul let compile_sym_out = dpr.get_compiles(); if !compile_sym_out.is_empty() { - write_sym_output(&compile_sym_out, &symbol_table_output).ok(); + write_sym_output(&compile_sym_out, &parsed.symbol_table_output).ok(); } stdout.write_str(&format!("{output}\n")); diff --git a/src/classic/clvm_tools/comp_input.rs b/src/classic/clvm_tools/comp_input.rs index b2d4d138a..6a171e878 100644 --- a/src/classic/clvm_tools/comp_input.rs +++ b/src/classic/clvm_tools/comp_input.rs @@ -9,8 +9,22 @@ use crate::classic::platform::argparse::ArgumentValue; use crate::classic::clvm_tools::binutils::assemble_from_ir; use crate::classic::clvm_tools::ir::reader::read_ir; +use crate::classic::clvm_tools::stages::stage_0::DefaultProgramRunner; +use crate::compiler::compiler::{compile_file, DefaultCompilerOpts}; +use crate::compiler::comptypes::{CompilerOpts, CompileErr}; +use crate::compiler::debug::build_symbol_table_mut; use crate::compiler::dialect::{detect_modern, AcceptedDialect}; +use crate::compiler::optimize::maybe_finalize_program_via_classic_optimizer; +use crate::compiler::sexp; + +pub fn get_disassembly_ver(p: &HashMap) -> Option { + if let Some(ArgumentValue::ArgInt(x)) = p.get("operators_version") { + return Some(*x as usize); + } + + None +} fn get_string_and_filename_with_default( parsed_args: &HashMap, @@ -30,6 +44,15 @@ pub struct ParsedInputPathOrCode { pub parsed: NodePtr, } +impl ParsedInputPathOrCode { + pub fn use_filename(&self) -> String { + self + .path + .clone() + .unwrap_or_else(|| "*command*".to_string()) + } +} + pub fn parse_tool_input_sexp( allocator: &mut Allocator, argument: &str, @@ -92,6 +115,10 @@ pub struct RunAndCompileInputData { pub program: ParsedInputPathOrCode, pub args: ParsedInputPathOrCode, pub dialect: AcceptedDialect, + pub opts: Rc, + pub do_optimize: bool, + pub search_paths: Vec, + pub symbol_table_output: String, } impl RunAndCompileInputData { @@ -102,10 +129,90 @@ impl RunAndCompileInputData { let program = parse_tool_input_sexp(allocator, "path_or_code", parsed_args, None, None)?; let args = parse_tool_input_sexp(allocator, "env", parsed_args, Some("80"), Some("()"))?; let dialect = detect_modern(allocator, program.parsed); + + let do_optimize = parsed_args + .get("optimize") + .map(|x| matches!(x, ArgumentValue::ArgBool(true))) + .unwrap_or_else(|| false); + + let search_paths = if let Some(ArgumentValue::ArgArray(v)) = parsed_args.get("include") { + v.iter() + .filter_map(|p| { + if let ArgumentValue::ArgString(_, s) = p { + Some(s.to_string()) + } else { + None + } + }) + .collect() + } else { + Vec::new() + }; + + let mut opts: Rc = Rc::new(DefaultCompilerOpts::new(&program.use_filename())) + .set_dialect(dialect.clone()) + .set_search_paths(&search_paths) + .set_optimize(do_optimize) + .set_disassembly_ver(get_disassembly_ver(&parsed_args)); + + if let Some(stepping) = dialect.stepping { + opts = opts + .set_optimize(do_optimize || stepping > 22) + .set_frontend_opt(stepping == 22); + } + + let symbol_table_output = parsed_args + .get("symbol_output_file") + .and_then(|s| { + if let ArgumentValue::ArgString(_, v) = s { + Some(v.clone()) + } else { + None + } + }) + .unwrap_or_else(|| "main.sym".to_string()); + Ok(RunAndCompileInputData { program, args, dialect, + do_optimize, + search_paths, + opts, + symbol_table_output, }) } + + pub fn use_filename(&self) -> String { + self.program.path.clone().unwrap_or_else(|| "*command*".to_string()) + } + + pub fn compile_modern( + &self, + allocator: &mut Allocator, + symbol_table: &mut HashMap, + ) -> Result, CompileErr> { + let runner = Rc::new(DefaultProgramRunner::new()); + + let unopt_res = compile_file( + allocator, + runner.clone(), + self.opts.clone(), + &self.program.content, + symbol_table, + ); + let res = unopt_res.and_then(|x| { + maybe_finalize_program_via_classic_optimizer( + allocator, + runner, + self.opts.clone(), + self.do_optimize, + &x, + ) + })?; + + build_symbol_table_mut(symbol_table, &res); + + Ok(res) + } } From 3e16cb2b46b9070a314a99e11872ea1f2cb2cc6b Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 5 Jul 2024 12:37:28 -0700 Subject: [PATCH 05/14] Tighten further by eliminating some duplicated computations --- src/classic/clvm_tools/cmds.rs | 136 +++++++++++++-------------- src/classic/clvm_tools/comp_input.rs | 23 ++--- 2 files changed, 76 insertions(+), 83 deletions(-) diff --git a/src/classic/clvm_tools/cmds.rs b/src/classic/clvm_tools/cmds.rs index a9b314587..374c2f1f6 100644 --- a/src/classic/clvm_tools/cmds.rs +++ b/src/classic/clvm_tools/cmds.rs @@ -594,10 +594,7 @@ pub fn cldb(args: &[String]) { _ => None, }); - let parsed = match RunAndCompileInputData::new( - &mut allocator, - &parsed_args - ) { + let parsed = match RunAndCompileInputData::new(&mut allocator, &parsed_args) { Ok(r) => r, Err(e) => { println!("FAIL: {e}"); @@ -619,9 +616,7 @@ pub fn cldb(args: &[String]) { &parsed.program.content, ) .map_err(|_| CompileErr(prog_srcloc, "Failed to parse hex".to_string())), - _ => { - parsed.compile_modern(&mut allocator, &mut use_symbol_table) - } + _ => parsed.compile_modern(&mut allocator, &mut use_symbol_table), }; let program = match res { @@ -633,47 +628,46 @@ pub fn cldb(args: &[String]) { }; let env_loc = Srcloc::start("*args*"); - let env = - match parsed_args.get("hex") { - Some(ArgumentValue::ArgBool(true)) => { - match hex_to_modern_sexp( - &mut allocator, - &HashMap::new(), - args_srcloc, - &parsed.args.content, - ) { - Ok(r) => r, - Err(p) => { - let mut parse_error = BTreeMap::new(); - parse_error.insert("Error".to_string(), YamlElement::String(p.to_string())); - output.push(parse_error.clone()); - println!("{}", yamlette_string(&output)); - return; - } - } - } - - _ => match parse_sexp(env_loc.clone(), parsed.args.content.bytes()) { - Ok(r) => { - if !r.is_empty() { - r[0].clone() - } else { - Rc::new(sexp::SExp::Nil(env_loc)) - } - } - Err(c) => { + let env = match parsed_args.get("hex") { + Some(ArgumentValue::ArgBool(true)) => { + match hex_to_modern_sexp( + &mut allocator, + &HashMap::new(), + args_srcloc, + &parsed.args.content, + ) { + Ok(r) => r, + Err(p) => { let mut parse_error = BTreeMap::new(); - parse_error.insert( - "Error-Location".to_string(), - YamlElement::String(c.0.to_string()), - ); - parse_error.insert("Error".to_string(), YamlElement::String(c.1)); + parse_error.insert("Error".to_string(), YamlElement::String(p.to_string())); output.push(parse_error.clone()); println!("{}", yamlette_string(&output)); return; } } - }; + } + + _ => match parse_sexp(env_loc.clone(), parsed.args.content.bytes()) { + Ok(r) => { + if !r.is_empty() { + r[0].clone() + } else { + Rc::new(sexp::SExp::Nil(env_loc)) + } + } + Err(c) => { + let mut parse_error = BTreeMap::new(); + parse_error.insert( + "Error-Location".to_string(), + YamlElement::String(c.0.to_string()), + ); + parse_error.insert("Error".to_string(), YamlElement::String(c.1)); + output.push(parse_error.clone()); + println!("{}", yamlette_string(&output)); + return; + } + }, + }; let mut prim_map = HashMap::new(); for p in prims::prims().iter() { @@ -1085,23 +1079,6 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul // more info). let extra_symbol_info = parsed_args.get("extra_syms").map(|_| true).unwrap_or(false); - // Get name of included file so we can use it to initialize the compiler's - // runner, which contains operators used at compile time in clvm to update - // symbols. This is the only means we have of communicating with the - // compilation process from this level. - let mut input_file = None; - let mut input_program = "()".to_string(); - - if let Some(ArgumentValue::ArgString(file, path_or_code)) = parsed_args.get("path_or_code") { - input_file.clone_from(file); - input_program = path_or_code.to_string(); - } - - let reported_input_file = input_file - .as_ref() - .cloned() - .unwrap_or_else(|| "*command*".to_string()); - let time_start = SystemTime::now(); let time_read_hex = SystemTime::now(); @@ -1132,8 +1109,11 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul return; } - let special_runner = - run_program_for_search_paths(&reported_input_file, &parsed.search_paths, extra_symbol_info); + let special_runner = run_program_for_search_paths( + &parsed.use_filename(), + &parsed.search_paths, + extra_symbol_info, + ); // Ensure we know the user's wishes about the disassembly version here. special_runner.set_operators_version(get_disassembly_ver(&parsed_args)); let dpr = special_runner.clone(); @@ -1190,9 +1170,9 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul }; if do_check_unused { - let opts = - Rc::new(DefaultCompilerOpts::new(&reported_input_file)).set_search_paths(&parsed.search_paths); - match check_unused(opts, &input_program) { + let opts = Rc::new(DefaultCompilerOpts::new(&parsed.use_filename())) + .set_search_paths(&parsed.search_paths); + match check_unused(opts, &parsed.program.content) { Ok((success, output)) => { stderr_output(output); if !success { @@ -1210,20 +1190,32 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul if parsed.dialect.stepping.is_some() { // Short circuit preprocessing display. if parsed_args.contains_key("preprocess") { - if let Err(e) = perform_preprocessing(stdout, parsed.opts.clone(), &parsed.use_filename(), &input_program) { + if let Err(e) = perform_preprocessing( + stdout, + parsed.opts.clone(), + &parsed.use_filename(), + &parsed.program.content, + ) { stdout.write_str(&format!("{}: {}", e.0, e.1)); } return; } let mut symbol_table = HashMap::new(); - match parsed.compile_modern(&mut allocator, &mut symbol_table).and_then(|r| { - write_sym_output(&symbol_table, &parsed.symbol_table_output).map_err(|e| { - CompileErr(Srcloc::start(&parsed.use_filename()), format!("writing symbols: {e:?}")) - })?; - - Ok(r) - }) { + let res = parsed + .compile_modern(&mut allocator, &mut symbol_table) + .and_then(|r| { + write_sym_output(&symbol_table, &parsed.symbol_table_output).map_err(|e| { + CompileErr( + Srcloc::start(&parsed.use_filename()), + format!("writing symbols: {e:?}"), + ) + })?; + + Ok(r) + }); + + match res { Ok(r) => { stdout.write_str(&r.to_string()); } diff --git a/src/classic/clvm_tools/comp_input.rs b/src/classic/clvm_tools/comp_input.rs index 6a171e878..7ce810e03 100644 --- a/src/classic/clvm_tools/comp_input.rs +++ b/src/classic/clvm_tools/comp_input.rs @@ -12,7 +12,7 @@ use crate::classic::clvm_tools::ir::reader::read_ir; use crate::classic::clvm_tools::stages::stage_0::DefaultProgramRunner; use crate::compiler::compiler::{compile_file, DefaultCompilerOpts}; -use crate::compiler::comptypes::{CompilerOpts, CompileErr}; +use crate::compiler::comptypes::{CompileErr, CompilerOpts}; use crate::compiler::debug::build_symbol_table_mut; use crate::compiler::dialect::{detect_modern, AcceptedDialect}; use crate::compiler::optimize::maybe_finalize_program_via_classic_optimizer; @@ -46,10 +46,7 @@ pub struct ParsedInputPathOrCode { impl ParsedInputPathOrCode { pub fn use_filename(&self) -> String { - self - .path - .clone() - .unwrap_or_else(|| "*command*".to_string()) + self.path.clone().unwrap_or_else(|| "*command*".to_string()) } } @@ -149,11 +146,12 @@ impl RunAndCompileInputData { Vec::new() }; - let mut opts: Rc = Rc::new(DefaultCompilerOpts::new(&program.use_filename())) - .set_dialect(dialect.clone()) - .set_search_paths(&search_paths) - .set_optimize(do_optimize) - .set_disassembly_ver(get_disassembly_ver(&parsed_args)); + let mut opts: Rc = + Rc::new(DefaultCompilerOpts::new(&program.use_filename())) + .set_dialect(dialect.clone()) + .set_search_paths(&search_paths) + .set_optimize(do_optimize) + .set_disassembly_ver(get_disassembly_ver(parsed_args)); if let Some(stepping) = dialect.stepping { opts = opts @@ -184,7 +182,10 @@ impl RunAndCompileInputData { } pub fn use_filename(&self) -> String { - self.program.path.clone().unwrap_or_else(|| "*command*".to_string()) + self.program + .path + .clone() + .unwrap_or_else(|| "*command*".to_string()) } pub fn compile_modern( From 78d2653d7971b76532e7aa5048505224fb55dfb7 Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 5 Jul 2024 13:06:07 -0700 Subject: [PATCH 06/14] Ensure that we take an empty argument as a default when a default value exists --- src/classic/clvm_tools/comp_input.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/classic/clvm_tools/comp_input.rs b/src/classic/clvm_tools/comp_input.rs index 7ce810e03..448a1b52c 100644 --- a/src/classic/clvm_tools/comp_input.rs +++ b/src/classic/clvm_tools/comp_input.rs @@ -59,7 +59,7 @@ pub fn parse_tool_input_sexp( ) -> Result { match parsed_args.get("hex") { Some(_) => { - let (path, use_sexp_text) = if let Some(r) = + let (path, sexp_text) = if let Some(r) = get_string_and_filename_with_default(parsed_args, argument, default_hex) { r @@ -67,6 +67,13 @@ pub fn parse_tool_input_sexp( return Err("missing argument {argument}".to_string()); }; + let use_sexp_text = + if sexp_text.is_empty() { + default_hex.unwrap_or_default() + } else { + &sexp_text + }.to_string(); + let sexp_serialized = Bytes::new_validated(Some(UnvalidatedBytesFromType::Hex(use_sexp_text.clone()))) .map_err(|e| format!("{e:?}"))?; From 9bb05ff43770ff42c6d6fff99efdddf4e7f99169 Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 5 Jul 2024 13:08:26 -0700 Subject: [PATCH 07/14] fmt + clippy --- src/classic/clvm_tools/comp_input.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/classic/clvm_tools/comp_input.rs b/src/classic/clvm_tools/comp_input.rs index 448a1b52c..3216b2535 100644 --- a/src/classic/clvm_tools/comp_input.rs +++ b/src/classic/clvm_tools/comp_input.rs @@ -67,12 +67,12 @@ pub fn parse_tool_input_sexp( return Err("missing argument {argument}".to_string()); }; - let use_sexp_text = - if sexp_text.is_empty() { - default_hex.unwrap_or_default() - } else { - &sexp_text - }.to_string(); + let use_sexp_text = if sexp_text.is_empty() { + default_hex.unwrap_or_default() + } else { + &sexp_text + } + .to_string(); let sexp_serialized = Bytes::new_validated(Some(UnvalidatedBytesFromType::Hex(use_sexp_text.clone()))) From 3cbda7e4f36f77b954296e604877bb44b65f0dec Mon Sep 17 00:00:00 2001 From: arty Date: Sat, 13 Jul 2024 16:04:23 -0700 Subject: [PATCH 08/14] Merge in fix for accidental parsing --- .github/workflows/build-test.yml | 2 +- .github/workflows/dependency-review.yml | 1 + resources/tests/bin-quote.bin | 1 + src/compiler/preprocessor/mod.rs | 3 +- src/compiler/sexp.rs | 2 + src/tests/classic/run.rs | 73 ++++++++++++++++--------- 6 files changed, 53 insertions(+), 29 deletions(-) create mode 100644 resources/tests/bin-quote.bin diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index d1c4d9864..aadb4cf36 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -153,7 +153,7 @@ jobs: # Check that recompiling deployed puzzles match with their deployed hashes cp support/install_deps.sh support/verify_compiler_version.sh chia-blockchain - (cd chia-blockchain && python -m venv venv && . venv/bin/activate && pip install --upgrade pip && \ + (cd chia-blockchain && python -m venv .venv && . .venv/bin/activate && pip install --upgrade pip && \ python -m pip install maturin==1.1.0 && \ cd .. && pip install --no-index --find-links target/wheels/ clvm_tools_rs && \ cd chia-blockchain && \ diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 3bc3e1a77..53a3bbdd8 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -21,4 +21,5 @@ jobs: - name: "Dependency Review" uses: actions/dependency-review-action@v4 with: + allow-dependencies-licenses: pkg:pypi/pylint, pkg:pypi/pyinstaller deny-licenses: AGPL-1.0-only, AGPL-1.0-or-later, AGPL-1.0-or-later, AGPL-3.0-or-later, GPL-1.0-only, GPL-1.0-or-later, GPL-2.0-only, GPL-2.0-or-later, GPL-3.0-only, GPL-3.0-or-later diff --git a/resources/tests/bin-quote.bin b/resources/tests/bin-quote.bin new file mode 100644 index 000000000..0c26c19c1 --- /dev/null +++ b/resources/tests/bin-quote.bin @@ -0,0 +1 @@ +'test \ No newline at end of file diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index 97895375b..9ba80e4a6 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -199,7 +199,8 @@ impl Preprocessor { desc: IncludeDesc, ) -> Result<(), CompileErr> { let name_string = decode_string(&desc.name); - if KNOWN_DIALECTS.contains_key(&name_string) { + // Terminate early checking anything with a processed include type. + if KNOWN_DIALECTS.contains_key(&name_string) || desc.kind.is_some() { return Ok(()); } diff --git a/src/compiler/sexp.rs b/src/compiler/sexp.rs index 173fd7419..eee39c7b5 100644 --- a/src/compiler/sexp.rs +++ b/src/compiler/sexp.rs @@ -276,6 +276,8 @@ fn normalize_int(v: Vec, base: u32) -> Number { fn from_hex(l: Srcloc, v: &[u8]) -> SExp { let mut result = vec![0; (v.len() - 1) / 2]; let mut hex_const; + // This assigns a new reference so the vec is copied only when we need + // to pad it. let v_ref = if v.len() % 2 == 1 { hex_const = v.to_vec(); hex_const.insert(2, b'0'); diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index 9a5823d6f..6ffce7057 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -2429,29 +2429,6 @@ fn test_assign_cse_tricky_2() { assert_eq!(program, wanted_repr); } -#[test] -fn test_include_zero_bin() { - let program = do_basic_run(&vec![ - "run".to_string(), - "-i".to_string(), - "resources/tests".to_string(), - "(mod (X) (include *standard-cl-23.1*) (embed-file lz bin lz.bin) (concat 1 lz))" - .to_string(), - ]); - assert_eq!(program, "(2 (1 14 (1 . 1) 2) (4 (1 . 0x0001) 1))"); -} - -#[test] -fn test_include_zero_bin_pre_fix() { - let program = do_basic_run(&vec![ - "run".to_string(), - "-i".to_string(), - "resources/tests".to_string(), - "(mod (X) (include *standard-cl-23*) (embed-file lz bin lz.bin) (concat 1 lz))".to_string(), - ]); - assert_eq!(program, "(2 (1 14 (1 . 1) 2) (4 (1 . 1) 1))"); -} - #[test] fn test_classic_modpow() { let result = do_basic_brun(&vec![ @@ -2474,20 +2451,26 @@ fn test_classic_mod_op() { #[test] fn test_modern_modpow() { - let result = do_basic_brun(&vec![ + let program = do_basic_run(&vec![ "run".to_string(), "(mod (X Y Z) (include *standard-cl-23*) (modpow X Y Z))".to_string(), ]); - assert_eq!(result.trim(), "(60 2 5 11)"); + assert_eq!(program.trim(), "(60 2 5 11)"); + let result = do_basic_brun(&vec!["brun".to_string(), program, "(2 7 10)".to_string()]); + // 128 % 10 == 8 + assert_eq!(result.trim(), "8"); } #[test] fn test_modern_mod_op() { - let result = do_basic_brun(&vec![ + let program = do_basic_run(&vec![ "run".to_string(), "(mod (X Y) (include *standard-cl-23*) (% X Y))".to_string(), ]); - assert_eq!(result.trim(), "(61 2 5)"); + assert_eq!(program.trim(), "(61 2 5)"); + let result = do_basic_brun(&vec!["brun".to_string(), program, "(137 6)".to_string()]); + // 137 % 6 == 5 + assert_eq!(result.trim(), "5"); } #[test] @@ -2506,3 +2489,39 @@ fn test_quote_string_generation() { "(0x7465737422 \"test'\" \"test hi\" 0x7465737422 \"test'\" \"test_hi\")" ); } + +#[test] +fn test_include_zero_bin() { + let program = do_basic_run(&vec![ + "run".to_string(), + "-i".to_string(), + "resources/tests".to_string(), + "(mod (X) (include *standard-cl-23.1*) (embed-file lz bin lz.bin) (concat 1 lz))" + .to_string(), + ]); + assert_eq!(program, "(2 (1 14 (1 . 1) 2) (4 (1 . 0x0001) 1))"); +} + +#[test] +fn test_include_zero_bin_pre_fix() { + let program = do_basic_run(&vec![ + "run".to_string(), + "-i".to_string(), + "resources/tests".to_string(), + "(mod (X) (include *standard-cl-23*) (embed-file lz bin lz.bin) (concat 1 lz))".to_string(), + ]); + assert_eq!(program, "(2 (1 14 (1 . 1) 2) (4 (1 . 1) 1))"); +} + +#[test] +fn test_include_bin_should_not_be_parsed() { + let program = do_basic_run(&vec![ + "run".to_string(), + "-i".to_string(), + "resources/tests".to_string(), + "(mod (X) (include *standard-cl-23.1*) (embed-file test bin bin-quote.bin) test)" + .to_string(), + ]); + let result = do_basic_brun(&vec!["brun".to_string(), program]); + assert_eq!(result.trim(), "\"'test\""); +} From 026fda13ce4d317dc9d3123703e380d665032724 Mon Sep 17 00:00:00 2001 From: arty Date: Sat, 20 Jul 2024 14:56:30 -0700 Subject: [PATCH 09/14] Enable ops outside guard. I had believed this flag would be removed but wasn't removed at the time of this clvmr update. Will do a more extensive upgrade pr as well. --- Cargo.lock | 74 +++++++++---------- Cargo.toml | 2 +- src/classic/clvm_tools/stages/stage_0.rs | 4 +- .../clvm_tools/stages/stage_2/operators.rs | 4 +- 4 files changed, 41 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 44150012d..d06048ea3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -105,9 +105,9 @@ dependencies = [ [[package]] name = "blst" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c94087b935a822949d3291a9989ad2b2051ea141eda0fd4e478a75f6aa3e604b" +checksum = "62dc83a094a71d43eeadd254b1ec2d24cb6a0bb6cadce00df51f0db594711a32" dependencies = [ "cc", "glob", @@ -186,14 +186,14 @@ dependencies = [ "proc-macro-crate", "proc-macro2 1.0.66", "quote 1.0.32", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] name = "clvm-traits" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b527f38a1a9ef23aafd982a89be3881fa5f0a0ba0a642fa94244c72a74b8c8" +checksum = "5548b7de9921c68b2ca1101c08dd416bfa9afdf319f5b45183ecb3574ee177a3" dependencies = [ "num-bigint", "thiserror", @@ -237,9 +237,9 @@ dependencies = [ [[package]] name = "clvmr" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dede49a536ac5ad83753cf241f674d0c7b251a2a58ae681577457be9c5ea7bc3" +checksum = "deabd4e4163efb64486937a7c80996e5bdfb88d21a6baa9c0f802e6b45906109" dependencies = [ "chia-bls", "clvm-traits", @@ -371,9 +371,9 @@ checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "elliptic-curve" -version = "0.13.5" +version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "968405c8fdc9b3bf4df0a6638858cc0b52462836ab6b1c87377785dd09cf1c0b" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ "base16ct", "crypto-bigint", @@ -585,9 +585,9 @@ dependencies = [ [[package]] name = "k256" -version = "0.13.1" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cadb76004ed8e97623117f3df85b17aaa6626ab0b0831e6573f104df16cd1bcc" +checksum = "956ff9b67e26e1a6a866cb758f12c6f8746208489e3e4a4b5580802f2f0a587b" dependencies = [ "cfg-if", "ecdsa", @@ -599,9 +599,9 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "lfsr" @@ -723,11 +723,10 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.3" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ - "autocfg", "num-integer", "num-traits", "serde", @@ -744,11 +743,10 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] @@ -777,9 +775,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] @@ -796,9 +794,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "opaque-debug" @@ -978,7 +976,7 @@ dependencies = [ "proc-macro2 1.0.66", "pyo3-macros-backend", "quote 1.0.32", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -990,7 +988,7 @@ dependencies = [ "heck", "proc-macro2 1.0.66", "quote 1.0.32", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -1172,7 +1170,7 @@ checksum = "24e744d7782b686ab3b73267ef05697159cc0e5abbed3f47f9933165e5219036" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.32", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -1266,9 +1264,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.28" +version = "2.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" +checksum = "239814284fd6f1a4ffe4ca893952cdd93c224b6a1571c9a9eadd670295c0c9e2" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.32", @@ -1317,7 +1315,7 @@ checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.32", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -1350,9 +1348,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" dependencies = [ "tinyvec_macros", ] @@ -1394,9 +1392,9 @@ checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] @@ -1454,7 +1452,7 @@ dependencies = [ "once_cell", "proc-macro2 1.0.66", "quote 1.0.32", - "syn 2.0.28", + "syn 2.0.32", "wasm-bindgen-shared", ] @@ -1488,7 +1486,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.32", - "syn 2.0.28", + "syn 2.0.32", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -1650,9 +1648,9 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.6.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" dependencies = [ "zeroize_derive", ] @@ -1665,5 +1663,5 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.32", - "syn 2.0.28", + "syn 2.0.32", ] diff --git a/Cargo.toml b/Cargo.toml index 9a31e179c..942085fc4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ do-notation = "0.1.3" serde_json = "1.0" sha2 = "0.9.5" tempfile = "3.3.0" -clvmr = { version = "0.3.2", features = ["pre-eval"] } +clvmr = { version = "0.3.3", features = ["pre-eval"] } binascii = "0.1.4" yaml-rust = "0.4" linked-hash-map = "0.5.6" diff --git a/src/classic/clvm_tools/stages/stage_0.rs b/src/classic/clvm_tools/stages/stage_0.rs index df5fd9b23..638b050d5 100644 --- a/src/classic/clvm_tools/stages/stage_0.rs +++ b/src/classic/clvm_tools/stages/stage_0.rs @@ -1,5 +1,5 @@ use clvm_rs::allocator::{Allocator, NodePtr}; -use clvm_rs::chia_dialect::{ChiaDialect, NO_UNKNOWN_OPS}; +use clvm_rs::chia_dialect::{ChiaDialect, NO_UNKNOWN_OPS, ENABLE_BLS_OPS_OUTSIDE_GUARD}; use clvm_rs::cost::Cost; use clvm_rs::reduction::Response; @@ -47,7 +47,7 @@ impl TRunProgram for DefaultProgramRunner { run_program_with_pre_eval( allocator, - &ChiaDialect::new(NO_UNKNOWN_OPS), + &ChiaDialect::new(NO_UNKNOWN_OPS | ENABLE_BLS_OPS_OUTSIDE_GUARD), program, args, max_cost, diff --git a/src/classic/clvm_tools/stages/stage_2/operators.rs b/src/classic/clvm_tools/stages/stage_2/operators.rs index 5b433395f..19f00c634 100644 --- a/src/classic/clvm_tools/stages/stage_2/operators.rs +++ b/src/classic/clvm_tools/stages/stage_2/operators.rs @@ -5,7 +5,7 @@ use std::path::PathBuf; use std::rc::Rc; use clvm_rs::allocator::{Allocator, NodePtr, SExp}; -use clvm_rs::chia_dialect::{ChiaDialect, NO_UNKNOWN_OPS}; +use clvm_rs::chia_dialect::{ChiaDialect, NO_UNKNOWN_OPS, ENABLE_BLS_OPS_OUTSIDE_GUARD}; use clvm_rs::cost::Cost; use clvm_rs::dialect::{Dialect, OperatorSet}; use clvm_rs::reduction::{EvalErr, Reduction, Response}; @@ -122,7 +122,7 @@ impl Drop for CompilerOperators { impl CompilerOperatorsInternal { pub fn new(source_file: &str, search_paths: Vec, symbols_extra_info: bool) -> Self { - let base_dialect = Rc::new(ChiaDialect::new(NO_UNKNOWN_OPS)); + let base_dialect = Rc::new(ChiaDialect::new(NO_UNKNOWN_OPS | ENABLE_BLS_OPS_OUTSIDE_GUARD)); let base_runner = Rc::new(DefaultProgramRunner::new()); CompilerOperatorsInternal { base_dialect, From 3db67345382152e9070dd814f99a9c73f9aa0729 Mon Sep 17 00:00:00 2001 From: arty Date: Sat, 20 Jul 2024 15:14:30 -0700 Subject: [PATCH 10/14] Add tests --- src/classic/clvm_tools/stages/stage_0.rs | 2 +- .../clvm_tools/stages/stage_2/operators.rs | 6 +++-- src/tests/classic/run.rs | 12 ++++++++++ src/tests/compiler/cldb.rs | 24 +++++++++++++++++++ 4 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/classic/clvm_tools/stages/stage_0.rs b/src/classic/clvm_tools/stages/stage_0.rs index 638b050d5..f99ee3f67 100644 --- a/src/classic/clvm_tools/stages/stage_0.rs +++ b/src/classic/clvm_tools/stages/stage_0.rs @@ -1,5 +1,5 @@ use clvm_rs::allocator::{Allocator, NodePtr}; -use clvm_rs::chia_dialect::{ChiaDialect, NO_UNKNOWN_OPS, ENABLE_BLS_OPS_OUTSIDE_GUARD}; +use clvm_rs::chia_dialect::{ChiaDialect, ENABLE_BLS_OPS_OUTSIDE_GUARD, NO_UNKNOWN_OPS}; use clvm_rs::cost::Cost; use clvm_rs::reduction::Response; diff --git a/src/classic/clvm_tools/stages/stage_2/operators.rs b/src/classic/clvm_tools/stages/stage_2/operators.rs index 19f00c634..11c3b8d00 100644 --- a/src/classic/clvm_tools/stages/stage_2/operators.rs +++ b/src/classic/clvm_tools/stages/stage_2/operators.rs @@ -5,7 +5,7 @@ use std::path::PathBuf; use std::rc::Rc; use clvm_rs::allocator::{Allocator, NodePtr, SExp}; -use clvm_rs::chia_dialect::{ChiaDialect, NO_UNKNOWN_OPS, ENABLE_BLS_OPS_OUTSIDE_GUARD}; +use clvm_rs::chia_dialect::{ChiaDialect, ENABLE_BLS_OPS_OUTSIDE_GUARD, NO_UNKNOWN_OPS}; use clvm_rs::cost::Cost; use clvm_rs::dialect::{Dialect, OperatorSet}; use clvm_rs::reduction::{EvalErr, Reduction, Response}; @@ -122,7 +122,9 @@ impl Drop for CompilerOperators { impl CompilerOperatorsInternal { pub fn new(source_file: &str, search_paths: Vec, symbols_extra_info: bool) -> Self { - let base_dialect = Rc::new(ChiaDialect::new(NO_UNKNOWN_OPS | ENABLE_BLS_OPS_OUTSIDE_GUARD)); + let base_dialect = Rc::new(ChiaDialect::new( + NO_UNKNOWN_OPS | ENABLE_BLS_OPS_OUTSIDE_GUARD, + )); let base_runner = Rc::new(DefaultProgramRunner::new()); CompilerOperatorsInternal { base_dialect, diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index ec6a31f25..4c3f687b0 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -2543,3 +2543,15 @@ fn test_include_bin_should_not_be_parsed() { let result = do_basic_brun(&vec!["brun".to_string(), program]); assert_eq!(result.trim(), "\"'test\""); } + +#[test] +fn test_coinid_outside_guard() { + let result = do_basic_run(&vec![ + "run".to_string(), + "(coinid (sha256 3) (sha256 3) 4)".to_string(), + ]); + assert_eq!( + result.trim(), + "0x9f7f12b86a583805a4442879b7b5b531469e45c7e753e5fd431058e90bf3fbec" + ); +} diff --git a/src/tests/compiler/cldb.rs b/src/tests/compiler/cldb.rs index c22074a22..1a72143bf 100644 --- a/src/tests/compiler/cldb.rs +++ b/src/tests/compiler/cldb.rs @@ -78,6 +78,7 @@ where } if let Some(result) = cldbrun.step(&mut allocator) { + eprintln!("{:?}", result); output = result; if !viewer.show(&cldbrun.current_step(), Some(output.clone())) { return None; @@ -455,3 +456,26 @@ fn test_cldb_hierarchy_before_hex() { compare_run_output(result, run_entries); } + +#[test] +fn test_cldb_coinid_outside_guard() { + let filename = "coinid.clvm"; + let loc = Srcloc::start(filename); + let program = "(coinid (sha256 (q . 3)) (sha256 (q . 3)) (q . 4))"; + let parsed = parse_sexp(loc.clone(), program.as_bytes().iter().copied()).expect("should parse"); + let args = Rc::new(SExp::Nil(loc)); + let program_lines = Rc::new(vec![program.to_string()]); + + assert_eq!( + run_clvm_in_cldb( + filename, + program_lines, + parsed[0].clone(), + HashMap::new(), + args, + &mut DoesntWatchCldb {}, + FAVOR_HEX, + ), + Some("0x9f7f12b86a583805a4442879b7b5b531469e45c7e753e5fd431058e90bf3fbec".to_string()) + ); +} From 860bbeb98243e36754b892bc95927ded648d0178 Mon Sep 17 00:00:00 2001 From: arty Date: Sat, 20 Jul 2024 15:19:05 -0700 Subject: [PATCH 11/14] Update to use = in version --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 942085fc4..b43c83e49 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ do-notation = "0.1.3" serde_json = "1.0" sha2 = "0.9.5" tempfile = "3.3.0" -clvmr = { version = "0.3.3", features = ["pre-eval"] } +clvmr = { version = "=0.3.3", features = ["pre-eval"] } binascii = "0.1.4" yaml-rust = "0.4" linked-hash-map = "0.5.6" From cb44f52375bcc8a85c2341e920eecb0259bba35c Mon Sep 17 00:00:00 2001 From: arty Date: Sat, 20 Jul 2024 15:32:09 -0700 Subject: [PATCH 12/14] Better: classic already handled this differently --- src/classic/clvm_tools/cmds.rs | 1 + src/classic/clvm_tools/stages/stage_0.rs | 5 ++++- src/compiler/clvm.rs | 7 +++++-- src/tests/classic/run.rs | 12 ------------ src/tests/compiler/optimizer/output.rs | 3 +-- 5 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/classic/clvm_tools/cmds.rs b/src/classic/clvm_tools/cmds.rs index af5073109..9811c19a6 100644 --- a/src/classic/clvm_tools/cmds.rs +++ b/src/classic/clvm_tools/cmds.rs @@ -1575,6 +1575,7 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul Some(max_cost as u64) }, pre_eval_f, + new_operators: false, strict: parsed_args .get("strict") .map(|_| true) diff --git a/src/classic/clvm_tools/stages/stage_0.rs b/src/classic/clvm_tools/stages/stage_0.rs index f99ee3f67..4bd73e60b 100644 --- a/src/classic/clvm_tools/stages/stage_0.rs +++ b/src/classic/clvm_tools/stages/stage_0.rs @@ -5,10 +5,12 @@ use clvm_rs::reduction::Response; use clvm_rs::run_program::{run_program_with_pre_eval, PreEval}; +#[derive(Default)] pub struct RunProgramOption { pub max_cost: Option, pub pre_eval_f: Option, pub strict: bool, + pub new_operators: bool, } pub trait TRunProgram { @@ -44,10 +46,11 @@ impl TRunProgram for DefaultProgramRunner { option: Option, ) -> Response { let max_cost = option.as_ref().and_then(|o| o.max_cost).unwrap_or(0); + let new_operators = option.as_ref().map(|o| o.new_operators).unwrap_or_default(); run_program_with_pre_eval( allocator, - &ChiaDialect::new(NO_UNKNOWN_OPS | ENABLE_BLS_OPS_OUTSIDE_GUARD), + &ChiaDialect::new(NO_UNKNOWN_OPS | ((new_operators as u32) * ENABLE_BLS_OPS_OUTSIDE_GUARD)), program, args, max_cost, diff --git a/src/compiler/clvm.rs b/src/compiler/clvm.rs index 41a8b4cbf..d38cd6774 100644 --- a/src/compiler/clvm.rs +++ b/src/compiler/clvm.rs @@ -13,7 +13,7 @@ use sha2::Digest; use sha2::Sha256; use crate::classic::clvm::__type_compatibility__::{bi_one, bi_zero}; -use crate::classic::clvm_tools::stages::stage_0::TRunProgram; +use crate::classic::clvm_tools::stages::stage_0::{TRunProgram, RunProgramOption}; use crate::compiler::prims; use crate::compiler::runtypes::RunFailure; @@ -396,7 +396,10 @@ fn apply_op( let converted_args = convert_to_clvm_rs(allocator, wrapped_args.clone())?; runner - .run_program(allocator, converted_app, converted_args, None) + .run_program(allocator, converted_app, converted_args, Some(RunProgramOption { + new_operators: true, + .. RunProgramOption::default() + })) .map_err(|e| { RunFailure::RunErr( head.loc(), diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index 4c3f687b0..ec6a31f25 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -2543,15 +2543,3 @@ fn test_include_bin_should_not_be_parsed() { let result = do_basic_brun(&vec!["brun".to_string(), program]); assert_eq!(result.trim(), "\"'test\""); } - -#[test] -fn test_coinid_outside_guard() { - let result = do_basic_run(&vec![ - "run".to_string(), - "(coinid (sha256 3) (sha256 3) 4)".to_string(), - ]); - assert_eq!( - result.trim(), - "0x9f7f12b86a583805a4442879b7b5b531469e45c7e753e5fd431058e90bf3fbec" - ); -} diff --git a/src/tests/compiler/optimizer/output.rs b/src/tests/compiler/optimizer/output.rs index 6ac41c882..90b01aef4 100644 --- a/src/tests/compiler/optimizer/output.rs +++ b/src/tests/compiler/optimizer/output.rs @@ -47,8 +47,7 @@ fn run_with_cost( as_classic_env, Some(RunProgramOption { max_cost: Some(MAX_RUN_COST), - pre_eval_f: None, - strict: false, + .. RunProgramOption::default() }), ) .map_err(|e| RunFailure::RunErr(sexp.loc(), format!("{} in {} {}", e.1, sexp, env))) From b5fb3d1c0c076f938a4e3841a7cb615126344ddd Mon Sep 17 00:00:00 2001 From: arty Date: Sat, 20 Jul 2024 15:33:25 -0700 Subject: [PATCH 13/14] fmt --- src/classic/clvm_tools/stages/stage_0.rs | 4 +++- src/compiler/clvm.rs | 15 ++++++++++----- src/tests/compiler/optimizer/output.rs | 2 +- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/classic/clvm_tools/stages/stage_0.rs b/src/classic/clvm_tools/stages/stage_0.rs index 4bd73e60b..6aba8c648 100644 --- a/src/classic/clvm_tools/stages/stage_0.rs +++ b/src/classic/clvm_tools/stages/stage_0.rs @@ -50,7 +50,9 @@ impl TRunProgram for DefaultProgramRunner { run_program_with_pre_eval( allocator, - &ChiaDialect::new(NO_UNKNOWN_OPS | ((new_operators as u32) * ENABLE_BLS_OPS_OUTSIDE_GUARD)), + &ChiaDialect::new( + NO_UNKNOWN_OPS | ((new_operators as u32) * ENABLE_BLS_OPS_OUTSIDE_GUARD), + ), program, args, max_cost, diff --git a/src/compiler/clvm.rs b/src/compiler/clvm.rs index d38cd6774..2d01ee413 100644 --- a/src/compiler/clvm.rs +++ b/src/compiler/clvm.rs @@ -13,7 +13,7 @@ use sha2::Digest; use sha2::Sha256; use crate::classic::clvm::__type_compatibility__::{bi_one, bi_zero}; -use crate::classic::clvm_tools::stages::stage_0::{TRunProgram, RunProgramOption}; +use crate::classic::clvm_tools::stages::stage_0::{RunProgramOption, TRunProgram}; use crate::compiler::prims; use crate::compiler::runtypes::RunFailure; @@ -396,10 +396,15 @@ fn apply_op( let converted_args = convert_to_clvm_rs(allocator, wrapped_args.clone())?; runner - .run_program(allocator, converted_app, converted_args, Some(RunProgramOption { - new_operators: true, - .. RunProgramOption::default() - })) + .run_program( + allocator, + converted_app, + converted_args, + Some(RunProgramOption { + new_operators: true, + ..RunProgramOption::default() + }), + ) .map_err(|e| { RunFailure::RunErr( head.loc(), diff --git a/src/tests/compiler/optimizer/output.rs b/src/tests/compiler/optimizer/output.rs index 90b01aef4..36ee6dec0 100644 --- a/src/tests/compiler/optimizer/output.rs +++ b/src/tests/compiler/optimizer/output.rs @@ -47,7 +47,7 @@ fn run_with_cost( as_classic_env, Some(RunProgramOption { max_cost: Some(MAX_RUN_COST), - .. RunProgramOption::default() + ..RunProgramOption::default() }), ) .map_err(|e| RunFailure::RunErr(sexp.loc(), format!("{} in {} {}", e.1, sexp, env))) From 5dd4de0147173c00a1f65b8743e240bfffdf350c Mon Sep 17 00:00:00 2001 From: arty Date: Sat, 20 Jul 2024 15:38:49 -0700 Subject: [PATCH 14/14] Reverse part of this because it's taken care of by operator selection --- src/classic/clvm_tools/stages/stage_2/operators.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/classic/clvm_tools/stages/stage_2/operators.rs b/src/classic/clvm_tools/stages/stage_2/operators.rs index 11c3b8d00..5b433395f 100644 --- a/src/classic/clvm_tools/stages/stage_2/operators.rs +++ b/src/classic/clvm_tools/stages/stage_2/operators.rs @@ -5,7 +5,7 @@ use std::path::PathBuf; use std::rc::Rc; use clvm_rs::allocator::{Allocator, NodePtr, SExp}; -use clvm_rs::chia_dialect::{ChiaDialect, ENABLE_BLS_OPS_OUTSIDE_GUARD, NO_UNKNOWN_OPS}; +use clvm_rs::chia_dialect::{ChiaDialect, NO_UNKNOWN_OPS}; use clvm_rs::cost::Cost; use clvm_rs::dialect::{Dialect, OperatorSet}; use clvm_rs::reduction::{EvalErr, Reduction, Response}; @@ -122,9 +122,7 @@ impl Drop for CompilerOperators { impl CompilerOperatorsInternal { pub fn new(source_file: &str, search_paths: Vec, symbols_extra_info: bool) -> Self { - let base_dialect = Rc::new(ChiaDialect::new( - NO_UNKNOWN_OPS | ENABLE_BLS_OPS_OUTSIDE_GUARD, - )); + let base_dialect = Rc::new(ChiaDialect::new(NO_UNKNOWN_OPS)); let base_runner = Rc::new(DefaultProgramRunner::new()); CompilerOperatorsInternal { base_dialect,