diff --git a/Cargo.lock b/Cargo.lock index 3f7163200ee68..eae7d5b3eef90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8281,6 +8281,27 @@ dependencies = [ "move-transactional-test-runner", ] +[[package]] +name = "move-compiler-v2" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 3.2.23", + "codespan", + "codespan-reporting", + "datatest-stable", + "itertools", + "move-binary-format", + "move-core-types", + "move-ir-to-bytecode", + "move-model", + "move-prover-test-utils", + "move-stdlib", + "num", + "once_cell", + "serde 1.0.149", +] + [[package]] name = "move-core-types" version = "0.0.4" diff --git a/Cargo.toml b/Cargo.toml index ba32a6fad3c61..388403e035c30 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -170,6 +170,7 @@ members = [ "third_party/move/move-bytecode-verifier/transactional-tests", "third_party/move/move-command-line-common", "third_party/move/move-compiler", + "third_party/move/move-compiler-v2", "third_party/move/move-compiler/transactional-tests", "third_party/move/move-core/types", "third_party/move/move-ir-compiler", @@ -615,6 +616,7 @@ move-cli = { path = "third_party/move/tools/move-cli" } move-command-line-common = { path = "third_party/move/move-command-line-common" } move-coverage = { path = "third_party/move/tools/move-coverage" } move-compiler = { path = "third_party/move/move-compiler" } +move-compiler-v2 = { path = "third_party/move/move-compiler-v2" } move-core-types = { path = "third_party/move/move-core/types" } move-docgen = { path = "third_party/move/move-prover/move-docgen" } move-disassembler = { path = "third_party/move/tools/move-disassembler" } diff --git a/third_party/move/move-compiler-v2/Cargo.toml b/third_party/move/move-compiler-v2/Cargo.toml new file mode 100644 index 0000000000000..ce0665227f41d --- /dev/null +++ b/third_party/move/move-compiler-v2/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "move-compiler-v2" +version = "0.1.0" +authors = ["Aptos Labs"] +description = "Move compiler based on stackless bytecode" +repository = "https://github.com/aptos-labs/aptos-core" +homepage = "https://aptosfoundation.org/" +license = "Apache-2.0" +publish = false +edition = "2021" + +[dependencies] +move-binary-format = { path = "../move-binary-format" } +#move-borrow-graph = { path = "../../move-borrow-graph" } +#move-bytecode-verifier = { path = "../../move-bytecode-verifier" } +#move-command-line-common = { path = "../../move-command-line-common" } +#move-compiler = { path = "../../move-compiler" } +move-core-types = { path = "../move-core/types" } +move-ir-to-bytecode = { path = "../move-ir-compiler/move-ir-to-bytecode" } +move-model = { path = "../move-model" } + +anyhow = "1.0.62" +clap = { version = "3.2.23", features = ["derive", "env"] } +codespan = "0.11.1" +codespan-reporting = { version = "0.11.1", features = ["serde", "serialization"] } +#ethnum = "1.0.4" +#im = "15.0.0" +itertools = "0.10.0" +#log = "0.4.14" +num = "0.4.0" +once_cell = "1.7.2" +#paste = "1.0.5" +#petgraph = "0.5.1" +serde = { version = "1.0.124", features = ["derive"] } + +[dev-dependencies] +anyhow = "1.0.52" +datatest-stable = "0.1.1" +move-prover-test-utils = { path = "../move-prover/test-utils" } +move-stdlib = { path = "../move-stdlib" } + +[[test]] +name = "testsuite" +harness = false diff --git a/third_party/move/move-compiler-v2/src/experiments.rs b/third_party/move/move-compiler-v2/src/experiments.rs new file mode 100644 index 0000000000000..1de5096036267 --- /dev/null +++ b/third_party/move/move-compiler-v2/src/experiments.rs @@ -0,0 +1,27 @@ +// Copyright © Aptos Foundation +// Parts of the project are originally copyright © Meta Platforms, Inc. +// SPDX-License-Identifier: Apache-2.0 + +/// Container for experiments in the compiler. Those can be activated +/// via the `--experiment=` flag. +/// +/// One can activate an experiment in a test source by using a comment as such: +/// ``` +/// // experiment: +/// ``` +/// This can be repeated an arbitrary number of times. The test will then be run for +/// the default configuration and for each of the named experiments separately (if it is a +/// baseline test, a different baseline file will be generated each time). +/// +/// Each new experiment should have a description and explicit note about its retention. +/// +/// - Permanent: the experiment is available indefinitely +/// - Temporary: the experiment is intended to be removed after some time. Please document +/// the condition under which it can be removed. +pub struct Experiment(); + +impl Experiment { + /// Whether to exit after type checking. + /// Retention: permanent + pub const CHECK_ONLY: &'static str = "check-only"; +} diff --git a/third_party/move/move-compiler-v2/src/lib.rs b/third_party/move/move-compiler-v2/src/lib.rs new file mode 100644 index 0000000000000..4b0d801e8dbe2 --- /dev/null +++ b/third_party/move/move-compiler-v2/src/lib.rs @@ -0,0 +1,63 @@ +// Copyright © Aptos Foundation +// Parts of the project are originally copyright © Meta Platforms, Inc. +// SPDX-License-Identifier: Apache-2.0 + +mod experiments; +mod options; + +use anyhow::anyhow; +use codespan_reporting::term::termcolor::{ColorChoice, StandardStream, WriteColor}; +pub use experiments::*; +use move_model::{model::GlobalEnv, PackageInfo}; +pub use options::*; + +/// Run Move compiler and print errors to stderr. +pub fn run_move_compiler_to_stderr(options: Options) -> anyhow::Result<()> { + let mut error_writer = StandardStream::stderr(ColorChoice::Auto); + run_move_compiler(&mut error_writer, options) +} + +/// Run move compiler and print errors to given writer. +pub fn run_move_compiler( + error_writer: &mut W, + options: Options, +) -> anyhow::Result<()> { + // Run the model builder, which performs context checking. + let addrs = move_model::parse_addresses_from_options(options.named_address_mapping.clone())?; + let env = move_model::run_model_builder_in_compiler_mode( + PackageInfo { + sources: options.sources.clone(), + address_map: addrs.clone(), + }, + vec![PackageInfo { + sources: options.dependencies.clone(), + address_map: addrs, + }], + )?; + // If the model contains any errors, report them now and exit. + check_errors( + &env, + &options, + error_writer, + "exiting with Move build errors", + )?; + if options.experiment_on(Experiment::CHECK_ONLY) { + // Stop here + return Ok(()); + } + panic!("code generation NYI") +} + +pub fn check_errors( + env: &GlobalEnv, + options: &Options, + error_writer: &mut W, + msg: &'static str, +) -> anyhow::Result<()> { + env.report_diag(error_writer, options.report_severity()); + if env.has_errors() { + Err(anyhow!(msg)) + } else { + Ok(()) + } +} diff --git a/third_party/move/move-compiler-v2/src/options.rs b/third_party/move/move-compiler-v2/src/options.rs new file mode 100644 index 0000000000000..05d694852f4cf --- /dev/null +++ b/third_party/move/move-compiler-v2/src/options.rs @@ -0,0 +1,70 @@ +// Copyright © Aptos Foundation +// Parts of the project are originally copyright © Meta Platforms, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use clap::Parser; +use codespan_reporting::diagnostic::Severity; + +/// Defines options for a run of the compiler. +#[derive(Parser, Debug)] +#[clap(author, version, about)] +pub struct Options { + /// Directories where to lookup dependencies. + #[clap( + short, + takes_value(true), + multiple_values(true), + multiple_occurrences(true) + )] + pub dependencies: Vec, + /// Named address mapping. + #[clap( + short, + takes_value(true), + multiple_values(true), + multiple_occurrences(true) + )] + pub named_address_mapping: Vec, + /// Output directory. + #[clap(short)] + #[clap(long, default_value = "")] + pub output_dir: String, + /// Whether to dump intermediate bytecode for debugging. + #[clap(long = "dump-bytecode")] + pub dump_bytecode: bool, + /// Whether we generate code for tests. This specifically guarantees stable output + /// for baseline testing. + #[clap(long)] + pub testing: bool, + /// Active experiments. Experiments alter default behavior of the compiler. + /// See `Experiment` struct. + #[clap(short)] + #[clap( + long = "experiment", + takes_value(true), + multiple_values(true), + multiple_occurrences(true) + )] + pub experiments: Vec, + /// Sources to compile (positional arg, therefore last) + pub sources: Vec, +} + +impl Default for Options { + fn default() -> Self { + Parser::parse_from(std::iter::empty::()) + } +} + +impl Options { + /// Returns the least severity of diagnosis which shall be reported. + /// This is currently hardwired. + pub fn report_severity(&self) -> Severity { + Severity::Warning + } + + /// Returns true if an experiment is on. + pub fn experiment_on(&self, name: &str) -> bool { + self.experiments.iter().any(|s| s == name) + } +} diff --git a/third_party/move/move-compiler-v2/tests/checking/basic.exp.check-only b/third_party/move/move-compiler-v2/tests/checking/basic.exp.check-only new file mode 100644 index 0000000000000..0e29d3e65a0f6 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/checking/basic.exp.check-only @@ -0,0 +1,7 @@ +failed: exiting with Move build errors +Diagnostics: +error: expected `u64` but found `address` in expression + ┌─ tests/checking/basic.move:3:22 + │ +3 │ fun foo(): u64 { @22 } + │ ^^^ diff --git a/third_party/move/move-compiler-v2/tests/checking/basic.move b/third_party/move/move-compiler-v2/tests/checking/basic.move new file mode 100644 index 0000000000000..1af99b6741d8a --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/checking/basic.move @@ -0,0 +1,4 @@ +// experiment: check-only +module 0x1::basic { + fun foo(): u64 { @22 } +} diff --git a/third_party/move/move-compiler-v2/tests/testsuite.rs b/third_party/move/move-compiler-v2/tests/testsuite.rs new file mode 100644 index 0000000000000..ac19d10f5e09d --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/testsuite.rs @@ -0,0 +1,63 @@ +// Copyright © Aptos Foundation +// Parts of the project are originally copyright © Meta Platforms, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use codespan_reporting::term::termcolor::Buffer; +use move_compiler_v2::Options; +use move_prover_test_utils::{baseline_test, extract_test_directives}; +use std::path::{Path, PathBuf}; + +/// Extension for expected output files +pub const EXP_EXT: &str = "exp"; + +fn test_runner(path: &Path) -> datatest_stable::Result<()> { + let mut experiments = extract_test_directives(path, "// experiment:")?; + if experiments.is_empty() { + // If there is no experiment, use "" as the 'default' experiment. + experiments.push("".to_string()) // default experiment + } + let mut sources = extract_test_directives(path, "// dep:")?; + sources.push(path.to_string_lossy().to_string()); + let deps = vec![path_from_crate_root("../move-stdlib/sources")]; + + // For each experiment, run the compiler. + for exp in experiments { + // Construct options, compiler and collect output. + let mut options = Options { + testing: true, + sources: sources.clone(), + dependencies: deps.clone(), + named_address_mapping: vec!["std=0x1".to_string()], + ..Options::default() + }; + let file_ext = if exp.is_empty() { + EXP_EXT.to_string() + } else { + options.experiments.push(exp.clone()); + format!("{}.{}", EXP_EXT, exp) + }; + + let mut error_writer = Buffer::no_color(); + let mut out = match move_compiler_v2::run_move_compiler(&mut error_writer, options) { + Ok(_) => "succeeded".to_owned(), + Err(e) => format!("failed: {}", e), + }; + let diag = String::from_utf8_lossy(&error_writer.into_inner()).to_string(); + if !diag.is_empty() { + out = format!("{}\nDiagnostics:\n{}", out, diag) + } + + // Generate/check baseline. + let baseline_path = path.with_extension(file_ext); + baseline_test::verify_or_update_baseline(baseline_path.as_path(), &out)?; + } + Ok(()) +} + +fn path_from_crate_root(path: &str) -> String { + let mut buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + buf.push(path); + buf.to_string_lossy().to_string() +} + +datatest_stable::harness!(test_runner, "tests", r".*\.move$"); diff --git a/third_party/move/move-compiler/src/shared/mod.rs b/third_party/move/move-compiler/src/shared/mod.rs index 83a08169f556b..7468f59c31fd9 100644 --- a/third_party/move/move-compiler/src/shared/mod.rs +++ b/third_party/move/move-compiler/src/shared/mod.rs @@ -353,6 +353,17 @@ impl Flags { } } + pub fn model_compilation() -> Self { + Self { + test: false, + verify: true, + shadow: true, // allows overlapping between sources and deps + flavor: "".to_string(), + bytecode_version: None, + keep_testing_functions: true, + } + } + pub fn set_flavor(self, flavor: impl ToString) -> Self { Self { flavor: flavor.to_string(), diff --git a/third_party/move/move-model/src/lib.rs b/third_party/move/move-model/src/lib.rs index a4dafd3ceb4e9..4bca6399f38a0 100644 --- a/third_party/move/move-model/src/lib.rs +++ b/third_party/move/move-model/src/lib.rs @@ -61,10 +61,49 @@ pub mod ty; pub mod well_known; // ================================================================================================= -// Entry Point +// Entry Point V2 + +/// Represents information about a package: the sources it contains and the package private +/// address mapping. +#[derive(Debug, Clone)] +pub struct PackageInfo { + pub sources: Vec, + pub address_map: BTreeMap, +} -/// Build the move model with default compilation flags and default options and no named addresses. -/// This collects transitive dependencies for move sources from the provided directory list. +/// Builds the Move model for the v2 compiler. This builds the model, compiling both code +/// and specs from sources into the type-checked AST. No bytecode is attached to the model. +/// This currently uses the v1 compiler as the parser (up to expansion AST), after that +/// a new type checker. +pub fn run_model_builder_in_compiler_mode( + source: PackageInfo, + deps: Vec, +) -> anyhow::Result { + let to_package_paths = |PackageInfo { + sources, + address_map, + }| PackagePaths { + name: None, + paths: sources, + named_address_map: address_map, + }; + run_model_builder_with_options_and_compilation_flags( + vec![to_package_paths(source)], + deps.into_iter().map(to_package_paths).collect(), + ModelBuilderOptions { + compile_via_model: true, + ..ModelBuilderOptions::default() + }, + Flags::model_compilation(), + ) +} + +// ================================================================================================= +// Entry Point V1 + +/// Builds the move model with default compilation flags and default options. This calls +/// the move compiler v1 to compile to bytecode first and attach the generated bytecode to +/// the model. pub fn run_model_builder< Paths: Into + Clone, NamedAddress: Into + Clone, @@ -75,9 +114,7 @@ pub fn run_model_builder< run_model_builder_with_options(move_sources, deps, ModelBuilderOptions::default()) } -/// Build the move model with default compilation flags and custom options and a set of provided -/// named addreses. -/// This collects transitive dependencies for move sources from the provided directory list. +/// Build the move model with default compilation flags and custom options. pub fn run_model_builder_with_options< Paths: Into + Clone, NamedAddress: Into + Clone, @@ -95,7 +132,6 @@ pub fn run_model_builder_with_options< } /// Build the move model with custom compilation flags and custom options -/// This collects transitive dependencies for move sources from the provided directory list. pub fn run_model_builder_with_options_and_compilation_flags< Paths: Into + Clone, NamedAddress: Into + Clone, @@ -106,6 +142,7 @@ pub fn run_model_builder_with_options_and_compilation_flags< flags: Flags, ) -> anyhow::Result { let mut env = GlobalEnv::new(); + let compile_via_model = options.compile_via_model; env.set_extension(options); // Step 1: parse the program to get comments and a separation of targets and dependencies. @@ -238,65 +275,96 @@ pub fn run_model_builder_with_options_and_compilation_flags< E::Program { modules, scripts } }; - // Step 4: retrospectively add lambda-lifted function to expansion AST - let (compiler, inlining_ast) = match compiler - .at_expansion(expansion_ast.clone()) - .run::() - { - Err(diags) => { - add_move_lang_diagnostics(&mut env, diags); - return Ok(env); - }, - Ok(compiler) => compiler.into_ast(), - }; - - for (loc, module_id, expansion_module) in expansion_ast.modules.iter_mut() { - match inlining_ast.modules.get_(module_id) { - None => { - env.error( - &env.to_loc(&loc), - "unable to find matching module in inlining AST", - ); - }, - Some(inlining_module) => { - retrospective_lambda_lifting(inlining_module, expansion_module); + if !compile_via_model { + // Legacy compilation via v1 compiler + // TODO: eventually remove this code and related helpers + + // Step 4: retrospectively add lambda-lifted function to expansion AST + let (compiler, inlining_ast) = match compiler + .at_expansion(expansion_ast.clone()) + .run::() + { + Err(diags) => { + add_move_lang_diagnostics(&mut env, diags); + return Ok(env); }, + Ok(compiler) => compiler.into_ast(), + }; + + for (loc, module_id, expansion_module) in expansion_ast.modules.iter_mut() { + match inlining_ast.modules.get_(module_id) { + None => { + env.error( + &env.to_loc(&loc), + "unable to find matching module in inlining AST", + ); + }, + Some(inlining_module) => { + retrospective_lambda_lifting(inlining_module, expansion_module); + }, + } } - } - // Step 5: Run the compiler from instrumented expansion AST fully to the compiled units + // Step 5: Run the compiler from instrumented expansion AST fully to the compiled units - let units = match compiler - .at_expansion(expansion_ast.clone()) - .run::() - { - Err(diags) => { + let units = match compiler + .at_expansion(expansion_ast.clone()) + .run::() + { + Err(diags) => { + add_move_lang_diagnostics(&mut env, diags); + return Ok(env); + }, + Ok(compiler) => { + let (units, warnings) = compiler.into_compiled_units(); + if !warnings.is_empty() { + // NOTE: these diagnostics are just warnings. it should be feasible to continue the + // model building here. But before that, register the warnings to the `GlobalEnv` + // first so we get a chance to report these warnings as well. + add_move_lang_diagnostics(&mut env, warnings); + } + units + }, + }; + + // Check for bytecode verifier errors (there should not be any) + let diags = compiled_unit::verify_units(&units); + if !diags.is_empty() { add_move_lang_diagnostics(&mut env, diags); return Ok(env); - }, - Ok(compiler) => { - let (units, warnings) = compiler.into_compiled_units(); - if !warnings.is_empty() { - // NOTE: these diagnostics are just warnings. it should be feasible to continue the - // model building here. But before that, register the warnings to the `GlobalEnv` - // first so we get a chance to report these warnings as well. - add_move_lang_diagnostics(&mut env, warnings); - } - units - }, - }; + } - // Check for bytecode verifier errors (there should not be any) - let diags = compiled_unit::verify_units(&units); - if !diags.is_empty() { - add_move_lang_diagnostics(&mut env, diags); - return Ok(env); + // Now that it is known that the program has no errors, run the spec checker on verified units + // plus expanded AST. This will populate the environment including any errors. + run_spec_checker(&mut env, units, expansion_ast); + Ok(env) + } else { + // New compilation via model (compiler v2). The expansion AST will be type checked. + // No bytecode is attached. + run_move_checker(&mut env, expansion_ast); + Ok(env) } +} - // Now that it is known that the program has no errors, run the spec checker on verified units - // plus expanded AST. This will populate the environment including any errors. - run_model_compiler(&mut env, units, expansion_ast); - Ok(env) +fn run_move_checker(env: &mut GlobalEnv, program: E::Program) { + // TODO: verify that the expansion AST has modules in bottom-up dependency order, since this + // is a requirement for the builder. + let mut builder = ModelBuilder::new(env); + for (module_count, (module_id, module_def)) in program.modules.into_iter().enumerate() { + let loc = builder.to_loc(&module_def.loc); + let addr_bytes = builder.resolve_address(&loc, &module_id.value.address); + let module_name = ModuleName::from_address_bytes_and_name( + addr_bytes, + builder + .env + .symbol_pool() + .make(&module_id.value.module.0.value), + ); + // Assign new module id in the model. + let module_id = ModuleId::new(module_count); + let mut module_translator = ModuleBuilder::new(&mut builder, module_id, module_name); + module_translator.translate(loc, module_def, None); + } } fn collect_related_modules_recursive<'a>( @@ -452,11 +520,7 @@ fn script_into_module(compiled_script: CompiledScript) -> CompiledModule { } #[allow(deprecated)] -fn run_model_compiler( - env: &mut GlobalEnv, - units: Vec, - mut eprog: E::Program, -) { +fn run_spec_checker(env: &mut GlobalEnv, units: Vec, mut eprog: E::Program) { let mut builder = ModelBuilder::new(env); // Merge the compiled units with source ASTs, preserving the order of the compiled