[compiler v2] Setup basic driver and test environment

This creates a new crate `move-compiler-v2`, containing a few basic components:

- A model of options and experiments for the compiler. This has been adapted from the Move-to-Yul compiler from last year.
- A test driver which supports experiments, with a first smoke test to ensure basic functionality of the setup.
- Top level entry function to call the compiler. It invokes the  new compilation mode of the move-model, which types check and constructs an AST. After that is unimplemented.

The PR also adds an additional entry point for the move-model to support the above.

Cargo.toml

@@ -170,6 +170,7 @@ members = [
     "third_party/move/move-bytecode-verifier/transactional-tests",
     "third_party/move/move-command-line-common",
     "third_party/move/move-compiler",
+    "third_party/move/move-compiler-v2",
     "third_party/move/move-compiler/transactional-tests",
     "third_party/move/move-core/types",
     "third_party/move/move-ir-compiler",
@@ -615,6 +616,7 @@ move-cli = { path = "third_party/move/tools/move-cli" }
 move-command-line-common = { path = "third_party/move/move-command-line-common" }
 move-coverage = { path = "third_party/move/tools/move-coverage" }
 move-compiler = { path = "third_party/move/move-compiler" }
+move-compiler-v2 = { path = "third_party/move/move-compiler-v2" }
 move-core-types = { path = "third_party/move/move-core/types" }
 move-docgen = { path = "third_party/move/move-prover/move-docgen" }
 move-disassembler = { path = "third_party/move/tools/move-disassembler" }

third_party/move/move-compiler-v2/Cargo.toml

@@ -0,0 +1,44 @@
+[package]
+name = "move-compiler-v2"
+version = "0.1.0"
+authors = ["Aptos Labs"]
+description = "Move compiler based on stackless bytecode"
+repository = "https://github.com/aptos-labs/aptos-core"
+homepage = "https://aptosfoundation.org/"
+license = "Apache-2.0"
+publish = false
+edition = "2021"
+
+[dependencies]
+move-binary-format = { path = "../move-binary-format" }
+#move-borrow-graph = { path = "../../move-borrow-graph" }
+#move-bytecode-verifier = { path = "../../move-bytecode-verifier" }
+#move-command-line-common = { path = "../../move-command-line-common" }
+#move-compiler = { path = "../../move-compiler" }
+move-core-types = { path = "../move-core/types" }
+move-ir-to-bytecode = { path = "../move-ir-compiler/move-ir-to-bytecode" }
+move-model = { path = "../move-model" }
+
+anyhow = "1.0.62"
+clap = { version = "3.2.23", features = ["derive", "env"] }
+codespan = "0.11.1"
+codespan-reporting = { version = "0.11.1", features = ["serde", "serialization"] }
+#ethnum = "1.0.4"
+#im = "15.0.0"
+itertools = "0.10.0"
+#log = "0.4.14"
+num = "0.4.0"
+once_cell = "1.7.2"
+#paste = "1.0.5"
+#petgraph = "0.5.1"
+serde = { version = "1.0.124", features = ["derive"] }
+
+[dev-dependencies]
+anyhow = "1.0.52"
+datatest-stable = "0.1.1"
+move-prover-test-utils = { path = "../move-prover/test-utils" }
+move-stdlib = { path = "../move-stdlib" }
+
+[[test]]
+name = "testsuite"
+harness = false

third_party/move/move-compiler-v2/src/experiments.rs

@@ -0,0 +1,27 @@
+// Copyright © Aptos Foundation
+// Parts of the project are originally copyright © Meta Platforms, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+/// Container for experiments in the compiler. Those can be activated
+/// via the `--experiment=<name>` flag.
+///
+/// One can activate an experiment in a test source by using a comment as such:
+/// ```
+///   // experiment: <name>
+/// ```
+/// This can be repeated an arbitrary number of times. The test will then be run for
+/// the default configuration and for each of the named experiments separately (if it is a
+/// baseline test, a different baseline file will be generated each time).
+///
+/// Each new experiment should have a description and explicit note about its retention.
+///
+/// - Permanent: the experiment is available indefinitely
+/// - Temporary: the experiment is intended to be removed after some time. Please document
+///   the condition under which it can be removed.
+pub struct Experiment();
+
+impl Experiment {
+    /// Whether to exit after type checking.
+    /// Retention: permanent
+    pub const CHECK_ONLY: &'static str = "check-only";
+}

third_party/move/move-compiler-v2/src/lib.rs

@@ -0,0 +1,63 @@
+// Copyright © Aptos Foundation
+// Parts of the project are originally copyright © Meta Platforms, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+mod experiments;
+mod options;
+
+use anyhow::anyhow;
+use codespan_reporting::term::termcolor::{ColorChoice, StandardStream, WriteColor};
+pub use experiments::*;
+use move_model::{model::GlobalEnv, PackageInfo};
+pub use options::*;
+
+/// Run Move compiler and print errors to stderr.
+pub fn run_move_compiler_to_stderr(options: Options) -> anyhow::Result<()> {
+    let mut error_writer = StandardStream::stderr(ColorChoice::Auto);
+    run_move_compiler(&mut error_writer, options)
+}
+
+/// Run move compiler and print errors to given writer.
+pub fn run_move_compiler<W: WriteColor>(
+    error_writer: &mut W,
+    options: Options,
+) -> anyhow::Result<()> {
+    // Run the model builder, which performs context checking.
+    let addrs = move_model::parse_addresses_from_options(options.named_address_mapping.clone())?;
+    let env = move_model::run_model_builder_in_compiler_mode(
+        PackageInfo {
+            sources: options.sources.clone(),
+            address_map: addrs.clone(),
+        },
+        vec![PackageInfo {
+            sources: options.dependencies.clone(),
+            address_map: addrs,
+        }],
+    )?;
+    // If the model contains any errors, report them now and exit.
+    check_errors(
+        &env,
+        &options,
+        error_writer,
+        "exiting with Move build errors",
+    )?;
+    if options.experiment_on(Experiment::CHECK_ONLY) {
+        // Stop here
+        return Ok(());
+    }
+    panic!("code generation NYI")
+}
+
+pub fn check_errors<W: WriteColor>(
+    env: &GlobalEnv,
+    options: &Options,
+    error_writer: &mut W,
+    msg: &'static str,
+) -> anyhow::Result<()> {
+    env.report_diag(error_writer, options.report_severity());
+    if env.has_errors() {
+        Err(anyhow!(msg))
+    } else {
+        Ok(())
+    }
+}

third_party/move/move-compiler-v2/src/options.rs

@@ -0,0 +1,70 @@
+// Copyright © Aptos Foundation
+// Parts of the project are originally copyright © Meta Platforms, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+use clap::Parser;
+use codespan_reporting::diagnostic::Severity;
+
+/// Defines options for a run of the compiler.
+#[derive(Parser, Debug)]
+#[clap(author, version, about)]
+pub struct Options {
+    /// Directories where to lookup dependencies.
+    #[clap(
+        short,
+        takes_value(true),
+        multiple_values(true),
+        multiple_occurrences(true)
+    )]
+    pub dependencies: Vec<String>,
+    /// Named address mapping.
+    #[clap(
+        short,
+        takes_value(true),
+        multiple_values(true),
+        multiple_occurrences(true)
+    )]
+    pub named_address_mapping: Vec<String>,
+    /// Output directory.
+    #[clap(short)]
+    #[clap(long, default_value = "")]
+    pub output_dir: String,
+    /// Whether to dump intermediate bytecode for debugging.
+    #[clap(long = "dump-bytecode")]
+    pub dump_bytecode: bool,
+    /// Whether we generate code for tests. This specifically guarantees stable output
+    /// for baseline testing.
+    #[clap(long)]
+    pub testing: bool,
+    /// Active experiments. Experiments alter default behavior of the compiler.
+    /// See `Experiment` struct.
+    #[clap(short)]
+    #[clap(
+        long = "experiment",
+        takes_value(true),
+        multiple_values(true),
+        multiple_occurrences(true)
+    )]
+    pub experiments: Vec<String>,
+    /// Sources to compile (positional arg, therefore last)
+    pub sources: Vec<String>,
+}
+
+impl Default for Options {
+    fn default() -> Self {
+        Parser::parse_from(std::iter::empty::<String>())
+    }
+}
+
+impl Options {
+    /// Returns the least severity of diagnosis which shall be reported.
+    /// This is currently hardwired.
+    pub fn report_severity(&self) -> Severity {
+        Severity::Warning
+    }
+
+    /// Returns true if an experiment is on.
+    pub fn experiment_on(&self, name: &str) -> bool {
+        self.experiments.iter().any(|s| s == name)
+    }
+}

third_party/move/move-compiler-v2/tests/checking/basic.exp.check-only

@@ -0,0 +1,7 @@
+failed: exiting with Move build errors
+Diagnostics:
+error: expected `u64` but found `address` in expression
+  ┌─ tests/checking/basic.move:3:22
+  │
+3 │     fun foo(): u64 { @22 }
+  │                      ^^^

third_party/move/move-compiler-v2/tests/checking/basic.move

@@ -0,0 +1,4 @@
+// experiment: check-only
+module 0x1::basic {
+    fun foo(): u64 { @22 }
+}

third_party/move/move-compiler-v2/tests/testsuite.rs

@@ -0,0 +1,63 @@
+// Copyright © Aptos Foundation
+// Parts of the project are originally copyright © Meta Platforms, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+use codespan_reporting::term::termcolor::Buffer;
+use move_compiler_v2::Options;
+use move_prover_test_utils::{baseline_test, extract_test_directives};
+use std::path::{Path, PathBuf};
+
+/// Extension for expected output files
+pub const EXP_EXT: &str = "exp";
+
+fn test_runner(path: &Path) -> datatest_stable::Result<()> {
+    let mut experiments = extract_test_directives(path, "// experiment:")?;
+    if experiments.is_empty() {
+        // If there is no experiment, use "" as the 'default' experiment.
+        experiments.push("".to_string()) // default experiment
+    }
+    let mut sources = extract_test_directives(path, "// dep:")?;
+    sources.push(path.to_string_lossy().to_string());
+    let deps = vec![path_from_crate_root("../move-stdlib/sources")];
+
+    // For each experiment, run the compiler.
+    for exp in experiments {
+        // Construct options, compiler and collect output.
+        let mut options = Options {
+            testing: true,
+            sources: sources.clone(),
+            dependencies: deps.clone(),
+            named_address_mapping: vec!["std=0x1".to_string()],
+            ..Options::default()
+        };
+        let file_ext = if exp.is_empty() {
+            EXP_EXT.to_string()
+        } else {
+            options.experiments.push(exp.clone());
+            format!("{}.{}", EXP_EXT, exp)
+        };
+
+        let mut error_writer = Buffer::no_color();
+        let mut out = match move_compiler_v2::run_move_compiler(&mut error_writer, options) {
+            Ok(_) => "succeeded".to_owned(),
+            Err(e) => format!("failed: {}", e),
+        };
+        let diag = String::from_utf8_lossy(&error_writer.into_inner()).to_string();
+        if !diag.is_empty() {
+            out = format!("{}\nDiagnostics:\n{}", out, diag)
+        }
+
+        // Generate/check baseline.
+        let baseline_path = path.with_extension(file_ext);
+        baseline_test::verify_or_update_baseline(baseline_path.as_path(), &out)?;
+    }
+    Ok(())
+}
+
+fn path_from_crate_root(path: &str) -> String {
+    let mut buf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    buf.push(path);
+    buf.to_string_lossy().to_string()
+}
+
+datatest_stable::harness!(test_runner, "tests", r".*\.move$");

third_party/move/move-compiler/src/shared/mod.rs

@@ -353,6 +353,17 @@ impl Flags {
         }
     }
 
+    pub fn model_compilation() -> Self {
+        Self {
+            test: false,
+            verify: true,
+            shadow: true, // allows overlapping between sources and deps
+            flavor: "".to_string(),
+            bytecode_version: None,
+            keep_testing_functions: true,
+        }
+    }
+
     pub fn set_flavor(self, flavor: impl ToString) -> Self {
         Self {
             flavor: flavor.to_string(),