[programmability] basic adapter + CLI

Prototype demonstrating how to hook the proposed programmability model up to the Move VM and the `OnDiskStateView` persistent storage used by the Move CLI.

- Refactor `ID` and `Authenticator` to use the Move `address` type under the hood
- Modify `Transfer::transfer<T>(address, T)` to emit a distinguished event containing the recipient address
- Implement an adapter that processes these events to store the `T` on disk under its ID
- In addition, the adapter wraps the Move CLI to enable persistent state, publishing modules, calling functions of published modules, etc.

This is limited/unsatisfactory in many ways (see the various TODO's), but it works well enough to enable experimenting with programmability and persistence via a basic CLI.

Cargo.toml

@@ -2,6 +2,8 @@
 members = [
         "fastpay_core",
         "fastpay",
+        "fastx_programmability/adapter",
+        "fastx_programmability/framework"
 ]
 
 [profile.release]

fastx_programmability/adapter/Cargo.toml

@@ -0,0 +1,40 @@
+[package]
+name = "fastx-adapter"
+version = "0.1.0"
+authors = ["Mysten Labs <[email protected]>"]
+description = "Adapter and accompanying CLI for local fastX development"
+license = "Apache-2.0"
+publish = false
+edition = "2018"
+
+[dependencies]
+anyhow = "1.0.38"
+bcs = "0.1.2"
+structopt = "0.3.21"
+sha3 = "0.9.1"
+
+## uncomment for debugging with local copy of diem repo
+# move-binary-format = { path = "../../../diem/language/move-binary-format" }
+# move-bytecode-utils = { path = "../../../diem/language/tools/move-bytecode-utils" }
+# move-core-types = { path = "../../../diem/language/move-core/types" }
+# move-cli = { path = "../../../diem/language/tools/move-cli" }
+# move-vm-runtime = { path = "../../../diem/language/move-vm/runtime" }
+
+move-binary-format = { git = "https://github.com/diem/diem", rev="661a2d1367a64a02027e4ed8f4b18f0a37cfaa17" }
+move-bytecode-utils = { git = "https://github.com/diem/diem", rev="661a2d1367a64a02027e4ed8f4b18f0a37cfaa17" }
+move-core-types = { git = "https://github.com/diem/diem", rev="661a2d1367a64a02027e4ed8f4b18f0a37cfaa17" }
+move-cli = { git = "https://github.com/diem/diem", rev="661a2d1367a64a02027e4ed8f4b18f0a37cfaa17" }
+move-vm-runtime = { git = "https://github.com/diem/diem", rev="661a2d1367a64a02027e4ed8f4b18f0a37cfaa17" }
+
+fastx-framework = { path = "../framework" }
+
+[dev-dependencies]
+datatest-stable = "0.1.1"
+
+[[bin]]
+name = "fastx"
+path = "src/main.rs"
+
+[[test]]
+name = "cli_testsuite"
+harness = false

fastx_programmability/adapter/src/adapter.rs

@@ -0,0 +1,204 @@
+// Copyright (c) Mysten Labs
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::{
+    state_view::FastXStateView, swap_authenticator_and_id, FASTX_FRAMEWORK_ADDRESS,
+    MOVE_STDLIB_ADDRESS,
+};
+use anyhow::Result;
+use fastx_framework::natives;
+use move_binary_format::errors::VMError;
+
+use move_cli::sandbox::utils::get_gas_status;
+use move_core_types::{
+    account_address::AccountAddress,
+    effects::ChangeSet,
+    gas_schedule::GasAlgebra,
+    identifier::{IdentStr, Identifier},
+    language_storage::{ModuleId, TypeTag},
+    resolver::MoveResolver,
+    transaction_argument::{convert_txn_args, TransactionArgument},
+};
+use move_vm_runtime::{move_vm::MoveVM, native_functions::NativeFunction};
+use sha3::{Digest, Sha3_256};
+
+pub struct FastXAdapter {
+    state_view: FastXStateView,
+}
+
+impl FastXAdapter {
+    pub fn create(build_dir: &str, storage_dir: &str) -> Result<Self> {
+        let state_view = FastXStateView::create(build_dir, storage_dir)?;
+        Ok(FastXAdapter { state_view })
+    }
+
+    /// Endpoint for local execution--no signature checking etc. is performed, and the result is saved on disk
+    // TODO: implement a wrapper of this with tx prologue + epilogue, bytecode verifier passes, etc.
+    pub fn execute_local(
+        &mut self,
+        module: Identifier,
+        function: Identifier,
+        sender: AccountAddress,
+        mut args: Vec<TransactionArgument>,
+        type_args: Vec<TypeTag>,
+        gas_budget: Option<u64>,
+    ) -> Result<()> {
+        // calculate `inputs_hash` based on address arguments. each address is the identifier of an object accessed by `function`
+        let mut hash_arg = Vec::new();
+        for arg in &args {
+            if let TransactionArgument::Address(a) = arg {
+                hash_arg.append(&mut a.to_vec())
+            }
+        }
+        // TODO: we should assert this eventually. but it makes testing difficult
+        // because of bootstrapping--the initial state contains no objects :)
+        //assert!(!hash_arg.is_empty(), "Need at least one object ID as input");
+        let inputs_hash = Sha3_256::digest(&hash_arg);
+        // assume that by convention, `inputs_hash` is the last argument
+        args.push(TransactionArgument::U8Vector(inputs_hash.to_vec()));
+        let script_args = convert_txn_args(&args);
+        let module_id = ModuleId::new(FASTX_FRAMEWORK_ADDRESS, module);
+        let natives = natives::all_natives(MOVE_STDLIB_ADDRESS, FASTX_FRAMEWORK_ADDRESS);
+        match execute_function(
+            &module_id,
+            &function,
+            type_args,
+            vec![sender],
+            script_args,
+            gas_budget,
+            &self.state_view,
+            natives,
+        )? {
+            ExecutionResult::Success {
+                change_set,
+                events,
+                gas_used: _,
+            } => {
+                // process change set. important to do this before processing events because it's where deletions happen
+                for (addr, addr_changes) in change_set.into_inner() {
+                    for (struct_tag, bytes_opt) in addr_changes.into_resources() {
+                        match bytes_opt {
+                            Some(bytes) => self
+                                .state_view
+                                .inner
+                                .save_resource(addr, struct_tag, &bytes)?,
+                            None => self.state_view.inner.delete_resource(addr, struct_tag)?,
+                        }
+                    }
+                }
+                // TODO: use CLI's explain_change_set here?
+                // process events
+                for e in events {
+                    if Self::is_transfer_event(&e) {
+                        let (guid, _seq_num, type_, event_bytes) = e;
+                        match type_ {
+                            TypeTag::Struct(s_type) => {
+                                // special transfer event. process by saving object under given authenticator
+                                let mut transferred_obj = event_bytes;
+                                let recipient = AccountAddress::from_bytes(guid)?;
+                                // hack: extract the ID from the object and use it as the address the object is saved under
+                                // replace the id with the object's new owner `recipient`
+                                let id = swap_authenticator_and_id(recipient, &mut transferred_obj);
+                                self.state_view
+                                    .inner
+                                    .save_resource(id, s_type, &transferred_obj)?
+                            }
+                            _ => unreachable!("Only structs can be transferred"),
+                        }
+                    } else {
+                        // the fastX framework doesn't support user-generated events yet, so shouldn't hit this
+                        unimplemented!("Processing user events")
+                    }
+                }
+            }
+            ExecutionResult::Fail { error, gas_used: _ } => {
+                // TODO: use CLI's error explanation features here
+                println!("Fail: {}", error)
+            }
+        }
+        Ok(())
+    }
+
+    /// Check if this is a special event type emitted when there is a transfer between fastX addresses
+    pub fn is_transfer_event(e: &Event) -> bool {
+        // TODO: hack that leverages implementation of Transfer::transfer_internal native function
+        !e.0.is_empty()
+    }
+}
+
+// TODO: Code below here probably wants to move into the VM or elsewhere in
+// the Diem codebase--seems generically useful + nothing similar exists
+
+type Event = (Vec<u8>, u64, TypeTag, Vec<u8>);
+
+/// Result of executing a script or script function in the VM
+pub enum ExecutionResult {
+    /// Execution completed successfully. Changes to global state are
+    /// captured in `change_set`, and `events` are recorded in the order
+    /// they were emitted. `gas_used` records the amount of gas expended
+    /// by execution. Note that this will be 0 in unmetered mode.
+    Success {
+        change_set: ChangeSet,
+        events: Vec<Event>,
+        gas_used: u64,
+    },
+    /// Execution failed for the reason described in `error`.
+    /// `gas_used` records the amount of gas expended by execution. Note
+    /// that this will be 0 in unmetered mode.
+    Fail { error: VMError, gas_used: u64 },
+}
+
+/// Execute the function named `script_function` in `module` with the given
+/// `type_args`, `signer_addresses`, and `args` as input.
+/// Execute the function according to the given `gas_budget`. If this budget
+/// is `Some(t)`, use `t` use `t`; if None, run the VM in unmetered mode
+/// Read published modules and global state from `resolver` and native functions
+/// from `natives`.
+#[allow(clippy::too_many_arguments)]
+pub fn execute_function<Resolver: MoveResolver>(
+    module: &ModuleId,
+    script_function: &IdentStr,
+    type_args: Vec<TypeTag>,
+    signer_addresses: Vec<AccountAddress>,
+    mut args: Vec<Vec<u8>>,
+    gas_budget: Option<u64>,
+    resolver: &Resolver,
+    natives: impl IntoIterator<Item = (AccountAddress, Identifier, Identifier, NativeFunction)>,
+) -> Result<ExecutionResult> {
+    let vm = MoveVM::new(natives).unwrap();
+    let mut gas_status = get_gas_status(gas_budget)?;
+    let mut session = vm.new_session(resolver);
+    // prepend signers to args
+    let mut signer_args: Vec<Vec<u8>> = signer_addresses
+        .iter()
+        .map(|s| bcs::to_bytes(s).unwrap())
+        .collect();
+    signer_args.append(&mut args);
+
+    let res = {
+        session
+            .execute_function(
+                module,
+                script_function,
+                type_args,
+                signer_args,
+                &mut gas_status,
+            )
+            .map(|_| ())
+    };
+    let gas_used = match gas_budget {
+        Some(budget) => budget - gas_status.remaining_gas().get(),
+        None => 0,
+    };
+    if let Err(error) = res {
+        println!("Failure: {}", error);
+        Ok(ExecutionResult::Fail { error, gas_used })
+    } else {
+        let (change_set, events) = session.finish().map_err(|e| e.into_vm_status())?;
+        Ok(ExecutionResult::Success {
+            change_set,
+            events,
+            gas_used,
+        })
+    }
+}

fastx_programmability/adapter/src/lib.rs

@@ -0,0 +1,36 @@
+// Copyright (c) Mysten Labs
+// SPDX-License-Identifier: Apache-2.0
+
+pub mod adapter;
+mod state_view;
+
+use move_core_types::account_address::AccountAddress;
+
+/// 0x1-- account address where Move stdlib modules are stored
+pub const MOVE_STDLIB_ADDRESS: AccountAddress = AccountAddress::new([
+    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
+]);
+
+/// 0x2-- account address where fastX framework modules are stored
+pub const FASTX_FRAMEWORK_ADDRESS: AccountAddress = AccountAddress::new([
+    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8,
+]);
+
+/// Extract + return an address from the first `authenticator.length()` bytes of `object`.
+/// Replace theses bytes with `authenticator`.
+/// copy the first authenticator.length() bytes out of `object`, turn them into
+/// an address. and return them. then, replace the first authenicator.length()
+/// bytes of `object` with `authenticator`
+pub(crate) fn swap_authenticator_and_id(
+    authenticator: AccountAddress,
+    object: &mut Vec<u8>,
+) -> AccountAddress {
+    assert!(object.len() > authenticator.len());
+
+    let authenticator_bytes = authenticator.into_bytes();
+    let mut id_bytes = [0u8; AccountAddress::LENGTH];
+
+    id_bytes[..authenticator.len()].clone_from_slice(&object[..authenticator.len()]);
+    object[..authenticator.len()].clone_from_slice(&authenticator_bytes[..authenticator.len()]);
+    AccountAddress::new(id_bytes)
+}

fastx_programmability/adapter/src/main.rs

@@ -0,0 +1,82 @@
+// Copyright (c) Mysten Labs
+// SPDX-License-Identifier: Apache-2.0
+
+use anyhow::Result;
+use fastx_adapter::{adapter::FastXAdapter, FASTX_FRAMEWORK_ADDRESS, MOVE_STDLIB_ADDRESS};
+use fastx_framework::natives;
+
+use move_cli::{Command, Move};
+use move_core_types::{
+    account_address::AccountAddress, errmap::ErrorMapping, identifier::Identifier,
+    language_storage::TypeTag, parser, transaction_argument::TransactionArgument,
+};
+
+use structopt::StructOpt;
+
+#[derive(StructOpt)]
+pub struct FastXCli {
+    #[structopt(flatten)]
+    move_args: Move,
+
+    #[structopt(subcommand)]
+    cmd: FastXCommand,
+}
+
+#[derive(StructOpt)]
+pub enum FastXCommand {
+    /// Command that delegates to the Move CLI
+    #[structopt(flatten)]
+    MoveCommand(Command),
+
+    // ... extra commands available only in fastX added below
+    #[structopt(name = "run")]
+    Run {
+        /// Path to module bytecode stored on disk
+        // TODO: We hardcode the module address to the fastX stdlib address for now, but will fix this
+        #[structopt(name = "module")]
+        module: Identifier,
+        /// Name of function in that module to call
+        #[structopt(name = "name", parse(try_from_str = Identifier::new))]
+        function: Identifier,
+        /// Sender of the transaction
+        #[structopt(name = "sender", parse(try_from_str = AccountAddress::from_hex_literal))]
+        sender: AccountAddress,
+        /// Arguments to the transaction
+        #[structopt(long = "args", parse(try_from_str = parser::parse_transaction_argument))]
+        args: Vec<TransactionArgument>,
+        /// Type arguments to the transaction
+        #[structopt(long = "type-args", parse(try_from_str = parser::parse_type_tag))]
+        type_args: Vec<TypeTag>,
+        /// Maximum number of gas units to be consumed by execution.
+        /// When the budget is exhaused, execution will abort.
+        /// By default, no `gas-budget` is specified and gas metering is disabled.
+        #[structopt(long = "gas-budget", short = "g")]
+        gas_budget: Option<u64>,
+    },
+}
+
+fn main() -> Result<()> {
+    // TODO: read this from the build artifacts so we can give better error messages
+    let error_descriptions: ErrorMapping = ErrorMapping::default();
+    // TODO: less hacky way of doing this?
+    let natives = natives::all_natives(MOVE_STDLIB_ADDRESS, FASTX_FRAMEWORK_ADDRESS);
+
+    let args = FastXCli::from_args();
+    use FastXCommand::*;
+    match args.cmd {
+        MoveCommand(cmd) => move_cli::run_cli(natives, &error_descriptions, &args.move_args, &cmd),
+        Run {
+            module,
+            function,
+            sender,
+            args,
+            type_args,
+            gas_budget,
+        } => {
+            // TODO: take build_dir and storage_dir as CLI inputs
+            let mut adapter = FastXAdapter::create("build", "storage")?;
+            adapter.execute_local(module, function, sender, args, type_args, gas_budget)?;
+            Ok(())
+        }
+    }
+}