[genesis] add logic for extracting genesis modules for FastX framework

- Add code in `framework/lib.rs` to build and verify the FastX framework Move modules + their dependencies in the Move stdlib and hand back a list of `CompiledModule`s. Add test to ensure that this code always works on the current set of framework modules.
- Add genesis.rs, which calls this code and wraps each module in an `Object` with a freshly generated ID
- Add an authority test that creates a genesis state with a module in it, then successfully processes a new `publish` order containing a module that depends on genesis one.
- Fix bugs in `publish` flow revealed by attempting to write this test.

A couple of issues came up during this that will need to be fixed later. One is #69; it basically stops us from calling the Move linker.

fastpay_core/Cargo.toml

@@ -14,6 +14,7 @@ serde = { version = "1.0.115", features = ["derive"] }
 tokio = { version = "0.2.22", features = ["full"] }
 
 fastx-adapter = { path = "../fastx_programmability/adapter" }
+fastx-framework = { path = "../fastx_programmability/framework" }
 fastx-types = { path = "../fastx_types" }
 
 move-binary-format = { git = "https://github.com/diem/diem", rev="5572eae5e35407f10316b473f95cb155a5e8d40b" }

fastpay_core/src/authority.rs

@@ -249,24 +249,24 @@ impl Authority for AuthorityState {
                 .map_err(|_| FastPayError::MoveExecutionFailure)?;
             }
             OrderKind::Publish(m) => {
+                // Fake the gas payment
+                let mut gas_object = temporary_store
+                    .read_object(&object_id)
+                    .expect("Checked existence at start of function.");
+                gas_object.next_sequence_number = gas_object.next_sequence_number.increment()?;
+                temporary_store.write_object(gas_object);
                 // TODO(https://github.com/MystenLabs/fastnft/issues/45): charge for gas
                 let sender = m.sender.to_address_hack();
                 match adapter::publish(&mut temporary_store, m.modules, &sender, &mut tx_ctx) {
                     Ok(outputs) => {
-                        // Fake the gas payment
-                        let mut gas_object = temporary_store
-                            .read_object(&object_id)
-                            .expect("Checked existance at start of function.");
-                        gas_object.next_sequence_number =
-                            gas_object.next_sequence_number.increment()?;
-                        temporary_store.write_object(gas_object);
-
                         // TODO: AccountInfoResponse should return all object ID outputs.
                         // but for now it only returns one, so use this hack
                         object_id = outputs[0].0;
                     }
                     Err(_e) => {
+                        println!("failure during publishing: {:?}", _e);
                         // TODO: return this error to the client
+                        object_id = m.gas_payment.0;
                     }
                 }
             }

fastpay_core/src/genesis.rs

@@ -0,0 +1,29 @@
+// Copyright (c) Mysten Labs
+// SPDX-License-Identifier: Apache-2.0
+
+use anyhow::Result;
+use fastx_adapter::adapter;
+use fastx_framework::{self};
+use fastx_types::{
+    base_types::{PublicKeyBytes, SequenceNumber, TransactionDigest, TxContext},
+    object::Object,
+    FASTX_FRAMEWORK_ADDRESS,
+};
+
+/// Create and return objects wrapping the genesis modules for fastX
+pub fn create_genesis_module_objects() -> Result<Vec<Object>> {
+    let mut tx_context = TxContext::new(TransactionDigest::genesis());
+    let mut modules = fastx_framework::get_framework_modules()?;
+    adapter::generate_module_ids(&mut modules, &mut tx_context)?;
+    let module_objects = modules
+        .into_iter()
+        .map(|m| {
+            Object::new_module(
+                m,
+                PublicKeyBytes::from_move_address_hack(&FASTX_FRAMEWORK_ADDRESS),
+                SequenceNumber::new(),
+            )
+        })
+        .collect();
+    Ok(module_objects)
+}

fastpay_core/src/lib.rs

@@ -12,3 +12,4 @@ pub mod authority;
 pub mod client;
 pub mod downloader;
 pub mod fastpay_smart_contract;
+pub mod genesis;

fastpay_core/src/unit_tests/authority_tests.rs

@@ -2,9 +2,13 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use super::*;
+use crate::genesis;
 #[cfg(test)]
 use fastx_types::base_types::dbg_addr;
-use move_binary_format::file_format;
+use move_binary_format::{
+    file_format::{self, AddressIdentifierIndex, IdentifierIndex, ModuleHandle},
+    CompiledModule,
+};
 use move_core_types::ident_str;
 #[cfg(test)]
 use move_core_types::language_storage::ModuleId;
@@ -159,17 +163,69 @@ fn send_and_confirm_order(
     authority.handle_confirmation_order(ConfirmationOrder::new(certificate))
 }
 
+/// Create a `CompiledModule` that depends on `m`
+fn make_dependent_module(m: &CompiledModule) -> CompiledModule {
+    let mut dependent_module = file_format::empty_module();
+    dependent_module
+        .identifiers
+        .push(m.self_id().name().to_owned());
+    dependent_module
+        .address_identifiers
+        .push(*m.self_id().address());
+    dependent_module.module_handles.push(ModuleHandle {
+        address: AddressIdentifierIndex((dependent_module.address_identifiers.len() - 1) as u16),
+        name: IdentifierIndex((dependent_module.identifiers.len() - 1) as u16),
+    });
+    dependent_module
+}
+
+// Test that publishing a module that depends on an existing one works
+#[test]
+fn test_publish_dependent_module_ok() {
+    let (sender, sender_key) = get_key_pair();
+    // create a dummy gas payment object. ok for now because we don't check gas
+    let gas_payment_object_id = ObjectID::random();
+    let gas_payment_object = Object::with_id_owner_for_testing(gas_payment_object_id, sender);
+    let gas_payment_object_ref = gas_payment_object.to_object_reference();
+    // create a genesis state that contains the gas object and genesis modules
+    let mut genesis_module_objects = genesis::create_genesis_module_objects().unwrap();
+    let genesis_module = match &genesis_module_objects[0].data {
+        Data::Module(m) => m,
+        _ => unreachable!(),
+    };
+    // create a module that depends on a genesis module
+    let dependent_module = make_dependent_module(genesis_module);
+    let dependent_module_bytes = {
+        let mut bytes = Vec::new();
+        dependent_module.serialize(&mut bytes).unwrap();
+        bytes
+    };
+    genesis_module_objects.push(gas_payment_object);
+    let mut authority = init_state_with_objects(genesis_module_objects);
+
+    let order = Order::new_module(
+        sender,
+        gas_payment_object_ref,
+        vec![dependent_module_bytes],
+        &sender_key,
+    );
+    let dependent_module_id = TxContext::new(order.digest()).fresh_id();
+    let response = send_and_confirm_order(&mut authority, order).unwrap();
+    // check that the dependent module got published
+    assert!(response.object_id == dependent_module_id);
+}
+
+// Test that publishing a module with no dependencies works
 #[test]
-fn test_publish_module_ok() {
+fn test_publish_module_no_dependencies_ok() {
     let (sender, sender_key) = get_key_pair();
     // create a dummy gas payment object. ok for now because we don't check gas
     let gas_payment_object_id = ObjectID::random();
     let gas_payment_object = Object::with_id_owner_for_testing(gas_payment_object_id, sender);
     let gas_payment_object_ref = gas_payment_object.to_object_reference();
     let mut authority = init_state_with_objects(vec![gas_payment_object]);
 
-    let mut module = file_format::empty_module();
-    module.address_identifiers[0] = sender.to_address_hack();
+    let module = file_format::empty_module();
     let mut module_bytes = Vec::new();
     module.serialize(&mut module_bytes).unwrap();
     let order = Order::new_module(
@@ -178,11 +234,10 @@ fn test_publish_module_ok() {
         vec![module_bytes],
         &sender_key,
     );
-    let module_object_id = TxContext::new(order.digest()).fresh_id();
+    let module_id = TxContext::new(order.digest()).fresh_id();
     let response = send_and_confirm_order(&mut authority, order).unwrap();
     // check that the module actually got published
-    assert!(response.object_id == module_object_id);
-    // TODO: eventually, `response` should return a vector of ID's created + we should check non-emptiness of this vector
+    assert!(response.object_id == module_id);
 }
 
 #[test]

fastx_programmability/adapter/Cargo.toml

@@ -14,6 +14,7 @@ structopt = "0.3.21"
 
 move-binary-format = { git = "https://github.com/diem/diem", rev="5572eae5e35407f10316b473f95cb155a5e8d40b" }
 move-bytecode-utils = { git = "https://github.com/diem/diem", rev="5572eae5e35407f10316b473f95cb155a5e8d40b" }
+move-bytecode-verifier = { git = "https://github.com/diem/diem", rev="5572eae5e35407f10316b473f95cb155a5e8d40b" }
 move-core-types = { git = "https://github.com/diem/diem", rev="5572eae5e35407f10316b473f95cb155a5e8d40b" }
 move-cli = { git = "https://github.com/diem/diem", rev="5572eae5e35407f10316b473f95cb155a5e8d40b" }
 move-vm-runtime = { git = "https://github.com/diem/diem", rev="5572eae5e35407f10316b473f95cb155a5e8d40b" }

fastx_programmability/adapter/src/adapter.rs

@@ -2,12 +2,13 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use anyhow::{bail, Result};
-use fastx_framework::{natives, FASTX_FRAMEWORK_ADDRESS, MOVE_STDLIB_ADDRESS};
+use fastx_framework::natives;
 use fastx_types::{
     base_types::{FastPayAddress, ObjectRef, SequenceNumber, TxContext},
     error::{FastPayError, FastPayResult},
     object::Object,
     storage::Storage,
+    FASTX_FRAMEWORK_ADDRESS, MOVE_STDLIB_ADDRESS,
 };
 use fastx_verifier::verifier;
 use move_binary_format::{errors::VMError, file_format::CompiledModule};
@@ -23,7 +24,9 @@ use move_core_types::{
     transaction_argument::{convert_txn_args, TransactionArgument},
 };
 use move_vm_runtime::move_vm::MoveVM;
-use std::fmt::Debug;
+use std::{collections::BTreeMap, fmt::Debug};
+
+use crate::bytecode_rewriter::ModuleHandleRewriter;
 
 /// Execute `module::function<type_args>(object_args ++ pure_args)` as a call from `sender` with the given `gas_budget`.
 /// Execution will read from/write to the store in `state_view`.
@@ -144,55 +147,67 @@ pub fn publish<E: Debug, S: ResourceResolver<Error = E> + ModuleResolver<Error =
             })
         })
         .collect::<FastPayResult<Vec<CompiledModule>>>()?;
-
-    // Use the Move VM's publish API to run the Move bytecode verifier and linker.
-    // It is important to do this before running the FastX verifier, since the fastX
-    // verifier may assume well-formedness conditions enforced by the Move verifier hold
-    // TODO(https://github.com/MystenLabs/fastnft/issues/57):
-    // it would be more efficient to call the linker/verifier directly instead of
-    // creating a VM. It will also save us from serializing/deserializing the modules twice
-    let vm = create_vm();
-    let mut session = vm.new_session(state_view);
-    let mut gas_status = get_gas_status(None).expect("Cannot fail when called with None");
-    session
-        .publish_module_bundle(module_bytes, *sender, &mut gas_status)
-        .map_err(|e| FastPayError::ModulePublishFailure {
-            error: e.to_string(),
+    generate_module_ids(&mut modules, ctx)?;
+    // verify and link modules, wrap them in objects, write them to the store
+    let mut written_refs = Vec::with_capacity(modules.len());
+    for module in modules {
+        // It is important to do this before running the FastX verifier, since the fastX
+        // verifier may assume well-formedness conditions enforced by the Move verifier hold
+        move_bytecode_verifier::verify_module(&module).map_err(|e| {
+            FastPayError::ModuleVerificationFailure {
+                error: e.to_string(),
+            }
         })?;
+        // Run FastX bytecode verifier
+        verifier::verify_module(&module)?;
 
-    // Run FastX bytecode verifier
-    for module in &modules {
-        verifier::verify_module(module)?
-    }
-
-    // derive fresh ID's for each module and mutate its self address to the ID.
-    // this ensures that each module can be uniquely identified/retrieved by its self address
-    // TODO: do this *before* passing the modules to the verifier. Right now, we can't because
-    // `publish_module_bundle` insists that the tx sender is equal to the module's self_address()
-    for module in modules.iter_mut() {
-        let fresh_id = ctx.fresh_id();
-        // TODO(https://github.com/MystenLabs/fastnft/issues/56):
-        // add a FastX bytecode verifier pass to ensure that no bytecodes reference `module.address_identifiers[0]`
-        // otherwise, code like `if (x == old_self_address)` could sneakily change to `if (x == fresh_id)` after the mutation below
-        module.address_identifiers[0] = fresh_id;
-        assert!(module.self_id().address() == &fresh_id);
-    }
+        // TODO(https://github.com/MystenLabs/fastnft/issues/69):
+        // run Move linker using state_view. it currently can only be called through the VM's publish or publish_module_bundle API's,
+        // but we can't use those because they require module.self_address() == sender, which is not true for FastX modules
+        let _ = state_view;
 
-    // Create and return module objects
-    let mut written_refs = Vec::with_capacity(modules.len());
-    for m in modules {
-        let new_module = Object::new_module(
-            m,
+        // Create module objects and write them to the store
+        let module_object = Object::new_module(
+            module,
             FastPayAddress::from_move_address_hack(sender),
             SequenceNumber::new(),
         );
-        written_refs.push(new_module.to_object_reference());
-        state_view.write_object(new_module);
+        written_refs.push(module_object.to_object_reference());
+        state_view.write_object(module_object);
     }
 
     Ok(written_refs)
 }
 
+/// Use `ctx` to generate fresh ID's for each module in `modules`.
+/// Mutate each module's self ID to the appropriate fresh ID and update its module handle tables
+/// to reflect the new ID's of its dependencies
+pub fn generate_module_ids(
+    modules: &mut Vec<CompiledModule>,
+    ctx: &mut TxContext,
+) -> Result<(), FastPayError> {
+    let mut sub_map = BTreeMap::new();
+    for module in modules.iter() {
+        // derive a fresh ID's for each module and mutate its self address to the ID.
+        // this ensures that the  module can be uniquely identified/retrieved by its self address
+        let old_module_id = module.self_id();
+        let fresh_object_id = ctx.fresh_id();
+        let new_module_id = ModuleId::new(fresh_object_id, old_module_id.name().to_owned());
+        if sub_map.insert(old_module_id, new_module_id).is_some() {
+            return Err(FastPayError::ModulePublishFailure {
+                error: "Publishing two modules with the same ID".to_string(),
+            });
+        }
+    }
+    // Safe to unwrap because we checked for duplicate domain entries above, and range entries are fresh ID's
+    let rewriter = ModuleHandleRewriter::new(sub_map).unwrap();
+    for module in modules.iter_mut() {
+        // rewrite module handles to reflect freshly generated ID's
+        rewriter.sub_module_ids(module);
+    }
+    Ok(())
+}
+
 /// Check if this is a special event type emitted when there is a transfer between fastX addresses
 pub fn is_transfer_event(e: &Event) -> bool {
     // TODO: hack that leverages implementation of Transfer::transfer_internal native function