Merge pull request #185 from lxfind/one-package-a-time

Publish one package at a time

fastx_programmability/adapter/src/adapter.rs

@@ -15,7 +15,7 @@ use fastx_types::{
     },
     object::{Data, MoveObject, Object},
     storage::Storage,
-    FASTX_FRAMEWORK_OBJECT_ID,
+    FASTX_FRAMEWORK_ADDRESS,
 };
 use fastx_verifier::verifier;
 use move_binary_format::{
@@ -148,59 +148,58 @@ pub fn publish<E: Debug, S: ResourceResolver<Error = E> + ModuleResolver<Error =
     state_view.write_object(gas_object);
     // TODO: Keep track the gas deducted so that we could give them to participants.
 
-    let modules = module_bytes
+    let mut modules = module_bytes
         .iter()
         .map(|b| {
             CompiledModule::deserialize(b).map_err(|e| FastPayError::ModuleDeserializationFailure {
                 error: e.to_string(),
             })
         })
         .collect::<FastPayResult<Vec<CompiledModule>>>()?;
-    let packages = generate_package_info_map(modules, ctx)?;
+    generate_package_id(&mut modules, ctx)?;
     // verify and link modules, wrap them in objects, write them to the store
-    for (_, modules) in packages.into_values() {
-        for module in modules.iter() {
-            // It is important to do this before running the FastX verifier, since the fastX
-            // verifier may assume well-formedness conditions enforced by the Move verifier hold
-            move_bytecode_verifier::verify_module(module).map_err(|e| {
-                FastPayError::ModuleVerificationFailure {
-                    error: e.to_string(),
-                }
-            })?;
-            // Run FastX bytecode verifier
-            verifier::verify_module(module)?;
+    for module in modules.iter() {
+        // It is important to do this before running the FastX verifier, since the fastX
+        // verifier may assume well-formedness conditions enforced by the Move verifier hold
+        move_bytecode_verifier::verify_module(module).map_err(|e| {
+            FastPayError::ModuleVerificationFailure {
+                error: e.to_string(),
+            }
+        })?;
+        // Run FastX bytecode verifier
+        verifier::verify_module(module)?;
 
-            // TODO(https://github.com/MystenLabs/fastnft/issues/69):
-            // run Move linker using state_view. it currently can only be called through the VM's publish or publish_module_bundle API's,
-            // but we can't use those because they require module.self_address() == sender, which is not true for FastX modules
-        }
-        let package_object = Object::new_package(modules, sender, ctx.digest());
-        state_view.write_object(package_object);
+        // TODO(https://github.com/MystenLabs/fastnft/issues/69):
+        // run Move linker using state_view. it currently can only be called through the VM's publish or publish_module_bundle API's,
+        // but we can't use those because they require module.self_address() == sender, which is not true for FastX modules
     }
+    let package_object = Object::new_package(modules, sender, ctx.digest());
+    state_view.write_object(package_object);
 
     Ok(())
 }
 
-/// Given a list of `modules`, regonize the packages from them (i.e. modules with the same address),
-/// use `ctx` to generate a fresh ID for each of those packages.
+/// Given a list of `modules`, use `ctx` to generate a fresh ID for the new packages.
+/// If `is_framework` is true, then the modules can have arbitrary user-defined address,
+/// otherwise their addresses must be 0.
 /// Mutate each module's self ID to the appropriate fresh ID and update its module handle tables
 /// to reflect the new ID's of its dependencies.
-/// Returns the mapping from the old package addresses to the new addresses.
-/// Note: id and address means the same thing here.
-pub fn generate_package_info_map(
-    modules: Vec<CompiledModule>,
+/// Returns the newly created package ID.
+pub fn generate_package_id(
+    modules: &mut Vec<CompiledModule>,
     ctx: &mut TxContext,
-) -> Result<BTreeMap<AccountAddress, (AccountAddress, Vec<CompiledModule>)>, FastPayError> {
+) -> FastPayResult {
     let mut sub_map = BTreeMap::new();
-    let mut packages = BTreeMap::new();
-    for module in modules {
+    let package_id = ctx.fresh_id();
+    for module in modules.iter() {
         let old_module_id = module.self_id();
         let old_address = *old_module_id.address();
-        let package_info = packages
-            .entry(old_address)
-            .or_insert((ctx.fresh_id(), vec![]));
-        package_info.1.push(module);
-        let new_module_id = ModuleId::new(package_info.0, old_module_id.name().to_owned());
+        if old_address != AccountAddress::ZERO {
+            return Err(FastPayError::ModulePublishFailure {
+                error: "Publishing modules with non-zero address is not allowed".to_string(),
+            });
+        }
+        let new_module_id = ModuleId::new(package_id, old_module_id.name().to_owned());
         if sub_map.insert(old_module_id, new_module_id).is_some() {
             return Err(FastPayError::ModulePublishFailure {
                 error: "Publishing two modules with the same ID".to_string(),
@@ -210,13 +209,11 @@ pub fn generate_package_info_map(
 
     // Safe to unwrap because we checked for duplicate domain entries above, and range entries are fresh ID's
     let rewriter = ModuleHandleRewriter::new(sub_map).unwrap();
-    for (_, modules) in packages.values_mut() {
-        for module in modules.iter_mut() {
-            // rewrite module handles to reflect freshly generated ID's
-            rewriter.sub_module_ids(module);
-        }
+    for module in modules.iter_mut() {
+        // rewrite module handles to reflect freshly generated ID's
+        rewriter.sub_module_ids(module);
     }
-    Ok(packages)
+    Ok(())
 }
 
 type Event = (Vec<u8>, u64, TypeTag, Vec<u8>);
@@ -404,15 +401,15 @@ fn resolve_and_type_check(
             module,
             name,
             type_arguments,
-        } if address == &FASTX_FRAMEWORK_OBJECT_ID
+        } if address == &FASTX_FRAMEWORK_ADDRESS
             && module.as_ident_str() == TX_CONTEXT_MODULE_NAME
             && name.as_ident_str() == TX_CONTEXT_STRUCT_NAME
             && type_arguments.is_empty() => {}
         t => {
             return Err(FastPayError::InvalidFunctionSignature {
                 error: format!(
                     "Expected last parameter of function signature to be &mut {}::{}::{}, but found {}",
-                    FASTX_FRAMEWORK_OBJECT_ID, TX_CONTEXT_MODULE_NAME, TX_CONTEXT_STRUCT_NAME, t
+                    FASTX_FRAMEWORK_ADDRESS, TX_CONTEXT_MODULE_NAME, TX_CONTEXT_STRUCT_NAME, t
                 ),
             })
         }
@@ -421,7 +418,7 @@ fn resolve_and_type_check(
         return Err(FastPayError::InvalidFunctionSignature {
             error: format!(
                 "Expected last parameter of function signature to be &mut {}::{}::{}, but found non-reference_type",
-                FASTX_FRAMEWORK_OBJECT_ID, TX_CONTEXT_MODULE_NAME, TX_CONTEXT_STRUCT_NAME
+                FASTX_FRAMEWORK_ADDRESS, TX_CONTEXT_MODULE_NAME, TX_CONTEXT_STRUCT_NAME
             ),
         });
     }

fastx_programmability/adapter/src/genesis.rs

@@ -1,20 +1,17 @@
 // Copyright (c) Mysten Labs
 // SPDX-License-Identifier: Apache-2.0
 
-use crate::adapter;
-use anyhow::Result;
 use fastx_framework::{self};
 use fastx_types::{
-    base_types::{FastPayAddress, TransactionDigest, TxContext},
+    base_types::{FastPayAddress, TransactionDigest},
     object::Object,
-    FASTX_FRAMEWORK_ADDRESS, FASTX_FRAMEWORK_OBJECT_ID, MOVE_STDLIB_ADDRESS, MOVE_STDLIB_OBJECT_ID,
+    FASTX_FRAMEWORK_ADDRESS, MOVE_STDLIB_ADDRESS,
 };
 use move_vm_runtime::native_functions::NativeFunctionTable;
 use once_cell::sync::Lazy;
 use std::sync::Mutex;
 
-static GENESIS: Lazy<Mutex<Genesis>> =
-    Lazy::new(|| Mutex::new(create_genesis_module_objects().unwrap()));
+static GENESIS: Lazy<Mutex<Genesis>> = Lazy::new(|| Mutex::new(create_genesis_module_objects()));
 
 struct Genesis {
     pub objects: Vec<Object>,
@@ -27,33 +24,18 @@ pub fn clone_genesis_data() -> (Vec<Object>, NativeFunctionTable) {
 }
 
 /// Create and return objects wrapping the genesis modules for fastX
-fn create_genesis_module_objects() -> Result<Genesis> {
-    let mut tx_context = TxContext::new(TransactionDigest::genesis());
-    let modules = fastx_framework::get_framework_packages()?;
-    let packages = adapter::generate_package_info_map(modules, &mut tx_context)?;
-    let fastx_framework_addr = packages[&FASTX_FRAMEWORK_ADDRESS].0;
-    let move_stdlib_addr = packages[&MOVE_STDLIB_ADDRESS].0;
-    if fastx_framework_addr != FASTX_FRAMEWORK_OBJECT_ID {
-        panic!(
-            "FastX framework address doesn't match, expecting: {:#X?}",
-            fastx_framework_addr
-        );
-    }
-    if move_stdlib_addr != MOVE_STDLIB_OBJECT_ID {
-        panic!(
-            "Move stdlib address doesn't match, expecting: {:#X?}",
-            move_stdlib_addr
-        );
-    }
+fn create_genesis_module_objects() -> Genesis {
+    let fastx_modules = fastx_framework::get_fastx_framework_modules();
+    let std_modules = fastx_framework::get_move_stdlib_modules();
     let native_functions =
-        fastx_framework::natives::all_natives(move_stdlib_addr, fastx_framework_addr);
+        fastx_framework::natives::all_natives(MOVE_STDLIB_ADDRESS, FASTX_FRAMEWORK_ADDRESS);
     let owner = FastPayAddress::default();
-    let objects = packages
-        .into_values()
-        .map(|(_, modules)| Object::new_package(modules, owner, TransactionDigest::genesis()))
-        .collect();
-    Ok(Genesis {
+    let objects = vec![
+        Object::new_package(fastx_modules, owner, TransactionDigest::genesis()),
+        Object::new_package(std_modules, owner, TransactionDigest::genesis()),
+    ];
+    Genesis {
         objects,
         native_functions,
-    })
+    }
 }

fastx_programmability/adapter/src/unit_tests/adapter_tests.rs

@@ -556,7 +556,7 @@ fn test_transfer_and_freeze() {
         &mut storage,
         &native_functions,
         "set_value",
-        gas_object.clone(),
+        gas_object,
         MAX_GAS,
         vec![obj1],
         pure_args,

fastx_programmability/framework/Move.toml

@@ -9,11 +9,7 @@ MoveStdlib = { local = "deps/move-stdlib" }
 #MoveStdlib = { git = "https://github.com/diem/diem.git", subdir="language/move-stdlib", rev = "346301f33b3489bb4e486ae6c0aa5e030223b492" }
 
 [addresses]
-Std = "0x1"
 FastX = "0x2"
-Examples = "0x2"
 
 [dev-addresses]
-Std = "0x1"
 FastX = "0x2"
-Examples = "0x2"

fastx_programmability/framework/deps/move-stdlib/Move.toml

@@ -3,7 +3,7 @@ name = "MoveStdlib"
 version = "1.5.0"
 
 [addresses]
-Std = "_"
+Std = "0x1"
 
 [dev-addresses]
 Std = "0x1"

fastx_programmability/framework/src/lib.rs

@@ -1,12 +1,10 @@
 // Copyright (c) Mysten Labs
 // SPDX-License-Identifier: Apache-2.0
 
-use anyhow::Result;
-use fastx_types::MOVE_STDLIB_ADDRESS;
 use fastx_verifier::verifier as fastx_bytecode_verifier;
 use move_binary_format::CompiledModule;
-use move_core_types::{ident_str, language_storage::ModuleId};
-use move_package::{compilation::compiled_package::CompiledPackage, BuildConfig};
+use move_core_types::ident_str;
+use move_package::BuildConfig;
 use std::path::PathBuf;
 
 pub mod natives;
@@ -15,78 +13,75 @@ pub mod natives;
 #[cfg(test)]
 const MAX_UNIT_TEST_INSTRUCTIONS: u64 = 100_000;
 
-/// Return all the modules of the fastX framework and its dependencies in topologically
-/// sorted dependency order (leaves first). The packages are organized
-/// as a map from the address to all modules in that address.
-pub fn get_framework_packages() -> Result<Vec<CompiledModule>> {
-    let include_examples = false;
-    let verify = true;
-    get_framework_packages_(include_examples, verify)
+pub fn get_fastx_framework_modules() -> Vec<CompiledModule> {
+    let modules = build(".");
+    veirfy_modules(&modules);
+    modules
 }
 
-fn get_framework_packages_(include_examples: bool, verify: bool) -> Result<Vec<CompiledModule>> {
-    // TODO: prune unused deps from Move stdlib instead of using an explicit denylist.
-    // The manually curated list are modules that do not pass the FastX verifier
+pub fn get_move_stdlib_modules() -> Vec<CompiledModule> {
     let denylist = vec![
-        ModuleId::new(MOVE_STDLIB_ADDRESS, ident_str!("Capability").to_owned()),
-        ModuleId::new(MOVE_STDLIB_ADDRESS, ident_str!("Event").to_owned()),
-        ModuleId::new(MOVE_STDLIB_ADDRESS, ident_str!("GUID").to_owned()),
+        ident_str!("Capability").to_owned(),
+        ident_str!("Event").to_owned(),
+        ident_str!("GUID").to_owned(),
     ];
-    let package = build(include_examples)?;
-    let filtered_modules: Vec<CompiledModule> = package
-        .transitive_compiled_modules()
-        .iter_modules_owned()
+    let modules: Vec<CompiledModule> = build("deps/move-stdlib")
         .into_iter()
-        .filter(|m| !denylist.contains(&m.self_id()))
+        .filter(|m| !denylist.contains(&m.self_id().name().to_owned()))
         .collect();
-    if verify {
-        for m in &filtered_modules {
-            move_bytecode_verifier::verify_module(m).unwrap();
-            fastx_bytecode_verifier::verify_module(m).unwrap();
-            // TODO(https://github.com/MystenLabs/fastnft/issues/69): Run Move linker
-        }
+    veirfy_modules(&modules);
+    modules
+}
+
+fn veirfy_modules(modules: &[CompiledModule]) {
+    for m in modules {
+        move_bytecode_verifier::verify_module(m).unwrap();
+        fastx_bytecode_verifier::verify_module(m).unwrap();
+        // TODO(https://github.com/MystenLabs/fastnft/issues/69): Run Move linker
     }
-    Ok(filtered_modules)
 }
 
-/// Include the Move package's `example` modules if `include_examples` is true, omits them otherwise
-fn build(include_examples: bool) -> Result<CompiledPackage> {
-    let framework_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+fn build(sub_dir: &str) -> Vec<CompiledModule> {
+    let mut framework_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    framework_dir.push(sub_dir);
     let build_config = BuildConfig {
-        dev_mode: include_examples,
+        dev_mode: false,
         ..Default::default()
     };
-    build_config.compile_package(&framework_dir, &mut Vec::new())
+    build_config
+        .compile_package(&framework_dir, &mut Vec::new())
+        .unwrap()
+        .compiled_modules()
+        .iter_modules_owned()
 }
 
 #[test]
-fn check_that_move_code_can_be_built_verified_testsd() {
-    let include_examples = true;
-    let verify = true;
-    get_framework_packages_(include_examples, verify).unwrap();
+fn check_that_move_code_can_be_built_verified_tested() {
+    get_fastx_framework_modules();
     // ideally this would be a separate test, but doing so introduces
     // races because of https://github.com/diem/diem/issues/10102
     run_move_unit_tests();
 }
 
 #[cfg(test)]
 fn run_move_unit_tests() {
-    use fastx_types::FASTX_FRAMEWORK_ADDRESS;
-    use move_cli::package::cli;
+    use fastx_types::{FASTX_FRAMEWORK_ADDRESS, MOVE_STDLIB_ADDRESS};
+    use move_cli::package::cli::{self, UnitTestResult};
+
     use move_unit_test::UnitTestingConfig;
     use std::path::Path;
 
     let framework_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
-    let include_examples = true;
-    cli::run_move_unit_tests(
+    let result = cli::run_move_unit_tests(
         framework_dir,
         BuildConfig {
-            dev_mode: include_examples,
+            dev_mode: false,
             ..Default::default()
         },
         UnitTestingConfig::default_with_bound(Some(MAX_UNIT_TEST_INSTRUCTIONS)),
         natives::all_natives(MOVE_STDLIB_ADDRESS, FASTX_FRAMEWORK_ADDRESS),
         /* compute_coverage */ false,
     )
     .unwrap();
+    assert!(result == UnitTestResult::Success);
 }