diff --git a/Cargo.lock b/Cargo.lock index 0c3d4637d524a..0da9d978f29cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1078,13 +1078,17 @@ name = "aptos-release-builder" version = "0.1.0" dependencies = [ "anyhow", + "aptos-crypto", "aptos-gas", + "aptos-rest-client", "aptos-temppath", "aptos-types", "bcs 0.1.3 (git+https://github.com/aptos-labs/bcs?rev=2cde3e8446c460cb17b0c1d6bac7e27e964ac169)", "clap 3.2.17", "move-core-types", "move-model", + "serde 1.0.144", + "serde_yaml 0.8.26", "tempfile", ] @@ -3392,9 +3396,9 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.3" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" +checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" dependencies = [ "block-buffer 0.10.2", "crypto-common", @@ -4537,7 +4541,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest 0.10.3", + "digest 0.10.5", ] [[package]] @@ -8064,11 +8068,11 @@ dependencies = [ [[package]] name = "ripemd" -version = "0.1.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1facec54cb5e0dc08553501fa740091086d0259ad0067e0d4103448e4cb22ed3" +checksum = "bd124222d17ad93a644ed9d011a40f4fb64aa54275c08cc216524a9ea82fb09f" dependencies = [ - "digest 0.10.3", + "digest 0.10.5", ] [[package]] @@ -8594,7 +8598,7 @@ checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.3", + "digest 0.10.5", ] [[package]] @@ -8618,7 +8622,7 @@ checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.3", + "digest 0.10.5", ] [[package]] diff --git a/aptos-move/aptos-release-builder/Cargo.toml b/aptos-move/aptos-release-builder/Cargo.toml index d3a277a5c4829..01d287f9cdd58 100644 --- a/aptos-move/aptos-release-builder/Cargo.toml +++ b/aptos-move/aptos-release-builder/Cargo.toml @@ -19,10 +19,14 @@ bcs = { git = "https://github.com/aptos-labs/bcs", rev = "2cde3e8446c460cb17b0c1 clap = { version = "3.1.17", features = ["derive"] } move-core-types = { workspace = true } move-model = { workspace = true } +serde = { version = "1.0.137", features = ["derive"], default-features = false } +serde_yaml = "0.8.24" tempfile = "3.3.0" aptos-temppath = { path = "../../crates/aptos-temppath" } aptos-types = { path = "../../types" } +aptos-crypto = { path = "../../crates/aptos-crypto" } +aptos-rest-client = { path = "../../crates/aptos-rest-client" } [[bin]] name = "aptos-release-builder" diff --git a/aptos-move/aptos-release-builder/data/example.yaml b/aptos-move/aptos-release-builder/data/example.yaml new file mode 100644 index 0000000000000..7348be4f6fc86 --- /dev/null +++ b/aptos-move/aptos-release-builder/data/example.yaml @@ -0,0 +1,473 @@ +--- +testnet: true +is_multi_step: false +framework_release: true +gas_schedule: + feature_version: 4 + entries: + - - instr.nop + - 200 + - - instr.ret + - 1200 + - - instr.abort + - 1200 + - - instr.br_true + - 2400 + - - instr.br_false + - 2400 + - - instr.branch + - 1600 + - - instr.pop + - 800 + - - instr.ld_u8 + - 1200 + - - instr.ld_u64 + - 1200 + - - instr.ld_u128 + - 1600 + - - instr.ld_true + - 1200 + - - instr.ld_false + - 1200 + - - instr.ld_const.base + - 13000 + - - instr.ld_const.per_byte + - 700 + - - instr.imm_borrow_loc + - 1200 + - - instr.mut_borrow_loc + - 1200 + - - instr.imm_borrow_field + - 4000 + - - instr.mut_borrow_field + - 4000 + - - instr.imm_borrow_field_generic + - 4000 + - - instr.mut_borrow_field_generic + - 4000 + - - instr.copy_loc.base + - 1600 + - - instr.copy_loc.per_abs_val_unit + - 80 + - - instr.move_loc.base + - 2400 + - - instr.st_loc.base + - 2400 + - - instr.call.base + - 20000 + - - instr.call.per_arg + - 2000 + - - instr.call.per_local + - 2000 + - - instr.call_generic.base + - 20000 + - - instr.call_generic.per_ty_arg + - 2000 + - - instr.call_generic.per_arg + - 2000 + - - instr.call_generic.per_local + - 2000 + - - instr.pack.base + - 4400 + - - instr.pack.per_field + - 800 + - - instr.pack_generic.base + - 4400 + - - instr.pack_generic.per_field + - 800 + - - instr.unpack.base + - 4400 + - - instr.unpack.per_field + - 800 + - - instr.unpack_generic.base + - 4400 + - - instr.unpack_generic.per_field + - 800 + - - instr.read_ref.base + - 4000 + - - instr.read_ref.per_abs_val_unit + - 80 + - - instr.write_ref.base + - 4000 + - - instr.freeze_ref + - 200 + - - instr.cast_u8 + - 2400 + - - instr.cast_u64 + - 2400 + - - instr.cast_u128 + - 2400 + - - instr.add + - 3200 + - - instr.sub + - 3200 + - - instr.mul + - 3200 + - - instr.mod + - 3200 + - - instr.div + - 3200 + - - instr.bit_or + - 3200 + - - instr.bit_and + - 3200 + - - instr.bit_xor + - 3200 + - - instr.bit_shl + - 3200 + - - instr.bit_shr + - 3200 + - - instr.or + - 3200 + - - instr.and + - 3200 + - - instr.not + - 3200 + - - instr.lt + - 3200 + - - instr.gt + - 3200 + - - instr.le + - 3200 + - - instr.ge + - 3200 + - - instr.eq.base + - 2000 + - - instr.eq.per_abs_val_unit + - 80 + - - instr.neq.base + - 2000 + - - instr.neq.per_abs_val_unit + - 80 + - - instr.imm_borrow_global.base + - 10000 + - - instr.imm_borrow_global_generic.base + - 10000 + - - instr.mut_borrow_global.base + - 10000 + - - instr.mut_borrow_global_generic.base + - 10000 + - - instr.exists.base + - 5000 + - - instr.exists_generic.base + - 5000 + - - instr.move_from.base + - 7000 + - - instr.move_from_generic.base + - 7000 + - - instr.move_to.base + - 10000 + - - instr.move_to_generic.base + - 10000 + - - instr.vec_len.base + - 4400 + - - instr.vec_imm_borrow.base + - 6600 + - - instr.vec_mut_borrow.base + - 6600 + - - instr.vec_push_back.base + - 7600 + - - instr.vec_pop_back.base + - 5200 + - - instr.vec_swap.base + - 6000 + - - instr.vec_pack.base + - 12000 + - - instr.vec_pack.per_elem + - 800 + - - instr.vec_unpack.base + - 10000 + - - instr.vec_unpack.per_expected_elem + - 800 + - - txn.min_transaction_gas_units + - 1500000 + - - txn.large_transaction_cutoff + - 600 + - - txn.intrinsic_gas_per_byte + - 2000 + - - txn.maximum_number_of_gas_units + - 2000000 + - - txn.min_price_per_gas_unit + - 100 + - - txn.max_price_per_gas_unit + - 10000000000 + - - txn.max_transaction_size_in_bytes + - 65536 + - - txn.gas_unit_scaling_factor + - 10000 + - - txn.load_data.base + - 16000 + - - txn.load_data.per_byte + - 1000 + - - txn.load_data.failure + - 0 + - - txn.write_data.per_op + - 160000 + - - txn.write_data.new_item + - 1280000 + - - txn.write_data.per_byte_in_key + - 10000 + - - txn.write_data.per_byte_in_val + - 10000 + - - txn.memory_quota + - 10000000 + - - move_stdlib.bcs.to_bytes.per_byte_serialized + - 200 + - - move_stdlib.bcs.to_bytes.failure + - 20000 + - - move_stdlib.hash.sha2_256.base + - 60000 + - - move_stdlib.hash.sha2_256.per_byte + - 1000 + - - move_stdlib.hash.sha3_256.base + - 80000 + - - move_stdlib.hash.sha3_256.per_byte + - 900 + - - move_stdlib.signer.borrow_address.base + - 4000 + - - move_stdlib.string.check_utf8.base + - 6000 + - - move_stdlib.string.check_utf8.per_byte + - 160 + - - move_stdlib.string.is_char_boundary.base + - 6000 + - - move_stdlib.string.sub_string.base + - 8000 + - - move_stdlib.string.sub_string.per_byte + - 60 + - - move_stdlib.string.index_of.base + - 8000 + - - move_stdlib.string.index_of.per_byte_pattern + - 400 + - - move_stdlib.string.index_of.per_byte_searched + - 200 + - - aptos_framework.account.create_address.base + - 6000 + - - aptos_framework.account.create_signer.base + - 6000 + - - aptos_framework.bls12381.base + - 3000 + - - aptos_framework.bls12381.per_pubkey_deserialize + - 2180000 + - - aptos_framework.bls12381.per_pubkey_aggregate + - 84000 + - - aptos_framework.bls12381.per_pubkey_subgroup_check + - 7400000 + - - aptos_framework.bls12381.per_sig_deserialize + - 4440000 + - - aptos_framework.bls12381.per_sig_aggregate + - 233000 + - - aptos_framework.bls12381.per_sig_subgroup_check + - 9210000 + - - aptos_framework.bls12381.per_sig_verify + - 169700000 + - - aptos_framework.bls12381.per_pop_verify + - 206000000 + - - aptos_framework.bls12381.per_pairing + - 80260000 + - - aptos_framework.bls12381.per_msg_hashing + - 30800000 + - - aptos_framework.bls12381.per_byte_hashing + - 1000 + - - aptos_framework.signature.base + - 3000 + - - aptos_framework.signature.per_pubkey_deserialize + - 760000 + - - aptos_framework.signature.per_pubkey_small_order_check + - 127000 + - - aptos_framework.signature.per_sig_deserialize + - 7500 + - - aptos_framework.signature.per_sig_strict_verify + - 5340000 + - - aptos_framework.signature.per_msg_hashing_base + - 64800 + - - aptos_framework.signature.per_msg_byte_hashing + - 1200 + - - aptos_framework.secp256k1.base + - 3000 + - - aptos_framework.secp256k1.ecdsa_recover + - 32200000 + - - aptos_framework.ristretto255.basepoint_mul + - 2560000 + - - aptos_framework.ristretto255.basepoint_double_mul + - 8800000 + - - aptos_framework.ristretto255.point_add + - 42700 + - - aptos_framework.ristretto255.point_compress + - 800000 + - - aptos_framework.ristretto255.point_decompress + - 810000 + - - aptos_framework.ristretto255.point_equals + - 46000 + - - aptos_framework.ristretto255.point_from_64_uniform_bytes + - 1630000 + - - aptos_framework.ristretto255.point_identity + - 3000 + - - aptos_framework.ristretto255.point_mul + - 9420000 + - - aptos_framework.ristretto255.point_neg + - 7200 + - - aptos_framework.ristretto255.point_sub + - 42600 + - - aptos_framework.ristretto255.point_parse_arg + - 3000 + - - aptos_framework.ristretto255.scalar_sha512_per_byte + - 1200 + - - aptos_framework.ristretto255.scalar_sha512_per_hash + - 64800 + - - aptos_framework.ristretto255.scalar_add + - 15400 + - - aptos_framework.ristretto255.scalar_reduced_from_32_bytes + - 14200 + - - aptos_framework.ristretto255.scalar_uniform_from_64_bytes + - 24900 + - - aptos_framework.ristretto255.scalar_from_u128 + - 3500 + - - aptos_framework.ristretto255.scalar_from_u64 + - 3500 + - - aptos_framework.ristretto255.scalar_invert + - 2200000 + - - aptos_framework.ristretto255.scalar_is_canonical + - 23000 + - - aptos_framework.ristretto255.scalar_mul + - 21300 + - - aptos_framework.ristretto255.scalar_neg + - 14500 + - - aptos_framework.ristretto255.scalar_sub + - 21200 + - - aptos_framework.ristretto255.scalar_parse_arg + - 3000 + - - aptos_framework.hash.sip_hash.base + - 20000 + - - aptos_framework.hash.sip_hash.per_byte + - 400 + - - aptos_framework.hash.keccak256.base + - 80000 + - - aptos_framework.hash.keccak256.per_byte + - 900 + - - aptos_framework.type_info.type_of.base + - 6000 + - - aptos_framework.type_info.type_of.per_abstract_memory_unit + - 100 + - - aptos_framework.type_info.type_name.base + - 6000 + - - aptos_framework.type_info.type_name.per_abstract_memory_unit + - 100 + - - aptos_framework.type_info.chain_id.base + - 3000 + - - aptos_framework.hash.sha2_512.base + - 3240 + - - aptos_framework.hash.sha2_512.per_byte + - 60 + - - aptos_framework.hash.sha3_512.base + - 4500 + - - aptos_framework.hash.sha3_512.per_byte + - 50 + - - aptos_framework.hash.ripemd160.base + - 3000 + - - aptos_framework.hash.ripemd160.per_byte + - 50 + - - aptos_framework.util.from_bytes.base + - 6000 + - - aptos_framework.util.from_bytes.per_byte + - 100 + - - aptos_framework.transaction_context.get_script_hash.base + - 4000 + - - aptos_framework.code.request_publish.base + - 10000 + - - aptos_framework.code.request_publish.per_byte + - 40 + - - aptos_framework.event.write_to_event_store.base + - 500000 + - - aptos_framework.event.write_to_event_store.per_abstract_memory_unit + - 5000 + - - aptos_framework.state_storage.get_usage.base + - 10000 + - - aptos_framework.aggregator.add.base + - 6000 + - - aptos_framework.aggregator.read.base + - 6000 + - - aptos_framework.aggregator.sub.base + - 6000 + - - aptos_framework.aggregator.destroy.base + - 10000 + - - aptos_framework.aggregator_factory.new_aggregator.base + - 10000 + - - table.common.load.base + - 8000 + - - table.common.load.per_byte + - 1000 + - - table.common.load.failure + - 0 + - - table.new_table_handle.base + - 20000 + - - table.add_box.base + - 24000 + - - table.add_box.per_byte_serialized + - 200 + - - table.borrow_box.base + - 24000 + - - table.borrow_box.per_byte_serialized + - 200 + - - table.contains_box.base + - 24000 + - - table.contains_box.per_byte_serialized + - 200 + - - table.remove_box.base + - 24000 + - - table.remove_box.per_byte_serialized + - 200 + - - table.destroy_empty_box.base + - 24000 + - - table.drop_unchecked_box.base + - 2000 + - - misc.abs_val.u8 + - 40 + - - misc.abs_val.u64 + - 40 + - - misc.abs_val.u128 + - 40 + - - misc.abs_val.bool + - 40 + - - misc.abs_val.address + - 40 + - - misc.abs_val.struct + - 40 + - - misc.abs_val.vector + - 40 + - - misc.abs_val.reference + - 40 + - - misc.abs_val.per_u8_packed + - 1 + - - misc.abs_val.per_u64_packed + - 8 + - - misc.abs_val.per_u128_packed + - 16 + - - misc.abs_val.per_bool_packed + - 1 + - - misc.abs_val.per_address_packed + - 32 +version: + major: 4 +feature_flags: + enabled: + - code_dependency_check + - treat_friend_as_private + disabled: [] +consensus_config: + V1: + decoupled_execution: true + back_pressure_limit: 10 + exclude_round: 20 + proposer_election_type: + leader_reputation: + proposer_and_voter_v2: + active_weight: 1000 + inactive_weight: 10 + failed_weight: 1 + failure_threshold_percent: 10 + proposer_window_num_validators_multiplier: 10 + voter_window_num_validators_multiplier: 1 + weight_by_voting_power: true + use_history_from_previous_epoch_max_count: 5 + max_failed_authors_to_store: 10 \ No newline at end of file diff --git a/aptos-move/aptos-release-builder/src/components/consensus_config.rs b/aptos-move/aptos-release-builder/src/components/consensus_config.rs new file mode 100644 index 0000000000000..75a0d9518c5ad --- /dev/null +++ b/aptos-move/aptos-release-builder/src/components/consensus_config.rs @@ -0,0 +1,42 @@ +// Copyright (c) Aptos +// SPDX-License-Identifier: Apache-2.0 + +use crate::utils::*; +use anyhow::Result; +use aptos_types::on_chain_config::OnChainConsensusConfig; +use move_model::{code_writer::CodeWriter, emit, emitln, model::Loc}; + +pub fn generate_consensus_upgrade_proposal( + consensus_config: &OnChainConsensusConfig, + is_testnet: bool, + next_execution_hash: String, +) -> Result> { + let mut result = vec![]; + + let writer = CodeWriter::new(Loc::default()); + + emitln!(writer, "// Consensus config upgrade proposal\n"); + + let proposal = generate_governance_proposal( + &writer, + is_testnet, + &next_execution_hash, + "aptos_framework::consensus_config", + |writer| { + let consensus_config_blob = bcs::to_bytes(consensus_config).unwrap(); + assert!(consensus_config_blob.len() < 65536); + + emit!(writer, "let consensus_blob: vector = "); + generate_blob(writer, &consensus_config_blob); + emitln!(writer, ";\n"); + + emitln!( + writer, + "consensus_config::set(framework_signer, consensus_blob);" + ); + }, + ); + + result.push(("consensus-config".to_string(), proposal)); + Ok(result) +} diff --git a/aptos-move/aptos-release-builder/src/components/feature_flags.rs b/aptos-move/aptos-release-builder/src/components/feature_flags.rs index e0339e2eb2570..b9c5434c61a67 100644 --- a/aptos-move/aptos-release-builder/src/components/feature_flags.rs +++ b/aptos-move/aptos-release-builder/src/components/feature_flags.rs @@ -3,14 +3,24 @@ use crate::utils::*; use anyhow::Result; -use aptos_types::on_chain_config::FeatureFlag; +use aptos_types::on_chain_config::{FeatureFlag as AptosFeatureFlag, Features as AptosFeatures}; use move_model::{code_writer::CodeWriter, emit, emitln, model::Loc}; +use serde::{Deserialize, Serialize}; +#[derive(Clone, Deserialize, PartialEq, Eq, Serialize)] pub struct Features { pub enabled: Vec, pub disabled: Vec, } +#[derive(Clone, Deserialize, PartialEq, Eq, Serialize)] +#[allow(non_camel_case_types)] +#[serde(rename_all = "snake_case")] +pub enum FeatureFlag { + CodeDependencyCheck, + TreatFriendAsPrivate, +} + fn generate_features_blob(writer: &CodeWriter, data: &[u64]) { emitln!(writer, "vector["); writer.indent(); @@ -32,18 +42,19 @@ fn generate_features_blob(writer: &CodeWriter, data: &[u64]) { pub fn generate_feature_upgrade_proposal( features: &Features, is_testnet: bool, + next_execution_hash: String, ) -> Result> { let mut result = vec![]; let enabled = features .enabled .iter() - .map(|f| *f as u64) + .map(|f| AptosFeatureFlag::from(f.clone()) as u64) .collect::>(); let disabled = features .disabled .iter() - .map(|f| *f as u64) + .map(|f| AptosFeatureFlag::from(f.clone()) as u64) .collect::>(); assert!(enabled.len() < u16::MAX as usize); @@ -51,25 +62,59 @@ pub fn generate_feature_upgrade_proposal( let writer = CodeWriter::new(Loc::default()); - if is_testnet { - generate_testnet_header(&writer, "std::features"); - } else { - generate_governance_proposal_header(&writer, "std::features"); - } - - emit!(writer, "let enabled_blob: vector = "); - generate_features_blob(&writer, &enabled); - emitln!(writer, ";\n"); + let proposal = generate_governance_proposal( + &writer, + is_testnet, + &next_execution_hash, + "std::features", + |writer| { + emit!(writer, "let enabled_blob: vector = "); + generate_features_blob(writer, &enabled); + emitln!(writer, ";\n"); - emit!(writer, "let disabled_blob: vector = "); - generate_features_blob(&writer, &disabled); - emitln!(writer, ";\n"); + emit!(writer, "let disabled_blob: vector = "); + generate_features_blob(writer, &disabled); + emitln!(writer, ";\n"); - emitln!( - writer, - "features::change_feature_flags(framework_signer, enabled_blob, disabled_blob);" + emitln!( + writer, + "features::change_feature_flags(framework_signer, enabled_blob, disabled_blob);" + ); + }, ); - result.push(("features".to_string(), finish_with_footer(&writer))); + result.push(("features".to_string(), proposal)); Ok(result) } + +impl From for AptosFeatureFlag { + fn from(f: FeatureFlag) -> Self { + match f { + FeatureFlag::CodeDependencyCheck => AFeatureFlag::CODE_DEPENDENCY_CHECK, + FeatureFlag::TreatFriendAsPrivate => AFeatureFlag::TREAT_FRIEND_AS_PRIVATE, + } + } +} + +// We don't need this implementation. Just to make sure we have an exhaustive 1-1 mapping between the two structs. +impl From for FeatureFlag { + fn from(f: AptosFeatureFlag) -> Self { + match f { + AFeatureFlag::CODE_DEPENDENCY_CHECK => FeatureFlag::CodeDependencyCheck, + AFeatureFlag::TREAT_FRIEND_AS_PRIVATE => FeatureFlag::TreatFriendAsPrivate, + } + } +} + +impl Features { + // Compare if the current feature set is different from features that has been enabled on chain. + pub(crate) fn has_modified(&self, on_chain_features: &AptosFeatures) -> bool { + self.enabled + .iter() + .any(|f| !on_chain_features.is_enabled(AptosFeatureFlag::from(f.clone()))) + || self + .disabled + .iter() + .any(|f| on_chain_features.is_enabled(AptosFeatureFlag::from(f.clone()))) + } +} diff --git a/aptos-move/aptos-release-builder/src/components/framework.rs b/aptos-move/aptos-release-builder/src/components/framework.rs index 8bde20897c540..db238b0e3cb4e 100644 --- a/aptos-move/aptos-release-builder/src/components/framework.rs +++ b/aptos-move/aptos-release-builder/src/components/framework.rs @@ -2,23 +2,34 @@ // SPDX-License-Identifier: Apache-2.0 use anyhow::Result; +use aptos_crypto::HashValue; use aptos_temppath::TempPath; use std::process::Command; -pub fn generate_upgrade_proposals(is_testnet: bool) -> Result> { - let package_path_list = vec![ +pub fn generate_upgrade_proposals( + is_testnet: bool, + next_execution_hash: String, +) -> Result> { + let mut package_path_list = vec![ ("0x1", "aptos-move/framework/move-stdlib"), ("0x1", "aptos-move/framework/aptos-stdlib"), ("0x1", "aptos-move/framework/aptos-framework"), ("0x3", "aptos-move/framework/aptos-token"), ]; - let mut result = vec![]; + let mut result: Vec<(String, String)> = vec![]; let mut root_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).to_path_buf(); root_path.pop(); root_path.pop(); + // For generating multi-step proposal files, we need to generate them in the reverse order since + // we need the hash of the next script. + // We will reverse the order back when writing the files into a directory. + if !next_execution_hash.is_empty() { + package_path_list.reverse(); + } + for (publish_addr, relative_package_path) in package_path_list.iter() { let temp_script_path = TempPath::new(); temp_script_path.create_as_file()?; @@ -35,23 +46,49 @@ pub fn generate_upgrade_proposals(is_testnet: bool) -> Result Result> { let mut result = vec![]; - let gas_schedule_blob = bcs::to_bytes(gas_schedule).unwrap(); - - assert!(gas_schedule_blob.len() < 65536); - let writer = CodeWriter::new(Loc::default()); emitln!(writer, "// Gas schedule upgrade proposal\n"); @@ -37,21 +34,25 @@ pub fn generate_gas_upgrade_proposal( } emitln!(writer); - if is_testnet { - generate_testnet_header(&writer, "aptos_framework::gas_schedule"); - } else { - generate_governance_proposal_header(&writer, "aptos_framework::gas_schedule"); - } - - emit!(writer, "let gas_schedule_blob: vector = "); - generate_blob(&writer, &gas_schedule_blob); - emitln!(writer, ";\n"); - - emitln!( - writer, - "gas_schedule::set_gas_schedule(framework_signer, gas_schedule_blob);" + let proposal = generate_governance_proposal( + &writer, + is_testnet, + &next_execution_hash, + "aptos_framework::gas_schedule", + |writer| { + let gas_schedule_blob = bcs::to_bytes(gas_schedule).unwrap(); + assert!(gas_schedule_blob.len() < 65536); + emit!(writer, "let gas_schedule_blob: vector = "); + generate_blob(writer, &gas_schedule_blob); + emitln!(writer, ";\n"); + + emitln!( + writer, + "gas_schedule::set_gas_schedule(framework_signer, gas_schedule_blob);" + ); + }, ); - result.push(("gas-schedule".to_string(), finish_with_footer(&writer))); + result.push(("gas-schedule".to_string(), proposal)); Ok(result) } diff --git a/aptos-move/aptos-release-builder/src/components/mod.rs b/aptos-move/aptos-release-builder/src/components/mod.rs index 08003c13ae70c..3ca1f486c37fa 100644 --- a/aptos-move/aptos-release-builder/src/components/mod.rs +++ b/aptos-move/aptos-release-builder/src/components/mod.rs @@ -3,49 +3,105 @@ use crate::components::feature_flags::Features; use anyhow::{anyhow, Result}; -use aptos_types::on_chain_config::{GasScheduleV2, Version}; -use std::path::Path; +use aptos_crypto::HashValue; +use aptos_rest_client::Client; +use aptos_types::{ + account_config::CORE_CODE_ADDRESS, + on_chain_config::{GasScheduleV2, OnChainConfig, OnChainConsensusConfig, Version}, +}; +use futures::executor::block_on; +use serde::{Deserialize, Serialize}; +use std::{ + fs::File, + io::{Read, Write}, + path::Path, +}; +use url::Url; +pub mod consensus_config; pub mod feature_flags; pub mod framework; pub mod gas; pub mod version; +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq)] pub struct ReleaseConfig { + pub testnet: bool, + pub remote_endpoint: Option, + pub framework_release: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] pub gas_schedule: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub version: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub feature_flags: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub consensus_config: Option, + #[serde(default)] + pub is_multi_step: bool, } -impl ReleaseConfig { - pub fn generate_release_proposal_scripts( - &self, - base_path: &Path, - is_testnet: bool, - ) -> Result<()> { - let mut result = vec![]; +// Compare the current on chain config with the value recorded on chain. Return false if there's a difference. +fn fetch_and_equals( + client: &Option, + expected: &T, +) -> Result { + match client { + Some(client) => { + let config = T::deserialize_into_config( + block_on(async { + client + .get_account_resource_bytes( + CORE_CODE_ADDRESS, + format!( + "{}::{}::{}", + T::ADDRESS, + T::MODULE_IDENTIFIER, + T::TYPE_IDENTIFIER + ) + .as_str(), + ) + .await + })? + .inner(), + )?; - // First create framework releases - result.append(&mut framework::generate_upgrade_proposals(is_testnet)?); + Ok(&config == expected) + } + None => Ok(false), + } +} - if let Some(gas_schedule) = &self.gas_schedule { - result.append(&mut gas::generate_gas_upgrade_proposal( - gas_schedule, - is_testnet, - )?); +impl ReleaseConfig { + pub fn generate_release_proposal_scripts(&self, base_path: &Path) -> Result<()> { + let mut result: Vec<(String, String)> = vec![]; + let mut release_generation_functions: Vec< + &dyn Fn(&Self, &Option, &mut Vec<(String, String)>) -> Result<()>, + > = vec![ + &Self::generate_framework_release, + &Self::generate_gas_schedule, + &Self::generate_version_file, + &Self::generate_feature_flag_file, + &Self::generate_consensus_file, + ]; + let client = self + .remote_endpoint + .as_ref() + .map(|url| Client::new(url.clone())); + + // If we are generating multi-step proposal files, we generate the files in reverse order, + // since we need to pass in the hash of the next file to the previous file. + if self.is_multi_step { + release_generation_functions.reverse(); } - if let Some(version) = &self.version { - result.append(&mut version::generate_version_upgrade_proposal( - version, is_testnet, - )?); + for f in &release_generation_functions { + (f)(self, &client, &mut result)?; } - if let Some(feature_flags) = &self.feature_flags { - result.append(&mut feature_flags::generate_feature_upgrade_proposal( - feature_flags, - is_testnet, - )?); + // Here we are reversing the results back, so the result would be in order. + if self.is_multi_step { + result.reverse(); } for (idx, (script_name, script)) in result.into_iter().enumerate() { @@ -57,16 +113,182 @@ impl ReleaseConfig { std::fs::write(script_path.as_path(), script.as_bytes()) .map_err(|err| anyhow!("Failed to write to file: {:?}", err))?; } + + Ok(()) + } + + fn generate_framework_release( + &self, + _client: &Option, + result: &mut Vec<(String, String)>, + ) -> Result<()> { + if self.framework_release { + result.append(&mut framework::generate_upgrade_proposals( + self.testnet, + if self.is_multi_step { + Self::get_execution_hash(result) + } else { + "".to_owned() + }, + )?); + } + Ok(()) + } + + fn generate_gas_schedule( + &self, + client: &Option, + result: &mut Vec<(String, String)>, + ) -> Result<()> { + if let Some(gas_schedule) = &self.gas_schedule { + if !fetch_and_equals::(client, gas_schedule)? { + result.append(&mut gas::generate_gas_upgrade_proposal( + gas_schedule, + self.testnet, + if self.is_multi_step { + Self::get_execution_hash(result) + } else { + "".to_owned() + }, + )?); + } + } + Ok(()) + } + + fn generate_version_file( + &self, + client: &Option, + result: &mut Vec<(String, String)>, + ) -> Result<()> { + if let Some(version) = &self.version { + if !fetch_and_equals::(client, version)? { + result.append(&mut version::generate_version_upgrade_proposal( + version, + self.testnet, + if self.is_multi_step { + Self::get_execution_hash(result) + } else { + "".to_owned() + }, + )?); + } + } + Ok(()) + } + + fn generate_feature_flag_file( + &self, + client: &Option, + result: &mut Vec<(String, String)>, + ) -> Result<()> { + if let Some(feature_flags) = &self.feature_flags { + let mut needs_update = false; + if let Some(client) = client { + let features = block_on(async { + client + .get_account_resource_bcs::( + CORE_CODE_ADDRESS, + "0x1::features::Features", + ) + .await + })?; + // Only update the feature flags section when there's a divergence between the local configs and on chain configs. + needs_update = feature_flags.has_modified(features.inner()); + } + if needs_update { + result.append(&mut feature_flags::generate_feature_upgrade_proposal( + feature_flags, + self.testnet, + if self.is_multi_step { + Self::get_execution_hash(result) + } else { + "".to_owned() + }, + )?); + } + } + Ok(()) + } + + fn generate_consensus_file( + &self, + client: &Option, + result: &mut Vec<(String, String)>, + ) -> Result<()> { + if let Some(consensus_config) = &self.consensus_config { + if fetch_and_equals(client, consensus_config)? { + result.append(&mut consensus_config::generate_consensus_upgrade_proposal( + consensus_config, + self.testnet, + if self.is_multi_step { + Self::get_execution_hash(result) + } else { + "".to_owned() + }, + )?); + } + } Ok(()) } + + pub fn load_config>(path: P) -> Result { + // Open the file and read it into a string + let config_path_string = path.as_ref().to_str().unwrap().to_string(); + let mut file = File::open(&path).map_err(|error| { + anyhow!( + "Failed to open config file: {:?}. Error: {:?}", + config_path_string, + error + ) + })?; + let mut contents = String::new(); + file.read_to_string(&mut contents).map_err(|error| { + anyhow!( + "Failed to read the config file into a string: {:?}. Error: {:?}", + config_path_string, + error + ) + })?; + + // Parse the file string + Self::parse(&contents) + } + + pub fn save_config>(&self, output_file: P) -> Result<()> { + let contents = + serde_yaml::to_vec(&self).map_err(|e| anyhow!("failed to generate config: {:?}", e))?; + let mut file = File::create(output_file.as_ref()) + .map_err(|e| anyhow!("failed to create file: {:?}", e))?; + file.write_all(&contents) + .map_err(|e| anyhow!("failed to write file: {:?}", e))?; + Ok(()) + } + + pub fn parse(serialized: &str) -> Result { + serde_yaml::from_str(serialized).map_err(|e| anyhow!("Failed to parse the config: {:?}", e)) + } + + fn get_execution_hash(result: &Vec<(String, String)>) -> String { + if result.is_empty() { + "vector::empty()".to_owned() + } else { + HashValue::sha3_256_of(result.last().unwrap().1.to_owned().as_bytes()).to_string() + } + } } impl Default for ReleaseConfig { fn default() -> Self { ReleaseConfig { + testnet: true, + framework_release: true, gas_schedule: Some(aptos_gas::gen::current_gas_schedule()), version: None, feature_flags: None, + consensus_config: Some(OnChainConsensusConfig::default()), + is_multi_step: false, + remote_endpoint: None, } } } diff --git a/aptos-move/aptos-release-builder/src/components/version.rs b/aptos-move/aptos-release-builder/src/components/version.rs index ad97b76f4efaf..c1da7146399fa 100644 --- a/aptos-move/aptos-release-builder/src/components/version.rs +++ b/aptos-move/aptos-release-builder/src/components/version.rs @@ -9,23 +9,26 @@ use move_model::{code_writer::CodeWriter, emitln, model::Loc}; pub fn generate_version_upgrade_proposal( version: &Version, is_testnet: bool, + next_execution_hash: String, ) -> Result> { let mut result = vec![]; let writer = CodeWriter::new(Loc::default()); - if is_testnet { - generate_testnet_header(&writer, "aptos_framework::version"); - } else { - generate_governance_proposal_header(&writer, "aptos_framework::version"); - } - - emitln!( - writer, - "version::set_version(framework_signer, {});", - version.major, + let proposal = generate_governance_proposal( + &writer, + is_testnet, + &next_execution_hash, + "aptos_framework::version", + |writer| { + emitln!( + writer, + "version::set_version(framework_signer, {});", + version.major, + ); + }, ); - result.push(("version".to_string(), finish_with_footer(&writer))); + result.push(("version".to_string(), proposal)); Ok(result) } diff --git a/aptos-move/aptos-release-builder/src/main.rs b/aptos-move/aptos-release-builder/src/main.rs index 21c9603da2a1d..cff8e96dbad0e 100644 --- a/aptos-move/aptos-release-builder/src/main.rs +++ b/aptos-move/aptos-release-builder/src/main.rs @@ -2,19 +2,42 @@ // SPDX-License-Identifier: Apache-2.0 use anyhow::Result; -use clap::Parser; +use clap::{Parser, Subcommand}; use std::path::PathBuf; -#[derive(Debug, Parser)] -pub struct GenArgs { - #[clap(short, long)] - pub output: Option, +#[derive(Parser)] +pub struct Argument { + #[clap(subcommand)] + cmd: Commands, } -fn main() -> Result<()> { - let args = GenArgs::parse(); +#[derive(Subcommand, Debug)] +pub enum Commands { + GenerateProposals { + #[clap(short, long)] + release_config: PathBuf, + #[clap(short, long)] + output_dir: PathBuf, + }, + WriteDefault { + #[clap(short, long)] + output_path: PathBuf, + }, +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = Argument::parse(); // TODO: Being able to parse the release config from a TOML file to generate the proposals. - aptos_release_builder::ReleaseConfig::default() - .generate_release_proposal_scripts(args.output.as_ref().unwrap(), true) + match args.cmd { + Commands::GenerateProposals { + release_config, + output_dir, + } => aptos_release_builder::ReleaseConfig::load_config(release_config.as_path())? + .generate_release_proposal_scripts(output_dir.as_path()), + Commands::WriteDefault { output_path } => { + aptos_release_builder::ReleaseConfig::default().save_config(output_path.as_path()) + } + } } diff --git a/aptos-move/aptos-release-builder/src/utils.rs b/aptos-move/aptos-release-builder/src/utils.rs index c3bf06ebd57ee..4e64add03f788 100644 --- a/aptos-move/aptos-release-builder/src/utils.rs +++ b/aptos-move/aptos-release-builder/src/utils.rs @@ -22,7 +22,43 @@ pub(crate) fn generate_blob(writer: &CodeWriter, data: &[u8]) { emit!(writer, "]") } -pub(crate) fn generate_governance_proposal_header(writer: &CodeWriter, deps_name: &str) { +pub(crate) fn generate_next_execution_hash_blob( + writer: &CodeWriter, + for_address: AccountAddress, + next_execution_hash: String, +) { + if next_execution_hash == "vector::empty()" { + emitln!( + writer, + "let framework_signer = aptos_governance::resolve_multi_step_proposal(proposal_id, @{}, {});\n", + for_address, + next_execution_hash, + ); + } else { + let next_execution_hash_bytes = next_execution_hash.as_bytes(); + emitln!( + writer, + "let framework_signer = aptos_governance::resolve_multi_step_proposal(" + ); + writer.indent(); + emitln!(writer, "proposal_id,"); + emitln!(writer, "@{},", for_address); + emit!(writer, "vector["); + for (_, b) in next_execution_hash_bytes.iter().enumerate() { + emit!(writer, "{}u8,", b); + } + emitln!(writer, "],"); + writer.unindent(); + emitln!(writer, "};"); + } +} + +pub(crate) fn generate_governance_proposal_header( + writer: &CodeWriter, + deps_name: &str, + is_multi_step: bool, + next_execution_hash: &str, +) { emitln!(writer, "script {"); writer.indent(); @@ -33,11 +69,19 @@ pub(crate) fn generate_governance_proposal_header(writer: &CodeWriter, deps_name emitln!(writer, "fun main(proposal_id: u64) {"); writer.indent(); - emitln!( - writer, - "let framework_signer = aptos_governance::resolve(proposal_id, @{});\n", - AccountAddress::ONE, - ); + if is_multi_step && !next_execution_hash.is_empty() { + generate_next_execution_hash_blob( + writer, + AccountAddress::ONE, + next_execution_hash.to_owned(), + ); + } else { + emitln!( + writer, + "let framework_signer = aptos_governance::resolve(proposal_id, @{});\n", + AccountAddress::ONE, + ); + } } pub(crate) fn generate_testnet_header(writer: &CodeWriter, deps_name: &str) { @@ -68,3 +112,27 @@ pub(crate) fn finish_with_footer(writer: &CodeWriter) -> String { writer.process_result(|s| s.to_string()) } + +pub(crate) fn generate_governance_proposal( + writer: &CodeWriter, + is_testnet: bool, + next_execution_hash: &str, + deps_name: &str, + body: F, +) -> String +where + F: FnOnce(&CodeWriter), +{ + if next_execution_hash.is_empty() { + if is_testnet { + generate_testnet_header(writer, deps_name); + } else { + generate_governance_proposal_header(writer, deps_name, false, ""); + } + } else { + generate_governance_proposal_header(writer, deps_name, true, next_execution_hash); + }; + + body(writer); + finish_with_footer(writer) +} diff --git a/aptos-move/framework/src/release_bundle.rs b/aptos-move/framework/src/release_bundle.rs index 2d04b2f2a076a..60ca6fdebf457 100644 --- a/aptos-move/framework/src/release_bundle.rs +++ b/aptos-move/framework/src/release_bundle.rs @@ -29,7 +29,7 @@ pub struct ReleaseBundle { pub source_dirs: Vec, } -/// A release package consists of package metdata and the code. +/// A release package consists of package metadata and the code. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct ReleasePackage { pub metadata: PackageMetadata, @@ -193,7 +193,7 @@ impl ReleasePackage { for_address: AccountAddress, out: PathBuf, ) -> anyhow::Result<()> { - self.generate_script_proposal_impl(for_address, out, false) + self.generate_script_proposal_impl(for_address, out, false, false, "".to_owned()) } pub fn generate_script_proposal_testnet( @@ -201,7 +201,16 @@ impl ReleasePackage { for_address: AccountAddress, out: PathBuf, ) -> anyhow::Result<()> { - self.generate_script_proposal_impl(for_address, out, true) + self.generate_script_proposal_impl(for_address, out, true, false, "".to_owned()) + } + + pub fn generate_script_proposal_multi_step( + &self, + for_address: AccountAddress, + out: PathBuf, + next_execution_hash: String, + ) -> anyhow::Result<()> { + self.generate_script_proposal_impl(for_address, out, true, true, next_execution_hash) } fn generate_script_proposal_impl( @@ -209,6 +218,8 @@ impl ReleasePackage { for_address: AccountAddress, out: PathBuf, is_testnet: bool, + is_multi_step: bool, + next_execution_hash: String, ) -> anyhow::Result<()> { let writer = CodeWriter::new(Loc::default()); emitln!( @@ -223,7 +234,7 @@ impl ReleasePackage { emitln!(writer, "use aptos_framework::aptos_governance;"); emitln!(writer, "use aptos_framework::code;\n"); - if is_testnet { + if is_testnet && !is_multi_step { emitln!(writer, "fun main(core_resources: &signer){"); writer.indent(); emitln!( @@ -231,7 +242,7 @@ impl ReleasePackage { "let framework_signer = aptos_governance::get_signer_testnet_only(core_resources, @{});", for_address ); - } else { + } else if !is_multi_step { emitln!(writer, "fun main(proposal_id: u64){"); writer.indent(); emitln!( @@ -239,6 +250,10 @@ impl ReleasePackage { "let framework_signer = aptos_governance::resolve(proposal_id, @{});", for_address ); + } else { + emitln!(writer, "fun main(proposal_id: u64){"); + writer.indent(); + Self::generate_next_execution_hash_blob(&writer, for_address, next_execution_hash); } emitln!(writer, "let code = vector::empty();"); @@ -299,4 +314,38 @@ impl ReleasePackage { writer.unindent(); emit!(writer, "]") } + + fn generate_next_execution_hash_blob( + writer: &CodeWriter, + for_address: AccountAddress, + next_execution_hash: String, + ) { + if next_execution_hash == "vector::empty()" { + emitln!( + writer, + "let framework_signer = aptos_governance::resolve_multi_step_proposal(proposal_id, @{}, {});\n", + for_address, + next_execution_hash, + ); + } else { + let next_execution_hash_bytes = next_execution_hash.as_bytes(); + emitln!( + writer, + "let framework_signer = aptos_governance::resolve_multi_step_proposal(" + ); + writer.indent(); + emitln!(writer, "proposal_id,"); + emitln!(writer, "@{},", for_address); + emit!(writer, "vector["); + for (i, b) in next_execution_hash_bytes.iter().enumerate() { + if (i + 1) % 20 == 0 { + emitln!(writer); + } + emit!(writer, "{}u8,", b); + } + emitln!(writer, "],"); + writer.unindent(); + emitln!(writer, "};"); + } + } } diff --git a/crates/aptos-rest-client/src/lib.rs b/crates/aptos-rest-client/src/lib.rs index c6ce0e557764f..e269bc08a897e 100644 --- a/crates/aptos-rest-client/src/lib.rs +++ b/crates/aptos-rest-client/src/lib.rs @@ -1000,6 +1000,17 @@ impl Client { Ok(response.map(|inner| inner.to_vec())) } + pub async fn get_account_resource_bytes( + &self, + address: AccountAddress, + resource_type: &str, + ) -> AptosResult>> { + let url = self.build_path(&format!("accounts/{}/resource/{}", address, resource_type))?; + + let response = self.get_bcs(url).await?; + Ok(response.map(|inner| inner.to_vec())) + } + pub async fn get_account_resource_at_version( &self, address: AccountAddress, diff --git a/crates/aptos/src/governance/mod.rs b/crates/aptos/src/governance/mod.rs index e5afaaadc84ce..e15470f2bbe62 100644 --- a/crates/aptos/src/governance/mod.rs +++ b/crates/aptos/src/governance/mod.rs @@ -737,6 +737,9 @@ pub struct GenerateUpgradeProposal { #[clap(long)] pub(crate) testnet: bool, + #[clap(long, default_value = "")] + pub(crate) next_execution_hash: String, + #[clap(flatten)] pub(crate) move_options: MovePackageDir, } @@ -754,16 +757,23 @@ impl CliCommand<()> for GenerateUpgradeProposal { included_artifacts, output, testnet, + next_execution_hash, } = self; let package_path = move_options.get_package_path()?; let options = included_artifacts.build_options(move_options.named_addresses()); let package = BuiltPackage::build(package_path, options)?; let release = ReleasePackage::new(package)?; - if testnet { + + // If we're generating a single-step proposal on testnet + if testnet && next_execution_hash.is_empty() { release.generate_script_proposal_testnet(account, output)?; - } else { + // If we're generating a single-step proposal on mainnet + } else if next_execution_hash.is_empty() { release.generate_script_proposal(account, output)?; - } + // If we're generating a multi-step proposal + } else { + release.generate_script_proposal_multi_step(account, output, next_execution_hash)?; + }; Ok(()) } } diff --git a/testsuite/smoke-test/src/upgrade.rs b/testsuite/smoke-test/src/upgrade.rs index 8d9027d99efec..d58dd0e0cb45c 100644 --- a/testsuite/smoke-test/src/upgrade.rs +++ b/testsuite/smoke-test/src/upgrade.rs @@ -8,10 +8,10 @@ use crate::{ use aptos_crypto::ValidCryptoMaterialStringExt; use aptos_gas::{AptosGasParameters, GasQuantity, InitialGasSchedule, ToOnChainGasSchedule}; use aptos_release_builder::components::{ - feature_flags::Features, gas::generate_gas_upgrade_proposal, + feature_flags::{FeatureFlag, Features}, + gas::generate_gas_upgrade_proposal, }; use aptos_temppath::TempPath; -use aptos_types::on_chain_config::FeatureFlag; use forge::Swarm; use std::fs; use std::process::Command; @@ -47,7 +47,7 @@ async fn test_upgrade_flow() { entries: gas_parameters.to_on_chain_gas_schedule(), }; - let (_, update_gas_script) = generate_gas_upgrade_proposal(&gas_schedule, true) + let (_, update_gas_script) = generate_gas_upgrade_proposal(&gas_schedule, true, "".to_owned()) .unwrap() .pop() .unwrap(); @@ -84,8 +84,8 @@ async fn test_upgrade_flow() { let config = aptos_release_builder::ReleaseConfig { feature_flags: Some(Features { enabled: vec![ - FeatureFlag::CODE_DEPENDENCY_CHECK, - FeatureFlag::TREAT_FRIEND_AS_PRIVATE, + FeatureFlag::CodeDependencyCheck, + FeatureFlag::TreatFriendAsPrivate, ], disabled: vec![], }), @@ -93,7 +93,7 @@ async fn test_upgrade_flow() { }; config - .generate_release_proposal_scripts(upgrade_scripts_folder.path(), true) + .generate_release_proposal_scripts(upgrade_scripts_folder.path()) .unwrap(); let mut scripts = fs::read_dir(upgrade_scripts_folder.path()) .unwrap()