From aa17f3e554dcc79d3dafca4bd86f47776409f528 Mon Sep 17 00:00:00 2001 From: Xavier Lau Date: Fri, 3 Feb 2023 00:15:23 +0800 Subject: [PATCH] Rebuild account reference counters (#249) * Rebuild account reference counters part.1 * part.2 * part.3 * TODO * Fix * Fixes Signed-off-by: Xavier Lau * Doc --------- Signed-off-by: Xavier Lau --- tool/state-processor/src/identity/mod.rs | 2 +- tool/state-processor/src/main.rs | 5 +- tool/state-processor/src/processor.rs | 71 +++++++++++------------ tool/state-processor/src/session/mod.rs | 23 -------- tool/state-processor/src/staking/mod.rs | 7 +++ tool/state-processor/src/system/README.md | 8 +-- tool/state-processor/src/system/mod.rs | 50 ++++------------ tool/state-processor/src/tests.rs | 3 +- tool/state-processor/src/util.rs | 4 -- 9 files changed, 61 insertions(+), 112 deletions(-) delete mode 100644 tool/state-processor/src/session/mod.rs diff --git a/tool/state-processor/src/identity/mod.rs b/tool/state-processor/src/identity/mod.rs index 385ada5ac..13225904b 100644 --- a/tool/state-processor/src/identity/mod.rs +++ b/tool/state-processor/src/identity/mod.rs @@ -38,7 +38,7 @@ where }) }); - self.shell_state.reserve(a, r + rj); + self.shell_state.reserve(&a, r + rj); self.shell_state.insert_value(b"AccountMigration", b"Identities", &k, v); }); diff --git a/tool/state-processor/src/main.rs b/tool/state-processor/src/main.rs index add7898e3..b64393358 100644 --- a/tool/state-processor/src/main.rs +++ b/tool/state-processor/src/main.rs @@ -16,7 +16,6 @@ use type_registry::*; mod balances; mod evm; mod identity; -mod session; mod staking; mod system; mod vesting; @@ -32,10 +31,10 @@ fn main() -> Result<()> { pretty_env_logger::init(); // >::new()?.process().save()?; - >::new()?.test().process().save()?; + // >::new()?.test().process().save()?; // >::new()?.process().save()?; // >::new()?.test().process().save()?; - // >::new()?.process().save()?; + >::new()?.process().save()?; // >::new()?.test().process().save()?; // >::new()?.process().save()?; // >::new()?.test().process().save()?; diff --git a/tool/state-processor/src/processor.rs b/tool/state-processor/src/processor.rs index 26948919c..0c3b86355 100644 --- a/tool/state-processor/src/processor.rs +++ b/tool/state-processor/src/processor.rs @@ -11,7 +11,7 @@ use std::{ use crate::*; // crates.io use anyhow::Result; -use fxhash::{FxHashMap, FxHashSet}; +use fxhash::FxHashMap; use once_cell::sync::Lazy; use parity_scale_codec::{Decode, Encode}; use serde::de::DeserializeOwned; @@ -19,7 +19,6 @@ use serde_json::Value; // hack-ink use subspector::ChainSpec; -pub type Set = FxHashSet; pub type Map = FxHashMap; pub static NOW: Lazy> = Lazy::new(|| RwLock::new(0)); @@ -193,23 +192,6 @@ impl State { self.map.keys().into_iter().any(|k| k.starts_with(&item_key(pallet, item))) } - pub fn reserve(&mut self, account_id_32: A, amount: u128) - where - A: AsRef<[u8]>, - { - let account_id_32 = account_id_32.as_ref(); - let (p, i, h) = if is_evm_address(account_id_32) { - (&b"System"[..], &b"Account"[..], &account_id_32[11..31]) - } else { - (&b"AccountMigration"[..], &b"Accounts"[..], account_id_32) - }; - - self.mutate_value(p, i, &blake2_128_concat_to_string(h), |a: &mut AccountInfo| { - a.data.free -= amount; - a.data.reserved += amount; - }); - } - pub fn take_raw_map( &mut self, prefix: &str, @@ -232,23 +214,6 @@ impl State { self } - pub fn take_keys(&mut self, prefix: &str, keys: &mut Set, process_key: F) -> &mut Self - where - F: Fn(&str, &str) -> String, - { - self.map.retain(|k, _| { - if k.starts_with(prefix) { - keys.insert(process_key(k, prefix)); - - false - } else { - true - } - }); - - self - } - pub fn take_value( &mut self, pallet: &[u8], @@ -336,6 +301,40 @@ impl State { self } + + fn mutate_account(&mut self, account_id_32: &str, f: F) + where + F: FnOnce(&mut AccountInfo), + { + let account_id_32 = array_bytes::hex2array_unchecked::<_, 32>(account_id_32); + let (p, i, h) = if is_evm_address(&account_id_32) { + (&b"System"[..], &b"Account"[..], &account_id_32[11..31]) + } else { + (&b"AccountMigration"[..], &b"Accounts"[..], &account_id_32[..]) + }; + + self.mutate_value(p, i, &blake2_128_concat_to_string(h), f); + } + + pub fn inc_consumers_by(&mut self, account_id_32: &str, x: RefCount) { + self.mutate_account(account_id_32, |a| a.consumers += x); + } + + pub fn reserve(&mut self, account_id_32: &str, amount: u128) { + self.mutate_account(account_id_32, |a| { + if a.data.free < amount { + log::warn!( + "`Account({account_id_32})` can't afford the latest runtime reservation amount" + ); + + a.data.reserved += a.data.free; + a.data.free = 0; + } else { + a.data.free -= amount; + a.data.reserved += amount; + } + }); + } } pub fn from_file(path: &str) -> Result diff --git a/tool/state-processor/src/session/mod.rs b/tool/state-processor/src/session/mod.rs deleted file mode 100644 index c7d3c4d55..000000000 --- a/tool/state-processor/src/session/mod.rs +++ /dev/null @@ -1,23 +0,0 @@ -// darwinia -use crate::*; - -impl Processor { - pub fn process_session(&mut self) -> (Set, Set) { - // Session storage items. - // https://github.dev/darwinia-network/substrate/blob/darwinia-v0.12.5/frame/session/src/lib.rs#L532 - - let prefix = item_key(b"Session", b"NextKeys"); - - let mut s_keys = Set::default(); - let mut p_keys = Set::default(); - - self.solo_state.take_keys(&prefix, &mut s_keys, |k, _| { - blake2_128_concat_to_string(key_to_account32(k)) - }); - self.para_state.take_keys(&prefix, &mut p_keys, |k, _| { - blake2_128_concat_to_string(key_to_account32(k)) - }); - - (s_keys, p_keys) - } -} diff --git a/tool/state-processor/src/staking/mod.rs b/tool/state-processor/src/staking/mod.rs index 1cdd19539..3cb20472f 100644 --- a/tool/state-processor/src/staking/mod.rs +++ b/tool/state-processor/src/staking/mod.rs @@ -30,9 +30,12 @@ impl Processor { let hash_k = blake2_128_concat_to_string(v.stash); let deposit_k = format!("{deposit_ik}{hash_k}"); let staking_k = format!("{staking_ik}{hash_k}"); + let mut consumers = 1; let mut staked_deposits = Vec::default(); if !v.deposit_items.is_empty() { + consumers += 1; + let mut deposit_ring = Balance::default(); self.shell_state.insert_raw_key_value( @@ -61,6 +64,10 @@ impl Processor { ring_pool += v.active; kton_pool += v.active_kton; + // Some accounts were killed. + // But their staking data didn't get deleted. + // TODO: https://github.com/darwinia-network/darwinia-2.0/issues/6 + self.shell_state.inc_consumers_by(&array_bytes::bytes2hex("", v.stash), consumers); self.shell_state.insert_raw_key_value( staking_k, Ledger { diff --git a/tool/state-processor/src/system/README.md b/tool/state-processor/src/system/README.md index 9467e4973..079a365f5 100644 --- a/tool/state-processor/src/system/README.md +++ b/tool/state-processor/src/system/README.md @@ -11,11 +11,11 @@ - take para `Balances::TotalIssuance` and `Balances::Locks` - there should not be any locks on parachain - check if there are any other locks -- decrease pallet-session account references -- use all previous data to build the new accounts and calculate total issuances +- burn parachain backing ring +- build the new accounts and calculate total issuances - merge solo and para account infos - - how to deal with the account references? - TODO - - burn parachain backing ring + - reset account reference counters + - increase `EVM::AccountCodes`'s `sufficients` - set `Balances::TotalIssuance` - compare the calculated one with the storage one - remove para backing account diff --git a/tool/state-processor/src/system/mod.rs b/tool/state-processor/src/system/mod.rs index 6a416e554..fa913f9e9 100644 --- a/tool/state-processor/src/system/mod.rs +++ b/tool/state-processor/src/system/mod.rs @@ -6,9 +6,6 @@ use crate::*; #[derive(Debug)] pub struct AccountAll { pub nonce: u32, - pub consumers: RefCount, - pub providers: RefCount, - pub sufficients: RefCount, pub ring: Balance, pub ring_locks: Vec, pub kton: Balance, @@ -22,31 +19,13 @@ where pub fn process_system(&mut self) -> &mut Self { // System storage items. // https://github.dev/darwinia-network/substrate/blob/darwinia-v0.12.5/frame/system/src/lib.rs#L545 - let mut solo_account_infos = self.process_solo_account_infos(); - let mut para_account_infos = self.process_para_account_infos(); + let solo_account_infos = self.process_solo_account_infos(); + let para_account_infos = self.process_para_account_infos(); let (ring_total_issuance_storage, kton_total_issuance_storage) = self.process_balances(); - let (solo_validators, para_collators) = self.process_session(); let mut accounts = Map::default(); let mut ring_total_issuance = Balance::default(); let mut kton_total_issuance = Balance::default(); - // Skip for testnets, due to https://github.com/paritytech/substrate/issues/13172. - // log::info!("decrease solo pallet-session account references"); - // solo_validators.into_iter().for_each(|k| { - // if let Some(a) = solo_account_infos.get_mut(&k) { - // a.consumers -= 1; - // } - // }); - - // Skip, due to https://github.com/paritytech/substrate/issues/13172. - // log::info!("decrease para pallet-session account references"); - // para_collators.into_iter().for_each(|k| { - // if let Some(a) = para_account_infos.get_mut(&k) { - // dbg!(get_last_64(&k)); - // a.consumers -= 1; - // } - // }); - log::info!("build accounts"); log::info!("calculate total issuance"); solo_account_infos.into_iter().for_each(|(k, v)| { @@ -57,12 +36,6 @@ where k, AccountAll { nonce: v.nonce, - // --- - // TODO: check if we could ignore para's. - consumers: v.consumers, - providers: v.providers, - sufficients: v.sufficients, - // --- ring, ring_locks: Default::default(), kton, @@ -84,9 +57,6 @@ where }) .or_insert(AccountAll { nonce: v.nonce, - consumers: v.consumers, - providers: v.providers, - sufficients: v.sufficients, ring, ring_locks: Default::default(), kton: Default::default(), @@ -128,14 +98,15 @@ where is_frozen: false, }; - log::info!("fix `EVM::AccountCodes`'s `sufficients` and set `Assets::Account`, `System::Account`, `AccountMigration::KtonAccounts` and `AccountMigration::Accounts`"); + log::info!("increase `EVM::AccountCodes`'s `sufficients` and set `Assets::Account`, `System::Account`, `AccountMigration::KtonAccounts` and `AccountMigration::Accounts`"); accounts.into_iter().for_each(|(k, v)| { let key = get_last_64(&k); let mut a = AccountInfo { nonce: v.nonce, - consumers: v.consumers, - providers: v.providers, - sufficients: v.sufficients, + consumers: Default::default(), + // https://github.com/paritytech/substrate/blob/3bc3742d5c0c5269353d7809d9f8f91104a93273/frame/system/src/lib.rs#L1708 + providers: 1, + sufficients: Default::default(), data: AccountData { free: v.ring, reserved: Default::default(), @@ -145,14 +116,12 @@ where }; if let Some(k) = try_get_evm_address(&key) { - // If the evm account is a contract contract with sufficients, then we should - // increase the sufficients by one. + // https://github.dev/paritytech/frontier/blob/ab0f4a47e42ad17e4d8551fb9b3c3a6b4c5df2db/frame/evm/src/lib.rs#L705 if self.solo_state.contains_key(&full_key( b"EVM", b"AccountCodes", &blake2_128_concat_to_string(k), - )) && a.sufficients == 0 - { + )) { a.sufficients += 1; } @@ -276,6 +245,7 @@ fn new_kton_account( asset_details: &mut AssetDetails, balance: Balance, ) -> AssetAccount { + // https://github.com/paritytech/substrate/blob/3bc3742d5c0c5269353d7809d9f8f91104a93273/frame/assets/src/functions.rs#L75 account_info.sufficients += 1; asset_details.accounts += 1; asset_details.sufficients += 1; diff --git a/tool/state-processor/src/tests.rs b/tool/state-processor/src/tests.rs index 2c54d6a09..40445043c 100644 --- a/tool/state-processor/src/tests.rs +++ b/tool/state-processor/src/tests.rs @@ -116,6 +116,7 @@ where // --- System & Balances & Assets --- +// TODO: more testcases about the account references and reservations #[test] fn solo_chain_substrate_account_adjust() { run_test(|tester| { @@ -132,7 +133,7 @@ fn solo_chain_substrate_account_adjust() { // after migrate let migrated_account = tester.migration_accounts.get(test_addr).unwrap(); - assert_eq!(solo_account.consumers, migrated_account.consumers); + assert_eq!(migrated_account.consumers, 2); assert_eq!(solo_account.providers, migrated_account.providers); assert_eq!(solo_account.sufficients + 1, migrated_account.sufficients); // nonce reset diff --git a/tool/state-processor/src/util.rs b/tool/state-processor/src/util.rs index f57f5cf2f..d469aeb6a 100644 --- a/tool/state-processor/src/util.rs +++ b/tool/state-processor/src/util.rs @@ -73,10 +73,6 @@ pub fn is_evm_address(address: &[u8]) -> bool { && address[1..31].iter().fold(address[0], |checksum, &b| checksum ^ b) == address[31] } -pub fn key_to_account32(key: &str) -> [u8; 32] { - array_bytes::hex2array_unchecked(get_last_64(key)) -} - pub fn build_spec(chain: &str) -> Result<()> { log::info!("build {chain} spec");