From 6a68a2b18a8e17c9f1946d5ccd49742e7e14b587 Mon Sep 17 00:00:00 2001 From: Anton Puhach Date: Mon, 4 Mar 2024 19:20:50 +0100 Subject: [PATCH] refactor: use NightshadeRuntime for chain garabage collection tests (#10694) Part of https://github.com/near/nearcore/issues/10678 and https://github.com/near/nearcore/issues/10634. Tests are updated to reflect the actual behaviour of chain garbage collection. --- chain/chain/Cargo.toml | 2 +- chain/chain/src/test_utils.rs | 34 ++-- chain/chain/src/tests/garbage_collection.rs | 167 ++++++++++---------- 3 files changed, 105 insertions(+), 98 deletions(-) diff --git a/chain/chain/Cargo.toml b/chain/chain/Cargo.toml index 8dfdcb831d2..418d433b4bc 100644 --- a/chain/chain/Cargo.toml +++ b/chain/chain/Cargo.toml @@ -29,6 +29,7 @@ thiserror.workspace = true tracing.workspace = true yansi.workspace = true easy-ext.workspace = true +tempfile.workspace = true near-async.workspace = true near-cache.workspace = true @@ -52,7 +53,6 @@ near-mainnet-res.workspace = true [dev-dependencies] serde_json.workspace = true primitive-types.workspace = true -tempfile.workspace = true insta.workspace = true assert_matches.workspace = true diff --git a/chain/chain/src/test_utils.rs b/chain/chain/src/test_utils.rs index 3ac474474ab..5d2264168c8 100644 --- a/chain/chain/src/test_utils.rs +++ b/chain/chain/src/test_utils.rs @@ -1,24 +1,21 @@ mod kv_runtime; mod validator_schedule; -use chrono::{DateTime, Utc}; -use near_chain_configs::GenesisConfig; -use num_rational::Ratio; use std::cmp::Ordering; use std::sync::Arc; -pub use self::kv_runtime::account_id_to_shard_id; -pub use self::kv_runtime::KeyValueRuntime; -pub use self::kv_runtime::MockEpochManager; -pub use self::validator_schedule::ValidatorSchedule; use crate::block_processing_utils::BlockNotInPoolError; use crate::chain::Chain; +use crate::runtime::NightshadeRuntime; use crate::store::ChainStoreAccess; use crate::types::{AcceptedBlock, ChainConfig, ChainGenesis}; use crate::DoomslugThresholdMode; use crate::{BlockProcessingArtifact, Provenance}; +use chrono::{DateTime, Utc}; +use near_chain_configs::Genesis; use near_chain_primitives::Error; use near_epoch_manager::shard_tracker::ShardTracker; +use near_epoch_manager::EpochManager; use near_primitives::block::Block; use near_primitives::hash::CryptoHash; use near_primitives::static_clock::StaticClock; @@ -27,10 +24,15 @@ use near_primitives::types::{AccountId, NumBlocks, NumShards}; use near_primitives::utils::MaybeValidated; use near_primitives::validator_signer::InMemoryValidatorSigner; use near_primitives::version::PROTOCOL_VERSION; +use near_store::genesis::initialize_genesis_state; use near_store::test_utils::create_test_store; use near_store::DBCol; +use num_rational::Ratio; use tracing::debug; +pub use self::kv_runtime::{account_id_to_shard_id, KeyValueRuntime, MockEpochManager}; +pub use self::validator_schedule::ValidatorSchedule; + pub fn get_chain() -> Chain { get_chain_with_epoch_length_and_num_shards(10, 1) } @@ -48,13 +50,19 @@ pub fn get_chain_with_epoch_length_and_num_shards( num_shards: NumShards, ) -> Chain { let store = create_test_store(); - let chain_genesis = ChainGenesis::new(&GenesisConfig::test()); - let vs = ValidatorSchedule::new() - .block_producers_per_epoch(vec![vec!["test1".parse().unwrap()]]) - .num_shards(num_shards); - let epoch_manager = MockEpochManager::new_with_validators(store.clone(), vs, epoch_length); + let mut genesis = Genesis::test_sharded( + vec!["test1".parse::().unwrap()], + 1, + vec![1; num_shards as usize], + ); + genesis.config.epoch_length = epoch_length; + let tempdir = tempfile::tempdir().unwrap(); + initialize_genesis_state(store.clone(), &genesis, Some(tempdir.path())); + let chain_genesis = ChainGenesis::new(&genesis.config); + let epoch_manager = EpochManager::new_arc_handle(store.clone(), &genesis.config); let shard_tracker = ShardTracker::new_empty(epoch_manager.clone()); - let runtime = KeyValueRuntime::new(store, epoch_manager.as_ref()); + let runtime = + NightshadeRuntime::test(tempdir.path(), store, &genesis.config, epoch_manager.clone()); Chain::new( epoch_manager, shard_tracker, diff --git a/chain/chain/src/tests/garbage_collection.rs b/chain/chain/src/tests/garbage_collection.rs index 28a9b888a11..2fa6fdcaebb 100644 --- a/chain/chain/src/tests/garbage_collection.rs +++ b/chain/chain/src/tests/garbage_collection.rs @@ -1,3 +1,4 @@ +use near_epoch_manager::types::BlockHeaderInfo; use rand::Rng; use std::sync::Arc; @@ -10,7 +11,7 @@ use crate::test_utils::{ use crate::types::Tip; use crate::{ChainStoreAccess, StoreValidator}; -use near_chain_configs::{GCConfig, GenesisConfig}; +use near_chain_configs::{GCConfig, GenesisConfig, DEFAULT_GC_NUM_EPOCHS_TO_KEEP}; use near_epoch_manager::EpochManagerAdapter; use near_primitives::block::Block; use near_primitives::epoch_manager::block_info::BlockInfo; @@ -32,6 +33,7 @@ fn do_fork( states: &mut Vec<(Block, Vec, Vec, Option>)>>)>, max_changes: usize, verbose: bool, + final_block_height: Option, ) { if verbose { println!( @@ -87,6 +89,7 @@ fn do_fork( } let head = chain.head().unwrap(); + let epoch_manager = chain.epoch_manager.clone(); let mut store_update = chain.mut_chain_store().store_update(); if i == 0 { store_update.save_block_merkle_tree(*prev_block.hash(), PartialMerkleTree::default()); @@ -98,6 +101,12 @@ fn do_fork( if head.height < tip.height { store_update.save_head(&tip).unwrap(); } + let final_height = + final_block_height.unwrap_or_else(|| block.header().height().saturating_sub(2)); + let epoch_manager_update = epoch_manager + .add_validator_proposals(BlockHeaderInfo::new(block.header(), final_height)) + .unwrap(); + store_update.merge(epoch_manager_update); let mut trie_changes_shards = Vec::new(); for shard_id in 0..num_shards { @@ -160,6 +169,7 @@ fn gc_fork_common(simple_chains: Vec, max_changes: usize) { vec![Vec::new(); num_shards as usize], )); + let mut final_height = None; for simple_chain in simple_chains.iter() { let (source_block1, state_root1, _) = states1[simple_chain.from as usize].clone(); do_fork( @@ -171,7 +181,11 @@ fn gc_fork_common(simple_chains: Vec, max_changes: usize) { &mut states1, max_changes, verbose, + final_height, ); + if final_height.is_none() { + final_height = states1.last().unwrap().0.header().height().checked_sub(2); + } } // GC execution @@ -182,7 +196,7 @@ fn gc_fork_common(simple_chains: Vec, max_changes: usize) { let tries2 = get_chain_with_num_shards(num_shards).runtime_adapter.get_tries(); // Find gc_height - let mut gc_height = simple_chains[0].length - 51; + let mut gc_height = simple_chains[0].length - 41; for (i, simple_chain) in simple_chains.iter().enumerate() { if (!simple_chain.is_removed) && gc_height > simple_chain.from && i > 0 { gc_height = simple_chain.from @@ -359,7 +373,7 @@ fn test_gc_remove_fork_fail_often() { // Creates simple shorter fork and NOT GCs it fn test_gc_not_remove_fork_common(max_changes_limit: usize) { for max_changes in 1..=max_changes_limit { - for fork_length in 41..=50 { + for fork_length in 51..=60 { let chains = vec![ SimpleChain { from: 0, length: 101, is_removed: false }, SimpleChain { from: 10, length: fork_length, is_removed: false }, @@ -409,14 +423,14 @@ fn test_gc_forks_from_genesis() { ]; gc_fork_common(chains, 1); } - for fork_length in 45..=50 { + for fork_length in 45..=60 { let chains = vec![ SimpleChain { from: 0, length: 101, is_removed: false }, SimpleChain { from: 0, length: fork_length, is_removed: true }, ]; gc_fork_common(chains, 1); } - for fork_length in 51..=55 { + for fork_length in 61..=65 { let chains = vec![ SimpleChain { from: 0, length: 101, is_removed: false }, SimpleChain { from: 0, length: fork_length, is_removed: false }, @@ -426,9 +440,9 @@ fn test_gc_forks_from_genesis() { for fork_length in 0..=10 { let chains = vec![ SimpleChain { from: 0, length: 101, is_removed: false }, - SimpleChain { from: 0, length: 51 + fork_length, is_removed: false }, + SimpleChain { from: 0, length: 61 + fork_length, is_removed: false }, SimpleChain { from: 0, length: fork_length, is_removed: true }, - SimpleChain { from: 0, length: 50 - fork_length, is_removed: true }, + SimpleChain { from: 0, length: 60 - fork_length, is_removed: true }, ]; gc_fork_common(chains, 1); } @@ -441,7 +455,7 @@ fn test_gc_overlap() { SimpleChain { from: 0, length: 101, is_removed: false }, SimpleChain { from: 10, length: 70, is_removed: false }, SimpleChain { from: 20, length: 25, is_removed: true }, - SimpleChain { from: 30, length: 30, is_removed: false }, + SimpleChain { from: 30, length: 31, is_removed: false }, SimpleChain { from: 40, length: 1, is_removed: true }, ]; gc_fork_common(chains, max_changes); @@ -454,7 +468,7 @@ fn test_gc_boundaries_common(max_changes_limit: usize) { for len in 1..=5 { let chains = vec![ SimpleChain { from: 0, length: 101, is_removed: false }, - SimpleChain { from: i, length: len, is_removed: i + len <= 50 }, + SimpleChain { from: i, length: len, is_removed: i + len <= 60 }, ]; gc_fork_common(chains, max_changes); } @@ -484,7 +498,7 @@ fn test_gc_random_common(runs: u64) { chains.push(SimpleChain { from, length: len, - is_removed: from + len < canonical_len - 50, + is_removed: from + len < canonical_len - 40, }); gc_fork_common(chains.clone(), rng.gen_range(0..20) + 1); } @@ -505,38 +519,26 @@ fn test_gc_random_large() { #[test] fn test_gc_pine_small() { let mut chains = vec![SimpleChain { from: 0, length: 101, is_removed: false }]; - for i in 1..50 { - chains.push(SimpleChain { from: i, length: 1, is_removed: true }); - } - for i in 50..100 { - chains.push(SimpleChain { from: i, length: 1, is_removed: false }); + for i in 1..100 { + chains.push(SimpleChain { from: i, length: 1, is_removed: i < 60 }); } gc_fork_common(chains, 3); let mut chains = vec![SimpleChain { from: 0, length: 101, is_removed: false }]; - for i in 1..49 { - chains.push(SimpleChain { from: i, length: 2, is_removed: true }); - } - for i in 49..99 { - chains.push(SimpleChain { from: i, length: 2, is_removed: false }); + for i in 1..99 { + chains.push(SimpleChain { from: i, length: 2, is_removed: i < 59 }); } gc_fork_common(chains, 2); let mut chains = vec![SimpleChain { from: 0, length: 101, is_removed: false }]; - for i in 1..48 { - chains.push(SimpleChain { from: i, length: 3, is_removed: true }); - } - for i in 48..98 { - chains.push(SimpleChain { from: i, length: 3, is_removed: false }); + for i in 1..98 { + chains.push(SimpleChain { from: i, length: 3, is_removed: i < 58 }); } gc_fork_common(chains, 1); let mut chains = vec![SimpleChain { from: 0, length: 101, is_removed: false }]; - for i in 1..40 { - chains.push(SimpleChain { from: i, length: 11, is_removed: true }); - } - for i in 40..90 { - chains.push(SimpleChain { from: i, length: 11, is_removed: false }); + for i in 1..90 { + chains.push(SimpleChain { from: i, length: 11, is_removed: i < 50 }); } gc_fork_common(chains, 1); } @@ -568,11 +570,8 @@ fn test_gc_pine() { fn test_gc_star_common(max_changes_limit: usize) { for max_changes in 1..=max_changes_limit { let mut chains = vec![SimpleChain { from: 0, length: 101, is_removed: false }]; - for i in 1..=17 { - chains.push(SimpleChain { from: 33, length: i, is_removed: true }); - } - for i in 18..67 { - chains.push(SimpleChain { from: 33, length: i, is_removed: false }); + for i in 1..=67 { + chains.push(SimpleChain { from: 33, length: i, is_removed: i <= 27 }); } gc_fork_common(chains, max_changes); } @@ -605,37 +604,38 @@ fn test_fork_far_away_from_epoch_end() { SimpleChain { from: 0, length: 5, is_removed: false }, SimpleChain { from: 5, length: 2, is_removed: true }, // We want the chain to end up exactly at the new epoch start. - SimpleChain { from: 5, length: 6 * epoch_length - 5 + 1, is_removed: false }, + SimpleChain { from: 5, length: 5 * epoch_length - 5 + 1, is_removed: false }, ]; let num_shards = 1; - let mut chain1 = get_chain_with_epoch_length_and_num_shards(epoch_length, num_shards); - let tries1 = chain1.runtime_adapter.get_tries(); - let genesis1 = chain1.get_block_by_height(0).unwrap(); - let mut states1 = vec![( - genesis1, + let mut chain = get_chain_with_epoch_length_and_num_shards(epoch_length, num_shards); + let tries = chain.runtime_adapter.get_tries(); + let genesis = chain.get_block_by_height(0).unwrap(); + let mut states = vec![( + genesis, vec![Trie::EMPTY_ROOT; num_shards as usize], vec![Vec::new(); num_shards as usize], )]; - for simple_chain in simple_chains.iter() { - let (source_block1, state_root1, _) = states1[simple_chain.from as usize].clone(); + for (simple_chain, final_height) in simple_chains.iter().zip([None, Some(3), None]) { + let (source_block, state_root, _) = states[simple_chain.from as usize].clone(); do_fork( - source_block1.clone(), - state_root1, - tries1.clone(), - &mut chain1, + source_block.clone(), + state_root, + tries.clone(), + &mut chain, simple_chain.length, - &mut states1, + &mut states, max_changes, verbose, + final_height, ); } // GC execution - chain1 + chain .clear_data( - tries1.clone(), + tries.clone(), &GCConfig { gc_blocks_limit: 100, gc_fork_clean_step: fork_clean_step, @@ -647,9 +647,9 @@ fn test_fork_far_away_from_epoch_end() { // The run above would clear just the first 5 blocks from the beginning, but shouldn't clear any forks // yet - as fork_tail only clears the 'last' 1k blocks. for i in 1..5 { - let (block, _, _) = states1[i as usize].clone(); + let (block, _, _) = states[i as usize].clone(); assert_eq!( - chain1.block_exists(block.hash()).unwrap(), + chain.block_exists(block.hash()).unwrap(), false, "Block {:?}@{} should have been removed.", block.hash(), @@ -657,10 +657,10 @@ fn test_fork_far_away_from_epoch_end() { ); } // But blocks from the fork - shouldn't be removed yet. - for i in 6..7 { - let (block, _, _) = states1[i as usize].clone(); + for i in 5..7 { + let (block, _, _) = states[i as usize].clone(); assert_eq!( - chain1.block_exists(block.hash()).unwrap(), + chain.block_exists(block.hash()).unwrap(), true, "Block {:?}@{} should NOT be removed.", block.hash(), @@ -669,26 +669,27 @@ fn test_fork_far_away_from_epoch_end() { } // Now let's add one more block - and now the fork (and the rest) should be successfully removed. { - let (source_block1, state_root1, _) = states1.last().unwrap().clone(); + let (source_block, state_root, _) = states.last().unwrap().clone(); do_fork( - source_block1, - state_root1, - tries1.clone(), - &mut chain1, + source_block, + state_root, + tries.clone(), + &mut chain, 1, - &mut states1, + &mut states, max_changes, verbose, + None, ); } - chain1 - .clear_data(tries1, &GCConfig { gc_blocks_limit: 100, ..GCConfig::default() }) + chain + .clear_data(tries, &GCConfig { gc_blocks_limit: 100, ..GCConfig::default() }) .expect("Clear data failed"); // And now all these blocks should be safely removed. for i in 6..50 { - let (block, _, _) = states1[i as usize].clone(); + let (block, _, _) = states[i as usize].clone(); assert_eq!( - chain1.block_exists(block.hash()).unwrap(), + chain.block_exists(block.hash()).unwrap(), false, "Block {:?}@{} should have been removed.", block.hash(), @@ -701,44 +702,35 @@ fn test_fork_far_away_from_epoch_end() { /// collected while the blocks that are ahead of it should not. #[test] fn test_clear_old_data() { + let max_height = 14usize; let mut chain = get_chain_with_epoch_length(1); let epoch_manager = chain.epoch_manager.clone(); let genesis = chain.get_block_by_height(0).unwrap(); let signer = Arc::new(create_test_signer("test1")); let mut prev_block = genesis; let mut blocks = vec![prev_block.clone()]; - for i in 1..15 { + for i in 1..=max_height { add_block( &mut chain, epoch_manager.as_ref(), &mut prev_block, &mut blocks, signer.clone(), - i, + i as BlockHeight, ); } let trie = chain.runtime_adapter.get_tries(); chain.clear_data(trie, &GCConfig { gc_blocks_limit: 100, ..GCConfig::default() }).unwrap(); - // epoch didn't change so no data is garbage collected. - for i in 0..15 { + for i in 0..=max_height { println!("height = {} hash = {}", i, blocks[i].hash()); - if i < 8 { - assert!(chain.get_block(blocks[i].hash()).is_err()); - assert!(chain - .mut_chain_store() - .get_all_block_hashes_by_height(i as BlockHeight) - .unwrap() - .is_empty()); - } else { - assert!(chain.get_block(blocks[i].hash()).is_ok()); - assert!(!chain - .mut_chain_store() - .get_all_block_hashes_by_height(i as BlockHeight) - .unwrap() - .is_empty()); - } + let expected_removed = i < max_height - DEFAULT_GC_NUM_EPOCHS_TO_KEEP as usize; + let get_block_result = chain.get_block(blocks[i].hash()); + let blocks_by_heigh = + chain.mut_chain_store().get_all_block_hashes_by_height(i as BlockHeight).unwrap(); + assert_eq!(get_block_result.is_err(), expected_removed); + assert_eq!(blocks_by_heigh.is_empty(), expected_removed); } } @@ -785,6 +777,13 @@ fn add_block( .height_to_hashes .insert(height, Some(*block.header().hash())); store_update.save_next_block_hash(prev_block.hash(), *block.hash()); + let epoch_manager_update = epoch_manager + .add_validator_proposals(BlockHeaderInfo::new( + block.header(), + block.header().height().saturating_sub(2), + )) + .unwrap(); + store_update.merge(epoch_manager_update); store_update.commit().unwrap(); *prev_block = block.clone(); }