diff --git a/ledger/src/ancestor_iterator.rs b/ledger/src/ancestor_iterator.rs index 16d4abe3249777..e991fd04635fd5 100644 --- a/ledger/src/ancestor_iterator.rs +++ b/ledger/src/ancestor_iterator.rs @@ -79,8 +79,8 @@ mod tests { use std::{collections::HashMap, path::Path}; use trees::tr; - fn setup_forks(blockstore_path: &Path) -> Blockstore { - let blockstore = Blockstore::open(blockstore_path).unwrap(); + fn setup_forks(ledger_path: &Path) -> Blockstore { + let blockstore = Blockstore::open(ledger_path).unwrap(); /* Build fork structure: @@ -102,28 +102,25 @@ mod tests { #[test] fn test_ancestor_iterator() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = setup_forks(&blockstore_path); - - // Test correctness - assert!(AncestorIterator::new(0, &blockstore).next().is_none()); - assert_eq!( - AncestorIterator::new(4, &blockstore).collect::>(), - vec![1, 0] - ); - assert_eq!( - AncestorIterator::new(3, &blockstore).collect::>(), - vec![2, 1, 0] - ); - } - Blockstore::destroy(&blockstore_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = setup_forks(ledger_path.path()); + + // Test correctness + assert!(AncestorIterator::new(0, &blockstore).next().is_none()); + assert_eq!( + AncestorIterator::new(4, &blockstore).collect::>(), + vec![1, 0] + ); + assert_eq!( + AncestorIterator::new(3, &blockstore).collect::>(), + vec![2, 1, 0] + ); } #[test] fn test_ancestor_iterator_inclusive() { - let blockstore_path = get_tmp_ledger_path!(); - let blockstore = Blockstore::open(&blockstore_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); let (shreds, _) = make_slot_entries(0, 0, 42); blockstore.insert_shreds(shreds, None, false).unwrap(); @@ -151,39 +148,36 @@ mod tests { #[test] fn test_ancestor_iterator_with_hash() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = setup_forks(&blockstore_path); - - // Insert frozen hashes - let mut slot_to_bank_hash = HashMap::new(); - for slot in 0..=4 { - let bank_hash = Hash::new_unique(); - slot_to_bank_hash.insert(slot, bank_hash); - blockstore.insert_bank_hash(slot, bank_hash, false); - } - - // Test correctness - assert!( - AncestorIteratorWithHash::from(AncestorIterator::new(0, &blockstore)) - .next() - .is_none() - ); - assert_eq!( - AncestorIteratorWithHash::from(AncestorIterator::new(4, &blockstore)) - .collect::>(), - vec![(1, slot_to_bank_hash[&1]), (0, slot_to_bank_hash[&0])] - ); - assert_eq!( - AncestorIteratorWithHash::from(AncestorIterator::new(3, &blockstore)) - .collect::>(), - vec![ - (2, slot_to_bank_hash[&2]), - (1, slot_to_bank_hash[&1]), - (0, slot_to_bank_hash[&0]) - ] - ); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = setup_forks(ledger_path.path()); + + // Insert frozen hashes + let mut slot_to_bank_hash = HashMap::new(); + for slot in 0..=4 { + let bank_hash = Hash::new_unique(); + slot_to_bank_hash.insert(slot, bank_hash); + blockstore.insert_bank_hash(slot, bank_hash, false); } - Blockstore::destroy(&blockstore_path).unwrap(); + + // Test correctness + assert!( + AncestorIteratorWithHash::from(AncestorIterator::new(0, &blockstore)) + .next() + .is_none() + ); + assert_eq!( + AncestorIteratorWithHash::from(AncestorIterator::new(4, &blockstore)) + .collect::>(), + vec![(1, slot_to_bank_hash[&1]), (0, slot_to_bank_hash[&0])] + ); + assert_eq!( + AncestorIteratorWithHash::from(AncestorIterator::new(3, &blockstore)) + .collect::>(), + vec![ + (2, slot_to_bank_hash[&2]), + (1, slot_to_bank_hash[&1]), + (0, slot_to_bank_hash[&0]) + ] + ); } } diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 120bb855dc9f47..09d96fa2053181 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -59,7 +59,7 @@ use { }, time::Instant, }, - tempfile::TempDir, + tempfile::{Builder, TempDir}, thiserror::Error, trees::{Tree, TreeWalk}, }; @@ -3809,9 +3809,16 @@ macro_rules! get_tmp_ledger_path_auto_delete { } pub fn get_ledger_path_from_name_auto_delete(name: &str) -> TempDir { - let path = get_ledger_path_from_name(name); + let mut path = get_ledger_path_from_name(name); + // path is a directory so .file_name() returns the last component of the path + let last = path.file_name().unwrap().to_str().unwrap().to_string(); + path.pop(); fs::create_dir_all(&path).unwrap(); - TempDir::new_in(path).unwrap() + Builder::new() + .prefix(&last) + .rand_bytes(0) + .tempdir_in(path) + .unwrap() } pub fn get_ledger_path_from_name(name: &str) -> PathBuf { @@ -3844,6 +3851,17 @@ macro_rules! create_new_tmp_ledger { }; } +#[macro_export] +macro_rules! create_new_tmp_ledger_auto_delete { + ($genesis_config:expr) => { + $crate::blockstore::create_new_ledger_from_name_auto_delete( + $crate::tmp_ledger_name!(), + $genesis_config, + $crate::blockstore_db::AccessType::PrimaryOnly, + ) + }; +} + pub fn verify_shred_slots(slot: Slot, parent_slot: Slot, last_root: Slot) -> bool { if !is_valid_write_to_slot_0(slot, parent_slot, last_root) { // Check that the parent_slot < slot @@ -3871,9 +3889,23 @@ pub fn create_new_ledger_from_name( genesis_config: &GenesisConfig, access_type: AccessType, ) -> (PathBuf, Hash) { - let ledger_path = get_ledger_path_from_name(name); + let (ledger_path, blockhash) = + create_new_ledger_from_name_auto_delete(name, genesis_config, access_type); + (ledger_path.into_path(), blockhash) +} + +// Same as `create_new_ledger()` but use a temporary ledger name based on the provided `name` +// +// Note: like `create_new_ledger` the returned ledger will have slot 0 full of ticks (and only +// ticks) +pub fn create_new_ledger_from_name_auto_delete( + name: &str, + genesis_config: &GenesisConfig, + access_type: AccessType, +) -> (TempDir, Hash) { + let ledger_path = get_ledger_path_from_name_auto_delete(name); let blockhash = create_new_ledger( - &ledger_path, + ledger_path.path(), genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, access_type, @@ -4043,19 +4075,16 @@ pub mod tests { #[test] fn test_create_new_ledger() { + solana_logger::setup(); let mint_total = 1_000_000_000_000; let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(mint_total); - let (ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_config); - let ledger = Blockstore::open(&ledger_path).unwrap(); + let (ledger_path, _blockhash) = create_new_tmp_ledger_auto_delete!(&genesis_config); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); //FINDME let ticks = create_ticks(genesis_config.ticks_per_slot, 0, genesis_config.hash()); - let entries = ledger.get_slot_entries(0, 0).unwrap(); + let entries = blockstore.get_slot_entries(0, 0).unwrap(); assert_eq!(ticks, entries); - - // Destroying database without closing it first is undefined behavior - drop(ledger); - Blockstore::destroy(&ledger_path).expect("Expected successful database destruction"); } #[test] @@ -4066,17 +4095,17 @@ pub mod tests { let (mut shreds, _) = make_slot_entries(0, 0, num_entries); - let ledger_path = get_tmp_ledger_path!(); - let ledger = Blockstore::open(&ledger_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); // Insert last shred, test we can retrieve it let last_shred = shreds.pop().unwrap(); assert!(last_shred.index() > 0); - ledger + blockstore .insert_shreds(vec![last_shred.clone()], None, false) .unwrap(); - let serialized_shred = ledger + let serialized_shred = blockstore .data_shred_cf .get_bytes((0, last_shred.index() as u64)) .unwrap() @@ -4084,119 +4113,114 @@ pub mod tests { let deserialized_shred = Shred::new_from_serialized_shred(serialized_shred).unwrap(); assert_eq!(last_shred, deserialized_shred); - // Destroying database without closing it first is undefined behavior - drop(ledger); - Blockstore::destroy(&ledger_path).expect("Expected successful database destruction"); } #[test] fn test_write_entries() { solana_logger::setup(); - let ledger_path = get_tmp_ledger_path!(); - { - let ticks_per_slot = 10; - let num_slots = 10; - let ledger = Blockstore::open(&ledger_path).unwrap(); - let mut ticks = vec![]; - //let mut shreds_per_slot = 0 as u64; - let mut shreds_per_slot = vec![]; - - for i in 0..num_slots { - let mut new_ticks = create_ticks(ticks_per_slot, 0, Hash::default()); - let num_shreds = ledger - .write_entries( - i, - 0, - 0, - ticks_per_slot, - Some(i.saturating_sub(1)), - true, - &Arc::new(Keypair::new()), - new_ticks.clone(), - 0, - ) - .unwrap() as u64; - shreds_per_slot.push(num_shreds); - ticks.append(&mut new_ticks); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + let ticks_per_slot = 10; + let num_slots = 10; + let mut ticks = vec![]; + //let mut shreds_per_slot = 0 as u64; + let mut shreds_per_slot = vec![]; + + for i in 0..num_slots { + let mut new_ticks = create_ticks(ticks_per_slot, 0, Hash::default()); + let num_shreds = blockstore + .write_entries( + i, + 0, + 0, + ticks_per_slot, + Some(i.saturating_sub(1)), + true, + &Arc::new(Keypair::new()), + new_ticks.clone(), + 0, + ) + .unwrap() as u64; + shreds_per_slot.push(num_shreds); + ticks.append(&mut new_ticks); + } + + for i in 0..num_slots { + let meta = blockstore.meta(i).unwrap().unwrap(); + let num_shreds = shreds_per_slot[i as usize]; + assert_eq!(meta.consumed, num_shreds); + assert_eq!(meta.received, num_shreds); + assert_eq!(meta.last_index, num_shreds - 1); + if i == num_slots - 1 { + assert!(meta.next_slots.is_empty()); + } else { + assert_eq!(meta.next_slots, vec![i + 1]); + } + if i == 0 { + assert_eq!(meta.parent_slot, 0); + } else { + assert_eq!(meta.parent_slot, i - 1); } - for i in 0..num_slots { - let meta = ledger.meta(i).unwrap().unwrap(); - let num_shreds = shreds_per_slot[i as usize]; - assert_eq!(meta.consumed, num_shreds); - assert_eq!(meta.received, num_shreds); - assert_eq!(meta.last_index, num_shreds - 1); - if i == num_slots - 1 { - assert!(meta.next_slots.is_empty()); - } else { - assert_eq!(meta.next_slots, vec![i + 1]); - } - if i == 0 { - assert_eq!(meta.parent_slot, 0); - } else { - assert_eq!(meta.parent_slot, i - 1); - } + assert_eq!( + &ticks[(i * ticks_per_slot) as usize..((i + 1) * ticks_per_slot) as usize], + &blockstore.get_slot_entries(i, 0).unwrap()[..] + ); + } - assert_eq!( - &ticks[(i * ticks_per_slot) as usize..((i + 1) * ticks_per_slot) as usize], - &ledger.get_slot_entries(i, 0).unwrap()[..] - ); - } + /* + // Simulate writing to the end of a slot with existing ticks + blockstore + .write_entries( + num_slots, + ticks_per_slot - 1, + ticks_per_slot - 2, + ticks_per_slot, + &ticks[0..2], + ) + .unwrap(); + + let meta = blockstore.meta(num_slots).unwrap().unwrap(); + assert_eq!(meta.consumed, 0); + // received shred was ticks_per_slot - 2, so received should be ticks_per_slot - 2 + 1 + assert_eq!(meta.received, ticks_per_slot - 1); + // last shred index ticks_per_slot - 2 because that's the shred that made tick_height == ticks_per_slot + // for the slot + assert_eq!(meta.last_index, ticks_per_slot - 2); + assert_eq!(meta.parent_slot, num_slots - 1); + assert_eq!(meta.next_slots, vec![num_slots + 1]); + assert_eq!( + &ticks[0..1], + &blockstore + .get_slot_entries(num_slots, ticks_per_slot - 2) + .unwrap()[..] + ); - /* - // Simulate writing to the end of a slot with existing ticks - ledger - .write_entries( - num_slots, - ticks_per_slot - 1, - ticks_per_slot - 2, - ticks_per_slot, - &ticks[0..2], - ) - .unwrap(); - - let meta = ledger.meta(num_slots).unwrap().unwrap(); - assert_eq!(meta.consumed, 0); - // received shred was ticks_per_slot - 2, so received should be ticks_per_slot - 2 + 1 - assert_eq!(meta.received, ticks_per_slot - 1); - // last shred index ticks_per_slot - 2 because that's the shred that made tick_height == ticks_per_slot - // for the slot - assert_eq!(meta.last_index, ticks_per_slot - 2); - assert_eq!(meta.parent_slot, num_slots - 1); - assert_eq!(meta.next_slots, vec![num_slots + 1]); - assert_eq!( - &ticks[0..1], - &ledger - .get_slot_entries(num_slots, ticks_per_slot - 2) - .unwrap()[..] - ); - - // We wrote two entries, the second should spill into slot num_slots + 1 - let meta = ledger.meta(num_slots + 1).unwrap().unwrap(); - assert_eq!(meta.consumed, 1); - assert_eq!(meta.received, 1); - assert_eq!(meta.last_index, std::u64::MAX); - assert_eq!(meta.parent_slot, num_slots); - assert!(meta.next_slots.is_empty()); - - assert_eq!( - &ticks[1..2], - &ledger.get_slot_entries(num_slots + 1, 0).unwrap()[..] - ); - */ - } - Blockstore::destroy(&ledger_path).expect("Expected successful database destruction"); + // We wrote two entries, the second should spill into slot num_slots + 1 + let meta = blockstore.meta(num_slots + 1).unwrap().unwrap(); + assert_eq!(meta.consumed, 1); + assert_eq!(meta.received, 1); + assert_eq!(meta.last_index, std::u64::MAX); + assert_eq!(meta.parent_slot, num_slots); + assert!(meta.next_slots.is_empty()); + + assert_eq!( + &ticks[1..2], + &blockstore.get_slot_entries(num_slots + 1, 0).unwrap()[..] + ); + */ } #[test] fn test_put_get_simple() { - let ledger_path = get_tmp_ledger_path!(); - let ledger = Blockstore::open(&ledger_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); // Test meta column family let meta = SlotMeta::new(0, 1); - ledger.meta_cf.put(0, &meta).unwrap(); - let result = ledger + blockstore.meta_cf.put(0, &meta).unwrap(); + let result = blockstore .meta_cf .get(0) .unwrap() @@ -4207,12 +4231,12 @@ pub mod tests { // Test erasure column family let erasure = vec![1u8; 16]; let erasure_key = (0, 0); - ledger + blockstore .code_shred_cf .put_bytes(erasure_key, &erasure) .unwrap(); - let result = ledger + let result = blockstore .code_shred_cf .get_bytes(erasure_key) .unwrap() @@ -4223,19 +4247,15 @@ pub mod tests { // Test data column family let data = vec![2u8; 16]; let data_key = (0, 0); - ledger.data_shred_cf.put_bytes(data_key, &data).unwrap(); + blockstore.data_shred_cf.put_bytes(data_key, &data).unwrap(); - let result = ledger + let result = blockstore .data_shred_cf .get_bytes(data_key) .unwrap() .expect("Expected data object to exist"); assert_eq!(result, data); - - // Destroying database without closing it first is undefined behavior - drop(ledger); - Blockstore::destroy(&ledger_path).expect("Expected successful database destruction"); } #[test] @@ -4245,15 +4265,15 @@ pub mod tests { let num_shreds = shreds.len() as u64; let shred_bufs: Vec<_> = shreds.iter().map(|shred| shred.payload.clone()).collect(); - let ledger_path = get_tmp_ledger_path!(); - let ledger = Blockstore::open(&ledger_path).unwrap(); - ledger.insert_shreds(shreds, None, false).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + blockstore.insert_shreds(shreds, None, false).unwrap(); let mut buf = [0; 4096]; - let (_, bytes) = ledger.get_data_shreds(slot, 0, 1, &mut buf).unwrap(); + let (_, bytes) = blockstore.get_data_shreds(slot, 0, 1, &mut buf).unwrap(); assert_eq!(buf[..bytes], shred_bufs[0][..bytes]); - let (last_index, bytes2) = ledger.get_data_shreds(slot, 0, 2, &mut buf).unwrap(); + let (last_index, bytes2) = blockstore.get_data_shreds(slot, 0, 2, &mut buf).unwrap(); assert_eq!(last_index, 1); assert!(bytes2 > bytes); { @@ -4266,17 +4286,17 @@ pub mod tests { // buf size part-way into shred[1], should just return shred[0] let mut buf = vec![0; bytes + 1]; - let (last_index, bytes3) = ledger.get_data_shreds(slot, 0, 2, &mut buf).unwrap(); + let (last_index, bytes3) = blockstore.get_data_shreds(slot, 0, 2, &mut buf).unwrap(); assert_eq!(last_index, 0); assert_eq!(bytes3, bytes); let mut buf = vec![0; bytes2 - 1]; - let (last_index, bytes4) = ledger.get_data_shreds(slot, 0, 2, &mut buf).unwrap(); + let (last_index, bytes4) = blockstore.get_data_shreds(slot, 0, 2, &mut buf).unwrap(); assert_eq!(last_index, 0); assert_eq!(bytes4, bytes); let mut buf = vec![0; bytes * 2]; - let (last_index, bytes6) = ledger + let (last_index, bytes6) = blockstore .get_data_shreds(slot, num_shreds - 1, num_shreds, &mut buf) .unwrap(); assert_eq!(last_index, num_shreds - 1); @@ -4287,15 +4307,11 @@ pub mod tests { } // Read out of range - let (last_index, bytes6) = ledger + let (last_index, bytes6) = blockstore .get_data_shreds(slot, num_shreds, num_shreds + 2, &mut buf) .unwrap(); assert_eq!(last_index, 0); assert_eq!(bytes6, 0); - - // Destroying database without closing it first is undefined behavior - drop(ledger); - Blockstore::destroy(&ledger_path).expect("Expected successful database destruction"); } #[test] @@ -4303,21 +4319,21 @@ pub mod tests { let slot = 1; let (shreds, _) = make_slot_entries(slot, 0, 100); - let ledger_path = get_tmp_ledger_path!(); - let ledger = Blockstore::open(&ledger_path).unwrap(); - ledger.insert_shreds(shreds, None, false).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + blockstore.insert_shreds(shreds, None, false).unwrap(); let mut buf = [0; 4096]; - assert!(ledger.get_data_shreds(slot, 0, 1, &mut buf).is_ok()); + assert!(blockstore.get_data_shreds(slot, 0, 1, &mut buf).is_ok()); let max_purge_slot = 1; - ledger + blockstore .run_purge(0, max_purge_slot, PurgeType::PrimaryIndex) .unwrap(); - *ledger.lowest_cleanup_slot.write().unwrap() = max_purge_slot; + *blockstore.lowest_cleanup_slot.write().unwrap() = max_purge_slot; let mut buf = [0; 4096]; - assert!(ledger.get_data_shreds(slot, 0, 1, &mut buf).is_err()); + assert!(blockstore.get_data_shreds(slot, 0, 1, &mut buf).is_err()); } #[test] @@ -4329,29 +4345,31 @@ pub mod tests { let (mut shreds, entries) = make_slot_entries(0, 0, num_entries); let num_shreds = shreds.len() as u64; - let ledger_path = get_tmp_ledger_path!(); - let ledger = Blockstore::open(&ledger_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); // Insert last shred, we're missing the other shreds, so no consecutive // shreds starting from slot 0, index 0 should exist. assert!(shreds.len() > 1); let last_shred = shreds.pop().unwrap(); - ledger.insert_shreds(vec![last_shred], None, false).unwrap(); - assert!(ledger.get_slot_entries(0, 0).unwrap().is_empty()); + blockstore + .insert_shreds(vec![last_shred], None, false) + .unwrap(); + assert!(blockstore.get_slot_entries(0, 0).unwrap().is_empty()); - let meta = ledger + let meta = blockstore .meta(0) .unwrap() .expect("Expected new metadata object to be created"); assert!(meta.consumed == 0 && meta.received == num_shreds); // Insert the other shreds, check for consecutive returned entries - ledger.insert_shreds(shreds, None, false).unwrap(); - let result = ledger.get_slot_entries(0, 0).unwrap(); + blockstore.insert_shreds(shreds, None, false).unwrap(); + let result = blockstore.get_slot_entries(0, 0).unwrap(); assert_eq!(result, entries); - let meta = ledger + let meta = blockstore .meta(0) .unwrap() .expect("Expected new metadata object to exist"); @@ -4361,10 +4379,6 @@ pub mod tests { assert_eq!(meta.last_index, num_shreds - 1); assert!(meta.next_slots.is_empty()); assert!(meta.is_connected); - - // Destroying database without closing it first is undefined behavior - drop(ledger); - Blockstore::destroy(&ledger_path).expect("Expected successful database destruction"); } #[test] @@ -4374,16 +4388,16 @@ pub mod tests { let (mut shreds, entries) = make_slot_entries(0, 0, num_entries); let num_shreds = shreds.len() as u64; - let ledger_path = get_tmp_ledger_path!(); - let ledger = Blockstore::open(&ledger_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); // Insert shreds in reverse, check for consecutive returned shreds for i in (0..num_shreds).rev() { let shred = shreds.pop().unwrap(); - ledger.insert_shreds(vec![shred], None, false).unwrap(); - let result = ledger.get_slot_entries(0, 0).unwrap(); + blockstore.insert_shreds(vec![shred], None, false).unwrap(); + let result = blockstore.get_slot_entries(0, 0).unwrap(); - let meta = ledger + let meta = blockstore .meta(0) .unwrap() .expect("Expected metadata object to exist"); @@ -4397,84 +4411,75 @@ pub mod tests { assert!(meta.consumed == num_shreds as u64 && meta.received == num_shreds as u64); } } - - // Destroying database without closing it first is undefined behavior - drop(ledger); - Blockstore::destroy(&ledger_path).expect("Expected successful database destruction"); } #[test] fn test_insert_slots() { - test_insert_data_shreds_slots("test_insert_data_shreds_slots_single", false); - test_insert_data_shreds_slots("test_insert_data_shreds_slots_bulk", true); + test_insert_data_shreds_slots(false); + test_insert_data_shreds_slots(true); } /* #[test] pub fn test_iteration_order() { let slot = 0; - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - // Write entries - let num_entries = 8; - let entries = make_tiny_test_entries(num_entries); - let mut shreds = entries.to_single_entry_shreds(); + // Write entries + let num_entries = 8; + let entries = make_tiny_test_entries(num_entries); + let mut shreds = entries.to_single_entry_shreds(); - for (i, b) in shreds.iter_mut().enumerate() { - b.set_index(1 << (i * 8)); - b.set_slot(0); - } + for (i, b) in shreds.iter_mut().enumerate() { + b.set_index(1 << (i * 8)); + b.set_slot(0); + } - blockstore - .write_shreds(&shreds) - .expect("Expected successful write of shreds"); - - let mut db_iterator = blockstore - .db - .cursor::() - .expect("Expected to be able to open database iterator"); - - db_iterator.seek((slot, 1)); - - // Iterate through ledger - for i in 0..num_entries { - assert!(db_iterator.valid()); - let (_, current_index) = db_iterator.key().expect("Expected a valid key"); - assert_eq!(current_index, (1 as u64) << (i * 8)); - db_iterator.next(); - } + blockstore + .write_shreds(&shreds) + .expect("Expected successful write of shreds"); + + let mut db_iterator = blockstore + .db + .cursor::() + .expect("Expected to be able to open database iterator"); + + db_iterator.seek((slot, 1)); + + // Iterate through blockstore + for i in 0..num_entries { + assert!(db_iterator.valid()); + let (_, current_index) = db_iterator.key().expect("Expected a valid key"); + assert_eq!(current_index, (1 as u64) << (i * 8)); + db_iterator.next(); } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + } */ #[test] pub fn test_get_slot_entries1() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - let entries = create_ticks(8, 0, Hash::default()); - let shreds = entries_to_test_shreds(entries[0..4].to_vec(), 1, 0, false, 0); - blockstore - .insert_shreds(shreds, None, false) - .expect("Expected successful write of shreds"); - - let mut shreds1 = entries_to_test_shreds(entries[4..].to_vec(), 1, 0, false, 0); - for (i, b) in shreds1.iter_mut().enumerate() { - b.set_index(8 + i as u32); - } - blockstore - .insert_shreds(shreds1, None, false) - .expect("Expected successful write of shreds"); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + let entries = create_ticks(8, 0, Hash::default()); + let shreds = entries_to_test_shreds(entries[0..4].to_vec(), 1, 0, false, 0); + blockstore + .insert_shreds(shreds, None, false) + .expect("Expected successful write of shreds"); - assert_eq!( - blockstore.get_slot_entries(1, 0).unwrap()[2..4], - entries[2..4], - ); + let mut shreds1 = entries_to_test_shreds(entries[4..].to_vec(), 1, 0, false, 0); + for (i, b) in shreds1.iter_mut().enumerate() { + b.set_index(8 + i as u32); } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + blockstore + .insert_shreds(shreds1, None, false) + .expect("Expected successful write of shreds"); + + assert_eq!( + blockstore.get_slot_entries(1, 0).unwrap()[2..4], + entries[2..4], + ); } // This test seems to be unnecessary with introduction of data shreds. There are no @@ -4482,133 +4487,123 @@ pub mod tests { #[test] #[ignore] pub fn test_get_slot_entries2() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - // Write entries - let num_slots = 5_u64; - let mut index = 0; - for slot in 0..num_slots { - let entries = create_ticks(slot + 1, 0, Hash::default()); - let last_entry = entries.last().unwrap().clone(); - let mut shreds = - entries_to_test_shreds(entries, slot, slot.saturating_sub(1), false, 0); - for b in shreds.iter_mut() { - b.set_index(index); - b.set_slot(slot as u64); - index += 1; - } - blockstore - .insert_shreds(shreds, None, false) - .expect("Expected successful write of shreds"); - assert_eq!( - blockstore - .get_slot_entries(slot, u64::from(index - 1)) - .unwrap(), - vec![last_entry], - ); + // Write entries + let num_slots = 5_u64; + let mut index = 0; + for slot in 0..num_slots { + let entries = create_ticks(slot + 1, 0, Hash::default()); + let last_entry = entries.last().unwrap().clone(); + let mut shreds = + entries_to_test_shreds(entries, slot, slot.saturating_sub(1), false, 0); + for b in shreds.iter_mut() { + b.set_index(index); + b.set_slot(slot as u64); + index += 1; } + blockstore + .insert_shreds(shreds, None, false) + .expect("Expected successful write of shreds"); + assert_eq!( + blockstore + .get_slot_entries(slot, u64::from(index - 1)) + .unwrap(), + vec![last_entry], + ); } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] pub fn test_get_slot_entries3() { // Test inserting/fetching shreds which contain multiple entries per shred - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - let num_slots = 5_u64; - let shreds_per_slot = 5_u64; - let entry_serialized_size = - bincode::serialized_size(&create_ticks(1, 0, Hash::default())).unwrap(); - let entries_per_slot = - (shreds_per_slot * PACKET_DATA_SIZE as u64) / entry_serialized_size; + let ledger_path = get_tmp_ledger_path_auto_delete!(); - // Write entries - for slot in 0..num_slots { - let entries = create_ticks(entries_per_slot, 0, Hash::default()); - let shreds = - entries_to_test_shreds(entries.clone(), slot, slot.saturating_sub(1), false, 0); - assert!(shreds.len() as u64 >= shreds_per_slot); - blockstore - .insert_shreds(shreds, None, false) - .expect("Expected successful write of shreds"); - assert_eq!(blockstore.get_slot_entries(slot, 0).unwrap(), entries); - } + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + let num_slots = 5_u64; + let shreds_per_slot = 5_u64; + let entry_serialized_size = + bincode::serialized_size(&create_ticks(1, 0, Hash::default())).unwrap(); + let entries_per_slot = (shreds_per_slot * PACKET_DATA_SIZE as u64) / entry_serialized_size; + + // Write entries + for slot in 0..num_slots { + let entries = create_ticks(entries_per_slot, 0, Hash::default()); + let shreds = + entries_to_test_shreds(entries.clone(), slot, slot.saturating_sub(1), false, 0); + assert!(shreds.len() as u64 >= shreds_per_slot); + blockstore + .insert_shreds(shreds, None, false) + .expect("Expected successful write of shreds"); + assert_eq!(blockstore.get_slot_entries(slot, 0).unwrap(), entries); } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] pub fn test_insert_data_shreds_consecutive() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - // Create enough entries to ensure there are at least two shreds created - let min_entries = max_ticks_per_n_shreds(1, None) + 1; - for i in 0..4 { - let slot = i; - let parent_slot = if i == 0 { 0 } else { i - 1 }; - // Write entries - let num_entries = min_entries * (i + 1); - let (shreds, original_entries) = make_slot_entries(slot, parent_slot, num_entries); - - let num_shreds = shreds.len() as u64; - assert!(num_shreds > 1); - let mut even_shreds = vec![]; - let mut odd_shreds = vec![]; - - for (i, shred) in shreds.into_iter().enumerate() { - if i % 2 == 0 { - even_shreds.push(shred); - } else { - odd_shreds.push(shred); - } - } - - blockstore.insert_shreds(odd_shreds, None, false).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + // Create enough entries to ensure there are at least two shreds created + let min_entries = max_ticks_per_n_shreds(1, None) + 1; + for i in 0..4 { + let slot = i; + let parent_slot = if i == 0 { 0 } else { i - 1 }; + // Write entries + let num_entries = min_entries * (i + 1); + let (shreds, original_entries) = make_slot_entries(slot, parent_slot, num_entries); - assert_eq!(blockstore.get_slot_entries(slot, 0).unwrap(), vec![]); + let num_shreds = shreds.len() as u64; + assert!(num_shreds > 1); + let mut even_shreds = vec![]; + let mut odd_shreds = vec![]; - let meta = blockstore.meta(slot).unwrap().unwrap(); - if num_shreds % 2 == 0 { - assert_eq!(meta.received, num_shreds); - } else { - trace!("got here"); - assert_eq!(meta.received, num_shreds - 1); - } - assert_eq!(meta.consumed, 0); - if num_shreds % 2 == 0 { - assert_eq!(meta.last_index, num_shreds - 1); + for (i, shred) in shreds.into_iter().enumerate() { + if i % 2 == 0 { + even_shreds.push(shred); } else { - assert_eq!(meta.last_index, std::u64::MAX); + odd_shreds.push(shred); } + } - blockstore.insert_shreds(even_shreds, None, false).unwrap(); + blockstore.insert_shreds(odd_shreds, None, false).unwrap(); - assert_eq!( - blockstore.get_slot_entries(slot, 0).unwrap(), - original_entries, - ); + assert_eq!(blockstore.get_slot_entries(slot, 0).unwrap(), vec![]); - let meta = blockstore.meta(slot).unwrap().unwrap(); + let meta = blockstore.meta(slot).unwrap().unwrap(); + if num_shreds % 2 == 0 { assert_eq!(meta.received, num_shreds); - assert_eq!(meta.consumed, num_shreds); - assert_eq!(meta.parent_slot, parent_slot); + } else { + trace!("got here"); + assert_eq!(meta.received, num_shreds - 1); + } + assert_eq!(meta.consumed, 0); + if num_shreds % 2 == 0 { assert_eq!(meta.last_index, num_shreds - 1); + } else { + assert_eq!(meta.last_index, std::u64::MAX); } - } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + blockstore.insert_shreds(even_shreds, None, false).unwrap(); + + assert_eq!( + blockstore.get_slot_entries(slot, 0).unwrap(), + original_entries, + ); + + let meta = blockstore.meta(slot).unwrap().unwrap(); + assert_eq!(meta.received, num_shreds); + assert_eq!(meta.consumed, num_shreds); + assert_eq!(meta.parent_slot, parent_slot); + assert_eq!(meta.last_index, num_shreds - 1); + } } #[test] fn test_data_set_completed_on_insert() { - let ledger_path = get_tmp_ledger_path!(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); let BlockstoreSignals { blockstore, .. } = - Blockstore::open_with_signal(&ledger_path, None, true).unwrap(); + Blockstore::open_with_signal(ledger_path.path(), None, true).unwrap(); // Create enough entries to fill 2 shreds, only the later one is data complete let slot = 0; @@ -4643,14 +4638,14 @@ pub mod tests { #[test] pub fn test_new_shreds_signal() { - // Initialize ledger - let ledger_path = get_tmp_ledger_path!(); + // Initialize blockstore + let ledger_path = get_tmp_ledger_path_auto_delete!(); let BlockstoreSignals { - blockstore: ledger, + blockstore, ledger_signal_receiver: recvr, .. - } = Blockstore::open_with_signal(&ledger_path, None, true).unwrap(); - let ledger = Arc::new(ledger); + } = Blockstore::open_with_signal(ledger_path.path(), None, true).unwrap(); + //let blockstore = Arc::new(blockstore); let entries_per_slot = 50; // Create entries for slot 0 @@ -4659,20 +4654,20 @@ pub mod tests { // Insert second shred, but we're missing the first shred, so no consecutive // shreds starting from slot 0, index 0 should exist. - ledger + blockstore .insert_shreds(vec![shreds.remove(1)], None, false) .unwrap(); let timer = Duration::new(1, 0); assert!(recvr.recv_timeout(timer).is_err()); // Insert first shred, now we've made a consecutive block - ledger + blockstore .insert_shreds(vec![shreds.remove(0)], None, false) .unwrap(); // Wait to get notified of update, should only be one update assert!(recvr.recv_timeout(timer).is_ok()); assert!(recvr.try_recv().is_err()); // Insert the rest of the ticks - ledger.insert_shreds(shreds, None, false).unwrap(); + blockstore.insert_shreds(shreds, None, false).unwrap(); // Wait to get notified of update, should only be one update assert!(recvr.recv_timeout(timer).is_ok()); assert!(recvr.try_recv().is_err()); @@ -4691,7 +4686,7 @@ pub mod tests { } // Should be no updates, since no new chains from block 0 were formed - ledger.insert_shreds(shreds, None, false).unwrap(); + blockstore.insert_shreds(shreds, None, false).unwrap(); assert!(recvr.recv_timeout(timer).is_err()); // Insert a shred for each slot that doesn't make a consecutive block, we @@ -4704,7 +4699,7 @@ pub mod tests { }) .collect(); - ledger.insert_shreds(shreds, None, false).unwrap(); + blockstore.insert_shreds(shreds, None, false).unwrap(); assert!(recvr.recv_timeout(timer).is_err()); // For slots 1..num_slots/2, fill in the holes in one batch insertion, @@ -4712,29 +4707,29 @@ pub mod tests { let missing_shreds2 = missing_shreds .drain((num_slots / 2) as usize..) .collect_vec(); - ledger.insert_shreds(missing_shreds, None, false).unwrap(); + blockstore + .insert_shreds(missing_shreds, None, false) + .unwrap(); assert!(recvr.recv_timeout(timer).is_ok()); assert!(recvr.try_recv().is_err()); // Fill in the holes for each of the remaining slots, we should get a single update // for each - ledger.insert_shreds(missing_shreds2, None, false).unwrap(); - - // Destroying database without closing it first is undefined behavior - drop(ledger); - Blockstore::destroy(&ledger_path).expect("Expected successful database destruction"); + blockstore + .insert_shreds(missing_shreds2, None, false) + .unwrap(); } #[test] pub fn test_completed_shreds_signal() { - // Initialize ledger - let ledger_path = get_tmp_ledger_path!(); + // Initialize blockstore + let ledger_path = get_tmp_ledger_path_auto_delete!(); let BlockstoreSignals { - blockstore: ledger, + blockstore, completed_slots_receiver: recvr, .. - } = Blockstore::open_with_signal(&ledger_path, None, true).unwrap(); - let ledger = Arc::new(ledger); + } = Blockstore::open_with_signal(ledger_path.path(), None, true).unwrap(); + // let blockstore = Arc::new(blockstore); let entries_per_slot = 10; @@ -4743,24 +4738,24 @@ pub mod tests { let shred0 = shreds.remove(0); // Insert all but the first shred in the slot, should not be considered complete - ledger.insert_shreds(shreds, None, false).unwrap(); + blockstore.insert_shreds(shreds, None, false).unwrap(); assert!(recvr.try_recv().is_err()); // Insert first shred, slot should now be considered complete - ledger.insert_shreds(vec![shred0], None, false).unwrap(); + blockstore.insert_shreds(vec![shred0], None, false).unwrap(); assert_eq!(recvr.try_recv().unwrap(), vec![0]); } #[test] pub fn test_completed_shreds_signal_orphans() { - // Initialize ledger - let ledger_path = get_tmp_ledger_path!(); + // Initialize blockstore + let ledger_path = get_tmp_ledger_path_auto_delete!(); let BlockstoreSignals { - blockstore: ledger, + blockstore, completed_slots_receiver: recvr, .. - } = Blockstore::open_with_signal(&ledger_path, None, true).unwrap(); - let ledger = Arc::new(ledger); + } = Blockstore::open_with_signal(ledger_path.path(), None, true).unwrap(); + // let blockstore = Arc::new(blockstore); let entries_per_slot = 10; let slots = vec![2, 5, 10]; @@ -4774,22 +4769,24 @@ pub mod tests { // Insert all but the first shred in the slot, should not be considered complete let orphan_child0 = orphan_child.remove(0); - ledger.insert_shreds(orphan_child, None, false).unwrap(); + blockstore.insert_shreds(orphan_child, None, false).unwrap(); assert!(recvr.try_recv().is_err()); // Insert first shred, slot should now be considered complete - ledger + blockstore .insert_shreds(vec![orphan_child0], None, false) .unwrap(); assert_eq!(recvr.try_recv().unwrap(), vec![slots[2]]); // Insert the shreds for the orphan_slot let orphan_shred0 = orphan_shreds.remove(0); - ledger.insert_shreds(orphan_shreds, None, false).unwrap(); + blockstore + .insert_shreds(orphan_shreds, None, false) + .unwrap(); assert!(recvr.try_recv().is_err()); // Insert first shred, slot should now be considered complete - ledger + blockstore .insert_shreds(vec![orphan_shred0], None, false) .unwrap(); assert_eq!(recvr.try_recv().unwrap(), vec![slots[1]]); @@ -4797,14 +4794,14 @@ pub mod tests { #[test] pub fn test_completed_shreds_signal_many() { - // Initialize ledger - let ledger_path = get_tmp_ledger_path!(); + // Initialize blockstore + let ledger_path = get_tmp_ledger_path_auto_delete!(); let BlockstoreSignals { - blockstore: ledger, + blockstore, completed_slots_receiver: recvr, .. - } = Blockstore::open_with_signal(&ledger_path, None, true).unwrap(); - let ledger = Arc::new(ledger); + } = Blockstore::open_with_signal(ledger_path.path(), None, true).unwrap(); + // let blockstore = Arc::new(blockstore); let entries_per_slot = 10; let mut slots = vec![2, 5, 10]; @@ -4822,7 +4819,7 @@ pub mod tests { .collect(); all_shreds.shuffle(&mut thread_rng()); - ledger.insert_shreds(all_shreds, None, false).unwrap(); + blockstore.insert_shreds(all_shreds, None, false).unwrap(); let mut result = recvr.try_recv().unwrap(); result.sort_unstable(); slots.push(disconnected_slot); @@ -4832,307 +4829,261 @@ pub mod tests { #[test] pub fn test_handle_chaining_basic() { - let blockstore_path = get_tmp_ledger_path!(); - { - let entries_per_slot = 5; - let num_slots = 3; - let blockstore = Blockstore::open(&blockstore_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - // Construct the shreds - let (mut shreds, _) = make_many_slot_entries(0, num_slots, entries_per_slot); - let shreds_per_slot = shreds.len() / num_slots as usize; - - // 1) Write to the first slot - let shreds1 = shreds - .drain(shreds_per_slot..2 * shreds_per_slot) - .collect_vec(); - blockstore.insert_shreds(shreds1, None, false).unwrap(); - let s1 = blockstore.meta(1).unwrap().unwrap(); - assert!(s1.next_slots.is_empty()); - // Slot 1 is not trunk because slot 0 hasn't been inserted yet - assert!(!s1.is_connected); - assert_eq!(s1.parent_slot, 0); - assert_eq!(s1.last_index, shreds_per_slot as u64 - 1); - - // 2) Write to the second slot - let shreds2 = shreds - .drain(shreds_per_slot..2 * shreds_per_slot) - .collect_vec(); - blockstore.insert_shreds(shreds2, None, false).unwrap(); - let s2 = blockstore.meta(2).unwrap().unwrap(); - assert!(s2.next_slots.is_empty()); - // Slot 2 is not trunk because slot 0 hasn't been inserted yet - assert!(!s2.is_connected); - assert_eq!(s2.parent_slot, 1); - assert_eq!(s2.last_index, shreds_per_slot as u64 - 1); - - // Check the first slot again, it should chain to the second slot, - // but still isn't part of the trunk - let s1 = blockstore.meta(1).unwrap().unwrap(); - assert_eq!(s1.next_slots, vec![2]); - assert!(!s1.is_connected); - assert_eq!(s1.parent_slot, 0); - assert_eq!(s1.last_index, shreds_per_slot as u64 - 1); - - // 3) Write to the zeroth slot, check that every slot - // is now part of the trunk - blockstore.insert_shreds(shreds, None, false).unwrap(); - for i in 0..3 { - let s = blockstore.meta(i).unwrap().unwrap(); - // The last slot will not chain to any other slots - if i != 2 { - assert_eq!(s.next_slots, vec![i + 1]); - } - if i == 0 { - assert_eq!(s.parent_slot, 0); - } else { - assert_eq!(s.parent_slot, i - 1); - } - assert_eq!(s.last_index, shreds_per_slot as u64 - 1); - assert!(s.is_connected); + let entries_per_slot = 5; + let num_slots = 3; + + // Construct the shreds + let (mut shreds, _) = make_many_slot_entries(0, num_slots, entries_per_slot); + let shreds_per_slot = shreds.len() / num_slots as usize; + + // 1) Write to the first slot + let shreds1 = shreds + .drain(shreds_per_slot..2 * shreds_per_slot) + .collect_vec(); + blockstore.insert_shreds(shreds1, None, false).unwrap(); + let s1 = blockstore.meta(1).unwrap().unwrap(); + assert!(s1.next_slots.is_empty()); + // Slot 1 is not trunk because slot 0 hasn't been inserted yet + assert!(!s1.is_connected); + assert_eq!(s1.parent_slot, 0); + assert_eq!(s1.last_index, shreds_per_slot as u64 - 1); + + // 2) Write to the second slot + let shreds2 = shreds + .drain(shreds_per_slot..2 * shreds_per_slot) + .collect_vec(); + blockstore.insert_shreds(shreds2, None, false).unwrap(); + let s2 = blockstore.meta(2).unwrap().unwrap(); + assert!(s2.next_slots.is_empty()); + // Slot 2 is not trunk because slot 0 hasn't been inserted yet + assert!(!s2.is_connected); + assert_eq!(s2.parent_slot, 1); + assert_eq!(s2.last_index, shreds_per_slot as u64 - 1); + + // Check the first slot again, it should chain to the second slot, + // but still isn't part of the trunk + let s1 = blockstore.meta(1).unwrap().unwrap(); + assert_eq!(s1.next_slots, vec![2]); + assert!(!s1.is_connected); + assert_eq!(s1.parent_slot, 0); + assert_eq!(s1.last_index, shreds_per_slot as u64 - 1); + + // 3) Write to the zeroth slot, check that every slot + // is now part of the trunk + blockstore.insert_shreds(shreds, None, false).unwrap(); + for i in 0..3 { + let s = blockstore.meta(i).unwrap().unwrap(); + // The last slot will not chain to any other slots + if i != 2 { + assert_eq!(s.next_slots, vec![i + 1]); + } + if i == 0 { + assert_eq!(s.parent_slot, 0); + } else { + assert_eq!(s.parent_slot, i - 1); } + assert_eq!(s.last_index, shreds_per_slot as u64 - 1); + assert!(s.is_connected); } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] pub fn test_handle_chaining_missing_slots() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - let num_slots = 30; - let entries_per_slot = 5; - - // Separate every other slot into two separate vectors - let mut slots = vec![]; - let mut missing_slots = vec![]; - let mut shreds_per_slot = 2; - for slot in 0..num_slots { - let parent_slot = { - if slot == 0 { - 0 - } else { - slot - 1 - } - }; - let (slot_shreds, _) = make_slot_entries(slot, parent_slot, entries_per_slot); - shreds_per_slot = slot_shreds.len(); - - if slot % 2 == 1 { - slots.extend(slot_shreds); - } else { - missing_slots.extend(slot_shreds); - } - } - - // Write the shreds for every other slot - blockstore.insert_shreds(slots, None, false).unwrap(); - - // Check metadata - for i in 0..num_slots { - // If "i" is the index of a slot we just inserted, then next_slots should be empty - // for slot "i" because no slots chain to that slot, because slot i + 1 is missing. - // However, if it's a slot we haven't inserted, aka one of the gaps, then one of the - // slots we just inserted will chain to that gap, so next_slots for that orphan slot - // won't be empty, but the parent slot is unknown so should equal std::u64::MAX. - let s = blockstore.meta(i as u64).unwrap().unwrap(); - if i % 2 == 0 { - assert_eq!(s.next_slots, vec![i as u64 + 1]); - assert_eq!(s.parent_slot, std::u64::MAX); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + let num_slots = 30; + let entries_per_slot = 5; + + // Separate every other slot into two separate vectors + let mut slots = vec![]; + let mut missing_slots = vec![]; + let mut shreds_per_slot = 2; + for slot in 0..num_slots { + let parent_slot = { + if slot == 0 { + 0 } else { - assert!(s.next_slots.is_empty()); - assert_eq!(s.parent_slot, i - 1); + slot - 1 } + }; + let (slot_shreds, _) = make_slot_entries(slot, parent_slot, entries_per_slot); + shreds_per_slot = slot_shreds.len(); - if i == 0 { - assert!(s.is_connected); - } else { - assert!(!s.is_connected); - } + if slot % 2 == 1 { + slots.extend(slot_shreds); + } else { + missing_slots.extend(slot_shreds); } + } - // Write the shreds for the other half of the slots that we didn't insert earlier - blockstore - .insert_shreds(missing_slots, None, false) - .unwrap(); + // Write the shreds for every other slot + blockstore.insert_shreds(slots, None, false).unwrap(); - for i in 0..num_slots { - // Check that all the slots chain correctly once the missing slots - // have been filled - let s = blockstore.meta(i as u64).unwrap().unwrap(); - if i != num_slots - 1 { - assert_eq!(s.next_slots, vec![i as u64 + 1]); - } else { - assert!(s.next_slots.is_empty()); - } + // Check metadata + for i in 0..num_slots { + // If "i" is the index of a slot we just inserted, then next_slots should be empty + // for slot "i" because no slots chain to that slot, because slot i + 1 is missing. + // However, if it's a slot we haven't inserted, aka one of the gaps, then one of the + // slots we just inserted will chain to that gap, so next_slots for that orphan slot + // won't be empty, but the parent slot is unknown so should equal std::u64::MAX. + let s = blockstore.meta(i as u64).unwrap().unwrap(); + if i % 2 == 0 { + assert_eq!(s.next_slots, vec![i as u64 + 1]); + assert_eq!(s.parent_slot, std::u64::MAX); + } else { + assert!(s.next_slots.is_empty()); + assert_eq!(s.parent_slot, i - 1); + } - if i == 0 { - assert_eq!(s.parent_slot, 0); - } else { - assert_eq!(s.parent_slot, i - 1); - } - assert_eq!(s.last_index, shreds_per_slot as u64 - 1); + if i == 0 { assert!(s.is_connected); + } else { + assert!(!s.is_connected); } } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + // Write the shreds for the other half of the slots that we didn't insert earlier + blockstore + .insert_shreds(missing_slots, None, false) + .unwrap(); + + for i in 0..num_slots { + // Check that all the slots chain correctly once the missing slots + // have been filled + let s = blockstore.meta(i as u64).unwrap().unwrap(); + if i != num_slots - 1 { + assert_eq!(s.next_slots, vec![i as u64 + 1]); + } else { + assert!(s.next_slots.is_empty()); + } + + if i == 0 { + assert_eq!(s.parent_slot, 0); + } else { + assert_eq!(s.parent_slot, i - 1); + } + assert_eq!(s.last_index, shreds_per_slot as u64 - 1); + assert!(s.is_connected); + } } #[test] #[allow(clippy::cognitive_complexity)] pub fn test_forward_chaining_is_connected() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - let num_slots = 15; - // Create enough entries to ensure there are at least two shreds created - let entries_per_slot = max_ticks_per_n_shreds(1, None) + 1; - assert!(entries_per_slot > 1); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - let (mut shreds, _) = make_many_slot_entries(0, num_slots, entries_per_slot); - let shreds_per_slot = shreds.len() / num_slots as usize; - assert!(shreds_per_slot > 1); + let num_slots = 15; + // Create enough entries to ensure there are at least two shreds created + let entries_per_slot = max_ticks_per_n_shreds(1, None) + 1; + assert!(entries_per_slot > 1); - // Write the shreds such that every 3rd slot has a gap in the beginning - let mut missing_shreds = vec![]; - for slot in 0..num_slots { - let mut shreds_for_slot = shreds.drain(..shreds_per_slot).collect_vec(); - if slot % 3 == 0 { - let shred0 = shreds_for_slot.remove(0); - missing_shreds.push(shred0); - } - blockstore - .insert_shreds(shreds_for_slot, None, false) - .unwrap(); + let (mut shreds, _) = make_many_slot_entries(0, num_slots, entries_per_slot); + let shreds_per_slot = shreds.len() / num_slots as usize; + assert!(shreds_per_slot > 1); + + // Write the shreds such that every 3rd slot has a gap in the beginning + let mut missing_shreds = vec![]; + for slot in 0..num_slots { + let mut shreds_for_slot = shreds.drain(..shreds_per_slot).collect_vec(); + if slot % 3 == 0 { + let shred0 = shreds_for_slot.remove(0); + missing_shreds.push(shred0); } + blockstore + .insert_shreds(shreds_for_slot, None, false) + .unwrap(); + } - // Check metadata - for i in 0..num_slots { - let s = blockstore.meta(i as u64).unwrap().unwrap(); - // The last slot will not chain to any other slots - if i as u64 != num_slots - 1 { - assert_eq!(s.next_slots, vec![i as u64 + 1]); - } else { - assert!(s.next_slots.is_empty()); - } + // Check metadata + for i in 0..num_slots { + let s = blockstore.meta(i as u64).unwrap().unwrap(); + // The last slot will not chain to any other slots + if i as u64 != num_slots - 1 { + assert_eq!(s.next_slots, vec![i as u64 + 1]); + } else { + assert!(s.next_slots.is_empty()); + } - if i == 0 { - assert_eq!(s.parent_slot, 0); - } else { - assert_eq!(s.parent_slot, i - 1); - } + if i == 0 { + assert_eq!(s.parent_slot, 0); + } else { + assert_eq!(s.parent_slot, i - 1); + } - assert_eq!(s.last_index, shreds_per_slot as u64 - 1); + assert_eq!(s.last_index, shreds_per_slot as u64 - 1); - // Other than slot 0, no slots should be part of the trunk - if i != 0 { - assert!(!s.is_connected); - } else { - assert!(s.is_connected); - } + // Other than slot 0, no slots should be part of the trunk + if i != 0 { + assert!(!s.is_connected); + } else { + assert!(s.is_connected); } + } - // Iteratively finish every 3rd slot, and check that all slots up to and including - // slot_index + 3 become part of the trunk - for slot_index in 0..num_slots { - if slot_index % 3 == 0 { - let shred = missing_shreds.remove(0); - blockstore.insert_shreds(vec![shred], None, false).unwrap(); - - for i in 0..num_slots { - let s = blockstore.meta(i as u64).unwrap().unwrap(); - if i != num_slots - 1 { - assert_eq!(s.next_slots, vec![i as u64 + 1]); - } else { - assert!(s.next_slots.is_empty()); - } - if i <= slot_index as u64 + 3 { - assert!(s.is_connected); - } else { - assert!(!s.is_connected); - } + // Iteratively finish every 3rd slot, and check that all slots up to and including + // slot_index + 3 become part of the trunk + for slot_index in 0..num_slots { + if slot_index % 3 == 0 { + let shred = missing_shreds.remove(0); + blockstore.insert_shreds(vec![shred], None, false).unwrap(); - if i == 0 { - assert_eq!(s.parent_slot, 0); - } else { - assert_eq!(s.parent_slot, i - 1); - } + for i in 0..num_slots { + let s = blockstore.meta(i as u64).unwrap().unwrap(); + if i != num_slots - 1 { + assert_eq!(s.next_slots, vec![i as u64 + 1]); + } else { + assert!(s.next_slots.is_empty()); + } + if i <= slot_index as u64 + 3 { + assert!(s.is_connected); + } else { + assert!(!s.is_connected); + } - assert_eq!(s.last_index, shreds_per_slot as u64 - 1); + if i == 0 { + assert_eq!(s.parent_slot, 0); + } else { + assert_eq!(s.parent_slot, i - 1); } + + assert_eq!(s.last_index, shreds_per_slot as u64 - 1); } } } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } /* #[test] pub fn test_chaining_tree() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - let num_tree_levels = 6; - assert!(num_tree_levels > 1); - let branching_factor: u64 = 4; - // Number of slots that will be in the tree - let num_slots = (branching_factor.pow(num_tree_levels) - 1) / (branching_factor - 1); - let erasure_config = ErasureConfig::default(); - let entries_per_slot = erasure_config.num_data() as u64; - assert!(entries_per_slot > 1); - - let (mut shreds, _) = make_many_slot_entries(0, num_slots, entries_per_slot); - - // Insert tree one slot at a time in a random order - let mut slots: Vec<_> = (0..num_slots).collect(); - - // Get shreds for the slot - slots.shuffle(&mut thread_rng()); - for slot in slots { - // Get shreds for the slot "slot" - let slot_shreds = &mut shreds - [(slot * entries_per_slot) as usize..((slot + 1) * entries_per_slot) as usize]; - for shred in slot_shreds.iter_mut() { - // Get the parent slot of the slot in the tree - let slot_parent = { - if slot == 0 { - 0 - } else { - (slot - 1) / branching_factor - } - }; - shred.set_parent(slot_parent); - } + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + let num_tree_levels = 6; + assert!(num_tree_levels > 1); + let branching_factor: u64 = 4; + // Number of slots that will be in the tree + let num_slots = (branching_factor.pow(num_tree_levels) - 1) / (branching_factor - 1); + let erasure_config = ErasureConfig::default(); + let entries_per_slot = erasure_config.num_data() as u64; + assert!(entries_per_slot > 1); - let shared_shreds: Vec<_> = slot_shreds - .iter() - .cloned() - .map(|shred| Arc::new(RwLock::new(shred))) - .collect(); - let mut coding_generator = CodingGenerator::new_from_config(&erasure_config); - let coding_shreds = coding_generator.next(&shared_shreds); - assert_eq!(coding_shreds.len(), erasure_config.num_coding()); - - let mut rng = thread_rng(); - - // Randomly pick whether to insert erasure or coding shreds first - if rng.gen_bool(0.5) { - blockstore.write_shreds(slot_shreds).unwrap(); - blockstore.put_shared_coding_shreds(&coding_shreds).unwrap(); - } else { - blockstore.put_shared_coding_shreds(&coding_shreds).unwrap(); - blockstore.write_shreds(slot_shreds).unwrap(); - } - } + let (mut shreds, _) = make_many_slot_entries(0, num_slots, entries_per_slot); - // Make sure everything chains correctly - let last_level = - (branching_factor.pow(num_tree_levels - 1) - 1) / (branching_factor - 1); - for slot in 0..num_slots { - let slot_meta = blockstore.meta(slot).unwrap().unwrap(); - assert_eq!(slot_meta.consumed, entries_per_slot); - assert_eq!(slot_meta.received, entries_per_slot); - assert!(slot_meta.is_connected); + // Insert tree one slot at a time in a random order + let mut slots: Vec<_> = (0..num_slots).collect(); + + // Get shreds for the slot + slots.shuffle(&mut thread_rng()); + for slot in slots { + // Get shreds for the slot "slot" + let slot_shreds = &mut shreds + [(slot * entries_per_slot) as usize..((slot + 1) * entries_per_slot) as usize]; + for shred in slot_shreds.iter_mut() { + // Get the parent slot of the slot in the tree let slot_parent = { if slot == 0 { 0 @@ -5140,210 +5091,236 @@ pub mod tests { (slot - 1) / branching_factor } }; - assert_eq!(slot_meta.parent_slot, slot_parent); + shred.set_parent(slot_parent); + } - let expected_children: HashSet<_> = { - if slot >= last_level { - HashSet::new() - } else { - let first_child_slot = min(num_slots - 1, slot * branching_factor + 1); - let last_child_slot = min(num_slots - 1, (slot + 1) * branching_factor); - (first_child_slot..last_child_slot + 1).collect() - } - }; + let shared_shreds: Vec<_> = slot_shreds + .iter() + .cloned() + .map(|shred| Arc::new(RwLock::new(shred))) + .collect(); + let mut coding_generator = CodingGenerator::new_from_config(&erasure_config); + let coding_shreds = coding_generator.next(&shared_shreds); + assert_eq!(coding_shreds.len(), erasure_config.num_coding()); + + let mut rng = thread_rng(); + + // Randomly pick whether to insert erasure or coding shreds first + if rng.gen_bool(0.5) { + blockstore.write_shreds(slot_shreds).unwrap(); + blockstore.put_shared_coding_shreds(&coding_shreds).unwrap(); + } else { + blockstore.put_shared_coding_shreds(&coding_shreds).unwrap(); + blockstore.write_shreds(slot_shreds).unwrap(); + } + } - let result: HashSet<_> = slot_meta.next_slots.iter().cloned().collect(); - if expected_children.len() != 0 { - assert_eq!(slot_meta.next_slots.len(), branching_factor as usize); + // Make sure everything chains correctly + let last_level = + (branching_factor.pow(num_tree_levels - 1) - 1) / (branching_factor - 1); + for slot in 0..num_slots { + let slot_meta = blockstore.meta(slot).unwrap().unwrap(); + assert_eq!(slot_meta.consumed, entries_per_slot); + assert_eq!(slot_meta.received, entries_per_slot); + assert!(slot_meta.is_connected); + let slot_parent = { + if slot == 0 { + 0 } else { - assert_eq!(slot_meta.next_slots.len(), 0); + (slot - 1) / branching_factor } - assert_eq!(expected_children, result); - } + }; + assert_eq!(slot_meta.parent_slot, slot_parent); + + let expected_children: HashSet<_> = { + if slot >= last_level { + HashSet::new() + } else { + let first_child_slot = min(num_slots - 1, slot * branching_factor + 1); + let last_child_slot = min(num_slots - 1, (slot + 1) * branching_factor); + (first_child_slot..last_child_slot + 1).collect() + } + }; - // No orphan slots should exist - assert!(blockstore.orphans_cf.is_empty().unwrap()) + let result: HashSet<_> = slot_meta.next_slots.iter().cloned().collect(); + if expected_children.len() != 0 { + assert_eq!(slot_meta.next_slots.len(), branching_factor as usize); + } else { + assert_eq!(slot_meta.next_slots.len(), 0); + } + assert_eq!(expected_children, result); } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + // No orphan slots should exist + assert!(blockstore.orphans_cf.is_empty().unwrap()) + } */ #[test] pub fn test_get_slots_since() { - let blockstore_path = get_tmp_ledger_path!(); - - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - - // Slot doesn't exist - assert!(blockstore.get_slots_since(&[0]).unwrap().is_empty()); - - let mut meta0 = SlotMeta::new(0, 0); - blockstore.meta_cf.put(0, &meta0).unwrap(); - - // Slot exists, chains to nothing - let expected: HashMap> = vec![(0, vec![])].into_iter().collect(); - assert_eq!(blockstore.get_slots_since(&[0]).unwrap(), expected); - meta0.next_slots = vec![1, 2]; - blockstore.meta_cf.put(0, &meta0).unwrap(); - - // Slot exists, chains to some other slots - let expected: HashMap> = vec![(0, vec![1, 2])].into_iter().collect(); - assert_eq!(blockstore.get_slots_since(&[0]).unwrap(), expected); - assert_eq!(blockstore.get_slots_since(&[0, 1]).unwrap(), expected); - - let mut meta3 = SlotMeta::new(3, 1); - meta3.next_slots = vec![10, 5]; - blockstore.meta_cf.put(3, &meta3).unwrap(); - let expected: HashMap> = vec![(0, vec![1, 2]), (3, vec![10, 5])] - .into_iter() - .collect(); - assert_eq!(blockstore.get_slots_since(&[0, 1, 3]).unwrap(), expected); - } - - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + // Slot doesn't exist + assert!(blockstore.get_slots_since(&[0]).unwrap().is_empty()); + + let mut meta0 = SlotMeta::new(0, 0); + blockstore.meta_cf.put(0, &meta0).unwrap(); + + // Slot exists, chains to nothing + let expected: HashMap> = vec![(0, vec![])].into_iter().collect(); + assert_eq!(blockstore.get_slots_since(&[0]).unwrap(), expected); + meta0.next_slots = vec![1, 2]; + blockstore.meta_cf.put(0, &meta0).unwrap(); + + // Slot exists, chains to some other slots + let expected: HashMap> = vec![(0, vec![1, 2])].into_iter().collect(); + assert_eq!(blockstore.get_slots_since(&[0]).unwrap(), expected); + assert_eq!(blockstore.get_slots_since(&[0, 1]).unwrap(), expected); + + let mut meta3 = SlotMeta::new(3, 1); + meta3.next_slots = vec![10, 5]; + blockstore.meta_cf.put(3, &meta3).unwrap(); + let expected: HashMap> = vec![(0, vec![1, 2]), (3, vec![10, 5])] + .into_iter() + .collect(); + assert_eq!(blockstore.get_slots_since(&[0, 1, 3]).unwrap(), expected); } #[test] fn test_orphans() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - // Create shreds and entries - let entries_per_slot = 1; - let (mut shreds, _) = make_many_slot_entries(0, 3, entries_per_slot); - let shreds_per_slot = shreds.len() / 3; + // Create shreds and entries + let entries_per_slot = 1; + let (mut shreds, _) = make_many_slot_entries(0, 3, entries_per_slot); + let shreds_per_slot = shreds.len() / 3; - // Write slot 2, which chains to slot 1. We're missing slot 0, - // so slot 1 is the orphan - let shreds_for_slot = shreds.drain((shreds_per_slot * 2)..).collect_vec(); - blockstore - .insert_shreds(shreds_for_slot, None, false) - .unwrap(); - let meta = blockstore - .meta(1) - .expect("Expect database get to succeed") - .unwrap(); - assert!(is_orphan(&meta)); - assert_eq!( - blockstore.orphans_iterator(0).unwrap().collect::>(), - vec![1] - ); + // Write slot 2, which chains to slot 1. We're missing slot 0, + // so slot 1 is the orphan + let shreds_for_slot = shreds.drain((shreds_per_slot * 2)..).collect_vec(); + blockstore + .insert_shreds(shreds_for_slot, None, false) + .unwrap(); + let meta = blockstore + .meta(1) + .expect("Expect database get to succeed") + .unwrap(); + assert!(is_orphan(&meta)); + assert_eq!( + blockstore.orphans_iterator(0).unwrap().collect::>(), + vec![1] + ); - // Write slot 1 which chains to slot 0, so now slot 0 is the - // orphan, and slot 1 is no longer the orphan. - let shreds_for_slot = shreds.drain(shreds_per_slot..).collect_vec(); - blockstore - .insert_shreds(shreds_for_slot, None, false) - .unwrap(); + // Write slot 1 which chains to slot 0, so now slot 0 is the + // orphan, and slot 1 is no longer the orphan. + let shreds_for_slot = shreds.drain(shreds_per_slot..).collect_vec(); + blockstore + .insert_shreds(shreds_for_slot, None, false) + .unwrap(); + let meta = blockstore + .meta(1) + .expect("Expect database get to succeed") + .unwrap(); + assert!(!is_orphan(&meta)); + let meta = blockstore + .meta(0) + .expect("Expect database get to succeed") + .unwrap(); + assert!(is_orphan(&meta)); + assert_eq!( + blockstore.orphans_iterator(0).unwrap().collect::>(), + vec![0] + ); + + // Write some slot that also chains to existing slots and orphan, + // nothing should change + let (shred4, _) = make_slot_entries(4, 0, 1); + let (shred5, _) = make_slot_entries(5, 1, 1); + blockstore.insert_shreds(shred4, None, false).unwrap(); + blockstore.insert_shreds(shred5, None, false).unwrap(); + assert_eq!( + blockstore.orphans_iterator(0).unwrap().collect::>(), + vec![0] + ); + + // Write zeroth slot, no more orphans + blockstore.insert_shreds(shreds, None, false).unwrap(); + for i in 0..3 { let meta = blockstore - .meta(1) + .meta(i) .expect("Expect database get to succeed") .unwrap(); assert!(!is_orphan(&meta)); - let meta = blockstore - .meta(0) - .expect("Expect database get to succeed") - .unwrap(); - assert!(is_orphan(&meta)); - assert_eq!( - blockstore.orphans_iterator(0).unwrap().collect::>(), - vec![0] - ); - - // Write some slot that also chains to existing slots and orphan, - // nothing should change - let (shred4, _) = make_slot_entries(4, 0, 1); - let (shred5, _) = make_slot_entries(5, 1, 1); - blockstore.insert_shreds(shred4, None, false).unwrap(); - blockstore.insert_shreds(shred5, None, false).unwrap(); - assert_eq!( - blockstore.orphans_iterator(0).unwrap().collect::>(), - vec![0] - ); - - // Write zeroth slot, no more orphans - blockstore.insert_shreds(shreds, None, false).unwrap(); - for i in 0..3 { - let meta = blockstore - .meta(i) - .expect("Expect database get to succeed") - .unwrap(); - assert!(!is_orphan(&meta)); - } - // Orphans cf is empty - assert!(blockstore.orphans_cf.is_empty().unwrap()) } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + // Orphans cf is empty + assert!(blockstore.orphans_cf.is_empty().unwrap()); } - fn test_insert_data_shreds_slots(name: &str, should_bulk_write: bool) { - let blockstore_path = get_ledger_path_from_name(name); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - - // Create shreds and entries - let num_entries = 20_u64; - let mut entries = vec![]; - let mut shreds = vec![]; - let mut num_shreds_per_slot = 0; - for slot in 0..num_entries { - let parent_slot = { - if slot == 0 { - 0 - } else { - slot - 1 - } - }; - - let (mut shred, entry) = make_slot_entries(slot, parent_slot, 1); - num_shreds_per_slot = shred.len() as u64; - shred - .iter_mut() - .enumerate() - .for_each(|(_, shred)| shred.set_index(0)); - shreds.extend(shred); - entries.extend(entry); - } + fn test_insert_data_shreds_slots(should_bulk_write: bool) { + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - let num_shreds = shreds.len(); - // Write shreds to the database - if should_bulk_write { - blockstore.insert_shreds(shreds, None, false).unwrap(); - } else { - for _ in 0..num_shreds { - let shred = shreds.remove(0); - blockstore.insert_shreds(vec![shred], None, false).unwrap(); + // Create shreds and entries + let num_entries = 20_u64; + let mut entries = vec![]; + let mut shreds = vec![]; + let mut num_shreds_per_slot = 0; + for slot in 0..num_entries { + let parent_slot = { + if slot == 0 { + 0 + } else { + slot - 1 } + }; + + let (mut shred, entry) = make_slot_entries(slot, parent_slot, 1); + num_shreds_per_slot = shred.len() as u64; + shred + .iter_mut() + .enumerate() + .for_each(|(_, shred)| shred.set_index(0)); + shreds.extend(shred); + entries.extend(entry); + } + + let num_shreds = shreds.len(); + // Write shreds to the database + if should_bulk_write { + blockstore.insert_shreds(shreds, None, false).unwrap(); + } else { + for _ in 0..num_shreds { + let shred = shreds.remove(0); + blockstore.insert_shreds(vec![shred], None, false).unwrap(); } + } - for i in 0..num_entries - 1 { - assert_eq!( - blockstore.get_slot_entries(i, 0).unwrap()[0], - entries[i as usize] - ); + for i in 0..num_entries - 1 { + assert_eq!( + blockstore.get_slot_entries(i, 0).unwrap()[0], + entries[i as usize] + ); - let meta = blockstore.meta(i).unwrap().unwrap(); - assert_eq!(meta.received, 1); - assert_eq!(meta.last_index, 0); - if i != 0 { - assert_eq!(meta.parent_slot, i - 1); - assert_eq!(meta.consumed, 1); - } else { - assert_eq!(meta.parent_slot, 0); - assert_eq!(meta.consumed, num_shreds_per_slot); - } + let meta = blockstore.meta(i).unwrap().unwrap(); + assert_eq!(meta.received, 1); + assert_eq!(meta.last_index, 0); + if i != 0 { + assert_eq!(meta.parent_slot, i - 1); + assert_eq!(meta.consumed, 1); + } else { + assert_eq!(meta.parent_slot, 0); + assert_eq!(meta.consumed, num_shreds_per_slot); } } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] fn test_find_missing_data_indexes() { let slot = 0; - let blockstore_path = get_tmp_ledger_path!(); - let blockstore = Blockstore::open(&blockstore_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); // Write entries let gap: u64 = 10; @@ -5424,16 +5401,13 @@ pub mod tests { ); } } - - drop(blockstore); - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] fn test_find_missing_data_indexes_timeout() { let slot = 0; - let blockstore_path = get_tmp_ledger_path!(); - let blockstore = Blockstore::open(&blockstore_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); // Write entries let gap: u64 = 10; @@ -5464,17 +5438,14 @@ pub mod tests { blockstore.find_missing_data_indexes(slot, timestamp() - 400, 0, 50, 9), expected ); - - drop(blockstore); - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] fn test_find_missing_data_indexes_sanity() { let slot = 0; - let blockstore_path = get_tmp_ledger_path!(); - let blockstore = Blockstore::open(&blockstore_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); // Early exit conditions let empty: Vec = vec![]; @@ -5524,16 +5495,13 @@ pub mod tests { let expected: Vec = (start..END).filter(|i| *i != ONE && *i != OTHER).collect(); assert_eq!(result, expected); } - - drop(blockstore); - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] pub fn test_no_missing_shred_indexes() { let slot = 0; - let blockstore_path = get_tmp_ledger_path!(); - let blockstore = Blockstore::open(&blockstore_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); // Write entries let num_entries = 10; @@ -5552,347 +5520,329 @@ pub mod tests { ); } } - - drop(blockstore); - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] pub fn test_should_insert_data_shred() { solana_logger::setup(); let (mut shreds, _) = make_slot_entries(0, 0, 200); - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - let last_root = RwLock::new(0); - - // Insert the first 5 shreds, we don't have a "is_last" shred yet - blockstore - .insert_shreds(shreds[0..5].to_vec(), None, false) - .unwrap(); - - let slot_meta = blockstore.meta(0).unwrap().unwrap(); - // Corrupt shred by making it too large - let mut shred5 = shreds[5].clone(); - shred5.payload.push(10); - shred5.data_header.size = shred5.payload.len() as u16; - assert!(!blockstore.should_insert_data_shred( - &shred5, - &slot_meta, - &HashMap::new(), - &last_root, - None, - ShredSource::Turbine - )); - - // Ensure that an empty shred (one with no data) would get inserted. Such shreds - // may be used as signals (broadcast does so to indicate a slot was interrupted) - // Reuse shred5's header values to avoid a false negative result - let mut empty_shred = Shred::new_from_data( - shred5.common_header.slot, - shred5.common_header.index, - shred5.data_header.parent_offset, - None, // data - true, // is_last_data - true, // is_last_in_slot - 0, // reference_tick - shred5.common_header.version, - shred5.common_header.fec_set_index, - ); - assert!(blockstore.should_insert_data_shred( - &empty_shred, - &slot_meta, - &HashMap::new(), - &last_root, - None, - ShredSource::Repaired, - )); - empty_shred.data_header.size = 0; - assert!(!blockstore.should_insert_data_shred( - &empty_shred, - &slot_meta, - &HashMap::new(), - &last_root, - None, - ShredSource::Recovered, - )); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - // Trying to insert another "is_last" shred with index < the received index should fail - // skip over shred 7 - blockstore - .insert_shreds(shreds[8..9].to_vec(), None, false) - .unwrap(); - let slot_meta = blockstore.meta(0).unwrap().unwrap(); - assert_eq!(slot_meta.received, 9); - let shred7 = { - if shreds[7].is_data() { - shreds[7].set_last_in_slot(); - shreds[7].clone() - } else { - panic!("Shred in unexpected format") - } - }; - assert!(!blockstore.should_insert_data_shred( - &shred7, - &slot_meta, - &HashMap::new(), - &last_root, - None, - ShredSource::Repaired, - )); - assert!(blockstore.has_duplicate_shreds_in_slot(0)); + let last_root = RwLock::new(0); - // Insert all pending shreds - let mut shred8 = shreds[8].clone(); - blockstore.insert_shreds(shreds, None, false).unwrap(); - let slot_meta = blockstore.meta(0).unwrap().unwrap(); + // Insert the first 5 shreds, we don't have a "is_last" shred yet + blockstore + .insert_shreds(shreds[0..5].to_vec(), None, false) + .unwrap(); - // Trying to insert a shred with index > the "is_last" shred should fail - if shred8.is_data() { - shred8.set_slot(slot_meta.last_index + 1); + let slot_meta = blockstore.meta(0).unwrap().unwrap(); + // Corrupt shred by making it too large + let mut shred5 = shreds[5].clone(); + shred5.payload.push(10); + shred5.data_header.size = shred5.payload.len() as u16; + assert!(!blockstore.should_insert_data_shred( + &shred5, + &slot_meta, + &HashMap::new(), + &last_root, + None, + ShredSource::Turbine + )); + + // Ensure that an empty shred (one with no data) would get inserted. Such shreds + // may be used as signals (broadcast does so to indicate a slot was interrupted) + // Reuse shred5's header values to avoid a false negative result + let mut empty_shred = Shred::new_from_data( + shred5.common_header.slot, + shred5.common_header.index, + shred5.data_header.parent_offset, + None, // data + true, // is_last_data + true, // is_last_in_slot + 0, // reference_tick + shred5.common_header.version, + shred5.common_header.fec_set_index, + ); + assert!(blockstore.should_insert_data_shred( + &empty_shred, + &slot_meta, + &HashMap::new(), + &last_root, + None, + ShredSource::Repaired, + )); + empty_shred.data_header.size = 0; + assert!(!blockstore.should_insert_data_shred( + &empty_shred, + &slot_meta, + &HashMap::new(), + &last_root, + None, + ShredSource::Recovered, + )); + + // Trying to insert another "is_last" shred with index < the received index should fail + // skip over shred 7 + blockstore + .insert_shreds(shreds[8..9].to_vec(), None, false) + .unwrap(); + let slot_meta = blockstore.meta(0).unwrap().unwrap(); + assert_eq!(slot_meta.received, 9); + let shred7 = { + if shreds[7].is_data() { + shreds[7].set_last_in_slot(); + shreds[7].clone() } else { panic!("Shred in unexpected format") } - assert!(!blockstore.should_insert_data_shred( - &shred7, - &slot_meta, - &HashMap::new(), - &last_root, - None, - ShredSource::Repaired, - )); + }; + assert!(!blockstore.should_insert_data_shred( + &shred7, + &slot_meta, + &HashMap::new(), + &last_root, + None, + ShredSource::Repaired, + )); + assert!(blockstore.has_duplicate_shreds_in_slot(0)); + + // Insert all pending shreds + let mut shred8 = shreds[8].clone(); + blockstore.insert_shreds(shreds, None, false).unwrap(); + let slot_meta = blockstore.meta(0).unwrap().unwrap(); + + // Trying to insert a shred with index > the "is_last" shred should fail + if shred8.is_data() { + shred8.set_slot(slot_meta.last_index + 1); + } else { + panic!("Shred in unexpected format") } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + assert!(!blockstore.should_insert_data_shred( + &shred7, + &slot_meta, + &HashMap::new(), + &last_root, + None, + ShredSource::Repaired, + )); } #[test] pub fn test_is_data_shred_present() { let (shreds, _) = make_slot_entries(0, 0, 200); - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - let index_cf = blockstore.db.column::(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + let index_cf = &blockstore.index_cf; - blockstore - .insert_shreds(shreds[0..5].to_vec(), None, false) - .unwrap(); - // Insert a shred less than `slot_meta.consumed`, check that - // it already exists - let slot_meta = blockstore.meta(0).unwrap().unwrap(); - let index = index_cf.get(0).unwrap().unwrap(); - assert_eq!(slot_meta.consumed, 5); - assert!(Blockstore::is_data_shred_present( - &shreds[1], - &slot_meta, - index.data(), - )); - - // Insert a shred, check that it already exists - blockstore - .insert_shreds(shreds[6..7].to_vec(), None, false) - .unwrap(); - let slot_meta = blockstore.meta(0).unwrap().unwrap(); - let index = index_cf.get(0).unwrap().unwrap(); - assert!(Blockstore::is_data_shred_present( - &shreds[6], - &slot_meta, - index.data() - ),); - } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + blockstore + .insert_shreds(shreds[0..5].to_vec(), None, false) + .unwrap(); + // Insert a shred less than `slot_meta.consumed`, check that + // it already exists + let slot_meta = blockstore.meta(0).unwrap().unwrap(); + let index = index_cf.get(0).unwrap().unwrap(); + assert_eq!(slot_meta.consumed, 5); + assert!(Blockstore::is_data_shred_present( + &shreds[1], + &slot_meta, + index.data(), + )); + + // Insert a shred, check that it already exists + blockstore + .insert_shreds(shreds[6..7].to_vec(), None, false) + .unwrap(); + let slot_meta = blockstore.meta(0).unwrap().unwrap(); + let index = index_cf.get(0).unwrap().unwrap(); + assert!(Blockstore::is_data_shred_present( + &shreds[6], + &slot_meta, + index.data() + ),); } #[test] pub fn test_check_cache_coding_shred() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - - let slot = 1; - let (shred, coding) = Shredder::new_coding_shred_header(slot, 11, 11, 11, 11, 0); - let coding_shred = - Shred::new_empty_from_header(shred, DataShredHeader::default(), coding); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - let mut erasure_metas = HashMap::new(); - let mut index_working_set = HashMap::new(); - let mut just_received_coding_shreds = HashMap::new(); - let mut index_meta_time = 0; - assert!(blockstore.check_cache_coding_shred( - coding_shred.clone(), - &mut erasure_metas, - &mut index_working_set, - &mut just_received_coding_shreds, - &mut index_meta_time, - &|_shred| { - panic!("no dupes"); - }, - false, - false, - )); + let slot = 1; + let (shred, coding) = Shredder::new_coding_shred_header(slot, 11, 11, 11, 11, 0); + let coding_shred = Shred::new_empty_from_header(shred, DataShredHeader::default(), coding); - // insert again fails on dupe - use std::sync::atomic::{AtomicUsize, Ordering}; - let counter = AtomicUsize::new(0); - assert!(!blockstore.check_cache_coding_shred( - coding_shred, - &mut erasure_metas, - &mut index_working_set, - &mut just_received_coding_shreds, - &mut index_meta_time, - &|_shred| { - counter.fetch_add(1, Ordering::Relaxed); - }, - false, - false, - )); - assert_eq!(counter.load(Ordering::Relaxed), 1); - } + let mut erasure_metas = HashMap::new(); + let mut index_working_set = HashMap::new(); + let mut just_received_coding_shreds = HashMap::new(); + let mut index_meta_time = 0; + assert!(blockstore.check_cache_coding_shred( + coding_shred.clone(), + &mut erasure_metas, + &mut index_working_set, + &mut just_received_coding_shreds, + &mut index_meta_time, + &|_shred| { + panic!("no dupes"); + }, + false, + false, + )); + + // insert again fails on dupe + use std::sync::atomic::{AtomicUsize, Ordering}; + let counter = AtomicUsize::new(0); + assert!(!blockstore.check_cache_coding_shred( + coding_shred, + &mut erasure_metas, + &mut index_working_set, + &mut just_received_coding_shreds, + &mut index_meta_time, + &|_shred| { + counter.fetch_add(1, Ordering::Relaxed); + }, + false, + false, + )); + assert_eq!(counter.load(Ordering::Relaxed), 1); } #[test] pub fn test_should_insert_coding_shred() { - let blockstore_path = get_tmp_ledger_path!(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + let last_root = RwLock::new(0); + + let slot = 1; + let (mut shred, coding) = Shredder::new_coding_shred_header(slot, 11, 11, 11, 11, 0); + let coding_shred = + Shred::new_empty_from_header(shred.clone(), DataShredHeader::default(), coding.clone()); + + // Insert a good coding shred + assert!(Blockstore::should_insert_coding_shred( + &coding_shred, + &last_root + )); + + // Insertion should succeed + blockstore + .insert_shreds(vec![coding_shred.clone()], None, false) + .unwrap(); + + // Trying to insert the same shred again should pass since this doesn't check for + // duplicate index { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - let last_root = RwLock::new(0); + assert!(Blockstore::should_insert_coding_shred( + &coding_shred, + &last_root + )); + } + + shred.index += 1; - let slot = 1; - let (mut shred, coding) = Shredder::new_coding_shred_header(slot, 11, 11, 11, 11, 0); + // Establish a baseline that works + { let coding_shred = Shred::new_empty_from_header( shred.clone(), DataShredHeader::default(), coding.clone(), ); - - // Insert a good coding shred assert!(Blockstore::should_insert_coding_shred( &coding_shred, &last_root )); + } - // Insertion should succeed - blockstore - .insert_shreds(vec![coding_shred.clone()], None, false) - .unwrap(); - - // Trying to insert the same shred again should pass since this doesn't check for - // duplicate index - { - assert!(Blockstore::should_insert_coding_shred( - &coding_shred, - &last_root - )); - } - - shred.index += 1; + // Trying to insert a shred with index < position should fail + { + let mut coding_shred = Shred::new_empty_from_header( + shred.clone(), + DataShredHeader::default(), + coding.clone(), + ); + let index = coding_shred.index() - coding_shred.common_header.fec_set_index - 1; + coding_shred.set_index(index as u32); - // Establish a baseline that works - { - let coding_shred = Shred::new_empty_from_header( - shred.clone(), - DataShredHeader::default(), - coding.clone(), - ); - assert!(Blockstore::should_insert_coding_shred( - &coding_shred, - &last_root - )); - } + assert!(!Blockstore::should_insert_coding_shred( + &coding_shred, + &last_root + )); + } - // Trying to insert a shred with index < position should fail - { - let mut coding_shred = Shred::new_empty_from_header( - shred.clone(), - DataShredHeader::default(), - coding.clone(), - ); - let index = coding_shred.index() - coding_shred.common_header.fec_set_index - 1; - coding_shred.set_index(index as u32); + // Trying to insert shred with num_coding == 0 should fail + { + let mut coding_shred = Shred::new_empty_from_header( + shred.clone(), + DataShredHeader::default(), + coding.clone(), + ); + coding_shred.coding_header.num_coding_shreds = 0; + assert!(!Blockstore::should_insert_coding_shred( + &coding_shred, + &last_root + )); + } - assert!(!Blockstore::should_insert_coding_shred( - &coding_shred, - &last_root - )); - } + // Trying to insert shred with pos >= num_coding should fail + { + let mut coding_shred = Shred::new_empty_from_header( + shred.clone(), + DataShredHeader::default(), + coding.clone(), + ); + let num_coding_shreds = + coding_shred.common_header.index - coding_shred.common_header.fec_set_index; + coding_shred.coding_header.num_coding_shreds = num_coding_shreds as u16; + assert!(!Blockstore::should_insert_coding_shred( + &coding_shred, + &last_root + )); + } - // Trying to insert shred with num_coding == 0 should fail - { - let mut coding_shred = Shred::new_empty_from_header( - shred.clone(), - DataShredHeader::default(), - coding.clone(), - ); - coding_shred.coding_header.num_coding_shreds = 0; - assert!(!Blockstore::should_insert_coding_shred( - &coding_shred, - &last_root - )); - } + // Trying to insert with set_index with num_coding that would imply the last shred + // has index > u32::MAX should fail + { + let mut coding_shred = Shred::new_empty_from_header( + shred.clone(), + DataShredHeader::default(), + coding.clone(), + ); + coding_shred.common_header.fec_set_index = std::u32::MAX - 1; + coding_shred.coding_header.num_coding_shreds = 3; + coding_shred.common_header.index = std::u32::MAX - 1; + assert!(!Blockstore::should_insert_coding_shred( + &coding_shred, + &last_root + )); - // Trying to insert shred with pos >= num_coding should fail - { - let mut coding_shred = Shred::new_empty_from_header( - shred.clone(), - DataShredHeader::default(), - coding.clone(), - ); - let num_coding_shreds = - coding_shred.common_header.index - coding_shred.common_header.fec_set_index; - coding_shred.coding_header.num_coding_shreds = num_coding_shreds as u16; - assert!(!Blockstore::should_insert_coding_shred( - &coding_shred, - &last_root - )); - } + coding_shred.coding_header.num_coding_shreds = 2000; + assert!(!Blockstore::should_insert_coding_shred( + &coding_shred, + &last_root + )); - // Trying to insert with set_index with num_coding that would imply the last shred - // has index > u32::MAX should fail - { - let mut coding_shred = Shred::new_empty_from_header( - shred.clone(), - DataShredHeader::default(), - coding.clone(), - ); - coding_shred.common_header.fec_set_index = std::u32::MAX - 1; - coding_shred.coding_header.num_coding_shreds = 3; - coding_shred.common_header.index = std::u32::MAX - 1; - assert!(!Blockstore::should_insert_coding_shred( - &coding_shred, - &last_root - )); - - coding_shred.coding_header.num_coding_shreds = 2000; - assert!(!Blockstore::should_insert_coding_shred( - &coding_shred, - &last_root - )); - - // Decreasing the number of num_coding_shreds will put it within the allowed limit - coding_shred.coding_header.num_coding_shreds = 2; - assert!(Blockstore::should_insert_coding_shred( - &coding_shred, - &last_root - )); - - // Insertion should succeed - blockstore - .insert_shreds(vec![coding_shred], None, false) - .unwrap(); - } + // Decreasing the number of num_coding_shreds will put it within the allowed limit + coding_shred.coding_header.num_coding_shreds = 2; + assert!(Blockstore::should_insert_coding_shred( + &coding_shred, + &last_root + )); - // Trying to insert value into slot <= than last root should fail - { - let mut coding_shred = - Shred::new_empty_from_header(shred, DataShredHeader::default(), coding); - coding_shred.set_slot(*last_root.read().unwrap()); - assert!(!Blockstore::should_insert_coding_shred( - &coding_shred, - &last_root - )); - } + // Insertion should succeed + blockstore + .insert_shreds(vec![coding_shred], None, false) + .unwrap(); } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + // Trying to insert value into slot <= than last root should fail + { + let mut coding_shred = + Shred::new_empty_from_header(shred, DataShredHeader::default(), coding); + coding_shred.set_slot(*last_root.read().unwrap()); + assert!(!Blockstore::should_insert_coding_shred( + &coding_shred, + &last_root + )); + } } #[test] @@ -5900,8 +5850,8 @@ pub mod tests { solana_logger::setup(); let (shreds, _) = make_slot_entries(0, 0, 20); let num_shreds = shreds.len() as u64; - let blockstore_path = get_tmp_ledger_path!(); - let blockstore = Blockstore::open(&blockstore_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); blockstore.insert_shreds(shreds, None, false).unwrap(); let slot_meta = blockstore.meta(0).unwrap().unwrap(); @@ -5921,16 +5871,13 @@ pub mod tests { assert!(slot_meta.is_full()); assert!(blockstore.has_duplicate_shreds_in_slot(0)); - - drop(blockstore); - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] fn test_slot_data_iterator() { // Construct the shreds - let blockstore_path = get_tmp_ledger_path!(); - let blockstore = Blockstore::open(&blockstore_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); let shreds_per_slot = 10; let slots = vec![2, 4, 8, 12]; let all_shreds = make_chaining_slot_entries(&slots, shreds_per_slot); @@ -5951,15 +5898,12 @@ pub mod tests { .collect(); assert_eq!(result.len(), slot_8_shreds.len()); assert_eq!(result, slot_8_shreds); - - drop(blockstore); - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] fn test_set_roots() { - let blockstore_path = get_tmp_ledger_path!(); - let blockstore = Blockstore::open(&blockstore_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); let chained_slots = vec![0, 2, 4, 7, 12, 15]; assert_eq!(blockstore.last_root(), 0); @@ -5970,15 +5914,12 @@ pub mod tests { for i in chained_slots { assert!(blockstore.is_root(i)); } - - drop(blockstore); - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] fn test_is_skipped() { - let blockstore_path = get_tmp_ledger_path!(); - let blockstore = Blockstore::open(&blockstore_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); let roots = vec![2, 4, 7, 12, 15]; blockstore.set_roots(roots.iter()).unwrap(); @@ -5989,24 +5930,18 @@ pub mod tests { assert!(blockstore.is_skipped(i)); } } - - drop(blockstore); - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] fn test_iter_bounds() { - let blockstore_path = get_tmp_ledger_path!(); - let blockstore = Blockstore::open(&blockstore_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); // slot 5 does not exist, iter should be ok and should be a noop blockstore .slot_meta_iterator(5) .unwrap() .for_each(|_| panic!()); - - drop(blockstore); - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] @@ -6071,43 +6006,40 @@ pub mod tests { #[test] fn test_get_slot_entries_with_shred_count_corruption() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - let num_ticks = 8; - let entries = create_ticks(num_ticks, 0, Hash::default()); - let slot = 1; - let shreds = entries_to_test_shreds(entries, slot, 0, false, 0); - let next_shred_index = shreds.len(); - blockstore - .insert_shreds(shreds, None, false) - .expect("Expected successful write of shreds"); - assert_eq!( - blockstore.get_slot_entries(slot, 0).unwrap().len() as u64, - num_ticks - ); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + let num_ticks = 8; + let entries = create_ticks(num_ticks, 0, Hash::default()); + let slot = 1; + let shreds = entries_to_test_shreds(entries, slot, 0, false, 0); + let next_shred_index = shreds.len(); + blockstore + .insert_shreds(shreds, None, false) + .expect("Expected successful write of shreds"); + assert_eq!( + blockstore.get_slot_entries(slot, 0).unwrap().len() as u64, + num_ticks + ); - // Insert an empty shred that won't deshred into entries - let shreds = vec![Shred::new_from_data( - slot, - next_shred_index as u32, - 1, - Some(&[1, 1, 1]), - true, - true, - 0, - 0, - next_shred_index as u32, - )]; + // Insert an empty shred that won't deshred into entries + let shreds = vec![Shred::new_from_data( + slot, + next_shred_index as u32, + 1, + Some(&[1, 1, 1]), + true, + true, + 0, + 0, + next_shred_index as u32, + )]; - // With the corruption, nothing should be returned, even though an - // earlier data block was valid - blockstore - .insert_shreds(shreds, None, false) - .expect("Expected successful write of shreds"); - assert!(blockstore.get_slot_entries(slot, 0).is_err()); - } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + // With the corruption, nothing should be returned, even though an + // earlier data block was valid + blockstore + .insert_shreds(shreds, None, false) + .expect("Expected successful write of shreds"); + assert!(blockstore.get_slot_entries(slot, 0).is_err()); } #[test] @@ -6115,54 +6047,51 @@ pub mod tests { // This tests correctness of the SlotMeta in various cases in which a shred // that gets filtered out by checks let (shreds0, _) = make_slot_entries(0, 0, 200); - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - // Insert the first 5 shreds, we don't have a "is_last" shred yet - blockstore - .insert_shreds(shreds0[0..5].to_vec(), None, false) - .unwrap(); + // Insert the first 5 shreds, we don't have a "is_last" shred yet + blockstore + .insert_shreds(shreds0[0..5].to_vec(), None, false) + .unwrap(); - // Insert a repetitive shred for slot 's', should get ignored, but also - // insert shreds that chains to 's', should see the update in the SlotMeta - // for 's'. - let (mut shreds2, _) = make_slot_entries(2, 0, 200); - let (mut shreds3, _) = make_slot_entries(3, 0, 200); - shreds2.push(shreds0[1].clone()); - shreds3.insert(0, shreds0[1].clone()); - blockstore.insert_shreds(shreds2, None, false).unwrap(); - let slot_meta = blockstore.meta(0).unwrap().unwrap(); - assert_eq!(slot_meta.next_slots, vec![2]); - blockstore.insert_shreds(shreds3, None, false).unwrap(); - let slot_meta = blockstore.meta(0).unwrap().unwrap(); - assert_eq!(slot_meta.next_slots, vec![2, 3]); - } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + // Insert a repetitive shred for slot 's', should get ignored, but also + // insert shreds that chains to 's', should see the update in the SlotMeta + // for 's'. + let (mut shreds2, _) = make_slot_entries(2, 0, 200); + let (mut shreds3, _) = make_slot_entries(3, 0, 200); + shreds2.push(shreds0[1].clone()); + shreds3.insert(0, shreds0[1].clone()); + blockstore.insert_shreds(shreds2, None, false).unwrap(); + let slot_meta = blockstore.meta(0).unwrap().unwrap(); + assert_eq!(slot_meta.next_slots, vec![2]); + blockstore.insert_shreds(shreds3, None, false).unwrap(); + let slot_meta = blockstore.meta(0).unwrap().unwrap(); + assert_eq!(slot_meta.next_slots, vec![2, 3]); } #[test] fn test_trusted_insert_shreds() { + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + // Make shred for slot 1 let (shreds1, _) = make_slot_entries(1, 0, 1); - let blockstore_path = get_tmp_ledger_path!(); let last_root = 100; - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - blockstore.set_roots(std::iter::once(&last_root)).unwrap(); - // Insert will fail, slot < root - blockstore - .insert_shreds(shreds1[..].to_vec(), None, false) - .unwrap(); - assert!(blockstore.get_data_shred(1, 0).unwrap().is_none()); + blockstore.set_roots(std::iter::once(&last_root)).unwrap(); - // Insert through trusted path will succeed - blockstore - .insert_shreds(shreds1[..].to_vec(), None, true) - .unwrap(); - assert!(blockstore.get_data_shred(1, 0).unwrap().is_some()); - } + // Insert will fail, slot < root + blockstore + .insert_shreds(shreds1[..].to_vec(), None, false) + .unwrap(); + assert!(blockstore.get_data_shred(1, 0).unwrap().is_none()); + + // Insert through trusted path will succeed + blockstore + .insert_shreds(shreds1[..].to_vec(), None, true) + .unwrap(); + assert!(blockstore.get_data_shred(1, 0).unwrap().is_some()); } #[test] @@ -6173,12 +6102,14 @@ pub mod tests { let shreds = entries_to_test_shreds(entries.clone(), slot, slot - 1, true, 0); let more_shreds = entries_to_test_shreds(entries.clone(), slot + 1, slot, true, 0); let unrooted_shreds = entries_to_test_shreds(entries.clone(), slot + 2, slot + 1, true, 0); - let ledger_path = get_tmp_ledger_path!(); - let ledger = Blockstore::open(&ledger_path).unwrap(); - ledger.insert_shreds(shreds, None, false).unwrap(); - ledger.insert_shreds(more_shreds, None, false).unwrap(); - ledger.insert_shreds(unrooted_shreds, None, false).unwrap(); - ledger + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + blockstore.insert_shreds(shreds, None, false).unwrap(); + blockstore.insert_shreds(more_shreds, None, false).unwrap(); + blockstore + .insert_shreds(unrooted_shreds, None, false) + .unwrap(); + blockstore .set_roots(vec![slot - 1, slot, slot + 1].iter()) .unwrap(); @@ -6186,7 +6117,7 @@ pub mod tests { parent_slot: std::u64::MAX, ..SlotMeta::default() }; - ledger + blockstore .put_meta_bytes(slot - 1, &serialize(&parent_meta).unwrap()) .unwrap(); @@ -6220,7 +6151,7 @@ pub mod tests { rewards: Some(vec![]), } .into(); - ledger + blockstore .transaction_status_cf .put_protobuf((0, signature, slot), &status) .unwrap(); @@ -6236,7 +6167,7 @@ pub mod tests { rewards: Some(vec![]), } .into(); - ledger + blockstore .transaction_status_cf .put_protobuf((0, signature, slot + 1), &status) .unwrap(); @@ -6252,7 +6183,7 @@ pub mod tests { rewards: Some(vec![]), } .into(); - ledger + blockstore .transaction_status_cf .put_protobuf((0, signature, slot + 2), &status) .unwrap(); @@ -6274,19 +6205,19 @@ pub mod tests { .collect(); // Even if marked as root, a slot that is empty of entries should return an error - let confirmed_block_err = ledger.get_rooted_block(slot - 1, true).unwrap_err(); + let confirmed_block_err = blockstore.get_rooted_block(slot - 1, true).unwrap_err(); assert_matches!(confirmed_block_err, BlockstoreError::SlotUnavailable); // The previous_blockhash of `expected_block` is default because its parent slot is a root, // but empty of entries (eg. snapshot root slots). This now returns an error. - let confirmed_block_err = ledger.get_rooted_block(slot, true).unwrap_err(); + let confirmed_block_err = blockstore.get_rooted_block(slot, true).unwrap_err(); assert_matches!( confirmed_block_err, BlockstoreError::ParentEntriesUnavailable ); // Test if require_previous_blockhash is false - let confirmed_block = ledger.get_rooted_block(slot, false).unwrap(); + let confirmed_block = blockstore.get_rooted_block(slot, false).unwrap(); assert_eq!(confirmed_block.transactions.len(), 100); let expected_block = ConfirmedBlock { transactions: expected_transactions.clone(), @@ -6299,7 +6230,7 @@ pub mod tests { }; assert_eq!(confirmed_block, expected_block); - let confirmed_block = ledger.get_rooted_block(slot + 1, true).unwrap(); + let confirmed_block = blockstore.get_rooted_block(slot + 1, true).unwrap(); assert_eq!(confirmed_block.transactions.len(), 100); let mut expected_block = ConfirmedBlock { @@ -6313,10 +6244,10 @@ pub mod tests { }; assert_eq!(confirmed_block, expected_block); - let not_root = ledger.get_rooted_block(slot + 2, true).unwrap_err(); + let not_root = blockstore.get_rooted_block(slot + 2, true).unwrap_err(); assert_matches!(not_root, BlockstoreError::SlotNotRooted); - let complete_block = ledger.get_complete_block(slot + 2, true).unwrap(); + let complete_block = blockstore.get_complete_block(slot + 2, true).unwrap(); assert_eq!(complete_block.transactions.len(), 100); let mut expected_complete_block = ConfirmedBlock { @@ -6332,557 +6263,544 @@ pub mod tests { // Test block_time & block_height return, if available let timestamp = 1_576_183_541; - ledger.blocktime_cf.put(slot + 1, ×tamp).unwrap(); + blockstore.blocktime_cf.put(slot + 1, ×tamp).unwrap(); expected_block.block_time = Some(timestamp); let block_height = slot - 2; - ledger.block_height_cf.put(slot + 1, &block_height).unwrap(); + blockstore + .block_height_cf + .put(slot + 1, &block_height) + .unwrap(); expected_block.block_height = Some(block_height); - let confirmed_block = ledger.get_rooted_block(slot + 1, true).unwrap(); + let confirmed_block = blockstore.get_rooted_block(slot + 1, true).unwrap(); assert_eq!(confirmed_block, expected_block); let timestamp = 1_576_183_542; - ledger.blocktime_cf.put(slot + 2, ×tamp).unwrap(); + blockstore.blocktime_cf.put(slot + 2, ×tamp).unwrap(); expected_complete_block.block_time = Some(timestamp); let block_height = slot - 1; - ledger.block_height_cf.put(slot + 2, &block_height).unwrap(); + blockstore + .block_height_cf + .put(slot + 2, &block_height) + .unwrap(); expected_complete_block.block_height = Some(block_height); - let complete_block = ledger.get_complete_block(slot + 2, true).unwrap(); + let complete_block = blockstore.get_complete_block(slot + 2, true).unwrap(); assert_eq!(complete_block, expected_complete_block); - - drop(ledger); - Blockstore::destroy(&ledger_path).expect("Expected successful database destruction"); } #[test] fn test_persist_transaction_status() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - let transaction_status_cf = blockstore.db.column::(); - - let pre_balances_vec = vec![1, 2, 3]; - let post_balances_vec = vec![3, 2, 1]; - let inner_instructions_vec = vec![InnerInstructions { - index: 0, - instructions: vec![CompiledInstruction::new(1, &(), vec![0])], - }]; - let log_messages_vec = vec![String::from("Test message\n")]; - let pre_token_balances_vec = vec![]; - let post_token_balances_vec = vec![]; - let rewards_vec = vec![]; - - // result not found - assert!(transaction_status_cf - .get_protobuf_or_bincode::(( - 0, - Signature::default(), - 0 - )) - .unwrap() - .is_none()); - - // insert value - let status = TransactionStatusMeta { - status: solana_sdk::transaction::Result::<()>::Err( - TransactionError::AccountNotFound, - ), - fee: 5u64, - pre_balances: pre_balances_vec.clone(), - post_balances: post_balances_vec.clone(), - inner_instructions: Some(inner_instructions_vec.clone()), - log_messages: Some(log_messages_vec.clone()), - pre_token_balances: Some(pre_token_balances_vec.clone()), - post_token_balances: Some(post_token_balances_vec.clone()), - rewards: Some(rewards_vec.clone()), - } - .into(); - assert!(transaction_status_cf - .put_protobuf((0, Signature::default(), 0), &status,) - .is_ok()); - - // result found - let TransactionStatusMeta { - status, - fee, - pre_balances, - post_balances, - inner_instructions, - log_messages, - pre_token_balances, - post_token_balances, - rewards, - } = transaction_status_cf - .get_protobuf_or_bincode::(( - 0, - Signature::default(), - 0, - )) - .unwrap() - .unwrap() - .try_into() - .unwrap(); - assert_eq!(status, Err(TransactionError::AccountNotFound)); - assert_eq!(fee, 5u64); - assert_eq!(pre_balances, pre_balances_vec); - assert_eq!(post_balances, post_balances_vec); - assert_eq!(inner_instructions.unwrap(), inner_instructions_vec); - assert_eq!(log_messages.unwrap(), log_messages_vec); - assert_eq!(pre_token_balances.unwrap(), pre_token_balances_vec); - assert_eq!(post_token_balances.unwrap(), post_token_balances_vec); - assert_eq!(rewards.unwrap(), rewards_vec); - - // insert value - let status = TransactionStatusMeta { - status: solana_sdk::transaction::Result::<()>::Ok(()), - fee: 9u64, - pre_balances: pre_balances_vec.clone(), - post_balances: post_balances_vec.clone(), - inner_instructions: Some(inner_instructions_vec.clone()), - log_messages: Some(log_messages_vec.clone()), - pre_token_balances: Some(pre_token_balances_vec.clone()), - post_token_balances: Some(post_token_balances_vec.clone()), - rewards: Some(rewards_vec.clone()), - } - .into(); - assert!(transaction_status_cf - .put_protobuf((0, Signature::new(&[2u8; 64]), 9), &status,) - .is_ok()); - - // result found - let TransactionStatusMeta { - status, - fee, - pre_balances, - post_balances, - inner_instructions, - log_messages, - pre_token_balances, - post_token_balances, - rewards, - } = transaction_status_cf - .get_protobuf_or_bincode::(( - 0, - Signature::new(&[2u8; 64]), - 9, - )) - .unwrap() - .unwrap() - .try_into() - .unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + let transaction_status_cf = &blockstore.transaction_status_cf; + + let pre_balances_vec = vec![1, 2, 3]; + let post_balances_vec = vec![3, 2, 1]; + let inner_instructions_vec = vec![InnerInstructions { + index: 0, + instructions: vec![CompiledInstruction::new(1, &(), vec![0])], + }]; + let log_messages_vec = vec![String::from("Test message\n")]; + let pre_token_balances_vec = vec![]; + let post_token_balances_vec = vec![]; + let rewards_vec = vec![]; + + // result not found + assert!(transaction_status_cf + .get_protobuf_or_bincode::((0, Signature::default(), 0)) + .unwrap() + .is_none()); + + // insert value + let status = TransactionStatusMeta { + status: solana_sdk::transaction::Result::<()>::Err(TransactionError::AccountNotFound), + fee: 5u64, + pre_balances: pre_balances_vec.clone(), + post_balances: post_balances_vec.clone(), + inner_instructions: Some(inner_instructions_vec.clone()), + log_messages: Some(log_messages_vec.clone()), + pre_token_balances: Some(pre_token_balances_vec.clone()), + post_token_balances: Some(post_token_balances_vec.clone()), + rewards: Some(rewards_vec.clone()), + } + .into(); + assert!(transaction_status_cf + .put_protobuf((0, Signature::default(), 0), &status,) + .is_ok()); + + // result found + let TransactionStatusMeta { + status, + fee, + pre_balances, + post_balances, + inner_instructions, + log_messages, + pre_token_balances, + post_token_balances, + rewards, + } = transaction_status_cf + .get_protobuf_or_bincode::((0, Signature::default(), 0)) + .unwrap() + .unwrap() + .try_into() + .unwrap(); + assert_eq!(status, Err(TransactionError::AccountNotFound)); + assert_eq!(fee, 5u64); + assert_eq!(pre_balances, pre_balances_vec); + assert_eq!(post_balances, post_balances_vec); + assert_eq!(inner_instructions.unwrap(), inner_instructions_vec); + assert_eq!(log_messages.unwrap(), log_messages_vec); + assert_eq!(pre_token_balances.unwrap(), pre_token_balances_vec); + assert_eq!(post_token_balances.unwrap(), post_token_balances_vec); + assert_eq!(rewards.unwrap(), rewards_vec); + + // insert value + let status = TransactionStatusMeta { + status: solana_sdk::transaction::Result::<()>::Ok(()), + fee: 9u64, + pre_balances: pre_balances_vec.clone(), + post_balances: post_balances_vec.clone(), + inner_instructions: Some(inner_instructions_vec.clone()), + log_messages: Some(log_messages_vec.clone()), + pre_token_balances: Some(pre_token_balances_vec.clone()), + post_token_balances: Some(post_token_balances_vec.clone()), + rewards: Some(rewards_vec.clone()), + } + .into(); + assert!(transaction_status_cf + .put_protobuf((0, Signature::new(&[2u8; 64]), 9), &status,) + .is_ok()); + + // result found + let TransactionStatusMeta { + status, + fee, + pre_balances, + post_balances, + inner_instructions, + log_messages, + pre_token_balances, + post_token_balances, + rewards, + } = transaction_status_cf + .get_protobuf_or_bincode::(( + 0, + Signature::new(&[2u8; 64]), + 9, + )) + .unwrap() + .unwrap() + .try_into() + .unwrap(); - // deserialize - assert_eq!(status, Ok(())); - assert_eq!(fee, 9u64); - assert_eq!(pre_balances, pre_balances_vec); - assert_eq!(post_balances, post_balances_vec); - assert_eq!(inner_instructions.unwrap(), inner_instructions_vec); - assert_eq!(log_messages.unwrap(), log_messages_vec); - assert_eq!(pre_token_balances.unwrap(), pre_token_balances_vec); - assert_eq!(post_token_balances.unwrap(), post_token_balances_vec); - assert_eq!(rewards.unwrap(), rewards_vec); - } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + // deserialize + assert_eq!(status, Ok(())); + assert_eq!(fee, 9u64); + assert_eq!(pre_balances, pre_balances_vec); + assert_eq!(post_balances, post_balances_vec); + assert_eq!(inner_instructions.unwrap(), inner_instructions_vec); + assert_eq!(log_messages.unwrap(), log_messages_vec); + assert_eq!(pre_token_balances.unwrap(), pre_token_balances_vec); + assert_eq!(post_token_balances.unwrap(), post_token_balances_vec); + assert_eq!(rewards.unwrap(), rewards_vec); } #[test] #[allow(clippy::cognitive_complexity)] fn test_transaction_status_index() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - let transaction_status_index_cf = blockstore.db.column::(); - let slot0 = 10; - - // Primary index column is initialized on Blockstore::open - assert!(transaction_status_index_cf.get(0).unwrap().is_some()); - assert!(transaction_status_index_cf.get(1).unwrap().is_some()); - - for _ in 0..5 { - let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); - blockstore - .write_transaction_status( - slot0, - Signature::new(&random_bytes), - vec![&Pubkey::new(&random_bytes[0..32])], - vec![&Pubkey::new(&random_bytes[32..])], - TransactionStatusMeta::default(), - ) - .unwrap(); - } - - // New statuses bump index 0 max_slot - assert_eq!( - transaction_status_index_cf.get(0).unwrap().unwrap(), - TransactionStatusIndexMeta { - max_slot: slot0, - frozen: false, - } - ); - assert_eq!( - transaction_status_index_cf.get(1).unwrap().unwrap(), - TransactionStatusIndexMeta::default() - ); - - let first_status_entry = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap() - .next() - .unwrap() - .0; - assert_eq!(first_status_entry.0, 0); - assert_eq!(first_status_entry.2, slot0); - let first_address_entry = blockstore - .db - .iter::(IteratorMode::From( - cf::AddressSignatures::as_index(0), - IteratorDirection::Forward, - )) - .unwrap() - .next() - .unwrap() - .0; - assert_eq!(first_address_entry.0, 0); - assert_eq!(first_address_entry.2, slot0); - - blockstore.run_purge(0, 8, PurgeType::PrimaryIndex).unwrap(); - // First successful prune freezes index 0 - assert_eq!( - transaction_status_index_cf.get(0).unwrap().unwrap(), - TransactionStatusIndexMeta { - max_slot: slot0, - frozen: true, - } - ); - assert_eq!( - transaction_status_index_cf.get(1).unwrap().unwrap(), - TransactionStatusIndexMeta::default() - ); - - let slot1 = 20; - for _ in 0..5 { - let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); - blockstore - .write_transaction_status( - slot1, - Signature::new(&random_bytes), - vec![&Pubkey::new(&random_bytes[0..32])], - vec![&Pubkey::new(&random_bytes[32..])], - TransactionStatusMeta::default(), - ) - .unwrap(); - } + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - assert_eq!( - transaction_status_index_cf.get(0).unwrap().unwrap(), - TransactionStatusIndexMeta { - max_slot: slot0, - frozen: true, - } - ); - // Index 0 is frozen, so new statuses bump index 1 max_slot - assert_eq!( - transaction_status_index_cf.get(1).unwrap().unwrap(), - TransactionStatusIndexMeta { - max_slot: slot1, - frozen: false, - } - ); + let transaction_status_index_cf = &blockstore.transaction_status_index_cf; + let slot0 = 10; - // Index 0 statuses and address records still exist - let first_status_entry = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap() - .next() - .unwrap() - .0; - assert_eq!(first_status_entry.0, 0); - assert_eq!(first_status_entry.2, 10); - let first_address_entry = blockstore - .db - .iter::(IteratorMode::From( - cf::AddressSignatures::as_index(0), - IteratorDirection::Forward, - )) - .unwrap() - .next() - .unwrap() - .0; - assert_eq!(first_address_entry.0, 0); - assert_eq!(first_address_entry.2, slot0); - // New statuses and address records are stored in index 1 - let index1_first_status_entry = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(1), - IteratorDirection::Forward, - )) - .unwrap() - .next() - .unwrap() - .0; - assert_eq!(index1_first_status_entry.0, 1); - assert_eq!(index1_first_status_entry.2, slot1); - let index1_first_address_entry = blockstore - .db - .iter::(IteratorMode::From( - cf::AddressSignatures::as_index(1), - IteratorDirection::Forward, - )) - .unwrap() - .next() - .unwrap() - .0; - assert_eq!(index1_first_address_entry.0, 1); - assert_eq!(index1_first_address_entry.2, slot1); + // Primary index column is initialized on Blockstore::open + assert!(transaction_status_index_cf.get(0).unwrap().is_some()); + assert!(transaction_status_index_cf.get(1).unwrap().is_some()); + for _ in 0..5 { + let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); blockstore - .run_purge(0, 18, PurgeType::PrimaryIndex) + .write_transaction_status( + slot0, + Signature::new(&random_bytes), + vec![&Pubkey::new(&random_bytes[0..32])], + vec![&Pubkey::new(&random_bytes[32..])], + TransactionStatusMeta::default(), + ) .unwrap(); - // Successful prune toggles TransactionStatusIndex - assert_eq!( - transaction_status_index_cf.get(0).unwrap().unwrap(), - TransactionStatusIndexMeta { - max_slot: 0, - frozen: false, - } - ); - assert_eq!( - transaction_status_index_cf.get(1).unwrap().unwrap(), - TransactionStatusIndexMeta { - max_slot: slot1, - frozen: true, - } - ); - - // Index 0 has been pruned, so first status and address entries are now index 1 - let first_status_entry = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap() - .next() - .unwrap() - .0; - assert_eq!(first_status_entry.0, 1); - assert_eq!(first_status_entry.2, slot1); - let first_address_entry = blockstore - .db - .iter::(IteratorMode::From( - cf::AddressSignatures::as_index(0), - IteratorDirection::Forward, - )) - .unwrap() - .next() - .unwrap() - .0; - assert_eq!(first_address_entry.0, 1); - assert_eq!(first_address_entry.2, slot1); } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); - } - #[test] - fn test_get_transaction_status() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - // TransactionStatus column opens initialized with one entry at index 2 - let transaction_status_cf = blockstore.db.column::(); - - let pre_balances_vec = vec![1, 2, 3]; - let post_balances_vec = vec![3, 2, 1]; - let status = TransactionStatusMeta { - status: solana_sdk::transaction::Result::<()>::Ok(()), - fee: 42u64, - pre_balances: pre_balances_vec, - post_balances: post_balances_vec, - inner_instructions: Some(vec![]), - log_messages: Some(vec![]), - pre_token_balances: Some(vec![]), - post_token_balances: Some(vec![]), - rewards: Some(vec![]), + // New statuses bump index 0 max_slot + assert_eq!( + transaction_status_index_cf.get(0).unwrap().unwrap(), + TransactionStatusIndexMeta { + max_slot: slot0, + frozen: false, } - .into(); + ); + assert_eq!( + transaction_status_index_cf.get(1).unwrap().unwrap(), + TransactionStatusIndexMeta::default() + ); - let signature1 = Signature::new(&[1u8; 64]); - let signature2 = Signature::new(&[2u8; 64]); - let signature3 = Signature::new(&[3u8; 64]); - let signature4 = Signature::new(&[4u8; 64]); - let signature5 = Signature::new(&[5u8; 64]); - let signature6 = Signature::new(&[6u8; 64]); - let signature7 = Signature::new(&[7u8; 64]); - - // Insert slots with fork - // 0 (root) - // / \ - // 1 | - // 2 (root) - // | - // 3 - let meta0 = SlotMeta::new(0, 0); - blockstore.meta_cf.put(0, &meta0).unwrap(); - let meta1 = SlotMeta::new(1, 0); - blockstore.meta_cf.put(1, &meta1).unwrap(); - let meta2 = SlotMeta::new(2, 0); - blockstore.meta_cf.put(2, &meta2).unwrap(); - let meta3 = SlotMeta::new(3, 2); - blockstore.meta_cf.put(3, &meta3).unwrap(); - - blockstore.set_roots(vec![0, 2].iter()).unwrap(); - - // Initialize index 0, including: - // signature2 in non-root and root, - // signature4 in non-root, - // signature5 in skipped slot and non-root, - // signature6 in skipped slot, - transaction_status_cf - .put_protobuf((0, signature2, 1), &status) - .unwrap(); + let first_status_entry = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(0), + IteratorDirection::Forward, + )) + .unwrap() + .next() + .unwrap() + .0; + assert_eq!(first_status_entry.0, 0); + assert_eq!(first_status_entry.2, slot0); + let first_address_entry = blockstore + .db + .iter::(IteratorMode::From( + cf::AddressSignatures::as_index(0), + IteratorDirection::Forward, + )) + .unwrap() + .next() + .unwrap() + .0; + assert_eq!(first_address_entry.0, 0); + assert_eq!(first_address_entry.2, slot0); - transaction_status_cf - .put_protobuf((0, signature2, 2), &status) - .unwrap(); + blockstore.run_purge(0, 8, PurgeType::PrimaryIndex).unwrap(); + // First successful prune freezes index 0 + assert_eq!( + transaction_status_index_cf.get(0).unwrap().unwrap(), + TransactionStatusIndexMeta { + max_slot: slot0, + frozen: true, + } + ); + assert_eq!( + transaction_status_index_cf.get(1).unwrap().unwrap(), + TransactionStatusIndexMeta::default() + ); - transaction_status_cf - .put_protobuf((0, signature4, 1), &status) + let slot1 = 20; + for _ in 0..5 { + let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); + blockstore + .write_transaction_status( + slot1, + Signature::new(&random_bytes), + vec![&Pubkey::new(&random_bytes[0..32])], + vec![&Pubkey::new(&random_bytes[32..])], + TransactionStatusMeta::default(), + ) .unwrap(); + } - transaction_status_cf - .put_protobuf((0, signature5, 1), &status) - .unwrap(); + assert_eq!( + transaction_status_index_cf.get(0).unwrap().unwrap(), + TransactionStatusIndexMeta { + max_slot: slot0, + frozen: true, + } + ); + // Index 0 is frozen, so new statuses bump index 1 max_slot + assert_eq!( + transaction_status_index_cf.get(1).unwrap().unwrap(), + TransactionStatusIndexMeta { + max_slot: slot1, + frozen: false, + } + ); - transaction_status_cf - .put_protobuf((0, signature5, 3), &status) - .unwrap(); + // Index 0 statuses and address records still exist + let first_status_entry = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(0), + IteratorDirection::Forward, + )) + .unwrap() + .next() + .unwrap() + .0; + assert_eq!(first_status_entry.0, 0); + assert_eq!(first_status_entry.2, 10); + let first_address_entry = blockstore + .db + .iter::(IteratorMode::From( + cf::AddressSignatures::as_index(0), + IteratorDirection::Forward, + )) + .unwrap() + .next() + .unwrap() + .0; + assert_eq!(first_address_entry.0, 0); + assert_eq!(first_address_entry.2, slot0); + // New statuses and address records are stored in index 1 + let index1_first_status_entry = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(1), + IteratorDirection::Forward, + )) + .unwrap() + .next() + .unwrap() + .0; + assert_eq!(index1_first_status_entry.0, 1); + assert_eq!(index1_first_status_entry.2, slot1); + let index1_first_address_entry = blockstore + .db + .iter::(IteratorMode::From( + cf::AddressSignatures::as_index(1), + IteratorDirection::Forward, + )) + .unwrap() + .next() + .unwrap() + .0; + assert_eq!(index1_first_address_entry.0, 1); + assert_eq!(index1_first_address_entry.2, slot1); - transaction_status_cf - .put_protobuf((0, signature6, 1), &status) - .unwrap(); + blockstore + .run_purge(0, 18, PurgeType::PrimaryIndex) + .unwrap(); + // Successful prune toggles TransactionStatusIndex + assert_eq!( + transaction_status_index_cf.get(0).unwrap().unwrap(), + TransactionStatusIndexMeta { + max_slot: 0, + frozen: false, + } + ); + assert_eq!( + transaction_status_index_cf.get(1).unwrap().unwrap(), + TransactionStatusIndexMeta { + max_slot: slot1, + frozen: true, + } + ); - // Initialize index 1, including: - // signature4 in root, - // signature6 in non-root, - // signature5 extra entries - transaction_status_cf - .put_protobuf((1, signature4, 2), &status) - .unwrap(); + // Index 0 has been pruned, so first status and address entries are now index 1 + let first_status_entry = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(0), + IteratorDirection::Forward, + )) + .unwrap() + .next() + .unwrap() + .0; + assert_eq!(first_status_entry.0, 1); + assert_eq!(first_status_entry.2, slot1); + let first_address_entry = blockstore + .db + .iter::(IteratorMode::From( + cf::AddressSignatures::as_index(0), + IteratorDirection::Forward, + )) + .unwrap() + .next() + .unwrap() + .0; + assert_eq!(first_address_entry.0, 1); + assert_eq!(first_address_entry.2, slot1); + } - transaction_status_cf - .put_protobuf((1, signature5, 4), &status) - .unwrap(); + #[test] + fn test_get_transaction_status() { + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + // TransactionStatus column opens initialized with one entry at index 2 + let transaction_status_cf = &blockstore.transaction_status_cf; + + let pre_balances_vec = vec![1, 2, 3]; + let post_balances_vec = vec![3, 2, 1]; + let status = TransactionStatusMeta { + status: solana_sdk::transaction::Result::<()>::Ok(()), + fee: 42u64, + pre_balances: pre_balances_vec, + post_balances: post_balances_vec, + inner_instructions: Some(vec![]), + log_messages: Some(vec![]), + pre_token_balances: Some(vec![]), + post_token_balances: Some(vec![]), + rewards: Some(vec![]), + } + .into(); + + let signature1 = Signature::new(&[1u8; 64]); + let signature2 = Signature::new(&[2u8; 64]); + let signature3 = Signature::new(&[3u8; 64]); + let signature4 = Signature::new(&[4u8; 64]); + let signature5 = Signature::new(&[5u8; 64]); + let signature6 = Signature::new(&[6u8; 64]); + let signature7 = Signature::new(&[7u8; 64]); + + // Insert slots with fork + // 0 (root) + // / \ + // 1 | + // 2 (root) + // | + // 3 + let meta0 = SlotMeta::new(0, 0); + blockstore.meta_cf.put(0, &meta0).unwrap(); + let meta1 = SlotMeta::new(1, 0); + blockstore.meta_cf.put(1, &meta1).unwrap(); + let meta2 = SlotMeta::new(2, 0); + blockstore.meta_cf.put(2, &meta2).unwrap(); + let meta3 = SlotMeta::new(3, 2); + blockstore.meta_cf.put(3, &meta3).unwrap(); + + blockstore.set_roots(vec![0, 2].iter()).unwrap(); + + // Initialize index 0, including: + // signature2 in non-root and root, + // signature4 in non-root, + // signature5 in skipped slot and non-root, + // signature6 in skipped slot, + transaction_status_cf + .put_protobuf((0, signature2, 1), &status) + .unwrap(); - transaction_status_cf - .put_protobuf((1, signature5, 5), &status) - .unwrap(); + transaction_status_cf + .put_protobuf((0, signature2, 2), &status) + .unwrap(); - transaction_status_cf - .put_protobuf((1, signature6, 3), &status) - .unwrap(); + transaction_status_cf + .put_protobuf((0, signature4, 1), &status) + .unwrap(); - // Signature exists, root found in index 0 - if let (Some((slot, _status)), counter) = blockstore - .get_transaction_status_with_counter(signature2, &[]) - .unwrap() - { - assert_eq!(slot, 2); - assert_eq!(counter, 2); - } + transaction_status_cf + .put_protobuf((0, signature5, 1), &status) + .unwrap(); - // Signature exists, root found although not required - if let (Some((slot, _status)), counter) = blockstore - .get_transaction_status_with_counter(signature2, &[3]) - .unwrap() - { - assert_eq!(slot, 2); - assert_eq!(counter, 2); - } + transaction_status_cf + .put_protobuf((0, signature5, 3), &status) + .unwrap(); - // Signature exists, root found in index 1 - if let (Some((slot, _status)), counter) = blockstore - .get_transaction_status_with_counter(signature4, &[]) - .unwrap() - { - assert_eq!(slot, 2); - assert_eq!(counter, 3); - } + transaction_status_cf + .put_protobuf((0, signature6, 1), &status) + .unwrap(); - // Signature exists, root found although not required, in index 1 - if let (Some((slot, _status)), counter) = blockstore - .get_transaction_status_with_counter(signature4, &[3]) - .unwrap() - { - assert_eq!(slot, 2); - assert_eq!(counter, 3); - } + // Initialize index 1, including: + // signature4 in root, + // signature6 in non-root, + // signature5 extra entries + transaction_status_cf + .put_protobuf((1, signature4, 2), &status) + .unwrap(); - // Signature exists, no root found - let (status, counter) = blockstore - .get_transaction_status_with_counter(signature5, &[]) - .unwrap(); - assert_eq!(status, None); - assert_eq!(counter, 6); + transaction_status_cf + .put_protobuf((1, signature5, 4), &status) + .unwrap(); - // Signature exists, root not required - if let (Some((slot, _status)), counter) = blockstore - .get_transaction_status_with_counter(signature5, &[3]) - .unwrap() - { - assert_eq!(slot, 3); - assert_eq!(counter, 2); - } + transaction_status_cf + .put_protobuf((1, signature5, 5), &status) + .unwrap(); - // Signature does not exist, smaller than existing entries - let (status, counter) = blockstore - .get_transaction_status_with_counter(signature1, &[]) - .unwrap(); - assert_eq!(status, None); - assert_eq!(counter, 2); + transaction_status_cf + .put_protobuf((1, signature6, 3), &status) + .unwrap(); - let (status, counter) = blockstore - .get_transaction_status_with_counter(signature1, &[3]) - .unwrap(); - assert_eq!(status, None); + // Signature exists, root found in index 0 + if let (Some((slot, _status)), counter) = blockstore + .get_transaction_status_with_counter(signature2, &[]) + .unwrap() + { + assert_eq!(slot, 2); assert_eq!(counter, 2); + } - // Signature does not exist, between existing entries - let (status, counter) = blockstore - .get_transaction_status_with_counter(signature3, &[]) - .unwrap(); - assert_eq!(status, None); + // Signature exists, root found although not required + if let (Some((slot, _status)), counter) = blockstore + .get_transaction_status_with_counter(signature2, &[3]) + .unwrap() + { + assert_eq!(slot, 2); assert_eq!(counter, 2); + } - let (status, counter) = blockstore - .get_transaction_status_with_counter(signature3, &[3]) - .unwrap(); - assert_eq!(status, None); - assert_eq!(counter, 2); + // Signature exists, root found in index 1 + if let (Some((slot, _status)), counter) = blockstore + .get_transaction_status_with_counter(signature4, &[]) + .unwrap() + { + assert_eq!(slot, 2); + assert_eq!(counter, 3); + } - // Signature does not exist, larger than existing entries - let (status, counter) = blockstore - .get_transaction_status_with_counter(signature7, &[]) - .unwrap(); - assert_eq!(status, None); - assert_eq!(counter, 2); + // Signature exists, root found although not required, in index 1 + if let (Some((slot, _status)), counter) = blockstore + .get_transaction_status_with_counter(signature4, &[3]) + .unwrap() + { + assert_eq!(slot, 2); + assert_eq!(counter, 3); + } - let (status, counter) = blockstore - .get_transaction_status_with_counter(signature7, &[3]) - .unwrap(); - assert_eq!(status, None); + // Signature exists, no root found + let (status, counter) = blockstore + .get_transaction_status_with_counter(signature5, &[]) + .unwrap(); + assert_eq!(status, None); + assert_eq!(counter, 6); + + // Signature exists, root not required + if let (Some((slot, _status)), counter) = blockstore + .get_transaction_status_with_counter(signature5, &[3]) + .unwrap() + { + assert_eq!(slot, 3); assert_eq!(counter, 2); } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + + // Signature does not exist, smaller than existing entries + let (status, counter) = blockstore + .get_transaction_status_with_counter(signature1, &[]) + .unwrap(); + assert_eq!(status, None); + assert_eq!(counter, 2); + + let (status, counter) = blockstore + .get_transaction_status_with_counter(signature1, &[3]) + .unwrap(); + assert_eq!(status, None); + assert_eq!(counter, 2); + + // Signature does not exist, between existing entries + let (status, counter) = blockstore + .get_transaction_status_with_counter(signature3, &[]) + .unwrap(); + assert_eq!(status, None); + assert_eq!(counter, 2); + + let (status, counter) = blockstore + .get_transaction_status_with_counter(signature3, &[3]) + .unwrap(); + assert_eq!(status, None); + assert_eq!(counter, 2); + + // Signature does not exist, larger than existing entries + let (status, counter) = blockstore + .get_transaction_status_with_counter(signature7, &[]) + .unwrap(); + assert_eq!(status, None); + assert_eq!(counter, 2); + + let (status, counter) = blockstore + .get_transaction_status_with_counter(signature7, &[3]) + .unwrap(); + assert_eq!(status, None); + assert_eq!(counter, 2); } fn do_test_lowest_cleanup_slot_and_special_cfs( @@ -6891,144 +6809,138 @@ pub mod tests { ) { solana_logger::setup(); - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - // TransactionStatus column opens initialized with one entry at index 2 - let transaction_status_cf = blockstore.db.column::(); - - let pre_balances_vec = vec![1, 2, 3]; - let post_balances_vec = vec![3, 2, 1]; - let status = TransactionStatusMeta { - status: solana_sdk::transaction::Result::<()>::Ok(()), - fee: 42u64, - pre_balances: pre_balances_vec, - post_balances: post_balances_vec, - inner_instructions: Some(vec![]), - log_messages: Some(vec![]), - pre_token_balances: Some(vec![]), - post_token_balances: Some(vec![]), - rewards: Some(vec![]), - } - .into(); - - let signature1 = Signature::new(&[2u8; 64]); - let signature2 = Signature::new(&[3u8; 64]); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + // TransactionStatus column opens initialized with one entry at index 2 + let transaction_status_cf = &blockstore.transaction_status_cf; + + let pre_balances_vec = vec![1, 2, 3]; + let post_balances_vec = vec![3, 2, 1]; + let status = TransactionStatusMeta { + status: solana_sdk::transaction::Result::<()>::Ok(()), + fee: 42u64, + pre_balances: pre_balances_vec, + post_balances: post_balances_vec, + inner_instructions: Some(vec![]), + log_messages: Some(vec![]), + pre_token_balances: Some(vec![]), + post_token_balances: Some(vec![]), + rewards: Some(vec![]), + } + .into(); + + let signature1 = Signature::new(&[2u8; 64]); + let signature2 = Signature::new(&[3u8; 64]); + + // Insert rooted slots 0..=3 with no fork + let meta0 = SlotMeta::new(0, 0); + blockstore.meta_cf.put(0, &meta0).unwrap(); + let meta1 = SlotMeta::new(1, 0); + blockstore.meta_cf.put(1, &meta1).unwrap(); + let meta2 = SlotMeta::new(2, 1); + blockstore.meta_cf.put(2, &meta2).unwrap(); + let meta3 = SlotMeta::new(3, 2); + blockstore.meta_cf.put(3, &meta3).unwrap(); + + blockstore.set_roots(vec![0, 1, 2, 3].iter()).unwrap(); + + let lowest_cleanup_slot = 1; + let lowest_available_slot = lowest_cleanup_slot + 1; + + transaction_status_cf + .put_protobuf((0, signature1, lowest_cleanup_slot), &status) + .unwrap(); - // Insert rooted slots 0..=3 with no fork - let meta0 = SlotMeta::new(0, 0); - blockstore.meta_cf.put(0, &meta0).unwrap(); - let meta1 = SlotMeta::new(1, 0); - blockstore.meta_cf.put(1, &meta1).unwrap(); - let meta2 = SlotMeta::new(2, 1); - blockstore.meta_cf.put(2, &meta2).unwrap(); - let meta3 = SlotMeta::new(3, 2); - blockstore.meta_cf.put(3, &meta3).unwrap(); + transaction_status_cf + .put_protobuf((0, signature2, lowest_available_slot), &status) + .unwrap(); - blockstore.set_roots(vec![0, 1, 2, 3].iter()).unwrap(); + let address0 = solana_sdk::pubkey::new_rand(); + let address1 = solana_sdk::pubkey::new_rand(); + blockstore + .write_transaction_status( + lowest_cleanup_slot, + signature1, + vec![&address0], + vec![], + TransactionStatusMeta::default(), + ) + .unwrap(); + blockstore + .write_transaction_status( + lowest_available_slot, + signature2, + vec![&address1], + vec![], + TransactionStatusMeta::default(), + ) + .unwrap(); - let lowest_cleanup_slot = 1; - let lowest_available_slot = lowest_cleanup_slot + 1; + let check_for_missing = || { + ( + blockstore + .get_transaction_status_with_counter(signature1, &[]) + .unwrap() + .0 + .is_none(), + blockstore + .find_address_signatures_for_slot(address0, lowest_cleanup_slot) + .unwrap() + .is_empty(), + blockstore + .find_address_signatures(address0, lowest_cleanup_slot, lowest_cleanup_slot) + .unwrap() + .is_empty(), + ) + }; - transaction_status_cf - .put_protobuf((0, signature1, lowest_cleanup_slot), &status) - .unwrap(); + let assert_existing_always = || { + let are_existing_always = ( + blockstore + .get_transaction_status_with_counter(signature2, &[]) + .unwrap() + .0 + .is_some(), + !blockstore + .find_address_signatures_for_slot(address1, lowest_available_slot) + .unwrap() + .is_empty(), + !blockstore + .find_address_signatures(address1, lowest_available_slot, lowest_available_slot) + .unwrap() + .is_empty(), + ); + assert_eq!(are_existing_always, (true, true, true)); + }; - transaction_status_cf - .put_protobuf((0, signature2, lowest_available_slot), &status) - .unwrap(); + let are_missing = check_for_missing(); + // should never be missing before the conditional compaction & simulation... + assert_eq!(are_missing, (false, false, false)); + assert_existing_always(); - let address0 = solana_sdk::pubkey::new_rand(); - let address1 = solana_sdk::pubkey::new_rand(); - blockstore - .write_transaction_status( - lowest_cleanup_slot, - signature1, - vec![&address0], - vec![], - TransactionStatusMeta::default(), - ) - .unwrap(); + if simulate_compaction { + blockstore.set_max_expired_slot(lowest_cleanup_slot); + // force compaction filters to run across whole key range. blockstore - .write_transaction_status( - lowest_available_slot, - signature2, - vec![&address1], - vec![], - TransactionStatusMeta::default(), - ) + .compact_storage(Slot::min_value(), Slot::max_value()) .unwrap(); + } - let check_for_missing = || { - ( - blockstore - .get_transaction_status_with_counter(signature1, &[]) - .unwrap() - .0 - .is_none(), - blockstore - .find_address_signatures_for_slot(address0, lowest_cleanup_slot) - .unwrap() - .is_empty(), - blockstore - .find_address_signatures(address0, lowest_cleanup_slot, lowest_cleanup_slot) - .unwrap() - .is_empty(), - ) - }; - - let assert_existing_always = || { - let are_existing_always = ( - blockstore - .get_transaction_status_with_counter(signature2, &[]) - .unwrap() - .0 - .is_some(), - !blockstore - .find_address_signatures_for_slot(address1, lowest_available_slot) - .unwrap() - .is_empty(), - !blockstore - .find_address_signatures( - address1, - lowest_available_slot, - lowest_available_slot, - ) - .unwrap() - .is_empty(), - ); - assert_eq!(are_existing_always, (true, true, true)); - }; + if simulate_ledger_cleanup_service { + *blockstore.lowest_cleanup_slot.write().unwrap() = lowest_cleanup_slot; + } - let are_missing = check_for_missing(); - // should never be missing before the conditional compaction & simulation... + let are_missing = check_for_missing(); + if simulate_compaction || simulate_ledger_cleanup_service { + // ... when either simulation (or both) is effective, we should observe to be missing + // consistently + assert_eq!(are_missing, (true, true, true)); + } else { + // ... otherwise, we should observe to be existing... assert_eq!(are_missing, (false, false, false)); - assert_existing_always(); - - if simulate_compaction { - blockstore.set_max_expired_slot(lowest_cleanup_slot); - // force compaction filters to run across whole key range. - blockstore - .compact_storage(Slot::min_value(), Slot::max_value()) - .unwrap(); - } - - if simulate_ledger_cleanup_service { - *blockstore.lowest_cleanup_slot.write().unwrap() = lowest_cleanup_slot; - } - - let are_missing = check_for_missing(); - if simulate_compaction || simulate_ledger_cleanup_service { - // ... when either simulation (or both) is effective, we should observe to be missing - // consistently - assert_eq!(are_missing, (true, true, true)); - } else { - // ... otherwise, we should observe to be existing... - assert_eq!(are_missing, (false, false, false)); - } - assert_existing_always(); } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + assert_existing_always(); } #[test] @@ -7060,8 +6972,8 @@ pub mod tests { let slot = 2; let entries = make_slot_entries_with_transactions(5); let shreds = entries_to_test_shreds(entries.clone(), slot, slot - 1, true, 0); - let ledger_path = get_tmp_ledger_path!(); - let blockstore = Blockstore::open(&ledger_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); blockstore.insert_shreds(shreds, None, false).unwrap(); blockstore.set_roots(vec![slot - 1, slot].iter()).unwrap(); @@ -7161,11 +7073,12 @@ pub mod tests { #[test] fn test_get_complete_transaction() { + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + let slot = 2; let entries = make_slot_entries_with_transactions(5); let shreds = entries_to_test_shreds(entries.clone(), slot, slot - 1, true, 0); - let ledger_path = get_tmp_ledger_path!(); - let blockstore = Blockstore::open(&ledger_path).unwrap(); blockstore.insert_shreds(shreds, None, false).unwrap(); let expected_transactions: Vec = entries @@ -7257,8 +7170,9 @@ pub mod tests { #[test] fn test_empty_transaction_status() { - let blockstore_path = get_tmp_ledger_path!(); - let blockstore = Blockstore::open(&blockstore_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + blockstore.set_roots(std::iter::once(&0)).unwrap(); assert_eq!( blockstore @@ -7270,645 +7184,637 @@ pub mod tests { #[test] fn test_get_confirmed_signatures_for_address() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - - let address0 = solana_sdk::pubkey::new_rand(); - let address1 = solana_sdk::pubkey::new_rand(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - let slot0 = 10; - for x in 1..5 { - let signature = Signature::new(&[x; 64]); - blockstore - .write_transaction_status( - slot0, - signature, - vec![&address0], - vec![&address1], - TransactionStatusMeta::default(), - ) - .unwrap(); - } - // Purge to freeze index 0 - blockstore.run_purge(0, 1, PurgeType::PrimaryIndex).unwrap(); - let slot1 = 20; - for x in 5..9 { - let signature = Signature::new(&[x; 64]); - blockstore - .write_transaction_status( - slot1, - signature, - vec![&address0], - vec![&address1], - TransactionStatusMeta::default(), - ) - .unwrap(); - } - blockstore.set_roots(vec![slot0, slot1].iter()).unwrap(); + let address0 = solana_sdk::pubkey::new_rand(); + let address1 = solana_sdk::pubkey::new_rand(); - let all0 = blockstore - .get_confirmed_signatures_for_address(address0, 0, 50) + let slot0 = 10; + for x in 1..5 { + let signature = Signature::new(&[x; 64]); + blockstore + .write_transaction_status( + slot0, + signature, + vec![&address0], + vec![&address1], + TransactionStatusMeta::default(), + ) .unwrap(); - assert_eq!(all0.len(), 8); - for x in 1..9 { - let expected_signature = Signature::new(&[x; 64]); - assert_eq!(all0[x as usize - 1], expected_signature); - } - assert_eq!( - blockstore - .get_confirmed_signatures_for_address(address0, 20, 50) - .unwrap() - .len(), - 4 - ); - assert_eq!( - blockstore - .get_confirmed_signatures_for_address(address0, 0, 10) - .unwrap() - .len(), - 4 - ); - assert!(blockstore - .get_confirmed_signatures_for_address(address0, 1, 5) - .unwrap() - .is_empty()); - assert_eq!( - blockstore - .get_confirmed_signatures_for_address(address0, 1, 15) - .unwrap() - .len(), - 4 - ); - - let all1 = blockstore - .get_confirmed_signatures_for_address(address1, 0, 50) + } + // Purge to freeze index 0 + blockstore.run_purge(0, 1, PurgeType::PrimaryIndex).unwrap(); + let slot1 = 20; + for x in 5..9 { + let signature = Signature::new(&[x; 64]); + blockstore + .write_transaction_status( + slot1, + signature, + vec![&address0], + vec![&address1], + TransactionStatusMeta::default(), + ) .unwrap(); - assert_eq!(all1.len(), 8); - for x in 1..9 { - let expected_signature = Signature::new(&[x; 64]); - assert_eq!(all1[x as usize - 1], expected_signature); - } + } + blockstore.set_roots(vec![slot0, slot1].iter()).unwrap(); - // Purge index 0 + let all0 = blockstore + .get_confirmed_signatures_for_address(address0, 0, 50) + .unwrap(); + assert_eq!(all0.len(), 8); + for x in 1..9 { + let expected_signature = Signature::new(&[x; 64]); + assert_eq!(all0[x as usize - 1], expected_signature); + } + assert_eq!( blockstore - .run_purge(0, 10, PurgeType::PrimaryIndex) - .unwrap(); - assert_eq!( - blockstore - .get_confirmed_signatures_for_address(address0, 0, 50) - .unwrap() - .len(), - 4 - ); - assert_eq!( - blockstore - .get_confirmed_signatures_for_address(address0, 20, 50) - .unwrap() - .len(), - 4 - ); - assert!(blockstore - .get_confirmed_signatures_for_address(address0, 0, 10) + .get_confirmed_signatures_for_address(address0, 20, 50) .unwrap() - .is_empty()); - assert!(blockstore - .get_confirmed_signatures_for_address(address0, 1, 5) + .len(), + 4 + ); + assert_eq!( + blockstore + .get_confirmed_signatures_for_address(address0, 0, 10) .unwrap() - .is_empty()); - assert_eq!( - blockstore - .get_confirmed_signatures_for_address(address0, 1, 25) - .unwrap() - .len(), - 4 - ); + .len(), + 4 + ); + assert!(blockstore + .get_confirmed_signatures_for_address(address0, 1, 5) + .unwrap() + .is_empty()); + assert_eq!( + blockstore + .get_confirmed_signatures_for_address(address0, 1, 15) + .unwrap() + .len(), + 4 + ); - // Test sort, regardless of entry order or signature value - for slot in (21..25).rev() { - let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); - let signature = Signature::new(&random_bytes); - blockstore - .write_transaction_status( - slot, - signature, - vec![&address0], - vec![&address1], - TransactionStatusMeta::default(), - ) - .unwrap(); - } - blockstore.set_roots(vec![21, 22, 23, 24].iter()).unwrap(); - let mut past_slot = 0; - for (slot, _) in blockstore.find_address_signatures(address0, 1, 25).unwrap() { - assert!(slot >= past_slot); - past_slot = slot; - } + let all1 = blockstore + .get_confirmed_signatures_for_address(address1, 0, 50) + .unwrap(); + assert_eq!(all1.len(), 8); + for x in 1..9 { + let expected_signature = Signature::new(&[x; 64]); + assert_eq!(all1[x as usize - 1], expected_signature); + } + + // Purge index 0 + blockstore + .run_purge(0, 10, PurgeType::PrimaryIndex) + .unwrap(); + assert_eq!( + blockstore + .get_confirmed_signatures_for_address(address0, 0, 50) + .unwrap() + .len(), + 4 + ); + assert_eq!( + blockstore + .get_confirmed_signatures_for_address(address0, 20, 50) + .unwrap() + .len(), + 4 + ); + assert!(blockstore + .get_confirmed_signatures_for_address(address0, 0, 10) + .unwrap() + .is_empty()); + assert!(blockstore + .get_confirmed_signatures_for_address(address0, 1, 5) + .unwrap() + .is_empty()); + assert_eq!( + blockstore + .get_confirmed_signatures_for_address(address0, 1, 25) + .unwrap() + .len(), + 4 + ); + + // Test sort, regardless of entry order or signature value + for slot in (21..25).rev() { + let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); + let signature = Signature::new(&random_bytes); + blockstore + .write_transaction_status( + slot, + signature, + vec![&address0], + vec![&address1], + TransactionStatusMeta::default(), + ) + .unwrap(); + } + blockstore.set_roots(vec![21, 22, 23, 24].iter()).unwrap(); + let mut past_slot = 0; + for (slot, _) in blockstore.find_address_signatures(address0, 1, 25).unwrap() { + assert!(slot >= past_slot); + past_slot = slot; } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] fn test_find_address_signatures_for_slot() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - let address0 = solana_sdk::pubkey::new_rand(); - let address1 = solana_sdk::pubkey::new_rand(); + let address0 = solana_sdk::pubkey::new_rand(); + let address1 = solana_sdk::pubkey::new_rand(); - let slot1 = 1; - for x in 1..5 { - let signature = Signature::new(&[x; 64]); - blockstore - .write_transaction_status( - slot1, - signature, - vec![&address0], - vec![&address1], - TransactionStatusMeta::default(), - ) - .unwrap(); - } - let slot2 = 2; - for x in 5..7 { - let signature = Signature::new(&[x; 64]); - blockstore - .write_transaction_status( - slot2, - signature, - vec![&address0], - vec![&address1], - TransactionStatusMeta::default(), - ) - .unwrap(); - } - // Purge to freeze index 0 - blockstore.run_purge(0, 1, PurgeType::PrimaryIndex).unwrap(); - for x in 7..9 { - let signature = Signature::new(&[x; 64]); - blockstore - .write_transaction_status( - slot2, - signature, - vec![&address0], - vec![&address1], - TransactionStatusMeta::default(), - ) - .unwrap(); - } - let slot3 = 3; - for x in 9..13 { - let signature = Signature::new(&[x; 64]); - blockstore - .write_transaction_status( - slot3, - signature, - vec![&address0], - vec![&address1], - TransactionStatusMeta::default(), - ) - .unwrap(); - } - blockstore.set_roots(std::iter::once(&slot1)).unwrap(); - - let slot1_signatures = blockstore - .find_address_signatures_for_slot(address0, 1) + let slot1 = 1; + for x in 1..5 { + let signature = Signature::new(&[x; 64]); + blockstore + .write_transaction_status( + slot1, + signature, + vec![&address0], + vec![&address1], + TransactionStatusMeta::default(), + ) .unwrap(); - for (i, (slot, signature)) in slot1_signatures.iter().enumerate() { - assert_eq!(*slot, slot1); - assert_eq!(*signature, Signature::new(&[i as u8 + 1; 64])); - } - - let slot2_signatures = blockstore - .find_address_signatures_for_slot(address0, 2) + } + let slot2 = 2; + for x in 5..7 { + let signature = Signature::new(&[x; 64]); + blockstore + .write_transaction_status( + slot2, + signature, + vec![&address0], + vec![&address1], + TransactionStatusMeta::default(), + ) .unwrap(); - for (i, (slot, signature)) in slot2_signatures.iter().enumerate() { - assert_eq!(*slot, slot2); - assert_eq!(*signature, Signature::new(&[i as u8 + 5; 64])); - } - - let slot3_signatures = blockstore - .find_address_signatures_for_slot(address0, 3) + } + // Purge to freeze index 0 + blockstore.run_purge(0, 1, PurgeType::PrimaryIndex).unwrap(); + for x in 7..9 { + let signature = Signature::new(&[x; 64]); + blockstore + .write_transaction_status( + slot2, + signature, + vec![&address0], + vec![&address1], + TransactionStatusMeta::default(), + ) .unwrap(); - for (i, (slot, signature)) in slot3_signatures.iter().enumerate() { - assert_eq!(*slot, slot3); - assert_eq!(*signature, Signature::new(&[i as u8 + 9; 64])); - } + } + let slot3 = 3; + for x in 9..13 { + let signature = Signature::new(&[x; 64]); + blockstore + .write_transaction_status( + slot3, + signature, + vec![&address0], + vec![&address1], + TransactionStatusMeta::default(), + ) + .unwrap(); + } + blockstore.set_roots(std::iter::once(&slot1)).unwrap(); + + let slot1_signatures = blockstore + .find_address_signatures_for_slot(address0, 1) + .unwrap(); + for (i, (slot, signature)) in slot1_signatures.iter().enumerate() { + assert_eq!(*slot, slot1); + assert_eq!(*signature, Signature::new(&[i as u8 + 1; 64])); + } + + let slot2_signatures = blockstore + .find_address_signatures_for_slot(address0, 2) + .unwrap(); + for (i, (slot, signature)) in slot2_signatures.iter().enumerate() { + assert_eq!(*slot, slot2); + assert_eq!(*signature, Signature::new(&[i as u8 + 5; 64])); + } + + let slot3_signatures = blockstore + .find_address_signatures_for_slot(address0, 3) + .unwrap(); + for (i, (slot, signature)) in slot3_signatures.iter().enumerate() { + assert_eq!(*slot, slot3); + assert_eq!(*signature, Signature::new(&[i as u8 + 9; 64])); } } #[test] fn test_get_confirmed_signatures_for_address2() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - - fn make_slot_entries_with_transaction_addresses(addresses: &[Pubkey]) -> Vec { - let mut entries: Vec = Vec::new(); - for address in addresses { - let transaction = Transaction::new_with_compiled_instructions( - &[&Keypair::new()], - &[*address], - Hash::default(), - vec![solana_sdk::pubkey::new_rand()], - vec![CompiledInstruction::new(1, &(), vec![0])], - ); - entries.push(next_entry_mut(&mut Hash::default(), 0, vec![transaction])); - let mut tick = create_ticks(1, 0, hash(&serialize(address).unwrap())); - entries.append(&mut tick); - } - entries + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + fn make_slot_entries_with_transaction_addresses(addresses: &[Pubkey]) -> Vec { + let mut entries: Vec = Vec::new(); + for address in addresses { + let transaction = Transaction::new_with_compiled_instructions( + &[&Keypair::new()], + &[*address], + Hash::default(), + vec![solana_sdk::pubkey::new_rand()], + vec![CompiledInstruction::new(1, &(), vec![0])], + ); + entries.push(next_entry_mut(&mut Hash::default(), 0, vec![transaction])); + let mut tick = create_ticks(1, 0, hash(&serialize(address).unwrap())); + entries.append(&mut tick); } + entries + } - let address0 = solana_sdk::pubkey::new_rand(); - let address1 = solana_sdk::pubkey::new_rand(); + let address0 = solana_sdk::pubkey::new_rand(); + let address1 = solana_sdk::pubkey::new_rand(); - for slot in 2..=8 { - let entries = make_slot_entries_with_transaction_addresses(&[ - address0, address1, address0, address1, - ]); - let shreds = entries_to_test_shreds(entries.clone(), slot, slot - 1, true, 0); - blockstore.insert_shreds(shreds, None, false).unwrap(); + for slot in 2..=8 { + let entries = make_slot_entries_with_transaction_addresses(&[ + address0, address1, address0, address1, + ]); + let shreds = entries_to_test_shreds(entries.clone(), slot, slot - 1, true, 0); + blockstore.insert_shreds(shreds, None, false).unwrap(); - for (i, entry) in entries.into_iter().enumerate() { - if slot == 4 && i == 2 { - // Purge to freeze index 0 and write address-signatures in new primary index - blockstore.run_purge(0, 1, PurgeType::PrimaryIndex).unwrap(); - } - for tx in entry.transactions { - let transaction = tx - .into_legacy_transaction() - .expect("versioned transactions not supported"); - assert_eq!(transaction.signatures.len(), 1); - blockstore - .write_transaction_status( - slot, - transaction.signatures[0], - transaction.message.account_keys.iter().collect(), - vec![], - TransactionStatusMeta::default(), - ) - .unwrap(); - } + for (i, entry) in entries.into_iter().enumerate() { + if slot == 4 && i == 2 { + // Purge to freeze index 0 and write address-signatures in new primary index + blockstore.run_purge(0, 1, PurgeType::PrimaryIndex).unwrap(); + } + for tx in entry.transactions { + let transaction = tx + .into_legacy_transaction() + .expect("versioned transactions not supported"); + assert_eq!(transaction.signatures.len(), 1); + blockstore + .write_transaction_status( + slot, + transaction.signatures[0], + transaction.message.account_keys.iter().collect(), + vec![], + TransactionStatusMeta::default(), + ) + .unwrap(); } } + } - // Add 2 slots that both descend from slot 8 - for slot in 9..=10 { - let entries = make_slot_entries_with_transaction_addresses(&[ - address0, address1, address0, address1, - ]); - let shreds = entries_to_test_shreds(entries.clone(), slot, 8, true, 0); - blockstore.insert_shreds(shreds, None, false).unwrap(); - - for entry in entries.into_iter() { - for tx in entry.transactions { - let transaction = tx - .into_legacy_transaction() - .expect("versioned transactions not supported"); - assert_eq!(transaction.signatures.len(), 1); - blockstore - .write_transaction_status( - slot, - transaction.signatures[0], - transaction.message.account_keys.iter().collect(), - vec![], - TransactionStatusMeta::default(), - ) - .unwrap(); - } + // Add 2 slots that both descend from slot 8 + for slot in 9..=10 { + let entries = make_slot_entries_with_transaction_addresses(&[ + address0, address1, address0, address1, + ]); + let shreds = entries_to_test_shreds(entries.clone(), slot, 8, true, 0); + blockstore.insert_shreds(shreds, None, false).unwrap(); + + for entry in entries.into_iter() { + for tx in entry.transactions { + let transaction = tx + .into_legacy_transaction() + .expect("versioned transactions not supported"); + assert_eq!(transaction.signatures.len(), 1); + blockstore + .write_transaction_status( + slot, + transaction.signatures[0], + transaction.message.account_keys.iter().collect(), + vec![], + TransactionStatusMeta::default(), + ) + .unwrap(); } } + } - // Leave one slot unrooted to test only returns confirmed signatures - blockstore - .set_roots(vec![1, 2, 4, 5, 6, 7, 8].iter()) - .unwrap(); - let highest_confirmed_root = 8; + // Leave one slot unrooted to test only returns confirmed signatures + blockstore + .set_roots(vec![1, 2, 4, 5, 6, 7, 8].iter()) + .unwrap(); + let highest_confirmed_root = 8; - // Fetch all rooted signatures for address 0 at once... - let all0 = blockstore - .get_confirmed_signatures_for_address2( - address0, - highest_confirmed_root, - None, - None, - usize::MAX, - ) - .unwrap(); - assert_eq!(all0.len(), 12); + // Fetch all rooted signatures for address 0 at once... + let all0 = blockstore + .get_confirmed_signatures_for_address2( + address0, + highest_confirmed_root, + None, + None, + usize::MAX, + ) + .unwrap(); + assert_eq!(all0.len(), 12); - // Fetch all rooted signatures for address 1 at once... - let all1 = blockstore - .get_confirmed_signatures_for_address2( - address1, - highest_confirmed_root, - None, - None, - usize::MAX, - ) - .unwrap(); - assert_eq!(all1.len(), 12); - - // Fetch all signatures for address 0 individually - for i in 0..all0.len() { - let results = blockstore - .get_confirmed_signatures_for_address2( - address0, - highest_confirmed_root, - if i == 0 { - None - } else { - Some(all0[i - 1].signature) - }, - None, - 1, - ) - .unwrap(); - assert_eq!(results.len(), 1); - assert_eq!(results[0], all0[i], "Unexpected result for {}", i); - } - // Fetch all signatures for address 0 individually using `until` - for i in 0..all0.len() { - let results = blockstore - .get_confirmed_signatures_for_address2( - address0, - highest_confirmed_root, - if i == 0 { - None - } else { - Some(all0[i - 1].signature) - }, - if i == all0.len() - 1 || i == all0.len() { - None - } else { - Some(all0[i + 1].signature) - }, - 10, - ) - .unwrap(); - assert_eq!(results.len(), 1); - assert_eq!(results[0], all0[i], "Unexpected result for {}", i); - } + // Fetch all rooted signatures for address 1 at once... + let all1 = blockstore + .get_confirmed_signatures_for_address2( + address1, + highest_confirmed_root, + None, + None, + usize::MAX, + ) + .unwrap(); + assert_eq!(all1.len(), 12); - assert!(blockstore + // Fetch all signatures for address 0 individually + for i in 0..all0.len() { + let results = blockstore .get_confirmed_signatures_for_address2( address0, highest_confirmed_root, - Some(all0[all0.len() - 1].signature), + if i == 0 { + None + } else { + Some(all0[i - 1].signature) + }, None, 1, ) - .unwrap() - .is_empty()); - - assert!(blockstore + .unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0], all0[i], "Unexpected result for {}", i); + } + // Fetch all signatures for address 0 individually using `until` + for i in 0..all0.len() { + let results = blockstore .get_confirmed_signatures_for_address2( address0, highest_confirmed_root, - None, - Some(all0[0].signature), - 2, + if i == 0 { + None + } else { + Some(all0[i - 1].signature) + }, + if i == all0.len() - 1 || i == all0.len() { + None + } else { + Some(all0[i + 1].signature) + }, + 10, ) - .unwrap() - .is_empty()); - - // Fetch all signatures for address 0, three at a time - assert!(all0.len() % 3 == 0); - for i in (0..all0.len()).step_by(3) { - let results = blockstore - .get_confirmed_signatures_for_address2( - address0, - highest_confirmed_root, - if i == 0 { - None - } else { - Some(all0[i - 1].signature) - }, - None, - 3, - ) - .unwrap(); - assert_eq!(results.len(), 3); - assert_eq!(results[0], all0[i]); - assert_eq!(results[1], all0[i + 1]); - assert_eq!(results[2], all0[i + 2]); - } + .unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0], all0[i], "Unexpected result for {}", i); + } - // Ensure that the signatures within a slot are reverse ordered by signature - // (current limitation of the .get_confirmed_signatures_for_address2()) - for i in (0..all1.len()).step_by(2) { - let results = blockstore - .get_confirmed_signatures_for_address2( - address1, - highest_confirmed_root, - if i == 0 { - None - } else { - Some(all1[i - 1].signature) - }, - None, - 2, - ) - .unwrap(); - assert_eq!(results.len(), 2); - assert_eq!(results[0].slot, results[1].slot); - assert!(results[0].signature >= results[1].signature); - assert_eq!(results[0], all1[i]); - assert_eq!(results[1], all1[i + 1]); - } + assert!(blockstore + .get_confirmed_signatures_for_address2( + address0, + highest_confirmed_root, + Some(all0[all0.len() - 1].signature), + None, + 1, + ) + .unwrap() + .is_empty()); - // A search for address 0 with `before` and/or `until` signatures from address1 should also work + assert!(blockstore + .get_confirmed_signatures_for_address2( + address0, + highest_confirmed_root, + None, + Some(all0[0].signature), + 2, + ) + .unwrap() + .is_empty()); + + // Fetch all signatures for address 0, three at a time + assert!(all0.len() % 3 == 0); + for i in (0..all0.len()).step_by(3) { let results = blockstore .get_confirmed_signatures_for_address2( address0, highest_confirmed_root, - Some(all1[0].signature), + if i == 0 { + None + } else { + Some(all0[i - 1].signature) + }, None, - usize::MAX, + 3, ) .unwrap(); - // The exact number of results returned is variable, based on the sort order of the - // random signatures that are generated - assert!(!results.is_empty()); + assert_eq!(results.len(), 3); + assert_eq!(results[0], all0[i]); + assert_eq!(results[1], all0[i + 1]); + assert_eq!(results[2], all0[i + 2]); + } - let results2 = blockstore + // Ensure that the signatures within a slot are reverse ordered by signature + // (current limitation of the .get_confirmed_signatures_for_address2()) + for i in (0..all1.len()).step_by(2) { + let results = blockstore .get_confirmed_signatures_for_address2( - address0, + address1, highest_confirmed_root, - Some(all1[0].signature), - Some(all1[4].signature), - usize::MAX, + if i == 0 { + None + } else { + Some(all1[i - 1].signature) + }, + None, + 2, ) .unwrap(); - assert!(results2.len() < results.len()); + assert_eq!(results.len(), 2); + assert_eq!(results[0].slot, results[1].slot); + assert!(results[0].signature >= results[1].signature); + assert_eq!(results[0], all1[i]); + assert_eq!(results[1], all1[i + 1]); + } + + // A search for address 0 with `before` and/or `until` signatures from address1 should also work + let results = blockstore + .get_confirmed_signatures_for_address2( + address0, + highest_confirmed_root, + Some(all1[0].signature), + None, + usize::MAX, + ) + .unwrap(); + // The exact number of results returned is variable, based on the sort order of the + // random signatures that are generated + assert!(!results.is_empty()); + + let results2 = blockstore + .get_confirmed_signatures_for_address2( + address0, + highest_confirmed_root, + Some(all1[0].signature), + Some(all1[4].signature), + usize::MAX, + ) + .unwrap(); + assert!(results2.len() < results.len()); - // Duplicate all tests using confirmed signatures - let highest_confirmed_slot = 10; + // Duplicate all tests using confirmed signatures + let highest_confirmed_slot = 10; - // Fetch all signatures for address 0 at once... - let all0 = blockstore - .get_confirmed_signatures_for_address2( - address0, - highest_confirmed_slot, - None, - None, - usize::MAX, - ) - .unwrap(); - assert_eq!(all0.len(), 14); + // Fetch all signatures for address 0 at once... + let all0 = blockstore + .get_confirmed_signatures_for_address2( + address0, + highest_confirmed_slot, + None, + None, + usize::MAX, + ) + .unwrap(); + assert_eq!(all0.len(), 14); - // Fetch all signatures for address 1 at once... - let all1 = blockstore - .get_confirmed_signatures_for_address2( - address1, - highest_confirmed_slot, - None, - None, - usize::MAX, - ) - .unwrap(); - assert_eq!(all1.len(), 14); - - // Fetch all signatures for address 0 individually - for i in 0..all0.len() { - let results = blockstore - .get_confirmed_signatures_for_address2( - address0, - highest_confirmed_slot, - if i == 0 { - None - } else { - Some(all0[i - 1].signature) - }, - None, - 1, - ) - .unwrap(); - assert_eq!(results.len(), 1); - assert_eq!(results[0], all0[i], "Unexpected result for {}", i); - } - // Fetch all signatures for address 0 individually using `until` - for i in 0..all0.len() { - let results = blockstore - .get_confirmed_signatures_for_address2( - address0, - highest_confirmed_slot, - if i == 0 { - None - } else { - Some(all0[i - 1].signature) - }, - if i == all0.len() - 1 || i == all0.len() { - None - } else { - Some(all0[i + 1].signature) - }, - 10, - ) - .unwrap(); - assert_eq!(results.len(), 1); - assert_eq!(results[0], all0[i], "Unexpected result for {}", i); - } + // Fetch all signatures for address 1 at once... + let all1 = blockstore + .get_confirmed_signatures_for_address2( + address1, + highest_confirmed_slot, + None, + None, + usize::MAX, + ) + .unwrap(); + assert_eq!(all1.len(), 14); - assert!(blockstore + // Fetch all signatures for address 0 individually + for i in 0..all0.len() { + let results = blockstore .get_confirmed_signatures_for_address2( address0, highest_confirmed_slot, - Some(all0[all0.len() - 1].signature), + if i == 0 { + None + } else { + Some(all0[i - 1].signature) + }, None, 1, ) - .unwrap() - .is_empty()); - - assert!(blockstore + .unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0], all0[i], "Unexpected result for {}", i); + } + // Fetch all signatures for address 0 individually using `until` + for i in 0..all0.len() { + let results = blockstore .get_confirmed_signatures_for_address2( address0, highest_confirmed_slot, - None, - Some(all0[0].signature), - 2, + if i == 0 { + None + } else { + Some(all0[i - 1].signature) + }, + if i == all0.len() - 1 || i == all0.len() { + None + } else { + Some(all0[i + 1].signature) + }, + 10, ) - .unwrap() - .is_empty()); - - // Fetch all signatures for address 0, three at a time - assert!(all0.len() % 3 == 2); - for i in (0..all0.len()).step_by(3) { - let results = blockstore - .get_confirmed_signatures_for_address2( - address0, - highest_confirmed_slot, - if i == 0 { - None - } else { - Some(all0[i - 1].signature) - }, - None, - 3, - ) - .unwrap(); - if i < 12 { - assert_eq!(results.len(), 3); - assert_eq!(results[2], all0[i + 2]); - } else { - assert_eq!(results.len(), 2); - } - assert_eq!(results[0], all0[i]); - assert_eq!(results[1], all0[i + 1]); - } + .unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0], all0[i], "Unexpected result for {}", i); + } - // Ensure that the signatures within a slot are reverse ordered by signature - // (current limitation of the .get_confirmed_signatures_for_address2()) - for i in (0..all1.len()).step_by(2) { - let results = blockstore - .get_confirmed_signatures_for_address2( - address1, - highest_confirmed_slot, - if i == 0 { - None - } else { - Some(all1[i - 1].signature) - }, - None, - 2, - ) - .unwrap(); - assert_eq!(results.len(), 2); - assert_eq!(results[0].slot, results[1].slot); - assert!(results[0].signature >= results[1].signature); - assert_eq!(results[0], all1[i]); - assert_eq!(results[1], all1[i + 1]); - } + assert!(blockstore + .get_confirmed_signatures_for_address2( + address0, + highest_confirmed_slot, + Some(all0[all0.len() - 1].signature), + None, + 1, + ) + .unwrap() + .is_empty()); + + assert!(blockstore + .get_confirmed_signatures_for_address2( + address0, + highest_confirmed_slot, + None, + Some(all0[0].signature), + 2, + ) + .unwrap() + .is_empty()); - // A search for address 0 with `before` and/or `until` signatures from address1 should also work + // Fetch all signatures for address 0, three at a time + assert!(all0.len() % 3 == 2); + for i in (0..all0.len()).step_by(3) { let results = blockstore .get_confirmed_signatures_for_address2( address0, highest_confirmed_slot, - Some(all1[0].signature), + if i == 0 { + None + } else { + Some(all0[i - 1].signature) + }, None, - usize::MAX, + 3, ) .unwrap(); - // The exact number of results returned is variable, based on the sort order of the - // random signatures that are generated - assert!(!results.is_empty()); + if i < 12 { + assert_eq!(results.len(), 3); + assert_eq!(results[2], all0[i + 2]); + } else { + assert_eq!(results.len(), 2); + } + assert_eq!(results[0], all0[i]); + assert_eq!(results[1], all0[i + 1]); + } - let results2 = blockstore + // Ensure that the signatures within a slot are reverse ordered by signature + // (current limitation of the .get_confirmed_signatures_for_address2()) + for i in (0..all1.len()).step_by(2) { + let results = blockstore .get_confirmed_signatures_for_address2( - address0, + address1, highest_confirmed_slot, - Some(all1[0].signature), - Some(all1[4].signature), - usize::MAX, + if i == 0 { + None + } else { + Some(all1[i - 1].signature) + }, + None, + 2, ) .unwrap(); - assert!(results2.len() < results.len()); - } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + assert_eq!(results.len(), 2); + assert_eq!(results[0].slot, results[1].slot); + assert!(results[0].signature >= results[1].signature); + assert_eq!(results[0], all1[i]); + assert_eq!(results[1], all1[i + 1]); + } + + // A search for address 0 with `before` and/or `until` signatures from address1 should also work + let results = blockstore + .get_confirmed_signatures_for_address2( + address0, + highest_confirmed_slot, + Some(all1[0].signature), + None, + usize::MAX, + ) + .unwrap(); + // The exact number of results returned is variable, based on the sort order of the + // random signatures that are generated + assert!(!results.is_empty()); + + let results2 = blockstore + .get_confirmed_signatures_for_address2( + address0, + highest_confirmed_slot, + Some(all1[0].signature), + Some(all1[4].signature), + usize::MAX, + ) + .unwrap(); + assert!(results2.len() < results.len()); } #[test] @@ -7930,144 +7836,136 @@ pub mod tests { #[test] fn test_map_transactions_to_statuses() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - let transaction_status_cf = blockstore.db.column::(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - let slot = 0; - let mut transactions: Vec = vec![]; - for x in 0..4 { - let transaction = Transaction::new_with_compiled_instructions( - &[&Keypair::new()], - &[solana_sdk::pubkey::new_rand()], - Hash::default(), - vec![solana_sdk::pubkey::new_rand()], - vec![CompiledInstruction::new(1, &(), vec![0])], - ); - let status = TransactionStatusMeta { - status: solana_sdk::transaction::Result::<()>::Err( - TransactionError::AccountNotFound, - ), - fee: x, - pre_balances: vec![], - post_balances: vec![], - inner_instructions: Some(vec![]), - log_messages: Some(vec![]), - pre_token_balances: Some(vec![]), - post_token_balances: Some(vec![]), - rewards: Some(vec![]), - } - .into(); - transaction_status_cf - .put_protobuf((0, transaction.signatures[0], slot), &status) - .unwrap(); - transactions.push(transaction.into()); - } - // Push transaction that will not have matching status, as a test case - transactions.push( - Transaction::new_with_compiled_instructions( - &[&Keypair::new()], - &[solana_sdk::pubkey::new_rand()], - Hash::default(), - vec![solana_sdk::pubkey::new_rand()], - vec![CompiledInstruction::new(1, &(), vec![0])], - ) - .into(), - ); + let transaction_status_cf = &blockstore.transaction_status_cf; - let map_result = - blockstore.map_transactions_to_statuses(slot, transactions.into_iter()); - assert!(map_result.is_ok()); - let map = map_result.unwrap(); - assert_eq!(map.len(), 5); - for (x, m) in map.iter().take(4).enumerate() { - assert_eq!(m.meta.as_ref().unwrap().fee, x as u64); + let slot = 0; + let mut transactions: Vec = vec![]; + for x in 0..4 { + let transaction = Transaction::new_with_compiled_instructions( + &[&Keypair::new()], + &[solana_sdk::pubkey::new_rand()], + Hash::default(), + vec![solana_sdk::pubkey::new_rand()], + vec![CompiledInstruction::new(1, &(), vec![0])], + ); + let status = TransactionStatusMeta { + status: solana_sdk::transaction::Result::<()>::Err( + TransactionError::AccountNotFound, + ), + fee: x, + pre_balances: vec![], + post_balances: vec![], + inner_instructions: Some(vec![]), + log_messages: Some(vec![]), + pre_token_balances: Some(vec![]), + post_token_balances: Some(vec![]), + rewards: Some(vec![]), } - assert_eq!(map[4].meta, None); + .into(); + transaction_status_cf + .put_protobuf((0, transaction.signatures[0], slot), &status) + .unwrap(); + transactions.push(transaction.into()); + } + // Push transaction that will not have matching status, as a test case + transactions.push( + Transaction::new_with_compiled_instructions( + &[&Keypair::new()], + &[solana_sdk::pubkey::new_rand()], + Hash::default(), + vec![solana_sdk::pubkey::new_rand()], + vec![CompiledInstruction::new(1, &(), vec![0])], + ) + .into(), + ); + + let map_result = blockstore.map_transactions_to_statuses(slot, transactions.into_iter()); + assert!(map_result.is_ok()); + let map = map_result.unwrap(); + assert_eq!(map.len(), 5); + for (x, m) in map.iter().take(4).enumerate() { + assert_eq!(m.meta.as_ref().unwrap().fee, x as u64); } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + assert_eq!(map[4].meta, None); } #[test] fn test_write_get_perf_samples() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - let num_entries: usize = 10; - let mut perf_samples: Vec<(Slot, PerfSample)> = vec![]; - for x in 1..num_entries + 1 { - perf_samples.push(( - x as u64 * 50, - PerfSample { - num_transactions: 1000 + x as u64, - num_slots: 50, - sample_period_secs: 20, - }, - )); - } - for (slot, sample) in perf_samples.iter() { - blockstore.write_perf_sample(*slot, sample).unwrap(); - } - for x in 0..num_entries { - let mut expected_samples = perf_samples[num_entries - 1 - x..].to_vec(); - expected_samples.sort_by(|a, b| b.0.cmp(&a.0)); - assert_eq!( - blockstore.get_recent_perf_samples(x + 1).unwrap(), - expected_samples - ); - } + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + let num_entries: usize = 10; + let mut perf_samples: Vec<(Slot, PerfSample)> = vec![]; + for x in 1..num_entries + 1 { + perf_samples.push(( + x as u64 * 50, + PerfSample { + num_transactions: 1000 + x as u64, + num_slots: 50, + sample_period_secs: 20, + }, + )); + } + for (slot, sample) in perf_samples.iter() { + blockstore.write_perf_sample(*slot, sample).unwrap(); + } + for x in 0..num_entries { + let mut expected_samples = perf_samples[num_entries - 1 - x..].to_vec(); + expected_samples.sort_by(|a, b| b.0.cmp(&a.0)); + assert_eq!( + blockstore.get_recent_perf_samples(x + 1).unwrap(), + expected_samples + ); } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] fn test_lowest_slot() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - for i in 0..10 { - let slot = i; - let (shreds, _) = make_slot_entries(slot, 0, 1); - blockstore.insert_shreds(shreds, None, false).unwrap(); - } - assert_eq!(blockstore.lowest_slot(), 1); - blockstore.run_purge(0, 5, PurgeType::PrimaryIndex).unwrap(); - assert_eq!(blockstore.lowest_slot(), 6); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + for i in 0..10 { + let slot = i; + let (shreds, _) = make_slot_entries(slot, 0, 1); + blockstore.insert_shreds(shreds, None, false).unwrap(); } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + assert_eq!(blockstore.lowest_slot(), 1); + blockstore.run_purge(0, 5, PurgeType::PrimaryIndex).unwrap(); + assert_eq!(blockstore.lowest_slot(), 6); } #[test] fn test_recovery() { + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + let slot = 1; let (data_shreds, coding_shreds, leader_schedule_cache) = setup_erasure_shreds(slot, 0, 100); - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - blockstore - .insert_shreds(coding_shreds, Some(&leader_schedule_cache), false) - .unwrap(); - let shred_bufs: Vec<_> = data_shreds - .iter() - .map(|shred| shred.payload.clone()) - .collect(); - // Check all the data shreds were recovered - for (s, buf) in data_shreds.iter().zip(shred_bufs) { - assert_eq!( - blockstore - .get_data_shred(s.slot(), s.index() as u64) - .unwrap() - .unwrap(), - buf - ); - } + blockstore + .insert_shreds(coding_shreds, Some(&leader_schedule_cache), false) + .unwrap(); + let shred_bufs: Vec<_> = data_shreds + .iter() + .map(|shred| shred.payload.clone()) + .collect(); - verify_index_integrity(&blockstore, slot); + // Check all the data shreds were recovered + for (s, buf) in data_shreds.iter().zip(shred_bufs) { + assert_eq!( + blockstore + .get_data_shred(s.slot(), s.index() as u64) + .unwrap() + .unwrap(), + buf + ); } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + + verify_index_integrity(&blockstore, slot); } #[test] @@ -8078,134 +7976,133 @@ pub mod tests { setup_erasure_shreds(slot, 0, num_entries); assert!(data_shreds.len() > 3); assert!(coding_shreds.len() > 3); - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - // Test inserting all the shreds - let all_shreds: Vec<_> = data_shreds - .iter() - .cloned() - .chain(coding_shreds.iter().cloned()) - .collect(); - blockstore - .insert_shreds(all_shreds, Some(&leader_schedule_cache), false) - .unwrap(); - verify_index_integrity(&blockstore, slot); - blockstore.purge_and_compact_slots(0, slot); - // Test inserting just the codes, enough for recovery - blockstore - .insert_shreds(coding_shreds.clone(), Some(&leader_schedule_cache), false) - .unwrap(); - verify_index_integrity(&blockstore, slot); - blockstore.purge_and_compact_slots(0, slot); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - // Test inserting some codes, but not enough for recovery - blockstore - .insert_shreds( - coding_shreds[..coding_shreds.len() - 1].to_vec(), - Some(&leader_schedule_cache), - false, - ) - .unwrap(); - verify_index_integrity(&blockstore, slot); - blockstore.purge_and_compact_slots(0, slot); + // Test inserting all the shreds + let all_shreds: Vec<_> = data_shreds + .iter() + .cloned() + .chain(coding_shreds.iter().cloned()) + .collect(); + blockstore + .insert_shreds(all_shreds, Some(&leader_schedule_cache), false) + .unwrap(); + verify_index_integrity(&blockstore, slot); + blockstore.purge_and_compact_slots(0, slot); - // Test inserting just the codes, and some data, enough for recovery - let shreds: Vec<_> = data_shreds[..data_shreds.len() - 1] - .iter() - .cloned() - .chain(coding_shreds[..coding_shreds.len() - 1].iter().cloned()) - .collect(); - blockstore - .insert_shreds(shreds, Some(&leader_schedule_cache), false) - .unwrap(); - verify_index_integrity(&blockstore, slot); - blockstore.purge_and_compact_slots(0, slot); + // Test inserting just the codes, enough for recovery + blockstore + .insert_shreds(coding_shreds.clone(), Some(&leader_schedule_cache), false) + .unwrap(); + verify_index_integrity(&blockstore, slot); + blockstore.purge_and_compact_slots(0, slot); - // Test inserting some codes, and some data, but enough for recovery - let shreds: Vec<_> = data_shreds[..data_shreds.len() / 2 - 1] - .iter() - .cloned() - .chain(coding_shreds[..coding_shreds.len() / 2 - 1].iter().cloned()) - .collect(); - blockstore - .insert_shreds(shreds, Some(&leader_schedule_cache), false) - .unwrap(); - verify_index_integrity(&blockstore, slot); - blockstore.purge_and_compact_slots(0, slot); + // Test inserting some codes, but not enough for recovery + blockstore + .insert_shreds( + coding_shreds[..coding_shreds.len() - 1].to_vec(), + Some(&leader_schedule_cache), + false, + ) + .unwrap(); + verify_index_integrity(&blockstore, slot); + blockstore.purge_and_compact_slots(0, slot); - // Test inserting all shreds in 2 rounds, make sure nothing is lost - let shreds1: Vec<_> = data_shreds[..data_shreds.len() / 2 - 1] - .iter() - .cloned() - .chain(coding_shreds[..coding_shreds.len() / 2 - 1].iter().cloned()) - .collect(); - let shreds2: Vec<_> = data_shreds[data_shreds.len() / 2 - 1..] - .iter() - .cloned() - .chain(coding_shreds[coding_shreds.len() / 2 - 1..].iter().cloned()) - .collect(); - blockstore - .insert_shreds(shreds1, Some(&leader_schedule_cache), false) - .unwrap(); - blockstore - .insert_shreds(shreds2, Some(&leader_schedule_cache), false) - .unwrap(); - verify_index_integrity(&blockstore, slot); - blockstore.purge_and_compact_slots(0, slot); + // Test inserting just the codes, and some data, enough for recovery + let shreds: Vec<_> = data_shreds[..data_shreds.len() - 1] + .iter() + .cloned() + .chain(coding_shreds[..coding_shreds.len() - 1].iter().cloned()) + .collect(); + blockstore + .insert_shreds(shreds, Some(&leader_schedule_cache), false) + .unwrap(); + verify_index_integrity(&blockstore, slot); + blockstore.purge_and_compact_slots(0, slot); - // Test not all, but enough data and coding shreds in 2 rounds to trigger recovery, - // make sure nothing is lost - let shreds1: Vec<_> = data_shreds[..data_shreds.len() / 2 - 1] - .iter() - .cloned() - .chain(coding_shreds[..coding_shreds.len() / 2 - 1].iter().cloned()) - .collect(); - let shreds2: Vec<_> = data_shreds[data_shreds.len() / 2 - 1..data_shreds.len() / 2] - .iter() - .cloned() - .chain( - coding_shreds[coding_shreds.len() / 2 - 1..coding_shreds.len() / 2] - .iter() - .cloned(), - ) - .collect(); - blockstore - .insert_shreds(shreds1, Some(&leader_schedule_cache), false) - .unwrap(); - blockstore - .insert_shreds(shreds2, Some(&leader_schedule_cache), false) - .unwrap(); - verify_index_integrity(&blockstore, slot); - blockstore.purge_and_compact_slots(0, slot); + // Test inserting some codes, and some data, but enough for recovery + let shreds: Vec<_> = data_shreds[..data_shreds.len() / 2 - 1] + .iter() + .cloned() + .chain(coding_shreds[..coding_shreds.len() / 2 - 1].iter().cloned()) + .collect(); + blockstore + .insert_shreds(shreds, Some(&leader_schedule_cache), false) + .unwrap(); + verify_index_integrity(&blockstore, slot); + blockstore.purge_and_compact_slots(0, slot); - // Test insert shreds in 2 rounds, but not enough to trigger - // recovery, make sure nothing is lost - let shreds1: Vec<_> = data_shreds[..data_shreds.len() / 2 - 2] - .iter() - .cloned() - .chain(coding_shreds[..coding_shreds.len() / 2 - 2].iter().cloned()) - .collect(); - let shreds2: Vec<_> = data_shreds[data_shreds.len() / 2 - 2..data_shreds.len() / 2 - 1] - .iter() - .cloned() - .chain( - coding_shreds[coding_shreds.len() / 2 - 2..coding_shreds.len() / 2 - 1] - .iter() - .cloned(), - ) - .collect(); - blockstore - .insert_shreds(shreds1, Some(&leader_schedule_cache), false) - .unwrap(); - blockstore - .insert_shreds(shreds2, Some(&leader_schedule_cache), false) - .unwrap(); - verify_index_integrity(&blockstore, slot); - blockstore.purge_and_compact_slots(0, slot); - } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + // Test inserting all shreds in 2 rounds, make sure nothing is lost + let shreds1: Vec<_> = data_shreds[..data_shreds.len() / 2 - 1] + .iter() + .cloned() + .chain(coding_shreds[..coding_shreds.len() / 2 - 1].iter().cloned()) + .collect(); + let shreds2: Vec<_> = data_shreds[data_shreds.len() / 2 - 1..] + .iter() + .cloned() + .chain(coding_shreds[coding_shreds.len() / 2 - 1..].iter().cloned()) + .collect(); + blockstore + .insert_shreds(shreds1, Some(&leader_schedule_cache), false) + .unwrap(); + blockstore + .insert_shreds(shreds2, Some(&leader_schedule_cache), false) + .unwrap(); + verify_index_integrity(&blockstore, slot); + blockstore.purge_and_compact_slots(0, slot); + + // Test not all, but enough data and coding shreds in 2 rounds to trigger recovery, + // make sure nothing is lost + let shreds1: Vec<_> = data_shreds[..data_shreds.len() / 2 - 1] + .iter() + .cloned() + .chain(coding_shreds[..coding_shreds.len() / 2 - 1].iter().cloned()) + .collect(); + let shreds2: Vec<_> = data_shreds[data_shreds.len() / 2 - 1..data_shreds.len() / 2] + .iter() + .cloned() + .chain( + coding_shreds[coding_shreds.len() / 2 - 1..coding_shreds.len() / 2] + .iter() + .cloned(), + ) + .collect(); + blockstore + .insert_shreds(shreds1, Some(&leader_schedule_cache), false) + .unwrap(); + blockstore + .insert_shreds(shreds2, Some(&leader_schedule_cache), false) + .unwrap(); + verify_index_integrity(&blockstore, slot); + blockstore.purge_and_compact_slots(0, slot); + + // Test insert shreds in 2 rounds, but not enough to trigger + // recovery, make sure nothing is lost + let shreds1: Vec<_> = data_shreds[..data_shreds.len() / 2 - 2] + .iter() + .cloned() + .chain(coding_shreds[..coding_shreds.len() / 2 - 2].iter().cloned()) + .collect(); + let shreds2: Vec<_> = data_shreds[data_shreds.len() / 2 - 2..data_shreds.len() / 2 - 1] + .iter() + .cloned() + .chain( + coding_shreds[coding_shreds.len() / 2 - 2..coding_shreds.len() / 2 - 1] + .iter() + .cloned(), + ) + .collect(); + blockstore + .insert_shreds(shreds1, Some(&leader_schedule_cache), false) + .unwrap(); + blockstore + .insert_shreds(shreds2, Some(&leader_schedule_cache), false) + .unwrap(); + verify_index_integrity(&blockstore, slot); + blockstore.purge_and_compact_slots(0, slot); } fn setup_erasure_shreds( @@ -8279,95 +8176,90 @@ pub mod tests { let duplicate_shred = duplicate_shreds[0].clone(); let non_duplicate_shred = shred.clone(); - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - blockstore - .insert_shreds(vec![shred.clone()], None, false) - .unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - // No duplicate shreds exist yet - assert!(!blockstore.has_duplicate_shreds_in_slot(slot)); + blockstore + .insert_shreds(vec![shred.clone()], None, false) + .unwrap(); - // Check if shreds are duplicated - assert_eq!( - blockstore.is_shred_duplicate( - slot, - 0, - &duplicate_shred.payload, - duplicate_shred.is_data() - ), - Some(shred.payload.to_vec()) - ); - assert!(blockstore - .is_shred_duplicate( - slot, - 0, - &non_duplicate_shred.payload, - duplicate_shred.is_data() - ) - .is_none()); + // No duplicate shreds exist yet + assert!(!blockstore.has_duplicate_shreds_in_slot(slot)); - // Store a duplicate shred - blockstore - .store_duplicate_slot(slot, shred.payload.clone(), duplicate_shred.payload.clone()) - .unwrap(); + // Check if shreds are duplicated + assert_eq!( + blockstore.is_shred_duplicate( + slot, + 0, + &duplicate_shred.payload, + duplicate_shred.is_data() + ), + Some(shred.payload.to_vec()) + ); + assert!(blockstore + .is_shred_duplicate( + slot, + 0, + &non_duplicate_shred.payload, + duplicate_shred.is_data() + ) + .is_none()); - // Slot is now marked as duplicate - assert!(blockstore.has_duplicate_shreds_in_slot(slot)); + // Store a duplicate shred + blockstore + .store_duplicate_slot(slot, shred.payload.clone(), duplicate_shred.payload.clone()) + .unwrap(); - // Check ability to fetch the duplicates - let duplicate_proof = blockstore.get_duplicate_slot(slot).unwrap(); - assert_eq!(duplicate_proof.shred1, shred.payload); - assert_eq!(duplicate_proof.shred2, duplicate_shred.payload); - } + // Slot is now marked as duplicate + assert!(blockstore.has_duplicate_shreds_in_slot(slot)); - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + // Check ability to fetch the duplicates + let duplicate_proof = blockstore.get_duplicate_slot(slot).unwrap(); + assert_eq!(duplicate_proof.shred1, shred.payload); + assert_eq!(duplicate_proof.shred2, duplicate_shred.payload); } #[test] fn test_clear_unconfirmed_slot() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - let unconfirmed_slot = 9; - let unconfirmed_child_slot = 10; - let slots = vec![2, unconfirmed_slot, unconfirmed_child_slot]; + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + let unconfirmed_slot = 9; + let unconfirmed_child_slot = 10; + let slots = vec![2, unconfirmed_slot, unconfirmed_child_slot]; + + // Insert into slot 9, mark it as dead + let shreds: Vec<_> = make_chaining_slot_entries(&slots, 1) + .into_iter() + .flat_map(|x| x.0) + .collect(); + blockstore.insert_shreds(shreds, None, false).unwrap(); + // Should only be one shred in slot 9 + assert!(blockstore + .get_data_shred(unconfirmed_slot, 0) + .unwrap() + .is_some()); + assert!(blockstore + .get_data_shred(unconfirmed_slot, 1) + .unwrap() + .is_none()); + blockstore.set_dead_slot(unconfirmed_slot).unwrap(); - // Insert into slot 9, mark it as dead - let shreds: Vec<_> = make_chaining_slot_entries(&slots, 1) - .into_iter() - .flat_map(|x| x.0) - .collect(); - blockstore.insert_shreds(shreds, None, false).unwrap(); - // Should only be one shred in slot 9 - assert!(blockstore - .get_data_shred(unconfirmed_slot, 0) - .unwrap() - .is_some()); - assert!(blockstore - .get_data_shred(unconfirmed_slot, 1) + // Purge the slot + blockstore.clear_unconfirmed_slot(unconfirmed_slot); + assert!(!blockstore.is_dead(unconfirmed_slot)); + assert_eq!( + blockstore + .meta(unconfirmed_slot) .unwrap() - .is_none()); - blockstore.set_dead_slot(unconfirmed_slot).unwrap(); - - // Purge the slot - blockstore.clear_unconfirmed_slot(unconfirmed_slot); - assert!(!blockstore.is_dead(unconfirmed_slot)); - assert_eq!( - blockstore - .meta(unconfirmed_slot) - .unwrap() - .unwrap() - .next_slots, - vec![unconfirmed_child_slot] - ); - assert!(blockstore - .get_data_shred(unconfirmed_slot, 0) .unwrap() - .is_none()); - } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + .next_slots, + vec![unconfirmed_child_slot] + ); + assert!(blockstore + .get_data_shred(unconfirmed_slot, 0) + .unwrap() + .is_none()); } #[test] @@ -8432,137 +8324,133 @@ pub mod tests { #[test] fn test_rewards_protobuf_backward_compatability() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - let rewards: Rewards = (0..100) - .map(|i| Reward { - pubkey: solana_sdk::pubkey::new_rand().to_string(), - lamports: 42 + i, - post_balance: std::u64::MAX, - reward_type: Some(RewardType::Fee), - commission: None, - }) - .collect(); - let protobuf_rewards: generated::Rewards = rewards.into(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + let rewards: Rewards = (0..100) + .map(|i| Reward { + pubkey: solana_sdk::pubkey::new_rand().to_string(), + lamports: 42 + i, + post_balance: std::u64::MAX, + reward_type: Some(RewardType::Fee), + commission: None, + }) + .collect(); + let protobuf_rewards: generated::Rewards = rewards.into(); - let deprecated_rewards: StoredExtendedRewards = protobuf_rewards.clone().into(); - for slot in 0..2 { - let data = serialize(&deprecated_rewards).unwrap(); - blockstore.rewards_cf.put_bytes(slot, &data).unwrap(); - } - for slot in 2..4 { + let deprecated_rewards: StoredExtendedRewards = protobuf_rewards.clone().into(); + for slot in 0..2 { + let data = serialize(&deprecated_rewards).unwrap(); + blockstore.rewards_cf.put_bytes(slot, &data).unwrap(); + } + for slot in 2..4 { + blockstore + .rewards_cf + .put_protobuf(slot, &protobuf_rewards) + .unwrap(); + } + for slot in 0..4 { + assert_eq!( blockstore .rewards_cf - .put_protobuf(slot, &protobuf_rewards) - .unwrap(); - } - for slot in 0..4 { - assert_eq!( - blockstore - .rewards_cf - .get_protobuf_or_bincode::(slot) - .unwrap() - .unwrap(), - protobuf_rewards - ); - } + .get_protobuf_or_bincode::(slot) + .unwrap() + .unwrap(), + protobuf_rewards + ); } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] fn test_transaction_status_protobuf_backward_compatability() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - let status = TransactionStatusMeta { - status: Ok(()), - fee: 42, - pre_balances: vec![1, 2, 3], - post_balances: vec![1, 2, 3], - inner_instructions: Some(vec![]), - log_messages: Some(vec![]), - pre_token_balances: Some(vec![TransactionTokenBalance { - account_index: 0, - mint: Pubkey::new_unique().to_string(), - ui_token_amount: UiTokenAmount { - ui_amount: Some(1.1), - decimals: 1, - amount: "11".to_string(), - ui_amount_string: "1.1".to_string(), - }, - }]), - post_token_balances: Some(vec![TransactionTokenBalance { - account_index: 0, - mint: Pubkey::new_unique().to_string(), - ui_token_amount: UiTokenAmount { - ui_amount: None, - decimals: 1, - amount: "11".to_string(), - ui_amount_string: "1.1".to_string(), - }, - }]), - rewards: Some(vec![Reward { - pubkey: "My11111111111111111111111111111111111111111".to_string(), - lamports: -42, - post_balance: 42, - reward_type: Some(RewardType::Rent), - commission: None, - }]), - }; - let deprecated_status: StoredTransactionStatusMeta = status.clone().into(); - let protobuf_status: generated::TransactionStatusMeta = status.into(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + let status = TransactionStatusMeta { + status: Ok(()), + fee: 42, + pre_balances: vec![1, 2, 3], + post_balances: vec![1, 2, 3], + inner_instructions: Some(vec![]), + log_messages: Some(vec![]), + pre_token_balances: Some(vec![TransactionTokenBalance { + account_index: 0, + mint: Pubkey::new_unique().to_string(), + ui_token_amount: UiTokenAmount { + ui_amount: Some(1.1), + decimals: 1, + amount: "11".to_string(), + ui_amount_string: "1.1".to_string(), + }, + }]), + post_token_balances: Some(vec![TransactionTokenBalance { + account_index: 0, + mint: Pubkey::new_unique().to_string(), + ui_token_amount: UiTokenAmount { + ui_amount: None, + decimals: 1, + amount: "11".to_string(), + ui_amount_string: "1.1".to_string(), + }, + }]), + rewards: Some(vec![Reward { + pubkey: "My11111111111111111111111111111111111111111".to_string(), + lamports: -42, + post_balance: 42, + reward_type: Some(RewardType::Rent), + commission: None, + }]), + }; + let deprecated_status: StoredTransactionStatusMeta = status.clone().into(); + let protobuf_status: generated::TransactionStatusMeta = status.into(); - for slot in 0..2 { - let data = serialize(&deprecated_status).unwrap(); - blockstore - .transaction_status_cf - .put_bytes((0, Signature::default(), slot), &data) - .unwrap(); - } - for slot in 2..4 { + for slot in 0..2 { + let data = serialize(&deprecated_status).unwrap(); + blockstore + .transaction_status_cf + .put_bytes((0, Signature::default(), slot), &data) + .unwrap(); + } + for slot in 2..4 { + blockstore + .transaction_status_cf + .put_protobuf((0, Signature::default(), slot), &protobuf_status) + .unwrap(); + } + for slot in 0..4 { + assert_eq!( blockstore .transaction_status_cf - .put_protobuf((0, Signature::default(), slot), &protobuf_status) - .unwrap(); - } - for slot in 0..4 { - assert_eq!( - blockstore - .transaction_status_cf - .get_protobuf_or_bincode::(( - 0, - Signature::default(), - slot - )) - .unwrap() - .unwrap(), - protobuf_status - ); - } + .get_protobuf_or_bincode::(( + 0, + Signature::default(), + slot + )) + .unwrap() + .unwrap(), + protobuf_status + ); } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] fn test_remove_shred_data_complete_flag() { - let (mut shreds, entries) = make_slot_entries(0, 0, 1); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - let ledger_path = get_tmp_ledger_path!(); - let ledger = Blockstore::open(&ledger_path).unwrap(); + let (mut shreds, entries) = make_slot_entries(0, 0, 1); // Remove the data complete flag from the last shred shreds[0].unset_data_complete(); - ledger.insert_shreds(shreds, None, false).unwrap(); + blockstore.insert_shreds(shreds, None, false).unwrap(); // Check that the `data_complete` flag was unset in the stored shred, but the // `last_in_slot` flag is set. - let stored_shred = &ledger.get_data_shreds_for_slot(0, 0).unwrap()[0]; + let stored_shred = &blockstore.get_data_shreds_for_slot(0, 0).unwrap()[0]; assert!(!stored_shred.data_complete()); assert!(stored_shred.last_in_slot()); - assert_eq!(entries, ledger.get_any_valid_slot_entries(0, 0)); + assert_eq!(entries, blockstore.get_any_valid_slot_entries(0, 0)); } fn make_large_tx_entry(num_txs: usize) -> Entry { @@ -8588,8 +8476,8 @@ pub mod tests { let shreds = entries_to_test_shreds(vec![entry], slot, parent, true, 0); assert!(shreds.len() > 1); - let ledger_path = get_tmp_ledger_path!(); - let ledger = Blockstore::open(&ledger_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); let coding1 = Shredder::generate_coding_shreds(&shreds, false); let coding2 = Shredder::generate_coding_shreds(&shreds, true); @@ -8602,13 +8490,13 @@ pub mod tests { for shred in &coding2 { info!("coding2 {:?}", shred); } - ledger + blockstore .insert_shreds(shreds[..shreds.len() - 2].to_vec(), None, false) .unwrap(); - ledger + blockstore .insert_shreds(vec![coding1[0].clone(), coding2[1].clone()], None, false) .unwrap(); - assert!(ledger.has_duplicate_shreds_in_slot(slot)); + assert!(blockstore.has_duplicate_shreds_in_slot(slot)); } #[test] @@ -8617,75 +8505,69 @@ pub mod tests { let slot = 1; let (_data_shreds, mut coding_shreds, leader_schedule_cache) = setup_erasure_shreds(slot, 0, 100); - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - coding_shreds[1].coding_header.num_coding_shreds = u16::MAX; - blockstore - .insert_shreds( - vec![coding_shreds[1].clone()], - Some(&leader_schedule_cache), - false, - ) - .unwrap(); - // Check no coding shreds are inserted - let res = blockstore.get_coding_shreds_for_slot(slot, 0).unwrap(); - assert!(res.is_empty()); - } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + coding_shreds[1].coding_header.num_coding_shreds = u16::MAX; + blockstore + .insert_shreds( + vec![coding_shreds[1].clone()], + Some(&leader_schedule_cache), + false, + ) + .unwrap(); + + // Check no coding shreds are inserted + let res = blockstore.get_coding_shreds_for_slot(slot, 0).unwrap(); + assert!(res.is_empty()); } #[test] pub fn test_insert_data_shreds_same_slot_last_index() { - // Create RocksDb ledger - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - - // Create enough entries to ensure there are at least two shreds created - let num_unique_entries = max_ticks_per_n_shreds(1, None) + 1; - let (mut original_shreds, original_entries) = - make_slot_entries(0, 0, num_unique_entries); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - // Discard first shred, so that the slot is not full - assert!(original_shreds.len() > 1); - let last_index = original_shreds.last().unwrap().index() as u64; - original_shreds.remove(0); + // Create enough entries to ensure there are at least two shreds created + let num_unique_entries = max_ticks_per_n_shreds(1, None) + 1; + let (mut original_shreds, original_entries) = make_slot_entries(0, 0, num_unique_entries); - // Insert the same shreds, including the last shred specifically, multiple - // times - for _ in 0..10 { - blockstore - .insert_shreds(original_shreds.clone(), None, false) - .unwrap(); - let meta = blockstore.meta(0).unwrap().unwrap(); - assert!(!blockstore.is_dead(0)); - assert_eq!(blockstore.get_slot_entries(0, 0).unwrap(), vec![]); - assert_eq!(meta.consumed, 0); - assert_eq!(meta.received, last_index + 1); - assert_eq!(meta.parent_slot, 0); - assert_eq!(meta.last_index, last_index); - assert!(!blockstore.is_full(0)); - } + // Discard first shred, so that the slot is not full + assert!(original_shreds.len() > 1); + let last_index = original_shreds.last().unwrap().index() as u64; + original_shreds.remove(0); - let duplicate_shreds = entries_to_test_shreds(original_entries.clone(), 0, 0, true, 0); - let num_shreds = duplicate_shreds.len() as u64; + // Insert the same shreds, including the last shred specifically, multiple + // times + for _ in 0..10 { blockstore - .insert_shreds(duplicate_shreds, None, false) + .insert_shreds(original_shreds.clone(), None, false) .unwrap(); - - assert_eq!(blockstore.get_slot_entries(0, 0).unwrap(), original_entries); - let meta = blockstore.meta(0).unwrap().unwrap(); - assert_eq!(meta.consumed, num_shreds); - assert_eq!(meta.received, num_shreds); - assert_eq!(meta.parent_slot, 0); - assert_eq!(meta.last_index, num_shreds - 1); - assert!(blockstore.is_full(0)); assert!(!blockstore.is_dead(0)); + assert_eq!(blockstore.get_slot_entries(0, 0).unwrap(), vec![]); + assert_eq!(meta.consumed, 0); + assert_eq!(meta.received, last_index + 1); + assert_eq!(meta.parent_slot, 0); + assert_eq!(meta.last_index, last_index); + assert!(!blockstore.is_full(0)); } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + + let duplicate_shreds = entries_to_test_shreds(original_entries.clone(), 0, 0, true, 0); + let num_shreds = duplicate_shreds.len() as u64; + blockstore + .insert_shreds(duplicate_shreds, None, false) + .unwrap(); + + assert_eq!(blockstore.get_slot_entries(0, 0).unwrap(), original_entries); + + let meta = blockstore.meta(0).unwrap().unwrap(); + assert_eq!(meta.consumed, num_shreds); + assert_eq!(meta.received, num_shreds); + assert_eq!(meta.parent_slot, 0); + assert_eq!(meta.last_index, num_shreds - 1); + assert!(blockstore.is_full(0)); + assert!(!blockstore.is_dead(0)); } #[test] @@ -8698,14 +8580,12 @@ pub mod tests { // Mark both as last shred shreds[0].set_last_in_slot(); shreds[1].set_last_in_slot(); - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - blockstore.insert_shreds(shreds, None, false).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - assert!(blockstore.get_duplicate_slot(slot).is_some()); - } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + blockstore.insert_shreds(shreds, None, false).unwrap(); + + assert!(blockstore.get_duplicate_slot(slot).is_some()); } #[test] @@ -8744,142 +8624,139 @@ pub mod tests { (meta, shreds_index) }; - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - let mut slot = 0; - let shreds = setup_test_shreds(slot); - - // Case 1: Insert in the same batch. Since we're inserting the shreds in order, - // any shreds > smaller_last_shred_index will not be inserted. Slot is not marked - // as dead because no slots > the first "last" index shred are inserted before - // the "last" index shred itself is inserted. - let (expected_slot_meta, expected_index) = get_expected_slot_meta_and_index_meta( - &blockstore, - shreds[..=smaller_last_shred_index].to_vec(), - ); - blockstore - .insert_shreds(shreds.clone(), None, false) - .unwrap(); - assert!(blockstore.get_duplicate_slot(slot).is_some()); - assert!(!blockstore.is_dead(slot)); - for i in 0..num_shreds { - if i <= smaller_last_shred_index as u64 { - assert_eq!( - blockstore.get_data_shred(slot, i).unwrap().unwrap(), - shreds[i as usize].payload - ); - } else { - assert!(blockstore.get_data_shred(slot, i).unwrap().is_none()); - } + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + let mut slot = 0; + let shreds = setup_test_shreds(slot); + + // Case 1: Insert in the same batch. Since we're inserting the shreds in order, + // any shreds > smaller_last_shred_index will not be inserted. Slot is not marked + // as dead because no slots > the first "last" index shred are inserted before + // the "last" index shred itself is inserted. + let (expected_slot_meta, expected_index) = get_expected_slot_meta_and_index_meta( + &blockstore, + shreds[..=smaller_last_shred_index].to_vec(), + ); + blockstore + .insert_shreds(shreds.clone(), None, false) + .unwrap(); + assert!(blockstore.get_duplicate_slot(slot).is_some()); + assert!(!blockstore.is_dead(slot)); + for i in 0..num_shreds { + if i <= smaller_last_shred_index as u64 { + assert_eq!( + blockstore.get_data_shred(slot, i).unwrap().unwrap(), + shreds[i as usize].payload + ); + } else { + assert!(blockstore.get_data_shred(slot, i).unwrap().is_none()); } - let mut meta = blockstore.meta(slot).unwrap().unwrap(); - meta.first_shred_timestamp = expected_slot_meta.first_shred_timestamp; - assert_eq!(meta, expected_slot_meta); - assert_eq!(blockstore.get_index(slot).unwrap().unwrap(), expected_index); - - // Case 2: Inserting a duplicate with an even smaller last shred index should not - // mark the slot as dead since the Slotmeta is full. - let mut even_smaller_last_shred_duplicate = - shreds[smaller_last_shred_index - 1].clone(); - even_smaller_last_shred_duplicate.set_last_in_slot(); - // Flip a byte to create a duplicate shred - even_smaller_last_shred_duplicate.payload[0] = - std::u8::MAX - even_smaller_last_shred_duplicate.payload[0]; - assert!(blockstore - .is_shred_duplicate( - slot, - even_smaller_last_shred_duplicate.index(), - &even_smaller_last_shred_duplicate.payload, - true - ) - .is_some()); - blockstore - .insert_shreds(vec![even_smaller_last_shred_duplicate], None, false) - .unwrap(); - assert!(!blockstore.is_dead(slot)); - for i in 0..num_shreds { - if i <= smaller_last_shred_index as u64 { - assert_eq!( - blockstore.get_data_shred(slot, i).unwrap().unwrap(), - shreds[i as usize].payload - ); - } else { - assert!(blockstore.get_data_shred(slot, i).unwrap().is_none()); - } + } + let mut meta = blockstore.meta(slot).unwrap().unwrap(); + meta.first_shred_timestamp = expected_slot_meta.first_shred_timestamp; + assert_eq!(meta, expected_slot_meta); + assert_eq!(blockstore.get_index(slot).unwrap().unwrap(), expected_index); + + // Case 2: Inserting a duplicate with an even smaller last shred index should not + // mark the slot as dead since the Slotmeta is full. + let mut even_smaller_last_shred_duplicate = shreds[smaller_last_shred_index - 1].clone(); + even_smaller_last_shred_duplicate.set_last_in_slot(); + // Flip a byte to create a duplicate shred + even_smaller_last_shred_duplicate.payload[0] = + std::u8::MAX - even_smaller_last_shred_duplicate.payload[0]; + assert!(blockstore + .is_shred_duplicate( + slot, + even_smaller_last_shred_duplicate.index(), + &even_smaller_last_shred_duplicate.payload, + true + ) + .is_some()); + blockstore + .insert_shreds(vec![even_smaller_last_shred_duplicate], None, false) + .unwrap(); + assert!(!blockstore.is_dead(slot)); + for i in 0..num_shreds { + if i <= smaller_last_shred_index as u64 { + assert_eq!( + blockstore.get_data_shred(slot, i).unwrap().unwrap(), + shreds[i as usize].payload + ); + } else { + assert!(blockstore.get_data_shred(slot, i).unwrap().is_none()); } - let mut meta = blockstore.meta(slot).unwrap().unwrap(); - meta.first_shred_timestamp = expected_slot_meta.first_shred_timestamp; - assert_eq!(meta, expected_slot_meta); - assert_eq!(blockstore.get_index(slot).unwrap().unwrap(), expected_index); - - // Case 3: Insert shreds in reverse so that consumed will not be updated. Now on insert, the - // the slot should be marked as dead - slot += 1; - let mut shreds = setup_test_shreds(slot); - shreds.reverse(); - blockstore - .insert_shreds(shreds.clone(), None, false) - .unwrap(); - assert!(blockstore.is_dead(slot)); - // All the shreds other than the two last index shreds because those two - // are marked as last, but less than the first received index == 10. - // The others will be inserted even after the slot is marked dead on attempted - // insert of the first last_index shred since dead slots can still be - // inserted into. - for i in 0..num_shreds { - let shred_to_check = &shreds[i as usize]; - let shred_index = shred_to_check.index() as u64; - if shred_index != smaller_last_shred_index as u64 - && shred_index != larger_last_shred_index as u64 - { - assert_eq!( - blockstore - .get_data_shred(slot, shred_index) - .unwrap() - .unwrap(), - shred_to_check.payload - ); - } else { - assert!(blockstore + } + let mut meta = blockstore.meta(slot).unwrap().unwrap(); + meta.first_shred_timestamp = expected_slot_meta.first_shred_timestamp; + assert_eq!(meta, expected_slot_meta); + assert_eq!(blockstore.get_index(slot).unwrap().unwrap(), expected_index); + + // Case 3: Insert shreds in reverse so that consumed will not be updated. Now on insert, the + // the slot should be marked as dead + slot += 1; + let mut shreds = setup_test_shreds(slot); + shreds.reverse(); + blockstore + .insert_shreds(shreds.clone(), None, false) + .unwrap(); + assert!(blockstore.is_dead(slot)); + // All the shreds other than the two last index shreds because those two + // are marked as last, but less than the first received index == 10. + // The others will be inserted even after the slot is marked dead on attempted + // insert of the first last_index shred since dead slots can still be + // inserted into. + for i in 0..num_shreds { + let shred_to_check = &shreds[i as usize]; + let shred_index = shred_to_check.index() as u64; + if shred_index != smaller_last_shred_index as u64 + && shred_index != larger_last_shred_index as u64 + { + assert_eq!( + blockstore .get_data_shred(slot, shred_index) .unwrap() - .is_none()); - } + .unwrap(), + shred_to_check.payload + ); + } else { + assert!(blockstore + .get_data_shred(slot, shred_index) + .unwrap() + .is_none()); } + } - // Case 4: Same as Case 3, but this time insert the shreds one at a time to test that the clearing - // of data shreds works even after they've been committed - slot += 1; - let mut shreds = setup_test_shreds(slot); - shreds.reverse(); - for shred in shreds.clone() { - blockstore.insert_shreds(vec![shred], None, false).unwrap(); - } - assert!(blockstore.is_dead(slot)); - // All the shreds will be inserted since dead slots can still be inserted into. - for i in 0..num_shreds { - let shred_to_check = &shreds[i as usize]; - let shred_index = shred_to_check.index() as u64; - if shred_index != smaller_last_shred_index as u64 - && shred_index != larger_last_shred_index as u64 - { - assert_eq!( - blockstore - .get_data_shred(slot, shred_index) - .unwrap() - .unwrap(), - shred_to_check.payload - ); - } else { - assert!(blockstore + // Case 4: Same as Case 3, but this time insert the shreds one at a time to test that the clearing + // of data shreds works even after they've been committed + slot += 1; + let mut shreds = setup_test_shreds(slot); + shreds.reverse(); + for shred in shreds.clone() { + blockstore.insert_shreds(vec![shred], None, false).unwrap(); + } + assert!(blockstore.is_dead(slot)); + // All the shreds will be inserted since dead slots can still be inserted into. + for i in 0..num_shreds { + let shred_to_check = &shreds[i as usize]; + let shred_index = shred_to_check.index() as u64; + if shred_index != smaller_last_shred_index as u64 + && shred_index != larger_last_shred_index as u64 + { + assert_eq!( + blockstore .get_data_shred(slot, shred_index) .unwrap() - .is_none()); - } + .unwrap(), + shred_to_check.payload + ); + } else { + assert!(blockstore + .get_data_shred(slot, shred_index) + .unwrap() + .is_none()); } } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] @@ -8901,9 +8778,9 @@ pub mod tests { shreds }; - let blockstore_path = get_tmp_ledger_path!(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); { - let blockstore = Arc::new(Blockstore::open(&blockstore_path).unwrap()); + let blockstore = Arc::new(Blockstore::open(ledger_path.path()).unwrap()); let (slot_sender, slot_receiver) = channel(); let (shred_sender, shred_receiver) = channel::>(); let (signal_sender, signal_receiver) = channel(); @@ -8935,21 +8812,25 @@ pub mod tests { .unwrap() }; - let t_shred_inserter = Builder::new() - .spawn(move || { - while let Ok(shreds) = shred_receiver.recv() { - let slot = shreds[0].slot(); - // Grab this lock to block `get_slot_entries` before it fetches completed datasets - // and then mark the slot as dead, but full, by inserting carefully crafted shreds. - let _lowest_cleanup_slot = blockstore.lowest_cleanup_slot.write().unwrap(); - blockstore.insert_shreds(shreds, None, false).unwrap(); - assert!(blockstore.get_duplicate_slot(slot).is_some()); - assert!(blockstore.is_dead(slot)); - assert!(blockstore.meta(slot).unwrap().unwrap().is_full()); - signal_sender.send(Ok(())).unwrap(); - } - }) - .unwrap(); + let t_shred_inserter = { + let blockstore = blockstore.clone(); + Builder::new() + .spawn(move || { + while let Ok(shreds) = shred_receiver.recv() { + let slot = shreds[0].slot(); + // Grab this lock to block `get_slot_entries` before it fetches completed datasets + // and then mark the slot as dead, but full, by inserting carefully crafted shreds. + let _lowest_cleanup_slot = + blockstore.lowest_cleanup_slot.write().unwrap(); + blockstore.insert_shreds(shreds, None, false).unwrap(); + assert!(blockstore.get_duplicate_slot(slot).is_some()); + assert!(blockstore.is_dead(slot)); + assert!(blockstore.meta(slot).unwrap().unwrap().is_full()); + signal_sender.send(Ok(())).unwrap(); + } + }) + .unwrap() + }; for slot in 0..100 { let shreds = setup_test_shreds(slot); @@ -8972,130 +8853,126 @@ pub mod tests { for handle in handles { assert!(handle.join().is_ok()); } - } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + assert!(Arc::strong_count(&blockstore) == 1); + } } #[test] fn test_read_write_cost_table() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - let num_entries: usize = 10; - let mut cost_table: HashMap = HashMap::new(); - for x in 1..num_entries + 1 { - cost_table.insert(Pubkey::new_unique(), (x + 100) as u64); - } + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - // write to db - for (key, cost) in cost_table.iter() { - blockstore - .write_program_cost(key, cost) - .expect("write a program"); - } - - // read back from db - let read_back = blockstore.read_program_costs().expect("read programs"); - // verify - assert_eq!(read_back.len(), cost_table.len()); - for (read_key, read_cost) in read_back { - assert_eq!(read_cost, *cost_table.get(&read_key).unwrap()); - } + let num_entries: usize = 10; + let mut cost_table: HashMap = HashMap::new(); + for x in 1..num_entries + 1 { + cost_table.insert(Pubkey::new_unique(), (x + 100) as u64); + } - // update value, write to db - for val in cost_table.values_mut() { - *val += 100; - } - for (key, cost) in cost_table.iter() { - blockstore - .write_program_cost(key, cost) - .expect("write a program"); - } - // add a new record - let new_program_key = Pubkey::new_unique(); - let new_program_cost = 999; + // write to db + for (key, cost) in cost_table.iter() { blockstore - .write_program_cost(&new_program_key, &new_program_cost) - .unwrap(); + .write_program_cost(key, cost) + .expect("write a program"); + } - // confirm value updated - let read_back = blockstore.read_program_costs().expect("read programs"); - // verify - assert_eq!(read_back.len(), cost_table.len() + 1); - for (key, cost) in cost_table.iter() { - assert_eq!(*cost, read_back.iter().find(|(k, _v)| k == key).unwrap().1); - } - assert_eq!( - new_program_cost, - read_back - .iter() - .find(|(k, _v)| *k == new_program_key) - .unwrap() - .1 - ); + // read back from db + let read_back = blockstore.read_program_costs().expect("read programs"); + // verify + assert_eq!(read_back.len(), cost_table.len()); + for (read_key, read_cost) in read_back { + assert_eq!(read_cost, *cost_table.get(&read_key).unwrap()); + } - // test delete + // update value, write to db + for val in cost_table.values_mut() { + *val += 100; + } + for (key, cost) in cost_table.iter() { blockstore - .delete_program_cost(&new_program_key) - .expect("delete a progrma"); - let read_back = blockstore.read_program_costs().expect("read programs"); - // verify - assert_eq!(read_back.len(), cost_table.len()); - for (read_key, read_cost) in read_back { - assert_eq!(read_cost, *cost_table.get(&read_key).unwrap()); - } + .write_program_cost(key, cost) + .expect("write a program"); + } + // add a new record + let new_program_key = Pubkey::new_unique(); + let new_program_cost = 999; + blockstore + .write_program_cost(&new_program_key, &new_program_cost) + .unwrap(); + + // confirm value updated + let read_back = blockstore.read_program_costs().expect("read programs"); + // verify + assert_eq!(read_back.len(), cost_table.len() + 1); + for (key, cost) in cost_table.iter() { + assert_eq!(*cost, read_back.iter().find(|(k, _v)| k == key).unwrap().1); + } + assert_eq!( + new_program_cost, + read_back + .iter() + .find(|(k, _v)| *k == new_program_key) + .unwrap() + .1 + ); + + // test delete + blockstore + .delete_program_cost(&new_program_key) + .expect("delete a progrma"); + let read_back = blockstore.read_program_costs().expect("read programs"); + // verify + assert_eq!(read_back.len(), cost_table.len()); + for (read_key, read_cost) in read_back { + assert_eq!(read_cost, *cost_table.get(&read_key).unwrap()); } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] fn test_delete_old_records_from_cost_table() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - let num_entries: usize = 10; - let mut cost_table: HashMap = HashMap::new(); - for x in 1..num_entries + 1 { - cost_table.insert(Pubkey::new_unique(), (x + 100) as u64); - } + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - // write to db - for (key, cost) in cost_table.iter() { - blockstore - .write_program_cost(key, cost) - .expect("write a program"); - } + let num_entries: usize = 10; + let mut cost_table: HashMap = HashMap::new(); + for x in 1..num_entries + 1 { + cost_table.insert(Pubkey::new_unique(), (x + 100) as u64); + } - // remove a record - let mut removed_key = Pubkey::new_unique(); - for (key, cost) in cost_table.iter() { - if *cost == 101_u64 { - removed_key = *key; - break; - } - } - cost_table.remove(&removed_key); + // write to db + for (key, cost) in cost_table.iter() { + blockstore + .write_program_cost(key, cost) + .expect("write a program"); + } - // delete records from blockstore if they are no longer in cost_table - let db_records = blockstore.read_program_costs().expect("read programs"); - db_records.iter().for_each(|(pubkey, _)| { - if !cost_table.iter().any(|(key, _)| key == pubkey) { - assert_eq!(*pubkey, removed_key); - blockstore - .delete_program_cost(pubkey) - .expect("delete old program"); - } - }); + // remove a record + let mut removed_key = Pubkey::new_unique(); + for (key, cost) in cost_table.iter() { + if *cost == 101_u64 { + removed_key = *key; + break; + } + } + cost_table.remove(&removed_key); - // read back from db - let read_back = blockstore.read_program_costs().expect("read programs"); - // verify - assert_eq!(read_back.len(), cost_table.len()); - for (read_key, read_cost) in read_back { - assert_eq!(read_cost, *cost_table.get(&read_key).unwrap()); + // delete records from blockstore if they are no longer in cost_table + let db_records = blockstore.read_program_costs().expect("read programs"); + db_records.iter().for_each(|(pubkey, _)| { + if !cost_table.iter().any(|(key, _)| key == pubkey) { + assert_eq!(*pubkey, removed_key); + blockstore + .delete_program_cost(pubkey) + .expect("delete old program"); } + }); + + // read back from db + let read_back = blockstore.read_program_costs().expect("read programs"); + // verify + assert_eq!(read_back.len(), cost_table.len()); + for (read_key, read_cost) in read_back { + assert_eq!(read_cost, *cost_table.get(&read_key).unwrap()); } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } } diff --git a/ledger/src/blockstore/blockstore_purge.rs b/ledger/src/blockstore/blockstore_purge.rs index 769b16520e62fc..bab308ac16de8e 100644 --- a/ledger/src/blockstore/blockstore_purge.rs +++ b/ledger/src/blockstore/blockstore_purge.rs @@ -393,7 +393,9 @@ impl Blockstore { #[cfg(test)] pub mod tests { use super::*; - use crate::{blockstore::tests::make_slot_entries_with_transactions, get_tmp_ledger_path}; + use crate::{ + blockstore::tests::make_slot_entries_with_transactions, get_tmp_ledger_path_auto_delete, + }; use bincode::serialize; use solana_entry::entry::next_entry_mut; use solana_sdk::{ @@ -497,8 +499,9 @@ pub mod tests { #[test] fn test_purge_slots() { - let blockstore_path = get_tmp_ledger_path!(); - let blockstore = Blockstore::open(&blockstore_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + let (shreds, _) = make_many_slot_entries(0, 50, 5); blockstore.insert_shreds(shreds, None, false).unwrap(); @@ -518,239 +521,235 @@ pub mod tests { .for_each(|(_, _)| { panic!(); }); - - drop(blockstore); - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] fn test_purge_huge() { - let blockstore_path = get_tmp_ledger_path!(); - let blockstore = Blockstore::open(&blockstore_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + let (shreds, _) = make_many_slot_entries(0, 5000, 10); blockstore.insert_shreds(shreds, None, false).unwrap(); blockstore.purge_and_compact_slots(0, 4999); - test_all_empty_or_min(&blockstore, 5000); - - drop(blockstore); - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] fn test_purge_front_of_ledger() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - let max_slot = 10; - for x in 0..max_slot { - let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); - blockstore - .write_transaction_status( - x, - Signature::new(&random_bytes), - vec![&Pubkey::new(&random_bytes[0..32])], - vec![&Pubkey::new(&random_bytes[32..])], - TransactionStatusMeta::default(), - ) - .unwrap(); - } - // Purge to freeze index 0 - blockstore.run_purge(0, 1, PurgeType::PrimaryIndex).unwrap(); - - for x in max_slot..2 * max_slot { - let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); - blockstore - .write_transaction_status( - x, - Signature::new(&random_bytes), - vec![&Pubkey::new(&random_bytes[0..32])], - vec![&Pubkey::new(&random_bytes[32..])], - TransactionStatusMeta::default(), - ) - .unwrap(); - } + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - // Purging range outside of TransactionStatus max slots should not affect TransactionStatus data - blockstore.run_purge(20, 30, PurgeType::Exact).unwrap(); + let max_slot = 10; + for x in 0..max_slot { + let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); + blockstore + .write_transaction_status( + x, + Signature::new(&random_bytes), + vec![&Pubkey::new(&random_bytes[0..32])], + vec![&Pubkey::new(&random_bytes[32..])], + TransactionStatusMeta::default(), + ) + .unwrap(); + } + // Purge to freeze index 0 + blockstore.run_purge(0, 1, PurgeType::PrimaryIndex).unwrap(); - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) + for x in max_slot..2 * max_slot { + let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); + blockstore + .write_transaction_status( + x, + Signature::new(&random_bytes), + vec![&Pubkey::new(&random_bytes[0..32])], + vec![&Pubkey::new(&random_bytes[32..])], + TransactionStatusMeta::default(), + ) .unwrap(); - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + + // Purging range outside of TransactionStatus max slots should not affect TransactionStatus data + blockstore.run_purge(20, 30, PurgeType::Exact).unwrap(); + + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(0), + IteratorDirection::Forward, + )) + .unwrap(); + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 0); } #[test] #[allow(clippy::cognitive_complexity)] fn test_purge_transaction_status() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - let transaction_status_index_cf = blockstore.db.column::(); - let slot = 10; - for _ in 0..5 { - let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); - blockstore - .write_transaction_status( - slot, - Signature::new(&random_bytes), - vec![&Pubkey::new(&random_bytes[0..32])], - vec![&Pubkey::new(&random_bytes[32..])], - TransactionStatusMeta::default(), - ) - .unwrap(); - } - // Purge to freeze index 0 - blockstore.run_purge(0, 1, PurgeType::PrimaryIndex).unwrap(); - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - for _ in 0..5 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert_eq!(entry.2, slot); - } - let mut address_transactions_iterator = blockstore - .db - .iter::(IteratorMode::From( - (0, Pubkey::default(), 0, Signature::default()), - IteratorDirection::Forward, - )) + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + let transaction_status_index_cf = &blockstore.transaction_status_index_cf; + let slot = 10; + for _ in 0..5 { + let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); + blockstore + .write_transaction_status( + slot, + Signature::new(&random_bytes), + vec![&Pubkey::new(&random_bytes[0..32])], + vec![&Pubkey::new(&random_bytes[32..])], + TransactionStatusMeta::default(), + ) .unwrap(); - for _ in 0..10 { - let entry = address_transactions_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert_eq!(entry.2, slot); + } + // Purge to freeze index 0 + blockstore.run_purge(0, 1, PurgeType::PrimaryIndex).unwrap(); + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(0), + IteratorDirection::Forward, + )) + .unwrap(); + for _ in 0..5 { + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 0); + assert_eq!(entry.2, slot); + } + let mut address_transactions_iterator = blockstore + .db + .iter::(IteratorMode::From( + (0, Pubkey::default(), 0, Signature::default()), + IteratorDirection::Forward, + )) + .unwrap(); + for _ in 0..10 { + let entry = address_transactions_iterator.next().unwrap().0; + assert_eq!(entry.0, 0); + assert_eq!(entry.2, slot); + } + assert_eq!( + transaction_status_index_cf.get(0).unwrap().unwrap(), + TransactionStatusIndexMeta { + max_slot: 10, + frozen: true, } - assert_eq!( - transaction_status_index_cf.get(0).unwrap().unwrap(), - TransactionStatusIndexMeta { - max_slot: 10, - frozen: true, - } - ); + ); + drop(status_entry_iterator); + drop(address_transactions_iterator); - // Low purge should not affect state - blockstore.run_purge(0, 5, PurgeType::PrimaryIndex).unwrap(); - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - for _ in 0..5 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert_eq!(entry.2, slot); + // Low purge should not affect state + blockstore.run_purge(0, 5, PurgeType::PrimaryIndex).unwrap(); + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(0), + IteratorDirection::Forward, + )) + .unwrap(); + for _ in 0..5 { + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 0); + assert_eq!(entry.2, slot); + } + let mut address_transactions_iterator = blockstore + .db + .iter::(IteratorMode::From( + cf::AddressSignatures::as_index(0), + IteratorDirection::Forward, + )) + .unwrap(); + for _ in 0..10 { + let entry = address_transactions_iterator.next().unwrap().0; + assert_eq!(entry.0, 0); + assert_eq!(entry.2, slot); + } + assert_eq!( + transaction_status_index_cf.get(0).unwrap().unwrap(), + TransactionStatusIndexMeta { + max_slot: 10, + frozen: true, } - let mut address_transactions_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::AddressSignatures::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - for _ in 0..10 { - let entry = address_transactions_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert_eq!(entry.2, slot); + ); + drop(status_entry_iterator); + drop(address_transactions_iterator); + + // Test boundary conditions: < slot should not purge statuses; <= slot should + blockstore.run_purge(0, 9, PurgeType::PrimaryIndex).unwrap(); + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(0), + IteratorDirection::Forward, + )) + .unwrap(); + for _ in 0..5 { + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 0); + assert_eq!(entry.2, slot); + } + let mut address_transactions_iterator = blockstore + .db + .iter::(IteratorMode::From( + cf::AddressSignatures::as_index(0), + IteratorDirection::Forward, + )) + .unwrap(); + for _ in 0..10 { + let entry = address_transactions_iterator.next().unwrap().0; + assert_eq!(entry.0, 0); + assert_eq!(entry.2, slot); + } + assert_eq!( + transaction_status_index_cf.get(0).unwrap().unwrap(), + TransactionStatusIndexMeta { + max_slot: 10, + frozen: true, } - assert_eq!( - transaction_status_index_cf.get(0).unwrap().unwrap(), - TransactionStatusIndexMeta { - max_slot: 10, - frozen: true, - } - ); + ); + drop(status_entry_iterator); + drop(address_transactions_iterator); - // Test boundary conditions: < slot should not purge statuses; <= slot should - blockstore.run_purge(0, 9, PurgeType::PrimaryIndex).unwrap(); - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - for _ in 0..5 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert_eq!(entry.2, slot); + blockstore + .run_purge(0, 10, PurgeType::PrimaryIndex) + .unwrap(); + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(0), + IteratorDirection::Forward, + )) + .unwrap(); + let padding_entry = status_entry_iterator.next().unwrap().0; + assert_eq!(padding_entry.0, 2); + assert_eq!(padding_entry.2, 0); + assert!(status_entry_iterator.next().is_none()); + let mut address_transactions_iterator = blockstore + .db + .iter::(IteratorMode::From( + cf::AddressSignatures::as_index(0), + IteratorDirection::Forward, + )) + .unwrap(); + let padding_entry = address_transactions_iterator.next().unwrap().0; + assert_eq!(padding_entry.0, 2); + assert_eq!(padding_entry.2, 0); + assert!(address_transactions_iterator.next().is_none()); + assert_eq!( + transaction_status_index_cf.get(0).unwrap().unwrap(), + TransactionStatusIndexMeta { + max_slot: 0, + frozen: false, } - let mut address_transactions_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::AddressSignatures::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - for _ in 0..10 { - let entry = address_transactions_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert_eq!(entry.2, slot); + ); + assert_eq!( + transaction_status_index_cf.get(1).unwrap().unwrap(), + TransactionStatusIndexMeta { + max_slot: 0, + frozen: true, } - assert_eq!( - transaction_status_index_cf.get(0).unwrap().unwrap(), - TransactionStatusIndexMeta { - max_slot: 10, - frozen: true, - } - ); - - blockstore - .run_purge(0, 10, PurgeType::PrimaryIndex) - .unwrap(); - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - let padding_entry = status_entry_iterator.next().unwrap().0; - assert_eq!(padding_entry.0, 2); - assert_eq!(padding_entry.2, 0); - assert!(status_entry_iterator.next().is_none()); - let mut address_transactions_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::AddressSignatures::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - let padding_entry = address_transactions_iterator.next().unwrap().0; - assert_eq!(padding_entry.0, 2); - assert_eq!(padding_entry.2, 0); - assert!(address_transactions_iterator.next().is_none()); - assert_eq!( - transaction_status_index_cf.get(0).unwrap().unwrap(), - TransactionStatusIndexMeta { - max_slot: 0, - frozen: false, - } - ); - assert_eq!( - transaction_status_index_cf.get(1).unwrap().unwrap(), - TransactionStatusIndexMeta { - max_slot: 0, - frozen: true, - } - ); - } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + ); } fn clear_and_repopulate_transaction_statuses( @@ -860,371 +859,365 @@ pub mod tests { #[test] #[allow(clippy::cognitive_complexity)] fn test_purge_transaction_status_exact() { - let blockstore_path = get_tmp_ledger_path!(); - { - let mut blockstore = Blockstore::open(&blockstore_path).unwrap(); - let index0_max_slot = 9; - let index1_max_slot = 19; + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let mut blockstore = Blockstore::open(ledger_path.path()).unwrap(); - // Test purge outside bounds - clear_and_repopulate_transaction_statuses( - &mut blockstore, - index0_max_slot, - index1_max_slot, - ); - blockstore.run_purge(20, 22, PurgeType::Exact).unwrap(); + let index0_max_slot = 9; + let index1_max_slot = 19; - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index0_max_slot, - frozen: true, - } - ); - for _ in 0..index0_max_slot + 1 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); + // Test purge outside bounds + clear_and_repopulate_transaction_statuses( + &mut blockstore, + index0_max_slot, + index1_max_slot, + ); + blockstore.run_purge(20, 22, PurgeType::Exact).unwrap(); + + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(0), + IteratorDirection::Forward, + )) + .unwrap(); + assert_eq!( + blockstore + .transaction_status_index_cf + .get(0) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: index0_max_slot, + frozen: true, } - assert_eq!( - blockstore - .transaction_status_index_cf - .get(1) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index1_max_slot, - frozen: false, - } - ); - for _ in index0_max_slot + 1..index1_max_slot + 1 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 1); + ); + for _ in 0..index0_max_slot + 1 { + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 0); + } + assert_eq!( + blockstore + .transaction_status_index_cf + .get(1) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: index1_max_slot, + frozen: false, } - drop(status_entry_iterator); + ); + for _ in index0_max_slot + 1..index1_max_slot + 1 { + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 1); + } + drop(status_entry_iterator); - // Test purge inside index 0 - clear_and_repopulate_transaction_statuses( - &mut blockstore, - index0_max_slot, - index1_max_slot, - ); - blockstore.run_purge(2, 4, PurgeType::Exact).unwrap(); + // Test purge inside index 0 + clear_and_repopulate_transaction_statuses( + &mut blockstore, + index0_max_slot, + index1_max_slot, + ); + blockstore.run_purge(2, 4, PurgeType::Exact).unwrap(); - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index0_max_slot, - frozen: true, - } - ); - for _ in 0..7 { - // 7 entries remaining - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert!(entry.2 < 2 || entry.2 > 4); + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(0), + IteratorDirection::Forward, + )) + .unwrap(); + assert_eq!( + blockstore + .transaction_status_index_cf + .get(0) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: index0_max_slot, + frozen: true, } - assert_eq!( - blockstore - .transaction_status_index_cf - .get(1) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index1_max_slot, - frozen: false, - } - ); - for _ in index0_max_slot + 1..index1_max_slot + 1 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 1); + ); + for _ in 0..7 { + // 7 entries remaining + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 0); + assert!(entry.2 < 2 || entry.2 > 4); + } + assert_eq!( + blockstore + .transaction_status_index_cf + .get(1) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: index1_max_slot, + frozen: false, } - drop(status_entry_iterator); + ); + for _ in index0_max_slot + 1..index1_max_slot + 1 { + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 1); + } + drop(status_entry_iterator); - // Test purge inside index 0 at upper boundary - clear_and_repopulate_transaction_statuses( - &mut blockstore, - index0_max_slot, - index1_max_slot, - ); - blockstore - .run_purge(7, index0_max_slot, PurgeType::Exact) - .unwrap(); + // Test purge inside index 0 at upper boundary + clear_and_repopulate_transaction_statuses( + &mut blockstore, + index0_max_slot, + index1_max_slot, + ); + blockstore + .run_purge(7, index0_max_slot, PurgeType::Exact) + .unwrap(); - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: 6, - frozen: true, - } - ); - for _ in 0..7 { - // 7 entries remaining - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert!(entry.2 < 7); + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(0), + IteratorDirection::Forward, + )) + .unwrap(); + assert_eq!( + blockstore + .transaction_status_index_cf + .get(0) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: 6, + frozen: true, } - assert_eq!( - blockstore - .transaction_status_index_cf - .get(1) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index1_max_slot, - frozen: false, - } - ); - for _ in index0_max_slot + 1..index1_max_slot + 1 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 1); + ); + for _ in 0..7 { + // 7 entries remaining + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 0); + assert!(entry.2 < 7); + } + assert_eq!( + blockstore + .transaction_status_index_cf + .get(1) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: index1_max_slot, + frozen: false, } - drop(status_entry_iterator); + ); + for _ in index0_max_slot + 1..index1_max_slot + 1 { + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 1); + } + drop(status_entry_iterator); - // Test purge inside index 1 at lower boundary - clear_and_repopulate_transaction_statuses( - &mut blockstore, - index0_max_slot, - index1_max_slot, - ); - blockstore.run_purge(10, 12, PurgeType::Exact).unwrap(); + // Test purge inside index 1 at lower boundary + clear_and_repopulate_transaction_statuses( + &mut blockstore, + index0_max_slot, + index1_max_slot, + ); + blockstore.run_purge(10, 12, PurgeType::Exact).unwrap(); - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index0_max_slot, - frozen: true, - } - ); - for _ in 0..index0_max_slot + 1 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(0), + IteratorDirection::Forward, + )) + .unwrap(); + assert_eq!( + blockstore + .transaction_status_index_cf + .get(0) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: index0_max_slot, + frozen: true, } - assert_eq!( - blockstore - .transaction_status_index_cf - .get(1) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index1_max_slot, - frozen: false, - } - ); - for _ in 13..index1_max_slot + 1 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 1); - assert!(entry.2 > 12); + ); + for _ in 0..index0_max_slot + 1 { + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 0); + } + assert_eq!( + blockstore + .transaction_status_index_cf + .get(1) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: index1_max_slot, + frozen: false, } - drop(status_entry_iterator); + ); + for _ in 13..index1_max_slot + 1 { + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 1); + assert!(entry.2 > 12); + } + drop(status_entry_iterator); - // Test purge across index boundaries - clear_and_repopulate_transaction_statuses( - &mut blockstore, - index0_max_slot, - index1_max_slot, - ); - blockstore.run_purge(7, 12, PurgeType::Exact).unwrap(); + // Test purge across index boundaries + clear_and_repopulate_transaction_statuses( + &mut blockstore, + index0_max_slot, + index1_max_slot, + ); + blockstore.run_purge(7, 12, PurgeType::Exact).unwrap(); - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: 6, - frozen: true, - } - ); - for _ in 0..7 { - // 7 entries remaining - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert!(entry.2 < 7); + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(0), + IteratorDirection::Forward, + )) + .unwrap(); + assert_eq!( + blockstore + .transaction_status_index_cf + .get(0) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: 6, + frozen: true, } - assert_eq!( - blockstore - .transaction_status_index_cf - .get(1) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index1_max_slot, - frozen: false, - } - ); - for _ in 13..index1_max_slot + 1 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 1); - assert!(entry.2 > 12); + ); + for _ in 0..7 { + // 7 entries remaining + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 0); + assert!(entry.2 < 7); + } + assert_eq!( + blockstore + .transaction_status_index_cf + .get(1) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: index1_max_slot, + frozen: false, } - drop(status_entry_iterator); + ); + for _ in 13..index1_max_slot + 1 { + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 1); + assert!(entry.2 > 12); + } + drop(status_entry_iterator); - // Test purge include complete index 1 - clear_and_repopulate_transaction_statuses( - &mut blockstore, - index0_max_slot, - index1_max_slot, - ); - blockstore.run_purge(7, 22, PurgeType::Exact).unwrap(); + // Test purge include complete index 1 + clear_and_repopulate_transaction_statuses( + &mut blockstore, + index0_max_slot, + index1_max_slot, + ); + blockstore.run_purge(7, 22, PurgeType::Exact).unwrap(); - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: 6, - frozen: true, - } - ); - for _ in 0..7 { - // 7 entries remaining - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert!(entry.2 < 7); + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(0), + IteratorDirection::Forward, + )) + .unwrap(); + assert_eq!( + blockstore + .transaction_status_index_cf + .get(0) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: 6, + frozen: true, } - assert_eq!( - blockstore - .transaction_status_index_cf - .get(1) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: 6, - frozen: false, - } - ); + ); + for _ in 0..7 { + // 7 entries remaining let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 2); // Buffer entry, no index 1 entries remaining - drop(status_entry_iterator); + assert_eq!(entry.0, 0); + assert!(entry.2 < 7); + } + assert_eq!( + blockstore + .transaction_status_index_cf + .get(1) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: 6, + frozen: false, + } + ); + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 2); // Buffer entry, no index 1 entries remaining + drop(status_entry_iterator); - // Test purge all - clear_and_repopulate_transaction_statuses( - &mut blockstore, - index0_max_slot, - index1_max_slot, - ); - blockstore.run_purge(0, 22, PurgeType::Exact).unwrap(); + // Test purge all + clear_and_repopulate_transaction_statuses( + &mut blockstore, + index0_max_slot, + index1_max_slot, + ); + blockstore.run_purge(0, 22, PurgeType::Exact).unwrap(); - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: 0, - frozen: true, - } - ); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(1) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: 0, - frozen: false, - } - ); - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 2); // Buffer entry, no index 0 or index 1 entries remaining - drop(status_entry_iterator); - } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(0), + IteratorDirection::Forward, + )) + .unwrap(); + assert_eq!( + blockstore + .transaction_status_index_cf + .get(0) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: 0, + frozen: true, + } + ); + assert_eq!( + blockstore + .transaction_status_index_cf + .get(1) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: 0, + frozen: false, + } + ); + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 2); // Buffer entry, no index 0 or index 1 entries remaining } #[test] fn test_purge_special_columns_exact_no_sigs() { - let blockstore_path = get_tmp_ledger_path!(); - { - let blockstore = Blockstore::open(&blockstore_path).unwrap(); - - let slot = 1; - let mut entries: Vec = vec![]; - for x in 0..5 { - let mut tx = Transaction::new_unsigned(Message::default()); - tx.signatures = vec![]; - entries.push(next_entry_mut(&mut Hash::default(), 0, vec![tx])); - let mut tick = create_ticks(1, 0, hash(&serialize(&x).unwrap())); - entries.append(&mut tick); - } - let shreds = entries_to_test_shreds(entries, slot, slot - 1, true, 0); - blockstore.insert_shreds(shreds, None, false).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - let mut write_batch = blockstore.db.batch().unwrap(); - blockstore - .purge_special_columns_exact(&mut write_batch, slot, slot + 1) - .unwrap(); + let slot = 1; + let mut entries: Vec = vec![]; + for x in 0..5 { + let mut tx = Transaction::new_unsigned(Message::default()); + tx.signatures = vec![]; + entries.push(next_entry_mut(&mut Hash::default(), 0, vec![tx])); + let mut tick = create_ticks(1, 0, hash(&serialize(&x).unwrap())); + entries.append(&mut tick); } - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); + let shreds = entries_to_test_shreds(entries, slot, slot - 1, true, 0); + blockstore.insert_shreds(shreds, None, false).unwrap(); + + let mut write_batch = blockstore.db.batch().unwrap(); + blockstore + .purge_special_columns_exact(&mut write_batch, slot, slot + 1) + .unwrap(); } } diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index 943bd782136c65..7a9981611c58be 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -1522,9 +1522,8 @@ pub mod tests { genesis_config.poh_config.hashes_per_tick = Some(hashes_per_tick); let ticks_per_slot = genesis_config.ticks_per_slot; - let (ledger_path, blockhash) = create_new_tmp_ledger!(&genesis_config); - let blockstore = - Blockstore::open(&ledger_path).expect("Expected to successfully open database ledger"); + let (ledger_path, blockhash) = create_new_tmp_ledger_auto_delete!(&genesis_config); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); let parent_slot = 0; let slot = 1; @@ -1563,8 +1562,8 @@ pub mod tests { let ticks_per_slot = genesis_config.ticks_per_slot; // Create a new ledger with slot 0 full of ticks - let (ledger_path, blockhash) = create_new_tmp_ledger!(&genesis_config); - let blockstore = Blockstore::open(&ledger_path).unwrap(); + let (ledger_path, blockhash) = create_new_tmp_ledger_auto_delete!(&genesis_config); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); // Write slot 1 with one tick missing let parent_slot = 0; @@ -1626,8 +1625,8 @@ pub mod tests { } = create_genesis_config(10_000); let ticks_per_slot = genesis_config.ticks_per_slot; - let (ledger_path, blockhash) = create_new_tmp_ledger!(&genesis_config); - let blockstore = Blockstore::open(&ledger_path).unwrap(); + let (ledger_path, blockhash) = create_new_tmp_ledger_auto_delete!(&genesis_config); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); let mut entries = create_ticks(ticks_per_slot, 0, blockhash); let trailing_entry = { @@ -1685,11 +1684,10 @@ pub mod tests { */ // Create a new ledger with slot 0 full of ticks - let (ledger_path, mut blockhash) = create_new_tmp_ledger!(&genesis_config); + let (ledger_path, mut blockhash) = create_new_tmp_ledger_auto_delete!(&genesis_config); debug!("ledger_path: {:?}", ledger_path); - let blockstore = - Blockstore::open(&ledger_path).expect("Expected to successfully open database ledger"); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); // Write slot 1 // slot 1, points at slot 0. Missing one tick @@ -1759,7 +1757,7 @@ pub mod tests { let ticks_per_slot = genesis_config.ticks_per_slot; // Create a new ledger with slot 0 full of ticks - let (ledger_path, blockhash) = create_new_tmp_ledger!(&genesis_config); + let (ledger_path, blockhash) = create_new_tmp_ledger_auto_delete!(&genesis_config); debug!("ledger_path: {:?}", ledger_path); let mut last_entry_hash = blockhash; @@ -1777,8 +1775,7 @@ pub mod tests { slot 4 <-- set_root(true) */ - let blockstore = - Blockstore::open(&ledger_path).expect("Expected to successfully open database ledger"); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); // Fork 1, ending at slot 3 let last_slot1_entry_hash = @@ -1838,7 +1835,7 @@ pub mod tests { let ticks_per_slot = genesis_config.ticks_per_slot; // Create a new ledger with slot 0 full of ticks - let (ledger_path, blockhash) = create_new_tmp_ledger!(&genesis_config); + let (ledger_path, blockhash) = create_new_tmp_ledger_auto_delete!(&genesis_config); debug!("ledger_path: {:?}", ledger_path); let mut last_entry_hash = blockhash; @@ -1856,8 +1853,7 @@ pub mod tests { slot 4 */ - let blockstore = - Blockstore::open(&ledger_path).expect("Expected to successfully open database ledger"); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); // Fork 1, ending at slot 3 let last_slot1_entry_hash = @@ -1926,7 +1922,7 @@ pub mod tests { let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000); let ticks_per_slot = genesis_config.ticks_per_slot; - let (ledger_path, blockhash) = create_new_tmp_ledger!(&genesis_config); + let (ledger_path, blockhash) = create_new_tmp_ledger_auto_delete!(&genesis_config); debug!("ledger_path: {:?}", ledger_path); /* @@ -1939,7 +1935,7 @@ pub mod tests { \ slot 3 */ - let blockstore = Blockstore::open(&ledger_path).unwrap(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); let slot1_blockhash = fill_blockstore_slot_with_ticks(&blockstore, ticks_per_slot, 1, 0, blockhash); fill_blockstore_slot_with_ticks(&blockstore, ticks_per_slot, 2, 1, slot1_blockhash); @@ -1968,7 +1964,7 @@ pub mod tests { let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000); let ticks_per_slot = genesis_config.ticks_per_slot; - let (ledger_path, blockhash) = create_new_tmp_ledger!(&genesis_config); + let (ledger_path, blockhash) = create_new_tmp_ledger_auto_delete!(&genesis_config); debug!("ledger_path: {:?}", ledger_path); /* @@ -1981,7 +1977,7 @@ pub mod tests { / \ slot 4 (dead) slot 3 */ - let blockstore = Blockstore::open(&ledger_path).unwrap(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); let slot1_blockhash = fill_blockstore_slot_with_ticks(&blockstore, ticks_per_slot, 1, 0, blockhash); let slot2_blockhash = @@ -2023,7 +2019,7 @@ pub mod tests { let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000); let ticks_per_slot = genesis_config.ticks_per_slot; - let (ledger_path, blockhash) = create_new_tmp_ledger!(&genesis_config); + let (ledger_path, blockhash) = create_new_tmp_ledger_auto_delete!(&genesis_config); debug!("ledger_path: {:?}", ledger_path); /* @@ -2032,7 +2028,7 @@ pub mod tests { / \ slot 1 (dead) slot 2 (dead) */ - let blockstore = Blockstore::open(&ledger_path).unwrap(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); fill_blockstore_slot_with_ticks(&blockstore, ticks_per_slot, 1, 0, blockhash); fill_blockstore_slot_with_ticks(&blockstore, ticks_per_slot, 2, 0, blockhash); blockstore.set_dead_slot(1).unwrap(); @@ -2053,11 +2049,10 @@ pub mod tests { let ticks_per_slot = genesis_config.ticks_per_slot; // Create a new ledger with slot 0 full of ticks - let (ledger_path, blockhash) = create_new_tmp_ledger!(&genesis_config); + let (ledger_path, blockhash) = create_new_tmp_ledger_auto_delete!(&genesis_config); let mut last_entry_hash = blockhash; - let blockstore = - Blockstore::open(&ledger_path).expect("Expected to successfully open database ledger"); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); // Let `last_slot` be the number of slots in the first two epochs let epoch_schedule = get_epoch_schedule(&genesis_config, Vec::new()); @@ -2178,7 +2173,8 @@ pub mod tests { .. } = create_genesis_config_with_leader(mint, &leader_pubkey, 50); genesis_config.poh_config.hashes_per_tick = Some(hashes_per_tick); - let (ledger_path, mut last_entry_hash) = create_new_tmp_ledger!(&genesis_config); + let (ledger_path, mut last_entry_hash) = + create_new_tmp_ledger_auto_delete!(&genesis_config); debug!("ledger_path: {:?}", ledger_path); let deducted_from_mint = 3; @@ -2212,8 +2208,7 @@ pub mod tests { )); let last_blockhash = entries.last().unwrap().hash; - let blockstore = - Blockstore::open(&ledger_path).expect("Expected to successfully open database ledger"); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); blockstore .write_entries( 1, @@ -2253,9 +2248,9 @@ pub mod tests { mut genesis_config, .. } = create_genesis_config(123); genesis_config.ticks_per_slot = 1; - let (ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_config); + let (ledger_path, _blockhash) = create_new_tmp_ledger_auto_delete!(&genesis_config); - let blockstore = Blockstore::open(&ledger_path).unwrap(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); let opts = ProcessOptions { poh_verify: true, accounts_db_test_hash_calculation: true, @@ -2271,9 +2266,9 @@ pub mod tests { #[test] fn test_process_ledger_options_override_threads() { let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(123); - let (ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_config); + let (ledger_path, _blockhash) = create_new_tmp_ledger_auto_delete!(&genesis_config); - let blockstore = Blockstore::open(&ledger_path).unwrap(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); let opts = ProcessOptions { override_num_threads: Some(1), accounts_db_test_hash_calculation: true, @@ -2288,9 +2283,9 @@ pub mod tests { #[test] fn test_process_ledger_options_full_leader_cache() { let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(123); - let (ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_config); + let (ledger_path, _blockhash) = create_new_tmp_ledger_auto_delete!(&genesis_config); - let blockstore = Blockstore::open(&ledger_path).unwrap(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); let opts = ProcessOptions { full_leader_cache: true, accounts_db_test_hash_calculation: true, @@ -2308,9 +2303,8 @@ pub mod tests { mint_keypair, .. } = create_genesis_config(100); - let (ledger_path, last_entry_hash) = create_new_tmp_ledger!(&genesis_config); - let blockstore = - Blockstore::open(&ledger_path).expect("Expected to successfully open database ledger"); + let (ledger_path, last_entry_hash) = create_new_tmp_ledger_auto_delete!(&genesis_config); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); let blockhash = genesis_config.hash(); let keypairs = [Keypair::new(), Keypair::new(), Keypair::new()]; @@ -2988,8 +2982,8 @@ pub mod tests { // Create roots at slots 0, 1 let forks = tr(0) / tr(1); - let ledger_path = get_tmp_ledger_path!(); - let blockstore = Blockstore::open(&ledger_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); blockstore.add_tree( forks, false, @@ -3021,8 +3015,8 @@ pub mod tests { let ticks_per_slot = 1; genesis_config.ticks_per_slot = ticks_per_slot; - let (ledger_path, blockhash) = create_new_tmp_ledger!(&genesis_config); - let blockstore = Blockstore::open(&ledger_path).unwrap(); + let (ledger_path, blockhash) = create_new_tmp_ledger_auto_delete!(&genesis_config); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); /* Build a blockstore in the ledger with the following fork structure: @@ -3615,8 +3609,8 @@ pub mod tests { vec![100], ); let ticks_per_slot = genesis_config.ticks_per_slot(); - let ledger_path = get_tmp_ledger_path!(); - let blockstore = Blockstore::open(&ledger_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); blockstore.add_tree(forks, false, true, ticks_per_slot, genesis_config.hash()); if let Some(blockstore_root) = blockstore_root { diff --git a/ledger/src/leader_schedule_cache.rs b/ledger/src/leader_schedule_cache.rs index 2c03eb3b1c9d7d..e040a95d3560cc 100644 --- a/ledger/src/leader_schedule_cache.rs +++ b/ledger/src/leader_schedule_cache.rs @@ -255,7 +255,7 @@ mod tests { bootstrap_validator_stake_lamports, create_genesis_config, create_genesis_config_with_leader, GenesisConfigInfo, }, - get_tmp_ledger_path, + get_tmp_ledger_path_auto_delete, staking_utils::tests::setup_vote_and_stake_accounts, }; use solana_runtime::bank::Bank; @@ -428,75 +428,71 @@ mod tests { let bank = Bank::new_for_tests(&genesis_config); let cache = Arc::new(LeaderScheduleCache::new_from_bank(&bank)); - let ledger_path = get_tmp_ledger_path!(); - { - let blockstore = Arc::new( - Blockstore::open(&ledger_path) - .expect("Expected to be able to open database ledger"), - ); + let ledger_path = get_tmp_ledger_path_auto_delete!(); - assert_eq!( - cache.slot_leader_at(bank.slot(), Some(&bank)).unwrap(), - pubkey - ); - // Check that the next leader slot after 0 is slot 1 - assert_eq!( - cache - .next_leader_slot(&pubkey, 0, &bank, Some(&blockstore), std::u64::MAX) - .unwrap() - .0, - 1 - ); + let blockstore = Blockstore::open(ledger_path.path()) + .expect("Expected to be able to open database ledger"); - // Write a shred into slot 2 that chains to slot 1, - // but slot 1 is empty so should not be skipped - let (shreds, _) = make_slot_entries(2, 1, 1); - blockstore.insert_shreds(shreds, None, false).unwrap(); - assert_eq!( - cache - .next_leader_slot(&pubkey, 0, &bank, Some(&blockstore), std::u64::MAX) - .unwrap() - .0, - 1 - ); + assert_eq!( + cache.slot_leader_at(bank.slot(), Some(&bank)).unwrap(), + pubkey + ); + // Check that the next leader slot after 0 is slot 1 + assert_eq!( + cache + .next_leader_slot(&pubkey, 0, &bank, Some(&blockstore), std::u64::MAX) + .unwrap() + .0, + 1 + ); - // Write a shred into slot 1 - let (shreds, _) = make_slot_entries(1, 0, 1); - - // Check that slot 1 and 2 are skipped - blockstore.insert_shreds(shreds, None, false).unwrap(); - assert_eq!( - cache - .next_leader_slot(&pubkey, 0, &bank, Some(&blockstore), std::u64::MAX) - .unwrap() - .0, - 3 - ); + // Write a shred into slot 2 that chains to slot 1, + // but slot 1 is empty so should not be skipped + let (shreds, _) = make_slot_entries(2, 1, 1); + blockstore.insert_shreds(shreds, None, false).unwrap(); + assert_eq!( + cache + .next_leader_slot(&pubkey, 0, &bank, Some(&blockstore), std::u64::MAX) + .unwrap() + .0, + 1 + ); - // Integrity checks - assert_eq!( - cache.next_leader_slot( - &pubkey, - 2 * genesis_config.epoch_schedule.slots_per_epoch - 1, // no schedule generated for epoch 2 - &bank, - Some(&blockstore), - std::u64::MAX - ), - None - ); + // Write a shred into slot 1 + let (shreds, _) = make_slot_entries(1, 0, 1); - assert_eq!( - cache.next_leader_slot( - &solana_sdk::pubkey::new_rand(), // not in leader_schedule - 0, - &bank, - Some(&blockstore), - std::u64::MAX - ), - None - ); - } - Blockstore::destroy(&ledger_path).unwrap(); + // Check that slot 1 and 2 are skipped + blockstore.insert_shreds(shreds, None, false).unwrap(); + assert_eq!( + cache + .next_leader_slot(&pubkey, 0, &bank, Some(&blockstore), std::u64::MAX) + .unwrap() + .0, + 3 + ); + + // Integrity checks + assert_eq!( + cache.next_leader_slot( + &pubkey, + 2 * genesis_config.epoch_schedule.slots_per_epoch - 1, // no schedule generated for epoch 2 + &bank, + Some(&blockstore), + std::u64::MAX + ), + None + ); + + assert_eq!( + cache.next_leader_slot( + &solana_sdk::pubkey::new_rand(), // not in leader_schedule + 0, + &bank, + Some(&blockstore), + std::u64::MAX + ), + None + ); } #[test] diff --git a/ledger/src/next_slots_iterator.rs b/ledger/src/next_slots_iterator.rs index f3c67a6cf0c643..842fae5cd1f982 100644 --- a/ledger/src/next_slots_iterator.rs +++ b/ledger/src/next_slots_iterator.rs @@ -41,8 +41,8 @@ mod tests { #[test] fn test_next_slots_iterator() { - let blockstore_path = get_tmp_ledger_path!(); - let blockstore = Blockstore::open(&blockstore_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); blockstore.set_roots(std::iter::once(&0)).unwrap(); let ticks_per_slot = 5; /* @@ -107,8 +107,5 @@ mod tests { .collect(); let expected = vec![4].into_iter().collect(); assert_eq!(result, expected); - - drop(blockstore); - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } } diff --git a/ledger/src/rooted_slot_iterator.rs b/ledger/src/rooted_slot_iterator.rs index cdc6f1c46efe8f..15c2cdb0ecfe5d 100644 --- a/ledger/src/rooted_slot_iterator.rs +++ b/ledger/src/rooted_slot_iterator.rs @@ -82,8 +82,8 @@ mod tests { #[test] fn test_rooted_slot_iterator() { - let blockstore_path = get_tmp_ledger_path!(); - let blockstore = Blockstore::open(&blockstore_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); blockstore.set_roots(std::iter::once(&0)).unwrap(); let ticks_per_slot = 5; /* @@ -150,15 +150,12 @@ mod tests { .collect(); let expected = vec![0, 1, 2, 3]; assert_eq!(result, expected); - - drop(blockstore); - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } #[test] fn test_skipping_rooted_slot_iterator() { - let blockstore_path = get_tmp_ledger_path!(); - let blockstore = Blockstore::open(&blockstore_path).unwrap(); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); let ticks_per_slot = 5; /* Build a blockstore in the ledger with the following fork structure: @@ -229,8 +226,5 @@ mod tests { (11, true), ]; assert_eq!(result, expected); - - drop(blockstore); - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); } } diff --git a/ledger/tests/blockstore.rs b/ledger/tests/blockstore.rs index aac9e072edccf9..a645c8fd0cf65e 100644 --- a/ledger/tests/blockstore.rs +++ b/ledger/tests/blockstore.rs @@ -1,7 +1,7 @@ use solana_entry::entry; use solana_ledger::{ blockstore::{self, Blockstore}, - get_tmp_ledger_path, + get_tmp_ledger_path_auto_delete, }; use solana_sdk::hash::Hash; use std::sync::Arc; @@ -9,8 +9,8 @@ use std::thread::Builder; #[test] fn test_multiple_threads_insert_shred() { - let blockstore_path = get_tmp_ledger_path!(); - let blockstore = Arc::new(Blockstore::open(&blockstore_path).unwrap()); + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Arc::new(Blockstore::open(ledger_path.path()).unwrap()); for _ in 0..100 { let num_threads = 10; @@ -44,8 +44,4 @@ fn test_multiple_threads_insert_shred() { // Delete slots for next iteration blockstore.purge_and_compact_slots(0, num_threads + 1); } - - // Cleanup - drop(blockstore); - Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); }