From 8b0541ed29d33d4d8a077d403bb91325bac99c37 Mon Sep 17 00:00:00 2001 From: ppca Date: Mon, 31 Jul 2023 11:14:51 -0700 Subject: [PATCH] fix(state-sync): save to BlockMisc:STATE_SNAPSHOT_KEY the latest snapshot's hash (#9343) and a few other fixes: - retry setting and deleting snapshots from rocksdb and file system until success - replace all unwrap() of state_snapshot locks that may result in unable to start node with returning errors --- core/store/src/db.rs | 1 + core/store/src/lib.rs | 13 +- core/store/src/trie/shard_tries.rs | 208 ++++++++------ integration-tests/src/tests/nearcore/mod.rs | 1 + .../src/tests/nearcore/state_snapshot.rs | 270 ++++++++++++++++++ 5 files changed, 410 insertions(+), 83 deletions(-) create mode 100644 integration-tests/src/tests/nearcore/state_snapshot.rs diff --git a/core/store/src/db.rs b/core/store/src/db.rs index 83c4b971803..75dce86ac4e 100644 --- a/core/store/src/db.rs +++ b/core/store/src/db.rs @@ -33,6 +33,7 @@ pub const GENESIS_JSON_HASH_KEY: &[u8; 17] = b"GENESIS_JSON_HASH"; pub const GENESIS_STATE_ROOTS_KEY: &[u8; 19] = b"GENESIS_STATE_ROOTS"; pub const COLD_HEAD_KEY: &[u8; 9] = b"COLD_HEAD"; pub const STATE_SYNC_DUMP_KEY: &[u8; 15] = b"STATE_SYNC_DUMP"; +pub const STATE_SNAPSHOT_KEY: &[u8; 18] = b"STATE_SNAPSHOT_KEY"; // `DBCol::Misc` keys pub const FLAT_STATE_VALUES_INLINING_MIGRATION_STATUS_KEY: &[u8] = diff --git a/core/store/src/lib.rs b/core/store/src/lib.rs index 357db5a1c4d..ef2d0d79359 100644 --- a/core/store/src/lib.rs +++ b/core/store/src/lib.rs @@ -15,7 +15,7 @@ pub use columns::DBCol; pub use db::{ CHUNK_TAIL_KEY, COLD_HEAD_KEY, FINAL_HEAD_KEY, FORK_TAIL_KEY, GENESIS_JSON_HASH_KEY, GENESIS_STATE_ROOTS_KEY, HEADER_HEAD_KEY, HEAD_KEY, LARGEST_TARGET_HEIGHT_KEY, - LATEST_KNOWN_KEY, STATE_SYNC_DUMP_KEY, TAIL_KEY, + LATEST_KNOWN_KEY, STATE_SNAPSHOT_KEY, STATE_SYNC_DUMP_KEY, TAIL_KEY, }; use near_crypto::PublicKey; use near_fmt::{AbbrBytes, StorageKey}; @@ -856,6 +856,17 @@ pub fn set_genesis_state_roots(store_update: &mut StoreUpdate, genesis_roots: &[ .expect("Borsh cannot fail"); } +fn option_to_not_found(res: io::Result>, field_name: F) -> io::Result +where + F: std::string::ToString, +{ + match res { + Ok(Some(o)) => Ok(o), + Ok(None) => Err(io::Error::new(io::ErrorKind::NotFound, field_name.to_string())), + Err(e) => Err(e), + } +} + pub struct StoreCompiledContractCache { db: Arc, } diff --git a/core/store/src/trie/shard_tries.rs b/core/store/src/trie/shard_tries.rs index 0c74dd4992a..bacd26ef0f8 100644 --- a/core/store/src/trie/shard_tries.rs +++ b/core/store/src/trie/shard_tries.rs @@ -1,4 +1,6 @@ +use crate::db::STATE_SNAPSHOT_KEY; use crate::flat::FlatStorageManager; +use crate::option_to_not_found; use crate::trie::config::TrieConfig; use crate::trie::prefetching_trie_storage::PrefetchingThreadsHandle; use crate::trie::trie_storage::{TrieCache, TrieCachingStorage}; @@ -18,9 +20,9 @@ use near_primitives::trie_key::TrieKey; use near_primitives::types::{ NumShards, RawStateChange, RawStateChangesWithTrieKey, StateChangeCause, StateRoot, }; +use std::io; use std::path::{Path, PathBuf}; use std::rc::Rc; -use std::str::FromStr; use std::sync::{Arc, RwLock, TryLockError}; struct ShardTriesInner { @@ -475,21 +477,55 @@ impl ShardTries { let _timer = metrics::MAKE_STATE_SNAPSHOT_ELAPSED.start_timer(); // `write()` lock is held for the whole duration of this function. // Accessing the snapshot in other parts of the system will fail. - let mut state_snapshot_lock = self.0.state_snapshot.write().unwrap(); + let mut state_snapshot_lock = self.0.state_snapshot.write().map_err(|_| { + anyhow::Error::msg("error accessing write lock of state_snapshot") + })?; + let db_snapshot_hash = self.get_state_snapshot_hash(); + if let Some(state_snapshot) = &*state_snapshot_lock { - if &state_snapshot.prev_block_hash == prev_block_hash { + // only return Ok() when the hash stored in STATE_SNAPSHOT_KEY and in state_snapshot_lock and prev_block_hash are the same + if db_snapshot_hash.is_ok() + && db_snapshot_hash.unwrap() == *prev_block_hash + && state_snapshot.prev_block_hash == *prev_block_hash + { tracing::warn!(target: "state_snapshot", ?prev_block_hash, "Requested a state snapshot but that is already available"); return Ok(()); } else { - let prev_block_hash = state_snapshot.prev_block_hash; // Drop Store before deleting the underlying data. *state_snapshot_lock = None; - self.delete_state_snapshot( - &prev_block_hash, - home_dir, - hot_store_path, - state_snapshot_subdir, - ); + + // This will delete all existing snapshots from file system. If failed, will retry until success + let mut delete_state_snapshots_from_file_system = false; + let mut file_system_delete_retries = 0; + while !delete_state_snapshots_from_file_system + && file_system_delete_retries < 3 + { + delete_state_snapshots_from_file_system = self + .delete_all_state_snapshots( + home_dir, + hot_store_path, + state_snapshot_subdir, + ); + file_system_delete_retries += 1; + } + + // this will delete the STATE_SNAPSHOT_KEY-value pair from db. If failed, will retry until success + let mut delete_state_snapshot_from_db = false; + let mut db_delete_retries = 0; + while !delete_state_snapshot_from_db && db_delete_retries < 3 { + delete_state_snapshot_from_db = match self.set_state_snapshot_hash(None) + { + Ok(_) => true, + Err(err) => { + // This will be retried. + tracing::debug!(target: "state_snapshot", ?err, "Failed to delete the old state snapshot for BlockMisc::STATE_SNAPSHOT_KEY in rocksdb"); + false + } + }; + db_delete_retries += 1; + } + + metrics::HAS_STATE_SNAPSHOT.set(0); } } @@ -526,6 +562,22 @@ impl ShardTries { shard_uids, Some(block), )); + + // this will set the new hash for state snapshot in rocksdb. will retry until success. + let mut set_state_snapshot_in_db = false; + while !set_state_snapshot_in_db { + set_state_snapshot_in_db = match self + .set_state_snapshot_hash(Some(*prev_block_hash)) + { + Ok(_) => true, + Err(err) => { + // This will be retried. + tracing::debug!(target: "state_snapshot", ?err, "Failed to set the new state snapshot for BlockMisc::STATE_SNAPSHOT_KEY in rocksdb"); + false + } + } + } + metrics::HAS_STATE_SNAPSHOT.set(1); tracing::info!(target: "state_snapshot", ?prev_block_hash, "Made a checkpoint"); Ok(()) @@ -538,7 +590,11 @@ impl ShardTries { let _span = tracing::info_span!(target: "state_snapshot", "compact_state_snapshot").entered(); // It's fine if the access to state snapshot blocks. - let state_snapshot_lock = self.0.state_snapshot.read().unwrap(); + let state_snapshot_lock = self + .0 + .state_snapshot + .read() + .map_err(|_| anyhow::Error::msg("error accessing read lock of state_snapshot"))?; if let Some(state_snapshot) = &*state_snapshot_lock { let _timer = metrics::COMPACT_STATE_SNAPSHOT_ELAPSED.start_timer(); Ok(state_snapshot.store.compact()?) @@ -548,30 +604,25 @@ impl ShardTries { } } - /// Deletes a previously open state snapshot. - /// Drops the Store and deletes the files. - fn delete_state_snapshot( + /// Deletes all existing state snapshots in the parent directory + fn delete_all_state_snapshots( &self, - prev_block_hash: &CryptoHash, home_dir: &Path, hot_store_path: &Path, state_snapshot_subdir: &Path, - ) { + ) -> bool { let _timer = metrics::DELETE_STATE_SNAPSHOT_ELAPSED.start_timer(); let _span = tracing::info_span!(target: "state_snapshot", "delete_state_snapshot").entered(); - let path = Self::get_state_snapshot_base_dir( - prev_block_hash, - home_dir, - hot_store_path, - state_snapshot_subdir, - ); + let path = home_dir.join(hot_store_path).join(state_snapshot_subdir); match std::fs::remove_dir_all(&path) { Ok(_) => { - tracing::info!(target: "state_snapshot", ?path, ?prev_block_hash, "Deleted a state snapshot"); + tracing::info!(target: "state_snapshot", ?path, "Deleted all state snapshots"); + true } Err(err) => { - tracing::warn!(target: "state_snapshot", ?err, ?path, ?prev_block_hash, "Failed to delete a state snapshot"); + tracing::warn!(target: "state_snapshot", ?err, ?path, "Failed to delete all state snapshots"); + false } } } @@ -588,8 +639,27 @@ impl ShardTries { home_dir.join(hot_store_path).join(state_snapshot_subdir).join(format!("{prev_block_hash}")) } - /// Looks for directories on disk with names that look like `prev_block_hash`. - /// Checks that there is at most one such directory. Opens it as a Store. + /// Retrieves STATE_SNAPSHOT_KEY + pub fn get_state_snapshot_hash(&self) -> Result { + option_to_not_found( + self.0.store.get_ser(DBCol::BlockMisc, STATE_SNAPSHOT_KEY), + "STATE_SNAPSHOT_KEY", + ) + } + + /// Updates STATE_SNAPSHOT_KEY. + pub fn set_state_snapshot_hash(&self, value: Option) -> Result<(), io::Error> { + let mut store_update = self.0.store.store_update(); + let key = STATE_SNAPSHOT_KEY; + match value { + None => store_update.delete(DBCol::BlockMisc, key), + Some(value) => store_update.set_ser(DBCol::BlockMisc, key, &value)?, + } + store_update.commit().map_err(|err| err.into()) + } + + /// Read RocksDB for the latest available snapshot hash, if available, open base_path+snapshot_hash for the state snapshot + /// we don't deal with multiple snapshots here because we will deal with it whenever a new snapshot is created and saved to file system pub fn maybe_open_state_snapshot( &self, get_shard_uids_fn: impl Fn(CryptoHash) -> Result, EpochError>, @@ -608,67 +678,41 @@ impl ShardTries { state_snapshot_subdir, compaction_enabled: _, } => { - let path = Self::get_state_snapshot_base_dir( - &CryptoHash::new(), + // directly return error if no snapshot is found + let snapshot_hash: CryptoHash = self.get_state_snapshot_hash()?; + + let snapshot_path = Self::get_state_snapshot_base_dir( + &snapshot_hash, &home_dir, &hot_store_path, &state_snapshot_subdir, ); - let parent_path = - path.parent().ok_or(anyhow::anyhow!("{path:?} needs to have a parent dir"))?; - tracing::debug!(target: "state_snapshot", ?path, ?parent_path); - - let snapshots = match std::fs::read_dir(parent_path) { - Err(err) => { - if err.kind() == std::io::ErrorKind::NotFound { - tracing::debug!(target: "state_snapshot", ?parent_path, "State Snapshot base directory doesn't exist."); - return Ok(()); - } else { - return Err(err.into()); - } - } - Ok(entries) => { - let mut snapshots = vec![]; - for entry in entries.filter_map(Result::ok) { - let file_name = entry.file_name().into_string().map_err(|err| { - anyhow::anyhow!("Can't display file_name: {err:?}") - })?; - if let Ok(prev_block_hash) = CryptoHash::from_str(&file_name) { - snapshots.push((prev_block_hash, entry.path())); - } - } - snapshots - } - }; + let parent_path = snapshot_path + .parent() + .ok_or(anyhow::anyhow!("{snapshot_path:?} needs to have a parent dir"))?; + tracing::debug!(target: "state_snapshot", ?snapshot_path, ?parent_path); - if snapshots.is_empty() { - Ok(()) - } else if snapshots.len() == 1 { - let (prev_block_hash, snapshot_dir) = &snapshots[0]; - - let store_config = StoreConfig::default(); - - let opener = NodeStorage::opener(&snapshot_dir, false, &store_config, None); - let storage = opener.open_in_mode(Mode::ReadOnly)?; - let store = storage.get_hot_store(); - let flat_storage_manager = FlatStorageManager::new(store.clone()); - - let shard_uids = get_shard_uids_fn(*prev_block_hash)?; - let mut guard = self.0.state_snapshot.write().unwrap(); - *guard = Some(StateSnapshot::new( - store, - *prev_block_hash, - flat_storage_manager, - &shard_uids, - None, - )); - metrics::HAS_STATE_SNAPSHOT.set(1); - tracing::info!(target: "runtime", ?prev_block_hash, ?snapshot_dir, "Detected and opened a state snapshot."); - Ok(()) - } else { - tracing::error!(target: "runtime", ?snapshots, "Detected multiple state snapshots. Please keep at most one snapshot and delete others."); - Err(anyhow::anyhow!("More than one state snapshot detected {:?}", snapshots)) - } + let store_config = StoreConfig::default(); + + let opener = NodeStorage::opener(&snapshot_path, false, &store_config, None); + let storage = opener.open_in_mode(Mode::ReadOnly)?; + let store = storage.get_hot_store(); + let flat_storage_manager = FlatStorageManager::new(store.clone()); + + let shard_uids = get_shard_uids_fn(snapshot_hash)?; + let mut guard = self.0.state_snapshot.write().map_err(|_| { + anyhow::Error::msg("error accessing write lock of state_snapshot") + })?; + *guard = Some(StateSnapshot::new( + store, + snapshot_hash, + flat_storage_manager, + &shard_uids, + None, + )); + metrics::HAS_STATE_SNAPSHOT.set(1); + tracing::info!(target: "runtime", ?snapshot_hash, ?snapshot_path, "Detected and opened a state snapshot."); + Ok(()) } } } diff --git a/integration-tests/src/tests/nearcore/mod.rs b/integration-tests/src/tests/nearcore/mod.rs index 19145d13676..80d5811916e 100644 --- a/integration-tests/src/tests/nearcore/mod.rs +++ b/integration-tests/src/tests/nearcore/mod.rs @@ -3,6 +3,7 @@ mod rpc_error_structs; mod rpc_nodes; mod run_nodes; mod stake_nodes; +mod state_snapshot; mod sync_nodes; mod sync_state_nodes; mod track_shards; diff --git a/integration-tests/src/tests/nearcore/state_snapshot.rs b/integration-tests/src/tests/nearcore/state_snapshot.rs new file mode 100644 index 00000000000..eaced459ca9 --- /dev/null +++ b/integration-tests/src/tests/nearcore/state_snapshot.rs @@ -0,0 +1,270 @@ +use near_chain::types::RuntimeAdapter; +use near_chain::{ChainGenesis, Provenance}; +use near_chain_configs::Genesis; +use near_client::test_utils::TestEnv; +use near_client::ProcessTxResponse; +use near_crypto::{InMemorySigner, KeyType, Signer}; +use near_epoch_manager::{EpochManager, EpochManagerHandle}; +use near_o11y::testonly::init_test_logger; +use near_primitives::block::Block; +use near_primitives::hash::CryptoHash; +use near_primitives::shard_layout::ShardUId; +use near_primitives::transaction::SignedTransaction; +use near_store::flat::FlatStorageManager; +use near_store::genesis::initialize_genesis_state; +use near_store::{ + config::TrieCacheConfig, test_utils::create_test_store, Mode, ShardTries, StateSnapshotConfig, + StoreConfig, TrieConfig, +}; +use near_store::{NodeStorage, Store}; +use nearcore::config::GenesisExt; +use nearcore::{NightshadeRuntime, NEAR_BASE}; +use std::path::PathBuf; +use std::sync::Arc; + +struct StateSnaptshotTestEnv { + home_dir: PathBuf, + hot_store_path: PathBuf, + state_snapshot_subdir: PathBuf, + shard_tries: ShardTries, +} + +impl StateSnaptshotTestEnv { + fn new( + home_dir: PathBuf, + hot_store_path: PathBuf, + state_snapshot_subdir: PathBuf, + store: &Store, + ) -> Self { + let trie_cache_config = TrieCacheConfig { + default_max_bytes: 50_000_000, + per_shard_max_bytes: Default::default(), + shard_cache_deletions_queue_capacity: 0, + }; + let trie_config = TrieConfig { + shard_cache_config: trie_cache_config.clone(), + view_shard_cache_config: trie_cache_config, + enable_receipt_prefetching: false, + sweat_prefetch_receivers: Vec::new(), + sweat_prefetch_senders: Vec::new(), + }; + let flat_storage_manager = FlatStorageManager::new(store.clone()); + let shard_uids = [ShardUId::single_shard()]; + let state_snapshot_config = StateSnapshotConfig::Enabled { + home_dir: home_dir.clone(), + hot_store_path: hot_store_path.clone(), + state_snapshot_subdir: state_snapshot_subdir.clone(), + compaction_enabled: true, + }; + let shard_tries = ShardTries::new_with_state_snapshot( + store.clone(), + trie_config, + &shard_uids, + flat_storage_manager, + state_snapshot_config, + ); + Self { home_dir, hot_store_path, state_snapshot_subdir, shard_tries } + } +} + +fn set_up_test_env_for_state_snapshots(store: &Store) -> StateSnaptshotTestEnv { + let home_dir = + tempfile::Builder::new().prefix("storage").tempdir().unwrap().path().to_path_buf(); + let hot_store_path = PathBuf::from("data"); + let state_snapshot_subdir = PathBuf::from("state_snapshot"); + StateSnaptshotTestEnv::new(home_dir, hot_store_path, state_snapshot_subdir, store) +} + +#[test] +// there's no entry in rocksdb for STATE_SNAPSHOT_KEY, maybe_open_state_snapshot should return error instead of panic +fn test_maybe_open_state_snapshot_no_state_snapshot_key_entry() { + init_test_logger(); + let store = create_test_store(); + let test_env = set_up_test_env_for_state_snapshots(&store); + let result = + test_env.shard_tries.maybe_open_state_snapshot(|_| Ok(vec![ShardUId::single_shard()])); + assert!(result.is_err()); +} + +#[test] +// there's no file present in the path for state snapshot, maybe_open_state_snapshot should return error instead of panic +fn test_maybe_open_state_snapshot_file_not_exist() { + init_test_logger(); + let store = create_test_store(); + let test_env = set_up_test_env_for_state_snapshots(&store); + let snapshot_hash = CryptoHash::new(); + test_env.shard_tries.set_state_snapshot_hash(Some(snapshot_hash)).unwrap(); + let result = + test_env.shard_tries.maybe_open_state_snapshot(|_| Ok(vec![ShardUId::single_shard()])); + assert!(result.is_err()); +} + +#[test] +// there's garbage in the path for state snapshot, maybe_open_state_snapshot should return error instead of panic +fn test_maybe_open_state_snapshot_garbage_snapshot() { + use std::fs::{create_dir_all, File}; + use std::io::Write; + use std::path::Path; + init_test_logger(); + let store = create_test_store(); + let test_env = set_up_test_env_for_state_snapshots(&store); + let snapshot_hash = CryptoHash::new(); + test_env.shard_tries.set_state_snapshot_hash(Some(snapshot_hash)).unwrap(); + let snapshot_path = ShardTries::get_state_snapshot_base_dir( + &snapshot_hash, + &test_env.home_dir, + &test_env.hot_store_path, + &test_env.state_snapshot_subdir, + ); + if let Some(parent) = Path::new(&snapshot_path).parent() { + create_dir_all(parent).unwrap(); + } + let mut file = File::create(snapshot_path).unwrap(); + // write some garbage + let data: Vec = vec![1, 2, 3, 4]; + file.write_all(&data).unwrap(); + + let result = + test_env.shard_tries.maybe_open_state_snapshot(|_| Ok(vec![ShardUId::single_shard()])); + assert!(result.is_err()); +} + +fn verify_make_snapshot( + state_snapshot_test_env: &StateSnaptshotTestEnv, + block_hash: CryptoHash, + block: &Block, +) -> Result<(), anyhow::Error> { + state_snapshot_test_env.shard_tries.make_state_snapshot( + &block_hash, + &vec![ShardUId::single_shard()], + block, + )?; + // check that make_state_snapshot does not panic or err out + // assert!(res.is_ok()); + let snapshot_path = ShardTries::get_state_snapshot_base_dir( + &block_hash, + &state_snapshot_test_env.home_dir, + &state_snapshot_test_env.hot_store_path, + &state_snapshot_test_env.state_snapshot_subdir, + ); + // check that the snapshot just made can be opened + state_snapshot_test_env + .shard_tries + .maybe_open_state_snapshot(|_| Ok(vec![ShardUId::single_shard()]))?; + // check that the entry of STATE_SNAPSHOT_KEY is the latest block hash + let db_state_snapshot_hash = state_snapshot_test_env.shard_tries.get_state_snapshot_hash()?; + if db_state_snapshot_hash != block_hash { + return Err(anyhow::Error::msg( + "the entry of STATE_SNAPSHOT_KEY does not equal to the prev block hash", + )); + } + // check that the stored snapshot in file system is an actual snapshot + let store_config = StoreConfig::default(); + let opener = NodeStorage::opener(&snapshot_path, false, &store_config, None); + let _storage = opener.open_in_mode(Mode::ReadOnly)?; + // check that there's only one snapshot at the parent directory of snapshot path + let parent_path = snapshot_path + .parent() + .ok_or(anyhow::anyhow!("{snapshot_path:?} needs to have a parent dir"))?; + let parent_path_result = std::fs::read_dir(parent_path)?; + if vec![parent_path_result.filter_map(Result::ok)].len() > 1 { + return Err(anyhow::Error::msg( + "there are more than 1 snapshot file in the snapshot parent directory", + )); + } + return Ok(()); +} + +fn delete_content_at_path(path: &str) -> std::io::Result<()> { + let metadata = std::fs::metadata(path)?; + if metadata.is_dir() { + std::fs::remove_dir_all(path)?; + } else { + std::fs::remove_file(path)?; + } + Ok(()) +} + +#[test] +fn test_make_state_snapshot() { + init_test_logger(); + let genesis = Genesis::test(vec!["test0".parse().unwrap()], 1); + let num_clients = 1; + let env_objects = (0..num_clients).map(|_|{ + let tmp_dir = tempfile::tempdir().unwrap(); + // Use default StoreConfig rather than NodeStorage::test_opener so we’re using the + // same configuration as in production. + let store = NodeStorage::opener(&tmp_dir.path(), false, &Default::default(), None) + .open() + .unwrap() + .get_hot_store(); + initialize_genesis_state(store.clone(), &genesis, Some(tmp_dir.path())); + let epoch_manager = EpochManager::new_arc_handle(store.clone(), &genesis.config); + let runtime = + NightshadeRuntime::test(tmp_dir.path(), store.clone(), &genesis.config, epoch_manager.clone()) + as Arc; + (tmp_dir, store, epoch_manager, runtime) + }).collect::, Arc)>>(); + + let stores = env_objects.iter().map(|x| x.1.clone()).collect::>(); + let epoch_managers = env_objects.iter().map(|x| x.2.clone()).collect::>(); + let runtimes = env_objects.iter().map(|x| x.3.clone()).collect::>(); + + let mut env = TestEnv::builder(ChainGenesis::test()) + .clients_count(env_objects.len()) + .stores(stores.clone()) + .epoch_managers(epoch_managers) + .runtimes(runtimes.clone()) + .use_state_snapshots() + .build(); + + let signer = InMemorySigner::from_seed("test0".parse().unwrap(), KeyType::ED25519, "test0"); + let genesis_block = env.clients[0].chain.get_block_by_height(0).unwrap(); + let genesis_hash = *genesis_block.hash(); + + let mut blocks = vec![]; + + let state_snapshot_test_env = set_up_test_env_for_state_snapshots(&stores[0]); + + for i in 1..=5 { + let new_account_id = format!("test_account_{i}"); + let nonce = i; + let tx = SignedTransaction::create_account( + nonce, + "test0".parse().unwrap(), + new_account_id.parse().unwrap(), + NEAR_BASE, + signer.public_key(), + &signer, + genesis_hash, + ); + assert_eq!(env.clients[0].process_tx(tx, false, false), ProcessTxResponse::ValidTx); + let block = env.clients[0].produce_block(i).unwrap().unwrap(); + blocks.push(block.clone()); + env.process_block(0, block.clone(), Provenance::PRODUCED); + assert!(verify_make_snapshot(&state_snapshot_test_env, *block.hash(), &block).is_ok()); + } + + // check that if the entry in DBCol::STATE_SNAPSHOT_KEY was missing while snapshot file exists, an overwrite of snapshot can succeed + state_snapshot_test_env.shard_tries.set_state_snapshot_hash(None).unwrap(); + let head = env.clients[0].chain.head().unwrap(); + let head_block_hash = head.last_block_hash; + let head_block = env.clients[0].chain.get_block(&head_block_hash).unwrap(); + assert!(verify_make_snapshot(&state_snapshot_test_env, head_block_hash, &head_block).is_ok()); + + // check that if the snapshot is deleted from file system while there's entry in DBCol::STATE_SNAPSHOT_KEY and write lock is nonempty, making a snpashot of the same hash will not write to the file system + let snapshot_hash = head.last_block_hash; + let snapshot_path = ShardTries::get_state_snapshot_base_dir( + &snapshot_hash, + &state_snapshot_test_env.home_dir, + &state_snapshot_test_env.hot_store_path, + &state_snapshot_test_env.state_snapshot_subdir, + ); + delete_content_at_path(snapshot_path.to_str().unwrap()).unwrap(); + assert!( + verify_make_snapshot(&state_snapshot_test_env, head.last_block_hash, &head_block).is_err() + ); + if let Ok(entries) = std::fs::read_dir(snapshot_path) { + assert_eq!(entries.count(), 0); + } +}