Revert "Create bank snapshots (solana-labs#3671)" (solana-labs#4243)

This reverts commit abf2b30.

core/src/bank_forks.rs

@@ -1,25 +1,17 @@
 //! The `bank_forks` module implments BankForks a DAG of checkpointed Banks
 
-use bincode::{deserialize_from, serialize_into};
+use hashbrown::{HashMap, HashSet};
 use solana_metrics::counter::Counter;
-use solana_runtime::bank::{Bank, BankRc};
+use solana_runtime::bank::Bank;
 use solana_sdk::timing;
-use std::collections::{HashMap, HashSet};
-use std::env;
-use std::fs;
-use std::fs::File;
-use std::io::{BufReader, BufWriter, Error, ErrorKind};
 use std::ops::Index;
-use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::Instant;
 
 pub struct BankForks {
     banks: HashMap<u64, Arc<Bank>>,
     working_bank: Arc<Bank>,
     root: u64,
-    slots: HashSet<u64>,
-    use_snapshot: bool,
 }
 
 impl Index<u64> for BankForks {
@@ -38,8 +30,6 @@ impl BankForks {
             banks,
             working_bank,
             root: 0,
-            slots: HashSet::new(),
-            use_snapshot: false,
         }
     }
 
@@ -55,7 +45,6 @@ impl BankForks {
     }
 
     /// Create a map of bank slot id to the set of all of its descendants
-    #[allow(clippy::or_fun_call)]
     pub fn descendants(&self) -> HashMap<u64, HashSet<u64>> {
         let mut descendants = HashMap::new();
         for bank in self.banks.values() {
@@ -102,8 +91,6 @@ impl BankForks {
             root,
             banks,
             working_bank,
-            slots: HashSet::new(),
-            use_snapshot: false,
         }
     }
 
@@ -141,162 +128,9 @@ impl BankForks {
     }
 
     fn prune_non_root(&mut self, root: u64) {
-        let slots: HashSet<u64> = self
-            .banks
-            .iter()
-            .filter(|(_, b)| b.is_frozen())
-            .map(|(k, _)| *k)
-            .collect();
         let descendants = self.descendants();
         self.banks
-            .retain(|slot, _| descendants[&root].contains(slot));
-        if self.use_snapshot {
-            let diff: HashSet<_> = slots.symmetric_difference(&self.slots).collect();
-            trace!("prune non root {} - {:?}", root, diff);
-            for slot in diff.iter() {
-                if **slot > root {
-                    let _ = self.add_snapshot(**slot, root);
-                } else if **slot > 0 {
-                    self.remove_snapshot(**slot);
-                }
-            }
-        }
-        self.slots = slots.clone();
-    }
-
-    fn get_io_error(error: &str) -> Error {
-        Error::new(ErrorKind::Other, error)
-    }
-
-    fn get_snapshot_path() -> PathBuf {
-        let out_dir = env::var("OUT_DIR").unwrap_or_else(|_| "target".to_string());
-        let snapshot_dir = format!("{}/snapshots/", out_dir);
-        Path::new(&snapshot_dir).to_path_buf()
-    }
-
-    pub fn add_snapshot(&self, slot: u64, root: u64) -> Result<(), Error> {
-        let path = BankForks::get_snapshot_path();
-        fs::create_dir_all(path.clone())?;
-        let bank_file = format!("{}", slot);
-        let bank_file_path = path.join(bank_file);
-        trace!("path: {:?}", bank_file_path);
-        let file = File::create(bank_file_path)?;
-        let mut stream = BufWriter::new(file);
-        let bank = self.get(slot).unwrap().clone();
-        serialize_into(&mut stream, &*bank)
-            .map_err(|_| BankForks::get_io_error("serialize bank error"))?;
-        let mut parent_slot: u64 = 0;
-        if let Some(parent_bank) = bank.parent() {
-            parent_slot = parent_bank.slot();
-        }
-        serialize_into(&mut stream, &parent_slot)
-            .map_err(|_| BankForks::get_io_error("serialize bank parent error"))?;
-        serialize_into(&mut stream, &bank.rc)
-            .map_err(|_| BankForks::get_io_error("serialize bank rc error"))?;
-        serialize_into(&mut stream, &root)
-            .map_err(|_| BankForks::get_io_error("serialize root error"))?;
-        Ok(())
-    }
-
-    pub fn remove_snapshot(&self, slot: u64) {
-        let path = BankForks::get_snapshot_path();
-        let bank_file = format!("{}", slot);
-        let bank_file_path = path.join(bank_file);
-        let _ = fs::remove_file(bank_file_path);
-    }
-
-    pub fn set_snapshot_config(&mut self, use_snapshot: bool) {
-        self.use_snapshot = use_snapshot;
-    }
-
-    fn setup_banks(
-        bank_maps: &mut Vec<(u64, u64, Bank)>,
-        bank_rc: &BankRc,
-    ) -> (HashMap<u64, Arc<Bank>>, HashSet<u64>, u64) {
-        let mut banks = HashMap::new();
-        let mut slots = HashSet::new();
-        let (last_slot, last_parent_slot, mut last_bank) = bank_maps.remove(0);
-        last_bank.set_bank_rc(&bank_rc);
-
-        while let Some((slot, parent_slot, mut bank)) = bank_maps.pop() {
-            bank.set_bank_rc(&bank_rc);
-            if parent_slot != 0 {
-                if let Some(parent) = banks.get(&parent_slot) {
-                    bank.set_parent(parent);
-                }
-            }
-            banks.insert(slot, Arc::new(bank));
-            slots.insert(slot);
-        }
-        if last_parent_slot != 0 {
-            if let Some(parent) = banks.get(&last_parent_slot) {
-                last_bank.set_parent(parent);
-            }
-        }
-        banks.insert(last_slot, Arc::new(last_bank));
-        slots.insert(last_slot);
-
-        (banks, slots, last_slot)
-    }
-
-    pub fn load_from_snapshot() -> Result<Self, Error> {
-        let path = BankForks::get_snapshot_path();
-        let paths = fs::read_dir(path.clone())?;
-        let mut names = paths
-            .filter_map(|entry| {
-                entry.ok().and_then(|e| {
-                    e.path()
-                        .file_name()
-                        .and_then(|n| n.to_str().map(|s| s.parse::<u64>().unwrap()))
-                })
-            })
-            .collect::<Vec<u64>>();
-
-        names.sort();
-        let mut bank_maps = vec![];
-        let mut bank_rc: Option<BankRc> = None;
-        let mut root: u64 = 0;
-        for bank_slot in names.iter().rev() {
-            let bank_path = format!("{}", bank_slot);
-            let bank_file_path = path.join(bank_path.clone());
-            info!("Load from {:?}", bank_file_path);
-            let file = File::open(bank_file_path)?;
-            let mut stream = BufReader::new(file);
-            let bank: Result<Bank, std::io::Error> = deserialize_from(&mut stream)
-                .map_err(|_| BankForks::get_io_error("deserialize bank error"));
-            let slot: Result<u64, std::io::Error> = deserialize_from(&mut stream)
-                .map_err(|_| BankForks::get_io_error("deserialize bank parent error"));
-            let parent_slot = if slot.is_ok() { slot.unwrap() } else { 0 };
-            if bank_rc.is_none() {
-                let rc: Result<BankRc, std::io::Error> = deserialize_from(&mut stream)
-                    .map_err(|_| BankForks::get_io_error("deserialize bank rc error"));
-                if rc.is_ok() {
-                    bank_rc = Some(rc.unwrap());
-                    let r: Result<u64, std::io::Error> = deserialize_from(&mut stream)
-                        .map_err(|_| BankForks::get_io_error("deserialize root error"));
-                    if r.is_ok() {
-                        root = r.unwrap();
-                    }
-                }
-            }
-            match bank {
-                Ok(v) => bank_maps.push((*bank_slot, parent_slot, v)),
-                Err(_) => warn!("Load snapshot failed for {}", bank_slot),
-            }
-        }
-        if bank_maps.is_empty() || bank_rc.is_none() {
-            return Err(Error::new(ErrorKind::Other, "no snapshots loaded"));
-        }
-
-        let (banks, slots, last_slot) = BankForks::setup_banks(&mut bank_maps, &bank_rc.unwrap());
-        let working_bank = banks[&last_slot].clone();
-        Ok(BankForks {
-            banks,
-            working_bank,
-            root,
-            slots,
-            use_snapshot: true,
-        })
+            .retain(|slot, _| descendants[&root].contains(slot))
     }
 }
 
@@ -306,10 +140,6 @@ mod tests {
     use crate::genesis_utils::create_genesis_block;
     use solana_sdk::hash::Hash;
     use solana_sdk::pubkey::Pubkey;
-    use solana_sdk::signature::{Keypair, KeypairUtil};
-    use solana_sdk::system_transaction;
-    use std::env;
-    use std::fs::remove_dir_all;
 
     #[test]
     fn test_bank_forks() {
@@ -334,8 +164,8 @@ mod tests {
         let bank = Bank::new_from_parent(&bank0, &Pubkey::default(), 2);
         bank_forks.insert(bank);
         let descendants = bank_forks.descendants();
-        let children: HashSet<u64> = [1u64, 2u64].to_vec().into_iter().collect();
-        assert_eq!(children, *descendants.get(&0).unwrap());
+        let children: Vec<u64> = descendants[&0].iter().cloned().collect();
+        assert_eq!(children, vec![1, 2]);
         assert!(descendants[&1].is_empty());
         assert!(descendants[&2].is_empty());
     }
@@ -379,89 +209,4 @@ mod tests {
         assert_eq!(bank_forks.active_banks(), vec![1]);
     }
 
-    struct TempPaths {
-        pub paths: String,
-    }
-
-    #[macro_export]
-    macro_rules! tmp_bank_accounts_name {
-        () => {
-            &format!("{}-{}", file!(), line!())
-        };
-    }
-
-    #[macro_export]
-    macro_rules! get_tmp_bank_accounts_path {
-        () => {
-            get_tmp_bank_accounts_path(tmp_bank_accounts_name!())
-        };
-    }
-
-    impl Drop for TempPaths {
-        fn drop(&mut self) {
-            let paths: Vec<String> = self.paths.split(',').map(|s| s.to_string()).collect();
-            paths.iter().for_each(|p| {
-                let _ignored = remove_dir_all(p);
-            });
-        }
-    }
-
-    fn get_paths_vec(paths: &str) -> Vec<String> {
-        paths.split(',').map(|s| s.to_string()).collect()
-    }
-
-    fn get_tmp_bank_accounts_path(paths: &str) -> TempPaths {
-        let vpaths = get_paths_vec(paths);
-        let out_dir = env::var("OUT_DIR").unwrap_or_else(|_| "target".to_string());
-        let vpaths: Vec<_> = vpaths
-            .iter()
-            .map(|path| format!("{}/{}", out_dir, path))
-            .collect();
-        TempPaths {
-            paths: vpaths.join(","),
-        }
-    }
-
-    fn save_and_load_snapshot(bank_forks: &BankForks) {
-        for (slot, _) in bank_forks.banks.iter() {
-            bank_forks.add_snapshot(*slot, 0).unwrap();
-        }
-
-        let new = BankForks::load_from_snapshot().unwrap();
-        for (slot, _) in bank_forks.banks.iter() {
-            let bank = bank_forks.banks.get(slot).unwrap().clone();
-            let new_bank = new.banks.get(slot).unwrap();
-            bank.compare_bank(&new_bank);
-        }
-        for (slot, _) in new.banks.iter() {
-            new.remove_snapshot(*slot);
-        }
-    }
-
-    #[test]
-    fn test_bank_forks_snapshot_n() {
-        solana_logger::setup();
-        let path = get_tmp_bank_accounts_path!();
-        let (genesis_block, mint_keypair) = create_genesis_block(10_000);
-        let bank0 = Bank::new_with_paths(&genesis_block, Some(path.paths.clone()));
-        bank0.freeze();
-        let mut bank_forks = BankForks::new(0, bank0);
-        bank_forks.set_snapshot_config(true);
-        for index in 0..10 {
-            let bank = Bank::new_from_parent(&bank_forks[index], &Pubkey::default(), index + 1);
-            let key1 = Keypair::new().pubkey();
-            let tx = system_transaction::create_user_account(
-                &mint_keypair,
-                &key1,
-                1,
-                genesis_block.hash(),
-                0,
-            );
-            assert_eq!(bank.process_transaction(&tx), Ok(()));
-            bank.freeze();
-            bank_forks.insert(bank);
-            save_and_load_snapshot(&bank_forks);
-        }
-        assert_eq!(bank_forks.working_bank().slot(), 10);
-    }
 }

core/src/blocktree.rs

@@ -11,7 +11,7 @@ use solana_kvstore as kvstore;
 
 use bincode::deserialize;
 
-use std::collections::HashMap;
+use hashbrown::HashMap;
 
 #[cfg(not(feature = "kvstore"))]
 use rocksdb;

core/src/cluster_info.rs

@@ -26,6 +26,7 @@ use crate::staking_utils;
 use crate::streamer::{BlobReceiver, BlobSender};
 use bincode::{deserialize, serialize};
 use core::cmp;
+use hashbrown::HashMap;
 use rand::{thread_rng, Rng};
 use rayon::prelude::*;
 use solana_metrics::counter::Counter;
@@ -40,7 +41,7 @@ use solana_sdk::signature::{Keypair, KeypairUtil, Signable, Signature};
 use solana_sdk::timing::{duration_as_ms, timestamp};
 use solana_sdk::transaction::Transaction;
 use std::cmp::min;
-use std::collections::{HashMap, HashSet};
+use std::collections::HashSet;
 use std::fmt;
 use std::io;
 use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket};

core/src/crds_gossip.rs

@@ -8,10 +8,10 @@ use crate::crds_gossip_error::CrdsGossipError;
 use crate::crds_gossip_pull::CrdsGossipPull;
 use crate::crds_gossip_push::{CrdsGossipPush, CRDS_GOSSIP_NUM_ACTIVE};
 use crate::crds_value::CrdsValue;
+use hashbrown::HashMap;
 use solana_runtime::bloom::Bloom;
 use solana_sdk::hash::Hash;
 use solana_sdk::pubkey::Pubkey;
-use std::collections::HashMap;
 
 ///The min size for bloom filters
 pub const CRDS_GOSSIP_BLOOM_SIZE: usize = 1000;

core/src/crds_gossip_pull.rs

@@ -16,13 +16,13 @@ use crate::crds_gossip_error::CrdsGossipError;
 use crate::crds_value::{CrdsValue, CrdsValueLabel};
 use crate::packet::BLOB_DATA_SIZE;
 use bincode::serialized_size;
+use hashbrown::HashMap;
 use rand;
 use rand::distributions::{Distribution, WeightedIndex};
 use solana_runtime::bloom::Bloom;
 use solana_sdk::hash::Hash;
 use solana_sdk::pubkey::Pubkey;
 use std::cmp;
-use std::collections::HashMap;
 use std::collections::VecDeque;
 
 pub const CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS: u64 = 15000;

core/src/crds_gossip_push.rs

@@ -15,6 +15,7 @@ use crate::crds_gossip_error::CrdsGossipError;
 use crate::crds_value::{CrdsValue, CrdsValueLabel};
 use crate::packet::BLOB_DATA_SIZE;
 use bincode::serialized_size;
+use hashbrown::HashMap;
 use indexmap::map::IndexMap;
 use rand;
 use rand::distributions::{Distribution, WeightedIndex};
@@ -24,7 +25,6 @@ use solana_sdk::hash::Hash;
 use solana_sdk::pubkey::Pubkey;
 use solana_sdk::timing::timestamp;
 use std::cmp;
-use std::collections::HashMap;
 
 pub const CRDS_GOSSIP_NUM_ACTIVE: usize = 30;
 pub const CRDS_GOSSIP_PUSH_FANOUT: usize = 6;