diff --git a/Cargo.lock b/Cargo.lock index 19be2f3611479c..2a8f77a9b3822f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -193,7 +193,7 @@ dependencies = [ "peeking_take_while 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "shlex 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "which 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -586,7 +586,7 @@ dependencies = [ "encode_unicode 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "termios 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -925,7 +925,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", "strsim 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1023,7 +1023,7 @@ dependencies = [ "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)", "humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1035,7 +1035,7 @@ dependencies = [ "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)", "humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1047,7 +1047,7 @@ dependencies = [ "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)", "humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1339,7 +1339,7 @@ dependencies = [ "bstr 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1640,7 +1640,7 @@ dependencies = [ "console 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "number_prefix 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1823,8 +1823,8 @@ dependencies = [ "itertools 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", "lalrpop-util 0.17.2 (registry+https://github.com/rust-lang/crates.io-index)", "petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", "sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2586,7 +2586,7 @@ dependencies = [ "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", "rusty-fork 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2657,7 +2657,7 @@ dependencies = [ "error-chain 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)", "idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2918,13 +2918,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.3.1" +version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", - "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2942,7 +2942,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "regex-syntax" -version = "0.6.12" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -3803,6 +3803,7 @@ dependencies = [ "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "rayon 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3857,7 +3858,7 @@ dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.9.24 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.42 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4032,6 +4033,7 @@ dependencies = [ "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "rayon 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "rocksdb 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", "serde_bytes 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4714,7 +4716,7 @@ dependencies = [ "lalrpop 0.17.2 (registry+https://github.com/rust-lang/crates.io-index)", "lalrpop-util 0.17.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "solana_libra_failure_ext 0.0.1-sol4 (registry+https://github.com/rust-lang/crates.io-index)", "solana_libra_types 0.0.1-sol4 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -5300,6 +5302,14 @@ dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "threshold_crypto" version = "0.3.2" @@ -5357,7 +5367,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -6374,10 +6384,10 @@ dependencies = [ "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" "checksum redox_users 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3fe5204c3a17e97dde73f285d49be585df59ed84b50a872baf416e73b62c3828" "checksum regex 0.1.80 (registry+https://github.com/rust-lang/crates.io-index)" = "4fd4ace6a8cf7860714a2c2280d6c1f7e6a413486c13298bbc86fd3da019402f" -"checksum regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dc220bd33bdce8f093101afe22a037b8eb0e5af33592e6a9caafff0d4cb81cbd" +"checksum regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "322cf97724bea3ee221b78fe25ac9c46114ebb51747ad5babd51a2fc6a8235a8" "checksum regex-automata 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "92b73c2a1770c255c240eaa4ee600df1704a38dc3feaa6e949e7fcd4f8dc09f9" "checksum regex-syntax 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "f9ec002c35e86791825ed294b50008eea9ddfc8def4420124fbc6b08db834957" -"checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716" +"checksum regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)" = "b28dfe3fe9badec5dbf0a79a9cccad2cfc2ab5484bdb3e44cbd1ae8b3ba2be06" "checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e" "checksum rental 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "01916ebd9fc2e81978a5dc9542a2fa47f5bb2ca3402e14c7cc42d6e3c5123e1f" "checksum rental-impl 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "82260d54cf2cbe9608df161f7e7c98e81fae702aa13af9e4d5d39dc2ffb25ab6" @@ -6493,6 +6503,7 @@ dependencies = [ "checksum thread-scoped 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bcbb6aa301e5d3b0b5ef639c9a9c7e2f1c944f177b460c04dc24c69b1fa2bd99" "checksum thread_local 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "8576dbbfcaef9641452d5cf0df9b0e7eeab7694956dd33bb61515fb8f18cfdd5" "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" +"checksum thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" "checksum threshold_crypto 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "95be1032c63011f20b01c5edb64930e2b51512782b43b458b1e3449613d70f87" "checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" "checksum tiny-bip39 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1cd1fb03fe8e07d17cd851a624a9fff74642a997b67fbd1ccd77533241640d92" diff --git a/core/Cargo.toml b/core/Cargo.toml index 81cabfd156cb34..8504494bf857fa 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -38,6 +38,7 @@ num-traits = "0.2" rand = "0.6.5" rand_chacha = "0.1.1" rayon = "1.2.0" +regex = "1.3.4" serde = "1.0.104" serde_derive = "1.0.103" serde_json = "1.0.44" diff --git a/core/src/rpc_service.rs b/core/src/rpc_service.rs index 5ddecebb341b7a..d0747ecb6c1b65 100644 --- a/core/src/rpc_service.rs +++ b/core/src/rpc_service.rs @@ -9,6 +9,7 @@ use jsonrpc_http_server::{ hyper, AccessControlAllowOrigin, CloseHandle, DomainsValidation, RequestMiddleware, RequestMiddlewareAction, ServerBuilder, }; +use regex::Regex; use solana_ledger::{bank_forks::BankForks, blockstore::Blockstore}; use solana_sdk::hash::Hash; use std::{ @@ -28,13 +29,17 @@ pub struct JsonRpcService { close_handle: Option, } -#[derive(Default)] struct RpcRequestMiddleware { ledger_path: PathBuf, + snapshot_archive_path_regex: Regex, } impl RpcRequestMiddleware { pub fn new(ledger_path: PathBuf) -> Self { - Self { ledger_path } + Self { + ledger_path, + snapshot_archive_path_regex: Regex::new(r"/snapshot-\d+-[[:alnum:]]+\.tar\.bz2$") + .unwrap(), + } } fn not_found() -> hyper::Response { @@ -51,9 +56,19 @@ impl RpcRequestMiddleware { .unwrap() } - fn get(&self, filename: &str) -> RequestMiddlewareAction { - info!("get {}", filename); - let filename = self.ledger_path.join(filename); + fn is_get_path(&self, path: &str) -> bool { + match path { + "/genesis.tar.bz2" => true, + _ => self.snapshot_archive_path_regex.is_match(path), + } + } + + fn get(&self, path: &str) -> RequestMiddlewareAction { + let filename = self.ledger_path.join( + path.split_at(1).1, // Drop leading '/' from path + ); + info!("get {} -> {:?}", path, filename); + RequestMiddlewareAction::Respond { should_validate_hosts: true, response: Box::new( @@ -73,13 +88,14 @@ impl RpcRequestMiddleware { impl RequestMiddleware for RpcRequestMiddleware { fn on_request(&self, request: hyper::Request) -> RequestMiddlewareAction { trace!("request uri: {}", request.uri()); - match request.uri().path() { - "/snapshot.tar.bz2" => self.get("snapshot.tar.bz2"), - "/genesis.tar.bz2" => self.get("genesis.tar.bz2"), - _ => RequestMiddlewareAction::Proceed { + + if self.is_get_path(request.uri().path()) { + self.get(request.uri().path()) + } else { + RequestMiddlewareAction::Proceed { should_continue_on_invalid_cors: false, request, - }, + } } } } @@ -234,4 +250,25 @@ mod tests { rpc_service.exit(); rpc_service.join().unwrap(); } + + #[test] + fn test_is_get_path() { + let rrm = RpcRequestMiddleware::new(PathBuf::from("/")); + + assert!(rrm.is_get_path("/genesis.tar.bz2")); + assert!(!rrm.is_get_path("genesis.tar.bz2")); + + assert!(!rrm.is_get_path("/snapshot.tar.bz2")); + + assert!( + rrm.is_get_path("/snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2") + ); + assert!(!rrm.is_get_path( + "/snapshot-notaslotnumber-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2" + )); + + assert!(!rrm.is_get_path("/")); + assert!(!rrm.is_get_path("..")); + assert!(!rrm.is_get_path("🎣")); + } } diff --git a/core/src/snapshot_packager_service.rs b/core/src/snapshot_packager_service.rs index 2b619cefaae343..12de848a037d00 100644 --- a/core/src/snapshot_packager_service.rs +++ b/core/src/snapshot_packager_service.rs @@ -154,8 +154,10 @@ mod tests { } // Create a packageable snapshot - let output_tar_path = - snapshot_utils::get_snapshot_archive_path(&snapshot_package_output_path); + let output_tar_path = snapshot_utils::get_snapshot_archive_path( + &snapshot_package_output_path, + &(42, Hash::default()), + ); let snapshot_package = SnapshotPackage::new( 5, vec![], diff --git a/core/tests/bank_forks.rs b/core/tests/bank_forks.rs index 8458b26f424053..e9862a5c2e7c1c 100644 --- a/core/tests/bank_forks.rs +++ b/core/tests/bank_forks.rs @@ -67,13 +67,19 @@ mod tests { } } - fn restore_from_snapshot(old_bank_forks: &BankForks, account_paths: Vec) { + fn restore_from_snapshot( + old_bank_forks: &BankForks, + old_last_slot: Slot, + account_paths: Vec, + ) { let (snapshot_path, snapshot_package_output_path) = old_bank_forks .snapshot_config .as_ref() .map(|c| (&c.snapshot_path, &c.snapshot_package_output_path)) .unwrap(); + let old_last_bank = old_bank_forks.get(old_last_slot).unwrap(); + let deserialized_bank = snapshot_utils::bank_from_archive( &account_paths, &old_bank_forks @@ -81,7 +87,10 @@ mod tests { .as_ref() .unwrap() .snapshot_path, - snapshot_utils::get_snapshot_archive_path(snapshot_package_output_path), + snapshot_utils::get_snapshot_archive_path( + snapshot_package_output_path, + &(old_last_bank.slot(), old_last_bank.get_accounts_hash()), + ), ) .unwrap(); @@ -139,18 +148,20 @@ mod tests { slot_snapshot_paths .last() .expect("no snapshots found in path"), - snapshot_utils::get_snapshot_archive_path( - &snapshot_config.snapshot_package_output_path, - ), &snapshot_config.snapshot_path, &last_bank.src.roots(), + &snapshot_config.snapshot_package_output_path, storages, ) .unwrap(); snapshot_utils::archive_snapshot_package(&snapshot_package).unwrap(); - restore_from_snapshot(bank_forks, vec![accounts_dir.path().to_path_buf()]); + restore_from_snapshot( + bank_forks, + last_slot, + vec![accounts_dir.path().to_path_buf()], + ); } #[test] @@ -221,9 +232,8 @@ mod tests { let saved_snapshots_dir = TempDir::new().unwrap(); let saved_accounts_dir = TempDir::new().unwrap(); let saved_slot = 4; - let saved_tar = snapshot_config - .snapshot_package_output_path - .join(saved_slot.to_string()); + let mut saved_archive_path = None; + for forks in 0..MAX_CACHE_ENTRIES + 2 { let bank = Bank::new_from_parent( &bank_forks[forks as u64], @@ -235,6 +245,7 @@ mod tests { let tx = system_transaction::transfer(&mint_keypair, &key1, 1, genesis_config.hash()); assert_eq!(bank.process_transaction(&tx), Ok(())); bank.squash(); + let accounts_hash = bank.update_accounts_hash(); bank_forks.insert(bank); let package_sender = { @@ -249,14 +260,7 @@ mod tests { }; bank_forks - .generate_snapshot( - slot, - &vec![], - &package_sender, - snapshot_config - .snapshot_package_output_path - .join(slot.to_string()), - ) + .generate_snapshot(slot, &vec![], &package_sender) .unwrap(); if slot == saved_slot as u64 { @@ -282,6 +286,11 @@ mod tests { &options, ) .unwrap(); + + saved_archive_path = Some(snapshot_utils::get_snapshot_archive_path( + &snapshot_config.snapshot_package_output_path, + &(slot, accounts_hash), + )); } } @@ -319,7 +328,7 @@ mod tests { .join() .expect("SnapshotPackagerService exited with error"); - // Check the tar we cached the state for earlier was generated correctly + // Check the archive we cached the state for earlier was generated correctly // before we compare, stick an empty status_cache in this dir so that the package comparision works // This is needed since the status_cache is added by the packager and is not collected from @@ -338,7 +347,7 @@ mod tests { .unwrap(); snapshot_utils::verify_snapshot_archive( - saved_tar, + saved_archive_path.unwrap(), saved_snapshots_dir.path(), saved_accounts_dir .path() diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 426a738905d5ac..79df5efa9bac1f 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -909,9 +909,9 @@ fn main() { snapshot_utils::package_snapshot( &bank, &slot_snapshot_paths, - snapshot_utils::get_snapshot_archive_path(output_directory), &temp_dir, &bank.src.roots(), + output_directory, storages, ) }) diff --git a/ledger/Cargo.toml b/ledger/Cargo.toml index d8681818c6b318..60516ce9aee32b 100644 --- a/ledger/Cargo.toml +++ b/ledger/Cargo.toml @@ -26,6 +26,7 @@ rand = "0.6.5" rand_chacha = "0.1.1" rayon = "1.2.0" reed-solomon-erasure = { package = "solana-reed-solomon-erasure", version = "4.0.1-3", features = ["simd-accel"] } +regex = "1.3.4" serde = "1.0.104" serde_bytes = "0.11.3" serde_derive = "1.0.103" diff --git a/ledger/src/bank_forks.rs b/ledger/src/bank_forks.rs index b60dd3eae5abef..dfa85cd865d9e5 100644 --- a/ledger/src/bank_forks.rs +++ b/ledger/src/bank_forks.rs @@ -10,7 +10,7 @@ use solana_sdk::{clock::Slot, timing}; use std::{ collections::{HashMap, HashSet}, ops::Index, - path::{Path, PathBuf}, + path::PathBuf, sync::Arc, time::Instant, }; @@ -192,7 +192,6 @@ impl BankForks { root, &root_bank.src.roots(), snapshot_package_sender.as_ref().unwrap(), - snapshot_utils::get_snapshot_archive_path(&config.snapshot_package_output_path), ); if r.is_err() { warn!("Error generating snapshot for bank: {}, err: {:?}", root, r); @@ -236,12 +235,11 @@ impl BankForks { } } - pub fn generate_snapshot>( + pub fn generate_snapshot( &self, root: Slot, slots_to_snapshot: &[Slot], snapshot_package_sender: &SnapshotPackageSender, - tar_output_file: P, ) -> Result<()> { let config = self.snapshot_config.as_ref().unwrap(); @@ -267,9 +265,9 @@ impl BankForks { let package = snapshot_utils::package_snapshot( &bank, latest_slot_snapshot_paths, - tar_output_file, &config.snapshot_path, slots_to_snapshot, + &config.snapshot_package_output_path, storages, )?; diff --git a/ledger/src/bank_forks_utils.rs b/ledger/src/bank_forks_utils.rs index bf7482652c7c27..a32227840cf31c 100644 --- a/ledger/src/bank_forks_utils.rs +++ b/ledger/src/bank_forks_utils.rs @@ -10,7 +10,7 @@ use crate::{ }; use log::*; use solana_sdk::{clock::Slot, genesis_config::GenesisConfig, hash::Hash}; -use std::{fs, path::PathBuf, result, sync::Arc}; +use std::{fs, path::PathBuf, process, result, sync::Arc}; pub type LoadResult = result::Result< ( @@ -52,37 +52,50 @@ pub fn load( fs::create_dir_all(&snapshot_config.snapshot_path) .expect("Couldn't create snapshot directory"); - let tar = snapshot_utils::get_snapshot_archive_path( + match snapshot_utils::get_highest_snapshot_archive_path( &snapshot_config.snapshot_package_output_path, - ); - if tar.exists() { - info!("Loading snapshot package: {:?}", tar); - // Fail hard here if snapshot fails to load, don't silently continue + ) { + Some((archive_filename, archive_snapshot_hash)) => { + info!("Loading snapshot package: {:?}", archive_filename); + // Fail hard here if snapshot fails to load, don't silently continue - if account_paths.is_empty() { - panic!("Account paths not present when booting from snapshot") - } + if account_paths.is_empty() { + error!("Account paths not present when booting from snapshot"); + process::exit(1); + } + + let deserialized_bank = snapshot_utils::bank_from_archive( + &account_paths, + &snapshot_config.snapshot_path, + &archive_filename, + ) + .expect("Load from snapshot failed"); - let deserialized_bank = snapshot_utils::bank_from_archive( - &account_paths, - &snapshot_config.snapshot_path, - &tar, - ) - .expect("Load from snapshot failed"); + let deserialized_snapshot_hash = ( + deserialized_bank.slot(), + deserialized_bank.get_accounts_hash(), + ); - let snapshot_hash = (deserialized_bank.slot(), deserialized_bank.hash()); - return to_loadresult( - blockstore_processor::process_blockstore_from_root( - genesis_config, - blockstore, - Arc::new(deserialized_bank), - &process_options, - &VerifyRecyclers::default(), - ), - Some(snapshot_hash), - ); - } else { - info!("Snapshot package does not exist: {:?}", tar); + if deserialized_snapshot_hash != archive_snapshot_hash { + error!( + "Snapshot has mismatch:\narchive: {:?}\ndeserialized: {:?}", + archive_snapshot_hash, deserialized_snapshot_hash + ); + process::exit(1); + } + + return to_loadresult( + blockstore_processor::process_blockstore_from_root( + genesis_config, + blockstore, + Arc::new(deserialized_bank), + &process_options, + &VerifyRecyclers::default(), + ), + Some(deserialized_snapshot_hash), + ); + } + None => info!("No snapshot package available"), } } else { info!("Snapshots disabled"); diff --git a/ledger/src/snapshot_utils.rs b/ledger/src/snapshot_utils.rs index bf6367cf8a091f..c62582dcd79c39 100644 --- a/ledger/src/snapshot_utils.rs +++ b/ledger/src/snapshot_utils.rs @@ -3,6 +3,7 @@ use bincode::serialize_into; use bzip2::bufread::BzDecoder; use fs_extra::dir::CopyOptions; use log::*; +use regex::Regex; use solana_measure::measure::Measure; use solana_runtime::{ accounts_db::{SnapshotStorage, SnapshotStorages}, @@ -11,7 +12,7 @@ use solana_runtime::{ MAX_SNAPSHOT_DATA_FILE_SIZE, }, }; -use solana_sdk::clock::Slot; +use solana_sdk::{clock::Slot, hash::Hash}; use std::{ cmp::Ordering, env, @@ -81,9 +82,9 @@ impl SlotSnapshotPaths { pub fn package_snapshot, Q: AsRef>( bank: &Bank, snapshot_files: &SlotSnapshotPaths, - snapshot_package_output_file: P, snapshot_path: Q, slots_to_snapshot: &[Slot], + snapshot_package_output_path: P, snapshot_storages: SnapshotStorages, ) -> Result { // Hard link all the snapshots we need for this package @@ -100,13 +101,18 @@ pub fn package_snapshot, Q: AsRef>( // any temporary state created for the SnapshotPackage (like the snapshot_hard_links_dir) snapshot_files.copy_snapshot_directory(snapshot_hard_links_dir.path())?; + let snapshot_package_output_file = get_snapshot_archive_path( + &snapshot_package_output_path, + &(bank.slot(), bank.get_accounts_hash()), + ); + let package = SnapshotPackage::new( bank.slot(), bank.src.slot_deltas(slots_to_snapshot), snapshot_hard_links_dir, snapshot_storages, - snapshot_package_output_file.as_ref().to_path_buf(), - bank.hash(), + snapshot_package_output_file, + bank.get_accounts_hash(), ); Ok(package) @@ -205,9 +211,17 @@ pub fn archive_snapshot_package(snapshot_package: &SnapshotPackage) -> Result<() let metadata = fs::metadata(&archive_path)?; fs::rename(&archive_path, &snapshot_package.tar_output_file)?; + // Keep around at most two snapshot archives + let archives = get_snapshot_archives(snapshot_package.tar_output_file.parent().unwrap()); + for old_archive in archives.into_iter().skip(2) { + fs::remove_file(old_archive.0) + .unwrap_or_else(|err| info!("Failed to remove old snapshot: {:}", err)); + } + timer.stop(); info!( - "Successfully created tarball. slot: {}, elapsed ms: {}, size={}", + "Successfully created {:?}. slot: {}, elapsed ms: {}, size={}", + snapshot_package.tar_output_file, snapshot_package.root, timer.as_ms(), metadata.len() @@ -415,32 +429,6 @@ pub fn remove_snapshot>(slot: Slot, snapshot_path: P) -> Result<( Ok(()) } -pub fn bank_slot_from_archive>(snapshot_tar: P) -> Result { - let tempdir = tempfile::TempDir::new()?; - untar_snapshot_in(&snapshot_tar, &tempdir)?; - let unpacked_snapshots_dir = tempdir.path().join(TAR_SNAPSHOTS_DIR); - let local_account_paths = vec![tempdir.path().join("account_dummy")]; - let unpacked_accounts_dir = tempdir.path().join(TAR_ACCOUNTS_DIR); - let snapshot_paths = get_snapshot_paths(&unpacked_snapshots_dir); - let last_root_paths = snapshot_paths - .last() - .ok_or_else(|| get_io_error("No snapshots found in snapshots directory"))?; - let bank = deserialize_snapshot_data_file( - &last_root_paths.snapshot_file_path, - MAX_SNAPSHOT_DATA_FILE_SIZE, - |stream| { - let bank: Bank = deserialize_from_snapshot(stream.by_ref())?; - bank.rc.accounts_from_stream( - stream.by_ref(), - &local_account_paths, - &unpacked_accounts_dir, - )?; - Ok(bank) - }, - )?; - Ok(bank.slot()) -} - pub fn bank_from_archive>( account_paths: &[PathBuf], snapshot_path: &PathBuf, @@ -503,8 +491,59 @@ fn is_snapshot_compression_disabled() -> bool { } } -pub fn get_snapshot_archive_path>(snapshot_output_dir: P) -> PathBuf { - snapshot_output_dir.as_ref().join("snapshot.tar.bz2") +pub fn get_snapshot_archive_path>( + snapshot_output_dir: P, + snapshot_hash: &(Slot, Hash), +) -> PathBuf { + snapshot_output_dir.as_ref().join(format!( + "snapshot-{}-{}.tar.bz2", + snapshot_hash.0, snapshot_hash.1 + )) +} + +fn snapshot_hash_of(archive_filename: &str) -> Option<(Slot, Hash)> { + let snapshot_filename_regex = Regex::new(r"snapshot-(\d+)-([[:alnum:]]+)\.tar\.bz2$").unwrap(); + + if let Some(captures) = snapshot_filename_regex.captures(archive_filename) { + let slot_str = captures.get(1).unwrap().as_str(); + let hash_str = captures.get(2).unwrap().as_str(); + + if let (Ok(slot), Ok(hash)) = (slot_str.parse::(), hash_str.parse::()) { + return Some((slot, hash)); + } + } + None +} + +fn get_snapshot_archives>(snapshot_output_dir: P) -> Vec<(PathBuf, (Slot, Hash))> { + let files = fs::read_dir(&snapshot_output_dir) + .unwrap_or_else(|err| panic!("Unable to read snapshot directory: {}", err)); + + let mut archives: Vec<_> = files + .filter_map(|entry| { + if let Ok(entry) = entry { + let path = entry.path(); + if path.is_file() { + if let Some(snapshot_hash) = + snapshot_hash_of(path.file_name().unwrap().to_str().unwrap()) + { + return Some((path, snapshot_hash)); + } + } + } + None + }) + .collect(); + + archives.sort_by(|a, b| (b.1).0.cmp(&(a.1).0)); // reverse sort by slot + archives +} + +pub fn get_highest_snapshot_archive_path>( + snapshot_output_dir: P, +) -> Option<(PathBuf, (Slot, Hash))> { + let archives = get_snapshot_archives(snapshot_output_dir); + archives.into_iter().next() } pub fn untar_snapshot_in, Q: AsRef>( @@ -607,7 +646,7 @@ fn get_io_error(error: &str) -> SnapshotError { } pub fn verify_snapshot_archive( - snapshot_tar: P, + snapshot_archive: P, snapshots_to_verify: Q, storages_to_verify: R, ) where @@ -617,7 +656,7 @@ pub fn verify_snapshot_archive( { let temp_dir = tempfile::TempDir::new().unwrap(); let unpack_dir = temp_dir.path(); - untar_snapshot_in(snapshot_tar, &unpack_dir).unwrap(); + untar_snapshot_in(snapshot_archive, &unpack_dir).unwrap(); // Check snapshots are the same let unpacked_snapshots = unpack_dir.join(&TAR_SNAPSHOTS_DIR); @@ -739,4 +778,13 @@ mod tests { ); assert_matches!(result, Err(SnapshotError::IO(ref message)) if message.to_string().starts_with("invalid snapshot data file")); } + + #[test] + fn test_snapshot_hash_of() { + assert_eq!( + snapshot_hash_of(&format!("snapshot-42-{}.tar.bz2", Hash::default())), + Some((42, Hash::default())) + ); + assert!(snapshot_hash_of("invalid").is_none()); + } } diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index 2ad5be6d2fe3ac..873894c6a2c996 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -18,10 +18,11 @@ use solana_local_cluster::{ }; use solana_sdk::{ client::SyncClient, - clock, + clock::{self, Slot}, commitment_config::CommitmentConfig, epoch_schedule::{EpochSchedule, MINIMUM_SLOTS_PER_EPOCH}, genesis_config::OperatingMode, + hash::Hash, poh_config::PohConfig, signature::{Keypair, Signer}, }; @@ -648,14 +649,16 @@ fn test_snapshot_restart_tower() { .as_ref() .unwrap() .snapshot_package_output_path; - let tar = snapshot_utils::get_snapshot_archive_path(&snapshot_package_output_path); - wait_for_next_snapshot(&cluster, &tar); - // Copy tar to validator's snapshot output directory - let validator_tar_path = snapshot_utils::get_snapshot_archive_path( + let (archive_filename, archive_snapshot_hash) = + wait_for_next_snapshot(&cluster, &snapshot_package_output_path); + + // Copy archive to validator's snapshot output directory + let validator_archive_path = snapshot_utils::get_snapshot_archive_path( &validator_snapshot_test_config.snapshot_output_path, + &archive_snapshot_hash, ); - fs::hard_link(tar, &validator_tar_path).unwrap(); + fs::hard_link(archive_filename, &validator_archive_path).unwrap(); // Restart validator from snapshot, the validator's tower state in this snapshot // will contain slots < the root bank of the snapshot. Validator should not panic. @@ -704,21 +707,23 @@ fn test_snapshots_blockstore_floor() { trace!("Waiting for snapshot tar to be generated with slot",); - let tar = snapshot_utils::get_snapshot_archive_path(&snapshot_package_output_path); - loop { - if tar.exists() { - trace!("snapshot tar exists"); - break; + let (archive_filename, (archive_slot, archive_hash)) = loop { + let archive = + snapshot_utils::get_highest_snapshot_archive_path(&snapshot_package_output_path); + if archive.is_some() { + trace!("snapshot exists"); + break archive.unwrap(); } sleep(Duration::from_millis(5000)); - } + }; - // Copy tar to validator's snapshot output directory - let validator_tar_path = snapshot_utils::get_snapshot_archive_path( + // Copy archive to validator's snapshot output directory + let validator_archive_path = snapshot_utils::get_snapshot_archive_path( &validator_snapshot_test_config.snapshot_output_path, + &(archive_slot, archive_hash), ); - fs::hard_link(tar, &validator_tar_path).unwrap(); - let slot_floor = snapshot_utils::bank_slot_from_archive(&validator_tar_path).unwrap(); + fs::hard_link(archive_filename, &validator_archive_path).unwrap(); + let slot_floor = archive_slot; // Start up a new node from a snapshot let validator_stake = 5; @@ -814,8 +819,7 @@ fn test_snapshots_restart_validity() { expected_balances.extend(new_balances); - let tar = snapshot_utils::get_snapshot_archive_path(&snapshot_package_output_path); - wait_for_next_snapshot(&cluster, &tar); + wait_for_next_snapshot(&cluster, &snapshot_package_output_path); // Create new account paths since validator exit is not guaranteed to cleanup RPC threads, // which may delete the old accounts on exit at any point @@ -952,7 +956,10 @@ fn test_no_voting() { } } -fn wait_for_next_snapshot>(cluster: &LocalCluster, tar: P) { +fn wait_for_next_snapshot( + cluster: &LocalCluster, + snapshot_package_output_path: &Path, +) -> (PathBuf, (Slot, Hash)) { // Get slot after which this was generated let client = cluster .get_validator_client(&cluster.entry_point_info.id) @@ -964,17 +971,18 @@ fn wait_for_next_snapshot>(cluster: &LocalCluster, tar: P) { // Wait for a snapshot for a bank >= last_slot to be made so we know that the snapshot // must include the transactions just pushed trace!( - "Waiting for snapshot tar to be generated with slot > {}", + "Waiting for snapshot archive to be generated with slot > {}", last_slot ); loop { - if tar.as_ref().exists() { - trace!("snapshot tar exists"); - let slot = snapshot_utils::bank_slot_from_archive(&tar).unwrap(); + if let Some((filename, (slot, hash))) = + snapshot_utils::get_highest_snapshot_archive_path(snapshot_package_output_path) + { + trace!("snapshot for slot {} exists", slot); if slot >= last_slot { - break; + return (filename, (slot, hash)); } - trace!("snapshot tar slot {} < last_slot {}", slot, last_slot); + trace!("snapshot slot {} < last_slot {}", slot, last_slot); } sleep(Duration::from_millis(5000)); } diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index adb2a07a3aa2ea..a29bec12a12d0b 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -496,6 +496,14 @@ impl Bank { *self.hash.read().unwrap() } + pub fn get_accounts_hash(&self) -> Hash { + self.hash() + } + + pub fn update_accounts_hash(&self) -> Hash { + self.hash() + } + pub fn is_frozen(&self) -> bool { *self.hash.read().unwrap() != Hash::default() } diff --git a/validator/src/main.rs b/validator/src/main.rs index 9d8d0ff6957baf..629eec22ac0e04 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -80,7 +80,7 @@ fn new_spinner_progress_bar() -> ProgressBar { progress_bar } -fn download_file(url: &str, destination_file: &Path, not_found_ok: bool) -> Result<(), String> { +fn download_file(url: &str, destination_file: &Path) -> Result<(), String> { if destination_file.is_file() { return Err(format!("{:?} already exists", destination_file)); } @@ -96,15 +96,9 @@ fn download_file(url: &str, destination_file: &Path, not_found_ok: bool) -> Resu let client = reqwest::blocking::Client::new(); let response = client.get(url).send().map_err(|err| err.to_string())?; - if response.status() == reqwest::StatusCode::NOT_FOUND && not_found_ok { - progress_bar.finish_and_clear(); - info!("Archive not found at {}", url); - return Ok(()); - } - let response = response .error_for_status() - .map_err(|err| format!("Unable to get: {:?}", err))?; + .map_err(|err| format!("Unable to download {}: {}", url, err))?; let download_size = { response .headers() @@ -438,7 +432,6 @@ fn download_genesis( download_file( &format!("http://{}/{}", rpc_addr, "genesis.tar.bz2"), &tmp_genesis_package, - false, )?; extract_archive(&tmp_genesis_package, &ledger_path)?; @@ -474,28 +467,22 @@ fn download_genesis( fn download_snapshot( rpc_addr: &SocketAddr, ledger_path: &Path, - snapshot_hash: Option<(Slot, Hash)>, + snapshot_hash: (Slot, Hash), ) -> Result<(), String> { - if snapshot_hash.is_none() { - return Ok(()); - } - - let snapshot_package = solana_ledger::snapshot_utils::get_snapshot_archive_path(ledger_path); + let snapshot_package = + solana_ledger::snapshot_utils::get_snapshot_archive_path(ledger_path, &snapshot_hash); if snapshot_package.exists() { - fs::remove_file(&snapshot_package) - .map_err(|err| format!("error removing {:?}: {}", snapshot_package, err))?; + Ok(()) + } else { + download_file( + &format!( + "http://{}/{}", + rpc_addr, + snapshot_package.file_name().unwrap().to_str().unwrap() + ), + &snapshot_package, + ) } - download_file( - &format!( - "http://{}/{}", - rpc_addr, - snapshot_package.file_name().unwrap().to_str().unwrap() - ), - &snapshot_package, - true, - )?; - - Ok(()) } // This function is duplicated in ledger-tool/src/main.rs... @@ -1122,7 +1109,13 @@ pub fn main() { } Ok(()) }) - .and_then(|_| download_snapshot(&rpc_contact_info.rpc, &ledger_path, snapshot_hash)) + .and_then(|_| { + if let Some(snapshot_hash) = snapshot_hash { + download_snapshot(&rpc_contact_info.rpc, &ledger_path, snapshot_hash) + } else { + Ok(()) + } + }) .and_then(|_| { if !validator_config.voting_disabled && !no_check_vote_account { check_vote_account(