diff --git a/Cargo.lock b/Cargo.lock index 32f5d56fd7..41b6bfe3c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -767,16 +767,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" -[[package]] -name = "byte-unit" -version = "4.0.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da78b32057b8fdfc352504708feeba7216dcd65a2c9ab02978cbd288d1279b6c" -dependencies = [ - "serde", - "utf8-width", -] - [[package]] name = "bytecount" version = "0.6.3" @@ -1049,7 +1039,6 @@ checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" dependencies = [ "atty", "bitflags", - "clap_derive 3.2.25", "clap_lex 0.2.4", "indexmap 1.9.3", "once_cell", @@ -1065,7 +1054,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c27cdf28c0f604ba3f512b0c9a409f8de8513e4816705deb0498b627e7c3a3fd" dependencies = [ "clap_builder", - "clap_derive 4.3.12", + "clap_derive", "once_cell", ] @@ -1090,19 +1079,6 @@ dependencies = [ "clap 4.3.21", ] -[[package]] -name = "clap_derive" -version = "3.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" -dependencies = [ - "heck 0.4.1", - "proc-macro-error", - "proc-macro2 1.0.63", - "quote 1.0.28", - "syn 1.0.109", -] - [[package]] name = "clap_derive" version = "4.3.12" @@ -4452,16 +4428,6 @@ dependencies = [ "rustc_version 0.2.3", ] -[[package]] -name = "parking_lot" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" -dependencies = [ - "lock_api 0.3.4", - "parking_lot_core 0.7.3", -] - [[package]] name = "parking_lot" version = "0.11.2" @@ -4498,20 +4464,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "parking_lot_core" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b93f386bb233083c799e6e642a9d73db98c24a5deeb95ffc85bf281255dffc98" -dependencies = [ - "cfg-if 0.1.10", - "cloudabi", - "libc", - "redox_syscall 0.1.57", - "smallvec 1.10.0", - "winapi 0.3.9", -] - [[package]] name = "parking_lot_core" version = "0.8.6" @@ -4653,19 +4605,6 @@ dependencies = [ "indexmap 1.9.3", ] -[[package]] -name = "pickledb" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9161694d67f6c5163519d42be942ae36bbdb55f439460144f105bc4f9f7d1d61" -dependencies = [ - "bincode", - "serde", - "serde_cbor", - "serde_json", - "serde_yaml", -] - [[package]] name = "pin-project" version = "1.1.0" @@ -5451,20 +5390,6 @@ dependencies = [ "num_cpus", ] -[[package]] -name = "rbpf-cli" -version = "1.10.41" -dependencies = [ - "clap 3.2.25", - "serde", - "serde_json", - "solana-bpf-loader-program", - "solana-logger 1.10.41", - "solana-program-runtime", - "solana-sdk 1.10.41", - "solana_rbpf", -] - [[package]] name = "rcgen" version = "0.9.3" @@ -6002,16 +5927,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_cbor" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" -dependencies = [ - "half", - "serde", -] - [[package]] name = "serde_derive" version = "1.0.164" @@ -6080,17 +5995,6 @@ dependencies = [ "yaml-rust", ] -[[package]] -name = "serial_test" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fef5f7c7434b2f2c598adc6f9494648a1e41274a75c0ba4056f680ae0c117fd6" -dependencies = [ - "lazy_static", - "parking_lot 0.10.2", - "serial_test_derive 0.4.0", -] - [[package]] name = "serial_test" version = "0.6.0" @@ -6116,17 +6020,6 @@ dependencies = [ "serial_test_derive 2.0.0", ] -[[package]] -name = "serial_test_derive" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d08338d8024b227c62bd68a12c7c9883f5c66780abaef15c550dc56f46ee6515" -dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.28", - "syn 1.0.109", -] - [[package]] name = "serial_test_derive" version = "0.6.0" @@ -6445,47 +6338,6 @@ dependencies = [ "zstd", ] -[[package]] -name = "solana-accounts-bench" -version = "1.10.41" -dependencies = [ - "clap 2.34.0", - "log 0.4.19", - "rayon", - "solana-logger 1.10.41", - "solana-measure", - "solana-runtime", - "solana-sdk 1.10.41", - "solana-version", -] - -[[package]] -name = "solana-accounts-cluster-bench" -version = "1.10.41" -dependencies = [ - "clap 2.34.0", - "log 0.4.19", - "rand 0.7.3", - "rayon", - "solana-account-decoder", - "solana-clap-utils", - "solana-client", - "solana-core", - "solana-faucet", - "solana-gossip", - "solana-local-cluster", - "solana-logger 1.10.41", - "solana-measure", - "solana-net-utils", - "solana-runtime", - "solana-sdk 1.10.41", - "solana-streamer", - "solana-test-validator", - "solana-transaction-status", - "solana-version", - "spl-token", -] - [[package]] name = "solana-address-lookup-table-program" version = "1.10.41" @@ -6516,29 +6368,6 @@ dependencies = [ "solana-sdk 1.10.41", ] -[[package]] -name = "solana-banking-bench" -version = "1.10.41" -dependencies = [ - "clap 3.2.25", - "crossbeam-channel", - "log 0.4.19", - "rand 0.7.3", - "rayon", - "solana-client", - "solana-core", - "solana-gossip", - "solana-ledger", - "solana-logger 1.10.41", - "solana-measure", - "solana-perf", - "solana-poh", - "solana-runtime", - "solana-sdk 1.10.41", - "solana-streamer", - "solana-version", -] - [[package]] name = "solana-banks-client" version = "1.10.41" @@ -6583,81 +6412,6 @@ dependencies = [ "tokio-stream", ] -[[package]] -name = "solana-bench-streamer" -version = "1.10.41" -dependencies = [ - "clap 2.34.0", - "crossbeam-channel", - "solana-net-utils", - "solana-streamer", - "solana-version", -] - -[[package]] -name = "solana-bench-tps" -version = "1.10.41" -dependencies = [ - "clap 2.34.0", - "crossbeam-channel", - "log 0.4.19", - "rayon", - "serde_json", - "serde_yaml", - "serial_test 0.6.0", - "solana-clap-utils", - "solana-cli-config", - "solana-client", - "solana-core", - "solana-faucet", - "solana-genesis", - "solana-gossip", - "solana-local-cluster", - "solana-logger 1.10.41", - "solana-measure", - "solana-metrics", - "solana-net-utils", - "solana-rpc", - "solana-runtime", - "solana-sdk 1.10.41", - "solana-streamer", - "solana-test-validator", - "solana-version", - "thiserror", -] - -[[package]] -name = "solana-bench-tps-evm" -version = "1.0.0" -dependencies = [ - "bincode", - "clap 2.34.0", - "log 0.4.19", - "rand_isaac", - "rayon", - "serde_json", - "serde_yaml", - "serial_test 0.4.0", - "serial_test_derive 0.4.0", - "simple_logger", - "solana-clap-utils", - "solana-client", - "solana-core", - "solana-evm-loader-program", - "solana-faucet", - "solana-genesis", - "solana-gossip", - "solana-local-cluster", - "solana-logger 1.10.41", - "solana-measure", - "solana-metrics", - "solana-net-utils", - "solana-runtime", - "solana-sdk 1.10.41", - "solana-streamer", - "solana-version", -] - [[package]] name = "solana-bloom" version = "1.10.41" @@ -7025,27 +6779,6 @@ dependencies = [ "velas-relying-party-program", ] -[[package]] -name = "solana-dos" -version = "1.10.41" -dependencies = [ - "bincode", - "clap 3.2.25", - "log 0.4.19", - "rand 0.7.3", - "serde", - "solana-client", - "solana-core", - "solana-gossip", - "solana-local-cluster", - "solana-logger 1.10.41", - "solana-net-utils", - "solana-perf", - "solana-sdk 1.10.41", - "solana-streamer", - "solana-version", -] - [[package]] name = "solana-download-utils" version = "1.10.41" @@ -7472,18 +7205,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "solana-log-analyzer" -version = "1.10.41" -dependencies = [ - "byte-unit", - "clap 2.34.0", - "serde", - "serde_json", - "solana-logger 1.10.41", - "solana-version", -] - [[package]] name = "solana-logger" version = "1.10.41" @@ -7513,19 +7234,6 @@ dependencies = [ "solana-sdk 1.10.41", ] -[[package]] -name = "solana-merkle-root-bench" -version = "1.10.41" -dependencies = [ - "clap 2.34.0", - "log 0.4.19", - "solana-logger 1.10.41", - "solana-measure", - "solana-runtime", - "solana-sdk 1.10.41", - "solana-version", -] - [[package]] name = "solana-merkle-tree" version = "1.10.41" @@ -7551,17 +7259,6 @@ dependencies = [ "solana-sdk 1.10.41", ] -[[package]] -name = "solana-net-shaper" -version = "1.10.41" -dependencies = [ - "clap 2.34.0", - "rand 0.7.3", - "serde", - "serde_json", - "solana-logger 1.10.41", -] - [[package]] name = "solana-net-utils" version = "1.10.41" @@ -7582,15 +7279,6 @@ dependencies = [ "url 2.4.0", ] -[[package]] -name = "solana-notifier" -version = "1.10.41" -dependencies = [ - "log 0.4.19", - "reqwest", - "serde_json", -] - [[package]] name = "solana-perf" version = "1.10.41" @@ -7640,22 +7328,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "solana-poh-bench" -version = "1.10.41" -dependencies = [ - "clap 2.34.0", - "log 0.4.19", - "rand 0.7.3", - "rayon", - "solana-entry", - "solana-logger 1.10.41", - "solana-measure", - "solana-perf", - "solana-sdk 1.10.41", - "solana-version", -] - [[package]] name = "solana-program" version = "1.10.41" @@ -8204,20 +7876,6 @@ dependencies = [ "solana-sdk 1.10.41", ] -[[package]] -name = "solana-stake-accounts" -version = "1.10.41" -dependencies = [ - "clap 2.34.0", - "solana-clap-utils", - "solana-cli-config", - "solana-client", - "solana-remote-wallet", - "solana-runtime", - "solana-sdk 1.10.41", - "solana-stake-program", -] - [[package]] name = "solana-stake-program" version = "1.10.41" @@ -8292,17 +7950,6 @@ dependencies = [ "tonic-build 0.7.2", ] -[[package]] -name = "solana-store-tool" -version = "1.10.41" -dependencies = [ - "clap 2.34.0", - "log 0.4.19", - "solana-logger 1.10.41", - "solana-runtime", - "solana-version", -] - [[package]] name = "solana-streamer" version = "1.10.41" @@ -8370,63 +8017,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "solana-tokens" -version = "1.10.41" -dependencies = [ - "bincode", - "chrono", - "clap 2.34.0", - "console 0.15.7", - "csv", - "ctrlc", - "indexmap 1.9.3", - "indicatif 0.16.2", - "pickledb", - "serde", - "solana-account-decoder", - "solana-clap-utils", - "solana-cli-config", - "solana-client", - "solana-logger 1.10.41", - "solana-remote-wallet", - "solana-sdk 1.10.41", - "solana-streamer", - "solana-test-validator", - "solana-transaction-status", - "solana-version", - "spl-associated-token-account", - "spl-token", - "tempfile", - "thiserror", -] - -[[package]] -name = "solana-transaction-dos" -version = "1.10.41" -dependencies = [ - "bincode", - "clap 2.34.0", - "log 0.4.19", - "rand 0.7.3", - "rayon", - "solana-clap-utils", - "solana-cli", - "solana-client", - "solana-core", - "solana-faucet", - "solana-gossip", - "solana-local-cluster", - "solana-logger 1.10.41", - "solana-measure", - "solana-net-utils", - "solana-runtime", - "solana-sdk 1.10.41", - "solana-streamer", - "solana-transaction-status", - "solana-version", -] - [[package]] name = "solana-transaction-status" version = "1.10.41" @@ -8456,14 +8046,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "solana-upload-perf" -version = "1.10.41" -dependencies = [ - "serde_json", - "solana-metrics", -] - [[package]] name = "solana-version" version = "0.7.0" @@ -8498,24 +8080,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "solana-watchtower" -version = "1.10.41" -dependencies = [ - "clap 2.34.0", - "humantime", - "log 0.4.19", - "solana-clap-utils", - "solana-cli-config", - "solana-cli-output", - "solana-client", - "solana-logger 1.10.41", - "solana-metrics", - "solana-notifier", - "solana-sdk 1.10.41", - "solana-version", -] - [[package]] name = "solana-zk-token-proof-program" version = "1.10.41" @@ -10017,12 +9581,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" -[[package]] -name = "utf8-width" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5190c9442dcdaf0ddd50f37420417d219ae5261bbf5db120d0f9bab996c9cba1" - [[package]] name = "utf8parse" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 823dd92dba..d3e1da373c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,15 +1,9 @@ [workspace] members = [ "account-decoder", - "accounts-bench", - "accounts-cluster-bench", - "banking-bench", "banks-client", "banks-interface", "banks-server", - "bench-streamer", - "bench-tps", - "bench-tps-evm", "bloom", "bucket_map", "clap-utils", @@ -19,7 +13,6 @@ members = [ "client", "client-test", "core", - "dos", "download-utils", "entry", "faucet", @@ -34,15 +27,11 @@ members = [ "ledger", "ledger-tool", "local-cluster", - "log-analyzer", "logger", "measure", - "merkle-root-bench", "merkle-tree", "metrics", - "net-shaper", "net-utils", - "notifier", "perf", "evm-utils/evm-block-recovery", "evm-utils/evm-bridge", @@ -50,7 +39,6 @@ members = [ "evm-utils/evm-state", "evm-utils/evm-rpc", "poh", - "poh-bench", "program-test", "programs/address-lookup-table", "programs/address-lookup-table-tests", @@ -63,31 +51,24 @@ members = [ "programs/vote", "programs/zk-token-proof", "rayon-threadlimit", - "rbpf-cli", "remote-wallet", "replica-lib", "replica-node", "rpc", "rpc-test", "runtime", - "runtime/store-tool", "sdk", "sdk/cargo-build-bpf", "sdk/cargo-test-bpf", "send-transaction-service", - "stake-accounts", "storage-bigtable", "storage-proto", "streamer", "sys-tuner", "test-validator", - "tokens", - "transaction-dos", "transaction-status", - "upload-perf", "validator", "version", - "watchtower", "zk-token-sdk", ] diff --git a/accounts-bench/Cargo.toml b/accounts-bench/Cargo.toml deleted file mode 100644 index bf1de19910..0000000000 --- a/accounts-bench/Cargo.toml +++ /dev/null @@ -1,22 +0,0 @@ -[package] -authors = ["Solana Maintainers "] -edition = "2021" -name = "solana-accounts-bench" -version = "1.10.41" -repository = "https://github.com/solana-labs/solana" -license = "Apache-2.0" -homepage = "https://solana.com/" -publish = false - -[dependencies] -clap = "2.33.1" -log = "0.4.14" -rayon = "1.5.1" -solana-logger = { path = "../logger", version = "=1.10.41" } -solana-measure = { path = "../measure", version = "=1.10.41" } -solana-runtime = { path = "../runtime", version = "=1.10.41" } -solana-sdk = { path = "../sdk", version = "=1.10.41" } -solana-version = { path = "../version", version = "=0.7.0" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/accounts-bench/src/main.rs b/accounts-bench/src/main.rs deleted file mode 100644 index 6e0a92a2fa..0000000000 --- a/accounts-bench/src/main.rs +++ /dev/null @@ -1,153 +0,0 @@ -#![allow(clippy::integer_arithmetic)] -#[macro_use] -extern crate log; -use { - clap::{crate_description, crate_name, value_t, App, Arg}, - rayon::prelude::*, - solana_measure::measure::Measure, - solana_runtime::{ - accounts::{create_test_accounts, update_accounts_bench, Accounts}, - accounts_db::AccountShrinkThreshold, - accounts_index::AccountSecondaryIndexes, - ancestors::Ancestors, - }, - solana_sdk::{genesis_config::ClusterType, pubkey::Pubkey}, - std::{env, fs, path::PathBuf}, -}; - -fn main() { - solana_logger::setup(); - - let matches = App::new(crate_name!()) - .about(crate_description!()) - .version(solana_version::version!()) - .arg( - Arg::with_name("num_slots") - .long("num_slots") - .takes_value(true) - .value_name("SLOTS") - .help("Number of slots to store to."), - ) - .arg( - Arg::with_name("num_accounts") - .long("num_accounts") - .takes_value(true) - .value_name("NUM_ACCOUNTS") - .help("Total number of accounts"), - ) - .arg( - Arg::with_name("iterations") - .long("iterations") - .takes_value(true) - .value_name("ITERATIONS") - .help("Number of bench iterations"), - ) - .arg( - Arg::with_name("clean") - .long("clean") - .takes_value(false) - .help("Run clean"), - ) - .get_matches(); - - let num_slots = value_t!(matches, "num_slots", usize).unwrap_or(4); - let num_accounts = value_t!(matches, "num_accounts", usize).unwrap_or(10_000); - let iterations = value_t!(matches, "iterations", usize).unwrap_or(20); - let clean = matches.is_present("clean"); - println!("clean: {:?}", clean); - - let path = PathBuf::from(env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_owned())) - .join("accounts-bench"); - println!("cleaning file system: {:?}", path); - if fs::remove_dir_all(path.clone()).is_err() { - println!("Warning: Couldn't remove {:?}", path); - } - let accounts = Accounts::new_with_config_for_benches( - vec![path], - &ClusterType::Testnet, - AccountSecondaryIndexes::default(), - false, - AccountShrinkThreshold::default(), - ); - println!("Creating {} accounts", num_accounts); - let mut create_time = Measure::start("create accounts"); - let pubkeys: Vec<_> = (0..num_slots) - .into_par_iter() - .map(|slot| { - let mut pubkeys: Vec = vec![]; - create_test_accounts( - &accounts, - &mut pubkeys, - num_accounts / num_slots, - slot as u64, - ); - pubkeys - }) - .collect(); - let pubkeys: Vec<_> = pubkeys.into_iter().flatten().collect(); - create_time.stop(); - println!( - "created {} accounts in {} slots {}", - (num_accounts / num_slots) * num_slots, - num_slots, - create_time - ); - let mut ancestors = Vec::with_capacity(num_slots); - ancestors.push(0); - for i in 1..num_slots { - ancestors.push(i as u64); - accounts.add_root(i as u64); - } - let ancestors = Ancestors::from(ancestors); - let mut elapsed = vec![0; iterations]; - let mut elapsed_store = vec![0; iterations]; - for x in 0..iterations { - if clean { - let mut time = Measure::start("clean"); - accounts.accounts_db.clean_accounts(None, false, None); - time.stop(); - println!("{}", time); - for slot in 0..num_slots { - update_accounts_bench(&accounts, &pubkeys, ((x + 1) * num_slots + slot) as u64); - accounts.add_root((x * num_slots + slot) as u64); - } - } else { - let mut pubkeys: Vec = vec![]; - let mut time = Measure::start("hash"); - let results = accounts.accounts_db.update_accounts_hash(0, &ancestors); - time.stop(); - let mut time_store = Measure::start("hash using store"); - let results_store = accounts.accounts_db.update_accounts_hash_with_index_option( - false, - false, - solana_sdk::clock::Slot::default(), - &ancestors, - None, - false, - None, - false, - ); - time_store.stop(); - if results != results_store { - error!("results different: \n{:?}\n{:?}", results, results_store); - } - println!( - "hash,{},{},{},{}%", - results.0, - time, - time_store, - (time_store.as_us() as f64 / time.as_us() as f64 * 100.0f64) as u32 - ); - create_test_accounts(&accounts, &mut pubkeys, 1, 0); - elapsed[x] = time.as_us(); - elapsed_store[x] = time_store.as_us(); - } - } - - for x in elapsed { - info!("update_accounts_hash(us),{}", x); - } - for x in elapsed_store { - info!("calculate_accounts_hash_without_index(us),{}", x); - } -} diff --git a/accounts-cluster-bench/.gitignore b/accounts-cluster-bench/.gitignore deleted file mode 100644 index b645148aa9..0000000000 --- a/accounts-cluster-bench/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/farf/ diff --git a/accounts-cluster-bench/Cargo.toml b/accounts-cluster-bench/Cargo.toml deleted file mode 100644 index 14f7c53694..0000000000 --- a/accounts-cluster-bench/Cargo.toml +++ /dev/null @@ -1,38 +0,0 @@ -[package] -authors = ["Solana Maintainers "] -edition = "2021" -name = "solana-accounts-cluster-bench" -version = "1.10.41" -repository = "https://github.com/solana-labs/solana" -license = "Apache-2.0" -homepage = "https://solana.com/" -publish = false - -[dependencies] -clap = "2.33.1" -log = "0.4.14" -rand = "0.7.0" -rayon = "1.5.1" -solana-account-decoder = { path = "../account-decoder", version = "=1.10.41" } -solana-clap-utils = { path = "../clap-utils", version = "=1.10.41" } -solana-client = { path = "../client", version = "=1.10.41" } -solana-faucet = { path = "../faucet", version = "=1.10.41" } -solana-gossip = { path = "../gossip", version = "=1.10.41" } -solana-logger = { path = "../logger", version = "=1.10.41" } -solana-measure = { path = "../measure", version = "=1.10.41" } -solana-net-utils = { path = "../net-utils", version = "=1.10.41" } -solana-runtime = { path = "../runtime", version = "=1.10.41" } -solana-sdk = { path = "../sdk", version = "=1.10.41" } -solana-streamer = { path = "../streamer", version = "=1.10.41" } -solana-test-validator = { path = "../test-validator", version = "=1.10.41" } -solana-transaction-status = { path = "../transaction-status", version = "=1.10.41" } -solana-version = { path = "../version", version = "=0.7.0" } -spl-token = { version = "=3.5.0", features = ["no-entrypoint"] } - -[dev-dependencies] -solana-core = { path = "../core", version = "=1.10.41" } -solana-local-cluster = { path = "../local-cluster", version = "=1.10.41" } -solana-test-validator = { path = "../test-validator", version = "=1.10.41" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/accounts-cluster-bench/src/main.rs b/accounts-cluster-bench/src/main.rs deleted file mode 100644 index 4eef2630af..0000000000 --- a/accounts-cluster-bench/src/main.rs +++ /dev/null @@ -1,812 +0,0 @@ -#![allow(clippy::integer_arithmetic)] -use { - clap::{crate_description, crate_name, value_t, values_t_or_exit, App, Arg}, - log::*, - rand::{thread_rng, Rng}, - rayon::prelude::*, - solana_account_decoder::parse_token::spl_token_pubkey, - solana_clap_utils::input_parsers::pubkey_of, - solana_client::{rpc_client::RpcClient, transaction_executor::TransactionExecutor}, - solana_faucet::faucet::{request_airdrop_transaction, FAUCET_PORT}, - solana_gossip::gossip_service::discover, - solana_runtime::inline_spl_token, - solana_sdk::{ - commitment_config::CommitmentConfig, - instruction::{AccountMeta, Instruction}, - message::Message, - pubkey::Pubkey, - rpc_port::DEFAULT_RPC_PORT, - signature::{read_keypair_file, Keypair, Signer}, - system_instruction, system_program, - transaction::Transaction, - }, - solana_streamer::socket::SocketAddrSpace, - solana_transaction_status::parse_token::spl_token_instruction, - std::{ - cmp::min, - net::SocketAddr, - process::exit, - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, - }, - thread::sleep, - time::{Duration, Instant}, - }, -}; - -pub fn airdrop_lamports( - client: &RpcClient, - faucet_addr: &SocketAddr, - id: &Keypair, - desired_balance: u64, -) -> bool { - let starting_balance = client.get_balance(&id.pubkey()).unwrap_or(0); - info!("starting balance {}", starting_balance); - - if starting_balance < desired_balance { - let airdrop_amount = desired_balance - starting_balance; - info!( - "Airdropping {:?} lamports from {} for {}", - airdrop_amount, - faucet_addr, - id.pubkey(), - ); - - let blockhash = client.get_latest_blockhash().unwrap(); - match request_airdrop_transaction(faucet_addr, &id.pubkey(), airdrop_amount, blockhash) { - Ok(transaction) => { - let mut tries = 0; - loop { - tries += 1; - let result = client.send_and_confirm_transaction_with_spinner(&transaction); - - if result.is_ok() { - break; - } - if tries >= 5 { - panic!( - "Error requesting airdrop: to addr: {:?} amount: {} {:?}", - faucet_addr, airdrop_amount, result - ) - } - } - } - Err(err) => { - panic!( - "Error requesting airdrop: {:?} to addr: {:?} amount: {}", - err, faucet_addr, airdrop_amount - ); - } - }; - - let current_balance = client.get_balance(&id.pubkey()).unwrap_or_else(|e| { - panic!("airdrop error {}", e); - }); - info!("current balance {}...", current_balance); - - if current_balance - starting_balance != airdrop_amount { - info!( - "Airdrop failed? {} {} {} {}", - id.pubkey(), - current_balance, - starting_balance, - airdrop_amount, - ); - } - } - true -} - -struct SeedTracker { - max_created: Arc, - max_closed: Arc, -} - -fn make_create_message( - keypair: &Keypair, - base_keypair: &Keypair, - max_created_seed: Arc, - num_instructions: usize, - balance: u64, - maybe_space: Option, - mint: Option, -) -> Message { - let space = maybe_space.unwrap_or_else(|| thread_rng().gen_range(0, 1000)); - - let instructions: Vec<_> = (0..num_instructions) - .into_iter() - .flat_map(|_| { - let program_id = if mint.is_some() { - inline_spl_token::id() - } else { - system_program::id() - }; - let seed = max_created_seed.fetch_add(1, Ordering::Relaxed).to_string(); - let to_pubkey = - Pubkey::create_with_seed(&base_keypair.pubkey(), &seed, &program_id).unwrap(); - let mut instructions = vec![system_instruction::create_account_with_seed( - &keypair.pubkey(), - &to_pubkey, - &base_keypair.pubkey(), - &seed, - balance, - space, - &program_id, - )]; - if let Some(mint_address) = mint { - instructions.push(spl_token_instruction( - spl_token::instruction::initialize_account( - &spl_token::id(), - &spl_token_pubkey(&to_pubkey), - &spl_token_pubkey(&mint_address), - &spl_token_pubkey(&base_keypair.pubkey()), - ) - .unwrap(), - )); - } - - instructions - }) - .collect(); - - Message::new(&instructions, Some(&keypair.pubkey())) -} - -fn make_close_message( - keypair: &Keypair, - base_keypair: &Keypair, - max_created: Arc, - max_closed: Arc, - num_instructions: usize, - balance: u64, - spl_token: bool, -) -> Message { - let instructions: Vec<_> = (0..num_instructions) - .into_iter() - .filter_map(|_| { - let program_id = if spl_token { - inline_spl_token::id() - } else { - system_program::id() - }; - let max_created_seed = max_created.load(Ordering::Relaxed); - let max_closed_seed = max_closed.load(Ordering::Relaxed); - if max_closed_seed >= max_created_seed { - return None; - } - let seed = max_closed.fetch_add(1, Ordering::Relaxed).to_string(); - let address = - Pubkey::create_with_seed(&base_keypair.pubkey(), &seed, &program_id).unwrap(); - if spl_token { - Some(spl_token_instruction( - spl_token::instruction::close_account( - &spl_token::id(), - &spl_token_pubkey(&address), - &spl_token_pubkey(&keypair.pubkey()), - &spl_token_pubkey(&base_keypair.pubkey()), - &[], - ) - .unwrap(), - )) - } else { - Some(system_instruction::transfer_with_seed( - &address, - &base_keypair.pubkey(), - seed, - &program_id, - &keypair.pubkey(), - balance, - )) - } - }) - .collect(); - - Message::new(&instructions, Some(&keypair.pubkey())) -} - -#[allow(clippy::too_many_arguments)] -fn run_accounts_bench( - entrypoint_addr: SocketAddr, - faucet_addr: SocketAddr, - payer_keypairs: &[&Keypair], - iterations: usize, - maybe_space: Option, - batch_size: usize, - close_nth_batch: u64, - maybe_lamports: Option, - num_instructions: usize, - mint: Option, - reclaim_accounts: bool, -) { - assert!(num_instructions > 0); - let client = - RpcClient::new_socket_with_commitment(entrypoint_addr, CommitmentConfig::confirmed()); - - info!("Targeting {}", entrypoint_addr); - - let mut latest_blockhash = Instant::now(); - let mut last_log = Instant::now(); - let mut count = 0; - let mut blockhash = client.get_latest_blockhash().expect("blockhash"); - let mut tx_sent_count = 0; - let mut total_accounts_created = 0; - let mut total_accounts_closed = 0; - let mut balances: Vec<_> = payer_keypairs - .iter() - .map(|keypair| client.get_balance(&keypair.pubkey()).unwrap_or(0)) - .collect(); - let mut last_balance = Instant::now(); - - let default_max_lamports = 1000; - let min_balance = maybe_lamports.unwrap_or_else(|| { - let space = maybe_space.unwrap_or(default_max_lamports); - client - .get_minimum_balance_for_rent_exemption(space as usize) - .expect("min balance") - }); - - let base_keypair = Keypair::new(); - let seed_tracker = SeedTracker { - max_created: Arc::new(AtomicU64::default()), - max_closed: Arc::new(AtomicU64::default()), - }; - - info!("Starting balance(s): {:?}", balances); - - let executor = TransactionExecutor::new(entrypoint_addr); - - // Create and close messages both require 2 signatures, fake a 2 signature message to calculate fees - let mut message = Message::new( - &[ - Instruction::new_with_bytes( - Pubkey::new_unique(), - &[], - vec![AccountMeta::new(Pubkey::new_unique(), true)], - ), - Instruction::new_with_bytes( - Pubkey::new_unique(), - &[], - vec![AccountMeta::new(Pubkey::new_unique(), true)], - ), - ], - None, - ); - - loop { - if latest_blockhash.elapsed().as_millis() > 10_000 { - blockhash = client.get_latest_blockhash().expect("blockhash"); - latest_blockhash = Instant::now(); - } - - message.recent_blockhash = blockhash; - let fee = client - .get_fee_for_message(&message) - .expect("get_fee_for_message"); - let lamports = min_balance + fee; - - for (i, balance) in balances.iter_mut().enumerate() { - if *balance < lamports || last_balance.elapsed().as_millis() > 2000 { - if let Ok(b) = client.get_balance(&payer_keypairs[i].pubkey()) { - *balance = b; - } - last_balance = Instant::now(); - if *balance < lamports * 2 { - info!( - "Balance {} is less than needed: {}, doing aidrop...", - balance, lamports - ); - if !airdrop_lamports( - &client, - &faucet_addr, - payer_keypairs[i], - lamports * 100_000, - ) { - warn!("failed airdrop, exiting"); - return; - } - } - } - } - - // Create accounts - let sigs_len = executor.num_outstanding(); - if sigs_len < batch_size { - let num_to_create = batch_size - sigs_len; - if num_to_create >= payer_keypairs.len() { - info!("creating {} new", num_to_create); - let chunk_size = num_to_create / payer_keypairs.len(); - if chunk_size > 0 { - for (i, keypair) in payer_keypairs.iter().enumerate() { - let txs: Vec<_> = (0..chunk_size) - .into_par_iter() - .map(|_| { - let message = make_create_message( - keypair, - &base_keypair, - seed_tracker.max_created.clone(), - num_instructions, - min_balance, - maybe_space, - mint, - ); - let signers: Vec<&Keypair> = vec![keypair, &base_keypair]; - Transaction::new(&signers, message, blockhash) - }) - .collect(); - balances[i] = balances[i].saturating_sub(lamports * txs.len() as u64); - info!("txs: {}", txs.len()); - let new_ids = executor.push_transactions(txs); - info!("ids: {}", new_ids.len()); - tx_sent_count += new_ids.len(); - total_accounts_created += num_instructions * new_ids.len(); - } - } - } - - if close_nth_batch > 0 { - let num_batches_to_close = - total_accounts_created as u64 / (close_nth_batch * batch_size as u64); - let expected_closed = num_batches_to_close * batch_size as u64; - let max_closed_seed = seed_tracker.max_closed.load(Ordering::Relaxed); - // Close every account we've created with seed between max_closed_seed..expected_closed - if max_closed_seed < expected_closed { - let txs: Vec<_> = (0..expected_closed - max_closed_seed) - .into_par_iter() - .map(|_| { - let message = make_close_message( - payer_keypairs[0], - &base_keypair, - seed_tracker.max_created.clone(), - seed_tracker.max_closed.clone(), - 1, - min_balance, - mint.is_some(), - ); - let signers: Vec<&Keypair> = vec![payer_keypairs[0], &base_keypair]; - Transaction::new(&signers, message, blockhash) - }) - .collect(); - balances[0] = balances[0].saturating_sub(fee * txs.len() as u64); - info!("close txs: {}", txs.len()); - let new_ids = executor.push_transactions(txs); - info!("close ids: {}", new_ids.len()); - tx_sent_count += new_ids.len(); - total_accounts_closed += new_ids.len() as u64; - } - } - } else { - let _ = executor.drain_cleared(); - } - - count += 1; - if last_log.elapsed().as_millis() > 3000 || count >= iterations { - info!( - "total_accounts_created: {} total_accounts_closed: {} tx_sent_count: {} loop_count: {} balance(s): {:?}", - total_accounts_created, total_accounts_closed, tx_sent_count, count, balances - ); - last_log = Instant::now(); - } - if iterations != 0 && count >= iterations { - break; - } - if executor.num_outstanding() >= batch_size { - sleep(Duration::from_millis(500)); - } - } - executor.close(); - - if reclaim_accounts { - let executor = TransactionExecutor::new(entrypoint_addr); - loop { - let max_closed_seed = seed_tracker.max_closed.load(Ordering::Relaxed); - let max_created_seed = seed_tracker.max_created.load(Ordering::Relaxed); - - if latest_blockhash.elapsed().as_millis() > 10_000 { - blockhash = client.get_latest_blockhash().expect("blockhash"); - latest_blockhash = Instant::now(); - } - message.recent_blockhash = blockhash; - let fee = client - .get_fee_for_message(&message) - .expect("get_fee_for_message"); - - let sigs_len = executor.num_outstanding(); - if sigs_len < batch_size && max_closed_seed < max_created_seed { - let num_to_close = min( - batch_size - sigs_len, - (max_created_seed - max_closed_seed) as usize, - ); - if num_to_close >= payer_keypairs.len() { - info!("closing {} accounts", num_to_close); - let chunk_size = num_to_close / payer_keypairs.len(); - info!("{:?} chunk_size", chunk_size); - if chunk_size > 0 { - for (i, keypair) in payer_keypairs.iter().enumerate() { - let txs: Vec<_> = (0..chunk_size) - .into_par_iter() - .filter_map(|_| { - let message = make_close_message( - keypair, - &base_keypair, - seed_tracker.max_created.clone(), - seed_tracker.max_closed.clone(), - num_instructions, - min_balance, - mint.is_some(), - ); - if message.instructions.is_empty() { - return None; - } - let signers: Vec<&Keypair> = vec![keypair, &base_keypair]; - Some(Transaction::new(&signers, message, blockhash)) - }) - .collect(); - balances[i] = balances[i].saturating_sub(fee * txs.len() as u64); - info!("close txs: {}", txs.len()); - let new_ids = executor.push_transactions(txs); - info!("close ids: {}", new_ids.len()); - tx_sent_count += new_ids.len(); - total_accounts_closed += (num_instructions * new_ids.len()) as u64; - } - } - } - } else { - let _ = executor.drain_cleared(); - } - count += 1; - if last_log.elapsed().as_millis() > 3000 || max_closed_seed >= max_created_seed { - info!( - "total_accounts_closed: {} tx_sent_count: {} loop_count: {} balance(s): {:?}", - total_accounts_closed, tx_sent_count, count, balances - ); - last_log = Instant::now(); - } - - if max_closed_seed >= max_created_seed { - break; - } - if executor.num_outstanding() >= batch_size { - sleep(Duration::from_millis(500)); - } - } - executor.close(); - } -} - -fn main() { - solana_logger::setup_with_default("solana=info"); - let matches = App::new(crate_name!()) - .about(crate_description!()) - .version(solana_version::version!()) - .arg( - Arg::with_name("entrypoint") - .long("entrypoint") - .takes_value(true) - .value_name("HOST:PORT") - .help("RPC entrypoint address. Usually :8899"), - ) - .arg( - Arg::with_name("faucet_addr") - .long("faucet") - .takes_value(true) - .value_name("HOST:PORT") - .help("Faucet entrypoint address. Usually :9900"), - ) - .arg( - Arg::with_name("space") - .long("space") - .takes_value(true) - .value_name("BYTES") - .help("Size of accounts to create"), - ) - .arg( - Arg::with_name("lamports") - .long("lamports") - .takes_value(true) - .value_name("LAMPORTS") - .help("How many lamports to fund each account"), - ) - .arg( - Arg::with_name("identity") - .long("identity") - .takes_value(true) - .multiple(true) - .value_name("FILE") - .help("keypair file"), - ) - .arg( - Arg::with_name("batch_size") - .long("batch-size") - .takes_value(true) - .value_name("BYTES") - .help("Number of transactions to send per batch"), - ) - .arg( - Arg::with_name("close_nth_batch") - .long("close-frequency") - .takes_value(true) - .value_name("BYTES") - .help( - "Every `n` batches, create a batch of close transactions for - the earliest remaining batch of accounts created. - Note: Should be > 1 to avoid situations where the close \ - transactions will be submitted before the corresponding \ - create transactions have been confirmed", - ), - ) - .arg( - Arg::with_name("num_instructions") - .long("num-instructions") - .takes_value(true) - .value_name("NUM") - .help("Number of accounts to create on each transaction"), - ) - .arg( - Arg::with_name("iterations") - .long("iterations") - .takes_value(true) - .value_name("NUM") - .help("Number of iterations to make. 0 = unlimited iterations."), - ) - .arg( - Arg::with_name("check_gossip") - .long("check-gossip") - .help("Just use entrypoint address directly"), - ) - .arg( - Arg::with_name("mint") - .long("mint") - .takes_value(true) - .help("Mint address to initialize account"), - ) - .arg( - Arg::with_name("reclaim_accounts") - .long("reclaim-accounts") - .takes_value(false) - .help("Reclaim accounts after session ends; incompatible with --iterations 0"), - ) - .get_matches(); - - let skip_gossip = !matches.is_present("check_gossip"); - - let port = if skip_gossip { DEFAULT_RPC_PORT } else { 8001 }; - let mut entrypoint_addr = SocketAddr::from(([127, 0, 0, 1], port)); - if let Some(addr) = matches.value_of("entrypoint") { - entrypoint_addr = solana_net_utils::parse_host_port(addr).unwrap_or_else(|e| { - eprintln!("failed to parse entrypoint address: {}", e); - exit(1) - }); - } - let mut faucet_addr = SocketAddr::from(([127, 0, 0, 1], FAUCET_PORT)); - if let Some(addr) = matches.value_of("faucet_addr") { - faucet_addr = solana_net_utils::parse_host_port(addr).unwrap_or_else(|e| { - eprintln!("failed to parse entrypoint address: {}", e); - exit(1) - }); - } - - let space = value_t!(matches, "space", u64).ok(); - let lamports = value_t!(matches, "lamports", u64).ok(); - let batch_size = value_t!(matches, "batch_size", usize).unwrap_or(4); - let close_nth_batch = value_t!(matches, "close_nth_batch", u64).unwrap_or(0); - let iterations = value_t!(matches, "iterations", usize).unwrap_or(10); - let num_instructions = value_t!(matches, "num_instructions", usize).unwrap_or(1); - if num_instructions == 0 || num_instructions > 500 { - eprintln!("bad num_instructions: {}", num_instructions); - exit(1); - } - - let mint = pubkey_of(&matches, "mint"); - - let payer_keypairs: Vec<_> = values_t_or_exit!(matches, "identity", String) - .iter() - .map(|keypair_string| { - read_keypair_file(keypair_string) - .unwrap_or_else(|_| panic!("bad keypair {:?}", keypair_string)) - }) - .collect(); - let mut payer_keypair_refs: Vec<&Keypair> = vec![]; - for keypair in payer_keypairs.iter() { - payer_keypair_refs.push(keypair); - } - - let rpc_addr = if !skip_gossip { - info!("Finding cluster entry: {:?}", entrypoint_addr); - let (gossip_nodes, _validators) = discover( - None, // keypair - Some(&entrypoint_addr), - None, // num_nodes - Duration::from_secs(60), // timeout - None, // find_node_by_pubkey - Some(&entrypoint_addr), // find_node_by_gossip_addr - None, // my_gossip_addr - 0, // my_shred_version - SocketAddrSpace::Unspecified, - ) - .unwrap_or_else(|err| { - eprintln!("Failed to discover {} node: {:?}", entrypoint_addr, err); - exit(1); - }); - - info!("done found {} nodes", gossip_nodes.len()); - gossip_nodes[0].rpc - } else { - info!("Using {:?} as the RPC address", entrypoint_addr); - entrypoint_addr - }; - - run_accounts_bench( - rpc_addr, - faucet_addr, - &payer_keypair_refs, - iterations, - space, - batch_size, - close_nth_batch, - lamports, - num_instructions, - mint, - matches.is_present("reclaim_accounts"), - ); -} - -#[cfg(test)] -pub mod test { - use { - super::*, - solana_core::validator::ValidatorConfig, - solana_faucet::faucet::run_local_faucet, - solana_local_cluster::{ - local_cluster::{ClusterConfig, LocalCluster}, - validator_configs::make_identical_validator_configs, - }, - solana_measure::measure::Measure, - solana_sdk::{native_token::sol_to_lamports, poh_config::PohConfig}, - solana_test_validator::TestValidator, - spl_token::{ - solana_program::program_pack::Pack, - state::{Account, Mint}, - }, - }; - - #[test] - fn test_accounts_cluster_bench() { - solana_logger::setup(); - let validator_config = ValidatorConfig::default_for_test(); - let num_nodes = 1; - let mut config = ClusterConfig { - cluster_lamports: 10_000_000, - poh_config: PohConfig::new_sleep(Duration::from_millis(50)), - node_stakes: vec![100; num_nodes], - validator_configs: make_identical_validator_configs(&validator_config, num_nodes), - ..ClusterConfig::default() - }; - - let faucet_addr = SocketAddr::from(([127, 0, 0, 1], 9900)); - let cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified); - let iterations = 10; - let maybe_space = None; - let batch_size = 100; - let close_nth_batch = 100; - let maybe_lamports = None; - let num_instructions = 2; - let mut start = Measure::start("total accounts run"); - run_accounts_bench( - cluster.entry_point_info.rpc, - faucet_addr, - &[&cluster.funding_keypair], - iterations, - maybe_space, - batch_size, - close_nth_batch, - maybe_lamports, - num_instructions, - None, - false, - ); - start.stop(); - info!("{}", start); - } - - #[test] - fn test_create_then_reclaim_spl_token_accounts() { - solana_logger::setup(); - let mint_keypair = Keypair::new(); - let mint_pubkey = mint_keypair.pubkey(); - let faucet_addr = run_local_faucet(mint_keypair, None); - let test_validator = TestValidator::with_custom_fees( - mint_pubkey, - 1, - Some(faucet_addr), - SocketAddrSpace::Unspecified, - ); - let rpc_client = - RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed()); - - // Created funder - let funder = Keypair::new(); - let latest_blockhash = rpc_client.get_latest_blockhash().unwrap(); - let signature = rpc_client - .request_airdrop_with_blockhash( - &funder.pubkey(), - sol_to_lamports(1.0), - &latest_blockhash, - ) - .unwrap(); - rpc_client - .confirm_transaction_with_spinner( - &signature, - &latest_blockhash, - CommitmentConfig::confirmed(), - ) - .unwrap(); - - // Create Mint - let spl_mint_keypair = Keypair::new(); - let spl_mint_len = Mint::get_packed_len(); - let spl_mint_rent = rpc_client - .get_minimum_balance_for_rent_exemption(spl_mint_len) - .unwrap(); - let transaction = Transaction::new_signed_with_payer( - &[ - system_instruction::create_account( - &funder.pubkey(), - &spl_mint_keypair.pubkey(), - spl_mint_rent, - spl_mint_len as u64, - &inline_spl_token::id(), - ), - spl_token_instruction( - spl_token::instruction::initialize_mint( - &spl_token::id(), - &spl_token_pubkey(&spl_mint_keypair.pubkey()), - &spl_token_pubkey(&spl_mint_keypair.pubkey()), - None, - 2, - ) - .unwrap(), - ), - ], - Some(&funder.pubkey()), - &[&funder, &spl_mint_keypair], - latest_blockhash, - ); - let _sig = rpc_client - .send_and_confirm_transaction_with_spinner(&transaction) - .unwrap(); - - let account_len = Account::get_packed_len(); - let minimum_balance = rpc_client - .get_minimum_balance_for_rent_exemption(account_len) - .unwrap(); - - let iterations = 5; - let batch_size = 100; - let close_nth_batch = 0; - let num_instructions = 4; - let mut start = Measure::start("total accounts run"); - let keypair0 = Keypair::new(); - let keypair1 = Keypair::new(); - let keypair2 = Keypair::new(); - run_accounts_bench( - test_validator - .rpc_url() - .replace("http://", "") - .parse() - .unwrap(), - faucet_addr, - &[&keypair0, &keypair1, &keypair2], - iterations, - Some(account_len as u64), - batch_size, - close_nth_batch, - Some(minimum_balance), - num_instructions, - Some(spl_mint_keypair.pubkey()), - true, - ); - start.stop(); - info!("{}", start); - } -} diff --git a/banking-bench/.gitignore b/banking-bench/.gitignore deleted file mode 100644 index 5404b132db..0000000000 --- a/banking-bench/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/target/ -/farf/ diff --git a/banking-bench/Cargo.toml b/banking-bench/Cargo.toml deleted file mode 100644 index 93945c3133..0000000000 --- a/banking-bench/Cargo.toml +++ /dev/null @@ -1,31 +0,0 @@ -[package] -authors = ["Solana Maintainers "] -edition = "2021" -name = "solana-banking-bench" -version = "1.10.41" -repository = "https://github.com/solana-labs/solana" -license = "Apache-2.0" -homepage = "https://solana.com/" -publish = false - -[dependencies] -clap = {version = "3.1.5", features = ["derive", "cargo"]} -crossbeam-channel = "0.5" -log = "0.4.14" -rand = "0.7.0" -rayon = "1.5.1" -solana-client = { path = "../client", version = "=1.10.41" } -solana-core = { path = "../core", version = "=1.10.41" } -solana-gossip = { path = "../gossip", version = "=1.10.41" } -solana-ledger = { path = "../ledger", version = "=1.10.41" } -solana-logger = { path = "../logger", version = "=1.10.41" } -solana-measure = { path = "../measure", version = "=1.10.41" } -solana-perf = { path = "../perf", version = "=1.10.41" } -solana-poh = { path = "../poh", version = "=1.10.41" } -solana-runtime = { path = "../runtime", version = "=1.10.41" } -solana-sdk = { path = "../sdk", version = "=1.10.41" } -solana-streamer = { path = "../streamer", version = "=1.10.41" } -solana-version = { path = "../version", version = "=0.7.0" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/banking-bench/src/main.rs b/banking-bench/src/main.rs deleted file mode 100644 index e0c877fbb8..0000000000 --- a/banking-bench/src/main.rs +++ /dev/null @@ -1,513 +0,0 @@ -#![allow(clippy::integer_arithmetic)] -use { - clap::{crate_description, crate_name, Arg, ArgEnum, Command}, - crossbeam_channel::{unbounded, Receiver}, - log::*, - rand::{thread_rng, Rng}, - rayon::prelude::*, - solana_client::connection_cache::{ConnectionCache, DEFAULT_TPU_CONNECTION_POOL_SIZE}, - solana_core::banking_stage::BankingStage, - solana_gossip::cluster_info::{ClusterInfo, Node}, - solana_ledger::{ - blockstore::Blockstore, - genesis_utils::{create_genesis_config, GenesisConfigInfo}, - get_tmp_ledger_path, - leader_schedule_cache::LeaderScheduleCache, - }, - solana_measure::measure::Measure, - solana_perf::packet::{to_packet_batches, PacketBatch}, - solana_poh::poh_recorder::{create_test_recorder, PohRecorder, WorkingBankEntry}, - solana_runtime::{ - accounts_background_service::AbsRequestSender, bank::Bank, bank_forks::BankForks, - cost_model::CostModel, - }, - solana_sdk::{ - hash::Hash, - signature::{Keypair, Signature}, - system_transaction, - timing::{duration_as_us, timestamp}, - transaction::Transaction, - }, - solana_streamer::socket::SocketAddrSpace, - std::{ - sync::{atomic::Ordering, Arc, Mutex, RwLock}, - thread::sleep, - time::{Duration, Instant}, - }, -}; - -fn check_txs( - receiver: &Arc>, - ref_tx_count: usize, - poh_recorder: &Arc>, -) -> bool { - let mut total = 0; - let now = Instant::now(); - let mut no_bank = false; - loop { - if let Ok((_bank, (entry, _tick_height))) = receiver.recv_timeout(Duration::from_millis(10)) - { - total += entry.transactions.len(); - } - if total >= ref_tx_count { - break; - } - if now.elapsed().as_secs() > 60 { - break; - } - if poh_recorder.lock().unwrap().bank().is_none() { - no_bank = true; - break; - } - } - if !no_bank { - assert!(total >= ref_tx_count); - } - no_bank -} - -#[derive(ArgEnum, Clone, Copy, PartialEq)] -enum WriteLockContention { - /// No transactions lock the same accounts. - None, - /// Transactions don't lock the same account, unless they belong to the same batch. - SameBatchOnly, - /// All transactions write lock the same account. - Full, -} - -impl WriteLockContention { - fn possible_values<'a>() -> impl Iterator> { - Self::value_variants() - .iter() - .filter_map(|v| v.to_possible_value()) - } -} - -impl std::str::FromStr for WriteLockContention { - type Err = String; - fn from_str(input: &str) -> Result { - ArgEnum::from_str(input, false) - } -} - -fn make_accounts_txs( - total_num_transactions: usize, - packets_per_batch: usize, - hash: Hash, - contention: WriteLockContention, -) -> Vec { - use solana_sdk::pubkey; - let to_pubkey = pubkey::new_rand(); - let chunk_pubkeys: Vec = (0..total_num_transactions / packets_per_batch) - .map(|_| pubkey::new_rand()) - .collect(); - let payer_key = Keypair::new(); - let dummy = system_transaction::transfer(&payer_key, &to_pubkey, 1, hash); - (0..total_num_transactions) - .into_par_iter() - .map(|i| { - let mut new = dummy.clone(); - let sig: Vec = (0..64).map(|_| thread_rng().gen::()).collect(); - new.message.account_keys[0] = pubkey::new_rand(); - new.message.account_keys[1] = match contention { - WriteLockContention::None => pubkey::new_rand(), - WriteLockContention::SameBatchOnly => chunk_pubkeys[i / packets_per_batch], - WriteLockContention::Full => to_pubkey, - }; - new.signatures = vec![Signature::new(&sig[0..64])]; - new - }) - .collect() -} - -struct PacketsPerIteration { - packet_batches: Vec, - transactions: Vec, - packets_per_batch: usize, -} - -impl PacketsPerIteration { - fn new( - packets_per_batch: usize, - batches_per_iteration: usize, - genesis_hash: Hash, - write_lock_contention: WriteLockContention, - ) -> Self { - let total_num_transactions = packets_per_batch * batches_per_iteration; - let transactions = make_accounts_txs( - total_num_transactions, - packets_per_batch, - genesis_hash, - write_lock_contention, - ); - - let packet_batches: Vec = to_packet_batches(&transactions, packets_per_batch); - assert_eq!(packet_batches.len(), batches_per_iteration); - Self { - packet_batches, - transactions, - packets_per_batch, - } - } - - fn refresh_blockhash(&mut self, new_blockhash: Hash) { - for tx in self.transactions.iter_mut() { - tx.message.recent_blockhash = new_blockhash; - let sig: Vec = (0..64).map(|_| thread_rng().gen::()).collect(); - tx.signatures[0] = Signature::new(&sig[0..64]); - } - self.packet_batches = to_packet_batches(&self.transactions, self.packets_per_batch); - } -} - -#[allow(clippy::cognitive_complexity)] -fn main() { - solana_logger::setup(); - - let matches = Command::new(crate_name!()) - .about(crate_description!()) - .version(solana_version::version!()) - .arg( - Arg::new("iterations") - .long("iterations") - .takes_value(true) - .help("Number of test iterations"), - ) - .arg( - Arg::new("num_chunks") - .long("num-chunks") - .takes_value(true) - .value_name("SIZE") - .help("Number of transaction chunks."), - ) - .arg( - Arg::new("packets_per_batch") - .long("packets-per-batch") - .takes_value(true) - .value_name("SIZE") - .help("Packets per batch"), - ) - .arg( - Arg::new("skip_sanity") - .long("skip-sanity") - .takes_value(false) - .help("Skip transaction sanity execution"), - ) - .arg( - Arg::new("write_lock_contention") - .long("write-lock-contention") - .takes_value(true) - .possible_values(WriteLockContention::possible_values()) - .help("Accounts that test transactions write lock"), - ) - .arg( - Arg::new("batches_per_iteration") - .long("batches-per-iteration") - .takes_value(true) - .help("Number of batches to send in each iteration"), - ) - .arg( - Arg::new("num_banking_threads") - .long("num-banking-threads") - .takes_value(true) - .help("Number of threads to use in the banking stage"), - ) - .arg( - Arg::new("tpu_use_quic") - .long("tpu-use-quic") - .takes_value(false) - .help("Forward messages to TPU using QUIC"), - ) - .get_matches(); - - let num_banking_threads = matches - .value_of_t::("num_banking_threads") - .unwrap_or_else(|_| BankingStage::num_threads()); - // a multiple of packet chunk duplicates to avoid races - let num_chunks = matches.value_of_t::("num_chunks").unwrap_or(16); - let packets_per_batch = matches - .value_of_t::("packets_per_batch") - .unwrap_or(192); - let iterations = matches.value_of_t::("iterations").unwrap_or(1000); - let batches_per_iteration = matches - .value_of_t::("batches_per_iteration") - .unwrap_or(BankingStage::num_threads() as usize); - let write_lock_contention = matches - .value_of_t::("write_lock_contention") - .unwrap_or(WriteLockContention::None); - - let mint_total = 1_000_000_000_000; - let GenesisConfigInfo { - genesis_config, - mint_keypair, - .. - } = create_genesis_config(mint_total); - - let (verified_sender, verified_receiver) = unbounded(); - let (vote_sender, vote_receiver) = unbounded(); - let (tpu_vote_sender, tpu_vote_receiver) = unbounded(); - let (replay_vote_sender, _replay_vote_receiver) = unbounded(); - let bank0 = Bank::new_for_benches(&genesis_config); - let bank_forks = Arc::new(RwLock::new(BankForks::new(bank0))); - let mut bank = bank_forks.read().unwrap().working_bank(); - - // set cost tracker limits to MAX so it will not filter out TXs - bank.write_cost_tracker() - .unwrap() - .set_limits(std::u64::MAX, std::u64::MAX, std::u64::MAX); - - let mut all_packets: Vec = std::iter::from_fn(|| { - Some(PacketsPerIteration::new( - packets_per_batch, - batches_per_iteration, - genesis_config.hash(), - write_lock_contention, - )) - }) - .take(num_chunks) - .collect(); - - // fund all the accounts - let total_num_transactions: u64 = all_packets - .iter() - .map(|packets_for_single_iteration| packets_for_single_iteration.transactions.len() as u64) - .sum(); - info!( - "threads: {} txs: {}", - num_banking_threads, total_num_transactions - ); - - all_packets.iter().for_each(|packets_for_single_iteration| { - packets_for_single_iteration - .transactions - .iter() - .for_each(|tx| { - let mut fund = system_transaction::transfer( - &mint_keypair, - &tx.message.account_keys[0], - mint_total / total_num_transactions, - genesis_config.hash(), - ); - // Ignore any pesky duplicate signature errors in the case we are using single-payer - let sig: Vec = (0..64).map(|_| thread_rng().gen::()).collect(); - fund.signatures = vec![Signature::new(&sig[0..64])]; - bank.process_transaction(&fund).unwrap(); - }); - }); - - let skip_sanity = matches.is_present("skip_sanity"); - if !skip_sanity { - all_packets.iter().for_each(|packets_for_single_iteration| { - //sanity check, make sure all the transactions can execute sequentially - packets_for_single_iteration - .transactions - .iter() - .for_each(|tx| { - let res = bank.process_transaction(tx); - assert!(res.is_ok(), "sanity test transactions error: {:?}", res); - }); - }); - bank.clear_signatures(); - - if write_lock_contention == WriteLockContention::None { - all_packets.iter().for_each(|packets_for_single_iteration| { - //sanity check, make sure all the transactions can execute in parallel - let res = - bank.process_transactions(packets_for_single_iteration.transactions.iter()); - for r in res { - assert!(r.is_ok(), "sanity parallel execution error: {:?}", r); - } - bank.clear_signatures(); - }); - } - } - - let ledger_path = get_tmp_ledger_path!(); - { - let blockstore = Arc::new( - Blockstore::open(&ledger_path).expect("Expected to be able to open database ledger"), - ); - let leader_schedule_cache = Arc::new(LeaderScheduleCache::new_from_bank(&bank)); - let (exit, poh_recorder, poh_service, signal_receiver) = create_test_recorder( - &bank, - &blockstore, - None, - Some(leader_schedule_cache.clone()), - ); - let cluster_info = ClusterInfo::new( - Node::new_localhost().info, - Arc::new(Keypair::new()), - SocketAddrSpace::Unspecified, - ); - let cluster_info = Arc::new(cluster_info); - let tpu_use_quic = matches.is_present("tpu_use_quic"); - let connection_cache = match tpu_use_quic { - true => ConnectionCache::new(DEFAULT_TPU_CONNECTION_POOL_SIZE), - false => ConnectionCache::with_udp(DEFAULT_TPU_CONNECTION_POOL_SIZE), - }; - let banking_stage = BankingStage::new_num_threads( - &cluster_info, - &poh_recorder, - verified_receiver, - tpu_vote_receiver, - vote_receiver, - num_banking_threads, - None, - replay_vote_sender, - Arc::new(RwLock::new(CostModel::default())), - Arc::new(connection_cache), - bank_forks.clone(), - ); - poh_recorder.lock().unwrap().set_bank(&bank); - - // This is so that the signal_receiver does not go out of scope after the closure. - // If it is dropped before poh_service, then poh_service will error when - // calling send() on the channel. - let signal_receiver = Arc::new(signal_receiver); - let mut total_us = 0; - let mut tx_total_us = 0; - let base_tx_count = bank.transaction_count(); - let mut txs_processed = 0; - let mut root = 1; - let collector = solana_sdk::pubkey::new_rand(); - let mut total_sent = 0; - for current_iteration_index in 0..iterations { - trace!("RUNNING ITERATION {}", current_iteration_index); - let now = Instant::now(); - let mut sent = 0; - - let packets_for_this_iteration = &all_packets[current_iteration_index % num_chunks]; - for (packet_batch_index, packet_batch) in - packets_for_this_iteration.packet_batches.iter().enumerate() - { - sent += packet_batch.len(); - trace!( - "Sending PacketBatch index {}, {}", - packet_batch_index, - timestamp(), - ); - verified_sender - .send((vec![packet_batch.clone()], None)) - .unwrap(); - } - - for tx in &packets_for_this_iteration.transactions { - loop { - if bank.get_signature_status(&tx.signatures[0]).is_some() { - break; - } - if poh_recorder.lock().unwrap().bank().is_none() { - break; - } - sleep(Duration::from_millis(5)); - } - } - if check_txs( - &signal_receiver, - packets_for_this_iteration.transactions.len(), - &poh_recorder, - ) { - debug!( - "resetting bank {} tx count: {} txs_proc: {}", - bank.slot(), - bank.transaction_count(), - txs_processed - ); - txs_processed = bank.transaction_count(); - tx_total_us += duration_as_us(&now.elapsed()); - - let mut poh_time = Measure::start("poh_time"); - poh_recorder - .lock() - .unwrap() - .reset(bank.clone(), Some((bank.slot(), bank.slot() + 1))); - poh_time.stop(); - - let mut new_bank_time = Measure::start("new_bank"); - let new_bank = Bank::new_from_parent(&bank, &collector, bank.slot() + 1); - new_bank_time.stop(); - - let mut insert_time = Measure::start("insert_time"); - bank_forks.write().unwrap().insert(new_bank); - bank = bank_forks.read().unwrap().working_bank(); - insert_time.stop(); - - // set cost tracker limits to MAX so it will not filter out TXs - bank.write_cost_tracker().unwrap().set_limits( - std::u64::MAX, - std::u64::MAX, - std::u64::MAX, - ); - - poh_recorder.lock().unwrap().set_bank(&bank); - assert!(poh_recorder.lock().unwrap().bank().is_some()); - if bank.slot() > 32 { - leader_schedule_cache.set_root(&bank); - bank_forks - .write() - .unwrap() - .set_root(root, &AbsRequestSender::default(), None); - root += 1; - } - debug!( - "new_bank_time: {}us insert_time: {}us poh_time: {}us", - new_bank_time.as_us(), - insert_time.as_us(), - poh_time.as_us(), - ); - } else { - tx_total_us += duration_as_us(&now.elapsed()); - } - - // This signature clear may not actually clear the signatures - // in this chunk, but since we rotate between CHUNKS then - // we should clear them by the time we come around again to re-use that chunk. - bank.clear_signatures(); - total_us += duration_as_us(&now.elapsed()); - debug!( - "time: {} us checked: {} sent: {}", - duration_as_us(&now.elapsed()), - total_num_transactions / num_chunks as u64, - sent, - ); - total_sent += sent; - - if current_iteration_index % 16 == 0 { - let last_blockhash = bank.last_blockhash(); - for packets_for_single_iteration in all_packets.iter_mut() { - packets_for_single_iteration.refresh_blockhash(last_blockhash); - } - } - } - let txs_processed = bank_forks - .read() - .unwrap() - .working_bank() - .transaction_count(); - debug!("processed: {} base: {}", txs_processed, base_tx_count); - eprintln!( - "{{'name': 'banking_bench_total', 'median': '{:.2}'}}", - (1000.0 * 1000.0 * total_sent as f64) / (total_us as f64), - ); - eprintln!( - "{{'name': 'banking_bench_tx_total', 'median': '{:.2}'}}", - (1000.0 * 1000.0 * total_sent as f64) / (tx_total_us as f64), - ); - eprintln!( - "{{'name': 'banking_bench_success_tx_total', 'median': '{:.2}'}}", - (1000.0 * 1000.0 * (txs_processed - base_tx_count) as f64) / (total_us as f64), - ); - - drop(verified_sender); - drop(tpu_vote_sender); - drop(vote_sender); - exit.store(true, Ordering::Relaxed); - banking_stage.join().unwrap(); - debug!("waited for banking_stage"); - poh_service.join().unwrap(); - sleep(Duration::from_secs(1)); - debug!("waited for poh_service"); - } - let _unused = Blockstore::destroy(&ledger_path); -} diff --git a/bench-streamer/.gitignore b/bench-streamer/.gitignore deleted file mode 100644 index 5404b132db..0000000000 --- a/bench-streamer/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/target/ -/farf/ diff --git a/bench-streamer/Cargo.toml b/bench-streamer/Cargo.toml deleted file mode 100644 index f0dd5c683f..0000000000 --- a/bench-streamer/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -authors = ["Solana Maintainers "] -edition = "2021" -name = "solana-bench-streamer" -version = "1.10.41" -repository = "https://github.com/solana-labs/solana" -license = "Apache-2.0" -homepage = "https://solana.com/" -publish = false - -[dependencies] -clap = "2.33.1" -crossbeam-channel = "0.5" -solana-net-utils = { path = "../net-utils", version = "=1.10.41" } -solana-streamer = { path = "../streamer", version = "=1.10.41" } -solana-version = { path = "../version", version = "=0.7.0" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/bench-streamer/src/main.rs b/bench-streamer/src/main.rs deleted file mode 100644 index b12afc1920..0000000000 --- a/bench-streamer/src/main.rs +++ /dev/null @@ -1,153 +0,0 @@ -#![allow(clippy::integer_arithmetic)] - -use { - clap::{crate_description, crate_name, value_t, App, Arg}, - crossbeam_channel::unbounded, - solana_streamer::{ - packet::{Packet, PacketBatch, PacketBatchRecycler, PACKET_DATA_SIZE}, - streamer::{receiver, PacketBatchReceiver, StreamerReceiveStats}, - }, - std::{ - cmp::max, - net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}, - sync::{ - atomic::{AtomicBool, AtomicUsize, Ordering}, - Arc, - }, - thread::{sleep, spawn, JoinHandle, Result}, - time::{Duration, SystemTime}, - }, -}; - -fn producer(addr: &SocketAddr, exit: Arc) -> JoinHandle<()> { - let send = UdpSocket::bind("0.0.0.0:0").unwrap(); - let batch_size = 10; - let mut packet_batch = PacketBatch::with_capacity(batch_size); - packet_batch.resize(batch_size, Packet::default()); - for w in packet_batch.iter_mut() { - w.meta.size = PACKET_DATA_SIZE; - w.meta.set_socket_addr(addr); - } - let packet_batch = Arc::new(packet_batch); - spawn(move || loop { - if exit.load(Ordering::Relaxed) { - return; - } - let mut num = 0; - for p in packet_batch.iter() { - let a = p.meta.socket_addr(); - assert!(p.meta.size <= PACKET_DATA_SIZE); - let data = p.data(..).unwrap_or_default(); - send.send_to(data, &a).unwrap(); - num += 1; - } - assert_eq!(num, 10); - }) -} - -fn sink(exit: Arc, rvs: Arc, r: PacketBatchReceiver) -> JoinHandle<()> { - spawn(move || loop { - if exit.load(Ordering::Relaxed) { - return; - } - let timer = Duration::new(1, 0); - if let Ok(packet_batch) = r.recv_timeout(timer) { - rvs.fetch_add(packet_batch.len(), Ordering::Relaxed); - } - }) -} - -fn main() -> Result<()> { - let mut num_sockets = 1usize; - - let matches = App::new(crate_name!()) - .about(crate_description!()) - .version(solana_version::version!()) - .arg( - Arg::with_name("num-recv-sockets") - .long("num-recv-sockets") - .value_name("NUM") - .takes_value(true) - .help("Use NUM receive sockets"), - ) - .arg( - Arg::with_name("num-producers") - .long("num-producers") - .value_name("NUM") - .takes_value(true) - .help("Use this many producer threads."), - ) - .get_matches(); - - if let Some(n) = matches.value_of("num-recv-sockets") { - num_sockets = max(num_sockets, n.to_string().parse().expect("integer")); - } - - let num_producers = value_t!(matches, "num_producers", u64).unwrap_or(4); - - let port = 0; - let ip_addr = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)); - let mut addr = SocketAddr::new(ip_addr, 0); - - let exit = Arc::new(AtomicBool::new(false)); - - let mut read_channels = Vec::new(); - let mut read_threads = Vec::new(); - let recycler = PacketBatchRecycler::default(); - let stats = Arc::new(StreamerReceiveStats::new("bench-streamer-test")); - let (_port, read_sockets) = solana_net_utils::multi_bind_in_range( - ip_addr, - (port, port + num_sockets as u16), - num_sockets, - ) - .unwrap(); - let stats = Arc::new(StreamerReceiveStats::new("bench-streamer-test")); - for read in read_sockets { - read.set_read_timeout(Some(Duration::new(1, 0))).unwrap(); - - addr = read.local_addr().unwrap(); - let (s_reader, r_reader) = unbounded(); - read_channels.push(r_reader); - read_threads.push(receiver( - Arc::new(read), - exit.clone(), - s_reader, - recycler.clone(), - stats.clone(), - 1, - true, - None, - )); - } - - let producer_threads: Vec<_> = (0..num_producers) - .into_iter() - .map(|_| producer(&addr, exit.clone())) - .collect(); - - let rvs = Arc::new(AtomicUsize::new(0)); - let sink_threads: Vec<_> = read_channels - .into_iter() - .map(|r_reader| sink(exit.clone(), rvs.clone(), r_reader)) - .collect(); - let start = SystemTime::now(); - let start_val = rvs.load(Ordering::Relaxed); - sleep(Duration::new(5, 0)); - let elapsed = start.elapsed().unwrap(); - let end_val = rvs.load(Ordering::Relaxed); - let time = elapsed.as_secs() * 10_000_000_000 + u64::from(elapsed.subsec_nanos()); - let ftime = (time as f64) / 10_000_000_000_f64; - let fcount = (end_val - start_val) as f64; - println!("performance: {:?}", fcount / ftime); - exit.store(true, Ordering::Relaxed); - for t_reader in read_threads { - t_reader.join()?; - } - for t_producer in producer_threads { - t_producer.join()?; - } - for t_sink in sink_threads { - t_sink.join()?; - } - Ok(()) -} diff --git a/bench-tps-evm/.gitignore b/bench-tps-evm/.gitignore deleted file mode 100644 index 252411f34d..0000000000 --- a/bench-tps-evm/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/target/ -/config/ -/config-local/ -/farf/ diff --git a/bench-tps-evm/Cargo.toml b/bench-tps-evm/Cargo.toml deleted file mode 100644 index f3b5e6a508..0000000000 --- a/bench-tps-evm/Cargo.toml +++ /dev/null @@ -1,41 +0,0 @@ -[package] -authors = ["Solana Maintainers "] -edition = "2018" -name = "solana-bench-tps-evm" -version = "1.0.0" -repository = "https://github.com/solana-labs/solana" -license = "Apache-2.0" -homepage = "https://solana.com/" - -[dependencies] -bincode = "1.3.1" -clap = "2.33.1" -log = "0.4.8" -rayon = "1.4.0" -serde_json = "1.0.56" -serde_yaml = "0.8.13" -solana-clap-utils = { path = "../clap-utils", version = "1.10.41" } -solana-core = { path = "../core", version = "1.4.0" } -solana-genesis = { path = "../genesis", version = "1.4.0" } -solana-client = { path = "../client", version = "1.4.0" } -solana-faucet = { path = "../faucet", version = "1.4.0" } -solana-logger = { path = "../logger", version = "1.4.0" } -solana-metrics = { path = "../metrics", version = "1.4.0" } -solana-measure = { path = "../measure", version = "1.4.0" } -solana-net-utils = { path = "../net-utils", version = "1.4.0" } -solana-runtime = { path = "../runtime", version = "1.4.0" } -solana-sdk = { path = "../sdk", version = "1.4.0" } -solana-gossip = { path = "../gossip", version = "1.9.23" } -solana-streamer = { path = "../streamer", version = "=1.10.41" } -solana-version = { path = "../version", version = "=0.7.0" } -solana-evm-loader-program = { path = "../evm-utils/programs/evm_loader" } -rand_isaac = "0.1.1" -simple_logger = "2.2" - -[dev-dependencies] -serial_test = "0.4.0" -serial_test_derive = "0.4.0" -solana-local-cluster = { path = "../local-cluster", version = "1.4.0" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/bench-tps-evm/src/bench.rs b/bench-tps-evm/src/bench.rs deleted file mode 100644 index 52f1a25621..0000000000 --- a/bench-tps-evm/src/bench.rs +++ /dev/null @@ -1,683 +0,0 @@ -use log::*; -use rayon::prelude::*; -use std::{ - collections::{HashSet, VecDeque}, - net::SocketAddr, - process::exit, - sync::{ - atomic::{AtomicBool, AtomicUsize, Ordering}, - Arc, Mutex, RwLock, - }, - thread::{sleep, Builder, JoinHandle}, - time::{Duration, Instant}, -}; - -use crate::perf_utils::{sample_txs, SampleStats}; -use solana_core::gen_keys::GenKeys; -use solana_faucet::faucet::request_airdrop_transaction; -use solana_measure::measure::Measure; -use solana_metrics::{self, datapoint_info}; -use solana_sdk::{ - client::Client, - clock::{ - DEFAULT_MS_PER_SLOT, DEFAULT_TICKS_PER_SECOND, DEFAULT_TICKS_PER_SLOT, MAX_PROCESSING_AGE, - }, - commitment_config::CommitmentConfig, - hash::Hash, - instruction::{AccountMeta, Instruction}, - message::Message, - pubkey::Pubkey, - signature::{Keypair, Signer}, - system_instruction, - timing::duration_as_s, - transaction::Transaction, -}; - -// The point at which transactions become "too old", in seconds. -pub const MAX_TX_QUEUE_AGE: u64 = - MAX_PROCESSING_AGE as u64 * DEFAULT_TICKS_PER_SLOT / DEFAULT_TICKS_PER_SECOND; - -pub const MAX_SPENDS_PER_TX: u64 = 4; - -#[derive(Debug)] -pub enum BenchTpsError { - AirdropFailure, -} - -pub type Result = std::result::Result; - -pub type SharedTransactions = Arc>>>; - -pub fn get_recent_blockhash(client: &T) -> Hash { - loop { - match client.get_latest_blockhash_with_commitment(CommitmentConfig::processed()) { - Ok((blockhash, _last_valid_height)) => return blockhash, - Err(err) => { - info!("Couldn't get recent blockhash: {:?}", err); - if cfg!(not(test)) { - sleep(Duration::from_secs(1)); - } - } - }; - } -} - -pub fn wait_for_target_slots_per_epoch(target_slots_per_epoch: u64, client: &Arc) -where - T: 'static + Client, -{ - if target_slots_per_epoch != 0 { - info!( - "Waiting until epochs are {} slots long..", - target_slots_per_epoch - ); - loop { - if let Ok(epoch_info) = client.get_epoch_info() { - if epoch_info.slots_in_epoch >= target_slots_per_epoch { - info!("Done epoch_info: {:?}", epoch_info); - return; - } - info!( - "Waiting for epoch: {} now: {}", - target_slots_per_epoch, epoch_info.slots_in_epoch - ); - } - sleep(Duration::from_secs(3)); - } - } -} - -pub fn create_sampler_thread( - client: &Arc, - exit_signal: &Arc, - sample_period: u64, - maxes: &Arc>>, -) -> JoinHandle<()> -where - T: 'static + Client + Send + Sync, -{ - info!("Sampling TPS every {} second...", sample_period); - let exit_signal = exit_signal.clone(); - let maxes = maxes.clone(); - let client = client.clone(); - Builder::new() - .name("solana-client-sample".to_string()) - .spawn(move || { - sample_txs(&exit_signal, &maxes, sample_period, &client); - }) - .unwrap() -} - -pub fn metrics_submit_lamport_balance(lamport_balance: u64) { - info!("Token balance: {}", lamport_balance); - datapoint_info!( - "bench-tps-lamport_balance", - ("balance", lamport_balance, i64) - ); -} - -fn get_new_latest_blockhash(client: &Arc, blockhash: &Hash) -> Option { - let start = Instant::now(); - while start.elapsed().as_secs() < 5 { - if let Ok(new_blockhash) = client.get_latest_blockhash() { - if new_blockhash != *blockhash { - return Some(new_blockhash); - } - } - debug!("Got same blockhash ({:?}), will retry...", blockhash); - - // Retry ~twice during a slot - sleep(Duration::from_millis(DEFAULT_MS_PER_SLOT / 2)); - } - None -} - -pub fn poll_blockhash( - exit_signal: &Arc, - blockhash: &Arc>, - client: &Arc, - id: &Pubkey, -) { - let mut blockhash_last_updated = Instant::now(); - let mut last_error_log = Instant::now(); - loop { - let blockhash_updated = { - let old_blockhash = *blockhash.read().unwrap(); - if let Some(new_blockhash) = get_new_latest_blockhash(client, &old_blockhash) { - *blockhash.write().unwrap() = new_blockhash; - blockhash_last_updated = Instant::now(); - true - } else { - if blockhash_last_updated.elapsed().as_secs() > 120 { - eprintln!("Blockhash is stuck"); - exit(1) - } else if blockhash_last_updated.elapsed().as_secs() > 30 - && last_error_log.elapsed().as_secs() >= 1 - { - last_error_log = Instant::now(); - error!("Blockhash is not updating"); - } - false - } - }; - - if blockhash_updated { - let balance = client.get_balance(id).unwrap_or(0); - metrics_submit_lamport_balance(balance); - } - - if exit_signal.load(Ordering::Relaxed) { - break; - } - - sleep(Duration::from_millis(50)); - } -} - -fn verify_funding_transfer(client: &Arc, tx: &Transaction, amount: u64) -> bool { - for a in &tx.message().account_keys[1..] { - match client.get_balance_with_commitment(a, CommitmentConfig::processed()) { - Ok(balance) => return balance >= amount, - Err(err) => error!("failed to get balance {:?}", err), - } - } - false -} - -trait FundingTransactions<'a> { - fn fund( - &mut self, - client: &Arc, - to_fund: &[(&'a Keypair, Vec<(Pubkey, u64)>)], - to_lamports: u64, - ); - fn make(&mut self, to_fund: &[(&'a Keypair, Vec<(Pubkey, u64)>)]); - fn sign(&mut self, blockhash: Hash); - fn send(&self, client: &Arc); - fn verify(&mut self, client: &Arc, to_lamports: u64); -} - -impl<'a> FundingTransactions<'a> for Vec<(&'a Keypair, Transaction)> { - fn fund( - &mut self, - client: &Arc, - to_fund: &[(&'a Keypair, Vec<(Pubkey, u64)>)], - to_lamports: u64, - ) { - self.make(to_fund); - - let mut tries = 0; - while !self.is_empty() { - info!( - "{} {} each to {} accounts in {} txs", - if tries == 0 { - "transferring" - } else { - " retrying" - }, - to_lamports, - self.len() * MAX_SPENDS_PER_TX as usize, - self.len(), - ); - - let blockhash = get_recent_blockhash(client.as_ref()); - - // re-sign retained to_fund_txes with updated blockhash - self.sign(blockhash); - self.send(client); - - // Sleep a few slots to allow transactions to process - sleep(Duration::from_secs(1)); - - self.verify(client, to_lamports); - - // retry anything that seems to have dropped through cracks - // again since these txs are all or nothing, they're fine to - // retry - tries += 1; - } - info!("transferred"); - } - - fn make(&mut self, to_fund: &[(&'a Keypair, Vec<(Pubkey, u64)>)]) { - let mut make_txs = Measure::start("make_txs"); - let to_fund_txs: Vec<(&Keypair, Transaction)> = to_fund - .par_iter() - .map(|(k, t)| { - let instructions = system_instruction::transfer_many(&k.pubkey(), t); - let message = Message::new(&instructions, Some(&k.pubkey())); - (*k, Transaction::new_unsigned(message)) - }) - .collect(); - make_txs.stop(); - debug!( - "make {} unsigned txs: {}us", - to_fund_txs.len(), - make_txs.as_us() - ); - self.extend(to_fund_txs); - } - - fn sign(&mut self, blockhash: Hash) { - let mut sign_txs = Measure::start("sign_txs"); - self.par_iter_mut().for_each(|(k, tx)| { - tx.sign(&[*k], blockhash); - }); - sign_txs.stop(); - debug!("sign {} txs: {}us", self.len(), sign_txs.as_us()); - } - - fn send(&self, client: &Arc) { - let mut send_txs = Measure::start("send_txs"); - self.iter().for_each(|(_, tx)| { - client.async_send_transaction(tx.clone()).expect("transfer"); - }); - send_txs.stop(); - debug!("send {} txs: {}us", self.len(), send_txs.as_us()); - } - - fn verify(&mut self, client: &Arc, to_lamports: u64) { - let starting_txs = self.len(); - let verified_txs = Arc::new(AtomicUsize::new(0)); - let too_many_failures = Arc::new(AtomicBool::new(false)); - let loops = if starting_txs < 1000 { 3 } else { 1 }; - // Only loop multiple times for small (quick) transaction batches - let time = Arc::new(Mutex::new(Instant::now())); - for _ in 0..loops { - let time = time.clone(); - let failed_verify = Arc::new(AtomicUsize::new(0)); - let client = client.clone(); - let verified_txs = &verified_txs; - let failed_verify = &failed_verify; - let too_many_failures = &too_many_failures; - let verified_set: HashSet = self - .par_iter() - .filter_map(move |(k, tx)| { - if too_many_failures.load(Ordering::Relaxed) { - return None; - } - - let verified = if verify_funding_transfer(&client, tx, to_lamports) { - verified_txs.fetch_add(1, Ordering::Relaxed); - Some(k.pubkey()) - } else { - failed_verify.fetch_add(1, Ordering::Relaxed); - None - }; - - let verified_txs = verified_txs.load(Ordering::Relaxed); - let failed_verify = failed_verify.load(Ordering::Relaxed); - let remaining_count = starting_txs.saturating_sub(verified_txs + failed_verify); - if failed_verify > 100 && failed_verify > verified_txs { - too_many_failures.store(true, Ordering::Relaxed); - warn!( - "Too many failed transfers... {} remaining, {} verified, {} failures", - remaining_count, verified_txs, failed_verify - ); - } - if remaining_count > 0 { - let mut time_l = time.lock().unwrap(); - if time_l.elapsed().as_secs() > 2 { - info!( - "Verifying transfers... {} remaining, {} verified, {} failures", - remaining_count, verified_txs, failed_verify - ); - *time_l = Instant::now(); - } - } - - verified - }) - .collect(); - - self.retain(|(k, _)| !verified_set.contains(&k.pubkey())); - if self.is_empty() { - break; - } - info!("Looping verifications"); - - let verified_txs = verified_txs.load(Ordering::Relaxed); - let failed_verify = failed_verify.load(Ordering::Relaxed); - let remaining_count = starting_txs.saturating_sub(verified_txs + failed_verify); - info!( - "Verifying transfers... {} remaining, {} verified, {} failures", - remaining_count, verified_txs, failed_verify - ); - sleep(Duration::from_millis(100)); - } - } -} - -/// fund the dests keys by spending all of the source keys into MAX_SPENDS_PER_TX -/// on every iteration. This allows us to replay the transfers because the source is either empty, -/// or full -pub fn fund_keys( - client: Arc, - source: &Keypair, - dests: &[Keypair], - total: u64, - max_fee: u64, - lamports_per_account: u64, -) { - let mut funded: Vec<&Keypair> = vec![source]; - let mut funded_funds = total; - let mut not_funded: Vec<&Keypair> = dests.iter().collect(); - while !not_funded.is_empty() { - // Build to fund list and prepare funding sources for next iteration - let mut new_funded: Vec<&Keypair> = vec![]; - let mut to_fund: Vec<(&Keypair, Vec<(Pubkey, u64)>)> = vec![]; - let to_lamports = (funded_funds - lamports_per_account - max_fee) / MAX_SPENDS_PER_TX; - for f in funded { - let start = not_funded.len() - MAX_SPENDS_PER_TX as usize; - let dests: Vec<_> = not_funded.drain(start..).collect(); - let spends: Vec<_> = dests.iter().map(|k| (k.pubkey(), to_lamports)).collect(); - to_fund.push((f, spends)); - new_funded.extend(dests.into_iter()); - } - - // try to transfer a "few" at a time with recent blockhash - // assume 4MB network buffers, and 512 byte packets - const FUND_CHUNK_LEN: usize = 4 * 1024 * 1024 / 512; - - to_fund.chunks(FUND_CHUNK_LEN).for_each(|chunk| { - Vec::<(&Keypair, Transaction)>::with_capacity(chunk.len()).fund( - &client, - chunk, - to_lamports, - ); - }); - - info!("funded: {} left: {}", new_funded.len(), not_funded.len()); - funded = new_funded; - funded_funds = to_lamports; - } -} - -pub fn airdrop_lamports( - client: &T, - faucet_addr: &SocketAddr, - id: &Keypair, - desired_balance: u64, -) -> Result<()> { - let starting_balance = client.get_balance(&id.pubkey()).unwrap_or(0); - metrics_submit_lamport_balance(starting_balance); - info!("starting balance {}", starting_balance); - - if starting_balance < desired_balance { - let airdrop_amount = desired_balance - starting_balance; - info!( - "Airdropping {:?} lamports from {} for {}", - airdrop_amount, - faucet_addr, - id.pubkey(), - ); - - let blockhash = get_recent_blockhash(client); - match request_airdrop_transaction(faucet_addr, &id.pubkey(), airdrop_amount, blockhash) { - Ok(transaction) => { - let mut tries = 0; - loop { - tries += 1; - let signature = client.async_send_transaction(transaction.clone()).unwrap(); - let result = client.poll_for_signature_confirmation(&signature, 1); - - if result.is_ok() { - break; - } - if tries >= 5 { - panic!( - "Error requesting airdrop: to addr: {:?} amount: {} {:?}", - faucet_addr, airdrop_amount, result - ) - } - } - } - Err(err) => { - panic!( - "Error requesting airdrop: {:?} to addr: {:?} amount: {}", - err, faucet_addr, airdrop_amount - ); - } - }; - - let current_balance = client - .get_balance_with_commitment(&id.pubkey(), CommitmentConfig::processed()) - .unwrap_or_else(|e| { - info!("airdrop error {}", e); - starting_balance - }); - info!("current balance {}...", current_balance); - - metrics_submit_lamport_balance(current_balance); - if current_balance - starting_balance != airdrop_amount { - info!( - "Airdrop failed! {} {} {}", - id.pubkey(), - current_balance, - starting_balance - ); - return Err(BenchTpsError::AirdropFailure); - } - } - Ok(()) -} - -pub fn compute_and_report_stats( - maxes: &Arc>>, - sample_period: u64, - tx_send_elapsed: &Duration, - total_tx_send_count: usize, -) { - // Compute/report stats - let mut max_of_maxes = 0.0; - let mut max_tx_count = 0; - let mut nodes_with_zero_tps = 0; - let mut total_maxes = 0.0; - info!(" Node address | Max TPS | Total Transactions"); - info!("---------------------+---------------+--------------------"); - - for (sock, stats) in maxes.read().unwrap().iter() { - let maybe_flag = match stats.txs { - 0 => "!!!!!", - _ => "", - }; - - info!( - "{:20} | {:13.2} | {} {}", - sock, stats.tps, stats.txs, maybe_flag - ); - - if stats.tps == 0.0 { - nodes_with_zero_tps += 1; - } - total_maxes += stats.tps; - - if stats.tps > max_of_maxes { - max_of_maxes = stats.tps; - } - if stats.txs > max_tx_count { - max_tx_count = stats.txs; - } - } - - if total_maxes > 0.0 { - let num_nodes_with_tps = maxes.read().unwrap().len() - nodes_with_zero_tps; - let average_max = total_maxes / num_nodes_with_tps as f32; - info!( - "\nAverage max TPS: {:.2}, {} nodes had 0 TPS", - average_max, nodes_with_zero_tps - ); - } - - let total_tx_send_count = total_tx_send_count as u64; - let drop_rate = if total_tx_send_count > max_tx_count { - (total_tx_send_count - max_tx_count) as f64 / total_tx_send_count as f64 - } else { - 0.0 - }; - info!( - "\nHighest TPS: {:.2} sampling period {}s max transactions: {} clients: {} drop rate: {:.2}", - max_of_maxes, - sample_period, - max_tx_count, - maxes.read().unwrap().len(), - drop_rate, - ); - info!( - "\tAverage TPS: {}", - max_tx_count as f32 / duration_as_s(tx_send_elapsed) - ); -} - -pub fn generate_keypairs(seed_keypair: &Keypair, count: u64) -> (Vec, u64) { - let mut seed = [0u8; 32]; - seed.copy_from_slice(&seed_keypair.to_bytes()[..32]); - let mut rnd = GenKeys::new(seed); - - let mut total_keys = 0; - let mut extra = 1; // This variable tracks the number of keypairs needing extra transaction fees funded - let mut delta = 1; - while total_keys < count { - extra += delta; - delta *= MAX_SPENDS_PER_TX; - total_keys += delta; - } - (rnd.gen_n_keypairs(total_keys), extra) -} - -pub fn generate_and_fund_keypairs( - client: Arc, - faucet_addr: Option, - funding_key: &Keypair, - keypair_count: usize, - lamports_per_account: u64, -) -> Result> { - info!("Creating {} keypairs...", keypair_count); - let (mut keypairs, extra) = generate_keypairs(funding_key, keypair_count as u64); - info!("Get lamports..."); - - // Sample the first keypair, to prevent lamport loss on repeated solana-bench-tps executions - let first_key = keypairs[0].pubkey(); - let first_keypair_balance = client.get_balance(&first_key).unwrap_or(0); - - // Sample the last keypair, to check if funding was already completed - let last_key = keypairs[keypair_count - 1].pubkey(); - let last_keypair_balance = client.get_balance(&last_key).unwrap_or(0); - - // Repeated runs will eat up keypair balances from transaction fees. In order to quickly - // start another bench-tps run without re-funding all of the keypairs, check if the - // keypairs still have at least 80% of the expected funds. That should be enough to - // pay for the transaction fees in a new run. - let enough_lamports = 8 * lamports_per_account / 10; - if first_keypair_balance < enough_lamports || last_keypair_balance < enough_lamports { - let single_sig_message = Message::new_with_blockhash( - &[Instruction::new_with_bytes( - Pubkey::new_unique(), - &[], - vec![AccountMeta::new(Pubkey::new_unique(), true)], - )], - None, - &client.get_latest_blockhash().unwrap(), - ); - let max_fee = client.get_fee_for_message(&single_sig_message).unwrap(); - let extra_fees = extra * max_fee; - let total_keypairs = keypairs.len() as u64 + 1; // Add one for funding keypair - - let total = lamports_per_account * total_keypairs + extra_fees; - let total = total * 2; - let funding_key_balance = client.get_balance(&funding_key.pubkey()).unwrap_or(0); - info!( - "Funding keypair balance: {} max_fee: {} lamports_per_account: {} extra: {} total: {}", - funding_key_balance, max_fee, lamports_per_account, extra, total - ); - - if client.get_balance(&funding_key.pubkey()).unwrap_or(0) < total { - airdrop_lamports(client.as_ref(), &faucet_addr.unwrap(), funding_key, total)?; - } - - fund_keys( - client, - funding_key, - &keypairs, - total, - max_fee, - lamports_per_account * 2, - ); - } - - // 'generate_keypairs' generates extra keys to be able to have size-aligned funding batches for fund_keys. - keypairs.truncate(keypair_count); - - Ok(keypairs) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::bench_evm::{do_bench_tps, Peer}; - use crate::cli::Config; - use solana_evm_loader_program::scope::evm::FromKey; - use solana_runtime::bank::Bank; - use solana_runtime::bank_client::BankClient; - use solana_sdk::client::SyncClient; - use solana_sdk::fee_calculator::FeeRateGovernor; - use solana_sdk::genesis_config::create_genesis_config; - - #[test] - #[ignore] - fn test_bench_tps_bank_client() { - let (genesis_config, id) = create_genesis_config(10_000); - let bank = Bank::new_for_tests(&genesis_config); - let client = Arc::new(BankClient::new(bank)); - - let config = Config { - id, - tx_count: 10, - duration: Duration::from_secs(5), - ..Config::default() - }; - - let keypair_count = config.tx_count * config.keypair_multiplier; - let keypairs = - generate_and_fund_keypairs(client.clone(), None, &config.id, keypair_count, 20) - .unwrap(); - - let keypairs = - crate::bench_evm::generate_and_fund_evm_keypairs(client.clone(), None, keypairs, 20) - .unwrap(); - let keypairs = keypairs - .into_iter() - .map(|(k, s)| Peer(std::sync::Arc::new(k), s, 0)) - .collect(); - do_bench_tps(client, config, keypairs); - } - #[test] - #[ignore] - fn test_bench_tps_fund_keys_with_fees() { - let (mut genesis_config, id) = create_genesis_config(10_000); - let fee_rate_governor = FeeRateGovernor::new(11, 0); - genesis_config.fee_rate_governor = fee_rate_governor; - let bank = Bank::new_for_tests(&genesis_config); - let client = Arc::new(BankClient::new(bank)); - let keypair_count = 20; - let lamports = 20; - - let keypairs = - generate_and_fund_keypairs(client.clone(), None, &id, keypair_count, lamports).unwrap(); - assert_eq!(keypairs.len(), keypair_count); - let keypairs = crate::bench_evm::generate_and_fund_evm_keypairs( - client.clone(), - None, - keypairs, - lamports, - ) - .unwrap(); - - for (_kp, secret_key) in &keypairs { - assert_eq!( - client.get_evm_balance(&secret_key.to_address()).unwrap(), - solana_evm_loader_program::scope::evm::lamports_to_gwei(lamports) - ); - } - } -} diff --git a/bench-tps-evm/src/bench_evm.rs b/bench-tps-evm/src/bench_evm.rs deleted file mode 100644 index 7b3b175d01..0000000000 --- a/bench-tps-evm/src/bench_evm.rs +++ /dev/null @@ -1,665 +0,0 @@ -use crate::cli::Config; -use log::*; -use rayon::prelude::*; -use std::{ - collections::{HashSet, VecDeque}, - net::SocketAddr, - sync::{ - atomic::{AtomicBool, AtomicIsize, AtomicUsize, Ordering}, - Arc, Mutex, RwLock, - }, - thread::{sleep, Builder, JoinHandle}, - time::{Duration, Instant}, -}; - -use crate::bench::Result; -use crate::bench::SharedTransactions; -use solana_measure::measure::Measure; -use solana_metrics::{self, datapoint_info}; -use solana_sdk::{ - client::Client, - hash::Hash, - message::Message, - pubkey::Pubkey, - signature::{Keypair, Signer}, - timing::{duration_as_ms, duration_as_s, duration_as_us, timestamp}, - transaction::Transaction, -}; - -pub const MAX_SPENDS_PER_TX: u64 = 4; -use solana_evm_loader_program::scope::evm::FromKey; -use solana_evm_loader_program::scope::evm::U256; -use solana_evm_loader_program::scope::*; - -pub const BENCH_SEED: &str = "authority"; - -#[derive(Clone, Debug)] -pub struct Peer(pub std::sync::Arc, pub evm::SecretKey, pub u64); - -pub fn generate_evm_key(seed_keypair: &Keypair) -> evm::SecretKey { - use solana_evm_loader_program::scope::evm::rand::SeedableRng; - - let mut seed = [0u8; 32]; - seed.copy_from_slice(&seed_keypair.to_bytes()[..32]); - let mut rng = rand_isaac::IsaacRng::from_seed(seed); - - evm::SecretKey::new(&mut rng) -} - -pub fn generate_and_fund_evm_keypairs( - client: Arc, - _faucet_addr: Option, - sources: Vec, - lamports_per_account: u64, -) -> Result> { - info!("Creating {} keypairs...", sources.len()); - let keypairs: Vec<_> = sources - .into_iter() - .map(|key| { - let evm_key = generate_evm_key(&key); - (key, evm_key) - }) - .collect(); - info!("Get lamports..."); - - // Sample the first keypair, to prevent lamport loss on repeated solana-bench-tps executions - let first_key = keypairs[0].1; - let first_keypair_balance = client - .get_evm_balance(&first_key.to_address()) - .unwrap_or_default(); - - // Sample the last keypair, to check if funding was already completed - let last_key = keypairs[keypairs.len() - 1].1; - let last_keypair_balance = client - .get_evm_balance(&last_key.to_address()) - .unwrap_or_default(); - - let enough_gweis = evm::lamports_to_gwei(lamports_per_account); - if first_keypair_balance < enough_gweis || last_keypair_balance < enough_gweis { - info!("Funding evm keys.",); - fund_evm_keys(client, &keypairs, lamports_per_account); - } - - Ok(keypairs) -} - -fn verify_funding_transfer( - client: &Arc, - evm_key: evm::Address, - _tx: &Transaction, - amount: u64, -) -> bool { - match client.get_evm_balance(&evm_key) { - Ok(balance) => return balance >= U256::from(amount), - Err(err) => error!("failed to get balance {:?}", err), - } - false -} - -trait FundingTransactions<'a> { - fn fund( - &mut self, - client: &Arc, - to_fund: &'a [(Keypair, evm::SecretKey)], - to_lamports: u64, - ); - fn make(&mut self, to_fund: &'a [(Keypair, evm::SecretKey)], to_lamports: u64); - fn sign(&mut self, blockhash: Hash); - fn send(&self, client: &Arc); - fn verify(&mut self, client: &Arc, to_lamports: u64); -} - -impl<'a> FundingTransactions<'a> for Vec<(&'a Keypair, &'a evm::SecretKey, Transaction)> { - fn fund( - &mut self, - client: &Arc, - to_fund: &'a [(Keypair, evm::SecretKey)], - to_lamports: u64, - ) { - self.make(to_fund, to_lamports); - - let mut tries = 0; - while !self.is_empty() { - info!( - "{} {} each to {} accounts in {} txs", - if tries == 0 { - "transferring" - } else { - " retrying" - }, - to_lamports, - self.len() * MAX_SPENDS_PER_TX as usize, - self.len(), - ); - - let blockhash = crate::bench::get_recent_blockhash(client.as_ref()); - - // re-sign retained to_fund_txes with updated blockhash - self.sign(blockhash); - self.send(client); - - // Sleep a few slots to allow transactions to process - if cfg!(not(test)) { - sleep(Duration::from_secs(1)); - } - - self.verify(client, to_lamports); - - // retry anything that seems to have dropped through cracks - // again since these txs are all or nothing, they're fine to - // retry - tries += 1; - } - info!("transferred"); - } - - fn make(&mut self, to_fund: &'a [(Keypair, evm::SecretKey)], to_lamports: u64) { - let mut make_txs = Measure::start("make_txs"); - let to_fund_txs: Vec<(&Keypair, &evm::SecretKey, Transaction)> = to_fund - .par_iter() - .map(|(k, evm)| { - let instructions = solana_evm_loader_program::transfer_native_to_evm_ixs( - k.pubkey(), - to_lamports, - evm.to_address(), - ); - let message = Message::new(&instructions, Some(&k.pubkey())); - (k, evm, Transaction::new_unsigned(message)) - }) - .collect(); - make_txs.stop(); - debug!( - "make {} unsigned txs: {}us", - to_fund_txs.len(), - make_txs.as_us() - ); - self.extend(to_fund_txs); - } - - fn sign(&mut self, blockhash: Hash) { - let mut sign_txs = Measure::start("sign_txs"); - self.par_iter_mut().for_each(|(k, _, tx)| { - tx.sign(&[*k], blockhash); - }); - sign_txs.stop(); - debug!("sign {} txs: {}us", self.len(), sign_txs.as_us()); - } - - fn send(&self, client: &Arc) { - let mut send_txs = Measure::start("send_txs"); - self.iter().for_each(|(_, _, tx)| { - client.async_send_transaction(tx.clone()).expect("transfer"); - }); - send_txs.stop(); - debug!("send {} txs: {}us", self.len(), send_txs.as_us()); - } - - fn verify(&mut self, client: &Arc, to_lamports: u64) { - let starting_txs = self.len(); - let verified_txs = Arc::new(AtomicUsize::new(0)); - let too_many_failures = Arc::new(AtomicBool::new(false)); - let loops = if starting_txs < 1000 { 3 } else { 1 }; - // Only loop multiple times for small (quick) transaction batches - let time = Arc::new(Mutex::new(Instant::now())); - for _ in 0..loops { - let time = time.clone(); - let failed_verify = Arc::new(AtomicUsize::new(0)); - let client = client.clone(); - let verified_txs = &verified_txs; - let failed_verify = &failed_verify; - let too_many_failures = &too_many_failures; - let verified_set: HashSet = self - .par_iter() - .filter_map(move |(k, evm_secret, tx)| { - if too_many_failures.load(Ordering::Relaxed) { - return None; - } - - let verified = if verify_funding_transfer( - &client, - evm_secret.to_address(), - tx, - to_lamports, - ) { - verified_txs.fetch_add(1, Ordering::Relaxed); - Some(k.pubkey()) - } else { - failed_verify.fetch_add(1, Ordering::Relaxed); - None - }; - - let verified_txs = verified_txs.load(Ordering::Relaxed); - let failed_verify = failed_verify.load(Ordering::Relaxed); - let remaining_count = starting_txs.saturating_sub(verified_txs + failed_verify); - if failed_verify > 100 && failed_verify > verified_txs { - too_many_failures.store(true, Ordering::Relaxed); - warn!( - "Too many failed transfers... {} remaining, {} verified, {} failures", - remaining_count, verified_txs, failed_verify - ); - } - if remaining_count > 0 { - let mut time_l = time.lock().unwrap(); - if time_l.elapsed().as_secs() > 2 { - info!( - "Verifying transfers... {} remaining, {} verified, {} failures", - remaining_count, verified_txs, failed_verify - ); - *time_l = Instant::now(); - } - } - - verified - }) - .collect(); - - self.retain(|(k, _, _)| !verified_set.contains(&k.pubkey())); - if self.is_empty() { - break; - } - info!("Looping verifications"); - - let verified_txs = verified_txs.load(Ordering::Relaxed); - let failed_verify = failed_verify.load(Ordering::Relaxed); - let remaining_count = starting_txs.saturating_sub(verified_txs + failed_verify); - info!( - "Verifying transfers... {} remaining, {} verified, {} failures", - remaining_count, verified_txs, failed_verify - ); - sleep(Duration::from_millis(100)); - } - } -} - -pub fn fund_evm_keys( - client: Arc, - keys: &[(Keypair, evm::SecretKey)], - lamports_per_account: u64, -) { - // try to transfer a "few" at a time with recent blockhash - // assume 4MB network buffers, and 512 byte packets - const FUND_CHUNK_LEN: usize = 4 * 1024 * 1024 / 512; - - keys.chunks(FUND_CHUNK_LEN).for_each(|chunk| { - Vec::<(&Keypair, _, Transaction)>::with_capacity(chunk.len()).fund( - &client, - chunk, - lamports_per_account, - ); - }); - - info!("evm funded: {}", keys.len(),); -} - -fn generate_system_txs( - source: &mut [Peer], - dest: &mut VecDeque, - reclaim: bool, - blockhash: &Hash, - chain_id: Option, -) -> Vec<(Transaction, u64)> { - let mut pairs: Vec<_> = if !reclaim { - source.iter_mut().zip(dest.iter_mut()).collect() - } else { - dest.iter_mut().zip(source.iter_mut()).collect() - }; - - pairs - .par_iter_mut() - .map(|(from, to)| { - let tx_address = to.1.to_address(); - - let tx_call = evm::UnsignedTransaction { - nonce: from.2.into(), - gas_price: 0.into(), - gas_limit: 300000.into(), - action: evm::TransactionAction::Call(tx_address), - value: 1.into(), - input: vec![], - }; - - let tx_call = tx_call.sign(&from.1, chain_id); - - from.2 += 1; - let ix = solana_evm_loader_program::send_raw_tx( - from.0.pubkey(), - tx_call, - None, - solana_evm_loader_program::instructions::FeePayerType::Evm, - ); - - let message = Message::new(&[ix], Some(&from.0.pubkey())); - - ( - Transaction::new(&[&*from.0], message, *blockhash), - timestamp(), - ) - }) - .collect() -} - -fn generate_txs( - shared_txs: &SharedTransactions, - blockhash: &Arc>, - source: &mut [Peer], - dest: &mut VecDeque, - threads: usize, - reclaim: bool, - chain_id: Option, -) { - let blockhash = *blockhash.read().unwrap(); - let tx_count = source.len(); - info!( - "Signing transactions... {} (reclaim={}, blockhash={})", - tx_count, reclaim, &blockhash - ); - let signing_start = Instant::now(); - - let transactions = generate_system_txs(source, dest, reclaim, &blockhash, chain_id); - - let duration = signing_start.elapsed(); - let ns = duration.as_secs() * 1_000_000_000 + u64::from(duration.subsec_nanos()); - let bsps = (tx_count) as f64 / ns as f64; - let nsps = ns as f64 / (tx_count) as f64; - info!( - "Done. {:.2} thousand signatures per second, {:.2} us per signature, {} ms total time, {}", - bsps * 1_000_000_f64, - nsps / 1_000_f64, - duration_as_ms(&duration), - blockhash, - ); - datapoint_info!( - "bench-tps-generate_txs", - ("duration", duration_as_us(&duration), i64) - ); - - let sz = transactions.len() / threads; - let chunks: Vec<_> = transactions.chunks(sz).collect(); - { - let mut shared_txs_wl = shared_txs.write().unwrap(); - for chunk in chunks { - shared_txs_wl.push_back(chunk.to_vec()); - } - } -} - -pub fn do_bench_tps(client: Arc, config: Config, gen_keypairs: Vec) -> u64 -where - T: 'static + Client + Send + Sync, -{ - let Config { - id, - threads, - thread_batch_sleep_ms, - duration, - tx_count, - sustained, - target_slots_per_epoch, - chain_id, - .. - } = config; - - let mut source_keypair_chunks: Vec> = Vec::new(); - let mut dest_keypair_chunks: Vec> = Vec::new(); - assert!(gen_keypairs.len() >= 2 * tx_count); - for chunk in gen_keypairs.chunks_exact(2 * tx_count) { - source_keypair_chunks.push(chunk[..tx_count].to_vec()); - dest_keypair_chunks.push(chunk[tx_count..].iter().cloned().collect()); - } - - let first_tx_count = loop { - match client.get_transaction_count() { - Ok(count) => break count, - Err(err) => { - info!("Couldn't get transaction count: {:?}", err); - sleep(Duration::from_secs(1)); - } - } - }; - info!("Initial transaction count {}", first_tx_count); - - let exit_signal = Arc::new(AtomicBool::new(false)); - - // Setup a thread per validator to sample every period - // collect the max transaction rate and total tx count seen - let maxes = Arc::new(RwLock::new(Vec::new())); - let sample_period = 1; // in seconds - let sample_thread = - crate::bench::create_sampler_thread(&client, &exit_signal, sample_period, &maxes); - - let shared_txs: SharedTransactions = Arc::new(RwLock::new(VecDeque::new())); - - let recent_blockhash = Arc::new(RwLock::new( - crate::bench::get_recent_blockhash(client.as_ref()), - )); - let shared_tx_active_thread_count = Arc::new(AtomicIsize::new(0)); - let total_tx_sent_count = Arc::new(AtomicUsize::new(0)); - - let blockhash_thread = { - let exit_signal = exit_signal.clone(); - let recent_blockhash = recent_blockhash.clone(); - let client = client.clone(); - let id = id.pubkey(); - Builder::new() - .name("solana-blockhash-poller".to_string()) - .spawn(move || { - crate::bench::poll_blockhash(&exit_signal, &recent_blockhash, &client, &id); - }) - .unwrap() - }; - - let s_threads = create_sender_threads( - &client, - &shared_txs, - thread_batch_sleep_ms, - &total_tx_sent_count, - threads, - &exit_signal, - &shared_tx_active_thread_count, - ); - - crate::bench::wait_for_target_slots_per_epoch(target_slots_per_epoch, &client); - - let start = Instant::now(); - - generate_chunked_transfers( - recent_blockhash, - &shared_txs, - shared_tx_active_thread_count, - source_keypair_chunks, - dest_keypair_chunks, - threads, - duration, - sustained, - chain_id, - ); - - // Stop the sampling threads so it will collect the stats - exit_signal.store(true, Ordering::Relaxed); - - info!("Waiting for sampler threads..."); - if let Err(err) = sample_thread.join() { - info!(" join() failed with: {:?}", err); - } - - // join the tx send threads - info!("Waiting for transmit threads..."); - for t in s_threads { - if let Err(err) = t.join() { - info!(" join() failed with: {:?}", err); - } - } - - info!("Waiting for blockhash thread..."); - if let Err(err) = blockhash_thread.join() { - info!(" join() failed with: {:?}", err); - } - - let balance = client.get_balance(&id.pubkey()).unwrap_or(0); - crate::bench::metrics_submit_lamport_balance(balance); - - crate::bench::compute_and_report_stats( - &maxes, - sample_period, - &start.elapsed(), - total_tx_sent_count.load(Ordering::Relaxed), - ); - - let r_maxes = maxes.read().unwrap(); - r_maxes.first().unwrap().1.txs -} - -fn generate_chunked_transfers( - recent_blockhash: Arc>, - shared_txs: &SharedTransactions, - shared_tx_active_thread_count: Arc, - mut source_keypair_chunks: Vec>, - mut dest_keypair_chunks: Vec>, - threads: usize, - duration: Duration, - sustained: bool, - chain_id: Option, -) { - // generate and send transactions for the specified duration - let start = Instant::now(); - let keypair_chunks = source_keypair_chunks.len(); - let mut reclaim_lamports_back_to_source_account = false; - let mut chunk_index = 0; - while start.elapsed() < duration { - generate_txs( - shared_txs, - &recent_blockhash, - &mut source_keypair_chunks[chunk_index], - &mut dest_keypair_chunks[chunk_index], - threads, - reclaim_lamports_back_to_source_account, - chain_id, - ); - - // In sustained mode, overlap the transfers with generation. This has higher average - // performance but lower peak performance in tested environments. - if sustained { - // Ensure that we don't generate more transactions than we can handle. - while shared_txs.read().unwrap().len() > 2 * threads { - sleep(Duration::from_millis(1)); - } - } else { - while !shared_txs.read().unwrap().is_empty() - || shared_tx_active_thread_count.load(Ordering::Relaxed) > 0 - { - sleep(Duration::from_millis(1)); - } - } - - // Rotate destination keypairs so that the next round of transactions will have different - // transaction signatures even when blockhash is reused. - dest_keypair_chunks[chunk_index].rotate_left(1); - - // Move on to next chunk - chunk_index = (chunk_index + 1) % keypair_chunks; - - // Switch directions after transfering for each "chunk" - if chunk_index == 0 { - reclaim_lamports_back_to_source_account = !reclaim_lamports_back_to_source_account; - } - } -} - -fn do_tx_transfers( - exit_signal: &Arc, - shared_txs: &SharedTransactions, - shared_tx_thread_count: &Arc, - total_tx_sent_count: &Arc, - thread_batch_sleep_ms: usize, - client: &Arc, -) { - loop { - if thread_batch_sleep_ms > 0 { - sleep(Duration::from_millis(thread_batch_sleep_ms as u64)); - } - let txs = { - let mut shared_txs_wl = shared_txs.write().expect("write lock in do_tx_transfers"); - shared_txs_wl.pop_front() - }; - if let Some(txs0) = txs { - shared_tx_thread_count.fetch_add(1, Ordering::Relaxed); - info!( - "Transferring 1 unit {} times... to {}", - txs0.len(), - client.as_ref().tpu_addr(), - ); - let tx_len = txs0.len(); - let transfer_start = Instant::now(); - let mut old_transactions = false; - for tx in txs0 { - let now = timestamp(); - // Transactions that are too old will be rejected by the cluster Don't bother - // sending them. - if now > tx.1 && now - tx.1 > 1000 * crate::bench::MAX_TX_QUEUE_AGE { - old_transactions = true; - continue; - } - client - .async_send_transaction(tx.0) - .expect("async_send_transaction in do_tx_transfers"); - } - if old_transactions { - let mut shared_txs_wl = shared_txs.write().expect("write lock in do_tx_transfers"); - shared_txs_wl.clear(); - } - shared_tx_thread_count.fetch_add(-1, Ordering::Relaxed); - total_tx_sent_count.fetch_add(tx_len, Ordering::Relaxed); - info!( - "Tx send done. {} ms {} tps", - duration_as_ms(&transfer_start.elapsed()), - tx_len as f32 / duration_as_s(&transfer_start.elapsed()), - ); - datapoint_info!( - "bench-tps-do_tx_transfers", - ("duration", duration_as_us(&transfer_start.elapsed()), i64), - ("count", tx_len, i64) - ); - } - if exit_signal.load(Ordering::Relaxed) { - break; - } - println!("Sleeping 1 sec"); - sleep(Duration::from_secs(1)); - } -} - -fn create_sender_threads( - client: &Arc, - shared_txs: &SharedTransactions, - thread_batch_sleep_ms: usize, - total_tx_sent_count: &Arc, - threads: usize, - exit_signal: &Arc, - shared_tx_active_thread_count: &Arc, -) -> Vec> -where - T: 'static + Client + Send + Sync, -{ - (0..threads) - .map(|_| { - let exit_signal = exit_signal.clone(); - let shared_txs = shared_txs.clone(); - let shared_tx_active_thread_count = shared_tx_active_thread_count.clone(); - let total_tx_sent_count = total_tx_sent_count.clone(); - let client = client.clone(); - Builder::new() - .name("solana-client-sender".to_string()) - .spawn(move || { - do_tx_transfers( - &exit_signal, - &shared_txs, - &shared_tx_active_thread_count, - &total_tx_sent_count, - thread_batch_sleep_ms, - &client, - ); - }) - .unwrap() - }) - .collect() -} diff --git a/bench-tps-evm/src/cli.rs b/bench-tps-evm/src/cli.rs deleted file mode 100644 index f6a7cd5bc7..0000000000 --- a/bench-tps-evm/src/cli.rs +++ /dev/null @@ -1,272 +0,0 @@ -use clap::{crate_description, crate_name, App, Arg, ArgMatches}; -use solana_faucet::faucet::FAUCET_PORT; -use solana_sdk::fee_calculator::FeeRateGovernor; -use solana_sdk::{ - pubkey::Pubkey, - signature::{read_keypair_file, Keypair}, -}; -use std::{net::SocketAddr, process::exit, time::Duration}; - -const NUM_LAMPORTS_PER_ACCOUNT_DEFAULT: u64 = solana_sdk::native_token::LAMPORTS_PER_VLX; - -/// Holds the configuration for a single run of the benchmark -pub struct Config { - pub entrypoint_addr: SocketAddr, - pub faucet_addr: SocketAddr, - pub id: Keypair, - pub threads: usize, - pub num_nodes: usize, - pub duration: Duration, - pub tx_count: usize, - pub keypair_multiplier: usize, - pub thread_batch_sleep_ms: usize, - pub sustained: bool, - pub client_ids_and_stake_file: String, - pub target_lamports_per_signature: u64, - pub multi_client: bool, - pub num_lamports_per_account: u64, - pub target_slots_per_epoch: u64, - pub target_node: Option, - pub chain_id: Option, -} - -impl Default for Config { - fn default() -> Config { - Config { - entrypoint_addr: SocketAddr::from(([127, 0, 0, 1], 8001)), - faucet_addr: SocketAddr::from(([127, 0, 0, 1], FAUCET_PORT)), - id: Keypair::new(), - threads: 4, - num_nodes: 1, - duration: Duration::new(std::u64::MAX, 0), - tx_count: 50_000, - keypair_multiplier: 8, - thread_batch_sleep_ms: 1000, - sustained: false, - client_ids_and_stake_file: String::new(), - target_lamports_per_signature: FeeRateGovernor::default().target_lamports_per_signature, - multi_client: true, - num_lamports_per_account: NUM_LAMPORTS_PER_ACCOUNT_DEFAULT, - target_slots_per_epoch: 0, - target_node: None, - chain_id: None, - } - } -} - -/// Defines and builds the CLI args for a run of the benchmark -pub fn build_args<'a, 'b>(version: &'b str) -> App<'a, 'b> { - App::new(crate_name!()).about(crate_description!()) - .version(version) - .arg( - Arg::with_name("entrypoint") - .short("n") - .long("entrypoint") - .value_name("HOST:PORT") - .takes_value(true) - .help("Rendezvous with the cluster at this entry point; defaults to 127.0.0.1:8001"), - ) - .arg( - Arg::with_name("faucet") - .short("d") - .long("faucet") - .value_name("HOST:PORT") - .takes_value(true) - .help("Location of the faucet; defaults to entrypoint:FAUCET_PORT"), - ) - .arg( - Arg::with_name("identity") - .short("i") - .long("identity") - .value_name("PATH") - .takes_value(true) - .help("File containing a client identity (keypair)"), - ) - .arg( - Arg::with_name("num-nodes") - .short("N") - .long("num-nodes") - .value_name("NUM") - .takes_value(true) - .help("Wait for NUM nodes to converge"), - ) - .arg( - Arg::with_name("threads") - .short("t") - .long("threads") - .value_name("NUM") - .takes_value(true) - .help("Number of threads"), - ) - .arg( - Arg::with_name("duration") - .long("duration") - .value_name("SECS") - .takes_value(true) - .help("Seconds to run benchmark, then exit; default is forever"), - ) - .arg( - Arg::with_name("chain_id") - .long("chain-id") - .takes_value(true) - .required(true) - .help("Evm chain id"), - ) - .arg( - Arg::with_name("sustained") - .long("sustained") - .help("Use sustained performance mode vs. peak mode. This overlaps the tx generation with transfers."), - ) - .arg( - Arg::with_name("no-multi-client") - .long("no-multi-client") - .help("Disable multi-client support, only transact with the entrypoint."), - ) - .arg( - Arg::with_name("target_node") - .long("target-node") - .requires("no-multi-client") - .takes_value(true) - .value_name("PUBKEY") - .help("Specify an exact node to send transactions to."), - ) - .arg( - Arg::with_name("tx_count") - .long("tx_count") - .value_name("NUM") - .takes_value(true) - .help("Number of transactions to send per batch") - ) - .arg( - Arg::with_name("keypair_multiplier") - .long("keypair-multiplier") - .value_name("NUM") - .takes_value(true) - .help("Multiply by transaction count to determine number of keypairs to create") - ) - .arg( - Arg::with_name("thread-batch-sleep-ms") - .short("z") - .long("thread-batch-sleep-ms") - .value_name("NUM") - .takes_value(true) - .help("Per-thread-per-iteration sleep in ms"), - ) - .arg( - Arg::with_name("target_lamports_per_signature") - .long("target-lamports-per-signature") - .value_name("LAMPORTS") - .takes_value(true) - .help( - "The cost in lamports that the cluster will charge for signature \ - verification when the cluster is operating at target-signatures-per-slot", - ), - ) - .arg( - Arg::with_name("num_lamports_per_account") - .long("num-lamports-per-account") - .value_name("LAMPORTS") - .takes_value(true) - .help( - "Number of lamports per account.", - ), - ) - .arg( - Arg::with_name("target_slots_per_epoch") - .long("target-slots-per-epoch") - .value_name("SLOTS") - .takes_value(true) - .help( - "Wait until epochs are this many slots long.", - ), - ) -} - -/// Parses a clap `ArgMatches` structure into a `Config` -/// # Arguments -/// * `matches` - command line arguments parsed by clap -/// # Panics -/// Panics if there is trouble parsing any of the arguments -pub fn extract_args(matches: &ArgMatches) -> Config { - let mut args = Config::default(); - - if let Some(addr) = matches.value_of("entrypoint") { - args.entrypoint_addr = solana_net_utils::parse_host_port(addr).unwrap_or_else(|e| { - eprintln!("failed to parse entrypoint address: {}", e); - exit(1) - }); - } - - if let Some(addr) = matches.value_of("faucet") { - args.faucet_addr = solana_net_utils::parse_host_port(addr).unwrap_or_else(|e| { - eprintln!("failed to parse faucet address: {}", e); - exit(1) - }); - } - - if matches.is_present("identity") { - args.id = read_keypair_file(matches.value_of("identity").unwrap()) - .expect("can't read client identity"); - } - - if let Some(t) = matches.value_of("threads") { - args.threads = t.to_string().parse().expect("can't parse threads"); - } - - if let Some(n) = matches.value_of("num-nodes") { - args.num_nodes = n.to_string().parse().expect("can't parse num-nodes"); - } - - if let Some(duration) = matches.value_of("duration") { - args.duration = Duration::new( - duration.to_string().parse().expect("can't parse duration"), - 0, - ); - } - if let Some(chain_id) = matches.value_of("chain_id") { - args.chain_id = Some(chain_id.parse().expect("can't parse chain-id")); - } - - if let Some(s) = matches.value_of("tx_count") { - args.tx_count = s.to_string().parse().expect("can't parse tx_count"); - } - - if let Some(s) = matches.value_of("keypair_multiplier") { - args.keypair_multiplier = s - .to_string() - .parse() - .expect("can't parse keypair-multiplier"); - assert!(args.keypair_multiplier >= 2); - } - - if let Some(t) = matches.value_of("thread-batch-sleep-ms") { - args.thread_batch_sleep_ms = t - .to_string() - .parse() - .expect("can't parse thread-batch-sleep-ms"); - } - - args.sustained = matches.is_present("sustained"); - - if let Some(v) = matches.value_of("target_lamports_per_signature") { - args.target_lamports_per_signature = v.to_string().parse().expect("can't parse lamports"); - } - - args.multi_client = !matches.is_present("no-multi-client"); - args.target_node = matches - .value_of("target_node") - .map(|target_str| target_str.parse().unwrap()); - - if let Some(v) = matches.value_of("num_lamports_per_account") { - args.num_lamports_per_account = v.to_string().parse().expect("can't parse lamports"); - } - - if let Some(t) = matches.value_of("target_slots_per_epoch") { - args.target_slots_per_epoch = t - .to_string() - .parse() - .expect("can't parse target slots per epoch"); - } - - args -} diff --git a/bench-tps-evm/src/lib.rs b/bench-tps-evm/src/lib.rs deleted file mode 100644 index 91f42c6aba..0000000000 --- a/bench-tps-evm/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod bench; -pub mod bench_evm; -pub mod cli; -pub mod perf_utils; diff --git a/bench-tps-evm/src/main.rs b/bench-tps-evm/src/main.rs deleted file mode 100644 index 842bc7ebb8..0000000000 --- a/bench-tps-evm/src/main.rs +++ /dev/null @@ -1,117 +0,0 @@ -use log::*; -use solana_client::connection_cache::ConnectionCache; -use std::{process::exit, sync::Arc}; - -use solana_bench_tps_evm::bench::generate_and_fund_keypairs; -use solana_bench_tps_evm::bench_evm::{self, Peer}; -use solana_bench_tps_evm::cli; -use solana_gossip::gossip_service::{discover_cluster, get_client, get_multi_client}; - -use solana_streamer::socket::SocketAddrSpace; - -/// Number of signatures for all transactions in ~1 week at ~100K TPS -pub const NUM_SIGNATURES_FOR_TXS: u64 = 100_000 * 60 * 60 * 24 * 7; - -fn main() { - solana_logger::setup_with_default("solana=info"); - solana_metrics::set_panic_hook("bench-tps", None); - - let matches = cli::build_args(solana_version::version!()).get_matches(); - let cli_config = cli::extract_args(&matches); - - let cli::Config { - entrypoint_addr, - faucet_addr, - id, - num_nodes, - tx_count, - keypair_multiplier, - multi_client, - num_lamports_per_account, - target_node, - .. - } = &cli_config; - - let keypair_count = *tx_count * keypair_multiplier; - - info!("Connecting to the cluster"); - let nodes = discover_cluster(entrypoint_addr, *num_nodes, SocketAddrSpace::Unspecified) - .unwrap_or_else(|err| { - eprintln!("Failed to discover {} nodes: {:?}", num_nodes, err); - exit(1); - }); - let use_quic = false; - let tpu_connection_pool_size = 4; - let connection_cache = match use_quic { - true => Arc::new(ConnectionCache::new(tpu_connection_pool_size)), - false => Arc::new(ConnectionCache::with_udp(tpu_connection_pool_size)), - }; - - let client = if *multi_client { - let (client, num_clients) = get_multi_client( - &nodes, - &SocketAddrSpace::Unspecified, - connection_cache.clone(), - ); - if nodes.len() < num_clients { - eprintln!( - "Error: Insufficient nodes discovered. Expecting {} or more", - num_nodes - ); - exit(1); - } - Arc::new(client) - } else if let Some(target_node) = target_node { - info!("Searching for target_node: {:?}", target_node); - let mut target_client = None; - for node in nodes { - if node.id == *target_node { - target_client = Some(Arc::new(get_client( - &[node], - &SocketAddrSpace::Unspecified, - connection_cache.clone(), - ))); - break; - } - } - target_client.unwrap_or_else(|| { - eprintln!("Target node {} not found", target_node); - exit(1); - }) - } else { - Arc::new(get_client( - &nodes, - &SocketAddrSpace::Unspecified, - connection_cache.clone(), - )) - }; - - let keypairs = generate_and_fund_keypairs( - client.clone(), - Some(*faucet_addr), - id, - keypair_count, - *num_lamports_per_account, - ) - .unwrap_or_else(|e| { - eprintln!("Error could not fund keys: {:?}", e); - exit(1); - }); - let keypairs = bench_evm::generate_and_fund_evm_keypairs( - client.clone(), - Some(*faucet_addr), - keypairs, - *num_lamports_per_account, - ) - .unwrap_or_else(|e| { - eprintln!("Error could not fund evm keys: {:?}", e); - exit(1); - }); - - // Init nonce = 0 - let keypairs = keypairs - .into_iter() - .map(|(k, s)| Peer(std::sync::Arc::new(k), s, 0)) - .collect(); - bench_evm::do_bench_tps(client, cli_config, keypairs); -} diff --git a/bench-tps-evm/src/perf_utils.rs b/bench-tps-evm/src/perf_utils.rs deleted file mode 100644 index 4cad7f86c9..0000000000 --- a/bench-tps-evm/src/perf_utils.rs +++ /dev/null @@ -1,94 +0,0 @@ -use solana_sdk::client::Client; - -use { - log::*, - solana_sdk::{commitment_config::CommitmentConfig, timing::duration_as_s}, - std::{ - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, RwLock, - }, - thread::sleep, - time::{Duration, Instant}, - }, -}; - -#[derive(Default)] -pub struct SampleStats { - /// Maximum TPS reported by this node - pub tps: f32, - /// Total time taken for those txs - pub elapsed: Duration, - /// Total transactions reported by this node - pub txs: u64, -} - -pub fn sample_txs( - exit_signal: &Arc, - sample_stats: &Arc>>, - sample_period: u64, - client: &Arc, -) where - T: 'static + Client + Send + Sync, -{ - let mut max_tps = 0.0; - let mut total_elapsed; - let mut total_txs; - let mut now = Instant::now(); - let start_time = now; - let initial_txs = client - .get_transaction_count_with_commitment(CommitmentConfig::processed()) - .expect("transaction count"); - let mut last_txs = initial_txs; - - loop { - total_elapsed = start_time.elapsed(); - let elapsed = now.elapsed(); - now = Instant::now(); - let mut txs = - match client.get_transaction_count_with_commitment(CommitmentConfig::processed()) { - Err(e) => { - // ThinClient with multiple options should pick a better one now. - info!("Couldn't get transaction count {:?}", e); - sleep(Duration::from_secs(sample_period)); - continue; - } - Ok(tx_count) => tx_count, - }; - - if txs < last_txs { - info!("Expected txs({}) >= last_txs({})", txs, last_txs); - txs = last_txs; - } - total_txs = txs - initial_txs; - let sample_txs = txs - last_txs; - last_txs = txs; - - let tps = sample_txs as f32 / duration_as_s(&elapsed); - if tps > max_tps { - max_tps = tps; - } - - info!( - "Sampler {:9.2} TPS, Transactions: {:6}, Total transactions: {} over {} s", - tps, - sample_txs, - total_txs, - total_elapsed.as_secs(), - ); - - if exit_signal.load(Ordering::Relaxed) { - let stats = SampleStats { - tps: max_tps, - elapsed: total_elapsed, - txs: total_txs, - }; - sample_stats - .write() - .unwrap() - .push((client.tpu_addr().to_string(), stats)); - return; - } - sleep(Duration::from_secs(sample_period)); - } -} diff --git a/bench-tps/.gitignore b/bench-tps/.gitignore deleted file mode 100644 index 252411f34d..0000000000 --- a/bench-tps/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/target/ -/config/ -/config-local/ -/farf/ diff --git a/bench-tps/Cargo.toml b/bench-tps/Cargo.toml deleted file mode 100644 index fa7fa5e40a..0000000000 --- a/bench-tps/Cargo.toml +++ /dev/null @@ -1,42 +0,0 @@ -[package] -authors = ["Solana Maintainers "] -edition = "2021" -name = "solana-bench-tps" -version = "1.10.41" -repository = "https://github.com/solana-labs/solana" -license = "Apache-2.0" -homepage = "https://solana.com/" -publish = false - -[dependencies] -clap = "2.33.1" -crossbeam-channel = "0.5" -log = "0.4.14" -rayon = "1.5.1" -serde_json = "1.0.79" -serde_yaml = "0.8.23" -solana-clap-utils = { path = "../clap-utils", version = "=1.10.41" } -solana-cli-config = { path = "../cli-config", version = "=1.10.41" } -solana-client = { path = "../client", version = "=1.10.41" } -solana-core = { path = "../core", version = "=1.10.41" } -solana-faucet = { path = "../faucet", version = "=1.10.41" } -solana-genesis = { path = "../genesis", version = "=1.10.41" } -solana-gossip = { path = "../gossip", version = "=1.10.41" } -solana-logger = { path = "../logger", version = "=1.10.41" } -solana-measure = { path = "../measure", version = "=1.10.41" } -solana-metrics = { path = "../metrics", version = "=1.10.41" } -solana-net-utils = { path = "../net-utils", version = "=1.10.41" } -solana-rpc = { path = "../rpc", version = "=1.10.41" } -solana-runtime = { path = "../runtime", version = "=1.10.41" } -solana-sdk = { path = "../sdk", version = "=1.10.41" } -solana-streamer = { path = "../streamer", version = "=1.10.41" } -solana-version = { path = "../version", version = "=0.7.0" } -thiserror = "1.0" - -[dev-dependencies] -serial_test = "0.6.0" -solana-local-cluster = { path = "../local-cluster", version = "=1.10.41" } -solana-test-validator = { path = "../test-validator", version = "=1.10.41" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/bench-tps/src/bench.rs b/bench-tps/src/bench.rs deleted file mode 100644 index 5a2143908c..0000000000 --- a/bench-tps/src/bench.rs +++ /dev/null @@ -1,986 +0,0 @@ -use { - crate::{ - bench_tps_client::*, - cli::Config, - perf_utils::{sample_txs, SampleStats}, - }, - log::*, - rayon::prelude::*, - solana_core::gen_keys::GenKeys, - solana_measure::measure::Measure, - solana_metrics::{self, datapoint_info}, - solana_sdk::{ - clock::{DEFAULT_MS_PER_SLOT, DEFAULT_S_PER_SLOT, MAX_PROCESSING_AGE}, - commitment_config::CommitmentConfig, - hash::Hash, - instruction::{AccountMeta, Instruction}, - message::Message, - native_token::Sol, - pubkey::Pubkey, - signature::{Keypair, Signer}, - system_instruction, system_transaction, - timing::{duration_as_ms, duration_as_s, duration_as_us, timestamp}, - transaction::Transaction, - }, - std::{ - collections::{HashSet, VecDeque}, - process::exit, - sync::{ - atomic::{AtomicBool, AtomicIsize, AtomicUsize, Ordering}, - Arc, Mutex, RwLock, - }, - thread::{sleep, Builder, JoinHandle}, - time::{Duration, Instant}, - }, -}; - -// The point at which transactions become "too old", in seconds. -const MAX_TX_QUEUE_AGE: u64 = (MAX_PROCESSING_AGE as f64 * DEFAULT_S_PER_SLOT) as u64; - -pub const MAX_SPENDS_PER_TX: u64 = 4; - -pub type SharedTransactions = Arc>>>; - -fn get_latest_blockhash(client: &T) -> Hash { - loop { - match client.get_latest_blockhash_with_commitment(CommitmentConfig::processed()) { - Ok((blockhash, _)) => return blockhash, - Err(err) => { - info!("Couldn't get last blockhash: {:?}", err); - sleep(Duration::from_secs(1)); - } - }; - } -} - -fn wait_for_target_slots_per_epoch(target_slots_per_epoch: u64, client: &Arc) -where - T: 'static + BenchTpsClient + Send + Sync, -{ - if target_slots_per_epoch != 0 { - info!( - "Waiting until epochs are {} slots long..", - target_slots_per_epoch - ); - loop { - if let Ok(epoch_info) = client.get_epoch_info() { - if epoch_info.slots_in_epoch >= target_slots_per_epoch { - info!("Done epoch_info: {:?}", epoch_info); - break; - } - info!( - "Waiting for epoch: {} now: {}", - target_slots_per_epoch, epoch_info.slots_in_epoch - ); - } - sleep(Duration::from_secs(3)); - } - } -} - -fn create_sampler_thread( - client: &Arc, - exit_signal: &Arc, - sample_period: u64, - maxes: &Arc>>, -) -> JoinHandle<()> -where - T: 'static + BenchTpsClient + Send + Sync, -{ - info!("Sampling TPS every {} second...", sample_period); - let exit_signal = exit_signal.clone(); - let maxes = maxes.clone(); - let client = client.clone(); - Builder::new() - .name("solana-client-sample".to_string()) - .spawn(move || { - sample_txs(&exit_signal, &maxes, sample_period, &client); - }) - .unwrap() -} - -#[allow(clippy::ptr_arg)] -fn generate_chunked_transfers( - recent_blockhash: Arc>, - shared_txs: &SharedTransactions, - shared_tx_active_thread_count: Arc, - source_keypair_chunks: Vec>, - dest_keypair_chunks: &mut [VecDeque<&Keypair>], - threads: usize, - duration: Duration, - sustained: bool, -) { - // generate and send transactions for the specified duration - let start = Instant::now(); - let keypair_chunks = source_keypair_chunks.len(); - let mut reclaim_lamports_back_to_source_account = false; - let mut chunk_index = 0; - while start.elapsed() < duration { - generate_txs( - shared_txs, - &recent_blockhash, - &source_keypair_chunks[chunk_index], - &dest_keypair_chunks[chunk_index], - threads, - reclaim_lamports_back_to_source_account, - ); - - // In sustained mode, overlap the transfers with generation. This has higher average - // performance but lower peak performance in tested environments. - if sustained { - // Ensure that we don't generate more transactions than we can handle. - while shared_txs.read().unwrap().len() > 2 * threads { - sleep(Duration::from_millis(1)); - } - } else { - while !shared_txs.read().unwrap().is_empty() - || shared_tx_active_thread_count.load(Ordering::Relaxed) > 0 - { - sleep(Duration::from_millis(1)); - } - } - - // Rotate destination keypairs so that the next round of transactions will have different - // transaction signatures even when blockhash is reused. - dest_keypair_chunks[chunk_index].rotate_left(1); - - // Move on to next chunk - chunk_index = (chunk_index + 1) % keypair_chunks; - - // Switch directions after transfering for each "chunk" - if chunk_index == 0 { - reclaim_lamports_back_to_source_account = !reclaim_lamports_back_to_source_account; - } - } -} - -fn create_sender_threads( - client: &Arc, - shared_txs: &SharedTransactions, - thread_batch_sleep_ms: usize, - total_tx_sent_count: &Arc, - threads: usize, - exit_signal: &Arc, - shared_tx_active_thread_count: &Arc, -) -> Vec> -where - T: 'static + BenchTpsClient + Send + Sync, -{ - (0..threads) - .map(|_| { - let exit_signal = exit_signal.clone(); - let shared_txs = shared_txs.clone(); - let shared_tx_active_thread_count = shared_tx_active_thread_count.clone(); - let total_tx_sent_count = total_tx_sent_count.clone(); - let client = client.clone(); - Builder::new() - .name("solana-client-sender".to_string()) - .spawn(move || { - do_tx_transfers( - &exit_signal, - &shared_txs, - &shared_tx_active_thread_count, - &total_tx_sent_count, - thread_batch_sleep_ms, - &client, - ); - }) - .unwrap() - }) - .collect() -} - -pub fn do_bench_tps(client: Arc, config: Config, gen_keypairs: Vec) -> u64 -where - T: 'static + BenchTpsClient + Send + Sync, -{ - let Config { - id, - threads, - thread_batch_sleep_ms, - duration, - tx_count, - sustained, - target_slots_per_epoch, - .. - } = config; - - let mut source_keypair_chunks: Vec> = Vec::new(); - let mut dest_keypair_chunks: Vec> = Vec::new(); - assert!(gen_keypairs.len() >= 2 * tx_count); - for chunk in gen_keypairs.chunks_exact(2 * tx_count) { - source_keypair_chunks.push(chunk[..tx_count].iter().collect()); - dest_keypair_chunks.push(chunk[tx_count..].iter().collect()); - } - - let first_tx_count = loop { - match client.get_transaction_count() { - Ok(count) => break count, - Err(err) => { - info!("Couldn't get transaction count: {:?}", err); - sleep(Duration::from_secs(1)); - } - } - }; - info!("Initial transaction count {}", first_tx_count); - - let exit_signal = Arc::new(AtomicBool::new(false)); - - // Setup a thread per validator to sample every period - // collect the max transaction rate and total tx count seen - let maxes = Arc::new(RwLock::new(Vec::new())); - let sample_period = 1; // in seconds - let sample_thread = create_sampler_thread(&client, &exit_signal, sample_period, &maxes); - - let shared_txs: SharedTransactions = Arc::new(RwLock::new(VecDeque::new())); - - let blockhash = Arc::new(RwLock::new(get_latest_blockhash(client.as_ref()))); - let shared_tx_active_thread_count = Arc::new(AtomicIsize::new(0)); - let total_tx_sent_count = Arc::new(AtomicUsize::new(0)); - - let blockhash_thread = { - let exit_signal = exit_signal.clone(); - let blockhash = blockhash.clone(); - let client = client.clone(); - let id = id.pubkey(); - Builder::new() - .name("solana-blockhash-poller".to_string()) - .spawn(move || { - poll_blockhash(&exit_signal, &blockhash, &client, &id); - }) - .unwrap() - }; - - let s_threads = create_sender_threads( - &client, - &shared_txs, - thread_batch_sleep_ms, - &total_tx_sent_count, - threads, - &exit_signal, - &shared_tx_active_thread_count, - ); - - wait_for_target_slots_per_epoch(target_slots_per_epoch, &client); - - let start = Instant::now(); - - generate_chunked_transfers( - blockhash, - &shared_txs, - shared_tx_active_thread_count, - source_keypair_chunks, - &mut dest_keypair_chunks, - threads, - duration, - sustained, - ); - - // Stop the sampling threads so it will collect the stats - exit_signal.store(true, Ordering::Relaxed); - - info!("Waiting for sampler threads..."); - if let Err(err) = sample_thread.join() { - info!(" join() failed with: {:?}", err); - } - - // join the tx send threads - info!("Waiting for transmit threads..."); - for t in s_threads { - if let Err(err) = t.join() { - info!(" join() failed with: {:?}", err); - } - } - - info!("Waiting for blockhash thread..."); - if let Err(err) = blockhash_thread.join() { - info!(" join() failed with: {:?}", err); - } - - let balance = client.get_balance(&id.pubkey()).unwrap_or(0); - metrics_submit_lamport_balance(balance); - - compute_and_report_stats( - &maxes, - sample_period, - &start.elapsed(), - total_tx_sent_count.load(Ordering::Relaxed), - ); - - let r_maxes = maxes.read().unwrap(); - r_maxes.first().unwrap().1.txs -} - -fn metrics_submit_lamport_balance(lamport_balance: u64) { - info!("Token balance: {}", lamport_balance); - datapoint_info!( - "bench-tps-lamport_balance", - ("balance", lamport_balance, i64) - ); -} - -fn generate_system_txs( - source: &[&Keypair], - dest: &VecDeque<&Keypair>, - reclaim: bool, - blockhash: &Hash, -) -> Vec<(Transaction, u64)> { - let pairs: Vec<_> = if !reclaim { - source.iter().zip(dest.iter()).collect() - } else { - dest.iter().zip(source.iter()).collect() - }; - - pairs - .par_iter() - .map(|(from, to)| { - ( - system_transaction::transfer(from, &to.pubkey(), 1, *blockhash), - timestamp(), - ) - }) - .collect() -} - -fn generate_txs( - shared_txs: &SharedTransactions, - blockhash: &Arc>, - source: &[&Keypair], - dest: &VecDeque<&Keypair>, - threads: usize, - reclaim: bool, -) { - let blockhash = *blockhash.read().unwrap(); - let tx_count = source.len(); - info!( - "Signing transactions... {} (reclaim={}, blockhash={})", - tx_count, reclaim, &blockhash - ); - let signing_start = Instant::now(); - - let transactions = generate_system_txs(source, dest, reclaim, &blockhash); - - let duration = signing_start.elapsed(); - let ns = duration.as_secs() * 1_000_000_000 + u64::from(duration.subsec_nanos()); - let bsps = (tx_count) as f64 / ns as f64; - let nsps = ns as f64 / (tx_count) as f64; - info!( - "Done. {:.2} thousand signatures per second, {:.2} us per signature, {} ms total time, {}", - bsps * 1_000_000_f64, - nsps / 1_000_f64, - duration_as_ms(&duration), - blockhash, - ); - datapoint_info!( - "bench-tps-generate_txs", - ("duration", duration_as_us(&duration), i64) - ); - - let sz = transactions.len() / threads; - let chunks: Vec<_> = transactions.chunks(sz).collect(); - { - let mut shared_txs_wl = shared_txs.write().unwrap(); - for chunk in chunks { - shared_txs_wl.push_back(chunk.to_vec()); - } - } -} - -fn get_new_latest_blockhash(client: &Arc, blockhash: &Hash) -> Option { - let start = Instant::now(); - while start.elapsed().as_secs() < 5 { - if let Ok(new_blockhash) = client.get_latest_blockhash() { - if new_blockhash != *blockhash { - return Some(new_blockhash); - } - } - debug!("Got same blockhash ({:?}), will retry...", blockhash); - - // Retry ~twice during a slot - sleep(Duration::from_millis(DEFAULT_MS_PER_SLOT / 2)); - } - None -} - -fn poll_blockhash( - exit_signal: &Arc, - blockhash: &Arc>, - client: &Arc, - id: &Pubkey, -) { - let mut blockhash_last_updated = Instant::now(); - let mut last_error_log = Instant::now(); - loop { - let blockhash_updated = { - let old_blockhash = *blockhash.read().unwrap(); - if let Some(new_blockhash) = get_new_latest_blockhash(client, &old_blockhash) { - *blockhash.write().unwrap() = new_blockhash; - blockhash_last_updated = Instant::now(); - true - } else { - if blockhash_last_updated.elapsed().as_secs() > 120 { - eprintln!("Blockhash is stuck"); - exit(1) - } else if blockhash_last_updated.elapsed().as_secs() > 30 - && last_error_log.elapsed().as_secs() >= 1 - { - last_error_log = Instant::now(); - error!("Blockhash is not updating"); - } - false - } - }; - - if blockhash_updated { - let balance = client.get_balance(id).unwrap_or(0); - metrics_submit_lamport_balance(balance); - } - - if exit_signal.load(Ordering::Relaxed) { - break; - } - - sleep(Duration::from_millis(50)); - } -} - -fn do_tx_transfers( - exit_signal: &Arc, - shared_txs: &SharedTransactions, - shared_tx_thread_count: &Arc, - total_tx_sent_count: &Arc, - thread_batch_sleep_ms: usize, - client: &Arc, -) { - loop { - if thread_batch_sleep_ms > 0 { - sleep(Duration::from_millis(thread_batch_sleep_ms as u64)); - } - let txs = { - let mut shared_txs_wl = shared_txs.write().expect("write lock in do_tx_transfers"); - shared_txs_wl.pop_front() - }; - if let Some(txs0) = txs { - shared_tx_thread_count.fetch_add(1, Ordering::Relaxed); - info!("Transferring 1 unit {} times...", txs0.len()); - let tx_len = txs0.len(); - let transfer_start = Instant::now(); - let mut old_transactions = false; - let mut transactions = Vec::<_>::new(); - for tx in txs0 { - let now = timestamp(); - // Transactions that are too old will be rejected by the cluster Don't bother - // sending them. - if now > tx.1 && now - tx.1 > 1000 * MAX_TX_QUEUE_AGE { - old_transactions = true; - continue; - } - transactions.push(tx.0); - } - - if let Err(error) = client.send_batch(transactions) { - warn!("send_batch_sync in do_tx_transfers failed: {}", error); - } - - if old_transactions { - let mut shared_txs_wl = shared_txs.write().expect("write lock in do_tx_transfers"); - shared_txs_wl.clear(); - } - shared_tx_thread_count.fetch_add(-1, Ordering::Relaxed); - total_tx_sent_count.fetch_add(tx_len, Ordering::Relaxed); - info!( - "Tx send done. {} ms {} tps", - duration_as_ms(&transfer_start.elapsed()), - tx_len as f32 / duration_as_s(&transfer_start.elapsed()), - ); - datapoint_info!( - "bench-tps-do_tx_transfers", - ("duration", duration_as_us(&transfer_start.elapsed()), i64), - ("count", tx_len, i64) - ); - } - if exit_signal.load(Ordering::Relaxed) { - break; - } - } -} - -fn verify_funding_transfer( - client: &Arc, - tx: &Transaction, - amount: u64, -) -> bool { - for a in &tx.message().account_keys[1..] { - match client.get_balance_with_commitment(a, CommitmentConfig::processed()) { - Ok(balance) => return balance >= amount, - Err(err) => error!("failed to get balance {:?}", err), - } - } - false -} - -trait FundingTransactions<'a> { - fn fund( - &mut self, - client: &Arc, - to_fund: &[(&'a Keypair, Vec<(Pubkey, u64)>)], - to_lamports: u64, - ); - fn make(&mut self, to_fund: &[(&'a Keypair, Vec<(Pubkey, u64)>)]); - fn sign(&mut self, blockhash: Hash); - fn send(&self, client: &Arc); - fn verify( - &mut self, - client: &Arc, - to_lamports: u64, - ); -} - -impl<'a> FundingTransactions<'a> for Vec<(&'a Keypair, Transaction)> { - fn fund( - &mut self, - client: &Arc, - to_fund: &[(&'a Keypair, Vec<(Pubkey, u64)>)], - to_lamports: u64, - ) { - self.make(to_fund); - - let mut tries = 0; - while !self.is_empty() { - info!( - "{} {} each to {} accounts in {} txs", - if tries == 0 { - "transferring" - } else { - " retrying" - }, - to_lamports, - self.len() * MAX_SPENDS_PER_TX as usize, - self.len(), - ); - - let blockhash = get_latest_blockhash(client.as_ref()); - - // re-sign retained to_fund_txes with updated blockhash - self.sign(blockhash); - self.send(client); - - // Sleep a few slots to allow transactions to process - sleep(Duration::from_secs(1)); - - self.verify(client, to_lamports); - - // retry anything that seems to have dropped through cracks - // again since these txs are all or nothing, they're fine to - // retry - tries += 1; - } - info!("transferred"); - } - - fn make(&mut self, to_fund: &[(&'a Keypair, Vec<(Pubkey, u64)>)]) { - let mut make_txs = Measure::start("make_txs"); - let to_fund_txs: Vec<(&Keypair, Transaction)> = to_fund - .par_iter() - .map(|(k, t)| { - let instructions = system_instruction::transfer_many(&k.pubkey(), t); - let message = Message::new(&instructions, Some(&k.pubkey())); - (*k, Transaction::new_unsigned(message)) - }) - .collect(); - make_txs.stop(); - debug!( - "make {} unsigned txs: {}us", - to_fund_txs.len(), - make_txs.as_us() - ); - self.extend(to_fund_txs); - } - - fn sign(&mut self, blockhash: Hash) { - let mut sign_txs = Measure::start("sign_txs"); - self.par_iter_mut().for_each(|(k, tx)| { - tx.sign(&[*k], blockhash); - }); - sign_txs.stop(); - debug!("sign {} txs: {}us", self.len(), sign_txs.as_us()); - } - - fn send(&self, client: &Arc) { - let mut send_txs = Measure::start("send_and_clone_txs"); - let batch: Vec<_> = self.iter().map(|(_keypair, tx)| tx.clone()).collect(); - client.send_batch(batch).expect("transfer"); - send_txs.stop(); - debug!("send {} {}", self.len(), send_txs); - } - - fn verify( - &mut self, - client: &Arc, - to_lamports: u64, - ) { - let starting_txs = self.len(); - let verified_txs = Arc::new(AtomicUsize::new(0)); - let too_many_failures = Arc::new(AtomicBool::new(false)); - let loops = if starting_txs < 1000 { 3 } else { 1 }; - // Only loop multiple times for small (quick) transaction batches - let time = Arc::new(Mutex::new(Instant::now())); - for _ in 0..loops { - let time = time.clone(); - let failed_verify = Arc::new(AtomicUsize::new(0)); - let client = client.clone(); - let verified_txs = &verified_txs; - let failed_verify = &failed_verify; - let too_many_failures = &too_many_failures; - let verified_set: HashSet = self - .par_iter() - .filter_map(move |(k, tx)| { - if too_many_failures.load(Ordering::Relaxed) { - return None; - } - - let verified = if verify_funding_transfer(&client, tx, to_lamports) { - verified_txs.fetch_add(1, Ordering::Relaxed); - Some(k.pubkey()) - } else { - failed_verify.fetch_add(1, Ordering::Relaxed); - None - }; - - let verified_txs = verified_txs.load(Ordering::Relaxed); - let failed_verify = failed_verify.load(Ordering::Relaxed); - let remaining_count = starting_txs.saturating_sub(verified_txs + failed_verify); - if failed_verify > 100 && failed_verify > verified_txs { - too_many_failures.store(true, Ordering::Relaxed); - warn!( - "Too many failed transfers... {} remaining, {} verified, {} failures", - remaining_count, verified_txs, failed_verify - ); - } - if remaining_count > 0 { - let mut time_l = time.lock().unwrap(); - if time_l.elapsed().as_secs() > 2 { - info!( - "Verifying transfers... {} remaining, {} verified, {} failures", - remaining_count, verified_txs, failed_verify - ); - *time_l = Instant::now(); - } - } - - verified - }) - .collect(); - - self.retain(|(k, _)| !verified_set.contains(&k.pubkey())); - if self.is_empty() { - break; - } - info!("Looping verifications"); - - let verified_txs = verified_txs.load(Ordering::Relaxed); - let failed_verify = failed_verify.load(Ordering::Relaxed); - let remaining_count = starting_txs.saturating_sub(verified_txs + failed_verify); - info!( - "Verifying transfers... {} remaining, {} verified, {} failures", - remaining_count, verified_txs, failed_verify - ); - sleep(Duration::from_millis(100)); - } - } -} - -/// fund the dests keys by spending all of the source keys into MAX_SPENDS_PER_TX -/// on every iteration. This allows us to replay the transfers because the source is either empty, -/// or full -pub fn fund_keys( - client: Arc, - source: &Keypair, - dests: &[Keypair], - total: u64, - max_fee: u64, - lamports_per_account: u64, -) { - let mut funded: Vec<&Keypair> = vec![source]; - let mut funded_funds = total; - let mut not_funded: Vec<&Keypair> = dests.iter().collect(); - while !not_funded.is_empty() { - // Build to fund list and prepare funding sources for next iteration - let mut new_funded: Vec<&Keypair> = vec![]; - let mut to_fund: Vec<(&Keypair, Vec<(Pubkey, u64)>)> = vec![]; - let to_lamports = (funded_funds - lamports_per_account - max_fee) / MAX_SPENDS_PER_TX; - for f in funded { - let start = not_funded.len() - MAX_SPENDS_PER_TX as usize; - let dests: Vec<_> = not_funded.drain(start..).collect(); - let spends: Vec<_> = dests.iter().map(|k| (k.pubkey(), to_lamports)).collect(); - to_fund.push((f, spends)); - new_funded.extend(dests.into_iter()); - } - - // try to transfer a "few" at a time with recent blockhash - // assume 4MB network buffers, and 512 byte packets - const FUND_CHUNK_LEN: usize = 4 * 1024 * 1024 / 512; - - to_fund.chunks(FUND_CHUNK_LEN).for_each(|chunk| { - Vec::<(&Keypair, Transaction)>::with_capacity(chunk.len()).fund( - &client, - chunk, - to_lamports, - ); - }); - - info!("funded: {} left: {}", new_funded.len(), not_funded.len()); - funded = new_funded; - funded_funds = to_lamports; - } -} - -fn compute_and_report_stats( - maxes: &Arc>>, - sample_period: u64, - tx_send_elapsed: &Duration, - total_tx_send_count: usize, -) { - // Compute/report stats - let mut max_of_maxes = 0.0; - let mut max_tx_count = 0; - let mut nodes_with_zero_tps = 0; - let mut total_maxes = 0.0; - info!(" Node address | Max TPS | Total Transactions"); - info!("---------------------+---------------+--------------------"); - - for (sock, stats) in maxes.read().unwrap().iter() { - let maybe_flag = match stats.txs { - 0 => "!!!!!", - _ => "", - }; - - info!( - "{:20} | {:13.2} | {} {}", - sock, stats.tps, stats.txs, maybe_flag - ); - - if stats.tps == 0.0 { - nodes_with_zero_tps += 1; - } - total_maxes += stats.tps; - - if stats.tps > max_of_maxes { - max_of_maxes = stats.tps; - } - if stats.txs > max_tx_count { - max_tx_count = stats.txs; - } - } - - if total_maxes > 0.0 { - let num_nodes_with_tps = maxes.read().unwrap().len() - nodes_with_zero_tps; - let average_max = total_maxes / num_nodes_with_tps as f32; - info!( - "\nAverage max TPS: {:.2}, {} nodes had 0 TPS", - average_max, nodes_with_zero_tps - ); - } - - let total_tx_send_count = total_tx_send_count as u64; - let drop_rate = if total_tx_send_count > max_tx_count { - (total_tx_send_count - max_tx_count) as f64 / total_tx_send_count as f64 - } else { - 0.0 - }; - info!( - "\nHighest TPS: {:.2} sampling period {}s max transactions: {} clients: {} drop rate: {:.2}", - max_of_maxes, - sample_period, - max_tx_count, - maxes.read().unwrap().len(), - drop_rate, - ); - info!( - "\tAverage TPS: {}", - max_tx_count as f32 / duration_as_s(tx_send_elapsed) - ); -} - -pub fn generate_keypairs(seed_keypair: &Keypair, count: u64) -> (Vec, u64) { - let mut seed = [0u8; 32]; - seed.copy_from_slice(&seed_keypair.to_bytes()[..32]); - let mut rnd = GenKeys::new(seed); - - let mut total_keys = 0; - let mut extra = 0; // This variable tracks the number of keypairs needing extra transaction fees funded - let mut delta = 1; - while total_keys < count { - extra += delta; - delta *= MAX_SPENDS_PER_TX; - total_keys += delta; - } - (rnd.gen_n_keypairs(total_keys), extra) -} - -pub fn generate_and_fund_keypairs( - client: Arc, - funding_key: &Keypair, - keypair_count: usize, - lamports_per_account: u64, -) -> Result> { - let rent = client.get_minimum_balance_for_rent_exemption(0)?; - let lamports_per_account = lamports_per_account + rent; - - info!("Creating {} keypairs...", keypair_count); - let (mut keypairs, extra) = generate_keypairs(funding_key, keypair_count as u64); - fund_keypairs(client, funding_key, &keypairs, extra, lamports_per_account)?; - - // 'generate_keypairs' generates extra keys to be able to have size-aligned funding batches for fund_keys. - keypairs.truncate(keypair_count); - - Ok(keypairs) -} - -pub fn fund_keypairs( - client: Arc, - funding_key: &Keypair, - keypairs: &[Keypair], - extra: u64, - lamports_per_account: u64, -) -> Result<()> { - let rent = client.get_minimum_balance_for_rent_exemption(0)?; - info!("Get lamports..."); - - // Sample the first keypair, to prevent lamport loss on repeated solana-bench-tps executions - let first_key = keypairs[0].pubkey(); - let first_keypair_balance = client.get_balance(&first_key).unwrap_or(0); - - // Sample the last keypair, to check if funding was already completed - let last_key = keypairs[keypairs.len() - 1].pubkey(); - let last_keypair_balance = client.get_balance(&last_key).unwrap_or(0); - - // Repeated runs will eat up keypair balances from transaction fees. In order to quickly - // start another bench-tps run without re-funding all of the keypairs, check if the - // keypairs still have at least 80% of the expected funds. That should be enough to - // pay for the transaction fees in a new run. - let enough_lamports = 8 * lamports_per_account / 10; - if first_keypair_balance < enough_lamports || last_keypair_balance < enough_lamports { - let single_sig_message = Message::new_with_blockhash( - &[Instruction::new_with_bytes( - Pubkey::new_unique(), - &[], - vec![AccountMeta::new(Pubkey::new_unique(), true)], - )], - None, - &client.get_latest_blockhash().unwrap(), - ); - let max_fee = client.get_fee_for_message(&single_sig_message).unwrap(); - let extra_fees = extra * max_fee; - let total_keypairs = keypairs.len() as u64 + 1; // Add one for funding keypair - let total = lamports_per_account * total_keypairs + extra_fees; - - let funding_key_balance = client.get_balance(&funding_key.pubkey()).unwrap_or(0); - info!( - "Funding keypair balance: {} max_fee: {} lamports_per_account: {} extra: {} total: {}", - funding_key_balance, max_fee, lamports_per_account, extra, total - ); - - if funding_key_balance < total + rent { - error!( - "funder has {}, needed {}", - Sol(funding_key_balance), - Sol(total) - ); - let latest_blockhash = get_latest_blockhash(client.as_ref()); - if client - .request_airdrop_with_blockhash( - &funding_key.pubkey(), - total + rent - funding_key_balance, - &latest_blockhash, - ) - .is_err() - { - return Err(BenchTpsError::AirdropFailure); - } - } - - fund_keys( - client, - funding_key, - keypairs, - total, - max_fee, - lamports_per_account, - ); - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use { - super::*, - solana_runtime::{bank::Bank, bank_client::BankClient}, - solana_sdk::{ - fee_calculator::FeeRateGovernor, genesis_config::create_genesis_config, - native_token::sol_to_lamports, - }, - }; - - #[test] - fn test_bench_tps_bank_client() { - let (genesis_config, id) = create_genesis_config(sol_to_lamports(10_000.0)); - let bank = Bank::new_for_tests(&genesis_config); - let client = Arc::new(BankClient::new(bank)); - - let config = Config { - id, - tx_count: 10, - duration: Duration::from_secs(5), - ..Config::default() - }; - - let keypair_count = config.tx_count * config.keypair_multiplier; - let keypairs = - generate_and_fund_keypairs(client.clone(), &config.id, keypair_count, 20).unwrap(); - - do_bench_tps(client, config, keypairs); - } - - #[test] - fn test_bench_tps_fund_keys() { - let (genesis_config, id) = create_genesis_config(sol_to_lamports(10_000.0)); - let bank = Bank::new_for_tests(&genesis_config); - let client = Arc::new(BankClient::new(bank)); - let keypair_count = 20; - let lamports = 20; - let rent = client.get_minimum_balance_for_rent_exemption(0).unwrap(); - - let keypairs = - generate_and_fund_keypairs(client.clone(), &id, keypair_count, lamports).unwrap(); - - for kp in &keypairs { - assert_eq!( - client - .get_balance_with_commitment(&kp.pubkey(), CommitmentConfig::processed()) - .unwrap(), - lamports + rent - ); - } - } - - #[test] - fn test_bench_tps_fund_keys_with_fees() { - let (mut genesis_config, id) = create_genesis_config(sol_to_lamports(10_000.0)); - let fee_rate_governor = FeeRateGovernor::new(11, 0); - genesis_config.fee_rate_governor = fee_rate_governor; - let bank = Bank::new_for_tests(&genesis_config); - let client = Arc::new(BankClient::new(bank)); - let keypair_count = 20; - let lamports = 20; - let rent = client.get_minimum_balance_for_rent_exemption(0).unwrap(); - - let keypairs = - generate_and_fund_keypairs(client.clone(), &id, keypair_count, lamports).unwrap(); - - for kp in &keypairs { - assert_eq!(client.get_balance(&kp.pubkey()).unwrap(), lamports + rent); - } - } -} diff --git a/bench-tps/src/bench_tps_client.rs b/bench-tps/src/bench_tps_client.rs deleted file mode 100644 index 1a4743cf35..0000000000 --- a/bench-tps/src/bench_tps_client.rs +++ /dev/null @@ -1,87 +0,0 @@ -use { - solana_client::{client_error::ClientError, tpu_client::TpuSenderError}, - solana_sdk::{ - commitment_config::CommitmentConfig, epoch_info::EpochInfo, hash::Hash, message::Message, - pubkey::Pubkey, signature::Signature, transaction::Transaction, transport::TransportError, - }, - thiserror::Error, -}; - -#[derive(Error, Debug)] -pub enum BenchTpsError { - #[error("Airdrop failure")] - AirdropFailure, - #[error("IO error: {0:?}")] - IoError(#[from] std::io::Error), - #[error("Client error: {0:?}")] - ClientError(#[from] ClientError), - #[error("TpuClient error: {0:?}")] - TpuSenderError(#[from] TpuSenderError), - #[error("Transport error: {0:?}")] - TransportError(#[from] TransportError), - #[error("Custom error: {0}")] - Custom(String), -} - -pub(crate) type Result = std::result::Result; - -pub trait BenchTpsClient { - /// Send a signed transaction without confirmation - fn send_transaction(&self, transaction: Transaction) -> Result; - - /// Send a batch of signed transactions without confirmation. - fn send_batch(&self, transactions: Vec) -> Result<()>; - - /// Get latest blockhash - fn get_latest_blockhash(&self) -> Result; - - /// Get latest blockhash and its last valid block height, using explicit commitment - fn get_latest_blockhash_with_commitment( - &self, - commitment_config: CommitmentConfig, - ) -> Result<(Hash, u64)>; - - /// Get transaction count - fn get_transaction_count(&self) -> Result; - - /// Get transaction count, using explicit commitment - fn get_transaction_count_with_commitment( - &self, - commitment_config: CommitmentConfig, - ) -> Result; - - /// Get epoch info - fn get_epoch_info(&self) -> Result; - - /// Get account balance - fn get_balance(&self, pubkey: &Pubkey) -> Result; - - /// Get account balance, using explicit commitment - fn get_balance_with_commitment( - &self, - pubkey: &Pubkey, - commitment_config: CommitmentConfig, - ) -> Result; - - /// Calculate the fee for a `Message` - fn get_fee_for_message(&self, message: &Message) -> Result; - - /// Get the rent-exempt minimum for an account - fn get_minimum_balance_for_rent_exemption(&self, data_len: usize) -> Result; - - /// Return the address of client - fn addr(&self) -> String; - - /// Request, submit, and confirm an airdrop transaction - fn request_airdrop_with_blockhash( - &self, - pubkey: &Pubkey, - lamports: u64, - recent_blockhash: &Hash, - ) -> Result; -} - -mod bank_client; -mod rpc_client; -mod thin_client; -mod tpu_client; diff --git a/bench-tps/src/bench_tps_client/bank_client.rs b/bench-tps/src/bench_tps_client/bank_client.rs deleted file mode 100644 index 2653fce311..0000000000 --- a/bench-tps/src/bench_tps_client/bank_client.rs +++ /dev/null @@ -1,85 +0,0 @@ -use { - crate::bench_tps_client::{BenchTpsClient, BenchTpsError, Result}, - solana_runtime::bank_client::BankClient, - solana_sdk::{ - client::{AsyncClient, SyncClient}, - commitment_config::CommitmentConfig, - epoch_info::EpochInfo, - hash::Hash, - message::Message, - pubkey::Pubkey, - signature::Signature, - transaction::Transaction, - }, -}; - -impl BenchTpsClient for BankClient { - fn send_transaction(&self, transaction: Transaction) -> Result { - AsyncClient::async_send_transaction(self, transaction).map_err(|err| err.into()) - } - fn send_batch(&self, transactions: Vec) -> Result<()> { - AsyncClient::async_send_batch(self, transactions).map_err(|err| err.into()) - } - fn get_latest_blockhash(&self) -> Result { - SyncClient::get_latest_blockhash(self).map_err(|err| err.into()) - } - - fn get_latest_blockhash_with_commitment( - &self, - commitment_config: CommitmentConfig, - ) -> Result<(Hash, u64)> { - SyncClient::get_latest_blockhash_with_commitment(self, commitment_config) - .map_err(|err| err.into()) - } - - fn get_transaction_count(&self) -> Result { - SyncClient::get_transaction_count(self).map_err(|err| err.into()) - } - - fn get_transaction_count_with_commitment( - &self, - commitment_config: CommitmentConfig, - ) -> Result { - SyncClient::get_transaction_count_with_commitment(self, commitment_config) - .map_err(|err| err.into()) - } - - fn get_epoch_info(&self) -> Result { - SyncClient::get_epoch_info(self).map_err(|err| err.into()) - } - - fn get_balance(&self, pubkey: &Pubkey) -> Result { - SyncClient::get_balance(self, pubkey).map_err(|err| err.into()) - } - - fn get_balance_with_commitment( - &self, - pubkey: &Pubkey, - commitment_config: CommitmentConfig, - ) -> Result { - SyncClient::get_balance_with_commitment(self, pubkey, commitment_config) - .map_err(|err| err.into()) - } - - fn get_fee_for_message(&self, message: &Message) -> Result { - SyncClient::get_fee_for_message(self, message).map_err(|err| err.into()) - } - - fn get_minimum_balance_for_rent_exemption(&self, data_len: usize) -> Result { - SyncClient::get_minimum_balance_for_rent_exemption(self, data_len).map_err(|err| err.into()) - } - - fn addr(&self) -> String { - "Local BankClient".to_string() - } - - fn request_airdrop_with_blockhash( - &self, - _pubkey: &Pubkey, - _lamports: u64, - _recent_blockhash: &Hash, - ) -> Result { - // BankClient doesn't support airdrops - Err(BenchTpsError::AirdropFailure) - } -} diff --git a/bench-tps/src/bench_tps_client/rpc_client.rs b/bench-tps/src/bench_tps_client/rpc_client.rs deleted file mode 100644 index 4eb4685a3e..0000000000 --- a/bench-tps/src/bench_tps_client/rpc_client.rs +++ /dev/null @@ -1,83 +0,0 @@ -use { - crate::bench_tps_client::{BenchTpsClient, Result}, - solana_client::rpc_client::RpcClient, - solana_sdk::{ - commitment_config::CommitmentConfig, epoch_info::EpochInfo, hash::Hash, message::Message, - pubkey::Pubkey, signature::Signature, transaction::Transaction, - }, -}; - -impl BenchTpsClient for RpcClient { - fn send_transaction(&self, transaction: Transaction) -> Result { - RpcClient::send_transaction(self, &transaction).map_err(|err| err.into()) - } - fn send_batch(&self, transactions: Vec) -> Result<()> { - for transaction in transactions { - BenchTpsClient::send_transaction(self, transaction)?; - } - Ok(()) - } - fn get_latest_blockhash(&self) -> Result { - RpcClient::get_latest_blockhash(self).map_err(|err| err.into()) - } - - fn get_latest_blockhash_with_commitment( - &self, - commitment_config: CommitmentConfig, - ) -> Result<(Hash, u64)> { - RpcClient::get_latest_blockhash_with_commitment(self, commitment_config) - .map_err(|err| err.into()) - } - - fn get_transaction_count(&self) -> Result { - RpcClient::get_transaction_count(self).map_err(|err| err.into()) - } - - fn get_transaction_count_with_commitment( - &self, - commitment_config: CommitmentConfig, - ) -> Result { - RpcClient::get_transaction_count_with_commitment(self, commitment_config) - .map_err(|err| err.into()) - } - - fn get_epoch_info(&self) -> Result { - RpcClient::get_epoch_info(self).map_err(|err| err.into()) - } - - fn get_balance(&self, pubkey: &Pubkey) -> Result { - RpcClient::get_balance(self, pubkey).map_err(|err| err.into()) - } - - fn get_balance_with_commitment( - &self, - pubkey: &Pubkey, - commitment_config: CommitmentConfig, - ) -> Result { - RpcClient::get_balance_with_commitment(self, pubkey, commitment_config) - .map(|res| res.value) - .map_err(|err| err.into()) - } - - fn get_fee_for_message(&self, message: &Message) -> Result { - RpcClient::get_fee_for_message(self, message).map_err(|err| err.into()) - } - - fn get_minimum_balance_for_rent_exemption(&self, data_len: usize) -> Result { - RpcClient::get_minimum_balance_for_rent_exemption(self, data_len).map_err(|err| err.into()) - } - - fn addr(&self) -> String { - self.url() - } - - fn request_airdrop_with_blockhash( - &self, - pubkey: &Pubkey, - lamports: u64, - recent_blockhash: &Hash, - ) -> Result { - RpcClient::request_airdrop_with_blockhash(self, pubkey, lamports, recent_blockhash) - .map_err(|err| err.into()) - } -} diff --git a/bench-tps/src/bench_tps_client/thin_client.rs b/bench-tps/src/bench_tps_client/thin_client.rs deleted file mode 100644 index 2ac7bca2e1..0000000000 --- a/bench-tps/src/bench_tps_client/thin_client.rs +++ /dev/null @@ -1,86 +0,0 @@ -use { - crate::bench_tps_client::{BenchTpsClient, Result}, - solana_client::thin_client::ThinClient, - solana_sdk::{ - client::{AsyncClient, Client, SyncClient}, - commitment_config::CommitmentConfig, - epoch_info::EpochInfo, - hash::Hash, - message::Message, - pubkey::Pubkey, - signature::Signature, - transaction::Transaction, - }, -}; - -impl BenchTpsClient for ThinClient { - fn send_transaction(&self, transaction: Transaction) -> Result { - AsyncClient::async_send_transaction(self, transaction).map_err(|err| err.into()) - } - fn send_batch(&self, transactions: Vec) -> Result<()> { - AsyncClient::async_send_batch(self, transactions).map_err(|err| err.into()) - } - fn get_latest_blockhash(&self) -> Result { - SyncClient::get_latest_blockhash(self).map_err(|err| err.into()) - } - - fn get_latest_blockhash_with_commitment( - &self, - commitment_config: CommitmentConfig, - ) -> Result<(Hash, u64)> { - SyncClient::get_latest_blockhash_with_commitment(self, commitment_config) - .map_err(|err| err.into()) - } - - fn get_transaction_count(&self) -> Result { - SyncClient::get_transaction_count(self).map_err(|err| err.into()) - } - - fn get_transaction_count_with_commitment( - &self, - commitment_config: CommitmentConfig, - ) -> Result { - SyncClient::get_transaction_count_with_commitment(self, commitment_config) - .map_err(|err| err.into()) - } - - fn get_epoch_info(&self) -> Result { - SyncClient::get_epoch_info(self).map_err(|err| err.into()) - } - - fn get_balance(&self, pubkey: &Pubkey) -> Result { - SyncClient::get_balance(self, pubkey).map_err(|err| err.into()) - } - - fn get_balance_with_commitment( - &self, - pubkey: &Pubkey, - commitment_config: CommitmentConfig, - ) -> Result { - SyncClient::get_balance_with_commitment(self, pubkey, commitment_config) - .map_err(|err| err.into()) - } - - fn get_fee_for_message(&self, message: &Message) -> Result { - SyncClient::get_fee_for_message(self, message).map_err(|err| err.into()) - } - - fn get_minimum_balance_for_rent_exemption(&self, data_len: usize) -> Result { - SyncClient::get_minimum_balance_for_rent_exemption(self, data_len).map_err(|err| err.into()) - } - - fn addr(&self) -> String { - Client::tpu_addr(self) - } - - fn request_airdrop_with_blockhash( - &self, - pubkey: &Pubkey, - lamports: u64, - recent_blockhash: &Hash, - ) -> Result { - self.rpc_client() - .request_airdrop_with_blockhash(pubkey, lamports, recent_blockhash) - .map_err(|err| err.into()) - } -} diff --git a/bench-tps/src/bench_tps_client/tpu_client.rs b/bench-tps/src/bench_tps_client/tpu_client.rs deleted file mode 100644 index d968338161..0000000000 --- a/bench-tps/src/bench_tps_client/tpu_client.rs +++ /dev/null @@ -1,99 +0,0 @@ -use { - crate::bench_tps_client::{BenchTpsClient, Result}, - solana_client::tpu_client::TpuClient, - solana_sdk::{ - commitment_config::CommitmentConfig, epoch_info::EpochInfo, hash::Hash, message::Message, - pubkey::Pubkey, signature::Signature, transaction::Transaction, - }, -}; - -impl BenchTpsClient for TpuClient { - fn send_transaction(&self, transaction: Transaction) -> Result { - let signature = transaction.signatures[0]; - self.try_send_transaction(&transaction)?; - Ok(signature) - } - fn send_batch(&self, transactions: Vec) -> Result<()> { - for transaction in transactions { - BenchTpsClient::send_transaction(self, transaction)?; - } - Ok(()) - } - fn get_latest_blockhash(&self) -> Result { - self.rpc_client() - .get_latest_blockhash() - .map_err(|err| err.into()) - } - - fn get_latest_blockhash_with_commitment( - &self, - commitment_config: CommitmentConfig, - ) -> Result<(Hash, u64)> { - self.rpc_client() - .get_latest_blockhash_with_commitment(commitment_config) - .map_err(|err| err.into()) - } - - fn get_transaction_count(&self) -> Result { - self.rpc_client() - .get_transaction_count() - .map_err(|err| err.into()) - } - - fn get_transaction_count_with_commitment( - &self, - commitment_config: CommitmentConfig, - ) -> Result { - self.rpc_client() - .get_transaction_count_with_commitment(commitment_config) - .map_err(|err| err.into()) - } - - fn get_epoch_info(&self) -> Result { - self.rpc_client().get_epoch_info().map_err(|err| err.into()) - } - - fn get_balance(&self, pubkey: &Pubkey) -> Result { - self.rpc_client() - .get_balance(pubkey) - .map_err(|err| err.into()) - } - - fn get_balance_with_commitment( - &self, - pubkey: &Pubkey, - commitment_config: CommitmentConfig, - ) -> Result { - self.rpc_client() - .get_balance_with_commitment(pubkey, commitment_config) - .map(|res| res.value) - .map_err(|err| err.into()) - } - - fn get_fee_for_message(&self, message: &Message) -> Result { - self.rpc_client() - .get_fee_for_message(message) - .map_err(|err| err.into()) - } - - fn get_minimum_balance_for_rent_exemption(&self, data_len: usize) -> Result { - self.rpc_client() - .get_minimum_balance_for_rent_exemption(data_len) - .map_err(|err| err.into()) - } - - fn addr(&self) -> String { - self.rpc_client().url() - } - - fn request_airdrop_with_blockhash( - &self, - pubkey: &Pubkey, - lamports: u64, - recent_blockhash: &Hash, - ) -> Result { - self.rpc_client() - .request_airdrop_with_blockhash(pubkey, lamports, recent_blockhash) - .map_err(|err| err.into()) - } -} diff --git a/bench-tps/src/cli.rs b/bench-tps/src/cli.rs deleted file mode 100644 index e226dd2d90..0000000000 --- a/bench-tps/src/cli.rs +++ /dev/null @@ -1,417 +0,0 @@ -use { - clap::{crate_description, crate_name, App, Arg, ArgMatches}, - solana_clap_utils::input_validators::{is_url, is_url_or_moniker}, - solana_cli_config::{ConfigInput, CONFIG_FILE}, - solana_client::connection_cache::{DEFAULT_TPU_CONNECTION_POOL_SIZE, DEFAULT_TPU_USE_QUIC}, - solana_sdk::{ - fee_calculator::FeeRateGovernor, - pubkey::Pubkey, - signature::{read_keypair_file, Keypair}, - }, - std::{net::SocketAddr, process::exit, time::Duration}, -}; - -const NUM_LAMPORTS_PER_ACCOUNT_DEFAULT: u64 = solana_sdk::native_token::LAMPORTS_PER_VLX; - -pub enum ExternalClientType { - // Submits transactions to an Rpc node using an RpcClient - RpcClient, - // Submits transactions directly to leaders using a ThinClient, broadcasting to multiple - // leaders when num_nodes > 1 - ThinClient, - // Submits transactions directly to leaders using a TpuClient, broadcasting to upcoming leaders - // via TpuClient default configuration - TpuClient, -} - -impl Default for ExternalClientType { - fn default() -> Self { - Self::ThinClient - } -} - -/// Holds the configuration for a single run of the benchmark -pub struct Config { - pub entrypoint_addr: SocketAddr, - pub json_rpc_url: String, - pub websocket_url: String, - pub id: Keypair, - pub threads: usize, - pub num_nodes: usize, - pub duration: Duration, - pub tx_count: usize, - pub keypair_multiplier: usize, - pub thread_batch_sleep_ms: usize, - pub sustained: bool, - pub client_ids_and_stake_file: String, - pub write_to_client_file: bool, - pub read_from_client_file: bool, - pub target_lamports_per_signature: u64, - pub multi_client: bool, - pub num_lamports_per_account: u64, - pub target_slots_per_epoch: u64, - pub target_node: Option, - pub external_client_type: ExternalClientType, - pub use_quic: bool, - pub tpu_connection_pool_size: usize, -} - -impl Default for Config { - fn default() -> Config { - Config { - entrypoint_addr: SocketAddr::from(([127, 0, 0, 1], 8001)), - json_rpc_url: ConfigInput::default().json_rpc_url, - websocket_url: ConfigInput::default().websocket_url, - id: Keypair::new(), - threads: 4, - num_nodes: 1, - duration: Duration::new(std::u64::MAX, 0), - tx_count: 50_000, - keypair_multiplier: 8, - thread_batch_sleep_ms: 1000, - sustained: false, - client_ids_and_stake_file: String::new(), - write_to_client_file: false, - read_from_client_file: false, - target_lamports_per_signature: FeeRateGovernor::default().target_lamports_per_signature, - multi_client: true, - num_lamports_per_account: NUM_LAMPORTS_PER_ACCOUNT_DEFAULT, - target_slots_per_epoch: 0, - target_node: None, - external_client_type: ExternalClientType::default(), - use_quic: DEFAULT_TPU_USE_QUIC, - tpu_connection_pool_size: DEFAULT_TPU_CONNECTION_POOL_SIZE, - } - } -} - -/// Defines and builds the CLI args for a run of the benchmark -pub fn build_args<'a, 'b>(version: &'b str) -> App<'a, 'b> { - App::new(crate_name!()).about(crate_description!()) - .version(version) - .arg({ - let arg = Arg::with_name("config_file") - .short("C") - .long("config") - .value_name("FILEPATH") - .takes_value(true) - .global(true) - .help("Configuration file to use"); - if let Some(ref config_file) = *CONFIG_FILE { - arg.default_value(config_file) - } else { - arg - } - }) - .arg( - Arg::with_name("json_rpc_url") - .short("u") - .long("url") - .value_name("URL_OR_MONIKER") - .takes_value(true) - .global(true) - .validator(is_url_or_moniker) - .help( - "URL for Solana's JSON RPC or moniker (or their first letter): \ - [mainnet-beta, testnet, devnet, localhost]", - ), - ) - .arg( - Arg::with_name("websocket_url") - .long("ws") - .value_name("URL") - .takes_value(true) - .global(true) - .validator(is_url) - .help("WebSocket URL for the solana cluster"), - ) - .arg( - Arg::with_name("entrypoint") - .short("n") - .long("entrypoint") - .value_name("HOST:PORT") - .takes_value(true) - .help("Rendezvous with the cluster at this entry point; defaults to 127.0.0.1:8001"), - ) - .arg( - Arg::with_name("faucet") - .short("d") - .long("faucet") - .value_name("HOST:PORT") - .takes_value(true) - .hidden(true) - .help("Deprecated. BenchTps no longer queries the faucet directly"), - ) - .arg( - Arg::with_name("identity") - .short("i") - .long("identity") - .value_name("PATH") - .takes_value(true) - .help("File containing a client identity (keypair)"), - ) - .arg( - Arg::with_name("num-nodes") - .short("N") - .long("num-nodes") - .value_name("NUM") - .takes_value(true) - .help("Wait for NUM nodes to converge"), - ) - .arg( - Arg::with_name("threads") - .short("t") - .long("threads") - .value_name("NUM") - .takes_value(true) - .help("Number of threads"), - ) - .arg( - Arg::with_name("duration") - .long("duration") - .value_name("SECS") - .takes_value(true) - .help("Seconds to run benchmark, then exit; default is forever"), - ) - .arg( - Arg::with_name("sustained") - .long("sustained") - .help("Use sustained performance mode vs. peak mode. This overlaps the tx generation with transfers."), - ) - .arg( - Arg::with_name("no-multi-client") - .long("no-multi-client") - .help("Disable multi-client support, only transact with the entrypoint."), - ) - .arg( - Arg::with_name("target_node") - .long("target-node") - .requires("no-multi-client") - .takes_value(true) - .value_name("PUBKEY") - .help("Specify an exact node to send transactions to."), - ) - .arg( - Arg::with_name("tx_count") - .long("tx_count") - .value_name("NUM") - .takes_value(true) - .help("Number of transactions to send per batch") - ) - .arg( - Arg::with_name("keypair_multiplier") - .long("keypair-multiplier") - .value_name("NUM") - .takes_value(true) - .help("Multiply by transaction count to determine number of keypairs to create") - ) - .arg( - Arg::with_name("thread-batch-sleep-ms") - .short("z") - .long("thread-batch-sleep-ms") - .value_name("NUM") - .takes_value(true) - .help("Per-thread-per-iteration sleep in ms"), - ) - .arg( - Arg::with_name("write-client-keys") - .long("write-client-keys") - .value_name("FILENAME") - .takes_value(true) - .help("Generate client keys and stakes and write the list to YAML file"), - ) - .arg( - Arg::with_name("read-client-keys") - .long("read-client-keys") - .value_name("FILENAME") - .takes_value(true) - .help("Read client keys and stakes from the YAML file"), - ) - .arg( - Arg::with_name("target_lamports_per_signature") - .long("target-lamports-per-signature") - .value_name("LAMPORTS") - .takes_value(true) - .help( - "The cost in lamports that the cluster will charge for signature \ - verification when the cluster is operating at target-signatures-per-slot", - ), - ) - .arg( - Arg::with_name("num_lamports_per_account") - .long("num-lamports-per-account") - .value_name("LAMPORTS") - .takes_value(true) - .help( - "Number of lamports per account.", - ), - ) - .arg( - Arg::with_name("target_slots_per_epoch") - .long("target-slots-per-epoch") - .value_name("SLOTS") - .takes_value(true) - .help( - "Wait until epochs are this many slots long.", - ), - ) - .arg( - Arg::with_name("rpc_client") - .long("use-rpc-client") - .conflicts_with("tpu_client") - .takes_value(false) - .help("Submit transactions with a RpcClient") - ) - .arg( - Arg::with_name("tpu_client") - .long("use-tpu-client") - .conflicts_with("rpc_client") - .takes_value(false) - .help("Submit transactions with a TpuClient") - ) - .arg( - Arg::with_name("tpu_use_quic") - .long("tpu-use-quic") - .takes_value(false) - .help("Submit transactions via QUIC; only affects ThinClient (default) \ - or TpuClient sends"), - ) - .arg( - Arg::with_name("tpu_connection_pool_size") - .long("tpu-connection-pool-size") - .takes_value(true) - .help("Controls the connection pool size per remote address; only affects ThinClient (default) \ - or TpuClient sends"), - ) -} - -/// Parses a clap `ArgMatches` structure into a `Config` -/// # Arguments -/// * `matches` - command line arguments parsed by clap -/// # Panics -/// Panics if there is trouble parsing any of the arguments -pub fn extract_args(matches: &ArgMatches) -> Config { - let mut args = Config::default(); - - let config = if let Some(config_file) = matches.value_of("config_file") { - solana_cli_config::Config::load(config_file).unwrap_or_default() - } else { - solana_cli_config::Config::default() - }; - let (_, json_rpc_url) = ConfigInput::compute_json_rpc_url_setting( - matches.value_of("json_rpc_url").unwrap_or(""), - &config.json_rpc_url, - ); - args.json_rpc_url = json_rpc_url; - - let (_, websocket_url) = ConfigInput::compute_websocket_url_setting( - matches.value_of("websocket_url").unwrap_or(""), - &config.websocket_url, - matches.value_of("json_rpc_url").unwrap_or(""), - &config.json_rpc_url, - ); - args.websocket_url = websocket_url; - - let (_, id_path) = ConfigInput::compute_keypair_path_setting( - matches.value_of("identity").unwrap_or(""), - &config.keypair_path, - ); - if let Ok(id) = read_keypair_file(id_path) { - args.id = id; - } else if matches.is_present("identity") { - panic!("could not parse identity path"); - } - - if matches.is_present("tpu_client") { - args.external_client_type = ExternalClientType::TpuClient; - } else if matches.is_present("rpc_client") { - args.external_client_type = ExternalClientType::RpcClient; - } - - if matches.is_present("tpu_use_quic") { - args.use_quic = true; - } - - if let Some(v) = matches.value_of("tpu_connection_pool_size") { - args.tpu_connection_pool_size = v - .to_string() - .parse() - .expect("can't parse tpu_connection_pool_size"); - } - - if let Some(addr) = matches.value_of("entrypoint") { - args.entrypoint_addr = solana_net_utils::parse_host_port(addr).unwrap_or_else(|e| { - eprintln!("failed to parse entrypoint address: {}", e); - exit(1) - }); - } - - if let Some(t) = matches.value_of("threads") { - args.threads = t.to_string().parse().expect("can't parse threads"); - } - - if let Some(n) = matches.value_of("num-nodes") { - args.num_nodes = n.to_string().parse().expect("can't parse num-nodes"); - } - - if let Some(duration) = matches.value_of("duration") { - args.duration = Duration::new( - duration.to_string().parse().expect("can't parse duration"), - 0, - ); - } - - if let Some(s) = matches.value_of("tx_count") { - args.tx_count = s.to_string().parse().expect("can't parse tx_count"); - } - - if let Some(s) = matches.value_of("keypair_multiplier") { - args.keypair_multiplier = s - .to_string() - .parse() - .expect("can't parse keypair-multiplier"); - assert!(args.keypair_multiplier >= 2); - } - - if let Some(t) = matches.value_of("thread-batch-sleep-ms") { - args.thread_batch_sleep_ms = t - .to_string() - .parse() - .expect("can't parse thread-batch-sleep-ms"); - } - - args.sustained = matches.is_present("sustained"); - - if let Some(s) = matches.value_of("write-client-keys") { - args.write_to_client_file = true; - args.client_ids_and_stake_file = s.to_string(); - } - - if let Some(s) = matches.value_of("read-client-keys") { - assert!(!args.write_to_client_file); - args.read_from_client_file = true; - args.client_ids_and_stake_file = s.to_string(); - } - - if let Some(v) = matches.value_of("target_lamports_per_signature") { - args.target_lamports_per_signature = v.to_string().parse().expect("can't parse lamports"); - } - - args.multi_client = !matches.is_present("no-multi-client"); - args.target_node = matches - .value_of("target_node") - .map(|target_str| target_str.parse().unwrap()); - - if let Some(v) = matches.value_of("num_lamports_per_account") { - args.num_lamports_per_account = v.to_string().parse().expect("can't parse lamports"); - } - - if let Some(t) = matches.value_of("target_slots_per_epoch") { - args.target_slots_per_epoch = t - .to_string() - .parse() - .expect("can't parse target slots per epoch"); - } - - args -} diff --git a/bench-tps/src/keypairs.rs b/bench-tps/src/keypairs.rs deleted file mode 100644 index e165e484d5..0000000000 --- a/bench-tps/src/keypairs.rs +++ /dev/null @@ -1,72 +0,0 @@ -use { - crate::{ - bench::{fund_keypairs, generate_and_fund_keypairs}, - bench_tps_client::BenchTpsClient, - }, - log::*, - solana_genesis::Base64Account, - solana_sdk::signature::{Keypair, Signer}, - std::{collections::HashMap, fs::File, path::Path, process::exit, sync::Arc}, -}; - -pub fn get_keypairs( - client: Arc, - id: &Keypair, - keypair_count: usize, - num_lamports_per_account: u64, - client_ids_and_stake_file: &str, - read_from_client_file: bool, -) -> Vec -where - T: 'static + BenchTpsClient + Send + Sync, -{ - if read_from_client_file { - let path = Path::new(client_ids_and_stake_file); - let file = File::open(path).unwrap(); - - info!("Reading {}", client_ids_and_stake_file); - let accounts: HashMap = serde_yaml::from_reader(file).unwrap(); - let mut keypairs = vec![]; - let mut last_balance = 0; - - accounts - .into_iter() - .for_each(|(keypair, primordial_account)| { - let bytes: Vec = serde_json::from_str(keypair.as_str()).unwrap(); - keypairs.push(Keypair::from_bytes(&bytes).unwrap()); - last_balance = primordial_account.balance; - }); - - if keypairs.len() < keypair_count { - eprintln!( - "Expected {} accounts in {}, only received {} (--tx_count mismatch?)", - keypair_count, - client_ids_and_stake_file, - keypairs.len(), - ); - exit(1); - } - // Sort keypairs so that do_bench_tps() uses the same subset of accounts for each run. - // This prevents the amount of storage needed for bench-tps accounts from creeping up - // across multiple runs. - keypairs.sort_by_key(|x| x.pubkey().to_string()); - fund_keypairs( - client, - id, - &keypairs, - keypairs.len().saturating_sub(keypair_count) as u64, - last_balance, - ) - .unwrap_or_else(|e| { - eprintln!("Error could not fund keys: {:?}", e); - exit(1); - }); - keypairs - } else { - generate_and_fund_keypairs(client, id, keypair_count, num_lamports_per_account) - .unwrap_or_else(|e| { - eprintln!("Error could not fund keys: {:?}", e); - exit(1); - }) - } -} diff --git a/bench-tps/src/lib.rs b/bench-tps/src/lib.rs deleted file mode 100644 index 06d5eaa1af..0000000000 --- a/bench-tps/src/lib.rs +++ /dev/null @@ -1,6 +0,0 @@ -#![allow(clippy::integer_arithmetic)] -pub mod bench; -pub mod bench_tps_client; -pub mod cli; -pub mod keypairs; -mod perf_utils; diff --git a/bench-tps/src/main.rs b/bench-tps/src/main.rs deleted file mode 100644 index 27db5a168b..0000000000 --- a/bench-tps/src/main.rs +++ /dev/null @@ -1,191 +0,0 @@ -#![allow(clippy::integer_arithmetic)] -use { - log::*, - solana_bench_tps::{ - bench::{do_bench_tps, generate_keypairs}, - cli::{self, ExternalClientType}, - keypairs::get_keypairs, - }, - solana_client::{ - connection_cache::ConnectionCache, - rpc_client::RpcClient, - tpu_client::{TpuClient, TpuClientConfig}, - }, - solana_genesis::Base64Account, - solana_gossip::gossip_service::{discover_cluster, get_client, get_multi_client}, - solana_sdk::{ - commitment_config::CommitmentConfig, fee_calculator::FeeRateGovernor, system_program, - }, - solana_streamer::socket::SocketAddrSpace, - std::{collections::HashMap, fs::File, io::prelude::*, path::Path, process::exit, sync::Arc}, -}; - -/// Number of signatures for all transactions in ~1 week at ~100K TPS -pub const NUM_SIGNATURES_FOR_TXS: u64 = 100_000 * 60 * 60 * 24 * 7; - -fn main() { - solana_logger::setup_with_default("solana=info"); - solana_metrics::set_panic_hook("bench-tps", /*version:*/ None); - - let matches = cli::build_args(solana_version::version!()).get_matches(); - let cli_config = cli::extract_args(&matches); - - let cli::Config { - entrypoint_addr, - json_rpc_url, - websocket_url, - id, - num_nodes, - tx_count, - keypair_multiplier, - client_ids_and_stake_file, - write_to_client_file, - read_from_client_file, - target_lamports_per_signature, - multi_client, - num_lamports_per_account, - target_node, - external_client_type, - use_quic, - tpu_connection_pool_size, - .. - } = &cli_config; - - let keypair_count = *tx_count * keypair_multiplier; - if *write_to_client_file { - info!("Generating {} keypairs", keypair_count); - let (keypairs, _) = generate_keypairs(id, keypair_count as u64); - let num_accounts = keypairs.len() as u64; - let max_fee = - FeeRateGovernor::new(*target_lamports_per_signature, 0).max_lamports_per_signature; - let num_lamports_per_account = (num_accounts - 1 + NUM_SIGNATURES_FOR_TXS * max_fee) - / num_accounts - + num_lamports_per_account; - let mut accounts = HashMap::new(); - keypairs.iter().for_each(|keypair| { - accounts.insert( - serde_json::to_string(&keypair.to_bytes().to_vec()).unwrap(), - Base64Account { - balance: num_lamports_per_account, - executable: false, - owner: system_program::id().to_string(), - data: String::new(), - }, - ); - }); - - info!("Writing {}", client_ids_and_stake_file); - let serialized = serde_yaml::to_string(&accounts).unwrap(); - let path = Path::new(&client_ids_and_stake_file); - let mut file = File::create(path).unwrap(); - file.write_all(&serialized.into_bytes()).unwrap(); - return; - } - - info!("Connecting to the cluster"); - - match external_client_type { - ExternalClientType::RpcClient => { - let client = Arc::new(RpcClient::new_with_commitment( - json_rpc_url.to_string(), - CommitmentConfig::confirmed(), - )); - let keypairs = get_keypairs( - client.clone(), - id, - keypair_count, - *num_lamports_per_account, - client_ids_and_stake_file, - *read_from_client_file, - ); - do_bench_tps(client, cli_config, keypairs); - } - ExternalClientType::ThinClient => { - let nodes = discover_cluster(entrypoint_addr, *num_nodes, SocketAddrSpace::Unspecified) - .unwrap_or_else(|err| { - eprintln!("Failed to discover {} nodes: {:?}", num_nodes, err); - exit(1); - }); - let connection_cache = match use_quic { - true => Arc::new(ConnectionCache::new(*tpu_connection_pool_size)), - false => Arc::new(ConnectionCache::with_udp(*tpu_connection_pool_size)), - }; - let client = if *multi_client { - let (client, num_clients) = - get_multi_client(&nodes, &SocketAddrSpace::Unspecified, connection_cache); - if nodes.len() < num_clients { - eprintln!( - "Error: Insufficient nodes discovered. Expecting {} or more", - num_nodes - ); - exit(1); - } - Arc::new(client) - } else if let Some(target_node) = target_node { - info!("Searching for target_node: {:?}", target_node); - let mut target_client = None; - for node in nodes { - if node.id == *target_node { - target_client = Some(Arc::new(get_client( - &[node], - &SocketAddrSpace::Unspecified, - connection_cache, - ))); - break; - } - } - target_client.unwrap_or_else(|| { - eprintln!("Target node {} not found", target_node); - exit(1); - }) - } else { - Arc::new(get_client( - &nodes, - &SocketAddrSpace::Unspecified, - connection_cache, - )) - }; - let keypairs = get_keypairs( - client.clone(), - id, - keypair_count, - *num_lamports_per_account, - client_ids_and_stake_file, - *read_from_client_file, - ); - do_bench_tps(client, cli_config, keypairs); - } - ExternalClientType::TpuClient => { - let rpc_client = Arc::new(RpcClient::new_with_commitment( - json_rpc_url.to_string(), - CommitmentConfig::confirmed(), - )); - let connection_cache = match use_quic { - true => ConnectionCache::new(*tpu_connection_pool_size), - false => ConnectionCache::with_udp(*tpu_connection_pool_size), - }; - - let client = Arc::new( - TpuClient::new_with_connection_cache( - rpc_client, - websocket_url, - TpuClientConfig::default(), - Arc::new(connection_cache), - ) - .unwrap_or_else(|err| { - eprintln!("Could not create TpuClient {:?}", err); - exit(1); - }), - ); - let keypairs = get_keypairs( - client.clone(), - id, - keypair_count, - *num_lamports_per_account, - client_ids_and_stake_file, - *read_from_client_file, - ); - do_bench_tps(client, cli_config, keypairs); - } - } -} diff --git a/bench-tps/src/perf_utils.rs b/bench-tps/src/perf_utils.rs deleted file mode 100644 index 7e31e6408e..0000000000 --- a/bench-tps/src/perf_utils.rs +++ /dev/null @@ -1,90 +0,0 @@ -use { - crate::bench_tps_client::BenchTpsClient, - log::*, - solana_sdk::{commitment_config::CommitmentConfig, timing::duration_as_s}, - std::{ - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, RwLock, - }, - thread::sleep, - time::{Duration, Instant}, - }, -}; - -#[derive(Default)] -pub struct SampleStats { - /// Maximum TPS reported by this node - pub tps: f32, - /// Total time taken for those txs - pub elapsed: Duration, - /// Total transactions reported by this node - pub txs: u64, -} - -pub fn sample_txs( - exit_signal: &Arc, - sample_stats: &Arc>>, - sample_period: u64, - client: &Arc, -) where - T: BenchTpsClient, -{ - let mut max_tps = 0.0; - let mut total_elapsed; - let mut total_txs; - let mut now = Instant::now(); - let start_time = now; - let initial_txs = client - .get_transaction_count_with_commitment(CommitmentConfig::processed()) - .expect("transaction count"); - let mut last_txs = initial_txs; - - loop { - total_elapsed = start_time.elapsed(); - let elapsed = now.elapsed(); - now = Instant::now(); - let mut txs = - match client.get_transaction_count_with_commitment(CommitmentConfig::processed()) { - Err(e) => { - // ThinClient with multiple options should pick a better one now. - info!("Couldn't get transaction count {:?}", e); - sleep(Duration::from_secs(sample_period)); - continue; - } - Ok(tx_count) => tx_count, - }; - - if txs < last_txs { - info!("Expected txs({}) >= last_txs({})", txs, last_txs); - txs = last_txs; - } - total_txs = txs - initial_txs; - let sample_txs = txs - last_txs; - last_txs = txs; - - let tps = sample_txs as f32 / duration_as_s(&elapsed); - if tps > max_tps { - max_tps = tps; - } - - info!( - "Sampler {:9.2} TPS, Transactions: {:6}, Total transactions: {} over {} s", - tps, - sample_txs, - total_txs, - total_elapsed.as_secs(), - ); - - if exit_signal.load(Ordering::Relaxed) { - let stats = SampleStats { - tps: max_tps, - elapsed: total_elapsed, - txs: total_txs, - }; - sample_stats.write().unwrap().push((client.addr(), stats)); - return; - } - sleep(Duration::from_secs(sample_period)); - } -} diff --git a/bench-tps/tests/bench_tps.rs b/bench-tps/tests/bench_tps.rs deleted file mode 100644 index 6d1c32b47a..0000000000 --- a/bench-tps/tests/bench_tps.rs +++ /dev/null @@ -1,148 +0,0 @@ -#![allow(clippy::integer_arithmetic)] -use { - serial_test::serial, - solana_bench_tps::{ - bench::{do_bench_tps, generate_and_fund_keypairs}, - cli::Config, - }, - solana_client::{ - connection_cache::ConnectionCache, - rpc_client::RpcClient, - thin_client::ThinClient, - tpu_client::{TpuClient, TpuClientConfig}, - }, - solana_core::validator::ValidatorConfig, - solana_faucet::faucet::run_local_faucet, - solana_local_cluster::{ - local_cluster::{ClusterConfig, LocalCluster}, - validator_configs::make_identical_validator_configs, - }, - solana_rpc::rpc::JsonRpcConfig, - solana_sdk::{ - commitment_config::CommitmentConfig, - signature::{Keypair, Signer}, - }, - solana_streamer::socket::SocketAddrSpace, - solana_test_validator::TestValidator, - std::{sync::Arc, time::Duration}, -}; - -fn test_bench_tps_local_cluster(config: Config) { - let native_instruction_processors = vec![]; - - solana_logger::setup(); - - let faucet_keypair = Keypair::new(); - let faucet_pubkey = faucet_keypair.pubkey(); - let faucet_addr = run_local_faucet(faucet_keypair, None); - - const NUM_NODES: usize = 1; - let cluster = LocalCluster::new( - &mut ClusterConfig { - node_stakes: vec![999_990; NUM_NODES], - cluster_lamports: 200_000_000, - validator_configs: make_identical_validator_configs( - &ValidatorConfig { - rpc_config: JsonRpcConfig { - faucet_addr: Some(faucet_addr), - ..JsonRpcConfig::default_for_test() - }, - ..ValidatorConfig::default_for_test() - }, - NUM_NODES, - ), - native_instruction_processors, - ..ClusterConfig::default() - }, - SocketAddrSpace::Unspecified, - ); - - cluster.transfer(&cluster.funding_keypair, &faucet_pubkey, 100_000_000); - - let client = Arc::new(ThinClient::new( - cluster.entry_point_info.rpc, - cluster.entry_point_info.tpu, - cluster.connection_cache.clone(), - )); - - let lamports_per_account = 100; - - let keypair_count = config.tx_count * config.keypair_multiplier; - let keypairs = generate_and_fund_keypairs( - client.clone(), - &config.id, - keypair_count, - lamports_per_account, - ) - .unwrap(); - - let _total = do_bench_tps(client, config, keypairs); - - #[cfg(not(debug_assertions))] - assert!(_total > 100); -} - -fn test_bench_tps_test_validator(config: Config) { - solana_logger::setup(); - - let mint_keypair = Keypair::new(); - let mint_pubkey = mint_keypair.pubkey(); - - let faucet_addr = run_local_faucet(mint_keypair, None); - - let test_validator = - TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr), SocketAddrSpace::Unspecified); - - let rpc_client = Arc::new(RpcClient::new_with_commitment( - test_validator.rpc_url(), - CommitmentConfig::processed(), - )); - let websocket_url = test_validator.rpc_pubsub_url(); - let connection_cache = Arc::new(ConnectionCache::default()); - - let client = Arc::new( - TpuClient::new_with_connection_cache( - rpc_client, - &websocket_url, - TpuClientConfig::default(), - connection_cache, - ) - .unwrap(), - ); - - let lamports_per_account = 100; - - let keypair_count = config.tx_count * config.keypair_multiplier; - let keypairs = generate_and_fund_keypairs( - client.clone(), - &config.id, - keypair_count, - lamports_per_account, - ) - .unwrap(); - - let _total = do_bench_tps(client, config, keypairs); - - #[cfg(not(debug_assertions))] - assert!(_total > 100); -} - -#[test] -#[serial] -fn test_bench_tps_local_cluster_solana() { - test_bench_tps_local_cluster(Config { - tx_count: 100, - duration: Duration::from_secs(10), - ..Config::default() - }); -} - -#[test] -#[serial] -fn test_bench_tps_tpu_client() { - test_bench_tps_test_validator(Config { - tx_count: 100, - duration: Duration::from_secs(10), - ..Config::default() - }); -} diff --git a/dos/Cargo.toml b/dos/Cargo.toml deleted file mode 100644 index c56be42fea..0000000000 --- a/dos/Cargo.toml +++ /dev/null @@ -1,32 +0,0 @@ -[package] -authors = ["Solana Maintainers "] -edition = "2021" -name = "solana-dos" -version = "1.10.41" -repository = "https://github.com/solana-labs/solana" -license = "Apache-2.0" -homepage = "https://solana.com/" -publish = false -description = "Tool to send various requests to cluster in order to evaluate the effect on performance" - -[dependencies] -bincode = "1.3.3" -clap = {version = "3.1.5", features = ["derive", "cargo"]} -log = "0.4.14" -rand = "0.7.0" -serde = "1.0.136" -solana-client = { path = "../client", version = "=1.10.41" } -solana-core = { path = "../core", version = "=1.10.41" } -solana-gossip = { path = "../gossip", version = "=1.10.41" } -solana-logger = { path = "../logger", version = "=1.10.41" } -solana-net-utils = { path = "../net-utils", version = "=1.10.41" } -solana-perf = { path = "../perf", version = "=1.10.41" } -solana-sdk = { path = "../sdk", version = "=1.10.41" } -solana-streamer = { path = "../streamer", version = "=1.10.41" } -solana-version = { path = "../version", version = "=0.7.0" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] - -[dev-dependencies] -solana-local-cluster = { path = "../local-cluster", version = "=1.10.41" } diff --git a/dos/src/main.rs b/dos/src/main.rs deleted file mode 100644 index 1587cfcbe6..0000000000 --- a/dos/src/main.rs +++ /dev/null @@ -1,737 +0,0 @@ -//! DoS tool -//! -//! Sends requests to cluster in a loop to measure -//! the effect of handling these requests on the performance of the cluster. -//! -//! * `mode` argument defines interface to use (e.g. rpc, tvu, tpu) -//! * `data-type` argument specifies the type of the request. -//! Some request types might be used only with particular `mode` value. -//! For example, `get-account-info` is valid only with `mode=rpc`. -//! -//! Most options are provided for `data-type = transaction`. -//! These options allow to compose transaction which fails at -//! a particular stage of the processing pipeline. -//! -//! Example 1: send random transactions to TPU -//! ```bash -//! solana-dos --entrypoint 127.0.0.1:8001 --mode tpu --data-type random -//! ``` -//! -//! Example 2: send unique transactions with valid recent blockhash to TPU -//! ```bash -//! solana-dos --entrypoint 127.0.0.1:8001 --mode tpu --data-type random -//! solana-dos --entrypoint 127.0.0.1:8001 --mode tpu \ -//! --data-type transaction --generate-unique-transactions -//! --payer config/bootstrap-validator/identity.json \ -//! --generate-valid-blockhash -//! ``` -//! -#![allow(clippy::integer_arithmetic)] -use { - clap::{crate_description, crate_name, crate_version, ArgEnum, Args, Parser}, - log::*, - rand::{thread_rng, Rng}, - serde::{Deserialize, Serialize}, - solana_client::rpc_client::RpcClient, - solana_core::serve_repair::{RepairProtocol, RepairRequestHeader, ServeRepair}, - solana_gossip::{contact_info::ContactInfo, gossip_service::discover}, - solana_sdk::{ - hash::Hash, - instruction::{AccountMeta, CompiledInstruction, Instruction}, - pubkey::Pubkey, - signature::{read_keypair_file, Keypair, Signature, Signer}, - stake, - system_instruction::SystemInstruction, - system_program, - timing::timestamp, - transaction::Transaction, - }, - solana_streamer::socket::SocketAddrSpace, - std::{ - net::{SocketAddr, UdpSocket}, - process::exit, - str::FromStr, - time::{Duration, Instant}, - }, -}; - -struct TransactionGenerator { - blockhash: Hash, - last_generated: Instant, - transaction_params: TransactionParams, - cached_transaction: Option, -} - -impl TransactionGenerator { - fn new(transaction_params: TransactionParams) -> Self { - TransactionGenerator { - blockhash: Hash::default(), - last_generated: (Instant::now() - Duration::from_secs(100)), - transaction_params, - cached_transaction: None, - } - } - - fn generate(&mut self, payer: Option<&Keypair>, rpc_client: &Option) -> Transaction { - if !self.transaction_params.unique_transactions && self.cached_transaction.is_some() { - return self.cached_transaction.as_ref().unwrap().clone(); - } - - // generate a new blockhash every 1sec - if self.transaction_params.valid_blockhash - && self.last_generated.elapsed().as_millis() > 1000 - { - self.blockhash = rpc_client.as_ref().unwrap().get_latest_blockhash().unwrap(); - self.last_generated = Instant::now(); - } - - // in order to evaluate the performance implications of the different transactions - // we create here transactions which are filtered out on different stages of processing pipeline - - // create an arbitrary valid instruction - let lamports = 5; - let transfer_instruction = SystemInstruction::Transfer { lamports }; - let program_ids = vec![system_program::id(), stake::program::id()]; - - // transaction with payer, in this case signatures are valid and num_signatures is irrelevant - // random payer will cause error "attempt to debit an account but found no record of a prior credit" - // if payer is correct, it will trigger error with not enough signatures - let transaction = if let Some(payer) = payer { - let instruction = Instruction::new_with_bincode( - program_ids[0], - &transfer_instruction, - vec![ - AccountMeta::new(program_ids[0], false), - AccountMeta::new(program_ids[1], false), - ], - ); - Transaction::new_signed_with_payer( - &[instruction], - Some(&payer.pubkey()), - &[payer], - self.blockhash, - ) - } else if self.transaction_params.valid_signatures { - // Since we don't provide a payer, this transaction will end up - // filtered at legacy.rs sanitize method (banking_stage) with error "a program cannot be payer" - let kpvals: Vec = (0..self.transaction_params.num_signatures) - .map(|_| Keypair::new()) - .collect(); - let keypairs: Vec<&Keypair> = kpvals.iter().collect(); - - let instructions = vec![CompiledInstruction::new( - 0, - &transfer_instruction, - vec![0, 1], - )]; - - Transaction::new_with_compiled_instructions( - &keypairs, - &[], - self.blockhash, - program_ids, - instructions, - ) - } else { - // Since we provided invalid signatures - // this transaction will end up filtered at legacy.rs (banking_stage) because - // num_required_signatures == 0 - let instructions = vec![CompiledInstruction::new( - 0, - &transfer_instruction, - vec![0, 1], - )]; - - let mut tx = Transaction::new_with_compiled_instructions( - &[] as &[&Keypair; 0], - &[], - self.blockhash, - program_ids, - instructions, - ); - tx.signatures = vec![Signature::new_unique(); self.transaction_params.num_signatures]; - tx - }; - - // if we need to generate only one transaction, we cache it to reuse later - if !self.transaction_params.unique_transactions { - self.cached_transaction = Some(transaction.clone()); - } - - transaction - } -} - -fn run_dos( - nodes: &[ContactInfo], - iterations: usize, - payer: Option<&Keypair>, - params: DosClientParameters, -) { - let mut target = None; - let mut rpc_client = None; - if nodes.is_empty() { - if params.mode == Mode::Rpc { - rpc_client = Some(RpcClient::new_socket(params.entrypoint_addr)); - } - target = Some((solana_sdk::pubkey::new_rand(), params.entrypoint_addr)); - } else { - info!("************ NODE ***********"); - for node in nodes { - info!("{:?}", node); - } - info!("ADDR = {}", params.entrypoint_addr); - - for node in nodes { - if node.gossip == params.entrypoint_addr { - info!("{}", node.gossip); - target = match params.mode { - Mode::Gossip => Some((node.id, node.gossip)), - Mode::Tvu => Some((node.id, node.tvu)), - Mode::TvuForwards => Some((node.id, node.tvu_forwards)), - Mode::Tpu => { - rpc_client = Some(RpcClient::new_socket(node.rpc)); - Some((node.id, node.tpu)) - } - Mode::TpuForwards => Some((node.id, node.tpu_forwards)), - Mode::Repair => Some((node.id, node.repair)), - Mode::ServeRepair => Some((node.id, node.serve_repair)), - Mode::Rpc => { - rpc_client = Some(RpcClient::new_socket(node.rpc)); - None - } - }; - break; - } - } - } - let (target_id, target_addr) = target.expect("should have target"); - info!("Targeting {}", target_addr); - let socket = UdpSocket::bind("0.0.0.0:0").unwrap(); - - let mut data = Vec::new(); - let mut transaction_generator = None; - - match params.data_type { - DataType::RepairHighest => { - let slot = 100; - let keypair = Keypair::new(); - let header = RepairRequestHeader::new(keypair.pubkey(), target_id, timestamp(), 0); - let req = RepairProtocol::WindowIndex { - header, - slot, - shred_index: 0, - }; - data = ServeRepair::repair_proto_to_bytes(&req, Some(&keypair)).unwrap(); - } - DataType::RepairShred => { - let slot = 100; - let keypair = Keypair::new(); - let header = RepairRequestHeader::new(keypair.pubkey(), target_id, timestamp(), 0); - let req = RepairProtocol::HighestWindowIndex { - header, - slot, - shred_index: 0, - }; - data = ServeRepair::repair_proto_to_bytes(&req, Some(&keypair)).unwrap(); - } - DataType::RepairOrphan => { - let slot = 100; - let keypair = Keypair::new(); - let header = RepairRequestHeader::new(keypair.pubkey(), target_id, timestamp(), 0); - let req = RepairProtocol::Orphan { header, slot }; - data = ServeRepair::repair_proto_to_bytes(&req, Some(&keypair)).unwrap(); - } - DataType::Random => { - data.resize(params.data_size, 0); - } - DataType::Transaction => { - let tp = params.transaction_params; - info!("{:?}", tp); - - transaction_generator = Some(TransactionGenerator::new(tp)); - let tx = transaction_generator - .as_mut() - .unwrap() - .generate(payer, &rpc_client); - info!("{:?}", tx); - data = bincode::serialize(&tx).unwrap(); - } - DataType::GetAccountInfo => {} - DataType::GetProgramAccounts => {} - } - - let mut last_log = Instant::now(); - let mut count = 0; - let mut error_count = 0; - loop { - if params.mode == Mode::Rpc { - match params.data_type { - DataType::GetAccountInfo => { - let res = rpc_client.as_ref().unwrap().get_account( - &Pubkey::from_str(params.data_input.as_ref().unwrap()).unwrap(), - ); - if res.is_err() { - error_count += 1; - } - } - DataType::GetProgramAccounts => { - let res = rpc_client.as_ref().unwrap().get_program_accounts( - &Pubkey::from_str(params.data_input.as_ref().unwrap()).unwrap(), - ); - if res.is_err() { - error_count += 1; - } - } - _ => { - panic!("unsupported data type"); - } - } - } else { - if params.data_type == DataType::Random { - thread_rng().fill(&mut data[..]); - } - if let Some(tg) = transaction_generator.as_mut() { - let tx = tg.generate(payer, &rpc_client); - info!("{:?}", tx); - data = bincode::serialize(&tx).unwrap(); - } - let res = socket.send_to(&data, target_addr); - if res.is_err() { - error_count += 1; - } - } - count += 1; - if last_log.elapsed().as_millis() > 10_000 { - info!("count: {} errors: {}", count, error_count); - last_log = Instant::now(); - count = 0; - } - if iterations != 0 && count >= iterations { - break; - } - } -} - -// command line parsing -#[derive(Parser)] -#[clap(name = crate_name!(), - version = crate_version!(), - about = crate_description!(), - rename_all = "kebab-case" -)] -struct DosClientParameters { - #[clap(long, arg_enum, help = "Interface to DoS")] - mode: Mode, - - #[clap(long, arg_enum, help = "Type of data to send")] - data_type: DataType, - - #[clap( - long = "entrypoint", - parse(try_from_str = addr_parser), - default_value = "127.0.0.1:8001", - help = "Gossip entrypoint address. Usually :8001" - )] - entrypoint_addr: SocketAddr, - - #[clap( - long, - default_value = "128", - required_if_eq("data-type", "random"), - help = "Size of packet to DoS with, relevant only for data-type=random" - )] - data_size: usize, - - #[clap(long, help = "Data to send [Optional]")] - data_input: Option, - - #[clap(long, help = "Just use entrypoint address directly")] - skip_gossip: bool, - - #[clap(long, help = "Allow contacting private ip addresses")] - allow_private_addr: bool, - - #[clap(flatten)] - transaction_params: TransactionParams, -} - -#[derive(Args, Serialize, Deserialize, Debug, Default)] -#[clap(rename_all = "kebab-case")] -struct TransactionParams { - #[clap( - long, - default_value = "2", - help = "Number of signatures in transaction" - )] - num_signatures: usize, - - #[clap(long, help = "Generate a valid blockhash for transaction")] - valid_blockhash: bool, - - #[clap(long, help = "Generate valid signature(s) for transaction")] - valid_signatures: bool, - - #[clap(long, help = "Generate unique transactions")] - unique_transactions: bool, - - #[clap( - long = "payer", - help = "Payer's keypair file to fund transactions [Optional]" - )] - payer_filename: Option, -} - -#[derive(ArgEnum, Clone, Eq, PartialEq)] -enum Mode { - Gossip, - Tvu, - TvuForwards, - Tpu, - TpuForwards, - Repair, - ServeRepair, - Rpc, -} - -#[derive(ArgEnum, Clone, Eq, PartialEq)] -enum DataType { - RepairHighest, - RepairShred, - RepairOrphan, - Random, - GetAccountInfo, - GetProgramAccounts, - Transaction, -} - -fn addr_parser(addr: &str) -> Result { - match solana_net_utils::parse_host_port(addr) { - Ok(v) => Ok(v), - Err(_) => Err("failed to parse entrypoint address"), - } -} - -/// input checks which are not covered by Clap -fn validate_input(params: &DosClientParameters) { - if params.mode == Mode::Rpc - && (params.data_type != DataType::GetAccountInfo - && params.data_type != DataType::GetProgramAccounts) - { - panic!("unsupported data type"); - } - - if params.data_type != DataType::Transaction { - let tp = ¶ms.transaction_params; - if tp.valid_blockhash - || tp.valid_signatures - || tp.unique_transactions - || tp.payer_filename.is_some() - { - println!("Arguments valid-blockhash, valid-sign, unique-trans, payer are ignored if data-type != transaction"); - } - } - - if params.transaction_params.payer_filename.is_some() - && params.transaction_params.valid_signatures - { - println!("Arguments valid-signatures is ignored if payer is provided"); - } -} - -fn main() { - solana_logger::setup_with_default("solana=info"); - let cmd_params = DosClientParameters::parse(); - validate_input(&cmd_params); - - let mut nodes = vec![]; - if !cmd_params.skip_gossip { - info!("Finding cluster entry: {:?}", cmd_params.entrypoint_addr); - let socket_addr_space = SocketAddrSpace::new(cmd_params.allow_private_addr); - let (gossip_nodes, _validators) = discover( - None, // keypair - Some(&cmd_params.entrypoint_addr), - None, // num_nodes - Duration::from_secs(60), // timeout - None, // find_node_by_pubkey - Some(&cmd_params.entrypoint_addr), // find_node_by_gossip_addr - None, // my_gossip_addr - 0, // my_shred_version - socket_addr_space, - ) - .unwrap_or_else(|err| { - eprintln!( - "Failed to discover {} node: {:?}", - cmd_params.entrypoint_addr, err - ); - exit(1); - }); - nodes = gossip_nodes; - } - - info!("done found {} nodes", nodes.len()); - let payer = cmd_params - .transaction_params - .payer_filename - .as_ref() - .map(|keypair_file_name| { - read_keypair_file(&keypair_file_name) - .unwrap_or_else(|_| panic!("bad keypair {:?}", keypair_file_name)) - }); - - run_dos(&nodes, 0, payer.as_ref(), cmd_params); -} - -#[cfg(test)] -pub mod test { - use { - super::*, - solana_local_cluster::{cluster::Cluster, local_cluster::LocalCluster}, - solana_sdk::timing::timestamp, - }; - - #[test] - fn test_dos() { - let nodes = [ContactInfo::new_localhost( - &solana_sdk::pubkey::new_rand(), - timestamp(), - )]; - let entrypoint_addr = nodes[0].gossip; - - run_dos( - &nodes, - 1, - None, - DosClientParameters { - entrypoint_addr, - mode: Mode::Tvu, - data_size: 10, - data_type: DataType::Random, - data_input: None, - skip_gossip: false, - allow_private_addr: false, - transaction_params: TransactionParams::default(), - }, - ); - - run_dos( - &nodes, - 1, - None, - DosClientParameters { - entrypoint_addr, - mode: Mode::Repair, - data_size: 10, - data_type: DataType::RepairHighest, - data_input: None, - skip_gossip: false, - allow_private_addr: false, - transaction_params: TransactionParams::default(), - }, - ); - - run_dos( - &nodes, - 1, - None, - DosClientParameters { - entrypoint_addr, - mode: Mode::ServeRepair, - data_size: 10, - data_type: DataType::RepairShred, - data_input: None, - skip_gossip: false, - allow_private_addr: false, - transaction_params: TransactionParams::default(), - }, - ); - } - - #[test] - fn test_dos_local_cluster_transactions() { - let num_nodes = 1; - let cluster = - LocalCluster::new_with_equal_stakes(num_nodes, 100, 3, SocketAddrSpace::Unspecified); - assert_eq!(cluster.validators.len(), num_nodes); - - let nodes = cluster.get_node_pubkeys(); - let node = cluster.get_contact_info(&nodes[0]).unwrap().clone(); - let nodes_slice = [node]; - - // send random transactions to TPU - // will be discarded on sigverify stage - run_dos( - &nodes_slice, - 1, - None, - DosClientParameters { - entrypoint_addr: cluster.entry_point_info.gossip, - mode: Mode::Tpu, - data_size: 1024, - data_type: DataType::Random, - data_input: None, - skip_gossip: false, - allow_private_addr: false, - transaction_params: TransactionParams::default(), - }, - ); - - // send transactions to TPU with 2 random signatures - // will be filtered on dedup (because transactions are not unique) - run_dos( - &nodes_slice, - 1, - None, - DosClientParameters { - entrypoint_addr: cluster.entry_point_info.gossip, - mode: Mode::Tpu, - data_size: 0, // irrelevant if not random - data_type: DataType::Transaction, - data_input: None, - skip_gossip: false, - allow_private_addr: false, - transaction_params: TransactionParams { - num_signatures: 2, - valid_blockhash: false, - valid_signatures: false, - unique_transactions: false, - payer_filename: None, - }, - }, - ); - - // send *unique* transactions to TPU with 4 random signatures - // will be discarded on banking stage in legacy.rs - // ("there should be at least 1 RW fee-payer account") - run_dos( - &nodes_slice, - 1, - None, - DosClientParameters { - entrypoint_addr: cluster.entry_point_info.gossip, - mode: Mode::Tpu, - data_size: 0, // irrelevant if not random - data_type: DataType::Transaction, - data_input: None, - skip_gossip: false, - allow_private_addr: false, - transaction_params: TransactionParams { - num_signatures: 4, - valid_blockhash: false, - valid_signatures: false, - unique_transactions: true, - payer_filename: None, - }, - }, - ); - - // send unique transactions to TPU with 2 random signatures - // will be discarded on banking stage in legacy.rs (A program cannot be a payer) - // because we haven't provided a valid payer - run_dos( - &nodes_slice, - 1, - None, - DosClientParameters { - entrypoint_addr: cluster.entry_point_info.gossip, - mode: Mode::Tpu, - data_size: 0, // irrelevant if not random - data_type: DataType::Transaction, - data_input: None, - skip_gossip: false, - allow_private_addr: false, - transaction_params: TransactionParams { - num_signatures: 2, - valid_blockhash: false, // irrelevant without valid payer, because - // it will be filtered before blockhash validity checks - valid_signatures: true, - unique_transactions: true, - payer_filename: None, - }, - }, - ); - - // send unique transaction to TPU with valid blockhash - // will be discarded due to invalid hash - run_dos( - &nodes_slice, - 1, - Some(&cluster.funding_keypair), - DosClientParameters { - entrypoint_addr: cluster.entry_point_info.gossip, - mode: Mode::Tpu, - data_size: 0, // irrelevant if not random - data_type: DataType::Transaction, - data_input: None, - skip_gossip: false, - allow_private_addr: false, - transaction_params: TransactionParams { - num_signatures: 2, - valid_blockhash: false, - valid_signatures: true, - unique_transactions: true, - payer_filename: None, - }, - }, - ); - - // send unique transaction to TPU with valid blockhash - // will fail with error processing Instruction 0: missing required signature for instruction - run_dos( - &nodes_slice, - 1, - Some(&cluster.funding_keypair), - DosClientParameters { - entrypoint_addr: cluster.entry_point_info.gossip, - mode: Mode::Tpu, - data_size: 0, // irrelevant if not random - data_type: DataType::Transaction, - data_input: None, - skip_gossip: false, - allow_private_addr: false, - transaction_params: TransactionParams { - num_signatures: 2, - valid_blockhash: true, - valid_signatures: true, - unique_transactions: true, - payer_filename: None, - }, - }, - ); - } - - #[test] - #[ignore] - fn test_dos_local_cluster() { - solana_logger::setup(); - let num_nodes = 1; - let cluster = - LocalCluster::new_with_equal_stakes(num_nodes, 100, 3, SocketAddrSpace::Unspecified); - assert_eq!(cluster.validators.len(), num_nodes); - - let nodes = cluster.get_node_pubkeys(); - let node = cluster.get_contact_info(&nodes[0]).unwrap().clone(); - - run_dos( - &[node], - 10_000_000, - Some(&cluster.funding_keypair), - DosClientParameters { - entrypoint_addr: cluster.entry_point_info.gossip, - mode: Mode::Tpu, - data_size: 0, // irrelevant if not random - data_type: DataType::Transaction, - data_input: None, - skip_gossip: false, - allow_private_addr: false, - transaction_params: TransactionParams { - num_signatures: 2, - valid_blockhash: true, - valid_signatures: true, - unique_transactions: true, - payer_filename: None, - }, - }, - ); - } -} diff --git a/log-analyzer/Cargo.toml b/log-analyzer/Cargo.toml deleted file mode 100644 index ad1a54ac9d..0000000000 --- a/log-analyzer/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -authors = ["Solana Maintainers "] -edition = "2021" -name = "solana-log-analyzer" -description = "The solana cluster network analysis tool" -version = "1.10.41" -repository = "https://github.com/solana-labs/solana" -license = "Apache-2.0" -homepage = "https://solana.com/" -publish = false - -[dependencies] -byte-unit = "4.0.14" -clap = "2.33.1" -serde = "1.0.136" -serde_json = "1.0.79" -solana-logger = { path = "../logger", version = "=1.10.41" } -solana-version = { path = "../version", version = "=0.7.0" } - -[[bin]] -name = "solana-log-analyzer" -path = "src/main.rs" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/log-analyzer/src/main.rs b/log-analyzer/src/main.rs deleted file mode 100644 index 6f0ea1f2e7..0000000000 --- a/log-analyzer/src/main.rs +++ /dev/null @@ -1,253 +0,0 @@ -#![allow(clippy::integer_arithmetic)] -extern crate byte_unit; - -use { - byte_unit::Byte, - clap::{crate_description, crate_name, value_t_or_exit, App, Arg, ArgMatches, SubCommand}, - serde::{Deserialize, Serialize}, - std::{collections::HashMap, fs, ops::Sub, path::PathBuf}, -}; - -#[derive(Deserialize, Serialize, Debug)] -struct IpAddrMapping { - private: String, - public: String, -} - -#[derive(Deserialize, Serialize, Debug)] -struct LogLine { - a: String, - b: String, - a_to_b: String, - b_to_a: String, -} - -impl Default for LogLine { - fn default() -> Self { - Self { - a: String::default(), - b: String::default(), - a_to_b: "0B".to_string(), - b_to_a: "0B".to_string(), - } - } -} - -impl LogLine { - fn output(a: &str, b: &str, v1: u128, v2: u128) -> String { - format!( - "Lost {}%, {}, ({} - {}), sender {}, receiver {}", - ((v1 - v2) * 100 / v1), - Byte::from_bytes(v1 - v2).get_appropriate_unit(true), - Byte::from_bytes(v1).get_appropriate_unit(true), - Byte::from_bytes(v2).get_appropriate_unit(true), - a, - b - ) - } -} - -impl Sub for &LogLine { - type Output = String; - - #[allow(clippy::comparison_chain)] - fn sub(self, rhs: Self) -> Self::Output { - let a_to_b = Byte::from_str(&self.a_to_b) - .expect("Failed to read a_to_b bytes") - .get_bytes(); - let b_to_a = Byte::from_str(&self.b_to_a) - .expect("Failed to read b_to_a bytes") - .get_bytes(); - let rhs_a_to_b = Byte::from_str(&rhs.a_to_b) - .expect("Failed to read a_to_b bytes") - .get_bytes(); - let rhs_b_to_a = Byte::from_str(&rhs.b_to_a) - .expect("Failed to read b_to_a bytes") - .get_bytes(); - let mut out1 = if a_to_b > rhs_b_to_a { - LogLine::output(&self.a, &self.b, a_to_b, rhs_b_to_a) - } else if a_to_b < rhs_b_to_a { - LogLine::output(&self.b, &self.a, rhs_b_to_a, a_to_b) - } else { - String::default() - }; - let out2 = if rhs_a_to_b > b_to_a { - LogLine::output(&self.a, &self.b, rhs_a_to_b, b_to_a) - } else if rhs_a_to_b < b_to_a { - LogLine::output(&self.b, &self.a, b_to_a, rhs_a_to_b) - } else { - String::default() - }; - if !out1.is_empty() && !out2.is_empty() { - out1.push('\n'); - } - out1.push_str(&out2); - out1 - } -} - -fn map_ip_address(mappings: &[IpAddrMapping], target: String) -> String { - for mapping in mappings { - if target.contains(&mapping.private) { - return target.replace(&mapping.private, mapping.public.as_str()); - } - } - target -} - -fn process_iftop_logs(matches: &ArgMatches) { - let mut map_list: Vec = vec![]; - if let ("map-IP", Some(args_matches)) = matches.subcommand() { - let mut list = args_matches - .value_of("list") - .expect("Missing list of IP address mappings") - .to_string(); - list.insert(0, '['); - let terminate_at = list - .rfind('}') - .expect("Didn't find a terminating '}' in IP list") - + 1; - let _ = list.split_off(terminate_at); - list.push(']'); - map_list = serde_json::from_str(&list).expect("Failed to parse IP address mapping list"); - }; - - let log_path = PathBuf::from(value_t_or_exit!(matches, "file", String)); - let mut log = fs::read_to_string(log_path).expect("Unable to read log file"); - log.insert(0, '['); - let terminate_at = log.rfind('}').expect("Didn't find a terminating '}'") + 1; - let _ = log.split_off(terminate_at); - log.push(']'); - let json_log: Vec = serde_json::from_str(&log).expect("Failed to parse log as JSON"); - - let mut unique_latest_logs = HashMap::new(); - - json_log.into_iter().rev().for_each(|l| { - if !l.a.is_empty() && !l.b.is_empty() && !l.a_to_b.is_empty() && !l.b_to_a.is_empty() { - let key = (l.a.clone(), l.b.clone()); - unique_latest_logs.entry(key).or_insert(l); - } - }); - let output: Vec = unique_latest_logs - .into_values() - .map(|l| { - if map_list.is_empty() { - l - } else { - LogLine { - a: map_ip_address(&map_list, l.a), - b: map_ip_address(&map_list, l.b), - a_to_b: l.a_to_b, - b_to_a: l.b_to_a, - } - } - }) - .collect(); - - println!("{}", serde_json::to_string(&output).unwrap()); -} - -fn analyze_logs(matches: &ArgMatches) { - let dir_path = PathBuf::from(value_t_or_exit!(matches, "folder", String)); - assert!( - dir_path.is_dir(), - "Need a folder that contains all log files" - ); - let list_all_diffs = matches.is_present("all"); - let files = fs::read_dir(dir_path).expect("Failed to read log folder"); - let logs: Vec<_> = files - .flat_map(|f| { - if let Ok(f) = f { - let log_str = fs::read_to_string(f.path()).expect("Unable to read log file"); - let log: Vec = - serde_json::from_str(log_str.as_str()).expect("Failed to deserialize log"); - log - } else { - vec![] - } - }) - .collect(); - let mut logs_hash = HashMap::new(); - logs.iter().for_each(|l| { - let key = (l.a.clone(), l.b.clone()); - logs_hash.entry(key).or_insert(l); - }); - - logs.iter().for_each(|l| { - let diff = logs_hash - .remove(&(l.a.clone(), l.b.clone())) - .map(|v1| { - logs_hash.remove(&(l.b.clone(), l.a.clone())).map_or( - if list_all_diffs { - v1 - &LogLine::default() - } else { - String::default() - }, - |v2| v1 - v2, - ) - }) - .unwrap_or_default(); - if !diff.is_empty() { - println!("{}", diff); - } - }); -} - -fn main() { - solana_logger::setup(); - - let matches = App::new(crate_name!()) - .about(crate_description!()) - .version(solana_version::version!()) - .subcommand( - SubCommand::with_name("iftop") - .about("Process iftop log file") - .arg( - Arg::with_name("file") - .short("f") - .long("file") - .value_name("iftop log file") - .takes_value(true) - .help("Location of the log file generated by iftop"), - ) - .subcommand( - SubCommand::with_name("map-IP") - .about("Map private IP to public IP Address") - .arg( - Arg::with_name("list") - .short("l") - .long("list") - .value_name("JSON string") - .takes_value(true) - .required(true) - .help("JSON string with a list of mapping"), - ), - ), - ) - .subcommand( - SubCommand::with_name("analyze") - .about("Compare processed network log files") - .arg( - Arg::with_name("folder") - .short("f") - .long("folder") - .value_name("DIR") - .takes_value(true) - .help("Location of processed log files"), - ) - .arg( - Arg::with_name("all") - .short("a") - .long("all") - .takes_value(false) - .help("List all differences"), - ), - ) - .get_matches(); - - match matches.subcommand() { - ("iftop", Some(args_matches)) => process_iftop_logs(args_matches), - ("analyze", Some(args_matches)) => analyze_logs(args_matches), - _ => {} - }; -} diff --git a/merkle-root-bench/Cargo.toml b/merkle-root-bench/Cargo.toml deleted file mode 100644 index be73f05170..0000000000 --- a/merkle-root-bench/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -authors = ["Solana Maintainers "] -edition = "2021" -name = "solana-merkle-root-bench" -version = "1.10.41" -repository = "https://github.com/solana-labs/solana" -license = "Apache-2.0" -homepage = "https://solana.com/" -publish = false - -[dependencies] -clap = "2.33.1" -log = "0.4.14" -solana-logger = { path = "../logger", version = "=1.10.41" } -solana-measure = { path = "../measure", version = "=1.10.41" } -solana-runtime = { path = "../runtime", version = "=1.10.41" } -solana-sdk = { path = "../sdk", version = "=1.10.41" } -solana-version = { path = "../version", version = "=0.7.0" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/merkle-root-bench/src/main.rs b/merkle-root-bench/src/main.rs deleted file mode 100644 index 26b4d8ffc3..0000000000 --- a/merkle-root-bench/src/main.rs +++ /dev/null @@ -1,54 +0,0 @@ -extern crate log; -use { - clap::{crate_description, crate_name, value_t, App, Arg}, - solana_measure::measure::Measure, - solana_runtime::accounts_hash::AccountsHash, - solana_sdk::{hash::Hash, pubkey::Pubkey}, -}; - -fn main() { - solana_logger::setup(); - - let matches = App::new(crate_name!()) - .about(crate_description!()) - .version(solana_version::version!()) - .arg( - Arg::with_name("num_accounts") - .long("num_accounts") - .takes_value(true) - .value_name("NUM_ACCOUNTS") - .help("Total number of accounts"), - ) - .arg( - Arg::with_name("iterations") - .long("iterations") - .takes_value(true) - .value_name("ITERATIONS") - .help("Number of bench iterations"), - ) - .get_matches(); - - let num_accounts = value_t!(matches, "num_accounts", usize).unwrap_or(10_000); - let iterations = value_t!(matches, "iterations", usize).unwrap_or(20); - let hashes: Vec<_> = (0..num_accounts) - .map(|_| (Pubkey::new_unique(), Hash::new_unique())) - .collect(); - let elapsed: Vec<_> = (0..iterations) - .map(|_| { - let hashes = hashes.clone(); // done outside timing - let mut time = Measure::start("compute_merkle_root"); - let fanout = 16; - AccountsHash::compute_merkle_root(hashes, fanout); - time.stop(); - time.as_us() - }) - .collect(); - - for result in &elapsed { - println!("compute_merkle_root(us),{}", result); - } - println!( - "compute_merkle_root(us) avg: {}", - elapsed.into_iter().sum::() as f64 / iterations as f64 - ); -} diff --git a/net-shaper/Cargo.toml b/net-shaper/Cargo.toml deleted file mode 100644 index 8c6e18237d..0000000000 --- a/net-shaper/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -authors = ["Solana Maintainers "] -edition = "2021" -name = "solana-net-shaper" -description = "The solana cluster network shaping tool" -version = "1.10.41" -repository = "https://github.com/solana-labs/solana" -license = "Apache-2.0" -homepage = "https://solana.com/" -publish = false - -[dependencies] -clap = "2.33.1" -rand = "0.7.0" -serde = "1.0.136" -serde_json = "1.0.79" -solana-logger = { path = "../logger", version = "=1.10.41" } - -[[bin]] -name = "solana-net-shaper" -path = "src/main.rs" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/net-shaper/src/main.rs b/net-shaper/src/main.rs deleted file mode 100644 index 47f80ebe36..0000000000 --- a/net-shaper/src/main.rs +++ /dev/null @@ -1,623 +0,0 @@ -#![allow(clippy::integer_arithmetic)] -use { - clap::{ - crate_description, crate_name, crate_version, value_t, value_t_or_exit, App, Arg, - ArgMatches, SubCommand, - }, - rand::{thread_rng, Rng}, - serde::{Deserialize, Serialize}, - std::{fs, io, path::PathBuf}, -}; - -#[derive(Deserialize, Serialize, Debug)] -struct NetworkInterconnect { - pub a: u8, - pub b: u8, - pub config: String, -} - -#[derive(Deserialize, Serialize, Debug)] -struct NetworkTopology { - pub partitions: Vec, - pub interconnects: Vec, -} - -impl Default for NetworkTopology { - fn default() -> Self { - Self { - partitions: vec![100], - interconnects: vec![], - } - } -} - -impl NetworkTopology { - pub fn verify(&self) -> bool { - let sum: u8 = self.partitions.iter().sum(); - if sum != 100 { - return false; - } - - for x in self.interconnects.iter() { - if x.a as usize > self.partitions.len() || x.b as usize > self.partitions.len() { - return false; - } - } - - true - } - - pub fn new_from_stdin() -> Self { - let mut input = String::new(); - println!("Configure partition map (must add up to 100, e.g. [70, 20, 10]):"); - let partitions_str = match io::stdin().read_line(&mut input) { - Ok(_) => input, - Err(error) => panic!("error: {}", error), - }; - - let partitions: Vec = serde_json::from_str(&partitions_str) - .expect("Failed to parse input. It must be a JSON string"); - - let mut interconnects: Vec = vec![]; - - for i in 0..partitions.len() - 1 { - for j in i + 1..partitions.len() { - println!("Configure interconnect ({} <-> {}):", i, j); - let mut input = String::new(); - let mut interconnect_config = match io::stdin().read_line(&mut input) { - Ok(_) => input, - Err(error) => panic!("error: {}", error), - }; - - if interconnect_config.ends_with('\n') { - interconnect_config.pop(); - if interconnect_config.ends_with('\r') { - interconnect_config.pop(); - } - } - - if !interconnect_config.is_empty() { - let interconnect = NetworkInterconnect { - a: i as u8, - b: j as u8, - config: interconnect_config.clone(), - }; - interconnects.push(interconnect); - let interconnect = NetworkInterconnect { - a: j as u8, - b: i as u8, - config: interconnect_config, - }; - interconnects.push(interconnect); - } - } - } - - Self { - partitions, - interconnects, - } - } - - fn new_random(max_partitions: usize, max_packet_drop: u8, max_packet_delay: u32) -> Self { - let mut rng = thread_rng(); - let num_partitions = rng.gen_range(0, max_partitions + 1); - - if num_partitions == 0 { - return NetworkTopology::default(); - } - - let mut partitions = vec![]; - let mut used_partition = 0; - for i in 0..num_partitions { - let partition = if i == num_partitions - 1 { - 100 - used_partition - } else { - rng.gen_range(0, 100 - used_partition - num_partitions + i) - }; - used_partition += partition; - partitions.push(partition as u8); - } - - let mut interconnects: Vec = vec![]; - for i in 0..partitions.len() - 1 { - for j in i + 1..partitions.len() { - let drop_config = if max_packet_drop > 0 { - let packet_drop = rng.gen_range(0, max_packet_drop + 1); - format!("loss {}% 25% ", packet_drop) - } else { - String::default() - }; - - let config = if max_packet_delay > 0 { - let packet_delay = rng.gen_range(0, max_packet_delay + 1); - format!("{}delay {}ms 10ms", drop_config, packet_delay) - } else { - drop_config - }; - - let interconnect = NetworkInterconnect { - a: i as u8, - b: j as u8, - config: config.clone(), - }; - interconnects.push(interconnect); - let interconnect = NetworkInterconnect { - a: j as u8, - b: i as u8, - config, - }; - interconnects.push(interconnect); - } - } - Self { - partitions, - interconnects, - } - } -} - -fn run( - cmd: &str, - args: &[&str], - launch_err_msg: &str, - status_err_msg: &str, - ignore_err: bool, -) -> bool { - println!("Running {:?}", std::process::Command::new(cmd).args(args)); - let output = std::process::Command::new(cmd) - .args(args) - .output() - .expect(launch_err_msg); - - if ignore_err { - return true; - } - - if !output.status.success() { - eprintln!( - "{} command failed with exit code: {}", - status_err_msg, output.status - ); - use std::str::from_utf8; - println!("stdout: {}", from_utf8(&output.stdout).unwrap_or("?")); - println!("stderr: {}", from_utf8(&output.stderr).unwrap_or("?")); - false - } else { - true - } -} - -fn insert_iptables_rule(tos: u8) -> bool { - let my_tos = tos.to_string(); - - // iptables -t mangle -A PREROUTING -p udp -j TOS --set-tos - run( - "iptables", - &[ - "-t", - "mangle", - "-A", - "OUTPUT", - "-p", - "udp", - "-j", - "TOS", - "--set-tos", - my_tos.as_str(), - ], - "Failed to add iptables rule", - "iptables", - false, - ) -} - -fn flush_iptables_rule() { - run( - "iptables", - &["-F", "-t", "mangle"], - "Failed to flush iptables", - "iptables flush", - true, - ); -} - -fn setup_ifb(interface: &str) -> bool { - // modprobe ifb numifbs=1 - run( - "modprobe", - &[ - "ifb", "numifbs=1", - ], - "Failed to load ifb module", - "modprobe ifb numifbs=1", - false - ) && - // ip link set dev ifb0 up - run( - "ip", - &[ - "link", "set", "dev", "ifb0", "up" - ], - "Failed to bring ifb0 online", - "ip link set dev ifb0 up", - false - ) && - // tc qdisc add dev handle ffff: ingress - run( - "tc", - &[ - "qdisc", "add", "dev", interface, "handle", "ffff:", "ingress" - ], - "Failed to setup ingress qdisc", - "tc qdisc add dev handle ffff: ingress", - false - ) - && - // tc filter add dev parent ffff: protocol ip u32 match u32 0 0 action mirred egress redirect dev ifb0 - run( - "tc", - &[ - "filter", "add", "dev", interface, "parent", "ffff:", "protocol", "ip", "u32", "match", "u32", "0", "0", "action", "mirred", "egress", "redirect", "dev", "ifb0" - ], - "Failed to redirect ingress traffc", - "tc filter add dev parent ffff: protocol ip u32 match u32 0 0 action mirred egress redirect dev ifb0", - false - ) -} - -fn delete_ifb(interface: &str) -> bool { - run( - "tc", - &[ - "qdisc", "delete", "dev", interface, "handle", "ffff:", "ingress", - ], - "Failed to setup ingress qdisc", - "tc qdisc delete dev handle ffff: ingress", - true, - ) && run( - "modprobe", - &["ifb", "--remove"], - "Failed to delete ifb module", - "modprobe ifb --remove", - true, - ) -} - -fn insert_tc_ifb_root(num_bands: &str) -> bool { - // tc qdisc add dev ifb0 root handle 1: prio bands - run( - "tc", - &[ - "qdisc", "add", "dev", "ifb0", "root", "handle", "1:", "prio", "bands", num_bands, - ], - "Failed to add root ifb qdisc", - "tc qdisc add dev ifb0 root handle 1: prio bands ", - false, - ) -} - -fn insert_tc_ifb_netem(class: &str, handle: &str, filter: &str) -> bool { - let mut filters: Vec<&str> = filter.split(' ').collect(); - let mut args = vec![ - "qdisc", "add", "dev", "ifb0", "parent", class, "handle", handle, "netem", - ]; - args.append(&mut filters); - // tc qdisc add dev ifb0 parent handle netem - run("tc", &args, "Failed to add tc child", "tc add child", false) -} - -fn insert_tos_ifb_filter(class: &str, tos: &str) -> bool { - // tc filter add dev ifb0 protocol ip parent 1: prio 1 u32 match ip tos 0xff flowid - run( - "tc", - &[ - "filter", "add", "dev", "ifb0", "protocol", "ip", "parent", "1:", "prio", "1", - "u32", "match", "ip", "tos", tos, "0xff", "flowid", class, - ], - "Failed to add tos filter", - "tc filter add dev ifb0 protocol ip parent 1: prio 1 u32 match ip tos 0xff flowid ", - false, - ) -} - -fn insert_default_ifb_filter(class: &str) -> bool { - // tc filter add dev ifb0 parent 1: protocol all prio 2 u32 match u32 0 0 flowid 1: - run( - "tc", - &[ - "filter", "add", "dev", "ifb0", "parent", "1:", "protocol", "all", "prio", "2", "u32", - "match", "u32", "0", "0", "flowid", class, - ], - "Failed to add catch-all filter", - "tc filter add dev ifb0 parent 1: protocol all prio 2 u32 match u32 0 0 flowid 1:", - false, - ) -} - -fn delete_all_filters(interface: &str) { - // tc filter delete dev - run( - "tc", - &["filter", "delete", "dev", interface], - "Failed to delete all filters", - "tc delete all filters", - true, - ); -} - -fn identify_my_partition(partitions: &[u8], index: u64, size: u64) -> usize { - let mut my_partition = 0; - let mut watermark = 0; - for (i, p) in partitions.iter().enumerate() { - watermark += *p; - if u64::from(watermark) >= index * 100 / size { - my_partition = i; - break; - } - } - - my_partition -} - -fn partition_id_to_tos(partition: usize) -> u8 { - if partition < 4 { - 2u8.pow(partition as u32 + 1) - } else { - 0 - } -} - -fn shape_network(matches: &ArgMatches) { - let config_path = PathBuf::from(value_t_or_exit!(matches, "file", String)); - let config = fs::read_to_string(config_path).expect("Unable to read config file"); - let topology: NetworkTopology = - serde_json::from_str(&config).expect("Failed to parse log as JSON"); - let interface = value_t_or_exit!(matches, "iface", String); - let network_size = value_t_or_exit!(matches, "size", u64); - let my_index = value_t_or_exit!(matches, "position", u64); - if !shape_network_steps(&topology, &interface, network_size, my_index) { - delete_ifb(interface.as_str()); - flush_iptables_rule(); - } -} - -fn shape_network_steps( - topology: &NetworkTopology, - interface: &str, - network_size: u64, - my_index: u64, -) -> bool { - // Integrity checks - assert!(topology.verify(), "Failed to verify the configuration file"); - assert!(my_index < network_size); - - // Figure out partition we belong in - let my_partition = identify_my_partition(&topology.partitions, my_index + 1, network_size); - - // Clear any lingering state - println!( - "my_index: {}, network_size: {}, partitions: {:?}", - my_index, network_size, topology.partitions - ); - println!("My partition is {}", my_partition); - - cleanup_network(interface); - - // Mark egress packets with our partition id - if !insert_iptables_rule(partition_id_to_tos(my_partition)) { - return false; - } - - let num_bands = topology.partitions.len() + 1; - let default_filter_class = format!("1:{}", num_bands); - if !topology.interconnects.is_empty() { - let num_bands_str = num_bands.to_string(); - // Redirect ingress traffic to the virtual interface ifb0 so we can - // apply egress rules - if !setup_ifb(interface) - // Setup root qdisc on ifb0 - || !insert_tc_ifb_root(num_bands_str.as_str()) - // Catch all so regular traffic/traffic within the same partition - // is not filtered out - || !insert_default_ifb_filter(default_filter_class.as_str()) - { - return false; - } - } - - println!("Setting up interconnects"); - for i in &topology.interconnects { - if i.b as usize == my_partition { - println!("interconnects: {:#?}", i); - let tos = partition_id_to_tos(i.a as usize); - if tos == 0 { - println!("Incorrect value of TOS/Partition in config {}", i.a); - return false; - } - let tos_string = tos.to_string(); - // First valid class is 1:1 - let class = format!("1:{}", i.a + 1); - if !insert_tc_ifb_netem(class.as_str(), tos_string.as_str(), i.config.as_str()) { - return false; - } - - if !insert_tos_ifb_filter(class.as_str(), tos_string.as_str()) { - return false; - } - } - } - - true -} - -fn parse_interface(interfaces: &str) -> &str { - for line in interfaces.lines() { - if line != "ifb0" { - return line; - } - } - - panic!("No valid interfaces"); -} - -fn cleanup_network(interface: &str) { - delete_all_filters("ifb0"); - delete_ifb(interface); - flush_iptables_rule(); -} - -fn configure(matches: &ArgMatches) { - let config = if !matches.is_present("random") { - NetworkTopology::new_from_stdin() - } else { - let max_partitions = value_t!(matches, "max-partitions", usize).unwrap_or(4); - let max_drop = value_t!(matches, "max-drop", u8).unwrap_or(100); - let max_delay = value_t!(matches, "max-delay", u32).unwrap_or(50); - NetworkTopology::new_random(max_partitions, max_drop, max_delay) - }; - - assert!(config.verify(), "Failed to verify the configuration"); - - let topology = serde_json::to_string(&config).expect("Failed to write as JSON"); - - println!("{}", topology); -} - -fn main() { - solana_logger::setup(); - - let matches = App::new(crate_name!()) - .about(crate_description!()) - .version(crate_version!()) - .subcommand( - SubCommand::with_name("shape") - .about("Shape the network using config file") - .arg( - Arg::with_name("file") - .short("f") - .long("file") - .value_name("config file") - .takes_value(true) - .required(true) - .help("Location of the network config file"), - ) - .arg( - Arg::with_name("size") - .short("s") - .long("size") - .value_name("network size") - .takes_value(true) - .required(true) - .help("Number of nodes in the network"), - ) - .arg( - Arg::with_name("iface") - .short("i") - .long("iface") - .value_name("network interface name") - .takes_value(true) - .required(true) - .help("Name of network interface"), - ) - .arg( - Arg::with_name("position") - .short("p") - .long("position") - .value_name("position of node") - .takes_value(true) - .required(true) - .help("Position of current node in the network"), - ), - ) - .subcommand( - SubCommand::with_name("cleanup") - .about("Remove the network filters using config file") - .arg( - Arg::with_name("file") - .short("f") - .long("file") - .value_name("config file") - .takes_value(true) - .required(true) - .help("Location of the network config file"), - ) - .arg( - Arg::with_name("size") - .short("s") - .long("size") - .value_name("network size") - .takes_value(true) - .required(true) - .help("Number of nodes in the network"), - ) - .arg( - Arg::with_name("iface") - .short("i") - .long("iface") - .value_name("network interface name") - .takes_value(true) - .required(true) - .help("Name of network interface"), - ) - .arg( - Arg::with_name("position") - .short("p") - .long("position") - .value_name("position of node") - .takes_value(true) - .required(true) - .help("Position of current node in the network"), - ), - ) - .subcommand( - SubCommand::with_name("configure") - .about("Generate a config file") - .arg( - Arg::with_name("random") - .short("r") - .long("random") - .required(false) - .help("Generate a random config file"), - ) - .arg( - Arg::with_name("max-partitions") - .short("p") - .long("max-partitions") - .value_name("count") - .takes_value(true) - .required(false) - .help("Maximum number of partitions. Used only with random configuration generation"), - ) - .arg( - Arg::with_name("max-drop") - .short("d") - .long("max-drop") - .value_name("percentage") - .takes_value(true) - .required(false) - .help("Maximum amount of packet drop. Used only with random configuration generation"), - ) - .arg( - Arg::with_name("max-delay") - .short("y") - .long("max-delay") - .value_name("ms") - .takes_value(true) - .required(false) - .help("Maximum amount of packet delay. Used only with random configuration generation"), - ), - ) - .get_matches(); - - match matches.subcommand() { - ("shape", Some(args_matches)) => shape_network(args_matches), - ("cleanup", Some(args_matches)) => { - let interfaces = value_t_or_exit!(args_matches, "iface", String); - let iface = parse_interface(&interfaces); - cleanup_network(iface) - } - ("configure", Some(args_matches)) => configure(args_matches), - _ => {} - }; -} diff --git a/notifier/.gitignore b/notifier/.gitignore deleted file mode 100644 index 5404b132db..0000000000 --- a/notifier/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/target/ -/farf/ diff --git a/notifier/Cargo.toml b/notifier/Cargo.toml deleted file mode 100644 index fab891df8a..0000000000 --- a/notifier/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "solana-notifier" -version = "1.10.41" -description = "Solana Notifier" -authors = ["Solana Maintainers "] -repository = "https://github.com/solana-labs/solana" -license = "Apache-2.0" -homepage = "https://solana.com/" -documentation = "https://docs.rs/solana-notifier" -edition = "2021" - -[dependencies] -log = "0.4.14" -reqwest = { version = "0.11.10", default-features = false, features = ["blocking", "brotli", "deflate", "gzip", "rustls-tls", "json"] } -serde_json = "1.0" - -[lib] -name = "solana_notifier" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/notifier/src/lib.rs b/notifier/src/lib.rs deleted file mode 100644 index b9b2529ab9..0000000000 --- a/notifier/src/lib.rs +++ /dev/null @@ -1,215 +0,0 @@ -/// To activate Slack, Discord and/or Telegram notifications, define these environment variables -/// before using the `Notifier` -/// ```bash -/// export SLACK_WEBHOOK=... -/// export DISCORD_WEBHOOK=... -/// ``` -/// -/// Telegram requires the following two variables: -/// ```bash -/// export TELEGRAM_BOT_TOKEN=... -/// export TELEGRAM_CHAT_ID=... -/// ``` -/// -/// To receive a Twilio SMS notification on failure, having a Twilio account, -/// and a sending number owned by that account, -/// define environment variable before running `solana-watchtower`: -/// ```bash -/// export TWILIO_CONFIG='ACCOUNT=,TOKEN=,TO=,FROM=' -/// ``` -use log::*; -use { - reqwest::{blocking::Client, StatusCode}, - serde_json::json, - std::{env, str::FromStr, thread::sleep, time::Duration}, -}; - -struct TelegramWebHook { - bot_token: String, - chat_id: String, -} - -#[derive(Debug, Default)] -struct TwilioWebHook { - account: String, - token: String, - to: String, - from: String, -} - -impl TwilioWebHook { - fn complete(&self) -> bool { - !(self.account.is_empty() - || self.token.is_empty() - || self.to.is_empty() - || self.from.is_empty()) - } -} - -fn get_twilio_config() -> Result, String> { - let config_var = env::var("TWILIO_CONFIG"); - - if config_var.is_err() { - info!("Twilio notifications disabled"); - return Ok(None); - } - - let mut config = TwilioWebHook::default(); - - for pair in config_var.unwrap().split(',') { - let nv: Vec<_> = pair.split('=').collect(); - if nv.len() != 2 { - return Err(format!("TWILIO_CONFIG is invalid: '{}'", pair)); - } - let v = nv[1].to_string(); - match nv[0] { - "ACCOUNT" => config.account = v, - "TOKEN" => config.token = v, - "TO" => config.to = v, - "FROM" => config.from = v, - _ => return Err(format!("TWILIO_CONFIG is invalid: '{}'", pair)), - } - } - - if !config.complete() { - return Err("TWILIO_CONFIG is incomplete".to_string()); - } - Ok(Some(config)) -} - -enum NotificationType { - Discord(String), - Slack(String), - Telegram(TelegramWebHook), - Twilio(TwilioWebHook), - Log(Level), -} - -pub struct Notifier { - client: Client, - notifiers: Vec, -} - -impl Default for Notifier { - fn default() -> Self { - Self::new("") - } -} - -impl Notifier { - pub fn new(env_prefix: &str) -> Self { - info!("Initializing {}Notifier", env_prefix); - - let mut notifiers = vec![]; - - if let Ok(webhook) = env::var(format!("{}DISCORD_WEBHOOK", env_prefix)) { - notifiers.push(NotificationType::Discord(webhook)); - } - if let Ok(webhook) = env::var(format!("{}SLACK_WEBHOOK", env_prefix)) { - notifiers.push(NotificationType::Slack(webhook)); - } - - if let (Ok(bot_token), Ok(chat_id)) = ( - env::var(format!("{}TELEGRAM_BOT_TOKEN", env_prefix)), - env::var(format!("{}TELEGRAM_CHAT_ID", env_prefix)), - ) { - notifiers.push(NotificationType::Telegram(TelegramWebHook { - bot_token, - chat_id, - })); - } - - if let Ok(Some(webhook)) = get_twilio_config() { - notifiers.push(NotificationType::Twilio(webhook)); - } - - if let Ok(log_level) = env::var(format!("{}LOG_NOTIFIER_LEVEL", env_prefix)) { - match Level::from_str(&log_level) { - Ok(level) => notifiers.push(NotificationType::Log(level)), - Err(e) => warn!( - "could not parse specified log notifier level string ({}): {}", - log_level, e - ), - } - } - - info!("{} notifiers", notifiers.len()); - - Notifier { - client: Client::new(), - notifiers, - } - } - - pub fn is_empty(&self) -> bool { - self.notifiers.is_empty() - } - - pub fn send(&self, msg: &str) { - for notifier in &self.notifiers { - match notifier { - NotificationType::Discord(webhook) => { - for line in msg.split('\n') { - // Discord rate limiting is aggressive, limit to 1 message a second - sleep(Duration::from_millis(1000)); - - info!("Sending {}", line); - let data = json!({ "content": line }); - - loop { - let response = self.client.post(webhook).json(&data).send(); - - if let Err(err) = response { - warn!("Failed to send Discord message: \"{}\": {:?}", line, err); - break; - } else if let Ok(response) = response { - info!("response status: {}", response.status()); - if response.status() == StatusCode::TOO_MANY_REQUESTS { - warn!("rate limited!..."); - warn!("response text: {:?}", response.text()); - sleep(Duration::from_secs(2)); - } else { - break; - } - } - } - } - } - NotificationType::Slack(webhook) => { - let data = json!({ "text": msg }); - if let Err(err) = self.client.post(webhook).json(&data).send() { - warn!("Failed to send Slack message: {:?}", err); - } - } - - NotificationType::Telegram(TelegramWebHook { chat_id, bot_token }) => { - let data = json!({ "chat_id": chat_id, "text": msg }); - let url = format!("https://api.telegram.org/bot{}/sendMessage", bot_token); - - if let Err(err) = self.client.post(url).json(&data).send() { - warn!("Failed to send Telegram message: {:?}", err); - } - } - - NotificationType::Twilio(TwilioWebHook { - account, - token, - to, - from, - }) => { - let url = format!( - "https://{}:{}@api.twilio.com/2010-04-01/Accounts/{}/Messages.json", - account, token, account - ); - let params = [("To", to), ("From", from), ("Body", &msg.to_string())]; - if let Err(err) = self.client.post(url).form(¶ms).send() { - warn!("Failed to send Twilio message: {:?}", err); - } - } - NotificationType::Log(level) => { - log!(*level, "{}", msg) - } - } - } - } -} diff --git a/poh-bench/Cargo.toml b/poh-bench/Cargo.toml deleted file mode 100644 index 645fc2d202..0000000000 --- a/poh-bench/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -authors = ["Solana Maintainers "] -edition = "2021" -name = "solana-poh-bench" -version = "1.10.41" -repository = "https://github.com/solana-labs/solana" -license = "Apache-2.0" -homepage = "https://solana.com/" -documentation = "https://docs.rs/solana-poh-bench" -publish = false - -[dependencies] -clap = "2.33.1" -log = "0.4.14" -rand = "0.7.0" -rayon = "1.5.1" -solana-entry = { path = "../entry", version = "=1.10.41" } -solana-logger = { path = "../logger", version = "=1.10.41" } -solana-measure = { path = "../measure", version = "=1.10.41" } -solana-perf = { path = "../perf", version = "=1.10.41" } -solana-sdk = { path = "../sdk", version = "=1.10.41" } -solana-version = { path = "../version", version = "=0.7.0" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/poh-bench/src/main.rs b/poh-bench/src/main.rs deleted file mode 100644 index 86328ef5bd..0000000000 --- a/poh-bench/src/main.rs +++ /dev/null @@ -1,148 +0,0 @@ -#![allow(clippy::integer_arithmetic)] -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -use solana_entry::entry::{self, create_ticks, init_poh, EntrySlice, VerifyRecyclers}; -#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] -use solana_entry::entry::{create_ticks, init_poh, EntrySlice, VerifyRecyclers}; -use { - clap::{crate_description, crate_name, value_t, App, Arg}, - solana_measure::measure::Measure, - solana_perf::perf_libs, - solana_sdk::hash::hash, -}; - -fn main() { - solana_logger::setup(); - - let matches = App::new(crate_name!()) - .about(crate_description!()) - .version(solana_version::version!()) - .arg( - Arg::with_name("max_num_entries") - .long("max-num-entries") - .takes_value(true) - .value_name("SIZE") - .help("Number of entries."), - ) - .arg( - Arg::with_name("start_num_entries") - .long("start-num-entries") - .takes_value(true) - .value_name("SIZE") - .help("Packets per chunk"), - ) - .arg( - Arg::with_name("hashes_per_tick") - .long("hashes-per-tick") - .takes_value(true) - .value_name("SIZE") - .help("hashes per tick"), - ) - .arg( - Arg::with_name("num_transactions_per_entry") - .long("num-transactions-per-entry") - .takes_value(true) - .value_name("NUM") - .help("Skip transaction sanity execution"), - ) - .arg( - Arg::with_name("iterations") - .long("iterations") - .takes_value(true) - .help("Number of iterations"), - ) - .arg( - Arg::with_name("num_threads") - .long("num-threads") - .takes_value(true) - .help("Number of threads"), - ) - .arg( - Arg::with_name("cuda") - .long("cuda") - .takes_value(false) - .help("Use cuda"), - ) - .get_matches(); - - let max_num_entries = value_t!(matches, "max_num_entries", u64).unwrap_or(64); - let start_num_entries = value_t!(matches, "start_num_entries", u64).unwrap_or(max_num_entries); - let iterations = value_t!(matches, "iterations", usize).unwrap_or(10); - let hashes_per_tick = value_t!(matches, "hashes_per_tick", u64).unwrap_or(10_000); - let start_hash = hash(&[1, 2, 3, 4]); - let ticks = create_ticks(max_num_entries, hashes_per_tick, start_hash); - let mut num_entries = start_num_entries as usize; - if matches.is_present("cuda") { - perf_libs::init_cuda(); - } - init_poh(); - while num_entries <= max_num_entries as usize { - let mut time = Measure::start("time"); - for _ in 0..iterations { - assert!(ticks[..num_entries] - .verify_cpu_generic(&start_hash) - .finish_verify()); - } - time.stop(); - println!( - "{},cpu_generic,{}", - num_entries, - time.as_us() / iterations as u64 - ); - - // A target_arch check is required here since calling - // is_x86_feature_detected from a non-x86_64 arch results in a build - // error. - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - if is_x86_feature_detected!("avx2") && entry::api().is_some() { - let mut time = Measure::start("time"); - for _ in 0..iterations { - assert!(ticks[..num_entries] - .verify_cpu_x86_simd(&start_hash, 8) - .finish_verify()); - } - time.stop(); - println!( - "{},cpu_simd_avx2,{}", - num_entries, - time.as_us() / iterations as u64 - ); - } - - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - if is_x86_feature_detected!("avx512f") && entry::api().is_some() { - let mut time = Measure::start("time"); - for _ in 0..iterations { - assert!(ticks[..num_entries] - .verify_cpu_x86_simd(&start_hash, 16) - .finish_verify()); - } - time.stop(); - println!( - "{},cpu_simd_avx512,{}", - num_entries, - time.as_us() / iterations as u64 - ); - } - } - - if perf_libs::api().is_some() { - let mut time = Measure::start("time"); - let recyclers = VerifyRecyclers::default(); - for _ in 0..iterations { - assert!(ticks[..num_entries] - .start_verify(&start_hash, recyclers.clone()) - .finish_verify()); - } - time.stop(); - println!( - "{},gpu_cuda,{}", - num_entries, - time.as_us() / iterations as u64 - ); - } - - println!(); - num_entries *= 2; - } -} diff --git a/programs/budget/Cargo.toml b/programs/budget/Cargo.toml deleted file mode 100644 index 4863a2523b..0000000000 --- a/programs/budget/Cargo.toml +++ /dev/null @@ -1,31 +0,0 @@ -[package] -name = "solana-budget-program" -version = "1.6.14" -description = "Solana Budget program" -authors = ["Solana Maintainers "] -repository = "https://github.com/solana-labs/solana" -license = "Apache-2.0" -homepage = "https://solana.com/" -documentation = "https://docs.rs/solana-budget-program" -edition = "2018" - -[dependencies] -bincode = "1.3.1" -chrono = { version = "0.4.11", features = ["serde"] } -log = "0.4.11" -num-derive = "0.3" -num-traits = "0.2" -serde = "1.0.122" -serde_derive = "1.0.103" -solana-sdk = { path = "../../sdk", version = "=1.10.41" } -thiserror = "1.0" - -[dev-dependencies] -solana-runtime = { path = "../../runtime", version = "=1.10.41" } - -[lib] -crate-type = ["lib", "cdylib"] -name = "solana_budget_program" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/rbpf-cli/Cargo.lock b/rbpf-cli/Cargo.lock deleted file mode 100644 index 5c203a182e..0000000000 --- a/rbpf-cli/Cargo.lock +++ /dev/null @@ -1,510 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -[[package]] -name = "ascii" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" - -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi", - "libc", - "winapi", -] - -[[package]] -name = "autocfg" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" - -[[package]] -name = "bitflags" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" - -[[package]] -name = "byteorder" -version = "1.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" - -[[package]] -name = "cfg-if" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" - -[[package]] -name = "clap" -version = "3.0.0-beta.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd1061998a501ee7d4b6d449020df3266ca3124b941ec56cf2005c3779ca142" -dependencies = [ - "atty", - "bitflags", - "clap_derive", - "indexmap", - "lazy_static", - "os_str_bytes", - "strsim", - "termcolor", - "textwrap", - "unicode-width", - "vec_map", -] - -[[package]] -name = "clap_derive" -version = "3.0.0-beta.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "370f715b81112975b1b69db93e0b56ea4cd4e5002ac43b2da8474106a54096a1" -dependencies = [ - "heck", - "proc-macro-error", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "combine" -version = "3.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3da6baa321ec19e1cc41d31bf599f00c783d0517095cdaf0332e3fe8d20680" -dependencies = [ - "ascii", - "byteorder", - "either", - "memchr", - "unreachable", -] - -[[package]] -name = "either" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" - -[[package]] -name = "getrandom" -version = "0.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", -] - -[[package]] -name = "goblin" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c69552f48b18aa6102ce0c82dd9bc9d3f8af5fc0a5797069b1b466b90570e39c" -dependencies = [ - "log", - "plain", - "scroll", -] - -[[package]] -name = "hash32" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4041af86e63ac4298ce40e5cca669066e75b6f1aa3390fe2561ffa5e1d9f4cc" -dependencies = [ - "byteorder", -] - -[[package]] -name = "hashbrown" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" - -[[package]] -name = "heck" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" -dependencies = [ - "unicode-segmentation", -] - -[[package]] -name = "hermit-abi" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" -dependencies = [ - "libc", -] - -[[package]] -name = "indexmap" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55e2e4c765aa53a0424761bf9f41aa7a6ac1efa87238f59560640e27fca028f2" -dependencies = [ - "autocfg", - "hashbrown", -] - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "libc" -version = "0.2.80" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614" - -[[package]] -name = "log" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "memchr" -version = "2.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" - -[[package]] -name = "os_str_bytes" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afb2e1c3ee07430c2cf76151675e583e0f19985fa6efae47d6848a3e2c824f85" - -[[package]] -name = "plain" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" - -[[package]] -name = "ppv-lite86" -version = "0.2.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" - -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - -[[package]] -name = "proc-macro2" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" -dependencies = [ - "unicode-xid", -] - -[[package]] -name = "quote" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom", - "libc", - "rand_chacha", - "rand_core", - "rand_hc", - "rand_pcg", -] - -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom", -] - -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core", -] - -[[package]] -name = "rand_pcg" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" -dependencies = [ - "rand_core", -] - -[[package]] -name = "rbpf_cli" -version = "0.2.8" -dependencies = [ - "clap", - "solana_rbpf", - "test_utils", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232" - -[[package]] -name = "scroll" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda28d4b4830b807a8b43f7b0e6b5df875311b3e7621d84577188c175b6ec1ec" -dependencies = [ - "scroll_derive", -] - -[[package]] -name = "scroll_derive" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b12bd20b94c7cdfda8c7ba9b92ad0d9a56e3fa018c25fca83b51aa664c9b4c0d" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "solana_rbpf" -version = "0.2.8" -dependencies = [ - "byteorder", - "combine", - "goblin", - "hash32", - "libc", - "log", - "rand", - "rustc-demangle", - "scroll", - "thiserror", - "time", -] - -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - -[[package]] -name = "syn" -version = "1.0.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac" -dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", -] - -[[package]] -name = "termcolor" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "test_utils" -version = "0.2.8" -dependencies = [ - "libc", - "solana_rbpf", -] - -[[package]] -name = "textwrap" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "203008d98caf094106cfaba70acfed15e18ed3ddb7d94e49baec153a2b462789" -dependencies = [ - "unicode-width", -] - -[[package]] -name = "thiserror" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e9ae34b84616eedaaf1e9dd6026dbe00dcafa92aa0c8077cb69df1fcfe5e53e" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba20f23e85b10754cd195504aebf6a27e2e6cbe28c17778a0c930724628dd56" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "time" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" -dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", -] - -[[package]] -name = "unicode-segmentation" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" - -[[package]] -name = "unicode-width" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" - -[[package]] -name = "unicode-xid" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" - -[[package]] -name = "unreachable" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" -dependencies = [ - "void", -] - -[[package]] -name = "vec_map" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" - -[[package]] -name = "version_check" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" - -[[package]] -name = "void" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" - -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-util" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" -dependencies = [ - "winapi", -] - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/rbpf-cli/Cargo.toml b/rbpf-cli/Cargo.toml deleted file mode 100644 index 67b3d3cc71..0000000000 --- a/rbpf-cli/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "rbpf-cli" -version = "1.10.41" -description = "CLI to test and analyze eBPF programs" -authors = ["Solana Maintainers "] -repository = "https://github.com/solana-labs/rbpf" -homepage = "https://solana.com/" -keywords = ["BPF", "eBPF", "interpreter", "JIT"] -edition = "2021" -publish = false - -[dependencies] -clap = { version = "3.1.5", features = ["cargo"] } -serde = "1.0.136" -serde_json = "1.0.79" -solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "=1.10.41" } -solana-logger = { path = "../logger", version = "=1.10.41" } -solana-program-runtime = { path = "../program-runtime", version = "=1.10.41" } -solana-sdk = { path = "../sdk", version = "=1.10.41" } -solana_rbpf = "=0.2.24" diff --git a/rbpf-cli/src/main.rs b/rbpf-cli/src/main.rs deleted file mode 100644 index b304dbeeca..0000000000 --- a/rbpf-cli/src/main.rs +++ /dev/null @@ -1,385 +0,0 @@ -use { - clap::{crate_version, Arg, Command}, - serde::{Deserialize, Serialize}, - serde_json::Result, - solana_bpf_loader_program::{ - create_vm, serialization::serialize_parameters, syscalls::register_syscalls, BpfError, - ThisInstructionMeter, - }, - solana_program_runtime::invoke_context::{prepare_mock_invoke_context, InvokeContext}, - solana_rbpf::{ - assembler::assemble, - elf::Executable, - static_analysis::Analysis, - verifier::check, - vm::{Config, DynamicAnalysis}, - }, - solana_sdk::{ - account::AccountSharedData, bpf_loader, instruction::AccountMeta, pubkey::Pubkey, - transaction_context::TransactionContext, - }, - std::{ - fmt::{Debug, Formatter}, - fs::File, - io::{Read, Seek, SeekFrom}, - path::Path, - time::{Duration, Instant}, - }, -}; - -#[derive(Serialize, Deserialize, Debug)] -struct Account { - key: Pubkey, - owner: Pubkey, - is_signer: bool, - is_writable: bool, - lamports: u64, - data: Vec, -} -#[derive(Serialize, Deserialize)] -struct Input { - accounts: Vec, - instruction_data: Vec, -} -fn load_accounts(path: &Path) -> Result { - let file = File::open(path).unwrap(); - let input: Input = serde_json::from_reader(file)?; - eprintln!("Program input:"); - eprintln!("accounts {:?}", &input.accounts); - eprintln!("instruction_data {:?}", &input.instruction_data); - eprintln!("----------------------------------------"); - Ok(input) -} - -fn main() { - solana_logger::setup(); - let matches = Command::new("Solana BPF CLI") - .version(crate_version!()) - .author("Solana Maintainers ") - .help( - r##"CLI to test and analyze eBPF programs. - -The tool executes eBPF programs in a mocked environment. -Some features, such as sysvars syscall and CPI, are not -available for the programs executed by the CLI tool. - -The input data for a program execution have to be in JSON format -and the following fields are required -{ - "accounts": [ - { - "key": [ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - ], - "owner": [ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - ], - "is_signer": false, - "is_writable": true, - "lamports": 1000, - "data": [0, 0, 0, 3] - } - ], - "instruction_data": [] -} -"##, - ) - .arg( - Arg::new("PROGRAM") - .help( - "Program file to use. This is either an ELF shared-object file to be executed, \ - or an assembly file to be assembled and executed.", - ) - .required(true) - .index(1) - ) - .arg( - Arg::new("input") - .help( - "Input for the program to run on, where FILE is a name of a JSON file \ -with input data, or BYTES is the number of 0-valued bytes to allocate for program parameters", - ) - .short('i') - .long("input") - .value_name("FILE / BYTES") - .takes_value(true) - .default_value("0"), - ) - .arg( - Arg::new("memory") - .help("Heap memory for the program to run on") - .short('m') - .long("memory") - .value_name("BYTES") - .takes_value(true) - .default_value("0"), - ) - .arg( - Arg::new("use") - .help( - "Method of execution to use, where 'cfg' generates Control Flow Graph \ -of the program, 'disassembler' dumps disassembled code of the program, 'interpreter' runs \ -the program in the virtual machine's interpreter, and 'jit' precompiles the program to \ -native machine code before execting it in the virtual machine.", - ) - .short('u') - .long("use") - .takes_value(true) - .value_name("VALUE") - .possible_values(["cfg", "disassembler", "interpreter", "jit"]) - .default_value("jit"), - ) - .arg( - Arg::new("instruction limit") - .help("Limit the number of instructions to execute") - .short('l') - .long("limit") - .takes_value(true) - .value_name("COUNT") - .default_value(&std::i64::MAX.to_string()), - ) - .arg( - Arg::new("trace") - .help("Output trace to 'trace.out' file using tracing instrumentation") - .short('t') - .long("trace"), - ) - .arg( - Arg::new("profile") - .help("Output profile to 'profile.dot' file using tracing instrumentation") - .short('p') - .long("profile"), - ) - .arg( - Arg::new("verify") - .help("Run the verifier before execution or disassembly") - .short('v') - .long("verify"), - ) - .arg( - Arg::new("output_format") - .help("Return information in specified output format") - .long("output") - .value_name("FORMAT") - .global(true) - .takes_value(true) - .possible_values(&["json", "json-compact"]), - ) - .get_matches(); - - let config = Config { - enable_instruction_tracing: matches.is_present("trace") || matches.is_present("profile"), - enable_symbol_and_section_labels: true, - ..Config::default() - }; - let loader_id = bpf_loader::id(); - let mut transaction_accounts = vec![ - ( - loader_id, - AccountSharedData::new(0, 0, &solana_sdk::native_loader::id()), - ), - ( - Pubkey::new_unique(), - AccountSharedData::new(0, 0, &loader_id), - ), - ]; - let mut instruction_accounts = Vec::new(); - let instruction_data = match matches.value_of("input").unwrap().parse::() { - Ok(allocation_size) => { - let pubkey = Pubkey::new_unique(); - transaction_accounts.push(( - pubkey, - AccountSharedData::new(0, allocation_size, &Pubkey::new_unique()), - )); - instruction_accounts.push(AccountMeta { - pubkey, - is_signer: false, - is_writable: true, - }); - vec![] - } - Err(_) => { - let input = load_accounts(Path::new(matches.value_of("input").unwrap())).unwrap(); - for account_info in input.accounts { - let mut account = AccountSharedData::new( - account_info.lamports, - account_info.data.len(), - &account_info.owner, - ); - account.set_data(account_info.data); - instruction_accounts.push(AccountMeta { - pubkey: account_info.key, - is_signer: account_info.is_signer, - is_writable: account_info.is_writable, - }); - transaction_accounts.push((account_info.key, account)); - } - input.instruction_data - } - }; - let program_indices = [0, 1]; - let preparation = - prepare_mock_invoke_context(transaction_accounts, instruction_accounts, &program_indices); - let mut transaction_context = TransactionContext::new(preparation.transaction_accounts, 1, 1); - let mut invoke_context = InvokeContext::new_mock(&mut transaction_context, &[]); - invoke_context - .push( - &preparation.instruction_accounts, - &program_indices, - &instruction_data, - ) - .unwrap(); - let (mut parameter_bytes, account_lengths) = serialize_parameters( - invoke_context.transaction_context, - invoke_context - .transaction_context - .get_current_instruction_context() - .unwrap(), - ) - .unwrap(); - let compute_meter = invoke_context.get_compute_meter(); - let mut instruction_meter = ThisInstructionMeter { compute_meter }; - - let program = matches.value_of("PROGRAM").unwrap(); - let mut file = File::open(Path::new(program)).unwrap(); - let mut magic = [0u8; 4]; - file.read_exact(&mut magic).unwrap(); - file.seek(SeekFrom::Start(0)).unwrap(); - let mut contents = Vec::new(); - file.read_to_end(&mut contents).unwrap(); - let syscall_registry = register_syscalls(&mut invoke_context).unwrap(); - let mut executable = if magic == [0x7f, 0x45, 0x4c, 0x46] { - Executable::::from_elf( - &contents, - None, - config, - syscall_registry, - ) - .map_err(|err| format!("Executable constructor failed: {:?}", err)) - } else { - assemble::( - std::str::from_utf8(contents.as_slice()).unwrap(), - None, - config, - syscall_registry, - ) - } - .unwrap(); - - if matches.is_present("verify") { - let text_bytes = executable.get_text_bytes().1; - check(text_bytes, &config).unwrap(); - } - Executable::::jit_compile(&mut executable).unwrap(); - let mut analysis = LazyAnalysis::new(&executable); - - match matches.value_of("use") { - Some("cfg") => { - let mut file = File::create("cfg.dot").unwrap(); - analysis - .analyze() - .visualize_graphically(&mut file, None) - .unwrap(); - return; - } - Some("disassembler") => { - let stdout = std::io::stdout(); - analysis.analyze().disassemble(&mut stdout.lock()).unwrap(); - return; - } - _ => {} - } - - let mut vm = create_vm( - &executable, - parameter_bytes.as_slice_mut(), - &mut invoke_context, - &account_lengths, - ) - .unwrap(); - let start_time = Instant::now(); - let result = if matches.value_of("use").unwrap() == "interpreter" { - vm.execute_program_interpreted(&mut instruction_meter) - } else { - vm.execute_program_jit(&mut instruction_meter) - }; - let duration = Instant::now() - start_time; - - let output = Output { - result: format!("{:?}", result), - instruction_count: vm.get_total_instruction_count(), - execution_time: duration, - }; - match matches.value_of("output_format") { - Some("json") => { - println!("{}", serde_json::to_string_pretty(&output).unwrap()); - } - Some("json-compact") => { - println!("{}", serde_json::to_string(&output).unwrap()); - } - _ => { - println!("Program output:"); - println!("{:?}", output); - } - } - - if matches.is_present("trace") { - eprintln!("Trace is saved in trace.out"); - let mut file = File::create("trace.out").unwrap(); - vm.get_tracer() - .write(&mut file, analysis.analyze()) - .unwrap(); - } - if matches.is_present("profile") { - eprintln!("Profile is saved in profile.dot"); - let tracer = &vm.get_tracer(); - let analysis = analysis.analyze(); - let dynamic_analysis = DynamicAnalysis::new(tracer, analysis); - let mut file = File::create("profile.dot").unwrap(); - analysis - .visualize_graphically(&mut file, Some(&dynamic_analysis)) - .unwrap(); - } -} - -#[derive(Serialize)] -struct Output { - result: String, - instruction_count: u64, - execution_time: Duration, -} - -impl Debug for Output { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - writeln!(f, "Result: {}", self.result)?; - writeln!(f, "Instruction Count: {}", self.instruction_count)?; - writeln!(f, "Execution time: {} us", self.execution_time.as_micros())?; - Ok(()) - } -} - -// Replace with std::lazy::Lazy when stabilized. -// https://github.com/rust-lang/rust/issues/74465 -struct LazyAnalysis<'a> { - analysis: Option>, - executable: &'a Executable, -} - -impl<'a> LazyAnalysis<'a> { - fn new(executable: &'a Executable) -> Self { - Self { - analysis: None, - executable, - } - } - - fn analyze(&mut self) -> &Analysis { - if let Some(ref analysis) = self.analysis { - return analysis; - } - self.analysis - .insert(Analysis::from_executable(self.executable)) - } -} diff --git a/runtime/store-tool/Cargo.toml b/runtime/store-tool/Cargo.toml deleted file mode 100644 index 9cb94f875d..0000000000 --- a/runtime/store-tool/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -authors = ["Solana Maintainers "] -edition = "2021" -name = "solana-store-tool" -description = "Tool to inspect append vecs" -version = "1.10.41" -repository = "https://github.com/solana-labs/solana" -license = "Apache-2.0" -homepage = "https://solana.com/" -publish = false - -[dependencies] -clap = "2.33.1" -log = { version = "0.4.14" } -solana-logger = { path = "../../logger", version = "=1.10.41" } -solana-runtime = { path = "..", version = "=1.10.41" } -solana-version = { path = "../../version", version = "=0.7.0" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/runtime/store-tool/src/main.rs b/runtime/store-tool/src/main.rs deleted file mode 100644 index 7c6c162ce4..0000000000 --- a/runtime/store-tool/src/main.rs +++ /dev/null @@ -1,47 +0,0 @@ -use { - clap::{crate_description, crate_name, value_t_or_exit, App, Arg}, - log::*, - solana_runtime::append_vec::AppendVec, -}; - -fn main() { - solana_logger::setup_with_default("solana=info"); - let matches = App::new(crate_name!()) - .about(crate_description!()) - .version(solana_version::version!()) - .arg( - Arg::with_name("file") - .long("file") - .takes_value(true) - .value_name("") - .help("store to open"), - ) - .arg( - Arg::with_name("len") - .long("len") - .takes_value(true) - .value_name("LEN") - .help("len of store to open"), - ) - .get_matches(); - - let file = value_t_or_exit!(matches, "file", String); - let len = value_t_or_exit!(matches, "len", usize); - let (mut store, num_accounts) = AppendVec::new_from_file(file, len).expect("should succeed"); - store.set_no_remove_on_drop(); - info!( - "store: len: {} capacity: {} accounts: {}", - store.len(), - store.capacity(), - num_accounts, - ); - for account in store.accounts(0) { - info!( - " account: {:?} version: {} data: {} hash: {:?}", - account.meta.pubkey, account.meta.write_version, account.meta.data_len, account.hash - ); - } -} - -#[cfg(test)] -pub mod test {} diff --git a/scripts/Cargo.toml b/scripts/Cargo.toml deleted file mode 100644 index 01944a758c..0000000000 --- a/scripts/Cargo.toml +++ /dev/null @@ -1,22 +0,0 @@ -[package] -authors = ["Solana Maintainers "] -edition = "2018" -name = "solana-scripts" -description = "Blockchain, Rebuilt for Scale" -version = "1.6.14" -repository = "https://github.com/solana-labs/solana" -license = "Apache-2.0" -homepage = "https://solana.com/" -publish = false - -[dependencies] -csv = "1.1" -serde = { version = "1.0.122", features = ["derive"] } - -[[bin]] -name = "solana-csv-to-validator-infos" -path = "src/csv_to_validator_infos.rs" - - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/stake-accounts/Cargo.toml b/stake-accounts/Cargo.toml deleted file mode 100644 index aa04ca9b7f..0000000000 --- a/stake-accounts/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "solana-stake-accounts" -description = "Blockchain, Rebuilt for Scale" -authors = ["Solana Maintainers "] -edition = "2021" -version = "1.10.41" -repository = "https://github.com/solana-labs/solana" -license = "Apache-2.0" -homepage = "https://solana.com/" -documentation = "https://docs.rs/solana-stake-accounts" - -[dependencies] -clap = "2.33.1" -solana-clap-utils = { path = "../clap-utils", version = "=1.10.41" } -solana-cli-config = { path = "../cli-config", version = "=1.10.41" } -solana-client = { path = "../client", version = "=1.10.41" } -solana-remote-wallet = { path = "../remote-wallet", version = "=1.10.41" } -solana-sdk = { path = "../sdk", version = "=1.10.41" } -solana-stake-program = { path = "../programs/stake", version = "=1.10.41" } - -[dev-dependencies] -solana-runtime = { path = "../runtime", version = "=1.10.41" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/stake-accounts/src/arg_parser.rs b/stake-accounts/src/arg_parser.rs deleted file mode 100644 index a29b880cbe..0000000000 --- a/stake-accounts/src/arg_parser.rs +++ /dev/null @@ -1,376 +0,0 @@ -use { - crate::args::{ - Args, AuthorizeArgs, Command, CountArgs, MoveArgs, NewArgs, QueryArgs, RebaseArgs, - SetLockupArgs, - }, - clap::{value_t, value_t_or_exit, App, Arg, ArgMatches, SubCommand}, - solana_clap_utils::{ - input_parsers::unix_timestamp_from_rfc3339_datetime, - input_validators::{is_amount, is_rfc3339_datetime, is_valid_pubkey, is_valid_signer}, - }, - solana_cli_config::CONFIG_FILE, - solana_sdk::native_token::sol_to_lamports, - std::{ffi::OsString, process::exit}, -}; - -fn fee_payer_arg<'a, 'b>() -> Arg<'a, 'b> { - solana_clap_utils::fee_payer::fee_payer_arg().required(true) -} - -fn funding_keypair_arg<'a, 'b>() -> Arg<'a, 'b> { - Arg::with_name("funding_keypair") - .required(true) - .takes_value(true) - .value_name("FUNDING_KEYPAIR") - .validator(is_valid_signer) - .help("Keypair to fund accounts") -} - -fn base_pubkey_arg<'a, 'b>() -> Arg<'a, 'b> { - Arg::with_name("base_pubkey") - .required(true) - .takes_value(true) - .value_name("BASE_PUBKEY") - .validator(is_valid_pubkey) - .help("Public key which stake account addresses are derived from") -} - -fn custodian_arg<'a, 'b>() -> Arg<'a, 'b> { - Arg::with_name("custodian") - .required(true) - .takes_value(true) - .value_name("KEYPAIR") - .validator(is_valid_signer) - .help("Authority to modify lockups") -} - -fn new_custodian_arg<'a, 'b>() -> Arg<'a, 'b> { - Arg::with_name("new_custodian") - .takes_value(true) - .value_name("PUBKEY") - .validator(is_valid_pubkey) - .help("New authority to modify lockups") -} - -fn new_base_keypair_arg<'a, 'b>() -> Arg<'a, 'b> { - Arg::with_name("new_base_keypair") - .required(true) - .takes_value(true) - .value_name("NEW_BASE_KEYPAIR") - .validator(is_valid_signer) - .help("New keypair which stake account addresses are derived from") -} - -fn stake_authority_arg<'a, 'b>() -> Arg<'a, 'b> { - Arg::with_name("stake_authority") - .long("stake-authority") - .required(true) - .takes_value(true) - .value_name("KEYPAIR") - .validator(is_valid_signer) - .help("Stake authority") -} - -fn withdraw_authority_arg<'a, 'b>() -> Arg<'a, 'b> { - Arg::with_name("withdraw_authority") - .long("withdraw-authority") - .required(true) - .takes_value(true) - .value_name("KEYPAIR") - .validator(is_valid_signer) - .help("Withdraw authority") -} - -fn new_stake_authority_arg<'a, 'b>() -> Arg<'a, 'b> { - Arg::with_name("new_stake_authority") - .long("new-stake-authority") - .required(true) - .takes_value(true) - .value_name("PUBKEY") - .validator(is_valid_pubkey) - .help("New stake authority") -} - -fn new_withdraw_authority_arg<'a, 'b>() -> Arg<'a, 'b> { - Arg::with_name("new_withdraw_authority") - .long("new-withdraw-authority") - .required(true) - .takes_value(true) - .value_name("PUBKEY") - .validator(is_valid_pubkey) - .help("New withdraw authority") -} - -fn lockup_epoch_arg<'a, 'b>() -> Arg<'a, 'b> { - Arg::with_name("lockup_epoch") - .long("lockup-epoch") - .takes_value(true) - .value_name("NUMBER") - .help("The epoch height at which each account will be available for withdrawl") -} - -fn lockup_date_arg<'a, 'b>() -> Arg<'a, 'b> { - Arg::with_name("lockup_date") - .long("lockup-date") - .value_name("RFC3339 DATETIME") - .validator(is_rfc3339_datetime) - .takes_value(true) - .help("The date and time at which each account will be available for withdrawl") -} - -fn num_accounts_arg<'a, 'b>() -> Arg<'a, 'b> { - Arg::with_name("num_accounts") - .long("num-accounts") - .required(true) - .takes_value(true) - .value_name("NUMBER") - .help("Number of derived stake accounts") -} - -pub(crate) fn get_matches<'a, I, T>(args: I) -> ArgMatches<'a> -where - I: IntoIterator, - T: Into + Clone, -{ - let default_config_file = CONFIG_FILE.as_ref().unwrap(); - App::new("solana-stake-accounts") - .about("about") - .version("version") - .arg( - Arg::with_name("config_file") - .long("config") - .takes_value(true) - .value_name("FILEPATH") - .default_value(default_config_file) - .help("Config file"), - ) - .arg( - Arg::with_name("url") - .long("url") - .global(true) - .takes_value(true) - .value_name("URL") - .help("RPC entrypoint address. i.e. http://api.devnet.solana.com"), - ) - .subcommand( - SubCommand::with_name("new") - .about("Create derived stake accounts") - .arg(fee_payer_arg()) - .arg(funding_keypair_arg().index(1)) - .arg( - Arg::with_name("base_keypair") - .required(true) - .index(2) - .takes_value(true) - .value_name("BASE_KEYPAIR") - .validator(is_valid_signer) - .help("Keypair which stake account addresses are derived from"), - ) - .arg( - Arg::with_name("amount") - .required(true) - .index(3) - .takes_value(true) - .value_name("AMOUNT") - .validator(is_amount) - .help("Amount to move into the new stake accounts, in SOL"), - ) - .arg( - Arg::with_name("stake_authority") - .long("stake-authority") - .required(true) - .takes_value(true) - .value_name("PUBKEY") - .validator(is_valid_pubkey) - .help("Stake authority"), - ) - .arg( - Arg::with_name("withdraw_authority") - .long("withdraw-authority") - .required(true) - .takes_value(true) - .value_name("PUBKEY") - .validator(is_valid_pubkey) - .help("Withdraw authority"), - ) - .arg( - Arg::with_name("index") - .long("index") - .takes_value(true) - .default_value("0") - .value_name("NUMBER") - .help("Index of the derived account to create"), - ), - ) - .subcommand( - SubCommand::with_name("count") - .about("Count derived stake accounts") - .arg(base_pubkey_arg().index(1)), - ) - .subcommand( - SubCommand::with_name("addresses") - .about("Show public keys of all derived stake accounts") - .arg(base_pubkey_arg().index(1)) - .arg(num_accounts_arg()), - ) - .subcommand( - SubCommand::with_name("balance") - .about("Sum balances of all derived stake accounts") - .arg(base_pubkey_arg().index(1)) - .arg(num_accounts_arg()), - ) - .subcommand( - SubCommand::with_name("authorize") - .about("Set new authorities in all derived stake accounts") - .arg(fee_payer_arg()) - .arg(base_pubkey_arg().index(1)) - .arg(stake_authority_arg()) - .arg(withdraw_authority_arg()) - .arg(new_stake_authority_arg()) - .arg(new_withdraw_authority_arg()) - .arg(num_accounts_arg()), - ) - .subcommand( - SubCommand::with_name("set-lockup") - .about("Set new lockups in all derived stake accounts") - .arg(fee_payer_arg()) - .arg(base_pubkey_arg().index(1)) - .arg(custodian_arg()) - .arg(lockup_epoch_arg()) - .arg(lockup_date_arg()) - .arg(new_custodian_arg()) - .arg(num_accounts_arg()) - .arg( - Arg::with_name("no_wait") - .long("no-wait") - .help("Send transactions without waiting for confirmation"), - ) - .arg( - Arg::with_name("unlock_years") - .long("unlock-years") - .takes_value(true) - .value_name("NUMBER") - .help("Years to unlock after the cliff"), - ), - ) - .subcommand( - SubCommand::with_name("rebase") - .about("Relocate derived stake accounts") - .arg(fee_payer_arg()) - .arg(base_pubkey_arg().index(1)) - .arg(new_base_keypair_arg().index(2)) - .arg(stake_authority_arg()) - .arg(num_accounts_arg()), - ) - .subcommand( - SubCommand::with_name("move") - .about("Rebase and set new authorities in all derived stake accounts") - .arg(fee_payer_arg()) - .arg(base_pubkey_arg().index(1)) - .arg(new_base_keypair_arg().index(2)) - .arg(stake_authority_arg()) - .arg(withdraw_authority_arg()) - .arg(new_stake_authority_arg()) - .arg(new_withdraw_authority_arg()) - .arg(num_accounts_arg()), - ) - .get_matches_from(args) -} - -fn parse_new_args(matches: &ArgMatches<'_>) -> NewArgs { - NewArgs { - fee_payer: value_t_or_exit!(matches, "fee_payer", String), - funding_keypair: value_t_or_exit!(matches, "funding_keypair", String), - lamports: sol_to_lamports(value_t_or_exit!(matches, "amount", f64)), - base_keypair: value_t_or_exit!(matches, "base_keypair", String), - stake_authority: value_t_or_exit!(matches, "stake_authority", String), - withdraw_authority: value_t_or_exit!(matches, "withdraw_authority", String), - index: value_t_or_exit!(matches, "index", usize), - } -} - -fn parse_count_args(matches: &ArgMatches<'_>) -> CountArgs { - CountArgs { - base_pubkey: value_t_or_exit!(matches, "base_pubkey", String), - } -} - -fn parse_query_args(matches: &ArgMatches<'_>) -> QueryArgs { - QueryArgs { - base_pubkey: value_t_or_exit!(matches, "base_pubkey", String), - num_accounts: value_t_or_exit!(matches, "num_accounts", usize), - } -} - -fn parse_authorize_args(matches: &ArgMatches<'_>) -> AuthorizeArgs { - AuthorizeArgs { - fee_payer: value_t_or_exit!(matches, "fee_payer", String), - base_pubkey: value_t_or_exit!(matches, "base_pubkey", String), - stake_authority: value_t_or_exit!(matches, "stake_authority", String), - withdraw_authority: value_t_or_exit!(matches, "withdraw_authority", String), - new_stake_authority: value_t_or_exit!(matches, "new_stake_authority", String), - new_withdraw_authority: value_t_or_exit!(matches, "new_withdraw_authority", String), - num_accounts: value_t_or_exit!(matches, "num_accounts", usize), - } -} - -fn parse_set_lockup_args(matches: &ArgMatches<'_>) -> SetLockupArgs { - SetLockupArgs { - fee_payer: value_t_or_exit!(matches, "fee_payer", String), - base_pubkey: value_t_or_exit!(matches, "base_pubkey", String), - custodian: value_t_or_exit!(matches, "custodian", String), - lockup_epoch: value_t!(matches, "lockup_epoch", u64).ok(), - lockup_date: unix_timestamp_from_rfc3339_datetime(matches, "lockup_date"), - new_custodian: value_t!(matches, "new_custodian", String).ok(), - num_accounts: value_t_or_exit!(matches, "num_accounts", usize), - no_wait: matches.is_present("no_wait"), - unlock_years: value_t!(matches, "unlock_years", f64).ok(), - } -} - -fn parse_rebase_args(matches: &ArgMatches<'_>) -> RebaseArgs { - RebaseArgs { - fee_payer: value_t_or_exit!(matches, "fee_payer", String), - base_pubkey: value_t_or_exit!(matches, "base_pubkey", String), - new_base_keypair: value_t_or_exit!(matches, "new_base_keypair", String), - stake_authority: value_t_or_exit!(matches, "stake_authority", String), - num_accounts: value_t_or_exit!(matches, "num_accounts", usize), - } -} - -fn parse_move_args(matches: &ArgMatches<'_>) -> MoveArgs { - MoveArgs { - rebase_args: parse_rebase_args(matches), - authorize_args: parse_authorize_args(matches), - } -} - -pub(crate) fn parse_args(args: I) -> Args -where - I: IntoIterator, - T: Into + Clone, -{ - let matches = get_matches(args); - let config_file = matches.value_of("config_file").unwrap().to_string(); - let url = matches.value_of("url").map(|x| x.to_string()); - - let command = match matches.subcommand() { - ("new", Some(matches)) => Command::New(parse_new_args(matches)), - ("count", Some(matches)) => Command::Count(parse_count_args(matches)), - ("addresses", Some(matches)) => Command::Addresses(parse_query_args(matches)), - ("balance", Some(matches)) => Command::Balance(parse_query_args(matches)), - ("authorize", Some(matches)) => Command::Authorize(parse_authorize_args(matches)), - ("set-lockup", Some(matches)) => Command::SetLockup(parse_set_lockup_args(matches)), - ("rebase", Some(matches)) => Command::Rebase(parse_rebase_args(matches)), - ("move", Some(matches)) => Command::Move(Box::new(parse_move_args(matches))), - _ => { - eprintln!("{}", matches.usage()); - exit(1); - } - }; - Args { - config_file, - url, - command, - } -} diff --git a/stake-accounts/src/args.rs b/stake-accounts/src/args.rs deleted file mode 100644 index de25dca870..0000000000 --- a/stake-accounts/src/args.rs +++ /dev/null @@ -1,295 +0,0 @@ -use { - clap::ArgMatches, - solana_clap_utils::keypair::{pubkey_from_path, signer_from_path}, - solana_remote_wallet::remote_wallet::RemoteWalletManager, - solana_sdk::{ - clock::{Epoch, UnixTimestamp}, - pubkey::Pubkey, - signature::Signer, - }, - std::{error::Error, sync::Arc}, -}; - -pub(crate) struct NewArgs { - pub fee_payer: K, - pub funding_keypair: K, - pub base_keypair: K, - pub lamports: u64, - pub stake_authority: P, - pub withdraw_authority: P, - pub index: usize, -} - -pub(crate) struct CountArgs

{ - pub base_pubkey: P, -} - -pub(crate) struct QueryArgs

{ - pub base_pubkey: P, - pub num_accounts: usize, -} - -pub(crate) struct AuthorizeArgs { - pub fee_payer: K, - pub base_pubkey: P, - pub stake_authority: K, - pub withdraw_authority: K, - pub new_stake_authority: P, - pub new_withdraw_authority: P, - pub num_accounts: usize, -} - -pub(crate) struct SetLockupArgs { - pub fee_payer: K, - pub base_pubkey: P, - pub custodian: K, - pub lockup_epoch: Option, - pub lockup_date: Option, - pub new_custodian: Option

, - pub num_accounts: usize, - pub no_wait: bool, - pub unlock_years: Option, -} - -pub(crate) struct RebaseArgs { - pub fee_payer: K, - pub base_pubkey: P, - pub new_base_keypair: K, - pub stake_authority: K, - pub num_accounts: usize, -} - -pub(crate) struct MoveArgs { - pub rebase_args: RebaseArgs, - pub authorize_args: AuthorizeArgs, -} - -pub(crate) enum Command { - New(NewArgs), - Count(CountArgs

), - Addresses(QueryArgs

), - Balance(QueryArgs

), - Authorize(AuthorizeArgs), - SetLockup(SetLockupArgs), - Rebase(RebaseArgs), - Move(Box>), -} - -pub(crate) struct Args { - pub config_file: String, - pub url: Option, - pub command: Command, -} - -fn resolve_stake_authority( - wallet_manager: &mut Option>, - key_url: &str, -) -> Result, Box> { - let matches = ArgMatches::default(); - signer_from_path(&matches, key_url, "stake authority", wallet_manager) -} - -fn resolve_withdraw_authority( - wallet_manager: &mut Option>, - key_url: &str, -) -> Result, Box> { - let matches = ArgMatches::default(); - signer_from_path(&matches, key_url, "withdraw authority", wallet_manager) -} - -fn resolve_new_stake_authority( - wallet_manager: &mut Option>, - key_url: &str, -) -> Result> { - let matches = ArgMatches::default(); - pubkey_from_path(&matches, key_url, "new stake authority", wallet_manager) -} - -fn resolve_new_withdraw_authority( - wallet_manager: &mut Option>, - key_url: &str, -) -> Result> { - let matches = ArgMatches::default(); - pubkey_from_path(&matches, key_url, "new withdraw authority", wallet_manager) -} - -fn resolve_fee_payer( - wallet_manager: &mut Option>, - key_url: &str, -) -> Result, Box> { - let matches = ArgMatches::default(); - signer_from_path(&matches, key_url, "fee-payer", wallet_manager) -} - -fn resolve_custodian( - wallet_manager: &mut Option>, - key_url: &str, -) -> Result, Box> { - let matches = ArgMatches::default(); - signer_from_path(&matches, key_url, "custodian", wallet_manager) -} - -fn resolve_new_custodian( - wallet_manager: &mut Option>, - key_url: &Option, -) -> Result, Box> { - let matches = ArgMatches::default(); - let pubkey = match key_url { - None => None, - Some(key_url) => { - let pubkey = pubkey_from_path(&matches, key_url, "new custodian", wallet_manager)?; - Some(pubkey) - } - }; - Ok(pubkey) -} - -fn resolve_base_pubkey( - wallet_manager: &mut Option>, - key_url: &str, -) -> Result> { - let matches = ArgMatches::default(); - pubkey_from_path(&matches, key_url, "base pubkey", wallet_manager) -} - -fn resolve_new_base_keypair( - wallet_manager: &mut Option>, - key_url: &str, -) -> Result, Box> { - let matches = ArgMatches::default(); - signer_from_path(&matches, key_url, "new base pubkey", wallet_manager) -} - -fn resolve_authorize_args( - wallet_manager: &mut Option>, - args: &AuthorizeArgs, -) -> Result>, Box> { - let resolved_args = AuthorizeArgs { - fee_payer: resolve_fee_payer(wallet_manager, &args.fee_payer)?, - base_pubkey: resolve_base_pubkey(wallet_manager, &args.base_pubkey)?, - stake_authority: resolve_stake_authority(wallet_manager, &args.stake_authority)?, - withdraw_authority: resolve_withdraw_authority(wallet_manager, &args.withdraw_authority)?, - new_stake_authority: resolve_new_stake_authority( - wallet_manager, - &args.new_stake_authority, - )?, - new_withdraw_authority: resolve_new_withdraw_authority( - wallet_manager, - &args.new_withdraw_authority, - )?, - num_accounts: args.num_accounts, - }; - Ok(resolved_args) -} - -fn resolve_set_lockup_args( - wallet_manager: &mut Option>, - args: &SetLockupArgs, -) -> Result>, Box> { - let resolved_args = SetLockupArgs { - fee_payer: resolve_fee_payer(wallet_manager, &args.fee_payer)?, - base_pubkey: resolve_base_pubkey(wallet_manager, &args.base_pubkey)?, - custodian: resolve_custodian(wallet_manager, &args.custodian)?, - lockup_epoch: args.lockup_epoch, - lockup_date: args.lockup_date, - new_custodian: resolve_new_custodian(wallet_manager, &args.new_custodian)?, - num_accounts: args.num_accounts, - no_wait: args.no_wait, - unlock_years: args.unlock_years, - }; - Ok(resolved_args) -} - -fn resolve_rebase_args( - wallet_manager: &mut Option>, - args: &RebaseArgs, -) -> Result>, Box> { - let resolved_args = RebaseArgs { - fee_payer: resolve_fee_payer(wallet_manager, &args.fee_payer)?, - base_pubkey: resolve_base_pubkey(wallet_manager, &args.base_pubkey)?, - new_base_keypair: resolve_new_base_keypair(wallet_manager, &args.new_base_keypair)?, - stake_authority: resolve_stake_authority(wallet_manager, &args.stake_authority)?, - num_accounts: args.num_accounts, - }; - Ok(resolved_args) -} - -pub(crate) fn resolve_command( - command: &Command, -) -> Result>, Box> { - let mut wallet_manager = None; - let matches = ArgMatches::default(); - match command { - Command::New(args) => { - let resolved_args = NewArgs { - fee_payer: resolve_fee_payer(&mut wallet_manager, &args.fee_payer)?, - funding_keypair: signer_from_path( - &matches, - &args.funding_keypair, - "funding keypair", - &mut wallet_manager, - )?, - base_keypair: signer_from_path( - &matches, - &args.base_keypair, - "base keypair", - &mut wallet_manager, - )?, - stake_authority: pubkey_from_path( - &matches, - &args.stake_authority, - "stake authority", - &mut wallet_manager, - )?, - withdraw_authority: pubkey_from_path( - &matches, - &args.withdraw_authority, - "withdraw authority", - &mut wallet_manager, - )?, - lamports: args.lamports, - index: args.index, - }; - Ok(Command::New(resolved_args)) - } - Command::Count(args) => { - let resolved_args = CountArgs { - base_pubkey: resolve_base_pubkey(&mut wallet_manager, &args.base_pubkey)?, - }; - Ok(Command::Count(resolved_args)) - } - Command::Addresses(args) => { - let resolved_args = QueryArgs { - base_pubkey: resolve_base_pubkey(&mut wallet_manager, &args.base_pubkey)?, - num_accounts: args.num_accounts, - }; - Ok(Command::Addresses(resolved_args)) - } - Command::Balance(args) => { - let resolved_args = QueryArgs { - base_pubkey: resolve_base_pubkey(&mut wallet_manager, &args.base_pubkey)?, - num_accounts: args.num_accounts, - }; - Ok(Command::Balance(resolved_args)) - } - Command::Authorize(args) => { - let resolved_args = resolve_authorize_args(&mut wallet_manager, args)?; - Ok(Command::Authorize(resolved_args)) - } - Command::SetLockup(args) => { - let resolved_args = resolve_set_lockup_args(&mut wallet_manager, args)?; - Ok(Command::SetLockup(resolved_args)) - } - Command::Rebase(args) => { - let resolved_args = resolve_rebase_args(&mut wallet_manager, args)?; - Ok(Command::Rebase(resolved_args)) - } - Command::Move(args) => { - let resolved_args = MoveArgs { - authorize_args: resolve_authorize_args(&mut wallet_manager, &args.authorize_args)?, - rebase_args: resolve_rebase_args(&mut wallet_manager, &args.rebase_args)?, - }; - Ok(Command::Move(Box::new(resolved_args))) - } - } -} diff --git a/stake-accounts/src/main.rs b/stake-accounts/src/main.rs deleted file mode 100644 index 39823cb100..0000000000 --- a/stake-accounts/src/main.rs +++ /dev/null @@ -1,282 +0,0 @@ -#![allow(clippy::integer_arithmetic)] -mod arg_parser; -mod args; -mod stake_accounts; - -use { - crate::{ - arg_parser::parse_args, - args::{ - resolve_command, AuthorizeArgs, Command, MoveArgs, NewArgs, RebaseArgs, SetLockupArgs, - }, - }, - solana_cli_config::Config, - solana_client::{client_error::ClientError, rpc_client::RpcClient}, - solana_sdk::{ - message::Message, - native_token::lamports_to_sol, - pubkey::Pubkey, - signature::{unique_signers, Signature, Signer}, - signers::Signers, - stake::{instruction::LockupArgs, state::Lockup}, - transaction::Transaction, - }, - solana_stake_program::stake_state, - std::{env, error::Error}, -}; - -fn get_balance_at(client: &RpcClient, pubkey: &Pubkey, i: usize) -> Result { - let address = stake_accounts::derive_stake_account_address(pubkey, i); - client.get_balance(&address) -} - -// Return the number of derived stake accounts with balances -fn count_stake_accounts(client: &RpcClient, base_pubkey: &Pubkey) -> Result { - let mut i = 0; - while get_balance_at(client, base_pubkey, i)? > 0 { - i += 1; - } - Ok(i) -} - -fn get_balances( - client: &RpcClient, - addresses: Vec, -) -> Result, ClientError> { - addresses - .into_iter() - .map(|pubkey| client.get_balance(&pubkey).map(|bal| (pubkey, bal))) - .collect() -} - -fn get_lockup(client: &RpcClient, address: &Pubkey) -> Result { - client - .get_account(address) - .map(|account| stake_state::lockup_from(&account).unwrap()) -} - -fn get_lockups( - client: &RpcClient, - addresses: Vec, -) -> Result, ClientError> { - addresses - .into_iter() - .map(|pubkey| get_lockup(client, &pubkey).map(|bal| (pubkey, bal))) - .collect() -} - -fn process_new_stake_account( - client: &RpcClient, - args: &NewArgs>, -) -> Result { - let message = stake_accounts::new_stake_account( - &args.fee_payer.pubkey(), - &args.funding_keypair.pubkey(), - &args.base_keypair.pubkey(), - args.lamports, - &args.stake_authority, - &args.withdraw_authority, - &Pubkey::default(), - args.index, - ); - let signers = unique_signers(vec![ - &*args.fee_payer, - &*args.funding_keypair, - &*args.base_keypair, - ]); - let signature = send_and_confirm_message(client, message, &signers, false)?; - Ok(signature) -} - -fn process_authorize_stake_accounts( - client: &RpcClient, - args: &AuthorizeArgs>, -) -> Result<(), ClientError> { - let messages = stake_accounts::authorize_stake_accounts( - &args.fee_payer.pubkey(), - &args.base_pubkey, - &args.stake_authority.pubkey(), - &args.withdraw_authority.pubkey(), - &args.new_stake_authority, - &args.new_withdraw_authority, - args.num_accounts, - ); - let signers = unique_signers(vec![ - &*args.fee_payer, - &*args.stake_authority, - &*args.withdraw_authority, - ]); - send_and_confirm_messages(client, messages, &signers, false)?; - Ok(()) -} - -fn process_lockup_stake_accounts( - client: &RpcClient, - args: &SetLockupArgs>, -) -> Result<(), ClientError> { - let addresses = - stake_accounts::derive_stake_account_addresses(&args.base_pubkey, args.num_accounts); - let existing_lockups = get_lockups(client, addresses)?; - - let lockup = LockupArgs { - epoch: args.lockup_epoch, - unix_timestamp: args.lockup_date, - custodian: args.new_custodian, - }; - let messages = stake_accounts::lockup_stake_accounts( - &args.fee_payer.pubkey(), - &args.custodian.pubkey(), - &lockup, - &existing_lockups, - args.unlock_years, - ); - if messages.is_empty() { - eprintln!("No work to do"); - return Ok(()); - } - let signers = unique_signers(vec![&*args.fee_payer, &*args.custodian]); - send_and_confirm_messages(client, messages, &signers, args.no_wait)?; - Ok(()) -} - -fn process_rebase_stake_accounts( - client: &RpcClient, - args: &RebaseArgs>, -) -> Result<(), ClientError> { - let addresses = - stake_accounts::derive_stake_account_addresses(&args.base_pubkey, args.num_accounts); - let balances = get_balances(client, addresses)?; - - let messages = stake_accounts::rebase_stake_accounts( - &args.fee_payer.pubkey(), - &args.new_base_keypair.pubkey(), - &args.stake_authority.pubkey(), - &balances, - ); - if messages.is_empty() { - eprintln!("No accounts found"); - return Ok(()); - } - let signers = unique_signers(vec![ - &*args.fee_payer, - &*args.new_base_keypair, - &*args.stake_authority, - ]); - send_and_confirm_messages(client, messages, &signers, false)?; - Ok(()) -} - -fn process_move_stake_accounts( - client: &RpcClient, - move_args: &MoveArgs>, -) -> Result<(), ClientError> { - let authorize_args = &move_args.authorize_args; - let args = &move_args.rebase_args; - let addresses = - stake_accounts::derive_stake_account_addresses(&args.base_pubkey, args.num_accounts); - let balances = get_balances(client, addresses)?; - - let messages = stake_accounts::move_stake_accounts( - &args.fee_payer.pubkey(), - &args.new_base_keypair.pubkey(), - &args.stake_authority.pubkey(), - &authorize_args.withdraw_authority.pubkey(), - &authorize_args.new_stake_authority, - &authorize_args.new_withdraw_authority, - &balances, - ); - if messages.is_empty() { - eprintln!("No accounts found"); - return Ok(()); - } - let signers = unique_signers(vec![ - &*args.fee_payer, - &*args.new_base_keypair, - &*args.stake_authority, - &*authorize_args.withdraw_authority, - ]); - send_and_confirm_messages(client, messages, &signers, false)?; - Ok(()) -} - -fn send_and_confirm_message( - client: &RpcClient, - message: Message, - signers: &S, - no_wait: bool, -) -> Result { - let mut transaction = Transaction::new_unsigned(message); - - let blockhash = client.get_new_latest_blockhash(&transaction.message().recent_blockhash)?; - transaction.try_sign(signers, blockhash)?; - - if no_wait { - client.send_transaction(&transaction) - } else { - client.send_and_confirm_transaction_with_spinner(&transaction) - } -} - -fn send_and_confirm_messages( - client: &RpcClient, - messages: Vec, - signers: &S, - no_wait: bool, -) -> Result, ClientError> { - let mut signatures = vec![]; - for message in messages { - let signature = send_and_confirm_message(client, message, signers, no_wait)?; - signatures.push(signature); - println!("{}", signature); - } - Ok(signatures) -} - -fn main() -> Result<(), Box> { - let command_args = parse_args(env::args_os()); - let config = Config::load(&command_args.config_file)?; - let json_rpc_url = command_args.url.unwrap_or(config.json_rpc_url); - let client = RpcClient::new(json_rpc_url); - - match resolve_command(&command_args.command)? { - Command::New(args) => { - process_new_stake_account(&client, &args)?; - } - Command::Count(args) => { - let num_accounts = count_stake_accounts(&client, &args.base_pubkey)?; - println!("{}", num_accounts); - } - Command::Addresses(args) => { - let addresses = stake_accounts::derive_stake_account_addresses( - &args.base_pubkey, - args.num_accounts, - ); - for address in addresses { - println!("{:?}", address); - } - } - Command::Balance(args) => { - let addresses = stake_accounts::derive_stake_account_addresses( - &args.base_pubkey, - args.num_accounts, - ); - let balances = get_balances(&client, addresses)?; - let lamports: u64 = balances.into_iter().map(|(_, bal)| bal).sum(); - let sol = lamports_to_sol(lamports); - println!("{} VLX", sol); - } - Command::Authorize(args) => { - process_authorize_stake_accounts(&client, &args)?; - } - Command::SetLockup(args) => { - process_lockup_stake_accounts(&client, &args)?; - } - Command::Rebase(args) => { - process_rebase_stake_accounts(&client, &args)?; - } - Command::Move(args) => { - process_move_stake_accounts(&client, &args)?; - } - } - Ok(()) -} diff --git a/stake-accounts/src/stake_accounts.rs b/stake-accounts/src/stake_accounts.rs deleted file mode 100644 index 7df019e3f2..0000000000 --- a/stake-accounts/src/stake_accounts.rs +++ /dev/null @@ -1,687 +0,0 @@ -use solana_sdk::{ - clock::SECONDS_PER_DAY, - instruction::Instruction, - message::Message, - pubkey::Pubkey, - stake::{ - self, - instruction::{self as stake_instruction, LockupArgs}, - state::{Authorized, Lockup, StakeAuthorize}, - }, -}; - -const DAYS_PER_YEAR: f64 = 365.25; -const SECONDS_PER_YEAR: i64 = (SECONDS_PER_DAY as f64 * DAYS_PER_YEAR) as i64; - -pub(crate) fn derive_stake_account_address(base_pubkey: &Pubkey, i: usize) -> Pubkey { - Pubkey::create_with_seed(base_pubkey, &i.to_string(), &stake::program::id()).unwrap() -} - -// Return derived addresses -pub(crate) fn derive_stake_account_addresses( - base_pubkey: &Pubkey, - num_accounts: usize, -) -> Vec { - (0..num_accounts) - .map(|i| derive_stake_account_address(base_pubkey, i)) - .collect() -} - -pub(crate) fn new_stake_account( - fee_payer_pubkey: &Pubkey, - funding_pubkey: &Pubkey, - base_pubkey: &Pubkey, - lamports: u64, - stake_authority_pubkey: &Pubkey, - withdraw_authority_pubkey: &Pubkey, - custodian_pubkey: &Pubkey, - index: usize, -) -> Message { - let stake_account_address = derive_stake_account_address(base_pubkey, index); - let authorized = Authorized { - staker: *stake_authority_pubkey, - withdrawer: *withdraw_authority_pubkey, - }; - let lockup = Lockup { - custodian: *custodian_pubkey, - ..Lockup::default() - }; - let instructions = stake_instruction::create_account_with_seed( - funding_pubkey, - &stake_account_address, - base_pubkey, - &index.to_string(), - &authorized, - &lockup, - lamports, - ); - Message::new(&instructions, Some(fee_payer_pubkey)) -} - -fn authorize_stake_accounts_instructions( - stake_account_address: &Pubkey, - stake_authority_pubkey: &Pubkey, - withdraw_authority_pubkey: &Pubkey, - new_stake_authority_pubkey: &Pubkey, - new_withdraw_authority_pubkey: &Pubkey, -) -> Vec { - let instruction0 = stake_instruction::authorize( - stake_account_address, - stake_authority_pubkey, - new_stake_authority_pubkey, - StakeAuthorize::Staker, - None, - ); - let instruction1 = stake_instruction::authorize( - stake_account_address, - withdraw_authority_pubkey, - new_withdraw_authority_pubkey, - StakeAuthorize::Withdrawer, - None, - ); - vec![instruction0, instruction1] -} - -fn rebase_stake_account( - stake_account_address: &Pubkey, - new_base_pubkey: &Pubkey, - i: usize, - fee_payer_pubkey: &Pubkey, - stake_authority_pubkey: &Pubkey, - lamports: u64, -) -> Option { - if lamports == 0 { - return None; - } - let new_stake_account_address = derive_stake_account_address(new_base_pubkey, i); - let instructions = stake_instruction::split_with_seed( - stake_account_address, - stake_authority_pubkey, - lamports, - &new_stake_account_address, - new_base_pubkey, - &i.to_string(), - ); - let message = Message::new(&instructions, Some(fee_payer_pubkey)); - Some(message) -} - -fn move_stake_account( - stake_account_address: &Pubkey, - new_base_pubkey: &Pubkey, - i: usize, - fee_payer_pubkey: &Pubkey, - stake_authority_pubkey: &Pubkey, - withdraw_authority_pubkey: &Pubkey, - new_stake_authority_pubkey: &Pubkey, - new_withdraw_authority_pubkey: &Pubkey, - lamports: u64, -) -> Option { - if lamports == 0 { - return None; - } - let new_stake_account_address = derive_stake_account_address(new_base_pubkey, i); - let mut instructions = stake_instruction::split_with_seed( - stake_account_address, - stake_authority_pubkey, - lamports, - &new_stake_account_address, - new_base_pubkey, - &i.to_string(), - ); - - let authorize_instructions = authorize_stake_accounts_instructions( - &new_stake_account_address, - stake_authority_pubkey, - withdraw_authority_pubkey, - new_stake_authority_pubkey, - new_withdraw_authority_pubkey, - ); - - instructions.extend(authorize_instructions.into_iter()); - let message = Message::new(&instructions, Some(fee_payer_pubkey)); - Some(message) -} - -pub(crate) fn authorize_stake_accounts( - fee_payer_pubkey: &Pubkey, - base_pubkey: &Pubkey, - stake_authority_pubkey: &Pubkey, - withdraw_authority_pubkey: &Pubkey, - new_stake_authority_pubkey: &Pubkey, - new_withdraw_authority_pubkey: &Pubkey, - num_accounts: usize, -) -> Vec { - let stake_account_addresses = derive_stake_account_addresses(base_pubkey, num_accounts); - stake_account_addresses - .iter() - .map(|stake_account_address| { - let instructions = authorize_stake_accounts_instructions( - stake_account_address, - stake_authority_pubkey, - withdraw_authority_pubkey, - new_stake_authority_pubkey, - new_withdraw_authority_pubkey, - ); - Message::new(&instructions, Some(fee_payer_pubkey)) - }) - .collect::>() -} - -fn extend_lockup(lockup: &LockupArgs, years: f64) -> LockupArgs { - let offset = (SECONDS_PER_YEAR as f64 * years) as i64; - let unix_timestamp = lockup.unix_timestamp.map(|x| x + offset); - let epoch = lockup.epoch.map(|_| todo!()); - LockupArgs { - unix_timestamp, - epoch, - custodian: lockup.custodian, - } -} - -fn apply_lockup_changes(lockup: &LockupArgs, existing_lockup: &Lockup) -> LockupArgs { - let custodian = match lockup.custodian { - Some(x) if x == existing_lockup.custodian => None, - x => x, - }; - let epoch = match lockup.epoch { - Some(x) if x == existing_lockup.epoch => None, - x => x, - }; - let unix_timestamp = match lockup.unix_timestamp { - Some(x) if x == existing_lockup.unix_timestamp => None, - x => x, - }; - LockupArgs { - unix_timestamp, - epoch, - custodian, - } -} - -pub(crate) fn lockup_stake_accounts( - fee_payer_pubkey: &Pubkey, - custodian_pubkey: &Pubkey, - lockup: &LockupArgs, - existing_lockups: &[(Pubkey, Lockup)], - unlock_years: Option, -) -> Vec { - let default_lockup = LockupArgs::default(); - existing_lockups - .iter() - .enumerate() - .filter_map(|(index, (address, existing_lockup))| { - let lockup = if let Some(unlock_years) = unlock_years { - let unlocks = existing_lockups.len() - 1; - let years = (unlock_years / unlocks as f64) * index as f64; - extend_lockup(lockup, years) - } else { - *lockup - }; - let lockup = apply_lockup_changes(&lockup, existing_lockup); - if lockup == default_lockup { - return None; - } - let instruction = stake_instruction::set_lockup(address, &lockup, custodian_pubkey); - let message = Message::new(&[instruction], Some(fee_payer_pubkey)); - Some(message) - }) - .collect() -} - -pub(crate) fn rebase_stake_accounts( - fee_payer_pubkey: &Pubkey, - new_base_pubkey: &Pubkey, - stake_authority_pubkey: &Pubkey, - balances: &[(Pubkey, u64)], -) -> Vec { - balances - .iter() - .enumerate() - .filter_map(|(i, (stake_account_address, lamports))| { - rebase_stake_account( - stake_account_address, - new_base_pubkey, - i, - fee_payer_pubkey, - stake_authority_pubkey, - *lamports, - ) - }) - .collect() -} - -pub(crate) fn move_stake_accounts( - fee_payer_pubkey: &Pubkey, - new_base_pubkey: &Pubkey, - stake_authority_pubkey: &Pubkey, - withdraw_authority_pubkey: &Pubkey, - new_stake_authority_pubkey: &Pubkey, - new_withdraw_authority_pubkey: &Pubkey, - balances: &[(Pubkey, u64)], -) -> Vec { - balances - .iter() - .enumerate() - .filter_map(|(i, (stake_account_address, lamports))| { - move_stake_account( - stake_account_address, - new_base_pubkey, - i, - fee_payer_pubkey, - stake_authority_pubkey, - withdraw_authority_pubkey, - new_stake_authority_pubkey, - new_withdraw_authority_pubkey, - *lamports, - ) - }) - .collect() -} - -#[cfg(test)] -mod tests { - use { - super::*, - solana_runtime::{bank::Bank, bank_client::BankClient}, - solana_sdk::{ - account::{AccountSharedData, ReadableAccount}, - client::SyncClient, - genesis_config::create_genesis_config, - signature::{Keypair, Signer}, - stake::state::StakeState, - }, - solana_stake_program::stake_state, - }; - - fn create_bank(lamports: u64) -> (Bank, Keypair, u64) { - let (genesis_config, mint_keypair) = create_genesis_config(lamports); - let bank = Bank::new_for_tests(&genesis_config); - let rent = bank.get_minimum_balance_for_rent_exemption(std::mem::size_of::()); - (bank, mint_keypair, rent) - } - - fn create_account( - client: &C, - funding_keypair: &Keypair, - lamports: u64, - ) -> Keypair { - let fee_payer_keypair = Keypair::new(); - client - .transfer_and_confirm(lamports, funding_keypair, &fee_payer_keypair.pubkey()) - .unwrap(); - fee_payer_keypair - } - - fn get_account_at( - client: &C, - base_pubkey: &Pubkey, - i: usize, - ) -> AccountSharedData { - let account_address = derive_stake_account_address(base_pubkey, i); - AccountSharedData::from(client.get_account(&account_address).unwrap().unwrap()) - } - - fn get_balances( - client: &C, - base_pubkey: &Pubkey, - num_accounts: usize, - ) -> Vec<(Pubkey, u64)> { - (0..num_accounts) - .map(|i| { - let address = derive_stake_account_address(base_pubkey, i); - (address, client.get_balance(&address).unwrap()) - }) - .collect() - } - - fn get_lockups( - client: &C, - base_pubkey: &Pubkey, - num_accounts: usize, - ) -> Vec<(Pubkey, Lockup)> { - (0..num_accounts) - .map(|i| { - let address = derive_stake_account_address(base_pubkey, i); - let account = - AccountSharedData::from(client.get_account(&address).unwrap().unwrap()); - (address, stake_state::lockup_from(&account).unwrap()) - }) - .collect() - } - - #[test] - fn test_new_derived_stake_account() { - let (bank, funding_keypair, rent) = create_bank(10_000_000); - let funding_pubkey = funding_keypair.pubkey(); - let bank_client = BankClient::new(bank); - let fee_payer_keypair = create_account(&bank_client, &funding_keypair, 1); - let fee_payer_pubkey = fee_payer_keypair.pubkey(); - - let base_keypair = Keypair::new(); - let base_pubkey = base_keypair.pubkey(); - let lamports = rent + 1; - let stake_authority_pubkey = solana_sdk::pubkey::new_rand(); - let withdraw_authority_pubkey = solana_sdk::pubkey::new_rand(); - - let message = new_stake_account( - &fee_payer_pubkey, - &funding_pubkey, - &base_pubkey, - lamports, - &stake_authority_pubkey, - &withdraw_authority_pubkey, - &Pubkey::default(), - 0, - ); - - let signers = [&funding_keypair, &fee_payer_keypair, &base_keypair]; - bank_client - .send_and_confirm_message(&signers, message) - .unwrap(); - - let account = get_account_at(&bank_client, &base_pubkey, 0); - assert_eq!(account.lamports(), lamports); - let authorized = stake_state::authorized_from(&account).unwrap(); - assert_eq!(authorized.staker, stake_authority_pubkey); - assert_eq!(authorized.withdrawer, withdraw_authority_pubkey); - } - - #[test] - fn test_authorize_stake_accounts() { - let (bank, funding_keypair, rent) = create_bank(10_000_000); - let funding_pubkey = funding_keypair.pubkey(); - let bank_client = BankClient::new(bank); - let fee_payer_keypair = create_account(&bank_client, &funding_keypair, 1); - let fee_payer_pubkey = fee_payer_keypair.pubkey(); - - let base_keypair = Keypair::new(); - let base_pubkey = base_keypair.pubkey(); - let lamports = rent + 1; - - let stake_authority_keypair = Keypair::new(); - let stake_authority_pubkey = stake_authority_keypair.pubkey(); - let withdraw_authority_keypair = Keypair::new(); - let withdraw_authority_pubkey = withdraw_authority_keypair.pubkey(); - - let message = new_stake_account( - &fee_payer_pubkey, - &funding_pubkey, - &base_pubkey, - lamports, - &stake_authority_pubkey, - &withdraw_authority_pubkey, - &Pubkey::default(), - 0, - ); - - let signers = [&funding_keypair, &fee_payer_keypair, &base_keypair]; - bank_client - .send_and_confirm_message(&signers, message) - .unwrap(); - - let new_stake_authority_pubkey = solana_sdk::pubkey::new_rand(); - let new_withdraw_authority_pubkey = solana_sdk::pubkey::new_rand(); - let messages = authorize_stake_accounts( - &fee_payer_pubkey, - &base_pubkey, - &stake_authority_pubkey, - &withdraw_authority_pubkey, - &new_stake_authority_pubkey, - &new_withdraw_authority_pubkey, - 1, - ); - - let signers = [ - &fee_payer_keypair, - &stake_authority_keypair, - &withdraw_authority_keypair, - ]; - for message in messages { - bank_client - .send_and_confirm_message(&signers, message) - .unwrap(); - } - - let account = get_account_at(&bank_client, &base_pubkey, 0); - let authorized = stake_state::authorized_from(&account).unwrap(); - assert_eq!(authorized.staker, new_stake_authority_pubkey); - assert_eq!(authorized.withdrawer, new_withdraw_authority_pubkey); - } - - #[test] - fn test_lockup_stake_accounts() { - let (bank, funding_keypair, rent) = create_bank(10_000_000); - let funding_pubkey = funding_keypair.pubkey(); - let bank_client = BankClient::new(bank); - let fee_payer_keypair = create_account(&bank_client, &funding_keypair, 1); - let fee_payer_pubkey = fee_payer_keypair.pubkey(); - - let base_keypair = Keypair::new(); - let base_pubkey = base_keypair.pubkey(); - let lamports = rent + 1; - - let custodian_keypair = Keypair::new(); - let custodian_pubkey = custodian_keypair.pubkey(); - - let withdrawer_keypair = Keypair::new(); - let withdrawer_pubkey = withdrawer_keypair.pubkey(); - - let message = new_stake_account( - &fee_payer_pubkey, - &funding_pubkey, - &base_pubkey, - lamports, - &Pubkey::default(), - &withdrawer_pubkey, - &custodian_pubkey, - 0, - ); - - let signers = [&funding_keypair, &fee_payer_keypair, &base_keypair]; - bank_client - .send_and_confirm_message(&signers, message) - .unwrap(); - - let lockups = get_lockups(&bank_client, &base_pubkey, 1); - let messages = lockup_stake_accounts( - &fee_payer_pubkey, - &withdrawer_pubkey, - &LockupArgs { - unix_timestamp: Some(1), - custodian: Some(custodian_pubkey), - ..LockupArgs::default() - }, - &lockups, - None, - ); - - let signers = [&fee_payer_keypair, &withdrawer_keypair]; - for message in messages { - bank_client - .send_and_confirm_message(&signers, message) - .unwrap(); - } - - let account = get_account_at(&bank_client, &base_pubkey, 0); - let lockup = stake_state::lockup_from(&account).unwrap(); - assert_eq!(lockup.custodian, custodian_pubkey); - assert_eq!(lockup.unix_timestamp, 1); - assert_eq!(lockup.epoch, 0); - - // Assert no work left to do - let lockups = get_lockups(&bank_client, &base_pubkey, 1); - let messages = lockup_stake_accounts( - &fee_payer_pubkey, - &custodian_pubkey, - &LockupArgs { - unix_timestamp: Some(1), - ..LockupArgs::default() - }, - &lockups, - None, - ); - assert_eq!(messages.len(), 0); - } - - #[test] - fn test_rebase_empty_account() { - let pubkey = Pubkey::default(); - let message = rebase_stake_account(&pubkey, &pubkey, 0, &pubkey, &pubkey, 0); - assert_eq!(message, None); - } - - #[test] - fn test_move_empty_account() { - let pubkey = Pubkey::default(); - let message = move_stake_account( - &pubkey, &pubkey, 0, &pubkey, &pubkey, &pubkey, &pubkey, &pubkey, 0, - ); - assert_eq!(message, None); - } - - #[test] - fn test_rebase_stake_accounts() { - let (bank, funding_keypair, rent) = create_bank(10_000_000); - let funding_pubkey = funding_keypair.pubkey(); - let bank_client = BankClient::new(bank); - let fee_payer_keypair = create_account(&bank_client, &funding_keypair, 1); - let fee_payer_pubkey = fee_payer_keypair.pubkey(); - - let base_keypair = Keypair::new(); - let base_pubkey = base_keypair.pubkey(); - let lamports = rent + 1; - - let stake_authority_keypair = Keypair::new(); - let stake_authority_pubkey = stake_authority_keypair.pubkey(); - let withdraw_authority_keypair = Keypair::new(); - let withdraw_authority_pubkey = withdraw_authority_keypair.pubkey(); - - let num_accounts = 1; - let message = new_stake_account( - &fee_payer_pubkey, - &funding_pubkey, - &base_pubkey, - lamports, - &stake_authority_pubkey, - &withdraw_authority_pubkey, - &Pubkey::default(), - 0, - ); - - let signers = [&funding_keypair, &fee_payer_keypair, &base_keypair]; - bank_client - .send_and_confirm_message(&signers, message) - .unwrap(); - - let new_base_keypair = Keypair::new(); - let new_base_pubkey = new_base_keypair.pubkey(); - let balances = get_balances(&bank_client, &base_pubkey, num_accounts); - let messages = rebase_stake_accounts( - &fee_payer_pubkey, - &new_base_pubkey, - &stake_authority_pubkey, - &balances, - ); - assert_eq!(messages.len(), num_accounts); - - let signers = [ - &fee_payer_keypair, - &new_base_keypair, - &stake_authority_keypair, - ]; - for message in messages { - bank_client - .send_and_confirm_message(&signers, message) - .unwrap(); - } - - // Ensure the new accounts are duplicates of the previous ones. - let account = get_account_at(&bank_client, &new_base_pubkey, 0); - let authorized = stake_state::authorized_from(&account).unwrap(); - assert_eq!(authorized.staker, stake_authority_pubkey); - assert_eq!(authorized.withdrawer, withdraw_authority_pubkey); - } - - #[test] - fn test_move_stake_accounts() { - let (bank, funding_keypair, rent) = create_bank(10_000_000); - let funding_pubkey = funding_keypair.pubkey(); - let bank_client = BankClient::new(bank); - let fee_payer_keypair = create_account(&bank_client, &funding_keypair, 1); - let fee_payer_pubkey = fee_payer_keypair.pubkey(); - - let base_keypair = Keypair::new(); - let base_pubkey = base_keypair.pubkey(); - let lamports = rent + 1; - - let stake_authority_keypair = Keypair::new(); - let stake_authority_pubkey = stake_authority_keypair.pubkey(); - let withdraw_authority_keypair = Keypair::new(); - let withdraw_authority_pubkey = withdraw_authority_keypair.pubkey(); - - let num_accounts = 1; - let message = new_stake_account( - &fee_payer_pubkey, - &funding_pubkey, - &base_pubkey, - lamports, - &stake_authority_pubkey, - &withdraw_authority_pubkey, - &Pubkey::default(), - 0, - ); - - let signers = [&funding_keypair, &fee_payer_keypair, &base_keypair]; - bank_client - .send_and_confirm_message(&signers, message) - .unwrap(); - - let new_base_keypair = Keypair::new(); - let new_base_pubkey = new_base_keypair.pubkey(); - let new_stake_authority_pubkey = solana_sdk::pubkey::new_rand(); - let new_withdraw_authority_pubkey = solana_sdk::pubkey::new_rand(); - let balances = get_balances(&bank_client, &base_pubkey, num_accounts); - let messages = move_stake_accounts( - &fee_payer_pubkey, - &new_base_pubkey, - &stake_authority_pubkey, - &withdraw_authority_pubkey, - &new_stake_authority_pubkey, - &new_withdraw_authority_pubkey, - &balances, - ); - assert_eq!(messages.len(), num_accounts); - - let signers = [ - &fee_payer_keypair, - &new_base_keypair, - &stake_authority_keypair, - &withdraw_authority_keypair, - ]; - for message in messages { - bank_client - .send_and_confirm_message(&signers, message) - .unwrap(); - } - - // Ensure the new accounts have the new authorities. - let account = get_account_at(&bank_client, &new_base_pubkey, 0); - let authorized = stake_state::authorized_from(&account).unwrap(); - assert_eq!(authorized.staker, new_stake_authority_pubkey); - assert_eq!(authorized.withdrawer, new_withdraw_authority_pubkey); - } - - #[test] - fn test_extend_lockup() { - let lockup = LockupArgs { - unix_timestamp: Some(1), - ..LockupArgs::default() - }; - let expected_lockup = LockupArgs { - unix_timestamp: Some(1 + SECONDS_PER_YEAR), - ..LockupArgs::default() - }; - assert_eq!(extend_lockup(&lockup, 1.0), expected_lockup); - } -} diff --git a/tokens/.gitignore b/tokens/.gitignore deleted file mode 100644 index f52cd4e258..0000000000 --- a/tokens/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -target/ -*.csv -/farf/ diff --git a/tokens/Cargo.toml b/tokens/Cargo.toml deleted file mode 100644 index 11709a43b9..0000000000 --- a/tokens/Cargo.toml +++ /dev/null @@ -1,39 +0,0 @@ -[package] -name = "solana-tokens" -description = "Blockchain, Rebuilt for Scale" -authors = ["Solana Maintainers "] -edition = "2021" -version = "1.10.41" -repository = "https://github.com/solana-labs/solana" -license = "Apache-2.0" -homepage = "https://solana.com/" -documentation = "https://docs.rs/solana-tokens" - -[dependencies] -chrono = { version = "0.4", features = ["serde"] } -clap = "2.33.0" -console = "0.15.0" -csv = "1.1.6" -ctrlc = { version = "3.2.1", features = ["termination"] } -indexmap = "1.8.0" -indicatif = "0.16.2" -pickledb = "0.4.1" -serde = { version = "1.0", features = ["derive"] } -solana-account-decoder = { path = "../account-decoder", version = "=1.10.41" } -solana-clap-utils = { path = "../clap-utils", version = "=1.10.41" } -solana-cli-config = { path = "../cli-config", version = "=1.10.41" } -solana-client = { path = "../client", version = "=1.10.41" } -solana-remote-wallet = { path = "../remote-wallet", version = "=1.10.41" } -solana-sdk = { path = "../sdk", version = "=1.10.41" } -solana-transaction-status = { path = "../transaction-status", version = "=1.10.41" } -solana-version = { path = "../version", version = "=0.7.0" } -spl-associated-token-account = { version = "=1.1.1" } -spl-token = { version = "=3.5.0", features = ["no-entrypoint"] } -tempfile = "3.3.0" -thiserror = "1.0" - -[dev-dependencies] -bincode = "1.3.3" -solana-logger = { path = "../logger", version = "=1.10.41" } -solana-streamer = { path = "../streamer", version = "=1.10.41" } -solana-test-validator = { path = "../test-validator", version = "=1.10.41" } diff --git a/tokens/README.md b/tokens/README.md deleted file mode 100644 index 626cc53932..0000000000 --- a/tokens/README.md +++ /dev/null @@ -1,240 +0,0 @@ -# Distribute Solana tokens - -A user may want to make payments to multiple accounts over multiple iterations. -The user will have a spreadsheet listing public keys and token amounts, and -some process for transferring tokens to them, and ensuring that no more than the -expected amount are sent. The command-line tool here automates that process. - -## Distribute tokens - -Send tokens to the recipients in ``. - -Example recipients.csv: - -```text -recipient,amount,lockup_date -3ihfUy1n9gaqihM5bJCiTAGLgWc5zo3DqVUS6T736NLM,42.0, -CYRJWqiSjLitBAcRxPvWpgX3s5TvmN2SuRY3eEYypFvT,43.0, -``` - -```bash -solana-tokens distribute-tokens --from --input-csv --fee-payer -``` - -Example transaction log before: - -```text -recipient,amount,finalized_date,signature -6Vo87BaDhp4v4GHwVDhw5huhxVF8CyxSXYtkUwVHbbPv,70.0,2020-09-15T23:29:26.879747Z,UB168XhBhecxzeD1w2ZRUhwTHpPSqv2WNh8NrZHqz1F2EqxxbSW6iFfVtsg3HkU9NX2cD7R92D8VRLSyArZ9xKQ -``` - -Send tokens to the recipients in `` if the distribution is -not already recorded in the transaction log. - -```bash -solana-tokens distribute-tokens --from --input-csv --fee-payer -``` - -Example output: - -```text -Recipient Expected Balance -3ihfUy1n9gaqihM5bJCiTAGLgWc5zo3DqVUS6T736NLM 42 -UKUcTXgbeTYh65RaVV5gSf6xBHevqHvAXMo3e8Q6np8k 43 -``` - - -Example transaction log after: - -```bash -solana-tokens transaction-log --output-path transactions.csv -``` - -```text -recipient,amount,signature -6Vo87BaDhp4v4GHwVDhw5huhxVF8CyxSXYtkUwVHbbPv,70.0,2020-09-15T23:29:26.879747Z,UB168XhBhecxzeD1w2ZRUhwTHpPSqv2WNh8NrZHqz1F2EqxxbSW6iFfVtsg3HkU9NX2cD7R92D8VRLSyArZ9xKQ -3ihfUy1n9gaqihM5bJCiTAGLgWc5zo3DqVUS6T736NLM,42.0,2020-09-15T23:31:50.264241Z,53AVNEVpQBteJBRAKp6naxXsgESDjqe1ge9Dg2HeCSpYWTuGTLqHrBpkHTnpvPJURNgKWxkJfihuRa5STVRjL2hy -CYRJWqiSjLitBAcRxPvWpgX3s5TvmN2SuRY3eEYypFvT,43.0,2020-09-15T23:33:53.680821Z,4XsMfLx9D2ZxVpdJ5xdkV2w4X4SKEQ5zbQhcH4NcRwgZDkdRNiZjvnMFaWaWHUh5eF1LwFPpQdjn6mzSsiCVj3L7 -``` - -### Calculate what tokens should be sent - -List the differences between a list of expected distributions and the record of what -transactions have already been sent. - -```bash -solana-tokens distribute-tokens --dry-run --input-csv -``` - -Example recipients.csv: - -```text -recipient,amount,lockup_date -6Vo87BaDhp4v4GHwVDhw5huhxVF8CyxSXYtkUwVHbbPv,80, -7aHDubg5FBYj1SgmyBgU3ZJdtfuqYCQsJQK2pTR5JUqr,42, -``` - -Example output: - -```text -Recipient Expected Balance -6Vo87BaDhp4v4GHwVDhw5huhxVF8CyxSXYtkUwVHbbPv 10 -7aHDubg5FBYj1SgmyBgU3ZJdtfuqYCQsJQK2pTR5JUqr 42 -``` - -## Distribute tokens: transfer-amount - -This tool also makes it straightforward to transfer the same amount of tokens to a simple list of recipients. Just add the `--transfer-amount` arg to specify the amount: - -Example recipients.csv: - -```text -recipient -6Vo87BaDhp4v4GHwVDhw5huhxVF8CyxSXYtkUwVHbbPv -7aHDubg5FBYj1SgmyBgU3ZJdtfuqYCQsJQK2pTR5JUqr -CYRJWqiSjLitBAcRxPvWpgX3s5TvmN2SuRY3eEYypFvT -``` - -```bash -solana-tokens distribute-tokens --transfer-amount 10 --from --input-csv --fee-payer -``` - -Example output: - -```text -Recipient Expected Balance -6Vo87BaDhp4v4GHwVDhw5huhxVF8CyxSXYtkUwVHbbPv 10 -7aHDubg5FBYj1SgmyBgU3ZJdtfuqYCQsJQK2pTR5JUqr 10 -CYRJWqiSjLitBAcRxPvWpgX3s5TvmN2SuRY3eEYypFvT 10 -``` - -## Distribute stake accounts - -Distributing tokens via stake accounts works similarly to how tokens are distributed. The -big difference is that new stake accounts are split from existing ones. By splitting, -the new accounts inherit any lockup or custodian settings of the original. - -```bash -solana-tokens distribute-stake --stake-account-address \ - --input-csv \ - --stake-authority --withdraw-authority --fee-payer -``` - -Currently, this will subtract 1 VLX from each allocation and store it the -recipient address. That VLX can be used to pay transaction fees on staking -operations such as delegating stake. The rest of the allocation is put in -a stake account. The new stake account address is output in the transaction -log. - -## Distribute SPL tokens - -Distributing SPL Tokens works very similarly to distributing VLX, but requires -the `--owner` parameter to sign transactions. Each recipient account must be an -system account that will own an Associated Token Account for the SPL Token mint. -The Associated Token Account will be created, and funded by the fee_payer, if it -does not already exist. - -Send SPL tokens to the recipients in ``. -*NOTE:* the CSV expects SPL-token amounts in raw format (no decimals) - -Example recipients.csv: - -```text -recipient,amount -CYRJWqiSjLitBAcRxPvWpgX3s5TvmN2SuRY3eEYypFvT,75400 -C56nwrDVFpPrqwGYsTgQxv1ZraTh81H14PV4RHvZe36s,10000 -7aHDubg5FBYj1SgmyBgU3ZJdtfuqYCQsJQK2pTR5JUqr,42100 -7qQPmVAQxEQ5djPDCtiEUrxaPf8wKtLG1m6SB1brejJ1,20000 -``` - -You can check the status of the recipients before beginning a distribution. You -must include the SPL Token mint address: - -```bash -solana-tokens spl-token-balances --mint

--input-csv -``` - -Example output: - -```text -Token: JDte736XZ1jGUtfAS32DLpBUWBR7WGSHy1hSZ36VRQ5V -Recipient Expected Balance Actual Balance Difference -CYRJWqiSjLitBAcRxPvWpgX3s5TvmN2SuRY3eEYypFvT 75.400 0.000 -75.400 -C56nwrDVFpPrqwGYsTgQxv1ZraTh81H14PV4RHvZe36s 10.000 Associated token account not yet created -7aHDubg5FBYj1SgmyBgU3ZJdtfuqYCQsJQK2pTR5JUqr 42.100 0.000 -42.100 -7qQPmVAQxEQ5djPDCtiEUrxaPf8wKtLG1m6SB1brejJ1 20.000 Associated token account not yet created -``` - -To run the distribution: - -```bash -solana-tokens distribute-spl-tokens --from
--owner \ - --input-csv --fee-payer -``` - -Example output: - -```text -Total in input_csv: 147.5 tokens -Distributed: 0 tokens -Undistributed: 147.5 tokens -Total: 147.5 tokens -Recipient Expected Balance -CYRJWqiSjLitBAcRxPvWpgX3s5TvmN2SuRY3eEYypFvT 75.400 -C56nwrDVFpPrqwGYsTgQxv1ZraTh81H14PV4RHvZe36s 10.000 -7aHDubg5FBYj1SgmyBgU3ZJdtfuqYCQsJQK2pTR5JUqr 42.100 -7qQPmVAQxEQ5djPDCtiEUrxaPf8wKtLG1m6SB1brejJ1 20.000 -``` - -### Calculate what tokens should be sent - -As with VLX, you can List the differences between a list of expected -distributions and the record of what transactions have already been sent using -the `--dry-run` parameter, or `solana-tokens balances`. - -Example updated recipients.csv: - -```text -recipient,amount -CYRJWqiSjLitBAcRxPvWpgX3s5TvmN2SuRY3eEYypFvT,100000 -C56nwrDVFpPrqwGYsTgQxv1ZraTh81H14PV4RHvZe36s,100000 -7aHDubg5FBYj1SgmyBgU3ZJdtfuqYCQsJQK2pTR5JUqr,100000 -7qQPmVAQxEQ5djPDCtiEUrxaPf8wKtLG1m6SB1brejJ1,100000 -``` - -Using dry-run: - -```bash -solana-tokens distribute-tokens --dry-run --input-csv -``` - -Example output: - -```text -Total in input_csv: 400 tokens -Distributed: 147.5 tokens -Undistributed: 252.5 tokens -Total: 400 tokens -Recipient Expected Balance -CYRJWqiSjLitBAcRxPvWpgX3s5TvmN2SuRY3eEYypFvT 24.600 -C56nwrDVFpPrqwGYsTgQxv1ZraTh81H14PV4RHvZe36s 90.000 -7aHDubg5FBYj1SgmyBgU3ZJdtfuqYCQsJQK2pTR5JUqr 57.900 -7qQPmVAQxEQ5djPDCtiEUrxaPf8wKtLG1m6SB1brejJ1 80.000 -``` - -Or: - -```bash -solana-tokens balances --mint
--input-csv -``` - -Example output: - -```text -Token: JDte736XZ1jGUtfAS32DLpBUWBR7WGSHy1hSZ36VRQ5V -Recipient Expected Balance Actual Balance Difference -CYRJWqiSjLitBAcRxPvWpgX3s5TvmN2SuRY3eEYypFvT 100.000 75.400 -24.600 -C56nwrDVFpPrqwGYsTgQxv1ZraTh81H14PV4RHvZe36s 100.000 10.000 -90.000 -7aHDubg5FBYj1SgmyBgU3ZJdtfuqYCQsJQK2pTR5JUqr 100.000 42.100 -57.900 -7qQPmVAQxEQ5djPDCtiEUrxaPf8wKtLG1m6SB1brejJ1 100.000 20.000 -80.000 -``` diff --git a/tokens/src/arg_parser.rs b/tokens/src/arg_parser.rs deleted file mode 100644 index f6bfd28b97..0000000000 --- a/tokens/src/arg_parser.rs +++ /dev/null @@ -1,679 +0,0 @@ -use { - crate::args::{ - Args, BalancesArgs, Command, DistributeTokensArgs, SenderStakeArgs, SplTokenArgs, - StakeArgs, TransactionLogArgs, - }, - clap::{ - crate_description, crate_name, value_t, value_t_or_exit, App, Arg, ArgMatches, SubCommand, - }, - solana_clap_utils::{ - input_parsers::{pubkey_of_signer, value_of}, - input_validators::{is_amount, is_valid_pubkey, is_valid_signer}, - keypair::{pubkey_from_path, signer_from_path}, - }, - solana_cli_config::CONFIG_FILE, - solana_remote_wallet::remote_wallet::maybe_wallet_manager, - solana_sdk::native_token::sol_to_lamports, - std::{error::Error, ffi::OsString, process::exit}, -}; - -fn get_matches<'a, I, T>(args: I) -> ArgMatches<'a> -where - I: IntoIterator, - T: Into + Clone, -{ - let default_config_file = CONFIG_FILE.as_ref().unwrap(); - App::new(crate_name!()) - .about(crate_description!()) - .version(solana_version::version!()) - .arg( - Arg::with_name("config_file") - .long("config") - .takes_value(true) - .value_name("FILEPATH") - .default_value(default_config_file) - .help("Config file"), - ) - .arg( - Arg::with_name("url") - .long("url") - .global(true) - .takes_value(true) - .value_name("URL") - .help("RPC entrypoint address. i.e. https://api.devnet.velas.com"), - ) - .subcommand( - SubCommand::with_name("distribute-tokens") - .about("Distribute VLX") - .arg( - Arg::with_name("db_path") - .long("db-path") - .required(true) - .takes_value(true) - .value_name("FILE") - .help( - "Location for storing distribution database. \ - The database is used for tracking transactions as they are finalized \ - and preventing double spends.", - ), - ) - .arg( - Arg::with_name("input_csv") - .long("input-csv") - .required(true) - .takes_value(true) - .value_name("FILE") - .help("Input CSV file"), - ) - .arg( - Arg::with_name("transfer_amount") - .long("transfer-amount") - .takes_value(true) - .value_name("AMOUNT") - .validator(is_amount) - .help("The amount to send to each recipient, in VLX"), - ) - .arg( - Arg::with_name("dry_run") - .long("dry-run") - .help("Do not execute any transfers"), - ) - .arg( - Arg::with_name("output_path") - .long("output-path") - .short("o") - .value_name("FILE") - .takes_value(true) - .help("Write the transaction log to this file"), - ) - .arg( - Arg::with_name("sender_keypair") - .long("from") - .required(true) - .takes_value(true) - .value_name("SENDING_KEYPAIR") - .validator(is_valid_signer) - .help("Keypair to fund accounts"), - ) - .arg( - Arg::with_name("fee_payer") - .long("fee-payer") - .required(true) - .takes_value(true) - .value_name("KEYPAIR") - .validator(is_valid_signer) - .help("Fee payer"), - ), - ) - .subcommand( - SubCommand::with_name("create-stake") - .about("Create stake accounts") - .arg( - Arg::with_name("db_path") - .long("db-path") - .required(true) - .takes_value(true) - .value_name("FILE") - .help( - "Location for storing distribution database. \ - The database is used for tracking transactions as they are finalized \ - and preventing double spends.", - ), - ) - .arg( - Arg::with_name("input_csv") - .long("input-csv") - .required(true) - .takes_value(true) - .value_name("FILE") - .help("Allocations CSV file"), - ) - .arg( - Arg::with_name("dry_run") - .long("dry-run") - .help("Do not execute any transfers"), - ) - .arg( - Arg::with_name("output_path") - .long("output-path") - .short("o") - .value_name("FILE") - .takes_value(true) - .help("Write the transaction log to this file"), - ) - .arg( - Arg::with_name("sender_keypair") - .long("from") - .required(true) - .takes_value(true) - .value_name("SENDING_KEYPAIR") - .validator(is_valid_signer) - .help("Keypair to fund accounts"), - ) - .arg( - Arg::with_name("unlocked_sol") - .default_value("1.0") - .long("unlocked-sol") - .takes_value(true) - .value_name("SOL_AMOUNT") - .help("Amount of SOL to put in system account to pay for fees"), - ) - .arg( - Arg::with_name("lockup_authority") - .long("lockup-authority") - .takes_value(true) - .value_name("PUBKEY") - .validator(is_valid_pubkey) - .help("Lockup Authority Address"), - ) - .arg( - Arg::with_name("fee_payer") - .long("fee-payer") - .required(true) - .takes_value(true) - .value_name("KEYPAIR") - .validator(is_valid_signer) - .help("Fee payer"), - ), - ) - .subcommand( - SubCommand::with_name("distribute-stake") - .about("Split to stake accounts") - .arg( - Arg::with_name("db_path") - .long("db-path") - .required(true) - .takes_value(true) - .value_name("FILE") - .help( - "Location for storing distribution database. \ - The database is used for tracking transactions as they are finalized \ - and preventing double spends.", - ), - ) - .arg( - Arg::with_name("input_csv") - .long("input-csv") - .required(true) - .takes_value(true) - .value_name("FILE") - .help("Allocations CSV file"), - ) - .arg( - Arg::with_name("dry_run") - .long("dry-run") - .help("Do not execute any transfers"), - ) - .arg( - Arg::with_name("output_path") - .long("output-path") - .short("o") - .value_name("FILE") - .takes_value(true) - .help("Write the transaction log to this file"), - ) - .arg( - Arg::with_name("sender_keypair") - .long("from") - .required(true) - .takes_value(true) - .value_name("SENDING_KEYPAIR") - .validator(is_valid_signer) - .help("Keypair to fund accounts"), - ) - .arg( - Arg::with_name("stake_account_address") - .required(true) - .long("stake-account-address") - .takes_value(true) - .value_name("ACCOUNT_ADDRESS") - .validator(is_valid_pubkey) - .help("Stake Account Address"), - ) - .arg( - Arg::with_name("unlocked_vlx") - .default_value("1.0") - .long("unlocked-vlx") - .takes_value(true) - .value_name("VLX_AMOUNT") - .help("Amount of VLX to put in system account to pay for fees"), - ) - .arg( - Arg::with_name("stake_authority") - .long("stake-authority") - .required(true) - .takes_value(true) - .value_name("KEYPAIR") - .validator(is_valid_signer) - .help("Stake Authority Keypair"), - ) - .arg( - Arg::with_name("withdraw_authority") - .long("withdraw-authority") - .required(true) - .takes_value(true) - .value_name("KEYPAIR") - .validator(is_valid_signer) - .help("Withdraw Authority Keypair"), - ) - .arg( - Arg::with_name("lockup_authority") - .long("lockup-authority") - .takes_value(true) - .value_name("KEYPAIR") - .validator(is_valid_signer) - .help("Lockup Authority Keypair"), - ) - .arg( - Arg::with_name("fee_payer") - .long("fee-payer") - .required(true) - .takes_value(true) - .value_name("KEYPAIR") - .validator(is_valid_signer) - .help("Fee payer"), - ), - ) - .subcommand( - SubCommand::with_name("distribute-spl-tokens") - .about("Distribute SPL tokens") - .arg( - Arg::with_name("db_path") - .long("db-path") - .required(true) - .takes_value(true) - .value_name("FILE") - .help( - "Location for storing distribution database. \ - The database is used for tracking transactions as they are finalized \ - and preventing double spends.", - ), - ) - .arg( - Arg::with_name("input_csv") - .long("input-csv") - .required(true) - .takes_value(true) - .value_name("FILE") - .help("Allocations CSV file"), - ) - .arg( - Arg::with_name("dry_run") - .long("dry-run") - .help("Do not execute any transfers"), - ) - .arg( - Arg::with_name("transfer_amount") - .long("transfer-amount") - .takes_value(true) - .value_name("AMOUNT") - .validator(is_amount) - .help("The amount of SPL tokens to send to each recipient"), - ) - .arg( - Arg::with_name("output_path") - .long("output-path") - .short("o") - .value_name("FILE") - .takes_value(true) - .help("Write the transaction log to this file"), - ) - .arg( - Arg::with_name("token_account_address") - .long("from") - .required(true) - .takes_value(true) - .value_name("TOKEN_ACCOUNT_ADDRESS") - .validator(is_valid_pubkey) - .help("SPL token account to send from"), - ) - .arg( - Arg::with_name("token_owner") - .long("owner") - .required(true) - .takes_value(true) - .value_name("TOKEN_ACCOUNT_OWNER_KEYPAIR") - .validator(is_valid_signer) - .help("SPL token account owner"), - ) - .arg( - Arg::with_name("fee_payer") - .long("fee-payer") - .required(true) - .takes_value(true) - .value_name("KEYPAIR") - .validator(is_valid_signer) - .help("Fee payer"), - ), - ) - .subcommand( - SubCommand::with_name("balances") - .about("Balance of each account") - .arg( - Arg::with_name("input_csv") - .long("input-csv") - .required(true) - .takes_value(true) - .value_name("FILE") - .help("Allocations CSV file"), - ), - ) - .subcommand( - SubCommand::with_name("spl-token-balances") - .about("Balance of SPL token associated accounts") - .arg( - Arg::with_name("input_csv") - .long("input-csv") - .required(true) - .takes_value(true) - .value_name("FILE") - .help("Allocations CSV file"), - ) - .arg( - Arg::with_name("mint_address") - .long("mint") - .required(true) - .takes_value(true) - .value_name("MINT_ADDRESS") - .validator(is_valid_pubkey) - .help("SPL token mint of distribution"), - ), - ) - .subcommand( - SubCommand::with_name("transaction-log") - .about("Print the database to a CSV file") - .arg( - Arg::with_name("db_path") - .long("db-path") - .required(true) - .takes_value(true) - .value_name("FILE") - .help("Location of database to query"), - ) - .arg( - Arg::with_name("output_path") - .long("output-path") - .required(true) - .takes_value(true) - .value_name("FILE") - .help("Output file"), - ), - ) - .get_matches_from(args) -} - -fn parse_distribute_tokens_args( - matches: &ArgMatches<'_>, -) -> Result> { - let mut wallet_manager = maybe_wallet_manager()?; - let signer_matches = ArgMatches::default(); // No default signer - - let sender_keypair_str = value_t_or_exit!(matches, "sender_keypair", String); - let sender_keypair = signer_from_path( - &signer_matches, - &sender_keypair_str, - "sender", - &mut wallet_manager, - )?; - - let fee_payer_str = value_t_or_exit!(matches, "fee_payer", String); - let fee_payer = signer_from_path( - &signer_matches, - &fee_payer_str, - "fee-payer", - &mut wallet_manager, - )?; - - Ok(DistributeTokensArgs { - input_csv: value_t_or_exit!(matches, "input_csv", String), - transaction_db: value_t_or_exit!(matches, "db_path", String), - output_path: matches.value_of("output_path").map(|path| path.to_string()), - dry_run: matches.is_present("dry_run"), - sender_keypair, - fee_payer, - stake_args: None, - spl_token_args: None, - transfer_amount: value_of(matches, "transfer_amount").map(sol_to_lamports), - }) -} - -fn parse_create_stake_args( - matches: &ArgMatches<'_>, -) -> Result> { - let mut wallet_manager = maybe_wallet_manager()?; - let signer_matches = ArgMatches::default(); // No default signer - - let sender_keypair_str = value_t_or_exit!(matches, "sender_keypair", String); - let sender_keypair = signer_from_path( - &signer_matches, - &sender_keypair_str, - "sender", - &mut wallet_manager, - )?; - - let fee_payer_str = value_t_or_exit!(matches, "fee_payer", String); - let fee_payer = signer_from_path( - &signer_matches, - &fee_payer_str, - "fee-payer", - &mut wallet_manager, - )?; - - let lockup_authority_str = value_t!(matches, "lockup_authority", String).ok(); - let lockup_authority = lockup_authority_str - .map(|path| { - pubkey_from_path( - &signer_matches, - &path, - "lockup authority", - &mut wallet_manager, - ) - }) - .transpose()?; - - let stake_args = StakeArgs { - unlocked_sol: sol_to_lamports(value_t_or_exit!(matches, "unlocked_sol", f64)), - lockup_authority, - sender_stake_args: None, - }; - Ok(DistributeTokensArgs { - input_csv: value_t_or_exit!(matches, "input_csv", String), - transaction_db: value_t_or_exit!(matches, "db_path", String), - output_path: matches.value_of("output_path").map(|path| path.to_string()), - dry_run: matches.is_present("dry_run"), - sender_keypair, - fee_payer, - stake_args: Some(stake_args), - spl_token_args: None, - transfer_amount: None, - }) -} - -fn parse_distribute_stake_args( - matches: &ArgMatches<'_>, -) -> Result> { - let mut wallet_manager = maybe_wallet_manager()?; - let signer_matches = ArgMatches::default(); // No default signer - - let sender_keypair_str = value_t_or_exit!(matches, "sender_keypair", String); - let sender_keypair = signer_from_path( - &signer_matches, - &sender_keypair_str, - "sender", - &mut wallet_manager, - )?; - - let fee_payer_str = value_t_or_exit!(matches, "fee_payer", String); - let fee_payer = signer_from_path( - &signer_matches, - &fee_payer_str, - "fee-payer", - &mut wallet_manager, - )?; - - let stake_account_address_str = value_t_or_exit!(matches, "stake_account_address", String); - let stake_account_address = pubkey_from_path( - &signer_matches, - &stake_account_address_str, - "stake account address", - &mut wallet_manager, - )?; - - let stake_authority_str = value_t_or_exit!(matches, "stake_authority", String); - let stake_authority = signer_from_path( - &signer_matches, - &stake_authority_str, - "stake authority", - &mut wallet_manager, - )?; - - let withdraw_authority_str = value_t_or_exit!(matches, "withdraw_authority", String); - let withdraw_authority = signer_from_path( - &signer_matches, - &withdraw_authority_str, - "withdraw authority", - &mut wallet_manager, - )?; - - let lockup_authority_str = value_t!(matches, "lockup_authority", String).ok(); - let lockup_authority = lockup_authority_str - .map(|path| { - signer_from_path( - &signer_matches, - &path, - "lockup authority", - &mut wallet_manager, - ) - }) - .transpose()?; - - let lockup_authority_address = lockup_authority.as_ref().map(|keypair| keypair.pubkey()); - let sender_stake_args = SenderStakeArgs { - stake_account_address, - stake_authority, - withdraw_authority, - lockup_authority, - }; - let stake_args = StakeArgs { - unlocked_sol: sol_to_lamports(value_t_or_exit!(matches, "unlocked_sol", f64)), - lockup_authority: lockup_authority_address, - sender_stake_args: Some(sender_stake_args), - }; - Ok(DistributeTokensArgs { - input_csv: value_t_or_exit!(matches, "input_csv", String), - transaction_db: value_t_or_exit!(matches, "db_path", String), - output_path: matches.value_of("output_path").map(|path| path.to_string()), - dry_run: matches.is_present("dry_run"), - sender_keypair, - fee_payer, - stake_args: Some(stake_args), - spl_token_args: None, - transfer_amount: None, - }) -} - -fn parse_distribute_spl_tokens_args( - matches: &ArgMatches<'_>, -) -> Result> { - let mut wallet_manager = maybe_wallet_manager()?; - let signer_matches = ArgMatches::default(); // No default signer - - let token_owner_str = value_t_or_exit!(matches, "token_owner", String); - let token_owner = signer_from_path( - &signer_matches, - &token_owner_str, - "owner", - &mut wallet_manager, - )?; - - let fee_payer_str = value_t_or_exit!(matches, "fee_payer", String); - let fee_payer = signer_from_path( - &signer_matches, - &fee_payer_str, - "fee-payer", - &mut wallet_manager, - )?; - - let token_account_address_str = value_t_or_exit!(matches, "token_account_address", String); - let token_account_address = pubkey_from_path( - &signer_matches, - &token_account_address_str, - "token account address", - &mut wallet_manager, - )?; - - Ok(DistributeTokensArgs { - input_csv: value_t_or_exit!(matches, "input_csv", String), - transaction_db: value_t_or_exit!(matches, "db_path", String), - output_path: matches.value_of("output_path").map(|path| path.to_string()), - dry_run: matches.is_present("dry_run"), - sender_keypair: token_owner, - fee_payer, - stake_args: None, - spl_token_args: Some(SplTokenArgs { - token_account_address, - ..SplTokenArgs::default() - }), - transfer_amount: value_of(matches, "transfer_amount"), - }) -} - -fn parse_balances_args(matches: &ArgMatches<'_>) -> Result> { - let mut wallet_manager = maybe_wallet_manager()?; - let spl_token_args = - pubkey_of_signer(matches, "mint_address", &mut wallet_manager)?.map(|mint| SplTokenArgs { - mint, - ..SplTokenArgs::default() - }); - Ok(BalancesArgs { - input_csv: value_t_or_exit!(matches, "input_csv", String), - spl_token_args, - }) -} - -fn parse_transaction_log_args(matches: &ArgMatches<'_>) -> TransactionLogArgs { - TransactionLogArgs { - transaction_db: value_t_or_exit!(matches, "db_path", String), - output_path: value_t_or_exit!(matches, "output_path", String), - } -} - -pub fn parse_args(args: I) -> Result> -where - I: IntoIterator, - T: Into + Clone, -{ - let matches = get_matches(args); - let config_file = matches.value_of("config_file").unwrap().to_string(); - let url = matches.value_of("url").map(|x| x.to_string()); - - let command = match matches.subcommand() { - ("distribute-tokens", Some(matches)) => { - Command::DistributeTokens(parse_distribute_tokens_args(matches)?) - } - ("create-stake", Some(matches)) => { - Command::DistributeTokens(parse_create_stake_args(matches)?) - } - ("distribute-stake", Some(matches)) => { - Command::DistributeTokens(parse_distribute_stake_args(matches)?) - } - ("distribute-spl-tokens", Some(matches)) => { - Command::DistributeTokens(parse_distribute_spl_tokens_args(matches)?) - } - ("balances", Some(matches)) => Command::Balances(parse_balances_args(matches)?), - ("spl-token-balances", Some(matches)) => Command::Balances(parse_balances_args(matches)?), - ("transaction-log", Some(matches)) => { - Command::TransactionLog(parse_transaction_log_args(matches)) - } - _ => { - eprintln!("{}", matches.usage()); - exit(1); - } - }; - let args = Args { - config_file, - url, - command, - }; - Ok(args) -} diff --git a/tokens/src/args.rs b/tokens/src/args.rs deleted file mode 100644 index b1f1522e15..0000000000 --- a/tokens/src/args.rs +++ /dev/null @@ -1,56 +0,0 @@ -use solana_sdk::{pubkey::Pubkey, signature::Signer}; - -pub struct SenderStakeArgs { - pub stake_account_address: Pubkey, - pub stake_authority: Box, - pub withdraw_authority: Box, - pub lockup_authority: Option>, -} - -pub struct StakeArgs { - pub unlocked_sol: u64, - pub lockup_authority: Option, - pub sender_stake_args: Option, -} - -pub struct DistributeTokensArgs { - pub input_csv: String, - pub transaction_db: String, - pub output_path: Option, - pub dry_run: bool, - pub sender_keypair: Box, - pub fee_payer: Box, - pub stake_args: Option, - pub spl_token_args: Option, - pub transfer_amount: Option, -} - -#[derive(Default)] -pub struct SplTokenArgs { - pub token_account_address: Pubkey, - pub mint: Pubkey, - pub decimals: u8, -} - -pub struct BalancesArgs { - pub input_csv: String, - pub spl_token_args: Option, -} - -pub struct TransactionLogArgs { - pub transaction_db: String, - pub output_path: String, -} - -#[allow(clippy::large_enum_variant)] -pub enum Command { - DistributeTokens(DistributeTokensArgs), - Balances(BalancesArgs), - TransactionLog(TransactionLogArgs), -} - -pub struct Args { - pub config_file: String, - pub url: Option, - pub command: Command, -} diff --git a/tokens/src/commands.rs b/tokens/src/commands.rs deleted file mode 100644 index b60a6e3988..0000000000 --- a/tokens/src/commands.rs +++ /dev/null @@ -1,2468 +0,0 @@ -use { - crate::{ - args::{ - BalancesArgs, DistributeTokensArgs, SenderStakeArgs, StakeArgs, TransactionLogArgs, - }, - db::{self, TransactionInfo}, - spl_token::*, - token_display::Token, - }, - chrono::prelude::*, - console::style, - csv::{ReaderBuilder, Trim}, - indexmap::IndexMap, - indicatif::{ProgressBar, ProgressStyle}, - pickledb::PickleDb, - serde::{Deserialize, Serialize}, - solana_account_decoder::parse_token::{ - pubkey_from_spl_token, real_number_string, spl_token_pubkey, - }, - solana_client::{ - client_error::{ClientError, Result as ClientResult}, - rpc_client::RpcClient, - rpc_config::RpcSendTransactionConfig, - rpc_request::MAX_GET_SIGNATURE_STATUSES_QUERY_ITEMS, - }, - solana_sdk::{ - clock::{Slot, DEFAULT_MS_PER_SLOT}, - commitment_config::CommitmentConfig, - hash::Hash, - instruction::Instruction, - message::Message, - native_token::{lamports_to_sol, sol_to_lamports}, - signature::{unique_signers, Signature, Signer}, - stake::{ - instruction::{self as stake_instruction, LockupArgs}, - state::{Authorized, Lockup, StakeAuthorize}, - }, - system_instruction, - transaction::Transaction, - }, - solana_transaction_status::TransactionStatus, - spl_associated_token_account::get_associated_token_address, - spl_token::solana_program::program_error::ProgramError, - std::{ - cmp::{self}, - io, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, - thread::sleep, - time::{Duration, Instant}, - }, -}; - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct Allocation { - pub recipient: String, - pub amount: u64, - pub lockup_date: String, -} - -#[derive(Debug, PartialEq)] -pub enum FundingSource { - FeePayer, - SplTokenAccount, - StakeAccount, - SystemAccount, -} - -pub struct FundingSources(Vec); - -impl std::fmt::Debug for FundingSources { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - for (i, source) in self.0.iter().enumerate() { - if i > 0 { - write!(f, "/")?; - } - write!(f, "{:?}", source)?; - } - Ok(()) - } -} - -impl PartialEq for FundingSources { - fn eq(&self, other: &Self) -> bool { - self.0 == other.0 - } -} - -impl From> for FundingSources { - fn from(sources_vec: Vec) -> Self { - Self(sources_vec) - } -} - -type StakeExtras = Vec<(Keypair, Option>)>; - -#[derive(thiserror::Error, Debug)] -pub enum Error { - #[error("I/O error")] - IoError(#[from] io::Error), - #[error("CSV error")] - CsvError(#[from] csv::Error), - #[error("PickleDb error")] - PickleDbError(#[from] pickledb::error::Error), - #[error("Transport error")] - ClientError(#[from] ClientError), - #[error("Missing lockup authority")] - MissingLockupAuthority, - #[error("insufficient funds in {0:?}, requires {1}")] - InsufficientFunds(FundingSources, String), - #[error("Program error")] - ProgramError(#[from] ProgramError), - #[error("Exit signal received")] - ExitSignal, -} - -fn merge_allocations(allocations: &[Allocation]) -> Vec { - let mut allocation_map = IndexMap::new(); - for allocation in allocations { - allocation_map - .entry(&allocation.recipient) - .or_insert(Allocation { - recipient: allocation.recipient.clone(), - amount: 0, - lockup_date: "".to_string(), - }) - .amount += allocation.amount; - } - allocation_map.values().cloned().collect() -} - -/// Return true if the recipient and lockups are the same -fn has_same_recipient(allocation: &Allocation, transaction_info: &TransactionInfo) -> bool { - allocation.recipient == transaction_info.recipient.to_string() - && allocation.lockup_date.parse().ok() == transaction_info.lockup_date -} - -fn apply_previous_transactions( - allocations: &mut Vec, - transaction_infos: &[TransactionInfo], -) { - for transaction_info in transaction_infos { - let mut amount = transaction_info.amount; - for allocation in allocations.iter_mut() { - if !has_same_recipient(allocation, transaction_info) { - continue; - } - if allocation.amount >= amount { - allocation.amount -= amount; - break; - } else { - amount -= allocation.amount; - allocation.amount = 0; - } - } - } - allocations.retain(|x| x.amount > 0); -} - -fn transfer( - client: &RpcClient, - lamports: u64, - sender_keypair: &S, - to_pubkey: &Pubkey, -) -> ClientResult { - let create_instruction = - system_instruction::transfer(&sender_keypair.pubkey(), to_pubkey, lamports); - let message = Message::new(&[create_instruction], Some(&sender_keypair.pubkey())); - let recent_blockhash = client.get_latest_blockhash()?; - Ok(Transaction::new( - &[sender_keypair], - message, - recent_blockhash, - )) -} - -fn distribution_instructions( - allocation: &Allocation, - new_stake_account_address: &Pubkey, - args: &DistributeTokensArgs, - lockup_date: Option>, - do_create_associated_token_account: bool, -) -> Vec { - if args.spl_token_args.is_some() { - return build_spl_token_instructions(allocation, args, do_create_associated_token_account); - } - - match &args.stake_args { - // No stake args; a simple token transfer. - None => { - let from = args.sender_keypair.pubkey(); - let to = allocation.recipient.parse().unwrap(); - let lamports = allocation.amount; - let instruction = system_instruction::transfer(&from, &to, lamports); - vec![instruction] - } - - // Stake args provided, so create a recipient stake account. - Some(stake_args) => { - let unlocked_sol = stake_args.unlocked_sol; - let sender_pubkey = args.sender_keypair.pubkey(); - let recipient = allocation.recipient.parse().unwrap(); - - let mut instructions = match &stake_args.sender_stake_args { - // No source stake account, so create a recipient stake account directly. - None => { - // Make the recipient both the new stake and withdraw authority - let authorized = Authorized { - staker: recipient, - withdrawer: recipient, - }; - let mut lockup = Lockup::default(); - if let Some(lockup_date) = lockup_date { - lockup.unix_timestamp = lockup_date.timestamp(); - } - if let Some(lockup_authority) = stake_args.lockup_authority { - lockup.custodian = lockup_authority; - } - stake_instruction::create_account( - &sender_pubkey, - new_stake_account_address, - &authorized, - &lockup, - allocation.amount - unlocked_sol, - ) - } - - // A sender stake account was provided, so create a recipient stake account by - // splitting the sender account. - Some(sender_stake_args) => { - let stake_authority = sender_stake_args.stake_authority.pubkey(); - let withdraw_authority = sender_stake_args.withdraw_authority.pubkey(); - let mut instructions = stake_instruction::split( - &sender_stake_args.stake_account_address, - &stake_authority, - allocation.amount - unlocked_sol, - new_stake_account_address, - ); - - // Make the recipient the new stake authority - instructions.push(stake_instruction::authorize( - new_stake_account_address, - &stake_authority, - &recipient, - StakeAuthorize::Staker, - None, - )); - - // Make the recipient the new withdraw authority - instructions.push(stake_instruction::authorize( - new_stake_account_address, - &withdraw_authority, - &recipient, - StakeAuthorize::Withdrawer, - None, - )); - - // Add lockup - if let Some(lockup_date) = lockup_date { - let lockup = LockupArgs { - unix_timestamp: Some(lockup_date.timestamp()), - epoch: None, - custodian: None, - }; - instructions.push(stake_instruction::set_lockup( - new_stake_account_address, - &lockup, - &stake_args.lockup_authority.unwrap(), - )); - } - - instructions - } - }; - - // Transfer some unlocked tokens to recipient, which they can use for transaction fees. - instructions.push(system_instruction::transfer( - &sender_pubkey, - &recipient, - unlocked_sol, - )); - - instructions - } - } -} - -fn build_messages( - client: &RpcClient, - db: &mut PickleDb, - allocations: &[Allocation], - args: &DistributeTokensArgs, - exit: Arc, - messages: &mut Vec, - stake_extras: &mut StakeExtras, - created_accounts: &mut u64, -) -> Result<(), Error> { - for allocation in allocations.iter() { - if exit.load(Ordering::SeqCst) { - db.dump()?; - return Err(Error::ExitSignal); - } - let new_stake_account_keypair = Keypair::new(); - let lockup_date = if allocation.lockup_date.is_empty() { - None - } else { - Some(allocation.lockup_date.parse::>().unwrap()) - }; - - let do_create_associated_token_account = if let Some(spl_token_args) = &args.spl_token_args - { - let wallet_address = allocation.recipient.parse().unwrap(); - let associated_token_address = get_associated_token_address( - &wallet_address, - &spl_token_pubkey(&spl_token_args.mint), - ); - let do_create_associated_token_account = client - .get_multiple_accounts(&[pubkey_from_spl_token(&associated_token_address)])?[0] - .is_none(); - if do_create_associated_token_account { - *created_accounts += 1; - } - println!( - "{:<44} {:>24}", - allocation.recipient, - real_number_string(allocation.amount, spl_token_args.decimals) - ); - do_create_associated_token_account - } else { - println!( - "{:<44} {:>24.9}", - allocation.recipient, - lamports_to_sol(allocation.amount) - ); - false - }; - let instructions = distribution_instructions( - allocation, - &new_stake_account_keypair.pubkey(), - args, - lockup_date, - do_create_associated_token_account, - ); - let fee_payer_pubkey = args.fee_payer.pubkey(); - let message = Message::new_with_blockhash( - &instructions, - Some(&fee_payer_pubkey), - &Hash::default(), // populated by a real blockhash for balance check and submission - ); - messages.push(message); - stake_extras.push((new_stake_account_keypair, lockup_date)); - } - Ok(()) -} - -fn send_messages( - client: &RpcClient, - db: &mut PickleDb, - allocations: &[Allocation], - args: &DistributeTokensArgs, - exit: Arc, - messages: Vec, - stake_extras: StakeExtras, -) -> Result<(), Error> { - for ((allocation, message), (new_stake_account_keypair, lockup_date)) in - allocations.iter().zip(messages).zip(stake_extras) - { - if exit.load(Ordering::SeqCst) { - db.dump()?; - return Err(Error::ExitSignal); - } - let new_stake_account_address = new_stake_account_keypair.pubkey(); - - let mut signers = vec![&*args.fee_payer, &*args.sender_keypair]; - if let Some(stake_args) = &args.stake_args { - signers.push(&new_stake_account_keypair); - if let Some(sender_stake_args) = &stake_args.sender_stake_args { - signers.push(&*sender_stake_args.stake_authority); - signers.push(&*sender_stake_args.withdraw_authority); - signers.push(&new_stake_account_keypair); - if !allocation.lockup_date.is_empty() { - if let Some(lockup_authority) = &sender_stake_args.lockup_authority { - signers.push(&**lockup_authority); - } else { - return Err(Error::MissingLockupAuthority); - } - } - } - } - let signers = unique_signers(signers); - let result: ClientResult<(Transaction, u64)> = { - if args.dry_run { - Ok((Transaction::new_unsigned(message), std::u64::MAX)) - } else { - let (blockhash, last_valid_block_height) = - client.get_latest_blockhash_with_commitment(CommitmentConfig::default())?; - let transaction = Transaction::new(&signers, message, blockhash); - let config = RpcSendTransactionConfig { - skip_preflight: true, - ..RpcSendTransactionConfig::default() - }; - client.send_transaction_with_config(&transaction, config)?; - Ok((transaction, last_valid_block_height)) - } - }; - match result { - Ok((transaction, last_valid_block_height)) => { - let new_stake_account_address_option = - args.stake_args.as_ref().map(|_| &new_stake_account_address); - db::set_transaction_info( - db, - &allocation.recipient.parse().unwrap(), - allocation.amount, - &transaction, - new_stake_account_address_option, - false, - last_valid_block_height, - lockup_date, - )?; - } - Err(e) => { - eprintln!("Error sending tokens to {}: {}", allocation.recipient, e); - } - }; - } - Ok(()) -} - -fn distribute_allocations( - client: &RpcClient, - db: &mut PickleDb, - allocations: &[Allocation], - args: &DistributeTokensArgs, - exit: Arc, -) -> Result<(), Error> { - let mut messages: Vec = vec![]; - let mut stake_extras: StakeExtras = vec![]; - let mut created_accounts = 0; - - build_messages( - client, - db, - allocations, - args, - exit.clone(), - &mut messages, - &mut stake_extras, - &mut created_accounts, - )?; - - if args.spl_token_args.is_some() { - check_spl_token_balances(&messages, allocations, client, args, created_accounts)?; - } else { - check_payer_balances(&messages, allocations, client, args)?; - } - - send_messages(client, db, allocations, args, exit, messages, stake_extras)?; - - db.dump()?; - Ok(()) -} - -#[allow(clippy::needless_collect)] -fn read_allocations( - input_csv: &str, - transfer_amount: Option, - require_lockup_heading: bool, - raw_amount: bool, -) -> io::Result> { - let mut rdr = ReaderBuilder::new().trim(Trim::All).from_path(input_csv)?; - let allocations = if let Some(amount) = transfer_amount { - let recipients: Vec = rdr - .deserialize() - .map(|recipient| recipient.unwrap()) - .collect(); - recipients - .into_iter() - .map(|recipient| Allocation { - recipient, - amount, - lockup_date: "".to_string(), - }) - .collect() - } else if require_lockup_heading { - let recipients: Vec<(String, f64, String)> = rdr - .deserialize() - .map(|recipient| recipient.unwrap()) - .collect(); - recipients - .into_iter() - .map(|(recipient, amount, lockup_date)| Allocation { - recipient, - amount: sol_to_lamports(amount), - lockup_date, - }) - .collect() - } else if raw_amount { - let recipients: Vec<(String, u64)> = rdr - .deserialize() - .map(|recipient| recipient.unwrap()) - .collect(); - recipients - .into_iter() - .map(|(recipient, amount)| Allocation { - recipient, - amount, - lockup_date: "".to_string(), - }) - .collect() - } else { - let recipients: Vec<(String, f64)> = rdr - .deserialize() - .map(|recipient| recipient.unwrap()) - .collect(); - recipients - .into_iter() - .map(|(recipient, amount)| Allocation { - recipient, - amount: sol_to_lamports(amount), - lockup_date: "".to_string(), - }) - .collect() - }; - Ok(allocations) -} - -fn new_spinner_progress_bar() -> ProgressBar { - let progress_bar = ProgressBar::new(42); - progress_bar - .set_style(ProgressStyle::default_spinner().template("{spinner:.green} {wide_msg}")); - progress_bar.enable_steady_tick(100); - progress_bar -} - -pub fn process_allocations( - client: &RpcClient, - args: &DistributeTokensArgs, - exit: Arc, -) -> Result, Error> { - let require_lockup_heading = args.stake_args.is_some(); - let mut allocations: Vec = read_allocations( - &args.input_csv, - args.transfer_amount, - require_lockup_heading, - args.spl_token_args.is_some(), - )?; - - let starting_total_tokens = allocations.iter().map(|x| x.amount).sum(); - let starting_total_tokens = if let Some(spl_token_args) = &args.spl_token_args { - Token::spl_token(starting_total_tokens, spl_token_args.decimals) - } else { - Token::sol(starting_total_tokens) - }; - println!( - "{} {}", - style("Total in input_csv:").bold(), - starting_total_tokens, - ); - - let mut db = db::open_db(&args.transaction_db, args.dry_run)?; - - // Start by finalizing any transactions from the previous run. - let confirmations = finalize_transactions(client, &mut db, args.dry_run, exit.clone())?; - - let transaction_infos = db::read_transaction_infos(&db); - apply_previous_transactions(&mut allocations, &transaction_infos); - - if allocations.is_empty() { - eprintln!("No work to do"); - return Ok(confirmations); - } - - let distributed_tokens = transaction_infos.iter().map(|x| x.amount).sum(); - let undistributed_tokens = allocations.iter().map(|x| x.amount).sum(); - let (distributed_tokens, undistributed_tokens) = - if let Some(spl_token_args) = &args.spl_token_args { - ( - Token::spl_token(distributed_tokens, spl_token_args.decimals), - Token::spl_token(undistributed_tokens, spl_token_args.decimals), - ) - } else { - ( - Token::sol(distributed_tokens), - Token::sol(undistributed_tokens), - ) - }; - println!("{} {}", style("Distributed:").bold(), distributed_tokens,); - println!( - "{} {}", - style("Undistributed:").bold(), - undistributed_tokens, - ); - println!( - "{} {}", - style("Total:").bold(), - distributed_tokens + undistributed_tokens, - ); - - println!( - "{}", - style(format!("{:<44} {:>24}", "Recipient", "Expected Balance",)).bold() - ); - - distribute_allocations(client, &mut db, &allocations, args, exit.clone())?; - - let opt_confirmations = finalize_transactions(client, &mut db, args.dry_run, exit)?; - - if !args.dry_run { - if let Some(output_path) = &args.output_path { - db::write_transaction_log(&db, &output_path)?; - } - } - - Ok(opt_confirmations) -} - -fn finalize_transactions( - client: &RpcClient, - db: &mut PickleDb, - dry_run: bool, - exit: Arc, -) -> Result, Error> { - if dry_run { - return Ok(None); - } - - let mut opt_confirmations = update_finalized_transactions(client, db, exit.clone())?; - - let progress_bar = new_spinner_progress_bar(); - - while opt_confirmations.is_some() { - if let Some(confirmations) = opt_confirmations { - progress_bar.set_message(format!( - "[{}/{}] Finalizing transactions", - confirmations, 32, - )); - } - - // Sleep for about 1 slot - sleep(Duration::from_millis(500)); - let opt_conf = update_finalized_transactions(client, db, exit.clone())?; - opt_confirmations = opt_conf; - } - - Ok(opt_confirmations) -} - -// Update the finalized bit on any transactions that are now rooted -// Return the lowest number of confirmations on the unfinalized transactions or None if all are finalized. -fn update_finalized_transactions( - client: &RpcClient, - db: &mut PickleDb, - exit: Arc, -) -> Result, Error> { - let transaction_infos = db::read_transaction_infos(db); - let unconfirmed_transactions: Vec<_> = transaction_infos - .iter() - .filter_map(|info| { - if info.finalized_date.is_some() { - None - } else { - Some((&info.transaction, info.last_valid_block_height)) - } - }) - .collect(); - let unconfirmed_signatures: Vec<_> = unconfirmed_transactions - .iter() - .map(|(tx, _slot)| tx.signatures[0]) - .filter(|sig| *sig != Signature::default()) // Filter out dry-run signatures - .collect(); - let mut statuses = vec![]; - for unconfirmed_signatures_chunk in - unconfirmed_signatures.chunks(MAX_GET_SIGNATURE_STATUSES_QUERY_ITEMS - 1) - { - statuses.extend( - client - .get_signature_statuses(unconfirmed_signatures_chunk)? - .value - .into_iter(), - ); - } - - let mut confirmations = None; - log_transaction_confirmations( - client, - db, - exit, - unconfirmed_transactions, - statuses, - &mut confirmations, - )?; - db.dump()?; - Ok(confirmations) -} - -fn log_transaction_confirmations( - client: &RpcClient, - db: &mut PickleDb, - exit: Arc, - unconfirmed_transactions: Vec<(&Transaction, Slot)>, - statuses: Vec>, - confirmations: &mut Option, -) -> Result<(), Error> { - let finalized_block_height = client.get_block_height()?; - for ((transaction, last_valid_block_height), opt_transaction_status) in unconfirmed_transactions - .into_iter() - .zip(statuses.into_iter()) - { - match db::update_finalized_transaction( - db, - &transaction.signatures[0], - opt_transaction_status, - last_valid_block_height, - finalized_block_height, - ) { - Ok(Some(confs)) => { - *confirmations = Some(cmp::min(confs, confirmations.unwrap_or(usize::MAX))); - } - result => { - result?; - } - } - if exit.load(Ordering::SeqCst) { - db.dump()?; - return Err(Error::ExitSignal); - } - } - Ok(()) -} - -pub fn get_fees_for_messages(messages: &[Message], client: &RpcClient) -> Result { - // This is an arbitrary value to get regular blockhash updates for balance checks without - // hitting the RPC node with too many requests - const BLOCKHASH_REFRESH_MILLIS: u64 = DEFAULT_MS_PER_SLOT * 32; - - let mut latest_blockhash = client.get_latest_blockhash()?; - let mut now = Instant::now(); - let mut fees = 0; - for mut message in messages.iter().cloned() { - if now.elapsed() > Duration::from_millis(BLOCKHASH_REFRESH_MILLIS) { - latest_blockhash = client.get_latest_blockhash()?; - now = Instant::now(); - } - message.recent_blockhash = latest_blockhash; - fees += client.get_fee_for_message(&message)?; - } - Ok(fees) -} - -fn check_payer_balances( - messages: &[Message], - allocations: &[Allocation], - client: &RpcClient, - args: &DistributeTokensArgs, -) -> Result<(), Error> { - let mut undistributed_tokens: u64 = allocations.iter().map(|x| x.amount).sum(); - let fees = get_fees_for_messages(messages, client)?; - - let (distribution_source, unlocked_sol_source) = if let Some(stake_args) = &args.stake_args { - let total_unlocked_sol = allocations.len() as u64 * stake_args.unlocked_sol; - undistributed_tokens -= total_unlocked_sol; - let from_pubkey = if let Some(sender_stake_args) = &stake_args.sender_stake_args { - sender_stake_args.stake_account_address - } else { - args.sender_keypair.pubkey() - }; - ( - from_pubkey, - Some((args.sender_keypair.pubkey(), total_unlocked_sol)), - ) - } else { - (args.sender_keypair.pubkey(), None) - }; - - let fee_payer_balance = client.get_balance(&args.fee_payer.pubkey())?; - if let Some((unlocked_sol_source, total_unlocked_sol)) = unlocked_sol_source { - let staker_balance = client.get_balance(&distribution_source)?; - if staker_balance < undistributed_tokens { - return Err(Error::InsufficientFunds( - vec![FundingSource::StakeAccount].into(), - lamports_to_sol(undistributed_tokens).to_string(), - )); - } - if args.fee_payer.pubkey() == unlocked_sol_source { - if fee_payer_balance < fees + total_unlocked_sol { - return Err(Error::InsufficientFunds( - vec![FundingSource::SystemAccount, FundingSource::FeePayer].into(), - lamports_to_sol(fees + total_unlocked_sol).to_string(), - )); - } - } else { - if fee_payer_balance < fees { - return Err(Error::InsufficientFunds( - vec![FundingSource::FeePayer].into(), - lamports_to_sol(fees).to_string(), - )); - } - let unlocked_sol_balance = client.get_balance(&unlocked_sol_source)?; - if unlocked_sol_balance < total_unlocked_sol { - return Err(Error::InsufficientFunds( - vec![FundingSource::SystemAccount].into(), - lamports_to_sol(total_unlocked_sol).to_string(), - )); - } - } - } else if args.fee_payer.pubkey() == distribution_source { - if fee_payer_balance < fees + undistributed_tokens { - return Err(Error::InsufficientFunds( - vec![FundingSource::SystemAccount, FundingSource::FeePayer].into(), - lamports_to_sol(fees + undistributed_tokens).to_string(), - )); - } - } else { - if fee_payer_balance < fees { - return Err(Error::InsufficientFunds( - vec![FundingSource::FeePayer].into(), - lamports_to_sol(fees).to_string(), - )); - } - let sender_balance = client.get_balance(&distribution_source)?; - if sender_balance < undistributed_tokens { - return Err(Error::InsufficientFunds( - vec![FundingSource::SystemAccount].into(), - lamports_to_sol(undistributed_tokens).to_string(), - )); - } - } - Ok(()) -} - -pub fn process_balances(client: &RpcClient, args: &BalancesArgs) -> Result<(), Error> { - let allocations: Vec = - read_allocations(&args.input_csv, None, false, args.spl_token_args.is_some())?; - let allocations = merge_allocations(&allocations); - - let token = if let Some(spl_token_args) = &args.spl_token_args { - spl_token_args.mint.to_string() - } else { - "◎".to_string() - }; - println!("{} {}", style("Token:").bold(), token); - - println!( - "{}", - style(format!( - "{:<44} {:>24} {:>24} {:>24}", - "Recipient", "Expected Balance", "Actual Balance", "Difference" - )) - .bold() - ); - - for allocation in &allocations { - if let Some(spl_token_args) = &args.spl_token_args { - print_token_balances(client, allocation, spl_token_args)?; - } else { - let address: Pubkey = allocation.recipient.parse().unwrap(); - let expected = lamports_to_sol(allocation.amount); - let actual = lamports_to_sol(client.get_balance(&address).unwrap()); - println!( - "{:<44} {:>24.9} {:>24.9} {:>24.9}", - allocation.recipient, - expected, - actual, - actual - expected, - ); - } - } - - Ok(()) -} - -pub fn process_transaction_log(args: &TransactionLogArgs) -> Result<(), Error> { - let db = db::open_db(&args.transaction_db, true)?; - db::write_transaction_log(&db, &args.output_path)?; - Ok(()) -} - -use { - crate::db::check_output_file, - solana_sdk::{pubkey::Pubkey, signature::Keypair}, - tempfile::{tempdir, NamedTempFile}, -}; -pub fn test_process_distribute_tokens_with_client( - client: &RpcClient, - sender_keypair: Keypair, - transfer_amount: Option, -) { - let exit = Arc::new(AtomicBool::default()); - let fee_payer = Keypair::new(); - let transaction = transfer( - client, - sol_to_lamports(1.0), - &sender_keypair, - &fee_payer.pubkey(), - ) - .unwrap(); - client - .send_and_confirm_transaction_with_spinner(&transaction) - .unwrap(); - assert_eq!( - client.get_balance(&fee_payer.pubkey()).unwrap(), - sol_to_lamports(1.0), - ); - - let expected_amount = if let Some(amount) = transfer_amount { - amount - } else { - sol_to_lamports(1000.0) - }; - let alice_pubkey = solana_sdk::pubkey::new_rand(); - let allocations_file = NamedTempFile::new().unwrap(); - let input_csv = allocations_file.path().to_str().unwrap().to_string(); - let mut wtr = csv::WriterBuilder::new().from_writer(allocations_file); - wtr.write_record(["recipient", "amount"]).unwrap(); - wtr.write_record(&[ - alice_pubkey.to_string(), - lamports_to_sol(expected_amount).to_string(), - ]) - .unwrap(); - wtr.flush().unwrap(); - - let dir = tempdir().unwrap(); - let transaction_db = dir - .path() - .join("transactions.db") - .to_str() - .unwrap() - .to_string(); - - let output_file = NamedTempFile::new().unwrap(); - let output_path = output_file.path().to_str().unwrap().to_string(); - - let args = DistributeTokensArgs { - sender_keypair: Box::new(sender_keypair), - fee_payer: Box::new(fee_payer), - dry_run: false, - input_csv, - transaction_db: transaction_db.clone(), - output_path: Some(output_path.clone()), - stake_args: None, - spl_token_args: None, - transfer_amount, - }; - let confirmations = process_allocations(client, &args, exit.clone()).unwrap(); - assert_eq!(confirmations, None); - - let transaction_infos = - db::read_transaction_infos(&db::open_db(&transaction_db, true).unwrap()); - assert_eq!(transaction_infos.len(), 1); - assert_eq!(transaction_infos[0].recipient, alice_pubkey); - assert_eq!(transaction_infos[0].amount, expected_amount); - - assert_eq!(client.get_balance(&alice_pubkey).unwrap(), expected_amount); - - check_output_file(&output_path, &db::open_db(&transaction_db, true).unwrap()); - - // Now, run it again, and check there's no double-spend. - process_allocations(client, &args, exit).unwrap(); - let transaction_infos = - db::read_transaction_infos(&db::open_db(&transaction_db, true).unwrap()); - assert_eq!(transaction_infos.len(), 1); - assert_eq!(transaction_infos[0].recipient, alice_pubkey); - assert_eq!(transaction_infos[0].amount, expected_amount); - - assert_eq!(client.get_balance(&alice_pubkey).unwrap(), expected_amount); - - check_output_file(&output_path, &db::open_db(&transaction_db, true).unwrap()); -} - -pub fn test_process_create_stake_with_client(client: &RpcClient, sender_keypair: Keypair) { - let exit = Arc::new(AtomicBool::default()); - let fee_payer = Keypair::new(); - let transaction = transfer( - client, - sol_to_lamports(1.0), - &sender_keypair, - &fee_payer.pubkey(), - ) - .unwrap(); - client - .send_and_confirm_transaction_with_spinner(&transaction) - .unwrap(); - - let stake_account_keypair = Keypair::new(); - let stake_account_address = stake_account_keypair.pubkey(); - let stake_authority = Keypair::new(); - let withdraw_authority = Keypair::new(); - - let authorized = Authorized { - staker: stake_authority.pubkey(), - withdrawer: withdraw_authority.pubkey(), - }; - let lockup = Lockup::default(); - let instructions = stake_instruction::create_account( - &sender_keypair.pubkey(), - &stake_account_address, - &authorized, - &lockup, - sol_to_lamports(3000.0), - ); - let message = Message::new(&instructions, Some(&sender_keypair.pubkey())); - let signers = [&sender_keypair, &stake_account_keypair]; - let blockhash = client.get_latest_blockhash().unwrap(); - let transaction = Transaction::new(&signers, message, blockhash); - client - .send_and_confirm_transaction_with_spinner(&transaction) - .unwrap(); - - let expected_amount = sol_to_lamports(1000.0); - let alice_pubkey = solana_sdk::pubkey::new_rand(); - let file = NamedTempFile::new().unwrap(); - let input_csv = file.path().to_str().unwrap().to_string(); - let mut wtr = csv::WriterBuilder::new().from_writer(file); - wtr.write_record(["recipient", "amount", "lockup_date"]) - .unwrap(); - wtr.write_record(&[ - alice_pubkey.to_string(), - lamports_to_sol(expected_amount).to_string(), - "".to_string(), - ]) - .unwrap(); - wtr.flush().unwrap(); - - let dir = tempdir().unwrap(); - let transaction_db = dir - .path() - .join("transactions.db") - .to_str() - .unwrap() - .to_string(); - - let output_file = NamedTempFile::new().unwrap(); - let output_path = output_file.path().to_str().unwrap().to_string(); - - let stake_args = StakeArgs { - lockup_authority: None, - unlocked_sol: sol_to_lamports(1.0), - sender_stake_args: None, - }; - let args = DistributeTokensArgs { - fee_payer: Box::new(fee_payer), - dry_run: false, - input_csv, - transaction_db: transaction_db.clone(), - output_path: Some(output_path.clone()), - stake_args: Some(stake_args), - spl_token_args: None, - sender_keypair: Box::new(sender_keypair), - transfer_amount: None, - }; - let confirmations = process_allocations(client, &args, exit.clone()).unwrap(); - assert_eq!(confirmations, None); - - let transaction_infos = - db::read_transaction_infos(&db::open_db(&transaction_db, true).unwrap()); - assert_eq!(transaction_infos.len(), 1); - assert_eq!(transaction_infos[0].recipient, alice_pubkey); - assert_eq!(transaction_infos[0].amount, expected_amount); - - assert_eq!( - client.get_balance(&alice_pubkey).unwrap(), - sol_to_lamports(1.0), - ); - let new_stake_account_address = transaction_infos[0].new_stake_account_address.unwrap(); - assert_eq!( - client.get_balance(&new_stake_account_address).unwrap(), - expected_amount - sol_to_lamports(1.0), - ); - - check_output_file(&output_path, &db::open_db(&transaction_db, true).unwrap()); - - // Now, run it again, and check there's no double-spend. - process_allocations(client, &args, exit).unwrap(); - let transaction_infos = - db::read_transaction_infos(&db::open_db(&transaction_db, true).unwrap()); - assert_eq!(transaction_infos.len(), 1); - assert_eq!(transaction_infos[0].recipient, alice_pubkey); - assert_eq!(transaction_infos[0].amount, expected_amount); - - assert_eq!( - client.get_balance(&alice_pubkey).unwrap(), - sol_to_lamports(1.0), - ); - assert_eq!( - client.get_balance(&new_stake_account_address).unwrap(), - expected_amount - sol_to_lamports(1.0), - ); - - check_output_file(&output_path, &db::open_db(&transaction_db, true).unwrap()); -} - -pub fn test_process_distribute_stake_with_client(client: &RpcClient, sender_keypair: Keypair) { - let exit = Arc::new(AtomicBool::default()); - let fee_payer = Keypair::new(); - let transaction = transfer( - client, - sol_to_lamports(1.0), - &sender_keypair, - &fee_payer.pubkey(), - ) - .unwrap(); - client - .send_and_confirm_transaction_with_spinner(&transaction) - .unwrap(); - - let stake_account_keypair = Keypair::new(); - let stake_account_address = stake_account_keypair.pubkey(); - let stake_authority = Keypair::new(); - let withdraw_authority = Keypair::new(); - - let authorized = Authorized { - staker: stake_authority.pubkey(), - withdrawer: withdraw_authority.pubkey(), - }; - let lockup = Lockup::default(); - let instructions = stake_instruction::create_account( - &sender_keypair.pubkey(), - &stake_account_address, - &authorized, - &lockup, - sol_to_lamports(3000.0), - ); - let message = Message::new(&instructions, Some(&sender_keypair.pubkey())); - let signers = [&sender_keypair, &stake_account_keypair]; - let blockhash = client.get_latest_blockhash().unwrap(); - let transaction = Transaction::new(&signers, message, blockhash); - client - .send_and_confirm_transaction_with_spinner(&transaction) - .unwrap(); - - let expected_amount = sol_to_lamports(1000.0); - let alice_pubkey = solana_sdk::pubkey::new_rand(); - let file = NamedTempFile::new().unwrap(); - let input_csv = file.path().to_str().unwrap().to_string(); - let mut wtr = csv::WriterBuilder::new().from_writer(file); - wtr.write_record(["recipient", "amount", "lockup_date"]) - .unwrap(); - wtr.write_record(&[ - alice_pubkey.to_string(), - lamports_to_sol(expected_amount).to_string(), - "".to_string(), - ]) - .unwrap(); - wtr.flush().unwrap(); - - let dir = tempdir().unwrap(); - let transaction_db = dir - .path() - .join("transactions.db") - .to_str() - .unwrap() - .to_string(); - - let output_file = NamedTempFile::new().unwrap(); - let output_path = output_file.path().to_str().unwrap().to_string(); - - let sender_stake_args = SenderStakeArgs { - stake_account_address, - stake_authority: Box::new(stake_authority), - withdraw_authority: Box::new(withdraw_authority), - lockup_authority: None, - }; - let stake_args = StakeArgs { - unlocked_sol: sol_to_lamports(1.0), - lockup_authority: None, - sender_stake_args: Some(sender_stake_args), - }; - let args = DistributeTokensArgs { - fee_payer: Box::new(fee_payer), - dry_run: false, - input_csv, - transaction_db: transaction_db.clone(), - output_path: Some(output_path.clone()), - stake_args: Some(stake_args), - spl_token_args: None, - sender_keypair: Box::new(sender_keypair), - transfer_amount: None, - }; - let confirmations = process_allocations(client, &args, exit.clone()).unwrap(); - assert_eq!(confirmations, None); - - let transaction_infos = - db::read_transaction_infos(&db::open_db(&transaction_db, true).unwrap()); - assert_eq!(transaction_infos.len(), 1); - assert_eq!(transaction_infos[0].recipient, alice_pubkey); - assert_eq!(transaction_infos[0].amount, expected_amount); - - assert_eq!( - client.get_balance(&alice_pubkey).unwrap(), - sol_to_lamports(1.0), - ); - let new_stake_account_address = transaction_infos[0].new_stake_account_address.unwrap(); - assert_eq!( - client.get_balance(&new_stake_account_address).unwrap(), - expected_amount - sol_to_lamports(1.0), - ); - - check_output_file(&output_path, &db::open_db(&transaction_db, true).unwrap()); - - // Now, run it again, and check there's no double-spend. - process_allocations(client, &args, exit).unwrap(); - let transaction_infos = - db::read_transaction_infos(&db::open_db(&transaction_db, true).unwrap()); - assert_eq!(transaction_infos.len(), 1); - assert_eq!(transaction_infos[0].recipient, alice_pubkey); - assert_eq!(transaction_infos[0].amount, expected_amount); - - assert_eq!( - client.get_balance(&alice_pubkey).unwrap(), - sol_to_lamports(1.0), - ); - assert_eq!( - client.get_balance(&new_stake_account_address).unwrap(), - expected_amount - sol_to_lamports(1.0), - ); - - check_output_file(&output_path, &db::open_db(&transaction_db, true).unwrap()); -} - -#[cfg(test)] -mod tests { - use { - super::*, - solana_sdk::{ - instruction::AccountMeta, - signature::{read_keypair_file, write_keypair_file, Signer}, - stake::instruction::StakeInstruction, - }, - solana_streamer::socket::SocketAddrSpace, - solana_test_validator::TestValidator, - solana_transaction_status::TransactionConfirmationStatus, - }; - - fn one_signer_message(client: &RpcClient) -> Message { - Message::new_with_blockhash( - &[Instruction::new_with_bytes( - Pubkey::new_unique(), - &[], - vec![AccountMeta::new(Pubkey::default(), true)], - )], - None, - &client.get_latest_blockhash().unwrap(), - ) - } - - #[test] - fn test_process_token_allocations() { - let alice = Keypair::new(); - let test_validator = - TestValidator::with_no_fees(alice.pubkey(), None, SocketAddrSpace::Unspecified); - let url = test_validator.rpc_url(); - - let client = RpcClient::new_with_commitment(url, CommitmentConfig::processed()); - test_process_distribute_tokens_with_client(&client, alice, None); - } - - #[test] - fn test_process_transfer_amount_allocations() { - let alice = Keypair::new(); - let test_validator = - TestValidator::with_no_fees(alice.pubkey(), None, SocketAddrSpace::Unspecified); - let url = test_validator.rpc_url(); - - let client = RpcClient::new_with_commitment(url, CommitmentConfig::processed()); - test_process_distribute_tokens_with_client(&client, alice, Some(sol_to_lamports(1.5))); - } - - #[test] - fn test_create_stake_allocations() { - let alice = Keypair::new(); - let test_validator = - TestValidator::with_no_fees(alice.pubkey(), None, SocketAddrSpace::Unspecified); - let url = test_validator.rpc_url(); - - let client = RpcClient::new_with_commitment(url, CommitmentConfig::processed()); - test_process_create_stake_with_client(&client, alice); - } - - #[test] - fn test_process_stake_allocations() { - let alice = Keypair::new(); - let test_validator = - TestValidator::with_no_fees(alice.pubkey(), None, SocketAddrSpace::Unspecified); - let url = test_validator.rpc_url(); - - let client = RpcClient::new_with_commitment(url, CommitmentConfig::processed()); - test_process_distribute_stake_with_client(&client, alice); - } - - #[test] - fn test_read_allocations() { - let alice_pubkey = solana_sdk::pubkey::new_rand(); - let allocation = Allocation { - recipient: alice_pubkey.to_string(), - amount: 42, - lockup_date: "".to_string(), - }; - let file = NamedTempFile::new().unwrap(); - let input_csv = file.path().to_str().unwrap().to_string(); - let mut wtr = csv::WriterBuilder::new().from_writer(file); - wtr.serialize(&allocation).unwrap(); - wtr.flush().unwrap(); - - assert_eq!( - read_allocations(&input_csv, None, false, true).unwrap(), - vec![allocation] - ); - - let allocation_sol = Allocation { - recipient: alice_pubkey.to_string(), - amount: sol_to_lamports(42.0), - lockup_date: "".to_string(), - }; - - assert_eq!( - read_allocations(&input_csv, None, true, true).unwrap(), - vec![allocation_sol.clone()] - ); - assert_eq!( - read_allocations(&input_csv, None, false, false).unwrap(), - vec![allocation_sol.clone()] - ); - assert_eq!( - read_allocations(&input_csv, None, true, false).unwrap(), - vec![allocation_sol] - ); - } - - #[test] - fn test_read_allocations_no_lockup() { - let pubkey0 = solana_sdk::pubkey::new_rand(); - let pubkey1 = solana_sdk::pubkey::new_rand(); - let file = NamedTempFile::new().unwrap(); - let input_csv = file.path().to_str().unwrap().to_string(); - let mut wtr = csv::WriterBuilder::new().from_writer(file); - wtr.serialize(("recipient".to_string(), "amount".to_string())) - .unwrap(); - wtr.serialize((&pubkey0.to_string(), 42.0)).unwrap(); - wtr.serialize((&pubkey1.to_string(), 43.0)).unwrap(); - wtr.flush().unwrap(); - - let expected_allocations = vec![ - Allocation { - recipient: pubkey0.to_string(), - amount: sol_to_lamports(42.0), - lockup_date: "".to_string(), - }, - Allocation { - recipient: pubkey1.to_string(), - amount: sol_to_lamports(43.0), - lockup_date: "".to_string(), - }, - ]; - assert_eq!( - read_allocations(&input_csv, None, false, false).unwrap(), - expected_allocations - ); - } - - #[test] - #[should_panic] - fn test_read_allocations_malformed() { - let pubkey0 = solana_sdk::pubkey::new_rand(); - let pubkey1 = solana_sdk::pubkey::new_rand(); - let file = NamedTempFile::new().unwrap(); - let input_csv = file.path().to_str().unwrap().to_string(); - let mut wtr = csv::WriterBuilder::new().from_writer(file); - wtr.serialize(("recipient".to_string(), "amount".to_string())) - .unwrap(); - wtr.serialize((&pubkey0.to_string(), 42.0)).unwrap(); - wtr.serialize((&pubkey1.to_string(), 43.0)).unwrap(); - wtr.flush().unwrap(); - - let expected_allocations = vec![ - Allocation { - recipient: pubkey0.to_string(), - amount: sol_to_lamports(42.0), - lockup_date: "".to_string(), - }, - Allocation { - recipient: pubkey1.to_string(), - amount: sol_to_lamports(43.0), - lockup_date: "".to_string(), - }, - ]; - assert_eq!( - read_allocations(&input_csv, None, true, false).unwrap(), - expected_allocations - ); - } - - #[test] - fn test_read_allocations_transfer_amount() { - let pubkey0 = solana_sdk::pubkey::new_rand(); - let pubkey1 = solana_sdk::pubkey::new_rand(); - let pubkey2 = solana_sdk::pubkey::new_rand(); - let file = NamedTempFile::new().unwrap(); - let input_csv = file.path().to_str().unwrap().to_string(); - let mut wtr = csv::WriterBuilder::new().from_writer(file); - wtr.serialize("recipient".to_string()).unwrap(); - wtr.serialize(&pubkey0.to_string()).unwrap(); - wtr.serialize(&pubkey1.to_string()).unwrap(); - wtr.serialize(&pubkey2.to_string()).unwrap(); - wtr.flush().unwrap(); - - let amount = sol_to_lamports(1.5); - - let expected_allocations = vec![ - Allocation { - recipient: pubkey0.to_string(), - amount, - lockup_date: "".to_string(), - }, - Allocation { - recipient: pubkey1.to_string(), - amount, - lockup_date: "".to_string(), - }, - Allocation { - recipient: pubkey2.to_string(), - amount, - lockup_date: "".to_string(), - }, - ]; - assert_eq!( - read_allocations(&input_csv, Some(amount), false, false).unwrap(), - expected_allocations - ); - } - - #[test] - fn test_apply_previous_transactions() { - let alice = solana_sdk::pubkey::new_rand(); - let bob = solana_sdk::pubkey::new_rand(); - let mut allocations = vec![ - Allocation { - recipient: alice.to_string(), - amount: sol_to_lamports(1.0), - lockup_date: "".to_string(), - }, - Allocation { - recipient: bob.to_string(), - amount: sol_to_lamports(1.0), - lockup_date: "".to_string(), - }, - ]; - let transaction_infos = vec![TransactionInfo { - recipient: bob, - amount: sol_to_lamports(1.0), - ..TransactionInfo::default() - }]; - apply_previous_transactions(&mut allocations, &transaction_infos); - assert_eq!(allocations.len(), 1); - - // Ensure that we applied the transaction to the allocation with - // a matching recipient address (to bob, not alice). - assert_eq!(allocations[0].recipient, alice.to_string()); - } - - #[test] - fn test_has_same_recipient() { - let alice_pubkey = solana_sdk::pubkey::new_rand(); - let bob_pubkey = solana_sdk::pubkey::new_rand(); - let lockup0 = "2021-01-07T00:00:00Z".to_string(); - let lockup1 = "9999-12-31T23:59:59Z".to_string(); - let alice_alloc = Allocation { - recipient: alice_pubkey.to_string(), - amount: sol_to_lamports(1.0), - lockup_date: "".to_string(), - }; - let alice_alloc_lockup0 = Allocation { - recipient: alice_pubkey.to_string(), - amount: sol_to_lamports(1.0), - lockup_date: lockup0.clone(), - }; - let alice_info = TransactionInfo { - recipient: alice_pubkey, - lockup_date: None, - ..TransactionInfo::default() - }; - let alice_info_lockup0 = TransactionInfo { - recipient: alice_pubkey, - lockup_date: lockup0.parse().ok(), - ..TransactionInfo::default() - }; - let alice_info_lockup1 = TransactionInfo { - recipient: alice_pubkey, - lockup_date: lockup1.parse().ok(), - ..TransactionInfo::default() - }; - let bob_info = TransactionInfo { - recipient: bob_pubkey, - lockup_date: None, - ..TransactionInfo::default() - }; - assert!(!has_same_recipient(&alice_alloc, &bob_info)); // Different recipient, no lockup - assert!(!has_same_recipient(&alice_alloc, &alice_info_lockup0)); // One with no lockup, one locked up - assert!(!has_same_recipient( - &alice_alloc_lockup0, - &alice_info_lockup1 - )); // Different lockups - assert!(has_same_recipient(&alice_alloc, &alice_info)); // Same recipient, no lockups - assert!(has_same_recipient( - &alice_alloc_lockup0, - &alice_info_lockup0 - )); // Same recipient, same lockups - } - - const SET_LOCKUP_INDEX: usize = 5; - - #[test] - fn test_set_split_stake_lockup() { - let lockup_date_str = "2021-01-07T00:00:00Z"; - let allocation = Allocation { - recipient: Pubkey::default().to_string(), - amount: sol_to_lamports(1.0), - lockup_date: lockup_date_str.to_string(), - }; - let stake_account_address = solana_sdk::pubkey::new_rand(); - let new_stake_account_address = solana_sdk::pubkey::new_rand(); - let lockup_authority = Keypair::new(); - let lockup_authority_address = lockup_authority.pubkey(); - let sender_stake_args = SenderStakeArgs { - stake_account_address, - stake_authority: Box::new(Keypair::new()), - withdraw_authority: Box::new(Keypair::new()), - lockup_authority: Some(Box::new(lockup_authority)), - }; - let stake_args = StakeArgs { - lockup_authority: Some(lockup_authority_address), - unlocked_sol: sol_to_lamports(1.0), - sender_stake_args: Some(sender_stake_args), - }; - let args = DistributeTokensArgs { - fee_payer: Box::new(Keypair::new()), - dry_run: false, - input_csv: "".to_string(), - transaction_db: "".to_string(), - output_path: None, - stake_args: Some(stake_args), - spl_token_args: None, - sender_keypair: Box::new(Keypair::new()), - transfer_amount: None, - }; - let lockup_date = lockup_date_str.parse().unwrap(); - let instructions = distribution_instructions( - &allocation, - &new_stake_account_address, - &args, - Some(lockup_date), - false, - ); - let lockup_instruction = - bincode::deserialize(&instructions[SET_LOCKUP_INDEX].data).unwrap(); - if let StakeInstruction::SetLockup(lockup_args) = lockup_instruction { - assert_eq!(lockup_args.unix_timestamp, Some(lockup_date.timestamp())); - assert_eq!(lockup_args.epoch, None); // Don't change the epoch - assert_eq!(lockup_args.custodian, None); // Don't change the lockup authority - } else { - panic!("expected SetLockup instruction"); - } - } - - fn tmp_file_path(name: &str, pubkey: &Pubkey) -> String { - use std::env; - let out_dir = env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_string()); - - format!("{}/tmp/{}-{}", out_dir, name, pubkey) - } - - fn initialize_check_payer_balances_inputs( - allocation_amount: u64, - sender_keypair_file: &str, - fee_payer: &str, - stake_args: Option, - ) -> (Vec, DistributeTokensArgs) { - let recipient = solana_sdk::pubkey::new_rand(); - let allocations = vec![Allocation { - recipient: recipient.to_string(), - amount: allocation_amount, - lockup_date: "".to_string(), - }]; - let args = DistributeTokensArgs { - sender_keypair: read_keypair_file(sender_keypair_file).unwrap().into(), - fee_payer: read_keypair_file(fee_payer).unwrap().into(), - dry_run: false, - input_csv: "".to_string(), - transaction_db: "".to_string(), - output_path: None, - stake_args, - spl_token_args: None, - transfer_amount: None, - }; - (allocations, args) - } - - #[test] - fn test_check_payer_balances_distribute_tokens_single_payer() { - let alice = Keypair::new(); - let test_validator = TestValidator::with_custom_fees( - alice.pubkey(), - 10_000, - None, - SocketAddrSpace::Unspecified, - ); - let url = test_validator.rpc_url(); - - let client = RpcClient::new_with_commitment(url, CommitmentConfig::processed()); - let sender_keypair_file = tmp_file_path("keypair_file", &alice.pubkey()); - write_keypair_file(&alice, &sender_keypair_file).unwrap(); - - let fees = client - .get_fee_for_message(&one_signer_message(&client)) - .unwrap(); - let fees_in_sol = lamports_to_sol(fees); - - let allocation_amount = 1000.0; - - // Fully funded payer - let (allocations, mut args) = initialize_check_payer_balances_inputs( - sol_to_lamports(allocation_amount), - &sender_keypair_file, - &sender_keypair_file, - None, - ); - check_payer_balances(&[one_signer_message(&client)], &allocations, &client, &args).unwrap(); - - // Unfunded payer - let unfunded_payer = Keypair::new(); - let unfunded_payer_keypair_file = tmp_file_path("keypair_file", &unfunded_payer.pubkey()); - write_keypair_file(&unfunded_payer, &unfunded_payer_keypair_file).unwrap(); - args.sender_keypair = read_keypair_file(&unfunded_payer_keypair_file) - .unwrap() - .into(); - args.fee_payer = read_keypair_file(&unfunded_payer_keypair_file) - .unwrap() - .into(); - - let err_result = - check_payer_balances(&[one_signer_message(&client)], &allocations, &client, &args) - .unwrap_err(); - if let Error::InsufficientFunds(sources, amount) = err_result { - assert_eq!( - sources, - vec![FundingSource::SystemAccount, FundingSource::FeePayer].into() - ); - assert_eq!(amount, (allocation_amount + fees_in_sol).to_string()); - } else { - panic!("check_payer_balances should have errored"); - } - - // Payer funded enough for distribution only - let partially_funded_payer = Keypair::new(); - let partially_funded_payer_keypair_file = - tmp_file_path("keypair_file", &partially_funded_payer.pubkey()); - write_keypair_file( - &partially_funded_payer, - &partially_funded_payer_keypair_file, - ) - .unwrap(); - let transaction = transfer( - &client, - sol_to_lamports(allocation_amount), - &alice, - &partially_funded_payer.pubkey(), - ) - .unwrap(); - client - .send_and_confirm_transaction_with_spinner(&transaction) - .unwrap(); - - args.sender_keypair = read_keypair_file(&partially_funded_payer_keypair_file) - .unwrap() - .into(); - args.fee_payer = read_keypair_file(&partially_funded_payer_keypair_file) - .unwrap() - .into(); - let err_result = - check_payer_balances(&[one_signer_message(&client)], &allocations, &client, &args) - .unwrap_err(); - if let Error::InsufficientFunds(sources, amount) = err_result { - assert_eq!( - sources, - vec![FundingSource::SystemAccount, FundingSource::FeePayer].into() - ); - assert_eq!(amount, (allocation_amount + fees_in_sol).to_string()); - } else { - panic!("check_payer_balances should have errored"); - } - } - - #[test] - fn test_check_payer_balances_distribute_tokens_separate_payers() { - let alice = Keypair::new(); - let test_validator = TestValidator::with_custom_fees( - alice.pubkey(), - 10_000, - None, - SocketAddrSpace::Unspecified, - ); - let url = test_validator.rpc_url(); - - let client = RpcClient::new_with_commitment(url, CommitmentConfig::processed()); - - let fees = client - .get_fee_for_message(&one_signer_message(&client)) - .unwrap(); - let fees_in_sol = lamports_to_sol(fees); - - let sender_keypair_file = tmp_file_path("keypair_file", &alice.pubkey()); - write_keypair_file(&alice, &sender_keypair_file).unwrap(); - - let allocation_amount = 1000.0; - - let funded_payer = Keypair::new(); - let funded_payer_keypair_file = tmp_file_path("keypair_file", &funded_payer.pubkey()); - write_keypair_file(&funded_payer, &funded_payer_keypair_file).unwrap(); - let transaction = transfer( - &client, - sol_to_lamports(allocation_amount), - &alice, - &funded_payer.pubkey(), - ) - .unwrap(); - client - .send_and_confirm_transaction_with_spinner(&transaction) - .unwrap(); - - // Fully funded payers - let (allocations, mut args) = initialize_check_payer_balances_inputs( - sol_to_lamports(allocation_amount), - &funded_payer_keypair_file, - &sender_keypair_file, - None, - ); - check_payer_balances(&[one_signer_message(&client)], &allocations, &client, &args).unwrap(); - - // Unfunded sender - let unfunded_payer = Keypair::new(); - let unfunded_payer_keypair_file = tmp_file_path("keypair_file", &unfunded_payer.pubkey()); - write_keypair_file(&unfunded_payer, &unfunded_payer_keypair_file).unwrap(); - args.sender_keypair = read_keypair_file(&unfunded_payer_keypair_file) - .unwrap() - .into(); - args.fee_payer = read_keypair_file(&sender_keypair_file).unwrap().into(); - - let err_result = - check_payer_balances(&[one_signer_message(&client)], &allocations, &client, &args) - .unwrap_err(); - if let Error::InsufficientFunds(sources, amount) = err_result { - assert_eq!(sources, vec![FundingSource::SystemAccount].into()); - assert_eq!(amount, allocation_amount.to_string()); - } else { - panic!("check_payer_balances should have errored"); - } - - // Unfunded fee payer - args.sender_keypair = read_keypair_file(&sender_keypair_file).unwrap().into(); - args.fee_payer = read_keypair_file(&unfunded_payer_keypair_file) - .unwrap() - .into(); - - let err_result = - check_payer_balances(&[one_signer_message(&client)], &allocations, &client, &args) - .unwrap_err(); - if let Error::InsufficientFunds(sources, amount) = err_result { - assert_eq!(sources, vec![FundingSource::FeePayer].into()); - assert_eq!(amount, fees_in_sol.to_string()); - } else { - panic!("check_payer_balances should have errored"); - } - } - - fn initialize_stake_account( - stake_account_amount: u64, - unlocked_sol: u64, - sender_keypair: &Keypair, - client: &RpcClient, - ) -> StakeArgs { - let stake_account_keypair = Keypair::new(); - let stake_account_address = stake_account_keypair.pubkey(); - let stake_authority = Keypair::new(); - let withdraw_authority = Keypair::new(); - - let authorized = Authorized { - staker: stake_authority.pubkey(), - withdrawer: withdraw_authority.pubkey(), - }; - let lockup = Lockup::default(); - let instructions = stake_instruction::create_account( - &sender_keypair.pubkey(), - &stake_account_address, - &authorized, - &lockup, - stake_account_amount, - ); - let message = Message::new(&instructions, Some(&sender_keypair.pubkey())); - let signers = [sender_keypair, &stake_account_keypair]; - let blockhash = client.get_latest_blockhash().unwrap(); - let transaction = Transaction::new(&signers, message, blockhash); - client - .send_and_confirm_transaction_with_spinner(&transaction) - .unwrap(); - - let sender_stake_args = SenderStakeArgs { - stake_account_address, - stake_authority: Box::new(stake_authority), - withdraw_authority: Box::new(withdraw_authority), - lockup_authority: None, - }; - - StakeArgs { - lockup_authority: None, - unlocked_sol, - sender_stake_args: Some(sender_stake_args), - } - } - - #[test] - fn test_check_payer_balances_distribute_stakes_single_payer() { - let alice = Keypair::new(); - let test_validator = TestValidator::with_custom_fees( - alice.pubkey(), - 10_000, - None, - SocketAddrSpace::Unspecified, - ); - let url = test_validator.rpc_url(); - let client = RpcClient::new_with_commitment(url, CommitmentConfig::processed()); - - let fees = client - .get_fee_for_message(&one_signer_message(&client)) - .unwrap(); - let fees_in_sol = lamports_to_sol(fees); - - let sender_keypair_file = tmp_file_path("keypair_file", &alice.pubkey()); - write_keypair_file(&alice, &sender_keypair_file).unwrap(); - - let allocation_amount = 1000.0; - let unlocked_sol = 1.0; - let stake_args = initialize_stake_account( - sol_to_lamports(allocation_amount), - sol_to_lamports(unlocked_sol), - &alice, - &client, - ); - - // Fully funded payer & stake account - let (allocations, mut args) = initialize_check_payer_balances_inputs( - sol_to_lamports(allocation_amount), - &sender_keypair_file, - &sender_keypair_file, - Some(stake_args), - ); - check_payer_balances(&[one_signer_message(&client)], &allocations, &client, &args).unwrap(); - - // Underfunded stake-account - let expensive_allocation_amount = 5000.0; - let expensive_allocations = vec![Allocation { - recipient: solana_sdk::pubkey::new_rand().to_string(), - amount: sol_to_lamports(expensive_allocation_amount), - lockup_date: "".to_string(), - }]; - let err_result = check_payer_balances( - &[one_signer_message(&client)], - &expensive_allocations, - &client, - &args, - ) - .unwrap_err(); - if let Error::InsufficientFunds(sources, amount) = err_result { - assert_eq!(sources, vec![FundingSource::StakeAccount].into()); - assert_eq!( - amount, - (expensive_allocation_amount - unlocked_sol).to_string() - ); - } else { - panic!("check_payer_balances should have errored"); - } - - // Unfunded payer - let unfunded_payer = Keypair::new(); - let unfunded_payer_keypair_file = tmp_file_path("keypair_file", &unfunded_payer.pubkey()); - write_keypair_file(&unfunded_payer, &unfunded_payer_keypair_file).unwrap(); - args.sender_keypair = read_keypair_file(&unfunded_payer_keypair_file) - .unwrap() - .into(); - args.fee_payer = read_keypair_file(&unfunded_payer_keypair_file) - .unwrap() - .into(); - - let err_result = - check_payer_balances(&[one_signer_message(&client)], &allocations, &client, &args) - .unwrap_err(); - if let Error::InsufficientFunds(sources, amount) = err_result { - assert_eq!( - sources, - vec![FundingSource::SystemAccount, FundingSource::FeePayer].into() - ); - assert_eq!(amount, (unlocked_sol + fees_in_sol).to_string()); - } else { - panic!("check_payer_balances should have errored"); - } - - // Payer funded enough for distribution only - let partially_funded_payer = Keypair::new(); - let partially_funded_payer_keypair_file = - tmp_file_path("keypair_file", &partially_funded_payer.pubkey()); - write_keypair_file( - &partially_funded_payer, - &partially_funded_payer_keypair_file, - ) - .unwrap(); - let transaction = transfer( - &client, - sol_to_lamports(unlocked_sol), - &alice, - &partially_funded_payer.pubkey(), - ) - .unwrap(); - client - .send_and_confirm_transaction_with_spinner(&transaction) - .unwrap(); - - args.sender_keypair = read_keypair_file(&partially_funded_payer_keypair_file) - .unwrap() - .into(); - args.fee_payer = read_keypair_file(&partially_funded_payer_keypair_file) - .unwrap() - .into(); - let err_result = - check_payer_balances(&[one_signer_message(&client)], &allocations, &client, &args) - .unwrap_err(); - if let Error::InsufficientFunds(sources, amount) = err_result { - assert_eq!( - sources, - vec![FundingSource::SystemAccount, FundingSource::FeePayer].into() - ); - assert_eq!(amount, (unlocked_sol + fees_in_sol).to_string()); - } else { - panic!("check_payer_balances should have errored"); - } - } - - #[test] - fn test_check_payer_balances_distribute_stakes_separate_payers() { - let alice = Keypair::new(); - let test_validator = TestValidator::with_custom_fees( - alice.pubkey(), - 10_000, - None, - SocketAddrSpace::Unspecified, - ); - let url = test_validator.rpc_url(); - - let client = RpcClient::new_with_commitment(url, CommitmentConfig::processed()); - - let fees = client - .get_fee_for_message(&one_signer_message(&client)) - .unwrap(); - let fees_in_sol = lamports_to_sol(fees); - - let sender_keypair_file = tmp_file_path("keypair_file", &alice.pubkey()); - write_keypair_file(&alice, &sender_keypair_file).unwrap(); - - let allocation_amount = 1000.0; - let unlocked_sol = 1.0; - let stake_args = initialize_stake_account( - sol_to_lamports(allocation_amount), - sol_to_lamports(unlocked_sol), - &alice, - &client, - ); - - let funded_payer = Keypair::new(); - let funded_payer_keypair_file = tmp_file_path("keypair_file", &funded_payer.pubkey()); - write_keypair_file(&funded_payer, &funded_payer_keypair_file).unwrap(); - let transaction = transfer( - &client, - sol_to_lamports(unlocked_sol), - &alice, - &funded_payer.pubkey(), - ) - .unwrap(); - client - .send_and_confirm_transaction_with_spinner(&transaction) - .unwrap(); - - // Fully funded payers - let (allocations, mut args) = initialize_check_payer_balances_inputs( - sol_to_lamports(allocation_amount), - &funded_payer_keypair_file, - &sender_keypair_file, - Some(stake_args), - ); - check_payer_balances(&[one_signer_message(&client)], &allocations, &client, &args).unwrap(); - - // Unfunded sender - let unfunded_payer = Keypair::new(); - let unfunded_payer_keypair_file = tmp_file_path("keypair_file", &unfunded_payer.pubkey()); - write_keypair_file(&unfunded_payer, &unfunded_payer_keypair_file).unwrap(); - args.sender_keypair = read_keypair_file(&unfunded_payer_keypair_file) - .unwrap() - .into(); - args.fee_payer = read_keypair_file(&sender_keypair_file).unwrap().into(); - - let err_result = - check_payer_balances(&[one_signer_message(&client)], &allocations, &client, &args) - .unwrap_err(); - if let Error::InsufficientFunds(sources, amount) = err_result { - assert_eq!(sources, vec![FundingSource::SystemAccount].into()); - assert_eq!(amount, unlocked_sol.to_string()); - } else { - panic!("check_payer_balances should have errored"); - } - - // Unfunded fee payer - args.sender_keypair = read_keypair_file(&sender_keypair_file).unwrap().into(); - args.fee_payer = read_keypair_file(&unfunded_payer_keypair_file) - .unwrap() - .into(); - - let err_result = - check_payer_balances(&[one_signer_message(&client)], &allocations, &client, &args) - .unwrap_err(); - if let Error::InsufficientFunds(sources, amount) = err_result { - assert_eq!(sources, vec![FundingSource::FeePayer].into()); - assert_eq!(amount, fees_in_sol.to_string()); - } else { - panic!("check_payer_balances should have errored"); - } - } - - #[test] - fn test_build_messages_dump_db() { - let client = RpcClient::new_mock("mock_client".to_string()); - let dir = tempdir().unwrap(); - let db_file = dir - .path() - .join("build_messages.db") - .to_str() - .unwrap() - .to_string(); - let mut db = db::open_db(&db_file, false).unwrap(); - - let sender = Keypair::new(); - let recipient = Pubkey::new_unique(); - let amount = sol_to_lamports(1.0); - let last_valid_block_height = 222; - let transaction = transfer(&client, amount, &sender, &recipient).unwrap(); - - // Queue db data - db::set_transaction_info( - &mut db, - &recipient, - amount, - &transaction, - None, - false, - last_valid_block_height, - None, - ) - .unwrap(); - - // Check that data has not been dumped - let read_db = db::open_db(&db_file, true).unwrap(); - assert!(db::read_transaction_infos(&read_db).is_empty()); - - // This is just dummy data; Args will not affect messages built - let args = DistributeTokensArgs { - sender_keypair: Box::new(Keypair::new()), - fee_payer: Box::new(Keypair::new()), - dry_run: true, - input_csv: "".to_string(), - transaction_db: "".to_string(), - output_path: None, - stake_args: None, - spl_token_args: None, - transfer_amount: None, - }; - let allocation = Allocation { - recipient: recipient.to_string(), - amount: sol_to_lamports(1.0), - lockup_date: "".to_string(), - }; - - let mut messages: Vec = vec![]; - let mut stake_extras: StakeExtras = vec![]; - let mut created_accounts = 0; - - // Exit false will not dump data - build_messages( - &client, - &mut db, - &[allocation.clone()], - &args, - Arc::new(AtomicBool::new(false)), - &mut messages, - &mut stake_extras, - &mut created_accounts, - ) - .unwrap(); - let read_db = db::open_db(&db_file, true).unwrap(); - assert!(db::read_transaction_infos(&read_db).is_empty()); - assert_eq!(messages.len(), 1); - - // Empty allocations will not dump data - let mut messages: Vec = vec![]; - let exit = Arc::new(AtomicBool::new(true)); - build_messages( - &client, - &mut db, - &[], - &args, - exit.clone(), - &mut messages, - &mut stake_extras, - &mut created_accounts, - ) - .unwrap(); - let read_db = db::open_db(&db_file, true).unwrap(); - assert!(db::read_transaction_infos(&read_db).is_empty()); - assert!(messages.is_empty()); - - // Any allocation should prompt data dump - let mut messages: Vec = vec![]; - build_messages( - &client, - &mut db, - &[allocation], - &args, - exit, - &mut messages, - &mut stake_extras, - &mut created_accounts, - ) - .unwrap_err(); - let read_db = db::open_db(&db_file, true).unwrap(); - let transaction_info = db::read_transaction_infos(&read_db); - assert_eq!(transaction_info.len(), 1); - assert_eq!( - transaction_info[0], - TransactionInfo { - recipient, - amount, - new_stake_account_address: None, - finalized_date: None, - transaction, - last_valid_block_height, - lockup_date: None, - } - ); - assert_eq!(messages.len(), 0); - } - - #[test] - fn test_send_messages_dump_db() { - let client = RpcClient::new_mock("mock_client".to_string()); - let dir = tempdir().unwrap(); - let db_file = dir - .path() - .join("send_messages.db") - .to_str() - .unwrap() - .to_string(); - let mut db = db::open_db(&db_file, false).unwrap(); - - let sender = Keypair::new(); - let recipient = Pubkey::new_unique(); - let amount = sol_to_lamports(1.0); - let last_valid_block_height = 222; - let transaction = transfer(&client, amount, &sender, &recipient).unwrap(); - - // Queue db data - db::set_transaction_info( - &mut db, - &recipient, - amount, - &transaction, - None, - false, - last_valid_block_height, - None, - ) - .unwrap(); - - // Check that data has not been dumped - let read_db = db::open_db(&db_file, true).unwrap(); - assert!(db::read_transaction_infos(&read_db).is_empty()); - - // This is just dummy data; Args will not affect messages - let args = DistributeTokensArgs { - sender_keypair: Box::new(Keypair::new()), - fee_payer: Box::new(Keypair::new()), - dry_run: true, - input_csv: "".to_string(), - transaction_db: "".to_string(), - output_path: None, - stake_args: None, - spl_token_args: None, - transfer_amount: None, - }; - let allocation = Allocation { - recipient: recipient.to_string(), - amount: sol_to_lamports(1.0), - lockup_date: "".to_string(), - }; - let message = transaction.message.clone(); - - // Exit false will not dump data - send_messages( - &client, - &mut db, - &[allocation.clone()], - &args, - Arc::new(AtomicBool::new(false)), - vec![message.clone()], - vec![(Keypair::new(), None)], - ) - .unwrap(); - let read_db = db::open_db(&db_file, true).unwrap(); - assert!(db::read_transaction_infos(&read_db).is_empty()); - // The method above will, however, write a record to the in-memory db - // Grab that expected value to test successful dump - let num_records = db::read_transaction_infos(&db).len(); - - // Empty messages/allocations will not dump data - let exit = Arc::new(AtomicBool::new(true)); - send_messages(&client, &mut db, &[], &args, exit.clone(), vec![], vec![]).unwrap(); - let read_db = db::open_db(&db_file, true).unwrap(); - assert!(db::read_transaction_infos(&read_db).is_empty()); - - // Message/allocation should prompt data dump at start of loop - send_messages( - &client, - &mut db, - &[allocation], - &args, - exit, - vec![message.clone()], - vec![(Keypair::new(), None)], - ) - .unwrap_err(); - let read_db = db::open_db(&db_file, true).unwrap(); - let transaction_info = db::read_transaction_infos(&read_db); - assert_eq!(transaction_info.len(), num_records); - assert!(transaction_info.contains(&TransactionInfo { - recipient, - amount, - new_stake_account_address: None, - finalized_date: None, - transaction, - last_valid_block_height, - lockup_date: None, - })); - assert!(transaction_info.contains(&TransactionInfo { - recipient, - amount, - new_stake_account_address: None, - finalized_date: None, - transaction: Transaction::new_unsigned(message), - last_valid_block_height: std::u64::MAX, - lockup_date: None, - })); - - // Next dump should write record written in last send_messages call - let num_records = db::read_transaction_infos(&db).len(); - db.dump().unwrap(); - let read_db = db::open_db(&db_file, true).unwrap(); - let transaction_info = db::read_transaction_infos(&read_db); - assert_eq!(transaction_info.len(), num_records); - } - - #[test] - fn test_distribute_allocations_dump_db() { - let sender_keypair = Keypair::new(); - let test_validator = TestValidator::with_no_fees( - sender_keypair.pubkey(), - None, - SocketAddrSpace::Unspecified, - ); - let url = test_validator.rpc_url(); - let client = RpcClient::new_with_commitment(url, CommitmentConfig::processed()); - - let fee_payer = Keypair::new(); - let transaction = transfer( - &client, - sol_to_lamports(1.0), - &sender_keypair, - &fee_payer.pubkey(), - ) - .unwrap(); - client - .send_and_confirm_transaction_with_spinner(&transaction) - .unwrap(); - - let dir = tempdir().unwrap(); - let db_file = dir - .path() - .join("dist_allocations.db") - .to_str() - .unwrap() - .to_string(); - let mut db = db::open_db(&db_file, false).unwrap(); - let recipient = Pubkey::new_unique(); - let allocation = Allocation { - recipient: recipient.to_string(), - amount: sol_to_lamports(1.0), - lockup_date: "".to_string(), - }; - // This is just dummy data; Args will not affect messages - let args = DistributeTokensArgs { - sender_keypair: Box::new(sender_keypair), - fee_payer: Box::new(fee_payer), - dry_run: true, - input_csv: "".to_string(), - transaction_db: "".to_string(), - output_path: None, - stake_args: None, - spl_token_args: None, - transfer_amount: None, - }; - - let exit = Arc::new(AtomicBool::new(false)); - - // Ensure data is always dumped after distribute_allocations - distribute_allocations(&client, &mut db, &[allocation], &args, exit).unwrap(); - let read_db = db::open_db(&db_file, true).unwrap(); - let transaction_info = db::read_transaction_infos(&read_db); - assert_eq!(transaction_info.len(), 1); - } - - #[test] - fn test_log_transaction_confirmations_dump_db() { - let client = RpcClient::new_mock("mock_client".to_string()); - let dir = tempdir().unwrap(); - let db_file = dir - .path() - .join("log_transaction_confirmations.db") - .to_str() - .unwrap() - .to_string(); - let mut db = db::open_db(&db_file, false).unwrap(); - - let sender = Keypair::new(); - let recipient = Pubkey::new_unique(); - let amount = sol_to_lamports(1.0); - let last_valid_block_height = 222; - let transaction = transfer(&client, amount, &sender, &recipient).unwrap(); - - // Queue unconfirmed transaction into db - db::set_transaction_info( - &mut db, - &recipient, - amount, - &transaction, - None, - false, - last_valid_block_height, - None, - ) - .unwrap(); - - // Check that data has not been dumped - let read_db = db::open_db(&db_file, true).unwrap(); - assert!(db::read_transaction_infos(&read_db).is_empty()); - - // Empty unconfirmed_transactions will not dump data - let mut confirmations = None; - let exit = Arc::new(AtomicBool::new(true)); - log_transaction_confirmations( - &client, - &mut db, - exit.clone(), - vec![], - vec![], - &mut confirmations, - ) - .unwrap(); - let read_db = db::open_db(&db_file, true).unwrap(); - assert!(db::read_transaction_infos(&read_db).is_empty()); - assert_eq!(confirmations, None); - - // Exit false will not dump data - log_transaction_confirmations( - &client, - &mut db, - Arc::new(AtomicBool::new(false)), - vec![(&transaction, 111)], - vec![Some(TransactionStatus { - slot: 40, - confirmations: Some(15), - status: Ok(()), - err: None, - confirmation_status: Some(TransactionConfirmationStatus::Finalized), - })], - &mut confirmations, - ) - .unwrap(); - let read_db = db::open_db(&db_file, true).unwrap(); - assert!(db::read_transaction_infos(&read_db).is_empty()); - assert_eq!(confirmations, Some(15)); - - // Exit true should dump data - log_transaction_confirmations( - &client, - &mut db, - exit, - vec![(&transaction, 111)], - vec![Some(TransactionStatus { - slot: 55, - confirmations: None, - status: Ok(()), - err: None, - confirmation_status: Some(TransactionConfirmationStatus::Finalized), - })], - &mut confirmations, - ) - .unwrap_err(); - let read_db = db::open_db(&db_file, true).unwrap(); - let transaction_info = db::read_transaction_infos(&read_db); - assert_eq!(transaction_info.len(), 1); - assert!(transaction_info[0].finalized_date.is_some()); - } - - #[test] - fn test_update_finalized_transactions_dump_db() { - let client = RpcClient::new_mock("mock_client".to_string()); - let dir = tempdir().unwrap(); - let db_file = dir - .path() - .join("update_finalized_transactions.db") - .to_str() - .unwrap() - .to_string(); - let mut db = db::open_db(&db_file, false).unwrap(); - - let sender = Keypair::new(); - let recipient = Pubkey::new_unique(); - let amount = sol_to_lamports(1.0); - let last_valid_block_height = 222; - let transaction = transfer(&client, amount, &sender, &recipient).unwrap(); - - // Queue unconfirmed transaction into db - db::set_transaction_info( - &mut db, - &recipient, - amount, - &transaction, - None, - false, - last_valid_block_height, - None, - ) - .unwrap(); - - // Ensure data is always dumped after update_finalized_transactions - let confs = - update_finalized_transactions(&client, &mut db, Arc::new(AtomicBool::new(false))) - .unwrap(); - let read_db = db::open_db(&db_file, true).unwrap(); - let transaction_info = db::read_transaction_infos(&read_db); - assert_eq!(transaction_info.len(), 1); - assert_eq!(confs, None); - } -} diff --git a/tokens/src/db.rs b/tokens/src/db.rs deleted file mode 100644 index 881c476d13..0000000000 --- a/tokens/src/db.rs +++ /dev/null @@ -1,380 +0,0 @@ -use { - chrono::prelude::*, - pickledb::{error::Error, PickleDb, PickleDbDumpPolicy}, - serde::{Deserialize, Serialize}, - solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature, transaction::Transaction}, - solana_transaction_status::TransactionStatus, - std::{cmp::Ordering, fs, io, path::Path}, -}; - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct TransactionInfo { - pub recipient: Pubkey, - pub amount: u64, - pub new_stake_account_address: Option, - pub finalized_date: Option>, - pub transaction: Transaction, - pub last_valid_block_height: Slot, - pub lockup_date: Option>, -} - -#[derive(Serialize, Deserialize, Debug, Default, PartialEq)] -struct SignedTransactionInfo { - recipient: String, - amount: u64, - #[serde(skip_serializing_if = "String::is_empty", default)] - new_stake_account_address: String, - finalized_date: Option>, - signature: String, -} - -impl Default for TransactionInfo { - fn default() -> Self { - let transaction = Transaction { - signatures: vec![Signature::default()], - ..Transaction::default() - }; - Self { - recipient: Pubkey::default(), - amount: 0, - new_stake_account_address: None, - finalized_date: None, - transaction, - last_valid_block_height: 0, - lockup_date: None, - } - } -} - -pub fn open_db(path: &str, dry_run: bool) -> Result { - let policy = if dry_run { - PickleDbDumpPolicy::NeverDump - } else { - PickleDbDumpPolicy::DumpUponRequest - }; - let path = Path::new(path); - let db = if path.exists() { - PickleDb::load_yaml(path, policy)? - } else { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent).unwrap(); - } - PickleDb::new_yaml(path, policy) - }; - Ok(db) -} - -pub fn compare_transaction_infos(a: &TransactionInfo, b: &TransactionInfo) -> Ordering { - let ordering = match (a.finalized_date, b.finalized_date) { - (Some(a), Some(b)) => a.cmp(&b), - (None, Some(_)) => Ordering::Greater, - (Some(_), None) => Ordering::Less, // Future finalized date will be greater - _ => Ordering::Equal, - }; - if ordering == Ordering::Equal { - return a.recipient.to_string().cmp(&b.recipient.to_string()); - } - ordering -} - -pub fn write_transaction_log>(db: &PickleDb, path: &P) -> Result<(), io::Error> { - let mut wtr = csv::WriterBuilder::new().from_path(path).unwrap(); - let mut transaction_infos = read_transaction_infos(db); - transaction_infos.sort_by(compare_transaction_infos); - for info in transaction_infos { - let signed_info = SignedTransactionInfo { - recipient: info.recipient.to_string(), - amount: info.amount, - new_stake_account_address: info - .new_stake_account_address - .map(|x| x.to_string()) - .unwrap_or_else(|| "".to_string()), - finalized_date: info.finalized_date, - signature: info.transaction.signatures[0].to_string(), - }; - wtr.serialize(&signed_info)?; - } - wtr.flush() -} - -pub fn read_transaction_infos(db: &PickleDb) -> Vec { - db.iter() - .map(|kv| kv.get_value::().unwrap()) - .collect() -} - -pub fn set_transaction_info( - db: &mut PickleDb, - recipient: &Pubkey, - amount: u64, - transaction: &Transaction, - new_stake_account_address: Option<&Pubkey>, - finalized: bool, - last_valid_block_height: u64, - lockup_date: Option>, -) -> Result<(), Error> { - let finalized_date = if finalized { Some(Utc::now()) } else { None }; - let transaction_info = TransactionInfo { - recipient: *recipient, - amount, - new_stake_account_address: new_stake_account_address.cloned(), - finalized_date, - transaction: transaction.clone(), - last_valid_block_height, - lockup_date, - }; - let signature = transaction.signatures[0]; - db.set(&signature.to_string(), &transaction_info)?; - Ok(()) -} - -// Set the finalized bit in the database if the transaction is rooted. -// Remove the TransactionInfo from the database if the transaction failed. -// Return the number of confirmations on the transaction or None if either -// finalized or discarded. -pub fn update_finalized_transaction( - db: &mut PickleDb, - signature: &Signature, - opt_transaction_status: Option, - last_valid_block_height: u64, - finalized_block_height: u64, -) -> Result, Error> { - if opt_transaction_status.is_none() { - if finalized_block_height > last_valid_block_height { - eprintln!( - "Signature not found {} and blockhash expired. Transaction either dropped or the validator purged the transaction status.", - signature - ); - eprintln!(); - - // Don't discard the transaction, because we are not certain the - // blockhash is expired. Instead, return None to signal that - // we don't need to wait for confirmations. - return Ok(None); - } - - // Return zero to signal the transaction may still be in flight. - return Ok(Some(0)); - } - let transaction_status = opt_transaction_status.unwrap(); - - if let Some(confirmations) = transaction_status.confirmations { - // The transaction was found but is not yet finalized. - return Ok(Some(confirmations)); - } - - if let Some(e) = &transaction_status.err { - // The transaction was finalized, but execution failed. Drop it. - eprintln!("Error in transaction with signature {}: {}", signature, e); - eprintln!("Discarding transaction record"); - eprintln!(); - db.rem(&signature.to_string())?; - return Ok(None); - } - - // Transaction is rooted. Set the finalized date in the database. - let mut transaction_info = db.get::(&signature.to_string()).unwrap(); - transaction_info.finalized_date = Some(Utc::now()); - db.set(&signature.to_string(), &transaction_info)?; - Ok(None) -} - -use csv::{ReaderBuilder, Trim}; -pub(crate) fn check_output_file(path: &str, db: &PickleDb) { - let mut rdr = ReaderBuilder::new() - .trim(Trim::All) - .from_path(path) - .unwrap(); - let logged_infos: Vec = - rdr.deserialize().map(|entry| entry.unwrap()).collect(); - - let mut transaction_infos = read_transaction_infos(db); - transaction_infos.sort_by(compare_transaction_infos); - let transaction_infos: Vec = transaction_infos - .iter() - .map(|info| SignedTransactionInfo { - recipient: info.recipient.to_string(), - amount: info.amount, - new_stake_account_address: info - .new_stake_account_address - .map(|x| x.to_string()) - .unwrap_or_else(|| "".to_string()), - finalized_date: info.finalized_date, - signature: info.transaction.signatures[0].to_string(), - }) - .collect(); - assert_eq!(logged_infos, transaction_infos); -} - -#[cfg(test)] -mod tests { - use { - super::*, - csv::{ReaderBuilder, Trim}, - solana_sdk::transaction::TransactionError, - solana_transaction_status::TransactionConfirmationStatus, - tempfile::NamedTempFile, - }; - - #[test] - fn test_sort_transaction_infos_finalized_first() { - let info0 = TransactionInfo { - finalized_date: Some(Utc.ymd(2014, 7, 8).and_hms(9, 10, 11)), - ..TransactionInfo::default() - }; - let info1 = TransactionInfo { - finalized_date: Some(Utc.ymd(2014, 7, 8).and_hms(9, 10, 42)), - ..TransactionInfo::default() - }; - let info2 = TransactionInfo::default(); - let info3 = TransactionInfo { - recipient: solana_sdk::pubkey::new_rand(), - ..TransactionInfo::default() - }; - - // Sorted first by date - assert_eq!(compare_transaction_infos(&info0, &info1), Ordering::Less); - - // Finalized transactions should be before unfinalized ones - assert_eq!(compare_transaction_infos(&info1, &info2), Ordering::Less); - - // Then sorted by recipient - assert_eq!(compare_transaction_infos(&info2, &info3), Ordering::Less); - } - - #[test] - fn test_write_transaction_log() { - let mut db = - PickleDb::new_yaml(NamedTempFile::new().unwrap(), PickleDbDumpPolicy::NeverDump); - let signature = Signature::default(); - let transaction_info = TransactionInfo::default(); - db.set(&signature.to_string(), &transaction_info).unwrap(); - - let csv_file = NamedTempFile::new().unwrap(); - write_transaction_log(&db, &csv_file).unwrap(); - - let mut rdr = ReaderBuilder::new().trim(Trim::All).from_reader(csv_file); - let signed_infos: Vec = - rdr.deserialize().map(|entry| entry.unwrap()).collect(); - - let signed_info = SignedTransactionInfo { - recipient: Pubkey::default().to_string(), - signature: Signature::default().to_string(), - ..SignedTransactionInfo::default() - }; - assert_eq!(signed_infos, vec![signed_info]); - } - - #[test] - fn test_update_finalized_transaction_not_landed() { - // Keep waiting for a transaction that hasn't landed yet. - let mut db = - PickleDb::new_yaml(NamedTempFile::new().unwrap(), PickleDbDumpPolicy::NeverDump); - let signature = Signature::default(); - let transaction_info = TransactionInfo::default(); - db.set(&signature.to_string(), &transaction_info).unwrap(); - assert!(matches!( - update_finalized_transaction(&mut db, &signature, None, 0, 0).unwrap(), - Some(0) - )); - - // Unchanged - assert_eq!( - db.get::(&signature.to_string()).unwrap(), - transaction_info - ); - - // Same as before, but now with an expired blockhash - assert_eq!( - update_finalized_transaction(&mut db, &signature, None, 0, 1).unwrap(), - None - ); - - // Ensure TransactionInfo has not been purged. - assert_eq!( - db.get::(&signature.to_string()).unwrap(), - transaction_info - ); - } - - #[test] - fn test_update_finalized_transaction_confirming() { - // Keep waiting for a transaction that is still being confirmed. - let mut db = - PickleDb::new_yaml(NamedTempFile::new().unwrap(), PickleDbDumpPolicy::NeverDump); - let signature = Signature::default(); - let transaction_info = TransactionInfo::default(); - db.set(&signature.to_string(), &transaction_info).unwrap(); - let transaction_status = TransactionStatus { - slot: 0, - confirmations: Some(1), - err: None, - status: Ok(()), - confirmation_status: Some(TransactionConfirmationStatus::Confirmed), - }; - assert_eq!( - update_finalized_transaction(&mut db, &signature, Some(transaction_status), 0, 0) - .unwrap(), - Some(1) - ); - - // Unchanged - assert_eq!( - db.get::(&signature.to_string()).unwrap(), - transaction_info - ); - } - - #[test] - fn test_update_finalized_transaction_failed() { - // Don't wait if the transaction failed to execute. - let mut db = - PickleDb::new_yaml(NamedTempFile::new().unwrap(), PickleDbDumpPolicy::NeverDump); - let signature = Signature::default(); - let transaction_info = TransactionInfo::default(); - db.set(&signature.to_string(), &transaction_info).unwrap(); - let transaction_status = TransactionStatus { - slot: 0, - confirmations: None, - err: Some(TransactionError::AccountNotFound), - status: Ok(()), - confirmation_status: Some(TransactionConfirmationStatus::Finalized), - }; - assert_eq!( - update_finalized_transaction(&mut db, &signature, Some(transaction_status), 0, 0) - .unwrap(), - None - ); - - // Ensure TransactionInfo has been purged. - assert_eq!(db.get::(&signature.to_string()), None); - } - - #[test] - fn test_update_finalized_transaction_finalized() { - // Don't wait once the transaction has been finalized. - let mut db = - PickleDb::new_yaml(NamedTempFile::new().unwrap(), PickleDbDumpPolicy::NeverDump); - let signature = Signature::default(); - let transaction_info = TransactionInfo::default(); - db.set(&signature.to_string(), &transaction_info).unwrap(); - let transaction_status = TransactionStatus { - slot: 0, - confirmations: None, - err: None, - status: Ok(()), - confirmation_status: Some(TransactionConfirmationStatus::Finalized), - }; - assert_eq!( - update_finalized_transaction(&mut db, &signature, Some(transaction_status), 0, 0) - .unwrap(), - None - ); - - assert!(db - .get::(&signature.to_string()) - .unwrap() - .finalized_date - .is_some()); - } -} diff --git a/tokens/src/lib.rs b/tokens/src/lib.rs deleted file mode 100644 index 8df0d4f482..0000000000 --- a/tokens/src/lib.rs +++ /dev/null @@ -1,7 +0,0 @@ -#![allow(clippy::integer_arithmetic)] -pub mod arg_parser; -pub mod args; -pub mod commands; -mod db; -pub mod spl_token; -pub mod token_display; diff --git a/tokens/src/main.rs b/tokens/src/main.rs deleted file mode 100644 index bbd8c0e435..0000000000 --- a/tokens/src/main.rs +++ /dev/null @@ -1,54 +0,0 @@ -use { - solana_cli_config::{Config, CONFIG_FILE}, - solana_client::rpc_client::RpcClient, - solana_tokens::{arg_parser::parse_args, args::Command, commands, spl_token}, - std::{ - env, - error::Error, - path::Path, - process, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, - }, -}; - -fn main() -> Result<(), Box> { - let command_args = parse_args(env::args_os())?; - let config = if Path::new(&command_args.config_file).exists() { - Config::load(&command_args.config_file)? - } else { - let default_config_file = CONFIG_FILE.as_ref().unwrap(); - if command_args.config_file != *default_config_file { - eprintln!("Error: config file not found"); - process::exit(1); - } - Config::default() - }; - let json_rpc_url = command_args.url.unwrap_or(config.json_rpc_url); - let client = RpcClient::new(json_rpc_url); - - let exit = Arc::new(AtomicBool::default()); - let _exit = exit.clone(); - // Initialize CTRL-C handler to ensure db changes are written before exit. - ctrlc::set_handler(move || { - _exit.store(true, Ordering::SeqCst); - }) - .expect("Error setting Ctrl-C handler"); - - match command_args.command { - Command::DistributeTokens(mut args) => { - spl_token::update_token_args(&client, &mut args.spl_token_args)?; - commands::process_allocations(&client, &args, exit)?; - } - Command::Balances(mut args) => { - spl_token::update_decimals(&client, &mut args.spl_token_args)?; - commands::process_balances(&client, &args)?; - } - Command::TransactionLog(args) => { - commands::process_transaction_log(&args)?; - } - } - Ok(()) -} diff --git a/tokens/src/spl_token.rs b/tokens/src/spl_token.rs deleted file mode 100644 index 8d6f4be0a6..0000000000 --- a/tokens/src/spl_token.rs +++ /dev/null @@ -1,177 +0,0 @@ -use { - crate::{ - args::{DistributeTokensArgs, SplTokenArgs}, - commands::{get_fees_for_messages, Allocation, Error, FundingSource}, - }, - console::style, - solana_account_decoder::parse_token::{ - pubkey_from_spl_token, real_number_string, real_number_string_trimmed, spl_token_pubkey, - }, - solana_client::rpc_client::RpcClient, - solana_sdk::{instruction::Instruction, message::Message, native_token::lamports_to_sol}, - solana_transaction_status::parse_token::spl_token_instruction, - spl_associated_token_account::{ - get_associated_token_address, instruction::create_associated_token_account, - }, - spl_token::{ - solana_program::program_pack::Pack, - state::{Account as SplTokenAccount, Mint}, - }, -}; - -pub fn update_token_args(client: &RpcClient, args: &mut Option) -> Result<(), Error> { - if let Some(spl_token_args) = args { - let sender_account = client - .get_account(&spl_token_args.token_account_address) - .unwrap_or_default(); - let mint_address = - pubkey_from_spl_token(&SplTokenAccount::unpack(&sender_account.data)?.mint); - spl_token_args.mint = mint_address; - update_decimals(client, args)?; - } - Ok(()) -} - -pub fn update_decimals(client: &RpcClient, args: &mut Option) -> Result<(), Error> { - if let Some(spl_token_args) = args { - let mint_account = client.get_account(&spl_token_args.mint).unwrap_or_default(); - let mint = Mint::unpack(&mint_account.data)?; - spl_token_args.decimals = mint.decimals; - } - Ok(()) -} - -pub fn spl_token_amount(amount: f64, decimals: u8) -> u64 { - (amount * 10_usize.pow(decimals as u32) as f64) as u64 -} - -pub fn build_spl_token_instructions( - allocation: &Allocation, - args: &DistributeTokensArgs, - do_create_associated_token_account: bool, -) -> Vec { - let spl_token_args = args - .spl_token_args - .as_ref() - .expect("spl_token_args must be some"); - let wallet_address = allocation.recipient.parse().unwrap(); - let associated_token_address = - get_associated_token_address(&wallet_address, &spl_token_pubkey(&spl_token_args.mint)); - let mut instructions = vec![]; - if do_create_associated_token_account { - let create_associated_token_account_instruction = create_associated_token_account( - &spl_token_pubkey(&args.fee_payer.pubkey()), - &wallet_address, - &spl_token_pubkey(&spl_token_args.mint), - &spl_token::id(), - ); - instructions.push(spl_token_instruction( - create_associated_token_account_instruction, - )); - } - let spl_instruction = spl_token::instruction::transfer_checked( - &spl_token::id(), - &spl_token_pubkey(&spl_token_args.token_account_address), - &spl_token_pubkey(&spl_token_args.mint), - &associated_token_address, - &spl_token_pubkey(&args.sender_keypair.pubkey()), - &[], - allocation.amount, - spl_token_args.decimals, - ) - .unwrap(); - instructions.push(spl_token_instruction(spl_instruction)); - instructions -} - -pub fn check_spl_token_balances( - messages: &[Message], - allocations: &[Allocation], - client: &RpcClient, - args: &DistributeTokensArgs, - created_accounts: u64, -) -> Result<(), Error> { - let spl_token_args = args - .spl_token_args - .as_ref() - .expect("spl_token_args must be some"); - let allocation_amount: u64 = allocations.iter().map(|x| x.amount).sum(); - let fees = get_fees_for_messages(messages, client)?; - - let token_account_rent_exempt_balance = - client.get_minimum_balance_for_rent_exemption(SplTokenAccount::LEN)?; - let account_creation_amount = created_accounts * token_account_rent_exempt_balance; - let fee_payer_balance = client.get_balance(&args.fee_payer.pubkey())?; - if fee_payer_balance < fees + account_creation_amount { - return Err(Error::InsufficientFunds( - vec![FundingSource::FeePayer].into(), - lamports_to_sol(fees + account_creation_amount).to_string(), - )); - } - let source_token_account = client - .get_account(&spl_token_args.token_account_address) - .unwrap_or_default(); - let source_token = SplTokenAccount::unpack(&source_token_account.data)?; - if source_token.amount < allocation_amount { - return Err(Error::InsufficientFunds( - vec![FundingSource::SplTokenAccount].into(), - real_number_string_trimmed(allocation_amount, spl_token_args.decimals), - )); - } - Ok(()) -} - -pub fn print_token_balances( - client: &RpcClient, - allocation: &Allocation, - spl_token_args: &SplTokenArgs, -) -> Result<(), Error> { - let address = allocation.recipient.parse().unwrap(); - let expected = allocation.amount; - let associated_token_address = get_associated_token_address( - &spl_token_pubkey(&address), - &spl_token_pubkey(&spl_token_args.mint), - ); - let recipient_account = client - .get_account(&pubkey_from_spl_token(&associated_token_address)) - .unwrap_or_default(); - let (actual, difference) = if let Ok(recipient_token) = - SplTokenAccount::unpack(&recipient_account.data) - { - let actual_ui_amount = real_number_string(recipient_token.amount, spl_token_args.decimals); - let delta_string = - real_number_string(recipient_token.amount - expected, spl_token_args.decimals); - ( - style(format!("{:>24}", actual_ui_amount)), - format!("{:>24}", delta_string), - ) - } else { - ( - style("Associated token account not yet created".to_string()).yellow(), - "".to_string(), - ) - }; - println!( - "{:<44} {:>24} {:>24} {:>24}", - allocation.recipient, - real_number_string(expected, spl_token_args.decimals), - actual, - difference, - ); - Ok(()) -} - -#[cfg(test)] -mod tests { - // The following unit tests were written for v1.4 using the ProgramTest framework, passing its - // BanksClient into the `solana-tokens` methods. With the revert to RpcClient in this module - // (https://github.com/solana-labs/solana/pull/13623), that approach was no longer viable. - // These tests were removed rather than rewritten to avoid accruing technical debt. Once a new - // rpc/client framework is implemented, they should be restored. - // - // async fn test_process_spl_token_allocations() - // async fn test_process_spl_token_transfer_amount_allocations() - // async fn test_check_spl_token_balances() - // - // https://github.com/solana-labs/solana/blob/5511d52c6284013a24ced10966d11d8f4585799e/tokens/src/spl_token.rs#L490-L685 -} diff --git a/tokens/src/token_display.rs b/tokens/src/token_display.rs deleted file mode 100644 index 32828f0be1..0000000000 --- a/tokens/src/token_display.rs +++ /dev/null @@ -1,81 +0,0 @@ -use { - solana_account_decoder::parse_token::real_number_string_trimmed, - solana_sdk::native_token::lamports_to_sol, - std::{ - fmt::{Debug, Display, Formatter, Result}, - ops::Add, - }, -}; - -const VLX_SYMBOL: &str = "◎"; - -#[derive(PartialEq)] -pub enum TokenType { - Sol, - SplToken, -} - -pub struct Token { - amount: u64, - decimals: u8, - token_type: TokenType, -} - -impl Token { - fn write_with_symbol(&self, f: &mut Formatter) -> Result { - match &self.token_type { - TokenType::Sol => { - let amount = lamports_to_sol(self.amount); - write!(f, "{}{}", VLX_SYMBOL, amount) - } - TokenType::SplToken => { - let amount = real_number_string_trimmed(self.amount, self.decimals); - write!(f, "{} tokens", amount) - } - } - } - - pub fn sol(amount: u64) -> Self { - Self { - amount, - decimals: 9, - token_type: TokenType::Sol, - } - } - - pub fn spl_token(amount: u64, decimals: u8) -> Self { - Self { - amount, - decimals, - token_type: TokenType::SplToken, - } - } -} - -impl Display for Token { - fn fmt(&self, f: &mut Formatter) -> Result { - self.write_with_symbol(f) - } -} - -impl Debug for Token { - fn fmt(&self, f: &mut Formatter) -> Result { - self.write_with_symbol(f) - } -} - -impl Add for Token { - type Output = Token; - - fn add(self, other: Self) -> Self { - if self.token_type == other.token_type { - Self { - amount: self.amount + other.amount, - decimals: self.decimals, - token_type: self.token_type, - } - } else { - self - } - } -} diff --git a/tokens/tests/commands.rs b/tokens/tests/commands.rs deleted file mode 100644 index ea0288283e..0000000000 --- a/tokens/tests/commands.rs +++ /dev/null @@ -1,19 +0,0 @@ -use { - solana_client::rpc_client::RpcClient, - solana_sdk::signature::{Keypair, Signer}, - solana_streamer::socket::SocketAddrSpace, - solana_test_validator::TestValidator, - solana_tokens::commands::test_process_distribute_tokens_with_client, -}; - -#[test] -fn test_process_distribute_with_rpc_client() { - solana_logger::setup(); - - let mint_keypair = Keypair::new(); - let test_validator = - TestValidator::with_no_fees(mint_keypair.pubkey(), None, SocketAddrSpace::Unspecified); - - let client = RpcClient::new(test_validator.rpc_url()); - test_process_distribute_tokens_with_client(&client, mint_keypair, None); -} diff --git a/transaction-dos/.gitignore b/transaction-dos/.gitignore deleted file mode 100644 index 5404b132db..0000000000 --- a/transaction-dos/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/target/ -/farf/ diff --git a/transaction-dos/Cargo.toml b/transaction-dos/Cargo.toml deleted file mode 100644 index f2758f86b7..0000000000 --- a/transaction-dos/Cargo.toml +++ /dev/null @@ -1,36 +0,0 @@ -[package] -authors = ["Solana Maintainers "] -edition = "2021" -name = "solana-transaction-dos" -version = "1.10.41" -repository = "https://github.com/solana-labs/solana" -license = "Apache-2.0" -homepage = "https://solana.com/" -publish = false - -[dependencies] -bincode = "1.3.3" -clap = "2.33.1" -log = "0.4.14" -rand = "0.7.0" -rayon = "1.5.1" -solana-clap-utils = { path = "../clap-utils", version = "=1.10.41" } -solana-cli = { path = "../cli", version = "=1.10.41" } -solana-client = { path = "../client", version = "=1.10.41" } -solana-core = { path = "../core", version = "=1.10.41" } -solana-faucet = { path = "../faucet", version = "=1.10.41" } -solana-gossip = { path = "../gossip", version = "=1.10.41" } -solana-logger = { path = "../logger", version = "=1.10.41" } -solana-measure = { path = "../measure", version = "=1.10.41" } -solana-net-utils = { path = "../net-utils", version = "=1.10.41" } -solana-runtime = { path = "../runtime", version = "=1.10.41" } -solana-sdk = { path = "../sdk", version = "=1.10.41" } -solana-streamer = { path = "../streamer", version = "=1.10.41" } -solana-transaction-status = { path = "../transaction-status", version = "=1.10.41" } -solana-version = { path = "../version", version = "=0.7.0" } - -[dev-dependencies] -solana-local-cluster = { path = "../local-cluster", version = "=1.10.41" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/transaction-dos/src/main.rs b/transaction-dos/src/main.rs deleted file mode 100644 index 360afa6a1a..0000000000 --- a/transaction-dos/src/main.rs +++ /dev/null @@ -1,735 +0,0 @@ -#![allow(clippy::integer_arithmetic)] -use { - clap::{crate_description, crate_name, value_t, values_t_or_exit, App, Arg}, - log::*, - rand::{thread_rng, Rng}, - rayon::prelude::*, - solana_clap_utils::input_parsers::pubkey_of, - solana_cli::{cli::CliConfig, program::process_deploy}, - solana_client::{rpc_client::RpcClient, transaction_executor::TransactionExecutor}, - solana_faucet::faucet::{request_airdrop_transaction, FAUCET_PORT}, - solana_gossip::gossip_service::discover, - solana_sdk::{ - commitment_config::CommitmentConfig, - instruction::{AccountMeta, Instruction}, - message::Message, - packet::PACKET_DATA_SIZE, - pubkey::Pubkey, - rpc_port::DEFAULT_RPC_PORT, - signature::{read_keypair_file, Keypair, Signer}, - system_instruction, - transaction::Transaction, - }, - solana_streamer::socket::SocketAddrSpace, - std::{ - net::SocketAddr, - process::exit, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, - thread::sleep, - time::{Duration, Instant}, - }, -}; - -pub fn airdrop_lamports( - client: &RpcClient, - faucet_addr: &SocketAddr, - id: &Keypair, - desired_balance: u64, -) -> bool { - let starting_balance = client.get_balance(&id.pubkey()).unwrap_or(0); - info!("starting balance {}", starting_balance); - - if starting_balance < desired_balance { - let airdrop_amount = desired_balance - starting_balance; - info!( - "Airdropping {:?} lamports from {} for {}", - airdrop_amount, - faucet_addr, - id.pubkey(), - ); - - let blockhash = client.get_latest_blockhash().unwrap(); - match request_airdrop_transaction(faucet_addr, &id.pubkey(), airdrop_amount, blockhash) { - Ok(transaction) => { - let mut tries = 0; - loop { - tries += 1; - let result = client.send_and_confirm_transaction_with_spinner(&transaction); - - if result.is_ok() { - break; - } - if tries >= 5 { - panic!( - "Error requesting airdrop: to addr: {:?} amount: {} {:?}", - faucet_addr, airdrop_amount, result - ) - } - } - } - Err(err) => { - panic!( - "Error requesting airdrop: {:?} to addr: {:?} amount: {}", - err, faucet_addr, airdrop_amount - ); - } - }; - - let current_balance = client.get_balance(&id.pubkey()).unwrap_or_else(|e| { - panic!("airdrop error {}", e); - }); - info!("current balance {}...", current_balance); - - if current_balance - starting_balance != airdrop_amount { - info!( - "Airdrop failed? {} {} {} {}", - id.pubkey(), - current_balance, - starting_balance, - airdrop_amount, - ); - } - } - true -} - -fn make_create_message( - keypair: &Keypair, - base_keypair: &Keypair, - balance: u64, - space: u64, - program_id: Pubkey, -) -> Message { - let instructions = vec![system_instruction::create_account( - &keypair.pubkey(), - &base_keypair.pubkey(), - balance, - space, - &program_id, - )]; - - Message::new(&instructions, Some(&keypair.pubkey())) -} - -fn make_dos_message( - keypair: &Keypair, - num_instructions: usize, - program_id: Pubkey, - num_program_iterations: u8, - account_metas: &[AccountMeta], -) -> Message { - let instructions: Vec<_> = (0..num_instructions) - .into_iter() - .map(|_| { - let data = [num_program_iterations, thread_rng().gen_range(0, 255)]; - Instruction::new_with_bytes(program_id, &data, account_metas.to_vec()) - }) - .collect(); - - Message::new(&instructions, Some(&keypair.pubkey())) -} - -/// creates large transactions that all touch the same set of accounts, -/// so they can't be parallelized -/// -#[allow(clippy::too_many_arguments)] -fn run_transactions_dos( - entrypoint_addr: SocketAddr, - faucet_addr: SocketAddr, - payer_keypairs: &[&Keypair], - iterations: usize, - maybe_space: Option, - batch_size: usize, - maybe_lamports: Option, - num_instructions: usize, - num_program_iterations: usize, - program_id: Pubkey, - program_options: Option<(Keypair, String)>, - account_keypairs: &[&Keypair], - maybe_account_groups: Option, - just_calculate_fees: bool, - batch_sleep_ms: u64, -) { - assert!(num_instructions > 0); - let client = Arc::new(RpcClient::new_socket_with_commitment( - entrypoint_addr, - CommitmentConfig::confirmed(), - )); - - info!("Targeting {}", entrypoint_addr); - - let space = maybe_space.unwrap_or(1000); - - let min_balance = maybe_lamports.unwrap_or_else(|| { - client - .get_minimum_balance_for_rent_exemption(space as usize) - .expect("min balance") - }); - assert!(min_balance > 0); - - let account_groups = maybe_account_groups.unwrap_or(1); - - assert!(account_keypairs.len() % account_groups == 0); - - let account_group_size = account_keypairs.len() / account_groups; - - let program_account = client.get_account(&program_id); - - let mut blockhash = client.get_latest_blockhash().expect("blockhash"); - let mut message = Message::new_with_blockhash( - &[ - Instruction::new_with_bytes( - Pubkey::new_unique(), - &[], - vec![AccountMeta::new(Pubkey::new_unique(), true)], - ), - Instruction::new_with_bytes( - Pubkey::new_unique(), - &[], - vec![AccountMeta::new(Pubkey::new_unique(), true)], - ), - ], - None, - &blockhash, - ); - - let mut latest_blockhash = Instant::now(); - let mut last_log = Instant::now(); - let mut count = 0; - - if just_calculate_fees { - let fee = client - .get_fee_for_message(&message) - .expect("get_fee_for_message"); - - let account_space_fees = min_balance * account_keypairs.len() as u64; - let program_fees = if program_account.is_ok() { - 0 - } else { - // todo, dynamic real size - client.get_minimum_balance_for_rent_exemption(2400).unwrap() - }; - let transaction_fees = - account_keypairs.len() as u64 * fee + iterations as u64 * batch_size as u64 * fee; - info!( - "Accounts fees: {} program_account fees: {} transaction fees: {} total: {}", - account_space_fees, - program_fees, - transaction_fees, - account_space_fees + program_fees + transaction_fees, - ); - return; - } - - if program_account.is_err() { - let mut config = CliConfig::default(); - let (program_keypair, program_location) = program_options - .as_ref() - .expect("If the program doesn't exist, need to provide program keypair to deploy"); - info!( - "processing deploy: {:?} key: {}", - program_account, - program_keypair.pubkey() - ); - config.signers = vec![payer_keypairs[0], program_keypair]; - process_deploy( - client.clone(), - &config, - program_location, - Some(1), - false, - true, - true, /* skip_fee_check */ - ) - .expect("deploy didn't pass"); - } else { - info!("Found program account. Skipping deploy.."); - assert!(program_account.unwrap().executable); - } - - let mut tx_sent_count = 0; - let mut total_dos_messages_sent = 0; - let mut balances: Vec<_> = payer_keypairs - .iter() - .map(|keypair| client.get_balance(&keypair.pubkey()).unwrap_or(0)) - .collect(); - let mut last_balance = Instant::now(); - - info!("Starting balance(s): {:?}", balances); - - let executor = TransactionExecutor::new(entrypoint_addr); - - let mut accounts_created = false; - let tested_size = Arc::new(AtomicBool::new(false)); - - let account_metas: Vec<_> = account_keypairs - .iter() - .map(|kp| AccountMeta::new(kp.pubkey(), false)) - .collect(); - - loop { - if latest_blockhash.elapsed().as_secs() > 10 { - blockhash = client.get_latest_blockhash().expect("blockhash"); - message.recent_blockhash = blockhash; - latest_blockhash = Instant::now(); - } - - let fee = client - .get_fee_for_message(&message) - .expect("get_fee_for_message"); - let lamports = min_balance + fee; - - for (i, balance) in balances.iter_mut().enumerate() { - if *balance < lamports || last_balance.elapsed().as_secs() > 2 { - if let Ok(b) = client.get_balance(&payer_keypairs[i].pubkey()) { - *balance = b; - } - last_balance = Instant::now(); - if *balance < lamports * 2 { - info!( - "Balance {} is less than needed: {}, doing aidrop...", - balance, lamports - ); - if !airdrop_lamports( - &client, - &faucet_addr, - payer_keypairs[i], - lamports * 100_000, - ) { - warn!("failed airdrop, exiting"); - return; - } - } - } - } - - if !accounts_created { - let mut accounts_to_create = vec![]; - for kp in account_keypairs { - if let Ok(account) = client.get_account(&kp.pubkey()) { - if account.data.len() as u64 != space { - info!( - "account {} doesn't have space specified. Has {} requested: {}", - kp.pubkey(), - account.data.len(), - space, - ); - } - } else { - accounts_to_create.push(kp); - } - } - - if !accounts_to_create.is_empty() { - info!("creating accounts {}", accounts_to_create.len()); - let txs: Vec<_> = accounts_to_create - .par_iter() - .enumerate() - .map(|(i, keypair)| { - let message = make_create_message( - payer_keypairs[i % payer_keypairs.len()], - keypair, - min_balance, - space, - program_id, - ); - let signers: Vec<&Keypair> = - vec![payer_keypairs[i % payer_keypairs.len()], keypair]; - Transaction::new(&signers, message, blockhash) - }) - .collect(); - let mut new_ids = executor.push_transactions(txs); - warn!("sent account creation {}", new_ids.len()); - let start = Instant::now(); - loop { - let cleared = executor.drain_cleared(); - new_ids.retain(|x| !cleared.contains(x)); - if new_ids.is_empty() { - break; - } - if start.elapsed().as_secs() > 60 { - info!("Some creation failed"); - break; - } - sleep(Duration::from_millis(500)); - } - for kp in account_keypairs { - let account = client.get_account(&kp.pubkey()).unwrap(); - info!("{} => {:?}", kp.pubkey(), account); - assert!(account.data.len() as u64 == space); - } - } else { - info!("All accounts created."); - } - accounts_created = true; - } else { - // Create dos transactions - info!("creating new batch of size: {}", batch_size); - let chunk_size = batch_size / payer_keypairs.len(); - for (i, keypair) in payer_keypairs.iter().enumerate() { - let txs: Vec<_> = (0..chunk_size) - .into_par_iter() - .map(|x| { - let message = make_dos_message( - keypair, - num_instructions, - program_id, - num_program_iterations as u8, - &account_metas[(x % account_groups) * account_group_size - ..(x % account_groups) * account_group_size + account_group_size], - ); - let signers: Vec<&Keypair> = vec![keypair]; - let tx = Transaction::new(&signers, message, blockhash); - if !tested_size.load(Ordering::Relaxed) { - let ser_size = bincode::serialized_size(&tx).unwrap(); - assert!(ser_size < PACKET_DATA_SIZE as u64, "{}", ser_size); - tested_size.store(true, Ordering::Relaxed); - } - tx - }) - .collect(); - balances[i] = balances[i].saturating_sub(fee * txs.len() as u64); - info!("txs: {}", txs.len()); - let new_ids = executor.push_transactions(txs); - info!("ids: {}", new_ids.len()); - tx_sent_count += new_ids.len(); - total_dos_messages_sent += num_instructions * new_ids.len(); - } - let _ = executor.drain_cleared(); - } - - count += 1; - if last_log.elapsed().as_secs() > 3 { - info!( - "total_dos_messages_sent: {} tx_sent_count: {} loop_count: {} balance(s): {:?}", - total_dos_messages_sent, tx_sent_count, count, balances - ); - last_log = Instant::now(); - } - if iterations != 0 && count >= iterations { - break; - } - if executor.num_outstanding() >= batch_size { - sleep(Duration::from_millis(batch_sleep_ms)); - } - } - executor.close(); -} - -fn main() { - solana_logger::setup_with_default("solana=info"); - let matches = App::new(crate_name!()) - .about(crate_description!()) - .version(solana_version::version!()) - .arg( - Arg::with_name("entrypoint") - .long("entrypoint") - .takes_value(true) - .value_name("HOST:PORT") - .help("RPC entrypoint address. Usually :8899"), - ) - .arg( - Arg::with_name("faucet_addr") - .long("faucet") - .takes_value(true) - .value_name("HOST:PORT") - .help("Faucet entrypoint address. Usually :9900"), - ) - .arg( - Arg::with_name("space") - .long("space") - .takes_value(true) - .value_name("BYTES") - .help("Size of accounts to create"), - ) - .arg( - Arg::with_name("lamports") - .long("lamports") - .takes_value(true) - .value_name("LAMPORTS") - .help("How many lamports to fund each account"), - ) - .arg( - Arg::with_name("payer") - .long("payer") - .takes_value(true) - .multiple(true) - .value_name("FILE") - .help("One or more payer keypairs to fund account creation."), - ) - .arg( - Arg::with_name("account") - .long("account") - .takes_value(true) - .multiple(true) - .value_name("FILE") - .help("One or more keypairs to create accounts owned by the program and which the program will write to."), - ) - .arg( - Arg::with_name("account_groups") - .long("account_groups") - .takes_value(true) - .value_name("NUM") - .help("Number of groups of accounts to split the accounts into") - ) - .arg( - Arg::with_name("batch_size") - .long("batch-size") - .takes_value(true) - .value_name("NUM") - .help("Number of transactions to send per batch"), - ) - .arg( - Arg::with_name("num_instructions") - .long("num-instructions") - .takes_value(true) - .value_name("NUM") - .help("Number of accounts to create on each transaction"), - ) - .arg( - Arg::with_name("num_program_iterations") - .long("num-program-iterations") - .takes_value(true) - .value_name("NUM") - .help("Number of iterations in the smart contract"), - ) - .arg( - Arg::with_name("iterations") - .long("iterations") - .takes_value(true) - .value_name("NUM") - .help("Number of iterations to make"), - ) - .arg( - Arg::with_name("batch_sleep_ms") - .long("batch-sleep-ms") - .takes_value(true) - .value_name("NUM") - .help("Sleep for this long the num outstanding transctions is greater than the batch size."), - ) - .arg( - Arg::with_name("check_gossip") - .long("check-gossip") - .help("Just use entrypoint address directly"), - ) - .arg( - Arg::with_name("just_calculate_fees") - .long("just-calculate-fees") - .help("Just print the necessary fees and exit"), - ) - .arg( - Arg::with_name("program_id") - .long("program-id") - .takes_value(true) - .required(true) - .help("program_id address to initialize account"), - ) - .get_matches(); - - let skip_gossip = !matches.is_present("check_gossip"); - let just_calculate_fees = matches.is_present("just_calculate_fees"); - - let port = if skip_gossip { DEFAULT_RPC_PORT } else { 8001 }; - let mut entrypoint_addr = SocketAddr::from(([127, 0, 0, 1], port)); - if let Some(addr) = matches.value_of("entrypoint") { - entrypoint_addr = solana_net_utils::parse_host_port(addr).unwrap_or_else(|e| { - eprintln!("failed to parse entrypoint address: {}", e); - exit(1) - }); - } - let mut faucet_addr = SocketAddr::from(([127, 0, 0, 1], FAUCET_PORT)); - if let Some(addr) = matches.value_of("faucet_addr") { - faucet_addr = solana_net_utils::parse_host_port(addr).unwrap_or_else(|e| { - eprintln!("failed to parse entrypoint address: {}", e); - exit(1) - }); - } - - let space = value_t!(matches, "space", u64).ok(); - let lamports = value_t!(matches, "lamports", u64).ok(); - let batch_size = value_t!(matches, "batch_size", usize).unwrap_or(4); - let iterations = value_t!(matches, "iterations", usize).unwrap_or(10); - let num_program_iterations = value_t!(matches, "num_program_iterations", usize).unwrap_or(10); - let num_instructions = value_t!(matches, "num_instructions", usize).unwrap_or(1); - if num_instructions == 0 || num_instructions > 500 { - eprintln!("bad num_instructions: {}", num_instructions); - exit(1); - } - let batch_sleep_ms = value_t!(matches, "batch_sleep_ms", u64).unwrap_or(500); - - let program_id = pubkey_of(&matches, "program_id").unwrap(); - - let payer_keypairs: Vec<_> = values_t_or_exit!(matches, "payer", String) - .iter() - .map(|keypair_string| { - read_keypair_file(keypair_string) - .unwrap_or_else(|_| panic!("bad keypair {:?}", keypair_string)) - }) - .collect(); - - let account_keypairs: Vec<_> = values_t_or_exit!(matches, "account", String) - .iter() - .map(|keypair_string| { - read_keypair_file(keypair_string) - .unwrap_or_else(|_| panic!("bad keypair {:?}", keypair_string)) - }) - .collect(); - - let account_groups = value_t!(matches, "account_groups", usize).ok(); - let payer_keypair_refs: Vec<&Keypair> = payer_keypairs.iter().collect(); - let account_keypair_refs: Vec<&Keypair> = account_keypairs.iter().collect(); - - let rpc_addr = if !skip_gossip { - info!("Finding cluster entry: {:?}", entrypoint_addr); - let (gossip_nodes, _validators) = discover( - None, // keypair - Some(&entrypoint_addr), - None, // num_nodes - Duration::from_secs(60), // timeout - None, // find_node_by_pubkey - Some(&entrypoint_addr), // find_node_by_gossip_addr - None, // my_gossip_addr - 0, // my_shred_version - SocketAddrSpace::Unspecified, - ) - .unwrap_or_else(|err| { - eprintln!("Failed to discover {} node: {:?}", entrypoint_addr, err); - exit(1); - }); - - info!("done found {} nodes", gossip_nodes.len()); - gossip_nodes[0].rpc - } else { - info!("Using {:?} as the RPC address", entrypoint_addr); - entrypoint_addr - }; - - run_transactions_dos( - rpc_addr, - faucet_addr, - &payer_keypair_refs, - iterations, - space, - batch_size, - lamports, - num_instructions, - num_program_iterations, - program_id, - None, - &account_keypair_refs, - account_groups, - just_calculate_fees, - batch_sleep_ms, - ); -} - -#[cfg(test)] -pub mod test { - use { - super::*, - solana_core::validator::ValidatorConfig, - solana_local_cluster::{ - local_cluster::{ClusterConfig, LocalCluster}, - validator_configs::make_identical_validator_configs, - }, - solana_measure::measure::Measure, - solana_sdk::poh_config::PohConfig, - }; - - #[test] - fn test_tx_size() { - solana_logger::setup(); - let keypair = Keypair::new(); - let num_instructions = 20; - let program_id = Pubkey::new_unique(); - let num_accounts = 17; - - let account_metas: Vec<_> = (0..num_accounts) - .into_iter() - .map(|_| AccountMeta::new(Pubkey::new_unique(), false)) - .collect(); - let num_program_iterations = 10; - let message = make_dos_message( - &keypair, - num_instructions, - program_id, - num_program_iterations, - &account_metas, - ); - let signers: Vec<&Keypair> = vec![&keypair]; - let blockhash = solana_sdk::hash::Hash::default(); - let tx = Transaction::new(&signers, message, blockhash); - let size = bincode::serialized_size(&tx).unwrap(); - info!("size:{}", size); - assert!(size < PACKET_DATA_SIZE as u64); - } - - #[test] - #[ignore] - fn test_transaction_dos() { - solana_logger::setup(); - - let validator_config = ValidatorConfig::default_for_test(); - let num_nodes = 1; - let mut config = ClusterConfig { - cluster_lamports: 10_000_000, - poh_config: PohConfig::new_sleep(Duration::from_millis(50)), - node_stakes: vec![100; num_nodes], - validator_configs: make_identical_validator_configs(&validator_config, num_nodes), - ..ClusterConfig::default() - }; - - let faucet_addr = SocketAddr::from(([127, 0, 0, 1], 9900)); - let cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified); - - let program_keypair = Keypair::new(); - - let iterations = 1000; - let maybe_space = Some(10_000_000); - let batch_size = 1; - let maybe_lamports = Some(10); - let maybe_account_groups = Some(1); - // 85 inst, 142 iterations, 5 accounts - // 20 inst, 30 * 20 iterations, 1 account - // - // 100 inst, 7 * 20 iterations, 1 account - let num_instructions = 70; - let num_program_iterations = 10; - let num_accounts = 7; - let account_keypairs: Vec<_> = (0..num_accounts) - .into_iter() - .map(|_| Keypair::new()) - .collect(); - let account_keypair_refs: Vec<_> = account_keypairs.iter().collect(); - let mut start = Measure::start("total accounts run"); - run_transactions_dos( - cluster.entry_point_info.rpc, - faucet_addr, - &[&cluster.funding_keypair], - iterations, - maybe_space, - batch_size, - maybe_lamports, - num_instructions, - num_program_iterations, - program_keypair.pubkey(), - Some(( - program_keypair, - format!( - "{}{}", - env!("CARGO_MANIFEST_DIR"), - "/../programs/bpf/c/out/tuner.so" - ), - )), - &account_keypair_refs, - maybe_account_groups, - false, - 100, - ); - start.stop(); - info!("{}", start); - } -} diff --git a/upload-perf/.gitignore b/upload-perf/.gitignore deleted file mode 100644 index 5404b132db..0000000000 --- a/upload-perf/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/target/ -/farf/ diff --git a/upload-perf/Cargo.toml b/upload-perf/Cargo.toml deleted file mode 100644 index bdf9b49e5e..0000000000 --- a/upload-perf/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "solana-upload-perf" -version = "1.10.41" -description = "Metrics Upload Utility" -authors = ["Solana Maintainers "] -repository = "https://github.com/solana-labs/solana" -license = "Apache-2.0" -edition = "2021" -homepage = "https://solana.com/" -publish = false - -[dependencies] -serde_json = "1.0.79" -solana-metrics = { path = "../metrics", version = "=1.10.41" } - -[[bin]] -name = "solana-upload-perf" -path = "src/upload-perf.rs" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/upload-perf/src/upload-perf.rs b/upload-perf/src/upload-perf.rs deleted file mode 100644 index 16f8a433db..0000000000 --- a/upload-perf/src/upload-perf.rs +++ /dev/null @@ -1,116 +0,0 @@ -use { - serde_json::Value, - std::{ - collections::HashMap, - env, - fs::File, - io::{BufRead, BufReader}, - process::Command, - }, -}; - -fn get_last_metrics(metric: &str, db: &str, name: &str, branch: &str) -> Result { - let query = format!( - r#"SELECT last("{}") FROM "{}"."autogen"."{}" WHERE "branch"='{}'"#, - metric, db, name, branch - ); - - let response = solana_metrics::query(&query)?; - - match serde_json::from_str(&response) { - Result::Ok(v) => { - let v: Value = v; - let data = &v["results"][0]["series"][0]["values"][0][1]; - if data.is_null() { - return Result::Err("Key not found".to_string()); - } - Result::Ok(data.to_string()) - } - Result::Err(err) => Result::Err(err.to_string()), - } -} - -fn main() { - let args: Vec = env::args().collect(); - // Open the path in read-only mode, returns `io::Result` - let fname = &args[1]; - let file = match File::open(fname) { - Err(why) => panic!("couldn't open {}: {:?}", fname, why), - Ok(file) => file, - }; - - let branch = &args[2]; - let upload_metrics = args.len() > 2; - - let git_output = Command::new("git") - .args(["rev-parse", "HEAD"]) - .output() - .expect("failed to execute git rev-parse"); - let git_commit_hash = String::from_utf8_lossy(&git_output.stdout); - let trimmed_hash = git_commit_hash.trim().to_string(); - - let mut last_commit = None; - let mut results = HashMap::new(); - - let db = env::var("INFLUX_DATABASE").unwrap_or_else(|_| "scratch".to_string()); - - for line in BufReader::new(file).lines() { - if let Ok(v) = serde_json::from_str(&line.unwrap()) { - let v: Value = v; - if v["type"] == "bench" { - let name = v["name"].as_str().unwrap().trim_matches('\"').to_string(); - - if last_commit.is_none() { - last_commit = get_last_metrics("commit", &db, &name, branch).ok(); - } - - let median: i64 = v["median"].to_string().parse().unwrap(); - let deviation: i64 = v["deviation"].to_string().parse().unwrap(); - assert!(!upload_metrics, "TODO"); - /* - solana_metrics::datapoint_info!( - &v["name"].as_str().unwrap().trim_matches('\"'), - ("test", "bench", String), - ("branch", branch.to_string(), String), - ("median", median, i64), - ("deviation", deviation, i64), - ("commit", git_commit_hash.trim().to_string(), String) - ); - */ - - let last_median = - get_last_metrics("median", &db, &name, branch).unwrap_or_default(); - let last_deviation = - get_last_metrics("deviation", &db, &name, branch).unwrap_or_default(); - - results.insert(name, (median, deviation, last_median, last_deviation)); - } - } - } - - if let Some(commit) = last_commit { - println!( - "Comparing current commits: {} against baseline {} on {} branch", - trimmed_hash, commit, branch - ); - println!("bench_name, median, last_median, deviation, last_deviation"); - for (entry, values) in results { - println!( - "{}, {:#10?}, {:#10?}, {:#10?}, {:#10?}", - entry, - values.0, - values.2.parse::().unwrap_or_default(), - values.1, - values.3.parse::().unwrap_or_default(), - ); - } - } else { - println!("No previous results found for {} branch", branch); - println!("hash: {}", trimmed_hash); - println!("bench_name, median, deviation"); - for (entry, values) in results { - println!("{}, {:10?}, {:10?}", entry, values.0, values.1); - } - } - solana_metrics::flush(); -} diff --git a/watchtower/.gitignore b/watchtower/.gitignore deleted file mode 100644 index 5404b132db..0000000000 --- a/watchtower/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/target/ -/farf/ diff --git a/watchtower/Cargo.toml b/watchtower/Cargo.toml deleted file mode 100644 index b283fbf489..0000000000 --- a/watchtower/Cargo.toml +++ /dev/null @@ -1,27 +0,0 @@ -[package] -authors = ["Solana Maintainers "] -edition = "2021" -name = "solana-watchtower" -description = "Blockchain, Rebuilt for Scale" -version = "1.10.41" -repository = "https://github.com/solana-labs/solana" -license = "Apache-2.0" -homepage = "https://solana.com/" -documentation = "https://docs.rs/solana-watchtower" - -[dependencies] -clap = "2.33.1" -humantime = "2.0.1" -log = "0.4.14" -solana-clap-utils = { path = "../clap-utils", version = "=1.10.41" } -solana-cli-config = { path = "../cli-config", version = "=1.10.41" } -solana-cli-output = { path = "../cli-output", version = "=1.10.41" } -solana-client = { path = "../client", version = "=1.10.41" } -solana-logger = { path = "../logger", version = "=1.10.41" } -solana-metrics = { path = "../metrics", version = "=1.10.41" } -solana-notifier = { path = "../notifier", version = "=1.10.41" } -solana-sdk = { path = "../sdk", version = "=1.10.41" } -solana-version = { path = "../version", version = "=0.7.0" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/watchtower/README.md b/watchtower/README.md deleted file mode 100644 index 33bad3e458..0000000000 --- a/watchtower/README.md +++ /dev/null @@ -1,25 +0,0 @@ -The `solana-watchtower` program is used to monitor the health of a cluster. It -periodically polls the cluster over an RPC API to confirm that the transaction -count is advancing, new blockhashes are available, and no validators are -delinquent. Results are reported as InfluxDB metrics, with an optional push -notification on sanity failure. - -If you only care about the health of one specific validator, the -`--validator-identity` command-line argument can be used to restrict failure -notifications to issues only affecting that validator. - -If you do not want duplicate notifications, for example if you have elected to -recieve notifications by SMS the -`--no-duplicate-notifications` command-line argument will suppress identical -failure notifications. - -### Metrics -#### `watchtower-sanity` -On every iteration this data point will be emitted indicating the overall result -using a boolean `ok` field. - -#### `watchtower-sanity-failure` -On failure this data point contains details about the specific test that failed via -the following fields: -* `test`: name of the sanity test that failed -* `err`: exact sanity failure message diff --git a/watchtower/src/main.rs b/watchtower/src/main.rs deleted file mode 100644 index 965b51e4de..0000000000 --- a/watchtower/src/main.rs +++ /dev/null @@ -1,379 +0,0 @@ -//! A command-line executable for monitoring the health of a cluster -#![allow(clippy::integer_arithmetic)] - -use { - clap::{crate_description, crate_name, value_t, value_t_or_exit, App, Arg}, - log::*, - solana_clap_utils::{ - input_parsers::pubkeys_of, - input_validators::{is_parsable, is_pubkey_or_keypair, is_url}, - }, - solana_cli_output::display::format_labeled_address, - solana_client::{client_error, rpc_client::RpcClient, rpc_response::RpcVoteAccountStatus}, - solana_metrics::{datapoint_error, datapoint_info}, - solana_notifier::Notifier, - solana_sdk::{ - hash::Hash, - native_token::{sol_to_lamports, Sol}, - pubkey::Pubkey, - }, - std::{ - collections::HashMap, - error, - thread::sleep, - time::{Duration, Instant}, - }, -}; - -struct Config { - address_labels: HashMap, - ignore_http_bad_gateway: bool, - interval: Duration, - json_rpc_url: String, - minimum_validator_identity_balance: u64, - monitor_active_stake: bool, - unhealthy_threshold: usize, - validator_identity_pubkeys: Vec, -} - -fn get_config() -> Config { - let matches = App::new(crate_name!()) - .about(crate_description!()) - .version(solana_version::version!()) - .after_help("ADDITIONAL HELP: - To receive a Slack, Discord and/or Telegram notification on sanity failure, - define environment variables before running `solana-watchtower`: - - export SLACK_WEBHOOK=... - export DISCORD_WEBHOOK=... - - Telegram requires the following two variables: - - export TELEGRAM_BOT_TOKEN=... - export TELEGRAM_CHAT_ID=... - - To receive a Twilio SMS notification on failure, having a Twilio account, - and a sending number owned by that account, - define environment variable before running `solana-watchtower`: - - export TWILIO_CONFIG='ACCOUNT=,TOKEN=,TO=,FROM='") - .arg({ - let arg = Arg::with_name("config_file") - .short("C") - .long("config") - .value_name("PATH") - .takes_value(true) - .global(true) - .help("Configuration file to use"); - if let Some(ref config_file) = *solana_cli_config::CONFIG_FILE { - arg.default_value(config_file) - } else { - arg - } - }) - .arg( - Arg::with_name("json_rpc_url") - .long("url") - .value_name("URL") - .takes_value(true) - .validator(is_url) - .help("JSON RPC URL for the cluster"), - ) - .arg( - Arg::with_name("interval") - .long("interval") - .value_name("SECONDS") - .takes_value(true) - .default_value("60") - .help("Wait interval seconds between checking the cluster"), - ) - .arg( - Arg::with_name("unhealthy_threshold") - .long("unhealthy-threshold") - .value_name("COUNT") - .takes_value(true) - .default_value("1") - .help("How many consecutive failures must occur to trigger a notification") - ) - .arg( - Arg::with_name("validator_identities") - .long("validator-identity") - .value_name("VALIDATOR IDENTITY PUBKEY") - .takes_value(true) - .validator(is_pubkey_or_keypair) - .multiple(true) - .help("Validator identities to monitor for delinquency") - ) - .arg( - Arg::with_name("minimum_validator_identity_balance") - .long("minimum-validator-identity-balance") - .value_name("VLX") - .takes_value(true) - .default_value("10") - .validator(is_parsable::) - .help("Alert when the validator identity balance is less than this amount of VLX") - ) - .arg( - // Deprecated parameter, now always enabled - Arg::with_name("no_duplicate_notifications") - .long("no-duplicate-notifications") - .hidden(true) - ) - .arg( - Arg::with_name("monitor_active_stake") - .long("monitor-active-stake") - .takes_value(false) - .help("Alert when the current stake for the cluster drops below 80%"), - ) - .arg( - Arg::with_name("ignore_http_bad_gateway") - .long("ignore-http-bad-gateway") - .takes_value(false) - .help("Ignore HTTP 502 Bad Gateway errors from the JSON RPC URL. \ - This flag can help reduce false positives, at the expense of \ - no alerting should a Bad Gateway error be a side effect of \ - the real problem") - ) - .get_matches(); - - let config = if let Some(config_file) = matches.value_of("config_file") { - solana_cli_config::Config::load(config_file).unwrap_or_default() - } else { - solana_cli_config::Config::default() - }; - - let interval = Duration::from_secs(value_t_or_exit!(matches, "interval", u64)); - let unhealthy_threshold = value_t_or_exit!(matches, "unhealthy_threshold", usize); - let minimum_validator_identity_balance = sol_to_lamports(value_t_or_exit!( - matches, - "minimum_validator_identity_balance", - f64 - )); - let json_rpc_url = - value_t!(matches, "json_rpc_url", String).unwrap_or_else(|_| config.json_rpc_url.clone()); - let validator_identity_pubkeys: Vec<_> = pubkeys_of(&matches, "validator_identities") - .unwrap_or_default(); - - let monitor_active_stake = matches.is_present("monitor_active_stake"); - let ignore_http_bad_gateway = matches.is_present("ignore_http_bad_gateway"); - - let config = Config { - address_labels: config.address_labels, - ignore_http_bad_gateway, - interval, - json_rpc_url, - minimum_validator_identity_balance, - monitor_active_stake, - unhealthy_threshold, - validator_identity_pubkeys, - }; - - info!("RPC URL: {}", config.json_rpc_url); - info!( - "Monitored validators: {:?}", - config.validator_identity_pubkeys - ); - config -} - -fn get_cluster_info( - config: &Config, - rpc_client: &RpcClient, -) -> client_error::Result<(u64, Hash, RpcVoteAccountStatus, HashMap)> { - let transaction_count = rpc_client.get_transaction_count()?; - let recent_blockhash = rpc_client.get_latest_blockhash()?; - let vote_accounts = rpc_client.get_vote_accounts()?; - - let mut validator_balances = HashMap::new(); - for validator_identity in &config.validator_identity_pubkeys { - validator_balances.insert( - *validator_identity, - rpc_client.get_balance(validator_identity)?, - ); - } - - Ok(( - transaction_count, - recent_blockhash, - vote_accounts, - validator_balances, - )) -} - -fn main() -> Result<(), Box> { - solana_logger::setup_with_default("solana=info"); - solana_metrics::set_panic_hook("watchtower", /*version:*/ None); - - let config = get_config(); - - let rpc_client = RpcClient::new(config.json_rpc_url.clone()); - let notifier = Notifier::default(); - let mut last_transaction_count = 0; - let mut last_recent_blockhash = Hash::default(); - let mut last_notification_msg = "".into(); - let mut num_consecutive_failures = 0; - let mut last_success = Instant::now(); - - loop { - let failure = match get_cluster_info(&config, &rpc_client) { - Ok((transaction_count, recent_blockhash, vote_accounts, validator_balances)) => { - info!("Current transaction count: {}", transaction_count); - info!("Recent blockhash: {}", recent_blockhash); - info!("Current validator count: {}", vote_accounts.current.len()); - info!( - "Delinquent validator count: {}", - vote_accounts.delinquent.len() - ); - - let mut failures = vec![]; - - let total_current_stake = vote_accounts - .current - .iter() - .map(|vote_account| vote_account.activated_stake) - .sum(); - let total_delinquent_stake = vote_accounts - .delinquent - .iter() - .map(|vote_account| vote_account.activated_stake) - .sum(); - - let total_stake = total_current_stake + total_delinquent_stake; - let current_stake_percent = total_current_stake as f64 * 100. / total_stake as f64; - info!( - "Current stake: {:.2}% | Total stake: {}, current stake: {}, delinquent: {}", - current_stake_percent, - Sol(total_stake), - Sol(total_current_stake), - Sol(total_delinquent_stake) - ); - - if transaction_count > last_transaction_count { - last_transaction_count = transaction_count; - } else { - failures.push(( - "transaction-count", - format!( - "Transaction count is not advancing: {} <= {}", - transaction_count, last_transaction_count - ), - )); - } - - if recent_blockhash != last_recent_blockhash { - last_recent_blockhash = recent_blockhash; - } else { - failures.push(( - "recent-blockhash", - format!("Unable to get new blockhash: {}", recent_blockhash), - )); - } - - if config.monitor_active_stake && current_stake_percent < 80. { - failures.push(( - "current-stake", - format!("Current stake is {:.2}%", current_stake_percent), - )); - } - - let mut validator_errors = vec![]; - for validator_identity in config.validator_identity_pubkeys.iter() { - let formatted_validator_identity = format_labeled_address( - &validator_identity.to_string(), - &config.address_labels, - ); - if vote_accounts - .delinquent - .iter() - .any(|vai| vai.node_pubkey == *validator_identity.to_string()) - { - validator_errors - .push(format!("{} delinquent", formatted_validator_identity)); - } else if !vote_accounts - .current - .iter() - .any(|vai| vai.node_pubkey == *validator_identity.to_string()) - { - validator_errors.push(format!("{} missing", formatted_validator_identity)); - } - - if let Some(balance) = validator_balances.get(validator_identity) { - if *balance < config.minimum_validator_identity_balance { - failures.push(( - "balance", - format!("{} has {}", formatted_validator_identity, Sol(*balance)), - )); - } - } - } - - if !validator_errors.is_empty() { - failures.push(("delinquent", validator_errors.join(","))); - } - - for failure in failures.iter() { - error!("{} sanity failure: {}", failure.0, failure.1); - } - failures.into_iter().next() // Only report the first failure if any - } - Err(err) => { - let mut failure = Some(("rpc-error", err.to_string())); - - if let client_error::ClientErrorKind::Reqwest(reqwest_err) = err.kind() { - if let Some(client_error::reqwest::StatusCode::BAD_GATEWAY) = - reqwest_err.status() - { - if config.ignore_http_bad_gateway { - warn!("Error suppressed: {}", err); - failure = None; - } - } - } - failure - } - }; - - if let Some((failure_test_name, failure_error_message)) = &failure { - let notification_msg = format!( - "solana-watchtower: Error: {}: {}", - failure_test_name, failure_error_message - ); - num_consecutive_failures += 1; - if num_consecutive_failures > config.unhealthy_threshold { - datapoint_info!("watchtower-sanity", ("ok", false, bool)); - if last_notification_msg != notification_msg { - notifier.send(¬ification_msg); - } - datapoint_error!( - "watchtower-sanity-failure", - ("test", failure_test_name, String), - ("err", failure_error_message, String) - ); - last_notification_msg = notification_msg; - } else { - info!( - "Failure {} of {}: {}", - num_consecutive_failures, config.unhealthy_threshold, notification_msg - ); - } - } else { - datapoint_info!("watchtower-sanity", ("ok", true, bool)); - if !last_notification_msg.is_empty() { - let alarm_duration = Instant::now().duration_since(last_success); - let alarm_duration = alarm_duration - config.interval; // Subtract the period before the first error - let alarm_duration = Duration::from_secs(alarm_duration.as_secs()); // Drop milliseconds in message - - let all_clear_msg = format!( - "All clear after {}", - humantime::format_duration(alarm_duration) - ); - info!("{}", all_clear_msg); - notifier.send(&format!("solana-watchtower: {}", all_clear_msg)); - } - last_notification_msg = "".into(); - last_success = Instant::now(); - num_consecutive_failures = 0; - } - sleep(config.interval); - } -}