diff --git a/Cargo.lock b/Cargo.lock index 71e6400cc51a39..9c6bd49727158e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6262,6 +6262,7 @@ dependencies = [ "solana-clap-utils", "solana-clap-v3-utils", "solana-client", + "solana-core", "solana-entry", "solana-frozen-abi", "solana-frozen-abi-macro", diff --git a/k8s-cluster/Cargo.toml b/k8s-cluster/Cargo.toml index 34c51c6c9c29be..232be52e173de8 100644 --- a/k8s-cluster/Cargo.toml +++ b/k8s-cluster/Cargo.toml @@ -49,6 +49,7 @@ solana-bloom = { workspace = true } solana-clap-utils = { workspace = true } solana-clap-v3-utils = { workspace = true } solana-client = { workspace = true } +solana-core = { workspace = true } solana-entry = { workspace = true } solana-frozen-abi = { workspace = true } solana-frozen-abi-macro = { workspace = true } diff --git a/k8s-cluster/src/genesis.rs b/k8s-cluster/src/genesis.rs index 7c483cd9538925..8c349905d2ec96 100644 --- a/k8s-cluster/src/genesis.rs +++ b/k8s-cluster/src/genesis.rs @@ -230,6 +230,10 @@ impl Genesis { let keypair = generate_keypair()?; self.all_pubkeys.push(keypair.pubkey()); + if account == "identity" { + info!("identity: {:?}", keypair.pubkey()); + } + if let Some(outfile) = outfile.to_str() { output_keypair(&keypair, outfile) .map_err(|err| format!("Unable to write {outfile}: {err}"))?; diff --git a/k8s-cluster/src/kubernetes.rs b/k8s-cluster/src/kubernetes.rs index a04b0f2c204897..a2e9a48f2c1400 100644 --- a/k8s-cluster/src/kubernetes.rs +++ b/k8s-cluster/src/kubernetes.rs @@ -22,9 +22,9 @@ use { std::{collections::BTreeMap, error::Error, fs::File, io::Read}, }; -pub struct RuntimeConfig<'a> { - pub enable_udp: bool, - pub disable_quic: bool, +pub struct ValidatorConfig<'a> { + pub tpu_enable_udp: bool, + pub tpu_disable_quic: bool, pub gpu_mode: &'a str, // TODO: this is not implemented yet pub internal_node_sol: f64, pub internal_node_stake_sol: f64, @@ -32,24 +32,34 @@ pub struct RuntimeConfig<'a> { pub warp_slot: Option, pub shred_version: Option, pub bank_hash: Option, + pub max_ledger_size: Option, + pub skip_poh_verify: bool, + pub no_snapshot_fetch: bool, + pub skip_require_tower: bool, + pub enable_full_rpc: bool, } -impl<'a> std::fmt::Display for RuntimeConfig<'a> { +impl<'a> std::fmt::Display for ValidatorConfig<'a> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( f, "Runtime Config\n\ - enable_udp: {}\n\ - disable_quic: {}\n\ + tpu_enable_udp: {}\n\ + tpu_disable_quic: {}\n\ gpu_mode: {}\n\ internal_node_sol: {}\n\ internal_node_stake_sol: {}\n\ wait_for_supermajority: {:?}\n\ warp_slot: {:?}\n\ shred_version: {:?}\n\ - bank_hash: {:?}", - self.enable_udp, - self.disable_quic, + bank_hash: {:?}\n\ + max_ledger_size: {:?}\n\ + skip_poh_verify: {}\n\ + no_snapshot_fetch: {}\n\ + skip_require_tower: {}\n\ + enable_full_rpc: {}", + self.tpu_enable_udp, + self.tpu_disable_quic, self.gpu_mode, self.internal_node_sol, self.internal_node_stake_sol, @@ -57,6 +67,11 @@ impl<'a> std::fmt::Display for RuntimeConfig<'a> { self.warp_slot, self.shred_version, self.bank_hash, + self.max_ledger_size, + self.skip_poh_verify, + self.no_snapshot_fetch, + self.skip_require_tower, + self.enable_full_rpc, ) } } @@ -64,49 +79,73 @@ impl<'a> std::fmt::Display for RuntimeConfig<'a> { pub struct Kubernetes<'a> { client: Client, namespace: &'a str, - runtime_config: &'a mut RuntimeConfig<'a>, + validator_config: &'a mut ValidatorConfig<'a>, } impl<'a> Kubernetes<'a> { pub async fn new( namespace: &'a str, - runtime_config: &'a mut RuntimeConfig<'a>, + validator_config: &'a mut ValidatorConfig<'a>, ) -> Kubernetes<'a> { Kubernetes { client: Client::try_default().await.unwrap(), namespace, - runtime_config, + validator_config, } } pub fn set_shred_version(&mut self, shred_version: u16) { - self.runtime_config.shred_version = Some(shred_version); + self.validator_config.shred_version = Some(shred_version); } pub fn set_bank_hash(&mut self, bank_hash: Hash) { - self.runtime_config.bank_hash = Some(bank_hash); + self.validator_config.bank_hash = Some(bank_hash); } fn generate_command_flags(&mut self) -> Vec { let mut flags = Vec::new(); - if self.runtime_config.enable_udp { + if self.validator_config.tpu_enable_udp { flags.push("--tpu-enable-udp".to_string()); } - if self.runtime_config.disable_quic { + if self.validator_config.tpu_disable_quic { flags.push("--tpu-disable-quic".to_string()); } + if self.validator_config.skip_poh_verify { + flags.push("--skip-poh-verify".to_string()); + } + if self.validator_config.no_snapshot_fetch { + flags.push("--no-snapshot-fetch".to_string()); + } + if self.validator_config.skip_require_tower { + flags.push("--skip-require-tower".to_string()); + } + if self.validator_config.enable_full_rpc { + flags.push("--enable-rpc-transaction-history".to_string()); + flags.push("--enable-extended-tx-metadata-storage".to_string()); + } + if self.validator_config.enable_full_rpc { + flags.push("--enable-rpc-transaction-history".to_string()); + flags.push("--enable-extended-tx-metadata-storage".to_string()); + } - if let Some(slot) = self.runtime_config.wait_for_supermajority { + if let Some(slot) = self.validator_config.wait_for_supermajority { flags.push("--wait-for-supermajority".to_string()); flags.push(slot.to_string()); } - if let Some(bank_hash) = self.runtime_config.bank_hash { + if let Some(bank_hash) = self.validator_config.bank_hash { flags.push("--expected-bank-hash".to_string()); flags.push(bank_hash.to_string()); } + if let Some(limit_ledger_size) = self.validator_config.max_ledger_size { + flags.push("--limit-ledger-size".to_string()); + flags.push(limit_ledger_size.to_string()); + } + + + flags } @@ -118,11 +157,11 @@ impl<'a> Kubernetes<'a> { let mut flags = self.generate_command_flags(); flags.push("--internal-node-stake-sol".to_string()); - flags.push(self.runtime_config.internal_node_stake_sol.to_string()); + flags.push(self.validator_config.internal_node_stake_sol.to_string()); flags.push("--internal-node-sol".to_string()); - flags.push(self.runtime_config.internal_node_sol.to_string()); + flags.push(self.validator_config.internal_node_sol.to_string()); - if let Some(shred_version) = self.runtime_config.shred_version { + if let Some(shred_version) = self.validator_config.shred_version { flags.push("--expected-shred-version".to_string()); flags.push(shred_version.to_string()); } diff --git a/k8s-cluster/src/main.rs b/k8s-cluster/src/main.rs index d681e180ecf3e0..7527d6967201df 100644 --- a/k8s-cluster/src/main.rs +++ b/k8s-cluster/src/main.rs @@ -8,11 +8,12 @@ use { DEFAULT_INTERNAL_NODE_STAKE_SOL, }, initialize_globals, - kubernetes::{Kubernetes, RuntimeConfig}, + kubernetes::{Kubernetes, ValidatorConfig}, ledger_helper::LedgerHelper, release::{BuildConfig, Deploy}, ValidatorType, }, + solana_core::ledger_cleanup_service::{DEFAULT_MAX_LEDGER_SHREDS, DEFAULT_MIN_MAX_LEDGER_SHREDS}, std::{thread, time::Duration}, }; @@ -189,7 +190,7 @@ fn parse_matches() -> ArgMatches<'static> { Arg::with_name("internal_node_stake_sol") .long("internal-node-stake-sol") .takes_value(true) - .help(" Amount to stake internal nodes (Sol)."), + .help("Amount to stake internal nodes (Sol)."), ) .arg( Arg::with_name("enable_warmup_epochs") @@ -218,12 +219,12 @@ fn parse_matches() -> ArgMatches<'static> { .arg( Arg::with_name("tpu_enable_udp") .long("tpu-enable-udp") - .help("Runtime config. Enable UDP for tpu transactions."), + .help("Validator config. Enable UDP for tpu transactions."), ) .arg( Arg::with_name("tpu_disable_quic") .long("tpu-disable-quic") - .help("Genesis config. Disable quic for tpu packet forwarding"), + .help("Validator config. Disable quic for tpu packet forwarding"), ) .arg( Arg::with_name("gpu_mode") @@ -245,6 +246,41 @@ fn parse_matches() -> ArgMatches<'static> { .takes_value(true) .help("Boot from a snapshot that has warped ahead to WARP_SLOT rather than a slot 0 genesis"), ) + .arg( + Arg::with_name("limit_ledger_size") + .long("limit-ledger-size") + .takes_value(true) + .help("Validator Config. The `--limit-ledger-size` parameter allows you to specify how many ledger + shreds your node retains on disk. If you do not + include this parameter, the validator will keep the entire ledger until it runs + out of disk space. The default value attempts to keep the ledger disk usage + under 500GB. More or less disk usage may be requested by adding an argument to + `--limit-ledger-size` if desired. Check `solana-validator --help` for the + default limit value used by `--limit-ledger-size`. More information about + selecting a custom limit value is at : https://github.com/solana-labs/solana/blob/583cec922b6107e0f85c7e14cb5e642bc7dfb340/core/src/ledger_cleanup_service.rs#L15-L26"), + ) + .arg( + Arg::with_name("skip_poh_verify") + .long("skip-poh-verify") + .help("Validator config. If set, validators will skip verifying + the ledger they already have saved to disk at + boot (results in a much faster boot)"), + ) + .arg( + Arg::with_name("no_snapshot_fetch") + .long("no-snapshot-fetch") + .help("Validator config. If set, disables booting validators from a snapshot"), + ) + .arg( + Arg::with_name("skip_require_tower") + .long("skip-require-tower") + .help("Validator config. Disable require tower"), + ) + .arg( + Arg::with_name("full_rpc") + .long("full-rpc") + .help("Validator config. Support full RPC services on all nodes"), + ) .get_matches() } @@ -321,9 +357,9 @@ async fn main() { ), }; - let mut runtime_config = RuntimeConfig { - enable_udp: matches.is_present("tpu_enable_udp"), - disable_quic: matches.is_present("tpu_disable_quic"), + let mut validator_config = ValidatorConfig { + tpu_enable_udp: matches.is_present("tpu_enable_udp"), + tpu_disable_quic: matches.is_present("tpu_disable_quic"), gpu_mode: matches.value_of("gpu_mode").unwrap(), internal_node_sol: matches .value_of("internal_node_sol") @@ -344,15 +380,35 @@ async fn main() { warp_slot: matches .value_of("warp_slot") .map(|value_str| value_str.parse().expect("Invalid value for warp_slot")), + shred_version: None, // set after genesis created bank_hash: None, // set after genesis created + max_ledger_size: if matches.is_present("limit_ledger_size") { + let limit_ledger_size = match matches.value_of("limit_ledger_size") { + Some(_) => value_t_or_exit!(matches, "limit_ledger_size", u64), + None => DEFAULT_MAX_LEDGER_SHREDS, + }; + if limit_ledger_size < DEFAULT_MIN_MAX_LEDGER_SHREDS { + error!( + "The provided --limit-ledger-size value was too small, the minimum value is {DEFAULT_MIN_MAX_LEDGER_SHREDS}" + ); + return; + } + Some(limit_ledger_size) + } else { + None + }, + skip_poh_verify: matches.is_present("skip_poh_verify"), + no_snapshot_fetch: matches.is_present("no_snapshot_fetch"), + skip_require_tower: matches.is_present("skip_require_tower"), + enable_full_rpc: matches.is_present("enable_full_rpc"), }; - let wait_for_supermajority: Option = runtime_config.wait_for_supermajority; - let warp_slot: Option = runtime_config.warp_slot; + let wait_for_supermajority: Option = validator_config.wait_for_supermajority; + let warp_slot: Option = validator_config.warp_slot; if !match ( - runtime_config.wait_for_supermajority, - runtime_config.warp_slot, + validator_config.wait_for_supermajority, + validator_config.warp_slot, ) { (Some(slot1), Some(slot2)) => slot1 == slot2, (None, None) => true, // Both are None, consider them equal @@ -361,14 +417,14 @@ async fn main() { panic!( "Error: When specifying both --wait-for-supermajority and --warp-slot, \ they must use the same slot. ({:?} != {:?})", - runtime_config.wait_for_supermajority, runtime_config.warp_slot + validator_config.wait_for_supermajority, validator_config.warp_slot ); } - info!("Runtime Config: {}", runtime_config); + info!("Runtime Config: {}", validator_config); // Check if namespace exists - let mut kub_controller = Kubernetes::new(setup_config.namespace, &mut runtime_config).await; + let mut kub_controller = Kubernetes::new(setup_config.namespace, &mut validator_config).await; match kub_controller.namespace_exists().await { Ok(res) => { if !res { diff --git a/k8s-cluster/src/scripts/bootstrap-startup-script.sh b/k8s-cluster/src/scripts/bootstrap-startup-script.sh index ffbd54fc8b7b19..1c96a2918d5df6 100755 --- a/k8s-cluster/src/scripts/bootstrap-startup-script.sh +++ b/k8s-cluster/src/scripts/bootstrap-startup-script.sh @@ -47,67 +47,64 @@ while [[ -n $1 ]]; do elif [[ $1 = --limit-ledger-size ]]; then # Done args+=("$1" "$2") shift 2 - elif [[ $1 = --no-rocksdb-compaction ]]; then + elif [[ $1 = --no-rocksdb-compaction ]]; then # not enabled in net.sh args+=("$1") shift - elif [[ $1 = --enable-rpc-transaction-history ]]; then + elif [[ $1 = --enable-rpc-transaction-history ]]; then # enabled through full-rpc args+=("$1") shift - elif [[ $1 = --rpc-pubsub-enable-block-subscription ]]; then + elif [[ $1 = --rpc-pubsub-enable-block-subscription ]]; then # not enabled in net.sh args+=("$1") shift - elif [[ $1 = --enable-cpi-and-log-storage ]]; then + elif [[ $1 = --enable-cpi-and-log-storage ]]; then # not enabled in net.sh args+=("$1") shift - elif [[ $1 = --enable-extended-tx-metadata-storage ]]; then + elif [[ $1 = --enable-extended-tx-metadata-storage ]]; then # enabled through full-rpc args+=("$1") shift - elif [[ $1 = --enable-rpc-bigtable-ledger-storage ]]; then + elif [[ $1 = --enable-rpc-bigtable-ledger-storage ]]; then # not enabled in net.sh args+=("$1") shift - elif [[ $1 = --tpu-disable-quic ]]; then + elif [[ $1 = --tpu-disable-quic ]]; then # Done args+=("$1") shift - elif [[ $1 = --tpu-enable-udp ]]; then + elif [[ $1 = --tpu-enable-udp ]]; then # Done args+=("$1") shift - elif [[ $1 = --rpc-send-batch-ms ]]; then + elif [[ $1 = --rpc-send-batch-ms ]]; then # not enabled in net.sh args+=("$1" "$2") shift 2 - elif [[ $1 = --rpc-send-batch-size ]]; then + elif [[ $1 = --rpc-send-batch-size ]]; then # not enabled in net.sh args+=("$1" "$2") shift 2 - elif [[ $1 = --skip-poh-verify ]]; then + elif [[ $1 = --skip-poh-verify ]]; then # Done args+=("$1") shift - elif [[ $1 = --log ]]; then - args+=("$1" "$2") - shift 2 - elif [[ $1 = --no-restart ]]; then + elif [[ $1 = --no-restart ]]; then # not enabled in net.sh no_restart=1 shift - elif [[ $1 == --wait-for-supermajority ]]; then + elif [[ $1 == --wait-for-supermajority ]]; then # Done args+=("$1" "$2") shift 2 - elif [[ $1 == --expected-bank-hash ]]; then + elif [[ $1 == --expected-bank-hash ]]; then # Done args+=("$1" "$2") shift 2 elif [[ $1 == --accounts ]]; then args+=("$1" "$2") shift 2 - elif [[ $1 == --maximum-snapshots-to-retain ]]; then + elif [[ $1 == --maximum-snapshots-to-retain ]]; then # not enabled in net.sh args+=("$1" "$2") shift 2 - elif [[ $1 == --no-snapshot-fetch ]]; then + elif [[ $1 == --no-snapshot-fetch ]]; then # Done args+=("$1") shift elif [[ $1 == --accounts-db-skip-shrink ]]; then args+=("$1") shift - elif [[ $1 == --skip-require-tower ]]; then + elif [[ $1 == --skip-require-tower ]]; then # Done maybeRequireTower=false shift - elif [[ $1 = --log-messages-bytes-limit ]]; then + elif [[ $1 = --log-messages-bytes-limit ]]; then # not enabled in net.sh args+=("$1" "$2") shift 2 else