Skip to content

Commit

Permalink
Merge pull request solana-labs#30 from gregcusack/k8s-cluster-officia…
Browse files Browse the repository at this point in the history 
…l-args

add in additional args for validator config
  • Loading branch information
Greg Cusack authored Oct 11, 2023
2 parents 341e832 + d3ba7fc commit d05782e
Show file tree
Hide file tree
Showing 6 changed files with 154 additions and 56 deletions.
1 change: 1 addition & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions k8s-cluster/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ solana-bloom = { workspace = true }
solana-clap-utils = { workspace = true }
solana-clap-v3-utils = { workspace = true }
solana-client = { workspace = true }
solana-core = { workspace = true }
solana-entry = { workspace = true }
solana-frozen-abi = { workspace = true }
solana-frozen-abi-macro = { workspace = true }
Expand Down
4 changes: 4 additions & 0 deletions k8s-cluster/src/genesis.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,10 @@ impl Genesis {
let keypair = generate_keypair()?;
self.all_pubkeys.push(keypair.pubkey());

if account == "identity" {
info!("identity: {:?}", keypair.pubkey());
}

if let Some(outfile) = outfile.to_str() {
output_keypair(&keypair, outfile)
.map_err(|err| format!("Unable to write {outfile}: {err}"))?;
Expand Down
81 changes: 60 additions & 21 deletions k8s-cluster/src/kubernetes.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,91 +22,130 @@ use {
std::{collections::BTreeMap, error::Error, fs::File, io::Read},
};

pub struct RuntimeConfig<'a> {
pub enable_udp: bool,
pub disable_quic: bool,
pub struct ValidatorConfig<'a> {
pub tpu_enable_udp: bool,
pub tpu_disable_quic: bool,
pub gpu_mode: &'a str, // TODO: this is not implemented yet
pub internal_node_sol: f64,
pub internal_node_stake_sol: f64,
pub wait_for_supermajority: Option<u64>,
pub warp_slot: Option<u64>,
pub shred_version: Option<u16>,
pub bank_hash: Option<Hash>,
pub max_ledger_size: Option<u64>,
pub skip_poh_verify: bool,
pub no_snapshot_fetch: bool,
pub skip_require_tower: bool,
pub enable_full_rpc: bool,
}

impl<'a> std::fmt::Display for RuntimeConfig<'a> {
impl<'a> std::fmt::Display for ValidatorConfig<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"Runtime Config\n\
enable_udp: {}\n\
disable_quic: {}\n\
tpu_enable_udp: {}\n\
tpu_disable_quic: {}\n\
gpu_mode: {}\n\
internal_node_sol: {}\n\
internal_node_stake_sol: {}\n\
wait_for_supermajority: {:?}\n\
warp_slot: {:?}\n\
shred_version: {:?}\n\
bank_hash: {:?}",
self.enable_udp,
self.disable_quic,
bank_hash: {:?}\n\
max_ledger_size: {:?}\n\
skip_poh_verify: {}\n\
no_snapshot_fetch: {}\n\
skip_require_tower: {}\n\
enable_full_rpc: {}",
self.tpu_enable_udp,
self.tpu_disable_quic,
self.gpu_mode,
self.internal_node_sol,
self.internal_node_stake_sol,
self.wait_for_supermajority,
self.warp_slot,
self.shred_version,
self.bank_hash,
self.max_ledger_size,
self.skip_poh_verify,
self.no_snapshot_fetch,
self.skip_require_tower,
self.enable_full_rpc,
)
}
}

pub struct Kubernetes<'a> {
client: Client,
namespace: &'a str,
runtime_config: &'a mut RuntimeConfig<'a>,
validator_config: &'a mut ValidatorConfig<'a>,
}

impl<'a> Kubernetes<'a> {
pub async fn new(
namespace: &'a str,
runtime_config: &'a mut RuntimeConfig<'a>,
validator_config: &'a mut ValidatorConfig<'a>,
) -> Kubernetes<'a> {
Kubernetes {
client: Client::try_default().await.unwrap(),
namespace,
runtime_config,
validator_config,
}
}

pub fn set_shred_version(&mut self, shred_version: u16) {
self.runtime_config.shred_version = Some(shred_version);
self.validator_config.shred_version = Some(shred_version);
}

pub fn set_bank_hash(&mut self, bank_hash: Hash) {
self.runtime_config.bank_hash = Some(bank_hash);
self.validator_config.bank_hash = Some(bank_hash);
}

fn generate_command_flags(&mut self) -> Vec<String> {
let mut flags = Vec::new();

if self.runtime_config.enable_udp {
if self.validator_config.tpu_enable_udp {
flags.push("--tpu-enable-udp".to_string());
}
if self.runtime_config.disable_quic {
if self.validator_config.tpu_disable_quic {
flags.push("--tpu-disable-quic".to_string());
}
if self.validator_config.skip_poh_verify {
flags.push("--skip-poh-verify".to_string());
}
if self.validator_config.no_snapshot_fetch {
flags.push("--no-snapshot-fetch".to_string());
}
if self.validator_config.skip_require_tower {
flags.push("--skip-require-tower".to_string());
}
if self.validator_config.enable_full_rpc {
flags.push("--enable-rpc-transaction-history".to_string());
flags.push("--enable-extended-tx-metadata-storage".to_string());
}
if self.validator_config.enable_full_rpc {
flags.push("--enable-rpc-transaction-history".to_string());
flags.push("--enable-extended-tx-metadata-storage".to_string());
}

if let Some(slot) = self.runtime_config.wait_for_supermajority {
if let Some(slot) = self.validator_config.wait_for_supermajority {
flags.push("--wait-for-supermajority".to_string());
flags.push(slot.to_string());
}

if let Some(bank_hash) = self.runtime_config.bank_hash {
if let Some(bank_hash) = self.validator_config.bank_hash {
flags.push("--expected-bank-hash".to_string());
flags.push(bank_hash.to_string());
}

if let Some(limit_ledger_size) = self.validator_config.max_ledger_size {
flags.push("--limit-ledger-size".to_string());
flags.push(limit_ledger_size.to_string());
}



flags
}

Expand All @@ -118,11 +157,11 @@ impl<'a> Kubernetes<'a> {
let mut flags = self.generate_command_flags();

flags.push("--internal-node-stake-sol".to_string());
flags.push(self.runtime_config.internal_node_stake_sol.to_string());
flags.push(self.validator_config.internal_node_stake_sol.to_string());
flags.push("--internal-node-sol".to_string());
flags.push(self.runtime_config.internal_node_sol.to_string());
flags.push(self.validator_config.internal_node_sol.to_string());

if let Some(shred_version) = self.runtime_config.shred_version {
if let Some(shred_version) = self.validator_config.shred_version {
flags.push("--expected-shred-version".to_string());
flags.push(shred_version.to_string());
}
Expand Down
84 changes: 70 additions & 14 deletions k8s-cluster/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,12 @@ use {
DEFAULT_INTERNAL_NODE_STAKE_SOL,
},
initialize_globals,
kubernetes::{Kubernetes, RuntimeConfig},
kubernetes::{Kubernetes, ValidatorConfig},
ledger_helper::LedgerHelper,
release::{BuildConfig, Deploy},
ValidatorType,
},
solana_core::ledger_cleanup_service::{DEFAULT_MAX_LEDGER_SHREDS, DEFAULT_MIN_MAX_LEDGER_SHREDS},
std::{thread, time::Duration},
};

Expand Down Expand Up @@ -189,7 +190,7 @@ fn parse_matches() -> ArgMatches<'static> {
Arg::with_name("internal_node_stake_sol")
.long("internal-node-stake-sol")
.takes_value(true)
.help(" Amount to stake internal nodes (Sol)."),
.help("Amount to stake internal nodes (Sol)."),
)
.arg(
Arg::with_name("enable_warmup_epochs")
Expand Down Expand Up @@ -218,12 +219,12 @@ fn parse_matches() -> ArgMatches<'static> {
.arg(
Arg::with_name("tpu_enable_udp")
.long("tpu-enable-udp")
.help("Runtime config. Enable UDP for tpu transactions."),
.help("Validator config. Enable UDP for tpu transactions."),
)
.arg(
Arg::with_name("tpu_disable_quic")
.long("tpu-disable-quic")
.help("Genesis config. Disable quic for tpu packet forwarding"),
.help("Validator config. Disable quic for tpu packet forwarding"),
)
.arg(
Arg::with_name("gpu_mode")
Expand All @@ -245,6 +246,41 @@ fn parse_matches() -> ArgMatches<'static> {
.takes_value(true)
.help("Boot from a snapshot that has warped ahead to WARP_SLOT rather than a slot 0 genesis"),
)
.arg(
Arg::with_name("limit_ledger_size")
.long("limit-ledger-size")
.takes_value(true)
.help("Validator Config. The `--limit-ledger-size` parameter allows you to specify how many ledger
shreds your node retains on disk. If you do not
include this parameter, the validator will keep the entire ledger until it runs
out of disk space. The default value attempts to keep the ledger disk usage
under 500GB. More or less disk usage may be requested by adding an argument to
`--limit-ledger-size` if desired. Check `solana-validator --help` for the
default limit value used by `--limit-ledger-size`. More information about
selecting a custom limit value is at : https://github.com/solana-labs/solana/blob/583cec922b6107e0f85c7e14cb5e642bc7dfb340/core/src/ledger_cleanup_service.rs#L15-L26"),
)
.arg(
Arg::with_name("skip_poh_verify")
.long("skip-poh-verify")
.help("Validator config. If set, validators will skip verifying
the ledger they already have saved to disk at
boot (results in a much faster boot)"),
)
.arg(
Arg::with_name("no_snapshot_fetch")
.long("no-snapshot-fetch")
.help("Validator config. If set, disables booting validators from a snapshot"),
)
.arg(
Arg::with_name("skip_require_tower")
.long("skip-require-tower")
.help("Validator config. Disable require tower"),
)
.arg(
Arg::with_name("full_rpc")
.long("full-rpc")
.help("Validator config. Support full RPC services on all nodes"),
)
.get_matches()
}

Expand Down Expand Up @@ -321,9 +357,9 @@ async fn main() {
),
};

let mut runtime_config = RuntimeConfig {
enable_udp: matches.is_present("tpu_enable_udp"),
disable_quic: matches.is_present("tpu_disable_quic"),
let mut validator_config = ValidatorConfig {
tpu_enable_udp: matches.is_present("tpu_enable_udp"),
tpu_disable_quic: matches.is_present("tpu_disable_quic"),
gpu_mode: matches.value_of("gpu_mode").unwrap(),
internal_node_sol: matches
.value_of("internal_node_sol")
Expand All @@ -344,15 +380,35 @@ async fn main() {
warp_slot: matches
.value_of("warp_slot")
.map(|value_str| value_str.parse().expect("Invalid value for warp_slot")),

shred_version: None, // set after genesis created
bank_hash: None, // set after genesis created
max_ledger_size: if matches.is_present("limit_ledger_size") {
let limit_ledger_size = match matches.value_of("limit_ledger_size") {
Some(_) => value_t_or_exit!(matches, "limit_ledger_size", u64),
None => DEFAULT_MAX_LEDGER_SHREDS,
};
if limit_ledger_size < DEFAULT_MIN_MAX_LEDGER_SHREDS {
error!(
"The provided --limit-ledger-size value was too small, the minimum value is {DEFAULT_MIN_MAX_LEDGER_SHREDS}"
);
return;
}
Some(limit_ledger_size)
} else {
None
},
skip_poh_verify: matches.is_present("skip_poh_verify"),
no_snapshot_fetch: matches.is_present("no_snapshot_fetch"),
skip_require_tower: matches.is_present("skip_require_tower"),
enable_full_rpc: matches.is_present("enable_full_rpc"),
};

let wait_for_supermajority: Option<u64> = runtime_config.wait_for_supermajority;
let warp_slot: Option<u64> = runtime_config.warp_slot;
let wait_for_supermajority: Option<u64> = validator_config.wait_for_supermajority;
let warp_slot: Option<u64> = validator_config.warp_slot;
if !match (
runtime_config.wait_for_supermajority,
runtime_config.warp_slot,
validator_config.wait_for_supermajority,
validator_config.warp_slot,
) {
(Some(slot1), Some(slot2)) => slot1 == slot2,
(None, None) => true, // Both are None, consider them equal
Expand All @@ -361,14 +417,14 @@ async fn main() {
panic!(
"Error: When specifying both --wait-for-supermajority and --warp-slot, \
they must use the same slot. ({:?} != {:?})",
runtime_config.wait_for_supermajority, runtime_config.warp_slot
validator_config.wait_for_supermajority, validator_config.warp_slot
);
}

info!("Runtime Config: {}", runtime_config);
info!("Runtime Config: {}", validator_config);

// Check if namespace exists
let mut kub_controller = Kubernetes::new(setup_config.namespace, &mut runtime_config).await;
let mut kub_controller = Kubernetes::new(setup_config.namespace, &mut validator_config).await;
match kub_controller.namespace_exists().await {
Ok(res) => {
if !res {
Expand Down
Loading

0 comments on commit d05782e

Please sign in to comment.