From 6a220fff183478f2570798f687f93aca2e9bc3ac Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 5 Sep 2023 21:15:10 -0700 Subject: [PATCH] add credentials for pushing and pulling from docker repo. some refactor --- net/k8s-cluster/README.md | 3 + net/k8s-cluster/src/docker.rs | 148 ++++++++++++++++++++---------- net/k8s-cluster/src/genesis.rs | 37 ++++---- net/k8s-cluster/src/kubernetes.rs | 67 ++++++++++++-- net/k8s-cluster/src/lib.rs | 5 + net/k8s-cluster/src/main.rs | 141 ++++++++++++++++++++++------ 6 files changed, 297 insertions(+), 104 deletions(-) diff --git a/net/k8s-cluster/README.md b/net/k8s-cluster/README.md index f7e3b96b710669..66d249271bffbd 100644 --- a/net/k8s-cluster/README.md +++ b/net/k8s-cluster/README.md @@ -1,5 +1,8 @@ # Run this thing ``` +export REGISTRY_USERNAME= REGISTRY_PASSWORD= +``` +``` kubectl create ns greg ``` ``` diff --git a/net/k8s-cluster/src/docker.rs b/net/k8s-cluster/src/docker.rs index 1cc8be257a1373..9e06c44848b351 100644 --- a/net/k8s-cluster/src/docker.rs +++ b/net/k8s-cluster/src/docker.rs @@ -1,5 +1,8 @@ use { - crate::{boxed_error, initialize_globals, SOLANA_ROOT}, + crate::{ + boxed_error, initialize_globals, load_env_variable_by_name, new_spinner_progress_bar, + DOCKER_WHALE, SOLANA_ROOT, + }, docker_api::{self, opts, Docker}, log::*, std::{ @@ -16,12 +19,46 @@ const URI_ENV_VAR: &str = "unix:///var/run/docker.sock"; #[derive(Clone, Debug)] pub struct DockerImageConfig<'a> { pub base_image: &'a str, + pub image_name: &'a str, pub tag: &'a str, + pub registry: &'a str, } pub struct DockerConfig<'a> { image_config: DockerImageConfig<'a>, deploy_method: &'a str, + docker: Docker, + registry_username: Option, + registry_password: Option, +} + +fn init_runtime() -> Docker { + let _ = env_logger::try_init(); + if let Ok(uri) = env::var(URI_ENV_VAR) { + Docker::new(uri).unwrap() + } else { + #[cfg(unix)] + { + let uid = nix::unistd::Uid::effective(); + let docker_dir = PathBuf::from(format!("/run/user/{uid}/docker")); + let docker_root_dir = PathBuf::from("/var/run"); + if docker_dir.exists() { + Docker::unix(docker_dir.join("docker.sock")) + } else if docker_root_dir.exists() { + Docker::unix(docker_root_dir.join("docker.sock")) + } else { + panic!( + "Docker socket not found. Tried {URI_ENV_VAR} env variable, {} and {}", + docker_dir.display(), + docker_root_dir.display() + ); + } + } + #[cfg(not(unix))] + { + panic!("Docker socket not found. Try setting the {URI_ENV_VAR} env variable",); + } + } } impl<'a> DockerConfig<'a> { @@ -30,41 +67,27 @@ impl<'a> DockerConfig<'a> { DockerConfig { image_config, deploy_method, + docker: init_runtime(), + registry_username: match load_env_variable_by_name("REGISTRY_USERNAME") { + Ok(username) => Some(username), + Err(_) => None, + }, + registry_password: match load_env_variable_by_name("REGISTRY_PASSWORD") { + Ok(password) => Some(password), + Err(_) => None, + }, } } - pub fn init_runtime(&self) -> Docker { - let _ = env_logger::try_init(); - if let Ok(uri) = env::var(URI_ENV_VAR) { - Docker::new(uri).unwrap() - } else { - #[cfg(unix)] - { - let uid = nix::unistd::Uid::effective(); - let docker_dir = PathBuf::from(format!("/run/user/{uid}/docker")); - let docker_root_dir = PathBuf::from("/var/run"); - if docker_dir.exists() { - Docker::unix(docker_dir.join("docker.sock")) - } else if docker_root_dir.exists() { - Docker::unix(docker_root_dir.join("docker.sock")) - } else { - panic!( - "Docker socket not found. Tried {URI_ENV_VAR} env variable, {} and {}", - docker_dir.display(), - docker_root_dir.display() - ); - } - } - #[cfg(not(unix))] - { - panic!("Docker socket not found. Try setting the {URI_ENV_VAR} env variable",); - } + pub fn registry_credentials_set(&self) -> bool { + if self.registry_username.is_none() || self.registry_username.is_none() { + return false; } + true } pub async fn build_image(&self, validator_type: &str) -> Result<(), Box> { - let docker = self.init_runtime(); - match self.create_base_image(&docker, validator_type).await { + match self.create_base_image(validator_type).await { Ok(res) => { if res.status.success() { info!("Successfully created base Image"); @@ -78,24 +101,8 @@ impl<'a> DockerConfig<'a> { }; } - pub async fn create_base_image( - &self, - docker: &Docker, - validator_type: &str, - ) -> Result> { - let tag = format!("{}-{}", validator_type, self.image_config.tag); - - let images = docker.images(); - let _ = images - .get(tag.as_str()) - .remove( - &opts::ImageRemoveOpts::builder() - .force(true) - .noprune(true) - .build(), - ) - .await; - + pub async fn create_base_image(&self, validator_type: &str) -> Result> { + let image_name = format!("{}-{}", validator_type, self.image_config.image_name); let docker_path = SOLANA_ROOT.join(format!("{}/{}", "docker-build", validator_type)); let dockerfile_path = match self.create_dockerfile(validator_type, docker_path, None) { @@ -112,8 +119,8 @@ impl<'a> DockerConfig<'a> { let dockerfile = dockerfile_path.join("Dockerfile"); let context_path = SOLANA_ROOT.display().to_string(); let command = format!( - "docker build -t {}/{} -f {:?} {}", - "gregcusack", tag, dockerfile, context_path + "docker build -t {}/{}:{} -f {:?} {}", + self.image_config.registry, image_name, self.image_config.tag, dockerfile, context_path ); match Command::new("sh") .arg("-c") @@ -189,6 +196,49 @@ WORKDIR /home/solana .expect("saved Dockerfile"); Ok(docker_path) } + + pub async fn push_image(&self, validator_type: &str) -> Result<(), Box> { + let username = match &self.registry_username { + Some(username) => username, + None => { + return Err(boxed_error!( + "No username set for registry! Is REGISTRY_USERNAME set?" + )) + } + }; + let password = match &self.registry_password { + Some(password) => password, + None => { + return Err(boxed_error!( + "No password set for registry! Is REGISTRY_PASSWORD set?" + )) + } + }; + + // self.docker + let image = format!( + "{}/{}-{}", + self.image_config.registry, validator_type, self.image_config.image_name + ); + let auth = opts::RegistryAuth::Password { + username: password.to_string(), + password: username.to_string(), + email: None, + server_address: None, + }; + + let options = opts::ImagePushOpts::builder() + .tag(self.image_config.tag) + .auth(auth) + .build(); + let progress_bar = new_spinner_progress_bar(); + progress_bar.set_message(format!("{DOCKER_WHALE}Pushing image {} to registry", image)); + + match self.docker.images().push(image, &options).await { + Ok(res) => Ok(res), + Err(err) => Err(boxed_error!(format!("{}", err))), + } + } } // RUN apt install -y iputils-ping curl vim bzip2 psmisc \ diff --git a/net/k8s-cluster/src/genesis.rs b/net/k8s-cluster/src/genesis.rs index d002a3ed71b4de..9f901ca8c4cb6a 100644 --- a/net/k8s-cluster/src/genesis.rs +++ b/net/k8s-cluster/src/genesis.rs @@ -18,22 +18,13 @@ use { native_token::sol_to_lamports, poh_config::PohConfig, rent::Rent, - signature::{ - keypair_from_seed, write_keypair, write_keypair_file, Keypair, - Signer, - }, + signature::{keypair_from_seed, write_keypair, write_keypair_file, Keypair, Signer}, stake::state::StakeStateV2, system_program, timing, }, solana_stake_program::stake_state, solana_vote_program::vote_state::{self, VoteState}, - std::{ - error::Error, - fs::File, - io::Read, - path::PathBuf, - time::Duration, - }, + std::{error::Error, fs::File, io::Read, path::PathBuf, time::Duration}, }; pub const DEFAULT_WORD_COUNT: usize = 12; @@ -106,9 +97,9 @@ impl<'a> Genesis<'a> { } pub fn generate_accounts( - &mut self, - validator_type: &str, - number_of_accounts: i32 + &mut self, + validator_type: &str, + number_of_accounts: i32, ) -> Result<(), Box> { let mut filename_prefix = "validator".to_string(); if validator_type == "bootstrap" { @@ -116,18 +107,25 @@ impl<'a> Genesis<'a> { } else if validator_type == "validator" { filename_prefix = "validator".to_string(); } else { - return Err(boxed_error!(format!("Invalid validator type: {}", validator_type))); + return Err(boxed_error!(format!( + "Invalid validator type: {}", + validator_type + ))); } for i in 0..number_of_accounts { self.generate_account(validator_type, filename_prefix.as_str(), i)?; } - Ok(()) } - fn generate_account(&mut self, validator_type: &str, filename_prefix: &str, i: i32) -> Result<(), Box> { + fn generate_account( + &mut self, + validator_type: &str, + filename_prefix: &str, + i: i32, + ) -> Result<(), Box> { let account_types = vec!["identity", "vote-account", "stake-account"]; let mut identity: Option = None; let mut vote: Option = None; @@ -139,7 +137,10 @@ impl<'a> Genesis<'a> { } else if validator_type == "validator" { filename = format!("{}-{}-{}.json", filename_prefix, account, i); } else { - return Err(boxed_error!(format!("Invalid validator type: {}", validator_type))); + return Err(boxed_error!(format!( + "Invalid validator type: {}", + validator_type + ))); } let outfile = self.config_dir.join(filename); diff --git a/net/k8s-cluster/src/kubernetes.rs b/net/k8s-cluster/src/kubernetes.rs index 69d4787fea497d..f39c338e512b4a 100644 --- a/net/k8s-cluster/src/kubernetes.rs +++ b/net/k8s-cluster/src/kubernetes.rs @@ -1,15 +1,16 @@ use { - crate::{boxed_error, ValidatorType}, + crate::{boxed_error, load_env_variable_by_name, ValidatorType}, k8s_openapi::{ api::{ apps::v1::{ReplicaSet, ReplicaSetSpec}, core::v1::{ - ConfigMap, ConfigMapVolumeSource, Container, EnvVar, EnvVarSource, Namespace, - ObjectFieldSelector, PodSpec, PodTemplateSpec, Service, ServicePort, ServiceSpec, - Volume, VolumeMount, + ConfigMap, ConfigMapVolumeSource, Container, EnvVar, EnvVarSource, + LocalObjectReference, Namespace, ObjectFieldSelector, PodSpec, PodTemplateSpec, + Secret, Service, ServicePort, ServiceSpec, Volume, VolumeMount, }, }, apimachinery::pkg::apis::meta::v1::LabelSelector, + ByteString, }, kube::{ api::{Api, ObjectMeta, PostParams}, @@ -95,7 +96,7 @@ impl<'a> Kubernetes<'a> { } } - pub fn create_bootstrap_validator_replicas_set( + pub async fn create_bootstrap_validator_replicas_set( &self, container_name: &str, image_name: &str, @@ -128,9 +129,10 @@ impl<'a> Kubernetes<'a> { &command, config_map_name, ) + .await } - fn create_replicas_set( + async fn create_replicas_set( &self, app_name: &str, label_selector: &BTreeMap, @@ -171,13 +173,18 @@ impl<'a> Kubernetes<'a> { containers: vec![Container { name: container_name.to_string(), image: Some(image_name.to_string()), - image_pull_policy: Some("Never".to_string()), // Set the image pull policy to "Never" + image_pull_policy: Some("Always".to_string()), // Set the image pull policy to "Never" env: Some(env_vars), command: Some(command.clone()), volume_mounts: Some(vec![volume_mount]), + ..Default::default() }], volumes: Some(vec![volume]), + image_pull_secrets: Some(vec![LocalObjectReference { + name: Some("dockerhub-login".to_string()), + ..Default::default() + }]), ..Default::default() }), ..Default::default() @@ -204,6 +211,44 @@ impl<'a> Kubernetes<'a> { }) } + //TODO: this duplicates code seen in docker.rs -> loading registry info. + // could be ok if we don't build docker and want to just pull from registry! + pub async fn create_secret(&self) -> Result> { + let username = match load_env_variable_by_name("REGISTRY_USERNAME") { + Ok(username) => username, + Err(_) => return Err(boxed_error!("REGISTRY_USERNAME not set")), + }; + let password = match load_env_variable_by_name("REGISTRY_PASSWORD") { + Ok(password) => password, + Err(_) => return Err(boxed_error!("REGISTRY_PASSWORD not set")), + }; + let secret_name = "dockerhub-login"; + let mut data = BTreeMap::new(); + data.insert( + "username".to_string(), + ByteString(username.as_bytes().to_vec()), + ); + data.insert( + "password".to_string(), + ByteString(password.as_bytes().to_vec()), + ); + + let secret = Secret { + metadata: ObjectMeta { + name: Some(secret_name.to_string()), + ..Default::default() + }, + data: Some(data), + ..Default::default() + }; + Ok(secret) + } + + pub async fn deploy_secret(&self, secret: &Secret) -> Result { + let secrets_api: Api = Api::namespaced(self.client.clone(), self.namespace); + secrets_api.create(&PostParams::default(), &secret).await + } + pub async fn deploy_replicas_set( &self, replica_set: &ReplicaSet, @@ -266,7 +311,10 @@ impl<'a> Kubernetes<'a> { service_api.create(&post_params, &service).await } - pub async fn check_replica_set_ready(&self, replica_set_name: &str) -> Result { + pub async fn check_replica_set_ready( + &self, + replica_set_name: &str, + ) -> Result { let replica_sets: Api = Api::namespaced(self.client.clone(), self.namespace); let replica_set = replica_sets.get(replica_set_name).await?; @@ -281,7 +329,7 @@ impl<'a> Kubernetes<'a> { Ok(available_validators >= desired_validators) } - pub fn create_validator_replicas_set( + pub async fn create_validator_replicas_set( &self, container_name: &str, image_name: &str, @@ -336,6 +384,7 @@ impl<'a> Kubernetes<'a> { &command, config_map_name, ) + .await } pub fn create_validator_service(&self) -> Service { diff --git a/net/k8s-cluster/src/lib.rs b/net/k8s-cluster/src/lib.rs index 0348e5529c6a29..917c320ddc3078 100644 --- a/net/k8s-cluster/src/lib.rs +++ b/net/k8s-cluster/src/lib.rs @@ -54,6 +54,10 @@ macro_rules! boxed_error { }; } +pub fn load_env_variable_by_name(name: &str) -> Result { + env::var(name) +} + // pub fn get_rust_flags() -> &'static str { // // env::var("RUSTFLAGS").ok().unwrap_or_default()) // match env::var("RUSTFLAGS").ok() { @@ -67,6 +71,7 @@ macro_rules! boxed_error { static TRUCK: Emoji = Emoji("🚚 ", ""); static PACKAGE: Emoji = Emoji("📦 ", ""); +static DOCKER_WHALE: Emoji = Emoji("🐳 ", ""); /// Creates a new process bar for processing that will take an unknown amount of time pub fn new_spinner_progress_bar() -> ProgressBar { diff --git a/net/k8s-cluster/src/main.rs b/net/k8s-cluster/src/main.rs index d69e81132807ae..b3b6aa1b9e9968 100644 --- a/net/k8s-cluster/src/main.rs +++ b/net/k8s-cluster/src/main.rs @@ -1,5 +1,5 @@ use { - clap::{crate_description, crate_name, value_t_or_exit, App, Arg, ArgMatches, value_t}, + clap::{crate_description, crate_name, value_t, value_t_or_exit, App, Arg, ArgMatches}, log::*, solana_k8s_cluster::{ docker::{DockerConfig, DockerImageConfig}, @@ -42,6 +42,7 @@ fn parse_matches() -> ArgMatches<'static> { .long("bootstrap-container") .takes_value(true) .required(true) + .default_value("bootstrap-container") .help("Bootstrap Validator Container name"), ) .arg( @@ -56,6 +57,7 @@ fn parse_matches() -> ArgMatches<'static> { .long("validator-container") .takes_value(true) .required(true) + .default_value("validator-container") .help("Validator Container name"), ) .arg( @@ -99,19 +101,36 @@ fn parse_matches() -> ArgMatches<'static> { .long("docker-build") .help("Build Docker images. If not set, will assume local docker image should be used"), ) + .arg( + Arg::with_name("registry_name") + .long("registry") + .takes_value(true) + .requires("docker_build") + .help("Registry to push docker image to"), + ) + .arg( + Arg::with_name("image_name") + .long("image-name") + .takes_value(true) + .requires("docker_build") + .help("Docker image name. Will be prepended with validator_type (bootstrap or validator)"), + ) .arg( Arg::with_name("base_image") .long("base-image") .takes_value(true) .default_value("ubuntu:20.04") + .requires("docker_build") .help("Docker base image"), ) .arg( Arg::with_name("image_tag") - .long("image-tag") + .long("tag") .takes_value(true) - .default_value("k8s-cluster-image") - .help("Docker image tag. tag will be prepended with `bootstrap-` and `validator-` to distinguish between both images"), + // .default_value("k8s-cluster-image") + .default_value("latest") + .requires("docker_build") + .help("Docker image tag."), ) .arg( Arg::with_name("prebuild_genesis") @@ -165,7 +184,9 @@ async fn main() { // Download validator version and Build docker image let docker_image_config = DockerImageConfig { base_image: matches.value_of("base_image").unwrap_or_default(), + image_name: matches.value_of("image_name").unwrap(), tag: matches.value_of("image_tag").unwrap_or_default(), + registry: matches.value_of("registry_name").unwrap(), }; let deploy = Deploy::new(build_config.clone()); @@ -179,18 +200,44 @@ async fn main() { if build_config.docker_build { let docker = DockerConfig::new(docker_image_config, build_config.deploy_method); + // just exit if we don't have credentials set + if !docker.registry_credentials_set() { + error!( + "Registry Credentials not set. Are REGISTRY_USERNAME and REGISTRY_PASSWORD set?" + ); + return; + } let image_types = vec!["bootstrap", "validator"]; for image_type in image_types { match docker.build_image(image_type).await { - Ok(_) => info!("Validator Docker image built successfully"), + Ok(_) => info!("Docker image built successfully"), Err(err) => { error!("Exiting........ {}", err); return; } } } + + // Need to push image to registry so Monogon nodes can pull image from registry to local + match docker.push_image("bootstrap").await { + Ok(_) => info!("Bootstrap Image pushed successfully to registry"), + Err(err) => { + error!("{}", err); + return; + } + } + + // Need to push image to registry so Monogon nodes can pull image from registry to local + match docker.push_image("validator").await { + Ok(_) => info!("Validator Image pushed successfully to registry"), + Err(err) => { + error!("{}", err); + return; + } + } } + info!("Creating Genesis"); let mut genesis = Genesis::new(setup_config.clone()); // genesis.generate(); match genesis.generate_faucet() { @@ -266,29 +313,46 @@ async fn main() { let bootstrap_container_name = matches .value_of("bootstrap_container_name") - .expect("Bootstrap container name is required"); + .unwrap_or_default(); let bootstrap_image_name = matches .value_of("bootstrap_image_name") .expect("Bootstrap image name is required"); let validator_container_name = matches .value_of("validator_container_name") - .expect("Validator container name is required"); + .unwrap_or_default(); let validator_image_name = matches .value_of("validator_image_name") .expect("Validator image name is required"); + let secret = match kub_controller.create_secret().await { + Ok(secret) => secret, + Err(err) => { + error!("Failed to create secret! {}", err); + return; + } + }; + match kub_controller.deploy_secret(&secret).await { + Ok(_) => (), + Err(err) => { + error!("{}", err); + return; + } + } + kub_controller.create_selector( &ValidatorType::Bootstrap, "app.kubernetes.io/name", "bootstrap-validator", ); - let bootstrap_replica_set = match kub_controller.create_bootstrap_validator_replicas_set( - bootstrap_container_name, - bootstrap_image_name, - BOOTSTRAP_VALIDATOR_REPLICAS, - config_map.metadata.name.clone(), - - ) { + let bootstrap_replica_set = match kub_controller + .create_bootstrap_validator_replicas_set( + bootstrap_container_name, + bootstrap_image_name, + BOOTSTRAP_VALIDATOR_REPLICAS, + config_map.metadata.name.clone(), + ) + .await + { Ok(replica_set) => replica_set, Err(err) => { error!("Error creating bootstrap validator replicas_set: {}", err); @@ -308,7 +372,7 @@ async fn main() { "Error! Failed to deploy bootstrap validator replicas_set. err: {:?}", err ); - err.to_string() //TODO: fix this, should handle this error better. shoudn't just return a string. should exit or something better + return; } }; @@ -337,12 +401,15 @@ async fn main() { "app.kubernetes.io/name", "validator", ); - let validator_replica_set = match kub_controller.create_validator_replicas_set( - validator_container_name, - validator_image_name, - setup_config.num_validators, - config_map.metadata.name, - ) { + let validator_replica_set = match kub_controller + .create_validator_replicas_set( + validator_container_name, + validator_image_name, + setup_config.num_validators, + config_map.metadata.name, + ) + .await + { Ok(replica_set) => replica_set, Err(err) => { error!("Error creating validator replicas_set: {}", err); @@ -350,16 +417,22 @@ async fn main() { } }; - match kub_controller + let validator_rs_name = match kub_controller .deploy_replicas_set(&validator_replica_set) .await { - Ok(_) => info!("validator replicas_set deployed successfully"), - Err(err) => error!( - "Error! Failed to deploy validator replicas_set. err: {:?}", - err - ), - } + Ok(rs) => { + info!("validator replicas_sets deployed successfully"); + rs.metadata.name.unwrap() + } + Err(err) => { + error!( + "Error! Failed to deploy validator replicas_sets. err: {:?}", + err + ); + return; + } + }; let validator_service = kub_controller.create_validator_service(); match kub_controller.deploy_service(&validator_service).await { @@ -367,6 +440,18 @@ async fn main() { Err(err) => error!("Error! Failed to deploy validator service. err: {:?}", err), } + //TODO: handle this return val properly, don't just unwrap + //TODO: not sure this checks for all replica sets + while !kub_controller + .check_replica_set_ready(validator_rs_name.as_str()) + .await + .unwrap() + { + info!("replica set: {} not ready...", validator_rs_name); + thread::sleep(Duration::from_secs(1)); + } + info!("replica set: {} Ready!", rs_name); + let _ = kub_controller .check_service_matching_replica_set("bootstrap-validator") .await;