Merge pull request solana-labs#17 from gregcusack/solana-k8s-cluster-…

…genesis Solana k8s cluster genesis
ryleung-solana · Sep 6, 2023 · 585bd0b · 585bd0b
2 parents da8ee55 + 94973ab
commit 585bd0b
Show file tree

Hide file tree

Showing 6 changed files with 272 additions and 179 deletions.
diff --git a/net/k8s-cluster/README.md b/net/k8s-cluster/README.md
@@ -1,4 +1,9 @@
 # Run this thing
+Login to Docker for pushing/pulling repos. we assume auth for registryies are already setup.
+```
+docker login
+```
+
 ```
 kubectl create ns greg
 ```

diff --git a/net/k8s-cluster/src/docker.rs b/net/k8s-cluster/src/docker.rs
@@ -1,22 +1,22 @@
 use {
-    crate::{boxed_error, initialize_globals, SOLANA_ROOT},
-    docker_api::{self, opts, Docker},
+    crate::{
+        boxed_error, initialize_globals, load_env_variable_by_name, SOLANA_ROOT,
+    },
     log::*,
     std::{
-        env,
         error::Error,
         fs,
         path::PathBuf,
         process::{Command, Output, Stdio},
     },
 };
 
-const URI_ENV_VAR: &str = "unix:///var/run/docker.sock";
-
 #[derive(Clone, Debug)]
 pub struct DockerImageConfig<'a> {
     pub base_image: &'a str,
+    pub image_name: &'a str,
     pub tag: &'a str,
+    pub registry: &'a str,
 }
 
 pub struct DockerConfig<'a> {
@@ -33,38 +33,8 @@ impl<'a> DockerConfig<'a> {
         }
     }
 
-    pub fn init_runtime(&self) -> Docker {
-        let _ = env_logger::try_init();
-        if let Ok(uri) = env::var(URI_ENV_VAR) {
-            Docker::new(uri).unwrap()
-        } else {
-            #[cfg(unix)]
-            {
-                let uid = nix::unistd::Uid::effective();
-                let docker_dir = PathBuf::from(format!("/run/user/{uid}/docker"));
-                let docker_root_dir = PathBuf::from("/var/run");
-                if docker_dir.exists() {
-                    Docker::unix(docker_dir.join("docker.sock"))
-                } else if docker_root_dir.exists() {
-                    Docker::unix(docker_root_dir.join("docker.sock"))
-                } else {
-                    panic!(
-                        "Docker socket not found. Tried {URI_ENV_VAR} env variable, {} and {}",
-                        docker_dir.display(),
-                        docker_root_dir.display()
-                    );
-                }
-            }
-            #[cfg(not(unix))]
-            {
-                panic!("Docker socket not found. Try setting the {URI_ENV_VAR} env variable",);
-            }
-        }
-    }
-
     pub async fn build_image(&self, validator_type: &str) -> Result<(), Box<dyn Error>> {
-        let docker = self.init_runtime();
-        match self.create_base_image(&docker, validator_type).await {
+        match self.create_base_image(validator_type).await {
             Ok(res) => {
                 if res.status.success() {
                     info!("Successfully created base Image");
@@ -78,24 +48,8 @@ impl<'a> DockerConfig<'a> {
         };
     }
 
-    pub async fn create_base_image(
-        &self,
-        docker: &Docker,
-        validator_type: &str,
-    ) -> Result<Output, Box<dyn Error>> {
-        let tag = format!("{}-{}", validator_type, self.image_config.tag);
-
-        let images = docker.images();
-        let _ = images
-            .get(tag.as_str())
-            .remove(
-                &opts::ImageRemoveOpts::builder()
-                    .force(true)
-                    .noprune(true)
-                    .build(),
-            )
-            .await;
-
+    pub async fn create_base_image(&self, validator_type: &str) -> Result<Output, Box<dyn Error>> {
+        let image_name = format!("{}-{}", validator_type, self.image_config.image_name);
         let docker_path = SOLANA_ROOT.join(format!("{}/{}", "docker-build", validator_type));
 
         let dockerfile_path = match self.create_dockerfile(validator_type, docker_path, None) {
@@ -112,8 +66,8 @@ impl<'a> DockerConfig<'a> {
         let dockerfile = dockerfile_path.join("Dockerfile");
         let context_path = SOLANA_ROOT.display().to_string();
         let command = format!(
-            "docker build -t {} -f {:?} {}",
-            tag, dockerfile, context_path
+            "docker build -t {}/{}:{} -f {:?} {}",
+            self.image_config.registry, image_name, self.image_config.tag, dockerfile, context_path
         );
         match Command::new("sh")
             .arg("-c")
@@ -189,8 +143,35 @@ WORKDIR /home/solana
         .expect("saved Dockerfile");
         Ok(docker_path)
     }
+
+    pub async fn push_image(&self, validator_type: &str) -> Result<(), Box<dyn Error>> {
+        let image = format!(
+            "{}/{}-{}:{}",
+            self.image_config.registry, validator_type, self.image_config.image_name, self.image_config.tag
+        );
+
+        let command = format!(
+            "docker push '{}'",
+            image
+        );
+        let output = Command::new("sh")
+            .arg("-c")
+            .arg(&command)
+            .stdout(Stdio::inherit())
+            .stderr(Stdio::inherit())
+            .spawn()
+            .expect("Failed to execute command")
+            .wait_with_output()
+            .expect("Failed to push image");
+
+        if !output.status.success() {
+            return Err(boxed_error!(output.status.to_string()));
+        }
+        Ok(())
+    }
+
 }
 
 // RUN apt install -y iputils-ping curl vim bzip2 psmisc \
 // iproute2 software-properties-common apt-transport-https \
-// ca-certificates openssh-client openssh-server
+// ca-certificates openssh-client openssh-server
diff --git a/net/k8s-cluster/src/genesis.rs b/net/k8s-cluster/src/genesis.rs
@@ -5,9 +5,7 @@ use {
         Engine as _,
     },
     bip39::{Language, Mnemonic, MnemonicType, Seed},
-    clap::ArgMatches,
     log::*,
-    solana_accounts_db::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
     solana_clap_v3_utils::{input_parsers::STDOUT_OUTFILE_TOKEN, keygen, keypair},
     solana_entry::poh::{compute_hashes_per_tick, Poh},
     solana_genesis::genesis_accounts::add_genesis_accounts,
@@ -19,10 +17,9 @@ use {
         genesis_config::{ClusterType, GenesisConfig, DEFAULT_GENESIS_FILE},
         native_token::sol_to_lamports,
         poh_config::PohConfig,
-        pubkey::Pubkey,
         rent::Rent,
         signature::{
-            keypair_from_seed, read_keypair_file, write_keypair, write_keypair_file, Keypair,
+            keypair_from_seed, write_keypair, write_keypair_file, Keypair,
             Signer,
         },
         stake::state::StakeStateV2,
@@ -33,9 +30,8 @@ use {
     std::{
         error::Error,
         fs::File,
-        io::{Read, Write},
+        io::Read,
         path::PathBuf,
-        process,
         time::Duration,
     },
 };
@@ -339,7 +335,10 @@ impl<'a> Genesis<'a> {
 mod tests {
     use {
         super::*,
-        solana_sdk::signature::{Keypair, Signer},
+        solana_sdk::{
+            signature::{Keypair, Signer},
+            pubkey::Pubkey,
+        },
         std::path::PathBuf,
     };
     fn make_tmp_path(name: &str) -> PathBuf {