From c061774207084f1f63e6c8f0e9956844ad96a92a Mon Sep 17 00:00:00 2001 From: Kevin Heavey Date: Thu, 7 Nov 2024 03:55:52 +0400 Subject: [PATCH] extract file-download crate to remove solana-runtime dep from cargo-build-sbf (#3460) * extract file-downloader crate * replace download-utils with file-downloader in cargo-build-sbf * remove duplicate definition of DownloadProgressRecord * unused import * remove unnecessary allow * rename to solana-file-download Co-authored-by: Jon C * update paths after rename * move to sdk/file-download * update lock files --------- Co-authored-by: Jon C --- Cargo.lock | 16 ++- Cargo.toml | 2 + download-utils/Cargo.toml | 4 +- download-utils/src/lib.rs | 223 +------------------------------ programs/sbf/Cargo.lock | 14 +- sdk/cargo-build-sbf/Cargo.toml | 2 +- sdk/cargo-build-sbf/src/main.rs | 2 +- sdk/file-download/Cargo.toml | 19 +++ sdk/file-download/src/lib.rs | 226 ++++++++++++++++++++++++++++++++ 9 files changed, 276 insertions(+), 232 deletions(-) create mode 100644 sdk/file-download/Cargo.toml create mode 100644 sdk/file-download/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index cc437e886f238b..1bee15388c9c2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6290,7 +6290,7 @@ dependencies = [ "reqwest", "semver 1.0.23", "serial_test", - "solana-download-utils", + "solana-file-download", "solana-logger", "solana-sdk", "tar", @@ -6791,10 +6791,8 @@ dependencies = [ name = "solana-download-utils" version = "2.2.0" dependencies = [ - "console", - "indicatif", "log", - "reqwest", + "solana-file-download", "solana-runtime", "solana-sdk", ] @@ -6931,6 +6929,16 @@ dependencies = [ "solana-program", ] +[[package]] +name = "solana-file-download" +version = "2.2.0" +dependencies = [ + "console", + "indicatif", + "log", + "reqwest", +] + [[package]] name = "solana-frozen-abi" version = "2.2.0" diff --git a/Cargo.toml b/Cargo.toml index 992250ce885fb4..0979c3061618a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -115,6 +115,7 @@ members = [ "sdk/feature-set", "sdk/fee-calculator", "sdk/fee-structure", + "sdk/file-download", "sdk/frozen-abi", "sdk/frozen-abi/macro", "sdk/gen-headers", @@ -444,6 +445,7 @@ solana-fee-structure = { path = "sdk/fee-structure", version = "=2.2.0" } solana-frozen-abi = { path = "sdk/frozen-abi", version = "=2.2.0" } solana-frozen-abi-macro = { path = "sdk/frozen-abi/macro", version = "=2.2.0" } solana-tps-client = { path = "tps-client", version = "=2.2.0" } +solana-file-download = { path = "sdk/file-download", version = "=2.2.0" } solana-genesis = { path = "genesis", version = "=2.2.0" } solana-genesis-utils = { path = "genesis-utils", version = "=2.2.0" } agave-geyser-plugin-interface = { path = "geyser-plugin-interface", version = "=2.2.0" } diff --git a/download-utils/Cargo.toml b/download-utils/Cargo.toml index 9321eb4c88db92..ef714a6fb2ee2c 100644 --- a/download-utils/Cargo.toml +++ b/download-utils/Cargo.toml @@ -10,10 +10,8 @@ license = { workspace = true } edition = { workspace = true } [dependencies] -console = { workspace = true } -indicatif = { workspace = true } log = { workspace = true } -reqwest = { workspace = true, features = ["blocking", "brotli", "deflate", "gzip", "rustls-tls", "json"] } +solana-file-download = { workspace = true } solana-runtime = { workspace = true } solana-sdk = { workspace = true } diff --git a/download-utils/src/lib.rs b/download-utils/src/lib.rs index 17d50d31b55628..a0a717ac7f86a0 100644 --- a/download-utils/src/lib.rs +++ b/download-utils/src/lib.rs @@ -1,8 +1,7 @@ -#![allow(clippy::arithmetic_side_effects)] +pub use solana_file_download::DownloadProgressRecord; use { - console::Emoji, - indicatif::{ProgressBar, ProgressStyle}, log::*, + solana_file_download::{download_file, DownloadProgressCallbackOption}, solana_runtime::{ snapshot_hash::SnapshotHash, snapshot_package::SnapshotKind, @@ -10,229 +9,13 @@ use { }, solana_sdk::{clock::Slot, genesis_config::DEFAULT_GENESIS_ARCHIVE}, std::{ - fs::{self, File}, - io::{self, Read}, + fs, net::SocketAddr, num::NonZeroUsize, path::{Path, PathBuf}, - time::{Duration, Instant}, }, }; -static TRUCK: Emoji = Emoji("🚚 ", ""); -static SPARKLE: Emoji = Emoji("✨ ", ""); - -/// Creates a new process bar for processing that will take an unknown amount of time -fn new_spinner_progress_bar() -> ProgressBar { - let progress_bar = ProgressBar::new(42); - progress_bar.set_style( - ProgressStyle::default_spinner() - .template("{spinner:.green} {wide_msg}") - .expect("ProgresStyle::template direct input to be correct"), - ); - progress_bar.enable_steady_tick(Duration::from_millis(100)); - progress_bar -} - -/// Structure modeling information about download progress -#[derive(Debug)] -pub struct DownloadProgressRecord { - // Duration since the beginning of the download - pub elapsed_time: Duration, - // Duration since the the last notification - pub last_elapsed_time: Duration, - // the bytes/sec speed measured for the last notification period - pub last_throughput: f32, - // the bytes/sec speed measured from the beginning - pub total_throughput: f32, - // total bytes of the download - pub total_bytes: usize, - // bytes downloaded so far - pub current_bytes: usize, - // percentage downloaded - pub percentage_done: f32, - // Estimated remaining time (in seconds) to finish the download if it keeps at the the last download speed - pub estimated_remaining_time: f32, - // The times of the progress is being notified, it starts from 1 and increments by 1 each time - pub notification_count: u64, -} - -type DownloadProgressCallback<'a> = Box bool + 'a>; -type DownloadProgressCallbackOption<'a> = Option>; - -/// This callback allows the caller to get notified of the download progress modelled by DownloadProgressRecord -/// Return "true" to continue the download -/// Return "false" to abort the download -pub fn download_file<'a, 'b>( - url: &str, - destination_file: &Path, - use_progress_bar: bool, - progress_notify_callback: &'a mut DownloadProgressCallbackOption<'b>, -) -> Result<(), String> { - if destination_file.is_file() { - return Err(format!("{destination_file:?} already exists")); - } - let download_start = Instant::now(); - - fs::create_dir_all(destination_file.parent().expect("parent")) - .map_err(|err| err.to_string())?; - - let mut temp_destination_file = destination_file.to_path_buf(); - temp_destination_file.set_file_name(format!( - "tmp-{}", - destination_file - .file_name() - .expect("file_name") - .to_str() - .expect("to_str") - )); - - let progress_bar = new_spinner_progress_bar(); - if use_progress_bar { - progress_bar.set_message(format!("{TRUCK}Downloading {url}...")); - } - - let response = reqwest::blocking::Client::new() - .get(url) - .send() - .and_then(|response| response.error_for_status()) - .map_err(|err| { - progress_bar.finish_and_clear(); - err.to_string() - })?; - - let download_size = { - response - .headers() - .get(reqwest::header::CONTENT_LENGTH) - .and_then(|content_length| content_length.to_str().ok()) - .and_then(|content_length| content_length.parse().ok()) - .unwrap_or(0) - }; - - if use_progress_bar { - progress_bar.set_length(download_size); - progress_bar.set_style( - ProgressStyle::default_bar() - .template( - "{spinner:.green}{msg_wide}[{bar:40.cyan/blue}] {bytes}/{total_bytes} ({eta})", - ) - .expect("ProgresStyle::template direct input to be correct") - .progress_chars("=> "), - ); - progress_bar.set_message(format!("{TRUCK}Downloading~ {url}")); - } else { - info!("Downloading {} bytes from {}", download_size, url); - } - - struct DownloadProgress<'e, 'f, R> { - progress_bar: ProgressBar, - response: R, - last_print: Instant, - current_bytes: usize, - last_print_bytes: usize, - download_size: f32, - use_progress_bar: bool, - start_time: Instant, - callback: &'f mut DownloadProgressCallbackOption<'e>, - notification_count: u64, - } - - impl<'e, 'f, R: Read> Read for DownloadProgress<'e, 'f, R> { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - let n = self.response.read(buf)?; - - self.current_bytes += n; - let total_bytes_f32 = self.current_bytes as f32; - let diff_bytes_f32 = (self.current_bytes - self.last_print_bytes) as f32; - let last_throughput = diff_bytes_f32 / self.last_print.elapsed().as_secs_f32(); - let estimated_remaining_time = if last_throughput > 0_f32 { - (self.download_size - self.current_bytes as f32) / last_throughput - } else { - f32::MAX - }; - - let mut progress_record = DownloadProgressRecord { - elapsed_time: self.start_time.elapsed(), - last_elapsed_time: self.last_print.elapsed(), - last_throughput, - total_throughput: self.current_bytes as f32 - / self.start_time.elapsed().as_secs_f32(), - total_bytes: self.download_size as usize, - current_bytes: self.current_bytes, - percentage_done: 100f32 * (total_bytes_f32 / self.download_size), - estimated_remaining_time, - notification_count: self.notification_count, - }; - let mut to_update_progress = false; - if progress_record.last_elapsed_time.as_secs() > 5 { - self.last_print = Instant::now(); - self.last_print_bytes = self.current_bytes; - to_update_progress = true; - self.notification_count += 1; - progress_record.notification_count = self.notification_count - } - - if self.use_progress_bar { - self.progress_bar.inc(n as u64); - } else if to_update_progress { - info!( - "downloaded {} bytes {:.1}% {:.1} bytes/s", - self.current_bytes, - progress_record.percentage_done, - progress_record.last_throughput, - ); - } - - if let Some(callback) = self.callback { - if to_update_progress && !callback(&progress_record) { - info!("Download is aborted by the caller"); - return Err(io::Error::new( - io::ErrorKind::Other, - "Download is aborted by the caller", - )); - } - } - - Ok(n) - } - } - - let mut source = DownloadProgress::<'b, 'a> { - progress_bar, - response, - last_print: Instant::now(), - current_bytes: 0, - last_print_bytes: 0, - download_size: (download_size as f32).max(1f32), - use_progress_bar, - start_time: Instant::now(), - callback: progress_notify_callback, - notification_count: 0, - }; - - File::create(&temp_destination_file) - .and_then(|mut file| std::io::copy(&mut source, &mut file)) - .map_err(|err| format!("Unable to write {temp_destination_file:?}: {err:?}"))?; - - source.progress_bar.finish_and_clear(); - info!( - " {}{}", - SPARKLE, - format!( - "Downloaded {} ({} bytes) in {:?}", - url, - download_size, - Instant::now().duration_since(download_start), - ) - ); - - std::fs::rename(temp_destination_file, destination_file) - .map_err(|err| format!("Unable to rename: {err:?}"))?; - - Ok(()) -} - pub fn download_genesis_if_missing( rpc_addr: &SocketAddr, genesis_package: &Path, diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 498441efb8766f..f33328a6375dbb 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -5442,10 +5442,8 @@ dependencies = [ name = "solana-download-utils" version = "2.2.0" dependencies = [ - "console", - "indicatif", "log", - "reqwest", + "solana-file-download", "solana-runtime", "solana-sdk", ] @@ -5553,6 +5551,16 @@ dependencies = [ "solana-program", ] +[[package]] +name = "solana-file-download" +version = "2.2.0" +dependencies = [ + "console", + "indicatif", + "log", + "reqwest", +] + [[package]] name = "solana-genesis-utils" version = "2.2.0" diff --git a/sdk/cargo-build-sbf/Cargo.toml b/sdk/cargo-build-sbf/Cargo.toml index 36ce44d0d5c7d3..baa8008c3ba454 100644 --- a/sdk/cargo-build-sbf/Cargo.toml +++ b/sdk/cargo-build-sbf/Cargo.toml @@ -18,7 +18,7 @@ log = { workspace = true, features = ["std"] } regex = { workspace = true } reqwest = { workspace = true, features = ["blocking", "rustls-tls"] } semver = { workspace = true } -solana-download-utils = { workspace = true } +solana-file-download = { workspace = true } solana-logger = { workspace = true } solana-sdk = { workspace = true } tar = { workspace = true } diff --git a/sdk/cargo-build-sbf/src/main.rs b/sdk/cargo-build-sbf/src/main.rs index 03bfce2cabd5dd..744238c2337b7b 100644 --- a/sdk/cargo-build-sbf/src/main.rs +++ b/sdk/cargo-build-sbf/src/main.rs @@ -5,7 +5,7 @@ use { itertools::Itertools, log::*, regex::Regex, - solana_download_utils::download_file, + solana_file_download::download_file, solana_sdk::signature::{write_keypair_file, Keypair}, std::{ borrow::Cow, diff --git a/sdk/file-download/Cargo.toml b/sdk/file-download/Cargo.toml new file mode 100644 index 00000000000000..3e471ec6cf3e2e --- /dev/null +++ b/sdk/file-download/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "solana-file-download" +description = "Solana File Download Utility" +documentation = "https://docs.rs/solana-file-download" +version = { workspace = true } +authors = { workspace = true } +repository = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +edition = { workspace = true } + +[dependencies] +console = { workspace = true } +indicatif = { workspace = true } +log = { workspace = true } +reqwest = { workspace = true, features = ["blocking", "brotli", "deflate", "gzip", "rustls-tls", "json"] } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/sdk/file-download/src/lib.rs b/sdk/file-download/src/lib.rs new file mode 100644 index 00000000000000..77fd6c3bff88cc --- /dev/null +++ b/sdk/file-download/src/lib.rs @@ -0,0 +1,226 @@ +#![allow(clippy::arithmetic_side_effects)] +use { + console::Emoji, + indicatif::{ProgressBar, ProgressStyle}, + log::*, + std::{ + fs::{self, File}, + io::{self, Read}, + path::Path, + time::{Duration, Instant}, + }, +}; + +static TRUCK: Emoji = Emoji("🚚 ", ""); +static SPARKLE: Emoji = Emoji("✨ ", ""); + +/// Creates a new process bar for processing that will take an unknown amount of time +fn new_spinner_progress_bar() -> ProgressBar { + let progress_bar = ProgressBar::new(42); + progress_bar.set_style( + ProgressStyle::default_spinner() + .template("{spinner:.green} {wide_msg}") + .expect("ProgresStyle::template direct input to be correct"), + ); + progress_bar.enable_steady_tick(Duration::from_millis(100)); + progress_bar +} + +/// Structure modeling information about download progress +#[derive(Debug)] +pub struct DownloadProgressRecord { + // Duration since the beginning of the download + pub elapsed_time: Duration, + // Duration since the the last notification + pub last_elapsed_time: Duration, + // the bytes/sec speed measured for the last notification period + pub last_throughput: f32, + // the bytes/sec speed measured from the beginning + pub total_throughput: f32, + // total bytes of the download + pub total_bytes: usize, + // bytes downloaded so far + pub current_bytes: usize, + // percentage downloaded + pub percentage_done: f32, + // Estimated remaining time (in seconds) to finish the download if it keeps at the the last download speed + pub estimated_remaining_time: f32, + // The times of the progress is being notified, it starts from 1 and increments by 1 each time + pub notification_count: u64, +} + +type DownloadProgressCallback<'a> = Box bool + 'a>; +pub type DownloadProgressCallbackOption<'a> = Option>; + +/// This callback allows the caller to get notified of the download progress modelled by DownloadProgressRecord +/// Return "true" to continue the download +/// Return "false" to abort the download +pub fn download_file<'a, 'b>( + url: &str, + destination_file: &Path, + use_progress_bar: bool, + progress_notify_callback: &'a mut DownloadProgressCallbackOption<'b>, +) -> Result<(), String> { + if destination_file.is_file() { + return Err(format!("{destination_file:?} already exists")); + } + let download_start = Instant::now(); + + fs::create_dir_all(destination_file.parent().expect("parent")) + .map_err(|err| err.to_string())?; + + let mut temp_destination_file = destination_file.to_path_buf(); + temp_destination_file.set_file_name(format!( + "tmp-{}", + destination_file + .file_name() + .expect("file_name") + .to_str() + .expect("to_str") + )); + + let progress_bar = new_spinner_progress_bar(); + if use_progress_bar { + progress_bar.set_message(format!("{TRUCK}Downloading {url}...")); + } + + let response = reqwest::blocking::Client::new() + .get(url) + .send() + .and_then(|response| response.error_for_status()) + .map_err(|err| { + progress_bar.finish_and_clear(); + err.to_string() + })?; + + let download_size = { + response + .headers() + .get(reqwest::header::CONTENT_LENGTH) + .and_then(|content_length| content_length.to_str().ok()) + .and_then(|content_length| content_length.parse().ok()) + .unwrap_or(0) + }; + + if use_progress_bar { + progress_bar.set_length(download_size); + progress_bar.set_style( + ProgressStyle::default_bar() + .template( + "{spinner:.green}{msg_wide}[{bar:40.cyan/blue}] {bytes}/{total_bytes} ({eta})", + ) + .expect("ProgresStyle::template direct input to be correct") + .progress_chars("=> "), + ); + progress_bar.set_message(format!("{TRUCK}Downloading~ {url}")); + } else { + info!("Downloading {} bytes from {}", download_size, url); + } + + struct DownloadProgress<'e, 'f, R> { + progress_bar: ProgressBar, + response: R, + last_print: Instant, + current_bytes: usize, + last_print_bytes: usize, + download_size: f32, + use_progress_bar: bool, + start_time: Instant, + callback: &'f mut DownloadProgressCallbackOption<'e>, + notification_count: u64, + } + + impl<'e, 'f, R: Read> Read for DownloadProgress<'e, 'f, R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let n = self.response.read(buf)?; + + self.current_bytes += n; + let total_bytes_f32 = self.current_bytes as f32; + let diff_bytes_f32 = (self.current_bytes - self.last_print_bytes) as f32; + let last_throughput = diff_bytes_f32 / self.last_print.elapsed().as_secs_f32(); + let estimated_remaining_time = if last_throughput > 0_f32 { + (self.download_size - self.current_bytes as f32) / last_throughput + } else { + f32::MAX + }; + + let mut progress_record = DownloadProgressRecord { + elapsed_time: self.start_time.elapsed(), + last_elapsed_time: self.last_print.elapsed(), + last_throughput, + total_throughput: self.current_bytes as f32 + / self.start_time.elapsed().as_secs_f32(), + total_bytes: self.download_size as usize, + current_bytes: self.current_bytes, + percentage_done: 100f32 * (total_bytes_f32 / self.download_size), + estimated_remaining_time, + notification_count: self.notification_count, + }; + let mut to_update_progress = false; + if progress_record.last_elapsed_time.as_secs() > 5 { + self.last_print = Instant::now(); + self.last_print_bytes = self.current_bytes; + to_update_progress = true; + self.notification_count += 1; + progress_record.notification_count = self.notification_count + } + + if self.use_progress_bar { + self.progress_bar.inc(n as u64); + } else if to_update_progress { + info!( + "downloaded {} bytes {:.1}% {:.1} bytes/s", + self.current_bytes, + progress_record.percentage_done, + progress_record.last_throughput, + ); + } + + if let Some(callback) = self.callback { + if to_update_progress && !callback(&progress_record) { + info!("Download is aborted by the caller"); + return Err(io::Error::new( + io::ErrorKind::Other, + "Download is aborted by the caller", + )); + } + } + + Ok(n) + } + } + + let mut source = DownloadProgress::<'b, 'a> { + progress_bar, + response, + last_print: Instant::now(), + current_bytes: 0, + last_print_bytes: 0, + download_size: (download_size as f32).max(1f32), + use_progress_bar, + start_time: Instant::now(), + callback: progress_notify_callback, + notification_count: 0, + }; + + File::create(&temp_destination_file) + .and_then(|mut file| std::io::copy(&mut source, &mut file)) + .map_err(|err| format!("Unable to write {temp_destination_file:?}: {err:?}"))?; + + source.progress_bar.finish_and_clear(); + info!( + " {}{}", + SPARKLE, + format!( + "Downloaded {} ({} bytes) in {:?}", + url, + download_size, + Instant::now().duration_since(download_start), + ) + ); + + std::fs::rename(temp_destination_file, destination_file) + .map_err(|err| format!("Unable to rename: {err:?}"))?; + + Ok(()) +}