extract file-download crate to remove solana-runtime dep from cargo-b…

…uild-sbf (#3460) * extract file-downloader crate * replace download-utils with file-downloader in cargo-build-sbf * remove duplicate definition of DownloadProgressRecord * unused import * remove unnecessary allow * rename to solana-file-download Co-authored-by: Jon C <[email protected]> * update paths after rename * move to sdk/file-download * update lock files --------- Co-authored-by: Jon C <[email protected]>
anza-xyz · Nov 6, 2024 · c061774 · c061774
1 parent 03f0b00
commit c061774
Show file tree

Hide file tree

Showing 9 changed files with 276 additions and 232 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -115,6 +115,7 @@ members = [
     "sdk/feature-set",
     "sdk/fee-calculator",
     "sdk/fee-structure",
+    "sdk/file-download",
     "sdk/frozen-abi",
     "sdk/frozen-abi/macro",
     "sdk/gen-headers",
@@ -444,6 +445,7 @@ solana-fee-structure = { path = "sdk/fee-structure", version = "=2.2.0" }
 solana-frozen-abi = { path = "sdk/frozen-abi", version = "=2.2.0" }
 solana-frozen-abi-macro = { path = "sdk/frozen-abi/macro", version = "=2.2.0" }
 solana-tps-client = { path = "tps-client", version = "=2.2.0" }
+solana-file-download = { path = "sdk/file-download", version = "=2.2.0" }
 solana-genesis = { path = "genesis", version = "=2.2.0" }
 solana-genesis-utils = { path = "genesis-utils", version = "=2.2.0" }
 agave-geyser-plugin-interface = { path = "geyser-plugin-interface", version = "=2.2.0" }

diff --git a/download-utils/Cargo.toml b/download-utils/Cargo.toml
@@ -10,10 +10,8 @@ license = { workspace = true }
 edition = { workspace = true }
 
 [dependencies]
-console = { workspace = true }
-indicatif = { workspace = true }
 log = { workspace = true }
-reqwest = { workspace = true, features = ["blocking", "brotli", "deflate", "gzip", "rustls-tls", "json"] }
+solana-file-download = { workspace = true }
 solana-runtime = { workspace = true }
 solana-sdk = { workspace = true }
 

diff --git a/download-utils/src/lib.rs b/download-utils/src/lib.rs
@@ -1,238 +1,21 @@
-#![allow(clippy::arithmetic_side_effects)]
+pub use solana_file_download::DownloadProgressRecord;
 use {
-    console::Emoji,
-    indicatif::{ProgressBar, ProgressStyle},
     log::*,
+    solana_file_download::{download_file, DownloadProgressCallbackOption},
     solana_runtime::{
         snapshot_hash::SnapshotHash,
         snapshot_package::SnapshotKind,
         snapshot_utils::{self, ArchiveFormat},
     },
     solana_sdk::{clock::Slot, genesis_config::DEFAULT_GENESIS_ARCHIVE},
     std::{
-        fs::{self, File},
-        io::{self, Read},
+        fs,
         net::SocketAddr,
         num::NonZeroUsize,
         path::{Path, PathBuf},
-        time::{Duration, Instant},
     },
 };
 
-static TRUCK: Emoji = Emoji("🚚 ", "");
-static SPARKLE: Emoji = Emoji("✨ ", "");
-
-/// Creates a new process bar for processing that will take an unknown amount of time
-fn new_spinner_progress_bar() -> ProgressBar {
-    let progress_bar = ProgressBar::new(42);
-    progress_bar.set_style(
-        ProgressStyle::default_spinner()
-            .template("{spinner:.green} {wide_msg}")
-            .expect("ProgresStyle::template direct input to be correct"),
-    );
-    progress_bar.enable_steady_tick(Duration::from_millis(100));
-    progress_bar
-}
-
-/// Structure modeling information about download progress
-#[derive(Debug)]
-pub struct DownloadProgressRecord {
-    // Duration since the beginning of the download
-    pub elapsed_time: Duration,
-    // Duration since the the last notification
-    pub last_elapsed_time: Duration,
-    // the bytes/sec speed measured for the last notification period
-    pub last_throughput: f32,
-    // the bytes/sec speed measured from the beginning
-    pub total_throughput: f32,
-    // total bytes of the download
-    pub total_bytes: usize,
-    // bytes downloaded so far
-    pub current_bytes: usize,
-    // percentage downloaded
-    pub percentage_done: f32,
-    // Estimated remaining time (in seconds) to finish the download if it keeps at the the last download speed
-    pub estimated_remaining_time: f32,
-    // The times of the progress is being notified, it starts from 1 and increments by 1 each time
-    pub notification_count: u64,
-}
-
-type DownloadProgressCallback<'a> = Box<dyn FnMut(&DownloadProgressRecord) -> bool + 'a>;
-type DownloadProgressCallbackOption<'a> = Option<DownloadProgressCallback<'a>>;
-
-/// This callback allows the caller to get notified of the download progress modelled by DownloadProgressRecord
-/// Return "true" to continue the download
-/// Return "false" to abort the download
-pub fn download_file<'a, 'b>(
-    url: &str,
-    destination_file: &Path,
-    use_progress_bar: bool,
-    progress_notify_callback: &'a mut DownloadProgressCallbackOption<'b>,
-) -> Result<(), String> {
-    if destination_file.is_file() {
-        return Err(format!("{destination_file:?} already exists"));
-    }
-    let download_start = Instant::now();
-
-    fs::create_dir_all(destination_file.parent().expect("parent"))
-        .map_err(|err| err.to_string())?;
-
-    let mut temp_destination_file = destination_file.to_path_buf();
-    temp_destination_file.set_file_name(format!(
-        "tmp-{}",
-        destination_file
-            .file_name()
-            .expect("file_name")
-            .to_str()
-            .expect("to_str")
-    ));
-
-    let progress_bar = new_spinner_progress_bar();
-    if use_progress_bar {
-        progress_bar.set_message(format!("{TRUCK}Downloading {url}..."));
-    }
-
-    let response = reqwest::blocking::Client::new()
-        .get(url)
-        .send()
-        .and_then(|response| response.error_for_status())
-        .map_err(|err| {
-            progress_bar.finish_and_clear();
-            err.to_string()
-        })?;
-
-    let download_size = {
-        response
-            .headers()
-            .get(reqwest::header::CONTENT_LENGTH)
-            .and_then(|content_length| content_length.to_str().ok())
-            .and_then(|content_length| content_length.parse().ok())
-            .unwrap_or(0)
-    };
-
-    if use_progress_bar {
-        progress_bar.set_length(download_size);
-        progress_bar.set_style(
-            ProgressStyle::default_bar()
-                .template(
-                    "{spinner:.green}{msg_wide}[{bar:40.cyan/blue}] {bytes}/{total_bytes} ({eta})",
-                )
-                .expect("ProgresStyle::template direct input to be correct")
-                .progress_chars("=> "),
-        );
-        progress_bar.set_message(format!("{TRUCK}Downloading~ {url}"));
-    } else {
-        info!("Downloading {} bytes from {}", download_size, url);
-    }
-
-    struct DownloadProgress<'e, 'f, R> {
-        progress_bar: ProgressBar,
-        response: R,
-        last_print: Instant,
-        current_bytes: usize,
-        last_print_bytes: usize,
-        download_size: f32,
-        use_progress_bar: bool,
-        start_time: Instant,
-        callback: &'f mut DownloadProgressCallbackOption<'e>,
-        notification_count: u64,
-    }
-
-    impl<'e, 'f, R: Read> Read for DownloadProgress<'e, 'f, R> {
-        fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
-            let n = self.response.read(buf)?;
-
-            self.current_bytes += n;
-            let total_bytes_f32 = self.current_bytes as f32;
-            let diff_bytes_f32 = (self.current_bytes - self.last_print_bytes) as f32;
-            let last_throughput = diff_bytes_f32 / self.last_print.elapsed().as_secs_f32();
-            let estimated_remaining_time = if last_throughput > 0_f32 {
-                (self.download_size - self.current_bytes as f32) / last_throughput
-            } else {
-                f32::MAX
-            };
-
-            let mut progress_record = DownloadProgressRecord {
-                elapsed_time: self.start_time.elapsed(),
-                last_elapsed_time: self.last_print.elapsed(),
-                last_throughput,
-                total_throughput: self.current_bytes as f32
-                    / self.start_time.elapsed().as_secs_f32(),
-                total_bytes: self.download_size as usize,
-                current_bytes: self.current_bytes,
-                percentage_done: 100f32 * (total_bytes_f32 / self.download_size),
-                estimated_remaining_time,
-                notification_count: self.notification_count,
-            };
-            let mut to_update_progress = false;
-            if progress_record.last_elapsed_time.as_secs() > 5 {
-                self.last_print = Instant::now();
-                self.last_print_bytes = self.current_bytes;
-                to_update_progress = true;
-                self.notification_count += 1;
-                progress_record.notification_count = self.notification_count
-            }
-
-            if self.use_progress_bar {
-                self.progress_bar.inc(n as u64);
-            } else if to_update_progress {
-                info!(
-                    "downloaded {} bytes {:.1}% {:.1} bytes/s",
-                    self.current_bytes,
-                    progress_record.percentage_done,
-                    progress_record.last_throughput,
-                );
-            }
-
-            if let Some(callback) = self.callback {
-                if to_update_progress && !callback(&progress_record) {
-                    info!("Download is aborted by the caller");
-                    return Err(io::Error::new(
-                        io::ErrorKind::Other,
-                        "Download is aborted by the caller",
-                    ));
-                }
-            }
-
-            Ok(n)
-        }
-    }
-
-    let mut source = DownloadProgress::<'b, 'a> {
-        progress_bar,
-        response,
-        last_print: Instant::now(),
-        current_bytes: 0,
-        last_print_bytes: 0,
-        download_size: (download_size as f32).max(1f32),
-        use_progress_bar,
-        start_time: Instant::now(),
-        callback: progress_notify_callback,
-        notification_count: 0,
-    };
-
-    File::create(&temp_destination_file)
-        .and_then(|mut file| std::io::copy(&mut source, &mut file))
-        .map_err(|err| format!("Unable to write {temp_destination_file:?}: {err:?}"))?;
-
-    source.progress_bar.finish_and_clear();
-    info!(
-        "  {}{}",
-        SPARKLE,
-        format!(
-            "Downloaded {} ({} bytes) in {:?}",
-            url,
-            download_size,
-            Instant::now().duration_since(download_start),
-        )
-    );
-
-    std::fs::rename(temp_destination_file, destination_file)
-        .map_err(|err| format!("Unable to rename: {err:?}"))?;
-
-    Ok(())
-}
-
 pub fn download_genesis_if_missing(
     rpc_addr: &SocketAddr,
     genesis_package: &Path,

diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock
diff --git a/sdk/cargo-build-sbf/Cargo.toml b/sdk/cargo-build-sbf/Cargo.toml
@@ -18,7 +18,7 @@ log = { workspace = true, features = ["std"] }
 regex = { workspace = true }
 reqwest = { workspace = true, features = ["blocking", "rustls-tls"] }
 semver = { workspace = true }
-solana-download-utils = { workspace = true }
+solana-file-download = { workspace = true }
 solana-logger = { workspace = true }
 solana-sdk = { workspace = true }
 tar = { workspace = true }

diff --git a/sdk/cargo-build-sbf/src/main.rs b/sdk/cargo-build-sbf/src/main.rs
@@ -5,7 +5,7 @@ use {
     itertools::Itertools,
     log::*,
     regex::Regex,
-    solana_download_utils::download_file,
+    solana_file_download::download_file,
     solana_sdk::signature::{write_keypair_file, Keypair},
     std::{
         borrow::Cow,

diff --git a/sdk/file-download/Cargo.toml b/sdk/file-download/Cargo.toml
@@ -0,0 +1,19 @@
+[package]
+name = "solana-file-download"
+description = "Solana File Download Utility"
+documentation = "https://docs.rs/solana-file-download"
+version = { workspace = true }
+authors = { workspace = true }
+repository = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+edition = { workspace = true }
+
+[dependencies]
+console = { workspace = true }
+indicatif = { workspace = true }
+log = { workspace = true }
+reqwest = { workspace = true, features = ["blocking", "brotli", "deflate", "gzip", "rustls-tls", "json"] }
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]