Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move snapshot output to the rich print system. #1531

Merged
merged 1 commit into from
Aug 8, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/soroban-cli/src/commands/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ impl Root {
Cmd::Events(events) => events.run().await?,
Cmd::Xdr(xdr) => xdr.run()?,
Cmd::Network(network) => network.run().await?,
Cmd::Snapshot(snapshot) => snapshot.run().await?,
Cmd::Snapshot(snapshot) => snapshot.run(&self.global_args).await?,
Cmd::Version(version) => version.run(),
Cmd::Keys(id) => id.run().await?,
Cmd::Tx(tx) => tx.run(&self.global_args).await?,
Expand Down
105 changes: 69 additions & 36 deletions cmd/soroban-cli/src/commands/snapshot/create.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use sha2::{Digest, Sha256};
use soroban_ledger_snapshot::LedgerSnapshot;
use std::{
collections::HashSet,
fs::{self},
fs,
io::{self},
path::PathBuf,
str::FromStr,
Expand All @@ -26,8 +26,9 @@ use stellar_xdr::curr::{
use tokio::fs::OpenOptions;

use crate::{
commands::{config::data, HEADING_RPC},
commands::{config::data, global, HEADING_RPC},
config::{self, locator, network::passphrase},
print,
utils::{get_name_from_stellar_asset_contract_storage, parsing::parse_asset},
};

Expand Down Expand Up @@ -140,18 +141,20 @@ const CHECKPOINT_FREQUENCY: u32 = 64;

impl Cmd {
#[allow(clippy::too_many_lines)]
pub async fn run(&self) -> Result<(), Error> {
pub async fn run(&self, global_args: &global::Args) -> Result<(), Error> {
let print = print::Print::new(global_args.quiet);
let start = Instant::now();

let archive_url = self.archive_url()?;
let history = get_history(&archive_url, self.ledger).await?;
let history = get_history(&print, &archive_url, self.ledger).await?;

let ledger = history.current_ledger;
let network_passphrase = &history.network_passphrase;
let network_id = Sha256::digest(network_passphrase);
println!("ℹ️ Ledger: {ledger}");
println!("ℹ️ Network Passphrase: {network_passphrase}");
println!("ℹ️ Network ID: {}", hex::encode(network_id));

print.infoln(format!("Ledger: {ledger}"));
print.infoln(format!("Network Passphrase: {network_passphrase}"));
print.infoln(format!("Network id: {}", hex::encode(network_id)));

// Prepare a flat list of buckets to read. They'll be ordered by their
// level so that they can iterated higher level to lower level.
Expand All @@ -164,7 +167,7 @@ impl Cmd {

// Pre-cache the buckets.
for (i, bucket) in buckets.iter().enumerate() {
cache_bucket(&archive_url, i, bucket).await?;
cache_bucket(&print, &archive_url, i, bucket).await?;
}

// The snapshot is what will be written to file at the end. Fields will
Expand Down Expand Up @@ -215,29 +218,35 @@ impl Cmd {
wasm_hashes: self.wasm_hashes.iter().cloned().collect(),
};
let mut next = SearchInputs::default();

loop {
if current.is_empty() {
break;
}
println!(
"ℹ️ Searching for {} accounts, {} contracts, {} wasms",

print.infoln(format!(
"Searching for {} accounts, {} contracts, {} wasms",
current.account_ids.len(),
current.contract_ids.len(),
current.wasm_hashes.len(),
);
));

for (i, bucket) in buckets.iter().enumerate() {
// Defined where the bucket will be read from, either from cache on
// disk, or streamed from the archive.
let cache_path = cache_bucket(&archive_url, i, bucket).await?;
let cache_path = cache_bucket(&print, &archive_url, i, bucket).await?;
let file = std::fs::OpenOptions::new()
.read(true)
.open(&cache_path)
.map_err(Error::ReadOpeningCachedBucket)?;
print!("🔎 Searching bucket {i} {bucket}");

let message = format!("Searching bucket {i} {bucket}");
print.search(format!("{message}…"));

if let Ok(metadata) = file.metadata() {
print!(" ({})", ByteSize(metadata.len()));
print.clear_line();
leighmcculloch marked this conversation as resolved.
Show resolved Hide resolved
print.searchln(format!("{message} ({})", ByteSize(metadata.len())));
}
println!();

// Stream the bucket entries from the bucket, identifying
// entries that match the filters, and including only the
Expand Down Expand Up @@ -288,10 +297,10 @@ impl Cmd {
}) => {
if !current.wasm_hashes.contains(hash) {
next.wasm_hashes.insert(hash.clone());
println!(
"ℹ️ Adding wasm {} to search",
print.infoln(format!(
"Adding wasm {} to search",
hex::encode(hash)
);
));
}
}
ScVal::ContractInstance(ScContractInstance {
Expand All @@ -312,9 +321,9 @@ impl Cmd {
Some(a12.issuer.clone())
}
} {
println!(
"ℹ️ Adding asset issuer {issuer} to search"
);
print.infoln(format!(
"Adding asset issuer {issuer} to search"
));
next.account_ids.insert(issuer);
}
}
Expand All @@ -332,7 +341,7 @@ impl Cmd {
count_saved += 1;
}
if count_saved > 0 {
println!("ℹ️ Found {count_saved} entries");
print.infoln(format!("Found {count_saved} entries"));
}
}
current = next;
Expand All @@ -343,14 +352,14 @@ impl Cmd {
snapshot
.write_file(&self.out)
.map_err(Error::WriteLedgerSnapshot)?;
println!(
"💾 Saved {} entries to {:?}",
print.saveln(format!(
"Saved {} entries to {:?}",
snapshot.ledger_entries.len(),
self.out
);
));

let duration = Duration::from_secs(start.elapsed().as_secs());
println!("Completed in {}", format_duration(duration));
print.checkln(format!("Completed in {}", format_duration(duration)));

Ok(())
}
Expand Down Expand Up @@ -380,7 +389,11 @@ impl Cmd {
}
}

async fn get_history(archive_url: &Uri, ledger: Option<u32>) -> Result<History, Error> {
async fn get_history(
print: &print::Print,
archive_url: &Uri,
ledger: Option<u32>,
) -> Result<History, Error> {
let archive_url = archive_url.to_string();
let archive_url = archive_url.strip_suffix('/').unwrap_or(&archive_url);
let history_url = if let Some(ledger) = ledger {
Expand All @@ -394,34 +407,44 @@ async fn get_history(archive_url: &Uri, ledger: Option<u32>) -> Result<History,
};
let history_url = Uri::from_str(&history_url).unwrap();

println!("🌎 Downloading history {history_url}");
print.globe(format!("Downloading history {history_url}"));

let https = hyper_tls::HttpsConnector::new();
let response = hyper::Client::builder()
.build::<_, hyper::Body>(https)
.get(history_url)
.get(history_url.clone())
.await
.map_err(Error::DownloadingHistory)?;

if !response.status().is_success() {
// Check ledger is a checkpoint ledger and available in archives.
if let Some(ledger) = ledger {
let ledger_offset = (ledger + 1) % CHECKPOINT_FREQUENCY;

if ledger_offset != 0 {
println!(
"ℹ️ Ledger {ledger} may not be a checkpoint ledger, try {} or {}",
print.println("");
print.errorln(format!(
"Ledger {ledger} may not be a checkpoint ledger, try {} or {}",
ledger - ledger_offset,
ledger + (CHECKPOINT_FREQUENCY - ledger_offset),
);
));
}
}
return Err(Error::DownloadingHistoryGotStatusCode(response.status()));
}

let body = hyper::body::to_bytes(response.into_body())
.await
.map_err(Error::ReadHistoryHttpStream)?;

print.clear_line();
print.globeln(format!("Downloaded history {}", &history_url));

serde_json::from_slice::<History>(&body).map_err(Error::JsonDecodingHistory)
}

async fn cache_bucket(
print: &print::Print,
archive_url: &Uri,
bucket_index: usize,
bucket: &str,
Expand All @@ -434,26 +457,36 @@ async fn cache_bucket(
let bucket_2 = &bucket[4..=5];
let bucket_url =
format!("{archive_url}/bucket/{bucket_0}/{bucket_1}/{bucket_2}/bucket-{bucket}.xdr.gz");
print!("🪣 Downloading bucket {bucket_index} {bucket}");

print.globe(format!("Downloading bucket {bucket_index} {bucket}…"));

let bucket_url = Uri::from_str(&bucket_url).map_err(Error::ParsingBucketUrl)?;
let https = hyper_tls::HttpsConnector::new();
let response = hyper::Client::builder()
.build::<_, hyper::Body>(https)
.get(bucket_url)
.await
.map_err(Error::GettingBucket)?;

if !response.status().is_success() {
println!();
print.println("");
return Err(Error::GettingBucketGotStatusCode(response.status()));
}

if let Some(val) = response.headers().get("Content-Length") {
if let Ok(str) = val.to_str() {
if let Ok(len) = str.parse::<u64>() {
print!(" ({})", ByteSize(len));
print.clear_line();
print.globe(format!(
"Downloaded bucket {bucket_index} {bucket} ({})",
ByteSize(len)
));
}
}
}
println!();

print.println("");

let read = response
.into_body()
.map(|result| result.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)))
Expand Down
6 changes: 4 additions & 2 deletions cmd/soroban-cli/src/commands/snapshot/mod.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
use clap::Parser;

use super::global;

pub mod create;

/// Create and operate on ledger snapshots.
Expand All @@ -15,9 +17,9 @@ pub enum Error {
}

impl Cmd {
pub async fn run(&self) -> Result<(), Error> {
pub async fn run(&self, global_args: &global::Args) -> Result<(), Error> {
match self {
Cmd::Create(cmd) => cmd.run().await?,
Cmd::Create(cmd) => cmd.run(global_args).await?,
};
Ok(())
}
Expand Down
20 changes: 20 additions & 0 deletions cmd/soroban-cli/src/print.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,26 @@ impl Print {
Print { quiet }
}

pub fn print<T: Display + Sized>(&self, message: T) {
if !self.quiet {
print!("{message}");
}
}

pub fn println<T: Display + Sized>(&self, message: T) {
if !self.quiet {
println!("{message}");
}
}

pub fn clear_line(&self) {
if cfg!(windows) {
print!("\r");
} else {
print!("\r\x1b[2K");
}
}
leighmcculloch marked this conversation as resolved.
Show resolved Hide resolved

/// # Errors
///
/// Might return an error
Expand Down
Loading