Add ability for ledger-tool to dump ledger transactions to CSV

ledger-tool/Cargo.toml

@@ -11,7 +11,9 @@ homepage = "https://solana.com/"
 [dependencies]
 bs58 = "0.3.0"
 clap = "2.33.0"
+csv = "1.1.3"
 histogram = "*"
+serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0.46"
 serde_yaml = "0.8.11"
 solana-clap-utils = { path = "../clap-utils", version = "1.0.25" }

ledger-tool/src/main.rs

@@ -14,9 +14,17 @@ use solana_ledger::{
     snapshot_utils,
 };
 use solana_sdk::{
-    clock::Slot, genesis_config::GenesisConfig, native_token::lamports_to_sol, pubkey::Pubkey,
+    clock::Slot,
+    clock::DEFAULT_TICKS_PER_SECOND,
+    clock::DEFAULT_TICKS_PER_SLOT,
+    genesis_config::GenesisConfig,
+    instruction::CompiledInstruction,
+    native_token::lamports_to_sol, pubkey::Pubkey,
     shred_version::compute_shred_version,
+    program_utils::limited_deserialize,
+    transaction::Transaction,
 };
+use solana_transaction_status::RpcTransactionStatusMeta;
 use solana_vote_program::vote_state::VoteState;
 use std::{
     collections::{BTreeMap, HashMap, HashSet},
@@ -28,13 +36,198 @@ use std::{
     process::{exit, Command, Stdio},
     str::FromStr,
 };
+use csv::Writer;
+
+use serde::{
+    Serialize,
+    Deserialize,
+};
 
 #[derive(PartialEq)]
 enum LedgerOutputMethod {
     Print,
     Json,
 }
 
+#[derive(Serialize, Deserialize, Debug, Default, PartialEq)]
+struct EntryInfo {
+    index: usize,
+    num_hashes: u64,
+    hash: String,
+    num_transactions: usize,
+}
+
+#[derive(Serialize, Deserialize, Debug, Default, PartialEq)]
+struct InstructionInfo {
+    txn_sig: Option<String>,
+    program_pubkey: Option<String>,
+    program_instruction: Option<String>,
+    instruction_accounts: Vec<String>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Default, PartialEq)]
+struct TransactionInfo {
+    slot: Option<Slot>,
+    cluster_unix_timestamp: Option<i64>,
+    recent_blockhash: Option<String>,
+    txn_sig: Option<String>,
+    fee: Option<f64>,
+    num_instructions: Option<usize>,
+    num_accounts: Option<usize>,
+    account_balance_infos: Vec<AccountBalanceInfo>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Default, PartialEq)]
+struct AccountBalanceInfo {
+    account_pubkey: String,
+    account_pre_balance: f64,
+    account_post_balance: f64,
+}
+
+fn output_slot_to_csv(
+    blockstore: &Blockstore,
+    slot: Slot,
+    txn_wtr: &mut Writer<File>,
+    instruction_wtr: &mut Writer<File>,
+    ledger_path: &PathBuf,
+) -> Result<(), String> {
+    let entries = blockstore
+        .get_slot_entries(slot, 0, None)
+        .map_err(|err| format!("Failed to load entries for slot {}: {}", slot, err))?;
+
+    for entry in entries {
+        for transaction in entry.transactions {
+            // Skip any vote transactions
+            let program_pubkey = transaction
+                .message
+                .account_keys[transaction.message.instructions[0].program_id_index as usize];
+            if program_pubkey == solana_vote_program::id() { continue; }
+
+            let transaction_status: Option<RpcTransactionStatusMeta> = blockstore
+                .read_transaction_status((transaction.signatures[0], slot))
+                .unwrap_or_else(|err| {
+                    eprintln!(
+                        "Failed to read transaction status for {} at slot {}: {}",
+                        transaction.signatures[0], slot, err
+                    );
+                    None
+                })
+                .map(|transaction_status| transaction_status.into());
+
+            for instruction in &transaction.message.instructions {
+                let instruction_info = build_instruction_info(
+                    &instruction,
+                    &transaction,
+                    &transaction_status
+                );
+                instruction_wtr.serialize(&instruction_info);
+            }
+
+            let txn_info = build_txn_info(
+                slot,
+                &ledger_path,
+                &transaction,
+                &transaction_status
+            );
+            txn_wtr.serialize(&txn_info);
+        }
+    }
+    instruction_wtr.flush();
+    txn_wtr.flush();
+    Ok(())
+}
+
+fn build_instruction_info(
+    instruction: &CompiledInstruction,
+    transaction: &Transaction,
+    transaction_status: &Option<RpcTransactionStatusMeta>
+) -> InstructionInfo {
+    let program_pubkey = transaction
+        .message
+        .account_keys[instruction.program_id_index as usize];
+
+    let mut instruction_info = InstructionInfo{
+        txn_sig: Some(transaction.signatures[0].to_string()),
+        program_pubkey: Some(bs58::encode(program_pubkey).into_string()),
+        ..Default::default()
+    };
+
+    // Need to figure out how to decode program instruction to String
+    instruction_info.program_instruction = Some("foo".to_string());
+//    instruction_info.program_instruction =
+//                Some(std::str::from_utf8(&instruction.data).unwrap().to_string());
+
+
+//    if program_pubkey == solana_stake_program::id() {
+//        if let Ok(stake_instruction) = limited_deserialize::<
+//            solana_stake_program::stake_instruction::StakeInstruction,
+//        >(&instruction.data)
+//        {
+//            instruction_info.program_instruction =
+//                Some(std::str::from_utf8(&stake_instruction).unwrap().to_string());
+//        }
+//    } else if program_pubkey == solana_sdk::system_program::id() {
+//        if let Ok(system_instruction) = limited_deserialize::<
+//            solana_sdk::system_instruction::SystemInstruction,
+//        >(&instruction.data)
+//        {
+//            instruction_info.program_instruction =
+//                Some(std::str::from_utf8(&system_instruction).unwrap().to_string());
+//        }
+//    }
+
+
+    for account in &instruction.accounts {
+        instruction_info
+            .instruction_accounts
+            .push(bs58::encode(transaction
+                .message
+                .account_keys[*account as usize])
+                .into_string())
+    }
+
+    instruction_info
+}
+
+fn build_txn_info(
+    slot: Slot,
+    ledger_path: &PathBuf,
+    transaction: &Transaction,
+    transaction_status: &Option<RpcTransactionStatusMeta>
+) -> TransactionInfo {
+    let genesis_creation_time = open_genesis_config(&ledger_path).creation_time;
+    let seconds_per_slot = DEFAULT_TICKS_PER_SLOT / DEFAULT_TICKS_PER_SECOND;
+    let seconds_since_genesis = (slot * seconds_per_slot) as i64;
+
+    let mut txn_info = TransactionInfo{
+        slot: Some(slot),
+        cluster_unix_timestamp: Some(genesis_creation_time + seconds_since_genesis),
+        recent_blockhash: Some(transaction.message.recent_blockhash.to_string()),
+        txn_sig: Some(transaction.signatures[0].to_string()),
+        num_instructions: Some(transaction.message.instructions.len()),
+        num_accounts: Some(transaction.message.account_keys.len()),
+        ..Default::default()
+    };
+
+    if let Some(transaction_status) = transaction_status {
+        txn_info.fee = Some(lamports_to_sol(transaction_status.fee));
+
+        for (i, (pre, post)) in transaction_status
+        .pre_balances
+        .iter()
+        .zip(transaction_status.post_balances.iter())
+        .enumerate()
+        {
+            txn_info.account_balance_infos.push(AccountBalanceInfo{
+                account_pubkey: bs58::encode(transaction.message.account_keys[i]).into_string(),
+                account_pre_balance: lamports_to_sol(*pre),
+                account_post_balance: lamports_to_sol(*post),
+            })
+        };
+    }
+    txn_info
+}
+
 fn output_slot_rewards(
     blockstore: &Blockstore,
     slot: Slot,
@@ -112,6 +305,42 @@ fn output_slot(
     output_slot_rewards(blockstore, slot, method)
 }
 
+fn output_ledger_to_csv(blockstore: &Blockstore,
+                        ledger_path: &PathBuf,
+                        starting_slot: Slot,
+                        txn_csv_file: String,
+                        instruction_csv_file: String) {
+    let rooted_slot_iterator =
+        RootedSlotIterator::new(starting_slot, &blockstore).unwrap_or_else(|err| {
+            eprintln!(
+                "Failed to load entries starting from slot {}: {:?}",
+                starting_slot, err
+            );
+            exit(1);
+        });
+
+    let mut txn_wtr = csv::WriterBuilder::new()
+        .has_headers(false)
+        .flexible(true)
+        .from_path(txn_csv_file)
+        .unwrap();
+    let mut instruction_wtr = csv::WriterBuilder::new()
+        .has_headers(false)
+        .flexible(true)
+        .from_path(instruction_csv_file)
+        .unwrap();
+
+    for (slot, slot_meta) in rooted_slot_iterator {
+        output_slot_to_csv(
+            blockstore,
+            slot,
+            &mut txn_wtr,
+            &mut instruction_wtr,
+            ledger_path,
+        );
+    }
+}
+
 fn output_ledger(blockstore: Blockstore, starting_slot: Slot, method: LedgerOutputMethod) {
     let rooted_slot_iterator =
         RootedSlotIterator::new(starting_slot, &blockstore).unwrap_or_else(|err| {
@@ -584,6 +813,18 @@ fn main() {
         .multiple(true)
         .takes_value(true)
         .help("Add a hard fork at this slot");
+    let txn_csv_file_arg = Arg::with_name("txn_csv_file")
+        .short("t")
+        .long("txn-csv-file")
+        .value_name("PATH")
+        .takes_value(true)
+        .help("File path to new or existing CSV file for writing");
+    let instruction_csv_file_arg = Arg::with_name("instruction_csv_file")
+        .short("i")
+        .long("instruction-csv-file")
+        .value_name("PATH")
+        .takes_value(true)
+        .help("File path to new or existing CSV file for writing");
 
     let matches = App::new(crate_name!())
         .about(crate_description!())
@@ -602,6 +843,13 @@ fn main() {
             .about("Print the ledger")
             .arg(&starting_slot_arg)
         )
+        .subcommand(
+            SubCommand::with_name("print-csv")
+                .about("Write the ledger to CSV files")
+                .arg(&starting_slot_arg)
+                .arg(&txn_csv_file_arg)
+                .arg(&instruction_csv_file_arg)
+        )
         .subcommand(
             SubCommand::with_name("slot")
             .about("Print the contents of one or more slots")
@@ -616,6 +864,22 @@ fn main() {
                     .help("Slots to print"),
             )
         )
+        .subcommand(
+            SubCommand::with_name("slot-csv")
+                .about("Print the contents of one or more slots into a CSV file")
+                .arg(
+                    Arg::with_name("slots")
+                        .index(1)
+                        .value_name("SLOTS")
+                        .validator(is_slot)
+                        .takes_value(true)
+                        .multiple(true)
+                        .required(true)
+                        .help("Slots to print"),
+                )
+                .arg(&txn_csv_file_arg)
+                .arg(&instruction_csv_file_arg)
+        )
         .subcommand(
             SubCommand::with_name("set-dead-slot")
             .about("Mark one or more slots dead")
@@ -802,6 +1066,20 @@ fn main() {
                 LedgerOutputMethod::Print,
             );
         }
+        ("print-csv", Some(arg_matches)) => {
+            let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
+            let txn_csv_file = value_t_or_exit!(arg_matches, "txn_csv_file", String);
+            let instruction_csv_file = value_t_or_exit!(arg_matches, "instruction_csv_file", String);
+            let blockstore = open_blockstore(&ledger_path);
+            output_ledger_to_csv(
+                &blockstore,
+                &ledger_path,
+                starting_slot,
+                txn_csv_file,
+                instruction_csv_file,
+            );
+        }
+
         ("genesis", Some(_arg_matches)) => {
             println!("{}", open_genesis_config(&ledger_path));
         }
@@ -845,6 +1123,29 @@ fn main() {
                 }
             }
         }
+        ("slot-csv", Some(arg_matches)) => {
+            let slots = values_t_or_exit!(arg_matches, "slots", Slot);
+            let blockstore = open_blockstore(&ledger_path);
+            let txn_csv_file = value_t_or_exit!(arg_matches, "txn_csv_file", String);
+            let instruction_csv_file = value_t_or_exit!(arg_matches, "instruction_csv_file", String);
+            let mut txn_wtr = csv::WriterBuilder::new()
+                .has_headers(false)
+                .flexible(true)
+                .from_path(txn_csv_file)
+                .unwrap();
+            let mut instruction_wtr = csv::WriterBuilder::new()
+                .has_headers(false)
+                .flexible(true)
+                .from_path(instruction_csv_file)
+                .unwrap();
+
+            for slot in slots {
+                println!("Slot {}", slot);
+                if let Err(err) = output_slot_to_csv(&blockstore, slot, &mut txn_wtr, &mut instruction_wtr, &ledger_path) {
+                    eprintln!("{}", err);
+                }
+            }
+        }
         ("json", Some(arg_matches)) => {
             let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
             output_ledger(