From d40bb382f49004709a3651cda92cd2801abac642 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Fri, 3 Sep 2021 21:05:30 -0600 Subject: [PATCH 1/2] Demote write locks on transaction program ids (#19593) * Add feature * Demote write lock on program ids * Fixup bpf tests * Update MappedMessage::is_writable * Comma nit * Review comments (cherry picked from commit decec3cd8b27dca05814ac884f6d61c23acd3131) # Conflicts: # core/src/banking_stage.rs # core/src/cost_model.rs # core/src/cost_tracker.rs # ledger-tool/src/main.rs # program-runtime/src/instruction_processor.rs # programs/bpf/tests/programs.rs # programs/bpf_loader/src/syscalls.rs # rpc/src/transaction_status_service.rs # runtime/src/accounts.rs # runtime/src/bank.rs # runtime/src/message_processor.rs # sdk/benches/serialize_instructions.rs # sdk/program/src/message/mapped.rs # sdk/program/src/message/sanitized.rs # sdk/src/transaction/sanitized.rs --- cli-output/src/display.rs | 2 +- core/src/banking_stage.rs | 91 ++ core/src/cost_model.rs | 504 +++++++ core/src/cost_tracker.rs | 423 ++++++ ledger-tool/src/main.rs | 72 + program-runtime/src/instruction_processor.rs | 1242 ++++++++++++++++++ program-test/src/lib.rs | 7 +- programs/bpf/tests/programs.rs | 9 +- programs/bpf_loader/src/syscalls.rs | 18 +- rpc/src/transaction_status_service.rs | 6 + runtime/src/accounts.rs | 146 +- runtime/src/bank.rs | 60 + runtime/src/message_processor.rs | 24 +- sdk/benches/serialize_instructions.rs | 18 +- sdk/program/src/message.rs | 21 +- sdk/program/src/message/mapped.rs | 289 ++++ sdk/program/src/message/sanitized.rs | 597 +++++++++ sdk/src/feature_set.rs | 5 + sdk/src/transaction/sanitized.rs | 231 ++++ transaction-status/src/parse_accounts.rs | 2 +- 20 files changed, 3736 insertions(+), 31 deletions(-) create mode 100644 core/src/cost_model.rs create mode 100644 core/src/cost_tracker.rs create mode 100644 program-runtime/src/instruction_processor.rs create mode 100644 sdk/program/src/message/mapped.rs create mode 100644 sdk/program/src/message/sanitized.rs create mode 100644 sdk/src/transaction/sanitized.rs diff --git a/cli-output/src/display.rs b/cli-output/src/display.rs index f8f7e8274a1694..b44af327c00382 100644 --- a/cli-output/src/display.rs +++ b/cli-output/src/display.rs @@ -140,7 +140,7 @@ fn format_account_mode(message: &Message, index: usize) -> String { } else { "-" }, - if message.is_writable(index) { + if message.is_writable(index, /*demote_program_write_locks=*/ true) { "w" // comment for consistent rust fmt (no joking; lol) } else { "-" diff --git a/core/src/banking_stage.rs b/core/src/banking_stage.rs index 3215bc67f57695..007021f7d6a353 100644 --- a/core/src/banking_stage.rs +++ b/core/src/banking_stage.rs @@ -965,8 +965,19 @@ impl BankingStage { msgs: &Packets, transaction_indexes: &[usize], libsecp256k1_0_5_upgrade_enabled: bool, +<<<<<<< HEAD ) -> (Vec>, Vec) { transaction_indexes +======= + libsecp256k1_fail_on_bad_count: bool, + cost_tracker: &Arc>, + banking_stage_stats: &BankingStageStats, + demote_program_write_locks: bool, + ) -> (Vec, Vec, Vec) { + let mut retryable_transaction_packet_indexes: Vec = vec![]; + + let verified_transactions_with_packet_indexes: Vec<_> = transaction_indexes +>>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) .iter() .filter_map(|tx_index| { let p = &msgs.packets[*tx_index]; @@ -980,7 +991,44 @@ impl BankingStage { tx_index, )) }) +<<<<<<< HEAD .unzip() +======= + .collect(); + banking_stage_stats.cost_tracker_check_count.fetch_add( + verified_transactions_with_packet_indexes.len(), + Ordering::Relaxed, + ); + + let mut cost_tracker_check_time = Measure::start("cost_tracker_check_time"); + let (filtered_transactions, filter_transaction_packet_indexes) = { + let cost_tracker_readonly = cost_tracker.read().unwrap(); + verified_transactions_with_packet_indexes + .into_iter() + .filter_map(|(tx, tx_index)| { + let result = cost_tracker_readonly + .would_transaction_fit(&tx, demote_program_write_locks); + if result.is_err() { + debug!("transaction {:?} would exceed limit: {:?}", tx, result); + retryable_transaction_packet_indexes.push(tx_index); + return None; + } + Some((tx, tx_index)) + }) + .unzip() + }; + cost_tracker_check_time.stop(); + + banking_stage_stats + .cost_tracker_check_elapsed + .fetch_add(cost_tracker_check_time.as_us(), Ordering::Relaxed); + + ( + filtered_transactions, + filter_transaction_packet_indexes, + retryable_transaction_packet_indexes, + ) +>>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) } /// This function filters pending packets that are still valid @@ -1033,11 +1081,24 @@ impl BankingStage { banking_stage_stats: &BankingStageStats, ) -> (usize, usize, Vec) { let mut packet_conversion_time = Measure::start("packet_conversion"); +<<<<<<< HEAD let (transactions, transaction_to_packet_indexes) = Self::transactions_from_packets( msgs, &packet_indexes, bank.libsecp256k1_0_5_upgrade_enabled(), ); +======= + let (transactions, transaction_to_packet_indexes, retryable_packet_indexes) = + Self::transactions_from_packets( + msgs, + &packet_indexes, + bank.libsecp256k1_0_5_upgrade_enabled(), + bank.libsecp256k1_fail_on_bad_count(), + cost_tracker, + banking_stage_stats, + bank.demote_program_write_locks(), + ); +>>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) packet_conversion_time.stop(); debug!( @@ -1059,7 +1120,21 @@ impl BankingStage { ); process_tx_time.stop(); +<<<<<<< HEAD let unprocessed_tx_count = unprocessed_tx_indexes.len(); +======= + // applying cost of processed transactions to shared cost_tracker + let mut cost_tracking_time = Measure::start("cost_tracking_time"); + transactions.iter().enumerate().for_each(|(index, tx)| { + if unprocessed_tx_indexes.iter().all(|&i| i != index) { + cost_tracker + .write() + .unwrap() + .add_transaction_cost(tx, bank.demote_program_write_locks()); + } + }); + cost_tracking_time.stop(); +>>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) let mut filter_pending_packets_time = Measure::start("filter_pending_packets_time"); let filtered_unprocessed_packet_indexes = Self::filter_pending_packets_from_pending_txs( @@ -1104,11 +1179,27 @@ impl BankingStage { } } +<<<<<<< HEAD let (transactions, transaction_to_packet_indexes) = Self::transactions_from_packets( msgs, &transaction_indexes, bank.libsecp256k1_0_5_upgrade_enabled(), ); +======= + let mut unprocessed_packet_conversion_time = + Measure::start("unprocessed_packet_conversion"); + let (transactions, transaction_to_packet_indexes, retry_packet_indexes) = + Self::transactions_from_packets( + msgs, + transaction_indexes, + bank.libsecp256k1_0_5_upgrade_enabled(), + bank.libsecp256k1_fail_on_bad_count(), + cost_tracker, + banking_stage_stats, + bank.demote_program_write_locks(), + ); + unprocessed_packet_conversion_time.stop(); +>>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) let tx_count = transaction_to_packet_indexes.len(); diff --git a/core/src/cost_model.rs b/core/src/cost_model.rs new file mode 100644 index 00000000000000..ade82ea8e746b1 --- /dev/null +++ b/core/src/cost_model.rs @@ -0,0 +1,504 @@ +//! 'cost_model` provides service to estimate a transaction's cost +//! It does so by analyzing accounts the transaction touches, and instructions +//! it includes. Using historical data as guideline, it estimates cost of +//! reading/writing account, the sum of that comes up to "account access cost"; +//! Instructions take time to execute, both historical and runtime data are +//! used to determine each instruction's execution time, the sum of that +//! is transaction's "execution cost" +//! The main function is `calculate_cost` which returns &TransactionCost. +//! +use crate::execute_cost_table::ExecuteCostTable; +use log::*; +use solana_ledger::block_cost_limits::*; +use solana_sdk::{pubkey::Pubkey, transaction::SanitizedTransaction}; +use std::collections::HashMap; + +const MAX_WRITABLE_ACCOUNTS: usize = 256; + +#[derive(Debug, Clone)] +pub enum CostModelError { + /// transaction that would fail sanitize, cost model is not able to process + /// such transaction. + InvalidTransaction, + + /// would exceed block max limit + WouldExceedBlockMaxLimit, + + /// would exceed account max limit + WouldExceedAccountMaxLimit, +} + +// cost of transaction is made of account_access_cost and instruction execution_cost +// where +// account_access_cost is the sum of read/write/sign all accounts included in the transaction +// read is cheaper than write. +// execution_cost is the sum of all instructions execution cost, which is +// observed during runtime and feedback by Replay +#[derive(Default, Debug)] +pub struct TransactionCost { + pub writable_accounts: Vec, + pub account_access_cost: u64, + pub execution_cost: u64, +} + +impl TransactionCost { + pub fn new_with_capacity(capacity: usize) -> Self { + Self { + writable_accounts: Vec::with_capacity(capacity), + ..Self::default() + } + } + + pub fn reset(&mut self) { + self.writable_accounts.clear(); + self.account_access_cost = 0; + self.execution_cost = 0; + } +} + +#[derive(Debug)] +pub struct CostModel { + account_cost_limit: u64, + block_cost_limit: u64, + instruction_execution_cost_table: ExecuteCostTable, + + // reusable variables + transaction_cost: TransactionCost, +} + +impl Default for CostModel { + fn default() -> Self { + CostModel::new(ACCOUNT_COST_MAX, BLOCK_COST_MAX) + } +} + +impl CostModel { + pub fn new(chain_max: u64, block_max: u64) -> Self { + Self { + account_cost_limit: chain_max, + block_cost_limit: block_max, + instruction_execution_cost_table: ExecuteCostTable::default(), + transaction_cost: TransactionCost::new_with_capacity(MAX_WRITABLE_ACCOUNTS), + } + } + + pub fn get_account_cost_limit(&self) -> u64 { + self.account_cost_limit + } + + pub fn get_block_cost_limit(&self) -> u64 { + self.block_cost_limit + } + + pub fn initialize_cost_table(&mut self, cost_table: &[(Pubkey, u64)]) { + for (program_id, cost) in cost_table { + match self.upsert_instruction_cost(program_id, *cost) { + Ok(c) => { + debug!( + "initiating cost table, instruction {:?} has cost {}", + program_id, c + ); + } + Err(err) => { + debug!( + "initiating cost table, failed for instruction {:?}, err: {}", + program_id, err + ); + } + } + } + debug!( + "restored cost model instruction cost table from blockstore, current values: {:?}", + self.get_instruction_cost_table() + ); + } + + pub fn calculate_cost( + &mut self, + transaction: &SanitizedTransaction, + demote_program_write_locks: bool, + ) -> &TransactionCost { + self.transaction_cost.reset(); + + // calculate transaction exeution cost + self.transaction_cost.execution_cost = self.find_transaction_cost(transaction); + + // calculate account access cost + let message = transaction.message(); + message.account_keys_iter().enumerate().for_each(|(i, k)| { + let is_writable = message.is_writable(i, demote_program_write_locks); + + if is_writable { + self.transaction_cost.writable_accounts.push(*k); + self.transaction_cost.account_access_cost += ACCOUNT_WRITE_COST; + } else { + self.transaction_cost.account_access_cost += ACCOUNT_READ_COST; + } + }); + debug!( + "transaction {:?} has cost {:?}", + transaction, self.transaction_cost + ); + &self.transaction_cost + } + + // To update or insert instruction cost to table. + pub fn upsert_instruction_cost( + &mut self, + program_key: &Pubkey, + cost: u64, + ) -> Result { + self.instruction_execution_cost_table + .upsert(program_key, cost); + match self.instruction_execution_cost_table.get_cost(program_key) { + Some(cost) => Ok(*cost), + None => Err("failed to upsert to ExecuteCostTable"), + } + } + + pub fn get_instruction_cost_table(&self) -> &HashMap { + self.instruction_execution_cost_table.get_cost_table() + } + + fn find_instruction_cost(&self, program_key: &Pubkey) -> u64 { + match self.instruction_execution_cost_table.get_cost(program_key) { + Some(cost) => *cost, + None => { + let default_value = self.instruction_execution_cost_table.get_mode(); + debug!( + "Program key {:?} does not have assigned cost, using mode {}", + program_key, default_value + ); + default_value + } + } + } + + fn find_transaction_cost(&self, transaction: &SanitizedTransaction) -> u64 { + let mut cost: u64 = 0; + + for (program_id, instruction) in transaction.message().program_instructions_iter() { + let instruction_cost = self.find_instruction_cost(program_id); + trace!( + "instruction {:?} has cost of {}", + instruction, + instruction_cost + ); + cost += instruction_cost; + } + cost + } +} + +#[cfg(test)] +mod tests { + use super::*; + use solana_runtime::{ + bank::Bank, + genesis_utils::{create_genesis_config, GenesisConfigInfo}, + }; + use solana_sdk::{ + bpf_loader, + hash::Hash, + instruction::CompiledInstruction, + message::Message, + signature::{Keypair, Signer}, + system_instruction::{self}, + system_program, system_transaction, + transaction::Transaction, + }; + use std::{ + convert::{TryFrom, TryInto}, + str::FromStr, + sync::{Arc, RwLock}, + thread::{self, JoinHandle}, + }; + + fn test_setup() -> (Keypair, Hash) { + solana_logger::setup(); + let GenesisConfigInfo { + genesis_config, + mint_keypair, + .. + } = create_genesis_config(10); + let bank = Arc::new(Bank::new_no_wallclock_throttle_for_tests(&genesis_config)); + let start_hash = bank.last_blockhash(); + (mint_keypair, start_hash) + } + + #[test] + fn test_cost_model_instruction_cost() { + let mut testee = CostModel::default(); + + let known_key = Pubkey::from_str("known11111111111111111111111111111111111111").unwrap(); + testee.upsert_instruction_cost(&known_key, 100).unwrap(); + // find cost for known programs + assert_eq!(100, testee.find_instruction_cost(&known_key)); + + testee + .upsert_instruction_cost(&bpf_loader::id(), 1999) + .unwrap(); + assert_eq!(1999, testee.find_instruction_cost(&bpf_loader::id())); + + // unknown program is assigned with default cost + assert_eq!( + testee.instruction_execution_cost_table.get_mode(), + testee.find_instruction_cost( + &Pubkey::from_str("unknown111111111111111111111111111111111111").unwrap() + ) + ); + } + + #[test] + fn test_cost_model_simple_transaction() { + let (mint_keypair, start_hash) = test_setup(); + + let keypair = Keypair::new(); + let simple_transaction: SanitizedTransaction = + system_transaction::transfer(&mint_keypair, &keypair.pubkey(), 2, start_hash) + .try_into() + .unwrap(); + debug!( + "system_transaction simple_transaction {:?}", + simple_transaction + ); + + // expected cost for one system transfer instructions + let expected_cost = 8; + + let mut testee = CostModel::default(); + testee + .upsert_instruction_cost(&system_program::id(), expected_cost) + .unwrap(); + assert_eq!( + expected_cost, + testee.find_transaction_cost(&simple_transaction) + ); + } + + #[test] + fn test_cost_model_transaction_many_transfer_instructions() { + let (mint_keypair, start_hash) = test_setup(); + + let key1 = solana_sdk::pubkey::new_rand(); + let key2 = solana_sdk::pubkey::new_rand(); + let instructions = + system_instruction::transfer_many(&mint_keypair.pubkey(), &[(key1, 1), (key2, 1)]); + let message = Message::new(&instructions, Some(&mint_keypair.pubkey())); + let tx: SanitizedTransaction = Transaction::new(&[&mint_keypair], message, start_hash) + .try_into() + .unwrap(); + debug!("many transfer transaction {:?}", tx); + + // expected cost for two system transfer instructions + let program_cost = 8; + let expected_cost = program_cost * 2; + + let mut testee = CostModel::default(); + testee + .upsert_instruction_cost(&system_program::id(), program_cost) + .unwrap(); + assert_eq!(expected_cost, testee.find_transaction_cost(&tx)); + } + + #[test] + fn test_cost_model_message_many_different_instructions() { + let (mint_keypair, start_hash) = test_setup(); + + // construct a transaction with multiple random instructions + let key1 = solana_sdk::pubkey::new_rand(); + let key2 = solana_sdk::pubkey::new_rand(); + let prog1 = solana_sdk::pubkey::new_rand(); + let prog2 = solana_sdk::pubkey::new_rand(); + let instructions = vec![ + CompiledInstruction::new(3, &(), vec![0, 1]), + CompiledInstruction::new(4, &(), vec![0, 2]), + ]; + let tx: SanitizedTransaction = Transaction::new_with_compiled_instructions( + &[&mint_keypair], + &[key1, key2], + start_hash, + vec![prog1, prog2], + instructions, + ) + .try_into() + .unwrap(); + debug!("many random transaction {:?}", tx); + + let testee = CostModel::default(); + let result = testee.find_transaction_cost(&tx); + + // expected cost for two random/unknown program is + let expected_cost = testee.instruction_execution_cost_table.get_mode() * 2; + assert_eq!(expected_cost, result); + } + + #[test] + fn test_cost_model_sort_message_accounts_by_type() { + // construct a transaction with two random instructions with same signer + let signer1 = Keypair::new(); + let signer2 = Keypair::new(); + let key1 = Pubkey::new_unique(); + let key2 = Pubkey::new_unique(); + let prog1 = Pubkey::new_unique(); + let prog2 = Pubkey::new_unique(); + let instructions = vec![ + CompiledInstruction::new(4, &(), vec![0, 2]), + CompiledInstruction::new(5, &(), vec![1, 3]), + ]; + let tx: SanitizedTransaction = Transaction::new_with_compiled_instructions( + &[&signer1, &signer2], + &[key1, key2], + Hash::new_unique(), + vec![prog1, prog2], + instructions, + ) + .try_into() + .unwrap(); + + let mut cost_model = CostModel::default(); + let tx_cost = cost_model.calculate_cost(&tx, /*demote_program_write_locks=*/ true); + assert_eq!(2 + 2, tx_cost.writable_accounts.len()); + assert_eq!(signer1.pubkey(), tx_cost.writable_accounts[0]); + assert_eq!(signer2.pubkey(), tx_cost.writable_accounts[1]); + assert_eq!(key1, tx_cost.writable_accounts[2]); + assert_eq!(key2, tx_cost.writable_accounts[3]); + } + + #[test] + fn test_cost_model_insert_instruction_cost() { + let key1 = Pubkey::new_unique(); + let cost1 = 100; + + let mut cost_model = CostModel::default(); + // Using default cost for unknown instruction + assert_eq!( + cost_model.instruction_execution_cost_table.get_mode(), + cost_model.find_instruction_cost(&key1) + ); + + // insert instruction cost to table + assert!(cost_model.upsert_instruction_cost(&key1, cost1).is_ok()); + + // now it is known insturction with known cost + assert_eq!(cost1, cost_model.find_instruction_cost(&key1)); + } + + #[test] + fn test_cost_model_calculate_cost() { + let (mint_keypair, start_hash) = test_setup(); + let tx: SanitizedTransaction = + system_transaction::transfer(&mint_keypair, &Keypair::new().pubkey(), 2, start_hash) + .try_into() + .unwrap(); + + let expected_account_cost = ACCOUNT_WRITE_COST + ACCOUNT_WRITE_COST + ACCOUNT_READ_COST; + let expected_execution_cost = 8; + + let mut cost_model = CostModel::default(); + cost_model + .upsert_instruction_cost(&system_program::id(), expected_execution_cost) + .unwrap(); + let tx_cost = cost_model.calculate_cost(&tx, /*demote_program_write_locks=*/ true); + assert_eq!(expected_account_cost, tx_cost.account_access_cost); + assert_eq!(expected_execution_cost, tx_cost.execution_cost); + assert_eq!(2, tx_cost.writable_accounts.len()); + } + + #[test] + fn test_cost_model_update_instruction_cost() { + let key1 = Pubkey::new_unique(); + let cost1 = 100; + let cost2 = 200; + let updated_cost = (cost1 + cost2) / 2; + + let mut cost_model = CostModel::default(); + + // insert instruction cost to table + assert!(cost_model.upsert_instruction_cost(&key1, cost1).is_ok()); + assert_eq!(cost1, cost_model.find_instruction_cost(&key1)); + + // update instruction cost + assert!(cost_model.upsert_instruction_cost(&key1, cost2).is_ok()); + assert_eq!(updated_cost, cost_model.find_instruction_cost(&key1)); + } + + #[test] + fn test_cost_model_can_be_shared_concurrently_with_rwlock() { + let (mint_keypair, start_hash) = test_setup(); + // construct a transaction with multiple random instructions + let key1 = solana_sdk::pubkey::new_rand(); + let key2 = solana_sdk::pubkey::new_rand(); + let prog1 = solana_sdk::pubkey::new_rand(); + let prog2 = solana_sdk::pubkey::new_rand(); + let instructions = vec![ + CompiledInstruction::new(3, &(), vec![0, 1]), + CompiledInstruction::new(4, &(), vec![0, 2]), + ]; + let tx = Arc::new( + SanitizedTransaction::try_from(Transaction::new_with_compiled_instructions( + &[&mint_keypair], + &[key1, key2], + start_hash, + vec![prog1, prog2], + instructions, + )) + .unwrap(), + ); + + let number_threads = 10; + let expected_account_cost = + ACCOUNT_WRITE_COST + ACCOUNT_WRITE_COST * 2 + ACCOUNT_READ_COST * 2; + let cost1 = 100; + let cost2 = 200; + // execution cost can be either 2 * Default (before write) or cost1+cost2 (after write) + + let cost_model: Arc> = Arc::new(RwLock::new(CostModel::default())); + + let thread_handlers: Vec> = (0..number_threads) + .map(|i| { + let cost_model = cost_model.clone(); + let tx = tx.clone(); + + if i == 5 { + thread::spawn(move || { + let mut cost_model = cost_model.write().unwrap(); + assert!(cost_model.upsert_instruction_cost(&prog1, cost1).is_ok()); + assert!(cost_model.upsert_instruction_cost(&prog2, cost2).is_ok()); + }) + } else { + thread::spawn(move || { + let mut cost_model = cost_model.write().unwrap(); + let tx_cost = cost_model + .calculate_cost(&tx, /*demote_program_write_locks=*/ true); + assert_eq!(3, tx_cost.writable_accounts.len()); + assert_eq!(expected_account_cost, tx_cost.account_access_cost); + }) + } + }) + .collect(); + + for th in thread_handlers { + th.join().unwrap(); + } + } + + #[test] + fn test_cost_model_init_cost_table() { + // build cost table + let cost_table = vec![ + (Pubkey::new_unique(), 10), + (Pubkey::new_unique(), 20), + (Pubkey::new_unique(), 30), + ]; + + // init cost model + let mut cost_model = CostModel::default(); + cost_model.initialize_cost_table(&cost_table); + + // verify + for (id, cost) in cost_table.iter() { + assert_eq!(*cost, cost_model.find_instruction_cost(id)); + } + } +} diff --git a/core/src/cost_tracker.rs b/core/src/cost_tracker.rs new file mode 100644 index 00000000000000..40a86133adb7d3 --- /dev/null +++ b/core/src/cost_tracker.rs @@ -0,0 +1,423 @@ +//! `cost_tracker` keeps tracking transaction cost per chained accounts as well as for entire block +//! It aggregates `cost_model`, which provides service of calculating transaction cost. +//! The main functions are: +//! - would_transaction_fit(&tx), immutable function to test if `tx` would fit into current block +//! - add_transaction_cost(&tx), mutable function to accumulate `tx` cost to tracker. +//! +use crate::cost_model::{CostModel, CostModelError, TransactionCost}; +use solana_sdk::{clock::Slot, pubkey::Pubkey, transaction::SanitizedTransaction}; +use std::{ + collections::HashMap, + sync::{Arc, RwLock}, +}; + +const WRITABLE_ACCOUNTS_PER_BLOCK: usize = 512; + +#[derive(Debug)] +pub struct CostTracker { + cost_model: Arc>, + account_cost_limit: u64, + block_cost_limit: u64, + current_bank_slot: Slot, + cost_by_writable_accounts: HashMap, + block_cost: u64, +} + +impl CostTracker { + pub fn new(cost_model: Arc>) -> Self { + let (account_cost_limit, block_cost_limit) = { + let cost_model = cost_model.read().unwrap(); + ( + cost_model.get_account_cost_limit(), + cost_model.get_block_cost_limit(), + ) + }; + assert!(account_cost_limit <= block_cost_limit); + Self { + cost_model, + account_cost_limit, + block_cost_limit, + current_bank_slot: 0, + cost_by_writable_accounts: HashMap::with_capacity(WRITABLE_ACCOUNTS_PER_BLOCK), + block_cost: 0, + } + } + + pub fn would_transaction_fit( + &self, + transaction: &SanitizedTransaction, + demote_program_write_locks: bool, + ) -> Result<(), CostModelError> { + let mut cost_model = self.cost_model.write().unwrap(); + let tx_cost = cost_model.calculate_cost(transaction, demote_program_write_locks); + self.would_fit( + &tx_cost.writable_accounts, + &(tx_cost.account_access_cost + tx_cost.execution_cost), + ) + } + + pub fn add_transaction_cost( + &mut self, + transaction: &SanitizedTransaction, + demote_program_write_locks: bool, + ) { + let mut cost_model = self.cost_model.write().unwrap(); + let tx_cost = cost_model.calculate_cost(transaction, demote_program_write_locks); + let cost = tx_cost.account_access_cost + tx_cost.execution_cost; + for account_key in tx_cost.writable_accounts.iter() { + *self + .cost_by_writable_accounts + .entry(*account_key) + .or_insert(0) += cost; + } + self.block_cost += cost; + } + + pub fn reset_if_new_bank(&mut self, slot: Slot) { + if slot != self.current_bank_slot { + self.current_bank_slot = slot; + self.cost_by_writable_accounts.clear(); + self.block_cost = 0; + } + } + + pub fn try_add(&mut self, transaction_cost: &TransactionCost) -> Result { + let cost = transaction_cost.account_access_cost + transaction_cost.execution_cost; + self.would_fit(&transaction_cost.writable_accounts, &cost)?; + + self.add_transaction(&transaction_cost.writable_accounts, &cost); + Ok(self.block_cost) + } + + fn would_fit(&self, keys: &[Pubkey], cost: &u64) -> Result<(), CostModelError> { + // check against the total package cost + if self.block_cost + cost > self.block_cost_limit { + return Err(CostModelError::WouldExceedBlockMaxLimit); + } + + // check if the transaction itself is more costly than the account_cost_limit + if *cost > self.account_cost_limit { + return Err(CostModelError::WouldExceedAccountMaxLimit); + } + + // check each account against account_cost_limit, + for account_key in keys.iter() { + match self.cost_by_writable_accounts.get(account_key) { + Some(chained_cost) => { + if chained_cost + cost > self.account_cost_limit { + return Err(CostModelError::WouldExceedAccountMaxLimit); + } else { + continue; + } + } + None => continue, + } + } + + Ok(()) + } + + fn add_transaction(&mut self, keys: &[Pubkey], cost: &u64) { + for account_key in keys.iter() { + *self + .cost_by_writable_accounts + .entry(*account_key) + .or_insert(0) += cost; + } + self.block_cost += cost; + } +} + +// CostStats can be collected by util, such as ledger_tool +#[derive(Default, Debug)] +pub struct CostStats { + pub bank_slot: Slot, + pub total_cost: u64, + pub number_of_accounts: usize, + pub costliest_account: Pubkey, + pub costliest_account_cost: u64, +} + +impl CostTracker { + pub fn get_stats(&self) -> CostStats { + let mut stats = CostStats { + bank_slot: self.current_bank_slot, + total_cost: self.block_cost, + number_of_accounts: self.cost_by_writable_accounts.len(), + costliest_account: Pubkey::default(), + costliest_account_cost: 0, + }; + + for (key, cost) in self.cost_by_writable_accounts.iter() { + if cost > &stats.costliest_account_cost { + stats.costliest_account = *key; + stats.costliest_account_cost = *cost; + } + } + + stats + } +} + +#[cfg(test)] +mod tests { + use super::*; + use solana_runtime::{ + bank::Bank, + genesis_utils::{create_genesis_config, GenesisConfigInfo}, + }; + use solana_sdk::{ + hash::Hash, + signature::{Keypair, Signer}, + system_transaction, + transaction::Transaction, + }; + use std::{cmp, sync::Arc}; + + fn test_setup() -> (Keypair, Hash) { + solana_logger::setup(); + let GenesisConfigInfo { + genesis_config, + mint_keypair, + .. + } = create_genesis_config(10); + let bank = Arc::new(Bank::new_no_wallclock_throttle_for_tests(&genesis_config)); + let start_hash = bank.last_blockhash(); + (mint_keypair, start_hash) + } + + fn build_simple_transaction( + mint_keypair: &Keypair, + start_hash: &Hash, + ) -> (Transaction, Vec, u64) { + let keypair = Keypair::new(); + let simple_transaction = + system_transaction::transfer(mint_keypair, &keypair.pubkey(), 2, *start_hash); + + (simple_transaction, vec![mint_keypair.pubkey()], 5) + } + + #[test] + fn test_cost_tracker_initialization() { + let testee = CostTracker::new(Arc::new(RwLock::new(CostModel::new(10, 11)))); + assert_eq!(10, testee.account_cost_limit); + assert_eq!(11, testee.block_cost_limit); + assert_eq!(0, testee.cost_by_writable_accounts.len()); + assert_eq!(0, testee.block_cost); + } + + #[test] + fn test_cost_tracker_ok_add_one() { + let (mint_keypair, start_hash) = test_setup(); + let (_tx, keys, cost) = build_simple_transaction(&mint_keypair, &start_hash); + + // build testee to have capacity for one simple transaction + let mut testee = CostTracker::new(Arc::new(RwLock::new(CostModel::new(cost, cost)))); + assert!(testee.would_fit(&keys, &cost).is_ok()); + testee.add_transaction(&keys, &cost); + assert_eq!(cost, testee.block_cost); + } + + #[test] + fn test_cost_tracker_ok_add_two_same_accounts() { + let (mint_keypair, start_hash) = test_setup(); + // build two transactions with same signed account + let (_tx1, keys1, cost1) = build_simple_transaction(&mint_keypair, &start_hash); + let (_tx2, keys2, cost2) = build_simple_transaction(&mint_keypair, &start_hash); + + // build testee to have capacity for two simple transactions, with same accounts + let mut testee = CostTracker::new(Arc::new(RwLock::new(CostModel::new( + cost1 + cost2, + cost1 + cost2, + )))); + { + assert!(testee.would_fit(&keys1, &cost1).is_ok()); + testee.add_transaction(&keys1, &cost1); + } + { + assert!(testee.would_fit(&keys2, &cost2).is_ok()); + testee.add_transaction(&keys2, &cost2); + } + assert_eq!(cost1 + cost2, testee.block_cost); + assert_eq!(1, testee.cost_by_writable_accounts.len()); + } + + #[test] + fn test_cost_tracker_ok_add_two_diff_accounts() { + let (mint_keypair, start_hash) = test_setup(); + // build two transactions with diff accounts + let (_tx1, keys1, cost1) = build_simple_transaction(&mint_keypair, &start_hash); + let second_account = Keypair::new(); + let (_tx2, keys2, cost2) = build_simple_transaction(&second_account, &start_hash); + + // build testee to have capacity for two simple transactions, with same accounts + let mut testee = CostTracker::new(Arc::new(RwLock::new(CostModel::new( + cmp::max(cost1, cost2), + cost1 + cost2, + )))); + { + assert!(testee.would_fit(&keys1, &cost1).is_ok()); + testee.add_transaction(&keys1, &cost1); + } + { + assert!(testee.would_fit(&keys2, &cost2).is_ok()); + testee.add_transaction(&keys2, &cost2); + } + assert_eq!(cost1 + cost2, testee.block_cost); + assert_eq!(2, testee.cost_by_writable_accounts.len()); + } + + #[test] + fn test_cost_tracker_chain_reach_limit() { + let (mint_keypair, start_hash) = test_setup(); + // build two transactions with same signed account + let (_tx1, keys1, cost1) = build_simple_transaction(&mint_keypair, &start_hash); + let (_tx2, keys2, cost2) = build_simple_transaction(&mint_keypair, &start_hash); + + // build testee to have capacity for two simple transactions, but not for same accounts + let mut testee = CostTracker::new(Arc::new(RwLock::new(CostModel::new( + cmp::min(cost1, cost2), + cost1 + cost2, + )))); + // should have room for first transaction + { + assert!(testee.would_fit(&keys1, &cost1).is_ok()); + testee.add_transaction(&keys1, &cost1); + } + // but no more sapce on the same chain (same signer account) + { + assert!(testee.would_fit(&keys2, &cost2).is_err()); + } + } + + #[test] + fn test_cost_tracker_reach_limit() { + let (mint_keypair, start_hash) = test_setup(); + // build two transactions with diff accounts + let (_tx1, keys1, cost1) = build_simple_transaction(&mint_keypair, &start_hash); + let second_account = Keypair::new(); + let (_tx2, keys2, cost2) = build_simple_transaction(&second_account, &start_hash); + + // build testee to have capacity for each chain, but not enough room for both transactions + let mut testee = CostTracker::new(Arc::new(RwLock::new(CostModel::new( + cmp::max(cost1, cost2), + cost1 + cost2 - 1, + )))); + // should have room for first transaction + { + assert!(testee.would_fit(&keys1, &cost1).is_ok()); + testee.add_transaction(&keys1, &cost1); + } + // but no more room for package as whole + { + assert!(testee.would_fit(&keys2, &cost2).is_err()); + } + } + + #[test] + fn test_cost_tracker_reset() { + let (mint_keypair, start_hash) = test_setup(); + // build two transactions with same signed account + let (_tx1, keys1, cost1) = build_simple_transaction(&mint_keypair, &start_hash); + let (_tx2, keys2, cost2) = build_simple_transaction(&mint_keypair, &start_hash); + + // build testee to have capacity for two simple transactions, but not for same accounts + let mut testee = CostTracker::new(Arc::new(RwLock::new(CostModel::new( + cmp::min(cost1, cost2), + cost1 + cost2, + )))); + // should have room for first transaction + { + assert!(testee.would_fit(&keys1, &cost1).is_ok()); + testee.add_transaction(&keys1, &cost1); + assert_eq!(1, testee.cost_by_writable_accounts.len()); + assert_eq!(cost1, testee.block_cost); + } + // but no more sapce on the same chain (same signer account) + { + assert!(testee.would_fit(&keys2, &cost2).is_err()); + } + // reset the tracker + { + testee.reset_if_new_bank(100); + assert_eq!(0, testee.cost_by_writable_accounts.len()); + assert_eq!(0, testee.block_cost); + } + //now the second transaction can be added + { + assert!(testee.would_fit(&keys2, &cost2).is_ok()); + } + } + + #[test] + fn test_cost_tracker_try_add_is_atomic() { + let acct1 = Pubkey::new_unique(); + let acct2 = Pubkey::new_unique(); + let acct3 = Pubkey::new_unique(); + let cost = 100; + let account_max = cost * 2; + let block_max = account_max * 3; // for three accts + + let mut testee = CostTracker::new(Arc::new(RwLock::new(CostModel::new( + account_max, + block_max, + )))); + + // case 1: a tx writes to 3 accounts, should success, we will have: + // | acct1 | $cost | + // | acct2 | $cost | + // | acct2 | $cost | + // and block_cost = $cost + { + let tx_cost = TransactionCost { + writable_accounts: vec![acct1, acct2, acct3], + account_access_cost: 0, + execution_cost: cost, + }; + assert!(testee.try_add(&tx_cost).is_ok()); + let stat = testee.get_stats(); + assert_eq!(cost, stat.total_cost); + assert_eq!(3, stat.number_of_accounts); + assert_eq!(cost, stat.costliest_account_cost); + } + + // case 2: add tx writes to acct2 with $cost, should succeed, result to + // | acct1 | $cost | + // | acct2 | $cost * 2 | + // | acct2 | $cost | + // and block_cost = $cost * 2 + { + let tx_cost = TransactionCost { + writable_accounts: vec![acct2], + account_access_cost: 0, + execution_cost: cost, + }; + assert!(testee.try_add(&tx_cost).is_ok()); + let stat = testee.get_stats(); + assert_eq!(cost * 2, stat.total_cost); + assert_eq!(3, stat.number_of_accounts); + assert_eq!(cost * 2, stat.costliest_account_cost); + assert_eq!(acct2, stat.costliest_account); + } + + // case 3: add tx writes to [acct1, acct2], acct2 exceeds limit, should failed atomically, + // we shoudl still have: + // | acct1 | $cost | + // | acct2 | $cost | + // | acct2 | $cost | + // and block_cost = $cost + { + let tx_cost = TransactionCost { + writable_accounts: vec![acct1, acct2], + account_access_cost: 0, + execution_cost: cost, + }; + assert!(testee.try_add(&tx_cost).is_err()); + let stat = testee.get_stats(); + assert_eq!(cost * 2, stat.total_cost); + assert_eq!(3, stat.number_of_accounts); + assert_eq!(cost * 2, stat.costliest_account_cost); + assert_eq!(acct2, stat.costliest_account); + } + } +} diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 9aa14f8a5cd99b..0a88e8b833ea17 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -725,6 +725,78 @@ fn load_bank_forks( ) } +<<<<<<< HEAD +======= +fn compute_slot_cost(blockstore: &Blockstore, slot: Slot) -> Result<(), String> { + if blockstore.is_dead(slot) { + return Err("Dead slot".to_string()); + } + + let (entries, _num_shreds, _is_full) = blockstore + .get_slot_entries_with_shred_info(slot, 0, false) + .map_err(|err| format!(" Slot: {}, Failed to load entries, err {:?}", slot, err))?; + + let num_entries = entries.len(); + let mut num_transactions = 0; + let mut num_programs = 0; + + let mut program_ids = HashMap::new(); + let mut cost_model = CostModel::default(); + cost_model.initialize_cost_table(&blockstore.read_program_costs().unwrap()); + let cost_model = Arc::new(RwLock::new(cost_model)); + let mut cost_tracker = CostTracker::new(cost_model.clone()); + + for entry in entries { + num_transactions += entry.transactions.len(); + let mut cost_model = cost_model.write().unwrap(); + entry + .transactions + .into_iter() + .filter_map(|transaction| { + SanitizedTransaction::try_create(transaction, Hash::default(), |_| { + Err(TransactionError::UnsupportedVersion) + }) + .map_err(|err| { + warn!("Failed to compute cost of transaction: {:?}", err); + }) + .ok() + }) + .for_each(|transaction| { + num_programs += transaction.message().instructions().len(); + + let tx_cost = cost_model.calculate_cost( + &transaction, + true, // demote_program_write_locks + ); + if cost_tracker.try_add(tx_cost).is_err() { + println!( + "Slot: {}, CostModel rejected transaction {:?}, stats {:?}!", + slot, + transaction, + cost_tracker.get_stats() + ); + } + for (program_id, _instruction) in transaction.message().program_instructions_iter() + { + *program_ids.entry(*program_id).or_insert(0) += 1; + } + }); + } + + println!( + "Slot: {}, Entries: {}, Transactions: {}, Programs {}, {:?}", + slot, + num_entries, + num_transactions, + num_programs, + cost_tracker.get_stats() + ); + println!(" Programs: {:?}", program_ids); + + Ok(()) +} + +>>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) fn open_genesis_config_by(ledger_path: &Path, matches: &ArgMatches<'_>) -> GenesisConfig { let max_genesis_archive_unpacked_size = value_t_or_exit!(matches, "max_genesis_archive_unpacked_size", u64); diff --git a/program-runtime/src/instruction_processor.rs b/program-runtime/src/instruction_processor.rs new file mode 100644 index 00000000000000..49c6381be1e4b2 --- /dev/null +++ b/program-runtime/src/instruction_processor.rs @@ -0,0 +1,1242 @@ +use crate::native_loader::NativeLoader; +use serde::{Deserialize, Serialize}; +use solana_sdk::{ + account::{AccountSharedData, ReadableAccount, WritableAccount}, + account_utils::StateMut, + bpf_loader_upgradeable::{self, UpgradeableLoaderState}, + feature_set::demote_program_write_locks, + ic_logger_msg, ic_msg, + instruction::{CompiledInstruction, Instruction, InstructionError}, + keyed_account::{keyed_account_at_index, KeyedAccount}, + message::Message, + process_instruction::{Executor, InvokeContext, Logger, ProcessInstructionWithContext}, + pubkey::Pubkey, + rent::Rent, + system_program, +}; +use std::{ + cell::{Ref, RefCell}, + collections::HashMap, + rc::Rc, + sync::Arc, +}; + +pub struct Executors { + pub executors: HashMap>, + pub is_dirty: bool, +} +impl Default for Executors { + fn default() -> Self { + Self { + executors: HashMap::default(), + is_dirty: false, + } + } +} +impl Executors { + pub fn insert(&mut self, key: Pubkey, executor: Arc) { + let _ = self.executors.insert(key, executor); + self.is_dirty = true; + } + pub fn get(&self, key: &Pubkey) -> Option> { + self.executors.get(key).cloned() + } +} + +#[derive(Default, Debug)] +pub struct ProgramTiming { + pub accumulated_us: u64, + pub accumulated_units: u64, + pub count: u32, +} + +#[derive(Default, Debug)] +pub struct ExecuteDetailsTimings { + pub serialize_us: u64, + pub create_vm_us: u64, + pub execute_us: u64, + pub deserialize_us: u64, + pub changed_account_count: u64, + pub total_account_count: u64, + pub total_data_size: usize, + pub data_size_changed: usize, + pub per_program_timings: HashMap, +} +impl ExecuteDetailsTimings { + pub fn accumulate(&mut self, other: &ExecuteDetailsTimings) { + self.serialize_us += other.serialize_us; + self.create_vm_us += other.create_vm_us; + self.execute_us += other.execute_us; + self.deserialize_us += other.deserialize_us; + self.changed_account_count += other.changed_account_count; + self.total_account_count += other.total_account_count; + self.total_data_size += other.total_data_size; + self.data_size_changed += other.data_size_changed; + for (id, other) in &other.per_program_timings { + let program_timing = self.per_program_timings.entry(*id).or_default(); + program_timing.accumulated_us = program_timing + .accumulated_us + .saturating_add(other.accumulated_us); + program_timing.accumulated_units = program_timing + .accumulated_units + .saturating_add(other.accumulated_units); + program_timing.count = program_timing.count.saturating_add(other.count); + } + } + pub fn accumulate_program(&mut self, program_id: &Pubkey, us: u64, units: u64) { + let program_timing = self.per_program_timings.entry(*program_id).or_default(); + program_timing.accumulated_us = program_timing.accumulated_us.saturating_add(us); + program_timing.accumulated_units = program_timing.accumulated_units.saturating_add(units); + program_timing.count = program_timing.count.saturating_add(1); + } +} + +// The relevant state of an account before an Instruction executes, used +// to verify account integrity after the Instruction completes +#[derive(Clone, Debug, Default)] +pub struct PreAccount { + key: Pubkey, + account: Rc>, + changed: bool, +} +impl PreAccount { + pub fn new(key: &Pubkey, account: &AccountSharedData) -> Self { + Self { + key: *key, + account: Rc::new(RefCell::new(account.clone())), + changed: false, + } + } + + pub fn verify( + &self, + program_id: &Pubkey, + is_writable: bool, + rent: &Rent, + post: &AccountSharedData, + timings: &mut ExecuteDetailsTimings, + outermost_call: bool, + updated_verify_policy: bool, + ) -> Result<(), InstructionError> { + let pre = self.account.borrow(); + + // Only the owner of the account may change owner and + // only if the account is writable and + // only if the account is not executable and + // only if the data is zero-initialized or empty + let owner_changed = pre.owner() != post.owner(); + if owner_changed + && (!is_writable // line coverage used to get branch coverage + || pre.executable() + || program_id != pre.owner() + || !Self::is_zeroed(post.data())) + { + return Err(InstructionError::ModifiedProgramId); + } + + // An account not assigned to the program cannot have its balance decrease. + if program_id != pre.owner() // line coverage used to get branch coverage + && pre.lamports() > post.lamports() + { + return Err(InstructionError::ExternalAccountLamportSpend); + } + + // The balance of read-only and executable accounts may not change + let lamports_changed = pre.lamports() != post.lamports(); + if lamports_changed { + if !is_writable { + return Err(InstructionError::ReadonlyLamportChange); + } + if pre.executable() { + return Err(InstructionError::ExecutableLamportChange); + } + } + + // Only the system program can change the size of the data + // and only if the system program owns the account + let data_len_changed = pre.data().len() != post.data().len(); + if data_len_changed + && (!system_program::check_id(program_id) // line coverage used to get branch coverage + || !system_program::check_id(pre.owner())) + { + return Err(InstructionError::AccountDataSizeChanged); + } + + // Only the owner may change account data + // and if the account is writable + // and if the account is not executable + if !(program_id == pre.owner() + && is_writable // line coverage used to get branch coverage + && !pre.executable()) + && pre.data() != post.data() + { + if pre.executable() { + return Err(InstructionError::ExecutableDataModified); + } else if is_writable { + return Err(InstructionError::ExternalAccountDataModified); + } else { + return Err(InstructionError::ReadonlyDataModified); + } + } + + // executable is one-way (false->true) and only the account owner may set it. + let executable_changed = pre.executable() != post.executable(); + if executable_changed { + if !rent.is_exempt(post.lamports(), post.data().len()) { + return Err(InstructionError::ExecutableAccountNotRentExempt); + } + let owner = if updated_verify_policy { + post.owner() + } else { + pre.owner() + }; + if !is_writable // line coverage used to get branch coverage + || pre.executable() + || program_id != owner + { + return Err(InstructionError::ExecutableModified); + } + } + + // No one modifies rent_epoch (yet). + let rent_epoch_changed = pre.rent_epoch() != post.rent_epoch(); + if rent_epoch_changed { + return Err(InstructionError::RentEpochModified); + } + + if outermost_call { + timings.total_account_count += 1; + timings.total_data_size += post.data().len(); + if owner_changed + || lamports_changed + || data_len_changed + || executable_changed + || rent_epoch_changed + || self.changed + { + timings.changed_account_count += 1; + timings.data_size_changed += post.data().len(); + } + } + + Ok(()) + } + + pub fn update(&mut self, account: &AccountSharedData) { + let mut pre = self.account.borrow_mut(); + let rent_epoch = pre.rent_epoch(); + *pre = account.clone(); + pre.set_rent_epoch(rent_epoch); + + self.changed = true; + } + + pub fn key(&self) -> &Pubkey { + &self.key + } + + pub fn data(&self) -> Ref<[u8]> { + Ref::map(self.account.borrow(), |account| account.data()) + } + + pub fn lamports(&self) -> u64 { + self.account.borrow().lamports() + } + + pub fn executable(&self) -> bool { + self.account.borrow().executable() + } + + pub fn is_zeroed(buf: &[u8]) -> bool { + const ZEROS_LEN: usize = 1024; + static ZEROS: [u8; ZEROS_LEN] = [0; ZEROS_LEN]; + let mut chunks = buf.chunks_exact(ZEROS_LEN); + + chunks.all(|chunk| chunk == &ZEROS[..]) + && chunks.remainder() == &ZEROS[..chunks.remainder().len()] + } +} + +#[derive(Deserialize, Serialize)] +pub struct InstructionProcessor { + #[serde(skip)] + programs: Vec<(Pubkey, ProcessInstructionWithContext)>, + #[serde(skip)] + native_loader: NativeLoader, +} + +impl std::fmt::Debug for InstructionProcessor { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + #[derive(Debug)] + struct MessageProcessor<'a> { + programs: Vec, + native_loader: &'a NativeLoader, + } + + // These are just type aliases for work around of Debug-ing above pointers + type ErasedProcessInstructionWithContext = fn( + &'static Pubkey, + &'static [u8], + &'static mut dyn InvokeContext, + ) -> Result<(), InstructionError>; + + // rustc doesn't compile due to bug without this work around + // https://github.com/rust-lang/rust/issues/50280 + // https://users.rust-lang.org/t/display-function-pointer/17073/2 + let processor = MessageProcessor { + programs: self + .programs + .iter() + .map(|(pubkey, instruction)| { + let erased_instruction: ErasedProcessInstructionWithContext = *instruction; + format!("{}: {:p}", pubkey, erased_instruction) + }) + .collect::>(), + native_loader: &self.native_loader, + }; + + write!(f, "{:?}", processor) + } +} + +impl Default for InstructionProcessor { + fn default() -> Self { + Self { + programs: vec![], + native_loader: NativeLoader::default(), + } + } +} +impl Clone for InstructionProcessor { + fn clone(&self) -> Self { + InstructionProcessor { + programs: self.programs.clone(), + native_loader: NativeLoader::default(), + } + } +} + +#[cfg(RUSTC_WITH_SPECIALIZATION)] +impl ::solana_frozen_abi::abi_example::AbiExample for InstructionProcessor { + fn example() -> Self { + // MessageProcessor's fields are #[serde(skip)]-ed and not Serialize + // so, just rely on Default anyway. + InstructionProcessor::default() + } +} + +impl InstructionProcessor { + pub fn programs(&self) -> &[(Pubkey, ProcessInstructionWithContext)] { + &self.programs + } + + /// Add a static entrypoint to intercept instructions before the dynamic loader. + pub fn add_program( + &mut self, + program_id: Pubkey, + process_instruction: ProcessInstructionWithContext, + ) { + match self.programs.iter_mut().find(|(key, _)| program_id == *key) { + Some((_, processor)) => *processor = process_instruction, + None => self.programs.push((program_id, process_instruction)), + } + } + + /// Create the KeyedAccounts that will be passed to the program + pub fn create_keyed_accounts<'a>( + message: &'a Message, + instruction: &'a CompiledInstruction, + executable_accounts: &'a [(Pubkey, Rc>)], + accounts: &'a [(Pubkey, Rc>)], + demote_program_write_locks: bool, + ) -> Vec<(bool, bool, &'a Pubkey, &'a RefCell)> { + executable_accounts + .iter() + .map(|(key, account)| (false, false, key, account as &RefCell)) + .chain(instruction.accounts.iter().map(|index| { + let index = *index as usize; + ( + message.is_signer(index), + message.is_writable(index, demote_program_write_locks), + &accounts[index].0, + &accounts[index].1 as &RefCell, + ) + })) + .collect::>() + } + + /// Process an instruction + /// This method calls the instruction's program entrypoint method + pub fn process_instruction( + &self, + program_id: &Pubkey, + instruction_data: &[u8], + invoke_context: &mut dyn InvokeContext, + ) -> Result<(), InstructionError> { + if let Some(root_account) = invoke_context.get_keyed_accounts()?.iter().next() { + let root_id = root_account.unsigned_key(); + if solana_sdk::native_loader::check_id(&root_account.owner()?) { + for (id, process_instruction) in &self.programs { + if id == root_id { + invoke_context.remove_first_keyed_account()?; + // Call the builtin program + return process_instruction(program_id, instruction_data, invoke_context); + } + } + // Call the program via the native loader + return self.native_loader.process_instruction( + &solana_sdk::native_loader::id(), + instruction_data, + invoke_context, + ); + } else { + let owner_id = &root_account.owner()?; + for (id, process_instruction) in &self.programs { + if id == owner_id { + // Call the program via a builtin loader + return process_instruction(program_id, instruction_data, invoke_context); + } + } + } + } + Err(InstructionError::UnsupportedProgramId) + } + + pub fn create_message( + instruction: &Instruction, + keyed_accounts: &[&KeyedAccount], + signers: &[Pubkey], + invoke_context: &Ref<&mut dyn InvokeContext>, + ) -> Result<(Message, Pubkey, usize), InstructionError> { + // Check for privilege escalation + for account in instruction.accounts.iter() { + let keyed_account = keyed_accounts + .iter() + .find_map(|keyed_account| { + if &account.pubkey == keyed_account.unsigned_key() { + Some(keyed_account) + } else { + None + } + }) + .ok_or_else(|| { + ic_msg!( + invoke_context, + "Instruction references an unknown account {}", + account.pubkey + ); + InstructionError::MissingAccount + })?; + // Readonly account cannot become writable + if account.is_writable && !keyed_account.is_writable() { + ic_msg!( + invoke_context, + "{}'s writable privilege escalated", + account.pubkey + ); + return Err(InstructionError::PrivilegeEscalation); + } + + if account.is_signer && // If message indicates account is signed + !( // one of the following needs to be true: + keyed_account.signer_key().is_some() // Signed in the parent instruction + || signers.contains(&account.pubkey) // Signed by the program + ) { + ic_msg!( + invoke_context, + "{}'s signer privilege escalated", + account.pubkey + ); + return Err(InstructionError::PrivilegeEscalation); + } + } + + // validate the caller has access to the program account and that it is executable + let program_id = instruction.program_id; + match keyed_accounts + .iter() + .find(|keyed_account| &program_id == keyed_account.unsigned_key()) + { + Some(keyed_account) => { + if !keyed_account.executable()? { + ic_msg!( + invoke_context, + "Account {} is not executable", + keyed_account.unsigned_key() + ); + return Err(InstructionError::AccountNotExecutable); + } + } + None => { + ic_msg!(invoke_context, "Unknown program {}", program_id); + return Err(InstructionError::MissingAccount); + } + } + + let message = Message::new(&[instruction.clone()], None); + let program_id_index = message.instructions[0].program_id_index as usize; + + Ok((message, program_id, program_id_index)) + } + + /// Entrypoint for a cross-program invocation from a native program + pub fn native_invoke( + invoke_context: &mut dyn InvokeContext, + instruction: Instruction, + keyed_account_indices: &[usize], + signers: &[Pubkey], + ) -> Result<(), InstructionError> { + let invoke_context = RefCell::new(invoke_context); + + let ( + message, + executable_accounts, + accounts, + keyed_account_indices_reordered, + caller_write_privileges, + ) = { + let invoke_context = invoke_context.borrow(); + + // Translate and verify caller's data + let keyed_accounts = invoke_context.get_keyed_accounts()?; + let keyed_accounts = keyed_account_indices + .iter() + .map(|index| keyed_account_at_index(keyed_accounts, *index)) + .collect::, InstructionError>>()?; + let (message, callee_program_id, _) = + Self::create_message(&instruction, &keyed_accounts, signers, &invoke_context)?; + let keyed_accounts = invoke_context.get_keyed_accounts()?; + let mut caller_write_privileges = keyed_account_indices + .iter() + .map(|index| keyed_accounts[*index].is_writable()) + .collect::>(); + caller_write_privileges.insert(0, false); + let mut accounts = vec![]; + let mut keyed_account_indices_reordered = vec![]; + let keyed_accounts = invoke_context.get_keyed_accounts()?; + 'root: for account_key in message.account_keys.iter() { + for keyed_account_index in keyed_account_indices { + let keyed_account = &keyed_accounts[*keyed_account_index]; + if account_key == keyed_account.unsigned_key() { + accounts.push((*account_key, Rc::new(keyed_account.account.clone()))); + keyed_account_indices_reordered.push(*keyed_account_index); + continue 'root; + } + } + ic_msg!( + invoke_context, + "Instruction references an unknown account {}", + account_key + ); + return Err(InstructionError::MissingAccount); + } + + // Process instruction + + invoke_context.record_instruction(&instruction); + + let program_account = + invoke_context + .get_account(&callee_program_id) + .ok_or_else(|| { + ic_msg!(invoke_context, "Unknown program {}", callee_program_id); + InstructionError::MissingAccount + })?; + if !program_account.borrow().executable() { + ic_msg!( + invoke_context, + "Account {} is not executable", + callee_program_id + ); + return Err(InstructionError::AccountNotExecutable); + } + let programdata = if program_account.borrow().owner() == &bpf_loader_upgradeable::id() { + if let UpgradeableLoaderState::Program { + programdata_address, + } = program_account.borrow().state()? + { + if let Some(account) = invoke_context.get_account(&programdata_address) { + Some((programdata_address, account)) + } else { + ic_msg!( + invoke_context, + "Unknown upgradeable programdata account {}", + programdata_address, + ); + return Err(InstructionError::MissingAccount); + } + } else { + ic_msg!( + invoke_context, + "Upgradeable program account state not valid {}", + callee_program_id, + ); + return Err(InstructionError::MissingAccount); + } + } else { + None + }; + let mut executable_accounts = vec![(callee_program_id, program_account)]; + if let Some(programdata) = programdata { + executable_accounts.push(programdata); + } + ( + message, + executable_accounts, + accounts, + keyed_account_indices_reordered, + caller_write_privileges, + ) + }; + + #[allow(clippy::deref_addrof)] + InstructionProcessor::process_cross_program_instruction( + &message, + &executable_accounts, + &accounts, + &caller_write_privileges, + *(&mut *(invoke_context.borrow_mut())), + )?; + + // Copy results back to caller + + { + let invoke_context = invoke_context.borrow(); + let demote_program_write_locks = + invoke_context.is_feature_active(&demote_program_write_locks::id()); + let keyed_accounts = invoke_context.get_keyed_accounts()?; + for (src_keyed_account_index, ((_key, account), dst_keyed_account_index)) in accounts + .iter() + .zip(keyed_account_indices_reordered) + .enumerate() + { + let dst_keyed_account = &keyed_accounts[dst_keyed_account_index]; + let src_keyed_account = account.borrow(); + if message.is_writable(src_keyed_account_index, demote_program_write_locks) + && !src_keyed_account.executable() + { + if dst_keyed_account.data_len()? != src_keyed_account.data().len() + && dst_keyed_account.data_len()? != 0 + { + // Only support for `CreateAccount` at this time. + // Need a way to limit total realloc size across multiple CPI calls + ic_msg!( + invoke_context, + "Inner instructions do not support realloc, only SystemProgram::CreateAccount", + ); + return Err(InstructionError::InvalidRealloc); + } + dst_keyed_account + .try_account_ref_mut()? + .set_lamports(src_keyed_account.lamports()); + dst_keyed_account + .try_account_ref_mut()? + .set_owner(*src_keyed_account.owner()); + dst_keyed_account + .try_account_ref_mut()? + .set_data(src_keyed_account.data().to_vec()); + } + } + } + + Ok(()) + } + + /// Process a cross-program instruction + /// This method calls the instruction's program entrypoint function + pub fn process_cross_program_instruction( + message: &Message, + executable_accounts: &[(Pubkey, Rc>)], + accounts: &[(Pubkey, Rc>)], + caller_write_privileges: &[bool], + invoke_context: &mut dyn InvokeContext, + ) -> Result<(), InstructionError> { + if let Some(instruction) = message.instructions.get(0) { + let program_id = instruction.program_id(&message.account_keys); + + // Verify the calling program hasn't misbehaved + invoke_context.verify_and_update(instruction, accounts, caller_write_privileges)?; + + // Construct keyed accounts + let demote_program_write_locks = + invoke_context.is_feature_active(&demote_program_write_locks::id()); + let keyed_accounts = Self::create_keyed_accounts( + message, + instruction, + executable_accounts, + accounts, + demote_program_write_locks, + ); + + // Invoke callee + invoke_context.push(program_id, &keyed_accounts)?; + + let mut instruction_processor = InstructionProcessor::default(); + for (program_id, process_instruction) in invoke_context.get_programs().iter() { + instruction_processor.add_program(*program_id, *process_instruction); + } + + let mut result = instruction_processor.process_instruction( + program_id, + &instruction.data, + invoke_context, + ); + if result.is_ok() { + // Verify the called program has not misbehaved + let write_privileges: Vec = (0..message.account_keys.len()) + .map(|i| message.is_writable(i, demote_program_write_locks)) + .collect(); + result = invoke_context.verify_and_update(instruction, accounts, &write_privileges); + } + + // Restore previous state + invoke_context.pop(); + result + } else { + // This function is always called with a valid instruction, if that changes return an error + Err(InstructionError::GenericError) + } + } + + /// Record the initial state of the accounts so that they can be compared + /// after the instruction is processed + pub fn create_pre_accounts( + message: &Message, + instruction: &CompiledInstruction, + accounts: &[(Pubkey, Rc>)], + ) -> Vec { + let mut pre_accounts = Vec::with_capacity(instruction.accounts.len()); + { + let mut work = |_unique_index: usize, account_index: usize| { + if account_index < message.account_keys.len() && account_index < accounts.len() { + let account = accounts[account_index].1.borrow(); + pre_accounts.push(PreAccount::new(&accounts[account_index].0, &account)); + return Ok(()); + } + Err(InstructionError::MissingAccount) + }; + let _ = instruction.visit_each_account(&mut work); + } + pre_accounts + } + + /// Verify the results of a cross-program instruction + #[allow(clippy::too_many_arguments)] + pub fn verify_and_update( + instruction: &CompiledInstruction, + pre_accounts: &mut [PreAccount], + accounts: &[(Pubkey, Rc>)], + program_id: &Pubkey, + rent: &Rent, + write_privileges: &[bool], + timings: &mut ExecuteDetailsTimings, + logger: Rc>, + updated_verify_policy: bool, + ) -> Result<(), InstructionError> { + // Verify the per-account instruction results + let (mut pre_sum, mut post_sum) = (0_u128, 0_u128); + let mut work = |_unique_index: usize, account_index: usize| { + if account_index < write_privileges.len() && account_index < accounts.len() { + let (key, account) = &accounts[account_index]; + let is_writable = write_privileges[account_index]; + // Find the matching PreAccount + for pre_account in pre_accounts.iter_mut() { + if key == pre_account.key() { + { + // Verify account has no outstanding references + let _ = account + .try_borrow_mut() + .map_err(|_| InstructionError::AccountBorrowOutstanding)?; + } + let account = account.borrow(); + pre_account + .verify( + program_id, + is_writable, + rent, + &account, + timings, + false, + updated_verify_policy, + ) + .map_err(|err| { + ic_logger_msg!(logger, "failed to verify account {}: {}", key, err); + err + })?; + pre_sum += u128::from(pre_account.lamports()); + post_sum += u128::from(account.lamports()); + if is_writable && !pre_account.executable() { + pre_account.update(&account); + } + return Ok(()); + } + } + } + Err(InstructionError::MissingAccount) + }; + instruction.visit_each_account(&mut work)?; + + // Verify that the total sum of all the lamports did not change + if pre_sum != post_sum { + return Err(InstructionError::UnbalancedInstruction); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use solana_sdk::{account::Account, instruction::InstructionError}; + + #[test] + fn test_is_zeroed() { + const ZEROS_LEN: usize = 1024; + let mut buf = [0; ZEROS_LEN]; + assert!(PreAccount::is_zeroed(&buf)); + buf[0] = 1; + assert!(!PreAccount::is_zeroed(&buf)); + + let mut buf = [0; ZEROS_LEN - 1]; + assert!(PreAccount::is_zeroed(&buf)); + buf[0] = 1; + assert!(!PreAccount::is_zeroed(&buf)); + + let mut buf = [0; ZEROS_LEN + 1]; + assert!(PreAccount::is_zeroed(&buf)); + buf[0] = 1; + assert!(!PreAccount::is_zeroed(&buf)); + + let buf = vec![]; + assert!(PreAccount::is_zeroed(&buf)); + } + + struct Change { + program_id: Pubkey, + is_writable: bool, + rent: Rent, + pre: PreAccount, + post: AccountSharedData, + } + impl Change { + pub fn new(owner: &Pubkey, program_id: &Pubkey) -> Self { + Self { + program_id: *program_id, + rent: Rent::default(), + is_writable: true, + pre: PreAccount::new( + &solana_sdk::pubkey::new_rand(), + &AccountSharedData::from(Account { + owner: *owner, + lamports: std::u64::MAX, + data: vec![], + ..Account::default() + }), + ), + post: AccountSharedData::from(Account { + owner: *owner, + lamports: std::u64::MAX, + ..Account::default() + }), + } + } + pub fn read_only(mut self) -> Self { + self.is_writable = false; + self + } + pub fn executable(mut self, pre: bool, post: bool) -> Self { + self.pre.account.borrow_mut().set_executable(pre); + self.post.set_executable(post); + self + } + pub fn lamports(mut self, pre: u64, post: u64) -> Self { + self.pre.account.borrow_mut().set_lamports(pre); + self.post.set_lamports(post); + self + } + pub fn owner(mut self, post: &Pubkey) -> Self { + self.post.set_owner(*post); + self + } + pub fn data(mut self, pre: Vec, post: Vec) -> Self { + self.pre.account.borrow_mut().set_data(pre); + self.post.set_data(post); + self + } + pub fn rent_epoch(mut self, pre: u64, post: u64) -> Self { + self.pre.account.borrow_mut().set_rent_epoch(pre); + self.post.set_rent_epoch(post); + self + } + pub fn verify(&self) -> Result<(), InstructionError> { + self.pre.verify( + &self.program_id, + self.is_writable, + &self.rent, + &self.post, + &mut ExecuteDetailsTimings::default(), + false, + true, + ) + } + } + + #[test] + fn test_verify_account_changes_owner() { + let system_program_id = system_program::id(); + let alice_program_id = solana_sdk::pubkey::new_rand(); + let mallory_program_id = solana_sdk::pubkey::new_rand(); + + assert_eq!( + Change::new(&system_program_id, &system_program_id) + .owner(&alice_program_id) + .verify(), + Ok(()), + "system program should be able to change the account owner" + ); + assert_eq!( + Change::new(&system_program_id, &system_program_id) + .owner(&alice_program_id) + .read_only() + .verify(), + Err(InstructionError::ModifiedProgramId), + "system program should not be able to change the account owner of a read-only account" + ); + assert_eq!( + Change::new(&mallory_program_id, &system_program_id) + .owner(&alice_program_id) + .verify(), + Err(InstructionError::ModifiedProgramId), + "system program should not be able to change the account owner of a non-system account" + ); + assert_eq!( + Change::new(&mallory_program_id, &mallory_program_id) + .owner(&alice_program_id) + .verify(), + Ok(()), + "mallory should be able to change the account owner, if she leaves clear data" + ); + assert_eq!( + Change::new(&mallory_program_id, &mallory_program_id) + .owner(&alice_program_id) + .data(vec![42], vec![0]) + .verify(), + Ok(()), + "mallory should be able to change the account owner, if she leaves clear data" + ); + assert_eq!( + Change::new(&mallory_program_id, &mallory_program_id) + .owner(&alice_program_id) + .executable(true, true) + .data(vec![42], vec![0]) + .verify(), + Err(InstructionError::ModifiedProgramId), + "mallory should not be able to change the account owner, if the account executable" + ); + assert_eq!( + Change::new(&mallory_program_id, &mallory_program_id) + .owner(&alice_program_id) + .data(vec![42], vec![42]) + .verify(), + Err(InstructionError::ModifiedProgramId), + "mallory should not be able to inject data into the alice program" + ); + } + + #[test] + fn test_verify_account_changes_executable() { + let owner = solana_sdk::pubkey::new_rand(); + let mallory_program_id = solana_sdk::pubkey::new_rand(); + let system_program_id = system_program::id(); + + assert_eq!( + Change::new(&owner, &system_program_id) + .executable(false, true) + .verify(), + Err(InstructionError::ExecutableModified), + "system program can't change executable if system doesn't own the account" + ); + assert_eq!( + Change::new(&owner, &system_program_id) + .executable(true, true) + .data(vec![1], vec![2]) + .verify(), + Err(InstructionError::ExecutableDataModified), + "system program can't change executable data if system doesn't own the account" + ); + assert_eq!( + Change::new(&owner, &owner).executable(false, true).verify(), + Ok(()), + "owner should be able to change executable" + ); + assert_eq!( + Change::new(&owner, &owner) + .executable(false, true) + .read_only() + .verify(), + Err(InstructionError::ExecutableModified), + "owner can't modify executable of read-only accounts" + ); + assert_eq!( + Change::new(&owner, &owner).executable(true, false).verify(), + Err(InstructionError::ExecutableModified), + "owner program can't reverse executable" + ); + assert_eq!( + Change::new(&owner, &mallory_program_id) + .executable(false, true) + .verify(), + Err(InstructionError::ExecutableModified), + "malicious Mallory should not be able to change the account executable" + ); + assert_eq!( + Change::new(&owner, &owner) + .executable(false, true) + .data(vec![1], vec![2]) + .verify(), + Ok(()), + "account data can change in the same instruction that sets the bit" + ); + assert_eq!( + Change::new(&owner, &owner) + .executable(true, true) + .data(vec![1], vec![2]) + .verify(), + Err(InstructionError::ExecutableDataModified), + "owner should not be able to change an account's data once its marked executable" + ); + assert_eq!( + Change::new(&owner, &owner) + .executable(true, true) + .lamports(1, 2) + .verify(), + Err(InstructionError::ExecutableLamportChange), + "owner should not be able to add lamports once marked executable" + ); + assert_eq!( + Change::new(&owner, &owner) + .executable(true, true) + .lamports(1, 2) + .verify(), + Err(InstructionError::ExecutableLamportChange), + "owner should not be able to add lamports once marked executable" + ); + assert_eq!( + Change::new(&owner, &owner) + .executable(true, true) + .lamports(2, 1) + .verify(), + Err(InstructionError::ExecutableLamportChange), + "owner should not be able to subtract lamports once marked executable" + ); + let data = vec![1; 100]; + let min_lamports = Rent::default().minimum_balance(data.len()); + assert_eq!( + Change::new(&owner, &owner) + .executable(false, true) + .lamports(0, min_lamports) + .data(data.clone(), data.clone()) + .verify(), + Ok(()), + ); + assert_eq!( + Change::new(&owner, &owner) + .executable(false, true) + .lamports(0, min_lamports - 1) + .data(data.clone(), data) + .verify(), + Err(InstructionError::ExecutableAccountNotRentExempt), + "owner should not be able to change an account's data once its marked executable" + ); + } + + #[test] + fn test_verify_account_changes_data_len() { + let alice_program_id = solana_sdk::pubkey::new_rand(); + + assert_eq!( + Change::new(&system_program::id(), &system_program::id()) + .data(vec![0], vec![0, 0]) + .verify(), + Ok(()), + "system program should be able to change the data len" + ); + assert_eq!( + Change::new(&alice_program_id, &system_program::id()) + .data(vec![0], vec![0,0]) + .verify(), + Err(InstructionError::AccountDataSizeChanged), + "system program should not be able to change the data length of accounts it does not own" + ); + } + + #[test] + fn test_verify_account_changes_data() { + let alice_program_id = solana_sdk::pubkey::new_rand(); + let mallory_program_id = solana_sdk::pubkey::new_rand(); + + assert_eq!( + Change::new(&alice_program_id, &alice_program_id) + .data(vec![0], vec![42]) + .verify(), + Ok(()), + "alice program should be able to change the data" + ); + assert_eq!( + Change::new(&mallory_program_id, &alice_program_id) + .data(vec![0], vec![42]) + .verify(), + Err(InstructionError::ExternalAccountDataModified), + "non-owner mallory should not be able to change the account data" + ); + assert_eq!( + Change::new(&alice_program_id, &alice_program_id) + .data(vec![0], vec![42]) + .read_only() + .verify(), + Err(InstructionError::ReadonlyDataModified), + "alice isn't allowed to touch a CO account" + ); + } + + #[test] + fn test_verify_account_changes_rent_epoch() { + let alice_program_id = solana_sdk::pubkey::new_rand(); + + assert_eq!( + Change::new(&alice_program_id, &system_program::id()).verify(), + Ok(()), + "nothing changed!" + ); + assert_eq!( + Change::new(&alice_program_id, &system_program::id()) + .rent_epoch(0, 1) + .verify(), + Err(InstructionError::RentEpochModified), + "no one touches rent_epoch" + ); + } + + #[test] + fn test_verify_account_changes_deduct_lamports_and_reassign_account() { + let alice_program_id = solana_sdk::pubkey::new_rand(); + let bob_program_id = solana_sdk::pubkey::new_rand(); + + // positive test of this capability + assert_eq!( + Change::new(&alice_program_id, &alice_program_id) + .owner(&bob_program_id) + .lamports(42, 1) + .data(vec![42], vec![0]) + .verify(), + Ok(()), + "alice should be able to deduct lamports and give the account to bob if the data is zeroed", + ); + } + + #[test] + fn test_verify_account_changes_lamports() { + let alice_program_id = solana_sdk::pubkey::new_rand(); + + assert_eq!( + Change::new(&alice_program_id, &system_program::id()) + .lamports(42, 0) + .read_only() + .verify(), + Err(InstructionError::ExternalAccountLamportSpend), + "debit should fail, even if system program" + ); + assert_eq!( + Change::new(&alice_program_id, &alice_program_id) + .lamports(42, 0) + .read_only() + .verify(), + Err(InstructionError::ReadonlyLamportChange), + "debit should fail, even if owning program" + ); + assert_eq!( + Change::new(&alice_program_id, &system_program::id()) + .lamports(42, 0) + .owner(&system_program::id()) + .verify(), + Err(InstructionError::ModifiedProgramId), + "system program can't debit the account unless it was the pre.owner" + ); + assert_eq!( + Change::new(&system_program::id(), &system_program::id()) + .lamports(42, 0) + .owner(&alice_program_id) + .verify(), + Ok(()), + "system can spend (and change owner)" + ); + } + + #[test] + fn test_verify_account_changes_data_size_changed() { + let alice_program_id = solana_sdk::pubkey::new_rand(); + + assert_eq!( + Change::new(&alice_program_id, &system_program::id()) + .data(vec![0], vec![0, 0]) + .verify(), + Err(InstructionError::AccountDataSizeChanged), + "system program should not be able to change another program's account data size" + ); + assert_eq!( + Change::new(&alice_program_id, &alice_program_id) + .data(vec![0], vec![0, 0]) + .verify(), + Err(InstructionError::AccountDataSizeChanged), + "non-system programs cannot change their data size" + ); + assert_eq!( + Change::new(&system_program::id(), &system_program::id()) + .data(vec![0], vec![0, 0]) + .verify(), + Ok(()), + "system program should be able to change account data size" + ); + } + + #[test] + fn test_verify_account_changes_owner_executable() { + let alice_program_id = solana_sdk::pubkey::new_rand(); + let bob_program_id = solana_sdk::pubkey::new_rand(); + + assert_eq!( + Change::new(&alice_program_id, &alice_program_id) + .owner(&bob_program_id) + .executable(false, true) + .verify(), + Err(InstructionError::ExecutableModified), + "Program should not be able to change owner and executable at the same time" + ); + } + + #[test] + fn test_debug() { + let mut instruction_processor = InstructionProcessor::default(); + #[allow(clippy::unnecessary_wraps)] + fn mock_process_instruction( + _program_id: &Pubkey, + _data: &[u8], + _invoke_context: &mut dyn InvokeContext, + ) -> Result<(), InstructionError> { + Ok(()) + } + #[allow(clippy::unnecessary_wraps)] + fn mock_ix_processor( + _pubkey: &Pubkey, + _data: &[u8], + _context: &mut dyn InvokeContext, + ) -> Result<(), InstructionError> { + Ok(()) + } + let program_id = solana_sdk::pubkey::new_rand(); + instruction_processor.add_program(program_id, mock_process_instruction); + instruction_processor.add_program(program_id, mock_ix_processor); + + assert!(!format!("{:?}", instruction_processor).is_empty()); + } +} diff --git a/program-test/src/lib.rs b/program-test/src/lib.rs index 636fa68d6040cc..f839014024fdf8 100644 --- a/program-test/src/lib.rs +++ b/program-test/src/lib.rs @@ -19,6 +19,7 @@ use { clock::{Clock, Slot}, entrypoint::{ProgramResult, SUCCESS}, epoch_schedule::EpochSchedule, + feature_set::demote_program_write_locks, fee_calculator::{FeeCalculator, FeeRateGovernor}, genesis_config::{ClusterType, GenesisConfig}, hash::Hash, @@ -258,12 +259,14 @@ impl solana_sdk::program_stubs::SyscallStubs for SyscallStubs { } panic!("Program id {} wasn't found in account_infos", program_id); }; + let demote_program_write_locks = + invoke_context.is_feature_active(&demote_program_write_locks::id()); // TODO don't have the caller's keyed_accounts so can't validate writer or signer escalation or deescalation yet let caller_privileges = message .account_keys .iter() .enumerate() - .map(|(i, _)| message.is_writable(i)) + .map(|(i, _)| message.is_writable(i, demote_program_write_locks)) .collect::>(); stable_log::program_invoke(&logger, &program_id, invoke_context.invoke_depth()); @@ -334,7 +337,7 @@ impl solana_sdk::program_stubs::SyscallStubs for SyscallStubs { // Copy writeable account modifications back into the caller's AccountInfos for (i, (pubkey, account)) in accounts.iter().enumerate().take(message.account_keys.len()) { - if !message.is_writable(i) { + if !message.is_writable(i, demote_program_write_locks) { continue; } for account_info in account_infos { diff --git a/programs/bpf/tests/programs.rs b/programs/bpf/tests/programs.rs index c394f72b98e5ca..b362de4b672155 100644 --- a/programs/bpf/tests/programs.rs +++ b/programs/bpf/tests/programs.rs @@ -1782,7 +1782,7 @@ fn test_program_bpf_upgrade_and_invoke_in_same_tx() { "solana_bpf_rust_panic", ); - // Invoke, then upgrade the program, and then invoke again in same tx + // Attempt to invoke, then upgrade the program in same tx let message = Message::new( &[ invoke_instruction.clone(), @@ -1801,10 +1801,12 @@ fn test_program_bpf_upgrade_and_invoke_in_same_tx() { message.clone(), bank.last_blockhash(), ); + // program_id is automatically demoted to readonly, preventing the upgrade, which requires + // writeability let (result, _) = process_transaction_and_record_inner(&bank, tx); assert_eq!( result.unwrap_err(), - TransactionError::InstructionError(2, InstructionError::ProgramFailedToComplete) + TransactionError::InstructionError(1, InstructionError::InvalidArgument) ); } @@ -2105,6 +2107,7 @@ fn test_program_bpf_upgrade_via_cpi() { #[cfg(feature = "bpf_rust")] #[test] +<<<<<<< HEAD fn test_program_bpf_upgrade_self_via_cpi() { solana_logger::setup(); @@ -2196,6 +2199,8 @@ fn test_program_bpf_upgrade_self_via_cpi() { #[cfg(feature = "bpf_rust")] #[test] +======= +>>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) fn test_program_bpf_set_upgrade_authority_via_cpi() { solana_logger::setup(); diff --git a/programs/bpf_loader/src/syscalls.rs b/programs/bpf_loader/src/syscalls.rs index 3145a14cc170d7..862e34f24de281 100644 --- a/programs/bpf_loader/src/syscalls.rs +++ b/programs/bpf_loader/src/syscalls.rs @@ -19,10 +19,16 @@ use solana_sdk::{ entrypoint::{MAX_PERMITTED_DATA_INCREASE, SUCCESS}, epoch_schedule::EpochSchedule, feature_set::{ +<<<<<<< HEAD close_upgradeable_program_accounts, cpi_data_cost, enforce_aligned_host_addrs, keccak256_syscall_enabled, libsecp256k1_0_5_upgrade_enabled, mem_overlap_fix, memory_ops_syscalls, secp256k1_recover_syscall_enabled, set_upgrade_authority_via_cpi_enabled, sysvar_via_syscall, update_data_on_realloc, +======= + blake3_syscall_enabled, close_upgradeable_program_accounts, demote_program_write_locks, + disable_fees_sysvar, enforce_aligned_host_addrs, libsecp256k1_0_5_upgrade_enabled, + mem_overlap_fix, secp256k1_recover_syscall_enabled, +>>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) }, hash::{Hasher, HASH_BYTES}, ic_msg, @@ -2244,7 +2250,14 @@ fn call<'a>( signers_seeds_len: u64, memory_mapping: &MemoryMapping, ) -> Result> { - let (message, executables, accounts, account_refs, caller_write_privileges) = { + let ( + message, + executables, + accounts, + account_refs, + caller_write_privileges, + demote_program_write_locks, + ) = { let invoke_context = syscall.get_context()?; invoke_context @@ -2335,6 +2348,7 @@ fn call<'a>( accounts, account_refs, caller_write_privileges, + invoke_context.is_feature_active(&demote_program_write_locks::id()), ) }; @@ -2360,7 +2374,7 @@ fn call<'a>( for (i, ((_key, account), account_ref)) in accounts.iter().zip(account_refs).enumerate() { let account = account.borrow(); if let Some(mut account_ref) = account_ref { - if message.is_writable(i) && !account.executable() { + if message.is_writable(i, demote_program_write_locks) && !account.executable() { *account_ref.lamports = account.lamports(); *account_ref.owner = *account.owner(); if account_ref.data.len() != account.data().len() { diff --git a/rpc/src/transaction_status_service.rs b/rpc/src/transaction_status_service.rs index e79f4d1c6f07b2..fd3f3d702aabd8 100644 --- a/rpc/src/transaction_status_service.rs +++ b/rpc/src/transaction_status_service.rs @@ -111,9 +111,15 @@ impl TransactionStatusService { bank.get_fee_calculator(&transaction.message().recent_blockhash) }) .expect("FeeCalculator must exist"); +<<<<<<< HEAD let fee = fee_calculator.calculate_fee(transaction.message()); let (writable_keys, readonly_keys) = transaction.message.get_account_keys_by_lock_type(); +======= + let fee = transaction.message().calculate_fee(&fee_calculator); + let tx_account_locks = + transaction.get_account_locks(bank.demote_program_write_locks()); +>>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) let inner_instructions = inner_instructions.map(|inner_instructions| { inner_instructions diff --git a/runtime/src/accounts.rs b/runtime/src/accounts.rs index 57c9960e8d7819..3dcbe5da9413ca 100644 --- a/runtime/src/accounts.rs +++ b/runtime/src/accounts.rs @@ -166,6 +166,7 @@ impl Accounts { } } +<<<<<<< HEAD /// Return true if the slice has any duplicate elements pub fn has_duplicates(xs: &[T]) -> bool { // Note: This is an O(n^2) algorithm, but requires no heap allocations. The benchmark @@ -181,6 +182,14 @@ impl Accounts { fn construct_instructions_account(message: &Message) -> AccountSharedData { let mut data = message.serialize_instructions(); +======= + fn construct_instructions_account( + message: &SanitizedMessage, + is_owned_by_sysvar: bool, + demote_program_write_locks: bool, + ) -> AccountSharedData { + let mut data = message.serialize_instructions(demote_program_write_locks); +>>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) // add room for current instruction index. data.resize(data.len() + 2, 0); AccountSharedData::from(Account { @@ -212,6 +221,8 @@ impl Accounts { let mut key_check = MessageProgramIdsCache::new(message); let mut rent_debits = RentDebits::default(); let rent_for_sysvars = feature_set.is_active(&feature_set::rent_for_sysvars::id()); + let demote_program_write_locks = + feature_set.is_active(&feature_set::demote_program_write_locks::id()); for (i, key) in message.account_keys.iter().enumerate() { let account = if key_check.is_non_loader_key(key, i) { @@ -219,6 +230,7 @@ impl Accounts { payer_index = Some(i); } +<<<<<<< HEAD if solana_sdk::sysvar::instructions::check_id(key) && feature_set.is_active(&feature_set::instructions_sysvar_enabled::id()) { @@ -226,12 +238,24 @@ impl Accounts { return Err(TransactionError::InvalidAccountIndex); } Self::construct_instructions_account(message) +======= + if solana_sdk::sysvar::instructions::check_id(key) { + if message.is_writable(i, demote_program_write_locks) { + return Err(TransactionError::InvalidAccountIndex); + } + Self::construct_instructions_account( + message, + feature_set + .is_active(&feature_set::instructions_sysvar_owned_by_sysvar::id()), + demote_program_write_locks, + ) +>>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) } else { let (account, rent) = self .accounts_db .load_with_fixed_root(ancestors, key) .map(|(mut account, _)| { - if message.is_writable(i) { + if message.is_writable(i, demote_program_write_locks) { let rent_due = rent_collector.collect_from_existing_account( key, &mut account, @@ -860,6 +884,7 @@ impl Accounts { /// same time #[must_use] #[allow(clippy::needless_collect)] +<<<<<<< HEAD pub fn lock_accounts<'a>(&self, txs: impl Iterator) -> Vec> { use solana_sdk::sanitize::Sanitize; let keys: Vec> = txs @@ -872,6 +897,15 @@ impl Accounts { Ok(tx.message().get_account_keys_by_lock_type()) }) +======= + pub fn lock_accounts<'a>( + &self, + txs: impl Iterator, + demote_program_write_locks: bool, + ) -> Vec> { + let keys: Vec<_> = txs + .map(|tx| tx.get_account_locks(demote_program_write_locks)) +>>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) .collect(); let mut account_locks = &mut self.account_locks.lock().unwrap(); keys.into_iter() @@ -890,6 +924,7 @@ impl Accounts { &self, txs: impl Iterator, results: &[Result<()>], + demote_program_write_locks: bool, ) { let keys: Vec<_> = txs .zip(results) @@ -897,7 +932,11 @@ impl Accounts { Err(TransactionError::AccountInUse) => None, Err(TransactionError::SanitizeFailure) => None, Err(TransactionError::AccountLoadedTwice) => None, +<<<<<<< HEAD _ => Some(tx.message.get_account_keys_by_lock_type()), +======= + _ => Some(tx.get_account_locks(demote_program_write_locks)), +>>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) }) .collect(); let mut account_locks = self.account_locks.lock().unwrap(); @@ -921,6 +960,7 @@ impl Accounts { fix_recent_blockhashes_sysvar_delay: bool, rent_for_sysvars: bool, merge_nonce_error_into_system_error: bool, + demote_program_write_locks: bool, ) { let accounts_to_store = self.collect_accounts_to_store( txs, @@ -931,6 +971,7 @@ impl Accounts { fix_recent_blockhashes_sysvar_delay, rent_for_sysvars, merge_nonce_error_into_system_error, + demote_program_write_locks, ); self.accounts_db.store_cached(slot, &accounts_to_store); } @@ -957,6 +998,7 @@ impl Accounts { fix_recent_blockhashes_sysvar_delay: bool, rent_for_sysvars: bool, merge_nonce_error_into_system_error: bool, + demote_program_write_locks: bool, ) -> Vec<(&'a Pubkey, &'a AccountSharedData)> { let mut accounts = Vec::with_capacity(loaded.len()); for (i, ((raccs, _nonce_rollback), tx)) in loaded.iter_mut().zip(txs).enumerate() { @@ -1006,7 +1048,7 @@ impl Accounts { fee_payer_index = Some(i); } let is_fee_payer = Some(i) == fee_payer_index; - if message.is_writable(i) + if message.is_writable(i, demote_program_write_locks) && (res.is_ok() || (maybe_nonce_rollback.is_some() && (is_nonce_account || is_fee_payer))) { @@ -1775,6 +1817,8 @@ mod tests { accounts.store_slow_uncached(0, &keypair2.pubkey(), &account2); accounts.store_slow_uncached(0, &keypair3.pubkey(), &account3); + let demote_program_write_locks = true; + let instructions = vec![CompiledInstruction::new(2, &(), vec![0, 1])]; let message = Message::new_with_compiled_instructions( 1, @@ -1784,8 +1828,13 @@ mod tests { Hash::default(), instructions, ); +<<<<<<< HEAD let tx = Transaction::new(&[&keypair0], message, Hash::default()); let results0 = accounts.lock_accounts([tx.clone()].iter()); +======= + let tx = new_sanitized_tx(&[&keypair0], message, Hash::default()); + let results0 = accounts.lock_accounts([tx.clone()].iter(), demote_program_write_locks); +>>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) assert!(results0[0].is_ok()); assert_eq!( @@ -1820,7 +1869,7 @@ mod tests { ); let tx1 = Transaction::new(&[&keypair1], message, Hash::default()); let txs = vec![tx0, tx1]; - let results1 = accounts.lock_accounts(txs.iter()); + let results1 = accounts.lock_accounts(txs.iter(), demote_program_write_locks); assert!(results1[0].is_ok()); // Read-only account (keypair1) can be referenced multiple times assert!(results1[1].is_err()); // Read-only account (keypair1) cannot also be locked as writable @@ -1835,8 +1884,8 @@ mod tests { 2 ); - accounts.unlock_accounts([tx].iter(), &results0); - accounts.unlock_accounts(txs.iter(), &results1); + accounts.unlock_accounts([tx].iter(), &results0, demote_program_write_locks); + accounts.unlock_accounts(txs.iter(), &results1, demote_program_write_locks); let instructions = vec![CompiledInstruction::new(2, &(), vec![0, 1])]; let message = Message::new_with_compiled_instructions( 1, @@ -1846,8 +1895,13 @@ mod tests { Hash::default(), instructions, ); +<<<<<<< HEAD let tx = Transaction::new(&[&keypair1], message, Hash::default()); let results2 = accounts.lock_accounts([tx].iter()); +======= + let tx = new_sanitized_tx(&[&keypair1], message, Hash::default()); + let results2 = accounts.lock_accounts([tx].iter(), demote_program_write_locks); +>>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) assert!(results2[0].is_ok()); // Now keypair1 account can be locked as writable // Check that read-only lock with zero references is deleted @@ -1884,6 +1938,8 @@ mod tests { accounts.store_slow_uncached(0, &keypair1.pubkey(), &account1); accounts.store_slow_uncached(0, &keypair2.pubkey(), &account2); + let demote_program_write_locks = true; + let accounts_arc = Arc::new(accounts); let instructions = vec![CompiledInstruction::new(2, &(), vec![0, 1])]; @@ -1916,13 +1972,15 @@ mod tests { let exit_clone = exit_clone.clone(); loop { let txs = vec![writable_tx.clone()]; - let results = accounts_clone.clone().lock_accounts(txs.iter()); + let results = accounts_clone + .clone() + .lock_accounts(txs.iter(), demote_program_write_locks); for result in results.iter() { if result.is_ok() { counter_clone.clone().fetch_add(1, Ordering::SeqCst); } } - accounts_clone.unlock_accounts(txs.iter(), &results); + accounts_clone.unlock_accounts(txs.iter(), &results, demote_program_write_locks); if exit_clone.clone().load(Ordering::Relaxed) { break; } @@ -1931,18 +1989,85 @@ mod tests { let counter_clone = counter; for _ in 0..5 { let txs = vec![readonly_tx.clone()]; - let results = accounts_arc.clone().lock_accounts(txs.iter()); + let results = accounts_arc + .clone() + .lock_accounts(txs.iter(), demote_program_write_locks); if results[0].is_ok() { let counter_value = counter_clone.clone().load(Ordering::SeqCst); thread::sleep(time::Duration::from_millis(50)); assert_eq!(counter_value, counter_clone.clone().load(Ordering::SeqCst)); } - accounts_arc.unlock_accounts(txs.iter(), &results); + accounts_arc.unlock_accounts(txs.iter(), &results, demote_program_write_locks); thread::sleep(time::Duration::from_millis(50)); } exit.store(true, Ordering::Relaxed); } + #[test] + fn test_demote_program_write_locks() { + let keypair0 = Keypair::new(); + let keypair1 = Keypair::new(); + let keypair2 = Keypair::new(); + let keypair3 = Keypair::new(); + + let account0 = AccountSharedData::new(1, 0, &Pubkey::default()); + let account1 = AccountSharedData::new(2, 0, &Pubkey::default()); + let account2 = AccountSharedData::new(3, 0, &Pubkey::default()); + let account3 = AccountSharedData::new(4, 0, &Pubkey::default()); + + let accounts = Accounts::new_with_config_for_tests( + Vec::new(), + &ClusterType::Development, + AccountSecondaryIndexes::default(), + false, + AccountShrinkThreshold::default(), + ); + accounts.store_slow_uncached(0, &keypair0.pubkey(), &account0); + accounts.store_slow_uncached(0, &keypair1.pubkey(), &account1); + accounts.store_slow_uncached(0, &keypair2.pubkey(), &account2); + accounts.store_slow_uncached(0, &keypair3.pubkey(), &account3); + + let demote_program_write_locks = true; + + let instructions = vec![CompiledInstruction::new(2, &(), vec![0, 1])]; + let message = Message::new_with_compiled_instructions( + 1, + 0, + 0, // All accounts marked as writable + vec![keypair0.pubkey(), keypair1.pubkey(), native_loader::id()], + Hash::default(), + instructions, + ); + let tx = new_sanitized_tx(&[&keypair0], message, Hash::default()); + let results0 = accounts.lock_accounts([tx].iter(), demote_program_write_locks); + + assert!(results0[0].is_ok()); + // Instruction program-id account demoted to readonly + assert_eq!( + *accounts + .account_locks + .lock() + .unwrap() + .readonly_locks + .get(&native_loader::id()) + .unwrap(), + 1 + ); + // Non-program accounts remain writable + assert!(accounts + .account_locks + .lock() + .unwrap() + .write_locks + .contains(&keypair0.pubkey())); + assert!(accounts + .account_locks + .lock() + .unwrap() + .write_locks + .contains(&keypair1.pubkey())); + } + #[test] fn test_collect_accounts_to_store() { let keypair0 = Keypair::new(); @@ -2036,6 +2161,7 @@ mod tests { true, true, true, // merge_nonce_error_into_system_error + true, // demote_program_write_locks ); assert_eq!(collected_accounts.len(), 2); assert!(collected_accounts @@ -2422,6 +2548,7 @@ mod tests { true, true, true, // merge_nonce_error_into_system_error + true, // demote_program_write_locks ); assert_eq!(collected_accounts.len(), 2); assert_eq!( @@ -2540,6 +2667,7 @@ mod tests { true, true, true, // merge_nonce_error_into_system_error + true, // demote_program_write_locks ); assert_eq!(collected_accounts.len(), 1); let collected_nonce_account = collected_accounts diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 9659987daa871d..b134c85af6f821 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -2610,6 +2610,7 @@ impl Bank { tick_height % self.ticks_per_slot == 0 } +<<<<<<< HEAD pub fn prepare_batch<'a, 'b>( &'a self, txs: impl Iterator, @@ -2620,6 +2621,46 @@ impl Bank { .accounts .lock_accounts(hashed_txs.as_transactions_iter()); TransactionBatch::new(lock_results, self, Cow::Owned(hashed_txs)) +======= + /// Prepare a transaction batch from a list of legacy transactionsy. Used for tests only. + pub fn prepare_batch(&self, txs: Vec) -> Result { + let sanitized_txs = txs + .into_iter() + .map(SanitizedTransaction::try_from) + .collect::>>()?; + let lock_results = self + .rc + .accounts + .lock_accounts(sanitized_txs.iter(), self.demote_program_write_locks()); + Ok(TransactionBatch::new( + lock_results, + self, + Cow::Owned(sanitized_txs), + )) + } + + /// Prepare a transaction batch from a list of versioned transactions from + /// an entry. Used for tests only. + pub fn prepare_entry_batch(&self, txs: Vec) -> Result { + let sanitized_txs = txs + .into_iter() + .map(|tx| { + let message_hash = tx.message.hash(); + SanitizedTransaction::try_create(tx, message_hash, |_| { + Err(TransactionError::UnsupportedVersion) + }) + }) + .collect::>>()?; + let lock_results = self + .rc + .accounts + .lock_accounts(sanitized_txs.iter(), self.demote_program_write_locks()); + Ok(TransactionBatch::new( + lock_results, + self, + Cow::Owned(sanitized_txs), + )) +>>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) } pub fn prepare_hashed_batch<'a, 'b>( @@ -2629,8 +2670,13 @@ impl Bank { let lock_results = self .rc .accounts +<<<<<<< HEAD .lock_accounts(hashed_txs.as_transactions_iter()); TransactionBatch::new(lock_results, self, Cow::Borrowed(hashed_txs)) +======= + .lock_accounts(txs.iter(), self.demote_program_write_locks()); + TransactionBatch::new(lock_results, self, Cow::Borrowed(txs)) +>>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) } pub(crate) fn prepare_simulation_batch<'a, 'b>( @@ -2708,9 +2754,17 @@ impl Bank { pub fn unlock_accounts(&self, batch: &mut TransactionBatch) { if batch.needs_unlock { batch.needs_unlock = false; +<<<<<<< HEAD self.rc .accounts .unlock_accounts(batch.transactions_iter(), batch.lock_results()) +======= + self.rc.accounts.unlock_accounts( + batch.sanitized_transactions().iter(), + batch.lock_results(), + self.demote_program_write_locks(), + ) +>>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) } } @@ -3428,6 +3482,7 @@ impl Bank { self.fix_recent_blockhashes_sysvar_delay(), self.rent_for_sysvars(), self.merge_nonce_error_into_system_error(), + self.demote_program_write_locks(), ); let rent_debits = self.collect_rent(executed, loaded_txs); @@ -5084,6 +5139,11 @@ impl Bank { .is_active(&feature_set::stake_program_advance_activating_credits_observed::id()) } + pub fn demote_program_write_locks(&self) -> bool { + self.feature_set + .is_active(&feature_set::demote_program_write_locks::id()) + } + // Check if the wallclock time from bank creation to now has exceeded the allotted // time for transaction processing pub fn should_bank_still_be_processing_txs( diff --git a/runtime/src/message_processor.rs b/runtime/src/message_processor.rs index fe5348a9395bc4..91e7a86e73028c 100644 --- a/runtime/src/message_processor.rs +++ b/runtime/src/message_processor.rs @@ -9,7 +9,12 @@ use solana_sdk::{ account_utils::StateMut, bpf_loader_upgradeable::{self, UpgradeableLoaderState}, feature_set::{ +<<<<<<< HEAD instructions_sysvar_enabled, neon_evm_compute_budget, updated_verify_policy, FeatureSet, +======= + demote_program_write_locks, neon_evm_compute_budget, tx_wide_compute_cap, + updated_verify_policy, FeatureSet, +>>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) }, ic_logger_msg, ic_msg, instruction::{CompiledInstruction, Instruction, InstructionError}, @@ -300,6 +305,7 @@ impl<'a> ThisInvokeContext<'a> { instruction, executable_accounts, accounts, + feature_set.is_active(&demote_program_write_locks::id()), ); let mut invoke_context = Self { invoke_stack: Vec::with_capacity(bpf_compute_budget.max_invoke_depth), @@ -984,6 +990,7 @@ impl MessageProcessor { } /// Verify the results of an instruction + #[allow(clippy::too_many_arguments)] pub fn verify( message: &Message, instruction: &CompiledInstruction, @@ -994,6 +1001,7 @@ impl MessageProcessor { timings: &mut ExecuteDetailsTimings, logger: Rc>, updated_verify_policy: bool, + demote_program_write_locks: bool, ) -> Result<(), InstructionError> { // Verify all executable accounts have zero outstanding refs Self::verify_account_references(executable_accounts)?; @@ -1014,7 +1022,7 @@ impl MessageProcessor { pre_accounts[unique_index] .verify( program_id, - message.is_writable(account_index), + message.is_writable(account_index, demote_program_write_locks), rent, &account, timings, @@ -1183,6 +1191,7 @@ impl MessageProcessor { timings, invoke_context.get_logger(), invoke_context.is_feature_active(&updated_verify_policy::id()), + invoke_context.is_feature_active(&demote_program_write_locks::id()), )?; timings.accumulate(&invoke_context.timings); @@ -1338,7 +1347,7 @@ mod tests { ))), )); let write_privileges: Vec = (0..message.account_keys.len()) - .map(|i| message.is_writable(i)) + .map(|i| message.is_writable(i, /*demote_program_write_locks=*/ true)) .collect(); invoke_context .verify_and_update(&message.instructions[0], &these_accounts, &write_privileges) @@ -2234,6 +2243,8 @@ mod tests { metas.clone(), ); let message = Message::new(&[instruction], None); + let feature_set = FeatureSet::all_enabled(); + let demote_program_write_locks = feature_set.is_active(&demote_program_write_locks::id()); let ancestors = Ancestors::default(); let mut invoke_context = ThisInvokeContext::new( @@ -2248,8 +2259,13 @@ mod tests { BpfComputeBudget::default(), Rc::new(RefCell::new(Executors::default())), None, +<<<<<<< HEAD Arc::new(FeatureSet::all_enabled()), Arc::new(Accounts::default()), +======= + Arc::new(feature_set), + Arc::new(Accounts::default_for_tests()), +>>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) &ancestors, ); @@ -2258,7 +2274,7 @@ mod tests { .account_keys .iter() .enumerate() - .map(|(i, _)| message.is_writable(i)) + .map(|(i, _)| message.is_writable(i, demote_program_write_locks)) .collect::>(); accounts[0].1.borrow_mut().data_as_mut_slice()[0] = 1; assert_eq!( @@ -2313,7 +2329,7 @@ mod tests { .account_keys .iter() .enumerate() - .map(|(i, _)| message.is_writable(i)) + .map(|(i, _)| message.is_writable(i, demote_program_write_locks)) .collect::>(); assert_eq!( MessageProcessor::process_cross_program_instruction( diff --git a/sdk/benches/serialize_instructions.rs b/sdk/benches/serialize_instructions.rs index dae549c98ee009..1d6b062d95b2f6 100644 --- a/sdk/benches/serialize_instructions.rs +++ b/sdk/benches/serialize_instructions.rs @@ -14,6 +14,8 @@ fn make_instructions() -> Vec { vec![inst; 4] } +const DEMOTE_PROGRAM_WRITE_LOCKS: bool = true; + #[bench] fn bench_bincode_instruction_serialize(b: &mut Bencher) { let instructions = make_instructions(); @@ -27,7 +29,7 @@ fn bench_manual_instruction_serialize(b: &mut Bencher) { let instructions = make_instructions(); let message = Message::new(&instructions, None); b.iter(|| { - test::black_box(message.serialize_instructions()); + test::black_box(message.serialize_instructions(DEMOTE_PROGRAM_WRITE_LOCKS)); }); } @@ -43,8 +45,15 @@ fn bench_bincode_instruction_deserialize(b: &mut Bencher) { #[bench] fn bench_manual_instruction_deserialize(b: &mut Bencher) { let instructions = make_instructions(); +<<<<<<< HEAD let message = Message::new(&instructions, None); let serialized = message.serialize_instructions(); +======= + let message = + SanitizedMessage::try_from(Message::new(&instructions, Some(&Pubkey::new_unique()))) + .unwrap(); + let serialized = message.serialize_instructions(DEMOTE_PROGRAM_WRITE_LOCKS); +>>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) b.iter(|| { for i in 0..instructions.len() { test::black_box(instructions::load_instruction_at(i, &serialized).unwrap()); @@ -55,8 +64,15 @@ fn bench_manual_instruction_deserialize(b: &mut Bencher) { #[bench] fn bench_manual_instruction_deserialize_single(b: &mut Bencher) { let instructions = make_instructions(); +<<<<<<< HEAD let message = Message::new(&instructions, None); let serialized = message.serialize_instructions(); +======= + let message = + SanitizedMessage::try_from(Message::new(&instructions, Some(&Pubkey::new_unique()))) + .unwrap(); + let serialized = message.serialize_instructions(DEMOTE_PROGRAM_WRITE_LOCKS); +>>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) b.iter(|| { test::black_box(instructions::load_instruction_at(3, &serialized).unwrap()); }); diff --git a/sdk/program/src/message.rs b/sdk/program/src/message.rs index 28b44eec3773c6..f23d1d241a369e 100644 --- a/sdk/program/src/message.rs +++ b/sdk/program/src/message.rs @@ -391,7 +391,7 @@ impl Message { self.program_position(i).is_some() } - pub fn is_writable(&self, i: usize) -> bool { + pub fn is_writable(&self, i: usize, demote_program_write_locks: bool) -> bool { (i < (self.header.num_required_signatures - self.header.num_readonly_signed_accounts) as usize || (i >= self.header.num_required_signatures as usize @@ -401,6 +401,7 @@ impl Message { let key = self.account_keys[i]; sysvar::is_sysvar_id(&key) || BUILTIN_PROGRAMS_KEYS.contains(&key) } + && !(demote_program_write_locks && self.is_key_called_as_program(i)) } pub fn is_signer(&self, i: usize) -> bool { @@ -411,7 +412,7 @@ impl Message { let mut writable_keys = vec![]; let mut readonly_keys = vec![]; for (i, key) in self.account_keys.iter().enumerate() { - if self.is_writable(i) { + if self.is_writable(i, /*demote_program_write_locks=*/ true) { writable_keys.push(key); } else { readonly_keys.push(key); @@ -448,7 +449,8 @@ impl Message { for account_index in &instruction.accounts { let account_index = *account_index as usize; let is_signer = self.is_signer(account_index); - let is_writable = self.is_writable(account_index); + let is_writable = + self.is_writable(account_index, /*demote_program_write_locks=*/ true); let mut meta_byte = 0; if is_signer { meta_byte |= 1 << Self::IS_SIGNER_BIT; @@ -887,12 +889,13 @@ mod tests { recent_blockhash: Hash::default(), instructions: vec![], }; - assert!(message.is_writable(0)); - assert!(!message.is_writable(1)); - assert!(!message.is_writable(2)); - assert!(message.is_writable(3)); - assert!(message.is_writable(4)); - assert!(!message.is_writable(5)); + let demote_program_write_locks = true; + assert!(message.is_writable(0, demote_program_write_locks)); + assert!(!message.is_writable(1, demote_program_write_locks)); + assert!(!message.is_writable(2, demote_program_write_locks)); + assert!(message.is_writable(3, demote_program_write_locks)); + assert!(message.is_writable(4, demote_program_write_locks)); + assert!(!message.is_writable(5, demote_program_write_locks)); } #[test] diff --git a/sdk/program/src/message/mapped.rs b/sdk/program/src/message/mapped.rs new file mode 100644 index 00000000000000..0f30e35238db07 --- /dev/null +++ b/sdk/program/src/message/mapped.rs @@ -0,0 +1,289 @@ +use { + crate::{ + message::{legacy::BUILTIN_PROGRAMS_KEYS, v0}, + pubkey::Pubkey, + sysvar, + }, + std::{collections::HashSet, convert::TryFrom}, +}; + +/// Combination of a version #0 message and its mapped addresses +#[derive(Debug, Clone)] +pub struct MappedMessage { + /// Message which loaded a collection of mapped addresses + pub message: v0::Message, + /// Collection of mapped addresses loaded by this message + pub mapped_addresses: MappedAddresses, +} + +/// Collection of mapped addresses loaded succinctly by a transaction using +/// on-chain address map accounts. +#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] +pub struct MappedAddresses { + /// List of addresses for writable loaded accounts + pub writable: Vec, + /// List of addresses for read-only loaded accounts + pub readonly: Vec, +} + +impl MappedMessage { + /// Returns an iterator of account key segments. The ordering of segments + /// affects how account indexes from compiled instructions are resolved and + /// so should not be changed. + fn account_keys_segment_iter(&self) -> impl Iterator> { + vec![ + &self.message.account_keys, + &self.mapped_addresses.writable, + &self.mapped_addresses.readonly, + ] + .into_iter() + } + + /// Returns the total length of loaded accounts for this message + pub fn account_keys_len(&self) -> usize { + let mut len = 0usize; + for key_segment in self.account_keys_segment_iter() { + len = len.saturating_add(key_segment.len()); + } + len + } + + /// Iterator for the addresses of the loaded accounts for this message + pub fn account_keys_iter(&self) -> impl Iterator { + self.account_keys_segment_iter().flatten() + } + + /// Returns true if any account keys are duplicates + pub fn has_duplicates(&self) -> bool { + let mut uniq = HashSet::new(); + self.account_keys_iter().any(|x| !uniq.insert(x)) + } + + /// Returns the address of the account at the specified index of the list of + /// message account keys constructed from unmapped keys, followed by mapped + /// writable addresses, and lastly the list of mapped readonly addresses. + pub fn get_account_key(&self, mut index: usize) -> Option<&Pubkey> { + for key_segment in self.account_keys_segment_iter() { + if index < key_segment.len() { + return Some(&key_segment[index]); + } + index = index.saturating_sub(key_segment.len()); + } + + None + } + + /// Returns true if the account at the specified index was requested to be + /// writable. This method should not be used directly. + fn is_writable_index(&self, key_index: usize) -> bool { + let header = &self.message.header; + let num_account_keys = self.message.account_keys.len(); + let num_signed_accounts = usize::from(header.num_required_signatures); + if key_index >= num_account_keys { + let mapped_addresses_index = key_index.saturating_sub(num_account_keys); + mapped_addresses_index < self.mapped_addresses.writable.len() + } else if key_index >= num_signed_accounts { + let num_unsigned_accounts = num_account_keys.saturating_sub(num_signed_accounts); + let num_writable_unsigned_accounts = num_unsigned_accounts + .saturating_sub(usize::from(header.num_readonly_unsigned_accounts)); + let unsigned_account_index = key_index.saturating_sub(num_signed_accounts); + unsigned_account_index < num_writable_unsigned_accounts + } else { + let num_writable_signed_accounts = num_signed_accounts + .saturating_sub(usize::from(header.num_readonly_signed_accounts)); + key_index < num_writable_signed_accounts + } + } + + /// Returns true if the account at the specified index was loaded as writable + pub fn is_writable(&self, key_index: usize, demote_program_write_locks: bool) -> bool { + if self.is_writable_index(key_index) { + if let Some(key) = self.get_account_key(key_index) { + return !(sysvar::is_sysvar_id(key) || BUILTIN_PROGRAMS_KEYS.contains(key) + || (demote_program_write_locks && self.is_key_called_as_program(key_index))); + } + } + false + } + + /// Returns true if the account at the specified index is called as a program by an instruction + pub fn is_key_called_as_program(&self, key_index: usize) -> bool { + if let Ok(key_index) = u8::try_from(key_index) { + self.message.instructions + .iter() + .any(|ix| ix.program_id_index == key_index) + } else { + false + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{instruction::CompiledInstruction, message::MessageHeader, system_program, sysvar}; + use itertools::Itertools; + + fn create_test_mapped_message() -> (MappedMessage, [Pubkey; 6]) { + let key0 = Pubkey::new_unique(); + let key1 = Pubkey::new_unique(); + let key2 = Pubkey::new_unique(); + let key3 = Pubkey::new_unique(); + let key4 = Pubkey::new_unique(); + let key5 = Pubkey::new_unique(); + + let message = MappedMessage { + message: v0::Message { + header: MessageHeader { + num_required_signatures: 2, + num_readonly_signed_accounts: 1, + num_readonly_unsigned_accounts: 1, + }, + account_keys: vec![key0, key1, key2, key3], + ..v0::Message::default() + }, + mapped_addresses: MappedAddresses { + writable: vec![key4], + readonly: vec![key5], + }, + }; + + (message, [key0, key1, key2, key3, key4, key5]) + } + + #[test] + fn test_account_keys_segment_iter() { + let (message, keys) = create_test_mapped_message(); + + let expected_segments = vec![ + vec![keys[0], keys[1], keys[2], keys[3]], + vec![keys[4]], + vec![keys[5]], + ]; + + let mut iter = message.account_keys_segment_iter(); + for expected_segment in expected_segments { + assert_eq!(iter.next(), Some(&expected_segment)); + } + } + + #[test] + fn test_account_keys_len() { + let (message, keys) = create_test_mapped_message(); + + assert_eq!(message.account_keys_len(), keys.len()); + } + + #[test] + fn test_account_keys_iter() { + let (message, keys) = create_test_mapped_message(); + + let mut iter = message.account_keys_iter(); + for expected_key in keys { + assert_eq!(iter.next(), Some(&expected_key)); + } + } + + #[test] + fn test_has_duplicates() { + let message = create_test_mapped_message().0; + + assert!(!message.has_duplicates()); + } + + #[test] + fn test_has_duplicates_with_dupe_keys() { + let create_message_with_dupe_keys = |mut keys: Vec| MappedMessage { + message: v0::Message { + account_keys: keys.split_off(2), + ..v0::Message::default() + }, + mapped_addresses: MappedAddresses { + writable: keys.split_off(2), + readonly: keys, + }, + }; + + let key0 = Pubkey::new_unique(); + let key1 = Pubkey::new_unique(); + let key2 = Pubkey::new_unique(); + let key3 = Pubkey::new_unique(); + let dupe_key = Pubkey::new_unique(); + + let keys = vec![key0, key1, key2, key3, dupe_key, dupe_key]; + let keys_len = keys.len(); + for keys in keys.into_iter().permutations(keys_len).unique() { + let message = create_message_with_dupe_keys(keys); + assert!(message.has_duplicates()); + } + } + + #[test] + fn test_get_account_key() { + let (message, keys) = create_test_mapped_message(); + + assert_eq!(message.get_account_key(0), Some(&keys[0])); + assert_eq!(message.get_account_key(1), Some(&keys[1])); + assert_eq!(message.get_account_key(2), Some(&keys[2])); + assert_eq!(message.get_account_key(3), Some(&keys[3])); + assert_eq!(message.get_account_key(4), Some(&keys[4])); + assert_eq!(message.get_account_key(5), Some(&keys[5])); + } + + #[test] + fn test_is_writable_index() { + let message = create_test_mapped_message().0; + + assert!(message.is_writable_index(0)); + assert!(!message.is_writable_index(1)); + assert!(message.is_writable_index(2)); + assert!(!message.is_writable_index(3)); + assert!(message.is_writable_index(4)); + assert!(!message.is_writable_index(5)); + } + + #[test] + fn test_is_writable() { + let mut mapped_msg = create_test_mapped_message().0; + + mapped_msg.message.account_keys[0] = sysvar::clock::id(); + assert!(mapped_msg.is_writable_index(0)); + assert!(!mapped_msg.is_writable(0, /*demote_program_write_locks=*/ true)); + + mapped_msg.message.account_keys[0] = system_program::id(); + assert!(mapped_msg.is_writable_index(0)); + assert!(!mapped_msg.is_writable(0, /*demote_program_write_locks=*/ true)); + } + + #[test] + fn test_demote_writable_program() { + let key0 = Pubkey::new_unique(); + let key1 = Pubkey::new_unique(); + let key2 = Pubkey::new_unique(); + let mapped_msg = MappedMessage { + message: v0::Message { + header: MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 0, + }, + account_keys: vec![key0], + instructions: vec![ + CompiledInstruction { + program_id_index: 2, + accounts: vec![1], + data: vec![], + } + ], + ..v0::Message::default() + }, + mapped_addresses: MappedAddresses { + writable: vec![key1, key2], + readonly: vec![], + }, + }; + + assert!(mapped_msg.is_writable_index(2)); + assert!(!mapped_msg.is_writable(2, /*demote_program_write_locks=*/ true)); + } +} diff --git a/sdk/program/src/message/sanitized.rs b/sdk/program/src/message/sanitized.rs new file mode 100644 index 00000000000000..3b8d3d79755c99 --- /dev/null +++ b/sdk/program/src/message/sanitized.rs @@ -0,0 +1,597 @@ +use { + crate::{ + fee_calculator::FeeCalculator, + hash::Hash, + instruction::{CompiledInstruction, Instruction}, + message::{MappedAddresses, MappedMessage, Message, MessageHeader}, + pubkey::Pubkey, + sanitize::{Sanitize, SanitizeError}, + secp256k1_program, + serialize_utils::{append_slice, append_u16, append_u8}, + }, + bitflags::bitflags, + std::convert::TryFrom, + thiserror::Error, +}; + +/// Sanitized message of a transaction which includes a set of atomic +/// instructions to be executed on-chain +#[derive(Debug, Clone)] +pub enum SanitizedMessage { + /// Sanitized legacy message + Legacy(Message), + /// Sanitized version #0 message with mapped addresses + V0(MappedMessage), +} + +#[derive(PartialEq, Debug, Error, Eq, Clone)] +pub enum SanitizeMessageError { + #[error("index out of bounds")] + IndexOutOfBounds, + #[error("value out of bounds")] + ValueOutOfBounds, + #[error("invalid value")] + InvalidValue, + #[error("duplicate account key")] + DuplicateAccountKey, +} + +impl From for SanitizeMessageError { + fn from(err: SanitizeError) -> Self { + match err { + SanitizeError::IndexOutOfBounds => Self::IndexOutOfBounds, + SanitizeError::ValueOutOfBounds => Self::ValueOutOfBounds, + SanitizeError::InvalidValue => Self::InvalidValue, + } + } +} + +impl TryFrom for SanitizedMessage { + type Error = SanitizeMessageError; + fn try_from(message: Message) -> Result { + message.sanitize()?; + + let sanitized_msg = Self::Legacy(message); + if sanitized_msg.has_duplicates() { + return Err(SanitizeMessageError::DuplicateAccountKey); + } + + Ok(sanitized_msg) + } +} + +bitflags! { + struct InstructionsSysvarAccountMeta: u8 { + const NONE = 0b00000000; + const IS_SIGNER = 0b00000001; + const IS_WRITABLE = 0b00000010; + } +} + +impl SanitizedMessage { + /// Return true if this message contains duplicate account keys + pub fn has_duplicates(&self) -> bool { + match self { + SanitizedMessage::Legacy(message) => message.has_duplicates(), + SanitizedMessage::V0(message) => message.has_duplicates(), + } + } + + /// Message header which identifies the number of signer and writable or + /// readonly accounts + pub fn header(&self) -> &MessageHeader { + match self { + Self::Legacy(message) => &message.header, + Self::V0(mapped_msg) => &mapped_msg.message.header, + } + } + + /// Returns a legacy message if this sanitized message wraps one + pub fn legacy_message(&self) -> Option<&Message> { + if let Self::Legacy(message) = &self { + Some(message) + } else { + None + } + } + + /// Returns the fee payer for the transaction + pub fn fee_payer(&self) -> &Pubkey { + self.get_account_key(0) + .expect("sanitized message always has non-program fee payer at index 0") + } + + /// The hash of a recent block, used for timing out a transaction + pub fn recent_blockhash(&self) -> &Hash { + match self { + Self::Legacy(message) => &message.recent_blockhash, + Self::V0(mapped_msg) => &mapped_msg.message.recent_blockhash, + } + } + + /// Program instructions that will be executed in sequence and committed in + /// one atomic transaction if all succeed. + pub fn instructions(&self) -> &[CompiledInstruction] { + match self { + Self::Legacy(message) => &message.instructions, + Self::V0(mapped_msg) => &mapped_msg.message.instructions, + } + } + + /// Program instructions iterator which includes each instruction's program + /// id. + pub fn program_instructions_iter( + &self, + ) -> impl Iterator { + match self { + Self::Legacy(message) => message.instructions.iter(), + Self::V0(mapped_msg) => mapped_msg.message.instructions.iter(), + } + .map(move |ix| { + ( + self.get_account_key(usize::from(ix.program_id_index)) + .expect("program id index is sanitized"), + ix, + ) + }) + } + + /// Iterator of all account keys referenced in this message, included mapped keys. + pub fn account_keys_iter(&self) -> Box + '_> { + match self { + Self::Legacy(message) => Box::new(message.account_keys.iter()), + Self::V0(mapped_msg) => Box::new(mapped_msg.account_keys_iter()), + } + } + + /// Length of all account keys referenced in this message, included mapped keys. + pub fn account_keys_len(&self) -> usize { + match self { + Self::Legacy(message) => message.account_keys.len(), + Self::V0(mapped_msg) => mapped_msg.account_keys_len(), + } + } + + /// Returns the address of the account at the specified index. + pub fn get_account_key(&self, index: usize) -> Option<&Pubkey> { + match self { + Self::Legacy(message) => message.account_keys.get(index), + Self::V0(message) => message.get_account_key(index), + } + } + + /// Returns true if the account at the specified index is an input to some + /// program instruction in this message. + fn is_key_passed_to_program(&self, key_index: usize) -> bool { + if let Ok(key_index) = u8::try_from(key_index) { + self.instructions() + .iter() + .any(|ix| ix.accounts.contains(&key_index)) + } else { + false + } + } + + /// Returns true if the account at the specified index is invoked as a + /// program in this message. + pub fn is_invoked(&self, key_index: usize) -> bool { + match self { + Self::Legacy(message) => message.is_key_called_as_program(key_index), + Self::V0(message) => message.is_key_called_as_program(key_index), + } + } + + /// Returns true if the account at the specified index is not invoked as a + /// program or, if invoked, is passed to a program. + pub fn is_non_loader_key(&self, key_index: usize) -> bool { + !self.is_invoked(key_index) || self.is_key_passed_to_program(key_index) + } + + /// Returns true if the account at the specified index is writable by the + /// instructions in this message. + pub fn is_writable(&self, index: usize, demote_program_write_locks: bool) -> bool { + match self { + Self::Legacy(message) => message.is_writable(index, demote_program_write_locks), + Self::V0(message) => message.is_writable(index, demote_program_write_locks), + } + } + + /// Returns true if the account at the specified index signed this + /// message. + pub fn is_signer(&self, index: usize) -> bool { + index < usize::from(self.header().num_required_signatures) + } + + // First encode the number of instructions: + // [0..2 - num_instructions + // + // Then a table of offsets of where to find them in the data + // 3..2 * num_instructions table of instruction offsets + // + // Each instruction is then encoded as: + // 0..2 - num_accounts + // 2 - meta_byte -> (bit 0 signer, bit 1 is_writable) + // 3..35 - pubkey - 32 bytes + // 35..67 - program_id + // 67..69 - data len - u16 + // 69..data_len - data + #[allow(clippy::integer_arithmetic)] + pub fn serialize_instructions(&self, demote_program_write_locks: bool) -> Vec { + // 64 bytes is a reasonable guess, calculating exactly is slower in benchmarks + let mut data = Vec::with_capacity(self.instructions().len() * (32 * 2)); + append_u16(&mut data, self.instructions().len() as u16); + for _ in 0..self.instructions().len() { + append_u16(&mut data, 0); + } + for (i, (program_id, instruction)) in self.program_instructions_iter().enumerate() { + let start_instruction_offset = data.len() as u16; + let start = 2 + (2 * i); + data[start..start + 2].copy_from_slice(&start_instruction_offset.to_le_bytes()); + append_u16(&mut data, instruction.accounts.len() as u16); + for account_index in &instruction.accounts { + let account_index = *account_index as usize; + let is_signer = self.is_signer(account_index); + let is_writable = self.is_writable(account_index, demote_program_write_locks); + let mut account_meta = InstructionsSysvarAccountMeta::NONE; + if is_signer { + account_meta |= InstructionsSysvarAccountMeta::IS_SIGNER; + } + if is_writable { + account_meta |= InstructionsSysvarAccountMeta::IS_WRITABLE; + } + append_u8(&mut data, account_meta.bits()); + append_slice( + &mut data, + self.get_account_key(account_index).unwrap().as_ref(), + ); + } + + append_slice(&mut data, program_id.as_ref()); + append_u16(&mut data, instruction.data.len() as u16); + append_slice(&mut data, &instruction.data); + } + data + } + + /// Return the mapped addresses for this message if it has any. + fn mapped_addresses(&self) -> Option<&MappedAddresses> { + match &self { + SanitizedMessage::V0(message) => Some(&message.mapped_addresses), + _ => None, + } + } + + /// Return the number of readonly accounts loaded by this message. + pub fn num_readonly_accounts(&self) -> usize { + let mapped_readonly_addresses = self + .mapped_addresses() + .map(|keys| keys.readonly.len()) + .unwrap_or_default(); + mapped_readonly_addresses + .saturating_add(usize::from(self.header().num_readonly_signed_accounts)) + .saturating_add(usize::from(self.header().num_readonly_unsigned_accounts)) + } + + fn try_position(&self, key: &Pubkey) -> Option { + u8::try_from(self.account_keys_iter().position(|k| k == key)?).ok() + } + + /// Try to compile an instruction using the account keys in this message. + pub fn try_compile_instruction(&self, ix: &Instruction) -> Option { + let accounts: Vec<_> = ix + .accounts + .iter() + .map(|account_meta| self.try_position(&account_meta.pubkey)) + .collect::>()?; + + Some(CompiledInstruction { + program_id_index: self.try_position(&ix.program_id)?, + data: ix.data.clone(), + accounts, + }) + } + + /// Calculate the total fees for a transaction given a fee calculator + pub fn calculate_fee(&self, fee_calculator: &FeeCalculator) -> u64 { + let mut num_secp256k1_signatures: u64 = 0; + for (program_id, instruction) in self.program_instructions_iter() { + if secp256k1_program::check_id(program_id) { + if let Some(num_signatures) = instruction.data.get(0) { + num_secp256k1_signatures = + num_secp256k1_signatures.saturating_add(u64::from(*num_signatures)); + } + } + } + + fee_calculator.lamports_per_signature.saturating_mul( + u64::from(self.header().num_required_signatures) + .saturating_add(num_secp256k1_signatures), + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + instruction::{AccountMeta, Instruction}, + message::v0, + secp256k1_program, system_instruction, + }; + + #[test] + fn test_try_from_message() { + let dupe_key = Pubkey::new_unique(); + let legacy_message_with_dupes = Message { + header: MessageHeader { + num_required_signatures: 1, + ..MessageHeader::default() + }, + account_keys: vec![dupe_key, dupe_key], + ..Message::default() + }; + + assert_eq!( + SanitizedMessage::try_from(legacy_message_with_dupes).err(), + Some(SanitizeMessageError::DuplicateAccountKey), + ); + + let legacy_message_with_no_signers = Message { + account_keys: vec![Pubkey::new_unique()], + ..Message::default() + }; + + assert_eq!( + SanitizedMessage::try_from(legacy_message_with_no_signers).err(), + Some(SanitizeMessageError::IndexOutOfBounds), + ); + } + + #[test] + fn test_is_non_loader_key() { + let key0 = Pubkey::new_unique(); + let key1 = Pubkey::new_unique(); + let loader_key = Pubkey::new_unique(); + let instructions = vec![ + CompiledInstruction::new(1, &(), vec![0]), + CompiledInstruction::new(2, &(), vec![0, 1]), + ]; + + let message = SanitizedMessage::try_from(Message::new_with_compiled_instructions( + 1, + 0, + 2, + vec![key0, key1, loader_key], + Hash::default(), + instructions, + )) + .unwrap(); + + assert!(message.is_non_loader_key(0)); + assert!(message.is_non_loader_key(1)); + assert!(!message.is_non_loader_key(2)); + } + + #[test] + fn test_num_readonly_accounts() { + let key0 = Pubkey::new_unique(); + let key1 = Pubkey::new_unique(); + let key2 = Pubkey::new_unique(); + let key3 = Pubkey::new_unique(); + let key4 = Pubkey::new_unique(); + let key5 = Pubkey::new_unique(); + + let legacy_message = SanitizedMessage::try_from(Message { + header: MessageHeader { + num_required_signatures: 2, + num_readonly_signed_accounts: 1, + num_readonly_unsigned_accounts: 1, + }, + account_keys: vec![key0, key1, key2, key3], + ..Message::default() + }) + .unwrap(); + + assert_eq!(legacy_message.num_readonly_accounts(), 2); + + let mapped_message = SanitizedMessage::V0(MappedMessage { + message: v0::Message { + header: MessageHeader { + num_required_signatures: 2, + num_readonly_signed_accounts: 1, + num_readonly_unsigned_accounts: 1, + }, + account_keys: vec![key0, key1, key2, key3], + ..v0::Message::default() + }, + mapped_addresses: MappedAddresses { + writable: vec![key4], + readonly: vec![key5], + }, + }); + + assert_eq!(mapped_message.num_readonly_accounts(), 3); + } + + #[test] + #[allow(deprecated)] + fn test_serialize_instructions() { + let program_id0 = Pubkey::new_unique(); + let program_id1 = Pubkey::new_unique(); + let id0 = Pubkey::new_unique(); + let id1 = Pubkey::new_unique(); + let id2 = Pubkey::new_unique(); + let id3 = Pubkey::new_unique(); + let instructions = vec![ + Instruction::new_with_bincode(program_id0, &0, vec![AccountMeta::new(id0, false)]), + Instruction::new_with_bincode(program_id0, &0, vec![AccountMeta::new(id1, true)]), + Instruction::new_with_bincode( + program_id1, + &0, + vec![AccountMeta::new_readonly(id2, false)], + ), + Instruction::new_with_bincode( + program_id1, + &0, + vec![AccountMeta::new_readonly(id3, true)], + ), + ]; + + let demote_program_write_locks = true; + let message = Message::new(&instructions, Some(&id1)); + let sanitized_message = SanitizedMessage::try_from(message.clone()).unwrap(); + let serialized = sanitized_message.serialize_instructions(demote_program_write_locks); + + // assert that SanitizedMessage::serialize_instructions has the same behavior as the + // deprecated Message::serialize_instructions method + assert_eq!(serialized, message.serialize_instructions()); + + // assert that Message::deserialize_instruction is compatible with SanitizedMessage::serialize_instructions + for (i, instruction) in instructions.iter().enumerate() { + assert_eq!( + Message::deserialize_instruction(i, &serialized).unwrap(), + *instruction + ); + } + } + + #[test] + fn test_calculate_fee() { + // Default: no fee. + let message = + SanitizedMessage::try_from(Message::new(&[], Some(&Pubkey::new_unique()))).unwrap(); + assert_eq!(message.calculate_fee(&FeeCalculator::default()), 0); + + // One signature, a fee. + assert_eq!(message.calculate_fee(&FeeCalculator::new(1)), 1); + + // Two signatures, double the fee. + let key0 = Pubkey::new_unique(); + let key1 = Pubkey::new_unique(); + let ix0 = system_instruction::transfer(&key0, &key1, 1); + let ix1 = system_instruction::transfer(&key1, &key0, 1); + let message = SanitizedMessage::try_from(Message::new(&[ix0, ix1], Some(&key0))).unwrap(); + assert_eq!(message.calculate_fee(&FeeCalculator::new(2)), 4); + } + + #[test] + fn test_try_compile_instruction() { + let key0 = Pubkey::new_unique(); + let key1 = Pubkey::new_unique(); + let key2 = Pubkey::new_unique(); + let program_id = Pubkey::new_unique(); + + let valid_instruction = Instruction { + program_id, + accounts: vec![ + AccountMeta::new_readonly(key0, false), + AccountMeta::new_readonly(key1, false), + AccountMeta::new_readonly(key2, false), + ], + data: vec![], + }; + + let invalid_program_id_instruction = Instruction { + program_id: Pubkey::new_unique(), + accounts: vec![ + AccountMeta::new_readonly(key0, false), + AccountMeta::new_readonly(key1, false), + AccountMeta::new_readonly(key2, false), + ], + data: vec![], + }; + + let invalid_account_key_instruction = Instruction { + program_id: Pubkey::new_unique(), + accounts: vec![ + AccountMeta::new_readonly(key0, false), + AccountMeta::new_readonly(key1, false), + AccountMeta::new_readonly(Pubkey::new_unique(), false), + ], + data: vec![], + }; + + let legacy_message = SanitizedMessage::try_from(Message { + header: MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 0, + }, + account_keys: vec![key0, key1, key2, program_id], + ..Message::default() + }) + .unwrap(); + + let mapped_message = SanitizedMessage::V0(MappedMessage { + message: v0::Message { + header: MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 0, + }, + account_keys: vec![key0, key1], + ..v0::Message::default() + }, + mapped_addresses: MappedAddresses { + writable: vec![key2], + readonly: vec![program_id], + }, + }); + + for message in vec![legacy_message, mapped_message] { + assert_eq!( + message.try_compile_instruction(&valid_instruction), + Some(CompiledInstruction { + program_id_index: 3, + accounts: vec![0, 1, 2], + data: vec![], + }) + ); + + assert!(message + .try_compile_instruction(&invalid_program_id_instruction) + .is_none()); + assert!(message + .try_compile_instruction(&invalid_account_key_instruction) + .is_none()); + } + } + + #[test] + fn test_calculate_fee_secp256k1() { + let key0 = Pubkey::new_unique(); + let key1 = Pubkey::new_unique(); + let ix0 = system_instruction::transfer(&key0, &key1, 1); + + let mut secp_instruction1 = Instruction { + program_id: secp256k1_program::id(), + accounts: vec![], + data: vec![], + }; + let mut secp_instruction2 = Instruction { + program_id: secp256k1_program::id(), + accounts: vec![], + data: vec![1], + }; + + let message = SanitizedMessage::try_from(Message::new( + &[ + ix0.clone(), + secp_instruction1.clone(), + secp_instruction2.clone(), + ], + Some(&key0), + )) + .unwrap(); + assert_eq!(message.calculate_fee(&FeeCalculator::new(1)), 2); + + secp_instruction1.data = vec![0]; + secp_instruction2.data = vec![10]; + let message = SanitizedMessage::try_from(Message::new( + &[ix0, secp_instruction1, secp_instruction2], + Some(&key0), + )) + .unwrap(); + assert_eq!(message.calculate_fee(&FeeCalculator::new(1)), 11); + } +} diff --git a/sdk/src/feature_set.rs b/sdk/src/feature_set.rs index 1e9f6f96a94402..98ed71a7e85a9e 100644 --- a/sdk/src/feature_set.rs +++ b/sdk/src/feature_set.rs @@ -187,6 +187,10 @@ pub mod stake_program_advance_activating_credits_observed { solana_sdk::declare_id!("SAdVFw3RZvzbo6DvySbSdBnHN4gkzSTH9dSxesyKKPj"); } +pub mod demote_program_write_locks { + solana_sdk::declare_id!("3E3jV7v9VcdJL8iYZUMax9DiDno8j7EWUVbhm9RtShj2"); +} + lazy_static! { /// Map of feature identifiers to user-visible description pub static ref FEATURE_NAMES: HashMap = [ @@ -234,6 +238,7 @@ lazy_static! { (mem_overlap_fix::id(), "Memory overlap fix"), (close_upgradeable_program_accounts::id(), "enable closing upgradeable program accounts"), (stake_program_advance_activating_credits_observed::id(), "Enable advancing credits observed for activation epoch #19309"), + (demote_program_write_locks::id(), "demote program write locks to readonly #19593"), /*************** ADD NEW FEATURES HERE ***************/ ] .iter() diff --git a/sdk/src/transaction/sanitized.rs b/sdk/src/transaction/sanitized.rs new file mode 100644 index 00000000000000..d3ffc77af6da4d --- /dev/null +++ b/sdk/src/transaction/sanitized.rs @@ -0,0 +1,231 @@ +#![cfg(feature = "full")] + +use { + crate::{ + hash::Hash, + message::{v0, MappedAddresses, MappedMessage, SanitizedMessage, VersionedMessage}, + nonce::NONCED_TX_MARKER_IX_INDEX, + program_utils::limited_deserialize, + pubkey::Pubkey, + sanitize::Sanitize, + secp256k1_instruction::verify_eth_addresses, + secp256k1_program, + signature::Signature, + transaction::{Result, Transaction, TransactionError, VersionedTransaction}, + }, + solana_program::{system_instruction::SystemInstruction, system_program}, + std::convert::TryFrom, +}; + +/// Sanitized transaction and the hash of its message +#[derive(Debug, Clone)] +pub struct SanitizedTransaction { + message: SanitizedMessage, + message_hash: Hash, + signatures: Vec, +} + +/// Set of accounts that must be locked for safe transaction processing +#[derive(Debug, Clone, Default)] +pub struct TransactionAccountLocks<'a> { + /// List of readonly account key locks + pub readonly: Vec<&'a Pubkey>, + /// List of writable account key locks + pub writable: Vec<&'a Pubkey>, +} + +impl TryFrom for SanitizedTransaction { + type Error = TransactionError; + fn try_from(tx: Transaction) -> Result { + tx.sanitize()?; + + if tx.message.has_duplicates() { + return Err(TransactionError::AccountLoadedTwice); + } + + Ok(Self { + message_hash: tx.message.hash(), + message: SanitizedMessage::Legacy(tx.message), + signatures: tx.signatures, + }) + } +} + +impl SanitizedTransaction { + /// Create a sanitized transaction from an unsanitized transaction. + /// If the input transaction uses address maps, attempt to map the + /// transaction keys to full addresses. + pub fn try_create( + tx: VersionedTransaction, + message_hash: Hash, + address_mapper: impl Fn(&v0::Message) -> Result, + ) -> Result { + tx.sanitize()?; + + let signatures = tx.signatures; + let message = match tx.message { + VersionedMessage::Legacy(message) => SanitizedMessage::Legacy(message), + VersionedMessage::V0(message) => SanitizedMessage::V0(MappedMessage { + mapped_addresses: address_mapper(&message)?, + message, + }), + }; + + if message.has_duplicates() { + return Err(TransactionError::AccountLoadedTwice); + } + + Ok(Self { + message, + message_hash, + signatures, + }) + } + + /// Return the first signature for this transaction. + /// + /// Notes: + /// + /// Sanitized transactions must have at least one signature because the + /// number of signatures must be greater than or equal to the message header + /// value `num_required_signatures` which must be greater than 0 itself. + pub fn signature(&self) -> &Signature { + &self.signatures[0] + } + + /// Return the list of signatures for this transaction + pub fn signatures(&self) -> &[Signature] { + &self.signatures + } + + /// Return the signed message + pub fn message(&self) -> &SanitizedMessage { + &self.message + } + + /// Return the hash of the signed message + pub fn message_hash(&self) -> &Hash { + &self.message_hash + } + + /// Convert this sanitized transaction into a versioned transaction for + /// recording in the ledger. + pub fn to_versioned_transaction(&self) -> VersionedTransaction { + let signatures = self.signatures.clone(); + match &self.message { + SanitizedMessage::V0(mapped_msg) => VersionedTransaction { + signatures, + message: VersionedMessage::V0(mapped_msg.message.clone()), + }, + SanitizedMessage::Legacy(message) => VersionedTransaction { + signatures, + message: VersionedMessage::Legacy(message.clone()), + }, + } + } + + /// Return the list of accounts that must be locked during processing this transaction. + pub fn get_account_locks(&self, demote_program_write_locks: bool) -> TransactionAccountLocks { + let message = &self.message; + let num_readonly_accounts = message.num_readonly_accounts(); + let num_writable_accounts = message + .account_keys_len() + .saturating_sub(num_readonly_accounts); + + let mut account_locks = TransactionAccountLocks { + writable: Vec::with_capacity(num_writable_accounts), + readonly: Vec::with_capacity(num_readonly_accounts), + }; + + for (i, key) in message.account_keys_iter().enumerate() { + if message.is_writable(i, demote_program_write_locks) { + account_locks.writable.push(key); + } else { + account_locks.readonly.push(key); + } + } + + account_locks + } + + /// If the transaction uses a durable nonce, return the pubkey of the nonce account + pub fn get_durable_nonce(&self) -> Option<&Pubkey> { + self.message + .instructions() + .get(NONCED_TX_MARKER_IX_INDEX as usize) + .filter( + |ix| match self.message.get_account_key(ix.program_id_index as usize) { + Some(program_id) => system_program::check_id(program_id), + _ => false, + }, + ) + .filter(|ix| { + matches!( + limited_deserialize(&ix.data), + Ok(SystemInstruction::AdvanceNonceAccount) + ) + }) + .and_then(|ix| { + ix.accounts.get(0).and_then(|idx| { + let idx = *idx as usize; + self.message.get_account_key(idx) + }) + }) + } + + /// Return the serialized message data to sign. + fn message_data(&self) -> Vec { + match &self.message { + SanitizedMessage::Legacy(message) => message.serialize(), + SanitizedMessage::V0(mapped_msg) => mapped_msg.message.serialize(), + } + } + + /// Verify the length of signatures matches the value in the message header + pub fn verify_signatures_len(&self) -> bool { + self.signatures.len() == self.message.header().num_required_signatures as usize + } + + /// Verify the transaction signatures + pub fn verify(&self) -> Result<()> { + let message_bytes = self.message_data(); + if self + .signatures + .iter() + .zip(self.message.account_keys_iter()) + .map(|(signature, pubkey)| signature.verify(pubkey.as_ref(), &message_bytes)) + .any(|verified| !verified) + { + Err(TransactionError::SignatureFailure) + } else { + Ok(()) + } + } + + /// Verify the encoded secp256k1 signatures in this transaction + pub fn verify_precompiles( + &self, + libsecp256k1_0_5_upgrade_enabled: bool, + libsecp256k1_fail_on_bad_count: bool, + ) -> Result<()> { + for (program_id, instruction) in self.message.program_instructions_iter() { + if secp256k1_program::check_id(program_id) { + let instruction_datas: Vec<_> = self + .message + .instructions() + .iter() + .map(|instruction| instruction.data.as_ref()) + .collect(); + let data = &instruction.data; + let e = verify_eth_addresses( + data, + &instruction_datas, + libsecp256k1_0_5_upgrade_enabled, + libsecp256k1_fail_on_bad_count, + ); + e.map_err(|_| TransactionError::InvalidAccountIndex)?; + } + } + Ok(()) + } +} diff --git a/transaction-status/src/parse_accounts.rs b/transaction-status/src/parse_accounts.rs index ebef018e716695..197643450b6eab 100644 --- a/transaction-status/src/parse_accounts.rs +++ b/transaction-status/src/parse_accounts.rs @@ -13,7 +13,7 @@ pub fn parse_accounts(message: &Message) -> Vec { for (i, account_key) in message.account_keys.iter().enumerate() { accounts.push(ParsedAccount { pubkey: account_key.to_string(), - writable: message.is_writable(i), + writable: message.is_writable(i, /*demote_program_write_locks=*/ true), signer: message.is_signer(i), }); } From 1c238a1a7f3cae7700f61d3e9954d9d5fb269c6d Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Fri, 3 Sep 2021 22:10:26 -0600 Subject: [PATCH 2/2] Fix conflicts --- core/src/banking_stage.rs | 91 -- core/src/cost_model.rs | 504 ------- core/src/cost_tracker.rs | 423 ------ ledger-tool/src/main.rs | 72 - program-runtime/src/instruction_processor.rs | 1242 ------------------ programs/bpf/tests/programs.rs | 94 -- programs/bpf_loader/src/syscalls.rs | 12 +- rpc/src/transaction_status_service.rs | 11 +- runtime/src/accounts.rs | 67 +- runtime/src/bank.rs | 70 +- runtime/src/message_processor.rs | 33 +- sdk/benches/serialize_instructions.rs | 14 - sdk/program/src/message.rs | 28 +- sdk/program/src/message/mapped.rs | 289 ---- sdk/program/src/message/sanitized.rs | 597 --------- sdk/src/transaction/sanitized.rs | 231 ---- 16 files changed, 70 insertions(+), 3708 deletions(-) delete mode 100644 core/src/cost_model.rs delete mode 100644 core/src/cost_tracker.rs delete mode 100644 program-runtime/src/instruction_processor.rs delete mode 100644 sdk/program/src/message/mapped.rs delete mode 100644 sdk/program/src/message/sanitized.rs delete mode 100644 sdk/src/transaction/sanitized.rs diff --git a/core/src/banking_stage.rs b/core/src/banking_stage.rs index 007021f7d6a353..3215bc67f57695 100644 --- a/core/src/banking_stage.rs +++ b/core/src/banking_stage.rs @@ -965,19 +965,8 @@ impl BankingStage { msgs: &Packets, transaction_indexes: &[usize], libsecp256k1_0_5_upgrade_enabled: bool, -<<<<<<< HEAD ) -> (Vec>, Vec) { transaction_indexes -======= - libsecp256k1_fail_on_bad_count: bool, - cost_tracker: &Arc>, - banking_stage_stats: &BankingStageStats, - demote_program_write_locks: bool, - ) -> (Vec, Vec, Vec) { - let mut retryable_transaction_packet_indexes: Vec = vec![]; - - let verified_transactions_with_packet_indexes: Vec<_> = transaction_indexes ->>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) .iter() .filter_map(|tx_index| { let p = &msgs.packets[*tx_index]; @@ -991,44 +980,7 @@ impl BankingStage { tx_index, )) }) -<<<<<<< HEAD .unzip() -======= - .collect(); - banking_stage_stats.cost_tracker_check_count.fetch_add( - verified_transactions_with_packet_indexes.len(), - Ordering::Relaxed, - ); - - let mut cost_tracker_check_time = Measure::start("cost_tracker_check_time"); - let (filtered_transactions, filter_transaction_packet_indexes) = { - let cost_tracker_readonly = cost_tracker.read().unwrap(); - verified_transactions_with_packet_indexes - .into_iter() - .filter_map(|(tx, tx_index)| { - let result = cost_tracker_readonly - .would_transaction_fit(&tx, demote_program_write_locks); - if result.is_err() { - debug!("transaction {:?} would exceed limit: {:?}", tx, result); - retryable_transaction_packet_indexes.push(tx_index); - return None; - } - Some((tx, tx_index)) - }) - .unzip() - }; - cost_tracker_check_time.stop(); - - banking_stage_stats - .cost_tracker_check_elapsed - .fetch_add(cost_tracker_check_time.as_us(), Ordering::Relaxed); - - ( - filtered_transactions, - filter_transaction_packet_indexes, - retryable_transaction_packet_indexes, - ) ->>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) } /// This function filters pending packets that are still valid @@ -1081,24 +1033,11 @@ impl BankingStage { banking_stage_stats: &BankingStageStats, ) -> (usize, usize, Vec) { let mut packet_conversion_time = Measure::start("packet_conversion"); -<<<<<<< HEAD let (transactions, transaction_to_packet_indexes) = Self::transactions_from_packets( msgs, &packet_indexes, bank.libsecp256k1_0_5_upgrade_enabled(), ); -======= - let (transactions, transaction_to_packet_indexes, retryable_packet_indexes) = - Self::transactions_from_packets( - msgs, - &packet_indexes, - bank.libsecp256k1_0_5_upgrade_enabled(), - bank.libsecp256k1_fail_on_bad_count(), - cost_tracker, - banking_stage_stats, - bank.demote_program_write_locks(), - ); ->>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) packet_conversion_time.stop(); debug!( @@ -1120,21 +1059,7 @@ impl BankingStage { ); process_tx_time.stop(); -<<<<<<< HEAD let unprocessed_tx_count = unprocessed_tx_indexes.len(); -======= - // applying cost of processed transactions to shared cost_tracker - let mut cost_tracking_time = Measure::start("cost_tracking_time"); - transactions.iter().enumerate().for_each(|(index, tx)| { - if unprocessed_tx_indexes.iter().all(|&i| i != index) { - cost_tracker - .write() - .unwrap() - .add_transaction_cost(tx, bank.demote_program_write_locks()); - } - }); - cost_tracking_time.stop(); ->>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) let mut filter_pending_packets_time = Measure::start("filter_pending_packets_time"); let filtered_unprocessed_packet_indexes = Self::filter_pending_packets_from_pending_txs( @@ -1179,27 +1104,11 @@ impl BankingStage { } } -<<<<<<< HEAD let (transactions, transaction_to_packet_indexes) = Self::transactions_from_packets( msgs, &transaction_indexes, bank.libsecp256k1_0_5_upgrade_enabled(), ); -======= - let mut unprocessed_packet_conversion_time = - Measure::start("unprocessed_packet_conversion"); - let (transactions, transaction_to_packet_indexes, retry_packet_indexes) = - Self::transactions_from_packets( - msgs, - transaction_indexes, - bank.libsecp256k1_0_5_upgrade_enabled(), - bank.libsecp256k1_fail_on_bad_count(), - cost_tracker, - banking_stage_stats, - bank.demote_program_write_locks(), - ); - unprocessed_packet_conversion_time.stop(); ->>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) let tx_count = transaction_to_packet_indexes.len(); diff --git a/core/src/cost_model.rs b/core/src/cost_model.rs deleted file mode 100644 index ade82ea8e746b1..00000000000000 --- a/core/src/cost_model.rs +++ /dev/null @@ -1,504 +0,0 @@ -//! 'cost_model` provides service to estimate a transaction's cost -//! It does so by analyzing accounts the transaction touches, and instructions -//! it includes. Using historical data as guideline, it estimates cost of -//! reading/writing account, the sum of that comes up to "account access cost"; -//! Instructions take time to execute, both historical and runtime data are -//! used to determine each instruction's execution time, the sum of that -//! is transaction's "execution cost" -//! The main function is `calculate_cost` which returns &TransactionCost. -//! -use crate::execute_cost_table::ExecuteCostTable; -use log::*; -use solana_ledger::block_cost_limits::*; -use solana_sdk::{pubkey::Pubkey, transaction::SanitizedTransaction}; -use std::collections::HashMap; - -const MAX_WRITABLE_ACCOUNTS: usize = 256; - -#[derive(Debug, Clone)] -pub enum CostModelError { - /// transaction that would fail sanitize, cost model is not able to process - /// such transaction. - InvalidTransaction, - - /// would exceed block max limit - WouldExceedBlockMaxLimit, - - /// would exceed account max limit - WouldExceedAccountMaxLimit, -} - -// cost of transaction is made of account_access_cost and instruction execution_cost -// where -// account_access_cost is the sum of read/write/sign all accounts included in the transaction -// read is cheaper than write. -// execution_cost is the sum of all instructions execution cost, which is -// observed during runtime and feedback by Replay -#[derive(Default, Debug)] -pub struct TransactionCost { - pub writable_accounts: Vec, - pub account_access_cost: u64, - pub execution_cost: u64, -} - -impl TransactionCost { - pub fn new_with_capacity(capacity: usize) -> Self { - Self { - writable_accounts: Vec::with_capacity(capacity), - ..Self::default() - } - } - - pub fn reset(&mut self) { - self.writable_accounts.clear(); - self.account_access_cost = 0; - self.execution_cost = 0; - } -} - -#[derive(Debug)] -pub struct CostModel { - account_cost_limit: u64, - block_cost_limit: u64, - instruction_execution_cost_table: ExecuteCostTable, - - // reusable variables - transaction_cost: TransactionCost, -} - -impl Default for CostModel { - fn default() -> Self { - CostModel::new(ACCOUNT_COST_MAX, BLOCK_COST_MAX) - } -} - -impl CostModel { - pub fn new(chain_max: u64, block_max: u64) -> Self { - Self { - account_cost_limit: chain_max, - block_cost_limit: block_max, - instruction_execution_cost_table: ExecuteCostTable::default(), - transaction_cost: TransactionCost::new_with_capacity(MAX_WRITABLE_ACCOUNTS), - } - } - - pub fn get_account_cost_limit(&self) -> u64 { - self.account_cost_limit - } - - pub fn get_block_cost_limit(&self) -> u64 { - self.block_cost_limit - } - - pub fn initialize_cost_table(&mut self, cost_table: &[(Pubkey, u64)]) { - for (program_id, cost) in cost_table { - match self.upsert_instruction_cost(program_id, *cost) { - Ok(c) => { - debug!( - "initiating cost table, instruction {:?} has cost {}", - program_id, c - ); - } - Err(err) => { - debug!( - "initiating cost table, failed for instruction {:?}, err: {}", - program_id, err - ); - } - } - } - debug!( - "restored cost model instruction cost table from blockstore, current values: {:?}", - self.get_instruction_cost_table() - ); - } - - pub fn calculate_cost( - &mut self, - transaction: &SanitizedTransaction, - demote_program_write_locks: bool, - ) -> &TransactionCost { - self.transaction_cost.reset(); - - // calculate transaction exeution cost - self.transaction_cost.execution_cost = self.find_transaction_cost(transaction); - - // calculate account access cost - let message = transaction.message(); - message.account_keys_iter().enumerate().for_each(|(i, k)| { - let is_writable = message.is_writable(i, demote_program_write_locks); - - if is_writable { - self.transaction_cost.writable_accounts.push(*k); - self.transaction_cost.account_access_cost += ACCOUNT_WRITE_COST; - } else { - self.transaction_cost.account_access_cost += ACCOUNT_READ_COST; - } - }); - debug!( - "transaction {:?} has cost {:?}", - transaction, self.transaction_cost - ); - &self.transaction_cost - } - - // To update or insert instruction cost to table. - pub fn upsert_instruction_cost( - &mut self, - program_key: &Pubkey, - cost: u64, - ) -> Result { - self.instruction_execution_cost_table - .upsert(program_key, cost); - match self.instruction_execution_cost_table.get_cost(program_key) { - Some(cost) => Ok(*cost), - None => Err("failed to upsert to ExecuteCostTable"), - } - } - - pub fn get_instruction_cost_table(&self) -> &HashMap { - self.instruction_execution_cost_table.get_cost_table() - } - - fn find_instruction_cost(&self, program_key: &Pubkey) -> u64 { - match self.instruction_execution_cost_table.get_cost(program_key) { - Some(cost) => *cost, - None => { - let default_value = self.instruction_execution_cost_table.get_mode(); - debug!( - "Program key {:?} does not have assigned cost, using mode {}", - program_key, default_value - ); - default_value - } - } - } - - fn find_transaction_cost(&self, transaction: &SanitizedTransaction) -> u64 { - let mut cost: u64 = 0; - - for (program_id, instruction) in transaction.message().program_instructions_iter() { - let instruction_cost = self.find_instruction_cost(program_id); - trace!( - "instruction {:?} has cost of {}", - instruction, - instruction_cost - ); - cost += instruction_cost; - } - cost - } -} - -#[cfg(test)] -mod tests { - use super::*; - use solana_runtime::{ - bank::Bank, - genesis_utils::{create_genesis_config, GenesisConfigInfo}, - }; - use solana_sdk::{ - bpf_loader, - hash::Hash, - instruction::CompiledInstruction, - message::Message, - signature::{Keypair, Signer}, - system_instruction::{self}, - system_program, system_transaction, - transaction::Transaction, - }; - use std::{ - convert::{TryFrom, TryInto}, - str::FromStr, - sync::{Arc, RwLock}, - thread::{self, JoinHandle}, - }; - - fn test_setup() -> (Keypair, Hash) { - solana_logger::setup(); - let GenesisConfigInfo { - genesis_config, - mint_keypair, - .. - } = create_genesis_config(10); - let bank = Arc::new(Bank::new_no_wallclock_throttle_for_tests(&genesis_config)); - let start_hash = bank.last_blockhash(); - (mint_keypair, start_hash) - } - - #[test] - fn test_cost_model_instruction_cost() { - let mut testee = CostModel::default(); - - let known_key = Pubkey::from_str("known11111111111111111111111111111111111111").unwrap(); - testee.upsert_instruction_cost(&known_key, 100).unwrap(); - // find cost for known programs - assert_eq!(100, testee.find_instruction_cost(&known_key)); - - testee - .upsert_instruction_cost(&bpf_loader::id(), 1999) - .unwrap(); - assert_eq!(1999, testee.find_instruction_cost(&bpf_loader::id())); - - // unknown program is assigned with default cost - assert_eq!( - testee.instruction_execution_cost_table.get_mode(), - testee.find_instruction_cost( - &Pubkey::from_str("unknown111111111111111111111111111111111111").unwrap() - ) - ); - } - - #[test] - fn test_cost_model_simple_transaction() { - let (mint_keypair, start_hash) = test_setup(); - - let keypair = Keypair::new(); - let simple_transaction: SanitizedTransaction = - system_transaction::transfer(&mint_keypair, &keypair.pubkey(), 2, start_hash) - .try_into() - .unwrap(); - debug!( - "system_transaction simple_transaction {:?}", - simple_transaction - ); - - // expected cost for one system transfer instructions - let expected_cost = 8; - - let mut testee = CostModel::default(); - testee - .upsert_instruction_cost(&system_program::id(), expected_cost) - .unwrap(); - assert_eq!( - expected_cost, - testee.find_transaction_cost(&simple_transaction) - ); - } - - #[test] - fn test_cost_model_transaction_many_transfer_instructions() { - let (mint_keypair, start_hash) = test_setup(); - - let key1 = solana_sdk::pubkey::new_rand(); - let key2 = solana_sdk::pubkey::new_rand(); - let instructions = - system_instruction::transfer_many(&mint_keypair.pubkey(), &[(key1, 1), (key2, 1)]); - let message = Message::new(&instructions, Some(&mint_keypair.pubkey())); - let tx: SanitizedTransaction = Transaction::new(&[&mint_keypair], message, start_hash) - .try_into() - .unwrap(); - debug!("many transfer transaction {:?}", tx); - - // expected cost for two system transfer instructions - let program_cost = 8; - let expected_cost = program_cost * 2; - - let mut testee = CostModel::default(); - testee - .upsert_instruction_cost(&system_program::id(), program_cost) - .unwrap(); - assert_eq!(expected_cost, testee.find_transaction_cost(&tx)); - } - - #[test] - fn test_cost_model_message_many_different_instructions() { - let (mint_keypair, start_hash) = test_setup(); - - // construct a transaction with multiple random instructions - let key1 = solana_sdk::pubkey::new_rand(); - let key2 = solana_sdk::pubkey::new_rand(); - let prog1 = solana_sdk::pubkey::new_rand(); - let prog2 = solana_sdk::pubkey::new_rand(); - let instructions = vec![ - CompiledInstruction::new(3, &(), vec![0, 1]), - CompiledInstruction::new(4, &(), vec![0, 2]), - ]; - let tx: SanitizedTransaction = Transaction::new_with_compiled_instructions( - &[&mint_keypair], - &[key1, key2], - start_hash, - vec![prog1, prog2], - instructions, - ) - .try_into() - .unwrap(); - debug!("many random transaction {:?}", tx); - - let testee = CostModel::default(); - let result = testee.find_transaction_cost(&tx); - - // expected cost for two random/unknown program is - let expected_cost = testee.instruction_execution_cost_table.get_mode() * 2; - assert_eq!(expected_cost, result); - } - - #[test] - fn test_cost_model_sort_message_accounts_by_type() { - // construct a transaction with two random instructions with same signer - let signer1 = Keypair::new(); - let signer2 = Keypair::new(); - let key1 = Pubkey::new_unique(); - let key2 = Pubkey::new_unique(); - let prog1 = Pubkey::new_unique(); - let prog2 = Pubkey::new_unique(); - let instructions = vec![ - CompiledInstruction::new(4, &(), vec![0, 2]), - CompiledInstruction::new(5, &(), vec![1, 3]), - ]; - let tx: SanitizedTransaction = Transaction::new_with_compiled_instructions( - &[&signer1, &signer2], - &[key1, key2], - Hash::new_unique(), - vec![prog1, prog2], - instructions, - ) - .try_into() - .unwrap(); - - let mut cost_model = CostModel::default(); - let tx_cost = cost_model.calculate_cost(&tx, /*demote_program_write_locks=*/ true); - assert_eq!(2 + 2, tx_cost.writable_accounts.len()); - assert_eq!(signer1.pubkey(), tx_cost.writable_accounts[0]); - assert_eq!(signer2.pubkey(), tx_cost.writable_accounts[1]); - assert_eq!(key1, tx_cost.writable_accounts[2]); - assert_eq!(key2, tx_cost.writable_accounts[3]); - } - - #[test] - fn test_cost_model_insert_instruction_cost() { - let key1 = Pubkey::new_unique(); - let cost1 = 100; - - let mut cost_model = CostModel::default(); - // Using default cost for unknown instruction - assert_eq!( - cost_model.instruction_execution_cost_table.get_mode(), - cost_model.find_instruction_cost(&key1) - ); - - // insert instruction cost to table - assert!(cost_model.upsert_instruction_cost(&key1, cost1).is_ok()); - - // now it is known insturction with known cost - assert_eq!(cost1, cost_model.find_instruction_cost(&key1)); - } - - #[test] - fn test_cost_model_calculate_cost() { - let (mint_keypair, start_hash) = test_setup(); - let tx: SanitizedTransaction = - system_transaction::transfer(&mint_keypair, &Keypair::new().pubkey(), 2, start_hash) - .try_into() - .unwrap(); - - let expected_account_cost = ACCOUNT_WRITE_COST + ACCOUNT_WRITE_COST + ACCOUNT_READ_COST; - let expected_execution_cost = 8; - - let mut cost_model = CostModel::default(); - cost_model - .upsert_instruction_cost(&system_program::id(), expected_execution_cost) - .unwrap(); - let tx_cost = cost_model.calculate_cost(&tx, /*demote_program_write_locks=*/ true); - assert_eq!(expected_account_cost, tx_cost.account_access_cost); - assert_eq!(expected_execution_cost, tx_cost.execution_cost); - assert_eq!(2, tx_cost.writable_accounts.len()); - } - - #[test] - fn test_cost_model_update_instruction_cost() { - let key1 = Pubkey::new_unique(); - let cost1 = 100; - let cost2 = 200; - let updated_cost = (cost1 + cost2) / 2; - - let mut cost_model = CostModel::default(); - - // insert instruction cost to table - assert!(cost_model.upsert_instruction_cost(&key1, cost1).is_ok()); - assert_eq!(cost1, cost_model.find_instruction_cost(&key1)); - - // update instruction cost - assert!(cost_model.upsert_instruction_cost(&key1, cost2).is_ok()); - assert_eq!(updated_cost, cost_model.find_instruction_cost(&key1)); - } - - #[test] - fn test_cost_model_can_be_shared_concurrently_with_rwlock() { - let (mint_keypair, start_hash) = test_setup(); - // construct a transaction with multiple random instructions - let key1 = solana_sdk::pubkey::new_rand(); - let key2 = solana_sdk::pubkey::new_rand(); - let prog1 = solana_sdk::pubkey::new_rand(); - let prog2 = solana_sdk::pubkey::new_rand(); - let instructions = vec![ - CompiledInstruction::new(3, &(), vec![0, 1]), - CompiledInstruction::new(4, &(), vec![0, 2]), - ]; - let tx = Arc::new( - SanitizedTransaction::try_from(Transaction::new_with_compiled_instructions( - &[&mint_keypair], - &[key1, key2], - start_hash, - vec![prog1, prog2], - instructions, - )) - .unwrap(), - ); - - let number_threads = 10; - let expected_account_cost = - ACCOUNT_WRITE_COST + ACCOUNT_WRITE_COST * 2 + ACCOUNT_READ_COST * 2; - let cost1 = 100; - let cost2 = 200; - // execution cost can be either 2 * Default (before write) or cost1+cost2 (after write) - - let cost_model: Arc> = Arc::new(RwLock::new(CostModel::default())); - - let thread_handlers: Vec> = (0..number_threads) - .map(|i| { - let cost_model = cost_model.clone(); - let tx = tx.clone(); - - if i == 5 { - thread::spawn(move || { - let mut cost_model = cost_model.write().unwrap(); - assert!(cost_model.upsert_instruction_cost(&prog1, cost1).is_ok()); - assert!(cost_model.upsert_instruction_cost(&prog2, cost2).is_ok()); - }) - } else { - thread::spawn(move || { - let mut cost_model = cost_model.write().unwrap(); - let tx_cost = cost_model - .calculate_cost(&tx, /*demote_program_write_locks=*/ true); - assert_eq!(3, tx_cost.writable_accounts.len()); - assert_eq!(expected_account_cost, tx_cost.account_access_cost); - }) - } - }) - .collect(); - - for th in thread_handlers { - th.join().unwrap(); - } - } - - #[test] - fn test_cost_model_init_cost_table() { - // build cost table - let cost_table = vec![ - (Pubkey::new_unique(), 10), - (Pubkey::new_unique(), 20), - (Pubkey::new_unique(), 30), - ]; - - // init cost model - let mut cost_model = CostModel::default(); - cost_model.initialize_cost_table(&cost_table); - - // verify - for (id, cost) in cost_table.iter() { - assert_eq!(*cost, cost_model.find_instruction_cost(id)); - } - } -} diff --git a/core/src/cost_tracker.rs b/core/src/cost_tracker.rs deleted file mode 100644 index 40a86133adb7d3..00000000000000 --- a/core/src/cost_tracker.rs +++ /dev/null @@ -1,423 +0,0 @@ -//! `cost_tracker` keeps tracking transaction cost per chained accounts as well as for entire block -//! It aggregates `cost_model`, which provides service of calculating transaction cost. -//! The main functions are: -//! - would_transaction_fit(&tx), immutable function to test if `tx` would fit into current block -//! - add_transaction_cost(&tx), mutable function to accumulate `tx` cost to tracker. -//! -use crate::cost_model::{CostModel, CostModelError, TransactionCost}; -use solana_sdk::{clock::Slot, pubkey::Pubkey, transaction::SanitizedTransaction}; -use std::{ - collections::HashMap, - sync::{Arc, RwLock}, -}; - -const WRITABLE_ACCOUNTS_PER_BLOCK: usize = 512; - -#[derive(Debug)] -pub struct CostTracker { - cost_model: Arc>, - account_cost_limit: u64, - block_cost_limit: u64, - current_bank_slot: Slot, - cost_by_writable_accounts: HashMap, - block_cost: u64, -} - -impl CostTracker { - pub fn new(cost_model: Arc>) -> Self { - let (account_cost_limit, block_cost_limit) = { - let cost_model = cost_model.read().unwrap(); - ( - cost_model.get_account_cost_limit(), - cost_model.get_block_cost_limit(), - ) - }; - assert!(account_cost_limit <= block_cost_limit); - Self { - cost_model, - account_cost_limit, - block_cost_limit, - current_bank_slot: 0, - cost_by_writable_accounts: HashMap::with_capacity(WRITABLE_ACCOUNTS_PER_BLOCK), - block_cost: 0, - } - } - - pub fn would_transaction_fit( - &self, - transaction: &SanitizedTransaction, - demote_program_write_locks: bool, - ) -> Result<(), CostModelError> { - let mut cost_model = self.cost_model.write().unwrap(); - let tx_cost = cost_model.calculate_cost(transaction, demote_program_write_locks); - self.would_fit( - &tx_cost.writable_accounts, - &(tx_cost.account_access_cost + tx_cost.execution_cost), - ) - } - - pub fn add_transaction_cost( - &mut self, - transaction: &SanitizedTransaction, - demote_program_write_locks: bool, - ) { - let mut cost_model = self.cost_model.write().unwrap(); - let tx_cost = cost_model.calculate_cost(transaction, demote_program_write_locks); - let cost = tx_cost.account_access_cost + tx_cost.execution_cost; - for account_key in tx_cost.writable_accounts.iter() { - *self - .cost_by_writable_accounts - .entry(*account_key) - .or_insert(0) += cost; - } - self.block_cost += cost; - } - - pub fn reset_if_new_bank(&mut self, slot: Slot) { - if slot != self.current_bank_slot { - self.current_bank_slot = slot; - self.cost_by_writable_accounts.clear(); - self.block_cost = 0; - } - } - - pub fn try_add(&mut self, transaction_cost: &TransactionCost) -> Result { - let cost = transaction_cost.account_access_cost + transaction_cost.execution_cost; - self.would_fit(&transaction_cost.writable_accounts, &cost)?; - - self.add_transaction(&transaction_cost.writable_accounts, &cost); - Ok(self.block_cost) - } - - fn would_fit(&self, keys: &[Pubkey], cost: &u64) -> Result<(), CostModelError> { - // check against the total package cost - if self.block_cost + cost > self.block_cost_limit { - return Err(CostModelError::WouldExceedBlockMaxLimit); - } - - // check if the transaction itself is more costly than the account_cost_limit - if *cost > self.account_cost_limit { - return Err(CostModelError::WouldExceedAccountMaxLimit); - } - - // check each account against account_cost_limit, - for account_key in keys.iter() { - match self.cost_by_writable_accounts.get(account_key) { - Some(chained_cost) => { - if chained_cost + cost > self.account_cost_limit { - return Err(CostModelError::WouldExceedAccountMaxLimit); - } else { - continue; - } - } - None => continue, - } - } - - Ok(()) - } - - fn add_transaction(&mut self, keys: &[Pubkey], cost: &u64) { - for account_key in keys.iter() { - *self - .cost_by_writable_accounts - .entry(*account_key) - .or_insert(0) += cost; - } - self.block_cost += cost; - } -} - -// CostStats can be collected by util, such as ledger_tool -#[derive(Default, Debug)] -pub struct CostStats { - pub bank_slot: Slot, - pub total_cost: u64, - pub number_of_accounts: usize, - pub costliest_account: Pubkey, - pub costliest_account_cost: u64, -} - -impl CostTracker { - pub fn get_stats(&self) -> CostStats { - let mut stats = CostStats { - bank_slot: self.current_bank_slot, - total_cost: self.block_cost, - number_of_accounts: self.cost_by_writable_accounts.len(), - costliest_account: Pubkey::default(), - costliest_account_cost: 0, - }; - - for (key, cost) in self.cost_by_writable_accounts.iter() { - if cost > &stats.costliest_account_cost { - stats.costliest_account = *key; - stats.costliest_account_cost = *cost; - } - } - - stats - } -} - -#[cfg(test)] -mod tests { - use super::*; - use solana_runtime::{ - bank::Bank, - genesis_utils::{create_genesis_config, GenesisConfigInfo}, - }; - use solana_sdk::{ - hash::Hash, - signature::{Keypair, Signer}, - system_transaction, - transaction::Transaction, - }; - use std::{cmp, sync::Arc}; - - fn test_setup() -> (Keypair, Hash) { - solana_logger::setup(); - let GenesisConfigInfo { - genesis_config, - mint_keypair, - .. - } = create_genesis_config(10); - let bank = Arc::new(Bank::new_no_wallclock_throttle_for_tests(&genesis_config)); - let start_hash = bank.last_blockhash(); - (mint_keypair, start_hash) - } - - fn build_simple_transaction( - mint_keypair: &Keypair, - start_hash: &Hash, - ) -> (Transaction, Vec, u64) { - let keypair = Keypair::new(); - let simple_transaction = - system_transaction::transfer(mint_keypair, &keypair.pubkey(), 2, *start_hash); - - (simple_transaction, vec![mint_keypair.pubkey()], 5) - } - - #[test] - fn test_cost_tracker_initialization() { - let testee = CostTracker::new(Arc::new(RwLock::new(CostModel::new(10, 11)))); - assert_eq!(10, testee.account_cost_limit); - assert_eq!(11, testee.block_cost_limit); - assert_eq!(0, testee.cost_by_writable_accounts.len()); - assert_eq!(0, testee.block_cost); - } - - #[test] - fn test_cost_tracker_ok_add_one() { - let (mint_keypair, start_hash) = test_setup(); - let (_tx, keys, cost) = build_simple_transaction(&mint_keypair, &start_hash); - - // build testee to have capacity for one simple transaction - let mut testee = CostTracker::new(Arc::new(RwLock::new(CostModel::new(cost, cost)))); - assert!(testee.would_fit(&keys, &cost).is_ok()); - testee.add_transaction(&keys, &cost); - assert_eq!(cost, testee.block_cost); - } - - #[test] - fn test_cost_tracker_ok_add_two_same_accounts() { - let (mint_keypair, start_hash) = test_setup(); - // build two transactions with same signed account - let (_tx1, keys1, cost1) = build_simple_transaction(&mint_keypair, &start_hash); - let (_tx2, keys2, cost2) = build_simple_transaction(&mint_keypair, &start_hash); - - // build testee to have capacity for two simple transactions, with same accounts - let mut testee = CostTracker::new(Arc::new(RwLock::new(CostModel::new( - cost1 + cost2, - cost1 + cost2, - )))); - { - assert!(testee.would_fit(&keys1, &cost1).is_ok()); - testee.add_transaction(&keys1, &cost1); - } - { - assert!(testee.would_fit(&keys2, &cost2).is_ok()); - testee.add_transaction(&keys2, &cost2); - } - assert_eq!(cost1 + cost2, testee.block_cost); - assert_eq!(1, testee.cost_by_writable_accounts.len()); - } - - #[test] - fn test_cost_tracker_ok_add_two_diff_accounts() { - let (mint_keypair, start_hash) = test_setup(); - // build two transactions with diff accounts - let (_tx1, keys1, cost1) = build_simple_transaction(&mint_keypair, &start_hash); - let second_account = Keypair::new(); - let (_tx2, keys2, cost2) = build_simple_transaction(&second_account, &start_hash); - - // build testee to have capacity for two simple transactions, with same accounts - let mut testee = CostTracker::new(Arc::new(RwLock::new(CostModel::new( - cmp::max(cost1, cost2), - cost1 + cost2, - )))); - { - assert!(testee.would_fit(&keys1, &cost1).is_ok()); - testee.add_transaction(&keys1, &cost1); - } - { - assert!(testee.would_fit(&keys2, &cost2).is_ok()); - testee.add_transaction(&keys2, &cost2); - } - assert_eq!(cost1 + cost2, testee.block_cost); - assert_eq!(2, testee.cost_by_writable_accounts.len()); - } - - #[test] - fn test_cost_tracker_chain_reach_limit() { - let (mint_keypair, start_hash) = test_setup(); - // build two transactions with same signed account - let (_tx1, keys1, cost1) = build_simple_transaction(&mint_keypair, &start_hash); - let (_tx2, keys2, cost2) = build_simple_transaction(&mint_keypair, &start_hash); - - // build testee to have capacity for two simple transactions, but not for same accounts - let mut testee = CostTracker::new(Arc::new(RwLock::new(CostModel::new( - cmp::min(cost1, cost2), - cost1 + cost2, - )))); - // should have room for first transaction - { - assert!(testee.would_fit(&keys1, &cost1).is_ok()); - testee.add_transaction(&keys1, &cost1); - } - // but no more sapce on the same chain (same signer account) - { - assert!(testee.would_fit(&keys2, &cost2).is_err()); - } - } - - #[test] - fn test_cost_tracker_reach_limit() { - let (mint_keypair, start_hash) = test_setup(); - // build two transactions with diff accounts - let (_tx1, keys1, cost1) = build_simple_transaction(&mint_keypair, &start_hash); - let second_account = Keypair::new(); - let (_tx2, keys2, cost2) = build_simple_transaction(&second_account, &start_hash); - - // build testee to have capacity for each chain, but not enough room for both transactions - let mut testee = CostTracker::new(Arc::new(RwLock::new(CostModel::new( - cmp::max(cost1, cost2), - cost1 + cost2 - 1, - )))); - // should have room for first transaction - { - assert!(testee.would_fit(&keys1, &cost1).is_ok()); - testee.add_transaction(&keys1, &cost1); - } - // but no more room for package as whole - { - assert!(testee.would_fit(&keys2, &cost2).is_err()); - } - } - - #[test] - fn test_cost_tracker_reset() { - let (mint_keypair, start_hash) = test_setup(); - // build two transactions with same signed account - let (_tx1, keys1, cost1) = build_simple_transaction(&mint_keypair, &start_hash); - let (_tx2, keys2, cost2) = build_simple_transaction(&mint_keypair, &start_hash); - - // build testee to have capacity for two simple transactions, but not for same accounts - let mut testee = CostTracker::new(Arc::new(RwLock::new(CostModel::new( - cmp::min(cost1, cost2), - cost1 + cost2, - )))); - // should have room for first transaction - { - assert!(testee.would_fit(&keys1, &cost1).is_ok()); - testee.add_transaction(&keys1, &cost1); - assert_eq!(1, testee.cost_by_writable_accounts.len()); - assert_eq!(cost1, testee.block_cost); - } - // but no more sapce on the same chain (same signer account) - { - assert!(testee.would_fit(&keys2, &cost2).is_err()); - } - // reset the tracker - { - testee.reset_if_new_bank(100); - assert_eq!(0, testee.cost_by_writable_accounts.len()); - assert_eq!(0, testee.block_cost); - } - //now the second transaction can be added - { - assert!(testee.would_fit(&keys2, &cost2).is_ok()); - } - } - - #[test] - fn test_cost_tracker_try_add_is_atomic() { - let acct1 = Pubkey::new_unique(); - let acct2 = Pubkey::new_unique(); - let acct3 = Pubkey::new_unique(); - let cost = 100; - let account_max = cost * 2; - let block_max = account_max * 3; // for three accts - - let mut testee = CostTracker::new(Arc::new(RwLock::new(CostModel::new( - account_max, - block_max, - )))); - - // case 1: a tx writes to 3 accounts, should success, we will have: - // | acct1 | $cost | - // | acct2 | $cost | - // | acct2 | $cost | - // and block_cost = $cost - { - let tx_cost = TransactionCost { - writable_accounts: vec![acct1, acct2, acct3], - account_access_cost: 0, - execution_cost: cost, - }; - assert!(testee.try_add(&tx_cost).is_ok()); - let stat = testee.get_stats(); - assert_eq!(cost, stat.total_cost); - assert_eq!(3, stat.number_of_accounts); - assert_eq!(cost, stat.costliest_account_cost); - } - - // case 2: add tx writes to acct2 with $cost, should succeed, result to - // | acct1 | $cost | - // | acct2 | $cost * 2 | - // | acct2 | $cost | - // and block_cost = $cost * 2 - { - let tx_cost = TransactionCost { - writable_accounts: vec![acct2], - account_access_cost: 0, - execution_cost: cost, - }; - assert!(testee.try_add(&tx_cost).is_ok()); - let stat = testee.get_stats(); - assert_eq!(cost * 2, stat.total_cost); - assert_eq!(3, stat.number_of_accounts); - assert_eq!(cost * 2, stat.costliest_account_cost); - assert_eq!(acct2, stat.costliest_account); - } - - // case 3: add tx writes to [acct1, acct2], acct2 exceeds limit, should failed atomically, - // we shoudl still have: - // | acct1 | $cost | - // | acct2 | $cost | - // | acct2 | $cost | - // and block_cost = $cost - { - let tx_cost = TransactionCost { - writable_accounts: vec![acct1, acct2], - account_access_cost: 0, - execution_cost: cost, - }; - assert!(testee.try_add(&tx_cost).is_err()); - let stat = testee.get_stats(); - assert_eq!(cost * 2, stat.total_cost); - assert_eq!(3, stat.number_of_accounts); - assert_eq!(cost * 2, stat.costliest_account_cost); - assert_eq!(acct2, stat.costliest_account); - } - } -} diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 0a88e8b833ea17..9aa14f8a5cd99b 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -725,78 +725,6 @@ fn load_bank_forks( ) } -<<<<<<< HEAD -======= -fn compute_slot_cost(blockstore: &Blockstore, slot: Slot) -> Result<(), String> { - if blockstore.is_dead(slot) { - return Err("Dead slot".to_string()); - } - - let (entries, _num_shreds, _is_full) = blockstore - .get_slot_entries_with_shred_info(slot, 0, false) - .map_err(|err| format!(" Slot: {}, Failed to load entries, err {:?}", slot, err))?; - - let num_entries = entries.len(); - let mut num_transactions = 0; - let mut num_programs = 0; - - let mut program_ids = HashMap::new(); - let mut cost_model = CostModel::default(); - cost_model.initialize_cost_table(&blockstore.read_program_costs().unwrap()); - let cost_model = Arc::new(RwLock::new(cost_model)); - let mut cost_tracker = CostTracker::new(cost_model.clone()); - - for entry in entries { - num_transactions += entry.transactions.len(); - let mut cost_model = cost_model.write().unwrap(); - entry - .transactions - .into_iter() - .filter_map(|transaction| { - SanitizedTransaction::try_create(transaction, Hash::default(), |_| { - Err(TransactionError::UnsupportedVersion) - }) - .map_err(|err| { - warn!("Failed to compute cost of transaction: {:?}", err); - }) - .ok() - }) - .for_each(|transaction| { - num_programs += transaction.message().instructions().len(); - - let tx_cost = cost_model.calculate_cost( - &transaction, - true, // demote_program_write_locks - ); - if cost_tracker.try_add(tx_cost).is_err() { - println!( - "Slot: {}, CostModel rejected transaction {:?}, stats {:?}!", - slot, - transaction, - cost_tracker.get_stats() - ); - } - for (program_id, _instruction) in transaction.message().program_instructions_iter() - { - *program_ids.entry(*program_id).or_insert(0) += 1; - } - }); - } - - println!( - "Slot: {}, Entries: {}, Transactions: {}, Programs {}, {:?}", - slot, - num_entries, - num_transactions, - num_programs, - cost_tracker.get_stats() - ); - println!(" Programs: {:?}", program_ids); - - Ok(()) -} - ->>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) fn open_genesis_config_by(ledger_path: &Path, matches: &ArgMatches<'_>) -> GenesisConfig { let max_genesis_archive_unpacked_size = value_t_or_exit!(matches, "max_genesis_archive_unpacked_size", u64); diff --git a/program-runtime/src/instruction_processor.rs b/program-runtime/src/instruction_processor.rs deleted file mode 100644 index 49c6381be1e4b2..00000000000000 --- a/program-runtime/src/instruction_processor.rs +++ /dev/null @@ -1,1242 +0,0 @@ -use crate::native_loader::NativeLoader; -use serde::{Deserialize, Serialize}; -use solana_sdk::{ - account::{AccountSharedData, ReadableAccount, WritableAccount}, - account_utils::StateMut, - bpf_loader_upgradeable::{self, UpgradeableLoaderState}, - feature_set::demote_program_write_locks, - ic_logger_msg, ic_msg, - instruction::{CompiledInstruction, Instruction, InstructionError}, - keyed_account::{keyed_account_at_index, KeyedAccount}, - message::Message, - process_instruction::{Executor, InvokeContext, Logger, ProcessInstructionWithContext}, - pubkey::Pubkey, - rent::Rent, - system_program, -}; -use std::{ - cell::{Ref, RefCell}, - collections::HashMap, - rc::Rc, - sync::Arc, -}; - -pub struct Executors { - pub executors: HashMap>, - pub is_dirty: bool, -} -impl Default for Executors { - fn default() -> Self { - Self { - executors: HashMap::default(), - is_dirty: false, - } - } -} -impl Executors { - pub fn insert(&mut self, key: Pubkey, executor: Arc) { - let _ = self.executors.insert(key, executor); - self.is_dirty = true; - } - pub fn get(&self, key: &Pubkey) -> Option> { - self.executors.get(key).cloned() - } -} - -#[derive(Default, Debug)] -pub struct ProgramTiming { - pub accumulated_us: u64, - pub accumulated_units: u64, - pub count: u32, -} - -#[derive(Default, Debug)] -pub struct ExecuteDetailsTimings { - pub serialize_us: u64, - pub create_vm_us: u64, - pub execute_us: u64, - pub deserialize_us: u64, - pub changed_account_count: u64, - pub total_account_count: u64, - pub total_data_size: usize, - pub data_size_changed: usize, - pub per_program_timings: HashMap, -} -impl ExecuteDetailsTimings { - pub fn accumulate(&mut self, other: &ExecuteDetailsTimings) { - self.serialize_us += other.serialize_us; - self.create_vm_us += other.create_vm_us; - self.execute_us += other.execute_us; - self.deserialize_us += other.deserialize_us; - self.changed_account_count += other.changed_account_count; - self.total_account_count += other.total_account_count; - self.total_data_size += other.total_data_size; - self.data_size_changed += other.data_size_changed; - for (id, other) in &other.per_program_timings { - let program_timing = self.per_program_timings.entry(*id).or_default(); - program_timing.accumulated_us = program_timing - .accumulated_us - .saturating_add(other.accumulated_us); - program_timing.accumulated_units = program_timing - .accumulated_units - .saturating_add(other.accumulated_units); - program_timing.count = program_timing.count.saturating_add(other.count); - } - } - pub fn accumulate_program(&mut self, program_id: &Pubkey, us: u64, units: u64) { - let program_timing = self.per_program_timings.entry(*program_id).or_default(); - program_timing.accumulated_us = program_timing.accumulated_us.saturating_add(us); - program_timing.accumulated_units = program_timing.accumulated_units.saturating_add(units); - program_timing.count = program_timing.count.saturating_add(1); - } -} - -// The relevant state of an account before an Instruction executes, used -// to verify account integrity after the Instruction completes -#[derive(Clone, Debug, Default)] -pub struct PreAccount { - key: Pubkey, - account: Rc>, - changed: bool, -} -impl PreAccount { - pub fn new(key: &Pubkey, account: &AccountSharedData) -> Self { - Self { - key: *key, - account: Rc::new(RefCell::new(account.clone())), - changed: false, - } - } - - pub fn verify( - &self, - program_id: &Pubkey, - is_writable: bool, - rent: &Rent, - post: &AccountSharedData, - timings: &mut ExecuteDetailsTimings, - outermost_call: bool, - updated_verify_policy: bool, - ) -> Result<(), InstructionError> { - let pre = self.account.borrow(); - - // Only the owner of the account may change owner and - // only if the account is writable and - // only if the account is not executable and - // only if the data is zero-initialized or empty - let owner_changed = pre.owner() != post.owner(); - if owner_changed - && (!is_writable // line coverage used to get branch coverage - || pre.executable() - || program_id != pre.owner() - || !Self::is_zeroed(post.data())) - { - return Err(InstructionError::ModifiedProgramId); - } - - // An account not assigned to the program cannot have its balance decrease. - if program_id != pre.owner() // line coverage used to get branch coverage - && pre.lamports() > post.lamports() - { - return Err(InstructionError::ExternalAccountLamportSpend); - } - - // The balance of read-only and executable accounts may not change - let lamports_changed = pre.lamports() != post.lamports(); - if lamports_changed { - if !is_writable { - return Err(InstructionError::ReadonlyLamportChange); - } - if pre.executable() { - return Err(InstructionError::ExecutableLamportChange); - } - } - - // Only the system program can change the size of the data - // and only if the system program owns the account - let data_len_changed = pre.data().len() != post.data().len(); - if data_len_changed - && (!system_program::check_id(program_id) // line coverage used to get branch coverage - || !system_program::check_id(pre.owner())) - { - return Err(InstructionError::AccountDataSizeChanged); - } - - // Only the owner may change account data - // and if the account is writable - // and if the account is not executable - if !(program_id == pre.owner() - && is_writable // line coverage used to get branch coverage - && !pre.executable()) - && pre.data() != post.data() - { - if pre.executable() { - return Err(InstructionError::ExecutableDataModified); - } else if is_writable { - return Err(InstructionError::ExternalAccountDataModified); - } else { - return Err(InstructionError::ReadonlyDataModified); - } - } - - // executable is one-way (false->true) and only the account owner may set it. - let executable_changed = pre.executable() != post.executable(); - if executable_changed { - if !rent.is_exempt(post.lamports(), post.data().len()) { - return Err(InstructionError::ExecutableAccountNotRentExempt); - } - let owner = if updated_verify_policy { - post.owner() - } else { - pre.owner() - }; - if !is_writable // line coverage used to get branch coverage - || pre.executable() - || program_id != owner - { - return Err(InstructionError::ExecutableModified); - } - } - - // No one modifies rent_epoch (yet). - let rent_epoch_changed = pre.rent_epoch() != post.rent_epoch(); - if rent_epoch_changed { - return Err(InstructionError::RentEpochModified); - } - - if outermost_call { - timings.total_account_count += 1; - timings.total_data_size += post.data().len(); - if owner_changed - || lamports_changed - || data_len_changed - || executable_changed - || rent_epoch_changed - || self.changed - { - timings.changed_account_count += 1; - timings.data_size_changed += post.data().len(); - } - } - - Ok(()) - } - - pub fn update(&mut self, account: &AccountSharedData) { - let mut pre = self.account.borrow_mut(); - let rent_epoch = pre.rent_epoch(); - *pre = account.clone(); - pre.set_rent_epoch(rent_epoch); - - self.changed = true; - } - - pub fn key(&self) -> &Pubkey { - &self.key - } - - pub fn data(&self) -> Ref<[u8]> { - Ref::map(self.account.borrow(), |account| account.data()) - } - - pub fn lamports(&self) -> u64 { - self.account.borrow().lamports() - } - - pub fn executable(&self) -> bool { - self.account.borrow().executable() - } - - pub fn is_zeroed(buf: &[u8]) -> bool { - const ZEROS_LEN: usize = 1024; - static ZEROS: [u8; ZEROS_LEN] = [0; ZEROS_LEN]; - let mut chunks = buf.chunks_exact(ZEROS_LEN); - - chunks.all(|chunk| chunk == &ZEROS[..]) - && chunks.remainder() == &ZEROS[..chunks.remainder().len()] - } -} - -#[derive(Deserialize, Serialize)] -pub struct InstructionProcessor { - #[serde(skip)] - programs: Vec<(Pubkey, ProcessInstructionWithContext)>, - #[serde(skip)] - native_loader: NativeLoader, -} - -impl std::fmt::Debug for InstructionProcessor { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - #[derive(Debug)] - struct MessageProcessor<'a> { - programs: Vec, - native_loader: &'a NativeLoader, - } - - // These are just type aliases for work around of Debug-ing above pointers - type ErasedProcessInstructionWithContext = fn( - &'static Pubkey, - &'static [u8], - &'static mut dyn InvokeContext, - ) -> Result<(), InstructionError>; - - // rustc doesn't compile due to bug without this work around - // https://github.com/rust-lang/rust/issues/50280 - // https://users.rust-lang.org/t/display-function-pointer/17073/2 - let processor = MessageProcessor { - programs: self - .programs - .iter() - .map(|(pubkey, instruction)| { - let erased_instruction: ErasedProcessInstructionWithContext = *instruction; - format!("{}: {:p}", pubkey, erased_instruction) - }) - .collect::>(), - native_loader: &self.native_loader, - }; - - write!(f, "{:?}", processor) - } -} - -impl Default for InstructionProcessor { - fn default() -> Self { - Self { - programs: vec![], - native_loader: NativeLoader::default(), - } - } -} -impl Clone for InstructionProcessor { - fn clone(&self) -> Self { - InstructionProcessor { - programs: self.programs.clone(), - native_loader: NativeLoader::default(), - } - } -} - -#[cfg(RUSTC_WITH_SPECIALIZATION)] -impl ::solana_frozen_abi::abi_example::AbiExample for InstructionProcessor { - fn example() -> Self { - // MessageProcessor's fields are #[serde(skip)]-ed and not Serialize - // so, just rely on Default anyway. - InstructionProcessor::default() - } -} - -impl InstructionProcessor { - pub fn programs(&self) -> &[(Pubkey, ProcessInstructionWithContext)] { - &self.programs - } - - /// Add a static entrypoint to intercept instructions before the dynamic loader. - pub fn add_program( - &mut self, - program_id: Pubkey, - process_instruction: ProcessInstructionWithContext, - ) { - match self.programs.iter_mut().find(|(key, _)| program_id == *key) { - Some((_, processor)) => *processor = process_instruction, - None => self.programs.push((program_id, process_instruction)), - } - } - - /// Create the KeyedAccounts that will be passed to the program - pub fn create_keyed_accounts<'a>( - message: &'a Message, - instruction: &'a CompiledInstruction, - executable_accounts: &'a [(Pubkey, Rc>)], - accounts: &'a [(Pubkey, Rc>)], - demote_program_write_locks: bool, - ) -> Vec<(bool, bool, &'a Pubkey, &'a RefCell)> { - executable_accounts - .iter() - .map(|(key, account)| (false, false, key, account as &RefCell)) - .chain(instruction.accounts.iter().map(|index| { - let index = *index as usize; - ( - message.is_signer(index), - message.is_writable(index, demote_program_write_locks), - &accounts[index].0, - &accounts[index].1 as &RefCell, - ) - })) - .collect::>() - } - - /// Process an instruction - /// This method calls the instruction's program entrypoint method - pub fn process_instruction( - &self, - program_id: &Pubkey, - instruction_data: &[u8], - invoke_context: &mut dyn InvokeContext, - ) -> Result<(), InstructionError> { - if let Some(root_account) = invoke_context.get_keyed_accounts()?.iter().next() { - let root_id = root_account.unsigned_key(); - if solana_sdk::native_loader::check_id(&root_account.owner()?) { - for (id, process_instruction) in &self.programs { - if id == root_id { - invoke_context.remove_first_keyed_account()?; - // Call the builtin program - return process_instruction(program_id, instruction_data, invoke_context); - } - } - // Call the program via the native loader - return self.native_loader.process_instruction( - &solana_sdk::native_loader::id(), - instruction_data, - invoke_context, - ); - } else { - let owner_id = &root_account.owner()?; - for (id, process_instruction) in &self.programs { - if id == owner_id { - // Call the program via a builtin loader - return process_instruction(program_id, instruction_data, invoke_context); - } - } - } - } - Err(InstructionError::UnsupportedProgramId) - } - - pub fn create_message( - instruction: &Instruction, - keyed_accounts: &[&KeyedAccount], - signers: &[Pubkey], - invoke_context: &Ref<&mut dyn InvokeContext>, - ) -> Result<(Message, Pubkey, usize), InstructionError> { - // Check for privilege escalation - for account in instruction.accounts.iter() { - let keyed_account = keyed_accounts - .iter() - .find_map(|keyed_account| { - if &account.pubkey == keyed_account.unsigned_key() { - Some(keyed_account) - } else { - None - } - }) - .ok_or_else(|| { - ic_msg!( - invoke_context, - "Instruction references an unknown account {}", - account.pubkey - ); - InstructionError::MissingAccount - })?; - // Readonly account cannot become writable - if account.is_writable && !keyed_account.is_writable() { - ic_msg!( - invoke_context, - "{}'s writable privilege escalated", - account.pubkey - ); - return Err(InstructionError::PrivilegeEscalation); - } - - if account.is_signer && // If message indicates account is signed - !( // one of the following needs to be true: - keyed_account.signer_key().is_some() // Signed in the parent instruction - || signers.contains(&account.pubkey) // Signed by the program - ) { - ic_msg!( - invoke_context, - "{}'s signer privilege escalated", - account.pubkey - ); - return Err(InstructionError::PrivilegeEscalation); - } - } - - // validate the caller has access to the program account and that it is executable - let program_id = instruction.program_id; - match keyed_accounts - .iter() - .find(|keyed_account| &program_id == keyed_account.unsigned_key()) - { - Some(keyed_account) => { - if !keyed_account.executable()? { - ic_msg!( - invoke_context, - "Account {} is not executable", - keyed_account.unsigned_key() - ); - return Err(InstructionError::AccountNotExecutable); - } - } - None => { - ic_msg!(invoke_context, "Unknown program {}", program_id); - return Err(InstructionError::MissingAccount); - } - } - - let message = Message::new(&[instruction.clone()], None); - let program_id_index = message.instructions[0].program_id_index as usize; - - Ok((message, program_id, program_id_index)) - } - - /// Entrypoint for a cross-program invocation from a native program - pub fn native_invoke( - invoke_context: &mut dyn InvokeContext, - instruction: Instruction, - keyed_account_indices: &[usize], - signers: &[Pubkey], - ) -> Result<(), InstructionError> { - let invoke_context = RefCell::new(invoke_context); - - let ( - message, - executable_accounts, - accounts, - keyed_account_indices_reordered, - caller_write_privileges, - ) = { - let invoke_context = invoke_context.borrow(); - - // Translate and verify caller's data - let keyed_accounts = invoke_context.get_keyed_accounts()?; - let keyed_accounts = keyed_account_indices - .iter() - .map(|index| keyed_account_at_index(keyed_accounts, *index)) - .collect::, InstructionError>>()?; - let (message, callee_program_id, _) = - Self::create_message(&instruction, &keyed_accounts, signers, &invoke_context)?; - let keyed_accounts = invoke_context.get_keyed_accounts()?; - let mut caller_write_privileges = keyed_account_indices - .iter() - .map(|index| keyed_accounts[*index].is_writable()) - .collect::>(); - caller_write_privileges.insert(0, false); - let mut accounts = vec![]; - let mut keyed_account_indices_reordered = vec![]; - let keyed_accounts = invoke_context.get_keyed_accounts()?; - 'root: for account_key in message.account_keys.iter() { - for keyed_account_index in keyed_account_indices { - let keyed_account = &keyed_accounts[*keyed_account_index]; - if account_key == keyed_account.unsigned_key() { - accounts.push((*account_key, Rc::new(keyed_account.account.clone()))); - keyed_account_indices_reordered.push(*keyed_account_index); - continue 'root; - } - } - ic_msg!( - invoke_context, - "Instruction references an unknown account {}", - account_key - ); - return Err(InstructionError::MissingAccount); - } - - // Process instruction - - invoke_context.record_instruction(&instruction); - - let program_account = - invoke_context - .get_account(&callee_program_id) - .ok_or_else(|| { - ic_msg!(invoke_context, "Unknown program {}", callee_program_id); - InstructionError::MissingAccount - })?; - if !program_account.borrow().executable() { - ic_msg!( - invoke_context, - "Account {} is not executable", - callee_program_id - ); - return Err(InstructionError::AccountNotExecutable); - } - let programdata = if program_account.borrow().owner() == &bpf_loader_upgradeable::id() { - if let UpgradeableLoaderState::Program { - programdata_address, - } = program_account.borrow().state()? - { - if let Some(account) = invoke_context.get_account(&programdata_address) { - Some((programdata_address, account)) - } else { - ic_msg!( - invoke_context, - "Unknown upgradeable programdata account {}", - programdata_address, - ); - return Err(InstructionError::MissingAccount); - } - } else { - ic_msg!( - invoke_context, - "Upgradeable program account state not valid {}", - callee_program_id, - ); - return Err(InstructionError::MissingAccount); - } - } else { - None - }; - let mut executable_accounts = vec![(callee_program_id, program_account)]; - if let Some(programdata) = programdata { - executable_accounts.push(programdata); - } - ( - message, - executable_accounts, - accounts, - keyed_account_indices_reordered, - caller_write_privileges, - ) - }; - - #[allow(clippy::deref_addrof)] - InstructionProcessor::process_cross_program_instruction( - &message, - &executable_accounts, - &accounts, - &caller_write_privileges, - *(&mut *(invoke_context.borrow_mut())), - )?; - - // Copy results back to caller - - { - let invoke_context = invoke_context.borrow(); - let demote_program_write_locks = - invoke_context.is_feature_active(&demote_program_write_locks::id()); - let keyed_accounts = invoke_context.get_keyed_accounts()?; - for (src_keyed_account_index, ((_key, account), dst_keyed_account_index)) in accounts - .iter() - .zip(keyed_account_indices_reordered) - .enumerate() - { - let dst_keyed_account = &keyed_accounts[dst_keyed_account_index]; - let src_keyed_account = account.borrow(); - if message.is_writable(src_keyed_account_index, demote_program_write_locks) - && !src_keyed_account.executable() - { - if dst_keyed_account.data_len()? != src_keyed_account.data().len() - && dst_keyed_account.data_len()? != 0 - { - // Only support for `CreateAccount` at this time. - // Need a way to limit total realloc size across multiple CPI calls - ic_msg!( - invoke_context, - "Inner instructions do not support realloc, only SystemProgram::CreateAccount", - ); - return Err(InstructionError::InvalidRealloc); - } - dst_keyed_account - .try_account_ref_mut()? - .set_lamports(src_keyed_account.lamports()); - dst_keyed_account - .try_account_ref_mut()? - .set_owner(*src_keyed_account.owner()); - dst_keyed_account - .try_account_ref_mut()? - .set_data(src_keyed_account.data().to_vec()); - } - } - } - - Ok(()) - } - - /// Process a cross-program instruction - /// This method calls the instruction's program entrypoint function - pub fn process_cross_program_instruction( - message: &Message, - executable_accounts: &[(Pubkey, Rc>)], - accounts: &[(Pubkey, Rc>)], - caller_write_privileges: &[bool], - invoke_context: &mut dyn InvokeContext, - ) -> Result<(), InstructionError> { - if let Some(instruction) = message.instructions.get(0) { - let program_id = instruction.program_id(&message.account_keys); - - // Verify the calling program hasn't misbehaved - invoke_context.verify_and_update(instruction, accounts, caller_write_privileges)?; - - // Construct keyed accounts - let demote_program_write_locks = - invoke_context.is_feature_active(&demote_program_write_locks::id()); - let keyed_accounts = Self::create_keyed_accounts( - message, - instruction, - executable_accounts, - accounts, - demote_program_write_locks, - ); - - // Invoke callee - invoke_context.push(program_id, &keyed_accounts)?; - - let mut instruction_processor = InstructionProcessor::default(); - for (program_id, process_instruction) in invoke_context.get_programs().iter() { - instruction_processor.add_program(*program_id, *process_instruction); - } - - let mut result = instruction_processor.process_instruction( - program_id, - &instruction.data, - invoke_context, - ); - if result.is_ok() { - // Verify the called program has not misbehaved - let write_privileges: Vec = (0..message.account_keys.len()) - .map(|i| message.is_writable(i, demote_program_write_locks)) - .collect(); - result = invoke_context.verify_and_update(instruction, accounts, &write_privileges); - } - - // Restore previous state - invoke_context.pop(); - result - } else { - // This function is always called with a valid instruction, if that changes return an error - Err(InstructionError::GenericError) - } - } - - /// Record the initial state of the accounts so that they can be compared - /// after the instruction is processed - pub fn create_pre_accounts( - message: &Message, - instruction: &CompiledInstruction, - accounts: &[(Pubkey, Rc>)], - ) -> Vec { - let mut pre_accounts = Vec::with_capacity(instruction.accounts.len()); - { - let mut work = |_unique_index: usize, account_index: usize| { - if account_index < message.account_keys.len() && account_index < accounts.len() { - let account = accounts[account_index].1.borrow(); - pre_accounts.push(PreAccount::new(&accounts[account_index].0, &account)); - return Ok(()); - } - Err(InstructionError::MissingAccount) - }; - let _ = instruction.visit_each_account(&mut work); - } - pre_accounts - } - - /// Verify the results of a cross-program instruction - #[allow(clippy::too_many_arguments)] - pub fn verify_and_update( - instruction: &CompiledInstruction, - pre_accounts: &mut [PreAccount], - accounts: &[(Pubkey, Rc>)], - program_id: &Pubkey, - rent: &Rent, - write_privileges: &[bool], - timings: &mut ExecuteDetailsTimings, - logger: Rc>, - updated_verify_policy: bool, - ) -> Result<(), InstructionError> { - // Verify the per-account instruction results - let (mut pre_sum, mut post_sum) = (0_u128, 0_u128); - let mut work = |_unique_index: usize, account_index: usize| { - if account_index < write_privileges.len() && account_index < accounts.len() { - let (key, account) = &accounts[account_index]; - let is_writable = write_privileges[account_index]; - // Find the matching PreAccount - for pre_account in pre_accounts.iter_mut() { - if key == pre_account.key() { - { - // Verify account has no outstanding references - let _ = account - .try_borrow_mut() - .map_err(|_| InstructionError::AccountBorrowOutstanding)?; - } - let account = account.borrow(); - pre_account - .verify( - program_id, - is_writable, - rent, - &account, - timings, - false, - updated_verify_policy, - ) - .map_err(|err| { - ic_logger_msg!(logger, "failed to verify account {}: {}", key, err); - err - })?; - pre_sum += u128::from(pre_account.lamports()); - post_sum += u128::from(account.lamports()); - if is_writable && !pre_account.executable() { - pre_account.update(&account); - } - return Ok(()); - } - } - } - Err(InstructionError::MissingAccount) - }; - instruction.visit_each_account(&mut work)?; - - // Verify that the total sum of all the lamports did not change - if pre_sum != post_sum { - return Err(InstructionError::UnbalancedInstruction); - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use solana_sdk::{account::Account, instruction::InstructionError}; - - #[test] - fn test_is_zeroed() { - const ZEROS_LEN: usize = 1024; - let mut buf = [0; ZEROS_LEN]; - assert!(PreAccount::is_zeroed(&buf)); - buf[0] = 1; - assert!(!PreAccount::is_zeroed(&buf)); - - let mut buf = [0; ZEROS_LEN - 1]; - assert!(PreAccount::is_zeroed(&buf)); - buf[0] = 1; - assert!(!PreAccount::is_zeroed(&buf)); - - let mut buf = [0; ZEROS_LEN + 1]; - assert!(PreAccount::is_zeroed(&buf)); - buf[0] = 1; - assert!(!PreAccount::is_zeroed(&buf)); - - let buf = vec![]; - assert!(PreAccount::is_zeroed(&buf)); - } - - struct Change { - program_id: Pubkey, - is_writable: bool, - rent: Rent, - pre: PreAccount, - post: AccountSharedData, - } - impl Change { - pub fn new(owner: &Pubkey, program_id: &Pubkey) -> Self { - Self { - program_id: *program_id, - rent: Rent::default(), - is_writable: true, - pre: PreAccount::new( - &solana_sdk::pubkey::new_rand(), - &AccountSharedData::from(Account { - owner: *owner, - lamports: std::u64::MAX, - data: vec![], - ..Account::default() - }), - ), - post: AccountSharedData::from(Account { - owner: *owner, - lamports: std::u64::MAX, - ..Account::default() - }), - } - } - pub fn read_only(mut self) -> Self { - self.is_writable = false; - self - } - pub fn executable(mut self, pre: bool, post: bool) -> Self { - self.pre.account.borrow_mut().set_executable(pre); - self.post.set_executable(post); - self - } - pub fn lamports(mut self, pre: u64, post: u64) -> Self { - self.pre.account.borrow_mut().set_lamports(pre); - self.post.set_lamports(post); - self - } - pub fn owner(mut self, post: &Pubkey) -> Self { - self.post.set_owner(*post); - self - } - pub fn data(mut self, pre: Vec, post: Vec) -> Self { - self.pre.account.borrow_mut().set_data(pre); - self.post.set_data(post); - self - } - pub fn rent_epoch(mut self, pre: u64, post: u64) -> Self { - self.pre.account.borrow_mut().set_rent_epoch(pre); - self.post.set_rent_epoch(post); - self - } - pub fn verify(&self) -> Result<(), InstructionError> { - self.pre.verify( - &self.program_id, - self.is_writable, - &self.rent, - &self.post, - &mut ExecuteDetailsTimings::default(), - false, - true, - ) - } - } - - #[test] - fn test_verify_account_changes_owner() { - let system_program_id = system_program::id(); - let alice_program_id = solana_sdk::pubkey::new_rand(); - let mallory_program_id = solana_sdk::pubkey::new_rand(); - - assert_eq!( - Change::new(&system_program_id, &system_program_id) - .owner(&alice_program_id) - .verify(), - Ok(()), - "system program should be able to change the account owner" - ); - assert_eq!( - Change::new(&system_program_id, &system_program_id) - .owner(&alice_program_id) - .read_only() - .verify(), - Err(InstructionError::ModifiedProgramId), - "system program should not be able to change the account owner of a read-only account" - ); - assert_eq!( - Change::new(&mallory_program_id, &system_program_id) - .owner(&alice_program_id) - .verify(), - Err(InstructionError::ModifiedProgramId), - "system program should not be able to change the account owner of a non-system account" - ); - assert_eq!( - Change::new(&mallory_program_id, &mallory_program_id) - .owner(&alice_program_id) - .verify(), - Ok(()), - "mallory should be able to change the account owner, if she leaves clear data" - ); - assert_eq!( - Change::new(&mallory_program_id, &mallory_program_id) - .owner(&alice_program_id) - .data(vec![42], vec![0]) - .verify(), - Ok(()), - "mallory should be able to change the account owner, if she leaves clear data" - ); - assert_eq!( - Change::new(&mallory_program_id, &mallory_program_id) - .owner(&alice_program_id) - .executable(true, true) - .data(vec![42], vec![0]) - .verify(), - Err(InstructionError::ModifiedProgramId), - "mallory should not be able to change the account owner, if the account executable" - ); - assert_eq!( - Change::new(&mallory_program_id, &mallory_program_id) - .owner(&alice_program_id) - .data(vec![42], vec![42]) - .verify(), - Err(InstructionError::ModifiedProgramId), - "mallory should not be able to inject data into the alice program" - ); - } - - #[test] - fn test_verify_account_changes_executable() { - let owner = solana_sdk::pubkey::new_rand(); - let mallory_program_id = solana_sdk::pubkey::new_rand(); - let system_program_id = system_program::id(); - - assert_eq!( - Change::new(&owner, &system_program_id) - .executable(false, true) - .verify(), - Err(InstructionError::ExecutableModified), - "system program can't change executable if system doesn't own the account" - ); - assert_eq!( - Change::new(&owner, &system_program_id) - .executable(true, true) - .data(vec![1], vec![2]) - .verify(), - Err(InstructionError::ExecutableDataModified), - "system program can't change executable data if system doesn't own the account" - ); - assert_eq!( - Change::new(&owner, &owner).executable(false, true).verify(), - Ok(()), - "owner should be able to change executable" - ); - assert_eq!( - Change::new(&owner, &owner) - .executable(false, true) - .read_only() - .verify(), - Err(InstructionError::ExecutableModified), - "owner can't modify executable of read-only accounts" - ); - assert_eq!( - Change::new(&owner, &owner).executable(true, false).verify(), - Err(InstructionError::ExecutableModified), - "owner program can't reverse executable" - ); - assert_eq!( - Change::new(&owner, &mallory_program_id) - .executable(false, true) - .verify(), - Err(InstructionError::ExecutableModified), - "malicious Mallory should not be able to change the account executable" - ); - assert_eq!( - Change::new(&owner, &owner) - .executable(false, true) - .data(vec![1], vec![2]) - .verify(), - Ok(()), - "account data can change in the same instruction that sets the bit" - ); - assert_eq!( - Change::new(&owner, &owner) - .executable(true, true) - .data(vec![1], vec![2]) - .verify(), - Err(InstructionError::ExecutableDataModified), - "owner should not be able to change an account's data once its marked executable" - ); - assert_eq!( - Change::new(&owner, &owner) - .executable(true, true) - .lamports(1, 2) - .verify(), - Err(InstructionError::ExecutableLamportChange), - "owner should not be able to add lamports once marked executable" - ); - assert_eq!( - Change::new(&owner, &owner) - .executable(true, true) - .lamports(1, 2) - .verify(), - Err(InstructionError::ExecutableLamportChange), - "owner should not be able to add lamports once marked executable" - ); - assert_eq!( - Change::new(&owner, &owner) - .executable(true, true) - .lamports(2, 1) - .verify(), - Err(InstructionError::ExecutableLamportChange), - "owner should not be able to subtract lamports once marked executable" - ); - let data = vec![1; 100]; - let min_lamports = Rent::default().minimum_balance(data.len()); - assert_eq!( - Change::new(&owner, &owner) - .executable(false, true) - .lamports(0, min_lamports) - .data(data.clone(), data.clone()) - .verify(), - Ok(()), - ); - assert_eq!( - Change::new(&owner, &owner) - .executable(false, true) - .lamports(0, min_lamports - 1) - .data(data.clone(), data) - .verify(), - Err(InstructionError::ExecutableAccountNotRentExempt), - "owner should not be able to change an account's data once its marked executable" - ); - } - - #[test] - fn test_verify_account_changes_data_len() { - let alice_program_id = solana_sdk::pubkey::new_rand(); - - assert_eq!( - Change::new(&system_program::id(), &system_program::id()) - .data(vec![0], vec![0, 0]) - .verify(), - Ok(()), - "system program should be able to change the data len" - ); - assert_eq!( - Change::new(&alice_program_id, &system_program::id()) - .data(vec![0], vec![0,0]) - .verify(), - Err(InstructionError::AccountDataSizeChanged), - "system program should not be able to change the data length of accounts it does not own" - ); - } - - #[test] - fn test_verify_account_changes_data() { - let alice_program_id = solana_sdk::pubkey::new_rand(); - let mallory_program_id = solana_sdk::pubkey::new_rand(); - - assert_eq!( - Change::new(&alice_program_id, &alice_program_id) - .data(vec![0], vec![42]) - .verify(), - Ok(()), - "alice program should be able to change the data" - ); - assert_eq!( - Change::new(&mallory_program_id, &alice_program_id) - .data(vec![0], vec![42]) - .verify(), - Err(InstructionError::ExternalAccountDataModified), - "non-owner mallory should not be able to change the account data" - ); - assert_eq!( - Change::new(&alice_program_id, &alice_program_id) - .data(vec![0], vec![42]) - .read_only() - .verify(), - Err(InstructionError::ReadonlyDataModified), - "alice isn't allowed to touch a CO account" - ); - } - - #[test] - fn test_verify_account_changes_rent_epoch() { - let alice_program_id = solana_sdk::pubkey::new_rand(); - - assert_eq!( - Change::new(&alice_program_id, &system_program::id()).verify(), - Ok(()), - "nothing changed!" - ); - assert_eq!( - Change::new(&alice_program_id, &system_program::id()) - .rent_epoch(0, 1) - .verify(), - Err(InstructionError::RentEpochModified), - "no one touches rent_epoch" - ); - } - - #[test] - fn test_verify_account_changes_deduct_lamports_and_reassign_account() { - let alice_program_id = solana_sdk::pubkey::new_rand(); - let bob_program_id = solana_sdk::pubkey::new_rand(); - - // positive test of this capability - assert_eq!( - Change::new(&alice_program_id, &alice_program_id) - .owner(&bob_program_id) - .lamports(42, 1) - .data(vec![42], vec![0]) - .verify(), - Ok(()), - "alice should be able to deduct lamports and give the account to bob if the data is zeroed", - ); - } - - #[test] - fn test_verify_account_changes_lamports() { - let alice_program_id = solana_sdk::pubkey::new_rand(); - - assert_eq!( - Change::new(&alice_program_id, &system_program::id()) - .lamports(42, 0) - .read_only() - .verify(), - Err(InstructionError::ExternalAccountLamportSpend), - "debit should fail, even if system program" - ); - assert_eq!( - Change::new(&alice_program_id, &alice_program_id) - .lamports(42, 0) - .read_only() - .verify(), - Err(InstructionError::ReadonlyLamportChange), - "debit should fail, even if owning program" - ); - assert_eq!( - Change::new(&alice_program_id, &system_program::id()) - .lamports(42, 0) - .owner(&system_program::id()) - .verify(), - Err(InstructionError::ModifiedProgramId), - "system program can't debit the account unless it was the pre.owner" - ); - assert_eq!( - Change::new(&system_program::id(), &system_program::id()) - .lamports(42, 0) - .owner(&alice_program_id) - .verify(), - Ok(()), - "system can spend (and change owner)" - ); - } - - #[test] - fn test_verify_account_changes_data_size_changed() { - let alice_program_id = solana_sdk::pubkey::new_rand(); - - assert_eq!( - Change::new(&alice_program_id, &system_program::id()) - .data(vec![0], vec![0, 0]) - .verify(), - Err(InstructionError::AccountDataSizeChanged), - "system program should not be able to change another program's account data size" - ); - assert_eq!( - Change::new(&alice_program_id, &alice_program_id) - .data(vec![0], vec![0, 0]) - .verify(), - Err(InstructionError::AccountDataSizeChanged), - "non-system programs cannot change their data size" - ); - assert_eq!( - Change::new(&system_program::id(), &system_program::id()) - .data(vec![0], vec![0, 0]) - .verify(), - Ok(()), - "system program should be able to change account data size" - ); - } - - #[test] - fn test_verify_account_changes_owner_executable() { - let alice_program_id = solana_sdk::pubkey::new_rand(); - let bob_program_id = solana_sdk::pubkey::new_rand(); - - assert_eq!( - Change::new(&alice_program_id, &alice_program_id) - .owner(&bob_program_id) - .executable(false, true) - .verify(), - Err(InstructionError::ExecutableModified), - "Program should not be able to change owner and executable at the same time" - ); - } - - #[test] - fn test_debug() { - let mut instruction_processor = InstructionProcessor::default(); - #[allow(clippy::unnecessary_wraps)] - fn mock_process_instruction( - _program_id: &Pubkey, - _data: &[u8], - _invoke_context: &mut dyn InvokeContext, - ) -> Result<(), InstructionError> { - Ok(()) - } - #[allow(clippy::unnecessary_wraps)] - fn mock_ix_processor( - _pubkey: &Pubkey, - _data: &[u8], - _context: &mut dyn InvokeContext, - ) -> Result<(), InstructionError> { - Ok(()) - } - let program_id = solana_sdk::pubkey::new_rand(); - instruction_processor.add_program(program_id, mock_process_instruction); - instruction_processor.add_program(program_id, mock_ix_processor); - - assert!(!format!("{:?}", instruction_processor).is_empty()); - } -} diff --git a/programs/bpf/tests/programs.rs b/programs/bpf/tests/programs.rs index b362de4b672155..5183252c216514 100644 --- a/programs/bpf/tests/programs.rs +++ b/programs/bpf/tests/programs.rs @@ -2107,100 +2107,6 @@ fn test_program_bpf_upgrade_via_cpi() { #[cfg(feature = "bpf_rust")] #[test] -<<<<<<< HEAD -fn test_program_bpf_upgrade_self_via_cpi() { - solana_logger::setup(); - - let GenesisConfigInfo { - genesis_config, - mint_keypair, - .. - } = create_genesis_config(50); - let mut bank = Bank::new(&genesis_config); - let (name, id, entrypoint) = solana_bpf_loader_program!(); - bank.add_builtin(&name, id, entrypoint); - let (name, id, entrypoint) = solana_bpf_loader_upgradeable_program!(); - bank.add_builtin(&name, id, entrypoint); - let bank = Arc::new(bank); - let bank_client = BankClient::new_shared(&bank); - let noop_program_id = load_bpf_program( - &bank_client, - &bpf_loader::id(), - &mint_keypair, - "solana_bpf_rust_noop", - ); - - // Deploy upgradeable program - let buffer_keypair = Keypair::new(); - let program_keypair = Keypair::new(); - let program_id = program_keypair.pubkey(); - let authority_keypair = Keypair::new(); - load_upgradeable_bpf_program( - &bank_client, - &mint_keypair, - &buffer_keypair, - &program_keypair, - &authority_keypair, - "solana_bpf_rust_invoke_and_return", - ); - - let mut invoke_instruction = Instruction::new_with_bytes( - program_id, - &[0], - vec![ - AccountMeta::new(noop_program_id, false), - AccountMeta::new(noop_program_id, false), - AccountMeta::new(clock::id(), false), - AccountMeta::new(fees::id(), false), - ], - ); - - // Call the upgraded program - invoke_instruction.data[0] += 1; - let result = - bank_client.send_and_confirm_instruction(&mint_keypair, invoke_instruction.clone()); - assert!(result.is_ok()); - - // Prepare for upgrade - let buffer_keypair = Keypair::new(); - load_upgradeable_buffer( - &bank_client, - &mint_keypair, - &buffer_keypair, - &authority_keypair, - "solana_bpf_rust_panic", - ); - - // Invoke, then upgrade the program, and then invoke again in same tx - let message = Message::new( - &[ - invoke_instruction.clone(), - bpf_loader_upgradeable::upgrade( - &program_id, - &buffer_keypair.pubkey(), - &authority_keypair.pubkey(), - &mint_keypair.pubkey(), - ), - invoke_instruction, - ], - Some(&mint_keypair.pubkey()), - ); - let tx = Transaction::new( - &[&mint_keypair, &authority_keypair], - message.clone(), - bank.last_blockhash(), - ); - let (result, _) = process_transaction_and_record_inner(&bank, tx); - assert_eq!( - result.unwrap_err(), - TransactionError::InstructionError(2, InstructionError::ProgramFailedToComplete) - ); -} - -#[cfg(feature = "bpf_rust")] -#[test] -======= ->>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) fn test_program_bpf_set_upgrade_authority_via_cpi() { solana_logger::setup(); diff --git a/programs/bpf_loader/src/syscalls.rs b/programs/bpf_loader/src/syscalls.rs index 862e34f24de281..ed238654def9bb 100644 --- a/programs/bpf_loader/src/syscalls.rs +++ b/programs/bpf_loader/src/syscalls.rs @@ -19,16 +19,10 @@ use solana_sdk::{ entrypoint::{MAX_PERMITTED_DATA_INCREASE, SUCCESS}, epoch_schedule::EpochSchedule, feature_set::{ -<<<<<<< HEAD - close_upgradeable_program_accounts, cpi_data_cost, enforce_aligned_host_addrs, - keccak256_syscall_enabled, libsecp256k1_0_5_upgrade_enabled, mem_overlap_fix, - memory_ops_syscalls, secp256k1_recover_syscall_enabled, + close_upgradeable_program_accounts, cpi_data_cost, demote_program_write_locks, + enforce_aligned_host_addrs, keccak256_syscall_enabled, libsecp256k1_0_5_upgrade_enabled, + mem_overlap_fix, memory_ops_syscalls, secp256k1_recover_syscall_enabled, set_upgrade_authority_via_cpi_enabled, sysvar_via_syscall, update_data_on_realloc, -======= - blake3_syscall_enabled, close_upgradeable_program_accounts, demote_program_write_locks, - disable_fees_sysvar, enforce_aligned_host_addrs, libsecp256k1_0_5_upgrade_enabled, - mem_overlap_fix, secp256k1_recover_syscall_enabled, ->>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) }, hash::{Hasher, HASH_BYTES}, ic_msg, diff --git a/rpc/src/transaction_status_service.rs b/rpc/src/transaction_status_service.rs index fd3f3d702aabd8..60b1f1f7e84a74 100644 --- a/rpc/src/transaction_status_service.rs +++ b/rpc/src/transaction_status_service.rs @@ -111,15 +111,10 @@ impl TransactionStatusService { bank.get_fee_calculator(&transaction.message().recent_blockhash) }) .expect("FeeCalculator must exist"); -<<<<<<< HEAD let fee = fee_calculator.calculate_fee(transaction.message()); - let (writable_keys, readonly_keys) = - transaction.message.get_account_keys_by_lock_type(); -======= - let fee = transaction.message().calculate_fee(&fee_calculator); - let tx_account_locks = - transaction.get_account_locks(bank.demote_program_write_locks()); ->>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) + let (writable_keys, readonly_keys) = transaction + .message + .get_account_keys_by_lock_type(bank.demote_program_write_locks()); let inner_instructions = inner_instructions.map(|inner_instructions| { inner_instructions diff --git a/runtime/src/accounts.rs b/runtime/src/accounts.rs index 3dcbe5da9413ca..ebdeb5623d2bd6 100644 --- a/runtime/src/accounts.rs +++ b/runtime/src/accounts.rs @@ -166,7 +166,6 @@ impl Accounts { } } -<<<<<<< HEAD /// Return true if the slice has any duplicate elements pub fn has_duplicates(xs: &[T]) -> bool { // Note: This is an O(n^2) algorithm, but requires no heap allocations. The benchmark @@ -180,16 +179,11 @@ impl Accounts { false } - fn construct_instructions_account(message: &Message) -> AccountSharedData { - let mut data = message.serialize_instructions(); -======= fn construct_instructions_account( - message: &SanitizedMessage, - is_owned_by_sysvar: bool, + message: &Message, demote_program_write_locks: bool, ) -> AccountSharedData { let mut data = message.serialize_instructions(demote_program_write_locks); ->>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) // add room for current instruction index. data.resize(data.len() + 2, 0); AccountSharedData::from(Account { @@ -230,26 +224,13 @@ impl Accounts { payer_index = Some(i); } -<<<<<<< HEAD if solana_sdk::sysvar::instructions::check_id(key) && feature_set.is_active(&feature_set::instructions_sysvar_enabled::id()) { - if message.is_writable(i) { - return Err(TransactionError::InvalidAccountIndex); - } - Self::construct_instructions_account(message) -======= - if solana_sdk::sysvar::instructions::check_id(key) { if message.is_writable(i, demote_program_write_locks) { return Err(TransactionError::InvalidAccountIndex); } - Self::construct_instructions_account( - message, - feature_set - .is_active(&feature_set::instructions_sysvar_owned_by_sysvar::id()), - demote_program_write_locks, - ) ->>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) + Self::construct_instructions_account(message, demote_program_write_locks) } else { let (account, rent) = self .accounts_db @@ -884,8 +865,11 @@ impl Accounts { /// same time #[must_use] #[allow(clippy::needless_collect)] -<<<<<<< HEAD - pub fn lock_accounts<'a>(&self, txs: impl Iterator) -> Vec> { + pub fn lock_accounts<'a>( + &self, + txs: impl Iterator, + demote_program_write_locks: bool, + ) -> Vec> { use solana_sdk::sanitize::Sanitize; let keys: Vec> = txs .map(|tx| { @@ -895,17 +879,10 @@ impl Accounts { return Err(TransactionError::AccountLoadedTwice); } - Ok(tx.message().get_account_keys_by_lock_type()) + Ok(tx + .message() + .get_account_keys_by_lock_type(demote_program_write_locks)) }) -======= - pub fn lock_accounts<'a>( - &self, - txs: impl Iterator, - demote_program_write_locks: bool, - ) -> Vec> { - let keys: Vec<_> = txs - .map(|tx| tx.get_account_locks(demote_program_write_locks)) ->>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) .collect(); let mut account_locks = &mut self.account_locks.lock().unwrap(); keys.into_iter() @@ -932,11 +909,10 @@ impl Accounts { Err(TransactionError::AccountInUse) => None, Err(TransactionError::SanitizeFailure) => None, Err(TransactionError::AccountLoadedTwice) => None, -<<<<<<< HEAD - _ => Some(tx.message.get_account_keys_by_lock_type()), -======= - _ => Some(tx.get_account_locks(demote_program_write_locks)), ->>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) + _ => Some( + tx.message + .get_account_keys_by_lock_type(demote_program_write_locks), + ), }) .collect(); let mut account_locks = self.account_locks.lock().unwrap(); @@ -988,6 +964,7 @@ impl Accounts { self.accounts_db.add_root(slot) } + #[allow(clippy::too_many_arguments)] fn collect_accounts_to_store<'a>( &self, txs: impl Iterator, @@ -1828,13 +1805,8 @@ mod tests { Hash::default(), instructions, ); -<<<<<<< HEAD let tx = Transaction::new(&[&keypair0], message, Hash::default()); - let results0 = accounts.lock_accounts([tx.clone()].iter()); -======= - let tx = new_sanitized_tx(&[&keypair0], message, Hash::default()); let results0 = accounts.lock_accounts([tx.clone()].iter(), demote_program_write_locks); ->>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) assert!(results0[0].is_ok()); assert_eq!( @@ -1895,13 +1867,8 @@ mod tests { Hash::default(), instructions, ); -<<<<<<< HEAD let tx = Transaction::new(&[&keypair1], message, Hash::default()); - let results2 = accounts.lock_accounts([tx].iter()); -======= - let tx = new_sanitized_tx(&[&keypair1], message, Hash::default()); let results2 = accounts.lock_accounts([tx].iter(), demote_program_write_locks); ->>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) assert!(results2[0].is_ok()); // Now keypair1 account can be locked as writable // Check that read-only lock with zero references is deleted @@ -2015,7 +1982,7 @@ mod tests { let account2 = AccountSharedData::new(3, 0, &Pubkey::default()); let account3 = AccountSharedData::new(4, 0, &Pubkey::default()); - let accounts = Accounts::new_with_config_for_tests( + let accounts = Accounts::new_with_config( Vec::new(), &ClusterType::Development, AccountSecondaryIndexes::default(), @@ -2038,7 +2005,7 @@ mod tests { Hash::default(), instructions, ); - let tx = new_sanitized_tx(&[&keypair0], message, Hash::default()); + let tx = Transaction::new(&[&keypair0], message, Hash::default()); let results0 = accounts.lock_accounts([tx].iter(), demote_program_write_locks); assert!(results0[0].is_ok()); diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index b134c85af6f821..d810260044d04f 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -2610,73 +2610,27 @@ impl Bank { tick_height % self.ticks_per_slot == 0 } -<<<<<<< HEAD pub fn prepare_batch<'a, 'b>( &'a self, txs: impl Iterator, ) -> TransactionBatch<'a, 'b> { let hashed_txs: Vec = txs.map(HashedTransaction::from).collect(); - let lock_results = self - .rc - .accounts - .lock_accounts(hashed_txs.as_transactions_iter()); + let lock_results = self.rc.accounts.lock_accounts( + hashed_txs.as_transactions_iter(), + self.demote_program_write_locks(), + ); TransactionBatch::new(lock_results, self, Cow::Owned(hashed_txs)) -======= - /// Prepare a transaction batch from a list of legacy transactionsy. Used for tests only. - pub fn prepare_batch(&self, txs: Vec) -> Result { - let sanitized_txs = txs - .into_iter() - .map(SanitizedTransaction::try_from) - .collect::>>()?; - let lock_results = self - .rc - .accounts - .lock_accounts(sanitized_txs.iter(), self.demote_program_write_locks()); - Ok(TransactionBatch::new( - lock_results, - self, - Cow::Owned(sanitized_txs), - )) - } - - /// Prepare a transaction batch from a list of versioned transactions from - /// an entry. Used for tests only. - pub fn prepare_entry_batch(&self, txs: Vec) -> Result { - let sanitized_txs = txs - .into_iter() - .map(|tx| { - let message_hash = tx.message.hash(); - SanitizedTransaction::try_create(tx, message_hash, |_| { - Err(TransactionError::UnsupportedVersion) - }) - }) - .collect::>>()?; - let lock_results = self - .rc - .accounts - .lock_accounts(sanitized_txs.iter(), self.demote_program_write_locks()); - Ok(TransactionBatch::new( - lock_results, - self, - Cow::Owned(sanitized_txs), - )) ->>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) } pub fn prepare_hashed_batch<'a, 'b>( &'a self, hashed_txs: &'b [HashedTransaction], ) -> TransactionBatch<'a, 'b> { - let lock_results = self - .rc - .accounts -<<<<<<< HEAD - .lock_accounts(hashed_txs.as_transactions_iter()); + let lock_results = self.rc.accounts.lock_accounts( + hashed_txs.as_transactions_iter(), + self.demote_program_write_locks(), + ); TransactionBatch::new(lock_results, self, Cow::Borrowed(hashed_txs)) -======= - .lock_accounts(txs.iter(), self.demote_program_write_locks()); - TransactionBatch::new(lock_results, self, Cow::Borrowed(txs)) ->>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) } pub(crate) fn prepare_simulation_batch<'a, 'b>( @@ -2754,17 +2708,11 @@ impl Bank { pub fn unlock_accounts(&self, batch: &mut TransactionBatch) { if batch.needs_unlock { batch.needs_unlock = false; -<<<<<<< HEAD - self.rc - .accounts - .unlock_accounts(batch.transactions_iter(), batch.lock_results()) -======= self.rc.accounts.unlock_accounts( - batch.sanitized_transactions().iter(), + batch.transactions_iter(), batch.lock_results(), self.demote_program_write_locks(), ) ->>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) } } diff --git a/runtime/src/message_processor.rs b/runtime/src/message_processor.rs index 91e7a86e73028c..2f44221176f78f 100644 --- a/runtime/src/message_processor.rs +++ b/runtime/src/message_processor.rs @@ -9,12 +9,8 @@ use solana_sdk::{ account_utils::StateMut, bpf_loader_upgradeable::{self, UpgradeableLoaderState}, feature_set::{ -<<<<<<< HEAD - instructions_sysvar_enabled, neon_evm_compute_budget, updated_verify_policy, FeatureSet, -======= - demote_program_write_locks, neon_evm_compute_budget, tx_wide_compute_cap, + demote_program_write_locks, instructions_sysvar_enabled, neon_evm_compute_budget, updated_verify_policy, FeatureSet, ->>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) }, ic_logger_msg, ic_msg, instruction::{CompiledInstruction, Instruction, InstructionError}, @@ -617,6 +613,7 @@ impl MessageProcessor { instruction: &'a CompiledInstruction, executable_accounts: &'a [(Pubkey, Rc>)], accounts: &'a [(Pubkey, Rc>)], + demote_program_write_locks: bool, ) -> Vec<(bool, bool, &'a Pubkey, &'a RefCell)> { executable_accounts .iter() @@ -625,7 +622,7 @@ impl MessageProcessor { let index = *index as usize; ( message.is_signer(index), - message.is_writable(index), + message.is_writable(index, demote_program_write_locks), &accounts[index].0, &accounts[index].1 as &RefCell, ) @@ -870,6 +867,8 @@ impl MessageProcessor { { let invoke_context = invoke_context.borrow(); + let demote_program_write_locks = + invoke_context.is_feature_active(&demote_program_write_locks::id()); let keyed_accounts = invoke_context.get_keyed_accounts()?; for (src_keyed_account_index, ((_key, account), dst_keyed_account_index)) in accounts .iter() @@ -878,7 +877,9 @@ impl MessageProcessor { { let dst_keyed_account = &keyed_accounts[dst_keyed_account_index]; let src_keyed_account = account.borrow(); - if message.is_writable(src_keyed_account_index) && !src_keyed_account.executable() { + if message.is_writable(src_keyed_account_index, demote_program_write_locks) + && !src_keyed_account.executable() + { if dst_keyed_account.data_len()? != src_keyed_account.data().len() && dst_keyed_account.data_len()? != 0 { @@ -922,8 +923,15 @@ impl MessageProcessor { invoke_context.verify_and_update(instruction, accounts, caller_write_privileges)?; // Construct keyed accounts - let keyed_accounts = - Self::create_keyed_accounts(message, instruction, executable_accounts, accounts); + let demote_program_write_locks = + invoke_context.is_feature_active(&demote_program_write_locks::id()); + let keyed_accounts = Self::create_keyed_accounts( + message, + instruction, + executable_accounts, + accounts, + demote_program_write_locks, + ); // Invoke callee invoke_context.push(program_id, &keyed_accounts)?; @@ -941,7 +949,7 @@ impl MessageProcessor { if result.is_ok() { // Verify the called program has not misbehaved let write_privileges: Vec = (0..message.account_keys.len()) - .map(|i| message.is_writable(i)) + .map(|i| message.is_writable(i, demote_program_write_locks)) .collect(); result = invoke_context.verify_and_update(instruction, accounts, &write_privileges); } @@ -2259,13 +2267,8 @@ mod tests { BpfComputeBudget::default(), Rc::new(RefCell::new(Executors::default())), None, -<<<<<<< HEAD Arc::new(FeatureSet::all_enabled()), Arc::new(Accounts::default()), -======= - Arc::new(feature_set), - Arc::new(Accounts::default_for_tests()), ->>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) &ancestors, ); diff --git a/sdk/benches/serialize_instructions.rs b/sdk/benches/serialize_instructions.rs index 1d6b062d95b2f6..201089a850caae 100644 --- a/sdk/benches/serialize_instructions.rs +++ b/sdk/benches/serialize_instructions.rs @@ -45,15 +45,8 @@ fn bench_bincode_instruction_deserialize(b: &mut Bencher) { #[bench] fn bench_manual_instruction_deserialize(b: &mut Bencher) { let instructions = make_instructions(); -<<<<<<< HEAD let message = Message::new(&instructions, None); - let serialized = message.serialize_instructions(); -======= - let message = - SanitizedMessage::try_from(Message::new(&instructions, Some(&Pubkey::new_unique()))) - .unwrap(); let serialized = message.serialize_instructions(DEMOTE_PROGRAM_WRITE_LOCKS); ->>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) b.iter(|| { for i in 0..instructions.len() { test::black_box(instructions::load_instruction_at(i, &serialized).unwrap()); @@ -64,15 +57,8 @@ fn bench_manual_instruction_deserialize(b: &mut Bencher) { #[bench] fn bench_manual_instruction_deserialize_single(b: &mut Bencher) { let instructions = make_instructions(); -<<<<<<< HEAD let message = Message::new(&instructions, None); - let serialized = message.serialize_instructions(); -======= - let message = - SanitizedMessage::try_from(Message::new(&instructions, Some(&Pubkey::new_unique()))) - .unwrap(); let serialized = message.serialize_instructions(DEMOTE_PROGRAM_WRITE_LOCKS); ->>>>>>> decec3cd8 (Demote write locks on transaction program ids (#19593)) b.iter(|| { test::black_box(instructions::load_instruction_at(3, &serialized).unwrap()); }); diff --git a/sdk/program/src/message.rs b/sdk/program/src/message.rs index f23d1d241a369e..429c69e91edb09 100644 --- a/sdk/program/src/message.rs +++ b/sdk/program/src/message.rs @@ -356,6 +356,16 @@ impl Message { .collect() } + pub fn is_key_called_as_program(&self, key_index: usize) -> bool { + if let Ok(key_index) = u8::try_from(key_index) { + self.instructions + .iter() + .any(|ix| ix.program_id_index == key_index) + } else { + false + } + } + pub fn is_key_passed_to_program(&self, index: usize) -> bool { if let Ok(index) = u8::try_from(index) { for ix in self.instructions.iter() { @@ -408,11 +418,14 @@ impl Message { i < self.header.num_required_signatures as usize } - pub fn get_account_keys_by_lock_type(&self) -> (Vec<&Pubkey>, Vec<&Pubkey>) { + pub fn get_account_keys_by_lock_type( + &self, + demote_program_write_locks: bool, + ) -> (Vec<&Pubkey>, Vec<&Pubkey>) { let mut writable_keys = vec![]; let mut readonly_keys = vec![]; for (i, key) in self.account_keys.iter().enumerate() { - if self.is_writable(i, /*demote_program_write_locks=*/ true) { + if self.is_writable(i, demote_program_write_locks) { writable_keys.push(key); } else { readonly_keys.push(key); @@ -434,7 +447,7 @@ impl Message { // 35..67 - program_id // 67..69 - data len - u16 // 69..data_len - data - pub fn serialize_instructions(&self) -> Vec { + pub fn serialize_instructions(&self, demote_program_write_locks: bool) -> Vec { // 64 bytes is a reasonable guess, calculating exactly is slower in benchmarks let mut data = Vec::with_capacity(self.instructions.len() * (32 * 2)); append_u16(&mut data, self.instructions.len() as u16); @@ -449,8 +462,7 @@ impl Message { for account_index in &instruction.accounts { let account_index = *account_index as usize; let is_signer = self.is_signer(account_index); - let is_writable = - self.is_writable(account_index, /*demote_program_write_locks=*/ true); + let is_writable = self.is_writable(account_index, demote_program_write_locks); let mut meta_byte = 0; if is_signer { meta_byte |= 1 << Self::IS_SIGNER_BIT; @@ -923,7 +935,7 @@ mod tests { Some(&id1), ); assert_eq!( - message.get_account_keys_by_lock_type(), + message.get_account_keys_by_lock_type(/*demote_program_write_locks=*/ true), (vec![&id1, &id0], vec![&id3, &id2, &program_id]) ); } @@ -953,7 +965,7 @@ mod tests { ]; let message = Message::new(&instructions, Some(&id1)); - let serialized = message.serialize_instructions(); + let serialized = message.serialize_instructions(/*demote_program_write_locks=*/ true); for (i, instruction) in instructions.iter().enumerate() { assert_eq!( Message::deserialize_instruction(i, &serialized).unwrap(), @@ -974,7 +986,7 @@ mod tests { ]; let message = Message::new(&instructions, Some(&id1)); - let serialized = message.serialize_instructions(); + let serialized = message.serialize_instructions(/*demote_program_write_locks=*/ true); assert_eq!( Message::deserialize_instruction(instructions.len(), &serialized).unwrap_err(), SanitizeError::IndexOutOfBounds, diff --git a/sdk/program/src/message/mapped.rs b/sdk/program/src/message/mapped.rs deleted file mode 100644 index 0f30e35238db07..00000000000000 --- a/sdk/program/src/message/mapped.rs +++ /dev/null @@ -1,289 +0,0 @@ -use { - crate::{ - message::{legacy::BUILTIN_PROGRAMS_KEYS, v0}, - pubkey::Pubkey, - sysvar, - }, - std::{collections::HashSet, convert::TryFrom}, -}; - -/// Combination of a version #0 message and its mapped addresses -#[derive(Debug, Clone)] -pub struct MappedMessage { - /// Message which loaded a collection of mapped addresses - pub message: v0::Message, - /// Collection of mapped addresses loaded by this message - pub mapped_addresses: MappedAddresses, -} - -/// Collection of mapped addresses loaded succinctly by a transaction using -/// on-chain address map accounts. -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct MappedAddresses { - /// List of addresses for writable loaded accounts - pub writable: Vec, - /// List of addresses for read-only loaded accounts - pub readonly: Vec, -} - -impl MappedMessage { - /// Returns an iterator of account key segments. The ordering of segments - /// affects how account indexes from compiled instructions are resolved and - /// so should not be changed. - fn account_keys_segment_iter(&self) -> impl Iterator> { - vec![ - &self.message.account_keys, - &self.mapped_addresses.writable, - &self.mapped_addresses.readonly, - ] - .into_iter() - } - - /// Returns the total length of loaded accounts for this message - pub fn account_keys_len(&self) -> usize { - let mut len = 0usize; - for key_segment in self.account_keys_segment_iter() { - len = len.saturating_add(key_segment.len()); - } - len - } - - /// Iterator for the addresses of the loaded accounts for this message - pub fn account_keys_iter(&self) -> impl Iterator { - self.account_keys_segment_iter().flatten() - } - - /// Returns true if any account keys are duplicates - pub fn has_duplicates(&self) -> bool { - let mut uniq = HashSet::new(); - self.account_keys_iter().any(|x| !uniq.insert(x)) - } - - /// Returns the address of the account at the specified index of the list of - /// message account keys constructed from unmapped keys, followed by mapped - /// writable addresses, and lastly the list of mapped readonly addresses. - pub fn get_account_key(&self, mut index: usize) -> Option<&Pubkey> { - for key_segment in self.account_keys_segment_iter() { - if index < key_segment.len() { - return Some(&key_segment[index]); - } - index = index.saturating_sub(key_segment.len()); - } - - None - } - - /// Returns true if the account at the specified index was requested to be - /// writable. This method should not be used directly. - fn is_writable_index(&self, key_index: usize) -> bool { - let header = &self.message.header; - let num_account_keys = self.message.account_keys.len(); - let num_signed_accounts = usize::from(header.num_required_signatures); - if key_index >= num_account_keys { - let mapped_addresses_index = key_index.saturating_sub(num_account_keys); - mapped_addresses_index < self.mapped_addresses.writable.len() - } else if key_index >= num_signed_accounts { - let num_unsigned_accounts = num_account_keys.saturating_sub(num_signed_accounts); - let num_writable_unsigned_accounts = num_unsigned_accounts - .saturating_sub(usize::from(header.num_readonly_unsigned_accounts)); - let unsigned_account_index = key_index.saturating_sub(num_signed_accounts); - unsigned_account_index < num_writable_unsigned_accounts - } else { - let num_writable_signed_accounts = num_signed_accounts - .saturating_sub(usize::from(header.num_readonly_signed_accounts)); - key_index < num_writable_signed_accounts - } - } - - /// Returns true if the account at the specified index was loaded as writable - pub fn is_writable(&self, key_index: usize, demote_program_write_locks: bool) -> bool { - if self.is_writable_index(key_index) { - if let Some(key) = self.get_account_key(key_index) { - return !(sysvar::is_sysvar_id(key) || BUILTIN_PROGRAMS_KEYS.contains(key) - || (demote_program_write_locks && self.is_key_called_as_program(key_index))); - } - } - false - } - - /// Returns true if the account at the specified index is called as a program by an instruction - pub fn is_key_called_as_program(&self, key_index: usize) -> bool { - if let Ok(key_index) = u8::try_from(key_index) { - self.message.instructions - .iter() - .any(|ix| ix.program_id_index == key_index) - } else { - false - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{instruction::CompiledInstruction, message::MessageHeader, system_program, sysvar}; - use itertools::Itertools; - - fn create_test_mapped_message() -> (MappedMessage, [Pubkey; 6]) { - let key0 = Pubkey::new_unique(); - let key1 = Pubkey::new_unique(); - let key2 = Pubkey::new_unique(); - let key3 = Pubkey::new_unique(); - let key4 = Pubkey::new_unique(); - let key5 = Pubkey::new_unique(); - - let message = MappedMessage { - message: v0::Message { - header: MessageHeader { - num_required_signatures: 2, - num_readonly_signed_accounts: 1, - num_readonly_unsigned_accounts: 1, - }, - account_keys: vec![key0, key1, key2, key3], - ..v0::Message::default() - }, - mapped_addresses: MappedAddresses { - writable: vec![key4], - readonly: vec![key5], - }, - }; - - (message, [key0, key1, key2, key3, key4, key5]) - } - - #[test] - fn test_account_keys_segment_iter() { - let (message, keys) = create_test_mapped_message(); - - let expected_segments = vec![ - vec![keys[0], keys[1], keys[2], keys[3]], - vec![keys[4]], - vec![keys[5]], - ]; - - let mut iter = message.account_keys_segment_iter(); - for expected_segment in expected_segments { - assert_eq!(iter.next(), Some(&expected_segment)); - } - } - - #[test] - fn test_account_keys_len() { - let (message, keys) = create_test_mapped_message(); - - assert_eq!(message.account_keys_len(), keys.len()); - } - - #[test] - fn test_account_keys_iter() { - let (message, keys) = create_test_mapped_message(); - - let mut iter = message.account_keys_iter(); - for expected_key in keys { - assert_eq!(iter.next(), Some(&expected_key)); - } - } - - #[test] - fn test_has_duplicates() { - let message = create_test_mapped_message().0; - - assert!(!message.has_duplicates()); - } - - #[test] - fn test_has_duplicates_with_dupe_keys() { - let create_message_with_dupe_keys = |mut keys: Vec| MappedMessage { - message: v0::Message { - account_keys: keys.split_off(2), - ..v0::Message::default() - }, - mapped_addresses: MappedAddresses { - writable: keys.split_off(2), - readonly: keys, - }, - }; - - let key0 = Pubkey::new_unique(); - let key1 = Pubkey::new_unique(); - let key2 = Pubkey::new_unique(); - let key3 = Pubkey::new_unique(); - let dupe_key = Pubkey::new_unique(); - - let keys = vec![key0, key1, key2, key3, dupe_key, dupe_key]; - let keys_len = keys.len(); - for keys in keys.into_iter().permutations(keys_len).unique() { - let message = create_message_with_dupe_keys(keys); - assert!(message.has_duplicates()); - } - } - - #[test] - fn test_get_account_key() { - let (message, keys) = create_test_mapped_message(); - - assert_eq!(message.get_account_key(0), Some(&keys[0])); - assert_eq!(message.get_account_key(1), Some(&keys[1])); - assert_eq!(message.get_account_key(2), Some(&keys[2])); - assert_eq!(message.get_account_key(3), Some(&keys[3])); - assert_eq!(message.get_account_key(4), Some(&keys[4])); - assert_eq!(message.get_account_key(5), Some(&keys[5])); - } - - #[test] - fn test_is_writable_index() { - let message = create_test_mapped_message().0; - - assert!(message.is_writable_index(0)); - assert!(!message.is_writable_index(1)); - assert!(message.is_writable_index(2)); - assert!(!message.is_writable_index(3)); - assert!(message.is_writable_index(4)); - assert!(!message.is_writable_index(5)); - } - - #[test] - fn test_is_writable() { - let mut mapped_msg = create_test_mapped_message().0; - - mapped_msg.message.account_keys[0] = sysvar::clock::id(); - assert!(mapped_msg.is_writable_index(0)); - assert!(!mapped_msg.is_writable(0, /*demote_program_write_locks=*/ true)); - - mapped_msg.message.account_keys[0] = system_program::id(); - assert!(mapped_msg.is_writable_index(0)); - assert!(!mapped_msg.is_writable(0, /*demote_program_write_locks=*/ true)); - } - - #[test] - fn test_demote_writable_program() { - let key0 = Pubkey::new_unique(); - let key1 = Pubkey::new_unique(); - let key2 = Pubkey::new_unique(); - let mapped_msg = MappedMessage { - message: v0::Message { - header: MessageHeader { - num_required_signatures: 1, - num_readonly_signed_accounts: 0, - num_readonly_unsigned_accounts: 0, - }, - account_keys: vec![key0], - instructions: vec![ - CompiledInstruction { - program_id_index: 2, - accounts: vec![1], - data: vec![], - } - ], - ..v0::Message::default() - }, - mapped_addresses: MappedAddresses { - writable: vec![key1, key2], - readonly: vec![], - }, - }; - - assert!(mapped_msg.is_writable_index(2)); - assert!(!mapped_msg.is_writable(2, /*demote_program_write_locks=*/ true)); - } -} diff --git a/sdk/program/src/message/sanitized.rs b/sdk/program/src/message/sanitized.rs deleted file mode 100644 index 3b8d3d79755c99..00000000000000 --- a/sdk/program/src/message/sanitized.rs +++ /dev/null @@ -1,597 +0,0 @@ -use { - crate::{ - fee_calculator::FeeCalculator, - hash::Hash, - instruction::{CompiledInstruction, Instruction}, - message::{MappedAddresses, MappedMessage, Message, MessageHeader}, - pubkey::Pubkey, - sanitize::{Sanitize, SanitizeError}, - secp256k1_program, - serialize_utils::{append_slice, append_u16, append_u8}, - }, - bitflags::bitflags, - std::convert::TryFrom, - thiserror::Error, -}; - -/// Sanitized message of a transaction which includes a set of atomic -/// instructions to be executed on-chain -#[derive(Debug, Clone)] -pub enum SanitizedMessage { - /// Sanitized legacy message - Legacy(Message), - /// Sanitized version #0 message with mapped addresses - V0(MappedMessage), -} - -#[derive(PartialEq, Debug, Error, Eq, Clone)] -pub enum SanitizeMessageError { - #[error("index out of bounds")] - IndexOutOfBounds, - #[error("value out of bounds")] - ValueOutOfBounds, - #[error("invalid value")] - InvalidValue, - #[error("duplicate account key")] - DuplicateAccountKey, -} - -impl From for SanitizeMessageError { - fn from(err: SanitizeError) -> Self { - match err { - SanitizeError::IndexOutOfBounds => Self::IndexOutOfBounds, - SanitizeError::ValueOutOfBounds => Self::ValueOutOfBounds, - SanitizeError::InvalidValue => Self::InvalidValue, - } - } -} - -impl TryFrom for SanitizedMessage { - type Error = SanitizeMessageError; - fn try_from(message: Message) -> Result { - message.sanitize()?; - - let sanitized_msg = Self::Legacy(message); - if sanitized_msg.has_duplicates() { - return Err(SanitizeMessageError::DuplicateAccountKey); - } - - Ok(sanitized_msg) - } -} - -bitflags! { - struct InstructionsSysvarAccountMeta: u8 { - const NONE = 0b00000000; - const IS_SIGNER = 0b00000001; - const IS_WRITABLE = 0b00000010; - } -} - -impl SanitizedMessage { - /// Return true if this message contains duplicate account keys - pub fn has_duplicates(&self) -> bool { - match self { - SanitizedMessage::Legacy(message) => message.has_duplicates(), - SanitizedMessage::V0(message) => message.has_duplicates(), - } - } - - /// Message header which identifies the number of signer and writable or - /// readonly accounts - pub fn header(&self) -> &MessageHeader { - match self { - Self::Legacy(message) => &message.header, - Self::V0(mapped_msg) => &mapped_msg.message.header, - } - } - - /// Returns a legacy message if this sanitized message wraps one - pub fn legacy_message(&self) -> Option<&Message> { - if let Self::Legacy(message) = &self { - Some(message) - } else { - None - } - } - - /// Returns the fee payer for the transaction - pub fn fee_payer(&self) -> &Pubkey { - self.get_account_key(0) - .expect("sanitized message always has non-program fee payer at index 0") - } - - /// The hash of a recent block, used for timing out a transaction - pub fn recent_blockhash(&self) -> &Hash { - match self { - Self::Legacy(message) => &message.recent_blockhash, - Self::V0(mapped_msg) => &mapped_msg.message.recent_blockhash, - } - } - - /// Program instructions that will be executed in sequence and committed in - /// one atomic transaction if all succeed. - pub fn instructions(&self) -> &[CompiledInstruction] { - match self { - Self::Legacy(message) => &message.instructions, - Self::V0(mapped_msg) => &mapped_msg.message.instructions, - } - } - - /// Program instructions iterator which includes each instruction's program - /// id. - pub fn program_instructions_iter( - &self, - ) -> impl Iterator { - match self { - Self::Legacy(message) => message.instructions.iter(), - Self::V0(mapped_msg) => mapped_msg.message.instructions.iter(), - } - .map(move |ix| { - ( - self.get_account_key(usize::from(ix.program_id_index)) - .expect("program id index is sanitized"), - ix, - ) - }) - } - - /// Iterator of all account keys referenced in this message, included mapped keys. - pub fn account_keys_iter(&self) -> Box + '_> { - match self { - Self::Legacy(message) => Box::new(message.account_keys.iter()), - Self::V0(mapped_msg) => Box::new(mapped_msg.account_keys_iter()), - } - } - - /// Length of all account keys referenced in this message, included mapped keys. - pub fn account_keys_len(&self) -> usize { - match self { - Self::Legacy(message) => message.account_keys.len(), - Self::V0(mapped_msg) => mapped_msg.account_keys_len(), - } - } - - /// Returns the address of the account at the specified index. - pub fn get_account_key(&self, index: usize) -> Option<&Pubkey> { - match self { - Self::Legacy(message) => message.account_keys.get(index), - Self::V0(message) => message.get_account_key(index), - } - } - - /// Returns true if the account at the specified index is an input to some - /// program instruction in this message. - fn is_key_passed_to_program(&self, key_index: usize) -> bool { - if let Ok(key_index) = u8::try_from(key_index) { - self.instructions() - .iter() - .any(|ix| ix.accounts.contains(&key_index)) - } else { - false - } - } - - /// Returns true if the account at the specified index is invoked as a - /// program in this message. - pub fn is_invoked(&self, key_index: usize) -> bool { - match self { - Self::Legacy(message) => message.is_key_called_as_program(key_index), - Self::V0(message) => message.is_key_called_as_program(key_index), - } - } - - /// Returns true if the account at the specified index is not invoked as a - /// program or, if invoked, is passed to a program. - pub fn is_non_loader_key(&self, key_index: usize) -> bool { - !self.is_invoked(key_index) || self.is_key_passed_to_program(key_index) - } - - /// Returns true if the account at the specified index is writable by the - /// instructions in this message. - pub fn is_writable(&self, index: usize, demote_program_write_locks: bool) -> bool { - match self { - Self::Legacy(message) => message.is_writable(index, demote_program_write_locks), - Self::V0(message) => message.is_writable(index, demote_program_write_locks), - } - } - - /// Returns true if the account at the specified index signed this - /// message. - pub fn is_signer(&self, index: usize) -> bool { - index < usize::from(self.header().num_required_signatures) - } - - // First encode the number of instructions: - // [0..2 - num_instructions - // - // Then a table of offsets of where to find them in the data - // 3..2 * num_instructions table of instruction offsets - // - // Each instruction is then encoded as: - // 0..2 - num_accounts - // 2 - meta_byte -> (bit 0 signer, bit 1 is_writable) - // 3..35 - pubkey - 32 bytes - // 35..67 - program_id - // 67..69 - data len - u16 - // 69..data_len - data - #[allow(clippy::integer_arithmetic)] - pub fn serialize_instructions(&self, demote_program_write_locks: bool) -> Vec { - // 64 bytes is a reasonable guess, calculating exactly is slower in benchmarks - let mut data = Vec::with_capacity(self.instructions().len() * (32 * 2)); - append_u16(&mut data, self.instructions().len() as u16); - for _ in 0..self.instructions().len() { - append_u16(&mut data, 0); - } - for (i, (program_id, instruction)) in self.program_instructions_iter().enumerate() { - let start_instruction_offset = data.len() as u16; - let start = 2 + (2 * i); - data[start..start + 2].copy_from_slice(&start_instruction_offset.to_le_bytes()); - append_u16(&mut data, instruction.accounts.len() as u16); - for account_index in &instruction.accounts { - let account_index = *account_index as usize; - let is_signer = self.is_signer(account_index); - let is_writable = self.is_writable(account_index, demote_program_write_locks); - let mut account_meta = InstructionsSysvarAccountMeta::NONE; - if is_signer { - account_meta |= InstructionsSysvarAccountMeta::IS_SIGNER; - } - if is_writable { - account_meta |= InstructionsSysvarAccountMeta::IS_WRITABLE; - } - append_u8(&mut data, account_meta.bits()); - append_slice( - &mut data, - self.get_account_key(account_index).unwrap().as_ref(), - ); - } - - append_slice(&mut data, program_id.as_ref()); - append_u16(&mut data, instruction.data.len() as u16); - append_slice(&mut data, &instruction.data); - } - data - } - - /// Return the mapped addresses for this message if it has any. - fn mapped_addresses(&self) -> Option<&MappedAddresses> { - match &self { - SanitizedMessage::V0(message) => Some(&message.mapped_addresses), - _ => None, - } - } - - /// Return the number of readonly accounts loaded by this message. - pub fn num_readonly_accounts(&self) -> usize { - let mapped_readonly_addresses = self - .mapped_addresses() - .map(|keys| keys.readonly.len()) - .unwrap_or_default(); - mapped_readonly_addresses - .saturating_add(usize::from(self.header().num_readonly_signed_accounts)) - .saturating_add(usize::from(self.header().num_readonly_unsigned_accounts)) - } - - fn try_position(&self, key: &Pubkey) -> Option { - u8::try_from(self.account_keys_iter().position(|k| k == key)?).ok() - } - - /// Try to compile an instruction using the account keys in this message. - pub fn try_compile_instruction(&self, ix: &Instruction) -> Option { - let accounts: Vec<_> = ix - .accounts - .iter() - .map(|account_meta| self.try_position(&account_meta.pubkey)) - .collect::>()?; - - Some(CompiledInstruction { - program_id_index: self.try_position(&ix.program_id)?, - data: ix.data.clone(), - accounts, - }) - } - - /// Calculate the total fees for a transaction given a fee calculator - pub fn calculate_fee(&self, fee_calculator: &FeeCalculator) -> u64 { - let mut num_secp256k1_signatures: u64 = 0; - for (program_id, instruction) in self.program_instructions_iter() { - if secp256k1_program::check_id(program_id) { - if let Some(num_signatures) = instruction.data.get(0) { - num_secp256k1_signatures = - num_secp256k1_signatures.saturating_add(u64::from(*num_signatures)); - } - } - } - - fee_calculator.lamports_per_signature.saturating_mul( - u64::from(self.header().num_required_signatures) - .saturating_add(num_secp256k1_signatures), - ) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - instruction::{AccountMeta, Instruction}, - message::v0, - secp256k1_program, system_instruction, - }; - - #[test] - fn test_try_from_message() { - let dupe_key = Pubkey::new_unique(); - let legacy_message_with_dupes = Message { - header: MessageHeader { - num_required_signatures: 1, - ..MessageHeader::default() - }, - account_keys: vec![dupe_key, dupe_key], - ..Message::default() - }; - - assert_eq!( - SanitizedMessage::try_from(legacy_message_with_dupes).err(), - Some(SanitizeMessageError::DuplicateAccountKey), - ); - - let legacy_message_with_no_signers = Message { - account_keys: vec![Pubkey::new_unique()], - ..Message::default() - }; - - assert_eq!( - SanitizedMessage::try_from(legacy_message_with_no_signers).err(), - Some(SanitizeMessageError::IndexOutOfBounds), - ); - } - - #[test] - fn test_is_non_loader_key() { - let key0 = Pubkey::new_unique(); - let key1 = Pubkey::new_unique(); - let loader_key = Pubkey::new_unique(); - let instructions = vec![ - CompiledInstruction::new(1, &(), vec![0]), - CompiledInstruction::new(2, &(), vec![0, 1]), - ]; - - let message = SanitizedMessage::try_from(Message::new_with_compiled_instructions( - 1, - 0, - 2, - vec![key0, key1, loader_key], - Hash::default(), - instructions, - )) - .unwrap(); - - assert!(message.is_non_loader_key(0)); - assert!(message.is_non_loader_key(1)); - assert!(!message.is_non_loader_key(2)); - } - - #[test] - fn test_num_readonly_accounts() { - let key0 = Pubkey::new_unique(); - let key1 = Pubkey::new_unique(); - let key2 = Pubkey::new_unique(); - let key3 = Pubkey::new_unique(); - let key4 = Pubkey::new_unique(); - let key5 = Pubkey::new_unique(); - - let legacy_message = SanitizedMessage::try_from(Message { - header: MessageHeader { - num_required_signatures: 2, - num_readonly_signed_accounts: 1, - num_readonly_unsigned_accounts: 1, - }, - account_keys: vec![key0, key1, key2, key3], - ..Message::default() - }) - .unwrap(); - - assert_eq!(legacy_message.num_readonly_accounts(), 2); - - let mapped_message = SanitizedMessage::V0(MappedMessage { - message: v0::Message { - header: MessageHeader { - num_required_signatures: 2, - num_readonly_signed_accounts: 1, - num_readonly_unsigned_accounts: 1, - }, - account_keys: vec![key0, key1, key2, key3], - ..v0::Message::default() - }, - mapped_addresses: MappedAddresses { - writable: vec![key4], - readonly: vec![key5], - }, - }); - - assert_eq!(mapped_message.num_readonly_accounts(), 3); - } - - #[test] - #[allow(deprecated)] - fn test_serialize_instructions() { - let program_id0 = Pubkey::new_unique(); - let program_id1 = Pubkey::new_unique(); - let id0 = Pubkey::new_unique(); - let id1 = Pubkey::new_unique(); - let id2 = Pubkey::new_unique(); - let id3 = Pubkey::new_unique(); - let instructions = vec![ - Instruction::new_with_bincode(program_id0, &0, vec![AccountMeta::new(id0, false)]), - Instruction::new_with_bincode(program_id0, &0, vec![AccountMeta::new(id1, true)]), - Instruction::new_with_bincode( - program_id1, - &0, - vec![AccountMeta::new_readonly(id2, false)], - ), - Instruction::new_with_bincode( - program_id1, - &0, - vec![AccountMeta::new_readonly(id3, true)], - ), - ]; - - let demote_program_write_locks = true; - let message = Message::new(&instructions, Some(&id1)); - let sanitized_message = SanitizedMessage::try_from(message.clone()).unwrap(); - let serialized = sanitized_message.serialize_instructions(demote_program_write_locks); - - // assert that SanitizedMessage::serialize_instructions has the same behavior as the - // deprecated Message::serialize_instructions method - assert_eq!(serialized, message.serialize_instructions()); - - // assert that Message::deserialize_instruction is compatible with SanitizedMessage::serialize_instructions - for (i, instruction) in instructions.iter().enumerate() { - assert_eq!( - Message::deserialize_instruction(i, &serialized).unwrap(), - *instruction - ); - } - } - - #[test] - fn test_calculate_fee() { - // Default: no fee. - let message = - SanitizedMessage::try_from(Message::new(&[], Some(&Pubkey::new_unique()))).unwrap(); - assert_eq!(message.calculate_fee(&FeeCalculator::default()), 0); - - // One signature, a fee. - assert_eq!(message.calculate_fee(&FeeCalculator::new(1)), 1); - - // Two signatures, double the fee. - let key0 = Pubkey::new_unique(); - let key1 = Pubkey::new_unique(); - let ix0 = system_instruction::transfer(&key0, &key1, 1); - let ix1 = system_instruction::transfer(&key1, &key0, 1); - let message = SanitizedMessage::try_from(Message::new(&[ix0, ix1], Some(&key0))).unwrap(); - assert_eq!(message.calculate_fee(&FeeCalculator::new(2)), 4); - } - - #[test] - fn test_try_compile_instruction() { - let key0 = Pubkey::new_unique(); - let key1 = Pubkey::new_unique(); - let key2 = Pubkey::new_unique(); - let program_id = Pubkey::new_unique(); - - let valid_instruction = Instruction { - program_id, - accounts: vec![ - AccountMeta::new_readonly(key0, false), - AccountMeta::new_readonly(key1, false), - AccountMeta::new_readonly(key2, false), - ], - data: vec![], - }; - - let invalid_program_id_instruction = Instruction { - program_id: Pubkey::new_unique(), - accounts: vec![ - AccountMeta::new_readonly(key0, false), - AccountMeta::new_readonly(key1, false), - AccountMeta::new_readonly(key2, false), - ], - data: vec![], - }; - - let invalid_account_key_instruction = Instruction { - program_id: Pubkey::new_unique(), - accounts: vec![ - AccountMeta::new_readonly(key0, false), - AccountMeta::new_readonly(key1, false), - AccountMeta::new_readonly(Pubkey::new_unique(), false), - ], - data: vec![], - }; - - let legacy_message = SanitizedMessage::try_from(Message { - header: MessageHeader { - num_required_signatures: 1, - num_readonly_signed_accounts: 0, - num_readonly_unsigned_accounts: 0, - }, - account_keys: vec![key0, key1, key2, program_id], - ..Message::default() - }) - .unwrap(); - - let mapped_message = SanitizedMessage::V0(MappedMessage { - message: v0::Message { - header: MessageHeader { - num_required_signatures: 1, - num_readonly_signed_accounts: 0, - num_readonly_unsigned_accounts: 0, - }, - account_keys: vec![key0, key1], - ..v0::Message::default() - }, - mapped_addresses: MappedAddresses { - writable: vec![key2], - readonly: vec![program_id], - }, - }); - - for message in vec![legacy_message, mapped_message] { - assert_eq!( - message.try_compile_instruction(&valid_instruction), - Some(CompiledInstruction { - program_id_index: 3, - accounts: vec![0, 1, 2], - data: vec![], - }) - ); - - assert!(message - .try_compile_instruction(&invalid_program_id_instruction) - .is_none()); - assert!(message - .try_compile_instruction(&invalid_account_key_instruction) - .is_none()); - } - } - - #[test] - fn test_calculate_fee_secp256k1() { - let key0 = Pubkey::new_unique(); - let key1 = Pubkey::new_unique(); - let ix0 = system_instruction::transfer(&key0, &key1, 1); - - let mut secp_instruction1 = Instruction { - program_id: secp256k1_program::id(), - accounts: vec![], - data: vec![], - }; - let mut secp_instruction2 = Instruction { - program_id: secp256k1_program::id(), - accounts: vec![], - data: vec![1], - }; - - let message = SanitizedMessage::try_from(Message::new( - &[ - ix0.clone(), - secp_instruction1.clone(), - secp_instruction2.clone(), - ], - Some(&key0), - )) - .unwrap(); - assert_eq!(message.calculate_fee(&FeeCalculator::new(1)), 2); - - secp_instruction1.data = vec![0]; - secp_instruction2.data = vec![10]; - let message = SanitizedMessage::try_from(Message::new( - &[ix0, secp_instruction1, secp_instruction2], - Some(&key0), - )) - .unwrap(); - assert_eq!(message.calculate_fee(&FeeCalculator::new(1)), 11); - } -} diff --git a/sdk/src/transaction/sanitized.rs b/sdk/src/transaction/sanitized.rs deleted file mode 100644 index d3ffc77af6da4d..00000000000000 --- a/sdk/src/transaction/sanitized.rs +++ /dev/null @@ -1,231 +0,0 @@ -#![cfg(feature = "full")] - -use { - crate::{ - hash::Hash, - message::{v0, MappedAddresses, MappedMessage, SanitizedMessage, VersionedMessage}, - nonce::NONCED_TX_MARKER_IX_INDEX, - program_utils::limited_deserialize, - pubkey::Pubkey, - sanitize::Sanitize, - secp256k1_instruction::verify_eth_addresses, - secp256k1_program, - signature::Signature, - transaction::{Result, Transaction, TransactionError, VersionedTransaction}, - }, - solana_program::{system_instruction::SystemInstruction, system_program}, - std::convert::TryFrom, -}; - -/// Sanitized transaction and the hash of its message -#[derive(Debug, Clone)] -pub struct SanitizedTransaction { - message: SanitizedMessage, - message_hash: Hash, - signatures: Vec, -} - -/// Set of accounts that must be locked for safe transaction processing -#[derive(Debug, Clone, Default)] -pub struct TransactionAccountLocks<'a> { - /// List of readonly account key locks - pub readonly: Vec<&'a Pubkey>, - /// List of writable account key locks - pub writable: Vec<&'a Pubkey>, -} - -impl TryFrom for SanitizedTransaction { - type Error = TransactionError; - fn try_from(tx: Transaction) -> Result { - tx.sanitize()?; - - if tx.message.has_duplicates() { - return Err(TransactionError::AccountLoadedTwice); - } - - Ok(Self { - message_hash: tx.message.hash(), - message: SanitizedMessage::Legacy(tx.message), - signatures: tx.signatures, - }) - } -} - -impl SanitizedTransaction { - /// Create a sanitized transaction from an unsanitized transaction. - /// If the input transaction uses address maps, attempt to map the - /// transaction keys to full addresses. - pub fn try_create( - tx: VersionedTransaction, - message_hash: Hash, - address_mapper: impl Fn(&v0::Message) -> Result, - ) -> Result { - tx.sanitize()?; - - let signatures = tx.signatures; - let message = match tx.message { - VersionedMessage::Legacy(message) => SanitizedMessage::Legacy(message), - VersionedMessage::V0(message) => SanitizedMessage::V0(MappedMessage { - mapped_addresses: address_mapper(&message)?, - message, - }), - }; - - if message.has_duplicates() { - return Err(TransactionError::AccountLoadedTwice); - } - - Ok(Self { - message, - message_hash, - signatures, - }) - } - - /// Return the first signature for this transaction. - /// - /// Notes: - /// - /// Sanitized transactions must have at least one signature because the - /// number of signatures must be greater than or equal to the message header - /// value `num_required_signatures` which must be greater than 0 itself. - pub fn signature(&self) -> &Signature { - &self.signatures[0] - } - - /// Return the list of signatures for this transaction - pub fn signatures(&self) -> &[Signature] { - &self.signatures - } - - /// Return the signed message - pub fn message(&self) -> &SanitizedMessage { - &self.message - } - - /// Return the hash of the signed message - pub fn message_hash(&self) -> &Hash { - &self.message_hash - } - - /// Convert this sanitized transaction into a versioned transaction for - /// recording in the ledger. - pub fn to_versioned_transaction(&self) -> VersionedTransaction { - let signatures = self.signatures.clone(); - match &self.message { - SanitizedMessage::V0(mapped_msg) => VersionedTransaction { - signatures, - message: VersionedMessage::V0(mapped_msg.message.clone()), - }, - SanitizedMessage::Legacy(message) => VersionedTransaction { - signatures, - message: VersionedMessage::Legacy(message.clone()), - }, - } - } - - /// Return the list of accounts that must be locked during processing this transaction. - pub fn get_account_locks(&self, demote_program_write_locks: bool) -> TransactionAccountLocks { - let message = &self.message; - let num_readonly_accounts = message.num_readonly_accounts(); - let num_writable_accounts = message - .account_keys_len() - .saturating_sub(num_readonly_accounts); - - let mut account_locks = TransactionAccountLocks { - writable: Vec::with_capacity(num_writable_accounts), - readonly: Vec::with_capacity(num_readonly_accounts), - }; - - for (i, key) in message.account_keys_iter().enumerate() { - if message.is_writable(i, demote_program_write_locks) { - account_locks.writable.push(key); - } else { - account_locks.readonly.push(key); - } - } - - account_locks - } - - /// If the transaction uses a durable nonce, return the pubkey of the nonce account - pub fn get_durable_nonce(&self) -> Option<&Pubkey> { - self.message - .instructions() - .get(NONCED_TX_MARKER_IX_INDEX as usize) - .filter( - |ix| match self.message.get_account_key(ix.program_id_index as usize) { - Some(program_id) => system_program::check_id(program_id), - _ => false, - }, - ) - .filter(|ix| { - matches!( - limited_deserialize(&ix.data), - Ok(SystemInstruction::AdvanceNonceAccount) - ) - }) - .and_then(|ix| { - ix.accounts.get(0).and_then(|idx| { - let idx = *idx as usize; - self.message.get_account_key(idx) - }) - }) - } - - /// Return the serialized message data to sign. - fn message_data(&self) -> Vec { - match &self.message { - SanitizedMessage::Legacy(message) => message.serialize(), - SanitizedMessage::V0(mapped_msg) => mapped_msg.message.serialize(), - } - } - - /// Verify the length of signatures matches the value in the message header - pub fn verify_signatures_len(&self) -> bool { - self.signatures.len() == self.message.header().num_required_signatures as usize - } - - /// Verify the transaction signatures - pub fn verify(&self) -> Result<()> { - let message_bytes = self.message_data(); - if self - .signatures - .iter() - .zip(self.message.account_keys_iter()) - .map(|(signature, pubkey)| signature.verify(pubkey.as_ref(), &message_bytes)) - .any(|verified| !verified) - { - Err(TransactionError::SignatureFailure) - } else { - Ok(()) - } - } - - /// Verify the encoded secp256k1 signatures in this transaction - pub fn verify_precompiles( - &self, - libsecp256k1_0_5_upgrade_enabled: bool, - libsecp256k1_fail_on_bad_count: bool, - ) -> Result<()> { - for (program_id, instruction) in self.message.program_instructions_iter() { - if secp256k1_program::check_id(program_id) { - let instruction_datas: Vec<_> = self - .message - .instructions() - .iter() - .map(|instruction| instruction.data.as_ref()) - .collect(); - let data = &instruction.data; - let e = verify_eth_addresses( - data, - &instruction_datas, - libsecp256k1_0_5_upgrade_enabled, - libsecp256k1_fail_on_bad_count, - ); - e.map_err(|_| TransactionError::InvalidAccountIndex)?; - } - } - Ok(()) - } -}