From b694187bba89dbd2bf07d32c405a06cc05923a3a Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Fri, 8 Nov 2019 16:54:37 -0700 Subject: [PATCH 1/4] Add getBlocksSince rpc method, and initial stub of getBlock method --- core/src/rpc.rs | 62 ++++++++++++++++++++++++++++++++++------- core/src/rpc_service.rs | 33 +++++++++++++--------- core/src/validator.rs | 11 ++++---- 3 files changed, 78 insertions(+), 28 deletions(-) diff --git a/core/src/rpc.rs b/core/src/rpc.rs index 0177c35ec85f27..9d885335243d73 100644 --- a/core/src/rpc.rs +++ b/core/src/rpc.rs @@ -13,7 +13,9 @@ use jsonrpc_core::{Error, Metadata, Result}; use jsonrpc_derive::rpc; use solana_client::rpc_request::{RpcEpochInfo, RpcVoteAccountInfo, RpcVoteAccountStatus}; use solana_drone::drone::request_airdrop_transaction; -use solana_ledger::bank_forks::BankForks; +use solana_ledger::{ + bank_forks::BankForks, blocktree::Blocktree, rooted_slot_iterator::RootedSlotIterator, +}; use solana_runtime::bank::Bank; use solana_sdk::{ account::Account, @@ -54,8 +56,9 @@ impl Default for JsonRpcConfig { pub struct JsonRpcRequestProcessor { bank_forks: Arc>, block_commitment_cache: Arc>, - storage_state: StorageState, + blocktree: Arc, config: JsonRpcConfig, + storage_state: StorageState, validator_exit: Arc>>, } @@ -75,17 +78,19 @@ impl JsonRpcRequestProcessor { } pub fn new( - storage_state: StorageState, config: JsonRpcConfig, bank_forks: Arc>, block_commitment_cache: Arc>, + blocktree: Arc, + storage_state: StorageState, validator_exit: &Arc>>, ) -> Self { JsonRpcRequestProcessor { + config, bank_forks, block_commitment_cache, + blocktree, storage_state, - config, validator_exit: validator_exit.clone(), } } @@ -258,6 +263,14 @@ impl JsonRpcRequestProcessor { Ok(false) } } + + pub fn get_blocks_since(&self, slot: Slot) -> Result> { + Ok(RootedSlotIterator::new(slot, &self.blocktree) + .map_err(|err| Error::invalid_params(format!("Slot {:?}: {:?}", slot, err)))? + .into_iter() + .map(|(slot, _)| slot) + .collect()) + } } fn get_tpu_addr(cluster_info: &Arc>) -> Result { @@ -479,6 +492,9 @@ pub trait RpcSol { #[rpc(meta, name = "setLogFilter")] fn set_log_filter(&self, _meta: Self::Metadata, filter: String) -> Result<()>; + + #[rpc(meta, name = "getBlocksSince")] + fn get_blocks_since(&self, meta: Self::Metadata, slot: Slot) -> Result>; } pub struct RpcSolImpl; @@ -929,6 +945,13 @@ impl RpcSol for RpcSolImpl { solana_logger::setup_with_filter(&filter); Ok(()) } + + fn get_blocks_since(&self, meta: Self::Metadata, slot: Slot) -> Result> { + meta.request_processor + .read() + .unwrap() + .get_blocks_since(slot) + } } #[cfg(test)] @@ -939,6 +962,7 @@ pub mod tests { genesis_utils::{create_genesis_config, GenesisConfigInfo}, }; use jsonrpc_core::{MetaIoHandler, Output, Response, Value}; + use solana_ledger::blocktree::get_tmp_ledger_path; use solana_sdk::{ fee_calculator::DEFAULT_BURN_PERCENT, hash::{hash, Hash}, @@ -980,6 +1004,8 @@ pub mod tests { .or_insert(commitment_slot1.clone()); let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::new(block_commitment, 42))); + let ledger_path = get_tmp_ledger_path!(); + let blocktree = Blocktree::open(&ledger_path).unwrap(); let leader_pubkey = *bank.collector_id(); let exit = Arc::new(AtomicBool::new(false)); @@ -993,10 +1019,11 @@ pub mod tests { let _ = bank.process_transaction(&tx); let request_processor = Arc::new(RwLock::new(JsonRpcRequestProcessor::new( - StorageState::default(), JsonRpcConfig::default(), bank_forks, block_commitment_cache.clone(), + Arc::new(blocktree), + StorageState::default(), &validator_exit, ))); let cluster_info = Arc::new(RwLock::new(ClusterInfo::new_with_invalid_keypair( @@ -1038,11 +1065,14 @@ pub mod tests { let (bank_forks, alice) = new_bank_forks(); let bank = bank_forks.read().unwrap().working_bank(); let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default())); + let ledger_path = get_tmp_ledger_path!(); + let blocktree = Blocktree::open(&ledger_path).unwrap(); let request_processor = JsonRpcRequestProcessor::new( - StorageState::default(), JsonRpcConfig::default(), bank_forks, block_commitment_cache, + Arc::new(blocktree), + StorageState::default(), &validator_exit, ); thread::spawn(move || { @@ -1453,6 +1483,8 @@ pub mod tests { let exit = Arc::new(AtomicBool::new(false)); let validator_exit = create_validator_exit(&exit); let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default())); + let ledger_path = get_tmp_ledger_path!(); + let blocktree = Blocktree::open(&ledger_path).unwrap(); let mut io = MetaIoHandler::default(); let rpc = RpcSolImpl; @@ -1460,10 +1492,11 @@ pub mod tests { let meta = Meta { request_processor: { let request_processor = JsonRpcRequestProcessor::new( - StorageState::default(), JsonRpcConfig::default(), new_bank_forks().0, block_commitment_cache, + Arc::new(blocktree), + StorageState::default(), &validator_exit, ); Arc::new(RwLock::new(request_processor)) @@ -1551,11 +1584,14 @@ pub mod tests { let exit = Arc::new(AtomicBool::new(false)); let validator_exit = create_validator_exit(&exit); let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default())); + let ledger_path = get_tmp_ledger_path!(); + let blocktree = Blocktree::open(&ledger_path).unwrap(); let request_processor = JsonRpcRequestProcessor::new( - StorageState::default(), JsonRpcConfig::default(), new_bank_forks().0, block_commitment_cache, + Arc::new(blocktree), + StorageState::default(), &validator_exit, ); assert_eq!(request_processor.validator_exit(), Ok(false)); @@ -1567,13 +1603,16 @@ pub mod tests { let exit = Arc::new(AtomicBool::new(false)); let validator_exit = create_validator_exit(&exit); let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default())); + let ledger_path = get_tmp_ledger_path!(); + let blocktree = Blocktree::open(&ledger_path).unwrap(); let mut config = JsonRpcConfig::default(); config.enable_validator_exit = true; let request_processor = JsonRpcRequestProcessor::new( - StorageState::default(), config, new_bank_forks().0, block_commitment_cache, + Arc::new(blocktree), + StorageState::default(), &validator_exit, ); assert_eq!(request_processor.validator_exit(), Ok(true)); @@ -1616,14 +1655,17 @@ pub mod tests { .or_insert(commitment_slot1.clone()); let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::new(block_commitment, 42))); + let ledger_path = get_tmp_ledger_path!(); + let blocktree = Blocktree::open(&ledger_path).unwrap(); let mut config = JsonRpcConfig::default(); config.enable_validator_exit = true; let request_processor = JsonRpcRequestProcessor::new( - StorageState::default(), config, new_bank_forks().0, block_commitment_cache, + Arc::new(blocktree), + StorageState::default(), &validator_exit, ); assert_eq!( diff --git a/core/src/rpc_service.rs b/core/src/rpc_service.rs index 08b0b980616b05..b2aa22dc0b98e7 100644 --- a/core/src/rpc_service.rs +++ b/core/src/rpc_service.rs @@ -9,13 +9,12 @@ use jsonrpc_http_server::{ hyper, AccessControlAllowOrigin, CloseHandle, DomainsValidation, RequestMiddleware, RequestMiddlewareAction, ServerBuilder, }; -use solana_ledger::bank_forks::BankForks; +use solana_ledger::{bank_forks::BankForks, blocktree::Blocktree}; use solana_sdk::hash::Hash; use std::{ net::SocketAddr, path::{Path, PathBuf}, - sync::mpsc::channel, - sync::{Arc, RwLock}, + sync::{mpsc::channel, Arc, RwLock}, thread::{self, Builder, JoinHandle}, }; use tokio::prelude::Future; @@ -87,23 +86,25 @@ impl RequestMiddleware for RpcRequestMiddleware { impl JsonRpcService { pub fn new( - cluster_info: &Arc>, rpc_addr: SocketAddr, - storage_state: StorageState, config: JsonRpcConfig, bank_forks: Arc>, block_commitment_cache: Arc>, - ledger_path: &Path, + blocktree: Arc, + cluster_info: &Arc>, genesis_hash: Hash, + ledger_path: &Path, + storage_state: StorageState, validator_exit: &Arc>>, ) -> Self { info!("rpc bound to {:?}", rpc_addr); info!("rpc configuration: {:?}", config); let request_processor = Arc::new(RwLock::new(JsonRpcRequestProcessor::new( - storage_state, config, bank_forks, block_commitment_cache, + blocktree, + storage_state, validator_exit, ))); let request_processor_ = request_processor.clone(); @@ -174,9 +175,12 @@ impl Service for JsonRpcService { #[cfg(test)] mod tests { use super::*; - use crate::contact_info::ContactInfo; - use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo}; - use crate::rpc::tests::create_validator_exit; + use crate::{ + contact_info::ContactInfo, + genesis_utils::{create_genesis_config, GenesisConfigInfo}, + rpc::tests::create_validator_exit, + }; + use solana_ledger::blocktree::get_tmp_ledger_path; use solana_runtime::bank::Bank; use solana_sdk::signature::KeypairUtil; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; @@ -201,15 +205,18 @@ mod tests { ); let bank_forks = Arc::new(RwLock::new(BankForks::new(bank.slot(), bank))); let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default())); + let ledger_path = get_tmp_ledger_path!(); + let blocktree = Blocktree::open(&ledger_path).unwrap(); let mut rpc_service = JsonRpcService::new( - &cluster_info, rpc_addr, - StorageState::default(), JsonRpcConfig::default(), bank_forks, block_commitment_cache, - &PathBuf::from("farf"), + Arc::new(blocktree), + &cluster_info, Hash::default(), + &PathBuf::from("farf"), + StorageState::default(), &validator_exit, ); let thread = rpc_service.thread_hdl.thread(); diff --git a/core/src/validator.rs b/core/src/validator.rs index 2dc0430aa5721a..7254088785fc57 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -199,18 +199,21 @@ impl Validator { bank.slots_per_segment(), ); + let blocktree = Arc::new(blocktree); + let rpc_service = if node.info.rpc.port() == 0 { None } else { Some(JsonRpcService::new( - &cluster_info, SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), node.info.rpc.port()), - storage_state.clone(), config.rpc_config.clone(), bank_forks.clone(), block_commitment_cache.clone(), - ledger_path, + blocktree.clone(), + &cluster_info, genesis_hash, + ledger_path, + storage_state.clone(), &validator_exit, )) }; @@ -244,8 +247,6 @@ impl Validator { std::thread::park(); } - let blocktree = Arc::new(blocktree); - let poh_config = Arc::new(poh_config); let (mut poh_recorder, entry_receiver) = PohRecorder::new_with_clear_signal( bank.tick_height(), From 633235f810e65f852ac5dddda63c4547308b2449 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Fri, 8 Nov 2019 19:34:58 -0700 Subject: [PATCH 2/4] Return test transactions from getBlock method --- core/src/rpc.rs | 57 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/core/src/rpc.rs b/core/src/rpc.rs index 9d885335243d73..4261984b84fc2c 100644 --- a/core/src/rpc.rs +++ b/core/src/rpc.rs @@ -4,6 +4,7 @@ use crate::{ cluster_info::ClusterInfo, commitment::{BlockCommitment, BlockCommitmentCache}, contact_info::ContactInfo, + gen_keys::GenKeys, packet::PACKET_DATA_SIZE, storage_stage::StorageState, validator::ValidatorExit, @@ -25,9 +26,11 @@ use solana_sdk::{ fee_calculator::FeeCalculator, hash::Hash, inflation::Inflation, + instruction::InstructionError, pubkey::Pubkey, - signature::Signature, - transaction::{self, Transaction}, + signature::{KeypairUtil, Signature}, + system_transaction, + transaction::{self, Transaction, TransactionError}, }; use solana_vote_api::vote_state::{VoteState, MAX_LOCKOUT_HISTORY}; use std::{ @@ -271,6 +274,10 @@ impl JsonRpcRequestProcessor { .map(|(slot, _)| slot) .collect()) } + + pub fn get_block(&self, slot: Slot) -> Result)>> { + Ok(make_test_transactions(slot)) + } } fn get_tpu_addr(cluster_info: &Arc>) -> Result { @@ -495,6 +502,13 @@ pub trait RpcSol { #[rpc(meta, name = "getBlocksSince")] fn get_blocks_since(&self, meta: Self::Metadata, slot: Slot) -> Result>; + + #[rpc(meta, name = "getBlock")] + fn get_block( + &self, + meta: Self::Metadata, + slot: Slot, + ) -> Result)>>; } pub struct RpcSolImpl; @@ -952,6 +966,43 @@ impl RpcSol for RpcSolImpl { .unwrap() .get_blocks_since(slot) } + + fn get_block( + &self, + meta: Self::Metadata, + slot: Slot, + ) -> Result)>> { + meta.request_processor.read().unwrap().get_block(slot) + } +} + +fn make_test_transactions(count: u64) -> Vec<(Transaction, transaction::Result<()>)> { + let seed = [42u8; 32]; + let keys = GenKeys::new(seed).gen_n_keypairs(count + 1); + let mut transactions: Vec<(Transaction, transaction::Result<()>)> = Vec::new(); + for x in 0..count { + let tx = system_transaction::transfer( + &keys[x as usize], + &keys[(x + 1) as usize].pubkey(), + 123, + Hash::default(), + ); + let status = if x % 3 == 0 { + Ok(()) + } else if x % 3 == 1 { + Err(TransactionError::InstructionError( + 0, + InstructionError::InsufficientFunds, + )) + } else { + Err(TransactionError::InstructionError( + 0, + InstructionError::CustomError(3), + )) + }; + transactions.push((tx, status)) + } + transactions } #[cfg(test)] @@ -967,7 +1018,7 @@ pub mod tests { fee_calculator::DEFAULT_BURN_PERCENT, hash::{hash, Hash}, instruction::InstructionError, - signature::{Keypair, KeypairUtil}, + signature::Keypair, system_transaction, transaction::TransactionError, }; From bfbcfde54b208a6bf1498a7f4982b1f6fc4752e4 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Sat, 9 Nov 2019 10:14:43 -0500 Subject: [PATCH 3/4] clippy --- core/src/rpc.rs | 1 - core/src/rpc_service.rs | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/rpc.rs b/core/src/rpc.rs index 4261984b84fc2c..7bc49bba6fa267 100644 --- a/core/src/rpc.rs +++ b/core/src/rpc.rs @@ -270,7 +270,6 @@ impl JsonRpcRequestProcessor { pub fn get_blocks_since(&self, slot: Slot) -> Result> { Ok(RootedSlotIterator::new(slot, &self.blocktree) .map_err(|err| Error::invalid_params(format!("Slot {:?}: {:?}", slot, err)))? - .into_iter() .map(|(slot, _)| slot) .collect()) } diff --git a/core/src/rpc_service.rs b/core/src/rpc_service.rs index b2aa22dc0b98e7..3f2456cca566fd 100644 --- a/core/src/rpc_service.rs +++ b/core/src/rpc_service.rs @@ -85,6 +85,7 @@ impl RequestMiddleware for RpcRequestMiddleware { } impl JsonRpcService { + #[allow(clippy::too_many_arguments)] pub fn new( rpc_addr: SocketAddr, config: JsonRpcConfig, From cd2571524a02de4754e4b421b0bdfddab1f801f6 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 11 Nov 2019 12:39:52 -0500 Subject: [PATCH 4/4] Add comment on get_block method --- core/src/rpc.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/src/rpc.rs b/core/src/rpc.rs index 7bc49bba6fa267..edcb2f0501e417 100644 --- a/core/src/rpc.rs +++ b/core/src/rpc.rs @@ -274,6 +274,10 @@ impl JsonRpcRequestProcessor { .collect()) } + // The `get_block` method is not fully implemented. It currenlty returns a batch of test transaction + // tuples (Transaction, transaction::Result) to demonstrate message format and + // TransactionErrors. Transaction count == slot, and transaction keys are derived + // deterministically to allow testers to track the pubkeys across slots. pub fn get_block(&self, slot: Slot) -> Result)>> { Ok(make_test_transactions(slot)) }