From 95bf68f3f52966e9e8be380b238b52050606d29d Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Fri, 11 May 2018 16:15:14 -0600 Subject: [PATCH 01/25] Correct some strange naming --- src/bin/client-demo.rs | 12 ++++++------ src/thin_client.rs | 33 +++++++++++++-------------------- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/src/bin/client-demo.rs b/src/bin/client-demo.rs index 3e4f01e88b8bda..5b308a0f6c9a90 100644 --- a/src/bin/client-demo.rs +++ b/src/bin/client-demo.rs @@ -87,10 +87,10 @@ fn main() { println!("Binding to {}", client_addr); let socket = UdpSocket::bind(&client_addr).unwrap(); socket.set_read_timeout(Some(Duration::new(5, 0))).unwrap(); - let mut accountant = ThinClient::new(addr.parse().unwrap(), socket); + let mut client = ThinClient::new(addr.parse().unwrap(), socket); println!("Get last ID..."); - let last_id = accountant.get_last_id().wait().unwrap(); + let last_id = client.get_last_id().wait().unwrap(); println!("Got last ID {:?}", last_id); let rnd = GenKeys::new(demo.mint.keypair().public_key_bytes()); @@ -122,7 +122,7 @@ fn main() { nsps / 1_000_f64 ); - let initial_tx_count = accountant.transaction_count(); + let initial_tx_count = client.transaction_count(); println!("initial count {}", initial_tx_count); println!("Transfering {} transactions in {} batches", txs, threads); @@ -134,16 +134,16 @@ fn main() { let mut client_addr: SocketAddr = client_addr.parse().unwrap(); client_addr.set_port(0); let socket = UdpSocket::bind(client_addr).unwrap(); - let accountant = ThinClient::new(addr.parse().unwrap(), socket); + let client = ThinClient::new(addr.parse().unwrap(), socket); for tr in trs { - accountant.transfer_signed(tr.clone()).unwrap(); + client.transfer_signed(tr.clone()).unwrap(); } }); println!("Waiting for transactions to complete...",); let mut tx_count; for _ in 0..10 { - tx_count = accountant.transaction_count(); + tx_count = client.transaction_count(); duration = now.elapsed(); let txs = tx_count - initial_tx_count; println!("Transactions processed {}", txs); diff --git a/src/thin_client.rs b/src/thin_client.rs index e1ee2b4dcd1dbf..d7ee25ab4f9f56 100644 --- a/src/thin_client.rs +++ b/src/thin_client.rs @@ -188,29 +188,22 @@ mod tests { let bob_pubkey = KeyPair::new().pubkey(); let exit = Arc::new(AtomicBool::new(false)); let accounting_stage = AccountingStage::new(accountant, &alice.last_id(), Some(30)); - let accountant = Arc::new(Tpu::new(accounting_stage)); - let threads = Tpu::serve( - &accountant, - d, - serve, - events_socket, - gossip, - exit.clone(), - sink(), - ).unwrap(); + let tpu = Arc::new(Tpu::new(accounting_stage)); + let threads = + Tpu::serve(&tpu, d, serve, events_socket, gossip, exit.clone(), sink()).unwrap(); sleep(Duration::from_millis(300)); let socket = UdpSocket::bind("0.0.0.0:0").unwrap(); - let mut accountant = ThinClient::new(addr, socket); - let last_id = accountant.get_last_id().wait().unwrap(); - let _sig = accountant + let mut client = ThinClient::new(addr, socket); + let last_id = client.get_last_id().wait().unwrap(); + let _sig = client .transfer(500, &alice.keypair(), bob_pubkey, &last_id) .unwrap(); let mut balance; let now = Instant::now(); loop { - balance = accountant.get_balance(&bob_pubkey); + balance = client.get_balance(&bob_pubkey); if balance.is_ok() { break; } @@ -372,15 +365,15 @@ mod tests { let socket = UdpSocket::bind("0.0.0.0:0").unwrap(); socket.set_read_timeout(Some(Duration::new(1, 0))).unwrap(); - let mut accountant = ThinClient::new(leader.0.serve_addr, socket); + let mut client = ThinClient::new(leader.0.serve_addr, socket); info!("getting leader last_id"); - let last_id = accountant.get_last_id().wait().unwrap(); + let last_id = client.get_last_id().wait().unwrap(); info!("executing leader transer"); - let _sig = accountant + let _sig = client .transfer(500, &alice.keypair(), bob_pubkey, &last_id) .unwrap(); info!("getting leader balance"); - accountant.get_balance(&bob_pubkey).unwrap() + client.get_balance(&bob_pubkey).unwrap() }; assert_eq!(leader_balance, 500); //verify replicant has the same balance @@ -389,9 +382,9 @@ mod tests { let socket = UdpSocket::bind("0.0.0.0:0").unwrap(); socket.set_read_timeout(Some(Duration::new(1, 0))).unwrap(); - let mut accountant = ThinClient::new(replicant.0.serve_addr, socket); + let mut client = ThinClient::new(replicant.0.serve_addr, socket); info!("getting replicant balance"); - if let Ok(bal) = accountant.get_balance(&bob_pubkey) { + if let Ok(bal) = client.get_balance(&bob_pubkey) { replicant_balance = bal; } info!("replicant balance {}", replicant_balance); From 600a1f88662e57968b5ac3ae87ebaf1a17c7d40d Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Fri, 11 May 2018 16:35:53 -0600 Subject: [PATCH 02/25] Initialize thin client with events port --- src/bin/client-demo.rs | 34 ++++++++++++++++++-------- src/thin_client.rs | 55 ++++++++++++++++++++++++++---------------- 2 files changed, 58 insertions(+), 31 deletions(-) diff --git a/src/bin/client-demo.rs b/src/bin/client-demo.rs index 5b308a0f6c9a90..da47e0e6325583 100644 --- a/src/bin/client-demo.rs +++ b/src/bin/client-demo.rs @@ -35,7 +35,7 @@ fn print_usage(program: &str, opts: Options) { fn main() { let mut threads = 4usize; let mut addr: String = "127.0.0.1:8000".to_string(); - let mut client_addr: String = "127.0.0.1:8010".to_string(); + let mut requests_addr: String = "127.0.0.1:8010".to_string(); let mut opts = Options::new(); opts.optopt("s", "", "server address", "host:port"); @@ -60,12 +60,16 @@ fn main() { addr = matches.opt_str("s").unwrap(); } if matches.opt_present("c") { - client_addr = matches.opt_str("c").unwrap(); + requests_addr = matches.opt_str("c").unwrap(); } if matches.opt_present("t") { threads = matches.opt_str("t").unwrap().parse().expect("integer"); } + let mut events_addr: SocketAddr = requests_addr.parse().unwrap(); + let requests_port = events_addr.port(); + events_addr.set_port(requests_port + 1); + if stdin_isatty() { eprintln!("nothing found on stdin, expected a json file"); exit(1); @@ -84,10 +88,13 @@ fn main() { exit(1); }); - println!("Binding to {}", client_addr); - let socket = UdpSocket::bind(&client_addr).unwrap(); - socket.set_read_timeout(Some(Duration::new(5, 0))).unwrap(); - let mut client = ThinClient::new(addr.parse().unwrap(), socket); + println!("Binding to {}", requests_addr); + let requests_socket = UdpSocket::bind(&requests_addr).unwrap(); + requests_socket + .set_read_timeout(Some(Duration::new(5, 0))) + .unwrap(); + let events_socket = UdpSocket::bind(&events_addr).unwrap(); + let mut client = ThinClient::new(addr.parse().unwrap(), requests_socket, events_socket); println!("Get last ID..."); let last_id = client.get_last_id().wait().unwrap(); @@ -131,10 +138,17 @@ fn main() { let chunks: Vec<_> = transactions.chunks(sz).collect(); chunks.into_par_iter().for_each(|trs| { println!("Transferring 1 unit {} times... to", trs.len()); - let mut client_addr: SocketAddr = client_addr.parse().unwrap(); - client_addr.set_port(0); - let socket = UdpSocket::bind(client_addr).unwrap(); - let client = ThinClient::new(addr.parse().unwrap(), socket); + let mut requests_addr: SocketAddr = requests_addr.parse().unwrap(); + requests_addr.set_port(0); + let requests_socket = UdpSocket::bind(requests_addr).unwrap(); + requests_socket + .set_read_timeout(Some(Duration::new(5, 0))) + .unwrap(); + let mut events_addr: SocketAddr = requests_addr.clone(); + let requests_port = events_addr.port(); + events_addr.set_port(requests_port + 1); + let events_socket = UdpSocket::bind(&events_addr).unwrap(); + let client = ThinClient::new(addr.parse().unwrap(), requests_socket, events_socket); for tr in trs { client.transfer_signed(tr.clone()).unwrap(); } diff --git a/src/thin_client.rs b/src/thin_client.rs index d7ee25ab4f9f56..852338f7c53f4e 100644 --- a/src/thin_client.rs +++ b/src/thin_client.rs @@ -15,7 +15,8 @@ use transaction::Transaction; pub struct ThinClient { pub addr: SocketAddr, - pub socket: UdpSocket, + pub requests_socket: UdpSocket, + pub events_socket: UdpSocket, last_id: Option, num_events: u64, balances: HashMap>, @@ -23,12 +24,13 @@ pub struct ThinClient { impl ThinClient { /// Create a new ThinClient that will interface with Tpu - /// over `socket`. To receive responses, the caller must bind `socket` + /// over `requests_socket` and `events_socket`. To receive responses, the caller must bind `socket` /// to a public address before invoking ThinClient methods. - pub fn new(addr: SocketAddr, socket: UdpSocket) -> Self { + pub fn new(addr: SocketAddr, requests_socket: UdpSocket, events_socket: UdpSocket) -> Self { let client = ThinClient { addr: addr, - socket, + requests_socket, + events_socket, last_id: None, num_events: 0, balances: HashMap::new(), @@ -42,13 +44,13 @@ impl ThinClient { let req = Request::Subscribe { subscriptions }; let data = serialize(&req).expect("serialize Subscribe in thin_client"); trace!("subscribing to {}", self.addr); - let _res = self.socket.send_to(&data, &self.addr); + let _res = self.requests_socket.send_to(&data, &self.addr); } pub fn recv_response(&self) -> io::Result { let mut buf = vec![0u8; 1024]; info!("start recv_from"); - self.socket.recv_from(&mut buf)?; + self.requests_socket.recv_from(&mut buf)?; info!("end recv_from"); let resp = deserialize(&buf).expect("deserialize balance in thin_client"); Ok(resp) @@ -73,7 +75,7 @@ impl ThinClient { pub fn transfer_signed(&self, tr: Transaction) -> io::Result { let req = Request::Transaction(tr); let data = serialize(&req).expect("serialize Transaction in pub fn transfer_signed"); - self.socket.send_to(&data, &self.addr) + self.requests_socket.send_to(&data, &self.addr) } /// Creates, signs, and processes a Transaction. Useful for writing unit-tests. @@ -96,7 +98,7 @@ impl ThinClient { info!("get_balance"); let req = Request::GetBalance { key: *pubkey }; let data = serialize(&req).expect("serialize GetBalance in pub fn get_balance"); - self.socket + self.requests_socket .send_to(&data, &self.addr) .expect("buffer error in pub fn get_balance"); let mut done = false; @@ -133,7 +135,7 @@ impl ThinClient { } // Then take the rest. - self.socket + self.requests_socket .set_nonblocking(true) .expect("set_nonblocking in pub fn transaction_count"); loop { @@ -142,7 +144,7 @@ impl ThinClient { Ok(resp) => self.process_response(resp), } } - self.socket + self.requests_socket .set_nonblocking(false) .expect("set_nonblocking in pub fn transaction_count"); self.num_events @@ -193,9 +195,10 @@ mod tests { Tpu::serve(&tpu, d, serve, events_socket, gossip, exit.clone(), sink()).unwrap(); sleep(Duration::from_millis(300)); - let socket = UdpSocket::bind("0.0.0.0:0").unwrap(); + let requests_socket = UdpSocket::bind("0.0.0.0:0").unwrap(); + let events_socket = UdpSocket::bind("0.0.0.0:0").unwrap(); - let mut client = ThinClient::new(addr, socket); + let mut client = ThinClient::new(addr, requests_socket, events_socket); let last_id = client.get_last_id().wait().unwrap(); let _sig = client .transfer(500, &alice.keypair(), bob_pubkey, &last_id) @@ -239,9 +242,12 @@ mod tests { ).unwrap(); sleep(Duration::from_millis(300)); - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - socket.set_read_timeout(Some(Duration::new(5, 0))).unwrap(); - let mut client = ThinClient::new(serve_addr, socket); + let requests_socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + requests_socket + .set_read_timeout(Some(Duration::new(5, 0))) + .unwrap(); + let events_socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let mut client = ThinClient::new(serve_addr, requests_socket, events_socket); let last_id = client.get_last_id().wait().unwrap(); trace!("doing stuff"); @@ -362,10 +368,13 @@ mod tests { //verify leader can do transfer let leader_balance = { - let socket = UdpSocket::bind("0.0.0.0:0").unwrap(); - socket.set_read_timeout(Some(Duration::new(1, 0))).unwrap(); + let requests_socket = UdpSocket::bind("0.0.0.0:0").unwrap(); + requests_socket + .set_read_timeout(Some(Duration::new(1, 0))) + .unwrap(); + let events_socket = UdpSocket::bind("0.0.0.0:0").unwrap(); - let mut client = ThinClient::new(leader.0.serve_addr, socket); + let mut client = ThinClient::new(leader.0.serve_addr, requests_socket, events_socket); info!("getting leader last_id"); let last_id = client.get_last_id().wait().unwrap(); info!("executing leader transer"); @@ -379,10 +388,14 @@ mod tests { //verify replicant has the same balance let mut replicant_balance = 0; for _ in 0..10 { - let socket = UdpSocket::bind("0.0.0.0:0").unwrap(); - socket.set_read_timeout(Some(Duration::new(1, 0))).unwrap(); + let requests_socket = UdpSocket::bind("0.0.0.0:0").unwrap(); + requests_socket + .set_read_timeout(Some(Duration::new(1, 0))) + .unwrap(); + let events_socket = UdpSocket::bind("0.0.0.0:0").unwrap(); - let mut client = ThinClient::new(replicant.0.serve_addr, socket); + let mut client = + ThinClient::new(replicant.0.serve_addr, requests_socket, events_socket); info!("getting replicant balance"); if let Ok(bal) = client.get_balance(&bob_pubkey) { replicant_balance = bal; From 55100854d60fa36ec74652e5e40c2186cf62f4a4 Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Fri, 11 May 2018 16:41:35 -0600 Subject: [PATCH 03/25] Better names --- src/tpu.rs | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/tpu.rs b/src/tpu.rs index 95bfbc48ad35a5..394282c566f0b7 100644 --- a/src/tpu.rs +++ b/src/tpu.rs @@ -202,7 +202,7 @@ impl Tpu { pub fn serve( obj: &SharedTpu, me: ReplicatedData, - serve: UdpSocket, + requests_socket: UdpSocket, _events_socket: UdpSocket, gossip: UdpSocket, exit: Arc, @@ -213,15 +213,19 @@ impl Tpu { let t_listen = Crdt::listen(crdt.clone(), gossip, exit.clone()); // make sure we are on the same interface - let mut local = serve.local_addr()?; + let mut local = requests_socket.local_addr()?; local.set_port(0); let respond_socket = UdpSocket::bind(local.clone())?; let packet_recycler = packet::PacketRecycler::default(); let blob_recycler = packet::BlobRecycler::default(); let (packet_sender, packet_receiver) = channel(); - let t_receiver = - streamer::receiver(serve, exit.clone(), packet_recycler.clone(), packet_sender)?; + let t_receiver = streamer::receiver( + requests_socket, + exit.clone(), + packet_recycler.clone(), + packet_sender, + )?; let (responder_sender, responder_receiver) = channel(); let t_responder = streamer::responder( respond_socket, @@ -317,7 +321,7 @@ impl Tpu { obj: &SharedTpu, me: ReplicatedData, gossip: UdpSocket, - serve: UdpSocket, + requests_socket: UdpSocket, replicate: UdpSocket, leader: ReplicatedData, exit: Arc, @@ -380,15 +384,19 @@ impl Tpu { //serve pipeline // make sure we are on the same interface - let mut local = serve.local_addr()?; + let mut local = requests_socket.local_addr()?; local.set_port(0); let respond_socket = UdpSocket::bind(local.clone())?; let packet_recycler = packet::PacketRecycler::default(); let blob_recycler = packet::BlobRecycler::default(); let (packet_sender, packet_receiver) = channel(); - let t_packet_receiver = - streamer::receiver(serve, exit.clone(), packet_recycler.clone(), packet_sender)?; + let t_packet_receiver = streamer::receiver( + requests_socket, + exit.clone(), + packet_recycler.clone(), + packet_sender, + )?; let (responder_sender, responder_receiver) = channel(); let t_responder = streamer::responder( respond_socket, @@ -458,15 +466,15 @@ pub fn test_node() -> (ReplicatedData, UdpSocket, UdpSocket, UdpSocket, UdpSocke let events_socket = UdpSocket::bind("127.0.0.1:0").unwrap(); let gossip = UdpSocket::bind("127.0.0.1:0").unwrap(); let replicate = UdpSocket::bind("127.0.0.1:0").unwrap(); - let serve = UdpSocket::bind("127.0.0.1:0").unwrap(); + let requests_socket = UdpSocket::bind("127.0.0.1:0").unwrap(); let pubkey = KeyPair::new().pubkey(); let d = ReplicatedData::new( pubkey, gossip.local_addr().unwrap(), replicate.local_addr().unwrap(), - serve.local_addr().unwrap(), + requests_socket.local_addr().unwrap(), ); - (d, gossip, replicate, serve, events_socket) + (d, gossip, replicate, requests_socket, events_socket) } #[cfg(test)] From f0be595e4c164e3f75e5538587275a3b2c0264f0 Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Fri, 11 May 2018 17:58:27 -0600 Subject: [PATCH 04/25] Create function for thin client thread --- src/tpu.rs | 77 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 43 insertions(+), 34 deletions(-) diff --git a/src/tpu.rs b/src/tpu.rs index 394282c566f0b7..7865e6b4002dd1 100644 --- a/src/tpu.rs +++ b/src/tpu.rs @@ -16,7 +16,7 @@ use std::io::Write; use std::io::sink; use std::net::UdpSocket; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::mpsc::{channel, Sender}; +use std::sync::mpsc::{channel, Receiver, Sender}; use std::sync::{Arc, Mutex, RwLock}; use std::thread::{spawn, JoinHandle}; use std::time::Duration; @@ -196,6 +196,30 @@ impl Tpu { Ok(()) } + fn thin_client_service( + obj: SharedTpu, + exit: Arc, + verified_receiver: Receiver)>>, + responder_sender: streamer::BlobSender, + packet_recycler: packet::PacketRecycler, + blob_recycler: packet::BlobRecycler, + ) -> JoinHandle<()> { + spawn(move || loop { + let e = obj.thin_client_service.process_request_packets( + &obj.accounting_stage, + &verified_receiver, + &responder_sender, + &packet_recycler, + &blob_recycler, + ); + if e.is_err() { + if exit.load(Ordering::Relaxed) { + break; + } + } + }) + } + /// Create a UDP microservice that forwards messages the given Tpu. /// This service is the network leader /// Set `exit` to shutdown its threads. @@ -270,26 +294,19 @@ impl Tpu { Mutex::new(writer), ); - let tpu = obj.clone(); - let t_server = spawn(move || loop { - let e = tpu.thin_client_service.process_request_packets( - &tpu.accounting_stage, - &verified_receiver, - &responder_sender, - &packet_recycler, - &blob_recycler, - ); - if e.is_err() { - if exit.load(Ordering::Relaxed) { - break; - } - } - }); + let t_thin_client = Self::thin_client_service( + obj.clone(), + exit.clone(), + verified_receiver, + responder_sender, + packet_recycler.clone(), + blob_recycler.clone(), + ); let mut threads = vec![ t_receiver, t_responder, - t_server, + t_thin_client, t_sync, t_gossip, t_listen, @@ -423,22 +440,14 @@ impl Tpu { } let t_sync = Self::sync_no_broadcast_service(obj.clone(), exit.clone()); - let tpu = obj.clone(); - let s_exit = exit.clone(); - let t_server = spawn(move || loop { - let e = tpu.thin_client_service.process_request_packets( - &tpu.accounting_stage, - &verified_receiver, - &responder_sender, - &packet_recycler, - &blob_recycler, - ); - if e.is_err() { - if s_exit.load(Ordering::Relaxed) { - break; - } - } - }); + let t_thin_client = Self::thin_client_service( + obj.clone(), + exit.clone(), + verified_receiver, + responder_sender, + packet_recycler.clone(), + blob_recycler.clone(), + ); let mut threads = vec![ //replicate threads @@ -451,7 +460,7 @@ impl Tpu { //serve threads t_packet_receiver, t_responder, - t_server, + t_thin_client, t_sync, ]; threads.extend(verify_threads.into_iter()); From 0488d0a82f655eeb29ddf8cadecc07e87abe8bd2 Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Fri, 11 May 2018 19:59:40 -0600 Subject: [PATCH 05/25] Extract sig verify functions --- src/tpu.rs | 61 +++++++++++++++++++++++++++--------------------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/src/tpu.rs b/src/tpu.rs index 7865e6b4002dd1..b979d8742a17a8 100644 --- a/src/tpu.rs +++ b/src/tpu.rs @@ -220,6 +220,31 @@ impl Tpu { }) } + fn verifier_service( + exit: Arc, + packets_receiver: Arc>, + verified_sender: Arc)>>>>, + ) -> JoinHandle<()> { + spawn(move || loop { + let e = Self::verifier(&packets_receiver.clone(), &verified_sender.clone()); + if e.is_err() && exit.load(Ordering::Relaxed) { + break; + } + }) + } + + fn verifier_services( + exit: Arc, + packets_receiver: streamer::PacketReceiver, + verified_sender: Sender)>>, + ) -> Vec> { + let sender = Arc::new(Mutex::new(verified_sender)); + let receiver = Arc::new(Mutex::new(packets_receiver)); + (0..4) + .map(|_| Self::verifier_service(exit.clone(), receiver.clone(), sender.clone())) + .collect() + } + /// Create a UDP microservice that forwards messages the given Tpu. /// This service is the network leader /// Set `exit` to shutdown its threads. @@ -257,23 +282,10 @@ impl Tpu { blob_recycler.clone(), responder_receiver, ); - let (verified_sender, verified_receiver) = channel(); - let mut verify_threads = Vec::new(); - let shared_verified_sender = Arc::new(Mutex::new(verified_sender)); - let shared_packet_receiver = Arc::new(Mutex::new(packet_receiver)); - for _ in 0..4 { - let exit_ = exit.clone(); - let recv = shared_packet_receiver.clone(); - let sender = shared_verified_sender.clone(); - let thread = spawn(move || loop { - let e = Self::verifier(&recv, &sender); - if e.is_err() && exit_.load(Ordering::Relaxed) { - break; - } - }); - verify_threads.push(thread); - } + let (verified_sender, verified_receiver) = channel(); + let verify_threads: Vec<_> = + Self::verifier_services(exit.clone(), packet_receiver, verified_sender); let (broadcast_sender, broadcast_receiver) = channel(); @@ -422,22 +434,9 @@ impl Tpu { responder_receiver, ); let (verified_sender, verified_receiver) = channel(); + let verify_threads: Vec<_> = + Self::verifier_services(exit.clone(), packet_receiver, verified_sender); - let mut verify_threads = Vec::new(); - let shared_verified_sender = Arc::new(Mutex::new(verified_sender)); - let shared_packet_receiver = Arc::new(Mutex::new(packet_receiver)); - for _ in 0..4 { - let exit_ = exit.clone(); - let recv = shared_packet_receiver.clone(); - let sender = shared_verified_sender.clone(); - let thread = spawn(move || loop { - let e = Self::verifier(&recv, &sender); - if e.is_err() && exit_.load(Ordering::Relaxed) { - break; - } - }); - verify_threads.push(thread); - } let t_sync = Self::sync_no_broadcast_service(obj.clone(), exit.clone()); let t_thin_client = Self::thin_client_service( From 3cedbc493e0194aeb036f6c44023992456a332a9 Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Fri, 11 May 2018 20:11:25 -0600 Subject: [PATCH 06/25] Reorder to reflect the pipeline order --- src/tpu.rs | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/tpu.rs b/src/tpu.rs index b979d8742a17a8..67a4cac86ef530 100644 --- a/src/tpu.rs +++ b/src/tpu.rs @@ -268,6 +268,7 @@ impl Tpu { let packet_recycler = packet::PacketRecycler::default(); let blob_recycler = packet::BlobRecycler::default(); + let (packet_sender, packet_receiver) = channel(); let t_receiver = streamer::receiver( requests_socket, @@ -275,19 +276,19 @@ impl Tpu { packet_recycler.clone(), packet_sender, )?; - let (responder_sender, responder_receiver) = channel(); - let t_responder = streamer::responder( - respond_socket, - exit.clone(), - blob_recycler.clone(), - responder_receiver, - ); let (verified_sender, verified_receiver) = channel(); let verify_threads: Vec<_> = Self::verifier_services(exit.clone(), packet_receiver, verified_sender); let (broadcast_sender, broadcast_receiver) = channel(); + let t_sync = Self::sync_service( + obj.clone(), + exit.clone(), + broadcast_sender, + blob_recycler.clone(), + Mutex::new(writer), + ); let broadcast_socket = UdpSocket::bind(local)?; let t_broadcast = streamer::broadcaster( @@ -298,12 +299,12 @@ impl Tpu { broadcast_receiver, ); - let t_sync = Self::sync_service( - obj.clone(), + let (responder_sender, responder_receiver) = channel(); + let t_responder = streamer::responder( + respond_socket, exit.clone(), - broadcast_sender, blob_recycler.clone(), - Mutex::new(writer), + responder_receiver, ); let t_thin_client = Self::thin_client_service( From 765d90153085a9be0d8a8e30a31be0ccad40afe3 Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Fri, 11 May 2018 20:18:04 -0600 Subject: [PATCH 07/25] Better names --- src/tpu.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/tpu.rs b/src/tpu.rs index 67a4cac86ef530..3b8fa0fba03f2c 100644 --- a/src/tpu.rs +++ b/src/tpu.rs @@ -80,7 +80,7 @@ impl Tpu { /// Process any Entry items that have been published by the Historian. /// continuosly broadcast blobs of entries out - fn run_sync( + fn broadcast_entries( &self, broadcast: &streamer::BlobSender, blob_recycler: &packet::BlobRecycler, @@ -96,7 +96,7 @@ impl Tpu { Ok(()) } - pub fn sync_service( + pub fn broadcast_service( obj: SharedTpu, exit: Arc, broadcast: streamer::BlobSender, @@ -104,9 +104,9 @@ impl Tpu { writer: Mutex, ) -> JoinHandle<()> { spawn(move || loop { - let _ = obj.run_sync(&broadcast, &blob_recycler, &writer); + let _ = obj.broadcast_entries(&broadcast, &blob_recycler, &writer); if exit.load(Ordering::Relaxed) { - info!("sync_service exiting"); + info!("broadcat_service exiting"); break; } }) @@ -114,16 +114,16 @@ impl Tpu { /// Process any Entry items that have been published by the Historian. /// continuosly broadcast blobs of entries out - fn run_sync_no_broadcast(&self) -> Result<()> { + fn drain_entries(&self) -> Result<()> { self.write_entries(&Arc::new(Mutex::new(sink())))?; Ok(()) } - pub fn sync_no_broadcast_service(obj: SharedTpu, exit: Arc) -> JoinHandle<()> { + pub fn write_service(obj: SharedTpu, exit: Arc) -> JoinHandle<()> { spawn(move || loop { - let _ = obj.run_sync_no_broadcast(); + let _ = obj.drain_entries(); if exit.load(Ordering::Relaxed) { - info!("sync_no_broadcast_service exiting"); + info!("write_service exiting"); break; } }) @@ -282,7 +282,7 @@ impl Tpu { Self::verifier_services(exit.clone(), packet_receiver, verified_sender); let (broadcast_sender, broadcast_receiver) = channel(); - let t_sync = Self::sync_service( + let t_sync = Self::broadcast_service( obj.clone(), exit.clone(), broadcast_sender, @@ -438,7 +438,7 @@ impl Tpu { let verify_threads: Vec<_> = Self::verifier_services(exit.clone(), packet_receiver, verified_sender); - let t_sync = Self::sync_no_broadcast_service(obj.clone(), exit.clone()); + let t_sync = Self::write_service(obj.clone(), exit.clone()); let t_thin_client = Self::thin_client_service( obj.clone(), From b781fdbd043f359eefeb0e89f338377fb5bde428 Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Fri, 11 May 2018 20:50:50 -0600 Subject: [PATCH 08/25] Reorganize --- src/tpu.rs | 129 ++++++++++++++++++++++++++--------------------------- 1 file changed, 64 insertions(+), 65 deletions(-) diff --git a/src/tpu.rs b/src/tpu.rs index 3b8fa0fba03f2c..d701e1590de8ea 100644 --- a/src/tpu.rs +++ b/src/tpu.rs @@ -80,7 +80,7 @@ impl Tpu { /// Process any Entry items that have been published by the Historian. /// continuosly broadcast blobs of entries out - fn broadcast_entries( + fn write_and_send_entries( &self, broadcast: &streamer::BlobSender, blob_recycler: &packet::BlobRecycler, @@ -96,7 +96,7 @@ impl Tpu { Ok(()) } - pub fn broadcast_service( + pub fn write_service( obj: SharedTpu, exit: Arc, broadcast: streamer::BlobSender, @@ -104,7 +104,7 @@ impl Tpu { writer: Mutex, ) -> JoinHandle<()> { spawn(move || loop { - let _ = obj.broadcast_entries(&broadcast, &blob_recycler, &writer); + let _ = obj.write_and_send_entries(&broadcast, &blob_recycler, &writer); if exit.load(Ordering::Relaxed) { info!("broadcat_service exiting"); break; @@ -119,11 +119,11 @@ impl Tpu { Ok(()) } - pub fn write_service(obj: SharedTpu, exit: Arc) -> JoinHandle<()> { + pub fn drain_service(obj: SharedTpu, exit: Arc) -> JoinHandle<()> { spawn(move || loop { let _ = obj.drain_entries(); if exit.load(Ordering::Relaxed) { - info!("write_service exiting"); + info!("drain_service exiting"); break; } }) @@ -176,24 +176,29 @@ impl Tpu { Ok(()) } - /// Process verified blobs, already in order - /// Respond with a signed hash of the state - fn replicate_state( - obj: &Tpu, - verified_receiver: &streamer::BlobReceiver, - blob_recycler: &packet::BlobRecycler, - ) -> Result<()> { - let timer = Duration::new(1, 0); - let blobs = verified_receiver.recv_timeout(timer)?; - trace!("replicating blobs {}", blobs.len()); - let entries = ledger::reconstruct_entries_from_blobs(&blobs); - obj.accounting_stage - .accountant - .process_verified_entries(entries)?; - for blob in blobs { - blob_recycler.recycle(blob); - } - Ok(()) + fn verifier_service( + exit: Arc, + packets_receiver: Arc>, + verified_sender: Arc)>>>>, + ) -> JoinHandle<()> { + spawn(move || loop { + let e = Self::verifier(&packets_receiver.clone(), &verified_sender.clone()); + if e.is_err() && exit.load(Ordering::Relaxed) { + break; + } + }) + } + + fn verifier_services( + exit: Arc, + packets_receiver: streamer::PacketReceiver, + verified_sender: Sender)>>, + ) -> Vec> { + let sender = Arc::new(Mutex::new(verified_sender)); + let receiver = Arc::new(Mutex::new(packets_receiver)); + (0..4) + .map(|_| Self::verifier_service(exit.clone(), receiver.clone(), sender.clone())) + .collect() } fn thin_client_service( @@ -220,31 +225,6 @@ impl Tpu { }) } - fn verifier_service( - exit: Arc, - packets_receiver: Arc>, - verified_sender: Arc)>>>>, - ) -> JoinHandle<()> { - spawn(move || loop { - let e = Self::verifier(&packets_receiver.clone(), &verified_sender.clone()); - if e.is_err() && exit.load(Ordering::Relaxed) { - break; - } - }) - } - - fn verifier_services( - exit: Arc, - packets_receiver: streamer::PacketReceiver, - verified_sender: Sender)>>, - ) -> Vec> { - let sender = Arc::new(Mutex::new(verified_sender)); - let receiver = Arc::new(Mutex::new(packets_receiver)); - (0..4) - .map(|_| Self::verifier_service(exit.clone(), receiver.clone(), sender.clone())) - .collect() - } - /// Create a UDP microservice that forwards messages the given Tpu. /// This service is the network leader /// Set `exit` to shutdown its threads. @@ -264,11 +244,8 @@ impl Tpu { // make sure we are on the same interface let mut local = requests_socket.local_addr()?; local.set_port(0); - let respond_socket = UdpSocket::bind(local.clone())?; let packet_recycler = packet::PacketRecycler::default(); - let blob_recycler = packet::BlobRecycler::default(); - let (packet_sender, packet_receiver) = channel(); let t_receiver = streamer::receiver( requests_socket, @@ -281,8 +258,19 @@ impl Tpu { let verify_threads: Vec<_> = Self::verifier_services(exit.clone(), packet_receiver, verified_sender); + let blob_recycler = packet::BlobRecycler::default(); + let (responder_sender, responder_receiver) = channel(); + let t_thin_client = Self::thin_client_service( + obj.clone(), + exit.clone(), + verified_receiver, + responder_sender, + packet_recycler.clone(), + blob_recycler.clone(), + ); + let (broadcast_sender, broadcast_receiver) = channel(); - let t_sync = Self::broadcast_service( + let t_write = Self::write_service( obj.clone(), exit.clone(), broadcast_sender, @@ -299,7 +287,7 @@ impl Tpu { broadcast_receiver, ); - let (responder_sender, responder_receiver) = channel(); + let respond_socket = UdpSocket::bind(local.clone())?; let t_responder = streamer::responder( respond_socket, exit.clone(), @@ -307,20 +295,11 @@ impl Tpu { responder_receiver, ); - let t_thin_client = Self::thin_client_service( - obj.clone(), - exit.clone(), - verified_receiver, - responder_sender, - packet_recycler.clone(), - blob_recycler.clone(), - ); - let mut threads = vec![ t_receiver, t_responder, t_thin_client, - t_sync, + t_write, t_gossip, t_listen, t_broadcast, @@ -329,6 +308,26 @@ impl Tpu { Ok(threads) } + /// Process verified blobs, already in order + /// Respond with a signed hash of the state + fn replicate_state( + obj: &Tpu, + verified_receiver: &streamer::BlobReceiver, + blob_recycler: &packet::BlobRecycler, + ) -> Result<()> { + let timer = Duration::new(1, 0); + let blobs = verified_receiver.recv_timeout(timer)?; + trace!("replicating blobs {}", blobs.len()); + let entries = ledger::reconstruct_entries_from_blobs(&blobs); + obj.accounting_stage + .accountant + .process_verified_entries(entries)?; + for blob in blobs { + blob_recycler.recycle(blob); + } + Ok(()) + } + /// This service receives messages from a leader in the network and processes the transactions /// on the accountant state. /// # Arguments @@ -438,7 +437,7 @@ impl Tpu { let verify_threads: Vec<_> = Self::verifier_services(exit.clone(), packet_receiver, verified_sender); - let t_sync = Self::write_service(obj.clone(), exit.clone()); + let t_write = Self::drain_service(obj.clone(), exit.clone()); let t_thin_client = Self::thin_client_service( obj.clone(), @@ -461,7 +460,7 @@ impl Tpu { t_packet_receiver, t_responder, t_thin_client, - t_sync, + t_write, ]; threads.extend(verify_threads.into_iter()); Ok(threads) From 3c11a91f7778b57a12830f443e2cbe12b70431a6 Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Fri, 11 May 2018 21:01:07 -0600 Subject: [PATCH 09/25] Cleanup verifier error handling --- src/tpu.rs | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/tpu.rs b/src/tpu.rs index d701e1590de8ea..d2be36783b1401 100644 --- a/src/tpu.rs +++ b/src/tpu.rs @@ -129,18 +129,9 @@ impl Tpu { }) } - fn verify_batch( - batch: Vec, - sendr: &Arc)>>>>, - ) -> Result<()> { + fn verify_batch(batch: Vec) -> Vec<(SharedPackets, Vec)> { let r = ecdsa::ed25519_verify(&batch); - let res = batch.into_iter().zip(r).collect(); - sendr - .lock() - .expect("lock in fn verify_batch in tpu") - .send(res)?; - // TODO: fix error handling here? - Ok(()) + batch.into_iter().zip(r).collect() } fn verifier( @@ -160,7 +151,11 @@ impl Tpu { rand_id ); - Self::verify_batch(batch, sendr).expect("verify_batch in fn verifier"); + let verified_batch = Self::verify_batch(batch); + sendr + .lock() + .expect("lock in fn verify_batch in tpu") + .send(verified_batch)?; let total_time_ms = timing::duration_as_ms(&now.elapsed()); let total_time_s = timing::duration_as_s(&now.elapsed()); From 19607886f77d35c6d83fed46952750cb301735da Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Fri, 11 May 2018 21:51:37 -0600 Subject: [PATCH 10/25] Move sig verification stage into its own module --- src/lib.rs | 1 + src/sig_verify_stage.rs | 96 +++++++++++++++++++++++++++++++++++++++++ src/tpu.rs | 91 ++++---------------------------------- 3 files changed, 106 insertions(+), 82 deletions(-) create mode 100644 src/sig_verify_stage.rs diff --git a/src/lib.rs b/src/lib.rs index 10716a9eb8eef4..51e6471e7d0c47 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,6 +16,7 @@ pub mod packet; pub mod plan; pub mod recorder; pub mod result; +pub mod sig_verify_stage; pub mod signature; pub mod streamer; pub mod thin_client; diff --git a/src/sig_verify_stage.rs b/src/sig_verify_stage.rs new file mode 100644 index 00000000000000..db6c6dbab717ae --- /dev/null +++ b/src/sig_verify_stage.rs @@ -0,0 +1,96 @@ +//! The `sig_verify_stage` implements the signature verification stage of the TPU. + +use ecdsa; +use packet::SharedPackets; +use rand::{thread_rng, Rng}; +use result::Result; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{channel, Receiver, Sender}; +use std::sync::{Arc, Mutex}; +use std::thread::{spawn, JoinHandle}; +use std::time::Instant; +use streamer; +use timing; + +pub struct SigVerifyStage { + pub output: Receiver)>>, + pub thread_hdls: Vec>, +} + +impl SigVerifyStage { + pub fn new(exit: Arc, packets_receiver: Receiver) -> Self { + let (verified_sender, output) = channel(); + let thread_hdls = Self::verifier_services(exit, packets_receiver, verified_sender); + SigVerifyStage { + thread_hdls, + output, + } + } + + fn verify_batch(batch: Vec) -> Vec<(SharedPackets, Vec)> { + let r = ecdsa::ed25519_verify(&batch); + batch.into_iter().zip(r).collect() + } + + fn verifier( + recvr: &Arc>, + sendr: &Arc)>>>>, + ) -> Result<()> { + let (batch, len) = + streamer::recv_batch(&recvr.lock().expect("'recvr' lock in fn verifier"))?; + + let now = Instant::now(); + let batch_len = batch.len(); + let rand_id = thread_rng().gen_range(0, 100); + info!( + "@{:?} verifier: verifying: {} id: {}", + timing::timestamp(), + batch.len(), + rand_id + ); + + let verified_batch = Self::verify_batch(batch); + sendr + .lock() + .expect("lock in fn verify_batch in tpu") + .send(verified_batch)?; + + let total_time_ms = timing::duration_as_ms(&now.elapsed()); + let total_time_s = timing::duration_as_s(&now.elapsed()); + info!( + "@{:?} verifier: done. batches: {} total verify time: {:?} id: {} verified: {} v/s {}", + timing::timestamp(), + batch_len, + total_time_ms, + rand_id, + len, + (len as f32 / total_time_s) + ); + Ok(()) + } + + fn verifier_service( + exit: Arc, + packets_receiver: Arc>, + verified_sender: Arc)>>>>, + ) -> JoinHandle<()> { + spawn(move || loop { + let e = Self::verifier(&packets_receiver.clone(), &verified_sender.clone()); + if e.is_err() && exit.load(Ordering::Relaxed) { + break; + } + }) + } + + fn verifier_services( + exit: Arc, + packets_receiver: streamer::PacketReceiver, + verified_sender: Sender)>>, + ) -> Vec> { + let sender = Arc::new(Mutex::new(verified_sender)); + let receiver = Arc::new(Mutex::new(packets_receiver)); + (0..4) + .map(|_| Self::verifier_service(exit.clone(), receiver.clone(), sender.clone())) + .collect() + } +} diff --git a/src/tpu.rs b/src/tpu.rs index d2be36783b1401..a41da61e76f769 100644 --- a/src/tpu.rs +++ b/src/tpu.rs @@ -3,27 +3,24 @@ use accounting_stage::AccountingStage; use crdt::{Crdt, ReplicatedData}; -use ecdsa; use entry::Entry; use ledger; use packet; use packet::SharedPackets; -use rand::{thread_rng, Rng}; use result::Result; use serde_json; +use sig_verify_stage::SigVerifyStage; use std::collections::VecDeque; use std::io::Write; use std::io::sink; use std::net::UdpSocket; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::mpsc::{channel, Receiver, Sender}; +use std::sync::mpsc::{channel, Receiver}; use std::sync::{Arc, Mutex, RwLock}; use std::thread::{spawn, JoinHandle}; use std::time::Duration; -use std::time::Instant; use streamer; use thin_client_service::ThinClientService; -use timing; pub struct Tpu { accounting_stage: AccountingStage, @@ -129,73 +126,6 @@ impl Tpu { }) } - fn verify_batch(batch: Vec) -> Vec<(SharedPackets, Vec)> { - let r = ecdsa::ed25519_verify(&batch); - batch.into_iter().zip(r).collect() - } - - fn verifier( - recvr: &Arc>, - sendr: &Arc)>>>>, - ) -> Result<()> { - let (batch, len) = - streamer::recv_batch(&recvr.lock().expect("'recvr' lock in fn verifier"))?; - - let now = Instant::now(); - let batch_len = batch.len(); - let rand_id = thread_rng().gen_range(0, 100); - info!( - "@{:?} verifier: verifying: {} id: {}", - timing::timestamp(), - batch.len(), - rand_id - ); - - let verified_batch = Self::verify_batch(batch); - sendr - .lock() - .expect("lock in fn verify_batch in tpu") - .send(verified_batch)?; - - let total_time_ms = timing::duration_as_ms(&now.elapsed()); - let total_time_s = timing::duration_as_s(&now.elapsed()); - info!( - "@{:?} verifier: done. batches: {} total verify time: {:?} id: {} verified: {} v/s {}", - timing::timestamp(), - batch_len, - total_time_ms, - rand_id, - len, - (len as f32 / total_time_s) - ); - Ok(()) - } - - fn verifier_service( - exit: Arc, - packets_receiver: Arc>, - verified_sender: Arc)>>>>, - ) -> JoinHandle<()> { - spawn(move || loop { - let e = Self::verifier(&packets_receiver.clone(), &verified_sender.clone()); - if e.is_err() && exit.load(Ordering::Relaxed) { - break; - } - }) - } - - fn verifier_services( - exit: Arc, - packets_receiver: streamer::PacketReceiver, - verified_sender: Sender)>>, - ) -> Vec> { - let sender = Arc::new(Mutex::new(verified_sender)); - let receiver = Arc::new(Mutex::new(packets_receiver)); - (0..4) - .map(|_| Self::verifier_service(exit.clone(), receiver.clone(), sender.clone())) - .collect() - } - fn thin_client_service( obj: SharedTpu, exit: Arc, @@ -249,16 +179,14 @@ impl Tpu { packet_sender, )?; - let (verified_sender, verified_receiver) = channel(); - let verify_threads: Vec<_> = - Self::verifier_services(exit.clone(), packet_receiver, verified_sender); + let sig_verify_stage = SigVerifyStage::new(exit.clone(), packet_receiver); let blob_recycler = packet::BlobRecycler::default(); let (responder_sender, responder_receiver) = channel(); let t_thin_client = Self::thin_client_service( obj.clone(), exit.clone(), - verified_receiver, + sig_verify_stage.output, responder_sender, packet_recycler.clone(), blob_recycler.clone(), @@ -299,7 +227,7 @@ impl Tpu { t_listen, t_broadcast, ]; - threads.extend(verify_threads.into_iter()); + threads.extend(sig_verify_stage.thread_hdls.into_iter()); Ok(threads) } @@ -428,16 +356,15 @@ impl Tpu { blob_recycler.clone(), responder_receiver, ); - let (verified_sender, verified_receiver) = channel(); - let verify_threads: Vec<_> = - Self::verifier_services(exit.clone(), packet_receiver, verified_sender); + + let sig_verify_stage = SigVerifyStage::new(exit.clone(), packet_receiver); let t_write = Self::drain_service(obj.clone(), exit.clone()); let t_thin_client = Self::thin_client_service( obj.clone(), exit.clone(), - verified_receiver, + sig_verify_stage.output, responder_sender, packet_recycler.clone(), blob_recycler.clone(), @@ -457,7 +384,7 @@ impl Tpu { t_thin_client, t_write, ]; - threads.extend(verify_threads.into_iter()); + threads.extend(sig_verify_stage.thread_hdls.into_iter()); Ok(threads) } } From ca80bc33c617c2a907d6944f0a9dd2d52e9801b1 Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Fri, 11 May 2018 22:36:16 -0600 Subject: [PATCH 11/25] Move the writer stage's utilities to its own module --- src/entry_writer.rs | 94 +++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + src/tpu.rs | 82 ++++++--------------------------------- 3 files changed, 106 insertions(+), 71 deletions(-) create mode 100644 src/entry_writer.rs diff --git a/src/entry_writer.rs b/src/entry_writer.rs new file mode 100644 index 00000000000000..38421a72e03ff2 --- /dev/null +++ b/src/entry_writer.rs @@ -0,0 +1,94 @@ +//! The `entry_writer` module helps implement the TPU's write stage. + +use accounting_stage::AccountingStage; +use entry::Entry; +use ledger; +use packet; +use result::Result; +use serde_json; +use std::collections::VecDeque; +use std::io::Write; +use std::io::sink; +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use streamer; +use thin_client_service::ThinClientService; + +pub struct EntryWriter<'a> { + accounting_stage: &'a AccountingStage, + thin_client_service: &'a ThinClientService, +} + +impl<'a> EntryWriter<'a> { + /// Create a new Tpu that wraps the given Accountant. + pub fn new( + accounting_stage: &'a AccountingStage, + thin_client_service: &'a ThinClientService, + ) -> Self { + EntryWriter { + accounting_stage, + thin_client_service, + } + } + + fn write_entry(&self, writer: &Mutex, entry: &Entry) { + trace!("write_entry entry"); + self.accounting_stage + .accountant + .register_entry_id(&entry.id); + writeln!( + writer.lock().expect("'writer' lock in fn fn write_entry"), + "{}", + serde_json::to_string(&entry).expect("'entry' to_strong in fn write_entry") + ).expect("writeln! in fn write_entry"); + self.thin_client_service + .notify_entry_info_subscribers(&entry); + } + + fn write_entries(&self, writer: &Mutex) -> Result> { + //TODO implement a serialize for channel that does this without allocations + let mut l = vec![]; + let entry = self.accounting_stage + .output + .lock() + .expect("'ouput' lock in fn receive_all") + .recv_timeout(Duration::new(1, 0))?; + self.write_entry(writer, &entry); + l.push(entry); + while let Ok(entry) = self.accounting_stage + .output + .lock() + .expect("'output' lock in fn write_entries") + .try_recv() + { + self.write_entry(writer, &entry); + l.push(entry); + } + Ok(l) + } + + /// Process any Entry items that have been published by the Historian. + /// continuosly broadcast blobs of entries out + pub fn write_and_send_entries( + &self, + broadcast: &streamer::BlobSender, + blob_recycler: &packet::BlobRecycler, + writer: &Mutex, + ) -> Result<()> { + let mut q = VecDeque::new(); + let list = self.write_entries(writer)?; + trace!("New blobs? {}", list.len()); + ledger::process_entry_list_into_blobs(&list, blob_recycler, &mut q); + if !q.is_empty() { + broadcast.send(q)?; + } + Ok(()) + } + + /// Process any Entry items that have been published by the Historian. + /// continuosly broadcast blobs of entries out + pub fn drain_entries(&self) -> Result<()> { + self.write_entries(&Arc::new(Mutex::new(sink())))?; + Ok(()) + } +} diff --git a/src/lib.rs b/src/lib.rs index 51e6471e7d0c47..17d730500f74fd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ pub mod accounting_stage; pub mod crdt; pub mod ecdsa; pub mod entry; +pub mod entry_writer; #[cfg(feature = "erasure")] pub mod erasure; pub mod event; diff --git a/src/tpu.rs b/src/tpu.rs index a41da61e76f769..4e25e4e2753431 100644 --- a/src/tpu.rs +++ b/src/tpu.rs @@ -3,16 +3,13 @@ use accounting_stage::AccountingStage; use crdt::{Crdt, ReplicatedData}; -use entry::Entry; +use entry_writer::EntryWriter; use ledger; use packet; use packet::SharedPackets; use result::Result; -use serde_json; use sig_verify_stage::SigVerifyStage; -use std::collections::VecDeque; use std::io::Write; -use std::io::sink; use std::net::UdpSocket; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::{channel, Receiver}; @@ -39,60 +36,6 @@ impl Tpu { } } - fn write_entry(&self, writer: &Mutex, entry: &Entry) { - trace!("write_entry entry"); - self.accounting_stage - .accountant - .register_entry_id(&entry.id); - writeln!( - writer.lock().expect("'writer' lock in fn fn write_entry"), - "{}", - serde_json::to_string(&entry).expect("'entry' to_strong in fn write_entry") - ).expect("writeln! in fn write_entry"); - self.thin_client_service - .notify_entry_info_subscribers(&entry); - } - - fn write_entries(&self, writer: &Mutex) -> Result> { - //TODO implement a serialize for channel that does this without allocations - let mut l = vec![]; - let entry = self.accounting_stage - .output - .lock() - .expect("'ouput' lock in fn receive_all") - .recv_timeout(Duration::new(1, 0))?; - self.write_entry(writer, &entry); - l.push(entry); - while let Ok(entry) = self.accounting_stage - .output - .lock() - .expect("'output' lock in fn write_entries") - .try_recv() - { - self.write_entry(writer, &entry); - l.push(entry); - } - Ok(l) - } - - /// Process any Entry items that have been published by the Historian. - /// continuosly broadcast blobs of entries out - fn write_and_send_entries( - &self, - broadcast: &streamer::BlobSender, - blob_recycler: &packet::BlobRecycler, - writer: &Mutex, - ) -> Result<()> { - let mut q = VecDeque::new(); - let list = self.write_entries(writer)?; - trace!("New blobs? {}", list.len()); - ledger::process_entry_list_into_blobs(&list, blob_recycler, &mut q); - if !q.is_empty() { - broadcast.send(q)?; - } - Ok(()) - } - pub fn write_service( obj: SharedTpu, exit: Arc, @@ -101,7 +44,8 @@ impl Tpu { writer: Mutex, ) -> JoinHandle<()> { spawn(move || loop { - let _ = obj.write_and_send_entries(&broadcast, &blob_recycler, &writer); + let entry_writer = EntryWriter::new(&obj.accounting_stage, &obj.thin_client_service); + let _ = entry_writer.write_and_send_entries(&broadcast, &blob_recycler, &writer); if exit.load(Ordering::Relaxed) { info!("broadcat_service exiting"); break; @@ -109,19 +53,15 @@ impl Tpu { }) } - /// Process any Entry items that have been published by the Historian. - /// continuosly broadcast blobs of entries out - fn drain_entries(&self) -> Result<()> { - self.write_entries(&Arc::new(Mutex::new(sink())))?; - Ok(()) - } - pub fn drain_service(obj: SharedTpu, exit: Arc) -> JoinHandle<()> { - spawn(move || loop { - let _ = obj.drain_entries(); - if exit.load(Ordering::Relaxed) { - info!("drain_service exiting"); - break; + spawn(move || { + let entry_writer = EntryWriter::new(&obj.accounting_stage, &obj.thin_client_service); + loop { + let _ = entry_writer.drain_entries(); + if exit.load(Ordering::Relaxed) { + info!("drain_service exiting"); + break; + } } }) } From cd968436992d72e99c74df65a4e1f9a7f32a7179 Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Fri, 11 May 2018 23:07:44 -0600 Subject: [PATCH 12/25] Free up name ThinClientService --- src/entry_writer.rs | 11 +++++------ src/thin_client_service.rs | 6 +++--- src/tpu.rs | 14 +++++++------- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/entry_writer.rs b/src/entry_writer.rs index 38421a72e03ff2..274e1ac3443ca2 100644 --- a/src/entry_writer.rs +++ b/src/entry_writer.rs @@ -12,22 +12,22 @@ use std::io::sink; use std::sync::{Arc, Mutex}; use std::time::Duration; use streamer; -use thin_client_service::ThinClientService; +use thin_client_service::RequestProcessor; pub struct EntryWriter<'a> { accounting_stage: &'a AccountingStage, - thin_client_service: &'a ThinClientService, + request_processor: &'a RequestProcessor, } impl<'a> EntryWriter<'a> { /// Create a new Tpu that wraps the given Accountant. pub fn new( accounting_stage: &'a AccountingStage, - thin_client_service: &'a ThinClientService, + request_processor: &'a RequestProcessor, ) -> Self { EntryWriter { accounting_stage, - thin_client_service, + request_processor, } } @@ -41,8 +41,7 @@ impl<'a> EntryWriter<'a> { "{}", serde_json::to_string(&entry).expect("'entry' to_strong in fn write_entry") ).expect("writeln! in fn write_entry"); - self.thin_client_service - .notify_entry_info_subscribers(&entry); + self.request_processor.notify_entry_info_subscribers(&entry); } fn write_entries(&self, writer: &Mutex) -> Result> { diff --git a/src/thin_client_service.rs b/src/thin_client_service.rs index f86ff014264337..e6eda81334af62 100644 --- a/src/thin_client_service.rs +++ b/src/thin_client_service.rs @@ -62,18 +62,18 @@ pub enum Response { EntryInfo(EntryInfo), } -pub struct ThinClientService { +pub struct RequestProcessor { //pub output: Mutex>, //response_sender: Mutex>, accountant: Arc, entry_info_subscribers: Mutex>, } -impl ThinClientService { +impl RequestProcessor { /// Create a new Tpu that wraps the given Accountant. pub fn new(accountant: Arc) -> Self { //let (response_sender, output) = channel(); - ThinClientService { + RequestProcessor { //output: Mutex::new(output), //response_sender: Mutex::new(response_sender), accountant, diff --git a/src/tpu.rs b/src/tpu.rs index 4e25e4e2753431..dec9116e08f1b9 100644 --- a/src/tpu.rs +++ b/src/tpu.rs @@ -17,11 +17,11 @@ use std::sync::{Arc, Mutex, RwLock}; use std::thread::{spawn, JoinHandle}; use std::time::Duration; use streamer; -use thin_client_service::ThinClientService; +use thin_client_service::RequestProcessor; pub struct Tpu { accounting_stage: AccountingStage, - thin_client_service: ThinClientService, + request_processor: RequestProcessor, } type SharedTpu = Arc; @@ -29,10 +29,10 @@ type SharedTpu = Arc; impl Tpu { /// Create a new Tpu that wraps the given Accountant. pub fn new(accounting_stage: AccountingStage) -> Self { - let thin_client_service = ThinClientService::new(accounting_stage.accountant.clone()); + let request_processor = RequestProcessor::new(accounting_stage.accountant.clone()); Tpu { accounting_stage, - thin_client_service, + request_processor, } } @@ -44,7 +44,7 @@ impl Tpu { writer: Mutex, ) -> JoinHandle<()> { spawn(move || loop { - let entry_writer = EntryWriter::new(&obj.accounting_stage, &obj.thin_client_service); + let entry_writer = EntryWriter::new(&obj.accounting_stage, &obj.request_processor); let _ = entry_writer.write_and_send_entries(&broadcast, &blob_recycler, &writer); if exit.load(Ordering::Relaxed) { info!("broadcat_service exiting"); @@ -55,7 +55,7 @@ impl Tpu { pub fn drain_service(obj: SharedTpu, exit: Arc) -> JoinHandle<()> { spawn(move || { - let entry_writer = EntryWriter::new(&obj.accounting_stage, &obj.thin_client_service); + let entry_writer = EntryWriter::new(&obj.accounting_stage, &obj.request_processor); loop { let _ = entry_writer.drain_entries(); if exit.load(Ordering::Relaxed) { @@ -75,7 +75,7 @@ impl Tpu { blob_recycler: packet::BlobRecycler, ) -> JoinHandle<()> { spawn(move || loop { - let e = obj.thin_client_service.process_request_packets( + let e = obj.request_processor.process_request_packets( &obj.accounting_stage, &verified_receiver, &responder_sender, From d2f95d531966fe36030218ec171eb4fff5d79765 Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Fri, 11 May 2018 23:37:44 -0600 Subject: [PATCH 13/25] Move thin client service thread into thin_client_service.rs --- src/thin_client_service.rs | 39 ++++++++++++++++++++++++++--- src/tpu.rs | 51 +++++++++++--------------------------- 2 files changed, 49 insertions(+), 41 deletions(-) diff --git a/src/thin_client_service.rs b/src/thin_client_service.rs index e6eda81334af62..059d44c790667a 100644 --- a/src/thin_client_service.rs +++ b/src/thin_client_service.rs @@ -14,13 +14,12 @@ use result::Result; use signature::PublicKey; use std::collections::VecDeque; use std::net::{SocketAddr, UdpSocket}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread::{spawn, JoinHandle}; use transaction::Transaction; -//use std::io::{Cursor, Write}; -//use std::sync::atomic::{AtomicBool, Ordering}; //use std::sync::mpsc::{channel, Receiver, Sender}; use std::sync::mpsc::Receiver; -use std::sync::{Arc, Mutex}; -//use std::thread::{spawn, JoinHandle}; use std::time::Duration; use std::time::Instant; use streamer; @@ -277,6 +276,38 @@ impl RequestProcessor { } } +pub struct ThinClientService { + pub thread_hdl: JoinHandle<()>, +} + +impl ThinClientService { + pub fn new( + request_processor: Arc, + accounting_stage: Arc, + exit: Arc, + verified_receiver: Receiver)>>, + responder_sender: streamer::BlobSender, + packet_recycler: packet::PacketRecycler, + blob_recycler: packet::BlobRecycler, + ) -> Self { + let thread_hdl = spawn(move || loop { + let e = request_processor.process_request_packets( + &accounting_stage, + &verified_receiver, + &responder_sender, + &packet_recycler, + &blob_recycler, + ); + if e.is_err() { + if exit.load(Ordering::Relaxed) { + break; + } + } + }); + ThinClientService { thread_hdl } + } +} + #[cfg(test)] pub fn to_request_packets(r: &packet::PacketRecycler, reqs: Vec) -> Vec { let mut out = vec![]; diff --git a/src/tpu.rs b/src/tpu.rs index dec9116e08f1b9..f512148962a8c7 100644 --- a/src/tpu.rs +++ b/src/tpu.rs @@ -6,22 +6,21 @@ use crdt::{Crdt, ReplicatedData}; use entry_writer::EntryWriter; use ledger; use packet; -use packet::SharedPackets; use result::Result; use sig_verify_stage::SigVerifyStage; use std::io::Write; use std::net::UdpSocket; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::mpsc::{channel, Receiver}; +use std::sync::mpsc::channel; use std::sync::{Arc, Mutex, RwLock}; use std::thread::{spawn, JoinHandle}; use std::time::Duration; use streamer; -use thin_client_service::RequestProcessor; +use thin_client_service::{RequestProcessor, ThinClientService}; pub struct Tpu { - accounting_stage: AccountingStage, - request_processor: RequestProcessor, + accounting_stage: Arc, + request_processor: Arc, } type SharedTpu = Arc; @@ -31,8 +30,8 @@ impl Tpu { pub fn new(accounting_stage: AccountingStage) -> Self { let request_processor = RequestProcessor::new(accounting_stage.accountant.clone()); Tpu { - accounting_stage, - request_processor, + accounting_stage: Arc::new(accounting_stage), + request_processor: Arc::new(request_processor), } } @@ -66,30 +65,6 @@ impl Tpu { }) } - fn thin_client_service( - obj: SharedTpu, - exit: Arc, - verified_receiver: Receiver)>>, - responder_sender: streamer::BlobSender, - packet_recycler: packet::PacketRecycler, - blob_recycler: packet::BlobRecycler, - ) -> JoinHandle<()> { - spawn(move || loop { - let e = obj.request_processor.process_request_packets( - &obj.accounting_stage, - &verified_receiver, - &responder_sender, - &packet_recycler, - &blob_recycler, - ); - if e.is_err() { - if exit.load(Ordering::Relaxed) { - break; - } - } - }) - } - /// Create a UDP microservice that forwards messages the given Tpu. /// This service is the network leader /// Set `exit` to shutdown its threads. @@ -123,8 +98,9 @@ impl Tpu { let blob_recycler = packet::BlobRecycler::default(); let (responder_sender, responder_receiver) = channel(); - let t_thin_client = Self::thin_client_service( - obj.clone(), + let thin_client_service = ThinClientService::new( + obj.request_processor.clone(), + obj.accounting_stage.clone(), exit.clone(), sig_verify_stage.output, responder_sender, @@ -161,7 +137,7 @@ impl Tpu { let mut threads = vec![ t_receiver, t_responder, - t_thin_client, + thin_client_service.thread_hdl, t_write, t_gossip, t_listen, @@ -301,8 +277,9 @@ impl Tpu { let t_write = Self::drain_service(obj.clone(), exit.clone()); - let t_thin_client = Self::thin_client_service( - obj.clone(), + let thin_client_service = ThinClientService::new( + obj.request_processor.clone(), + obj.accounting_stage.clone(), exit.clone(), sig_verify_stage.output, responder_sender, @@ -321,7 +298,7 @@ impl Tpu { //serve threads t_packet_receiver, t_responder, - t_thin_client, + thin_client_service.thread_hdl, t_write, ]; threads.extend(sig_verify_stage.thread_hdls.into_iter()); From 2376dfc139dea6ae70eac438c4e728b3f0879cb2 Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Fri, 11 May 2018 23:45:57 -0600 Subject: [PATCH 14/25] Let thin client own the receiver channel --- src/thin_client_service.rs | 15 +++++---------- src/tpu.rs | 23 ++++++++++------------- 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/src/thin_client_service.rs b/src/thin_client_service.rs index 059d44c790667a..ef07bafddf10c7 100644 --- a/src/thin_client_service.rs +++ b/src/thin_client_service.rs @@ -15,15 +15,14 @@ use signature::PublicKey; use std::collections::VecDeque; use std::net::{SocketAddr, UdpSocket}; use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{channel, Receiver}; use std::sync::{Arc, Mutex}; use std::thread::{spawn, JoinHandle}; -use transaction::Transaction; -//use std::sync::mpsc::{channel, Receiver, Sender}; -use std::sync::mpsc::Receiver; use std::time::Duration; use std::time::Instant; use streamer; use timing; +use transaction::Transaction; #[cfg_attr(feature = "cargo-clippy", allow(large_enum_variant))] #[derive(Serialize, Deserialize, Debug, Clone)] @@ -62,8 +61,6 @@ pub enum Response { } pub struct RequestProcessor { - //pub output: Mutex>, - //response_sender: Mutex>, accountant: Arc, entry_info_subscribers: Mutex>, } @@ -71,10 +68,7 @@ pub struct RequestProcessor { impl RequestProcessor { /// Create a new Tpu that wraps the given Accountant. pub fn new(accountant: Arc) -> Self { - //let (response_sender, output) = channel(); RequestProcessor { - //output: Mutex::new(output), - //response_sender: Mutex::new(response_sender), accountant, entry_info_subscribers: Mutex::new(vec![]), } @@ -278,6 +272,7 @@ impl RequestProcessor { pub struct ThinClientService { pub thread_hdl: JoinHandle<()>, + pub output: streamer::BlobReceiver, } impl ThinClientService { @@ -286,10 +281,10 @@ impl ThinClientService { accounting_stage: Arc, exit: Arc, verified_receiver: Receiver)>>, - responder_sender: streamer::BlobSender, packet_recycler: packet::PacketRecycler, blob_recycler: packet::BlobRecycler, ) -> Self { + let (responder_sender, output) = channel(); let thread_hdl = spawn(move || loop { let e = request_processor.process_request_packets( &accounting_stage, @@ -304,7 +299,7 @@ impl ThinClientService { } } }); - ThinClientService { thread_hdl } + ThinClientService { thread_hdl, output } } } diff --git a/src/tpu.rs b/src/tpu.rs index f512148962a8c7..111ee34284b27e 100644 --- a/src/tpu.rs +++ b/src/tpu.rs @@ -97,13 +97,11 @@ impl Tpu { let sig_verify_stage = SigVerifyStage::new(exit.clone(), packet_receiver); let blob_recycler = packet::BlobRecycler::default(); - let (responder_sender, responder_receiver) = channel(); let thin_client_service = ThinClientService::new( obj.request_processor.clone(), obj.accounting_stage.clone(), exit.clone(), sig_verify_stage.output, - responder_sender, packet_recycler.clone(), blob_recycler.clone(), ); @@ -131,7 +129,7 @@ impl Tpu { respond_socket, exit.clone(), blob_recycler.clone(), - responder_receiver, + thin_client_service.output, ); let mut threads = vec![ @@ -265,28 +263,27 @@ impl Tpu { packet_recycler.clone(), packet_sender, )?; - let (responder_sender, responder_receiver) = channel(); - let t_responder = streamer::responder( - respond_socket, - exit.clone(), - blob_recycler.clone(), - responder_receiver, - ); let sig_verify_stage = SigVerifyStage::new(exit.clone(), packet_receiver); - let t_write = Self::drain_service(obj.clone(), exit.clone()); - let thin_client_service = ThinClientService::new( obj.request_processor.clone(), obj.accounting_stage.clone(), exit.clone(), sig_verify_stage.output, - responder_sender, packet_recycler.clone(), blob_recycler.clone(), ); + let t_write = Self::drain_service(obj.clone(), exit.clone()); + + let t_responder = streamer::responder( + respond_socket, + exit.clone(), + blob_recycler.clone(), + thin_client_service.output, + ); + let mut threads = vec![ //replicate threads t_blob_receiver, From 73abea088adde9643c6f5ab8269e4a319966323e Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Fri, 11 May 2018 23:51:35 -0600 Subject: [PATCH 15/25] No need for TPU dependency --- src/tpu.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/tpu.rs b/src/tpu.rs index 111ee34284b27e..b45087ad000e12 100644 --- a/src/tpu.rs +++ b/src/tpu.rs @@ -36,14 +36,15 @@ impl Tpu { } pub fn write_service( - obj: SharedTpu, + accounting_stage: Arc, + request_processor: Arc, exit: Arc, broadcast: streamer::BlobSender, blob_recycler: packet::BlobRecycler, writer: Mutex, ) -> JoinHandle<()> { spawn(move || loop { - let entry_writer = EntryWriter::new(&obj.accounting_stage, &obj.request_processor); + let entry_writer = EntryWriter::new(&accounting_stage, &request_processor); let _ = entry_writer.write_and_send_entries(&broadcast, &blob_recycler, &writer); if exit.load(Ordering::Relaxed) { info!("broadcat_service exiting"); @@ -52,9 +53,13 @@ impl Tpu { }) } - pub fn drain_service(obj: SharedTpu, exit: Arc) -> JoinHandle<()> { + pub fn drain_service( + accounting_stage: Arc, + request_processor: Arc, + exit: Arc, + ) -> JoinHandle<()> { spawn(move || { - let entry_writer = EntryWriter::new(&obj.accounting_stage, &obj.request_processor); + let entry_writer = EntryWriter::new(&accounting_stage, &request_processor); loop { let _ = entry_writer.drain_entries(); if exit.load(Ordering::Relaxed) { @@ -108,7 +113,8 @@ impl Tpu { let (broadcast_sender, broadcast_receiver) = channel(); let t_write = Self::write_service( - obj.clone(), + obj.accounting_stage.clone(), + obj.request_processor.clone(), exit.clone(), broadcast_sender, blob_recycler.clone(), @@ -275,7 +281,11 @@ impl Tpu { blob_recycler.clone(), ); - let t_write = Self::drain_service(obj.clone(), exit.clone()); + let t_write = Self::drain_service( + obj.accounting_stage.clone(), + obj.request_processor.clone(), + exit.clone(), + ); let t_responder = streamer::responder( respond_socket, From b4ca41449282a81d1af94c09a11242f560125fca Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Sat, 12 May 2018 00:19:12 -0600 Subject: [PATCH 16/25] More object-oriented --- src/bin/testnode.rs | 3 +-- src/thin_client.rs | 4 ++-- src/tpu.rs | 10 +++++----- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/bin/testnode.rs b/src/bin/testnode.rs index 0eb38e195ecbaa..4df3e67ca0ca12 100644 --- a/src/bin/testnode.rs +++ b/src/bin/testnode.rs @@ -130,8 +130,7 @@ fn main() { serve_sock.local_addr().unwrap(), ); eprintln!("starting server..."); - let threads = Tpu::serve( - &tpu, + let threads = tpu.serve( d, serve_sock, events_sock, diff --git a/src/thin_client.rs b/src/thin_client.rs index 852338f7c53f4e..ce5135cada7fa6 100644 --- a/src/thin_client.rs +++ b/src/thin_client.rs @@ -191,8 +191,8 @@ mod tests { let exit = Arc::new(AtomicBool::new(false)); let accounting_stage = AccountingStage::new(accountant, &alice.last_id(), Some(30)); let tpu = Arc::new(Tpu::new(accounting_stage)); - let threads = - Tpu::serve(&tpu, d, serve, events_socket, gossip, exit.clone(), sink()).unwrap(); + let threads = tpu.serve(d, serve, events_socket, gossip, exit.clone(), sink()) + .unwrap(); sleep(Duration::from_millis(300)); let requests_socket = UdpSocket::bind("0.0.0.0:0").unwrap(); diff --git a/src/tpu.rs b/src/tpu.rs index b45087ad000e12..4f2ae287aa32e7 100644 --- a/src/tpu.rs +++ b/src/tpu.rs @@ -74,7 +74,7 @@ impl Tpu { /// This service is the network leader /// Set `exit` to shutdown its threads. pub fn serve( - obj: &SharedTpu, + &self, me: ReplicatedData, requests_socket: UdpSocket, _events_socket: UdpSocket, @@ -103,8 +103,8 @@ impl Tpu { let blob_recycler = packet::BlobRecycler::default(); let thin_client_service = ThinClientService::new( - obj.request_processor.clone(), - obj.accounting_stage.clone(), + self.request_processor.clone(), + self.accounting_stage.clone(), exit.clone(), sig_verify_stage.output, packet_recycler.clone(), @@ -113,8 +113,8 @@ impl Tpu { let (broadcast_sender, broadcast_receiver) = channel(); let t_write = Self::write_service( - obj.accounting_stage.clone(), - obj.request_processor.clone(), + self.accounting_stage.clone(), + self.request_processor.clone(), exit.clone(), broadcast_sender, blob_recycler.clone(), From 7ab3331f0104fb939b7148c6edc903fd94496841 Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Sat, 12 May 2018 00:31:32 -0600 Subject: [PATCH 17/25] Move validation processor to its own module --- src/lib.rs | 1 + src/thin_client.rs | 28 ++-- src/tpu.rs | 347 --------------------------------------- src/tvu.rs | 394 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 410 insertions(+), 360 deletions(-) create mode 100644 src/tvu.rs diff --git a/src/lib.rs b/src/lib.rs index 17d730500f74fd..6cde272d60f699 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,6 +25,7 @@ pub mod thin_client_service; pub mod timing; pub mod tpu; pub mod transaction; +pub mod tvu; extern crate bincode; extern crate byteorder; extern crate chrono; diff --git a/src/thin_client.rs b/src/thin_client.rs index ce5135cada7fa6..82a693a5852d67 100644 --- a/src/thin_client.rs +++ b/src/thin_client.rs @@ -168,7 +168,8 @@ mod tests { use std::thread::sleep; use std::time::Duration; use std::time::Instant; - use tpu::{self, Tpu}; + use tpu::Tpu; + use tvu::{self, Tvu}; #[test] fn test_thin_client() { @@ -223,7 +224,7 @@ mod tests { #[test] fn test_bad_sig() { - let (leader_data, leader_gossip, _, leader_serve, leader_events) = tpu::test_node(); + let (leader_data, leader_gossip, _, leader_serve, leader_events) = tvu::test_node(); let alice = Mint::new(10_000); let accountant = Accountant::new(&alice); let bob_pubkey = KeyPair::new().pubkey(); @@ -307,19 +308,20 @@ mod tests { let replicant_acc = { let accountant = Accountant::new(&alice); let accounting_stage = AccountingStage::new(accountant, &alice.last_id(), Some(30)); - Arc::new(Tpu::new(accounting_stage)) + Arc::new(Tvu::new(accounting_stage)) }; - let leader_threads = Tpu::serve( - &leader_acc, - leader.0.clone(), - leader.2, - leader.4, - leader.1, - exit.clone(), - sink(), - ).unwrap(); - let replicant_threads = Tpu::replicate( + let leader_threads = leader_acc + .serve( + leader.0.clone(), + leader.2, + leader.4, + leader.1, + exit.clone(), + sink(), + ) + .unwrap(); + let replicant_threads = Tvu::serve( &replicant_acc, replicant.0.clone(), replicant.1, diff --git a/src/tpu.rs b/src/tpu.rs index 4f2ae287aa32e7..0843139ab77f32 100644 --- a/src/tpu.rs +++ b/src/tpu.rs @@ -4,7 +4,6 @@ use accounting_stage::AccountingStage; use crdt::{Crdt, ReplicatedData}; use entry_writer::EntryWriter; -use ledger; use packet; use result::Result; use sig_verify_stage::SigVerifyStage; @@ -14,7 +13,6 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::channel; use std::sync::{Arc, Mutex, RwLock}; use std::thread::{spawn, JoinHandle}; -use std::time::Duration; use streamer; use thin_client_service::{RequestProcessor, ThinClientService}; @@ -23,8 +21,6 @@ pub struct Tpu { request_processor: Arc, } -type SharedTpu = Arc; - impl Tpu { /// Create a new Tpu that wraps the given Accountant. pub fn new(accounting_stage: AccountingStage) -> Self { @@ -150,347 +146,4 @@ impl Tpu { threads.extend(sig_verify_stage.thread_hdls.into_iter()); Ok(threads) } - - /// Process verified blobs, already in order - /// Respond with a signed hash of the state - fn replicate_state( - obj: &Tpu, - verified_receiver: &streamer::BlobReceiver, - blob_recycler: &packet::BlobRecycler, - ) -> Result<()> { - let timer = Duration::new(1, 0); - let blobs = verified_receiver.recv_timeout(timer)?; - trace!("replicating blobs {}", blobs.len()); - let entries = ledger::reconstruct_entries_from_blobs(&blobs); - obj.accounting_stage - .accountant - .process_verified_entries(entries)?; - for blob in blobs { - blob_recycler.recycle(blob); - } - Ok(()) - } - - /// This service receives messages from a leader in the network and processes the transactions - /// on the accountant state. - /// # Arguments - /// * `obj` - The accountant state. - /// * `me` - my configuration - /// * `leader` - leader configuration - /// * `exit` - The exit signal. - /// # Remarks - /// The pipeline is constructed as follows: - /// 1. receive blobs from the network, these are out of order - /// 2. verify blobs, PoH, signatures (TODO) - /// 3. reconstruct contiguous window - /// a. order the blobs - /// b. use erasure coding to reconstruct missing blobs - /// c. ask the network for missing blobs, if erasure coding is insufficient - /// d. make sure that the blobs PoH sequences connect (TODO) - /// 4. process the transaction state machine - /// 5. respond with the hash of the state back to the leader - pub fn replicate( - obj: &SharedTpu, - me: ReplicatedData, - gossip: UdpSocket, - requests_socket: UdpSocket, - replicate: UdpSocket, - leader: ReplicatedData, - exit: Arc, - ) -> Result>> { - //replicate pipeline - let crdt = Arc::new(RwLock::new(Crdt::new(me))); - crdt.write() - .expect("'crdt' write lock in pub fn replicate") - .set_leader(leader.id); - crdt.write() - .expect("'crdt' write lock before insert() in pub fn replicate") - .insert(leader); - let t_gossip = Crdt::gossip(crdt.clone(), exit.clone()); - let t_listen = Crdt::listen(crdt.clone(), gossip, exit.clone()); - - // make sure we are on the same interface - let mut local = replicate.local_addr()?; - local.set_port(0); - let write = UdpSocket::bind(local)?; - - let blob_recycler = packet::BlobRecycler::default(); - let (blob_sender, blob_receiver) = channel(); - let t_blob_receiver = streamer::blob_receiver( - exit.clone(), - blob_recycler.clone(), - replicate, - blob_sender.clone(), - )?; - let (window_sender, window_receiver) = channel(); - let (retransmit_sender, retransmit_receiver) = channel(); - - let t_retransmit = streamer::retransmitter( - write, - exit.clone(), - crdt.clone(), - blob_recycler.clone(), - retransmit_receiver, - ); - - //TODO - //the packets coming out of blob_receiver need to be sent to the GPU and verified - //then sent to the window, which does the erasure coding reconstruction - let t_window = streamer::window( - exit.clone(), - crdt.clone(), - blob_recycler.clone(), - blob_receiver, - window_sender, - retransmit_sender, - ); - - let tpu = obj.clone(); - let s_exit = exit.clone(); - let t_replicator = spawn(move || loop { - let e = Self::replicate_state(&tpu, &window_receiver, &blob_recycler); - if e.is_err() && s_exit.load(Ordering::Relaxed) { - break; - } - }); - - //serve pipeline - // make sure we are on the same interface - let mut local = requests_socket.local_addr()?; - local.set_port(0); - let respond_socket = UdpSocket::bind(local.clone())?; - - let packet_recycler = packet::PacketRecycler::default(); - let blob_recycler = packet::BlobRecycler::default(); - let (packet_sender, packet_receiver) = channel(); - let t_packet_receiver = streamer::receiver( - requests_socket, - exit.clone(), - packet_recycler.clone(), - packet_sender, - )?; - - let sig_verify_stage = SigVerifyStage::new(exit.clone(), packet_receiver); - - let thin_client_service = ThinClientService::new( - obj.request_processor.clone(), - obj.accounting_stage.clone(), - exit.clone(), - sig_verify_stage.output, - packet_recycler.clone(), - blob_recycler.clone(), - ); - - let t_write = Self::drain_service( - obj.accounting_stage.clone(), - obj.request_processor.clone(), - exit.clone(), - ); - - let t_responder = streamer::responder( - respond_socket, - exit.clone(), - blob_recycler.clone(), - thin_client_service.output, - ); - - let mut threads = vec![ - //replicate threads - t_blob_receiver, - t_retransmit, - t_window, - t_replicator, - t_gossip, - t_listen, - //serve threads - t_packet_receiver, - t_responder, - thin_client_service.thread_hdl, - t_write, - ]; - threads.extend(sig_verify_stage.thread_hdls.into_iter()); - Ok(threads) - } -} - -#[cfg(test)] -pub fn test_node() -> (ReplicatedData, UdpSocket, UdpSocket, UdpSocket, UdpSocket) { - use signature::{KeyPair, KeyPairUtil}; - - let events_socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - let gossip = UdpSocket::bind("127.0.0.1:0").unwrap(); - let replicate = UdpSocket::bind("127.0.0.1:0").unwrap(); - let requests_socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - let pubkey = KeyPair::new().pubkey(); - let d = ReplicatedData::new( - pubkey, - gossip.local_addr().unwrap(), - replicate.local_addr().unwrap(), - requests_socket.local_addr().unwrap(), - ); - (d, gossip, replicate, requests_socket, events_socket) -} - -#[cfg(test)] -mod tests { - use accountant::Accountant; - use accounting_stage::AccountingStage; - use bincode::serialize; - use chrono::prelude::*; - use crdt::Crdt; - use entry; - use event::Event; - use hash::{hash, Hash}; - use logger; - use mint::Mint; - use packet::BlobRecycler; - use signature::{KeyPair, KeyPairUtil}; - use std::collections::VecDeque; - use std::sync::atomic::{AtomicBool, Ordering}; - use std::sync::mpsc::channel; - use std::sync::{Arc, RwLock}; - use std::time::Duration; - use streamer; - use tpu::{test_node, Tpu}; - use transaction::Transaction; - - /// Test that mesasge sent from leader to target1 and repliated to target2 - #[test] - #[ignore] - fn test_replicate() { - logger::setup(); - let (leader_data, leader_gossip, _, leader_serve, _) = test_node(); - let (target1_data, target1_gossip, target1_replicate, target1_serve, _) = test_node(); - let (target2_data, target2_gossip, target2_replicate, _, _) = test_node(); - let exit = Arc::new(AtomicBool::new(false)); - - //start crdt_leader - let mut crdt_l = Crdt::new(leader_data.clone()); - crdt_l.set_leader(leader_data.id); - - let cref_l = Arc::new(RwLock::new(crdt_l)); - let t_l_gossip = Crdt::gossip(cref_l.clone(), exit.clone()); - let t_l_listen = Crdt::listen(cref_l, leader_gossip, exit.clone()); - - //start crdt2 - let mut crdt2 = Crdt::new(target2_data.clone()); - crdt2.insert(leader_data.clone()); - crdt2.set_leader(leader_data.id); - let leader_id = leader_data.id; - let cref2 = Arc::new(RwLock::new(crdt2)); - let t2_gossip = Crdt::gossip(cref2.clone(), exit.clone()); - let t2_listen = Crdt::listen(cref2, target2_gossip, exit.clone()); - - // setup some blob services to send blobs into the socket - // to simulate the source peer and get blobs out of the socket to - // simulate target peer - let recv_recycler = BlobRecycler::default(); - let resp_recycler = BlobRecycler::default(); - let (s_reader, r_reader) = channel(); - let t_receiver = streamer::blob_receiver( - exit.clone(), - recv_recycler.clone(), - target2_replicate, - s_reader, - ).unwrap(); - - // simulate leader sending messages - let (s_responder, r_responder) = channel(); - let t_responder = streamer::responder( - leader_serve, - exit.clone(), - resp_recycler.clone(), - r_responder, - ); - - let starting_balance = 10_000; - let alice = Mint::new(starting_balance); - let accountant = Accountant::new(&alice); - let accounting_stage = AccountingStage::new(accountant, &alice.last_id(), Some(30)); - let tpu = Arc::new(Tpu::new(accounting_stage)); - let replicate_addr = target1_data.replicate_addr; - let threads = Tpu::replicate( - &tpu, - target1_data, - target1_gossip, - target1_serve, - target1_replicate, - leader_data, - exit.clone(), - ).unwrap(); - - let mut alice_ref_balance = starting_balance; - let mut msgs = VecDeque::new(); - let mut cur_hash = Hash::default(); - let num_blobs = 10; - let transfer_amount = 501; - let bob_keypair = KeyPair::new(); - for i in 0..num_blobs { - let b = resp_recycler.allocate(); - let b_ = b.clone(); - let mut w = b.write().unwrap(); - w.set_index(i).unwrap(); - w.set_id(leader_id).unwrap(); - - let accountant = &tpu.accounting_stage.accountant; - - let tr0 = Event::new_timestamp(&bob_keypair, Utc::now()); - let entry0 = entry::create_entry(&cur_hash, i, vec![tr0]); - accountant.register_entry_id(&cur_hash); - cur_hash = hash(&cur_hash); - - let tr1 = Transaction::new( - &alice.keypair(), - bob_keypair.pubkey(), - transfer_amount, - cur_hash, - ); - accountant.register_entry_id(&cur_hash); - cur_hash = hash(&cur_hash); - let entry1 = - entry::create_entry(&cur_hash, i + num_blobs, vec![Event::Transaction(tr1)]); - accountant.register_entry_id(&cur_hash); - cur_hash = hash(&cur_hash); - - alice_ref_balance -= transfer_amount; - - let serialized_entry = serialize(&vec![entry0, entry1]).unwrap(); - - w.data_mut()[..serialized_entry.len()].copy_from_slice(&serialized_entry); - w.set_size(serialized_entry.len()); - w.meta.set_addr(&replicate_addr); - drop(w); - msgs.push_back(b_); - } - - // send the blobs into the socket - s_responder.send(msgs).expect("send"); - - // receive retransmitted messages - let timer = Duration::new(1, 0); - let mut msgs: Vec<_> = Vec::new(); - while let Ok(msg) = r_reader.recv_timeout(timer) { - trace!("msg: {:?}", msg); - msgs.push(msg); - } - - let accountant = &tpu.accounting_stage.accountant; - let alice_balance = accountant.get_balance(&alice.keypair().pubkey()).unwrap(); - assert_eq!(alice_balance, alice_ref_balance); - - let bob_balance = accountant.get_balance(&bob_keypair.pubkey()).unwrap(); - assert_eq!(bob_balance, starting_balance - alice_ref_balance); - - exit.store(true, Ordering::Relaxed); - for t in threads { - t.join().expect("join"); - } - t2_gossip.join().expect("join"); - t2_listen.join().expect("join"); - t_receiver.join().expect("join"); - t_responder.join().expect("join"); - t_l_gossip.join().expect("join"); - t_l_listen.join().expect("join"); - } - } diff --git a/src/tvu.rs b/src/tvu.rs new file mode 100644 index 00000000000000..e947d52cfb2a69 --- /dev/null +++ b/src/tvu.rs @@ -0,0 +1,394 @@ +//! The `tvu` module implements the Transaction Validation Unit, a +//! 5-stage transaction validation pipeline in software. + +use accounting_stage::AccountingStage; +use crdt::{Crdt, ReplicatedData}; +use entry_writer::EntryWriter; +use ledger; +use packet; +use result::Result; +use sig_verify_stage::SigVerifyStage; +use std::net::UdpSocket; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::channel; +use std::sync::{Arc, RwLock}; +use std::thread::{spawn, JoinHandle}; +use std::time::Duration; +use streamer; +use thin_client_service::{RequestProcessor, ThinClientService}; + +pub struct Tvu { + accounting_stage: Arc, + request_processor: Arc, +} + +impl Tvu { + /// Create a new Tvu that wraps the given Accountant. + pub fn new(accounting_stage: AccountingStage) -> Self { + let request_processor = RequestProcessor::new(accounting_stage.accountant.clone()); + Tvu { + accounting_stage: Arc::new(accounting_stage), + request_processor: Arc::new(request_processor), + } + } + + pub fn drain_service( + accounting_stage: Arc, + request_processor: Arc, + exit: Arc, + ) -> JoinHandle<()> { + spawn(move || { + let entry_writer = EntryWriter::new(&accounting_stage, &request_processor); + loop { + let _ = entry_writer.drain_entries(); + if exit.load(Ordering::Relaxed) { + info!("drain_service exiting"); + break; + } + } + }) + } + + /// Process verified blobs, already in order + /// Respond with a signed hash of the state + fn replicate_state( + obj: &Tvu, + verified_receiver: &streamer::BlobReceiver, + blob_recycler: &packet::BlobRecycler, + ) -> Result<()> { + let timer = Duration::new(1, 0); + let blobs = verified_receiver.recv_timeout(timer)?; + trace!("replicating blobs {}", blobs.len()); + let entries = ledger::reconstruct_entries_from_blobs(&blobs); + obj.accounting_stage + .accountant + .process_verified_entries(entries)?; + for blob in blobs { + blob_recycler.recycle(blob); + } + Ok(()) + } + + /// This service receives messages from a leader in the network and processes the transactions + /// on the accountant state. + /// # Arguments + /// * `obj` - The accountant state. + /// * `me` - my configuration + /// * `leader` - leader configuration + /// * `exit` - The exit signal. + /// # Remarks + /// The pipeline is constructed as follows: + /// 1. receive blobs from the network, these are out of order + /// 2. verify blobs, PoH, signatures (TODO) + /// 3. reconstruct contiguous window + /// a. order the blobs + /// b. use erasure coding to reconstruct missing blobs + /// c. ask the network for missing blobs, if erasure coding is insufficient + /// d. make sure that the blobs PoH sequences connect (TODO) + /// 4. process the transaction state machine + /// 5. respond with the hash of the state back to the leader + pub fn serve( + obj: &Arc, + me: ReplicatedData, + gossip: UdpSocket, + requests_socket: UdpSocket, + replicate: UdpSocket, + leader: ReplicatedData, + exit: Arc, + ) -> Result>> { + //replicate pipeline + let crdt = Arc::new(RwLock::new(Crdt::new(me))); + crdt.write() + .expect("'crdt' write lock in pub fn replicate") + .set_leader(leader.id); + crdt.write() + .expect("'crdt' write lock before insert() in pub fn replicate") + .insert(leader); + let t_gossip = Crdt::gossip(crdt.clone(), exit.clone()); + let t_listen = Crdt::listen(crdt.clone(), gossip, exit.clone()); + + // make sure we are on the same interface + let mut local = replicate.local_addr()?; + local.set_port(0); + let write = UdpSocket::bind(local)?; + + let blob_recycler = packet::BlobRecycler::default(); + let (blob_sender, blob_receiver) = channel(); + let t_blob_receiver = streamer::blob_receiver( + exit.clone(), + blob_recycler.clone(), + replicate, + blob_sender.clone(), + )?; + let (window_sender, window_receiver) = channel(); + let (retransmit_sender, retransmit_receiver) = channel(); + + let t_retransmit = streamer::retransmitter( + write, + exit.clone(), + crdt.clone(), + blob_recycler.clone(), + retransmit_receiver, + ); + + //TODO + //the packets coming out of blob_receiver need to be sent to the GPU and verified + //then sent to the window, which does the erasure coding reconstruction + let t_window = streamer::window( + exit.clone(), + crdt.clone(), + blob_recycler.clone(), + blob_receiver, + window_sender, + retransmit_sender, + ); + + let tvu = obj.clone(); + let s_exit = exit.clone(); + let t_replicator = spawn(move || loop { + let e = Self::replicate_state(&tvu, &window_receiver, &blob_recycler); + if e.is_err() && s_exit.load(Ordering::Relaxed) { + break; + } + }); + + //serve pipeline + // make sure we are on the same interface + let mut local = requests_socket.local_addr()?; + local.set_port(0); + let respond_socket = UdpSocket::bind(local.clone())?; + + let packet_recycler = packet::PacketRecycler::default(); + let blob_recycler = packet::BlobRecycler::default(); + let (packet_sender, packet_receiver) = channel(); + let t_packet_receiver = streamer::receiver( + requests_socket, + exit.clone(), + packet_recycler.clone(), + packet_sender, + )?; + + let sig_verify_stage = SigVerifyStage::new(exit.clone(), packet_receiver); + + let thin_client_service = ThinClientService::new( + obj.request_processor.clone(), + obj.accounting_stage.clone(), + exit.clone(), + sig_verify_stage.output, + packet_recycler.clone(), + blob_recycler.clone(), + ); + + let t_write = Self::drain_service( + obj.accounting_stage.clone(), + obj.request_processor.clone(), + exit.clone(), + ); + + let t_responder = streamer::responder( + respond_socket, + exit.clone(), + blob_recycler.clone(), + thin_client_service.output, + ); + + let mut threads = vec![ + //replicate threads + t_blob_receiver, + t_retransmit, + t_window, + t_replicator, + t_gossip, + t_listen, + //serve threads + t_packet_receiver, + t_responder, + thin_client_service.thread_hdl, + t_write, + ]; + threads.extend(sig_verify_stage.thread_hdls.into_iter()); + Ok(threads) + } +} + +#[cfg(test)] +pub fn test_node() -> (ReplicatedData, UdpSocket, UdpSocket, UdpSocket, UdpSocket) { + use signature::{KeyPair, KeyPairUtil}; + + let events_socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let gossip = UdpSocket::bind("127.0.0.1:0").unwrap(); + let replicate = UdpSocket::bind("127.0.0.1:0").unwrap(); + let requests_socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let pubkey = KeyPair::new().pubkey(); + let d = ReplicatedData::new( + pubkey, + gossip.local_addr().unwrap(), + replicate.local_addr().unwrap(), + requests_socket.local_addr().unwrap(), + ); + (d, gossip, replicate, requests_socket, events_socket) +} + +#[cfg(test)] +mod tests { + use accountant::Accountant; + use accounting_stage::AccountingStage; + use bincode::serialize; + use chrono::prelude::*; + use crdt::Crdt; + use entry; + use event::Event; + use hash::{hash, Hash}; + use logger; + use mint::Mint; + use packet::BlobRecycler; + use signature::{KeyPair, KeyPairUtil}; + use std::collections::VecDeque; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::mpsc::channel; + use std::sync::{Arc, RwLock}; + use std::time::Duration; + use streamer; + use transaction::Transaction; + use tvu::{test_node, Tvu}; + + /// Test that mesasge sent from leader to target1 and repliated to target2 + #[test] + #[ignore] + fn test_replicate() { + logger::setup(); + let (leader_data, leader_gossip, _, leader_serve, _) = test_node(); + let (target1_data, target1_gossip, target1_replicate, target1_serve, _) = test_node(); + let (target2_data, target2_gossip, target2_replicate, _, _) = test_node(); + let exit = Arc::new(AtomicBool::new(false)); + + //start crdt_leader + let mut crdt_l = Crdt::new(leader_data.clone()); + crdt_l.set_leader(leader_data.id); + + let cref_l = Arc::new(RwLock::new(crdt_l)); + let t_l_gossip = Crdt::gossip(cref_l.clone(), exit.clone()); + let t_l_listen = Crdt::listen(cref_l, leader_gossip, exit.clone()); + + //start crdt2 + let mut crdt2 = Crdt::new(target2_data.clone()); + crdt2.insert(leader_data.clone()); + crdt2.set_leader(leader_data.id); + let leader_id = leader_data.id; + let cref2 = Arc::new(RwLock::new(crdt2)); + let t2_gossip = Crdt::gossip(cref2.clone(), exit.clone()); + let t2_listen = Crdt::listen(cref2, target2_gossip, exit.clone()); + + // setup some blob services to send blobs into the socket + // to simulate the source peer and get blobs out of the socket to + // simulate target peer + let recv_recycler = BlobRecycler::default(); + let resp_recycler = BlobRecycler::default(); + let (s_reader, r_reader) = channel(); + let t_receiver = streamer::blob_receiver( + exit.clone(), + recv_recycler.clone(), + target2_replicate, + s_reader, + ).unwrap(); + + // simulate leader sending messages + let (s_responder, r_responder) = channel(); + let t_responder = streamer::responder( + leader_serve, + exit.clone(), + resp_recycler.clone(), + r_responder, + ); + + let starting_balance = 10_000; + let alice = Mint::new(starting_balance); + let accountant = Accountant::new(&alice); + let accounting_stage = AccountingStage::new(accountant, &alice.last_id(), Some(30)); + let tvu = Arc::new(Tvu::new(accounting_stage)); + let replicate_addr = target1_data.replicate_addr; + let threads = Tvu::serve( + &tvu, + target1_data, + target1_gossip, + target1_serve, + target1_replicate, + leader_data, + exit.clone(), + ).unwrap(); + + let mut alice_ref_balance = starting_balance; + let mut msgs = VecDeque::new(); + let mut cur_hash = Hash::default(); + let num_blobs = 10; + let transfer_amount = 501; + let bob_keypair = KeyPair::new(); + for i in 0..num_blobs { + let b = resp_recycler.allocate(); + let b_ = b.clone(); + let mut w = b.write().unwrap(); + w.set_index(i).unwrap(); + w.set_id(leader_id).unwrap(); + + let accountant = &tvu.accounting_stage.accountant; + + let tr0 = Event::new_timestamp(&bob_keypair, Utc::now()); + let entry0 = entry::create_entry(&cur_hash, i, vec![tr0]); + accountant.register_entry_id(&cur_hash); + cur_hash = hash(&cur_hash); + + let tr1 = Transaction::new( + &alice.keypair(), + bob_keypair.pubkey(), + transfer_amount, + cur_hash, + ); + accountant.register_entry_id(&cur_hash); + cur_hash = hash(&cur_hash); + let entry1 = + entry::create_entry(&cur_hash, i + num_blobs, vec![Event::Transaction(tr1)]); + accountant.register_entry_id(&cur_hash); + cur_hash = hash(&cur_hash); + + alice_ref_balance -= transfer_amount; + + let serialized_entry = serialize(&vec![entry0, entry1]).unwrap(); + + w.data_mut()[..serialized_entry.len()].copy_from_slice(&serialized_entry); + w.set_size(serialized_entry.len()); + w.meta.set_addr(&replicate_addr); + drop(w); + msgs.push_back(b_); + } + + // send the blobs into the socket + s_responder.send(msgs).expect("send"); + + // receive retransmitted messages + let timer = Duration::new(1, 0); + let mut msgs: Vec<_> = Vec::new(); + while let Ok(msg) = r_reader.recv_timeout(timer) { + trace!("msg: {:?}", msg); + msgs.push(msg); + } + + let accountant = &tvu.accounting_stage.accountant; + let alice_balance = accountant.get_balance(&alice.keypair().pubkey()).unwrap(); + assert_eq!(alice_balance, alice_ref_balance); + + let bob_balance = accountant.get_balance(&bob_keypair.pubkey()).unwrap(); + assert_eq!(bob_balance, starting_balance - alice_ref_balance); + + exit.store(true, Ordering::Relaxed); + for t in threads { + t.join().expect("join"); + } + t2_gossip.join().expect("join"); + t2_listen.join().expect("join"); + t_receiver.join().expect("join"); + t_responder.join().expect("join"); + t_l_gossip.join().expect("join"); + t_l_listen.join().expect("join"); + } + +} From 898f4971a28bf7438912e7902397c6cc2b46d250 Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Sat, 12 May 2018 10:31:28 -0600 Subject: [PATCH 18/25] Free up name 'thin_client_service' --- src/accounting_stage.rs | 2 +- src/ecdsa.rs | 2 +- src/entry_writer.rs | 2 +- src/lib.rs | 2 +- src/{thin_client_service.rs => request_stage.rs} | 11 +++++------ src/thin_client.rs | 2 +- src/tpu.rs | 8 ++++---- src/tvu.rs | 8 ++++---- 8 files changed, 18 insertions(+), 19 deletions(-) rename src/{thin_client_service.rs => request_stage.rs} (97%) diff --git a/src/accounting_stage.rs b/src/accounting_stage.rs index 105d5b667d83b1..20a95b6a33311b 100644 --- a/src/accounting_stage.rs +++ b/src/accounting_stage.rs @@ -19,7 +19,7 @@ pub struct AccountingStage { } impl AccountingStage { - /// Create a new Tpu that wraps the given Accountant. + /// Create a new stage of the TPU for event and transaction processing pub fn new(accountant: Accountant, start_hash: &Hash, ms_per_tick: Option) -> Self { let (historian_input, event_receiver) = channel(); let historian = Historian::new(event_receiver, start_hash, ms_per_tick); diff --git a/src/ecdsa.rs b/src/ecdsa.rs index ea10be9471fc36..14237e6cb7cc30 100644 --- a/src/ecdsa.rs +++ b/src/ecdsa.rs @@ -137,8 +137,8 @@ mod tests { use bincode::serialize; use ecdsa; use packet::{Packet, Packets, SharedPackets}; + use request_stage::Request; use std::sync::RwLock; - use thin_client_service::Request; use transaction::Transaction; use transaction::test_tx; diff --git a/src/entry_writer.rs b/src/entry_writer.rs index 274e1ac3443ca2..48b5d195fa4024 100644 --- a/src/entry_writer.rs +++ b/src/entry_writer.rs @@ -4,6 +4,7 @@ use accounting_stage::AccountingStage; use entry::Entry; use ledger; use packet; +use request_stage::RequestProcessor; use result::Result; use serde_json; use std::collections::VecDeque; @@ -12,7 +13,6 @@ use std::io::sink; use std::sync::{Arc, Mutex}; use std::time::Duration; use streamer; -use thin_client_service::RequestProcessor; pub struct EntryWriter<'a> { accounting_stage: &'a AccountingStage, diff --git a/src/lib.rs b/src/lib.rs index 6cde272d60f699..793c21eef81b83 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,12 +16,12 @@ pub mod mint; pub mod packet; pub mod plan; pub mod recorder; +pub mod request_stage; pub mod result; pub mod sig_verify_stage; pub mod signature; pub mod streamer; pub mod thin_client; -pub mod thin_client_service; pub mod timing; pub mod tpu; pub mod transaction; diff --git a/src/thin_client_service.rs b/src/request_stage.rs similarity index 97% rename from src/thin_client_service.rs rename to src/request_stage.rs index ef07bafddf10c7..92ea3fc53e4902 100644 --- a/src/thin_client_service.rs +++ b/src/request_stage.rs @@ -1,5 +1,4 @@ -//! The `thin_client_service` sits alongside the TPU and queries it for information -//! on behalf of thing clients. +//! The `request_stage` processes thin client Request messages. use accountant::Accountant; use accounting_stage::AccountingStage; @@ -270,12 +269,12 @@ impl RequestProcessor { } } -pub struct ThinClientService { +pub struct RequestStage { pub thread_hdl: JoinHandle<()>, pub output: streamer::BlobReceiver, } -impl ThinClientService { +impl RequestStage { pub fn new( request_processor: Arc, accounting_stage: Arc, @@ -299,7 +298,7 @@ impl ThinClientService { } } }); - ThinClientService { thread_hdl, output } + RequestStage { thread_hdl, output } } } @@ -328,7 +327,7 @@ mod tests { use bincode::serialize; use ecdsa; use packet::{PacketRecycler, NUM_PACKETS}; - use thin_client_service::{to_request_packets, Request}; + use request_stage::{to_request_packets, Request}; use transaction::{memfind, test_tx}; #[test] diff --git a/src/thin_client.rs b/src/thin_client.rs index 82a693a5852d67..ad2501b55f853c 100644 --- a/src/thin_client.rs +++ b/src/thin_client.rs @@ -6,11 +6,11 @@ use bincode::{deserialize, serialize}; use futures::future::{ok, FutureResult}; use hash::Hash; +use request_stage::{Request, Response, Subscription}; use signature::{KeyPair, PublicKey, Signature}; use std::collections::HashMap; use std::io; use std::net::{SocketAddr, UdpSocket}; -use thin_client_service::{Request, Response, Subscription}; use transaction::Transaction; pub struct ThinClient { diff --git a/src/tpu.rs b/src/tpu.rs index 0843139ab77f32..24d499bb134b7e 100644 --- a/src/tpu.rs +++ b/src/tpu.rs @@ -5,6 +5,7 @@ use accounting_stage::AccountingStage; use crdt::{Crdt, ReplicatedData}; use entry_writer::EntryWriter; use packet; +use request_stage::{RequestProcessor, RequestStage}; use result::Result; use sig_verify_stage::SigVerifyStage; use std::io::Write; @@ -14,7 +15,6 @@ use std::sync::mpsc::channel; use std::sync::{Arc, Mutex, RwLock}; use std::thread::{spawn, JoinHandle}; use streamer; -use thin_client_service::{RequestProcessor, ThinClientService}; pub struct Tpu { accounting_stage: Arc, @@ -98,7 +98,7 @@ impl Tpu { let sig_verify_stage = SigVerifyStage::new(exit.clone(), packet_receiver); let blob_recycler = packet::BlobRecycler::default(); - let thin_client_service = ThinClientService::new( + let request_stage = RequestStage::new( self.request_processor.clone(), self.accounting_stage.clone(), exit.clone(), @@ -131,13 +131,13 @@ impl Tpu { respond_socket, exit.clone(), blob_recycler.clone(), - thin_client_service.output, + request_stage.output, ); let mut threads = vec![ t_receiver, t_responder, - thin_client_service.thread_hdl, + request_stage.thread_hdl, t_write, t_gossip, t_listen, diff --git a/src/tvu.rs b/src/tvu.rs index e947d52cfb2a69..37f88e17f97fcc 100644 --- a/src/tvu.rs +++ b/src/tvu.rs @@ -6,6 +6,7 @@ use crdt::{Crdt, ReplicatedData}; use entry_writer::EntryWriter; use ledger; use packet; +use request_stage::{RequestProcessor, RequestStage}; use result::Result; use sig_verify_stage::SigVerifyStage; use std::net::UdpSocket; @@ -15,7 +16,6 @@ use std::sync::{Arc, RwLock}; use std::thread::{spawn, JoinHandle}; use std::time::Duration; use streamer; -use thin_client_service::{RequestProcessor, ThinClientService}; pub struct Tvu { accounting_stage: Arc, @@ -170,7 +170,7 @@ impl Tvu { let sig_verify_stage = SigVerifyStage::new(exit.clone(), packet_receiver); - let thin_client_service = ThinClientService::new( + let request_stage = RequestStage::new( obj.request_processor.clone(), obj.accounting_stage.clone(), exit.clone(), @@ -189,7 +189,7 @@ impl Tvu { respond_socket, exit.clone(), blob_recycler.clone(), - thin_client_service.output, + request_stage.output, ); let mut threads = vec![ @@ -203,7 +203,7 @@ impl Tvu { //serve threads t_packet_receiver, t_responder, - thin_client_service.thread_hdl, + request_stage.thread_hdl, t_write, ]; threads.extend(sig_verify_stage.thread_hdls.into_iter()); From 421d9aa501d850542a6f59bf05fe8a5d55b9d6fd Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Sat, 12 May 2018 10:53:25 -0600 Subject: [PATCH 19/25] Free up the name 'tpu' --- src/bin/testnode.rs | 6 +++--- src/lib.rs | 2 +- src/{tpu.rs => rpu.rs} | 12 ++++++------ src/thin_client.rs | 15 +++++++-------- 4 files changed, 17 insertions(+), 18 deletions(-) rename src/{tpu.rs => rpu.rs} (96%) diff --git a/src/bin/testnode.rs b/src/bin/testnode.rs index 4df3e67ca0ca12..68eaca991bc817 100644 --- a/src/bin/testnode.rs +++ b/src/bin/testnode.rs @@ -11,8 +11,8 @@ use solana::accounting_stage::AccountingStage; use solana::crdt::ReplicatedData; use solana::entry::Entry; use solana::event::Event; +use solana::rpu::Rpu; use solana::signature::{KeyPair, KeyPairUtil}; -use solana::tpu::Tpu; use std::env; use std::io::{stdin, stdout, Read}; use std::net::UdpSocket; @@ -117,7 +117,7 @@ fn main() { let accounting_stage = AccountingStage::new(accountant, &last_id, Some(1000)); let exit = Arc::new(AtomicBool::new(false)); - let tpu = Arc::new(Tpu::new(accounting_stage)); + let rpu = Arc::new(Rpu::new(accounting_stage)); let serve_sock = UdpSocket::bind(&serve_addr).unwrap(); let gossip_sock = UdpSocket::bind(&gossip_addr).unwrap(); let replicate_sock = UdpSocket::bind(&replicate_addr).unwrap(); @@ -130,7 +130,7 @@ fn main() { serve_sock.local_addr().unwrap(), ); eprintln!("starting server..."); - let threads = tpu.serve( + let threads = rpu.serve( d, serve_sock, events_sock, diff --git a/src/lib.rs b/src/lib.rs index 793c21eef81b83..30f87959fa9ede 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,12 +18,12 @@ pub mod plan; pub mod recorder; pub mod request_stage; pub mod result; +pub mod rpu; pub mod sig_verify_stage; pub mod signature; pub mod streamer; pub mod thin_client; pub mod timing; -pub mod tpu; pub mod transaction; pub mod tvu; extern crate bincode; diff --git a/src/tpu.rs b/src/rpu.rs similarity index 96% rename from src/tpu.rs rename to src/rpu.rs index 24d499bb134b7e..bccc92f5f5f916 100644 --- a/src/tpu.rs +++ b/src/rpu.rs @@ -1,4 +1,4 @@ -//! The `tpu` module implements the Transaction Processing Unit, a +//! The `rpu` module implements the Request Processing Unit, a //! 5-stage transaction processing pipeline in software. use accounting_stage::AccountingStage; @@ -16,16 +16,16 @@ use std::sync::{Arc, Mutex, RwLock}; use std::thread::{spawn, JoinHandle}; use streamer; -pub struct Tpu { +pub struct Rpu { accounting_stage: Arc, request_processor: Arc, } -impl Tpu { - /// Create a new Tpu that wraps the given Accountant. +impl Rpu { + /// Create a new Rpu that wraps the given Accountant. pub fn new(accounting_stage: AccountingStage) -> Self { let request_processor = RequestProcessor::new(accounting_stage.accountant.clone()); - Tpu { + Rpu { accounting_stage: Arc::new(accounting_stage), request_processor: Arc::new(request_processor), } @@ -66,7 +66,7 @@ impl Tpu { }) } - /// Create a UDP microservice that forwards messages the given Tpu. + /// Create a UDP microservice that forwards messages the given Rpu. /// This service is the network leader /// Set `exit` to shutdown its threads. pub fn serve( diff --git a/src/thin_client.rs b/src/thin_client.rs index ad2501b55f853c..ee51860c1a5213 100644 --- a/src/thin_client.rs +++ b/src/thin_client.rs @@ -23,7 +23,7 @@ pub struct ThinClient { } impl ThinClient { - /// Create a new ThinClient that will interface with Tpu + /// Create a new ThinClient that will interface with Rpu /// over `requests_socket` and `events_socket`. To receive responses, the caller must bind `socket` /// to a public address before invoking ThinClient methods. pub fn new(addr: SocketAddr, requests_socket: UdpSocket, events_socket: UdpSocket) -> Self { @@ -161,6 +161,7 @@ mod tests { use logger; use mint::Mint; use plan::Plan; + use rpu::Rpu; use signature::{KeyPair, KeyPairUtil}; use std::io::sink; use std::sync::atomic::{AtomicBool, Ordering}; @@ -168,7 +169,6 @@ mod tests { use std::thread::sleep; use std::time::Duration; use std::time::Instant; - use tpu::Tpu; use tvu::{self, Tvu}; #[test] @@ -191,8 +191,8 @@ mod tests { let bob_pubkey = KeyPair::new().pubkey(); let exit = Arc::new(AtomicBool::new(false)); let accounting_stage = AccountingStage::new(accountant, &alice.last_id(), Some(30)); - let tpu = Arc::new(Tpu::new(accounting_stage)); - let threads = tpu.serve(d, serve, events_socket, gossip, exit.clone(), sink()) + let rpu = Arc::new(Rpu::new(accounting_stage)); + let threads = rpu.serve(d, serve, events_socket, gossip, exit.clone(), sink()) .unwrap(); sleep(Duration::from_millis(300)); @@ -230,10 +230,9 @@ mod tests { let bob_pubkey = KeyPair::new().pubkey(); let exit = Arc::new(AtomicBool::new(false)); let accounting_stage = AccountingStage::new(accountant, &alice.last_id(), Some(30)); - let tpu = Arc::new(Tpu::new(accounting_stage)); + let rpu = Arc::new(Rpu::new(accounting_stage)); let serve_addr = leader_serve.local_addr().unwrap(); - let threads = Tpu::serve( - &tpu, + let threads = rpu.serve( leader_data, leader_serve, leader_events, @@ -302,7 +301,7 @@ mod tests { let leader_acc = { let accountant = Accountant::new(&alice); let accounting_stage = AccountingStage::new(accountant, &alice.last_id(), Some(30)); - Arc::new(Tpu::new(accounting_stage)) + Arc::new(Rpu::new(accounting_stage)) }; let replicant_acc = { From 4180571660b897c9b2480d1eaacf09fcd96e76b2 Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Sat, 12 May 2018 11:11:30 -0600 Subject: [PATCH 20/25] Don't pass events_socket to RPU --- src/bin/testnode.rs | 12 +++--------- src/rpu.rs | 1 - src/thin_client.rs | 17 ++++------------- 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/src/bin/testnode.rs b/src/bin/testnode.rs index 68eaca991bc817..41a871ea001bcf 100644 --- a/src/bin/testnode.rs +++ b/src/bin/testnode.rs @@ -121,7 +121,7 @@ fn main() { let serve_sock = UdpSocket::bind(&serve_addr).unwrap(); let gossip_sock = UdpSocket::bind(&gossip_addr).unwrap(); let replicate_sock = UdpSocket::bind(&replicate_addr).unwrap(); - let events_sock = UdpSocket::bind(&events_addr).unwrap(); + let _events_sock = UdpSocket::bind(&events_addr).unwrap(); let pubkey = KeyPair::new().pubkey(); let d = ReplicatedData::new( pubkey, @@ -130,14 +130,8 @@ fn main() { serve_sock.local_addr().unwrap(), ); eprintln!("starting server..."); - let threads = rpu.serve( - d, - serve_sock, - events_sock, - gossip_sock, - exit.clone(), - stdout(), - ).unwrap(); + let threads = rpu.serve(d, serve_sock, gossip_sock, exit.clone(), stdout()) + .unwrap(); eprintln!("Ready. Listening on {}", serve_addr); for t in threads { t.join().expect("join"); diff --git a/src/rpu.rs b/src/rpu.rs index bccc92f5f5f916..adc5f2f5eb3d40 100644 --- a/src/rpu.rs +++ b/src/rpu.rs @@ -73,7 +73,6 @@ impl Rpu { &self, me: ReplicatedData, requests_socket: UdpSocket, - _events_socket: UdpSocket, gossip: UdpSocket, exit: Arc, writer: W, diff --git a/src/thin_client.rs b/src/thin_client.rs index ee51860c1a5213..e4d56be7e142f7 100644 --- a/src/thin_client.rs +++ b/src/thin_client.rs @@ -176,7 +176,7 @@ mod tests { logger::setup(); let gossip = UdpSocket::bind("0.0.0.0:0").unwrap(); let serve = UdpSocket::bind("0.0.0.0:0").unwrap(); - let events_socket = UdpSocket::bind("0.0.0.0:0").unwrap(); + let _events_socket = UdpSocket::bind("0.0.0.0:0").unwrap(); let addr = serve.local_addr().unwrap(); let pubkey = KeyPair::new().pubkey(); let d = ReplicatedData::new( @@ -192,8 +192,7 @@ mod tests { let exit = Arc::new(AtomicBool::new(false)); let accounting_stage = AccountingStage::new(accountant, &alice.last_id(), Some(30)); let rpu = Arc::new(Rpu::new(accounting_stage)); - let threads = rpu.serve(d, serve, events_socket, gossip, exit.clone(), sink()) - .unwrap(); + let threads = rpu.serve(d, serve, gossip, exit.clone(), sink()).unwrap(); sleep(Duration::from_millis(300)); let requests_socket = UdpSocket::bind("0.0.0.0:0").unwrap(); @@ -224,7 +223,7 @@ mod tests { #[test] fn test_bad_sig() { - let (leader_data, leader_gossip, _, leader_serve, leader_events) = tvu::test_node(); + let (leader_data, leader_gossip, _, leader_serve, _leader_events) = tvu::test_node(); let alice = Mint::new(10_000); let accountant = Accountant::new(&alice); let bob_pubkey = KeyPair::new().pubkey(); @@ -235,7 +234,6 @@ mod tests { let threads = rpu.serve( leader_data, leader_serve, - leader_events, leader_gossip, exit.clone(), sink(), @@ -311,14 +309,7 @@ mod tests { }; let leader_threads = leader_acc - .serve( - leader.0.clone(), - leader.2, - leader.4, - leader.1, - exit.clone(), - sink(), - ) + .serve(leader.0.clone(), leader.2, leader.1, exit.clone(), sink()) .unwrap(); let replicant_threads = Tvu::serve( &replicant_acc, From 3d82807965454f003190a10249ba8121736476d5 Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Sat, 12 May 2018 11:24:40 -0600 Subject: [PATCH 21/25] Delete dead code --- src/rpu.rs | 19 +------------------ src/tvu.rs | 2 +- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/src/rpu.rs b/src/rpu.rs index adc5f2f5eb3d40..36dd636964becf 100644 --- a/src/rpu.rs +++ b/src/rpu.rs @@ -31,7 +31,7 @@ impl Rpu { } } - pub fn write_service( + fn write_service( accounting_stage: Arc, request_processor: Arc, exit: Arc, @@ -49,23 +49,6 @@ impl Rpu { }) } - pub fn drain_service( - accounting_stage: Arc, - request_processor: Arc, - exit: Arc, - ) -> JoinHandle<()> { - spawn(move || { - let entry_writer = EntryWriter::new(&accounting_stage, &request_processor); - loop { - let _ = entry_writer.drain_entries(); - if exit.load(Ordering::Relaxed) { - info!("drain_service exiting"); - break; - } - } - }) - } - /// Create a UDP microservice that forwards messages the given Rpu. /// This service is the network leader /// Set `exit` to shutdown its threads. diff --git a/src/tvu.rs b/src/tvu.rs index 37f88e17f97fcc..b4feb477bbe02a 100644 --- a/src/tvu.rs +++ b/src/tvu.rs @@ -32,7 +32,7 @@ impl Tvu { } } - pub fn drain_service( + fn drain_service( accounting_stage: Arc, request_processor: Arc, exit: Arc, From 1511dc43d7eb35174235ce8ba4cef1e827c7c5d7 Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Sat, 12 May 2018 11:39:24 -0600 Subject: [PATCH 22/25] Move RequestProcessor out of Rpu/Tvu state --- src/request_stage.rs | 13 ++++++++++--- src/rpu.rs | 8 +++----- src/tvu.rs | 8 +++----- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/request_stage.rs b/src/request_stage.rs index 92ea3fc53e4902..30e6dae528da81 100644 --- a/src/request_stage.rs +++ b/src/request_stage.rs @@ -272,20 +272,23 @@ impl RequestProcessor { pub struct RequestStage { pub thread_hdl: JoinHandle<()>, pub output: streamer::BlobReceiver, + pub request_processor: Arc, } impl RequestStage { pub fn new( - request_processor: Arc, + request_processor: RequestProcessor, accounting_stage: Arc, exit: Arc, verified_receiver: Receiver)>>, packet_recycler: packet::PacketRecycler, blob_recycler: packet::BlobRecycler, ) -> Self { + let request_processor = Arc::new(request_processor); + let request_processor_ = request_processor.clone(); let (responder_sender, output) = channel(); let thread_hdl = spawn(move || loop { - let e = request_processor.process_request_packets( + let e = request_processor_.process_request_packets( &accounting_stage, &verified_receiver, &responder_sender, @@ -298,7 +301,11 @@ impl RequestStage { } } }); - RequestStage { thread_hdl, output } + RequestStage { + thread_hdl, + output, + request_processor, + } } } diff --git a/src/rpu.rs b/src/rpu.rs index 36dd636964becf..5e5bcb03790248 100644 --- a/src/rpu.rs +++ b/src/rpu.rs @@ -18,16 +18,13 @@ use streamer; pub struct Rpu { accounting_stage: Arc, - request_processor: Arc, } impl Rpu { /// Create a new Rpu that wraps the given Accountant. pub fn new(accounting_stage: AccountingStage) -> Self { - let request_processor = RequestProcessor::new(accounting_stage.accountant.clone()); Rpu { accounting_stage: Arc::new(accounting_stage), - request_processor: Arc::new(request_processor), } } @@ -80,8 +77,9 @@ impl Rpu { let sig_verify_stage = SigVerifyStage::new(exit.clone(), packet_receiver); let blob_recycler = packet::BlobRecycler::default(); + let request_processor = RequestProcessor::new(self.accounting_stage.accountant.clone()); let request_stage = RequestStage::new( - self.request_processor.clone(), + request_processor, self.accounting_stage.clone(), exit.clone(), sig_verify_stage.output, @@ -92,7 +90,7 @@ impl Rpu { let (broadcast_sender, broadcast_receiver) = channel(); let t_write = Self::write_service( self.accounting_stage.clone(), - self.request_processor.clone(), + request_stage.request_processor.clone(), exit.clone(), broadcast_sender, blob_recycler.clone(), diff --git a/src/tvu.rs b/src/tvu.rs index b4feb477bbe02a..b085494cbceb93 100644 --- a/src/tvu.rs +++ b/src/tvu.rs @@ -19,16 +19,13 @@ use streamer; pub struct Tvu { accounting_stage: Arc, - request_processor: Arc, } impl Tvu { /// Create a new Tvu that wraps the given Accountant. pub fn new(accounting_stage: AccountingStage) -> Self { - let request_processor = RequestProcessor::new(accounting_stage.accountant.clone()); Tvu { accounting_stage: Arc::new(accounting_stage), - request_processor: Arc::new(request_processor), } } @@ -170,8 +167,9 @@ impl Tvu { let sig_verify_stage = SigVerifyStage::new(exit.clone(), packet_receiver); + let request_processor = RequestProcessor::new(obj.accounting_stage.accountant.clone()); let request_stage = RequestStage::new( - obj.request_processor.clone(), + request_processor, obj.accounting_stage.clone(), exit.clone(), sig_verify_stage.output, @@ -181,7 +179,7 @@ impl Tvu { let t_write = Self::drain_service( obj.accounting_stage.clone(), - obj.request_processor.clone(), + request_stage.request_processor.clone(), exit.clone(), ); From a3d2831f8c07c0dff99cd244bc60e151ac416cb0 Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Sat, 12 May 2018 14:05:57 -0600 Subject: [PATCH 23/25] Free up the name 'accounting_stage' --- src/bin/testnode.rs | 6 ++-- src/entry_writer.rs | 16 +++++----- ...accounting_stage.rs => event_processor.rs} | 30 +++++++++---------- src/lib.rs | 2 +- src/request_stage.rs | 10 +++---- src/rpu.rs | 18 +++++------ src/thin_client.rs | 18 +++++------ src/tvu.rs | 30 +++++++++---------- 8 files changed, 64 insertions(+), 66 deletions(-) rename src/{accounting_stage.rs => event_processor.rs} (87%) diff --git a/src/bin/testnode.rs b/src/bin/testnode.rs index 41a871ea001bcf..74056b13b86ddb 100644 --- a/src/bin/testnode.rs +++ b/src/bin/testnode.rs @@ -7,10 +7,10 @@ extern crate solana; use getopts::Options; use isatty::stdin_isatty; use solana::accountant::Accountant; -use solana::accounting_stage::AccountingStage; use solana::crdt::ReplicatedData; use solana::entry::Entry; use solana::event::Event; +use solana::event_processor::EventProcessor; use solana::rpu::Rpu; use solana::signature::{KeyPair, KeyPairUtil}; use std::env; @@ -115,9 +115,9 @@ fn main() { eprintln!("creating networking stack..."); - let accounting_stage = AccountingStage::new(accountant, &last_id, Some(1000)); + let event_processor = EventProcessor::new(accountant, &last_id, Some(1000)); let exit = Arc::new(AtomicBool::new(false)); - let rpu = Arc::new(Rpu::new(accounting_stage)); + let rpu = Rpu::new(event_processor); let serve_sock = UdpSocket::bind(&serve_addr).unwrap(); let gossip_sock = UdpSocket::bind(&gossip_addr).unwrap(); let replicate_sock = UdpSocket::bind(&replicate_addr).unwrap(); diff --git a/src/entry_writer.rs b/src/entry_writer.rs index 48b5d195fa4024..42fe5d53fcda5d 100644 --- a/src/entry_writer.rs +++ b/src/entry_writer.rs @@ -1,7 +1,7 @@ //! The `entry_writer` module helps implement the TPU's write stage. -use accounting_stage::AccountingStage; use entry::Entry; +use event_processor::EventProcessor; use ledger; use packet; use request_stage::RequestProcessor; @@ -15,27 +15,25 @@ use std::time::Duration; use streamer; pub struct EntryWriter<'a> { - accounting_stage: &'a AccountingStage, + event_processor: &'a EventProcessor, request_processor: &'a RequestProcessor, } impl<'a> EntryWriter<'a> { /// Create a new Tpu that wraps the given Accountant. pub fn new( - accounting_stage: &'a AccountingStage, + event_processor: &'a EventProcessor, request_processor: &'a RequestProcessor, ) -> Self { EntryWriter { - accounting_stage, + event_processor, request_processor, } } fn write_entry(&self, writer: &Mutex, entry: &Entry) { trace!("write_entry entry"); - self.accounting_stage - .accountant - .register_entry_id(&entry.id); + self.event_processor.accountant.register_entry_id(&entry.id); writeln!( writer.lock().expect("'writer' lock in fn fn write_entry"), "{}", @@ -47,14 +45,14 @@ impl<'a> EntryWriter<'a> { fn write_entries(&self, writer: &Mutex) -> Result> { //TODO implement a serialize for channel that does this without allocations let mut l = vec![]; - let entry = self.accounting_stage + let entry = self.event_processor .output .lock() .expect("'ouput' lock in fn receive_all") .recv_timeout(Duration::new(1, 0))?; self.write_entry(writer, &entry); l.push(entry); - while let Ok(entry) = self.accounting_stage + while let Ok(entry) = self.event_processor .output .lock() .expect("'output' lock in fn write_entries") diff --git a/src/accounting_stage.rs b/src/event_processor.rs similarity index 87% rename from src/accounting_stage.rs rename to src/event_processor.rs index 20a95b6a33311b..df547af3d1450e 100644 --- a/src/accounting_stage.rs +++ b/src/event_processor.rs @@ -1,4 +1,4 @@ -//! The `accounting_stage` module implements the accounting stage of the TPU. +//! The `event_processor` module implements the accounting stage of the TPU. use accountant::Accountant; use entry::Entry; @@ -10,7 +10,7 @@ use result::Result; use std::sync::mpsc::{channel, Receiver, Sender}; use std::sync::{Arc, Mutex}; -pub struct AccountingStage { +pub struct EventProcessor { pub output: Mutex>, entry_sender: Mutex>, pub accountant: Arc, @@ -18,13 +18,13 @@ pub struct AccountingStage { historian: Mutex, } -impl AccountingStage { +impl EventProcessor { /// Create a new stage of the TPU for event and transaction processing pub fn new(accountant: Accountant, start_hash: &Hash, ms_per_tick: Option) -> Self { let (historian_input, event_receiver) = channel(); let historian = Historian::new(event_receiver, start_hash, ms_per_tick); let (entry_sender, output) = channel(); - AccountingStage { + EventProcessor { output: Mutex::new(output), entry_sender: Mutex::new(entry_sender), accountant: Arc::new(accountant), @@ -52,9 +52,9 @@ impl AccountingStage { #[cfg(test)] mod tests { use accountant::Accountant; - use accounting_stage::AccountingStage; use entry::Entry; use event::Event; + use event_processor::EventProcessor; use mint::Mint; use signature::{KeyPair, KeyPairUtil}; use transaction::Transaction; @@ -66,22 +66,22 @@ mod tests { // Entry OR if the verifier tries to parallelize across multiple Entries. let mint = Mint::new(2); let accountant = Accountant::new(&mint); - let accounting_stage = AccountingStage::new(accountant, &mint.last_id(), None); + let event_processor = EventProcessor::new(accountant, &mint.last_id(), None); // Process a batch that includes a transaction that receives two tokens. let alice = KeyPair::new(); let tr = Transaction::new(&mint.keypair(), alice.pubkey(), 2, mint.last_id()); let events = vec![Event::Transaction(tr)]; - assert!(accounting_stage.process_events(events).is_ok()); + assert!(event_processor.process_events(events).is_ok()); // Process a second batch that spends one of those tokens. let tr = Transaction::new(&alice, mint.pubkey(), 1, mint.last_id()); let events = vec![Event::Transaction(tr)]; - assert!(accounting_stage.process_events(events).is_ok()); + assert!(event_processor.process_events(events).is_ok()); // Collect the ledger and feed it to a new accountant. - drop(accounting_stage.entry_sender); - let entries: Vec = accounting_stage.output.lock().unwrap().iter().collect(); + drop(event_processor.entry_sender); + let entries: Vec = event_processor.output.lock().unwrap().iter().collect(); // Assert the user holds one token, not two. If the server only output one // entry, then the second transaction will be rejected, because it drives @@ -104,8 +104,8 @@ mod bench { extern crate test; use self::test::Bencher; use accountant::{Accountant, MAX_ENTRY_IDS}; - use accounting_stage::*; use bincode::serialize; + use event_processor::*; use hash::hash; use mint::Mint; use rayon::prelude::*; @@ -154,17 +154,17 @@ mod bench { .map(|tr| Event::Transaction(tr)) .collect(); - let accounting_stage = AccountingStage::new(accountant, &mint.last_id(), None); + let event_processor = EventProcessor::new(accountant, &mint.last_id(), None); let now = Instant::now(); - assert!(accounting_stage.process_events(events).is_ok()); + assert!(event_processor.process_events(events).is_ok()); let duration = now.elapsed(); let sec = duration.as_secs() as f64 + duration.subsec_nanos() as f64 / 1_000_000_000.0; let tps = txs as f64 / sec; // Ensure that all transactions were successfully logged. - drop(accounting_stage.historian_input); - let entries: Vec = accounting_stage.output.lock().unwrap().iter().collect(); + drop(event_processor.historian_input); + let entries: Vec = event_processor.output.lock().unwrap().iter().collect(); assert_eq!(entries.len(), 1); assert_eq!(entries[0].events.len(), txs as usize); diff --git a/src/lib.rs b/src/lib.rs index 30f87959fa9ede..522cb33fc85d57 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,5 @@ #![cfg_attr(feature = "unstable", feature(test))] pub mod accountant; -pub mod accounting_stage; pub mod crdt; pub mod ecdsa; pub mod entry; @@ -8,6 +7,7 @@ pub mod entry_writer; #[cfg(feature = "erasure")] pub mod erasure; pub mod event; +pub mod event_processor; pub mod hash; pub mod historian; pub mod ledger; diff --git a/src/request_stage.rs b/src/request_stage.rs index 30e6dae528da81..3695fd01a25d80 100644 --- a/src/request_stage.rs +++ b/src/request_stage.rs @@ -1,10 +1,10 @@ //! The `request_stage` processes thin client Request messages. use accountant::Accountant; -use accounting_stage::AccountingStage; use bincode::{deserialize, serialize}; use entry::Entry; use event::Event; +use event_processor::EventProcessor; use hash::Hash; use packet; use packet::SharedPackets; @@ -205,7 +205,7 @@ impl RequestProcessor { pub fn process_request_packets( &self, - accounting_stage: &AccountingStage, + event_processor: &EventProcessor, verified_receiver: &Receiver)>>, responder_sender: &streamer::BlobSender, packet_recycler: &packet::PacketRecycler, @@ -240,7 +240,7 @@ impl RequestProcessor { debug!("events: {} reqs: {}", events.len(), reqs.len()); debug!("process_events"); - accounting_stage.process_events(events)?; + event_processor.process_events(events)?; debug!("done process_events"); debug!("process_requests"); @@ -278,7 +278,7 @@ pub struct RequestStage { impl RequestStage { pub fn new( request_processor: RequestProcessor, - accounting_stage: Arc, + event_processor: Arc, exit: Arc, verified_receiver: Receiver)>>, packet_recycler: packet::PacketRecycler, @@ -289,7 +289,7 @@ impl RequestStage { let (responder_sender, output) = channel(); let thread_hdl = spawn(move || loop { let e = request_processor_.process_request_packets( - &accounting_stage, + &event_processor, &verified_receiver, &responder_sender, &packet_recycler, diff --git a/src/rpu.rs b/src/rpu.rs index 5e5bcb03790248..5fee29e0b5fe24 100644 --- a/src/rpu.rs +++ b/src/rpu.rs @@ -1,9 +1,9 @@ //! The `rpu` module implements the Request Processing Unit, a //! 5-stage transaction processing pipeline in software. -use accounting_stage::AccountingStage; use crdt::{Crdt, ReplicatedData}; use entry_writer::EntryWriter; +use event_processor::EventProcessor; use packet; use request_stage::{RequestProcessor, RequestStage}; use result::Result; @@ -17,19 +17,19 @@ use std::thread::{spawn, JoinHandle}; use streamer; pub struct Rpu { - accounting_stage: Arc, + event_processor: Arc, } impl Rpu { /// Create a new Rpu that wraps the given Accountant. - pub fn new(accounting_stage: AccountingStage) -> Self { + pub fn new(event_processor: EventProcessor) -> Self { Rpu { - accounting_stage: Arc::new(accounting_stage), + event_processor: Arc::new(event_processor), } } fn write_service( - accounting_stage: Arc, + event_processor: Arc, request_processor: Arc, exit: Arc, broadcast: streamer::BlobSender, @@ -37,7 +37,7 @@ impl Rpu { writer: Mutex, ) -> JoinHandle<()> { spawn(move || loop { - let entry_writer = EntryWriter::new(&accounting_stage, &request_processor); + let entry_writer = EntryWriter::new(&event_processor, &request_processor); let _ = entry_writer.write_and_send_entries(&broadcast, &blob_recycler, &writer); if exit.load(Ordering::Relaxed) { info!("broadcat_service exiting"); @@ -77,10 +77,10 @@ impl Rpu { let sig_verify_stage = SigVerifyStage::new(exit.clone(), packet_receiver); let blob_recycler = packet::BlobRecycler::default(); - let request_processor = RequestProcessor::new(self.accounting_stage.accountant.clone()); + let request_processor = RequestProcessor::new(self.event_processor.accountant.clone()); let request_stage = RequestStage::new( request_processor, - self.accounting_stage.clone(), + self.event_processor.clone(), exit.clone(), sig_verify_stage.output, packet_recycler.clone(), @@ -89,7 +89,7 @@ impl Rpu { let (broadcast_sender, broadcast_receiver) = channel(); let t_write = Self::write_service( - self.accounting_stage.clone(), + self.event_processor.clone(), request_stage.request_processor.clone(), exit.clone(), broadcast_sender, diff --git a/src/thin_client.rs b/src/thin_client.rs index e4d56be7e142f7..318377ae35630c 100644 --- a/src/thin_client.rs +++ b/src/thin_client.rs @@ -155,8 +155,8 @@ impl ThinClient { mod tests { use super::*; use accountant::Accountant; - use accounting_stage::AccountingStage; use crdt::{Crdt, ReplicatedData}; + use event_processor::EventProcessor; use futures::Future; use logger; use mint::Mint; @@ -190,8 +190,8 @@ mod tests { let accountant = Accountant::new(&alice); let bob_pubkey = KeyPair::new().pubkey(); let exit = Arc::new(AtomicBool::new(false)); - let accounting_stage = AccountingStage::new(accountant, &alice.last_id(), Some(30)); - let rpu = Arc::new(Rpu::new(accounting_stage)); + let event_processor = EventProcessor::new(accountant, &alice.last_id(), Some(30)); + let rpu = Arc::new(Rpu::new(event_processor)); let threads = rpu.serve(d, serve, gossip, exit.clone(), sink()).unwrap(); sleep(Duration::from_millis(300)); @@ -228,8 +228,8 @@ mod tests { let accountant = Accountant::new(&alice); let bob_pubkey = KeyPair::new().pubkey(); let exit = Arc::new(AtomicBool::new(false)); - let accounting_stage = AccountingStage::new(accountant, &alice.last_id(), Some(30)); - let rpu = Arc::new(Rpu::new(accounting_stage)); + let event_processor = EventProcessor::new(accountant, &alice.last_id(), Some(30)); + let rpu = Arc::new(Rpu::new(event_processor)); let serve_addr = leader_serve.local_addr().unwrap(); let threads = rpu.serve( leader_data, @@ -298,14 +298,14 @@ mod tests { let leader_acc = { let accountant = Accountant::new(&alice); - let accounting_stage = AccountingStage::new(accountant, &alice.last_id(), Some(30)); - Arc::new(Rpu::new(accounting_stage)) + let event_processor = EventProcessor::new(accountant, &alice.last_id(), Some(30)); + Arc::new(Rpu::new(event_processor)) }; let replicant_acc = { let accountant = Accountant::new(&alice); - let accounting_stage = AccountingStage::new(accountant, &alice.last_id(), Some(30)); - Arc::new(Tvu::new(accounting_stage)) + let event_processor = EventProcessor::new(accountant, &alice.last_id(), Some(30)); + Arc::new(Tvu::new(event_processor)) }; let leader_threads = leader_acc diff --git a/src/tvu.rs b/src/tvu.rs index b085494cbceb93..0d59ee3716bff7 100644 --- a/src/tvu.rs +++ b/src/tvu.rs @@ -1,9 +1,9 @@ //! The `tvu` module implements the Transaction Validation Unit, a //! 5-stage transaction validation pipeline in software. -use accounting_stage::AccountingStage; use crdt::{Crdt, ReplicatedData}; use entry_writer::EntryWriter; +use event_processor::EventProcessor; use ledger; use packet; use request_stage::{RequestProcessor, RequestStage}; @@ -18,24 +18,24 @@ use std::time::Duration; use streamer; pub struct Tvu { - accounting_stage: Arc, + event_processor: Arc, } impl Tvu { /// Create a new Tvu that wraps the given Accountant. - pub fn new(accounting_stage: AccountingStage) -> Self { + pub fn new(event_processor: EventProcessor) -> Self { Tvu { - accounting_stage: Arc::new(accounting_stage), + event_processor: Arc::new(event_processor), } } fn drain_service( - accounting_stage: Arc, + event_processor: Arc, request_processor: Arc, exit: Arc, ) -> JoinHandle<()> { spawn(move || { - let entry_writer = EntryWriter::new(&accounting_stage, &request_processor); + let entry_writer = EntryWriter::new(&event_processor, &request_processor); loop { let _ = entry_writer.drain_entries(); if exit.load(Ordering::Relaxed) { @@ -57,7 +57,7 @@ impl Tvu { let blobs = verified_receiver.recv_timeout(timer)?; trace!("replicating blobs {}", blobs.len()); let entries = ledger::reconstruct_entries_from_blobs(&blobs); - obj.accounting_stage + obj.event_processor .accountant .process_verified_entries(entries)?; for blob in blobs { @@ -167,10 +167,10 @@ impl Tvu { let sig_verify_stage = SigVerifyStage::new(exit.clone(), packet_receiver); - let request_processor = RequestProcessor::new(obj.accounting_stage.accountant.clone()); + let request_processor = RequestProcessor::new(obj.event_processor.accountant.clone()); let request_stage = RequestStage::new( request_processor, - obj.accounting_stage.clone(), + obj.event_processor.clone(), exit.clone(), sig_verify_stage.output, packet_recycler.clone(), @@ -178,7 +178,7 @@ impl Tvu { ); let t_write = Self::drain_service( - obj.accounting_stage.clone(), + obj.event_processor.clone(), request_stage.request_processor.clone(), exit.clone(), ); @@ -230,12 +230,12 @@ pub fn test_node() -> (ReplicatedData, UdpSocket, UdpSocket, UdpSocket, UdpSocke #[cfg(test)] mod tests { use accountant::Accountant; - use accounting_stage::AccountingStage; use bincode::serialize; use chrono::prelude::*; use crdt::Crdt; use entry; use event::Event; + use event_processor::EventProcessor; use hash::{hash, Hash}; use logger; use mint::Mint; @@ -302,8 +302,8 @@ mod tests { let starting_balance = 10_000; let alice = Mint::new(starting_balance); let accountant = Accountant::new(&alice); - let accounting_stage = AccountingStage::new(accountant, &alice.last_id(), Some(30)); - let tvu = Arc::new(Tvu::new(accounting_stage)); + let event_processor = EventProcessor::new(accountant, &alice.last_id(), Some(30)); + let tvu = Arc::new(Tvu::new(event_processor)); let replicate_addr = target1_data.replicate_addr; let threads = Tvu::serve( &tvu, @@ -328,7 +328,7 @@ mod tests { w.set_index(i).unwrap(); w.set_id(leader_id).unwrap(); - let accountant = &tvu.accounting_stage.accountant; + let accountant = &tvu.event_processor.accountant; let tr0 = Event::new_timestamp(&bob_keypair, Utc::now()); let entry0 = entry::create_entry(&cur_hash, i, vec![tr0]); @@ -370,7 +370,7 @@ mod tests { msgs.push(msg); } - let accountant = &tvu.accounting_stage.accountant; + let accountant = &tvu.event_processor.accountant; let alice_balance = accountant.get_balance(&alice.keypair().pubkey()).unwrap(); assert_eq!(alice_balance, alice_ref_balance); From a3869dd4c194d5ce86778ad6c75566ef9d7051e2 Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Sat, 12 May 2018 15:14:10 -0600 Subject: [PATCH 24/25] Move entry_receiver to RequestStage This can move to AccountingStage once RequestStage stops calling process_events(). --- src/entry_writer.rs | 27 ++++++++++++--------------- src/event_processor.rs | 22 ++++++++-------------- src/request_stage.rs | 10 ++++++++-- src/rpu.rs | 12 ++++++++++-- src/tvu.rs | 7 +++++-- 5 files changed, 43 insertions(+), 35 deletions(-) diff --git a/src/entry_writer.rs b/src/entry_writer.rs index 42fe5d53fcda5d..24a13e1792db63 100644 --- a/src/entry_writer.rs +++ b/src/entry_writer.rs @@ -10,6 +10,7 @@ use serde_json; use std::collections::VecDeque; use std::io::Write; use std::io::sink; +use std::sync::mpsc::Receiver; use std::sync::{Arc, Mutex}; use std::time::Duration; use streamer; @@ -42,22 +43,17 @@ impl<'a> EntryWriter<'a> { self.request_processor.notify_entry_info_subscribers(&entry); } - fn write_entries(&self, writer: &Mutex) -> Result> { + fn write_entries( + &self, + writer: &Mutex, + entry_receiver: &Receiver, + ) -> Result> { //TODO implement a serialize for channel that does this without allocations let mut l = vec![]; - let entry = self.event_processor - .output - .lock() - .expect("'ouput' lock in fn receive_all") - .recv_timeout(Duration::new(1, 0))?; + let entry = entry_receiver.recv_timeout(Duration::new(1, 0))?; self.write_entry(writer, &entry); l.push(entry); - while let Ok(entry) = self.event_processor - .output - .lock() - .expect("'output' lock in fn write_entries") - .try_recv() - { + while let Ok(entry) = entry_receiver.try_recv() { self.write_entry(writer, &entry); l.push(entry); } @@ -71,9 +67,10 @@ impl<'a> EntryWriter<'a> { broadcast: &streamer::BlobSender, blob_recycler: &packet::BlobRecycler, writer: &Mutex, + entry_receiver: &Receiver, ) -> Result<()> { let mut q = VecDeque::new(); - let list = self.write_entries(writer)?; + let list = self.write_entries(writer, entry_receiver)?; trace!("New blobs? {}", list.len()); ledger::process_entry_list_into_blobs(&list, blob_recycler, &mut q); if !q.is_empty() { @@ -84,8 +81,8 @@ impl<'a> EntryWriter<'a> { /// Process any Entry items that have been published by the Historian. /// continuosly broadcast blobs of entries out - pub fn drain_entries(&self) -> Result<()> { - self.write_entries(&Arc::new(Mutex::new(sink())))?; + pub fn drain_entries(&self, entry_receiver: &Receiver) -> Result<()> { + self.write_entries(&Arc::new(Mutex::new(sink())), entry_receiver)?; Ok(()) } } diff --git a/src/event_processor.rs b/src/event_processor.rs index df547af3d1450e..19f69c9d6cef62 100644 --- a/src/event_processor.rs +++ b/src/event_processor.rs @@ -7,12 +7,10 @@ use hash::Hash; use historian::Historian; use recorder::Signal; use result::Result; -use std::sync::mpsc::{channel, Receiver, Sender}; +use std::sync::mpsc::{channel, Sender}; use std::sync::{Arc, Mutex}; pub struct EventProcessor { - pub output: Mutex>, - entry_sender: Mutex>, pub accountant: Arc, historian_input: Mutex>, historian: Mutex, @@ -23,10 +21,7 @@ impl EventProcessor { pub fn new(accountant: Accountant, start_hash: &Hash, ms_per_tick: Option) -> Self { let (historian_input, event_receiver) = channel(); let historian = Historian::new(event_receiver, start_hash, ms_per_tick); - let (entry_sender, output) = channel(); EventProcessor { - output: Mutex::new(output), - entry_sender: Mutex::new(entry_sender), accountant: Arc::new(accountant), historian_input: Mutex::new(historian_input), historian: Mutex::new(historian), @@ -34,7 +29,7 @@ impl EventProcessor { } /// Process the transactions in parallel and then log the successful ones. - pub fn process_events(&self, events: Vec) -> Result<()> { + pub fn process_events(&self, events: Vec) -> Result { let historian = self.historian.lock().unwrap(); let results = self.accountant.process_verified_events(events); let events = results.into_iter().filter_map(|x| x.ok()).collect(); @@ -44,15 +39,13 @@ impl EventProcessor { // Wait for the historian to tag our Events with an ID and then register it. let entry = historian.output.lock().unwrap().recv()?; self.accountant.register_entry_id(&entry.id); - self.entry_sender.lock().unwrap().send(entry)?; - Ok(()) + Ok(entry) } } #[cfg(test)] mod tests { use accountant::Accountant; - use entry::Entry; use event::Event; use event_processor::EventProcessor; use mint::Mint; @@ -60,6 +53,8 @@ mod tests { use transaction::Transaction; #[test] + // TODO: Move this test accounting_stage. Calling process_events() directly + // defeats the purpose of this test. fn test_accounting_sequential_consistency() { // In this attack we'll demonstrate that a verifier can interpret the ledger // differently if either the server doesn't signal the ledger to add an @@ -72,16 +67,15 @@ mod tests { let alice = KeyPair::new(); let tr = Transaction::new(&mint.keypair(), alice.pubkey(), 2, mint.last_id()); let events = vec![Event::Transaction(tr)]; - assert!(event_processor.process_events(events).is_ok()); + let entry0 = event_processor.process_events(events).unwrap(); // Process a second batch that spends one of those tokens. let tr = Transaction::new(&alice, mint.pubkey(), 1, mint.last_id()); let events = vec![Event::Transaction(tr)]; - assert!(event_processor.process_events(events).is_ok()); + let entry1 = event_processor.process_events(events).unwrap(); // Collect the ledger and feed it to a new accountant. - drop(event_processor.entry_sender); - let entries: Vec = event_processor.output.lock().unwrap().iter().collect(); + let entries = vec![entry0, entry1]; // Assert the user holds one token, not two. If the server only output one // entry, then the second transaction will be rejected, because it drives diff --git a/src/request_stage.rs b/src/request_stage.rs index 3695fd01a25d80..068ce3e81295db 100644 --- a/src/request_stage.rs +++ b/src/request_stage.rs @@ -14,7 +14,7 @@ use signature::PublicKey; use std::collections::VecDeque; use std::net::{SocketAddr, UdpSocket}; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::mpsc::{channel, Receiver}; +use std::sync::mpsc::{channel, Receiver, Sender}; use std::sync::{Arc, Mutex}; use std::thread::{spawn, JoinHandle}; use std::time::Duration; @@ -207,6 +207,7 @@ impl RequestProcessor { &self, event_processor: &EventProcessor, verified_receiver: &Receiver)>>, + entry_sender: &Sender, responder_sender: &streamer::BlobSender, packet_recycler: &packet::PacketRecycler, blob_recycler: &packet::BlobRecycler, @@ -240,7 +241,8 @@ impl RequestProcessor { debug!("events: {} reqs: {}", events.len(), reqs.len()); debug!("process_events"); - event_processor.process_events(events)?; + let entry = event_processor.process_events(events)?; + entry_sender.send(entry)?; debug!("done process_events"); debug!("process_requests"); @@ -271,6 +273,7 @@ impl RequestProcessor { pub struct RequestStage { pub thread_hdl: JoinHandle<()>, + pub entry_receiver: Receiver, pub output: streamer::BlobReceiver, pub request_processor: Arc, } @@ -286,11 +289,13 @@ impl RequestStage { ) -> Self { let request_processor = Arc::new(request_processor); let request_processor_ = request_processor.clone(); + let (entry_sender, entry_receiver) = channel(); let (responder_sender, output) = channel(); let thread_hdl = spawn(move || loop { let e = request_processor_.process_request_packets( &event_processor, &verified_receiver, + &entry_sender, &responder_sender, &packet_recycler, &blob_recycler, @@ -303,6 +308,7 @@ impl RequestStage { }); RequestStage { thread_hdl, + entry_receiver, output, request_processor, } diff --git a/src/rpu.rs b/src/rpu.rs index 5fee29e0b5fe24..e4e275eee76ae5 100644 --- a/src/rpu.rs +++ b/src/rpu.rs @@ -2,6 +2,7 @@ //! 5-stage transaction processing pipeline in software. use crdt::{Crdt, ReplicatedData}; +use entry::Entry; use entry_writer::EntryWriter; use event_processor::EventProcessor; use packet; @@ -11,7 +12,7 @@ use sig_verify_stage::SigVerifyStage; use std::io::Write; use std::net::UdpSocket; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::mpsc::channel; +use std::sync::mpsc::{channel, Receiver}; use std::sync::{Arc, Mutex, RwLock}; use std::thread::{spawn, JoinHandle}; use streamer; @@ -35,10 +36,16 @@ impl Rpu { broadcast: streamer::BlobSender, blob_recycler: packet::BlobRecycler, writer: Mutex, + entry_receiver: Receiver, ) -> JoinHandle<()> { spawn(move || loop { let entry_writer = EntryWriter::new(&event_processor, &request_processor); - let _ = entry_writer.write_and_send_entries(&broadcast, &blob_recycler, &writer); + let _ = entry_writer.write_and_send_entries( + &broadcast, + &blob_recycler, + &writer, + &entry_receiver, + ); if exit.load(Ordering::Relaxed) { info!("broadcat_service exiting"); break; @@ -95,6 +102,7 @@ impl Rpu { broadcast_sender, blob_recycler.clone(), Mutex::new(writer), + request_stage.entry_receiver, ); let broadcast_socket = UdpSocket::bind(local)?; diff --git a/src/tvu.rs b/src/tvu.rs index 0d59ee3716bff7..869a9ba4ea0c8e 100644 --- a/src/tvu.rs +++ b/src/tvu.rs @@ -2,6 +2,7 @@ //! 5-stage transaction validation pipeline in software. use crdt::{Crdt, ReplicatedData}; +use entry::Entry; use entry_writer::EntryWriter; use event_processor::EventProcessor; use ledger; @@ -11,7 +12,7 @@ use result::Result; use sig_verify_stage::SigVerifyStage; use std::net::UdpSocket; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::mpsc::channel; +use std::sync::mpsc::{channel, Receiver}; use std::sync::{Arc, RwLock}; use std::thread::{spawn, JoinHandle}; use std::time::Duration; @@ -33,11 +34,12 @@ impl Tvu { event_processor: Arc, request_processor: Arc, exit: Arc, + entry_receiver: Receiver, ) -> JoinHandle<()> { spawn(move || { let entry_writer = EntryWriter::new(&event_processor, &request_processor); loop { - let _ = entry_writer.drain_entries(); + let _ = entry_writer.drain_entries(&entry_receiver); if exit.load(Ordering::Relaxed) { info!("drain_service exiting"); break; @@ -181,6 +183,7 @@ impl Tvu { obj.event_processor.clone(), request_stage.request_processor.clone(), exit.clone(), + request_stage.entry_receiver, ); let t_responder = streamer::responder( From 6264508f5e14f3b42cccb4d1e6dcddf889edd1e1 Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Sat, 12 May 2018 15:24:17 -0600 Subject: [PATCH 25/25] Consistent naming of senders and receivers --- src/event_processor.rs | 2 +- src/historian.rs | 20 ++++++++++---------- src/request_stage.rs | 12 ++++++------ src/rpu.rs | 4 ++-- src/sig_verify_stage.rs | 6 +++--- src/tvu.rs | 4 ++-- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/event_processor.rs b/src/event_processor.rs index 19f69c9d6cef62..8d3b9cdafefa74 100644 --- a/src/event_processor.rs +++ b/src/event_processor.rs @@ -37,7 +37,7 @@ impl EventProcessor { sender.send(Signal::Events(events))?; // Wait for the historian to tag our Events with an ID and then register it. - let entry = historian.output.lock().unwrap().recv()?; + let entry = historian.entry_receiver.lock().unwrap().recv()?; self.accountant.register_entry_id(&entry.id); Ok(entry) } diff --git a/src/historian.rs b/src/historian.rs index 351fca9c085636..b44fabece9b8ea 100644 --- a/src/historian.rs +++ b/src/historian.rs @@ -10,7 +10,7 @@ use std::thread::{spawn, JoinHandle}; use std::time::Instant; pub struct Historian { - pub output: Mutex>, + pub entry_receiver: Mutex>, pub thread_hdl: JoinHandle, } @@ -20,11 +20,11 @@ impl Historian { start_hash: &Hash, ms_per_tick: Option, ) -> Self { - let (entry_sender, output) = channel(); + let (entry_sender, entry_receiver) = channel(); let thread_hdl = Historian::create_recorder(*start_hash, ms_per_tick, event_receiver, entry_sender); Historian { - output: Mutex::new(output), + entry_receiver: Mutex::new(entry_receiver), thread_hdl, } } @@ -52,9 +52,9 @@ impl Historian { } pub fn receive(self: &Self) -> Result { - self.output + self.entry_receiver .lock() - .expect("'output' lock in pub fn receive") + .expect("'entry_receiver' lock in pub fn receive") .try_recv() } } @@ -78,9 +78,9 @@ mod tests { sleep(Duration::new(0, 1_000_000)); input.send(Signal::Tick).unwrap(); - let entry0 = hist.output.lock().unwrap().recv().unwrap(); - let entry1 = hist.output.lock().unwrap().recv().unwrap(); - let entry2 = hist.output.lock().unwrap().recv().unwrap(); + let entry0 = hist.entry_receiver.lock().unwrap().recv().unwrap(); + let entry1 = hist.entry_receiver.lock().unwrap().recv().unwrap(); + let entry2 = hist.entry_receiver.lock().unwrap().recv().unwrap(); assert_eq!(entry0.num_hashes, 0); assert_eq!(entry1.num_hashes, 0); @@ -100,7 +100,7 @@ mod tests { let (input, event_receiver) = channel(); let zero = Hash::default(); let hist = Historian::new(event_receiver, &zero, None); - drop(hist.output); + drop(hist.entry_receiver); input.send(Signal::Tick).unwrap(); assert_eq!( hist.thread_hdl.join().unwrap(), @@ -116,7 +116,7 @@ mod tests { sleep(Duration::from_millis(300)); input.send(Signal::Tick).unwrap(); drop(input); - let entries: Vec = hist.output.lock().unwrap().iter().collect(); + let entries: Vec = hist.entry_receiver.lock().unwrap().iter().collect(); assert!(entries.len() > 1); // Ensure the ID is not the seed. diff --git a/src/request_stage.rs b/src/request_stage.rs index 068ce3e81295db..6b9a693cd9d4c6 100644 --- a/src/request_stage.rs +++ b/src/request_stage.rs @@ -208,7 +208,7 @@ impl RequestProcessor { event_processor: &EventProcessor, verified_receiver: &Receiver)>>, entry_sender: &Sender, - responder_sender: &streamer::BlobSender, + blob_sender: &streamer::BlobSender, packet_recycler: &packet::PacketRecycler, blob_recycler: &packet::BlobRecycler, ) -> Result<()> { @@ -253,7 +253,7 @@ impl RequestProcessor { if !blobs.is_empty() { info!("process: sending blobs: {}", blobs.len()); //don't wake up the other side if there is nothing - responder_sender.send(blobs)?; + blob_sender.send(blobs)?; } packet_recycler.recycle(msgs); } @@ -274,7 +274,7 @@ impl RequestProcessor { pub struct RequestStage { pub thread_hdl: JoinHandle<()>, pub entry_receiver: Receiver, - pub output: streamer::BlobReceiver, + pub blob_receiver: streamer::BlobReceiver, pub request_processor: Arc, } @@ -290,13 +290,13 @@ impl RequestStage { let request_processor = Arc::new(request_processor); let request_processor_ = request_processor.clone(); let (entry_sender, entry_receiver) = channel(); - let (responder_sender, output) = channel(); + let (blob_sender, blob_receiver) = channel(); let thread_hdl = spawn(move || loop { let e = request_processor_.process_request_packets( &event_processor, &verified_receiver, &entry_sender, - &responder_sender, + &blob_sender, &packet_recycler, &blob_recycler, ); @@ -309,7 +309,7 @@ impl RequestStage { RequestStage { thread_hdl, entry_receiver, - output, + blob_receiver, request_processor, } } diff --git a/src/rpu.rs b/src/rpu.rs index e4e275eee76ae5..fe7a2e5eb4db43 100644 --- a/src/rpu.rs +++ b/src/rpu.rs @@ -89,7 +89,7 @@ impl Rpu { request_processor, self.event_processor.clone(), exit.clone(), - sig_verify_stage.output, + sig_verify_stage.verified_receiver, packet_recycler.clone(), blob_recycler.clone(), ); @@ -119,7 +119,7 @@ impl Rpu { respond_socket, exit.clone(), blob_recycler.clone(), - request_stage.output, + request_stage.blob_receiver, ); let mut threads = vec![ diff --git a/src/sig_verify_stage.rs b/src/sig_verify_stage.rs index db6c6dbab717ae..6528537c6a0512 100644 --- a/src/sig_verify_stage.rs +++ b/src/sig_verify_stage.rs @@ -13,17 +13,17 @@ use streamer; use timing; pub struct SigVerifyStage { - pub output: Receiver)>>, + pub verified_receiver: Receiver)>>, pub thread_hdls: Vec>, } impl SigVerifyStage { pub fn new(exit: Arc, packets_receiver: Receiver) -> Self { - let (verified_sender, output) = channel(); + let (verified_sender, verified_receiver) = channel(); let thread_hdls = Self::verifier_services(exit, packets_receiver, verified_sender); SigVerifyStage { thread_hdls, - output, + verified_receiver, } } diff --git a/src/tvu.rs b/src/tvu.rs index 869a9ba4ea0c8e..59bb599a3a3ea2 100644 --- a/src/tvu.rs +++ b/src/tvu.rs @@ -174,7 +174,7 @@ impl Tvu { request_processor, obj.event_processor.clone(), exit.clone(), - sig_verify_stage.output, + sig_verify_stage.verified_receiver, packet_recycler.clone(), blob_recycler.clone(), ); @@ -190,7 +190,7 @@ impl Tvu { respond_socket, exit.clone(), blob_recycler.clone(), - request_stage.output, + request_stage.blob_receiver, ); let mut threads = vec![