From 9041c9f495de367458fccf18a87512c2dc240669 Mon Sep 17 00:00:00 2001 From: Justin Starry Date: Mon, 28 Sep 2020 23:45:42 +0800 Subject: [PATCH 1/6] Use protobufs to store confirmed blocks in BigTable --- Cargo.lock | 63 ++++++ storage-bigtable/Cargo.toml | 4 + storage-bigtable/build.rs | 17 ++ storage-bigtable/src/bigtable.rs | 95 ++++++++- storage-bigtable/src/confirmed_block.proto | 49 +++++ storage-bigtable/src/generated.rs | 68 ++++++ storage-bigtable/src/lib.rs | 236 +++++++++++++++++++-- 7 files changed, 516 insertions(+), 16 deletions(-) create mode 100644 storage-bigtable/build.rs create mode 100644 storage-bigtable/src/confirmed_block.proto create mode 100644 storage-bigtable/src/generated.rs diff --git a/Cargo.lock b/Cargo.lock index fed6c3040321cc..c02ae56661d1e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -968,6 +968,12 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "fixedbitset" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" + [[package]] name = "flate2" version = "1.0.14" @@ -1306,6 +1312,15 @@ dependencies = [ "autocfg 1.0.0", ] +[[package]] +name = "heck" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "hermit-abi" version = "0.1.13" @@ -2069,6 +2084,12 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "multimap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1255076139a83bb467426e7f8d0134968a8118844faa755985e077cf31850333" + [[package]] name = "native-tls" version = "0.2.4" @@ -2376,6 +2397,16 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "petgraph" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7" +dependencies = [ + "fixedbitset", + "indexmap", +] + [[package]] name = "pickledb" version = "0.4.1" @@ -2520,6 +2551,24 @@ dependencies = [ "prost-derive", ] +[[package]] +name = "prost-build" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b10678c913ecbd69350e8535c3aef91a8676c0773fc1d7b95cdd196d7f2f26" +dependencies = [ + "bytes 0.5.4", + "heck", + "itertools 0.8.2", + "log 0.4.8", + "multimap", + "petgraph", + "prost", + "prost-types", + "tempfile", + "which", +] + [[package]] name = "prost-derive" version = "0.6.1" @@ -2543,6 +2592,12 @@ dependencies = [ "prost", ] +[[package]] +name = "protobuf" +version = "1.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ccd6b79ec748412d4f2dfde1a80fa363a67def4062969f8aed3d790a30f28" + [[package]] name = "quick-error" version = "1.2.3" @@ -4552,7 +4607,9 @@ dependencies = [ "goauth", "log 0.4.8", "prost", + "prost-build", "prost-types", + "protobuf", "serde", "serde_derive", "smpl_jwt", @@ -5833,6 +5890,12 @@ dependencies = [ "smallvec 1.4.0", ] +[[package]] +name = "unicode-segmentation" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" + [[package]] name = "unicode-width" version = "0.1.7" diff --git a/storage-bigtable/Cargo.toml b/storage-bigtable/Cargo.toml index d1635d34224678..d7b9f42395e942 100644 --- a/storage-bigtable/Cargo.toml +++ b/storage-bigtable/Cargo.toml @@ -28,6 +28,10 @@ futures = "0.3.5" tonic = {version="0.3.0", features = ["tls", "transport"]} zstd = "0.5.1" +[build-dependencies] +prost-build = "0.6" +protobuf = "1.2" + [lib] crate-type = ["lib"] name = "solana_storage_bigtable" diff --git a/storage-bigtable/build.rs b/storage-bigtable/build.rs new file mode 100644 index 00000000000000..b050bd390963cf --- /dev/null +++ b/storage-bigtable/build.rs @@ -0,0 +1,17 @@ +use std::path::PathBuf; + +fn main() { + let src = PathBuf::from("./src"); + let includes = &[src.clone()]; + + // Generate BTreeMap fields for all messages. This forces encoded output to be consistent, so + // that encode/decode roundtrips can use encoded output for comparison. Otherwise trying to + // compare based on the Rust PartialEq implementations is difficult, due to presence of NaN + // values. + let mut config = prost_build::Config::new(); + config.btree_map(&["."]); + config.out_dir(&src); + config + .compile_protos(&[src.join("confirmed_block.proto")], includes) + .unwrap(); +} diff --git a/storage-bigtable/src/bigtable.rs b/storage-bigtable/src/bigtable.rs index 9d2b1d5740aaeb..d7132096e274ce 100644 --- a/storage-bigtable/src/bigtable.rs +++ b/storage-bigtable/src/bigtable.rs @@ -26,10 +26,14 @@ mod google { use google::bigtable::v2::*; pub type RowKey = String; -pub type CellName = String; -pub type CellValue = Vec; pub type RowData = Vec<(CellName, CellValue)>; pub type RowDataSlice<'a> = &'a [(CellName, CellValue)]; +pub type CellName = String; +pub type CellValue = Vec; +pub enum CellData { + Bincode(B), + Protobuf(P), +} #[derive(Debug, Error)] pub enum Error { @@ -196,6 +200,23 @@ impl BigTableConnection { .retry(ExponentialBackoff::default()) .await } + + pub async fn put_protobuf_cells_with_retry( + &self, + table: &str, + cells: &[(RowKey, T)], + ) -> Result + where + T: prost::Message, + { + use backoff::{future::FutureOperation as _, ExponentialBackoff}; + (|| async { + let mut client = self.client(); + Ok(client.put_protobuf_cells(table, cells).await?) + }) + .retry(ExponentialBackoff::default()) + .await + } } pub struct BigTable { @@ -484,7 +505,30 @@ impl BigTable { T: serde::de::DeserializeOwned, { let row_data = self.get_single_row_data(table, key.clone()).await?; - deserialize_cell_data(&row_data, table, key.to_string()) + deserialize_bincode_cell_data(&row_data, table, key.to_string()) + } + + pub async fn get_bincode_or_protobuf_cell( + &mut self, + table: &str, + key: RowKey, + ) -> Result> + where + B: serde::de::DeserializeOwned, + P: prost::Message + Default, + { + let row_data = self.get_single_row_data(table, key.clone()).await?; + match deserialize_protobuf_cell_data(&row_data, table, key.to_string()) { + Ok(result) => return Ok(CellData::Protobuf(result)), + Err(err) => match err { + Error::ObjectNotFound(_) => {} + _ => return Err(err), + }, + } + match deserialize_bincode_cell_data(&row_data, table, key.to_string()) { + Ok(result) => Ok(CellData::Bincode(result)), + Err(err) => Err(err), + } } pub async fn put_bincode_cells( @@ -506,9 +550,52 @@ impl BigTable { self.put_row_data(table, "x", &new_row_data).await?; Ok(bytes_written) } + + pub async fn put_protobuf_cells( + &mut self, + table: &str, + cells: &[(RowKey, T)], + ) -> Result + where + T: prost::Message, + { + let mut bytes_written = 0; + let mut new_row_data = vec![]; + for (row_key, data) in cells { + let mut buf = Vec::with_capacity(data.encoded_len()); + data.encode(&mut buf).unwrap(); + let data = compress_best(&buf)?; + bytes_written += data.len(); + new_row_data.push((row_key, vec![("proto".to_string(), data)])); + } + + self.put_row_data(table, "x", &new_row_data).await?; + Ok(bytes_written) + } +} + +pub(crate) fn deserialize_protobuf_cell_data( + row_data: RowDataSlice, + table: &str, + key: RowKey, +) -> Result +where + T: prost::Message + Default, +{ + let value = &row_data + .iter() + .find(|(name, _)| name == "proto") + .ok_or_else(|| Error::ObjectNotFound(format!("{}/{}", table, key)))? + .1; + + let data = decompress(&value)?; + T::decode(&data[..]).map_err(|err| { + warn!("Failed to deserialize {}/{}: {}", table, key, err); + Error::ObjectCorrupt(format!("{}/{}", table, key)) + }) } -pub(crate) fn deserialize_cell_data( +pub(crate) fn deserialize_bincode_cell_data( row_data: RowDataSlice, table: &str, key: RowKey, diff --git a/storage-bigtable/src/confirmed_block.proto b/storage-bigtable/src/confirmed_block.proto new file mode 100644 index 00000000000000..dd4fb35cee4b81 --- /dev/null +++ b/storage-bigtable/src/confirmed_block.proto @@ -0,0 +1,49 @@ +syntax = "proto3"; + +package Generated; + +message ConfirmedBlock { + string previous_blockhash = 1; + string blockhash = 2; + uint64 parent_slot = 3; + repeated ConfirmedTransaction transactions = 4; + repeated Reward rewards = 5; + UnixTimestamp block_time = 6; +} + +message ConfirmedTransaction { + bytes transaction = 1; + TransactionStatusMeta meta = 2; +} + +message TransactionStatusMeta { + TransactionError err = 1; + uint64 fee = 2; + repeated uint64 pre_balances = 3; + repeated uint64 post_balances = 4; + repeated InnerInstructions inner_instructions = 5; +} + +message TransactionError { + bytes err = 1; +} + +message InnerInstructions { + uint32 index = 1; + repeated CompiledInstruction instructions = 2; +} + +message CompiledInstruction { + uint32 program_id_index = 1; + bytes accounts = 2; + bytes data = 3; +} + +message Reward { + string pubkey = 1; + int64 lamports = 2; +} + +message UnixTimestamp { + int64 timestamp = 1; +} \ No newline at end of file diff --git a/storage-bigtable/src/generated.rs b/storage-bigtable/src/generated.rs new file mode 100644 index 00000000000000..3ddd2023ba51fd --- /dev/null +++ b/storage-bigtable/src/generated.rs @@ -0,0 +1,68 @@ +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ConfirmedBlock { + #[prost(string, tag = "1")] + pub previous_blockhash: std::string::String, + #[prost(string, tag = "2")] + pub blockhash: std::string::String, + #[prost(uint64, tag = "3")] + pub parent_slot: u64, + #[prost(message, repeated, tag = "4")] + pub transactions: ::std::vec::Vec, + #[prost(message, repeated, tag = "5")] + pub rewards: ::std::vec::Vec, + #[prost(message, optional, tag = "6")] + pub block_time: ::std::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ConfirmedTransaction { + #[prost(bytes, tag = "1")] + pub transaction: std::vec::Vec, + #[prost(message, optional, tag = "2")] + pub meta: ::std::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionStatusMeta { + #[prost(message, optional, tag = "1")] + pub err: ::std::option::Option, + #[prost(uint64, tag = "2")] + pub fee: u64, + #[prost(uint64, repeated, tag = "3")] + pub pre_balances: ::std::vec::Vec, + #[prost(uint64, repeated, tag = "4")] + pub post_balances: ::std::vec::Vec, + #[prost(message, repeated, tag = "5")] + pub inner_instructions: ::std::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionError { + #[prost(bytes, tag = "1")] + pub err: std::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct InnerInstructions { + #[prost(uint32, tag = "1")] + pub index: u32, + #[prost(message, repeated, tag = "2")] + pub instructions: ::std::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CompiledInstruction { + #[prost(uint32, tag = "1")] + pub program_id_index: u32, + #[prost(bytes, tag = "2")] + pub accounts: std::vec::Vec, + #[prost(bytes, tag = "3")] + pub data: std::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Reward { + #[prost(string, tag = "1")] + pub pubkey: std::string::String, + #[prost(int64, tag = "2")] + pub lamports: i64, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct UnixTimestamp { + #[prost(int64, tag = "1")] + pub timestamp: i64, +} diff --git a/storage-bigtable/src/lib.rs b/storage-bigtable/src/lib.rs index bc2bc4cf863d6f..5825385000616d 100644 --- a/storage-bigtable/src/lib.rs +++ b/storage-bigtable/src/lib.rs @@ -2,6 +2,7 @@ use log::*; use serde::{Deserialize, Serialize}; use solana_sdk::{ clock::{Slot, UnixTimestamp}, + instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature, sysvar::is_sysvar_id, @@ -9,10 +10,13 @@ use solana_sdk::{ }; use solana_transaction_status::{ ConfirmedBlock, ConfirmedTransaction, ConfirmedTransactionStatusWithSignature, - InnerInstructions, Rewards, TransactionStatus, TransactionStatusMeta, + InnerInstructions, Reward, Rewards, TransactionStatus, TransactionStatusMeta, TransactionWithStatusMeta, }; -use std::collections::HashMap; +use std::{ + collections::HashMap, + convert::{TryFrom, TryInto}, +}; use thiserror::Error; #[macro_use] @@ -21,6 +25,7 @@ extern crate serde_derive; mod access_token; mod bigtable; mod compression; +mod generated; mod root_ca_certificate; #[derive(Debug, Error)] @@ -88,6 +93,24 @@ struct StoredConfirmedBlock { block_time: Option, } +impl From for generated::Reward { + fn from(reward: Reward) -> Self { + Self { + pubkey: reward.pubkey, + lamports: reward.lamports, + } + } +} + +impl From for Reward { + fn from(reward: generated::Reward) -> Self { + Self { + pubkey: reward.pubkey, + lamports: reward.lamports, + } + } +} + impl From for StoredConfirmedBlock { fn from(confirmed_block: ConfirmedBlock) -> Self { let ConfirmedBlock { @@ -132,6 +155,61 @@ impl From for ConfirmedBlock { } } +impl TryFrom for generated::ConfirmedBlock { + type Error = bincode::Error; + fn try_from(confirmed_block: ConfirmedBlock) -> std::result::Result { + let ConfirmedBlock { + previous_blockhash, + blockhash, + parent_slot, + transactions, + rewards, + block_time, + } = confirmed_block; + + Ok(Self { + previous_blockhash, + blockhash, + parent_slot, + transactions: transactions + .into_iter() + .map(|tx| tx.try_into()) + .collect::, Self::Error>>( + )?, + rewards: rewards.into_iter().map(|r| r.into()).collect(), + block_time: block_time.map(|timestamp| generated::UnixTimestamp { timestamp }), + }) + } +} + +impl TryFrom for ConfirmedBlock { + type Error = bincode::Error; + fn try_from( + confirmed_block: generated::ConfirmedBlock, + ) -> std::result::Result { + let generated::ConfirmedBlock { + previous_blockhash, + blockhash, + parent_slot, + transactions, + rewards, + block_time, + } = confirmed_block; + + Ok(Self { + previous_blockhash, + blockhash, + parent_slot, + transactions: transactions + .into_iter() + .map(|tx| tx.try_into()) + .collect::, Self::Error>>()?, + rewards: rewards.into_iter().map(|r| r.into()).collect(), + block_time: block_time.map(|generated::UnixTimestamp { timestamp }| timestamp), + }) + } +} + #[derive(Serialize, Deserialize)] struct StoredConfirmedBlockTransaction { transaction: Transaction, @@ -156,13 +234,42 @@ impl From for TransactionWithStatusMeta { } } +impl TryFrom for generated::ConfirmedTransaction { + type Error = bincode::Error; + fn try_from(value: TransactionWithStatusMeta) -> std::result::Result { + let meta = if let Some(meta) = value.meta { + Some(meta.try_into()?) + } else { + None + }; + Ok(Self { + transaction: bincode::serialize(&value.transaction)?, + meta, + }) + } +} + +impl TryFrom for TransactionWithStatusMeta { + type Error = bincode::Error; + fn try_from(value: generated::ConfirmedTransaction) -> std::result::Result { + let meta = if let Some(meta) = value.meta { + Some(meta.try_into()?) + } else { + None + }; + Ok(Self { + transaction: bincode::deserialize(&value.transaction)?, + meta, + }) + } +} + #[derive(Serialize, Deserialize)] struct StoredConfirmedBlockTransactionStatusMeta { err: Option, fee: u64, pre_balances: Vec, post_balances: Vec, - inner_instructions: Option>, } impl From for TransactionStatusMeta { @@ -172,7 +279,6 @@ impl From for TransactionStatusMeta { fee, pre_balances, post_balances, - inner_instructions, } = value; let status = match &err { None => Ok(()), @@ -183,7 +289,7 @@ impl From for TransactionStatusMeta { fee, pre_balances, post_balances, - inner_instructions, + inner_instructions: None, } } } @@ -195,7 +301,6 @@ impl From for StoredConfirmedBlockTransactionStatusMeta { fee, pre_balances, post_balances, - inner_instructions, .. } = value; Self { @@ -203,11 +308,110 @@ impl From for StoredConfirmedBlockTransactionStatusMeta { fee, pre_balances, post_balances, + } + } +} + +impl TryFrom for TransactionStatusMeta { + type Error = bincode::Error; + + fn try_from(value: generated::TransactionStatusMeta) -> std::result::Result { + let generated::TransactionStatusMeta { + err, + fee, + pre_balances, + post_balances, + inner_instructions, + } = value; + let status = match &err { + None => Ok(()), + Some(tx_error) => Err(bincode::deserialize(&tx_error.err)?), + }; + let inner_instructions = Some( + inner_instructions + .into_iter() + .map(|inner| inner.into()) + .collect(), + ); + Ok(Self { + status, + fee, + pre_balances, + post_balances, inner_instructions, + }) + } +} + +impl From for generated::InnerInstructions { + fn from(value: InnerInstructions) -> Self { + Self { + index: value.index as u32, + instructions: value.instructions.into_iter().map(|i| i.into()).collect(), + } + } +} + +impl From for InnerInstructions { + fn from(value: generated::InnerInstructions) -> Self { + Self { + index: value.index as u8, + instructions: value.instructions.into_iter().map(|i| i.into()).collect(), + } + } +} + +impl From for generated::CompiledInstruction { + fn from(value: CompiledInstruction) -> Self { + Self { + program_id_index: value.program_id_index as u32, + accounts: value.accounts, + data: value.data, + } + } +} + +impl From for CompiledInstruction { + fn from(value: generated::CompiledInstruction) -> Self { + Self { + program_id_index: value.program_id_index as u8, + accounts: value.accounts, + data: value.data, } } } +impl TryFrom for generated::TransactionStatusMeta { + type Error = bincode::Error; + fn try_from(value: TransactionStatusMeta) -> std::result::Result { + let TransactionStatusMeta { + status, + fee, + pre_balances, + post_balances, + inner_instructions, + } = value; + let err = match status { + Ok(()) => None, + Err(err) => Some(generated::TransactionError { + err: bincode::serialize(&err)?, + }), + }; + let inner_instructions = inner_instructions + .unwrap_or_default() + .into_iter() + .map(|ii| ii.into()) + .collect(); + Ok(Self { + err, + fee, + pre_balances, + post_balances, + inner_instructions, + }) + } +} + // A serialized `TransactionInfo` is stored in the `tx` table #[derive(Serialize, Deserialize)] struct TransactionInfo { @@ -279,10 +483,18 @@ impl LedgerStorage { /// Fetch the confirmed block from the desired slot pub async fn get_confirmed_block(&self, slot: Slot) -> Result { let mut bigtable = self.connection.client(); - let block = bigtable - .get_bincode_cell::("blocks", slot_to_key(slot)) + let block_cell_data = bigtable + .get_bincode_or_protobuf_cell::( + "blocks", + slot_to_key(slot), + ) .await?; - Ok(block.into()) + Ok(match block_cell_data { + bigtable::CellData::Bincode(block) => block.into(), + bigtable::CellData::Protobuf(block) => block + .try_into() + .map_err(|_err| bigtable::Error::ObjectCorrupt("todo".into()))?, + }) } pub async fn get_signature_status(&self, signature: &Signature) -> Result { @@ -405,7 +617,7 @@ impl LedgerStorage { )) })?; let mut cell_data: Vec = - bigtable::deserialize_cell_data(&data, "tx-by-addr", row_key)?; + bigtable::deserialize_bincode_cell_data(&data, "tx-by-addr", row_key)?; cell_data.reverse(); for tx_by_addr_info in cell_data.into_iter() { // Filter out records before `before_transaction_index` @@ -508,10 +720,10 @@ impl LedgerStorage { // Store the block itself last, after all other metadata about the block has been // successfully stored. This avoids partial uploaded blocks from becoming visible to // `get_confirmed_block()` and `get_confirmed_blocks()` - let blocks_cells = [(slot_to_key(slot), confirmed_block.into())]; + let blocks_cells = [(slot_to_key(slot), confirmed_block.try_into().unwrap())]; bytes_written += self .connection - .put_bincode_cells_with_retry::("blocks", &blocks_cells) + .put_protobuf_cells_with_retry::("blocks", &blocks_cells) .await?; info!( "uploaded block for slot {}: {} transactions, {} bytes", From 403f82620fcc20090b730f8b3832270c8165c505 Mon Sep 17 00:00:00 2001 From: Justin Starry Date: Wed, 30 Sep 2020 01:14:22 +0800 Subject: [PATCH 2/6] Cleanup --- Cargo.lock | 63 ---- storage-bigtable/Cargo.toml | 4 - storage-bigtable/build-proto/src/main.rs | 16 + storage-bigtable/build.rs | 17 -- storage-bigtable/src/lib.rs | 229 +------------- .../src/{ => proto}/confirmed_block.proto | 20 +- storage-bigtable/src/{ => proto}/generated.rs | 31 +- storage-bigtable/src/proto/mod.rs | 286 ++++++++++++++++++ 8 files changed, 361 insertions(+), 305 deletions(-) delete mode 100644 storage-bigtable/build.rs rename storage-bigtable/src/{ => proto}/confirmed_block.proto (67%) rename storage-bigtable/src/{ => proto}/generated.rs (67%) create mode 100644 storage-bigtable/src/proto/mod.rs diff --git a/Cargo.lock b/Cargo.lock index c02ae56661d1e0..fed6c3040321cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -968,12 +968,6 @@ dependencies = [ "winapi 0.3.8", ] -[[package]] -name = "fixedbitset" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" - [[package]] name = "flate2" version = "1.0.14" @@ -1312,15 +1306,6 @@ dependencies = [ "autocfg 1.0.0", ] -[[package]] -name = "heck" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" -dependencies = [ - "unicode-segmentation", -] - [[package]] name = "hermit-abi" version = "0.1.13" @@ -2084,12 +2069,6 @@ dependencies = [ "winapi 0.3.8", ] -[[package]] -name = "multimap" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1255076139a83bb467426e7f8d0134968a8118844faa755985e077cf31850333" - [[package]] name = "native-tls" version = "0.2.4" @@ -2397,16 +2376,6 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" -[[package]] -name = "petgraph" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7" -dependencies = [ - "fixedbitset", - "indexmap", -] - [[package]] name = "pickledb" version = "0.4.1" @@ -2551,24 +2520,6 @@ dependencies = [ "prost-derive", ] -[[package]] -name = "prost-build" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b10678c913ecbd69350e8535c3aef91a8676c0773fc1d7b95cdd196d7f2f26" -dependencies = [ - "bytes 0.5.4", - "heck", - "itertools 0.8.2", - "log 0.4.8", - "multimap", - "petgraph", - "prost", - "prost-types", - "tempfile", - "which", -] - [[package]] name = "prost-derive" version = "0.6.1" @@ -2592,12 +2543,6 @@ dependencies = [ "prost", ] -[[package]] -name = "protobuf" -version = "1.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ccd6b79ec748412d4f2dfde1a80fa363a67def4062969f8aed3d790a30f28" - [[package]] name = "quick-error" version = "1.2.3" @@ -4607,9 +4552,7 @@ dependencies = [ "goauth", "log 0.4.8", "prost", - "prost-build", "prost-types", - "protobuf", "serde", "serde_derive", "smpl_jwt", @@ -5890,12 +5833,6 @@ dependencies = [ "smallvec 1.4.0", ] -[[package]] -name = "unicode-segmentation" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" - [[package]] name = "unicode-width" version = "0.1.7" diff --git a/storage-bigtable/Cargo.toml b/storage-bigtable/Cargo.toml index d7b9f42395e942..d1635d34224678 100644 --- a/storage-bigtable/Cargo.toml +++ b/storage-bigtable/Cargo.toml @@ -28,10 +28,6 @@ futures = "0.3.5" tonic = {version="0.3.0", features = ["tls", "transport"]} zstd = "0.5.1" -[build-dependencies] -prost-build = "0.6" -protobuf = "1.2" - [lib] crate-type = ["lib"] name = "solana_storage_bigtable" diff --git a/storage-bigtable/build-proto/src/main.rs b/storage-bigtable/build-proto/src/main.rs index a61afd1e183484..3cb4a42c9cf6c8 100644 --- a/storage-bigtable/build-proto/src/main.rs +++ b/storage-bigtable/build-proto/src/main.rs @@ -15,5 +15,21 @@ fn main() -> Result<(), std::io::Error> { .compile( &[googleapis.join("google/bigtable/v2/bigtable.proto")], &[googleapis], + )?; + + let out_dir = manifest_dir.join("../src/proto"); + let proto_files = manifest_dir.join("../src/proto"); + + println!("Protobuf directory: {}", proto_files.display()); + println!("output directory: {}", out_dir.display()); + + tonic_build::configure() + .build_client(true) + .build_server(false) + .format(true) + .out_dir(&out_dir) + .compile( + &[proto_files.join("confirmed_block.proto")], + &[proto_files], ) } diff --git a/storage-bigtable/build.rs b/storage-bigtable/build.rs deleted file mode 100644 index b050bd390963cf..00000000000000 --- a/storage-bigtable/build.rs +++ /dev/null @@ -1,17 +0,0 @@ -use std::path::PathBuf; - -fn main() { - let src = PathBuf::from("./src"); - let includes = &[src.clone()]; - - // Generate BTreeMap fields for all messages. This forces encoded output to be consistent, so - // that encode/decode roundtrips can use encoded output for comparison. Otherwise trying to - // compare based on the Rust PartialEq implementations is difficult, due to presence of NaN - // values. - let mut config = prost_build::Config::new(); - config.btree_map(&["."]); - config.out_dir(&src); - config - .compile_protos(&[src.join("confirmed_block.proto")], includes) - .unwrap(); -} diff --git a/storage-bigtable/src/lib.rs b/storage-bigtable/src/lib.rs index 5825385000616d..26bef199a89ee1 100644 --- a/storage-bigtable/src/lib.rs +++ b/storage-bigtable/src/lib.rs @@ -2,21 +2,16 @@ use log::*; use serde::{Deserialize, Serialize}; use solana_sdk::{ clock::{Slot, UnixTimestamp}, - instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature, sysvar::is_sysvar_id, transaction::{Transaction, TransactionError}, }; use solana_transaction_status::{ - ConfirmedBlock, ConfirmedTransaction, ConfirmedTransactionStatusWithSignature, - InnerInstructions, Reward, Rewards, TransactionStatus, TransactionStatusMeta, - TransactionWithStatusMeta, -}; -use std::{ - collections::HashMap, - convert::{TryFrom, TryInto}, + ConfirmedBlock, ConfirmedTransaction, ConfirmedTransactionStatusWithSignature, Rewards, + TransactionStatus, TransactionStatusMeta, TransactionWithStatusMeta, }; +use std::{collections::HashMap, convert::TryInto}; use thiserror::Error; #[macro_use] @@ -25,8 +20,9 @@ extern crate serde_derive; mod access_token; mod bigtable; mod compression; -mod generated; +mod proto; mod root_ca_certificate; +use proto::generated as prost; #[derive(Debug, Error)] pub enum Error { @@ -93,24 +89,6 @@ struct StoredConfirmedBlock { block_time: Option, } -impl From for generated::Reward { - fn from(reward: Reward) -> Self { - Self { - pubkey: reward.pubkey, - lamports: reward.lamports, - } - } -} - -impl From for Reward { - fn from(reward: generated::Reward) -> Self { - Self { - pubkey: reward.pubkey, - lamports: reward.lamports, - } - } -} - impl From for StoredConfirmedBlock { fn from(confirmed_block: ConfirmedBlock) -> Self { let ConfirmedBlock { @@ -155,61 +133,6 @@ impl From for ConfirmedBlock { } } -impl TryFrom for generated::ConfirmedBlock { - type Error = bincode::Error; - fn try_from(confirmed_block: ConfirmedBlock) -> std::result::Result { - let ConfirmedBlock { - previous_blockhash, - blockhash, - parent_slot, - transactions, - rewards, - block_time, - } = confirmed_block; - - Ok(Self { - previous_blockhash, - blockhash, - parent_slot, - transactions: transactions - .into_iter() - .map(|tx| tx.try_into()) - .collect::, Self::Error>>( - )?, - rewards: rewards.into_iter().map(|r| r.into()).collect(), - block_time: block_time.map(|timestamp| generated::UnixTimestamp { timestamp }), - }) - } -} - -impl TryFrom for ConfirmedBlock { - type Error = bincode::Error; - fn try_from( - confirmed_block: generated::ConfirmedBlock, - ) -> std::result::Result { - let generated::ConfirmedBlock { - previous_blockhash, - blockhash, - parent_slot, - transactions, - rewards, - block_time, - } = confirmed_block; - - Ok(Self { - previous_blockhash, - blockhash, - parent_slot, - transactions: transactions - .into_iter() - .map(|tx| tx.try_into()) - .collect::, Self::Error>>()?, - rewards: rewards.into_iter().map(|r| r.into()).collect(), - block_time: block_time.map(|generated::UnixTimestamp { timestamp }| timestamp), - }) - } -} - #[derive(Serialize, Deserialize)] struct StoredConfirmedBlockTransaction { transaction: Transaction, @@ -234,36 +157,6 @@ impl From for TransactionWithStatusMeta { } } -impl TryFrom for generated::ConfirmedTransaction { - type Error = bincode::Error; - fn try_from(value: TransactionWithStatusMeta) -> std::result::Result { - let meta = if let Some(meta) = value.meta { - Some(meta.try_into()?) - } else { - None - }; - Ok(Self { - transaction: bincode::serialize(&value.transaction)?, - meta, - }) - } -} - -impl TryFrom for TransactionWithStatusMeta { - type Error = bincode::Error; - fn try_from(value: generated::ConfirmedTransaction) -> std::result::Result { - let meta = if let Some(meta) = value.meta { - Some(meta.try_into()?) - } else { - None - }; - Ok(Self { - transaction: bincode::deserialize(&value.transaction)?, - meta, - }) - } -} - #[derive(Serialize, Deserialize)] struct StoredConfirmedBlockTransactionStatusMeta { err: Option, @@ -312,106 +205,6 @@ impl From for StoredConfirmedBlockTransactionStatusMeta { } } -impl TryFrom for TransactionStatusMeta { - type Error = bincode::Error; - - fn try_from(value: generated::TransactionStatusMeta) -> std::result::Result { - let generated::TransactionStatusMeta { - err, - fee, - pre_balances, - post_balances, - inner_instructions, - } = value; - let status = match &err { - None => Ok(()), - Some(tx_error) => Err(bincode::deserialize(&tx_error.err)?), - }; - let inner_instructions = Some( - inner_instructions - .into_iter() - .map(|inner| inner.into()) - .collect(), - ); - Ok(Self { - status, - fee, - pre_balances, - post_balances, - inner_instructions, - }) - } -} - -impl From for generated::InnerInstructions { - fn from(value: InnerInstructions) -> Self { - Self { - index: value.index as u32, - instructions: value.instructions.into_iter().map(|i| i.into()).collect(), - } - } -} - -impl From for InnerInstructions { - fn from(value: generated::InnerInstructions) -> Self { - Self { - index: value.index as u8, - instructions: value.instructions.into_iter().map(|i| i.into()).collect(), - } - } -} - -impl From for generated::CompiledInstruction { - fn from(value: CompiledInstruction) -> Self { - Self { - program_id_index: value.program_id_index as u32, - accounts: value.accounts, - data: value.data, - } - } -} - -impl From for CompiledInstruction { - fn from(value: generated::CompiledInstruction) -> Self { - Self { - program_id_index: value.program_id_index as u8, - accounts: value.accounts, - data: value.data, - } - } -} - -impl TryFrom for generated::TransactionStatusMeta { - type Error = bincode::Error; - fn try_from(value: TransactionStatusMeta) -> std::result::Result { - let TransactionStatusMeta { - status, - fee, - pre_balances, - post_balances, - inner_instructions, - } = value; - let err = match status { - Ok(()) => None, - Err(err) => Some(generated::TransactionError { - err: bincode::serialize(&err)?, - }), - }; - let inner_instructions = inner_instructions - .unwrap_or_default() - .into_iter() - .map(|ii| ii.into()) - .collect(); - Ok(Self { - err, - fee, - pre_balances, - post_balances, - inner_instructions, - }) - } -} - // A serialized `TransactionInfo` is stored in the `tx` table #[derive(Serialize, Deserialize)] struct TransactionInfo { @@ -484,16 +277,16 @@ impl LedgerStorage { pub async fn get_confirmed_block(&self, slot: Slot) -> Result { let mut bigtable = self.connection.client(); let block_cell_data = bigtable - .get_bincode_or_protobuf_cell::( + .get_bincode_or_protobuf_cell::( "blocks", slot_to_key(slot), ) .await?; Ok(match block_cell_data { bigtable::CellData::Bincode(block) => block.into(), - bigtable::CellData::Protobuf(block) => block - .try_into() - .map_err(|_err| bigtable::Error::ObjectCorrupt("todo".into()))?, + bigtable::CellData::Protobuf(block) => block.try_into().map_err(|_err| { + bigtable::Error::ObjectCorrupt(format!("blocks/{}", slot_to_key(slot))) + })?, }) } @@ -720,10 +513,10 @@ impl LedgerStorage { // Store the block itself last, after all other metadata about the block has been // successfully stored. This avoids partial uploaded blocks from becoming visible to // `get_confirmed_block()` and `get_confirmed_blocks()` - let blocks_cells = [(slot_to_key(slot), confirmed_block.try_into().unwrap())]; + let blocks_cells = [(slot_to_key(slot), confirmed_block.into())]; bytes_written += self .connection - .put_protobuf_cells_with_retry::("blocks", &blocks_cells) + .put_protobuf_cells_with_retry::("blocks", &blocks_cells) .await?; info!( "uploaded block for slot {}: {} transactions, {} bytes", diff --git a/storage-bigtable/src/confirmed_block.proto b/storage-bigtable/src/proto/confirmed_block.proto similarity index 67% rename from storage-bigtable/src/confirmed_block.proto rename to storage-bigtable/src/proto/confirmed_block.proto index dd4fb35cee4b81..e3b085627925e0 100644 --- a/storage-bigtable/src/confirmed_block.proto +++ b/storage-bigtable/src/proto/confirmed_block.proto @@ -12,10 +12,28 @@ message ConfirmedBlock { } message ConfirmedTransaction { - bytes transaction = 1; + Transaction transaction = 1; TransactionStatusMeta meta = 2; } +message Transaction { + repeated bytes signatures = 1; + Message message = 2; +} + +message Message { + MessageHeader header = 1; + repeated bytes account_keys = 2; + bytes recent_blockhash = 3; + repeated CompiledInstruction instructions = 4; +} + +message MessageHeader { + uint32 num_required_signatures = 1; + uint32 num_readonly_signed_accounts = 2; + uint32 num_readonly_unsigned_accounts = 3; +} + message TransactionStatusMeta { TransactionError err = 1; uint64 fee = 2; diff --git a/storage-bigtable/src/generated.rs b/storage-bigtable/src/proto/generated.rs similarity index 67% rename from storage-bigtable/src/generated.rs rename to storage-bigtable/src/proto/generated.rs index 3ddd2023ba51fd..f49b1a7a40dd90 100644 --- a/storage-bigtable/src/generated.rs +++ b/storage-bigtable/src/proto/generated.rs @@ -15,12 +15,39 @@ pub struct ConfirmedBlock { } #[derive(Clone, PartialEq, ::prost::Message)] pub struct ConfirmedTransaction { - #[prost(bytes, tag = "1")] - pub transaction: std::vec::Vec, + #[prost(message, optional, tag = "1")] + pub transaction: ::std::option::Option, #[prost(message, optional, tag = "2")] pub meta: ::std::option::Option, } #[derive(Clone, PartialEq, ::prost::Message)] +pub struct Transaction { + #[prost(bytes, repeated, tag = "1")] + pub signatures: ::std::vec::Vec>, + #[prost(message, optional, tag = "2")] + pub message: ::std::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Message { + #[prost(message, optional, tag = "1")] + pub header: ::std::option::Option, + #[prost(bytes, repeated, tag = "2")] + pub account_keys: ::std::vec::Vec>, + #[prost(bytes, tag = "3")] + pub recent_blockhash: std::vec::Vec, + #[prost(message, repeated, tag = "4")] + pub instructions: ::std::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MessageHeader { + #[prost(uint32, tag = "1")] + pub num_required_signatures: u32, + #[prost(uint32, tag = "2")] + pub num_readonly_signed_accounts: u32, + #[prost(uint32, tag = "3")] + pub num_readonly_unsigned_accounts: u32, +} +#[derive(Clone, PartialEq, ::prost::Message)] pub struct TransactionStatusMeta { #[prost(message, optional, tag = "1")] pub err: ::std::option::Option, diff --git a/storage-bigtable/src/proto/mod.rs b/storage-bigtable/src/proto/mod.rs new file mode 100644 index 00000000000000..2fc9b14374af6e --- /dev/null +++ b/storage-bigtable/src/proto/mod.rs @@ -0,0 +1,286 @@ +use solana_sdk::{ + hash::Hash, + instruction::CompiledInstruction, + message::{Message, MessageHeader}, + pubkey::Pubkey, + signature::Signature, + transaction::Transaction, +}; +use solana_transaction_status::{ + ConfirmedBlock, InnerInstructions, Reward, TransactionStatusMeta, TransactionWithStatusMeta, +}; +use std::convert::{TryFrom, TryInto}; + +pub mod generated; + +impl From for generated::Reward { + fn from(reward: Reward) -> Self { + Self { + pubkey: reward.pubkey, + lamports: reward.lamports, + } + } +} + +impl From for Reward { + fn from(reward: generated::Reward) -> Self { + Self { + pubkey: reward.pubkey, + lamports: reward.lamports, + } + } +} + +impl From for generated::ConfirmedBlock { + fn from(confirmed_block: ConfirmedBlock) -> Self { + let ConfirmedBlock { + previous_blockhash, + blockhash, + parent_slot, + transactions, + rewards, + block_time, + } = confirmed_block; + + Self { + previous_blockhash, + blockhash, + parent_slot, + transactions: transactions.into_iter().map(|tx| tx.into()).collect(), + rewards: rewards.into_iter().map(|r| r.into()).collect(), + block_time: block_time.map(|timestamp| generated::UnixTimestamp { timestamp }), + } + } +} + +impl TryFrom for ConfirmedBlock { + type Error = bincode::Error; + fn try_from( + confirmed_block: generated::ConfirmedBlock, + ) -> std::result::Result { + let generated::ConfirmedBlock { + previous_blockhash, + blockhash, + parent_slot, + transactions, + rewards, + block_time, + } = confirmed_block; + + Ok(Self { + previous_blockhash, + blockhash, + parent_slot, + transactions: transactions + .into_iter() + .map(|tx| tx.try_into()) + .collect::, Self::Error>>()?, + rewards: rewards.into_iter().map(|r| r.into()).collect(), + block_time: block_time.map(|generated::UnixTimestamp { timestamp }| timestamp), + }) + } +} + +impl From for generated::ConfirmedTransaction { + fn from(value: TransactionWithStatusMeta) -> Self { + let meta = if let Some(meta) = value.meta { + Some(meta.into()) + } else { + None + }; + Self { + transaction: Some(value.transaction.into()), + meta, + } + } +} + +impl TryFrom for TransactionWithStatusMeta { + type Error = bincode::Error; + fn try_from(value: generated::ConfirmedTransaction) -> std::result::Result { + let meta = if let Some(meta) = value.meta { + Some(meta.try_into()?) + } else { + None + }; + Ok(Self { + transaction: value.transaction.expect("transaction is required").into(), + meta, + }) + } +} + +impl From for generated::Transaction { + fn from(value: Transaction) -> Self { + Self { + signatures: value + .signatures + .into_iter() + .map(|signature| >::as_ref(&signature).into()) + .collect(), + message: Some(value.message.into()), + } + } +} + +impl From for Transaction { + fn from(value: generated::Transaction) -> Self { + Self { + signatures: value + .signatures + .into_iter() + .map(|x| Signature::new(&x)) + .collect(), + message: value.message.expect("message is required").into(), + } + } +} + +impl From for generated::Message { + fn from(value: Message) -> Self { + Self { + header: Some(value.header.into()), + account_keys: value + .account_keys + .into_iter() + .map(|key| >::as_ref(&key).into()) + .collect(), + recent_blockhash: value.recent_blockhash.to_bytes().into(), + instructions: value.instructions.into_iter().map(|ix| ix.into()).collect(), + } + } +} + +impl From for Message { + fn from(value: generated::Message) -> Self { + Self { + header: value.header.expect("header is required").into(), + account_keys: value + .account_keys + .into_iter() + .map(|key| Pubkey::new(&key)) + .collect(), + recent_blockhash: Hash::new(&value.recent_blockhash), + instructions: value.instructions.into_iter().map(|ix| ix.into()).collect(), + } + } +} + +impl From for generated::MessageHeader { + fn from(value: MessageHeader) -> Self { + Self { + num_required_signatures: value.num_required_signatures as u32, + num_readonly_signed_accounts: value.num_readonly_signed_accounts as u32, + num_readonly_unsigned_accounts: value.num_readonly_unsigned_accounts as u32, + } + } +} + +impl From for MessageHeader { + fn from(value: generated::MessageHeader) -> Self { + Self { + num_required_signatures: value.num_required_signatures as u8, + num_readonly_signed_accounts: value.num_readonly_signed_accounts as u8, + num_readonly_unsigned_accounts: value.num_readonly_unsigned_accounts as u8, + } + } +} + +impl From for generated::TransactionStatusMeta { + fn from(value: TransactionStatusMeta) -> Self { + let TransactionStatusMeta { + status, + fee, + pre_balances, + post_balances, + inner_instructions, + } = value; + let err = match status { + Ok(()) => None, + Err(err) => Some(generated::TransactionError { + err: bincode::serialize(&err).expect("transaction error to serialize to bytes"), + }), + }; + let inner_instructions = inner_instructions + .unwrap_or_default() + .into_iter() + .map(|ii| ii.into()) + .collect(); + Self { + err, + fee, + pre_balances, + post_balances, + inner_instructions, + } + } +} + +impl TryFrom for TransactionStatusMeta { + type Error = bincode::Error; + + fn try_from(value: generated::TransactionStatusMeta) -> std::result::Result { + let generated::TransactionStatusMeta { + err, + fee, + pre_balances, + post_balances, + inner_instructions, + } = value; + let status = match &err { + None => Ok(()), + Some(tx_error) => Err(bincode::deserialize(&tx_error.err)?), + }; + let inner_instructions = Some( + inner_instructions + .into_iter() + .map(|inner| inner.into()) + .collect(), + ); + Ok(Self { + status, + fee, + pre_balances, + post_balances, + inner_instructions, + }) + } +} + +impl From for generated::InnerInstructions { + fn from(value: InnerInstructions) -> Self { + Self { + index: value.index as u32, + instructions: value.instructions.into_iter().map(|i| i.into()).collect(), + } + } +} + +impl From for InnerInstructions { + fn from(value: generated::InnerInstructions) -> Self { + Self { + index: value.index as u8, + instructions: value.instructions.into_iter().map(|i| i.into()).collect(), + } + } +} + +impl From for generated::CompiledInstruction { + fn from(value: CompiledInstruction) -> Self { + Self { + program_id_index: value.program_id_index as u32, + accounts: value.accounts, + data: value.data, + } + } +} + +impl From for CompiledInstruction { + fn from(value: generated::CompiledInstruction) -> Self { + Self { + program_id_index: value.program_id_index as u8, + accounts: value.accounts, + data: value.data, + } + } +} From ef3e98d341cef38b0d111ba11f667c5a3bff4034 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Tue, 29 Sep 2020 16:42:46 -0600 Subject: [PATCH 3/6] Reorganize proto --- storage-bigtable/build-proto/src/main.rs | 4 ++-- .../solana.bigtable.confirmed_block.rs} | 0 storage-bigtable/src/{proto => }/confirmed_block.proto | 4 ++-- storage-bigtable/src/lib.rs | 8 ++++---- storage-bigtable/src/{proto/mod.rs => utils.rs} | 7 ++++++- 5 files changed, 14 insertions(+), 9 deletions(-) rename storage-bigtable/{src/proto/generated.rs => proto/solana.bigtable.confirmed_block.rs} (100%) rename storage-bigtable/src/{proto => }/confirmed_block.proto (97%) rename storage-bigtable/src/{proto/mod.rs => utils.rs} (98%) diff --git a/storage-bigtable/build-proto/src/main.rs b/storage-bigtable/build-proto/src/main.rs index 3cb4a42c9cf6c8..1afc36df440503 100644 --- a/storage-bigtable/build-proto/src/main.rs +++ b/storage-bigtable/build-proto/src/main.rs @@ -17,8 +17,8 @@ fn main() -> Result<(), std::io::Error> { &[googleapis], )?; - let out_dir = manifest_dir.join("../src/proto"); - let proto_files = manifest_dir.join("../src/proto"); + let out_dir = manifest_dir.join("../proto"); + let proto_files = manifest_dir.join("../src"); println!("Protobuf directory: {}", proto_files.display()); println!("output directory: {}", out_dir.display()); diff --git a/storage-bigtable/src/proto/generated.rs b/storage-bigtable/proto/solana.bigtable.confirmed_block.rs similarity index 100% rename from storage-bigtable/src/proto/generated.rs rename to storage-bigtable/proto/solana.bigtable.confirmed_block.rs diff --git a/storage-bigtable/src/proto/confirmed_block.proto b/storage-bigtable/src/confirmed_block.proto similarity index 97% rename from storage-bigtable/src/proto/confirmed_block.proto rename to storage-bigtable/src/confirmed_block.proto index e3b085627925e0..762dd1faf4b825 100644 --- a/storage-bigtable/src/proto/confirmed_block.proto +++ b/storage-bigtable/src/confirmed_block.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package Generated; +package solana.bigtable.ConfirmedBlock; message ConfirmedBlock { string previous_blockhash = 1; @@ -64,4 +64,4 @@ message Reward { message UnixTimestamp { int64 timestamp = 1; -} \ No newline at end of file +} diff --git a/storage-bigtable/src/lib.rs b/storage-bigtable/src/lib.rs index 26bef199a89ee1..2e496bd4ee17a9 100644 --- a/storage-bigtable/src/lib.rs +++ b/storage-bigtable/src/lib.rs @@ -20,9 +20,9 @@ extern crate serde_derive; mod access_token; mod bigtable; mod compression; -mod proto; mod root_ca_certificate; -use proto::generated as prost; +mod utils; +use utils::generated; #[derive(Debug, Error)] pub enum Error { @@ -277,7 +277,7 @@ impl LedgerStorage { pub async fn get_confirmed_block(&self, slot: Slot) -> Result { let mut bigtable = self.connection.client(); let block_cell_data = bigtable - .get_bincode_or_protobuf_cell::( + .get_bincode_or_protobuf_cell::( "blocks", slot_to_key(slot), ) @@ -516,7 +516,7 @@ impl LedgerStorage { let blocks_cells = [(slot_to_key(slot), confirmed_block.into())]; bytes_written += self .connection - .put_protobuf_cells_with_retry::("blocks", &blocks_cells) + .put_protobuf_cells_with_retry::("blocks", &blocks_cells) .await?; info!( "uploaded block for slot {}: {} transactions, {} bytes", diff --git a/storage-bigtable/src/proto/mod.rs b/storage-bigtable/src/utils.rs similarity index 98% rename from storage-bigtable/src/proto/mod.rs rename to storage-bigtable/src/utils.rs index 2fc9b14374af6e..cafe71552b6b08 100644 --- a/storage-bigtable/src/proto/mod.rs +++ b/storage-bigtable/src/utils.rs @@ -11,7 +11,12 @@ use solana_transaction_status::{ }; use std::convert::{TryFrom, TryInto}; -pub mod generated; +pub mod generated { + include!(concat!( + env!("CARGO_MANIFEST_DIR"), + concat!("/proto/solana.bigtable.confirmed_block.rs") + )); +} impl From for generated::Reward { fn from(reward: Reward) -> Self { From 364954b620ebfe96bf682255c44b187f06550beb Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Tue, 29 Sep 2020 20:42:12 -0600 Subject: [PATCH 4/6] Clean up use statements --- storage-bigtable/src/bigtable.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/storage-bigtable/src/bigtable.rs b/storage-bigtable/src/bigtable.rs index d7132096e274ce..15688516918921 100644 --- a/storage-bigtable/src/bigtable.rs +++ b/storage-bigtable/src/bigtable.rs @@ -1,8 +1,10 @@ // Primitives for reading/writing BigTable tables -use crate::access_token::{AccessToken, Scope}; -use crate::compression::{compress_best, decompress}; -use crate::root_ca_certificate; +use crate::{ + access_token::{AccessToken, Scope}, + compression::{compress_best, decompress}, + root_ca_certificate, +}; use log::*; use thiserror::Error; use tonic::{metadata::MetadataValue, transport::ClientTlsConfig, Request}; From 0625a9e5e9fdc42dc3bb005ec5112df00528257c Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Tue, 29 Sep 2020 22:41:16 -0600 Subject: [PATCH 5/6] Split out function for unit testing --- storage-bigtable/src/bigtable.rs | 144 ++++++++++++++++++++++++++++--- storage-bigtable/src/lib.rs | 2 +- transaction-status/src/lib.rs | 4 +- 3 files changed, 135 insertions(+), 15 deletions(-) diff --git a/storage-bigtable/src/bigtable.rs b/storage-bigtable/src/bigtable.rs index 15688516918921..3c5c7587188261 100644 --- a/storage-bigtable/src/bigtable.rs +++ b/storage-bigtable/src/bigtable.rs @@ -510,7 +510,7 @@ impl BigTable { deserialize_bincode_cell_data(&row_data, table, key.to_string()) } - pub async fn get_bincode_or_protobuf_cell( + pub async fn get_protobuf_or_bincode_cell( &mut self, table: &str, key: RowKey, @@ -520,17 +520,7 @@ impl BigTable { P: prost::Message + Default, { let row_data = self.get_single_row_data(table, key.clone()).await?; - match deserialize_protobuf_cell_data(&row_data, table, key.to_string()) { - Ok(result) => return Ok(CellData::Protobuf(result)), - Err(err) => match err { - Error::ObjectNotFound(_) => {} - _ => return Err(err), - }, - } - match deserialize_bincode_cell_data(&row_data, table, key.to_string()) { - Ok(result) => Ok(CellData::Bincode(result)), - Err(err) => Err(err), - } + deserialize_protobuf_or_bincode_cell_data(&row_data, table, key) } pub async fn put_bincode_cells( @@ -576,6 +566,28 @@ impl BigTable { } } +pub(crate) fn deserialize_protobuf_or_bincode_cell_data( + row_data: RowDataSlice, + table: &str, + key: RowKey, +) -> Result> +where + B: serde::de::DeserializeOwned, + P: prost::Message + Default, +{ + match deserialize_protobuf_cell_data(row_data, table, key.to_string()) { + Ok(result) => return Ok(CellData::Protobuf(result)), + Err(err) => match err { + Error::ObjectNotFound(_) => {} + _ => return Err(err), + }, + } + match deserialize_bincode_cell_data(row_data, table, key) { + Ok(result) => Ok(CellData::Bincode(result)), + Err(err) => Err(err), + } +} + pub(crate) fn deserialize_protobuf_cell_data( row_data: RowDataSlice, table: &str, @@ -617,3 +629,111 @@ where Error::ObjectCorrupt(format!("{}/{}", table, key)) }) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::{utils::generated, StoredConfirmedBlock}; + use prost::Message; + use solana_sdk::{hash::Hash, pubkey::Pubkey, signature::Keypair, system_transaction}; + use solana_transaction_status::{ + ConfirmedBlock, TransactionStatusMeta, TransactionWithStatusMeta, + }; + use std::convert::TryInto; + + #[test] + fn test_deserialize_protobuf_or_bincode_cell_data() { + let from = Keypair::new(); + let recipient = Pubkey::new_rand(); + let transaction = system_transaction::transfer(&from, &recipient, 42, Hash::default()); + let with_meta = TransactionWithStatusMeta { + transaction, + meta: Some(TransactionStatusMeta { + status: Ok(()), + fee: 1, + pre_balances: vec![43, 0, 1], + post_balances: vec![0, 42, 1], + inner_instructions: Some(vec![]), + }), + }; + let block = ConfirmedBlock { + transactions: vec![with_meta], + parent_slot: 1, + blockhash: Hash::default().to_string(), + previous_blockhash: Hash::default().to_string(), + rewards: vec![], + block_time: Some(1_234_567_890), + }; + let bincode_block = compress_best( + &bincode::serialize::(&block.clone().into()).unwrap(), + ) + .unwrap(); + + let protobuf_block = generated::ConfirmedBlock::from(block.clone()); + let mut buf = Vec::with_capacity(protobuf_block.encoded_len()); + protobuf_block.encode(&mut buf).unwrap(); + let protobuf_block = compress_best(&buf).unwrap(); + + let deserialized = deserialize_protobuf_or_bincode_cell_data::< + StoredConfirmedBlock, + generated::ConfirmedBlock, + >( + &[("proto".to_string(), protobuf_block.clone())], + "", + "".to_string(), + ) + .unwrap(); + if let CellData::Protobuf(protobuf_block) = deserialized { + assert_eq!(block, protobuf_block.try_into().unwrap()); + } else { + panic!("deserialization should produce CellData::Protobuf"); + } + + let deserialized = deserialize_protobuf_or_bincode_cell_data::< + StoredConfirmedBlock, + generated::ConfirmedBlock, + >( + &[("bin".to_string(), bincode_block.clone())], + "", + "".to_string(), + ) + .unwrap(); + if let CellData::Bincode(bincode_block) = deserialized { + let mut block = block; + if let Some(meta) = &mut block.transactions[0].meta { + meta.inner_instructions = None; // Legacy bincode implementation does not suport inner_instructions + } + assert_eq!(block, bincode_block.into()); + } else { + panic!("deserialization should produce CellData::Bincode"); + } + + let result = deserialize_protobuf_or_bincode_cell_data::< + StoredConfirmedBlock, + generated::ConfirmedBlock, + >(&[("proto".to_string(), bincode_block)], "", "".to_string()); + assert!(result.is_err()); + + let result = deserialize_protobuf_or_bincode_cell_data::< + StoredConfirmedBlock, + generated::ConfirmedBlock, + >( + &[("proto".to_string(), vec![1, 2, 3, 4])], + "", + "".to_string(), + ); + assert!(result.is_err()); + + let result = deserialize_protobuf_or_bincode_cell_data::< + StoredConfirmedBlock, + generated::ConfirmedBlock, + >(&[("bin".to_string(), protobuf_block)], "", "".to_string()); + assert!(result.is_err()); + + let result = deserialize_protobuf_or_bincode_cell_data::< + StoredConfirmedBlock, + generated::ConfirmedBlock, + >(&[("bin".to_string(), vec![1, 2, 3, 4])], "", "".to_string()); + assert!(result.is_err()); + } +} diff --git a/storage-bigtable/src/lib.rs b/storage-bigtable/src/lib.rs index 2e496bd4ee17a9..93b1db7556fae0 100644 --- a/storage-bigtable/src/lib.rs +++ b/storage-bigtable/src/lib.rs @@ -277,7 +277,7 @@ impl LedgerStorage { pub async fn get_confirmed_block(&self, slot: Slot) -> Result { let mut bigtable = self.connection.client(); let block_cell_data = bigtable - .get_bincode_or_protobuf_cell::( + .get_protobuf_or_bincode_cell::( "blocks", slot_to_key(slot), ) diff --git a/transaction-status/src/lib.rs b/transaction-status/src/lib.rs index e7ee0c10bbdb15..95ae4b638e4835 100644 --- a/transaction-status/src/lib.rs +++ b/transaction-status/src/lib.rs @@ -225,7 +225,7 @@ pub struct ConfirmedTransactionStatusWithSignature { pub memo: Option, } -#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct Reward { pub pubkey: String, pub lamports: i64, @@ -233,7 +233,7 @@ pub struct Reward { pub type Rewards = Vec; -#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ConfirmedBlock { pub previous_blockhash: String, From 84d9e57d1778caea6e4931e017d614f5b3160e0c Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Wed, 30 Sep 2020 10:29:25 -0600 Subject: [PATCH 6/6] s/utils/convert --- storage-bigtable/src/bigtable.rs | 2 +- storage-bigtable/src/{utils.rs => convert.rs} | 0 storage-bigtable/src/lib.rs | 5 +++-- 3 files changed, 4 insertions(+), 3 deletions(-) rename storage-bigtable/src/{utils.rs => convert.rs} (100%) diff --git a/storage-bigtable/src/bigtable.rs b/storage-bigtable/src/bigtable.rs index 3c5c7587188261..0cfbd16f43e94d 100644 --- a/storage-bigtable/src/bigtable.rs +++ b/storage-bigtable/src/bigtable.rs @@ -633,7 +633,7 @@ where #[cfg(test)] mod tests { use super::*; - use crate::{utils::generated, StoredConfirmedBlock}; + use crate::{convert::generated, StoredConfirmedBlock}; use prost::Message; use solana_sdk::{hash::Hash, pubkey::Pubkey, signature::Keypair, system_transaction}; use solana_transaction_status::{ diff --git a/storage-bigtable/src/utils.rs b/storage-bigtable/src/convert.rs similarity index 100% rename from storage-bigtable/src/utils.rs rename to storage-bigtable/src/convert.rs diff --git a/storage-bigtable/src/lib.rs b/storage-bigtable/src/lib.rs index 93b1db7556fae0..7c8761be3d945c 100644 --- a/storage-bigtable/src/lib.rs +++ b/storage-bigtable/src/lib.rs @@ -20,9 +20,10 @@ extern crate serde_derive; mod access_token; mod bigtable; mod compression; +mod convert; mod root_ca_certificate; -mod utils; -use utils::generated; + +use convert::generated; #[derive(Debug, Error)] pub enum Error {