From 7f51b96c1ca2bab85eadb28fe9d458d05672690e Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Thu, 23 Nov 2023 22:11:52 -0500 Subject: [PATCH] remove plerkle from program_transformers --- Cargo.lock | 4 +- Cargo.toml | 2 +- nft_ingester/src/account_updates.rs | 4 +- nft_ingester/src/plerkle.rs | 88 +++++++++++++++---- nft_ingester/src/transaction_notifications.rs | 25 +++++- program_transformers/Cargo.toml | 2 +- .../src/bubblegum/decompress.rs | 16 +--- program_transformers/src/bubblegum/mint_v1.rs | 4 +- program_transformers/src/lib.rs | 60 +++++++------ 9 files changed, 138 insertions(+), 67 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5cf0c6eee..60d423344 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -985,6 +985,7 @@ checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" [[package]] name = "blockbuster" version = "0.9.0-beta.1" +source = "git+https://github.com/rpcpool/blockbuster.git?tag=blockbuster-rm-plerkle-v0.9.0-beta.1#acbb64c569c700cccedad3c9289e1000deb0ab01" dependencies = [ "anchor-lang", "async-trait", @@ -999,6 +1000,7 @@ dependencies = [ "mpl-token-metadata", "plerkle_serialization", "solana-sdk", + "solana-transaction-status", "spl-account-compression", "spl-noop", "spl-token 4.0.0", @@ -4068,10 +4070,10 @@ dependencies = [ "futures", "mpl-bubblegum", "num-traits", - "plerkle_serialization", "sea-orm", "serde_json", "solana-sdk", + "solana-transaction-status", "spl-account-compression", "spl-token 4.0.0", "sqlx", diff --git a/Cargo.toml b/Cargo.toml index 1f93607d7..5c31592f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -94,7 +94,7 @@ txn_forwarder = { path = "tools/txn_forwarder" } url = "2.3.1" [patch.crates-io] -blockbuster = { path = "/home/kirill/projects/blockbuster/blockbuster" } +blockbuster = { git = "https://github.com/rpcpool/blockbuster.git", tag = "blockbuster-rm-plerkle-v0.9.0-beta.1" } [workspace.lints.clippy] clone_on_ref_ptr = "deny" diff --git a/nft_ingester/src/account_updates.rs b/nft_ingester/src/account_updates.rs index 02cf4bdb6..c49b6cd5e 100644 --- a/nft_ingester/src/account_updates.rs +++ b/nft_ingester/src/account_updates.rs @@ -2,7 +2,7 @@ use { crate::{ metric, metrics::capture_result, - plerkle::{parse_pubkey, parse_vector}, + plerkle::{parse_pubkey, parse_slice}, tasks::{create_download_metadata_notifier, TaskData}, }, cadence_macros::{is_global_default_set, statsd_count, statsd_time}, @@ -131,7 +131,7 @@ async fn handle_account_update<'a>( slot: account_update.slot(), pubkey: &parse_pubkey(account_update.pubkey())?, owner: &parse_pubkey(account_update.owner())?, - data: parse_vector(account_update.data())?, + data: parse_slice(account_update.data())?, }) .await } diff --git a/nft_ingester/src/plerkle.rs b/nft_ingester/src/plerkle.rs index 530a8bc98..c206583e6 100644 --- a/nft_ingester/src/plerkle.rs +++ b/nft_ingester/src/plerkle.rs @@ -1,26 +1,78 @@ use { - flatbuffers::Vector, - plerkle_serialization::Pubkey as FBPubkey, + flatbuffers::{ForwardsUOffset, Vector}, + plerkle_serialization::{ + CompiledInnerInstructions as FBCompiledInnerInstructions, + CompiledInstruction as FBCompiledInstruction, Pubkey as FBPubkey, + }, program_transformers::error::{ProgramTransformerError, ProgramTransformerResult}, - solana_sdk::pubkey::Pubkey, + solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature}, + solana_transaction_status::{InnerInstruction, InnerInstructions}, }; +fn deser_err() -> ProgramTransformerError { + ProgramTransformerError::DeserializationError("Could not deserialize data".to_owned()) +} + pub fn parse_pubkey(pubkey: Option<&FBPubkey>) -> ProgramTransformerResult { - Ok(Pubkey::try_from( - pubkey - .ok_or_else(|| { - ProgramTransformerError::DeserializationError( - "Could not deserialize data".to_owned(), - ) - })? - .0 - .as_slice(), - ) - .expect("valid key from FlatBuffer")) + Ok(Pubkey::try_from(pubkey.ok_or_else(deser_err)?.0.as_slice()) + .expect("valid key from FlatBuffer")) +} + +pub fn parse_slice(data: Option>) -> ProgramTransformerResult<&[u8]> { + data.map(|data| data.bytes()).ok_or_else(deser_err) +} + +pub fn parse_signature(data: Option<&str>) -> ProgramTransformerResult { + data.ok_or_else(deser_err)? + .parse() + .map_err(|_error| deser_err()) +} + +pub fn parse_account_keys( + keys: Option>, +) -> ProgramTransformerResult> { + let mut account_keys = vec![]; + for key in keys.ok_or_else(deser_err)? { + account_keys.push(Pubkey::try_from(key.0.as_slice()).expect("valid key from FlatBuffer")); + } + Ok(account_keys) +} + +pub fn parse_message_instructions( + vec_cix: Option>>, +) -> ProgramTransformerResult> { + let mut message_instructions = vec![]; + for cix in vec_cix.ok_or_else(deser_err)? { + message_instructions.push(CompiledInstruction { + program_id_index: cix.program_id_index(), + accounts: cix.accounts().ok_or_else(deser_err)?.bytes().to_vec(), + data: cix.data().ok_or_else(deser_err)?.bytes().to_vec(), + }) + } + Ok(message_instructions) } -pub fn parse_vector(data: Option>) -> ProgramTransformerResult<&[u8]> { - data.map(|data| data.bytes()).ok_or_else(|| { - ProgramTransformerError::DeserializationError("Could not deserialize data".to_owned()) - }) +pub fn parse_meta_inner_instructions( + vec_ixs: Option>>, +) -> ProgramTransformerResult> { + let mut meta_inner_instructions = vec![]; + for ixs in vec_ixs.ok_or_else(deser_err)? { + let mut instructions = vec![]; + for ix in ixs.instructions().ok_or_else(deser_err)? { + let cix = ix.compiled_instruction().ok_or_else(deser_err)?; + instructions.push(InnerInstruction { + instruction: CompiledInstruction { + program_id_index: cix.program_id_index(), + accounts: cix.accounts().ok_or_else(deser_err)?.bytes().to_vec(), + data: cix.data().ok_or_else(deser_err)?.bytes().to_vec(), + }, + stack_height: Some(ix.stack_height() as u32), + }); + } + meta_inner_instructions.push(InnerInstructions { + index: ixs.index(), + instructions, + }) + } + Ok(meta_inner_instructions) } diff --git a/nft_ingester/src/transaction_notifications.rs b/nft_ingester/src/transaction_notifications.rs index e3932eaf6..0859a658c 100644 --- a/nft_ingester/src/transaction_notifications.rs +++ b/nft_ingester/src/transaction_notifications.rs @@ -2,6 +2,10 @@ use { crate::{ metric, metrics::capture_result, + plerkle::{ + parse_account_keys, parse_message_instructions, parse_meta_inner_instructions, + parse_signature, + }, tasks::{create_download_metadata_notifier, TaskData}, }, cadence_macros::{is_global_default_set, statsd_count, statsd_time}, @@ -9,7 +13,7 @@ use { log::{debug, error}, plerkle_messenger::{ConsumptionType, Messenger, MessengerConfig, RecvData}, plerkle_serialization::root_as_transaction_info, - program_transformers::ProgramTransformer, + program_transformers::{error::ProgramTransformerResult, ProgramTransformer, TransactionInfo}, sqlx::{Pool, Postgres}, std::sync::Arc, tokio::{ @@ -101,7 +105,7 @@ async fn handle_transaction( } let begin = Instant::now(); - let res = manager.handle_transaction(&tx).await; + let res = handle_transaction_update(manager, tx).await; let should_ack = capture_result( id.clone(), stream_key, @@ -118,3 +122,20 @@ async fn handle_transaction( } ret_id } + +async fn handle_transaction_update<'a>( + manager: Arc, + tx: plerkle_serialization::TransactionInfo<'_>, +) -> ProgramTransformerResult<()> { + manager + .handle_transaction(&TransactionInfo { + slot: tx.slot(), + signature: &parse_signature(tx.signature())?, + account_keys: parse_account_keys(tx.account_keys())?, + message_instructions: &parse_message_instructions(tx.outer_instructions())?, + meta_inner_instructions: &parse_meta_inner_instructions( + tx.compiled_inner_instructions(), + )?, + }) + .await +} diff --git a/program_transformers/Cargo.toml b/program_transformers/Cargo.toml index 330e85b43..31d35871f 100644 --- a/program_transformers/Cargo.toml +++ b/program_transformers/Cargo.toml @@ -12,10 +12,10 @@ digital_asset_types = { workspace = true } futures = { workspace = true } mpl-bubblegum = { workspace = true } num-traits = { workspace = true } -plerkle_serialization = { workspace = true } sea-orm = { workspace = true, features = [] } serde_json = { workspace = true } solana-sdk = { workspace = true } +solana-transaction-status = { workspace = true } spl-account-compression = { workspace = true, features = ["no-entrypoint"] } spl-token = { workspace = true, features = ["no-entrypoint"] } sqlx = { workspace = true, features = [] } diff --git a/program_transformers/src/bubblegum/decompress.rs b/program_transformers/src/bubblegum/decompress.rs index 9db67e8b8..208e9aabc 100644 --- a/program_transformers/src/bubblegum/decompress.rs +++ b/program_transformers/src/bubblegum/decompress.rs @@ -17,18 +17,10 @@ pub async fn decompress<'c, T>( where T: ConnectionTrait + TransactionTrait, { - let id_bytes = bundle.keys.get(3).unwrap().0.as_slice(); + let id_bytes = bundle.keys.get(3).unwrap().to_bytes().to_vec(); // Partial update of asset table with just leaf. - upsert_asset_with_leaf_info_for_decompression(txn, id_bytes.to_vec()).await?; - upsert_asset_with_compression_info( - txn, - id_bytes.to_vec(), - false, - false, - 1, - Some(id_bytes.to_vec()), - true, - ) - .await + upsert_asset_with_leaf_info_for_decompression(txn, id_bytes.clone()).await?; + upsert_asset_with_compression_info(txn, id_bytes.clone(), false, false, 1, Some(id_bytes), true) + .await } diff --git a/program_transformers/src/bubblegum/mint_v1.rs b/program_transformers/src/bubblegum/mint_v1.rs index 73e1b7d16..b8b13016f 100644 --- a/program_transformers/src/bubblegum/mint_v1.rs +++ b/program_transformers/src/bubblegum/mint_v1.rs @@ -130,7 +130,7 @@ where } else { Some(delegate.to_bytes().to_vec()) }; - let tree_id = bundle.keys.get(3).unwrap().0.to_vec(); + let tree_id = bundle.keys.get(3).unwrap().to_bytes().to_vec(); // ActiveValue::Set initial mint info. let asset_model = asset::ActiveModel { @@ -311,7 +311,7 @@ where // Insert into `asset_authority` table. let model = asset_authority::ActiveModel { asset_id: ActiveValue::Set(id_bytes.to_vec()), - authority: ActiveValue::Set(bundle.keys.get(0).unwrap().0.to_vec()), //TODO - we need to rem,ove the optional bubblegum signer logic + authority: ActiveValue::Set(bundle.keys.get(0).unwrap().to_bytes().to_vec()), //TODO - we need to rem,ove the optional bubblegum signer logic seq: ActiveValue::Set(seq as i64), slot_updated: ActiveValue::Set(slot_i), ..Default::default() diff --git a/program_transformers/src/lib.rs b/program_transformers/src/lib.rs index bf4ac0623..5cf1c0ced 100644 --- a/program_transformers/src/lib.rs +++ b/program_transformers/src/lib.rs @@ -14,9 +14,9 @@ use { }, }, futures::future::BoxFuture, - plerkle_serialization::{Pubkey as FBPubkey, TransactionInfo}, sea_orm::{DatabaseConnection, SqlxPostgresConnector}, - solana_sdk::pubkey::Pubkey, + solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey, signature::Signature}, + solana_transaction_status::InnerInstructions, sqlx::PgPool, std::collections::{HashMap, HashSet, VecDeque}, tracing::{debug, error, info}, @@ -35,6 +35,15 @@ pub struct AccountInfo<'a> { pub data: &'a [u8], } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TransactionInfo<'a> { + pub slot: u64, + pub signature: &'a Signature, + pub account_keys: Vec, + pub message_instructions: &'a [CompiledInstruction], + pub meta_inner_instructions: &'a [InnerInstructions], +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct DownloadMetadataInfo { asset_data_id: Vec, @@ -96,12 +105,16 @@ impl ProgramTransformer { } } - pub fn break_transaction<'i>( + pub fn break_transaction<'a>( &self, - tx: &'i TransactionInfo<'i>, - ) -> VecDeque<(IxPair<'i>, Option>>)> { - let ref_set: HashSet<&[u8]> = self.key_set.iter().map(|k| k.as_ref()).collect(); - order_instructions(ref_set, tx) + tx_info: &'a TransactionInfo<'_>, + ) -> VecDeque<(IxPair<'a>, Option>>)> { + order_instructions( + &self.key_set, + &tx_info.account_keys, + tx_info.message_instructions, + tx_info.meta_inner_instructions, + ) } #[allow(clippy::borrowed_box)] @@ -109,33 +122,25 @@ impl ProgramTransformer { self.parsers.get(key) } - pub async fn handle_transaction<'a>( + pub async fn handle_transaction( &self, - tx: &'a TransactionInfo<'a>, + tx_info: &TransactionInfo<'_>, ) -> ProgramTransformerResult<()> { - let sig: Option<&str> = tx.signature(); - info!("Handling Transaction: {:?}", sig); - let instructions = self.break_transaction(tx); - let accounts = tx.account_keys().unwrap_or_default(); - let slot = tx.slot(); - let txn_id = tx.signature().unwrap_or(""); - let mut keys: Vec = Vec::with_capacity(accounts.len()); - for k in accounts.into_iter() { - keys.push(*k); - } + info!("Handling Transaction: {:?}", tx_info.signature); + let instructions = self.break_transaction(tx_info); let mut not_impl = 0; let ixlen = instructions.len(); debug!("Instructions: {}", ixlen); let contains = instructions .iter() - .filter(|(ib, _inner)| ib.0 .0.as_ref() == mpl_bubblegum::ID.as_ref()); + .filter(|(ib, _inner)| ib.0 == mpl_bubblegum::ID); debug!("Instructions bgum: {}", contains.count()); for (outer_ix, inner_ix) in instructions { let (program, instruction) = outer_ix; - let ix_accounts = instruction.accounts().unwrap().iter().collect::>(); + let ix_accounts = &instruction.accounts; let ix_account_len = ix_accounts.len(); let max = ix_accounts.iter().max().copied().unwrap_or(0) as usize; - if keys.len() < max { + if tx_info.account_keys.len() < max { return Err(ProgramTransformerError::DeserializationError( "Missing Accounts in Serialized Ixn/Txn".to_string(), )); @@ -144,22 +149,21 @@ impl ProgramTransformer { ix_accounts .iter() .fold(Vec::with_capacity(ix_account_len), |mut acc, a| { - if let Some(key) = keys.get(*a as usize) { + if let Some(key) = tx_info.account_keys.get(*a as usize) { acc.push(*key); } acc }); let ix = InstructionBundle { - txn_id, + txn_id: &tx_info.signature.to_string(), program, instruction: Some(instruction), inner_ix, keys: ix_accounts.as_slice(), - slot, + slot: tx_info.slot, }; - let program_key = - Pubkey::try_from(ix.program.0.as_slice()).expect("valid key from FlatBuffer"); + let program_key = ix.program; if let Some(program) = self.match_program(&program_key) { debug!("Found a ix for program: {:?}", program.key()); let result = program.handle_instruction(&ix)?; @@ -177,7 +181,7 @@ impl ProgramTransformer { .map_err(|err| { error!( "Failed to handle bubblegum instruction for txn {:?}: {:?}", - sig, err + tx_info.signature, err ); err })?;