diff --git a/Cargo.lock b/Cargo.lock index ed7d71e484b6db..8badd70fb5acf7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4265,6 +4265,7 @@ dependencies = [ "serde_json", "solana-accountsdb-plugin-interface", "solana-logger 1.9.0", + "solana-measure", "solana-metrics", "solana-rpc", "solana-runtime", @@ -4286,6 +4287,8 @@ dependencies = [ "serde_json", "solana-accountsdb-plugin-interface", "solana-logger 1.9.0", + "solana-metrics", + "solana-sdk", "thiserror", ] diff --git a/accountsdb-plugin-manager/Cargo.toml b/accountsdb-plugin-manager/Cargo.toml index c55aade08a2170..7799c8e6bc88f5 100644 --- a/accountsdb-plugin-manager/Cargo.toml +++ b/accountsdb-plugin-manager/Cargo.toml @@ -19,6 +19,7 @@ serde_derive = "1.0.103" serde_json = "1.0.67" solana-accountsdb-plugin-interface = { path = "../accountsdb-plugin-interface", version = "=1.9.0" } solana-logger = { path = "../logger", version = "=1.9.0" } +solana-measure = { path = "../measure", version = "=1.9.0" } solana-metrics = { path = "../metrics", version = "=1.9.0" } solana-rpc = { path = "../rpc", version = "=1.9.0" } solana-runtime = { path = "../runtime", version = "=1.9.0" } diff --git a/accountsdb-plugin-manager/src/accounts_update_notifier.rs b/accountsdb-plugin-manager/src/accounts_update_notifier.rs index 1896dca013a600..dc1204703d2d43 100644 --- a/accountsdb-plugin-manager/src/accounts_update_notifier.rs +++ b/accountsdb-plugin-manager/src/accounts_update_notifier.rs @@ -5,6 +5,8 @@ use { solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{ ReplicaAccountInfo, ReplicaAccountInfoVersions, SlotStatus, }, + solana_measure::measure::Measure, + solana_metrics::*, solana_runtime::{ accounts_update_notifier_interface::AccountsUpdateNotifierInterface, append_vec::StoredAccountMeta, @@ -88,21 +90,33 @@ impl AccountsUpdateNotifierImpl { return; } for plugin in plugin_manager.plugins.iter_mut() { + let mut measure = Measure::start("accountsdb-plugin-update-account"); match plugin.update_account(ReplicaAccountInfoVersions::V0_0_1(&account), slot) { Err(err) => { error!( - "Failed to update account {:?} at slot {:?}, error: {:?}", - account.pubkey, slot, err + "Failed to update account {} at slot {}, error: {} to plugin {}", + bs58::encode(account.pubkey).into_string(), + slot, + err, + plugin.name() ) } Ok(_) => { trace!( - "Successfully updated account {:?} at slot {:?}", - account.pubkey, - slot + "Successfully updated account {} at slot {} to plugin {}", + bs58::encode(account.pubkey).into_string(), + slot, + plugin.name() ); } } + measure.stop(); + inc_new_counter_info!( + "accountsdb-plugin-update-account-ms", + measure.as_ms() as usize, + 100000, + 100000 + ); } } @@ -113,17 +127,31 @@ impl AccountsUpdateNotifierImpl { } for plugin in plugin_manager.plugins.iter_mut() { + let mut measure = Measure::start("accountsdb-plugin-update-slot"); match plugin.update_slot_status(slot, parent, slot_status.clone()) { Err(err) => { error!( - "Failed to update slot status at slot {:?}, error: {:?}", - slot, err + "Failed to update slot status at slot {}, error: {} to plugin {}", + slot, + err, + plugin.name() ) } Ok(_) => { - trace!("Successfully updated slot status at slot {:?}", slot); + trace!( + "Successfully updated slot status at slot {} to plugin {}", + slot, + plugin.name() + ); } } + measure.stop(); + inc_new_counter_info!( + "accountsdb-plugin-update-slot-ms", + measure.as_ms() as usize, + 1000, + 1000 + ); } } } diff --git a/accountsdb-plugin-postgres/Cargo.toml b/accountsdb-plugin-postgres/Cargo.toml index 36d63d9d8479f1..e9e2d86a797aa3 100644 --- a/accountsdb-plugin-postgres/Cargo.toml +++ b/accountsdb-plugin-postgres/Cargo.toml @@ -23,6 +23,8 @@ serde_derive = "1.0.103" serde_json = "1.0.67" solana-accountsdb-plugin-interface = { path = "../accountsdb-plugin-interface", version = "=1.9.0" } solana-logger = { path = "../logger", version = "=1.9.0" } +solana-metrics = { path = "../metrics", version = "=1.9.0" } +solana-sdk = { path = "../sdk", version = "=1.9.0" } thiserror = "1.0.30" [package.metadata.docs.rs] diff --git a/accountsdb-plugin-postgres/src/postgres_client.rs b/accountsdb-plugin-postgres/src/postgres_client.rs index 3cc8310eae12a1..e17834b8a8d4f5 100644 --- a/accountsdb-plugin-postgres/src/postgres_client.rs +++ b/accountsdb-plugin-postgres/src/postgres_client.rs @@ -10,6 +10,8 @@ use { solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{ AccountsDbPluginError, ReplicaAccountInfo, SlotStatus, }, + solana_metrics::datapoint_info, + solana_sdk::timing::AtomicInterval, std::{ sync::{ atomic::{AtomicBool, Ordering}, @@ -188,10 +190,10 @@ impl PostgresClient for SimplePostgresClient { account: &T, slot: u64, ) -> Result<(), AccountsDbPluginError> { - debug!( - "Updating account {:?} {:?} at slot {:?}", - account.pubkey(), - account.owner(), + trace!( + "Updating account {} with owner {} at slot {}", + bs58::encode(account.pubkey()).into_string(), + bs58::encode(account.owner()).into_string(), slot, ); @@ -215,9 +217,12 @@ impl PostgresClient for SimplePostgresClient { ); if let Err(err) = result { - return Err(AccountsDbPluginError::AccountsUpdateError { - msg: format!("Failed to persist the update of account to the PostgreSQL database. Error: {:?}", err) - }); + let msg = format!( + "Failed to persist the update of account to the PostgreSQL database. Error: {:?}", + err + ); + error!("{}", msg); + return Err(AccountsDbPluginError::AccountsUpdateError { msg }); } Ok(()) } @@ -266,9 +271,12 @@ impl PostgresClient for SimplePostgresClient { match result { Err(err) => { - return Err(AccountsDbPluginError::SlotStatusUpdateError{ - msg: format!("Failed to persist the update of slot to the PostgreSQL database. Error: {:?}", err) - }); + let msg = format!( + "Failed to persist the update of slot to the PostgreSQL database. Error: {:?}", + err + ); + error!("{:?}", msg); + return Err(AccountsDbPluginError::SlotStatusUpdateError { msg }); } Ok(rows) => { assert_eq!(1, rows, "Expected one rows to be updated a time"); @@ -340,6 +348,7 @@ pub struct ParallelPostgresClient { workers: Vec>>, exit_worker: Arc, sender: Sender, + last_report: AtomicInterval, } impl ParallelPostgresClient { @@ -365,6 +374,7 @@ impl ParallelPostgresClient { } Ok(Self { + last_report: AtomicInterval::default(), workers, exit_worker, sender, @@ -395,6 +405,12 @@ impl PostgresClient for ParallelPostgresClient { account: &T, slot: u64, ) -> Result<(), AccountsDbPluginError> { + if self.last_report.should_update(30000) { + datapoint_info!( + "postgres-plugin-stats", + ("message-queue-length", self.sender.len() as i64, i64), + ); + } if let Err(err) = self .sender .send(DbWorkItem::UpdateAccount(UpdateAccountRequest { @@ -405,7 +421,7 @@ impl PostgresClient for ParallelPostgresClient { return Err(AccountsDbPluginError::AccountsUpdateError { msg: format!( "Failed to update the account {:?}, error: {:?}", - account.pubkey(), + bs58::encode(account.pubkey()).into_string(), err ), });