From ae2bea036f663050cb3d61bd3bd50613094833ba Mon Sep 17 00:00:00 2001 From: Steven Czabaniuk Date: Fri, 10 Nov 2023 14:46:52 -0600 Subject: [PATCH 1/2] Remove key_size() method from Column trait This helper simply called std::mem::size_of(). However, all of the underlying functions that create keys manually copy fields into a byte array. The fields are copied in end-to-end whereas size_of() might include alignment bytes. That is, a (u64, u32) only has 12 bytes of "data", but it would have size 16 due to the 4 alignment padding bytes that would be added to get the u32 (size 4) aligned with the u64 (size 8). --- ledger/src/blockstore_db.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 0b2b14445539d6..376471b76afeae 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -719,10 +719,6 @@ impl Rocks { pub trait Column { type Index; - fn key_size() -> usize { - std::mem::size_of::() - } - fn key(index: Self::Index) -> Vec; fn index(key: &[u8]) -> Self::Index; // This trait method is primarily used by `Database::delete_range_cf()`, and is therefore only From 4dc7f099c5715e3a45c3403609a8c1e59b47be02 Mon Sep 17 00:00:00 2001 From: Steven Czabaniuk Date: Thu, 16 Nov 2023 23:41:21 -0600 Subject: [PATCH 2/2] Remove key_size() method from ledger-tool analyze_storage() We iterate through key-value pairs anyways, so just get the key size from there. --- ledger-tool/src/main.rs | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index a6b34d39d63e76..47cac69f7a94c3 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -743,17 +743,23 @@ fn analyze_column< db: &Database, name: &str, ) { + let mut key_len: u64 = 0; let mut key_tot: u64 = 0; let mut val_hist = histogram::Histogram::new(); let mut val_tot: u64 = 0; let mut row_hist = histogram::Histogram::new(); - let a = C::key_size() as u64; - for (_x, y) in db.iter::(blockstore_db::IteratorMode::Start).unwrap() { - let b = y.len() as u64; - key_tot += a; - val_hist.increment(b).unwrap(); - val_tot += b; - row_hist.increment(a + b).unwrap(); + for (key, val) in db.iter::(blockstore_db::IteratorMode::Start).unwrap() { + // Key length is fixed, only need to calculate it once + if key_len == 0 { + key_len = C::key(key).len() as u64; + } + let val_len = val.len() as u64; + + key_tot += key_len; + val_hist.increment(val_len).unwrap(); + val_tot += val_len; + + row_hist.increment(key_len + val_len).unwrap(); } let json_result = if val_hist.entries() > 0 { @@ -761,7 +767,7 @@ fn analyze_column< "column":name, "entries":val_hist.entries(), "key_stats":{ - "max":a, + "max":key_len, "total_bytes":key_tot, }, "val_stats":{ @@ -790,7 +796,7 @@ fn analyze_column< "column":name, "entries":val_hist.entries(), "key_stats":{ - "max":a, + "max":key_len, "total_bytes":0, }, "val_stats":{