From df28c0d57a71eaa840c57f387bbc580d24a3cc5d Mon Sep 17 00:00:00 2001 From: Yueh-Hsuan Chiang Date: Sat, 12 Feb 2022 18:27:50 -0800 Subject: [PATCH] (LedgerStore/FIFO) Refactor FIFO options and sanity check. --- core/tests/ledger_cleanup.rs | 8 ++- ledger/src/blockstore_db.rs | 113 +++++++++++++++++++---------------- 2 files changed, 66 insertions(+), 55 deletions(-) diff --git a/core/tests/ledger_cleanup.rs b/core/tests/ledger_cleanup.rs index 904a9fc3777214..012b76058ed313 100644 --- a/core/tests/ledger_cleanup.rs +++ b/core/tests/ledger_cleanup.rs @@ -9,7 +9,7 @@ mod tests { solana_core::ledger_cleanup_service::LedgerCleanupService, solana_ledger::{ blockstore::{make_many_slot_shreds, Blockstore}, - blockstore_db::{BlockstoreOptions, ShredStorageType}, + blockstore_db::{BlockstoreOptions, BlockstoreRocksFifoOptions, ShredStorageType}, get_tmp_ledger_path, }, solana_measure::measure::Measure, @@ -309,8 +309,10 @@ mod tests { &ledger_path, if config.fifo_compaction { BlockstoreOptions { - shred_storage_type: ShredStorageType::RocksFifo, - shred_data_cf_size: config.shred_data_cf_size, + shred_storage_type: ShredStorageType::RocksFifo(BlockstoreRocksFifoOptions { + shred_data_cf_size: config.shred_data_cf_size, + ..BlockstoreRocksFifoOptions::default() + }), ..BlockstoreOptions::default() } } else { diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 1b6d2a9ed39f4e..44f721a5158e9d 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -309,6 +309,12 @@ impl Rocks { let oldest_slot = OldestSlot::default(); // Get column family descriptors and names + let (cf_descriptor_shred_data, cf_descriptor_shred_code) = + new_cf_descriptor_pair_shreds::( + &options.shred_storage_type, + &access_type, + &oldest_slot, + ); let cfs = vec![ new_cf_descriptor::(&access_type, &oldest_slot), new_cf_descriptor::(&access_type, &oldest_slot), @@ -318,18 +324,8 @@ impl Rocks { new_cf_descriptor::(&access_type, &oldest_slot), new_cf_descriptor::(&access_type, &oldest_slot), new_cf_descriptor::(&access_type, &oldest_slot), - new_cf_descriptor_shreds::( - &options.shred_storage_type, - &access_type, - &oldest_slot, - &options.shred_data_cf_size, - ), - new_cf_descriptor_shreds::( - &options.shred_storage_type, - &access_type, - &oldest_slot, - &options.shred_code_cf_size, - ), + cf_descriptor_shred_data, + cf_descriptor_shred_code, new_cf_descriptor::(&access_type, &oldest_slot), new_cf_descriptor::(&access_type, &oldest_slot), new_cf_descriptor::(&access_type, &oldest_slot), @@ -981,7 +977,7 @@ pub enum ShredStorageType { // (Experimental) Stores shreds under RocksDB's FIFO compaction which // allows ledger store to reclaim storage more efficiently with // lower I/O overhead. - RocksFifo, + RocksFifo(BlockstoreRocksFifoOptions), } pub struct BlockstoreOptions { @@ -993,37 +989,47 @@ pub struct BlockstoreOptions { pub enforce_ulimit_nofile: bool, // Determine how to store both data and coding shreds. Default: RocksLevel. pub shred_storage_type: ShredStorageType, +} + +impl Default for BlockstoreOptions { + /// The default options are the values used by [`Blockstore::open`]. + fn default() -> Self { + Self { + access_type: AccessType::PrimaryOnly, + recovery_mode: None, + enforce_ulimit_nofile: true, + shred_storage_type: ShredStorageType::RocksLevel, + } + } +} + +pub struct BlockstoreRocksFifoOptions { // The maximum storage size for storing data shreds in column family // [`cf::DataShred`]. Typically, data shreds contribute around 25% of the // ledger store storage size if the RPC service is enabled, or 50% if RPC // service is not enabled. // - // Currently, this setting is only used when shred_storage_type is set to - // [`ShredStorageType::RocksFifo`]. + // Note that this number must be greater than FIFO_WRITE_BUFFER_SIZE + // otherwise we won't be able to write any file. If not, the blockstore + // will panic. pub shred_data_cf_size: u64, // The maximum storage size for storing coding shreds in column family // [`cf::CodeShred`]. Typically, coding shreds contribute around 20% of the // ledger store storage size if the RPC service is enabled, or 40% if RPC // service is not enabled. // - // Currently, this setting is only used when shred_storage_type is set to - // [`ShredStorageType::RocksFifo`]. + // Note that this number must be greater than FIFO_WRITE_BUFFER_SIZE + // otherwise we won't be able to write any file. If not, the blockstore + // will panic. pub shred_code_cf_size: u64, } -impl Default for BlockstoreOptions { - /// The default options are the values used by [`Blockstore::open`]. +impl Default for BlockstoreRocksFifoOptions { fn default() -> Self { Self { - access_type: AccessType::PrimaryOnly, - recovery_mode: None, - enforce_ulimit_nofile: true, - shred_storage_type: ShredStorageType::RocksLevel, - // Maximum size of cf::DataShred. Used when `shred_storage_type` - // is set to ShredStorageType::RocksFifo. + // Maximum size of cf::ShredData. shred_data_cf_size: DEFAULT_FIFO_COMPACTION_DATA_CF_SIZE, - // Maximum size of cf::CodeShred. Used when `shred_storage_type` - // is set to ShredStorageType::RocksFifo. + // Maximum size of cf::ShredCode. shred_code_cf_size: DEFAULT_FIFO_COMPACTION_CODING_CF_SIZE, } } @@ -1415,39 +1421,42 @@ fn get_cf_options( options } -/// Constructs and returns a ColumnFamilyDescriptor based on the -/// specified ShredStorageType. -fn new_cf_descriptor_shreds( - storage_type: &ShredStorageType, +/// Creates and returns the column family descriptors for both data shreds and +/// coding shreds column families. +/// +/// @return a pair of ColumnFamilyDescriptor where the first / second elements +/// are associated to the first / second template class respectively. +fn new_cf_descriptor_pair_shreds< + D: 'static + Column + ColumnName, // Column Family for Data Shred + C: 'static + Column + ColumnName, // Column Family for Coding Shred +>( + shred_storage_type: &ShredStorageType, access_type: &AccessType, oldest_slot: &OldestSlot, - max_cf_size: &u64, -) -> ColumnFamilyDescriptor { - match storage_type { - ShredStorageType::RocksLevel => new_cf_descriptor::(access_type, oldest_slot), - ShredStorageType::RocksFifo => { - if *max_cf_size > FIFO_WRITE_BUFFER_SIZE { - new_cf_descriptor_fifo::(max_cf_size) - } else { - warn!( - "{} cf_size must be greater than {} when using ShredStorageType::RocksFifo.", - C::NAME, - FIFO_WRITE_BUFFER_SIZE - ); - warn!( - "Fall back to ShredStorageType::RocksLevel for cf::{}.", - C::NAME - ); - new_cf_descriptor::(access_type, oldest_slot) - } - } +) -> (ColumnFamilyDescriptor, ColumnFamilyDescriptor) { + match shred_storage_type { + ShredStorageType::RocksLevel => ( + new_cf_descriptor::(access_type, oldest_slot), + new_cf_descriptor::(access_type, oldest_slot), + ), + ShredStorageType::RocksFifo(fifo_options) => ( + new_cf_descriptor_fifo::(&fifo_options.shred_data_cf_size), + new_cf_descriptor_fifo::(&fifo_options.shred_code_cf_size), + ), } } fn new_cf_descriptor_fifo( max_cf_size: &u64, ) -> ColumnFamilyDescriptor { - ColumnFamilyDescriptor::new(C::NAME, get_cf_options_fifo::(max_cf_size)) + if *max_cf_size > FIFO_WRITE_BUFFER_SIZE { + ColumnFamilyDescriptor::new(C::NAME, get_cf_options_fifo::(max_cf_size)) + } else { + panic!( + "{} cf_size must be greater than write buffer size {} when using ShredStorageType::RocksFifo.", + C::NAME, FIFO_WRITE_BUFFER_SIZE + ); + } } /// Returns the RocksDB Column Family Options which use FIFO Compaction.