From cbafde81c1e848584cb241d65fbc5d4ec61d2cd2 Mon Sep 17 00:00:00 2001 From: Haoran Yi Date: Wed, 23 Nov 2022 09:46:24 -0600 Subject: [PATCH 01/21] log number of open files when panic --- Cargo.lock | 16 ++++++++++ bucket_map/Cargo.toml | 1 + bucket_map/src/bucket_storage.rs | 54 +++++++++++++++++++++++++++----- programs/sbf/Cargo.lock | 22 +++++++++++++ 4 files changed, 86 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 45f74ff6242e71..55838e117ba933 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3541,6 +3541,21 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "procfs" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfb6451c91904606a1abe93e83a8ec851f45827fa84273f256ade45dc095818" +dependencies = [ + "bitflags", + "byteorder", + "chrono", + "flate2", + "hex", + "lazy_static", + "rustix", +] + [[package]] name = "proptest" version = "1.0.0" @@ -4901,6 +4916,7 @@ dependencies = [ "log", "memmap2", "modular-bitfield", + "procfs", "rand 0.7.3", "rayon", "solana-logger 1.15.0", diff --git a/bucket_map/Cargo.toml b/bucket_map/Cargo.toml index 40f91d60b8fcf0..9799821b2e3844 100644 --- a/bucket_map/Cargo.toml +++ b/bucket_map/Cargo.toml @@ -14,6 +14,7 @@ edition = "2021" log = { version = "0.4.17" } memmap2 = "0.5.3" modular-bitfield = "0.11.2" +procfs = "0.14.1" rand = "0.7.0" solana-measure = { path = "../measure", version = "=1.15.0" } solana-sdk = { path = "../sdk", version = "=1.15.0" } diff --git a/bucket_map/src/bucket_storage.rs b/bucket_map/src/bucket_storage.rs index dac4e9b11caf03..f7471459922540 100644 --- a/bucket_map/src/bucket_storage.rs +++ b/bucket_map/src/bucket_storage.rs @@ -1,6 +1,7 @@ use { crate::{bucket_stats::BucketStats, MaxSearch}, memmap2::MmapMut, + procfs::process::{Limit, Process}, rand::{thread_rng, Rng}, solana_measure::measure::Measure, std::{ @@ -262,6 +263,14 @@ impl BucketStorage { } } + fn get_number_of_open_files_and_limit() -> Option<(usize, Limit)> { + let proc = Process::myself().ok()?; + let max_open_files_limit = proc.limits().unwrap().max_open_files; + let num_open_files = proc.fd_count().unwrap(); + + Some((num_open_files, max_open_files_limit)) + } + fn new_map( drives: &[PathBuf], cell_size: usize, @@ -280,12 +289,24 @@ impl BucketStorage { .create(true) .open(file.clone()) .map_err(|e| { - panic!( - "Unable to create data file {} in current dir({:?}): {:?}", - file.display(), - std::env::current_dir(), - e - ); + if let Some((num_open_files, max_open_files_limit)) = Self::get_number_of_open_files_and_limit() { + panic!( + "Unable to create data file {} in current dir({:?}): {:?}, current number of open files: {}, max limit of open files: {:?}", + file.display(), + std::env::current_dir(), + e, + num_open_files, + max_open_files_limit, + ); + } + else { + panic!( + "Unable to create data file {} in current dir({:?}): {:?}", + file.display(), + std::env::current_dir(), + e, + ); + } }) .unwrap(); @@ -381,7 +402,7 @@ impl BucketStorage { #[cfg(test)] mod test { - use {super::*, tempfile::tempdir}; + use {super::*, procfs::process::LimitValue, tempfile::tempdir}; #[test] fn test_bucket_storage() { @@ -412,5 +433,24 @@ mod test { storage.free(ix, uid); assert!(storage.is_free(ix)); assert_eq!(storage.uid(ix), None); + + // test get_number_of_open_files_and_limit + if let Some((num_open_files, max_open_files_limit)) = + BucketStorage::get_number_of_open_files_and_limit() + { + assert!(num_open_files > 0); + match max_open_files_limit.soft_limit { + LimitValue::Unlimited => {} + LimitValue::Value(x) => assert!(x > 0), + } + + match max_open_files_limit.hard_limit { + LimitValue::Unlimited => {} + LimitValue::Value(x) => assert!(x > 0), + } + + println!("{:?}", num_open_files); + println!("{:?}", max_open_files_limit); + } } } diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 42b5e1a4767728..cf3968139feb81 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -1801,6 +1801,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "histogram" version = "0.6.9" @@ -3259,6 +3265,21 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "procfs" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfb6451c91904606a1abe93e83a8ec851f45827fa84273f256ade45dc095818" +dependencies = [ + "bitflags", + "byteorder 1.4.3", + "chrono", + "flate2", + "hex", + "lazy_static", + "rustix", +] + [[package]] name = "prost" version = "0.9.0" @@ -4336,6 +4357,7 @@ dependencies = [ "log", "memmap2", "modular-bitfield", + "procfs", "rand 0.7.3", "solana-measure", "solana-sdk 1.15.0", From d01d1166f2bb87b405cae5a5ac50bf9c48c10a21 Mon Sep 17 00:00:00 2001 From: Haoran Yi Date: Wed, 23 Nov 2022 13:58:33 -0600 Subject: [PATCH 02/21] add monitoring for open file descriptors stat --- Cargo.lock | 1 + core/Cargo.toml | 1 + core/src/system_monitor_service.rs | 43 ++++++++++++++++++++++++++ core/src/validator.rs | 3 ++ ledger-tool/src/main.rs | 1 + local-cluster/src/validator_configs.rs | 1 + programs/sbf/Cargo.lock | 1 + validator/src/cli.rs | 5 +++ validator/src/main.rs | 1 + 9 files changed, 57 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 55838e117ba933..0bd2719eca72e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5202,6 +5202,7 @@ dependencies = [ "matches", "min-max-heap", "num_enum", + "procfs", "rand 0.7.3", "rand_chacha 0.2.2", "raptorq", diff --git a/core/Cargo.toml b/core/Cargo.toml index 8829d4b9509f8b..5c42d5f615eefd 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -31,6 +31,7 @@ log = "0.4.17" lru = "0.7.7" min-max-heap = "1.3.0" num_enum = "0.5.7" +procfs = "0.14.1" rand = "0.7.0" rand_chacha = "0.2.2" rayon = "1.5.3" diff --git a/core/src/system_monitor_service.rs b/core/src/system_monitor_service.rs index 8806b62bc85e06..9911597dce9c43 100644 --- a/core/src/system_monitor_service.rs +++ b/core/src/system_monitor_service.rs @@ -7,6 +7,7 @@ use num_enum::{IntoPrimitive, TryFromPrimitive}; #[cfg(target_os = "linux")] use std::{fs::File, io::BufReader}; use { + procfs::process::{LimitValue, Process}, solana_sdk::timing::AtomicInterval, std::{ collections::HashMap, @@ -29,6 +30,7 @@ const SAMPLE_INTERVAL_OS_NETWORK_LIMITS_MS: u64 = MS_PER_H; const SAMPLE_INTERVAL_MEM_MS: u64 = 5 * MS_PER_S; const SAMPLE_INTERVAL_CPU_MS: u64 = 10 * MS_PER_S; const SAMPLE_INTERVAL_DISK_MS: u64 = 5 * MS_PER_S; +const SAMPLE_INTERVAL_OPEN_FD_MS: u64 = 30 * MS_PER_S; const SLEEP_INTERVAL: Duration = Duration::from_millis(500); #[cfg(target_os = "linux")] @@ -392,6 +394,7 @@ impl SystemMonitorService { report_os_network_stats: bool, report_os_cpu_stats: bool, report_os_disk_stats: bool, + report_os_open_fd_stats: bool, ) -> Self { info!("Starting SystemMonitorService"); let thread_hdl = Builder::new() @@ -403,6 +406,7 @@ impl SystemMonitorService { report_os_network_stats, report_os_cpu_stats, report_os_disk_stats, + report_os_open_fd_stats, ); }) .unwrap(); @@ -832,6 +836,40 @@ impl SystemMonitorService { Self::report_cpuid_values(); } + fn get_open_fd_stats() -> Option<(usize, usize, usize)> { + let proc = Process::myself().ok()?; + let curr_num_open_fd = proc.fd_count().unwrap(); + let max_open_fd_limit = proc.limits().unwrap().max_open_files; + + let max_open_fd_soft_limit = match max_open_fd_limit.soft_limit { + LimitValue::Unlimited => usize::MAX, + LimitValue::Value(x) => x as usize, + }; + let max_open_fd_hard_limit = match max_open_fd_limit.hard_limit { + LimitValue::Unlimited => usize::MAX, + LimitValue::Value(x) => x as usize, + }; + + Some(( + curr_num_open_fd, + max_open_fd_soft_limit, + max_open_fd_hard_limit, + )) + } + + fn report_open_fd_stats() { + if let Some((curr_num_open_fd, max_open_fd_soft_limit, max_open_fd_hard_limit)) = + Self::get_open_fd_stats() + { + datapoint_info!( + "open-fd-stats", + ("number_open_files", curr_num_open_fd, i64), + ("max_open_files_hard_limit", max_open_fd_hard_limit, i64), + ("max_open_files_soft_limit", max_open_fd_soft_limit, i64), + ); + } + } + #[cfg(target_os = "linux")] fn process_disk_stats(disk_stats: &mut Option) { match read_disk_stats() { @@ -973,6 +1011,7 @@ impl SystemMonitorService { report_os_network_stats: bool, report_os_cpu_stats: bool, report_os_disk_stats: bool, + report_os_open_fd_stats: bool, ) { let mut udp_stats = None; let mut disk_stats = None; @@ -981,6 +1020,7 @@ impl SystemMonitorService { let mem_timer = AtomicInterval::default(); let cpu_timer = AtomicInterval::default(); let disk_timer = AtomicInterval::default(); + let open_fd_timer = AtomicInterval::default(); loop { if exit.load(Ordering::Relaxed) { @@ -1003,6 +1043,9 @@ impl SystemMonitorService { if report_os_disk_stats && disk_timer.should_update(SAMPLE_INTERVAL_DISK_MS) { Self::process_disk_stats(&mut disk_stats); } + if report_os_open_fd_stats && open_fd_timer.should_update(SAMPLE_INTERVAL_OPEN_FD_MS) { + Self::report_open_fd_stats(); + } sleep(SLEEP_INTERVAL); } } diff --git a/core/src/validator.rs b/core/src/validator.rs index 36a5cf334f015b..60f7447234632b 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -156,6 +156,7 @@ pub struct ValidatorConfig { pub no_os_network_stats_reporting: bool, pub no_os_cpu_stats_reporting: bool, pub no_os_disk_stats_reporting: bool, + pub no_os_open_fd_stats_reporting: bool, pub poh_pinned_cpu_core: usize, pub poh_hashes_per_batch: u64, pub process_ledger_before_services: bool, @@ -218,6 +219,7 @@ impl Default for ValidatorConfig { no_os_network_stats_reporting: true, no_os_cpu_stats_reporting: true, no_os_disk_stats_reporting: true, + no_os_open_fd_stats_reporting: true, poh_pinned_cpu_core: poh_service::DEFAULT_PINNED_CPU_CORE, poh_hashes_per_batch: poh_service::DEFAULT_HASHES_PER_BATCH, process_ledger_before_services: false, @@ -500,6 +502,7 @@ impl Validator { !config.no_os_network_stats_reporting, !config.no_os_cpu_stats_reporting, !config.no_os_disk_stats_reporting, + !config.no_os_open_fd_stats_reporting, )); let (poh_timing_point_sender, poh_timing_point_receiver) = unbounded(); diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 0da06d773462e5..c2638ceb0c28f9 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -2711,6 +2711,7 @@ fn main() { false, false, false, + false, ); accounts_index_config.index_limit_mb = if let Some(limit) = diff --git a/local-cluster/src/validator_configs.rs b/local-cluster/src/validator_configs.rs index 88a6977a48e9d5..eda2952f1ffd7e 100644 --- a/local-cluster/src/validator_configs.rs +++ b/local-cluster/src/validator_configs.rs @@ -46,6 +46,7 @@ pub fn safe_clone_config(config: &ValidatorConfig) -> ValidatorConfig { no_os_network_stats_reporting: config.no_os_network_stats_reporting, no_os_cpu_stats_reporting: config.no_os_cpu_stats_reporting, no_os_disk_stats_reporting: config.no_os_disk_stats_reporting, + no_os_open_fd_stats_reporting: config.no_os_open_fd_stats_reporting, poh_pinned_cpu_core: config.poh_pinned_cpu_core, account_indexes: config.account_indexes.clone(), accounts_db_caching_enabled: config.accounts_db_caching_enabled, diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index cf3968139feb81..9e3a973f07a71a 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -4491,6 +4491,7 @@ dependencies = [ "lru", "min-max-heap", "num_enum", + "procfs", "rand 0.7.3", "rand_chacha 0.2.2", "rayon", diff --git a/validator/src/cli.rs b/validator/src/cli.rs index af919e53745b3a..3d050ab92a4927 100644 --- a/validator/src/cli.rs +++ b/validator/src/cli.rs @@ -513,6 +513,11 @@ pub fn app<'a>(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> { .long("no-os-disk-stats-reporting") .help("Disable reporting of OS disk statistics.") ) + .arg( + Arg::with_name("no_os_open_fd_stats_reporting") + .long("no-os-open-fd-stats-reporting") + .help("Disable reporting of open file descriptors statistics for current process.") + ) .arg( Arg::with_name("accounts-hash-interval-slots") .long("accounts-hash-interval-slots") diff --git a/validator/src/main.rs b/validator/src/main.rs index b1f07fdb81e66b..be3d77ef47e1c4 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -1141,6 +1141,7 @@ pub fn main() { no_os_network_stats_reporting: matches.is_present("no_os_network_stats_reporting"), no_os_cpu_stats_reporting: matches.is_present("no_os_cpu_stats_reporting"), no_os_disk_stats_reporting: matches.is_present("no_os_disk_stats_reporting"), + no_os_open_fd_stats_reporting: matches.is_present("no_os_open_fd_stats_reporting"), poh_pinned_cpu_core: value_of(&matches, "poh_pinned_cpu_core") .unwrap_or(poh_service::DEFAULT_PINNED_CPU_CORE), poh_hashes_per_batch: value_of(&matches, "poh_hashes_per_batch") From 6e8dd0eeb5863eb6e6a906468a60b49f4a8d6ec8 Mon Sep 17 00:00:00 2001 From: Haoran Yi Date: Mon, 28 Nov 2022 17:47:47 -0600 Subject: [PATCH 03/21] add mmmap file count --- core/src/system_monitor_service.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/core/src/system_monitor_service.rs b/core/src/system_monitor_service.rs index 9911597dce9c43..a739521657545e 100644 --- a/core/src/system_monitor_service.rs +++ b/core/src/system_monitor_service.rs @@ -839,6 +839,7 @@ impl SystemMonitorService { fn get_open_fd_stats() -> Option<(usize, usize, usize)> { let proc = Process::myself().ok()?; let curr_num_open_fd = proc.fd_count().unwrap(); + let curr_mmap_count = proc.maps().unwrap().len(); let max_open_fd_limit = proc.limits().unwrap().max_open_files; let max_open_fd_soft_limit = match max_open_fd_limit.soft_limit { @@ -852,18 +853,24 @@ impl SystemMonitorService { Some(( curr_num_open_fd, + curr_mmap_count, max_open_fd_soft_limit, max_open_fd_hard_limit, )) } fn report_open_fd_stats() { - if let Some((curr_num_open_fd, max_open_fd_soft_limit, max_open_fd_hard_limit)) = - Self::get_open_fd_stats() + if let Some(( + curr_num_open_fd, + curr_mmap_count, + max_open_fd_soft_limit, + max_open_fd_hard_limit, + )) = Self::get_open_fd_stats() { datapoint_info!( "open-fd-stats", ("number_open_files", curr_num_open_fd, i64), + ("number_mmap_files", curr_mmap_count, i64), ("max_open_files_hard_limit", max_open_fd_hard_limit, i64), ("max_open_files_soft_limit", max_open_fd_soft_limit, i64), ); From 680218e4e7e8c678905473da9b518e507e0026b8 Mon Sep 17 00:00:00 2001 From: Haoran Yi Date: Mon, 28 Nov 2022 18:57:47 -0600 Subject: [PATCH 04/21] fix --- core/src/system_monitor_service.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/system_monitor_service.rs b/core/src/system_monitor_service.rs index a739521657545e..46b70ad4ab8c72 100644 --- a/core/src/system_monitor_service.rs +++ b/core/src/system_monitor_service.rs @@ -836,7 +836,7 @@ impl SystemMonitorService { Self::report_cpuid_values(); } - fn get_open_fd_stats() -> Option<(usize, usize, usize)> { + fn get_open_fd_stats() -> Option<(usize, usize, usize, usize)> { let proc = Process::myself().ok()?; let curr_num_open_fd = proc.fd_count().unwrap(); let curr_mmap_count = proc.maps().unwrap().len(); From 37ef3402db458ea5d1378e670cd1dcda3385756f Mon Sep 17 00:00:00 2001 From: Haoran Yi Date: Mon, 28 Nov 2022 18:52:37 -0600 Subject: [PATCH 05/21] log mmap count --- bucket_map/src/bucket_storage.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/bucket_map/src/bucket_storage.rs b/bucket_map/src/bucket_storage.rs index f7471459922540..479ab855bc2d50 100644 --- a/bucket_map/src/bucket_storage.rs +++ b/bucket_map/src/bucket_storage.rs @@ -263,12 +263,13 @@ impl BucketStorage { } } - fn get_number_of_open_files_and_limit() -> Option<(usize, Limit)> { + fn get_mmap_fd_stats() -> Option<(usize, usize, Limit)> { let proc = Process::myself().ok()?; + let mmap_count = proc.maps().unwrap().len(); let max_open_files_limit = proc.limits().unwrap().max_open_files; let num_open_files = proc.fd_count().unwrap(); - Some((num_open_files, max_open_files_limit)) + Some((mmap_count, num_open_files, max_open_files_limit)) } fn new_map( @@ -289,12 +290,13 @@ impl BucketStorage { .create(true) .open(file.clone()) .map_err(|e| { - if let Some((num_open_files, max_open_files_limit)) = Self::get_number_of_open_files_and_limit() { + if let Some((mmap_count, num_open_files, max_open_files_limit)) = Self::get_mmap_fd_stats() { panic!( - "Unable to create data file {} in current dir({:?}): {:?}, current number of open files: {}, max limit of open files: {:?}", + "Unable to create data file {} in current dir({:?}): {:?}, current mmap_count: {}, current number of open files: {}, max limit of open files: {:?}", file.display(), std::env::current_dir(), e, + mmap_count, num_open_files, max_open_files_limit, ); @@ -434,10 +436,11 @@ mod test { assert!(storage.is_free(ix)); assert_eq!(storage.uid(ix), None); - // test get_number_of_open_files_and_limit - if let Some((num_open_files, max_open_files_limit)) = - BucketStorage::get_number_of_open_files_and_limit() + // test get_mmap_fd_stats + if let Some((mmap_count, num_open_files, max_open_files_limit)) = + BucketStorage::get_mmap_fd_stats() { + assert!(mmap_count > 0); assert!(num_open_files > 0); match max_open_files_limit.soft_limit { LimitValue::Unlimited => {} From fb615a2d93b0270bde744cfe1a41ad0ee04b5716 Mon Sep 17 00:00:00 2001 From: HaoranYi Date: Tue, 29 Nov 2022 10:45:06 -0600 Subject: [PATCH 06/21] Update validator/src/cli.rs Co-authored-by: Michael Vines --- validator/src/cli.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/src/cli.rs b/validator/src/cli.rs index 3d050ab92a4927..e2fcc4a9835c06 100644 --- a/validator/src/cli.rs +++ b/validator/src/cli.rs @@ -516,7 +516,7 @@ pub fn app<'a>(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> { .arg( Arg::with_name("no_os_open_fd_stats_reporting") .long("no-os-open-fd-stats-reporting") - .help("Disable reporting of open file descriptors statistics for current process.") + .help("Disable reporting of open file descriptor statistics.") ) .arg( Arg::with_name("accounts-hash-interval-slots") From b389a8168179df45d7bc787e4291129a67fb9262 Mon Sep 17 00:00:00 2001 From: haoran Date: Tue, 29 Nov 2022 17:42:18 +0000 Subject: [PATCH 07/21] review feedbacks --- bucket_map/src/bucket_storage.rs | 62 ++++++++++++++------------------ 1 file changed, 27 insertions(+), 35 deletions(-) diff --git a/bucket_map/src/bucket_storage.rs b/bucket_map/src/bucket_storage.rs index 479ab855bc2d50..e59520b8d9901a 100644 --- a/bucket_map/src/bucket_storage.rs +++ b/bucket_map/src/bucket_storage.rs @@ -290,25 +290,21 @@ impl BucketStorage { .create(true) .open(file.clone()) .map_err(|e| { - if let Some((mmap_count, num_open_files, max_open_files_limit)) = Self::get_mmap_fd_stats() { - panic!( - "Unable to create data file {} in current dir({:?}): {:?}, current mmap_count: {}, current number of open files: {}, max limit of open files: {:?}", - file.display(), - std::env::current_dir(), - e, - mmap_count, - num_open_files, - max_open_files_limit, - ); - } - else { - panic!( - "Unable to create data file {} in current dir({:?}): {:?}", - file.display(), - std::env::current_dir(), - e, - ); - } + let mmap_msg = Self::get_mmap_fd_stats() + .map(|(mmap_count, num_open_files, max_open_files_limit)| { + format!("current mmap_count: {}, current number of open files: {}, max limit of open files: {:?}", + mmap_count, + num_open_files, + max_open_files_limit, + ) + }).unwrap_or_default(); + panic!( + "Unable to create data file {} in current dir({:?}): {:?}. {}", + file.display(), + std::env::current_dir(), + e, + mmap_msg, + ); }) .unwrap(); @@ -437,23 +433,19 @@ mod test { assert_eq!(storage.uid(ix), None); // test get_mmap_fd_stats - if let Some((mmap_count, num_open_files, max_open_files_limit)) = - BucketStorage::get_mmap_fd_stats() - { - assert!(mmap_count > 0); - assert!(num_open_files > 0); - match max_open_files_limit.soft_limit { - LimitValue::Unlimited => {} - LimitValue::Value(x) => assert!(x > 0), - } - - match max_open_files_limit.hard_limit { - LimitValue::Unlimited => {} - LimitValue::Value(x) => assert!(x > 0), - } + let (mmap_count, num_open_files, max_open_files_limit) = + BucketStorage::get_mmap_fd_stats().unwrap(); + + assert!(mmap_count > 0); + assert!(num_open_files > 0); + match max_open_files_limit.soft_limit { + LimitValue::Unlimited => {} + LimitValue::Value(x) => assert!(x > 0), + } - println!("{:?}", num_open_files); - println!("{:?}", max_open_files_limit); + match max_open_files_limit.hard_limit { + LimitValue::Unlimited => {} + LimitValue::Value(x) => assert!(x > 0), } } } From 30d9c878ea579355175dfb50b81951fc9c0c9621 Mon Sep 17 00:00:00 2001 From: haoran Date: Tue, 29 Nov 2022 17:58:27 +0000 Subject: [PATCH 08/21] refactor SystemMonitorReportConfig --- core/src/system_monitor_service.rs | 47 ++++++++++++------------------ core/src/validator.rs | 16 ++++++---- 2 files changed, 28 insertions(+), 35 deletions(-) diff --git a/core/src/system_monitor_service.rs b/core/src/system_monitor_service.rs index 46b70ad4ab8c72..ce190c65122e13 100644 --- a/core/src/system_monitor_service.rs +++ b/core/src/system_monitor_service.rs @@ -387,27 +387,21 @@ fn parse_disk_stats(reader_diskstats: &mut impl BufRead) -> Result, - report_os_memory_stats: bool, - report_os_network_stats: bool, - report_os_cpu_stats: bool, - report_os_disk_stats: bool, - report_os_open_fd_stats: bool, - ) -> Self { + pub fn new(exit: Arc, config: SystemMonitorStatsReportConfig) -> Self { info!("Starting SystemMonitorService"); let thread_hdl = Builder::new() .name("solSystemMonitr".to_string()) .spawn(move || { - Self::run( - exit, - report_os_memory_stats, - report_os_network_stats, - report_os_cpu_stats, - report_os_disk_stats, - report_os_open_fd_stats, - ); + Self::run(exit, config); }) .unwrap(); @@ -1012,14 +1006,7 @@ impl SystemMonitorService { ) } - pub fn run( - exit: Arc, - report_os_memory_stats: bool, - report_os_network_stats: bool, - report_os_cpu_stats: bool, - report_os_disk_stats: bool, - report_os_open_fd_stats: bool, - ) { + pub fn run(exit: Arc, config: SystemMonitorStatsReportConfig) { let mut udp_stats = None; let mut disk_stats = None; let network_limits_timer = AtomicInterval::default(); @@ -1033,7 +1020,7 @@ impl SystemMonitorService { if exit.load(Ordering::Relaxed) { break; } - if report_os_network_stats { + if config.report_os_network_stats { if network_limits_timer.should_update(SAMPLE_INTERVAL_OS_NETWORK_LIMITS_MS) { Self::check_os_network_limits(); } @@ -1041,16 +1028,18 @@ impl SystemMonitorService { Self::process_net_stats(&mut udp_stats); } } - if report_os_memory_stats && mem_timer.should_update(SAMPLE_INTERVAL_MEM_MS) { + if config.report_os_memory_stats && mem_timer.should_update(SAMPLE_INTERVAL_MEM_MS) { Self::report_mem_stats(); } - if report_os_cpu_stats && cpu_timer.should_update(SAMPLE_INTERVAL_CPU_MS) { + if config.report_os_cpu_stats && cpu_timer.should_update(SAMPLE_INTERVAL_CPU_MS) { Self::report_cpu_stats(); } - if report_os_disk_stats && disk_timer.should_update(SAMPLE_INTERVAL_DISK_MS) { + if config.report_os_disk_stats && disk_timer.should_update(SAMPLE_INTERVAL_DISK_MS) { Self::process_disk_stats(&mut disk_stats); } - if report_os_open_fd_stats && open_fd_timer.should_update(SAMPLE_INTERVAL_OPEN_FD_MS) { + if config.report_os_open_fd_stats + && open_fd_timer.should_update(SAMPLE_INTERVAL_OPEN_FD_MS) + { Self::report_open_fd_stats(); } sleep(SLEEP_INTERVAL); diff --git a/core/src/validator.rs b/core/src/validator.rs index 60f7447234632b..5bbd61a7ff9861 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -18,7 +18,9 @@ use { sigverify, snapshot_packager_service::SnapshotPackagerService, stats_reporter_service::StatsReporterService, - system_monitor_service::{verify_net_stats_access, SystemMonitorService}, + system_monitor_service::{ + verify_net_stats_access, SystemMonitorService, SystemMonitorStatsReportConfig, + }, tower_storage::TowerStorage, tpu::{Tpu, TpuSockets, DEFAULT_TPU_COALESCE_MS}, tvu::{Tvu, TvuConfig, TvuSockets}, @@ -498,11 +500,13 @@ impl Validator { let system_monitor_service = Some(SystemMonitorService::new( Arc::clone(&exit), - !config.no_os_memory_stats_reporting, - !config.no_os_network_stats_reporting, - !config.no_os_cpu_stats_reporting, - !config.no_os_disk_stats_reporting, - !config.no_os_open_fd_stats_reporting, + SystemMonitorStatsReportConfig { + report_os_memory_stats: !config.no_os_memory_stats_reporting, + report_os_network_stats: !config.no_os_network_stats_reporting, + report_os_cpu_stats: !config.no_os_cpu_stats_reporting, + report_os_disk_stats: !config.no_os_disk_stats_reporting, + report_os_open_fd_stats: !config.no_os_open_fd_stats_reporting, + }, )); let (poh_timing_point_sender, poh_timing_point_receiver) = unbounded(); From a4dfa36efe9cad2e2b86325f7caa2dc7416a8af2 Mon Sep 17 00:00:00 2001 From: haoran Date: Tue, 29 Nov 2022 19:23:52 +0000 Subject: [PATCH 09/21] fix build --- ledger-tool/src/main.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index c2638ceb0c28f9..81b9374aa7c648 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -2707,11 +2707,13 @@ fn main() { arg_matches.is_present("no_os_memory_stats_reporting"); let system_monitor_service = SystemMonitorService::new( Arc::clone(&exit_signal), - !no_os_memory_stats_reporting, - false, - false, - false, - false, + SystemMonitorStatsReportConfig { + report_os_memory_stats: !no_os_memory_stats_reporting, + report_os_network_stats: false, + report_os_cpu_stats: false, + report_os_disk_stats: false, + report_os_open_fd_stats: false, + }, ); accounts_index_config.index_limit_mb = if let Some(limit) = From b294dae45ff926a6b2ad3feda32d49a2fd87b273 Mon Sep 17 00:00:00 2001 From: haoran Date: Tue, 29 Nov 2022 19:28:35 +0000 Subject: [PATCH 10/21] increase fd report interval to 3 minutes --- core/src/system_monitor_service.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/system_monitor_service.rs b/core/src/system_monitor_service.rs index ce190c65122e13..657ee5552229a1 100644 --- a/core/src/system_monitor_service.rs +++ b/core/src/system_monitor_service.rs @@ -31,6 +31,8 @@ const SAMPLE_INTERVAL_MEM_MS: u64 = 5 * MS_PER_S; const SAMPLE_INTERVAL_CPU_MS: u64 = 10 * MS_PER_S; const SAMPLE_INTERVAL_DISK_MS: u64 = 5 * MS_PER_S; const SAMPLE_INTERVAL_OPEN_FD_MS: u64 = 30 * MS_PER_S; +const SAMPLE_INTERVAL_DISK_MS: u64 = 5 * MS_PER_S; +const SAMPLE_INTERVAL_OPEN_FD_MS: u64 = 60 * MS_PER_S; const SLEEP_INTERVAL: Duration = Duration::from_millis(500); #[cfg(target_os = "linux")] From 4cade8df2f3a9265908cde9867099ab54d01d701 Mon Sep 17 00:00:00 2001 From: haoran Date: Tue, 29 Nov 2022 20:19:45 +0000 Subject: [PATCH 11/21] fix --- ledger-tool/src/main.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 81b9374aa7c648..60b9fb7b3d985b 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -25,7 +25,8 @@ use { }, solana_cli_output::{CliAccount, CliAccountNewConfig, OutputFormat}, solana_core::{ - system_monitor_service::SystemMonitorService, validator::move_and_async_delete_path, + system_monitor_service::{SystemMonitorService, SystemMonitorStatsReportConfig}, + validator::move_and_async_delete_path, }, solana_entry::entry::Entry, solana_geyser_plugin_manager::geyser_plugin_service::GeyserPluginService, From d4034df9571c0cb848cc638ebaa58d27faa83896 Mon Sep 17 00:00:00 2001 From: haoran Date: Tue, 29 Nov 2022 20:59:09 +0000 Subject: [PATCH 12/21] get mmap with wc-l --- bucket_map/src/bucket_storage.rs | 56 +++++++++++++++++++++++++++++- core/src/system_monitor_service.rs | 24 ++++++++++++- 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/bucket_map/src/bucket_storage.rs b/bucket_map/src/bucket_storage.rs index e59520b8d9901a..fc08caa50e2b8d 100644 --- a/bucket_map/src/bucket_storage.rs +++ b/bucket_map/src/bucket_storage.rs @@ -263,9 +263,31 @@ impl BucketStorage { } } + fn get_mmap_count() -> Option { + let pid = std::process::id(); + let map_path = format!("/proc/{}/maps", pid); + let output = std::process::Command::new("wc") + .args(["-l", &map_path]) + .output() + .unwrap(); + if output.status.success() { + let n: usize = std::str::from_utf8(&output.stdout) + .unwrap() + .split_whitespace() + .next() + .unwrap() + .parse() + .unwrap(); + + Some(n) + } else { + None + } + } + fn get_mmap_fd_stats() -> Option<(usize, usize, Limit)> { let proc = Process::myself().ok()?; - let mmap_count = proc.maps().unwrap().len(); + let mmap_count = Self::get_mmap_count().unwrap(); let max_open_files_limit = proc.limits().unwrap().max_open_files; let num_open_files = proc.fd_count().unwrap(); @@ -448,4 +470,36 @@ mod test { LimitValue::Value(x) => assert!(x > 0), } } + + #[test] + fn test_time_mmap() { + use std::time::{Duration, Instant}; + + let mut v = vec![]; + for i in 1..1900000 { + if i % 100 == 0 { + println!("{}", i); + } + + let tmpdir = tempdir().unwrap(); + let paths: Vec = vec![tmpdir.path().to_path_buf()]; + assert!(!paths.is_empty()); + let mut s = + BucketStorage::new(Arc::new(paths), 1, 1, 1, Arc::default(), Arc::default()); + v.push(s); + } + + // test get_mmap_fd_stats + let start = Instant::now(); + let (mmap_count, num_open_files, max_open_files_limit) = + BucketStorage::get_mmap_fd_stats().unwrap(); + let duration = start.elapsed(); + + println!( + "{} {} {:?}", + mmap_count, num_open_files, max_open_files_limit + ); + + println!("Time elapsed is: {:?}", duration); + } } diff --git a/core/src/system_monitor_service.rs b/core/src/system_monitor_service.rs index 657ee5552229a1..2502d56f4d3644 100644 --- a/core/src/system_monitor_service.rs +++ b/core/src/system_monitor_service.rs @@ -832,10 +832,32 @@ impl SystemMonitorService { Self::report_cpuid_values(); } + fn get_mmap_count() -> Option { + let pid = std::process::id(); + let map_path = format!("/proc/{}/maps", pid); + let output = std::process::Command::new("wc") + .args(["-l", &map_path]) + .output() + .unwrap(); + if output.status.success() { + let n: usize = std::str::from_utf8(&output.stdout) + .unwrap() + .split_whitespace() + .next() + .unwrap() + .parse() + .unwrap(); + + Some(n) + } else { + None + } + } + fn get_open_fd_stats() -> Option<(usize, usize, usize, usize)> { let proc = Process::myself().ok()?; let curr_num_open_fd = proc.fd_count().unwrap(); - let curr_mmap_count = proc.maps().unwrap().len(); + let curr_mmap_count = Self::get_mmap_count().unwrap(); let max_open_fd_limit = proc.limits().unwrap().max_open_files; let max_open_fd_soft_limit = match max_open_fd_limit.soft_limit { From e7d069782b8112e80031856ca7b17cd6edd10f80 Mon Sep 17 00:00:00 2001 From: haoran Date: Tue, 29 Nov 2022 22:22:44 +0000 Subject: [PATCH 13/21] clippy --- bucket_map/src/bucket_storage.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bucket_map/src/bucket_storage.rs b/bucket_map/src/bucket_storage.rs index fc08caa50e2b8d..af32ae882cefa6 100644 --- a/bucket_map/src/bucket_storage.rs +++ b/bucket_map/src/bucket_storage.rs @@ -473,9 +473,9 @@ mod test { #[test] fn test_time_mmap() { - use std::time::{Duration, Instant}; + use std::time::Instant; - let mut v = vec![]; + let v = vec![]; for i in 1..1900000 { if i % 100 == 0 { println!("{}", i); From a8b5ffba7cc7f1333a71ecc77103cabdb5f76250 Mon Sep 17 00:00:00 2001 From: haoran Date: Wed, 30 Nov 2022 15:54:02 +0000 Subject: [PATCH 14/21] share code --- Cargo.lock | 2 +- bucket_map/src/bucket_map.rs | 48 +++++++++++++++++++++++++++++ bucket_map/src/bucket_storage.rs | 46 +++++----------------------- core/Cargo.toml | 2 +- core/src/system_monitor_service.rs | 49 ++---------------------------- 5 files changed, 59 insertions(+), 88 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0bd2719eca72e2..8ed6e277dcbafa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5202,7 +5202,6 @@ dependencies = [ "matches", "min-max-heap", "num_enum", - "procfs", "rand 0.7.3", "rand_chacha 0.2.2", "raptorq", @@ -5214,6 +5213,7 @@ dependencies = [ "serial_test", "solana-address-lookup-table-program", "solana-bloom", + "solana-bucket-map", "solana-client", "solana-entry", "solana-frozen-abi 1.15.0", diff --git a/bucket_map/src/bucket_map.rs b/bucket_map/src/bucket_map.rs index 02666316f55a4a..4351e183b729cc 100644 --- a/bucket_map/src/bucket_map.rs +++ b/bucket_map/src/bucket_map.rs @@ -2,6 +2,7 @@ use { crate::{bucket_api::BucketApi, bucket_stats::BucketMapStats, MaxSearch, RefCount}, + procfs::process::{LimitValue, Process}, solana_sdk::pubkey::Pubkey, std::{convert::TryInto, fmt::Debug, fs, path::PathBuf, sync::Arc}, tempfile::TempDir, @@ -181,6 +182,53 @@ fn read_be_u64(input: &[u8]) -> u64 { u64::from_be_bytes(input[0..std::mem::size_of::()].try_into().unwrap()) } +/// Utility function to get number of Mmap files for current process +pub fn get_mmap_count() -> Option { + let pid = std::process::id(); + let map_path = format!("/proc/{}/maps", pid); + let output = std::process::Command::new("wc") + .args(["-l", &map_path]) + .output() + .unwrap(); + if output.status.success() { + let n: usize = std::str::from_utf8(&output.stdout) + .unwrap() + .split_whitespace() + .next() + .unwrap() + .parse() + .unwrap(); + + Some(n) + } else { + None + } +} + +/// Utility function to get open_fd stats +pub fn get_open_fd_stats() -> Option<(usize, usize, usize, usize)> { + let proc = Process::myself().ok()?; + let curr_num_open_fd = proc.fd_count().unwrap(); + let curr_mmap_count = get_mmap_count().unwrap(); + let max_open_fd_limit = proc.limits().unwrap().max_open_files; + + let max_open_fd_soft_limit = match max_open_fd_limit.soft_limit { + LimitValue::Unlimited => usize::MAX, + LimitValue::Value(x) => x as usize, + }; + let max_open_fd_hard_limit = match max_open_fd_limit.hard_limit { + LimitValue::Unlimited => usize::MAX, + LimitValue::Value(x) => x as usize, + }; + + Some(( + curr_num_open_fd, + curr_mmap_count, + max_open_fd_soft_limit, + max_open_fd_hard_limit, + )) +} + #[cfg(test)] mod tests { use { diff --git a/bucket_map/src/bucket_storage.rs b/bucket_map/src/bucket_storage.rs index af32ae882cefa6..d58be7b0a42583 100644 --- a/bucket_map/src/bucket_storage.rs +++ b/bucket_map/src/bucket_storage.rs @@ -1,7 +1,6 @@ use { - crate::{bucket_stats::BucketStats, MaxSearch}, + crate::{bucket_map::get_open_fd_stats, bucket_stats::BucketStats, MaxSearch}, memmap2::MmapMut, - procfs::process::{Limit, Process}, rand::{thread_rng, Rng}, solana_measure::measure::Measure, std::{ @@ -263,37 +262,6 @@ impl BucketStorage { } } - fn get_mmap_count() -> Option { - let pid = std::process::id(); - let map_path = format!("/proc/{}/maps", pid); - let output = std::process::Command::new("wc") - .args(["-l", &map_path]) - .output() - .unwrap(); - if output.status.success() { - let n: usize = std::str::from_utf8(&output.stdout) - .unwrap() - .split_whitespace() - .next() - .unwrap() - .parse() - .unwrap(); - - Some(n) - } else { - None - } - } - - fn get_mmap_fd_stats() -> Option<(usize, usize, Limit)> { - let proc = Process::myself().ok()?; - let mmap_count = Self::get_mmap_count().unwrap(); - let max_open_files_limit = proc.limits().unwrap().max_open_files; - let num_open_files = proc.fd_count().unwrap(); - - Some((mmap_count, num_open_files, max_open_files_limit)) - } - fn new_map( drives: &[PathBuf], cell_size: usize, @@ -312,12 +280,13 @@ impl BucketStorage { .create(true) .open(file.clone()) .map_err(|e| { - let mmap_msg = Self::get_mmap_fd_stats() - .map(|(mmap_count, num_open_files, max_open_files_limit)| { - format!("current mmap_count: {}, current number of open files: {}, max limit of open files: {:?}", + let mmap_msg = get_open_fd_stats() + .map(|(mmap_count, num_open_files, soft_limit, hard_limit)| { + format!("current mmap_count: {}, current number of open files: {}, soft_limit: {}, hard_limit: {}", mmap_count, num_open_files, - max_open_files_limit, + soft_limit, + hard_limit, ) }).unwrap_or_default(); panic!( @@ -491,8 +460,7 @@ mod test { // test get_mmap_fd_stats let start = Instant::now(); - let (mmap_count, num_open_files, max_open_files_limit) = - BucketStorage::get_mmap_fd_stats().unwrap(); + let (mmap_count, num_open_files, soft_limit, hard_limit) = get_open_fd_stats().unwrap(); let duration = start.elapsed(); println!( diff --git a/core/Cargo.toml b/core/Cargo.toml index 5c42d5f615eefd..0d72623222a290 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -31,13 +31,13 @@ log = "0.4.17" lru = "0.7.7" min-max-heap = "1.3.0" num_enum = "0.5.7" -procfs = "0.14.1" rand = "0.7.0" rand_chacha = "0.2.2" rayon = "1.5.3" serde = "1.0.144" serde_derive = "1.0.103" solana-address-lookup-table-program = { path = "../programs/address-lookup-table", version = "=1.15.0" } +solana-bucket-map = { path = "../bucket_map", version = "=1.15.0" } solana-bloom = { path = "../bloom", version = "=1.15.0" } solana-client = { path = "../client", version = "=1.15.0" } solana-entry = { path = "../entry", version = "=1.15.0" } diff --git a/core/src/system_monitor_service.rs b/core/src/system_monitor_service.rs index 2502d56f4d3644..be9c65f59e0d5d 100644 --- a/core/src/system_monitor_service.rs +++ b/core/src/system_monitor_service.rs @@ -7,7 +7,7 @@ use num_enum::{IntoPrimitive, TryFromPrimitive}; #[cfg(target_os = "linux")] use std::{fs::File, io::BufReader}; use { - procfs::process::{LimitValue, Process}, + solana_bucket_map::bucket_map::get_open_fd_stats, solana_sdk::timing::AtomicInterval, std::{ collections::HashMap, @@ -832,58 +832,13 @@ impl SystemMonitorService { Self::report_cpuid_values(); } - fn get_mmap_count() -> Option { - let pid = std::process::id(); - let map_path = format!("/proc/{}/maps", pid); - let output = std::process::Command::new("wc") - .args(["-l", &map_path]) - .output() - .unwrap(); - if output.status.success() { - let n: usize = std::str::from_utf8(&output.stdout) - .unwrap() - .split_whitespace() - .next() - .unwrap() - .parse() - .unwrap(); - - Some(n) - } else { - None - } - } - - fn get_open_fd_stats() -> Option<(usize, usize, usize, usize)> { - let proc = Process::myself().ok()?; - let curr_num_open_fd = proc.fd_count().unwrap(); - let curr_mmap_count = Self::get_mmap_count().unwrap(); - let max_open_fd_limit = proc.limits().unwrap().max_open_files; - - let max_open_fd_soft_limit = match max_open_fd_limit.soft_limit { - LimitValue::Unlimited => usize::MAX, - LimitValue::Value(x) => x as usize, - }; - let max_open_fd_hard_limit = match max_open_fd_limit.hard_limit { - LimitValue::Unlimited => usize::MAX, - LimitValue::Value(x) => x as usize, - }; - - Some(( - curr_num_open_fd, - curr_mmap_count, - max_open_fd_soft_limit, - max_open_fd_hard_limit, - )) - } - fn report_open_fd_stats() { if let Some(( curr_num_open_fd, curr_mmap_count, max_open_fd_soft_limit, max_open_fd_hard_limit, - )) = Self::get_open_fd_stats() + )) = get_open_fd_stats() { datapoint_info!( "open-fd-stats", From 18acb4aa2539216628834b7e424a3945e4616b9a Mon Sep 17 00:00:00 2001 From: haoran Date: Wed, 30 Nov 2022 16:29:22 +0000 Subject: [PATCH 15/21] alternative impl --- bucket_map/src/bucket_map.rs | 43 +++++++++++++++++++------------- bucket_map/src/bucket_storage.rs | 25 +++++++------------ 2 files changed, 35 insertions(+), 33 deletions(-) diff --git a/bucket_map/src/bucket_map.rs b/bucket_map/src/bucket_map.rs index 4351e183b729cc..88832f99e143b0 100644 --- a/bucket_map/src/bucket_map.rs +++ b/bucket_map/src/bucket_map.rs @@ -184,25 +184,34 @@ fn read_be_u64(input: &[u8]) -> u64 { /// Utility function to get number of Mmap files for current process pub fn get_mmap_count() -> Option { + use std::io::BufRead; + let pid = std::process::id(); let map_path = format!("/proc/{}/maps", pid); - let output = std::process::Command::new("wc") - .args(["-l", &map_path]) - .output() - .unwrap(); - if output.status.success() { - let n: usize = std::str::from_utf8(&output.stdout) - .unwrap() - .split_whitespace() - .next() - .unwrap() - .parse() - .unwrap(); - - Some(n) - } else { - None - } + let tmp_dir = tempfile::TempDir::new().ok()?; + let copy_path = tmp_dir.path().join("maps"); + std::fs::copy(map_path, copy_path.as_os_str()).unwrap(); + + let file = std::fs::File::open(copy_path).ok()?; + Some(std::io::BufReader::new(file).lines().count()) + + // let output = std::process::Command::new("wc") + // .args(["-l", &map_path]) + // .output() + // .unwrap(); + // if output.status.success() { + // let n: usize = std::str::from_utf8(&output.stdout) + // .unwrap() + // .split_whitespace() + // .next() + // .unwrap() + // .parse() + // .unwrap(); + // + // Some(n) + // } else { + // None + // } } /// Utility function to get open_fd stats diff --git a/bucket_map/src/bucket_storage.rs b/bucket_map/src/bucket_storage.rs index d58be7b0a42583..eb6fbe77c0a964 100644 --- a/bucket_map/src/bucket_storage.rs +++ b/bucket_map/src/bucket_storage.rs @@ -391,7 +391,7 @@ impl BucketStorage { #[cfg(test)] mod test { - use {super::*, procfs::process::LimitValue, tempfile::tempdir}; + use {super::*, tempfile::tempdir}; #[test] fn test_bucket_storage() { @@ -424,27 +424,20 @@ mod test { assert_eq!(storage.uid(ix), None); // test get_mmap_fd_stats - let (mmap_count, num_open_files, max_open_files_limit) = - BucketStorage::get_mmap_fd_stats().unwrap(); + let (mmap_count, num_open_files, soft_limit, hard_limit) = + get_open_fd_stats().unwrap(); assert!(mmap_count > 0); assert!(num_open_files > 0); - match max_open_files_limit.soft_limit { - LimitValue::Unlimited => {} - LimitValue::Value(x) => assert!(x > 0), - } - - match max_open_files_limit.hard_limit { - LimitValue::Unlimited => {} - LimitValue::Value(x) => assert!(x > 0), - } + assert!(soft_limit > 0); + assert!(hard_limit > 0); } #[test] fn test_time_mmap() { use std::time::Instant; - let v = vec![]; + let mut v = vec![]; for i in 1..1900000 { if i % 100 == 0 { println!("{}", i); @@ -453,7 +446,7 @@ mod test { let tmpdir = tempdir().unwrap(); let paths: Vec = vec![tmpdir.path().to_path_buf()]; assert!(!paths.is_empty()); - let mut s = + let s = BucketStorage::new(Arc::new(paths), 1, 1, 1, Arc::default(), Arc::default()); v.push(s); } @@ -464,8 +457,8 @@ mod test { let duration = start.elapsed(); println!( - "{} {} {:?}", - mmap_count, num_open_files, max_open_files_limit + "{} {} {} {}", + mmap_count, num_open_files, soft_limit, hard_limit ); println!("Time elapsed is: {:?}", duration); From b01c5096764473ff77a9a0bc9c4c03ea074cbc8a Mon Sep 17 00:00:00 2001 From: haoran Date: Thu, 1 Dec 2022 01:31:38 +0000 Subject: [PATCH 16/21] wc-l-copy --- bucket_map/src/bucket_map.rs | 74 +++++++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 22 deletions(-) diff --git a/bucket_map/src/bucket_map.rs b/bucket_map/src/bucket_map.rs index 88832f99e143b0..b54ba7ed7c664c 100644 --- a/bucket_map/src/bucket_map.rs +++ b/bucket_map/src/bucket_map.rs @@ -184,34 +184,64 @@ fn read_be_u64(input: &[u8]) -> u64 { /// Utility function to get number of Mmap files for current process pub fn get_mmap_count() -> Option { - use std::io::BufRead; - + // 1. readlines + // use std::io::BufRead; + + // let pid = std::process::id(); + // let map_path = format!("/proc/{}/maps", pid); + // let tmp_dir = tempfile::TempDir::new().ok()?; + // let copy_path = tmp_dir.path().join("maps"); + // std::fs::copy(map_path, copy_path.as_os_str()).unwrap(); + + // let file = std::fs::File::open(copy_path).ok()?; + // Some(std::io::BufReader::new(file).lines().count()) + + // wc-l + // let pid = std::process::id(); + // let map_path = format!("/proc/{}/maps", pid); + // let output = std::process::Command::new("wc") + // .args(["-l", &map_path]) + // .output() + // .unwrap(); + // if output.status.success() { + // let n: usize = std::str::from_utf8(&output.stdout) + // .unwrap() + // .split_whitespace() + // .next() + // .unwrap() + // .parse() + // .unwrap(); + + // Some(n) + // } else { + // None + // } + + // wc-l-copy let pid = std::process::id(); let map_path = format!("/proc/{}/maps", pid); + let tmp_dir = tempfile::TempDir::new().ok()?; let copy_path = tmp_dir.path().join("maps"); std::fs::copy(map_path, copy_path.as_os_str()).unwrap(); - let file = std::fs::File::open(copy_path).ok()?; - Some(std::io::BufReader::new(file).lines().count()) - - // let output = std::process::Command::new("wc") - // .args(["-l", &map_path]) - // .output() - // .unwrap(); - // if output.status.success() { - // let n: usize = std::str::from_utf8(&output.stdout) - // .unwrap() - // .split_whitespace() - // .next() - // .unwrap() - // .parse() - // .unwrap(); - // - // Some(n) - // } else { - // None - // } + let output = std::process::Command::new("wc") + .args(["-l", copy_path.to_str().unwrap()]) + .output() + .unwrap(); + if output.status.success() { + let n: usize = std::str::from_utf8(&output.stdout) + .unwrap() + .split_whitespace() + .next() + .unwrap() + .parse() + .unwrap(); + + Some(n) + } else { + None + } } /// Utility function to get open_fd stats From 5617c02d874513f6525da960e38dad25137cfcff Mon Sep 17 00:00:00 2001 From: haoran Date: Thu, 1 Dec 2022 18:07:17 +0000 Subject: [PATCH 17/21] don't use procfs as it is not supported on mac and windows. make open_fd stats only on linux platform --- Cargo.lock | 16 ----- bucket_map/Cargo.toml | 1 - bucket_map/src/bucket_map.rs | 93 +++++++++++------------------- bucket_map/src/bucket_storage.rs | 47 ++++++++------- core/src/system_monitor_service.rs | 19 +++--- programs/sbf/Cargo.lock | 24 +------- 6 files changed, 66 insertions(+), 134 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8ed6e277dcbafa..06a91d7b8a063e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3541,21 +3541,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "procfs" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dfb6451c91904606a1abe93e83a8ec851f45827fa84273f256ade45dc095818" -dependencies = [ - "bitflags", - "byteorder", - "chrono", - "flate2", - "hex", - "lazy_static", - "rustix", -] - [[package]] name = "proptest" version = "1.0.0" @@ -4916,7 +4901,6 @@ dependencies = [ "log", "memmap2", "modular-bitfield", - "procfs", "rand 0.7.3", "rayon", "solana-logger 1.15.0", diff --git a/bucket_map/Cargo.toml b/bucket_map/Cargo.toml index 9799821b2e3844..40f91d60b8fcf0 100644 --- a/bucket_map/Cargo.toml +++ b/bucket_map/Cargo.toml @@ -14,7 +14,6 @@ edition = "2021" log = { version = "0.4.17" } memmap2 = "0.5.3" modular-bitfield = "0.11.2" -procfs = "0.14.1" rand = "0.7.0" solana-measure = { path = "../measure", version = "=1.15.0" } solana-sdk = { path = "../sdk", version = "=1.15.0" } diff --git a/bucket_map/src/bucket_map.rs b/bucket_map/src/bucket_map.rs index b54ba7ed7c664c..1b2fe2009672d4 100644 --- a/bucket_map/src/bucket_map.rs +++ b/bucket_map/src/bucket_map.rs @@ -2,7 +2,6 @@ use { crate::{bucket_api::BucketApi, bucket_stats::BucketMapStats, MaxSearch, RefCount}, - procfs::process::{LimitValue, Process}, solana_sdk::pubkey::Pubkey, std::{convert::TryInto, fmt::Debug, fs, path::PathBuf, sync::Arc}, tempfile::TempDir, @@ -183,50 +182,12 @@ fn read_be_u64(input: &[u8]) -> u64 { } /// Utility function to get number of Mmap files for current process +#[cfg(target_os = "linux")] pub fn get_mmap_count() -> Option { - // 1. readlines - // use std::io::BufRead; - - // let pid = std::process::id(); - // let map_path = format!("/proc/{}/maps", pid); - // let tmp_dir = tempfile::TempDir::new().ok()?; - // let copy_path = tmp_dir.path().join("maps"); - // std::fs::copy(map_path, copy_path.as_os_str()).unwrap(); - - // let file = std::fs::File::open(copy_path).ok()?; - // Some(std::io::BufReader::new(file).lines().count()) - - // wc-l - // let pid = std::process::id(); - // let map_path = format!("/proc/{}/maps", pid); - // let output = std::process::Command::new("wc") - // .args(["-l", &map_path]) - // .output() - // .unwrap(); - // if output.status.success() { - // let n: usize = std::str::from_utf8(&output.stdout) - // .unwrap() - // .split_whitespace() - // .next() - // .unwrap() - // .parse() - // .unwrap(); - - // Some(n) - // } else { - // None - // } - - // wc-l-copy let pid = std::process::id(); let map_path = format!("/proc/{}/maps", pid); - - let tmp_dir = tempfile::TempDir::new().ok()?; - let copy_path = tmp_dir.path().join("maps"); - std::fs::copy(map_path, copy_path.as_os_str()).unwrap(); - let output = std::process::Command::new("wc") - .args(["-l", copy_path.to_str().unwrap()]) + .args(["-l", &map_path]) .output() .unwrap(); if output.status.success() { @@ -244,28 +205,40 @@ pub fn get_mmap_count() -> Option { } } -/// Utility function to get open_fd stats -pub fn get_open_fd_stats() -> Option<(usize, usize, usize, usize)> { - let proc = Process::myself().ok()?; - let curr_num_open_fd = proc.fd_count().unwrap(); - let curr_mmap_count = get_mmap_count().unwrap(); - let max_open_fd_limit = proc.limits().unwrap().max_open_files; +#[cfg(not(target_os = "linux"))] +pub fn get_mmap_count() -> Option { + None +} - let max_open_fd_soft_limit = match max_open_fd_limit.soft_limit { - LimitValue::Unlimited => usize::MAX, - LimitValue::Value(x) => x as usize, - }; - let max_open_fd_hard_limit = match max_open_fd_limit.hard_limit { - LimitValue::Unlimited => usize::MAX, - LimitValue::Value(x) => x as usize, +/// Utility function to get open_fd limits +#[cfg(target_os = "linux")] +pub fn get_open_fd_limits() -> Option<(usize, usize)> { + let run = |cmd, args| -> Option { + let output = std::process::Command::new(cmd).args(args).output().unwrap(); + if output.status.success() { + let n: usize = std::str::from_utf8(&output.stdout) + .unwrap() + .split_whitespace() + .next() + .unwrap() + .parse() + .unwrap(); + + Some(n) + } else { + None + } }; - Some(( - curr_num_open_fd, - curr_mmap_count, - max_open_fd_soft_limit, - max_open_fd_hard_limit, - )) + let soft_limit = run("sh", ["-c", "ulimit -Sn"])?; + let hard_limit = run("sh", ["-c", "ulimit -Hn"])?; + + Some((soft_limit, hard_limit)) +} + +#[cfg(not(target_os = "linux"))] +pub fn get_open_fd_limits() -> Option<(usize, usize, usize)> { + None } #[cfg(test)] diff --git a/bucket_map/src/bucket_storage.rs b/bucket_map/src/bucket_storage.rs index eb6fbe77c0a964..bc6218ff365a52 100644 --- a/bucket_map/src/bucket_storage.rs +++ b/bucket_map/src/bucket_storage.rs @@ -1,5 +1,9 @@ use { - crate::{bucket_map::get_open_fd_stats, bucket_stats::BucketStats, MaxSearch}, + crate::{ + bucket_map::{get_mmap_count, get_open_fd_limits}, + bucket_stats::BucketStats, + MaxSearch, + }, memmap2::MmapMut, rand::{thread_rng, Rng}, solana_measure::measure::Measure, @@ -280,21 +284,23 @@ impl BucketStorage { .create(true) .open(file.clone()) .map_err(|e| { - let mmap_msg = get_open_fd_stats() - .map(|(mmap_count, num_open_files, soft_limit, hard_limit)| { - format!("current mmap_count: {}, current number of open files: {}, soft_limit: {}, hard_limit: {}", - mmap_count, - num_open_files, - soft_limit, - hard_limit, - ) - }).unwrap_or_default(); + let mmap_msg = get_mmap_count() + .map(|mmap_count| format!("current mmap_count: {}", mmap_count)) + .unwrap_or_default(); + + let limit_msg = get_open_fd_limits() + .map(|(soft_limit, hard_limit)| { + format!("soft_limit: {}, hard_limit: {}", soft_limit, hard_limit,) + }) + .unwrap_or_default(); + panic!( - "Unable to create data file {} in current dir({:?}): {:?}. {}", + "Unable to create data file {} in current dir({:?}): {:?}. {}, {}", file.display(), std::env::current_dir(), e, mmap_msg, + limit_msg, ); }) .unwrap(); @@ -393,6 +399,7 @@ impl BucketStorage { mod test { use {super::*, tempfile::tempdir}; + #[cfg(target_os = "linux")] #[test] fn test_bucket_storage() { let tmpdir = tempdir().unwrap(); @@ -424,15 +431,16 @@ mod test { assert_eq!(storage.uid(ix), None); // test get_mmap_fd_stats - let (mmap_count, num_open_files, soft_limit, hard_limit) = - get_open_fd_stats().unwrap(); + let mmap_count = get_mmap_count().unwrap(); + let (soft_limit, hard_limit) = get_open_fd_limits().unwrap(); assert!(mmap_count > 0); - assert!(num_open_files > 0); assert!(soft_limit > 0); assert!(hard_limit > 0); } + #[cfg(target_os = "linux")] + #[ignore] #[test] fn test_time_mmap() { use std::time::Instant; @@ -446,21 +454,16 @@ mod test { let tmpdir = tempdir().unwrap(); let paths: Vec = vec![tmpdir.path().to_path_buf()]; assert!(!paths.is_empty()); - let s = - BucketStorage::new(Arc::new(paths), 1, 1, 1, Arc::default(), Arc::default()); + let s = BucketStorage::new(Arc::new(paths), 1, 1, 1, Arc::default(), Arc::default()); v.push(s); } // test get_mmap_fd_stats let start = Instant::now(); - let (mmap_count, num_open_files, soft_limit, hard_limit) = get_open_fd_stats().unwrap(); + let mmap_count = get_mmap_count().unwrap(); let duration = start.elapsed(); - println!( - "{} {} {} {}", - mmap_count, num_open_files, soft_limit, hard_limit - ); - + println!("{}", mmap_count); println!("Time elapsed is: {:?}", duration); } } diff --git a/core/src/system_monitor_service.rs b/core/src/system_monitor_service.rs index be9c65f59e0d5d..2748dc45c0dc2e 100644 --- a/core/src/system_monitor_service.rs +++ b/core/src/system_monitor_service.rs @@ -7,7 +7,7 @@ use num_enum::{IntoPrimitive, TryFromPrimitive}; #[cfg(target_os = "linux")] use std::{fs::File, io::BufReader}; use { - solana_bucket_map::bucket_map::get_open_fd_stats, + solana_bucket_map::bucket_map::get_mmap_count, solana_sdk::timing::AtomicInterval, std::{ collections::HashMap, @@ -832,24 +832,19 @@ impl SystemMonitorService { Self::report_cpuid_values(); } + #[cfg(target_os = "linux")] fn report_open_fd_stats() { - if let Some(( - curr_num_open_fd, - curr_mmap_count, - max_open_fd_soft_limit, - max_open_fd_hard_limit, - )) = get_open_fd_stats() - { + if let Some(curr_mmap_count) = get_mmap_count() { datapoint_info!( - "open-fd-stats", - ("number_open_files", curr_num_open_fd, i64), + "open-mmap-stats", ("number_mmap_files", curr_mmap_count, i64), - ("max_open_files_hard_limit", max_open_fd_hard_limit, i64), - ("max_open_files_soft_limit", max_open_fd_soft_limit, i64), ); } } + #[cfg(not(target_os = "linux"))] + fn report_open_fd_stats() {} + #[cfg(target_os = "linux")] fn process_disk_stats(disk_stats: &mut Option) { match read_disk_stats() { diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 9e3a973f07a71a..270f3e417bf602 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -1801,12 +1801,6 @@ dependencies = [ "libc", ] -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - [[package]] name = "histogram" version = "0.6.9" @@ -3265,21 +3259,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "procfs" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dfb6451c91904606a1abe93e83a8ec851f45827fa84273f256ade45dc095818" -dependencies = [ - "bitflags", - "byteorder 1.4.3", - "chrono", - "flate2", - "hex", - "lazy_static", - "rustix", -] - [[package]] name = "prost" version = "0.9.0" @@ -4357,7 +4336,6 @@ dependencies = [ "log", "memmap2", "modular-bitfield", - "procfs", "rand 0.7.3", "solana-measure", "solana-sdk 1.15.0", @@ -4491,7 +4469,6 @@ dependencies = [ "lru", "min-max-heap", "num_enum", - "procfs", "rand 0.7.3", "rand_chacha 0.2.2", "rayon", @@ -4500,6 +4477,7 @@ dependencies = [ "serde_derive", "solana-address-lookup-table-program", "solana-bloom", + "solana-bucket-map", "solana-client", "solana-entry", "solana-frozen-abi 1.15.0", From a8274defac7ce5e181966da94237c86455126029 Mon Sep 17 00:00:00 2001 From: haoran Date: Thu, 1 Dec 2022 18:23:48 +0000 Subject: [PATCH 18/21] sort deps --- core/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/Cargo.toml b/core/Cargo.toml index 0d72623222a290..b9034a513641d1 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -37,8 +37,8 @@ rayon = "1.5.3" serde = "1.0.144" serde_derive = "1.0.103" solana-address-lookup-table-program = { path = "../programs/address-lookup-table", version = "=1.15.0" } -solana-bucket-map = { path = "../bucket_map", version = "=1.15.0" } solana-bloom = { path = "../bloom", version = "=1.15.0" } +solana-bucket-map = { path = "../bucket_map", version = "=1.15.0" } solana-client = { path = "../client", version = "=1.15.0" } solana-entry = { path = "../entry", version = "=1.15.0" } solana-frozen-abi = { path = "../frozen-abi", version = "=1.15.0" } From 768ab6beef56127373e22a8386c196a535c8a2a6 Mon Sep 17 00:00:00 2001 From: haoran Date: Fri, 2 Dec 2022 15:21:28 +0000 Subject: [PATCH 19/21] report number of open files --- bucket_map/src/bucket_map.rs | 35 ++++++++++++++++++++++++++++-- bucket_map/src/bucket_storage.rs | 11 ++++++++-- core/src/system_monitor_service.rs | 13 ++++++----- 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/bucket_map/src/bucket_map.rs b/bucket_map/src/bucket_map.rs index 1b2fe2009672d4..698ec18f318874 100644 --- a/bucket_map/src/bucket_map.rs +++ b/bucket_map/src/bucket_map.rs @@ -210,7 +210,7 @@ pub fn get_mmap_count() -> Option { None } -/// Utility function to get open_fd limits +/// Utility function to get open files limits #[cfg(target_os = "linux")] pub fn get_open_fd_limits() -> Option<(usize, usize)> { let run = |cmd, args| -> Option { @@ -237,7 +237,38 @@ pub fn get_open_fd_limits() -> Option<(usize, usize)> { } #[cfg(not(target_os = "linux"))] -pub fn get_open_fd_limits() -> Option<(usize, usize, usize)> { +pub fn get_open_fd_limits() -> Option<(usize, usize)> { + None +} + +/// Utility function to get num of open file descriptors +#[cfg(target_os = "linux")] +pub fn get_num_open_fd() -> Option { + let pid = std::process::id(); + let fd_path = format!("/proc/{}/fd", pid); + let cmd = format!("ls -l {} | wc -l", fd_path); + + let output = std::process::Command::new("sh") + .args(["-c", &cmd]) + .output() + .unwrap(); + if output.status.success() { + let n: usize = std::str::from_utf8(&output.stdout) + .unwrap() + .split_whitespace() + .next() + .unwrap() + .parse() + .unwrap(); + + Some(n) + } else { + None + } +} + +#[cfg(not(target_os = "linux"))] +pub fn get_num_open_fd() -> Option { None } diff --git a/bucket_map/src/bucket_storage.rs b/bucket_map/src/bucket_storage.rs index bc6218ff365a52..a9e09e3293a0cd 100644 --- a/bucket_map/src/bucket_storage.rs +++ b/bucket_map/src/bucket_storage.rs @@ -1,6 +1,6 @@ use { crate::{ - bucket_map::{get_mmap_count, get_open_fd_limits}, + bucket_map::{get_mmap_count, get_num_open_fd, get_open_fd_limits}, bucket_stats::BucketStats, MaxSearch, }, @@ -288,6 +288,10 @@ impl BucketStorage { .map(|mmap_count| format!("current mmap_count: {}", mmap_count)) .unwrap_or_default(); + let open_fd_msg = get_num_open_fd() + .map(|open_fd| format!("current open_fd: {}", open_fd)) + .unwrap_or_default(); + let limit_msg = get_open_fd_limits() .map(|(soft_limit, hard_limit)| { format!("soft_limit: {}, hard_limit: {}", soft_limit, hard_limit,) @@ -295,10 +299,11 @@ impl BucketStorage { .unwrap_or_default(); panic!( - "Unable to create data file {} in current dir({:?}): {:?}. {}, {}", + "Unable to create data file {} in current dir({:?}): {:?}. {}, {}, {}", file.display(), std::env::current_dir(), e, + open_fd_msg, mmap_msg, limit_msg, ); @@ -432,9 +437,11 @@ mod test { // test get_mmap_fd_stats let mmap_count = get_mmap_count().unwrap(); + let open_fd = get_num_open_fd().unwrap(); let (soft_limit, hard_limit) = get_open_fd_limits().unwrap(); assert!(mmap_count > 0); + assert!(open_fd > 0); assert!(soft_limit > 0); assert!(hard_limit > 0); } diff --git a/core/src/system_monitor_service.rs b/core/src/system_monitor_service.rs index 2748dc45c0dc2e..42f0d5940c1547 100644 --- a/core/src/system_monitor_service.rs +++ b/core/src/system_monitor_service.rs @@ -7,7 +7,7 @@ use num_enum::{IntoPrimitive, TryFromPrimitive}; #[cfg(target_os = "linux")] use std::{fs::File, io::BufReader}; use { - solana_bucket_map::bucket_map::get_mmap_count, + solana_bucket_map::bucket_map::{get_mmap_count, get_num_open_fd}, solana_sdk::timing::AtomicInterval, std::{ collections::HashMap, @@ -835,10 +835,13 @@ impl SystemMonitorService { #[cfg(target_os = "linux")] fn report_open_fd_stats() { if let Some(curr_mmap_count) = get_mmap_count() { - datapoint_info!( - "open-mmap-stats", - ("number_mmap_files", curr_mmap_count, i64), - ); + if let Some(curr_open_fd) = get_num_open_fd() { + datapoint_info!( + "open-mmap-stats", + ("number_mmap_files", curr_mmap_count, i64), + ("number_open_fd", curr_open_fd, i64), + ); + } } } From 46d4becde23c2b0f8dde41958c5776995062f74b Mon Sep 17 00:00:00 2001 From: haoran Date: Fri, 2 Dec 2022 15:24:38 +0000 Subject: [PATCH 20/21] comments --- bucket_map/src/bucket_map.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bucket_map/src/bucket_map.rs b/bucket_map/src/bucket_map.rs index 698ec18f318874..a3811a93484f60 100644 --- a/bucket_map/src/bucket_map.rs +++ b/bucket_map/src/bucket_map.rs @@ -210,7 +210,7 @@ pub fn get_mmap_count() -> Option { None } -/// Utility function to get open files limits +/// Utility function to get open files limits on the system #[cfg(target_os = "linux")] pub fn get_open_fd_limits() -> Option<(usize, usize)> { let run = |cmd, args| -> Option { @@ -241,7 +241,7 @@ pub fn get_open_fd_limits() -> Option<(usize, usize)> { None } -/// Utility function to get num of open file descriptors +/// Utility function to get number of open file descriptors for current process #[cfg(target_os = "linux")] pub fn get_num_open_fd() -> Option { let pid = std::process::id(); From 4f0fc12af16f44521a178bef4594530190c6b521 Mon Sep 17 00:00:00 2001 From: Haoran Yi Date: Mon, 5 Dec 2022 08:19:27 -0600 Subject: [PATCH 21/21] fix merge error --- bucket_map/src/bucket_storage.rs | 5 +++-- core/src/system_monitor_service.rs | 2 -- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/bucket_map/src/bucket_storage.rs b/bucket_map/src/bucket_storage.rs index a9e09e3293a0cd..d528f26fc96d80 100644 --- a/bucket_map/src/bucket_storage.rs +++ b/bucket_map/src/bucket_storage.rs @@ -435,7 +435,7 @@ mod test { assert!(storage.is_free(ix)); assert_eq!(storage.uid(ix), None); - // test get_mmap_fd_stats + // test get_open_fd stats let mmap_count = get_mmap_count().unwrap(); let open_fd = get_num_open_fd().unwrap(); let (soft_limit, hard_limit) = get_open_fd_limits().unwrap(); @@ -446,6 +446,8 @@ mod test { assert!(hard_limit > 0); } + /// bench get_mmap_count + /// 2M mmaps takes 1.5s #[cfg(target_os = "linux")] #[ignore] #[test] @@ -465,7 +467,6 @@ mod test { v.push(s); } - // test get_mmap_fd_stats let start = Instant::now(); let mmap_count = get_mmap_count().unwrap(); let duration = start.elapsed(); diff --git a/core/src/system_monitor_service.rs b/core/src/system_monitor_service.rs index 42f0d5940c1547..b90e36acf68e09 100644 --- a/core/src/system_monitor_service.rs +++ b/core/src/system_monitor_service.rs @@ -30,8 +30,6 @@ const SAMPLE_INTERVAL_OS_NETWORK_LIMITS_MS: u64 = MS_PER_H; const SAMPLE_INTERVAL_MEM_MS: u64 = 5 * MS_PER_S; const SAMPLE_INTERVAL_CPU_MS: u64 = 10 * MS_PER_S; const SAMPLE_INTERVAL_DISK_MS: u64 = 5 * MS_PER_S; -const SAMPLE_INTERVAL_OPEN_FD_MS: u64 = 30 * MS_PER_S; -const SAMPLE_INTERVAL_DISK_MS: u64 = 5 * MS_PER_S; const SAMPLE_INTERVAL_OPEN_FD_MS: u64 = 60 * MS_PER_S; const SLEEP_INTERVAL: Duration = Duration::from_millis(500);