Tool to tune system parameters like PoH service priority (#7155)

* New daemon to tune system parameters like PoH service priority

* fixes for Linux

* integrate with poh_service

* fixes

* address review comments

* remove `dead_code` directive

Cargo.toml

@@ -44,6 +44,7 @@ members = [
     "runtime",
     "sdk",
     "sdk-c",
+    "sys-tuner",
     "upload-perf",
     "net-utils",
     "fixed-buf",

core/Cargo.toml

@@ -60,6 +60,7 @@ solana-stake-program = { path = "../programs/stake", version = "0.22.0" }
 solana-storage-program = { path = "../programs/storage", version = "0.22.0" }
 solana-vote-program = { path = "../programs/vote", version = "0.22.0" }
 solana-vote-signer = { path = "../vote-signer", version = "0.22.0" }
+solana-sys-tuner = { path = "../sys-tuner", version = "0.22.0" }
 symlink = "0.1.0"
 sys-info = "0.5.8"
 tempfile = "3.1.0"

core/src/poh_service.rs

@@ -3,6 +3,7 @@
 use crate::poh_recorder::PohRecorder;
 use core_affinity;
 use solana_sdk::poh_config::PohConfig;
+use solana_sys_tuner;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
 use std::thread::{self, sleep, Builder, JoinHandle};
@@ -30,6 +31,7 @@ impl PohService {
         let tick_producer = Builder::new()
             .name("solana-poh-service-tick_producer".to_string())
             .spawn(move || {
+                solana_sys_tuner::request_realtime_poh();
                 if poh_config.hashes_per_tick.is_none() {
                     if poh_config.target_tick_count.is_none() {
                         Self::sleepy_tick_producer(poh_recorder, &poh_config, &poh_exit_);

net/remote/remote-node.sh

@@ -129,6 +129,9 @@ cat >> ~/solana/on-reboot <<EOF
   # shellcheck source=/dev/null
   SUDO_OK=1 source scripts/tune-system.sh
 
+  sudo RUST_LOG=info ~solana/.cargo/bin/solana-sys-tuner > sys-tuner.log 2>&1 &
+  echo \$! > sys-tuner.pid
+
   (
     sudo SOLANA_METRICS_CONFIG="$SOLANA_METRICS_CONFIG" scripts/oom-monitor.sh
   ) > oom-monitor.log 2>&1 &

scripts/cargo-install-all.sh

@@ -82,6 +82,7 @@ BINS=(
   solana-log-analyzer
   solana-net-shaper
   solana-archiver
+  solana-sys-tuner
   solana-validator
 )
 

sys-tuner/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+authors = ["Solana Maintainers <[email protected]>"]
+edition = "2018"
+name = "solana-sys-tuner"
+description = "The solana cluster system tuner daemon"
+version = "0.22.0"
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+publish = true
+
+[dependencies]
+clap = "2.33.0"
+log = "0.4.8"
+semver = "0.9.0"
+solana-logger = { path = "../logger", version = "0.22.0" }
+
+[target."cfg(unix)".dependencies]
+unix_socket2 = "0.5.4"
+users = "0.9.1"
+nix = "0.16.0"
+
+[lib]
+name = "solana_sys_tuner"
+
+[[bin]]
+name = "solana-sys-tuner"
+path = "src/main.rs"

sys-tuner/src/lib.rs

@@ -0,0 +1,10 @@
+use log::*;
+use unix_socket::UnixStream;
+
+pub const SOLANA_SYS_TUNER_PATH: &str = "/tmp/solana-sys-tuner";
+
+pub fn request_realtime_poh() {
+    info!("Sending tuning request");
+    let status = UnixStream::connect(SOLANA_SYS_TUNER_PATH);
+    info!("Tuning request status {:?}", status);
+}

sys-tuner/src/main.rs

@@ -0,0 +1,113 @@
+use log::*;
+use std::{fs, io};
+
+use solana_sys_tuner::SOLANA_SYS_TUNER_PATH;
+
+#[cfg(unix)]
+use unix_socket::UnixListener;
+
+#[cfg(target_os = "linux")]
+use std::fs::DirEntry;
+#[cfg(target_os = "linux")]
+use std::path::Path;
+
+#[cfg(target_os = "linux")]
+fn find_pid<P: AsRef<Path>, F>(name: &str, path: P, processor: F) -> Option<u64>
+where
+    F: Fn(&DirEntry) -> Option<u64>,
+{
+    for entry in fs::read_dir(path).expect("Failed to read /proc folder") {
+        if let Ok(dir) = entry {
+            let mut path = dir.path();
+            path.push("comm");
+            if let Ok(comm) = fs::read_to_string(path.as_path()) {
+                if comm.starts_with(name) {
+                    if let Some(pid) = processor(&dir) {
+                        return Some(pid);
+                    }
+                }
+            }
+        }
+    }
+
+    None
+}
+
+#[cfg(target_os = "linux")]
+fn tune_system() {
+    use std::process::Command;
+    use std::str::from_utf8;
+
+    if let Some(pid) = find_pid("solana-validato", "/proc", |dir| {
+        let mut path = dir.path();
+        path.push("task");
+        find_pid("solana-poh-serv", path, |dir1| {
+            if let Ok(pid) = dir1.file_name().into_string() {
+                pid.parse::<u64>().ok()
+            } else {
+                None
+            }
+        })
+    }) {
+        info!("POH thread PID is {}", pid);
+        let pid = format!("{}", pid);
+        let output = Command::new("chrt")
+            .args(&["-r", "-p", "99", pid.as_str()])
+            .output()
+            .expect("Expected to set priority of thread");
+        if output.status.success() {
+            info!("Done setting thread priority");
+        } else {
+            error!("chrt stderr: {}", from_utf8(&output.stderr).unwrap_or("?"));
+        }
+    } else {
+        error!("Could not find pid for POH thread");
+    }
+}
+
+#[cfg(any(not(unix), target_os = "macos"))]
+fn tune_system() {}
+
+#[allow(dead_code)]
+#[cfg(target_os = "linux")]
+fn set_socket_permissions() {
+    if let Some(user) = users::get_user_by_name("solana") {
+        let uid = format!("{}", user.uid());
+        info!("UID for solana is {}", uid);
+        nix::unistd::chown(
+            SOLANA_SYS_TUNER_PATH,
+            Some(nix::unistd::Uid::from_raw(user.uid())),
+            None,
+        )
+        .expect("Expected to change UID of the socket file");
+    } else {
+        error!("Could not find UID for solana user");
+    }
+}
+
+#[cfg(any(not(unix), target_os = "macos"))]
+fn set_socket_permissions() {}
+
+fn main() {
+    solana_logger::setup();
+    if let Err(e) = fs::remove_file(SOLANA_SYS_TUNER_PATH) {
+        if e.kind() != io::ErrorKind::NotFound {
+            panic!("Failed to remove stale socket file: {:?}", e)
+        }
+    }
+
+    let listener =
+        UnixListener::bind(SOLANA_SYS_TUNER_PATH).expect("Failed to bind to the socket file");
+
+    set_socket_permissions();
+
+    info!("Waiting for tuning requests");
+    for stream in listener.incoming() {
+        if stream.is_ok() {
+            info!("Tuning the system now");
+            tune_system();
+        }
+    }
+
+    info!("exiting");
+}