Set receive_window per quic connection (solana-labs#26936)

This change sets the receive_window for non-staked node to 1 * PACKET_DATA_SIZE, and maps the staked nodes's connection's receive_window between 1.2 * PACKET_DATA_SIZE to 10 * PACKET_DATA_SIZE based on the stakes. The changes is based on Quinn library change to support per connection receive_window tweak at the server side. quinn-rs/quinn#1393
jeffwashington · Aug 9, 2022 · a69470f · a69470f
1 parent f7c6901
commit a69470f
Show file tree

Hide file tree

Showing 7 changed files with 263 additions and 32 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/core/src/staked_nodes_updater_service.rs b/core/src/staked_nodes_updater_service.rs
@@ -36,11 +36,15 @@ impl StakedNodesUpdaterService {
                     let mut new_ip_to_stake = HashMap::new();
                     let mut new_id_to_stake = HashMap::new();
                     let mut total_stake = 0;
+                    let mut max_stake: u64 = 0;
+                    let mut min_stake: u64 = u64::MAX;
                     if Self::try_refresh_stake_maps(
                         &mut last_stakes,
                         &mut new_ip_to_stake,
                         &mut new_id_to_stake,
                         &mut total_stake,
+                        &mut max_stake,
+                        &mut min_stake,
                         &bank_forks,
                         &cluster_info,
                     ) {
@@ -61,16 +65,21 @@ impl StakedNodesUpdaterService {
         ip_to_stake: &mut HashMap<IpAddr, u64>,
         id_to_stake: &mut HashMap<Pubkey, u64>,
         total_stake: &mut u64,
+        max_stake: &mut u64,
+        min_stake: &mut u64,
         bank_forks: &RwLock<BankForks>,
         cluster_info: &ClusterInfo,
     ) -> bool {
         if last_stakes.elapsed() > IP_TO_STAKE_REFRESH_DURATION {
             let root_bank = bank_forks.read().unwrap().root_bank();
             let staked_nodes = root_bank.staked_nodes();
-            *total_stake = staked_nodes
-                .iter()
-                .map(|(_pubkey, stake)| stake)
-                .sum::<u64>();
+
+            for stake in staked_nodes.values() {
+                *total_stake += stake;
+                *max_stake = *stake.max(max_stake);
+                *min_stake = *stake.min(min_stake);
+            }
+
             *id_to_stake = cluster_info
                 .tvu_peers()
                 .into_iter()

diff --git a/programs/bpf/Cargo.lock b/programs/bpf/Cargo.lock
diff --git a/sdk/src/quic.rs b/sdk/src/quic.rs
@@ -12,3 +12,15 @@ pub const QUIC_KEEP_ALIVE_MS: u64 = 1_000;
 // applications. Different applications vary, but most seem to
 // be in the 30-60 second range
 pub const QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS: u64 = 60_000;
+
+/// The receive window for QUIC connection from unstaked nodes is
+/// set to this ratio times [`solana_sdk::packet::PACKET_DATA_SIZE`]
+pub const QUIC_UNSTAKED_RECEIVE_WINDOW_RATIO: u64 = 1;
+
+/// The receive window for QUIC connection from minimum staked nodes is
+/// set to this ratio times [`solana_sdk::packet::PACKET_DATA_SIZE`]
+pub const QUIC_MIN_STAKED_RECEIVE_WINDOW_RATIO: u64 = 2;
+
+/// The receive window for QUIC connection from maximum staked nodes is
+/// set to this ratio times [`solana_sdk::packet::PACKET_DATA_SIZE`]
+pub const QUIC_MAX_STAKED_RECEIVE_WINDOW_RATIO: u64 = 10;
diff --git a/streamer/Cargo.toml b/streamer/Cargo.toml
@@ -21,7 +21,9 @@ nix = "0.24.2"
 pem = "1.0.2"
 percentage = "0.1.0"
 pkcs8 = { version = "0.8.0", features = ["alloc"] }
-quinn = "0.8.3"
+quinn = {git = "https://github.com/quinn-rs/quinn.git", branch = "0.8.x", commit = "37c19743cc881cf71369946d572849d5d2ffc3fd"}
+quinn-proto = {git = "https://github.com/quinn-rs/quinn.git", branch = "0.8.x", commit = "37c19743cc881cf71369946d572849d5d2ffc3fd"}
+
 rand = "0.7.0"
 rcgen = "0.9.2"
 rustls = { version = "0.20.6", features = ["dangerous_configuration"] }