From 5a5383917fe90302796a06c90dfcb3ebb4078f84 Mon Sep 17 00:00:00 2001 From: Will Chandler Date: Thu, 26 Sep 2024 10:37:40 -0400 Subject: [PATCH 1/3] Add doc comment to set_interface_mtu Fix comment type on `set_interface_mtu`. --- illumos-utils/src/ipadm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/illumos-utils/src/ipadm.rs b/illumos-utils/src/ipadm.rs index d5e0053ba9..218fb2815b 100644 --- a/illumos-utils/src/ipadm.rs +++ b/illumos-utils/src/ipadm.rs @@ -141,7 +141,7 @@ impl Ipadm { Ok(Self::addrobj_addr(addrobj)?.is_some()) } - // Set MTU to 9000 on both IPv4 and IPv6 + /// Set MTU to 9000 on both IPv4 and IPv6 pub fn set_interface_mtu(datalink: &str) -> Result<(), ExecutionError> { let mut cmd = std::process::Command::new(PFEXEC); let cmd = cmd.args(&[ From c7a2b55998abe6229505d5f25fdf4c5efbdb1acc Mon Sep 17 00:00:00 2001 From: Will Chandler Date: Thu, 26 Sep 2024 11:26:42 -0400 Subject: [PATCH 2/3] Enlarge TCP recv_buf to improve throughput Image uploads performed via the web console are 3-4x slower than uploads performed via the Oxide CLI[0]. We found that the CLI creates 8 separate TCP connections to upload the image chunks, while the console uses HTTP/2 to multiplex a single TCP connection six ways. The default TCP `recv_buf` size on Helios is 128 KB, which limits window size and therefore the number of packets that can be sent in parallel. By increasing this value to 1 MB, we can increase single-connection throughput by ~3x, bringing console performance to rough parity with the CLI. This does increase the amount of memory a potential DoS attack could consume, but 1 MB is still quite small relative to the total resources available on a compute sled. While we're at it, also update the TCP congestion control algorithm to `cubic` from its default value of `sunreno`, which may also help improve throughput. Closes https://github.com/oxidecomputer/omicron/issues/6601 [0] https://github.com/oxidecomputer/console/issues/2096 Assisted-by: David Crespo --- illumos-utils/src/ipadm.rs | 32 ++++++++++++++++++++++++++++++++ zone-setup/src/bin/zone-setup.rs | 11 +++++++++++ 2 files changed, 43 insertions(+) diff --git a/illumos-utils/src/ipadm.rs b/illumos-utils/src/ipadm.rs index 218fb2815b..1633f31953 100644 --- a/illumos-utils/src/ipadm.rs +++ b/illumos-utils/src/ipadm.rs @@ -197,4 +197,36 @@ impl Ipadm { Self::ensure_ip_addrobj_exists(&addrobj, AddrObjType::DHCP)?; Ok(()) } + + /// Set TCP recv_buf to 1 MB. + pub fn set_tcp_recv_buf() -> Result<(), ExecutionError> { + let mut cmd = std::process::Command::new(PFEXEC); + let cmd = cmd.args(&[ + IPADM, + "set-prop", + "-t", + "-p", + "recv_buf=1000000", + "tcp", + ]); + execute(cmd)?; + + Ok(()) + } + + /// Set TCP congestion control algorithm to `cubic`. + pub fn set_tcp_congestion_control() -> Result<(), ExecutionError> { + let mut cmd = std::process::Command::new(PFEXEC); + let cmd = cmd.args(&[ + IPADM, + "set-prop", + "-t", + "-p", + "congestion_control=cubic", + "tcp", + ]); + execute(cmd)?; + + Ok(()) + } } diff --git a/zone-setup/src/bin/zone-setup.rs b/zone-setup/src/bin/zone-setup.rs index 97bb2af60d..4441431605 100644 --- a/zone-setup/src/bin/zone-setup.rs +++ b/zone-setup/src/bin/zone-setup.rs @@ -563,6 +563,17 @@ async fn common_nw_set_up( Ipadm::set_interface_mtu(&datalink) .with_context(|| format!("failed to set MTU on datalink {datalink}"))?; + info!( + log, "Setting TCP recv_buf size to 1 MB"; + ); + Ipadm::set_tcp_recv_buf().context("failed to set TCP recv_buf")?; + + info!( + log, "Setting TCP congestion control algorithm to cubic"; + ); + Ipadm::set_tcp_congestion_control() + .context("failed to set TCP congestion_control")?; + if static_addrs.is_empty() { info!( log, From 552d1c584b9025c67f6386d29da2abe6c26c8fcd Mon Sep 17 00:00:00 2001 From: Will Chandler Date: Mon, 30 Sep 2024 16:38:11 -0400 Subject: [PATCH 3/3] Comment on why we're increasing recv_buf --- illumos-utils/src/ipadm.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/illumos-utils/src/ipadm.rs b/illumos-utils/src/ipadm.rs index 1633f31953..ff99cd7a26 100644 --- a/illumos-utils/src/ipadm.rs +++ b/illumos-utils/src/ipadm.rs @@ -201,6 +201,14 @@ impl Ipadm { /// Set TCP recv_buf to 1 MB. pub fn set_tcp_recv_buf() -> Result<(), ExecutionError> { let mut cmd = std::process::Command::new(PFEXEC); + + // This is to improve single-connection throughput on large uploads + // from clients, e.g., images. Modern browsers will almost always use + // HTTP/2, which will multiplex concurrent writes to the same host over + // a single TCP connection. The small default receive window size is a + // major bottleneck, see + // https://github.com/oxidecomputer/console/issues/2096 for further + // details. let cmd = cmd.args(&[ IPADM, "set-prop",