Self assembling zones should set their own MTU

Bump crucible rev to pick up: - Self assembling zones must set their own MTU - fix build break caused by merging oxidecomputer#801 - Issue extent flushes in parallel - Add Logger Option to Volume construct method - Update Rust crate libc to 0.2.147 - Update Rust crate percent-encoding to 2.3 - Retry jobs until they succeed - Reorder select arms so pings can't be starved out - Treat a skipped IO like an error IO for ACK results. - Retry pantry requests - Remove panics and asserts in dummy tests - Update Rust crate csv to 1.2.2 - Update Rust crate reedline to 0.21.0 - Set open file resource limit to the max - Update Rust crate ringbuffer to 0.14 - DTrace meet cmon - Widen assert values to u128 to deal with u64::MAX - Change size_to_validate from usize to u64 - Turn on live-repair test in CI - Increase flush_timeout for some tests, collect cores - Update to latest dropshot Bump propolis rev to pick up: - Self assembling zones must set their own MTU - Bump crucible rev to latest - Make the propolis zone self-assembling - Flesh out more PIIX3-PM to suppress log gripes - Bump crucible rev to latest - Restructure PM-timer under PIIX3 device - Fix inventory handling for nested child entities - Centralize vmm-data interface into bhyve_api - Clean up PCI device classes - Update openssl dep to 0.10.55 - Allow propolis-standalone to use VMM reservoir - only allow one request to reboot to be enqueued at a time Nexus is currently setting the MTU inside self-assembling zones, but this goes against the idea that self-assembling zones perform their own configuration. Remove the code in `RunningZone::boot` that performs commands on self-assembling zones, and set the MTU of $DATALINK in the Clickhouse and CockroachDB method scripts.
jmpesp · Jul 7, 2023 · 2e3e9fd · 2e3e9fd
1 parent 0325e96
commit 2e3e9fd
Show file tree

Hide file tree

Showing 5 changed files with 71 additions and 75 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -151,10 +151,10 @@ cookie = "0.16"
 criterion = { version = "0.5.1", features = [ "async_tokio" ] }
 crossbeam = "0.8"
 crossterm = { version = "0.26.1", features = ["event-stream"] }
-crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "df7e274edf0a1df287770e9567b27676f3ae4cc5" }
-crucible-client-types = { git = "https://github.com/oxidecomputer/crucible", rev = "df7e274edf0a1df287770e9567b27676f3ae4cc5" }
-crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "df7e274edf0a1df287770e9567b27676f3ae4cc5" }
-crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "df7e274edf0a1df287770e9567b27676f3ae4cc5" }
+crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "c574ff1232aa66eb60069795fe45f0c47f5da51d" }
+crucible-client-types = { git = "https://github.com/oxidecomputer/crucible", rev = "c574ff1232aa66eb60069795fe45f0c47f5da51d" }
+crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "c574ff1232aa66eb60069795fe45f0c47f5da51d" }
+crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "c574ff1232aa66eb60069795fe45f0c47f5da51d" }
 curve25519-dalek = "3"
 datatest-stable = "0.1.3"
 display-error-chain = "0.1.1"
@@ -264,9 +264,9 @@ pretty-hex = "0.3.0"
 proc-macro2 = "1.0"
 progenitor = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" }
 progenitor-client = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" }
-bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "04a275736e9f3316de6bf2a4077d03acfa4e2cdf" }
-propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "04a275736e9f3316de6bf2a4077d03acfa4e2cdf", features = [ "generated-migration" ] }
-propolis-server = { git = "https://github.com/oxidecomputer/propolis", rev = "04a275736e9f3316de6bf2a4077d03acfa4e2cdf", default-features = false, features = ["mock-only"] }
+bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "9e12522341048d7b8c38394f0ddeb18083685c2f" }
+propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "9e12522341048d7b8c38394f0ddeb18083685c2f", features = [ "generated-migration" ] }
+propolis-server = { git = "https://github.com/oxidecomputer/propolis", rev = "9e12522341048d7b8c38394f0ddeb18083685c2f", default-features = false, features = ["mock-only"] }
 #propolis-server = { path = "../propolis/bin/propolis-server" }
 proptest = "1.2.0"
 quote = "1.0"

diff --git a/illumos-utils/src/running_zone.rs b/illumos-utils/src/running_zone.rs
@@ -372,78 +372,60 @@ impl RunningZone {
             }
         })?;
 
-        // Pull the zone ID.
+        // If the zone is self-assembling, then SMF service(s) inside the zone
+        // will be creating the listen address for the zone's service(s),
+        // setting the appropriate ifprop MTU, and so on. The idea behind
+        // self-assembling zones is that once they boot there should be *no*
+        // zlogin required.
+
+        // Use the zone ID in order to check if /var/svc/profile/site.xml
+        // exists.
         let id = Zones::id(&zone.name)
             .await?
             .ok_or_else(|| BootError::NoZoneId { zone: zone.name.clone() })?;
         let site_profile_xml_exists =
             std::path::Path::new(&zone.site_profile_xml_path()).exists();
-        let running_zone = RunningZone { id: Some(id), inner: zone };
 
-        // Make sure the control vnic has an IP MTU of 9000 inside the zone
-        const CONTROL_VNIC_MTU: usize = 9000;
-        let vnic = running_zone.inner.control_vnic.name().to_string();
-
-        // If the zone is self-assembling, then SMF service(s) inside the zone
-        // will be creating the listen address for the zone's service(s). This
-        // will create IP interfaces, and means that `create-if` here will fail
-        // due to the interface already existing. Checking the output of
-        // `show-if` is also problematic due to TOCTOU. Use the check for the
-        // existence of site.xml, which means the zone is performing this
-        // self-assembly, and skip create-if if so.
+        let running_zone = RunningZone { id: Some(id), inner: zone };
 
         if !site_profile_xml_exists {
-            let args = vec![
-                IPADM.to_string(),
-                "create-if".to_string(),
-                "-t".to_string(),
-                vnic.clone(),
+            // If the zone is not self-assembling, make sure the control vnic
+            // has an IP MTU of 9000 inside the zone.
+            const CONTROL_VNIC_MTU: usize = 9000;
+            let vnic = running_zone.inner.control_vnic.name().to_string();
+
+            let commands = vec![
+                vec![
+                    IPADM.to_string(),
+                    "create-if".to_string(),
+                    "-t".to_string(),
+                    vnic.clone(),
+                ],
+                vec![
+                    IPADM.to_string(),
+                    "set-ifprop".to_string(),
+                    "-t".to_string(),
+                    "-p".to_string(),
+                    format!("mtu={}", CONTROL_VNIC_MTU),
+                    "-m".to_string(),
+                    "ipv4".to_string(),
+                    vnic.clone(),
+                ],
+                vec![
+                    IPADM.to_string(),
+                    "set-ifprop".to_string(),
+                    "-t".to_string(),
+                    "-p".to_string(),
+                    format!("mtu={}", CONTROL_VNIC_MTU),
+                    "-m".to_string(),
+                    "ipv6".to_string(),
+                    vnic,
+                ],
             ];
 
-            running_zone.run_cmd(args)?;
-        } else {
-            // If the zone is self-assembling, then it's possible that the IP
-            // interface does not exist yet because it has not been brought up
-            // by the software in the zone. Run `create-if` here, but eat the
-            // error if there is one: this is safe unless the software that's
-            // part of self-assembly inside the zone is also trying to run
-            // `create-if` (instead of `create-addr`), and required for the
-            // `set-ifprop` commands below to pass.
-            let args = vec![
-                IPADM.to_string(),
-                "create-if".to_string(),
-                "-t".to_string(),
-                vnic.clone(),
-            ];
-
-            let _result = running_zone.run_cmd(args);
-        }
-
-        let commands = vec![
-            vec![
-                IPADM.to_string(),
-                "set-ifprop".to_string(),
-                "-t".to_string(),
-                "-p".to_string(),
-                format!("mtu={}", CONTROL_VNIC_MTU),
-                "-m".to_string(),
-                "ipv4".to_string(),
-                vnic.clone(),
-            ],
-            vec![
-                IPADM.to_string(),
-                "set-ifprop".to_string(),
-                "-t".to_string(),
-                "-p".to_string(),
-                format!("mtu={}", CONTROL_VNIC_MTU),
-                "-m".to_string(),
-                "ipv6".to_string(),
-                vnic,
-            ],
-        ];
-
-        for args in &commands {
-            running_zone.run_cmd(args)?;
+            for args in &commands {
+                running_zone.run_cmd(args)?;
+            }
         }
 
         Ok(running_zone)

diff --git a/package-manifest.toml b/package-manifest.toml
@@ -241,21 +241,21 @@ only_for_targets.image = "standard"
 # 3. Use source.type = "manual" instead of "prebuilt"
 source.type = "prebuilt"
 source.repo = "crucible"
-source.commit = "df7e274edf0a1df287770e9567b27676f3ae4cc5"
+source.commit = "c574ff1232aa66eb60069795fe45f0c47f5da51d"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image/<commit>/crucible.sha256.txt
-source.sha256 = "c1a7996dfdded9476d5a595f16fcd07e902bb02f2ec0858bb3625f9b2195a919"
+source.sha256 = "72363fbbfde15689bdbafee53e415ca994903644a9fdbb33ca0bc72d65927f0f"
 output.type = "zone"
 
 [package.crucible-pantry]
 service_name = "crucible_pantry"
 only_for_targets.image = "standard"
 source.type = "prebuilt"
 source.repo = "crucible"
-source.commit = "df7e274edf0a1df287770e9567b27676f3ae4cc5"
+source.commit = "c574ff1232aa66eb60069795fe45f0c47f5da51d"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image/<commit>/crucible-pantry.sha256.txt
-source.sha256 = "23611f3770a797b829c340e6dba25a4898678ece79db8534e06f76e438214e4c"
+source.sha256 = "4645454e3af72ee68bc3916042f78deaac44a8c07bb6c1a1e5766b83f5326e07"
 output.type = "zone"
 
 # Refer to
@@ -266,10 +266,10 @@ service_name = "propolis-server"
 only_for_targets.image = "standard"
 source.type = "prebuilt"
 source.repo = "propolis"
-source.commit = "21ac8a9f5005f96caa384e3de0bd38283fc0188f"
+source.commit = "9e12522341048d7b8c38394f0ddeb18083685c2f"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/image/<commit>/propolis-server.sha256.txt
-source.sha256 = "2a7b4bfa6d2b13714f68ec0095419218257ab584e763faac1d34baf38c8b09de"
+source.sha256 = "084df2cf5866a17b5467dccdaddef5088cc021a1ee0eb3bea99472bcf67ea7fa"
 output.type = "zone"
 
 [package.maghemite]

diff --git a/smf/clickhouse/method_script.sh b/smf/clickhouse/method_script.sh
@@ -17,6 +17,13 @@ if [[ $DATALINK == unknown ]] || [[ $GATEWAY == unknown ]]; then
     exit "$SMF_EXIT_ERR_CONFIG"
 fi
 
+# TODO remove when https://github.com/oxidecomputer/stlouis/issues/435 is addressed
+ipadm delete-if "$DATALINK" || true
+ipadm create-if -t "$DATALINK"
+
+ipadm set-ifprop -t -p mtu=9000 -m ipv4 "$DATALINK"
+ipadm set-ifprop -t -p mtu=9000 -m ipv6 "$DATALINK"
+
 ipadm show-addr "$DATALINK/ll" || ipadm create-addr -t -T addrconf "$DATALINK/ll"
 ipadm show-addr "$DATALINK/omicron6"  || ipadm create-addr -t -T static -a "$LISTEN_ADDR" "$DATALINK/omicron6"
 route get -inet6 default -inet6 "$GATEWAY" || route add -inet6 default -inet6 "$GATEWAY"

diff --git a/smf/cockroachdb/method_script.sh b/smf/cockroachdb/method_script.sh
@@ -17,6 +17,13 @@ if [[ $DATALINK == unknown ]] || [[ $GATEWAY == unknown ]]; then
     exit "$SMF_EXIT_ERR_CONFIG"
 fi
 
+# TODO remove when https://github.com/oxidecomputer/stlouis/issues/435 is addressed
+ipadm delete-if "$DATALINK" || true
+ipadm create-if -t "$DATALINK"
+
+ipadm set-ifprop -t -p mtu=9000 -m ipv4 "$DATALINK"
+ipadm set-ifprop -t -p mtu=9000 -m ipv6 "$DATALINK"
+
 ipadm show-addr "$DATALINK/ll" || ipadm create-addr -t -T addrconf "$DATALINK/ll"
 ipadm show-addr "$DATALINK/omicron6"  || ipadm create-addr -t -T static -a "$LISTEN_ADDR" "$DATALINK/omicron6"
 route get -inet6 default -inet6 "$GATEWAY" || route add -inet6 default -inet6 "$GATEWAY"