Use net0 vnic for maghemite (#1342)

sled-agent/src/bootstrap/server.rs

@@ -13,13 +13,11 @@ use super::trust_quorum::ShareDistribution;
 use super::views::Response;
 use super::views::ResponseEnvelope;
 use crate::bootstrap::maghemite;
+use crate::common::underlay;
 use crate::config::Config as SledConfig;
-use crate::illumos::addrobj::AddrObject;
-use crate::illumos::dladm::VnicSource;
 use crate::sp::AsyncReadWrite;
 use crate::sp::SpHandle;
 use crate::sp::SprocketsRole;
-use crate::zone::Zones;
 use slog::Drain;
 use slog::Logger;
 use std::net::Ipv6Addr;
@@ -67,27 +65,21 @@ impl Server {
             debug!(log, "registered DTrace probes");
         }
 
-        // Ensure we have a link-local inet6 address.
-        let link = sled_config
-            .get_link()
-            .map_err(|err| format!("Failed to find physical link: {err}"))?;
-
-        let mg_interface = AddrObject::new(link.name(), "linklocal")
-            .expect("unexpected failure creating AddrObject");
-        Zones::ensure_has_link_local_v6_address(None, &mg_interface).map_err(
-            |err| {
-                format!(
-                    "Failed to ensure link-local address for {}: {}",
-                    mg_interface, err
-                )
-            },
-        )?;
+        // Find address objects to pass to maghemite.
+        let mg_addr_objs = underlay::find_nics().map_err(|err| {
+            format!("Failed to find address objects for maghemite: {err}")
+        })?;
+        if mg_addr_objs.is_empty() {
+            return Err(
+                "underlay::find_nics() returned 0 address objects".to_string()
+            );
+        }
 
         // Turn on the maghemite routing service.
         // TODO-correctness Eventually we need mg-ddm to listen on multiple
         // interfaces (link-local addresses of both NICs).
         info!(log, "Starting mg-ddm service");
-        maghemite::enable_mg_ddm_service(log.clone(), mg_interface)
+        maghemite::enable_mg_ddm_service(log.clone(), mg_addr_objs[0].clone())
             .await
             .map_err(|err| format!("Failed to start mg-ddm: {err}"))?;
 

sled-agent/src/common/mod.rs

@@ -6,4 +6,5 @@
 
 pub mod disk;
 pub mod instance;
+pub mod underlay;
 pub mod vlan;

sled-agent/src/common/underlay.rs

@@ -0,0 +1,48 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Finding the underlay network physical links and address objects.
+
+use crate::illumos::addrobj;
+use crate::illumos::addrobj::AddrObject;
+use crate::illumos::dladm::PhysicalLink;
+use crate::zone::Zones;
+
+// Names of VNICs used as underlay devices for the xde driver.
+const XDE_VNIC_NAMES: [&str; 2] = ["net0", "net1"];
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+    #[error(
+        "Failed to create an IPv6 link-local address for underlay devices: {0}"
+    )]
+    UnderlayDeviceAddress(#[from] crate::illumos::ExecutionError),
+
+    #[error(transparent)]
+    BadAddrObj(#[from] addrobj::ParseError),
+}
+
+pub fn find_nics() -> Result<Vec<AddrObject>, Error> {
+    let underlay_nics = find_chelsio_links()?;
+
+    let mut addr_objs = Vec::with_capacity(underlay_nics.len());
+    for nic in underlay_nics {
+        let addrobj = AddrObject::new(&nic.0, "linklocal")?;
+        Zones::ensure_has_link_local_v6_address(None, &addrobj)?;
+        addr_objs.push(addrobj);
+    }
+
+    Ok(addr_objs)
+}
+
+fn find_chelsio_links() -> Result<Vec<PhysicalLink>, Error> {
+    // TODO-correctness: This should eventually be determined by a call to
+    // `Dladm` to get the real Chelsio links on a Gimlet. These will likely be
+    // called `cxgbeN`, but we explicitly call them `netN` to be clear that
+    // they're likely VNICs for the time being.
+    Ok(XDE_VNIC_NAMES
+        .into_iter()
+        .map(|name| PhysicalLink(name.to_string()))
+        .collect())
+}

sled-agent/src/opte/opte.rs

@@ -4,13 +4,11 @@
 
 //! Interactions with the Oxide Packet Transformation Engine (OPTE)
 
-use crate::illumos::addrobj;
-use crate::illumos::addrobj::AddrObject;
+use crate::common::underlay;
 use crate::illumos::dladm;
 use crate::illumos::dladm::Dladm;
 use crate::illumos::dladm::PhysicalLink;
 use crate::illumos::vnic::Vnic;
-use crate::illumos::zone::Zones;
 use crate::params::ExternalIp;
 use ipnetwork::IpNetwork;
 use macaddr::MacAddr6;
@@ -30,9 +28,6 @@ use std::sync::atomic::AtomicU64;
 use std::sync::atomic::Ordering;
 use std::sync::Arc;
 
-// Names of VNICs used as underlay devices for the xde driver.
-const XDE_VNIC_NAMES: [&str; 2] = ["net0", "net1"];
-
 // Prefix used to identify xde data links.
 const XDE_LINK_PREFIX: &str = "opte";
 
@@ -44,11 +39,8 @@ pub enum Error {
     #[error("Failed to wrap OPTE port in a VNIC: {0}")]
     CreateVnic(#[from] dladm::CreateVnicError),
 
-    #[error("Failed to create an IPv6 link-local address for xde underlay devices: {0}")]
-    UnderlayDeviceAddress(#[from] crate::illumos::ExecutionError),
-
     #[error("Failed to get VNICs for xde underlay devices: {0}")]
-    GetVnic(#[from] crate::illumos::dladm::GetVnicError),
+    GetVnic(#[from] underlay::Error),
 
     #[error(
         "No xde driver configuration file exists at '/kernel/drv/xde.conf'"
@@ -61,9 +53,6 @@ pub enum Error {
         driver which are compatible."
     )]
     IncompatibleKernel,
-
-    #[error(transparent)]
-    BadAddrObj(#[from] addrobj::ParseError),
 }
 
 #[derive(Debug, Clone)]
@@ -352,7 +341,7 @@ pub fn initialize_xde_driver(log: &Logger) -> Result<(), Error> {
     if !std::path::Path::new("/kernel/drv/xde.conf").exists() {
         return Err(Error::NoXdeConf);
     }
-    let underlay_nics = find_chelsio_links()?;
+    let underlay_nics = underlay::find_nics()?;
     info!(log, "using '{:?}' as data links for xde driver", underlay_nics);
     if underlay_nics.len() < 2 {
         const MESSAGE: &str = concat!(
@@ -366,13 +355,10 @@ pub fn initialize_xde_driver(log: &Logger) -> Result<(), Error> {
             String::from(MESSAGE),
         )));
     }
-    for nic in &underlay_nics {
-        let addrobj = AddrObject::new(&nic.0, "linklocal")?;
-        Zones::ensure_has_link_local_v6_address(None, &addrobj)?;
-    }
-    match OpteHdl::open(OpteHdl::DLD_CTL)?
-        .set_xde_underlay(&underlay_nics[0].0, &underlay_nics[1].0)
-    {
+    match OpteHdl::open(OpteHdl::DLD_CTL)?.set_xde_underlay(
+        underlay_nics[0].interface(),
+        underlay_nics[1].interface(),
+    ) {
         Ok(_) => Ok(()),
         // Handle the specific case where the kernel appears to be unaware of
         // xde at all. This implies the developer has not installed the correct
@@ -397,14 +383,3 @@ pub fn initialize_xde_driver(log: &Logger) -> Result<(), Error> {
         Err(e) => Err(e.into()),
     }
 }
-
-fn find_chelsio_links() -> Result<Vec<PhysicalLink>, Error> {
-    // TODO-correctness: This should eventually be determined by a call to
-    // `Dladm` to get the real Chelsio links on a Gimlet. These will likely be
-    // called `cxgbeN`, but we explicitly call them `netN` to be clear that
-    // they're likely VNICs for the time being.
-    Ok(XDE_VNIC_NAMES
-        .into_iter()
-        .map(|name| PhysicalLink(name.to_string()))
-        .collect())
-}