diff --git a/sled-agent/src/bootstrap/server.rs b/sled-agent/src/bootstrap/server.rs index b375541599..45a8cc349c 100644 --- a/sled-agent/src/bootstrap/server.rs +++ b/sled-agent/src/bootstrap/server.rs @@ -13,13 +13,11 @@ use super::trust_quorum::ShareDistribution; use super::views::Response; use super::views::ResponseEnvelope; use crate::bootstrap::maghemite; +use crate::common::underlay; use crate::config::Config as SledConfig; -use crate::illumos::addrobj::AddrObject; -use crate::illumos::dladm::VnicSource; use crate::sp::AsyncReadWrite; use crate::sp::SpHandle; use crate::sp::SprocketsRole; -use crate::zone::Zones; use slog::Drain; use slog::Logger; use std::net::Ipv6Addr; @@ -67,27 +65,21 @@ impl Server { debug!(log, "registered DTrace probes"); } - // Ensure we have a link-local inet6 address. - let link = sled_config - .get_link() - .map_err(|err| format!("Failed to find physical link: {err}"))?; - - let mg_interface = AddrObject::new(link.name(), "linklocal") - .expect("unexpected failure creating AddrObject"); - Zones::ensure_has_link_local_v6_address(None, &mg_interface).map_err( - |err| { - format!( - "Failed to ensure link-local address for {}: {}", - mg_interface, err - ) - }, - )?; + // Find address objects to pass to maghemite. + let mg_addr_objs = underlay::find_nics().map_err(|err| { + format!("Failed to find address objects for maghemite: {err}") + })?; + if mg_addr_objs.is_empty() { + return Err( + "underlay::find_nics() returned 0 address objects".to_string() + ); + } // Turn on the maghemite routing service. // TODO-correctness Eventually we need mg-ddm to listen on multiple // interfaces (link-local addresses of both NICs). info!(log, "Starting mg-ddm service"); - maghemite::enable_mg_ddm_service(log.clone(), mg_interface) + maghemite::enable_mg_ddm_service(log.clone(), mg_addr_objs[0].clone()) .await .map_err(|err| format!("Failed to start mg-ddm: {err}"))?; diff --git a/sled-agent/src/common/mod.rs b/sled-agent/src/common/mod.rs index 2df5e37c14..79ed6e177d 100644 --- a/sled-agent/src/common/mod.rs +++ b/sled-agent/src/common/mod.rs @@ -6,4 +6,5 @@ pub mod disk; pub mod instance; +pub mod underlay; pub mod vlan; diff --git a/sled-agent/src/common/underlay.rs b/sled-agent/src/common/underlay.rs new file mode 100644 index 0000000000..42e22f6691 --- /dev/null +++ b/sled-agent/src/common/underlay.rs @@ -0,0 +1,48 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Finding the underlay network physical links and address objects. + +use crate::illumos::addrobj; +use crate::illumos::addrobj::AddrObject; +use crate::illumos::dladm::PhysicalLink; +use crate::zone::Zones; + +// Names of VNICs used as underlay devices for the xde driver. +const XDE_VNIC_NAMES: [&str; 2] = ["net0", "net1"]; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error( + "Failed to create an IPv6 link-local address for underlay devices: {0}" + )] + UnderlayDeviceAddress(#[from] crate::illumos::ExecutionError), + + #[error(transparent)] + BadAddrObj(#[from] addrobj::ParseError), +} + +pub fn find_nics() -> Result, Error> { + let underlay_nics = find_chelsio_links()?; + + let mut addr_objs = Vec::with_capacity(underlay_nics.len()); + for nic in underlay_nics { + let addrobj = AddrObject::new(&nic.0, "linklocal")?; + Zones::ensure_has_link_local_v6_address(None, &addrobj)?; + addr_objs.push(addrobj); + } + + Ok(addr_objs) +} + +fn find_chelsio_links() -> Result, Error> { + // TODO-correctness: This should eventually be determined by a call to + // `Dladm` to get the real Chelsio links on a Gimlet. These will likely be + // called `cxgbeN`, but we explicitly call them `netN` to be clear that + // they're likely VNICs for the time being. + Ok(XDE_VNIC_NAMES + .into_iter() + .map(|name| PhysicalLink(name.to_string())) + .collect()) +} diff --git a/sled-agent/src/opte/opte.rs b/sled-agent/src/opte/opte.rs index 630098f6c8..db91c9377a 100644 --- a/sled-agent/src/opte/opte.rs +++ b/sled-agent/src/opte/opte.rs @@ -4,13 +4,11 @@ //! Interactions with the Oxide Packet Transformation Engine (OPTE) -use crate::illumos::addrobj; -use crate::illumos::addrobj::AddrObject; +use crate::common::underlay; use crate::illumos::dladm; use crate::illumos::dladm::Dladm; use crate::illumos::dladm::PhysicalLink; use crate::illumos::vnic::Vnic; -use crate::illumos::zone::Zones; use crate::params::ExternalIp; use ipnetwork::IpNetwork; use macaddr::MacAddr6; @@ -30,9 +28,6 @@ use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; use std::sync::Arc; -// Names of VNICs used as underlay devices for the xde driver. -const XDE_VNIC_NAMES: [&str; 2] = ["net0", "net1"]; - // Prefix used to identify xde data links. const XDE_LINK_PREFIX: &str = "opte"; @@ -44,11 +39,8 @@ pub enum Error { #[error("Failed to wrap OPTE port in a VNIC: {0}")] CreateVnic(#[from] dladm::CreateVnicError), - #[error("Failed to create an IPv6 link-local address for xde underlay devices: {0}")] - UnderlayDeviceAddress(#[from] crate::illumos::ExecutionError), - #[error("Failed to get VNICs for xde underlay devices: {0}")] - GetVnic(#[from] crate::illumos::dladm::GetVnicError), + GetVnic(#[from] underlay::Error), #[error( "No xde driver configuration file exists at '/kernel/drv/xde.conf'" @@ -61,9 +53,6 @@ pub enum Error { driver which are compatible." )] IncompatibleKernel, - - #[error(transparent)] - BadAddrObj(#[from] addrobj::ParseError), } #[derive(Debug, Clone)] @@ -352,7 +341,7 @@ pub fn initialize_xde_driver(log: &Logger) -> Result<(), Error> { if !std::path::Path::new("/kernel/drv/xde.conf").exists() { return Err(Error::NoXdeConf); } - let underlay_nics = find_chelsio_links()?; + let underlay_nics = underlay::find_nics()?; info!(log, "using '{:?}' as data links for xde driver", underlay_nics); if underlay_nics.len() < 2 { const MESSAGE: &str = concat!( @@ -366,13 +355,10 @@ pub fn initialize_xde_driver(log: &Logger) -> Result<(), Error> { String::from(MESSAGE), ))); } - for nic in &underlay_nics { - let addrobj = AddrObject::new(&nic.0, "linklocal")?; - Zones::ensure_has_link_local_v6_address(None, &addrobj)?; - } - match OpteHdl::open(OpteHdl::DLD_CTL)? - .set_xde_underlay(&underlay_nics[0].0, &underlay_nics[1].0) - { + match OpteHdl::open(OpteHdl::DLD_CTL)?.set_xde_underlay( + underlay_nics[0].interface(), + underlay_nics[1].interface(), + ) { Ok(_) => Ok(()), // Handle the specific case where the kernel appears to be unaware of // xde at all. This implies the developer has not installed the correct @@ -397,14 +383,3 @@ pub fn initialize_xde_driver(log: &Logger) -> Result<(), Error> { Err(e) => Err(e.into()), } } - -fn find_chelsio_links() -> Result, Error> { - // TODO-correctness: This should eventually be determined by a call to - // `Dladm` to get the real Chelsio links on a Gimlet. These will likely be - // called `cxgbeN`, but we explicitly call them `netN` to be clear that - // they're likely VNICs for the time being. - Ok(XDE_VNIC_NAMES - .into_iter() - .map(|name| PhysicalLink(name.to_string())) - .collect()) -}