diff --git a/sled-agent/src/bootstrap/agent.rs b/sled-agent/src/bootstrap/agent.rs index 224f4c1e252..b337e203ffc 100644 --- a/sled-agent/src/bootstrap/agent.rs +++ b/sled-agent/src/bootstrap/agent.rs @@ -157,23 +157,19 @@ impl Agent { err, })?; - let etherstub = Dladm::create_etherstub().map_err(|e| { - BootstrapError::SledError(format!( - "Can't access etherstub device: {}", - e - )) - })?; - - let etherstub_vnic = - Dladm::create_etherstub_vnic(ðerstub).map_err(|e| { + let data_link = if let Some(link) = sled_config.data_link.clone() { + link + } else { + Dladm::find_physical().map_err(|err| { BootstrapError::SledError(format!( - "Can't access etherstub VNIC device: {}", - e + "Can't access physical link, and none in config: {}", + err )) - })?; + })? + }; Zones::ensure_has_global_zone_v6_address( - etherstub_vnic, + data_link, address, "bootstrap6", ) diff --git a/sled-agent/src/config.rs b/sled-agent/src/config.rs index d67dd088e8a..cfa6e84927b 100644 --- a/sled-agent/src/config.rs +++ b/sled-agent/src/config.rs @@ -27,7 +27,7 @@ pub struct Config { /// Optional list of zpools to be used as "discovered disks". pub zpools: Option>, - /// The data link on which we infer the bootstrap address. + /// The data link on which to allocate VNICs. /// /// If unsupplied, we default to the first physical device. pub data_link: Option, diff --git a/sled-agent/src/illumos/dladm.rs b/sled-agent/src/illumos/dladm.rs index 2df5edc9586..69e22ca8161 100644 --- a/sled-agent/src/illumos/dladm.rs +++ b/sled-agent/src/illumos/dladm.rs @@ -19,15 +19,8 @@ pub const VNIC_PREFIX_CONTROL: &str = "oxControl"; // Viona, and thus plumbed directly to guests. pub const VNIC_PREFIX_GUEST: &str = "vopte"; -/// Path to the DLADM command. pub const DLADM: &str = "/usr/sbin/dladm"; -/// The name of the etherstub to be created for the underlay. -pub const ETHERSTUB_NAME: &str = "stub0"; - -/// The name of the etherstub VNIC to be created in the global zone. -pub const ETHERSTUB_VNIC_NAME: &str = "underlay0"; - /// Errors returned from [`Dladm::find_physical`]. #[derive(thiserror::Error, Debug)] pub enum FindPhysicalLinkError { @@ -56,7 +49,7 @@ pub enum GetMacError { #[error("Failed to create VNIC {name} on link {link:?}: {err}")] pub struct CreateVnicError { name: String, - link: String, + link: PhysicalLink, #[source] err: ExecutionError, } @@ -82,77 +75,11 @@ pub struct DeleteVnicError { #[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] pub struct PhysicalLink(pub String); -/// The name of an etherstub -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] -pub struct Etherstub(pub String); - -/// The name of an etherstub's underlay VNIC -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] -pub struct EtherstubVnic(pub String); - -/// Identifies that an object may be used to create a VNIC. -pub trait VnicSource { - fn name(&self) -> &str; -} - -impl VnicSource for Etherstub { - fn name(&self) -> &str { - &self.0 - } -} - -impl VnicSource for PhysicalLink { - fn name(&self) -> &str { - &self.0 - } -} - /// Wraps commands for interacting with data links. pub struct Dladm {} #[cfg_attr(test, mockall::automock, allow(dead_code))] impl Dladm { - /// Creates an etherstub, or returns one which already exists. - pub fn create_etherstub() -> Result { - if let Ok(stub) = Self::get_etherstub() { - return Ok(stub); - } - let mut command = std::process::Command::new(PFEXEC); - let cmd = - command.args(&[DLADM, "create-etherstub", "-t", ETHERSTUB_NAME]); - execute(cmd)?; - Ok(Etherstub(ETHERSTUB_NAME.to_string())) - } - - /// Finds an etherstub. - fn get_etherstub() -> Result { - let mut command = std::process::Command::new(PFEXEC); - let cmd = command.args(&[DLADM, "show-etherstub", ETHERSTUB_NAME]); - execute(cmd)?; - Ok(Etherstub(ETHERSTUB_NAME.to_string())) - } - - /// Creates a VNIC on top of the etherstub. - /// - /// This VNIC is not tracked like [`crate::illumos::vnic::Vnic`], because - /// it is expected to exist for the lifetime of the sled. - pub fn create_etherstub_vnic( - source: &Etherstub, - ) -> Result { - if let Ok(vnic) = Self::get_etherstub_vnic() { - return Ok(vnic); - } - Self::create_vnic(source, ETHERSTUB_VNIC_NAME, None, None)?; - Ok(EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string())) - } - - fn get_etherstub_vnic() -> Result { - let mut command = std::process::Command::new(PFEXEC); - let cmd = command.args(&[DLADM, "show-vnic", ETHERSTUB_VNIC_NAME]); - execute(cmd)?; - Ok(EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string())) - } - /// Returns the name of the first observed physical data link. pub fn find_physical() -> Result { let mut command = std::process::Command::new(PFEXEC); @@ -208,8 +135,8 @@ impl Dladm { /// * `vnic_name`: Exact name of the VNIC to be created. /// * `mac`: An optional unicast MAC address for the newly created NIC. /// * `vlan`: An optional VLAN ID for VLAN tagging. - pub fn create_vnic( - source: &T, + pub fn create_vnic( + physical: &PhysicalLink, vnic_name: &str, mac: Option, vlan: Option, @@ -220,7 +147,7 @@ impl Dladm { "create-vnic".to_string(), "-t".to_string(), "-l".to_string(), - source.name().to_string(), + physical.0.to_string(), ]; if let Some(mac) = mac { @@ -237,7 +164,7 @@ impl Dladm { let cmd = command.args(&args); execute(cmd).map_err(|err| CreateVnicError { name: vnic_name.to_string(), - link: source.name().to_string(), + link: physical.clone(), err, })?; Ok(()) diff --git a/sled-agent/src/illumos/mod.rs b/sled-agent/src/illumos/mod.rs index 5baa7bd4421..a59521579ad 100644 --- a/sled-agent/src/illumos/mod.rs +++ b/sled-agent/src/illumos/mod.rs @@ -15,7 +15,7 @@ pub mod zfs; pub mod zone; pub mod zpool; -pub const PFEXEC: &str = "/usr/bin/pfexec"; +const PFEXEC: &str = "/usr/bin/pfexec"; #[derive(thiserror::Error, Debug)] pub enum ExecutionError { @@ -23,12 +23,11 @@ pub enum ExecutionError { ExecutionStart { command: String, err: std::io::Error }, #[error( - "Command [{command}] executed and failed with status: {status}. Stdout: {stdout}, Stderr: {stderr}" + "Command [{command}] executed and failed with status: {status}. Output: {stderr}" )] CommandFailure { command: String, status: std::process::ExitStatus, - stdout: String, stderr: String, }, } @@ -64,7 +63,6 @@ mod inner { .collect::>() .join(" "), status: output.status, - stdout: String::from_utf8_lossy(&output.stdout).to_string(), stderr: String::from_utf8_lossy(&output.stderr).to_string(), }); } diff --git a/sled-agent/src/illumos/running_zone.rs b/sled-agent/src/illumos/running_zone.rs index a58b4c9d185..68a0a28a995 100644 --- a/sled-agent/src/illumos/running_zone.rs +++ b/sled-agent/src/illumos/running_zone.rs @@ -171,21 +171,6 @@ impl RunningZone { Ok(network) } - pub async fn add_route( - &self, - destination: ipnetwork::Ipv6Network, - ) -> Result<(), RunCommandError> { - self.run_cmd(&[ - "/usr/sbin/route", - "add", - "-inet6", - &format!("{}/{}", destination.network(), destination.prefix()), - "-inet6", - &destination.ip().to_string(), - ])?; - Ok(()) - } - /// Looks up a running zone based on the `zone_prefix`, if one already exists. /// /// - If the zone was found, is running, and has a network interface, it is diff --git a/sled-agent/src/illumos/vnic.rs b/sled-agent/src/illumos/vnic.rs index 85bb32880f8..03aa872fce5 100644 --- a/sled-agent/src/illumos/vnic.rs +++ b/sled-agent/src/illumos/vnic.rs @@ -5,7 +5,7 @@ //! API for controlling a single instance. use crate::illumos::dladm::{ - CreateVnicError, DeleteVnicError, Etherstub, VNIC_PREFIX, + CreateVnicError, DeleteVnicError, PhysicalLink, VNIC_PREFIX, VNIC_PREFIX_CONTROL, VNIC_PREFIX_GUEST, }; use omicron_common::api::external::MacAddr; @@ -26,7 +26,7 @@ use crate::illumos::dladm::MockDladm as Dladm; pub struct VnicAllocator { value: Arc, scope: String, - data_link: Etherstub, + data_link: PhysicalLink, } impl VnicAllocator { @@ -41,11 +41,11 @@ impl VnicAllocator { /// /// VnicAllocator::new("Storage") produces /// - oxControlStorage[NNN] - pub fn new>(scope: S, etherstub: Etherstub) -> Self { + pub fn new>(scope: S, physical_link: PhysicalLink) -> Self { Self { value: Arc::new(AtomicU64::new(0)), scope: scope.as_ref().to_string(), - data_link: etherstub, + data_link: physical_link, } } @@ -171,7 +171,7 @@ mod test { #[test] fn test_allocate() { let allocator = - VnicAllocator::new("Foo", Etherstub("mystub".to_string())); + VnicAllocator::new("Foo", PhysicalLink("mylink".to_string())); assert_eq!("oxFoo0", allocator.next()); assert_eq!("oxFoo1", allocator.next()); assert_eq!("oxFoo2", allocator.next()); @@ -180,7 +180,7 @@ mod test { #[test] fn test_allocate_within_scopes() { let allocator = - VnicAllocator::new("Foo", Etherstub("mystub".to_string())); + VnicAllocator::new("Foo", PhysicalLink("mylink".to_string())); assert_eq!("oxFoo0", allocator.next()); let allocator = allocator.new_superscope("Baz"); assert_eq!("oxBazFoo1", allocator.next()); diff --git a/sled-agent/src/illumos/zone.rs b/sled-agent/src/illumos/zone.rs index edd57c8e0cc..c2ed47cf784 100644 --- a/sled-agent/src/illumos/zone.rs +++ b/sled-agent/src/illumos/zone.rs @@ -10,10 +10,9 @@ use slog::Logger; use std::net::{IpAddr, Ipv6Addr}; use crate::illumos::addrobj::AddrObject; -use crate::illumos::dladm::{EtherstubVnic, VNIC_PREFIX_CONTROL}; +use crate::illumos::dladm::{PhysicalLink, VNIC_PREFIX_CONTROL}; use crate::illumos::zfs::ZONE_ZFS_DATASET_MOUNTPOINT; use crate::illumos::{execute, PFEXEC}; -use omicron_common::address::SLED_PREFIX; const DLADM: &str = "/usr/sbin/dladm"; const IPADM: &str = "/usr/sbin/ipadm"; @@ -99,14 +98,13 @@ pub struct EnsureAddressError { /// Errors from [`Zones::ensure_has_global_zone_v6_address`]. #[derive(thiserror::Error, Debug)] -#[error("Failed to create address {address} with name {name} in the GZ on {link:?}: {err}. Note to developers: {extra_note}")] +#[error("Failed to create address {address} with name {name} in the GZ on {link:?}: {err}")] pub struct EnsureGzAddressError { address: Ipv6Addr, - link: EtherstubVnic, + link: PhysicalLink, name: String, #[source] err: anyhow::Error, - extra_note: String, } /// Describes the type of addresses which may be requested from a zone. @@ -124,7 +122,7 @@ impl AddressRequest { pub fn new_static(ip: IpAddr, prefix: Option) -> Self { let prefix = prefix.unwrap_or_else(|| match ip { IpAddr::V4(_) => 24, - IpAddr::V6(_) => SLED_PREFIX, + IpAddr::V6(_) => 64, }); let addr = IpNetwork::new(ip, prefix).unwrap(); AddressRequest::Static(addr) @@ -545,13 +543,13 @@ impl Zones { // should remove this function when Sled Agents are provided IPv6 addresses // from RSS. pub fn ensure_has_global_zone_v6_address( - link: EtherstubVnic, + link: PhysicalLink, address: Ipv6Addr, name: &str, ) -> Result<(), EnsureGzAddressError> { // Call the guts of this function within a closure to make it easier // to wrap the error with appropriate context. - |link: EtherstubVnic, address, name| -> Result<(), anyhow::Error> { + |link: PhysicalLink, address, name| -> Result<(), anyhow::Error> { let gz_link_local_addrobj = AddrObject::new(&link.0, "linklocal") .map_err(|err| anyhow!(err))?; Self::ensure_has_link_local_v6_address( @@ -584,22 +582,6 @@ impl Zones { link, name: name.to_string(), err, - extra_note: - r#"As of https://github.com/oxidecomputer/omicron/pull/1066, we are changing the - physical device on which Global Zone addresses are allocated. - - Before this PR, we allocated addresses and VNICs directly on a physical link. - After this PR, we are allocating them on etherstubs. - - As a result, however, if your machine previously ran Omicron, it - may have addresses on the physical link which we'd like to - allocate from the etherstub instead. - - This can be fixed with the following commands: - - $ pfexec ipadm delete-addr /bootstrap6 - $ pfexec ipadm delete-addr /sled6 - $ pfexec ipadm delete-addr /internaldns"#.to_string() })?; Ok(()) } diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index 2ba4d435b5e..ab350edf11c 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -715,7 +715,7 @@ impl Instance { #[cfg(test)] mod test { use super::*; - use crate::illumos::dladm::Etherstub; + use crate::illumos::dladm::PhysicalLink; use crate::mocks::MockNexusClient; use crate::opte::OptePortAllocator; use crate::params::InstanceStateRequested; @@ -786,7 +786,7 @@ mod test { let log = logger(); let vnic_allocator = VnicAllocator::new( "Test".to_string(), - Etherstub("mylink".to_string()), + PhysicalLink("mylink".to_string()), ); let port_allocator = OptePortAllocator::new(); let nexus_client = MockNexusClient::default(); diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs index 69a0c79c354..ad67e674559 100644 --- a/sled-agent/src/instance_manager.rs +++ b/sled-agent/src/instance_manager.rs @@ -4,7 +4,7 @@ //! API for controlling multiple instances on a sled. -use crate::illumos::dladm::Etherstub; +use crate::illumos::dladm::PhysicalLink; use crate::illumos::vnic::VnicAllocator; use crate::nexus::NexusClient; use crate::opte::OptePortAllocator; @@ -54,7 +54,7 @@ impl InstanceManager { pub fn new( log: Logger, nexus_client: Arc, - etherstub: Etherstub, + physical_link: PhysicalLink, underlay_addr: Ipv6Addr, ) -> InstanceManager { InstanceManager { @@ -62,7 +62,7 @@ impl InstanceManager { log: log.new(o!("component" => "InstanceManager")), nexus_client, instances: Mutex::new(BTreeMap::new()), - vnic_allocator: VnicAllocator::new("Instance", etherstub), + vnic_allocator: VnicAllocator::new("Instance", physical_link), underlay_addr, port_allocator: OptePortAllocator::new(), }), @@ -196,7 +196,7 @@ impl Drop for InstanceTicket { #[cfg(test)] mod test { use super::*; - use crate::illumos::dladm::Etherstub; + use crate::illumos::dladm::PhysicalLink; use crate::illumos::{dladm::MockDladm, zone::MockZones}; use crate::instance::MockInstance; use crate::mocks::MockNexusClient; @@ -260,7 +260,7 @@ mod test { let im = InstanceManager::new( log, nexus_client, - Etherstub("mylink".to_string()), + PhysicalLink("mylink".to_string()), std::net::Ipv6Addr::new( 0xfd00, 0x1de, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, ), @@ -342,7 +342,7 @@ mod test { let im = InstanceManager::new( log, nexus_client, - Etherstub("mylink".to_string()), + PhysicalLink("mylink".to_string()), std::net::Ipv6Addr::new( 0xfd00, 0x1de, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, ), diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index 34ebda537fd..87fde05d7c6 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; use std::fmt::{Debug, Display, Formatter, Result as FormatResult}; use std::net::IpAddr; use std::net::Ipv6Addr; -use std::net::{SocketAddr, SocketAddrV6}; +use std::net::SocketAddr; use uuid::Uuid; /// Information required to construct a virtual network interface for a guest @@ -218,7 +218,7 @@ pub struct DatasetEnsureBody { // The type of the filesystem. pub dataset_kind: DatasetKind, // The address on which the zone will listen for requests. - pub address: SocketAddrV6, + pub address: SocketAddr, // NOTE: We could insert a UUID here, if we want that to be set by the // caller explicitly? Currently, the lack of a UUID implies that // "at most one dataset type" exists within a zpool. diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index aafd89b221b..c680e435c06 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -4,18 +4,16 @@ //! Support for miscellaneous services managed by the sled. -use crate::illumos::dladm::{Etherstub, EtherstubVnic}; +use crate::illumos::dladm::PhysicalLink; use crate::illumos::running_zone::{InstalledZone, RunningZone}; use crate::illumos::vnic::VnicAllocator; -use crate::illumos::zone::AddressRequest; +use crate::illumos::zone::{AddressRequest, Zones}; use crate::params::{ServiceEnsureBody, ServiceRequest}; -use crate::zone::Zones; -use ipnetwork::Ipv6Network; -use omicron_common::address::{AZ_PREFIX, DNS_PORT, DNS_SERVER_PORT}; +use omicron_common::address::{DNS_PORT, DNS_SERVER_PORT}; use slog::Logger; use std::collections::HashSet; use std::iter::FromIterator; -use std::net::{IpAddr, Ipv6Addr}; +use std::net::IpAddr; use std::path::{Path, PathBuf}; use tokio::sync::Mutex; @@ -79,8 +77,7 @@ pub struct ServiceManager { config_path: Option, zones: Mutex>, vnic_allocator: VnicAllocator, - underlay_vnic: EtherstubVnic, - underlay_address: Ipv6Addr, + physical_link: PhysicalLink, } impl ServiceManager { @@ -89,16 +86,13 @@ impl ServiceManager { /// /// Args: /// - `log`: The logger - /// - `etherstub`: An etherstub on which to allocate VNICs. - /// - `underlay_vnic`: The underlay's VNIC in the Global Zone. + /// - `physical_link`: A physical link on which to allocate datalinks. /// - `config_path`: An optional path to a configuration file to store /// the record of services. By default, [`default_services_config_path`] /// is used. pub async fn new( log: Logger, - etherstub: Etherstub, - underlay_vnic: EtherstubVnic, - underlay_address: Ipv6Addr, + physical_link: PhysicalLink, config_path: Option, ) -> Result { debug!(log, "Creating new ServiceManager"); @@ -106,9 +100,11 @@ impl ServiceManager { log: log.new(o!("component" => "ServiceManager")), config_path, zones: Mutex::new(vec![]), - vnic_allocator: VnicAllocator::new("Service", etherstub), - underlay_vnic, - underlay_address, + vnic_allocator: VnicAllocator::new( + "Service", + physical_link.clone(), + ), + physical_link, }; let config_path = mgr.services_config_path(); @@ -216,7 +212,7 @@ impl ServiceManager { let addr_name = service.name.replace(&['-', '_'][..], ""); Zones::ensure_has_global_zone_v6_address( - self.underlay_vnic.clone(), + self.physical_link.clone(), *addr, &addr_name, ) @@ -229,25 +225,6 @@ impl ServiceManager { })?; } - let gz_route_subnet = if !service.gz_addresses.is_empty() { - // If this service supplies its own GZ address, add a route. - // - // This is currently being used for the DNS service. - // - // TODO: consider limitng the number of GZ addresses which - // can be supplied - now that we're actively using it, we - // aren't really handling the "many GZ addresses" case, and it - // doesn't seem necessary now. - Ipv6Network::new(service.gz_addresses[0], AZ_PREFIX).unwrap() - } else { - // Otherwise, add a route to the global Zone's sled address for - // everything within the AZ. - Ipv6Network::new(self.underlay_address, AZ_PREFIX).unwrap() - }; - running_zone.add_route(gz_route_subnet).await.map_err(|err| { - Error::ZoneCommand { intent: "Adding Route".to_string(), err } - })?; - debug!(self.log, "importing manifest"); running_zone @@ -433,25 +410,25 @@ impl ServiceManager { mod test { use super::*; use crate::illumos::{ - dladm::{Etherstub, MockDladm, ETHERSTUB_NAME, ETHERSTUB_VNIC_NAME}, - svc, - zone::MockZones, + dladm::MockDladm, dladm::PhysicalLink, svc, zone::MockZones, }; use std::os::unix::process::ExitStatusExt; const SVC_NAME: &str = "my_svc"; const EXPECTED_ZONE_NAME: &str = "oxz_my_svc"; + const EXPECTED_LINK_NAME: &str = "my_link"; // Returns the expectations for a new service to be created. fn expect_new_service() -> Vec> { // Create a VNIC let create_vnic_ctx = MockDladm::create_vnic_context(); - create_vnic_ctx.expect().return_once( - |physical_link: &Etherstub, _, _, _| { - assert_eq!(&physical_link.0, ÐERSTUB_NAME); - Ok(()) - }, - ); + create_vnic_ctx.expect().return_once(|physical_link, _, _, _| { + assert_eq!( + physical_link, + &PhysicalLink(EXPECTED_LINK_NAME.to_string()) + ); + Ok(()) + }); // Install the Omicron Zone let install_ctx = MockZones::install_omicron_zone_context(); install_ctx.expect().return_once(|_, name, _, _, _, _| { @@ -469,7 +446,7 @@ mod test { wait_ctx.expect().return_once(|_, _| Ok(())); // Import the manifest, enable the service let execute_ctx = crate::illumos::execute_context(); - execute_ctx.expect().times(3).returning(|_| { + execute_ctx.expect().times(2).returning(|_| { Ok(std::process::Output { status: std::process::ExitStatus::from_raw(0), stdout: vec![], @@ -543,9 +520,7 @@ mod test { let config = config_dir.path().join("services.toml"); let mgr = ServiceManager::new( log, - Etherstub(ETHERSTUB_NAME.to_string()), - EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), - Ipv6Addr::LOCALHOST, + PhysicalLink(EXPECTED_LINK_NAME.to_string()), Some(config), ) .await @@ -569,9 +544,7 @@ mod test { let config = config_dir.path().join("services.toml"); let mgr = ServiceManager::new( log, - Etherstub(ETHERSTUB_NAME.to_string()), - EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), - Ipv6Addr::LOCALHOST, + PhysicalLink(EXPECTED_LINK_NAME.to_string()), Some(config), ) .await @@ -598,9 +571,7 @@ mod test { // down. let mgr = ServiceManager::new( logctx.log.clone(), - Etherstub(ETHERSTUB_NAME.to_string()), - EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), - Ipv6Addr::LOCALHOST, + PhysicalLink(EXPECTED_LINK_NAME.to_string()), Some(config.clone()), ) .await @@ -613,9 +584,7 @@ mod test { let _expectations = expect_new_service(); let mgr = ServiceManager::new( logctx.log.clone(), - Etherstub(ETHERSTUB_NAME.to_string()), - EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), - Ipv6Addr::LOCALHOST, + PhysicalLink(EXPECTED_LINK_NAME.to_string()), Some(config.clone()), ) .await @@ -639,9 +608,7 @@ mod test { // down. let mgr = ServiceManager::new( logctx.log.clone(), - Etherstub(ETHERSTUB_NAME.to_string()), - EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), - Ipv6Addr::LOCALHOST, + PhysicalLink(EXPECTED_LINK_NAME.to_string()), Some(config.clone()), ) .await @@ -656,9 +623,7 @@ mod test { // Observe that the old service is not re-initialized. let mgr = ServiceManager::new( logctx.log.clone(), - Etherstub(ETHERSTUB_NAME.to_string()), - EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), - Ipv6Addr::LOCALHOST, + PhysicalLink(EXPECTED_LINK_NAME.to_string()), Some(config.clone()), ) .await diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index c0c2ff649c8..4655db63f97 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -9,7 +9,6 @@ use crate::illumos::vnic::VnicKind; use crate::illumos::zfs::{ Mountpoint, ZONE_ZFS_DATASET, ZONE_ZFS_DATASET_MOUNTPOINT, }; -use crate::illumos::{execute, PFEXEC}; use crate::instance_manager::InstanceManager; use crate::nexus::NexusClient; use crate::params::{ @@ -23,7 +22,7 @@ use omicron_common::api::{ internal::nexus::UpdateArtifact, }; use slog::Logger; -use std::net::SocketAddrV6; +use std::net::{SocketAddr, SocketAddrV6}; use std::sync::Arc; use uuid::Uuid; @@ -39,15 +38,6 @@ pub enum Error { #[error("Physical link not in config, nor found automatically: {0}")] FindPhysicalLink(#[from] crate::illumos::dladm::FindPhysicalLinkError), - #[error("Failed to enable routing: {0}")] - EnablingRouting(crate::illumos::ExecutionError), - - #[error("Failed to acquire etherstub: {0}")] - Etherstub(crate::illumos::ExecutionError), - - #[error("Failed to acquire etherstub VNIC: {0}")] - EtherstubVnic(crate::illumos::dladm::CreateVnicError), - #[error("Failed to lookup VNICs on boot: {0}")] GetVnics(#[from] crate::illumos::dladm::GetVnicError), @@ -127,10 +117,11 @@ impl SledAgent { )); info!(&log, "created sled agent"); - let etherstub = - Dladm::create_etherstub().map_err(|e| Error::Etherstub(e))?; - let etherstub_vnic = Dladm::create_etherstub_vnic(ðerstub) - .map_err(|e| Error::EtherstubVnic(e))?; + let data_link = if let Some(link) = config.data_link.clone() { + link + } else { + Dladm::find_physical()? + }; // Before we start creating zones, we need to ensure that the // necessary ZFS and Zone resources are ready. @@ -150,7 +141,7 @@ impl SledAgent { // RSS-provided IP address. In the meantime, we use one from the // configuration file. Zones::ensure_has_global_zone_v6_address( - etherstub_vnic.clone(), + data_link.clone(), *sled_address.ip(), "sled6", ) @@ -204,24 +195,11 @@ impl SledAgent { // https://github.com/oxidecomputer/omicron/issues/725. crate::opte::delete_all_xde_devices(&log)?; - // Ipv6 forwarding must be enabled to route traffic between zones. - // - // This should be a no-op if already enabled. - let mut command = std::process::Command::new(PFEXEC); - let cmd = command.args(&[ - "/usr/sbin/routeadm", - "-e", - "ipv6-forwarding", - "-u", - ]); - execute(cmd).map_err(|e| Error::EnablingRouting(e))?; - let storage = StorageManager::new( &parent_log, *id, nexus_client.clone(), - etherstub.clone(), - *sled_address.ip(), + data_link.clone(), ) .await; if let Some(pools) = &config.zpools { @@ -237,17 +215,12 @@ impl SledAgent { let instances = InstanceManager::new( parent_log.clone(), nexus_client.clone(), - etherstub.clone(), + data_link.clone(), *sled_address.ip(), ); - let services = ServiceManager::new( - parent_log.clone(), - etherstub.clone(), - etherstub_vnic.clone(), - *sled_address.ip(), - None, - ) - .await?; + let services = + ServiceManager::new(parent_log.clone(), data_link.clone(), None) + .await?; Ok(SledAgent { id: config.id, @@ -279,7 +252,7 @@ impl SledAgent { &self, zpool_uuid: Uuid, dataset_kind: DatasetKind, - address: SocketAddrV6, + address: SocketAddr, ) -> Result<(), Error> { self.storage .upsert_filesystem(zpool_uuid, dataset_kind, address) diff --git a/sled-agent/src/storage_manager.rs b/sled-agent/src/storage_manager.rs index ee364b40310..8de0e2ec9ce 100644 --- a/sled-agent/src/storage_manager.rs +++ b/sled-agent/src/storage_manager.rs @@ -4,7 +4,7 @@ //! Management of sled-local storage. -use crate::illumos::dladm::Etherstub; +use crate::illumos::dladm::PhysicalLink; use crate::illumos::running_zone::{InstalledZone, RunningZone}; use crate::illumos::vnic::VnicAllocator; use crate::illumos::zone::AddressRequest; @@ -15,9 +15,7 @@ use crate::params::DatasetKind; use futures::stream::FuturesOrdered; use futures::FutureExt; use futures::StreamExt; -use ipnetwork::Ipv6Network; use nexus_client::types::{DatasetPutRequest, ZpoolPutRequest}; -use omicron_common::address::AZ_PREFIX; use omicron_common::api::external::{ByteCount, ByteCountRangeError}; use omicron_common::backoff; use schemars::JsonSchema; @@ -25,7 +23,7 @@ use serde::{Deserialize, Serialize}; use slog::Logger; use std::collections::HashMap; use std::convert::TryFrom; -use std::net::{IpAddr, Ipv6Addr, SocketAddrV6}; +use std::net::SocketAddr; use std::path::PathBuf; use std::pin::Pin; use std::sync::Arc; @@ -202,17 +200,13 @@ impl DatasetName { // by the Sled Agent. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] struct DatasetInfo { - address: SocketAddrV6, + address: SocketAddr, kind: DatasetKind, name: DatasetName, } impl DatasetInfo { - fn new( - pool: &str, - kind: DatasetKind, - address: SocketAddrV6, - ) -> DatasetInfo { + fn new(pool: &str, kind: DatasetKind, address: SocketAddr) -> DatasetInfo { match kind { DatasetKind::CockroachDb { .. } => DatasetInfo { name: DatasetName::new(pool, "cockroachdb"), @@ -240,7 +234,7 @@ impl DatasetInfo { &self, log: &Logger, zone: &RunningZone, - address: SocketAddrV6, + address: SocketAddr, do_format: bool, ) -> Result<(), Error> { match self.kind { @@ -309,8 +303,7 @@ impl DatasetInfo { // Await liveness of the cluster. info!(log, "start_zone: awaiting liveness of CRDB"); let check_health = || async { - let http_addr = - SocketAddrV6::new(*address.ip(), 8080, 0, 0); + let http_addr = SocketAddr::new(address.ip(), 8080); reqwest::get(format!("http://{}/health?ready=1", http_addr)) .await .map_err(backoff::BackoffError::transient) @@ -462,12 +455,9 @@ async fn ensure_running_zone( dataset_info: &DatasetInfo, dataset_name: &DatasetName, do_format: bool, - underlay_address: Ipv6Addr, ) -> Result { - let address_request = AddressRequest::new_static( - IpAddr::V6(*dataset_info.address.ip()), - None, - ); + let address_request = + AddressRequest::new_static(dataset_info.address.ip(), None); let err = RunningZone::get(log, &dataset_info.zone_prefix(), address_request) @@ -496,11 +486,6 @@ async fn ensure_running_zone( let zone = RunningZone::boot(installed_zone).await?; zone.ensure_address(address_request).await?; - - let gz_subnet = - Ipv6Network::new(underlay_address, AZ_PREFIX).unwrap(); - zone.add_route(gz_subnet).await.map_err(Error::ZoneCommand)?; - dataset_info .start_zone(log, &zone, dataset_info.address, do_format) .await?; @@ -531,7 +516,7 @@ type NotifyFut = dyn futures::Future< struct NewFilesystemRequest { zpool_id: Uuid, dataset_kind: DatasetKind, - address: SocketAddrV6, + address: SocketAddr, responder: oneshot::Sender>, } @@ -544,7 +529,6 @@ struct StorageWorker { new_pools_rx: mpsc::Receiver, new_filesystems_rx: mpsc::Receiver, vnic_allocator: VnicAllocator, - underlay_address: Ipv6Addr, } impl StorageWorker { @@ -609,7 +593,6 @@ impl StorageWorker { dataset_info, &dataset_name, do_format, - self.underlay_address, ) .await?; @@ -671,7 +654,7 @@ impl StorageWorker { fn add_datasets_notify( &self, nexus_notifications: &mut FuturesOrdered>>, - datasets: Vec<(Uuid, SocketAddrV6, DatasetKind)>, + datasets: Vec<(Uuid, SocketAddr, DatasetKind)>, pool_id: Uuid, ) { let nexus = self.nexus_client.clone(); @@ -781,7 +764,7 @@ impl StorageWorker { &self, pool: &mut Pool, dataset_name: &DatasetName, - ) -> Result<(Uuid, SocketAddrV6, DatasetKind), Error> { + ) -> Result<(Uuid, SocketAddr, DatasetKind), Error> { let name = dataset_name.full(); let id = Zfs::get_oxide_value(&name, "uuid")? .parse::() @@ -906,8 +889,7 @@ impl StorageManager { log: &Logger, sled_id: Uuid, nexus_client: Arc, - etherstub: Etherstub, - underlay_address: Ipv6Addr, + physical_link: PhysicalLink, ) -> Self { let log = log.new(o!("component" => "StorageManager")); let pools = Arc::new(Mutex::new(HashMap::new())); @@ -920,8 +902,7 @@ impl StorageManager { pools: pools.clone(), new_pools_rx, new_filesystems_rx, - vnic_allocator: VnicAllocator::new("Storage", etherstub), - underlay_address, + vnic_allocator: VnicAllocator::new("Storage", physical_link), }; StorageManager { pools, @@ -962,7 +943,7 @@ impl StorageManager { &self, zpool_id: Uuid, dataset_kind: DatasetKind, - address: SocketAddrV6, + address: SocketAddr, ) -> Result<(), Error> { let (tx, rx) = oneshot::channel(); let request = NewFilesystemRequest { @@ -1003,7 +984,7 @@ mod test { #[test] fn serialize_dataset_info() { let dataset_info = DatasetInfo { - address: "[::1]:8080".parse().unwrap(), + address: "127.0.0.1:8080".parse().unwrap(), kind: DatasetKind::Crucible, name: DatasetName::new("pool", "dataset"), }; diff --git a/sled-agent/tests/integration_tests/multicast.rs b/sled-agent/tests/integration_tests/multicast.rs index a66eb8d0be6..45012d087fd 100644 --- a/sled-agent/tests/integration_tests/multicast.rs +++ b/sled-agent/tests/integration_tests/multicast.rs @@ -40,12 +40,8 @@ async fn test_multicast_bootstrap_address() { // This modifies global state of the target machine, creating // an address named "testbootstrap6", akin to what the bootstrap // agent should do. - let etherstub = dladm::Dladm::create_etherstub().unwrap(); - let link = dladm::Dladm::create_etherstub_vnic(ðerstub).unwrap(); - - let phys_link = dladm::Dladm::find_physical().unwrap(); - let address = - bootstrap::agent::bootstrap_address(phys_link.clone()).unwrap(); + let link = dladm::Dladm::find_physical().unwrap(); + let address = bootstrap::agent::bootstrap_address(link.clone()).unwrap(); let address_name = "testbootstrap6"; let addrobj = AddrObject::new(&link.0, address_name).unwrap(); zone::Zones::ensure_has_global_zone_v6_address( diff --git a/smf/sled-agent/config.toml b/smf/sled-agent/config.toml index 53ec733e9ec..c1f5927c4be 100644 --- a/smf/sled-agent/config.toml +++ b/smf/sled-agent/config.toml @@ -19,9 +19,7 @@ zpools = [ "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03", ] -# An optional data link from which we extract a MAC address. -# This is used as a unique identifier for the bootstrap address. -# +# An optional data link on which VNICs should be created. # If empty, this will be equivalent to the first result from: # $ dladm show-phys -p -o LINK # data_link = "igb0"