[sled-agent] Allocate VNICs over etherstubs, fix inter-zone routing

sled-agent/src/bootstrap/agent.rs

@@ -138,19 +138,23 @@ impl Agent {
             "server" => sled_config.id.to_string(),
         ));
 
-        let data_link = if let Some(link) = sled_config.data_link.clone() {
-            link
-        } else {
-            Dladm::find_physical().map_err(|err| {
+        let etherstub = Dladm::create_etherstub().map_err(|e| {
+            BootstrapError::SledError(format!(
+                "Can't access etherstub device: {}",
+                e
+            ))
+        })?;
+
+        let etherstub_vnic =
+            Dladm::create_etherstub_vnic(&etherstub).map_err(|e| {
                 BootstrapError::SledError(format!(
-                    "Can't access physical link, and none in config: {}",
-                    err
+                    "Can't access etherstub VNIC device: {}",
+                    e
                 ))
-            })?
-        };
+            })?;
 
         Zones::ensure_has_global_zone_v6_address(
-            data_link,
+            etherstub_vnic,
             address,
             "bootstrap6",
         )

sled-agent/src/config.rs

@@ -27,7 +27,7 @@ pub struct Config {
     /// Optional list of zpools to be used as "discovered disks".
     pub zpools: Option<Vec<ZpoolName>>,
 
-    /// The data link on which to allocate VNICs.
+    /// The data link on which we infer the bootstrap address.
     ///
     /// If unsupplied, we default to the first physical device.
     pub data_link: Option<PhysicalLink>,

sled-agent/src/illumos/dladm.rs

@@ -21,6 +21,9 @@ pub const VNIC_PREFIX_GUEST: &str = "vopte";
 
 pub const DLADM: &str = "/usr/sbin/dladm";
 
+pub const ETHERSTUB_NAME: &str = "stub0";
+pub const ETHERSTUB_VNIC_NAME: &str = "underlay0";
+
 /// Errors returned from [`Dladm::find_physical`].
 #[derive(thiserror::Error, Debug)]
 pub enum FindPhysicalLinkError {
@@ -49,7 +52,7 @@ pub enum GetMacError {
 #[error("Failed to create VNIC {name} on link {link:?}: {err}")]
 pub struct CreateVnicError {
     name: String,
-    link: PhysicalLink,
+    link: String,
     #[source]
     err: ExecutionError,
 }
@@ -75,11 +78,77 @@ pub struct DeleteVnicError {
 #[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
 pub struct PhysicalLink(pub String);
 
+/// The name of an etherstub
+#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
+pub struct Etherstub(pub String);
+
+/// The name of an etherstub's underlay VNIC
+#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
+pub struct EtherstubVnic(pub String);
+
+/// Identifies that an object may be used to create a VNIC.
+pub trait VnicSource {
+    fn name(&self) -> &str;
+}
+
+impl VnicSource for Etherstub {
+    fn name(&self) -> &str {
+        &self.0
+    }
+}
+
+impl VnicSource for PhysicalLink {
+    fn name(&self) -> &str {
+        &self.0
+    }
+}
+
 /// Wraps commands for interacting with data links.
 pub struct Dladm {}
 
 #[cfg_attr(test, mockall::automock, allow(dead_code))]
 impl Dladm {
+    /// Creates an etherstub, or returns one which already exists.
+    pub fn create_etherstub() -> Result<Etherstub, ExecutionError> {
+        if let Ok(stub) = Self::get_etherstub() {
+            return Ok(stub);
+        }
+        let mut command = std::process::Command::new(PFEXEC);
+        let cmd =
+            command.args(&[DLADM, "create-etherstub", "-t", ETHERSTUB_NAME]);
+        execute(cmd)?;
+        Ok(Etherstub(ETHERSTUB_NAME.to_string()))
+    }
+
+    /// Finds an etherstub.
+    fn get_etherstub() -> Result<Etherstub, ExecutionError> {
+        let mut command = std::process::Command::new(PFEXEC);
+        let cmd = command.args(&[DLADM, "show-etherstub", ETHERSTUB_NAME]);
+        execute(cmd)?;
+        Ok(Etherstub(ETHERSTUB_NAME.to_string()))
+    }
+
+    /// Creates a VNIC on top of the etherstub.
+    ///
+    /// This VNIC is not tracked like [`crate::illumos::vnic::Vnic`], because
+    /// it is expected to exist for the lifetime of the sled.
+    pub fn create_etherstub_vnic(
+        source: &Etherstub,
+    ) -> Result<EtherstubVnic, CreateVnicError> {
+        if let Ok(vnic) = Self::get_etherstub_vnic() {
+            return Ok(vnic);
+        }
+        Self::create_vnic(source, ETHERSTUB_VNIC_NAME, None, None)?;
+        Ok(EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()))
+    }
+
+    fn get_etherstub_vnic() -> Result<EtherstubVnic, ExecutionError> {
+        let mut command = std::process::Command::new(PFEXEC);
+        let cmd = command.args(&[DLADM, "show-vnic", ETHERSTUB_VNIC_NAME]);
+        execute(cmd)?;
+        Ok(EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()))
+    }
+
     /// Returns the name of the first observed physical data link.
     pub fn find_physical() -> Result<PhysicalLink, FindPhysicalLinkError> {
         let mut command = std::process::Command::new(PFEXEC);
@@ -135,8 +204,8 @@ impl Dladm {
     /// * `vnic_name`: Exact name of the VNIC to be created.
     /// * `mac`: An optional unicast MAC address for the newly created NIC.
     /// * `vlan`: An optional VLAN ID for VLAN tagging.
-    pub fn create_vnic(
-        physical: &PhysicalLink,
+    pub fn create_vnic<T: VnicSource + 'static>(
+        source: &T,
         vnic_name: &str,
         mac: Option<MacAddr>,
         vlan: Option<VlanID>,
@@ -147,7 +216,7 @@ impl Dladm {
             "create-vnic".to_string(),
             "-t".to_string(),
             "-l".to_string(),
-            physical.0.to_string(),
+            source.name().to_string(),
         ];
 
         if let Some(mac) = mac {
@@ -164,7 +233,7 @@ impl Dladm {
         let cmd = command.args(&args);
         execute(cmd).map_err(|err| CreateVnicError {
             name: vnic_name.to_string(),
-            link: physical.clone(),
+            link: source.name().to_string(),
             err,
         })?;
         Ok(())

sled-agent/src/illumos/vnic.rs

@@ -5,7 +5,7 @@
 //! API for controlling a single instance.
 
 use crate::illumos::dladm::{
-    CreateVnicError, DeleteVnicError, PhysicalLink, VNIC_PREFIX,
+    CreateVnicError, DeleteVnicError, Etherstub, VNIC_PREFIX,
     VNIC_PREFIX_CONTROL, VNIC_PREFIX_GUEST,
 };
 use omicron_common::api::external::MacAddr;
@@ -26,7 +26,7 @@ use crate::illumos::dladm::MockDladm as Dladm;
 pub struct VnicAllocator {
     value: Arc<AtomicU64>,
     scope: String,
-    data_link: PhysicalLink,
+    data_link: Etherstub,
 }
 
 impl VnicAllocator {
@@ -41,11 +41,11 @@ impl VnicAllocator {
     ///
     /// VnicAllocator::new("Storage") produces
     /// - oxControlStorage[NNN]
-    pub fn new<S: AsRef<str>>(scope: S, physical_link: PhysicalLink) -> Self {
+    pub fn new<S: AsRef<str>>(scope: S, etherstub: Etherstub) -> Self {
         Self {
             value: Arc::new(AtomicU64::new(0)),
             scope: scope.as_ref().to_string(),
-            data_link: physical_link,
+            data_link: etherstub,
         }
     }
 
@@ -171,7 +171,7 @@ mod test {
     #[test]
     fn test_allocate() {
         let allocator =
-            VnicAllocator::new("Foo", PhysicalLink("mylink".to_string()));
+            VnicAllocator::new("Foo", Etherstub("mystub".to_string()));
         assert_eq!("oxFoo0", allocator.next());
         assert_eq!("oxFoo1", allocator.next());
         assert_eq!("oxFoo2", allocator.next());
@@ -180,7 +180,7 @@ mod test {
     #[test]
     fn test_allocate_within_scopes() {
         let allocator =
-            VnicAllocator::new("Foo", PhysicalLink("mylink".to_string()));
+            VnicAllocator::new("Foo", Etherstub("mystub".to_string()));
         assert_eq!("oxFoo0", allocator.next());
         let allocator = allocator.new_superscope("Baz");
         assert_eq!("oxBazFoo1", allocator.next());

sled-agent/src/illumos/zone.rs

@@ -10,9 +10,10 @@ use slog::Logger;
 use std::net::{IpAddr, Ipv6Addr};
 
 use crate::illumos::addrobj::AddrObject;
-use crate::illumos::dladm::{PhysicalLink, VNIC_PREFIX_CONTROL};
+use crate::illumos::dladm::{EtherstubVnic, VNIC_PREFIX_CONTROL};
 use crate::illumos::zfs::ZONE_ZFS_DATASET_MOUNTPOINT;
 use crate::illumos::{execute, PFEXEC};
+use omicron_common::address::AZ_PREFIX;
 
 const DLADM: &str = "/usr/sbin/dladm";
 const IPADM: &str = "/usr/sbin/ipadm";
@@ -101,7 +102,7 @@ pub struct EnsureAddressError {
 #[error("Failed to create address {address} with name {name} in the GZ on {link:?}: {err}")]
 pub struct EnsureGzAddressError {
     address: Ipv6Addr,
-    link: PhysicalLink,
+    link: EtherstubVnic,
     name: String,
     #[source]
     err: anyhow::Error,
@@ -122,7 +123,7 @@ impl AddressRequest {
     pub fn new_static(ip: IpAddr, prefix: Option<u8>) -> Self {
         let prefix = prefix.unwrap_or_else(|| match ip {
             IpAddr::V4(_) => 24,
-            IpAddr::V6(_) => 64,
+            IpAddr::V6(_) => AZ_PREFIX,
         });
         let addr = IpNetwork::new(ip, prefix).unwrap();
         AddressRequest::Static(addr)
@@ -543,13 +544,13 @@ impl Zones {
     // should remove this function when Sled Agents are provided IPv6 addresses
     // from RSS.
     pub fn ensure_has_global_zone_v6_address(
-        link: PhysicalLink,
+        link: EtherstubVnic,
         address: Ipv6Addr,
         name: &str,
     ) -> Result<(), EnsureGzAddressError> {
         // Call the guts of this function within a closure to make it easier
         // to wrap the error with appropriate context.
-        |link: PhysicalLink, address, name| -> Result<(), anyhow::Error> {
+        |link: EtherstubVnic, address, name| -> Result<(), anyhow::Error> {
             let gz_link_local_addrobj = AddrObject::new(&link.0, "linklocal")
                 .map_err(|err| anyhow!(err))?;
             Self::ensure_has_link_local_v6_address(

sled-agent/src/instance.rs

@@ -671,7 +671,7 @@ impl Instance {
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::illumos::dladm::PhysicalLink;
+    use crate::illumos::dladm::Etherstub;
     use crate::mocks::MockNexusClient;
     use crate::opte::OptePortAllocator;
     use crate::params::InstanceStateRequested;
@@ -742,7 +742,7 @@ mod test {
         let log = logger();
         let vnic_allocator = VnicAllocator::new(
             "Test".to_string(),
-            PhysicalLink("mylink".to_string()),
+            Etherstub("mylink".to_string()),
         );
         let port_allocator = OptePortAllocator::new();
         let nexus_client = MockNexusClient::default();

sled-agent/src/instance_manager.rs

@@ -4,7 +4,7 @@
 
 //! API for controlling multiple instances on a sled.
 
-use crate::illumos::dladm::PhysicalLink;
+use crate::illumos::dladm::Etherstub;
 use crate::illumos::vnic::VnicAllocator;
 use crate::nexus::NexusClient;
 use crate::opte::OptePortAllocator;
@@ -54,15 +54,15 @@ impl InstanceManager {
     pub fn new(
         log: Logger,
         nexus_client: Arc<NexusClient>,
-        physical_link: PhysicalLink,
+        etherstub: Etherstub,
         underlay_addr: Ipv6Addr,
     ) -> InstanceManager {
         InstanceManager {
             inner: Arc::new(InstanceManagerInternal {
                 log: log.new(o!("component" => "InstanceManager")),
                 nexus_client,
                 instances: Mutex::new(BTreeMap::new()),
-                vnic_allocator: VnicAllocator::new("Instance", physical_link),
+                vnic_allocator: VnicAllocator::new("Instance", etherstub),
                 underlay_addr,
                 port_allocator: OptePortAllocator::new(),
             }),
@@ -196,7 +196,7 @@ impl Drop for InstanceTicket {
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::illumos::dladm::PhysicalLink;
+    use crate::illumos::dladm::Etherstub;
     use crate::illumos::{dladm::MockDladm, zone::MockZones};
     use crate::instance::MockInstance;
     use crate::mocks::MockNexusClient;
@@ -260,7 +260,7 @@ mod test {
         let im = InstanceManager::new(
             log,
             nexus_client,
-            PhysicalLink("mylink".to_string()),
+            Etherstub("mylink".to_string()),
             std::net::Ipv6Addr::new(
                 0xfd00, 0x1de, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
             ),
@@ -342,7 +342,7 @@ mod test {
         let im = InstanceManager::new(
             log,
             nexus_client,
-            PhysicalLink("mylink".to_string()),
+            Etherstub("mylink".to_string()),
             std::net::Ipv6Addr::new(
                 0xfd00, 0x1de, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
             ),