RPW for OPTE v2p Mappings

.github/buildomat/jobs/deploy.sh

@@ -2,7 +2,7 @@
 #:
 #: name = "helios / deploy"
 #: variety = "basic"
-#: target = "lab-2.0-opte-0.28"
+#: target = "lab-2.0-opte-0.29"
 #: output_rules = [
 #:  "%/var/svc/log/oxide-sled-agent:default.log*",
 #:  "%/zone/oxz_*/root/var/svc/log/oxide-*.log*",

Cargo.toml

@@ -304,14 +304,14 @@ omicron-sled-agent = { path = "sled-agent" }
 omicron-test-utils = { path = "test-utils" }
 omicron-zone-package = "0.11.0"
 oxide-client = { path = "clients/oxide-client" }
-oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "7ee353a470ea59529ee1b34729681da887aa88ce", features = [ "api", "std" ] }
+oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "4cc823b50d3e4a629cdfaab2b3d3382514174ba9", features = [ "api", "std" ] }
 once_cell = "1.19.0"
 openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" }
 openapiv3 = "2.0.0"
 # must match samael's crate!
 openssl = "0.10"
 openssl-sys = "0.9"
-opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "7ee353a470ea59529ee1b34729681da887aa88ce" }
+opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "4cc823b50d3e4a629cdfaab2b3d3382514174ba9" }
 oso = "0.27"
 owo-colors = "4.0.0"
 oximeter = { path = "oximeter/oximeter" }

clients/sled-agent-client/src/lib.rs

@@ -35,6 +35,7 @@ progenitor::generate_api!(
         PortConfigV1 = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] },
         RouteConfig = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] },
         IpNet = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] },
+        VirtualNetworkInterfaceHost = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] },
     },
     //TODO trade the manual transformations later in this file for the
     //     replace directives below?

dev-tools/omdb/tests/env.out

@@ -110,6 +110,10 @@ task: "switch_port_config_manager"
     manages switch port settings for rack switches
 
 
+task: "v2p_manager"
+    manages opte v2p mappings for vpc networking
+
+
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT
@@ -217,6 +221,10 @@ task: "switch_port_config_manager"
     manages switch port settings for rack switches
 
 
+task: "v2p_manager"
+    manages opte v2p mappings for vpc networking
+
+
 ---------------------------------------------
 stderr:
 note: Nexus URL not specified.  Will pick one from DNS.
@@ -311,6 +319,10 @@ task: "switch_port_config_manager"
     manages switch port settings for rack switches
 
 
+task: "v2p_manager"
+    manages opte v2p mappings for vpc networking
+
+
 ---------------------------------------------
 stderr:
 note: Nexus URL not specified.  Will pick one from DNS.

dev-tools/omdb/tests/successes.out

@@ -287,6 +287,10 @@ task: "switch_port_config_manager"
     manages switch port settings for rack switches
 
 
+task: "v2p_manager"
+    manages opte v2p mappings for vpc networking
+
+
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT/
@@ -455,6 +459,13 @@ task: "switch_port_config_manager"
     started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
 warning: unknown background task: "switch_port_config_manager" (don't know how to interpret details: Object {})
 
+task: "v2p_manager"
+  configured period: every 30s
+  currently executing: no
+  last completed activation: <REDACTED ITERATIONS>, triggered by an explicit signal
+    started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+warning: unknown background task: "v2p_manager" (don't know how to interpret details: Object {})
+
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT/

dev-tools/oxlog/src/bin/oxlog.rs

@@ -47,7 +47,7 @@ struct FilterArgs {
     #[arg(short, long)]
     archived: bool,
 
-    // Print only the extra log files
+    /// Print only the extra log files
     #[arg(short, long)]
     extra: bool,
 

illumos-utils/src/opte/params.rs

@@ -31,26 +31,16 @@ pub struct VpcFirewallRule {
 }
 
 /// A mapping from a virtual NIC to a physical host
-#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)]
-pub struct SetVirtualNetworkInterfaceHost {
+#[derive(
+    Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq, Hash,
+)]
+pub struct VirtualNetworkInterfaceHost {
     pub virtual_ip: IpAddr,
     pub virtual_mac: external::MacAddr,
     pub physical_host_ip: Ipv6Addr,
     pub vni: external::Vni,
 }
 
-/// The data needed to identify a virtual IP for which a sled maintains an OPTE
-/// virtual-to-physical mapping such that that mapping can be deleted.
-#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)]
-pub struct DeleteVirtualNetworkInterfaceHost {
-    /// The virtual IP whose mapping should be deleted.
-    pub virtual_ip: IpAddr,
-
-    /// The VNI for the network containing the virtual IP whose mapping should
-    /// be deleted.
-    pub vni: external::Vni,
-}
-
 /// DHCP configuration for a port
 ///
 /// Not present here: Hostname (DHCPv4 option 12; used in DHCPv6 option 39); we

illumos-utils/src/opte/port_manager.rs

@@ -5,8 +5,7 @@
 //! Manager for all OPTE ports on a Helios system
 
 use crate::opte::opte_firewall_rules;
-use crate::opte::params::DeleteVirtualNetworkInterfaceHost;
-use crate::opte::params::SetVirtualNetworkInterfaceHost;
+use crate::opte::params::VirtualNetworkInterfaceHost;
 use crate::opte::params::VpcFirewallRule;
 use crate::opte::Error;
 use crate::opte::Gateway;
@@ -570,10 +569,62 @@ impl PortManager {
         Ok(())
     }
 
+    #[cfg(target_os = "illumos")]
+    pub fn list_virtual_nics(
+        &self,
+    ) -> Result<Vec<VirtualNetworkInterfaceHost>, Error> {
+        use macaddr::MacAddr6;
+        use opte_ioctl::OpteHdl;
+
+        let hdl = OpteHdl::open(OpteHdl::XDE_CTL)?;
+        let v2p =
+            hdl.dump_v2p(&oxide_vpc::api::DumpVirt2PhysReq { unused: 99 })?;
+        let mut mappings: Vec<_> = vec![];
+
+        for mapping in v2p.mappings {
+            let vni = mapping
+                .vni
+                .as_u32()
+                .try_into()
+                .expect("opte VNI should be 24 bits");
+
+            for entry in mapping.ip4 {
+                mappings.push(VirtualNetworkInterfaceHost {
+                    virtual_ip: IpAddr::V4(entry.0.into()),
+                    virtual_mac: MacAddr6::from(entry.1.ether.bytes()).into(),
+                    physical_host_ip: entry.1.ip.into(),
+                    vni,
+                });
+            }
+
+            for entry in mapping.ip6 {
+                mappings.push(VirtualNetworkInterfaceHost {
+                    virtual_ip: IpAddr::V6(entry.0.into()),
+                    virtual_mac: MacAddr6::from(entry.1.ether.bytes()).into(),
+                    physical_host_ip: entry.1.ip.into(),
+                    vni,
+                });
+            }
+        }
+
+        Ok(mappings)
+    }
+
+    #[cfg(not(target_os = "illumos"))]
+    pub fn list_virtual_nics(
+        &self,
+    ) -> Result<Vec<VirtualNetworkInterfaceHost>, Error> {
+        info!(
+            self.inner.log,
+            "Listing virtual nics (ignored)";
+        );
+        Ok(vec![])
+    }
+
     #[cfg(target_os = "illumos")]
     pub fn set_virtual_nic_host(
         &self,
-        mapping: &SetVirtualNetworkInterfaceHost,
+        mapping: &VirtualNetworkInterfaceHost,
     ) -> Result<(), Error> {
         use opte_ioctl::OpteHdl;
 
@@ -600,7 +651,7 @@ impl PortManager {
     #[cfg(not(target_os = "illumos"))]
     pub fn set_virtual_nic_host(
         &self,
-        mapping: &SetVirtualNetworkInterfaceHost,
+        mapping: &VirtualNetworkInterfaceHost,
     ) -> Result<(), Error> {
         info!(
             self.inner.log,
@@ -613,20 +664,41 @@ impl PortManager {
     #[cfg(target_os = "illumos")]
     pub fn unset_virtual_nic_host(
         &self,
-        _mapping: &DeleteVirtualNetworkInterfaceHost,
+        mapping: &VirtualNetworkInterfaceHost,
     ) -> Result<(), Error> {
-        // TODO requires https://github.com/oxidecomputer/opte/issues/332
+        use opte_ioctl::OpteHdl;
+
+        info!(
+            self.inner.log,
+            "Clearing mapping of virtual NIC to physical host";
+            "mapping" => ?&mapping,
+        );
+
+        let hdl = OpteHdl::open(OpteHdl::XDE_CTL)?;
+        hdl.clear_v2p(&oxide_vpc::api::ClearVirt2PhysReq {
+            vip: mapping.virtual_ip.into(),
+            phys: oxide_vpc::api::PhysNet {
+                ether: oxide_vpc::api::MacAddr::from(
+                    (*mapping.virtual_mac).into_array(),
+                ),
+                ip: mapping.physical_host_ip.into(),
+                vni: Vni::new(mapping.vni).unwrap(),
+            },
+        })?;
 
-        slog::warn!(self.inner.log, "unset_virtual_nic_host unimplmented");
         Ok(())
     }
 
     #[cfg(not(target_os = "illumos"))]
     pub fn unset_virtual_nic_host(
         &self,
-        _mapping: &DeleteVirtualNetworkInterfaceHost,
+        mapping: &VirtualNetworkInterfaceHost,
     ) -> Result<(), Error> {
-        info!(self.inner.log, "Ignoring unset of virtual NIC mapping");
+        info!(
+            self.inner.log,
+            "Ignoring unset of virtual NIC mapping";
+            "mapping" => ?&mapping,
+        );
         Ok(())
     }
 }

nexus/db-model/src/lib.rs

@@ -55,6 +55,7 @@ mod project;
 mod semver_version;
 mod switch_interface;
 mod switch_port;
+mod v2p_mapping;
 // These actually represent subqueries, not real table.
 // However, they must be defined in the same crate as our tables
 // for join-based marker trait generation.
@@ -188,6 +189,7 @@ pub use typed_uuid::to_db_typed_uuid;
 pub use upstairs_repair::*;
 pub use user_builtin::*;
 pub use utilization::*;
+pub use v2p_mapping::*;
 pub use virtual_provisioning_collection::*;
 pub use virtual_provisioning_resource::*;
 pub use vmm::*;

nexus/db-model/src/schema.rs

@@ -285,6 +285,17 @@ table! {
     }
 }
 
+table! {
+    v2p_mapping_view (nic_id) {
+        nic_id -> Uuid,
+        sled_id -> Uuid,
+        sled_ip -> Inet,
+        vni -> Int4,
+        mac -> Int8,
+        ip -> Inet,
+    }
+}
+
 table! {
     bgp_announce_set (id) {
         id -> Uuid,

nexus/db-model/src/schema_versions.rs

@@ -17,7 +17,7 @@ use std::collections::BTreeMap;
 ///
 /// This must be updated when you change the database schema.  Refer to
 /// schema/crdb/README.adoc in the root of this repository for details.
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(59, 0, 0);
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(60, 0, 0);
 
 /// List of all past database schema versions, in *reverse* order
 ///
@@ -29,6 +29,7 @@ static KNOWN_VERSIONS: Lazy<Vec<KnownVersion>> = Lazy::new(|| {
         // |  leaving the first copy as an example for the next person.
         // v
         // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"),
+        KnownVersion::new(60, "add-view-for-v2p-mappings"),
         KnownVersion::new(59, "enforce-first-as-default"),
         KnownVersion::new(58, "insert-default-allowlist"),
         KnownVersion::new(57, "add-allowed-source-ips"),

nexus/db-model/src/v2p_mapping.rs

@@ -0,0 +1,16 @@
+use crate::schema::v2p_mapping_view;
+use crate::{MacAddr, Vni};
+use ipnetwork::IpNetwork;
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+#[derive(Queryable, Selectable, Clone, Debug, Serialize, Deserialize)]
+#[diesel(table_name = v2p_mapping_view)]
+pub struct V2PMappingView {
+    pub nic_id: Uuid,
+    pub sled_id: Uuid,
+    pub sled_ip: IpNetwork,
+    pub vni: Vni,
+    pub mac: MacAddr,
+    pub ip: IpNetwork,
+}