diff --git a/Cargo.lock b/Cargo.lock index 24300970557..f2ff3dee11a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10136,6 +10136,7 @@ dependencies = [ "installinator-artifact-client", "installinator-artifactd", "installinator-common", + "ipnetwork", "itertools 0.11.0", "omicron-certificates", "omicron-common 0.1.0", diff --git a/clients/bootstrap-agent-client/src/lib.rs b/clients/bootstrap-agent-client/src/lib.rs index 3f8b20e1f5e..19ecb599f30 100644 --- a/clients/bootstrap-agent-client/src/lib.rs +++ b/clients/bootstrap-agent-client/src/lib.rs @@ -20,6 +20,8 @@ progenitor::generate_api!( derives = [schemars::JsonSchema], replace = { Ipv4Network = ipnetwork::Ipv4Network, + Ipv6Network = ipnetwork::Ipv6Network, + IpNetwork = ipnetwork::IpNetwork, } ); diff --git a/clients/nexus-client/src/lib.rs b/clients/nexus-client/src/lib.rs index 412ca704970..38de3bf0794 100644 --- a/clients/nexus-client/src/lib.rs +++ b/clients/nexus-client/src/lib.rs @@ -23,6 +23,8 @@ progenitor::generate_api!( }), replace = { Ipv4Network = ipnetwork::Ipv4Network, + Ipv6Network = ipnetwork::Ipv6Network, + IpNetwork = ipnetwork::IpNetwork, MacAddr = omicron_common::api::external::MacAddr, Name = omicron_common::api::external::Name, NewPasswordHash = omicron_passwords::NewPasswordHash, diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs index 8567ab56bbd..5f3b6be53d4 100644 --- a/clients/sled-agent-client/src/lib.rs +++ b/clients/sled-agent-client/src/lib.rs @@ -5,11 +5,33 @@ //! Interface for making API requests to a Sled Agent use async_trait::async_trait; -use omicron_common::generate_logging_api; use std::convert::TryFrom; use uuid::Uuid; -generate_logging_api!("../../openapi/sled-agent.json"); +progenitor::generate_api!( + spec = "../../openapi/sled-agent.json", + inner_type = slog::Logger, + pre_hook = (|log: &slog::Logger, request: &reqwest::Request| { + slog::debug!(log, "client request"; + "method" => %request.method(), + "uri" => %request.url(), + "body" => ?&request.body(), + ); + }), + post_hook = (|log: &slog::Logger, result: &Result<_, _>| { + slog::debug!(log, "client response"; "result" => ?result); + }), + //TODO trade the manual transformations later in this file for the + // replace directives below? + replace = { + //Ipv4Network = ipnetwork::Ipv4Network, + SwitchLocation = omicron_common::api::external::SwitchLocation, + Ipv6Network = ipnetwork::Ipv6Network, + IpNetwork = ipnetwork::IpNetwork, + PortFec = omicron_common::api::internal::shared::PortFec, + PortSpeed = omicron_common::api::internal::shared::PortSpeed, + } +); impl omicron_common::api::external::ClientError for types::Error { fn message(&self) -> String { diff --git a/clients/wicketd-client/src/lib.rs b/clients/wicketd-client/src/lib.rs index ff452325202..e4325bdb697 100644 --- a/clients/wicketd-client/src/lib.rs +++ b/clients/wicketd-client/src/lib.rs @@ -44,6 +44,10 @@ progenitor::generate_api!( RackOperationStatus = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, RackNetworkConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, UplinkConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, + PortConfigV1 = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, + BgpPeerConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, + BgpConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, + RouteConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, CurrentRssUserConfigInsensitive = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, CurrentRssUserConfigSensitive = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, CurrentRssUserConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, @@ -52,6 +56,8 @@ progenitor::generate_api!( replace = { Duration = std::time::Duration, Ipv4Network = ipnetwork::Ipv4Network, + Ipv6Network = ipnetwork::Ipv6Network, + IpNetwork = ipnetwork::IpNetwork, PutRssUserConfigInsensitive = wicket_common::rack_setup::PutRssUserConfigInsensitive, EventReportForWicketdEngineSpec = wicket_common::update_events::EventReport, StepEventForWicketdEngineSpec = wicket_common::update_events::StepEvent, diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 23833610c94..1300a8d5ff7 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -5,7 +5,7 @@ //! Types shared between Nexus and Sled Agent. use crate::api::external::{self, Name}; -use ipnetwork::Ipv4Network; +use ipnetwork::{IpNetwork, Ipv4Network}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::{ @@ -77,9 +77,57 @@ pub struct RackNetworkConfig { /// Last ip address to be used for configuring network infrastructure pub infra_ip_last: Ipv4Addr, /// Uplinks for connecting the rack to external networks - pub uplinks: Vec, + pub ports: Vec, + /// BGP configurations for connecting the rack to external networks + pub bgp: Vec, } +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +pub struct BgpConfig { + /// The autonomous system number for the BGP configuration. + pub asn: u32, + /// The set of prefixes for the BGP router to originate. + pub originate: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +pub struct BgpPeerConfig { + /// Switch port the peer is reachable on. + pub asn: u32, + /// Switch port the peer is reachable on. + pub port: String, + /// Address of the peer. + pub addr: Ipv4Addr, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +pub struct RouteConfig { + /// The destination of the route. + pub destination: IpNetwork, + /// The nexthop/gateway address. + pub nexthop: IpAddr, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +pub struct PortConfigV1 { + /// The set of routes associated with this port. + pub routes: Vec, + /// This port's addresses. + pub addresses: Vec, + /// Switch the port belongs to. + pub switch: SwitchLocation, + /// Nmae of the port this config applies to. + pub port: String, + /// Port speed. + pub uplink_port_speed: PortSpeed, + /// Port forward error correction type. + pub uplink_port_fec: PortFec, + /// BGP peers on this port + pub bgp_peers: Vec, +} + +/// Deprecated, use PortConfigV1 instead. Cannot actually deprecate due to +/// #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] pub struct UplinkConfig { /// Gateway address @@ -100,25 +148,24 @@ pub struct UplinkConfig { } /// A set of switch uplinks. -#[derive(Clone, Debug, Deserialize, JsonSchema)] -pub struct SwitchUplinks { - pub uplinks: Vec, +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct SwitchPorts { + pub uplinks: Vec, } #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] -pub struct HostUplinkConfig { +pub struct HostPortConfig { /// Switchport to use for external connectivity - pub uplink_port: String, + pub port: String, - //TODO should suppot IPv4 and IPv6 with IpNetwork /// IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport /// (must be in infra_ip pool) - pub uplink_cidr: Ipv4Network, + pub addrs: Vec, } -impl From for HostUplinkConfig { - fn from(u: UplinkConfig) -> Self { - Self { uplink_port: u.uplink_port, uplink_cidr: u.uplink_cidr } +impl From for HostPortConfig { + fn from(x: PortConfigV1) -> Self { + Self { port: x.port, addrs: x.addresses } } } diff --git a/nexus/db-model/src/switch_port.rs b/nexus/db-model/src/switch_port.rs index f49ffb6a10a..8f818833014 100644 --- a/nexus/db-model/src/switch_port.rs +++ b/nexus/db-model/src/switch_port.rs @@ -439,7 +439,11 @@ impl Into for SwitchPortRouteConfig { #[diesel(table_name = switch_port_settings_bgp_peer_config)] pub struct SwitchPortBgpPeerConfig { pub port_settings_id: Uuid, + + //TODO(ry) this should be associated with the BGP configuration + // not an individual peer. pub bgp_announce_set_id: Uuid, + pub bgp_config_id: Uuid, pub interface_name: String, pub addr: IpNetwork, diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index 3ac4b9063d9..3faae7f0657 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -33,8 +33,6 @@ use omicron_common::api::external::AddressLotKind; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; use omicron_common::api::external::IdentityMetadataCreateParams; -use omicron_common::api::external::IpNet; -use omicron_common::api::external::Ipv4Net; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_common::api::external::Name; @@ -380,7 +378,7 @@ impl super::Nexus { })?; for (idx, uplink_config) in - rack_network_config.uplinks.iter().enumerate() + rack_network_config.ports.iter().enumerate() { let switch = uplink_config.switch.to_string(); let switch_location = Name::from_str(&switch).map_err(|e| { @@ -449,31 +447,32 @@ impl super::Nexus { addresses: HashMap::new(), }; - let uplink_address = - IpNet::V4(Ipv4Net(uplink_config.uplink_cidr)); - let address = Address { - address_lot: NameOrId::Name(address_lot_name.clone()), - address: uplink_address, - }; - port_settings_params.addresses.insert( - "phy0".to_string(), - AddressConfig { addresses: vec![address] }, - ); - - let dst = IpNet::from_str("0.0.0.0/0").map_err(|e| { - Error::internal_error(&format!( - "failed to parse provided default route CIDR: {e}" - )) - })?; - - let gw = IpAddr::V4(uplink_config.gateway_ip); - let vid = uplink_config.uplink_vid; - let route = Route { dst, gw, vid }; - - port_settings_params.routes.insert( - "phy0".to_string(), - RouteConfig { routes: vec![route] }, - ); + let addresses: Vec
= uplink_config + .addresses + .iter() + .map(|a| Address { + address_lot: NameOrId::Name(address_lot_name.clone()), + address: (*a).into(), + }) + .collect(); + + port_settings_params + .addresses + .insert("phy0".to_string(), AddressConfig { addresses }); + + let routes: Vec = uplink_config + .routes + .iter() + .map(|r| Route { + dst: r.destination.into(), + gw: r.nexthop, + vid: None, + }) + .collect(); + + port_settings_params + .routes + .insert("phy0".to_string(), RouteConfig { routes }); match self .db_datastore @@ -498,9 +497,7 @@ impl super::Nexus { opctx, rack_id, switch_location.into(), - Name::from_str(&uplink_config.uplink_port) - .unwrap() - .into(), + Name::from_str(&uplink_config.port).unwrap().into(), ) .await?; diff --git a/nexus/src/app/sagas/switch_port_settings_apply.rs b/nexus/src/app/sagas/switch_port_settings_apply.rs index ea97a85750c..55ed1c4f5f9 100644 --- a/nexus/src/app/sagas/switch_port_settings_apply.rs +++ b/nexus/src/app/sagas/switch_port_settings_apply.rs @@ -23,9 +23,16 @@ use nexus_db_queries::{authn, db}; use nexus_types::external_api::params; use omicron_common::api::external::{self, NameOrId}; use omicron_common::api::internal::shared::{ - ParseSwitchLocationError, SwitchLocation, + ParseSwitchLocationError, PortFec as OmicronPortFec, + PortSpeed as OmicronPortSpeed, SwitchLocation, }; use serde::{Deserialize, Serialize}; +use sled_agent_client::types::BgpConfig; +use sled_agent_client::types::BgpPeerConfig as OmicronBgpPeerConfig; +use sled_agent_client::types::EarlyNetworkPortChange; +use sled_agent_client::types::EarlyNetworkPortUpdate; +use sled_agent_client::types::PortConfigV1; +use sled_agent_client::types::RouteConfig; use std::collections::HashMap; use std::net::IpAddr; use std::net::SocketAddrV6; @@ -67,6 +74,10 @@ declare_saga_actions! { + spa_ensure_switch_port_bgp_settings - spa_undo_ensure_switch_port_bgp_settings } + ENSURE_SWITCH_PORT_EARLY_NETWORK_SETTINGS -> "ensure_switch_port_early_network_settings" { + + spa_ensure_switch_port_early_network_settings + - spa_undo_ensure_switch_port_early_network_settings + } } // switch port settings apply saga: definition @@ -476,6 +487,149 @@ async fn spa_undo_ensure_switch_port_bgp_settings( Ok(()) } +async fn spa_ensure_switch_port_early_network_settings( + sagactx: NexusActionContext, +) -> Result { + let osagactx = sagactx.user_data(); + let nexus = osagactx.nexus(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let settings = sagactx + .lookup::("switch_port_settings") + .map_err(|e| { + ActionError::action_failed(format!( + "lookup switch port settings (bgp undo): {e}" + )) + })?; + + // Just choosing the sled agent associated with switch0 for no reason. + let sled_agent_addr = + get_scrimlet_address(SwitchLocation::Switch0, nexus).await?; + let sa = sled_agent_client::Client::new( + &format!("http://{}", sled_agent_addr), + sagactx.user_data().log().clone(), + ); + + let switch_port = nexus + .get_switch_port(&opctx, params.switch_port_id) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "get switch port for uplink: {e}" + )) + })?; + + let switch_location: SwitchLocation = + switch_port.switch_location.parse().map_err( + |e: ParseSwitchLocationError| { + ActionError::action_failed(format!( + "get switch location for uplink: {e:?}", + )) + }, + )?; + + let mut peer_info = Vec::new(); + let mut bgp_configs = Vec::new(); + for p in &settings.bgp_peers { + let bgp_config = nexus + .bgp_config_get(&opctx, p.bgp_config_id.into()) + .await + .map_err(|e| { + ActionError::action_failed(format!("get bgp config: {e}")) + })?; + + let announcements = nexus + .bgp_announce_list( + &opctx, + ¶ms::BgpAnnounceSetSelector { + name_or_id: NameOrId::Id(p.bgp_announce_set_id), + }, + ) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "get bgp announcements: {e}" + )) + })?; + + peer_info.push((p, bgp_config.asn.0)); + bgp_configs.push(BgpConfig { + asn: bgp_config.asn.0, + originate: announcements + .iter() + .filter_map(|a| match a.network { + IpNetwork::V4(net) => Some(net.into()), + //TODO v6 + _ => None, + }) + .collect(), + }); + } + + let update = EarlyNetworkPortUpdate { + port: PortConfigV1 { + routes: settings + .routes + .iter() + .map(|r| RouteConfig { destination: r.dst, nexthop: r.gw.ip() }) + .collect(), + addresses: settings.addresses.iter().map(|a| a.address).collect(), + switch: switch_location, + port: params.switch_port_name.clone(), + uplink_port_fec: OmicronPortFec::None, //TODO hardcode + uplink_port_speed: OmicronPortSpeed::Speed100G, //TODO hardcode + bgp_peers: peer_info + .iter() + .filter_map(|(p, asn)| { + //TODO v6 + if let IpAddr::V4(addr) = p.addr.ip() { + Some(OmicronBgpPeerConfig { + asn: *asn, + port: params.switch_port_name.clone(), + addr, + }) + } else { + None + } + }) + .collect(), + }, + bgp_configs, + }; + + let change = sa + .early_network_port_update(&update) + .await + .map_err(|e| { + ActionError::action_failed(format!("early network update: {e}")) + })? + .into_inner(); + + Ok(change) +} + +async fn spa_undo_ensure_switch_port_early_network_settings( + sagactx: NexusActionContext, +) -> Result<(), Error> { + let _change = sagactx + .lookup::( + "ensure_switch_port_early_network_settings", + ) + .map_err(|e| { + ActionError::action_failed(format!( + "lookup changed port early network settings: {e}" + )) + })?; + + //TODO roll back the change + + Ok(()) +} + async fn spa_ensure_switch_port_uplink( sagactx: NexusActionContext, ) -> Result<(), ActionError> { @@ -517,28 +671,16 @@ async fn spa_ensure_switch_port_uplink( let sled_agent_addr = get_scrimlet_address(switch_location, nexus).await?; - let mut uplinks = Vec::new(); - - for a in &settings.addresses { - let uplink_cidr = match a.address { - IpNetwork::V4(uplink) => uplink, - IpNetwork::V6(_) => { - return Err(ActionError::action_failed( - "IPv6 uplinks not supported".to_string(), - )) - } - }; - uplinks.push(sled_agent_client::types::HostUplinkConfig { - uplink_port: tfport.clone(), - uplink_cidr: uplink_cidr.into(), - }); - } + let uplinks = vec![sled_agent_client::types::HostPortConfig { + port: tfport.clone(), + addrs: settings.addresses.iter().map(|a| a.address).collect(), + }]; let sc = sled_agent_client::Client::new( &format!("http://{}", sled_agent_addr), sagactx.user_data().log().clone(), ); - sc.uplink_ensure(&sled_agent_client::types::SwitchUplinks { uplinks }) + sc.uplink_ensure(&sled_agent_client::types::SwitchPorts { uplinks }) .await .map_err(|e| { ActionError::action_failed(format!("ensure uplink: {e}")) diff --git a/nexus/src/app/sagas/switch_port_settings_clear.rs b/nexus/src/app/sagas/switch_port_settings_clear.rs index 0c0f4ec01b6..6d5673b23dc 100644 --- a/nexus/src/app/sagas/switch_port_settings_clear.rs +++ b/nexus/src/app/sagas/switch_port_settings_clear.rs @@ -36,6 +36,11 @@ declare_saga_actions! { + spa_clear_switch_port_settings - spa_undo_clear_switch_port_settings } + //TODO + // - Clear switch port uplinnk + // - Clear BGP settings + // - Clear early network settings. + // * I think bgp configs need to be ref counted .... } #[derive(Debug)] diff --git a/nexus/src/app/switch_port.rs b/nexus/src/app/switch_port.rs index 2f3e3fe564f..73b1c4ffd3b 100644 --- a/nexus/src/app/switch_port.rs +++ b/nexus/src/app/switch_port.rs @@ -97,9 +97,6 @@ impl super::Nexus { .await?; } - // TODO update bootstore goes here - // Need a sled-agent API to update early network config - Ok(result) } diff --git a/openapi/bootstrap-agent.json b/openapi/bootstrap-agent.json index 682512cc242..91b8ae91307 100644 --- a/openapi/bootstrap-agent.json +++ b/openapi/bootstrap-agent.json @@ -241,6 +241,53 @@ } ] }, + "BgpConfig": { + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number for the BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "originate": { + "description": "The set of prefixes for the BGP router to originate.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4Network" + } + } + }, + "required": [ + "asn", + "originate" + ] + }, + "BgpPeerConfig": { + "type": "object", + "properties": { + "addr": { + "description": "Address of the peer.", + "type": "string", + "format": "ipv4" + }, + "asn": { + "description": "Switch port the peer is reachable on.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "port": { + "description": "Switch port the peer is reachable on.", + "type": "string" + } + }, + "required": [ + "addr", + "asn", + "port" + ] + }, "BootstrapAddressDiscovery": { "oneOf": [ { @@ -333,6 +380,26 @@ "request_id" ] }, + "IpNetwork": { + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Network" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Network" + } + ] + } + ] + }, "IpRange": { "oneOf": [ { @@ -375,6 +442,10 @@ "last" ] }, + "Ipv6Network": { + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\")[/](12[0-8]|1[0-1][0-9]|[0-9]?[0-9])$" + }, "Ipv6Range": { "description": "A non-decreasing IPv6 address range, inclusive of both ends.\n\nThe first address must be less than or equal to the last address.", "type": "object", @@ -406,6 +477,69 @@ "description": "Password hashes must be in PHC (Password Hashing Competition) string format. Passwords must be hashed with Argon2id. Password hashes may be rejected if the parameters appear not to be secure enough.", "type": "string" }, + "PortConfigV1": { + "type": "object", + "properties": { + "addresses": { + "description": "This port's addresses.", + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNetwork" + } + }, + "bgp_peers": { + "description": "BGP peers on this port", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpPeerConfig" + } + }, + "port": { + "description": "Nmae of the port this config applies to.", + "type": "string" + }, + "routes": { + "description": "The set of routes associated with this port.", + "type": "array", + "items": { + "$ref": "#/components/schemas/RouteConfig" + } + }, + "switch": { + "description": "Switch the port belongs to.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] + }, + "uplink_port_fec": { + "description": "Port forward error correction type.", + "allOf": [ + { + "$ref": "#/components/schemas/PortFec" + } + ] + }, + "uplink_port_speed": { + "description": "Port speed.", + "allOf": [ + { + "$ref": "#/components/schemas/PortSpeed" + } + ] + } + }, + "required": [ + "addresses", + "bgp_peers", + "port", + "routes", + "switch", + "uplink_port_fec", + "uplink_port_speed" + ] + }, "PortFec": { "description": "Switchport FEC options", "type": "string", @@ -533,6 +667,13 @@ "description": "Initial network configuration", "type": "object", "properties": { + "bgp": { + "description": "BGP configurations for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpConfig" + } + }, "infra_ip_first": { "description": "First ip address to be used for configuring network infrastructure", "type": "string", @@ -543,18 +684,19 @@ "type": "string", "format": "ipv4" }, - "uplinks": { + "ports": { "description": "Uplinks for connecting the rack to external networks", "type": "array", "items": { - "$ref": "#/components/schemas/UplinkConfig" + "$ref": "#/components/schemas/PortConfigV1" } } }, "required": [ + "bgp", "infra_ip_first", "infra_ip_last", - "uplinks" + "ports" ] }, "RackOperationStatus": { @@ -747,6 +889,28 @@ "user_password_hash" ] }, + "RouteConfig": { + "type": "object", + "properties": { + "destination": { + "description": "The destination of the route.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNetwork" + } + ] + }, + "nexthop": { + "description": "The nexthop/gateway address.", + "type": "string", + "format": "ip" + } + }, + "required": [ + "destination", + "nexthop" + ] + }, "SemverVersion": { "type": "string", "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" @@ -770,67 +934,6 @@ } ] }, - "UplinkConfig": { - "type": "object", - "properties": { - "gateway_ip": { - "description": "Gateway address", - "type": "string", - "format": "ipv4" - }, - "switch": { - "description": "Switch to use for uplink", - "allOf": [ - { - "$ref": "#/components/schemas/SwitchLocation" - } - ] - }, - "uplink_cidr": { - "description": "IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport (must be in infra_ip pool)", - "allOf": [ - { - "$ref": "#/components/schemas/Ipv4Network" - } - ] - }, - "uplink_port": { - "description": "Switchport to use for external connectivity", - "type": "string" - }, - "uplink_port_fec": { - "description": "Forward Error Correction setting for the uplink port", - "allOf": [ - { - "$ref": "#/components/schemas/PortFec" - } - ] - }, - "uplink_port_speed": { - "description": "Speed for the Switchport", - "allOf": [ - { - "$ref": "#/components/schemas/PortSpeed" - } - ] - }, - "uplink_vid": { - "nullable": true, - "description": "VLAN id to use for uplink", - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "gateway_ip", - "switch", - "uplink_cidr", - "uplink_port", - "uplink_port_fec", - "uplink_port_speed" - ] - }, "UserId": { "description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID though they may contain a UUID.", "type": "string" diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 3167a274f4f..aed90369d3f 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -767,6 +767,53 @@ "serial_number" ] }, + "BgpConfig": { + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number for the BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "originate": { + "description": "The set of prefixes for the BGP router to originate.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4Network" + } + } + }, + "required": [ + "asn", + "originate" + ] + }, + "BgpPeerConfig": { + "type": "object", + "properties": { + "addr": { + "description": "Address of the peer.", + "type": "string", + "format": "ipv4" + }, + "asn": { + "description": "Switch port the peer is reachable on.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "port": { + "description": "Switch port the peer is reachable on.", + "type": "string" + } + }, + "required": [ + "addr", + "asn", + "port" + ] + }, "BinRangedouble": { "description": "A type storing a range over `T`.\n\nThis type supports ranges similar to the `RangeTo`, `Range` and `RangeFrom` types in the standard library. Those cover `(..end)`, `(start..end)`, and `(start..)` respectively.", "oneOf": [ @@ -3711,6 +3758,26 @@ } ] }, + "IpNetwork": { + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Network" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Network" + } + ] + } + ] + }, "IpRange": { "oneOf": [ { @@ -3753,6 +3820,10 @@ "last" ] }, + "Ipv6Network": { + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\")[/](12[0-8]|1[0-1][0-9]|[0-9]?[0-9])$" + }, "Ipv6Range": { "description": "A non-decreasing IPv6 address range, inclusive of both ends.\n\nThe first address must be less than or equal to the last address.", "type": "object", @@ -4096,6 +4167,69 @@ "PhysicalDiskPutResponse": { "type": "object" }, + "PortConfigV1": { + "type": "object", + "properties": { + "addresses": { + "description": "This port's addresses.", + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNetwork" + } + }, + "bgp_peers": { + "description": "BGP peers on this port", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpPeerConfig" + } + }, + "port": { + "description": "Nmae of the port this config applies to.", + "type": "string" + }, + "routes": { + "description": "The set of routes associated with this port.", + "type": "array", + "items": { + "$ref": "#/components/schemas/RouteConfig" + } + }, + "switch": { + "description": "Switch the port belongs to.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] + }, + "uplink_port_fec": { + "description": "Port forward error correction type.", + "allOf": [ + { + "$ref": "#/components/schemas/PortFec" + } + ] + }, + "uplink_port_speed": { + "description": "Port speed.", + "allOf": [ + { + "$ref": "#/components/schemas/PortSpeed" + } + ] + } + }, + "required": [ + "addresses", + "bgp_peers", + "port", + "routes", + "switch", + "uplink_port_fec", + "uplink_port_speed" + ] + }, "PortFec": { "description": "Switchport FEC options", "type": "string", @@ -4361,6 +4495,13 @@ "description": "Initial network configuration", "type": "object", "properties": { + "bgp": { + "description": "BGP configurations for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpConfig" + } + }, "infra_ip_first": { "description": "First ip address to be used for configuring network infrastructure", "type": "string", @@ -4371,18 +4512,19 @@ "type": "string", "format": "ipv4" }, - "uplinks": { + "ports": { "description": "Uplinks for connecting the rack to external networks", "type": "array", "items": { - "$ref": "#/components/schemas/UplinkConfig" + "$ref": "#/components/schemas/PortConfigV1" } } }, "required": [ + "bgp", "infra_ip_first", "infra_ip_last", - "uplinks" + "ports" ] }, "RecoverySiloConfig": { @@ -4404,6 +4546,28 @@ "user_password_hash" ] }, + "RouteConfig": { + "type": "object", + "properties": { + "destination": { + "description": "The destination of the route.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNetwork" + } + ] + }, + "nexthop": { + "description": "The nexthop/gateway address.", + "type": "string", + "format": "ip" + } + }, + "required": [ + "destination", + "nexthop" + ] + }, "Saga": { "description": "Sagas\n\nThese are currently only intended for observability by developers. We will eventually want to flesh this out into something more observable for end users.", "type": "object", @@ -5130,67 +5294,6 @@ "SwitchPutResponse": { "type": "object" }, - "UplinkConfig": { - "type": "object", - "properties": { - "gateway_ip": { - "description": "Gateway address", - "type": "string", - "format": "ipv4" - }, - "switch": { - "description": "Switch to use for uplink", - "allOf": [ - { - "$ref": "#/components/schemas/SwitchLocation" - } - ] - }, - "uplink_cidr": { - "description": "IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport (must be in infra_ip pool)", - "allOf": [ - { - "$ref": "#/components/schemas/Ipv4Network" - } - ] - }, - "uplink_port": { - "description": "Switchport to use for external connectivity", - "type": "string" - }, - "uplink_port_fec": { - "description": "Forward Error Correction setting for the uplink port", - "allOf": [ - { - "$ref": "#/components/schemas/PortFec" - } - ] - }, - "uplink_port_speed": { - "description": "Speed for the Switchport", - "allOf": [ - { - "$ref": "#/components/schemas/PortSpeed" - } - ] - }, - "uplink_vid": { - "nullable": true, - "description": "VLAN id to use for uplink", - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "gateway_ip", - "switch", - "uplink_cidr", - "uplink_port", - "uplink_port_fec", - "uplink_port_speed" - ] - }, "UserId": { "title": "A name unique within the parent collection", "description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID though they may contain a UUID.", diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 7772afc33fc..4f088156720 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -71,6 +71,39 @@ } } }, + "/early-network-port-update": { + "post": { + "operationId": "early_network_port_update", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EarlyNetworkPortUpdate" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EarlyNetworkPortChange" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/instances/{instance_id}": { "put": { "operationId": "instance_register", @@ -338,14 +371,14 @@ } } }, - "/switch-uplinks": { + "/switch-ports": { "post": { "operationId": "uplink_ensure", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SwitchUplinks" + "$ref": "#/components/schemas/SwitchPorts" } } }, @@ -889,6 +922,53 @@ } }, "schemas": { + "BgpConfig": { + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number for the BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "originate": { + "description": "The set of prefixes for the BGP router to originate.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4Network" + } + } + }, + "required": [ + "asn", + "originate" + ] + }, + "BgpPeerConfig": { + "type": "object", + "properties": { + "addr": { + "description": "Address of the peer.", + "type": "string", + "format": "ipv4" + }, + "asn": { + "description": "Switch port the peer is reachable on.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "port": { + "description": "Switch port the peer is reachable on.", + "type": "string" + } + }, + "required": [ + "addr", + "asn", + "port" + ] + }, "BundleUtilization": { "description": "The portion of a debug dataset used for zone bundles.", "type": "object", @@ -1597,6 +1677,53 @@ "secs" ] }, + "EarlyNetworkPortChange": { + "type": "object", + "properties": { + "added_bgp_configs": { + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpConfig" + } + }, + "changed_bgp_configs": { + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpConfig" + } + }, + "previous_port_config": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/PortConfigV1" + } + ] + } + }, + "required": [ + "added_bgp_configs", + "changed_bgp_configs" + ] + }, + "EarlyNetworkPortUpdate": { + "type": "object", + "properties": { + "bgp_configs": { + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpConfig" + } + }, + "port": { + "$ref": "#/components/schemas/PortConfigV1" + } + }, + "required": [ + "bgp_configs", + "port" + ] + }, "Error": { "description": "Error information from a response.", "type": "object", @@ -1663,25 +1790,24 @@ } ] }, - "HostUplinkConfig": { + "HostPortConfig": { "type": "object", "properties": { - "uplink_cidr": { + "addrs": { "description": "IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport (must be in infra_ip pool)", - "allOf": [ - { - "$ref": "#/components/schemas/Ipv4Network" - } - ] + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNetwork" + } }, - "uplink_port": { + "port": { "description": "Switchport to use for external connectivity", "type": "string" } }, "required": [ - "uplink_cidr", - "uplink_port" + "addrs", + "port" ] }, "InstanceCpuCount": { @@ -2148,6 +2274,26 @@ } ] }, + "IpNetwork": { + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Network" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Network" + } + ] + } + ] + }, "Ipv4Net": { "example": "192.168.1.0/24", "title": "An IPv4 subnet", @@ -2166,6 +2312,10 @@ "type": "string", "pattern": "^([fF][dD])[0-9a-fA-F]{2}:(([0-9a-fA-F]{1,4}:){6}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,6}:)([0-9a-fA-F]{1,4})?\\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])$" }, + "Ipv6Network": { + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\")[/](12[0-8]|1[0-1][0-9]|[0-9]?[0-9])$" + }, "KnownArtifactKind": { "description": "Kinds of update artifacts, as used by Nexus to determine what updates are available and by sled-agent to determine how to apply an update when asked.", "type": "string", @@ -2300,6 +2450,93 @@ } ] }, + "PortConfigV1": { + "type": "object", + "properties": { + "addresses": { + "description": "This port's addresses.", + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNetwork" + } + }, + "bgp_peers": { + "description": "BGP peers on this port", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpPeerConfig" + } + }, + "port": { + "description": "Nmae of the port this config applies to.", + "type": "string" + }, + "routes": { + "description": "The set of routes associated with this port.", + "type": "array", + "items": { + "$ref": "#/components/schemas/RouteConfig" + } + }, + "switch": { + "description": "Switch the port belongs to.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] + }, + "uplink_port_fec": { + "description": "Port forward error correction type.", + "allOf": [ + { + "$ref": "#/components/schemas/PortFec" + } + ] + }, + "uplink_port_speed": { + "description": "Port speed.", + "allOf": [ + { + "$ref": "#/components/schemas/PortSpeed" + } + ] + } + }, + "required": [ + "addresses", + "bgp_peers", + "port", + "routes", + "switch", + "uplink_port_fec", + "uplink_port_speed" + ] + }, + "PortFec": { + "description": "Switchport FEC options", + "type": "string", + "enum": [ + "firecode", + "none", + "rs" + ] + }, + "PortSpeed": { + "description": "Switchport Speed options", + "type": "string", + "enum": [ + "speed0_g", + "speed1_g", + "speed10_g", + "speed25_g", + "speed40_g", + "speed50_g", + "speed100_g", + "speed200_g", + "speed400_g" + ] + }, "PriorityDimension": { "description": "A dimension along with bundles can be sorted, to determine priority.", "oneOf": [ @@ -2328,6 +2565,28 @@ "minItems": 2, "maxItems": 2 }, + "RouteConfig": { + "type": "object", + "properties": { + "destination": { + "description": "The destination of the route.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNetwork" + } + ] + }, + "nexthop": { + "description": "The nexthop/gateway address.", + "type": "string", + "format": "ip" + } + }, + "required": [ + "destination", + "nexthop" + ] + }, "SemverVersion": { "type": "string", "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" @@ -2810,14 +3069,33 @@ "format": "uint8", "minimum": 0 }, - "SwitchUplinks": { + "SwitchLocation": { + "description": "Identifies switch physical location", + "oneOf": [ + { + "description": "Switch in upper slot", + "type": "string", + "enum": [ + "switch0" + ] + }, + { + "description": "Switch in lower slot", + "type": "string", + "enum": [ + "switch1" + ] + } + ] + }, + "SwitchPorts": { "description": "A set of switch uplinks.", "type": "object", "properties": { "uplinks": { "type": "array", "items": { - "$ref": "#/components/schemas/HostUplinkConfig" + "$ref": "#/components/schemas/HostPortConfig" } } }, diff --git a/openapi/wicketd.json b/openapi/wicketd.json index d67fc79f7a0..1bd73d3fd40 100644 --- a/openapi/wicketd.json +++ b/openapi/wicketd.json @@ -820,6 +820,53 @@ } ] }, + "BgpConfig": { + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number for the BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "originate": { + "description": "The set of prefixes for the BGP router to originate.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4Network" + } + } + }, + "required": [ + "asn", + "originate" + ] + }, + "BgpPeerConfig": { + "type": "object", + "properties": { + "addr": { + "description": "Address of the peer.", + "type": "string", + "format": "ipv4" + }, + "asn": { + "description": "Switch port the peer is reachable on.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "port": { + "description": "Switch port the peer is reachable on.", + "type": "string" + } + }, + "required": [ + "addr", + "asn", + "port" + ] + }, "BootstrapSledDescription": { "type": "object", "properties": { @@ -1321,6 +1368,26 @@ "installable" ] }, + "IpNetwork": { + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Network" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Network" + } + ] + } + ] + }, "IpRange": { "oneOf": [ { @@ -1363,6 +1430,10 @@ "last" ] }, + "Ipv6Network": { + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\")[/](12[0-8]|1[0-1][0-9]|[0-9]?[0-9])$" + }, "Ipv6Range": { "description": "A non-decreasing IPv6 address range, inclusive of both ends.\n\nThe first address must be less than or equal to the last address.", "type": "object", @@ -1386,6 +1457,69 @@ "description": "Password hashes must be in PHC (Password Hashing Competition) string format. Passwords must be hashed with Argon2id. Password hashes may be rejected if the parameters appear not to be secure enough.", "type": "string" }, + "PortConfigV1": { + "type": "object", + "properties": { + "addresses": { + "description": "This port's addresses.", + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNetwork" + } + }, + "bgp_peers": { + "description": "BGP peers on this port", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpPeerConfig" + } + }, + "port": { + "description": "Nmae of the port this config applies to.", + "type": "string" + }, + "routes": { + "description": "The set of routes associated with this port.", + "type": "array", + "items": { + "$ref": "#/components/schemas/RouteConfig" + } + }, + "switch": { + "description": "Switch the port belongs to.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] + }, + "uplink_port_fec": { + "description": "Port forward error correction type.", + "allOf": [ + { + "$ref": "#/components/schemas/PortFec" + } + ] + }, + "uplink_port_speed": { + "description": "Port speed.", + "allOf": [ + { + "$ref": "#/components/schemas/PortSpeed" + } + ] + } + }, + "required": [ + "addresses", + "bgp_peers", + "port", + "routes", + "switch", + "uplink_port_fec", + "uplink_port_speed" + ] + }, "PortFec": { "description": "Switchport FEC options", "type": "string", @@ -1976,6 +2110,13 @@ "description": "Initial network configuration", "type": "object", "properties": { + "bgp": { + "description": "BGP configurations for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpConfig" + } + }, "infra_ip_first": { "description": "First ip address to be used for configuring network infrastructure", "type": "string", @@ -1986,18 +2127,19 @@ "type": "string", "format": "ipv4" }, - "uplinks": { + "ports": { "description": "Uplinks for connecting the rack to external networks", "type": "array", "items": { - "$ref": "#/components/schemas/UplinkConfig" + "$ref": "#/components/schemas/PortConfigV1" } } }, "required": [ + "bgp", "infra_ip_first", "infra_ip_last", - "uplinks" + "ports" ] }, "RackOperationStatus": { @@ -2314,6 +2456,28 @@ } ] }, + "RouteConfig": { + "type": "object", + "properties": { + "destination": { + "description": "The destination of the route.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNetwork" + } + ] + }, + "nexthop": { + "description": "The nexthop/gateway address.", + "type": "string", + "format": "ip" + } + }, + "required": [ + "destination", + "nexthop" + ] + }, "SemverVersion": { "type": "string", "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" @@ -4439,67 +4603,6 @@ } ] }, - "UplinkConfig": { - "type": "object", - "properties": { - "gateway_ip": { - "description": "Gateway address", - "type": "string", - "format": "ipv4" - }, - "switch": { - "description": "Switch to use for uplink", - "allOf": [ - { - "$ref": "#/components/schemas/SwitchLocation" - } - ] - }, - "uplink_cidr": { - "description": "IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport (must be in infra_ip pool)", - "allOf": [ - { - "$ref": "#/components/schemas/Ipv4Network" - } - ] - }, - "uplink_port": { - "description": "Switchport to use for external connectivity", - "type": "string" - }, - "uplink_port_fec": { - "description": "Forward Error Correction setting for the uplink port", - "allOf": [ - { - "$ref": "#/components/schemas/PortFec" - } - ] - }, - "uplink_port_speed": { - "description": "Speed for the Switchport", - "allOf": [ - { - "$ref": "#/components/schemas/PortSpeed" - } - ] - }, - "uplink_vid": { - "nullable": true, - "description": "VLAN id to use for uplink", - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "gateway_ip", - "switch", - "uplink_cidr", - "uplink_port", - "uplink_port_fec", - "uplink_port_speed" - ] - }, "IgnitionCommand": { "description": "Ignition command.", "type": "string", diff --git a/package-manifest.toml b/package-manifest.toml index febff4aa0c3..5fc22d34059 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -422,10 +422,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "f1813a2a07a9857f862daeed2bebe51060734c69" +source.commit = "7b88dbcc7810f4fe9c82a7459862a7340d7e09ce" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//maghemite.sha256.txt -source.sha256 = "5a61c753d744621f02e4e4c0c32030545fd3f28ef72fd69f8d11fa1703e94755" +source.sha256 = "ca7dad9723aae0507546065e77748186d06ebcf29d6217d1d430ccfff0678c2c" output.type = "tarball" [package.mg-ddm] @@ -438,10 +438,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "f1813a2a07a9857f862daeed2bebe51060734c69" +source.commit = "7b88dbcc7810f4fe9c82a7459862a7340d7e09ce" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "7c84a4ae14f561b17a61bcf96049d16c5c4ff2a023875e16128c24b493e09950" +source.sha256 = "47b7b9b70fa2564ebd9fca4e591e1dfe7a48ef8dd413b678de08bbfc33b27d50" output.type = "zone" output.intermediate_only = true @@ -453,10 +453,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "f1813a2a07a9857f862daeed2bebe51060734c69" +source.commit = "7b88dbcc7810f4fe9c82a7459862a7340d7e09ce" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "212c7d09958d8578dc1f6b5556e1d0dbfbbcc542aced90051195b25a4b20905e" +source.sha256 = "82e965486e7acdbc4661258f1bc8d7751155f406f0973d55e61f6506d18984c2" output.type = "zone" output.intermediate_only = true diff --git a/sled-agent/src/bootstrap/early_networking.rs b/sled-agent/src/bootstrap/early_networking.rs index 61d4c84af37..9e53aca0382 100644 --- a/sled-agent/src/bootstrap/early_networking.rs +++ b/sled-agent/src/bootstrap/early_networking.rs @@ -7,20 +7,20 @@ use anyhow::{anyhow, Context}; use bootstore::schemes::v0 as bootstore; use ddm_admin_client::{Client as DdmAdminClient, DdmError}; -use dpd_client::types::Ipv6Entry; +use dpd_client::types::{Ipv6Entry, RouteSettingsV6}; use dpd_client::types::{ LinkCreate, LinkId, LinkSettings, PortId, PortSettings, RouteSettingsV4, }; use dpd_client::Client as DpdClient; -use dpd_client::Ipv4Cidr; use futures::future; use gateway_client::Client as MgsClient; use internal_dns::resolver::{ResolveError, Resolver as DnsResolver}; use internal_dns::ServiceName; +use ipnetwork::IpNetwork; use omicron_common::address::{Ipv6Subnet, AZ_PREFIX, MGS_PORT}; use omicron_common::address::{DDMD_PORT, DENDRITE_PORT}; use omicron_common::api::internal::shared::{ - PortFec, PortSpeed, RackNetworkConfig, SwitchLocation, UplinkConfig, + PortConfigV1, PortFec, PortSpeed, RackNetworkConfig, SwitchLocation, }; use omicron_common::backoff::{ retry_notify, retry_policy_local, BackoffError, ExponentialBackoff, @@ -107,11 +107,11 @@ impl<'a> EarlyNetworkSetup<'a> { resolver: &DnsResolver, config: &RackNetworkConfig, ) -> HashSet { - // Which switches have uplinks? + // Which switches have configured ports? let uplinked_switches = config - .uplinks + .ports .iter() - .map(|uplink_config| uplink_config.switch) + .map(|port_config| port_config.switch) .collect::>(); // If we have no uplinks, we have nothing to look up. @@ -342,7 +342,7 @@ impl<'a> EarlyNetworkSetup<'a> { &mut self, rack_network_config: &RackNetworkConfig, switch_zone_underlay_ip: Ipv6Addr, - ) -> Result, EarlyNetworkSetupError> { + ) -> Result, EarlyNetworkSetupError> { // First, we have to know which switch we are: ask MGS. info!( self.log, @@ -385,10 +385,10 @@ impl<'a> EarlyNetworkSetup<'a> { }; // We now know which switch we are: filter the uplinks to just ours. - let our_uplinks = rack_network_config - .uplinks + let our_ports = rack_network_config + .ports .iter() - .filter(|uplink| uplink.switch == switch_location) + .filter(|port| port.switch == switch_location) .cloned() .collect::>(); @@ -396,7 +396,7 @@ impl<'a> EarlyNetworkSetup<'a> { self.log, "Initializing {} Uplinks on {switch_location:?} at \ {switch_zone_underlay_ip}", - our_uplinks.len(), + our_ports.len(), ); let dpd = DpdClient::new( &format!("http://[{}]:{}", switch_zone_underlay_ip, DENDRITE_PORT), @@ -408,9 +408,9 @@ impl<'a> EarlyNetworkSetup<'a> { // configure uplink for each requested uplink in configuration that // matches our switch_location - for uplink_config in &our_uplinks { + for port_config in &our_ports { let (ipv6_entry, dpd_port_settings, port_id) = - self.build_uplink_config(uplink_config)?; + self.build_port_config(port_config)?; self.wait_for_dendrite(&dpd).await; @@ -446,14 +446,14 @@ impl<'a> EarlyNetworkSetup<'a> { ddmd_client.advertise_prefix(Ipv6Subnet::new(ipv6_entry.addr)); } - Ok(our_uplinks) + Ok(our_ports) } - fn build_uplink_config( + fn build_port_config( &self, - uplink_config: &UplinkConfig, + port_config: &PortConfigV1, ) -> Result<(Ipv6Entry, PortSettings, PortId), EarlyNetworkSetupError> { - info!(self.log, "Building Uplink Configuration"); + info!(self.log, "Building Port Configuration"); let ipv6_entry = Ipv6Entry { addr: BOUNDARY_SERVICES_ADDR.parse().map_err(|e| { EarlyNetworkSetupError::BadConfig(format!( @@ -469,41 +469,57 @@ impl<'a> EarlyNetworkSetup<'a> { v6_routes: HashMap::new(), }; let link_id = LinkId(0); + + let mut addrs = Vec::new(); + for a in &port_config.addresses { + addrs.push(a.ip()); + } + // TODO We're discarding the `uplink_cidr.prefix()` here and only using // the IP address; at some point we probably need to give the full CIDR // to dendrite? - let addr = IpAddr::V4(uplink_config.uplink_cidr.ip()); let link_settings = LinkSettings { // TODO Allow user to configure link properties // https://github.com/oxidecomputer/omicron/issues/3061 params: LinkCreate { autoneg: false, kr: false, - fec: convert_fec(&uplink_config.uplink_port_fec), - speed: convert_speed(&uplink_config.uplink_port_speed), + fec: convert_fec(&port_config.uplink_port_fec), + speed: convert_speed(&port_config.uplink_port_speed), }, - addrs: vec![addr], + //addrs: vec![addr], + addrs, }; dpd_port_settings.links.insert(link_id.to_string(), link_settings); - let port_id: PortId = - uplink_config.uplink_port.parse().map_err(|e| { - EarlyNetworkSetupError::BadConfig(format!( - concat!( - "could not use value provided to", - "rack_network_config.uplink_port as PortID: {}" - ), - e - )) - })?; - dpd_port_settings.v4_routes.insert( - Ipv4Cidr { prefix: "0.0.0.0".parse().unwrap(), prefix_len: 0 } - .to_string(), - RouteSettingsV4 { - link_id: link_id.0, - vid: uplink_config.uplink_vid, - nexthop: uplink_config.gateway_ip, - }, - ); + let port_id: PortId = port_config.port.parse().map_err(|e| { + EarlyNetworkSetupError::BadConfig(format!( + concat!( + "could not use value provided to", + "rack_network_config.uplink_port as PortID: {}" + ), + e + )) + })?; + + for r in &port_config.routes { + if let (IpNetwork::V4(dst), IpAddr::V4(nexthop)) = + (r.destination, r.nexthop) + { + dpd_port_settings.v4_routes.insert( + dst.to_string(), + RouteSettingsV4 { link_id: link_id.0, nexthop, vid: None }, + ); + } + if let (IpNetwork::V6(dst), IpAddr::V6(nexthop)) = + (r.destination, r.nexthop) + { + dpd_port_settings.v6_routes.insert( + dst.to_string(), + RouteSettingsV6 { link_id: link_id.0, nexthop, vid: None }, + ); + } + } + Ok((ipv6_entry, dpd_port_settings, port_id)) } diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index d689e94d8bc..0bbcf48617b 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -5,6 +5,7 @@ //! HTTP entrypoint functions for the sled agent's exposed API use super::sled_agent::SledAgent; +use crate::bootstrap::early_networking::EarlyNetworkConfig; use crate::params::{ CleanupContextUpdate, DiskEnsureBody, InstanceEnsureBody, InstancePutMigrationIdsBody, InstancePutStateBody, @@ -14,6 +15,7 @@ use crate::params::{ }; use crate::sled_agent::Error as SledAgentError; use crate::zone_bundle; +use bootstore::schemes::v0::NetworkConfig; use camino::Utf8PathBuf; use dropshot::{ endpoint, ApiDescription, FreeformBody, HttpError, HttpResponseCreated, @@ -27,7 +29,9 @@ use omicron_common::api::external::Error; use omicron_common::api::internal::nexus::{ DiskRuntimeState, InstanceRuntimeState, UpdateArtifactId, }; -use omicron_common::api::internal::shared::SwitchUplinks; +use omicron_common::api::internal::shared::{ + BgpConfig, PortConfigV1, SwitchPorts, +}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; @@ -64,6 +68,7 @@ pub fn api() -> SledApiDescription { api.register(vpc_firewall_rules_put)?; api.register(zpools_get)?; api.register(uplink_ensure)?; + api.register(early_network_port_update)?; Ok(()) } @@ -627,13 +632,114 @@ async fn timesync_get( #[endpoint { method = POST, - path = "/switch-uplinks", + path = "/switch-ports", }] async fn uplink_ensure( rqctx: RequestContext, - body: TypedBody, + body: TypedBody, ) -> Result { let sa = rqctx.context(); - sa.ensure_scrimlet_host_uplinks(body.into_inner().uplinks).await?; + sa.ensure_scrimlet_host_ports(body.into_inner().uplinks).await?; Ok(HttpResponseUpdatedNoContent()) } + +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)] +pub struct EarlyNetworkPortUpdate { + port: PortConfigV1, + bgp_configs: Vec, +} + +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, Default)] +pub struct EarlyNetworkPortChange { + previous_port_config: Option, + changed_bgp_configs: Vec, + added_bgp_configs: Vec, +} + +#[endpoint { + method = POST, + path = "/early-network-port-update", +}] +async fn early_network_port_update( + rqctx: RequestContext, + body: TypedBody, +) -> Result, HttpError> { + let rq = body.into_inner(); + let sa = rqctx.context(); + let bs = sa.bootstore(); + + // Read the current network config from the boot store. + + let config = bs.get_network_config().await.map_err(|e| { + HttpError::for_internal_error(format!("failed to get bootstore: {e}")) + })?; + + let mut config = match config { + Some(config) => EarlyNetworkConfig::try_from(config).map_err(|e| { + HttpError::for_internal_error(format!( + "deserialize early network config: {e}" + )) + })?, + None => { + return Err(HttpError::for_unavail( + None, + "early network config does not exist yet".into(), + )); + } + }; + + let mut rack_net_config = match config.rack_network_config { + Some(cfg) => cfg, + None => { + return Err(HttpError::for_unavail( + None, + "rack network config does not exist yet".into(), + )); + } + }; + + let mut result = EarlyNetworkPortChange::default(); + + // Update the config with this new information. + + for port in &mut rack_net_config.ports { + if port.port == rq.port.port { + result.previous_port_config = Some(port.clone()); + *port = rq.port.clone(); + break; + } + } + if result.previous_port_config.is_none() { + rack_net_config.ports.push(rq.port); + } + + for updated_bgp in &rq.bgp_configs { + let mut exists = false; + for resident_bgp in &mut rack_net_config.bgp { + if resident_bgp.asn == updated_bgp.asn { + result.changed_bgp_configs.push(resident_bgp.clone()); + *resident_bgp = updated_bgp.clone(); + exists = true; + break; + } + } + if !exists { + result.added_bgp_configs.push(updated_bgp.clone()); + } + } + rack_net_config.bgp.extend_from_slice(&result.added_bgp_configs); + + // Write the network config back to the boot store. + + config.generation += 1; + config.rack_network_config = Some(rack_net_config); + bs.update_network_config(NetworkConfig::from(config)).await.map_err( + |e| { + HttpError::for_internal_error(format!( + "failed to write updated config to boot store: {e}" + )) + }, + )?; + + Ok(HttpResponseOk(result)) +} diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 9b9f9130c15..212a554c473 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -578,14 +578,21 @@ impl ServiceInner { let value = NexusTypes::RackNetworkConfig { infra_ip_first: config.infra_ip_first, infra_ip_last: config.infra_ip_last, - uplinks: config - .uplinks + ports: config + .ports .iter() - .map(|config| NexusTypes::UplinkConfig { - gateway_ip: config.gateway_ip, + .map(|config| NexusTypes::PortConfigV1 { + port: config.port.clone(), + routes: config + .routes + .iter() + .map(|r| NexusTypes::RouteConfig { + destination: r.destination, + nexthop: r.nexthop, + }) + .collect(), + addresses: config.addresses.clone(), switch: config.switch.into(), - uplink_cidr: config.uplink_cidr, - uplink_port: config.uplink_port.clone(), uplink_port_speed: config .uplink_port_speed .clone() @@ -594,7 +601,23 @@ impl ServiceInner { .uplink_port_fec .clone() .into(), - uplink_vid: config.uplink_vid, + bgp_peers: config + .bgp_peers + .iter() + .map(|b| NexusTypes::BgpPeerConfig { + addr: b.addr, + asn: b.asn, + port: b.port.clone(), + }) + .collect(), + }) + .collect(), + bgp: config + .bgp + .iter() + .map(|config| NexusTypes::BgpConfig { + asn: config.asn, + originate: config.originate.clone(), }) .collect(), }; diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 58180e9b6f9..81976684487 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -77,7 +77,7 @@ use omicron_common::address::SLED_PREFIX; use omicron_common::address::WICKETD_PORT; use omicron_common::api::external::Generation; use omicron_common::api::internal::shared::{ - HostUplinkConfig, RackNetworkConfig, + HostPortConfig, RackNetworkConfig, }; use omicron_common::backoff::{ retry_notify, retry_policy_internal_service_aggressive, retry_policy_local, @@ -2567,19 +2567,19 @@ impl ServiceManager { let log = &self.inner.log; // Configure uplinks via DPD in our switch zone. - let our_uplinks = EarlyNetworkSetup::new(log) + let our_ports = EarlyNetworkSetup::new(log) .init_switch_config(rack_network_config, switch_zone_ip) .await? .into_iter() .map(From::from) .collect(); - self.ensure_scrimlet_host_uplinks(our_uplinks).await + self.ensure_scrimlet_host_ports(our_ports).await } - pub async fn ensure_scrimlet_host_uplinks( + pub async fn ensure_scrimlet_host_ports( &self, - our_uplinks: Vec, + our_ports: Vec, ) -> Result<(), Error> { // We expect the switch zone to be running, as we're called immediately // after `ensure_zone()` above and we just successfully configured @@ -2611,12 +2611,14 @@ impl ServiceManager { smfh.delpropgroup("uplinks")?; smfh.addpropgroup("uplinks", "application")?; - for uplink_config in &our_uplinks { - smfh.addpropvalue_type( - &format!("uplinks/{}_0", uplink_config.uplink_port,), - &uplink_config.uplink_cidr.to_string(), - "astring", - )?; + for port_config in &our_ports { + for addr in &port_config.addrs { + smfh.addpropvalue_type( + &format!("uplinks/{}_0", port_config.port,), + &addr.to_string(), + "astring", + )?; + } } smfh.refresh()?; diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index 7fb45b03898..d4da14f72b2 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -22,7 +22,7 @@ use illumos_utils::opte::params::SetVirtualNetworkInterfaceHost; use omicron_common::api::internal::nexus::DiskRuntimeState; use omicron_common::api::internal::nexus::InstanceRuntimeState; use omicron_common::api::internal::nexus::UpdateArtifactId; -use omicron_common::api::internal::shared::SwitchUplinks; +use omicron_common::api::internal::shared::SwitchPorts; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::sync::Arc; @@ -329,7 +329,7 @@ async fn del_v2p( }] async fn uplink_ensure( _rqctx: RequestContext>, - _body: TypedBody, + _body: TypedBody, ) -> Result { //let sa = rqctx.context(); //sa.ensure_scrimlet_host_uplinks(body.into_inner().uplinks).await?; diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index c1f388589ff..3cf3516e0f9 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -36,7 +36,7 @@ use omicron_common::address::{ }; use omicron_common::api::external::Vni; use omicron_common::api::internal::shared::{ - HostUplinkConfig, RackNetworkConfig, + HostPortConfig, RackNetworkConfig, }; use omicron_common::api::{ internal::nexus::DiskRuntimeState, internal::nexus::InstanceRuntimeState, @@ -236,6 +236,9 @@ struct SledAgentInner { // Object managing zone bundles. zone_bundler: zone_bundle::ZoneBundler, + + // A handle to the bootstore. + bootstore: bootstore::NodeHandle, } impl SledAgentInner { @@ -457,6 +460,7 @@ impl SledAgent { nexus_request_queue: NexusRequestQueue::new(), rack_network_config, zone_bundler, + bootstore: bootstore.clone(), }), log: log.clone(), }; @@ -907,14 +911,18 @@ impl SledAgent { self.inner.services.timesync_get().await.map_err(Error::from) } - pub async fn ensure_scrimlet_host_uplinks( + pub async fn ensure_scrimlet_host_ports( &self, - uplinks: Vec, + uplinks: Vec, ) -> Result<(), Error> { self.inner .services - .ensure_scrimlet_host_uplinks(uplinks) + .ensure_scrimlet_host_ports(uplinks) .await .map_err(Error::from) } + + pub fn bootstore(&self) -> bootstore::NodeHandle { + self.inner.bootstore.clone() + } } diff --git a/smf/sled-agent/non-gimlet/config-rss.toml b/smf/sled-agent/non-gimlet/config-rss.toml index 8a009dd6872..646320c52dd 100644 --- a/smf/sled-agent/non-gimlet/config-rss.toml +++ b/smf/sled-agent/non-gimlet/config-rss.toml @@ -41,7 +41,7 @@ external_dns_zone_name = "oxide.test" # the DNS domain delegated to the rack by the customer. Each of these addresses # must be contained in one of the "internal services" IP Pool ranges listed # below. -external_dns_ips = [ "192.168.1.20", "192.168.1.21" ] +external_dns_ips = [ "192.168.20.20", "192.168.20.21" ] # Initial TLS certificates for the external API # @@ -77,8 +77,8 @@ external_certificates = [] # # For more on this and what to put here, see docs/how-to-run.adoc. [[internal_services_ip_pool_ranges]] -first = "192.168.1.20" -last = "192.168.1.29" +first = "192.168.20.20" +last = "192.168.20.30" # TODO - this configuration is subject to change going forward. Ultimately these # parameters should be provided to the control plane via wicket, but we need to @@ -91,20 +91,20 @@ last = "192.168.1.29" # A range of IP addresses used by Boundary Services on the external network. In # a real system, these would be addresses of the uplink ports on the Sidecar. # With softnpu, only one address is used. -infra_ip_first = "192.168.1.30" -infra_ip_last = "192.168.1.30" +infra_ip_first = "192.168.20.3" +infra_ip_last = "192.168.20.3" # You can configure multiple uplinks by repeating the following stanza [[rack_network_config.uplinks]] # The gateway IP for the rack's external network -gateway_ip = "192.168.1.199" +gateway_ip = "192.168.20.1" # Name of the uplink port. This should always be "qsfp0" when using softnpu. uplink_port = "qsfp0" -uplink_port_speed = "40G" +uplink_port_speed = "100G" uplink_port_fec="none" # For softnpu, an address within the "infra" block above that will be used for # the softnpu uplink port. You can just pick the first address in that pool. -uplink_cidr = "192.168.1.30/32" +uplink_cidr = "192.168.20.3/29" # Switch to use for the uplink. For single-rack deployments this can be # "switch0" (upper slot) or "switch1" (lower slot). For single-node softnpu # and dendrite stub environments, use "switch0" diff --git a/tools/ci_download_softnpu_machinery b/tools/ci_download_softnpu_machinery index d37d4284767..cb5ea40210c 100755 --- a/tools/ci_download_softnpu_machinery +++ b/tools/ci_download_softnpu_machinery @@ -15,7 +15,7 @@ OUT_DIR="out/npuzone" # Pinned commit for softnpu ASIC simulator SOFTNPU_REPO="softnpu" -SOFTNPU_COMMIT="41b3a67b3d44f51528816ff8e539b4001df48305" +SOFTNPU_COMMIT="c1c42398c82b0220c8b5fa3bfba9c7a3bcaa0943" # This is the softnpu ASIC simulator echo "fetching npuzone" diff --git a/tools/create_virtual_hardware.sh b/tools/create_virtual_hardware.sh index d2b22e05c2f..7b4ed926fbe 100755 --- a/tools/create_virtual_hardware.sh +++ b/tools/create_virtual_hardware.sh @@ -59,7 +59,6 @@ function ensure_softnpu_zone { exit 1 fi out/npuzone/npuzone create sidecar \ - --sidecar-lite-commit be162e11b3e297371a3d3221a3bc0e625e6623f7 \ --omicron-zone \ --ports sc0_0,tfportrear0_0 \ --ports sc0_1,tfportqsfp0_0 diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version index d1a6c7ad95c..8f1537fa628 100644 --- a/tools/maghemite_ddm_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1,2 +1,2 @@ -COMMIT="f1813a2a07a9857f862daeed2bebe51060734c69" +COMMIT="7b88dbcc7810f4fe9c82a7459862a7340d7e09ce" SHA2="9737906555a60911636532f00f1dc2866dc7cd6553beb106e9e57beabad41cdf" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version index 1e13f8c5501..390560e2416 100644 --- a/tools/maghemite_mg_openapi_version +++ b/tools/maghemite_mg_openapi_version @@ -1,2 +1,2 @@ -COMMIT="f1813a2a07a9857f862daeed2bebe51060734c69" -SHA2="5588abe9b0d206014b7e28739f81b2a1a683e19ddbb3265bb7eae7adbd11e2d5" +COMMIT="7b88dbcc7810f4fe9c82a7459862a7340d7e09ce" +SHA2="72a6344a16f98d384c50258f354d32aa52e4c3c413c6c3ecebc128d56b06abc6" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums index 2fefd55292e..876c8befc54 100644 --- a/tools/maghemite_mgd_checksums +++ b/tools/maghemite_mgd_checksums @@ -1,2 +1,2 @@ -CIDL_SHA256="212c7d09958d8578dc1f6b5556e1d0dbfbbcc542aced90051195b25a4b20905e" -MGD_LINUX_SHA256="e77660ceb8c239d0d6f8d5614d40a5ef1e03df8503f12e2de3298f121e422d68" +CIDL_SHA256="82e965486e7acdbc4661258f1bc8d7751155f406f0973d55e61f6506d18984c2" +MGD_LINUX_SHA256="7f42776e053f415cba4b4b52171381af9da6f3aeec3cc8bcc745a2c9684ca0e5" diff --git a/wicket/src/rack_setup/config_template.toml b/wicket/src/rack_setup/config_template.toml index 4cb204b6942..48f100dd95f 100644 --- a/wicket/src/rack_setup/config_template.toml +++ b/wicket/src/rack_setup/config_template.toml @@ -67,3 +67,17 @@ uplink_cidr = "" # VLAN ID for this uplink; omit if no VLAN ID is needed uplink_vid = 1234 + +# Optional BGP configuration. Remove this section if not needed. +[[rack_network_config.bgp]] +# The autonomous system numer +asn = 47 + +# Prefixes to originate e.g., ["10.0.0.0/16"] +originated = [] + +# The neighbors to peer wth ``[{"addr": "10.2.3.4", "port": "port0"}]`` +peers = [] + +# Either `switch0` or `switch1`, matching the hardware. +switch = "switch0" diff --git a/wicket/src/rack_setup/config_toml.rs b/wicket/src/rack_setup/config_toml.rs index 5f0bb9e876a..5e6ee9532b0 100644 --- a/wicket/src/rack_setup/config_toml.rs +++ b/wicket/src/rack_setup/config_toml.rs @@ -202,20 +202,21 @@ fn populate_network_table( Value::String(Formatted::new(value)); } + //TODO i'm not sure what to do about the array valued data structures here // If `config.uplinks` is empty, we'll leave the template uplinks in place; // otherwise, replace it with the user's uplinks. - if !config.uplinks.is_empty() { + if !config.ports.is_empty() { *table.get_mut("uplinks").unwrap().as_array_of_tables_mut().unwrap() = config - .uplinks + .ports .iter() .map(|cfg| { let mut uplink = Table::new(); - let mut last_key = None; + let mut _last_key = None; for (property, value) in [ ("switch", cfg.switch.to_string()), - ("gateway_ip", cfg.gateway_ip.to_string()), - ("uplink_port", cfg.uplink_port.to_string()), + //TODO(ry) ("addresses", cfg.gateway_ip.to_string()), + //TODO(ry) ("uplink_port", cfg.uplink_port.to_string()), ( "uplink_port_speed", enum_to_toml_string(&cfg.uplink_port_speed), @@ -224,15 +225,17 @@ fn populate_network_table( "uplink_port_fec", enum_to_toml_string(&cfg.uplink_port_fec), ), - ("uplink_cidr", cfg.uplink_cidr.to_string()), + //TODO(ry) ("uplink_cidr", cfg.uplink_cidr.to_string()), ] { uplink.insert( property, Item::Value(Value::String(Formatted::new(value))), ); - last_key = Some(property); + _last_key = Some(property); } + /* TODO(ry) + if let Some(uplink_vid) = cfg.uplink_vid { uplink.insert( "uplink_vid", @@ -254,6 +257,7 @@ fn populate_network_table( .decor_mut() .set_suffix("\n# uplink_vid ="); } + */ uplink }) @@ -268,19 +272,25 @@ mod tests { use std::net::Ipv6Addr; use wicket_common::rack_setup::PutRssUserConfigInsensitive; use wicketd_client::types::Baseboard; + use wicketd_client::types::BgpConfig; + use wicketd_client::types::BgpPeerConfig; + use wicketd_client::types::PortConfigV1; use wicketd_client::types::PortFec; use wicketd_client::types::PortSpeed; + use wicketd_client::types::RouteConfig; use wicketd_client::types::SpIdentifier; use wicketd_client::types::SwitchLocation; - use wicketd_client::types::UplinkConfig; fn put_config_from_current_config( value: CurrentRssUserConfigInsensitive, ) -> PutRssUserConfigInsensitive { + use omicron_common::api::internal::shared::BgpConfig as InternalBgpConfig; + use omicron_common::api::internal::shared::BgpPeerConfig as InternalBgpPeerConfig; + use omicron_common::api::internal::shared::PortConfigV1 as InternalPortConfig; use omicron_common::api::internal::shared::PortFec as InternalPortFec; use omicron_common::api::internal::shared::PortSpeed as InternalPortSpeed; + use omicron_common::api::internal::shared::RouteConfig as InternalRouteConfig; use omicron_common::api::internal::shared::SwitchLocation as InternalSwitchLocation; - use omicron_common::api::internal::shared::UplinkConfig as InternalUplinkConfig; let rnc = value.rack_network_config.unwrap(); @@ -312,12 +322,29 @@ mod tests { rack_network_config: InternalRackNetworkConfig { infra_ip_first: rnc.infra_ip_first, infra_ip_last: rnc.infra_ip_last, - uplinks: rnc - .uplinks + ports: rnc + .ports .iter() - .map(|config| InternalUplinkConfig { - gateway_ip: config.gateway_ip, - uplink_port: config.uplink_port.clone(), + .map(|config| InternalPortConfig { + routes: config + .routes + .iter() + .map(|r| InternalRouteConfig { + destination: r.destination, + nexthop: r.nexthop, + }) + .collect(), + addresses: config.addresses.clone(), + bgp_peers: config + .bgp_peers + .iter() + .map(|p| InternalBgpPeerConfig { + asn: p.asn, + port: p.port.clone(), + addr: p.addr, + }) + .collect(), + port: config.port.clone(), uplink_port_speed: match config.uplink_port_speed { PortSpeed::Speed0G => InternalPortSpeed::Speed0G, PortSpeed::Speed1G => InternalPortSpeed::Speed1G, @@ -340,8 +367,6 @@ mod tests { PortFec::None => InternalPortFec::None, PortFec::Rs => InternalPortFec::Rs, }, - uplink_cidr: config.uplink_cidr, - uplink_vid: config.uplink_vid, switch: match config.switch { SwitchLocation::Switch0 => { InternalSwitchLocation::Switch0 @@ -352,6 +377,14 @@ mod tests { }, }) .collect(), + bgp: rnc + .bgp + .iter() + .map(|config| InternalBgpConfig { + asn: config.asn, + originate: config.originate.clone(), + }) + .collect(), }, } } @@ -395,15 +428,28 @@ mod tests { rack_network_config: Some(RackNetworkConfig { infra_ip_first: "172.30.0.1".parse().unwrap(), infra_ip_last: "172.30.0.10".parse().unwrap(), - uplinks: vec![UplinkConfig { - gateway_ip: "172.30.0.10".parse().unwrap(), - uplink_cidr: "172.30.0.1/24".parse().unwrap(), + ports: vec![PortConfigV1 { + addresses: vec!["172.30.0.1/24".parse().unwrap()], + routes: vec![RouteConfig { + destination: "0.0.0.0/0".parse().unwrap(), + nexthop: "172.30.0.10".parse().unwrap(), + }], + bgp_peers: vec![BgpPeerConfig { + asn: 47, + addr: "10.2.3.4".parse().unwrap(), + port: "port0".into(), + }], + //gateway_ip: "172.30.0.10".parse().unwrap(), + //uplink_cidr: "172.30.0.1/24".parse().unwrap(), uplink_port_speed: PortSpeed::Speed400G, uplink_port_fec: PortFec::Firecode, - uplink_port: "port0".into(), - uplink_vid: None, + port: "port0".into(), switch: SwitchLocation::Switch0, }], + bgp: vec![BgpConfig { + asn: 47, + originate: vec!["10.0.0.0/16".parse().unwrap()], + }], }), }; let template = TomlTemplate::populate(&config).to_string(); diff --git a/wicket/src/ui/panes/rack_setup.rs b/wicket/src/ui/panes/rack_setup.rs index 212ddff4da8..1600bc40ac8 100644 --- a/wicket/src/ui/panes/rack_setup.rs +++ b/wicket/src/ui/panes/rack_setup.rs @@ -695,12 +695,13 @@ fn rss_config_text<'a>( }; if let Some(cfg) = insensitive.rack_network_config.as_ref() { - for (i, uplink) in cfg.uplinks.iter().enumerate() { - let mut items = vec![ + for (i, uplink) in cfg.ports.iter().enumerate() { + let /*mut*/ items = vec![ vec![ Span::styled(" • Switch : ", label_style), Span::styled(uplink.switch.to_string(), ok_style), ], + /* TODO(ry) vec![ Span::styled(" • Gateway IP : ", label_style), Span::styled(uplink.gateway_ip.to_string(), ok_style), @@ -713,6 +714,7 @@ fn rss_config_text<'a>( Span::styled(" • Uplink port : ", label_style), Span::styled(uplink.uplink_port.clone(), ok_style), ], + */ vec![ Span::styled(" • Uplink port speed: ", label_style), Span::styled( @@ -725,6 +727,7 @@ fn rss_config_text<'a>( Span::styled(uplink.uplink_port_fec.to_string(), ok_style), ], ]; + /* TODO(ry) if let Some(uplink_vid) = uplink.uplink_vid { items.push(vec![ Span::styled(" • Uplink VLAN id : ", label_style), @@ -736,6 +739,7 @@ fn rss_config_text<'a>( Span::styled("none", ok_style), ]); } + */ append_list( &mut spans, diff --git a/wicketd/Cargo.toml b/wicketd/Cargo.toml index 1044e1ff51f..56388afd344 100644 --- a/wicketd/Cargo.toml +++ b/wicketd/Cargo.toml @@ -24,6 +24,7 @@ hubtools.workspace = true http.workspace = true hyper.workspace = true illumos-utils.workspace = true +ipnetwork.workspace = true itertools.workspace = true reqwest.workspace = true schemars.workspace = true diff --git a/wicketd/src/preflight_check/uplink.rs b/wicketd/src/preflight_check/uplink.rs index 58955d04d62..ebcba90645f 100644 --- a/wicketd/src/preflight_check/uplink.rs +++ b/wicketd/src/preflight_check/uplink.rs @@ -17,12 +17,13 @@ use dpd_client::ClientState as DpdClientState; use either::Either; use illumos_utils::zone::SVCCFG; use illumos_utils::PFEXEC; +use ipnetwork::IpNetwork; use omicron_common::address::DENDRITE_PORT; +use omicron_common::api::internal::shared::PortConfigV1; use omicron_common::api::internal::shared::PortFec as OmicronPortFec; use omicron_common::api::internal::shared::PortSpeed as OmicronPortSpeed; use omicron_common::api::internal::shared::RackNetworkConfig; use omicron_common::api::internal::shared::SwitchLocation; -use omicron_common::api::internal::shared::UplinkConfig; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; @@ -32,7 +33,6 @@ use slog::Logger; use std::collections::BTreeSet; use std::collections::HashMap; use std::net::IpAddr; -use std::net::Ipv4Addr; use std::str::FromStr; use std::sync::Arc; use std::sync::Mutex; @@ -66,8 +66,6 @@ const CHRONYD: &str = "/usr/sbin/chronyd"; const IPADM: &str = "/usr/sbin/ipadm"; const ROUTE: &str = "/usr/sbin/route"; -const DPD_DEFAULT_IPV4_CIDR: &str = "0.0.0.0/0"; - pub(super) async fn run_local_uplink_preflight_check( network_config: RackNetworkConfig, dns_servers: Vec, @@ -90,7 +88,7 @@ pub(super) async fn run_local_uplink_preflight_check( let mut engine = UpdateEngine::new(log, sender); for uplink in network_config - .uplinks + .ports .iter() .filter(|uplink| uplink.switch == our_switch_location) { @@ -131,7 +129,7 @@ pub(super) async fn run_local_uplink_preflight_check( fn add_steps_for_single_local_uplink_preflight_check<'a>( engine: &mut UpdateEngine<'a>, dpd_client: &'a DpdClient, - uplink: &'a UplinkConfig, + uplink: &'a PortConfigV1, dns_servers: &'a [IpAddr], ntp_servers: &'a [String], dns_name_to_query: Option<&'a str>, @@ -153,7 +151,7 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( // Timeout we give to chronyd during the NTP check, in seconds. const CHRONYD_CHECK_TIMEOUT_SECS: &str = "30"; - let registrar = engine.for_component(uplink.uplink_port.clone()); + let registrar = engine.for_component(uplink.port.clone()); let prev_step = registrar .new_step( @@ -162,7 +160,7 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( |_cx| async { // Check that the port name is valid and that it has no links // configured already. - let port_id = PortId::from_str(&uplink.uplink_port) + let port_id = PortId::from_str(&uplink.port) .map_err(UplinkPreflightTerminalError::InvalidPortName)?; let links = dpd_client .link_list(&port_id) @@ -192,11 +190,11 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( { Ok(_response) => { let metadata = vec![format!( - "configured {}/{}: ip {}, gateway {}", + "configured {}/{}: ips {:#?}, routes {:#?}", *port_id, link_id.0, - uplink.uplink_cidr, - uplink.gateway_ip + uplink.addresses, + uplink.routes )]; StepSuccess::new((port_id, link_id)) .with_metadata(metadata) @@ -298,93 +296,99 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( // Tell the `uplink` service about the IP address we created on // the switch when configuring the uplink. let uplink_property = - UplinkProperty(format!("uplinks/{}_0", uplink.uplink_port)); - let uplink_cidr = uplink.uplink_cidr.to_string(); - - if let Err(err) = execute_command(&[ - SVCCFG, - "-s", - UPLINK_SMF_NAME, - "addpropvalue", - &uplink_property.0, - "astring:", - &uplink_cidr, - ]) - .await - { - return StepWarning::new( - Err(L2Failure::UplinkAddProperty(level1)), - format!("could not add uplink property: {err}"), - ) - .into(); - }; - - if let Err(err) = execute_command(&[ - SVCCFG, - "-s", - UPLINK_DEFAULT_SMF_NAME, - "refresh", - ]) - .await - { - return StepWarning::new( - Err(L2Failure::UplinkRefresh(level1, uplink_property)), - format!("could not add uplink property: {err}"), - ) - .into(); - }; - - // Wait for the `uplink` service to create the IP address. - let start_waiting_addr = Instant::now(); - 'waiting_for_addr: loop { - let ipadm_out = match execute_command(&[ - IPADM, - "show-addr", - "-p", - "-o", - "addr", + UplinkProperty(format!("uplinks/{}_0", uplink.port)); + + for addr in &uplink.addresses { + let uplink_cidr = addr.to_string(); + if let Err(err) = execute_command(&[ + SVCCFG, + "-s", + UPLINK_SMF_NAME, + "addpropvalue", + &uplink_property.0, + "astring:", + &uplink_cidr, ]) .await { - Ok(stdout) => stdout, - Err(err) => { - return StepWarning::new( - Err(L2Failure::RunIpadm( - level1, - uplink_property, - )), - format!("failed running ipadm: {err}"), - ) - .into(); - } + return StepWarning::new( + Err(L2Failure::UplinkAddProperty(level1)), + format!("could not add uplink property: {err}"), + ) + .into(); }; - for line in ipadm_out.split('\n') { - if line == uplink_cidr { - break 'waiting_for_addr; - } - } - - // We did not find `uplink_cidr` in the output of ipadm; - // sleep a bit and try again, unless we've been waiting too - // long already. - if start_waiting_addr.elapsed() < UPLINK_SVC_WAIT_TIMEOUT { - tokio::time::sleep(UPLINK_SVC_RETRY_DELAY).await; - } else { + if let Err(err) = execute_command(&[ + SVCCFG, + "-s", + UPLINK_DEFAULT_SMF_NAME, + "refresh", + ]) + .await + { return StepWarning::new( - Err(L2Failure::WaitingForHostAddr( + Err(L2Failure::UplinkRefresh( level1, uplink_property, )), - format!( - "timed out waiting for `uplink` to \ - create {uplink_cidr}" - ), + format!("could not add uplink property: {err}"), ) .into(); + }; + + // Wait for the `uplink` service to create the IP address. + let start_waiting_addr = Instant::now(); + 'waiting_for_addr: loop { + let ipadm_out = match execute_command(&[ + IPADM, + "show-addr", + "-p", + "-o", + "addr", + ]) + .await + { + Ok(stdout) => stdout, + Err(err) => { + return StepWarning::new( + Err(L2Failure::RunIpadm( + level1, + uplink_property, + )), + format!("failed running ipadm: {err}"), + ) + .into(); + } + }; + + for line in ipadm_out.split('\n') { + if line == uplink_cidr { + break 'waiting_for_addr; + } + } + + // We did not find `uplink_cidr` in the output of ipadm; + // sleep a bit and try again, unless we've been waiting too + // long already. + if start_waiting_addr.elapsed() + < UPLINK_SVC_WAIT_TIMEOUT + { + tokio::time::sleep(UPLINK_SVC_RETRY_DELAY).await; + } else { + return StepWarning::new( + Err(L2Failure::WaitingForHostAddr( + level1, + uplink_property, + )), + format!( + "timed out waiting for `uplink` to \ + create {uplink_cidr}" + ), + ) + .into(); + } } } - let metadata = vec![format!("configured {}", uplink_property.0)]; StepSuccess::new(Ok(L2Success { level1, uplink_property })) @@ -410,27 +414,29 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( } }; - // Add the gateway as the default route in illumos. - if let Err(err) = execute_command(&[ - ROUTE, - "add", - "-inet", - "default", - &uplink.gateway_ip.to_string(), - ]) - .await - { - return StepWarning::new( - Err(RoutingFailure::HostDefaultRoute(level2)), - format!("could not add default route: {err}"), - ) - .into(); - }; + for r in &uplink.routes { + // Add the gateway as the default route in illumos. + if let Err(err) = execute_command(&[ + ROUTE, + "add", + "-inet", + &r.destination.to_string(), + &r.nexthop.to_string(), + ]) + .await + { + return StepWarning::new( + Err(RoutingFailure::HostDefaultRoute(level2)), + format!("could not add default route: {err}"), + ) + .into(); + }; + } StepSuccess::new(Ok(RoutingSuccess { level2 })) .with_metadata(vec![format!( - "added default route to {}", - uplink.gateway_ip + "added routes {:#?}", + uplink.routes, )]) .into() }, @@ -595,21 +601,24 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( } }; - if remove_host_route { - execute_command(&[ - ROUTE, - "delete", - "-inet", - "default", - &uplink.gateway_ip.to_string(), - ]) - .await - .map_err(|err| { - UplinkPreflightTerminalError::RemoveHostRoute { - err, - gateway_ip: uplink.gateway_ip, - } - })?; + for r in &uplink.routes { + if remove_host_route { + execute_command(&[ + ROUTE, + "delete", + "-inet", + &r.destination.to_string(), + &r.nexthop.to_string(), + ]) + .await + .map_err(|err| { + UplinkPreflightTerminalError::RemoveHostRoute { + err, + destination: r.destination, + nexthop: r.nexthop, + } + })?; + } } StepSuccess::new(Ok(level2)).into() @@ -730,7 +739,7 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( } fn build_port_settings( - uplink: &UplinkConfig, + uplink: &PortConfigV1, link_id: &LinkId, ) -> PortSettings { // Map from omicron_common types to dpd_client types @@ -758,10 +767,12 @@ fn build_port_settings( v6_routes: HashMap::new(), }; + let addrs = uplink.addresses.iter().map(|a| a.ip()).collect(); + port_settings.links.insert( link_id.to_string(), LinkSettings { - addrs: vec![IpAddr::V4(uplink.uplink_cidr.ip())], + addrs, params: LinkCreate { // TODO we should take these parameters too // https://github.com/oxidecomputer/omicron/issues/3061 @@ -773,14 +784,16 @@ fn build_port_settings( }, ); - port_settings.v4_routes.insert( - DPD_DEFAULT_IPV4_CIDR.parse().unwrap(), - RouteSettingsV4 { - link_id: link_id.0, - nexthop: uplink.gateway_ip, - vid: uplink.uplink_vid, - }, - ); + for r in &uplink.routes { + if let (IpNetwork::V4(dst), IpAddr::V4(nexthop)) = + (r.destination, r.nexthop) + { + port_settings.v4_routes.insert( + dst.to_string(), + RouteSettingsV4 { link_id: link_id.0, nexthop, vid: None }, + ); + } + } port_settings } @@ -890,8 +903,10 @@ pub(crate) enum UplinkPreflightTerminalError { err: DpdError, port_id: PortId, }, - #[error("failed to remove host OS route to gateway {gateway_ip}: {err}")] - RemoveHostRoute { err: String, gateway_ip: Ipv4Addr }, + #[error( + "failed to remove host OS route {destination} -> {nexthop}: {err}" + )] + RemoveHostRoute { err: String, destination: IpNetwork, nexthop: IpAddr }, #[error("failed to remove uplink SMF property {property:?}: {err}")] RemoveSmfProperty { property: String, err: String }, #[error("failed to refresh uplink service config: {0}")] diff --git a/wicketd/src/rss_config.rs b/wicketd/src/rss_config.rs index 1dc9f849857..f335754318a 100644 --- a/wicketd/src/rss_config.rs +++ b/wicketd/src/rss_config.rs @@ -454,17 +454,20 @@ impl From<&'_ CurrentRssConfig> for CurrentRssUserConfig { fn validate_rack_network_config( config: &RackNetworkConfig, ) -> Result { + use bootstrap_agent_client::types::BgpConfig as BaBgpConfig; + use bootstrap_agent_client::types::BgpPeerConfig as BaBgpPeerConfig; + use bootstrap_agent_client::types::PortConfigV1 as BaPortConfigV1; use bootstrap_agent_client::types::PortFec as BaPortFec; use bootstrap_agent_client::types::PortSpeed as BaPortSpeed; + use bootstrap_agent_client::types::RouteConfig as BaRouteConfig; use bootstrap_agent_client::types::SwitchLocation as BaSwitchLocation; - use bootstrap_agent_client::types::UplinkConfig as BaUplinkConfig; use omicron_common::api::internal::shared::PortFec; use omicron_common::api::internal::shared::PortSpeed; use omicron_common::api::internal::shared::SwitchLocation; // Ensure that there is at least one uplink - if config.uplinks.is_empty() { - return Err(anyhow!("Must have at least one uplink configured")); + if config.ports.is_empty() { + return Err(anyhow!("Must have at least one port configured")); } // Make sure `infra_ip_first`..`infra_ip_last` is a well-defined range... @@ -475,16 +478,20 @@ fn validate_rack_network_config( }, )?; - // iterate through each UplinkConfig - for uplink_config in &config.uplinks { - // ... and check that it contains `uplink_ip`. - if uplink_config.uplink_cidr.ip() < infra_ip_range.first - || uplink_config.uplink_cidr.ip() > infra_ip_range.last - { - bail!( + // TODO this implies a single contiguous range for port IPs which is over + // constraining + // iterate through each port config + for port_config in &config.ports { + for addr in &port_config.addresses { + // ... and check that it contains `uplink_ip`. + if addr.ip() < infra_ip_range.first + || addr.ip() > infra_ip_range.last + { + bail!( "`uplink_cidr`'s IP address must be in the range defined by \ `infra_ip_first` and `infra_ip_last`" ); + } } } // TODO Add more client side checks on `rack_network_config` contents? @@ -492,17 +499,33 @@ fn validate_rack_network_config( Ok(bootstrap_agent_client::types::RackNetworkConfig { infra_ip_first: config.infra_ip_first, infra_ip_last: config.infra_ip_last, - uplinks: config - .uplinks + ports: config + .ports .iter() - .map(|config| BaUplinkConfig { - gateway_ip: config.gateway_ip, + .map(|config| BaPortConfigV1 { + port: config.port.clone(), + routes: config + .routes + .iter() + .map(|r| BaRouteConfig { + destination: r.destination, + nexthop: r.nexthop, + }) + .collect(), + addresses: config.addresses.clone(), + bgp_peers: config + .bgp_peers + .iter() + .map(|p| BaBgpPeerConfig { + addr: p.addr, + asn: p.asn, + port: p.port.clone(), + }) + .collect(), switch: match config.switch { SwitchLocation::Switch0 => BaSwitchLocation::Switch0, SwitchLocation::Switch1 => BaSwitchLocation::Switch1, }, - uplink_cidr: config.uplink_cidr, - uplink_port: config.uplink_port.clone(), uplink_port_speed: match config.uplink_port_speed { PortSpeed::Speed0G => BaPortSpeed::Speed0G, PortSpeed::Speed1G => BaPortSpeed::Speed1G, @@ -519,7 +542,14 @@ fn validate_rack_network_config( PortFec::None => BaPortFec::None, PortFec::Rs => BaPortFec::Rs, }, - uplink_vid: config.uplink_vid, + }) + .collect(), + bgp: config + .bgp + .iter() + .map(|config| BaBgpConfig { + asn: config.asn, + originate: config.originate.clone(), }) .collect(), })