diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh index b58d009aa00..4b95a5275bd 100755 --- a/.github/buildomat/jobs/deploy.sh +++ b/.github/buildomat/jobs/deploy.sh @@ -110,7 +110,7 @@ z_swadm () { # only set this if you want to override the version of opte/xde installed by the # install_opte.sh script -OPTE_COMMIT="7bba530f897f69755723be2218de2628af0eb771" +OPTE_COMMIT="986b796324c1bec791335e46404d8f9327c21a6c" if [[ "x$OPTE_COMMIT" != "x" ]]; then curl -sSfOL https://buildomat.eng.oxide.computer/public/file/oxidecomputer/opte/module/$OPTE_COMMIT/xde pfexec rem_drv xde || true diff --git a/Cargo.lock b/Cargo.lock index 3b5aea76b30..8eba5c16474 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4185,7 +4185,7 @@ dependencies = [ [[package]] name = "illumos-sys-hdrs" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=7bba530f897f69755723be2218de2628af0eb771#7bba530f897f69755723be2218de2628af0eb771" +source = "git+https://github.com/oxidecomputer/opte?rev=986b796324c1bec791335e46404d8f9327c21a6c#986b796324c1bec791335e46404d8f9327c21a6c" [[package]] name = "illumos-utils" @@ -4626,7 +4626,7 @@ dependencies = [ [[package]] name = "kstat-macro" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=7bba530f897f69755723be2218de2628af0eb771#7bba530f897f69755723be2218de2628af0eb771" +source = "git+https://github.com/oxidecomputer/opte?rev=986b796324c1bec791335e46404d8f9327c21a6c#986b796324c1bec791335e46404d8f9327c21a6c" dependencies = [ "quote", "syn 2.0.77", @@ -7260,7 +7260,7 @@ dependencies = [ [[package]] name = "opte" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=7bba530f897f69755723be2218de2628af0eb771#7bba530f897f69755723be2218de2628af0eb771" +source = "git+https://github.com/oxidecomputer/opte?rev=986b796324c1bec791335e46404d8f9327c21a6c#986b796324c1bec791335e46404d8f9327c21a6c" dependencies = [ "cfg-if", "dyn-clone", @@ -7277,7 +7277,7 @@ dependencies = [ [[package]] name = "opte-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=7bba530f897f69755723be2218de2628af0eb771#7bba530f897f69755723be2218de2628af0eb771" +source = "git+https://github.com/oxidecomputer/opte?rev=986b796324c1bec791335e46404d8f9327c21a6c#986b796324c1bec791335e46404d8f9327c21a6c" dependencies = [ "illumos-sys-hdrs", "ipnetwork", @@ -7289,7 +7289,7 @@ dependencies = [ [[package]] name = "opte-ioctl" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=7bba530f897f69755723be2218de2628af0eb771#7bba530f897f69755723be2218de2628af0eb771" +source = "git+https://github.com/oxidecomputer/opte?rev=986b796324c1bec791335e46404d8f9327c21a6c#986b796324c1bec791335e46404d8f9327c21a6c" dependencies = [ "libc", "libnet 0.1.0 (git+https://github.com/oxidecomputer/netadm-sys)", @@ -7357,7 +7357,7 @@ dependencies = [ [[package]] name = "oxide-vpc" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=7bba530f897f69755723be2218de2628af0eb771#7bba530f897f69755723be2218de2628af0eb771" +source = "git+https://github.com/oxidecomputer/opte?rev=986b796324c1bec791335e46404d8f9327c21a6c#986b796324c1bec791335e46404d8f9327c21a6c" dependencies = [ "cfg-if", "illumos-sys-hdrs", diff --git a/Cargo.toml b/Cargo.toml index e6d44caa612..2c1fbf65356 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -468,7 +468,7 @@ omicron-test-utils = { path = "test-utils" } omicron-workspace-hack = "0.1.0" omicron-zone-package = "0.11.1" oxide-client = { path = "clients/oxide-client" } -oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "7bba530f897f69755723be2218de2628af0eb771", features = [ "api", "std" ] } +oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "986b796324c1bec791335e46404d8f9327c21a6c", features = [ "api", "std" ] } oxlog = { path = "dev-tools/oxlog" } oxnet = { git = "https://github.com/oxidecomputer/oxnet" } once_cell = "1.19.0" @@ -478,7 +478,7 @@ openapiv3 = "2.0.0" # must match samael's crate! openssl = "0.10" openssl-sys = "0.9" -opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "7bba530f897f69755723be2218de2628af0eb771" } +opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "986b796324c1bec791335e46404d8f9327c21a6c" } oso = "0.27" owo-colors = "4.1.0" oximeter = { path = "oximeter/oximeter" } diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 246b82efadb..0eb8eb5fb53 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -841,7 +841,7 @@ pub struct ResolvedVpcRouteSet { /// which can choose them as a source. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)] pub struct ExternalIpGatewayMap { - pub mappings: HashMap>, + pub mappings: HashMap>>, } /// Describes the purpose of the dataset. diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index d6585cd39d0..746c5a36d82 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -83,7 +83,7 @@ struct PortManagerInner { /// attached to each NIC. /// /// IGW IDs are specific to the VPC of each NIC. - eip_gateways: Mutex>>, + eip_gateways: Mutex>>>, } impl PortManagerInner { @@ -692,7 +692,11 @@ impl PortManager { } let inet_gw_map = if let Some(map) = inet_gw_map { - Some(map.into_iter().map(|(k, v)| (k.into(), v)).collect()) + Some( + map.into_iter() + .map(|(k, v)| (k.into(), v.into_iter().collect())) + .collect(), + ) } else { None }; diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index add444b948f..3811ef95101 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -59,6 +59,7 @@ use nexus_db_model::InternetGateway; use nexus_db_model::InternetGatewayIpAddress; use nexus_db_model::InternetGatewayIpPool; use nexus_db_model::IpPoolRange; +use nexus_db_model::NetworkInterfaceKind; use nexus_types::deployment::BlueprintZoneFilter; use nexus_types::deployment::SledFilter; use omicron_common::api::external::http_pagination::PaginatedBy; @@ -2318,13 +2319,6 @@ impl DataStore { } // XXX: maybe wants to live in external IP? - // XXX: this assumes that only one IGW can be associated - // with each IP per VPC. I would be... surprised if - // we allow a pool to be referenced by several IGWs - // in the same VPC? But maybe that's worth discussion. - // There's nothing stopping us from changing OPTE to - // insert several nat rules if there exists >1 IGW value - // (or just implementing an OR predicate). /// Returns a (Ip, NicId) -> InetGwId map which identifies, for a given sled: /// * a) which external IPs belong to any of its services/instances/probe NICs. /// * b) whether each IP is linked to an Internet Gateway via its parent pool. @@ -2332,12 +2326,14 @@ impl DataStore { &self, opctx: &OpContext, sled_id: Uuid, - ) -> Result>, Error> { + ) -> Result>>, Error> { // TODO: give GW-bound addresses preferential treatment. use db::schema::external_ip as eip; use db::schema::external_ip::dsl as eip_dsl; use db::schema::internet_gateway as igw; use db::schema::internet_gateway::dsl as igw_dsl; + use db::schema::internet_gateway_ip_address as igw_ip; + use db::schema::internet_gateway_ip_address::dsl as igw_ip_dsl; use db::schema::internet_gateway_ip_pool as igw_pool; use db::schema::internet_gateway_ip_pool::dsl as igw_pool_dsl; use db::schema::network_interface as ni; @@ -2380,29 +2376,66 @@ impl DataStore { .load_async::<(IpNetwork, Uuid, Uuid)>(&*conn) .await; - // TODO: service & probe mappings. - // note that the current sled-agent design - // does not yet allow us to re-ensure the set of - // external IPs for non-instance entities. - // if we insert those here, we need to be sure that - // the mappings are ignored by sled-agent for new - // services/probes/etc. - match instance_mappings { Ok(map) => { for (ip, nic_id, inet_gw_id) in map { let per_nic: &mut HashMap<_, _> = out.entry(nic_id).or_default(); + let igw_list: &mut HashSet<_> = + per_nic.entry(ip.ip()).or_default(); - per_nic.insert(ip.ip(), inet_gw_id); + igw_list.insert(inet_gw_id); } + } + Err(e) => { + return Err(Error::non_resourcetype_not_found(&format!( + "unable to find IGW mappings for sled {sled_id}: {e}" + ))) + } + } + + // Map all individual IPs bound to IGWs to NICs in their VPC. + let indiv_ip_mappings = ni::table + .inner_join(igw::table.on(igw_dsl::vpc_id.eq(ni::vpc_id))) + .inner_join( + igw_ip::table + .on(igw_ip_dsl::internet_gateway_id.eq(igw_dsl::id)), + ) + .filter(ni::time_deleted.is_null()) + .filter(ni::kind.eq(NetworkInterfaceKind::Instance)) + .filter(igw_dsl::time_deleted.is_null()) + .filter(igw_ip_dsl::time_deleted.is_null()) + .select((igw_ip_dsl::address, ni::id, igw_dsl::id)) + .load_async::<(IpNetwork, Uuid, Uuid)>(&*conn) + .await; - Ok(out) + match indiv_ip_mappings { + Ok(map) => { + for (ip, nic_id, inet_gw_id) in map { + let per_nic: &mut HashMap<_, _> = + out.entry(nic_id).or_default(); + let igw_list: &mut HashSet<_> = + per_nic.entry(ip.ip()).or_default(); + + igw_list.insert(inet_gw_id); + } + } + Err(e) => { + return Err(Error::non_resourcetype_not_found(&format!( + "unable to find IGW mappings for sled {sled_id}: {e}" + ))) } - Err(e) => Err(Error::non_resourcetype_not_found(&format!( - "unable to find IGW mappings for sled {sled_id}: {e}" - ))), } + + // TODO: service & probe EIP mappings. + // note that the current sled-agent design + // does not yet allow us to re-ensure the set of + // external IPs for non-instance entities. + // if we insert those here, we need to be sure that + // the mappings are ignored by sled-agent for new + // services/probes/etc. + + Ok(out) } /// Resolve all targets in a router into concrete details. diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 902a8e35050..c80f2770fa6 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -3018,8 +3018,12 @@ "additionalProperties": { "type": "object", "additionalProperties": { - "type": "string", - "format": "uuid" + "type": "array", + "items": { + "type": "string", + "format": "uuid" + }, + "uniqueItems": true } } }