Skip to content

Commit

Permalink
fix instance igw vpc route resolution
Browse files Browse the repository at this point in the history
  • Loading branch information
@rcgoodfellow
rcgoodfellow committed Sep 28, 2024
1 parent 5e90c43 commit 564ac71
Show file tree
Hide file tree
Showing 2 changed files with 118 additions and 46 deletions.
2 changes: 0 additions & 2 deletions .github/buildomat/jobs/deploy.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,6 @@ z_swadm () {
pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm $@
}

set +x
# only set this if you want to override the version of opte/xde installed by the
# install_opte.sh script
OPTE_COMMIT="d2acf326ae1216b3bd78e5bfd677aea813176314"
Expand All @@ -122,7 +121,6 @@ if [[ "x$OPTE_COMMIT" != "x" ]]; then
cp opteadm /tmp/opteadm
pfexec mv opteadm /opt/oxide/opte/bin/opteadm
fi
set -x

#
# XXX work around 14537 (UFS should not allow directories to be unlinked) which
Expand Down
162 changes: 118 additions & 44 deletions nexus/db-queries/src/db/datastore/vpc.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ use crate::db::collection_attach::AttachError;
use crate::db::collection_attach::DatastoreAttachTarget;
use crate::db::collection_insert::AsyncInsertError;
use crate::db::collection_insert::DatastoreCollection;
use crate::db::datastore::InstanceAndActiveVmm;
use crate::db::error::public_error_from_diesel;
use crate::db::error::ErrorHandler;
use crate::db::identity::Resource;
Expand Down Expand Up @@ -2614,7 +2615,12 @@ impl DataStore {
// destination -> target rule where the target is the
// internet gateway parameterized by the external ip.
if let RouteTarget::InternetGateway(name) = &rule.target.0 {
info!(opctx.log, "Internet gateway '{name}' resolving targets");
info!(
opctx.log,
"Internet gateway resolving targets";
"name" => name.to_string(),
"vpc" => authz_vpc.id().to_string(),
);
let conn = self.pool_connection_authorized(opctx).await?;
let igw = db::lookup::LookupPath::new(opctx, self)
.vpc_id(authz_vpc.id())
Expand All @@ -2627,8 +2633,11 @@ impl DataStore {
Err(e) => {
warn!(
opctx.log,
"Internet gateway '{name}' lookup failed \
when resolving vpc router rules: {e}",
"Internet gateway lookup failed \
when resolving vpc router rules:";
"error" => e.to_string(),
"name" => name.to_string(),
"vpc" => authz_vpc.id().to_string(),
);
continue;
}
Expand All @@ -2647,35 +2656,58 @@ impl DataStore {
Err(e) => {
warn!(
opctx.log,
"Internet gateway '{name}' pool lookup failed \
when resolving vpc router rules: {e}"
"Internet gateway pool lookup failed \
when resolving vpc router rules:";
"error" => e.to_string(),
"name" => name.to_string(),
"vpc" => authz_vpc.id().to_string(),
"igw" => authz_igw.id().to_string(),
);
continue;
}
};
info!(
opctx.log,
"Internet gateway '{name}' found {} pools",
igw_pools.len()
"Internet gateway found {} pools",
igw_pools.len();
"name" => name.to_string(),
"vpc" => authz_vpc.id().to_string(),
"igw" => authz_igw.id().to_string(),
);

let mut parents: Vec<Uuid> =
match opctx.authn.silo_or_builtin()? {
None => self
.service_network_interfaces_all_list_batched(opctx)
.await?
.iter()
.map(|x| x.service_id)
.collect(),
Some(_) => Vec::new(),
};

parents.extend(instances.values().map(|x| x.1.instance_id));
let parents: Vec<Uuid> = if authz_vpc.id() == *SERVICES_VPC_ID {
self.service_network_interfaces_all_list_batched(opctx)
.await?
.iter()
.map(|x| x.service_id)
.collect()
} else {
let mut instances = Vec::new();
let mut paginator = Paginator::new(SQL_BATCH_SIZE);
while let Some(p) = paginator.next() {
let batch = self
.instance_list(
opctx,
&authz_project,
&PaginatedBy::Id(p.current_pagparams()),
)
.await?;
paginator = p.found_batch(
&batch,
&|s: &InstanceAndActiveVmm| s.instance.id(),
);
instances
.extend(batch.into_iter().map(|x| x.instance.id()));
}
instances
};

info!(
opctx.log,
"Internet gateway '{name}' found {} parents",
parents.len()
"Internet gateway found {} parents",
parents.len();
"name" => name.to_string(),
"vpc" => authz_vpc.id().to_string(),
);

for igw_pool in &igw_pools {
Expand All @@ -2691,14 +2723,26 @@ impl DataStore {
Err(e) => {
warn!(
opctx.log,
"Internet gateway '{name}' pool range lookup failed \
for pool id {} when resolving vpc router rules: {e}",
igw_pool.ip_pool_id,
"Internet gateway pool range lookup failed \
when resolving vpc router rules";
"name" => name.to_string(),
"vpc" => authz_vpc.id().to_string(),
"ip_pool" => igw_pool.ip_pool_id.to_string(),
"error" => e.to_string(),
);
continue;
}
};

info!(
opctx.log,
"Internet gateway found {} ip pool ranges",
instances.len();
"name" => name.to_string(),
"vpc" => authz_vpc.id().to_string(),
"ip_pool" => igw_pool.ip_pool_id.to_string(),
);

for parent_id in &parents {
use db::schema::external_ip::dsl as xip;
let ext_ips = xip::external_ip
Expand All @@ -2713,25 +2757,35 @@ impl DataStore {
Err(e) => {
warn!(
opctx.log,
"Internet gateway '{name}' external ip look \
up failed when resolving vpc router rules: {e}",
"Internet gateway external ip look \
up failed when resolving vpc router rules";
"name" => name.to_string(),
"vpc" => authz_vpc.id().to_string(),
"parent" => parent_id.to_string(),
"error" => e.to_string(),
);
continue;
}
};

info!(
opctx.log,
"Internet gateway '{name}' found {} external_ips",
ext_ips.len()
"Internet gateway found {} external_ips for parent",
ext_ips.len();
"name" => name.to_string(),
"vpc" => authz_vpc.id().to_string(),
"parent" => parent_id.to_string(),
);

for ext_ip in &ext_ips {
info!(
opctx.log,
"Internet gateway '{name}' \
"Internet gateway \
v4_dest: {v4_dest:?}, v6_dest: {v6_dest:?}, ext_ip: {}",
ext_ip.ip.ip(),
ext_ip.ip.ip();
"name" => name.to_string(),
"vpc" => authz_vpc.id().to_string(),
"parent" => parent_id.to_string(),
);

match ext_ip.ip.ip() {
Expand All @@ -2755,8 +2809,11 @@ impl DataStore {
Err(e) => {
warn!(
opctx.log,
"Internet gateway '{name}' find sled id for ip {}: {e}",
ext_ip.ip.ip(),
"Internet gateway find sled id for ip {}: {e}",
ext_ip.ip.ip();
"name" => name.to_string(),
"vpc" => authz_vpc.id().to_string(),
"parent" => parent_id.to_string(),
);
continue;
}
Expand All @@ -2772,8 +2829,11 @@ impl DataStore {
}
info!(
opctx.log,
"Internet gateway '{name}' adding route \
{dest} -> {v4}"
"Internet gateway adding route \
{dest} -> {v4}";
"name" => name.to_string(),
"vpc" => authz_vpc.id().to_string(),
"parent" => parent_id.to_string(),
);
found = true;
break;
Expand All @@ -2782,9 +2842,13 @@ impl DataStore {
if !found {
warn!(
opctx.log,
"Internet gateway '{name}' no suitable \
ipv4 range found for {} in pool {}",
ext_ip.ip.ip(), igw_pool.ip_pool_id,
"Internet gateway no suitable \
ipv4 range found for {} in pool",
ext_ip.ip.ip();
"name" => name.to_string(),
"vpc" => authz_vpc.id().to_string(),
"parent" => parent_id.to_string(),
"pool" => igw_pool.ip_pool_id.to_string(),
);
}
}
Expand All @@ -2809,8 +2873,11 @@ impl DataStore {
Err(e) => {
warn!(
opctx.log,
"Internet gateway '{name}' find sled id for ip {}: {e}",
ext_ip.ip.ip(),
"Internet gateway find sled id for ip {}: {e}",
ext_ip.ip.ip();
"name" => name.to_string(),
"vpc" => authz_vpc.id().to_string(),
"parent" => parent_id.to_string(),
);
continue;
}
Expand All @@ -2826,8 +2893,11 @@ impl DataStore {
}
info!(
opctx.log,
"Internet gateway '{name}' adding route \
{dest} -> {v6}"
"Internet gateway adding route \
{dest} -> {v6}";
"name" => name.to_string(),
"vpc" => authz_vpc.id().to_string(),
"parent" => parent_id.to_string(),
);
found = true;
break;
Expand All @@ -2836,9 +2906,13 @@ impl DataStore {
if !found {
warn!(
opctx.log,
"Internet gateway '{name}' no suitable \
ipv6 range found for {} in pool {}",
ext_ip.ip.ip(), igw_pool.ip_pool_id,
"Internet gateway no suitable \
ipv4 range found for {} in pool",
ext_ip.ip.ip();
"name" => name.to_string(),
"vpc" => authz_vpc.id().to_string(),
"parent" => parent_id.to_string(),
"pool" => igw_pool.ip_pool_id.to_string(),
);
}
}
Expand Down

0 comments on commit 564ac71

Please sign in to comment.