Skip to content

Commit

Permalink
Move instance networking functions into their own module (#4123)
Browse files Browse the repository at this point in the history 
`nexus/src/app/instance.rs` and `sled.rs` implement several
networking-related functions that are less directly concerned with
instance or sled management than their sibling routines. Tidy things up
a bit by creating an `instance_network` module and moving instance V2P
and NAT management functions there. Also, move instance NAT entry
deletion logic into its own function that's called from the instance
delete saga instead of implementing it inline in that saga.

These changes aim to reduce clutter in `instance.rs` and to move NAT
entry deletion to a function that can be reused by subsequent changes to
the way Nexus handles instance stop.

Except for some minor edits to error handling in
`instance_delete_dpd_config` (needed because this function no longer
returns a `steno::ActionError`), this PR only rearranges existing code
and has no functional changes.
  • Loading branch information
@gjcolombo
gjcolombo authored Sep 21, 2023
1 parent 88785de commit 79fee6a
Show file tree
Hide file tree
Showing 5 changed files with 503 additions and 457 deletions.
144 changes: 0 additions & 144 deletions nexus/src/app/instance.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ use super::MAX_NICS_PER_INSTANCE;
use super::MAX_VCPU_PER_INSTANCE;
use super::MIN_MEMORY_BYTES_PER_INSTANCE;
use crate::app::sagas;
use crate::app::sagas::retry_until_known_result;
use crate::cidata::InstanceCiData;
use crate::external_api::params;
use cancel_safe_futures::prelude::*;
Expand Down Expand Up @@ -41,7 +40,6 @@ use omicron_common::api::external::NameOrId;
use omicron_common::api::external::UpdateResult;
use omicron_common::api::external::Vni;
use omicron_common::api::internal::nexus;
use omicron_common::api::internal::shared::SwitchLocation;
use propolis_client::support::tungstenite::protocol::frame::coding::CloseCode;
use propolis_client::support::tungstenite::protocol::CloseFrame;
use propolis_client::support::tungstenite::Message as WebSocketMessage;
Expand All @@ -54,9 +52,7 @@ use sled_agent_client::types::InstancePutStateBody;
use sled_agent_client::types::InstanceStateRequested;
use sled_agent_client::types::SourceNatConfig;
use sled_agent_client::Client as SledAgentClient;
use std::collections::HashSet;
use std::net::SocketAddr;
use std::str::FromStr;
use std::sync::Arc;
use tokio::io::{AsyncRead, AsyncWrite};
use uuid::Uuid;
Expand Down Expand Up @@ -1232,146 +1228,6 @@ impl super::Nexus {
Ok(())
}

// Switches with uplinks configured and boundary services enabled
pub(crate) async fn boundary_switches(
&self,
opctx: &OpContext,
) -> Result<HashSet<SwitchLocation>, Error> {
let mut boundary_switches: HashSet<SwitchLocation> = HashSet::new();
let uplinks = self.list_switch_ports_with_uplinks(opctx).await?;
for uplink in &uplinks {
let location: SwitchLocation =
uplink.switch_location.parse().map_err(|_| {
Error::internal_error(&format!(
"invalid switch location in uplink config: {}",
uplink.switch_location
))
})?;
boundary_switches.insert(location);
}
Ok(boundary_switches)
}

/// Ensures that the Dendrite configuration for the supplied instance is
/// up-to-date.
///
/// # Parameters
///
/// - `opctx`: An operation context that grants read and list-children
/// permissions on the identified instance.
/// - `instance_id`: The ID of the instance to act on.
/// - `sled_ip_address`: The internal IP address assigned to the sled's
/// sled agent.
/// - `ip_index_filter`: An optional filter on the index into the instance's
/// external IP array.
/// - If this is `Some(n)`, this routine configures DPD state for only the
/// Nth external IP in the collection returned from CRDB. The caller is
/// responsible for ensuring that the IP collection has stable indices
/// when making this call.
/// - If this is `None`, this routine configures DPD for all external
/// IPs.
pub(crate) async fn instance_ensure_dpd_config(
&self,
opctx: &OpContext,
instance_id: Uuid,
sled_ip_address: &std::net::SocketAddrV6,
ip_index_filter: Option<usize>,
dpd_client: &Arc<dpd_client::Client>,
) -> Result<(), Error> {
let log = &self.log;

info!(log, "looking up instance's primary network interface";
"instance_id" => %instance_id);

let (.., authz_instance) = LookupPath::new(opctx, &self.db_datastore)
.instance_id(instance_id)
.lookup_for(authz::Action::ListChildren)
.await?;

// All external IPs map to the primary network interface, so find that
// interface. If there is no such interface, there's no way to route
// traffic destined to those IPs, so there's nothing to configure and
// it's safe to return early.
let network_interface = match self
.db_datastore
.derive_guest_network_interface_info(&opctx, &authz_instance)
.await?
.into_iter()
.find(|interface| interface.primary)
{
Some(interface) => interface,
None => {
info!(log, "Instance has no primary network interface";
"instance_id" => %instance_id);
return Ok(());
}
};

let mac_address =
macaddr::MacAddr6::from_str(&network_interface.mac.to_string())
.map_err(|e| {
Error::internal_error(&format!(
"failed to convert mac address: {e}"
))
})?;

let vni: u32 = network_interface.vni.into();

info!(log, "looking up instance's external IPs";
"instance_id" => %instance_id);

let ips = self
.db_datastore
.instance_lookup_external_ips(&opctx, instance_id)
.await?;

if let Some(wanted_index) = ip_index_filter {
if let None = ips.get(wanted_index) {
return Err(Error::internal_error(&format!(
"failed to find external ip address at index: {}",
wanted_index
)));
}
}

for target_ip in ips
.iter()
.enumerate()
.filter(|(index, _)| {
if let Some(wanted_index) = ip_index_filter {
*index == wanted_index
} else {
true
}
})
.map(|(_, ip)| ip)
{
retry_until_known_result(log, || async {
dpd_client
.ensure_nat_entry(
&log,
target_ip.ip,
dpd_client::types::MacAddr {
a: mac_address.into_array(),
},
*target_ip.first_port,
*target_ip.last_port,
vni,
sled_ip_address.ip(),
)
.await
})
.await
.map_err(|e| {
Error::internal_error(&format!(
"failed to ensure dpd entry: {e}"
))
})?;
}

Ok(())
}

/// Returns the requested range of serial console output bytes,
/// provided they are still in the propolis-server's cache.
pub(crate) async fn instance_serial_console_data(
Expand Down
Loading

0 comments on commit 79fee6a

Please sign in to comment.