From d95d371e40512e07b674a27a7b143941e58069d5 Mon Sep 17 00:00:00 2001 From: bnaecker Date: Fri, 8 Apr 2022 13:00:55 -0700 Subject: [PATCH] Adds basic per-sled sequential IP address allocation (#891) * Adds basic per-sled sequential IP address allocation - Adds the `last_used_address` column to the `omicron.sled` table, which tracks the last IP address within the sled's prefix allocated to a service running on the sled - Adds method for selecting the next IP address from the `sled` table, with a few basic tests for it - Uses a static address when launching guest instances, providing it to the propolis server managing them. * Review feedback - Adds some comments and issue links - Make allocation of IP addresses a separate saga action, to ensure idempotency. Also adds a generic helper, since this will likely be a common saga node. --- common/src/sql/dbinit.sql | 6 +- nexus/src/db/datastore.rs | 103 ++++++++++++++++++++++++++++++++ nexus/src/db/model.rs | 38 +++++++++++- nexus/src/db/schema.rs | 3 +- nexus/src/sagas.rs | 104 ++++++++++++++++++++++++++++++--- nexus/test-utils/src/lib.rs | 12 ++-- sled-agent/src/illumos/zone.rs | 3 + sled-agent/src/instance.rs | 13 +++-- 8 files changed, 260 insertions(+), 22 deletions(-) diff --git a/common/src/sql/dbinit.sql b/common/src/sql/dbinit.sql index b67212eb2d..3a8f79c162 100644 --- a/common/src/sql/dbinit.sql +++ b/common/src/sql/dbinit.sql @@ -64,8 +64,12 @@ CREATE TABLE omicron.public.sled ( time_deleted TIMESTAMPTZ, rcgen INT NOT NULL, + /* The IP address and bound port of the sled agent server. */ ip INET NOT NULL, - port INT4 NOT NULL + port INT4 NOT NULL, + + /* The last address allocated to an Oxide service on this sled. */ + last_used_address INET NOT NULL ); /* diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index f1893f4c25..2472c55a02 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -56,6 +56,7 @@ use omicron_common::api::external::{ use omicron_common::api::internal::nexus::UpdateArtifact; use omicron_common::bail_unless; use std::convert::{TryFrom, TryInto}; +use std::net::Ipv6Addr; use std::sync::Arc; use uuid::Uuid; @@ -2833,6 +2834,42 @@ impl DataStore { Ok(()) } + + /// Return the next available IPv6 address for an Oxide service running on + /// the provided sled. + pub async fn next_ipv6_address( + &self, + opctx: &OpContext, + sled_id: Uuid, + ) -> Result { + use db::schema::sled::dsl; + let net = diesel::update( + dsl::sled.find(sled_id).filter(dsl::time_deleted.is_null()), + ) + .set(dsl::last_used_address.eq(dsl::last_used_address + 1)) + .returning(dsl::last_used_address) + .get_result_async(self.pool_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel_pool( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Sled, + LookupType::ById(sled_id), + ), + ) + })?; + + // TODO-correctness: We need to ensure that this address is actually + // within the sled's underlay prefix, once that's included in the + // database record. + match net { + ipnetwork::IpNetwork::V6(net) => Ok(net.ip()), + _ => Err(Error::InternalError { + internal_message: String::from("Sled IP address must be IPv6"), + }), + } + } } /// Constructs a DataStore for use in test suites that has preloaded the @@ -3335,4 +3372,70 @@ mod test { let _ = db.cleanup().await; } + + // Test sled-specific IPv6 address allocation + #[tokio::test] + async fn test_sled_ipv6_address_allocation() { + use crate::db::model::STATIC_IPV6_ADDRESS_OFFSET; + use std::net::Ipv6Addr; + + let logctx = dev::test_setup_log("test_sled_ipv6_address_allocation"); + let mut db = test_setup_database(&logctx.log).await; + let cfg = db::Config { url: db.pg_config().clone() }; + let pool = Arc::new(db::Pool::new(&cfg)); + let datastore = Arc::new(DataStore::new(Arc::clone(&pool))); + let opctx = + OpContext::for_tests(logctx.log.new(o!()), datastore.clone()); + + let addr1 = "[fd00:1de::1]:12345".parse().unwrap(); + let sled1_id = "0de4b299-e0b4-46f0-d528-85de81a7095f".parse().unwrap(); + let sled1 = db::model::Sled::new(sled1_id, addr1); + datastore.sled_upsert(sled1).await.unwrap(); + + let addr2 = "[fd00:1df::1]:12345".parse().unwrap(); + let sled2_id = "66285c18-0c79-43e0-e54f-95271f271314".parse().unwrap(); + let sled2 = db::model::Sled::new(sled2_id, addr2); + datastore.sled_upsert(sled2).await.unwrap(); + + let ip = datastore.next_ipv6_address(&opctx, sled1_id).await.unwrap(); + let expected_ip = Ipv6Addr::new( + 0xfd00, + 0x1de, + 0, + 0, + 0, + 0, + 0, + 2 + STATIC_IPV6_ADDRESS_OFFSET, + ); + assert_eq!(ip, expected_ip); + let ip = datastore.next_ipv6_address(&opctx, sled1_id).await.unwrap(); + let expected_ip = Ipv6Addr::new( + 0xfd00, + 0x1de, + 0, + 0, + 0, + 0, + 0, + 3 + STATIC_IPV6_ADDRESS_OFFSET, + ); + assert_eq!(ip, expected_ip); + + let ip = datastore.next_ipv6_address(&opctx, sled2_id).await.unwrap(); + let expected_ip = Ipv6Addr::new( + 0xfd00, + 0x1df, + 0, + 0, + 0, + 0, + 0, + 2 + STATIC_IPV6_ADDRESS_OFFSET, + ); + assert_eq!(ip, expected_ip); + + let _ = db.cleanup().await; + logctx.cleanup_successful(); + } } diff --git a/nexus/src/db/model.rs b/nexus/src/db/model.rs index 4d65040ad0..a897e01bbc 100644 --- a/nexus/src/db/model.rs +++ b/nexus/src/db/model.rs @@ -631,16 +631,50 @@ pub struct Sled { pub ip: ipnetwork::IpNetwork, // TODO: Make use of SqlU16 pub port: i32, + + /// The last IP address provided to an Oxide service on this sled + pub last_used_address: IpNetwork, } +// TODO-correctness: We need a small offset here, while services and +// their addresses are still hardcoded in the mock RSS config file at +// `./smf/sled-agent/config-rss.toml`. This avoids conflicts with those +// addresses, but should be removed when they are entirely under the +// control of Nexus or RSS. +// +// See https://github.com/oxidecomputer/omicron/issues/732 for tracking issue. +pub(crate) const STATIC_IPV6_ADDRESS_OFFSET: u16 = 20; impl Sled { + // TODO-cleanup: We should be using IPv6 only for Oxide services, including + // `std::net::Ipv6Addr` and `SocketAddrV6`. The v4/v6 enums should only be + // used for managing customer addressing information, or when needed to + // interact with the database. pub fn new(id: Uuid, addr: SocketAddr) -> Self { + let last_used_address = { + match addr.ip() { + IpAddr::V6(ip) => { + let mut segments = ip.segments(); + segments[7] += STATIC_IPV6_ADDRESS_OFFSET; + ipnetwork::IpNetwork::from(IpAddr::from(Ipv6Addr::from( + segments, + ))) + } + IpAddr::V4(ip) => { + // TODO-correctness: This match arm should disappear when we + // support only IPv6 for underlay addressing. + let x = u32::from_be_bytes(ip.octets()) + + u32::from(STATIC_IPV6_ADDRESS_OFFSET); + ipnetwork::IpNetwork::from(IpAddr::from(Ipv4Addr::from(x))) + } + } + }; Self { identity: SledIdentity::new(id), time_deleted: None, rcgen: Generation::new(), ip: addr.ip().into(), port: addr.port().into(), + last_used_address, } } @@ -1143,10 +1177,10 @@ pub struct InstanceRuntimeState { pub sled_uuid: Uuid, #[column_name = "active_propolis_id"] pub propolis_uuid: Uuid, - #[column_name = "target_propolis_id"] - pub dst_propolis_uuid: Option, #[column_name = "active_propolis_ip"] pub propolis_ip: Option, + #[column_name = "target_propolis_id"] + pub dst_propolis_uuid: Option, #[column_name = "migration_id"] pub migration_uuid: Option, #[column_name = "ncpus"] diff --git a/nexus/src/db/schema.rs b/nexus/src/db/schema.rs index a102de7c29..08e7dac0f8 100644 --- a/nexus/src/db/schema.rs +++ b/nexus/src/db/schema.rs @@ -71,8 +71,8 @@ table! { state_generation -> Int8, active_server_id -> Uuid, active_propolis_id -> Uuid, - target_propolis_id -> Nullable, active_propolis_ip -> Nullable, + target_propolis_id -> Nullable, migration_id -> Nullable, ncpus -> Int8, memory -> Int8, @@ -222,6 +222,7 @@ table! { ip -> Inet, port -> Int4, + last_used_address -> Inet, } } diff --git a/nexus/src/sagas.rs b/nexus/src/sagas.rs index 12dc1a9d97..11a49cb4c6 100644 --- a/nexus/src/sagas.rs +++ b/nexus/src/sagas.rs @@ -44,6 +44,7 @@ use slog::warn; use slog::Logger; use std::collections::BTreeMap; use std::convert::{TryFrom, TryInto}; +use std::net::Ipv6Addr; use std::sync::Arc; use steno::new_action_noop_undo; use steno::ActionContext; @@ -110,6 +111,26 @@ async fn saga_generate_uuid( Ok(Uuid::new_v4()) } +/// A trait for sagas with serialized authentication information. +/// +/// This allows sharing code in different sagas which rely on some +/// authentication information, for example when doing database lookups. +trait AuthenticatedSagaParams { + fn serialized_authn(&self) -> &authn::saga::Serialized; +} + +/// A helper macro which implements the `AuthenticatedSagaParams` trait for saga +/// parameter types which have a field called `serialized_authn`. +macro_rules! impl_authenticated_saga_params { + ($typ:ty) => { + impl AuthenticatedSagaParams for <$typ as SagaType>::SagaParamsType { + fn serialized_authn(&self) -> &authn::saga::Serialized { + &self.serialized_authn + } + } + }; +} + // "Create Instance" saga template #[derive(Debug, Deserialize, Serialize)] @@ -127,6 +148,7 @@ impl SagaType for SagaInstanceCreate { type SagaParamsType = Arc; type ExecContextType = Arc; } +impl_authenticated_saga_params!(SagaInstanceCreate); pub fn saga_instance_create() -> SagaTemplate { let mut template_builder = SagaTemplateBuilder::new(); @@ -152,6 +174,12 @@ pub fn saga_instance_create() -> SagaTemplate { new_action_noop_undo(sic_alloc_server), ); + template_builder.append( + "propolis_ip", + "AllocatePropolisIp", + new_action_noop_undo(sic_allocate_propolis_ip), + ); + template_builder.append( "instance_name", "CreateInstanceRecord", @@ -651,21 +679,56 @@ async fn ensure_instance_disk_attach_state( Ok(()) } +/// Helper function to allocate a new IPv6 address for an Oxide service running +/// on the provided sled. +/// +/// `sled_id_name` is the name of the serialized output containing the UUID for +/// the targeted sled. +async fn allocate_sled_ipv6( + sagactx: ActionContext, + sled_id_name: &str, +) -> Result +where + T: SagaType>, + T::SagaParamsType: AuthenticatedSagaParams, +{ + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params(); + let opctx = OpContext::for_saga_action(&sagactx, params.serialized_authn()); + let sled_uuid = sagactx.lookup::(sled_id_name)?; + osagactx + .datastore() + .next_ipv6_address(&opctx, sled_uuid) + .await + .map_err(ActionError::action_failed) +} + +// Allocate an IP address on the destination sled for the Propolis server +async fn sic_allocate_propolis_ip( + sagactx: ActionContext, +) -> Result { + allocate_sled_ipv6(sagactx, "server_id").await +} + async fn sic_create_instance_record( sagactx: ActionContext, ) -> Result { let osagactx = sagactx.user_data(); let params = sagactx.saga_params(); - let sled_uuid = sagactx.lookup::("server_id"); - let instance_id = sagactx.lookup::("instance_id"); - let propolis_uuid = sagactx.lookup::("propolis_id"); + let sled_uuid = sagactx.lookup::("server_id")?; + let instance_id = sagactx.lookup::("instance_id")?; + let propolis_uuid = sagactx.lookup::("propolis_id")?; + let propolis_addr = sagactx.lookup::("propolis_ip")?; let runtime = InstanceRuntimeState { run_state: InstanceState::Creating, - sled_uuid: sled_uuid?, - propolis_uuid: propolis_uuid?, + sled_uuid, + propolis_uuid, dst_propolis_uuid: None, - propolis_addr: None, + propolis_addr: Some(std::net::SocketAddr::new( + propolis_addr.into(), + 12400, + )), migration_uuid: None, hostname: params.create_params.hostname.clone(), memory: params.create_params.memory, @@ -675,7 +738,7 @@ async fn sic_create_instance_record( }; let new_instance = db::model::Instance::new( - instance_id?, + instance_id, params.project_id, ¶ms.create_params, runtime.into(), @@ -792,6 +855,7 @@ impl SagaType for SagaInstanceMigrate { type SagaParamsType = Arc; type ExecContextType = Arc; } +impl_authenticated_saga_params!(SagaInstanceMigrate); pub fn saga_instance_migrate() -> SagaTemplate { let mut template_builder = SagaTemplateBuilder::new(); @@ -808,6 +872,12 @@ pub fn saga_instance_migrate() -> SagaTemplate { new_action_noop_undo(saga_generate_uuid), ); + template_builder.append( + "dst_propolis_ip", + "AllocatePropolisIp", + new_action_noop_undo(sim_allocate_propolis_ip), + ); + template_builder.append( "migrate_instance", "MigratePrep", @@ -861,11 +931,19 @@ async fn sim_migrate_prep( Ok((instance_id, instance.runtime_state.into())) } +// Allocate an IP address on the destination sled for the Propolis server. +async fn sim_allocate_propolis_ip( + sagactx: ActionContext, +) -> Result { + allocate_sled_ipv6(sagactx, "dst_sled_uuid").await +} + async fn sim_instance_migrate( sagactx: ActionContext, ) -> Result<(), ActionError> { let osagactx = sagactx.user_data(); let params = sagactx.saga_params(); + let opctx = OpContext::for_saga_action(&sagactx, ¶ms.serialized_authn); let migration_id = sagactx.lookup::("migrate_id")?; let dst_sled_uuid = params.migrate_params.dst_sled_uuid; @@ -873,10 +951,20 @@ async fn sim_instance_migrate( let (instance_id, old_runtime) = sagactx.lookup::<(Uuid, InstanceRuntimeState)>("migrate_instance")?; + // Allocate an IP address the destination sled for the new Propolis server. + let propolis_addr = osagactx + .datastore() + .next_ipv6_address(&opctx, dst_sled_uuid) + .await + .map_err(ActionError::action_failed)?; + let runtime = InstanceRuntimeState { sled_uuid: dst_sled_uuid, propolis_uuid: dst_propolis_uuid, - propolis_addr: None, + propolis_addr: Some(std::net::SocketAddr::new( + propolis_addr.into(), + 12400, + )), ..old_runtime }; let instance_hardware = InstanceHardware { diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index 983425c5d7..a53ad85d58 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -17,7 +17,7 @@ use oximeter_collector::Oximeter; use oximeter_producer::Server as ProducerServer; use slog::o; use slog::Logger; -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::net::{IpAddr, Ipv6Addr, SocketAddr}; use std::path::Path; use std::time::Duration; use uuid::Uuid; @@ -176,7 +176,7 @@ pub async fn start_sled_agent( sim_mode: sim::SimMode::Explicit, nexus_address, dropshot: ConfigDropshot { - bind_address: SocketAddr::new("127.0.0.1".parse().unwrap(), 0), + bind_address: SocketAddr::new(Ipv6Addr::LOCALHOST.into(), 0), request_body_max_bytes: 1024 * 1024, ..Default::default() }, @@ -184,7 +184,7 @@ pub async fn start_sled_agent( log: ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Debug }, storage: sim::ConfigStorage { zpools: vec![], - ip: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), + ip: IpAddr::from(Ipv6Addr::LOCALHOST), }, }; @@ -197,7 +197,7 @@ pub async fn start_oximeter( id: Uuid, ) -> Result { let db = oximeter_collector::DbConfig { - address: SocketAddr::new("::1".parse().unwrap(), db_port), + address: SocketAddr::new(Ipv6Addr::LOCALHOST.into(), db_port), batch_size: 10, batch_interval: 1, }; @@ -206,7 +206,7 @@ pub async fn start_oximeter( nexus_address, db, dropshot: ConfigDropshot { - bind_address: SocketAddr::new("::1".parse().unwrap(), 0), + bind_address: SocketAddr::new(Ipv6Addr::LOCALHOST.into(), 0), ..Default::default() }, log: ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Error }, @@ -254,7 +254,7 @@ pub async fn start_producer_server( // // This listens on any available port, and the server internally updates this to the actual // bound port of the Dropshot HTTP server. - let producer_address = SocketAddr::new("::1".parse().unwrap(), 0); + let producer_address = SocketAddr::new(Ipv6Addr::LOCALHOST.into(), 0); let server_info = ProducerEndpoint { id, address: producer_address, diff --git a/sled-agent/src/illumos/zone.rs b/sled-agent/src/illumos/zone.rs index 298457e071..6ce91132b6 100644 --- a/sled-agent/src/illumos/zone.rs +++ b/sled-agent/src/illumos/zone.rs @@ -75,6 +75,9 @@ pub enum Error { /// Describes the type of addresses which may be requested from a zone. #[derive(Copy, Clone, Debug)] +// TODO-cleanup: Remove, along with moving to IPv6 addressing everywhere. +// See https://github.com/oxidecomputer/omicron/issues/889. +#[allow(dead_code)] pub enum AddressRequest { Dhcp, Static(IpNetwork), diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index 6fa788bbcf..4b681fb7f5 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -25,6 +25,7 @@ use omicron_common::backoff; use propolis_client::api::DiskRequest; use propolis_client::Client as PropolisClient; use slog::Logger; +use std::net::IpAddr; use std::net::SocketAddr; use std::sync::Arc; use tokio::task::JoinHandle; @@ -181,6 +182,9 @@ struct InstanceInner { // The ID of the Propolis server (and zone) running this instance propolis_id: Uuid, + // The IP address of the Propolis server running this instance + propolis_ip: IpAddr, + // NIC-related properties vnic_allocator: VnicAllocator, requested_nics: Vec, @@ -422,6 +426,7 @@ impl Instance { vcpus: initial.runtime.ncpus.0 as u8, }, propolis_id: initial.runtime.propolis_uuid, + propolis_ip: initial.runtime.propolis_addr.unwrap().ip(), vnic_allocator, requested_nics: initial.nics, requested_disks: initial.disks, @@ -480,12 +485,12 @@ impl Instance { .await?; let running_zone = RunningZone::boot(installed_zone).await?; - let network = running_zone.ensure_address(AddressRequest::Dhcp).await?; + let addr_request = AddressRequest::new_static(inner.propolis_ip, None); + let network = running_zone.ensure_address(addr_request).await?; info!(inner.log, "Created address {} for zone: {}", network, zname); // Run Propolis in the Zone. - let server_addr = SocketAddr::new(network.ip(), PROPOLIS_PORT); - + let server_addr = SocketAddr::new(inner.propolis_ip, PROPOLIS_PORT); running_zone.run_cmd(&[ crate::illumos::zone::SVCCFG, "import", @@ -678,7 +683,7 @@ mod test { sled_uuid: Uuid::new_v4(), propolis_uuid: test_propolis_uuid(), dst_propolis_uuid: None, - propolis_addr: None, + propolis_addr: Some("[fd00:1de::74]:12400".parse().unwrap()), migration_uuid: None, ncpus: InstanceCpuCount(2), memory: ByteCount::from_mebibytes_u32(512),