From 0c7be4c9459b59a8d00b078985f2bc1dd41c2074 Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Wed, 10 Jan 2024 16:42:54 -0500 Subject: [PATCH] Addresses for propolis instances at SLED_PREFIX + 0xFFFF (#4777) Rather than allocating instance IPs starting at `SLED_PREFIX` + `RSS_RESERVED_ADDRESSES` + 1 where the `1` is the sled-agent allocated address of the GZ, we begin allocation from a larger block: `SLED_PREFIX` + `CP_SERVICES_RESERVED_ADDRESSES`. This gives us more room for nexus to allocate control plane services. Implements #4765 --- common/src/address.rs | 3 + nexus/db-model/src/schema.rs | 2 +- nexus/db-model/src/sled.rs | 6 +- nexus/db-queries/src/db/datastore/mod.rs | 38 ++----------- nexus/src/app/sagas/instance_common.rs | 6 +- nexus/src/app/sagas/instance_migrate.rs | 4 +- nexus/src/app/sagas/instance_start.rs | 4 +- nexus/tests/integration_tests/schema.rs | 72 ++++++++++++++++++++++++ schema/crdb/24.0.0/up.sql | 3 + schema/crdb/dbinit.sql | 4 +- 10 files changed, 97 insertions(+), 45 deletions(-) create mode 100644 schema/crdb/24.0.0/up.sql diff --git a/common/src/address.rs b/common/src/address.rs index 94361a2705..78eaee0bb4 100644 --- a/common/src/address.rs +++ b/common/src/address.rs @@ -165,6 +165,9 @@ const GZ_ADDRESS_INDEX: usize = 2; /// The maximum number of addresses per sled reserved for RSS. pub const RSS_RESERVED_ADDRESSES: u16 = 32; +// The maximum number of addresses per sled reserved for control plane services. +pub const CP_SERVICES_RESERVED_ADDRESSES: u16 = 0xFFFF; + /// Wraps an [`Ipv6Network`] with a compile-time prefix length. #[derive(Debug, Clone, Copy, JsonSchema, Serialize, Hash, PartialEq, Eq)] #[schemars(rename = "Ipv6Subnet")] diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 02bdd2c349..ed819cba80 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -13,7 +13,7 @@ use omicron_common::api::external::SemverVersion; /// /// This should be updated whenever the schema is changed. For more details, /// refer to: schema/crdb/README.adoc -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(23, 0, 1); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(24, 0, 0); table! { disk (id) { diff --git a/nexus/db-model/src/sled.rs b/nexus/db-model/src/sled.rs index 85a6b3139c..52968c27d5 100644 --- a/nexus/db-model/src/sled.rs +++ b/nexus/db-model/src/sled.rs @@ -57,7 +57,7 @@ pub struct Sled { pub ip: ipv6::Ipv6Addr, pub port: SqlU16, - /// The last IP address provided to an Oxide service on this sled + /// The last IP address provided to a propolis instance on this sled pub last_used_address: ipv6::Ipv6Addr, provision_state: SledProvisionState, @@ -183,7 +183,9 @@ impl SledUpdate { pub fn into_insertable(self) -> Sled { let last_used_address = { let mut segments = self.ip().segments(); - segments[7] += omicron_common::address::RSS_RESERVED_ADDRESSES; + // We allocate the entire last segment to control plane services + segments[7] = + omicron_common::address::CP_SERVICES_RESERVED_ADDRESSES; ipv6::Ipv6Addr::from(Ipv6Addr::from(segments)) }; Sled { diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index 93486771b5..d61ff15a3d 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -275,8 +275,8 @@ impl DataStore { self.pool_connection_unauthorized().await } - /// Return the next available IPv6 address for an Oxide service running on - /// the provided sled. + /// Return the next available IPv6 address for a propolis instance running + /// on the provided sled. pub async fn next_ipv6_address( &self, opctx: &OpContext, @@ -1286,7 +1286,6 @@ mod test { // Test sled-specific IPv6 address allocation #[tokio::test] async fn test_sled_ipv6_address_allocation() { - use omicron_common::address::RSS_RESERVED_ADDRESSES as STATIC_IPV6_ADDRESS_OFFSET; use std::net::Ipv6Addr; let logctx = dev::test_setup_log("test_sled_ipv6_address_allocation"); @@ -1322,41 +1321,14 @@ mod test { datastore.sled_upsert(sled2).await.unwrap(); let ip = datastore.next_ipv6_address(&opctx, sled1_id).await.unwrap(); - let expected_ip = Ipv6Addr::new( - 0xfd00, - 0x1de, - 0, - 0, - 0, - 0, - 0, - 2 + STATIC_IPV6_ADDRESS_OFFSET, - ); + let expected_ip = Ipv6Addr::new(0xfd00, 0x1de, 0, 0, 0, 0, 1, 0); assert_eq!(ip, expected_ip); let ip = datastore.next_ipv6_address(&opctx, sled1_id).await.unwrap(); - let expected_ip = Ipv6Addr::new( - 0xfd00, - 0x1de, - 0, - 0, - 0, - 0, - 0, - 3 + STATIC_IPV6_ADDRESS_OFFSET, - ); + let expected_ip = Ipv6Addr::new(0xfd00, 0x1de, 0, 0, 0, 0, 1, 1); assert_eq!(ip, expected_ip); let ip = datastore.next_ipv6_address(&opctx, sled2_id).await.unwrap(); - let expected_ip = Ipv6Addr::new( - 0xfd00, - 0x1df, - 0, - 0, - 0, - 0, - 0, - 2 + STATIC_IPV6_ADDRESS_OFFSET, - ); + let expected_ip = Ipv6Addr::new(0xfd00, 0x1df, 0, 0, 0, 0, 1, 0); assert_eq!(ip, expected_ip); let _ = db.cleanup().await; diff --git a/nexus/src/app/sagas/instance_common.rs b/nexus/src/app/sagas/instance_common.rs index 438b92cb84..8f9197b03b 100644 --- a/nexus/src/app/sagas/instance_common.rs +++ b/nexus/src/app/sagas/instance_common.rs @@ -121,9 +121,9 @@ pub async fn destroy_vmm_record( Ok(()) } -/// Allocates a new IPv6 address for a service that will run on the supplied -/// sled. -pub(super) async fn allocate_sled_ipv6( +/// Allocates a new IPv6 address for a propolis instance that will run on the +/// supplied sled. +pub(super) async fn allocate_vmm_ipv6( opctx: &OpContext, datastore: &DataStore, sled_uuid: Uuid, diff --git a/nexus/src/app/sagas/instance_migrate.rs b/nexus/src/app/sagas/instance_migrate.rs index 29c189efb4..1716953f04 100644 --- a/nexus/src/app/sagas/instance_migrate.rs +++ b/nexus/src/app/sagas/instance_migrate.rs @@ -7,7 +7,7 @@ use crate::app::instance::{ InstanceStateChangeError, InstanceStateChangeRequest, }; use crate::app::sagas::{ - declare_saga_actions, instance_common::allocate_sled_ipv6, + declare_saga_actions, instance_common::allocate_vmm_ipv6, }; use crate::external_api::params; use nexus_db_queries::db::{identity::Resource, lookup::LookupPath}; @@ -181,7 +181,7 @@ async fn sim_allocate_propolis_ip( &sagactx, ¶ms.serialized_authn, ); - allocate_sled_ipv6( + allocate_vmm_ipv6( &opctx, sagactx.user_data().datastore(), params.migrate_params.dst_sled_id, diff --git a/nexus/src/app/sagas/instance_start.rs b/nexus/src/app/sagas/instance_start.rs index 8957a838e7..9d12bd8031 100644 --- a/nexus/src/app/sagas/instance_start.rs +++ b/nexus/src/app/sagas/instance_start.rs @@ -7,7 +7,7 @@ use std::net::Ipv6Addr; use super::{ - instance_common::allocate_sled_ipv6, NexusActionContext, NexusSaga, + instance_common::allocate_vmm_ipv6, NexusActionContext, NexusSaga, SagaInitError, ACTION_GENERATE_ID, }; use crate::app::instance::InstanceStateChangeError; @@ -159,7 +159,7 @@ async fn sis_alloc_propolis_ip( ¶ms.serialized_authn, ); let sled_uuid = sagactx.lookup::("sled_id")?; - allocate_sled_ipv6(&opctx, sagactx.user_data().datastore(), sled_uuid).await + allocate_vmm_ipv6(&opctx, sagactx.user_data().datastore(), sled_uuid).await } async fn sis_create_vmm_record( diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index 21ed99e010..f183b53282 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -20,6 +20,7 @@ use pretty_assertions::{assert_eq, assert_ne}; use similar_asserts; use slog::Logger; use std::collections::{BTreeMap, BTreeSet}; +use std::net::IpAddr; use std::path::PathBuf; use tokio::time::timeout; use tokio::time::Duration; @@ -192,6 +193,7 @@ enum AnySqlType { String(String), TextArray(Vec), Uuid(Uuid), + Inet(IpAddr), // TODO: This isn't exhaustive, feel free to add more. // // These should only be necessary for rows where the database schema changes also choose to @@ -234,6 +236,12 @@ impl From for AnySqlType { } } +impl From for AnySqlType { + fn from(value: IpAddr) -> Self { + Self::Inet(value) + } +} + impl AnySqlType { fn as_str(&self) -> &str { match self { @@ -279,6 +287,9 @@ impl<'a> tokio_postgres::types::FromSql<'a> for AnySqlType { ty, raw, )?)); } + if IpAddr::accepts(ty) { + return Ok(AnySqlType::Inet(IpAddr::from_sql(ty, raw)?)); + } use tokio_postgres::types::Kind; match ty.kind() { @@ -941,6 +952,13 @@ const POOL1: Uuid = Uuid::from_u128(0x11116001_5c3d_4647_83b0_8f3515da7be1); const POOL2: Uuid = Uuid::from_u128(0x22226001_5c3d_4647_83b0_8f3515da7be1); const POOL3: Uuid = Uuid::from_u128(0x33336001_5c3d_4647_83b0_8f3515da7be1); +// "513D" -> "Sled" +const SLED1: Uuid = Uuid::from_u128(0x1111513d_5c3d_4647_83b0_8f3515da7be1); +const SLED2: Uuid = Uuid::from_u128(0x2222513d_5c3d_4647_83b0_8f3515da7be1); + +// "7AC4" -> "Rack" +const RACK1: Uuid = Uuid::from_u128(0x11117ac4_5c3d_4647_83b0_8f3515da7be1); + fn before_23_0_0(client: &Client) -> BoxFuture<'_, ()> { Box::pin(async move { // Create two silos @@ -1024,6 +1042,56 @@ fn after_23_0_0(client: &Client) -> BoxFuture<'_, ()> { }) } +fn before_24_0_0(client: &Client) -> BoxFuture<'_, ()> { + // IP addresses were pulled off dogfood sled 16 + Box::pin(async move { + // Create two sleds + client + .batch_execute(&format!( + "INSERT INTO sled + (id, time_created, time_modified, time_deleted, rcgen, rack_id, + is_scrimlet, serial_number, part_number, revision, + usable_hardware_threads, usable_physical_ram, reservoir_size, ip, + port, last_used_address, provision_state) VALUES + + ('{SLED1}', now(), now(), NULL, 1, '{RACK1}', true, 'abcd', 'defg', + '1', 64, 12345678, 77, 'fd00:1122:3344:104::1', 12345, + 'fd00:1122:3344:104::1ac', 'provisionable'), + ('{SLED2}', now(), now(), NULL, 1, '{RACK1}', false, 'zzzz', 'xxxx', + '2', 64, 12345678, 77,'fd00:1122:3344:107::1', 12345, + 'fd00:1122:3344:107::d4', 'provisionable'); + " + )) + .await + .expect("Failed to create sleds"); + }) +} + +fn after_24_0_0(client: &Client) -> BoxFuture<'_, ()> { + Box::pin(async { + // Confirm that the IP Addresses have the last 2 bytes changed to `0xFFFF` + let rows = client + .query("SELECT last_used_address FROM sled ORDER BY id", &[]) + .await + .expect("Failed to sled last_used_address"); + let last_used_addresses = process_rows(&rows); + + let expected_addr_1: IpAddr = + "fd00:1122:3344:104::ffff".parse().unwrap(); + let expected_addr_2: IpAddr = + "fd00:1122:3344:107::ffff".parse().unwrap(); + + assert_eq!( + last_used_addresses[0].values, + vec![ColumnValue::new("last_used_address", expected_addr_1)] + ); + assert_eq!( + last_used_addresses[1].values, + vec![ColumnValue::new("last_used_address", expected_addr_2)] + ); + }) +} + // Lazily initializes all migration checks. The combination of Rust function // pointers and async makes defining a static table fairly painful, so we're // using lazy initialization instead. @@ -1037,6 +1105,10 @@ fn get_migration_checks() -> BTreeMap { SemverVersion(semver::Version::parse("23.0.0").unwrap()), DataMigrationFns { before: Some(before_23_0_0), after: after_23_0_0 }, ); + map.insert( + SemverVersion(semver::Version::parse("24.0.0").unwrap()), + DataMigrationFns { before: Some(before_24_0_0), after: after_24_0_0 }, + ); map } diff --git a/schema/crdb/24.0.0/up.sql b/schema/crdb/24.0.0/up.sql new file mode 100644 index 0000000000..91bd10ab9f --- /dev/null +++ b/schema/crdb/24.0.0/up.sql @@ -0,0 +1,3 @@ +UPDATE omicron.public.sled + SET last_used_address = (netmask(set_masklen(ip, 64)) & ip) + 0xFFFF + WHERE time_deleted is null; diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index e40c97972f..2105caabef 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -108,7 +108,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.sled ( ip INET NOT NULL, port INT4 CHECK (port BETWEEN 0 AND 65535) NOT NULL, - /* The last address allocated to an Oxide service on this sled. */ + /* The last address allocated to a propolis instance on this sled. */ last_used_address INET NOT NULL, /* The state of whether resources should be provisioned onto the sled */ @@ -3258,7 +3258,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - ( TRUE, NOW(), NOW(), '23.0.1', NULL) + ( TRUE, NOW(), NOW(), '24.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT;