diff --git a/Cargo.lock b/Cargo.lock index bc5f53f759..bdf1edf268 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5749,6 +5749,7 @@ dependencies = [ "debug-ignore", "expectorate", "gateway-client", + "illumos-utils", "indexmap 2.6.0", "internal-dns-resolver", "ipnet", diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs index dfa89f4cc6..304a93439a 100644 --- a/clients/sled-agent-client/src/lib.rs +++ b/clients/sled-agent-client/src/lib.rs @@ -43,6 +43,7 @@ progenitor::generate_api!( replace = { Baseboard = nexus_sled_agent_shared::inventory::Baseboard, ByteCount = omicron_common::api::external::ByteCount, + DatasetsConfig = omicron_common::disk::DatasetsConfig, DatasetKind = omicron_common::api::internal::shared::DatasetKind, DiskIdentity = omicron_common::disk::DiskIdentity, DiskVariant = omicron_common::disk::DiskVariant, diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 7776958254..a5d93ada2b 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -860,8 +860,8 @@ pub enum DatasetKind { InternalDns, // Zone filesystems - ZoneRoot, - Zone { + TransientZoneRoot, + TransientZone { name: String, }, @@ -929,7 +929,7 @@ impl DatasetKind { match self { Cockroach | Crucible | Clickhouse | ClickhouseKeeper | ClickhouseServer | ExternalDns | InternalDns => true, - ZoneRoot | Zone { .. } | Debug | Update => false, + TransientZoneRoot | TransientZone { .. } | Debug | Update => false, } } @@ -937,7 +937,7 @@ impl DatasetKind { /// /// Otherwise, returns "None". pub fn zone_name(&self) -> Option<&str> { - if let DatasetKind::Zone { name } = self { + if let DatasetKind::TransientZone { name } = self { Some(name) } else { None @@ -961,8 +961,8 @@ impl fmt::Display for DatasetKind { ClickhouseServer => "clickhouse_server", ExternalDns => "external_dns", InternalDns => "internal_dns", - ZoneRoot => "zone", - Zone { name } => { + TransientZoneRoot => "zone", + TransientZone { name } => { write!(f, "zone/{}", name)?; return Ok(()); } @@ -992,12 +992,12 @@ impl FromStr for DatasetKind { "clickhouse_server" => ClickhouseServer, "external_dns" => ExternalDns, "internal_dns" => InternalDns, - "zone" => ZoneRoot, + "zone" => TransientZoneRoot, "debug" => Debug, "update" => Update, other => { if let Some(name) = other.strip_prefix("zone/") { - Zone { name: name.to_string() } + TransientZone { name: name.to_string() } } else { return Err(DatasetKindParseError::UnknownDataset( s.to_string(), @@ -1087,8 +1087,8 @@ mod tests { DatasetKind::ClickhouseServer, DatasetKind::ExternalDns, DatasetKind::InternalDns, - DatasetKind::ZoneRoot, - DatasetKind::Zone { name: String::from("myzone") }, + DatasetKind::TransientZoneRoot, + DatasetKind::TransientZone { name: String::from("myzone") }, DatasetKind::Debug, DatasetKind::Update, ]; diff --git a/common/src/disk.rs b/common/src/disk.rs index ac9232e257..79e5a46f23 100644 --- a/common/src/disk.rs +++ b/common/src/disk.rs @@ -12,6 +12,7 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::fmt; +use std::str::FromStr; use uuid::Uuid; use crate::{ @@ -186,6 +187,18 @@ impl GzipLevel { } } +impl FromStr for GzipLevel { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let level = s.parse::()?; + if level < GZIP_LEVEL_MIN || level > GZIP_LEVEL_MAX { + bail!("Invalid gzip compression level: {level}"); + } + Ok(Self(level)) + } +} + #[derive( Copy, Clone, @@ -224,6 +237,7 @@ pub enum CompressionAlgorithm { Zle, } +/// These match the arguments which can be passed to "zfs set compression=..." impl fmt::Display for CompressionAlgorithm { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { use CompressionAlgorithm::*; @@ -242,6 +256,29 @@ impl fmt::Display for CompressionAlgorithm { } } +impl FromStr for CompressionAlgorithm { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + use CompressionAlgorithm::*; + let c = match s { + "on" => On, + "" | "off" => Off, + "gzip" => Gzip, + "lz4" => Lz4, + "lzjb" => Lzjb, + "zle" => Zle, + _ => { + let Some(suffix) = s.strip_prefix("gzip-") else { + bail!("Unknown compression algorithm {s}"); + }; + GzipN { level: suffix.parse()? } + } + }; + Ok(c) + } +} + /// Configuration information necessary to request a single dataset #[derive( Clone, diff --git a/dev-tools/reconfigurator-cli/src/main.rs b/dev-tools/reconfigurator-cli/src/main.rs index c973fa5af2..c23ad8e4be 100644 --- a/dev-tools/reconfigurator-cli/src/main.rs +++ b/dev-tools/reconfigurator-cli/src/main.rs @@ -686,9 +686,12 @@ fn cmd_sled_show( swriteln!(s, "sled {}", sled_id); swriteln!(s, "subnet {}", sled_resources.subnet.net()); swriteln!(s, "zpools ({}):", sled_resources.zpools.len()); - for (zpool, disk) in &sled_resources.zpools { + for (zpool, (disk, datasets)) in &sled_resources.zpools { swriteln!(s, " {:?}", zpool); - swriteln!(s, " ↳ {:?}", disk); + swriteln!(s, " {:?}", disk); + for dataset in datasets { + swriteln!(s, " ↳ {:?}", dataset); + } } Ok(Some(s)) } @@ -840,7 +843,12 @@ fn cmd_blueprint_edit( .context("failed to add Nexus zone")?; assert_matches::assert_matches!( added, - EnsureMultiple::Changed { added: 1, removed: 0 } + EnsureMultiple::Changed { + added: 1, + updated: 0, + expunged: 0, + removed: 0 + } ); format!("added Nexus zone to sled {}", sled_id) } @@ -852,7 +860,12 @@ fn cmd_blueprint_edit( .context("failed to add CockroachDB zone")?; assert_matches::assert_matches!( added, - EnsureMultiple::Changed { added: 1, removed: 0 } + EnsureMultiple::Changed { + added: 1, + updated: 0, + expunged: 0, + removed: 0 + } ); format!("added CockroachDB zone to sled {}", sled_id) } diff --git a/dev-tools/reconfigurator-cli/tests/output/cmd-example-stdout b/dev-tools/reconfigurator-cli/tests/output/cmd-example-stdout index 838695cd80..93e705c6af 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmd-example-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmd-example-stdout @@ -40,25 +40,25 @@ sled 2eb69596-f081-4e2d-9425-9994926e0832 subnet fd00:1122:3344:102::/64 zpools (10): 088ed702-551e-453b-80d7-57700372a844 (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-088ed702-551e-453b-80d7-57700372a844" }, disk_id: b2850ccb-4ac7-4034-aeab-b1cd582d407b (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-088ed702-551e-453b-80d7-57700372a844" }, disk_id: b2850ccb-4ac7-4034-aeab-b1cd582d407b (physical_disk), policy: InService, state: Active } 09e51697-abad-47c0-a193-eaf74bc5d3cd (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-09e51697-abad-47c0-a193-eaf74bc5d3cd" }, disk_id: c6d1fe0d-5226-4318-a55a-e86e20612277 (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-09e51697-abad-47c0-a193-eaf74bc5d3cd" }, disk_id: c6d1fe0d-5226-4318-a55a-e86e20612277 (physical_disk), policy: InService, state: Active } 3a512d49-edbe-47f3-8d0b-6051bfdc4044 (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-3a512d49-edbe-47f3-8d0b-6051bfdc4044" }, disk_id: 24510d37-20b1-4bdc-9ca7-c37fff39abb2 (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-3a512d49-edbe-47f3-8d0b-6051bfdc4044" }, disk_id: 24510d37-20b1-4bdc-9ca7-c37fff39abb2 (physical_disk), policy: InService, state: Active } 40517680-aa77-413c-bcf4-b9041dcf6612 (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-40517680-aa77-413c-bcf4-b9041dcf6612" }, disk_id: 30ed317f-1717-4df6-8c1c-69f9d438705e (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-40517680-aa77-413c-bcf4-b9041dcf6612" }, disk_id: 30ed317f-1717-4df6-8c1c-69f9d438705e (physical_disk), policy: InService, state: Active } 78d3cb96-9295-4644-bf78-2e32191c71f9 (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-78d3cb96-9295-4644-bf78-2e32191c71f9" }, disk_id: 5ac39660-8149-48a2-a6df-aebb0f30352a (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-78d3cb96-9295-4644-bf78-2e32191c71f9" }, disk_id: 5ac39660-8149-48a2-a6df-aebb0f30352a (physical_disk), policy: InService, state: Active } 853595e7-77da-404e-bc35-aba77478d55c (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-853595e7-77da-404e-bc35-aba77478d55c" }, disk_id: 43083372-c7d0-4df3-ac4e-96c45cde28d9 (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-853595e7-77da-404e-bc35-aba77478d55c" }, disk_id: 43083372-c7d0-4df3-ac4e-96c45cde28d9 (physical_disk), policy: InService, state: Active } 8926e0e7-65d9-4e2e-ac6d-f1298af81ef1 (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-8926e0e7-65d9-4e2e-ac6d-f1298af81ef1" }, disk_id: 13e65865-2a6e-41f7-aa18-6ef8dff59b4e (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-8926e0e7-65d9-4e2e-ac6d-f1298af81ef1" }, disk_id: 13e65865-2a6e-41f7-aa18-6ef8dff59b4e (physical_disk), policy: InService, state: Active } 9c0b9151-17f3-4857-94cc-b5bfcd402326 (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-9c0b9151-17f3-4857-94cc-b5bfcd402326" }, disk_id: 40383e60-18f6-4423-94e7-7b91ce939b43 (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-9c0b9151-17f3-4857-94cc-b5bfcd402326" }, disk_id: 40383e60-18f6-4423-94e7-7b91ce939b43 (physical_disk), policy: InService, state: Active } d61354fa-48d2-47c6-90bf-546e3ed1708b (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-d61354fa-48d2-47c6-90bf-546e3ed1708b" }, disk_id: e02ae523-7b66-4188-93c8-c5808c01c795 (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-d61354fa-48d2-47c6-90bf-546e3ed1708b" }, disk_id: e02ae523-7b66-4188-93c8-c5808c01c795 (physical_disk), policy: InService, state: Active } d792c8cb-7490-40cb-bb1c-d4917242edf4 (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-d792c8cb-7490-40cb-bb1c-d4917242edf4" }, disk_id: c19e5610-a3a2-4cc6-af4d-517a49ef610b (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-d792c8cb-7490-40cb-bb1c-d4917242edf4" }, disk_id: c19e5610-a3a2-4cc6-af4d-517a49ef610b (physical_disk), policy: InService, state: Active } > blueprint-show ade5749d-bdf3-4fab-a8ae-00bea01b3a5a @@ -494,13 +494,13 @@ sled 89d02b1b-478c-401a-8e28-7a26f74fa41b subnet fd00:1122:3344:101::/64 zpools (4): 44fa7024-c2bc-4d2c-b478-c4997e4aece8 (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-44fa7024-c2bc-4d2c-b478-c4997e4aece8" }, disk_id: 2a15b33c-dd0e-45b7-aba9-d05f40f030ff (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-44fa7024-c2bc-4d2c-b478-c4997e4aece8" }, disk_id: 2a15b33c-dd0e-45b7-aba9-d05f40f030ff (physical_disk), policy: InService, state: Active } 8562317c-4736-4cfc-9292-7dcab96a6fee (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-8562317c-4736-4cfc-9292-7dcab96a6fee" }, disk_id: cad6faa6-9409-4496-9aeb-392b3c50bed4 (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-8562317c-4736-4cfc-9292-7dcab96a6fee" }, disk_id: cad6faa6-9409-4496-9aeb-392b3c50bed4 (physical_disk), policy: InService, state: Active } ce1c13f3-bef2-4306-b0f2-4e39bd4a18b6 (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-ce1c13f3-bef2-4306-b0f2-4e39bd4a18b6" }, disk_id: 7d89a66e-0dcd-47ab-824d-62186812b8bd (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-ce1c13f3-bef2-4306-b0f2-4e39bd4a18b6" }, disk_id: 7d89a66e-0dcd-47ab-824d-62186812b8bd (physical_disk), policy: InService, state: Active } f931ec80-a3e3-4adb-a8ba-fa5adbd2294c (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-f931ec80-a3e3-4adb-a8ba-fa5adbd2294c" }, disk_id: 41755be9-2c77-4deb-87a4-cb53f09263fa (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-f931ec80-a3e3-4adb-a8ba-fa5adbd2294c" }, disk_id: 41755be9-2c77-4deb-87a4-cb53f09263fa (physical_disk), policy: InService, state: Active } > blueprint-show ade5749d-bdf3-4fab-a8ae-00bea01b3a5a diff --git a/dev-tools/reconfigurator-cli/tests/output/cmd-stdout b/dev-tools/reconfigurator-cli/tests/output/cmd-stdout index dc1ba0c1ca..40489caeb5 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmd-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmd-stdout @@ -24,25 +24,25 @@ sled ..................... subnet fd00:1122:3344:101::/64 zpools (10): ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } > sled-add ..................... @@ -98,24 +98,24 @@ sled ..................... subnet fd00:1122:3344:101::/64 zpools (10): ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } diff --git a/illumos-utils/src/running_zone.rs b/illumos-utils/src/running_zone.rs index 5dbe4338cf..1dda130c95 100644 --- a/illumos-utils/src/running_zone.rs +++ b/illumos-utils/src/running_zone.rs @@ -12,12 +12,13 @@ use crate::dladm::Etherstub; use crate::link::{Link, VnicAllocator}; use crate::opte::{Port, PortTicket}; use crate::svc::wait_for_service; -use crate::zone::{AddressRequest, ZONE_PREFIX}; +use crate::zone::AddressRequest; use crate::zpool::{PathInPool, ZpoolName}; use camino::{Utf8Path, Utf8PathBuf}; use camino_tempfile::Utf8TempDir; use ipnetwork::IpNetwork; use omicron_common::backoff; +use omicron_uuid_kinds::OmicronZoneUuid; pub use oxlog::is_oxide_smf_log_file; use slog::{error, info, o, warn, Logger}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; @@ -26,7 +27,6 @@ use std::sync::Arc; use std::sync::OnceLock; #[cfg(target_os = "illumos")] use std::thread; -use uuid::Uuid; #[cfg(any(test, feature = "testing"))] use crate::zone::MockZones as Zones; @@ -947,12 +947,11 @@ impl InstalledZone { /// /// This results in a zone name which is distinct across different zpools, /// but stable and predictable across reboots. - pub fn get_zone_name(zone_type: &str, unique_name: Option) -> String { - let mut zone_name = format!("{}{}", ZONE_PREFIX, zone_type); - if let Some(suffix) = unique_name { - zone_name.push_str(&format!("_{}", suffix)); - } - zone_name + pub fn get_zone_name( + zone_type: &str, + unique_name: Option, + ) -> String { + crate::zone::zone_name(zone_type, unique_name) } /// Get the name of the bootstrap VNIC in the zone, if any. @@ -1055,7 +1054,7 @@ pub struct ZoneBuilder<'a> { // builder purposes - that is, skipping this field in the builder will // still result in an `Ok(InstalledZone)` from `.install()`, rather than // an `Err(InstallZoneError::IncompleteBuilder)`. - unique_name: Option, + unique_name: Option, /// ZFS datasets to be accessed from within the zone. datasets: Option<&'a [zone::Dataset]>, /// Filesystems to mount within the zone. @@ -1119,7 +1118,7 @@ impl<'a> ZoneBuilder<'a> { } /// Unique ID of the instance of the zone being created. (optional) - pub fn with_unique_name(mut self, uuid: Uuid) -> Self { + pub fn with_unique_name(mut self, uuid: OmicronZoneUuid) -> Self { self.unique_name = Some(uuid); self } diff --git a/illumos-utils/src/zone.rs b/illumos-utils/src/zone.rs index 47cc84dce6..da08c7b7df 100644 --- a/illumos-utils/src/zone.rs +++ b/illumos-utils/src/zone.rs @@ -17,6 +17,7 @@ use crate::dladm::{EtherstubVnic, VNIC_PREFIX_BOOTSTRAP, VNIC_PREFIX_CONTROL}; use crate::zpool::PathInPool; use crate::{execute, PFEXEC}; use omicron_common::address::SLED_PREFIX; +use omicron_uuid_kinds::OmicronZoneUuid; const DLADM: &str = "/usr/sbin/dladm"; pub const IPADM: &str = "/usr/sbin/ipadm"; @@ -29,6 +30,14 @@ pub const ROUTE: &str = "/usr/sbin/route"; pub const ZONE_PREFIX: &str = "oxz_"; pub const PROPOLIS_ZONE_PREFIX: &str = "oxz_propolis-server_"; +pub fn zone_name(prefix: &str, id: Option) -> String { + if let Some(id) = id { + format!("{ZONE_PREFIX}{}_{}", prefix, id) + } else { + format!("{ZONE_PREFIX}{}", prefix) + } +} + #[derive(thiserror::Error, Debug)] enum Error { #[error("Zone execution error: {0}")] diff --git a/live-tests/tests/test_nexus_add_remove.rs b/live-tests/tests/test_nexus_add_remove.rs index 6a0cdd93d0..81586496fd 100644 --- a/live-tests/tests/test_nexus_add_remove.rs +++ b/live-tests/tests/test_nexus_add_remove.rs @@ -70,7 +70,12 @@ async fn test_nexus_add_remove(lc: &LiveTestContext) { .context("adding Nexus zone")?; assert_matches!( count, - EnsureMultiple::Changed { added: 1, removed: 0 } + EnsureMultiple::Changed { + added: 1, + removed: 0, + updated: 0, + expunged: 0 + } ); Ok(()) }, diff --git a/nexus/db-model/src/dataset.rs b/nexus/db-model/src/dataset.rs index f896f11c5b..ad351fe612 100644 --- a/nexus/db-model/src/dataset.rs +++ b/nexus/db-model/src/dataset.rs @@ -2,13 +2,18 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use super::{DatasetKind, Generation, Region, SqlU16}; +use super::{ByteCount, DatasetKind, Generation, Region, SqlU16}; use crate::collection::DatastoreCollectionConfig; use crate::ipv6; use crate::schema::{dataset, region}; use chrono::{DateTime, Utc}; use db_macros::Asset; +use nexus_types::deployment::BlueprintDatasetConfig; +use omicron_common::api::external::Error; use omicron_common::api::internal::shared::DatasetKind as ApiDatasetKind; +use omicron_uuid_kinds::DatasetUuid; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::ZpoolUuid; use serde::{Deserialize, Serialize}; use std::net::{Ipv6Addr, SocketAddrV6}; use uuid::Uuid; @@ -43,6 +48,16 @@ pub struct Dataset { pub kind: DatasetKind, pub size_used: Option, zone_name: Option, + + quota: Option, + reservation: Option, + // This is the stringified form of + // "omicron_common::disk::CompressionAlgorithm". + // + // It can't serialize to the database without forcing omicron_common to + // depend on Diesel -- we could create a newtype, but "to_string" and + // "parse" cover this usage similarly. + compression: Option, } impl Dataset { @@ -55,7 +70,7 @@ impl Dataset { let kind = DatasetKind::from(&api_kind); let (size_used, zone_name) = match api_kind { ApiDatasetKind::Crucible => (Some(0), None), - ApiDatasetKind::Zone { name } => (None, Some(name)), + ApiDatasetKind::TransientZone { name } => (None, Some(name)), _ => (None, None), }; @@ -69,6 +84,9 @@ impl Dataset { kind, size_used, zone_name, + quota: None, + reservation: None, + compression: None, } } @@ -81,6 +99,69 @@ impl Dataset { } } +impl From for Dataset { + fn from(bp: BlueprintDatasetConfig) -> Self { + let kind = DatasetKind::from(&bp.kind); + let zone_name = bp.kind.zone_name().map(|s| s.to_string()); + // Only Crucible uses this "size_used" field. + let size_used = match bp.kind { + ApiDatasetKind::Crucible => Some(0), + ApiDatasetKind::Cockroach + | ApiDatasetKind::Clickhouse + | ApiDatasetKind::ClickhouseKeeper + | ApiDatasetKind::ClickhouseServer + | ApiDatasetKind::ExternalDns + | ApiDatasetKind::InternalDns + | ApiDatasetKind::TransientZone { .. } + | ApiDatasetKind::TransientZoneRoot + | ApiDatasetKind::Debug + | ApiDatasetKind::Update => None, + }; + let addr = bp.address; + Self { + identity: DatasetIdentity::new(bp.id.into_untyped_uuid()), + time_deleted: None, + rcgen: Generation::new(), + pool_id: bp.pool.id().into_untyped_uuid(), + kind, + ip: addr.map(|addr| addr.ip().into()), + port: addr.map(|addr| addr.port().into()), + size_used, + zone_name, + quota: bp.quota.map(ByteCount::from), + reservation: bp.reservation.map(ByteCount::from), + compression: Some(bp.compression.to_string()), + } + } +} + +impl TryFrom for omicron_common::disk::DatasetConfig { + type Error = Error; + + fn try_from(dataset: Dataset) -> Result { + let compression = if let Some(c) = dataset.compression { + c.parse().map_err(|e: anyhow::Error| { + Error::internal_error(&e.to_string()) + })? + } else { + omicron_common::disk::CompressionAlgorithm::Off + }; + + Ok(Self { + id: DatasetUuid::from_untyped_uuid(dataset.identity.id), + name: omicron_common::disk::DatasetName::new( + omicron_common::zpool_name::ZpoolName::new_external( + ZpoolUuid::from_untyped_uuid(dataset.pool_id), + ), + dataset.kind.try_into_api(dataset.zone_name)?, + ), + quota: dataset.quota.map(|q| q.into()), + reservation: dataset.reservation.map(|r| r.into()), + compression, + }) + } +} + // Datasets contain regions impl DatastoreCollectionConfig for Dataset { type CollectionId = Uuid; diff --git a/nexus/db-model/src/dataset_kind.rs b/nexus/db-model/src/dataset_kind.rs index 3f0b7c39bb..e90f0d1db3 100644 --- a/nexus/db-model/src/dataset_kind.rs +++ b/nexus/db-model/src/dataset_kind.rs @@ -3,6 +3,7 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use super::impl_enum_type; +use omicron_common::api::external::Error; use omicron_common::api::internal; use serde::{Deserialize, Serialize}; @@ -23,12 +24,44 @@ impl_enum_type!( ClickhouseServer => b"clickhouse_server" ExternalDns => b"external_dns" InternalDns => b"internal_dns" - ZoneRoot => b"zone_root" - Zone => b"zone" + TransientZoneRoot => b"zone_root" + TransientZone => b"zone" Debug => b"debug" Update => b"update" ); +impl DatasetKind { + pub fn try_into_api( + self, + zone_name: Option, + ) -> Result { + use internal::shared::DatasetKind as ApiKind; + let k = match (self, zone_name) { + (Self::Crucible, None) => ApiKind::Crucible, + (Self::Cockroach, None) => ApiKind::Cockroach, + (Self::Clickhouse, None) => ApiKind::Clickhouse, + (Self::ClickhouseKeeper, None) => ApiKind::ClickhouseKeeper, + (Self::ClickhouseServer, None) => ApiKind::ClickhouseServer, + (Self::ExternalDns, None) => ApiKind::ExternalDns, + (Self::InternalDns, None) => ApiKind::InternalDns, + (Self::TransientZoneRoot, None) => ApiKind::TransientZoneRoot, + (Self::TransientZone, Some(name)) => { + ApiKind::TransientZone { name } + } + (Self::Debug, None) => ApiKind::Debug, + (Self::Update, None) => ApiKind::Update, + (Self::TransientZone, None) => { + return Err(Error::internal_error("Zone kind needs name")) + } + (_, Some(_)) => { + return Err(Error::internal_error("Only zone kind needs name")) + } + }; + + Ok(k) + } +} + impl From<&internal::shared::DatasetKind> for DatasetKind { fn from(k: &internal::shared::DatasetKind) -> Self { match k { @@ -49,12 +82,16 @@ impl From<&internal::shared::DatasetKind> for DatasetKind { internal::shared::DatasetKind::InternalDns => { DatasetKind::InternalDns } - internal::shared::DatasetKind::ZoneRoot => DatasetKind::ZoneRoot, + internal::shared::DatasetKind::TransientZoneRoot => { + DatasetKind::TransientZoneRoot + } // Enums in the database do not have associated data, so this drops // the "name" of the zone and only considers the type. // // The zone name, if it exists, is stored in a separate column. - internal::shared::DatasetKind::Zone { .. } => DatasetKind::Zone, + internal::shared::DatasetKind::TransientZone { .. } => { + DatasetKind::TransientZone + } internal::shared::DatasetKind::Debug => DatasetKind::Debug, internal::shared::DatasetKind::Update => DatasetKind::Update, } diff --git a/nexus/db-model/src/deployment.rs b/nexus/db-model/src/deployment.rs index 693bec334d..fff829f548 100644 --- a/nexus/db-model/src/deployment.rs +++ b/nexus/db-model/src/deployment.rs @@ -10,29 +10,34 @@ use crate::omicron_zone_config::{self, OmicronZoneNic}; use crate::schema::{ blueprint, bp_clickhouse_cluster_config, bp_clickhouse_keeper_zone_id_to_node_id, - bp_clickhouse_server_zone_id_to_node_id, bp_omicron_physical_disk, - bp_omicron_zone, bp_omicron_zone_nic, bp_sled_omicron_physical_disks, + bp_clickhouse_server_zone_id_to_node_id, bp_omicron_dataset, + bp_omicron_physical_disk, bp_omicron_zone, bp_omicron_zone_nic, + bp_sled_omicron_datasets, bp_sled_omicron_physical_disks, bp_sled_omicron_zones, bp_sled_state, bp_target, }; use crate::typed_uuid::DbTypedUuid; use crate::{ - impl_enum_type, ipv6, Generation, MacAddr, Name, SledState, SqlU16, SqlU32, - SqlU8, + impl_enum_type, ipv6, ByteCount, Generation, MacAddr, Name, SledState, + SqlU16, SqlU32, SqlU8, }; use anyhow::{anyhow, bail, Context, Result}; use chrono::{DateTime, Utc}; use clickhouse_admin_types::{KeeperId, ServerId}; use ipnetwork::IpNetwork; use nexus_sled_agent_shared::inventory::OmicronZoneDataset; +use nexus_types::deployment::blueprint_zone_type; +use nexus_types::deployment::BlueprintDatasetConfig; +use nexus_types::deployment::BlueprintDatasetDisposition; +use nexus_types::deployment::BlueprintDatasetsConfig; +use nexus_types::deployment::BlueprintPhysicalDiskConfig; +use nexus_types::deployment::BlueprintPhysicalDisksConfig; use nexus_types::deployment::BlueprintTarget; use nexus_types::deployment::BlueprintZoneConfig; use nexus_types::deployment::BlueprintZoneDisposition; +use nexus_types::deployment::BlueprintZoneType; use nexus_types::deployment::BlueprintZonesConfig; +use nexus_types::deployment::ClickhouseClusterConfig; use nexus_types::deployment::CockroachDbPreserveDowngrade; -use nexus_types::deployment::{ - blueprint_zone_type, BlueprintPhysicalDisksConfig, ClickhouseClusterConfig, -}; -use nexus_types::deployment::{BlueprintPhysicalDiskConfig, BlueprintZoneType}; use nexus_types::deployment::{ OmicronZoneExternalFloatingAddr, OmicronZoneExternalFloatingIp, OmicronZoneExternalSnatIp, @@ -41,7 +46,7 @@ use omicron_common::api::internal::shared::NetworkInterface; use omicron_common::disk::DiskIdentity; use omicron_common::zpool_name::ZpoolName; use omicron_uuid_kinds::{ - ExternalIpKind, ExternalIpUuid, GenericUuid, OmicronZoneKind, + DatasetKind, ExternalIpKind, ExternalIpUuid, GenericUuid, OmicronZoneKind, OmicronZoneUuid, SledKind, SledUuid, ZpoolKind, ZpoolUuid, }; use std::net::{IpAddr, SocketAddrV6}; @@ -210,6 +215,154 @@ impl From for BlueprintPhysicalDiskConfig { } } +impl_enum_type!( + #[derive(Clone, SqlType, Debug, QueryId)] + #[diesel(postgres_type(name = "bp_dataset_disposition", schema = "public"))] + pub struct DbBpDatasetDispositionEnum; + + /// This type is not actually public, because [`BlueprintDatasetDisposition`] + /// interacts with external logic. + /// + /// However, it must be marked `pub` to avoid errors like `crate-private + /// type `BpDatasetDispositionEnum` in public interface`. Marking this type `pub`, + /// without actually making it public, tricks rustc in a desirable way. + #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq)] + #[diesel(sql_type = DbBpDatasetDispositionEnum)] + pub enum DbBpDatasetDisposition; + + // Enum values + InService => b"in_service" + Expunged => b"expunged" +); + +/// Converts a [`BlueprintDatasetDisposition`] to a version that can be inserted +/// into a database. +pub fn to_db_bp_dataset_disposition( + disposition: BlueprintDatasetDisposition, +) -> DbBpDatasetDisposition { + match disposition { + BlueprintDatasetDisposition::InService => { + DbBpDatasetDisposition::InService + } + BlueprintDatasetDisposition::Expunged => { + DbBpDatasetDisposition::Expunged + } + } +} + +impl From for BlueprintDatasetDisposition { + fn from(disposition: DbBpDatasetDisposition) -> Self { + match disposition { + DbBpDatasetDisposition::InService => { + BlueprintDatasetDisposition::InService + } + DbBpDatasetDisposition::Expunged => { + BlueprintDatasetDisposition::Expunged + } + } + } +} + +#[derive(Queryable, Clone, Debug, Selectable, Insertable)] +#[diesel(table_name = bp_sled_omicron_datasets)] +pub struct BpSledOmicronDatasets { + pub blueprint_id: Uuid, + pub sled_id: DbTypedUuid, + pub generation: Generation, +} + +impl BpSledOmicronDatasets { + pub fn new( + blueprint_id: Uuid, + sled_id: SledUuid, + datasets_config: &BlueprintDatasetsConfig, + ) -> Self { + Self { + blueprint_id, + sled_id: sled_id.into(), + generation: Generation(datasets_config.generation), + } + } +} + +/// DB representation of [BlueprintDatasetConfig] +#[derive(Queryable, Clone, Debug, Selectable, Insertable)] +#[diesel(table_name = bp_omicron_dataset)] +pub struct BpOmicronDataset { + pub blueprint_id: Uuid, + pub sled_id: DbTypedUuid, + pub id: DbTypedUuid, + + pub disposition: DbBpDatasetDisposition, + + pub pool_id: DbTypedUuid, + pub kind: crate::DatasetKind, + zone_name: Option, + pub ip: Option, + pub port: Option, + + pub quota: Option, + pub reservation: Option, + pub compression: String, +} + +impl BpOmicronDataset { + pub fn new( + blueprint_id: Uuid, + sled_id: SledUuid, + dataset_config: &BlueprintDatasetConfig, + ) -> Self { + Self { + blueprint_id, + sled_id: sled_id.into(), + id: dataset_config.id.into(), + disposition: to_db_bp_dataset_disposition( + dataset_config.disposition, + ), + pool_id: dataset_config.pool.id().into(), + kind: (&dataset_config.kind).into(), + zone_name: dataset_config.kind.zone_name().map(String::from), + ip: dataset_config.address.map(|addr| addr.ip().into()), + port: dataset_config.address.map(|addr| addr.port().into()), + quota: dataset_config.quota.map(|q| q.into()), + reservation: dataset_config.reservation.map(|r| r.into()), + compression: dataset_config.compression.to_string(), + } + } +} + +impl TryFrom for BlueprintDatasetConfig { + type Error = anyhow::Error; + + fn try_from(dataset: BpOmicronDataset) -> Result { + let address = match (dataset.ip, dataset.port) { + (Some(ip), Some(port)) => { + Some(SocketAddrV6::new(ip.into(), port.into(), 0, 0)) + } + (None, None) => None, + (_, _) => anyhow::bail!( + "Either both 'ip' and 'port' should be set, or neither" + ), + }; + + Ok(Self { + disposition: dataset.disposition.into(), + id: dataset.id.into(), + pool: omicron_common::zpool_name::ZpoolName::new_external( + dataset.pool_id.into(), + ), + kind: crate::DatasetKind::try_into_api( + dataset.kind, + dataset.zone_name, + )?, + address, + quota: dataset.quota.map(|b| b.into()), + reservation: dataset.reservation.map(|b| b.into()), + compression: dataset.compression.parse()?, + }) + } +} + /// See [`nexus_types::deployment::BlueprintZonesConfig`]. #[derive(Queryable, Clone, Debug, Selectable, Insertable)] #[diesel(table_name = bp_sled_omicron_zones)] diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 84e0c7e4f2..aea9c6a6bd 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -1038,6 +1038,10 @@ table! { kind -> crate::DatasetKindEnum, size_used -> Nullable, zone_name -> Nullable, + + quota -> Nullable, + reservation -> Nullable, + compression -> Nullable, } } @@ -1649,6 +1653,35 @@ table! { } } +table! { + bp_sled_omicron_datasets (blueprint_id, sled_id) { + blueprint_id -> Uuid, + sled_id -> Uuid, + + generation -> Int8, + } +} + +table! { + bp_omicron_dataset (blueprint_id, id) { + blueprint_id -> Uuid, + sled_id -> Uuid, + id -> Uuid, + + disposition -> crate::DbBpDatasetDispositionEnum, + + pool_id -> Uuid, + kind -> crate::DatasetKindEnum, + zone_name -> Nullable, + ip -> Nullable, + port -> Nullable, + + quota -> Nullable, + reservation -> Nullable, + compression -> Text, + } +} + table! { bp_sled_omicron_zones (blueprint_id, sled_id) { blueprint_id -> Uuid, diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index b09a8b7a6b..bc43152ace 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -17,7 +17,7 @@ use std::collections::BTreeMap; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(111, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(112, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -29,6 +29,7 @@ static KNOWN_VERSIONS: Lazy> = Lazy::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(112, "blueprint-dataset"), KnownVersion::new(111, "drop-omicron-zone-underlay-address"), KnownVersion::new(110, "clickhouse-policy"), KnownVersion::new(109, "inv-clickhouse-keeper-membership"), diff --git a/nexus/db-queries/src/db/datastore/dataset.rs b/nexus/db-queries/src/db/datastore/dataset.rs index 1843df0c7d..430e831a27 100644 --- a/nexus/db-queries/src/db/datastore/dataset.rs +++ b/nexus/db-queries/src/db/datastore/dataset.rs @@ -12,6 +12,7 @@ use crate::db; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; use crate::db::error::public_error_from_diesel; +use crate::db::error::retryable; use crate::db::error::ErrorHandler; use crate::db::identity::Asset; use crate::db::model::Dataset; @@ -20,18 +21,24 @@ use crate::db::model::PhysicalDiskPolicy; use crate::db::model::Zpool; use crate::db::pagination::paginated; use crate::db::pagination::Paginator; +use crate::db::TransactionError; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; use diesel::upsert::excluded; +use futures::FutureExt; use nexus_db_model::DatasetKind; use omicron_common::api::external::CreateResult; use omicron_common::api::external::DataPageParams; +use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_common::api::external::LookupType; use omicron_common::api::external::ResourceType; +use omicron_uuid_kinds::BlueprintUuid; +use omicron_uuid_kinds::DatasetUuid; +use omicron_uuid_kinds::GenericUuid; use uuid::Uuid; impl DataStore { @@ -53,6 +60,45 @@ impl DataStore { &self, dataset: Dataset, ) -> CreateResult { + let conn = &*self.pool_connection_unauthorized().await?; + Self::dataset_upsert_on_connection(&conn, dataset).await.map_err(|e| { + match e { + TransactionError::CustomError(e) => e, + TransactionError::Database(e) => { + public_error_from_diesel(e, ErrorHandler::Server) + } + } + }) + } + + pub async fn dataset_upsert_if_blueprint_is_current_target( + &self, + opctx: &OpContext, + bp_id: BlueprintUuid, + dataset: Dataset, + ) -> CreateResult { + let conn = self.pool_connection_unauthorized().await?; + + self.transaction_if_current_blueprint_is( + &conn, + "dataset_upsert_if_blueprint_is_current_target", + opctx, + bp_id, + |conn| { + let dataset = dataset.clone(); + async move { + Self::dataset_upsert_on_connection(&conn, dataset).await + } + .boxed() + }, + ) + .await + } + + async fn dataset_upsert_on_connection( + conn: &async_bb8_diesel::Connection, + dataset: Dataset, + ) -> Result> { use db::schema::dataset::dsl; let dataset_id = dataset.id(); @@ -69,24 +115,33 @@ impl DataStore { dsl::ip.eq(excluded(dsl::ip)), dsl::port.eq(excluded(dsl::port)), dsl::kind.eq(excluded(dsl::kind)), + dsl::zone_name.eq(excluded(dsl::zone_name)), + dsl::quota.eq(excluded(dsl::quota)), + dsl::reservation.eq(excluded(dsl::reservation)), + dsl::compression.eq(excluded(dsl::compression)), )), ) - .insert_and_get_result_async( - &*self.pool_connection_unauthorized().await?, - ) + .insert_and_get_result_async(conn) .await .map_err(|e| match e { - AsyncInsertError::CollectionNotFound => Error::ObjectNotFound { - type_name: ResourceType::Zpool, - lookup_type: LookupType::ById(zpool_id), - }, - AsyncInsertError::DatabaseError(e) => public_error_from_diesel( - e, - ErrorHandler::Conflict( - ResourceType::Dataset, - &dataset_id.to_string(), - ), - ), + AsyncInsertError::CollectionNotFound => { + TransactionError::CustomError(Error::ObjectNotFound { + type_name: ResourceType::Zpool, + lookup_type: LookupType::ById(zpool_id), + }) + } + AsyncInsertError::DatabaseError(e) => { + if retryable(&e) { + return TransactionError::Database(e); + } + TransactionError::CustomError(public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::Dataset, + &dataset_id.to_string(), + ), + )) + } }) } @@ -183,6 +238,63 @@ impl DataStore { Ok(all_datasets) } + pub async fn dataset_delete( + &self, + opctx: &OpContext, + id: DatasetUuid, + ) -> DeleteResult { + opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + let conn = self.pool_connection_authorized(&opctx).await?; + + Self::dataset_delete_on_connection(&conn, id) + .await + .map_err(|e| e.into()) + } + + pub async fn dataset_delete_if_blueprint_is_current_target( + &self, + opctx: &OpContext, + bp_id: BlueprintUuid, + id: DatasetUuid, + ) -> DeleteResult { + opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + let conn = self.pool_connection_authorized(&opctx).await?; + + self.transaction_if_current_blueprint_is( + &conn, + "dataset_delete_if_blueprint_is_current_target", + opctx, + bp_id, + |conn| { + async move { + Self::dataset_delete_on_connection(&conn, id).await + } + .boxed() + }, + ) + .await + } + + async fn dataset_delete_on_connection( + conn: &async_bb8_diesel::Connection, + id: DatasetUuid, + ) -> Result<(), TransactionError> { + use db::schema::dataset::dsl as dataset_dsl; + let now = Utc::now(); + + let id = *id.as_untyped_uuid(); + diesel::update(dataset_dsl::dataset) + .filter(dataset_dsl::time_deleted.is_null()) + .filter(dataset_dsl::id.eq(id)) + .set(dataset_dsl::time_deleted.eq(now)) + .execute_async(conn) + .await + .map(|_rows_modified| ()) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(()) + } + pub async fn dataset_physical_disk_in_service( &self, dataset_id: Uuid, @@ -240,25 +352,22 @@ mod test { use nexus_db_model::SledBaseboard; use nexus_db_model::SledSystemHardware; use nexus_db_model::SledUpdate; + use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder; + use nexus_types::deployment::Blueprint; + use nexus_types::deployment::BlueprintTarget; use omicron_common::api::internal::shared::DatasetKind as ApiDatasetKind; use omicron_test_utils::dev; + use omicron_uuid_kinds::SledUuid; + use omicron_uuid_kinds::ZpoolUuid; - #[tokio::test] - async fn test_insert_if_not_exists() { - let logctx = dev::test_setup_log("inventory_insert"); - let db = TestDatabase::new_with_datastore(&logctx.log).await; - let (opctx, datastore) = (db.opctx(), db.datastore()); - - // There should be no datasets initially. - assert_eq!( - datastore.dataset_list_all_batched(opctx, None).await.unwrap(), - [] - ); - + async fn create_sled_and_zpool( + datastore: &DataStore, + opctx: &OpContext, + ) -> (SledUuid, ZpoolUuid) { // Create a fake sled that holds our fake zpool. - let sled_id = Uuid::new_v4(); + let sled_id = SledUuid::new_v4(); let sled = SledUpdate::new( - sled_id, + *sled_id.as_untyped_uuid(), "[::1]:0".parse().unwrap(), SledBaseboard { serial_number: "test-sn".to_string(), @@ -277,18 +386,40 @@ mod test { datastore.sled_upsert(sled).await.expect("failed to upsert sled"); // Create a fake zpool that backs our fake datasets. - let zpool_id = Uuid::new_v4(); - let zpool = Zpool::new(zpool_id, sled_id, Uuid::new_v4()); + let zpool_id = ZpoolUuid::new_v4(); + let zpool = Zpool::new( + *zpool_id.as_untyped_uuid(), + *sled_id.as_untyped_uuid(), + Uuid::new_v4(), + ); datastore .zpool_insert(opctx, zpool) .await .expect("failed to upsert zpool"); + (sled_id, zpool_id) + } + + #[tokio::test] + async fn test_insert_if_not_exists() { + let logctx = dev::test_setup_log("insert_if_not_exists"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // There should be no datasets initially. + assert_eq!( + datastore.dataset_list_all_batched(opctx, None).await.unwrap(), + [] + ); + + let (_sled_id, zpool_id) = + create_sled_and_zpool(&datastore, opctx).await; + // Inserting a new dataset should succeed. let dataset1 = datastore .dataset_insert_if_not_exists(Dataset::new( Uuid::new_v4(), - zpool_id, + *zpool_id.as_untyped_uuid(), Some("[::1]:0".parse().unwrap()), ApiDatasetKind::Crucible, )) @@ -321,7 +452,7 @@ mod test { let insert_again_result = datastore .dataset_insert_if_not_exists(Dataset::new( dataset1.id(), - zpool_id, + *zpool_id.as_untyped_uuid(), Some("[::1]:12345".parse().unwrap()), ApiDatasetKind::Cockroach, )) @@ -337,7 +468,7 @@ mod test { let dataset2 = datastore .dataset_upsert(Dataset::new( Uuid::new_v4(), - zpool_id, + *zpool_id.as_untyped_uuid(), Some("[::1]:0".parse().unwrap()), ApiDatasetKind::Cockroach, )) @@ -369,7 +500,7 @@ mod test { let insert_again_result = datastore .dataset_insert_if_not_exists(Dataset::new( dataset1.id(), - zpool_id, + *zpool_id.as_untyped_uuid(), Some("[::1]:12345".parse().unwrap()), ApiDatasetKind::Cockroach, )) @@ -384,4 +515,114 @@ mod test { db.terminate().await; logctx.cleanup_successful(); } + + async fn bp_insert_and_make_target( + opctx: &OpContext, + datastore: &DataStore, + bp: &Blueprint, + ) { + datastore + .blueprint_insert(opctx, bp) + .await + .expect("inserted blueprint"); + datastore + .blueprint_target_set_current( + opctx, + BlueprintTarget { + target_id: bp.id, + enabled: true, + time_made_target: Utc::now(), + }, + ) + .await + .expect("made blueprint the target"); + } + + fn new_dataset_on(zpool_id: ZpoolUuid) -> Dataset { + Dataset::new( + Uuid::new_v4(), + *zpool_id.as_untyped_uuid(), + Some("[::1]:0".parse().unwrap()), + ApiDatasetKind::Cockroach, + ) + } + + #[tokio::test] + async fn test_upsert_and_delete_while_blueprint_changes() { + let logctx = + dev::test_setup_log("upsert_and_delete_while_blueprint_changes"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + let (sled_id, zpool_id) = + create_sled_and_zpool(&datastore, opctx).await; + + // The datastore methods don't actually read the blueprint, but they do + // guard against concurrent changes to the current target. + // + // We can test behavior by swapping between empty blueprints. + let bp0 = BlueprintBuilder::build_empty_with_sleds( + [sled_id].into_iter(), + "test", + ); + bp_insert_and_make_target(&opctx, &datastore, &bp0).await; + + let bp1 = { + let mut bp1 = bp0.clone(); + bp1.id = Uuid::new_v4(); + bp1.parent_blueprint_id = Some(bp0.id); + bp1 + }; + bp_insert_and_make_target(&opctx, &datastore, &bp1).await; + + let old_blueprint_id = BlueprintUuid::from_untyped_uuid(bp0.id); + let current_blueprint_id = BlueprintUuid::from_untyped_uuid(bp1.id); + + // Upsert referencing old blueprint: Error + datastore + .dataset_upsert_if_blueprint_is_current_target( + &opctx, + old_blueprint_id, + new_dataset_on(zpool_id), + ) + .await + .expect_err( + "Shouldn't be able to insert referencing old blueprint", + ); + + // Upsert referencing current blueprint: OK + let dataset = datastore + .dataset_upsert_if_blueprint_is_current_target( + &opctx, + current_blueprint_id, + new_dataset_on(zpool_id), + ) + .await + .expect("Should be able to insert while blueprint is active"); + + // Delete referencing old blueprint: Error + datastore + .dataset_delete_if_blueprint_is_current_target( + &opctx, + old_blueprint_id, + DatasetUuid::from_untyped_uuid(dataset.id()), + ) + .await + .expect_err( + "Shouldn't be able to delete referencing old blueprint", + ); + + // Delete referencing current blueprint: OK + datastore + .dataset_delete_if_blueprint_is_current_target( + &opctx, + current_blueprint_id, + DatasetUuid::from_untyped_uuid(dataset.id()), + ) + .await + .expect("Should be able to delete while blueprint is active"); + + db.terminate().await; + logctx.cleanup_successful(); + } } diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs index 34fcd45f96..19fff92aae 100644 --- a/nexus/db-queries/src/db/datastore/deployment.rs +++ b/nexus/db-queries/src/db/datastore/deployment.rs @@ -21,6 +21,8 @@ use async_bb8_diesel::AsyncRunQueryDsl; use chrono::DateTime; use chrono::Utc; use clickhouse_admin_types::{KeeperId, ServerId}; +use core::future::Future; +use core::pin::Pin; use diesel::expression::SelectableHelper; use diesel::pg::Pg; use diesel::query_builder::AstPass; @@ -36,18 +38,22 @@ use diesel::IntoSql; use diesel::OptionalExtension; use diesel::QueryDsl; use diesel::RunQueryDsl; +use futures::FutureExt; use nexus_db_model::Blueprint as DbBlueprint; use nexus_db_model::BpClickhouseClusterConfig; use nexus_db_model::BpClickhouseKeeperZoneIdToNodeId; use nexus_db_model::BpClickhouseServerZoneIdToNodeId; +use nexus_db_model::BpOmicronDataset; use nexus_db_model::BpOmicronPhysicalDisk; use nexus_db_model::BpOmicronZone; use nexus_db_model::BpOmicronZoneNic; +use nexus_db_model::BpSledOmicronDatasets; use nexus_db_model::BpSledOmicronPhysicalDisks; use nexus_db_model::BpSledOmicronZones; use nexus_db_model::BpSledState; use nexus_db_model::BpTarget; use nexus_types::deployment::Blueprint; +use nexus_types::deployment::BlueprintDatasetsConfig; use nexus_types::deployment::BlueprintMetadata; use nexus_types::deployment::BlueprintPhysicalDisksConfig; use nexus_types::deployment::BlueprintTarget; @@ -62,6 +68,7 @@ use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupType; use omicron_common::api::external::ResourceType; use omicron_common::bail_unless; +use omicron_uuid_kinds::BlueprintUuid; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::SledUuid; @@ -102,6 +109,76 @@ impl DataStore { Self::blueprint_insert_on_connection(&conn, opctx, blueprint).await } + /// Creates a transaction iff the current blueprint is "bp_id". + /// + /// - The transaction is retryable and named "name" + /// - The "bp_id" value is checked as the first operation within the + /// transaction. + /// - If "bp_id" is still the current target, then "f" is called, + /// within a transactional context. + pub async fn transaction_if_current_blueprint_is( + &self, + conn: &async_bb8_diesel::Connection, + name: &'static str, + opctx: &OpContext, + bp_id: BlueprintUuid, + f: Func, + ) -> Result + where + Func: for<'t> Fn( + &'t async_bb8_diesel::Connection, + ) -> Pin< + Box< + dyn Future>> + + Send + + 't, + >, + > + Send + + Sync + + Clone, + R: Send + 'static, + { + let err = OptionalError::new(); + let r = self + .transaction_retry_wrapper(name) + .transaction(&conn, |conn| { + let err = err.clone(); + let f = f.clone(); + async move { + // Bail if `bp_id` is no longer the target + let target = + Self::blueprint_target_get_current_on_connection( + &conn, opctx, + ) + .await + .map_err(|txn_error| txn_error.into_diesel(&err))?; + let bp_id_current = + BlueprintUuid::from_untyped_uuid(target.target_id); + if bp_id_current != bp_id { + return Err(err.bail( + Error::invalid_request(format!( + "blueprint target has changed from {} -> {}", + bp_id, bp_id_current + )) + .into(), + )); + } + + // Otherwise, perform our actual operation + f(&conn) + .await + .map_err(|txn_error| txn_error.into_diesel(&err)) + } + .boxed() + }) + .await + .map_err(|e| match err.take() { + Some(txn_error) => txn_error.into(), + None => public_error_from_diesel(e, ErrorHandler::Server), + })?; + Ok(r) + } + /// Variant of [Self::blueprint_insert] which may be called from a /// transaction context. pub(crate) async fn blueprint_insert_on_connection( @@ -156,6 +233,28 @@ impl DataStore { }) }) .collect::>(); + + let sled_omicron_datasets = blueprint + .blueprint_datasets + .iter() + .map(|(sled_id, datasets_config)| { + BpSledOmicronDatasets::new( + blueprint_id, + *sled_id, + datasets_config, + ) + }) + .collect::>(); + let omicron_datasets = blueprint + .blueprint_datasets + .iter() + .flat_map(|(sled_id, datasets_config)| { + datasets_config.datasets.values().map(move |dataset| { + BpOmicronDataset::new(blueprint_id, *sled_id, dataset) + }) + }) + .collect::>(); + let sled_omicron_zones = blueprint .blueprint_zones .iter() @@ -274,6 +373,24 @@ impl DataStore { .await?; } + // Insert all datasets for this blueprint. + + { + use db::schema::bp_sled_omicron_datasets::dsl as sled_datasets; + let _ = diesel::insert_into(sled_datasets::bp_sled_omicron_datasets) + .values(sled_omicron_datasets) + .execute_async(&conn) + .await?; + } + + { + use db::schema::bp_omicron_dataset::dsl as omicron_dataset; + let _ = diesel::insert_into(omicron_dataset::bp_omicron_dataset) + .values(omicron_datasets) + .execute_async(&conn) + .await?; + } + // Insert all the Omicron zones for this blueprint. { use db::schema::bp_sled_omicron_zones::dsl as sled_zones; @@ -522,6 +639,50 @@ impl DataStore { blueprint_physical_disks }; + // Do the same thing we just did for zones, but for datasets too. + let mut blueprint_datasets: BTreeMap< + SledUuid, + BlueprintDatasetsConfig, + > = { + use db::schema::bp_sled_omicron_datasets::dsl; + + let mut blueprint_datasets = BTreeMap::new(); + let mut paginator = Paginator::new(SQL_BATCH_SIZE); + while let Some(p) = paginator.next() { + let batch = paginated( + dsl::bp_sled_omicron_datasets, + dsl::sled_id, + &p.current_pagparams(), + ) + .filter(dsl::blueprint_id.eq(blueprint_id)) + .select(BpSledOmicronDatasets::as_select()) + .load_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })?; + + paginator = p.found_batch(&batch, &|s| s.sled_id); + + for s in batch { + let old = blueprint_datasets.insert( + s.sled_id.into(), + BlueprintDatasetsConfig { + generation: *s.generation, + datasets: BTreeMap::new(), + }, + ); + bail_unless!( + old.is_none(), + "found duplicate sled ID in bp_sled_omicron_datasets: {}", + s.sled_id + ); + } + } + + blueprint_datasets + }; + // Assemble a mutable map of all the NICs found, by NIC id. As we // match these up with the corresponding zone below, we'll remove items // from this set. That way we can tell if the same NIC was used twice @@ -685,6 +846,58 @@ impl DataStore { } } + // Load all the datasets for each sled + { + use db::schema::bp_omicron_dataset::dsl; + + let mut paginator = Paginator::new(SQL_BATCH_SIZE); + while let Some(p) = paginator.next() { + // `paginated` implicitly orders by our `id`, which is also + // handy for testing: the datasets are always consistently ordered + let batch = paginated( + dsl::bp_omicron_dataset, + dsl::id, + &p.current_pagparams(), + ) + .filter(dsl::blueprint_id.eq(blueprint_id)) + .select(BpOmicronDataset::as_select()) + .load_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })?; + + paginator = p.found_batch(&batch, &|d| d.id); + + for d in batch { + let sled_datasets = blueprint_datasets + .get_mut(&d.sled_id.into()) + .ok_or_else(|| { + // This error means that we found a row in + // bp_omicron_dataset with no associated record in + // bp_sled_omicron_datasets. This should be + // impossible and reflects either a bug or database + // corruption. + Error::internal_error(&format!( + "dataset {}: unknown sled: {}", + d.id, d.sled_id + )) + })?; + + let dataset_id = d.id; + sled_datasets.datasets.insert( + dataset_id.into(), + d.try_into().map_err(|e| { + Error::internal_error(&format!( + "Cannot parse dataset {}: {e}", + dataset_id + )) + })?, + ); + } + } + } + // Sort all disks to match what blueprint builders do. for (_, disks_config) in blueprint_disks.iter_mut() { disks_config.disks.sort_unstable_by_key(|d| d.id); @@ -844,6 +1057,7 @@ impl DataStore { id: blueprint_id, blueprint_zones, blueprint_disks, + blueprint_datasets, sled_state, parent_blueprint_id, internal_dns_version, @@ -879,6 +1093,8 @@ impl DataStore { nsled_states, nsled_physical_disks, nphysical_disks, + nsled_datasets, + ndatasets, nsled_agent_zones, nzones, nnics, @@ -890,7 +1106,7 @@ impl DataStore { // Ensure that blueprint we're about to delete is not the // current target. let current_target = - self.blueprint_current_target_only(&conn).await?; + Self::blueprint_current_target_only(&conn).await?; if current_target.target_id == blueprint_id { return Err(TransactionError::CustomError( Error::conflict(format!( @@ -950,6 +1166,26 @@ impl DataStore { .await? }; + // Remove rows associated with Omicron datasets + let nsled_datasets = { + use db::schema::bp_sled_omicron_datasets::dsl; + diesel::delete( + dsl::bp_sled_omicron_datasets + .filter(dsl::blueprint_id.eq(blueprint_id)), + ) + .execute_async(&conn) + .await? + }; + let ndatasets = { + use db::schema::bp_omicron_dataset::dsl; + diesel::delete( + dsl::bp_omicron_dataset + .filter(dsl::blueprint_id.eq(blueprint_id)), + ) + .execute_async(&conn) + .await? + }; + // Remove rows associated with Omicron zones let nsled_agent_zones = { use db::schema::bp_sled_omicron_zones::dsl; @@ -1014,6 +1250,8 @@ impl DataStore { nsled_states, nsled_physical_disks, nphysical_disks, + nsled_datasets, + ndatasets, nsled_agent_zones, nzones, nnics, @@ -1036,6 +1274,8 @@ impl DataStore { "nsled_states" => nsled_states, "nsled_physical_disks" => nsled_physical_disks, "nphysical_disks" => nphysical_disks, + "nsled_datasets" => nsled_datasets, + "ndatasets" => ndatasets, "nsled_agent_zones" => nsled_agent_zones, "nzones" => nzones, "nnics" => nnics, @@ -1102,15 +1342,17 @@ impl DataStore { async move { // Bail out if `blueprint` isn't the current target. - let current_target = self - .blueprint_current_target_only(&conn) + let current_target = Self::blueprint_current_target_only(&conn) .await .map_err(|e| err.bail(e))?; if current_target.target_id != blueprint.id { - return Err(err.bail(Error::invalid_request(format!( + return Err(err.bail( + Error::invalid_request(format!( "blueprint {} is not the current target blueprint ({})", blueprint.id, current_target.target_id - )))); + )) + .into(), + )); } // See the comment on this method; this lets us notify our test @@ -1151,7 +1393,7 @@ impl DataStore { .map(|(_sled_id, zone)| zone), ) .await - .map_err(|e| err.bail(e))?; + .map_err(|e| err.bail(e.into()))?; self.ensure_zone_external_networking_allocated_on_connection( &conn, opctx, @@ -1162,7 +1404,7 @@ impl DataStore { .map(|(_sled_id, zone)| zone), ) .await - .map_err(|e| err.bail(e))?; + .map_err(|e| err.bail(e.into()))?; Ok(()) } @@ -1170,7 +1412,7 @@ impl DataStore { .await .map_err(|e| { if let Some(err) = err.take() { - err + err.into() } else { public_error_from_diesel(e, ErrorHandler::Server) } @@ -1320,7 +1562,7 @@ impl DataStore { opctx.authorize(authz::Action::Read, &authz::BLUEPRINT_CONFIG).await?; let conn = self.pool_connection_authorized(opctx).await?; - let target = self.blueprint_current_target_only(&conn).await?; + let target = Self::blueprint_current_target_only(&conn).await?; // The blueprint for the current target cannot be deleted while it is // the current target, but it's possible someone else (a) made a new @@ -1334,6 +1576,15 @@ impl DataStore { Ok((target, blueprint)) } + /// Get the current target blueprint, if one exists + pub async fn blueprint_target_get_current_on_connection( + conn: &async_bb8_diesel::Connection, + opctx: &OpContext, + ) -> Result> { + opctx.authorize(authz::Action::Read, &authz::BLUEPRINT_CONFIG).await?; + Self::blueprint_current_target_only(&conn).await + } + /// Get the current target blueprint, if one exists pub async fn blueprint_target_get_current( &self, @@ -1341,7 +1592,7 @@ impl DataStore { ) -> Result { opctx.authorize(authz::Action::Read, &authz::BLUEPRINT_CONFIG).await?; let conn = self.pool_connection_authorized(opctx).await?; - self.blueprint_current_target_only(&conn).await + Self::blueprint_current_target_only(&conn).await.map_err(|e| e.into()) } // Helper to fetch the current blueprint target (without fetching the entire @@ -1349,9 +1600,8 @@ impl DataStore { // // Caller is responsible for checking authz for this operation. async fn blueprint_current_target_only( - &self, conn: &async_bb8_diesel::Connection, - ) -> Result { + ) -> Result> { use db::schema::bp_target::dsl; let current_target = dsl::bp_target @@ -1821,6 +2071,12 @@ mod tests { for (table_name, result) in [ query_count!(blueprint, id), + query_count!(bp_sled_state, blueprint_id), + query_count!(bp_sled_omicron_datasets, blueprint_id), + query_count!(bp_sled_omicron_physical_disks, blueprint_id), + query_count!(bp_sled_omicron_zones, blueprint_id), + query_count!(bp_omicron_dataset, blueprint_id), + query_count!(bp_omicron_physical_disk, blueprint_id), query_count!(bp_omicron_zone, blueprint_id), query_count!(bp_omicron_zone_nic, blueprint_id), ] { @@ -1840,16 +2096,20 @@ mod tests { .map(|i| { ( ZpoolUuid::new_v4(), - SledDisk { - disk_identity: DiskIdentity { - vendor: String::from("v"), - serial: format!("s-{i}"), - model: String::from("m"), + ( + SledDisk { + disk_identity: DiskIdentity { + vendor: String::from("v"), + serial: format!("s-{i}"), + model: String::from("m"), + }, + disk_id: PhysicalDiskUuid::new_v4(), + policy: PhysicalDiskPolicy::InService, + state: PhysicalDiskState::Active, }, - disk_id: PhysicalDiskUuid::new_v4(), - policy: PhysicalDiskPolicy::InService, - state: PhysicalDiskState::Active, - }, + // Datasets + vec![], + ), ) }) .collect(); @@ -2074,7 +2334,12 @@ mod tests { .resources, ) .unwrap(), - EnsureMultiple::Changed { added: 4, removed: 0 } + EnsureMultiple::Changed { + added: 4, + updated: 0, + expunged: 0, + removed: 0 + } ); // Add zones to our new sled. diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index fe7ee5a6d3..5386582df4 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -1059,6 +1059,7 @@ mod test { id: Uuid::new_v4(), blueprint_zones: BTreeMap::new(), blueprint_disks: BTreeMap::new(), + blueprint_datasets: BTreeMap::new(), sled_state: BTreeMap::new(), cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade::DoNotModify, @@ -1539,6 +1540,7 @@ mod test { sled_state: sled_states_active(blueprint_zones.keys().copied()), blueprint_zones, blueprint_disks: BTreeMap::new(), + blueprint_datasets: BTreeMap::new(), cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade::DoNotModify, parent_blueprint_id: None, @@ -1799,6 +1801,7 @@ mod test { sled_state: sled_states_active(blueprint_zones.keys().copied()), blueprint_zones, blueprint_disks: BTreeMap::new(), + blueprint_datasets: BTreeMap::new(), cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade::DoNotModify, parent_blueprint_id: None, @@ -2010,6 +2013,7 @@ mod test { sled_state: sled_states_active(blueprint_zones.keys().copied()), blueprint_zones, blueprint_disks: BTreeMap::new(), + blueprint_datasets: BTreeMap::new(), cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade::DoNotModify, parent_blueprint_id: None, @@ -2152,6 +2156,7 @@ mod test { sled_state: sled_states_active(blueprint_zones.keys().copied()), blueprint_zones, blueprint_disks: BTreeMap::new(), + blueprint_datasets: BTreeMap::new(), cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade::DoNotModify, parent_blueprint_id: None, diff --git a/nexus/db-queries/tests/output/region_allocate_distinct_sleds.sql b/nexus/db-queries/tests/output/region_allocate_distinct_sleds.sql index 4e7dde244b..9ee71b403f 100644 --- a/nexus/db-queries/tests/output/region_allocate_distinct_sleds.sql +++ b/nexus/db-queries/tests/output/region_allocate_distinct_sleds.sql @@ -271,7 +271,10 @@ WITH dataset.port, dataset.kind, dataset.size_used, - dataset.zone_name + dataset.zone_name, + dataset.quota, + dataset.reservation, + dataset.compression ) ( SELECT @@ -286,6 +289,9 @@ WITH dataset.kind, dataset.size_used, dataset.zone_name, + dataset.quota, + dataset.reservation, + dataset.compression, old_regions.id, old_regions.time_created, old_regions.time_modified, @@ -313,6 +319,9 @@ UNION updated_datasets.kind, updated_datasets.size_used, updated_datasets.zone_name, + updated_datasets.quota, + updated_datasets.reservation, + updated_datasets.compression, inserted_regions.id, inserted_regions.time_created, inserted_regions.time_modified, diff --git a/nexus/db-queries/tests/output/region_allocate_random_sleds.sql b/nexus/db-queries/tests/output/region_allocate_random_sleds.sql index b2c164a6d9..369410c68c 100644 --- a/nexus/db-queries/tests/output/region_allocate_random_sleds.sql +++ b/nexus/db-queries/tests/output/region_allocate_random_sleds.sql @@ -269,7 +269,10 @@ WITH dataset.port, dataset.kind, dataset.size_used, - dataset.zone_name + dataset.zone_name, + dataset.quota, + dataset.reservation, + dataset.compression ) ( SELECT @@ -284,6 +287,9 @@ WITH dataset.kind, dataset.size_used, dataset.zone_name, + dataset.quota, + dataset.reservation, + dataset.compression, old_regions.id, old_regions.time_created, old_regions.time_modified, @@ -311,6 +317,9 @@ UNION updated_datasets.kind, updated_datasets.size_used, updated_datasets.zone_name, + updated_datasets.quota, + updated_datasets.reservation, + updated_datasets.compression, inserted_regions.id, inserted_regions.time_created, inserted_regions.time_modified, diff --git a/nexus/db-queries/tests/output/region_allocate_with_snapshot_distinct_sleds.sql b/nexus/db-queries/tests/output/region_allocate_with_snapshot_distinct_sleds.sql index 97ee23f82e..9251139c4e 100644 --- a/nexus/db-queries/tests/output/region_allocate_with_snapshot_distinct_sleds.sql +++ b/nexus/db-queries/tests/output/region_allocate_with_snapshot_distinct_sleds.sql @@ -282,7 +282,10 @@ WITH dataset.port, dataset.kind, dataset.size_used, - dataset.zone_name + dataset.zone_name, + dataset.quota, + dataset.reservation, + dataset.compression ) ( SELECT @@ -297,6 +300,9 @@ WITH dataset.kind, dataset.size_used, dataset.zone_name, + dataset.quota, + dataset.reservation, + dataset.compression, old_regions.id, old_regions.time_created, old_regions.time_modified, @@ -324,6 +330,9 @@ UNION updated_datasets.kind, updated_datasets.size_used, updated_datasets.zone_name, + updated_datasets.quota, + updated_datasets.reservation, + updated_datasets.compression, inserted_regions.id, inserted_regions.time_created, inserted_regions.time_modified, diff --git a/nexus/db-queries/tests/output/region_allocate_with_snapshot_random_sleds.sql b/nexus/db-queries/tests/output/region_allocate_with_snapshot_random_sleds.sql index a1cc103594..c8aa8adf2e 100644 --- a/nexus/db-queries/tests/output/region_allocate_with_snapshot_random_sleds.sql +++ b/nexus/db-queries/tests/output/region_allocate_with_snapshot_random_sleds.sql @@ -280,7 +280,10 @@ WITH dataset.port, dataset.kind, dataset.size_used, - dataset.zone_name + dataset.zone_name, + dataset.quota, + dataset.reservation, + dataset.compression ) ( SELECT @@ -295,6 +298,9 @@ WITH dataset.kind, dataset.size_used, dataset.zone_name, + dataset.quota, + dataset.reservation, + dataset.compression, old_regions.id, old_regions.time_created, old_regions.time_modified, @@ -322,6 +328,9 @@ UNION updated_datasets.kind, updated_datasets.size_used, updated_datasets.zone_name, + updated_datasets.quota, + updated_datasets.reservation, + updated_datasets.compression, inserted_regions.id, inserted_regions.time_created, inserted_regions.time_modified, diff --git a/nexus/reconfigurator/execution/src/datasets.rs b/nexus/reconfigurator/execution/src/datasets.rs index cf53a24a8f..5d3bb30a37 100644 --- a/nexus/reconfigurator/execution/src/datasets.rs +++ b/nexus/reconfigurator/execution/src/datasets.rs @@ -4,36 +4,136 @@ //! Ensures dataset records required by a given blueprint +use crate::Sled; + +use anyhow::anyhow; use anyhow::Context; +use futures::stream; +use futures::StreamExt; use nexus_db_model::Dataset; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; -use nexus_types::deployment::BlueprintZoneConfig; -use nexus_types::deployment::DurableDataset; +use nexus_types::deployment::BlueprintDatasetConfig; +use nexus_types::deployment::BlueprintDatasetDisposition; +use nexus_types::deployment::BlueprintDatasetsConfig; use nexus_types::identity::Asset; +use omicron_common::disk::DatasetConfig; +use omicron_common::disk::DatasetsConfig; +use omicron_uuid_kinds::BlueprintUuid; +use omicron_uuid_kinds::DatasetUuid; use omicron_uuid_kinds::GenericUuid; -use omicron_uuid_kinds::OmicronZoneUuid; +use omicron_uuid_kinds::SledUuid; use slog::info; +use slog::o; use slog::warn; -use std::collections::BTreeSet; +use std::collections::BTreeMap; + +/// Idempotently ensures that the specified datasets are deployed to the +/// corresponding sleds +pub(crate) async fn deploy_datasets( + opctx: &OpContext, + sleds_by_id: &BTreeMap, + sled_configs: &BTreeMap, +) -> Result<(), Vec> { + let errors: Vec<_> = stream::iter(sled_configs) + .filter_map(|(sled_id, config)| async move { + let log = opctx.log.new(o!( + "sled_id" => sled_id.to_string(), + "generation" => config.generation.to_string(), + )); + + let db_sled = match sleds_by_id.get(&sled_id) { + Some(sled) => sled, + None => { + let err = anyhow!("sled not found in db list: {}", sled_id); + warn!(log, "{err:#}"); + return Some(err); + } + }; + + let client = nexus_networking::sled_client_from_address( + sled_id.into_untyped_uuid(), + db_sled.sled_agent_address(), + &log, + ); + + let config: DatasetsConfig = config.clone().into(); + let result = + client.datasets_put(&config).await.with_context( + || format!("Failed to put {config:#?} to sled {sled_id}"), + ); + match result { + Err(error) => { + warn!(log, "{error:#}"); + Some(error) + } + Ok(result) => { + let (errs, successes): (Vec<_>, Vec<_>) = result + .into_inner() + .status + .into_iter() + .partition(|status| status.err.is_some()); + + if !errs.is_empty() { + warn!( + log, + "Failed to deploy datasets for sled agent"; + "successfully configured datasets" => successes.len(), + "failed dataset configurations" => errs.len(), + ); + for err in &errs { + warn!(log, "{err:?}"); + } + return Some(anyhow!( + "failure deploying datasets: {:?}", + errs + )); + } + + info!( + log, + "Successfully deployed datasets for sled agent"; + "successfully configured datasets" => successes.len(), + ); + None + } + } + }) + .collect() + .await; + + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } +} + +#[allow(dead_code)] +pub(crate) struct EnsureDatasetsResult { + pub(crate) inserted: usize, + pub(crate) updated: usize, + pub(crate) removed: usize, +} -/// For each zone in `all_omicron_zones` that has an associated durable dataset, -/// ensure that a corresponding dataset record exists in `datastore`. +/// For all datasets we expect to see in the blueprint, ensure that a corresponding +/// database record exists in `datastore`. /// -/// Does not modify any existing dataset records. Returns the number of -/// datasets inserted. +/// Updates all existing dataset records that don't match the blueprint. +/// Returns the number of datasets changed. pub(crate) async fn ensure_dataset_records_exist( opctx: &OpContext, datastore: &DataStore, - all_omicron_zones: impl Iterator, -) -> anyhow::Result { + bp_id: BlueprintUuid, + bp_datasets: impl Iterator, +) -> anyhow::Result { // Before attempting to insert any datasets, first query for any existing // dataset records so we can filter them out. This looks like a typical // TOCTOU issue, but it is purely a performance optimization. We expect // almost all executions of this function to do nothing: new datasets are // created very rarely relative to how frequently blueprint realization // happens. We could remove this check and filter and instead run the below - // "insert if not exists" query on every zone, and the behavior would still + // "insert if not exists" query on every dataset, and the behavior would still // be correct. However, that would issue far more queries than necessary in // the very common case of "we don't need to do anything at all". let mut existing_datasets = datastore @@ -41,60 +141,90 @@ pub(crate) async fn ensure_dataset_records_exist( .await .context("failed to list all datasets")? .into_iter() - .map(|dataset| OmicronZoneUuid::from_untyped_uuid(dataset.id())) - .collect::>(); + .map(|dataset| (DatasetUuid::from_untyped_uuid(dataset.id()), dataset)) + .collect::>(); let mut num_inserted = 0; - let mut num_already_exist = 0; + let mut num_updated = 0; + let mut num_unchanged = 0; + let mut num_removed = 0; - for zone in all_omicron_zones { - let Some(DurableDataset { dataset, kind, address }) = - zone.zone_type.durable_dataset() - else { - continue; - }; + let (wanted_datasets, unwanted_datasets): (Vec<_>, Vec<_>) = bp_datasets + .partition(|d| match d.disposition { + BlueprintDatasetDisposition::InService => true, + BlueprintDatasetDisposition::Expunged => false, + }); - let id = zone.id; + for bp_dataset in wanted_datasets { + let id = bp_dataset.id; + let kind = &bp_dataset.kind; - // If already present in the datastore, move on. - if existing_datasets.remove(&id) { - num_already_exist += 1; - continue; - } + // If this dataset already exists, only update it if it appears different from what exists + // in the database already. + let action = if let Some(db_dataset) = existing_datasets.remove(&id) { + let db_config: DatasetConfig = db_dataset.try_into()?; + let bp_config: DatasetConfig = bp_dataset.clone().try_into()?; - let pool_id = dataset.pool_name.id(); - let dataset = Dataset::new( - id.into_untyped_uuid(), - pool_id.into_untyped_uuid(), - Some(address), - kind.clone(), - ); - let maybe_inserted = datastore - .dataset_insert_if_not_exists(dataset) + if db_config == bp_config { + num_unchanged += 1; + continue; + } + num_updated += 1; + "update" + } else { + num_inserted += 1; + "insert" + }; + + let dataset = Dataset::from(bp_dataset.clone()); + datastore + .dataset_upsert_if_blueprint_is_current_target( + &opctx, bp_id, dataset, + ) .await .with_context(|| { - format!("failed to insert dataset record for dataset {id}") + format!("failed to upsert dataset record for dataset {id}") })?; - // If we succeeded in inserting, log it; if `maybe_dataset` is `None`, - // we must have lost the TOCTOU race described above, and another Nexus - // must have inserted this dataset before we could. - if maybe_inserted.is_some() { - info!( - opctx.log, - "inserted new dataset for Omicron zone"; - "id" => %id, - "kind" => ?kind, - ); - num_inserted += 1; - } else { - num_already_exist += 1; + info!( + opctx.log, + "ensuring dataset record in database"; + "action" => action, + "id" => %id, + "kind" => ?kind, + ); + } + + for bp_dataset in unwanted_datasets { + if existing_datasets.remove(&bp_dataset.id).is_some() { + if matches!( + bp_dataset.kind, + omicron_common::disk::DatasetKind::Crucible + ) { + // Region and snapshot replacement cannot happen without the + // database record, even if the dataset has been expunged. + // + // This record will still be deleted, but it will happen as a + // part of the "decommissioned_disk_cleaner" background task. + continue; + } + + datastore + .dataset_delete_if_blueprint_is_current_target( + &opctx, + bp_id, + bp_dataset.id, + ) + .await?; + num_removed += 1; } } - // We don't currently support removing datasets, so this would be - // surprising: the database contains dataset records that are no longer in - // our blueprint. We can't do anything about this, so just warn. + // We support removing expunged datasets - if we read a dataset that hasn't + // been explicitly expunged, log this as an oddity. + // + // This could be possible in conditions where multiple Nexuses are executing + // distinct blueprints. if !existing_datasets.is_empty() { warn!( opctx.log, @@ -106,12 +236,18 @@ pub(crate) async fn ensure_dataset_records_exist( info!( opctx.log, - "ensured all Omicron zones have dataset records"; + "ensured all Omicron datasets have database records"; "num_inserted" => num_inserted, - "num_already_existed" => num_already_exist, + "num_updated" => num_updated, + "num_unchanged" => num_unchanged, + "num_removed" => num_removed, ); - Ok(num_inserted) + Ok(EnsureDatasetsResult { + inserted: num_inserted, + updated: num_updated, + removed: num_removed, + }) } #[cfg(test)] @@ -119,12 +255,12 @@ mod tests { use super::*; use nexus_db_model::Zpool; use nexus_reconfigurator_planning::example::ExampleSystemBuilder; - use nexus_sled_agent_shared::inventory::OmicronZoneDataset; use nexus_test_utils_macros::nexus_test; - use nexus_types::deployment::blueprint_zone_type; - use nexus_types::deployment::BlueprintZoneDisposition; + use nexus_types::deployment::Blueprint; use nexus_types::deployment::BlueprintZoneFilter; - use nexus_types::deployment::BlueprintZoneType; + use omicron_common::api::external::ByteCount; + use omicron_common::api::internal::shared::DatasetKind; + use omicron_common::disk::CompressionAlgorithm; use omicron_common::zpool_name::ZpoolName; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::ZpoolUuid; @@ -133,11 +269,30 @@ mod tests { type ControlPlaneTestContext = nexus_test_utils::ControlPlaneTestContext; + fn get_all_datasets_from_zones( + blueprint: &Blueprint, + ) -> Vec { + blueprint + .all_omicron_zones(BlueprintZoneFilter::All) + .filter_map(|(_, zone)| { + let dataset = zone.zone_type.durable_dataset()?; + Some(BlueprintDatasetConfig { + disposition: BlueprintDatasetDisposition::InService, + id: DatasetUuid::new_v4(), + pool: dataset.dataset.pool_name.clone(), + kind: dataset.kind, + address: Some(dataset.address), + quota: None, + reservation: None, + compression: CompressionAlgorithm::Off, + }) + }) + .collect::>() + } + #[nexus_test] - async fn test_ensure_dataset_records_exist( - cptestctx: &ControlPlaneTestContext, - ) { - const TEST_NAME: &str = "test_ensure_dataset_records_exist"; + async fn test_dataset_record_create(cptestctx: &ControlPlaneTestContext) { + const TEST_NAME: &str = "test_dataset_record_create"; // Set up. let nexus = &cptestctx.server.server_context().nexus; @@ -149,8 +304,12 @@ mod tests { let opctx = &opctx; // Use the standard example system. - let (example, blueprint) = + let (example, mut blueprint) = ExampleSystemBuilder::new(&opctx.log, TEST_NAME).nsleds(5).build(); + + // Set the target so our database-modifying operations know they + // can safely act on the current target blueprint. + update_blueprint_target(&datastore, &opctx, &mut blueprint).await; let collection = example.collection; // Record the sleds and zpools. @@ -170,29 +329,32 @@ mod tests { 0 ); - // Collect all the blueprint zones. - let all_omicron_zones = blueprint - .all_omicron_zones(BlueprintZoneFilter::All) - .map(|(_, zone)| zone) - .collect::>(); + // Let's allocate datasets for all the zones with durable datasets. + // + // Finding these datasets is normally the responsibility of the planner, + // but we're kinda hand-rolling it. + let all_datasets = get_all_datasets_from_zones(&blueprint); // How many zones are there with durable datasets? - let nzones_with_durable_datasets = all_omicron_zones - .iter() - .filter(|z| z.zone_type.durable_dataset().is_some()) - .count(); + let nzones_with_durable_datasets = all_datasets.len(); + assert!(nzones_with_durable_datasets > 0); - let ndatasets_inserted = ensure_dataset_records_exist( - opctx, - datastore, - all_omicron_zones.iter().copied(), - ) - .await - .expect("failed to ensure datasets"); + let bp_id = BlueprintUuid::from_untyped_uuid(blueprint.id); + let EnsureDatasetsResult { inserted, updated, removed } = + ensure_dataset_records_exist( + opctx, + datastore, + bp_id, + all_datasets.iter(), + ) + .await + .expect("failed to ensure datasets"); // We should have inserted a dataset for each zone with a durable // dataset. - assert_eq!(nzones_with_durable_datasets, ndatasets_inserted); + assert_eq!(inserted, nzones_with_durable_datasets); + assert_eq!(updated, 0); + assert_eq!(removed, 0); assert_eq!( datastore .dataset_list_all_batched(opctx, None) @@ -203,14 +365,18 @@ mod tests { ); // Ensuring the same datasets again should insert no new records. - let ndatasets_inserted = ensure_dataset_records_exist( - opctx, - datastore, - all_omicron_zones.iter().copied(), - ) - .await - .expect("failed to ensure datasets"); - assert_eq!(0, ndatasets_inserted); + let EnsureDatasetsResult { inserted, updated, removed } = + ensure_dataset_records_exist( + opctx, + datastore, + bp_id, + all_datasets.iter(), + ) + .await + .expect("failed to ensure datasets"); + assert_eq!(inserted, 0); + assert_eq!(updated, 0); + assert_eq!(removed, 0); assert_eq!( datastore .dataset_list_all_batched(opctx, None) @@ -235,44 +401,44 @@ mod tests { .expect("failed to upsert zpool"); } - // Call `ensure_dataset_records_exist` again, adding new crucible and - // cockroach zones. It should insert only these new zones. + // Call `ensure_dataset_records_exist` again, adding new datasets. + // + // It should only insert these new zones. let new_zones = [ - BlueprintZoneConfig { - disposition: BlueprintZoneDisposition::InService, - id: OmicronZoneUuid::new_v4(), - filesystem_pool: Some(ZpoolName::new_external(new_zpool_id)), - zone_type: BlueprintZoneType::Crucible( - blueprint_zone_type::Crucible { - address: "[::1]:0".parse().unwrap(), - dataset: OmicronZoneDataset { - pool_name: ZpoolName::new_external(new_zpool_id), - }, - }, - ), + BlueprintDatasetConfig { + disposition: BlueprintDatasetDisposition::InService, + id: DatasetUuid::new_v4(), + pool: ZpoolName::new_external(new_zpool_id), + kind: DatasetKind::Debug, + address: None, + quota: None, + reservation: None, + compression: CompressionAlgorithm::Off, }, - BlueprintZoneConfig { - disposition: BlueprintZoneDisposition::InService, - id: OmicronZoneUuid::new_v4(), - filesystem_pool: Some(ZpoolName::new_external(new_zpool_id)), - zone_type: BlueprintZoneType::CockroachDb( - blueprint_zone_type::CockroachDb { - address: "[::1]:0".parse().unwrap(), - dataset: OmicronZoneDataset { - pool_name: ZpoolName::new_external(new_zpool_id), - }, - }, - ), + BlueprintDatasetConfig { + disposition: BlueprintDatasetDisposition::InService, + id: DatasetUuid::new_v4(), + pool: ZpoolName::new_external(new_zpool_id), + kind: DatasetKind::TransientZoneRoot, + address: None, + quota: None, + reservation: None, + compression: CompressionAlgorithm::Off, }, ]; - let ndatasets_inserted = ensure_dataset_records_exist( - opctx, - datastore, - all_omicron_zones.iter().copied().chain(&new_zones), - ) - .await - .expect("failed to ensure datasets"); - assert_eq!(ndatasets_inserted, 2); + + let EnsureDatasetsResult { inserted, updated, removed } = + ensure_dataset_records_exist( + opctx, + datastore, + bp_id, + all_datasets.iter().chain(&new_zones), + ) + .await + .expect("failed to ensure datasets"); + assert_eq!(inserted, 2); + assert_eq!(updated, 0); + assert_eq!(removed, 0); assert_eq!( datastore .dataset_list_all_batched(opctx, None) @@ -282,4 +448,320 @@ mod tests { nzones_with_durable_datasets + 2, ); } + + // Sets the target blueprint to "blueprint" + // + // Reads the current target, and uses it as the "parent" blueprint + async fn update_blueprint_target( + datastore: &DataStore, + opctx: &OpContext, + blueprint: &mut Blueprint, + ) { + // Fetch the initial blueprint installed during rack initialization. + let parent_blueprint_target = datastore + .blueprint_target_get_current(&opctx) + .await + .expect("failed to read current target blueprint"); + blueprint.parent_blueprint_id = Some(parent_blueprint_target.target_id); + datastore.blueprint_insert(&opctx, &blueprint).await.unwrap(); + datastore + .blueprint_target_set_current( + &opctx, + nexus_types::deployment::BlueprintTarget { + target_id: blueprint.id, + enabled: true, + time_made_target: nexus_inventory::now_db_precision(), + }, + ) + .await + .unwrap(); + } + + #[nexus_test] + async fn test_dataset_records_update(cptestctx: &ControlPlaneTestContext) { + const TEST_NAME: &str = "test_dataset_records_update"; + + // Set up. + let nexus = &cptestctx.server.server_context().nexus; + let datastore = nexus.datastore(); + let opctx = OpContext::for_tests( + cptestctx.logctx.log.clone(), + datastore.clone(), + ); + let opctx = &opctx; + + // Use the standard example system. + let (_example, mut blueprint) = + ExampleSystemBuilder::new(&opctx.log, TEST_NAME).nsleds(5).build(); + + // Set the target so our database-modifying operations know they + // can safely act on the current target blueprint. + update_blueprint_target(&datastore, &opctx, &mut blueprint).await; + + // Record the sleds and zpools. + crate::tests::insert_sled_records(datastore, &blueprint).await; + crate::tests::create_disks_for_zones_using_datasets( + datastore, opctx, &blueprint, + ) + .await; + + let mut all_datasets = get_all_datasets_from_zones(&blueprint); + let bp_id = BlueprintUuid::from_untyped_uuid(blueprint.id); + let EnsureDatasetsResult { inserted, updated, removed } = + ensure_dataset_records_exist( + opctx, + datastore, + bp_id, + all_datasets.iter(), + ) + .await + .expect("failed to ensure datasets"); + assert_eq!(inserted, all_datasets.len()); + assert_eq!(updated, 0); + assert_eq!(removed, 0); + + // These values don't *really* matter, we just want to make sure we can + // change them and see the update. + let first_dataset = &mut all_datasets[0]; + assert_eq!(first_dataset.quota, None); + assert_eq!(first_dataset.reservation, None); + assert_eq!(first_dataset.compression, CompressionAlgorithm::Off); + + first_dataset.quota = Some(ByteCount::from_kibibytes_u32(1)); + first_dataset.reservation = Some(ByteCount::from_kibibytes_u32(2)); + first_dataset.compression = CompressionAlgorithm::Lz4; + + // Update the datastore + let EnsureDatasetsResult { inserted, updated, removed } = + ensure_dataset_records_exist( + opctx, + datastore, + bp_id, + all_datasets.iter(), + ) + .await + .expect("failed to ensure datasets"); + assert_eq!(inserted, 0); + assert_eq!(updated, 1); + assert_eq!(removed, 0); + + // Observe that the update stuck + let observed_datasets = + datastore.dataset_list_all_batched(opctx, None).await.unwrap(); + let first_dataset = &mut all_datasets[0]; + let observed_dataset = observed_datasets + .into_iter() + .find(|dataset| { + dataset.id() == first_dataset.id.into_untyped_uuid() + }) + .expect("Couldn't find dataset we tried to update?"); + let observed_dataset: DatasetConfig = + observed_dataset.try_into().unwrap(); + assert_eq!(observed_dataset.quota, first_dataset.quota); + assert_eq!(observed_dataset.reservation, first_dataset.reservation); + assert_eq!(observed_dataset.compression, first_dataset.compression); + } + + #[nexus_test] + async fn test_dataset_records_delete(cptestctx: &ControlPlaneTestContext) { + const TEST_NAME: &str = "test_dataset_records_delete"; + + // Set up. + let nexus = &cptestctx.server.server_context().nexus; + let datastore = nexus.datastore(); + let opctx = OpContext::for_tests( + cptestctx.logctx.log.clone(), + datastore.clone(), + ); + let opctx = &opctx; + + // Use the standard example system. + let (_example, mut blueprint) = + ExampleSystemBuilder::new(&opctx.log, TEST_NAME).nsleds(5).build(); + + // Set the target so our database-modifying operations know they + // can safely act on the current target blueprint. + update_blueprint_target(&datastore, &opctx, &mut blueprint).await; + + // Record the sleds and zpools. + crate::tests::insert_sled_records(datastore, &blueprint).await; + crate::tests::create_disks_for_zones_using_datasets( + datastore, opctx, &blueprint, + ) + .await; + + let mut all_datasets = get_all_datasets_from_zones(&blueprint); + + // Ensure that a non-crucible dataset exists + all_datasets.push(BlueprintDatasetConfig { + disposition: BlueprintDatasetDisposition::InService, + id: DatasetUuid::new_v4(), + pool: all_datasets[0].pool.clone(), + kind: DatasetKind::Debug, + address: None, + quota: None, + reservation: None, + compression: CompressionAlgorithm::Off, + }); + let bp_id = BlueprintUuid::from_untyped_uuid(blueprint.id); + let EnsureDatasetsResult { inserted, updated, removed } = + ensure_dataset_records_exist( + opctx, + datastore, + bp_id, + all_datasets.iter(), + ) + .await + .expect("failed to ensure datasets"); + assert_eq!(inserted, all_datasets.len()); + assert_eq!(updated, 0); + assert_eq!(removed, 0); + + // Expunge two datasets -- one for Crucible, and one for any other + // service. + + let crucible_dataset = all_datasets + .iter_mut() + .find(|dataset| matches!(dataset.kind, DatasetKind::Crucible)) + .expect("No crucible dataset found"); + assert_eq!( + crucible_dataset.disposition, + BlueprintDatasetDisposition::InService + ); + crucible_dataset.disposition = BlueprintDatasetDisposition::Expunged; + let crucible_dataset_id = crucible_dataset.id; + + let non_crucible_dataset = all_datasets + .iter_mut() + .find(|dataset| !matches!(dataset.kind, DatasetKind::Crucible)) + .expect("No non-crucible dataset found"); + assert_eq!( + non_crucible_dataset.disposition, + BlueprintDatasetDisposition::InService + ); + non_crucible_dataset.disposition = + BlueprintDatasetDisposition::Expunged; + let non_crucible_dataset_id = non_crucible_dataset.id; + + // Observe that we only remove one dataset. + // + // This is a property of "special-case handling" of the Crucible + // dataset, where we punt the deletion to a background task. + + let EnsureDatasetsResult { inserted, updated, removed } = + ensure_dataset_records_exist( + opctx, + datastore, + bp_id, + all_datasets.iter(), + ) + .await + .expect("failed to ensure datasets"); + assert_eq!(inserted, 0); + assert_eq!(updated, 0); + assert_eq!(removed, 1); + + // Make sure the Crucible dataset still exists, even if the other + // dataset got deleted. + + let observed_datasets = + datastore.dataset_list_all_batched(opctx, None).await.unwrap(); + assert!(observed_datasets + .iter() + .any(|d| d.id() == crucible_dataset_id.into_untyped_uuid())); + assert!(!observed_datasets + .iter() + .any(|d| d.id() == non_crucible_dataset_id.into_untyped_uuid())); + } + + #[nexus_test] + async fn test_dataset_record_blueprint_removal_without_expunging( + cptestctx: &ControlPlaneTestContext, + ) { + const TEST_NAME: &str = + "test_dataset_record_blueprint_removal_without_expunging"; + + // Set up. + let nexus = &cptestctx.server.server_context().nexus; + let datastore = nexus.datastore(); + let opctx = OpContext::for_tests( + cptestctx.logctx.log.clone(), + datastore.clone(), + ); + let opctx = &opctx; + + // Use the standard example system. + let (_example, mut blueprint) = + ExampleSystemBuilder::new(&opctx.log, TEST_NAME).nsleds(5).build(); + + // Set the target so our database-modifying operations know they + // can safely act on the current target blueprint. + update_blueprint_target(&datastore, &opctx, &mut blueprint).await; + + // Record the sleds and zpools. + crate::tests::insert_sled_records(datastore, &blueprint).await; + crate::tests::create_disks_for_zones_using_datasets( + datastore, opctx, &blueprint, + ) + .await; + + let mut all_datasets = get_all_datasets_from_zones(&blueprint); + + // Ensure that a deletable dataset exists + let dataset_id = DatasetUuid::new_v4(); + all_datasets.push(BlueprintDatasetConfig { + disposition: BlueprintDatasetDisposition::InService, + id: dataset_id, + pool: all_datasets[0].pool.clone(), + kind: DatasetKind::Debug, + address: None, + quota: None, + reservation: None, + compression: CompressionAlgorithm::Off, + }); + + let bp_id = BlueprintUuid::from_untyped_uuid(blueprint.id); + let EnsureDatasetsResult { inserted, updated, removed } = + ensure_dataset_records_exist( + opctx, + datastore, + bp_id, + all_datasets.iter(), + ) + .await + .expect("failed to ensure datasets"); + assert_eq!(inserted, all_datasets.len()); + assert_eq!(updated, 0); + assert_eq!(removed, 0); + + // Rather than expunging a dataset, which is the normal way to "delete" + // a dataset, we'll just remove it from the "blueprint". + // + // This situation mimics a scenario where we are an "old Nexus, + // executing an old blueprint" - more datasets might be created + // concurrently with our execution, and we should leave them alone. + assert_eq!(dataset_id, all_datasets.pop().unwrap().id); + + // Observe that no datasets are removed. + let EnsureDatasetsResult { inserted, updated, removed } = + ensure_dataset_records_exist( + opctx, + datastore, + bp_id, + all_datasets.iter(), + ) + .await + .expect("failed to ensure datasets"); + assert_eq!(inserted, 0); + assert_eq!(updated, 0); + assert_eq!(removed, 0); + + // Make sure the dataset still exists, even if it isn't tracked by our + // "blueprint". + let observed_datasets = + datastore.dataset_list_all_batched(opctx, None).await.unwrap(); + assert!(observed_datasets + .iter() + .any(|d| d.id() == dataset_id.into_untyped_uuid())); + } } diff --git a/nexus/reconfigurator/execution/src/dns.rs b/nexus/reconfigurator/execution/src/dns.rs index cd14069d50..e0d3414f1b 100644 --- a/nexus/reconfigurator/execution/src/dns.rs +++ b/nexus/reconfigurator/execution/src/dns.rs @@ -661,6 +661,7 @@ mod test { id: Uuid::new_v4(), blueprint_zones, blueprint_disks: BTreeMap::new(), + blueprint_datasets: BTreeMap::new(), sled_state, cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade::DoNotModify, @@ -1336,6 +1337,8 @@ mod test { .unwrap(); let zpool_rows = datastore.zpool_list_all_external_batched(&opctx).await.unwrap(); + let dataset_rows = + datastore.dataset_list_all_batched(&opctx, None).await.unwrap(); let ip_pool_range_rows = { let (authz_service_ip_pool, _) = datastore.ip_pools_service_lookup(&opctx).await.unwrap(); @@ -1348,6 +1351,7 @@ mod test { let mut builder = PlanningInputFromDb { sled_rows: &sled_rows, zpool_rows: &zpool_rows, + dataset_rows: &dataset_rows, ip_pool_range_rows: &ip_pool_range_rows, internal_dns_version: dns_initial_internal.generation.into(), external_dns_version: dns_latest_external.generation.into(), @@ -1395,7 +1399,15 @@ mod test { let rv = builder .sled_ensure_zone_multiple_nexus(sled_id, nalready + 1) .unwrap(); - assert_eq!(rv, EnsureMultiple::Changed { added: 1, removed: 0 }); + assert_eq!( + rv, + EnsureMultiple::Changed { + added: 1, + updated: 0, + expunged: 0, + removed: 0 + } + ); let blueprint2 = builder.build(); eprintln!("blueprint2: {}", blueprint2.display()); // Figure out the id of the new zone. diff --git a/nexus/reconfigurator/execution/src/lib.rs b/nexus/reconfigurator/execution/src/lib.rs index e160ddc9a0..48ab3996ff 100644 --- a/nexus/reconfigurator/execution/src/lib.rs +++ b/nexus/reconfigurator/execution/src/lib.rs @@ -12,11 +12,13 @@ use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; use nexus_types::deployment::execution::*; use nexus_types::deployment::Blueprint; +use nexus_types::deployment::BlueprintDatasetFilter; use nexus_types::deployment::BlueprintZoneFilter; use nexus_types::deployment::SledFilter; use nexus_types::external_api::views::SledState; use nexus_types::identity::Asset; use omicron_physical_disks::DeployDisksDone; +use omicron_uuid_kinds::BlueprintUuid; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::SledUuid; @@ -115,6 +117,13 @@ pub async fn realize_blueprint_with_overrides( sled_list.clone(), ); + register_deploy_datasets_step( + &engine.for_component(ExecutionComponent::Datasets), + &opctx, + blueprint, + sled_list.clone(), + ); + register_deploy_zones_step( &engine.for_component(ExecutionComponent::OmicronZones), &opctx, @@ -283,6 +292,32 @@ fn register_deploy_disks_step<'a>( .register() } +fn register_deploy_datasets_step<'a>( + registrar: &ComponentRegistrar<'_, 'a>, + opctx: &'a OpContext, + blueprint: &'a Blueprint, + sleds: SharedStepHandle>>, +) { + registrar + .new_step( + ExecutionStepId::Ensure, + "Deploy datasets", + move |cx| async move { + let sleds_by_id = sleds.into_value(cx.token()).await; + datasets::deploy_datasets( + &opctx, + &sleds_by_id, + &blueprint.blueprint_datasets, + ) + .await + .map_err(merge_anyhow_list)?; + + StepSuccess::new(()).into() + }, + ) + .register(); +} + fn register_deploy_zones_step<'a>( registrar: &ComponentRegistrar<'_, 'a>, opctx: &'a OpContext, @@ -348,6 +383,7 @@ fn register_dataset_records_step<'a>( datastore: &'a DataStore, blueprint: &'a Blueprint, ) { + let bp_id = BlueprintUuid::from_untyped_uuid(blueprint.id); registrar .new_step( ExecutionStepId::Ensure, @@ -356,9 +392,8 @@ fn register_dataset_records_step<'a>( datasets::ensure_dataset_records_exist( &opctx, datastore, - blueprint - .all_omicron_zones(BlueprintZoneFilter::ShouldBeRunning) - .map(|(_sled_id, zone)| zone), + bp_id, + blueprint.all_omicron_datasets(BlueprintDatasetFilter::All), ) .await?; diff --git a/nexus/reconfigurator/execution/src/omicron_physical_disks.rs b/nexus/reconfigurator/execution/src/omicron_physical_disks.rs index 895b7a2d9d..b56c9c0433 100644 --- a/nexus/reconfigurator/execution/src/omicron_physical_disks.rs +++ b/nexus/reconfigurator/execution/src/omicron_physical_disks.rs @@ -66,7 +66,7 @@ pub(crate) async fn deploy_disks( if !errs.is_empty() { warn!( log, - "Failed to deploy storage for sled agent"; + "Failed to deploy physical disk for sled agent"; "successfully configured disks" => successes.len(), "failed disk configurations" => errs.len(), ); @@ -81,7 +81,7 @@ pub(crate) async fn deploy_disks( info!( log, - "Successfully deployed storage for sled agent"; + "Successfully deployed physical disks for sled agent"; "successfully configured disks" => successes.len(), ); None @@ -181,6 +181,7 @@ mod test { id, blueprint_zones: BTreeMap::new(), blueprint_disks, + blueprint_datasets: BTreeMap::new(), sled_state: BTreeMap::new(), cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade::DoNotModify, diff --git a/nexus/reconfigurator/execution/src/omicron_zones.rs b/nexus/reconfigurator/execution/src/omicron_zones.rs index 0e299cebd3..b7587377c9 100644 --- a/nexus/reconfigurator/execution/src/omicron_zones.rs +++ b/nexus/reconfigurator/execution/src/omicron_zones.rs @@ -383,6 +383,7 @@ mod test { id, blueprint_zones, blueprint_disks: BTreeMap::new(), + blueprint_datasets: BTreeMap::new(), sled_state: BTreeMap::new(), cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade::DoNotModify, diff --git a/nexus/reconfigurator/planning/Cargo.toml b/nexus/reconfigurator/planning/Cargo.toml index 9607e26394..19e429dcd9 100644 --- a/nexus/reconfigurator/planning/Cargo.toml +++ b/nexus/reconfigurator/planning/Cargo.toml @@ -12,6 +12,7 @@ clickhouse-admin-types.workspace = true chrono.workspace = true debug-ignore.workspace = true gateway-client.workspace = true +illumos-utils.workspace = true indexmap.workspace = true internal-dns-resolver.workspace = true ipnet.workspace = true diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs index fb7d1f2dca..11a44321d9 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs @@ -15,6 +15,9 @@ use nexus_sled_agent_shared::inventory::OmicronZoneDataset; use nexus_sled_agent_shared::inventory::ZoneKind; use nexus_types::deployment::blueprint_zone_type; use nexus_types::deployment::Blueprint; +use nexus_types::deployment::BlueprintDatasetConfig; +use nexus_types::deployment::BlueprintDatasetDisposition; +use nexus_types::deployment::BlueprintDatasetsConfig; use nexus_types::deployment::BlueprintPhysicalDiskConfig; use nexus_types::deployment::BlueprintPhysicalDisksConfig; use nexus_types::deployment::BlueprintZoneConfig; @@ -45,11 +48,17 @@ use omicron_common::address::DNS_HTTP_PORT; use omicron_common::address::DNS_PORT; use omicron_common::address::NTP_PORT; use omicron_common::address::SLED_RESERVED_ADDRESSES; +use omicron_common::api::external::ByteCount; use omicron_common::api::external::Generation; use omicron_common::api::external::Vni; +use omicron_common::api::internal::shared::DatasetKind; use omicron_common::api::internal::shared::NetworkInterface; use omicron_common::api::internal::shared::NetworkInterfaceKind; +use omicron_common::disk::CompressionAlgorithm; +use omicron_common::disk::DatasetConfig; +use omicron_common::disk::DatasetName; use omicron_common::policy::INTERNAL_DNS_REDUNDANCY; +use omicron_uuid_kinds::DatasetUuid; use omicron_uuid_kinds::ExternalIpKind; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::OmicronZoneKind; @@ -140,8 +149,19 @@ pub enum Ensure { /// actions taken or no action was necessary #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum EnsureMultiple { - /// action was taken, and multiple items were added - Changed { added: usize, removed: usize }, + /// action was taken within the operation + Changed { + /// An item was added to the blueprint + added: usize, + /// An item was updated within the blueprint + updated: usize, + /// An item was expunged in the blueprint + expunged: usize, + /// An item was removed from the blueprint. + /// + /// This usually happens after the work of expungment has completed. + removed: usize, + }, /// no action was necessary NotNeeded, @@ -154,9 +174,28 @@ pub enum EnsureMultiple { /// "comment", identifying which operations have occurred on the blueprint. #[derive(Debug, Clone, Eq, PartialEq)] pub(crate) enum Operation { - AddZone { sled_id: SledUuid, kind: ZoneKind }, - UpdateDisks { sled_id: SledUuid, added: usize, removed: usize }, - ZoneExpunged { sled_id: SledUuid, reason: ZoneExpungeReason, count: usize }, + AddZone { + sled_id: SledUuid, + kind: ZoneKind, + }, + UpdateDisks { + sled_id: SledUuid, + added: usize, + updated: usize, + removed: usize, + }, + UpdateDatasets { + sled_id: SledUuid, + added: usize, + updated: usize, + expunged: usize, + removed: usize, + }, + ZoneExpunged { + sled_id: SledUuid, + reason: ZoneExpungeReason, + count: usize, + }, } impl fmt::Display for Operation { @@ -165,8 +204,17 @@ impl fmt::Display for Operation { Self::AddZone { sled_id, kind } => { write!(f, "sled {sled_id}: added zone: {}", kind.report_str()) } - Self::UpdateDisks { sled_id, added, removed } => { - write!(f, "sled {sled_id}: added {added} disks, removed {removed} disks") + Self::UpdateDisks { sled_id, added, updated, removed } => { + write!(f, "sled {sled_id}: added {added} disks, updated {updated}, removed {removed} disks") + } + Self::UpdateDatasets { + sled_id, + added, + updated, + expunged, + removed, + } => { + write!(f, "sled {sled_id}: added {added} datasets, updated: {updated}, expunged {expunged}, removed {removed} datasets") } Self::ZoneExpunged { sled_id, reason, count } => { let reason = match reason { @@ -195,6 +243,13 @@ impl fmt::Display for Operation { } } +fn zone_name(zone: &BlueprintZoneConfig) -> String { + illumos_utils::zone::zone_name( + zone.zone_type.kind().zone_prefix(), + Some(zone.id), + ) +} + /// Helper for assembling a blueprint /// /// There are two basic ways to assemble a new blueprint: @@ -232,6 +287,7 @@ pub struct BlueprintBuilder<'a> { // corresponding fields in `Blueprint`. pub(super) zones: BlueprintZonesBuilder<'a>, disks: BlueprintDisksBuilder<'a>, + datasets: BlueprintDatasetsBuilder<'a>, sled_state: BTreeMap, cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade, @@ -294,6 +350,7 @@ impl<'a> BlueprintBuilder<'a> { id: rng.blueprint_rng.next(), blueprint_zones, blueprint_disks: BTreeMap::new(), + blueprint_datasets: BTreeMap::new(), sled_state, parent_blueprint_id: None, internal_dns_version: Generation::new(), @@ -394,6 +451,7 @@ impl<'a> BlueprintBuilder<'a> { internal_dns_subnets: OnceCell::new(), zones: BlueprintZonesBuilder::new(parent_blueprint), disks: BlueprintDisksBuilder::new(parent_blueprint), + datasets: BlueprintDatasetsBuilder::new(parent_blueprint), sled_state, cockroachdb_setting_preserve_downgrade: parent_blueprint .cockroachdb_setting_preserve_downgrade, @@ -478,6 +536,9 @@ impl<'a> BlueprintBuilder<'a> { let blueprint_disks = self .disks .into_disks_map(self.input.all_sled_ids(SledFilter::InService)); + let blueprint_datasets = self + .datasets + .into_datasets_map(self.input.all_sled_ids(SledFilter::InService)); // If we have an allocator, use it to generate a new config. If an error // is returned then log it and carry over the parent_config. @@ -494,6 +555,7 @@ impl<'a> BlueprintBuilder<'a> { id: self.rng.blueprint_rng.next(), blueprint_zones, blueprint_disks, + blueprint_datasets, sled_state: self.sled_state, parent_blueprint_id: Some(self.parent_blueprint.id), internal_dns_version: self.input.internal_dns_version(), @@ -778,7 +840,208 @@ impl<'a> BlueprintBuilder<'a> { !removals.contains(&PhysicalDiskUuid::from_untyped_uuid(config.id)) }); - Ok(EnsureMultiple::Changed { added, removed }) + Ok(EnsureMultiple::Changed { added, updated: 0, expunged: 0, removed }) + } + + /// Ensures that a sled in the blueprint has all the datasets it should. + /// + /// We perform the following process to decide what datasets should exist + /// in the blueprint during the planning phase: + /// + /// INPUT | OUTPUT + /// ---------------------------------------------------------------------- + /// zpools in the blueprint | blueprint datasets for debug, root filesystem + /// | (All zpools should have these datasets) + /// ---------------------------------------------------------------------- + /// zones in the blueprint | blueprint datasets for filesystems, durable data + /// | (These datasets are needed for zones) + /// ---------------------------------------------------------------------- + /// discretionary datasets | blueprint datasets for discretionary datasets + /// NOTE: These don't exist, | + /// at the moment | + /// ---------------------------------------------------------------------- + /// + /// From this process, we should be able to construct "all datasets that + /// should exist in the new blueprint". + /// + /// - If new datasets are proposed, they are added to the blueprint. + /// - If datasets are changed, they are updated in the blueprint. + /// - If datasets are not proposed, but they exist in the parent blueprint, + /// they are expunged. + pub fn sled_ensure_datasets( + &mut self, + sled_id: SledUuid, + resources: &SledResources, + ) -> Result { + const DEBUG_QUOTA_SIZE_GB: u32 = 100; + + let (mut additions, mut updates, mut expunges, removals) = { + let mut datasets_builder = BlueprintSledDatasetsBuilder::new( + self.log.clone(), + sled_id, + &self.datasets, + resources, + ); + + // Ensure each zpool has a "Debug" and "Zone Root" dataset. + let bp_zpools = self + .disks + .current_sled_disks(sled_id) + .map(|disk_config| disk_config.pool_id) + .collect::>(); + for zpool_id in bp_zpools { + let zpool = ZpoolName::new_external(zpool_id); + let address = None; + datasets_builder.ensure( + DatasetName::new(zpool.clone(), DatasetKind::Debug), + address, + Some(ByteCount::from_gibibytes_u32(DEBUG_QUOTA_SIZE_GB)), + None, + CompressionAlgorithm::Off, + ); + datasets_builder.ensure( + DatasetName::new(zpool, DatasetKind::TransientZoneRoot), + address, + None, + None, + CompressionAlgorithm::Off, + ); + } + + // Ensure that datasets needed for zones exist. + for (zone, _zone_state) in self.zones.current_sled_zones( + sled_id, + BlueprintZoneFilter::ShouldBeRunning, + ) { + // Dataset for transient zone filesystem + if let Some(fs_zpool) = &zone.filesystem_pool { + let name = zone_name(&zone); + let address = None; + datasets_builder.ensure( + DatasetName::new( + fs_zpool.clone(), + DatasetKind::TransientZone { name }, + ), + address, + None, + None, + CompressionAlgorithm::Off, + ); + } + + // Dataset for durable dataset co-located with zone + if let Some(dataset) = zone.zone_type.durable_dataset() { + let zpool = &dataset.dataset.pool_name; + let address = match zone.zone_type { + BlueprintZoneType::Crucible( + blueprint_zone_type::Crucible { address, .. }, + ) => Some(address), + _ => None, + }; + datasets_builder.ensure( + DatasetName::new(zpool.clone(), dataset.kind), + address, + None, + None, + CompressionAlgorithm::Off, + ); + } + } + + // TODO: Note that we also have datasets in "zone/" for propolis + // zones, but these are not currently being tracked by blueprints. + + let expunges = datasets_builder.get_expungeable_datasets(); + let removals = datasets_builder.get_removable_datasets(); + + let additions = datasets_builder + .new_datasets + .into_values() + .flat_map(|datasets| datasets.into_values().map(|d| (d.id, d))) + .collect::>(); + let updates = datasets_builder + .updated_datasets + .into_values() + .flat_map(|datasets| { + datasets.into_values().map(|dataset| (dataset.id, dataset)) + }) + .collect::>(); + (additions, updates, expunges, removals) + }; + + if additions.is_empty() + && updates.is_empty() + && expunges.is_empty() + && removals.is_empty() + { + return Ok(EnsureMultiple::NotNeeded); + } + let added = additions.len(); + let updated = updates.len(); + // - When a dataset is expunged, for whatever reason, it is a part of + // "expunges". This leads to it getting removed from a sled. + // - When we know that we've safely destroyed all traces of the dataset, + // it becomes a part of "removals". This means we can remove it from the + // blueprint. + let expunged = expunges.len(); + let removed = removals.len(); + + let datasets = + &mut self.datasets.change_sled_datasets(sled_id).datasets; + + // Add all new datasets + datasets.append(&mut additions); + + for config in datasets.values_mut() { + // Apply updates + if let Some(new_config) = updates.remove(&config.id) { + *config = new_config; + }; + + // Mark unused datasets as expunged. + // + // This indicates that the dataset should be removed from the database. + if expunges.remove(&config.id) { + config.disposition = BlueprintDatasetDisposition::Expunged; + } + + // Small optimization -- if no expungement nor updates are left, + // bail + if expunges.is_empty() && updates.is_empty() { + break; + } + } + + // These conditions should be dead-code, and arguably could be + // assertions, but are safety nets to catch programming errors. + if !expunges.is_empty() { + return Err(Error::Planner(anyhow!( + "Should have marked all expunged datasets" + ))); + } + if !updates.is_empty() { + return Err(Error::Planner(anyhow!( + "Should have applied all updates" + ))); + } + + // Remove all datasets that we've finished expunging. + datasets.retain(|_id, d| { + if removals.contains(&d.id) { + debug_assert_eq!( + d.disposition, + BlueprintDatasetDisposition::Expunged, + "Should only remove datasets that are expunged, but dataset {} is {:?}", + d.id, d.disposition, + ); + return false; + }; + true + }); + + // We sort in the call to "BlueprintDatasetsBuilder::into_datasets_map", + // so we don't need to sort "datasets" now. + Ok(EnsureMultiple::Changed { added, updated, expunged, removed }) } fn sled_add_zone_internal_dns( @@ -840,7 +1103,12 @@ impl<'a> BlueprintBuilder<'a> { )?; } - Ok(EnsureMultiple::Changed { added: to_add, removed: 0 }) + Ok(EnsureMultiple::Changed { + added: to_add, + removed: 0, + expunged: 0, + updated: 0, + }) } fn sled_add_zone_external_dns( @@ -930,7 +1198,12 @@ impl<'a> BlueprintBuilder<'a> { } } - Ok(EnsureMultiple::Changed { added, removed: 0 }) + Ok(EnsureMultiple::Changed { + added, + updated: 0, + removed: 0, + expunged: 0, + }) } pub fn sled_ensure_zone_ntp( @@ -1148,7 +1421,12 @@ impl<'a> BlueprintBuilder<'a> { self.sled_add_zone(sled_id, zone)?; } - Ok(EnsureMultiple::Changed { added: num_nexus_to_add, removed: 0 }) + Ok(EnsureMultiple::Changed { + added: num_nexus_to_add, + updated: 0, + expunged: 0, + removed: 0, + }) } pub fn sled_ensure_zone_multiple_oximeter( @@ -1193,7 +1471,12 @@ impl<'a> BlueprintBuilder<'a> { self.sled_add_zone(sled_id, zone)?; } - Ok(EnsureMultiple::Changed { added: num_oximeter_to_add, removed: 0 }) + Ok(EnsureMultiple::Changed { + added: num_oximeter_to_add, + updated: 0, + expunged: 0, + removed: 0, + }) } pub fn sled_ensure_zone_multiple_crucible_pantry( @@ -1237,7 +1520,12 @@ impl<'a> BlueprintBuilder<'a> { self.sled_add_zone(sled_id, zone)?; } - Ok(EnsureMultiple::Changed { added: num_pantry_to_add, removed: 0 }) + Ok(EnsureMultiple::Changed { + added: num_pantry_to_add, + updated: 0, + expunged: 0, + removed: 0, + }) } pub fn cockroachdb_preserve_downgrade( @@ -1292,7 +1580,12 @@ impl<'a> BlueprintBuilder<'a> { self.sled_add_zone(sled_id, zone)?; } - Ok(EnsureMultiple::Changed { added: num_crdb_to_add, removed: 0 }) + Ok(EnsureMultiple::Changed { + added: num_crdb_to_add, + updated: 0, + expunged: 0, + removed: 0, + }) } fn sled_add_zone_clickhouse( @@ -1343,7 +1636,12 @@ impl<'a> BlueprintBuilder<'a> { for _ in 0..to_add { self.sled_add_zone_clickhouse(sled_id)?; } - Ok(EnsureMultiple::Changed { added: to_add, removed: 0 }) + Ok(EnsureMultiple::Changed { + added: to_add, + updated: 0, + expunged: 0, + removed: 0, + }) } pub fn sled_ensure_zone_multiple_clickhouse_server( @@ -1396,6 +1694,8 @@ impl<'a> BlueprintBuilder<'a> { Ok(EnsureMultiple::Changed { added: num_clickhouse_servers_to_add, + updated: 0, + expunged: 0, removed: 0, }) } @@ -1451,6 +1751,8 @@ impl<'a> BlueprintBuilder<'a> { Ok(EnsureMultiple::Changed { added: num_clickhouse_keepers_to_add, + updated: 0, + expunged: 0, removed: 0, }) } @@ -1583,7 +1885,12 @@ impl<'a> BlueprintBuilder<'a> { }, )?; - Ok(EnsureMultiple::Changed { added: 1, removed: 1 }) + Ok(EnsureMultiple::Changed { + added: 1, + updated: 0, + expunged: 0, + removed: 1, + }) } pub fn sled_expunge_zone( @@ -2018,6 +2325,335 @@ impl<'a> BlueprintDisksBuilder<'a> { } } +/// Helper for working with sets of datasets on each sled +struct BlueprintDatasetsBuilder<'a> { + changed_datasets: BTreeMap, + parent_datasets: &'a BTreeMap, +} + +impl<'a> BlueprintDatasetsBuilder<'a> { + pub fn new(parent_blueprint: &'a Blueprint) -> BlueprintDatasetsBuilder { + BlueprintDatasetsBuilder { + changed_datasets: BTreeMap::new(), + parent_datasets: &parent_blueprint.blueprint_datasets, + } + } + + pub fn change_sled_datasets( + &mut self, + sled_id: SledUuid, + ) -> &mut BlueprintDatasetsConfig { + self.changed_datasets.entry(sled_id).or_insert_with(|| { + if let Some(old_sled_datasets) = self.parent_datasets.get(&sled_id) + { + BlueprintDatasetsConfig { + generation: old_sled_datasets.generation.next(), + datasets: old_sled_datasets.datasets.clone(), + } + } else { + BlueprintDatasetsConfig { + generation: Generation::new(), + datasets: BTreeMap::new(), + } + } + }) + } + + /// Iterates over the list of Omicron datasets currently configured for this + /// sled in the blueprint that's being built + pub fn current_sled_datasets( + &self, + sled_id: SledUuid, + ) -> Box + '_> { + if let Some(sled_datasets) = self + .changed_datasets + .get(&sled_id) + .or_else(|| self.parent_datasets.get(&sled_id)) + { + Box::new(sled_datasets.datasets.values()) + } else { + Box::new(std::iter::empty()) + } + } + + /// Produces an owned map of datasets for the requested sleds + pub fn into_datasets_map( + mut self, + sled_ids: impl Iterator, + ) -> BTreeMap { + sled_ids + .map(|sled_id| { + // Start with self.changed_datasets, which contains entries for any + // sled whose datasets config is changing in this blueprint. + let datasets = self + .changed_datasets + .remove(&sled_id) + // If it's not there, use the config from the parent + // blueprint. + .or_else(|| self.parent_datasets.get(&sled_id).cloned()) + // If it's not there either, then this must be a new sled + // and we haven't added any datasets to it yet. Use the + // standard initial config. + .unwrap_or_else(|| BlueprintDatasetsConfig { + generation: Generation::new(), + datasets: BTreeMap::new(), + }); + + (sled_id, datasets) + }) + .collect() + } +} + +/// Helper for working with sets of datasets on a single sled +#[derive(Debug)] +struct BlueprintSledDatasetsBuilder<'a> { + log: Logger, + blueprint_datasets: + BTreeMap>, + database_datasets: + BTreeMap>, + + // Datasets which are unchanged from the prior blueprint + unchanged_datasets: + BTreeMap>, + // Datasets which are new in this blueprint + new_datasets: + BTreeMap>, + // Datasets which existed in the old blueprint, but which are + // changing in this one + updated_datasets: + BTreeMap>, +} + +impl<'a> BlueprintSledDatasetsBuilder<'a> { + pub fn new( + log: Logger, + sled_id: SledUuid, + datasets: &'a BlueprintDatasetsBuilder<'_>, + resources: &'a SledResources, + ) -> Self { + // Gather all datasets known to the blueprint + let mut blueprint_datasets: BTreeMap< + ZpoolUuid, + BTreeMap, + > = BTreeMap::new(); + for dataset in datasets.current_sled_datasets(sled_id) { + blueprint_datasets + .entry(dataset.pool.id()) + .or_default() + .insert(dataset.kind.clone(), dataset); + } + + // Gather all datasets known to the database + let mut database_datasets = BTreeMap::new(); + for (zpool, datasets) in resources.all_datasets(ZpoolFilter::InService) + { + let datasets_by_kind = datasets + .into_iter() + .map(|dataset| (dataset.name.dataset().clone(), dataset)) + .collect(); + + database_datasets.insert(*zpool, datasets_by_kind); + } + + Self { + log, + blueprint_datasets, + database_datasets, + unchanged_datasets: BTreeMap::new(), + new_datasets: BTreeMap::new(), + updated_datasets: BTreeMap::new(), + } + } + + /// Attempts to add a dataset to the builder. + /// + /// - If the dataset exists in the blueprint already, use it. + /// - Otherwise, if the dataset exists in the database, re-use the UUID, but + /// add it to the blueprint. + /// - Otherwse, create a new dataset in the blueprint, which will propagate + /// to the database during execution. + pub fn ensure( + &mut self, + dataset: DatasetName, + address: Option, + quota: Option, + reservation: Option, + compression: CompressionAlgorithm, + ) { + let zpool = dataset.pool(); + let zpool_id = zpool.id(); + let kind = dataset.dataset(); + + let make_config = |id: DatasetUuid| BlueprintDatasetConfig { + disposition: BlueprintDatasetDisposition::InService, + id, + pool: zpool.clone(), + kind: kind.clone(), + address, + quota, + reservation, + compression, + }; + + // This dataset already exists in the blueprint + if let Some(old_config) = self.get_from_bp(zpool_id, kind) { + let new_config = make_config(old_config.id); + + // If it needs updating, add it + let target = if *old_config != new_config { + &mut self.updated_datasets + } else { + &mut self.unchanged_datasets + }; + target + .entry(zpool_id) + .or_default() + .insert(new_config.kind.clone(), new_config); + return; + } + + // If the dataset exists in the datastore, re-use the UUID. + // + // TODO(https://github.com/oxidecomputer/omicron/issues/6645): We + // could avoid reading from the datastore if we were confident all + // provisioned datasets existed in the parent blueprint. + let id = if let Some(old_config) = self.get_from_db(zpool_id, kind) { + old_config.id + } else { + DatasetUuid::new_v4() + }; + + let new_config = make_config(id); + self.new_datasets + .entry(zpool_id) + .or_default() + .insert(new_config.kind.clone(), new_config); + } + + /// Returns all datasets in the old blueprint that are not planned to be + /// part of the new blueprint. + pub fn get_expungeable_datasets(&self) -> BTreeSet { + let dataset_exists_in = + |group: &BTreeMap< + ZpoolUuid, + BTreeMap, + >, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid| { + let Some(datasets) = group.get(&zpool_id) else { + return false; + }; + + datasets.values().any(|config| config.id == dataset_id) + }; + + let mut expunges = BTreeSet::new(); + + for (zpool_id, datasets) in &self.blueprint_datasets { + for dataset_config in datasets.values() { + match dataset_config.disposition { + // Already expunged; ignore + BlueprintDatasetDisposition::Expunged => continue, + // Potentially expungeable + BlueprintDatasetDisposition::InService => (), + }; + + let dataset_id = dataset_config.id; + if !dataset_exists_in(&self.new_datasets, *zpool_id, dataset_id) + && !dataset_exists_in( + &self.updated_datasets, + *zpool_id, + dataset_id, + ) + && !dataset_exists_in( + &self.unchanged_datasets, + *zpool_id, + dataset_id, + ) + { + info!(self.log, "dataset expungeable (not needed in blueprint)"; "id" => ?dataset_id); + expunges.insert(dataset_id); + } + } + } + + expunges + } + + /// TODO: + /// This function SHOULD do the following: + /// + /// Returns all datasets that have been expunged in a prior blueprint, and + /// which have also been removed from the database and from inventory. + /// This is our sign that the work of expungement has completed. + /// + /// TODO: In reality, however, this function actually implements the + /// following: + /// + /// - It returns an empty BTreeSet, effectively saying "no datasets are + /// removable from the blueprint". + pub fn get_removable_datasets(&self) -> BTreeSet { + let dataset_exists_in = + |group: &BTreeMap< + ZpoolUuid, + BTreeMap, + >, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid| { + let Some(datasets) = group.get(&zpool_id) else { + return false; + }; + + datasets.values().any(|config| config.id == dataset_id) + }; + + let removals = BTreeSet::new(); + for (zpool_id, datasets) in &self.blueprint_datasets { + for (_kind, config) in datasets { + if config.disposition == BlueprintDatasetDisposition::Expunged + && !dataset_exists_in( + &self.database_datasets, + *zpool_id, + config.id, + ) + { + info!(self.log, "dataset removable (expunged, not in database)"; "id" => ?config.id); + + // TODO(https://github.com/oxidecomputer/omicron/issues/6646): + // We could call `removals.insert(config.id)` here, but + // instead, opt to just log that the dataset is removable + // and keep it in the blueprint. + } + } + } + removals + } + + fn get_from_bp( + &self, + zpool: ZpoolUuid, + kind: &DatasetKind, + ) -> Option<&'a BlueprintDatasetConfig> { + self.blueprint_datasets + .get(&zpool) + .and_then(|datasets| datasets.get(kind)) + .copied() + } + + fn get_from_db( + &self, + zpool: ZpoolUuid, + kind: &DatasetKind, + ) -> Option<&'a DatasetConfig> { + self.database_datasets + .get(&zpool) + .and_then(|datasets| datasets.get(kind)) + .copied() + } +} + #[cfg(test)] pub mod test { use super::*; @@ -2036,6 +2672,35 @@ pub mod test { use std::collections::BTreeSet; use std::mem; + pub const DEFAULT_N_SLEDS: usize = 3; + + fn datasets_for_sled( + blueprint: &Blueprint, + sled_id: SledUuid, + ) -> &BTreeMap { + &blueprint + .blueprint_datasets + .get(&sled_id) + .unwrap_or_else(|| { + panic!("Cannot find datasets on missing sled: {sled_id}") + }) + .datasets + } + + fn find_dataset<'a>( + datasets: &'a BTreeMap, + zpool: &ZpoolName, + kind: DatasetKind, + ) -> &'a BlueprintDatasetConfig { + datasets.values().find(|dataset| { + &dataset.pool == zpool && + dataset.kind == kind + }).unwrap_or_else(|| { + let kinds = datasets.values().map(|d| (&d.id, &d.pool, &d.kind)).collect::>(); + panic!("Cannot find dataset of type {kind}\nFound the following: {kinds:#?}") + }) + } + /// Checks various conditions that should be true for all blueprints #[track_caller] pub fn verify_blueprint(blueprint: &Blueprint) { @@ -2088,6 +2753,66 @@ pub mod test { } } } + + // All commissioned disks should have debug and zone root datasets. + for (sled_id, disk_config) in &blueprint.blueprint_disks { + for disk in &disk_config.disks { + let zpool = ZpoolName::new_external(disk.pool_id); + let datasets = datasets_for_sled(&blueprint, *sled_id); + + let dataset = + find_dataset(&datasets, &zpool, DatasetKind::Debug); + assert_eq!( + dataset.disposition, + BlueprintDatasetDisposition::InService + ); + let dataset = find_dataset( + &datasets, + &zpool, + DatasetKind::TransientZoneRoot, + ); + assert_eq!( + dataset.disposition, + BlueprintDatasetDisposition::InService + ); + } + } + // All zones should have dataset records. + for (sled_id, zone_config) in + blueprint.all_omicron_zones(BlueprintZoneFilter::ShouldBeRunning) + { + match blueprint.sled_state.get(&sled_id) { + // Decommissioned sleds don't keep dataset state around. + // + // Normally we wouldn't observe zones from decommissioned sleds + // anyway, but that's the responsibility of the Planner, not the + // BlueprintBuilder. + None | Some(SledState::Decommissioned) => continue, + Some(SledState::Active) => (), + } + let datasets = datasets_for_sled(&blueprint, sled_id); + + let zpool = zone_config.filesystem_pool.as_ref().unwrap(); + let kind = + DatasetKind::TransientZone { name: zone_name(&zone_config) }; + let dataset = find_dataset(&datasets, &zpool, kind); + assert_eq!( + dataset.disposition, + BlueprintDatasetDisposition::InService + ); + + if let Some(durable_dataset) = + zone_config.zone_type.durable_dataset() + { + let zpool = &durable_dataset.dataset.pool_name; + let dataset = + find_dataset(&datasets, &zpool, durable_dataset.kind); + assert_eq!( + dataset.disposition, + BlueprintDatasetDisposition::InService + ); + } + } } #[track_caller] @@ -2217,6 +2942,7 @@ pub mod test { for pool_id in new_sled_resources.zpools.keys() { builder.sled_ensure_zone_crucible(new_sled_id, *pool_id).unwrap(); } + builder.sled_ensure_datasets(new_sled_id, new_sled_resources).unwrap(); let blueprint3 = builder.build(); verify_blueprint(&blueprint3); @@ -2444,6 +3170,8 @@ pub mod test { .unwrap(), EnsureMultiple::Changed { added: usize::from(SledBuilder::DEFAULT_NPOOLS), + updated: 0, + expunged: 0, removed: 0 }, ); @@ -2490,6 +3218,163 @@ pub mod test { logctx.cleanup_successful(); } + #[test] + fn test_datasets_for_zpools_and_zones() { + static TEST_NAME: &str = "test_datasets_for_zpools_and_zones"; + let logctx = test_setup_log(TEST_NAME); + let (collection, input, blueprint) = example(&logctx.log, TEST_NAME); + + // Creating the "example" blueprint should already invoke + // `sled_ensure_datasets`. + // + // Verify that it has created the datasets we expect to exist. + verify_blueprint(&blueprint); + + let mut builder = BlueprintBuilder::new_based_on( + &logctx.log, + &blueprint, + &input, + &collection, + "test", + ) + .expect("failed to create builder"); + + // Before we make any modifications, there should be no work to do. + // + // If we haven't changed inputs, the output should be the same! + for (sled_id, resources) in + input.all_sled_resources(SledFilter::Commissioned) + { + let r = builder.sled_ensure_datasets(sled_id, resources).unwrap(); + assert_eq!(r, EnsureMultiple::NotNeeded); + } + + // Expunge a zone from the blueprint, observe that the dataset is + // removed. + let sled_id = input + .all_sled_ids(SledFilter::Commissioned) + .next() + .expect("at least one sled present"); + let sled_details = + input.sled_lookup(SledFilter::Commissioned, sled_id).unwrap(); + let crucible_zone_id = builder + .zones + .current_sled_zones(sled_id, BlueprintZoneFilter::ShouldBeRunning) + .find_map(|(zone_config, _)| { + if zone_config.zone_type.is_crucible() { + return Some(zone_config.id); + } + None + }) + .expect("at least one crucible must be present"); + let change = builder.zones.change_sled_zones(sled_id); + println!("Expunging crucible zone: {crucible_zone_id}"); + change.expunge_zones(BTreeSet::from([crucible_zone_id])).unwrap(); + + // In the case of Crucible, we have a durable dataset and a transient + // zone filesystem, so we expect two datasets to be expunged. + let r = builder + .sled_ensure_datasets(sled_id, &sled_details.resources) + .unwrap(); + assert_eq!( + r, + EnsureMultiple::Changed { + added: 0, + updated: 0, + expunged: 2, + removed: 0 + } + ); + // Once the datasets are expunged, no further changes will be proposed. + let r = builder + .sled_ensure_datasets(sled_id, &sled_details.resources) + .unwrap(); + assert_eq!(r, EnsureMultiple::NotNeeded); + + let blueprint = builder.build(); + verify_blueprint(&blueprint); + + let mut builder = BlueprintBuilder::new_based_on( + &logctx.log, + &blueprint, + &input, + &collection, + "test", + ) + .expect("failed to create builder"); + + // While the datasets still exist in the input (effectively, the db) we + // cannot remove them. + let r = builder + .sled_ensure_datasets(sled_id, &sled_details.resources) + .unwrap(); + assert_eq!(r, EnsureMultiple::NotNeeded); + + let blueprint = builder.build(); + verify_blueprint(&blueprint); + + // Find the datasets we've expunged in the blueprint + let expunged_datasets = blueprint + .blueprint_datasets + .get(&sled_id) + .unwrap() + .datasets + .values() + .filter_map(|dataset_config| { + if dataset_config.disposition + == BlueprintDatasetDisposition::Expunged + { + Some(dataset_config.id) + } else { + None + } + }) + .collect::>(); + // We saw two datasets being expunged earlier when we called + // `sled_ensure_datasets` -- validate that this is true when inspecting + // the blueprint too. + assert_eq!(expunged_datasets.len(), 2); + + // Remove these two datasets from the input. + let mut input_builder = input.into_builder(); + let zpools = &mut input_builder + .sleds_mut() + .get_mut(&sled_id) + .unwrap() + .resources + .zpools; + for (_, (_, datasets)) in zpools { + datasets.retain(|dataset| !expunged_datasets.contains(&dataset.id)); + } + let input = input_builder.build(); + + let mut builder = BlueprintBuilder::new_based_on( + &logctx.log, + &blueprint, + &input, + &collection, + "test", + ) + .expect("failed to create builder"); + + // Now, we should see the datasets "removed" from the blueprint, since + // we no longer need to keep around records of their expungement. + let sled_details = + input.sled_lookup(SledFilter::Commissioned, sled_id).unwrap(); + let r = builder + .sled_ensure_datasets(sled_id, &sled_details.resources) + .unwrap(); + + // TODO(https://github.com/oxidecomputer/omicron/issues/6646): + // Because of the workaround for #6646, we don't actually remove + // datasets yet. + // + // In the future, however, we will. + assert_eq!(r, EnsureMultiple::NotNeeded); + + logctx.cleanup_successful(); + } + #[test] fn test_add_nexus_with_no_existing_nexus_zones() { static TEST_NAME: &str = @@ -2620,7 +3505,15 @@ pub mod test { .sled_ensure_zone_multiple_nexus(sled_id, 1) .expect("failed to ensure nexus zone"); - assert_eq!(added, EnsureMultiple::Changed { added: 1, removed: 0 }); + assert_eq!( + added, + EnsureMultiple::Changed { + added: 1, + updated: 0, + expunged: 0, + removed: 0 + } + ); } { @@ -2639,7 +3532,15 @@ pub mod test { .sled_ensure_zone_multiple_nexus(sled_id, 3) .expect("failed to ensure nexus zone"); - assert_eq!(added, EnsureMultiple::Changed { added: 3, removed: 0 }); + assert_eq!( + added, + EnsureMultiple::Changed { + added: 3, + updated: 0, + expunged: 0, + removed: 0 + } + ); } { @@ -2907,8 +3808,14 @@ pub mod test { .expect("ensured multiple CRDB zones"); assert_eq!( ensure_result, - EnsureMultiple::Changed { added: num_sled_zpools, removed: 0 } + EnsureMultiple::Changed { + added: num_sled_zpools, + updated: 0, + expunged: 0, + removed: 0 + } ); + builder.sled_ensure_datasets(target_sled_id, sled_resources).unwrap(); let blueprint = builder.build(); verify_blueprint(&blueprint); diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/zones.rs b/nexus/reconfigurator/planning/src/blueprint_builder/zones.rs index 3b95a60ad8..2d9194ee52 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/zones.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/zones.rs @@ -271,16 +271,20 @@ mod tests { ), zpools: BTreeMap::from([( ZpoolUuid::new_v4(), - SledDisk { - disk_identity: DiskIdentity { - vendor: String::from("fake-vendor"), - serial: String::from("fake-serial"), - model: String::from("fake-model"), + ( + SledDisk { + disk_identity: DiskIdentity { + vendor: String::from("fake-vendor"), + serial: String::from("fake-serial"), + model: String::from("fake-model"), + }, + disk_id: PhysicalDiskUuid::new_v4(), + policy: PhysicalDiskPolicy::InService, + state: PhysicalDiskState::Active, }, - disk_id: PhysicalDiskUuid::new_v4(), - policy: PhysicalDiskPolicy::InService, - state: PhysicalDiskState::Active, - }, + // Datasets: Leave empty + vec![], + ), )]), }, }, @@ -421,6 +425,13 @@ mod tests { } ); + // Ensure all datasets are created for the zones we've provisioned + for (sled_id, resources) in + input2.all_sled_resources(SledFilter::Commissioned) + { + builder.sled_ensure_datasets(sled_id, resources).unwrap(); + } + // Now build the blueprint and ensure that all the changes we described // above are present. let blueprint = builder.build(); diff --git a/nexus/reconfigurator/planning/src/example.rs b/nexus/reconfigurator/planning/src/example.rs index 55919b1c79..a0761a9b6c 100644 --- a/nexus/reconfigurator/planning/src/example.rs +++ b/nexus/reconfigurator/planning/src/example.rs @@ -345,6 +345,7 @@ impl ExampleSystemBuilder { .unwrap(); } } + builder.sled_ensure_datasets(sled_id, &sled_resources).unwrap(); } let blueprint = builder.build(); @@ -388,6 +389,27 @@ impl ExampleSystemBuilder { .unwrap(); } + // Ensure that our "input" contains the datasets we would have + // provisioned. + // + // This mimics them existing within the database. + let input_sleds = input_builder.sleds_mut(); + for (sled_id, bp_datasets_config) in &blueprint.blueprint_datasets { + let sled = input_sleds.get_mut(sled_id).unwrap(); + for (_, bp_dataset) in &bp_datasets_config.datasets { + let (_, datasets) = sled + .resources + .zpools + .get_mut(&bp_dataset.pool.id()) + .unwrap(); + let bp_config: omicron_common::disk::DatasetConfig = + bp_dataset.clone().try_into().unwrap(); + if !datasets.contains(&bp_config) { + datasets.push(bp_config); + } + } + } + let mut builder = system.to_collection_builder().expect("failed to build collection"); builder.set_rng_seed((&self.test_name, "ExampleSystem collection")); diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index 7a20f57017..647d883ef1 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -231,8 +231,12 @@ impl<'a> Planner<'a> { { // First, we need to ensure that sleds are using their expected // disks. This is necessary before we can allocate any zones. - if let EnsureMultiple::Changed { added, removed } = - self.blueprint.sled_ensure_disks(sled_id, &sled_resources)? + if let EnsureMultiple::Changed { + added, + updated, + expunged: _, + removed, + } = self.blueprint.sled_ensure_disks(sled_id, &sled_resources)? { info!( &self.log, @@ -242,6 +246,7 @@ impl<'a> Planner<'a> { self.blueprint.record_operation(Operation::UpdateDisks { sled_id, added, + updated, removed, }); @@ -344,7 +349,46 @@ impl<'a> Planner<'a> { } } - self.do_plan_add_discretionary_zones(&sleds_waiting_for_ntp_zone) + self.do_plan_add_discretionary_zones(&sleds_waiting_for_ntp_zone)?; + + // Now that we've added all the disks and zones we plan on adding, + // ensure that all sleds have the datasets they need to have. + self.do_plan_datasets()?; + + Ok(()) + } + + fn do_plan_datasets(&mut self) -> Result<(), Error> { + for (sled_id, sled_resources) in + self.input.all_sled_resources(SledFilter::InService) + { + if let EnsureMultiple::Changed { + added, + updated, + expunged, + removed, + } = + self.blueprint.sled_ensure_datasets(sled_id, &sled_resources)? + { + info!( + &self.log, + "altered datasets"; + "sled_id" => %sled_id, + "added" => added, + "updated" => updated, + "expunged" => expunged, + "removed" => removed, + ); + self.blueprint.record_operation(Operation::UpdateDatasets { + sled_id, + added, + updated, + expunged, + removed, + }); + } + } + Ok(()) } fn do_plan_add_discretionary_zones( @@ -598,12 +642,19 @@ impl<'a> Planner<'a> { } }; match result { - EnsureMultiple::Changed { added, removed } => { + EnsureMultiple::Changed { + added, + updated, + expunged, + removed, + } => { info!( self.log, "modified zones on sled"; "sled_id" => %sled_id, "kind" => ?kind, "added" => added, + "updated" => updated, + "expunged" => expunged, "removed" => removed, ); new_zones_added += added; @@ -1445,7 +1496,12 @@ mod test { builder .sled_ensure_zone_multiple_external_dns(sled_id, 3) .expect("can't add external DNS zones"), - EnsureMultiple::Changed { added: 0, removed: 0 }, + EnsureMultiple::Changed { + added: 0, + updated: 0, + removed: 0, + expunged: 0 + }, ); // Build a builder for a modfied blueprint that will include @@ -1484,13 +1540,23 @@ mod test { blueprint_builder .sled_ensure_zone_multiple_external_dns(sled_1, 2) .expect("can't add external DNS zones to blueprint"), - EnsureMultiple::Changed { added: 2, removed: 0 } + EnsureMultiple::Changed { + added: 2, + updated: 0, + removed: 0, + expunged: 0 + } )); assert!(matches!( blueprint_builder .sled_ensure_zone_multiple_external_dns(sled_2, 1) .expect("can't add external DNS zones to blueprint"), - EnsureMultiple::Changed { added: 1, removed: 0 } + EnsureMultiple::Changed { + added: 1, + updated: 0, + removed: 0, + expunged: 0 + } )); let blueprint1a = blueprint_builder.build(); @@ -1641,13 +1707,13 @@ mod test { for _ in 0..NEW_IN_SERVICE_DISKS { sled_details.resources.zpools.insert( ZpoolUuid::from(zpool_rng.next()), - new_sled_disk(PhysicalDiskPolicy::InService), + (new_sled_disk(PhysicalDiskPolicy::InService), vec![]), ); } for _ in 0..NEW_EXPUNGED_DISKS { sled_details.resources.zpools.insert( ZpoolUuid::from(zpool_rng.next()), - new_sled_disk(PhysicalDiskPolicy::Expunged), + (new_sled_disk(PhysicalDiskPolicy::Expunged), vec![]), ); } @@ -1726,7 +1792,7 @@ mod test { } } let (_, sled_details) = builder.sleds_mut().iter_mut().next().unwrap(); - let (_, disk) = sled_details + let (_, (disk, _datasets)) = sled_details .resources .zpools .iter_mut() @@ -1851,7 +1917,7 @@ mod test { // For that pool, find the physical disk behind it, and mark it // expunged. let (_, sled_details) = builder.sleds_mut().iter_mut().next().unwrap(); - let disk = sled_details + let (disk, _datasets) = sled_details .resources .zpools .get_mut(&pool_to_expunge.id()) diff --git a/nexus/reconfigurator/planning/src/system.rs b/nexus/reconfigurator/planning/src/system.rs index 10f879e7d6..9ffd99bd5f 100644 --- a/nexus/reconfigurator/planning/src/system.rs +++ b/nexus/reconfigurator/planning/src/system.rs @@ -585,7 +585,8 @@ impl Sled { policy: PhysicalDiskPolicy::InService, state: PhysicalDiskState::Active, }; - (zpool, disk) + let datasets = vec![]; + (zpool, (disk, datasets)) }) .collect(); let inventory_sp = match hardware { @@ -648,8 +649,8 @@ impl Sled { disks: zpools .values() .enumerate() - .map(|(i, d)| InventoryDisk { - identity: d.disk_identity.clone(), + .map(|(i, (disk, _datasets))| InventoryDisk { + identity: disk.disk_identity.clone(), variant: DiskVariant::U2, slot: i64::try_from(i).unwrap(), active_firmware_slot: 1, diff --git a/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt b/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt index b0b82a31f8..072e26df49 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt @@ -114,7 +114,7 @@ WARNING: Zones exist without physical disks! METADATA: created by::::::::::: test_blueprint2 created at::::::::::: 1970-01-01T00:00:00.000Z - comment:::::::::::::: sled a1b477db-b629-48eb-911d-1ccdafca75b9: expunged 15 zones because: sled policy is expunged + comment:::::::::::::: sled a1b477db-b629-48eb-911d-1ccdafca75b9: expunged 15 zones because: sled policy is expunged, sled d67ce8f0-a691-4010-b414-420d82e80527: added 2 datasets, updated: 0, expunged 0, removed 0 datasets, sled fefcf4cf-f7e7-46b3-b629-058526ce440e: added 4 datasets, updated: 0, expunged 0, removed 0 datasets internal DNS version: 1 external DNS version: 1 diff --git a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt index 04c119ea9f..5d777cd9d9 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt @@ -175,7 +175,7 @@ WARNING: Zones exist without physical disks! METADATA: created by::::::::::: test_blueprint2 created at::::::::::: 1970-01-01T00:00:00.000Z - comment:::::::::::::: sled 48d95fef-bc9f-4f50-9a53-1e075836291d: expunged 14 zones because: sled policy is expunged + comment:::::::::::::: sled 48d95fef-bc9f-4f50-9a53-1e075836291d: expunged 14 zones because: sled policy is expunged, sled 75bc286f-2b4b-482c-9431-59272af529da: added 3 datasets, updated: 0, expunged 0, removed 0 datasets, sled affab35f-600a-4109-8ea0-34a067a4e0bc: added 3 datasets, updated: 0, expunged 0, removed 0 datasets internal DNS version: 1 external DNS version: 1 diff --git a/nexus/reconfigurator/preparation/src/lib.rs b/nexus/reconfigurator/preparation/src/lib.rs index 24f32e9187..39367824b9 100644 --- a/nexus/reconfigurator/preparation/src/lib.rs +++ b/nexus/reconfigurator/preparation/src/lib.rs @@ -38,6 +38,7 @@ use omicron_common::address::SLED_PREFIX; use omicron_common::api::external::Error; use omicron_common::api::external::InternalContext; use omicron_common::api::external::LookupType; +use omicron_common::disk::DatasetConfig; use omicron_common::disk::DiskIdentity; use omicron_common::policy::BOUNDARY_NTP_REDUNDANCY; use omicron_common::policy::COCKROACHDB_REDUNDANCY; @@ -63,6 +64,7 @@ pub struct PlanningInputFromDb<'a> { pub sled_rows: &'a [nexus_db_model::Sled], pub zpool_rows: &'a [(nexus_db_model::Zpool, nexus_db_model::PhysicalDisk)], + pub dataset_rows: &'a [nexus_db_model::Dataset], pub ip_pool_range_rows: &'a [nexus_db_model::IpPoolRange], pub external_ip_rows: &'a [nexus_db_model::ExternalIp], pub service_nic_rows: &'a [nexus_db_model::ServiceNetworkInterface], @@ -107,6 +109,10 @@ impl PlanningInputFromDb<'_> { .zpool_list_all_external_batched(opctx) .await .internal_context("fetching all external zpool rows")?; + let dataset_rows = datastore + .dataset_list_all_batched(opctx, None) + .await + .internal_context("fetching all datasets")?; let ip_pool_range_rows = { let (authz_service_ip_pool, _) = datastore .ip_pools_service_lookup(opctx) @@ -148,6 +154,7 @@ impl PlanningInputFromDb<'_> { let planning_input = PlanningInputFromDb { sled_rows: &sled_rows, zpool_rows: &zpool_rows, + dataset_rows: &dataset_rows, ip_pool_range_rows: &ip_pool_range_rows, target_boundary_ntp_zone_count: BOUNDARY_NTP_REDUNDANCY, target_nexus_zone_count: NEXUS_REDUNDANCY, @@ -195,6 +202,27 @@ impl PlanningInputFromDb<'_> { ); let mut zpools_by_sled_id = { + // Gather all the datasets first, by Zpool ID + let mut datasets: Vec<_> = self + .dataset_rows + .iter() + .map(|dataset| { + ( + ZpoolUuid::from_untyped_uuid(dataset.pool_id), + dataset.clone(), + ) + }) + .collect(); + datasets.sort_unstable_by_key(|(zpool_id, _)| *zpool_id); + let mut datasets_by_zpool: BTreeMap<_, Vec<_>> = BTreeMap::new(); + for (zpool_id, dataset) in datasets { + datasets_by_zpool + .entry(zpool_id) + .or_default() + .push(DatasetConfig::try_from(dataset)?); + } + + // Iterate over all Zpools, identifying their disks and datasets let mut zpools = BTreeMap::new(); for (zpool, disk) in self.zpool_rows { let sled_zpool_names = @@ -211,7 +239,10 @@ impl PlanningInputFromDb<'_> { state: disk.disk_state.into(), }; - sled_zpool_names.insert(zpool_id, disk); + let datasets = datasets_by_zpool + .remove(&zpool_id) + .unwrap_or_else(|| vec![]); + sled_zpool_names.insert(zpool_id, (disk, datasets)); } zpools }; diff --git a/nexus/src/app/background/tasks/blueprint_execution.rs b/nexus/src/app/background/tasks/blueprint_execution.rs index fa0283c942..a9d47af117 100644 --- a/nexus/src/app/background/tasks/blueprint_execution.rs +++ b/nexus/src/app/background/tasks/blueprint_execution.rs @@ -185,9 +185,10 @@ mod test { }; use nexus_types::deployment::BlueprintZoneFilter; use nexus_types::deployment::{ - blueprint_zone_type, Blueprint, BlueprintPhysicalDisksConfig, - BlueprintTarget, BlueprintZoneConfig, BlueprintZoneDisposition, - BlueprintZoneType, BlueprintZonesConfig, CockroachDbPreserveDowngrade, + blueprint_zone_type, Blueprint, BlueprintDatasetsConfig, + BlueprintPhysicalDisksConfig, BlueprintTarget, BlueprintZoneConfig, + BlueprintZoneDisposition, BlueprintZoneType, BlueprintZonesConfig, + CockroachDbPreserveDowngrade, }; use nexus_types::external_api::views::SledState; use omicron_common::api::external::Generation; @@ -213,6 +214,7 @@ mod test { opctx: &OpContext, blueprint_zones: BTreeMap, blueprint_disks: BTreeMap, + blueprint_datasets: BTreeMap, dns_version: Generation, ) -> (BlueprintTarget, Blueprint) { let id = Uuid::new_v4(); @@ -240,6 +242,7 @@ mod test { id, blueprint_zones, blueprint_disks, + blueprint_datasets, sled_state, cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade::DoNotModify, @@ -367,6 +370,7 @@ mod test { &opctx, BTreeMap::new(), BTreeMap::new(), + BTreeMap::new(), generation, ) .await, @@ -433,6 +437,7 @@ mod test { (sled_id2, make_zones(BlueprintZoneDisposition::Quiesced)), ]), BTreeMap::new(), + BTreeMap::new(), generation, ) .await; diff --git a/nexus/src/app/background/tasks/blueprint_load.rs b/nexus/src/app/background/tasks/blueprint_load.rs index 8b5c02dd80..9ce1e89df4 100644 --- a/nexus/src/app/background/tasks/blueprint_load.rs +++ b/nexus/src/app/background/tasks/blueprint_load.rs @@ -218,6 +218,7 @@ mod test { id, blueprint_zones: BTreeMap::new(), blueprint_disks: BTreeMap::new(), + blueprint_datasets: BTreeMap::new(), sled_state: BTreeMap::new(), cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade::DoNotModify, diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index 631450db20..e0601c51a0 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -825,6 +825,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { // // However, for now, this isn't necessary. blueprint_disks: BTreeMap::new(), + blueprint_datasets: BTreeMap::new(), sled_state, parent_blueprint_id: None, internal_dns_version: dns_config.generation, diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs index ef2f366a28..365fe3f19e 100644 --- a/nexus/types/src/deployment.rs +++ b/nexus/types/src/deployment.rs @@ -22,10 +22,17 @@ use nexus_sled_agent_shared::inventory::OmicronZoneConfig; use nexus_sled_agent_shared::inventory::OmicronZoneType; use nexus_sled_agent_shared::inventory::OmicronZonesConfig; use nexus_sled_agent_shared::inventory::ZoneKind; +use omicron_common::api::external::ByteCount; use omicron_common::api::external::Generation; +use omicron_common::api::internal::shared::DatasetKind; +use omicron_common::disk::CompressionAlgorithm; +use omicron_common::disk::DatasetConfig; +use omicron_common::disk::DatasetName; +use omicron_common::disk::DatasetsConfig; use omicron_common::disk::DiskIdentity; use omicron_common::disk::OmicronPhysicalDisksConfig; use omicron_uuid_kinds::CollectionUuid; +use omicron_uuid_kinds::DatasetUuid; use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::SledUuid; use schemars::JsonSchema; @@ -35,6 +42,7 @@ use std::collections::BTreeMap; use std::collections::BTreeSet; use std::fmt; use std::net::Ipv6Addr; +use std::net::SocketAddrV6; use strum::EnumIter; use strum::IntoEnumIterator; use uuid::Uuid; @@ -149,6 +157,9 @@ pub struct Blueprint { /// A map of sled id -> disks in use on each sled. pub blueprint_disks: BTreeMap, + /// A map of sled id -> datasets in use on each sled + pub blueprint_datasets: BTreeMap, + /// which blueprint this blueprint is based on pub parent_blueprint_id: Option, @@ -226,6 +237,17 @@ impl Blueprint { }) } + /// Iterate over the [`BlueprintDatasetsConfig`] instances in the blueprint. + pub fn all_omicron_datasets( + &self, + filter: BlueprintDatasetFilter, + ) -> impl Iterator { + self.blueprint_datasets + .iter() + .flat_map(move |(_, datasets)| datasets.datasets.values()) + .filter(move |d| d.disposition.matches(filter)) + } + /// Iterate over the [`BlueprintZoneConfig`] instances in the blueprint /// that do not match the provided filter, along with the associated sled /// id. @@ -800,6 +822,22 @@ pub enum BlueprintZoneFilter { ShouldDeployVpcFirewallRules, } +/// Filters that apply to blueprint datasets. +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub enum BlueprintDatasetFilter { + // --- + // Prefer to keep this list in alphabetical order. + // --- + /// All datasets + All, + + /// Datasets that have been expunged. + Expunged, + + /// Datasets that are in-service. + InService, +} + /// Information about an Omicron physical disk as recorded in a blueprint. /// /// Part of [`Blueprint`]. @@ -809,6 +847,95 @@ pub type BlueprintPhysicalDisksConfig = pub type BlueprintPhysicalDiskConfig = omicron_common::disk::OmicronPhysicalDiskConfig; +/// Information about Omicron datasets as recorded in a blueprint. +#[derive(Debug, Clone, Eq, PartialEq, JsonSchema, Deserialize, Serialize)] +pub struct BlueprintDatasetsConfig { + pub generation: Generation, + pub datasets: BTreeMap, +} + +impl From for DatasetsConfig { + fn from(config: BlueprintDatasetsConfig) -> Self { + Self { + generation: config.generation, + datasets: config + .datasets + .into_iter() + .map(|(id, d)| (id, d.into())) + .collect(), + } + } +} + +/// The desired state of an Omicron-managed dataset in a blueprint. +/// +/// Part of [`BlueprintDatasetConfig`]. +#[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + JsonSchema, + Deserialize, + Serialize, + EnumIter, +)] +#[serde(rename_all = "snake_case")] +pub enum BlueprintDatasetDisposition { + /// The dataset is in-service. + InService, + + /// The dataset is permanently gone. + Expunged, +} + +impl BlueprintDatasetDisposition { + pub fn matches(self, filter: BlueprintDatasetFilter) -> bool { + match self { + Self::InService => match filter { + BlueprintDatasetFilter::All => true, + BlueprintDatasetFilter::Expunged => false, + BlueprintDatasetFilter::InService => true, + }, + Self::Expunged => match filter { + BlueprintDatasetFilter::All => true, + BlueprintDatasetFilter::Expunged => true, + BlueprintDatasetFilter::InService => false, + }, + } + } +} + +/// Information about a dataset as recorded in a blueprint +#[derive(Debug, Clone, Eq, PartialEq, JsonSchema, Deserialize, Serialize)] +pub struct BlueprintDatasetConfig { + pub disposition: BlueprintDatasetDisposition, + + pub id: DatasetUuid, + pub pool: ZpoolName, + pub kind: DatasetKind, + pub address: Option, + pub quota: Option, + pub reservation: Option, + pub compression: CompressionAlgorithm, +} + +impl From for DatasetConfig { + fn from(config: BlueprintDatasetConfig) -> Self { + Self { + id: config.id, + name: DatasetName::new(config.pool, config.kind), + quota: config.quota, + reservation: config.reservation, + compression: config.compression, + } + } +} + /// Describe high-level metadata about a blueprint // These fields are a subset of [`Blueprint`], and include only the data we can // quickly fetch from the main blueprint table (e.g., when listing all diff --git a/nexus/types/src/deployment/execution/spec.rs b/nexus/types/src/deployment/execution/spec.rs index 4b64477bf2..2472a47d03 100644 --- a/nexus/types/src/deployment/execution/spec.rs +++ b/nexus/types/src/deployment/execution/spec.rs @@ -33,6 +33,7 @@ pub enum ExecutionComponent { OmicronZones, FirewallRules, DatasetRecords, + Datasets, Dns, Cockroach, Clickhouse, diff --git a/nexus/types/src/deployment/planning_input.rs b/nexus/types/src/deployment/planning_input.rs index ea23465183..57cc99709e 100644 --- a/nexus/types/src/deployment/planning_input.rs +++ b/nexus/types/src/deployment/planning_input.rs @@ -23,6 +23,7 @@ use omicron_common::address::Ipv6Subnet; use omicron_common::address::SLED_PREFIX; use omicron_common::api::external::Generation; use omicron_common::api::internal::shared::SourceNatConfigError; +use omicron_common::disk::DatasetConfig; use omicron_common::disk::DiskIdentity; use omicron_common::policy::SINGLE_NODE_CLICKHOUSE_REDUNDANCY; use omicron_uuid_kinds::OmicronZoneUuid; @@ -604,7 +605,7 @@ pub struct SledResources { /// storage) // NOTE: I'd really like to make this private, to make it harder to // accidentally pick a zpool that is not in-service. - pub zpools: BTreeMap, + pub zpools: BTreeMap)>, /// the IPv6 subnet of this sled on the underlay network /// @@ -616,7 +617,9 @@ pub struct SledResources { impl SledResources { /// Returns if the zpool is provisionable (known, in-service, and active). pub fn zpool_is_provisionable(&self, zpool: &ZpoolUuid) -> bool { - let Some(disk) = self.zpools.get(zpool) else { return false }; + let Some((disk, _datasets)) = self.zpools.get(zpool) else { + return false; + }; disk.provisionable() } @@ -625,7 +628,7 @@ impl SledResources { &self, filter: ZpoolFilter, ) -> impl Iterator + '_ { - self.zpools.iter().filter_map(move |(zpool, disk)| { + self.zpools.iter().filter_map(move |(zpool, (disk, _datasets))| { filter .matches_policy_and_state(disk.policy, disk.state) .then_some(zpool) @@ -636,12 +639,23 @@ impl SledResources { &self, filter: DiskFilter, ) -> impl Iterator + '_ { - self.zpools.iter().filter_map(move |(zpool, disk)| { + self.zpools.iter().filter_map(move |(zpool, (disk, _datasets))| { filter .matches_policy_and_state(disk.policy, disk.state) .then_some((zpool, disk)) }) } + + pub fn all_datasets( + &self, + filter: ZpoolFilter, + ) -> impl Iterator + '_ { + self.zpools.iter().filter_map(move |(zpool, (disk, datasets))| { + filter + .matches_policy_and_state(disk.policy, disk.state) + .then_some((zpool, datasets.as_slice())) + }) + } } /// Filters that apply to sleds. diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 2d1465d2a7..1ec5eeea73 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -1907,6 +1907,13 @@ "description": "Describes a complete set of software and configuration for the system", "type": "object", "properties": { + "blueprint_datasets": { + "description": "A map of sled id -> datasets in use on each sled", + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/BlueprintDatasetsConfig" + } + }, "blueprint_disks": { "description": "A map of sled id -> disks in use on each sled.", "type": "object", @@ -1991,6 +1998,7 @@ } }, "required": [ + "blueprint_datasets", "blueprint_disks", "blueprint_zones", "cockroachdb_fingerprint", @@ -2004,6 +2012,92 @@ "time_created" ] }, + "BlueprintDatasetConfig": { + "description": "Information about a dataset as recorded in a blueprint", + "type": "object", + "properties": { + "address": { + "nullable": true, + "type": "string" + }, + "compression": { + "$ref": "#/components/schemas/CompressionAlgorithm" + }, + "disposition": { + "$ref": "#/components/schemas/BlueprintDatasetDisposition" + }, + "id": { + "$ref": "#/components/schemas/TypedUuidForDatasetKind" + }, + "kind": { + "$ref": "#/components/schemas/DatasetKind" + }, + "pool": { + "$ref": "#/components/schemas/ZpoolName" + }, + "quota": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/ByteCount" + } + ] + }, + "reservation": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/ByteCount" + } + ] + } + }, + "required": [ + "compression", + "disposition", + "id", + "kind", + "pool" + ] + }, + "BlueprintDatasetDisposition": { + "description": "The desired state of an Omicron-managed dataset in a blueprint.\n\nPart of [`BlueprintDatasetConfig`].", + "oneOf": [ + { + "description": "The dataset is in-service.", + "type": "string", + "enum": [ + "in_service" + ] + }, + { + "description": "The dataset is permanently gone.", + "type": "string", + "enum": [ + "expunged" + ] + } + ] + }, + "BlueprintDatasetsConfig": { + "description": "Information about Omicron datasets as recorded in a blueprint.", + "type": "object", + "properties": { + "datasets": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/BlueprintDatasetConfig" + } + }, + "generation": { + "$ref": "#/components/schemas/Generation" + } + }, + "required": [ + "datasets", + "generation" + ] + }, "BlueprintMetadata": { "description": "Describe high-level metadata about a blueprint", "type": "object", @@ -2830,6 +2924,112 @@ } ] }, + "CompressionAlgorithm": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "on" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "off" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "gzip" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "level": { + "$ref": "#/components/schemas/GzipLevel" + }, + "type": { + "type": "string", + "enum": [ + "gzip_n" + ] + } + }, + "required": [ + "level", + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "lz4" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "lzjb" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "zle" + ] + } + }, + "required": [ + "type" + ] + } + ] + }, "CurrentStatus": { "description": "Describes the current status of a background task", "oneOf": [ @@ -3475,6 +3675,11 @@ "format": "uint64", "minimum": 0 }, + "GzipLevel": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, "ImportExportPolicy": { "description": "Define policy relating to the import and export of prefixes from a BGP peer.", "oneOf": [ @@ -5541,6 +5746,10 @@ "SwitchPutResponse": { "type": "object" }, + "TypedUuidForDatasetKind": { + "type": "string", + "format": "uuid" + }, "TypedUuidForDemoSagaKind": { "type": "string", "format": "uuid" diff --git a/schema/crdb/blueprint-dataset/up01.sql b/schema/crdb/blueprint-dataset/up01.sql new file mode 100644 index 0000000000..cfdde5bacd --- /dev/null +++ b/schema/crdb/blueprint-dataset/up01.sql @@ -0,0 +1,4 @@ +ALTER TABLE omicron.public.dataset + ADD COLUMN IF NOT EXISTS quota INT8, + ADD COLUMN IF NOT EXISTS reservation INT8, + ADD COLUMN IF NOT EXISTS compression TEXT diff --git a/schema/crdb/blueprint-dataset/up02.sql b/schema/crdb/blueprint-dataset/up02.sql new file mode 100644 index 0000000000..a1a0dc7cb7 --- /dev/null +++ b/schema/crdb/blueprint-dataset/up02.sql @@ -0,0 +1,4 @@ +CREATE TYPE IF NOT EXISTS omicron.public.bp_dataset_disposition AS ENUM ( + 'in_service', + 'expunged' +) diff --git a/schema/crdb/blueprint-dataset/up03.sql b/schema/crdb/blueprint-dataset/up03.sql new file mode 100644 index 0000000000..2ce95db275 --- /dev/null +++ b/schema/crdb/blueprint-dataset/up03.sql @@ -0,0 +1,9 @@ +-- description of a collection of omicron datasets stored in a blueprint +CREATE TABLE IF NOT EXISTS omicron.public.bp_sled_omicron_datasets ( + -- foreign key into the `blueprint` table + blueprint_id UUID NOT NULL, + sled_id UUID NOT NULL, + generation INT8 NOT NULL, + + PRIMARY KEY (blueprint_id, sled_id) +) diff --git a/schema/crdb/blueprint-dataset/up04.sql b/schema/crdb/blueprint-dataset/up04.sql new file mode 100644 index 0000000000..1d21ff5db1 --- /dev/null +++ b/schema/crdb/blueprint-dataset/up04.sql @@ -0,0 +1,35 @@ +-- description of an omicron dataset specified in a blueprint. +CREATE TABLE IF NOT EXISTS omicron.public.bp_omicron_dataset ( + -- foreign key into the `blueprint` table + blueprint_id UUID NOT NULL, + sled_id UUID NOT NULL, + id UUID NOT NULL, + + -- Dataset disposition + disposition omicron.public.bp_dataset_disposition NOT NULL, + + pool_id UUID NOT NULL, + kind omicron.public.dataset_kind NOT NULL, + -- Only valid if kind = zone + zone_name TEXT, + + -- Only valid if kind = crucible + ip INET, + port INT4 CHECK (port BETWEEN 0 AND 65535), + + quota INT8, + reservation INT8, + compression TEXT NOT NULL, + + CONSTRAINT zone_name_for_zone_kind CHECK ( + (kind != 'zone') OR + (kind = 'zone' AND zone_name IS NOT NULL) + ), + + CONSTRAINT ip_and_port_set_for_crucible CHECK ( + (kind != 'crucible') OR + (kind = 'crucible' AND ip IS NOT NULL and port IS NOT NULL) + ), + + PRIMARY KEY (blueprint_id, id) +) diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index e4c82c77f9..a5a36156b4 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -582,6 +582,10 @@ CREATE TABLE IF NOT EXISTS omicron.public.dataset ( /* Only valid if kind = zone -- the name of this zone */ zone_name TEXT, + quota INT8, + reservation INT8, + compression TEXT, + /* Crucible must make use of 'size_used'; other datasets manage their own storage */ CONSTRAINT size_used_column_set_for_crucible CHECK ( (kind != 'crucible') OR @@ -3600,6 +3604,11 @@ CREATE TYPE IF NOT EXISTS omicron.public.bp_zone_disposition AS ENUM ( 'expunged' ); +CREATE TYPE IF NOT EXISTS omicron.public.bp_dataset_disposition AS ENUM ( + 'in_service', + 'expunged' +); + -- list of all blueprints CREATE TABLE IF NOT EXISTS omicron.public.blueprint ( id UUID PRIMARY KEY, @@ -3701,6 +3710,52 @@ CREATE TABLE IF NOT EXISTS omicron.public.bp_omicron_physical_disk ( PRIMARY KEY (blueprint_id, id) ); +-- description of a collection of omicron datasets stored in a blueprint +CREATE TABLE IF NOT EXISTS omicron.public.bp_sled_omicron_datasets ( + -- foreign key into the `blueprint` table + blueprint_id UUID NOT NULL, + sled_id UUID NOT NULL, + generation INT8 NOT NULL, + + PRIMARY KEY (blueprint_id, sled_id) +); + +-- description of an omicron dataset specified in a blueprint. +CREATE TABLE IF NOT EXISTS omicron.public.bp_omicron_dataset ( + -- foreign key into the `blueprint` table + blueprint_id UUID NOT NULL, + sled_id UUID NOT NULL, + id UUID NOT NULL, + + -- Dataset disposition + disposition omicron.public.bp_dataset_disposition NOT NULL, + + pool_id UUID NOT NULL, + kind omicron.public.dataset_kind NOT NULL, + -- Only valid if kind = zone + zone_name TEXT, + + -- Only valid if kind = crucible + ip INET, + port INT4 CHECK (port BETWEEN 0 AND 65535), + + quota INT8, + reservation INT8, + compression TEXT NOT NULL, + + CONSTRAINT zone_name_for_zone_kind CHECK ( + (kind != 'zone') OR + (kind = 'zone' AND zone_name IS NOT NULL) + ), + + CONSTRAINT ip_and_port_set_for_crucible CHECK ( + (kind != 'crucible') OR + (kind = 'crucible' AND ip IS NOT NULL and port IS NOT NULL) + ), + + PRIMARY KEY (blueprint_id, id) +); + -- see inv_sled_omicron_zones, which is identical except it references a -- collection whereas this table references a blueprint CREATE TABLE IF NOT EXISTS omicron.public.bp_sled_omicron_zones ( @@ -4536,7 +4591,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '111.0.0', NULL) + (TRUE, NOW(), NOW(), '112.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/schema/rss-service-plan-v5.json b/schema/rss-service-plan-v5.json new file mode 100644 index 0000000000..132b27c10e --- /dev/null +++ b/schema/rss-service-plan-v5.json @@ -0,0 +1,1197 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Plan", + "type": "object", + "required": [ + "dns_config", + "services" + ], + "properties": { + "dns_config": { + "$ref": "#/definitions/DnsConfigParams" + }, + "services": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/SledConfig" + } + } + }, + "definitions": { + "BlueprintZoneConfig": { + "description": "Describes one Omicron-managed zone in a blueprint.\n\nPart of [`BlueprintZonesConfig`].", + "type": "object", + "required": [ + "disposition", + "id", + "zone_type" + ], + "properties": { + "disposition": { + "description": "The disposition (desired state) of this zone recorded in the blueprint.", + "allOf": [ + { + "$ref": "#/definitions/BlueprintZoneDisposition" + } + ] + }, + "filesystem_pool": { + "description": "zpool used for the zone's (transient) root filesystem", + "anyOf": [ + { + "$ref": "#/definitions/ZpoolName" + }, + { + "type": "null" + } + ] + }, + "id": { + "$ref": "#/definitions/TypedUuidForOmicronZoneKind" + }, + "zone_type": { + "$ref": "#/definitions/BlueprintZoneType" + } + } + }, + "BlueprintZoneDisposition": { + "description": "The desired state of an Omicron-managed zone in a blueprint.\n\nPart of [`BlueprintZoneConfig`].", + "oneOf": [ + { + "description": "The zone is in-service.", + "type": "string", + "enum": [ + "in_service" + ] + }, + { + "description": "The zone is not in service.", + "type": "string", + "enum": [ + "quiesced" + ] + }, + { + "description": "The zone is permanently gone.", + "type": "string", + "enum": [ + "expunged" + ] + } + ] + }, + "BlueprintZoneType": { + "oneOf": [ + { + "type": "object", + "required": [ + "address", + "dns_servers", + "external_ip", + "nic", + "ntp_servers", + "type" + ], + "properties": { + "address": { + "type": "string" + }, + "dns_servers": { + "type": "array", + "items": { + "type": "string", + "format": "ip" + } + }, + "domain": { + "type": [ + "string", + "null" + ] + }, + "external_ip": { + "$ref": "#/definitions/OmicronZoneExternalSnatIp" + }, + "nic": { + "description": "The service vNIC providing outbound connectivity using OPTE.", + "allOf": [ + { + "$ref": "#/definitions/NetworkInterface" + } + ] + }, + "ntp_servers": { + "type": "array", + "items": { + "type": "string" + } + }, + "type": { + "type": "string", + "enum": [ + "boundary_ntp" + ] + } + } + }, + { + "description": "Used in single-node clickhouse setups", + "type": "object", + "required": [ + "address", + "dataset", + "type" + ], + "properties": { + "address": { + "type": "string" + }, + "dataset": { + "$ref": "#/definitions/OmicronZoneDataset" + }, + "type": { + "type": "string", + "enum": [ + "clickhouse" + ] + } + } + }, + { + "type": "object", + "required": [ + "address", + "dataset", + "type" + ], + "properties": { + "address": { + "type": "string" + }, + "dataset": { + "$ref": "#/definitions/OmicronZoneDataset" + }, + "type": { + "type": "string", + "enum": [ + "clickhouse_keeper" + ] + } + } + }, + { + "description": "Used in replicated clickhouse setups", + "type": "object", + "required": [ + "address", + "dataset", + "type" + ], + "properties": { + "address": { + "type": "string" + }, + "dataset": { + "$ref": "#/definitions/OmicronZoneDataset" + }, + "type": { + "type": "string", + "enum": [ + "clickhouse_server" + ] + } + } + }, + { + "type": "object", + "required": [ + "address", + "dataset", + "type" + ], + "properties": { + "address": { + "type": "string" + }, + "dataset": { + "$ref": "#/definitions/OmicronZoneDataset" + }, + "type": { + "type": "string", + "enum": [ + "cockroach_db" + ] + } + } + }, + { + "type": "object", + "required": [ + "address", + "dataset", + "type" + ], + "properties": { + "address": { + "type": "string" + }, + "dataset": { + "$ref": "#/definitions/OmicronZoneDataset" + }, + "type": { + "type": "string", + "enum": [ + "crucible" + ] + } + } + }, + { + "type": "object", + "required": [ + "address", + "type" + ], + "properties": { + "address": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "crucible_pantry" + ] + } + } + }, + { + "type": "object", + "required": [ + "dataset", + "dns_address", + "http_address", + "nic", + "type" + ], + "properties": { + "dataset": { + "$ref": "#/definitions/OmicronZoneDataset" + }, + "dns_address": { + "description": "The address at which the external DNS server is reachable.", + "allOf": [ + { + "$ref": "#/definitions/OmicronZoneExternalFloatingAddr" + } + ] + }, + "http_address": { + "description": "The address at which the external DNS server API is reachable.", + "type": "string" + }, + "nic": { + "description": "The service vNIC providing external connectivity using OPTE.", + "allOf": [ + { + "$ref": "#/definitions/NetworkInterface" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "external_dns" + ] + } + } + }, + { + "type": "object", + "required": [ + "dataset", + "dns_address", + "gz_address", + "gz_address_index", + "http_address", + "type" + ], + "properties": { + "dataset": { + "$ref": "#/definitions/OmicronZoneDataset" + }, + "dns_address": { + "type": "string" + }, + "gz_address": { + "description": "The addresses in the global zone which should be created\n\nFor the DNS service, which exists outside the sleds's typical subnet - adding an address in the GZ is necessary to allow inter-zone traffic routing.", + "type": "string", + "format": "ipv6" + }, + "gz_address_index": { + "description": "The address is also identified with an auxiliary bit of information to ensure that the created global zone address can have a unique name.", + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "http_address": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "internal_dns" + ] + } + } + }, + { + "type": "object", + "required": [ + "address", + "type" + ], + "properties": { + "address": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "internal_ntp" + ] + } + } + }, + { + "type": "object", + "required": [ + "external_dns_servers", + "external_ip", + "external_tls", + "internal_address", + "nic", + "type" + ], + "properties": { + "external_dns_servers": { + "description": "External DNS servers Nexus can use to resolve external hosts.", + "type": "array", + "items": { + "type": "string", + "format": "ip" + } + }, + "external_ip": { + "description": "The address at which the external nexus server is reachable.", + "allOf": [ + { + "$ref": "#/definitions/OmicronZoneExternalFloatingIp" + } + ] + }, + "external_tls": { + "description": "Whether Nexus's external endpoint should use TLS", + "type": "boolean" + }, + "internal_address": { + "description": "The address at which the internal nexus server is reachable.", + "type": "string" + }, + "nic": { + "description": "The service vNIC providing external connectivity using OPTE.", + "allOf": [ + { + "$ref": "#/definitions/NetworkInterface" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "nexus" + ] + } + } + }, + { + "type": "object", + "required": [ + "address", + "type" + ], + "properties": { + "address": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "oximeter" + ] + } + } + } + ] + }, + "ByteCount": { + "description": "Byte count to express memory or storage capacity.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "CompressionAlgorithm": { + "oneOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "on" + ] + } + } + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "off" + ] + } + } + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "gzip" + ] + } + } + }, + { + "type": "object", + "required": [ + "level", + "type" + ], + "properties": { + "level": { + "$ref": "#/definitions/GzipLevel" + }, + "type": { + "type": "string", + "enum": [ + "gzip_n" + ] + } + } + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "lz4" + ] + } + } + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "lzjb" + ] + } + } + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "zle" + ] + } + } + } + ] + }, + "DatasetConfig": { + "description": "Configuration information necessary to request a single dataset", + "type": "object", + "required": [ + "compression", + "id", + "name" + ], + "properties": { + "compression": { + "description": "The compression mode to be used by the dataset", + "allOf": [ + { + "$ref": "#/definitions/CompressionAlgorithm" + } + ] + }, + "id": { + "description": "The UUID of the dataset being requested", + "allOf": [ + { + "$ref": "#/definitions/TypedUuidForDatasetKind" + } + ] + }, + "name": { + "description": "The dataset's name", + "allOf": [ + { + "$ref": "#/definitions/DatasetName" + } + ] + }, + "quota": { + "description": "The upper bound on the amount of storage used by this dataset", + "anyOf": [ + { + "$ref": "#/definitions/ByteCount" + }, + { + "type": "null" + } + ] + }, + "reservation": { + "description": "The lower bound on the amount of storage usable by this dataset", + "anyOf": [ + { + "$ref": "#/definitions/ByteCount" + }, + { + "type": "null" + } + ] + } + } + }, + "DatasetKind": { + "description": "The kind of dataset. See the `DatasetKind` enum in omicron-common for possible values.", + "type": "string" + }, + "DatasetName": { + "type": "object", + "required": [ + "kind", + "pool_name" + ], + "properties": { + "kind": { + "$ref": "#/definitions/DatasetKind" + }, + "pool_name": { + "$ref": "#/definitions/ZpoolName" + } + } + }, + "DatasetsConfig": { + "type": "object", + "required": [ + "datasets", + "generation" + ], + "properties": { + "datasets": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/DatasetConfig" + } + }, + "generation": { + "description": "generation number of this configuration\n\nThis generation number is owned by the control plane (i.e., RSS or Nexus, depending on whether RSS-to-Nexus handoff has happened). It should not be bumped within Sled Agent.\n\nSled Agent rejects attempts to set the configuration to a generation older than the one it's currently running.\n\nNote that \"Generation::new()\", AKA, the first generation number, is reserved for \"no datasets\". This is the default configuration for a sled before any requests have been made.", + "allOf": [ + { + "$ref": "#/definitions/Generation" + } + ] + } + } + }, + "DiskIdentity": { + "description": "Uniquely identifies a disk.", + "type": "object", + "required": [ + "model", + "serial", + "vendor" + ], + "properties": { + "model": { + "type": "string" + }, + "serial": { + "type": "string" + }, + "vendor": { + "type": "string" + } + } + }, + "DnsConfigParams": { + "type": "object", + "required": [ + "generation", + "time_created", + "zones" + ], + "properties": { + "generation": { + "$ref": "#/definitions/Generation" + }, + "time_created": { + "type": "string", + "format": "date-time" + }, + "zones": { + "type": "array", + "items": { + "$ref": "#/definitions/DnsConfigZone" + } + } + } + }, + "DnsConfigZone": { + "type": "object", + "required": [ + "records", + "zone_name" + ], + "properties": { + "records": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "$ref": "#/definitions/DnsRecord" + } + } + }, + "zone_name": { + "type": "string" + } + } + }, + "DnsRecord": { + "oneOf": [ + { + "type": "object", + "required": [ + "data", + "type" + ], + "properties": { + "data": { + "type": "string", + "format": "ipv4" + }, + "type": { + "type": "string", + "enum": [ + "A" + ] + } + } + }, + { + "type": "object", + "required": [ + "data", + "type" + ], + "properties": { + "data": { + "type": "string", + "format": "ipv6" + }, + "type": { + "type": "string", + "enum": [ + "AAAA" + ] + } + } + }, + { + "type": "object", + "required": [ + "data", + "type" + ], + "properties": { + "data": { + "$ref": "#/definitions/Srv" + }, + "type": { + "type": "string", + "enum": [ + "SRV" + ] + } + } + } + ] + }, + "Generation": { + "description": "Generation numbers stored in the database, used for optimistic concurrency control", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "GzipLevel": { + "type": "integer", + "format": "uint8", + "minimum": 0.0 + }, + "IpNet": { + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/definitions/Ipv4Net" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/definitions/Ipv6Net" + } + ] + } + ], + "x-rust-type": { + "crate": "oxnet", + "path": "oxnet::IpNet", + "version": "0.1.0" + } + }, + "Ipv4Net": { + "title": "An IPv4 subnet", + "description": "An IPv4 subnet, including prefix and prefix length", + "examples": [ + "192.168.1.0/24" + ], + "type": "string", + "pattern": "^(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])/([0-9]|1[0-9]|2[0-9]|3[0-2])$", + "x-rust-type": { + "crate": "oxnet", + "path": "oxnet::Ipv4Net", + "version": "0.1.0" + } + }, + "Ipv6Net": { + "title": "An IPv6 subnet", + "description": "An IPv6 subnet, including prefix and subnet mask", + "examples": [ + "fd12:3456::/64" + ], + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])$", + "x-rust-type": { + "crate": "oxnet", + "path": "oxnet::Ipv6Net", + "version": "0.1.0" + } + }, + "MacAddr": { + "title": "A MAC address", + "description": "A Media Access Control address, in EUI-48 format", + "examples": [ + "ff:ff:ff:ff:ff:ff" + ], + "type": "string", + "maxLength": 17, + "minLength": 5, + "pattern": "^([0-9a-fA-F]{0,2}:){5}[0-9a-fA-F]{0,2}$" + }, + "Name": { + "title": "A name unique within the parent collection", + "description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID, but they may contain a UUID. They can be at most 63 characters long.", + "type": "string", + "maxLength": 63, + "minLength": 1, + "pattern": "^(?![0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$)^[a-z]([a-zA-Z0-9-]*[a-zA-Z0-9]+)?$" + }, + "NetworkInterface": { + "description": "Information required to construct a virtual network interface", + "type": "object", + "required": [ + "id", + "ip", + "kind", + "mac", + "name", + "primary", + "slot", + "subnet", + "vni" + ], + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "ip": { + "type": "string", + "format": "ip" + }, + "kind": { + "$ref": "#/definitions/NetworkInterfaceKind" + }, + "mac": { + "$ref": "#/definitions/MacAddr" + }, + "name": { + "$ref": "#/definitions/Name" + }, + "primary": { + "type": "boolean" + }, + "slot": { + "type": "integer", + "format": "uint8", + "minimum": 0.0 + }, + "subnet": { + "$ref": "#/definitions/IpNet" + }, + "transit_ips": { + "default": [], + "type": "array", + "items": { + "$ref": "#/definitions/IpNet" + } + }, + "vni": { + "$ref": "#/definitions/Vni" + } + } + }, + "NetworkInterfaceKind": { + "description": "The type of network interface", + "oneOf": [ + { + "description": "A vNIC attached to a guest instance", + "type": "object", + "required": [ + "id", + "type" + ], + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "type": { + "type": "string", + "enum": [ + "instance" + ] + } + } + }, + { + "description": "A vNIC associated with an internal service", + "type": "object", + "required": [ + "id", + "type" + ], + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "type": { + "type": "string", + "enum": [ + "service" + ] + } + } + }, + { + "description": "A vNIC associated with a probe", + "type": "object", + "required": [ + "id", + "type" + ], + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "type": { + "type": "string", + "enum": [ + "probe" + ] + } + } + } + ] + }, + "OmicronPhysicalDiskConfig": { + "type": "object", + "required": [ + "id", + "identity", + "pool_id" + ], + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "identity": { + "$ref": "#/definitions/DiskIdentity" + }, + "pool_id": { + "$ref": "#/definitions/TypedUuidForZpoolKind" + } + } + }, + "OmicronPhysicalDisksConfig": { + "type": "object", + "required": [ + "disks", + "generation" + ], + "properties": { + "disks": { + "type": "array", + "items": { + "$ref": "#/definitions/OmicronPhysicalDiskConfig" + } + }, + "generation": { + "description": "generation number of this configuration\n\nThis generation number is owned by the control plane (i.e., RSS or Nexus, depending on whether RSS-to-Nexus handoff has happened). It should not be bumped within Sled Agent.\n\nSled Agent rejects attempts to set the configuration to a generation older than the one it's currently running.", + "allOf": [ + { + "$ref": "#/definitions/Generation" + } + ] + } + } + }, + "OmicronZoneDataset": { + "description": "Describes a persistent ZFS dataset associated with an Omicron zone", + "type": "object", + "required": [ + "pool_name" + ], + "properties": { + "pool_name": { + "$ref": "#/definitions/ZpoolName" + } + } + }, + "OmicronZoneExternalFloatingAddr": { + "description": "Floating external address with port allocated to an Omicron-managed zone.", + "type": "object", + "required": [ + "addr", + "id" + ], + "properties": { + "addr": { + "type": "string" + }, + "id": { + "$ref": "#/definitions/TypedUuidForExternalIpKind" + } + } + }, + "OmicronZoneExternalFloatingIp": { + "description": "Floating external IP allocated to an Omicron-managed zone.\n\nThis is a slimmer `nexus_db_model::ExternalIp` that only stores the fields necessary for blueprint planning, and requires that the zone have a single IP.", + "type": "object", + "required": [ + "id", + "ip" + ], + "properties": { + "id": { + "$ref": "#/definitions/TypedUuidForExternalIpKind" + }, + "ip": { + "type": "string", + "format": "ip" + } + } + }, + "OmicronZoneExternalSnatIp": { + "description": "SNAT (outbound) external IP allocated to an Omicron-managed zone.\n\nThis is a slimmer `nexus_db_model::ExternalIp` that only stores the fields necessary for blueprint planning, and requires that the zone have a single IP.", + "type": "object", + "required": [ + "id", + "snat_cfg" + ], + "properties": { + "id": { + "$ref": "#/definitions/TypedUuidForExternalIpKind" + }, + "snat_cfg": { + "$ref": "#/definitions/SourceNatConfig" + } + } + }, + "SledConfig": { + "type": "object", + "required": [ + "datasets", + "disks", + "zones" + ], + "properties": { + "datasets": { + "description": "Datasets configured for this sled", + "allOf": [ + { + "$ref": "#/definitions/DatasetsConfig" + } + ] + }, + "disks": { + "description": "Control plane disks configured for this sled", + "allOf": [ + { + "$ref": "#/definitions/OmicronPhysicalDisksConfig" + } + ] + }, + "zones": { + "description": "zones configured for this sled", + "type": "array", + "items": { + "$ref": "#/definitions/BlueprintZoneConfig" + } + } + } + }, + "SourceNatConfig": { + "description": "An IP address and port range used for source NAT, i.e., making outbound network connections from guests or services.", + "type": "object", + "required": [ + "first_port", + "ip", + "last_port" + ], + "properties": { + "first_port": { + "description": "The first port used for source NAT, inclusive.", + "type": "integer", + "format": "uint16", + "minimum": 0.0 + }, + "ip": { + "description": "The external address provided to the instance or service.", + "type": "string", + "format": "ip" + }, + "last_port": { + "description": "The last port used for source NAT, also inclusive.", + "type": "integer", + "format": "uint16", + "minimum": 0.0 + } + } + }, + "Srv": { + "type": "object", + "required": [ + "port", + "prio", + "target", + "weight" + ], + "properties": { + "port": { + "type": "integer", + "format": "uint16", + "minimum": 0.0 + }, + "prio": { + "type": "integer", + "format": "uint16", + "minimum": 0.0 + }, + "target": { + "type": "string" + }, + "weight": { + "type": "integer", + "format": "uint16", + "minimum": 0.0 + } + } + }, + "TypedUuidForDatasetKind": { + "type": "string", + "format": "uuid" + }, + "TypedUuidForExternalIpKind": { + "type": "string", + "format": "uuid" + }, + "TypedUuidForOmicronZoneKind": { + "type": "string", + "format": "uuid" + }, + "TypedUuidForZpoolKind": { + "type": "string", + "format": "uuid" + }, + "Vni": { + "description": "A Geneve Virtual Network Identifier", + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "ZpoolName": { + "title": "The name of a Zpool", + "description": "Zpool names are of the format ox{i,p}_. They are either Internal or External, and should be unique", + "type": "string", + "pattern": "^ox[ip]_[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" + } + } +} \ No newline at end of file diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index b47aeb6508..f0c540f8e1 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -32,7 +32,9 @@ use omicron_common::backoff; use omicron_common::backoff::BackoffError; use omicron_common::zpool_name::ZpoolName; use omicron_common::NoDebug; -use omicron_uuid_kinds::{GenericUuid, InstanceUuid, PropolisUuid}; +use omicron_uuid_kinds::{ + GenericUuid, InstanceUuid, OmicronZoneUuid, PropolisUuid, +}; use propolis_api_types::ErrorCode as PropolisErrorCode; use propolis_client::Client as PropolisClient; use rand::prelude::IteratorRandom; @@ -1661,7 +1663,9 @@ impl InstanceRunner { .with_zone_root_path(root) .with_zone_image_paths(&["/opt/oxide".into()]) .with_zone_type("propolis-server") - .with_unique_name(self.propolis_id.into_untyped_uuid()) + .with_unique_name(OmicronZoneUuid::from_untyped_uuid( + self.propolis_id.into_untyped_uuid(), + )) .with_datasets(&[]) .with_filesystems(&[]) .with_data_links(&[]) diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index f9c0f117ba..de0b086752 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -4,7 +4,6 @@ use nexus_sled_agent_shared::inventory::{OmicronZoneConfig, OmicronZoneType}; use omicron_common::disk::{DatasetKind, DatasetName}; -use omicron_uuid_kinds::GenericUuid; pub use sled_hardware::DendriteAsic; use std::net::SocketAddrV6; @@ -20,7 +19,7 @@ impl OmicronZoneConfigExt for OmicronZoneConfig { fn zone_name(&self) -> String { illumos_utils::running_zone::InstalledZone::get_zone_name( self.zone_type.kind().zone_prefix(), - Some(self.id.into_untyped_uuid()), + Some(self.id), ) } } diff --git a/sled-agent/src/probe_manager.rs b/sled-agent/src/probe_manager.rs index 42186f66e9..fb1399a9c2 100644 --- a/sled-agent/src/probe_manager.rs +++ b/sled-agent/src/probe_manager.rs @@ -16,6 +16,7 @@ use omicron_common::api::external::{ use omicron_common::api::internal::shared::{ NetworkInterface, ResolvedVpcFirewallRule, }; +use omicron_uuid_kinds::{GenericUuid, OmicronZoneUuid}; use rand::prelude::IteratorRandom; use rand::SeedableRng; use sled_storage::dataset::ZONE_DATASET; @@ -330,7 +331,7 @@ impl ProbeManagerInner { .with_zone_root_path(zone_root_path) .with_zone_image_paths(&["/opt/oxide".into()]) .with_zone_type("probe") - .with_unique_name(probe.id) + .with_unique_name(OmicronZoneUuid::from_untyped_uuid(probe.id)) .with_datasets(&[]) .with_filesystems(&[]) .with_data_links(&[]) diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index a963375381..478bbcbc01 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -34,8 +34,8 @@ use omicron_common::backoff::{ retry_notify_ext, retry_policy_internal_service_aggressive, BackoffError, }; use omicron_common::disk::{ - DatasetKind, DatasetName, DiskVariant, OmicronPhysicalDiskConfig, - OmicronPhysicalDisksConfig, + DatasetKind, DatasetName, DatasetsConfig, DiskVariant, + OmicronPhysicalDiskConfig, OmicronPhysicalDisksConfig, }; use omicron_common::ledger::{self, Ledger, Ledgerable}; use omicron_common::policy::{ @@ -99,6 +99,12 @@ pub enum PlanError { #[error("Found only v2 service plan")] FoundV2, + + #[error("Found only v3 service plan")] + FoundV3, + + #[error("Found only v4 service plan")] + FoundV4, } #[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema)] @@ -106,6 +112,9 @@ pub struct SledConfig { /// Control plane disks configured for this sled pub disks: BlueprintPhysicalDisksConfig, + /// Datasets configured for this sled + pub datasets: DatasetsConfig, + /// zones configured for this sled pub zones: Vec, } @@ -125,7 +134,8 @@ impl Ledgerable for Plan { const RSS_SERVICE_PLAN_V1_FILENAME: &str = "rss-service-plan.json"; const RSS_SERVICE_PLAN_V2_FILENAME: &str = "rss-service-plan-v2.json"; const RSS_SERVICE_PLAN_V3_FILENAME: &str = "rss-service-plan-v3.json"; -const RSS_SERVICE_PLAN_FILENAME: &str = "rss-service-plan-v4.json"; +const RSS_SERVICE_PLAN_V4_FILENAME: &str = "rss-service-plan-v4.json"; +const RSS_SERVICE_PLAN_FILENAME: &str = "rss-service-plan-v5.json"; pub fn from_sockaddr_to_external_floating_addr( addr: SocketAddr, @@ -237,7 +247,15 @@ impl Plan { err, } })? { - Err(PlanError::FoundV2) + Err(PlanError::FoundV3) + } else if Self::has_v4(storage_manager).await.map_err(|err| { + // Same as the comment above, but for version 4. + PlanError::Io { + message: String::from("looking for v4 RSS plan"), + err, + } + })? { + Err(PlanError::FoundV4) } else { Ok(None) } @@ -300,6 +318,25 @@ impl Plan { Ok(false) } + async fn has_v4( + storage_manager: &StorageHandle, + ) -> Result { + let paths = storage_manager + .get_latest_disks() + .await + .all_m2_mountpoints(CONFIG_DATASET) + .into_iter() + .map(|p| p.join(RSS_SERVICE_PLAN_V4_FILENAME)); + + for p in paths { + if p.try_exists()? { + return Ok(true); + } + } + + Ok(false) + } + async fn is_sled_scrimlet( log: &Logger, address: SocketAddrV6, @@ -1344,10 +1381,10 @@ mod tests { } #[test] - fn test_rss_service_plan_v4_schema() { + fn test_rss_service_plan_v5_schema() { let schema = schemars::schema_for!(Plan); expectorate::assert_contents( - "../schema/rss-service-plan-v4.json", + "../schema/rss-service-plan-v5.json", &serde_json::to_string_pretty(&schema).unwrap(), ); } diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index a5ba8d9d7f..c1e29b0e0f 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -17,7 +17,7 @@ //! state files that get generated as RSS executes: //! //! - /pool/int/UUID/config/rss-sled-plan.json (Sled Plan) -//! - /pool/int/UUID/config/rss-service-plan-v3.json (Service Plan) +//! - /pool/int/UUID/config/rss-service-plan-v5.json (Service Plan) //! - /pool/int/UUID/config/rss-plan-completed.marker (Plan Execution Complete) //! //! These phases are described below. As each phase completes, a corresponding @@ -90,8 +90,9 @@ use nexus_sled_agent_shared::inventory::{ OmicronZoneConfig, OmicronZoneType, OmicronZonesConfig, }; use nexus_types::deployment::{ - blueprint_zone_type, Blueprint, BlueprintZoneType, BlueprintZonesConfig, - CockroachDbPreserveDowngrade, + blueprint_zone_type, Blueprint, BlueprintDatasetConfig, + BlueprintDatasetDisposition, BlueprintDatasetsConfig, BlueprintZoneType, + BlueprintZonesConfig, CockroachDbPreserveDowngrade, }; use nexus_types::external_api::views::SledState; use omicron_common::address::get_sled_address; @@ -1442,6 +1443,47 @@ pub(crate) fn build_initial_blueprint_from_sled_configs( .map(|(sled_id, sled_config)| (*sled_id, sled_config.disks.clone())) .collect(); + let mut blueprint_datasets = BTreeMap::new(); + for (sled_id, sled_config) in sled_configs_by_id { + let mut datasets = BTreeMap::new(); + for d in sled_config.datasets.datasets.values() { + // Only the "Crucible" dataset needs to know the address + let address = sled_config.zones.iter().find_map(|z| { + if let BlueprintZoneType::Crucible( + blueprint_zone_type::Crucible { address, dataset }, + ) = &z.zone_type + { + if &dataset.pool_name == d.name.pool() { + return Some(*address); + } + }; + None + }); + + datasets.insert( + d.id, + BlueprintDatasetConfig { + disposition: BlueprintDatasetDisposition::InService, + id: d.id, + pool: d.name.pool().clone(), + kind: d.name.dataset().clone(), + address, + compression: d.compression, + quota: d.quota, + reservation: d.reservation, + }, + ); + } + + blueprint_datasets.insert( + *sled_id, + BlueprintDatasetsConfig { + generation: sled_config.datasets.generation, + datasets, + }, + ); + } + let mut blueprint_zones = BTreeMap::new(); let mut sled_state = BTreeMap::new(); for (sled_id, sled_config) in sled_configs_by_id { @@ -1468,6 +1510,7 @@ pub(crate) fn build_initial_blueprint_from_sled_configs( id: Uuid::new_v4(), blueprint_zones, blueprint_disks, + blueprint_datasets, sled_state, parent_blueprint_id: None, internal_dns_version, diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 0243060546..7c4fb55151 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -89,7 +89,6 @@ use omicron_common::backoff::{ use omicron_common::disk::{DatasetKind, DatasetName}; use omicron_common::ledger::{self, Ledger, Ledgerable}; use omicron_ddm_admin_client::{Client as DdmAdminClient, DdmError}; -use omicron_uuid_kinds::GenericUuid; use once_cell::sync::OnceCell; use rand::prelude::SliceRandom; use sled_agent_types::{ @@ -1516,8 +1515,7 @@ impl ServiceManager { Some(dir) => ZoneBuilderFactory::fake(Some(dir)).builder(), }; if let Some(uuid) = unique_name { - zone_builder = - zone_builder.with_unique_name(uuid.into_untyped_uuid()); + zone_builder = zone_builder.with_unique_name(uuid); } if let Some(vnic) = bootstrap_vnic { zone_builder = zone_builder.with_bootstrap_vnic(vnic); diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index ca7f5e3410..af9b016370 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -73,7 +73,6 @@ pub fn api() -> SledApiDescription { api.register(instance_poke_single_step_post)?; api.register(instance_post_sim_migration_source)?; api.register(disk_poke_post)?; - Ok(api) } diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs index 7336182744..6301228efe 100644 --- a/sled-agent/src/sim/server.rs +++ b/sled-agent/src/sim/server.rs @@ -534,9 +534,15 @@ pub async fn run_standalone_server( None => vec![], }; - let disks = server.sled_agent.omicron_physical_disks_list().await?; let mut sled_configs = BTreeMap::new(); - sled_configs.insert(config.id, SledConfig { disks, zones }); + sled_configs.insert( + config.id, + SledConfig { + disks: server.sled_agent.omicron_physical_disks_list().await?, + datasets: server.sled_agent.datasets_config_list().await?, + zones, + }, + ); let rack_init_request = NexusTypes::RackInitializationRequest { blueprint: build_initial_blueprint_from_sled_configs( diff --git a/uuid-kinds/src/lib.rs b/uuid-kinds/src/lib.rs index ba586c03a5..7947062a82 100644 --- a/uuid-kinds/src/lib.rs +++ b/uuid-kinds/src/lib.rs @@ -49,6 +49,7 @@ macro_rules! impl_typed_uuid_kind { // Please keep this list in alphabetical order. impl_typed_uuid_kind! { + Blueprint => "blueprint", Collection => "collection", Dataset => "dataset", DemoSaga => "demo_saga",