Extract storage manager related code from sled-agent

Cargo.toml

@@ -57,6 +57,7 @@ members = [
     "rpaths",
     "sled-agent",
     "sled-hardware",
+    "sled-storage",
     "sp-sim",
     "test-utils",
     "tufaceous-lib",
@@ -122,6 +123,7 @@ default-members = [
     "rpaths",
     "sled-agent",
     "sled-hardware",
+    "sled-storage",
     "sp-sim",
     "test-utils",
     "tufaceous-lib",
@@ -329,6 +331,7 @@ similar-asserts = "1.5.0"
 sled = "0.34"
 sled-agent-client = { path = "clients/sled-agent-client" }
 sled-hardware = { path = "sled-hardware" }
+sled-storage = { path = "sled-storage" }
 slog = { version = "2.7", features = [ "dynamic-keys", "max_level_trace", "release_max_level_debug" ] }
 slog-async = "2.8"
 slog-dtrace = "0.2"

clients/nexus-client/Cargo.toml

@@ -10,6 +10,8 @@ futures.workspace = true
 ipnetwork.workspace = true
 omicron-common.workspace = true
 omicron-passwords.workspace = true
+sled-hardware.workspace = true
+sled-storage.workspace = true
 progenitor.workspace = true
 regress.workspace = true
 reqwest = { workspace = true, features = ["rustls-tls", "stream"] }

clients/nexus-client/src/lib.rs

@@ -388,3 +388,36 @@ impl From<omicron_common::api::internal::shared::ExternalPortDiscovery>
         }
     }
 }
+
+impl From<sled_hardware::DiskVariant> for types::PhysicalDiskKind {
+    fn from(value: sled_hardware::DiskVariant) -> Self {
+        match value {
+            sled_hardware::DiskVariant::U2 => types::PhysicalDiskKind::U2,
+            sled_hardware::DiskVariant::M2 => types::PhysicalDiskKind::M2,
+        }
+    }
+}
+
+impl From<sled_hardware::Baseboard> for types::Baseboard {
+    fn from(b: sled_hardware::Baseboard) -> types::Baseboard {
+        types::Baseboard {
+            serial_number: b.identifier().to_string(),
+            part_number: b.model().to_string(),
+            revision: b.revision(),
+        }
+    }
+}
+
+impl From<sled_storage::dataset::DatasetKind> for types::DatasetKind {
+    fn from(k: sled_storage::dataset::DatasetKind) -> Self {
+        use sled_storage::dataset::DatasetKind::*;
+        match k {
+            CockroachDb => Self::Cockroach,
+            Crucible => Self::Crucible,
+            Clickhouse => Self::Clickhouse,
+            ClickhouseKeeper => Self::ClickhouseKeeper,
+            ExternalDns => Self::ExternalDns,
+            InternalDns => Self::InternalDns,
+        }
+    }
+}

clients/sled-agent-client/Cargo.toml

@@ -14,5 +14,6 @@ regress.workspace = true
 reqwest = { workspace = true, features = [ "json", "rustls-tls", "stream" ] }
 serde.workspace = true
 slog.workspace = true
+sled-storage.workspace = true
 uuid.workspace = true
 omicron-workspace-hack.workspace = true

clients/sled-agent-client/src/lib.rs

@@ -6,6 +6,7 @@
 
 use async_trait::async_trait;
 use std::convert::TryFrom;
+use std::str::FromStr;
 use uuid::Uuid;
 
 progenitor::generate_api!(
@@ -528,3 +529,27 @@ impl TestInterfaces for Client {
             .expect("disk_finish_transition() failed unexpectedly");
     }
 }
+
+impl From<sled_storage::dataset::DatasetKind> for types::DatasetKind {
+    fn from(k: sled_storage::dataset::DatasetKind) -> Self {
+        use sled_storage::dataset::DatasetKind::*;
+        match k {
+            CockroachDb => Self::CockroachDb,
+            Crucible => Self::Crucible,
+            Clickhouse => Self::Clickhouse,
+            ClickhouseKeeper => Self::ClickhouseKeeper,
+            ExternalDns => Self::ExternalDns,
+            InternalDns => Self::InternalDns,
+        }
+    }
+}
+
+impl From<sled_storage::dataset::DatasetName> for types::DatasetName {
+    fn from(n: sled_storage::dataset::DatasetName) -> Self {
+        Self {
+            pool_name: types::ZpoolName::from_str(&n.pool().to_string())
+                .unwrap(),
+            kind: n.dataset().clone().into(),
+        }
+    }
+}

common/src/disk.rs

@@ -5,7 +5,7 @@
 //! Disk related types shared among crates
 
 /// Uniquely identifies a disk.
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Ord, PartialOrd)]
 pub struct DiskIdentity {
     pub vendor: String,
     pub serial: String,

illumos-utils/Cargo.toml

@@ -44,3 +44,6 @@ toml.workspace = true
 [features]
 # Enable to generate MockZones
 testing = ["mockall"]
+# Useful for tests that want real functionality and ability to run without
+# pfexec
+tmp_keypath = []

illumos-utils/src/lib.rs

@@ -4,6 +4,9 @@
 
 //! Wrappers around illumos-specific commands.
 
+#[allow(unused)]
+use std::sync::atomic::{AtomicBool, Ordering};
+
 use cfg_if::cfg_if;
 
 pub mod addrobj;
@@ -93,7 +96,7 @@ mod inner {
 
     // Helper function for starting the process and checking the
     // exit code result.
-    pub fn execute(
+    pub fn execute_helper(
         command: &mut std::process::Command,
     ) -> Result<std::process::Output, ExecutionError> {
         let output = command.output().map_err(|err| {
@@ -108,6 +111,34 @@ mod inner {
     }
 }
 
+// Due to feature unification, the `testing` feature is enabled when some tests
+// don't actually want to use it. We allow them to opt out of the use of the
+// free function here. We also explicitly opt-in where mocks are used.
+//
+// Note that this only works if the tests that use mocks and those that  don't
+// are run sequentially. However, this is how we do things in CI with nextest,
+// so there is no problem currently.
+//
+// We can remove all this when we get rid of the mocks.
+#[cfg(any(test, feature = "testing"))]
+pub static USE_MOCKS: AtomicBool = AtomicBool::new(false);
+
+pub fn execute(
+    command: &mut std::process::Command,
+) -> Result<std::process::Output, ExecutionError> {
+    cfg_if! {
+        if #[cfg(any(test, feature = "testing"))] {
+            if USE_MOCKS.load(Ordering::SeqCst) {
+                mock_inner::execute_helper(command)
+            } else {
+                inner::execute_helper(command)
+            }
+        } else {
+            inner::execute_helper(command)
+        }
+    }
+}
+
 cfg_if! {
     if #[cfg(any(test, feature = "testing"))] {
         pub use mock_inner::*;

illumos-utils/src/zfs.rs

@@ -20,7 +20,16 @@ pub const ZONE_ZFS_RAMDISK_DATASET_MOUNTPOINT: &str = "/zone";
 pub const ZONE_ZFS_RAMDISK_DATASET: &str = "rpool/zone";
 
 pub const ZFS: &str = "/usr/sbin/zfs";
+
+/// This path is intentionally on a `tmpfs` to prevent copy-on-write behavior
+/// and to ensure it goes away on power off.
+///
+/// We want minimize the time the key files are in memory, and so we rederive
+/// the keys and recreate the files on demand when creating and mounting
+/// encrypted filesystems. We then zero them and unlink them.
 pub const KEYPATH_ROOT: &str = "/var/run/oxide/";
+// Use /tmp so we don't have to worry about running tests with pfexec
+pub const TEST_KEYPATH_ROOT: &str = "/tmp";
 
 /// Error returned by [`Zfs::list_datasets`].
 #[derive(thiserror::Error, Debug)]
@@ -158,19 +167,27 @@ impl fmt::Display for Keypath {
     }
 }
 
+#[cfg(not(feature = "tmp_keypath"))]
+impl From<&DiskIdentity> for Keypath {
+    fn from(id: &DiskIdentity) -> Self {
+        build_keypath(id, KEYPATH_ROOT)
+    }
+}
+
+#[cfg(feature = "tmp_keypath")]
 impl From<&DiskIdentity> for Keypath {
     fn from(id: &DiskIdentity) -> Self {
-        let filename = format!(
-            "{}-{}-{}-zfs-aes-256-gcm.key",
-            id.vendor, id.serial, id.model
-        );
-        let mut path = Utf8PathBuf::new();
-        path.push(KEYPATH_ROOT);
-        path.push(filename);
-        Keypath(path)
+        build_keypath(id, TEST_KEYPATH_ROOT)
     }
 }
 
+fn build_keypath(id: &DiskIdentity, root: &str) -> Keypath {
+    let filename =
+        format!("{}-{}-{}-zfs-aes-256-gcm.key", id.vendor, id.serial, id.model);
+    let path: Utf8PathBuf = [root, &filename].iter().collect();
+    Keypath(path)
+}
+
 #[derive(Debug)]
 pub struct EncryptionDetails {
     pub keypath: Keypath,

illumos-utils/src/zpool.rs

@@ -39,6 +39,13 @@ pub struct CreateError {
     err: Error,
 }
 
+#[derive(thiserror::Error, Debug)]
+#[error("Failed to destroy zpool: {err}")]
+pub struct DestroyError {
+    #[from]
+    err: Error,
+}
+
 #[derive(thiserror::Error, Debug)]
 #[error("Failed to list zpools: {err}")]
 pub struct ListError {
@@ -89,7 +96,7 @@ impl FromStr for ZpoolHealth {
 }
 
 /// Describes a Zpool.
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ZpoolInfo {
     name: String,
     size: u64,
@@ -121,6 +128,17 @@ impl ZpoolInfo {
     pub fn health(&self) -> ZpoolHealth {
         self.health
     }
+
+    #[cfg(any(test, feature = "testing"))]
+    pub fn new_hardcoded(name: String) -> ZpoolInfo {
+        ZpoolInfo {
+            name,
+            size: 1024 * 1024 * 64,
+            allocated: 1024,
+            free: 1024 * 1023 * 64,
+            health: ZpoolHealth::Online,
+        }
+    }
 }
 
 impl FromStr for ZpoolInfo {
@@ -167,7 +185,10 @@ pub struct Zpool {}
 
 #[cfg_attr(any(test, feature = "testing"), mockall::automock, allow(dead_code))]
 impl Zpool {
-    pub fn create(name: ZpoolName, vdev: &Utf8Path) -> Result<(), CreateError> {
+    pub fn create(
+        name: &ZpoolName,
+        vdev: &Utf8Path,
+    ) -> Result<(), CreateError> {
         let mut cmd = std::process::Command::new(PFEXEC);
         cmd.env_clear();
         cmd.env("LC_ALL", "C.UTF-8");
@@ -189,7 +210,17 @@ impl Zpool {
         Ok(())
     }
 
-    pub fn import(name: ZpoolName) -> Result<(), Error> {
+    pub fn destroy(name: &ZpoolName) -> Result<(), DestroyError> {
+        let mut cmd = std::process::Command::new(PFEXEC);
+        cmd.env_clear();
+        cmd.env("LC_ALL", "C.UTF-8");
+        cmd.arg(ZPOOL).arg("destroy");
+        cmd.arg(&name.to_string());
+        execute(&mut cmd).map_err(Error::from)?;
+        Ok(())
+    }
+
+    pub fn import(name: &ZpoolName) -> Result<(), Error> {
         let mut cmd = std::process::Command::new(PFEXEC);
         cmd.env_clear();
         cmd.env("LC_ALL", "C.UTF-8");

installinator/Cargo.toml

@@ -27,6 +27,7 @@ omicron-common.workspace = true
 reqwest.workspace = true
 sha2.workspace = true
 sled-hardware.workspace = true
+sled-storage.workspace = true
 slog.workspace = true
 slog-async.workspace = true
 slog-envlogger.workspace = true