From a477ac21754df95753cdfcdc9ed9e1a73b131aba Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 2 May 2023 18:52:27 -0400 Subject: [PATCH] [sled-agent] Store service configuration data in duplicate in M.2s (#2972) - Creates a `Ledger` structure which makes it easy to write toml-serializable data to and from M.2s. - Uses this `Ledger` structure to store all service configuration information in duplicate on the M.2s. Fixes https://github.com/oxidecomputer/omicron/issues/2969 --- sled-agent/src/ledger.rs | 376 ++++++++++++++++++++++++++++++ sled-agent/src/lib.rs | 1 + sled-agent/src/services.rs | 241 ++++++++++--------- sled-agent/src/storage_manager.rs | 9 + 4 files changed, 513 insertions(+), 114 deletions(-) create mode 100644 sled-agent/src/ledger.rs diff --git a/sled-agent/src/ledger.rs b/sled-agent/src/ledger.rs new file mode 100644 index 0000000000..197c7607e4 --- /dev/null +++ b/sled-agent/src/ledger.rs @@ -0,0 +1,376 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Utilities to help reading/writing toml files from/to multiple paths + +use async_trait::async_trait; +use serde::{de::DeserializeOwned, Serialize}; +use slog::Logger; +use std::path::{Path, PathBuf}; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("Cannot serialize TOML to file {path}: {err}")] + TomlSerialize { path: PathBuf, err: toml::ser::Error }, + + #[error("Cannot deserialize TOML from file {path}: {err}")] + TomlDeserialize { path: PathBuf, err: toml::de::Error }, + + #[error("Failed to perform I/O: {message}: {err}")] + Io { + message: String, + #[source] + err: std::io::Error, + }, + + #[error("Failed to write the ledger to storage")] + FailedToAccessStorage, +} + +impl Error { + fn io_path(path: &Path, err: std::io::Error) -> Self { + Self::Io { message: format!("Error accessing {}", path.display()), err } + } +} + +impl From for omicron_common::api::external::Error { + fn from(err: Error) -> Self { + omicron_common::api::external::Error::InternalError { + internal_message: err.to_string(), + } + } +} + +// TODO: .json EXPECTORATE test? +// +// ... yes, but maybe not here? Seems like we gotta know the type of "T" to pull +// this off. + +/// Manage the serialization and deserialization of a ledger of information. +/// +/// This structure is intended to help with serialization and deserialization +/// of configuration information to both M.2s. +pub struct Ledger { + log: Logger, + ledger: T, + paths: Vec, +} + +impl Ledger { + /// Reads the ledger from any of the provided `paths`. + /// + /// Returns the following, in order: + /// - The ledger with the highest generation number + /// - If none exists, returns a default ledger + pub async fn new(log: &Logger, paths: Vec) -> Result { + // Read all the ledgers that we can. + let mut ledgers = vec![]; + for path in paths.iter() { + if let Ok(ledger) = T::read_from(log, &path).await { + ledgers.push(ledger); + } + } + + // Return the ledger with the highest generation number. + let ledger = ledgers.into_iter().reduce(|prior, ledger| { + if ledger.is_newer_than(&prior) { + ledger + } else { + prior + } + }); + + // If we can't read either ledger, start a new one. + let ledger = ledger.unwrap_or_else(|| T::default()); + + Ok(Self { log: log.clone(), ledger, paths }) + } + + pub fn data(&self) -> &T { + &self.ledger + } + + pub fn data_mut(&mut self) -> &mut T { + &mut self.ledger + } + + /// Writes the ledger back to all config directories. + /// + /// Succeeds if at least one of the writes succeeds. + pub async fn commit(&mut self) -> Result<(), Error> { + // Bump the generation number any time we want to commit the ledger. + self.ledger.generation_bump(); + + let mut one_successful_write = false; + for path in self.paths.iter() { + if let Err(e) = self.atomic_write(&path).await { + warn!(self.log, "Failed to write to {}: {e}", path.display()); + } else { + one_successful_write = true; + } + } + + if !one_successful_write { + return Err(Error::FailedToAccessStorage); + } + Ok(()) + } + + // Atomically serialize and write the ledger to storage. + // + // We accomplish this by first writing to a temporary file, then + // renaming to the target location. + async fn atomic_write(&self, path: &Path) -> Result<(), Error> { + let mut tmp_path = path.to_path_buf(); + let tmp_filename = format!( + ".{}.tmp", + tmp_path + .file_name() + .expect("Should have file name") + .to_string_lossy() + ); + tmp_path.set_file_name(tmp_filename); + + self.ledger.write_to(&self.log, &tmp_path).await?; + + tokio::fs::rename(&tmp_path, &path) + .await + .map_err(|err| Error::io_path(&path, err))?; + + Ok(()) + } +} + +#[async_trait] +pub trait Ledgerable: + Default + DeserializeOwned + Serialize + Send + Sync +{ + /// Returns true if [Self] is newer than `other`. + fn is_newer_than(&self, other: &Self) -> bool; + + /// Increments the gneration number. + fn generation_bump(&mut self); + + /// Reads from `path` as a toml-serialized version of `Self`. + async fn read_from(log: &Logger, path: &Path) -> Result { + if path.exists() { + debug!(log, "Reading ledger from {}", path.display()); + toml::from_str( + &tokio::fs::read_to_string(&path) + .await + .map_err(|err| Error::io_path(&path, err))?, + ) + .map_err(|err| Error::TomlDeserialize { + path: path.to_path_buf(), + err, + }) + } else { + debug!(log, "No ledger in {}", path.display()); + Ok(Self::default()) + } + } + + /// Writes to `path` as a toml-serialized version of `Self`. + async fn write_to(&self, log: &Logger, path: &Path) -> Result<(), Error> { + debug!(log, "Writing ledger to {}", path.display()); + let serialized = + toml::Value::try_from(&self).expect("Cannot serialize ledger"); + let as_str = toml::to_string(&serialized).map_err(|err| { + Error::TomlSerialize { path: path.to_path_buf(), err } + })?; + tokio::fs::write(&path, as_str) + .await + .map_err(|err| Error::io_path(&path, err))?; + Ok(()) + } +} + +#[cfg(test)] +mod test { + use super::*; + use omicron_test_utils::dev::test_setup_log; + + #[derive(Serialize, serde::Deserialize, Default, Eq, PartialEq, Debug)] + struct Data { + generation: u64, + contents: String, + } + + impl Ledgerable for Data { + fn is_newer_than(&self, other: &Self) -> bool { + self.generation > other.generation + } + + fn generation_bump(&mut self) { + self.generation = self.generation + 1; + } + } + + #[tokio::test] + async fn test_create_default_ledger() { + let logctx = test_setup_log("create_default_ledger"); + let log = &logctx.log; + + let config_dir = tempfile::TempDir::new().unwrap(); + let ledger = + Ledger::::new(&log, vec![config_dir.path().to_path_buf()]) + .await + .expect("Failed to create ledger"); + + // Since we haven't previously stored anything, expect to read a default + // value. + assert_eq!(ledger.data(), &Data::default()); + + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_create_ledger_reads_from_storage() { + let logctx = test_setup_log("create_ledger_reads_from_storage"); + let log = &logctx.log; + + let config_dir = tempfile::TempDir::new().unwrap(); + let config_path = config_dir.path().join("ledger.toml"); + + // Create the ledger within a configuration directory + let mut ledger = Ledger::::new(&log, vec![config_path.clone()]) + .await + .expect("Failed to create ledger"); + ledger.data_mut().contents = "new contents".to_string(); + ledger.commit().await.expect("Failed to write ledger"); + assert!(config_path.exists()); + + drop(ledger); + + // Re-create the ledger, observe the new contents. + let ledger = Ledger::::new(&log, vec![config_path.clone()]) + .await + .expect("Failed to create ledger"); + + assert_eq!(ledger.data().contents, "new contents"); + assert_eq!(ledger.data().generation, 1); + + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_create_ledger_reads_latest_from_storage() { + let logctx = test_setup_log("create_ledger_reads_latest_from_storage"); + let log = &logctx.log; + + // Create the ledger, initialize contents. + let config_dirs = vec![ + tempfile::TempDir::new().unwrap(), + tempfile::TempDir::new().unwrap(), + ]; + let config_paths = config_dirs + .iter() + .map(|d| d.path().join("ledger.toml")) + .collect::>(); + + let mut ledger = Ledger::::new(&log, config_paths.clone()) + .await + .expect("Failed to create ledger"); + ledger.data_mut().contents = "new contents".to_string(); + ledger.commit().await.expect("Failed to write ledger"); + + assert!(config_paths[0].exists()); + assert!(config_paths[1].exists()); + + drop(ledger); + + // Let's write again, but only using one of the two config dirs. + let mut ledger = Ledger::::new(&log, config_paths[..1].to_vec()) + .await + .expect("Failed to create ledger"); + ledger.data_mut().contents = "even newer contents".to_string(); + ledger.commit().await.expect("Failed to write ledger"); + + drop(ledger); + + // Re-create the ledger (using both config dirs), observe the newest contents. + let ledger = Ledger::::new(&log, config_paths.clone()) + .await + .expect("Failed to create ledger"); + + assert_eq!(ledger.data().contents, "even newer contents"); + assert_eq!(ledger.data().generation, 2); + + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_commit_handles_write_failures() { + let logctx = test_setup_log("create_commit_handles_write_failures"); + let log = &logctx.log; + + // Create the ledger, initialize contents. + let mut config_dirs = vec![ + tempfile::TempDir::new().unwrap(), + tempfile::TempDir::new().unwrap(), + ]; + let config_paths = config_dirs + .iter() + .map(|d| d.path().join("ledger.toml")) + .collect::>(); + + let mut ledger = Ledger::::new(&log, config_paths.clone()) + .await + .expect("Failed to create ledger"); + ledger.data_mut().contents = "written to both configs".to_string(); + ledger.commit().await.expect("Failed to write ledger"); + + assert!(config_paths[0].exists()); + assert!(config_paths[1].exists()); + + drop(ledger); + + // Remove one of the config directories, try again. + // + // We should still be able to read and write the ledger. + config_dirs.remove(1); + assert!(config_paths[0].exists()); + assert!(!config_paths[1].exists()); + + let mut ledger = Ledger::::new(&log, config_paths.clone()) + .await + .expect("Failed to create ledger"); + + assert_eq!(ledger.data().contents, "written to both configs"); + assert_eq!(ledger.data().generation, 1); + ledger.data_mut().contents = "written to one config".to_string(); + ledger.commit().await.expect("Failed to write ledger"); + + drop(ledger); + + // We can still parse the ledger from a single path + let ledger = Ledger::::new(&log, config_paths.clone()) + .await + .expect("Failed to create ledger"); + assert_eq!(ledger.data().contents, "written to one config"); + assert_eq!(ledger.data().generation, 2); + + drop(ledger); + + // Remove the last config directory, try again. + // + // We should not be able to write the ledger. + drop(config_dirs); + assert!(!config_paths[0].exists()); + assert!(!config_paths[1].exists()); + + let mut ledger = Ledger::::new(&log, config_paths.clone()) + .await + .expect("Failed to create ledger"); + + assert_eq!(ledger.data(), &Data::default()); + let err = ledger.commit().await.unwrap_err(); + assert!( + matches!(err, Error::FailedToAccessStorage), + "Unexpected error: {err}" + ); + + logctx.cleanup_successful(); + } +} diff --git a/sled-agent/src/lib.rs b/sled-agent/src/lib.rs index 9682fa3cc8..0a17f99e28 100644 --- a/sled-agent/src/lib.rs +++ b/sled-agent/src/lib.rs @@ -22,6 +22,7 @@ pub mod config; mod http_entrypoints; mod instance; mod instance_manager; +mod ledger; mod nexus; pub mod params; mod profile; diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 78b01895f3..7dd31f35c1 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -25,6 +25,7 @@ //! - [ServiceManager::activate_switch] exposes an API to specifically enable //! or disable (via [ServiceManager::deactivate_switch]) the switch zone. +use crate::ledger::{Ledger, Ledgerable}; use crate::params::{ DendriteAsic, ServiceEnsureBody, ServiceType, ServiceZoneRequest, ServiceZoneService, TimeSync, ZoneType, @@ -61,6 +62,7 @@ use omicron_common::address::OXIMETER_PORT; use omicron_common::address::RACK_PREFIX; use omicron_common::address::SLED_PREFIX; use omicron_common::address::WICKETD_PORT; +use omicron_common::api::external::Generation; use omicron_common::backoff::{ retry_notify, retry_policy_internal_service_aggressive, retry_policy_local, BackoffError, @@ -108,6 +110,9 @@ pub enum Error { #[error("Failed to find device {device}")] MissingDevice { device: String }, + #[error("Failed to access ledger: {0}")] + Ledger(#[from] crate::ledger::Error), + #[error("Sled Agent not initialized yet")] SledAgentNotReady, @@ -197,88 +202,39 @@ pub struct Config { /// Identifies the revision of the sidecar to be used. pub sidecar_revision: String, - - // The path for the ServiceManager to store information about - // all running services. - all_svcs_ledger_path: PathBuf, - storage_svcs_ledger_path: PathBuf, } impl Config { pub fn new(sled_id: Uuid, sidecar_revision: String) -> Self { - Self { - sled_id, - sidecar_revision, - all_svcs_ledger_path: default_services_ledger_path(), - storage_svcs_ledger_path: default_storage_services_ledger_path(), - } + Self { sled_id, sidecar_revision } } } -// The filename of ServiceManager's internal storage. +// The filename of the ledger, within the provided directory. const SERVICES_LEDGER_FILENAME: &str = "services.toml"; const STORAGE_SERVICES_LEDGER_FILENAME: &str = "storage-services.toml"; -// The default path to service configuration -fn default_services_ledger_path() -> PathBuf { - Path::new(omicron_common::OMICRON_CONFIG_PATH) - .join(SERVICES_LEDGER_FILENAME) -} - -// The default path to storage service configuration -fn default_storage_services_ledger_path() -> PathBuf { - Path::new(omicron_common::OMICRON_CONFIG_PATH) - .join(STORAGE_SERVICES_LEDGER_FILENAME) -} - // A wrapper around `ZoneRequest`, which allows it to be serialized // to a toml file. #[derive(Clone, serde::Serialize, serde::Deserialize)] struct AllZoneRequests { + generation: Generation, requests: Vec, } -impl AllZoneRequests { - fn new() -> Self { - Self { requests: vec![] } +impl Default for AllZoneRequests { + fn default() -> Self { + Self { generation: Generation::new(), requests: vec![] } } +} - // Reads from `path` as a toml-serialized version of `Self`. - async fn read_from(log: &Logger, path: &Path) -> Result { - if path.exists() { - debug!( - log, - "Reading old storage service requests from {}", - path.display() - ); - toml::from_str( - &tokio::fs::read_to_string(&path) - .await - .map_err(|err| Error::io_path(&path, err))?, - ) - .map_err(|err| Error::TomlDeserialize { - path: path.to_path_buf(), - err, - }) - } else { - debug!(log, "No old storage service requests"); - Ok(AllZoneRequests::new()) - } +impl Ledgerable for AllZoneRequests { + fn is_newer_than(&self, other: &AllZoneRequests) -> bool { + self.generation >= other.generation } - // Writes to `path` as a toml-serialized version of `Self`. - async fn write_to(&self, log: &Logger, path: &Path) -> Result<(), Error> { - debug!(log, "Writing zone request configuration to {}", path.display()); - let serialized_services = toml::Value::try_from(&self) - .expect("Cannot serialize service list"); - let services_str = - toml::to_string(&serialized_services).map_err(|err| { - Error::TomlSerialize { path: path.to_path_buf(), err } - })?; - tokio::fs::write(&path, services_str) - .await - .map_err(|err| Error::io_path(&path, err))?; - Ok(()) + fn generation_bump(&mut self) { + self.generation = self.generation.next(); } } @@ -357,8 +313,8 @@ pub struct ServiceManagerInner { // TODO(https://github.com/oxidecomputer/omicron/issues/2888): We will // need this interface to provision Zone filesystems on explicit U.2s, // rather than simply placing them on the ramdisk. - #[allow(dead_code)] storage: StorageManager, + ledger_directory_override: OnceCell, } // Late-binding information, only known once the sled agent is up and @@ -433,23 +389,62 @@ impl ServiceManager { sled_info: OnceCell::new(), switch_zone_bootstrap_address, storage, + ledger_directory_override: OnceCell::new(), }), }; Ok(mgr) } + #[cfg(test)] + async fn override_ledger_directory(&self, path: PathBuf) { + self.inner.ledger_directory_override.set(path).unwrap(); + } + pub fn switch_zone_bootstrap_address(&self) -> Ipv6Addr { self.inner.switch_zone_bootstrap_address } + async fn all_service_ledgers(&self) -> Vec { + if let Some(dir) = self.inner.ledger_directory_override.get() { + return vec![dir.join(SERVICES_LEDGER_FILENAME)]; + } + self.inner + .storage + .resources() + .all_m2_mountpoints(sled_hardware::disk::CONFIG_DATASET) + .await + .into_iter() + .map(|p| p.join(SERVICES_LEDGER_FILENAME)) + .collect() + } + + async fn all_storage_service_ledgers(&self) -> Vec { + if let Some(dir) = self.inner.ledger_directory_override.get() { + return vec![dir.join(STORAGE_SERVICES_LEDGER_FILENAME)]; + } + + self.inner + .storage + .resources() + .all_m2_mountpoints(sled_hardware::disk::CONFIG_DATASET) + .await + .into_iter() + .map(|p| p.join(STORAGE_SERVICES_LEDGER_FILENAME)) + .collect() + } + pub async fn load_non_storage_services(&self) -> Result<(), Error> { let log = &self.inner.log; - let ledger = self.services_ledger_path()?; - if !ledger.exists() { + let mut existing_zones = self.inner.zones.lock().await; + let ledger = Ledger::::new( + log, + self.all_service_ledgers().await, + ) + .await?; + let services = ledger.data(); + if services.requests.is_empty() { return Ok(()); } - let services = AllZoneRequests::read_from(log, &ledger).await?; - let mut existing_zones = self.inner.zones.lock().await; // Initialize and DNS and NTP services first as they are required // for time synchronization, which is a pre-requisite for the other @@ -517,12 +512,16 @@ impl ServiceManager { pub async fn load_storage_services(&self) -> Result<(), Error> { let log = &self.inner.log; - let ledger = self.storage_services_ledger_path()?; - if !ledger.exists() { + let mut existing_zones = self.inner.dataset_zones.lock().await; + let ledger = Ledger::::new( + log, + self.all_storage_service_ledgers().await, + ) + .await?; + let services = ledger.data(); + if services.requests.is_empty() { return Ok(()); } - let services = AllZoneRequests::read_from(log, &ledger).await?; - let mut existing_zones = self.inner.dataset_zones.lock().await; self.initialize_services_locked( &mut existing_zones, &services.requests, @@ -574,26 +573,6 @@ impl ServiceManager { self.inner.sled_mode } - // Returns either the path to the explicitly provided ledger path, or - // chooses the default one. - fn services_ledger_path(&self) -> Result { - if let Some(info) = self.inner.sled_info.get() { - Ok(info.config.all_svcs_ledger_path.clone()) - } else { - Err(Error::SledAgentNotReady) - } - } - - // Returns either the path to the explicitly provided ledger path, or - // chooses the default one. - fn storage_services_ledger_path(&self) -> Result { - if let Some(info) = self.inner.sled_info.get() { - Ok(info.config.storage_svcs_ledger_path.clone()) - } else { - Err(Error::SledAgentNotReady) - } - } - // Advertise the /64 prefix of `address`, unless we already have. // // This method only blocks long enough to check our HashSet of @@ -1804,15 +1783,20 @@ impl ServiceManager { &self, request: ServiceEnsureBody, ) -> Result<(), Error> { + let log = &self.inner.log; let mut existing_zones = self.inner.zones.lock().await; - let ledger_path = self.services_ledger_path()?; - let old_zone_requests = - AllZoneRequests::read_from(&self.inner.log, &ledger_path).await?; + // Read the existing set of services from the ledger. + let mut ledger = Ledger::::new( + log, + self.all_service_ledgers().await, + ) + .await?; + let ledger_zone_requests = ledger.data_mut(); let new_zone_requests: Vec = { let known_set: HashSet<&ServiceZoneRequest> = HashSet::from_iter( - old_zone_requests.requests.iter().map(|r| &r.zone), + ledger_zone_requests.requests.iter().map(|r| &r.zone), ); let requested_set = HashSet::from_iter(request.services.iter()); @@ -1824,7 +1808,7 @@ impl ServiceManager { // the case of changing configurations, rather than just doing // that removal implicitly. warn!( - self.inner.log, + log, "Cannot request services on this sled, differing configurations: {:#?}", known_set.symmetric_difference(&requested_set) ); @@ -1836,7 +1820,7 @@ impl ServiceManager { .collect::>() }; - let mut zone_requests = AllZoneRequests::new(); + let mut zone_requests = AllZoneRequests::default(); for zone in new_zone_requests.into_iter() { let root = PathBuf::from(ZONE_ZFS_RAMDISK_DATASET_MOUNTPOINT); zone_requests.requests.push(ZoneRequest { zone, root }); @@ -1848,8 +1832,9 @@ impl ServiceManager { ) .await?; - zone_requests.requests.append(&mut old_zone_requests.requests.clone()); - zone_requests.write_to(&self.inner.log, &ledger_path).await?; + // Update the services in the ledger and write it back to both M.2s + ledger_zone_requests.requests.append(&mut zone_requests.requests); + ledger.commit().await?; Ok(()) } @@ -1863,13 +1848,18 @@ impl ServiceManager { &self, request: ServiceZoneRequest, ) -> Result<(), Error> { + let log = &self.inner.log; let mut existing_zones = self.inner.dataset_zones.lock().await; - let ledger_path = self.storage_services_ledger_path()?; - let mut zone_requests = - AllZoneRequests::read_from(&self.inner.log, &ledger_path).await?; + // Read the existing set of services from the ledger. + let mut ledger = Ledger::::new( + log, + self.all_storage_service_ledgers().await, + ) + .await?; + let ledger_zone_requests = ledger.data_mut(); - if !zone_requests + if !ledger_zone_requests .requests .iter() .any(|zone_request| zone_request.zone.id == request.id) @@ -1883,16 +1873,18 @@ impl ServiceManager { let root = dataset .pool() .dataset_mountpoint(sled_hardware::disk::ZONE_DATASET); - zone_requests.requests.push(ZoneRequest { zone: request, root }); + ledger_zone_requests + .requests + .push(ZoneRequest { zone: request, root }); } self.initialize_services_locked( &mut existing_zones, - &zone_requests.requests, + &ledger_zone_requests.requests, ) .await?; - zone_requests.write_to(&self.inner.log, &ledger_path).await?; + ledger.commit().await?; Ok(()) } @@ -2476,15 +2468,9 @@ mod test { } fn make_config(&self) -> Config { - let all_svcs_ledger_path = - self.config_dir.path().join(SERVICES_LEDGER_FILENAME); - let storage_svcs_ledger_path = - self.config_dir.path().join(STORAGE_SERVICES_LEDGER_FILENAME); Config { sled_id: Uuid::new_v4(), sidecar_revision: "rev_whatever_its_a_test".to_string(), - all_svcs_ledger_path, - storage_svcs_ledger_path, } } } @@ -2512,6 +2498,10 @@ mod test { ) .await .unwrap(); + mgr.override_ledger_directory( + test_config.config_dir.path().to_path_buf(), + ) + .await; let port_manager = PortManager::new( logctx.log.new(o!("component" => "PortManager")), @@ -2557,6 +2547,10 @@ mod test { ) .await .unwrap(); + mgr.override_ledger_directory( + test_config.config_dir.path().to_path_buf(), + ) + .await; let port_manager = PortManager::new( logctx.log.new(o!("component" => "PortManager")), @@ -2605,6 +2599,10 @@ mod test { ) .await .unwrap(); + mgr.override_ledger_directory( + test_config.config_dir.path().to_path_buf(), + ) + .await; let port_manager = PortManager::new( logctx.log.new(o!("component" => "PortManager")), @@ -2641,6 +2639,10 @@ mod test { ) .await .unwrap(); + mgr.override_ledger_directory( + test_config.config_dir.path().to_path_buf(), + ) + .await; let port_manager = PortManager::new( logctx.log.new(o!("component" => "PortManager")), @@ -2686,6 +2688,10 @@ mod test { ) .await .unwrap(); + mgr.override_ledger_directory( + test_config.config_dir.path().to_path_buf(), + ) + .await; let port_manager = PortManager::new( logctx.log.new(o!("component" => "PortManager")), Ipv6Addr::new(0xfd00, 0x1de, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01), @@ -2703,10 +2709,12 @@ mod test { ensure_new_service(&mgr, id).await; drop_service_manager(mgr); - // Next, delete the config. This means the service we just created will + // Next, delete the ledger. This means the service we just created will // not be remembered on the next initialization. - let config = test_config.make_config(); - std::fs::remove_file(&config.all_svcs_ledger_path).unwrap(); + std::fs::remove_file( + test_config.config_dir.path().join(SERVICES_LEDGER_FILENAME), + ) + .unwrap(); // Observe that the old service is not re-initialized. let mgr = ServiceManager::new( @@ -2724,6 +2732,11 @@ mod test { ) .await .unwrap(); + mgr.override_ledger_directory( + test_config.config_dir.path().to_path_buf(), + ) + .await; + let port_manager = PortManager::new( logctx.log.new(o!("component" => "PortManager")), Ipv6Addr::new(0xfd00, 0x1de, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01), diff --git a/sled-agent/src/storage_manager.rs b/sled-agent/src/storage_manager.rs index 75df7ffeb6..0c58405e6d 100644 --- a/sled-agent/src/storage_manager.rs +++ b/sled-agent/src/storage_manager.rs @@ -225,6 +225,15 @@ impl StorageResources { self.all_zpools(DiskVariant::M2).await } + /// Returns all mountpoints within all M.2s for a particular dataset. + pub async fn all_m2_mountpoints(&self, dataset: &str) -> Vec { + let m2_zpools = self.all_m2_zpools().await; + m2_zpools + .iter() + .map(|zpool| zpool.dataset_mountpoint(dataset)) + .collect() + } + pub async fn all_zpools(&self, variant: DiskVariant) -> Vec { let disks = self.disks.lock().await; disks