[sled-agent] Store service configuration data in duplicate in M.2s

illumos-utils/src/running_zone.rs

@@ -592,8 +592,8 @@ impl InstalledZone {
     ///
     /// This results in a zone name which is distinct across different zpools,
     /// but stable and predictable across reboots.
-    pub fn get_zone_name(zone_name: &str, unique_name: Option<&str>) -> String {
-        let mut zone_name = format!("{}{}", ZONE_PREFIX, zone_name);
+    pub fn get_zone_name(zone_type: &str, unique_name: Option<&str>) -> String {
+        let mut zone_name = format!("{}{}", ZONE_PREFIX, zone_type);
         if let Some(suffix) = unique_name {
             zone_name.push_str(&format!("_{}", suffix));
         }
@@ -618,7 +618,7 @@ impl InstalledZone {
         log: &Logger,
         underlay_vnic_allocator: &VnicAllocator<Etherstub>,
         zone_root_path: &Path,
-        zone_name: &str,
+        zone_type: &str,
         unique_name: Option<&str>,
         datasets: &[zone::Dataset],
         filesystems: &[zone::Fs],
@@ -631,14 +631,14 @@ impl InstalledZone {
         let control_vnic =
             underlay_vnic_allocator.new_control(None).map_err(|err| {
                 InstallZoneError::CreateVnic {
-                    zone: zone_name.to_string(),
+                    zone: zone_type.to_string(),
                     err,
                 }
             })?;
 
-        let full_zone_name = Self::get_zone_name(zone_name, unique_name);
+        let full_zone_name = Self::get_zone_name(zone_type, unique_name);
         let zone_image_path =
-            PathBuf::from(&format!("/opt/oxide/{}.tar.gz", zone_name));
+            PathBuf::from(&format!("/opt/oxide/{}.tar.gz", zone_type));
 
         let net_device_names: Vec<String> = opte_ports
             .iter()

illumos-utils/src/zpool.rs

@@ -249,6 +249,7 @@ impl Zpool {
 }
 
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, JsonSchema)]
+#[serde(rename_all = "snake_case")]
 pub enum ZpoolKind {
     // This zpool is used for external storage (u.2)
     External,

openapi/bootstrap-agent.json

@@ -113,6 +113,49 @@
       }
     },
     "schemas": {
+      "BootstrapAddressDiscovery": {
+        "oneOf": [
+          {
+            "description": "Ignore all bootstrap addresses except our own.",
+            "type": "object",
+            "properties": {
+              "type": {
+                "type": "string",
+                "enum": [
+                  "only_ours"
+                ]
+              }
+            },
+            "required": [
+              "type"
+            ]
+          },
+          {
+            "description": "Ignore all bootstrap addresses except the following.",
+            "type": "object",
+            "properties": {
+              "addrs": {
+                "type": "array",
+                "items": {
+                  "type": "string",
+                  "format": "ipv6"
+                },
+                "uniqueItems": true
+              },
+              "type": {
+                "type": "string",
+                "enum": [
+                  "only_these"
+                ]
+              }
+            },
+            "required": [
+              "addrs",
+              "type"
+            ]
+          }
+        ]
+      },
       "Component": {
         "type": "object",
         "properties": {
@@ -239,6 +282,14 @@
         "description": "Configuration for the \"rack setup service\".\n\nThe Rack Setup Service should be responsible for one-time setup actions, such as CockroachDB placement and initialization.  Without operator intervention, however, these actions need a way to be automated in our deployment.",
         "type": "object",
         "properties": {
+          "bootstrap_discovery": {
+            "description": "Describes how bootstrap addresses should be collected during RSS.",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/BootstrapAddressDiscovery"
+              }
+            ]
+          },
           "dns_servers": {
             "description": "The external DNS server addresses.",
             "type": "array",
@@ -281,6 +332,7 @@
           }
         },
         "required": [
+          "bootstrap_discovery",
           "dns_servers",
           "internal_services_ip_pool_ranges",
           "ntp_servers",

openapi/sled-agent.json

@@ -640,13 +640,6 @@
           {
             "type": "object",
             "properties": {
-              "all_addresses": {
-                "description": "The addresses of all nodes within the cluster.",
-                "type": "array",
-                "items": {
-                  "type": "string"
-                }
-              },
               "type": {
                 "type": "string",
                 "enum": [
@@ -655,7 +648,6 @@
               }
             },
             "required": [
-              "all_addresses",
               "type"
             ]
           },
@@ -689,6 +681,21 @@
           }
         ]
       },
+      "DatasetName": {
+        "type": "object",
+        "properties": {
+          "kind": {
+            "$ref": "#/components/schemas/DatasetKind"
+          },
+          "pool_name": {
+            "$ref": "#/components/schemas/ZpoolName"
+          }
+        },
+        "required": [
+          "kind",
+          "pool_name"
+        ]
+      },
       "DendriteAsic": {
         "type": "string",
         "enum": [
@@ -1738,7 +1745,7 @@
         "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$"
       },
       "ServiceEnsureBody": {
-        "description": "Used to request that the Sled initialize certain services on initialization.\n\nThis may be used to record that certain sleds are responsible for launching services which may not be associated with a dataset, such as Nexus.",
+        "description": "Used to request that the Sled initialize multiple services.\n\nThis may be used to record that certain sleds are responsible for launching services which may not be associated with a dataset, such as Nexus.",
         "type": "object",
         "properties": {
           "services": {
@@ -2036,6 +2043,48 @@
               "mode",
               "type"
             ]
+          },
+          {
+            "type": "object",
+            "properties": {
+              "type": {
+                "type": "string",
+                "enum": [
+                  "clickhouse"
+                ]
+              }
+            },
+            "required": [
+              "type"
+            ]
+          },
+          {
+            "type": "object",
+            "properties": {
+              "type": {
+                "type": "string",
+                "enum": [
+                  "cockroach_db"
+                ]
+              }
+            },
+            "required": [
+              "type"
+            ]
+          },
+          {
+            "type": "object",
+            "properties": {
+              "type": {
+                "type": "string",
+                "enum": [
+                  "crucible"
+                ]
+              }
+            },
+            "required": [
+              "type"
+            ]
           }
         ]
       },
@@ -2050,6 +2099,15 @@
               "format": "ipv6"
             }
           },
+          "dataset": {
+            "nullable": true,
+            "default": null,
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/DatasetName"
+              }
+            ]
+          },
           "gz_addresses": {
             "default": [],
             "type": "array",
@@ -2080,6 +2138,7 @@
         ]
       },
       "ServiceZoneService": {
+        "description": "Used to request that the Sled initialize a single service.",
         "type": "object",
         "properties": {
           "details": {
@@ -2481,13 +2540,16 @@
         "description": "The type of zone which may be requested from Sled Agent",
         "type": "string",
         "enum": [
+          "clickhouse",
+          "cockroach_db",
+          "crucible_pantry",
+          "crucible",
           "external_dns",
           "internal_dns",
           "nexus",
+          "ntp",
           "oximeter",
-          "switch",
-          "crucible_pantry",
-          "ntp"
+          "switch"
         ]
       },
       "Zpool": {
@@ -2505,6 +2567,30 @@
           "disk_type",
           "id"
         ]
+      },
+      "ZpoolKind": {
+        "type": "string",
+        "enum": [
+          "external",
+          "internal"
+        ]
+      },
+      "ZpoolName": {
+        "description": "A wrapper around a zpool name.\n\nThis expects that the format will be: `ox{i,p}_<UUID>` - we parse the prefix when reading the structure, and validate that the UUID can be utilized.",
+        "type": "object",
+        "properties": {
+          "id": {
+            "type": "string",
+            "format": "uuid"
+          },
+          "kind": {
+            "$ref": "#/components/schemas/ZpoolKind"
+          }
+        },
+        "required": [
+          "id",
+          "kind"
+        ]
       }
     }
   }

sled-agent/src/bootstrap/hardware.rs

@@ -178,10 +178,7 @@ impl HardwareMonitor {
         let hardware = HardwareManager::new(log, sled_mode)
             .map_err(|e| Error::Hardware(e))?;
 
-        // TODO: The coupling between the storage and service manager is growing
-        // pretty tight; we should consider merging them together.
-        let storage_manager =
-            StorageManager::new(&log, underlay_etherstub.clone()).await;
+        let storage_manager = StorageManager::new(&log).await;
 
         let service_manager = ServiceManager::new(
             log.clone(),

sled-agent/src/bootstrap/params.rs

@@ -11,9 +11,19 @@ use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use serde_with::serde_as;
 use std::borrow::Cow;
+use std::collections::HashSet;
 use std::net::{Ipv4Addr, Ipv6Addr, SocketAddrV6};
 use uuid::Uuid;
 
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)]
+#[serde(rename_all = "snake_case", tag = "type")]
+pub enum BootstrapAddressDiscovery {
+    /// Ignore all bootstrap addresses except our own.
+    OnlyOurs,
+    /// Ignore all bootstrap addresses except the following.
+    OnlyThese { addrs: HashSet<Ipv6Addr> },
+}
+
 /// Configuration for the "rack setup service".
 ///
 /// The Rack Setup Service should be responsible for one-time setup actions,
@@ -24,6 +34,9 @@ use uuid::Uuid;
 pub struct RackInitializeRequest {
     pub rack_subnet: Ipv6Addr,
 
+    /// Describes how bootstrap addresses should be collected during RSS.
+    pub bootstrap_discovery: BootstrapAddressDiscovery,
+
     /// The minimum number of sleds required to unlock the rack secret.
     ///
     /// If this value is less than 2, no rack secret will be created on startup;

sled-agent/src/http_entrypoints.rs

@@ -103,6 +103,7 @@ async fn filesystem_put(
     let sa = rqctx.context();
     let body_args = body.into_inner();
     sa.filesystem_ensure(
+        body_args.id,
         body_args.zpool_id,
         body_args.dataset_kind,
         body_args.address,