User data support

common/src/sql/dbinit.sql

@@ -303,6 +303,9 @@ CREATE TABLE omicron.public.instance (
     /* Every Instance is in exactly one Project at a time. */
     project_id UUID NOT NULL,
 
+    /* user data for instance initialization systems (e.g. cloud-init) */
+    user_data BYTES NOT NULL,
+
     /*
      * TODO Would it make sense for the runtime state to live in a separate
      * table?

nexus/Cargo.toml

@@ -18,6 +18,7 @@ cookie = "0.16"
 crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "945daedb88cefa790f1d994b3a038b8fa9ac514a" }
 # Tracking pending 2.0 version.
 diesel = { git = "https://github.com/diesel-rs/diesel", rev = "ce77c382", features = ["postgres", "r2d2", "chrono", "serde_json", "network-address", "uuid"] }
+fatfs = "0.3.5"
 futures = "0.3.21"
 headers = "0.3.7"
 hex = "0.4.3"
@@ -30,6 +31,7 @@ libc = "0.2.122"
 macaddr = { version = "1.0.1", features = [ "serde_std" ]}
 mime_guess = "2.0.4"
 newtype_derive = "0.1.6"
+num-integer = "0.1.44"
 oso = "0.26"
 oximeter-client = { path = "../oximeter-client" }
 oximeter-db = { path = "../oximeter/db/" }

nexus/src/cidata.rs

@@ -0,0 +1,105 @@
+use crate::db::{identity::Resource, model::Instance};
+use fatfs::{FatType, FileSystem, FormatVolumeOptions, FsOptions};
+use num_integer::Integer;
+use omicron_common::api::external::Error;
+use serde::Serialize;
+use std::io::{self, Cursor, Write};
+use uuid::Uuid;
+
+pub const MAX_USER_DATA_BYTES: usize = 32 * 1024; // 32 KiB
+
+impl Instance {
+    pub fn generate_cidata(&self) -> Result<Vec<u8>, Error> {
+        // cloud-init meta-data is YAML, but YAML is a strict superset of JSON.
+        let meta_data = serde_json::to_vec(&MetaData {
+            instance_id: self.id(),
+            local_hostname: &self.runtime().hostname,
+            public_keys: &[], // TODO
+        })
+        .map_err(|_| Error::internal_error("failed to serialize meta-data"))?;
+        let cidata =
+            build_vfat(&meta_data, &self.user_data).map_err(|err| {
+                Error::internal_error(&format!(
+                    "failed to create cidata volume: {}",
+                    err
+                ))
+            })?;
+        Ok(cidata)
+    }
+}
+
+#[derive(Serialize)]
+#[serde(rename_all = "kebab-case")]
+struct MetaData<'a> {
+    instance_id: Uuid,
+    local_hostname: &'a str,
+    public_keys: &'a [String],
+}
+
+fn build_vfat(meta_data: &[u8], user_data: &[u8]) -> io::Result<Vec<u8>> {
+    let file_sectors =
+        meta_data.len().div_ceil(&512) + user_data.len().div_ceil(&512);
+    // vfat can hold more data than this, but we don't expect to ever need that for cloud-init
+    // purposes.
+    if file_sectors > 512 {
+        return Err(io::Error::new(io::ErrorKind::Other, "too much vfat data"));
+    }
+
+    // https://github.com/oxidecomputer/omicron/pull/911#discussion_r851354213
+    // If we're storing < 170 KiB of clusters, the FAT overhead is 35 sectors;
+    // if we're storing < 341 KiB of clusters, the overhead is 37. With a limit
+    // of 512 sectors (error check above), we can assume an overhead of 37.
+    // Additionally, fatfs refuses to format a disk that is smaller than 42
+    // sectors.
+    let sectors = 42.max(file_sectors + 37);
+
+    let mut disk = Cursor::new(vec![0; sectors * 512]);
+    fatfs::format_volume(
+        &mut disk,
+        FormatVolumeOptions::new()
+            .bytes_per_cluster(512)
+            .fat_type(FatType::Fat12)
+            .volume_label(*b"cidata     "),
+    )?;
+
+    {
+        let fs = FileSystem::new(&mut disk, FsOptions::new())?;
+        let root_dir = fs.root_dir();
+        for (file, data) in [("meta-data", meta_data), ("user-data", user_data)]
+        {
+            if !data.is_empty() {
+                let mut file = root_dir.create_file(file)?;
+                file.write_all(data)?;
+            }
+        }
+    }
+
+    Ok(disk.into_inner())
+}
+
+#[cfg(test)]
+mod tests {
+    /// the fatfs crate has some unfortunate panics if you ask it to do
+    /// incredibly stupid things, like format an empty disk or create a
+    /// filesystem with an invalid cluster size.
+    ///
+    /// to ensure that our math for the filesystem size is correct, and also to
+    /// ensure that we don't ask fatfs to do incredibly stupid things, this
+    /// test checks that `build_vfat` works on a representative sample of weird
+    /// file sizes. (32 KiB is our enforced limit for user_data, so push it a
+    /// little further.)
+    #[test]
+    fn build_vfat_works_with_arbitrarily_sized_input() {
+        let upper = crate::cidata::MAX_USER_DATA_BYTES + 4096;
+        // somewhat arbitrarily-chosen prime numbers near 1 KiB and 256 bytes
+        for md_size in (0..upper).step_by(1019) {
+            for ud_size in (0..upper).step_by(269) {
+                assert!(super::build_vfat(
+                    &vec![0x5a; md_size],
+                    &vec![0xa5; ud_size]
+                )
+                .is_ok());
+            }
+        }
+    }
+}

nexus/src/db/model.rs

@@ -1118,6 +1118,9 @@ pub struct Instance {
     /// id for the project containing this Instance
     pub project_id: Uuid,
 
+    /// user data for instance initialization systems (e.g. cloud-init)
+    pub user_data: Vec<u8>,
+
     /// runtime state of the Instance
     #[diesel(embed)]
     pub runtime_state: InstanceRuntimeState,
@@ -1132,7 +1135,12 @@ impl Instance {
     ) -> Self {
         let identity =
             InstanceIdentity::new(instance_id, params.identity.clone());
-        Self { identity, project_id, runtime_state: runtime }
+        Self {
+            identity,
+            project_id,
+            user_data: params.user_data.clone(),
+            runtime_state: runtime,
+        }
     }
 
     pub fn runtime(&self) -> &InstanceRuntimeState {

nexus/src/db/schema.rs

@@ -68,6 +68,7 @@ table! {
         time_modified -> Timestamptz,
         time_deleted -> Nullable<Timestamptz>,
         project_id -> Uuid,
+        user_data -> Binary,
         state -> crate::db::model::InstanceStateEnum,
         time_state_updated -> Timestamptz,
         state_generation -> Int8,

nexus/src/external_api/params.rs

@@ -138,6 +138,15 @@ pub struct InstanceCreate {
     pub memory: ByteCount,
     pub hostname: String, // TODO-cleanup different type?
 
+    /// User data for instance initialization systems (such as cloud-init).
+    /// Must be a Base64-encoded string, as specified in RFC 4648 § 4 (+ and /
+    /// characters with padding). Maximum 32 KiB unencoded data.
+    // TODO: this should emit `"format": "byte"`, but progenitor doesn't
+    // understand that yet.
+    #[schemars(default, with = "String")]
+    #[serde(default, with = "serde_user_data")]
+    pub user_data: Vec<u8>,
+
     /// The network interfaces to be created for this instance.
     #[serde(default)]
     pub network_interfaces: InstanceNetworkInterfaceAttachment,
@@ -147,6 +156,43 @@ pub struct InstanceCreate {
     pub disks: Vec<InstanceDiskAttachment>,
 }
 
+mod serde_user_data {
+    use serde::{de::Error, Deserialize, Deserializer, Serialize, Serializer};
+
+    pub fn serialize<S>(
+        data: &Vec<u8>,
+        serializer: S,
+    ) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        base64::encode(data).serialize(serializer)
+    }
+
+    pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        match base64::decode(<&str>::deserialize(deserializer)?) {
+            Ok(buf) => {
+                // if you change this, also update the stress test in crate::cidata
+                if buf.len() > crate::cidata::MAX_USER_DATA_BYTES {
+                    Err(D::Error::invalid_length(
+                        buf.len(),
+                        &"less than 32 KiB",
+                    ))
+                } else {
+                    Ok(buf)
+                }
+            }
+            Err(_) => Err(D::Error::invalid_value(
+                serde::de::Unexpected::Other("invalid base64 string"),
+                &"a valid base64 string",
+            )),
+        }
+    }
+}
+
 /// Migration parameters for an [`Instance`](omicron_common::api::external::Instance)
 #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)]
 pub struct InstanceMigrate {

nexus/src/lib.rs

@@ -15,6 +15,7 @@
 
 pub mod authn; // Public only for testing
 pub mod authz; // Public for documentation examples
+mod cidata;
 pub mod config; // Public for testing
 pub mod context; // Public for documentation examples
 pub mod db; // Public for documentation examples

nexus/src/nexus.rs

@@ -1697,6 +1697,9 @@ impl Nexus {
             ),
             nics: nics.iter().map(|nic| nic.clone().into()).collect(),
             disks: disk_reqs,
+            cloud_init_bytes: Some(base64::encode(
+                db_instance.generate_cidata()?,
+            )),
         };
 
         let sa = self.instance_sled(&db_instance).await?;

nexus/src/sagas.rs

@@ -973,6 +973,8 @@ async fn sim_instance_migrate(
         nics: vec![],
         // TODO: populate disks
         disks: vec![],
+        // TODO: populate cloud init bytes
+        cloud_init_bytes: None,
     };
     let target = InstanceRuntimeStateRequested {
         run_state: InstanceStateRequested::Migrating,

nexus/test-utils/src/resource_helpers.rs

@@ -162,6 +162,9 @@ pub async fn create_instance(
             ncpus: InstanceCpuCount(4),
             memory: ByteCount::from_mebibytes_u32(256),
             hostname: String::from("the_host"),
+            user_data:
+                b"#cloud-config\nsystem_info:\n  default_user:\n    name: oxide"
+                    .to_vec(),
             network_interfaces:
                 params::InstanceNetworkInterfaceAttachment::Default,
             disks: vec![],

nexus/tests/integration_tests/endpoints.rs

@@ -181,6 +181,7 @@ lazy_static! {
             ncpus: InstanceCpuCount(1),
             memory: ByteCount::from_gibibytes_u32(16),
             hostname: String::from("demo-instance"),
+            user_data: vec![],
             network_interfaces:
                 params::InstanceNetworkInterfaceAttachment::Default,
             disks: vec![],

nexus/tests/integration_tests/instances.rs

@@ -146,6 +146,7 @@ async fn test_instances_create_reboot_halt(
                 ncpus: instance.ncpus,
                 memory: instance.memory,
                 hostname: instance.hostname.clone(),
+                user_data: vec![],
                 network_interfaces:
                     params::InstanceNetworkInterfaceAttachment::Default,
                 disks: vec![],
@@ -563,6 +564,7 @@ async fn test_instance_create_saga_removes_instance_database_record(
         ncpus: InstanceCpuCount::try_from(2).unwrap(),
         memory: ByteCount::from_mebibytes_u32(4),
         hostname: String::from("inst"),
+        user_data: vec![],
         network_interfaces: interface_params.clone(),
         disks: vec![],
     };
@@ -584,6 +586,7 @@ async fn test_instance_create_saga_removes_instance_database_record(
         ncpus: InstanceCpuCount::try_from(2).unwrap(),
         memory: ByteCount::from_mebibytes_u32(4),
         hostname: String::from("inst2"),
+        user_data: vec![],
         network_interfaces: interface_params,
         disks: vec![],
     };
@@ -669,6 +672,7 @@ async fn test_instance_with_single_explicit_ip_address(
         ncpus: InstanceCpuCount::try_from(2).unwrap(),
         memory: ByteCount::from_mebibytes_u32(4),
         hostname: String::from("nic-test"),
+        user_data: vec![],
         network_interfaces: interface_params,
         disks: vec![],
     };
@@ -785,6 +789,7 @@ async fn test_instance_with_new_custom_network_interfaces(
         ncpus: InstanceCpuCount::try_from(2).unwrap(),
         memory: ByteCount::from_mebibytes_u32(4),
         hostname: String::from("nic-test"),
+        user_data: vec![],
         network_interfaces: interface_params,
         disks: vec![],
     };
@@ -878,6 +883,7 @@ async fn test_instance_create_delete_network_interface(
         ncpus: InstanceCpuCount::try_from(2).unwrap(),
         memory: ByteCount::from_mebibytes_u32(4),
         hostname: String::from("nic-test"),
+        user_data: vec![],
         network_interfaces: params::InstanceNetworkInterfaceAttachment::None,
         disks: vec![],
     };
@@ -1061,6 +1067,7 @@ async fn test_instance_with_multiple_nics_unwinds_completely(
         ncpus: InstanceCpuCount::try_from(2).unwrap(),
         memory: ByteCount::from_mebibytes_u32(4),
         hostname: String::from("nic-test"),
+        user_data: vec![],
         network_interfaces: interface_params,
         disks: vec![],
     };
@@ -1134,6 +1141,7 @@ async fn test_attach_one_disk_to_instance(cptestctx: &ControlPlaneTestContext) {
         ncpus: InstanceCpuCount::try_from(2).unwrap(),
         memory: ByteCount::from_mebibytes_u32(4),
         hostname: String::from("nfs"),
+        user_data: vec![],
         network_interfaces: params::InstanceNetworkInterfaceAttachment::Default,
         disks: vec![params::InstanceDiskAttachment::Attach(
             params::InstanceDiskAttach {
@@ -1229,6 +1237,7 @@ async fn test_attach_eight_disks_to_instance(
         ncpus: InstanceCpuCount::try_from(2).unwrap(),
         memory: ByteCount::from_mebibytes_u32(4),
         hostname: String::from("nfs"),
+        user_data: vec![],
         network_interfaces: params::InstanceNetworkInterfaceAttachment::Default,
         disks: (0..8)
             .map(|i| {
@@ -1334,6 +1343,7 @@ async fn test_cannot_attach_nine_disks_to_instance(
         ncpus: InstanceCpuCount::try_from(2).unwrap(),
         memory: ByteCount::from_mebibytes_u32(4),
         hostname: String::from("nfs"),
+        user_data: vec![],
         network_interfaces: params::InstanceNetworkInterfaceAttachment::Default,
         disks: (0..9)
             .map(|i| {
@@ -1460,6 +1470,7 @@ async fn test_cannot_attach_faulted_disks(cptestctx: &ControlPlaneTestContext) {
         ncpus: InstanceCpuCount::try_from(2).unwrap(),
         memory: ByteCount::from_mebibytes_u32(4),
         hostname: String::from("nfs"),
+        user_data: vec![],
         network_interfaces: params::InstanceNetworkInterfaceAttachment::Default,
         disks: (0..8)
             .map(|i| {

nexus/tests/integration_tests/subnet_allocation.rs

@@ -39,6 +39,7 @@ async fn create_instance_expect_failure(
         ncpus: InstanceCpuCount(1),
         memory: ByteCount::from_mebibytes_u32(256),
         hostname: name.to_string(),
+        user_data: vec![],
         network_interfaces: params::InstanceNetworkInterfaceAttachment::Default,
         disks: vec![],
     };