User data support #911
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| use crate::db::{identity::Resource, model::Instance}; | ||
| use fatfs::{FileSystem, FormatVolumeOptions, FsOptions}; | ||
| use omicron_common::api::external::Error; | ||
| use serde::Serialize; | ||
| use std::io::{Cursor, Write}; | ||
| use uuid::Uuid; | ||
|
|
||
| impl Instance { | ||
| pub fn generate_cidata(&self) -> Result<Vec<u8>, Error> { | ||
| // cloud-init meta-data is YAML, but YAML is a strict superset of JSON. | ||
| let meta_data = serde_json::to_vec(&MetaData { | ||
| instance_id: self.id(), | ||
| local_hostname: &self.runtime().hostname, | ||
| public_keys: "", // TODO | ||
| }) | ||
| .map_err(|_| Error::internal_error("failed to serialize meta-data"))?; | ||
| let cidata = | ||
| build_vfat(&meta_data, &self.user_data).map_err(|err| { | ||
| Error::internal_error(&format!( | ||
| "failed to create cidata volume: {}", | ||
| err | ||
| )) | ||
| })?; | ||
| Ok(cidata) | ||
| } | ||
| } | ||
|
|
||
| #[derive(Serialize)] | ||
| #[serde(rename_all = "kebab-case")] | ||
| struct MetaData<'a> { | ||
| instance_id: Uuid, | ||
| local_hostname: &'a str, | ||
| public_keys: &'a str, | ||
iliana marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| } | ||
|
|
||
| fn build_vfat(meta_data: &[u8], user_data: &[u8]) -> std::io::Result<Vec<u8>> { | ||
| // requires #![feature(int_roundings)]. | ||
| // https://github.com/rust-lang/rust/issues/88581 | ||
| let file_sectors = meta_data.len().unstable_div_ceil(512) | ||
| + user_data.len().unstable_div_ceil(512); | ||
| // this was reverse engineered by making the numbers go lower until the | ||
| // code failed (usually because of low disk space). this only works for | ||
| // FAT12 filesystems | ||
| let sectors = 42.max(file_sectors + 35 + ((file_sectors + 1) / 341 * 2)); | ||
|
There was a problem hiding this comment. what do the numbers in this calculation represent? my FAT* filesystem ignorant read is "the lowest number of sectors we support is 42, and otherwise we have to compute enough space for FAT* formatting / metadata plus our file data", but that implies that FAT* needs 35 sectors for something? There was a problem hiding this comment. I actually have no idea because the only reference I could easily find is [[Design of the FAT file system]], which I repeatedly tried to read and my eyes kept falling off the page. I wrote some code using fatfs to find out what the smallest numbers are that work with it. This more or less spells out to:
There was a problem hiding this comment. I recommend https://www.win.tue.nl/~aeb/linux/fs/fat/fatgen103.pdf as a reference - it is the FAT spec. Page 14 has a section on FAT type determination:
Since it seems we're seeing It also mentions the calculation: I kinda despise this calculation, because to solve the equation for "am I using a FAT12 filesystem", you must first provide input that knows if you are using FAT12/FAT16/FAT32. For example, (This is literally what the fatfs crate does - guess, then check) They even acknowledge this is kinda fucked up on page 19, under "FAT Volume Initialization":
So, anyway, "worked out by hand on a piece of paper" aside, we can make some assumptions and try calculating this for a limited size:
So, anyway, how do we calculate the size of the actual "FAT"s on the FS? With a "simple" calculation (see: So, recalling our calculation earlier:
So: TotalSec = 35 + DataSec, if we're storing < 170 KiB of clusters TL:DR:If we think it's okay to put a bound on user_data + meta_data, my inclination would be:
There was a problem hiding this comment. Though reading your comment earlier, it seems like you experimentally noticed the minimum FS size was 42, no? Any idea where that's coming from? There was a problem hiding this comment. No idea where the 42 is coming from, no. But if you try to give fatfs fewer than 42 sectors, it gives you an "unfortunate disk size" error. |
||
|
|
||
| let mut disk = Cursor::new(vec![0; sectors * 512]); | ||
| fatfs::format_volume( | ||
| &mut disk, | ||
| FormatVolumeOptions::new() | ||
|
There was a problem hiding this comment. We can explicitly specify that this needs to be FAT12 using There was a problem hiding this comment. Edit; Unless we think it's okay to impose an upper size bound, then setting the fat type would be totally reasonable here. |
||
| .bytes_per_cluster(512) | ||
| .volume_label(*b"CIDATA "), | ||
| )?; | ||
|
|
||
| { | ||
| let fs = FileSystem::new(&mut disk, FsOptions::new())?; | ||
| let root_dir = fs.root_dir(); | ||
| for (file, data) in [("meta-data", meta_data), ("user-data", user_data)] | ||
| { | ||
| if !data.is_empty() { | ||
| let mut file = root_dir.create_file(file)?; | ||
| file.write_all(data)?; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| Ok(disk.into_inner()) | ||
| } | ||
|
|
||
| #[cfg(test)] | ||
| mod tests { | ||
| /// the fatfs crate has some unfortunate panics if you ask it to do | ||
| /// incredibly stupid things, like format a filesystem with an invalid | ||
| /// cluster size. | ||
| /// | ||
| /// to ensure that our math for the filesystem size is correct, and also to | ||
| /// ensure that we don't ask fatfs to do incredibly stupid things, this | ||
| /// test checks that `build_vfat` works on a representative sample of weird | ||
| /// file sizes. (32 KiB is our enforced limit for user_data, so push it a | ||
| /// little further.) | ||
| #[test] | ||
| fn build_vfat_works_with_arbitrarily_sized_input() { | ||
| // somewhat arbitrarily-chosen prime numbers near 1 KiB and 256 bytes | ||
| for md_size in (0..36 * 1024).step_by(1019) { | ||
| for ud_size in (0..36 * 1024).step_by(269) { | ||
| assert!(super::build_vfat( | ||
| &vec![0x5a; md_size], | ||
| &vec![0xa5; ud_size] | ||
| ) | ||
| .is_ok()); | ||
| } | ||
| } | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
https://noyaml.com/
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🙏