Automatically manage FEX rootfs

This adds a mechanism for automatically managing the FEX rootfs,
mounting and overlayfs'ing the relevant squashfs files from the host (as
many as desired).

Depends on:

 * libkrun containers/libkrun#217
 * libkrunfw containers/libkrunfw#65

Signed-off-by: Alyssa Rosenzweig <[email protected]>

crates/krun/src/bin/krun.rs

@@ -11,10 +11,10 @@ use krun::launch::{launch_or_lock, LaunchResult};
 use krun::net::{connect_to_passt, start_passt};
 use krun::types::MiB;
 use krun_sys::{
-    krun_add_vsock_port, krun_create_ctx, krun_set_exec, krun_set_gpu_options, krun_set_log_level,
-    krun_set_passt_fd, krun_set_root, krun_set_vm_config, krun_set_workdir, krun_start_enter,
-    VIRGLRENDERER_DRM, VIRGLRENDERER_THREAD_SYNC, VIRGLRENDERER_USE_ASYNC_FENCE_CB,
-    VIRGLRENDERER_USE_EGL,
+    krun_add_disk, krun_add_vsock_port, krun_create_ctx, krun_set_exec, krun_set_gpu_options,
+    krun_set_log_level, krun_set_passt_fd, krun_set_root, krun_set_vm_config, krun_set_workdir,
+    krun_start_enter, VIRGLRENDERER_DRM, VIRGLRENDERER_THREAD_SYNC,
+    VIRGLRENDERER_USE_ASYNC_FENCE_CB, VIRGLRENDERER_USE_EGL,
 };
 use log::debug;
 use nix::sys::sysinfo::sysinfo;
@@ -24,6 +24,23 @@ use rustix::process::{
     geteuid, getgid, getrlimit, getuid, sched_setaffinity, setrlimit, CpuSet, Resource,
 };
 
+fn add_ro_disk(ctx_id: u32, label: &str, path: &str) -> Result<()> {
+    let path_cstr = CString::new(path).unwrap();
+    let path_ptr = path_cstr.as_ptr();
+
+    let label_cstr = CString::new(label).unwrap();
+    let label_ptr = label_cstr.as_ptr();
+
+    // SAFETY: `path_ptr` and `label_ptr` are live pointers to C-strings
+    let err = unsafe { krun_add_disk(ctx_id, label_ptr, path_ptr, true) };
+
+    if err < 0 {
+        Err(Errno::from_raw_os_error(-err).into())
+    } else {
+        Ok(())
+    }
+}
+
 fn main() -> Result<()> {
     env_logger::init();
 
@@ -122,6 +139,32 @@ fn main() -> Result<()> {
         setrlimit(Resource::Nofile, rlim).context("Failed to raise `RLIMIT_NOFILE`")?;
     }
 
+    // If the user specified a disk image, we want to load and fail if it's missing. If the user
+    // did not specify a disk image, we want to load the system images if installed but fail
+    // gracefully if missing. This follows the principle of least surprise.
+    //
+    // What we don't want is a clever autodiscovery mechanism that searches $HOME for images.
+    // That's liable to blow up in exciting ways. Instead we require images to be selected
+    // explicitly, either on the CLI or hardcoded here.
+    let disks: Vec<String> = if !options.fex_images.is_empty() {
+        options.fex_images
+    } else {
+        let default_disks = vec![
+            "/usr/share/fex-emu/rootfs.ero",
+            "/usr/share/fex-emu/mesa.ero",
+        ];
+
+        default_disks
+            .iter()
+            .map(|x| x.to_string())
+            .filter(|x| Path::new(x).exists())
+            .collect()
+    };
+
+    for path in disks {
+        add_ro_disk(ctx_id, &path, &path).context("Failed to configure disk")?;
+    }
+
     {
         // SAFETY: `root_path` is a pointer to a C-string literal.
         let err = unsafe { krun_set_root(ctx_id, c"/".as_ptr()) };

crates/krun/src/cli_options.rs

@@ -13,6 +13,7 @@ pub struct Options {
     pub mem: Option<MiB>,
     pub passt_socket: Option<PathBuf>,
     pub server_port: u32,
+    pub fex_images: Vec<String>,
     pub command: PathBuf,
     pub command_args: Vec<String>,
 }
@@ -66,6 +67,15 @@ pub fn options() -> OptionParser<Options> {
         )
         .argument("MEM")
         .optional();
+    let fex_images = long("fex-image")
+        .short('f')
+        .help(
+            "Adds an erofs file to be mounted as a FEX rootfs.
+            May be specified multiple times.
+            First the base image, then overlays in order.",
+        )
+        .argument::<String>("FEX_IMAGE")
+        .many();
     let passt_socket = long("passt-socket")
         .help("Instead of starting passt, connect to passt socket at PATH")
         .argument("PATH")
@@ -89,6 +99,7 @@ pub fn options() -> OptionParser<Options> {
         mem,
         passt_socket,
         server_port,
+        fex_images,
         // positionals
         command,
         command_args,

crates/krun/src/guest/mount.rs

@@ -1,13 +1,72 @@
-use std::fs::File;
+use std::ffi::CString;
+use std::fs::{read_dir, File};
 use std::os::fd::AsFd;
 use std::path::Path;
 
 use anyhow::{Context, Result};
-use rustix::fs::CWD;
+use rustix::fs::{mkdir, symlink, Mode, CWD};
 use rustix::mount::{
-    mount2, mount_bind, move_mount, open_tree, MountFlags, MoveMountFlags, OpenTreeFlags,
+    mount2, mount_bind, move_mount, open_tree, MountFlags, MoveMountFlags, OpenTreeFlags
 };
 
+fn mkdir_fex(dir: &str) {
+    // Must succeed since /run/ was just mounted and is now an empty tmpfs.
+    mkdir(
+        dir,
+        Mode::RUSR | Mode::XUSR | Mode::RGRP | Mode::XGRP | Mode::ROTH | Mode::XOTH,
+    )
+    .unwrap();
+}
+
+fn mount_fex_rootfs() -> Result<()> {
+    let dir = "/run/fex-emu/";
+    let dir_rootfs = dir.to_string() + "rootfs";
+
+    // Make base directories
+    mkdir_fex(dir);
+
+    let flags = MountFlags::RDONLY;
+    let mut images = Vec::new();
+
+    // Find /dev/vd*
+    for x in read_dir("/dev").unwrap() {
+        let file = x.unwrap();
+        let name = file.file_name().into_string().unwrap();
+        if !name.starts_with("vd") {
+            continue;
+        }
+
+        let path = file.path().into_os_string().into_string().unwrap();
+        let dir = dir.to_string() + &name;
+
+        // Mount the erofs images.
+        mkdir_fex(&dir);
+        mount2(Some(path), dir.clone(), Some("erofs"), flags, None)
+            .context("Failed to mount erofs")
+            .unwrap();
+        images.push(dir);
+    }
+
+    if images.len() >= 2 {
+        // Overlay the mounts together.
+        let opts = format!(
+            "lowerdir={}",
+            images.into_iter().rev().collect::<Vec<String>>().join(":")
+        );
+        let opts = CString::new(opts).unwrap();
+        let overlay = "overlay".to_string();
+        let overlay_ = Some(&overlay);
+
+        mkdir_fex(&dir_rootfs);
+        mount2(overlay_, &dir_rootfs, overlay_, flags, Some(&opts)).context("Failed to overlay")?;
+    } else if images.len() == 1 {
+        // Just expose the one mount
+        symlink(&images[0], &dir_rootfs)?;
+    }
+
+    Ok(())
+}
+
 pub fn mount_filesystems() -> Result<()> {
     mount2(
         Some("tmpfs"),
@@ -18,6 +77,10 @@ pub fn mount_filesystems() -> Result<()> {
     )
     .context("Failed to mount `/var/run`")?;
 
+    if let Err(_) = mount_fex_rootfs() {
+        println!("Failed to mount FEX rootfs, carrying on without.")
+    }
+
     let _ = File::options()
         .write(true)
         .create(true)