overlay: add overlay implementation

Cargo.toml

@@ -35,7 +35,7 @@ path = "src/lib.rs"
 anyhow = "1"
 clap = { version = "4.0.18", features = ["derive", "cargo"] }
 flexi_logger = { version = "0.25", features = ["compress"] }
-fuse-backend-rs = "^0.11.0"
+fuse-backend-rs = "^0.12.0"
 hex = "0.4.3"
 hyper = "0.14.11"
 hyperlocal = "0.8.0"

api/src/config.rs

@@ -29,6 +29,8 @@ pub struct ConfigV2 {
     pub cache: Option<CacheConfigV2>,
     /// Configuration information for RAFS filesystem.
     pub rafs: Option<RafsConfigV2>,
+    /// Overlay configuration information for the instance.
+    pub overlay: Option<OverlayConfig>,
     /// Internal runtime configuration.
     #[serde(skip)]
     pub internal: ConfigV2Internal,
@@ -42,6 +44,7 @@ impl Default for ConfigV2 {
             backend: None,
             cache: None,
             rafs: None,
+            overlay: None,
             internal: ConfigV2Internal::default(),
         }
     }
@@ -56,6 +59,7 @@ impl ConfigV2 {
             backend: None,
             cache: None,
             rafs: None,
+            overlay: None,
             internal: ConfigV2Internal::default(),
         }
     }
@@ -1024,6 +1028,7 @@ impl From<&BlobCacheEntryConfigV2> for ConfigV2 {
             backend: Some(c.backend.clone()),
             cache: Some(c.cache.clone()),
             rafs: None,
+            overlay: None,
             internal: ConfigV2Internal::default(),
         }
     }
@@ -1395,6 +1400,7 @@ impl TryFrom<RafsConfig> for ConfigV2 {
             backend: Some(backend),
             cache: Some(cache),
             rafs: Some(rafs),
+            overlay: None,
             internal: ConfigV2Internal::default(),
         })
     }
@@ -1523,6 +1529,15 @@ impl TryFrom<&BlobCacheEntryConfig> for BlobCacheEntryConfigV2 {
     }
 }
 
+/// Configuration information for Overlay filesystem.
+/// OverlayConfig is used to configure the writable layer(upper layer),
+/// The filesystem will be writable when OverlayConfig is set.
+#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
+pub struct OverlayConfig {
+    pub upper_dir: String,
+    pub work_dir: String,
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

builder/src/core/context.rs

@@ -1516,6 +1516,7 @@ mod tests {
             id: "id".to_owned(),
             cache: None,
             rafs: None,
+            overlay: None,
             internal: ConfigV2Internal {
                 blob_accessible: Arc::new(AtomicBool::new(true)),
             },

clib/Cargo.toml

@@ -15,7 +15,7 @@ crate-type = ["cdylib", "staticlib"]
 [dependencies]
 libc = "0.2.137"
 log = "0.4.17"
-fuse-backend-rs = "^0.11.0"
+fuse-backend-rs = "^0.12.0"
 nydus-api = { version = "0.3", path = "../api" }
 nydus-rafs = { version = "0.3.1", path = "../rafs" }
 nydus-storage = { version = "0.6.3", path = "../storage" }

docs/nydus-overlayfs.md

@@ -39,3 +39,4 @@ mount -t overlay overlay ./foo/merged -o lowerdir=./foo/lower2:./foo/lower1,uppe
 Meanwhile, when containerd passes the `nydus-snapshotter` mount slice to `containerd-shim-kata-v2`, it can parse the mount slice and pass the `extraoption` to `nydusd`, to support nydus image format natively.
 
 So in summary, `containerd` and `containerd-shim-runc-v2` rely on the `nydus-overlay` mount helper to handle the mount slice returned by `nydus-snapshotter`, while `containerd-shim-kata-v2` can parse and handle it on its own.
+

docs/nydusd.md

@@ -409,6 +409,53 @@ Once the configuration is loaded successfully on nydusd starting, we will see th
 INFO [storage/src/backend/connection.rs:136] backend config: CommonConfig { proxy: ProxyConfig { url: "http://p2p-proxy:65001", ping_url: "http://p2p-proxy:40901/server/ping", fallback: true, check_interval: 5 }, timeout: 5, connect_timeout: 5, retry_limit: 0 }
 ```
 
+### Mount writable Overlay FS
+
+`Nydusd` itself has a native userspace Overlay FS implementation, which can be enabled with several extra configurations. 
+
+An example configuration `/etc/nydus/nydusd-config.overlay.json` is as follows:
+
+```json
+{
+	"version": 2,
+	"backend": {
+		"type": "localfs",
+		"localfs": "/var/lib/nydus/blobs"
+	},
+	"cache": {
+		"type": "blobcache",
+		"filecache": {
+			"work_dir": "/var/lib/nydus/cache"
+		}
+	},
+	"rafs": {
+		"mode": "direct",
+		"enable_xattr": true
+	},
+	"overlay": {
+		"upper_dir": "/path/to/upperdir",
+		"work_dir": "/path/to/workdir"
+	}
+}
+```
+
+An extra field `overlay` is added to the Nydusd configuration, which specifies the `upper_dir` and `work_dir` for the overlay filesystem.
+
+You can start `nydusd` with the above configuration and such command:
+
+```bash
+sudo nydusd \
+    --config /etc/nydus/nydusd-config.overlay.json \
+    --mountpoint /path/to/mnt/ \
+    --bootstrap /path/to/bootstrap \
+    --log-level info \
+    --writable
+```
+
+This will create a FUSE overlay mountpoint at `/path/to/mnt/`, with one `Nydus` image as readonly lower layer and the `/path/to/upperdir` as writable upper layer, so that it can take over whole rootfs of a container, any contents writen from container will be stored in `/path/to/upperdir`.
+
+Removing `--writable` flag will make the overlay filesystem readonly if you wish.
+
 ### Mount Bootstrap Via API
 
 To mount a bootstrap via api, first launch nydusd without a bootstrap:

rafs/Cargo.toml

@@ -19,7 +19,7 @@ nix = "0.24"
 serde = { version = "1.0.110", features = ["serde_derive", "rc"] }
 serde_json = "1.0.53"
 vm-memory = "0.10"
-fuse-backend-rs = "^0.11.0"
+fuse-backend-rs = "^0.12.0"
 thiserror = "1"
 
 nydus-api = { version = "0.3", path = "../api" }

rafs/src/fs.rs

@@ -291,7 +291,8 @@ impl Rafs {
         // since nydusify gives root directory permission of 0o750 and fuse mount
         // options `rootmode=` does not affect root directory's permission bits, ending
         // up with preventing other users from accessing the container rootfs.
-        if attr.ino == self.root_ino() {
+        let root_ino = self.root_ino();
+        if attr.ino == root_ino {
             attr.mode = attr.mode & !0o777 | 0o755;
         }
 
@@ -684,9 +685,9 @@ impl FileSystem for Rafs {
         _inode: Self::Inode,
         _flags: u32,
         _fuse_flags: u32,
-    ) -> Result<(Option<Self::Handle>, OpenOptions)> {
+    ) -> Result<(Option<Self::Handle>, OpenOptions, Option<u32>)> {
         // Keep cache since we are readonly
-        Ok((None, OpenOptions::KEEP_CACHE))
+        Ok((None, OpenOptions::KEEP_CACHE, None))
     }
 
     fn release(
@@ -886,6 +887,14 @@ impl FileSystem for Rafs {
     }
 }
 
+#[cfg(target_os = "linux")]
+// Let Rafs works as an OverlayFs layer.
+impl Layer for Rafs {
+    fn root_inode(&self) -> Self::Inode {
+        self.root_ino()
+    }
+}
+
 #[cfg(all(test, feature = "backend-oss"))]
 pub(crate) mod tests {
     use super::*;

service/Cargo.toml

@@ -12,7 +12,7 @@ resolver = "2"
 [dependencies]
 bytes = { version = "1", optional = true }
 dbs-allocator = { version = "0.1.1", optional = true }
-fuse-backend-rs = { version = "^0.11.0", features = ["persist"] }
+fuse-backend-rs = { version = "^0.12.0", features = ["persist"] }
 libc = "0.2"
 log = "0.4.8"
 mio = { version = "0.8", features = ["os-poll", "os-ext"] }

service/src/fs_service.rs

@@ -13,10 +13,14 @@ use std::path::{Path, PathBuf};
 use std::str::FromStr;
 use std::sync::{Arc, MutexGuard};
 
+#[cfg(target_os = "linux")]
+use fuse_backend_rs::api::filesystem::{FileSystem, FsOptions, Layer};
 use fuse_backend_rs::api::vfs::VfsError;
 use fuse_backend_rs::api::{BackFileSystem, Vfs};
 #[cfg(target_os = "linux")]
-use fuse_backend_rs::passthrough::{Config, PassthroughFs};
+use fuse_backend_rs::overlayfs::{config::Config as overlay_config, OverlayFs};
+#[cfg(target_os = "linux")]
+use fuse_backend_rs::passthrough::{CachePolicy, Config as passthrough_config, PassthroughFs};
 use nydus_api::ConfigV2;
 use nydus_rafs::fs::Rafs;
 use nydus_rafs::{RafsError, RafsIoRead};
@@ -244,8 +248,77 @@ fn fs_backend_factory(cmd: &FsBackendMountCmd) -> Result<BackFileSystem> {
             let config = Arc::new(config);
             let (mut rafs, reader) = Rafs::new(&config, &cmd.mountpoint, Path::new(&cmd.source))?;
             rafs.import(reader, prefetch_files)?;
-            info!("RAFS filesystem imported");
-            Ok(Box::new(rafs))
+
+            // Put a writable upper layer above the rafs to create an OverlayFS with two layers.
+            match &config.overlay {
+                Some(ovl_conf) => {
+                    // check workdir and upperdir params.
+                    if ovl_conf.work_dir.is_empty() || ovl_conf.upper_dir.is_empty() {
+                        return Err(Error::InvalidArguments(String::from(
+                            "workdir and upperdir must be specified for overlayfs",
+                        )));
+                    }
+
+                    // Create an overlay upper layer with passthroughfs.
+                    #[cfg(target_os = "macos")]
+                    return Err(Error::InvalidArguments(String::from(
+                        "not support OverlayFs since passthroughfs isn't supported on MacOS",
+                    )));
+                    #[cfg(target_os = "linux")]
+                    {
+                        let fs_cfg = passthrough_config {
+                            // Use upper_dir as root_dir as rw layer.
+                            root_dir: ovl_conf.upper_dir.clone(),
+                            do_import: true,
+                            writeback: true,
+                            no_open: true,
+                            no_opendir: true,
+                            xattr: true,
+                            cache_policy: CachePolicy::Always,
+                            ..Default::default()
+                        };
+                        let fsopts = FsOptions::WRITEBACK_CACHE
+                            | FsOptions::ZERO_MESSAGE_OPEN
+                            | FsOptions::ZERO_MESSAGE_OPENDIR;
+
+                        let passthrough_fs = PassthroughFs::<()>::new(fs_cfg)
+                            .map_err(|e| Error::InvalidConfig(format!("{}", e)))?;
+                        passthrough_fs.init(fsopts).map_err(Error::PassthroughFs)?;
+
+                        type BoxedLayer = Box<dyn Layer<Inode = u64, Handle = u64> + Send + Sync>;
+                        let upper_layer = Arc::new(Box::new(passthrough_fs) as BoxedLayer);
+
+                        // Create overlay lower layer with rafs, use lower_dir as root_dir of rafs.
+                        let lower_layers = vec![Arc::new(Box::new(rafs) as BoxedLayer)];
+
+                        let overlay_config = overlay_config {
+                            work: ovl_conf.work_dir.clone(),
+                            mountpoint: cmd.mountpoint.clone(),
+                            do_import: false,
+                            no_open: true,
+                            no_opendir: true,
+                            ..Default::default()
+                        };
+                        let overlayfs =
+                            OverlayFs::new(Some(upper_layer), lower_layers, overlay_config)
+                                .map_err(|e| Error::InvalidConfig(format!("{}", e)))?;
+                        info!(
+                            "init overlay fs inode, upper {}, work {}\n",
+                            ovl_conf.upper_dir.clone(),
+                            ovl_conf.work_dir.clone()
+                        );
+                        overlayfs
+                            .import()
+                            .map_err(|e| Error::InvalidConfig(format!("{}", e)))?;
+                        info!("Overlay filesystem imported");
+                        Ok(Box::new(overlayfs))
+                    }
+                }
+                None => {
+                    info!("RAFS filesystem imported");
+                    Ok(Box::new(rafs))
+                }
+            }
         }
         FsBackendType::PassthroughFs => {
             #[cfg(target_os = "macos")]
@@ -257,7 +330,7 @@ fn fs_backend_factory(cmd: &FsBackendMountCmd) -> Result<BackFileSystem> {
                 // Vfs by default enables no_open and writeback, passthroughfs
                 // needs to specify them explicitly.
                 // TODO(liubo): enable no_open_dir.
-                let fs_cfg = Config {
+                let fs_cfg = passthrough_config {
                     root_dir: cmd.source.to_string(),
                     do_import: false,
                     writeback: true,