Skip to content

Commit

Permalink
overlay: add overlay implementation
Browse files Browse the repository at this point in the history
With help of newly introduced Overlay FileSystem in `fuse-backend-rs`
library, now we can create writable rootfs in Nydus. Implementation of
writable rootfs is based on one passthrough FS(as upper layer) over one
readonly rafs(as lower layer).

To do so, configuration is extended with some Overlay options.

Signed-off-by: Wei Zhang <[email protected]>
  • Loading branch information
WeiZhang555 committed Mar 7, 2024
1 parent eba6afe commit 157da95
Show file tree
Hide file tree
Showing 12 changed files with 321 additions and 21 deletions.
29 changes: 27 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,9 @@ time = { version = "0.3.14", features = ["formatting"] }
xattr = "1.0.1"
vmm-sys-util = "0.11.0"

[patch.crates-io]
fuse-backend-rs = { git = 'https://github.com/weizhang555/fuse-backend-rs.git', branch = 'overlay-impl' }

[features]
default = [
"fuse-backend-rs/fusedev",
Expand Down
15 changes: 15 additions & 0 deletions api/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ pub struct ConfigV2 {
pub cache: Option<CacheConfigV2>,
/// Configuration information for RAFS filesystem.
pub rafs: Option<RafsConfigV2>,
/// Overlay configuration information for the instance.
pub overlay: Option<OverlayConfig>,
/// Internal runtime configuration.
#[serde(skip)]
pub internal: ConfigV2Internal,
Expand All @@ -42,6 +44,7 @@ impl Default for ConfigV2 {
backend: None,
cache: None,
rafs: None,
overlay: None,
internal: ConfigV2Internal::default(),
}
}
Expand All @@ -56,6 +59,7 @@ impl ConfigV2 {
backend: None,
cache: None,
rafs: None,
overlay: None,
internal: ConfigV2Internal::default(),
}
}
Expand Down Expand Up @@ -1024,6 +1028,7 @@ impl From<&BlobCacheEntryConfigV2> for ConfigV2 {
backend: Some(c.backend.clone()),
cache: Some(c.cache.clone()),
rafs: None,
overlay: None,
internal: ConfigV2Internal::default(),
}
}
Expand Down Expand Up @@ -1395,6 +1400,7 @@ impl TryFrom<RafsConfig> for ConfigV2 {
backend: Some(backend),
cache: Some(cache),
rafs: Some(rafs),
overlay: None,
internal: ConfigV2Internal::default(),
})
}
Expand Down Expand Up @@ -1523,6 +1529,15 @@ impl TryFrom<&BlobCacheEntryConfig> for BlobCacheEntryConfigV2 {
}
}

/// Configuration information for Overlay filesystem.
/// OverlayConfig is used to configure the writable layer(upper layer),
/// The filesystem will be writable when OverlayConfig is set.
#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
pub struct OverlayConfig {
pub upper_dir: String,
pub work_dir: String,
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down
1 change: 1 addition & 0 deletions builder/src/core/context.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1516,6 +1516,7 @@ mod tests {
id: "id".to_owned(),
cache: None,
rafs: None,
overlay: None,
internal: ConfigV2Internal {
blob_accessible: Arc::new(AtomicBool::new(true)),
},
Expand Down
15 changes: 12 additions & 3 deletions rafs/src/fs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -291,7 +291,8 @@ impl Rafs {
// since nydusify gives root directory permission of 0o750 and fuse mount
// options `rootmode=` does not affect root directory's permission bits, ending
// up with preventing other users from accessing the container rootfs.
if attr.ino == self.root_ino() {
let root_ino = self.root_ino();
if attr.ino == root_ino {
attr.mode = attr.mode & !0o777 | 0o755;
}

Expand Down Expand Up @@ -684,9 +685,9 @@ impl FileSystem for Rafs {
_inode: Self::Inode,
_flags: u32,
_fuse_flags: u32,
) -> Result<(Option<Self::Handle>, OpenOptions)> {
) -> Result<(Option<Self::Handle>, OpenOptions, Option<u32>)> {
// Keep cache since we are readonly
Ok((None, OpenOptions::KEEP_CACHE))
Ok((None, OpenOptions::KEEP_CACHE, None))
}

fn release(
Expand Down Expand Up @@ -886,6 +887,14 @@ impl FileSystem for Rafs {
}
}

#[cfg(target_os = "linux")]
// Let Rafs works as an OverlayFs layer.
impl Layer for Rafs {
fn root_inode(&self) -> Self::Inode {
self.root_ino()
}
}

#[cfg(all(test, feature = "backend-oss"))]
pub(crate) mod tests {
use super::*;
Expand Down
81 changes: 77 additions & 4 deletions service/src/fs_service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,14 @@ use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::sync::{Arc, MutexGuard};

#[cfg(target_os = "linux")]
use fuse_backend_rs::api::filesystem::{FileSystem, FsOptions, Layer};
use fuse_backend_rs::api::vfs::VfsError;
use fuse_backend_rs::api::{BackFileSystem, Vfs};
#[cfg(target_os = "linux")]
use fuse_backend_rs::passthrough::{Config, PassthroughFs};
use fuse_backend_rs::overlayfs::{config::Config as overlay_config, OverlayFs};
#[cfg(target_os = "linux")]
use fuse_backend_rs::passthrough::{CachePolicy, Config as passthrough_config, PassthroughFs};
use nydus_api::ConfigV2;
use nydus_rafs::fs::Rafs;
use nydus_rafs::{RafsError, RafsIoRead};
Expand Down Expand Up @@ -244,8 +248,77 @@ fn fs_backend_factory(cmd: &FsBackendMountCmd) -> Result<BackFileSystem> {
let config = Arc::new(config);
let (mut rafs, reader) = Rafs::new(&config, &cmd.mountpoint, Path::new(&cmd.source))?;
rafs.import(reader, prefetch_files)?;
info!("RAFS filesystem imported");
Ok(Box::new(rafs))

// Put a writable upper layer above the rafs to create an OverlayFS with two layers.
match &config.overlay {
Some(ovl_conf) => {
// check workdir and upperdir params.
if ovl_conf.work_dir.is_empty() || ovl_conf.upper_dir.is_empty() {
return Err(Error::InvalidArguments(String::from(
"workdir and upperdir must be specified for overlayfs",
)));
}

// Create an overlay upper layer with passthroughfs.
#[cfg(target_os = "macos")]
return Err(Error::InvalidArguments(String::from(
"not support OverlayFs since passthroughfs isn't supported on MacOS",
)));
#[cfg(target_os = "linux")]
{
let fs_cfg = passthrough_config {
// Use upper_dir as root_dir as rw layer.
root_dir: ovl_conf.upper_dir.clone(),
do_import: true,
writeback: true,
no_open: true,
no_opendir: true,
xattr: true,
cache_policy: CachePolicy::Always,
..Default::default()
};
let fsopts = FsOptions::WRITEBACK_CACHE
| FsOptions::ZERO_MESSAGE_OPEN
| FsOptions::ZERO_MESSAGE_OPENDIR;

let passthrough_fs = PassthroughFs::<()>::new(fs_cfg)
.map_err(|e| Error::InvalidConfig(format!("{}", e)))?;
passthrough_fs.init(fsopts).map_err(Error::PassthroughFs)?;

type BoxedLayer = Box<dyn Layer<Inode = u64, Handle = u64> + Send + Sync>;
let upper_layer = Arc::new(Box::new(passthrough_fs) as BoxedLayer);

// Create overlay lower layer with rafs, use lower_dir as root_dir of rafs.
let lower_layers = vec![Arc::new(Box::new(rafs) as BoxedLayer)];

let overlay_config = overlay_config {
work: ovl_conf.work_dir.clone(),
mountpoint: cmd.mountpoint.clone(),
do_import: false,
no_open: true,
no_opendir: true,
..Default::default()
};
let overlayfs =
OverlayFs::new(Some(upper_layer), lower_layers, overlay_config)
.map_err(|e| Error::InvalidConfig(format!("{}", e)))?;
info!(
"init overlay fs inode, upper {}, work {}\n",
ovl_conf.upper_dir.clone(),
ovl_conf.work_dir.clone()
);
overlayfs
.import()
.map_err(|e| Error::InvalidConfig(format!("{}", e)))?;
info!("Overlay filesystem imported");
Ok(Box::new(overlayfs))
}
}
None => {
info!("RAFS filesystem imported");
Ok(Box::new(rafs))
}
}
}
FsBackendType::PassthroughFs => {
#[cfg(target_os = "macos")]
Expand All @@ -257,7 +330,7 @@ fn fs_backend_factory(cmd: &FsBackendMountCmd) -> Result<BackFileSystem> {
// Vfs by default enables no_open and writeback, passthroughfs
// needs to specify them explicitly.
// TODO(liubo): enable no_open_dir.
let fs_cfg = Config {
let fs_cfg = passthrough_config {
root_dir: cmd.source.to_string(),
do_import: false,
writeback: true,
Expand Down
110 changes: 110 additions & 0 deletions smoke/tests/overlay_fs_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
// Copyright 2023 Nydus Developers. All rights reserved.
//
// SPDX-License-Identifier: Apache-2.0

package tests

import (
"fmt"
"os"
"path/filepath"
"testing"

"github.com/containerd/log"
"github.com/containerd/nydus-snapshotter/pkg/converter"
"github.com/dragonflyoss/nydus/smoke/tests/texture"
"github.com/dragonflyoss/nydus/smoke/tests/tool"
"github.com/dragonflyoss/nydus/smoke/tests/tool/test"
"github.com/opencontainers/go-digest"
"github.com/stretchr/testify/require"
)

type OverlayFsTestSuite struct {
t *testing.T
}

func (ovl_ts *OverlayFsTestSuite) prepareTestEnv(t *testing.T) *tool.Context {
ctx := tool.DefaultContext(t)
ctx.PrepareWorkDir(t)

packOption := converter.PackOption{
BuilderPath: ctx.Binary.Builder,
Compressor: ctx.Build.Compressor,
FsVersion: ctx.Build.FSVersion,
ChunkSize: ctx.Build.ChunkSize,
}

lowerLayer := texture.MakeLowerLayer(t, filepath.Join(ctx.Env.WorkDir, "lower"))
lowerBlobDigest := lowerLayer.Pack(t, packOption, ctx.Env.BlobDir)
mergeOption := converter.MergeOption{
BuilderPath: ctx.Binary.Builder,
ChunkDictPath: "",
OCIRef: true,
}
actualDigests, lowerBootstrap := tool.MergeLayers(t, *ctx, mergeOption, []converter.Layer{
{
Digest: lowerBlobDigest,
},
})
require.Equal(t, []digest.Digest{lowerBlobDigest}, actualDigests)

// Verify lower layer mounted by nydusd
ctx.Env.BootstrapPath = lowerBootstrap
tool.Verify(t, *ctx, lowerLayer.FileTree)
return ctx
}

func (ovl_ts *OverlayFsTestSuite) TestSimpleOverlayFs(t *testing.T) {
ctx := ovl_ts.prepareTestEnv(t)
fmt.Printf("Workdir is %v\n", ctx.Env.WorkDir)
//defer ctx.Destroy(t)

nydusd, err := tool.NewNydusdWithOverlay(tool.NydusdConfig{
NydusdPath: ctx.Binary.Nydusd,
BootstrapPath: ctx.Env.BootstrapPath,
ConfigPath: filepath.Join(ctx.Env.WorkDir, "nydusd-config.fusedev.json"),
MountPath: ctx.Env.MountDir,
APISockPath: filepath.Join(ctx.Env.WorkDir, "nydusd-api.sock"),
BackendType: "localfs",
BackendConfig: fmt.Sprintf(`{"dir": "%s"}`, ctx.Env.BlobDir),
EnablePrefetch: ctx.Runtime.EnablePrefetch,
BlobCacheDir: ctx.Env.CacheDir,
CacheType: ctx.Runtime.CacheType,
CacheCompressed: ctx.Runtime.CacheCompressed,
RafsMode: ctx.Runtime.RafsMode,
OvlUpperDir: ctx.Env.OvlUpperDir,
OvlWorkDir: ctx.Env.OvlWorkDir,
DigestValidate: false,
Writable: true,
})
require.NoError(t, err)

err = nydusd.Mount()
require.NoError(t, err)
defer func() {
if err := nydusd.Umount(); err != nil {
log.L.WithError(err).Errorf("umount")
}
}()

// Write some file under mounted dir.
mountedDir := ctx.Env.MountDir
file := filepath.Join(mountedDir, "test.txt")
err = os.WriteFile(file, []byte("hello world"), 0644)
require.NoError(t, err)

// Read it back
data, err := os.ReadFile(file)
require.NoError(t, err)
require.Equal(t, "hello world", string(data))

// Try to read from upper dir.
upperFile := filepath.Join(ctx.Env.OvlUpperDir, "test.txt")
data, err = os.ReadFile(upperFile)
require.NoError(t, err)
require.Equal(t, "hello world", string(data))
}

func TestOverlayFs(t *testing.T) {
test.Run(t, &OverlayFsTestSuite{t: t})
}
Loading

0 comments on commit 157da95

Please sign in to comment.