From f02521659db8ea543b803b6f095082ac96365a13 Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 17 Dec 2024 10:57:38 +0800 Subject: [PATCH 01/47] Add SimpleFilesystem --- clients/filesystem-fuse/.cargo/config.toml | 1 - clients/filesystem-fuse/Cargo.toml | 6 +- clients/filesystem-fuse/Makefile | 70 +++ clients/filesystem-fuse/build.gradle.kts | 32 +- clients/filesystem-fuse/src/filesystem.rs | 427 +++++++++++++++++- .../filesystem-fuse/src/fuse_api_handle.rs | 7 +- clients/filesystem-fuse/src/lib.rs | 2 + clients/filesystem-fuse/src/main.rs | 2 + .../src/opened_file_manager.rs | 65 +++ clients/filesystem-fuse/src/utils.rs | 35 ++ 10 files changed, 625 insertions(+), 22 deletions(-) create mode 100644 clients/filesystem-fuse/Makefile create mode 100644 clients/filesystem-fuse/src/opened_file_manager.rs create mode 100644 clients/filesystem-fuse/src/utils.rs diff --git a/clients/filesystem-fuse/.cargo/config.toml b/clients/filesystem-fuse/.cargo/config.toml index 78bc9f7fe48..9d5bb048edc 100644 --- a/clients/filesystem-fuse/.cargo/config.toml +++ b/clients/filesystem-fuse/.cargo/config.toml @@ -16,5 +16,4 @@ # under the License. [build] -target-dir = "build" rustflags = ["-Adead_code", "-Aclippy::redundant-field-names"] diff --git a/clients/filesystem-fuse/Cargo.toml b/clients/filesystem-fuse/Cargo.toml index 2883cecc656..3bcf20f37ef 100644 --- a/clients/filesystem-fuse/Cargo.toml +++ b/clients/filesystem-fuse/Cargo.toml @@ -30,13 +30,15 @@ name = "gvfs-fuse" path = "src/main.rs" [lib] -name="gvfs_fuse" +name = "gvfs_fuse" [dependencies] async-trait = "0.1" bytes = "1.6.0" -futures-util = "0.3.30" +dashmap = "6.1.0" fuse3 = { version = "0.8.1", "features" = ["tokio-runtime", "unprivileged"] } +futures-util = "0.3.30" +libc = "0.2.168" log = "0.4.22" tokio = { version = "1.38.0", features = ["full"] } tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } diff --git a/clients/filesystem-fuse/Makefile b/clients/filesystem-fuse/Makefile new file mode 100644 index 00000000000..2e48a3e1fd8 --- /dev/null +++ b/clients/filesystem-fuse/Makefile @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +.EXPORT_ALL_VARIABLES: + +.PHONY: build +build: + cargo build --all-features --workspace + +fmt: + cargo fmt --all + +cargo-sort: install-cargo-sort + cargo sort -w + +check-fmt: + cargo fmt --all -- --check + +check-clippy: + #cargo clippy --all-targets --all-features --workspace -- -D warnings + cargo clippy --all-targets --all-features --workspace -- + +install-cargo-sort: + cargo install cargo-sort@1.0.9 + +check-cargo-sort: install-cargo-sort + cargo sort -c + +install-cargo-machete: + cargo install cargo-machete + +cargo-machete: install-cargo-machete + cargo machete + +install-taplo-cli: + cargo install taplo-cli@0.9.0 + +fix-toml: install-taplo-cli + taplo fmt + +check-toml: install-taplo-cli + taplo check + +check: check-fmt check-clippy check-cargo-sort check-toml cargo-machete + +doc-test: + cargo test --no-fail-fast --doc --all-features --workspace + +unit-test: doc-test + cargo test --no-fail-fast --lib --all-features --workspace + +test: doc-test + cargo test --no-fail-fast --all-targets --all-features --workspace + +clean: + cargo clean diff --git a/clients/filesystem-fuse/build.gradle.kts b/clients/filesystem-fuse/build.gradle.kts index 08693ddc5bd..201397c19e7 100644 --- a/clients/filesystem-fuse/build.gradle.kts +++ b/clients/filesystem-fuse/build.gradle.kts @@ -32,7 +32,7 @@ val buildRustProject by tasks.registering(Exec::class) { dependsOn(checkRustEnvironment) description = "Compile the Rust project" workingDir = file("$projectDir") - commandLine("bash", "-c", "cargo build --release") + commandLine("bash", "-c", "make build") } val checkRustProject by tasks.registering(Exec::class) { @@ -40,18 +40,7 @@ val checkRustProject by tasks.registering(Exec::class) { description = "Check the Rust project" workingDir = file("$projectDir") - commandLine( - "bash", - "-c", - """ - set -e - echo "Checking the code format" - cargo fmt --all -- --check - - echo "Running clippy" - cargo clippy --all-targets --all-features --workspace -- -D warnings - """.trimIndent() - ) + commandLine( "bash", "-c", "make check") } val testRustProject by tasks.registering(Exec::class) { @@ -59,7 +48,18 @@ val testRustProject by tasks.registering(Exec::class) { description = "Run tests in the Rust project" group = "verification" workingDir = file("$projectDir") - commandLine("bash", "-c", "cargo test --release") + commandLine("bash", "-c", "make test") + + standardOutput = System.out + errorOutput = System.err +} + +val cleanRustProject by tasks.registering(Exec::class) { + dependsOn(checkRustEnvironment) + description = "Run tests in the Rust project" + group = "verification" + workingDir = file("$projectDir") + commandLine("bash", "-c", "make clean") standardOutput = System.out errorOutput = System.err @@ -85,3 +85,7 @@ tasks.named("check") { tasks.named("test") { dependsOn(testRustProject) } + +tasks.named("clean") { + dependsOn(cleanRustProject) +} diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 6d1d8fa2538..6a2bf80c6dc 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -16,9 +16,15 @@ * specific language governing permissions and limitations * under the License. */ +use crate::opened_file_manager::OpenedFileManager; +use crate::utils::{join_file_path, split_file_path}; use async_trait::async_trait; use bytes::Bytes; use fuse3::{Errno, FileType, Timestamp}; +use std::collections::HashMap; +use std::sync::atomic::AtomicU64; +use std::time::SystemTime; +use tokio::sync::RwLock; pub(crate) type Result = std::result::Result; @@ -174,9 +180,6 @@ pub struct FileStat { // file type like regular file or directory and so on pub(crate) kind: FileType, - // file permission - pub(crate) perm: u16, - // file access time pub(crate) atime: Timestamp, @@ -190,6 +193,48 @@ pub struct FileStat { pub(crate) nlink: u32, } +impl FileStat { + pub fn new_file_with_path(path: &str, size: u64) -> Self { + let (parent, name) = split_file_path(path); + Self::new_file(parent, name, size) + } + + pub fn new_dir_with_path(path: &str) -> Self { + let (parent, name) = split_file_path(path); + Self::new_dir(parent, name) + } + + pub fn new_file(parent: &str, name: &str, size: u64) -> Self { + Self::new_file_entry(parent, name, size, FileType::RegularFile) + } + + pub fn new_dir(parent: &str, name: &str) -> Self { + Self::new_file_entry(parent, name, 0, FileType::Directory) + } + + pub fn new_file_entry(parent: &str, name: &str, size: u64, kind: FileType) -> Self { + let atime = Timestamp::from(SystemTime::now()); + Self { + file_id: 0, + parent_file_id: 0, + name: name.into(), + path: join_file_path(parent, name), + size: size, + kind: kind, + atime: atime, + mtime: atime, + ctime: atime, + nlink: 1, + } + } + + pub(crate) fn set_file_id(&mut self, parent_file_id: u64, file_id: u64) { + debug_assert!(file_id != 0 && parent_file_id != 0); + self.parent_file_id = parent_file_id; + self.file_id = file_id; + } +} + /// Opened file for read or write, it is used to read or write the file content. pub(crate) struct OpenedFile { pub(crate) file_stat: FileStat, @@ -201,6 +246,66 @@ pub(crate) struct OpenedFile { pub writer: Option>, } +impl OpenedFile { + pub fn new(file_stat: FileStat) -> Self { + OpenedFile { + file_stat: file_stat, + handle_id: 0, + reader: None, + writer: None, + } + } + + async fn read(&mut self, offset: u64, size: u32) -> Result { + self.file_stat.atime = Timestamp::from(SystemTime::now()); + self.reader.as_mut().unwrap().read(offset, size).await + } + + async fn write(&mut self, offset: u64, data: &[u8]) -> Result { + let end = offset + data.len() as u64; + + if end > self.file_stat.size { + self.file_stat.size = end; + } + self.file_stat.atime = Timestamp::from(SystemTime::now()); + self.file_stat.mtime = self.file_stat.atime; + + self.writer.as_mut().unwrap().write(offset, data).await + } + + async fn close(&mut self) -> Result<()> { + if let Some(mut reader) = self.reader.take() { + reader.close().await?; + } + if let Some(mut writer) = self.writer.take() { + self.flush().await?; + writer.close().await? + } + Ok(()) + } + + async fn flush(&mut self) -> Result<()> { + if let Some(writer) = &mut self.writer { + writer.flush().await?; + } + Ok(()) + } + + fn file_handle(&self) -> FileHandle { + debug_assert!(self.handle_id != 0); + debug_assert!(self.file_stat.file_id != 0); + FileHandle { + file_id: self.file_stat.file_id, + handle_id: self.handle_id, + } + } + + pub(crate) fn set_file_id(&mut self, parent_file_id: u64, file_id: u64) { + debug_assert!(file_id != 0 && parent_file_id != 0); + self.file_stat.set_file_id(parent_file_id, file_id) + } +} + // FileHandle is the file handle for the opened file. pub(crate) struct FileHandle { pub(crate) file_id: u64, @@ -239,3 +344,319 @@ pub trait FileWriter: Sync + Send { Ok(()) } } + +/// SimpleFileSystem is a simple implementation for the file system. +/// it is used to manage the file metadata and file handle. +/// The operations of the file system are implemented by the PathFileSystem. +/// Note: This class is not use in the production code, it is used for the demo and testing +pub struct SimpleFileSystem { + /// file entries + file_entry_manager: RwLock, + /// opened files + opened_file_manager: OpenedFileManager, + /// inode id generator + inode_id_generator: AtomicU64, + + /// real system + fs: T, +} + +impl SimpleFileSystem { + const INITIAL_INODE_ID: u64 = 10000; + const ROOT_DIR_PARENT_FILE_ID: u64 = 0; + const ROOT_DIR_FILE_ID: u64 = 1; + const ROOT_DIR_NAME: &'static str = ""; + + pub(crate) fn new(fs: T) -> Self { + Self { + file_entry_manager: RwLock::new(FileEntryManager::new()), + opened_file_manager: OpenedFileManager::new(), + inode_id_generator: AtomicU64::new(Self::INITIAL_INODE_ID), + fs, + } + } + + fn next_inode_id(&self) -> u64 { + self.inode_id_generator + .fetch_add(1, std::sync::atomic::Ordering::SeqCst) + } + + async fn get_file_node(&self, file_id: u64) -> Result { + self.file_entry_manager + .read() + .await + .get_file_by_id(file_id) + .ok_or(Errno::from(libc::ENOENT)) + } + + async fn get_file_node_by_path(&self, path: &str) -> Option { + self.file_entry_manager.read().await.get_file_by_name(path) + } + + async fn fill_file_id(&self, file_stat: &mut FileStat, parent_file_id: u64) { + let mut file_manager = self.file_entry_manager.write().await; + let file = file_manager.get_file_by_name(&file_stat.path); + match file { + None => { + file_stat.set_file_id(parent_file_id, self.next_inode_id()); + file_manager.insert(file_stat.parent_file_id, file_stat.file_id, &file_stat.path); + } + Some(file) => { + file_stat.set_file_id(file.parent_file_id, file.file_id); + } + } + } + + async fn open_file_internal( + &self, + file_id: u64, + flags: u32, + kind: FileType, + ) -> Result { + let file_node = self.get_file_node(file_id).await?; + + let mut file = { + match kind { + FileType::Directory => { + self.fs + .open_dir(&file_node.file_name, OpenFileFlags(flags)) + .await? + } + FileType::RegularFile => { + self.fs + .open_file(&file_node.file_name, OpenFileFlags(flags)) + .await? + } + _ => return Err(Errno::from(libc::EINVAL)), + } + }; + file.set_file_id(file_node.parent_file_id, file_id); + let file = self.opened_file_manager.put_file(file); + let file = file.lock().await; + Ok(file.file_handle()) + } +} + +#[async_trait] +impl RawFileSystem for SimpleFileSystem { + async fn init(&self) -> Result<()> { + self.file_entry_manager.write().await.insert( + Self::ROOT_DIR_PARENT_FILE_ID, + Self::ROOT_DIR_FILE_ID, + Self::ROOT_DIR_NAME, + ); + self.fs.init().await + } + + async fn get_file_path(&self, file_id: u64) -> String { + let file = self.get_file_node(file_id).await; + file.map(|x| x.file_name).unwrap_or_else(|_| "".to_string()) + } + + async fn valid_file_id(&self, _file_id: u64, fh: u64) -> Result<()> { + let file_id = self + .opened_file_manager + .get_file(fh) + .ok_or(Errno::from(libc::EBADF))? + .lock() + .await + .file_stat + .file_id; + + (file_id == _file_id) + .then_some(()) + .ok_or(Errno::from(libc::EBADF)) + } + + async fn stat(&self, file_id: u64) -> Result { + let file_node = self.get_file_node(file_id).await?; + let mut stat = self.fs.stat(&file_node.file_name).await?; + stat.set_file_id(file_node.parent_file_id, file_node.file_id); + Ok(stat) + } + + async fn lookup(&self, parent_file_id: u64, name: &str) -> Result { + let parent_file_node = self.get_file_node(parent_file_id).await?; + let mut stat = self.fs.lookup(&parent_file_node.file_name, name).await?; + self.fill_file_id(&mut stat, parent_file_id).await; + Ok(stat) + } + + async fn read_dir(&self, file_id: u64) -> Result> { + let file_node = self.get_file_node(file_id).await?; + let mut files = self.fs.read_dir(&file_node.file_name).await?; + for file in files.iter_mut() { + self.fill_file_id(file, file_node.file_id).await; + } + Ok(files) + } + + async fn open_file(&self, file_id: u64, flags: u32) -> Result { + self.open_file_internal(file_id, flags, FileType::RegularFile) + .await + } + + async fn open_dir(&self, file_id: u64, flags: u32) -> Result { + self.open_file_internal(file_id, flags, FileType::Directory) + .await + } + + async fn create_file(&self, parent_file_id: u64, name: &str, flags: u32) -> Result { + let parent_node = self.get_file_node(parent_file_id).await?; + let mut file = self + .fs + .create_file(&parent_node.file_name, name, OpenFileFlags(flags)) + .await?; + + file.set_file_id(parent_file_id, self.next_inode_id()); + { + let mut file_node_manager = self.file_entry_manager.write().await; + file_node_manager.insert( + file.file_stat.parent_file_id, + file.file_stat.file_id, + &file.file_stat.path, + ); + } + let file = self.opened_file_manager.put_file(file); + let file = file.lock().await; + Ok(file.file_handle()) + } + + async fn create_dir(&self, parent_file_id: u64, name: &str) -> Result { + let parent_node = self.get_file_node(parent_file_id).await?; + let mut file = self.fs.create_dir(&parent_node.file_name, name).await?; + + file.set_file_id(parent_file_id, self.next_inode_id()); + { + let mut file_node_manager = self.file_entry_manager.write().await; + file_node_manager.insert(file.parent_file_id, file.file_id, &file.path); + } + Ok(file.file_id) + } + + async fn set_attr(&self, file_id: u64, file_stat: &FileStat) -> Result<()> { + let file_node = self.get_file_node(file_id).await?; + self.fs + .set_attr(&file_node.file_name, file_stat, true) + .await + } + + async fn remove_file(&self, parent_file_id: u64, name: &str) -> Result<()> { + let parent_file_node = self.get_file_node(parent_file_id).await?; + self.fs + .remove_file(&parent_file_node.file_name, name) + .await?; + + { + let mut file_id_manager = self.file_entry_manager.write().await; + file_id_manager.remove(&join_file_path(&parent_file_node.file_name, name)); + } + Ok(()) + } + + async fn remove_dir(&self, parent_file_id: u64, name: &str) -> Result<()> { + let parent_file_node = self.get_file_node(parent_file_id).await?; + self.fs + .remove_dir(&parent_file_node.file_name, name) + .await?; + + { + let mut file_id_manager = self.file_entry_manager.write().await; + file_id_manager.remove(&join_file_path(&parent_file_node.file_name, name)); + } + Ok(()) + } + + async fn close_file(&self, _file_id: u64, fh: u64) -> Result<()> { + let file = self + .opened_file_manager + .remove_file(fh) + .ok_or(Errno::from(libc::EBADF))?; + let mut file = file.lock().await; + file.close().await?; + Ok(()) + } + + async fn read(&self, _file_id: u64, fh: u64, offset: u64, size: u32) -> Result { + let file_stat: FileStat; + let data = { + let opened_file = self + .opened_file_manager + .get_file(fh) + .ok_or(Errno::from(libc::EBADF))?; + let mut opened_file = opened_file.lock().await; + file_stat = opened_file.file_stat.clone(); + opened_file.read(offset, size).await + }; + + self.fs.set_attr(&file_stat.path, &file_stat, false).await?; + + data + } + + async fn write(&self, _file_id: u64, fh: u64, offset: u64, data: &[u8]) -> Result { + let (len, file_stat) = { + let opened_file = self + .opened_file_manager + .get_file(fh) + .ok_or(Errno::from(libc::EBADF))?; + let mut opened_file = opened_file.lock().await; + let len = opened_file.write(offset, data).await; + (len, opened_file.file_stat.clone()) + }; + + self.fs.set_attr(&file_stat.path, &file_stat, false).await?; + + len + } +} + +/// File entry is represent the abstract file. +#[derive(Debug, Clone)] +struct FileEntry { + file_id: u64, + parent_file_id: u64, + file_name: String, +} + +/// FileEntryManager is manage all the file entries in memory. it is used manger the file relationship and name mapping. +struct FileEntryManager { + // file_id_map is a map of file_id to file name. + file_id_map: HashMap, + + // file_name_map is a map of file name to file id. + file_name_map: HashMap, +} + +impl FileEntryManager { + fn new() -> Self { + Self { + file_id_map: HashMap::new(), + file_name_map: HashMap::new(), + } + } + + fn get_file_by_id(&self, file_id: u64) -> Option { + self.file_id_map.get(&file_id).cloned() + } + + fn get_file_by_name(&self, file_name: &str) -> Option { + self.file_name_map.get(file_name).cloned() + } + + fn insert(&mut self, parent_file_id: u64, file_id: u64, file_name: &str) { + let file_node = FileEntry { + file_id, + parent_file_id, + file_name: file_name.to_string(), + }; + self.file_id_map.insert(file_id, file_node.clone()); + self.file_name_map.insert(file_name.to_string(), file_node); + } + + fn remove(&mut self, file_name: &str) { + if let Some(node) = self.file_name_map.remove(file_name) { + self.file_id_map.remove(&node.file_id); + } + } +} diff --git a/clients/filesystem-fuse/src/fuse_api_handle.rs b/clients/filesystem-fuse/src/fuse_api_handle.rs index 8c065df0227..800930f4ed3 100644 --- a/clients/filesystem-fuse/src/fuse_api_handle.rs +++ b/clients/filesystem-fuse/src/fuse_api_handle.rs @@ -401,6 +401,10 @@ impl Filesystem for FuseApiHandle { const fn fstat_to_file_attr(file_st: &FileStat, context: &FileSystemContext) -> FileAttr { debug_assert!(file_st.file_id != 0 && file_st.parent_file_id != 0); + let perm = match file_st.kind { + Directory => context.default_dir_perm, + _ => context.default_file_perm, + }; FileAttr { ino: file_st.file_id, size: file_st.size, @@ -409,7 +413,7 @@ const fn fstat_to_file_attr(file_st: &FileStat, context: &FileSystemContext) -> mtime: file_st.mtime, ctime: file_st.ctime, kind: file_st.kind, - perm: file_st.perm, + perm: perm, nlink: file_st.nlink, uid: context.uid, gid: context.gid, @@ -469,7 +473,6 @@ mod test { path: "".to_string(), size: 10032, kind: FileType::RegularFile, - perm: 0, atime: Timestamp { sec: 10, nsec: 3 }, mtime: Timestamp { sec: 12, nsec: 5 }, ctime: Timestamp { sec: 15, nsec: 7 }, diff --git a/clients/filesystem-fuse/src/lib.rs b/clients/filesystem-fuse/src/lib.rs index 54fb59a5107..b531df82f79 100644 --- a/clients/filesystem-fuse/src/lib.rs +++ b/clients/filesystem-fuse/src/lib.rs @@ -18,3 +18,5 @@ */ mod filesystem; mod fuse_api_handle; +mod opened_file_manager; +mod utils; diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index f6a7e69ec67..24358840df5 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -18,6 +18,8 @@ */ mod filesystem; mod fuse_api_handle; +mod opened_file_manager; +mod utils; use log::debug; use log::info; diff --git a/clients/filesystem-fuse/src/opened_file_manager.rs b/clients/filesystem-fuse/src/opened_file_manager.rs new file mode 100644 index 00000000000..664f0a89875 --- /dev/null +++ b/clients/filesystem-fuse/src/opened_file_manager.rs @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use crate::filesystem::OpenedFile; +use dashmap::DashMap; +use std::sync::atomic::AtomicU64; +use std::sync::Arc; +use tokio::sync::Mutex; + +// OpenedFileManager is a manager all the opened files. and allocate a file handle id for the opened file. +pub(crate) struct OpenedFileManager { + // file_handle_map is a map of file_handle_id to opened file. + file_handle_map: DashMap>>, + + // file_handle_id_generator is used to generate unique file handle IDs. + handle_id_generator: AtomicU64, +} + +impl OpenedFileManager { + pub fn new() -> Self { + Self { + file_handle_map: Default::default(), + handle_id_generator: AtomicU64::new(1), + } + } + + pub(crate) fn next_handle_id(&self) -> u64 { + self.handle_id_generator + .fetch_add(1, std::sync::atomic::Ordering::SeqCst) + } + + pub(crate) fn put_file(&self, mut file: OpenedFile) -> Arc> { + let file_handle_id = self.next_handle_id(); + file.handle_id = file_handle_id; + let file_handle = Arc::new(Mutex::new(file)); + self.file_handle_map + .insert(file_handle_id, file_handle.clone()); + file_handle + } + + pub(crate) fn get_file(&self, handle_id: u64) -> Option>> { + self.file_handle_map + .get(&handle_id) + .map(|x| x.value().clone()) + } + + pub(crate) fn remove_file(&self, handle_id: u64) -> Option>> { + self.file_handle_map.remove(&handle_id).map(|x| x.1) + } +} diff --git a/clients/filesystem-fuse/src/utils.rs b/clients/filesystem-fuse/src/utils.rs new file mode 100644 index 00000000000..6118b672362 --- /dev/null +++ b/clients/filesystem-fuse/src/utils.rs @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// join the parent and name to a path +pub fn join_file_path(parent: &str, name: &str) -> String { + if parent.is_empty() { + name.to_string() + } else { + format!("{}/{}", parent, name) + } +} + +// split the path to parent and name +pub fn split_file_path(path: &str) -> (&str, &str) { + match path.rfind('/') { + Some(pos) => (&path[..pos], &path[pos + 1..]), + None => ("", path), + } +} From 97e066d5020c535129a9e2204324de907bdec9a2 Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 17 Dec 2024 11:48:01 +0800 Subject: [PATCH 02/47] Update --- clients/filesystem-fuse/src/filesystem.rs | 109 +++++++++++----------- 1 file changed, 56 insertions(+), 53 deletions(-) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 6a2bf80c6dc..ba0c04dc5d9 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -257,20 +257,28 @@ impl OpenedFile { } async fn read(&mut self, offset: u64, size: u32) -> Result { + let reader = self.reader.as_mut().ok_or(Errno::from(libc::EBADF))?; + let result = reader.read(offset, size).await?; + + // update the access time self.file_stat.atime = Timestamp::from(SystemTime::now()); - self.reader.as_mut().unwrap().read(offset, size).await + + Ok(result) } async fn write(&mut self, offset: u64, data: &[u8]) -> Result { - let end = offset + data.len() as u64; + let writer = self.writer.as_mut().ok_or(Errno::from(libc::EBADF))?; + let written = writer.write(offset, data).await?; + // update the file size + let end = offset + data.len() as u64; if end > self.file_stat.size { self.file_stat.size = end; } self.file_stat.atime = Timestamp::from(SystemTime::now()); self.file_stat.mtime = self.file_stat.atime; - self.writer.as_mut().unwrap().write(offset, data).await + Ok(written) } async fn close(&mut self) -> Result<()> { @@ -355,14 +363,14 @@ pub struct SimpleFileSystem { /// opened files opened_file_manager: OpenedFileManager, /// inode id generator - inode_id_generator: AtomicU64, + file_id_generator: AtomicU64, /// real system fs: T, } impl SimpleFileSystem { - const INITIAL_INODE_ID: u64 = 10000; + const INITIAL_FILE_ID: u64 = 10000; const ROOT_DIR_PARENT_FILE_ID: u64 = 0; const ROOT_DIR_FILE_ID: u64 = 1; const ROOT_DIR_NAME: &'static str = ""; @@ -371,17 +379,17 @@ impl SimpleFileSystem { Self { file_entry_manager: RwLock::new(FileEntryManager::new()), opened_file_manager: OpenedFileManager::new(), - inode_id_generator: AtomicU64::new(Self::INITIAL_INODE_ID), + file_id_generator: AtomicU64::new(Self::INITIAL_FILE_ID), fs, } } - fn next_inode_id(&self) -> u64 { - self.inode_id_generator + fn next_file_id(&self) -> u64 { + self.file_id_generator .fetch_add(1, std::sync::atomic::Ordering::SeqCst) } - async fn get_file_node(&self, file_id: u64) -> Result { + async fn get_file_entry(&self, file_id: u64) -> Result { self.file_entry_manager .read() .await @@ -389,7 +397,7 @@ impl SimpleFileSystem { .ok_or(Errno::from(libc::ENOENT)) } - async fn get_file_node_by_path(&self, path: &str) -> Option { + async fn get_file_entry_by_path(&self, path: &str) -> Option { self.file_entry_manager.read().await.get_file_by_name(path) } @@ -398,7 +406,8 @@ impl SimpleFileSystem { let file = file_manager.get_file_by_name(&file_stat.path); match file { None => { - file_stat.set_file_id(parent_file_id, self.next_inode_id()); + // allocate new file id + file_stat.set_file_id(parent_file_id, self.next_file_id()); file_manager.insert(file_stat.parent_file_id, file_stat.file_id, &file_stat.path); } Some(file) => { @@ -413,24 +422,24 @@ impl SimpleFileSystem { flags: u32, kind: FileType, ) -> Result { - let file_node = self.get_file_node(file_id).await?; + let file_entry = self.get_file_entry(file_id).await?; let mut file = { match kind { FileType::Directory => { self.fs - .open_dir(&file_node.file_name, OpenFileFlags(flags)) + .open_dir(&file_entry.file_name, OpenFileFlags(flags)) .await? } FileType::RegularFile => { self.fs - .open_file(&file_node.file_name, OpenFileFlags(flags)) + .open_file(&file_entry.file_name, OpenFileFlags(flags)) .await? } _ => return Err(Errno::from(libc::EINVAL)), } }; - file.set_file_id(file_node.parent_file_id, file_id); + file.set_file_id(file_entry.parent_file_id, file_id); let file = self.opened_file_manager.put_file(file); let file = file.lock().await; Ok(file.file_handle()) @@ -449,7 +458,7 @@ impl RawFileSystem for SimpleFileSystem { } async fn get_file_path(&self, file_id: u64) -> String { - let file = self.get_file_node(file_id).await; + let file = self.get_file_entry(file_id).await; file.map(|x| x.file_name).unwrap_or_else(|_| "".to_string()) } @@ -469,24 +478,24 @@ impl RawFileSystem for SimpleFileSystem { } async fn stat(&self, file_id: u64) -> Result { - let file_node = self.get_file_node(file_id).await?; - let mut stat = self.fs.stat(&file_node.file_name).await?; - stat.set_file_id(file_node.parent_file_id, file_node.file_id); + let file = self.get_file_entry(file_id).await?; + let mut stat = self.fs.stat(&file.file_name).await?; + stat.set_file_id(file.parent_file_id, file.file_id); Ok(stat) } async fn lookup(&self, parent_file_id: u64, name: &str) -> Result { - let parent_file_node = self.get_file_node(parent_file_id).await?; - let mut stat = self.fs.lookup(&parent_file_node.file_name, name).await?; + let parent_file = self.get_file_entry(parent_file_id).await?; + let mut stat = self.fs.lookup(&parent_file.file_name, name).await?; self.fill_file_id(&mut stat, parent_file_id).await; Ok(stat) } async fn read_dir(&self, file_id: u64) -> Result> { - let file_node = self.get_file_node(file_id).await?; - let mut files = self.fs.read_dir(&file_node.file_name).await?; + let file = self.get_file_entry(file_id).await?; + let mut files = self.fs.read_dir(&file.file_name).await?; for file in files.iter_mut() { - self.fill_file_id(file, file_node.file_id).await; + self.fill_file_id(file, file.file_id).await; } Ok(files) } @@ -502,16 +511,16 @@ impl RawFileSystem for SimpleFileSystem { } async fn create_file(&self, parent_file_id: u64, name: &str, flags: u32) -> Result { - let parent_node = self.get_file_node(parent_file_id).await?; + let parent = self.get_file_entry(parent_file_id).await?; let mut file = self .fs - .create_file(&parent_node.file_name, name, OpenFileFlags(flags)) + .create_file(&parent.file_name, name, OpenFileFlags(flags)) .await?; - file.set_file_id(parent_file_id, self.next_inode_id()); + file.set_file_id(parent_file_id, self.next_file_id()); { - let mut file_node_manager = self.file_entry_manager.write().await; - file_node_manager.insert( + let mut file_manager = self.file_entry_manager.write().await; + file_manager.insert( file.file_stat.parent_file_id, file.file_stat.file_id, &file.file_stat.path, @@ -523,46 +532,40 @@ impl RawFileSystem for SimpleFileSystem { } async fn create_dir(&self, parent_file_id: u64, name: &str) -> Result { - let parent_node = self.get_file_node(parent_file_id).await?; - let mut file = self.fs.create_dir(&parent_node.file_name, name).await?; + let parent = self.get_file_entry(parent_file_id).await?; + let mut file = self.fs.create_dir(&parent.file_name, name).await?; - file.set_file_id(parent_file_id, self.next_inode_id()); + file.set_file_id(parent_file_id, self.next_file_id()); { - let mut file_node_manager = self.file_entry_manager.write().await; - file_node_manager.insert(file.parent_file_id, file.file_id, &file.path); + let mut file_manager = self.file_entry_manager.write().await; + file_manager.insert(file.parent_file_id, file.file_id, &file.path); } Ok(file.file_id) } async fn set_attr(&self, file_id: u64, file_stat: &FileStat) -> Result<()> { - let file_node = self.get_file_node(file_id).await?; - self.fs - .set_attr(&file_node.file_name, file_stat, true) - .await + let file = self.get_file_entry(file_id).await?; + self.fs.set_attr(&file.file_name, file_stat, true).await } async fn remove_file(&self, parent_file_id: u64, name: &str) -> Result<()> { - let parent_file_node = self.get_file_node(parent_file_id).await?; - self.fs - .remove_file(&parent_file_node.file_name, name) - .await?; + let parent_file = self.get_file_entry(parent_file_id).await?; + self.fs.remove_file(&parent_file.file_name, name).await?; { let mut file_id_manager = self.file_entry_manager.write().await; - file_id_manager.remove(&join_file_path(&parent_file_node.file_name, name)); + file_id_manager.remove(&join_file_path(&parent_file.file_name, name)); } Ok(()) } async fn remove_dir(&self, parent_file_id: u64, name: &str) -> Result<()> { - let parent_file_node = self.get_file_node(parent_file_id).await?; - self.fs - .remove_dir(&parent_file_node.file_name, name) - .await?; + let parent_file = self.get_file_entry(parent_file_id).await?; + self.fs.remove_dir(&parent_file.file_name, name).await?; { let mut file_id_manager = self.file_entry_manager.write().await; - file_id_manager.remove(&join_file_path(&parent_file_node.file_name, name)); + file_id_manager.remove(&join_file_path(&parent_file.file_name, name)); } Ok(()) } @@ -645,18 +648,18 @@ impl FileEntryManager { } fn insert(&mut self, parent_file_id: u64, file_id: u64, file_name: &str) { - let file_node = FileEntry { + let file = FileEntry { file_id, parent_file_id, file_name: file_name.to_string(), }; - self.file_id_map.insert(file_id, file_node.clone()); - self.file_name_map.insert(file_name.to_string(), file_node); + self.file_id_map.insert(file_id, file.clone()); + self.file_name_map.insert(file_name.to_string(), file); } fn remove(&mut self, file_name: &str) { - if let Some(node) = self.file_name_map.remove(file_name) { - self.file_id_map.remove(&node.file_id); + if let Some(file) = self.file_name_map.remove(file_name) { + self.file_id_map.remove(&file.file_id); } } } From d0b6958025d19e69f0b02e92205bffae76704d50 Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 17 Dec 2024 15:17:51 +0800 Subject: [PATCH 03/47] Add uts --- clients/filesystem-fuse/Makefile | 3 +- clients/filesystem-fuse/src/filesystem.rs | 82 +++++++++++++++++++ .../src/opened_file_manager.rs | 62 ++++++++++++++ clients/filesystem-fuse/src/utils.rs | 23 ++++++ 4 files changed, 168 insertions(+), 2 deletions(-) diff --git a/clients/filesystem-fuse/Makefile b/clients/filesystem-fuse/Makefile index 2e48a3e1fd8..9a9cbe042a6 100644 --- a/clients/filesystem-fuse/Makefile +++ b/clients/filesystem-fuse/Makefile @@ -31,8 +31,7 @@ check-fmt: cargo fmt --all -- --check check-clippy: - #cargo clippy --all-targets --all-features --workspace -- -D warnings - cargo clippy --all-targets --all-features --workspace -- + cargo clippy --all-targets --all-features --workspace -- -D warnings install-cargo-sort: cargo install cargo-sort@1.0.9 diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index ba0c04dc5d9..4c0d716d91c 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -663,3 +663,85 @@ impl FileEntryManager { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_create_file_stat() { + //test new file + let file_stat = FileStat::new_file("a", "b", 10); + assert_eq!(file_stat.name, "b"); + assert_eq!(file_stat.path, "a/b"); + assert_eq!(file_stat.size, 10); + assert_eq!(file_stat.kind, FileType::RegularFile); + + //test new dir + let file_stat = FileStat::new_dir("a", "b"); + assert_eq!(file_stat.name, "b"); + assert_eq!(file_stat.path, "a/b"); + assert_eq!(file_stat.size, 0); + assert_eq!(file_stat.kind, FileType::Directory); + + //test new file with path + let file_stat = FileStat::new_file_with_path("a/b", 10); + assert_eq!(file_stat.name, "b"); + assert_eq!(file_stat.path, "a/b"); + assert_eq!(file_stat.size, 10); + assert_eq!(file_stat.kind, FileType::RegularFile); + + //teset new dir with path + let file_stat = FileStat::new_dir_with_path("a/b"); + assert_eq!(file_stat.name, "b"); + assert_eq!(file_stat.path, "a/b"); + assert_eq!(file_stat.size, 0); + assert_eq!(file_stat.kind, FileType::Directory); + } + + #[test] + fn test_file_stat_set_file_id() { + let mut file_stat = FileStat::new_file("a", "b", 10); + file_stat.set_file_id(1, 2); + assert_eq!(file_stat.file_id, 2); + assert_eq!(file_stat.parent_file_id, 1); + } + + #[test] + #[should_panic(expected = "assertion failed: file_id != 0 && parent_file_id != 0")] + fn test_file_stat_set_file_id_panic() { + let mut file_stat = FileStat::new_file("a", "b", 10); + file_stat.set_file_id(1, 0); + } + + #[test] + fn test_open_file() { + let mut open_file = OpenedFile::new(FileStat::new_file("a", "b", 10)); + assert_eq!(open_file.file_stat.name, "b"); + assert_eq!(open_file.file_stat.size, 10); + + open_file.set_file_id(1, 2); + + assert_eq!(open_file.file_stat.file_id, 2); + assert_eq!(open_file.file_stat.parent_file_id, 1); + } + + #[test] + fn test_file_entry_manager() { + let mut manager = FileEntryManager::new(); + manager.insert(1, 2, "a/b"); + let file = manager.get_file_by_id(2).unwrap(); + assert_eq!(file.file_id, 2); + assert_eq!(file.parent_file_id, 1); + assert_eq!(file.file_name, "a/b"); + + let file = manager.get_file_by_name("a/b").unwrap(); + assert_eq!(file.file_id, 2); + assert_eq!(file.parent_file_id, 1); + assert_eq!(file.file_name, "a/b"); + + manager.remove("a/b"); + assert!(manager.get_file_by_id(2).is_none()); + assert!(manager.get_file_by_name("a/b").is_none()); + } +} diff --git a/clients/filesystem-fuse/src/opened_file_manager.rs b/clients/filesystem-fuse/src/opened_file_manager.rs index 664f0a89875..03567485a4c 100644 --- a/clients/filesystem-fuse/src/opened_file_manager.rs +++ b/clients/filesystem-fuse/src/opened_file_manager.rs @@ -63,3 +63,65 @@ impl OpenedFileManager { self.file_handle_map.remove(&handle_id).map(|x| x.1) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::filesystem::FileStat; + + #[tokio::test] + async fn test_opened_file_manager() { + let manager = OpenedFileManager::new(); + + let file1_stat = FileStat::new_file("", "a.txt", 13); + let file2_stat = FileStat::new_file("", "b.txt", 18); + + let file1 = OpenedFile::new(file1_stat.clone()); + let file2 = OpenedFile::new(file2_stat.clone()); + + let handle_id1 = manager.put_file(file1).lock().await.handle_id; + let handle_id2 = manager.put_file(file2).lock().await.handle_id; + + // Test the file handle id is assigned. + assert!(handle_id1 > 0 && handle_id2 > 0); + assert_ne!(handle_id1, handle_id2); + + // test get file by handle id + assert_eq!( + manager + .get_file(handle_id1) + .unwrap() + .lock() + .await + .file_stat + .name, + file1_stat.name + ); + + assert_eq!( + manager + .get_file(handle_id2) + .unwrap() + .lock() + .await + .file_stat + .name, + file2_stat.name + ); + + // test remove file by handle id + assert_eq!( + manager + .remove_file(handle_id1) + .unwrap() + .lock() + .await + .handle_id, + handle_id1 + ); + + // test get file by handle id after remove + assert!(manager.get_file(handle_id1).is_none()); + assert!(manager.get_file(handle_id2).is_some()); + } +} diff --git a/clients/filesystem-fuse/src/utils.rs b/clients/filesystem-fuse/src/utils.rs index 6118b672362..7ac0c7212a7 100644 --- a/clients/filesystem-fuse/src/utils.rs +++ b/clients/filesystem-fuse/src/utils.rs @@ -33,3 +33,26 @@ pub fn split_file_path(path: &str) -> (&str, &str) { None => ("", path), } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_join_file_path() { + assert_eq!(join_file_path("", "a"), "a"); + assert_eq!(join_file_path("", "a.txt"), "a.txt"); + assert_eq!(join_file_path("a", "b"), "a/b"); + assert_eq!(join_file_path("a/b", "c"), "a/b/c"); + assert_eq!(join_file_path("a/b", "c.txt"), "a/b/c.txt"); + } + + #[test] + fn test_split_file_path() { + assert_eq!(split_file_path("a"), ("", "a")); + assert_eq!(split_file_path("a.txt"), ("", "a.txt")); + assert_eq!(split_file_path("a/b"), ("a", "b")); + assert_eq!(split_file_path("a/b/c"), ("a/b", "c")); + assert_eq!(split_file_path("a/b/c.txt"), ("a/b", "c.txt")); + } +} From 3769400a32a0cb109323f40e46e61ae0a6a9ff7a Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 17 Dec 2024 15:26:37 +0800 Subject: [PATCH 04/47] Fix test error --- clients/filesystem-fuse/Makefile | 6 +++--- clients/filesystem-fuse/src/filesystem.rs | 2 +- clients/filesystem-fuse/src/fuse_api_handle.rs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/clients/filesystem-fuse/Makefile b/clients/filesystem-fuse/Makefile index 9a9cbe042a6..f4a4cef20ae 100644 --- a/clients/filesystem-fuse/Makefile +++ b/clients/filesystem-fuse/Makefile @@ -27,6 +27,9 @@ fmt: cargo-sort: install-cargo-sort cargo sort -w +fix-toml: install-taplo-cli + taplo fmt + check-fmt: cargo fmt --all -- --check @@ -48,9 +51,6 @@ cargo-machete: install-cargo-machete install-taplo-cli: cargo install taplo-cli@0.9.0 -fix-toml: install-taplo-cli - taplo fmt - check-toml: install-taplo-cli taplo check diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 4c0d716d91c..f42418fde03 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -691,7 +691,7 @@ mod tests { assert_eq!(file_stat.size, 10); assert_eq!(file_stat.kind, FileType::RegularFile); - //teset new dir with path + //test new dir with path let file_stat = FileStat::new_dir_with_path("a/b"); assert_eq!(file_stat.name, "b"); assert_eq!(file_stat.path, "a/b"); diff --git a/clients/filesystem-fuse/src/fuse_api_handle.rs b/clients/filesystem-fuse/src/fuse_api_handle.rs index 800930f4ed3..e9f58f55045 100644 --- a/clients/filesystem-fuse/src/fuse_api_handle.rs +++ b/clients/filesystem-fuse/src/fuse_api_handle.rs @@ -496,7 +496,7 @@ mod test { assert_eq!(file_attr.mtime, Timestamp { sec: 12, nsec: 5 }); assert_eq!(file_attr.ctime, Timestamp { sec: 15, nsec: 7 }); assert_eq!(file_attr.kind, FileType::RegularFile); - assert_eq!(file_attr.perm, 0); + assert_eq!(file_attr.perm, context.default_file_perm); assert_eq!(file_attr.nlink, 0); assert_eq!(file_attr.uid, 1); assert_eq!(file_attr.gid, 2); From 3ecb9805ae7d0cf1fe8096595d6875591596933d Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 17 Dec 2024 15:44:48 +0800 Subject: [PATCH 05/47] Update comments --- clients/filesystem-fuse/src/filesystem.rs | 34 ++++++++++++------- .../filesystem-fuse/src/fuse_api_handle.rs | 4 +-- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index f42418fde03..2578ab0b4d3 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -44,7 +44,7 @@ pub(crate) trait RawFileSystem: Send + Sync { async fn get_file_path(&self, file_id: u64) -> String; /// Validate the file id and file handle, if file id and file handle is valid and it associated, return Ok - async fn valid_file_id(&self, file_id: u64, fh: u64) -> Result<()>; + async fn valid_file_handle_id(&self, file_id: u64, fh: u64) -> Result<()>; /// Get the file stat by file id. if the file id is valid, return the file stat async fn stat(&self, file_id: u64) -> Result; @@ -260,7 +260,7 @@ impl OpenedFile { let reader = self.reader.as_mut().ok_or(Errno::from(libc::EBADF))?; let result = reader.read(offset, size).await?; - // update the access time + // update the atime self.file_stat.atime = Timestamp::from(SystemTime::now()); Ok(result) @@ -270,7 +270,7 @@ impl OpenedFile { let writer = self.writer.as_mut().ok_or(Errno::from(libc::EBADF))?; let written = writer.write(offset, data).await?; - // update the file size + // update the file size ,mtime and atime let end = offset + data.len() as u64; if end > self.file_stat.size { self.file_stat.size = end; @@ -365,7 +365,7 @@ pub struct SimpleFileSystem { /// inode id generator file_id_generator: AtomicU64, - /// real system + /// real filesystem fs: T, } @@ -411,6 +411,7 @@ impl SimpleFileSystem { file_manager.insert(file_stat.parent_file_id, file_stat.file_id, &file_stat.path); } Some(file) => { + // use the exist file id file_stat.set_file_id(file.parent_file_id, file.file_id); } } @@ -439,6 +440,7 @@ impl SimpleFileSystem { _ => return Err(Errno::from(libc::EINVAL)), } }; + // set the exists file id file.set_file_id(file_entry.parent_file_id, file_id); let file = self.opened_file_manager.put_file(file); let file = file.lock().await; @@ -449,6 +451,7 @@ impl SimpleFileSystem { #[async_trait] impl RawFileSystem for SimpleFileSystem { async fn init(&self) -> Result<()> { + // init root directory self.file_entry_manager.write().await.insert( Self::ROOT_DIR_PARENT_FILE_ID, Self::ROOT_DIR_FILE_ID, @@ -462,8 +465,8 @@ impl RawFileSystem for SimpleFileSystem { file.map(|x| x.file_name).unwrap_or_else(|_| "".to_string()) } - async fn valid_file_id(&self, _file_id: u64, fh: u64) -> Result<()> { - let file_id = self + async fn valid_file_handle_id(&self, file_id: u64, fh: u64) -> Result<()> { + let fh_file_id = self .opened_file_manager .get_file(fh) .ok_or(Errno::from(libc::EBADF))? @@ -472,7 +475,7 @@ impl RawFileSystem for SimpleFileSystem { .file_stat .file_id; - (file_id == _file_id) + (file_id == fh_file_id) .then_some(()) .ok_or(Errno::from(libc::EBADF)) } @@ -487,6 +490,7 @@ impl RawFileSystem for SimpleFileSystem { async fn lookup(&self, parent_file_id: u64, name: &str) -> Result { let parent_file = self.get_file_entry(parent_file_id).await?; let mut stat = self.fs.lookup(&parent_file.file_name, name).await?; + // fill the file id to file stat self.fill_file_id(&mut stat, parent_file_id).await; Ok(stat) } @@ -518,14 +522,14 @@ impl RawFileSystem for SimpleFileSystem { .await?; file.set_file_id(parent_file_id, self.next_file_id()); + + // insert the new file to file entry manager { let mut file_manager = self.file_entry_manager.write().await; - file_manager.insert( - file.file_stat.parent_file_id, - file.file_stat.file_id, - &file.file_stat.path, - ); + file_manager.insert(parent_file_id, file.file_stat.file_id, &file.file_stat.path); } + + // put the file to the opened file manager let file = self.opened_file_manager.put_file(file); let file = file.lock().await; Ok(file.file_handle()) @@ -536,6 +540,8 @@ impl RawFileSystem for SimpleFileSystem { let mut file = self.fs.create_dir(&parent.file_name, name).await?; file.set_file_id(parent_file_id, self.next_file_id()); + + // insert the new file to file entry manager { let mut file_manager = self.file_entry_manager.write().await; file_manager.insert(file.parent_file_id, file.file_id, &file.path); @@ -552,6 +558,7 @@ impl RawFileSystem for SimpleFileSystem { let parent_file = self.get_file_entry(parent_file_id).await?; self.fs.remove_file(&parent_file.file_name, name).await?; + // remove the file from file entry manager { let mut file_id_manager = self.file_entry_manager.write().await; file_id_manager.remove(&join_file_path(&parent_file.file_name, name)); @@ -563,6 +570,7 @@ impl RawFileSystem for SimpleFileSystem { let parent_file = self.get_file_entry(parent_file_id).await?; self.fs.remove_dir(&parent_file.file_name, name).await?; + // remove the dir from file entry manager { let mut file_id_manager = self.file_entry_manager.write().await; file_id_manager.remove(&join_file_path(&parent_file.file_name, name)); @@ -592,6 +600,7 @@ impl RawFileSystem for SimpleFileSystem { opened_file.read(offset, size).await }; + // update the file atime self.fs.set_attr(&file_stat.path, &file_stat, false).await?; data @@ -608,6 +617,7 @@ impl RawFileSystem for SimpleFileSystem { (len, opened_file.file_stat.clone()) }; + // update the file size, mtime and atime self.fs.set_attr(&file_stat.path, &file_stat, false).await?; len diff --git a/clients/filesystem-fuse/src/fuse_api_handle.rs b/clients/filesystem-fuse/src/fuse_api_handle.rs index e9f58f55045..76af581446b 100644 --- a/clients/filesystem-fuse/src/fuse_api_handle.rs +++ b/clients/filesystem-fuse/src/fuse_api_handle.rs @@ -117,7 +117,7 @@ impl Filesystem for FuseApiHandle { ) -> fuse3::Result { // check the fh is associated with the file_id if let Some(fh) = fh { - self.fs.valid_file_id(inode, fh).await?; + self.fs.valid_file_handle_id(inode, fh).await?; } let file_stat = self.fs.stat(inode).await?; @@ -136,7 +136,7 @@ impl Filesystem for FuseApiHandle { ) -> fuse3::Result { // check the fh is associated with the file_id if let Some(fh) = fh { - self.fs.valid_file_id(inode, fh).await?; + self.fs.valid_file_handle_id(inode, fh).await?; } let new_file_stat = self From 4fe58f27a5637104d69046b347d8818bd5395328 Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 17 Dec 2024 15:53:14 +0800 Subject: [PATCH 06/47] Fix ci error --- clients/filesystem-fuse/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/filesystem-fuse/build.gradle.kts b/clients/filesystem-fuse/build.gradle.kts index 201397c19e7..7e20e13cedd 100644 --- a/clients/filesystem-fuse/build.gradle.kts +++ b/clients/filesystem-fuse/build.gradle.kts @@ -40,7 +40,7 @@ val checkRustProject by tasks.registering(Exec::class) { description = "Check the Rust project" workingDir = file("$projectDir") - commandLine( "bash", "-c", "make check") + commandLine("bash", "-c", "make check") } val testRustProject by tasks.registering(Exec::class) { From a55bff87a27c9a098e457ac8f24b1d83ef3b712c Mon Sep 17 00:00:00 2001 From: yuhui Date: Wed, 18 Dec 2024 11:11:33 +0800 Subject: [PATCH 07/47] Add description of gvfs-fuse filesystem struct --- clients/filesystem-fuse/src/main.rs | 48 +++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index 24358840df5..e713e743fb8 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -32,3 +32,51 @@ async fn main() { debug!("Shutdown filesystem..."); exit(0); } + +async fn create_gvfs_fuse_filesystem() { + // Gvfs-fuse filesystem structure: + // FuseApiHandle + // ├─ SimpleFileSystem (RawFileSystem) + // │ └─ FileSystemLog (PathFileSystem) + // ├─ GravitinoComposedFileSystem (PathFileSystem) + // │ ├─ GravitinoFilesetFileSystem (PathFileSystem) + // │ │ └─ OpenDALFileSystem (PathFileSystem) + // │ │ └─ S3FileSystem (PathFileSystem) + // │ ├─ GravitinoFilesetFileSystem (PathFileSystem) + // │ │ └─ OpenDALFileSystem (PathFileSystem) + // │ │ └─ HDFSFileSystem (PathFileSystem) + // │ ├─ GravitinoFilesetFileSystem (PathFileSystem) + // │ │ └─ NasFileSystem (PathFileSystem) + // │ │ └─ JuiceFileSystem (PathFileSystem) + // │ ├─ GravitinoFilesetFileSystem (PathFileSystem) + // │ │ └─ XXXFileSystem (PathFileSystem) + // + // `SimpleFileSystem` is a low-level filesystem designed to communicate with FUSE APIs. + // It manages file and directory relationships, as well as file mappings. + // It delegates file operations to the PathFileSystem + // + // `FileSystemLog` is a decorator that adds extra debug logging functionality to file system APIs. + // Similar implementations include permissions, caching, and metrics. + // + // `GravitinoComposeFileSystem` is a composite file system that can combine multiple `GravitinoFilesetFileSystem`. + // It translates the part of catalog and schema of fileset path to a signal GravitinoFilesetFileSystem path. + // If the user only mounts a fileset, this layer is not present. There will only be one below layer. + // + // `GravitinoFilesetFileSystem` is a file system that can access a fileset.It translates the fileset path to the real storage path. + // and delegate the operation to the real storage. + // + // `OpenDALFileSystem` is a file system that use the OpenDAL to access real storage. + // it can assess the S3, HDFS, gcs, azblob and other storage. + // + // `S3FileSystem` is a file system that use to access S3 storage. + // + // `HDFSFileSystem` is a file system that use to access HDFS storage. + + // `NasFileSystem` is a filesystem that uses a locally accessible path mounted by NAS tools, such as JuiceFS. + + // `JuiceFileSystem` is a file system that use to manage juice mount and access JuiceFS storage. + + // `XXXXFileSystem` is a file system that use implement access file by your extent. + + todo!("Implement the createGvfsFuseFileSystem function"); +} From 58922e14f14f276e51ecbb57b769eaed85d9ec97 Mon Sep 17 00:00:00 2001 From: yuhui Date: Wed, 18 Dec 2024 11:16:13 +0800 Subject: [PATCH 08/47] Update --- clients/filesystem-fuse/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index e713e743fb8..69b0a75e054 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -74,9 +74,9 @@ async fn create_gvfs_fuse_filesystem() { // `NasFileSystem` is a filesystem that uses a locally accessible path mounted by NAS tools, such as JuiceFS. - // `JuiceFileSystem` is a file system that use to manage juice mount and access JuiceFS storage. + // `JuiceFileSystem` is a file system that use to manage JuiceFS mount and access JuiceFS storage. - // `XXXXFileSystem` is a file system that use implement access file by your extent. + // `XXXFileSystem is a filesystem that allows you to implement file access through your own extensions. todo!("Implement the createGvfsFuseFileSystem function"); } From cda8275d86da50816d70367e142a2fde0fd27b9c Mon Sep 17 00:00:00 2001 From: yuhui Date: Wed, 18 Dec 2024 11:18:11 +0800 Subject: [PATCH 09/47] Update --- clients/filesystem-fuse/src/main.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index 69b0a75e054..6935a6b8e2b 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -37,19 +37,19 @@ async fn create_gvfs_fuse_filesystem() { // Gvfs-fuse filesystem structure: // FuseApiHandle // ├─ SimpleFileSystem (RawFileSystem) - // │ └─ FileSystemLog (PathFileSystem) - // ├─ GravitinoComposedFileSystem (PathFileSystem) - // │ ├─ GravitinoFilesetFileSystem (PathFileSystem) - // │ │ └─ OpenDALFileSystem (PathFileSystem) - // │ │ └─ S3FileSystem (PathFileSystem) - // │ ├─ GravitinoFilesetFileSystem (PathFileSystem) - // │ │ └─ OpenDALFileSystem (PathFileSystem) - // │ │ └─ HDFSFileSystem (PathFileSystem) - // │ ├─ GravitinoFilesetFileSystem (PathFileSystem) - // │ │ └─ NasFileSystem (PathFileSystem) - // │ │ └─ JuiceFileSystem (PathFileSystem) - // │ ├─ GravitinoFilesetFileSystem (PathFileSystem) - // │ │ └─ XXXFileSystem (PathFileSystem) + // │ └─ FileSystemLog (PathFileSystem) + // │ ├─ GravitinoComposedFileSystem (PathFileSystem) + // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) + // │ │ │ └─ OpenDALFileSystem (PathFileSystem) + // │ │ │ └─ S3FileSystem (PathFileSystem) + // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) + // │ │ │ └─ OpenDALFileSystem (PathFileSystem) + // │ │ │ └─ HDFSFileSystem (PathFileSystem) + // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) + // │ │ │ └─ NasFileSystem (PathFileSystem) + // │ │ │ └─ JuiceFileSystem (PathFileSystem) + // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) + // │ │ │ └─ XXXFileSystem (PathFileSystem) // // `SimpleFileSystem` is a low-level filesystem designed to communicate with FUSE APIs. // It manages file and directory relationships, as well as file mappings. From 5f9371787add08280bc66546fe2145aa817949e3 Mon Sep 17 00:00:00 2001 From: yuhui Date: Wed, 18 Dec 2024 14:14:17 +0800 Subject: [PATCH 10/47] Update --- clients/filesystem-fuse/src/main.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index 6935a6b8e2b..244de85a7bd 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -40,14 +40,14 @@ async fn create_gvfs_fuse_filesystem() { // │ └─ FileSystemLog (PathFileSystem) // │ ├─ GravitinoComposedFileSystem (PathFileSystem) // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) - // │ │ │ └─ OpenDALFileSystem (PathFileSystem) - // │ │ │ └─ S3FileSystem (PathFileSystem) + // │ │ │ └─ S3FileSystem (PathFileSystem) + // │ │ │ └─ OpenDALFileSystem (PathFileSystem) // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) - // │ │ │ └─ OpenDALFileSystem (PathFileSystem) - // │ │ │ └─ HDFSFileSystem (PathFileSystem) + // │ │ │ └─ HDFSFileSystem (PathFileSystem) + // │ │ │ └─ OpenDALFileSystem (PathFileSystem) // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) - // │ │ │ └─ NasFileSystem (PathFileSystem) - // │ │ │ └─ JuiceFileSystem (PathFileSystem) + // │ │ │ └─ JuiceFileSystem (PathFileSystem) + // │ │ │ └─ NasFileSystem (PathFileSystem) // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) // │ │ │ └─ XXXFileSystem (PathFileSystem) // @@ -59,7 +59,7 @@ async fn create_gvfs_fuse_filesystem() { // Similar implementations include permissions, caching, and metrics. // // `GravitinoComposeFileSystem` is a composite file system that can combine multiple `GravitinoFilesetFileSystem`. - // It translates the part of catalog and schema of fileset path to a signal GravitinoFilesetFileSystem path. + // It use the part of catalog and schema of fileset path to a find actual GravitinoFilesetFileSystem. delegate the operation to the real storage. // If the user only mounts a fileset, this layer is not present. There will only be one below layer. // // `GravitinoFilesetFileSystem` is a file system that can access a fileset.It translates the fileset path to the real storage path. @@ -68,14 +68,14 @@ async fn create_gvfs_fuse_filesystem() { // `OpenDALFileSystem` is a file system that use the OpenDAL to access real storage. // it can assess the S3, HDFS, gcs, azblob and other storage. // - // `S3FileSystem` is a file system that use to access S3 storage. + // `S3FileSystem` is a file system that use `OpenDALFileSystem` to access S3 storage. + // + // `HDFSFileSystem` is a file system that use `OpenDALFileSystem` to access HDFS storage. // - // `HDFSFileSystem` is a file system that use to access HDFS storage. - // `NasFileSystem` is a filesystem that uses a locally accessible path mounted by NAS tools, such as JuiceFS. - - // `JuiceFileSystem` is a file system that use to manage JuiceFS mount and access JuiceFS storage. - + // + // `JuiceFileSystem` is a file that use `NasFileSystem` to access JuiceFS storage. + // // `XXXFileSystem is a filesystem that allows you to implement file access through your own extensions. todo!("Implement the createGvfsFuseFileSystem function"); From 39e55fed30507d05098858ff5751ebdfa578fb25 Mon Sep 17 00:00:00 2001 From: yuhui Date: Wed, 18 Dec 2024 19:28:32 +0800 Subject: [PATCH 11/47] Update for review --- clients/filesystem-fuse/src/filesystem.rs | 196 ++++++++++-------- clients/filesystem-fuse/src/main.rs | 2 +- .../src/opened_file_manager.rs | 4 +- 3 files changed, 110 insertions(+), 92 deletions(-) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 2578ab0b4d3..3e56b9e20f6 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -194,25 +194,25 @@ pub struct FileStat { } impl FileStat { - pub fn new_file_with_path(path: &str, size: u64) -> Self { + pub fn new_file_filestat_with_path(path: &str, size: u64) -> Self { let (parent, name) = split_file_path(path); - Self::new_file(parent, name, size) + Self::new_file_filestat(parent, name, size) } - pub fn new_dir_with_path(path: &str) -> Self { + pub fn new_dir_filestat_with_path(path: &str) -> Self { let (parent, name) = split_file_path(path); - Self::new_dir(parent, name) + Self::new_dir_filestat(parent, name) } - pub fn new_file(parent: &str, name: &str, size: u64) -> Self { - Self::new_file_entry(parent, name, size, FileType::RegularFile) + pub fn new_file_filestat(parent: &str, name: &str, size: u64) -> Self { + Self::new_filestat(parent, name, size, FileType::RegularFile) } - pub fn new_dir(parent: &str, name: &str) -> Self { - Self::new_file_entry(parent, name, 0, FileType::Directory) + pub fn new_dir_filestat(parent: &str, name: &str) -> Self { + Self::new_filestat(parent, name, 0, FileType::Directory) } - pub fn new_file_entry(parent: &str, name: &str, size: u64, kind: FileType) -> Self { + pub fn new_filestat(parent: &str, name: &str, size: u64, kind: FileType) -> Self { let atime = Timestamp::from(SystemTime::now()); Self { file_id: 0, @@ -282,12 +282,24 @@ impl OpenedFile { } async fn close(&mut self) -> Result<()> { + let mut errors = Vec::new(); if let Some(mut reader) = self.reader.take() { - reader.close().await?; + if let Err(e) = reader.close().await { + errors.push(e); + } } + if let Some(mut writer) = self.writer.take() { - self.flush().await?; - writer.close().await? + if let Err(e) = self.flush().await { + errors.push(e); + } + if let Err(e) = writer.close().await { + errors.push(e); + } + } + + if !errors.is_empty() { + return Err(errors.remove(0)); } Ok(()) } @@ -357,7 +369,7 @@ pub trait FileWriter: Sync + Send { /// it is used to manage the file metadata and file handle. /// The operations of the file system are implemented by the PathFileSystem. /// Note: This class is not use in the production code, it is used for the demo and testing -pub struct SimpleFileSystem { +pub struct DefaultRawFileSystem { /// file entries file_entry_manager: RwLock, /// opened files @@ -369,7 +381,7 @@ pub struct SimpleFileSystem { fs: T, } -impl SimpleFileSystem { +impl DefaultRawFileSystem { const INITIAL_FILE_ID: u64 = 10000; const ROOT_DIR_PARENT_FILE_ID: u64 = 0; const ROOT_DIR_FILE_ID: u64 = 1; @@ -398,13 +410,13 @@ impl SimpleFileSystem { } async fn get_file_entry_by_path(&self, path: &str) -> Option { - self.file_entry_manager.read().await.get_file_by_name(path) + self.file_entry_manager.read().await.get_file_by_path(path) } - async fn fill_file_id(&self, file_stat: &mut FileStat, parent_file_id: u64) { + async fn resolve_file_id_to_filestat(&self, file_stat: &mut FileStat, parent_file_id: u64) { let mut file_manager = self.file_entry_manager.write().await; - let file = file_manager.get_file_by_name(&file_stat.path); - match file { + let file_entry = file_manager.get_file_by_path(&file_stat.path); + match file_entry { None => { // allocate new file id file_stat.set_file_id(parent_file_id, self.next_file_id()); @@ -425,31 +437,31 @@ impl SimpleFileSystem { ) -> Result { let file_entry = self.get_file_entry(file_id).await?; - let mut file = { + let mut opened_file = { match kind { FileType::Directory => { self.fs - .open_dir(&file_entry.file_name, OpenFileFlags(flags)) + .open_dir(&file_entry.path, OpenFileFlags(flags)) .await? } FileType::RegularFile => { self.fs - .open_file(&file_entry.file_name, OpenFileFlags(flags)) + .open_file(&file_entry.path, OpenFileFlags(flags)) .await? } _ => return Err(Errno::from(libc::EINVAL)), } }; // set the exists file id - file.set_file_id(file_entry.parent_file_id, file_id); - let file = self.opened_file_manager.put_file(file); + opened_file.set_file_id(file_entry.parent_file_id, file_id); + let file = self.opened_file_manager.put_file(opened_file); let file = file.lock().await; Ok(file.file_handle()) } } #[async_trait] -impl RawFileSystem for SimpleFileSystem { +impl RawFileSystem for DefaultRawFileSystem { async fn init(&self) -> Result<()> { // init root directory self.file_entry_manager.write().await.insert( @@ -461,8 +473,10 @@ impl RawFileSystem for SimpleFileSystem { } async fn get_file_path(&self, file_id: u64) -> String { - let file = self.get_file_entry(file_id).await; - file.map(|x| x.file_name).unwrap_or_else(|_| "".to_string()) + let file_entry = self.get_file_entry(file_id).await; + file_entry + .map(|x| x.path) + .unwrap_or_else(|_| "".to_string()) } async fn valid_file_handle_id(&self, file_id: u64, fh: u64) -> Result<()> { @@ -481,27 +495,28 @@ impl RawFileSystem for SimpleFileSystem { } async fn stat(&self, file_id: u64) -> Result { - let file = self.get_file_entry(file_id).await?; - let mut stat = self.fs.stat(&file.file_name).await?; - stat.set_file_id(file.parent_file_id, file.file_id); - Ok(stat) + let file_entry = self.get_file_entry(file_id).await?; + let mut file_stat = self.fs.stat(&file_entry.path).await?; + file_stat.set_file_id(file_entry.parent_file_id, file_entry.file_id); + Ok(file_stat) } async fn lookup(&self, parent_file_id: u64, name: &str) -> Result { - let parent_file = self.get_file_entry(parent_file_id).await?; - let mut stat = self.fs.lookup(&parent_file.file_name, name).await?; + let parent_file_entry = self.get_file_entry(parent_file_id).await?; + let mut file_stat = self.fs.lookup(&parent_file_entry.path, name).await?; // fill the file id to file stat - self.fill_file_id(&mut stat, parent_file_id).await; - Ok(stat) + self.resolve_file_id_to_filestat(&mut file_stat, parent_file_id) + .await; + Ok(file_stat) } async fn read_dir(&self, file_id: u64) -> Result> { - let file = self.get_file_entry(file_id).await?; - let mut files = self.fs.read_dir(&file.file_name).await?; - for file in files.iter_mut() { - self.fill_file_id(file, file.file_id).await; + let file_entry = self.get_file_entry(file_id).await?; + let mut child_filestats = self.fs.read_dir(&file_entry.path).await?; + for file in child_filestats.iter_mut() { + self.resolve_file_id_to_filestat(file, file.file_id).await; } - Ok(files) + Ok(child_filestats) } async fn open_file(&self, file_id: u64, flags: u32) -> Result { @@ -515,77 +530,80 @@ impl RawFileSystem for SimpleFileSystem { } async fn create_file(&self, parent_file_id: u64, name: &str, flags: u32) -> Result { - let parent = self.get_file_entry(parent_file_id).await?; - let mut file = self + let parent_file_entry = self.get_file_entry(parent_file_id).await?; + let mut opened_file = self .fs - .create_file(&parent.file_name, name, OpenFileFlags(flags)) + .create_file(&parent_file_entry.path, name, OpenFileFlags(flags)) .await?; - file.set_file_id(parent_file_id, self.next_file_id()); + opened_file.set_file_id(parent_file_id, self.next_file_id()); // insert the new file to file entry manager { let mut file_manager = self.file_entry_manager.write().await; - file_manager.insert(parent_file_id, file.file_stat.file_id, &file.file_stat.path); + file_manager.insert( + parent_file_id, + opened_file.file_stat.file_id, + &opened_file.file_stat.path, + ); } // put the file to the opened file manager - let file = self.opened_file_manager.put_file(file); - let file = file.lock().await; - Ok(file.file_handle()) + let opened_file = self.opened_file_manager.put_file(opened_file); + let opened_file = opened_file.lock().await; + Ok(opened_file.file_handle()) } async fn create_dir(&self, parent_file_id: u64, name: &str) -> Result { - let parent = self.get_file_entry(parent_file_id).await?; - let mut file = self.fs.create_dir(&parent.file_name, name).await?; + let parent_file_entry = self.get_file_entry(parent_file_id).await?; + let mut filestat = self.fs.create_dir(&parent_file_entry.path, name).await?; - file.set_file_id(parent_file_id, self.next_file_id()); + filestat.set_file_id(parent_file_id, self.next_file_id()); // insert the new file to file entry manager { let mut file_manager = self.file_entry_manager.write().await; - file_manager.insert(file.parent_file_id, file.file_id, &file.path); + file_manager.insert(filestat.parent_file_id, filestat.file_id, &filestat.path); } - Ok(file.file_id) + Ok(filestat.file_id) } async fn set_attr(&self, file_id: u64, file_stat: &FileStat) -> Result<()> { - let file = self.get_file_entry(file_id).await?; - self.fs.set_attr(&file.file_name, file_stat, true).await + let file_entry = self.get_file_entry(file_id).await?; + self.fs.set_attr(&file_entry.path, file_stat, true).await } async fn remove_file(&self, parent_file_id: u64, name: &str) -> Result<()> { - let parent_file = self.get_file_entry(parent_file_id).await?; - self.fs.remove_file(&parent_file.file_name, name).await?; + let parent_file_entry = self.get_file_entry(parent_file_id).await?; + self.fs.remove_file(&parent_file_entry.path, name).await?; // remove the file from file entry manager { - let mut file_id_manager = self.file_entry_manager.write().await; - file_id_manager.remove(&join_file_path(&parent_file.file_name, name)); + let mut file_manager = self.file_entry_manager.write().await; + file_manager.remove(&join_file_path(&parent_file_entry.path, name)); } Ok(()) } async fn remove_dir(&self, parent_file_id: u64, name: &str) -> Result<()> { - let parent_file = self.get_file_entry(parent_file_id).await?; - self.fs.remove_dir(&parent_file.file_name, name).await?; + let parent_file_entry = self.get_file_entry(parent_file_id).await?; + self.fs.remove_dir(&parent_file_entry.path, name).await?; // remove the dir from file entry manager { - let mut file_id_manager = self.file_entry_manager.write().await; - file_id_manager.remove(&join_file_path(&parent_file.file_name, name)); + let mut file_manager = self.file_entry_manager.write().await; + file_manager.remove(&join_file_path(&parent_file_entry.path, name)); } Ok(()) } async fn close_file(&self, _file_id: u64, fh: u64) -> Result<()> { - let file = self + let opened_file = self .opened_file_manager .remove_file(fh) .ok_or(Errno::from(libc::EBADF))?; - let mut file = file.lock().await; - file.close().await?; - Ok(()) + let mut file = opened_file.lock().await; + file.close().await } async fn read(&self, _file_id: u64, fh: u64, offset: u64, size: u32) -> Result { @@ -629,23 +647,23 @@ impl RawFileSystem for SimpleFileSystem { struct FileEntry { file_id: u64, parent_file_id: u64, - file_name: String, + path: String, } /// FileEntryManager is manage all the file entries in memory. it is used manger the file relationship and name mapping. struct FileEntryManager { - // file_id_map is a map of file_id to file name. + // file_id_map is a map of file_id to file entry. file_id_map: HashMap, - // file_name_map is a map of file name to file id. - file_name_map: HashMap, + // file_path_map is a map of file path to file entry. + file_path_map: HashMap, } impl FileEntryManager { fn new() -> Self { Self { file_id_map: HashMap::new(), - file_name_map: HashMap::new(), + file_path_map: HashMap::new(), } } @@ -653,22 +671,22 @@ impl FileEntryManager { self.file_id_map.get(&file_id).cloned() } - fn get_file_by_name(&self, file_name: &str) -> Option { - self.file_name_map.get(file_name).cloned() + fn get_file_by_path(&self, path: &str) -> Option { + self.file_path_map.get(path).cloned() } - fn insert(&mut self, parent_file_id: u64, file_id: u64, file_name: &str) { + fn insert(&mut self, parent_file_id: u64, file_id: u64, path: &str) { let file = FileEntry { file_id, parent_file_id, - file_name: file_name.to_string(), + path: path.to_string(), }; self.file_id_map.insert(file_id, file.clone()); - self.file_name_map.insert(file_name.to_string(), file); + self.file_path_map.insert(path.to_string(), file); } - fn remove(&mut self, file_name: &str) { - if let Some(file) = self.file_name_map.remove(file_name) { + fn remove(&mut self, path: &str) { + if let Some(file) = self.file_path_map.remove(path) { self.file_id_map.remove(&file.file_id); } } @@ -681,28 +699,28 @@ mod tests { #[test] fn test_create_file_stat() { //test new file - let file_stat = FileStat::new_file("a", "b", 10); + let file_stat = FileStat::new_file_filestat("a", "b", 10); assert_eq!(file_stat.name, "b"); assert_eq!(file_stat.path, "a/b"); assert_eq!(file_stat.size, 10); assert_eq!(file_stat.kind, FileType::RegularFile); //test new dir - let file_stat = FileStat::new_dir("a", "b"); + let file_stat = FileStat::new_dir_filestat("a", "b"); assert_eq!(file_stat.name, "b"); assert_eq!(file_stat.path, "a/b"); assert_eq!(file_stat.size, 0); assert_eq!(file_stat.kind, FileType::Directory); //test new file with path - let file_stat = FileStat::new_file_with_path("a/b", 10); + let file_stat = FileStat::new_file_filestat_with_path("a/b", 10); assert_eq!(file_stat.name, "b"); assert_eq!(file_stat.path, "a/b"); assert_eq!(file_stat.size, 10); assert_eq!(file_stat.kind, FileType::RegularFile); //test new dir with path - let file_stat = FileStat::new_dir_with_path("a/b"); + let file_stat = FileStat::new_dir_filestat_with_path("a/b"); assert_eq!(file_stat.name, "b"); assert_eq!(file_stat.path, "a/b"); assert_eq!(file_stat.size, 0); @@ -711,7 +729,7 @@ mod tests { #[test] fn test_file_stat_set_file_id() { - let mut file_stat = FileStat::new_file("a", "b", 10); + let mut file_stat = FileStat::new_file_filestat("a", "b", 10); file_stat.set_file_id(1, 2); assert_eq!(file_stat.file_id, 2); assert_eq!(file_stat.parent_file_id, 1); @@ -720,13 +738,13 @@ mod tests { #[test] #[should_panic(expected = "assertion failed: file_id != 0 && parent_file_id != 0")] fn test_file_stat_set_file_id_panic() { - let mut file_stat = FileStat::new_file("a", "b", 10); + let mut file_stat = FileStat::new_file_filestat("a", "b", 10); file_stat.set_file_id(1, 0); } #[test] fn test_open_file() { - let mut open_file = OpenedFile::new(FileStat::new_file("a", "b", 10)); + let mut open_file = OpenedFile::new(FileStat::new_file_filestat("a", "b", 10)); assert_eq!(open_file.file_stat.name, "b"); assert_eq!(open_file.file_stat.size, 10); @@ -743,15 +761,15 @@ mod tests { let file = manager.get_file_by_id(2).unwrap(); assert_eq!(file.file_id, 2); assert_eq!(file.parent_file_id, 1); - assert_eq!(file.file_name, "a/b"); + assert_eq!(file.path, "a/b"); - let file = manager.get_file_by_name("a/b").unwrap(); + let file = manager.get_file_by_path("a/b").unwrap(); assert_eq!(file.file_id, 2); assert_eq!(file.parent_file_id, 1); - assert_eq!(file.file_name, "a/b"); + assert_eq!(file.path, "a/b"); manager.remove("a/b"); assert!(manager.get_file_by_id(2).is_none()); - assert!(manager.get_file_by_name("a/b").is_none()); + assert!(manager.get_file_by_path("a/b").is_none()); } } diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index 244de85a7bd..25a3b1579ce 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -36,7 +36,7 @@ async fn main() { async fn create_gvfs_fuse_filesystem() { // Gvfs-fuse filesystem structure: // FuseApiHandle - // ├─ SimpleFileSystem (RawFileSystem) + // ├─ DefaultRawFileSystem (RawFileSystem) // │ └─ FileSystemLog (PathFileSystem) // │ ├─ GravitinoComposedFileSystem (PathFileSystem) // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) diff --git a/clients/filesystem-fuse/src/opened_file_manager.rs b/clients/filesystem-fuse/src/opened_file_manager.rs index 03567485a4c..c7a47cb306e 100644 --- a/clients/filesystem-fuse/src/opened_file_manager.rs +++ b/clients/filesystem-fuse/src/opened_file_manager.rs @@ -73,8 +73,8 @@ mod tests { async fn test_opened_file_manager() { let manager = OpenedFileManager::new(); - let file1_stat = FileStat::new_file("", "a.txt", 13); - let file2_stat = FileStat::new_file("", "b.txt", 18); + let file1_stat = FileStat::new_file_filestat("", "a.txt", 13); + let file2_stat = FileStat::new_file_filestat("", "b.txt", 18); let file1 = OpenedFile::new(file1_stat.clone()); let file2 = OpenedFile::new(file2_stat.clone()); From 366c77a3d95db4d77835d6392aee2af238eaea66 Mon Sep 17 00:00:00 2001 From: yuhui Date: Wed, 18 Dec 2024 19:56:20 +0800 Subject: [PATCH 12/47] Update --- clients/filesystem-fuse/src/filesystem.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 3e56b9e20f6..bec9ccc004a 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -383,7 +383,7 @@ pub struct DefaultRawFileSystem { impl DefaultRawFileSystem { const INITIAL_FILE_ID: u64 = 10000; - const ROOT_DIR_PARENT_FILE_ID: u64 = 0; + const ROOT_DIR_PARENT_FILE_ID: u64 = 1; const ROOT_DIR_FILE_ID: u64 = 1; const ROOT_DIR_NAME: &'static str = ""; From 33dc3bd8142ae58e6dfda61ab3590d08ec4553df Mon Sep 17 00:00:00 2001 From: yuhui Date: Wed, 18 Dec 2024 21:50:43 +0800 Subject: [PATCH 13/47] Update --- clients/filesystem-fuse/src/filesystem.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index bec9ccc004a..a1625504707 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -271,7 +271,7 @@ impl OpenedFile { let written = writer.write(offset, data).await?; // update the file size ,mtime and atime - let end = offset + data.len() as u64; + let end = offset + written as u64; if end > self.file_stat.size { self.file_stat.size = end; } @@ -365,7 +365,7 @@ pub trait FileWriter: Sync + Send { } } -/// SimpleFileSystem is a simple implementation for the file system. +/// DefaultRawFileSystem is a simple implementation for the file system. /// it is used to manage the file metadata and file handle. /// The operations of the file system are implemented by the PathFileSystem. /// Note: This class is not use in the production code, it is used for the demo and testing @@ -676,13 +676,13 @@ impl FileEntryManager { } fn insert(&mut self, parent_file_id: u64, file_id: u64, path: &str) { - let file = FileEntry { + let file_entry = FileEntry { file_id, parent_file_id, path: path.to_string(), }; - self.file_id_map.insert(file_id, file.clone()); - self.file_path_map.insert(path.to_string(), file); + self.file_id_map.insert(file_id, file_entry.clone()); + self.file_path_map.insert(path.to_string(), file_entry); } fn remove(&mut self, path: &str) { From 865f21315d0b4705b4bd33d370aef36da113c7a4 Mon Sep 17 00:00:00 2001 From: yuhui Date: Wed, 18 Dec 2024 21:56:26 +0800 Subject: [PATCH 14/47] Update --- clients/filesystem-fuse/src/filesystem.rs | 30 +++++++-------- .../src/opened_file_manager.rs | 37 +++++-------------- 2 files changed, 25 insertions(+), 42 deletions(-) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index a1625504707..fbc0386282c 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -405,17 +405,17 @@ impl DefaultRawFileSystem { self.file_entry_manager .read() .await - .get_file_by_id(file_id) + .get_by_id(file_id) .ok_or(Errno::from(libc::ENOENT)) } async fn get_file_entry_by_path(&self, path: &str) -> Option { - self.file_entry_manager.read().await.get_file_by_path(path) + self.file_entry_manager.read().await.get_by_path(path) } async fn resolve_file_id_to_filestat(&self, file_stat: &mut FileStat, parent_file_id: u64) { let mut file_manager = self.file_entry_manager.write().await; - let file_entry = file_manager.get_file_by_path(&file_stat.path); + let file_entry = file_manager.get_by_path(&file_stat.path); match file_entry { None => { // allocate new file id @@ -454,7 +454,7 @@ impl DefaultRawFileSystem { }; // set the exists file id opened_file.set_file_id(file_entry.parent_file_id, file_id); - let file = self.opened_file_manager.put_file(opened_file); + let file = self.opened_file_manager.put(opened_file); let file = file.lock().await; Ok(file.file_handle()) } @@ -482,7 +482,7 @@ impl RawFileSystem for DefaultRawFileSystem { async fn valid_file_handle_id(&self, file_id: u64, fh: u64) -> Result<()> { let fh_file_id = self .opened_file_manager - .get_file(fh) + .get(fh) .ok_or(Errno::from(libc::EBADF))? .lock() .await @@ -549,7 +549,7 @@ impl RawFileSystem for DefaultRawFileSystem { } // put the file to the opened file manager - let opened_file = self.opened_file_manager.put_file(opened_file); + let opened_file = self.opened_file_manager.put(opened_file); let opened_file = opened_file.lock().await; Ok(opened_file.file_handle()) } @@ -600,7 +600,7 @@ impl RawFileSystem for DefaultRawFileSystem { async fn close_file(&self, _file_id: u64, fh: u64) -> Result<()> { let opened_file = self .opened_file_manager - .remove_file(fh) + .remove(fh) .ok_or(Errno::from(libc::EBADF))?; let mut file = opened_file.lock().await; file.close().await @@ -611,7 +611,7 @@ impl RawFileSystem for DefaultRawFileSystem { let data = { let opened_file = self .opened_file_manager - .get_file(fh) + .get(fh) .ok_or(Errno::from(libc::EBADF))?; let mut opened_file = opened_file.lock().await; file_stat = opened_file.file_stat.clone(); @@ -628,7 +628,7 @@ impl RawFileSystem for DefaultRawFileSystem { let (len, file_stat) = { let opened_file = self .opened_file_manager - .get_file(fh) + .get(fh) .ok_or(Errno::from(libc::EBADF))?; let mut opened_file = opened_file.lock().await; let len = opened_file.write(offset, data).await; @@ -667,11 +667,11 @@ impl FileEntryManager { } } - fn get_file_by_id(&self, file_id: u64) -> Option { + fn get_by_id(&self, file_id: u64) -> Option { self.file_id_map.get(&file_id).cloned() } - fn get_file_by_path(&self, path: &str) -> Option { + fn get_by_path(&self, path: &str) -> Option { self.file_path_map.get(path).cloned() } @@ -758,18 +758,18 @@ mod tests { fn test_file_entry_manager() { let mut manager = FileEntryManager::new(); manager.insert(1, 2, "a/b"); - let file = manager.get_file_by_id(2).unwrap(); + let file = manager.get_by_id(2).unwrap(); assert_eq!(file.file_id, 2); assert_eq!(file.parent_file_id, 1); assert_eq!(file.path, "a/b"); - let file = manager.get_file_by_path("a/b").unwrap(); + let file = manager.get_by_path("a/b").unwrap(); assert_eq!(file.file_id, 2); assert_eq!(file.parent_file_id, 1); assert_eq!(file.path, "a/b"); manager.remove("a/b"); - assert!(manager.get_file_by_id(2).is_none()); - assert!(manager.get_file_by_path("a/b").is_none()); + assert!(manager.get_by_id(2).is_none()); + assert!(manager.get_by_path("a/b").is_none()); } } diff --git a/clients/filesystem-fuse/src/opened_file_manager.rs b/clients/filesystem-fuse/src/opened_file_manager.rs index c7a47cb306e..695d69adca4 100644 --- a/clients/filesystem-fuse/src/opened_file_manager.rs +++ b/clients/filesystem-fuse/src/opened_file_manager.rs @@ -44,7 +44,7 @@ impl OpenedFileManager { .fetch_add(1, std::sync::atomic::Ordering::SeqCst) } - pub(crate) fn put_file(&self, mut file: OpenedFile) -> Arc> { + pub(crate) fn put(&self, mut file: OpenedFile) -> Arc> { let file_handle_id = self.next_handle_id(); file.handle_id = file_handle_id; let file_handle = Arc::new(Mutex::new(file)); @@ -53,13 +53,13 @@ impl OpenedFileManager { file_handle } - pub(crate) fn get_file(&self, handle_id: u64) -> Option>> { + pub(crate) fn get(&self, handle_id: u64) -> Option>> { self.file_handle_map .get(&handle_id) .map(|x| x.value().clone()) } - pub(crate) fn remove_file(&self, handle_id: u64) -> Option>> { + pub(crate) fn remove(&self, handle_id: u64) -> Option>> { self.file_handle_map.remove(&handle_id).map(|x| x.1) } } @@ -79,8 +79,8 @@ mod tests { let file1 = OpenedFile::new(file1_stat.clone()); let file2 = OpenedFile::new(file2_stat.clone()); - let handle_id1 = manager.put_file(file1).lock().await.handle_id; - let handle_id2 = manager.put_file(file2).lock().await.handle_id; + let handle_id1 = manager.put(file1).lock().await.handle_id; + let handle_id2 = manager.put(file2).lock().await.handle_id; // Test the file handle id is assigned. assert!(handle_id1 > 0 && handle_id2 > 0); @@ -88,40 +88,23 @@ mod tests { // test get file by handle id assert_eq!( - manager - .get_file(handle_id1) - .unwrap() - .lock() - .await - .file_stat - .name, + manager.get(handle_id1).unwrap().lock().await.file_stat.name, file1_stat.name ); assert_eq!( - manager - .get_file(handle_id2) - .unwrap() - .lock() - .await - .file_stat - .name, + manager.get(handle_id2).unwrap().lock().await.file_stat.name, file2_stat.name ); // test remove file by handle id assert_eq!( - manager - .remove_file(handle_id1) - .unwrap() - .lock() - .await - .handle_id, + manager.remove(handle_id1).unwrap().lock().await.handle_id, handle_id1 ); // test get file by handle id after remove - assert!(manager.get_file(handle_id1).is_none()); - assert!(manager.get_file(handle_id2).is_some()); + assert!(manager.get(handle_id1).is_none()); + assert!(manager.get(handle_id2).is_some()); } } From 2570eb8f7845ff1d239346ed25947c8a42e23575 Mon Sep 17 00:00:00 2001 From: yuhui Date: Wed, 18 Dec 2024 22:24:13 +0800 Subject: [PATCH 15/47] Update --- clients/filesystem-fuse/src/filesystem.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index fbc0386282c..219dbfae0f9 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -368,7 +368,6 @@ pub trait FileWriter: Sync + Send { /// DefaultRawFileSystem is a simple implementation for the file system. /// it is used to manage the file metadata and file handle. /// The operations of the file system are implemented by the PathFileSystem. -/// Note: This class is not use in the production code, it is used for the demo and testing pub struct DefaultRawFileSystem { /// file entries file_entry_manager: RwLock, From bfd3945c939df6566c5ead3c74280f7e8fa6d779 Mon Sep 17 00:00:00 2001 From: yuhui Date: Thu, 19 Dec 2024 09:53:11 +0800 Subject: [PATCH 16/47] Update --- clients/filesystem-fuse/src/filesystem.rs | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 219dbfae0f9..45fa5392c99 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -404,17 +404,20 @@ impl DefaultRawFileSystem { self.file_entry_manager .read() .await - .get_by_id(file_id) + .get_file_entry_by_id(file_id) .ok_or(Errno::from(libc::ENOENT)) } async fn get_file_entry_by_path(&self, path: &str) -> Option { - self.file_entry_manager.read().await.get_by_path(path) + self.file_entry_manager + .read() + .await + .get_file_entry_by_path(path) } async fn resolve_file_id_to_filestat(&self, file_stat: &mut FileStat, parent_file_id: u64) { let mut file_manager = self.file_entry_manager.write().await; - let file_entry = file_manager.get_by_path(&file_stat.path); + let file_entry = file_manager.get_file_entry_by_path(&file_stat.path); match file_entry { None => { // allocate new file id @@ -666,11 +669,11 @@ impl FileEntryManager { } } - fn get_by_id(&self, file_id: u64) -> Option { + fn get_file_entry_by_id(&self, file_id: u64) -> Option { self.file_id_map.get(&file_id).cloned() } - fn get_by_path(&self, path: &str) -> Option { + fn get_file_entry_by_path(&self, path: &str) -> Option { self.file_path_map.get(path).cloned() } @@ -757,18 +760,18 @@ mod tests { fn test_file_entry_manager() { let mut manager = FileEntryManager::new(); manager.insert(1, 2, "a/b"); - let file = manager.get_by_id(2).unwrap(); + let file = manager.get_file_entry_by_id(2).unwrap(); assert_eq!(file.file_id, 2); assert_eq!(file.parent_file_id, 1); assert_eq!(file.path, "a/b"); - let file = manager.get_by_path("a/b").unwrap(); + let file = manager.get_file_entry_by_path("a/b").unwrap(); assert_eq!(file.file_id, 2); assert_eq!(file.parent_file_id, 1); assert_eq!(file.path, "a/b"); manager.remove("a/b"); - assert!(manager.get_by_id(2).is_none()); - assert!(manager.get_by_path("a/b").is_none()); + assert!(manager.get_file_entry_by_id(2).is_none()); + assert!(manager.get_file_entry_by_path("a/b").is_none()); } } From 4fb77e1e2d0739515987cd7966c8bd9ccf48666e Mon Sep 17 00:00:00 2001 From: yuhui Date: Thu, 19 Dec 2024 12:05:23 +0800 Subject: [PATCH 17/47] Update --- clients/filesystem-fuse/src/utils.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/clients/filesystem-fuse/src/utils.rs b/clients/filesystem-fuse/src/utils.rs index 7ac0c7212a7..7619775c949 100644 --- a/clients/filesystem-fuse/src/utils.rs +++ b/clients/filesystem-fuse/src/utils.rs @@ -19,6 +19,7 @@ // join the parent and name to a path pub fn join_file_path(parent: &str, name: &str) -> String { + //TODO handle corner cases if parent.is_empty() { name.to_string() } else { From 8cf0c4a25c47170e8ec579846c208034fe617fe9 Mon Sep 17 00:00:00 2001 From: yuhui Date: Thu, 19 Dec 2024 12:09:51 +0800 Subject: [PATCH 18/47] Update --- clients/filesystem-fuse/src/filesystem.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 45fa5392c99..846f51631fb 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -94,19 +94,19 @@ pub(crate) trait PathFileSystem: Send + Sync { async fn init(&self) -> Result<()>; /// Get the file stat by file path, if the file is exist, return the file stat - async fn stat(&self, name: &str) -> Result; + async fn stat(&self, path: &str) -> Result; /// Get the file stat by parent file path and file name, if the file is exist, return the file stat async fn lookup(&self, parent: &str, name: &str) -> Result; /// Read the directory by file path, if the file is a valid directory, return the file stat list - async fn read_dir(&self, name: &str) -> Result>; + async fn read_dir(&self, path: &str) -> Result>; /// Open the file by file path and flags, if the file is exist, return the opened file - async fn open_file(&self, name: &str, flags: OpenFileFlags) -> Result; + async fn open_file(&self, path: &str, flags: OpenFileFlags) -> Result; /// Open the directory by file path and flags, if the file is exist, return the opened file - async fn open_dir(&self, name: &str, flags: OpenFileFlags) -> Result; + async fn open_dir(&self, path: &str, flags: OpenFileFlags) -> Result; /// Create the file by parent file path and file name and flags, if successful, return the opened file async fn create_file( @@ -120,7 +120,7 @@ pub(crate) trait PathFileSystem: Send + Sync { async fn create_dir(&self, parent: &str, name: &str) -> Result; /// Set the file attribute by file path and file stat - async fn set_attr(&self, name: &str, file_stat: &FileStat, flush: bool) -> Result<()>; + async fn set_attr(&self, path: &str, file_stat: &FileStat, flush: bool) -> Result<()>; /// Remove the file by parent file path and file name async fn remove_file(&self, parent: &str, name: &str) -> Result<()>; From 21b080d6edd7cc0fc076a337d1703ef9c63e102e Mon Sep 17 00:00:00 2001 From: yuhui Date: Thu, 19 Dec 2024 19:29:47 +0800 Subject: [PATCH 19/47] Update --- clients/filesystem-fuse/build.gradle.kts | 8 - .../src/default_raw_filesystem.rs | 398 +++++++++++++++ clients/filesystem-fuse/src/filesystem.rs | 480 +----------------- clients/filesystem-fuse/src/lib.rs | 2 + clients/filesystem-fuse/src/main.rs | 2 + clients/filesystem-fuse/src/opened_file.rs | 123 +++++ .../src/opened_file_manager.rs | 2 +- 7 files changed, 534 insertions(+), 481 deletions(-) create mode 100644 clients/filesystem-fuse/src/default_raw_filesystem.rs create mode 100644 clients/filesystem-fuse/src/opened_file.rs diff --git a/clients/filesystem-fuse/build.gradle.kts b/clients/filesystem-fuse/build.gradle.kts index 7e20e13cedd..7d24c86a5b0 100644 --- a/clients/filesystem-fuse/build.gradle.kts +++ b/clients/filesystem-fuse/build.gradle.kts @@ -20,8 +20,6 @@ import org.gradle.api.tasks.Exec val checkRustEnvironment by tasks.registering(Exec::class) { - description = "Check if Rust environment." - group = "verification" commandLine("bash", "-c", "cargo --version") standardOutput = System.out errorOutput = System.err @@ -30,14 +28,12 @@ val checkRustEnvironment by tasks.registering(Exec::class) { val buildRustProject by tasks.registering(Exec::class) { dependsOn(checkRustEnvironment) - description = "Compile the Rust project" workingDir = file("$projectDir") commandLine("bash", "-c", "make build") } val checkRustProject by tasks.registering(Exec::class) { dependsOn(checkRustEnvironment) - description = "Check the Rust project" workingDir = file("$projectDir") commandLine("bash", "-c", "make check") @@ -45,8 +41,6 @@ val checkRustProject by tasks.registering(Exec::class) { val testRustProject by tasks.registering(Exec::class) { dependsOn(checkRustEnvironment) - description = "Run tests in the Rust project" - group = "verification" workingDir = file("$projectDir") commandLine("bash", "-c", "make test") @@ -56,8 +50,6 @@ val testRustProject by tasks.registering(Exec::class) { val cleanRustProject by tasks.registering(Exec::class) { dependsOn(checkRustEnvironment) - description = "Run tests in the Rust project" - group = "verification" workingDir = file("$projectDir") commandLine("bash", "-c", "make clean") diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs new file mode 100644 index 00000000000..2ea2f126305 --- /dev/null +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -0,0 +1,398 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use crate::filesystem::{FileStat, PathFileSystem, RawFileSystem}; +use crate::opened_file::{FileHandle, OpenFileFlags}; +use crate::opened_file_manager::OpenedFileManager; +use crate::utils::join_file_path; +use async_trait::async_trait; +use bytes::Bytes; +use fuse3::{Errno, FileType}; +use std::collections::HashMap; +use std::sync::atomic::AtomicU64; +use tokio::sync::RwLock; + +/// DefaultRawFileSystem is a simple implementation for the file system. +/// it is used to manage the file metadata and file handle. +/// The operations of the file system are implemented by the PathFileSystem. +pub struct DefaultRawFileSystem { + /// file entries + file_entry_manager: RwLock, + /// opened files + opened_file_manager: OpenedFileManager, + /// inode id generator + file_id_generator: AtomicU64, + + /// real filesystem + fs: T, +} + +impl DefaultRawFileSystem { + const INITIAL_FILE_ID: u64 = 10000; + const ROOT_DIR_PARENT_FILE_ID: u64 = 1; + const ROOT_DIR_FILE_ID: u64 = 1; + const ROOT_DIR_NAME: &'static str = ""; + + pub(crate) fn new(fs: T) -> Self { + Self { + file_entry_manager: RwLock::new(FileEntryManager::new()), + opened_file_manager: OpenedFileManager::new(), + file_id_generator: AtomicU64::new(Self::INITIAL_FILE_ID), + fs, + } + } + + fn next_file_id(&self) -> u64 { + self.file_id_generator + .fetch_add(1, std::sync::atomic::Ordering::SeqCst) + } + + async fn get_file_entry(&self, file_id: u64) -> crate::filesystem::Result { + self.file_entry_manager + .read() + .await + .get_file_entry_by_id(file_id) + .ok_or(Errno::from(libc::ENOENT)) + } + + async fn get_file_entry_by_path(&self, path: &str) -> Option { + self.file_entry_manager + .read() + .await + .get_file_entry_by_path(path) + } + + async fn resolve_file_id_to_filestat(&self, file_stat: &mut FileStat, parent_file_id: u64) { + let mut file_manager = self.file_entry_manager.write().await; + let file_entry = file_manager.get_file_entry_by_path(&file_stat.path); + match file_entry { + None => { + // allocate new file id + file_stat.set_file_id(parent_file_id, self.next_file_id()); + file_manager.insert(file_stat.parent_file_id, file_stat.file_id, &file_stat.path); + } + Some(file) => { + // use the exist file id + file_stat.set_file_id(file.parent_file_id, file.file_id); + } + } + } + + async fn open_file_internal( + &self, + file_id: u64, + flags: u32, + kind: FileType, + ) -> crate::filesystem::Result { + let file_entry = self.get_file_entry(file_id).await?; + + let mut opened_file = { + match kind { + FileType::Directory => { + self.fs + .open_dir(&file_entry.path, OpenFileFlags(flags)) + .await? + } + FileType::RegularFile => { + self.fs + .open_file(&file_entry.path, OpenFileFlags(flags)) + .await? + } + _ => return Err(Errno::from(libc::EINVAL)), + } + }; + // set the exists file id + opened_file.set_file_id(file_entry.parent_file_id, file_id); + let file = self.opened_file_manager.put(opened_file); + let file = file.lock().await; + Ok(file.file_handle()) + } +} + +#[async_trait] +impl RawFileSystem for DefaultRawFileSystem { + async fn init(&self) -> crate::filesystem::Result<()> { + // init root directory + self.file_entry_manager.write().await.insert( + Self::ROOT_DIR_PARENT_FILE_ID, + Self::ROOT_DIR_FILE_ID, + Self::ROOT_DIR_NAME, + ); + self.fs.init().await + } + + async fn get_file_path(&self, file_id: u64) -> String { + let file_entry = self.get_file_entry(file_id).await; + file_entry + .map(|x| x.path) + .unwrap_or_else(|_| "".to_string()) + } + + async fn valid_file_handle_id(&self, file_id: u64, fh: u64) -> crate::filesystem::Result<()> { + let fh_file_id = self + .opened_file_manager + .get(fh) + .ok_or(Errno::from(libc::EBADF))? + .lock() + .await + .file_stat + .file_id; + + (file_id == fh_file_id) + .then_some(()) + .ok_or(Errno::from(libc::EBADF)) + } + + async fn stat(&self, file_id: u64) -> crate::filesystem::Result { + let file_entry = self.get_file_entry(file_id).await?; + let mut file_stat = self.fs.stat(&file_entry.path).await?; + file_stat.set_file_id(file_entry.parent_file_id, file_entry.file_id); + Ok(file_stat) + } + + async fn lookup(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result { + let parent_file_entry = self.get_file_entry(parent_file_id).await?; + let mut file_stat = self.fs.lookup(&parent_file_entry.path, name).await?; + // fill the file id to file stat + self.resolve_file_id_to_filestat(&mut file_stat, parent_file_id) + .await; + Ok(file_stat) + } + + async fn read_dir(&self, file_id: u64) -> crate::filesystem::Result> { + let file_entry = self.get_file_entry(file_id).await?; + let mut child_filestats = self.fs.read_dir(&file_entry.path).await?; + for file in child_filestats.iter_mut() { + self.resolve_file_id_to_filestat(file, file.file_id).await; + } + Ok(child_filestats) + } + + async fn open_file(&self, file_id: u64, flags: u32) -> crate::filesystem::Result { + self.open_file_internal(file_id, flags, FileType::RegularFile) + .await + } + + async fn open_dir(&self, file_id: u64, flags: u32) -> crate::filesystem::Result { + self.open_file_internal(file_id, flags, FileType::Directory) + .await + } + + async fn create_file( + &self, + parent_file_id: u64, + name: &str, + flags: u32, + ) -> crate::filesystem::Result { + let parent_file_entry = self.get_file_entry(parent_file_id).await?; + let mut opened_file = self + .fs + .create_file(&parent_file_entry.path, name, OpenFileFlags(flags)) + .await?; + + opened_file.set_file_id(parent_file_id, self.next_file_id()); + + // insert the new file to file entry manager + { + let mut file_manager = self.file_entry_manager.write().await; + file_manager.insert( + parent_file_id, + opened_file.file_stat.file_id, + &opened_file.file_stat.path, + ); + } + + // put the file to the opened file manager + let opened_file = self.opened_file_manager.put(opened_file); + let opened_file = opened_file.lock().await; + Ok(opened_file.file_handle()) + } + + async fn create_dir(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result { + let parent_file_entry = self.get_file_entry(parent_file_id).await?; + let mut filestat = self.fs.create_dir(&parent_file_entry.path, name).await?; + + filestat.set_file_id(parent_file_id, self.next_file_id()); + + // insert the new file to file entry manager + { + let mut file_manager = self.file_entry_manager.write().await; + file_manager.insert(filestat.parent_file_id, filestat.file_id, &filestat.path); + } + Ok(filestat.file_id) + } + + async fn set_attr(&self, file_id: u64, file_stat: &FileStat) -> crate::filesystem::Result<()> { + let file_entry = self.get_file_entry(file_id).await?; + self.fs.set_attr(&file_entry.path, file_stat, true).await + } + + async fn remove_file(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result<()> { + let parent_file_entry = self.get_file_entry(parent_file_id).await?; + self.fs.remove_file(&parent_file_entry.path, name).await?; + + // remove the file from file entry manager + { + let mut file_manager = self.file_entry_manager.write().await; + file_manager.remove(&join_file_path(&parent_file_entry.path, name)); + } + Ok(()) + } + + async fn remove_dir(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result<()> { + let parent_file_entry = self.get_file_entry(parent_file_id).await?; + self.fs.remove_dir(&parent_file_entry.path, name).await?; + + // remove the dir from file entry manager + { + let mut file_manager = self.file_entry_manager.write().await; + file_manager.remove(&join_file_path(&parent_file_entry.path, name)); + } + Ok(()) + } + + async fn close_file(&self, _file_id: u64, fh: u64) -> crate::filesystem::Result<()> { + let opened_file = self + .opened_file_manager + .remove(fh) + .ok_or(Errno::from(libc::EBADF))?; + let mut file = opened_file.lock().await; + file.close().await + } + + async fn read( + &self, + _file_id: u64, + fh: u64, + offset: u64, + size: u32, + ) -> crate::filesystem::Result { + let file_stat: FileStat; + let data = { + let opened_file = self + .opened_file_manager + .get(fh) + .ok_or(Errno::from(libc::EBADF))?; + let mut opened_file = opened_file.lock().await; + file_stat = opened_file.file_stat.clone(); + opened_file.read(offset, size).await + }; + + // update the file atime + self.fs.set_attr(&file_stat.path, &file_stat, false).await?; + + data + } + + async fn write( + &self, + _file_id: u64, + fh: u64, + offset: u64, + data: &[u8], + ) -> crate::filesystem::Result { + let (len, file_stat) = { + let opened_file = self + .opened_file_manager + .get(fh) + .ok_or(Errno::from(libc::EBADF))?; + let mut opened_file = opened_file.lock().await; + let len = opened_file.write(offset, data).await; + (len, opened_file.file_stat.clone()) + }; + + // update the file size, mtime and atime + self.fs.set_attr(&file_stat.path, &file_stat, false).await?; + + len + } +} + +/// File entry is represent the abstract file. +#[derive(Debug, Clone)] +struct FileEntry { + file_id: u64, + parent_file_id: u64, + path: String, +} + +/// FileEntryManager is manage all the file entries in memory. it is used manger the file relationship and name mapping. +struct FileEntryManager { + // file_id_map is a map of file_id to file entry. + file_id_map: HashMap, + + // file_path_map is a map of file path to file entry. + file_path_map: HashMap, +} + +impl FileEntryManager { + fn new() -> Self { + Self { + file_id_map: HashMap::new(), + file_path_map: HashMap::new(), + } + } + + fn get_file_entry_by_id(&self, file_id: u64) -> Option { + self.file_id_map.get(&file_id).cloned() + } + + fn get_file_entry_by_path(&self, path: &str) -> Option { + self.file_path_map.get(path).cloned() + } + + fn insert(&mut self, parent_file_id: u64, file_id: u64, path: &str) { + let file_entry = FileEntry { + file_id, + parent_file_id, + path: path.to_string(), + }; + self.file_id_map.insert(file_id, file_entry.clone()); + self.file_path_map.insert(path.to_string(), file_entry); + } + + fn remove(&mut self, path: &str) { + if let Some(file) = self.file_path_map.remove(path) { + self.file_id_map.remove(&file.file_id); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_file_entry_manager() { + let mut manager = FileEntryManager::new(); + manager.insert(1, 2, "a/b"); + let file = manager.get_file_entry_by_id(2).unwrap(); + assert_eq!(file.file_id, 2); + assert_eq!(file.parent_file_id, 1); + assert_eq!(file.path, "a/b"); + + let file = manager.get_file_entry_by_path("a/b").unwrap(); + assert_eq!(file.file_id, 2); + assert_eq!(file.parent_file_id, 1); + assert_eq!(file.path, "a/b"); + + manager.remove("a/b"); + assert!(manager.get_file_entry_by_id(2).is_none()); + assert!(manager.get_file_entry_by_path("a/b").is_none()); + } +} diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 846f51631fb..ad552c16ebb 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -16,15 +16,12 @@ * specific language governing permissions and limitations * under the License. */ -use crate::opened_file_manager::OpenedFileManager; +use crate::opened_file::{FileHandle, OpenFileFlags, OpenedFile}; use crate::utils::{join_file_path, split_file_path}; use async_trait::async_trait; use bytes::Bytes; use fuse3::{Errno, FileType, Timestamp}; -use std::collections::HashMap; -use std::sync::atomic::AtomicU64; use std::time::SystemTime; -use tokio::sync::RwLock; pub(crate) type Result = std::result::Result; @@ -49,7 +46,7 @@ pub(crate) trait RawFileSystem: Send + Sync { /// Get the file stat by file id. if the file id is valid, return the file stat async fn stat(&self, file_id: u64) -> Result; - /// Lookup the file by parent file id and file name, if the file is exist, return the file stat + /// Lookup the file by parent file id and file name, if the file exists, return the file stat async fn lookup(&self, parent_file_id: u64, name: &str) -> Result; /// Read the directory by file id, if the file id is a valid directory, return the file stat list @@ -93,22 +90,22 @@ pub(crate) trait PathFileSystem: Send + Sync { /// Init the file system async fn init(&self) -> Result<()>; - /// Get the file stat by file path, if the file is exist, return the file stat + /// Get the file stat by file path, if the file exists, return the file stat async fn stat(&self, path: &str) -> Result; - /// Get the file stat by parent file path and file name, if the file is exist, return the file stat + /// Get the file stat by parent file path and file name, if the file exists, return the file stat async fn lookup(&self, parent: &str, name: &str) -> Result; - /// Read the directory by file path, if the file is a valid directory, return the file stat list + /// Read the directory by file path, if the is a valid directory, return the file stat list async fn read_dir(&self, path: &str) -> Result>; - /// Open the file by file path and flags, if the file is exist, return the opened file + /// Open the file by file path and flags, if the file exists, return the opened file async fn open_file(&self, path: &str, flags: OpenFileFlags) -> Result; - /// Open the directory by file path and flags, if the file is exist, return the opened file + /// Open the directory by file path and flags, if the file exists, return the opened file async fn open_dir(&self, path: &str, flags: OpenFileFlags) -> Result; - /// Create the file by parent file path and file name and flags, if successful, return the opened file + /// Create the file by parent file path and file name and flags, if successful return the opened file async fn create_file( &self, parent: &str, @@ -235,107 +232,6 @@ impl FileStat { } } -/// Opened file for read or write, it is used to read or write the file content. -pub(crate) struct OpenedFile { - pub(crate) file_stat: FileStat, - - pub(crate) handle_id: u64, - - pub reader: Option>, - - pub writer: Option>, -} - -impl OpenedFile { - pub fn new(file_stat: FileStat) -> Self { - OpenedFile { - file_stat: file_stat, - handle_id: 0, - reader: None, - writer: None, - } - } - - async fn read(&mut self, offset: u64, size: u32) -> Result { - let reader = self.reader.as_mut().ok_or(Errno::from(libc::EBADF))?; - let result = reader.read(offset, size).await?; - - // update the atime - self.file_stat.atime = Timestamp::from(SystemTime::now()); - - Ok(result) - } - - async fn write(&mut self, offset: u64, data: &[u8]) -> Result { - let writer = self.writer.as_mut().ok_or(Errno::from(libc::EBADF))?; - let written = writer.write(offset, data).await?; - - // update the file size ,mtime and atime - let end = offset + written as u64; - if end > self.file_stat.size { - self.file_stat.size = end; - } - self.file_stat.atime = Timestamp::from(SystemTime::now()); - self.file_stat.mtime = self.file_stat.atime; - - Ok(written) - } - - async fn close(&mut self) -> Result<()> { - let mut errors = Vec::new(); - if let Some(mut reader) = self.reader.take() { - if let Err(e) = reader.close().await { - errors.push(e); - } - } - - if let Some(mut writer) = self.writer.take() { - if let Err(e) = self.flush().await { - errors.push(e); - } - if let Err(e) = writer.close().await { - errors.push(e); - } - } - - if !errors.is_empty() { - return Err(errors.remove(0)); - } - Ok(()) - } - - async fn flush(&mut self) -> Result<()> { - if let Some(writer) = &mut self.writer { - writer.flush().await?; - } - Ok(()) - } - - fn file_handle(&self) -> FileHandle { - debug_assert!(self.handle_id != 0); - debug_assert!(self.file_stat.file_id != 0); - FileHandle { - file_id: self.file_stat.file_id, - handle_id: self.handle_id, - } - } - - pub(crate) fn set_file_id(&mut self, parent_file_id: u64, file_id: u64) { - debug_assert!(file_id != 0 && parent_file_id != 0); - self.file_stat.set_file_id(parent_file_id, file_id) - } -} - -// FileHandle is the file handle for the opened file. -pub(crate) struct FileHandle { - pub(crate) file_id: u64, - - pub(crate) handle_id: u64, -} - -// OpenFileFlags is the open file flags for the file system. -pub struct OpenFileFlags(u32); - /// File reader interface for read file content #[async_trait] pub(crate) trait FileReader: Sync + Send { @@ -365,335 +261,6 @@ pub trait FileWriter: Sync + Send { } } -/// DefaultRawFileSystem is a simple implementation for the file system. -/// it is used to manage the file metadata and file handle. -/// The operations of the file system are implemented by the PathFileSystem. -pub struct DefaultRawFileSystem { - /// file entries - file_entry_manager: RwLock, - /// opened files - opened_file_manager: OpenedFileManager, - /// inode id generator - file_id_generator: AtomicU64, - - /// real filesystem - fs: T, -} - -impl DefaultRawFileSystem { - const INITIAL_FILE_ID: u64 = 10000; - const ROOT_DIR_PARENT_FILE_ID: u64 = 1; - const ROOT_DIR_FILE_ID: u64 = 1; - const ROOT_DIR_NAME: &'static str = ""; - - pub(crate) fn new(fs: T) -> Self { - Self { - file_entry_manager: RwLock::new(FileEntryManager::new()), - opened_file_manager: OpenedFileManager::new(), - file_id_generator: AtomicU64::new(Self::INITIAL_FILE_ID), - fs, - } - } - - fn next_file_id(&self) -> u64 { - self.file_id_generator - .fetch_add(1, std::sync::atomic::Ordering::SeqCst) - } - - async fn get_file_entry(&self, file_id: u64) -> Result { - self.file_entry_manager - .read() - .await - .get_file_entry_by_id(file_id) - .ok_or(Errno::from(libc::ENOENT)) - } - - async fn get_file_entry_by_path(&self, path: &str) -> Option { - self.file_entry_manager - .read() - .await - .get_file_entry_by_path(path) - } - - async fn resolve_file_id_to_filestat(&self, file_stat: &mut FileStat, parent_file_id: u64) { - let mut file_manager = self.file_entry_manager.write().await; - let file_entry = file_manager.get_file_entry_by_path(&file_stat.path); - match file_entry { - None => { - // allocate new file id - file_stat.set_file_id(parent_file_id, self.next_file_id()); - file_manager.insert(file_stat.parent_file_id, file_stat.file_id, &file_stat.path); - } - Some(file) => { - // use the exist file id - file_stat.set_file_id(file.parent_file_id, file.file_id); - } - } - } - - async fn open_file_internal( - &self, - file_id: u64, - flags: u32, - kind: FileType, - ) -> Result { - let file_entry = self.get_file_entry(file_id).await?; - - let mut opened_file = { - match kind { - FileType::Directory => { - self.fs - .open_dir(&file_entry.path, OpenFileFlags(flags)) - .await? - } - FileType::RegularFile => { - self.fs - .open_file(&file_entry.path, OpenFileFlags(flags)) - .await? - } - _ => return Err(Errno::from(libc::EINVAL)), - } - }; - // set the exists file id - opened_file.set_file_id(file_entry.parent_file_id, file_id); - let file = self.opened_file_manager.put(opened_file); - let file = file.lock().await; - Ok(file.file_handle()) - } -} - -#[async_trait] -impl RawFileSystem for DefaultRawFileSystem { - async fn init(&self) -> Result<()> { - // init root directory - self.file_entry_manager.write().await.insert( - Self::ROOT_DIR_PARENT_FILE_ID, - Self::ROOT_DIR_FILE_ID, - Self::ROOT_DIR_NAME, - ); - self.fs.init().await - } - - async fn get_file_path(&self, file_id: u64) -> String { - let file_entry = self.get_file_entry(file_id).await; - file_entry - .map(|x| x.path) - .unwrap_or_else(|_| "".to_string()) - } - - async fn valid_file_handle_id(&self, file_id: u64, fh: u64) -> Result<()> { - let fh_file_id = self - .opened_file_manager - .get(fh) - .ok_or(Errno::from(libc::EBADF))? - .lock() - .await - .file_stat - .file_id; - - (file_id == fh_file_id) - .then_some(()) - .ok_or(Errno::from(libc::EBADF)) - } - - async fn stat(&self, file_id: u64) -> Result { - let file_entry = self.get_file_entry(file_id).await?; - let mut file_stat = self.fs.stat(&file_entry.path).await?; - file_stat.set_file_id(file_entry.parent_file_id, file_entry.file_id); - Ok(file_stat) - } - - async fn lookup(&self, parent_file_id: u64, name: &str) -> Result { - let parent_file_entry = self.get_file_entry(parent_file_id).await?; - let mut file_stat = self.fs.lookup(&parent_file_entry.path, name).await?; - // fill the file id to file stat - self.resolve_file_id_to_filestat(&mut file_stat, parent_file_id) - .await; - Ok(file_stat) - } - - async fn read_dir(&self, file_id: u64) -> Result> { - let file_entry = self.get_file_entry(file_id).await?; - let mut child_filestats = self.fs.read_dir(&file_entry.path).await?; - for file in child_filestats.iter_mut() { - self.resolve_file_id_to_filestat(file, file.file_id).await; - } - Ok(child_filestats) - } - - async fn open_file(&self, file_id: u64, flags: u32) -> Result { - self.open_file_internal(file_id, flags, FileType::RegularFile) - .await - } - - async fn open_dir(&self, file_id: u64, flags: u32) -> Result { - self.open_file_internal(file_id, flags, FileType::Directory) - .await - } - - async fn create_file(&self, parent_file_id: u64, name: &str, flags: u32) -> Result { - let parent_file_entry = self.get_file_entry(parent_file_id).await?; - let mut opened_file = self - .fs - .create_file(&parent_file_entry.path, name, OpenFileFlags(flags)) - .await?; - - opened_file.set_file_id(parent_file_id, self.next_file_id()); - - // insert the new file to file entry manager - { - let mut file_manager = self.file_entry_manager.write().await; - file_manager.insert( - parent_file_id, - opened_file.file_stat.file_id, - &opened_file.file_stat.path, - ); - } - - // put the file to the opened file manager - let opened_file = self.opened_file_manager.put(opened_file); - let opened_file = opened_file.lock().await; - Ok(opened_file.file_handle()) - } - - async fn create_dir(&self, parent_file_id: u64, name: &str) -> Result { - let parent_file_entry = self.get_file_entry(parent_file_id).await?; - let mut filestat = self.fs.create_dir(&parent_file_entry.path, name).await?; - - filestat.set_file_id(parent_file_id, self.next_file_id()); - - // insert the new file to file entry manager - { - let mut file_manager = self.file_entry_manager.write().await; - file_manager.insert(filestat.parent_file_id, filestat.file_id, &filestat.path); - } - Ok(filestat.file_id) - } - - async fn set_attr(&self, file_id: u64, file_stat: &FileStat) -> Result<()> { - let file_entry = self.get_file_entry(file_id).await?; - self.fs.set_attr(&file_entry.path, file_stat, true).await - } - - async fn remove_file(&self, parent_file_id: u64, name: &str) -> Result<()> { - let parent_file_entry = self.get_file_entry(parent_file_id).await?; - self.fs.remove_file(&parent_file_entry.path, name).await?; - - // remove the file from file entry manager - { - let mut file_manager = self.file_entry_manager.write().await; - file_manager.remove(&join_file_path(&parent_file_entry.path, name)); - } - Ok(()) - } - - async fn remove_dir(&self, parent_file_id: u64, name: &str) -> Result<()> { - let parent_file_entry = self.get_file_entry(parent_file_id).await?; - self.fs.remove_dir(&parent_file_entry.path, name).await?; - - // remove the dir from file entry manager - { - let mut file_manager = self.file_entry_manager.write().await; - file_manager.remove(&join_file_path(&parent_file_entry.path, name)); - } - Ok(()) - } - - async fn close_file(&self, _file_id: u64, fh: u64) -> Result<()> { - let opened_file = self - .opened_file_manager - .remove(fh) - .ok_or(Errno::from(libc::EBADF))?; - let mut file = opened_file.lock().await; - file.close().await - } - - async fn read(&self, _file_id: u64, fh: u64, offset: u64, size: u32) -> Result { - let file_stat: FileStat; - let data = { - let opened_file = self - .opened_file_manager - .get(fh) - .ok_or(Errno::from(libc::EBADF))?; - let mut opened_file = opened_file.lock().await; - file_stat = opened_file.file_stat.clone(); - opened_file.read(offset, size).await - }; - - // update the file atime - self.fs.set_attr(&file_stat.path, &file_stat, false).await?; - - data - } - - async fn write(&self, _file_id: u64, fh: u64, offset: u64, data: &[u8]) -> Result { - let (len, file_stat) = { - let opened_file = self - .opened_file_manager - .get(fh) - .ok_or(Errno::from(libc::EBADF))?; - let mut opened_file = opened_file.lock().await; - let len = opened_file.write(offset, data).await; - (len, opened_file.file_stat.clone()) - }; - - // update the file size, mtime and atime - self.fs.set_attr(&file_stat.path, &file_stat, false).await?; - - len - } -} - -/// File entry is represent the abstract file. -#[derive(Debug, Clone)] -struct FileEntry { - file_id: u64, - parent_file_id: u64, - path: String, -} - -/// FileEntryManager is manage all the file entries in memory. it is used manger the file relationship and name mapping. -struct FileEntryManager { - // file_id_map is a map of file_id to file entry. - file_id_map: HashMap, - - // file_path_map is a map of file path to file entry. - file_path_map: HashMap, -} - -impl FileEntryManager { - fn new() -> Self { - Self { - file_id_map: HashMap::new(), - file_path_map: HashMap::new(), - } - } - - fn get_file_entry_by_id(&self, file_id: u64) -> Option { - self.file_id_map.get(&file_id).cloned() - } - - fn get_file_entry_by_path(&self, path: &str) -> Option { - self.file_path_map.get(path).cloned() - } - - fn insert(&mut self, parent_file_id: u64, file_id: u64, path: &str) { - let file_entry = FileEntry { - file_id, - parent_file_id, - path: path.to_string(), - }; - self.file_id_map.insert(file_id, file_entry.clone()); - self.file_path_map.insert(path.to_string(), file_entry); - } - - fn remove(&mut self, path: &str) { - if let Some(file) = self.file_path_map.remove(path) { - self.file_id_map.remove(&file.file_id); - } - } -} - #[cfg(test)] mod tests { use super::*; @@ -743,35 +310,4 @@ mod tests { let mut file_stat = FileStat::new_file_filestat("a", "b", 10); file_stat.set_file_id(1, 0); } - - #[test] - fn test_open_file() { - let mut open_file = OpenedFile::new(FileStat::new_file_filestat("a", "b", 10)); - assert_eq!(open_file.file_stat.name, "b"); - assert_eq!(open_file.file_stat.size, 10); - - open_file.set_file_id(1, 2); - - assert_eq!(open_file.file_stat.file_id, 2); - assert_eq!(open_file.file_stat.parent_file_id, 1); - } - - #[test] - fn test_file_entry_manager() { - let mut manager = FileEntryManager::new(); - manager.insert(1, 2, "a/b"); - let file = manager.get_file_entry_by_id(2).unwrap(); - assert_eq!(file.file_id, 2); - assert_eq!(file.parent_file_id, 1); - assert_eq!(file.path, "a/b"); - - let file = manager.get_file_entry_by_path("a/b").unwrap(); - assert_eq!(file.file_id, 2); - assert_eq!(file.parent_file_id, 1); - assert_eq!(file.path, "a/b"); - - manager.remove("a/b"); - assert!(manager.get_file_entry_by_id(2).is_none()); - assert!(manager.get_file_entry_by_path("a/b").is_none()); - } } diff --git a/clients/filesystem-fuse/src/lib.rs b/clients/filesystem-fuse/src/lib.rs index b531df82f79..c1689bac476 100644 --- a/clients/filesystem-fuse/src/lib.rs +++ b/clients/filesystem-fuse/src/lib.rs @@ -16,7 +16,9 @@ * specific language governing permissions and limitations * under the License. */ +mod default_raw_filesystem; mod filesystem; mod fuse_api_handle; +mod opened_file; mod opened_file_manager; mod utils; diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index 25a3b1579ce..3d8e9dbb953 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -16,8 +16,10 @@ * specific language governing permissions and limitations * under the License. */ +mod default_raw_filesystem; mod filesystem; mod fuse_api_handle; +mod opened_file; mod opened_file_manager; mod utils; diff --git a/clients/filesystem-fuse/src/opened_file.rs b/clients/filesystem-fuse/src/opened_file.rs new file mode 100644 index 00000000000..61a9af4a3bc --- /dev/null +++ b/clients/filesystem-fuse/src/opened_file.rs @@ -0,0 +1,123 @@ +use crate::filesystem::{FileReader, FileStat, FileWriter, Result}; +use bytes::Bytes; +use fuse3::{Errno, Timestamp}; +use std::time::SystemTime; + +/// Opened file for read or write, it is used to read or write the file content. +pub(crate) struct OpenedFile { + pub(crate) file_stat: FileStat, + + pub(crate) handle_id: u64, + + pub reader: Option>, + + pub writer: Option>, +} + +impl OpenedFile { + pub(crate) fn new(file_stat: FileStat) -> Self { + OpenedFile { + file_stat: file_stat, + handle_id: 0, + reader: None, + writer: None, + } + } + + pub(crate) async fn read(&mut self, offset: u64, size: u32) -> Result { + let reader = self.reader.as_mut().ok_or(Errno::from(libc::EBADF))?; + let result = reader.read(offset, size).await?; + + // update the atime + self.file_stat.atime = Timestamp::from(SystemTime::now()); + + Ok(result) + } + + pub(crate) async fn write(&mut self, offset: u64, data: &[u8]) -> Result { + let writer = self.writer.as_mut().ok_or(Errno::from(libc::EBADF))?; + let written = writer.write(offset, data).await?; + + // update the file size ,mtime and atime + let end = offset + written as u64; + if end > self.file_stat.size { + self.file_stat.size = end; + } + self.file_stat.atime = Timestamp::from(SystemTime::now()); + self.file_stat.mtime = self.file_stat.atime; + + Ok(written) + } + + pub(crate) async fn close(&mut self) -> Result<()> { + let mut errors = Vec::new(); + if let Some(mut reader) = self.reader.take() { + if let Err(e) = reader.close().await { + errors.push(e); + } + } + + if let Some(mut writer) = self.writer.take() { + if let Err(e) = self.flush().await { + errors.push(e); + } + if let Err(e) = writer.close().await { + errors.push(e); + } + } + + if !errors.is_empty() { + return Err(errors.remove(0)); + } + Ok(()) + } + + pub(crate) async fn flush(&mut self) -> Result<()> { + if let Some(writer) = &mut self.writer { + writer.flush().await?; + } + Ok(()) + } + + pub(crate) fn file_handle(&self) -> FileHandle { + debug_assert!(self.handle_id != 0); + debug_assert!(self.file_stat.file_id != 0); + FileHandle { + file_id: self.file_stat.file_id, + handle_id: self.handle_id, + } + } + + pub(crate) fn set_file_id(&mut self, parent_file_id: u64, file_id: u64) { + debug_assert!(file_id != 0 && parent_file_id != 0); + self.file_stat.set_file_id(parent_file_id, file_id) + } +} + +// FileHandle is the file handle for the opened file. +pub(crate) struct FileHandle { + pub(crate) file_id: u64, + + pub(crate) handle_id: u64, +} + +// OpenFileFlags is the open file flags for the file system. +pub(crate) struct OpenFileFlags(pub(crate) u32); + +#[cfg(test)] +mod tests { + use super::*; + use crate::filesystem::FileStat; + + #[test] + fn test_open_file() { + let mut open_file = OpenedFile::new(FileStat::new_file_filestat("a", "b", 10)); + assert_eq!(open_file.file_stat.name, "b"); + assert_eq!(open_file.file_stat.size, 10); + + open_file.set_file_id(1, 2); + + assert_eq!(open_file.file_stat.file_id, 2); + assert_eq!(open_file.file_stat.parent_file_id, 1); + } +} diff --git a/clients/filesystem-fuse/src/opened_file_manager.rs b/clients/filesystem-fuse/src/opened_file_manager.rs index 695d69adca4..005ec629145 100644 --- a/clients/filesystem-fuse/src/opened_file_manager.rs +++ b/clients/filesystem-fuse/src/opened_file_manager.rs @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -use crate::filesystem::OpenedFile; +use crate::opened_file::OpenedFile; use dashmap::DashMap; use std::sync::atomic::AtomicU64; use std::sync::Arc; From 4fe5fe3717ef1acb71f46f4aa753448fec45df15 Mon Sep 17 00:00:00 2001 From: yuhui Date: Thu, 19 Dec 2024 19:35:12 +0800 Subject: [PATCH 20/47] Update --- clients/filesystem-fuse/src/default_raw_filesystem.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index 2ea2f126305..c559955b8e2 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -35,7 +35,7 @@ pub struct DefaultRawFileSystem { file_entry_manager: RwLock, /// opened files opened_file_manager: OpenedFileManager, - /// inode id generator + /// file id generator file_id_generator: AtomicU64, /// real filesystem From 2517afa5fb90322505d163005504c5e8921ade5d Mon Sep 17 00:00:00 2001 From: yuhui Date: Thu, 19 Dec 2024 20:33:21 +0800 Subject: [PATCH 21/47] Update --- clients/filesystem-fuse/src/filesystem.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index ad552c16ebb..7a74180159b 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -96,7 +96,7 @@ pub(crate) trait PathFileSystem: Send + Sync { /// Get the file stat by parent file path and file name, if the file exists, return the file stat async fn lookup(&self, parent: &str, name: &str) -> Result; - /// Read the directory by file path, if the is a valid directory, return the file stat list + /// Read the directory by file path, if the directory exists, return the file stat list async fn read_dir(&self, path: &str) -> Result>; /// Open the file by file path and flags, if the file exists, return the opened file From d32cd6a850db0a3747b0cdecb1cbd68729e09cb6 Mon Sep 17 00:00:00 2001 From: yuhui Date: Thu, 19 Dec 2024 20:43:03 +0800 Subject: [PATCH 22/47] Add --- clients/filesystem-fuse/Cargo.toml | 1 + .../src/default_raw_filesystem.rs | 22 +- clients/filesystem-fuse/src/filesystem.rs | 64 +++-- .../filesystem-fuse/src/fuse_api_handle.rs | 2 +- clients/filesystem-fuse/src/fuse_server.rs | 115 ++++++++ clients/filesystem-fuse/src/lib.rs | 9 + clients/filesystem-fuse/src/main.rs | 62 +---- .../filesystem-fuse/src/memory_filesystem.rs | 247 ++++++++++++++++++ clients/filesystem-fuse/src/mount.rs | 106 ++++++++ clients/filesystem-fuse/tests/fuse_test.rs | 141 ++++++++++ clients/filesystem-fuse/tests/it.rs | 23 -- 11 files changed, 684 insertions(+), 108 deletions(-) create mode 100644 clients/filesystem-fuse/src/fuse_server.rs create mode 100644 clients/filesystem-fuse/src/memory_filesystem.rs create mode 100644 clients/filesystem-fuse/src/mount.rs create mode 100644 clients/filesystem-fuse/tests/fuse_test.rs delete mode 100644 clients/filesystem-fuse/tests/it.rs diff --git a/clients/filesystem-fuse/Cargo.toml b/clients/filesystem-fuse/Cargo.toml index 3bcf20f37ef..047b3e60d5b 100644 --- a/clients/filesystem-fuse/Cargo.toml +++ b/clients/filesystem-fuse/Cargo.toml @@ -42,4 +42,5 @@ libc = "0.2.168" log = "0.4.22" tokio = { version = "1.38.0", features = ["full"] } tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } +regex = "1.11.1" diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index c559955b8e2..15d7d4b9eea 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -167,7 +167,8 @@ impl RawFileSystem for DefaultRawFileSystem { async fn lookup(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result { let parent_file_entry = self.get_file_entry(parent_file_id).await?; - let mut file_stat = self.fs.lookup(&parent_file_entry.path, name).await?; + let path = join_file_path(&parent_file_entry.path, name); + let mut file_stat = self.fs.lookup(&path).await?; // fill the file id to file stat self.resolve_file_id_to_filestat(&mut file_stat, parent_file_id) .await; @@ -200,10 +201,8 @@ impl RawFileSystem for DefaultRawFileSystem { flags: u32, ) -> crate::filesystem::Result { let parent_file_entry = self.get_file_entry(parent_file_id).await?; - let mut opened_file = self - .fs - .create_file(&parent_file_entry.path, name, OpenFileFlags(flags)) - .await?; + let path = join_file_path(&parent_file_entry.path, name); + let mut opened_file = self.fs.create_file(&path, OpenFileFlags(flags)).await?; opened_file.set_file_id(parent_file_id, self.next_file_id()); @@ -225,7 +224,8 @@ impl RawFileSystem for DefaultRawFileSystem { async fn create_dir(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result { let parent_file_entry = self.get_file_entry(parent_file_id).await?; - let mut filestat = self.fs.create_dir(&parent_file_entry.path, name).await?; + let path = join_file_path(&parent_file_entry.path, name); + let mut filestat = self.fs.create_dir(&path).await?; filestat.set_file_id(parent_file_id, self.next_file_id()); @@ -244,24 +244,26 @@ impl RawFileSystem for DefaultRawFileSystem { async fn remove_file(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result<()> { let parent_file_entry = self.get_file_entry(parent_file_id).await?; - self.fs.remove_file(&parent_file_entry.path, name).await?; + let path = join_file_path(&parent_file_entry.path, name); + self.fs.remove_file(&path).await?; // remove the file from file entry manager { let mut file_manager = self.file_entry_manager.write().await; - file_manager.remove(&join_file_path(&parent_file_entry.path, name)); + file_manager.remove(&path); } Ok(()) } async fn remove_dir(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result<()> { let parent_file_entry = self.get_file_entry(parent_file_id).await?; - self.fs.remove_dir(&parent_file_entry.path, name).await?; + let path = join_file_path(&parent_file_entry.path, name); + self.fs.remove_dir(&path).await?; // remove the dir from file entry manager { let mut file_manager = self.file_entry_manager.write().await; - file_manager.remove(&join_file_path(&parent_file_entry.path, name)); + file_manager.remove(&path); } Ok(()) } diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 7a74180159b..31d104824f3 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -46,7 +46,7 @@ pub(crate) trait RawFileSystem: Send + Sync { /// Get the file stat by file id. if the file id is valid, return the file stat async fn stat(&self, file_id: u64) -> Result; - /// Lookup the file by parent file id and file name, if the file exists, return the file stat + /// Lookup the file by parent file id and file name, if the file is exist, return the file stat async fn lookup(&self, parent_file_id: u64, name: &str) -> Result; /// Read the directory by file id, if the file id is a valid directory, return the file stat list @@ -90,40 +90,35 @@ pub(crate) trait PathFileSystem: Send + Sync { /// Init the file system async fn init(&self) -> Result<()>; - /// Get the file stat by file path, if the file exists, return the file stat + /// Get the file stat by file path, if the file is exist, return the file stat async fn stat(&self, path: &str) -> Result; - /// Get the file stat by parent file path and file name, if the file exists, return the file stat - async fn lookup(&self, parent: &str, name: &str) -> Result; + /// Get the file stat by file path, if the file is exist, return the file stat + async fn lookup(&self, path: &str) -> Result; /// Read the directory by file path, if the directory exists, return the file stat list async fn read_dir(&self, path: &str) -> Result>; - /// Open the file by file path and flags, if the file exists, return the opened file + /// Open the file by file path and flags, if the file is exist, return the opened file async fn open_file(&self, path: &str, flags: OpenFileFlags) -> Result; - /// Open the directory by file path and flags, if the file exists, return the opened file + /// Open the directory by file path and flags, if the file is exist, return the opened file async fn open_dir(&self, path: &str, flags: OpenFileFlags) -> Result; - /// Create the file by parent file path and file name and flags, if successful return the opened file - async fn create_file( - &self, - parent: &str, - name: &str, - flags: OpenFileFlags, - ) -> Result; + /// Create the file by file path and flags, if successful, return the opened file + async fn create_file(&self, path: &str, flags: OpenFileFlags) -> Result; - /// Create the directory by parent file path and file name, if successful, return the file stat - async fn create_dir(&self, parent: &str, name: &str) -> Result; + /// Create the directory by file path , if successful, return the file stat + async fn create_dir(&self, path: &str) -> Result; /// Set the file attribute by file path and file stat async fn set_attr(&self, path: &str, file_stat: &FileStat, flush: bool) -> Result<()>; - /// Remove the file by parent file path and file name - async fn remove_file(&self, parent: &str, name: &str) -> Result<()>; + /// Remove the file by file path + async fn remove_file(&self, path: &str) -> Result<()>; - /// Remove the directory by parent file path and file name - async fn remove_dir(&self, parent: &str, name: &str) -> Result<()>; + /// Remove the directory by file path + async fn remove_dir(&self, path: &str) -> Result<()>; } // FileSystemContext is the system environment for the fuse file system. @@ -310,4 +305,35 @@ mod tests { let mut file_stat = FileStat::new_file_filestat("a", "b", 10); file_stat.set_file_id(1, 0); } + + #[test] + fn test_open_file() { + let mut open_file = OpenedFile::new(FileStat::new_file_filestat("a", "b", 10)); + assert_eq!(open_file.file_stat.name, "b"); + assert_eq!(open_file.file_stat.size, 10); + + open_file.set_file_id(1, 2); + + assert_eq!(open_file.file_stat.file_id, 2); + assert_eq!(open_file.file_stat.parent_file_id, 1); + } + + #[test] + fn test_file_entry_manager() { + let mut manager = FileEntryManager::new(); + manager.insert(1, 2, "a/b"); + let file = manager.get_file_entry_by_id(2).unwrap(); + assert_eq!(file.file_id, 2); + assert_eq!(file.parent_file_id, 1); + assert_eq!(file.path, "a/b"); + + let file = manager.get_file_entry_by_path("a/b").unwrap(); + assert_eq!(file.file_id, 2); + assert_eq!(file.parent_file_id, 1); + assert_eq!(file.path, "a/b"); + + manager.remove("a/b"); + assert!(manager.get_file_entry_by_id(2).is_none()); + assert!(manager.get_file_entry_by_path("a/b").is_none()); + } } diff --git a/clients/filesystem-fuse/src/fuse_api_handle.rs b/clients/filesystem-fuse/src/fuse_api_handle.rs index 76af581446b..f1398b29438 100644 --- a/clients/filesystem-fuse/src/fuse_api_handle.rs +++ b/clients/filesystem-fuse/src/fuse_api_handle.rs @@ -34,7 +34,7 @@ use std::ffi::{OsStr, OsString}; use std::num::NonZeroU32; use std::time::{Duration, SystemTime}; -pub(crate) struct FuseApiHandle { +pub struct FuseApiHandle { fs: T, default_ttl: Duration, fs_context: FileSystemContext, diff --git a/clients/filesystem-fuse/src/fuse_server.rs b/clients/filesystem-fuse/src/fuse_server.rs new file mode 100644 index 00000000000..2aca413d586 --- /dev/null +++ b/clients/filesystem-fuse/src/fuse_server.rs @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use fuse3::raw::{Filesystem, MountHandle, Session}; +use fuse3::{MountOptions, Result}; +use log::{error, info}; +use std::process::exit; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::{Mutex, Notify}; +use tokio::time::timeout; + +/// Represents a FUSE server capable of starting and stopping the FUSE filesystem. +pub struct FuseServer { + // Notification for stop + close_notify: Arc, + + // Shared handle to manage FUSE unmounting + mount_handle: Arc>>, // Shared handle to manage FUSE unmounting + + // Mount point of the FUSE filesystem + mount_point: String, +} + +impl FuseServer { + /// Creates a new instance of `FuseServer`. + pub fn new(mount_point: &str) -> Self { + Self { + close_notify: Arc::new(Notify::new()), + mount_handle: Arc::new(Mutex::new(None)), + mount_point: mount_point.to_string(), + } + } + + /// Starts the FUSE filesystem and blocks until it is stopped. + pub async fn start(&self, fuse_fs: impl Filesystem + Sync + 'static) -> Result<()> { + //check if the mount point exists + if !std::path::Path::new(&self.mount_point).exists() { + error!("Mount point {} does not exist", self.mount_point); + exit(libc::ENOENT); + } + + info!( + "Starting FUSE filesystem and mounting at {}", + self.mount_point + ); + + let mount_options = MountOptions::default(); + let mount_handle = Session::new(mount_options) + .mount_with_unprivileged(fuse_fs, &self.mount_point) + .await?; + + { + let mut handle_guard = self.mount_handle.lock().await; + *handle_guard = Some(mount_handle); + } + + // Wait for stop notification + self.close_notify.notified().await; + + info!("Received stop notification, FUSE filesystem will be unmounted."); + Ok(()) + } + + /// Stops the FUSE filesystem and waits for unmounting to complete. + pub async fn stop(&self) -> Result<()> { + // Notify stop + self.close_notify.notify_one(); + + info!("Stopping FUSE filesystem..."); + let timeout_duration = Duration::from_secs(5); + + let handle = { + let mut handle_guard = self.mount_handle.lock().await; + handle_guard.take() // Take the handle out to unmount + }; + + if let Some(mount_handle) = handle { + let res = timeout(timeout_duration, mount_handle.unmount()).await; + + match res { + Ok(Ok(())) => { + info!("FUSE filesystem unmounted successfully."); + Ok(()) + } + Ok(Err(e)) => { + error!("Failed to unmount FUSE filesystem: {:?}", e); + Err(e.into()) + } + Err(_) => { + error!("Unmount timed out."); + Err(libc::ETIMEDOUT.into()) + } + } + } else { + error!("No active mount handle to unmount."); + Err(libc::EBADF.into()) + } + } +} diff --git a/clients/filesystem-fuse/src/lib.rs b/clients/filesystem-fuse/src/lib.rs index c1689bac476..6cd8552bbe2 100644 --- a/clients/filesystem-fuse/src/lib.rs +++ b/clients/filesystem-fuse/src/lib.rs @@ -19,6 +19,15 @@ mod default_raw_filesystem; mod filesystem; mod fuse_api_handle; +mod fuse_server; +mod memory_filesystem; +mod mount; mod opened_file; mod opened_file_manager; mod utils; + +pub async fn gvfs_mount() -> fuse3::Result<()> { + mount::mount().await +} + +pub fn gvfs_unmount() {} diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index 3d8e9dbb953..6212978fe95 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -19,66 +19,18 @@ mod default_raw_filesystem; mod filesystem; mod fuse_api_handle; +mod fuse_server; +mod memory_filesystem; +mod mount; mod opened_file; mod opened_file_manager; mod utils; -use log::debug; -use log::info; -use std::process::exit; +use gvfs_fuse::gvfs_mount; #[tokio::main] -async fn main() { +async fn main() -> fuse3::Result<()> { tracing_subscriber::fmt().init(); - info!("Starting filesystem..."); - debug!("Shutdown filesystem..."); - exit(0); -} - -async fn create_gvfs_fuse_filesystem() { - // Gvfs-fuse filesystem structure: - // FuseApiHandle - // ├─ DefaultRawFileSystem (RawFileSystem) - // │ └─ FileSystemLog (PathFileSystem) - // │ ├─ GravitinoComposedFileSystem (PathFileSystem) - // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) - // │ │ │ └─ S3FileSystem (PathFileSystem) - // │ │ │ └─ OpenDALFileSystem (PathFileSystem) - // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) - // │ │ │ └─ HDFSFileSystem (PathFileSystem) - // │ │ │ └─ OpenDALFileSystem (PathFileSystem) - // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) - // │ │ │ └─ JuiceFileSystem (PathFileSystem) - // │ │ │ └─ NasFileSystem (PathFileSystem) - // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) - // │ │ │ └─ XXXFileSystem (PathFileSystem) - // - // `SimpleFileSystem` is a low-level filesystem designed to communicate with FUSE APIs. - // It manages file and directory relationships, as well as file mappings. - // It delegates file operations to the PathFileSystem - // - // `FileSystemLog` is a decorator that adds extra debug logging functionality to file system APIs. - // Similar implementations include permissions, caching, and metrics. - // - // `GravitinoComposeFileSystem` is a composite file system that can combine multiple `GravitinoFilesetFileSystem`. - // It use the part of catalog and schema of fileset path to a find actual GravitinoFilesetFileSystem. delegate the operation to the real storage. - // If the user only mounts a fileset, this layer is not present. There will only be one below layer. - // - // `GravitinoFilesetFileSystem` is a file system that can access a fileset.It translates the fileset path to the real storage path. - // and delegate the operation to the real storage. - // - // `OpenDALFileSystem` is a file system that use the OpenDAL to access real storage. - // it can assess the S3, HDFS, gcs, azblob and other storage. - // - // `S3FileSystem` is a file system that use `OpenDALFileSystem` to access S3 storage. - // - // `HDFSFileSystem` is a file system that use `OpenDALFileSystem` to access HDFS storage. - // - // `NasFileSystem` is a filesystem that uses a locally accessible path mounted by NAS tools, such as JuiceFS. - // - // `JuiceFileSystem` is a file that use `NasFileSystem` to access JuiceFS storage. - // - // `XXXFileSystem is a filesystem that allows you to implement file access through your own extensions. - - todo!("Implement the createGvfsFuseFileSystem function"); + gvfs_mount().await?; + Ok(()) } diff --git a/clients/filesystem-fuse/src/memory_filesystem.rs b/clients/filesystem-fuse/src/memory_filesystem.rs new file mode 100644 index 00000000000..7371390e9fe --- /dev/null +++ b/clients/filesystem-fuse/src/memory_filesystem.rs @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use crate::filesystem::{FileReader, FileStat, FileWriter, PathFileSystem, Result}; +use crate::opened_file::{OpenFileFlags, OpenedFile}; +use async_trait::async_trait; +use bytes::Bytes; +use fuse3::FileType::{Directory, RegularFile}; +use fuse3::{Errno, FileType}; +use regex::Regex; +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex, RwLock}; + +// Simple in-memory file implementation of MemoryFileSystem +struct MemoryFile { + kind: FileType, + data: Arc>>, +} + +// MemoryFileSystem is a simple in-memory filesystem implementation +// It is used for testing purposes +pub struct MemoryFileSystem { + // file_map is a map of file name to file size + file_map: RwLock>, +} + +impl MemoryFileSystem { + const FS_META_FILE_NAME: &'static str = ".gvfs_meta"; + + pub(crate) async fn new() -> Self { + Self { + file_map: RwLock::new(Default::default()), + } + } + + fn create_file_stat(&self, path: &str, file: &MemoryFile) -> FileStat { + match file.kind { + Directory => FileStat::new_dir_filestat_with_path(path), + _ => { + FileStat::new_file_filestat_with_path(path, file.data.lock().unwrap().len() as u64) + } + } + } +} + +#[async_trait] +impl PathFileSystem for MemoryFileSystem { + async fn init(&self) -> Result<()> { + let root = MemoryFile { + kind: Directory, + data: Arc::new(Mutex::new(Vec::new())), + }; + self.file_map.write().unwrap().insert("/".to_string(), root); + + let meta = MemoryFile { + kind: RegularFile, + data: Arc::new(Mutex::new(Vec::new())), + }; + self.file_map + .write() + .unwrap() + .insert(Self::FS_META_FILE_NAME.to_string(), meta); + Ok(()) + } + + async fn stat(&self, name: &str) -> Result { + self.file_map + .read() + .unwrap() + .get(name) + .map(|x| self.create_file_stat(name, x)) + .ok_or(Errno::from(libc::ENOENT)) + } + + async fn lookup(&self, path: &str) -> Result { + self.stat(path).await + } + + async fn read_dir(&self, path: &str) -> Result> { + let file_map = self.file_map.read().unwrap(); + + let results: Vec = file_map + .iter() + .filter(|x| dir_child_reg_expr(path).is_match(x.0)) + .map(|(k, v)| self.create_file_stat(k, v)) + .collect(); + + Ok(results) + } + + async fn open_file(&self, path: &str, _flags: OpenFileFlags) -> Result { + let file_stat = self.stat(path).await?; + let mut file = OpenedFile::new(file_stat.clone()); + match file.file_stat.kind { + Directory => Ok(file), + RegularFile => { + let data = self + .file_map + .read() + .unwrap() + .get(&file.file_stat.path) + .unwrap() + .data + .clone(); + file.reader = Some(Box::new(MemoryFileReader { data: data.clone() })); + file.writer = Some(Box::new(MemoryFileWriter { data: data })); + Ok(file) + } + _ => Err(Errno::from(libc::EBADF)), + } + } + + async fn open_dir(&self, name: &str, flags: OpenFileFlags) -> Result { + self.open_file(name, flags).await + } + + async fn create_file(&self, path: &str, _flags: OpenFileFlags) -> Result { + { + let file_map = self.file_map.read().unwrap(); + if file_map.contains_key(path) { + return Err(Errno::from(libc::EEXIST)); + } + }; + + let mut file = OpenedFile::new(FileStat::new_file_filestat_with_path(path, 0)); + + let data = Arc::new(Mutex::new(Vec::new())); + self.file_map.write().unwrap().insert( + file.file_stat.path.clone(), + MemoryFile { + kind: RegularFile, + data: data.clone(), + }, + ); + file.reader = Some(Box::new(MemoryFileReader { data: data.clone() })); + file.writer = Some(Box::new(MemoryFileWriter { data: data })); + + Ok(file) + } + + async fn create_dir(&self, path: &str) -> Result { + { + let file_map = self.file_map.read().unwrap(); + if file_map.contains_key(path) { + return Err(Errno::from(libc::EEXIST)); + } + } + + let file = FileStat::new_dir_filestat_with_path(path); + self.file_map.write().unwrap().insert( + file.path.clone(), + MemoryFile { + kind: Directory, + data: Arc::new(Mutex::new(Vec::new())), + }, + ); + + Ok(file) + } + + async fn set_attr(&self, _name: &str, _file_stat: &FileStat, _flush: bool) -> Result<()> { + Ok(()) + } + + async fn remove_file(&self, path: &str) -> Result<()> { + let mut file_map = self.file_map.write().unwrap(); + if file_map.remove(path).is_none() { + return Err(Errno::from(libc::ENOENT)); + } + Ok(()) + } + + async fn remove_dir(&self, path: &str) -> Result<()> { + let mut file_map = self.file_map.write().unwrap(); + let count = file_map + .iter() + .filter(|x| dir_child_reg_expr(path).is_match(x.0)) + .count(); + + if count != 0 { + return Err(Errno::from(libc::ENOTEMPTY)); + } + + file_map.remove(path); + Ok(()) + } +} + +pub(crate) struct MemoryFileReader { + pub(crate) data: Arc>>, +} + +#[async_trait] +impl FileReader for MemoryFileReader { + async fn read(&mut self, offset: u64, size: u32) -> Result { + let v = self.data.lock().unwrap(); + let start = offset as usize; + let end = usize::min(start + size as usize, v.len()); + if start >= v.len() { + return Ok(Bytes::default()); + } + Ok(v[start..end].to_vec().into()) + } +} + +pub(crate) struct MemoryFileWriter { + pub(crate) data: Arc>>, +} + +#[async_trait] +impl FileWriter for MemoryFileWriter { + async fn write(&mut self, offset: u64, data: &[u8]) -> Result { + let mut v = self.data.lock().unwrap(); + let start = offset as usize; + let end = start + data.len(); + + if v.len() < end { + v.resize(end, 0); + } + v[start..end].copy_from_slice(data); + Ok(data.len() as u32) + } +} + +fn dir_child_reg_expr(name: &str) -> Regex { + let regex_pattern = if name.is_empty() { + r"^[^/]+$".to_string() + } else { + format!(r"^{}/[^/]+$", name) + }; + Regex::new(®ex_pattern).unwrap() +} diff --git a/clients/filesystem-fuse/src/mount.rs b/clients/filesystem-fuse/src/mount.rs new file mode 100644 index 00000000000..f3b451ba41e --- /dev/null +++ b/clients/filesystem-fuse/src/mount.rs @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use crate::default_raw_filesystem::DefaultRawFileSystem; +use crate::filesystem::FileSystemContext; +use crate::fuse_api_handle::FuseApiHandle; +use crate::fuse_server::FuseServer; +use crate::memory_filesystem::MemoryFileSystem; +use fuse3::raw::Filesystem; +use log::{debug, info}; +use std::sync::Arc; + +pub async fn mount() -> fuse3::Result<()> { + debug!("Starting gvfs-fuse server..."); + let server = Arc::new(FuseServer::new("gvfs")); + + let fs = create_fuse_fs().await; + server.start(fs).await?; + + tokio::signal::ctrl_c().await?; + info!("Received Ctrl+C, stopping server..."); + server.stop().await +} + +pub async fn unmount() { + todo!("Implement the unmount function"); +} + +pub async fn create_fuse_fs() -> impl Filesystem + Sync + 'static { + let uid = unsafe { libc::getuid() }; + let gid = unsafe { libc::getgid() }; + let fs_context = FileSystemContext { + uid: uid, + gid: gid, + default_file_perm: 0, + default_dir_perm: 0, + block_size: 0, + }; + + let gvfs = MemoryFileSystem::new().await; + let fs = DefaultRawFileSystem::new(gvfs); + FuseApiHandle::new(fs, fs_context) +} + +pub async fn create_gvfs_filesystem() { + // Gvfs-fuse filesystem structure: + // FuseApiHandle + // ├─ DefaultRawFileSystem (RawFileSystem) + // │ └─ FileSystemLog (PathFileSystem) + // │ ├─ GravitinoComposedFileSystem (PathFileSystem) + // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) + // │ │ │ └─ S3FileSystem (PathFileSystem) + // │ │ │ └─ OpenDALFileSystem (PathFileSystem) + // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) + // │ │ │ └─ HDFSFileSystem (PathFileSystem) + // │ │ │ └─ OpenDALFileSystem (PathFileSystem) + // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) + // │ │ │ └─ JuiceFileSystem (PathFileSystem) + // │ │ │ └─ NasFileSystem (PathFileSystem) + // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) + // │ │ │ └─ XXXFileSystem (PathFileSystem) + // + // `SimpleFileSystem` is a low-level filesystem designed to communicate with FUSE APIs. + // It manages file and directory relationships, as well as file mappings. + // It delegates file operations to the PathFileSystem + // + // `FileSystemLog` is a decorator that adds extra debug logging functionality to file system APIs. + // Similar implementations include permissions, caching, and metrics. + // + // `GravitinoComposeFileSystem` is a composite file system that can combine multiple `GravitinoFilesetFileSystem`. + // It use the part of catalog and schema of fileset path to a find actual GravitinoFilesetFileSystem. delegate the operation to the real storage. + // If the user only mounts a fileset, this layer is not present. There will only be one below layer. + // + // `GravitinoFilesetFileSystem` is a file system that can access a fileset.It translates the fileset path to the real storage path. + // and delegate the operation to the real storage. + // + // `OpenDALFileSystem` is a file system that use the OpenDAL to access real storage. + // it can assess the S3, HDFS, gcs, azblob and other storage. + // + // `S3FileSystem` is a file system that use `OpenDALFileSystem` to access S3 storage. + // + // `HDFSFileSystem` is a file system that use `OpenDALFileSystem` to access HDFS storage. + // + // `NasFileSystem` is a filesystem that uses a locally accessible path mounted by NAS tools, such as JuiceFS. + // + // `JuiceFileSystem` is a file that use `NasFileSystem` to access JuiceFS storage. + // + // `XXXFileSystem is a filesystem that allows you to implement file access through your own extensions. + + todo!("Implement the createGvfsFuseFileSystem function"); +} diff --git a/clients/filesystem-fuse/tests/fuse_test.rs b/clients/filesystem-fuse/tests/fuse_test.rs new file mode 100644 index 00000000000..d81c667c49c --- /dev/null +++ b/clients/filesystem-fuse/tests/fuse_test.rs @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use gvfs_fuse::{gvfs_mount, gvfs_unmount}; +use log::info; +use std::fs; +use std::fs::File; +use std::path::Path; +use std::sync::Arc; +use std::thread::sleep; +use std::time::{Duration, Instant}; +use tokio::runtime::Runtime; + +struct FuseTest { + runtime: Arc, + mount_point: String, +} + +impl FuseTest { + pub fn setup(&self) { + info!("Start gvfs fuse server"); + self.runtime.spawn(async move { gvfs_mount().await }); + let success = Self::wait_for_fuse_server_ready(&self.mount_point, Duration::from_secs(15)); + assert!(success, "Fuse server cannot start up at 15 seconds"); + } + + pub fn shutdown(&self) { + self.runtime.block_on(async { + gvfs_unmount(); + }); + } + + fn wait_for_fuse_server_ready(path: &str, timeout: Duration) -> bool { + let test_file = format!("{}/.gvfs_meta", path); + let start_time = Instant::now(); + + while start_time.elapsed() < timeout { + if let Ok(exists) = fs::exists(&test_file) { + info!("Wait for fuse server ready: {}", exists); + if exists { + return true; + } + } + sleep(Duration::from_secs(1)); + } + false + } +} + +impl Drop for FuseTest { + fn drop(&mut self) { + info!("Shutdown fuse server"); + self.shutdown(); + } +} + +#[test] +fn test_fuse_system_with_auto() { + tracing_subscriber::fmt().init(); + + let test = FuseTest { + runtime: Arc::new(Runtime::new().unwrap()), + mount_point: "build/gvfs".to_string(), + }; + + test.setup(); + + let mount_point = "build/gvfs"; + let _ = fs::create_dir_all(mount_point); + + test_fuse_filesystem(mount_point); +} + +fn test_fuse_system_with_manual() { + test_fuse_filesystem("build/gvfs"); +} + +fn test_fuse_filesystem(mount_point: &str) { + info!("Test startup"); + let base_path = Path::new(mount_point); + + //test create file + let test_file = base_path.join("test_create"); + let file = File::create(&test_file).expect("Failed to create file"); + assert!(file.metadata().is_ok(), "Failed to get file metadata"); + assert!(fs::exists(&test_file).expect("File is not created")); + + //test write file + fs::write(&test_file, "read test").expect("Failed to write file"); + + //test read file + let content = fs::read_to_string(test_file.clone()).expect("Failed to read file"); + assert_eq!(content, "read test", "File content mismatch"); + + //test delete file + fs::remove_file(test_file.clone()).expect("Failed to delete file"); + assert!(!fs::exists(test_file).expect("File is not deleted")); + + //test create directory + let test_dir = base_path.join("test_dir"); + fs::create_dir(&test_dir).expect("Failed to create directory"); + + //test create file in directory + let test_file = base_path.join("test_dir/test_file"); + let file = File::create(&test_file).expect("Failed to create file"); + assert!(file.metadata().is_ok(), "Failed to get file metadata"); + + //test write file in directory + let test_file = base_path.join("test_dir/test_read"); + fs::write(&test_file, "read test").expect("Failed to write file"); + + //test read file in directory + let content = fs::read_to_string(&test_file).expect("Failed to read file"); + assert_eq!(content, "read test", "File content mismatch"); + + //test delete file in directory + fs::remove_file(&test_file).expect("Failed to delete file"); + assert!(!fs::exists(&test_file).expect("File is not deleted")); + + //test delete directory + fs::remove_dir_all(&test_dir).expect("Failed to delete directory"); + assert!(!fs::exists(&test_dir).expect("Directory is not deleted")); + + info!("Success test") +} diff --git a/clients/filesystem-fuse/tests/it.rs b/clients/filesystem-fuse/tests/it.rs deleted file mode 100644 index 989e5f9895e..00000000000 --- a/clients/filesystem-fuse/tests/it.rs +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#[test] -fn test_math_add() { - assert_eq!(1, 1); -} From 53e4c4e095e8b18f41b79076bd31626e661b4f0b Mon Sep 17 00:00:00 2001 From: yuhui Date: Thu, 19 Dec 2024 20:45:09 +0800 Subject: [PATCH 23/47] remove unused mod  Conflicts:  clients/filesystem-fuse/src/main.rs  Conflicts:  clients/filesystem-fuse/src/filesystem.rs  clients/filesystem-fuse/src/main.rs --- clients/filesystem-fuse/Cargo.toml | 2 +- clients/filesystem-fuse/src/filesystem.rs | 41 +++---------------- .../filesystem-fuse/src/fuse_api_handle.rs | 2 +- clients/filesystem-fuse/tests/fuse_test.rs | 20 +++++---- 4 files changed, 18 insertions(+), 47 deletions(-) diff --git a/clients/filesystem-fuse/Cargo.toml b/clients/filesystem-fuse/Cargo.toml index 047b3e60d5b..7c87434456a 100644 --- a/clients/filesystem-fuse/Cargo.toml +++ b/clients/filesystem-fuse/Cargo.toml @@ -40,7 +40,7 @@ fuse3 = { version = "0.8.1", "features" = ["tokio-runtime", "unprivileged"] } futures-util = "0.3.30" libc = "0.2.168" log = "0.4.22" +regex = "1.11.1" tokio = { version = "1.38.0", features = ["full"] } tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } -regex = "1.11.1" diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 31d104824f3..55e6c854b76 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -46,7 +46,7 @@ pub(crate) trait RawFileSystem: Send + Sync { /// Get the file stat by file id. if the file id is valid, return the file stat async fn stat(&self, file_id: u64) -> Result; - /// Lookup the file by parent file id and file name, if the file is exist, return the file stat + /// Lookup the file by parent file id and file name, if the file exists, return the file stat async fn lookup(&self, parent_file_id: u64, name: &str) -> Result; /// Read the directory by file id, if the file id is a valid directory, return the file stat list @@ -90,19 +90,19 @@ pub(crate) trait PathFileSystem: Send + Sync { /// Init the file system async fn init(&self) -> Result<()>; - /// Get the file stat by file path, if the file is exist, return the file stat + /// Get the file stat by file path, if the file exists, return the file stat async fn stat(&self, path: &str) -> Result; - /// Get the file stat by file path, if the file is exist, return the file stat + /// Get the file stat by parent file path and file name, if the file exists, return the file stat async fn lookup(&self, path: &str) -> Result; /// Read the directory by file path, if the directory exists, return the file stat list async fn read_dir(&self, path: &str) -> Result>; - /// Open the file by file path and flags, if the file is exist, return the opened file + /// Open the file by file path and flags, if the file exists, return the opened file async fn open_file(&self, path: &str, flags: OpenFileFlags) -> Result; - /// Open the directory by file path and flags, if the file is exist, return the opened file + /// Open the directory by file path and flags, if the file exists, return the opened file async fn open_dir(&self, path: &str, flags: OpenFileFlags) -> Result; /// Create the file by file path and flags, if successful, return the opened file @@ -305,35 +305,4 @@ mod tests { let mut file_stat = FileStat::new_file_filestat("a", "b", 10); file_stat.set_file_id(1, 0); } - - #[test] - fn test_open_file() { - let mut open_file = OpenedFile::new(FileStat::new_file_filestat("a", "b", 10)); - assert_eq!(open_file.file_stat.name, "b"); - assert_eq!(open_file.file_stat.size, 10); - - open_file.set_file_id(1, 2); - - assert_eq!(open_file.file_stat.file_id, 2); - assert_eq!(open_file.file_stat.parent_file_id, 1); - } - - #[test] - fn test_file_entry_manager() { - let mut manager = FileEntryManager::new(); - manager.insert(1, 2, "a/b"); - let file = manager.get_file_entry_by_id(2).unwrap(); - assert_eq!(file.file_id, 2); - assert_eq!(file.parent_file_id, 1); - assert_eq!(file.path, "a/b"); - - let file = manager.get_file_entry_by_path("a/b").unwrap(); - assert_eq!(file.file_id, 2); - assert_eq!(file.parent_file_id, 1); - assert_eq!(file.path, "a/b"); - - manager.remove("a/b"); - assert!(manager.get_file_entry_by_id(2).is_none()); - assert!(manager.get_file_entry_by_path("a/b").is_none()); - } } diff --git a/clients/filesystem-fuse/src/fuse_api_handle.rs b/clients/filesystem-fuse/src/fuse_api_handle.rs index f1398b29438..76af581446b 100644 --- a/clients/filesystem-fuse/src/fuse_api_handle.rs +++ b/clients/filesystem-fuse/src/fuse_api_handle.rs @@ -34,7 +34,7 @@ use std::ffi::{OsStr, OsString}; use std::num::NonZeroU32; use std::time::{Duration, SystemTime}; -pub struct FuseApiHandle { +pub(crate) struct FuseApiHandle { fs: T, default_ttl: Duration, fs_context: FileSystemContext, diff --git a/clients/filesystem-fuse/tests/fuse_test.rs b/clients/filesystem-fuse/tests/fuse_test.rs index d81c667c49c..f83b1ea3c22 100644 --- a/clients/filesystem-fuse/tests/fuse_test.rs +++ b/clients/filesystem-fuse/tests/fuse_test.rs @@ -51,11 +51,9 @@ impl FuseTest { let start_time = Instant::now(); while start_time.elapsed() < timeout { - if let Ok(exists) = fs::exists(&test_file) { - info!("Wait for fuse server ready: {}", exists); - if exists { - return true; - } + if !file_exists(&test_file) { + info!("Wait for fuse server ready",); + return true; } sleep(Duration::from_secs(1)); } @@ -99,7 +97,7 @@ fn test_fuse_filesystem(mount_point: &str) { let test_file = base_path.join("test_create"); let file = File::create(&test_file).expect("Failed to create file"); assert!(file.metadata().is_ok(), "Failed to get file metadata"); - assert!(fs::exists(&test_file).expect("File is not created")); + assert!(file_exists(&test_file)); //test write file fs::write(&test_file, "read test").expect("Failed to write file"); @@ -110,7 +108,7 @@ fn test_fuse_filesystem(mount_point: &str) { //test delete file fs::remove_file(test_file.clone()).expect("Failed to delete file"); - assert!(!fs::exists(test_file).expect("File is not deleted")); + assert!(!file_exists(test_file)); //test create directory let test_dir = base_path.join("test_dir"); @@ -131,11 +129,15 @@ fn test_fuse_filesystem(mount_point: &str) { //test delete file in directory fs::remove_file(&test_file).expect("Failed to delete file"); - assert!(!fs::exists(&test_file).expect("File is not deleted")); + assert!(!file_exists(&test_file)); //test delete directory fs::remove_dir_all(&test_dir).expect("Failed to delete directory"); - assert!(!fs::exists(&test_dir).expect("Directory is not deleted")); + assert!(!file_exists(&test_dir)); info!("Success test") } + +fn file_exists>(path: P) -> bool { + fs::metadata(path).is_ok() +} From 568fefe1b24057d7616fc31ef2fa47e7ff198a22 Mon Sep 17 00:00:00 2001 From: yuhui Date: Fri, 20 Dec 2024 10:26:29 +0800 Subject: [PATCH 24/47] Update --- clients/filesystem-fuse/src/opened_file.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/clients/filesystem-fuse/src/opened_file.rs b/clients/filesystem-fuse/src/opened_file.rs index 61a9af4a3bc..c08f8107d2e 100644 --- a/clients/filesystem-fuse/src/opened_file.rs +++ b/clients/filesystem-fuse/src/opened_file.rs @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + use crate::filesystem::{FileReader, FileStat, FileWriter, Result}; use bytes::Bytes; use fuse3::{Errno, Timestamp}; From 76305a9980c941618cda5e45b83aa46934199650 Mon Sep 17 00:00:00 2001 From: yuhui Date: Fri, 20 Dec 2024 10:44:35 +0800 Subject: [PATCH 25/47] Update --- clients/filesystem-fuse/src/opened_file.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/clients/filesystem-fuse/src/opened_file.rs b/clients/filesystem-fuse/src/opened_file.rs index 61a9af4a3bc..ba3e41595da 100644 --- a/clients/filesystem-fuse/src/opened_file.rs +++ b/clients/filesystem-fuse/src/opened_file.rs @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ use crate::filesystem::{FileReader, FileStat, FileWriter, Result}; use bytes::Bytes; use fuse3::{Errno, Timestamp}; From 545962230905fb913d1b32bdc0e80ab204d3ff4b Mon Sep 17 00:00:00 2001 From: yuhui Date: Fri, 20 Dec 2024 14:58:03 +0800 Subject: [PATCH 26/47] Update path interface --- .../src/default_raw_filesystem.rs | 37 ++++++------ clients/filesystem-fuse/src/filesystem.rs | 50 ++++++++-------- .../filesystem-fuse/src/memory_filesystem.rs | 58 ++++++++++--------- clients/filesystem-fuse/src/mount.rs | 6 +- clients/filesystem-fuse/src/utils.rs | 40 +------------ 5 files changed, 80 insertions(+), 111 deletions(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index 15d7d4b9eea..94c1c45d10d 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -19,11 +19,11 @@ use crate::filesystem::{FileStat, PathFileSystem, RawFileSystem}; use crate::opened_file::{FileHandle, OpenFileFlags}; use crate::opened_file_manager::OpenedFileManager; -use crate::utils::join_file_path; use async_trait::async_trait; use bytes::Bytes; use fuse3::{Errno, FileType}; use std::collections::HashMap; +use std::path::{Path, PathBuf}; use std::sync::atomic::AtomicU64; use tokio::sync::RwLock; @@ -46,7 +46,7 @@ impl DefaultRawFileSystem { const INITIAL_FILE_ID: u64 = 10000; const ROOT_DIR_PARENT_FILE_ID: u64 = 1; const ROOT_DIR_FILE_ID: u64 = 1; - const ROOT_DIR_NAME: &'static str = ""; + const ROOT_DIR_PATH: &'static str = "/"; pub(crate) fn new(fs: T) -> Self { Self { @@ -70,7 +70,7 @@ impl DefaultRawFileSystem { .ok_or(Errno::from(libc::ENOENT)) } - async fn get_file_entry_by_path(&self, path: &str) -> Option { + async fn get_file_entry_by_path(&self, path: &Path) -> Option { self.file_entry_manager .read() .await @@ -131,7 +131,7 @@ impl RawFileSystem for DefaultRawFileSystem { self.file_entry_manager.write().await.insert( Self::ROOT_DIR_PARENT_FILE_ID, Self::ROOT_DIR_FILE_ID, - Self::ROOT_DIR_NAME, + Path::new(Self::ROOT_DIR_PATH), ); self.fs.init().await } @@ -139,7 +139,7 @@ impl RawFileSystem for DefaultRawFileSystem { async fn get_file_path(&self, file_id: u64) -> String { let file_entry = self.get_file_entry(file_id).await; file_entry - .map(|x| x.path) + .map(|x| x.path.to_string_lossy().to_string()) .unwrap_or_else(|_| "".to_string()) } @@ -167,11 +167,14 @@ impl RawFileSystem for DefaultRawFileSystem { async fn lookup(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result { let parent_file_entry = self.get_file_entry(parent_file_id).await?; - let path = join_file_path(&parent_file_entry.path, name); + + // assume the path is a regular file. Some filesystems may need to check whether it is a file or directory by lookup. + let path = parent_file_entry.path.join(name); let mut file_stat = self.fs.lookup(&path).await?; // fill the file id to file stat self.resolve_file_id_to_filestat(&mut file_stat, parent_file_id) .await; + Ok(file_stat) } @@ -201,7 +204,7 @@ impl RawFileSystem for DefaultRawFileSystem { flags: u32, ) -> crate::filesystem::Result { let parent_file_entry = self.get_file_entry(parent_file_id).await?; - let path = join_file_path(&parent_file_entry.path, name); + let path = parent_file_entry.path.join(name); let mut opened_file = self.fs.create_file(&path, OpenFileFlags(flags)).await?; opened_file.set_file_id(parent_file_id, self.next_file_id()); @@ -224,7 +227,7 @@ impl RawFileSystem for DefaultRawFileSystem { async fn create_dir(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result { let parent_file_entry = self.get_file_entry(parent_file_id).await?; - let path = join_file_path(&parent_file_entry.path, name); + let path = parent_file_entry.path.join(name); let mut filestat = self.fs.create_dir(&path).await?; filestat.set_file_id(parent_file_id, self.next_file_id()); @@ -244,7 +247,7 @@ impl RawFileSystem for DefaultRawFileSystem { async fn remove_file(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result<()> { let parent_file_entry = self.get_file_entry(parent_file_id).await?; - let path = join_file_path(&parent_file_entry.path, name); + let path = parent_file_entry.path.join(name); self.fs.remove_file(&path).await?; // remove the file from file entry manager @@ -257,7 +260,7 @@ impl RawFileSystem for DefaultRawFileSystem { async fn remove_dir(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result<()> { let parent_file_entry = self.get_file_entry(parent_file_id).await?; - let path = join_file_path(&parent_file_entry.path, name); + let path = parent_file_entry.path.join(name); self.fs.remove_dir(&path).await?; // remove the dir from file entry manager @@ -330,7 +333,7 @@ impl RawFileSystem for DefaultRawFileSystem { struct FileEntry { file_id: u64, parent_file_id: u64, - path: String, + path: PathBuf, } /// FileEntryManager is manage all the file entries in memory. it is used manger the file relationship and name mapping. @@ -339,7 +342,7 @@ struct FileEntryManager { file_id_map: HashMap, // file_path_map is a map of file path to file entry. - file_path_map: HashMap, + file_path_map: HashMap, } impl FileEntryManager { @@ -354,21 +357,21 @@ impl FileEntryManager { self.file_id_map.get(&file_id).cloned() } - fn get_file_entry_by_path(&self, path: &str) -> Option { + fn get_file_entry_by_path(&self, path: &Path) -> Option { self.file_path_map.get(path).cloned() } - fn insert(&mut self, parent_file_id: u64, file_id: u64, path: &str) { + fn insert(&mut self, parent_file_id: u64, file_id: u64, path: &Path) { let file_entry = FileEntry { file_id, parent_file_id, - path: path.to_string(), + path: path.into(), }; self.file_id_map.insert(file_id, file_entry.clone()); - self.file_path_map.insert(path.to_string(), file_entry); + self.file_path_map.insert(path.into(), file_entry); } - fn remove(&mut self, path: &str) { + fn remove(&mut self, path: &Path) { if let Some(file) = self.file_path_map.remove(path) { self.file_id_map.remove(&file.file_id); } diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 55e6c854b76..6601bed8aa1 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -17,10 +17,11 @@ * under the License. */ use crate::opened_file::{FileHandle, OpenFileFlags, OpenedFile}; -use crate::utils::{join_file_path, split_file_path}; use async_trait::async_trait; use bytes::Bytes; +use fuse3::FileType::{Directory, RegularFile}; use fuse3::{Errno, FileType, Timestamp}; +use std::path::{Path, PathBuf}; use std::time::SystemTime; pub(crate) type Result = std::result::Result; @@ -91,34 +92,34 @@ pub(crate) trait PathFileSystem: Send + Sync { async fn init(&self) -> Result<()>; /// Get the file stat by file path, if the file exists, return the file stat - async fn stat(&self, path: &str) -> Result; + async fn stat(&self, path: &Path) -> Result; /// Get the file stat by parent file path and file name, if the file exists, return the file stat - async fn lookup(&self, path: &str) -> Result; + async fn lookup(&self, path: &Path) -> Result; /// Read the directory by file path, if the directory exists, return the file stat list - async fn read_dir(&self, path: &str) -> Result>; + async fn read_dir(&self, path: &Path) -> Result>; /// Open the file by file path and flags, if the file exists, return the opened file - async fn open_file(&self, path: &str, flags: OpenFileFlags) -> Result; + async fn open_file(&self, path: &Path, flags: OpenFileFlags) -> Result; /// Open the directory by file path and flags, if the file exists, return the opened file - async fn open_dir(&self, path: &str, flags: OpenFileFlags) -> Result; + async fn open_dir(&self, path: &Path, flags: OpenFileFlags) -> Result; /// Create the file by file path and flags, if successful, return the opened file - async fn create_file(&self, path: &str, flags: OpenFileFlags) -> Result; + async fn create_file(&self, path: &Path, flags: OpenFileFlags) -> Result; /// Create the directory by file path , if successful, return the file stat - async fn create_dir(&self, path: &str) -> Result; + async fn create_dir(&self, path: &Path) -> Result; /// Set the file attribute by file path and file stat - async fn set_attr(&self, path: &str, file_stat: &FileStat, flush: bool) -> Result<()>; + async fn set_attr(&self, path: &Path, file_stat: &FileStat, flush: bool) -> Result<()>; /// Remove the file by file path - async fn remove_file(&self, path: &str) -> Result<()>; + async fn remove_file(&self, path: &Path) -> Result<()>; /// Remove the directory by file path - async fn remove_dir(&self, path: &str) -> Result<()>; + async fn remove_dir(&self, path: &Path) -> Result<()>; } // FileSystemContext is the system environment for the fuse file system. @@ -164,7 +165,7 @@ pub struct FileStat { pub(crate) name: String, // file path of the fuse file system root - pub(crate) path: String, + pub(crate) path: PathBuf, // file size pub(crate) size: u64, @@ -186,31 +187,32 @@ pub struct FileStat { } impl FileStat { - pub fn new_file_filestat_with_path(path: &str, size: u64) -> Self { - let (parent, name) = split_file_path(path); - Self::new_file_filestat(parent, name, size) + pub fn new_file_filestat_with_path(path: &Path, size: u64) -> Self { + Self::new_filestat(path, size, RegularFile) } - pub fn new_dir_filestat_with_path(path: &str) -> Self { - let (parent, name) = split_file_path(path); - Self::new_dir_filestat(parent, name) + pub fn new_dir_filestat_with_path(path: &Path) -> Self { + Self::new_filestat(path, 0, Directory) } - pub fn new_file_filestat(parent: &str, name: &str, size: u64) -> Self { - Self::new_filestat(parent, name, size, FileType::RegularFile) + pub fn new_file_filestat(parent: &Path, name: &str, size: u64) -> Self { + let path = parent.join(name); + Self::new_filestat(&path, size, RegularFile) } - pub fn new_dir_filestat(parent: &str, name: &str) -> Self { - Self::new_filestat(parent, name, 0, FileType::Directory) + pub fn new_dir_filestat(parent: &Path, name: &str) -> Self { + let path = parent.join(name); + Self::new_filestat(&path, 0, Directory) } - pub fn new_filestat(parent: &str, name: &str, size: u64, kind: FileType) -> Self { + pub fn new_filestat(path: &Path, size: u64, kind: FileType) -> Self { let atime = Timestamp::from(SystemTime::now()); + let name = path.file_name().unwrap().to_string_lossy(); Self { file_id: 0, parent_file_id: 0, name: name.into(), - path: join_file_path(parent, name), + path: path.into(), size: size, kind: kind, atime: atime, diff --git a/clients/filesystem-fuse/src/memory_filesystem.rs b/clients/filesystem-fuse/src/memory_filesystem.rs index 7371390e9fe..d9fffc92b7a 100644 --- a/clients/filesystem-fuse/src/memory_filesystem.rs +++ b/clients/filesystem-fuse/src/memory_filesystem.rs @@ -24,6 +24,7 @@ use fuse3::FileType::{Directory, RegularFile}; use fuse3::{Errno, FileType}; use regex::Regex; use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, RwLock}; // Simple in-memory file implementation of MemoryFileSystem @@ -36,7 +37,7 @@ struct MemoryFile { // It is used for testing purposes pub struct MemoryFileSystem { // file_map is a map of file name to file size - file_map: RwLock>, + file_map: RwLock>, } impl MemoryFileSystem { @@ -48,7 +49,7 @@ impl MemoryFileSystem { } } - fn create_file_stat(&self, path: &str, file: &MemoryFile) -> FileStat { + fn create_file_stat(&self, path: &Path, file: &MemoryFile) -> FileStat { match file.kind { Directory => FileStat::new_dir_filestat_with_path(path), _ => { @@ -65,45 +66,44 @@ impl PathFileSystem for MemoryFileSystem { kind: Directory, data: Arc::new(Mutex::new(Vec::new())), }; - self.file_map.write().unwrap().insert("/".to_string(), root); + let root_path = PathBuf::from("/"); + self.file_map.write().unwrap().insert(root_path, root); let meta = MemoryFile { kind: RegularFile, data: Arc::new(Mutex::new(Vec::new())), }; - self.file_map - .write() - .unwrap() - .insert(Self::FS_META_FILE_NAME.to_string(), meta); + let meta_file_path = Path::new(Self::FS_META_FILE_NAME).to_path_buf(); + self.file_map.write().unwrap().insert(meta_file_path, meta); Ok(()) } - async fn stat(&self, name: &str) -> Result { + async fn stat(&self, path: &Path) -> Result { self.file_map .read() .unwrap() - .get(name) - .map(|x| self.create_file_stat(name, x)) + .get(path) + .map(|x| self.create_file_stat(path, x)) .ok_or(Errno::from(libc::ENOENT)) } - async fn lookup(&self, path: &str) -> Result { + async fn lookup(&self, path: &Path) -> Result { self.stat(path).await } - async fn read_dir(&self, path: &str) -> Result> { + async fn read_dir(&self, path: &Path) -> Result> { let file_map = self.file_map.read().unwrap(); let results: Vec = file_map .iter() - .filter(|x| dir_child_reg_expr(path).is_match(x.0)) + .filter(|x| path_in_dir(path, x.0)) .map(|(k, v)| self.create_file_stat(k, v)) .collect(); Ok(results) } - async fn open_file(&self, path: &str, _flags: OpenFileFlags) -> Result { + async fn open_file(&self, path: &Path, _flags: OpenFileFlags) -> Result { let file_stat = self.stat(path).await?; let mut file = OpenedFile::new(file_stat.clone()); match file.file_stat.kind { @@ -125,11 +125,11 @@ impl PathFileSystem for MemoryFileSystem { } } - async fn open_dir(&self, name: &str, flags: OpenFileFlags) -> Result { - self.open_file(name, flags).await + async fn open_dir(&self, path: &Path, flags: OpenFileFlags) -> Result { + self.open_file(path, flags).await } - async fn create_file(&self, path: &str, _flags: OpenFileFlags) -> Result { + async fn create_file(&self, path: &Path, _flags: OpenFileFlags) -> Result { { let file_map = self.file_map.read().unwrap(); if file_map.contains_key(path) { @@ -153,7 +153,7 @@ impl PathFileSystem for MemoryFileSystem { Ok(file) } - async fn create_dir(&self, path: &str) -> Result { + async fn create_dir(&self, path: &Path) -> Result { { let file_map = self.file_map.read().unwrap(); if file_map.contains_key(path) { @@ -173,11 +173,11 @@ impl PathFileSystem for MemoryFileSystem { Ok(file) } - async fn set_attr(&self, _name: &str, _file_stat: &FileStat, _flush: bool) -> Result<()> { + async fn set_attr(&self, _name: &Path, _file_stat: &FileStat, _flush: bool) -> Result<()> { Ok(()) } - async fn remove_file(&self, path: &str) -> Result<()> { + async fn remove_file(&self, path: &Path) -> Result<()> { let mut file_map = self.file_map.write().unwrap(); if file_map.remove(path).is_none() { return Err(Errno::from(libc::ENOENT)); @@ -185,12 +185,9 @@ impl PathFileSystem for MemoryFileSystem { Ok(()) } - async fn remove_dir(&self, path: &str) -> Result<()> { + async fn remove_dir(&self, path: &Path) -> Result<()> { let mut file_map = self.file_map.write().unwrap(); - let count = file_map - .iter() - .filter(|x| dir_child_reg_expr(path).is_match(x.0)) - .count(); + let count = file_map.iter().filter(|x| path_in_dir(path, x.0)).count(); if count != 0 { return Err(Errno::from(libc::ENOTEMPTY)); @@ -237,11 +234,16 @@ impl FileWriter for MemoryFileWriter { } } -fn dir_child_reg_expr(name: &str) -> Regex { - let regex_pattern = if name.is_empty() { +fn path_in_dir(dir: &Path, path: &Path) -> bool { + let regex = dir_child_reg_expr(dir.to_str().unwrap()); + regex.is_match(path.to_str().unwrap()) +} + +fn dir_child_reg_expr(path: &str) -> Regex { + let regex_pattern = if path.is_empty() { r"^[^/]+$".to_string() } else { - format!(r"^{}/[^/]+$", name) + format!(r"^{}/[^/]+$", path) }; Regex::new(®ex_pattern).unwrap() } diff --git a/clients/filesystem-fuse/src/mount.rs b/clients/filesystem-fuse/src/mount.rs index f3b451ba41e..aa63d1322e7 100644 --- a/clients/filesystem-fuse/src/mount.rs +++ b/clients/filesystem-fuse/src/mount.rs @@ -47,9 +47,9 @@ pub async fn create_fuse_fs() -> impl Filesystem + Sync + 'static { let fs_context = FileSystemContext { uid: uid, gid: gid, - default_file_perm: 0, - default_dir_perm: 0, - block_size: 0, + default_file_perm: 0o644, + default_dir_perm: 0o755, + block_size: 4 * 1024, }; let gvfs = MemoryFileSystem::new().await; diff --git a/clients/filesystem-fuse/src/utils.rs b/clients/filesystem-fuse/src/utils.rs index 7619775c949..21e52f86af8 100644 --- a/clients/filesystem-fuse/src/utils.rs +++ b/clients/filesystem-fuse/src/utils.rs @@ -17,43 +17,5 @@ * under the License. */ -// join the parent and name to a path -pub fn join_file_path(parent: &str, name: &str) -> String { - //TODO handle corner cases - if parent.is_empty() { - name.to_string() - } else { - format!("{}/{}", parent, name) - } -} - -// split the path to parent and name -pub fn split_file_path(path: &str) -> (&str, &str) { - match path.rfind('/') { - Some(pos) => (&path[..pos], &path[pos + 1..]), - None => ("", path), - } -} - #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_join_file_path() { - assert_eq!(join_file_path("", "a"), "a"); - assert_eq!(join_file_path("", "a.txt"), "a.txt"); - assert_eq!(join_file_path("a", "b"), "a/b"); - assert_eq!(join_file_path("a/b", "c"), "a/b/c"); - assert_eq!(join_file_path("a/b", "c.txt"), "a/b/c.txt"); - } - - #[test] - fn test_split_file_path() { - assert_eq!(split_file_path("a"), ("", "a")); - assert_eq!(split_file_path("a.txt"), ("", "a.txt")); - assert_eq!(split_file_path("a/b"), ("a", "b")); - assert_eq!(split_file_path("a/b/c"), ("a/b", "c")); - assert_eq!(split_file_path("a/b/c.txt"), ("a/b", "c.txt")); - } -} +mod tests {} From dbca55706be83ffad60793937d3a53b13ce662fb Mon Sep 17 00:00:00 2001 From: yuhui Date: Fri, 20 Dec 2024 15:36:22 +0800 Subject: [PATCH 27/47] Fix error --- .../src/default_raw_filesystem.rs | 35 +++++++++----- clients/filesystem-fuse/src/filesystem.rs | 46 +++++++++++-------- .../filesystem-fuse/src/fuse_api_handle.rs | 23 ++++------ .../filesystem-fuse/src/memory_filesystem.rs | 4 +- clients/filesystem-fuse/src/opened_file.rs | 7 ++- .../src/opened_file_manager.rs | 5 +- 6 files changed, 70 insertions(+), 50 deletions(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index 94c1c45d10d..16806385dd1 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -23,6 +23,7 @@ use async_trait::async_trait; use bytes::Bytes; use fuse3::{Errno, FileType}; use std::collections::HashMap; +use std::ffi::OsStr; use std::path::{Path, PathBuf}; use std::sync::atomic::AtomicU64; use tokio::sync::RwLock; @@ -165,7 +166,11 @@ impl RawFileSystem for DefaultRawFileSystem { Ok(file_stat) } - async fn lookup(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result { + async fn lookup( + &self, + parent_file_id: u64, + name: &OsStr, + ) -> crate::filesystem::Result { let parent_file_entry = self.get_file_entry(parent_file_id).await?; // assume the path is a regular file. Some filesystems may need to check whether it is a file or directory by lookup. @@ -200,7 +205,7 @@ impl RawFileSystem for DefaultRawFileSystem { async fn create_file( &self, parent_file_id: u64, - name: &str, + name: &OsStr, flags: u32, ) -> crate::filesystem::Result { let parent_file_entry = self.get_file_entry(parent_file_id).await?; @@ -225,7 +230,11 @@ impl RawFileSystem for DefaultRawFileSystem { Ok(opened_file.file_handle()) } - async fn create_dir(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result { + async fn create_dir( + &self, + parent_file_id: u64, + name: &OsStr, + ) -> crate::filesystem::Result { let parent_file_entry = self.get_file_entry(parent_file_id).await?; let path = parent_file_entry.path.join(name); let mut filestat = self.fs.create_dir(&path).await?; @@ -245,7 +254,11 @@ impl RawFileSystem for DefaultRawFileSystem { self.fs.set_attr(&file_entry.path, file_stat, true).await } - async fn remove_file(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result<()> { + async fn remove_file( + &self, + parent_file_id: u64, + name: &OsStr, + ) -> crate::filesystem::Result<()> { let parent_file_entry = self.get_file_entry(parent_file_id).await?; let path = parent_file_entry.path.join(name); self.fs.remove_file(&path).await?; @@ -258,7 +271,7 @@ impl RawFileSystem for DefaultRawFileSystem { Ok(()) } - async fn remove_dir(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result<()> { + async fn remove_dir(&self, parent_file_id: u64, name: &OsStr) -> crate::filesystem::Result<()> { let parent_file_entry = self.get_file_entry(parent_file_id).await?; let path = parent_file_entry.path.join(name); self.fs.remove_dir(&path).await?; @@ -385,19 +398,19 @@ mod tests { #[test] fn test_file_entry_manager() { let mut manager = FileEntryManager::new(); - manager.insert(1, 2, "a/b"); + manager.insert(1, 2, Path::new("a/b")); let file = manager.get_file_entry_by_id(2).unwrap(); assert_eq!(file.file_id, 2); assert_eq!(file.parent_file_id, 1); - assert_eq!(file.path, "a/b"); + assert_eq!(file.path, Path::new("a/b")); - let file = manager.get_file_entry_by_path("a/b").unwrap(); + let file = manager.get_file_entry_by_path(Path::new("a/b")).unwrap(); assert_eq!(file.file_id, 2); assert_eq!(file.parent_file_id, 1); - assert_eq!(file.path, "a/b"); + assert_eq!(file.path, Path::new("a/b")); - manager.remove("a/b"); + manager.remove(Path::new("a/b")); assert!(manager.get_file_entry_by_id(2).is_none()); - assert!(manager.get_file_entry_by_path("a/b").is_none()); + assert!(manager.get_file_entry_by_path(Path::new("a/b")).is_none()); } } diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 6601bed8aa1..905f31e2274 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -21,6 +21,7 @@ use async_trait::async_trait; use bytes::Bytes; use fuse3::FileType::{Directory, RegularFile}; use fuse3::{Errno, FileType, Timestamp}; +use std::ffi::{OsStr, OsString}; use std::path::{Path, PathBuf}; use std::time::SystemTime; @@ -48,7 +49,7 @@ pub(crate) trait RawFileSystem: Send + Sync { async fn stat(&self, file_id: u64) -> Result; /// Lookup the file by parent file id and file name, if the file exists, return the file stat - async fn lookup(&self, parent_file_id: u64, name: &str) -> Result; + async fn lookup(&self, parent_file_id: u64, name: &OsStr) -> Result; /// Read the directory by file id, if the file id is a valid directory, return the file stat list async fn read_dir(&self, dir_file_id: u64) -> Result>; @@ -60,19 +61,24 @@ pub(crate) trait RawFileSystem: Send + Sync { async fn open_dir(&self, file_id: u64, flags: u32) -> Result; /// Create the file by parent file id and file name and flags, if successful, return the file handle - async fn create_file(&self, parent_file_id: u64, name: &str, flags: u32) -> Result; + async fn create_file( + &self, + parent_file_id: u64, + name: &OsStr, + flags: u32, + ) -> Result; /// Create the directory by parent file id and file name, if successful, return the file id - async fn create_dir(&self, parent_file_id: u64, name: &str) -> Result; + async fn create_dir(&self, parent_file_id: u64, name: &OsStr) -> Result; /// Set the file attribute by file id and file stat async fn set_attr(&self, file_id: u64, file_stat: &FileStat) -> Result<()>; /// Remove the file by parent file id and file name - async fn remove_file(&self, parent_file_id: u64, name: &str) -> Result<()>; + async fn remove_file(&self, parent_file_id: u64, name: &OsStr) -> Result<()>; /// Remove the directory by parent file id and file name - async fn remove_dir(&self, parent_file_id: u64, name: &str) -> Result<()>; + async fn remove_dir(&self, parent_file_id: u64, name: &OsStr) -> Result<()>; /// Close the file by file id and file handle, if successful async fn close_file(&self, file_id: u64, fh: u64) -> Result<()>; @@ -162,7 +168,7 @@ pub struct FileStat { pub(crate) parent_file_id: u64, // file name - pub(crate) name: String, + pub(crate) name: OsString, // file path of the fuse file system root pub(crate) path: PathBuf, @@ -195,23 +201,23 @@ impl FileStat { Self::new_filestat(path, 0, Directory) } - pub fn new_file_filestat(parent: &Path, name: &str, size: u64) -> Self { + pub fn new_file_filestat(parent: &Path, name: &OsStr, size: u64) -> Self { let path = parent.join(name); Self::new_filestat(&path, size, RegularFile) } - pub fn new_dir_filestat(parent: &Path, name: &str) -> Self { + pub fn new_dir_filestat(parent: &Path, name: &OsStr) -> Self { let path = parent.join(name); Self::new_filestat(&path, 0, Directory) } pub fn new_filestat(path: &Path, size: u64, kind: FileType) -> Self { let atime = Timestamp::from(SystemTime::now()); - let name = path.file_name().unwrap().to_string_lossy(); + let name = path.file_name().unwrap(); Self { file_id: 0, parent_file_id: 0, - name: name.into(), + name: name.to_os_string(), path: path.into(), size: size, kind: kind, @@ -265,37 +271,37 @@ mod tests { #[test] fn test_create_file_stat() { //test new file - let file_stat = FileStat::new_file_filestat("a", "b", 10); + let file_stat = FileStat::new_file_filestat(Path::new("a"), "b".as_ref(), 10); assert_eq!(file_stat.name, "b"); - assert_eq!(file_stat.path, "a/b"); + assert_eq!(file_stat.path, Path::new("a/b")); assert_eq!(file_stat.size, 10); assert_eq!(file_stat.kind, FileType::RegularFile); //test new dir - let file_stat = FileStat::new_dir_filestat("a", "b"); + let file_stat = FileStat::new_dir_filestat("a".as_ref(), "b".as_ref()); assert_eq!(file_stat.name, "b"); - assert_eq!(file_stat.path, "a/b"); + assert_eq!(file_stat.path, Path::new("a/b")); assert_eq!(file_stat.size, 0); assert_eq!(file_stat.kind, FileType::Directory); //test new file with path - let file_stat = FileStat::new_file_filestat_with_path("a/b", 10); + let file_stat = FileStat::new_file_filestat_with_path("a/b".as_ref(), 10); assert_eq!(file_stat.name, "b"); - assert_eq!(file_stat.path, "a/b"); + assert_eq!(file_stat.path, Path::new("a/b")); assert_eq!(file_stat.size, 10); assert_eq!(file_stat.kind, FileType::RegularFile); //test new dir with path - let file_stat = FileStat::new_dir_filestat_with_path("a/b"); + let file_stat = FileStat::new_dir_filestat_with_path("a/b".as_ref()); assert_eq!(file_stat.name, "b"); - assert_eq!(file_stat.path, "a/b"); + assert_eq!(file_stat.path, Path::new("a/b")); assert_eq!(file_stat.size, 0); assert_eq!(file_stat.kind, FileType::Directory); } #[test] fn test_file_stat_set_file_id() { - let mut file_stat = FileStat::new_file_filestat("a", "b", 10); + let mut file_stat = FileStat::new_file_filestat("a".as_ref(), "b".as_ref(), 10); file_stat.set_file_id(1, 2); assert_eq!(file_stat.file_id, 2); assert_eq!(file_stat.parent_file_id, 1); @@ -304,7 +310,7 @@ mod tests { #[test] #[should_panic(expected = "assertion failed: file_id != 0 && parent_file_id != 0")] fn test_file_stat_set_file_id_panic() { - let mut file_stat = FileStat::new_file_filestat("a", "b", 10); + let mut file_stat = FileStat::new_file_filestat("a".as_ref(), "b".as_ref(), 10); file_stat.set_file_id(1, 0); } } diff --git a/clients/filesystem-fuse/src/fuse_api_handle.rs b/clients/filesystem-fuse/src/fuse_api_handle.rs index 76af581446b..6d9d850be4d 100644 --- a/clients/filesystem-fuse/src/fuse_api_handle.rs +++ b/clients/filesystem-fuse/src/fuse_api_handle.rs @@ -99,8 +99,7 @@ impl Filesystem for FuseApiHandle { parent: Inode, name: &OsStr, ) -> fuse3::Result { - let name = name.to_string_lossy(); - let file_stat = self.fs.lookup(parent, &name).await?; + let file_stat = self.fs.lookup(parent, name).await?; Ok(ReplyEntry { ttl: self.default_ttl, attr: fstat_to_file_attr(&file_stat, &self.fs_context), @@ -158,8 +157,7 @@ impl Filesystem for FuseApiHandle { _mode: u32, _umask: u32, ) -> fuse3::Result { - let name = name.to_string_lossy(); - let handle_id = self.fs.create_dir(parent, &name).await?; + let handle_id = self.fs.create_dir(parent, name).await?; Ok(ReplyEntry { ttl: self.default_ttl, attr: dummy_file_attr( @@ -173,14 +171,12 @@ impl Filesystem for FuseApiHandle { } async fn unlink(&self, _req: Request, parent: Inode, name: &OsStr) -> fuse3::Result<()> { - let name = name.to_string_lossy(); - self.fs.remove_file(parent, &name).await?; + self.fs.remove_file(parent, name).await?; Ok(()) } async fn rmdir(&self, _req: Request, parent: Inode, name: &OsStr) -> fuse3::Result<()> { - let name = name.to_string_lossy(); - self.fs.remove_dir(parent, &name).await?; + self.fs.remove_dir(parent, name).await?; Ok(()) } @@ -271,7 +267,7 @@ impl Filesystem for FuseApiHandle { stream::iter(files.into_iter().enumerate().map(|(index, file_stat)| { Ok(DirectoryEntry { inode: file_stat.file_id, - name: file_stat.name.clone().into(), + name: file_stat.name.clone(), kind: file_stat.kind, offset: (index + 3) as i64, }) @@ -317,8 +313,7 @@ impl Filesystem for FuseApiHandle { _mode: u32, flags: u32, ) -> fuse3::Result { - let name = name.to_string_lossy(); - let file_handle = self.fs.create_file(parent, &name, flags).await?; + let file_handle = self.fs.create_file(parent, name, flags).await?; Ok(ReplyCreated { ttl: self.default_ttl, attr: dummy_file_attr( @@ -353,7 +348,7 @@ impl Filesystem for FuseApiHandle { stream::iter(files.into_iter().enumerate().map(|(index, file_stat)| { Ok(DirectoryEntryPlus { inode: file_stat.file_id, - name: file_stat.name.clone().into(), + name: file_stat.name.clone(), kind: file_stat.kind, offset: (index + 3) as i64, attr: fstat_to_file_attr(&file_stat, &self.fs_context), @@ -469,8 +464,8 @@ mod test { let file_stat = FileStat { file_id: 1, parent_file_id: 3, - name: "test".to_string(), - path: "".to_string(), + name: "test".into(), + path: "".into(), size: 10032, kind: FileType::RegularFile, atime: Timestamp { sec: 10, nsec: 3 }, diff --git a/clients/filesystem-fuse/src/memory_filesystem.rs b/clients/filesystem-fuse/src/memory_filesystem.rs index d9fffc92b7a..7c8488e37e4 100644 --- a/clients/filesystem-fuse/src/memory_filesystem.rs +++ b/clients/filesystem-fuse/src/memory_filesystem.rs @@ -235,8 +235,8 @@ impl FileWriter for MemoryFileWriter { } fn path_in_dir(dir: &Path, path: &Path) -> bool { - let regex = dir_child_reg_expr(dir.to_str().unwrap()); - regex.is_match(path.to_str().unwrap()) + let regex = dir_child_reg_expr(dir.to_string_lossy().as_ref()); + regex.is_match(path.to_string_lossy().as_ref()) } fn dir_child_reg_expr(path: &str) -> Regex { diff --git a/clients/filesystem-fuse/src/opened_file.rs b/clients/filesystem-fuse/src/opened_file.rs index c08f8107d2e..617645ad914 100644 --- a/clients/filesystem-fuse/src/opened_file.rs +++ b/clients/filesystem-fuse/src/opened_file.rs @@ -127,10 +127,15 @@ pub(crate) struct OpenFileFlags(pub(crate) u32); mod tests { use super::*; use crate::filesystem::FileStat; + use std::path::Path; #[test] fn test_open_file() { - let mut open_file = OpenedFile::new(FileStat::new_file_filestat("a", "b", 10)); + let mut open_file = OpenedFile::new(FileStat::new_file_filestat( + Path::new("a"), + "b".as_ref(), + 10, + )); assert_eq!(open_file.file_stat.name, "b"); assert_eq!(open_file.file_stat.size, 10); diff --git a/clients/filesystem-fuse/src/opened_file_manager.rs b/clients/filesystem-fuse/src/opened_file_manager.rs index 005ec629145..3c1308781b2 100644 --- a/clients/filesystem-fuse/src/opened_file_manager.rs +++ b/clients/filesystem-fuse/src/opened_file_manager.rs @@ -68,13 +68,14 @@ impl OpenedFileManager { mod tests { use super::*; use crate::filesystem::FileStat; + use std::path::Path; #[tokio::test] async fn test_opened_file_manager() { let manager = OpenedFileManager::new(); - let file1_stat = FileStat::new_file_filestat("", "a.txt", 13); - let file2_stat = FileStat::new_file_filestat("", "b.txt", 18); + let file1_stat = FileStat::new_file_filestat(Path::new(""), "a.txt".as_ref(), 13); + let file2_stat = FileStat::new_file_filestat(Path::new(""), "b.txt".as_ref(), 18); let file1 = OpenedFile::new(file1_stat.clone()); let file2 = OpenedFile::new(file2_stat.clone()); From d444ad3c159cd90dabdb704da819049052351564 Mon Sep 17 00:00:00 2001 From: yuhui Date: Fri, 20 Dec 2024 16:49:10 +0800 Subject: [PATCH 28/47] Fix --- clients/filesystem-fuse/src/default_raw_filesystem.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index c559955b8e2..d492143b18f 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -282,15 +282,14 @@ impl RawFileSystem for DefaultRawFileSystem { offset: u64, size: u32, ) -> crate::filesystem::Result { - let file_stat: FileStat; - let data = { + let (data, file_stat) = { let opened_file = self .opened_file_manager .get(fh) .ok_or(Errno::from(libc::EBADF))?; let mut opened_file = opened_file.lock().await; - file_stat = opened_file.file_stat.clone(); - opened_file.read(offset, size).await + let data = opened_file.read(offset, size).await; + (data, opened_file.file_stat.clone()) }; // update the file atime From 2570f54a25282dcae38c25b20d10cf0fd26deba0 Mon Sep 17 00:00:00 2001 From: yuhui Date: Mon, 23 Dec 2024 10:38:29 +0800 Subject: [PATCH 29/47] Update --- .../src/default_raw_filesystem.rs | 16 +-- clients/filesystem-fuse/src/filesystem.rs | 8 +- clients/filesystem-fuse/src/lib.rs | 105 ++++++++++++++++++ clients/filesystem-fuse/src/main.rs | 9 -- .../filesystem-fuse/src/memory_filesystem.rs | 4 - 5 files changed, 116 insertions(+), 26 deletions(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index 16806385dd1..c09a02a17cc 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -use crate::filesystem::{FileStat, PathFileSystem, RawFileSystem}; +use crate::filesystem::{FileStat, PathFileSystem, RawFileSystem, INITIAL_FILE_ID, ROOT_DIR_FILE_ID, ROOT_DIR_PARENT_FILE_ID, ROOT_DIR_PATH}; use crate::opened_file::{FileHandle, OpenFileFlags}; use crate::opened_file_manager::OpenedFileManager; use async_trait::async_trait; @@ -44,16 +44,12 @@ pub struct DefaultRawFileSystem { } impl DefaultRawFileSystem { - const INITIAL_FILE_ID: u64 = 10000; - const ROOT_DIR_PARENT_FILE_ID: u64 = 1; - const ROOT_DIR_FILE_ID: u64 = 1; - const ROOT_DIR_PATH: &'static str = "/"; pub(crate) fn new(fs: T) -> Self { Self { file_entry_manager: RwLock::new(FileEntryManager::new()), opened_file_manager: OpenedFileManager::new(), - file_id_generator: AtomicU64::new(Self::INITIAL_FILE_ID), + file_id_generator: AtomicU64::new(INITIAL_FILE_ID), fs, } } @@ -130,9 +126,9 @@ impl RawFileSystem for DefaultRawFileSystem { async fn init(&self) -> crate::filesystem::Result<()> { // init root directory self.file_entry_manager.write().await.insert( - Self::ROOT_DIR_PARENT_FILE_ID, - Self::ROOT_DIR_FILE_ID, - Path::new(Self::ROOT_DIR_PATH), + ROOT_DIR_PARENT_FILE_ID, + ROOT_DIR_FILE_ID, + Path::new(ROOT_DIR_PATH), ); self.fs.init().await } @@ -175,7 +171,7 @@ impl RawFileSystem for DefaultRawFileSystem { // assume the path is a regular file. Some filesystems may need to check whether it is a file or directory by lookup. let path = parent_file_entry.path.join(name); - let mut file_stat = self.fs.lookup(&path).await?; + let mut file_stat = self.fs.stat(&path).await?; // fill the file id to file stat self.resolve_file_id_to_filestat(&mut file_stat, parent_file_id) .await; diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 905f31e2274..b303a25884b 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -27,6 +27,11 @@ use std::time::SystemTime; pub(crate) type Result = std::result::Result; +pub(crate) const ROOT_DIR_PARENT_FILE_ID: u64 = 1; +pub(crate) const ROOT_DIR_FILE_ID: u64 = 1; +pub(crate) const ROOT_DIR_PATH: &'static str = "/"; +pub(crate) const INITIAL_FILE_ID: u64 = 10000; + /// RawFileSystem interface for the file system implementation. it use by FuseApiHandle, /// it ues the file id to operate the file system apis /// the `file_id` and `parent_file_id` it is the unique identifier for the file system, @@ -100,9 +105,6 @@ pub(crate) trait PathFileSystem: Send + Sync { /// Get the file stat by file path, if the file exists, return the file stat async fn stat(&self, path: &Path) -> Result; - /// Get the file stat by parent file path and file name, if the file exists, return the file stat - async fn lookup(&self, path: &Path) -> Result; - /// Read the directory by file path, if the directory exists, return the file stat list async fn read_dir(&self, path: &Path) -> Result>; diff --git a/clients/filesystem-fuse/src/lib.rs b/clients/filesystem-fuse/src/lib.rs index 6cd8552bbe2..75eaca5e769 100644 --- a/clients/filesystem-fuse/src/lib.rs +++ b/clients/filesystem-fuse/src/lib.rs @@ -31,3 +31,108 @@ pub async fn gvfs_mount() -> fuse3::Result<()> { } pub fn gvfs_unmount() {} + + +#[cfg(test)] +mod test { + use std::collections::HashSet; + use std::path::Path; + use fuse3::FileType; + use fuse3::FileType::Directory; + use libc::thread_identifier_info; + use crate::filesystem::{FileStat, PathFileSystem, INITIAL_FILE_ID, ROOT_DIR_FILE_ID, ROOT_DIR_PARENT_FILE_ID, ROOT_DIR_PATH}; + use crate::memory_filesystem::MemoryFileSystem; + use crate::opened_file::OpenFileFlags; + + #[tokio::test] + async fn test_memory_file_system() { + let fs = MemoryFileSystem::new().await; + test_file_system(&fs).await; + } + + async fn test_file_system(fs : &impl PathFileSystem) { + let mut test_file_stats = HashSet::new(); + // test root file + let root_dir_path = Path::new("/"); + let root_file_stat = fs.stat(root_dir_path).await; + assert_eq!(root_file_stat.is_ok(), true); + let root_file_stat = root_file_stat.unwrap(); + assert_file_stat(&root_file_stat,root_dir_path, Directory, 0); + test_file_stats.push(root_file_stat); + + // test meta file + let meta_file_path = Path::new("/.gvfs_meta"); + let meta_file_stat = fs.stat(meta_file_path).await; + assert_eq!(meta_file_stat.is_ok(), true); + let meta_file_stat = meta_file_stat.unwrap(); + assert_file_stat(&meta_file_stat, meta_file_path, FileType::RegularFile, 0); + test_file_stats.push(meta_file_stat); + + // test create file + let file_path = Path::new("/file1.txt"); + let opened_file = fs.create_file(file_path, OpenFileFlags(0)).await; + assert_eq!(opened_file.is_ok(), true); + let file = opened_file.unwrap(); + assert!(file.handle_id > 0); + assert_file_stat(&file.file_stat, file_path, FileType::RegularFile, 0); + test_file_stats.push(file.file_stat.clone()); + + // test create dir + let dir_path = Path::new("/dir1"); + let dir_stat = fs.create_dir(dir_path).await; + assert_eq!(dir_stat.is_ok(), true); + let dir_stat = dir_stat.unwrap(); + assert_file_stat(dir_stat, dir_path, FileType::Directory, 0); + test_file_stats.push(dir_path); + + // test list dir + let list_dir = fs.read_dir(Path::new("/")).await; + assert_eq!(list_dir.is_ok(), true); + let list_dir = list_dir.unwrap(); + assert_eq!(list_dir.len(), test_file_stats.len()); + for file_stat in list_dir { + assert!(test_file_stats.contains(file_stat.path)); + let actual_file_stat = test_file_stats.get(&file_stat.path).unwrap(); + assert_file_stat(&file_stat, actual_file_stat.path, actual_file_stat.kind, actual_file_stat.size); + } + + // test remove file + let remove_file = fs.remove_file(file_path).await; + assert_eq!(remove_file.is_ok(), true); + + // test remove dir + let remove_dir = fs.remove_dir(dir_path).await; + assert_eq!(remove_dir.is_ok(), true); + + // test list dir + let list_dir = fs.read_dir(Path::new("/")).await; + assert_eq!(list_dir.is_ok(), true); + + let list_dir = list_dir.unwrap(); + assert_eq!(list_dir.len(), test_file_stats.len() - 2); + for file_stat in list_dir { + assert!(test_file_stats.contains(file_stat.path)); + let actual_file_stat = test_file_stats.get(&file_stat.path).unwrap(); + assert_file_stat(&file_stat, actual_file_stat.path, actual_file_stat.kind, actual_file_stat.size); + } + + // test file not found + let not_found_file = fs.stat(Path::new("/not_found.txt")).await; + assert_eq!(not_found_file.is_err(), true); + + } + + fn assert_file_stat(file_stat: &FileStat, path: &Path, kind: FileType, size: u64) { + assert_eq!(file_stat.path, path); + assert_eq!(file_stat.kind, kind); + assert_eq!(file_stat.size, size); + if file_stat.file_id == 1 { + // root dir + assert_eq!(file_stat.parent_file_id, ROOT_DIR_PARENT_FILE_ID); + } else { + assert!(file_stat.file_id >= INITIAL_FILE_ID); + assert!(file_stat.parent_file_id >= INITIAL_FILE_ID); + } + } + +} \ No newline at end of file diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index 6212978fe95..dbed7149c31 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -16,15 +16,6 @@ * specific language governing permissions and limitations * under the License. */ -mod default_raw_filesystem; -mod filesystem; -mod fuse_api_handle; -mod fuse_server; -mod memory_filesystem; -mod mount; -mod opened_file; -mod opened_file_manager; -mod utils; use gvfs_fuse::gvfs_mount; diff --git a/clients/filesystem-fuse/src/memory_filesystem.rs b/clients/filesystem-fuse/src/memory_filesystem.rs index 7c8488e37e4..50e4a7c5282 100644 --- a/clients/filesystem-fuse/src/memory_filesystem.rs +++ b/clients/filesystem-fuse/src/memory_filesystem.rs @@ -87,10 +87,6 @@ impl PathFileSystem for MemoryFileSystem { .ok_or(Errno::from(libc::ENOENT)) } - async fn lookup(&self, path: &Path) -> Result { - self.stat(path).await - } - async fn read_dir(&self, path: &Path) -> Result> { let file_map = self.file_map.read().unwrap(); From f6ce0115b30605f1c489adfb3720068d5f0aad3f Mon Sep 17 00:00:00 2001 From: yuhui Date: Mon, 23 Dec 2024 16:49:02 +0800 Subject: [PATCH 30/47] Update --- .../src/default_raw_filesystem.rs | 8 +- clients/filesystem-fuse/src/filesystem.rs | 115 +++++++++++++++++- clients/filesystem-fuse/src/fuse_server.rs | 10 -- clients/filesystem-fuse/src/lib.rs | 110 +---------------- clients/filesystem-fuse/src/main.rs | 24 +++- .../filesystem-fuse/src/memory_filesystem.rs | 15 +-- clients/filesystem-fuse/src/mount.rs | 34 ++++-- clients/filesystem-fuse/tests/fuse_test.rs | 19 +-- 8 files changed, 180 insertions(+), 155 deletions(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index c09a02a17cc..fc537fc4b98 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -16,7 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -use crate::filesystem::{FileStat, PathFileSystem, RawFileSystem, INITIAL_FILE_ID, ROOT_DIR_FILE_ID, ROOT_DIR_PARENT_FILE_ID, ROOT_DIR_PATH}; +use crate::filesystem::{ + FileStat, PathFileSystem, RawFileSystem, INITIAL_FILE_ID, ROOT_DIR_FILE_ID, + ROOT_DIR_PARENT_FILE_ID, ROOT_DIR_PATH, +}; use crate::opened_file::{FileHandle, OpenFileFlags}; use crate::opened_file_manager::OpenedFileManager; use async_trait::async_trait; @@ -44,7 +47,6 @@ pub struct DefaultRawFileSystem { } impl DefaultRawFileSystem { - pub(crate) fn new(fs: T) -> Self { Self { file_entry_manager: RwLock::new(FileEntryManager::new()), @@ -183,7 +185,7 @@ impl RawFileSystem for DefaultRawFileSystem { let file_entry = self.get_file_entry(file_id).await?; let mut child_filestats = self.fs.read_dir(&file_entry.path).await?; for file in child_filestats.iter_mut() { - self.resolve_file_id_to_filestat(file, file.file_id).await; + self.resolve_file_id_to_filestat(file, file_id).await; } Ok(child_filestats) } diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index b303a25884b..fc138ca10b5 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -215,7 +215,7 @@ impl FileStat { pub fn new_filestat(path: &Path, size: u64, kind: FileType) -> Self { let atime = Timestamp::from(SystemTime::now()); - let name = path.file_name().unwrap(); + let name = path.file_name().unwrap_or(OsStr::new("")); Self { file_id: 0, parent_file_id: 0, @@ -268,6 +268,9 @@ pub trait FileWriter: Sync + Send { #[cfg(test)] mod tests { + use std::collections::HashMap; + use crate::default_raw_filesystem::DefaultRawFileSystem; + use crate::memory_filesystem::MemoryFileSystem; use super::*; #[test] @@ -315,4 +318,114 @@ mod tests { let mut file_stat = FileStat::new_file_filestat("a".as_ref(), "b".as_ref(), 10); file_stat.set_file_id(1, 0); } + + + #[tokio::test] + async fn test_memory_file_system() { + let fs = MemoryFileSystem::new().await; + let _ = fs.init().await; + test_path_file_system(&fs).await; + } + + async fn test_path_file_system(fs: &impl PathFileSystem) { + let mut root_dir_child_file_stats = HashMap::new(); + + // test root file + let root_dir_path = Path::new("/"); + let root_file_stat = fs.stat(root_dir_path).await; + assert_eq!(root_file_stat.is_ok(), true); + let root_file_stat = root_file_stat.unwrap(); + assert_file_stat(&root_file_stat, root_dir_path, Directory, 0); + + // test meta file + let meta_file_path = Path::new("/.gvfs_meta"); + let meta_file_stat = fs.stat(meta_file_path).await; + assert_eq!(meta_file_stat.is_ok(), true); + let meta_file_stat = meta_file_stat.unwrap(); + assert_file_stat(&meta_file_stat, meta_file_path, FileType::RegularFile, 0); + root_dir_child_file_stats.insert(meta_file_stat.path.clone(), meta_file_stat); + + // test create file + let file_path = Path::new("/file1.txt"); + let opened_file = fs.create_file(file_path, OpenFileFlags(0)).await; + assert_eq!(opened_file.is_ok(), true); + let file = opened_file.unwrap(); + assert_file_stat(&file.file_stat, file_path, FileType::RegularFile, 0); + root_dir_child_file_stats.insert(file.file_stat.path.clone(), file.file_stat.clone()); + + // test create dir + let dir_path = Path::new("/dir1"); + let dir_stat = fs.create_dir(dir_path).await; + assert_eq!(dir_stat.is_ok(), true); + let dir_stat = dir_stat.unwrap(); + assert_file_stat(&dir_stat, dir_path, Directory, 0); + root_dir_child_file_stats.insert(dir_stat.path.clone(), dir_stat); + + // test list dir + let list_dir = fs.read_dir(Path::new("/")).await; + assert_eq!(list_dir.is_ok(), true); + let list_dir = list_dir.unwrap(); + assert_eq!(list_dir.len(), root_dir_child_file_stats.len()); + for file_stat in list_dir { + assert!(root_dir_child_file_stats.contains_key(&file_stat.path)); + let actual_file_stat = root_dir_child_file_stats.get(&file_stat.path).unwrap(); + assert_file_stat( + &file_stat, + &actual_file_stat.path, + actual_file_stat.kind, + actual_file_stat.size, + ); + } + + // test remove file + let remove_file = fs.remove_file(file_path).await; + assert_eq!(remove_file.is_ok(), true); + root_dir_child_file_stats.remove(file_path); + + // test remove dir + let remove_dir = fs.remove_dir(dir_path).await; + assert_eq!(remove_dir.is_ok(), true); + root_dir_child_file_stats.remove(dir_path); + + // test list dir + let list_dir = fs.read_dir(Path::new("/")).await; + assert_eq!(list_dir.is_ok(), true); + + let list_dir = list_dir.unwrap(); + assert_eq!(list_dir.len(), root_dir_child_file_stats.len()); + for file_stat in list_dir { + assert!(root_dir_child_file_stats.contains_key(&file_stat.path)); + let actual_file_stat = root_dir_child_file_stats.get(&file_stat.path).unwrap(); + assert_file_stat( + &file_stat, + &actual_file_stat.path, + actual_file_stat.kind, + actual_file_stat.size, + ); + } + + // test file not found + let not_found_file = fs.stat(Path::new("/not_found.txt")).await; + assert_eq!(not_found_file.is_err(), true); + } + + fn assert_file_stat(file_stat: &FileStat, path: &Path, kind: FileType, size: u64) { + assert_eq!(file_stat.path, path); + assert_eq!(file_stat.kind, kind); + assert_eq!(file_stat.size, size); + } + + + #[tokio::test] + async fn test_default_raw_file_system() { + let memory_fs = MemoryFileSystem::new().await; + let raw_fs = DefaultRawFileSystem::new(memory_fs).await; + let _ = raw_fs.init().await; + test_raw_file_system(&raw_fs).await; + } + + async fn test_raw_file_system(fs: &impl PathFileSystem) { + let mut root_dir_child_file_stats = HashMap::new(); + + } } diff --git a/clients/filesystem-fuse/src/fuse_server.rs b/clients/filesystem-fuse/src/fuse_server.rs index 2aca413d586..9f5c6efdc03 100644 --- a/clients/filesystem-fuse/src/fuse_server.rs +++ b/clients/filesystem-fuse/src/fuse_server.rs @@ -27,9 +27,6 @@ use tokio::time::timeout; /// Represents a FUSE server capable of starting and stopping the FUSE filesystem. pub struct FuseServer { - // Notification for stop - close_notify: Arc, - // Shared handle to manage FUSE unmounting mount_handle: Arc>>, // Shared handle to manage FUSE unmounting @@ -41,7 +38,6 @@ impl FuseServer { /// Creates a new instance of `FuseServer`. pub fn new(mount_point: &str) -> Self { Self { - close_notify: Arc::new(Notify::new()), mount_handle: Arc::new(Mutex::new(None)), mount_point: mount_point.to_string(), } @@ -70,18 +66,12 @@ impl FuseServer { *handle_guard = Some(mount_handle); } - // Wait for stop notification - self.close_notify.notified().await; - info!("Received stop notification, FUSE filesystem will be unmounted."); Ok(()) } /// Stops the FUSE filesystem and waits for unmounting to complete. pub async fn stop(&self) -> Result<()> { - // Notify stop - self.close_notify.notify_one(); - info!("Stopping FUSE filesystem..."); let timeout_duration = Duration::from_secs(5); diff --git a/clients/filesystem-fuse/src/lib.rs b/clients/filesystem-fuse/src/lib.rs index 75eaca5e769..c7a9c17b126 100644 --- a/clients/filesystem-fuse/src/lib.rs +++ b/clients/filesystem-fuse/src/lib.rs @@ -26,113 +26,11 @@ mod opened_file; mod opened_file_manager; mod utils; -pub async fn gvfs_mount() -> fuse3::Result<()> { - mount::mount().await +pub async fn gvfs_mount(mount_point: &str) -> fuse3::Result<()> { + mount::mount(mount_point).await } -pub fn gvfs_unmount() {} - - -#[cfg(test)] -mod test { - use std::collections::HashSet; - use std::path::Path; - use fuse3::FileType; - use fuse3::FileType::Directory; - use libc::thread_identifier_info; - use crate::filesystem::{FileStat, PathFileSystem, INITIAL_FILE_ID, ROOT_DIR_FILE_ID, ROOT_DIR_PARENT_FILE_ID, ROOT_DIR_PATH}; - use crate::memory_filesystem::MemoryFileSystem; - use crate::opened_file::OpenFileFlags; - - #[tokio::test] - async fn test_memory_file_system() { - let fs = MemoryFileSystem::new().await; - test_file_system(&fs).await; - } - - async fn test_file_system(fs : &impl PathFileSystem) { - let mut test_file_stats = HashSet::new(); - // test root file - let root_dir_path = Path::new("/"); - let root_file_stat = fs.stat(root_dir_path).await; - assert_eq!(root_file_stat.is_ok(), true); - let root_file_stat = root_file_stat.unwrap(); - assert_file_stat(&root_file_stat,root_dir_path, Directory, 0); - test_file_stats.push(root_file_stat); - - // test meta file - let meta_file_path = Path::new("/.gvfs_meta"); - let meta_file_stat = fs.stat(meta_file_path).await; - assert_eq!(meta_file_stat.is_ok(), true); - let meta_file_stat = meta_file_stat.unwrap(); - assert_file_stat(&meta_file_stat, meta_file_path, FileType::RegularFile, 0); - test_file_stats.push(meta_file_stat); - - // test create file - let file_path = Path::new("/file1.txt"); - let opened_file = fs.create_file(file_path, OpenFileFlags(0)).await; - assert_eq!(opened_file.is_ok(), true); - let file = opened_file.unwrap(); - assert!(file.handle_id > 0); - assert_file_stat(&file.file_stat, file_path, FileType::RegularFile, 0); - test_file_stats.push(file.file_stat.clone()); - - // test create dir - let dir_path = Path::new("/dir1"); - let dir_stat = fs.create_dir(dir_path).await; - assert_eq!(dir_stat.is_ok(), true); - let dir_stat = dir_stat.unwrap(); - assert_file_stat(dir_stat, dir_path, FileType::Directory, 0); - test_file_stats.push(dir_path); - - // test list dir - let list_dir = fs.read_dir(Path::new("/")).await; - assert_eq!(list_dir.is_ok(), true); - let list_dir = list_dir.unwrap(); - assert_eq!(list_dir.len(), test_file_stats.len()); - for file_stat in list_dir { - assert!(test_file_stats.contains(file_stat.path)); - let actual_file_stat = test_file_stats.get(&file_stat.path).unwrap(); - assert_file_stat(&file_stat, actual_file_stat.path, actual_file_stat.kind, actual_file_stat.size); - } - - // test remove file - let remove_file = fs.remove_file(file_path).await; - assert_eq!(remove_file.is_ok(), true); - - // test remove dir - let remove_dir = fs.remove_dir(dir_path).await; - assert_eq!(remove_dir.is_ok(), true); - - // test list dir - let list_dir = fs.read_dir(Path::new("/")).await; - assert_eq!(list_dir.is_ok(), true); - - let list_dir = list_dir.unwrap(); - assert_eq!(list_dir.len(), test_file_stats.len() - 2); - for file_stat in list_dir { - assert!(test_file_stats.contains(file_stat.path)); - let actual_file_stat = test_file_stats.get(&file_stat.path).unwrap(); - assert_file_stat(&file_stat, actual_file_stat.path, actual_file_stat.kind, actual_file_stat.size); - } - - // test file not found - let not_found_file = fs.stat(Path::new("/not_found.txt")).await; - assert_eq!(not_found_file.is_err(), true); - - } - - fn assert_file_stat(file_stat: &FileStat, path: &Path, kind: FileType, size: u64) { - assert_eq!(file_stat.path, path); - assert_eq!(file_stat.kind, kind); - assert_eq!(file_stat.size, size); - if file_stat.file_id == 1 { - // root dir - assert_eq!(file_stat.parent_file_id, ROOT_DIR_PARENT_FILE_ID); - } else { - assert!(file_stat.file_id >= INITIAL_FILE_ID); - assert!(file_stat.parent_file_id >= INITIAL_FILE_ID); - } - } +pub async fn gvfs_unmount() { + mount::unmount().await; } \ No newline at end of file diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index dbed7149c31..6e4a458b398 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -16,12 +16,30 @@ * specific language governing permissions and limitations * under the License. */ - -use gvfs_fuse::gvfs_mount; +use log::{error, info}; +use tokio::signal; +use gvfs_fuse::{gvfs_mount, gvfs_unmount}; #[tokio::main] async fn main() -> fuse3::Result<()> { tracing_subscriber::fmt().init(); - gvfs_mount().await?; + let handle = tokio::spawn(async { + gvfs_mount("gvfs").await + }); + + tokio::select! { + hd = handle => { + match hd { + Ok(Ok(_)) => info!("Mount succeeded."), + Ok(Err(e)) => error!("Mount failed: {:?}", e), + Err(e) => error!("Mount failed: {:?}", e), + } + } + _ = signal::ctrl_c() => { + info!("Received Ctrl+C, stopping server..."); + info!("Unmounting gvfs..."); + gvfs_unmount().await; + } + } Ok(()) } diff --git a/clients/filesystem-fuse/src/memory_filesystem.rs b/clients/filesystem-fuse/src/memory_filesystem.rs index 50e4a7c5282..bba19604e33 100644 --- a/clients/filesystem-fuse/src/memory_filesystem.rs +++ b/clients/filesystem-fuse/src/memory_filesystem.rs @@ -22,7 +22,6 @@ use async_trait::async_trait; use bytes::Bytes; use fuse3::FileType::{Directory, RegularFile}; use fuse3::{Errno, FileType}; -use regex::Regex; use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, RwLock}; @@ -41,7 +40,7 @@ pub struct MemoryFileSystem { } impl MemoryFileSystem { - const FS_META_FILE_NAME: &'static str = ".gvfs_meta"; + const FS_META_FILE_NAME: &'static str = "/.gvfs_meta"; pub(crate) async fn new() -> Self { Self { @@ -231,15 +230,5 @@ impl FileWriter for MemoryFileWriter { } fn path_in_dir(dir: &Path, path: &Path) -> bool { - let regex = dir_child_reg_expr(dir.to_string_lossy().as_ref()); - regex.is_match(path.to_string_lossy().as_ref()) -} - -fn dir_child_reg_expr(path: &str) -> Regex { - let regex_pattern = if path.is_empty() { - r"^[^/]+$".to_string() - } else { - format!(r"^{}/[^/]+$", path) - }; - Regex::new(®ex_pattern).unwrap() + path.starts_with(dir) && path != dir } diff --git a/clients/filesystem-fuse/src/mount.rs b/clients/filesystem-fuse/src/mount.rs index aa63d1322e7..5d32d766878 100644 --- a/clients/filesystem-fuse/src/mount.rs +++ b/clients/filesystem-fuse/src/mount.rs @@ -22,23 +22,37 @@ use crate::fuse_api_handle::FuseApiHandle; use crate::fuse_server::FuseServer; use crate::memory_filesystem::MemoryFileSystem; use fuse3::raw::Filesystem; -use log::{debug, info}; -use std::sync::Arc; +use log::info; +use std::sync::{Arc, LazyLock}; +use tokio::sync::Mutex; -pub async fn mount() -> fuse3::Result<()> { - debug!("Starting gvfs-fuse server..."); - let server = Arc::new(FuseServer::new("gvfs")); +static SERVER: LazyLock>>> = LazyLock::new(|| Mutex::new(None)); +pub async fn mount(mount_point: &str) -> fuse3::Result<()> { + info!("Starting gvfs-fuse server..."); + let mut server = SERVER.lock().await; + + let svr = Arc::new(FuseServer::new(mount_point)); let fs = create_fuse_fs().await; - server.start(fs).await?; + let result = svr.start(fs).await; + + *server = Some(svr); + result - tokio::signal::ctrl_c().await?; - info!("Received Ctrl+C, stopping server..."); - server.stop().await } pub async fn unmount() { - todo!("Implement the unmount function"); + info!("Stop gvfs-fuse server..."); + let mut server = SERVER.lock().await; + info!("Stop gvfs-fuse server...0"); + if server.is_none() { + info!("Stop gvfs-fuse server...1"); + return; + } + info!("Stop gvfs-fuse server...2"); + let svr = server.take().unwrap(); + info!("Stop gvfs-fuse server...3"); + let _ = svr.stop().await; } pub async fn create_fuse_fs() -> impl Filesystem + Sync + 'static { diff --git a/clients/filesystem-fuse/tests/fuse_test.rs b/clients/filesystem-fuse/tests/fuse_test.rs index f83b1ea3c22..47dc6c1036e 100644 --- a/clients/filesystem-fuse/tests/fuse_test.rs +++ b/clients/filesystem-fuse/tests/fuse_test.rs @@ -35,7 +35,8 @@ struct FuseTest { impl FuseTest { pub fn setup(&self) { info!("Start gvfs fuse server"); - self.runtime.spawn(async move { gvfs_mount().await }); + let mount_point = self.mount_point.clone(); + self.runtime.spawn(async move { gvfs_mount(&mount_point).await }); let success = Self::wait_for_fuse_server_ready(&self.mount_point, Duration::from_secs(15)); assert!(success, "Fuse server cannot start up at 15 seconds"); } @@ -51,10 +52,10 @@ impl FuseTest { let start_time = Instant::now(); while start_time.elapsed() < timeout { - if !file_exists(&test_file) { - info!("Wait for fuse server ready",); + if file_exists(&test_file) { return true; } + info!("Wait for fuse server ready",); sleep(Duration::from_secs(1)); } false @@ -72,16 +73,15 @@ impl Drop for FuseTest { fn test_fuse_system_with_auto() { tracing_subscriber::fmt().init(); + let mount_point = "build/gvfs"; + let _ = fs::create_dir_all(mount_point); + let test = FuseTest { runtime: Arc::new(Runtime::new().unwrap()), - mount_point: "build/gvfs".to_string(), + mount_point: mount_point.to_string(), }; test.setup(); - - let mount_point = "build/gvfs"; - let _ = fs::create_dir_all(mount_point); - test_fuse_filesystem(mount_point); } @@ -135,7 +135,8 @@ fn test_fuse_filesystem(mount_point: &str) { fs::remove_dir_all(&test_dir).expect("Failed to delete directory"); assert!(!file_exists(&test_dir)); - info!("Success test") + info!("Success test"); + sleep(Duration::from_secs(10)); } fn file_exists>(path: P) -> bool { From 8fb13bb2116c637bf017191d0451bd81aef4b5be Mon Sep 17 00:00:00 2001 From: yuhui Date: Mon, 23 Dec 2024 17:02:33 +0800 Subject: [PATCH 31/47] Update --- .../src/default_raw_filesystem.rs | 34 +++++++++---------- clients/filesystem-fuse/src/filesystem.rs | 2 +- .../filesystem-fuse/src/fuse_api_handle.rs | 4 --- clients/filesystem-fuse/src/utils.rs | 8 +++++ 4 files changed, 25 insertions(+), 23 deletions(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index d492143b18f..d08ffc77b5b 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -use crate::filesystem::{FileStat, PathFileSystem, RawFileSystem}; +use crate::filesystem::{FileStat, PathFileSystem, RawFileSystem, Result}; use crate::opened_file::{FileHandle, OpenFileFlags}; use crate::opened_file_manager::OpenedFileManager; use crate::utils::join_file_path; @@ -62,7 +62,7 @@ impl DefaultRawFileSystem { .fetch_add(1, std::sync::atomic::Ordering::SeqCst) } - async fn get_file_entry(&self, file_id: u64) -> crate::filesystem::Result { + async fn get_file_entry(&self, file_id: u64) -> Result { self.file_entry_manager .read() .await @@ -126,7 +126,7 @@ impl DefaultRawFileSystem { #[async_trait] impl RawFileSystem for DefaultRawFileSystem { - async fn init(&self) -> crate::filesystem::Result<()> { + async fn init(&self) -> Result<()> { // init root directory self.file_entry_manager.write().await.insert( Self::ROOT_DIR_PARENT_FILE_ID, @@ -136,14 +136,12 @@ impl RawFileSystem for DefaultRawFileSystem { self.fs.init().await } - async fn get_file_path(&self, file_id: u64) -> String { + async fn get_file_path(&self, file_id: u64) -> Result { let file_entry = self.get_file_entry(file_id).await; - file_entry - .map(|x| x.path) - .unwrap_or_else(|_| "".to_string()) + Ok(file_entry?.path) } - async fn valid_file_handle_id(&self, file_id: u64, fh: u64) -> crate::filesystem::Result<()> { + async fn valid_file_handle_id(&self, file_id: u64, fh: u64) -> Result<()> { let fh_file_id = self .opened_file_manager .get(fh) @@ -158,14 +156,14 @@ impl RawFileSystem for DefaultRawFileSystem { .ok_or(Errno::from(libc::EBADF)) } - async fn stat(&self, file_id: u64) -> crate::filesystem::Result { + async fn stat(&self, file_id: u64) -> Result { let file_entry = self.get_file_entry(file_id).await?; let mut file_stat = self.fs.stat(&file_entry.path).await?; file_stat.set_file_id(file_entry.parent_file_id, file_entry.file_id); Ok(file_stat) } - async fn lookup(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result { + async fn lookup(&self, parent_file_id: u64, name: &str) -> Result { let parent_file_entry = self.get_file_entry(parent_file_id).await?; let mut file_stat = self.fs.lookup(&parent_file_entry.path, name).await?; // fill the file id to file stat @@ -174,7 +172,7 @@ impl RawFileSystem for DefaultRawFileSystem { Ok(file_stat) } - async fn read_dir(&self, file_id: u64) -> crate::filesystem::Result> { + async fn read_dir(&self, file_id: u64) -> Result> { let file_entry = self.get_file_entry(file_id).await?; let mut child_filestats = self.fs.read_dir(&file_entry.path).await?; for file in child_filestats.iter_mut() { @@ -183,12 +181,12 @@ impl RawFileSystem for DefaultRawFileSystem { Ok(child_filestats) } - async fn open_file(&self, file_id: u64, flags: u32) -> crate::filesystem::Result { + async fn open_file(&self, file_id: u64, flags: u32) -> Result { self.open_file_internal(file_id, flags, FileType::RegularFile) .await } - async fn open_dir(&self, file_id: u64, flags: u32) -> crate::filesystem::Result { + async fn open_dir(&self, file_id: u64, flags: u32) -> Result { self.open_file_internal(file_id, flags, FileType::Directory) .await } @@ -223,7 +221,7 @@ impl RawFileSystem for DefaultRawFileSystem { Ok(opened_file.file_handle()) } - async fn create_dir(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result { + async fn create_dir(&self, parent_file_id: u64, name: &str) -> Result { let parent_file_entry = self.get_file_entry(parent_file_id).await?; let mut filestat = self.fs.create_dir(&parent_file_entry.path, name).await?; @@ -237,12 +235,12 @@ impl RawFileSystem for DefaultRawFileSystem { Ok(filestat.file_id) } - async fn set_attr(&self, file_id: u64, file_stat: &FileStat) -> crate::filesystem::Result<()> { + async fn set_attr(&self, file_id: u64, file_stat: &FileStat) -> Result<()> { let file_entry = self.get_file_entry(file_id).await?; self.fs.set_attr(&file_entry.path, file_stat, true).await } - async fn remove_file(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result<()> { + async fn remove_file(&self, parent_file_id: u64, name: &str) -> Result<()> { let parent_file_entry = self.get_file_entry(parent_file_id).await?; self.fs.remove_file(&parent_file_entry.path, name).await?; @@ -254,7 +252,7 @@ impl RawFileSystem for DefaultRawFileSystem { Ok(()) } - async fn remove_dir(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result<()> { + async fn remove_dir(&self, parent_file_id: u64, name: &str) -> Result<()> { let parent_file_entry = self.get_file_entry(parent_file_id).await?; self.fs.remove_dir(&parent_file_entry.path, name).await?; @@ -266,7 +264,7 @@ impl RawFileSystem for DefaultRawFileSystem { Ok(()) } - async fn close_file(&self, _file_id: u64, fh: u64) -> crate::filesystem::Result<()> { + async fn close_file(&self, _file_id: u64, fh: u64) -> Result<()> { let opened_file = self .opened_file_manager .remove(fh) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 7a74180159b..b0d32ded233 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -38,7 +38,7 @@ pub(crate) trait RawFileSystem: Send + Sync { async fn init(&self) -> Result<()>; /// Get the file path by file id, if the file id is valid, return the file path - async fn get_file_path(&self, file_id: u64) -> String; + async fn get_file_path(&self, file_id: u64) -> Result; /// Validate the file id and file handle, if file id and file handle is valid and it associated, return Ok async fn valid_file_handle_id(&self, file_id: u64, fh: u64) -> Result<()>; diff --git a/clients/filesystem-fuse/src/fuse_api_handle.rs b/clients/filesystem-fuse/src/fuse_api_handle.rs index 76af581446b..7dc5461ce7f 100644 --- a/clients/filesystem-fuse/src/fuse_api_handle.rs +++ b/clients/filesystem-fuse/src/fuse_api_handle.rs @@ -52,10 +52,6 @@ impl FuseApiHandle { } } - pub async fn get_file_path(&self, file_id: u64) -> String { - self.fs.get_file_path(file_id).await - } - async fn get_modified_file_stat( &self, file_id: u64, diff --git a/clients/filesystem-fuse/src/utils.rs b/clients/filesystem-fuse/src/utils.rs index 7619775c949..6aeb2d5e767 100644 --- a/clients/filesystem-fuse/src/utils.rs +++ b/clients/filesystem-fuse/src/utils.rs @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ +use crate::filesystem::RawFileSystem; // join the parent and name to a path pub fn join_file_path(parent: &str, name: &str) -> String { @@ -35,6 +36,13 @@ pub fn split_file_path(path: &str) -> (&str, &str) { } } +// convert file id to string if file id is invalid return "Unknown" +pub async fn file_id_to_string(file_id: u64, fs: &impl RawFileSystem) -> String { + fs.get_file_path(file_id) + .await + .unwrap_or("Unknown".to_string()) +} + #[cfg(test)] mod tests { use super::*; From e23a5891739ed62838ad0c2c0ea17ebf598acbca Mon Sep 17 00:00:00 2001 From: yuhui Date: Mon, 23 Dec 2024 17:06:34 +0800 Subject: [PATCH 32/47] Update --- clients/filesystem-fuse/src/utils.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/filesystem-fuse/src/utils.rs b/clients/filesystem-fuse/src/utils.rs index 6aeb2d5e767..0c0cc80a162 100644 --- a/clients/filesystem-fuse/src/utils.rs +++ b/clients/filesystem-fuse/src/utils.rs @@ -36,8 +36,8 @@ pub fn split_file_path(path: &str) -> (&str, &str) { } } -// convert file id to string if file id is invalid return "Unknown" -pub async fn file_id_to_string(file_id: u64, fs: &impl RawFileSystem) -> String { +// convert file id to file path string if file id is invalid return "Unknown" +pub async fn file_id_to_file_path_string(file_id: u64, fs: &impl RawFileSystem) -> String { fs.get_file_path(file_id) .await .unwrap_or("Unknown".to_string()) From e8c74f6c703b07335a4831d28d67312c95d9582e Mon Sep 17 00:00:00 2001 From: yuhui Date: Mon, 23 Dec 2024 17:22:38 +0800 Subject: [PATCH 33/47] Update --- .../src/default_raw_filesystem.rs | 56 +++++++++---------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index d08ffc77b5b..4e3c9ed41d3 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -98,7 +98,7 @@ impl DefaultRawFileSystem { file_id: u64, flags: u32, kind: FileType, - ) -> crate::filesystem::Result { + ) -> Result { let file_entry = self.get_file_entry(file_id).await?; let mut opened_file = { @@ -122,17 +122,28 @@ impl DefaultRawFileSystem { let file = file.lock().await; Ok(file.file_handle()) } + + async fn remove_file_entry_locked(&self, path: &str) { + let mut file_manager = self.file_entry_manager.write().await; + file_manager.remove(path); + } + + async fn insert_file_entry_locked(&self, parent_file_id: u64, file_id: u64, path: &str) { + let mut file_manager = self.file_entry_manager.write().await; + file_manager.insert(parent_file_id, file_id, path); + } } #[async_trait] impl RawFileSystem for DefaultRawFileSystem { async fn init(&self) -> Result<()> { // init root directory - self.file_entry_manager.write().await.insert( + self.insert_file_entry_locked( Self::ROOT_DIR_PARENT_FILE_ID, Self::ROOT_DIR_FILE_ID, Self::ROOT_DIR_NAME, - ); + ) + .await; self.fs.init().await } @@ -191,12 +202,7 @@ impl RawFileSystem for DefaultRawFileSystem { .await } - async fn create_file( - &self, - parent_file_id: u64, - name: &str, - flags: u32, - ) -> crate::filesystem::Result { + async fn create_file(&self, parent_file_id: u64, name: &str, flags: u32) -> Result { let parent_file_entry = self.get_file_entry(parent_file_id).await?; let mut opened_file = self .fs @@ -206,14 +212,12 @@ impl RawFileSystem for DefaultRawFileSystem { opened_file.set_file_id(parent_file_id, self.next_file_id()); // insert the new file to file entry manager - { - let mut file_manager = self.file_entry_manager.write().await; - file_manager.insert( - parent_file_id, - opened_file.file_stat.file_id, - &opened_file.file_stat.path, - ); - } + self.insert_file_entry_locked( + parent_file_id, + opened_file.file_stat.file_id, + &opened_file.file_stat.path, + ) + .await; // put the file to the opened file manager let opened_file = self.opened_file_manager.put(opened_file); @@ -228,10 +232,8 @@ impl RawFileSystem for DefaultRawFileSystem { filestat.set_file_id(parent_file_id, self.next_file_id()); // insert the new file to file entry manager - { - let mut file_manager = self.file_entry_manager.write().await; - file_manager.insert(filestat.parent_file_id, filestat.file_id, &filestat.path); - } + self.insert_file_entry_locked(parent_file_id, filestat.file_id, &filestat.path) + .await; Ok(filestat.file_id) } @@ -245,10 +247,8 @@ impl RawFileSystem for DefaultRawFileSystem { self.fs.remove_file(&parent_file_entry.path, name).await?; // remove the file from file entry manager - { - let mut file_manager = self.file_entry_manager.write().await; - file_manager.remove(&join_file_path(&parent_file_entry.path, name)); - } + self.remove_file_entry_locked(&join_file_path(&parent_file_entry.path, name)) + .await; Ok(()) } @@ -257,10 +257,8 @@ impl RawFileSystem for DefaultRawFileSystem { self.fs.remove_dir(&parent_file_entry.path, name).await?; // remove the dir from file entry manager - { - let mut file_manager = self.file_entry_manager.write().await; - file_manager.remove(&join_file_path(&parent_file_entry.path, name)); - } + self.remove_file_entry_locked(&join_file_path(&parent_file_entry.path, name)) + .await; Ok(()) } From b99af102f77c10339acae55ae272e02f2982e477 Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 24 Dec 2024 09:54:43 +0800 Subject: [PATCH 34/47] Update --- clients/filesystem-fuse/src/filesystem.rs | 33 +++++------ clients/filesystem-fuse/src/fuse_server.rs | 66 ++++++++-------------- clients/filesystem-fuse/src/lib.rs | 3 +- clients/filesystem-fuse/src/main.rs | 26 +++------ clients/filesystem-fuse/src/mount.rs | 29 +++++----- clients/filesystem-fuse/tests/fuse_test.rs | 17 ++++-- 6 files changed, 70 insertions(+), 104 deletions(-) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index fc138ca10b5..8f4da86820a 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -29,7 +29,7 @@ pub(crate) type Result = std::result::Result; pub(crate) const ROOT_DIR_PARENT_FILE_ID: u64 = 1; pub(crate) const ROOT_DIR_FILE_ID: u64 = 1; -pub(crate) const ROOT_DIR_PATH: &'static str = "/"; +pub(crate) const ROOT_DIR_PATH: &str = "/"; pub(crate) const INITIAL_FILE_ID: u64 = 10000; /// RawFileSystem interface for the file system implementation. it use by FuseApiHandle, @@ -268,10 +268,10 @@ pub trait FileWriter: Sync + Send { #[cfg(test)] mod tests { - use std::collections::HashMap; + use super::*; use crate::default_raw_filesystem::DefaultRawFileSystem; use crate::memory_filesystem::MemoryFileSystem; - use super::*; + use std::collections::HashMap; #[test] fn test_create_file_stat() { @@ -319,7 +319,6 @@ mod tests { file_stat.set_file_id(1, 0); } - #[tokio::test] async fn test_memory_file_system() { let fs = MemoryFileSystem::new().await; @@ -333,14 +332,14 @@ mod tests { // test root file let root_dir_path = Path::new("/"); let root_file_stat = fs.stat(root_dir_path).await; - assert_eq!(root_file_stat.is_ok(), true); + assert!(root_file_stat.is_ok()); let root_file_stat = root_file_stat.unwrap(); assert_file_stat(&root_file_stat, root_dir_path, Directory, 0); // test meta file let meta_file_path = Path::new("/.gvfs_meta"); let meta_file_stat = fs.stat(meta_file_path).await; - assert_eq!(meta_file_stat.is_ok(), true); + assert!(meta_file_stat.is_ok()); let meta_file_stat = meta_file_stat.unwrap(); assert_file_stat(&meta_file_stat, meta_file_path, FileType::RegularFile, 0); root_dir_child_file_stats.insert(meta_file_stat.path.clone(), meta_file_stat); @@ -348,7 +347,7 @@ mod tests { // test create file let file_path = Path::new("/file1.txt"); let opened_file = fs.create_file(file_path, OpenFileFlags(0)).await; - assert_eq!(opened_file.is_ok(), true); + assert!(opened_file.is_ok()); let file = opened_file.unwrap(); assert_file_stat(&file.file_stat, file_path, FileType::RegularFile, 0); root_dir_child_file_stats.insert(file.file_stat.path.clone(), file.file_stat.clone()); @@ -356,14 +355,14 @@ mod tests { // test create dir let dir_path = Path::new("/dir1"); let dir_stat = fs.create_dir(dir_path).await; - assert_eq!(dir_stat.is_ok(), true); + assert!(dir_stat.is_ok()); let dir_stat = dir_stat.unwrap(); assert_file_stat(&dir_stat, dir_path, Directory, 0); root_dir_child_file_stats.insert(dir_stat.path.clone(), dir_stat); // test list dir let list_dir = fs.read_dir(Path::new("/")).await; - assert_eq!(list_dir.is_ok(), true); + assert!(list_dir.is_ok()); let list_dir = list_dir.unwrap(); assert_eq!(list_dir.len(), root_dir_child_file_stats.len()); for file_stat in list_dir { @@ -379,17 +378,17 @@ mod tests { // test remove file let remove_file = fs.remove_file(file_path).await; - assert_eq!(remove_file.is_ok(), true); + assert!(remove_file.is_ok()); root_dir_child_file_stats.remove(file_path); // test remove dir let remove_dir = fs.remove_dir(dir_path).await; - assert_eq!(remove_dir.is_ok(), true); + assert!(remove_dir.is_ok()); root_dir_child_file_stats.remove(dir_path); // test list dir let list_dir = fs.read_dir(Path::new("/")).await; - assert_eq!(list_dir.is_ok(), true); + assert!(list_dir.is_ok()); let list_dir = list_dir.unwrap(); assert_eq!(list_dir.len(), root_dir_child_file_stats.len()); @@ -406,7 +405,7 @@ mod tests { // test file not found let not_found_file = fs.stat(Path::new("/not_found.txt")).await; - assert_eq!(not_found_file.is_err(), true); + assert!(not_found_file.is_err()); } fn assert_file_stat(file_stat: &FileStat, path: &Path, kind: FileType, size: u64) { @@ -415,17 +414,15 @@ mod tests { assert_eq!(file_stat.size, size); } - #[tokio::test] async fn test_default_raw_file_system() { let memory_fs = MemoryFileSystem::new().await; - let raw_fs = DefaultRawFileSystem::new(memory_fs).await; + let raw_fs = DefaultRawFileSystem::new(memory_fs); let _ = raw_fs.init().await; test_raw_file_system(&raw_fs).await; } - async fn test_raw_file_system(fs: &impl PathFileSystem) { - let mut root_dir_child_file_stats = HashMap::new(); - + async fn test_raw_file_system(fs: &impl RawFileSystem) { + let _ = fs.init().await; } } diff --git a/clients/filesystem-fuse/src/fuse_server.rs b/clients/filesystem-fuse/src/fuse_server.rs index 9f5c6efdc03..9259b51c7f6 100644 --- a/clients/filesystem-fuse/src/fuse_server.rs +++ b/clients/filesystem-fuse/src/fuse_server.rs @@ -16,19 +16,18 @@ * specific language governing permissions and limitations * under the License. */ -use fuse3::raw::{Filesystem, MountHandle, Session}; +use fuse3::raw::{Filesystem, Session}; use fuse3::{MountOptions, Result}; use log::{error, info}; use std::process::exit; use std::sync::Arc; -use std::time::Duration; -use tokio::sync::{Mutex, Notify}; -use tokio::time::timeout; +use tokio::select; +use tokio::sync::Notify; /// Represents a FUSE server capable of starting and stopping the FUSE filesystem. pub struct FuseServer { - // Shared handle to manage FUSE unmounting - mount_handle: Arc>>, // Shared handle to manage FUSE unmounting + // Notification for stop + close_notify: Arc, // Mount point of the FUSE filesystem mount_point: String, @@ -38,7 +37,7 @@ impl FuseServer { /// Creates a new instance of `FuseServer`. pub fn new(mount_point: &str) -> Self { Self { - mount_handle: Arc::new(Mutex::new(None)), + close_notify: Arc::new(Default::default()), mount_point: mount_point.to_string(), } } @@ -57,49 +56,30 @@ impl FuseServer { ); let mount_options = MountOptions::default(); - let mount_handle = Session::new(mount_options) + let mut mount_handle = Session::new(mount_options) .mount_with_unprivileged(fuse_fs, &self.mount_point) .await?; - { - let mut handle_guard = self.mount_handle.lock().await; - *handle_guard = Some(mount_handle); - } - - info!("Received stop notification, FUSE filesystem will be unmounted."); - Ok(()) - } - - /// Stops the FUSE filesystem and waits for unmounting to complete. - pub async fn stop(&self) -> Result<()> { - info!("Stopping FUSE filesystem..."); - let timeout_duration = Duration::from_secs(5); - - let handle = { - let mut handle_guard = self.mount_handle.lock().await; - handle_guard.take() // Take the handle out to unmount - }; - - if let Some(mount_handle) = handle { - let res = timeout(timeout_duration, mount_handle.unmount()).await; + let handle = &mut mount_handle; - match res { - Ok(Ok(())) => { - info!("FUSE filesystem unmounted successfully."); - Ok(()) - } - Ok(Err(e)) => { + select! { + res = handle => res?, + _ = self.close_notify.notified() => { + if let Err(e) = mount_handle.unmount().await { error!("Failed to unmount FUSE filesystem: {:?}", e); - Err(e.into()) - } - Err(_) => { - error!("Unmount timed out."); - Err(libc::ETIMEDOUT.into()) + } else { + info!("FUSE filesystem unmounted successfully."); } } - } else { - error!("No active mount handle to unmount."); - Err(libc::EBADF.into()) } + self.close_notify.notify_one(); + Ok(()) + } + + /// Stops the FUSE filesystem. + pub async fn stop(&self) { + info!("Stopping FUSE filesystem..."); + self.close_notify.notify_one(); + self.close_notify.notified().await; } } diff --git a/clients/filesystem-fuse/src/lib.rs b/clients/filesystem-fuse/src/lib.rs index c7a9c17b126..36e8c28d343 100644 --- a/clients/filesystem-fuse/src/lib.rs +++ b/clients/filesystem-fuse/src/lib.rs @@ -32,5 +32,4 @@ pub async fn gvfs_mount(mount_point: &str) -> fuse3::Result<()> { pub async fn gvfs_unmount() { mount::unmount().await; - -} \ No newline at end of file +} diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index 6e4a458b398..28866a9bb1c 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -16,30 +16,18 @@ * specific language governing permissions and limitations * under the License. */ -use log::{error, info}; -use tokio::signal; use gvfs_fuse::{gvfs_mount, gvfs_unmount}; +use log::info; +use tokio::signal; #[tokio::main] async fn main() -> fuse3::Result<()> { tracing_subscriber::fmt().init(); - let handle = tokio::spawn(async { - gvfs_mount("gvfs").await - }); + tokio::spawn(async { gvfs_mount("gvfs").await }); + + let _ = signal::ctrl_c().await; + info!("Received Ctrl+C, Unmounting gvfs..."); + gvfs_unmount().await; - tokio::select! { - hd = handle => { - match hd { - Ok(Ok(_)) => info!("Mount succeeded."), - Ok(Err(e)) => error!("Mount failed: {:?}", e), - Err(e) => error!("Mount failed: {:?}", e), - } - } - _ = signal::ctrl_c() => { - info!("Received Ctrl+C, stopping server..."); - info!("Unmounting gvfs..."); - gvfs_unmount().await; - } - } Ok(()) } diff --git a/clients/filesystem-fuse/src/mount.rs b/clients/filesystem-fuse/src/mount.rs index 5d32d766878..6070118d798 100644 --- a/clients/filesystem-fuse/src/mount.rs +++ b/clients/filesystem-fuse/src/mount.rs @@ -30,28 +30,25 @@ static SERVER: LazyLock>>> = LazyLock::new(|| Mutex pub async fn mount(mount_point: &str) -> fuse3::Result<()> { info!("Starting gvfs-fuse server..."); - let mut server = SERVER.lock().await; - let svr = Arc::new(FuseServer::new(mount_point)); + { + let mut server = SERVER.lock().await; + *server = Some(svr.clone()); + } let fs = create_fuse_fs().await; - let result = svr.start(fs).await; - - *server = Some(svr); - result - + svr.start(fs).await } pub async fn unmount() { info!("Stop gvfs-fuse server..."); - let mut server = SERVER.lock().await; - info!("Stop gvfs-fuse server...0"); - if server.is_none() { - info!("Stop gvfs-fuse server...1"); - return; - } - info!("Stop gvfs-fuse server...2"); - let svr = server.take().unwrap(); - info!("Stop gvfs-fuse server...3"); + let svr = { + let mut server = SERVER.lock().await; + if server.is_none() { + info!("Server is already stopped."); + return; + } + server.take().unwrap() + }; let _ = svr.stop().await; } diff --git a/clients/filesystem-fuse/tests/fuse_test.rs b/clients/filesystem-fuse/tests/fuse_test.rs index 47dc6c1036e..80ac1f07b3e 100644 --- a/clients/filesystem-fuse/tests/fuse_test.rs +++ b/clients/filesystem-fuse/tests/fuse_test.rs @@ -26,24 +26,27 @@ use std::sync::Arc; use std::thread::sleep; use std::time::{Duration, Instant}; use tokio::runtime::Runtime; +use tokio::task::JoinHandle; struct FuseTest { runtime: Arc, mount_point: String, + gvfs_mount: Option>>, } impl FuseTest { - pub fn setup(&self) { + pub fn setup(&mut self) { info!("Start gvfs fuse server"); let mount_point = self.mount_point.clone(); - self.runtime.spawn(async move { gvfs_mount(&mount_point).await }); + self.runtime + .spawn(async move { gvfs_mount(&mount_point).await }); let success = Self::wait_for_fuse_server_ready(&self.mount_point, Duration::from_secs(15)); assert!(success, "Fuse server cannot start up at 15 seconds"); } - pub fn shutdown(&self) { + pub fn shutdown(&mut self) { self.runtime.block_on(async { - gvfs_unmount(); + gvfs_unmount().await; }); } @@ -76,9 +79,10 @@ fn test_fuse_system_with_auto() { let mount_point = "build/gvfs"; let _ = fs::create_dir_all(mount_point); - let test = FuseTest { + let mut test = FuseTest { runtime: Arc::new(Runtime::new().unwrap()), mount_point: mount_point.to_string(), + gvfs_mount: None, }; test.setup(); @@ -136,7 +140,8 @@ fn test_fuse_filesystem(mount_point: &str) { assert!(!file_exists(&test_dir)); info!("Success test"); - sleep(Duration::from_secs(10)); + + sleep(Duration::from_secs(15)); } fn file_exists>(path: P) -> bool { From 4dd5bf4fb936c0e4fae039dfc6bd607149e3f5e3 Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 24 Dec 2024 09:59:40 +0800 Subject: [PATCH 35/47] Update --- clients/filesystem-fuse/src/default_raw_filesystem.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index 4e3c9ed41d3..f524e1878e0 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -148,8 +148,8 @@ impl RawFileSystem for DefaultRawFileSystem { } async fn get_file_path(&self, file_id: u64) -> Result { - let file_entry = self.get_file_entry(file_id).await; - Ok(file_entry?.path) + let file_entry = self.get_file_entry(file_id).await?; + Ok(file_entry.path) } async fn valid_file_handle_id(&self, file_id: u64, fh: u64) -> Result<()> { @@ -186,8 +186,9 @@ impl RawFileSystem for DefaultRawFileSystem { async fn read_dir(&self, file_id: u64) -> Result> { let file_entry = self.get_file_entry(file_id).await?; let mut child_filestats = self.fs.read_dir(&file_entry.path).await?; - for file in child_filestats.iter_mut() { - self.resolve_file_id_to_filestat(file, file.file_id).await; + for file_stat in child_filestats.iter_mut() { + self.resolve_file_id_to_filestat(file_stat, file_stat.file_id) + .await; } Ok(child_filestats) } From 01d90a68416a68d67bd2b9eb124f9b32186920ab Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 24 Dec 2024 10:12:34 +0800 Subject: [PATCH 36/47] Update --- .../src/default_raw_filesystem.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index f524e1878e0..02a36b95985 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -205,25 +205,27 @@ impl RawFileSystem for DefaultRawFileSystem { async fn create_file(&self, parent_file_id: u64, name: &str, flags: u32) -> Result { let parent_file_entry = self.get_file_entry(parent_file_id).await?; - let mut opened_file = self + let mut opened_file_with_out_file_handle_id = self .fs .create_file(&parent_file_entry.path, name, OpenFileFlags(flags)) .await?; - opened_file.set_file_id(parent_file_id, self.next_file_id()); + opened_file_with_out_file_handle_id.set_file_id(parent_file_id, self.next_file_id()); // insert the new file to file entry manager self.insert_file_entry_locked( parent_file_id, - opened_file.file_stat.file_id, - &opened_file.file_stat.path, + opened_file_with_out_file_handle_id.file_stat.file_id, + &opened_file_with_out_file_handle_id.file_stat.path, ) .await; - // put the file to the opened file manager - let opened_file = self.opened_file_manager.put(opened_file); - let opened_file = opened_file.lock().await; - Ok(opened_file.file_handle()) + // put the openfile to the opened file manager and allocate a file handle id + let opened_file_with_file_handle_id = self + .opened_file_manager + .put(opened_file_with_out_file_handle_id); + let opened_file_with_file_handle_id = opened_file_with_file_handle_id.lock().await; + Ok(opened_file_with_file_handle_id.file_handle()) } async fn create_dir(&self, parent_file_id: u64, name: &str) -> Result { From 14609cf09d1b7cfc128ab693af9e62ea06efe1b6 Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 24 Dec 2024 10:13:54 +0800 Subject: [PATCH 37/47] Update --- clients/filesystem-fuse/src/opened_file_manager.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/clients/filesystem-fuse/src/opened_file_manager.rs b/clients/filesystem-fuse/src/opened_file_manager.rs index 005ec629145..17bfe00a397 100644 --- a/clients/filesystem-fuse/src/opened_file_manager.rs +++ b/clients/filesystem-fuse/src/opened_file_manager.rs @@ -45,6 +45,7 @@ impl OpenedFileManager { } pub(crate) fn put(&self, mut file: OpenedFile) -> Arc> { + // Put the file into the file handle map, and allocate a file handle id for the file. let file_handle_id = self.next_handle_id(); file.handle_id = file_handle_id; let file_handle = Arc::new(Mutex::new(file)); From 127cb41fd408e2cf1cafca381f454229ebc63185 Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 24 Dec 2024 10:54:34 +0800 Subject: [PATCH 38/47] Update --- .../filesystem-fuse/src/default_raw_filesystem.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index 02a36b95985..8add88d7aa4 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -205,26 +205,24 @@ impl RawFileSystem for DefaultRawFileSystem { async fn create_file(&self, parent_file_id: u64, name: &str, flags: u32) -> Result { let parent_file_entry = self.get_file_entry(parent_file_id).await?; - let mut opened_file_with_out_file_handle_id = self + let mut file_with_out_id = self .fs .create_file(&parent_file_entry.path, name, OpenFileFlags(flags)) .await?; - opened_file_with_out_file_handle_id.set_file_id(parent_file_id, self.next_file_id()); + file_with_out_id.set_file_id(parent_file_id, self.next_file_id()); // insert the new file to file entry manager self.insert_file_entry_locked( parent_file_id, - opened_file_with_out_file_handle_id.file_stat.file_id, - &opened_file_with_out_file_handle_id.file_stat.path, + file_with_out_id.file_stat.file_id, + &file_with_out_id.file_stat.path, ) .await; // put the openfile to the opened file manager and allocate a file handle id - let opened_file_with_file_handle_id = self - .opened_file_manager - .put(opened_file_with_out_file_handle_id); - let opened_file_with_file_handle_id = opened_file_with_file_handle_id.lock().await; + let file_with_id = self.opened_file_manager.put(file_with_out_id); + let opened_file_with_file_handle_id = file_with_id.lock().await; Ok(opened_file_with_file_handle_id.file_handle()) } From df48db4622d070b2ca4080c441ad337e6333f597 Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 24 Dec 2024 10:55:31 +0800 Subject: [PATCH 39/47] Update --- clients/filesystem-fuse/src/default_raw_filesystem.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index 8add88d7aa4..9a66cd551f0 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -205,23 +205,23 @@ impl RawFileSystem for DefaultRawFileSystem { async fn create_file(&self, parent_file_id: u64, name: &str, flags: u32) -> Result { let parent_file_entry = self.get_file_entry(parent_file_id).await?; - let mut file_with_out_id = self + let mut file_without_id = self .fs .create_file(&parent_file_entry.path, name, OpenFileFlags(flags)) .await?; - file_with_out_id.set_file_id(parent_file_id, self.next_file_id()); + file_without_id.set_file_id(parent_file_id, self.next_file_id()); // insert the new file to file entry manager self.insert_file_entry_locked( parent_file_id, - file_with_out_id.file_stat.file_id, - &file_with_out_id.file_stat.path, + file_without_id.file_stat.file_id, + &file_without_id.file_stat.path, ) .await; // put the openfile to the opened file manager and allocate a file handle id - let file_with_id = self.opened_file_manager.put(file_with_out_id); + let file_with_id = self.opened_file_manager.put(file_without_id); let opened_file_with_file_handle_id = file_with_id.lock().await; Ok(opened_file_with_file_handle_id.file_handle()) } From 37a46d6f58e1ff111d39f4d33a781e9c59a8459e Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 24 Dec 2024 11:01:22 +0800 Subject: [PATCH 40/47] Update --- clients/filesystem-fuse/Cargo.toml | 2 - .../filesystem-fuse/src/memory_filesystem.rs | 59 +++++++++---------- 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/clients/filesystem-fuse/Cargo.toml b/clients/filesystem-fuse/Cargo.toml index 7c87434456a..2d02639fc4a 100644 --- a/clients/filesystem-fuse/Cargo.toml +++ b/clients/filesystem-fuse/Cargo.toml @@ -40,7 +40,5 @@ fuse3 = { version = "0.8.1", "features" = ["tokio-runtime", "unprivileged"] } futures-util = "0.3.30" libc = "0.2.168" log = "0.4.22" -regex = "1.11.1" tokio = { version = "1.38.0", features = ["full"] } tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } - diff --git a/clients/filesystem-fuse/src/memory_filesystem.rs b/clients/filesystem-fuse/src/memory_filesystem.rs index bba19604e33..0592900624c 100644 --- a/clients/filesystem-fuse/src/memory_filesystem.rs +++ b/clients/filesystem-fuse/src/memory_filesystem.rs @@ -61,19 +61,19 @@ impl MemoryFileSystem { #[async_trait] impl PathFileSystem for MemoryFileSystem { async fn init(&self) -> Result<()> { - let root = MemoryFile { + let root_file = MemoryFile { kind: Directory, data: Arc::new(Mutex::new(Vec::new())), }; let root_path = PathBuf::from("/"); - self.file_map.write().unwrap().insert(root_path, root); + self.file_map.write().unwrap().insert(root_path, root_file); - let meta = MemoryFile { + let meta_file = MemoryFile { kind: RegularFile, data: Arc::new(Mutex::new(Vec::new())), }; let meta_file_path = Path::new(Self::FS_META_FILE_NAME).to_path_buf(); - self.file_map.write().unwrap().insert(meta_file_path, meta); + self.file_map.write().unwrap().insert(meta_file_path, meta_file); Ok(()) } @@ -100,21 +100,21 @@ impl PathFileSystem for MemoryFileSystem { async fn open_file(&self, path: &Path, _flags: OpenFileFlags) -> Result { let file_stat = self.stat(path).await?; - let mut file = OpenedFile::new(file_stat.clone()); - match file.file_stat.kind { - Directory => Ok(file), + let mut opened_file = OpenedFile::new(file_stat); + match opened_file.file_stat.kind { + Directory => Ok(opened_file), RegularFile => { let data = self .file_map .read() .unwrap() - .get(&file.file_stat.path) + .get(&opened_file.file_stat.path) .unwrap() .data .clone(); - file.reader = Some(Box::new(MemoryFileReader { data: data.clone() })); - file.writer = Some(Box::new(MemoryFileWriter { data: data })); - Ok(file) + opened_file.reader = Some(Box::new(MemoryFileReader { data: data.clone() })); + opened_file.writer = Some(Box::new(MemoryFileWriter { data: data })); + Ok(opened_file) } _ => Err(Errno::from(libc::EBADF)), } @@ -125,39 +125,36 @@ impl PathFileSystem for MemoryFileSystem { } async fn create_file(&self, path: &Path, _flags: OpenFileFlags) -> Result { - { - let file_map = self.file_map.read().unwrap(); - if file_map.contains_key(path) { - return Err(Errno::from(libc::EEXIST)); - } - }; + let mut file_map = self.file_map.write().unwrap(); + if file_map.contains_key(path) { + return Err(Errno::from(libc::EEXIST)); + } - let mut file = OpenedFile::new(FileStat::new_file_filestat_with_path(path, 0)); + let mut opened_file = OpenedFile::new(FileStat::new_file_filestat_with_path(path, 0)); let data = Arc::new(Mutex::new(Vec::new())); - self.file_map.write().unwrap().insert( - file.file_stat.path.clone(), + file_map.insert( + opened_file.file_stat.path.clone(), MemoryFile { kind: RegularFile, data: data.clone(), }, ); - file.reader = Some(Box::new(MemoryFileReader { data: data.clone() })); - file.writer = Some(Box::new(MemoryFileWriter { data: data })); - Ok(file) + opened_file.reader = Some(Box::new(MemoryFileReader { data: data.clone() })); + opened_file.writer = Some(Box::new(MemoryFileWriter { data: data })); + + Ok(opened_file) } async fn create_dir(&self, path: &Path) -> Result { - { - let file_map = self.file_map.read().unwrap(); - if file_map.contains_key(path) { - return Err(Errno::from(libc::EEXIST)); - } + let mut file_map = self.file_map.write().unwrap(); + if file_map.contains_key(path) { + return Err(Errno::from(libc::EEXIST)); } let file = FileStat::new_dir_filestat_with_path(path); - self.file_map.write().unwrap().insert( + file_map.insert( file.path.clone(), MemoryFile { kind: Directory, @@ -188,7 +185,9 @@ impl PathFileSystem for MemoryFileSystem { return Err(Errno::from(libc::ENOTEMPTY)); } - file_map.remove(path); + if file_map.remove(path).is_none() { + return Err(Errno::from(libc::ENOENT)); + } Ok(()) } } From 3a863edeed843b252a2b37f1aa855ec84975f664 Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 24 Dec 2024 11:55:34 +0800 Subject: [PATCH 41/47] Update --- clients/filesystem-fuse/Cargo.toml | 2 +- clients/filesystem-fuse/src/default_raw_filesystem.rs | 7 +------ clients/filesystem-fuse/src/main.rs | 1 - 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/clients/filesystem-fuse/Cargo.toml b/clients/filesystem-fuse/Cargo.toml index 6a1c635b847..75a4dd71301 100644 --- a/clients/filesystem-fuse/Cargo.toml +++ b/clients/filesystem-fuse/Cargo.toml @@ -40,6 +40,6 @@ fuse3 = { version = "0.8.1", "features" = ["tokio-runtime", "unprivileged"] } futures-util = "0.3.30" libc = "0.2.168" log = "0.4.22" +once_cell = "1.20.2" tokio = { version = "1.38.0", features = ["full"] } tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } -once_cell = "1.20.2" diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index dbf1f3f297b..37b6678e261 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -47,7 +47,6 @@ pub struct DefaultRawFileSystem { } impl DefaultRawFileSystem { - pub(crate) fn new(fs: T) -> Self { Self { file_entry_manager: RwLock::new(FileEntryManager::new()), @@ -174,11 +173,7 @@ impl RawFileSystem for DefaultRawFileSystem { Ok(file_stat) } - async fn lookup( - &self, - parent_file_id: u64, - name: &OsStr, - ) -> Result { + async fn lookup(&self, parent_file_id: u64, name: &OsStr) -> Result { let parent_file_entry = self.get_file_entry(parent_file_id).await?; // assume the path is a regular file. Some filesystems may need to check whether it is a file or directory by lookup. diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index 42b130c923e..28866a9bb1c 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -31,4 +31,3 @@ async fn main() -> fuse3::Result<()> { Ok(()) } - From 406641b7f226b07d73edebe3368d4f44f4ddb121 Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 24 Dec 2024 14:18:36 +0800 Subject: [PATCH 42/47] Update --- clients/filesystem-fuse/src/default_raw_filesystem.rs | 1 - clients/filesystem-fuse/src/filesystem.rs | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index 37b6678e261..0b9652cff38 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -176,7 +176,6 @@ impl RawFileSystem for DefaultRawFileSystem { async fn lookup(&self, parent_file_id: u64, name: &OsStr) -> Result { let parent_file_entry = self.get_file_entry(parent_file_id).await?; - // assume the path is a regular file. Some filesystems may need to check whether it is a file or directory by lookup. let path = parent_file_entry.path.join(name); let mut file_stat = self.fs.stat(&path).await?; // fill the file id to file stat diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index e74cd84fe2f..0d27c3574aa 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -29,6 +29,7 @@ pub(crate) type Result = std::result::Result; pub(crate) const ROOT_DIR_PARENT_FILE_ID: u64 = 1; pub(crate) const ROOT_DIR_FILE_ID: u64 = 1; +pub(crate) const ROOT_DIR_NAME: &str = ""; pub(crate) const ROOT_DIR_PATH: &str = "/"; pub(crate) const INITIAL_FILE_ID: u64 = 10000; @@ -215,7 +216,8 @@ impl FileStat { pub fn new_filestat(path: &Path, size: u64, kind: FileType) -> Self { let atime = Timestamp::from(SystemTime::now()); - let name = path.file_name().unwrap_or(OsStr::new("")); + // root directory name is "" + let name = path.file_name().unwrap_or(OsStr::new(ROOT_DIR_NAME)); Self { file_id: 0, parent_file_id: 0, From 0e512fc09c488402877ef444d59963bd9224afae Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 24 Dec 2024 14:59:22 +0800 Subject: [PATCH 43/47] Update --- clients/filesystem-fuse/src/fuse_server.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/clients/filesystem-fuse/src/fuse_server.rs b/clients/filesystem-fuse/src/fuse_server.rs index 9259b51c7f6..dae7c28a631 100644 --- a/clients/filesystem-fuse/src/fuse_server.rs +++ b/clients/filesystem-fuse/src/fuse_server.rs @@ -63,7 +63,11 @@ impl FuseServer { let handle = &mut mount_handle; select! { - res = handle => res?, + res = handle => { + if res.is_err() { + error!("Failed to mount FUSE filesystem: {:?}", res.err()); + } + }, _ = self.close_notify.notified() => { if let Err(e) = mount_handle.unmount().await { error!("Failed to unmount FUSE filesystem: {:?}", e); @@ -72,6 +76,8 @@ impl FuseServer { } } } + + // notify that the filesystem is stopped self.close_notify.notify_one(); Ok(()) } @@ -80,6 +86,8 @@ impl FuseServer { pub async fn stop(&self) { info!("Stopping FUSE filesystem..."); self.close_notify.notify_one(); + + // wait for the filesystem to stop self.close_notify.notified().await; } } From da02b01d411fc1cf137d5652ba5f539f7226a12f Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 24 Dec 2024 15:37:13 +0800 Subject: [PATCH 44/47] Add testers --- .../src/default_raw_filesystem.rs | 10 + clients/filesystem-fuse/src/filesystem.rs | 371 ++++++++++++------ .../filesystem-fuse/src/memory_filesystem.rs | 13 + 3 files changed, 284 insertions(+), 110 deletions(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index 0b9652cff38..d00a2995af1 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -382,6 +382,8 @@ impl FileEntryManager { #[cfg(test)] mod tests { use super::*; + use crate::filesystem::tests::TestRawFileSystem; + use crate::memory_filesystem::MemoryFileSystem; #[test] fn test_file_entry_manager() { @@ -401,4 +403,12 @@ mod tests { assert!(manager.get_file_entry_by_id(2).is_none()); assert!(manager.get_file_entry_by_path(Path::new("a/b")).is_none()); } + + #[tokio::test] + async fn test_default_raw_file_system() { + let memory_fs = MemoryFileSystem::new().await; + let raw_fs = DefaultRawFileSystem::new(memory_fs); + let _ = raw_fs.init().await; + TestRawFileSystem::test_raw_file_system(&raw_fs).await; + } } diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 0d27c3574aa..70da7ba8404 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -269,12 +269,270 @@ pub trait FileWriter: Sync + Send { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; - use crate::default_raw_filesystem::DefaultRawFileSystem; - use crate::memory_filesystem::MemoryFileSystem; use std::collections::HashMap; + pub(crate) struct TestPathFileSystem(); + + impl TestPathFileSystem { + pub(crate) async fn test_path_file_system(fs: &impl PathFileSystem) { + let mut root_dir_child_file_stats = HashMap::new(); + + // test root file + let root_dir_path = Path::new("/"); + let root_file_stat = fs.stat(root_dir_path).await; + assert!(root_file_stat.is_ok()); + let root_file_stat = root_file_stat.unwrap(); + TestPathFileSystem::assert_file_stat(&root_file_stat, root_dir_path, Directory, 0); + + // test meta file + let meta_file_path = Path::new("/.gvfs_meta"); + let meta_file_stat = fs.stat(meta_file_path).await; + assert!(meta_file_stat.is_ok()); + let meta_file_stat = meta_file_stat.unwrap(); + TestPathFileSystem::assert_file_stat( + &meta_file_stat, + meta_file_path, + FileType::RegularFile, + 0, + ); + root_dir_child_file_stats.insert(meta_file_stat.path.clone(), meta_file_stat); + + // test create file + let file_path = Path::new("/file1.txt"); + let opened_file = fs.create_file(file_path, OpenFileFlags(0)).await; + assert!(opened_file.is_ok()); + let file = opened_file.unwrap(); + TestPathFileSystem::assert_file_stat( + &file.file_stat, + file_path, + FileType::RegularFile, + 0, + ); + root_dir_child_file_stats.insert(file.file_stat.path.clone(), file.file_stat.clone()); + + // test create dir + let dir_path = Path::new("/dir1"); + let dir_stat = fs.create_dir(dir_path).await; + assert!(dir_stat.is_ok()); + let dir_stat = dir_stat.unwrap(); + TestPathFileSystem::assert_file_stat(&dir_stat, dir_path, Directory, 0); + root_dir_child_file_stats.insert(dir_stat.path.clone(), dir_stat); + + // test list dir + let list_dir = fs.read_dir(Path::new("/")).await; + assert!(list_dir.is_ok()); + let list_dir = list_dir.unwrap(); + assert_eq!(list_dir.len(), root_dir_child_file_stats.len()); + for file_stat in list_dir { + assert!(root_dir_child_file_stats.contains_key(&file_stat.path)); + let actual_file_stat = root_dir_child_file_stats.get(&file_stat.path).unwrap(); + TestPathFileSystem::assert_file_stat( + &file_stat, + &actual_file_stat.path, + actual_file_stat.kind, + actual_file_stat.size, + ); + } + + // test remove file + let remove_file = fs.remove_file(file_path).await; + assert!(remove_file.is_ok()); + root_dir_child_file_stats.remove(file_path); + + // test remove dir + let remove_dir = fs.remove_dir(dir_path).await; + assert!(remove_dir.is_ok()); + root_dir_child_file_stats.remove(dir_path); + + // test list dir + let list_dir = fs.read_dir(Path::new("/")).await; + assert!(list_dir.is_ok()); + + let list_dir = list_dir.unwrap(); + assert_eq!(list_dir.len(), root_dir_child_file_stats.len()); + for file_stat in list_dir { + assert!(root_dir_child_file_stats.contains_key(&file_stat.path)); + let actual_file_stat = root_dir_child_file_stats.get(&file_stat.path).unwrap(); + TestPathFileSystem::assert_file_stat( + &file_stat, + &actual_file_stat.path, + actual_file_stat.kind, + actual_file_stat.size, + ); + } + + // test file not found + let not_found_file = fs.stat(Path::new("/not_found.txt")).await; + assert!(not_found_file.is_err()); + } + + fn assert_file_stat(file_stat: &FileStat, path: &Path, kind: FileType, size: u64) { + assert_eq!(file_stat.path, path); + assert_eq!(file_stat.kind, kind); + assert_eq!(file_stat.size, size); + } + } + + pub(crate) struct TestRawFileSystem(); + + impl TestRawFileSystem { + pub(crate) async fn test_raw_file_system(fs: &impl RawFileSystem) { + let mut root_dir_child_file_stats = HashMap::new(); + + // test root file + let root_file_stat = fs.stat(ROOT_DIR_FILE_ID).await; + assert!(root_file_stat.is_ok()); + let root_file_stat = root_file_stat.unwrap(); + TestRawFileSystem::assert_file_stat( + &root_file_stat, + Path::new(ROOT_DIR_PATH), + Directory, + 0, + ); + + // test lookup meta file + let meta_file_stat = fs.lookup(ROOT_DIR_FILE_ID, ".gvfs_meta".as_ref()).await; + assert!(meta_file_stat.is_ok()); + let meta_file_stat = meta_file_stat.unwrap(); + TestRawFileSystem::assert_file_stat( + &meta_file_stat, + Path::new("/.gvfs_meta"), + RegularFile, + 0, + ); + + //test get file path + let file_path = fs.get_file_path(meta_file_stat.file_id).await; + assert!(file_path.is_ok()); + assert_eq!(file_path.unwrap(), meta_file_stat.path.to_string_lossy()); + + // test stat meta file + let meta_file_stat = fs.stat(meta_file_stat.file_id).await; + assert!(meta_file_stat.is_ok()); + let meta_file_stat = meta_file_stat.unwrap(); + TestRawFileSystem::assert_file_stat( + &meta_file_stat, + Path::new("/.gvfs_meta"), + RegularFile, + 0, + ); + root_dir_child_file_stats.insert(meta_file_stat.path.clone(), meta_file_stat); + + // test create file + let file = fs + .create_file(ROOT_DIR_FILE_ID, "file1.txt".as_ref(), 0) + .await; + assert!(file.is_ok()); + let file = file.unwrap(); + assert!(file.handle_id > 0); + assert!(file.file_id >= INITIAL_FILE_ID); + + //test open file + let file_handle = fs.open_file(file.file_id, 0).await; + assert!(file_handle.is_ok()); + let file_handle = file_handle.unwrap(); + assert_eq!(file_handle.file_id, file.file_id); + + // test stat file + let file_stat = fs.stat(file.file_id).await; + assert!(file_stat.is_ok()); + let file_stat = file_stat.unwrap(); + TestRawFileSystem::assert_file_stat( + &file_stat, + Path::new("/file1.txt"), + RegularFile, + 0, + ); + root_dir_child_file_stats.insert(file_stat.path.clone(), file_stat); + + // test create dir + let dir = fs.create_dir(ROOT_DIR_FILE_ID, "dir1".as_ref()).await; + assert!(dir.is_ok()); + let dir_file_id = dir.unwrap(); + assert!(dir_file_id >= INITIAL_FILE_ID); + + let dir = fs.stat(dir_file_id).await; + assert!(dir.is_ok()); + let dir = dir.unwrap(); + TestRawFileSystem::assert_file_stat(&dir, Path::new("/dir1"), Directory, 0); + root_dir_child_file_stats.insert(dir.path.clone(), dir); + + // test list dir + let list_dir = fs.read_dir(ROOT_DIR_FILE_ID).await; + assert!(list_dir.is_ok()); + let list_dir = list_dir.unwrap(); + assert_eq!(list_dir.len(), root_dir_child_file_stats.len()); + for file_stat in list_dir { + assert!(root_dir_child_file_stats.contains_key(&file_stat.path)); + let actual_file_stat = root_dir_child_file_stats.get(&file_stat.path).unwrap(); + TestRawFileSystem::assert_file_stat( + &file_stat, + &actual_file_stat.path, + actual_file_stat.kind, + actual_file_stat.size, + ); + } + + // test remove file + let remove_file = fs.remove_file(ROOT_DIR_FILE_ID, "file1.txt".as_ref()).await; + assert!(remove_file.is_ok()); + root_dir_child_file_stats.remove(Path::new("/file1.txt")); + + // test remove dir + let remove_dir = fs.remove_dir(ROOT_DIR_FILE_ID, "dir1".as_ref()).await; + assert!(remove_dir.is_ok()); + root_dir_child_file_stats.remove(Path::new("/dir1")); + + // test list dir + let list_dir = fs.read_dir(ROOT_DIR_FILE_ID).await; + assert!(list_dir.is_ok()); + let list_dir = list_dir.unwrap(); + assert_eq!(list_dir.len(), root_dir_child_file_stats.len()); + for file_stat in list_dir { + assert!(root_dir_child_file_stats.contains_key(&file_stat.path)); + let actual_file_stat = root_dir_child_file_stats.get(&file_stat.path).unwrap(); + TestRawFileSystem::assert_file_stat( + &file_stat, + &actual_file_stat.path, + actual_file_stat.kind, + actual_file_stat.size, + ); + } + + // test file not found + let not_found_file = fs.stat(ROOT_DIR_FILE_ID + 1).await; + assert!(not_found_file.is_err()); + + //test write file + let write_size = fs + .write(file.file_id, file_handle.handle_id, 0, "test".as_bytes()) + .await; + assert!(write_size.is_ok()); + assert_eq!(write_size.unwrap(), 4); + + // test read file + let read_data = fs.read(file.file_id, file_handle.handle_id, 0, 4).await; + assert!(read_data.is_ok()); + assert_eq!(read_data.unwrap(), "test".as_bytes()); + } + + fn assert_file_stat(file_stat: &FileStat, path: &Path, kind: FileType, size: u64) { + assert_eq!(file_stat.path, path); + assert_eq!(file_stat.kind, kind); + assert_eq!(file_stat.size, size); + if file_stat.file_id == 1 { + assert_eq!(file_stat.parent_file_id, 1); + } else { + assert!(file_stat.file_id >= INITIAL_FILE_ID); + assert!( + file_stat.parent_file_id == 1 || file_stat.parent_file_id >= INITIAL_FILE_ID + ); + } + } + } + #[test] fn test_create_file_stat() { //test new file @@ -320,111 +578,4 @@ mod tests { let mut file_stat = FileStat::new_file_filestat("a".as_ref(), "b".as_ref(), 10); file_stat.set_file_id(1, 0); } - - #[tokio::test] - async fn test_memory_file_system() { - let fs = MemoryFileSystem::new().await; - let _ = fs.init().await; - test_path_file_system(&fs).await; - } - - async fn test_path_file_system(fs: &impl PathFileSystem) { - let mut root_dir_child_file_stats = HashMap::new(); - - // test root file - let root_dir_path = Path::new("/"); - let root_file_stat = fs.stat(root_dir_path).await; - assert!(root_file_stat.is_ok()); - let root_file_stat = root_file_stat.unwrap(); - assert_file_stat(&root_file_stat, root_dir_path, Directory, 0); - - // test meta file - let meta_file_path = Path::new("/.gvfs_meta"); - let meta_file_stat = fs.stat(meta_file_path).await; - assert!(meta_file_stat.is_ok()); - let meta_file_stat = meta_file_stat.unwrap(); - assert_file_stat(&meta_file_stat, meta_file_path, FileType::RegularFile, 0); - root_dir_child_file_stats.insert(meta_file_stat.path.clone(), meta_file_stat); - - // test create file - let file_path = Path::new("/file1.txt"); - let opened_file = fs.create_file(file_path, OpenFileFlags(0)).await; - assert!(opened_file.is_ok()); - let file = opened_file.unwrap(); - assert_file_stat(&file.file_stat, file_path, FileType::RegularFile, 0); - root_dir_child_file_stats.insert(file.file_stat.path.clone(), file.file_stat.clone()); - - // test create dir - let dir_path = Path::new("/dir1"); - let dir_stat = fs.create_dir(dir_path).await; - assert!(dir_stat.is_ok()); - let dir_stat = dir_stat.unwrap(); - assert_file_stat(&dir_stat, dir_path, Directory, 0); - root_dir_child_file_stats.insert(dir_stat.path.clone(), dir_stat); - - // test list dir - let list_dir = fs.read_dir(Path::new("/")).await; - assert!(list_dir.is_ok()); - let list_dir = list_dir.unwrap(); - assert_eq!(list_dir.len(), root_dir_child_file_stats.len()); - for file_stat in list_dir { - assert!(root_dir_child_file_stats.contains_key(&file_stat.path)); - let actual_file_stat = root_dir_child_file_stats.get(&file_stat.path).unwrap(); - assert_file_stat( - &file_stat, - &actual_file_stat.path, - actual_file_stat.kind, - actual_file_stat.size, - ); - } - - // test remove file - let remove_file = fs.remove_file(file_path).await; - assert!(remove_file.is_ok()); - root_dir_child_file_stats.remove(file_path); - - // test remove dir - let remove_dir = fs.remove_dir(dir_path).await; - assert!(remove_dir.is_ok()); - root_dir_child_file_stats.remove(dir_path); - - // test list dir - let list_dir = fs.read_dir(Path::new("/")).await; - assert!(list_dir.is_ok()); - - let list_dir = list_dir.unwrap(); - assert_eq!(list_dir.len(), root_dir_child_file_stats.len()); - for file_stat in list_dir { - assert!(root_dir_child_file_stats.contains_key(&file_stat.path)); - let actual_file_stat = root_dir_child_file_stats.get(&file_stat.path).unwrap(); - assert_file_stat( - &file_stat, - &actual_file_stat.path, - actual_file_stat.kind, - actual_file_stat.size, - ); - } - - // test file not found - let not_found_file = fs.stat(Path::new("/not_found.txt")).await; - assert!(not_found_file.is_err()); - } - - fn assert_file_stat(file_stat: &FileStat, path: &Path, kind: FileType, size: u64) { - assert_eq!(file_stat.path, path); - assert_eq!(file_stat.kind, kind); - assert_eq!(file_stat.size, size); - } - - #[tokio::test] - async fn test_default_raw_file_system() { - let memory_fs = MemoryFileSystem::new().await; - let raw_fs = DefaultRawFileSystem::new(memory_fs); - let _ = raw_fs.init().await; - test_raw_file_system(&raw_fs).await; - } - - async fn test_raw_file_system(fs: &impl RawFileSystem) { - let _ = fs.init().await; - } } diff --git a/clients/filesystem-fuse/src/memory_filesystem.rs b/clients/filesystem-fuse/src/memory_filesystem.rs index c33f1d12597..3bc7915d60a 100644 --- a/clients/filesystem-fuse/src/memory_filesystem.rs +++ b/clients/filesystem-fuse/src/memory_filesystem.rs @@ -234,3 +234,16 @@ impl FileWriter for MemoryFileWriter { fn path_in_dir(dir: &Path, path: &Path) -> bool { path.starts_with(dir) && path != dir } + +#[cfg(test)] +mod tests { + use super::*; + use crate::filesystem::tests::TestPathFileSystem; + + #[tokio::test] + async fn test_memory_file_system() { + let fs = MemoryFileSystem::new().await; + let _ = fs.init().await; + TestPathFileSystem::test_path_file_system(&fs).await; + } +} From c043b95c551d0f7e7ecf0268d77baf3d194ec612 Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 24 Dec 2024 15:47:06 +0800 Subject: [PATCH 45/47] Fix test --- clients/filesystem-fuse/src/filesystem.rs | 42 +++++++++++++---------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 70da7ba8404..baaa9175635 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -279,7 +279,7 @@ pub(crate) mod tests { pub(crate) async fn test_path_file_system(fs: &impl PathFileSystem) { let mut root_dir_child_file_stats = HashMap::new(); - // test root file + // test root dir let root_dir_path = Path::new("/"); let root_file_stat = fs.stat(root_dir_path).await; assert!(root_file_stat.is_ok()); @@ -381,7 +381,7 @@ pub(crate) mod tests { pub(crate) async fn test_raw_file_system(fs: &impl RawFileSystem) { let mut root_dir_child_file_stats = HashMap::new(); - // test root file + // test root dir let root_file_stat = fs.stat(ROOT_DIR_FILE_ID).await; assert!(root_file_stat.is_ok()); let root_file_stat = root_file_stat.unwrap(); @@ -403,7 +403,7 @@ pub(crate) mod tests { 0, ); - //test get file path + //test get meta file path let file_path = fs.get_file_path(meta_file_stat.file_id).await; assert!(file_path.is_ok()); assert_eq!(file_path.unwrap(), meta_file_stat.path.to_string_lossy()); @@ -420,7 +420,7 @@ pub(crate) mod tests { ); root_dir_child_file_stats.insert(meta_file_stat.path.clone(), meta_file_stat); - // test create file + // test create file file1.txt let file = fs .create_file(ROOT_DIR_FILE_ID, "file1.txt".as_ref(), 0) .await; @@ -429,13 +429,29 @@ pub(crate) mod tests { assert!(file.handle_id > 0); assert!(file.file_id >= INITIAL_FILE_ID); - //test open file + //test open file file1.txt let file_handle = fs.open_file(file.file_id, 0).await; assert!(file_handle.is_ok()); let file_handle = file_handle.unwrap(); assert_eq!(file_handle.file_id, file.file_id); - // test stat file + //test write file file1.txt + let write_size = fs + .write(file.file_id, file_handle.handle_id, 0, "test".as_bytes()) + .await; + assert!(write_size.is_ok()); + assert_eq!(write_size.unwrap(), 4); + + // test read file file1.txt + let read_data = fs.read(file.file_id, file_handle.handle_id, 0, 4).await; + assert!(read_data.is_ok()); + assert_eq!(read_data.unwrap(), "test".as_bytes()); + + // test close file file1.txt + let close_file = fs.close_file(file.file_id, file_handle.handle_id).await; + assert!(close_file.is_ok()); + + // test stat file file1.txt let file_stat = fs.stat(file.file_id).await; assert!(file_stat.is_ok()); let file_stat = file_stat.unwrap(); @@ -443,7 +459,7 @@ pub(crate) mod tests { &file_stat, Path::new("/file1.txt"), RegularFile, - 0, + write_size.unwrap() as u64, ); root_dir_child_file_stats.insert(file_stat.path.clone(), file_stat); @@ -504,18 +520,6 @@ pub(crate) mod tests { // test file not found let not_found_file = fs.stat(ROOT_DIR_FILE_ID + 1).await; assert!(not_found_file.is_err()); - - //test write file - let write_size = fs - .write(file.file_id, file_handle.handle_id, 0, "test".as_bytes()) - .await; - assert!(write_size.is_ok()); - assert_eq!(write_size.unwrap(), 4); - - // test read file - let read_data = fs.read(file.file_id, file_handle.handle_id, 0, 4).await; - assert!(read_data.is_ok()); - assert_eq!(read_data.unwrap(), "test".as_bytes()); } fn assert_file_stat(file_stat: &FileStat, path: &Path, kind: FileType, size: u64) { From d4da50477775ad694646037798c1c09f71068caa Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 24 Dec 2024 17:40:16 +0800 Subject: [PATCH 46/47] Add testers --- .../src/default_raw_filesystem.rs | 9 +- clients/filesystem-fuse/src/filesystem.rs | 432 +++++++++++------- .../filesystem-fuse/src/memory_filesystem.rs | 3 +- 3 files changed, 275 insertions(+), 169 deletions(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index d00a2995af1..0ab92e91640 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -257,8 +257,7 @@ impl RawFileSystem for DefaultRawFileSystem { self.fs.remove_file(&path).await?; // remove the file from file entry manager - self.remove_file_entry_locked(&parent_file_entry.path.join(name)) - .await; + self.remove_file_entry_locked(&path).await; Ok(()) } @@ -268,8 +267,7 @@ impl RawFileSystem for DefaultRawFileSystem { self.fs.remove_dir(&path).await?; // remove the dir from file entry manager - self.remove_file_entry_locked(&parent_file_entry.path.join(name)) - .await; + self.remove_file_entry_locked(&path).await; Ok(()) } @@ -409,6 +407,7 @@ mod tests { let memory_fs = MemoryFileSystem::new().await; let raw_fs = DefaultRawFileSystem::new(memory_fs); let _ = raw_fs.init().await; - TestRawFileSystem::test_raw_file_system(&raw_fs).await; + let mut tester = TestRawFileSystem::new(raw_fs); + tester.test_raw_file_system().await; } } diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index baaa9175635..d9440b0e652 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -273,256 +273,362 @@ pub(crate) mod tests { use super::*; use std::collections::HashMap; - pub(crate) struct TestPathFileSystem(); + pub(crate) struct TestPathFileSystem { + files: HashMap, + fs: F, + } + + impl TestPathFileSystem { + pub(crate) fn new(fs: F) -> Self { + Self { + files: HashMap::new(), + fs, + } + } + + pub(crate) async fn test_path_file_system(&mut self) { + // Test root dir + self.test_root_dir().await; + + // Test stat file + self.test_stat_file(Path::new("/.gvfs_meta"), RegularFile, 0) + .await; + + // Test create file + self.test_create_file(Path::new("/file1.txt")).await; - impl TestPathFileSystem { - pub(crate) async fn test_path_file_system(fs: &impl PathFileSystem) { - let mut root_dir_child_file_stats = HashMap::new(); + // Test create dir + self.test_create_dir(Path::new("/dir1")).await; - // test root dir + // Test list dir + self.test_list_dir(Path::new("/")).await; + + // Test remove file + self.test_remove_file(Path::new("/file1.txt")).await; + + // Test remove dir + self.test_remove_dir(Path::new("/dir1")).await; + + // Test file not found + self.test_file_not_found(Path::new("unknown")).await; + + // Test list dir + self.test_list_dir(Path::new("/")).await; + } + + async fn test_root_dir(&mut self) { let root_dir_path = Path::new("/"); - let root_file_stat = fs.stat(root_dir_path).await; + let root_file_stat = self.fs.stat(root_dir_path).await; assert!(root_file_stat.is_ok()); let root_file_stat = root_file_stat.unwrap(); - TestPathFileSystem::assert_file_stat(&root_file_stat, root_dir_path, Directory, 0); - - // test meta file - let meta_file_path = Path::new("/.gvfs_meta"); - let meta_file_stat = fs.stat(meta_file_path).await; - assert!(meta_file_stat.is_ok()); - let meta_file_stat = meta_file_stat.unwrap(); - TestPathFileSystem::assert_file_stat( - &meta_file_stat, - meta_file_path, - FileType::RegularFile, - 0, - ); - root_dir_child_file_stats.insert(meta_file_stat.path.clone(), meta_file_stat); + self.assert_file_stat(&root_file_stat, root_dir_path, Directory, 0); + } - // test create file - let file_path = Path::new("/file1.txt"); - let opened_file = fs.create_file(file_path, OpenFileFlags(0)).await; + async fn test_stat_file(&mut self, path: &Path, expect_kind: FileType, expect_size: u64) { + let file_stat = self.fs.stat(path).await; + assert!(file_stat.is_ok()); + let file_stat = file_stat.unwrap(); + self.assert_file_stat(&file_stat, path, expect_kind, expect_size); + self.files.insert(file_stat.path.clone(), file_stat); + } + + async fn test_create_file(&mut self, path: &Path) { + let opened_file = self.fs.create_file(path, OpenFileFlags(0)).await; assert!(opened_file.is_ok()); let file = opened_file.unwrap(); - TestPathFileSystem::assert_file_stat( - &file.file_stat, - file_path, - FileType::RegularFile, - 0, - ); - root_dir_child_file_stats.insert(file.file_stat.path.clone(), file.file_stat.clone()); + self.assert_file_stat(&file.file_stat, path, FileType::RegularFile, 0); + self.test_stat_file(path, RegularFile, 0).await; + } - // test create dir - let dir_path = Path::new("/dir1"); - let dir_stat = fs.create_dir(dir_path).await; + async fn test_create_dir(&mut self, path: &Path) { + let dir_stat = self.fs.create_dir(path).await; assert!(dir_stat.is_ok()); let dir_stat = dir_stat.unwrap(); - TestPathFileSystem::assert_file_stat(&dir_stat, dir_path, Directory, 0); - root_dir_child_file_stats.insert(dir_stat.path.clone(), dir_stat); + self.assert_file_stat(&dir_stat, path, Directory, 0); + self.test_stat_file(path, Directory, 0).await; + } - // test list dir - let list_dir = fs.read_dir(Path::new("/")).await; + async fn test_list_dir(&self, path: &Path) { + let list_dir = self.fs.read_dir(path).await; assert!(list_dir.is_ok()); let list_dir = list_dir.unwrap(); - assert_eq!(list_dir.len(), root_dir_child_file_stats.len()); + assert_eq!(list_dir.len(), self.files.len()); for file_stat in list_dir { - assert!(root_dir_child_file_stats.contains_key(&file_stat.path)); - let actual_file_stat = root_dir_child_file_stats.get(&file_stat.path).unwrap(); - TestPathFileSystem::assert_file_stat( + assert!(self.files.contains_key(&file_stat.path)); + let actual_file_stat = self.files.get(&file_stat.path).unwrap(); + self.assert_file_stat( &file_stat, &actual_file_stat.path, actual_file_stat.kind, actual_file_stat.size, ); } + } - // test remove file - let remove_file = fs.remove_file(file_path).await; + async fn test_remove_file(&mut self, path: &Path) { + let remove_file = self.fs.remove_file(path).await; assert!(remove_file.is_ok()); - root_dir_child_file_stats.remove(file_path); + self.files.remove(path); - // test remove dir - let remove_dir = fs.remove_dir(dir_path).await; - assert!(remove_dir.is_ok()); - root_dir_child_file_stats.remove(dir_path); + self.test_file_not_found(path).await; + } - // test list dir - let list_dir = fs.read_dir(Path::new("/")).await; - assert!(list_dir.is_ok()); + async fn test_remove_dir(&mut self, path: &Path) { + let remove_dir = self.fs.remove_dir(path).await; + assert!(remove_dir.is_ok()); + self.files.remove(path); - let list_dir = list_dir.unwrap(); - assert_eq!(list_dir.len(), root_dir_child_file_stats.len()); - for file_stat in list_dir { - assert!(root_dir_child_file_stats.contains_key(&file_stat.path)); - let actual_file_stat = root_dir_child_file_stats.get(&file_stat.path).unwrap(); - TestPathFileSystem::assert_file_stat( - &file_stat, - &actual_file_stat.path, - actual_file_stat.kind, - actual_file_stat.size, - ); - } + self.test_file_not_found(path).await; + } - // test file not found - let not_found_file = fs.stat(Path::new("/not_found.txt")).await; + async fn test_file_not_found(&self, path: &Path) { + let not_found_file = self.fs.stat(path).await; assert!(not_found_file.is_err()); } - fn assert_file_stat(file_stat: &FileStat, path: &Path, kind: FileType, size: u64) { + fn assert_file_stat(&self, file_stat: &FileStat, path: &Path, kind: FileType, size: u64) { assert_eq!(file_stat.path, path); assert_eq!(file_stat.kind, kind); assert_eq!(file_stat.size, size); } } - pub(crate) struct TestRawFileSystem(); + pub(crate) struct TestRawFileSystem { + fs: F, + files: HashMap, + } - impl TestRawFileSystem { - pub(crate) async fn test_raw_file_system(fs: &impl RawFileSystem) { - let mut root_dir_child_file_stats = HashMap::new(); + impl TestRawFileSystem { + pub(crate) fn new(fs: F) -> Self { + Self { + fs, + files: HashMap::new(), + } + } + + pub(crate) async fn test_raw_file_system(&mut self) { + // Test root dir + self.test_root_dir().await; - // test root dir - let root_file_stat = fs.stat(ROOT_DIR_FILE_ID).await; + let parent_file_id = ROOT_DIR_FILE_ID; + // Test lookup file + let file_id = self + .test_lookup_file(parent_file_id, ".gvfs_meta".as_ref(), RegularFile, 0) + .await; + + // Test get file stat + self.test_stat_file(file_id, Path::new("/.gvfs_meta"), RegularFile, 0) + .await; + + // Test get file path + self.test_get_file_path(file_id, "/.gvfs_meta").await; + + // Test create file + self.test_create_file(parent_file_id, "file1.txt".as_ref()) + .await; + + // Test open file + let file_handle = self + .test_open_file(parent_file_id, "file1.txt".as_ref()) + .await; + + // Test write file + self.test_write_file(&file_handle, "test").await; + + // Test read file + self.test_read_file(&file_handle, "test").await; + + // Test close file + self.test_close_file(&file_handle).await; + + // Test create dir + self.test_create_dir(parent_file_id, "dir1".as_ref()).await; + + // Test list dir + self.test_list_dir(parent_file_id).await; + + // Test remove file + self.test_remove_file(parent_file_id, "file1.txt".as_ref()) + .await; + + // Test remove dir + self.test_remove_dir(parent_file_id, "dir1".as_ref()).await; + + // Test list dir again + self.test_list_dir(parent_file_id).await; + + // Test file not found + self.test_file_not_found(23).await; + } + + async fn test_root_dir(&self) { + let root_file_stat = self.fs.stat(ROOT_DIR_FILE_ID).await; assert!(root_file_stat.is_ok()); let root_file_stat = root_file_stat.unwrap(); - TestRawFileSystem::assert_file_stat( + self.assert_file_stat( &root_file_stat, Path::new(ROOT_DIR_PATH), - Directory, + FileType::Directory, 0, ); + } - // test lookup meta file - let meta_file_stat = fs.lookup(ROOT_DIR_FILE_ID, ".gvfs_meta".as_ref()).await; - assert!(meta_file_stat.is_ok()); - let meta_file_stat = meta_file_stat.unwrap(); - TestRawFileSystem::assert_file_stat( - &meta_file_stat, - Path::new("/.gvfs_meta"), - RegularFile, - 0, - ); + async fn test_lookup_file( + &mut self, + parent_file_id: u64, + expect_name: &OsStr, + expect_kind: FileType, + expect_size: u64, + ) -> u64 { + let file_stat = self.fs.lookup(parent_file_id, expect_name).await; + assert!(file_stat.is_ok()); + let file_stat = file_stat.unwrap(); + self.assert_file_stat(&file_stat, &file_stat.path, expect_kind, expect_size); + assert_eq!(file_stat.name, expect_name); + let file_id = file_stat.file_id; + self.files.insert(file_stat.file_id, file_stat); + file_id + } - //test get meta file path - let file_path = fs.get_file_path(meta_file_stat.file_id).await; + async fn test_get_file_path(&mut self, file_id: u64, expect_path: &str) { + let file_path = self.fs.get_file_path(file_id).await; assert!(file_path.is_ok()); - assert_eq!(file_path.unwrap(), meta_file_stat.path.to_string_lossy()); - - // test stat meta file - let meta_file_stat = fs.stat(meta_file_stat.file_id).await; - assert!(meta_file_stat.is_ok()); - let meta_file_stat = meta_file_stat.unwrap(); - TestRawFileSystem::assert_file_stat( - &meta_file_stat, - Path::new("/.gvfs_meta"), - RegularFile, - 0, - ); - root_dir_child_file_stats.insert(meta_file_stat.path.clone(), meta_file_stat); + assert_eq!(file_path.unwrap(), expect_path); + } - // test create file file1.txt - let file = fs - .create_file(ROOT_DIR_FILE_ID, "file1.txt".as_ref(), 0) - .await; + async fn test_stat_file( + &mut self, + file_id: u64, + expect_path: &Path, + expect_kind: FileType, + expect_size: u64, + ) { + let file_stat = self.fs.stat(file_id).await; + assert!(file_stat.is_ok()); + let file_stat = file_stat.unwrap(); + self.assert_file_stat(&file_stat, expect_path, expect_kind, expect_size); + self.files.insert(file_stat.file_id, file_stat); + } + + async fn test_create_file(&mut self, root_file_id: u64, name: &OsStr) { + let file = self.fs.create_file(root_file_id, name, 0).await; assert!(file.is_ok()); let file = file.unwrap(); assert!(file.handle_id > 0); assert!(file.file_id >= INITIAL_FILE_ID); + let file_stat = self.fs.stat(file.file_id).await; + assert!(file_stat.is_ok()); + + self.test_stat_file(file.file_id, &file_stat.unwrap().path, RegularFile, 0) + .await; + } - //test open file file1.txt - let file_handle = fs.open_file(file.file_id, 0).await; + async fn test_open_file(&self, root_file_id: u64, name: &OsStr) -> FileHandle { + let file = self.fs.lookup(root_file_id, name).await.unwrap(); + let file_handle = self.fs.open_file(file.file_id, 0).await; assert!(file_handle.is_ok()); let file_handle = file_handle.unwrap(); assert_eq!(file_handle.file_id, file.file_id); + file_handle + } - //test write file file1.txt - let write_size = fs - .write(file.file_id, file_handle.handle_id, 0, "test".as_bytes()) + async fn test_write_file(&mut self, file_handle: &FileHandle, content: &str) { + let write_size = self + .fs + .write( + file_handle.file_id, + file_handle.handle_id, + 0, + content.as_bytes(), + ) .await; assert!(write_size.is_ok()); - assert_eq!(write_size.unwrap(), 4); + assert_eq!(write_size.unwrap(), content.len() as u32); + + self.files.get_mut(&file_handle.file_id).unwrap().size = content.len() as u64; + } - // test read file file1.txt - let read_data = fs.read(file.file_id, file_handle.handle_id, 0, 4).await; + async fn test_read_file(&self, file_handle: &FileHandle, expected_content: &str) { + let read_data = self + .fs + .read( + file_handle.file_id, + file_handle.handle_id, + 0, + expected_content.len() as u32, + ) + .await; assert!(read_data.is_ok()); - assert_eq!(read_data.unwrap(), "test".as_bytes()); + assert_eq!(read_data.unwrap(), expected_content.as_bytes()); + } - // test close file file1.txt - let close_file = fs.close_file(file.file_id, file_handle.handle_id).await; + async fn test_close_file(&self, file_handle: &FileHandle) { + let close_file = self + .fs + .close_file(file_handle.file_id, file_handle.handle_id) + .await; assert!(close_file.is_ok()); + } - // test stat file file1.txt - let file_stat = fs.stat(file.file_id).await; - assert!(file_stat.is_ok()); - let file_stat = file_stat.unwrap(); - TestRawFileSystem::assert_file_stat( - &file_stat, - Path::new("/file1.txt"), - RegularFile, - write_size.unwrap() as u64, - ); - root_dir_child_file_stats.insert(file_stat.path.clone(), file_stat); - - // test create dir - let dir = fs.create_dir(ROOT_DIR_FILE_ID, "dir1".as_ref()).await; + async fn test_create_dir(&mut self, parent_file_id: u64, name: &OsStr) { + let dir = self.fs.create_dir(parent_file_id, name).await; assert!(dir.is_ok()); let dir_file_id = dir.unwrap(); assert!(dir_file_id >= INITIAL_FILE_ID); + let dir_stat = self.fs.stat(dir_file_id).await; + assert!(dir_stat.is_ok()); - let dir = fs.stat(dir_file_id).await; - assert!(dir.is_ok()); - let dir = dir.unwrap(); - TestRawFileSystem::assert_file_stat(&dir, Path::new("/dir1"), Directory, 0); - root_dir_child_file_stats.insert(dir.path.clone(), dir); + self.test_stat_file(dir_file_id, &dir_stat.unwrap().path, Directory, 0) + .await; + } - // test list dir - let list_dir = fs.read_dir(ROOT_DIR_FILE_ID).await; + async fn test_list_dir(&self, root_file_id: u64) { + let list_dir = self.fs.read_dir(root_file_id).await; assert!(list_dir.is_ok()); let list_dir = list_dir.unwrap(); - assert_eq!(list_dir.len(), root_dir_child_file_stats.len()); + assert_eq!(list_dir.len(), self.files.len()); for file_stat in list_dir { - assert!(root_dir_child_file_stats.contains_key(&file_stat.path)); - let actual_file_stat = root_dir_child_file_stats.get(&file_stat.path).unwrap(); - TestRawFileSystem::assert_file_stat( + assert!(self.files.contains_key(&file_stat.file_id)); + let actual_file_stat = self.files.get(&file_stat.file_id).unwrap(); + self.assert_file_stat( &file_stat, &actual_file_stat.path, actual_file_stat.kind, actual_file_stat.size, ); } + } - // test remove file - let remove_file = fs.remove_file(ROOT_DIR_FILE_ID, "file1.txt".as_ref()).await; + async fn test_remove_file(&mut self, root_file_id: u64, name: &OsStr) { + let file_stat = self.fs.lookup(root_file_id, name).await; + assert!(file_stat.is_ok()); + let file_stat = file_stat.unwrap(); + + let remove_file = self.fs.remove_file(root_file_id, name).await; assert!(remove_file.is_ok()); - root_dir_child_file_stats.remove(Path::new("/file1.txt")); + self.files.remove(&file_stat.file_id); + + self.test_file_not_found(file_stat.file_id).await; + } + + async fn test_remove_dir(&mut self, root_file_id: u64, name: &OsStr) { + let file_stat = self.fs.lookup(root_file_id, name).await; + assert!(file_stat.is_ok()); + let file_stat = file_stat.unwrap(); - // test remove dir - let remove_dir = fs.remove_dir(ROOT_DIR_FILE_ID, "dir1".as_ref()).await; + let remove_dir = self.fs.remove_dir(root_file_id, name).await; assert!(remove_dir.is_ok()); - root_dir_child_file_stats.remove(Path::new("/dir1")); + self.files.remove(&file_stat.file_id); - // test list dir - let list_dir = fs.read_dir(ROOT_DIR_FILE_ID).await; - assert!(list_dir.is_ok()); - let list_dir = list_dir.unwrap(); - assert_eq!(list_dir.len(), root_dir_child_file_stats.len()); - for file_stat in list_dir { - assert!(root_dir_child_file_stats.contains_key(&file_stat.path)); - let actual_file_stat = root_dir_child_file_stats.get(&file_stat.path).unwrap(); - TestRawFileSystem::assert_file_stat( - &file_stat, - &actual_file_stat.path, - actual_file_stat.kind, - actual_file_stat.size, - ); - } + self.test_file_not_found(file_stat.file_id).await; + } - // test file not found - let not_found_file = fs.stat(ROOT_DIR_FILE_ID + 1).await; + async fn test_file_not_found(&self, file_id: u64) { + let not_found_file = self.fs.stat(file_id).await; assert!(not_found_file.is_err()); } - fn assert_file_stat(file_stat: &FileStat, path: &Path, kind: FileType, size: u64) { + fn assert_file_stat(&self, file_stat: &FileStat, path: &Path, kind: FileType, size: u64) { assert_eq!(file_stat.path, path); assert_eq!(file_stat.kind, kind); assert_eq!(file_stat.size, size); diff --git a/clients/filesystem-fuse/src/memory_filesystem.rs b/clients/filesystem-fuse/src/memory_filesystem.rs index 3bc7915d60a..a7a6bf5b416 100644 --- a/clients/filesystem-fuse/src/memory_filesystem.rs +++ b/clients/filesystem-fuse/src/memory_filesystem.rs @@ -244,6 +244,7 @@ mod tests { async fn test_memory_file_system() { let fs = MemoryFileSystem::new().await; let _ = fs.init().await; - TestPathFileSystem::test_path_file_system(&fs).await; + let mut tester = TestPathFileSystem::new(fs); + tester.test_path_file_system().await; } } From 7bdd1663f550628ea8fbf9af0a2cb4ac2949ffb5 Mon Sep 17 00:00:00 2001 From: yuhui Date: Wed, 25 Dec 2024 14:29:36 +0800 Subject: [PATCH 47/47] Fix list_dir can list file in the subdir --- .../filesystem-fuse/src/memory_filesystem.rs | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/clients/filesystem-fuse/src/memory_filesystem.rs b/clients/filesystem-fuse/src/memory_filesystem.rs index a7a6bf5b416..ca3f13fd9a6 100644 --- a/clients/filesystem-fuse/src/memory_filesystem.rs +++ b/clients/filesystem-fuse/src/memory_filesystem.rs @@ -232,7 +232,11 @@ impl FileWriter for MemoryFileWriter { } fn path_in_dir(dir: &Path, path: &Path) -> bool { - path.starts_with(dir) && path != dir + if let Ok(relative_path) = path.strip_prefix(dir) { + relative_path.components().count() == 1 + } else { + false + } } #[cfg(test)] @@ -240,6 +244,33 @@ mod tests { use super::*; use crate::filesystem::tests::TestPathFileSystem; + #[test] + fn test_path_in_dir() { + let dir = Path::new("/parent"); + + let path1 = Path::new("/parent/child1"); + let path2 = Path::new("/parent/a.txt"); + let path3 = Path::new("/parent/child1/grandchild"); + let path4 = Path::new("/other"); + + assert!(!path_in_dir(dir, dir)); + assert!(path_in_dir(dir, path1)); + assert!(path_in_dir(dir, path2)); + assert!(!path_in_dir(dir, path3)); + assert!(!path_in_dir(dir, path4)); + + let dir = Path::new("/"); + + let path1 = Path::new("/child1"); + let path2 = Path::new("/a.txt"); + let path3 = Path::new("/child1/grandchild"); + + assert!(!path_in_dir(dir, dir)); + assert!(path_in_dir(dir, path1)); + assert!(path_in_dir(dir, path2)); + assert!(!path_in_dir(dir, path3)); + } + #[tokio::test] async fn test_memory_file_system() { let fs = MemoryFileSystem::new().await;