feat: add intermediate file system (#8631)

crates/node_binding/binding.d.ts

@@ -211,7 +211,7 @@ export declare class RawExternalItemFnCtx {
 }
 
 export declare class Rspack {
-  constructor(options: RawOptions, builtinPlugins: Array<BuiltinPlugin>, registerJsTaps: RegisterJsTaps, outputFilesystem: ThreadsafeNodeFS, resolverFactoryReference: JsResolverFactory)
+  constructor(options: RawOptions, builtinPlugins: Array<BuiltinPlugin>, registerJsTaps: RegisterJsTaps, outputFilesystem: ThreadsafeNodeFS, intermediateFilesystem: ThreadsafeNodeFS, resolverFactoryReference: JsResolverFactory)
   setNonSkippableRegisters(kinds: Array<RegisterJsTapKind>): void
   /** Build with the given option passed to the constructor */
   build(callback: (err: null | Error) => void): void
@@ -2136,4 +2136,12 @@ export interface ThreadsafeNodeFS {
   readFile: (name: string) => Promise<Buffer | string | void> | Buffer | string | void
   stat: (name: string) => Promise<NodeFsStats | void> | NodeFsStats | void
   lstat: (name: string) => Promise<NodeFsStats | void> | NodeFsStats | void
+  open: (name: string, flags: string) => Promise<number | void> | number | void
+  rename: (from: string, to: string) => Promise<void> | void
+  close: (fd: number) => Promise<void> | void
+  write: (fd: number, content: Buffer, position: number) => Promise<number | void> | number | void
+  writeAll: (fd: number, content: Buffer) => Promise<number | void> | number | void
+  read: (fd: number, length: number, position: number) => Promise<Buffer | void> | Buffer | void
+  readUntil: (fd: number, code: number, position: number) => Promise<Buffer | void> | Buffer | void
+  readToEnd: (fd: number, position: number) => Promise<Buffer | void> | Buffer | void
 }

crates/node_binding/src/lib.rs

@@ -44,6 +44,7 @@ impl Rspack {
     builtin_plugins: Vec<BuiltinPlugin>,
     register_js_taps: RegisterJsTaps,
     output_filesystem: ThreadsafeNodeFS,
+    intermediate_filesystem: ThreadsafeNodeFS,
     mut resolver_factory_reference: Reference<JsResolverFactory>,
   ) -> Result<Self> {
     tracing::info!("raw_options: {:#?}", &options);
@@ -73,6 +74,11 @@ impl Rspack {
       Some(Box::new(NodeFileSystem::new(output_filesystem).map_err(
         |e| Error::from_reason(format!("Failed to create writable filesystem: {e}",)),
       )?)),
+      Some(Box::new(
+        NodeFileSystem::new(intermediate_filesystem).map_err(|e| {
+          Error::from_reason(format!("Failed to create intermediate filesystem: {e}",))
+        })?,
+      )),
       None,
       Some(resolver_factory),
       Some(loader_resolver_factory),

crates/rspack_core/src/compiler/mod.rs

@@ -54,6 +54,7 @@ pub struct CompilerHooks {
 pub struct Compiler {
   pub options: Arc<CompilerOptions>,
   pub output_filesystem: Box<dyn WritableFileSystem>,
+  pub intermediate_filesystem: Box<dyn WritableFileSystem>,
   pub input_filesystem: Arc<dyn FileSystem>,
   pub compilation: Compilation,
   pub plugin_driver: SharedPluginDriver,
@@ -69,11 +70,13 @@ pub struct Compiler {
 
 impl Compiler {
   #[instrument(skip_all)]
+  #[allow(clippy::too_many_arguments)]
   pub fn new(
     options: CompilerOptions,
     plugins: Vec<BoxPlugin>,
     buildtime_plugins: Vec<BoxPlugin>,
     output_filesystem: Option<Box<dyn WritableFileSystem>>,
+    intermediate_filesystem: Option<Box<dyn WritableFileSystem>>,
     // only supports passing input_filesystem in rust api, no support for js api
     input_filesystem: Option<Arc<dyn FileSystem + Send + Sync>>,
     // no need to pass resolve_factory in rust api
@@ -109,6 +112,8 @@ impl Compiler {
     let old_cache = Arc::new(OldCache::new(options.clone()));
     let module_executor = ModuleExecutor::default();
     let output_filesystem = output_filesystem.unwrap_or_else(|| Box::new(NativeFileSystem {}));
+    let intermediate_filesystem =
+      intermediate_filesystem.unwrap_or_else(|| Box::new(NativeFileSystem {}));
 
     Self {
       options: options.clone(),
@@ -127,6 +132,7 @@ impl Compiler {
         input_filesystem.clone(),
       ),
       output_filesystem,
+      intermediate_filesystem,
       plugin_driver,
       buildtime_plugin_driver,
       resolver_factory,

crates/rspack_fs/src/intermediate.rs

@@ -0,0 +1,29 @@
+use std::fmt::Debug;
+
+use rspack_paths::Utf8Path;
+
+use super::Result;
+
+#[async_trait::async_trait]
+pub trait IntermediateFileSystemExtras: Debug + Send + Sync {
+  async fn rename(&self, from: &Utf8Path, to: &Utf8Path) -> Result<()>;
+  async fn create_read_stream(&self, file: &Utf8Path) -> Result<Box<dyn ReadStream>>;
+  async fn create_write_stream(&self, file: &Utf8Path) -> Result<Box<dyn WriteStream>>;
+}
+
+#[async_trait::async_trait]
+pub trait ReadStream: Debug + Sync + Send {
+  async fn read(&mut self, buf: &mut [u8]) -> Result<()>;
+  async fn read_until(&mut self, byte: u8, buf: &mut Vec<u8>) -> Result<usize>;
+  async fn read_to_end(&mut self, buf: &mut Vec<u8>) -> Result<usize>;
+  async fn skip(&mut self, offset: usize) -> Result<()>;
+  async fn close(&mut self) -> Result<()>;
+}
+
+#[async_trait::async_trait]
+pub trait WriteStream: Debug + Sync + Send {
+  async fn write(&mut self, buf: &[u8]) -> Result<usize>;
+  async fn write_all(&mut self, buf: &[u8]) -> Result<()>;
+  async fn flush(&mut self) -> Result<()>;
+  async fn close(&mut self) -> Result<()>;
+}

crates/rspack_fs/src/lib.rs

@@ -5,17 +5,19 @@ pub use read::ReadableFileSystem;
 
 mod write;
 pub use write::WritableFileSystem;
+mod intermediate;
+pub use intermediate::{IntermediateFileSystemExtras, ReadStream, WriteStream};
 
 mod file_metadata;
 pub use file_metadata::FileMetadata;
 
 mod macros;
 
 mod native_fs;
-pub use native_fs::NativeFileSystem;
+pub use native_fs::{NativeFileSystem, NativeReadStream, NativeWriteStream};
 
 mod memory_fs;
-pub use memory_fs::MemoryFileSystem;
+pub use memory_fs::{MemoryFileSystem, MemoryReadStream, MemoryWriteStream};
 
 mod error;
 pub use error::{Error, Result};

crates/rspack_fs/src/memory_fs.rs

@@ -1,13 +1,17 @@
 use std::{
   collections::{HashMap, HashSet},
-  sync::Mutex,
+  io::{BufRead, Cursor, Read, Seek},
+  sync::{Arc, Mutex},
   time::{SystemTime, UNIX_EPOCH},
 };
 
 use futures::future::BoxFuture;
 use rspack_paths::{AssertUtf8, Utf8Path, Utf8PathBuf};
 
-use crate::{Error, FileMetadata, FileSystem, ReadableFileSystem, Result, WritableFileSystem};
+use crate::{
+  Error, FileMetadata, FileSystem, IntermediateFileSystemExtras, ReadStream, ReadableFileSystem,
+  Result, WritableFileSystem, WriteStream,
+};
 
 fn current_time() -> u64 {
   SystemTime::now()
@@ -67,9 +71,9 @@ impl FileType {
   }
 }
 
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Clone)]
 pub struct MemoryFileSystem {
-  files: Mutex<HashMap<Utf8PathBuf, FileType>>,
+  files: Arc<Mutex<HashMap<Utf8PathBuf, FileType>>>,
 }
 impl FileSystem for MemoryFileSystem {}
 
@@ -135,6 +139,17 @@ impl MemoryFileSystem {
     }
     Ok(res.into_iter().collect())
   }
+
+  fn _rename_file(&self, from: &Utf8Path, to: &Utf8Path) -> Result<()> {
+    if !self.contains_file(from)? {
+      return Err(new_error("from dir not exist"));
+    }
+    let mut files = self.files.lock().expect("should get lock");
+    let file = files.remove(from).expect("should have file");
+    files.insert(to.into(), file);
+
+    Ok(())
+  }
 }
 #[async_trait::async_trait]
 impl WritableFileSystem for MemoryFileSystem {
@@ -253,6 +268,90 @@ impl ReadableFileSystem for MemoryFileSystem {
   }
 }
 
+#[async_trait::async_trait]
+impl IntermediateFileSystemExtras for MemoryFileSystem {
+  async fn rename(&self, from: &Utf8Path, to: &Utf8Path) -> Result<()> {
+    self._rename_file(from, to)?;
+    Ok(())
+  }
+
+  async fn create_read_stream(&self, file: &Utf8Path) -> Result<Box<dyn ReadStream>> {
+    let contents = self.read(file)?;
+    let reader = MemoryReadStream::new(contents);
+    Ok(Box::new(reader))
+  }
+
+  async fn create_write_stream(&self, file: &Utf8Path) -> Result<Box<dyn WriteStream>> {
+    let writer = MemoryWriteStream::new(file, self.clone());
+    Ok(Box::new(writer))
+  }
+}
+
+#[derive(Debug)]
+pub struct MemoryReadStream(Cursor<Vec<u8>>);
+
+impl MemoryReadStream {
+  pub fn new(contents: Vec<u8>) -> Self {
+    Self(Cursor::new(contents))
+  }
+}
+
+#[async_trait::async_trait]
+impl ReadStream for MemoryReadStream {
+  async fn read(&mut self, buf: &mut [u8]) -> Result<()> {
+    self.0.read_exact(buf).map_err(Error::from)
+  }
+
+  async fn read_until(&mut self, byte: u8, buf: &mut Vec<u8>) -> Result<usize> {
+    self.0.read_until(byte, buf).map_err(Error::from)
+  }
+  async fn read_to_end(&mut self, buf: &mut Vec<u8>) -> Result<usize> {
+    self.0.read_to_end(buf).map_err(Error::from)
+  }
+  async fn skip(&mut self, offset: usize) -> Result<()> {
+    self.0.seek_relative(offset as i64).map_err(Error::from)
+  }
+  async fn close(&mut self) -> Result<()> {
+    Ok(())
+  }
+}
+
+#[derive(Debug)]
+pub struct MemoryWriteStream {
+  file: Utf8PathBuf,
+  contents: Vec<u8>,
+  fs: MemoryFileSystem,
+}
+
+impl MemoryWriteStream {
+  pub fn new(file: &Utf8Path, fs: MemoryFileSystem) -> Self {
+    Self {
+      file: file.to_path_buf(),
+      contents: vec![],
+      fs,
+    }
+  }
+}
+
+#[async_trait::async_trait]
+impl WriteStream for MemoryWriteStream {
+  async fn write(&mut self, buf: &[u8]) -> Result<usize> {
+    self.contents.extend(buf);
+    Ok(buf.len())
+  }
+  async fn write_all(&mut self, buf: &[u8]) -> Result<()> {
+    self.contents = buf.to_vec();
+    Ok(())
+  }
+  async fn flush(&mut self) -> Result<()> {
+    self.fs.write(&self.file, &self.contents).await?;
+    Ok(())
+  }
+  async fn close(&mut self) -> Result<()> {
+    Ok(())
+  }
+}
+
 #[cfg(test)]
 mod tests {
   use rspack_paths::Utf8Path;

crates/rspack_fs/src/native_fs.rs

@@ -1,9 +1,15 @@
-use std::fs;
+use std::{
+  fs::{self, File},
+  io::{BufRead, BufReader, BufWriter, Read, Write},
+};
 
 use futures::future::BoxFuture;
 use rspack_paths::{AssertUtf8, Utf8Path, Utf8PathBuf};
 
-use crate::{Error, FileMetadata, FileSystem, ReadableFileSystem, Result, WritableFileSystem};
+use crate::{
+  Error, FileMetadata, FileSystem, IntermediateFileSystemExtras, ReadStream, ReadableFileSystem,
+  Result, WritableFileSystem, WriteStream,
+};
 
 #[derive(Debug)]
 pub struct NativeFileSystem;
@@ -85,3 +91,76 @@ impl ReadableFileSystem for NativeFileSystem {
     Box::pin(fut)
   }
 }
+
+#[async_trait::async_trait]
+impl IntermediateFileSystemExtras for NativeFileSystem {
+  async fn rename(&self, from: &Utf8Path, to: &Utf8Path) -> Result<()> {
+    fs::rename(from, to).map_err(Error::from)
+  }
+
+  async fn create_read_stream(&self, file: &Utf8Path) -> Result<Box<dyn ReadStream>> {
+    let reader = NativeReadStream::try_new(file)?;
+    Ok(Box::new(reader))
+  }
+
+  async fn create_write_stream(&self, file: &Utf8Path) -> Result<Box<dyn WriteStream>> {
+    let writer = NativeWriteStream::try_new(file)?;
+    Ok(Box::new(writer))
+  }
+}
+
+#[derive(Debug)]
+pub struct NativeReadStream(BufReader<File>);
+
+impl NativeReadStream {
+  pub fn try_new(file: &Utf8Path) -> Result<Self> {
+    let file = File::open(file).map_err(Error::from)?;
+    Ok(Self(BufReader::new(file)))
+  }
+}
+
+#[async_trait::async_trait]
+impl ReadStream for NativeReadStream {
+  async fn read(&mut self, buf: &mut [u8]) -> Result<()> {
+    self.0.read_exact(buf).map_err(Error::from)
+  }
+
+  async fn read_until(&mut self, byte: u8, buf: &mut Vec<u8>) -> Result<usize> {
+    self.0.read_until(byte, buf).map_err(Error::from)
+  }
+  async fn read_to_end(&mut self, buf: &mut Vec<u8>) -> Result<usize> {
+    self.0.read_to_end(buf).map_err(Error::from)
+  }
+  async fn skip(&mut self, offset: usize) -> Result<()> {
+    self.0.seek_relative(offset as i64).map_err(Error::from)
+  }
+  async fn close(&mut self) -> Result<()> {
+    Ok(())
+  }
+}
+
+#[derive(Debug)]
+pub struct NativeWriteStream(BufWriter<File>);
+
+impl NativeWriteStream {
+  pub fn try_new(file: &Utf8Path) -> Result<Self> {
+    let file = File::open(file).map_err(Error::from)?;
+    Ok(Self(BufWriter::new(file)))
+  }
+}
+
+#[async_trait::async_trait]
+impl WriteStream for NativeWriteStream {
+  async fn write(&mut self, buf: &[u8]) -> Result<usize> {
+    self.0.write(buf).map_err(Error::from)
+  }
+  async fn write_all(&mut self, buf: &[u8]) -> Result<()> {
+    self.0.write_all(buf).map_err(Error::from)
+  }
+  async fn flush(&mut self) -> Result<()> {
+    self.0.flush().map_err(Error::from)
+  }
+  async fn close(&mut self) -> Result<()> {
+    Ok(())
+  }
+}