Sync wasi-filesystem with wit definitions in standards repo (#6791)

* filesystem.wit: upstream main + pch/blocking_is_a_stream_concern + pch/metadata_hash_fixes

upstream main changes the interface to be named `types`, and has
changes to descriptor-stat, directory-entry, adds the metadata hash
methods.

pch/blocking_is_a_stream_concern upstreams the deletion of non-blocking
from descriptor flags.

pch/metadata_hash_fixes adds missing this: descriptor arguments to
the metadata-hash and metadata-hash-at methods.

* copy in preopens.wit and world.wit from wasi-filesystem repo, and delete wasi-cli-base/preopens.wit

* other wits: fix names of filesystem/{types, preopens}

* fix bindgen invocations and import paths for filesystem changes

* component adapter: fix bindings paths for preopens and filesystem

* wip

* wippp

* fill in a reasonable-ish metadata hash impl

* preview 1 adapter: we need async in the dir entries, so collect the whole thing to a vec

this isnt great but i dont have the time or energy to do this better
right now

* component adapter: use metadata hash to fake ino

* wasi-tests: fix warnings

* reactor-tests: fix filesystem paths

* only create a BuildHasher impl once

* consistiency

* component adapter: use descriptor to call filesystem funcs, not the fd

* reactor test: fix filesystem types paths

* metadata hash: use DefaultHasher

* fix missed trappable-error-type merge conflicts

* preview1-in-preview2: dont use to_le in Filestat assignment of ino

wiggle takes care of endianness translation. The to_le in fd_readdir
entries is because that implementation is writing structs directly to
memory (very cursed)

s390x CI caught this bug

* debugging: forward stdio for fd_readdir test so we can figure out CI failure on s390x

prtest:full

* Don't convert filestat ino to little-endian, as it's not written to memory

fd_readdir will write its results to a buffer, and needs to be careful
with endianness as a result. However, fd_filestat_get returns the value
directly, and as such does not need to call `to_le`.

---------

Co-authored-by: Trevor Elliott <[email protected]>

crates/test-programs/reactor-tests/src/lib.rs

@@ -31,7 +31,7 @@ impl TestReactor for T {
             Ok(())
         }
     }
-    fn pass_an_imported_record(stat: wasi::filesystem::filesystem::DescriptorStat) -> String {
+    fn pass_an_imported_record(stat: wasi::filesystem::types::DescriptorStat) -> String {
         format!("{stat:?}")
     }
 }

crates/test-programs/tests/reactor.rs

@@ -4,7 +4,7 @@ use wasmtime::{
     Config, Engine, Store,
 };
 use wasmtime_wasi::preview2::bindings::clocks::wall_clock;
-use wasmtime_wasi::preview2::bindings::filesystem::filesystem;
+use wasmtime_wasi::preview2::bindings::filesystem::types as filesystem;
 use wasmtime_wasi::preview2::{self, Table, WasiCtx, WasiCtxBuilder, WasiView};
 
 lazy_static::lazy_static! {
@@ -28,9 +28,9 @@ wasmtime::component::bindgen!({
     async: true,
     with: {
        "wasi:io/streams": preview2::bindings::io::streams,
-       "wasi:filesystem/filesystem": preview2::bindings::filesystem::filesystem,
+       "wasi:filesystem/types": preview2::bindings::filesystem::types,
+       "wasi:filesystem/preopens": preview2::bindings::filesystem::preopens,
        "wasi:cli-base/environment": preview2::bindings::cli_base::environment,
-       "wasi:cli-base/preopens": preview2::bindings::cli_base::preopens,
        "wasi:cli-base/exit": preview2::bindings::cli_base::exit,
        "wasi:cli-base/stdin": preview2::bindings::cli_base::stdin,
        "wasi:cli-base/stdout": preview2::bindings::cli_base::stdout,
@@ -68,10 +68,10 @@ async fn instantiate(
     let mut linker = Linker::new(&ENGINE);
 
     // All of the imports available to the world are provided by the wasi-common crate:
-    preview2::bindings::filesystem::filesystem::add_to_linker(&mut linker, |x| x)?;
+    preview2::bindings::filesystem::types::add_to_linker(&mut linker, |x| x)?;
+    preview2::bindings::filesystem::preopens::add_to_linker(&mut linker, |x| x)?;
     preview2::bindings::io::streams::add_to_linker(&mut linker, |x| x)?;
     preview2::bindings::cli_base::environment::add_to_linker(&mut linker, |x| x)?;
-    preview2::bindings::cli_base::preopens::add_to_linker(&mut linker, |x| x)?;
     preview2::bindings::cli_base::exit::add_to_linker(&mut linker, |x| x)?;
     preview2::bindings::cli_base::stdin::add_to_linker(&mut linker, |x| x)?;
     preview2::bindings::cli_base::stdout::add_to_linker(&mut linker, |x| x)?;
@@ -129,8 +129,6 @@ async fn reactor_tests() -> Result<()> {
             nanoseconds: 789,
             seconds: 10,
         },
-        device: 0,
-        inode: 0,
         link_count: 0,
         size: 0,
         status_change_timestamp: wall_clock::Datetime {

crates/test-programs/tests/wasi-preview1-host-in-preview2.rs

@@ -165,7 +165,7 @@ async fn fd_flags_set() {
 }
 #[test_log::test(tokio::test(flavor = "multi_thread"))]
 async fn fd_readdir() {
-    run("fd_readdir", false).await.unwrap()
+    run("fd_readdir", true).await.unwrap()
 }
 #[test_log::test(tokio::test(flavor = "multi_thread"))]
 async fn file_allocate() {

crates/test-programs/wasi-tests/src/bin/path_open_read_write.rs

@@ -38,7 +38,6 @@ unsafe fn test_path_open_read_write(dir_fd: wasi::Fd) {
     );
 
     wasi::fd_close(f_readonly).expect("close readonly");
-    drop(f_readonly);
 
     // =============== WRITE ONLY ==================
     let f_writeonly = wasi::path_open(dir_fd, 0, "file", 0, wasi::RIGHTS_FD_WRITE, 0, 0)
@@ -64,7 +63,6 @@ unsafe fn test_path_open_read_write(dir_fd: wasi::Fd) {
     assert_eq!(bytes_written, write_buffer.len());
 
     wasi::fd_close(f_writeonly).expect("close writeonly");
-    drop(f_writeonly);
 
     // ============== READ WRITE =======================
 
@@ -111,7 +109,6 @@ unsafe fn test_path_open_read_write(dir_fd: wasi::Fd) {
     );
 
     wasi::fd_close(f_readwrite).expect("close readwrite");
-    drop(f_readwrite);
 
     wasi::path_unlink_file(dir_fd, "file").expect("removing a file");
 }

crates/wasi-preview1-component-adapter/src/descriptors.rs

@@ -1,5 +1,5 @@
 use crate::bindings::wasi::cli_base::{stderr, stdin, stdout};
-use crate::bindings::wasi::filesystem::filesystem;
+use crate::bindings::wasi::filesystem::types as filesystem;
 use crate::bindings::wasi::io::streams::{self, InputStream, OutputStream};
 use crate::bindings::wasi::sockets::tcp;
 use crate::{set_stderr_stream, BumpArena, File, ImportAlloc, TrappingUnwrap, WasmStr};
@@ -166,7 +166,7 @@ impl Descriptors {
         }))
         .trapping_unwrap();
 
-        #[link(wasm_import_module = "wasi:cli-base/preopens")]
+        #[link(wasm_import_module = "wasi:filesystem/preopens")]
         extern "C" {
             #[link_name = "get-directories"]
             fn get_preopens_import(rval: *mut PreopenList);

crates/wasi-preview1-component-adapter/src/lib.rs

@@ -2,7 +2,7 @@
 
 use crate::bindings::wasi::cli_base::exit;
 use crate::bindings::wasi::clocks::{monotonic_clock, wall_clock};
-use crate::bindings::wasi::filesystem::filesystem;
+use crate::bindings::wasi::filesystem::types as filesystem;
 use crate::bindings::wasi::io::streams;
 use crate::bindings::wasi::poll::poll;
 use crate::bindings::wasi::random::random;
@@ -641,10 +641,11 @@ pub unsafe extern "C" fn fd_filestat_get(fd: Fd, buf: *mut Filestat) -> Errno {
                 ..
             }) => {
                 let stat = filesystem::stat(file.fd)?;
+                let metadata_hash = filesystem::metadata_hash(file.fd)?;
                 let filetype = stat.type_.into();
                 *buf = Filestat {
-                    dev: stat.device,
-                    ino: stat.inode,
+                    dev: 1,
+                    ino: metadata_hash.lower,
                     filetype,
                     nlink: stat.link_count,
                     size: stat.size,
@@ -959,8 +960,6 @@ pub unsafe extern "C" fn fd_readdir(
         // for it.
         let ds = state.descriptors();
         let dir = ds.get_dir(fd)?;
-        let stat = filesystem::stat(dir.fd)?;
-        let dot_inode = stat.inode;
 
         let mut iter;
         match stream {
@@ -974,7 +973,7 @@ pub unsafe extern "C" fn fd_readdir(
                     state,
                     cookie,
                     use_cache: true,
-                    dot_inode,
+                    dir_descriptor: dir.fd,
                 }
             }
 
@@ -989,7 +988,7 @@ pub unsafe extern "C" fn fd_readdir(
                     cookie: wasi::DIRCOOKIE_START,
                     use_cache: false,
                     stream: DirectoryEntryStream(filesystem::read_directory(dir.fd)?),
-                    dot_inode,
+                    dir_descriptor: dir.fd,
                 };
 
                 // Skip to the entry that is requested by the `cookie`
@@ -1068,7 +1067,7 @@ pub unsafe extern "C" fn fd_readdir(
         use_cache: bool,
         cookie: Dircookie,
         stream: DirectoryEntryStream,
-        dot_inode: wasi::Inode,
+        dir_descriptor: filesystem::Descriptor,
     }
 
     impl<'a> Iterator for DirectoryEntryIterator<'a> {
@@ -1085,9 +1084,13 @@ pub unsafe extern "C" fn fd_readdir(
             // Preview2 excludes them, so re-add them.
             match current_cookie {
                 0 => {
+                    let metadata_hash = match filesystem::metadata_hash(self.dir_descriptor) {
+                        Ok(h) => h,
+                        Err(e) => return Some(Err(e.into())),
+                    };
                     let dirent = wasi::Dirent {
                         d_next: self.cookie,
-                        d_ino: self.dot_inode,
+                        d_ino: metadata_hash.lower,
                         d_type: wasi::FILETYPE_DIRECTORY,
                         d_namlen: 1,
                     };
@@ -1127,11 +1130,18 @@ pub unsafe extern "C" fn fd_readdir(
                 Err(e) => return Some(Err(e.into())),
             };
 
-            let filesystem::DirectoryEntry { inode, type_, name } = entry;
+            let filesystem::DirectoryEntry { type_, name } = entry;
+            let d_ino = filesystem::metadata_hash_at(
+                self.dir_descriptor,
+                filesystem::PathFlags::empty(),
+                &name,
+            )
+            .map(|h| h.lower)
+            .unwrap_or(0);
             let name = ManuallyDrop::new(name);
             let dirent = wasi::Dirent {
                 d_next: self.cookie,
-                d_ino: inode.unwrap_or(0),
+                d_ino,
                 d_namlen: u32::try_from(name.len()).trapping_unwrap(),
                 d_type: type_.into(),
             };
@@ -1330,10 +1340,11 @@ pub unsafe extern "C" fn path_filestat_get(
         let ds = state.descriptors();
         let file = ds.get_dir(fd)?;
         let stat = filesystem::stat_at(file.fd, at_flags, path)?;
+        let metadata_hash = filesystem::metadata_hash_at(file.fd, at_flags, path)?;
         let filetype = stat.type_.into();
         *buf = Filestat {
-            dev: stat.device,
-            ino: stat.inode,
+            dev: 1,
+            ino: metadata_hash.lower,
             filetype,
             nlink: stat.link_count,
             size: stat.size,

crates/wasi/src/preview2/command.rs

@@ -5,11 +5,12 @@ wasmtime::component::bindgen!({
     tracing: true,
     async: true,
     trappable_error_type: {
-        "wasi:filesystem/filesystem"::"error-code": Error,
+        "wasi:filesystem/types"::"error-code": Error,
         "wasi:io/streams"::"stream-error": Error,
     },
     with: {
-       "wasi:filesystem/filesystem": crate::preview2::bindings::filesystem::filesystem,
+       "wasi:filesystem/types": crate::preview2::bindings::filesystem::types,
+       "wasi:filesystem/preopens": crate::preview2::bindings::filesystem::preopens,
        "wasi:clocks/monotonic_clock": crate::preview2::bindings::clocks::monotonic_clock,
        "wasi:poll/poll": crate::preview2::bindings::poll::poll,
        "wasi:io/streams": crate::preview2::bindings::io::streams,
@@ -18,7 +19,6 @@ wasmtime::component::bindgen!({
        "wasi:random/random": crate::preview2::bindings::random::random,
        "wasi:cli_base/environment": crate::preview2::bindings::cli_base::environment,
        "wasi:cli_base/exit": crate::preview2::bindings::cli_base::exit,
-       "wasi:cli_base/preopens": crate::preview2::bindings::cli_base::preopens,
        "wasi:cli_base/stdin": crate::preview2::bindings::cli_base::stdin,
        "wasi:cli_base/stdout": crate::preview2::bindings::cli_base::stdout,
        "wasi:cli_base/stderr": crate::preview2::bindings::cli_base::stderr,
@@ -29,13 +29,13 @@ pub fn add_to_linker<T: WasiView>(l: &mut wasmtime::component::Linker<T>) -> any
     crate::preview2::bindings::clocks::wall_clock::add_to_linker(l, |t| t)?;
     crate::preview2::bindings::clocks::monotonic_clock::add_to_linker(l, |t| t)?;
     crate::preview2::bindings::clocks::timezone::add_to_linker(l, |t| t)?;
-    crate::preview2::bindings::filesystem::filesystem::add_to_linker(l, |t| t)?;
+    crate::preview2::bindings::filesystem::types::add_to_linker(l, |t| t)?;
+    crate::preview2::bindings::filesystem::preopens::add_to_linker(l, |t| t)?;
     crate::preview2::bindings::poll::poll::add_to_linker(l, |t| t)?;
     crate::preview2::bindings::io::streams::add_to_linker(l, |t| t)?;
     crate::preview2::bindings::random::random::add_to_linker(l, |t| t)?;
     crate::preview2::bindings::cli_base::exit::add_to_linker(l, |t| t)?;
     crate::preview2::bindings::cli_base::environment::add_to_linker(l, |t| t)?;
-    crate::preview2::bindings::cli_base::preopens::add_to_linker(l, |t| t)?;
     crate::preview2::bindings::cli_base::stdin::add_to_linker(l, |t| t)?;
     crate::preview2::bindings::cli_base::stdout::add_to_linker(l, |t| t)?;
     crate::preview2::bindings::cli_base::stderr::add_to_linker(l, |t| t)?;
@@ -50,11 +50,12 @@ pub mod sync {
         tracing: true,
         async: false,
         trappable_error_type: {
-            "wasi:filesystem/filesystem"::"error-code": Error,
+            "wasi:filesystem/types"::"error-code": Error,
             "wasi:io/streams"::"stream-error": Error,
         },
         with: {
-           "wasi:filesystem/filesystem": crate::preview2::bindings::sync_io::filesystem::filesystem,
+           "wasi:filesystem/types": crate::preview2::bindings::sync_io::filesystem::types,
+           "wasi:filesystem/preopens": crate::preview2::bindings::filesystem::preopens,
            "wasi:clocks/monotonic_clock": crate::preview2::bindings::clocks::monotonic_clock,
            "wasi:poll/poll": crate::preview2::bindings::sync_io::poll::poll,
            "wasi:io/streams": crate::preview2::bindings::sync_io::io::streams,
@@ -63,7 +64,6 @@ pub mod sync {
            "wasi:random/random": crate::preview2::bindings::random::random,
            "wasi:cli_base/environment": crate::preview2::bindings::cli_base::environment,
            "wasi:cli_base/exit": crate::preview2::bindings::cli_base::exit,
-           "wasi:cli_base/preopens": crate::preview2::bindings::cli_base::preopens,
            "wasi:cli_base/stdin": crate::preview2::bindings::cli_base::stdin,
            "wasi:cli_base/stdout": crate::preview2::bindings::cli_base::stdout,
            "wasi:cli_base/stderr": crate::preview2::bindings::cli_base::stderr,
@@ -76,13 +76,13 @@ pub mod sync {
         crate::preview2::bindings::clocks::wall_clock::add_to_linker(l, |t| t)?;
         crate::preview2::bindings::clocks::monotonic_clock::add_to_linker(l, |t| t)?;
         crate::preview2::bindings::clocks::timezone::add_to_linker(l, |t| t)?;
-        crate::preview2::bindings::sync_io::filesystem::filesystem::add_to_linker(l, |t| t)?;
+        crate::preview2::bindings::sync_io::filesystem::types::add_to_linker(l, |t| t)?;
+        crate::preview2::bindings::filesystem::preopens::add_to_linker(l, |t| t)?;
         crate::preview2::bindings::sync_io::poll::poll::add_to_linker(l, |t| t)?;
         crate::preview2::bindings::sync_io::io::streams::add_to_linker(l, |t| t)?;
         crate::preview2::bindings::random::random::add_to_linker(l, |t| t)?;
         crate::preview2::bindings::cli_base::exit::add_to_linker(l, |t| t)?;
         crate::preview2::bindings::cli_base::environment::add_to_linker(l, |t| t)?;
-        crate::preview2::bindings::cli_base::preopens::add_to_linker(l, |t| t)?;
         crate::preview2::bindings::cli_base::stdin::add_to_linker(l, |t| t)?;
         crate::preview2::bindings::cli_base::stdout::add_to_linker(l, |t| t)?;
         crate::preview2::bindings::cli_base::stderr::add_to_linker(l, |t| t)?;

crates/wasi/src/preview2/mod.rs

@@ -49,12 +49,12 @@ pub mod bindings {
                 interfaces: "
               import wasi:poll/poll
               import wasi:io/streams
-              import wasi:filesystem/filesystem
+              import wasi:filesystem/types
             ",
                 tracing: true,
                 trappable_error_type: {
                     "wasi:io/streams"::"stream-error": Error,
-                    "wasi:filesystem/filesystem"::"error-code": Error,
+                    "wasi:filesystem/types"::"error-code": Error,
                 },
                 with: {
                     "wasi:clocks/wall-clock": crate::preview2::bindings::clocks::wall_clock,
@@ -99,53 +99,54 @@ pub mod bindings {
             interfaces: "
               import wasi:poll/poll
               import wasi:io/streams
-              import wasi:filesystem/filesystem
+              import wasi:filesystem/types
             ",
             tracing: true,
             async: true,
             trappable_error_type: {
                 "wasi:io/streams"::"stream-error": Error,
-                "wasi:filesystem/filesystem"::"error-code": Error,
+                "wasi:filesystem/types"::"error-code": Error,
             },
             with: {
                 "wasi:clocks/wall-clock": crate::preview2::bindings::clocks::wall_clock,
             }
         });
     }
-    pub use self::_internal_io::wasi::{filesystem, io, poll};
+    pub use self::_internal_io::wasi::{io, poll};
 
     pub(crate) mod _internal_rest {
         wasmtime::component::bindgen!({
         path: "wit",
         interfaces: "
-              import wasi:clocks/wall-clock
-              import wasi:clocks/monotonic-clock
-              import wasi:clocks/timezone
+              import wasi:filesystem/preopens
               import wasi:random/random
               import wasi:random/insecure
               import wasi:random/insecure-seed
               import wasi:cli-base/environment
-              import wasi:cli-base/preopens
               import wasi:cli-base/exit
               import wasi:cli-base/stdin
               import wasi:cli-base/stdout
               import wasi:cli-base/stderr
             ",
         tracing: true,
         trappable_error_type: {
-            "wasi:filesystem/filesystem"::"error-code": Error,
+            "wasi:filesystem/types"::"error-code": Error,
             "wasi:io/streams"::"stream-error": Error,
         },
         with: {
             "wasi:clocks/wall-clock": crate::preview2::bindings::clocks::wall_clock,
             "wasi:poll/poll": crate::preview2::bindings::poll::poll,
             "wasi:io/streams": crate::preview2::bindings::io::streams,
-            "wasi:filesystem/filesystem": crate::preview2::bindings::filesystem::filesystem
+            "wasi:filesystem/types": crate::preview2::bindings::filesystem::types,
         }
         });
     }
 
-    pub use self::_internal_rest::wasi::*;
+    pub use self::_internal_rest::wasi::{cli_base, random};
+    pub mod filesystem {
+        pub use super::_internal_io::wasi::filesystem::types;
+        pub use super::_internal_rest::wasi::filesystem::preopens;
+    }
 }
 
 pub(crate) static RUNTIME: once_cell::sync::Lazy<tokio::runtime::Runtime> =