Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sync wasi-filesystem with wit definitions in standards repo #6791

Merged
merged 24 commits into from
Aug 8, 2023
Merged

Sync wasi-filesystem with wit definitions in standards repo #6791

Show file tree
Hide file tree
Changes from 17 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
8665160
filesystem.wit: upstream main + pch/blocking_is_a_stream_concern + pc…
Jul 31, 2023
ebdd350
copy in preopens.wit and world.wit from wasi-filesystem repo, and del…
Aug 1, 2023
707e3fc
other wits: fix names of filesystem/{types, preopens}
Aug 1, 2023
5b77f4d
fix bindgen invocations and import paths for filesystem changes
Aug 1, 2023
db023d4
component adapter: fix bindings paths for preopens and filesystem
Aug 1, 2023
bd92000
wip
Aug 1, 2023
b136f7f
wippp
Aug 1, 2023
b7aff46
fill in a reasonable-ish metadata hash impl
Aug 1, 2023
0c4af0c
preview 1 adapter: we need async in the dir entries, so collect the w…
Aug 1, 2023
ee811c0
component adapter: use metadata hash to fake ino
Aug 2, 2023
8a179a9
wasi-tests: fix warnings
Aug 2, 2023
d66f545
reactor-tests: fix filesystem paths
Aug 2, 2023
67f0359
only create a BuildHasher impl once
Aug 2, 2023
4e9599e
consistiency
Aug 2, 2023
b5144e9
component adapter: use descriptor to call filesystem funcs, not the fd
Aug 2, 2023
a91561b
reactor test: fix filesystem types paths
Aug 2, 2023
64588fc
Merge remote-tracking branch 'origin/main' into pch/sync_wasi_filesys…
Aug 2, 2023
cb34e7e
Merge remote-tracking branch 'origin/main' into pch/sync_wasi_filesys…
Aug 3, 2023
5cc751b
metadata hash: use DefaultHasher
Aug 3, 2023
d7f1a78
Merge remote-tracking branch 'origin/main' into pch/sync_wasi_filesys…
Aug 3, 2023
7d55030
fix missed trappable-error-type merge conflicts
Aug 4, 2023
9ae2212
preview1-in-preview2: dont use to_le in Filestat assignment of ino
Aug 4, 2023
54a1607
debugging: forward stdio for fd_readdir test so we can figure out CI …
Aug 7, 2023
9899228
Don't convert filestat ino to little-endian, as it's not written to m…
elliottt Aug 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/test-programs/reactor-tests/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ impl TestReactor for T {
Ok(())
}
}
fn pass_an_imported_record(stat: wasi::filesystem::filesystem::DescriptorStat) -> String {
fn pass_an_imported_record(stat: wasi::filesystem::types::DescriptorStat) -> String {
format!("{stat:?}")
}
}
12 changes: 5 additions & 7 deletions crates/test-programs/tests/reactor.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use wasmtime::{
Config, Engine, Store,
};
use wasmtime_wasi::preview2::bindings::clocks::wall_clock;
use wasmtime_wasi::preview2::bindings::filesystem::filesystem;
use wasmtime_wasi::preview2::bindings::filesystem::types as filesystem;
use wasmtime_wasi::preview2::{self, Table, WasiCtx, WasiCtxBuilder, WasiView};

lazy_static::lazy_static! {
Expand All @@ -28,9 +28,9 @@ wasmtime::component::bindgen!({
async: true,
with: {
"wasi:io/streams": preview2::bindings::io::streams,
"wasi:filesystem/filesystem": preview2::bindings::filesystem::filesystem,
"wasi:filesystem/types": preview2::bindings::filesystem::types,
"wasi:filesystem/preopens": preview2::bindings::filesystem::preopens,
"wasi:cli-base/environment": preview2::bindings::cli_base::environment,
"wasi:cli-base/preopens": preview2::bindings::cli_base::preopens,
"wasi:cli-base/exit": preview2::bindings::cli_base::exit,
"wasi:cli-base/stdin": preview2::bindings::cli_base::stdin,
"wasi:cli-base/stdout": preview2::bindings::cli_base::stdout,
Expand Down Expand Up @@ -68,10 +68,10 @@ async fn instantiate(
let mut linker = Linker::new(&ENGINE);

// All of the imports available to the world are provided by the wasi-common crate:
preview2::bindings::filesystem::filesystem::add_to_linker(&mut linker, |x| x)?;
preview2::bindings::filesystem::types::add_to_linker(&mut linker, |x| x)?;
preview2::bindings::filesystem::preopens::add_to_linker(&mut linker, |x| x)?;
preview2::bindings::io::streams::add_to_linker(&mut linker, |x| x)?;
preview2::bindings::cli_base::environment::add_to_linker(&mut linker, |x| x)?;
preview2::bindings::cli_base::preopens::add_to_linker(&mut linker, |x| x)?;
preview2::bindings::cli_base::exit::add_to_linker(&mut linker, |x| x)?;
preview2::bindings::cli_base::stdin::add_to_linker(&mut linker, |x| x)?;
preview2::bindings::cli_base::stdout::add_to_linker(&mut linker, |x| x)?;
Expand Down Expand Up @@ -129,8 +129,6 @@ async fn reactor_tests() -> Result<()> {
nanoseconds: 789,
seconds: 10,
},
device: 0,
inode: 0,
link_count: 0,
size: 0,
status_change_timestamp: wall_clock::Datetime {
Expand Down
3 changes: 0 additions & 3 deletions crates/test-programs/wasi-tests/src/bin/path_open_read_write.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ unsafe fn test_path_open_read_write(dir_fd: wasi::Fd) {
);

wasi::fd_close(f_readonly).expect("close readonly");
drop(f_readonly);

// =============== WRITE ONLY ==================
let f_writeonly = wasi::path_open(dir_fd, 0, "file", 0, wasi::RIGHTS_FD_WRITE, 0, 0)
Expand All @@ -64,7 +63,6 @@ unsafe fn test_path_open_read_write(dir_fd: wasi::Fd) {
assert_eq!(bytes_written, write_buffer.len());

wasi::fd_close(f_writeonly).expect("close writeonly");
drop(f_writeonly);

// ============== READ WRITE =======================

Expand Down Expand Up @@ -111,7 +109,6 @@ unsafe fn test_path_open_read_write(dir_fd: wasi::Fd) {
);

wasi::fd_close(f_readwrite).expect("close readwrite");
drop(f_readwrite);

wasi::path_unlink_file(dir_fd, "file").expect("removing a file");
}
Expand Down
4 changes: 2 additions & 2 deletions crates/wasi-preview1-component-adapter/src/descriptors.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use crate::bindings::wasi::cli_base::{stderr, stdin, stdout};
use crate::bindings::wasi::filesystem::filesystem;
use crate::bindings::wasi::filesystem::types as filesystem;
use crate::bindings::wasi::io::streams::{self, InputStream, OutputStream};
use crate::bindings::wasi::sockets::tcp;
use crate::{set_stderr_stream, BumpArena, File, ImportAlloc, TrappingUnwrap, WasmStr};
Expand Down Expand Up @@ -166,7 +166,7 @@ impl Descriptors {
}))
.trapping_unwrap();

#[link(wasm_import_module = "wasi:cli-base/preopens")]
#[link(wasm_import_module = "wasi:filesystem/preopens")]
extern "C" {
#[link_name = "get-directories"]
fn get_preopens_import(rval: *mut PreopenList);
Expand Down
37 changes: 24 additions & 13 deletions crates/wasi-preview1-component-adapter/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

use crate::bindings::wasi::cli_base::exit;
use crate::bindings::wasi::clocks::{monotonic_clock, wall_clock};
use crate::bindings::wasi::filesystem::filesystem;
use crate::bindings::wasi::filesystem::types as filesystem;
use crate::bindings::wasi::io::streams;
use crate::bindings::wasi::poll::poll;
use crate::bindings::wasi::random::random;
Expand Down Expand Up @@ -597,10 +597,11 @@ pub unsafe extern "C" fn fd_filestat_get(fd: Fd, buf: *mut Filestat) -> Errno {
..
}) => {
let stat = filesystem::stat(file.fd)?;
let metadata_hash = filesystem::metadata_hash(file.fd)?;
let filetype = stat.type_.into();
*buf = Filestat {
dev: stat.device,
ino: stat.inode,
dev: 1,
ino: metadata_hash.lower,
filetype,
nlink: stat.link_count,
size: stat.size,
Expand Down Expand Up @@ -915,8 +916,6 @@ pub unsafe extern "C" fn fd_readdir(
// for it.
let ds = state.descriptors();
let dir = ds.get_dir(fd)?;
let stat = filesystem::stat(dir.fd)?;
let dot_inode = stat.inode;

let mut iter;
match stream {
Expand All @@ -930,7 +929,7 @@ pub unsafe extern "C" fn fd_readdir(
state,
cookie,
use_cache: true,
dot_inode,
dir_descriptor: dir.fd,
}
}

Expand All @@ -945,7 +944,7 @@ pub unsafe extern "C" fn fd_readdir(
cookie: wasi::DIRCOOKIE_START,
use_cache: false,
stream: DirectoryEntryStream(filesystem::read_directory(dir.fd)?),
dot_inode,
dir_descriptor: dir.fd,
};

// Skip to the entry that is requested by the `cookie`
Expand Down Expand Up @@ -1024,7 +1023,7 @@ pub unsafe extern "C" fn fd_readdir(
use_cache: bool,
cookie: Dircookie,
stream: DirectoryEntryStream,
dot_inode: wasi::Inode,
dir_descriptor: filesystem::Descriptor,
}

impl<'a> Iterator for DirectoryEntryIterator<'a> {
Expand All @@ -1041,9 +1040,13 @@ pub unsafe extern "C" fn fd_readdir(
// Preview2 excludes them, so re-add them.
match current_cookie {
0 => {
let metadata_hash = match filesystem::metadata_hash(self.dir_descriptor) {
Ok(h) => h,
Err(e) => return Some(Err(e.into())),
};
let dirent = wasi::Dirent {
d_next: self.cookie,
d_ino: self.dot_inode,
d_ino: metadata_hash.lower,
d_type: wasi::FILETYPE_DIRECTORY,
d_namlen: 1,
};
Expand Down Expand Up @@ -1083,11 +1086,18 @@ pub unsafe extern "C" fn fd_readdir(
Err(e) => return Some(Err(e.into())),
};

let filesystem::DirectoryEntry { inode, type_, name } = entry;
let filesystem::DirectoryEntry { type_, name } = entry;
let d_ino = filesystem::metadata_hash_at(
self.dir_descriptor,
filesystem::PathFlags::empty(),
&name,
)
.map(|h| h.lower)
.unwrap_or(0);
let name = ManuallyDrop::new(name);
let dirent = wasi::Dirent {
d_next: self.cookie,
d_ino: inode.unwrap_or(0),
d_ino,
d_namlen: u32::try_from(name.len()).trapping_unwrap(),
d_type: type_.into(),
};
Expand Down Expand Up @@ -1286,10 +1296,11 @@ pub unsafe extern "C" fn path_filestat_get(
let ds = state.descriptors();
let file = ds.get_dir(fd)?;
let stat = filesystem::stat_at(file.fd, at_flags, path)?;
let metadata_hash = filesystem::metadata_hash_at(file.fd, at_flags, path)?;
let filetype = stat.type_.into();
*buf = Filestat {
dev: stat.device,
ino: stat.inode,
dev: 1,
ino: metadata_hash.lower,
filetype,
nlink: stat.link_count,
size: stat.size,
Expand Down
16 changes: 8 additions & 8 deletions crates/wasi/src/preview2/command.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@ wasmtime::component::bindgen!({
"streams"::"stream-error": Error,
},
with: {
"wasi:filesystem/filesystem": crate::preview2::bindings::filesystem::filesystem,
"wasi:filesystem/types": crate::preview2::bindings::filesystem::types,
"wasi:filesystem/preopens": crate::preview2::bindings::filesystem::preopens,
"wasi:clocks/monotonic_clock": crate::preview2::bindings::clocks::monotonic_clock,
"wasi:poll/poll": crate::preview2::bindings::poll::poll,
"wasi:io/streams": crate::preview2::bindings::io::streams,
Expand All @@ -18,7 +19,6 @@ wasmtime::component::bindgen!({
"wasi:random/random": crate::preview2::bindings::random::random,
"wasi:cli_base/environment": crate::preview2::bindings::cli_base::environment,
"wasi:cli_base/exit": crate::preview2::bindings::cli_base::exit,
"wasi:cli_base/preopens": crate::preview2::bindings::cli_base::preopens,
"wasi:cli_base/stdin": crate::preview2::bindings::cli_base::stdin,
"wasi:cli_base/stdout": crate::preview2::bindings::cli_base::stdout,
"wasi:cli_base/stderr": crate::preview2::bindings::cli_base::stderr,
Expand All @@ -29,13 +29,13 @@ pub fn add_to_linker<T: WasiView>(l: &mut wasmtime::component::Linker<T>) -> any
crate::preview2::bindings::clocks::wall_clock::add_to_linker(l, |t| t)?;
crate::preview2::bindings::clocks::monotonic_clock::add_to_linker(l, |t| t)?;
crate::preview2::bindings::clocks::timezone::add_to_linker(l, |t| t)?;
crate::preview2::bindings::filesystem::filesystem::add_to_linker(l, |t| t)?;
crate::preview2::bindings::filesystem::types::add_to_linker(l, |t| t)?;
crate::preview2::bindings::filesystem::preopens::add_to_linker(l, |t| t)?;
crate::preview2::bindings::poll::poll::add_to_linker(l, |t| t)?;
crate::preview2::bindings::io::streams::add_to_linker(l, |t| t)?;
crate::preview2::bindings::random::random::add_to_linker(l, |t| t)?;
crate::preview2::bindings::cli_base::exit::add_to_linker(l, |t| t)?;
crate::preview2::bindings::cli_base::environment::add_to_linker(l, |t| t)?;
crate::preview2::bindings::cli_base::preopens::add_to_linker(l, |t| t)?;
crate::preview2::bindings::cli_base::stdin::add_to_linker(l, |t| t)?;
crate::preview2::bindings::cli_base::stdout::add_to_linker(l, |t| t)?;
crate::preview2::bindings::cli_base::stderr::add_to_linker(l, |t| t)?;
Expand All @@ -54,7 +54,8 @@ pub mod sync {
"streams"::"stream-error": Error,
},
with: {
"wasi:filesystem/filesystem": crate::preview2::bindings::sync_io::filesystem::filesystem,
"wasi:filesystem/types": crate::preview2::bindings::sync_io::filesystem::types,
"wasi:filesystem/preopens": crate::preview2::bindings::filesystem::preopens,
"wasi:clocks/monotonic_clock": crate::preview2::bindings::clocks::monotonic_clock,
"wasi:poll/poll": crate::preview2::bindings::sync_io::poll::poll,
"wasi:io/streams": crate::preview2::bindings::sync_io::io::streams,
Expand All @@ -63,7 +64,6 @@ pub mod sync {
"wasi:random/random": crate::preview2::bindings::random::random,
"wasi:cli_base/environment": crate::preview2::bindings::cli_base::environment,
"wasi:cli_base/exit": crate::preview2::bindings::cli_base::exit,
"wasi:cli_base/preopens": crate::preview2::bindings::cli_base::preopens,
"wasi:cli_base/stdin": crate::preview2::bindings::cli_base::stdin,
"wasi:cli_base/stdout": crate::preview2::bindings::cli_base::stdout,
"wasi:cli_base/stderr": crate::preview2::bindings::cli_base::stderr,
Expand All @@ -76,13 +76,13 @@ pub mod sync {
crate::preview2::bindings::clocks::wall_clock::add_to_linker(l, |t| t)?;
crate::preview2::bindings::clocks::monotonic_clock::add_to_linker(l, |t| t)?;
crate::preview2::bindings::clocks::timezone::add_to_linker(l, |t| t)?;
crate::preview2::bindings::sync_io::filesystem::filesystem::add_to_linker(l, |t| t)?;
crate::preview2::bindings::sync_io::filesystem::types::add_to_linker(l, |t| t)?;
crate::preview2::bindings::filesystem::preopens::add_to_linker(l, |t| t)?;
crate::preview2::bindings::sync_io::poll::poll::add_to_linker(l, |t| t)?;
crate::preview2::bindings::sync_io::io::streams::add_to_linker(l, |t| t)?;
crate::preview2::bindings::random::random::add_to_linker(l, |t| t)?;
crate::preview2::bindings::cli_base::exit::add_to_linker(l, |t| t)?;
crate::preview2::bindings::cli_base::environment::add_to_linker(l, |t| t)?;
crate::preview2::bindings::cli_base::preopens::add_to_linker(l, |t| t)?;
crate::preview2::bindings::cli_base::stdin::add_to_linker(l, |t| t)?;
crate::preview2::bindings::cli_base::stdout::add_to_linker(l, |t| t)?;
crate::preview2::bindings::cli_base::stderr::add_to_linker(l, |t| t)?;
Expand Down
25 changes: 13 additions & 12 deletions crates/wasi/src/preview2/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,12 @@ pub mod bindings {
interfaces: "
import wasi:poll/poll
import wasi:io/streams
import wasi:filesystem/filesystem
import wasi:filesystem/types
",
tracing: true,
trappable_error_type: {
"streams"::"stream-error": Error,
"filesystem"::"error-code": Error,
"types"::"error-code": Error,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These will need to be updated after #6795 merges.

},
with: {
"wasi:clocks/wall-clock": crate::preview2::bindings::clocks::wall_clock,
Expand Down Expand Up @@ -99,53 +99,54 @@ pub mod bindings {
interfaces: "
import wasi:poll/poll
import wasi:io/streams
import wasi:filesystem/filesystem
import wasi:filesystem/types
",
tracing: true,
async: true,
trappable_error_type: {
"streams"::"stream-error": Error,
"filesystem"::"error-code": Error,
"types"::"error-code": Error,
},
with: {
"wasi:clocks/wall-clock": crate::preview2::bindings::clocks::wall_clock,
}
});
}
pub use self::_internal_io::wasi::{filesystem, io, poll};
pub use self::_internal_io::wasi::{io, poll};

pub(crate) mod _internal_rest {
wasmtime::component::bindgen!({
path: "wit",
interfaces: "
import wasi:clocks/wall-clock
import wasi:clocks/monotonic-clock
import wasi:clocks/timezone
import wasi:filesystem/preopens
import wasi:random/random
import wasi:random/insecure
import wasi:random/insecure-seed
import wasi:cli-base/environment
import wasi:cli-base/preopens
import wasi:cli-base/exit
import wasi:cli-base/stdin
import wasi:cli-base/stdout
import wasi:cli-base/stderr
",
tracing: true,
trappable_error_type: {
"filesystem"::"error-code": Error,
"types"::"error-code": Error,
"streams"::"stream-error": Error,
},
with: {
"wasi:clocks/wall-clock": crate::preview2::bindings::clocks::wall_clock,
"wasi:poll/poll": crate::preview2::bindings::poll::poll,
"wasi:io/streams": crate::preview2::bindings::io::streams,
"wasi:filesystem/filesystem": crate::preview2::bindings::filesystem::filesystem
"wasi:filesystem/types": crate::preview2::bindings::filesystem::types,
}
});
}

pub use self::_internal_rest::wasi::*;
pub use self::_internal_rest::wasi::{cli_base, random};
pub mod filesystem {
pub use super::_internal_io::wasi::filesystem::types;
pub use super::_internal_rest::wasi::filesystem::preopens;
}
}

pub(crate) static RUNTIME: once_cell::sync::Lazy<tokio::runtime::Runtime> =
Expand Down
Loading