Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

A proof of concept to create standalone C library #3

Draft
wants to merge 41 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
59c1481
use WASI Preview 1 module instead of WASI Preview 2 component
dicej Sep 27, 2023
cedbc1b
ignore zero exit code
dicej Sep 27, 2023
477bdee
Modify path
certik Sep 28, 2023
326b37f
Remove custom build step
certik Sep 28, 2023
2d1a713
Add a build script
certik Sep 28, 2023
e3c66d0
Work on exposing it as a pointer
certik Sep 28, 2023
9040a75
Use Vec<u8>
certik Sep 28, 2023
dcb463d
Refactor
certik Sep 28, 2023
6f3b5ec
Borrow
certik Sep 28, 2023
0944730
Rearrange
certik Sep 28, 2023
62ea06c
Export via C
certik Sep 28, 2023
ce4e659
Call the C function
certik Sep 28, 2023
b9c2a38
Simplify
certik Sep 28, 2023
c5e3981
Simplify
certik Sep 28, 2023
4864ab4
Rework
certik Sep 28, 2023
1add678
Add mylib
certik Sep 28, 2023
629bd92
Use it
certik Sep 28, 2023
d188e50
X
certik Sep 28, 2023
4a7b54d
Runs
certik Sep 28, 2023
183a3c1
OK
certik Sep 28, 2023
9236862
ok
certik Sep 28, 2023
bd6b3dc
This works, but only sometimes...
certik Sep 28, 2023
292a4a2
Now it works
certik Sep 28, 2023
802cd88
X
certik Sep 28, 2023
67f1534
works
certik Sep 28, 2023
67e6f9e
X
certik Sep 28, 2023
7301089
offline
certik Sep 29, 2023
4a0bf3b
remove lock
certik Sep 29, 2023
592a58e
X
certik Sep 29, 2023
bb145de
X
certik Sep 29, 2023
432fff6
X
certik Sep 29, 2023
2de6e03
X
certik Sep 29, 2023
ad27e87
X
certik Sep 29, 2023
fad9323
X
certik Sep 29, 2023
6ddaf4f
X?
certik Sep 29, 2023
1815454
rename
certik Sep 29, 2023
c5c7aea
compile
certik Sep 29, 2023
0aae4da
X
certik Sep 29, 2023
24b9d9c
X
certik Sep 29, 2023
6e335be
X
certik Sep 29, 2023
72a772d
release
certik Sep 29, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2,725 changes: 0 additions & 2,725 deletions Cargo.lock

This file was deleted.

13 changes: 1 addition & 12 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,4 @@
name = "cwasm-standalone"
version = "0.1.0"
edition = "2021"

[dependencies]
anyhow = "1.0.75"
clap = { version = "4.4.5", features = ["derive"] }
wasmtime = { version = "13.0.0", default-features = false, features = ["component-model"] }
wasmtime-wasi = "13.0.0"

[build-dependencies]
anyhow = "1.0.75"
reqwest = { version = "0.11.20", features = ["blocking"] }
wasmtime = { version = "13.0.0", features = ["component-model"] }
wit-component = "0.14.3"
build = "build.rs"
45 changes: 3 additions & 42 deletions build.rs
Original file line number Diff line number Diff line change
@@ -1,43 +1,4 @@
use {
anyhow::Result,
std::{env, fs, path::PathBuf, process::Command},
wasmtime::{Config, Engine},
wit_component::ComponentEncoder,
};

fn main() -> Result<()> {
let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap());

assert!(Command::new("cargo")
.current_dir("guest")
.env("CARGO_TARGET_DIR", &out_dir)
.args(["build", "--release", "--target", "wasm32-wasi"])
.status()?
.success());

let adapter = &reqwest::blocking::get(
"https://github.com/bytecodealliance/wasmtime/releases/download/\
v13.0.0/wasi_snapshot_preview1.command.wasm",
)?
.error_for_status()?
.bytes()?;

let component = &ComponentEncoder::default()
.module(&fs::read(out_dir.join("wasm32-wasi/release/guest.wasm"))?)?
.validate(true)
.adapter("wasi_snapshot_preview1", adapter)?
.encode()?;

let mut config = Config::new();
config.wasm_component_model(true);

let engine = &Engine::new(&config)?;

let cwasm = engine.precompile_component(component)?;

fs::write(out_dir.join("wasm32-wasi/release/guest.cwasm"), cwasm)?;

println!("cargo:rerun-if-changed=guest");

Ok(())
fn main() {
println!(r"cargo:rustc-link-search=/Users/ondrej/repos/cwasm-standalone/mylib/target/release/");
println!(r"cargo:rustc-link-lib=mylib");
}
24 changes: 24 additions & 0 deletions build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
#! /bin/bash

set -ex

cd src
lfortran $HOME/repos/lfortran/lfortran/examples/expr2.f90 --backend=wasm -o guest.wasm
wasmtime compile guest.wasm -o guest.cwasm
cd ..

cd mylib
cargo build --release --offline
cd ..

strip mylib/target/release/libmylib.a

cargo run --release --offline

bin2c guest < src/guest.cwasm > guest.c
clang -c guest.c -o guest.o
clang -c driver.c -o driver.o
clang -o driver driver.o guest.o mylib/target/release/libmylib.a
ls -h driver
strip driver
ls -h driver
11 changes: 11 additions & 0 deletions driver.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
#include <stdint.h>
#include <stdio.h>

extern int32_t run_wasm_native_binary(size_t len, uint8_t *guest);

int main() {
extern const char guest[];
extern const size_t guest_len;
int32_t result = run_wasm_native_binary(guest_len, (uint8_t*)guest);
return result;
}
18 changes: 18 additions & 0 deletions mylib/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
[package]
name = "mylib"
version = "0.1.0"
edition = "2021"

[dependencies]
anyhow = "1.0.75"
clap = { version = "4.4.5", features = ["derive"] }
wasmtime = { version = "13.0.0", default-features = false }
wasmtime-wasi = "13.0.0"

[build-dependencies]
anyhow = "1.0.75"
reqwest = { version = "0.11.20", features = ["blocking"] }
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think reqwest is used anymore, so you could remove this.

wasmtime = { version = "13.0.0" }

[lib]
crate-type = ["staticlib"]
102 changes: 102 additions & 0 deletions mylib/src/lib.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
use {
anyhow::{anyhow, Result},
clap::Parser,
wasmtime::{Config, Engine, Linker, Module, Store},
wasmtime_wasi::{Dir, I32Exit, WasiCtx, WasiCtxBuilder},
};

fn parse_mapdir(s: &str) -> Result<(String, String)> {
if let Some((guest_dir, host_dir)) = s.split_once("::") {
Ok((guest_dir.into(), host_dir.into()))
} else {
Err(anyhow!(
"expected string of form GUEST_DIR::HOST_DIR; got {s}"
))
}
}

fn parse_env(s: &str) -> Result<(String, String)> {
if let Some((name, value)) = s.split_once('=') {
Ok((name.into(), value.into()))
} else {
Err(anyhow!("expected string of form NAME=VALUE; got {s}"))
}
}

#[derive(Parser)]
struct Options {
#[clap(long, value_name = "GUEST_DIR::HOST_DIR", value_parser = parse_mapdir)]
mapdir: Vec<(String, String)>,

#[clap(long, value_name = "NAME=VALUE", value_parser = parse_env)]
env: Vec<(String, String)>,
}

struct Ctx {
wasi: WasiCtx,
}

fn ignore_successful_proc_exit_trap(guest_err: anyhow::Error) -> Result<()> {
match guest_err.root_cause().downcast_ref::<I32Exit>() {
Some(trap) => match trap.0 {
0 => Ok(()),
_ => Err(guest_err),
},
None => Err(guest_err),
}
}

fn run_internal(guest: &[u8]) -> Result<()> {
let options = Options::parse();
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This probably won't do anything useful, since the Rust std::env::args won't have been initialized when using this as a library from C. You could update run_rust_wasm_native_binary to accept CLI parameters and pass them to https://docs.rs/clap/latest/clap/trait.Parser.html#method.try_parse_from. Or you could just delete all the options parsing and WASI init code if you don't care about giving the guest access to host environment variables and directories.

Copy link
Author

@certik certik Oct 3, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah I see. Actually I do care about that, although I think we have not implemented that yet in our WASM backend. This poses an interesting problem --- in our LLVM backend we allow access to the whole file system. Here we have a nice option to actually restrict it, but it's not clear to me right now what the default should be.


let engine = &Engine::new(&Config::new())?;

let mut linker = Linker::<Ctx>::new(engine);
wasmtime_wasi::add_to_linker(&mut linker, |ctx| &mut ctx.wasi)?;

let mut wasi = WasiCtxBuilder::new();
wasi.inherit_stdio();

for (guest_dir, host_dir) in options.mapdir {
wasi.preopened_dir(
Dir::from_std_file(std::fs::File::open(host_dir)?),
guest_dir,
)?;
}

for (name, value) in options.env {
wasi.env(&name, &value)?;
}

let wasi = wasi.build();
let mut store = Store::new(engine, Ctx { wasi });

let instance = linker.instantiate(&mut store, &unsafe {
Module::deserialize(
engine,
guest,
)
}?)?;

let start = instance
.get_func(&mut store, "_start")
.ok_or_else(|| anyhow!("unable to find `_start` function"))?;

start
.call(&mut store, &[], &mut [])
.or_else(ignore_successful_proc_exit_trap)?;

Ok(())
}

#[no_mangle]
pub extern "C" fn run_wasm_native_binary(len: usize, guest: *const u8) -> i32 {
let guest_slice: &[u8] = unsafe {
std::slice::from_raw_parts(guest, len)
};

match run_internal(guest_slice) {
Ok(_) => 0,
Err(_) => 1,
}
}
116 changes: 11 additions & 105 deletions src/main.rs
Original file line number Diff line number Diff line change
@@ -1,112 +1,18 @@
use {
anyhow::{anyhow, Result},
clap::Parser,
wasmtime::{
component::{Component, Linker},
Config, Engine, Store,
},
wasmtime_wasi::{
preview2::{
command::{self, sync::Command},
DirPerms, FilePerms, Table, WasiCtx, WasiCtxBuilder, WasiView,
},
Dir,
},
};
use std::io;

fn parse_mapdir(s: &str) -> Result<(String, String)> {
if let Some((guest_dir, host_dir)) = s.split_once("::") {
Ok((guest_dir.into(), host_dir.into()))
} else {
Err(anyhow!(
"expected string of form GUEST_DIR::HOST_DIR; got {s}"
))
}
}

fn parse_env(s: &str) -> Result<(String, String)> {
if let Some((name, value)) = s.split_once('=') {
Ok((name.into(), value.into()))
} else {
Err(anyhow!("expected string of form NAME=VALUE; got {s}"))
}
}

#[derive(Parser)]
pub struct Options {
#[clap(long, value_name = "GUEST_DIR::HOST_DIR", value_parser = parse_mapdir)]
mapdir: Vec<(String, String)>,

#[clap(long, value_name = "NAME=VALUE", value_parser = parse_env)]
env: Vec<(String, String)>,
}

struct Ctx {
wasi: WasiCtx,
table: Table,
}

impl WasiView for Ctx {
fn ctx(&self) -> &WasiCtx {
&self.wasi
}
fn ctx_mut(&mut self) -> &mut WasiCtx {
&mut self.wasi
}
fn table(&self) -> &Table {
&self.table
}
fn table_mut(&mut self) -> &mut Table {
&mut self.table
}
extern "C" {
fn run_wasm_native_binary(len: usize, guest: *const u8) -> i32;
}

fn main() -> Result<()> {
let options = Options::parse();

let mut config = Config::new();
config.wasm_component_model(true);

let engine = &Engine::new(&config)?;

let mut linker = Linker::new(engine);
command::sync::add_to_linker(&mut linker)?;
fn main() -> Result<(), std::io::Error> {
let guest: Vec<u8> = include_bytes!("guest.cwasm").to_vec();

let mut table = Table::new();
let mut wasi = WasiCtxBuilder::new();
wasi.inherit_stdio();
let result: i32 = unsafe {
run_wasm_native_binary(guest.len(), guest.as_ptr())
};

for (guest_dir, host_dir) in options.mapdir {
wasi.preopened_dir(
Dir::from_std_file(std::fs::File::open(host_dir)?),
DirPerms::all(),
FilePerms::all(),
guest_dir,
);
match result {
0 => Ok(()),
_ => Err(io::ErrorKind::Other.into()),
}

for (name, value) in options.env {
wasi.env(name, value);
}

let wasi = wasi.build(&mut table)?;
let mut store = Store::new(engine, Ctx { wasi, table });

let (command, _) = Command::instantiate(
&mut store,
&unsafe {
Component::deserialize(
engine,
include_bytes!(concat!(env!("OUT_DIR"), "/wasm32-wasi/release/guest.cwasm")),
)
}?,
&linker,
)?;

command
.wasi_cli_run()
.call_run(&mut store)?
.map_err(|()| anyhow!("guest command returned error"))?;

Ok(())
}