From 6adca9fe285efe0286929c2e926c1d61c52158a0 Mon Sep 17 00:00:00 2001 From: Doehyun Baek Date: Thu, 15 Feb 2024 11:13:10 +0900 Subject: [PATCH] shuffle wasm-merge arguments to prioritize main exports --- crates/replay_gen/src/wasmgen.rs | 149 +++++++++++++++++-------------- tests/eval-wasm-replay.cts | 17 ++-- tests/online/wasmsh/test.js | 4 +- tests/online_filter.cts | 28 ++++++ tests/run-tests.cts | 29 +----- 5 files changed, 118 insertions(+), 109 deletions(-) create mode 100644 tests/online_filter.cts diff --git a/crates/replay_gen/src/wasmgen.rs b/crates/replay_gen/src/wasmgen.rs index 2af15d7a..894963c2 100644 --- a/crates/replay_gen/src/wasmgen.rs +++ b/crates/replay_gen/src/wasmgen.rs @@ -1,10 +1,10 @@ -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{BTreeMap, HashSet}; use std::io::Write; use std::process::Command; use std::{fs, vec}; use std::{fs::File, path::Path}; -use walrus::{Import, Module}; +use walrus::Module; use crate::irgen::{FunctionTy, HostEvent, INIT_INDEX}; use crate::trace::{ValType, F64}; @@ -287,7 +287,7 @@ pub fn generate_replay_wasm(replay_path: &Path, code: &Replay) -> std::io::Resul } generate_replay_js(replay_path, &module_set, code)?; - generate_single_wasm(replay_path, &module_set)?; + generate_single_wasm(replay_path, &module_set, code)?; Ok(()) } @@ -424,20 +424,28 @@ return data; fn generate_single_wasm( replay_path: &Path, module_set: &HashSet<&String>, + code: &Replay, ) -> Result<(), std::io::Error> { - let module_args = module_set + let mut module_args = module_set .iter() .map(|module| vec![format!("{}.wasm", module), module.to_string()]) .flatten() .collect::>(); + // some shuffling of module args to make --rename-export-conflicts work + if let Some(index) = module_args.iter().position(|x| *x == "main.wasm") { + let main_wasm = module_args.remove(index); + let main = module_args.remove(index); + module_args.insert(0, main_wasm); + module_args.insert(1, main); + module_args.insert(2, "index.wasm".to_string()); + module_args.insert(3, "index".to_string()); + } let args = [ "--rename-export-conflicts", "--enable-reference-types", "--enable-multimemory", "--enable-bulk-memory", "--debuginfo", - "index.wasm", - "index", ] .iter() .cloned() @@ -449,77 +457,80 @@ fn generate_single_wasm( .output() .expect("Failed to execute wasm-merge"); - let wasm_path = replay_path - .parent() - .unwrap() - .join(&format!("merged_1.wasm")); - let buffer = &fs::read(wasm_path).unwrap(); - let walrus_module = Module::from_buffer(buffer).unwrap(); - let mut module_map: HashMap> = HashMap::new(); - for import in walrus_module.imports.iter() { - let import_vec = module_map - .entry(import.module.clone()) - .or_insert_with(Vec::new); - if !import_vec.iter().any(|i| i.name == import.name) { - import_vec.push(import); + let module_list = code.imported_modules(); + for module in &module_list { + let module_wat_path = replay_path + .parent() + .unwrap() + .join(&format!("{module}_merge.wat")); + let stream = &mut File::create(&module_wat_path).unwrap(); + + write!(stream, "(module\n")?; + for (_, mem) in code.imported_mems() { + match mem.import { + Some(import) => { + if import.module == *module { + let name = import.name; + let initial = mem.initial; + let maximum = match mem.maximum { + Some(max) => max.to_string(), + None => "".to_string(), + }; + write!(stream, "(memory (export \"{name}\") {initial} {maximum})\n")?; + } + } + None => {} + } } - } - for (module, import_list) in &module_map { + for (_, table) in code.imported_tables() { + match table.import { + Some(import) => { + if import.module == *module { + let name = import.name; + let initial = table.initial; + let maximum = match table.maximum { + Some(max) => max.to_string(), + None => "".to_string(), + }; + let reftype = &table.reftype; + write!( + stream, + "(table (export \"{name}\") {initial} {maximum} {reftype})\n" + )?; + } + } + None => {} + } + } + for (_, global) in code.imported_globals() { + match global.import { + Some(import) => { + if import.module == *module { + let name = import.name; + let initial = global.initial; + let valtype = global.valtype.clone(); + write!( + stream, + "(global (export \"{name}\") {valtype} ({valtype}.const {initial:?}))\n" + )?; + } + } + None => {} + } + } + write!(stream, ")\n")?; + let binary = wat::parse_file(module_wat_path.clone()).unwrap(); let module_wasm_path = replay_path .parent() .unwrap() .join(&format!("{module}_merge.wasm")); - let stream = &mut File::create(&module_wasm_path).unwrap(); - - let export_list = - import_list - .iter() - .map(|import| { - let name = &import.name; - match &import.kind { - walrus::ImportKind::Memory(mid) => { - let memory = walrus_module.memories.get(*mid); - let initial = memory.initial; - let maximum = match memory.maximum { - Some(max) => max.to_string(), - None => "".to_string(), - }; - format!("(memory (export \"{name}\") {initial} {maximum})\n",) - } - walrus::ImportKind::Global(gid) => { - let global = walrus_module.globals.get(*gid); - let valtype = global.ty; - // TODO: this might be wrong - let initial = 0; - format!("(global (export \"{name}\") {valtype} ({valtype}.const {initial:?}))\n",) - } - walrus::ImportKind::Table(tid) => { - let table = walrus_module.tables.get(*tid); - let initial = table.initial; - let maximum = match table.maximum { - Some(max) => max.to_string(), - None => "".to_string(), - }; - let reftype = match table.element_ty { - walrus::ValType::Externref => "externref", - walrus::ValType::Funcref => "funcref", - _ => unreachable!("table cannot contain non-funcref or externref"), - }; - format!("(table (export \"{name}\") {initial} {maximum} {reftype})\n",) - }, - walrus::ImportKind::Function(_) => { - unreachable!("it never imports function here") - } - } - }) - .collect::>() - .join("\n"); - write!(stream, "(module {export_list})")?; + let mut modle_wasm_file = File::create(&module_wasm_path).unwrap(); + modle_wasm_file.write_all(&binary).unwrap(); } - let module_args = module_map + let module_args = module_list .iter() - .map(|(module, _)| vec![format!("{}_merge.wasm", module), module.to_string()]) + .map(|module| vec![format!("{module}_merge.wasm"), module.to_string()]) .flatten() .collect::>(); let args = [ diff --git a/tests/eval-wasm-replay.cts b/tests/eval-wasm-replay.cts index a39b687e..55df2221 100644 --- a/tests/eval-wasm-replay.cts +++ b/tests/eval-wasm-replay.cts @@ -2,8 +2,7 @@ import cp from 'child_process' import fs from 'fs/promises' import path from 'path' import { writeWithSpaces } from './test-utils.cjs' -import { exit } from 'process' - +import { online_filter } from './online_filter.cjs' const onlinePath = './tests/online' const wizeng = process.env.WIZENG || 'wizeng.x86-64-linux'; @@ -15,15 +14,11 @@ async function run() { return } let folders = await fs.readdir(onlinePath); - let exclude = [ - 'commanderkeen', // wizard STACK_OVERFLOW - 'jqkungfu', // wizard doesnt end - 'fib', // takes too long - 'onnxjs', // unknown func: failed to find name `$1000008`" - 'hnset-bench', // no benchmark generated - 'fractals' // no benchmark generated - ]; - folders = folders.filter(folder => !exclude.includes(folder)); + let custom_filter = online_filter.concat([ + 'fib' // takes to long + ]) + folders = folders.filter(folder => !custom_filter.includes(folder)); + // folders = ['mandelbrot'] for (let folder of folders) { let benchmarkPath = path.join(onlinePath, folder, 'benchmark') let subBenchmarks diff --git a/tests/online/wasmsh/test.js b/tests/online/wasmsh/test.js index 031de2e6..6f503309 100644 --- a/tests/online/wasmsh/test.js +++ b/tests/online/wasmsh/test.js @@ -2,10 +2,10 @@ import { delay } from '../../../dist/tests/test-utils.cjs' export default async function test(analyser) { const url = 'https://webassembly.sh/' - const page = await analyser.start(url, { headless: false }) + const page = await analyser.start(url, { headless: true }) const textInput = page.locator('#wasm-terminal') - await textInput.waitFor({state: 'visible'}) + await textInput.waitFor({ state: 'visible' }) console.log(1) await textInput.pressSequentially('quickjs\n') diff --git a/tests/online_filter.cts b/tests/online_filter.cts new file mode 100644 index 00000000..94736412 --- /dev/null +++ b/tests/online_filter.cts @@ -0,0 +1,28 @@ + +export const online_filter = [ + 'ogv', // TODO: additional ER at end of original trace + 'heatmap', // works fine, but too long so we skip it + 'uarm', // doesn't work for js because string is too long + 'image-convolute', // asm2wasm - f64-to-int is too large + 'lichess', // failing test + 'livesplit', // uses simd, filter for now + 'onnxjs', // // unknown func: failed to find name `$1000008`" + 'gotemplate', // timeout for locator('#output') + 'commanderkeen', // unreachable + 'playnox', // test doesn't end + 'hnset-bench', // no benchmark generated + 'fractals', // no benchmark generated + 'rfxgen', // not working + 'rguiicons', // not working + 'rguilayout', // not working + 'rguistyler', // not working + 'roslyn', // not working + 'rtexpacker', //not working + 'rtexviewer', // not working + 'rustpython', // not working + 'skeletal', // not working + 'sqlpractice', // not working + 'takahirox', // not working + 'timestretch', // not working + 'wheel', // not working +] diff --git a/tests/run-tests.cts b/tests/run-tests.cts index eb333b29..fe39d916 100644 --- a/tests/run-tests.cts +++ b/tests/run-tests.cts @@ -15,6 +15,7 @@ import commandLineArgs from 'command-line-args' import { initPerformance } from '../src/performance.cjs' import { generateJavascript } from '../src/js-generator.cjs' import { createMeasure } from '../src/performance.cjs' +import { online_filter } from './online_filter.cjs' let extended = false @@ -217,33 +218,7 @@ async function runOnlineTests(names: string[], options) { console.log('WARNING: You need a working internet connection') console.log('WARNING: Tests depend on third party websites. If those websites changed since this testsuite was created, it might not work') // ignore specific tests - let filter = [ - 'ogv', // TODO: additional ER at end of original trace - 'heatmap', // works fine, but too long so we skip it - 'uarm', // doesn't work for js because string is too long - 'image-convolute', // asm2wasm - f64-to-int is too large - 'lichess', // failing test - 'livesplit', // uses simd, filter for now - 'onnxjs', // // unknown func: failed to find name `$1000008`" - 'gotemplate', // timeout for locator('#output') - 'commanderkeen', // unreachable - 'hnset-bench', // no benchmark generated - 'fractals', // no benchmark generated - 'rfxgen', // not working - 'rguiicons', // not working - 'rguilayout', // not working - 'rguistyler', // not working - 'roslyn', // not working - 'rtexpacker', //not working - 'rtexviewer', // not working - 'rustpython', // not working - 'skeletal', // not working - 'sqlpractice', // not working - 'takahirox', // not working - 'timestretch', // not working - 'wheel', // not working - ] - names = names.filter((n) => !filter.includes(n)) + names = names.filter((n) => !online_filter.includes(n)) let successfull = 0; let roundTripTimes = [] for (let name of names) {