add dyn and jit to eval-wasm-replay

tests/eval-wasm-replay.cts

@@ -2,7 +2,7 @@ import cp from 'child_process'
 import fs from 'fs/promises'
 import path from 'path'
 import { writeWithSpaces } from './test-utils.cjs'
-import { online_filter } from './online_filter.cjs'
+import { online_filter } from './filter.cjs'
 const onlinePath = './tests/online'
 const wizeng = process.env.WIZENG || 'wizeng.x86-64-linux';
 
@@ -15,10 +15,14 @@ async function run() {
     }
     let folders = await fs.readdir(onlinePath);
     let custom_filter = online_filter.concat([
-        'fib' // takes to long
+        'fib', // takes to long
+        'guiicons', // jit panicks
+        "mandelbrot", // jit, dyn panicks
+        "virtualkc", // jit panicks
+        "kittygame", // jit, dyn gets stuck
     ])
     folders = folders.filter(folder => !custom_filter.includes(folder));
-    // folders = ['mandelbrot']
+    // folders = ['ffmpeg']
     for (let folder of folders) {
         let benchmarkPath = path.join(onlinePath, folder, 'benchmark')
         let subBenchmarks
@@ -33,33 +37,42 @@ async function run() {
         writeWithSpaces(folder + ':')
         for (let subBenchmark of subBenchmarks) {
             let replay = path.join(benchmarkPath, subBenchmark, 'replay.wasm')
-            const command = `${wizeng} --monitors=profile ${replay}`
-            let buffer = cp.execSync(command, { maxBuffer: 1024 * 5000 * 1000 })
-            let lines = buffer.toString().split('\n')
-            let total = 0;
-            let r3 = 0;
-            for (let line of lines) {
-                line = line.replace(/\x1B\[\d+m/g, '');
-                let percentageIndex = line.indexOf('%');
+            let config = {
+                'int': { command: `${wizeng} --monitors=profile ${replay}`, r3: -1, total: -1 },
+                'dyn': { command: `${wizeng} -mode=dyn --monitors=profile ${replay}`, r3: -1, total: -1 },
+                'jit': { command: `${wizeng} -mode=jit --monitors=profile ${replay}`, r3: -1, total: -1 },
+            }
+            for (let key in config) {
+                let command = config[key].command
+                let buffer = cp.execSync(command, { maxBuffer: 1024 * 5000 * 1000 })
+                let lines = buffer.toString().split('\n')
+                let total = 0;
+                let r3 = 0;
+                for (let line of lines) {
+                    line = line.replace(/\x1B\[\d+m/g, '');
+                    let percentageIndex = line.indexOf('%');
 
-                if (percentageIndex !== -1) {
-                    // Get the substring that ends with the '%' character
-                    let percentage = line.slice(0, percentageIndex);
+                    if (percentageIndex !== -1) {
+                        // Get the substring that ends with the '%' character
+                        let percentage = line.slice(0, percentageIndex);
 
-                    // Split the substring into words
-                    let words = percentage.split(/\s+/);
+                        // Split the substring into words
+                        let words = percentage.split(/\s+/);
 
-                    // Get the third last word (the percentage number)
-                    let number = parseFloat(words[words.length - 1]);
-                    if (isNaN(number)) continue
-                    total += number
-                    if (words[2].startsWith('"r3')) {
-                        r3 += number;
+                        // Get the third last word (the percentage number)
+                        let number = parseFloat(words[words.length - 1]);
+                        if (isNaN(number)) continue
+                        total += number
+                        if (words[2].startsWith('"r3')) {
+                            r3 += number;
+                        }
                     }
                 }
+                config[key].r3 = r3
+                config[key].total = total
             }
 
-            console.log(`r3 time: ${r3}% / ${total}%`);
+            console.log(`int: ${config.int.r3.toFixed(2)}% / ${config.int.total.toFixed(2)}% | dyn: ${config.dyn.r3.toFixed(2)}% / ${config.dyn.total.toFixed(2)}% | jit: ${config.jit.r3.toFixed(2)}% / ${config.jit.total.toFixed(2)}%`);
         }
     }
 }

tests/online_filter.cts → tests/filter.cts

@@ -1,3 +1,20 @@
+export const node_filter = [
+    'rust-tictactoe',
+    'rust-game-of-life', // trace keeps changing.
+    'mem-exp-vec-store-no-host-mod',
+    'mem-exp-init-no-host-mod',
+    'mem-exp-copy-no-host-mod',
+    'mem-exp-fill-no-host-mod',
+    'mem-exp-host-mod-load-vec',
+    'table-exp-host-mod',
+    'table-exp-host-grow',
+    'funky-kart',
+    'table-exp-host-add-friend',
+]
+
+export const offline_filter = [
+    'sqllite',
+]
 
 export const online_filter = [
     'ogv', // TODO: additional ER at end of original trace
@@ -12,6 +29,8 @@ export const online_filter = [
     'playnox', // test doesn't end
     'hnset-bench', // no benchmark generated
     'fractals', // no benchmark generated
+    'video', // empty benchmark generated
+    'wasmsh', // empty benchmark generated
     'rfxgen', // not working
     'rguiicons', // not working
     'rguilayout', // not working

tests/run-tests.cts

@@ -15,7 +15,7 @@ import commandLineArgs from 'command-line-args'
 import { initPerformance } from '../src/performance.cjs'
 import { generateJavascript } from '../src/js-generator.cjs'
 import { createMeasure } from '../src/performance.cjs'
-import { online_filter } from './online_filter.cjs'
+import { node_filter, offline_filter, online_filter } from './filter.cjs'
 
 let extended = false
 
@@ -163,20 +163,8 @@ async function runNodeTests(names: string[], options) {
   console.log('Run node tests')
   console.log('==============')
   // ignore specific tests
-  let filter = [
-    'rust-tictactoe',
-    'rust-game-of-life', // trace keeps changing.
-    'mem-exp-vec-store-no-host-mod',
-    'mem-exp-init-no-host-mod',
-    'mem-exp-copy-no-host-mod',
-    'mem-exp-fill-no-host-mod',
-    'mem-exp-host-mod-load-vec',
-    'table-exp-host-mod',
-    'table-exp-host-grow',
-    'funky-kart',
-    'table-exp-host-add-friend',
-  ]
-  names = names.filter((n) => !filter.includes(n))
+
+  names = names.filter((n) => !node_filter.includes(n))
   let successfull = 0;
   // names = ["mem-imp-host-grow"]
   let roundTripTimes = []
@@ -248,10 +236,7 @@ async function runOfflineTests(names: string[], options) {
   console.log('Run offline tests')
   console.log('=================')
   // ignore specific tests
-  let filter = [
-    'sqllite',
-  ]
-  names = names.filter((n) => !filter.includes(n))
+  names = names.filter((n) => !offline_filter.includes(n))
   let successfull = 0;
   let roundTripTimes = []
   for (let name of names) {