Skip to content

Commit

Permalink
initial reduction experiments
Browse files Browse the repository at this point in the history
  • Loading branch information
@doehyunbaek
doehyunbaek committed Sep 3, 2024
1 parent 260e876 commit 299be52
Show file tree
Hide file tree
Showing 5 changed files with 264 additions and 8 deletions.
129 changes: 129 additions & 0 deletions evaluation/eval.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,135 @@
"!TEST_SUBSET=True TEST_NAME=jqkungfu python run_slicedice.py"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Failed to run boa - wasm-slice\n",
"timeout 120s python /home/don/wasm-r3/evaluation/run_reduction_tool.py wasm-slice /home/don/wasm-r3/benchmarks/boa/boa.wasm\n",
"Failed to run boa - wasm-reduce\n",
"timeout 120s python /home/don/wasm-r3/evaluation/run_reduction_tool.py wasm-reduce /home/don/wasm-r3/benchmarks/boa/boa.wasm\n",
"Failed to run funky-kart - wasm-reduce\n",
"timeout 120s python /home/don/wasm-r3/evaluation/run_reduction_tool.py wasm-reduce /home/don/wasm-r3/benchmarks/funky-kart/funky-kart.wasm\n",
"Failed to run rfxgen - wasm-reduce\n",
"timeout 120s python /home/don/wasm-r3/evaluation/run_reduction_tool.py wasm-reduce /home/don/wasm-r3/benchmarks/rfxgen/rfxgen.wasm\n",
"Failed to run rguilayout - wasm-reduce\n",
"timeout 120s python /home/don/wasm-r3/evaluation/run_reduction_tool.py wasm-reduce /home/don/wasm-r3/benchmarks/rguilayout/rguilayout.wasm\n",
"Failed to run rguistyler - wasm-reduce\n",
"timeout 120s python /home/don/wasm-r3/evaluation/run_reduction_tool.py wasm-reduce /home/don/wasm-r3/benchmarks/rguistyler/rguistyler.wasm\n",
"Failed to run riconpacker - wasm-reduce\n",
"timeout 120s python /home/don/wasm-r3/evaluation/run_reduction_tool.py wasm-reduce /home/don/wasm-r3/benchmarks/riconpacker/riconpacker.wasm\n",
"Failed to run rguilayout - wasm-shrink\n",
"timeout 120s python /home/don/wasm-r3/evaluation/run_reduction_tool.py wasm-shrink /home/don/wasm-r3/benchmarks/rguilayout/rguilayout.wasm\n",
"Failed to run sqlgui - wasm-reduce\n",
"timeout 120s python /home/don/wasm-r3/evaluation/run_reduction_tool.py wasm-reduce /home/don/wasm-r3/benchmarks/sqlgui/sqlgui.wasm\n",
"Failed to run commanderkeen - wasm-reduce\n",
"timeout 120s python /home/don/wasm-r3/evaluation/run_reduction_tool.py wasm-reduce /home/don/wasm-r3/benchmarks/commanderkeen/commanderkeen.wasm\n",
"Failed to run hydro - wasm-reduce\n",
"timeout 120s python /home/don/wasm-r3/evaluation/run_reduction_tool.py wasm-reduce /home/don/wasm-r3/benchmarks/hydro/hydro.wasm\n",
"Failed to run rtexviewer - wasm-reduce\n",
"timeout 120s python /home/don/wasm-r3/evaluation/run_reduction_tool.py wasm-reduce /home/don/wasm-r3/benchmarks/rtexviewer/rtexviewer.wasm\n",
"Failed to run mandelbrot - wasm-reduce\n",
"timeout 120s python /home/don/wasm-r3/evaluation/run_reduction_tool.py wasm-reduce /home/don/wasm-r3/benchmarks/mandelbrot/mandelbrot.wasm\n",
"Failed to run mandelbrot - wasm-shrink\n",
"timeout 120s python /home/don/wasm-r3/evaluation/run_reduction_tool.py wasm-shrink /home/don/wasm-r3/benchmarks/mandelbrot/mandelbrot.wasm\n"
]
}
],
"source": [
"import time, subprocess, json, os, concurrent\n",
"\n",
"TIMEOUT = 120\n",
"WASMR3_PATH = os.getenv(\"WASMR3_PATH\", \"~/wasm-r3\")\n",
"\n",
"with open(\"metrics.json\", \"r\") as f:\n",
" metrics = json.load(f)\n",
"\n",
"testset = [\n",
" 'boa', # this doesn't work for wasm-slice\n",
" \"guiicons\",\n",
" \"funky-kart\",\n",
" \"jsc\",\n",
" \"rfxgen\",\n",
" \"rguilayout\",\n",
" \"rguistyler\",\n",
" \"riconpacker\",\n",
" \"sqlgui\",\n",
" \"commanderkeen\",\n",
" \"hydro\",\n",
" \"rtexviewer\",\n",
" \"mandelbrot\",\n",
"]\n",
"\n",
"our_tool = [\"wasm-slice\"]\n",
"\n",
"toolset = [\n",
" \"wasm-reduce\",\n",
" \"wasm-shrink\",\n",
"]\n",
"\n",
"tool_to_suffix = {\n",
" \"wasm-slice\": \"sliced\",\n",
" \"wasm-reduce\": \"reduced\",\n",
" \"wasm-shrink\": \"shrunken\",\n",
"}\n",
"\n",
"\n",
"def run_reduction_tool(testname, tool):\n",
" try:\n",
" command = f\"timeout {TIMEOUT}s python {WASMR3_PATH}/evaluation/run_reduction_tool.py {tool} {WASMR3_PATH}/benchmarks/{testname}/{testname}.wasm\"\n",
" start_time = time.time()\n",
" result = subprocess.run(command, shell=True, capture_output=True, text=True)\n",
" end_time = time.time()\n",
" elapsed = end_time - start_time\n",
" reduced_size = os.path.getsize(\n",
" f\"{WASMR3_PATH}/benchmarks/{testname}/{testname}.{tool_to_suffix[tool]}.wasm\"\n",
" )\n",
" return [testname, tool, elapsed, reduced_size]\n",
" except Exception as e:\n",
" print(f\"Failed to run {testname} - {tool}\")\n",
" print(f\"timeout {TIMEOUT}s python {WASMR3_PATH}/evaluation/run_reduction_tool.py {tool} {WASMR3_PATH}/benchmarks/{testname}/{testname}.wasm\")\n",
" return [testname, tool, \"fail\", \"fail\"]\n",
"\n",
"\n",
"for testname in testset:\n",
" metrics[testname][\"reduction_comparison\"] = {\n",
" \"original_size\": os.path.getsize(f\"{WASMR3_PATH}/benchmarks/{testname}/{testname}.wasm\")\n",
" }\n",
"\n",
"with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:\n",
" futures = [\n",
" executor.submit(run_reduction_tool, testname, tool)\n",
" for testname in testset\n",
" for tool in our_tool\n",
" ]\n",
" results = [future.result() for future in concurrent.futures.as_completed(futures)]\n",
"\n",
"with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:\n",
" futures = [\n",
" executor.submit(run_reduction_tool, testname, tool)\n",
" for testname in testset\n",
" for tool in toolset\n",
" ]\n",
" results = results + [\n",
" future.result() for future in concurrent.futures.as_completed(futures)\n",
" ]\n",
"\n",
"for result in results:\n",
" testname, tool, elapsed, reduced_size = result\n",
" metrics[testname][\"reduction_comparison\"][f\"{tool}-size\"] = reduced_size\n",
" metrics[testname][\"reduction_comparison\"][f\"{tool}-time\"] = elapsed\n",
"\n",
"with open(\"metrics.json\", \"w\") as f:\n",
" sorted_metrics = {k: metrics[k] for k in sorted(metrics)}\n",
" json.dump(sorted_metrics, f, indent=4)"
]
},
{
"cell_type": "code",
"execution_count": 4,
Expand Down
4 changes: 3 additions & 1 deletion evaluation/interesting.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,15 +50,17 @@ def run_command(engine, test_input):

def crash_on_wizard():
result = run_command(ENGINE, WASM)
# print('crash_on_wizard: ', result.returncode)
return result.returncode != 0 and result.stdout.find('no main export from module') == -1

def ok_on_wasmtime():
command = ['timeout', '1', 'wasmtime', '--invoke', 'main', WASM]
command = ['timeout', '5', 'wasmtime', '--invoke', 'main', WASM]
result = subprocess.run(
command,
capture_output=True,
text=True,
)
# print('ok_on_wasmtime: ', result.returncode)
return result.returncode == 0

if crash_on_wizard() and ok_on_wasmtime():
Expand Down
117 changes: 117 additions & 0 deletions evaluation/metrics.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1847,6 +1847,15 @@
"4558": {
"slice-dice": 7313,
"wasm-r3": 11075
},
"reduction_comparison": {
"original_size": 22264896,
"wasm-slice-size": "fail",
"wasm-slice-time": "fail",
"wasm-reduce-size": 22180313,
"wasm-reduce-time": 120,
"wasm-shrink-size": 22216362,
"wasm-shrink-time": 120.01304936408997
}
},
"bullet": {
Expand Down Expand Up @@ -9175,6 +9184,15 @@
"1968": {
"slice-dice": 6984,
"wasm-r3": 94225
},
"reduction_comparison": {
"original_size": 4331017,
"wasm-slice-size": 995605,
"wasm-slice-time": 90.20325756072998,
"wasm-reduce-size": 676743,
"wasm-reduce-time": 120,
"wasm-shrink-size": 4328380,
"wasm-shrink-time": 120.00192189216614
}
},
"factorial": {
Expand Down Expand Up @@ -13425,6 +13443,15 @@
"1586": {
"slice-dice": 1228,
"wasm-r3": 80029
},
"reduction_comparison": {
"original_size": 3725095,
"wasm-slice-size": 2053785,
"wasm-slice-time": 69.60664987564087,
"wasm-reduce-size": 2299553,
"wasm-reduce-time": 120,
"wasm-shrink-size": 3675826,
"wasm-shrink-time": 120.00278520584106
}
},
"game-of-life": {
Expand Down Expand Up @@ -14105,6 +14132,15 @@
"504": {
"slice-dice": 446,
"wasm-r3": 14199
},
"reduction_comparison": {
"original_size": 376838,
"wasm-slice-size": 452103,
"wasm-slice-time": 30.84325075149536,
"wasm-reduce-size": 375265,
"wasm-reduce-time": 120,
"wasm-shrink-size": 315993,
"wasm-shrink-time": 120.00191926956177
}
},
"hydro": {
Expand Down Expand Up @@ -14319,6 +14355,15 @@
"1586": {
"slice-dice": 893,
"wasm-r3": 2385
},
"reduction_comparison": {
"original_size": 824085,
"wasm-slice-size": 380686,
"wasm-slice-time": 6.540843486785889,
"wasm-reduce-size": 645689,
"wasm-reduce-time": 120,
"wasm-shrink-size": 734868,
"wasm-shrink-time": 120.00204253196716
}
},
"jqkungfu": {
Expand Down Expand Up @@ -24091,6 +24136,15 @@
"20796": {
"slice-dice": 6564,
"wasm-r3": 24387
},
"reduction_comparison": {
"original_size": 5470509,
"wasm-slice-size": 4900022,
"wasm-slice-time": 44.25752401351929,
"wasm-reduce-size": 5286120,
"wasm-reduce-time": 120,
"wasm-shrink-size": 5159781,
"wasm-shrink-time": 120.00331735610962
}
},
"mandelbrot": {
Expand Down Expand Up @@ -24241,6 +24295,15 @@
"164": {
"slice-dice": 174,
"wasm-r3": 31976
},
"reduction_comparison": {
"original_size": 240403,
"wasm-slice-size": 404385,
"wasm-slice-time": 97.6673104763031,
"wasm-reduce-size": 237739,
"wasm-reduce-time": 120,
"wasm-shrink-size": 232459,
"wasm-shrink-time": 120
}
},
"multiplyDouble": {
Expand Down Expand Up @@ -26257,6 +26320,15 @@
"572": {
"slice-dice": 568,
"wasm-r3": 11958
},
"reduction_comparison": {
"original_size": 502035,
"wasm-slice-size": 540367,
"wasm-slice-time": 53.18854331970215,
"wasm-reduce-size": 485322,
"wasm-reduce-time": 120,
"wasm-shrink-size": 492377,
"wasm-shrink-time": 120.00193786621094
}
},
"rguilayout": {
Expand Down Expand Up @@ -27003,6 +27075,15 @@
"502": {
"slice-dice": 464,
"wasm-r3": 15897
},
"reduction_comparison": {
"original_size": 516369,
"wasm-slice-size": 596183,
"wasm-slice-time": 31.644596576690674,
"wasm-reduce-size": 514791,
"wasm-reduce-time": 120,
"wasm-shrink-size": 514240,
"wasm-shrink-time": 120
}
},
"rguistyler": {
Expand Down Expand Up @@ -27765,6 +27846,15 @@
"573": {
"slice-dice": 627,
"wasm-r3": 11898
},
"reduction_comparison": {
"original_size": 558009,
"wasm-slice-size": 845480,
"wasm-slice-time": 32.91276717185974,
"wasm-reduce-size": 551339,
"wasm-reduce-time": 120,
"wasm-shrink-size": 546181,
"wasm-shrink-time": 120.00490522384644
}
},
"riconpacker": {
Expand Down Expand Up @@ -28415,6 +28505,15 @@
"575": {
"slice-dice": 582,
"wasm-r3": 1990
},
"reduction_comparison": {
"original_size": 486543,
"wasm-slice-size": 425501,
"wasm-slice-time": 6.774238348007202,
"wasm-reduce-size": 486626,
"wasm-reduce-time": 120,
"wasm-shrink-size": 457528,
"wasm-shrink-time": 120.00188422203064
}
},
"rtexpacker": {
Expand All @@ -28435,6 +28534,15 @@
"365": {
"slice-dice": 381,
"wasm-r3": 3042
},
"reduction_comparison": {
"original_size": 386612,
"wasm-slice-size": 209441,
"wasm-slice-time": 3.9366116523742676,
"wasm-reduce-size": 239415,
"wasm-reduce-time": 120,
"wasm-shrink-size": 217342,
"wasm-shrink-time": 120.00243353843689
}
},
"sandspiel": {
Expand Down Expand Up @@ -31051,6 +31159,15 @@
"1893": {
"slice-dice": 1058,
"wasm-r3": 3728
},
"reduction_comparison": {
"original_size": 712714,
"wasm-slice-size": 551302,
"wasm-slice-time": 7.06463360786438,
"wasm-reduce-size": 648272,
"wasm-reduce-time": 120,
"wasm-shrink-size": 664983,
"wasm-shrink-time": 120.00179672241211
}
}
}
Loading

0 comments on commit 299be52

Please sign in to comment.