Fixes std lib cases actually benchmarkable (#1885)

Signed-off-by: Takeshi Yoneda <[email protected]>
tetratelabs · Dec 21, 2023 · 6fdb893 · 6fdb893
1 parent 8c71d4d
commit 6fdb893
Show file tree

Hide file tree

Showing 2 changed files with 38 additions and 30 deletions.
diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
@@ -391,7 +391,7 @@ jobs:
           path:
             ${{ env.STDLIB_TESTS }}/testdata/go
           # Use precise Go version from setup-go as patch version differences can effect tests.
-          key: go-wasip1-binaries-${{ matrix.os }}-${{ steps.setup-go.outputs.go-version }}
+          key: go-wasip1-binaries-${{ matrix.os }}-${{ steps.setup-go.outputs.go-version }}-${{ matrix.arch }}
 
       - if: ${{ matrix.compiler == 'optimizing' }}
         name: Build wazero

diff --git a/internal/integration_test/stdlibs/bench_test.go b/internal/integration_test/stdlibs/bench_test.go
@@ -20,38 +20,38 @@ func BenchmarkZig(b *testing.B) {
 	if runtime.GOARCH == "arm64" {
 		b.Run("optimizing", func(b *testing.B) {
 			c := opt.NewRuntimeConfigOptimizingCompiler()
-			runtBench(b, context.Background(), c, zigTestCase)
+			runtBenches(b, context.Background(), c, zigTestCase)
 		})
 	}
 	b.Run("baseline", func(b *testing.B) {
 		c := wazero.NewRuntimeConfigCompiler()
-		runtBench(b, context.Background(), c, zigTestCase)
+		runtBenches(b, context.Background(), c, zigTestCase)
 	})
 }
 
 func BenchmarkTinyGo(b *testing.B) {
 	if runtime.GOARCH == "arm64" {
 		b.Run("optimizing", func(b *testing.B) {
 			c := opt.NewRuntimeConfigOptimizingCompiler()
-			runtBench(b, context.Background(), c, tinyGoTestCase)
+			runtBenches(b, context.Background(), c, tinyGoTestCase)
 		})
 	}
 	b.Run("baseline", func(b *testing.B) {
 		c := wazero.NewRuntimeConfigCompiler()
-		runtBench(b, context.Background(), c, tinyGoTestCase)
+		runtBenches(b, context.Background(), c, tinyGoTestCase)
 	})
 }
 
 func BenchmarkWasip1(b *testing.B) {
 	if runtime.GOARCH == "arm64" {
 		b.Run("optimizing", func(b *testing.B) {
 			c := opt.NewRuntimeConfigOptimizingCompiler()
-			runtBench(b, context.Background(), c, wasip1TestCase)
+			runtBenches(b, context.Background(), c, wasip1TestCase)
 		})
 	}
 	b.Run("baseline", func(b *testing.B) {
 		c := wazero.NewRuntimeConfigCompiler()
-		runtBench(b, context.Background(), c, wasip1TestCase)
+		runtBenches(b, context.Background(), c, wasip1TestCase)
 	})
 }
 
@@ -136,7 +136,7 @@ var (
 	}
 )
 
-func runtBench(b *testing.B, ctx context.Context, rc wazero.RuntimeConfig, tc testCase) {
+func runtBenches(b *testing.B, ctx context.Context, rc wazero.RuntimeConfig, tc testCase) {
 	cwd, _ := os.Getwd()
 	files, err := os.ReadDir(tc.dir)
 	require.NoError(b, err)
@@ -152,27 +152,36 @@ func runtBench(b *testing.B, ctx context.Context, rc wazero.RuntimeConfig, tc te
 		if bin == nil {
 			continue
 		}
-		b.Run(fname, func(b *testing.B) {
-			r := wazero.NewRuntimeWithConfig(ctx, rc)
-			wasi_snapshot_preview1.MustInstantiate(ctx, r)
-			b.Cleanup(func() { r.Close(ctx) })
-
-			var cm wazero.CompiledModule
-			b.Run("Compile", func(b *testing.B) {
-				b.ResetTimer()
-				var err error
-				cm, err = r.CompileModule(ctx, bin)
-				require.NoError(b, err)
-			})
 
-			im, err := r.InstantiateModule(ctx, cm, modCfg)
-			require.NoError(b, err)
-			b.Run("Run", func(b *testing.B) {
-				b.ResetTimer()
-				_, err := im.ExportedFunction("_start").Call(ctx)
-				requireZeroExitCode(b, err)
-			})
-		})
+		for _, compile := range []bool{false, true} {
+			if compile {
+				b.Run("Compile/"+fname, func(b *testing.B) {
+					b.ResetTimer()
+					for i := 0; i < b.N; i++ {
+						r := wazero.NewRuntimeWithConfig(ctx, rc)
+						_, err := r.CompileModule(ctx, bin)
+						require.NoError(b, err)
+						require.NoError(b, r.Close(ctx))
+					}
+				})
+			} else {
+				r := wazero.NewRuntimeWithConfig(ctx, rc)
+				wasi_snapshot_preview1.MustInstantiate(ctx, r)
+				b.Cleanup(func() { r.Close(ctx) })
+
+				cm, err := r.CompileModule(ctx, bin)
+				require.NoError(b, err)
+				b.Run("Run/"+fname, func(b *testing.B) {
+					b.ResetTimer()
+					for i := 0; i < b.N; i++ {
+						// Instantiate in the loop as _start cannot be called multiple times.
+						m, err := r.InstantiateModule(ctx, cm, modCfg)
+						requireZeroExitCode(b, err)
+						require.NoError(b, m.Close(ctx))
+					}
+				})
+			}
+		}
 	}
 }
 
@@ -194,8 +203,7 @@ func defaultModuleConfig() wazero.ModuleConfig {
 		WithRandSource(rand.Reader).
 		// Some tests require Stdout and Stderr to be present.
 		WithStdout(os.Stdout).
-		WithStderr(os.Stderr).
-		WithStartFunctions()
+		WithStderr(os.Stderr)
 }
 
 func requireZeroExitCode(b *testing.B, err error) {