Removes integration_test/vs (#2270)

vs was introduced to bench various things including
the tests against other runtimes. However, the
cases are not representative, and also some are
just unable to build plus some are simply failing.

This removes the entire vs directory, and reduces
the maintenance burden.

Signed-off-by: Takeshi Yoneda <[email protected]>

.github/workflows/commit.yaml

@@ -128,9 +128,8 @@ jobs:
           go-version: ${{ matrix.go-version }}
 
       - name: Build test binaries
-        # Exclude benchmarks as we don't run those in Docker
         run: |
-          go list -f '{{.Dir}}' ./... | egrep -v '(bench|vs|spectest)' | xargs -Ipkg go test pkg -c -o pkg.test
+          go list -f '{{.Dir}}' ./... | egrep -v '(vs|spectest)' | xargs -Ipkg go test pkg -c -o pkg.test
           go build -o wazerocli ./cmd/wazero
         env:
           GOARCH: ${{ matrix.arch }}
@@ -167,28 +166,6 @@ jobs:
       - run: tinygo build ./cmd/wazero
       - run: tinygo build -size short -target pico -stack-size=8kb ./cmd/wazero
 
-  bench:
-    name: Benchmark
-    runs-on: ubuntu-22.04
-
-    steps:
-      # Unlike the other CGO libraries, WasmEdge requires offline installation.
-      - name: Install WasmEdge
-        run: |
-          wget -qO- https://raw.githubusercontent.com/WasmEdge/WasmEdge/master/utils/install.sh | sudo bash -s -- -p /usr/local -v ${WASMEDGE_VERSION}
-        # The version here is coupled to internal/integration_test/go.mod, but it
-        # isn't always the same as sometimes the Go layer has a broken release.
-        env:
-          WASMEDGE_VERSION: 0.12.1
-
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: ${{ env.GO_VERSION }}
-
-      - run: make bench
-
   # This ensures that internal/integration_test/fuzz is runnable, and is not intended to
   # run full-length fuzzing while trying to find low-hanging frontend bugs.
   fuzz:

.github/workflows/examples.yaml

@@ -104,9 +104,5 @@ jobs:
           source ./emsdk/emsdk_env.sh
           make build.examples.emscripten
 
-      - name: Build bench cases
-        run: make build.bench
-        if: matrix.go-version != '1.20' # fails with TinyGo v0.32.0
-
       - name: Run example tests
         run: make test.examples

Makefile

@@ -20,22 +20,6 @@ main_packages := $(sort $(foreach f,$(dir $(main_sources)),$(if $(findstring ./,
 
 go_test_options ?= -timeout 300s
 
-ensureCompilerFastest := -ldflags '-X github.com/tetratelabs/wazero/internal/integration_test/vs.ensureCompilerFastest=true'
-.PHONY: bench
-bench:
-	@go build ./internal/integration_test/bench/...
-	@# Don't use -test.benchmem as it isn't accurate when comparing against CGO libs
-	@for d in vs/time vs/wasmedge vs/wasmtime ; do \
-		cd ./internal/integration_test/$$d ; \
-		go test -bench=. . -tags='wasmedge' $(ensureCompilerFastest) ; \
-		cd - ;\
-	done
-
-bench_testdata_dir := internal/integration_test/bench/testdata
-.PHONY: build.bench
-build.bench:
-	@tinygo build -o $(bench_testdata_dir)/case.wasm -scheduler=none --no-debug -target=wasi $(bench_testdata_dir)/case.go
-
 .PHONY: test.examples
 test.examples:
 	@go test $(go_test_options) ./examples/... ./imports/assemblyscript/example/... ./imports/emscripten/... ./imports/wasi_snapshot_preview1/example/...

cache_test.go

@@ -14,10 +14,10 @@ import (
 	"github.com/tetratelabs/wazero/internal/wasm"
 )
 
-//go:embed internal/integration_test/vs/testdata/fac.wasm
+//go:embed testdata/fac.wasm
 var facWasm []byte
 
-//go:embed internal/integration_test/vs/testdata/mem_grow.wasm
+//go:embed testdata/mem_grow.wasm
 var memGrowWasm []byte
 
 func TestCompilationCache(t *testing.T) {