Update LLVM to 18.1.2 and add wasm-component-ld (#402)

* Update LLVM to 18.1.2 and add wasm-component-ld

This commit has two intertwined changes within it. First the LLVM
submodule is updated to the 18.1.2 release branch. This alone would
cause CI and tests to fail due to differing behavior for the
`wasm32-wasip2` target. To fix these test failures the
`wasm-component-ld` tool is added to the build. This tool is a
Rust-written tool and installed via `cargo install` as part of the build
at a pinned version written in the `Makefile`. This linker, used for
components, is then used for the `wasm32-wasip2` target.

Tests and CI are then updated to skip the need to have `wasm-tools` or
the adapter for WASI when making components since that's now the job of
`wasm-component-ld`. This then necessitated some changes to tests too.

* Add back accidentally deleted CI

* Inherit tools on PATH on Windows

* Add Rust to docker build

* Put rust in a different location in docker

* Try to fix permissions

* Review comments

* Revert changes to test outputs

.github/workflows/main.yml

@@ -42,12 +42,6 @@ jobs:
         uses: bytecodealliance/actions/wasmtime/setup@v1
         with:
           version: "18.0.2"
-      - name: Setup `wasm-tools` for tests
-        uses: bytecodealliance/actions/wasm-tools/setup@v1
-        with:
-          version: "1.201.0"
-      - name: Download command adapter for tests
-        run: curl -f -L --retry 5 -o ${{ runner.temp }}/wasi_snapshot_preview1.command.wasm https://github.com/bytecodealliance/wasmtime/releases/download/v18.0.2/wasi_snapshot_preview1.command.wasm
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
@@ -67,7 +61,7 @@ jobs:
         run: NINJA_FLAGS=-v make package LLVM_CMAKE_FLAGS=-DLLVM_CCACHE_BUILD=ON
         shell: bash
       - name: Run the testsuite
-        run: NINJA_FLAGS=-v make check RUNTIME=wasmtime ADAPTER=${{ runner.temp }}/wasi_snapshot_preview1.command.wasm WASM_TOOLS=wasm-tools
+        run: NINJA_FLAGS=-v make check RUNTIME=wasmtime
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
@@ -107,6 +101,7 @@ jobs:
           msystem: ${{ matrix.sys }}
           update: true
           release: false
+          path-type: inherit
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0

Dockerfile

@@ -28,6 +28,11 @@ RUN curl -sSLO https://github.com/Kitware/CMake/releases/download/v3.25.1/cmake-
   && mv cmake-3.25.1-linux-x86_64 /opt/cmake
 ENV PATH /opt/cmake/bin:$PATH
 
+ENV RUSTUP_HOME=/rust/rustup CARGO_HOME=/rust/cargo PATH=$PATH:/rust/cargo/bin
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
+  sh -s -- -y --profile=minimal && \
+  chmod -R a+w /rust
+
 RUN groupadd -g ${GID} builder && \
   useradd --create-home --uid ${UID} --gid ${GID} builder
 USER builder

Makefile

@@ -54,7 +54,7 @@ default: build
 	@echo "Use -fdebug-prefix-map=$(ROOT_DIR)=wasisdk://v$(VERSION)"
 
 check:
-	TARGETS="$(TARGETS)" tests/run.sh "$(BUILD_PREFIX)" "$(RUNTIME)" "$(ADAPTER)" "$(WASM_TOOLS)"
+	TARGETS="$(TARGETS)" tests/run.sh "$(BUILD_PREFIX)" "$(RUNTIME)"
 
 clean:
 	rm -rf build $(DESTDIR)
@@ -122,6 +122,14 @@ build/llvm.BUILT:
 		llvm-config
 	touch build/llvm.BUILT
 
+# Build the `wasm-component-ld` linker from source via `cargo install`. This is
+# used for the `wasm32-wasip2` target natively by Clang. Note that `--root`
+# passed to `cargo install` will place it in the output directory automatically.
+build/wasm-component-ld.BUILT: build/llvm.BUILT
+	cargo install [email protected] --root $(BUILD_PREFIX)
+	touch build/wasm-component-ld.BUILT
+
+
 # Flags for running `make` in wasi-libc
 # $(1): the target that's being built
 WASI_LIBC_MAKEFLAGS = \
@@ -132,7 +140,7 @@ WASI_LIBC_MAKEFLAGS = \
 	SYSROOT=$(BUILD_PREFIX)/share/wasi-sysroot \
 	TARGET_TRIPLE=$(1)
 
-build/wasi-libc.BUILT: build/compiler-rt.BUILT
+build/wasi-libc.BUILT: build/compiler-rt.BUILT build/wasm-component-ld.BUILT
 	$(MAKE) $(call WASI_LIBC_MAKEFLAGS,wasm32-wasi) default libc_so
 	$(MAKE) $(call WASI_LIBC_MAKEFLAGS,wasm32-wasip1) default libc_so
 	$(MAKE) $(call WASI_LIBC_MAKEFLAGS,wasm32-wasip2) WASI_SNAPSHOT=p2 default libc_so

tests/run.sh

@@ -17,17 +17,8 @@ wasi_sdk="$1"
 # Determine the wasm runtime to use, if one is provided.
 if [ $# -gt 1 ]; then
     runwasi="$2"
-    if [ $# -gt 3 ]; then
-        adapter="$3"
-        wasm_tools="$4"
-    else
-        adapter=""
-        wasm_tools=""
-    fi
 else
     runwasi=""
-    adapter=""
-    wasm_tools=""
 fi
 
 testdir=$(dirname $0)
@@ -45,13 +36,13 @@ for target in $TARGETS; do
         echo "===== Testing compile-only with $options ====="
         for file in *.c; do
             echo "Testing compile-only $file..."
-            ../testcase.sh "$target" "" "" "" "$wasi_sdk/bin/clang" "$options --target=$target" "$file"
-            ../testcase.sh "$target" "" "" "" "$wasi_sdk/bin/$target-clang" "$options" "$file"
+            ../testcase.sh "$target" "" "$wasi_sdk/bin/clang" "$options --target=$target" "$file"
+            ../testcase.sh "$target" "" "$wasi_sdk/bin/$target-clang" "$options" "$file"
         done
         for file in *.cc; do
             echo "Testing compile-only $file..."
-            ../testcase.sh "$target" "" "" "" "$wasi_sdk/bin/clang++" "$options --target=$target -fno-exceptions" "$file"
-            ../testcase.sh "$target" "" "" "" "$wasi_sdk/bin/$target-clang++" "$options -fno-exceptions" "$file"
+            ../testcase.sh "$target" "" "$wasi_sdk/bin/clang++" "$options --target=$target -fno-exceptions" "$file"
+            ../testcase.sh "$target" "" "$wasi_sdk/bin/$target-clang++" "$options -fno-exceptions" "$file"
         done
     done
     cd - >/dev/null
@@ -61,13 +52,13 @@ for target in $TARGETS; do
         echo "===== Testing with $options ====="
         for file in *.c; do
             echo "Testing $file..."
-            ../testcase.sh "$target" "$runwasi" "$adapter" "$wasm_tools" "$wasi_sdk/bin/clang" "$options --target=$target" "$file"
-            ../testcase.sh "$target" "$runwasi" "$adapter" "$wasm_tools" "$wasi_sdk/bin/$target-clang" "$options" "$file"
+            ../testcase.sh "$target" "$runwasi" "$wasi_sdk/bin/clang" "$options --target=$target" "$file"
+            ../testcase.sh "$target" "$runwasi" "$wasi_sdk/bin/$target-clang" "$options" "$file"
         done
         for file in *.cc; do
             echo "Testing $file..."
-            ../testcase.sh "$target" "$runwasi" "$adapter" "$wasm_tools" "$wasi_sdk/bin/clang++" "$options --target=$target -fno-exceptions" "$file"
-            ../testcase.sh "$target" "$runwasi" "$adapter" "$wasm_tools" "$wasi_sdk/bin/$target-clang++" "$options -fno-exceptions" "$file"
+            ../testcase.sh "$target" "$runwasi" "$wasi_sdk/bin/clang++" "$options --target=$target -fno-exceptions" "$file"
+            ../testcase.sh "$target" "$runwasi" "$wasi_sdk/bin/$target-clang++" "$options -fno-exceptions" "$file"
         done
     done
     cd - >/dev/null

tests/testcase.sh

@@ -8,18 +8,15 @@ set -ueo pipefail
 # script, so don't do anything fancy.
 target="$1"
 runwasi="$2"
-adapter="$3"
-wasm_tools="$4"
-compiler="$5"
-options="$6"
-input="$7"
+compiler="$3"
+options="$4"
+input="$5"
 
 # Compile names for generated files.
 wasm="$input.$options.wasm"
 stdout_observed="$input.$options.stdout.observed"
 stderr_observed="$input.$options.stderr.observed"
 exit_status_observed="$input.$options.exit_status.observed"
-run_args=""
 
 # Optionally load compiler options from a file.
 if [ -e "$input.options" ]; then
@@ -44,11 +41,6 @@ if [ "$runwasi" == "" ]; then
     exit 0
 fi
 
-if [ "$target" == "wasm32-wasip2" -a -n "$adapter" -a -n "$wasm_tools" ]; then
-    "$wasm_tools" component new --adapt "$adapter" "$wasm" -o "$wasm"
-    run_args="--wasm component-model"
-fi
-
 # Determine the input file to write to stdin.
 if [ -e "$input.stdin" ]; then
   stdin="$input.stdin"
@@ -74,7 +66,7 @@ fi
 
 # Run the test, capturing stdout, stderr, and the exit status.
 exit_status=0
-"$runwasi" $run_args $env $dir "$wasm" $dirarg \
+"$runwasi" $env $dir "$wasm" $dirarg \
     < "$stdin" \
     > "$stdout_observed" \
     2> "$stderr_observed" \