From ee0839aca4e0b665398a67796088205c37ab0eed Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Thu, 29 Feb 2024 14:39:39 +0000 Subject: [PATCH 1/4] build: Add `-threads` variant of Wasm stdlib build This patch adds a `-threads` variant of the Wasm stdlib build, which has completely different ABI and target triple. Now we build non-threaded and threaded variants when `--build-wasm-stdlib` is enabled. --- .../build_script_invocation.py | 4 ++ .../swift_build_support/products/__init__.py | 8 ++- .../products/wasisysroot.py | 63 +++++++++++++++---- .../products/wasmstdlib.py | 19 +++++- 4 files changed, 79 insertions(+), 15 deletions(-) diff --git a/utils/swift_build_support/swift_build_support/build_script_invocation.py b/utils/swift_build_support/swift_build_support/build_script_invocation.py index 3e5a9a306b1eb..0365dbaab96d6 100644 --- a/utils/swift_build_support/swift_build_support/build_script_invocation.py +++ b/utils/swift_build_support/swift_build_support/build_script_invocation.py @@ -676,10 +676,14 @@ def compute_product_pipelines(self): is_enabled=self.args.build_wasmstdlib) builder.add_product(products.WasmLLVMRuntimeLibs, is_enabled=self.args.build_wasmstdlib) + builder.add_product(products.WasmThreadsLLVMRuntimeLibs, + is_enabled=self.args.build_wasmstdlib) builder.add_product(products.WasmKit, is_enabled=self.args.build_wasmkit) builder.add_product(products.WasmStdlib, is_enabled=self.args.build_wasmstdlib) + builder.add_product(products.WasmThreadsStdlib, + is_enabled=self.args.build_wasmstdlib) # Keep SwiftDriver at last. # swift-driver's integration with the build scripts is not fully diff --git a/utils/swift_build_support/swift_build_support/products/__init__.py b/utils/swift_build_support/swift_build_support/products/__init__.py index 583f4eafaa5e4..51dc3ba018d9b 100644 --- a/utils/swift_build_support/swift_build_support/products/__init__.py +++ b/utils/swift_build_support/swift_build_support/products/__init__.py @@ -37,9 +37,9 @@ from .swiftpm import SwiftPM from .swiftsyntax import SwiftSyntax from .tsan_libdispatch import TSanLibDispatch -from .wasisysroot import WASILibc, WasmLLVMRuntimeLibs +from .wasisysroot import WASILibc, WasmLLVMRuntimeLibs, WasmThreadsLLVMRuntimeLibs from .wasmkit import WasmKit -from .wasmstdlib import WasmStdlib +from .wasmstdlib import WasmStdlib, WasmThreadsStdlib from .xctest import XCTest from .zlib import Zlib @@ -76,5 +76,7 @@ 'WASILibc', 'WasmLLVMRuntimeLibs', 'WasmKit', - 'WasmStdlib' + 'WasmStdlib', + 'WasmThreadsLLVMRuntimeLibs', + 'WasmThreadsStdlib', ] diff --git a/utils/swift_build_support/swift_build_support/products/wasisysroot.py b/utils/swift_build_support/swift_build_support/products/wasisysroot.py index 11abc9f4cff0a..a16871b7eb681 100644 --- a/utils/swift_build_support/swift_build_support/products/wasisysroot.py +++ b/utils/swift_build_support/swift_build_support/products/wasisysroot.py @@ -44,6 +44,11 @@ def should_install(self, host_target): return False def build(self, host_target): + self._build(host_target) + self._build(host_target, thread_model='posix', + target_triple='wasm32-wasip1-threads') + + def _build(self, host_target, thread_model='single', target_triple='wasm32-wasi'): build_root = os.path.dirname(self.build_dir) llvm_build_bin_dir = os.path.join( '..', build_root, '%s-%s' % ('llvm', host_target), 'bin') @@ -66,12 +71,14 @@ def build(self, host_target): # https://github.com/llvm/llvm-project/commit/7dd387d2971d7759cadfffeb2082439f6c7ddd49 '--old-file=check-symbols', '-C', self.source_dir, - 'OBJDIR=' + os.path.join(self.build_dir, 'obj'), + 'OBJDIR=' + os.path.join(self.build_dir, 'obj-' + thread_model), 'SYSROOT=' + sysroot_build_dir, 'INSTALL_DIR=' + WASILibc.sysroot_install_path(build_root), 'CC=' + os.path.join(clang_tools_path, 'clang'), 'AR=' + os.path.join(llvm_tools_path, 'llvm-ar'), 'NM=' + os.path.join(llvm_tools_path, 'llvm-nm'), + 'THREAD_MODEL=' + thread_model, + 'TARGET_TRIPLE=' + target_triple, ]) @classmethod @@ -121,18 +128,31 @@ def should_install(self, host_target): return False def build(self, host_target): + self._build(host_target) + + def _build(self, host_target, enable_wasi_threads=False): build_root = os.path.dirname(self.build_dir) llvm_build_bin_dir = os.path.join( '..', build_root, '%s-%s' % ('llvm', host_target), 'bin') llvm_tools_path = self.args.native_llvm_tools_path or llvm_build_bin_dir clang_tools_path = self.args.native_clang_tools_path or llvm_build_bin_dir + cmake_has_threads = 'TRUE' if enable_wasi_threads else 'FALSE' + self.cmake_options.define('CMAKE_SYSROOT:PATH', WASILibc.sysroot_build_path(build_root, host_target)) + enable_runtimes = ['libcxx', 'libcxxabi'] + if not enable_wasi_threads: + # compiler-rt can be shared between wasi and wasip1-threads + enable_runtimes.append('compiler-rt') self.cmake_options.define('LLVM_ENABLE_RUNTIMES:STRING', - 'libcxx;libcxxabi;compiler-rt') - self.cmake_options.define('LIBCXX_LIBDIR_SUFFIX:STRING', '/wasm32-wasi') - self.cmake_options.define('LIBCXXABI_LIBDIR_SUFFIX:STRING', '/wasm32-wasi') + ';'.join(enable_runtimes)) + + libdir_suffix = '/wasm32-wasi' + if enable_wasi_threads: + libdir_suffix = '/wasm32-wasi-threads' + self.cmake_options.define('LIBCXX_LIBDIR_SUFFIX:STRING', libdir_suffix) + self.cmake_options.define('LIBCXXABI_LIBDIR_SUFFIX:STRING', libdir_suffix) self.cmake_options.define('CMAKE_STAGING_PREFIX:PATH', '/') self.cmake_options.define('COMPILER_RT_DEFAULT_TARGET_ARCH:STRING', 'wasm32') @@ -157,19 +177,27 @@ def build(self, host_target): os.path.join(clang_tools_path, 'clang')) self.cmake_options.define('CMAKE_CXX_COMPILER:STRING', os.path.join(clang_tools_path, 'clang++')) + + c_flags = [] # Explicitly disable exceptions even though it's usually implicitly disabled by # LIBCXX_ENABLE_EXCEPTIONS because the CMake feature check fails to detect # -fno-exceptions support in clang due to missing compiler-rt while configuring # as mono project. - self.cmake_options.define('CMAKE_CXX_FLAGS:STRING', '-fno-exceptions') + cxx_flags = ['-fno-exceptions'] + if enable_wasi_threads: + c_flags.append('-pthread') + cxx_flags.append('-pthread') + self.cmake_options.define('CMAKE_C_FLAGS:STRING', ' '.join(c_flags)) + self.cmake_options.define('CMAKE_CXX_FLAGS:STRING', ' '.join(cxx_flags)) - self.cmake_options.define('CMAKE_C_COMPILER_TARGET:STRING', 'wasm32-wasi') - self.cmake_options.define('CMAKE_CXX_COMPILER_TARGET:STRING', 'wasm32-wasi') + target_triple = 'wasm32-wasi-threads' if enable_wasi_threads else 'wasm32-wasi' + self.cmake_options.define('CMAKE_C_COMPILER_TARGET:STRING', target_triple) + self.cmake_options.define('CMAKE_CXX_COMPILER_TARGET:STRING', target_triple) self.cmake_options.define('CXX_SUPPORTS_CXX11:BOOL', 'TRUE') - self.cmake_options.define('LIBCXX_ENABLE_THREADS:BOOL', 'FALSE') - self.cmake_options.define('LIBCXX_HAS_PTHREAD_API:BOOL', 'FALSE') + self.cmake_options.define('LIBCXX_ENABLE_THREADS:BOOL', cmake_has_threads) + self.cmake_options.define('LIBCXX_HAS_PTHREAD_API:BOOL', cmake_has_threads) self.cmake_options.define('LIBCXX_HAS_EXTERNAL_THREAD_API:BOOL', 'FALSE') self.cmake_options.define('LIBCXX_BUILD_EXTERNAL_THREAD_LIBRARY:BOOL', 'FALSE') self.cmake_options.define('LIBCXX_HAS_WIN32_THREAD_API:BOOL', 'FALSE') @@ -184,8 +212,8 @@ def build(self, host_target): self.cmake_options.define('LIBCXXABI_ENABLE_EXCEPTIONS:BOOL', 'FALSE') self.cmake_options.define('LIBCXXABI_ENABLE_SHARED:BOOL', 'FALSE') self.cmake_options.define('LIBCXXABI_SILENT_TERMINATE:BOOL', 'TRUE') - self.cmake_options.define('LIBCXXABI_ENABLE_THREADS:BOOL', 'FALSE') - self.cmake_options.define('LIBCXXABI_HAS_PTHREAD_API:BOOL', 'FALSE') + self.cmake_options.define('LIBCXXABI_ENABLE_THREADS:BOOL', cmake_has_threads) + self.cmake_options.define('LIBCXXABI_HAS_PTHREAD_API:BOOL', cmake_has_threads) self.cmake_options.define('LIBCXXABI_HAS_EXTERNAL_THREAD_API:BOOL', 'FALSE') self.cmake_options.define('LIBCXXABI_BUILD_EXTERNAL_THREAD_LIBRARY:BOOL', 'FALSE') @@ -201,3 +229,16 @@ def build(self, host_target): @classmethod def get_dependencies(cls): return [WASILibc, llvm.LLVM] + + +class WasmThreadsLLVMRuntimeLibs(WasmLLVMRuntimeLibs): + def build(self, host_target): + self._build(host_target, enable_wasi_threads=True) + + build_root = os.path.dirname(self.build_dir) + wasi_sysroot = WASILibc.sysroot_install_path(build_root) + # Copy compiler-rt os dirs to the WASI variant + os_dir = os.path.join(wasi_sysroot, 'lib', 'wasip1') + if os.path.exists(os_dir): + shell.rmtree(os_dir) + shell.copytree(os.path.join(wasi_sysroot, 'lib', 'wasi'), os_dir) diff --git a/utils/swift_build_support/swift_build_support/products/wasmstdlib.py b/utils/swift_build_support/swift_build_support/products/wasmstdlib.py index 9282b9183e35a..4168629c6770b 100644 --- a/utils/swift_build_support/swift_build_support/products/wasmstdlib.py +++ b/utils/swift_build_support/swift_build_support/products/wasmstdlib.py @@ -85,7 +85,6 @@ def build(self, host_target): self.cmake_options.define( 'SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY:BOOL', 'TRUE') self.cmake_options.define('SWIFT_ENABLE_DISPATCH:BOOL', 'FALSE') - self.cmake_options.define('SWIFT_THREADING_PACKAGE:STRING', 'none') self.cmake_options.define( 'SWIFT_STDLIB_SUPPORTS_BACKTRACE_REPORTING:BOOL', 'FALSE') self.cmake_options.define('SWIFT_STDLIB_HAS_DLADDR:BOOL', 'FALSE') @@ -98,6 +97,8 @@ def build(self, host_target): os.path.join(self.source_dir, '..', 'swift-experimental-string-processing')) + self.add_extra_cmake_options() + # Test configuration self.cmake_options.define('SWIFT_INCLUDE_TESTS:BOOL', 'TRUE') self.cmake_options.define('SWIFT_ENABLE_SOURCEKIT_TESTS:BOOL', 'FALSE') @@ -119,6 +120,9 @@ def build(self, host_target): self.build_with_cmake([], self._build_variant, [], prefer_native_toolchain=True) + def add_extra_cmake_options(self): + self.cmake_options.define('SWIFT_THREADING_PACKAGE:STRING', 'none') + def test(self, host_target): build_root = os.path.dirname(self.build_dir) bin_paths = [ @@ -170,3 +174,16 @@ def get_dependencies(cls): wasisysroot.WasmLLVMRuntimeLibs, wasmkit.WasmKit, swift.Swift] + + +class WasmThreadsStdlib(WasmStdlib): + def add_extra_cmake_options(self): + self.cmake_options.define('SWIFT_THREADING_PACKAGE:STRING', 'pthreads') + self.cmake_options.define('SWIFT_STDLIB_EXTRA_C_COMPILE_FLAGS:STRING', + '-mthread-model;posix;-pthread;' + '-ftls-model=local-exec') + self.cmake_options.define('SWIFT_STDLIB_EXTRA_SWIFT_COMPILE_FLAGS:STRING', + '-Xcc;-matomics;-Xcc;-mbulk-memory;' + '-Xcc;-mthread-model;-Xcc;posix;' + '-Xcc;-pthread;-Xcc;-ftls-model=local-exec') + self.cmake_options.define('SWIFT_ENABLE_WASI_THREADS:BOOL', 'TRUE') From ca5e7a6bdda5080332d94a80e75402f08d0354bf Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Thu, 21 Mar 2024 01:46:36 +0000 Subject: [PATCH 2/4] build: Rename wasi-threads to wasip1-threads The WASI community is transitioning to a new naming for the "preview" version in the target triple: wasm32-wasi -> wasm32-wasip1. At this moment, we keep the old triple wasm32-wasi because it's already widely used, but we should start using the new triple threaded target. LLVM checks only if the OS field *starts* with "wasi", so "wasip1" is still considered a valid `isOSWASI()` target. See: https://github.com/WebAssembly/wasi-libc/pull/478 --- cmake/modules/SwiftConfigureSDK.cmake | 2 +- .../swift_build_support/products/wasisysroot.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cmake/modules/SwiftConfigureSDK.cmake b/cmake/modules/SwiftConfigureSDK.cmake index f2923d7c371c3..e18dd0b7524a9 100644 --- a/cmake/modules/SwiftConfigureSDK.cmake +++ b/cmake/modules/SwiftConfigureSDK.cmake @@ -423,7 +423,7 @@ macro(configure_sdk_unix name architectures) endif() set(SWIFT_SDK_WASI_ARCH_wasm32_PATH "${SWIFT_WASI_SYSROOT_PATH}") if(SWIFT_ENABLE_WASI_THREADS) - set(SWIFT_SDK_WASI_ARCH_wasm32_TRIPLE "wasm32-unknown-wasi-threads") + set(SWIFT_SDK_WASI_ARCH_wasm32_TRIPLE "wasm32-unknown-wasip1-threads") else() set(SWIFT_SDK_WASI_ARCH_wasm32_TRIPLE "wasm32-unknown-wasi") endif() diff --git a/utils/swift_build_support/swift_build_support/products/wasisysroot.py b/utils/swift_build_support/swift_build_support/products/wasisysroot.py index a16871b7eb681..b77d2ec14468f 100644 --- a/utils/swift_build_support/swift_build_support/products/wasisysroot.py +++ b/utils/swift_build_support/swift_build_support/products/wasisysroot.py @@ -150,7 +150,7 @@ def _build(self, host_target, enable_wasi_threads=False): libdir_suffix = '/wasm32-wasi' if enable_wasi_threads: - libdir_suffix = '/wasm32-wasi-threads' + libdir_suffix = '/wasm32-wasip1-threads' self.cmake_options.define('LIBCXX_LIBDIR_SUFFIX:STRING', libdir_suffix) self.cmake_options.define('LIBCXXABI_LIBDIR_SUFFIX:STRING', libdir_suffix) self.cmake_options.define('CMAKE_STAGING_PREFIX:PATH', '/') @@ -190,7 +190,10 @@ def _build(self, host_target, enable_wasi_threads=False): self.cmake_options.define('CMAKE_C_FLAGS:STRING', ' '.join(c_flags)) self.cmake_options.define('CMAKE_CXX_FLAGS:STRING', ' '.join(cxx_flags)) - target_triple = 'wasm32-wasi-threads' if enable_wasi_threads else 'wasm32-wasi' + if enable_wasi_threads: + target_triple = 'wasm32-wasip1-threads' + else: + target_triple = 'wasm32-wasi' self.cmake_options.define('CMAKE_C_COMPILER_TARGET:STRING', target_triple) self.cmake_options.define('CMAKE_CXX_COMPILER_TARGET:STRING', target_triple) From c8b1dc7ed0adef3a20fee4cf97eb7e5bd4643173 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Fri, 29 Mar 2024 10:22:33 +0000 Subject: [PATCH 3/4] test: Disable executable lit tests with WASI threads for now We should enable them once WasmKit supports WASI threads. --- .../swift_build_support/products/wasmstdlib.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/utils/swift_build_support/swift_build_support/products/wasmstdlib.py b/utils/swift_build_support/swift_build_support/products/wasmstdlib.py index 4168629c6770b..f6c1d46aa4042 100644 --- a/utils/swift_build_support/swift_build_support/products/wasmstdlib.py +++ b/utils/swift_build_support/swift_build_support/products/wasmstdlib.py @@ -133,7 +133,7 @@ def test(self, host_target): wasmkit_build_path = os.path.join( build_root, '%s-%s' % ('wasmkit', host_target)) wasmkit_bin_path = wasmkit.WasmKit.cli_file_path(wasmkit_build_path) - if not os.path.exists(wasmkit_bin_path): + if not os.path.exists(wasmkit_bin_path) or not self.should_test_executable(): test_target = "check-swift-only_non_executable-wasi-wasm32-custom" else: test_target = "check-swift-wasi-wasm32-custom" @@ -147,6 +147,9 @@ def test(self, host_target): } self.test_with_cmake(None, [test_target], self._build_variant, [], test_env=env) + def should_test_executable(self): + return True + @property def _build_variant(self): return self.args.build_variant @@ -177,6 +180,10 @@ def get_dependencies(cls): class WasmThreadsStdlib(WasmStdlib): + def should_test_executable(self): + # TODO(katei): Enable tests once WasmKit supports WASI threads + return False + def add_extra_cmake_options(self): self.cmake_options.define('SWIFT_THREADING_PACKAGE:STRING', 'pthreads') self.cmake_options.define('SWIFT_STDLIB_EXTRA_C_COMPILE_FLAGS:STRING', From e73bc1202c2036ebb992c090a2e4e4411fe28995 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Fri, 29 Mar 2024 11:15:03 +0000 Subject: [PATCH 4/4] test: Don't distinguish between wasi, wasip1 to use OS=wasi condition We already have several uses of `OS=wasi` in the test suites, and the condition should match all the wasi targets regardless of the version. --- test/lit.cfg | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/lit.cfg b/test/lit.cfg index 59781fc96d28c..b8fd2c1125d59 100644 --- a/test/lit.cfg +++ b/test/lit.cfg @@ -400,6 +400,11 @@ if run_vers.endswith('-simulator'): else: run_environment='' +# Don't distinguish between wasi, wasip1, and so on to use OS=wasi condition in +# the test suites. +if run_os.startswith('wasi'): + run_os = 'wasi' + # Parse the host triple (host_cpu, host_vendor, host_os, host_vers) = re.match('([^-]+)-([^-]+)-([^0-9-]+)(.*)', config.host_triple).groups()