build: Add -threads variant of Wasm stdlib build

cmake/modules/SwiftConfigureSDK.cmake

@@ -423,7 +423,7 @@ macro(configure_sdk_unix name architectures)
         endif()
         set(SWIFT_SDK_WASI_ARCH_wasm32_PATH "${SWIFT_WASI_SYSROOT_PATH}")
         if(SWIFT_ENABLE_WASI_THREADS)
-          set(SWIFT_SDK_WASI_ARCH_wasm32_TRIPLE "wasm32-unknown-wasi-threads")
+          set(SWIFT_SDK_WASI_ARCH_wasm32_TRIPLE "wasm32-unknown-wasip1-threads")
         else()
           set(SWIFT_SDK_WASI_ARCH_wasm32_TRIPLE "wasm32-unknown-wasi")
         endif()

test/lit.cfg

@@ -400,6 +400,11 @@ if run_vers.endswith('-simulator'):
 else:
     run_environment=''
 
+# Don't distinguish between wasi, wasip1, and so on to use OS=wasi condition in
+# the test suites.
+if run_os.startswith('wasi'):
+    run_os = 'wasi'
+
 # Parse the host triple
 (host_cpu, host_vendor, host_os, host_vers) = re.match('([^-]+)-([^-]+)-([^0-9-]+)(.*)', config.host_triple).groups()
 

utils/swift_build_support/swift_build_support/build_script_invocation.py

@@ -676,10 +676,14 @@ def compute_product_pipelines(self):
                             is_enabled=self.args.build_wasmstdlib)
         builder.add_product(products.WasmLLVMRuntimeLibs,
                             is_enabled=self.args.build_wasmstdlib)
+        builder.add_product(products.WasmThreadsLLVMRuntimeLibs,
+                            is_enabled=self.args.build_wasmstdlib)
         builder.add_product(products.WasmKit,
                             is_enabled=self.args.build_wasmkit)
         builder.add_product(products.WasmStdlib,
                             is_enabled=self.args.build_wasmstdlib)
+        builder.add_product(products.WasmThreadsStdlib,
+                            is_enabled=self.args.build_wasmstdlib)
 
         # Keep SwiftDriver at last.
         # swift-driver's integration with the build scripts is not fully

utils/swift_build_support/swift_build_support/products/__init__.py

@@ -37,9 +37,9 @@
 from .swiftpm import SwiftPM
 from .swiftsyntax import SwiftSyntax
 from .tsan_libdispatch import TSanLibDispatch
-from .wasisysroot import WASILibc, WasmLLVMRuntimeLibs
+from .wasisysroot import WASILibc, WasmLLVMRuntimeLibs, WasmThreadsLLVMRuntimeLibs
 from .wasmkit import WasmKit
-from .wasmstdlib import WasmStdlib
+from .wasmstdlib import WasmStdlib, WasmThreadsStdlib
 from .xctest import XCTest
 from .zlib import Zlib
 
@@ -76,5 +76,7 @@
     'WASILibc',
     'WasmLLVMRuntimeLibs',
     'WasmKit',
-    'WasmStdlib'
+    'WasmStdlib',
+    'WasmThreadsLLVMRuntimeLibs',
+    'WasmThreadsStdlib',
 ]

utils/swift_build_support/swift_build_support/products/wasisysroot.py

@@ -44,6 +44,11 @@ def should_install(self, host_target):
         return False
 
     def build(self, host_target):
+        self._build(host_target)
+        self._build(host_target, thread_model='posix',
+                    target_triple='wasm32-wasip1-threads')
+
+    def _build(self, host_target, thread_model='single', target_triple='wasm32-wasi'):
         build_root = os.path.dirname(self.build_dir)
         llvm_build_bin_dir = os.path.join(
             '..', build_root, '%s-%s' % ('llvm', host_target), 'bin')
@@ -66,12 +71,14 @@ def build(self, host_target):
             # https://github.com/llvm/llvm-project/commit/7dd387d2971d7759cadfffeb2082439f6c7ddd49
             '--old-file=check-symbols',
             '-C', self.source_dir,
-            'OBJDIR=' + os.path.join(self.build_dir, 'obj'),
+            'OBJDIR=' + os.path.join(self.build_dir, 'obj-' + thread_model),
             'SYSROOT=' + sysroot_build_dir,
             'INSTALL_DIR=' + WASILibc.sysroot_install_path(build_root),
             'CC=' + os.path.join(clang_tools_path, 'clang'),
             'AR=' + os.path.join(llvm_tools_path, 'llvm-ar'),
             'NM=' + os.path.join(llvm_tools_path, 'llvm-nm'),
+            'THREAD_MODEL=' + thread_model,
+            'TARGET_TRIPLE=' + target_triple,
         ])
 
     @classmethod
@@ -121,18 +128,31 @@ def should_install(self, host_target):
         return False
 
     def build(self, host_target):
+        self._build(host_target)
+
+    def _build(self, host_target, enable_wasi_threads=False):
         build_root = os.path.dirname(self.build_dir)
         llvm_build_bin_dir = os.path.join(
             '..', build_root, '%s-%s' % ('llvm', host_target), 'bin')
         llvm_tools_path = self.args.native_llvm_tools_path or llvm_build_bin_dir
         clang_tools_path = self.args.native_clang_tools_path or llvm_build_bin_dir
 
+        cmake_has_threads = 'TRUE' if enable_wasi_threads else 'FALSE'
+
         self.cmake_options.define('CMAKE_SYSROOT:PATH',
                                   WASILibc.sysroot_build_path(build_root, host_target))
+        enable_runtimes = ['libcxx', 'libcxxabi']
+        if not enable_wasi_threads:
+            # compiler-rt can be shared between wasi and wasip1-threads
+            enable_runtimes.append('compiler-rt')
         self.cmake_options.define('LLVM_ENABLE_RUNTIMES:STRING',
-                                  'libcxx;libcxxabi;compiler-rt')
-        self.cmake_options.define('LIBCXX_LIBDIR_SUFFIX:STRING', '/wasm32-wasi')
-        self.cmake_options.define('LIBCXXABI_LIBDIR_SUFFIX:STRING', '/wasm32-wasi')
+                                  ';'.join(enable_runtimes))
+
+        libdir_suffix = '/wasm32-wasi'
+        if enable_wasi_threads:
+            libdir_suffix = '/wasm32-wasip1-threads'
+        self.cmake_options.define('LIBCXX_LIBDIR_SUFFIX:STRING', libdir_suffix)
+        self.cmake_options.define('LIBCXXABI_LIBDIR_SUFFIX:STRING', libdir_suffix)
         self.cmake_options.define('CMAKE_STAGING_PREFIX:PATH', '/')
 
         self.cmake_options.define('COMPILER_RT_DEFAULT_TARGET_ARCH:STRING', 'wasm32')
@@ -157,19 +177,30 @@ def build(self, host_target):
                                   os.path.join(clang_tools_path, 'clang'))
         self.cmake_options.define('CMAKE_CXX_COMPILER:STRING',
                                   os.path.join(clang_tools_path, 'clang++'))
+
+        c_flags = []
         # Explicitly disable exceptions even though it's usually implicitly disabled by
         # LIBCXX_ENABLE_EXCEPTIONS because the CMake feature check fails to detect
         # -fno-exceptions support in clang due to missing compiler-rt while configuring
         # as mono project.
-        self.cmake_options.define('CMAKE_CXX_FLAGS:STRING', '-fno-exceptions')
-
-        self.cmake_options.define('CMAKE_C_COMPILER_TARGET:STRING', 'wasm32-wasi')
-        self.cmake_options.define('CMAKE_CXX_COMPILER_TARGET:STRING', 'wasm32-wasi')
+        cxx_flags = ['-fno-exceptions']
+        if enable_wasi_threads:
+            c_flags.append('-pthread')
+            cxx_flags.append('-pthread')
+        self.cmake_options.define('CMAKE_C_FLAGS:STRING', ' '.join(c_flags))
+        self.cmake_options.define('CMAKE_CXX_FLAGS:STRING', ' '.join(cxx_flags))
+
+        if enable_wasi_threads:
+            target_triple = 'wasm32-wasip1-threads'
+        else:
+            target_triple = 'wasm32-wasi'
+        self.cmake_options.define('CMAKE_C_COMPILER_TARGET:STRING', target_triple)
+        self.cmake_options.define('CMAKE_CXX_COMPILER_TARGET:STRING', target_triple)
 
         self.cmake_options.define('CXX_SUPPORTS_CXX11:BOOL', 'TRUE')
 
-        self.cmake_options.define('LIBCXX_ENABLE_THREADS:BOOL', 'FALSE')
-        self.cmake_options.define('LIBCXX_HAS_PTHREAD_API:BOOL', 'FALSE')
+        self.cmake_options.define('LIBCXX_ENABLE_THREADS:BOOL', cmake_has_threads)
+        self.cmake_options.define('LIBCXX_HAS_PTHREAD_API:BOOL', cmake_has_threads)
         self.cmake_options.define('LIBCXX_HAS_EXTERNAL_THREAD_API:BOOL', 'FALSE')
         self.cmake_options.define('LIBCXX_BUILD_EXTERNAL_THREAD_LIBRARY:BOOL', 'FALSE')
         self.cmake_options.define('LIBCXX_HAS_WIN32_THREAD_API:BOOL', 'FALSE')
@@ -184,8 +215,8 @@ def build(self, host_target):
         self.cmake_options.define('LIBCXXABI_ENABLE_EXCEPTIONS:BOOL', 'FALSE')
         self.cmake_options.define('LIBCXXABI_ENABLE_SHARED:BOOL', 'FALSE')
         self.cmake_options.define('LIBCXXABI_SILENT_TERMINATE:BOOL', 'TRUE')
-        self.cmake_options.define('LIBCXXABI_ENABLE_THREADS:BOOL', 'FALSE')
-        self.cmake_options.define('LIBCXXABI_HAS_PTHREAD_API:BOOL', 'FALSE')
+        self.cmake_options.define('LIBCXXABI_ENABLE_THREADS:BOOL', cmake_has_threads)
+        self.cmake_options.define('LIBCXXABI_HAS_PTHREAD_API:BOOL', cmake_has_threads)
         self.cmake_options.define('LIBCXXABI_HAS_EXTERNAL_THREAD_API:BOOL', 'FALSE')
         self.cmake_options.define('LIBCXXABI_BUILD_EXTERNAL_THREAD_LIBRARY:BOOL',
                                   'FALSE')
@@ -201,3 +232,16 @@ def build(self, host_target):
     @classmethod
     def get_dependencies(cls):
         return [WASILibc, llvm.LLVM]
+
+
+class WasmThreadsLLVMRuntimeLibs(WasmLLVMRuntimeLibs):
+    def build(self, host_target):
+        self._build(host_target, enable_wasi_threads=True)
+
+        build_root = os.path.dirname(self.build_dir)
+        wasi_sysroot = WASILibc.sysroot_install_path(build_root)
+        # Copy compiler-rt os dirs to the WASI variant
+        os_dir = os.path.join(wasi_sysroot, 'lib', 'wasip1')
+        if os.path.exists(os_dir):
+            shell.rmtree(os_dir)
+        shell.copytree(os.path.join(wasi_sysroot, 'lib', 'wasi'), os_dir)

utils/swift_build_support/swift_build_support/products/wasmstdlib.py

@@ -85,7 +85,6 @@ def build(self, host_target):
         self.cmake_options.define(
             'SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY:BOOL', 'TRUE')
         self.cmake_options.define('SWIFT_ENABLE_DISPATCH:BOOL', 'FALSE')
-        self.cmake_options.define('SWIFT_THREADING_PACKAGE:STRING', 'none')
         self.cmake_options.define(
             'SWIFT_STDLIB_SUPPORTS_BACKTRACE_REPORTING:BOOL', 'FALSE')
         self.cmake_options.define('SWIFT_STDLIB_HAS_DLADDR:BOOL', 'FALSE')
@@ -98,6 +97,8 @@ def build(self, host_target):
                                   os.path.join(self.source_dir, '..',
                                                'swift-experimental-string-processing'))
 
+        self.add_extra_cmake_options()
+
         # Test configuration
         self.cmake_options.define('SWIFT_INCLUDE_TESTS:BOOL', 'TRUE')
         self.cmake_options.define('SWIFT_ENABLE_SOURCEKIT_TESTS:BOOL', 'FALSE')
@@ -119,6 +120,9 @@ def build(self, host_target):
         self.build_with_cmake([], self._build_variant, [],
                               prefer_native_toolchain=True)
 
+    def add_extra_cmake_options(self):
+        self.cmake_options.define('SWIFT_THREADING_PACKAGE:STRING', 'none')
+
     def test(self, host_target):
         build_root = os.path.dirname(self.build_dir)
         bin_paths = [
@@ -129,7 +133,7 @@ def test(self, host_target):
         wasmkit_build_path = os.path.join(
             build_root, '%s-%s' % ('wasmkit', host_target))
         wasmkit_bin_path = wasmkit.WasmKit.cli_file_path(wasmkit_build_path)
-        if not os.path.exists(wasmkit_bin_path):
+        if not os.path.exists(wasmkit_bin_path) or not self.should_test_executable():
             test_target = "check-swift-only_non_executable-wasi-wasm32-custom"
         else:
             test_target = "check-swift-wasi-wasm32-custom"
@@ -143,6 +147,9 @@ def test(self, host_target):
         }
         self.test_with_cmake(None, [test_target], self._build_variant, [], test_env=env)
 
+    def should_test_executable(self):
+        return True
+
     @property
     def _build_variant(self):
         return self.args.build_variant
@@ -170,3 +177,20 @@ def get_dependencies(cls):
                 wasisysroot.WasmLLVMRuntimeLibs,
                 wasmkit.WasmKit,
                 swift.Swift]
+
+
+class WasmThreadsStdlib(WasmStdlib):
+    def should_test_executable(self):
+        # TODO(katei): Enable tests once WasmKit supports WASI threads
+        return False
+
+    def add_extra_cmake_options(self):
+        self.cmake_options.define('SWIFT_THREADING_PACKAGE:STRING', 'pthreads')
+        self.cmake_options.define('SWIFT_STDLIB_EXTRA_C_COMPILE_FLAGS:STRING',
+                                  '-mthread-model;posix;-pthread;'
+                                  '-ftls-model=local-exec')
+        self.cmake_options.define('SWIFT_STDLIB_EXTRA_SWIFT_COMPILE_FLAGS:STRING',
+                                  '-Xcc;-matomics;-Xcc;-mbulk-memory;'
+                                  '-Xcc;-mthread-model;-Xcc;posix;'
+                                  '-Xcc;-pthread;-Xcc;-ftls-model=local-exec')
+        self.cmake_options.define('SWIFT_ENABLE_WASI_THREADS:BOOL', 'TRUE')