Enable MAP_32BIT for macOS (bytecodealliance#2992)

On macOS, by default, the first 4GB is occupied by the pagezero.
While it can be controlled with link time options, as we are
an library, we usually don't have a control on how to link an
executable.

build-scripts/config_common.cmake

@@ -451,3 +451,9 @@ if (WAMR_BUILD_LINUX_PERF EQUAL 1)
   add_definitions (-DWASM_ENABLE_LINUX_PERF=1)
   message ("     Enable linux perf support")
 endif ()
+
+if (APPLE)
+  # On recent macOS versions, by default, the size of page zero is 4GB.
+  # Shrink it to make MAP_32BIT mmap can work.
+  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-pagezero_size,0x4000")
+endif ()

core/shared/platform/common/posix/posix_memmap.c

@@ -77,15 +77,19 @@ os_mmap(void *hint, size_t size, int prot, int flags, os_file_handle file)
         map_prot |= PROT_EXEC;
 
 #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
-#ifndef __APPLE__
     if (flags & MMAP_MAP_32BIT)
         map_flags |= MAP_32BIT;
-#endif
 #endif
 
     if (flags & MMAP_MAP_FIXED)
         map_flags |= MAP_FIXED;
 
+#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
+#if defined(__APPLE__)
+retry_without_map_32bit:
+#endif
+#endif
+
 #if defined(BUILD_TARGET_RISCV64_LP64D) || defined(BUILD_TARGET_RISCV64_LP64)
     /* As AOT relocation in RISCV64 may require that the code/data mapped
      * is in range 0 to 2GB, we try to map the memory with hint address
@@ -143,6 +147,14 @@ os_mmap(void *hint, size_t size, int prot, int flags, os_file_handle file)
     }
 
     if (addr == MAP_FAILED) {
+#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
+#if defined(__APPLE__)
+        if ((map_flags & MAP_32BIT) != 0) {
+            map_flags &= ~MAP_32BIT;
+            goto retry_without_map_32bit;
+        }
+#endif
+#endif
 #if BH_ENABLE_TRACE_MMAP != 0
         os_printf("mmap failed\n");
 #endif