Linking issue with spawn thread API for ESP-IDF #2263

skrovk · 2023-06-05T21:59:49Z

Description

Trying to build the (slightly modified for ESP) spawn thread example fails to build due to linking error - the spawn thread API is unrecognized. When these calls are removed the project builds and executes (except stalling in wasm_runtime_destroy unrelated to this).

Replicate

Target: ESP-IDF ESP32
Build platform: Ubuntu

main.c (the main changes involve execting the wasm setup from a thread and loading the example module from a buffer in module.h)

#include <stdio.h>

#include "wasm_export.h"
#include "bh_platform.h"
#include "pthread.h"
#include "module.h"

#define THREAD_NUM 10

typedef struct ThreadArgs {
    wasm_exec_env_t exec_env;
    int start;
    int length;
} ThreadArgs;

void *
thread(void *arg)
{
    ThreadArgs *thread_arg = (ThreadArgs *)arg;
    wasm_exec_env_t exec_env = thread_arg->exec_env;
    wasm_module_inst_t module_inst = get_module_inst(exec_env);
    wasm_function_inst_t func;
    uint32 argv[2];

    if (!wasm_runtime_init_thread_env()) {
        printf("failed to initialize thread environment");
        return NULL;
    }

    func = wasm_runtime_lookup_function(module_inst, "sum", NULL);
    if (!func) {
        printf("failed to lookup function sum");
        wasm_runtime_destroy_thread_env();
        return NULL;
    }
    argv[0] = thread_arg->start;
    argv[1] = thread_arg->length;

    /* call the WASM function */
    if (!wasm_runtime_call_wasm(exec_env, func, 2, argv)) {
        printf("%s\n", wasm_runtime_get_exception(module_inst));
        wasm_runtime_destroy_thread_env();
        return NULL;
    }

    wasm_runtime_destroy_thread_env();
    return (void *)(uintptr_t)argv[0];
}

void *
wamr_thread_cb(wasm_exec_env_t exec_env, void *arg)
{
    ThreadArgs *thread_arg = (ThreadArgs *)arg;
    wasm_module_inst_t module_inst = get_module_inst(exec_env);
    wasm_function_inst_t func;
    uint32 argv[2];

    func = wasm_runtime_lookup_function(module_inst, "sum", NULL);
    if (!func) {
        printf("failed to lookup function sum");
        return NULL;
    }
    argv[0] = thread_arg->start;
    argv[1] = thread_arg->length;

    /* call the WASM function */
    if (!wasm_runtime_call_wasm(exec_env, func, 2, argv)) {
        printf("%s\n", wasm_runtime_get_exception(module_inst));
        return NULL;
    }

    return (void *)(uintptr_t)argv[0];
}

void *
wasm_start(void *arg) {
    printf("Starting...\n");
    uint8_t *wasm_file_buf = NULL;
    unsigned wasm_file_buf_size = 0;
    uint32 stack_size = 16 * 1024, heap_size = 16 * 1024;

    wasm_module_t wasm_module = NULL;
    wasm_module_inst_t wasm_module_inst = NULL;
    wasm_exec_env_t exec_env = NULL;
    char error_buf[128];
    RuntimeInitArgs init_args;

    wasm_file_buf = (uint8_t *) hello_module_bytes;
    wasm_file_buf_size = sizeof(hello_module_bytes);

    /* configure memory allocation */
    memset(&init_args, 0, sizeof(RuntimeInitArgs));

    init_args.mem_alloc_type = Alloc_With_Allocator;
    init_args.mem_alloc_option.allocator.malloc_func = (void *)os_malloc;
    init_args.mem_alloc_option.allocator.realloc_func = (void *)os_realloc;
    init_args.mem_alloc_option.allocator.free_func = (void *)os_free;
    init_args.max_thread_num = THREAD_NUM;

    printf("Initialize environment...\n");

    /* initialize runtime environment */
    if (!wasm_runtime_full_init(&init_args)) {
        printf("Init runtime environment failed.\n");
        return NULL;
    }

    printf("Load module...\n");

    /* load WASM module */
    if (!(wasm_module = wasm_runtime_load(wasm_file_buf, wasm_file_buf_size,
                                          error_buf, sizeof(error_buf)))) {
        printf("%s\n", error_buf);
        goto fail2;
    }

    uint32_t wasm_argv[2], i, threads_created;
    ThreadArgs thread_arg[THREAD_NUM];
    pthread_t tid[THREAD_NUM];
    wasm_thread_t wasm_tid[THREAD_NUM];
    uint32_t result[THREAD_NUM], sum;
    wasm_function_inst_t func;

    memset(thread_arg, 0, sizeof(ThreadArgs) * THREAD_NUM);

    /* instantiate the module */
    if (!(wasm_module_inst =
              wasm_runtime_instantiate(wasm_module, stack_size, heap_size,
                                       error_buf, sizeof(error_buf)))) {
        printf("%s\n", error_buf);
        goto fail3;
    }

    /* Create the first exec_env */
    if (!(exec_env =
              wasm_runtime_create_exec_env(wasm_module_inst, stack_size))) {
        printf("failed to create exec_env\n");
        goto fail4;
    }

    func = wasm_runtime_lookup_function(wasm_module_inst, "sum", NULL);
    if (!func) {
        printf("failed to lookup function sum");
        goto fail5;
    }
    wasm_argv[0] = 1;
    wasm_argv[1] = 2;

    /*
     * Execute the wasm function in current thread, get the expect result
     */
    if (!wasm_runtime_call_wasm(exec_env, func, 2, wasm_argv)) {
        printf("%s\n", wasm_runtime_get_exception(wasm_module_inst));
    }
    printf("expect result: %d\n", wasm_argv[0]);

    /*
     * Run wasm function in multiple thread created by pthread_create
     */
    memset(thread_arg, 0, sizeof(ThreadArgs) * THREAD_NUM);
    for (i = 0; i < THREAD_NUM; i++) {
        wasm_exec_env_t new_exec_env;
        thread_arg[i].start = 10 * i;
        thread_arg[i].length = 10;

        /* spawn a new exec_env to be executed in other threads */
        new_exec_env = wasm_runtime_spawn_exec_env(exec_env);
        if (new_exec_env)
            thread_arg[i].exec_env = new_exec_env;
        else {
            printf("failed to spawn exec_env\n");
            break;
        }

        /* If we use:
            thread_arg[i].exec_env = exec_env,
            we may get wrong result */

        if (0 != pthread_create(&tid[i], NULL, thread, &thread_arg[i])) {
            printf("failed to create thread.\n");
            wasm_runtime_destroy_spawned_exec_env(new_exec_env);
            break;
        }
    }

    threads_created = i;

    sum = 0;
    memset(result, 0, sizeof(uint32) * THREAD_NUM);
    for (i = 0; i < threads_created; i++) {
        pthread_join(tid[i], (void **)&result[i]);
        sum += result[i];
        /* destroy the spawned exec_env */
        if (thread_arg[i].exec_env)
            wasm_runtime_destroy_spawned_exec_env(thread_arg[i].exec_env);
    }

    printf("[pthread]sum result: %d\n", sum);

    /*
     * Run wasm function in multiple thread created by wamr spawn API
     */
    memset(thread_arg, 0, sizeof(ThreadArgs) * THREAD_NUM);
    for (i = 0; i < THREAD_NUM; i++) {
        thread_arg[i].start = 10 * i;
        thread_arg[i].length = 10;

        /* No need to spawn exec_env manually */
        if (0
            != wasm_runtime_spawn_thread(exec_env, &wasm_tid[i], wamr_thread_cb,
                                         &thread_arg[i])) {
            printf("failed to spawn thread.\n");
            break;
        }
    }

    threads_created = i;

    sum = 0;
    memset(result, 0, sizeof(uint32) * THREAD_NUM);
    for (i = 0; i < threads_created; i++) {
        wasm_runtime_join_thread(wasm_tid[i], (void **)&result[i]);
        sum += result[i];
        /* No need to destroy the spawned exec_env */
    }
    printf("[spwan_thread]sum result: %d\n", sum);

fail5:
    wasm_runtime_destroy_exec_env(exec_env);

fail4:
    /* destroy the module instance */
    wasm_runtime_deinstantiate(wasm_module_inst);

fail3:
    /* unload the module */
    wasm_runtime_unload(wasm_module);

fail2:
    /* destroy runtime environment */
    wasm_runtime_destroy();
    return NULL;
}

void app_main(void) {
    pthread_t t_1;
    int res;

    pthread_attr_t tattr;
    pthread_attr_init(&tattr);
    pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_JOINABLE);
    pthread_attr_setstacksize(&tattr, 4096);

    res = pthread_create(&t_1, &tattr, wasm_start, (void *)NULL);
    assert(res == 0);

    res = pthread_join(t_1, NULL);
    assert(res == 0);

    printf("Exiting...");
}

CMakeLists.txt (ESP project)

idf_component_register(SRCS 
			"main.c"
                    INCLUDE_DIRS ".")

CMakeLists:txt (WAMR project build)

cmake_minimum_required(VERSION 3.5)

include($ENV{IDF_PATH}/tools/cmake/project.cmake)
set (COMPONENTS ${IDF_TARGET} main freertos esptool_py wamr)
list(APPEND EXTRA_COMPONENT_DIRS "$ENV{WAMR_PATH}/build-scripts/esp-idf")

project(spawn_thread)

set (WAMR_BUILD_PLATFORM "esp-idf")
set (WAMR_BUILD_TARGET "XTENSA")

set (WAMR_BUILD_THREAD_MGR 1)
set (WAMR_BUILD_SHARED_MEMORY 1)
set (WAMR_BUILD_BULK_MEMORY 1)
set (WAMR_BUILD_LIB_PTHREAD 1)

set (WAMR_BUILD_INTERP 1)
set (WAMR_BUILD_JIT 0)

Relevant output from build

[677/679] Linking CXX executable coop_demo.elf
FAILED: coop_demo.elf 
:

...

u newlib_include_heap_impl  -u newlib_include_syscalls_impl  -u newlib_include_pthread_impl  -u newlib_include_assert_impl  -Wl,--wrap=_Unwind_SetEnableExceptionFdeSorting  -Wl,--wrap=__register_frame_info_bases  -Wl,--wrap=__register_frame_info  -Wl,--wrap=__register_frame  -Wl,--wrap=__register_frame_info_table_bases  -Wl,--wrap=__register_frame_info_table  -Wl,--wrap=__register_frame_table  -Wl,--wrap=__deregister_frame_info_bases  -Wl,--wrap=__deregister_frame_info  -Wl,--wrap=_Unwind_Find_FDE  -Wl,--wrap=_Unwind_GetGR  -Wl,--wrap=_Unwind_GetCFA  -Wl,--wrap=_Unwind_GetIP  -Wl,--wrap=_Unwind_GetIPInfo  -Wl,--wrap=_Unwind_GetRegionStart  -Wl,--wrap=_Unwind_GetDataRelBase  -Wl,--wrap=_Unwind_GetTextRelBase  -Wl,--wrap=_Unwind_SetIP  -Wl,--wrap=_Unwind_SetGR  -Wl,--wrap=_Unwind_GetLanguageSpecificData  -Wl,--wrap=_Unwind_FindEnclosingFunction  -Wl,--wrap=_Unwind_Resume  -Wl,--wrap=_Unwind_RaiseException  -Wl,--wrap=_Unwind_DeleteException  -Wl,--wrap=_Unwind_ForcedUnwind  -Wl,--wrap=_Unwind_Resume_or_Rethrow  -Wl,--wrap=_Unwind_Backtrace  -Wl,--wrap=__cxa_call_unexpected  -Wl,--wrap=__gxx_personality_v0  -u __cxa_guard_dummy  -lstdc++  esp-idf/pthread/libpthread.a  -lgcc  esp-idf/cxx/libcxx.a  -u __cxx_fatal_exception  /home/amoeba/esp/esp-idf/components/xtensa/esp32/libxt_hal.a && :
/home/amoeba/.espressif/tools/xtensa-esp32-elf/esp-2021r2-8.4.0/xtensa-esp32-elf/bin/../lib/gcc/xtensa-esp32-elf/8.4.0/../../../../xtensa-esp32-elf/bin/ld: esp-idf/main/libmain.a(main.c.obj):(.literal.wasm_start+0x54): undefined reference to `wasm_runtime_spawn_exec_env'
/home/amoeba/.espressif/tools/xtensa-esp32-elf/esp-2021r2-8.4.0/xtensa-esp32-elf/bin/../lib/gcc/xtensa-esp32-elf/8.4.0/../../../../xtensa-esp32-elf/bin/ld: esp-idf/main/libmain.a(main.c.obj):(.literal.wasm_start+0x58): undefined reference to `wasm_runtime_destroy_spawned_exec_env'
/home/amoeba/.espressif/tools/xtensa-esp32-elf/esp-2021r2-8.4.0/xtensa-esp32-elf/bin/../lib/gcc/xtensa-esp32-elf/8.4.0/../../../../xtensa-esp32-elf/bin/ld: esp-idf/main/libmain.a(main.c.obj):(.literal.wasm_start+0x5c): undefined reference to `wasm_runtime_spawn_thread'
/home/amoeba/.espressif/tools/xtensa-esp32-elf/esp-2021r2-8.4.0/xtensa-esp32-elf/bin/../lib/gcc/xtensa-esp32-elf/8.4.0/../../../../xtensa-esp32-elf/bin/ld: esp-idf/main/libmain.a(main.c.obj):(.literal.wasm_start+0x60): undefined reference to `wasm_runtime_join_thread'
/home/amoeba/.espressif/tools/xtensa-esp32-elf/esp-2021r2-8.4.0/xtensa-esp32-elf/bin/../lib/gcc/xtensa-esp32-elf/8.4.0/../../../../xtensa-esp32-elf/bin/ld: esp-idf/main/libmain.a(main.c.obj): in function `wasm_start':
/home/amoeba/Code/phd-wasm-experiments/spawn-thread/main/main.c:161: undefined reference to `wasm_runtime_spawn_exec_env'
/home/amoeba/.espressif/tools/xtensa-esp32-elf/esp-2021r2-8.4.0/xtensa-esp32-elf/bin/../lib/gcc/xtensa-esp32-elf/8.4.0/../../../../xtensa-esp32-elf/bin/ld: /home/amoeba/Code/phd-wasm-experiments/spawn-thread/main/main.c:180: undefined reference to `wasm_runtime_destroy_spawned_exec_env'
/home/amoeba/.espressif/tools/xtensa-esp32-elf/esp-2021r2-8.4.0/xtensa-esp32-elf/bin/../lib/gcc/xtensa-esp32-elf/8.4.0/../../../../xtensa-esp32-elf/bin/ld: /home/amoeba/Code/phd-wasm-experiments/spawn-thread/main/main.c:192: undefined reference to `wasm_runtime_destroy_spawned_exec_env'
/home/amoeba/.espressif/tools/xtensa-esp32-elf/esp-2021r2-8.4.0/xtensa-esp32-elf/bin/../lib/gcc/xtensa-esp32-elf/8.4.0/../../../../xtensa-esp32-elf/bin/ld: /home/amoeba/Code/phd-wasm-experiments/spawn-thread/main/main.c:206: undefined reference to `wasm_runtime_spawn_thread'
/home/amoeba/.espressif/tools/xtensa-esp32-elf/esp-2021r2-8.4.0/xtensa-esp32-elf/bin/../lib/gcc/xtensa-esp32-elf/8.4.0/../../../../xtensa-esp32-elf/bin/ld: /home/amoeba/Code/phd-wasm-experiments/spawn-thread/main/main.c:221: undefined reference to `wasm_runtime_join_thread'
collect2: error: ld returned 1 exit status
ninja: build stopped: subcommand failed.
ninja failed with exit code 1

Any help would be appreciated, thank you!

The text was updated successfully, but these errors were encountered:

TianlongLiang · 2023-06-09T09:18:32Z

Hi, I will run this program and see what I can do, I will keep you updated :)

TianlongLiang · 2023-06-13T09:50:18Z

Hi, @skrovk I tried your example program, it's a bug in CMakeLists.txt, set the cache variable before the project statement should be able to fix it, you can change your CMakeLists.txt to:

cmake_minimum_required(VERSION 3.5)

include($ENV{IDF_PATH}/tools/cmake/project.cmake)

set (COMPONENTS ${IDF_TARGET} main freertos esptool_py wamr)
list(APPEND EXTRA_COMPONENT_DIRS "$ENV{WAMR_PATH}/build-scripts/esp-idf")

set (WAMR_BUILD_PLATFORM "esp-idf")
set (WAMR_BUILD_TARGET "XTENSA")

# set the cache variable WAMR_BUILD_LIB_PTHREAD will automatically set SHRAED_MEMORY, BULK_MEMORY and THREAD_MGR 
set (WAMR_BUILD_LIB_PTHREAD 1)

set (WAMR_BUILD_INTERP 1)
set (WAMR_BUILD_JIT 0)

project(wamr-simple)

Let me know if this solution works for you :)

- Provide a Dockerfile.old to fix issue of ESP32 custom linker scripts not working properly with the newer version of Zephyr, as reported in #2263 - Provide a Dockerfile with newer Zephyr for other boards - Update the corresponding document

- Provide a Dockerfile.old to fix issue of ESP32 custom linker scripts not working properly with the newer version of Zephyr, as reported in bytecodealliance#2263 - Provide a Dockerfile with newer Zephyr for other boards - Update the corresponding document

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Linking issue with spawn thread API for ESP-IDF #2263

Linking issue with spawn thread API for ESP-IDF #2263

skrovk commented Jun 5, 2023

TianlongLiang commented Jun 9, 2023

TianlongLiang commented Jun 13, 2023

Linking issue with spawn thread API for ESP-IDF #2263

Linking issue with spawn thread API for ESP-IDF #2263

Comments

skrovk commented Jun 5, 2023

Description

Replicate

Relevant output from build

TianlongLiang commented Jun 9, 2023

TianlongLiang commented Jun 13, 2023