From 2f2d3bbd82d94e341166b437afcbcb89b5cb6c30 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 31 Oct 2023 11:19:01 +0800 Subject: [PATCH] Support static linking onnxruntime lib for 32-bit arm (#401) --- .github/workflows/arm-linux-gnueabihf.yaml | 38 ++++++++++-- build-arm-linux-gnueabihf.sh | 7 ++- cmake/onnxruntime-linux-arm-static.cmake | 72 ++++++++++++++++++++++ cmake/onnxruntime.cmake | 6 +- sherpa-onnx/csrc/CMakeLists.txt | 2 +- 5 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 cmake/onnxruntime-linux-arm-static.cmake diff --git a/.github/workflows/arm-linux-gnueabihf.yaml b/.github/workflows/arm-linux-gnueabihf.yaml index 86e49810b..3cb430e4d 100644 --- a/.github/workflows/arm-linux-gnueabihf.yaml +++ b/.github/workflows/arm-linux-gnueabihf.yaml @@ -32,16 +32,23 @@ concurrency: jobs: arm_linux_gnueabihf: runs-on: ${{ matrix.os }} + name: ${{ matrix.os }} ${{ matrix.lib_type }} strategy: fail-fast: false matrix: os: [ubuntu-latest] + lib_type: [static, shared] steps: - uses: actions/checkout@v4 with: fetch-depth: 0 + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ matrix.os }}-arm-${{ matrix.lib_type }} + - name: cache-qemu id: cache-qemu uses: actions/cache@v3 @@ -107,6 +114,19 @@ jobs: run: | export PATH=$GITHUB_WORKSPACE/toolchain/bin:$PATH + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + + cmake --version + + lib_type=${{ matrix.lib_type }} + + if [[ $lib_type == "shared" ]]; then + export BUILD_SHARED_LIBS=ON + else + export BUILD_SHARED_LIBS=OFF + fi + ./build-arm-linux-gnueabihf.sh ls -lh build-arm-linux-gnueabihf/bin @@ -125,12 +145,14 @@ jobs: qemu-arm ./build-arm-linux-gnueabihf/bin/sherpa-onnx --help + readelf -d ./build-arm-linux-gnueabihf/bin/sherpa-onnx + - name: Copy files shell: bash run: | SHERPA_ONNX_VERSION=v$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) - dst=sherpa-onnx-${SHERPA_ONNX_VERSION}-linux-arm-gnueabihf + dst=sherpa-onnx-${SHERPA_ONNX_VERSION}-linux-arm-gnueabihf-${{ matrix.lib_type }} mkdir $dst cp -a build-arm-linux-gnueabihf/install/bin $dst/ @@ -141,13 +163,21 @@ jobs: tar cjvf ${dst}.tar.bz2 $dst - uses: actions/upload-artifact@v3 + if: matrix.lib_type == 'shared' + with: + name: sherpa-onnx-linux-arm-gnueabihf-shared + path: sherpa-onnx-*linux-arm-gnueabihf-shared.tar.bz2 + + - uses: actions/upload-artifact@v3 + if: matrix.lib_type == 'static' with: - path: sherpa-onnx-*linux-arm-gnueabihf.tar.bz2 + name: sherpa-onnx-linux-arm-gnueabihf-static + path: sherpa-onnx-*linux-arm-gnueabihf-static.tar.bz2 - - name: Release pre-compiled binaries and libs for arm linux gnueabihf + - name: Release pre-compiled binaries and libs for arm linux gnueabihf ${{ matrix.lib_type }} if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') uses: svenstaro/upload-release-action@v2 with: file_glob: true overwrite: true - file: sherpa-onnx-*linux-arm-gnueabihf.tar.bz2 + file: sherpa-onnx-*linux-arm-gnueabihf*.tar.bz2 diff --git a/build-arm-linux-gnueabihf.sh b/build-arm-linux-gnueabihf.sh index 544253b55..90b6e7c18 100755 --- a/build-arm-linux-gnueabihf.sh +++ b/build-arm-linux-gnueabihf.sh @@ -34,10 +34,15 @@ fi export CPLUS_INCLUDE_PATH=$PWD/alsa-lib/include:$CPLUS_INCLUDE_PATH export SHERPA_ONNX_ALSA_LIB_DIR=$PWD/alsa-lib/src/.libs +if [[ x"$BUILD_SHARED_LIBS" == x"" ]]; then + # By default, use static link + BUILD_SHARED_LIBS=OFF +fi + cmake \ -DCMAKE_INSTALL_PREFIX=./install \ -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_SHARED_LIBS=$BUILD_SHARED_LIBS \ -DSHERPA_ONNX_ENABLE_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ -DSHERPA_ONNX_ENABLE_CHECK=OFF \ diff --git a/cmake/onnxruntime-linux-arm-static.cmake b/cmake/onnxruntime-linux-arm-static.cmake new file mode 100644 index 000000000..3814f6054 --- /dev/null +++ b/cmake/onnxruntime-linux-arm-static.cmake @@ -0,0 +1,72 @@ +# Copyright (c) 2022-2023 Xiaomi Corporation +message(STATUS "CMAKE_SYSTEM_NAME: ${CMAKE_SYSTEM_NAME}") +message(STATUS "CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}") + +if(NOT CMAKE_SYSTEM_NAME STREQUAL Linux) + message(FATAL_ERROR "This file is for Linux only. Given: ${CMAKE_SYSTEM_NAME}") +endif() + +if(NOT (CMAKE_SYSTEM_PROCESSOR STREQUAL arm OR CMAKE_SYSTEM_PROCESSOR STREQUAL armv7l)) + message(FATAL_ERROR "This file is for arm only. Given: ${CMAKE_SYSTEM_PROCESSOR}") +endif() + +if(BUILD_SHARED_LIBS) + message(FATAL_ERROR "This file is for building static libraries. BUILD_SHARED_LIBS: ${BUILD_SHARED_LIBS}") +endif() + +set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.16.0/onnxruntime-linux-arm-static_lib-1.16.0.zip") +set(onnxruntime_URL2 "https://huggingface.co/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-linux-arm-static_lib-1.16.0.zip") +set(onnxruntime_HASH "SHA256=d2bf3d69a479ac786bf2f019e720218a74634b73cfda758ba50d0b1adef1c76a") + +# If you don't have access to the Internet, +# please download onnxruntime to one of the following locations. +# You can add more if you want. +set(possible_file_locations + $ENV{HOME}/Downloads/onnxruntime-linux-arm-static_lib-1.16.0.zip + ${PROJECT_SOURCE_DIR}/onnxruntime-linux-arm-static_lib-1.16.0.zip + ${PROJECT_BINARY_DIR}/onnxruntime-linux-arm-static_lib-1.16.0.zip + /tmp/onnxruntime-linux-arm-static_lib-1.16.0.zip + /star-fj/fangjun/download/github/onnxruntime-linux-arm-static_lib-1.16.0.zip +) + +foreach(f IN LISTS possible_file_locations) + if(EXISTS ${f}) + set(onnxruntime_URL "${f}") + file(TO_CMAKE_PATH "${onnxruntime_URL}" onnxruntime_URL) + message(STATUS "Found local downloaded onnxruntime: ${onnxruntime_URL}") + set(onnxruntime_URL2) + break() + endif() +endforeach() + +FetchContent_Declare(onnxruntime + URL + ${onnxruntime_URL} + ${onnxruntime_URL2} + URL_HASH ${onnxruntime_HASH} +) + +FetchContent_GetProperties(onnxruntime) +if(NOT onnxruntime_POPULATED) + message(STATUS "Downloading onnxruntime from ${onnxruntime_URL}") + FetchContent_Populate(onnxruntime) +endif() +message(STATUS "onnxruntime is downloaded to ${onnxruntime_SOURCE_DIR}") + +find_library(location_onnxruntime onnxruntime + PATHS + "${onnxruntime_SOURCE_DIR}/lib" + NO_CMAKE_SYSTEM_PATH +) + +message(STATUS "location_onnxruntime: ${location_onnxruntime}") + +# for static libraries, we use onnxruntime_lib_files directly below +include_directories(${onnxruntime_SOURCE_DIR}/include) + +file(GLOB onnxruntime_lib_files "${onnxruntime_SOURCE_DIR}/lib/lib*.a") + +set(onnxruntime_lib_files ${onnxruntime_lib_files} PARENT_SCOPE) + +message(STATUS "onnxruntime lib files: ${onnxruntime_lib_files}") +install(FILES ${onnxruntime_lib_files} DESTINATION lib) diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake index 75e147ca1..9d1a8f625 100644 --- a/cmake/onnxruntime.cmake +++ b/cmake/onnxruntime.cmake @@ -8,7 +8,11 @@ function(download_onnxruntime) if(CMAKE_SYSTEM_NAME STREQUAL Linux AND CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64) include(onnxruntime-linux-aarch64) elseif(CMAKE_SYSTEM_NAME STREQUAL Linux AND (CMAKE_SYSTEM_PROCESSOR STREQUAL arm OR CMAKE_SYSTEM_PROCESSOR STREQUAL armv7l)) - include(onnxruntime-linux-arm) + if(BUILD_SHARED_LIBS) + include(onnxruntime-linux-arm) + else() + include(onnxruntime-linux-arm-static) + endif() elseif(CMAKE_SYSTEM_NAME STREQUAL Linux AND CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64) if(SHERPA_ONNX_ENABLE_GPU) include(onnxruntime-linux-x86_64-gpu) diff --git a/sherpa-onnx/csrc/CMakeLists.txt b/sherpa-onnx/csrc/CMakeLists.txt index 174a3a394..44eec6aef 100644 --- a/sherpa-onnx/csrc/CMakeLists.txt +++ b/sherpa-onnx/csrc/CMakeLists.txt @@ -113,7 +113,7 @@ target_link_libraries(sherpa-onnx-core kaldi-native-fbank-core) target_link_libraries(sherpa-onnx-core kaldi-decoder-core) -if(BUILD_SHARED_LIBS OR APPLE OR CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64 OR CMAKE_SYSTEM_PROCESSOR STREQUAL arm) +if(BUILD_SHARED_LIBS OR APPLE OR CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64) target_link_libraries(sherpa-onnx-core onnxruntime) else() target_link_libraries(sherpa-onnx-core ${onnxruntime_lib_files})