diff --git a/.github/workflows/aarch64-linux-gnu.yaml b/.github/workflows/aarch64-linux-gnu-shared.yaml similarity index 83% rename from .github/workflows/aarch64-linux-gnu.yaml rename to .github/workflows/aarch64-linux-gnu-shared.yaml index ab2416a70..25eb3cbd7 100644 --- a/.github/workflows/aarch64-linux-gnu.yaml +++ b/.github/workflows/aarch64-linux-gnu-shared.yaml @@ -1,5 +1,5 @@ # Modified from https://github.com/Tencent/ncnn/blob/master/.github/workflows/linux-arm-cpu-gcc.yml -name: aarch64-linux-gnu +name: aarch64-linux-gnu-shared on: push: @@ -8,7 +8,7 @@ on: tags: - '*' paths: - - '.github/workflows/aarch64-linux-gnu.yaml' + - '.github/workflows/aarch64-linux-gnu-shared.yaml' - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' @@ -17,7 +17,7 @@ on: branches: - master paths: - - '.github/workflows/aarch64-linux-gnu.yaml' + - '.github/workflows/aarch64-linux-gnu-shared.yaml' - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' @@ -35,12 +35,13 @@ env: ${{ github.event.release.tag_name != '' || github.event.inputs.release == 'true' }} concurrency: - group: aarch64-linux-gnu-${{ github.ref }} + group: aarch64-linux-gnu-shared-${{ github.ref }} cancel-in-progress: true jobs: - aarch64_linux_gnu: + aarch64_linux_gnu_shared: runs-on: ${{ matrix.os }} + name: aarch64 shared lib test strategy: fail-fast: false matrix: @@ -51,6 +52,11 @@ jobs: with: fetch-depth: 0 + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ matrix.os }}-aarch64-shared + - name: cache-qemu id: cache-qemu uses: actions/cache@v3 @@ -114,6 +120,13 @@ jobs: run: | export PATH=$GITHUB_WORKSPACE/toolchain/bin:$PATH + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + + cmake --version + + export BUILD_SHARED_LIBS=ON + ./build-aarch64-linux-gnu.sh ls -lh build-aarch64-linux-gnu/bin @@ -132,12 +145,14 @@ jobs: qemu-aarch64 ./build-aarch64-linux-gnu/bin/sherpa-onnx --help + readelf -d ./build-aarch64-linux-gnu/bin/sherpa-onnx + - name: Copy files shell: bash run: | SHERPA_ONNX_VERSION=v$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) - dst=sherpa-onnx-${SHERPA_ONNX_VERSION}-linux-aarch64 + dst=sherpa-onnx-${SHERPA_ONNX_VERSION}-linux-aarch64-shared mkdir $dst cp -a build-aarch64-linux-gnu/install/bin $dst/ @@ -149,7 +164,8 @@ jobs: - uses: actions/upload-artifact@v3 with: - path: sherpa-onnx-*linux-aarch64.tar.bz2 + name: sherpa-onnx-linux-aarch64-shared + path: sherpa-onnx-*linux-aarch64-shared.tar.bz2 - name: Release pre-compiled binaries and libs for aarch64 linux if: env.RELEASE == 'true' @@ -157,4 +173,4 @@ jobs: with: file_glob: true overwrite: true - file: sherpa-onnx-*linux-aarch64.tar.bz2 + file: sherpa-onnx-*linux-aarch64*.tar.bz2 diff --git a/.github/workflows/aarch64-linux-gnu-static.yaml b/.github/workflows/aarch64-linux-gnu-static.yaml new file mode 100644 index 000000000..1d01bfde0 --- /dev/null +++ b/.github/workflows/aarch64-linux-gnu-static.yaml @@ -0,0 +1,177 @@ +# Modified from https://github.com/Tencent/ncnn/blob/master/.github/workflows/linux-arm-cpu-gcc.yml +name: aarch64-linux-gnu-static + +on: + push: + branches: + - master + tags: + - '*' + paths: + - '.github/workflows/aarch64-linux-gnu-static.yaml' + - 'CMakeLists.txt' + - 'cmake/**' + - 'sherpa-onnx/csrc/*' + - 'toolchains/aarch64-linux-gnu.toolchain.cmake' + pull_request: + branches: + - master + paths: + - '.github/workflows/aarch64-linux-gnu-static.yaml' + - 'CMakeLists.txt' + - 'cmake/**' + - 'sherpa-onnx/csrc/*' + - 'toolchains/aarch64-linux-gnu.toolchain.cmake' + + workflow_dispatch: + inputs: + release: + description: "Whether to release" + type: boolean + +env: + RELEASE: + |- # Release if there is a release tag name or a release flag in workflow_dispatch + ${{ github.event.release.tag_name != '' || github.event.inputs.release == 'true' }} + +concurrency: + group: aarch64-linux-gnu-static-${{ github.ref }} + cancel-in-progress: true + +jobs: + aarch64_linux_gnu_static: + runs-on: ${{ matrix.os }} + name: aarch64 static lib test + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ matrix.os }}-aarch64-static + + - name: cache-qemu + id: cache-qemu + uses: actions/cache@v3 + with: + path: qemu-install + key: qemu-aarch64-install-20220908 + + - name: install-qemu-build-deps + if: steps.cache-qemu.outputs.cache-hit != 'true' + run: | + sudo apt-get update + sudo apt-get install autoconf automake autotools-dev ninja-build + + - name: checkout-qemu + if: steps.cache-qemu.outputs.cache-hit != 'true' + uses: actions/checkout@v3 + with: + repository: qemu/qemu + path: qemu + ref: f5643914a9e8f79c606a76e6a9d7ea82a3fc3e65 + + - name: qemu + if: steps.cache-qemu.outputs.cache-hit != 'true' + run: | + cd qemu + ./configure --prefix=$GITHUB_WORKSPACE/qemu-install --target-list=aarch64-linux-user --disable-system + make -j2 + make install + + - name: cache-toolchain + id: cache-toolchain + uses: actions/cache@v3 + with: + path: toolchain + key: gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu.tar.xz + + - name: Download toolchain + if: steps.cache-toolchain.outputs.cache-hit != 'true' + shell: bash + run: | + # wget wget https://armkeil.blob.core.windows.net/developer/Files/downloads/gnu-a/9.2-2019.12/binrel/gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu.tar.xz + wget -qq https://huggingface.co/csukuangfj/sherpa-ncnn-toolchains/resolve/main/gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu.tar.xz + + mkdir $GITHUB_WORKSPACE/toolchain + tar xf ./gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu.tar.xz --strip-components 1 -C $GITHUB_WORKSPACE/toolchain + + - name: Display toolchain info + shell: bash + run: | + export PATH=$GITHUB_WORKSPACE/toolchain/bin:$PATH + aarch64-none-linux-gnu-gcc --version + + - name: Display qemu-aarch64 -h + shell: bash + run: | + export PATH=$GITHUB_WORKSPACE/qemu-install/bin:$PATH + export QEMU_LD_PREFIX=$GITHUB_WORKSPACE/toolchain/aarch64-none-linux-gnu/libc + qemu-aarch64 -h + + - name: build aarch64-linux-gnu + shell: bash + run: | + export PATH=$GITHUB_WORKSPACE/toolchain/bin:$PATH + + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + + cmake --version + + export BUILD_SHARED_LIBS=OFF + + ./build-aarch64-linux-gnu.sh + + ls -lh build-aarch64-linux-gnu/bin + ls -lh build-aarch64-linux-gnu/lib + + file build-aarch64-linux-gnu/bin/sherpa-onnx + + - name: Test sherpa-onnx + shell: bash + run: | + export PATH=$GITHUB_WORKSPACE/toolchain/bin:$PATH + export PATH=$GITHUB_WORKSPACE/qemu-install/bin:$PATH + export QEMU_LD_PREFIX=$GITHUB_WORKSPACE/toolchain/aarch64-none-linux-gnu/libc + + ls -lh ./build-aarch64-linux-gnu/bin + + qemu-aarch64 ./build-aarch64-linux-gnu/bin/sherpa-onnx --help + + readelf -d ./build-aarch64-linux-gnu/bin/sherpa-onnx + + - name: Copy files + shell: bash + run: | + SHERPA_ONNX_VERSION=v$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + + dst=sherpa-onnx-${SHERPA_ONNX_VERSION}-linux-aarch64-static + mkdir $dst + + cp -a build-aarch64-linux-gnu/install/bin $dst/ + cp -a build-aarch64-linux-gnu/install/lib $dst/ + + tree $dst + + tar cjvf ${dst}.tar.bz2 $dst + + - uses: actions/upload-artifact@v3 + with: + name: sherpa-onnx-linux-aarch64-static + path: sherpa-onnx-*linux-aarch64-static.tar.bz2 + + - name: Release pre-compiled binaries and libs for aarch64 linux + if: env.RELEASE == 'true' + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + overwrite: true + file: sherpa-onnx-*linux-aarch64*.tar.bz2 diff --git a/build-aarch64-linux-gnu.sh b/build-aarch64-linux-gnu.sh index ee7234ad0..469812384 100755 --- a/build-aarch64-linux-gnu.sh +++ b/build-aarch64-linux-gnu.sh @@ -1,5 +1,11 @@ #!/usr/bin/env bash +if command -v aarch64-none-linux-gnu-gcc &> /dev/null; then + ln -svf $(which aarch64-none-linux-gnu-gcc) ./aarch64-linux-gnu-gcc + ln -svf $(which aarch64-none-linux-gnu-g++) ./aarch64-linux-gnu-g++ + export PATH=$PWD:$PATH +fi + if ! command -v aarch64-linux-gnu-gcc &> /dev/null; then echo "Please install a toolchain for cross-compiling." echo "You can refer to: " @@ -33,10 +39,15 @@ fi export CPLUS_INCLUDE_PATH=$PWD/alsa-lib/include:$CPLUS_INCLUDE_PATH export SHERPA_ONNX_ALSA_LIB_DIR=$PWD/alsa-lib/src/.libs +if [[ x"$BUILD_SHARED_LIBS" == x"" ]]; then + # By default, use static link + BUILD_SHARED_LIBS=OFF +fi + cmake \ -DCMAKE_INSTALL_PREFIX=./install \ -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_SHARED_LIBS=$BUILD_SHARED_LIBS \ -DSHERPA_ONNX_ENABLE_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ -DSHERPA_ONNX_ENABLE_CHECK=OFF \ diff --git a/cmake/onnxruntime-linux-aarch64-static.cmake b/cmake/onnxruntime-linux-aarch64-static.cmake new file mode 100644 index 000000000..c9ef6f3fc --- /dev/null +++ b/cmake/onnxruntime-linux-aarch64-static.cmake @@ -0,0 +1,64 @@ +# Copyright (c) 2022-2023 Xiaomi Corporation +message(STATUS "CMAKE_SYSTEM_NAME: ${CMAKE_SYSTEM_NAME}") +message(STATUS "CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}") + +if(NOT CMAKE_SYSTEM_NAME STREQUAL Linux) + message(FATAL_ERROR "This file is for Linux only. Given: ${CMAKE_SYSTEM_NAME}") +endif() + +if(NOT CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64) + message(FATAL_ERROR "This file is for aarch64 only. Given: ${CMAKE_SYSTEM_PROCESSOR}") +endif() + +if(BUILD_SHARED_LIBS) + message(FATAL_ERROR "This file is for building static libraries. BUILD_SHARED_LIBS: ${BUILD_SHARED_LIBS}") +endif() + +set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.16.0/onnxruntime-linux-aarch64-static_lib-1.16.0.zip") +set(onnxruntime_URL2 "https://huggingface.co/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-linux-aarch64-static_lib-1.16.0.zip") +set(onnxruntime_HASH "SHA256=ddcb4e67bcdff53eb72265a12420c71f73acc8610054e06259e9b98770136bd5") + +# If you don't have access to the Internet, +# please download onnxruntime to one of the following locations. +# You can add more if you want. +set(possible_file_locations + $ENV{HOME}/Downloads/onnxruntime-linux-aarch64-static_lib-1.16.0.zip + ${PROJECT_SOURCE_DIR}/onnxruntime-linux-aarch64-static_lib-1.16.0.zip + ${PROJECT_BINARY_DIR}/onnxruntime-linux-aarch64-static_lib-1.16.0.zip + /tmp/onnxruntime-linux-aarch64-static_lib-1.16.0.zip + /star-fj/fangjun/download/github/onnxruntime-linux-aarch64-static_lib-1.16.0.zip +) + +foreach(f IN LISTS possible_file_locations) + if(EXISTS ${f}) + set(onnxruntime_URL "${f}") + file(TO_CMAKE_PATH "${onnxruntime_URL}" onnxruntime_URL) + message(STATUS "Found local downloaded onnxruntime: ${onnxruntime_URL}") + set(onnxruntime_URL2) + break() + endif() +endforeach() + +FetchContent_Declare(onnxruntime + URL + ${onnxruntime_URL} + ${onnxruntime_URL2} + URL_HASH ${onnxruntime_HASH} +) + +FetchContent_GetProperties(onnxruntime) +if(NOT onnxruntime_POPULATED) + message(STATUS "Downloading onnxruntime from ${onnxruntime_URL}") + FetchContent_Populate(onnxruntime) +endif() +message(STATUS "onnxruntime is downloaded to ${onnxruntime_SOURCE_DIR}") + +# for static libraries, we use onnxruntime_lib_files directly below +include_directories(${onnxruntime_SOURCE_DIR}/include) + +file(GLOB onnxruntime_lib_files "${onnxruntime_SOURCE_DIR}/lib/lib*.a") + +set(onnxruntime_lib_files ${onnxruntime_lib_files} PARENT_SCOPE) + +message(STATUS "onnxruntime lib files: ${onnxruntime_lib_files}") +install(FILES ${onnxruntime_lib_files} DESTINATION lib) diff --git a/cmake/onnxruntime-linux-aarch64.cmake b/cmake/onnxruntime-linux-aarch64.cmake index cd67f5077..cadc9ade7 100644 --- a/cmake/onnxruntime-linux-aarch64.cmake +++ b/cmake/onnxruntime-linux-aarch64.cmake @@ -10,6 +10,10 @@ if(NOT CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64) message(FATAL_ERROR "This file is for aarch64 only. Given: ${CMAKE_SYSTEM_PROCESSOR}") endif() +if(NOT BUILD_SHARED_LIBS) + message(FATAL_ERROR "This file is for building shared libraries. BUILD_SHARED_LIBS: ${BUILD_SHARED_LIBS}") +endif() + set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.16.0/onnxruntime-linux-aarch64-1.16.0.tgz") set(onnxruntime_URL2 "https://huggingface.co/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/onnxruntime-linux-aarch64-1.16.0.tgz") set(onnxruntime_HASH "SHA256=8b15781d974803203c09df7d52c84d8c9f1ac7d949a97f515e4d2f5dc978d8af") diff --git a/cmake/onnxruntime-linux-arm-static.cmake b/cmake/onnxruntime-linux-arm-static.cmake index 3814f6054..9634485df 100644 --- a/cmake/onnxruntime-linux-arm-static.cmake +++ b/cmake/onnxruntime-linux-arm-static.cmake @@ -53,14 +53,6 @@ if(NOT onnxruntime_POPULATED) endif() message(STATUS "onnxruntime is downloaded to ${onnxruntime_SOURCE_DIR}") -find_library(location_onnxruntime onnxruntime - PATHS - "${onnxruntime_SOURCE_DIR}/lib" - NO_CMAKE_SYSTEM_PATH -) - -message(STATUS "location_onnxruntime: ${location_onnxruntime}") - # for static libraries, we use onnxruntime_lib_files directly below include_directories(${onnxruntime_SOURCE_DIR}/include) diff --git a/cmake/onnxruntime-linux-arm.cmake b/cmake/onnxruntime-linux-arm.cmake index 27b8ee9bd..89e446c7e 100644 --- a/cmake/onnxruntime-linux-arm.cmake +++ b/cmake/onnxruntime-linux-arm.cmake @@ -10,6 +10,10 @@ if(NOT (CMAKE_SYSTEM_PROCESSOR STREQUAL arm OR CMAKE_SYSTEM_PROCESSOR STREQUAL a message(FATAL_ERROR "This file is for arm only. Given: ${CMAKE_SYSTEM_PROCESSOR}") endif() +if(NOT BUILD_SHARED_LIBS) + message(FATAL_ERROR "This file is for building shared libraries. BUILD_SHARED_LIBS: ${BUILD_SHARED_LIBS}") +endif() + set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.16.0/onnxruntime-linux-arm-1.16.0.zip") set(onnxruntime_URL2 "https://huggingface.co/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-linux-arm-1.16.0.zip") set(onnxruntime_HASH "SHA256=0a63ea99fbba3ee399d6626c36752844ae93ae19bab30e4bf00d45cc8a13da02") diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake index 9d1a8f625..418a030f1 100644 --- a/cmake/onnxruntime.cmake +++ b/cmake/onnxruntime.cmake @@ -6,7 +6,11 @@ function(download_onnxruntime) message(STATUS "CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}") if(CMAKE_SYSTEM_NAME STREQUAL Linux AND CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64) - include(onnxruntime-linux-aarch64) + if(BUILD_SHARED_LIBS) + include(onnxruntime-linux-aarch64) + else() + include(onnxruntime-linux-aarch64-static) + endif() elseif(CMAKE_SYSTEM_NAME STREQUAL Linux AND (CMAKE_SYSTEM_PROCESSOR STREQUAL arm OR CMAKE_SYSTEM_PROCESSOR STREQUAL armv7l)) if(BUILD_SHARED_LIBS) include(onnxruntime-linux-arm) diff --git a/sherpa-onnx/csrc/CMakeLists.txt b/sherpa-onnx/csrc/CMakeLists.txt index 44eec6aef..1c7031b5a 100644 --- a/sherpa-onnx/csrc/CMakeLists.txt +++ b/sherpa-onnx/csrc/CMakeLists.txt @@ -113,7 +113,7 @@ target_link_libraries(sherpa-onnx-core kaldi-native-fbank-core) target_link_libraries(sherpa-onnx-core kaldi-decoder-core) -if(BUILD_SHARED_LIBS OR APPLE OR CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64) +if(BUILD_SHARED_LIBS OR APPLE) target_link_libraries(sherpa-onnx-core onnxruntime) else() target_link_libraries(sherpa-onnx-core ${onnxruntime_lib_files})