From 664018dfbb46f95daad0ee4d80837318c60db98a Mon Sep 17 00:00:00 2001 From: MyungJoo Ham Date: Wed, 17 Jan 2024 15:40:06 +0900 Subject: [PATCH] github-action: rebuild only if it's required This adds a workflow routine that checks if rebuild is needed. Then, if there is no changes in source code, skip rebuilding Signed-off-by: MyungJoo Ham --- .../workflows/check_if_rebuild_requires.sh | 83 +++++++++++++++++++ .github/workflows/gbs_x64.yml | 39 +++++++-- .github/workflows/macos.yaml | 16 +++- .github/workflows/risc-v.yml | 14 +++- .github/workflows/ubuntu_clean_llvm_build.yml | 18 +++- .../workflows/ubuntu_clean_meson_build.yml | 41 ++++++--- 6 files changed, 186 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/check_if_rebuild_requires.sh diff --git a/.github/workflows/check_if_rebuild_requires.sh b/.github/workflows/check_if_rebuild_requires.sh new file mode 100644 index 0000000000..3114c2fb02 --- /dev/null +++ b/.github/workflows/check_if_rebuild_requires.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +## +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved. +# +# @file: check_if_rebuild_requires.sh +# @brief Check if rebuild & unit-test is required with the given PR. +# @see https://github.com/nnstreamer/nnstreamer +# @author MyungJoo Ham +# +# Argument 1 ($1): the file containing list of files to be checked. +# Argument 2 ($2): build mode to be checked +# gbs: check if Tizen GBS build is required +# debian: check if pdebuild is required +# android: check if jni rebuild is required +# build (default): check if general meson rebuild is required. + +if [ -z $1 ]; then + echo "::error The argument (file path) is not given." + exit 1 +fi + +if [ -z $2 ]; then + mode="build" +else + mode=$2 +fi + +rebuild=0 +regbs=0 +redebian=0 +reandroid=0 + +for file in `cat $1`; do + case $file in + *.md|*.png|*.webp|*.css|*.html ) + ;; + packaging/* ) + regbs='1' + ;; + debian/* ) + redebian='1' + ;; + jni/* ) + reandroid='1' + ;; + * ) + rebuild='1' + regbs='1' + redebian='1' + reandroid='1' + ;; + esac +done + +case $mode in + gbs) + if [[ "$regbs" == "1" ]]; then + echo "REBUILD=YES" + exit 0 + fi + ;; + debian) + if [[ "$redebian" == "1" ]]; then + echo "REBUILD=YES" + exit 0 + fi + ;; + android) + if [[ "$reandroid" == "1" ]]; then + echo "REBUILD=YES" + exit 0 + fi + ;; + *) + if [[ "$rebuild" == "1" ]]; then + echo "REBUILD=YES" + exit 0 + fi + ;; +esac + +echo "REBUILD=NO" diff --git a/.github/workflows/gbs_x64.yml b/.github/workflows/gbs_x64.yml index 5e24be6d8d..c8383ab3e8 100644 --- a/.github/workflows/gbs_x64.yml +++ b/.github/workflows/gbs_x64.yml @@ -1,5 +1,8 @@ name: GBS Tizen build for x64 from Ubuntu +# ${{ github.event.pull_request.commits }} : # commits in this PR +# - changed_file_list in GITHUB_ENV: the list of files updated in this pull-request. + on: pull_request: branches: [ main ] @@ -10,30 +13,48 @@ jobs: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Check if rebuild required + ## @todo This should become a reusable workflow. + run: | + tmpfile=$(mktemp) + git show --pretty="format:" --name-only --diff-filter=AMRC ${{ github.event.pull_request.head.sha}} -${{ github.event.pull_request.commits }} | sort | uniq | awk NF > ${tmpfile} + echo "changed_file_list=${tmpfile}" >> "$GITHUB_ENV" + rebuild=`bash .github/workflows/check_if_rebuild_requires.sh ${tmpfile} gbs | grep "REBUILD=YES" | wc -l` + echo "Rebuild required: ${rebuild}" + echo "rebuild=${rebuild}" >> "$GITHUB_ENV" - uses: actions/setup-python@v1 - - name: prepare deb sources for GBS - run: echo "deb [trusted=yes] http://download.tizen.org/tools/latest-release/Ubuntu_20.04/ /" | sudo tee /etc/apt/sources.list.d/tizen.list - - name: install GBS - run: sudo apt-get update && sudo apt-get install -y gbs - - name: configure GBS - run: cp .github/workflows/tizen.gbs.conf ~/.gbs.conf + - name: prepare GBS + if: env.rebuild == '1' + run: | + echo "deb [trusted=yes] http://download.tizen.org/tools/latest-release/Ubuntu_20.04/ /" | sudo tee /etc/apt/sources.list.d/tizen.list + sudo apt-get update && sudo apt-get install -y gbs + cp .github/workflows/tizen.gbs.conf ~/.gbs.conf - name: make cache key + if: env.rebuild == '1' id: make-key run: echo "cache_key=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT shell: bash - name: cache gbs cache id: cache-gbs-root + if: env.rebuild == '1' uses: actions/cache@v3 with: path: ~/GBS-ROOT/local/cache key: ${{ steps.make-key.outputs.cache_key }} - name: run GBS - run: gbs build --skip-srcrpm --define "_skip_debug_rpm 1" + if: env.rebuild == '1' + run: | + gbs build --skip-srcrpm --define "_skip_debug_rpm 1" - name: get nntrainer + if: env.rebuild == '1' uses: actions/checkout@v3 with: repository: nnstreamer/nntrainer path: nntrainer - name: run nntrainer GBS build - run: pushd nntrainer && gbs build --skip-srcrpm --define "unit_test 1" --define "_skip_debug_rpm 1" && popd + if: env.rebuild == '1' + run: | + pushd nntrainer && gbs build --skip-srcrpm --define "unit_test 1" --define "_skip_debug_rpm 1" && popd diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml index 7b76d1223a..3f80a285f4 100644 --- a/.github/workflows/macos.yaml +++ b/.github/workflows/macos.yaml @@ -9,16 +9,30 @@ jobs: runs-on: macos-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Check if rebuild required + ## @todo This should become a reusable workflow. + run: | + tmpfile=$(mktemp) + git show --pretty="format:" --name-only --diff-filter=AMRC ${{ github.event.pull_request.head.sha}} -${{ github.event.pull_request.commits }} | sort | uniq | awk NF > ${tmpfile} + echo "changed_file_list=${tmpfile}" >> "$GITHUB_ENV" + rebuild=`bash .github/workflows/check_if_rebuild_requires.sh ${tmpfile} rebuild | grep "REBUILD=YES" | wc -l` + echo "Rebuild required: ${rebuild}" + echo "rebuild=${rebuild}" >> "$GITHUB_ENV" - uses: actions/setup-python@v1 - name: homebrew + if: env.rebuild == '1' run: | # temporarily disabled, because it always fails these days. # brew update brew install cask - name: install minimal requirements + if: env.rebuild == '1' run: brew install meson ninja pkg-config cmake libffi glib gstreamer numpy json-glib - uses: BSFishy/meson-build@v1.0.3 + if: env.rebuild == '1' with: action: build diff --git a/.github/workflows/risc-v.yml b/.github/workflows/risc-v.yml index ec1eda43da..b12ab9ee0f 100644 --- a/.github/workflows/risc-v.yml +++ b/.github/workflows/risc-v.yml @@ -9,9 +9,21 @@ jobs: runs-on: ubuntu-20.04 name: Build on Ubuntu 20.04 RISC-V 64 steps: - - uses: actions/checkout@v2.1.0 + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Check if rebuild required + ## @todo This should become a reusable workflow. + run: | + tmpfile=$(mktemp) + git show --pretty="format:" --name-only --diff-filter=AMRC ${{ github.event.pull_request.head.sha}} -${{ github.event.pull_request.commits }} | sort | uniq | awk NF > ${tmpfile} + echo "changed_file_list=${tmpfile}" >> "$GITHUB_ENV" + rebuild=`bash .github/workflows/check_if_rebuild_requires.sh ${tmpfile} rebuild | grep "REBUILD=YES" | wc -l` + echo "Rebuild required: ${rebuild}" + echo "rebuild=${rebuild}" >> "$GITHUB_ENV" - uses: nnstreamer/run-on-arch-action@master name: Run commands + if: env.rebuild == '1' id: Build with: arch: riscv64 diff --git a/.github/workflows/ubuntu_clean_llvm_build.yml b/.github/workflows/ubuntu_clean_llvm_build.yml index dbaae2db0e..153acf88f6 100644 --- a/.github/workflows/ubuntu_clean_llvm_build.yml +++ b/.github/workflows/ubuntu_clean_llvm_build.yml @@ -13,22 +13,38 @@ jobs: os: [ ubuntu-22.04 ] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Check if rebuild required + ## @todo This should become a reusable workflow. + run: | + tmpfile=$(mktemp) + git show --pretty="format:" --name-only --diff-filter=AMRC ${{ github.event.pull_request.head.sha}} -${{ github.event.pull_request.commits }} | sort | uniq | awk NF > ${tmpfile} + echo "changed_file_list=${tmpfile}" >> "$GITHUB_ENV" + rebuild=`bash .github/workflows/check_if_rebuild_requires.sh ${tmpfile} rebuild | grep "REBUILD=YES" | wc -l` + echo "Rebuild required: ${rebuild}" + echo "rebuild=${rebuild}" >> "$GITHUB_ENV" - uses: actions/setup-python@v1 with: python-version: '3.x' - name: install minimal requirements + if: env.rebuild == '1' run: | sudo apt-get update && \ sudo apt-get install -y libglib2.0-dev libjson-glib-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libunwind-dev googletest \ gstreamer1.0-plugins-good clang - run: pip install meson ninja + if: env.rebuild == '1' - run: meson setup build/ + if: env.rebuild == '1' env: CC: clang CXX: clang++ - run: meson compile -C build/ + if: env.rebuild == '1' - run: meson test -C build/ -v + if: env.rebuild == '1' - uses: actions/upload-artifact@v1 if: failure() with: diff --git a/.github/workflows/ubuntu_clean_meson_build.yml b/.github/workflows/ubuntu_clean_meson_build.yml index 16de549421..4c4919eea4 100644 --- a/.github/workflows/ubuntu_clean_meson_build.yml +++ b/.github/workflows/ubuntu_clean_meson_build.yml @@ -13,32 +13,47 @@ jobs: os: [ ubuntu-20.04, ubuntu-22.04 ] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Check if rebuild required + ## @todo This should become a reusable workflow. + run: | + tmpfile=$(mktemp) + git show --pretty="format:" --name-only --diff-filter=AMRC ${{ github.event.pull_request.head.sha}} -${{ github.event.pull_request.commits }} | sort | uniq | awk NF > ${tmpfile} + echo "changed_file_list=${tmpfile}" >> "$GITHUB_ENV" + rebuild=`bash .github/workflows/check_if_rebuild_requires.sh ${tmpfile} rebuild | grep "REBUILD=YES" | wc -l` + echo "Rebuild required: ${rebuild}" + echo "rebuild=${rebuild}" >> "$GITHUB_ENV" - uses: actions/setup-python@v1 with: python-version: '3.x' - - name: install minimal requirements - run: sudo apt-get update && sudo apt-get install -y libglib2.0-dev libjson-glib-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libunwind-dev googletest - - name: install additional packages for features - run: sudo apt-get install -y liborc-0.4-dev flex bison libopencv-dev pkg-config python3-dev python3-numpy python3 - - name: install additional package from PPA for testing - run: sudo add-apt-repository -y ppa:nnstreamer/ppa && sudo apt-get update && sudo apt-get install -y ssat libpaho-mqtt-dev - - name: install additional package from Ubuntu for testing and valgrind - run: sudo apt-get install -y valgrind gstreamer1.0-tools gstreamer1.0-plugins-good gstreamer1.0-plugins-base libgtest-dev libpng-dev libc6-dbg binutils-x86-64-linux-gnu-dbg valgrind-dbg - - run: pip install meson ninja - - run: meson setup build/ + - name: install requirements + if: env.rebuild == '1' + run: | + sudo apt-get update && sudo apt-get install -y libglib2.0-dev libjson-glib-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libunwind-dev googletest + sudo apt-get install -y liborc-0.4-dev flex bison libopencv-dev pkg-config python3-dev python3-numpy python3 + sudo add-apt-repository -y ppa:nnstreamer/ppa && sudo apt-get update && sudo apt-get install -y ssat libpaho-mqtt-dev + sudo apt-get install -y valgrind gstreamer1.0-tools gstreamer1.0-plugins-good gstreamer1.0-plugins-base libgtest-dev libpng-dev libc6-dbg binutils-x86-64-linux-gnu-dbg valgrind-dbg + pip install meson ninja + - name: build and unit test + if: env.rebuild == '1' + run: | + meson setup build/ + meson compile -C build/ + meson test -C build/ -v env: CC: gcc - - run: meson compile -C build/ - - run: meson test -C build/ -v - uses: actions/upload-artifact@v1 if: failure() with: name: Meson_Testlog path: build/meson-logs/testlog.txt - name: SSAT run with Valgrind on decoder-bounding-box + if: env.rebuild == '1' run: if [ '${{ matrix.os }}' == 'ubuntu-22.04' ]; then export NNSTREAMER_BUILD_ROOT_PATH=`pwd`/build && export NNSTREAMER_FILTERS=`pwd`/build/ext/nnstreamer/tensor_filter && export NNSTREAMER_DECODERS=`pwd`/build/ext/nnstreamer/tensor_decoder && export NNSTREAMER_CONVERTERS=`pwd`/build/ext/nnstreamer/tensor_converter && export GST_PLUGIN_PATH=`pwd`/build/gst && export NNSTREAMER_CONF=`pwd`/build/nnstreamer-test.ini && pushd tests/nnstreamer_decoder_boundingbox && G_SLICE=always-malloc G_DEBUG=gc-friendly ssat -n -p=1 --enable-valgrind --valgrind-suppression ../../tools/debugging/valgrind_suppression --summary summary.txt -cn _n && popd; fi - name: GTEST run with Valgrind on a case + if: env.rebuild == '1' run: if [ '${{ matrix.os }}' == 'ubuntu-22.04' ]; then export NNSTREAMER_BUILD_ROOT_PATH=`pwd`/build && export NNSTREAMER_FILTERS=`pwd`/build/ext/nnstreamer/tensor_filter && export NNSTREAMER_DECODERS=`pwd`/build/ext/nnstreamer/tensor_decoder && export NNSTREAMER_CONVERTERS=`pwd`/build/ext/nnstreamer/tensor_converter && export GST_PLUGIN_PATH=`pwd`/build/gst && export NNSTREAMER_CONF=`pwd`/build/nnstreamer-test.ini && G_SLICE=always-malloc G_DEBUG=gc-friendly ./packaging/run_unittests_binaries.sh --valgrind ./tests/ || echo "There are Valgrind errors. Please fix it. As we have a lot of Valgrind errors from different libraries and possible from nnstreamer itself, we are not halting the build with Valgrind errors until we get them all."; fi # TODO: add more subplugins to be built