Skip to content

Commit

Permalink
github-action: rebuild only if it's required
Browse files Browse the repository at this point in the history
This adds a workflow routine that checks if rebuild is needed.
Then, if there is no changes in source code, skip rebuilding

Signed-off-by: MyungJoo Ham <[email protected]>
  • Loading branch information
myungjoo authored and jaeyun-jung committed Jan 24, 2024
1 parent daf22f1 commit 664018d
Show file tree
Hide file tree
Showing 6 changed files with 186 additions and 25 deletions.
83 changes: 83 additions & 0 deletions .github/workflows/check_if_rebuild_requires.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
#!/usr/bin/env bash

##
# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved.
#
# @file: check_if_rebuild_requires.sh
# @brief Check if rebuild & unit-test is required with the given PR.
# @see https://github.com/nnstreamer/nnstreamer
# @author MyungJoo Ham <[email protected]>
#
# Argument 1 ($1): the file containing list of files to be checked.
# Argument 2 ($2): build mode to be checked
# gbs: check if Tizen GBS build is required
# debian: check if pdebuild is required
# android: check if jni rebuild is required
# build (default): check if general meson rebuild is required.

if [ -z $1 ]; then
echo "::error The argument (file path) is not given."
exit 1
fi

if [ -z $2 ]; then
mode="build"
else
mode=$2
fi

rebuild=0
regbs=0
redebian=0
reandroid=0

for file in `cat $1`; do
case $file in
*.md|*.png|*.webp|*.css|*.html )
;;
packaging/* )
regbs='1'
;;
debian/* )
redebian='1'
;;
jni/* )
reandroid='1'
;;
* )
rebuild='1'
regbs='1'
redebian='1'
reandroid='1'
;;
esac
done

case $mode in
gbs)
if [[ "$regbs" == "1" ]]; then
echo "REBUILD=YES"
exit 0
fi
;;
debian)
if [[ "$redebian" == "1" ]]; then
echo "REBUILD=YES"
exit 0
fi
;;
android)
if [[ "$reandroid" == "1" ]]; then
echo "REBUILD=YES"
exit 0
fi
;;
*)
if [[ "$rebuild" == "1" ]]; then
echo "REBUILD=YES"
exit 0
fi
;;
esac

echo "REBUILD=NO"
39 changes: 30 additions & 9 deletions .github/workflows/gbs_x64.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
name: GBS Tizen build for x64 from Ubuntu

# ${{ github.event.pull_request.commits }} : # commits in this PR
# - changed_file_list in GITHUB_ENV: the list of files updated in this pull-request.

on:
pull_request:
branches: [ main ]
Expand All @@ -10,30 +13,48 @@ jobs:
runs-on: ubuntu-20.04

steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Check if rebuild required
## @todo This should become a reusable workflow.
run: |
tmpfile=$(mktemp)
git show --pretty="format:" --name-only --diff-filter=AMRC ${{ github.event.pull_request.head.sha}} -${{ github.event.pull_request.commits }} | sort | uniq | awk NF > ${tmpfile}
echo "changed_file_list=${tmpfile}" >> "$GITHUB_ENV"
rebuild=`bash .github/workflows/check_if_rebuild_requires.sh ${tmpfile} gbs | grep "REBUILD=YES" | wc -l`
echo "Rebuild required: ${rebuild}"
echo "rebuild=${rebuild}" >> "$GITHUB_ENV"
- uses: actions/setup-python@v1
- name: prepare deb sources for GBS
run: echo "deb [trusted=yes] http://download.tizen.org/tools/latest-release/Ubuntu_20.04/ /" | sudo tee /etc/apt/sources.list.d/tizen.list
- name: install GBS
run: sudo apt-get update && sudo apt-get install -y gbs
- name: configure GBS
run: cp .github/workflows/tizen.gbs.conf ~/.gbs.conf
- name: prepare GBS
if: env.rebuild == '1'
run: |
echo "deb [trusted=yes] http://download.tizen.org/tools/latest-release/Ubuntu_20.04/ /" | sudo tee /etc/apt/sources.list.d/tizen.list
sudo apt-get update && sudo apt-get install -y gbs
cp .github/workflows/tizen.gbs.conf ~/.gbs.conf
- name: make cache key
if: env.rebuild == '1'
id: make-key
run: echo "cache_key=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
shell: bash
- name: cache gbs cache
id: cache-gbs-root
if: env.rebuild == '1'
uses: actions/cache@v3
with:
path: ~/GBS-ROOT/local/cache
key: ${{ steps.make-key.outputs.cache_key }}
- name: run GBS
run: gbs build --skip-srcrpm --define "_skip_debug_rpm 1"
if: env.rebuild == '1'
run: |
gbs build --skip-srcrpm --define "_skip_debug_rpm 1"
- name: get nntrainer
if: env.rebuild == '1'
uses: actions/checkout@v3
with:
repository: nnstreamer/nntrainer
path: nntrainer
- name: run nntrainer GBS build
run: pushd nntrainer && gbs build --skip-srcrpm --define "unit_test 1" --define "_skip_debug_rpm 1" && popd
if: env.rebuild == '1'
run: |
pushd nntrainer && gbs build --skip-srcrpm --define "unit_test 1" --define "_skip_debug_rpm 1" && popd
16 changes: 15 additions & 1 deletion .github/workflows/macos.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,30 @@ jobs:
runs-on: macos-latest

steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Check if rebuild required
## @todo This should become a reusable workflow.
run: |
tmpfile=$(mktemp)
git show --pretty="format:" --name-only --diff-filter=AMRC ${{ github.event.pull_request.head.sha}} -${{ github.event.pull_request.commits }} | sort | uniq | awk NF > ${tmpfile}
echo "changed_file_list=${tmpfile}" >> "$GITHUB_ENV"
rebuild=`bash .github/workflows/check_if_rebuild_requires.sh ${tmpfile} rebuild | grep "REBUILD=YES" | wc -l`
echo "Rebuild required: ${rebuild}"
echo "rebuild=${rebuild}" >> "$GITHUB_ENV"
- uses: actions/setup-python@v1
- name: homebrew
if: env.rebuild == '1'
run: |
# temporarily disabled, because it always fails these days.
# brew update
brew install cask
- name: install minimal requirements
if: env.rebuild == '1'
run: brew install meson ninja pkg-config cmake libffi glib gstreamer numpy json-glib
- uses: BSFishy/[email protected]
if: env.rebuild == '1'
with:
action: build

Expand Down
14 changes: 13 additions & 1 deletion .github/workflows/risc-v.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,21 @@ jobs:
runs-on: ubuntu-20.04
name: Build on Ubuntu 20.04 RISC-V 64
steps:
- uses: actions/[email protected]
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Check if rebuild required
## @todo This should become a reusable workflow.
run: |
tmpfile=$(mktemp)
git show --pretty="format:" --name-only --diff-filter=AMRC ${{ github.event.pull_request.head.sha}} -${{ github.event.pull_request.commits }} | sort | uniq | awk NF > ${tmpfile}
echo "changed_file_list=${tmpfile}" >> "$GITHUB_ENV"
rebuild=`bash .github/workflows/check_if_rebuild_requires.sh ${tmpfile} rebuild | grep "REBUILD=YES" | wc -l`
echo "Rebuild required: ${rebuild}"
echo "rebuild=${rebuild}" >> "$GITHUB_ENV"
- uses: nnstreamer/run-on-arch-action@master
name: Run commands
if: env.rebuild == '1'
id: Build
with:
arch: riscv64
Expand Down
18 changes: 17 additions & 1 deletion .github/workflows/ubuntu_clean_llvm_build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,22 +13,38 @@ jobs:
os: [ ubuntu-22.04 ]

steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Check if rebuild required
## @todo This should become a reusable workflow.
run: |
tmpfile=$(mktemp)
git show --pretty="format:" --name-only --diff-filter=AMRC ${{ github.event.pull_request.head.sha}} -${{ github.event.pull_request.commits }} | sort | uniq | awk NF > ${tmpfile}
echo "changed_file_list=${tmpfile}" >> "$GITHUB_ENV"
rebuild=`bash .github/workflows/check_if_rebuild_requires.sh ${tmpfile} rebuild | grep "REBUILD=YES" | wc -l`
echo "Rebuild required: ${rebuild}"
echo "rebuild=${rebuild}" >> "$GITHUB_ENV"
- uses: actions/setup-python@v1
with:
python-version: '3.x'
- name: install minimal requirements
if: env.rebuild == '1'
run: |
sudo apt-get update && \
sudo apt-get install -y libglib2.0-dev libjson-glib-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libunwind-dev googletest \
gstreamer1.0-plugins-good clang
- run: pip install meson ninja
if: env.rebuild == '1'
- run: meson setup build/
if: env.rebuild == '1'
env:
CC: clang
CXX: clang++
- run: meson compile -C build/
if: env.rebuild == '1'
- run: meson test -C build/ -v
if: env.rebuild == '1'
- uses: actions/upload-artifact@v1
if: failure()
with:
Expand Down
41 changes: 28 additions & 13 deletions .github/workflows/ubuntu_clean_meson_build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,32 +13,47 @@ jobs:
os: [ ubuntu-20.04, ubuntu-22.04 ]

steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Check if rebuild required
## @todo This should become a reusable workflow.
run: |
tmpfile=$(mktemp)
git show --pretty="format:" --name-only --diff-filter=AMRC ${{ github.event.pull_request.head.sha}} -${{ github.event.pull_request.commits }} | sort | uniq | awk NF > ${tmpfile}
echo "changed_file_list=${tmpfile}" >> "$GITHUB_ENV"
rebuild=`bash .github/workflows/check_if_rebuild_requires.sh ${tmpfile} rebuild | grep "REBUILD=YES" | wc -l`
echo "Rebuild required: ${rebuild}"
echo "rebuild=${rebuild}" >> "$GITHUB_ENV"
- uses: actions/setup-python@v1
with:
python-version: '3.x'
- name: install minimal requirements
run: sudo apt-get update && sudo apt-get install -y libglib2.0-dev libjson-glib-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libunwind-dev googletest
- name: install additional packages for features
run: sudo apt-get install -y liborc-0.4-dev flex bison libopencv-dev pkg-config python3-dev python3-numpy python3
- name: install additional package from PPA for testing
run: sudo add-apt-repository -y ppa:nnstreamer/ppa && sudo apt-get update && sudo apt-get install -y ssat libpaho-mqtt-dev
- name: install additional package from Ubuntu for testing and valgrind
run: sudo apt-get install -y valgrind gstreamer1.0-tools gstreamer1.0-plugins-good gstreamer1.0-plugins-base libgtest-dev libpng-dev libc6-dbg binutils-x86-64-linux-gnu-dbg valgrind-dbg
- run: pip install meson ninja
- run: meson setup build/
- name: install requirements
if: env.rebuild == '1'
run: |
sudo apt-get update && sudo apt-get install -y libglib2.0-dev libjson-glib-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libunwind-dev googletest
sudo apt-get install -y liborc-0.4-dev flex bison libopencv-dev pkg-config python3-dev python3-numpy python3
sudo add-apt-repository -y ppa:nnstreamer/ppa && sudo apt-get update && sudo apt-get install -y ssat libpaho-mqtt-dev
sudo apt-get install -y valgrind gstreamer1.0-tools gstreamer1.0-plugins-good gstreamer1.0-plugins-base libgtest-dev libpng-dev libc6-dbg binutils-x86-64-linux-gnu-dbg valgrind-dbg
pip install meson ninja
- name: build and unit test
if: env.rebuild == '1'
run: |
meson setup build/
meson compile -C build/
meson test -C build/ -v
env:
CC: gcc
- run: meson compile -C build/
- run: meson test -C build/ -v
- uses: actions/upload-artifact@v1
if: failure()
with:
name: Meson_Testlog
path: build/meson-logs/testlog.txt
- name: SSAT run with Valgrind on decoder-bounding-box
if: env.rebuild == '1'
run: if [ '${{ matrix.os }}' == 'ubuntu-22.04' ]; then export NNSTREAMER_BUILD_ROOT_PATH=`pwd`/build && export NNSTREAMER_FILTERS=`pwd`/build/ext/nnstreamer/tensor_filter && export NNSTREAMER_DECODERS=`pwd`/build/ext/nnstreamer/tensor_decoder && export NNSTREAMER_CONVERTERS=`pwd`/build/ext/nnstreamer/tensor_converter && export GST_PLUGIN_PATH=`pwd`/build/gst && export NNSTREAMER_CONF=`pwd`/build/nnstreamer-test.ini && pushd tests/nnstreamer_decoder_boundingbox && G_SLICE=always-malloc G_DEBUG=gc-friendly ssat -n -p=1 --enable-valgrind --valgrind-suppression ../../tools/debugging/valgrind_suppression --summary summary.txt -cn _n && popd; fi
- name: GTEST run with Valgrind on a case
if: env.rebuild == '1'
run: if [ '${{ matrix.os }}' == 'ubuntu-22.04' ]; then export NNSTREAMER_BUILD_ROOT_PATH=`pwd`/build && export NNSTREAMER_FILTERS=`pwd`/build/ext/nnstreamer/tensor_filter && export NNSTREAMER_DECODERS=`pwd`/build/ext/nnstreamer/tensor_decoder && export NNSTREAMER_CONVERTERS=`pwd`/build/ext/nnstreamer/tensor_converter && export GST_PLUGIN_PATH=`pwd`/build/gst && export NNSTREAMER_CONF=`pwd`/build/nnstreamer-test.ini && G_SLICE=always-malloc G_DEBUG=gc-friendly ./packaging/run_unittests_binaries.sh --valgrind ./tests/ || echo "There are Valgrind errors. Please fix it. As we have a lot of Valgrind errors from different libraries and possible from nnstreamer itself, we are not halting the build with Valgrind errors until we get them all."; fi

# TODO: add more subplugins to be built
Expand Down

0 comments on commit 664018d

Please sign in to comment.