Skip to content

Commit

Permalink
36 Add CI using Github Actions (IntelLabs#37)
Browse files Browse the repository at this point in the history
* Modifications needed for CI using Github actions

* remove .gitlab-ci.yml
  • Loading branch information
cwlacewe authored Apr 26, 2022
1 parent 90df8e4 commit 9d44760
Show file tree
Hide file tree
Showing 5 changed files with 192 additions and 63 deletions.
144 changes: 144 additions & 0 deletions .github/workflows/cpp_coverage_source.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
name: Compare C++ Coverage

# Controls when the action will run. Triggers the workflow on push or pull request
# events but only for the master and develop branch
on:
pull_request:
types: [ opened, edited, synchronize, reopened ]
branches:
- develop
- master

# Environment variables
env:
GITHUB_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
coverage_job:
name: Coverage Test

# The type of runner that the job will run on
runs-on: self-hosted

strategy:
fail-fast: true
matrix:
include:
- coverage_type: Source
container_name: coverage_cpp_source_${GITHUB_PULL_REQUEST_NUMBER}
container_output: coverage_cpp_source_output
container_tag: "vdms:source_coverage"
output_name: source_coverage
report_name: source_coverage_report
branch_ref: ${{ github.event.pull_request.head.sha }}
- coverage_type: Target
container_name: coverage_cpp_target_${GITHUB_PULL_REQUEST_NUMBER}
container_output: coverage_cpp_target_output
container_tag: "vdms:target_coverage"
output_name: target_coverage
report_name: target_coverage_report
branch_ref: ${{ github.event.pull_request.base.ref }}

outputs:
source_coverage: ${{ steps.report_coverage.outputs.source_coverage}}
target_coverage: ${{ steps.report_coverage.outputs.target_coverage}}
# source_coverage_report: ${{ steps.report_coverage.outputs.source_coverage_report}}
# target_coverage_report: ${{ steps.report_coverage.outputs.target_coverage_report}}

# Steps represent a sequence of tasks that will be executed as part of the job
steps:
- name: Clean workspace if git module is found
run: git submodule status || rm -rf "$GITHUB_WORKSPACE"/* "$GITHUB_WORKSPACE"/.gi*

# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- name: Checkout ${{ matrix.coverage_type }} Branch
uses: actions/checkout@v3
with:
submodules: true
ref: ${{ matrix.branch_ref }}

- name: Build and Run Docker Container
run: |
set -x
docker stop $(docker ps -aqf "name=${{ matrix.container_name }}") || true
docker rm $(docker ps -aqf "name=${{ matrix.container_name }}") || true
docker build --build-arg MAVEN_OPTS='-Dhttps.proxyHost=proxy-chain.intel.com -Dhttps.proxyPort=912 -Dhttps.nonProxyHosts="localhost|127.0.0.1"' \
-f docker/check-in/Dockerfile -t ${{ matrix.container_tag }} .
docker run -d --name ${{ matrix.container_name }} ${{ matrix.container_tag }}
- name: Get ${{ matrix.coverage_type }} Coverage
shell: bash
run: |
set -x
mkdir -p coverage
echo "${{ matrix.container_name }}"
# Make sure coverage script is available
if [ ! -f docker/check-in/run_coverage.sh ]; then
docker exec ${{ matrix.container_name }} bash -c "touch /run_coverage.sh && echo 'cd /vdms/tests && chmod +x run_tests.sh && ./run_tests.sh' >> /run_coverage.sh && chmod +x /run_coverage.sh && mkdir -p /vdms/tests/coverage_report"
docker exec ${{ matrix.container_name }} bash -c "echo 'gcovr -k --root /vdms -e /vdms/src/pmgd -e /vdms/build/CMakeFiles -f "/vdms/client/.*\.cc" -f "/vdms/ext/.*\.cc" -f "/vdms/src/.*\.cc" -f src/SearchExpression.cc --exclude-unreachable-branches --xml-pretty --xml=/vdms/tests/coverage_report/c_coverage_report.xml --txt=/vdms/tests/coverage_report/c_coverage_report.txt' >> /run_coverage.sh && echo "echo 'DONE'" >> /run_coverage.sh"
docker exec ${{ matrix.container_name }} bash -c "echo 'cat /vdms/tests/coverage_report/c_coverage_report.txt' >> /run_coverage.sh"
fi
docker exec ${{ matrix.container_name }} bash -c "./run_coverage.sh"
docker cp $(docker ps -a | grep ${{ matrix.container_name }} | awk '{print $1}'):/vdms/tests/coverage_report/c_coverage_report.txt coverage/c_coverage_report_target.txt
# report="$(<coverage/c_coverage_report_target.txt)"
# report="${report//'%'/'%25'}"
# report="${report//$'\n'/'%0A'}"
# report="${report//$'\r'/'%0D'}"
# echo "coverage_report="$report"" >> $GITHUB_ENV
docker cp $(docker ps -a | grep ${{ matrix.container_name }} | awk '{print $1}'):/vdms/tests/coverage_report/c_coverage_report.xml coverage/c_coverage_report_target.xml
echo "coverage_value=$(cat coverage/c_coverage_report_target.xml | grep -oP 'coverage line-rate="([-+]?\d*\.\d+|\d+)"' | grep -oP "[-+]?\d*\.\d+|\d+" | awk '{print $1*100}')" >> $GITHUB_ENV
docker ps -aqf "name=${{ matrix.container_name }}" | xargs docker stop
docker ps -aqf "name=${{ matrix.container_name }}" | xargs docker rm
- name: Report ${{ matrix.coverage_type }} Coverage
id: report_coverage
run: |
set -x
if [[ -z $coverage_value ]]
then
exit 1
fi
echo "${{ matrix.coverage_type }} Coverage: ${coverage_value}"
echo "::set-output name=${{ matrix.output_name }}::${coverage_value}"
# echo "::set-output name=${{ matrix.report_name }}::${coverage_report}"
compare_coverage:
name: Compare Reported Coverage

# The type of runner that the job will run on
runs-on: self-hosted
needs: coverage_job
steps:
- name: Comment Coverage
if: (github.event_name == 'pull_request')
uses: actions/github-script@v3
with:
# \n\n\nTarget Report: ${{ needs.coverage_job.outputs.target_coverage_report }}\n\n\nSource Report: ${{ needs.coverage_job.outputs.source_coverage_report }}'
script: |
github.issues.createComment({
issue_number: ${{ github.event.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: 'Target Coverage: ${{ needs.coverage_job.outputs.target_coverage }}%\n\n\nSource Coverage: ${{ needs.coverage_job.outputs.source_coverage }}%'
})
- name: Compare Coverage
run: |
echo "Source Coverage: ${{needs.coverage_job.outputs.source_coverage}}"
echo "Target Coverage: ${{needs.coverage_job.outputs.target_coverage}}"
if ${{ needs.coverage_job.outputs.target_coverage > needs.coverage_job.outputs.source_coverage }}
then
echo 'Coverage below target'
exit 1
else
echo "Coverage threshold met!"
fi
34 changes: 0 additions & 34 deletions .gitlab-ci.yml

This file was deleted.

42 changes: 23 additions & 19 deletions docker/check-in/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ ARG BUILD_THREADS=-j16
ARG MAVEN_OPTS='-Dhttps.nonProxyHosts="localhost|127.0.0.1"'

#1
FROM ubuntu:${UBUNTU_VERSION}
FROM ubuntu:${UBUNTU_VERSION} as base_build

# Dockerfile limitations force a repetition of global args
ARG UBUNTU_VERSION
Expand All @@ -16,15 +16,15 @@ ARG MAVEN_OPTS

#Install Packages
RUN apt-get update && apt-get -y install software-properties-common && \
add-apt-repository "deb http://security.ubuntu.com/ubuntu xenial-security main" && \
add-apt-repository "deb http://security.ubuntu.com/ubuntu focal-security main" && \
apt-get -y install g++ git libssl-dev libc-ares-dev apt-transport-https \
ca-certificates curl gnupg-agent software-properties-common cmake python3-pip \
build-essential autoconf automake libtool g++ unzip bzip2 libarchive-tools \
cmake git pkg-config python python-dev wget libbz2-dev libssl-dev liblz4-dev \
mpich libjsoncpp-dev flex javacc bison openjdk-11-jdk-headless libleveldb-dev \
libsnappy-dev libhdf5-serial-dev libatlas-base-dev libboost-all-dev \
libgflags-dev libgoogle-glog-dev liblmdb-dev libjpeg8-dev libtiff5-dev \
libjasper-dev libgtk-3-dev libopenmpi-dev libgtest-dev ed libgtk2.0-dev \
libgtk-3-dev libopenmpi-dev libgtest-dev ed libgtk2.0-dev \
pkg-config libavcodec-dev libavformat-dev libswscale-dev libtbb2 libtbb-dev \
libjpeg-dev libpng-dev libtiff-dev libdc1394-22-dev libopenblas-dev maven lcov && \
pip3 install numpy coverage vdms gcovr
Expand All @@ -37,25 +37,25 @@ RUN git clone --branch v1.40.0 https://github.com/grpc/grpc.git && \
git clone --branch v0.6 https://github.com/tristanpenman/valijson.git && \
git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \
git clone --branch v1.7.1 https://github.com/facebookresearch/faiss.git && \
curl http://zlib.net/zlib-1.2.11.tar.gz -o zlib-1.2.11.tar.gz && \
wget -c http://zlib.net/zlib-1.2.12.tar.gz && \
curl https://downloads.apache.org/commons/codec/binaries/commons-codec-1.15-bin.tar.gz -o /commons-codec-1.15-bin.tar.gz && \
curl https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/json-simple/json-simple-1.1.1.jar -o /usr/share/java/json-simple-1.1.1.jar && \
wget https://github.com/TileDB-Inc/TileDB/archive/1.3.1.tar.gz

RUN cd /CMake && ./bootstrap && make ${BUILD_THREADS} && make install && \
cd /swig && ./autogen.sh && ./configure && make ${BUILD_THREADS} && make install && \
cd /faiss && mkdir build && cd build && cmake -DFAISS_ENABLE_GPU=OFF .. && make ${BUILD_THREADS} && make install && \
cd /grpc && git submodule update --init --recursive && cd third_party/protobuf/cmake && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make -j8 && make install && \
cd ../../../abseil-cpp && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make -j8 && make install && \
cd ../../re2/ && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make -j8 && make install && \
cd ../../zlib/ && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make -j8 && make install && \
cd /faiss && mkdir build && cd build && cmake -DFAISS_ENABLE_GPU=OFF .. && make ${BUILD_THREADS} && make ${BUILD_THREADS} && make install && \
cd /grpc && git submodule update --init --recursive && cd third_party/protobuf/cmake && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \
cd ../../../abseil-cpp && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \
cd ../../re2/ && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \
cd ../../zlib/ && mkdir build && cd build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. && make ${BUILD_THREADS} && make install && \
cd /grpc/cmake && mkdir build && cd build && cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_ABSL_PROVIDER=package \
-DgRPC_CARES_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package \
-DgRPC_RE2_PROVIDER=package -DgRPC_SSL_PROVIDER=package \
-DgRPC_ZLIB_PROVIDER=package -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ../.. && make -j8 && make install && \
cd / && gunzip zlib-1.2.11.tar.gz && tar -xvf zlib-1.2.11.tar && cd zlib-1.2.11 && ./configure && make -j4 && make install && \
cd / && rm -rf zlib-1.2.11.tar zlib-1.2.11 && \
cd /libpng && ./configure && make -j4 && make install
-DgRPC_ZLIB_PROVIDER=package -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ../.. && make ${BUILD_THREADS} && make install && \
cd / && tar -zxf zlib-1.2.12.tar.gz && cd zlib-1.2.12 && ./configure && make ${BUILD_THREADS} && make install && \
cd / && rm -rf zlib-1.2.12.tar zlib-1.2.12 && \
cd /libpng && git checkout libpng12 && ./configure && make ${BUILD_THREADS} && make install

# Google Test
RUN cd /usr/src/gtest && cmake . && make ${BUILD_THREADS} && mv lib/libgtest* /usr/lib/ && \
Expand All @@ -72,17 +72,21 @@ RUN ln -s /grpc/third_party/protobuf/cmake/build/protoc /grpc/third_party/protob
cp $(ls target/protobuf-java*.jar) /usr/share/java/protobuf.jar

# Valijson
RUN cd /valijson && cp -r include/* /usr/local/include/
RUN cd /valijson && cp -r include/* /usr/local/include/ && \
cd / && rm -rf valijson && rm -rf faiss && \
rm -rf grpc && rm -rf libpng && rm -rf opencv && rm -rf swig && rm -rf CMake

FROM base_build

COPY . /vdms
COPY docker/check-in/run_coverage.sh /run_coverage.sh
RUN [ -d /vdms/build ]; rm -rf /vdms/build
RUN cd /vdms && git submodule update --init --recursive && mkdir build && \
RUN cd /vdms && [ -d /vdms/.git ]; git submodule update --init --recursive && mkdir build && \
cd build && cmake -DCODE_COVERAGE=ON .. && make ${BUILD_THREADS} && \
cd / && rm -rf valijson && rm -rf faiss && \
rm -rf grpc && rm -rf libpng && rm -rf opencv && rm -rf swig && rm -rf CMake && \
cp /vdms/config-vdms.json /vdms/build/
cp /vdms/config-vdms.json /vdms/build/ && mkdir -p /vdms/tests/coverage_report

RUN echo '#!/bin/bash' > /start.sh && echo 'cd /vdms/build' >> /start.sh && \
RUN chmod +x /run_coverage.sh && \
echo '#!/bin/bash' > /start.sh && echo 'cd /vdms/build' >> /start.sh && \
echo './vdms' >> /start.sh && chmod 755 /start.sh

CMD ["/start.sh"]
24 changes: 24 additions & 0 deletions docker/check-in/run_coverage.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
cd /vdms/tests

chmod +x run_tests.sh
./run_tests.sh

# -k
# -e /vdms/src/pmgd -e "/vdms/.*\.h" \
# -f /vdms/client -f /vdms/ext -f /vdms/include -f /vdms/src \
# gcovr -d --root /vdms \
# -f /vdms/include \
# -f /vdms/src \
# -e /vdms/src/pmgd \

gcovr -k --root /vdms \
-e /vdms/src/pmgd -e /vdms/build/CMakeFiles \
-f "/vdms/client/.*\.cc" -f "/vdms/ext/.*\.cc" -f "/vdms/src/.*\.cc" \
-f src/SearchExpression.cc \
--exclude-unreachable-branches \
--txt=/vdms/tests/coverage_report/c_coverage_report.txt \
--xml-pretty --xml=/vdms/tests/coverage_report/c_coverage_report.xml

echo "DONE"

cat /vdms/tests/coverage_report/c_coverage_report.txt
11 changes: 1 addition & 10 deletions tests/run_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,20 +7,11 @@ mkdir videos_tests
echo 'not the vdms application - this file is needed for shared key' > vdms

# Gets coverage for files in ../src and ../include
# OMIT Descriptors_Add.add_1by1_and_search_1k due to duration
echo 'Running C++ tests...'
./../build/tests/unit_test \
--gtest_filter=-ImageTest.CreateNameTDB:ImageTest.NoMetadata:VideoTest.CreateUnique:Descriptors_Add.add_1by1_and_search_1k

#OMIT Descriptors_Add.add_1by1_and_search_1k due to duration

gcovr -k -p --root /vdms/ \
-f /vdms/src \
-f /vdms/include \
-e /vdms/src/pmgd \
-e "/vdms/.*\.h" \
--exclude-unreachable-branches --txt \
--xml-pretty --xml=c_coverage_report.xml

# echo 'Running Python tests...'
# cd python
# sh run_python_tests.sh

0 comments on commit 9d44760

Please sign in to comment.