From 0872c07dc0cd6cdff9f81442a9133446307d3bdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Sat, 5 Mar 2022 20:00:13 +0100 Subject: [PATCH] Fixed #129: Configure coredumping in GHA and print and archive backtraces --- .github/workflows/build.yaml | 30 ++++++++++++++++++++---------- scripts/gha_analyze_coredump.sh | 27 +++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 10 deletions(-) create mode 100755 scripts/gha_analyze_coredump.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ba996b9a0..86c6132de 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -186,6 +186,7 @@ jobs: LD_LIBRARY_PATH: ${{github.workspace}}/install/lib QPID_SYSTEM_TEST_TIMEOUT: 300 QPID_SYSTEM_TEST_SKIP_FALLBACK_SWITCHOVER_TEST: True + steps: - name: Show environment (Linux) @@ -209,7 +210,7 @@ jobs: - name: Install Linux runtime/test dependencies if: ${{ runner.os == 'Linux' }} run: | - sudo apt update; sudo apt install -y libsasl2-2 libsasl2-modules sasl2-bin libjsoncpp1 libwebsockets15 libbenchmark1 pixz bubblewrap curl + sudo apt update; sudo apt install -y libsasl2-2 libsasl2-modules sasl2-bin libjsoncpp1 libwebsockets15 libbenchmark1 pixz bubblewrap curl gdb elfutils findutils gawk - name: Unpack archive run: tar -I pixz -xf archive.tar.xz @@ -220,9 +221,16 @@ jobs: - name: CTest working-directory: ${{env.RouterBuildDir}} run: | + sudo sysctl -w kernel.core_pattern="coredump.%e.%p" ulimit -c unlimited + ctest --timeout 1200 -C ${BuildType} -V -T Test --no-compress-output -I ${{matrix.shard}},,${{matrix.shards}} -j2 ${{env.RouterCTestExtraArgs}} + - name: Report coredump stacktraces (if any tests failed) + if: ${{ failure() }} + run: | + find -name 'coredump*' -exec ${{github.workspace}}/skupper-router/scripts/gha_analyze_coredump.sh {} \; + - name: Upload test results uses: actions/upload-artifact@v2 if: ${{ ! cancelled() }} @@ -244,7 +252,7 @@ jobs: with: name: cores_${{matrix.os}}_${{matrix.buildType}}_${{matrix.runtimeCheck}}_${{matrix.protonGitRef}}_${{matrix.shard}} path: | - **/core + **/coredump* compile_and_test: name: "Compile and Test (${{matrix.container}}:${{matrix.containerTag}}, ${{matrix.runtimeCheck}}, proton ${{matrix.protonGitRef}}, shard ${{matrix.shard}} of ${{matrix.shards}})" @@ -294,6 +302,7 @@ jobs: image: 'quay.io/${{ matrix.container }}/${{ matrix.container }}:${{ matrix.containerTag }}' volumes: - ${{github.workspace}}:${{github.workspace}} + options: --privileged --ulimit core=-1 --security-opt apparmor:unconfined --security-opt seccomp=unconfined env: BuildType: ${{matrix.buildType}} @@ -325,7 +334,6 @@ jobs: CCACHE_COMPRESS: 'true' CCACHE_MAXSIZE: '400MB' QPID_SYSTEM_TEST_TIMEOUT: 300 - QPID_SYSTEM_TEST_SKIP_FALLBACK_SWITCHOVER_TEST: True VERBOSE: 1 steps: @@ -432,12 +440,7 @@ jobs: - name: Install Linux runtime/test dependencies if: ${{ runner.os == 'Linux' }} run: | - dnf install -y curl - - - name: Install Linux runtime/test dependencies (for sanitizers) - if: ${{ matrix.runtimeCheck != 'OFF' }} - run: | - dnf install -y binutils + dnf install -y curl gdb binutils elfutils elfutils-debuginfod-client findutils gawk - name: install qpid-proton python wheel run: python3 -m pip install ${ProtonBuildDir}/python/pkgs/python_qpid_proton*.whl @@ -445,9 +448,16 @@ jobs: - name: CTest working-directory: ${{env.RouterBuildDir}} run: | + echo "coredump.%e.%p" > /proc/sys/kernel/core_pattern ulimit -c unlimited + ctest --timeout 1200 -C ${BuildType} -V -T Test --output-on-failure --no-compress-output -I ${{matrix.shard}},,${{matrix.shards}} -j2 ${{env.RouterCTestExtraArgs}} + - name: Report coredump stacktraces (if any tests failed) + if: ${{ failure() }} + run: | + find -name 'coredump*' -exec ${{github.workspace}}/skupper-router/scripts/gha_analyze_coredump.sh {} \; + - name: Upload test results uses: actions/upload-artifact@v2 if: ${{ ! cancelled() }} @@ -469,7 +479,7 @@ jobs: with: name: cores_${{matrix.container}}_${{matrix.buildType}}_${{matrix.runtimeCheck}}_${{matrix.protonGitRef}}_${{matrix.shard}} path: | - **/core + **/coredump* python: name: 'Python Checker (${{ matrix.container }}:${{ matrix.containerTag }})' diff --git a/scripts/gha_analyze_coredump.sh b/scripts/gha_analyze_coredump.sh new file mode 100755 index 000000000..8b55550b1 --- /dev/null +++ b/scripts/gha_analyze_coredump.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -Exeuo pipefail + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +corefile="${1}" +executable=$(eu-readelf --notes "${corefile}" | awk '/psargs:/ { print $2 }') + +export DEBUGINFOD_URLS=https://debuginfod.elfutils.org +gdb "${executable}" --core "${corefile}" -iex 'set pagination off' -iex 'set debuginfod enabled on' -ex 'thread apply all bt' --batch