Fixed skupperproject#129: Configure coredumping in GHA and print and …

…archive backtraces
jiridanek · Mar 5, 2022 · 0872c07 · 0872c07
1 parent a843a8c
commit 0872c07
Show file tree

Hide file tree

Showing 2 changed files with 47 additions and 10 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -186,6 +186,7 @@ jobs:
       LD_LIBRARY_PATH: ${{github.workspace}}/install/lib
       QPID_SYSTEM_TEST_TIMEOUT: 300
       QPID_SYSTEM_TEST_SKIP_FALLBACK_SWITCHOVER_TEST: True
+
     steps:
 
       - name: Show environment (Linux)
@@ -209,7 +210,7 @@ jobs:
       - name: Install Linux runtime/test dependencies
         if: ${{ runner.os == 'Linux' }}
         run: |
-          sudo apt update; sudo apt install -y libsasl2-2 libsasl2-modules sasl2-bin libjsoncpp1 libwebsockets15 libbenchmark1 pixz bubblewrap curl
+          sudo apt update; sudo apt install -y libsasl2-2 libsasl2-modules sasl2-bin libjsoncpp1 libwebsockets15 libbenchmark1 pixz bubblewrap curl gdb elfutils findutils gawk
 
       - name: Unpack archive
         run: tar -I pixz -xf archive.tar.xz
@@ -220,9 +221,16 @@ jobs:
       - name: CTest
         working-directory: ${{env.RouterBuildDir}}
         run: |
+          sudo sysctl -w kernel.core_pattern="coredump.%e.%p"
           ulimit -c unlimited
+
           ctest --timeout 1200 -C ${BuildType} -V -T Test --no-compress-output -I ${{matrix.shard}},,${{matrix.shards}} -j2 ${{env.RouterCTestExtraArgs}}
 
+      - name: Report coredump stacktraces (if any tests failed)
+        if: ${{ failure() }}
+        run: |
+          find -name 'coredump*' -exec ${{github.workspace}}/skupper-router/scripts/gha_analyze_coredump.sh {} \;
+
       - name: Upload test results
         uses: actions/upload-artifact@v2
         if: ${{ ! cancelled() }}
@@ -244,7 +252,7 @@ jobs:
         with:
           name: cores_${{matrix.os}}_${{matrix.buildType}}_${{matrix.runtimeCheck}}_${{matrix.protonGitRef}}_${{matrix.shard}}
           path: |
-            **/core
+            **/coredump*
 
   compile_and_test:
     name: "Compile and Test (${{matrix.container}}:${{matrix.containerTag}}, ${{matrix.runtimeCheck}}, proton ${{matrix.protonGitRef}}, shard ${{matrix.shard}} of ${{matrix.shards}})"
@@ -294,6 +302,7 @@ jobs:
       image: 'quay.io/${{ matrix.container }}/${{ matrix.container }}:${{ matrix.containerTag }}'
       volumes:
         - ${{github.workspace}}:${{github.workspace}}
+      options: --privileged --ulimit core=-1 --security-opt apparmor:unconfined --security-opt seccomp=unconfined
 
     env:
       BuildType: ${{matrix.buildType}}
@@ -325,7 +334,6 @@ jobs:
       CCACHE_COMPRESS: 'true'
       CCACHE_MAXSIZE: '400MB'
       QPID_SYSTEM_TEST_TIMEOUT: 300
-      QPID_SYSTEM_TEST_SKIP_FALLBACK_SWITCHOVER_TEST: True
       VERBOSE: 1
 
     steps:
@@ -432,22 +440,24 @@ jobs:
       - name: Install Linux runtime/test dependencies
         if: ${{ runner.os == 'Linux' }}
         run: |
-          dnf install -y curl
-
-      - name: Install Linux runtime/test dependencies (for sanitizers)
-        if: ${{ matrix.runtimeCheck != 'OFF' }}
-        run: |
-          dnf install -y binutils
+          dnf install -y curl gdb binutils elfutils elfutils-debuginfod-client findutils gawk
 
       - name: install qpid-proton python wheel
         run: python3 -m pip install ${ProtonBuildDir}/python/pkgs/python_qpid_proton*.whl
 
       - name: CTest
         working-directory: ${{env.RouterBuildDir}}
         run: |
+          echo "coredump.%e.%p" > /proc/sys/kernel/core_pattern
           ulimit -c unlimited
+
           ctest --timeout 1200 -C ${BuildType} -V -T Test --output-on-failure --no-compress-output -I ${{matrix.shard}},,${{matrix.shards}} -j2 ${{env.RouterCTestExtraArgs}}
 
+      - name: Report coredump stacktraces (if any tests failed)
+        if: ${{ failure() }}
+        run: |
+          find -name 'coredump*' -exec ${{github.workspace}}/skupper-router/scripts/gha_analyze_coredump.sh {} \;
+
       - name: Upload test results
         uses: actions/upload-artifact@v2
         if: ${{ ! cancelled() }}
@@ -469,7 +479,7 @@ jobs:
         with:
           name: cores_${{matrix.container}}_${{matrix.buildType}}_${{matrix.runtimeCheck}}_${{matrix.protonGitRef}}_${{matrix.shard}}
           path: |
-            **/core
+            **/coredump*
 
   python:
     name: 'Python Checker (${{ matrix.container }}:${{ matrix.containerTag }})'

diff --git a/scripts/gha_analyze_coredump.sh b/scripts/gha_analyze_coredump.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -Exeuo pipefail
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+corefile="${1}"
+executable=$(eu-readelf --notes "${corefile}" | awk '/psargs:/ { print $2 }')
+
+export DEBUGINFOD_URLS=https://debuginfod.elfutils.org
+gdb "${executable}" --core "${corefile}" -iex 'set pagination off' -iex 'set debuginfod enabled on' -ex 'thread apply all bt' --batch