Add Support for Uploading Coredumps in Github Actions (babelfish-for-…

…postgresql#2148) In case of crash during the Tests run , if there is a server crash this action will create the text backtrace as well as upload the corefile as an artifact of Github action run. Example of workflow where coredump generated -https://github.com/babelfish-for-postgresql/babelfish_extensions/actions/runs/7194251970 Signed-off-by: Nirmit Shah [email protected]
amazon-aurora · Dec 24, 2023 · 887c8f0 · 887c8f0
1 parent 65c16ef
commit 887c8f0
Show file tree

Hide file tree

Showing 15 changed files with 136 additions and 14 deletions.
diff --git a/.github/composite-actions/build-modified-postgres/action.yml b/.github/composite-actions/build-modified-postgres/action.yml
@@ -90,3 +90,16 @@ runs:
       env:
         HEAD_OWNER: ${{ github.event.pull_request.head.repo.owner.login }}
       shell: bash
+
+    - name: Enable Ubuntu Coredumps
+      run: |
+        sudo mkdir -p /var/coredumps
+        sudo chmod a+w /var/coredumps
+        sudo sysctl kernel.core_pattern=/var/coredumps/core-%e-%p
+        echo 'kernel.core_pattern=/var/coredumps/core-%e-%p' | sudo tee -a /etc/sysctl.conf
+        echo '* soft core unlimited' | sudo tee -a /etc/security/limits.conf
+        sudo systemctl daemon-reload
+        ulimit -c unlimited
+        echo 'DefaultLimitCORE=Infinity' | sudo tee -a /etc/systemd/system.conf
+        sudo systemctl daemon-reexec
+      shell: bash
diff --git a/.github/composite-actions/dump-restore-util/action.yml b/.github/composite-actions/dump-restore-util/action.yml
@@ -46,6 +46,7 @@ runs:
       id: run-pg_dump-restore
       if: always() && steps.setup-new-version.outcome == 'success'
       run: |
+        ulimit -c unlimited
         echo 'Starting dump...'
         cd ~
         mkdir -p upgrade
@@ -70,8 +71,8 @@ runs:
         fi
 
         # Stop old server and start the new.
-        ~/${{ inputs.pg_old_dir }}/bin/pg_ctl -D ~/${{ inputs.pg_old_dir }}/data stop
-        ~/${{ inputs.pg_new_dir }}/bin/pg_ctl -D ~/${{ inputs.pg_new_dir }}/data -l ~/${{ inputs.pg_new_dir }}/data/logfile start
+        ~/${{ inputs.pg_old_dir }}/bin/pg_ctl -c -D ~/${{ inputs.pg_old_dir }}/data stop
+        ~/${{ inputs.pg_new_dir }}/bin/pg_ctl -c -D ~/${{ inputs.pg_new_dir }}/data -l ~/${{ inputs.pg_new_dir }}/data/logfile start
         cd ~/work/babelfish_extensions/babelfish_extensions/
         echo 'Database dump complete.'
 

diff --git a/.github/composite-actions/install-dependencies/action.yml b/.github/composite-actions/install-dependencies/action.yml
@@ -9,7 +9,7 @@ runs:
         curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
         curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list
         sudo apt-get update --fix-missing -y
-        sudo apt-get install uuid-dev openjdk-8-jre libicu-dev libxml2-dev openssl libssl-dev python3-dev libossp-uuid-dev libpq-dev cmake pkg-config g++ build-essential bison mssql-tools unixodbc-dev libsybdb5 freetds-dev freetds-common gdal-bin libgdal-dev libgeos-dev 
+        sudo apt-get install uuid-dev openjdk-8-jre libicu-dev libxml2-dev openssl libssl-dev python3-dev libossp-uuid-dev libpq-dev cmake pkg-config g++ build-essential bison mssql-tools unixodbc-dev libsybdb5 freetds-dev freetds-common gdal-bin libgdal-dev libgeos-dev gdb
         sudo apt install -y ccache
         sudo /usr/sbin/update-ccache-symlinks
         echo 'export PATH="/usr/lib/ccache:$PATH"' | tee -a ~/.bashrc
@@ -24,4 +24,4 @@ runs:
         path: ~/.ccache
         key: ccache-${{ runner.os }}-${{ env.NOW }}
         restore-keys: |
-          ccache-${{ runner.os }}
+          ccache-${{ runner.os }}
diff --git a/.github/composite-actions/install-extensions/action.yml b/.github/composite-actions/install-extensions/action.yml
@@ -19,10 +19,11 @@ runs:
   steps:
     - name: Install Extensions
       run: |
+        ulimit -c unlimited
         cd ~
         export PATH=/opt/mssql-tools/bin:$PATH
         ~/${{inputs.install_dir}}/bin/initdb -D ~/${{inputs.install_dir}}/data/
-        ~/${{inputs.install_dir}}/bin/pg_ctl -D ~/${{inputs.install_dir}}/data/ -l logfile start
+        ~/${{inputs.install_dir}}/bin/pg_ctl -c -D ~/${{inputs.install_dir}}/data/ -l logfile start
         cd ${{inputs.install_dir}}/data
         sudo sed -i "s/#listen_addresses = 'localhost'/listen_addresses = '*'/g" postgresql.conf
         sudo sed -i "s/#shared_preload_libraries = ''/shared_preload_libraries = 'babelfishpg_tds, pg_stat_statements'/g" postgresql.conf
@@ -36,7 +37,7 @@ runs:
           sudo echo "host    all             all             0.0.0.0/0                md5"
           sudo echo "host    all             all             ::/0                     md5"
         } > pg_hba.conf
-        ~/${{inputs.install_dir}}/bin/pg_ctl -D ~/${{inputs.install_dir}}/data/ -l logfile restart
+        ~/${{inputs.install_dir}}/bin/pg_ctl -c -D ~/${{inputs.install_dir}}/data/ -l logfile restart
         cd ~/work/babelfish_extensions/babelfish_extensions/
         sudo ~/${{inputs.install_dir}}/bin/psql -d postgres -U runner -v user="jdbc_user" -v db="jdbc_testdb" -v migration_mode=${{inputs.migration_mode}} -v parallel_query_mode=${{inputs.parallel_query_mode}} -f .github/scripts/create_extension.sql
         sqlcmd -S localhost -U "jdbc_user" -P 12345678 -Q "SELECT @@version GO"

diff --git a/.github/composite-actions/minor-version-upgrade-util/action.yml b/.github/composite-actions/minor-version-upgrade-util/action.yml
@@ -34,8 +34,9 @@ runs:
     - name: Update extensions
       if: always() && steps.build-extensions-newer.outcome == 'success'
       run: |
+        ulimit -c unlimited
         cd ~
-        ~/${{ inputs.install_dir }}/bin/pg_ctl -D ~/${{ inputs.install_dir }}/data/ -l logfile restart
+        ~/${{ inputs.install_dir }}/bin/pg_ctl -c -D ~/${{ inputs.install_dir }}/data/ -l logfile restart
         sudo ~/${{ inputs.install_dir }}/bin/psql -d jdbc_testdb -U runner -c "\dx"
         sudo ~/${{ inputs.install_dir }}/bin/psql -d jdbc_testdb -U runner -c "ALTER EXTENSION "babelfishpg_common" UPDATE; ALTER EXTENSION "babelfishpg_tsql" UPDATE;"
         sudo ~/${{ inputs.install_dir }}/bin/psql -d jdbc_testdb -U runner -c "\dx"

diff --git a/.github/composite-actions/run-pg-upgrade/action.yml b/.github/composite-actions/run-pg-upgrade/action.yml
@@ -19,19 +19,20 @@ runs:
   steps:
     - name: Run pg_upgrade
       run: |
+        ulimit -c unlimited
         echo 'Starting pg_upgrade...'
         cd ~
         mkdir -p upgrade
         cd upgrade
-        ~/${{ inputs.pg_old_dir }}/bin/pg_ctl -D ~/${{ inputs.pg_old_dir }}//data stop
+        ~/${{ inputs.pg_old_dir }}/bin/pg_ctl -c -D ~/${{ inputs.pg_old_dir }}//data stop
         export PATH="$HOME/${{ inputs.pg_new_dir }}/bin:$HOME/${{ inputs.pg_old_dir }}/bin:$PATH"
         export LD_LIBRARY_PATH="$HOME/${{ inputs.pg_new_dir }}/lib:$HOME/${{ inputs.pg_old_dir }}/lib:$LD_LIBRARY_PATH"
         ~/${{ inputs.pg_new_dir }}/bin/pg_upgrade -U runner -b ~/${{ inputs.pg_old_dir }}/bin -B ~/${{ inputs.pg_new_dir }}/bin \
         -d ~/${{ inputs.pg_old_dir }}/data -D ~/${{ inputs.pg_new_dir }}/data -p 5432 -P 5433 -j 4 --link --verbose --retain
         echo 'pg_upgrade completed!'
         echo 'Updating babelfish extensions...'
         cd ~/work/babelfish_extensions/babelfish_extensions/
-        ~/${{ inputs.pg_new_dir }}/bin/pg_ctl -D ~/${{ inputs.pg_new_dir }}/data -l ~/${{ inputs.pg_new_dir }}/data/logfile14 start
+        ~/${{ inputs.pg_new_dir }}/bin/pg_ctl -c -D ~/${{ inputs.pg_new_dir }}/data -l ~/${{ inputs.pg_new_dir }}/data/logfile14 start
         sudo ~/${{ inputs.pg_new_dir }}/bin/psql -d jdbc_testdb -U runner -c "ALTER EXTENSION babelfishpg_common UPDATE; ALTER EXTENSION babelfishpg_tsql UPDATE;"
         sudo ~/${{ inputs.pg_new_dir }}/bin/psql -d jdbc_testdb -U runner -c "\dx"
         echo 'Reset bbf database settings...'

diff --git a/.github/composite-actions/setup-base-version/action.yml b/.github/composite-actions/setup-base-version/action.yml
@@ -88,10 +88,11 @@ runs:
     # checked out a branch/tag which may not have the updated install-extension composite action
     - name: Install extensions
       run: |
+        ulimit -c unlimited
         cd ~
         export PATH=/opt/mssql-tools/bin:$PATH
         ~/${{ inputs.install_dir }}/bin/initdb -D ~/${{ inputs.install_dir }}/data/
-        ~/${{ inputs.install_dir }}/bin/pg_ctl -D ~/${{ inputs.install_dir }}/data/ -l logfile start
+        ~/${{ inputs.install_dir }}/bin/pg_ctl -c -D ~/${{ inputs.install_dir }}/data/ -l logfile start
         cd ${{ inputs.install_dir }}/data
         sudo sed -i "s/#listen_addresses = 'localhost'/listen_addresses = '*'/g" postgresql.conf
         sudo sed -i "s/#shared_preload_libraries = ''/shared_preload_libraries = 'babelfishpg_tds, pg_stat_statements'/g" postgresql.conf
@@ -105,7 +106,7 @@ runs:
           sudo echo "host    all             all             0.0.0.0/0                md5"
           sudo echo "host    all             all             ::/0                     md5"
         } > pg_hba.conf
-        ~/${{ inputs.install_dir }}/bin/pg_ctl -D ~/${{ inputs.install_dir }}/data/ -l logfile restart
+        ~/${{ inputs.install_dir }}/bin/pg_ctl -c -D ~/${{ inputs.install_dir }}/data/ -l logfile restart
         sudo ~/${{ inputs.install_dir }}/bin/psql -d postgres -U runner -c "CREATE USER jdbc_user WITH SUPERUSER CREATEDB CREATEROLE PASSWORD '12345678' INHERIT;"
         sudo ~/${{ inputs.install_dir }}/bin/psql -d postgres -U runner -c "DROP DATABASE IF EXISTS jdbc_testdb;"
         sudo ~/${{ inputs.install_dir }}/bin/psql -d postgres -U runner -c "CREATE DATABASE jdbc_testdb OWNER jdbc_user;"

diff --git a/.github/composite-actions/upload-coredump/action.yml b/.github/composite-actions/upload-coredump/action.yml
@@ -0,0 +1,24 @@
+name: 'Check for coredumps on the System'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Create coredump summaries
+      if: always()
+      run: |
+        gdb --quiet --core /var/coredumps/* --batch -ex 'info auxv'
+        ./.github/scripts/get_coredumps linux /var/coredumps
+        ls ./*.out
+      shell: bash
+
+
+
+    - name: Upload Coredumps
+      uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: coredumps
+        path: |
+          /var/coredumps/*
+          /home/runner/work/babelfish_extensions/babelfish_extensions/bt-*
+
diff --git a/.github/scripts/get_coredumps b/.github/scripts/get_coredumps
@@ -0,0 +1,54 @@
+ #! /bin/sh
+
+if [ $# -ne 2 ]; then
+    echo "cores_backtrace.sh <os> <directory>"
+    exit 1
+fi
+
+os=$1
+directory=$2
+
+case $os in
+    freebsd|linux|macos)
+    ;;
+    *)
+        echo "unsupported operating system ${os}"
+        exit 1
+    ;;
+esac
+
+first=1
+for corefile in $(find "$directory" -type f -name "core*") ; do
+    if [ "$first" -eq 1 ]; then
+        first=0
+    else
+        # to make it easier to separate the different crash reports
+        echo -e '\n\n'
+    fi
+
+    dump_name=$(basename $corefile)
+
+    if [ "$os" = 'macos' ]; then
+        lldb -c $corefile --batch -o 'thread backtrace all' -o 'quit'
+    else
+        auxv=$(gdb --quiet --core ${corefile} --batch -ex 'info auxv' 2>/dev/null)
+        if [ $? -ne 0 ]; then
+            echo "could not process ${corefile}"
+            continue
+        fi
+
+        if [ "$os" = 'freebsd' ]; then
+            binary=$(echo "$auxv" | grep AT_EXECPATH | perl -pe "s/^.*\"(.*)\"\$/\$1/g")
+        elif [ "$os" = 'linux' ]; then
+            binary=$(echo "$auxv" | grep AT_EXECFN | perl -pe "s/^.*\"(.*)\"\$/\$1/g")
+        else
+            echo 'should not get here'
+            exit 1
+        fi
+
+        echo "dumping ${corefile} for ${binary}"
+        gdb --batch --quiet -ex "thread apply all bt full" -ex "quit" "$binary" "$corefile" 2>/dev/null | tee "bt-$dump_name.out"
+        # to get 0 exit code
+        ls ./ 
+    fi
+done
diff --git a/.github/workflows/jdbc-tests-with-parallel-query.yml b/.github/workflows/jdbc-tests-with-parallel-query.yml
@@ -98,3 +98,7 @@ jobs:
         with:
           name: output-diff.diff
           path: test/JDBC/Info/output-diff.diff
+
+      - name: Check and upload coredumps
+        if: always() && steps.jdbc.outcome == 'failure'
+        uses: ./.github/composite-actions/upload-coredump
diff --git a/.github/workflows/jdbc-tests.yml b/.github/workflows/jdbc-tests.yml
@@ -88,3 +88,7 @@ jobs:
         with:
           name: output-diff.diff
           path: test/JDBC/Info/output-diff.diff
+
+      - name: Check and upload coredumps
+        if: always() && steps.jdbc.outcome == 'failure'
+        uses: ./.github/composite-actions/upload-coredump
diff --git a/.github/workflows/major-version-upgrade.yml b/.github/workflows/major-version-upgrade.yml
@@ -92,10 +92,11 @@ jobs:
         id: install-extensions-old
         if: always() && steps.build-extensions-old.outcome == 'success'
         run: |
+          ulimit -c unlimited
           cd ~
           export PATH=/opt/mssql-tools/bin:$PATH
           ~/${{env.OLD_INSTALL_DIR}}/bin/initdb -D ~/${{env.OLD_INSTALL_DIR}}/data
-          ~/${{env.OLD_INSTALL_DIR}}/bin/pg_ctl -D ~/${{env.OLD_INSTALL_DIR}}/data -l logfile13 start
+          ~/${{env.OLD_INSTALL_DIR}}/bin/pg_ctl -c -D ~/${{env.OLD_INSTALL_DIR}}/data -l logfile13 start
           cd ${{env.OLD_INSTALL_DIR}}/data
           sudo sed -i "s/#listen_addresses = 'localhost'/listen_addresses = '*'/g" postgresql.conf
           sudo sed -i "s/#shared_preload_libraries = ''/shared_preload_libraries = 'babelfishpg_tds, pg_stat_statements'/g" postgresql.conf
@@ -109,7 +110,7 @@ jobs:
             sudo echo "host    all             all             0.0.0.0/0                md5"
             sudo echo "host    all             all             ::/0                     md5"
           } > pg_hba.conf
-          ~/${{env.OLD_INSTALL_DIR}}/bin/pg_ctl -D ~/${{env.OLD_INSTALL_DIR}}/data -l logfile13 restart
+          ~/${{env.OLD_INSTALL_DIR}}/bin/pg_ctl -c -D ~/${{env.OLD_INSTALL_DIR}}/data -l logfile13 restart
           cd ~/work/babelfish_extensions/babelfish_extensions/
           sudo ~/${{env.OLD_INSTALL_DIR}}/bin/psql -d postgres -U runner -v user="jdbc_user" -v db="jdbc_testdb" -f .github/scripts/create_extension.sql
           sqlcmd -S localhost -U jdbc_user -P 12345678 -Q "SELECT @@version GO"
@@ -236,3 +237,7 @@ jobs:
         with:
           name: output-diff.diff
           path: test/JDBC/Info/output-diff.diff
+
+      - name: Check and upload coredumps
+        if: always() && steps.jdbc.outcome == 'failure'
+        uses: ./.github/composite-actions/upload-coredump
diff --git a/.github/workflows/minor-version-upgrade.yml b/.github/workflows/minor-version-upgrade.yml
@@ -105,8 +105,9 @@ jobs:
       - name: Update extensions
         if: always() && steps.build-postgis-extension.outcome == 'success'
         run: |
+          ulimit -c unlimited
           cd ~
-          ~/psql/bin/pg_ctl -D ~/psql/data/ -l logfile restart
+          ~/psql/bin/pg_ctl -c -D ~/psql/data/ -l logfile restart
           sudo ~/psql/bin/psql -d jdbc_testdb -U runner -c "\dx"
           sudo ~/psql/bin/psql -d jdbc_testdb -U runner -c "ALTER EXTENSION "babelfishpg_common" UPDATE; ALTER EXTENSION "babelfishpg_tsql" UPDATE;"
           sudo ~/psql/bin/psql -d jdbc_testdb -U runner -c "\dx"
@@ -152,3 +153,7 @@ jobs:
         with:
           name: upgrade-output-diff.diff
           path: test/JDBC/Info/upgrade-output-diff.diff
+
+      - name: Check and upload coredumps
+        if: always() && steps.jdbc.outcome == 'failure'
+        uses: ./.github/composite-actions/upload-coredump
diff --git a/.github/workflows/singledb-version-upgrade.yml b/.github/workflows/singledb-version-upgrade.yml
@@ -83,3 +83,7 @@ jobs:
             ~/upgrade/*.diff
             ~/failed-testscript-outputs
             ~/${{env.NEW_INSTALL_DIR}}/data/pg_upgrade_output.d/*
+      
+      - name: Check and upload coredumps
+        if: always() && ( steps.upgrade-and-verify.outcome == 'failure' || steps.setup-base-version.outcome == 'failure' )
+        uses: ./.github/composite-actions/upload-coredump
diff --git a/.github/workflows/upgrade-test.yml b/.github/workflows/upgrade-test.yml
@@ -133,4 +133,8 @@ jobs:
             ~/psql*/data/logfile
             ~/psql*/data/pg_upgrade_output.d/*
             ~/failed-testscript-outputs
+      
+      - name: Check and upload coredumps
+        if: always() && (steps.setup-base-version.outcome == 'failure' || steps.upgrade-and-test.outcome == 'failure')
+        uses: ./.github/composite-actions/upload-coredump