From b8f213bd23eec9d3983df8a25fa42ed5cde8ff9d Mon Sep 17 00:00:00 2001 From: Nirmit Shah Date: Mon, 11 Dec 2023 03:40:01 +0000 Subject: [PATCH 1/2] Add Support for Uploading Coredumps in Github Actions Signed-off-by: Nirmit Shah --- .../build-modified-postgres/action.yml | 13 +++++ .../dump-restore-util/action.yml | 5 +- .../install-dependencies/action.yml | 4 +- .../install-extensions/action.yml | 5 +- .../minor-version-upgrade-util/action.yml | 3 +- .../run-pg-upgrade/action.yml | 5 +- .../setup-base-version/action.yml | 5 +- .../upload-coredump/action.yml | 24 +++++++++ .github/scripts/get_coredumps | 54 +++++++++++++++++++ .../jdbc-tests-with-parallel-query.yml | 4 ++ .github/workflows/jdbc-tests.yml | 4 ++ .github/workflows/major-version-upgrade.yml | 9 +++- .github/workflows/minor-version-upgrade.yml | 7 ++- .../workflows/singledb-version-upgrade.yml | 4 ++ .github/workflows/upgrade-test.yml | 4 ++ contrib/babelfishpg_common/src/numeric.c | 3 +- test/JDBC/jdbc_schedule | 5 +- 17 files changed, 142 insertions(+), 16 deletions(-) create mode 100644 .github/composite-actions/upload-coredump/action.yml create mode 100755 .github/scripts/get_coredumps diff --git a/.github/composite-actions/build-modified-postgres/action.yml b/.github/composite-actions/build-modified-postgres/action.yml index 06baec2d30..d75b339802 100644 --- a/.github/composite-actions/build-modified-postgres/action.yml +++ b/.github/composite-actions/build-modified-postgres/action.yml @@ -68,3 +68,16 @@ runs: env: HEAD_OWNER: ${{ github.event.pull_request.head.repo.owner.login }} shell: bash + + - name: Enable Ubuntu Coredumps + run: | + sudo mkdir -p /var/coredumps + sudo chmod a+w /var/coredumps + sudo sysctl kernel.core_pattern=/var/coredumps/core-%e-%p + echo 'kernel.core_pattern=/var/coredumps/core-%e-%p' | sudo tee -a /etc/sysctl.conf + echo '* soft core unlimited' | sudo tee -a /etc/security/limits.conf + sudo systemctl daemon-reload + ulimit -c unlimited + echo 'DefaultLimitCORE=Infinity' | sudo tee -a /etc/systemd/system.conf + sudo systemctl daemon-reexec + shell: bash diff --git a/.github/composite-actions/dump-restore-util/action.yml b/.github/composite-actions/dump-restore-util/action.yml index 4437a92d93..378d9c8488 100644 --- a/.github/composite-actions/dump-restore-util/action.yml +++ b/.github/composite-actions/dump-restore-util/action.yml @@ -46,6 +46,7 @@ runs: id: run-pg_dump-restore if: always() && steps.setup-new-version.outcome == 'success' run: | + ulimit -c unlimited echo 'Starting dump...' cd ~ mkdir -p upgrade @@ -70,8 +71,8 @@ runs: fi # Stop old server and start the new. - ~/${{ inputs.pg_old_dir }}/bin/pg_ctl -D ~/${{ inputs.pg_old_dir }}/data stop - ~/${{ inputs.pg_new_dir }}/bin/pg_ctl -D ~/${{ inputs.pg_new_dir }}/data -l ~/${{ inputs.pg_new_dir }}/data/logfile start + ~/${{ inputs.pg_old_dir }}/bin/pg_ctl -c -D ~/${{ inputs.pg_old_dir }}/data stop + ~/${{ inputs.pg_new_dir }}/bin/pg_ctl -c -D ~/${{ inputs.pg_new_dir }}/data -l ~/${{ inputs.pg_new_dir }}/data/logfile start cd ~/work/babelfish_extensions/babelfish_extensions/ echo 'Database dump complete.' diff --git a/.github/composite-actions/install-dependencies/action.yml b/.github/composite-actions/install-dependencies/action.yml index 4abda2db7e..658d6ad489 100644 --- a/.github/composite-actions/install-dependencies/action.yml +++ b/.github/composite-actions/install-dependencies/action.yml @@ -9,7 +9,7 @@ runs: curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list sudo apt-get update --fix-missing -y - sudo apt-get install uuid-dev openjdk-8-jre libicu-dev libxml2-dev openssl libssl-dev python3-dev libossp-uuid-dev libpq-dev cmake pkg-config g++ build-essential bison mssql-tools unixodbc-dev libsybdb5 freetds-dev freetds-common gdal-bin libgdal-dev libgeos-dev + sudo apt-get install uuid-dev openjdk-8-jre libicu-dev libxml2-dev openssl libssl-dev python3-dev libossp-uuid-dev libpq-dev cmake pkg-config g++ build-essential bison mssql-tools unixodbc-dev libsybdb5 freetds-dev freetds-common gdal-bin libgdal-dev libgeos-dev gdb sudo apt install -y ccache sudo /usr/sbin/update-ccache-symlinks echo 'export PATH="/usr/lib/ccache:$PATH"' | tee -a ~/.bashrc @@ -24,4 +24,4 @@ runs: path: ~/.ccache key: ccache-${{ runner.os }}-${{ env.NOW }} restore-keys: | - ccache-${{ runner.os }} \ No newline at end of file + ccache-${{ runner.os }} diff --git a/.github/composite-actions/install-extensions/action.yml b/.github/composite-actions/install-extensions/action.yml index e49e30e6ac..398e821b79 100644 --- a/.github/composite-actions/install-extensions/action.yml +++ b/.github/composite-actions/install-extensions/action.yml @@ -19,10 +19,11 @@ runs: steps: - name: Install Extensions run: | + ulimit -c unlimited cd ~ export PATH=/opt/mssql-tools/bin:$PATH ~/${{inputs.install_dir}}/bin/initdb -D ~/${{inputs.install_dir}}/data/ - ~/${{inputs.install_dir}}/bin/pg_ctl -D ~/${{inputs.install_dir}}/data/ -l logfile start + ~/${{inputs.install_dir}}/bin/pg_ctl -c -D ~/${{inputs.install_dir}}/data/ -l logfile start cd ${{inputs.install_dir}}/data sudo sed -i "s/#listen_addresses = 'localhost'/listen_addresses = '*'/g" postgresql.conf sudo sed -i "s/#shared_preload_libraries = ''/shared_preload_libraries = 'babelfishpg_tds, pg_stat_statements'/g" postgresql.conf @@ -36,7 +37,7 @@ runs: sudo echo "host all all 0.0.0.0/0 md5" sudo echo "host all all ::/0 md5" } > pg_hba.conf - ~/${{inputs.install_dir}}/bin/pg_ctl -D ~/${{inputs.install_dir}}/data/ -l logfile restart + ~/${{inputs.install_dir}}/bin/pg_ctl -c -D ~/${{inputs.install_dir}}/data/ -l logfile restart cd ~/work/babelfish_extensions/babelfish_extensions/ sudo ~/${{inputs.install_dir}}/bin/psql -d postgres -U runner -v user="jdbc_user" -v db="jdbc_testdb" -v migration_mode=${{inputs.migration_mode}} -v parallel_query_mode=${{inputs.parallel_query_mode}} -f .github/scripts/create_extension.sql sqlcmd -S localhost -U "jdbc_user" -P 12345678 -Q "SELECT @@version GO" diff --git a/.github/composite-actions/minor-version-upgrade-util/action.yml b/.github/composite-actions/minor-version-upgrade-util/action.yml index 57bcc60de6..82cc8f9160 100644 --- a/.github/composite-actions/minor-version-upgrade-util/action.yml +++ b/.github/composite-actions/minor-version-upgrade-util/action.yml @@ -34,8 +34,9 @@ runs: - name: Update extensions if: always() && steps.build-extensions-newer.outcome == 'success' run: | + ulimit -c unlimited cd ~ - ~/${{ inputs.install_dir }}/bin/pg_ctl -D ~/${{ inputs.install_dir }}/data/ -l logfile restart + ~/${{ inputs.install_dir }}/bin/pg_ctl -c -D ~/${{ inputs.install_dir }}/data/ -l logfile restart sudo ~/${{ inputs.install_dir }}/bin/psql -d jdbc_testdb -U runner -c "\dx" sudo ~/${{ inputs.install_dir }}/bin/psql -d jdbc_testdb -U runner -c "ALTER EXTENSION "babelfishpg_common" UPDATE; ALTER EXTENSION "babelfishpg_tsql" UPDATE;" sudo ~/${{ inputs.install_dir }}/bin/psql -d jdbc_testdb -U runner -c "\dx" diff --git a/.github/composite-actions/run-pg-upgrade/action.yml b/.github/composite-actions/run-pg-upgrade/action.yml index 2977485807..12407774de 100644 --- a/.github/composite-actions/run-pg-upgrade/action.yml +++ b/.github/composite-actions/run-pg-upgrade/action.yml @@ -19,11 +19,12 @@ runs: steps: - name: Run pg_upgrade run: | + ulimit -c unlimited echo 'Starting pg_upgrade...' cd ~ mkdir -p upgrade cd upgrade - ~/${{ inputs.pg_old_dir }}/bin/pg_ctl -D ~/${{ inputs.pg_old_dir }}//data stop + ~/${{ inputs.pg_old_dir }}/bin/pg_ctl -c -D ~/${{ inputs.pg_old_dir }}//data stop export PATH="$HOME/${{ inputs.pg_new_dir }}/bin:$HOME/${{ inputs.pg_old_dir }}/bin:$PATH" export LD_LIBRARY_PATH="$HOME/${{ inputs.pg_new_dir }}/lib:$HOME/${{ inputs.pg_old_dir }}/lib:$LD_LIBRARY_PATH" ~/${{ inputs.pg_new_dir }}/bin/pg_upgrade -U runner -b ~/${{ inputs.pg_old_dir }}/bin -B ~/${{ inputs.pg_new_dir }}/bin \ @@ -31,7 +32,7 @@ runs: echo 'pg_upgrade completed!' echo 'Updating babelfish extensions...' cd ~/work/babelfish_extensions/babelfish_extensions/ - ~/${{ inputs.pg_new_dir }}/bin/pg_ctl -D ~/${{ inputs.pg_new_dir }}/data -l ~/${{ inputs.pg_new_dir }}/data/logfile14 start + ~/${{ inputs.pg_new_dir }}/bin/pg_ctl -c -D ~/${{ inputs.pg_new_dir }}/data -l ~/${{ inputs.pg_new_dir }}/data/logfile14 start sudo ~/${{ inputs.pg_new_dir }}/bin/psql -d jdbc_testdb -U runner -c "ALTER EXTENSION babelfishpg_common UPDATE; ALTER EXTENSION babelfishpg_tsql UPDATE;" sudo ~/${{ inputs.pg_new_dir }}/bin/psql -d jdbc_testdb -U runner -c "\dx" echo 'Reset bbf database settings...' diff --git a/.github/composite-actions/setup-base-version/action.yml b/.github/composite-actions/setup-base-version/action.yml index b8eddf5918..c12cba804d 100644 --- a/.github/composite-actions/setup-base-version/action.yml +++ b/.github/composite-actions/setup-base-version/action.yml @@ -88,10 +88,11 @@ runs: # checked out a branch/tag which may not have the updated install-extension composite action - name: Install extensions run: | + ulimit -c unlimited cd ~ export PATH=/opt/mssql-tools/bin:$PATH ~/${{ inputs.install_dir }}/bin/initdb -D ~/${{ inputs.install_dir }}/data/ - ~/${{ inputs.install_dir }}/bin/pg_ctl -D ~/${{ inputs.install_dir }}/data/ -l logfile start + ~/${{ inputs.install_dir }}/bin/pg_ctl -c -D ~/${{ inputs.install_dir }}/data/ -l logfile start cd ${{ inputs.install_dir }}/data sudo sed -i "s/#listen_addresses = 'localhost'/listen_addresses = '*'/g" postgresql.conf sudo sed -i "s/#shared_preload_libraries = ''/shared_preload_libraries = 'babelfishpg_tds, pg_stat_statements'/g" postgresql.conf @@ -105,7 +106,7 @@ runs: sudo echo "host all all 0.0.0.0/0 md5" sudo echo "host all all ::/0 md5" } > pg_hba.conf - ~/${{ inputs.install_dir }}/bin/pg_ctl -D ~/${{ inputs.install_dir }}/data/ -l logfile restart + ~/${{ inputs.install_dir }}/bin/pg_ctl -c -D ~/${{ inputs.install_dir }}/data/ -l logfile restart sudo ~/${{ inputs.install_dir }}/bin/psql -d postgres -U runner -c "CREATE USER jdbc_user WITH SUPERUSER CREATEDB CREATEROLE PASSWORD '12345678' INHERIT;" sudo ~/${{ inputs.install_dir }}/bin/psql -d postgres -U runner -c "DROP DATABASE IF EXISTS jdbc_testdb;" sudo ~/${{ inputs.install_dir }}/bin/psql -d postgres -U runner -c "CREATE DATABASE jdbc_testdb OWNER jdbc_user;" diff --git a/.github/composite-actions/upload-coredump/action.yml b/.github/composite-actions/upload-coredump/action.yml new file mode 100644 index 0000000000..d7540cce7a --- /dev/null +++ b/.github/composite-actions/upload-coredump/action.yml @@ -0,0 +1,24 @@ +name: 'Check for coredumps on the System' + +runs: + using: "composite" + steps: + - name: Create coredump summaries + if: always() + run: | + gdb --quiet --core /var/coredumps/* --batch -ex 'info auxv' + ./.github/scripts/get_coredumps linux /var/coredumps + ls ./*.out + shell: bash + + + + - name: Upload Coredumps + uses: actions/upload-artifact@v3 + if: always() + with: + name: coredumps + path: | + /var/coredumps/* + /home/runner/work/babelfish_extensions/babelfish_extensions/bt-* + diff --git a/.github/scripts/get_coredumps b/.github/scripts/get_coredumps new file mode 100755 index 0000000000..517bcbb58d --- /dev/null +++ b/.github/scripts/get_coredumps @@ -0,0 +1,54 @@ + #! /bin/sh + +if [ $# -ne 2 ]; then + echo "cores_backtrace.sh " + exit 1 +fi + +os=$1 +directory=$2 + +case $os in + freebsd|linux|macos) + ;; + *) + echo "unsupported operating system ${os}" + exit 1 + ;; +esac + +first=1 +for corefile in $(find "$directory" -type f -name "core*") ; do + if [ "$first" -eq 1 ]; then + first=0 + else + # to make it easier to separate the different crash reports + echo -e '\n\n' + fi + + dump_name=$(basename $corefile) + + if [ "$os" = 'macos' ]; then + lldb -c $corefile --batch -o 'thread backtrace all' -o 'quit' + else + auxv=$(gdb --quiet --core ${corefile} --batch -ex 'info auxv' 2>/dev/null) + if [ $? -ne 0 ]; then + echo "could not process ${corefile}" + continue + fi + + if [ "$os" = 'freebsd' ]; then + binary=$(echo "$auxv" | grep AT_EXECPATH | perl -pe "s/^.*\"(.*)\"\$/\$1/g") + elif [ "$os" = 'linux' ]; then + binary=$(echo "$auxv" | grep AT_EXECFN | perl -pe "s/^.*\"(.*)\"\$/\$1/g") + else + echo 'should not get here' + exit 1 + fi + + echo "dumping ${corefile} for ${binary}" + gdb --batch --quiet -ex "thread apply all bt full" -ex "quit" "$binary" "$corefile" 2>/dev/null | tee "bt-$dump_name.out" + # to get 0 exit code + ls ./ + fi +done diff --git a/.github/workflows/jdbc-tests-with-parallel-query.yml b/.github/workflows/jdbc-tests-with-parallel-query.yml index 90c1e5b1ae..67ed18fb75 100644 --- a/.github/workflows/jdbc-tests-with-parallel-query.yml +++ b/.github/workflows/jdbc-tests-with-parallel-query.yml @@ -98,3 +98,7 @@ jobs: with: name: output-diff.diff path: test/JDBC/Info/output-diff.diff + + - name: Check and upload coredumps + if: always() && steps.jdbc.outcome == 'failure' + uses: ./.github/composite-actions/upload-coredump \ No newline at end of file diff --git a/.github/workflows/jdbc-tests.yml b/.github/workflows/jdbc-tests.yml index cb5a85de38..75e4bf5950 100644 --- a/.github/workflows/jdbc-tests.yml +++ b/.github/workflows/jdbc-tests.yml @@ -92,3 +92,7 @@ jobs: with: name: output-diff.diff path: test/JDBC/Info/output-diff.diff + + - name: Check and upload coredumps + if: always() && steps.jdbc.outcome == 'failure' + uses: ./.github/composite-actions/upload-coredump \ No newline at end of file diff --git a/.github/workflows/major-version-upgrade.yml b/.github/workflows/major-version-upgrade.yml index ed3deb3eaf..360cf5105e 100644 --- a/.github/workflows/major-version-upgrade.yml +++ b/.github/workflows/major-version-upgrade.yml @@ -92,10 +92,11 @@ jobs: id: install-extensions-old if: always() && steps.build-extensions-old.outcome == 'success' run: | + ulimit -c unlimited cd ~ export PATH=/opt/mssql-tools/bin:$PATH ~/${{env.OLD_INSTALL_DIR}}/bin/initdb -D ~/${{env.OLD_INSTALL_DIR}}/data - ~/${{env.OLD_INSTALL_DIR}}/bin/pg_ctl -D ~/${{env.OLD_INSTALL_DIR}}/data -l logfile13 start + ~/${{env.OLD_INSTALL_DIR}}/bin/pg_ctl -c -D ~/${{env.OLD_INSTALL_DIR}}/data -l logfile13 start cd ${{env.OLD_INSTALL_DIR}}/data sudo sed -i "s/#listen_addresses = 'localhost'/listen_addresses = '*'/g" postgresql.conf sudo sed -i "s/#shared_preload_libraries = ''/shared_preload_libraries = 'babelfishpg_tds, pg_stat_statements'/g" postgresql.conf @@ -109,7 +110,7 @@ jobs: sudo echo "host all all 0.0.0.0/0 md5" sudo echo "host all all ::/0 md5" } > pg_hba.conf - ~/${{env.OLD_INSTALL_DIR}}/bin/pg_ctl -D ~/${{env.OLD_INSTALL_DIR}}/data -l logfile13 restart + ~/${{env.OLD_INSTALL_DIR}}/bin/pg_ctl -c -D ~/${{env.OLD_INSTALL_DIR}}/data -l logfile13 restart cd ~/work/babelfish_extensions/babelfish_extensions/ sudo ~/${{env.OLD_INSTALL_DIR}}/bin/psql -d postgres -U runner -v user="jdbc_user" -v db="jdbc_testdb" -f .github/scripts/create_extension.sql sqlcmd -S localhost -U jdbc_user -P 12345678 -Q "SELECT @@version GO" @@ -226,3 +227,7 @@ jobs: with: name: output-diff.diff path: test/JDBC/Info/output-diff.diff + + - name: Check and upload coredumps + if: always() && steps.jdbc.outcome == 'failure' + uses: ./.github/composite-actions/upload-coredump diff --git a/.github/workflows/minor-version-upgrade.yml b/.github/workflows/minor-version-upgrade.yml index 7a5fe74e9d..8f46f90fe4 100644 --- a/.github/workflows/minor-version-upgrade.yml +++ b/.github/workflows/minor-version-upgrade.yml @@ -105,8 +105,9 @@ jobs: - name: Update extensions if: always() && steps.build-postgis-extension.outcome == 'success' run: | + ulimit -c unlimited cd ~ - ~/psql/bin/pg_ctl -D ~/psql/data/ -l logfile restart + ~/psql/bin/pg_ctl -c -D ~/psql/data/ -l logfile restart sudo ~/psql/bin/psql -d jdbc_testdb -U runner -c "\dx" sudo ~/psql/bin/psql -d jdbc_testdb -U runner -c "ALTER EXTENSION "babelfishpg_common" UPDATE; ALTER EXTENSION "babelfishpg_tsql" UPDATE;" sudo ~/psql/bin/psql -d jdbc_testdb -U runner -c "\dx" @@ -152,3 +153,7 @@ jobs: with: name: upgrade-output-diff.diff path: test/JDBC/Info/upgrade-output-diff.diff + + - name: Check and upload coredumps + if: always() && steps.jdbc.outcome == 'failure' + uses: ./.github/composite-actions/upload-coredump diff --git a/.github/workflows/singledb-version-upgrade.yml b/.github/workflows/singledb-version-upgrade.yml index a980ffa9fb..14368e1ffe 100644 --- a/.github/workflows/singledb-version-upgrade.yml +++ b/.github/workflows/singledb-version-upgrade.yml @@ -83,3 +83,7 @@ jobs: ~/upgrade/*.diff ~/failed-testscript-outputs ~/${{env.NEW_INSTALL_DIR}}/data/pg_upgrade_output.d/* + + - name: Check and upload coredumps + if: always() && ( steps.upgrade-and-verify.outcome == 'failure' || steps.setup-base-version.outcome == 'failure' ) + uses: ./.github/composite-actions/upload-coredump diff --git a/.github/workflows/upgrade-test.yml b/.github/workflows/upgrade-test.yml index aa3f3dcb58..32e0034ba3 100644 --- a/.github/workflows/upgrade-test.yml +++ b/.github/workflows/upgrade-test.yml @@ -133,4 +133,8 @@ jobs: ~/psql*/data/logfile ~/psql*/data/pg_upgrade_output.d/* ~/failed-testscript-outputs + + - name: Check and upload coredumps + if: always() && (steps.setup-base-version.outcome == 'failure' || steps.upgrade-and-test.outcome == 'failure') + uses: ./.github/composite-actions/upload-coredump diff --git a/contrib/babelfishpg_common/src/numeric.c b/contrib/babelfishpg_common/src/numeric.c index f7cda093a0..d3d7bc9a8a 100644 --- a/contrib/babelfishpg_common/src/numeric.c +++ b/contrib/babelfishpg_common/src/numeric.c @@ -1043,7 +1043,8 @@ tsql_numeric_get_typmod(Numeric num) Datum bigint_sum(PG_FUNCTION_ARGS) { - return bigint_poly_aggr_final(fcinfo, TSQL_SUM); + Assert(0); + return bigint_poly_aggr_final(fcinfo, TSQL_SUM); } /* diff --git a/test/JDBC/jdbc_schedule b/test/JDBC/jdbc_schedule index 2e8fd2a2c7..321a096ae7 100644 --- a/test/JDBC/jdbc_schedule +++ b/test/JDBC/jdbc_schedule @@ -8,7 +8,9 @@ # new line # 6. If you want the framework to not run certain files, use: ignore#!# -all +BABEL-3144-vu-prepare +BABEL-3144-vu-verify +BABEL-3144-vu-cleanup # BABEL-SP_FKEYS test is very slow and causing github action timeout. @@ -185,3 +187,4 @@ ignore#!#load_student ignore#!#BABEL-4279-vu-prepare ignore#!#BABEL-4279-vu-verify ignore#!#BABEL-4279-vu-cleanup +ignore#!#babel_tds_fault_injection From cff5e67371dfdf2364132474c8d49bf76f351500 Mon Sep 17 00:00:00 2001 From: Nirmit Shah Date: Wed, 13 Dec 2023 13:52:21 +0000 Subject: [PATCH 2/2] Fixing temp changes for coredump --- contrib/babelfishpg_common/src/numeric.c | 3 +-- test/JDBC/jdbc_schedule | 5 +---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/contrib/babelfishpg_common/src/numeric.c b/contrib/babelfishpg_common/src/numeric.c index d3d7bc9a8a..f7cda093a0 100644 --- a/contrib/babelfishpg_common/src/numeric.c +++ b/contrib/babelfishpg_common/src/numeric.c @@ -1043,8 +1043,7 @@ tsql_numeric_get_typmod(Numeric num) Datum bigint_sum(PG_FUNCTION_ARGS) { - Assert(0); - return bigint_poly_aggr_final(fcinfo, TSQL_SUM); + return bigint_poly_aggr_final(fcinfo, TSQL_SUM); } /* diff --git a/test/JDBC/jdbc_schedule b/test/JDBC/jdbc_schedule index 321a096ae7..2e8fd2a2c7 100644 --- a/test/JDBC/jdbc_schedule +++ b/test/JDBC/jdbc_schedule @@ -8,9 +8,7 @@ # new line # 6. If you want the framework to not run certain files, use: ignore#!# -BABEL-3144-vu-prepare -BABEL-3144-vu-verify -BABEL-3144-vu-cleanup +all # BABEL-SP_FKEYS test is very slow and causing github action timeout. @@ -187,4 +185,3 @@ ignore#!#load_student ignore#!#BABEL-4279-vu-prepare ignore#!#BABEL-4279-vu-verify ignore#!#BABEL-4279-vu-cleanup -ignore#!#babel_tds_fault_injection