Rel16 cherry pick before fast forward

.git-blame-ignore-revs

@@ -0,0 +1,18 @@
+# As of git 2.23, git-blame supports ignoring specific commits.  This is useful
+# with commits that make bulk formatting changes without truly changing any
+# code.
+#
+# This file lists ignorable pgindent, pgperltidy, and reformat-dat-files
+# related commits only.  Please don't add commits that are not in this
+# category.
+#
+# You can use the ignore list file by running:
+#
+# $ git config blame.ignoreRevsFile .git-blame-ignore-revs
+#
+# Add new entries by adding the output of the following to the top of the file:
+#
+# $ git log --pretty=format:"%H # %cd%n# %s" $PGINDENTGITHASH -1 --date=iso
+
+243e3294c5fee7863e7846b30d8a821b7e4b4242 # 2023-03-22 21:34:35 -0700
+# Run pgindent on each babelfish extensions code (#1357)

.github/composite-actions/build-babelfishpg_unit/action.yml

@@ -0,0 +1,27 @@
+name: 'Build babelfishpg_unit'
+inputs:
+  install_dir:
+    description: 'Engine install directory'
+    required: no
+    default: psql
+  extension_branch: 
+    description: 'Extension Branch name which needs to checkout first'
+    required: no
+    default: 'latest'
+
+runs:
+  using: "composite"
+  steps:
+    - if: always() && (inputs.extension_branch == 'latest')
+      uses: actions/checkout@v2
+
+    - name: Build babelfishpg_unit
+      run: |
+        export PG_CONFIG=~/${{ inputs.install_dir }}/bin/pg_config
+        export PG_SRC=~/work/babelfish_extensions/postgresql_modified_for_babelfish
+        export cmake=$(which cmake)
+        export PATH=~/${{ inputs.install_dir }}/bin:$PATH
+        cd contrib/babelfishpg_unit
+        make clean && make && make install
+      shell: bash
+

.github/composite-actions/build-extensions/action.yml

@@ -36,5 +36,6 @@ runs:
         cd ../babelfishpg_tds
         make -j 4 && make install
         cd ../babelfishpg_tsql
-        make && make install
+        PG_CPPFLAGS='-I/usr/include -DENABLE_TDS_LIB' SHLIB_LINK='-lsybdb -L/usr/lib64' make
+        PG_CPPFLAGS='-I/usr/include -DENABLE_TDS_LIB' SHLIB_LINK='-lsybdb -L/usr/lib64' make install
       shell: bash

.github/composite-actions/build-modified-postgres/action.yml

@@ -59,7 +59,7 @@ runs:
         elif [[ ${{inputs.engine_branch}} != *"__PG_13_"* ]]; then
           cd ../..
           rm -rf pg_hint_plan
-          git clone --depth 1 --branch PG15 https://github.com/ossc-db/pg_hint_plan.git
+          git clone --depth 1 --branch REL15_1_5_1 https://github.com/ossc-db/pg_hint_plan.git
           cd pg_hint_plan
           export PATH=$HOME/${{ inputs.install_dir }}/bin:$PATH
           make

.github/composite-actions/dump-restore-util/action.yml

@@ -18,6 +18,17 @@ inputs:
   migration_mode:
     description: "Database migration mode for Babelfish"
     required: true
+  logical_database:
+    description: "Logical Babelfish database to dump and restore"
+    required: false
+    default: 'null'
+  dump_data_as:
+    description: "Dump table data using COPY or INSERT"
+    required: false
+    default: 'copy'
+  dump_format:
+    description: "Dump format (plain/custom/tar/directory)"
+    required: true
 
 runs:
   using: "composite"
@@ -39,54 +50,44 @@ runs:
         cd ~
         mkdir -p upgrade
         cd upgrade
-        echo 'Runinng pg_dumpall and pg_dump'
-        ~/${{ inputs.pg_new_dir }}/bin/pg_dumpall --username jdbc_user --globals-only --quote-all-identifiers --verbose -f pg_dump_globals.sql 2>>error.log
-        ~/${{ inputs.pg_new_dir }}/bin/pg_dump --username jdbc_user --column-inserts --quote-all-identifiers --verbose --file="pg_dump.sql" --dbname=jdbc_testdb 2>>error.log
+        export PGPASSWORD=12345678
+
+        if [[ '${{ inputs.dump_data_as }}' == 'inserts' ]];then
+          export DUMP_OPTS='--column-inserts'
+        else
+          export DUMP_OPTS=''
+        fi
+        export DUMP_OPTS="$DUMP_OPTS --format=${{ inputs.dump_format }}"
+
+        if [[ '${{ inputs.logical_database }}' == 'null' ]];then
+          echo 'Starting to dump whole Babelfish physical database'
+          ~/${{ inputs.pg_new_dir }}/bin/pg_dumpall -h localhost --database jdbc_testdb --username jdbc_user --roles-only --quote-all-identifiers --verbose --no-role-passwords -f pg_dump_globals.sql 2>>error.log
+          ~/${{ inputs.pg_new_dir }}/bin/pg_dump -h localhost --username jdbc_user $DUMP_OPTS --quote-all-identifiers --verbose --file="pg_dump.archive" --dbname=jdbc_testdb 2>>error.log
+        else
+          echo "Starting to dump Babelfish logical database ${{ inputs.logical_database }}"
+          ~/${{ inputs.pg_new_dir }}/bin/pg_dumpall -h localhost --database jdbc_testdb --username jdbc_user --roles-only --quote-all-identifiers --verbose --no-role-passwords --bbf-database-name='${{ inputs.logical_database }}' -f pg_dump_globals.sql 2>>error.log
+          ~/${{ inputs.pg_new_dir }}/bin/pg_dump -h localhost --username jdbc_user $DUMP_OPTS --quote-all-identifiers --verbose --bbf-database-name='${{ inputs.logical_database }}' --file="pg_dump.archive" --dbname=jdbc_testdb 2>>error.log
+        fi
+
+        # Stop old server and start the new.
         ~/${{ inputs.pg_old_dir }}/bin/pg_ctl -D ~/${{ inputs.pg_old_dir }}/data stop
-        echo 'Done dumping the database, proceeding to restore dumped SQL script on new server.'
-        echo 'Restoring from pg_dumpall'
         ~/${{ inputs.pg_new_dir }}/bin/pg_ctl -D ~/${{ inputs.pg_new_dir }}/data -l ~/${{ inputs.pg_new_dir }}/data/logfile start
-        sudo ~/${{ inputs.pg_new_dir }}/bin/psql -d postgres -U runner -f pg_dump_globals.sql 2>>error.log
-        sudo ~/${{ inputs.pg_new_dir }}/bin/psql -d postgres -U runner -c "CREATE DATABASE jdbc_testdb OWNER jdbc_user;"
-        echo 'Restoring from pg_dump'
-        sudo ~/${{ inputs.pg_new_dir }}/bin/psql -d jdbc_testdb -U jdbc_user -f pg_dump.sql 2>>error.log
-        sudo ~/${{ inputs.pg_new_dir }}/bin/psql -d jdbc_testdb -U jdbc_user -c "GRANT ALL ON SCHEMA sys to jdbc_user;"
-        sudo ~/${{ inputs.pg_new_dir }}/bin/psql -d jdbc_testdb -U jdbc_user -c "GRANT CREATE, CONNECT, TEMPORARY ON DATABASE jdbc_testdb TO sysadmin WITH GRANT OPTION;"
-        sudo ~/${{ inputs.pg_new_dir }}/bin/psql -d jdbc_testdb -U jdbc_user -c "ALTER USER jdbc_user CREATEDB;"
-        sudo ~/${{ inputs.pg_new_dir }}/bin/psql -d jdbc_testdb -U jdbc_user -c "ALTER SYSTEM SET babelfishpg_tsql.database_name = 'jdbc_testdb';"
-        sudo ~/${{ inputs.pg_new_dir }}/bin/psql -d jdbc_testdb -U jdbc_user -c "ALTER SYSTEM SET babelfishpg_tsql.migration_mode = '${{inputs.migration_mode}}';"
-        sudo ~/${{ inputs.pg_new_dir }}/bin/psql -d jdbc_testdb -U jdbc_user -c "SELECT pg_reload_conf();"
         cd ~/work/babelfish_extensions/babelfish_extensions/
-        export PATH=/opt/mssql-tools/bin:$PATH
-        sqlcmd -S localhost -U jdbc_user -P 12345678 -Q "SELECT @@version GO"
-      shell: bash
+        echo 'Database dump complete.'
 
-    # Temporary step
-    - name: Temporarily disable failing tests
-      if: always() && steps.run-pg_dump-restore.outcome == 'success' && inputs.is_final_ver == 'true'
-      env:
-        migr_mode: ${{ inputs.migration_mode }}
-      run: |
-        if [[ "$migr_mode" == "multi-db" ]];then
-          base_dir=${{ matrix.upgrade-path.path[0] }}
-          if [[ "$base_dir" == *"latest"* ]]; then
-            base_dir="latest"
-          fi
+        # Create and initialise Babelfish extensions in the new server to perform restore.
+        sudo ~/${{ inputs.pg_new_dir }}/bin/psql -d postgres -U runner -v user="jdbc_user" -v db="jdbc_testdb" -v migration_mode=${{inputs.migration_mode}} -f .github/scripts/create_extension.sql
+        echo 'Restoring from pg_dumpall'
+        sudo PGPASSWORD=12345678 ~/${{ inputs.pg_new_dir }}/bin/psql -h localhost -d jdbc_testdb -U jdbc_user --single-transaction -f ~/upgrade/pg_dump_globals.sql 2>> ~/upgrade/error.log
+        echo 'Restoring from pg_dump'
+        if [[ '${{ inputs.dump_format }}' == 'plain' ]];then
+          sudo PGPASSWORD=12345678 ~/${{ inputs.pg_new_dir }}/bin/psql -h localhost -d jdbc_testdb -U jdbc_user --single-transaction -f ~/upgrade/pg_dump.archive 2>> ~/upgrade/error.log
         else
-          base_dir="singledb"
+          ~/${{ inputs.pg_new_dir }}/bin/pg_restore -h localhost -d jdbc_testdb -U jdbc_user --single-transaction ~/upgrade/pg_dump.archive 2>> ~/upgrade/error.log
         fi
 
-        # Temporarily disable certain tests until fixed
-        sed -i "/BABEL-3513/d" test/JDBC/upgrade/$base_dir/schedule
-        sed -i "/BABEL_OBJECT_ID/d" test/JDBC/upgrade/$base_dir/schedule
-        sed -i "/ISC-Views/d" test/JDBC/upgrade/$base_dir/schedule
-        sed -i "/TestNotNull/d" test/JDBC/upgrade/$base_dir/schedule
-        sed -i "/TestSQLVariant/d" test/JDBC/upgrade/$base_dir/schedule
-        sed -i "/babel_datatype_sqlvariant/d" test/JDBC/upgrade/$base_dir/schedule
-        sed -i "/schema_resolution_proc/d" test/JDBC/upgrade/$base_dir/schedule
-        sed -i "/sp_tablecollations/d" test/JDBC/upgrade/$base_dir/schedule
-        sed -i "/sys-sql_modules/d" test/JDBC/upgrade/$base_dir/schedule
-        sed -i "/sys-system_sql_modules/d" test/JDBC/upgrade/$base_dir/schedule
+        export PATH=/opt/mssql-tools/bin:$PATH
+        sqlcmd -S localhost -U jdbc_user -P 12345678 -Q "SELECT @@version GO"
       shell: bash
 
     - name: Run Verify Tests
@@ -95,4 +96,5 @@ runs:
       with:
         is_final_ver: ${{ inputs.is_final_ver }}
         pg_new_dir: ${{ inputs.pg_new_dir }}
-        migration_mode: ${{ inputs.migration_mode }}
+        migration_mode: ${{ inputs.migration_mode }}
+        logical_database: ${{ inputs.logical_database }}

.github/composite-actions/install-extensions/action.yml

@@ -9,6 +9,10 @@ inputs:
     description: 'Database migration mode'
     required: no
     default: "single-db"
+  parallel_query_mode:
+    description: 'Postgres Parallel Query Mode'
+    required: no
+    default: false
 
 runs:
   using: "composite"
@@ -21,11 +25,19 @@ runs:
         ~/${{inputs.install_dir}}/bin/pg_ctl -D ~/${{inputs.install_dir}}/data/ -l logfile start
         cd ${{inputs.install_dir}}/data
         sudo sed -i "s/#listen_addresses = 'localhost'/listen_addresses = '*'/g" postgresql.conf
-        sudo sed -i "s/#shared_preload_libraries = ''/shared_preload_libraries = 'babelfishpg_tds'/g" postgresql.conf
+        sudo sed -i "s/#shared_preload_libraries = ''/shared_preload_libraries = 'babelfishpg_tds, pg_stat_statements'/g" postgresql.conf
         ipaddress=$(ifconfig eth0 | grep 'inet ' | cut -d: -f2 | awk '{ print $2}')
-        sudo echo "host    all             all             $ipaddress/32            trust" >> pg_hba.conf
+        # Allow only runner to have trust authentication, all other users must provide a password
+        {
+          sudo echo "local   all             runner                                   trust"
+          sudo echo "local   all             all                                      md5"
+          sudo echo "host    all             runner          127.0.0.1/32             trust"
+          sudo echo "host    all             runner          $ipaddress/32            trust"
+          sudo echo "host    all             all             0.0.0.0/0                md5"
+          sudo echo "host    all             all             ::/0                     md5"
+        } > pg_hba.conf
         ~/${{inputs.install_dir}}/bin/pg_ctl -D ~/${{inputs.install_dir}}/data/ -l logfile restart
         cd ~/work/babelfish_extensions/babelfish_extensions/
-        sudo ~/${{inputs.install_dir}}/bin/psql -d postgres -U runner -v user="jdbc_user" -v db="jdbc_testdb" -v migration_mode=${{inputs.migration_mode}} -f .github/scripts/create_extension.sql
+        sudo ~/${{inputs.install_dir}}/bin/psql -d postgres -U runner -v user="jdbc_user" -v db="jdbc_testdb" -v migration_mode=${{inputs.migration_mode}} -v parallel_query_mode=${{inputs.parallel_query_mode}} -f .github/scripts/create_extension.sql
         sqlcmd -S localhost -U "jdbc_user" -P 12345678 -Q "SELECT @@version GO"
       shell: bash

.github/composite-actions/minor-version-upgrade-util/action.yml

@@ -77,6 +77,4 @@ runs:
         done
         export scheduleFile=upgrade/$base_dir/schedule
         mvn test
-      shell: bash
-
-
+      shell: bash      

.github/composite-actions/run-verify-tests/action.yml

@@ -9,6 +9,10 @@ inputs:
   migration_mode:
     description: "Database migration mode for Babelfish"
     required: true
+  logical_database:
+    description: "Logical Babelfish database to dump and restore"
+    required: false
+    default: 'null'
 
 runs:
   using: "composite"
@@ -39,6 +43,12 @@ runs:
           base_dir="singledb"
           export inputFilesPath=upgrade/singledb/verification_cleanup
         fi
+
+        if [[ '${{ inputs.logical_database }}' != 'null' ]];then
+          base_dir="${{ inputs.logical_database }}"
+          export inputFilesPath=upgrade/$base_dir/verification_cleanup
+        fi
+
         mvn test
         export inputFilesPath=input
 

.github/composite-actions/setup-base-version/action.yml

@@ -12,6 +12,10 @@ inputs:
   migration_mode:
     description: "Database migration mode for Babelfish"
     required: true
+  logical_database:
+    description: "Logical Babelfish database to dump and restore"
+    required: false
+    default: 'null'
 
 runs:
   using: "composite"
@@ -65,9 +69,17 @@ runs:
         ~/${{ inputs.install_dir }}/bin/pg_ctl -D ~/${{ inputs.install_dir }}/data/ -l logfile start
         cd ${{ inputs.install_dir }}/data
         sudo sed -i "s/#listen_addresses = 'localhost'/listen_addresses = '*'/g" postgresql.conf
-        sudo sed -i "s/#shared_preload_libraries = ''/shared_preload_libraries = 'babelfishpg_tds'/g" postgresql.conf
+        sudo sed -i "s/#shared_preload_libraries = ''/shared_preload_libraries = 'babelfishpg_tds, pg_stat_statements'/g" postgresql.conf
         ipaddress=$(ifconfig eth0 | grep 'inet ' | cut -d: -f2 | awk '{ print $2}')
-        sudo echo "host    all             all             $ipaddress/32            trust" >> pg_hba.conf
+        # Allow only runner to have trust authentication, all other users must provide a password
+        {
+          sudo echo "local   all             runner                                   trust"
+          sudo echo "local   all             all                                      md5"
+          sudo echo "host    all             runner          127.0.0.1/32             trust"
+          sudo echo "host    all             runner          $ipaddress/32            trust"
+          sudo echo "host    all             all             0.0.0.0/0                md5"
+          sudo echo "host    all             all             ::/0                     md5"
+        } > pg_hba.conf
         ~/${{ inputs.install_dir }}/bin/pg_ctl -D ~/${{ inputs.install_dir }}/data/ -l logfile restart
         sudo ~/${{ inputs.install_dir }}/bin/psql -d postgres -U runner -c "CREATE USER jdbc_user WITH SUPERUSER CREATEDB CREATEROLE PASSWORD '12345678' INHERIT;"
         sudo ~/${{ inputs.install_dir }}/bin/psql -d postgres -U runner -c "DROP DATABASE IF EXISTS jdbc_testdb;"
@@ -111,6 +123,11 @@ runs:
         else
           base_dir="singledb"
         fi
+
+        if [[ '${{ inputs.logical_database }}' != 'null' ]];then
+          base_dir="${{ inputs.logical_database }}"
+        fi
+
         export inputFilesPath=upgrade/$base_dir/preparation
         mvn test
         for filename in $(grep -v "^ignore.*\|^#.*\|^cmd.*\|^all.*\|^$" upgrade/$base_dir/schedule); do
@@ -130,7 +147,7 @@ runs:
 
     - name: Install Python
       id: install-python
-      if: ${{ matrix.upgrade-path.path[0] == 'source_latest' && steps.jdbc-upgrade-tests.outcome == 'success' }}
+      if: ${{ matrix.upgrade-path.path[0] == 'source_latest' && steps.jdbc-upgrade-tests.outcome == 'success' && inputs.logical_database == 'null' }}
       uses: actions/setup-python@v2
       with:
         python-version: 3.7

.github/composite-actions/setup-dump-restore-ca/action.yml

@@ -23,10 +23,10 @@ runs:
             echo "ERROR: Dump restore path length less than 2" 1>&2
             exit 1
         fi
-        
+
         # Initial installed version is the base version
         previous_installed_version=${{ inputs.base_version }}
-        
+
         # For every next dump+restore, below 'For' loop will add a step in dump-restore composite action which will dump the current version and restore it into next version
         for (( i=1 ; i<$LEN ; i++ )); 
         do
@@ -43,7 +43,19 @@ runs:
           uses_file=./.github/composite-actions/dump-restore-util
           temp="&& steps.dump-restore-version-$(($i-1)).outcome == 'success'"; [[ i -eq 1 ]] && temp=""
 
-          printf "    - name: Dump and Restore to version $dump_restore_version\n      id: dump-restore-version-$i\n      if: always() $temp\n      uses: ${uses_file}\n      with: \n        engine_branch: ${engine_branch}\n        extension_branch: ${extension_branch}\n        is_final_ver: ${is_final_ver}\n        pg_old_dir: ${pg_old_dir}\n        pg_new_dir: ${pg_new_dir}\n        migration_mode: 'multi-db'\n\n"   >> $dump_restore_version_dir_path/action.yml
+          logical_database_var=".\"dump-restore-version\"[${{ matrix.upgrade-path.id }}][$i].\"logical-database\""
+          logical_database=$(yq $logical_database_var ${{ github.workspace }}/.github/configuration/dump-restore-test-configuration.yml)
+          dump_data_as_var=".\"dump-restore-version\"[${{ matrix.upgrade-path.id }}][$i].\"dump-data-as\""
+          dump_data_as=$(yq $dump_data_as_var ${{ github.workspace }}/.github/configuration/dump-restore-test-configuration.yml)
+          dump_format_var=".\"dump-restore-version\"[${{ matrix.upgrade-path.id }}][$i].\"dump-format\""
+          dump_format=$(yq $dump_format_var ${{ github.workspace }}/.github/configuration/dump-restore-test-configuration.yml)
+
+          if [[ $logical_database == 'null' ]]
+          then
+            printf "    - name: Dump and Restore to version $dump_restore_version\n      id: dump-restore-version-$i\n      if: always() $temp\n      uses: ${uses_file}\n      with: \n        engine_branch: ${engine_branch}\n        extension_branch: ${extension_branch}\n        is_final_ver: ${is_final_ver}\n        pg_old_dir: ${pg_old_dir}\n        pg_new_dir: ${pg_new_dir}\n        migration_mode: 'multi-db'\n        dump_data_as: ${dump_data_as}\n        dump_format: ${dump_format}\n\n"   >> $dump_restore_version_dir_path/action.yml
+          else
+            printf "    - name: Dump and Restore to version $dump_restore_version\n      id: dump-restore-version-$i\n      if: always() $temp\n      uses: ${uses_file}\n      with: \n        engine_branch: ${engine_branch}\n        extension_branch: ${extension_branch}\n        is_final_ver: ${is_final_ver}\n        pg_old_dir: ${pg_old_dir}\n        pg_new_dir: ${pg_new_dir}\n        migration_mode: 'multi-db'\n        logical_database: ${logical_database}\n        dump_data_as: ${dump_data_as}\n        dump_format: ${dump_format}\n\n"   >> $dump_restore_version_dir_path/action.yml
+          fi
 
           previous_installed_version=$dump_restore_version
         done

.github/composite-actions/setup-new-version/action.yml

@@ -52,7 +52,15 @@ runs:
         ~/${{ inputs.pg_new_dir }}/bin/initdb -D ~/${{ inputs.pg_new_dir }}/data
         cd ~/${{ inputs.pg_new_dir }}/data
         sudo sed -i "s/#listen_addresses = 'localhost'/listen_addresses = '*'/g" postgresql.conf
-        sudo sed -i "s/#shared_preload_libraries = ''/shared_preload_libraries = 'babelfishpg_tds'/g" postgresql.conf
+        sudo sed -i "s/#shared_preload_libraries = ''/shared_preload_libraries = 'babelfishpg_tds, pg_stat_statements'/g" postgresql.conf
         ipaddress=$(ifconfig eth0 | grep 'inet ' | cut -d: -f2 | awk '{ print $2}')
-        sudo echo "host    all             all             $ipaddress/32            trust" >> pg_hba.conf
+        # Allow only runner to have trust authentication, all other users must provide a password
+        {
+          sudo echo "local   all             runner                                   trust"
+          sudo echo "local   all             all                                      md5"
+          sudo echo "host    all             runner          127.0.0.1/32             trust"
+          sudo echo "host    all             runner          $ipaddress/32            trust"
+          sudo echo "host    all             all             0.0.0.0/0                md5"
+          sudo echo "host    all             all             ::/0                     md5"
+        } > pg_hba.conf
       shell: bash