Merge 4_X changes back to 5_X (babelfish-for-postgresql#3149)

This is a cherry-pick PR which merges newer 4_X engine commits into 5_X. Engine PR: babelfish-for-postgresql/postgresql_modified_for_babelfish#479 Signed-off-by Roshan Kanwar <rskanwar.amazon.com>
amazon-aurora · Nov 25, 2024 · 98c2dfc · 98c2dfc
1 parent 3154e13
commit 98c2dfc
Show file tree

Hide file tree

Showing 196 changed files with 18,141 additions and 1,157 deletions.
diff --git a/.github/composite-actions/compile-antlr/action.yml b/.github/composite-actions/compile-antlr/action.yml
@@ -1,6 +1,10 @@
 name: 'Compile ANTLR'
 
 inputs:
+  version:
+    description: 'version of ANTLR to install'
+    required: no
+    default: 4.13.2
   install_dir:
     description: 'Engine install directory'
     required: no
@@ -12,9 +16,12 @@ runs:
 
     - name: Compile ANTLR
       run: |
-        ANTLR_VERSION=4.9.3
+        ANTLR_VERSION=${{inputs.version}}
         
         cd contrib/babelfishpg_tsql/antlr/thirdparty/antlr/
+        if [ ! -f antlr-$ANTLR_VERSION-complete.jar ]; then
+          wget "http://www.antlr.org/download/antlr-$ANTLR_VERSION-complete.jar"
+        fi
         sudo cp "antlr-$ANTLR_VERSION-complete.jar" /usr/local/lib
         cd ..
         wget "http://www.antlr.org/download/antlr4-cpp-runtime-$ANTLR_VERSION-source.zip"

diff --git a/.github/composite-actions/dump-restore-util/action.yml b/.github/composite-actions/dump-restore-util/action.yml
@@ -33,6 +33,10 @@ inputs:
     description: "Dump type full or combination of schema-only and data-only"
     required: false
     default: 'full'
+  antlr_version:
+    description: "ANTLR version to use (based on Extension Branch)"
+    required: false
+    default: 4.13.2
 
 runs:
   using: "composite"
@@ -45,6 +49,7 @@ runs:
         engine_branch: ${{ inputs.engine_branch }}
         extension_branch: ${{ inputs.extension_branch }}
         pg_new_dir: ${{ inputs.pg_new_dir }}
+        antlr_version: ${{ inputs.antlr_version }}
 
     - name: Build latest dump/restore utilities
       id: build-dump-utils

diff --git a/.github/composite-actions/major-version-upgrade-util/action.yml b/.github/composite-actions/major-version-upgrade-util/action.yml
@@ -22,6 +22,10 @@ inputs:
     description: "Server collation name"
     required: false
     default: "default"
+  antlr_version:
+    description: "ANTLR version to use (based on Extension Branch)"
+    required: false
+    default: 4.13.2
 
 runs:
   using: "composite"
@@ -34,6 +38,7 @@ runs:
         engine_branch: ${{ inputs.engine_branch }}
         extension_branch: ${{ inputs.extension_branch }}
         pg_new_dir: ${{ inputs.pg_new_dir }}
+        antlr_version: ${{ inputs.antlr_version }}
 
     - name: Run pg_upgrade
       id: run-pg_upgrade

diff --git a/.github/composite-actions/minor-version-upgrade-util/action.yml b/.github/composite-actions/minor-version-upgrade-util/action.yml
@@ -16,18 +16,33 @@ inputs:
     description: "Server collation name"
     required: false
     default: "default"
+  antlr_version:
+    description: "ANTLR version to use (based on Extension Branch)"
+    required: false
+    default: 4.13.2
 
 runs:
   using: "composite"
   steps:    
     - name: Build and run tests for Postgres engine using ${{ inputs.engine_branch }}
+      id: build-modified-postgres-new
+      if: always()
       uses: ./.github/composite-actions/build-modified-postgres
       with:
         engine_branch: ${{ inputs.engine_branch }}
         install_dir: ${{ inputs.install_dir }}
 
+    - name: Compile new ANTLR
+      id: compile-new-antlr
+      if: always() && steps.build-modified-postgres-new.outcome == 'success'
+      uses: ./.github/composite-actions/compile-antlr
+      with:
+        version: ${{ inputs.antlr_version }}
+        install_dir: ${{inputs.install_dir}}
+
     - name: Set env variables and build extensions using ${{ inputs.extension_branch }}
       id: build-extensions-newer
+      if: always() && steps.compile-new-antlr.outcome == 'success'
       uses: ./.github/composite-actions/build-extensions
       with: 
         install_dir: ${{ inputs.install_dir }}

diff --git a/.github/composite-actions/setup-base-version/action.yml b/.github/composite-actions/setup-base-version/action.yml
@@ -20,6 +20,10 @@ inputs:
     description: "Whether it is version upgrade or dump/restore"
     required: false
     default: 'false'
+  antlr_version:
+    description: "ANTLR version to use (based on Extension Branch)"
+    required: false
+    default: 4.13.2
 
 runs:
   using: "composite"
@@ -42,6 +46,7 @@ runs:
       if: always() && steps.build-modified-postgres.outcome == 'success'
       uses: ./.github/composite-actions/compile-antlr
       with:
+        version: ${{ inputs.antlr_version }}
         install_dir: ${{ inputs.install_dir }}
 
     - name: Set env variables and build extensions

diff --git a/.github/composite-actions/setup-dump-restore-ca/action.yml b/.github/composite-actions/setup-dump-restore-ca/action.yml
@@ -40,6 +40,12 @@ runs:
           is_final_ver=false; [[ i -eq $LEN-1 ]] && is_final_ver=true
           pg_old_dir=$(echo psql$(awk -F. '{print $1}' <<< $previous_installed_version))
           pg_new_dir=$(echo psql$(awk -F. '{print $1}' <<< $dump_restore_version))
+          if [[ "$dump_restore_version" == 'target.latest' || "$dump_restore_version" -ge 16.7 ]]
+          then
+            antlr_version=4.13.2
+          else
+            antlr_version=4.9.3
+          fi
 
           if [[ "$previous_installed_version" == "$dump_restore_version" ]]; then
             pg_new_dir=$(echo $pg_new_dir.$i)
@@ -56,7 +62,7 @@ runs:
           type_var=".\"dump-restore-version\"[${{ matrix.upgrade-path.id }}][$i].\"type\""
           type=$(yq $type_var ${{ github.workspace }}/.github/configuration/dump-restore-test-configuration.yml)
 
-          printf "    - name: Dump and Restore to version $dump_restore_version\n      id: dump-restore-version-$i\n      if: always() $temp\n      uses: ${uses_file}\n      with: \n        engine_branch: ${engine_branch}\n        extension_branch: ${extension_branch}\n        is_final_ver: ${is_final_ver}\n        pg_old_dir: ${pg_old_dir}\n        pg_new_dir: ${pg_new_dir}\n        migration_mode: 'multi-db'\n        database_level: ${database_level}\n        dump_data_as: ${dump_data_as}\n        dump_format: ${dump_format}\n        type: ${type}\n\n"   >> $dump_restore_version_dir_path/action.yml
+          printf "    - name: Dump and Restore to version $dump_restore_version\n      id: dump-restore-version-$i\n      if: always() $temp\n      uses: ${uses_file}\n      with: \n        engine_branch: ${engine_branch}\n        extension_branch: ${extension_branch}\n        is_final_ver: ${is_final_ver}\n        pg_old_dir: ${pg_old_dir}\n        pg_new_dir: ${pg_new_dir}\n        migration_mode: 'multi-db'\n        database_level: ${database_level}\n        dump_data_as: ${dump_data_as}\n        dump_format: ${dump_format}\n        type: ${type}\n        antlr_version: ${antlr_version}\n\n"   >> $dump_restore_version_dir_path/action.yml
 
           previous_installed_version=$dump_restore_version
         done

diff --git a/.github/composite-actions/setup-new-version/action.yml b/.github/composite-actions/setup-new-version/action.yml
@@ -9,6 +9,10 @@ inputs:
   pg_new_dir:
     description: "Install new version in this directory"
     required: true
+  antlr_version:
+    description: "ANTLR version to use (based on Extension Branch)"
+    required: false
+    default: 4.13.2
 
 runs:
   using: "composite"
@@ -21,15 +25,17 @@ runs:
         engine_branch: ${{ inputs.engine_branch }}
         install_dir: ${{ inputs.pg_new_dir }}
 
-    - name: Copy ANTLR
-      id: copy-antlr
+    - name: Compile new ANTLR
+      id: compile-new-antlr
       if: always() && steps.build-modified-postgres-new.outcome == 'success'
-      run: cp "/usr/local/lib/libantlr4-runtime.so.4.9.3" ~/${{ inputs.pg_new_dir }}/lib/
-      shell: bash
+      uses: ./.github/composite-actions/compile-antlr
+      with:
+        version: ${{ inputs.antlr_version }}
+        install_dir: ${{inputs.pg_new_dir}}
 
     - name: Build Extensions
       id: build-extensions-new
-      if: always() && steps.copy-antlr.outcome == 'success'
+      if: always() && steps.compile-new-antlr.outcome == 'success'
       uses: ./.github/composite-actions/build-extensions
       with:
         install_dir: ${{ inputs.pg_new_dir }}

diff --git a/.github/composite-actions/setup-upgrade-version-ca/action.yml b/.github/composite-actions/setup-upgrade-version-ca/action.yml
@@ -45,6 +45,12 @@ runs:
           is_final_ver=false; [[ i -eq $LEN-1 ]] && is_final_ver=true
           pg_old_dir=$(echo psql$(awk -F. '{print $1}' <<< $previous_installed_version))
           pg_new_dir=$(echo psql$(awk -F. '{print $1}' <<< $upgrade_version))
+          if [[ "$upgrade_version" == 'target.latest' || "$upgrade_version" -ge 16.7 ]]
+          then
+            antlr_version=4.13.2
+          else
+            antlr_version=4.9.3
+          fi
           
           upgrade_type_var=".\"upgrade-version\"[${{ matrix.upgrade-path.id }}]."upgrade-path"[$i].\"upgrade-type\"" 
           upgrade_type=$(yq $upgrade_type_var ${{ github.workspace }}/.github/configuration/upgrade-test-configuration.yml)
@@ -60,9 +66,9 @@ runs:
           # Appending upgrade step in upgrade-version composite action based on upgrade type
           if [[ $upgrade_type == 'major' ]]
           then
-            printf "    - name: Upgrade Version to $upgrade_version\n      id: upgrade-version-$i\n      if: always() $temp\n      uses: ${uses_file}\n      with: \n        engine_branch: ${engine_branch}\n        extension_branch: ${extension_branch}\n        is_final_ver: ${is_final_ver}\n        pg_old_dir: ${pg_old_dir}\n        pg_new_dir: ${pg_new_dir}\n        migration_mode: 'multi-db'\n        server_collation_name: ${{ inputs.server_collation_name }}\n\n"   >> $upgrade_version_dir_path/action.yml
+            printf "    - name: Upgrade Version to $upgrade_version\n      id: upgrade-version-$i\n      if: always() $temp\n      uses: ${uses_file}\n      with: \n        engine_branch: ${engine_branch}\n        extension_branch: ${extension_branch}\n        is_final_ver: ${is_final_ver}\n        pg_old_dir: ${pg_old_dir}\n        pg_new_dir: ${pg_new_dir}\n        migration_mode: 'multi-db'\n        server_collation_name: ${{ inputs.server_collation_name }}\n        antlr_version: ${antlr_version}\n\n"   >> $upgrade_version_dir_path/action.yml
           else
-            printf "    - name: Upgrade Version to $upgrade_version\n      id: upgrade-version-$i\n      if: always() $temp\n      uses: ${uses_file}\n      with: \n        engine_branch: ${engine_branch}\n        extension_branch: ${extension_branch}\n        is_final_ver: ${is_final_ver}\n        install_dir: ${pg_new_dir}\n        server_collation_name: ${{ inputs.server_collation_name }}\n\n"   >> $upgrade_version_dir_path/action.yml
+            printf "    - name: Upgrade Version to $upgrade_version\n      id: upgrade-version-$i\n      if: always() $temp\n      uses: ${uses_file}\n      with: \n        engine_branch: ${engine_branch}\n        extension_branch: ${extension_branch}\n        is_final_ver: ${is_final_ver}\n        install_dir: ${pg_new_dir}\n        server_collation_name: ${{ inputs.server_collation_name }}\n        antlr_version: ${antlr_version}\n\n"   >> $upgrade_version_dir_path/action.yml
           fi
 
           previous_installed_version=$upgrade_version

diff --git a/.github/configuration/upgrade-test-configuration.yml b/.github/configuration/upgrade-test-configuration.yml
@@ -101,7 +101,7 @@ upgrade-version: [{
 {
   upgrade-path: [
     {
-      version: '14.15',
+      version: '14.16',
       upgrade-type: null
     },
     { 
@@ -125,7 +125,7 @@ upgrade-version: [{
 {
   upgrade-path: [
     {
-      version: 15.9,
+      version: '15.10',
       upgrade-type: null
     },
     { 
@@ -161,7 +161,7 @@ upgrade-version: [{
 {
   upgrade-path: [
     {
-      version: 16.5,
+      version: 16.6,
       upgrade-type: null
     },
     { 
@@ -177,11 +177,11 @@ upgrade-version: [{
       upgrade-type: null
     },
     {
-      version: '14.15',
+      version: '14.16',
       upgrade-type: major
     },
     {
-      version: '15.10',
+      version: '15.11',
       upgrade-type: major
     },
     { 
@@ -207,7 +207,7 @@ upgrade-version: [{
 {
   upgrade-path: [
     {
-      version: '15.10',
+      version: '15.11',
       upgrade-type: null
     },
     { 
@@ -220,7 +220,7 @@ upgrade-version: [{
 {
   upgrade-path: [
     {
-      version: '15.10',
+      version: '15.11',
       upgrade-type: null
     },
     { 
@@ -237,7 +237,7 @@ upgrade-version: [{
       upgrade-type: null
     },
     {
-      version: '15.10',
+      version: '15.11',
       upgrade-type: minor
     },
     { 

diff --git a/.github/template/version-branch-template.yml b/.github/template/version-branch-template.yml
@@ -46,10 +46,10 @@
 '14.13':
   engine_branch: BABEL_2_10_STABLE__PG_14_13
   extension_branch: BABEL_2_10_STABLE
-'14.14':
-  engine_branch: BABEL_2_11_STABLE__PG_14_14
-  extension_branch: BABEL_2_11_STABLE
 '14.15':
+  engine_branch: BABEL_2_11_STABLE__PG_14_15
+  extension_branch: BABEL_2_11_STABLE
+'14.16':
   engine_branch: BABEL_2_X_DEV__PG_14_X
   extension_branch: BABEL_2_X_DEV
 '15.2':
@@ -73,10 +73,10 @@
 '15.8':
   engine_branch: BABEL_3_7_STABLE__PG_15_8
   extension_branch: BABEL_3_7_STABLE
-'15.9':
-  engine_branch: BABEL_3_8_STABLE__PG_15_9
-  extension_branch: BABEL_3_8_STABLE
 '15.10':
+  engine_branch: BABEL_3_8_STABLE__PG_15_10
+  extension_branch: BABEL_3_8_STABLE
+'15.11':
   engine_branch: BABEL_3_X_DEV__PG_15_X
   extension_branch: BABEL_3_X_DEV
 '16.1':
@@ -91,8 +91,8 @@
 '16.4':
   engine_branch: BABEL_4_3_STABLE__PG_16_4
   extension_branch: BABEL_4_3_STABLE
-'16.5':
-  engine_branch: BABEL_4_4_STABLE__PG_16_5
+'16.6':
+  engine_branch: BABEL_4_4_STABLE__PG_16_6
   extension_branch: BABEL_4_4_STABLE
 'source.latest':
   engine_branch: latest

diff --git a/.github/workflows/isolation-tests.yml b/.github/workflows/isolation-tests.yml
@@ -53,12 +53,12 @@ jobs:
           curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list
           cd ~/work/babelfish_extensions/babelfish_extensions/test/python
           sudo ACCEPT_EULA=Y apt-get install -y msodbcsql17 python3-dev
-          pip3 install pyodbc pymssql pytest pytest-xdist antlr4-python3-runtime==4.9.3
+          pip3 install pyodbc pymssql pytest pytest-xdist antlr4-python3-runtime==4.13.2
       
       - name: Generate .spec file parser
         run: |
           cd ~/work/babelfish_extensions/babelfish_extensions/test/python/isolationtest/
-          java -Xmx500M -cp /usr/local/lib/antlr-4.9.3-complete.jar org.antlr.v4.Tool -Dlanguage=Python3 ./parser/*.g4 -visitor -no-listener
+          java -Xmx500M -cp /usr/local/lib/antlr-4.13.2-complete.jar org.antlr.v4.Tool -Dlanguage=Python3 ./parser/*.g4 -visitor -no-listener
 
       - name: Run Isolation tests
         id: run-isolation-tests

diff --git a/.github/workflows/major-version-upgrade.yml b/.github/workflows/major-version-upgrade.yml
@@ -38,6 +38,7 @@ jobs:
         if: always() && steps.build-modified-postgres-old.outcome == 'success'
         uses: ./.github/composite-actions/compile-antlr
         with:
+          version: 4.9.3
           install_dir: ${{env.OLD_INSTALL_DIR}}
 
       - uses: actions/checkout@v2
@@ -136,13 +137,17 @@ jobs:
         with:
           install_dir: ${{env.NEW_INSTALL_DIR}}
           release_mode: 'yes'
-
-      - name: Copy ANTLR
-        run: cp "/usr/local/lib/libantlr4-runtime.so.4.9.3" ~/${{env.NEW_INSTALL_DIR}}/lib/
+
+      - name: Compile new ANTLR
+        id: compile-new-antlr
+        if: always() && steps.build-modified-postgres-new.outcome == 'success'
+        uses: ./.github/composite-actions/compile-antlr
+        with:
+          install_dir: ${{env.NEW_INSTALL_DIR}}
 
       - name: Build Extensions using latest version
         id: build-extensions-new
-        if: always() && steps.build-modified-postgres-new.outcome == 'success'
+        if: always() && steps.compile-new-antlr.outcome == 'success'
         uses: ./.github/composite-actions/build-extensions
         with:
           install_dir: ${{env.NEW_INSTALL_DIR}}

diff --git a/.github/workflows/minor-version-upgrade.yml b/.github/workflows/minor-version-upgrade.yml
@@ -29,6 +29,8 @@ jobs:
         id: compile-antlr
         if: always() && steps.build-modified-postgres-older.outcome == 'success'
         uses: ./.github/composite-actions/compile-antlr
+        with:
+          version: 4.9.3
 
       - uses: actions/checkout@v2
         with:
@@ -115,9 +117,15 @@ jobs:
         id: build-modified-postgres-newer
         if: always() && steps.install-extensions-older.outcome == 'success'
         uses: ./.github/composite-actions/build-modified-postgres
+
+      - name: Compile new ANTLR
+        id: compile-new-antlr
+        if: always() && steps.build-modified-postgres-newer.outcome == 'success'
+        uses: ./.github/composite-actions/compile-antlr
 
       - name: Set env variables and build extensions
         id: build-extensions-newer
+        if: always() && steps.compile-new-antlr.outcome == 'success'
         uses: ./.github/composite-actions/build-extensions
 
       - name: Build PostGIS Extension

diff --git a/.github/workflows/pg_dump-restore-test.yml b/.github/workflows/pg_dump-restore-test.yml
@@ -74,6 +74,7 @@ jobs:
           install_dir: ${{ steps.find-branch.outputs.base-dir }}
           migration_mode: 'multi-db'
           dump_restore: 'true'
+          antlr_version: ${{ (steps.read-base-and-final-version.outputs.base-version == 'source.latest' || fromJSON(steps.read-base-and-final-version.outputs.base-version) >= 16.7 ) && '4.13.2' || '4.9.3' }}
 
       - name: Setup Dump Restore Composite Action
         id: setup-dump-restore-ca

diff --git a/.github/workflows/singledb-version-upgrade.yml b/.github/workflows/singledb-version-upgrade.yml
@@ -23,6 +23,7 @@ jobs:
           extension_branch: ${{ env.EXTENSION_BRANCH_FROM }}
           install_dir: ${{ env.OLD_INSTALL_DIR }}
           migration_mode: 'single-db'
+          antlr_version: 4.9.3
 
       - name: Check Babelfish metadata inconsistency before Major Version Upgrade
         id: check-babelfish-inconsistency