[OSS-ONLY] Update antlr to 4.13.2 (babelfish-for-postgresql#3067)

Update the antlr version to 4.13.2. This dramatically improves ANTLR parsing performance for certain queries, reducing their execution times by up to 40%. Note that this will require developers to have Java 11+ installed in order to develop locally. Also update the github actions to make it so that previous versions of Babelfish will be compiled using the original antlr-4.9.3 version. This is purely a fix for testing. Task: BABEL-5371 Signed-off-by: Jason Teng <[email protected]>
amazon-aurora · Nov 25, 2024 · 79a89b8 · 79a89b8
1 parent 86f693b
commit 79a89b8
Show file tree

Hide file tree

Showing 26 changed files with 119 additions and 40 deletions.
diff --git a/.github/composite-actions/compile-antlr/action.yml b/.github/composite-actions/compile-antlr/action.yml
@@ -1,6 +1,10 @@
 name: 'Compile ANTLR'
 
 inputs:
+  version:
+    description: 'version of ANTLR to install'
+    required: no
+    default: 4.13.2
   install_dir:
     description: 'Engine install directory'
     required: no
@@ -12,9 +16,12 @@ runs:
 
     - name: Compile ANTLR
       run: |
-        ANTLR_VERSION=4.9.3
+        ANTLR_VERSION=${{inputs.version}}
         
         cd contrib/babelfishpg_tsql/antlr/thirdparty/antlr/
+        if [ ! -f antlr-$ANTLR_VERSION-complete.jar ]; then
+          wget "http://www.antlr.org/download/antlr-$ANTLR_VERSION-complete.jar"
+        fi
         sudo cp "antlr-$ANTLR_VERSION-complete.jar" /usr/local/lib
         cd ..
         wget "http://www.antlr.org/download/antlr4-cpp-runtime-$ANTLR_VERSION-source.zip"

diff --git a/.github/composite-actions/dump-restore-util/action.yml b/.github/composite-actions/dump-restore-util/action.yml
@@ -33,6 +33,10 @@ inputs:
     description: "Dump type full or combination of schema-only and data-only"
     required: false
     default: 'full'
+  antlr_version:
+    description: "ANTLR version to use (based on Extension Branch)"
+    required: false
+    default: 4.13.2
 
 runs:
   using: "composite"
@@ -45,6 +49,7 @@ runs:
         engine_branch: ${{ inputs.engine_branch }}
         extension_branch: ${{ inputs.extension_branch }}
         pg_new_dir: ${{ inputs.pg_new_dir }}
+        antlr_version: ${{ inputs.antlr_version }}
 
     - name: Build latest dump/restore utilities
       id: build-dump-utils

diff --git a/.github/composite-actions/major-version-upgrade-util/action.yml b/.github/composite-actions/major-version-upgrade-util/action.yml
@@ -22,6 +22,10 @@ inputs:
     description: "Server collation name"
     required: false
     default: "default"
+  antlr_version:
+    description: "ANTLR version to use (based on Extension Branch)"
+    required: false
+    default: 4.13.2
 
 runs:
   using: "composite"
@@ -34,6 +38,7 @@ runs:
         engine_branch: ${{ inputs.engine_branch }}
         extension_branch: ${{ inputs.extension_branch }}
         pg_new_dir: ${{ inputs.pg_new_dir }}
+        antlr_version: ${{ inputs.antlr_version }}
 
     - name: Run pg_upgrade
       id: run-pg_upgrade

diff --git a/.github/composite-actions/minor-version-upgrade-util/action.yml b/.github/composite-actions/minor-version-upgrade-util/action.yml
@@ -16,18 +16,33 @@ inputs:
     description: "Server collation name"
     required: false
     default: "default"
+  antlr_version:
+    description: "ANTLR version to use (based on Extension Branch)"
+    required: false
+    default: 4.13.2
 
 runs:
   using: "composite"
   steps:    
     - name: Build and run tests for Postgres engine using ${{ inputs.engine_branch }}
+      id: build-modified-postgres-new
+      if: always()
       uses: ./.github/composite-actions/build-modified-postgres
       with:
         engine_branch: ${{ inputs.engine_branch }}
         install_dir: ${{ inputs.install_dir }}
 
+    - name: Compile new ANTLR
+      id: compile-new-antlr
+      if: always() && steps.build-modified-postgres-new.outcome == 'success'
+      uses: ./.github/composite-actions/compile-antlr
+      with:
+        version: ${{ inputs.antlr_version }}
+        install_dir: ${{inputs.install_dir}}
+
     - name: Set env variables and build extensions using ${{ inputs.extension_branch }}
       id: build-extensions-newer
+      if: always() && steps.compile-new-antlr.outcome == 'success'
       uses: ./.github/composite-actions/build-extensions
       with: 
         install_dir: ${{ inputs.install_dir }}

diff --git a/.github/composite-actions/setup-base-version/action.yml b/.github/composite-actions/setup-base-version/action.yml
@@ -20,6 +20,10 @@ inputs:
     description: "Whether it is version upgrade or dump/restore"
     required: false
     default: 'false'
+  antlr_version:
+    description: "ANTLR version to use (based on Extension Branch)"
+    required: false
+    default: 4.13.2
 
 runs:
   using: "composite"
@@ -42,6 +46,7 @@ runs:
       if: always() && steps.build-modified-postgres.outcome == 'success'
       uses: ./.github/composite-actions/compile-antlr
       with:
+        version: ${{ inputs.antlr_version }}
         install_dir: ${{ inputs.install_dir }}
 
     - name: Set env variables and build extensions

diff --git a/.github/composite-actions/setup-dump-restore-ca/action.yml b/.github/composite-actions/setup-dump-restore-ca/action.yml
@@ -40,6 +40,12 @@ runs:
           is_final_ver=false; [[ i -eq $LEN-1 ]] && is_final_ver=true
           pg_old_dir=$(echo psql$(awk -F. '{print $1}' <<< $previous_installed_version))
           pg_new_dir=$(echo psql$(awk -F. '{print $1}' <<< $dump_restore_version))
+          if [[ "$dump_restore_version" == 'target.latest' || "$dump_restore_version" -ge 16.6 ]]
+          then
+            antlr_version=4.13.2
+          else
+            antlr_version=4.9.3
+          fi
 
           if [[ "$previous_installed_version" == "$dump_restore_version" ]]; then
             pg_new_dir=$(echo $pg_new_dir.$i)
@@ -56,7 +62,7 @@ runs:
           type_var=".\"dump-restore-version\"[${{ matrix.upgrade-path.id }}][$i].\"type\""
           type=$(yq $type_var ${{ github.workspace }}/.github/configuration/dump-restore-test-configuration.yml)
 
-          printf "    - name: Dump and Restore to version $dump_restore_version\n      id: dump-restore-version-$i\n      if: always() $temp\n      uses: ${uses_file}\n      with: \n        engine_branch: ${engine_branch}\n        extension_branch: ${extension_branch}\n        is_final_ver: ${is_final_ver}\n        pg_old_dir: ${pg_old_dir}\n        pg_new_dir: ${pg_new_dir}\n        migration_mode: 'multi-db'\n        database_level: ${database_level}\n        dump_data_as: ${dump_data_as}\n        dump_format: ${dump_format}\n        type: ${type}\n\n"   >> $dump_restore_version_dir_path/action.yml
+          printf "    - name: Dump and Restore to version $dump_restore_version\n      id: dump-restore-version-$i\n      if: always() $temp\n      uses: ${uses_file}\n      with: \n        engine_branch: ${engine_branch}\n        extension_branch: ${extension_branch}\n        is_final_ver: ${is_final_ver}\n        pg_old_dir: ${pg_old_dir}\n        pg_new_dir: ${pg_new_dir}\n        migration_mode: 'multi-db'\n        database_level: ${database_level}\n        dump_data_as: ${dump_data_as}\n        dump_format: ${dump_format}\n        type: ${type}\n        antlr_version: ${antlr_version}\n\n"   >> $dump_restore_version_dir_path/action.yml
 
           previous_installed_version=$dump_restore_version
         done

diff --git a/.github/composite-actions/setup-new-version/action.yml b/.github/composite-actions/setup-new-version/action.yml
@@ -9,6 +9,10 @@ inputs:
   pg_new_dir:
     description: "Install new version in this directory"
     required: true
+  antlr_version:
+    description: "ANTLR version to use (based on Extension Branch)"
+    required: false
+    default: 4.13.2
 
 runs:
   using: "composite"
@@ -21,15 +25,17 @@ runs:
         engine_branch: ${{ inputs.engine_branch }}
         install_dir: ${{ inputs.pg_new_dir }}
 
-    - name: Copy ANTLR
-      id: copy-antlr
+    - name: Compile new ANTLR
+      id: compile-new-antlr
       if: always() && steps.build-modified-postgres-new.outcome == 'success'
-      run: cp "/usr/local/lib/libantlr4-runtime.so.4.9.3" ~/${{ inputs.pg_new_dir }}/lib/
-      shell: bash
+      uses: ./.github/composite-actions/compile-antlr
+      with:
+        version: ${{ inputs.antlr_version }}
+        install_dir: ${{inputs.pg_new_dir}}
 
     - name: Build Extensions
       id: build-extensions-new
-      if: always() && steps.copy-antlr.outcome == 'success'
+      if: always() && steps.compile-new-antlr.outcome == 'success'
       uses: ./.github/composite-actions/build-extensions
       with:
         install_dir: ${{ inputs.pg_new_dir }}

diff --git a/.github/composite-actions/setup-upgrade-version-ca/action.yml b/.github/composite-actions/setup-upgrade-version-ca/action.yml
@@ -45,6 +45,12 @@ runs:
           is_final_ver=false; [[ i -eq $LEN-1 ]] && is_final_ver=true
           pg_old_dir=$(echo psql$(awk -F. '{print $1}' <<< $previous_installed_version))
           pg_new_dir=$(echo psql$(awk -F. '{print $1}' <<< $upgrade_version))
+          if [[ "$upgrade_version" == 'target.latest' || "$upgrade_version" -ge 16.6 ]]
+          then
+            antlr_version=4.13.2
+          else
+            antlr_version=4.9.3
+          fi
           
           upgrade_type_var=".\"upgrade-version\"[${{ matrix.upgrade-path.id }}]."upgrade-path"[$i].\"upgrade-type\"" 
           upgrade_type=$(yq $upgrade_type_var ${{ github.workspace }}/.github/configuration/upgrade-test-configuration.yml)
@@ -60,9 +66,9 @@ runs:
           # Appending upgrade step in upgrade-version composite action based on upgrade type
           if [[ $upgrade_type == 'major' ]]
           then
-            printf "    - name: Upgrade Version to $upgrade_version\n      id: upgrade-version-$i\n      if: always() $temp\n      uses: ${uses_file}\n      with: \n        engine_branch: ${engine_branch}\n        extension_branch: ${extension_branch}\n        is_final_ver: ${is_final_ver}\n        pg_old_dir: ${pg_old_dir}\n        pg_new_dir: ${pg_new_dir}\n        migration_mode: 'multi-db'\n        server_collation_name: ${{ inputs.server_collation_name }}\n\n"   >> $upgrade_version_dir_path/action.yml
+            printf "    - name: Upgrade Version to $upgrade_version\n      id: upgrade-version-$i\n      if: always() $temp\n      uses: ${uses_file}\n      with: \n        engine_branch: ${engine_branch}\n        extension_branch: ${extension_branch}\n        is_final_ver: ${is_final_ver}\n        pg_old_dir: ${pg_old_dir}\n        pg_new_dir: ${pg_new_dir}\n        migration_mode: 'multi-db'\n        server_collation_name: ${{ inputs.server_collation_name }}\n        antlr_version: ${antlr_version}\n\n"   >> $upgrade_version_dir_path/action.yml
           else
-            printf "    - name: Upgrade Version to $upgrade_version\n      id: upgrade-version-$i\n      if: always() $temp\n      uses: ${uses_file}\n      with: \n        engine_branch: ${engine_branch}\n        extension_branch: ${extension_branch}\n        is_final_ver: ${is_final_ver}\n        install_dir: ${pg_new_dir}\n        server_collation_name: ${{ inputs.server_collation_name }}\n\n"   >> $upgrade_version_dir_path/action.yml
+            printf "    - name: Upgrade Version to $upgrade_version\n      id: upgrade-version-$i\n      if: always() $temp\n      uses: ${uses_file}\n      with: \n        engine_branch: ${engine_branch}\n        extension_branch: ${extension_branch}\n        is_final_ver: ${is_final_ver}\n        install_dir: ${pg_new_dir}\n        server_collation_name: ${{ inputs.server_collation_name }}\n        antlr_version: ${antlr_version}\n\n"   >> $upgrade_version_dir_path/action.yml
           fi
 
           previous_installed_version=$upgrade_version

diff --git a/.github/workflows/isolation-tests.yml b/.github/workflows/isolation-tests.yml
@@ -53,12 +53,12 @@ jobs:
           curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list
           cd ~/work/babelfish_extensions/babelfish_extensions/test/python
           sudo ACCEPT_EULA=Y apt-get install -y msodbcsql17 python3-dev
-          pip3 install pyodbc pymssql pytest pytest-xdist antlr4-python3-runtime==4.9.3
+          pip3 install pyodbc pymssql pytest pytest-xdist antlr4-python3-runtime==4.13.2
       
       - name: Generate .spec file parser
         run: |
           cd ~/work/babelfish_extensions/babelfish_extensions/test/python/isolationtest/
-          java -Xmx500M -cp /usr/local/lib/antlr-4.9.3-complete.jar org.antlr.v4.Tool -Dlanguage=Python3 ./parser/*.g4 -visitor -no-listener
+          java -Xmx500M -cp /usr/local/lib/antlr-4.13.2-complete.jar org.antlr.v4.Tool -Dlanguage=Python3 ./parser/*.g4 -visitor -no-listener
 
       - name: Run Isolation tests
         id: run-isolation-tests

diff --git a/.github/workflows/major-version-upgrade.yml b/.github/workflows/major-version-upgrade.yml
@@ -38,6 +38,7 @@ jobs:
         if: always() && steps.build-modified-postgres-old.outcome == 'success'
         uses: ./.github/composite-actions/compile-antlr
         with:
+          version: 4.9.3
           install_dir: ${{env.OLD_INSTALL_DIR}}
 
       - uses: actions/checkout@v2
@@ -136,13 +137,17 @@ jobs:
         with:
           install_dir: ${{env.NEW_INSTALL_DIR}}
           release_mode: 'yes'
-
-      - name: Copy ANTLR
-        run: cp "/usr/local/lib/libantlr4-runtime.so.4.9.3" ~/${{env.NEW_INSTALL_DIR}}/lib/
+
+      - name: Compile new ANTLR
+        id: compile-new-antlr
+        if: always() && steps.build-modified-postgres-new.outcome == 'success'
+        uses: ./.github/composite-actions/compile-antlr
+        with:
+          install_dir: ${{env.NEW_INSTALL_DIR}}
 
       - name: Build Extensions using latest version
         id: build-extensions-new
-        if: always() && steps.build-modified-postgres-new.outcome == 'success'
+        if: always() && steps.compile-new-antlr.outcome == 'success'
         uses: ./.github/composite-actions/build-extensions
         with:
           install_dir: ${{env.NEW_INSTALL_DIR}}

diff --git a/.github/workflows/minor-version-upgrade.yml b/.github/workflows/minor-version-upgrade.yml
@@ -29,6 +29,8 @@ jobs:
         id: compile-antlr
         if: always() && steps.build-modified-postgres-older.outcome == 'success'
         uses: ./.github/composite-actions/compile-antlr
+        with:
+          version: 4.9.3
 
       - uses: actions/checkout@v2
         with:
@@ -115,9 +117,15 @@ jobs:
         id: build-modified-postgres-newer
         if: always() && steps.install-extensions-older.outcome == 'success'
         uses: ./.github/composite-actions/build-modified-postgres
+
+      - name: Compile new ANTLR
+        id: compile-new-antlr
+        if: always() && steps.build-modified-postgres-newer.outcome == 'success'
+        uses: ./.github/composite-actions/compile-antlr
 
       - name: Set env variables and build extensions
         id: build-extensions-newer
+        if: always() && steps.compile-new-antlr.outcome == 'success'
         uses: ./.github/composite-actions/build-extensions
 
       - name: Build PostGIS Extension

diff --git a/.github/workflows/pg_dump-restore-test.yml b/.github/workflows/pg_dump-restore-test.yml
@@ -74,6 +74,7 @@ jobs:
           install_dir: ${{ steps.find-branch.outputs.base-dir }}
           migration_mode: 'multi-db'
           dump_restore: 'true'
+          antlr_version: ${{ (steps.read-base-and-final-version.outputs.base-version == 'source.latest' || fromJSON(steps.read-base-and-final-version.outputs.base-version) >= 16.6 ) && '4.13.2' || '4.9.3' }}
 
       - name: Setup Dump Restore Composite Action
         id: setup-dump-restore-ca

diff --git a/.github/workflows/singledb-version-upgrade.yml b/.github/workflows/singledb-version-upgrade.yml
@@ -23,6 +23,7 @@ jobs:
           extension_branch: ${{ env.EXTENSION_BRANCH_FROM }}
           install_dir: ${{ env.OLD_INSTALL_DIR }}
           migration_mode: 'single-db'
+          antlr_version: 4.9.3
 
       - name: Check Babelfish metadata inconsistency before Major Version Upgrade
         id: check-babelfish-inconsistency

diff --git a/.github/workflows/tap-tests.yml b/.github/workflows/tap-tests.yml
@@ -59,6 +59,7 @@ jobs:
         if: always() && steps.build-modified-postgres-old.outcome == 'success'
         uses: ./.github/composite-actions/compile-antlr
         with:
+          version: 4.9.3
           install_dir: ${{env.OLD_INSTALL_DIR}}
 
       - name: Build PostGIS Extension using ${{env.EXTENSION_BRANCH_OLD}}
@@ -116,13 +117,17 @@ jobs:
         with:
           tap_tests: 'yes'
           install_dir: ${{env.NEW_INSTALL_DIR}}
-
-      - name: Copy ANTLR
-        run: cp "/usr/local/lib/libantlr4-runtime.so.4.9.3" ~/${{env.NEW_INSTALL_DIR}}/lib/
+
+      - name: Compile new ANTLR
+        id: compile-new-antlr
+        if: always() && steps.build-modified-postgres-new.outcome == 'success'
+        uses: ./.github/composite-actions/compile-antlr
+        with:
+          install_dir: ${{env.NEW_INSTALL_DIR}}
 
       - name: Build Extensions using latest version
         id: build-extensions-new
-        if: always() && steps.build-modified-postgres-new.outcome == 'success'
+        if: always() && steps.compile-new-antlr.outcome == 'success'
         uses: ./.github/composite-actions/build-extensions
         with:
           install_dir: ${{env.NEW_INSTALL_DIR}}

diff --git a/.github/workflows/upgrade-test.yml b/.github/workflows/upgrade-test.yml
@@ -67,6 +67,7 @@ jobs:
           install_dir: ${{ steps.find-branch.outputs.base-dir }}
           migration_mode: 'multi-db'
           server_collation_name: ${{ matrix.upgrade-path.server_collation_name }}
+          antlr_version: ${{ (steps.read-base-and-final-version.outputs.base-version == 'source.latest' || fromJSON(steps.read-base-and-final-version.outputs.base-version) >= 16.6 ) && '4.13.2' || '4.9.3' }}
 
       - name: Setup Upgrade Version Composite Action
         id: setup-upgrade-ver-ca

diff --git a/INSTALLING.md.tmpl b/INSTALLING.md.tmpl
@@ -55,19 +55,19 @@ export PG_SRC=$(realpath $PWD)
 
 ### Compile ANTLR 4
 
-Unfortunately, there are [no prebuilt C++ binaries for the Antlr 4.9.3 runtime version](https://www.antlr.org/download.html) for Linux. You will need to compile and install ANTLR manually.
+Unfortunately, there are [no prebuilt C++ binaries for the Antlr 4.13.2 runtime version](https://www.antlr.org/download.html) for Linux. You will need to compile and install ANTLR manually.
 
 First, define the following variables in your environment:
 
 ```sh
-export ANTLR4_VERSION=4.9.3
+export ANTLR4_VERSION=4.13.2
 export ANTLR4_JAVA_BIN=/usr/bin/java
 export ANTLR4_RUNTIME_LIBRARIES=/usr/include/antlr4-runtime
 export ANTLR_EXECUTABLE=/usr/local/lib/antlr-${ANTLR4_VERSION}-complete.jar
 export ANTLR_RUNTIME=~/antlr4
 ```
 
-The [Antlr 4.9.3 Runtime](https://www.antlr.org/) files are distributed with the Babelfish source code.  Use the following commands to copy the files into place:
+The [Antlr 4.13.2 Runtime](https://www.antlr.org/) files are distributed with the Babelfish source code.  Use the following commands to copy the files into place:
 
 ```sh
 sudo cp ${PG_SRC}/contrib/babelfishpg_tsql/antlr/thirdparty/antlr/antlr-${ANTLR4_VERSION}-complete.jar /usr/local/lib

diff --git a/contrib/README.md b/contrib/README.md
@@ -72,30 +72,30 @@ The following build instructions comply with Ubuntu 20.04 and Amazon Linux 2 env
 
 2. Install ANTLR
 
-    The `babelfishpg_tsql` contrib has a plug-in parser generated by ANTLR, which depends on cmake and `antlr4-cpp-runtime-4.9.3`. Unfortunately, there aren't binaries for [C++ targets available](https://www.antlr.org/download.html). You'll also need have `uuid-devel` installed in order to install `antlr4-cpp-runtime-4.9.3`.
+    The `babelfishpg_tsql` contrib has a plug-in parser generated by ANTLR, which depends on cmake and `antlr4-cpp-runtime-4.13.2`. Unfortunately, there aren't binaries for [C++ targets available](https://www.antlr.org/download.html). You'll also need have `uuid-devel` installed in order to install `antlr4-cpp-runtime-4.13.2`.
 
     First copy the jar file in `contrib/babelfishpg_tsql/antlr/thirdparty/antlr/` to another location:
 
     ```
     cd babelfish_extensions/contrib/babelfishpg_tsql/antlr/thirdparty/antlr/
-    sudo cp antlr-4.9.3-complete.jar /usr/local/lib
+    sudo cp antlr-4.13.2-complete.jar /usr/local/lib
     ```
 
     Compile antlr4:
 
     ```
-    wget http://www.antlr.org/download/antlr4-cpp-runtime-4.9.3-source.zip
-    unzip -d antlr4 antlr4-cpp-runtime-4.9.3-source.zip 
+    wget http://www.antlr.org/download/antlr4-cpp-runtime-4.13.2-source.zip
+    unzip -d antlr4 antlr4-cpp-runtime-4.13.2-source.zip 
     cd antlr4
     mkdir build && cd build 
-    cmake .. -DANTLR_JAR_LOCATION=/usr/local/lib/antlr-4.9.3-complete.jar -DCMAKE_INSTALL_PREFIX=/usr/local -DWITH_DEMO=True
+    cmake .. -DANTLR_JAR_LOCATION=/usr/local/lib/antlr-4.13.2-complete.jar -DCMAKE_INSTALL_PREFIX=/usr/local -DWITH_DEMO=True
     make
     sudo make install
     ```
 
     Copy libantlr4-runtime to postgres/lib
     ```
-    cp /usr/local/lib/libantlr4-runtime.so.4.9.3 ~/postgres/lib/
+    cp /usr/local/lib/libantlr4-runtime.so.4.13.2 ~/postgres/lib/
     ```
 
     - If you come across the error

diff --git a/contrib/babelfishpg_tsql/Makefile b/contrib/babelfishpg_tsql/Makefile
@@ -90,6 +90,7 @@ PG_CXXFLAGS += -Wno-deprecated -Wno-error=attributes -Wno-suggest-attribute=form
 PG_CXXFLAGS += -Wno-undef -Wall -Wcpp
 PG_CXXFLAGS += -Wno-register # otherwise C++17 gags on PostgreSQL headers
 PG_CXXFLAGS += -I$(ANTLR4_RUNTIME_INCLUDE_DIR)
+PG_CXXFLAGS += -std=c++17
 PG_CFLAGS += -g -Werror -Wfloat-conversion
 PG_CFLAGS += -fstack-protector-strong
 PG_CPPFLAGS += -I$(TSQLSRC) -I$(PG_SRC) -DFAULT_INJECTOR -Wfloat-conversion