From 3c4dd6f21588fe5f7d34865f555ed2298d6b9a6f Mon Sep 17 00:00:00 2001
From: mchades <liminghuang@datastrato.com>
Date: Fri, 9 Aug 2024 17:07:13 +0800
Subject: [PATCH] CI debug

---
 .../workflows/backend-integration-test.yml    |   5 +
 .github/workflows/build.yml                   | 109 -----------------
 .github/workflows/flink-integration-test.yml  | 104 -----------------
 .../workflows/frontend-integration-test.yml   | 107 -----------------
 .github/workflows/python-integration-test.yml |  86 --------------
 .github/workflows/spark-integration-test.yml  | 110 ------------------
 6 files changed, 5 insertions(+), 516 deletions(-)
 delete mode 100644 .github/workflows/build.yml
 delete mode 100644 .github/workflows/flink-integration-test.yml
 delete mode 100644 .github/workflows/frontend-integration-test.yml
 delete mode 100644 .github/workflows/python-integration-test.yml
 delete mode 100644 .github/workflows/spark-integration-test.yml

diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
index 2464a06e1b1..3a2f0ec72ab 100644
--- a/.github/workflows/backend-integration-test.yml
+++ b/.github/workflows/backend-integration-test.yml
@@ -65,6 +65,7 @@ jobs:
         backend: [ mysql, h2]
     env:
       PLATFORM: ${{ matrix.architecture }}
+      ACTIVE_CI: true
     steps:
       - uses: actions/checkout@v3
 
@@ -98,6 +99,10 @@ jobs:
           -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test 
           -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
 
+      - name: Setup debug Github Action
+        if: ${{ contains(github.event.pull_request.labels.*.name, 'debug action') }}
+        uses: csexton/debugger-action@master
+
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3
         if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
deleted file mode 100644
index cbb9eaffb68..00000000000
--- a/.github/workflows/build.yml
+++ /dev/null
@@ -1,109 +0,0 @@
-name: build
-
-# Controls when the workflow will run
-on:
-  # Triggers the workflow on push or pull request events but only for the "main" branch
-  push:
-    branches: [ "main", "branch-*" ]
-  pull_request:
-    branches: [ "main", "branch-*" ]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
-jobs:
-  changes:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: dorny/paths-filter@v2
-        id: filter
-        with:
-          filters: |
-            source_changes:
-              - .github/**
-              - api/**
-              - bin/**
-              - catalogs/**
-              - clients/client-java/**
-              - clients/client-java-runtime/**
-              - clients/filesystem-hadoop3/**
-              - clients/filesystem-hadoop3-runtime/**
-              - common/**
-              - conf/**
-              - core/**
-              - dev/**
-              - gradle/**
-              - integration-test/**
-              - meta/**
-              - server/**
-              - server-common/**
-              - spark-connector/**
-              - flink-connector/**
-              - trino-connector/**
-              - web/**
-              - docs/open-api/**
-              - build.gradle.kts
-              - gradle.properties
-              - gradlew
-              - setting.gradle.kts
-    outputs:
-      source_changes: ${{ steps.filter.outputs.source_changes }}
-
-  compile-check:
-    runs-on: ubuntu-latest
-    needs: changes
-    if: needs.changes.outputs.source_changes != 'true'
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-java@v4
-        with:
-          java-version: 8
-          distribution: 'temurin'
-          cache: 'gradle'
-
-      - name: Build with Gradle
-        run: ./gradlew build -x test -PjdkVersion=8
-
-  build:
-    # The type of runner that the job will run on
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        java-version: [ 8, 11, 17 ]
-    timeout-minutes: 30
-    needs: changes
-    if: needs.changes.outputs.source_changes == 'true'
-    # Steps represent a sequence of tasks that will be executed as part of the job
-    steps:
-      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-java@v4
-        with:
-          java-version: ${{ matrix.java-version }}
-          distribution: 'temurin'
-          cache: 'gradle'
-
-      - name: Test publish to local
-        run: ./gradlew publishToMavenLocal -x test -PjdkVersion=${{ matrix.java-version }}
-
-      - name: Free up disk space
-        run: |
-          dev/ci/util_free_space.sh
-
-      - name: Build with Gradle
-        run: ./gradlew build -PskipITs -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false -x :clients:client-python:build
-
-      - name: Upload unit tests report
-        uses: actions/upload-artifact@v3
-        if: failure()
-        with:
-          name: unit test report
-          path: |
-            build/reports
-            catalogs/**/*.log
-            catalogs/**/*.tar
diff --git a/.github/workflows/flink-integration-test.yml b/.github/workflows/flink-integration-test.yml
deleted file mode 100644
index c59c0fd23f7..00000000000
--- a/.github/workflows/flink-integration-test.yml
+++ /dev/null
@@ -1,104 +0,0 @@
-name: Flink Integration Test
-
-# Controls when the workflow will run
-on:
-  # Triggers the workflow on push or pull request events but only for the "main" branch
-  push:
-    branches: [ "main", "branch-*" ]
-  pull_request:
-    branches: [ "main", "branch-*" ]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  changes:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: dorny/paths-filter@v2
-        id: filter
-        with:
-          filters: |
-            source_changes:
-              - .github/**
-              - api/**
-              - bin/**
-              - catalogs/**
-              - clients/client-java/**
-              - clients/client-java-runtime/**
-              - clients/filesystem-hadoop3/**
-              - clients/filesystem-hadoop3-runtime/**
-              - common/**
-              - conf/**
-              - core/**
-              - dev/**
-              - gradle/**
-              - meta/**
-              - server/**
-              - server-common/**
-              - flink-connector/**
-              - docs/open-api/**
-              - build.gradle.kts
-              - gradle.properties
-              - gradlew
-              - setting.gradle.kts
-    outputs:
-      source_changes: ${{ steps.filter.outputs.source_changes }}
-
-  # Integration test for AMD64 architecture
-  test-amd64-arch:
-    needs: changes
-    if: needs.changes.outputs.source_changes == 'true'
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    strategy:
-      matrix:
-        architecture: [linux/amd64]
-        java-version: [ 8, 11, 17 ]
-    env:
-      PLATFORM: ${{ matrix.architecture }}
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-java@v4
-        with:
-          java-version: ${{ matrix.java-version }}
-          distribution: 'temurin'
-          cache: 'gradle'
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Check required command
-        run: |
-          dev/ci/check_commands.sh
-
-      - name: Package Gravitino
-        run: |
-          ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
-
-      - name: Free up disk space
-        run: |
-          dev/ci/util_free_space.sh
-
-      - name: Flink Integration Test
-        id: integrationTest
-        run: |
-          ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:test --tests "org.apache.gravitino.flink.connector.integration.test.**"
-          ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:test --tests "org.apache.gravitino.flink.connector.integration.test.**"
-
-      - name: Upload integrate tests reports
-        uses: actions/upload-artifact@v3
-        if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
-        with:
-          name: flink-connector-integrate-test-reports-${{ matrix.java-version }}
-          path: |
-            build/reports
-            flink-connector/build/flink-connector-integration-test.log
-            flink-connector/build/*.tar
-            distribution/package/logs/gravitino-server.out
-            distribution/package/logs/gravitino-server.log
-            catalogs/**/*.log
-            catalogs/**/*.tar
\ No newline at end of file
diff --git a/.github/workflows/frontend-integration-test.yml b/.github/workflows/frontend-integration-test.yml
deleted file mode 100644
index 2d730f392ff..00000000000
--- a/.github/workflows/frontend-integration-test.yml
+++ /dev/null
@@ -1,107 +0,0 @@
-name: Frontend Integration Test
-
-# Controls when the workflow will run
-on:
-  # Triggers the workflow on push or pull request events but only for the "main" branch
-  push:
-    branches: [ "main", "branch-*" ]
-  pull_request:
-    branches: [ "main", "branch-*" ]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  changes:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: dorny/paths-filter@v2
-        id: filter
-        with:
-          filters: |
-            source_changes:
-              - .github/**
-              - api/**
-              - bin/**
-              - catalogs/**
-              - clients/client-java/**
-              - clients/client-java-runtime/**
-              - clients/filesystem-hadoop3/**
-              - clients/filesystem-hadoop3-runtime/**
-              - common/**
-              - conf/**
-              - core/**
-              - dev/**
-              - gradle/**
-              - integration-test/**
-              - meta/**
-              - server/**
-              - server-common/**
-              - spark-connector/**
-              - trino-connector/**
-              - web/**
-              - docs/open-api/**
-              - build.gradle.kts
-              - gradle.properties
-              - gradlew
-              - setting.gradle.kts
-    outputs:
-      source_changes: ${{ steps.filter.outputs.source_changes }}
-
-  # Integration test for AMD64 architecture
-  test-amd64-arch:
-    needs: changes
-    if: needs.changes.outputs.source_changes == 'true'
-    runs-on: ubuntu-latest
-    timeout-minutes: 60
-    strategy:
-      matrix:
-        architecture: [linux/amd64]
-        java-version: [ 8 ]
-    env:
-      PLATFORM: ${{ matrix.architecture }}
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-java@v4
-        with:
-          java-version: ${{ matrix.java-version }}
-          distribution: 'temurin'
-          cache: 'gradle'
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Check required command
-        run: |
-          dev/ci/check_commands.sh
-
-      - name: Package Gravitino
-        run: |
-          ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
-
-      - name: Free up disk space
-        run: |
-          dev/ci/util_free_space.sh
-
-      - name: Frontend Integration Test
-        id: integrationTest
-        run: |
-          ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :integration-test:test --tests "org.apache.gravitino.integration.test.web.ui.**"
-          ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :integration-test:test --tests "org.apache.gravitino.integration.test.web.ui.**"
-
-      - name: Upload integrate tests reports
-        uses: actions/upload-artifact@v3
-        if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
-        with:
-          name: integrate-test-reports-${{ matrix.java-version }}
-          path: |
-            build/reports
-            integration-test/build/integration-test-integration-test.log
-            integration-test/build/*.tar
-            distribution/package/logs/gravitino-server.out
-            distribution/package/logs/gravitino-server.log
-            catalogs/**/*.log
-            catalogs/**/*.tar
\ No newline at end of file
diff --git a/.github/workflows/python-integration-test.yml b/.github/workflows/python-integration-test.yml
deleted file mode 100644
index 4e9f96bc690..00000000000
--- a/.github/workflows/python-integration-test.yml
+++ /dev/null
@@ -1,86 +0,0 @@
-name: Python Client Integration Test
-
-# Controls when the workflow will run
-on:
-  # Triggers the workflow on push or pull request events but only for the "main" branch
-  push:
-    branches: [ "main", "branch-*" ]
-  pull_request:
-    branches: [ "main", "branch-*" ]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  changes:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: dorny/paths-filter@v2
-        id: filter
-        with:
-          filters: |
-            source_changes:
-              - .github/**
-              - api/**
-              - catalogs/catalog-hadoop/**
-              - clients/client-python/**
-              - common/**
-              - conf/**
-              - core/**
-              - meta/**
-              - server/**
-              - server-common/**
-    outputs:
-      source_changes: ${{ steps.filter.outputs.source_changes }}
-
-  # Integration test for AMD64 architecture
-  test-amd64-arch:
-    needs: changes
-    if: needs.changes.outputs.source_changes == 'true'
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    strategy:
-      matrix:
-        architecture: [linux/amd64]
-        java-version: [ 8 ]
-    env:
-      PLATFORM: ${{ matrix.architecture }}
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-java@v4
-        with:
-          java-version: ${{ matrix.java-version }}
-          distribution: 'temurin'
-          cache: 'gradle'
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Python Client Integration Test
-        id: integrationTest
-        run: |
-          ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
-
-          for pythonVersion in "3.8" "3.9" "3.10" "3.11"
-          do
-            echo "Use Python version ${pythonVersion} to test the Python client."
-            ./gradlew -PjdkVersion=${{ matrix.java-version }} -PpythonVersion=${pythonVersion} -PskipDockerTests=false :clients:client-python:test
-            # Clean Gravitino database to clean test data
-            rm -rf ./distribution/package/data
-          done
-
-      - name: Upload integrate tests reports
-        uses: actions/upload-artifact@v3
-        if: ${{ failure() && steps.integrationTest.outcome == 'failure' }}
-        with:
-          name: integrate test reports
-          path: |
-            build/reports
-            integration-test/build/integration-test.log
-            distribution/package/logs/gravitino-server.out
-            distribution/package/logs/gravitino-server.log
-            catalogs/**/*.log
-            catalogs/**/*.tar
\ No newline at end of file
diff --git a/.github/workflows/spark-integration-test.yml b/.github/workflows/spark-integration-test.yml
deleted file mode 100644
index a2f11061cda..00000000000
--- a/.github/workflows/spark-integration-test.yml
+++ /dev/null
@@ -1,110 +0,0 @@
-name: Spark Integration Test
-
-# Controls when the workflow will run
-on:
-  # Triggers the workflow on push or pull request events but only for the "main" branch
-  push:
-    branches: [ "main", "branch-*" ]
-  pull_request:
-    branches: [ "main", "branch-*" ]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  changes:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: dorny/paths-filter@v2
-        id: filter
-        with:
-          filters: |
-            source_changes:
-              - .github/**
-              - api/**
-              - bin/**
-              - catalogs/**
-              - clients/client-java/**
-              - clients/client-java-runtime/**
-              - clients/filesystem-hadoop3/**
-              - clients/filesystem-hadoop3-runtime/**
-              - common/**
-              - conf/**
-              - core/**
-              - dev/**
-              - gradle/**
-              - iceberg/**
-              - meta/**
-              - server/**
-              - server-common/**
-              - spark-connector/**
-              - docs/open-api/**
-              - build.gradle.kts
-              - gradle.properties
-              - gradlew
-              - setting.gradle.kts
-    outputs:
-      source_changes: ${{ steps.filter.outputs.source_changes }}
-
-  # Integration test for AMD64 architecture
-  test-amd64-arch:
-    needs: changes
-    if: needs.changes.outputs.source_changes == 'true'
-    runs-on: ubuntu-latest
-    timeout-minutes: 90
-    strategy:
-      matrix:
-        architecture: [linux/amd64]
-        java-version: [ 8, 11, 17 ]
-        scala-version: [ 2.12 ]
-        test-mode: [ embedded, deploy ]
-    env:
-      PLATFORM: ${{ matrix.architecture }}
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-java@v4
-        with:
-          java-version: ${{ matrix.java-version }}
-          distribution: 'temurin'
-          cache: 'gradle'
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Check required command
-        run: |
-          dev/ci/check_commands.sh
-
-      - name: Package Gravitino
-        if : ${{ matrix.test-mode == 'deploy' }}
-        run: |
-          ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
-
-      - name: Free up disk space
-        run: |
-          dev/ci/util_free_space.sh
-
-      - name: Spark Integration Test
-        id: integrationTest
-        run: |
-          if [ "${{ matrix.scala-version }}" == "2.12" ];then 
-            ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.3:test --tests "org.apache.gravitino.spark.connector.integration.test.**"
-          fi
-          ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.4:test --tests "org.apache.gravitino.spark.connector.integration.test.**"
-          ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.5:test --tests "org.apache.gravitino.spark.connector.integration.test.**"
-
-      - name: Upload integrate tests reports
-        uses: actions/upload-artifact@v3
-        if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
-        with:
-          name: spark-connector-integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}
-          path: |
-            build/reports
-            spark-connector/v3.3/spark/build/spark-3.3-integration-test.log
-            spark-connector/v3.4/spark/build/spark-3.4-integration-test.log
-            spark-connector/v3.5/spark/build/spark-3.5-integration-test.log
-            distribution/package/logs/*.out
-            distribution/package/logs/*.log