optimize

mchades · Jun 7, 2024 · 49f1e93 · 49f1e93
1 parent 5b86a3d
commit 49f1e93
Show file tree

Hide file tree

Showing 9 changed files with 289 additions and 76 deletions.
diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
@@ -50,27 +50,26 @@ jobs:
     outputs:
       source_changes: ${{ steps.filter.outputs.source_changes }}
 
-  # Integration test for AMD64 architecture
-  test-amd64-arch:
+  catalog-it:
     needs: changes
     if: needs.changes.outputs.source_changes == 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
       matrix:
         architecture: [linux/amd64]
-        java-version: [ 8, 11, 17 ]
-        test-mode: [ embedded, deploy ]
-        backend: [ jdbcBackend, kvBackend]
+        java-version: [ 8 ]
+        catalog: [ lakehouse-iceberg, hive, jdbc-doris, jdbc-mysql, jdbc-postgresql, hadoop, kafka ]
     env:
       PLATFORM: ${{ matrix.architecture }}
     steps:
       - uses: actions/checkout@v3
 
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java-version }}
           distribution: 'temurin'
+          cache: 'gradle'
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
@@ -80,7 +79,6 @@ jobs:
           dev/ci/check_commands.sh
 
       - name: Package Gravitino
-        if : ${{ matrix.test-mode == 'deploy' }}
         run: |
           ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
 
@@ -89,22 +87,26 @@ jobs:
         uses: csexton/debugger-action@master
 
       - name: Free up disk space
+        if : ${{ matrix.catalog == 'jdbc-doris' }}
         run: |
           dev/ci/util_free_space.sh
 
-      - name: Backend Integration Test
-        id: integrationTest
-        run: >
-          ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -P${{ matrix.backend }} -PskipWebITs 
-          -x :web:test -x :clients:client-python:test -x :flink-connector:test -x :spark-connector:test -x :spark-connector:spark-common:test 
-          -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test 
-          -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
+      - name: Test catalog
+        id: catalogTest
+        run: |
+          for testMode in "embedded" "deploy"
+          do
+            echo "Run catalog it in ${testMode} mode with kvBackend"
+            ./gradlew :catalogs:catalog-${{ matrix.catalog }}:test -PskipTests -PtestMode=${testMode} -PjdkVersion=${{ matrix.java-version }} -PkvBackend
+            echo "Run catalog it in ${testMode} mode with jdbcBackend"
+            ./gradlew :catalogs:catalog-${{ matrix.catalog }}:test -PskipTests -PtestMode=${testMode} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend
+          done
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3
-        if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
+        if: ${{ (failure() && steps.catalogTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
         with:
-          name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}
+          name: catalog-test-reports-${{ matrix.java-version }}-${{ matrix.catalog }}
           path: |
             build/reports
             integration-test/build/*.log
@@ -116,3 +118,71 @@ jobs:
             catalogs/**/*.log
             catalogs/**/*.tar
             distribution/**/*.log
+
+  integration-test:
+    needs: changes
+    if: needs.changes.outputs.source_changes == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        architecture: [ linux/amd64 ]
+        java-version: [ 8 ]
+    env:
+      PLATFORM: ${{ matrix.architecture }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.java-version }}
+          distribution: 'temurin'
+          cache: 'gradle'
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Check required command
+        run: |
+          dev/ci/check_commands.sh
+
+      - name: Package Gravitino
+        run: |
+          ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
+
+      - name: Setup debug Github Action
+        if: ${{ contains(github.event.pull_request.labels.*.name, 'debug action') }}
+        uses: csexton/debugger-action@master
+
+      - name: Free up disk space
+        if: ${{ matrix.catalog == 'jdbc-doris' }}
+        run: |
+          dev/ci/util_free_space.sh
+
+      - name: Integration test
+        id: integrationTest
+        run: |
+          for testMode in "embedded" "deploy"
+          do
+            echo "Run integration test in ${testMode} mode with kvBackend"
+            ./gradlew :integration-test:test -PskipTrinoITs -PskipWebITs -PtestMode=${testMode} -PjdkVersion=${{ matrix.java-version }} -PkvBackend
+            echo "Run integration test in ${testMode} mode with jdbcBackend"
+            ./gradlew :integration-test:test -PskipTrinoITs -PskipWebITs -PtestMode=${testMode} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend
+          done
+
+      - name: Upload integrate tests reports
+        uses: actions/upload-artifact@v3
+        if: ${{ (failure() && steps.catalogTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
+        with:
+          name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.catalog }}
+          path: |
+            build/reports
+            integration-test/build/*.log
+            integration-test/build/*.tar
+            integration-test/build/trino-ci-container-log/hive/*.*
+            integration-test/build/trino-ci-container-log/hdfs/*.*
+            distribution/package/logs/gravitino-server.out
+            distribution/package/logs/gravitino-server.log
+            catalogs/**/*.log
+            catalogs/**/*.tar
+            distribution/**/*.log
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -81,10 +81,11 @@ jobs:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v3
 
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java-version }}
           distribution: 'temurin'
+          cache: 'gradle'
 
       - name: Test publish to local
         run: ./gradlew publishToMavenLocal -x test -PjdkVersion=${{ matrix.java-version }}
@@ -94,7 +95,7 @@ jobs:
           dev/ci/util_free_space.sh
 
       - name: Build with Gradle
-        run: ./gradlew build -PskipITs -PjdkVersion=${{ matrix.java-version }}
+        run: ./gradlew build -PskipITs -PjdkVersion=${{ matrix.java-version }} -x :clients:client-python:build
 
       - name: Upload unit tests report
         uses: actions/upload-artifact@v3

diff --git a/.github/workflows/flink-integration-test.yml b/.github/workflows/flink-integration-test.yml
@@ -56,16 +56,16 @@ jobs:
     strategy:
       matrix:
         architecture: [linux/amd64]
-        java-version: [ 8, 11, 17 ]
     env:
       PLATFORM: ${{ matrix.architecture }}
     steps:
       - uses: actions/checkout@v3
 
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
-          java-version: ${{ matrix.java-version }}
+          java-version: '8'
           distribution: 'temurin'
+          cache: 'gradle'
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
@@ -76,8 +76,7 @@ jobs:
 
       - name: Package Gravitino
         run: |
-          ./gradlew build -x test -PjdkVersion=${{ matrix.java-version }}
-          ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
+          ./gradlew compileDistribution -x test -PjdkVersion=8
 
       - name: Setup debug Github Action
         if: ${{ contains(github.event.pull_request.labels.*.name, 'debug action') }}
@@ -87,15 +86,43 @@ jobs:
         run: |
           dev/ci/util_free_space.sh
 
-      - name: Flink Integration Test
-        id: integrationTest
+      - name: Flink Integration Test Java8
+        id: integrationTestJava8
         run: |
-          ./gradlew --rerun-tasks -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**"
-          ./gradlew --rerun-tasks -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**"
+          ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=8 :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**"
+          ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=8 :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**"
+
+      - uses: actions/setup-java@v4
+        if: ${{ steps.integrationTestJava8.outcome == 'success' }}
+        with:
+          java-version: '11'
+          distribution: 'temurin'
+          cache: 'gradle'
+
+      - name: Flink Integration Test Java11
+        if: ${{ steps.integrationTestJava8.outcome == 'success' }}
+        id: integrationTestJava11
+        run: |
+          ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=11 :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**"
+          ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=11 :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**"
+
+      - uses: actions/setup-java@v4
+        if: ${{ steps.integrationTestJava11.outcome == 'success' }}
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+          cache: 'gradle'
+
+      - name: Flink Integration Test Java17
+        if: ${{ steps.integrationTestJava11.outcome == 'success' }}
+        id: integrationTestJava17
+        run: |
+          ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=17 :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**"
+          ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=17 :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**"
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3
-        if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
+        if: ${{ (failure() && steps.integrationTestJava17.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
         with:
           name: flink-connector-integrate-test-reports-${{ matrix.java-version }}
           path: |

diff --git a/.github/workflows/frontend-integration-test.yml b/.github/workflows/frontend-integration-test.yml
@@ -65,10 +65,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java-version }}
           distribution: 'temurin'
+          cache: 'gradle'
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
@@ -79,7 +80,6 @@ jobs:
 
       - name: Package Gravitino
         run: |
-          ./gradlew build -x test -PjdkVersion=${{ matrix.java-version }}
           ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
 
       - name: Setup debug Github Action
@@ -93,8 +93,8 @@ jobs:
       - name: Frontend Integration Test
         id: integrationTest
         run: |
-          ./gradlew --rerun-tasks -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} :integration-test:test --tests "com.datastrato.gravitino.integration.test.web.ui.**"
-          ./gradlew --rerun-tasks -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} :integration-test:test --tests "com.datastrato.gravitino.integration.test.web.ui.**"
+          ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} :integration-test:test --tests "com.datastrato.gravitino.integration.test.web.ui.**"
+          ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} :integration-test:test --tests "com.datastrato.gravitino.integration.test.web.ui.**"
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3

diff --git a/.github/workflows/python-integration-test.yml b/.github/workflows/python-integration-test.yml
@@ -49,10 +49,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java-version }}
           distribution: 'temurin'
+          cache: 'gradle'
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2

diff --git a/.github/workflows/spark-integration-test.yml b/.github/workflows/spark-integration-test.yml
@@ -56,18 +56,19 @@ jobs:
     strategy:
       matrix:
         architecture: [linux/amd64]
-        java-version: [ 8, 11, 17 ]
+        java-version: [ 8 ]
         scala-version: [ 2.12 ]
         test-mode: [ embedded, deploy ]
     env:
       PLATFORM: ${{ matrix.architecture }}
     steps:
       - uses: actions/checkout@v3
 
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java-version }}
           distribution: 'temurin'
+          cache: 'gradle'
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
@@ -93,10 +94,10 @@ jobs:
         id: integrationTest
         run: |
           if [ "${{ matrix.scala-version }}" == "2.12" ];then 
-            ./gradlew --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.3:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
+            ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.3:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
           fi
-          ./gradlew --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.4:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
-          ./gradlew --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.5:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
+          ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.4:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
+          ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.5:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3