[apache#3837] improvement(CI): Optimize job of CI (apache#3838)

### What changes were proposed in this pull request? - use gradle cache - reuse intermediate compilation results - remove unnecessary command ### Why are the changes needed? Fix: apache#3837 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? CI passed
zhoukangcn · Jun 14, 2024 · 81e373d · 81e373d
1 parent 0653004
commit 81e373d
Show file tree

Hide file tree

Showing 8 changed files with 27 additions and 36 deletions.
diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
@@ -67,10 +67,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java-version }}
           distribution: 'temurin'
+          cache: 'gradle'
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
@@ -95,7 +96,7 @@ jobs:
       - name: Backend Integration Test
         id: integrationTest
         run: >
-          ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -P${{ matrix.backend }} -PskipWebITs 
+          ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -P${{ matrix.backend }} -PskipWebITs 
           -x :web:test -x :clients:client-python:test -x :flink-connector:test -x :spark-connector:test -x :spark-connector:spark-common:test 
           -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test 
           -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -59,10 +59,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           java-version: 8
           distribution: 'temurin'
+          cache: 'gradle'
 
       - name: Build with Gradle
         run: ./gradlew build -x test -PjdkVersion=8
@@ -81,10 +82,11 @@ jobs:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v3
 
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java-version }}
           distribution: 'temurin'
+          cache: 'gradle'
 
       - name: Test publish to local
         run: ./gradlew publishToMavenLocal -x test -PjdkVersion=${{ matrix.java-version }}
@@ -94,7 +96,7 @@ jobs:
           dev/ci/util_free_space.sh
 
       - name: Build with Gradle
-        run: ./gradlew build -PskipITs -PjdkVersion=${{ matrix.java-version }}
+        run: ./gradlew build -PskipITs -PjdkVersion=${{ matrix.java-version }} -x :clients:client-python:build
 
       - name: Upload unit tests report
         uses: actions/upload-artifact@v3

diff --git a/.github/workflows/cron-integration-test.yml b/.github/workflows/cron-integration-test.yml
@@ -62,17 +62,18 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java-version }}
           distribution: 'temurin'
+          cache: 'gradle'
+
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
 
       - name: Package Gravitino
         run: |
-          ./gradlew build -x test -PjdkVersion=${{ matrix.java-version }}
           ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
 
       - name: Setup debug Github Action
@@ -86,7 +87,7 @@ jobs:
       - name: Integration Test
         id: integrationTest
         run: |
-          ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }}
+          ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }}
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3

diff --git a/.github/workflows/flink-integration-test.yml b/.github/workflows/flink-integration-test.yml
@@ -62,10 +62,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java-version }}
           distribution: 'temurin'
+          cache: 'gradle'
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
@@ -76,7 +77,6 @@ jobs:
 
       - name: Package Gravitino
         run: |
-          ./gradlew build -x test -PjdkVersion=${{ matrix.java-version }}
           ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
 
       - name: Setup debug Github Action
@@ -90,8 +90,8 @@ jobs:
       - name: Flink Integration Test
         id: integrationTest
         run: |
-          ./gradlew --rerun-tasks -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**"
-          ./gradlew --rerun-tasks -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**"
+          ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**"
+          ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} :flink-connector:test --tests "com.datastrato.gravitino.flink.connector.integration.test.**"
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3

diff --git a/.github/workflows/frontend-integration-test.yml b/.github/workflows/frontend-integration-test.yml
@@ -65,10 +65,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java-version }}
           distribution: 'temurin'
+          cache: 'gradle'
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
@@ -79,7 +80,6 @@ jobs:
 
       - name: Package Gravitino
         run: |
-          ./gradlew build -x test -PjdkVersion=${{ matrix.java-version }}
           ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
 
       - name: Setup debug Github Action
@@ -93,8 +93,8 @@ jobs:
       - name: Frontend Integration Test
         id: integrationTest
         run: |
-          ./gradlew --rerun-tasks -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} :integration-test:test --tests "com.datastrato.gravitino.integration.test.web.ui.**"
-          ./gradlew --rerun-tasks -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} :integration-test:test --tests "com.datastrato.gravitino.integration.test.web.ui.**"
+          ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} :integration-test:test --tests "com.datastrato.gravitino.integration.test.web.ui.**"
+          ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} :integration-test:test --tests "com.datastrato.gravitino.integration.test.web.ui.**"
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3

diff --git a/.github/workflows/python-integration-test.yml b/.github/workflows/python-integration-test.yml
@@ -49,10 +49,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java-version }}
           distribution: 'temurin'
+          cache: 'gradle'
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2

diff --git a/.github/workflows/spark-integration-test.yml b/.github/workflows/spark-integration-test.yml
@@ -64,10 +64,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java-version }}
           distribution: 'temurin'
+          cache: 'gradle'
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
@@ -93,10 +94,10 @@ jobs:
         id: integrationTest
         run: |
           if [ "${{ matrix.scala-version }}" == "2.12" ];then 
-            ./gradlew --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.3:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
+            ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.3:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
           fi
-          ./gradlew --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.4:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
-          ./gradlew --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.5:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
+          ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.4:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
+          ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.5:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**"
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3

diff --git a/dev/ci/util_free_space.sh b/dev/ci/util_free_space.sh
@@ -23,12 +23,6 @@ set -eux
 
 if [ "${GITHUB_ACTIONS}" = "true" ]; then
   df -h
-  echo "::group::/usr/local/*"
-  du -hsc /usr/local/*
-  echo "::endgroup::"
-  echo "::group::/usr/local/bin/*"
-  du -hsc /usr/local/bin/*
-  echo "::endgroup::"
   # ~1GB (From 1.2GB to 214MB)
   sudo rm -rf \
     /usr/local/bin/aliyun \
@@ -45,17 +39,8 @@ if [ "${GITHUB_ACTIONS}" = "true" ]; then
     /usr/local/bin/pulumi* \
     /usr/local/bin/stack \
     /usr/local/bin/terraform || :
-  echo "::group::/usr/local/share/*"
-  du -hsc /usr/local/share/*
-  echo "::endgroup::"
   # 1.3GB
   sudo rm -rf /usr/local/share/powershell || :
-  echo "::group::/opt/*"
-  du -hsc /opt/*
-  echo "::endgroup::"
-  echo "::group::/opt/hostedtoolcache/*"
-  du -hsc /opt/hostedtoolcache/*
-  echo "::endgroup::"
   # 5.3GB
   sudo rm -rf /opt/hostedtoolcache/CodeQL || :
   # 1.4GB