From 64af1244e14098554bb3268709aa8fe245d040ee Mon Sep 17 00:00:00 2001 From: mchades Date: Sat, 31 Aug 2024 14:12:31 +0800 Subject: [PATCH] minimize changes --- .../access-control-integration-test.yml | 13 +++- .../workflows/backend-integration-test.yml | 10 +-- .github/workflows/build.yml | 14 +++- .github/workflows/flink-integration-test.yml | 73 +++++++++++++++++-- .../workflows/frontend-integration-test.yml | 12 +-- .github/workflows/python-integration-test.yml | 12 ++- .github/workflows/spark-integration-test.yml | 73 +++++++++++++++++-- .github/workflows/trino-integration-test.yml | 7 +- 8 files changed, 177 insertions(+), 37 deletions(-) diff --git a/.github/workflows/access-control-integration-test.yml b/.github/workflows/access-control-integration-test.yml index 9f7d5f2bdc4..6435b2dc8e5 100644 --- a/.github/workflows/access-control-integration-test.yml +++ b/.github/workflows/access-control-integration-test.yml @@ -22,16 +22,27 @@ jobs: with: filters: | source_changes: + - .github/** - api/** - - authorizations/** + - bin/** - catalogs/** - clients/client-java/** - clients/client-java-runtime/** - common/** + - conf/** - core/** + - dev/** + - gradle/** + - iceberg/** - integration-test-common/** + - meta/** + - scripts/** - server/** - server-common/** + - build.gradle.kts + - gradle.properties + - gradlew + - setting.gradle.kts outputs: source_changes: ${{ steps.filter.outputs.source_changes }} diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml index 0539aec0a01..a072a4f9f52 100644 --- a/.github/workflows/backend-integration-test.yml +++ b/.github/workflows/backend-integration-test.yml @@ -24,26 +24,22 @@ jobs: source_changes: - .github/** - api/** + - authorizations/** - bin/** - catalogs/** - clients/client-java/** - clients/client-java-runtime/** - - clients/filesystem-hadoop3/** - - clients/filesystem-hadoop3-runtime/** - common/** - conf/** - core/** - dev/** - gradle/** - - integration-test/** - - integration-test-common/** - iceberg/** + - integration-test-common/** - meta/** + - scripts/** - server/** - server-common/** - - trino-connector/** - - web/** - - docs/open-api/** - build.gradle.kts - gradle.properties - gradlew diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0cf17683a0d..d991e30525a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,6 +25,7 @@ jobs: source_changes: - .github/** - api/** + - authorizations/** - bin/** - catalogs/** - clients/client-java/** @@ -35,22 +36,27 @@ jobs: - conf/** - core/** - dev/** + - docs/open-api/** + - flink-connector/** - gradle/** - - integration-test/** + - iceberg/** + - integration-test-common/** - meta/** + - scripts/** - server/** - server-common/** - spark-connector/** - - flink-connector/** - trino-connector/** - web/** - - docs/open-api/** - build.gradle.kts - gradle.properties - gradlew - setting.gradle.kts + spark_connector_changes: + - spark-connector/** outputs: source_changes: ${{ steps.filter.outputs.source_changes }} + spark_connector_changes: ${{ steps.filter.outputs.spark_connector_changes }} compile-check: runs-on: ubuntu-latest @@ -73,7 +79,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 needs: changes - if: needs.changes.outputs.source_changes == 'true' + if: needs.changes.outputs.spark_connector_changes == 'true' steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/flink-integration-test.yml b/.github/workflows/flink-integration-test.yml index 6e1227664cb..53b8663da0e 100644 --- a/.github/workflows/flink-integration-test.yml +++ b/.github/workflows/flink-integration-test.yml @@ -24,22 +24,23 @@ jobs: source_changes: - .github/** - api/** + - authorizations/** - bin/** - catalogs/** - clients/client-java/** - clients/client-java-runtime/** - - clients/filesystem-hadoop3/** - - clients/filesystem-hadoop3-runtime/** - common/** - conf/** - core/** - dev/** + - flink-connector/** - gradle/** + - iceberg/** + - integration-test-common/** - meta/** + - scripts/** - server/** - server-common/** - - flink-connector/** - - docs/open-api/** - build.gradle.kts - gradle.properties - gradlew @@ -47,14 +48,14 @@ jobs: outputs: source_changes: ${{ steps.filter.outputs.source_changes }} - # Integration test for AMD64 architecture - test-amd64-arch: + test-on-push: needs: changes - if: needs.changes.outputs.source_changes == 'true' + if: (github.event_name == 'push' && needs.changes.outputs.source_changes == 'true') runs-on: ubuntu-latest timeout-minutes: 30 strategy: matrix: + # Integration test for AMD64 architecture architecture: [linux/amd64] java-version: [ 8, 11, 17 ] env: @@ -85,6 +86,7 @@ jobs: - name: Flink Integration Test id: integrationTest + # run embedded mode and deploy mode integration tests run: | ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**" ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**" @@ -102,3 +104,60 @@ jobs: distribution/package/logs/gravitino-server.log catalogs/**/*.log catalogs/**/*.tar + + test-on-pr: + needs: changes + if: (github.event_name == 'pull_request' && needs.changes.outputs.source_changes == 'true') + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + matrix: + # Integration test for AMD64 architecture + architecture: [linux/amd64] + java-version: [ 17 ] + env: + PLATFORM: ${{ matrix.architecture }} + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.java-version }} + distribution: 'temurin' + cache: 'gradle' + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Check required command + run: | + dev/ci/check_commands.sh + + - name: Package Gravitino + run: | + ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} + + - name: Free up disk space + run: | + dev/ci/util_free_space.sh + + - name: Flink Integration Test + id: integrationTest + # run embedded mode and deploy mode integration tests + run: | + ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**" + ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**" + + - name: Upload integrate tests reports + uses: actions/upload-artifact@v3 + if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} + with: + name: flink-connector-integrate-test-reports-${{ matrix.java-version }} + path: | + build/reports + flink-connector/flink/build/*.log + flink-connector/flink/build/*.tar + distribution/package/logs/gravitino-server.out + distribution/package/logs/gravitino-server.log + catalogs/**/*.log + catalogs/**/*.tar \ No newline at end of file diff --git a/.github/workflows/frontend-integration-test.yml b/.github/workflows/frontend-integration-test.yml index 98af8777db3..f4b3cece83e 100644 --- a/.github/workflows/frontend-integration-test.yml +++ b/.github/workflows/frontend-integration-test.yml @@ -24,25 +24,21 @@ jobs: source_changes: - .github/** - api/** + - authorizations/** - bin/** - catalogs/** - - clients/client-java/** - - clients/client-java-runtime/** - - clients/filesystem-hadoop3/** - - clients/filesystem-hadoop3-runtime/** - common/** - conf/** - core/** - dev/** - gradle/** - - integration-test/** + - iceberg/** + - integration-test-common/** - meta/** + - scripts/** - server/** - server-common/** - - spark-connector/** - - trino-connector/** - web/** - - docs/open-api/** - build.gradle.kts - gradle.properties - gradlew diff --git a/.github/workflows/python-integration-test.yml b/.github/workflows/python-integration-test.yml index 4e9f96bc690..179e529a5b7 100644 --- a/.github/workflows/python-integration-test.yml +++ b/.github/workflows/python-integration-test.yml @@ -24,14 +24,24 @@ jobs: source_changes: - .github/** - api/** - - catalogs/catalog-hadoop/** + - authorizations/** + - bin/** + - catalogs/** - clients/client-python/** - common/** - conf/** - core/** + - dev/** + - gradle/** + - iceberg/** - meta/** + - scripts/** - server/** - server-common/** + - build.gradle.kts + - gradle.properties + - gradlew + - setting.gradle.kts outputs: source_changes: ${{ steps.filter.outputs.source_changes }} diff --git a/.github/workflows/spark-integration-test.yml b/.github/workflows/spark-integration-test.yml index a2f11061cda..2a5b94c1979 100644 --- a/.github/workflows/spark-integration-test.yml +++ b/.github/workflows/spark-integration-test.yml @@ -24,23 +24,23 @@ jobs: source_changes: - .github/** - api/** + - authorizations/** - bin/** - catalogs/** - clients/client-java/** - clients/client-java-runtime/** - - clients/filesystem-hadoop3/** - - clients/filesystem-hadoop3-runtime/** - common/** - conf/** - core/** - dev/** - gradle/** - iceberg/** + - integration-test-common/** - meta/** + - scripts/** - server/** - server-common/** - spark-connector/** - - docs/open-api/** - build.gradle.kts - gradle.properties - gradlew @@ -48,14 +48,14 @@ jobs: outputs: source_changes: ${{ steps.filter.outputs.source_changes }} - # Integration test for AMD64 architecture - test-amd64-arch: + test-on-push: needs: changes - if: needs.changes.outputs.source_changes == 'true' + if: (github.event_name == 'push' && needs.changes.outputs.source_changes == 'true') runs-on: ubuntu-latest timeout-minutes: 90 strategy: matrix: + # Integration test for AMD64 architecture architecture: [linux/amd64] java-version: [ 8, 11, 17 ] scala-version: [ 2.12 ] @@ -108,3 +108,64 @@ jobs: spark-connector/v3.5/spark/build/spark-3.5-integration-test.log distribution/package/logs/*.out distribution/package/logs/*.log + + test-on-pr: + needs: changes + if: (github.event_name == 'pull_request' && needs.changes.outputs.source_changes == 'true') + runs-on: ubuntu-latest + timeout-minutes: 90 + strategy: + matrix: + # Integration test for AMD64 architecture + architecture: [linux/amd64] + java-version: [ 17 ] + scala-version: [ 2.12 ] + test-mode: [ embedded, deploy ] + env: + PLATFORM: ${{ matrix.architecture }} + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.java-version }} + distribution: 'temurin' + cache: 'gradle' + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Check required command + run: | + dev/ci/check_commands.sh + + - name: Package Gravitino + if : ${{ matrix.test-mode == 'deploy' }} + run: | + ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} + + - name: Free up disk space + run: | + dev/ci/util_free_space.sh + + - name: Spark Integration Test + id: integrationTest + run: | + if [ "${{ matrix.scala-version }}" == "2.12" ];then + ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.3:test --tests "org.apache.gravitino.spark.connector.integration.test.**" + fi + ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.4:test --tests "org.apache.gravitino.spark.connector.integration.test.**" + ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.5:test --tests "org.apache.gravitino.spark.connector.integration.test.**" + + - name: Upload integrate tests reports + uses: actions/upload-artifact@v3 + if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} + with: + name: spark-connector-integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }} + path: | + build/reports + spark-connector/v3.3/spark/build/spark-3.3-integration-test.log + spark-connector/v3.4/spark/build/spark-3.4-integration-test.log + spark-connector/v3.5/spark/build/spark-3.5-integration-test.log + distribution/package/logs/*.out + distribution/package/logs/*.log \ No newline at end of file diff --git a/.github/workflows/trino-integration-test.yml b/.github/workflows/trino-integration-test.yml index 6d899ee8cf7..6df205344d8 100644 --- a/.github/workflows/trino-integration-test.yml +++ b/.github/workflows/trino-integration-test.yml @@ -24,22 +24,23 @@ jobs: source_changes: - .github/** - api/** + - authorizations/** - bin/** - catalogs/** - clients/client-java/** - clients/client-java-runtime/** - - clients/filesystem-hadoop3/** - - clients/filesystem-hadoop3-runtime/** - common/** - conf/** - core/** - dev/** - gradle/** + - iceberg/** + - integration-test-common/** - meta/** + - scripts/** - server/** - server-common/** - trino-connector/** - - docs/open-api/** - build.gradle.kts - gradle.properties - gradlew