diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml index 1c4e60adc3f..3f582e084c5 100644 --- a/.github/workflows/backend-integration-test.yml +++ b/.github/workflows/backend-integration-test.yml @@ -50,7 +50,8 @@ jobs: outputs: source_changes: ${{ steps.filter.outputs.source_changes }} - catalog-test: + # for catalog integration test that requires long time to run + slow-catalog-test: needs: changes if: needs.changes.outputs.source_changes == 'true' runs-on: ubuntu-latest @@ -59,7 +60,7 @@ jobs: matrix: architecture: [linux/amd64] java-version: [ 8, 11, 17 ] - catalog: [ jdbc-doris, jdbc-mysql, jdbc-postgresql, lakehouse-iceberg, hadoop, hive, kafka ] + catalog: [ jdbc-doris, lakehouse-iceberg, hive ] env: PLATFORM: ${{ matrix.architecture }} steps: @@ -96,8 +97,9 @@ jobs: run: | for testMode in "embedded" "deploy" do - echo "Run catalog test in ${testMode} mode" + echo "Run catalog test in ${testMode} mode with kvBackend" ./gradlew :catalogs:catalog-${{ matrix.catalog }}:test -PskipTests -PtestMode=${testMode} -PjdkVersion=${{ matrix.java-version }} -PkvBackend + echo "Run catalog test in ${testMode} mode with jdbcBackend" ./gradlew :catalogs:catalog-${{ matrix.catalog }}:test -PskipTests -PtestMode=${testMode} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend done @@ -105,7 +107,7 @@ jobs: uses: actions/upload-artifact@v3 if: ${{ (failure() && steps.catalogTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} with: - name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }} + name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.catalog }} path: | build/reports integration-test/build/*.log @@ -117,3 +119,110 @@ jobs: catalogs/**/*.log catalogs/**/*.tar distribution/**/*.log + + # for catalog integration test that requires short time to run + fast-catalog-test: + needs: changes + if: needs.changes.outputs.source_changes == 'true' + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + matrix: + architecture: [linux/amd64] + catalog: [ jdbc-mysql, jdbc-postgresql, hadoop, kafka ] + env: + PLATFORM: ${{ matrix.architecture }} + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-java@v4 + with: + java-version: '8' + distribution: 'temurin' + cache: 'gradle' + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Check required command + run: | + dev/ci/check_commands.sh + + - name: Package Gravitino + run: | + ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} + + - name: Setup debug Github Action + if: ${{ contains(github.event.pull_request.labels.*.name, 'debug action') }} + uses: csexton/debugger-action@master + + - name: Free up disk space + if : ${{ matrix.catalog == 'jdbc-doris' }} + run: | + dev/ci/util_free_space.sh + + - name: Test catalog Java8 + id: catalogTestJava8 + run: | + for testMode in "embedded" "deploy" + do + echo "Run catalog test in ${testMode} mode with kvBackend" + ./gradlew :catalogs:catalog-${{ matrix.catalog }}:test -PskipTests -PtestMode=${testMode} -PjdkVersion=8 -PkvBackend + echo "Run catalog test in ${testMode} mode with jdbcBackend" + ./gradlew :catalogs:catalog-${{ matrix.catalog }}:test -PskipTests -PtestMode=${testMode} -PjdkVersion=8 -PjdbcBackend + done + + - uses: actions/setup-java@v4 + if: ${{ steps.catalogTestJava8.outcome == 'success' }} + with: + java-version: '11' + distribution: 'temurin' + cache: 'gradle' + + - name: Test catalog Java11 + if: ${{ steps.catalogTestJava8.outcome == 'success' }} + id: catalogTestJava11 + run: | + for testMode in "embedded" "deploy" + do + echo "Run catalog test in ${testMode} mode with kvBackend" + ./gradlew :catalogs:catalog-${{ matrix.catalog }}:test -PskipTests -PtestMode=${testMode} -PjdkVersion=11 -PkvBackend + echo "Run catalog test in ${testMode} mode with jdbcBackend" + ./gradlew :catalogs:catalog-${{ matrix.catalog }}:test -PskipTests -PtestMode=${testMode} -PjdkVersion=11 -PjdbcBackend + done + + - uses: actions/setup-java@v4 + if: ${{ steps.catalogTestJava11.outcome == 'success' }} + with: + java-version: '17' + distribution: 'temurin' + cache: 'gradle' + + - name: Test catalog Java17 + if: ${{ steps.catalogTestJava11.outcome == 'success' }} + id: catalogTestJava17 + run: | + for testMode in "embedded" "deploy" + do + echo "Run catalog test in ${testMode} mode with kvBackend" + ./gradlew :catalogs:catalog-${{ matrix.catalog }}:test -PskipTests -PtestMode=${testMode} -PjdkVersion=17 -PkvBackend + echo "Run catalog test in ${testMode} mode with jdbcBackend" + ./gradlew :catalogs:catalog-${{ matrix.catalog }}:test -PskipTests -PtestMode=${testMode} -PjdkVersion=17 -PjdbcBackend + done + + - name: Upload integrate tests reports + uses: actions/upload-artifact@v3 + if: ${{ (failure() && steps.catalogTestJava17.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} + with: + name: integrate-test-reports-${{ matrix.catalog }} + path: | + build/reports + integration-test/build/*.log + integration-test/build/*.tar + integration-test/build/trino-ci-container-log/hive/*.* + integration-test/build/trino-ci-container-log/hdfs/*.* + distribution/package/logs/gravitino-server.out + distribution/package/logs/gravitino-server.log + catalogs/**/*.log + catalogs/**/*.tar + distribution/**/*.log \ No newline at end of file