diff --git a/.github/workflows/backend-integration-action.yml b/.github/workflows/backend-integration-action.yml new file mode 100644 index 00000000000..fb4991904ef --- /dev/null +++ b/.github/workflows/backend-integration-action.yml @@ -0,0 +1,80 @@ +name: Backend Integration Action + +# run backend integration test +on: + workflow_call: + inputs: + architecture: + required: true + description: 'Architecture of the platform' + type: string + java-version: + required: true + description: 'Java version' + type: string + backend: + required: true + description: 'Backend storage for Gravitino' + type: string + test-mode: + required: true + description: 'run on embedded or deploy mode' + type: string + +jobs: + start-runner: + name: ${{ inputs.architecture }}-java${{ inputs.java-version }}-${{ inputs.test-mode }}-${{ inputs.backend }} + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + PLATFORM: ${{ inputs.architecture }} + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.java-version }} + distribution: 'temurin' + cache: 'gradle' + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Check required command + run: | + dev/ci/check_commands.sh + + - name: Package Gravitino + if: ${{ matrix.test-mode == 'deploy' }} + run: | + ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} + + - name: Free up disk space + run: | + dev/ci/util_free_space.sh + + - name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}) + id: integrationTest + run: > + ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipDockerTests=false + -x :web:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:test -x :spark-connector:spark-common:test + -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test + -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test + -x :authorizations:authorization-ranger:test -x :web:integration-test:test -x :trino-connector:integration-test:test -x :trino-connector:trino-connector:test + + - name: Upload integrate tests reports + uses: actions/upload-artifact@v3 + if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} + with: + name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }} + path: | + build/reports + iceberg/iceberg-rest-server/build/*.log + integration-test/build/*.log + integration-test/build/*.tar + integration-test/build/trino-ci-container-log + distribution/package/logs/*.out + distribution/package/logs/*.log + catalogs/**/*.log + catalogs/**/*.tar + distribution/**/*.log \ No newline at end of file diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml index 7dfbe5b902d..2f51e9a3fb0 100644 --- a/.github/workflows/backend-integration-test.yml +++ b/.github/workflows/backend-integration-test.yml @@ -2,7 +2,6 @@ name: Backend Integration Test # Controls when the workflow will run on: - # Triggers the workflow on push or pull request events but only for the "main" branch push: branches: [ "main", "branch-*" ] pull_request: @@ -24,37 +23,14 @@ jobs: source_changes: - .github/** - api/** - - bin/** - - catalogs/** - - clients/client-java/** - - clients/client-java-runtime/** - - clients/filesystem-hadoop3/** - - common/** - - conf/** - - core/** - - dev/** - - gradle/** - - iceberg/** - - integration-test-common/** - - meta/** - - scripts/** - - server/** - - server-common/** - - build.gradle.kts - - gradle.properties - - gradlew - - setting.gradle.kts outputs: source_changes: ${{ steps.filter.outputs.source_changes }} BackendIT-on-push: needs: changes if: (github.event_name == 'push' && needs.changes.outputs.source_changes == 'true') - runs-on: ubuntu-latest - timeout-minutes: 90 strategy: matrix: - # Integration test for AMD64 architecture architecture: [linux/amd64] java-version: [ 8, 11, 17 ] backend: [ h2, mysql, postgresql ] @@ -66,72 +42,22 @@ jobs: backend: 'postgresql' - test-mode: 'deploy' backend: 'h2' - - env: - PLATFORM: ${{ matrix.architecture }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - cache: 'gradle' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Check required command - run: | - dev/ci/check_commands.sh - - - name: Package Gravitino - if : ${{ matrix.test-mode == 'deploy' }} - run: | - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - - name: Free up disk space - run: | - dev/ci/util_free_space.sh - - - name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}) - id: integrationTest - run: > - ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipDockerTests=false - -x :web:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:test -x :spark-connector:spark-common:test - -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test - -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test - -x :authorizations:authorization-ranger:test -x :web:integration-test:test -x :trino-connector:integration-test:test -x :trino-connector:trino-connector:test - - - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 - if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} - with: - name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }} - path: | - build/reports - iceberg/iceberg-rest-server/build/*.log - integration-test/build/*.log - integration-test/build/*.tar - integration-test/build/trino-ci-container-log - distribution/package/logs/*.out - distribution/package/logs/*.log - catalogs/**/*.log - catalogs/**/*.tar - distribution/**/*.log + uses: ./.github/workflows/backend-integration-action.yml + with: + architecture: ${{ matrix.architecture }} + java-version: ${{ matrix.java-version }} + backend: ${{ matrix.backend }} + test-mode: ${{ matrix.test-mode }} BackendIT-on-pr: needs: changes if: (github.event_name == 'pull_request' && needs.changes.outputs.source_changes == 'true') - runs-on: ubuntu-latest - timeout-minutes: 90 strategy: matrix: - # Integration test for AMD64 architecture architecture: [ linux/amd64 ] java-version: [ 17 ] - test-mode: [ embedded, deploy ] backend: [ h2, mysql, postgresql ] + test-mode: [ embedded, deploy ] exclude: - test-mode: 'embedded' backend: 'mysql' @@ -139,53 +65,9 @@ jobs: backend: 'postgresql' - test-mode: 'deploy' backend: 'h2' - - env: - PLATFORM: ${{ matrix.architecture }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - cache: 'gradle' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Check required command - run: | - dev/ci/check_commands.sh - - - name: Package Gravitino - if: ${{ matrix.test-mode == 'deploy' }} - run: | - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - - name: Free up disk space - run: | - dev/ci/util_free_space.sh - - - name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}) - id: integrationTest - run: > - ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipDockerTests=false - -x :web:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:test -x :spark-connector:spark-common:test - -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test - -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test - -x :authorizations:authorization-ranger:test -x :web:integration-test:test -x :trino-connector:integration-test:test -x :trino-connector:test - - - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 - if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} - with: - name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }} - path: | - build/reports - iceberg/iceberg-rest-server/build/*.log - distribution/package/logs/*.out - distribution/package/logs/*.log - catalogs/**/*.log - catalogs/**/*.tar - distribution/**/*.log \ No newline at end of file + uses: ./.github/workflows/backend-integration-action.yml + with: + architecture: ${{ matrix.architecture }} + java-version: ${{ matrix.java-version }} + backend: ${{ matrix.backend }} + test-mode: ${{ matrix.test-mode }} \ No newline at end of file