Skip to content

Commit

Permalink
minimize changes
Browse files Browse the repository at this point in the history
  • Loading branch information
mchades committed Sep 1, 2024
1 parent becb0d9 commit 64af124
Show file tree
Hide file tree
Showing 8 changed files with 177 additions and 37 deletions.
13 changes: 12 additions & 1 deletion .github/workflows/access-control-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,16 +22,27 @@ jobs:
with:
filters: |
source_changes:
- .github/**
- api/**
- authorizations/**
- bin/**
- catalogs/**
- clients/client-java/**
- clients/client-java-runtime/**
- common/**
- conf/**
- core/**
- dev/**
- gradle/**
- iceberg/**
- integration-test-common/**
- meta/**
- scripts/**
- server/**
- server-common/**
- build.gradle.kts
- gradle.properties
- gradlew
- setting.gradle.kts
outputs:
source_changes: ${{ steps.filter.outputs.source_changes }}

Expand Down
10 changes: 3 additions & 7 deletions .github/workflows/backend-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,26 +24,22 @@ jobs:
source_changes:
- .github/**
- api/**
- authorizations/**
- bin/**
- catalogs/**
- clients/client-java/**
- clients/client-java-runtime/**
- clients/filesystem-hadoop3/**
- clients/filesystem-hadoop3-runtime/**
- common/**
- conf/**
- core/**
- dev/**
- gradle/**
- integration-test/**
- integration-test-common/**
- iceberg/**
- integration-test-common/**
- meta/**
- scripts/**
- server/**
- server-common/**
- trino-connector/**
- web/**
- docs/open-api/**
- build.gradle.kts
- gradle.properties
- gradlew
Expand Down
14 changes: 10 additions & 4 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ jobs:
source_changes:
- .github/**
- api/**
- authorizations/**
- bin/**
- catalogs/**
- clients/client-java/**
Expand All @@ -35,22 +36,27 @@ jobs:
- conf/**
- core/**
- dev/**
- docs/open-api/**
- flink-connector/**
- gradle/**
- integration-test/**
- iceberg/**
- integration-test-common/**
- meta/**
- scripts/**
- server/**
- server-common/**
- spark-connector/**
- flink-connector/**
- trino-connector/**
- web/**
- docs/open-api/**
- build.gradle.kts
- gradle.properties
- gradlew
- setting.gradle.kts
spark_connector_changes:
- spark-connector/**
outputs:
source_changes: ${{ steps.filter.outputs.source_changes }}
spark_connector_changes: ${{ steps.filter.outputs.spark_connector_changes }}

compile-check:
runs-on: ubuntu-latest
Expand All @@ -73,7 +79,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 30
needs: changes
if: needs.changes.outputs.source_changes == 'true'
if: needs.changes.outputs.spark_connector_changes == 'true'
steps:
- uses: actions/checkout@v3

Expand Down
73 changes: 66 additions & 7 deletions .github/workflows/flink-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,37 +24,38 @@ jobs:
source_changes:
- .github/**
- api/**
- authorizations/**
- bin/**
- catalogs/**
- clients/client-java/**
- clients/client-java-runtime/**
- clients/filesystem-hadoop3/**
- clients/filesystem-hadoop3-runtime/**
- common/**
- conf/**
- core/**
- dev/**
- flink-connector/**
- gradle/**
- iceberg/**
- integration-test-common/**
- meta/**
- scripts/**
- server/**
- server-common/**
- flink-connector/**
- docs/open-api/**
- build.gradle.kts
- gradle.properties
- gradlew
- setting.gradle.kts
outputs:
source_changes: ${{ steps.filter.outputs.source_changes }}

# Integration test for AMD64 architecture
test-amd64-arch:
test-on-push:
needs: changes
if: needs.changes.outputs.source_changes == 'true'
if: (github.event_name == 'push' && needs.changes.outputs.source_changes == 'true')
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
matrix:
# Integration test for AMD64 architecture
architecture: [linux/amd64]
java-version: [ 8, 11, 17 ]
env:
Expand Down Expand Up @@ -85,6 +86,7 @@ jobs:
- name: Flink Integration Test
id: integrationTest
# run embedded mode and deploy mode integration tests
run: |
./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**"
./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**"
Expand All @@ -102,3 +104,60 @@ jobs:
distribution/package/logs/gravitino-server.log
catalogs/**/*.log
catalogs/**/*.tar
test-on-pr:
needs: changes
if: (github.event_name == 'pull_request' && needs.changes.outputs.source_changes == 'true')
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
matrix:
# Integration test for AMD64 architecture
architecture: [linux/amd64]
java-version: [ 17 ]
env:
PLATFORM: ${{ matrix.architecture }}
steps:
- uses: actions/checkout@v3

- uses: actions/setup-java@v4
with:
java-version: ${{ matrix.java-version }}
distribution: 'temurin'
cache: 'gradle'

- name: Set up QEMU
uses: docker/setup-qemu-action@v2

- name: Check required command
run: |
dev/ci/check_commands.sh
- name: Package Gravitino
run: |
./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
- name: Free up disk space
run: |
dev/ci/util_free_space.sh
- name: Flink Integration Test
id: integrationTest
# run embedded mode and deploy mode integration tests
run: |
./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**"
./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**"
- name: Upload integrate tests reports
uses: actions/upload-artifact@v3
if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
with:
name: flink-connector-integrate-test-reports-${{ matrix.java-version }}
path: |
build/reports
flink-connector/flink/build/*.log
flink-connector/flink/build/*.tar
distribution/package/logs/gravitino-server.out
distribution/package/logs/gravitino-server.log
catalogs/**/*.log
catalogs/**/*.tar
12 changes: 4 additions & 8 deletions .github/workflows/frontend-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,25 +24,21 @@ jobs:
source_changes:
- .github/**
- api/**
- authorizations/**
- bin/**
- catalogs/**
- clients/client-java/**
- clients/client-java-runtime/**
- clients/filesystem-hadoop3/**
- clients/filesystem-hadoop3-runtime/**
- common/**
- conf/**
- core/**
- dev/**
- gradle/**
- integration-test/**
- iceberg/**
- integration-test-common/**
- meta/**
- scripts/**
- server/**
- server-common/**
- spark-connector/**
- trino-connector/**
- web/**
- docs/open-api/**
- build.gradle.kts
- gradle.properties
- gradlew
Expand Down
12 changes: 11 additions & 1 deletion .github/workflows/python-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,24 @@ jobs:
source_changes:
- .github/**
- api/**
- catalogs/catalog-hadoop/**
- authorizations/**
- bin/**
- catalogs/**
- clients/client-python/**
- common/**
- conf/**
- core/**
- dev/**
- gradle/**
- iceberg/**
- meta/**
- scripts/**
- server/**
- server-common/**
- build.gradle.kts
- gradle.properties
- gradlew
- setting.gradle.kts
outputs:
source_changes: ${{ steps.filter.outputs.source_changes }}

Expand Down
73 changes: 67 additions & 6 deletions .github/workflows/spark-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,38 +24,38 @@ jobs:
source_changes:
- .github/**
- api/**
- authorizations/**
- bin/**
- catalogs/**
- clients/client-java/**
- clients/client-java-runtime/**
- clients/filesystem-hadoop3/**
- clients/filesystem-hadoop3-runtime/**
- common/**
- conf/**
- core/**
- dev/**
- gradle/**
- iceberg/**
- integration-test-common/**
- meta/**
- scripts/**
- server/**
- server-common/**
- spark-connector/**
- docs/open-api/**
- build.gradle.kts
- gradle.properties
- gradlew
- setting.gradle.kts
outputs:
source_changes: ${{ steps.filter.outputs.source_changes }}

# Integration test for AMD64 architecture
test-amd64-arch:
test-on-push:
needs: changes
if: needs.changes.outputs.source_changes == 'true'
if: (github.event_name == 'push' && needs.changes.outputs.source_changes == 'true')
runs-on: ubuntu-latest
timeout-minutes: 90
strategy:
matrix:
# Integration test for AMD64 architecture
architecture: [linux/amd64]
java-version: [ 8, 11, 17 ]
scala-version: [ 2.12 ]
Expand Down Expand Up @@ -108,3 +108,64 @@ jobs:
spark-connector/v3.5/spark/build/spark-3.5-integration-test.log
distribution/package/logs/*.out
distribution/package/logs/*.log
test-on-pr:
needs: changes
if: (github.event_name == 'pull_request' && needs.changes.outputs.source_changes == 'true')
runs-on: ubuntu-latest
timeout-minutes: 90
strategy:
matrix:
# Integration test for AMD64 architecture
architecture: [linux/amd64]
java-version: [ 17 ]
scala-version: [ 2.12 ]
test-mode: [ embedded, deploy ]
env:
PLATFORM: ${{ matrix.architecture }}
steps:
- uses: actions/checkout@v3

- uses: actions/setup-java@v4
with:
java-version: ${{ matrix.java-version }}
distribution: 'temurin'
cache: 'gradle'

- name: Set up QEMU
uses: docker/setup-qemu-action@v2

- name: Check required command
run: |
dev/ci/check_commands.sh
- name: Package Gravitino
if : ${{ matrix.test-mode == 'deploy' }}
run: |
./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
- name: Free up disk space
run: |
dev/ci/util_free_space.sh
- name: Spark Integration Test
id: integrationTest
run: |
if [ "${{ matrix.scala-version }}" == "2.12" ];then
./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.3:test --tests "org.apache.gravitino.spark.connector.integration.test.**"
fi
./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.4:test --tests "org.apache.gravitino.spark.connector.integration.test.**"
./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.5:test --tests "org.apache.gravitino.spark.connector.integration.test.**"
- name: Upload integrate tests reports
uses: actions/upload-artifact@v3
if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
with:
name: spark-connector-integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}
path: |
build/reports
spark-connector/v3.3/spark/build/spark-3.3-integration-test.log
spark-connector/v3.4/spark/build/spark-3.4-integration-test.log
spark-connector/v3.5/spark/build/spark-3.5-integration-test.log
distribution/package/logs/*.out
distribution/package/logs/*.log
Loading

0 comments on commit 64af124

Please sign in to comment.