spark-smoke-test.yml

name: spark smoke test
on:
  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:
  push:
    branches:
      - master
    paths:
      - "metadata_models/**"
      - "metadata-integration/java/datahub-client/**"
      - "metadata-integration/java/spark-lineage"
      - ".github/workflows/spark-smoke-test.yml"
  pull_request:
    branches:
      - "**"
    paths:
      - "metadata_models/**"
      - "metadata-integration/java/datahub-client/**"
      - "metadata-integration/java/spark-lineage"
      - ".github/workflows/spark-smoke-test.yml"
  release:
    types: [published]

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  spark-smoke-test:
    runs-on: ubuntu-latest
    steps:
      - uses: acryldata/sane-checkout-action@v3
      - name: Set up JDK 17
        uses: actions/setup-java@v3
        with:
          distribution: "zulu"
          java-version: 17
      - uses: gradle/actions/setup-gradle@v3
      - uses: actions/setup-python@v4
        with:
          python-version: "3.10"
          cache: "pip"
      - name: Install dependencies
        run: ./metadata-ingestion/scripts/install_deps.sh
      - name: Disk Check
        run: df -h . && docker images
      - name: Free up disk space
        run: |
          sudo apt-get remove 'dotnet-*' azure-cli || true
          sudo rm -rf /usr/local/lib/android/ || true
          sudo docker image prune -a -f || true
      - name: Disk Check
        run: df -h . && docker images
      - name: Smoke test
        run: |
          ./gradlew :metadata-integration:java:spark-lineage:integrationTest \
             -x :datahub-web-react:yarnTest \
             -x :datahub-web-react:yarnLint \
             -x :datahub-web-react:yarnGenerate \
             -x :datahub-web-react:yarnInstall \
             -x :datahub-web-react:yarnBuild \
             -x :datahub-web-react:distZip \
             -x :datahub-web-react:jar
      - name: store logs
        if: failure()
        run: |
          docker ps -a
          docker logs datahub-gms >& gms-${{ matrix.test_strategy }}.log || true
          docker logs datahub-actions >& actions-${{ matrix.test_strategy }}.log || true
          docker logs broker >& broker-${{ matrix.test_strategy }}.log || true
          docker logs mysql >& mysql-${{ matrix.test_strategy }}.log || true
          docker logs elasticsearch >& elasticsearch-${{ matrix.test_strategy }}.log || true
          docker logs datahub-frontend-react >& frontend-${{ matrix.test_strategy }}.log || true
      - name: Upload logs
        uses: actions/upload-artifact@v3
        if: failure()
        with:
          name: docker logs
          path: |
            "**/build/container-logs/*.log"
            "*.log"
      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: Test Results (smoke tests)
          path: |
            **/build/reports/tests/test/**
            **/build/test-results/test/**
            **/junit.*.xml
            !**/binary/**