E2E Tests #7
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Tests | |
| on: | |
| # We'll record runs using Replay.io and their browser on a schedule as an experiment | |
| schedule: | |
| - cron: '0 22 * * 0' | |
| push: | |
| branches: | |
| - "master" | |
| - 'release-**' | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| test-run-id: | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| testRunId: ${{ steps.testRunId.outputs.testRunId }} | |
| steps: | |
| - id: testRunId | |
| run: echo testRunId=$(npx uuid) >> "$GITHUB_OUTPUT" | |
| files-changed: | |
| name: Check which files changed | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 3 | |
| outputs: | |
| e2e_all: ${{ steps.changes.outputs.e2e_all }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Test which files changed | |
| uses: dorny/[email protected] | |
| id: changes | |
| with: | |
| token: ${{ github.token }} | |
| filters: .github/file-paths.yaml | |
| build: | |
| needs: files-changed | |
| if: github.event.pull_request.draft == false && needs.files-changed.outputs.e2e_all == 'true' | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 25 | |
| strategy: | |
| matrix: | |
| edition: [oss, ee] | |
| env: | |
| MB_EDITION: ${{ matrix.edition }} | |
| INTERACTIVE: false | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Prepare front-end environment | |
| uses: ./.github/actions/prepare-frontend | |
| - name: Prepare back-end environment | |
| uses: ./.github/actions/prepare-backend | |
| with: | |
| m2-cache-key: e2e-tests | |
| - name: Build uberjar with ./bin/build.sh | |
| run: ./bin/build.sh | |
| - name: Prepare uberjar artifact | |
| uses: ./.github/actions/prepare-uberjar-artifact | |
| e2e-tests: | |
| needs: [build, files-changed, test-run-id] | |
| if: | | |
| always() && | |
| needs.files-changed.outputs.e2e_all == 'true' && | |
| needs.build.result == 'success' | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 90 | |
| name: e2e-tests-${{ matrix.folder }}${{ matrix.context }}-${{ matrix.edition }} | |
| env: | |
| MB_EDITION: ${{ matrix.edition }} | |
| DISPLAY: "" | |
| QA_DB_ENABLED: true | |
| # Any env starting with `CYPRESS_` will be available to all Cypress tests via `Cypress.env()` | |
| # Example: you can get `CYPRESS_FOO` with `Cypress.env("FOO")` | |
| CYPRESS_ALL_FEATURES_TOKEN: ${{ secrets.ENTERPRISE_TOKEN }} | |
| CYPRESS_NO_FEATURES_TOKEN: ${{ secrets.E2E_STARTER_TOKEN }} | |
| MB_SNOWPLOW_AVAILABLE: true | |
| MB_SNOWPLOW_URL: "http://localhost:9090" # Snowplow micro | |
| ELECTRON_EXTRA_LAUNCH_ARGS: "--remote-debugging-port=40500" # deploysentinel | |
| TZ: US/Pacific # to make node match the instance tz | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| java-version: [11] | |
| edition: [ee] | |
| folder: | |
| - "auditing" | |
| - "actions" | |
| - "admin" | |
| - "binning" | |
| - "collections" | |
| - "custom-column" | |
| - "dashboard" | |
| - "dashboard-filters" | |
| - "embedding" | |
| - "filters" | |
| - "joins" | |
| - "models" | |
| - "native" | |
| - "native-filters" | |
| - "onboarding" | |
| - "organization" | |
| - "permissions" | |
| - "question" | |
| - "sharing" | |
| - "visualizations" | |
| include: | |
| - edition: oss | |
| context: grep | |
| java-version: 11 | |
| services: | |
| maildev: | |
| image: maildev/maildev:2.0.5 | |
| ports: | |
| - "1080:1080" | |
| - "1025:1025" | |
| credentials: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| openldap: | |
| image: osixia/openldap:1.5.0 | |
| ports: | |
| - "389:389" | |
| credentials: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| postgres-sample: | |
| image: metabase/qa-databases:postgres-sample-12 | |
| ports: | |
| - "5404:5432" | |
| credentials: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| mongo-sample: | |
| image: metabase/qa-databases:mongo-sample-4.4 | |
| ports: | |
| - 27004:27017 | |
| credentials: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| mysql-sample: | |
| image: metabase/qa-databases:mysql-sample-8 | |
| ports: | |
| - 3304:3306 | |
| credentials: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Record runs using Deploysentinel except for the release branch | |
| if: ${{ github.ref == 'refs/heads/master' || !(startsWith(github.event.pull_request.base.ref, 'release')) }} | |
| run: | | |
| echo "CYPRESS_DEPLOYSENTINEL_KEY=${{ secrets.CYPRESS_DEPLOYSENTINEL_KEY }}" >> $GITHUB_ENV | |
| - name: Prepare front-end environment | |
| uses: ./.github/actions/prepare-frontend | |
| - name: Prepare JDK ${{ matrix.java-version }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| distribution: "temurin" | |
| - name: Prepare Cypress environment | |
| uses: ./.github/actions/prepare-cypress | |
| - name: Run Snowplow micro | |
| uses: ./.github/actions/run-snowplow-micro | |
| - uses: actions/download-artifact@v3 | |
| name: Retrieve uberjar artifact for ${{ matrix.edition }} | |
| with: | |
| name: metabase-${{ matrix.edition }}-uberjar | |
| - name: Get the version info | |
| run: | | |
| jar xf target/uberjar/metabase.jar version.properties | |
| mv version.properties resources/ | |
| - name: Run OSS-specific Cypress tests | |
| if: matrix.edition == 'oss' && github.event_name != 'schedule' | |
| run: | | |
| yarn run test-cypress-run \ | |
| --env grepTags=@OSS \ | |
| --spec './e2e/test/scenarios/**/*.cy.spec.js' | |
| env: | |
| TERM: xterm | |
| - name: Run EE Cypress tests on ${{ matrix.folder }} | |
| if: matrix.edition == 'ee' && github.event_name != 'schedule' | |
| run: | | |
| yarn run test-cypress-run \ | |
| --env grepTags="-@quarantine" \ | |
| --folder ${{ matrix.folder }} | |
| env: | |
| TERM: xterm | |
| # REPLAY.IO specific jobs | |
| - name: Install Replay.io browser | |
| if: ${{ github.event_name == 'schedule' }} | |
| run: npx @replayio/cypress install | |
| - name: Run OSS-specific Cypress tests using Replay.io browser | |
| if: matrix.edition == 'oss' && github.event_name == 'schedule' | |
| run: | | |
| yarn run test-cypress-run \ | |
| --env grepTags=@OSS \ | |
| --spec './e2e/test/scenarios/**/*.cy.spec.js' \ | |
| --browser "replay-chromium" | |
| env: | |
| TERM: xterm | |
| CYPRESS_REPLAYIO_ENABLED: 1 | |
| RECORD_REPLAY_METADATA_FILE: /tmp/replay-metadata.json | |
| RECORD_REPLAY_METADATA_TEST_RUN_ID: ${{ needs.test-run-id.outputs.testRunId }} | |
| - name: Run EE Cypress tests on ${{ matrix.folder }} using Replay.io browser | |
| if: matrix.edition == 'ee' && github.event_name == 'schedule' | |
| run: | | |
| yarn run test-cypress-run \ | |
| --env grepTags="-@quarantine" \ | |
| --folder ${{ matrix.folder }} \ | |
| --browser "replay-chromium" | |
| env: | |
| TERM: xterm | |
| CYPRESS_REPLAYIO_ENABLED: 1 | |
| RECORD_REPLAY_METADATA_FILE: /tmp/replay-metadata.json | |
| RECORD_REPLAY_METADATA_TEST_RUN_ID: ${{ needs.test-run-id.outputs.testRunId }} | |
| - name: Upload Replay.io recordings | |
| if: github.event_name == 'schedule' && always() | |
| uses: replayio/[email protected] | |
| with: | |
| api-key: rwk_gXbvYctIcR6RZyEzUvby3gtkO4esrB2L321lkY8FSuQ | |
| public: true | |
| - name: Upload Cypress Artifacts upon failure | |
| uses: actions/upload-artifact@v3 | |
| if: failure() | |
| with: | |
| name: cypress-recording-${{ matrix.folder }}${{ matrix.context }}-${{ matrix.edition }} | |
| path: | | |
| ./cypress | |
| ./logs/test.log | |
| if-no-files-found: ignore | |
| e2e-tests-skipped-stub: | |
| needs: [build, files-changed] | |
| if: | | |
| always() && | |
| needs.files-changed.outputs.e2e_all == 'false' && | |
| needs.build.result == 'skipped' | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 5 | |
| name: e2e-tests-${{ matrix.folder }}${{ matrix.context }}-${{ matrix.edition }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| java-version: [11] | |
| edition: [ee] | |
| folder: | |
| - "auditing" | |
| - "actions" | |
| - "admin" | |
| - "binning" | |
| - "collections" | |
| - "custom-column" | |
| - "dashboard" | |
| - "dashboard-filters" | |
| - "embedding" | |
| - "filters" | |
| - "joins" | |
| - "models" | |
| - "native" | |
| - "native-filters" | |
| - "onboarding" | |
| - "organization" | |
| - "permissions" | |
| - "question" | |
| - "sharing" | |
| - "visualizations" | |
| include: | |
| - edition: oss | |
| context: grep | |
| java-version: 11 | |
| steps: | |
| - run: | | |
| echo "Didn't run due to conditional filtering" | |
| visual-regression-tests: | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 60 | |
| needs: build | |
| if: ${{ github.event_name != 'schedule' }} | |
| name: percy-visual-regression-tests | |
| env: | |
| MB_EDITION: ${{ matrix.edition }} | |
| services: | |
| maildev: | |
| image: maildev/maildev:2.0.5 | |
| ports: | |
| - "1080:1080" | |
| - "1025:1025" | |
| credentials: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Enable Percy recording on master only | |
| if: github.ref == 'refs/heads/master' | |
| run: | | |
| echo "PERCY_TOKEN=${{ secrets.PERCY_TOKEN }}" >> $GITHUB_ENV | |
| - name: Prepare front-end environment | |
| uses: ./.github/actions/prepare-frontend | |
| - name: Prepare JDK 11 | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: 11 | |
| distribution: 'temurin' | |
| - name: Prepare Cypress environment | |
| uses: ./.github/actions/prepare-cypress | |
| - uses: actions/download-artifact@v3 | |
| name: Retrieve uberjar artifact for ee | |
| with: | |
| name: metabase-ee-uberjar | |
| - name: Get the version info | |
| run: | | |
| jar xf target/uberjar/metabase.jar version.properties | |
| mv version.properties resources/ | |
| - name: Percy Test | |
| run: yarn run test-visual-run | |
| env: | |
| PERCY_TOKEN: ${{ env.PERCY_TOKEN }} |