report.yml

name: Screen Reader Tests Report

on:
  push:
    branches: [main]
  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
permissions:
  contents: read
  deployments: write
  pages: write
  id-token: write
  pull-requests: write

# Allow one concurrent deployment
concurrency:
  group: "aria-at-tests-report"
  cancel-in-progress: true

jobs:
  macos-install:
    name: MacOS Install
    runs-on: macos-12
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: recursive

      - uses: actions/setup-node@v3
        with:
          node-version: 20

      - name: Cache node_modules
        uses: actions/cache@v3
        id: cache-macos-node-modules
        with:
          path: |
            node_modules
          key: macos-modules-${{ hashFiles('yarn.lock') }}-${{ hashFiles('package.json') }}

      - name: Cache Playwright binaries
        uses: actions/cache@v3
        id: cache-macos-playwright
        with:
          path: |
            ~/Library/Caches/ms-playwright
          key: macos-playwright-${{ hashFiles('yarn.lock') }}-${{ hashFiles('package.json') }}

      - name: Cache Test Suites
        uses: actions/cache@v3
        id: cache-macos-test-suites
        with:
          path: |
            testSuites.json
          key: macos-testSuites-${{ hashFiles('aria-at') }}-${{ hashFiles('ignoredTests.json') }}

      - name: Install Dependencies
        if: steps.cache-macos-node-modules.outputs.cache-hit != 'true'
        run: yarn install --frozen-lockfile

      - name: Install Playwright Browsers
        if: steps.cache-macos-playwright.outputs.cache-hit != 'true'
        run: yarn test:install

      - name: Generate Test Suites
        if: steps.cache-macos-test-suites.outputs.cache-hit != 'true'
        run: yarn test:generate

  windows-install:
    name: Windows Install
    runs-on: windows-2022
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: recursive

      - uses: actions/setup-node@v3
        with:
          node-version: 20

      - name: Cache node_modules
        uses: actions/cache@v3
        id: cache-windows-node-modules
        with:
          path: |
            node_modules
          key: windows-modules-${{ hashFiles('yarn.lock') }}-${{ hashFiles('package.json') }}

      - name: Cache Playwright binaries
        uses: actions/cache@v3
        id: cache-windows-playwright
        with:
          path: |
            C:\Users\runneradmin\AppData\Local\ms-playwright
          key: windows-playwright-${{ hashFiles('yarn.lock') }}-${{ hashFiles('package.json') }}

      - name: Cache Test Suites
        uses: actions/cache@v3
        id: cache-windows-test-suites
        with:
          path: |
            testSuites.json
          key: windows-testSuites-${{ hashFiles('aria-at') }}-${{ hashFiles('ignoredTests.json') }}

      - name: Install Dependencies
        if: steps.cache-windows-node-modules.outputs.cache-hit != 'true'
        run: yarn install --frozen-lockfile

      - name: Install Playwright Browsers
        if: steps.cache-windows-playwright.outputs.cache-hit != 'true'
        run: yarn test:install

      - name: Generate Test Suites
        if: steps.cache-windows-test-suites.outputs.cache-hit != 'true'
        run: yarn test:generate

  test-voiceover:
    name: Playwright VoiceOver Tests (${{ matrix.shardIndex }} / 20)
    needs: macos-install
    runs-on: macos-12
    strategy:
      fail-fast: false
      matrix:
        shardIndex:
          [
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            10,
            11,
            12,
            13,
            14,
            15,
            16,
            17,
            18,
            19,
            20,
          ]

    steps:
      - uses: actions/checkout@v3
        with:
          submodules: recursive

      - uses: actions/setup-node@v3
        with:
          node-version: 20

      - name: Guidepup Setup
        uses: guidepup/setup-action@0.13.0

      - name: Cache node_modules
        uses: actions/cache@v3
        with:
          path: |
            node_modules
          key: macos-modules-${{ hashFiles('yarn.lock') }}-${{ hashFiles('package.json') }}

      - name: Cache Playwright
        uses: actions/cache@v3
        with:
          path: |
            ~/Library/Caches/ms-playwright
          key: macos-playwright-${{ hashFiles('yarn.lock') }}-${{ hashFiles('package.json') }}

      - name: Cache Test Suites
        uses: actions/cache@v3
        with:
          path: |
            testSuites.json
          key: macos-testSuites-${{ hashFiles('aria-at') }}-${{ hashFiles('ignoredTests.json') }}

      - name: Run Tests
        run: yarn test ./src/macOsVoiceOver.spec.ts --config macos.config.ts --shard ${{ matrix.shardIndex }}/20
        continue-on-error: true

      - name: Upload Playwright Artifacts
        uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        continue-on-error: true
        with:
          name: artifacts-macos-voiceover-${{ matrix.shardIndex }}
          path: |
            ./test-results
            ./recordings

      - name: Upload HTML Report Artifacts
        uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        continue-on-error: true
        with:
          name: playwright-report-macos-voiceover-${{ matrix.shardIndex }}
          path: ./playwright-report

  test-nvda:
    name: Playwright NVDA Tests (${{ matrix.shardIndex }} / 20)
    needs: windows-install
    runs-on: windows-2022
    strategy:
      fail-fast: false
      matrix:
        shardIndex:
          [
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            10,
            11,
            12,
            13,
            14,
            15,
            16,
            17,
            18,
            19,
            20,
          ]

    steps:
      - uses: actions/checkout@v3
        with:
          submodules: recursive

      - uses: actions/setup-node@v3
        with:
          node-version: 20

      - name: Guidepup Setup
        uses: guidepup/setup-action@0.13.0

      - name: Cache node_modules
        uses: actions/cache@v3
        with:
          path: |
            node_modules
          key: windows-modules-${{ hashFiles('yarn.lock') }}-${{ hashFiles('package.json') }}

      - name: Cache Playwright
        uses: actions/cache@v3
        with:
          path: |
            C:\Users\runneradmin\AppData\Local\ms-playwright
          key: windows-playwright-${{ hashFiles('yarn.lock') }}-${{ hashFiles('package.json') }}

      - name: Cache Test Suites
        uses: actions/cache@v3
        with:
          path: |
            testSuites.json
          key: windows-testSuites-${{ hashFiles('aria-at') }}-${{ hashFiles('ignoredTests.json') }}

      - name: Run Tests
        run: yarn test ./src/windowsNvda.spec.ts --config windows.config.ts --shard ${{ matrix.shardIndex }}/20
        continue-on-error: true

      - name: Upload Playwright Artifacts
        uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        continue-on-error: true
        with:
          name: artifacts-windows-nvda-${{ matrix.shardIndex }}
          path: |
            ./test-results
            ./recordings

      - name: Upload HTML Report Artifacts
        uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        continue-on-error: true
        with:
          name: playwright-report-windows-nvda-${{ matrix.shardIndex }}
          path: ./playwright-report

  publish-html-report:
    name: Publish HTML Report
    if: ${{ !cancelled() }}
    needs: [test-voiceover, test-nvda]
    runs-on: ubuntu-latest
    steps:
      - name: Clear Space
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /opt/ghc
          sudo rm -rf /opt/hostedtoolcache/CodeQL
          sudo docker image prune --all --force
          sudo apt-get purge -y \
            '^llvm-.*' \
            'php.*' \
            '^mongodb-.*' \
            '^mysql-.*' \
            azure-cli \
            google-chrome-stable \
            firefox \
            powershell \
            microsoft-edge-stable \
            mono-devel
          sudo apt-get autoremove -y
          sudo apt-get clean

      - uses: actions/checkout@v3
        with:
          submodules: recursive

      - uses: actions/setup-node@v3
        with:
          node-version: 20

      - name: Update Deployment Status - Start
        uses: bobheadxi/deployments@v1
        id: deployment
        with:
          step: start
          env: aria-at-tests-report
          token: ${{ secrets.GITHUB_TOKEN }}

      - uses: actions/download-artifact@v4
        with:
          pattern: playwright-report-*
          path: ./playwright-report

      - name: Cache node_modules
        uses: actions/cache@v3
        id: cache-macos-node-modules
        with:
          path: |
            node_modules
          key: macos-modules-${{ hashFiles('yarn.lock') }}-${{ hashFiles('package.json') }}

      - name: Create Test Report
        run: yarn test:report

      - name: Cleanup Downloads
        run: rm -rf ./playwright-report || true

      - name: Deploy to Vercel
        uses: amondnet/vercel-action@v25
        id: vercel-action
        with:
          vercel-token: ${{ secrets.VERCEL_TOKEN }}
          vercel-org-id: ${{ secrets.VERCEL_ORG_ID }}
          vercel-project-id: ${{ secrets.VERCEL_PROJECT_ID }}
          github-comment: false
          vercel-args: "--prod"
          github-token: ${{ secrets.GITHUB_TOKEN }}
          working-directory: html-report

      - name: Update Deployment Status - Finish
        uses: bobheadxi/deployments@v1
        if: ${{ !cancelled() }}
        with:
          step: finish
          token: ${{ secrets.GITHUB_TOKEN }}
          status: ${{ job.status }}
          deployment_id: ${{ steps.deployment.outputs.deployment_id }}
          env: ${{ steps.deployment.outputs.env }}
          env_url: ${{ steps.vercel-action.outputs.preview-url }}