E2E: Enable E2E v2 in CI #22852
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| types: [opened, reopened, synchronize, ready_for_review] | |
| workflow_dispatch: | |
| inputs: | |
| unit_tests: | |
| type: boolean | |
| description: 'Run unit tests' | |
| required: false | |
| default: true | |
| integration_tests: | |
| type: boolean | |
| description: 'Run integration tests' | |
| required: false | |
| default: true | |
| e2e_tests: | |
| type: boolean | |
| description: 'Run e2e tests' | |
| required: false | |
| default: true | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build: | |
| name: Build | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version-file: .tool-versions | |
| - uses: pnpm/action-setup@fe02b34f77f8bc703788d5817da081398fad5dd2 # SECURITY: pin third-party action hashes | |
| - run: echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT | |
| shell: bash | |
| id: pnpm-cache | |
| - name: Cache pnpm store | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} | |
| key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} | |
| restore-keys: ${{ runner.os }}-pnpm-store- | |
| - run: pnpm install | |
| # Run Biome and capture the output. If biome crashes, it still reports a | |
| # successful CI run (exit code 0) but that will ripple down to errors in | |
| # the Biome VS Code extension. This ensures that we don't accidentally | |
| # break Biome. | |
| - name: Biome | |
| run: | | |
| set +e | |
| output=$(pnpm exec biome ci --error-on-warnings . 2>&1) | |
| status=$? | |
| echo "$output" | |
| if [ $status -ne 0 ]; then | |
| exit $status | |
| fi | |
| if echo "$output" | grep -q "Biome encountered an unexpected error"; then | |
| echo "Error string 'Biome encountered an unexpected error' detected in output." | |
| exit 1 | |
| fi | |
| shell: bash | |
| - run: pnpm run build | |
| - run: pnpm -C vscode run build | |
| - run: CODY_RELEASE_TYPE=stable pnpm -C vscode run release:dry-run | |
| - name: Upload build artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: vscode_dist | |
| path: vscode/dist | |
| retention-days: 1 | |
| # Sets a variable that is used to determine the matrix to run tests on. | |
| # Everything runs on ubuntu and windows, only commits to main run on macos. | |
| fast_tests_matrix_prep: | |
| name: 'Test Matrix (Unit, Integration)' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - id: set-matrix | |
| run: | | |
| if [ "$GITHUB_EVENT_NAME" == "push" ] && [ "$GITHUB_REF" == "refs/heads/main" ]; then | |
| echo 'matrix=["ubuntu","windows","macos"]' >> $GITHUB_OUTPUT | |
| else | |
| echo 'matrix=["ubuntu","windows"]' >> $GITHUB_OUTPUT | |
| fi | |
| test-unit: | |
| name: Test - Unit (${{matrix.runner}}, Node v${{matrix.node}}) | |
| needs: fast_tests_matrix_prep | |
| if: ${{ !cancelled() }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runner: ${{ fromJson(needs.fast_tests_matrix_prep.outputs.matrix) }} | |
| # Run on the most recently supported version of node for all bots. | |
| node: [20] | |
| include: | |
| # Additionally, run the oldest supported version on Ubuntu. We don't | |
| # need to run this on all platforms as we're only verifying we don't | |
| # call any APIs not available in this version. | |
| - runner: ubuntu | |
| node: 18 # VS Code started using Node 18 in Aug 2023 in v1.82: https://code.visualstudio.com/updates/v1_82#_engineering | |
| runs-on: ${{ matrix.runner }}-latest | |
| timeout-minutes: 10 | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node }} | |
| - uses: pnpm/action-setup@fe02b34f77f8bc703788d5817da081398fad5dd2 # SECURITY: pin third-party action hashes | |
| - id: auth | |
| uses: google-github-actions/auth@v0 | |
| with: | |
| workload_identity_provider: ${{ secrets.DATA_TEAM_PROVIDER_NAME }} | |
| service_account: ${{ secrets.DATA_TEAM_SA_EMAIL }} | |
| - uses: google-github-actions/setup-gcloud@v2 | |
| - run: echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT | |
| shell: bash | |
| id: pnpm-cache | |
| - name: Cache pnpm store | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} | |
| key: ${{ runner.os }}-${{ matrix.node }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} | |
| restore-keys: ${{ runner.os }}-${{ matrix.node }}-pnpm-store- | |
| - run: pnpm install | |
| - run: pnpm build | |
| - run: pnpm run test:unit --run | |
| env: | |
| CODY_NODE_VERSION: ${{ matrix.node }} | |
| test-integration: | |
| name: Test - Integration (${{matrix.runner}}) | |
| needs: fast_tests_matrix_prep | |
| if: ${{ !cancelled() }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runner: ${{ fromJson(needs.fast_tests_matrix_prep.outputs.matrix) }} | |
| runs-on: ${{ matrix.runner }}-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version-file: .tool-versions | |
| - uses: pnpm/action-setup@fe02b34f77f8bc703788d5817da081398fad5dd2 # SECURITY: pin third-party action hashes | |
| - run: echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT | |
| shell: bash | |
| id: pnpm-cache | |
| - name: Cache pnpm store | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} | |
| key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} | |
| restore-keys: ${{ runner.os }}-pnpm-store- | |
| - run: pnpm install | |
| - run: xvfb-run -a pnpm -C vscode run test:integration | |
| if: matrix.runner == 'ubuntu' | |
| - run: pnpm -C vscode run test:integration | |
| if: matrix.runner == 'windows' || matrix.runner == 'macos' | |
| test-e2e: | |
| name: 'Test - E2E (${{ matrix.os }} - ${{ matrix.shardIndex }}/${{ matrix.shardTotal }})' | |
| needs: build | |
| if: ${{ success() }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: ['windows', 'ubuntu', 'macos'] | |
| shardIndex: ['1', '2'] # increase this as test total increases | |
| shardTotal: ['2'] | |
| exclude: | |
| - os: ${{ github.event_name == 'pull_request' && github.event.pull_request.draft && 'windows' }} | |
| - os: ${{ github.event_name == 'pull_request' && github.event.pull_request.draft && 'macos' }} | |
| runs-on: ${{ matrix.os }}-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version-file: .tool-versions | |
| - uses: pnpm/action-setup@fe02b34f77f8bc703788d5817da081398fad5dd2 # SECURITY: pin third-party action hashes | |
| - run: echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT | |
| shell: bash | |
| id: pnpm-cache | |
| - name: Cache pnpm store | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} | |
| # we use a different key here because we install the Playwright dependencies | |
| # however, we can still restore from the shared cache if there's a miss | |
| key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} | |
| restore-keys: ${{ runner.os }}-pnpm-store- | |
| - name: Install dependencies | |
| run: pnpm install | |
| - name: Build lib/shared | |
| run: pnpm -C lib/shared build --force | |
| # lib shared is used by the test framework | |
| - name: Setup Playwright browser dependencies | |
| run: pnpm -C vscode run test:e2e2:deps # this installs chromium etc | |
| - name: Download build artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: vscode_dist | |
| path: vscode/dist | |
| - name: Run E2E tests | |
| run: pnpm -C vscode run test:e2e2:run --project e2e --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} | |
| env: | |
| PLAYWRIGHT_BLOB_OUTPUT_NAME: report-${{matrix.os}}-${{matrix.shardIndex}}.zip | |
| - name: Upload blob report to GitHub Actions Artifacts | |
| if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e-blob-report-${{matrix.os}}-${{ matrix.shardIndex }} | |
| path: vscode/.test/e2e/reports/blob | |
| retention-days: 1 | |
| # TODO: Trigger another workflow / daily workflow to pipe the telemetry | |
| # events to pubsub so that we can keep historical records even if not | |
| # tested | |
| test-e2e-report: | |
| name: 'Test - E2E (Report)' | |
| runs-on: ubuntu-latest | |
| if: ${{ !cancelled() }} | |
| needs: [test-e2e] | |
| permissions: | |
| id-token: write | |
| contents: read | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version-file: .tool-versions | |
| - uses: pnpm/action-setup@fe02b34f77f8bc703788d5817da081398fad5dd2 # SECURITY: pin third-party action hashes | |
| - id: auth | |
| uses: google-github-actions/auth@v2 | |
| # Skip auth if PR is from a fork | |
| if: ${{ !github.event.pull_request.head.repo.fork }} | |
| with: | |
| workload_identity_provider: ${{ secrets.DATA_TEAM_PROVIDER_NAME }} | |
| service_account: ${{ secrets.DATA_TEAM_SA_EMAIL }} | |
| - uses: google-github-actions/setup-gcloud@v2 | |
| - run: echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT | |
| shell: bash | |
| id: pnpm-cache | |
| - name: Cache pnpm store | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} | |
| # we use a different key here because we install the Playwright dependencies | |
| # however, we can still restore from the shared cache if there's a miss | |
| key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} | |
| restore-keys: ${{ runner.os }}-pnpm-store- | |
| - name: Install dependencies | |
| run: pnpm install | |
| - name: Download E2E reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: vscode/.test/e2e/reports/blob | |
| pattern: e2e-blob-report-* | |
| merge-multiple: true | |
| - name: Merge into HTML Report | |
| run: pnpm -C vscode exec playwright merge-reports --reporter html,json -c .test/e2e/reports/blob .test/e2e/reports/blob | |
| env: | |
| PLAYWRIGHT_HTML_OUTPUT_DIR: .test/e2e/reports/html | |
| PLAYWRIGHT_JSON_OUTPUT_NAME: report.json | |
| PLAYWRIGHT_JSON_OUTPUT_DIR: .test/e2e/reports/json | |
| - name: Upload HTML report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e-html-report | |
| path: vscode/.test/e2e/reports/html | |
| retention-days: 7 | |
| - name: Comment on PR | |
| uses: daun/playwright-report-summary@v3 | |
| with: | |
| report-file: vscode/.test/e2e/reports/json/report.json | |
| # TODO: Pipe telemetry to PubSub | |
| - name: Report to Services | |
| run: pnpm -C vscode exec playwright merge-reports --reporter github,@estruyf/github-actions-reporter -c .test/e2e/reports/blob .test/e2e/reports/blob | |
| env: | |
| BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }} | |
| # This is required because the test collector from Buildkite is imprecise when it | |
| # comes to infer the branch name on GitHub action. | |
| BRANCH_NAME: ${{ github.head_ref || github.ref_name }} | |
| GITHUB_REF: ${{ github.head_ref || github.ref_name }} | |
| # Sets a variable that is used to determine the matrix to run slow tests (e2e) on. | |
| # Everything runs on ubuntu, only commits to main run on macos and windows. | |
| legacy_slow_tests_matrix_prep: | |
| name: Test Matrix (E2E legacy) | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - id: set-matrix | |
| run: | | |
| if [ "$GITHUB_EVENT_NAME" == "push" ] && [ "$GITHUB_REF" == "refs/heads/main" ]; then | |
| echo 'matrix=[{"runner":"ubuntu"},{"runner":"windows"},{"runner":"macos"}]' >> $GITHUB_OUTPUT | |
| else | |
| echo 'matrix=[{"runner":"ubuntu","shard":"1/5"},{"runner":"ubuntu","shard":"2/5"},{"runner":"ubuntu","shard":"3/5"},{"runner":"ubuntu","shard":"4/5"},{"runner":"ubuntu","shard":"5/5"}]' >> $GITHUB_OUTPUT | |
| fi | |
| test-e2e-legacy: | |
| name: 'Test - Legacy E2E' | |
| needs: [legacy_slow_tests_matrix_prep, test-e2e-report] | |
| if: ${{ !cancelled() }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.legacy_slow_tests_matrix_prep.outputs.matrix) }} | |
| runs-on: ${{ matrix.runner }}-latest | |
| timeout-minutes: 20 | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version-file: .tool-versions | |
| - uses: pnpm/action-setup@fe02b34f77f8bc703788d5817da081398fad5dd2 # SECURITY: pin third-party action hashes | |
| - id: auth | |
| uses: google-github-actions/auth@v2 | |
| # Skip auth if PR is from a fork | |
| if: ${{ !github.event.pull_request.head.repo.fork }} | |
| with: | |
| workload_identity_provider: ${{ secrets.DATA_TEAM_PROVIDER_NAME }} | |
| service_account: ${{ secrets.DATA_TEAM_SA_EMAIL }} | |
| - uses: google-github-actions/setup-gcloud@v2 | |
| - run: echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT | |
| shell: bash | |
| id: pnpm-cache | |
| - name: Cache pnpm store | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} | |
| key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} | |
| restore-keys: ${{ runner.os }}-pnpm-store- | |
| - run: pnpm install | |
| - run: GITHUB_REF=$BRANCH_NAME xvfb-run -a pnpm -C vscode run test:e2e --shard=${{ matrix.shard }} | |
| if: matrix.runner == 'ubuntu' | |
| env: | |
| BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }} | |
| # This is required because the test collector from Buildkite is imprecise when it | |
| # comes to infer the branch name on GitHub action. | |
| BRANCH_NAME: ${{ github.head_ref || github.ref_name }} | |
| - run: GITHUB_REF=$BRANCH_NAME pnpm -C vscode run test:e2e | |
| if: matrix.runner == 'macos' | |
| env: | |
| BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }} | |
| # This is required because the test collector from Buildkite is imprecise when it | |
| # comes to infer the branch name on GitHub action. | |
| BRANCH_NAME: ${{ github.head_ref || github.ref_name }} | |
| - run: $env:GITHUB_REF=$env:BRANCH_NAME; pnpm -C vscode run test:e2e | |
| if: matrix.runner == 'windows' | |
| env: | |
| BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }} | |
| # This is required because the test collector from Buildkite is imprecise when it | |
| # comes to infer the branch name on GitHub action. | |
| BRANCH_NAME: ${{ github.head_ref || github.ref_name }} | |
| - uses: actions/upload-artifact@v4 | |
| if: ${{ failure() }} | |
| with: | |
| name: playwright-recordings ${{ matrix.runner }} | |
| path: playwright/ |