regression_tests.yml

# .github/workflows/update_regression_tests.yml

# for details on triggering a workflow from a comment, see:
# https://dev.to/zirkelc/trigger-github-workflow-for-comment-on-pull-request-45l2
name: Regression Tests

on:
  issue_comment: # trigger from comment; event runs on the default branch
    types: [created]

jobs:
  update_regression_tests:
    name: update_regression_tests
    runs-on: ubuntu-20.04
    # Trigger from a comment that contains '/test_regression'
    if: github.event.issue.pull_request && contains(github.event.comment.body, '/test_regression')
    # workflow needs permissions to write to the PR
    permissions:
      contents: write
      pull-requests: write
      issues: read
    
    steps:
    - name: Create initial status comment
      uses: actions/github-script@v7
      id: initial-comment
      with:
        github-token: ${{ secrets.GITHUB_TOKEN }}
        script: |
          const response = await github.rest.issues.createComment({
            issue_number: context.issue.number,
            owner: context.repo.owner,
            repo: context.repo.repo,
            body: '## Regression Test\n⏳ Workflow is currently running...'
          });
          return response.data.id;
      
    - name: Check if PR is from fork
      id: check-fork
      uses: actions/github-script@v7
      with:
        github-token: ${{ secrets.GITHUB_TOKEN }}
        script: |
          const pr = await github.rest.pulls.get({
            owner: context.repo.owner,
            repo: context.repo.repo,
            pull_number: context.issue.number
          });
          return pr.data.head.repo.fork;

    - uses: actions/checkout@v3
      with:
        ref: main
        lfs: true
        fetch-depth: 0

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.10'
        architecture: 'x64'
    
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -e ".[dev]"

    - name: Update baseline
      id: update-baseline
      run: |
        NEW_BASELINE=1 pytest -m regression
        cp tests/regression_test_baselines.json /tmp/regression_test_baselines.json

    - name: Get PR branch
      uses: xt0rted/pull-request-comment-branch@v3
      id: comment-branch
      with:
        repo_token: ${{ secrets.GITHUB_TOKEN }}

    - name: Checkout PR branch
      uses: actions/checkout@v3
      with:
        ref: ${{ steps.comment-branch.outputs.head_sha }} # using head_sha vs. head_ref makes this work for forks
        lfs: true
        fetch-depth: 0 # This ensures we can checkout main branch too

    - name: Run comparison
      id: comparison
      run: |
        cp /tmp/regression_test_baselines.json tests/regression_test_baselines.json
        pytest -m regression

    - name: Update comment with results
      uses: actions/github-script@v7
      if: always()  # Run this step even if previous steps fail
      with:
        github-token: ${{ secrets.GITHUB_TOKEN }}
        script: |
          const fs = require('fs');
          let status = '${{ steps.comparison.outcome }}' === 'success' ? '✅' : '❌';
          let message = '## Regression Baseline Update\n' + status + ' Process completed\n\n';
          
          try {
            const TestReport = fs.readFileSync('tests/regression_test_report.txt', 'utf8');
            message += '```\n' + TestReport + '\n```\n\n';
            
            // Add information about where the changes were pushed
            if ('${{ steps.comparison.outcome }}' === 'success') {
              if (!${{ fromJson(steps.check-fork.outputs.result) }}) {
                message += '✨ Changes have been pushed directly to this PR\n';
              } else {
                const prNumber = '${{ steps.create-pr.outputs.pull-request-number }}';
                message += `✨ Changes have been pushed to a new PR #${prNumber} because this PR is from a fork\n`;
              }
            }
          } catch (error) {
            message += '⚠️ No test report was generated\n';
          }
          
          await github.rest.issues.updateComment({
            comment_id: ${{ steps.initial-comment.outputs.result }},
            owner: context.repo.owner,
            repo: context.repo.repo,
            body: message
          });