update_regression_baseline.yml

# .github/workflows/update_regression_tests.yml

# for details on triggering a workflow from a comment, see:
# https://dev.to/zirkelc/trigger-github-workflow-for-comment-on-pull-request-45l2
name: Update Regression Baseline

on:
  issue_comment: # trigger from comment; event runs on the default branch
    types: [created]

jobs:
  update_regression_tests:
    name: update_regression_tests
    runs-on: ubuntu-20.04
    # Trigger from a comment that contains '/update_regression_baseline'
    if: github.event.issue.pull_request && contains(github.event.comment.body, '/update_regression_baseline')
    # workflow needs permissions to write to the PR
    permissions:
      contents: write
      pull-requests: write
      issues: read
    
    steps:
    - name: Create initial status comment
      uses: actions/github-script@v7
      id: initial-comment
      with:
        github-token: ${{ secrets.GITHUB_TOKEN }}
        script: |
          const response = await github.rest.issues.createComment({
            issue_number: context.issue.number,
            owner: context.repo.owner,
            repo: context.repo.repo,
            body: '## Updating Regression Baselines\n⏳ Workflow is currently running...'
          });
          return response.data.id;
      
    - name: Check if PR is from fork
      id: check-fork
      uses: actions/github-script@v7
      with:
        github-token: ${{ secrets.GITHUB_TOKEN }}
        script: |
          const pr = await github.rest.pulls.get({
            owner: context.repo.owner,
            repo: context.repo.repo,
            pull_number: context.issue.number
          });
          return pr.data.head.repo.fork;
  
    - name: Get PR branch
      uses: xt0rted/pull-request-comment-branch@v3
      id: comment-branch
      with:
        repo_token: ${{ secrets.GITHUB_TOKEN }}

    - name: Checkout PR branch
      uses: actions/checkout@v3
      with:
        ref: ${{ steps.comment-branch.outputs.head_sha }} # using head_sha vs. head_ref makes this work for forks
        lfs: true
        fetch-depth: 0 # This ensures we can checkout main branch too
    
    - uses: actions/setup-python@v4
      with:
        python-version: '3.10'
        architecture: 'x64'
    
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -e ".[dev]"
       
    - name: Update baseline
      id: update-baseline
      run: |
        git config --global user.name '${{ github.event.comment.user.login }}'
        git config --global user.email '${{ github.event.comment.user.login }}@users.noreply.github.com'
        # Check if regression test results exist in main branch
        if [ -f 'git cat-file -e main:tests/regression_test_baselines.json' ]; then
          git checkout main tests/regression_test_baselines.json
        else
          echo "No regression test results found in main branch"
        fi
        NEW_BASELINE=1 pytest -m regression
        
    # Pushing to the PR branch does not work if the PR is initiated from a fork. This is
    # because the GITHUB_TOKEN has read-only access by default for workflows triggered by 
    # fork PRs. Hence we have to create a new PR to update the baseline (see below).
    - name: Commit and push to PR branch (non-fork)
      # Only run if baseline generation succeeded
      if: success() && steps.update-baseline.outcome == 'success' && !fromJson(steps.check-fork.outputs.result)
      run: |
        git add -f tests/regression_test_baselines.json # since it's in .gitignore
        git commit -m "Update regression test baselines"
        git push origin HEAD:${{ steps.comment-branch.outputs.head_ref }} # head_ref will probably not work for forks!

    - name: Create PR with updates (fork)
      if: success() && steps.update-baseline.outcome == 'success' && fromJson(steps.check-fork.outputs.result)
      uses: peter-evans/create-pull-request@v5
      id: create-pr
      with:
        token: ${{ secrets.GITHUB_TOKEN }}
        commit-message: Update regression test baselines
        title: 'Update regression test baselines'
        branch: regression-baseline-update-${{ github.event.issue.number }}
        base: ${{ steps.comment-branch.outputs.head_ref }}

    - name: Update comment with results
      uses: actions/github-script@v7
      if: always()  # Run this step even if previous steps fail
      with:
        github-token: ${{ secrets.GITHUB_TOKEN }}
        script: |
          const fs = require('fs');
          let status = '${{ steps.update-baseline.outcome }}' === 'success' ? '✅' : '❌';
          let message = '## Regression Baseline Update\n' + status + ' Process completed\n\n';
          
          try {
            const TestReport = fs.readFileSync('tests/regression_test_report.txt', 'utf8');
            message += '```\n' + TestReport + '\n```\n\n';
            
            // Add information about where the changes were pushed
            if ('${{ steps.update-baseline.outcome }}' === 'success') {
              if (!${{ fromJson(steps.check-fork.outputs.result) }}) {
                message += '✨ Changes have been pushed directly to this PR\n';
              } else {
                const prNumber = '${{ steps.create-pr.outputs.pull-request-number }}';
                message += `✨ Changes have been pushed to a new PR #${prNumber} because this PR is from a fork\n`;
              }
            }
          } catch (error) {
            message += '⚠️ No test report was generated\n';
          }
          
          await github.rest.issues.updateComment({
            comment_id: ${{ steps.initial-comment.outputs.result }},
            owner: context.repo.owner,
            repo: context.repo.repo,
            body: message
          });