wip: save wip

.github/workflows/regression_tests.yml

@@ -1,23 +1,55 @@
-# .github/workflows/regression_tests.yml
-name: Regression Tests
+# .github/workflows/update_regression_tests.yml
+
+# for details on triggering a workflow from a comment, see:
+# https://dev.to/zirkelc/trigger-github-workflow-for-comment-on-pull-request-45l2
+name: Update Regression Baseline
 
 on:
-  # pull_request:
-  #   branches:
-  #     - main
+  issue_comment: # trigger from comment; event runs on the default branch
+    types: [created]
 
 jobs:
-  regression_tests:
-    name: regression_tests
+  update_regression_tests:
+    name: update_regression_tests
     runs-on: ubuntu-20.04
+    # Trigger from a comment that contains '/update_regression_baseline'
+    if: github.event.issue.pull_request && contains(github.event.comment.body, '/update_regression_baseline')
+    # workflow needs permissions to write to the PR
+    permissions:
+      contents: write
+      pull-requests: write
+      issues: read
 
     steps:
-    - uses: actions/checkout@v3
+    - name: Create initial status comment
+      uses: actions/github-script@v7
+      id: initial-comment
       with:
-        lfs: true
-        fetch-depth: 0 # This ensures we can checkout main branch too
-
-    - uses: actions/setup-python@v4
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        script: |
+          const response = await github.rest.issues.createComment({
+            issue_number: context.issue.number,
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            body: '## Updating Regression Baselines\n⏳ Workflow is currently running...'
+          });
+          return response.data.id;
+      
+    - name: Check if PR is from fork
+      id: check-fork
+      uses: actions/github-script@v7
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        script: |
+          const pr = await github.rest.pulls.get({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            pull_number: context.issue.number
+          });
+          return pr.data.head.repo.fork;
+  
+    - name: Set up Python
+      uses: actions/setup-python@v4
       with:
         python-version: '3.10'
         architecture: 'x64'
@@ -26,7 +58,28 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -e ".[dev]"
-       
+
+    - name: Update baseline
+      id: update-baseline
+      run: |
+        git config --global user.name '${{ github.event.comment.user.login }}'
+        git config --global user.email '${{ github.event.comment.user.login }}@users.noreply.github.com'
+        git checkout main
+        NEW_BASELINE=1 pytest -m regression
+
+    - name: Get PR branch
+      uses: xt0rted/pull-request-comment-branch@v3
+      id: comment-branch
+      with:
+        repo_token: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Checkout PR branch
+      uses: actions/checkout@v3
+      with:
+        ref: ${{ steps.comment-branch.outputs.head_sha }} # using head_sha vs. head_ref makes this work for forks
+        lfs: true
+        fetch-depth: 0 # This ensures we can checkout main branch too
+
     - name: Run benchmarks and compare to baseline
       if: github.event.pull_request.base.ref == 'main'
       run: |
@@ -36,4 +89,60 @@ jobs:
         else
           echo "No regression test results found in main branch"
         fi
-        pytest -m regression
+        pytest -m regression
+        
+    # Pushing to the PR branch does not work if the PR is initiated from a fork. This is
+    # because the GITHUB_TOKEN has read-only access by default for workflows triggered by 
+    # fork PRs. Hence we have to create a new PR to update the baseline (see below).
+    - name: Commit and push to PR branch (non-fork)
+      # Only run if baseline generation succeeded
+      if: success() && steps.update-baseline.outcome == 'success' && !fromJson(steps.check-fork.outputs.result)
+      run: |
+        git add -f tests/regression_test_baselines.json # since it's in .gitignore
+        git commit -m "Update regression test baselines"
+        git push origin HEAD:${{ steps.comment-branch.outputs.head_ref }} # head_ref will probably not work for forks!
+
+    - name: Create PR with updates (fork)
+      if: success() && steps.update-baseline.outcome == 'success' && fromJson(steps.check-fork.outputs.result)
+      uses: peter-evans/create-pull-request@v5
+      id: create-pr
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        commit-message: Update regression test baselines
+        title: 'Update regression test baselines'
+        branch: regression-baseline-update-${{ github.event.issue.number }}
+        base: ${{ steps.comment-branch.outputs.head_ref }}
+
+    - name: Update comment with results
+      uses: actions/github-script@v7
+      if: always()  # Run this step even if previous steps fail
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        script: |
+          const fs = require('fs');
+          let status = '${{ steps.update-baseline.outcome }}' === 'success' ? '✅' : '❌';
+          let message = '## Regression Baseline Update\n' + status + ' Process completed\n\n';
+          
+          try {
+            const TestReport = fs.readFileSync('tests/regression_test_report.txt', 'utf8');
+            message += '```\n' + TestReport + '\n```\n\n';
+            
+            // Add information about where the changes were pushed
+            if ('${{ steps.update-baseline.outcome }}' === 'success') {
+              if (!${{ fromJson(steps.check-fork.outputs.result) }}) {
+                message += '✨ Changes have been pushed directly to this PR\n';
+              } else {
+                const prNumber = '${{ steps.create-pr.outputs.pull-request-number }}';
+                message += `✨ Changes have been pushed to a new PR #${prNumber} because this PR is from a fork\n`;
+              }
+            }
+          } catch (error) {
+            message += '⚠️ No test report was generated\n';
+          }
+          
+          await github.rest.issues.updateComment({
+            comment_id: ${{ steps.initial-comment.outputs.result }},
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            body: message
+          });