enh: better regression tests. works locally, need to update baseline

.github/workflows/regression_tests.yml

@@ -31,26 +31,26 @@ jobs:
       if: github.event.pull_request.base.ref == 'main'
       run: |
         # Check if regression test results exist in main branch
-        if [ -f 'git cat-file -e main:tests/regression_test_results.json' ]; then
-          git checkout main tests/regression_test_results.json
+        if [ -f 'git cat-file -e main:tests/regression_test_baselines.json' ]; then
+          git checkout main tests/regression_test_baselines.json
         else
           echo "No regression test results found in main branch"
         fi
-        python tests/regression_test_runner.py > regression_test_report.txt
-        git checkout .
+        PYTHONHASHSEED=0 pytest tests/test_regression.py
+        git checkout
         
-    - name: Comment PR
-      if: github.event.pull_request.base.ref == 'main'
-      uses: actions/github-script@v7
-      with:
-        github-token: ${{ secrets.GITHUB_TOKEN }}
-        script: |
-          const fs = require('fs');
-          const TestReport = fs.readFileSync('regression_test_report.txt', 'utf8');
+    # - name: Comment PR
+    #   if: github.event.pull_request.base.ref == 'main'
+    #   uses: actions/github-script@v7
+    #   with:
+    #     github-token: ${{ secrets.GITHUB_TOKEN }}
+    #     script: |
+    #       const fs = require('fs');
+    #       const TestReport = fs.readFileSync('regression_test_report.txt', 'utf8');
 
-          await github.rest.issues.createComment({
-            issue_number: context.issue.number,
-            owner: context.repo.owner,
-            repo: context.repo.repo,
-            body: `## Regression Test Results\n\`\`\`\n${TestReport}\n\`\`\``
-          });
+    #       await github.rest.issues.createComment({
+    #         issue_number: context.issue.number,
+    #         owner: context.repo.owner,
+    #         repo: context.repo.repo,
+    #         body: `## Regression Test Results\n\`\`\`\n${TestReport}\n\`\`\``
+    #       });

.github/workflows/update_regression_baseline.yml

@@ -2,7 +2,9 @@
 name: Regression Tests
 
 on:
-  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
 
 jobs:
   regression_tests:
@@ -24,13 +26,8 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -e ".[dev]"
-    
-    - name: Run benchmarks on PR branch
+       
+    - name: Update baseline
+      if: github.event.pull_request.base.ref == 'main'
       run: |
-        python tests/regression_test_runner.py
-    
-    - name: Save new regression baseline
-      uses: actions/upload-artifact@v3
-      with:
-        name: regression-test-results
-        path: tests/regression_test_results.json
+        PYTHONHASHSEED=0 UPDATE_BASELINE=1 pytest tests/test_regression.py