feat[ci]: use coverage combine to reduce codecov uploads (#4452)

This commit updates the CI to combine code coverage to a single file
and perform just a single upload after all tests finish (rather than
each test run uploading its own coverage report). This should reduce
failures / rate limiting on the codecov app, and also prevent the
codecov app from producing an inaccurate coverage report before all
the tests finish.

references:
- https://coverage.readthedocs.io/en/7.6.10/cmd.html#cmd-combine
- https://coverage.readthedocs.io/en/7.6.10/cmd.html#re-mapping-paths
- https://coverage.readthedocs.io/en/7.6.10/config.html#config-run-relative-files

---------

Co-authored-by: Charles Cooper <[email protected]>

.github/workflows/test.yml

@@ -149,16 +149,17 @@ jobs:
         --evm-backend ${{ matrix.evm-backend || 'revm' }}
         ${{ matrix.debug && '--enable-compiler-debug-mode' || '' }}
         ${{ matrix.experimental-codegen && '--experimental-codegen' || '' }}
-        --cov-branch
-        --cov-report xml:coverage.xml
+        --cov-config=setup.cfg
         --cov=vyper
         tests/
 
-    - name: Upload Coverage
-      uses: codecov/codecov-action@v5
+    - name: Upload coverage artifact
+      uses: actions/upload-artifact@v4
       with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        file: ./coverage.xml
+        name: coverage-files-${{ github.job }}-${{ strategy.job-index }}
+        include-hidden-files: true
+        path: .coverage
+        if-no-files-found: error
 
   core-tests-success:
     if: always()
@@ -209,16 +210,17 @@ jobs:
           --splits 120 \
           --group ${{ matrix.group }} \
           --splitting-algorithm least_duration \
-          --cov-branch \
-          --cov-report xml:coverage.xml \
+          --cov-config=setup.cfg \
           --cov=vyper \
           tests/
 
-    - name: Upload Coverage
-      uses: codecov/codecov-action@v5
+    - name: Upload coverage artifact
+      uses: actions/upload-artifact@v4
       with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        file: ./coverage.xml
+        name: coverage-files-${{ github.job }}-${{ strategy.job-index }}
+        include-hidden-files: true
+        path: .coverage
+        if-no-files-found: error
 
   slow-tests-success:
     if: always()
@@ -231,3 +233,38 @@ jobs:
       - name: Check slow tests all succeeded
         if: ${{ needs.fuzzing.result != 'success' }}
         run: exit 1
+
+  consolidate-coverage:
+    # Consolidate code coverage using `coverage combine` and upload
+    # to the codecov app
+    runs-on: ubuntu-latest
+    needs: [tests, fuzzing]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.11"
+        cache: "pip"
+
+    - name: Install coverage
+      run: pip install coverage
+
+    - name: Download coverage artifacts
+      uses: actions/download-artifact@v4
+      with:
+        pattern: coverage-files-*
+        path: coverage-files
+
+    - name: Combine coverage
+      run: |
+        coverage combine coverage-files/**/.coverage
+        coverage xml
+
+    - name: Upload Coverage
+      uses: codecov/codecov-action@v5
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        file: coverage.xml

setup.cfg

@@ -33,3 +33,15 @@ markers =
 	fuzzing: Run Hypothesis fuzz test suite (deselect with '-m "not fuzzing"')
 	requires_evm_version(version): Mark tests that require at least a specific EVM version and would throw `EvmVersionException` otherwise
 	venom_xfail: mark a test case as a regression (expected to fail) under the venom pipeline
+
+
+[coverage:run]
+branch = True
+source = vyper
+
+# this is not available on the CI step that performs `coverage combine`
+omit = vyper/version.py
+
+# allow `coverage combine` to combine reports from heterogeneous OSes.
+# (mainly important for consolidating coverage reports in the CI).
+relative_files = True