Update benchmark workflow

PeterLombaers · Mar 28, 2024 · df5d731 · df5d731
1 parent 3418303
commit df5d731
Show file tree

Hide file tree

Showing 2 changed files with 39 additions and 29 deletions.
diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml
@@ -14,10 +14,9 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Set up Rust
-      uses: actions-rs/toolchain@v1
+      uses: dtolnay/rust-toolchain@stable
       with:
         toolchain: stable
-        override: true
     - name: Install Python
       uses: actions/setup-python@v5
       with:
@@ -53,18 +52,20 @@ jobs:
         command: test
     - name: Prepare benchmark results file with system info
       run: |
-        echo "============================= Test Session Info ==============================" >> benches/results.txt
-        echo "System Information:" >> benches/results.txt
-        echo "Number of Cores: $(nproc)" >> benches/results.txt
-        echo "============================== Rust Benchmarks ===============================" >> benches/results.txt
-
+        echo "============================= Test Session Info ==============================" | tee benches/results.txt
+        echo "System Information:" | tee -a benches/results.txt
+        echo "Number of Cores: $(nproc)" | tee -a benches/results.txt
+        echo "CPU Model: $(lscpu | grep "Model name" | cut -d ':' -f 2 | xargs)" | tee -a benches/results.txt
+        echo "Memory Information:" | tee -a benches/results.txt
+        free -h | tee -a benches/results.txt
+        echo " "
     - name: Run Rust benchmarks and write results
       run: |
-        cargo bench | tee benches/results.txt
+        echo "============================== Rust Benchmarks ===============================" | tee -a benches/results.txt
+        cargo bench | tee -a benches/results.txt
     - name: Run Python benchmarks and append results
       run: |
         source .venv/bin/activate
-        pip freeze
         pytest --benchmark-min-rounds=10 benches/python_bench.py | tee -a benches/results.txt
     - name: Commit results to repository
       run: |

diff --git a/benches/results.txt b/benches/results.txt
@@ -1,3 +1,12 @@
+============================= Test Session Info ==============================
+System Information:
+Number of Cores: 4
+CPU Model: AMD EPYC 7763 64-Core Processor
+Memory Information:
+               total        used        free      shared  buff/cache   available
+Mem:            15Gi       758Mi        11Gi        25Mi       3.7Gi        14Gi
+Swap:          4.0Gi          0B       4.0Gi
+============================== Rust Benchmarks ===============================
 
 running 14 tests
 test content_iter::tests::test_next ... ignored
@@ -18,17 +27,17 @@ test ref_iter::tests::test_take_tag ... ignored
 test result: ok. 0 passed; 0 failed; 14 ignored; 0 measured; 0 filtered out; finished in 0.00s
 
 appenzeller_herzog_handwritten
-                        time:   [6.4019 ms 6.4129 ms 6.4246 ms]
-Found 7 outliers among 100 measurements (7.00%)
-  3 (3.00%) low mild
-  1 (1.00%) high mild
-  3 (3.00%) high severe
-
-parse_reference         time:   [14.625 µs 14.841 µs 15.094 µs]
-Found 18 outliers among 100 measurements (18.00%)
-  3 (3.00%) low severe
-  7 (7.00%) high mild
-  8 (8.00%) high severe
+                        time:   [6.3250 ms 6.3319 ms 6.3388 ms]
+Found 4 outliers among 100 measurements (4.00%)
+  2 (2.00%) low mild
+  2 (2.00%) high mild
+
+parse_reference         time:   [18.767 µs 19.021 µs 19.219 µs]
+Found 10 outliers among 100 measurements (10.00%)
+  6 (6.00%) low severe
+  1 (1.00%) low mild
+  2 (2.00%) high mild
+  1 (1.00%) high severe
 
 ============================= test session starts ==============================
 platform linux -- Python 3.11.8, pytest-8.1.1, pluggy-1.4.0
@@ -41,16 +50,16 @@ collected 4 items
 benches/python_bench.py ....                                             [100%]
 
 
-------------------------------------------------------------------------------------------- benchmark: 4 tests ------------------------------------------------------------------------------------------
-Name (time in ms)                       Min                 Max                Mean             StdDev              Median                IQR            Outliers       OPS            Rounds  Iterations
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-test_appenzeller_herzog_rust         9.3645 (1.0)       10.3763 (1.0)        9.8526 (1.0)       0.1805 (1.0)        9.8396 (1.0)       0.2570 (1.0)          33;1  101.4962 (1.0)          89           1
-test_appenzeller_herzog_python      45.7026 (4.88)      54.7840 (5.28)      47.4704 (4.82)      2.3241 (12.87)     46.5769 (4.73)      1.5352 (5.97)          2;2   21.0658 (0.21)         21           1
-test_brouwer_rust                  202.3495 (21.61)    209.7994 (20.22)    205.0485 (20.81)     2.5606 (14.18)    204.2841 (20.76)     3.4019 (13.24)         3;0    4.8769 (0.05)         10           1
-test_brouwer_python                783.0379 (83.62)    814.2790 (78.48)    801.6843 (81.37)    10.6943 (59.24)    802.7930 (81.59)    12.2996 (47.86)         3;0    1.2474 (0.01)         10           1
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+------------------------------------------------------------------------------------------ benchmark: 4 tests -----------------------------------------------------------------------------------------
+Name (time in ms)                       Min                 Max                Mean            StdDev              Median               IQR            Outliers       OPS            Rounds  Iterations
+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+test_appenzeller_herzog_rust         8.4712 (1.0)        9.7380 (1.0)        9.3421 (1.0)      0.1338 (1.0)        9.3529 (1.0)      0.1113 (1.0)          13;4  107.0421 (1.0)          91           1
+test_appenzeller_herzog_python      41.5709 (4.91)      54.3731 (5.58)      47.5820 (5.09)     2.3792 (17.78)     47.2261 (5.05)     0.6837 (6.14)          3;3   21.0164 (0.20)         22           1
+test_brouwer_rust                  202.8412 (23.94)    210.3107 (21.60)    204.8059 (21.92)    2.0718 (15.48)    204.1495 (21.83)    0.9228 (8.29)          1;1    4.8827 (0.05)         10           1
+test_brouwer_python                782.6216 (92.39)    798.1454 (81.96)    793.0815 (84.89)    5.4265 (40.55)    795.3374 (85.04)    7.1895 (64.61)         2;0    1.2609 (0.01)         10           1
+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 
 Legend:
   Outliers: 1 Standard Deviation from Mean; 1.5 IQR (InterQuartile Range) from 1st Quartile and 3rd Quartile.
   OPS: Operations Per Second, computed as 1 / Mean
-============================== 4 passed in 15.12s ==============================
+============================== 4 passed in 15.06s ==============================