From 12386a32c2179727a6b4779d0ed96373571cfafd Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 11 Mar 2024 15:25:40 +0200
Subject: [PATCH 01/12] ci: Benchmark NewReno and Cubic

---
 .github/workflows/bench.yml | 45 +++++++++++++++++++++----------------
 1 file changed, 26 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 176a80deac..2376467908 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -77,19 +77,25 @@ jobs:
 
       - name: Profile client/server transfer
         run: |
-          { mkdir server; \
-            cd server; \
-            taskset -c 0 nice -n -20 \
-            cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" \
-              --bin neqo-server -- --db ../test-fixture/db $HOST:4433 || true; } &
-          mkdir client; \
-            cd client; \
-            time taskset -c 1 nice -n -20 \
-            cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" \
-              --bin neqo-client -- --output-dir . https://$HOST:4433/$SIZE
-          killall -INT neqo-server
-          cd ${{ github.workspace }}
-          [ "$(wc -c < client/"$SIZE")" -eq "$SIZE" ] || exit 1
+          for CC in newreno cubic; do
+            { mkdir server; \
+              cd server; \
+              taskset -c 0 nice -n -20 \
+              cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" \
+                --bin neqo-server -- --db ../test-fixture/db --cc "$CC" $HOST:4433 || true; } &
+            mkdir client; \
+              cd client; \
+              time taskset -c 1 nice -n -20 \
+              cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" \
+                --bin neqo-client -- --output-dir . --cc "$CC" https://$HOST:4433/$SIZE
+            killall -INT neqo-server
+            cd ${{ github.workspace }}
+            [ "$(wc -c < client/"$SIZE")" -eq "$SIZE" ] || exit 1
+            for PEER in server client; do
+              mv "$PEER/flamegraph.svg" "$PEER-$CC.svg"
+              mv "$PEER/perf.data" "$PEER-$CC.perf"
+            done
+          done
         env:
           HOST: localhost
           SIZE: 1073741824 # 1 GB
@@ -101,14 +107,15 @@ jobs:
 
       - name: Convert for profiler.firefox.com
         run: |
+          rm neqo.svg
           perf script -i perf.data -F +pid > transfer.perf &
-          perf script -i client/perf.data -F +pid > client.perf &
-          perf script -i server/perf.data -F +pid > server.perf &
-          wait
           mv flamegraph.svg transfer.svg
-          mv client/flamegraph.svg client.svg
-          mv server/flamegraph.svg server.svg
-          rm neqo.svg
+          for CC in newreno cubic; do
+            for PEER in server client; do
+              perf script -i "$PEER-$CC.perf" -F +pid > "$PEER-$CC.fx.perf" &
+            done
+          done
+          wait
 
       - name: Cache main-branch results
         if: github.ref == 'refs/heads/main'

From de64b3e1a0108a35af58cb8d6d433a576eaa13c4 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 11 Mar 2024 16:00:19 +0200
Subject: [PATCH 02/12] Wait for output before continuing

---
 .github/workflows/bench.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 2376467908..36c28b9653 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -77,6 +77,8 @@ jobs:
 
       - name: Profile client/server transfer
         run: |
+          cd ${{ github.workspace }}
+          rm -rf client server
           for CC in newreno cubic; do
             { mkdir server; \
               cd server; \
@@ -91,7 +93,8 @@ jobs:
             killall -INT neqo-server
             cd ${{ github.workspace }}
             [ "$(wc -c < client/"$SIZE")" -eq "$SIZE" ] || exit 1
-            for PEER in server client; do
+            while [ ! -f server/flamegraph.svg ]; do sleep 1; done
+            for PEER in client server; do
               mv "$PEER/flamegraph.svg" "$PEER-$CC.svg"
               mv "$PEER/perf.data" "$PEER-$CC.perf"
             done
@@ -111,7 +114,7 @@ jobs:
           perf script -i perf.data -F +pid > transfer.perf &
           mv flamegraph.svg transfer.svg
           for CC in newreno cubic; do
-            for PEER in server client; do
+            for PEER in client server; do
               perf script -i "$PEER-$CC.perf" -F +pid > "$PEER-$CC.fx.perf" &
             done
           done

From bc7b99fe0b156294590dda561318283693b88df6 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 11 Mar 2024 16:29:14 +0200
Subject: [PATCH 03/12] Fix

---
 .github/workflows/bench.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 5b4c53284b..e0fad65174 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -77,8 +77,6 @@ jobs:
 
       - name: Profile client/server transfer
         run: |
-          cd ${{ github.workspace }}
-          rm -rf client server
           for CC in newreno cubic; do
             { mkdir server; \
               cd server; \
@@ -110,7 +108,6 @@ jobs:
 
       - name: Convert for profiler.firefox.com
         run: |
-          rm neqo.svg
           perf script -i perf.data -F +pid > transfer.perf &
           mv flamegraph.svg transfer.svg
           for CC in newreno cubic; do
@@ -119,6 +116,7 @@ jobs:
             done
           done
           wait
+          rm neqo.svg
 
       - name: Generate perf reports
         run: |

From bd742af1b5b56422294a01a43ed50388d1bc31f5 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 11 Mar 2024 17:00:34 +0200
Subject: [PATCH 04/12] mkdir -p

---
 .github/workflows/bench.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index e0fad65174..345a25fbe7 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -78,12 +78,12 @@ jobs:
       - name: Profile client/server transfer
         run: |
           for CC in newreno cubic; do
-            { mkdir server; \
+            { mkdir -p server; \
               cd server; \
               taskset -c 0 nice -n -20 \
               cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" \
                 --bin neqo-server -- --db ../test-fixture/db --cc "$CC" $HOST:4433 || true; } &
-            mkdir client; \
+            mkdir -p client; \
               cd client; \
               time taskset -c 1 nice -n -20 \
               cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" \

From 9ea3a99119e9ad19c0e7157b8218d0a631d1de95 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Tue, 12 Mar 2024 09:15:05 +0200
Subject: [PATCH 05/12] Address comments from @martinthomson

---
 .github/workflows/bench.yml | 32 +++++++++++---------------------
 1 file changed, 11 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 6980af5534..ff4c3f979d 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -12,7 +12,7 @@ env:
   RUST_BACKTRACE: 1
   TOOLCHAIN: nightly
   RUSTFLAGS: -C link-arg=-fuse-ld=lld -C link-arg=-Wl,--no-rosegment, -C force-frame-pointers=yes
-  PERF_CMD: record -o perf.data -F997 --call-graph fp -g
+  PERF_OPT: record -F997 --call-graph fp -g
 
 jobs:
   bench:
@@ -72,7 +72,8 @@ jobs:
           mv target/criterion target/criterion-bench
           mv target/criterion-transfer-profile target/criterion || true
           taskset -c 0 nice -n -20 \
-            cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" --features bench --bench transfer -- \
+            cargo "+$TOOLCHAIN" flamegraph -o transfer.svg -v -c "$PERF_OPT -o transfer.perf" \
+              --features bench --bench transfer -- \
               --bench --exact "Run multiple transfers with varying seeds"
           # And now restore the directories.
           mv target/criterion target/criterion-transfer-profile
@@ -81,24 +82,14 @@ jobs:
       - name: Profile client/server transfer
         run: |
           for CC in newreno cubic; do
-            { mkdir -p server; \
-              cd server; \
-              taskset -c 0 nice -n -20 \
-              cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" \
-                --bin neqo-server -- --db ../test-fixture/db --cc "$CC" "$HOST:4433" || true; } &
-            mkdir -p client; \
-              cd client; \
-              time taskset -c 1 nice -n -20 \
-              cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" \
-                --bin neqo-client -- --output-dir . --cc "$CC" "https://$HOST:4433/$SIZE"
+            taskset -c 0 nice -n -20 \
+            cargo "+$TOOLCHAIN" flamegraph -o "server-$CC.svg" -v -c "$PERF_OPT -o server-$CC.perf" \
+              --bin neqo-server -- --db ../test-fixture/db --cc "$CC" "$HOST:4433" &
+            time taskset -c 1 nice -n -20 \
+            cargo "+$TOOLCHAIN" flamegraph -o "client-$CC.svg" -v -c "$PERF_OPT -o client-$CC.perf" \
+              --bin neqo-client -- --output-dir . --cc "$CC" "https://$HOST:4433/$SIZE"
             killall -INT neqo-server
-            cd ${{ github.workspace }}
             [ "$(wc -c < client/"$SIZE")" -eq "$SIZE" ] || exit 1
-            while [ ! -f server/flamegraph.svg ]; do sleep 1; done
-            for PEER in client server; do
-              mv "$PEER/flamegraph.svg" "$PEER-$CC.svg"
-              mv "$PEER/perf.data" "$PEER-$CC.perf"
-            done
           done
         env:
           HOST: localhost
@@ -111,8 +102,7 @@ jobs:
 
       - name: Convert for profiler.firefox.com
         run: |
-          perf script -i perf.data -F +pid > transfer.perf &
-          mv flamegraph.svg transfer.svg
+          perf script -i transfer.perf -F +pid > transfer.fx.perf &
           for CC in newreno cubic; do
             for PEER in client server; do
               perf script -i "$PEER-$CC.perf" -F +pid > "$PEER-$CC.fx.perf" &
@@ -123,7 +113,7 @@ jobs:
 
       - name: Generate perf reports
         run: |
-          perf report -i perf.data --no-children --stdio > transfer.perf.txt &
+          perf report -i transfer.perf --no-children --stdio > transfer.perf.txt &
           for CC in newreno cubic; do
             for PEER in client server; do
               perf report -i "$PEER-$CC.perf" --no-children --stdio > "$PEER-$CC.txt" &

From 02620a7c48bdfcad703a47b25f324554edb5de7c Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Tue, 12 Mar 2024 09:57:33 +0200
Subject: [PATCH 06/12] Try to kill via `$!`

---
 .github/workflows/bench.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index ff4c3f979d..e9fb6fb0da 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -85,10 +85,11 @@ jobs:
             taskset -c 0 nice -n -20 \
             cargo "+$TOOLCHAIN" flamegraph -o "server-$CC.svg" -v -c "$PERF_OPT -o server-$CC.perf" \
               --bin neqo-server -- --db ../test-fixture/db --cc "$CC" "$HOST:4433" &
+            PID=$!
             time taskset -c 1 nice -n -20 \
             cargo "+$TOOLCHAIN" flamegraph -o "client-$CC.svg" -v -c "$PERF_OPT -o client-$CC.perf" \
               --bin neqo-client -- --output-dir . --cc "$CC" "https://$HOST:4433/$SIZE"
-            killall -INT neqo-server
+            killall -INT $PID
             [ "$(wc -c < client/"$SIZE")" -eq "$SIZE" ] || exit 1
           done
         env:

From 9d3a8b792e8fed127503fd93ff0406c0b462768d Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Tue, 12 Mar 2024 13:57:51 +0200
Subject: [PATCH 07/12] Use temp dir, and fix path error

---
 .github/workflows/bench.yml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 5c9fac7f10..7897b605bd 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -77,15 +77,17 @@ jobs:
       - name: Profile client/server transfer
         run: |
           for CC in newreno cubic; do
+            TMP=$(mktemp -d)
             taskset -c 0 nice -n -20 \
             cargo "+$TOOLCHAIN" flamegraph -o "server-$CC.svg" -v -c "$PERF_OPT -o server-$CC.perf" \
-              --bin neqo-server -- --db ../test-fixture/db --cc "$CC" "$HOST:4433" &
+              --bin neqo-server -- --cc "$CC" "$HOST:4433" &
             PID=$!
             time taskset -c 1 nice -n -20 \
             cargo "+$TOOLCHAIN" flamegraph -o "client-$CC.svg" -v -c "$PERF_OPT -o client-$CC.perf" \
-              --bin neqo-client -- --output-dir . --cc "$CC" "https://$HOST:4433/$SIZE"
+              --bin neqo-client -- --output-dir "$TMP" --cc "$CC" "https://$HOST:4433/$SIZE"
             killall -INT $PID
-            [ "$(wc -c < client/"$SIZE")" -eq "$SIZE" ] || exit 1
+            [ "$(wc -c < "$TMP/$SIZE")" -eq "$SIZE" ] || exit 1
+            rm -rf "$TMP"
           done
         env:
           HOST: localhost

From 407bd4ff8cc34767063b234af1737dfc2ea06339 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Tue, 12 Mar 2024 14:33:52 +0200
Subject: [PATCH 08/12] kill -> killall

Also reduce test transfer size.
---
 .github/workflows/bench.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 7897b605bd..ae678feeee 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -85,13 +85,13 @@ jobs:
             time taskset -c 1 nice -n -20 \
             cargo "+$TOOLCHAIN" flamegraph -o "client-$CC.svg" -v -c "$PERF_OPT -o client-$CC.perf" \
               --bin neqo-client -- --output-dir "$TMP" --cc "$CC" "https://$HOST:4433/$SIZE"
-            killall -INT $PID
+            kill -INT $PID
             [ "$(wc -c < "$TMP/$SIZE")" -eq "$SIZE" ] || exit 1
             rm -rf "$TMP"
           done
         env:
           HOST: localhost
-          SIZE: 1073741824 # 1 GB
+          SIZE: 536870912 # 512 MB
 
       # Re-enable turboboost, hyperthreading and use powersave governor.
       - name: Restore machine

From 11320d0095e4a8d637c40ed7eb57886582ce4a21 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Tue, 12 Mar 2024 16:11:38 +0200
Subject: [PATCH 09/12] No -INT

---
 .github/workflows/bench.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index ae678feeee..e00b1e9b24 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -85,7 +85,7 @@ jobs:
             time taskset -c 1 nice -n -20 \
             cargo "+$TOOLCHAIN" flamegraph -o "client-$CC.svg" -v -c "$PERF_OPT -o client-$CC.perf" \
               --bin neqo-client -- --output-dir "$TMP" --cc "$CC" "https://$HOST:4433/$SIZE"
-            kill -INT $PID
+            kill $PID
             [ "$(wc -c < "$TMP/$SIZE")" -eq "$SIZE" ] || exit 1
             rm -rf "$TMP"
           done

From d37e7068e65ed407e0dc38b15e67707dfc628fd5 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Tue, 12 Mar 2024 16:37:50 +0200
Subject: [PATCH 10/12] Go back to `killall`

---
 .github/workflows/bench.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index e00b1e9b24..e945cd470b 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -85,7 +85,7 @@ jobs:
             time taskset -c 1 nice -n -20 \
             cargo "+$TOOLCHAIN" flamegraph -o "client-$CC.svg" -v -c "$PERF_OPT -o client-$CC.perf" \
               --bin neqo-client -- --output-dir "$TMP" --cc "$CC" "https://$HOST:4433/$SIZE"
-            kill $PID
+            killall -v -u bench neqo-server
             [ "$(wc -c < "$TMP/$SIZE")" -eq "$SIZE" ] || exit 1
             rm -rf "$TMP"
           done

From c82ff3af7dd92aa4b2218992f85faaeff0b7cecb Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Tue, 12 Mar 2024 16:41:59 +0200
Subject: [PATCH 11/12] `killall` -> `pkill`

---
 .github/workflows/bench.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index e945cd470b..0f6cf434c3 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -81,11 +81,10 @@ jobs:
             taskset -c 0 nice -n -20 \
             cargo "+$TOOLCHAIN" flamegraph -o "server-$CC.svg" -v -c "$PERF_OPT -o server-$CC.perf" \
               --bin neqo-server -- --cc "$CC" "$HOST:4433" &
-            PID=$!
             time taskset -c 1 nice -n -20 \
             cargo "+$TOOLCHAIN" flamegraph -o "client-$CC.svg" -v -c "$PERF_OPT -o client-$CC.perf" \
               --bin neqo-client -- --output-dir "$TMP" --cc "$CC" "https://$HOST:4433/$SIZE"
-            killall -v -u bench neqo-server
+            pkill -U bench -o -x -INT neqo-server
             [ "$(wc -c < "$TMP/$SIZE")" -eq "$SIZE" ] || exit 1
             rm -rf "$TMP"
           done

From d072504e5e80e2bb541f948c6c7733c78e93cb4a Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Tue, 12 Mar 2024 17:24:52 +0200
Subject: [PATCH 12/12] Reorder things so `results.ms` is included in the
 exported artifact

---
 .github/workflows/bench.yml | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 0f6cf434c3..ef16d18862 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -125,19 +125,6 @@ jobs:
           path: ./target/criterion
           key: ${{ steps.criterion-cache.outputs.cache-primary-key }}
 
-      - name: Export perf data
-        id: export
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ github.event.repository.name }}-${{ github.sha }}
-          path: |
-            *.svg
-            *.perf
-            *.txt
-            results.*
-            target/criterion*
-          compression-level: 9
-
       - name: Format results as Markdown
         id: results
         run: |
@@ -150,6 +137,19 @@ jobs:
               > results.md
           echo '' >> results.md
 
+      - name: Export bench data
+        id: export
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ github.event.repository.name }}-${{ github.sha }}
+          path: |
+            *.svg
+            *.perf
+            *.txt
+            results.*
+            target/criterion*
+          compression-level: 9
+
       - name: Export PR comment data
         uses: ./.github/actions/pr-comment-data-export
         with: