Skip to content

Commit

Permalink
ci: Don't fail the bench job if the cache is missing
Browse files Browse the repository at this point in the history
Thanks you GitHub for wiping our cache so we could debug this issue.
  • Loading branch information
larseggert committed Feb 11, 2025
1 parent 1d9de16 commit 39fa41f
Showing 1 changed file with 31 additions and 28 deletions.
59 changes: 31 additions & 28 deletions .github/workflows/bench.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ on:
workflow_call:
workflow_dispatch:
schedule:
# Run at 1 AM each day, so there is a `main`-branch baseline in the cache.
- cron: '0 1 * * *'
# Run at minute 0 past every 8th hour, so there is a `main`-branch baseline in the cache.
- cron: '0 */8 * * *'
env:
CARGO_PROFILE_BENCH_BUILD_OVERRIDE_DEBUG: true
CARGO_PROFILE_RELEASE_DEBUG: true
Expand Down Expand Up @@ -245,38 +245,41 @@ jobs:
perf $PERF_OPT -o "$FILENAME.client.perf" $CMD > /dev/null 2>&1
kill $PID
grep -Ev '^\|(:| Command)' < "hyperfine/$FILENAME.md" | \
sed -E 's/`//g; s/,/ \| /g;' | cut -f1-8 -d\| | tr -d '\n' >> steps.md
# Figure out if any performance difference to `main` is statistically relevant, and indicate that.
BASELINE="hyperfine-main/$FILENAME.json"
RESULT="hyperfine/$FILENAME.json"
BASELINE_MEAN=$(jq -r '.results[0].mean' "$BASELINE")
MEAN=$(jq -r '.results[0].mean' "$RESULT")
# Even though we tell hyperfine to use milliseconds, it still outputs in seconds when dumping to JSON.
DELTA=$(bc -l <<< "($MEAN - $BASELINE_MEAN) * 1000")
PERCENT=$(bc -l <<< "($MEAN - $BASELINE_MEAN) / ($BASELINE_MEAN + $MEAN)/2 * 100")
# If a performance change is statistically significant, highlight it.
jq -r '.results[0].times[]' "$BASELINE" > baseline.txt
jq -r '.results[0].times[]' "$RESULT" > result.txt
if ! Rscript welch.R baseline.txt result.txt 2> /dev/null; then
if (( $(bc -l <<< "$DELTA > 0") )); then
echo "Performance has regressed: $BASELINE_MEAN -> $MEAN"
SYMBOL=":broken_heart:"
FORMAT='**'
if [ -e "$BASELINE" ]; then
RESULT="hyperfine/$FILENAME.json"
BASELINE_MEAN=$(jq -r '.results[0].mean' "$BASELINE")
MEAN=$(jq -r '.results[0].mean' "$RESULT")
# Even though we tell hyperfine to use milliseconds, it still outputs in seconds when dumping to JSON.
DELTA=$(bc -l <<< "($MEAN - $BASELINE_MEAN) * 1000")
PERCENT=$(bc -l <<< "($MEAN - $BASELINE_MEAN) / ($BASELINE_MEAN + $MEAN)/2 * 100")
# If a performance change is statistically significant, highlight it.
jq -r '.results[0].times[]' "$BASELINE" > baseline.txt
jq -r '.results[0].times[]' "$RESULT" > result.txt
if ! Rscript welch.R baseline.txt result.txt 2> /dev/null; then
if (( $(bc -l <<< "$DELTA > 0") )); then
echo "Performance has regressed: $BASELINE_MEAN -> $MEAN"
SYMBOL=":broken_heart:"
FORMAT='**'
else
echo "Performance has improved: $BASELINE_MEAN -> $MEAN"
SYMBOL=":green_heart:"
FORMAT='**'
fi
else
echo "Performance has improved: $BASELINE_MEAN -> $MEAN"
SYMBOL=":green_heart:"
FORMAT='**'
echo "No statistically significant change: $BASELINE_MEAN -> $MEAN"
fi
printf "| %s %s%.1f%s | %s%.1f%%%s |\n" \
"$SYMBOL" "$FORMAT" "$DELTA" "$FORMAT" "$FORMAT" "$PERCENT" "$FORMAT" >> steps.md
else
echo "No statistically significant change: $BASELINE_MEAN -> $MEAN"
echo No cached baseline from main found.
echo '| :question: | :question: |' >> steps.md
fi
{
grep -Ev '^\|(:| Command)' < "hyperfine/$FILENAME.md" | \
sed -E 's/`//g; s/,/ \| /g;' | cut -f1-8 -d\| | tr -d '\n'
printf "| %s %s%.1f%s | %s%.1f%%%s |\n" \
"$SYMBOL" "$FORMAT" "$DELTA" "$FORMAT" "$FORMAT" "$PERCENT" "$FORMAT"
} >> steps.md
done
done
done
Expand Down

0 comments on commit 39fa41f

Please sign in to comment.