diff --git a/.github/workflows/ci-bench.yml b/.github/workflows/ci-bench.yml index ef8c6e253..1f23053f5 100644 --- a/.github/workflows/ci-bench.yml +++ b/.github/workflows/ci-bench.yml @@ -22,9 +22,7 @@ concurrency: jobs: benchmark: name: Performance benchmarks - runs-on: ubuntu-latest - steps: - uses: actions/checkout@v4 - uses: actions-rust-lang/setup-rust-toolchain@v1 @@ -44,7 +42,6 @@ jobs: ./run_benchmarks.sh cd .. - - name: Publish benchmark result uses: benchmark-action/github-action-benchmark@v1 with: diff --git a/examples/run_benchmarks.sh b/examples/run_benchmarks.sh index 28077783c..ab9f91d44 100755 --- a/examples/run_benchmarks.sh +++ b/examples/run_benchmarks.sh @@ -1,17 +1,18 @@ #!/bin/bash # This script runs all the benchmarks in the current directory except the ones in the exclusion list. -# Current script measures peak memory and wall time for each benchmark. +# It uses GNU time to measure peak memory (maximum resident size) and wall time for each benchmark. +# The benchmarks in the exclusion list are not run since they require a large amount of memory and result +# in the github runners getting killed. +# The results are stored in a JSON file $output_file. # -# TODOs - clarify requirements: -# - Are we measuring the correct the CPU time? -# - Build time is ignored and not included in the CPU time -# - Are we measuring the correct peak memory usage? MRS? Heap? Stack? -# - What about the benchmarks that require a lot of memory? These crash the github runners +# Nuances: +# - Measures wall time, but CPU time might be desired in some cases. +# - Build time is excluded by building the benchmarks before running them. +# - Maximum resident size is being used as a surrogate to peak memory usage. # Define the exclude list -# exclusion_list=("collatz" "overflow" "sha3-chain") -exclusion_list=("fibonacci" "multi-function" "collatz" "overflow" "sha3-chain" "stdlib" "alloc" "sha3-ex" "sha2-ex" "sha2-chain" "memory-ops") +exclusion_list=("collatz" "overflow" "sha3-chain") # JSON file to store results output_file="benchmark_results.json" @@ -20,6 +21,7 @@ output_file="benchmark_results.json" # $1: name of the benchmark # $2: metric value # $3: unit of the test +# $4: is_last: boolean to check if it's the last entry function write_to_json() { # Append execution time to JSON file printf " {\n" >>"$output_file" @@ -28,10 +30,13 @@ function write_to_json() { printf " \"unit\": \"%s\",\n" "$3" >>"$output_file" printf " \"value\": %.4f,\n" "$2" >>"$output_file" printf " \"extra\": \"\"\n" >>"$output_file" - + if [ "$4" = false ]; then + printf " },\n" >>"$output_file" + else + printf " }\n" >>"$output_file" + fi } - # Initialize an array to hold directories test_directories=() @@ -64,38 +69,23 @@ for i in "${!test_directories[@]}"; do cargo build --release -p "$file" echo "Running $file" - # command="cargo run --release -p \"$file\"" - # Use `time` to measure execution time - # Use `time` to measure execution time and memory. # Output the information in the below custom format: # wall: 0:02.00 (HH:MM:SS) # real: 2.00 s # MRS: 1964 KB - - # exec_time=$( (time -p eval "$command") 2>&1 | grep real | awk '{print $2}') - # output=$(/usr/bin/time "${command}" 2>&1) output=$(/usr/bin/time -f "wall: %E (HH:MM:SS)\nreal: %e s\nMRS: %M KB" cargo run --release -p "$file" 2>&1) - # output=$(time eval "$command" 2>&1) - echo "$output" # Extract 'real' time value using awk exec_time=$(echo "$output" | awk '/real:/ {print $2}') # in seconds - echo "$exec_time" # Extract 'MRS' value using awk mem_used=$(echo "$output" | awk '/MRS:/ {print $2}') # in KB - echo "$mem_used" - + # Append execution time to JSON file - write_to_json "${file}-time" "$exec_time" "s" - printf " },\n" >>"$output_file" - - write_to_json "${file}-mem" "$mem_used" "KB" + write_to_json "${file}-time" "$exec_time" "s" false # Add a comma if it's not the last entry - if [ "$i" -lt $((${#test_directories[@]} - 1)) ]; then - printf " },\n" >>"$output_file" - else - printf " }\n" >>"$output_file" - fi + is_last=$([ "$i" -lt $((${#test_directories[@]} - 1)) ] && echo false || echo true) + # Append memory usage to JSON file + write_to_json "${file}-mem" "$mem_used" "KB" $is_last done