From 7ef708138ab6ef47394bffdead333ceee747086f Mon Sep 17 00:00:00 2001 From: Dino Pacandi Date: Thu, 8 Aug 2024 16:53:46 +0200 Subject: [PATCH 1/4] Init commit --- .github/workflows/benchmark.yml | 3 +++ scripts/run_benchmarks.sh | 13 +++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 4fe6ea1eb2..a9984e785b 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -109,6 +109,9 @@ jobs: - name: Check targets are installed correctly run: rustup target list --installed + - name: Install frame-omni bencher tool + run: cargo install frame-omni-bencher + - name: Execute benchmarking run: | mkdir -p ./benchmark-results diff --git a/scripts/run_benchmarks.sh b/scripts/run_benchmarks.sh index c739e4762d..38e04bfcd6 100755 --- a/scripts/run_benchmarks.sh +++ b/scripts/run_benchmarks.sh @@ -73,11 +73,12 @@ done if [ "$skip_build" != true ] then echo "[+] Compiling astar-collator benchmarks..." - CARGO_PROFILE_RELEASE_LTO=true RUSTFLAGS="-C codegen-units=1" cargo build --release --verbose --features=runtime-benchmarks + CARGO_PROFILE_RELEASE_LTO=true RUSTFLAGS="-C codegen-units=1" cargo build --release --verbose --features=runtime-benchmarks \ + -p astar-runtime -p shiden-runtime -p shibuya-runtime fi # The executable to use. -ASTAR_COLLATOR=./target/release/astar-collator +BENCHMARK-TOOL="frame-omni-bencher v1" # Manually exclude some pallets. EXCLUDED_PALLETS=( @@ -86,7 +87,7 @@ EXCLUDED_PALLETS=( # Load all pallet names in an array. ALL_PALLETS=($( - $ASTAR_COLLATOR benchmark pallet --list --chain=$chain |\ + $BENCHMARK-TOOL benchmark pallet --list --runtime ./target/release/wbuild/astar-runtime/${chain}_runtime.compact.compressed.wasm |\ tail -n+2 |\ cut -d',' -f1 |\ sort |\ @@ -120,8 +121,8 @@ for chain in ${chains//,/ }; do echo "[+] Benchmarking $PALLET with weight file $WEIGHT_FILE"; OUTPUT=$( - $ASTAR_COLLATOR benchmark pallet \ - --chain=$chain \ + $BENCHMARK-TOOL benchmark pallet \ + --runtime ./target/release/wbuild/astar-runtime/${chain}_runtime.compact.compressed.wasm \ --steps=50 \ --repeat=20 \ --pallet="$PALLET" \ @@ -139,7 +140,7 @@ for chain in ${chains//,/ }; do echo "[+] Benchmarking the machine..." OUTPUT=$( - $ASTAR_COLLATOR benchmark machine --chain=$chain 2>&1 + $BENCHMARK-TOOL benchmark machine --chain=$chain 2>&1 ) if [ $? -ne 0 ]; then echo "[-] Failed the machine benchmark" From 60b7ed0839766d9f3a97b7e9194215bc0ac3eee4 Mon Sep 17 00:00:00 2001 From: Dino Pacandi Date: Fri, 9 Aug 2024 06:22:39 +0200 Subject: [PATCH 2/4] Disable machine benchmarks --- scripts/run_benchmarks.sh | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/scripts/run_benchmarks.sh b/scripts/run_benchmarks.sh index 38e04bfcd6..2ec32cb6db 100755 --- a/scripts/run_benchmarks.sh +++ b/scripts/run_benchmarks.sh @@ -138,16 +138,19 @@ for chain in ${chains//,/ }; do fi done - echo "[+] Benchmarking the machine..." - OUTPUT=$( - $BENCHMARK-TOOL benchmark machine --chain=$chain 2>&1 - ) - if [ $? -ne 0 ]; then - echo "[-] Failed the machine benchmark" - echo "$OUTPUT" >> "$ERR_FILE" - else - echo "$OUTPUT" >> "$output_path/$chain/$chain-machine-bench.txt" - fi + # Disabled for now - command isn't available (yet). + # With latest changes we also benchmark the HW each time the client starts. + # + # echo "[+] Benchmarking the machine..." + # OUTPUT=$( + # $BENCHMARK-TOOL benchmark machine --runtime ./target/release/wbuild/astar-runtime/${chain}_runtime.compact.compressed.wasm 2>&1 + # ) + # if [ $? -ne 0 ]; then + # echo "[-] Failed the machine benchmark" + # echo "$OUTPUT" >> "$ERR_FILE" + # else + # echo "$OUTPUT" >> "$output_path/$chain/$chain-machine-bench.txt" + # fi # Check if the error file exists. if [ -f "$ERR_FILE" ]; then From ec8d25464d3f27861fc6cf132f7ee818d9a109c4 Mon Sep 17 00:00:00 2001 From: Dino Pacandi Date: Fri, 9 Aug 2024 06:46:34 +0200 Subject: [PATCH 3/4] Fix --- scripts/run_benchmarks.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/run_benchmarks.sh b/scripts/run_benchmarks.sh index 2ec32cb6db..4b51f2a9de 100755 --- a/scripts/run_benchmarks.sh +++ b/scripts/run_benchmarks.sh @@ -30,7 +30,7 @@ while getopts 'bc:fo:p:v' flag; do ;; c) chains=$(echo ${OPTARG} | tr '[:upper:]' '[:lower:]') - chains_default=("astar-dev" "shiden-dev" "shibuya-dev" "dev") + chains_default=("astar" "shiden" "shibuya") for chain in ${chains//,/ }; do if [[ ! " ${chains_default[*]} " =~ " ${chain} " ]]; then echo "Chain input is invalid. ${chain} not included in ${chains_default[*]}" @@ -78,7 +78,7 @@ then fi # The executable to use. -BENCHMARK-TOOL="frame-omni-bencher v1" +BENCHMARK_TOOL="frame-omni-bencher v1" # Manually exclude some pallets. EXCLUDED_PALLETS=( @@ -87,7 +87,7 @@ EXCLUDED_PALLETS=( # Load all pallet names in an array. ALL_PALLETS=($( - $BENCHMARK-TOOL benchmark pallet --list --runtime ./target/release/wbuild/astar-runtime/${chain}_runtime.compact.compressed.wasm |\ + $BENCHMARK_TOOL benchmark pallet --list --runtime ./target/release/wbuild/${chain}-runtime/${chain}_runtime.compact.compressed.wasm |\ tail -n+2 |\ cut -d',' -f1 |\ sort |\ @@ -121,8 +121,8 @@ for chain in ${chains//,/ }; do echo "[+] Benchmarking $PALLET with weight file $WEIGHT_FILE"; OUTPUT=$( - $BENCHMARK-TOOL benchmark pallet \ - --runtime ./target/release/wbuild/astar-runtime/${chain}_runtime.compact.compressed.wasm \ + $BENCHMARK_TOOL benchmark pallet \ + --runtime ./target/release/wbuild/${chain}-runtime/${chain}_runtime.compact.compressed.wasm \ --steps=50 \ --repeat=20 \ --pallet="$PALLET" \ @@ -143,7 +143,7 @@ for chain in ${chains//,/ }; do # # echo "[+] Benchmarking the machine..." # OUTPUT=$( - # $BENCHMARK-TOOL benchmark machine --runtime ./target/release/wbuild/astar-runtime/${chain}_runtime.compact.compressed.wasm 2>&1 + # $BENCHMARK_TOOL benchmark machine --runtime ./target/release/wbuild/astar-runtime/${chain}_runtime.compact.compressed.wasm 2>&1 # ) # if [ $? -ne 0 ]; then # echo "[-] Failed the machine benchmark" From cdcfd716ba7bcfbf5bc153ddc6f050b63aef9581 Mon Sep 17 00:00:00 2001 From: Dino Pacandi Date: Fri, 9 Aug 2024 07:26:42 +0200 Subject: [PATCH 4/4] README --- .github/workflows/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index c3ba6487e0..25c2dc68cc 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -8,22 +8,22 @@ Latest commit hash in PR branch will be built and used for benchmarking. ### How to execute Usage ``` -# [chain names] - Use "," for multiple runtimes. Available values are: astar-dev, shiden-dev, shibuya-dev, dev +# [chain names] - Use "," for multiple runtimes. Available values are: astar, shiden, shibuya # [pallet names] - Use "," for multiple pallets, "all" for all pallets /bench [chain names] [pallet names] ``` ``` # benchmark a pallet -/bench astar-dev pallet_balances +/bench astar pallet_balances # benchmark multiple pallets -/bench astar-dev pallet_balances,pallet_dapps_staking +/bench astar pallet_balances,pallet_dapps_staking # benchmark all pallets -/bench astar-dev all +/bench astar all # benchmark multiple runtimes with multiple pallets -/bench astar-dev,shibuya-dev pallet_balances,pallet_dapps_staking +/bench astar,shibuya pallet_balances,pallet_dapps_staking ```