From 1276f390385a96ac9159038addd105099b5f2931 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 3 Jul 2024 15:47:23 -0700 Subject: [PATCH 01/67] [SYCL] Introduce PoC benchmarking option into sycl-linux-run-tests --- .github/workflows/sycl-linux-run-tests.yml | 15 ++- devops/scripts/sycl-bench.sh | 105 +++++++++++++++++++++ 2 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 devops/scripts/sycl-bench.sh diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index 2f8e62eabd701..50106a65a4e8b 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -28,7 +28,7 @@ on: required: False tests_selector: description: | - Two possible options: "e2e" and "cts". + Three options: "e2e", "cts", "benchmark". type: string default: "e2e" @@ -112,6 +112,7 @@ on: options: - e2e - cts + - benchmark env: description: | @@ -369,3 +370,15 @@ jobs: grep 'exit code: [^0]' -r logs >> $GITHUB_STEP_SUMMARY exit $ret + + - name: Run benchmarks (via sycl-bench) + id: run_benchmarks + if: inputs.tests_selector == 'benchmark' + run: ./devops/scripts/sycl-bench.sh https://github.com/ianayl/sycl-bench + - name: Upload benchmark results + if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success' + uses: actions/upload-artifact@v4 + with: + name: sycl_benchmark_res_${{ steps.run_benchmarks.outputs.TIMESTAMP }} + path: ${{ steps.run_benchmarks.outputs.BENCHMARK_RESULTS }} + retention-days: 7 \ No newline at end of file diff --git a/devops/scripts/sycl-bench.sh b/devops/scripts/sycl-bench.sh new file mode 100644 index 0000000000000..9c403a3e37120 --- /dev/null +++ b/devops/scripts/sycl-bench.sh @@ -0,0 +1,105 @@ +#!/bin/sh + +# sycl-bench.sh: Benchmark dpcpp using sycl-bench + +usage () { + >&2 echo "Usage: $0 [-B ] + -B Path to clone and build sycl-bench on + +This script builds and runs benchmarks from sycl-bench." + exit 1 +} + +clone() { + mkdir -p $SYCL_BENCH_PATH + git clone $SYCL_BENCH_GIT_REPO $SYCL_BENCH_PATH || return $? +} + +build() { + cd $SYCL_BENCH_PATH + cmake -DSYCL_IMPL=dpcpp -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_RUNTIME_OUTPUT_DIRECTORY=./bin -S . -B ./build && + cmake --build ./build || return $? + cd - +} + +get_csv_col_index() { + # Determine the index of a column in a CSV given its title + # Usage: get_csv_col_index + tmp_csv_col_i="$(cat "$1" | head -n 1 | grep -o "^.*$2," | grep -o ',' | wc -l)" +} + +print_bench_res() { + # Usage: print_bench_res + if [ ! -s $1 ]; then + printf "NO OUTPUT! (Status $2)\n" | tee -a $3 + return # Do not proceed if file is empty + fi + + get_csv_col_index $1 run-time-mean + tmp_run_time_mean_i=$tmp_csv_col_i + get_csv_col_index $1 run-time-median + tmp_run_time_median_i=$tmp_csv_col_i + get_csv_col_index $1 run-time-throughput + tmp_run_time_throughput_i=$tmp_csv_col_i + + # `sycl-bench` output seems to like inserting the header multiple times. + # Here we cache the header to make sure it prints only once: + tmp_header_title="$(cat $1 | head -n 1 | sed 's/^\# Benchmark name/benchmark/')" + tmp_result="$(cat $1 | grep '^[^\#]')" + + printf "%s\n%s" "$tmp_header_title" "$tmp_result" \ + | awk -F',' -v me="$tmp_run_time_mean_i" \ + -v md="$tmp_run_time_median_i" \ + -v th="$tmp_run_time_throughput_i" \ + '{printf "%-57s %-13s %-15s %-20s\n", $1, $me, $md, $th }' \ + | tee -a $3 # Print to summary file +} + +# run sycl bench step +run() { + TIMESTAMP="$(date '+%Y%m%d_%H%M%S')" + mkdir "$SYCL_BENCH_PATH/build/bench-$TIMESTAMP/" + tmp_summary_file="$SYCL_BENCH_PATH/build/bench-$TIMESTAMP/summary.txt" + + for file in $SYCL_BENCH_PATH/build/bin/*; do + # TODO -size should not be always 256, caution + tmp_bench_output="$SYCL_BENCH_PATH/build/bench-$TIMESTAMP/$(basename $file).csv" + tmp_bench_log="$SYCL_BENCH_PATH/build/bench-$TIMESTAMP/$(basename $file).log" + + tmp_err="0" + printf "\n### Results for $(basename $file) ###\n" | tee -a $tmp_summary_file + # The pipe here suppresses errors in a way that doesn't stop github actions: + $file --output=$tmp_bench_output --no-verification --size=256 2> "$tmp_bench_log" || tmp_err=$? + print_bench_res $tmp_bench_output $tmp_err $tmp_summary_file + # Remove log if nothing logged + [ ! -s "$tmp_bench_log" ] && rm "$tmp_bench_log" || cat "$tmp_bench_log" | tee -a $tmp_summary_file + done + + # Export timestamp for later use + [ -f "$GITHUB_OUTPUT" ] && echo TIMESTAMP=$TIMESTAMP >> $GITHUB_OUTPUT +} + +compress() { + tar -I gzip -cf "$SYCL_BENCH_PATH/build/bench-$TIMESTAMP.tar.gz" -C "$SYCL_BENCH_PATH/build/bench-$TIMESTAMP" . + if [ -f "$SYCL_BENCH_PATH/build/bench-$TIMESTAMP.tar.gz" ] && [ -f "$GITHUB_OUTPUT" ]; then + echo BENCHMARK_RESULTS="$SYCL_BENCH_PATH/build/bench-$TIMESTAMP.tar.gz" >> $GITHUB_OUTPUT + fi +} + +cleanup() { + rm -r $SYCL_BENCH_PATH +} + + +[ "$#" -lt "1" ] && usage + +SYCL_BENCH_GIT_REPO="$1"; shift +SYCL_BENCH_PATH="./" +while getopts "B:" opt; do + case $opt in + B) SYCL_BENCH_PATH=$OPTARG ;; + \?) usage ;; + esac +done + +clone && build && run && compress From 754c33abb32d4adc689ba2a23512f9ad68032241 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 3 Jul 2024 15:49:46 -0700 Subject: [PATCH 02/67] [SYCL] Fix EOF character for sycl-linux-run-tests --- .github/workflows/sycl-linux-run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index 50106a65a4e8b..c9b72343bf6ee 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -381,4 +381,4 @@ jobs: with: name: sycl_benchmark_res_${{ steps.run_benchmarks.outputs.TIMESTAMP }} path: ${{ steps.run_benchmarks.outputs.BENCHMARK_RESULTS }} - retention-days: 7 \ No newline at end of file + retention-days: 7 From 9981c3abdedaaaf00d3dcac54b01ceb2e3eb394f Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 3 Jul 2024 16:02:23 -0700 Subject: [PATCH 03/67] [SYCL] Fix: Make benchmarking script executable --- devops/scripts/sycl-bench.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 devops/scripts/sycl-bench.sh diff --git a/devops/scripts/sycl-bench.sh b/devops/scripts/sycl-bench.sh old mode 100644 new mode 100755 From 3ed35ca59bccb2016a061dd4e8e4992df08d4c95 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 3 Jul 2024 16:06:33 -0700 Subject: [PATCH 04/67] [SYCL] Bug fix on sycl-bench.sh --- devops/scripts/sycl-bench.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops/scripts/sycl-bench.sh b/devops/scripts/sycl-bench.sh index 9c403a3e37120..4b00a60f178aa 100755 --- a/devops/scripts/sycl-bench.sh +++ b/devops/scripts/sycl-bench.sh @@ -94,7 +94,7 @@ cleanup() { [ "$#" -lt "1" ] && usage SYCL_BENCH_GIT_REPO="$1"; shift -SYCL_BENCH_PATH="./" +SYCL_BENCH_PATH="./sycl-bench" while getopts "B:" opt; do case $opt in B) SYCL_BENCH_PATH=$OPTARG ;; From 6ea01102c2ce4d3fa3b3043200d60d40c9f21e06 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Thu, 4 Jul 2024 08:27:45 -0700 Subject: [PATCH 05/67] [SYCL] Amend workflow as per review in #14351 --- .github/workflows/sycl-linux-run-tests.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index c9b72343bf6ee..48200bfb2748c 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -218,11 +218,6 @@ jobs: uses: actions/download-artifact@v4 with: name: ${{ inputs.sycl_toolchain_artifact }} - - name: Debug prints [workflow_run] - if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run' - run: | - pwd - ls - name: Download SYCL toolchain [workflow_run] if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run' uses: actions/github-script@v7 @@ -248,8 +243,6 @@ jobs: - name: Unzip artifact [workflow_run] if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run' run: | - pwd - ls unzip ${{ inputs.sycl_toolchain_artifact }}.zip rm ${{ inputs.sycl_toolchain_artifact }}.zip - name: Extract/Setup SYCL toolchain @@ -371,11 +364,11 @@ jobs: exit $ret - - name: Run benchmarks (via sycl-bench) + - name: Run sycl-bench microbenchmarks id: run_benchmarks if: inputs.tests_selector == 'benchmark' run: ./devops/scripts/sycl-bench.sh https://github.com/ianayl/sycl-bench - - name: Upload benchmark results + - name: Upload sycl-bench microbenchmark results if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success' uses: actions/upload-artifact@v4 with: From 940e3beff55e1077e0f8a10792a2ac5eb92da27b Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Thu, 5 Sep 2024 23:21:48 -0700 Subject: [PATCH 06/67] Complete redo of workflow, switch to compute-benchmarks --- .github/workflows/sycl-linux-run-tests.yml | 2 +- devops/scripts/benchmarking/aggregate.py | 70 ++++++++ devops/scripts/benchmarking/benchmark-ci.conf | 26 +++ devops/scripts/benchmarking/benchmark.sh | 170 ++++++++++++++++++ devops/scripts/benchmarking/common.py | 43 +++++ devops/scripts/benchmarking/compare.py | 43 +++++ .../scripts/benchmarking/enabled_tests.conf | 3 + devops/scripts/sycl-bench.sh | 105 ----------- 8 files changed, 356 insertions(+), 106 deletions(-) create mode 100644 devops/scripts/benchmarking/aggregate.py create mode 100644 devops/scripts/benchmarking/benchmark-ci.conf create mode 100755 devops/scripts/benchmarking/benchmark.sh create mode 100644 devops/scripts/benchmarking/common.py create mode 100644 devops/scripts/benchmarking/compare.py create mode 100644 devops/scripts/benchmarking/enabled_tests.conf delete mode 100755 devops/scripts/sycl-bench.sh diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index 48200bfb2748c..089b6020d2577 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -367,7 +367,7 @@ jobs: - name: Run sycl-bench microbenchmarks id: run_benchmarks if: inputs.tests_selector == 'benchmark' - run: ./devops/scripts/sycl-bench.sh https://github.com/ianayl/sycl-bench + run: ./devops/scripts/benchmarking/benchmark.sh - name: Upload sycl-bench microbenchmark results if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success' uses: actions/upload-artifact@v4 diff --git a/devops/scripts/benchmarking/aggregate.py b/devops/scripts/benchmarking/aggregate.py new file mode 100644 index 0000000000000..95fd21964d896 --- /dev/null +++ b/devops/scripts/benchmarking/aggregate.py @@ -0,0 +1,70 @@ +import csv +import sys +from pathlib import Path +import heapq + +import common + +class StreamingMedian: + + def __init__(self): + self.minheap_larger = [] + self.maxheap_smaller = [] + # Note: numbers on maxheap should be negative, as heapq + # is minheap by default + + def add(self, n: float): + if len(self.maxheap_smaller) == 0 or -self.maxheap_smaller[0] >= n: + heapq.heappush(self.maxheap_smaller, -n) + else: + heapq.heappush(self.minheap_larger, n) + + if len(self.maxheap_smaller) > len(self.minheap_larger) + 1: + heapq.heappush(self.minheap_larger, + -heapq.heappop(self.maxheap_smaller)) + elif len(self.maxheap_smaller) < len(self.minheap_larger): + heapq.heappush(self.maxheap_smaller, + -heapq.heappop(self.minheap_larger)) + + def get_median(self) -> float: + if len(self.maxheap_smaller) == len(self.minheap_larger): + return (-self.maxheap_smaller[0] + self.minheap_larger[0]) / 2.0 + else: + return -self.maxheap_smaller[0] + + +def aggregate_median(benchmark: str): + + def csv_samples() -> list[str]: + # TODO check that the path below is valid directory + with Path(f"{common.PERF_RES_PATH}/{benchmark}") as cache_dir: + # TODO check for time range; What time range do I want? + return filter(lambda f: f.is_file(), + cache_dir.glob(f"{benchmark}-*.csv")) + + # Calculate median of every desired metric: + aggregate_s = dict() + for sample_path in csv_samples(): + with open(sample_path, mode='r') as sample_file: + for s in csv.DictReader(sample_file): + if s["TestCase"] not in aggregate_s: + aggregate_s[s["TestCase"]] = \ + { metric: StreamingMedian() for metric in common.metrics_variance } + for metric in common.metrics_variance: + aggregate_s[s["TestCase"]][metric].add(common.sanitize(s[metric])) + + with open(f"{common.PERF_RES_PATH}/{benchmark}/{benchmark}-median.csv", 'w') as output_csv: + writer = csv.DictWriter(output_csv, + fieldnames=["TestCase", *common.metrics_variance.keys()]) + writer.writeheader() + for test_case in aggregate_s: + writer.writerow({ "TestCase": test_case } | + { metric: aggregate_s[test_case][metric].get_median() + for metric in common.metrics_variance }) + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print(f"Usage: {sys.argv[0]} ") + exit() + aggregate_median(sys.argv[1]) diff --git a/devops/scripts/benchmarking/benchmark-ci.conf b/devops/scripts/benchmarking/benchmark-ci.conf new file mode 100644 index 0000000000000..679b93604b9d0 --- /dev/null +++ b/devops/scripts/benchmarking/benchmark-ci.conf @@ -0,0 +1,26 @@ +# Git branch settings for llvm-ci-perf-results +PERF_RES_GIT_REPO="https://github.com/intel-sandbox/llvm-ci-perf-results" +PERF_RES_BRANCH="test-compute-bench" +# Path where llvm-ci-perf-results are cloned +PERF_RES_PATH="./llvm-ci-perf-res" + +# Git branch settings for compute-benchmarks +COMPUTE_BENCH_GIT_REPO="https://github.com/ianayl/compute-benchmarks" +COMPUTE_BENCH_BRANCH="update-sycl" + +# Path to compile and build compute-benchmarks +COMPUTE_BENCH_PATH="./compute-benchmarks" + +# Path to temporarily store compute-benchmark results +OUTPUT_PATH="." + +# Metrics to benchmark, and their allowed variance as a Python dictionary +METRICS_VARIANCE='{"Median": 0.5}' +#METRICS_VARIANCE='{"Median": 0.5, "StdDev": 4.0}' + +# Metrics to record using aggregate.py +METRICS_RECORDED='["Median", "StdDev"]' + +# Threshold to store benchmark files before benchmarking +AVERAGE_THRESHOLD=7 +# TODO reconsider this \ No newline at end of file diff --git a/devops/scripts/benchmarking/benchmark.sh b/devops/scripts/benchmarking/benchmark.sh new file mode 100755 index 0000000000000..66c75fcdd8d80 --- /dev/null +++ b/devops/scripts/benchmarking/benchmark.sh @@ -0,0 +1,170 @@ +#!/bin/sh + +# +# benchmark.sh: Benchmark dpcpp using compute-benchmarks +# + +# TODO fix +usage () { + >&2 echo "Usage: $0 [-B ] + -B Path to clone and build compute-benchmarks on + +This script builds and runs benchmarks from compute-benchmarks." + exit 1 +} + +clone_perf_res() { + echo "### Cloning llvm-ci-perf-res ($PERF_RES_GIT_REPO:$PERF_RES_BRANCH) ###" + mkdir -p "$(dirname $PERF_RES_PATH)" + git clone -b $PERF_RES_BRANCH $PERF_RES_GIT_REPO $PERF_RES_PATH + [ "$?" -ne 0 ] && exit $? +} + +clone_compute_bench() { + echo "### Cloning compute-benchmarks ($COMPUTE_BENCH_GIT_REPO:$COMPUTE_BENCH_BRANCH) ###" + mkdir -p "$(dirname $COMPUTE_BENCH_PATH)" + git clone -b $COMPUTE_BENCH_BRANCH \ + --recurse-submodules $COMPUTE_BENCH_GIT_REPO \ + $COMPUTE_BENCH_PATH + [ "$?" -ne 0 ] && exit $? +} + +build_compute_bench() { + echo "### Building compute-benchmarks ($COMPUTE_BENCH_GIT_REPO:$COMPUTE_BENCH_BRANCH) ###" + mkdir $COMPUTE_BENCH_PATH/build && cd $COMPUTE_BENCH_PATH/build && + cmake .. -DBUILD_SYCL=ON && cmake --build . + compute_bench_build_stat=$? + cd - + [ "$compute_bench_build_stat" -ne 0 ] && exit $compute_bench_build_stat +} + +print_bench_res() { + # Usage: print_bench_res + if [ ! -s $1 ]; then + printf "NO OUTPUT! (Status $2)\n" | tee -a $3 + return # Do not proceed if file is empty + fi + + get_csv_col_index $1 run-time-mean + tmp_run_time_mean_i=$tmp_csv_col_i + get_csv_col_index $1 run-time-median + tmp_run_time_median_i=$tmp_csv_col_i + get_csv_col_index $1 run-time-throughput + tmp_run_time_throughput_i=$tmp_csv_col_i + + # `sycl-bench` output seems to like inserting the header multiple times. + # Here we cache the header to make sure it prints only once: + tmp_header_title="$(cat $1 | head -n 1 | sed 's/^\# Benchmark name/benchmark/')" + tmp_result="$(cat $1 | grep '^[^\#]')" + + printf "%s\n%s" "$tmp_header_title" "$tmp_result" \ + | awk -F',' -v me="$tmp_run_time_mean_i" \ + -v md="$tmp_run_time_median_i" \ + -v th="$tmp_run_time_throughput_i" \ + '{printf "%-57s %-13s %-15s %-20s\n", $1, $me, $md, $th }' \ + | tee -a $3 # Print to summary file +} + +### +STATUS_SUCCESS=0 +STATUS_FAILED=1 +### + +samples_under_threshold () { + mkdir -p $1 + file_count="$(find $1 -maxdepth 1 -type f | wc -l )" + [ "$file_count" -lt "$AVERAGE_THRESHOLD" ] +} + +check_regression() { + if samples_under_threshold "$PERF_RES_PATH/$1"; then + echo "Not enough samples to construct an average, performance check skipped!" + return $STATUS_SUCCESS + fi + BENCHMARKING_ROOT="$BENCHMARKING_ROOT" python "$BENCHMARKING_ROOT/compare.py" "$1" "$2" + return $? + # return $STATUS_FAILED +} + +cache() { + mv "$2" "$PERF_RES_PATH/$1/" +} + +# Check for a regression, and cache if no regression found +check_and_cache() { + echo "Checking $testcase..." + if check_regression $1 $2; then + echo "Caching $testcase..." + cache $1 $2 + else + echo "Not caching!" + fi +} + +process_benchmarks() { + TIMESTAMP="$(date '+%Y%m%d_%H%M%S')" + mkdir -p "$PERF_RES_PATH" + + echo "### Running and processing selected benchmarks ###" + if [ -z "$TESTS_CONFIG" ]; then + echo "Setting tests to run via cli is not currently supported." + exit $STATUS_FAILED + else + # Ignore lines in the test config starting with #'s + grep "^[^#]" "$TESTS_CONFIG" | while read -r testcase; do + echo "# Running $testcase..." + test_csv_output="$OUTPUT_PATH/$testcase-$TIMESTAMP.csv" + $COMPUTE_BENCH_PATH/build/bin/$testcase --csv | tail +8 > "$test_csv_output" + # The tail +8 filters out initial debug prints not in csv format + if [ "$?" -eq 0 ] && [ -s "$test_csv_output" ]; then + check_and_cache $testcase $test_csv_output + else + echo "ERROR @ $test_case" + fi + done + fi +} + +cleanup() { + rm -r $COMPUTE_BENCH_PATH +} + +load_configs() { + # This script needs to know where the "BENCHMARKING_ROOT" directory is, + # containing all the configuration files and the compare script. + # + # If this is not provided, this function tries to guess where the files + # are based on how the script is called, and verifies that all necessary + # configs and scripts are reachable. + [ -z "$BENCHMARKING_ROOT" ] && BENCHMARKING_ROOT="$(dirname $0)" + + BENCHMARK_CI_CONFIG="$BENCHMARKING_ROOT/benchmark-ci.conf" + TESTS_CONFIG="$BENCHMARKING_ROOT/enabled_tests.conf" + COMPARE_PATH="$BENCHMARKING_ROOT/compare.py" + + for file in "$BENCHMARK_CI_CONFIG" "$TESTS_CONFIG" "$COMPARE_PATH"; do + if [ ! -f "$file" ]; then + echo "$(basename $file) not found, please provide path to BENCHMARKING_ROOT." + exit -1 + fi + done + + . $BENCHMARK_CI_CONFIG +} + +load_configs + +# CLI overrides to configuration options +while getopts "p:b:r:" opt; do + case $opt in + p) COMPUTE_BENCH_PATH=$OPTARG ;; + r) COMPUTE_BENCH_GIT_REPO=$OPTARG ;; + b) COMPUTE_BENCH_BRANCH=$OPTARG ;; + \?) usage ;; + esac +done + +[ ! -d "$PERF_RES_PATH" ] && clone_perf_res +[ ! -d "$COMPUTE_BENCH_PATH" ] && clone_compute_bench +[ ! -d "$COMPUTE_BENCH_PATH/build" ] && build_compute_bench +process_benchmarks \ No newline at end of file diff --git a/devops/scripts/benchmarking/common.py b/devops/scripts/benchmarking/common.py new file mode 100644 index 0000000000000..61272db6db618 --- /dev/null +++ b/devops/scripts/benchmarking/common.py @@ -0,0 +1,43 @@ +import os +import re +import ast + +PERF_RES_PATH, metrics_variance, metrics_recorded = None, None, None + +def sanitize(stat: str) -> float: + # Get rid of % + if stat[-1] == '%': + stat = stat[:-1] + return float(stat) + + +def load_configs(): + BENCHMARKING_ROOT = os.getenv("BENCHMARKING_ROOT") + if BENCHMARKING_ROOT is None: + # Try to predict where BENCHMARKING_ROOT is based on executable + BENCHMARKING_ROOT = os.path.dirname(os.path.abspath(__file__)) + + benchmarking_ci_conf_path = f"{BENCHMARKING_ROOT}/benchmark-ci.conf" + if not os.path.isfile(benchmarking_ci_conf_path): + raise Exception(f"Please provide path to a valid BENCHMARKING_ROOT.") + + global PERF_RES_PATH, metrics_variance, metrics_recorded + perf_res_re = re.compile(r'^PERF_RES_PATH=(.*)$', re.M) + m_variance_re = re.compile(r'^METRICS_VARIANCE=(.*)$', re.M) + m_recorded_re = re.compile(r'^METRICS_RECORDED=(.*)$', re.M) + + with open(benchmarking_ci_conf_path, 'r') as configs_file: + configs_str = configs_file.read() + + for m_variance in m_variance_re.findall(configs_str): + metrics_variance = ast.literal_eval(m_variance.strip()[1:-1]) + if not isinstance(metrics_variance, dict): + raise TypeError("Error in benchmark-ci.conf: METRICS_VARIANCE is not a python dict.") + + for m_recorded in m_recorded_re.findall(configs_str): + metrics_recorded = ast.literal_eval(m_recorded.strip()[1:-1]) + if not isinstance(metrics_recorded, list): + raise TypeError("Error in benchmark-ci.conf: METRICS_RECORDED is not a python list.") + + for perf_res in perf_res_re.findall(configs_str): + PERF_RES_PATH = str(perf_res[1:-1]) \ No newline at end of file diff --git a/devops/scripts/benchmarking/compare.py b/devops/scripts/benchmarking/compare.py new file mode 100644 index 0000000000000..9987938256330 --- /dev/null +++ b/devops/scripts/benchmarking/compare.py @@ -0,0 +1,43 @@ +import csv +import sys +from pathlib import Path + +import common + +# TODO compare_to(metric) instead? +def compare_to_median(test_name: str, test_csv_path: str): + median = dict() + with open(f"{common.PERF_RES_PATH}/{test_name}/{test_name}-median.csv", mode='r') as median_csv: + for stat in csv.DictReader(median_csv): + median[stat["TestCase"]] = \ + { metric: float(stat[metric]) for metric in common.metrics_variance } + + # TODO read status codes from a config file + status = 0 + failure_counts = { metric: 0 for metric in common.metrics_variance } + with open(test_csv_path, mode='r') as sample_csv: + for sample in csv.DictReader(sample_csv): + # Ignore test cases we haven't profiled before + if sample["TestCase"] not in median: + continue + test_median = median[sample["TestCase"]] + for metric, threshold in common.metrics_variance.items(): + max_tolerated = test_median[metric] * (1 + threshold) + if common.sanitize(sample[metric]) > max_tolerated: + print("vvv FAILED vvv") + print(sample['TestCase']) + print(f"{metric}: {metric} {common.sanitize(sample[metric])} -- Historic avg. {test_median[metric]} (max tolerance {threshold*100}% -- {max_tolerated})") + print("^^^^^^^^^^^^^^") + status = 1 + failure_counts[metric] += 1 + if status != 0: + print(f"Failure counts: {failure_counts}") + return status + + +if __name__ == "__main__": + if len(sys.argv) < 3: + print(f"Usage: {sys.argv[0]} ") + exit(-1) + common.load_configs() + exit(compare_to_median(sys.argv[1], sys.argv[2])) diff --git a/devops/scripts/benchmarking/enabled_tests.conf b/devops/scripts/benchmarking/enabled_tests.conf new file mode 100644 index 0000000000000..7aaec4919a416 --- /dev/null +++ b/devops/scripts/benchmarking/enabled_tests.conf @@ -0,0 +1,3 @@ +# Test cases to be enabled: +api_overhead_benchmark_sycl +memory_benchmark_sycl diff --git a/devops/scripts/sycl-bench.sh b/devops/scripts/sycl-bench.sh deleted file mode 100755 index 4b00a60f178aa..0000000000000 --- a/devops/scripts/sycl-bench.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/sh - -# sycl-bench.sh: Benchmark dpcpp using sycl-bench - -usage () { - >&2 echo "Usage: $0 [-B ] - -B Path to clone and build sycl-bench on - -This script builds and runs benchmarks from sycl-bench." - exit 1 -} - -clone() { - mkdir -p $SYCL_BENCH_PATH - git clone $SYCL_BENCH_GIT_REPO $SYCL_BENCH_PATH || return $? -} - -build() { - cd $SYCL_BENCH_PATH - cmake -DSYCL_IMPL=dpcpp -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_RUNTIME_OUTPUT_DIRECTORY=./bin -S . -B ./build && - cmake --build ./build || return $? - cd - -} - -get_csv_col_index() { - # Determine the index of a column in a CSV given its title - # Usage: get_csv_col_index - tmp_csv_col_i="$(cat "$1" | head -n 1 | grep -o "^.*$2," | grep -o ',' | wc -l)" -} - -print_bench_res() { - # Usage: print_bench_res - if [ ! -s $1 ]; then - printf "NO OUTPUT! (Status $2)\n" | tee -a $3 - return # Do not proceed if file is empty - fi - - get_csv_col_index $1 run-time-mean - tmp_run_time_mean_i=$tmp_csv_col_i - get_csv_col_index $1 run-time-median - tmp_run_time_median_i=$tmp_csv_col_i - get_csv_col_index $1 run-time-throughput - tmp_run_time_throughput_i=$tmp_csv_col_i - - # `sycl-bench` output seems to like inserting the header multiple times. - # Here we cache the header to make sure it prints only once: - tmp_header_title="$(cat $1 | head -n 1 | sed 's/^\# Benchmark name/benchmark/')" - tmp_result="$(cat $1 | grep '^[^\#]')" - - printf "%s\n%s" "$tmp_header_title" "$tmp_result" \ - | awk -F',' -v me="$tmp_run_time_mean_i" \ - -v md="$tmp_run_time_median_i" \ - -v th="$tmp_run_time_throughput_i" \ - '{printf "%-57s %-13s %-15s %-20s\n", $1, $me, $md, $th }' \ - | tee -a $3 # Print to summary file -} - -# run sycl bench step -run() { - TIMESTAMP="$(date '+%Y%m%d_%H%M%S')" - mkdir "$SYCL_BENCH_PATH/build/bench-$TIMESTAMP/" - tmp_summary_file="$SYCL_BENCH_PATH/build/bench-$TIMESTAMP/summary.txt" - - for file in $SYCL_BENCH_PATH/build/bin/*; do - # TODO -size should not be always 256, caution - tmp_bench_output="$SYCL_BENCH_PATH/build/bench-$TIMESTAMP/$(basename $file).csv" - tmp_bench_log="$SYCL_BENCH_PATH/build/bench-$TIMESTAMP/$(basename $file).log" - - tmp_err="0" - printf "\n### Results for $(basename $file) ###\n" | tee -a $tmp_summary_file - # The pipe here suppresses errors in a way that doesn't stop github actions: - $file --output=$tmp_bench_output --no-verification --size=256 2> "$tmp_bench_log" || tmp_err=$? - print_bench_res $tmp_bench_output $tmp_err $tmp_summary_file - # Remove log if nothing logged - [ ! -s "$tmp_bench_log" ] && rm "$tmp_bench_log" || cat "$tmp_bench_log" | tee -a $tmp_summary_file - done - - # Export timestamp for later use - [ -f "$GITHUB_OUTPUT" ] && echo TIMESTAMP=$TIMESTAMP >> $GITHUB_OUTPUT -} - -compress() { - tar -I gzip -cf "$SYCL_BENCH_PATH/build/bench-$TIMESTAMP.tar.gz" -C "$SYCL_BENCH_PATH/build/bench-$TIMESTAMP" . - if [ -f "$SYCL_BENCH_PATH/build/bench-$TIMESTAMP.tar.gz" ] && [ -f "$GITHUB_OUTPUT" ]; then - echo BENCHMARK_RESULTS="$SYCL_BENCH_PATH/build/bench-$TIMESTAMP.tar.gz" >> $GITHUB_OUTPUT - fi -} - -cleanup() { - rm -r $SYCL_BENCH_PATH -} - - -[ "$#" -lt "1" ] && usage - -SYCL_BENCH_GIT_REPO="$1"; shift -SYCL_BENCH_PATH="./sycl-bench" -while getopts "B:" opt; do - case $opt in - B) SYCL_BENCH_PATH=$OPTARG ;; - \?) usage ;; - esac -done - -clone && build && run && compress From 7aafdf5d33860a328cfd435608bebfe647f30e1a Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 11 Sep 2024 18:34:50 -0700 Subject: [PATCH 07/67] Add safeguards and update usage(), names --- .github/workflows/sycl-linux-run-tests.yml | 16 ++-- devops/scripts/benchmarking/benchmark.sh | 87 ++++++++++++++-------- devops/scripts/benchmarking/compare.py | 13 +++- 3 files changed, 74 insertions(+), 42 deletions(-) diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index 089b6020d2577..3a691edb9858c 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -364,14 +364,14 @@ jobs: exit $ret - - name: Run sycl-bench microbenchmarks + - name: Run compute-benchmarks id: run_benchmarks if: inputs.tests_selector == 'benchmark' run: ./devops/scripts/benchmarking/benchmark.sh - - name: Upload sycl-bench microbenchmark results - if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success' - uses: actions/upload-artifact@v4 - with: - name: sycl_benchmark_res_${{ steps.run_benchmarks.outputs.TIMESTAMP }} - path: ${{ steps.run_benchmarks.outputs.BENCHMARK_RESULTS }} - retention-days: 7 + # - name: Upload sycl-bench microbenchmark results + # if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success' + # uses: actions/upload-artifact@v4 + # with: + # name: sycl_benchmark_res_${{ steps.run_benchmarks.outputs.TIMESTAMP }} + # path: ${{ steps.run_benchmarks.outputs.BENCHMARK_RESULTS }} + # retention-days: 7 diff --git a/devops/scripts/benchmarking/benchmark.sh b/devops/scripts/benchmarking/benchmark.sh index 66c75fcdd8d80..e550fe6892ae9 100755 --- a/devops/scripts/benchmarking/benchmark.sh +++ b/devops/scripts/benchmarking/benchmark.sh @@ -4,10 +4,15 @@ # benchmark.sh: Benchmark dpcpp using compute-benchmarks # -# TODO fix usage () { >&2 echo "Usage: $0 [-B ] -B Path to clone and build compute-benchmarks on + -p Path to compute-benchmarks (or directory to build compute-benchmarks in) + -r Git repo url to use for compute-benchmarks origin + -b Git branch to use within compute-benchmarks + -f Compile flags passed into building compute-benchmarks + -c _cleanup=1 ;; + -C _cleanup=1 && _exit_after_cleanup=1 ;; This script builds and runs benchmarks from compute-benchmarks." exit 1 @@ -32,38 +37,38 @@ clone_compute_bench() { build_compute_bench() { echo "### Building compute-benchmarks ($COMPUTE_BENCH_GIT_REPO:$COMPUTE_BENCH_BRANCH) ###" mkdir $COMPUTE_BENCH_PATH/build && cd $COMPUTE_BENCH_PATH/build && - cmake .. -DBUILD_SYCL=ON && cmake --build . - compute_bench_build_stat=$? + cmake .. -DBUILD_SYCL=ON && cmake --build . $COMPUTE_BENCH_COMPILE_FLAGS + #compute_bench_build_stat=$? cd - - [ "$compute_bench_build_stat" -ne 0 ] && exit $compute_bench_build_stat + #[ "$compute_bench_build_stat" -ne 0 ] && exit $compute_bench_build_stat } -print_bench_res() { - # Usage: print_bench_res - if [ ! -s $1 ]; then - printf "NO OUTPUT! (Status $2)\n" | tee -a $3 - return # Do not proceed if file is empty - fi - - get_csv_col_index $1 run-time-mean - tmp_run_time_mean_i=$tmp_csv_col_i - get_csv_col_index $1 run-time-median - tmp_run_time_median_i=$tmp_csv_col_i - get_csv_col_index $1 run-time-throughput - tmp_run_time_throughput_i=$tmp_csv_col_i - - # `sycl-bench` output seems to like inserting the header multiple times. - # Here we cache the header to make sure it prints only once: - tmp_header_title="$(cat $1 | head -n 1 | sed 's/^\# Benchmark name/benchmark/')" - tmp_result="$(cat $1 | grep '^[^\#]')" - - printf "%s\n%s" "$tmp_header_title" "$tmp_result" \ - | awk -F',' -v me="$tmp_run_time_mean_i" \ - -v md="$tmp_run_time_median_i" \ - -v th="$tmp_run_time_throughput_i" \ - '{printf "%-57s %-13s %-15s %-20s\n", $1, $me, $md, $th }' \ - | tee -a $3 # Print to summary file -} +# print_bench_res() { +# # Usage: print_bench_res +# if [ ! -s $1 ]; then +# printf "NO OUTPUT! (Status $2)\n" | tee -a $3 +# return # Do not proceed if file is empty +# fi +# +# get_csv_col_index $1 run-time-mean +# tmp_run_time_mean_i=$tmp_csv_col_i +# get_csv_col_index $1 run-time-median +# tmp_run_time_median_i=$tmp_csv_col_i +# get_csv_col_index $1 run-time-throughput +# tmp_run_time_throughput_i=$tmp_csv_col_i +# +# # `sycl-bench` output seems to like inserting the header multiple times. +# # Here we cache the header to make sure it prints only once: +# tmp_header_title="$(cat $1 | head -n 1 | sed 's/^\# Benchmark name/benchmark/')" +# tmp_result="$(cat $1 | grep '^[^\#]')" +# +# printf "%s\n%s" "$tmp_header_title" "$tmp_result" \ +# | awk -F',' -v me="$tmp_run_time_mean_i" \ +# -v md="$tmp_run_time_median_i" \ +# -v th="$tmp_run_time_throughput_i" \ +# '{printf "%-57s %-13s %-15s %-20s\n", $1, $me, $md, $th }' \ +# | tee -a $3 # Print to summary file +# } ### STATUS_SUCCESS=0 @@ -102,7 +107,6 @@ check_and_cache() { } process_benchmarks() { - TIMESTAMP="$(date '+%Y%m%d_%H%M%S')" mkdir -p "$PERF_RES_PATH" echo "### Running and processing selected benchmarks ###" @@ -126,7 +130,10 @@ process_benchmarks() { } cleanup() { - rm -r $COMPUTE_BENCH_PATH + echo "### Cleaning up compute-benchmark builds from prior runs ###" + rm -rf $COMPUTE_BENCH_PATH + #rm -rf $PERF_RES_PATH + [ ! -z "$_exit_after_cleanup" ] && exit } load_configs() { @@ -152,18 +159,32 @@ load_configs() { . $BENCHMARK_CI_CONFIG } +COMPUTE_BENCH_COMPILE_FLAGS="" +TIMESTAMP="$(date '+%Y%m%d_%H%M%S')" + load_configs # CLI overrides to configuration options -while getopts "p:b:r:" opt; do +while getopts "p:b:r:f:cC" opt; do case $opt in p) COMPUTE_BENCH_PATH=$OPTARG ;; r) COMPUTE_BENCH_GIT_REPO=$OPTARG ;; b) COMPUTE_BENCH_BRANCH=$OPTARG ;; + f) COMPUTE_BENCH_COMPILE_FLAGS=$OPTARG ;; + # Cleanup status is saved in a var to ensure all arguments are processed before + # performing cleanup + c) _cleanup=1 ;; + C) _cleanup=1 && _exit_after_cleanup=1 ;; \?) usage ;; esac done +if [ -z "$CMPLR_ROOT" ]; then + echo "Please set \$CMPLR_ROOT first; it is needed by compute-benchmarks to build." + exit 1 +fi +[ ! -z "$_cleanup" ] && cleanup + [ ! -d "$PERF_RES_PATH" ] && clone_perf_res [ ! -d "$COMPUTE_BENCH_PATH" ] && clone_compute_bench [ ! -d "$COMPUTE_BENCH_PATH/build" ] && build_compute_bench diff --git a/devops/scripts/benchmarking/compare.py b/devops/scripts/benchmarking/compare.py index 9987938256330..e2e9b12b0a8e4 100644 --- a/devops/scripts/benchmarking/compare.py +++ b/devops/scripts/benchmarking/compare.py @@ -1,3 +1,4 @@ +import os import csv import sys from pathlib import Path @@ -6,8 +7,18 @@ # TODO compare_to(metric) instead? def compare_to_median(test_name: str, test_csv_path: str): + median_path = f"{common.PERF_RES_PATH}/{test_name}/{test_name}-median.csv" + + if not os.path.isfile(test_csv_path): + print("Invalid test file provided: " + test_csv_path) + exit(-1) + if not os.path.isfile(median_path): + print(f"Median file for test {test_name} not found at {median_path}.\n" + + "Please build the median using the aggregate workflow.") + exit(-1) + median = dict() - with open(f"{common.PERF_RES_PATH}/{test_name}/{test_name}-median.csv", mode='r') as median_csv: + with open(median_path, mode='r') as median_csv: for stat in csv.DictReader(median_csv): median[stat["TestCase"]] = \ { metric: float(stat[metric]) for metric in common.metrics_variance } From 24b51690b0e2cfb90111fe23071f80b5a224efa0 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 11 Sep 2024 18:50:43 -0700 Subject: [PATCH 08/67] Change SYCL_PI_TRACE to SYCL_UR_TRACE --- .github/workflows/sycl-linux-run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index 3a691edb9858c..e47bfedf0f0cc 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -255,7 +255,7 @@ jobs: echo LD_LIBRARY_PATH=$PWD/toolchain/lib/:$LD_LIBRARY_PATH >> $GITHUB_ENV - run: which clang++ sycl-ls - run: sycl-ls --verbose - - run: SYCL_PI_TRACE=-1 sycl-ls + - run: SYCL_UR_TRACE=-1 sycl-ls - run: | if [ -f /usr/local/lib/igc/IGCTAG.txt ]; then cat /usr/local/lib/igc/IGCTAG.txt From b2d44637c62f2eb95c8e4616b41a0a3bd7676267 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 11 Sep 2024 19:41:05 -0700 Subject: [PATCH 09/67] Test commit to learn more about the workflow --- .github/workflows/sycl-linux-run-tests.yml | 24 ++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index e47bfedf0f0cc..a5a17830c4481 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -114,6 +114,21 @@ on: - cts - benchmark + sycl_toolchain_artifact: + description: | + If benchmarking, this artifact is what would be used in benchmarking. + type: string + default: '' + required: False + sycl_toolchain_archive: + type: string + default: '' + required: False + sycl_toolchain_decompress_command: + type: string + default: '' + required: False + env: description: | Suggested variables: for E2E tests - LIT_FILTER, LIT_FILTER_OUT. @@ -255,7 +270,7 @@ jobs: echo LD_LIBRARY_PATH=$PWD/toolchain/lib/:$LD_LIBRARY_PATH >> $GITHUB_ENV - run: which clang++ sycl-ls - run: sycl-ls --verbose - - run: SYCL_UR_TRACE=-1 sycl-ls + - run: SYCL_UR_TRACE=1 sycl-ls - run: | if [ -f /usr/local/lib/igc/IGCTAG.txt ]; then cat /usr/local/lib/igc/IGCTAG.txt @@ -367,7 +382,12 @@ jobs: - name: Run compute-benchmarks id: run_benchmarks if: inputs.tests_selector == 'benchmark' - run: ./devops/scripts/benchmarking/benchmark.sh + run: | + whereis clang++ + whereis sycl-ls + clang++ --version + ls + ./devops/scripts/benchmarking/benchmark.sh # - name: Upload sycl-bench microbenchmark results # if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success' # uses: actions/upload-artifact@v4 From 5f1cd5701254109b9f8746809fdd09a2234dce0b Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 11 Sep 2024 20:29:36 -0700 Subject: [PATCH 10/67] Test commit to integrate building into the workflow --- .github/workflows/sycl-linux-run-tests.yml | 36 ++++++++++++---------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index a5a17830c4481..07de82efd2525 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -114,21 +114,6 @@ on: - cts - benchmark - sycl_toolchain_artifact: - description: | - If benchmarking, this artifact is what would be used in benchmarking. - type: string - default: '' - required: False - sycl_toolchain_archive: - type: string - default: '' - required: False - sycl_toolchain_decompress_command: - type: string - default: '' - required: False - env: description: | Suggested variables: for E2E tests - LIT_FILTER, LIT_FILTER_OUT. @@ -228,6 +213,23 @@ jobs: env | sort > env_after comm -13 env_before env_after >> $GITHUB_ENV rm env_before env_after + + - name: Build SYCL from head for benchmarking, if no artifact provided + id: build_for_benchmarking + if: inputs.tests_selector == 'benchmark' && inputs.sycl_toolchain_artifact == '' + uses: ./.github/workflows/sycl-linux-build.yml + with: + build_ref: ${{ github.sha }} + merge_ref: '' + build_cache_root: "/__w/" + build_artifact_suffix: "default" + build_cache_suffix: "default" + + - name: Testing + shell: bash + run: | + ls + - name: Download SYCL toolchain if: inputs.sycl_toolchain_artifact != '' && github.event_name != 'workflow_run' uses: actions/download-artifact@v4 @@ -381,7 +383,7 @@ jobs: - name: Run compute-benchmarks id: run_benchmarks - if: inputs.tests_selector == 'benchmark' + if: inputs.tests_selector == 'benchmark' && success() run: | whereis clang++ whereis sycl-ls @@ -394,4 +396,4 @@ jobs: # with: # name: sycl_benchmark_res_${{ steps.run_benchmarks.outputs.TIMESTAMP }} # path: ${{ steps.run_benchmarks.outputs.BENCHMARK_RESULTS }} - # retention-days: 7 + # retention-days: 7 \ No newline at end of file From 7a889f882690add24bbf663ac483e52ac11decba Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 11 Sep 2024 22:46:42 -0700 Subject: [PATCH 11/67] Refactor benchmarks out of sycl e2e --- .github/workflows/sycl-linux-benchmark.yml | 322 +++++++++++++++++++++ .github/workflows/sycl-linux-run-tests.yml | 36 +-- 2 files changed, 332 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/sycl-linux-benchmark.yml diff --git a/.github/workflows/sycl-linux-benchmark.yml b/.github/workflows/sycl-linux-benchmark.yml new file mode 100644 index 0000000000000..ba28114bf7aa4 --- /dev/null +++ b/.github/workflows/sycl-linux-benchmark.yml @@ -0,0 +1,322 @@ +name: SYCL Benchmarking + +on: + workflow_call: + inputs: + runner: + type: string + required: True + image: + type: string + required: True + image_options: + type: string + required: True + + target_devices: + type: string + required: True + + ref: + type: string + required: True + merge_ref: + description: | + Commit-ish to merge post-checkout if non-empty. Must be reachable from + the default_branch input paramter. + type: string + default: 'FETCH_HEAD' + required: False + + sycl_toolchain_artifact: + type: string + default: '' + required: False + sycl_toolchain_archive: + type: string + default: '' + required: False + sycl_toolchain_decompress_command: + type: string + default: '' + required: False + + reset_gpu: + type: string + required: False + install_drivers: + type: string + required: False + use_dev_igc: + type: string + required: False + env: + type: string + default: '{}' + required: False + + workflow_dispatch: + inputs: + runner: + type: choice + options: + - '["Linux", "gen12"]' + - '["amdgpu"]' + - '["Linux", "arc"]' + image: + description: | + Use option ending with ":build" for AMDGPU, ":latest" for the rest. + type: choice + options: + - 'ghcr.io/intel/llvm/sycl_ubuntu2204_nightly:latest' + - 'ghcr.io/intel/llvm/sycl_ubuntu2204_nightly:build' + image_options: + description: | + Use option with "--device=/dev/kfd" for AMDGPU, without it for the rest. + type: choice + options: + - '-u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN' + - '-u 1001 --device=/dev/dri --device=/dev/kfd --privileged --cap-add SYS_ADMIN' + target_devices: + type: choice + options: + - 'opencl:cpu' + - 'opencl:gpu' + - 'opencl:fpga' + - 'ext_oneapi_level_zero:gpu' + - 'ext_oneapi_hip:gpu' + env: + description: | + Suggested variables: for E2E tests - LIT_FILTER, LIT_FILTER_OUT. + LIT_OPTS won't work as we redefine it as part of this workflow. + + For SYCL CTS - CTS_TESTS_TO_BUILD to specify which categories to + build. + + Format: '{"VAR1":"VAL1","VAR2":"VAL2",...}' + default: '{}' + + install_drivers: + type: choice + options: + - false + - true + + use_dev_igc: + type: choice + options: + - false + - true + + sycl_toolchain_artifact: + type: string + default: '' + required: False + sycl_toolchain_archive: + type: string + default: '' + required: False + sycl_toolchain_decompress_command: + type: string + default: '' + required: False + +permissions: + contents: read + +jobs: + build: + if: inputs.sycl_toolchain_artifact == '' + uses: ./.github/workflows/sycl-linux-build.yml + with: + build_ref: ${{ github.sha }} + merge_ref: '' + build_cache_root: "/__w/" + build_artifact_suffix: "default" + build_cache_suffix: "default" + build_image: ${{ inputs.image }} + + run_archive: + if: inputs.sycl_toolchain_archive != '' + runs-on: ${{ fromJSON(inputs.runner) }} + container: + image: ${{ inputs.image }} + options: ${{ inputs.image_options }} + env: ${{ fromJSON(inputs.env) }} + steps: + - name: Reset GPU + if: inputs.reset_gpu == 'true' + run: | + sudo mount -t debugfs none /sys/kernel/debug + sudo bash -c 'echo 1 > /sys/kernel/debug/dri/0/i915_wedged' + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + sparse-checkout: | + devops + - name: Register cleanup after job is finished + uses: ./devops/actions/cleanup + - name: Install drivers + if: inputs.install_drivers == 'true' + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + if [ "${{ inputs.use_dev_igc }}" = "true" ]; then + # If libllvm14 is already installed (dev igc docker), still return true. + sudo apt-get install -yqq libllvm14 || true; + fi + sudo -E bash devops/scripts/install_drivers.sh llvm/devops/dependencies.json ${{ inputs.use_dev_igc == 'true' && 'llvm/devops/dependencies-igc-dev.json --use-dev-igc' || '' }} --all + - name: Source OneAPI TBB vars.sh + shell: bash + run: | + # https://github.com/actions/runner/issues/1964 prevents us from using + # the ENTRYPOINT in the image. + env | sort > env_before + if [ -e /runtimes/oneapi-tbb/env/vars.sh ]; then + source /runtimes/oneapi-tbb/env/vars.sh; + elif [ -e /opt/runtimes/oneapi-tbb/env/vars.sh ]; then + source /opt/runtimes/oneapi-tbb/env/vars.sh; + else + echo "no TBB vars in /opt/runtimes or /runtimes"; + fi + env | sort > env_after + comm -13 env_before env_after >> $GITHUB_ENV + rm env_before env_after + + - name: Download SYCL toolchain + if: inputs.sycl_toolchain_artifact != '' && github.event_name != 'workflow_run' + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.sycl_toolchain_artifact }} + - name: Download SYCL toolchain [workflow_run] + if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run' + uses: actions/github-script@v7 + with: + script: | + const name = '${{ inputs.sycl_toolchain_artifact }}' + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == name + })[0]; + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/` + name + '.zip', Buffer.from(download.data)); + - name: Unzip artifact [workflow_run] + if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run' + run: | + unzip ${{ inputs.sycl_toolchain_artifact }}.zip + rm ${{ inputs.sycl_toolchain_artifact }}.zip + - name: Extract/Setup SYCL toolchain + if: inputs.sycl_toolchain_artifact != '' + run: | + mkdir toolchain + tar -I '${{ inputs.sycl_toolchain_decompress_command }}' -xf ${{ inputs.sycl_toolchain_archive }} -C toolchain + rm -f ${{ inputs.sycl_toolchain_archive }} + echo PATH=$PWD/toolchain/bin/:$PATH >> $GITHUB_ENV + echo LD_LIBRARY_PATH=$PWD/toolchain/lib/:$LD_LIBRARY_PATH >> $GITHUB_ENV + - run: which clang++ sycl-ls + - run: sycl-ls --verbose + - run: SYCL_UR_TRACE=1 sycl-ls + - run: | + if [ -f /usr/local/lib/igc/IGCTAG.txt ]; then + cat /usr/local/lib/igc/IGCTAG.txt + fi + + - name: Run compute-benchmarks + id: run_benchmarks + if: success() + run: | + whereis clang++ + whereis sycl-ls + clang++ --version + ls + ./devops/scripts/benchmarking/benchmark.sh + # - name: Upload sycl-bench microbenchmark results + # if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success' + # uses: actions/upload-artifact@v4 + # with: + # name: sycl_benchmark_res_${{ steps.run_benchmarks.outputs.TIMESTAMP }} + # path: ${{ steps.run_benchmarks.outputs.BENCHMARK_RESULTS }} + # retention-days: 7 + + run_build: + needs: [build] + runs-on: ${{ fromJSON(inputs.runner) }} + container: + image: ${{ inputs.image }} + options: ${{ inputs.image_options }} + env: ${{ fromJSON(inputs.env) }} + steps: + - name: Reset GPU + if: inputs.reset_gpu == 'true' + run: | + sudo mount -t debugfs none /sys/kernel/debug + sudo bash -c 'echo 1 > /sys/kernel/debug/dri/0/i915_wedged' + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + sparse-checkout: | + devops + - name: Register cleanup after job is finished + uses: ./devops/actions/cleanup + - name: Install drivers + if: inputs.install_drivers == 'true' + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + if [ "${{ inputs.use_dev_igc }}" = "true" ]; then + # If libllvm14 is already installed (dev igc docker), still return true. + sudo apt-get install -yqq libllvm14 || true; + fi + sudo -E bash devops/scripts/install_drivers.sh llvm/devops/dependencies.json ${{ inputs.use_dev_igc == 'true' && 'llvm/devops/dependencies-igc-dev.json --use-dev-igc' || '' }} --all + - name: Source OneAPI TBB vars.sh + shell: bash + run: | + # https://github.com/actions/runner/issues/1964 prevents us from using + # the ENTRYPOINT in the image. + env | sort > env_before + if [ -e /runtimes/oneapi-tbb/env/vars.sh ]; then + source /runtimes/oneapi-tbb/env/vars.sh; + elif [ -e /opt/runtimes/oneapi-tbb/env/vars.sh ]; then + source /opt/runtimes/oneapi-tbb/env/vars.sh; + else + echo "no TBB vars in /opt/runtimes or /runtimes"; + fi + env | sort > env_after + comm -13 env_before env_after >> $GITHUB_ENV + rm env_before env_after + + - run: ls + - run: which clang++ sycl-ls + - run: sycl-ls --verbose + - run: SYCL_UR_TRACE=1 sycl-ls + - run: | + if [ -f /usr/local/lib/igc/IGCTAG.txt ]; then + cat /usr/local/lib/igc/IGCTAG.txt + fi + + # - name: Run compute-benchmarks + # id: run_benchmarks + # if: success() + # run: | + # whereis clang++ + # whereis sycl-ls + # clang++ --version + # ls + # ./devops/scripts/benchmarking/benchmark.sh + # - name: Upload sycl-bench microbenchmark results + # if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success' + # uses: actions/upload-artifact@v4 + # with: + # name: sycl_benchmark_res_${{ steps.run_benchmarks.outputs.TIMESTAMP }} + # path: ${{ steps.run_benchmarks.outputs.BENCHMARK_RESULTS }} + # retention-days: 7 \ No newline at end of file diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index 07de82efd2525..1b00d5b5c31b3 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -112,7 +112,7 @@ on: options: - e2e - cts - - benchmark + #- benchmark env: description: | @@ -214,22 +214,6 @@ jobs: comm -13 env_before env_after >> $GITHUB_ENV rm env_before env_after - - name: Build SYCL from head for benchmarking, if no artifact provided - id: build_for_benchmarking - if: inputs.tests_selector == 'benchmark' && inputs.sycl_toolchain_artifact == '' - uses: ./.github/workflows/sycl-linux-build.yml - with: - build_ref: ${{ github.sha }} - merge_ref: '' - build_cache_root: "/__w/" - build_artifact_suffix: "default" - build_cache_suffix: "default" - - - name: Testing - shell: bash - run: | - ls - - name: Download SYCL toolchain if: inputs.sycl_toolchain_artifact != '' && github.event_name != 'workflow_run' uses: actions/download-artifact@v4 @@ -381,15 +365,15 @@ jobs: exit $ret - - name: Run compute-benchmarks - id: run_benchmarks - if: inputs.tests_selector == 'benchmark' && success() - run: | - whereis clang++ - whereis sycl-ls - clang++ --version - ls - ./devops/scripts/benchmarking/benchmark.sh + # - name: Run compute-benchmarks + # id: run_benchmarks + # if: inputs.tests_selector == 'benchmark' && success() + # run: | + # whereis clang++ + # whereis sycl-ls + # clang++ --version + # ls + # ./devops/scripts/benchmarking/benchmark.sh # - name: Upload sycl-bench microbenchmark results # if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success' # uses: actions/upload-artifact@v4 From 6e1b3bb84715d2f059e035e236f636019630ecec Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 11 Sep 2024 22:52:55 -0700 Subject: [PATCH 12/67] Re-enable benchmarks for testing purposes --- .github/workflows/sycl-linux-run-tests.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index 1b00d5b5c31b3..8e0e3b0fcefba 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -112,7 +112,7 @@ on: options: - e2e - cts - #- benchmark + - benchmark env: description: | @@ -365,15 +365,15 @@ jobs: exit $ret - # - name: Run compute-benchmarks - # id: run_benchmarks - # if: inputs.tests_selector == 'benchmark' && success() - # run: | - # whereis clang++ - # whereis sycl-ls - # clang++ --version - # ls - # ./devops/scripts/benchmarking/benchmark.sh + - name: Run compute-benchmarks + id: run_benchmarks + if: inputs.tests_selector == 'benchmark' && success() + run: | + whereis clang++ + whereis sycl-ls + clang++ --version + ls + ./devops/scripts/benchmarking/benchmark.sh # - name: Upload sycl-bench microbenchmark results # if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success' # uses: actions/upload-artifact@v4 From 5d1dea4e34c0cc98317c9d093cdea4ca6c9adc55 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 11 Sep 2024 22:58:59 -0700 Subject: [PATCH 13/67] temporarily hijack precommit for testing --- .github/workflows/sycl-linux-precommit.yml | 53 +++++++++++----------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml index 19d106fa23675..405bfbff0f6db 100644 --- a/.github/workflows/sycl-linux-precommit.yml +++ b/.github/workflows/sycl-linux-precommit.yml @@ -76,11 +76,11 @@ jobs: fail-fast: false matrix: include: - - name: AMD/HIP - runner: '["Linux", "amdgpu"]' - image: ghcr.io/intel/llvm/ubuntu2204_build:latest-0300ac924620a51f76c4929794637b82790f12ab - image_options: -u 1001 --device=/dev/dri --device=/dev/kfd - target_devices: ext_oneapi_hip:gpu + # - name: AMD/HIP + # runner: '["Linux", "amdgpu"]' + # image: ghcr.io/intel/llvm/ubuntu2204_build:latest-0300ac924620a51f76c4929794637b82790f12ab + # image_options: -u 1001 --device=/dev/dri --device=/dev/kfd + # target_devices: ext_oneapi_hip:gpu - name: Intel runner: '["Linux", "gen12"]' image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest @@ -89,27 +89,27 @@ jobs: reset_gpu: true install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }} extra_lit_opts: --param gpu-intel-gen12=True - - name: E2E tests on Intel Arc A-Series Graphics - runner: '["Linux", "arc"]' - image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest - image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN - target_devices: ext_oneapi_level_zero:gpu;opencl:gpu - reset_gpu: true - install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }} - extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True - env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }' - - name: E2E tests with dev igc on Intel Arc A-Series Graphics - runner: '["Linux", "arc"]' - image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:${{ contains(github.event.pull_request.labels.*.name, 'ci-no-devigc') && 'latest' || 'devigc' }} - image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN - target_devices: ext_oneapi_level_zero:gpu;opencl:gpu - reset_gpu: true - install_drivers: >- - ${{ contains(needs.detect_changes.outputs.filters, 'drivers') || - contains(needs.detect_changes.outputs.filters, 'devigccfg') }} - use_dev_igc: ${{ contains(needs.detect_changes.outputs.filters, 'devigccfg') }} - extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True - env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }' + # - name: E2E tests on Intel Arc A-Series Graphics + # runner: '["Linux", "arc"]' + # image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest + # image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN + # target_devices: ext_oneapi_level_zero:gpu;opencl:gpu + # reset_gpu: true + # install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }} + # extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True + # env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }' + # - name: E2E tests with dev igc on Intel Arc A-Series Graphics + # runner: '["Linux", "arc"]' + # image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:${{ contains(github.event.pull_request.labels.*.name, 'ci-no-devigc') && 'latest' || 'devigc' }} + # image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN + # target_devices: ext_oneapi_level_zero:gpu;opencl:gpu + # reset_gpu: true + # install_drivers: >- + # ${{ contains(needs.detect_changes.outputs.filters, 'drivers') || + # contains(needs.detect_changes.outputs.filters, 'devigccfg') }} + # use_dev_igc: ${{ contains(needs.detect_changes.outputs.filters, 'devigccfg') }} + # extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True + # env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }' uses: ./.github/workflows/sycl-linux-run-tests.yml with: @@ -123,6 +123,7 @@ jobs: use_dev_igc: ${{ matrix.use_dev_igc }} extra_lit_opts: ${{ matrix.extra_lit_opts }} env: ${{ matrix.env || '{}' }} + tests_selector: "benchmark" ref: ${{ github.sha }} merge_ref: '' From 991cd55d7ac9d8dbd05b35b05d90d6cc6a767e46 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Thu, 12 Sep 2024 07:23:39 -0700 Subject: [PATCH 14/67] temporarily enable benchmark --- .github/workflows/sycl-linux-benchmark.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/sycl-linux-benchmark.yml b/.github/workflows/sycl-linux-benchmark.yml index ba28114bf7aa4..efecc1c049843 100644 --- a/.github/workflows/sycl-linux-benchmark.yml +++ b/.github/workflows/sycl-linux-benchmark.yml @@ -304,15 +304,15 @@ jobs: cat /usr/local/lib/igc/IGCTAG.txt fi - # - name: Run compute-benchmarks - # id: run_benchmarks - # if: success() - # run: | - # whereis clang++ - # whereis sycl-ls - # clang++ --version - # ls - # ./devops/scripts/benchmarking/benchmark.sh + - name: Run compute-benchmarks + id: run_benchmarks + if: success() + run: | + whereis clang++ + whereis sycl-ls + clang++ --version + ls + ./devops/scripts/benchmarking/benchmark.sh # - name: Upload sycl-bench microbenchmark results # if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success' # uses: actions/upload-artifact@v4 From 8a5ecb669b92925808a1c26a9b18cbca6283d38e Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Thu, 12 Sep 2024 08:31:28 -0700 Subject: [PATCH 15/67] Set CMPLR_ROOT for run --- .github/workflows/sycl-linux-benchmark.yml | 2 ++ .github/workflows/sycl-linux-run-tests.yml | 5 +---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sycl-linux-benchmark.yml b/.github/workflows/sycl-linux-benchmark.yml index efecc1c049843..de2c5b64e82f3 100644 --- a/.github/workflows/sycl-linux-benchmark.yml +++ b/.github/workflows/sycl-linux-benchmark.yml @@ -239,6 +239,7 @@ jobs: whereis sycl-ls clang++ --version ls + export CMPLR_ROOT=$PWD/toolchain ./devops/scripts/benchmarking/benchmark.sh # - name: Upload sycl-bench microbenchmark results # if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success' @@ -312,6 +313,7 @@ jobs: whereis sycl-ls clang++ --version ls + export CMPLR_ROOT=$PWD/toolchain ./devops/scripts/benchmarking/benchmark.sh # - name: Upload sycl-bench microbenchmark results # if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success' diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index d5ed4e3808ee4..1f7accde29287 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -384,10 +384,7 @@ jobs: id: run_benchmarks if: inputs.tests_selector == 'benchmark' && success() run: | - whereis clang++ - whereis sycl-ls - clang++ --version - ls + export CMPLR_ROOT=$PWD/toolchain ./devops/scripts/benchmarking/benchmark.sh # - name: Upload sycl-bench microbenchmark results # if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success' From 606c02ae978bc8be52b108a461481e65ceb74fe0 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Thu, 12 Sep 2024 12:07:50 -0700 Subject: [PATCH 16/67] Test pushing --- .github/workflows/sycl-linux-run-tests.yml | 16 ++++++++++++++++ devops/scripts/benchmarking/benchmark-ci.conf | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index 1f7accde29287..cc2318559e0b4 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -386,6 +386,22 @@ jobs: run: | export CMPLR_ROOT=$PWD/toolchain ./devops/scripts/benchmarking/benchmark.sh + - name: Push compute-benchmarks + if: inputs.tests_selector == 'benchmark' && success() + env: + SSH_KEY: ${{secrets.ACTIONS_DEPLOY_KEY}} + run: | + mkdir -p ~/.ssh + echo "$SSH_KEY" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + eval "$(ssh-agent -s)" + ssh-add -k ~/.ssh/id_rsa + cd llvm-ci-perf-results + git config --global user.name "iclsrc" + git config --global user.email "ia.compiler.tools.git@intel.com" + git add . + git commit -m "Update results" + git push # - name: Upload sycl-bench microbenchmark results # if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success' # uses: actions/upload-artifact@v4 diff --git a/devops/scripts/benchmarking/benchmark-ci.conf b/devops/scripts/benchmarking/benchmark-ci.conf index 679b93604b9d0..a58fa0fd3de3e 100644 --- a/devops/scripts/benchmarking/benchmark-ci.conf +++ b/devops/scripts/benchmarking/benchmark-ci.conf @@ -1,5 +1,5 @@ # Git branch settings for llvm-ci-perf-results -PERF_RES_GIT_REPO="https://github.com/intel-sandbox/llvm-ci-perf-results" +PERF_RES_GIT_REPO="https://github.com/ianayl/llvm-ci-perf-results" PERF_RES_BRANCH="test-compute-bench" # Path where llvm-ci-perf-results are cloned PERF_RES_PATH="./llvm-ci-perf-res" From 5dd976ed9501860a990b52b30d6c032dad39985c Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Thu, 12 Sep 2024 13:29:26 -0700 Subject: [PATCH 17/67] Enable more debugging --- .github/workflows/sycl-linux-run-tests.yml | 6 ++++++ devops/scripts/benchmarking/benchmark-ci.conf | 2 ++ devops/scripts/benchmarking/benchmark.sh | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index cc2318559e0b4..6097f90f5876c 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -380,17 +380,23 @@ jobs: exit $ret + - name: Install deps + if: inputs.tests_selector == 'benchmark' && success() + run: | + sudo apt install -y ssh - name: Run compute-benchmarks id: run_benchmarks if: inputs.tests_selector == 'benchmark' && success() run: | export CMPLR_ROOT=$PWD/toolchain ./devops/scripts/benchmarking/benchmark.sh + echo $? - name: Push compute-benchmarks if: inputs.tests_selector == 'benchmark' && success() env: SSH_KEY: ${{secrets.ACTIONS_DEPLOY_KEY}} run: | + find -maxdepth 3 mkdir -p ~/.ssh echo "$SSH_KEY" > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa diff --git a/devops/scripts/benchmarking/benchmark-ci.conf b/devops/scripts/benchmarking/benchmark-ci.conf index a58fa0fd3de3e..810a6f29f1787 100644 --- a/devops/scripts/benchmarking/benchmark-ci.conf +++ b/devops/scripts/benchmarking/benchmark-ci.conf @@ -10,6 +10,8 @@ COMPUTE_BENCH_BRANCH="update-sycl" # Path to compile and build compute-benchmarks COMPUTE_BENCH_PATH="./compute-benchmarks" +# Compile flags used to build compute-benchmarks +COMPUTE_BENCH_COMPILE_FLAGS="-j8" # Path to temporarily store compute-benchmark results OUTPUT_PATH="." diff --git a/devops/scripts/benchmarking/benchmark.sh b/devops/scripts/benchmarking/benchmark.sh index e550fe6892ae9..03e3eb38e6e02 100755 --- a/devops/scripts/benchmarking/benchmark.sh +++ b/devops/scripts/benchmarking/benchmark.sh @@ -118,7 +118,7 @@ process_benchmarks() { grep "^[^#]" "$TESTS_CONFIG" | while read -r testcase; do echo "# Running $testcase..." test_csv_output="$OUTPUT_PATH/$testcase-$TIMESTAMP.csv" - $COMPUTE_BENCH_PATH/build/bin/$testcase --csv | tail +8 > "$test_csv_output" + $COMPUTE_BENCH_PATH/build/bin/$testcase --csv | tail +8 | tee "$test_csv_output" # The tail +8 filters out initial debug prints not in csv format if [ "$?" -eq 0 ] && [ -s "$test_csv_output" ]; then check_and_cache $testcase $test_csv_output From 6c83d2be03a5acee57ef8026c10aefcac9e0f61f Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Thu, 12 Sep 2024 13:57:16 -0700 Subject: [PATCH 18/67] disable post commit temporarily for testing purposes, turn down parallelism level --- .github/workflows/sycl-post-commit.yml | 257 +++++++++--------- devops/scripts/benchmarking/benchmark-ci.conf | 2 +- 2 files changed, 130 insertions(+), 129 deletions(-) diff --git a/.github/workflows/sycl-post-commit.yml b/.github/workflows/sycl-post-commit.yml index d67edc83a050c..24b0635474c29 100644 --- a/.github/workflows/sycl-post-commit.yml +++ b/.github/workflows/sycl-post-commit.yml @@ -1,128 +1,129 @@ -name: SYCL Post Commit - -on: - push: - branches: - - sycl - - sycl-devops-pr/** - - sycl-rel-** - - pull_request: - branches: - - sycl - - sycl-devops-pr/** - paths: - - .github/workflows/sycl-post-commit.yml - - .github/workflows/sycl-linux-build.yml - - .github/workflows/sycl-linux-run-tests.yml - - .github/workflows/sycl-macos-build-and-test.yml - - ./devops/actions/cleanup - - ./devops/actions/cached_checkout - -concurrency: - # Cancel a currently running workflow from the same PR or commit hash. - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: true - -permissions: read-all - -jobs: - build-lin: - name: Linux (Self build + no-assertions) - if: github.repository == 'intel/llvm' - uses: ./.github/workflows/sycl-linux-build.yml - with: - build_cache_root: "/__w/llvm" - build_cache_suffix: default - build_artifact_suffix: default - build_configure_extra_args: --no-assertions --hip --cuda --native_cpu --cmake-opt="-DSYCL_ENABLE_STACK_PRINTING=ON" --cmake-opt="-DSYCL_LIB_WITH_DEBUG_SYMBOL=ON" - # Docker image has last nightly pre-installed and added to the PATH - build_image: "ghcr.io/intel/llvm/sycl_ubuntu2204_nightly:build" - cc: clang - cxx: clang++ - merge_ref: '' - - e2e-lin: - needs: [build-lin] - if: ${{ always() && !cancelled() && needs.build-lin.outputs.build_conclusion == 'success' }} - strategy: - fail-fast: false - matrix: - include: - - name: Intel GEN12 Graphics with Level Zero - runner: '["Linux", "gen12"]' - extra_lit_opts: --param gpu-intel-gen12=True - target_devices: level_zero:gpu;opencl:fpga - reset_intel_gpu: true - - name: Intel Arc A-Series Graphics with Level Zero - runner: '["Linux", "arc"]' - extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True - reset_intel_gpu: true - - name: AMD/HIP - runner: '["Linux", "amdgpu"]' - image: ghcr.io/intel/llvm/ubuntu2204_build:latest - image_options: -u 1001 --device=/dev/dri --device=/dev/kfd - target_devices: ext_oneapi_hip:gpu - reset_intel_gpu: false - # Performance tests below. Specifics: - # - only run performance tests (use LIT_FILTER env) - # - ask llvm-lit to show all the output, even for PASS (-a) - # - run in single thread (-j 1) - # - enable the tests in LIT (--param enable-perf-tests=True) - # - run on all available devices. - - name: Perf tests on Intel GEN12 Graphics system - runner: '["Linux", "gen12"]' - env: '{"LIT_FILTER":"PerformanceTests/"}' - extra_lit_opts: -a -j 1 --param enable-perf-tests=True --param gpu-intel-gen12=True - target_devices: all - reset_intel_gpu: true - - name: Perf tests on Intel Arc A-Series Graphics system - runner: '["Linux", "arc"]' - env: '{"LIT_FILTER":"PerformanceTests/"}' - extra_lit_opts: -a -j 1 --param enable-perf-tests=True - target_devices: all - reset_intel_gpu: true - uses: ./.github/workflows/sycl-linux-run-tests.yml - with: - name: ${{ matrix.name }} - runner: ${{ matrix. runner }} - image: ${{ matrix.image || 'ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest' }} - image_options: ${{ matrix.image_options || '-u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN' }} - target_devices: ${{ matrix.target_devices || 'level_zero:gpu' }} - reset_intel_gpu: ${{ matrix.reset_intel_gpu }} - - extra_lit_opts: ${{ matrix.extra_lit_opts }} - env: ${{ matrix.env || '{}' }} - - ref: ${{ github.sha }} - merge_ref: '' - - sycl_toolchain_artifact: sycl_linux_default - sycl_toolchain_archive: ${{ needs.build-lin.outputs.artifact_archive_name }} - sycl_toolchain_decompress_command: ${{ needs.build-lin.outputs.artifact_decompress_command }} - - build-win: - if: | - always() - && success() - && github.repository == 'intel/llvm' - uses: ./.github/workflows/sycl-windows-build.yml - - e2e-win: - needs: build-win - # Continue if build was successful. - if: | - always() - && !cancelled() - && needs.build-win.outputs.build_conclusion == 'success' - uses: ./.github/workflows/sycl-windows-run-tests.yml - with: - name: Intel GEN12 Graphics with Level Zero - runner: '["Windows","gen12"]' - sycl_toolchain_archive: ${{ needs.build-win.outputs.artifact_archive_name }} - extra_lit_opts: --param gpu-intel-gen12=True - - macos_default: - name: macOS - if: github.repository == 'intel/llvm' - uses: ./.github/workflows/sycl-macos-build-and-test.yml +# name: SYCL Post Commit +# +# on: +# push: +# branches: +# - sycl +# - sycl-devops-pr/** +# - sycl-rel-** +# +# pull_request: +# branches: +# - sycl +# - sycl-devops-pr/** +# paths: +# - .github/workflows/sycl-post-commit.yml +# - .github/workflows/sycl-linux-build.yml +# - .github/workflows/sycl-linux-run-tests.yml +# - .github/workflows/sycl-macos-build-and-test.yml +# - ./devops/actions/cleanup +# - ./devops/actions/cached_checkout +# +# concurrency: +# # Cancel a currently running workflow from the same PR or commit hash. +# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} +# cancel-in-progress: true +# +# permissions: read-all +# +# jobs: +# build-lin: +# name: Linux (Self build + no-assertions) +# if: github.repository == 'intel/llvm' +# uses: ./.github/workflows/sycl-linux-build.yml +# with: +# build_cache_root: "/__w/llvm" +# build_cache_suffix: default +# build_artifact_suffix: default +# build_configure_extra_args: --no-assertions --hip --cuda --native_cpu --cmake-opt="-DSYCL_ENABLE_STACK_PRINTING=ON" --cmake-opt="-DSYCL_LIB_WITH_DEBUG_SYMBOL=ON" +# # Docker image has last nightly pre-installed and added to the PATH +# build_image: "ghcr.io/intel/llvm/sycl_ubuntu2204_nightly:build" +# cc: clang +# cxx: clang++ +# merge_ref: '' +# +# e2e-lin: +# needs: [build-lin] +# if: ${{ always() && !cancelled() && needs.build-lin.outputs.build_conclusion == 'success' }} +# strategy: +# fail-fast: false +# matrix: +# include: +# - name: Intel GEN12 Graphics with Level Zero +# runner: '["Linux", "gen12"]' +# extra_lit_opts: --param gpu-intel-gen12=True +# target_devices: level_zero:gpu;opencl:fpga +# reset_intel_gpu: true +# - name: Intel Arc A-Series Graphics with Level Zero +# runner: '["Linux", "arc"]' +# extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True +# reset_intel_gpu: true +# - name: AMD/HIP +# runner: '["Linux", "amdgpu"]' +# image: ghcr.io/intel/llvm/ubuntu2204_build:latest +# image_options: -u 1001 --device=/dev/dri --device=/dev/kfd +# target_devices: ext_oneapi_hip:gpu +# reset_intel_gpu: false +# # Performance tests below. Specifics: +# # - only run performance tests (use LIT_FILTER env) +# # - ask llvm-lit to show all the output, even for PASS (-a) +# # - run in single thread (-j 1) +# # - enable the tests in LIT (--param enable-perf-tests=True) +# # - run on all available devices. +# - name: Perf tests on Intel GEN12 Graphics system +# runner: '["Linux", "gen12"]' +# env: '{"LIT_FILTER":"PerformanceTests/"}' +# extra_lit_opts: -a -j 1 --param enable-perf-tests=True --param gpu-intel-gen12=True +# target_devices: all +# reset_intel_gpu: true +# - name: Perf tests on Intel Arc A-Series Graphics system +# runner: '["Linux", "arc"]' +# env: '{"LIT_FILTER":"PerformanceTests/"}' +# extra_lit_opts: -a -j 1 --param enable-perf-tests=True +# target_devices: all +# reset_intel_gpu: true +# uses: ./.github/workflows/sycl-linux-run-tests.yml +# with: +# name: ${{ matrix.name }} +# runner: ${{ matrix. runner }} +# image: ${{ matrix.image || 'ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest' }} +# image_options: ${{ matrix.image_options || '-u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN' }} +# target_devices: ${{ matrix.target_devices || 'level_zero:gpu' }} +# reset_intel_gpu: ${{ matrix.reset_intel_gpu }} +# +# extra_lit_opts: ${{ matrix.extra_lit_opts }} +# env: ${{ matrix.env || '{}' }} +# +# ref: ${{ github.sha }} +# merge_ref: '' +# +# sycl_toolchain_artifact: sycl_linux_default +# sycl_toolchain_archive: ${{ needs.build-lin.outputs.artifact_archive_name }} +# sycl_toolchain_decompress_command: ${{ needs.build-lin.outputs.artifact_decompress_command }} +# +# build-win: +# if: | +# always() +# && success() +# && github.repository == 'intel/llvm' +# uses: ./.github/workflows/sycl-windows-build.yml +# +# e2e-win: +# needs: build-win +# # Continue if build was successful. +# if: | +# always() +# && !cancelled() +# && needs.build-win.outputs.build_conclusion == 'success' +# uses: ./.github/workflows/sycl-windows-run-tests.yml +# with: +# name: Intel GEN12 Graphics with Level Zero +# runner: '["Windows","gen12"]' +# sycl_toolchain_archive: ${{ needs.build-win.outputs.artifact_archive_name }} +# extra_lit_opts: --param gpu-intel-gen12=True +# +# macos_default: +# name: macOS +# if: github.repository == 'intel/llvm' +# uses: ./.github/workflows/sycl-macos-build-and-test.yml +# \ No newline at end of file diff --git a/devops/scripts/benchmarking/benchmark-ci.conf b/devops/scripts/benchmarking/benchmark-ci.conf index 810a6f29f1787..36dc6b1b6dbcb 100644 --- a/devops/scripts/benchmarking/benchmark-ci.conf +++ b/devops/scripts/benchmarking/benchmark-ci.conf @@ -11,7 +11,7 @@ COMPUTE_BENCH_BRANCH="update-sycl" # Path to compile and build compute-benchmarks COMPUTE_BENCH_PATH="./compute-benchmarks" # Compile flags used to build compute-benchmarks -COMPUTE_BENCH_COMPILE_FLAGS="-j8" +COMPUTE_BENCH_COMPILE_FLAGS="-j2" # Path to temporarily store compute-benchmark results OUTPUT_PATH="." From 5266cacd0ccb779dfa7d87bd0100cb63630945a8 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Thu, 12 Sep 2024 16:11:17 -0700 Subject: [PATCH 19/67] disable ccache --- devops/scripts/benchmarking/benchmark.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/devops/scripts/benchmarking/benchmark.sh b/devops/scripts/benchmarking/benchmark.sh index 03e3eb38e6e02..8ff15801ff3af 100755 --- a/devops/scripts/benchmarking/benchmark.sh +++ b/devops/scripts/benchmarking/benchmark.sh @@ -37,7 +37,8 @@ clone_compute_bench() { build_compute_bench() { echo "### Building compute-benchmarks ($COMPUTE_BENCH_GIT_REPO:$COMPUTE_BENCH_BRANCH) ###" mkdir $COMPUTE_BENCH_PATH/build && cd $COMPUTE_BENCH_PATH/build && - cmake .. -DBUILD_SYCL=ON && cmake --build . $COMPUTE_BENCH_COMPILE_FLAGS + cmake .. -DBUILD_SYCL=ON -DCCACHE_ALLOWED=FALSE && cmake --build . $COMPUTE_BENCH_COMPILE_FLAGS + # No reason to turn on ccache, if this docker image will be disassembled later on #compute_bench_build_stat=$? cd - #[ "$compute_bench_build_stat" -ne 0 ] && exit $compute_bench_build_stat From ab17254cb0f5472c11a20c797acd064ca7f7b69d Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Fri, 13 Sep 2024 13:42:09 -0700 Subject: [PATCH 20/67] Take benchmarking out of sycl-linux-run-tests --- .github/workflows/sycl-linux-benchmark.yml | 242 +++++++++++++-------- .github/workflows/sycl-linux-build.yml | 24 ++ .github/workflows/sycl-linux-precommit.yml | 222 ++++++++++++------- .github/workflows/sycl-linux-run-tests.yml | 39 +--- 4 files changed, 321 insertions(+), 206 deletions(-) diff --git a/.github/workflows/sycl-linux-benchmark.yml b/.github/workflows/sycl-linux-benchmark.yml index de2c5b64e82f3..090c2b39c5f2b 100644 --- a/.github/workflows/sycl-linux-benchmark.yml +++ b/.github/workflows/sycl-linux-benchmark.yml @@ -125,19 +125,134 @@ permissions: contents: read jobs: - build: - if: inputs.sycl_toolchain_artifact == '' + check_build: + name: Build (if no sycl artifact provided) uses: ./.github/workflows/sycl-linux-build.yml with: build_ref: ${{ github.sha }} merge_ref: '' build_cache_root: "/__w/" - build_artifact_suffix: "default" - build_cache_suffix: "default" + build_artifact_suffix: "default_benchmark" + build_cache_suffix: "default_benchmark" build_image: ${{ inputs.image }} + skip_build: ${{ inputs.sycl_toolchain_archive != '' }} + skip_reason: "SYCL toolchain artifact already provided, no build necessary" - run_archive: - if: inputs.sycl_toolchain_archive != '' +# run_archive: +# if: inputs.sycl_toolchain_archive != '' +# runs-on: ${{ fromJSON(inputs.runner) }} +# container: +# image: ${{ inputs.image }} +# options: ${{ inputs.image_options }} +# env: ${{ fromJSON(inputs.env) }} +# steps: +# - name: Reset GPU +# if: inputs.reset_gpu == 'true' +# run: | +# sudo mount -t debugfs none /sys/kernel/debug +# sudo bash -c 'echo 1 > /sys/kernel/debug/dri/0/i915_wedged' +# - uses: actions/checkout@v4 +# with: +# ref: ${{ inputs.ref }} +# sparse-checkout: | +# devops +# - name: Register cleanup after job is finished +# uses: ./devops/actions/cleanup +# - name: Install drivers +# if: inputs.install_drivers == 'true' +# env: +# GITHUB_TOKEN: ${{ github.token }} +# run: | +# if [ "${{ inputs.use_dev_igc }}" = "true" ]; then +# # If libllvm14 is already installed (dev igc docker), still return true. +# sudo apt-get install -yqq libllvm14 || true; +# fi +# sudo -E bash devops/scripts/install_drivers.sh llvm/devops/dependencies.json ${{ inputs.use_dev_igc == 'true' && 'llvm/devops/dependencies-igc-dev.json --use-dev-igc' || '' }} --all +# - name: Source OneAPI TBB vars.sh +# shell: bash +# run: | +# # https://github.com/actions/runner/issues/1964 prevents us from using +# # the ENTRYPOINT in the image. +# env | sort > env_before +# if [ -e /runtimes/oneapi-tbb/env/vars.sh ]; then +# source /runtimes/oneapi-tbb/env/vars.sh; +# elif [ -e /opt/runtimes/oneapi-tbb/env/vars.sh ]; then +# source /opt/runtimes/oneapi-tbb/env/vars.sh; +# else +# echo "no TBB vars in /opt/runtimes or /runtimes"; +# fi +# env | sort > env_after +# comm -13 env_before env_after >> $GITHUB_ENV +# rm env_before env_after +# +# - name: Download SYCL toolchain +# if: inputs.sycl_toolchain_artifact != '' && github.event_name != 'workflow_run' +# uses: actions/download-artifact@v4 +# with: +# name: ${{ inputs.sycl_toolchain_artifact }} +# - name: Download SYCL toolchain [workflow_run] +# if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run' +# uses: actions/github-script@v7 +# with: +# script: | +# const name = '${{ inputs.sycl_toolchain_artifact }}' +# let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ +# owner: context.repo.owner, +# repo: context.repo.repo, +# run_id: context.payload.workflow_run.id, +# }); +# let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { +# return artifact.name == name +# })[0]; +# let download = await github.rest.actions.downloadArtifact({ +# owner: context.repo.owner, +# repo: context.repo.repo, +# artifact_id: matchArtifact.id, +# archive_format: 'zip', +# }); +# let fs = require('fs'); +# fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/` + name + '.zip', Buffer.from(download.data)); +# - name: Unzip artifact [workflow_run] +# if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run' +# run: | +# unzip ${{ inputs.sycl_toolchain_artifact }}.zip +# rm ${{ inputs.sycl_toolchain_artifact }}.zip +# - name: Extract/Setup SYCL toolchain +# if: inputs.sycl_toolchain_artifact != '' +# run: | +# mkdir toolchain +# tar -I '${{ inputs.sycl_toolchain_decompress_command }}' -xf ${{ inputs.sycl_toolchain_archive }} -C toolchain +# rm -f ${{ inputs.sycl_toolchain_archive }} +# echo PATH=$PWD/toolchain/bin/:$PATH >> $GITHUB_ENV +# echo LD_LIBRARY_PATH=$PWD/toolchain/lib/:$LD_LIBRARY_PATH >> $GITHUB_ENV +# - run: which clang++ sycl-ls +# - run: sycl-ls --verbose +# - run: SYCL_UR_TRACE=1 sycl-ls +# - run: | +# if [ -f /usr/local/lib/igc/IGCTAG.txt ]; then +# cat /usr/local/lib/igc/IGCTAG.txt +# fi +# +# - name: Run compute-benchmarks +# id: run_benchmarks +# if: success() +# run: | +# whereis clang++ +# whereis sycl-ls +# clang++ --version +# ls +# export CMPLR_ROOT=$PWD/toolchain +# ./devops/scripts/benchmarking/benchmark.sh +# # - name: Upload sycl-bench microbenchmark results +# # if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success' +# # uses: actions/upload-artifact@v4 +# # with: +# # name: sycl_benchmark_res_${{ steps.run_benchmarks.outputs.TIMESTAMP }} +# # path: ${{ steps.run_benchmarks.outputs.BENCHMARK_RESULTS }} +# # retention-days: 7 + + run_benchmarks: + needs: [ check_build ] runs-on: ${{ fromJSON(inputs.runner) }} container: image: ${{ inputs.image }} @@ -184,16 +299,16 @@ jobs: rm env_before env_after - name: Download SYCL toolchain - if: inputs.sycl_toolchain_artifact != '' && github.event_name != 'workflow_run' + if: github.event_name != 'workflow_run' uses: actions/download-artifact@v4 with: - name: ${{ inputs.sycl_toolchain_artifact }} + name: ${{ inputs.sycl_toolchain_artifact || 'sycl_linux_benchmark_ver' }} - name: Download SYCL toolchain [workflow_run] - if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run' + if: && github.event_name == 'workflow_run' uses: actions/github-script@v7 with: script: | - const name = '${{ inputs.sycl_toolchain_artifact }}' + const name = '${{ inputs.sycl_toolchain_artifact || 'sycl_linux_benchmark_ver' }}' let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ owner: context.repo.owner, repo: context.repo.repo, @@ -211,18 +326,19 @@ jobs: let fs = require('fs'); fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/` + name + '.zip', Buffer.from(download.data)); - name: Unzip artifact [workflow_run] - if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run' + if: github.event_name == 'workflow_run' run: | - unzip ${{ inputs.sycl_toolchain_artifact }}.zip - rm ${{ inputs.sycl_toolchain_artifact }}.zip + unzip ${{ inputs.sycl_toolchain_artifact || 'sycl_linux_benchmark_ver' }}.zip + rm ${{ inputs.sycl_toolchain_artifact || 'sycl_linux_benchmark_ver' }}.zip - name: Extract/Setup SYCL toolchain - if: inputs.sycl_toolchain_artifact != '' run: | mkdir toolchain - tar -I '${{ inputs.sycl_toolchain_decompress_command }}' -xf ${{ inputs.sycl_toolchain_archive }} -C toolchain - rm -f ${{ inputs.sycl_toolchain_archive }} + tar -I '${{ inputs.sycl_toolchain_decompress_command || needs.check_build.outputs.artifact_decompress_command }}' \ + -xf ${{ inputs.sycl_toolchain_archive || needs.check_build.outputs.artifact_archive_name }} -C toolchain + rm -f ${{ inputs.sycl_toolchain_archive || needs.check_build.outputs.artifact_archive_name }} echo PATH=$PWD/toolchain/bin/:$PATH >> $GITHUB_ENV echo LD_LIBRARY_PATH=$PWD/toolchain/lib/:$LD_LIBRARY_PATH >> $GITHUB_ENV + - run: ls - run: which clang++ sycl-ls - run: sycl-ls --verbose - run: SYCL_UR_TRACE=1 sycl-ls @@ -231,6 +347,10 @@ jobs: cat /usr/local/lib/igc/IGCTAG.txt fi + # - name: Install deps + # if: success() + # run: | + # sudo apt install -y ssh - name: Run compute-benchmarks id: run_benchmarks if: success() @@ -241,80 +361,26 @@ jobs: ls export CMPLR_ROOT=$PWD/toolchain ./devops/scripts/benchmarking/benchmark.sh - # - name: Upload sycl-bench microbenchmark results - # if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success' - # uses: actions/upload-artifact@v4 - # with: - # name: sycl_benchmark_res_${{ steps.run_benchmarks.outputs.TIMESTAMP }} - # path: ${{ steps.run_benchmarks.outputs.BENCHMARK_RESULTS }} - # retention-days: 7 - - run_build: - needs: [build] - runs-on: ${{ fromJSON(inputs.runner) }} - container: - image: ${{ inputs.image }} - options: ${{ inputs.image_options }} - env: ${{ fromJSON(inputs.env) }} - steps: - - name: Reset GPU - if: inputs.reset_gpu == 'true' - run: | - sudo mount -t debugfs none /sys/kernel/debug - sudo bash -c 'echo 1 > /sys/kernel/debug/dri/0/i915_wedged' - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref }} - sparse-checkout: | - devops - - name: Register cleanup after job is finished - uses: ./devops/actions/cleanup - - name: Install drivers - if: inputs.install_drivers == 'true' - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - if [ "${{ inputs.use_dev_igc }}" = "true" ]; then - # If libllvm14 is already installed (dev igc docker), still return true. - sudo apt-get install -yqq libllvm14 || true; - fi - sudo -E bash devops/scripts/install_drivers.sh llvm/devops/dependencies.json ${{ inputs.use_dev_igc == 'true' && 'llvm/devops/dependencies-igc-dev.json --use-dev-igc' || '' }} --all - - name: Source OneAPI TBB vars.sh - shell: bash - run: | - # https://github.com/actions/runner/issues/1964 prevents us from using - # the ENTRYPOINT in the image. - env | sort > env_before - if [ -e /runtimes/oneapi-tbb/env/vars.sh ]; then - source /runtimes/oneapi-tbb/env/vars.sh; - elif [ -e /opt/runtimes/oneapi-tbb/env/vars.sh ]; then - source /opt/runtimes/oneapi-tbb/env/vars.sh; - else - echo "no TBB vars in /opt/runtimes or /runtimes"; - fi - env | sort > env_after - comm -13 env_before env_after >> $GITHUB_ENV - rm env_before env_after - - - run: ls - - run: which clang++ sycl-ls - - run: sycl-ls --verbose - - run: SYCL_UR_TRACE=1 sycl-ls - - run: | - if [ -f /usr/local/lib/igc/IGCTAG.txt ]; then - cat /usr/local/lib/igc/IGCTAG.txt - fi - - - name: Run compute-benchmarks - id: run_benchmarks + exit $? + - name: Push compute-benchmarks if: success() + env: + SSH_KEY: ${{secrets.ACTIONS_DEPLOY_KEY}} run: | - whereis clang++ - whereis sycl-ls - clang++ --version - ls - export CMPLR_ROOT=$PWD/toolchain - ./devops/scripts/benchmarking/benchmark.sh + mkdir -p ~/.ssh + echo "$SSH_KEY" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + echo "###" + echo "$(ssh-agent -s)" + echo "###" + eval "$(ssh-agent -s)" + ssh-add -k ~/.ssh/id_rsa + cd llvm-ci-perf-results + git config --global user.name "iclsrc" + git config --global user.email "ia.compiler.tools.git@intel.com" + git add . + git commit -m "Update results" + git push # - name: Upload sycl-bench microbenchmark results # if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success' # uses: actions/upload-artifact@v4 diff --git a/.github/workflows/sycl-linux-build.yml b/.github/workflows/sycl-linux-build.yml index 17dcc83aeb437..e0d390dd3dba5 100644 --- a/.github/workflows/sycl-linux-build.yml +++ b/.github/workflows/sycl-linux-build.yml @@ -50,6 +50,18 @@ on: description: 'Artifacts retention period' type: string default: 3 + skip_build: + description: | + Build is skipped if set to true. This is used to implement building + based on a condition, as github currently does not provide a way to + have dynamic `needs` dependencies between jobs. + type: boolean + default: false + skip_reason: + description: | + Reason for skipping the build. + type: string + default: "inputs.skip-reason set to true." outputs: build_conclusion: @@ -109,6 +121,7 @@ jobs: build: name: Build + LIT runs-on: [Linux, build] + if: ${{ inputs.skip_build != 'true' }} container: image: ${{ inputs.build_image }} options: -u 1001:1001 @@ -242,3 +255,14 @@ jobs: name: sycl_linux_${{ inputs.build_artifact_suffix }} path: ${{ steps.artifact_info.outputs.ARCHIVE_NAME }} retention-days: ${{ inputs.retention-days }} + + skip_build: + name: Build skipped (${{ inputs.skip_reason }}) + runs-on: [Linux, aux-tasks] + if: ${{ inputs.skip_build == 'true' }} + outputs: + build_conclusion: "skipped" + artifact_archive_name: "" + artifact_decompress_command: "" + steps: + - run: echo "Build skipped -- ${{ inputs.skip_reason }}." \ No newline at end of file diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml index 7fe0f6e868ad1..0d5d7f97a142f 100644 --- a/.github/workflows/sycl-linux-precommit.yml +++ b/.github/workflows/sycl-linux-precommit.yml @@ -47,29 +47,146 @@ jobs: build_cache_suffix: "default" changes: ${{ needs.detect_changes.outputs.filters }} - determine_arc_tests: - name: Decide which Arc tests to run - needs: [build, detect_changes] - if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }} - runs-on: [Linux, aux-tasks] - timeout-minutes: 3 - outputs: - arc_tests: ${{ steps.arc_tests.outputs.arc_tests }} - steps: - - name: Determine Arc tests - id: arc_tests - run: | - if [ "${{ contains(needs.detect_changes.outputs.filters, 'devigccfg') }}" == "true" ]; then - echo 'arc_tests="(ESIMD|InvokeSimd|Matrix)/"' >> "$GITHUB_OUTPUT" - elif [ "${{ contains(needs.detect_changes.outputs.filters, 'drivers') }}" == "true" ]; then - echo 'arc_tests=""' >> "$GITHUB_OUTPUT" - elif [ "${{ contains(needs.detect_changes.outputs.filters, 'esimd') }}" == "true" ]; then - echo 'arc_tests="(ESIMD|InvokeSimd|Matrix)/"' >> "$GITHUB_OUTPUT" - else - echo 'arc_tests="Matrix/"' >> "$GITHUB_OUTPUT" - fi - test: - needs: [build, detect_changes, determine_arc_tests] +# determine_arc_tests: +# name: Decide which Arc tests to run +# needs: [build, detect_changes] +# if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }} +# runs-on: [Linux, aux-tasks] +# timeout-minutes: 3 +# outputs: +# arc_tests: ${{ steps.arc_tests.outputs.arc_tests }} +# steps: +# - name: Determine Arc tests +# id: arc_tests +# run: | +# if [ "${{ contains(needs.detect_changes.outputs.filters, 'devigccfg') }}" == "true" ]; then +# echo 'arc_tests="(ESIMD|InvokeSimd|Matrix)/"' >> "$GITHUB_OUTPUT" +# elif [ "${{ contains(needs.detect_changes.outputs.filters, 'drivers') }}" == "true" ]; then +# echo 'arc_tests=""' >> "$GITHUB_OUTPUT" +# elif [ "${{ contains(needs.detect_changes.outputs.filters, 'esimd') }}" == "true" ]; then +# echo 'arc_tests="(ESIMD|InvokeSimd|Matrix)/"' >> "$GITHUB_OUTPUT" +# else +# echo 'arc_tests="Matrix/"' >> "$GITHUB_OUTPUT" +# fi +# test: +# needs: [build, detect_changes, determine_arc_tests] +# if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }} +# strategy: +# fail-fast: false +# matrix: +# include: +# - name: NVIDIA/CUDA +# runner: '["Linux", "cuda"]' +# image: ghcr.io/intel/llvm/ubuntu2204_build:latest +# image_options: -u 1001 --gpus all --cap-add SYS_ADMIN +# target_devices: ext_oneapi_cuda:gpu +# - name: Intel +# runner: '["Linux", "gen12"]' +# image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest +# image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN +# target_devices: level_zero:gpu;opencl:gpu;opencl:cpu +# reset_intel_gpu: true +# install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }} +# extra_lit_opts: --param gpu-intel-gen12=True +# - name: E2E tests on Intel Arc A-Series Graphics +# runner: '["Linux", "arc"]' +# image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest +# image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN +# target_devices: level_zero:gpu;opencl:gpu +# reset_intel_gpu: true +# install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }} +# extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True +# env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }' +# - name: E2E tests with dev igc on Intel Arc A-Series Graphics +# runner: '["Linux", "arc"]' +# image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:devigc +# image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN +# target_devices: level_zero:gpu;opencl:gpu +# reset_intel_gpu: true +# install_drivers: >- +# ${{ contains(needs.detect_changes.outputs.filters, 'drivers') || +# contains(needs.detect_changes.outputs.filters, 'devigccfg') }} +# use_dev_igc: ${{ contains(needs.detect_changes.outputs.filters, 'devigccfg') }} +# extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True +# env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }' +# # Run only if the PR does not have the 'ci-no-devigc' label. +# skip_run: ${{ contains(github.event.pull_request.labels.*.name, 'ci-no-devigc') }} +# +# uses: ./.github/workflows/sycl-linux-run-tests.yml +# with: +# name: ${{ matrix.name }} +# runner: ${{ matrix. runner }} +# image: ${{ matrix.image }} +# image_options: ${{ matrix.image_options }} +# target_devices: ${{ matrix.target_devices }} +# reset_intel_gpu: ${{ matrix.reset_intel_gpu }} +# install_drivers: ${{ matrix.install_drivers }} +# use_dev_igc: ${{ matrix.use_dev_igc }} +# extra_lit_opts: ${{ matrix.extra_lit_opts }} +# env: ${{ matrix.env || '{}' }} +# skip_run: ${{ matrix.skip_run || 'false' }} +# tests_selector: "benchmark" +# +# ref: ${{ github.sha }} +# merge_ref: '' +# +# sycl_toolchain_artifact: sycl_linux_default +# sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }} +# sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }} +# +# +# test-perf: +# needs: [build, detect_changes] +# if: | +# always() && !cancelled() +# && needs.build.outputs.build_conclusion == 'success' +# && (contains(github.event.pull_request.labels.*.name, 'run-perf-tests') +# || contains(needs.detect_changes.outputs.filters, 'perf-tests')) +# strategy: +# fail-fast: false +# matrix: +# include: +# - name: Intel GEN12 Graphics system +# runner: '["Linux", "gen12"]' +# image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest +# image_extra_opts: --device=/dev/dri +# reset_intel_gpu: true +# - name: Intel Arc A-Series Graphics system +# runner: '["Linux", "arc"]' +# image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest +# image_extra_opts: --device=/dev/dri +# reset_intel_gpu: true +# - name: AMD system +# runner: '["Linux", "amdgpu"]' +# image: ghcr.io/intel/llvm/ubuntu2204_build:latest +# image_extra_opts: --device=/dev/dri --device=/dev/kfd +# extra_cmake_args: -DHIP_PLATFORM="AMD" -DAMD_ARCH="gfx1031" +# - name: CUDA system +# runner: '["Linux", "cuda"]' +# image: ghcr.io/intel/llvm/ubuntu2204_build:latest +# image_extra_opts: --gpus all +# uses: ./.github/workflows/sycl-linux-run-tests.yml +# with: +# name: Perf tests on ${{ matrix.name }} +# runner: ${{ matrix. runner }} +# image: ${{ matrix.image }} +# image_options: -u 1001 --privileged --cap-add SYS_ADMIN ${{ matrix.image_extra_opts }} +# target_devices: all +# reset_intel_gpu: ${{ matrix.reset_intel_gpu }} +# +# env: '{"LIT_FILTER":"PerformanceTests/"}' +# extra_lit_opts: -a -j 1 --param enable-perf-tests=True +# extra_cmake_args: ${{ matrix.extra_cmake_args }} +# +# ref: ${{ github.sha }} +# merge_ref: '' +# +# sycl_toolchain_artifact: sycl_linux_default +# sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }} +# sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }} + + benchmark: + needs: [build, detect_changes ] if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }} strategy: fail-fast: false @@ -87,7 +204,6 @@ jobs: target_devices: level_zero:gpu;opencl:gpu;opencl:cpu reset_intel_gpu: true install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }} - extra_lit_opts: --param gpu-intel-gen12=True # - name: E2E tests on Intel Arc A-Series Graphics # runner: '["Linux", "arc"]' # image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest @@ -95,7 +211,6 @@ jobs: # target_devices: level_zero:gpu;opencl:gpu # reset_intel_gpu: true # install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }} - # extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True # env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }' # - name: E2E tests with dev igc on Intel Arc A-Series Graphics # runner: '["Linux", "arc"]' @@ -107,22 +222,20 @@ jobs: # ${{ contains(needs.detect_changes.outputs.filters, 'drivers') || # contains(needs.detect_changes.outputs.filters, 'devigccfg') }} # use_dev_igc: ${{ contains(needs.detect_changes.outputs.filters, 'devigccfg') }} - # extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True # env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }' # # Run only if the PR does not have the 'ci-no-devigc' label. # skip_run: ${{ contains(github.event.pull_request.labels.*.name, 'ci-no-devigc') }} - uses: ./.github/workflows/sycl-linux-run-tests.yml + uses: ./.github/workflows/sycl-linux-benchmark.yml with: name: ${{ matrix.name }} - runner: ${{ matrix. runner }} + runner: ${{ matrix.runner }} image: ${{ matrix.image }} image_options: ${{ matrix.image_options }} target_devices: ${{ matrix.target_devices }} reset_intel_gpu: ${{ matrix.reset_intel_gpu }} install_drivers: ${{ matrix.install_drivers }} use_dev_igc: ${{ matrix.use_dev_igc }} - extra_lit_opts: ${{ matrix.extra_lit_opts }} env: ${{ matrix.env || '{}' }} skip_run: ${{ matrix.skip_run || 'false' }} tests_selector: "benchmark" @@ -133,54 +246,3 @@ jobs: sycl_toolchain_artifact: sycl_linux_default sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }} sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }} - - - test-perf: - needs: [build, detect_changes] - if: | - always() && !cancelled() - && needs.build.outputs.build_conclusion == 'success' - && (contains(github.event.pull_request.labels.*.name, 'run-perf-tests') - || contains(needs.detect_changes.outputs.filters, 'perf-tests')) - strategy: - fail-fast: false - matrix: - include: - - name: Intel GEN12 Graphics system - runner: '["Linux", "gen12"]' - image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest - image_extra_opts: --device=/dev/dri - reset_intel_gpu: true - - name: Intel Arc A-Series Graphics system - runner: '["Linux", "arc"]' - image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest - image_extra_opts: --device=/dev/dri - reset_intel_gpu: true - - name: AMD system - runner: '["Linux", "amdgpu"]' - image: ghcr.io/intel/llvm/ubuntu2204_build:latest - image_extra_opts: --device=/dev/dri --device=/dev/kfd - extra_cmake_args: -DHIP_PLATFORM="AMD" -DAMD_ARCH="gfx1031" - - name: CUDA system - runner: '["Linux", "cuda"]' - image: ghcr.io/intel/llvm/ubuntu2204_build:latest - image_extra_opts: --gpus all - uses: ./.github/workflows/sycl-linux-run-tests.yml - with: - name: Perf tests on ${{ matrix.name }} - runner: ${{ matrix. runner }} - image: ${{ matrix.image }} - image_options: -u 1001 --privileged --cap-add SYS_ADMIN ${{ matrix.image_extra_opts }} - target_devices: all - reset_intel_gpu: ${{ matrix.reset_intel_gpu }} - - env: '{"LIT_FILTER":"PerformanceTests/"}' - extra_lit_opts: -a -j 1 --param enable-perf-tests=True - extra_cmake_args: ${{ matrix.extra_cmake_args }} - - ref: ${{ github.sha }} - merge_ref: '' - - sycl_toolchain_artifact: sycl_linux_default - sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }} - sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }} diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index 6097f90f5876c..5efe424757465 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -28,7 +28,7 @@ on: required: False tests_selector: description: | - Three options: "e2e", "cts", "benchmark". + Two options: "e2e", "cts" type: string default: "e2e" @@ -117,7 +117,6 @@ on: options: - e2e - cts - - benchmark env: description: | @@ -379,39 +378,3 @@ jobs: grep 'exit code: [^0]' -r logs >> $GITHUB_STEP_SUMMARY exit $ret - - - name: Install deps - if: inputs.tests_selector == 'benchmark' && success() - run: | - sudo apt install -y ssh - - name: Run compute-benchmarks - id: run_benchmarks - if: inputs.tests_selector == 'benchmark' && success() - run: | - export CMPLR_ROOT=$PWD/toolchain - ./devops/scripts/benchmarking/benchmark.sh - echo $? - - name: Push compute-benchmarks - if: inputs.tests_selector == 'benchmark' && success() - env: - SSH_KEY: ${{secrets.ACTIONS_DEPLOY_KEY}} - run: | - find -maxdepth 3 - mkdir -p ~/.ssh - echo "$SSH_KEY" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - eval "$(ssh-agent -s)" - ssh-add -k ~/.ssh/id_rsa - cd llvm-ci-perf-results - git config --global user.name "iclsrc" - git config --global user.email "ia.compiler.tools.git@intel.com" - git add . - git commit -m "Update results" - git push - # - name: Upload sycl-bench microbenchmark results - # if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success' - # uses: actions/upload-artifact@v4 - # with: - # name: sycl_benchmark_res_${{ steps.run_benchmarks.outputs.TIMESTAMP }} - # path: ${{ steps.run_benchmarks.outputs.BENCHMARK_RESULTS }} - # retention-days: 7 \ No newline at end of file From 45a93ba818f4fda9648b5ff5d1b782b49b1e2ca1 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Fri, 13 Sep 2024 13:45:44 -0700 Subject: [PATCH 21/67] Fix leftover chars --- .github/workflows/sycl-linux-benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sycl-linux-benchmark.yml b/.github/workflows/sycl-linux-benchmark.yml index 090c2b39c5f2b..8331ada1dc3a6 100644 --- a/.github/workflows/sycl-linux-benchmark.yml +++ b/.github/workflows/sycl-linux-benchmark.yml @@ -304,7 +304,7 @@ jobs: with: name: ${{ inputs.sycl_toolchain_artifact || 'sycl_linux_benchmark_ver' }} - name: Download SYCL toolchain [workflow_run] - if: && github.event_name == 'workflow_run' + if: github.event_name == 'workflow_run' uses: actions/github-script@v7 with: script: | From a4a1c03d6b327a98a836c4e1d4ede8431b20ae99 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Fri, 13 Sep 2024 13:51:32 -0700 Subject: [PATCH 22/67] Add missing fields --- .github/workflows/sycl-linux-benchmark.yml | 3 +++ .github/workflows/sycl-linux-precommit.yml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sycl-linux-benchmark.yml b/.github/workflows/sycl-linux-benchmark.yml index 8331ada1dc3a6..55b5927f8e5a6 100644 --- a/.github/workflows/sycl-linux-benchmark.yml +++ b/.github/workflows/sycl-linux-benchmark.yml @@ -3,6 +3,8 @@ name: SYCL Benchmarking on: workflow_call: inputs: + name: + type: string runner: type: string required: True @@ -252,6 +254,7 @@ jobs: # # retention-days: 7 run_benchmarks: + name: Run benchmarks (${{ inputs.name }}) needs: [ check_build ] runs-on: ${{ fromJSON(inputs.runner) }} container: diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml index 0d5d7f97a142f..f088e92d9f26b 100644 --- a/.github/workflows/sycl-linux-precommit.yml +++ b/.github/workflows/sycl-linux-precommit.yml @@ -202,7 +202,7 @@ jobs: image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN target_devices: level_zero:gpu;opencl:gpu;opencl:cpu - reset_intel_gpu: true + reset_gpu: true install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }} # - name: E2E tests on Intel Arc A-Series Graphics # runner: '["Linux", "arc"]' From 52f5dc309ae9e9f5e4f742664b7606a787520a65 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Fri, 13 Sep 2024 13:55:46 -0700 Subject: [PATCH 23/67] Disable windows, fix missing fields --- .github/workflows/sycl-linux-build.yml | 2 + .github/workflows/sycl-linux-precommit.yml | 6 +- .github/workflows/sycl-windows-precommit.yml | 117 ++++++++++--------- 3 files changed, 63 insertions(+), 62 deletions(-) diff --git a/.github/workflows/sycl-linux-build.yml b/.github/workflows/sycl-linux-build.yml index e0d390dd3dba5..b663976297b87 100644 --- a/.github/workflows/sycl-linux-build.yml +++ b/.github/workflows/sycl-linux-build.yml @@ -57,11 +57,13 @@ on: have dynamic `needs` dependencies between jobs. type: boolean default: false + required: false skip_reason: description: | Reason for skipping the build. type: string default: "inputs.skip-reason set to true." + required: false outputs: build_conclusion: diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml index f088e92d9f26b..4e8954a543986 100644 --- a/.github/workflows/sycl-linux-precommit.yml +++ b/.github/workflows/sycl-linux-precommit.yml @@ -202,7 +202,7 @@ jobs: image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN target_devices: level_zero:gpu;opencl:gpu;opencl:cpu - reset_gpu: true + reset_intel_gpu: true install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }} # - name: E2E tests on Intel Arc A-Series Graphics # runner: '["Linux", "arc"]' @@ -233,12 +233,10 @@ jobs: image: ${{ matrix.image }} image_options: ${{ matrix.image_options }} target_devices: ${{ matrix.target_devices }} - reset_intel_gpu: ${{ matrix.reset_intel_gpu }} + reset_gpu: ${{ matrix.reset_intel_gpu }} install_drivers: ${{ matrix.install_drivers }} use_dev_igc: ${{ matrix.use_dev_igc }} env: ${{ matrix.env || '{}' }} - skip_run: ${{ matrix.skip_run || 'false' }} - tests_selector: "benchmark" ref: ${{ github.sha }} merge_ref: '' diff --git a/.github/workflows/sycl-windows-precommit.yml b/.github/workflows/sycl-windows-precommit.yml index a82c2b7814d75..226947b341a50 100644 --- a/.github/workflows/sycl-windows-precommit.yml +++ b/.github/workflows/sycl-windows-precommit.yml @@ -1,58 +1,59 @@ -name: SYCL Pre Commit on Windows - -on: - pull_request: - branches: - - sycl - - sycl-devops-pr/** - - llvmspirv_pulldown - - sycl-rel-** - # Do not run builds if changes are only in the following locations - paths-ignore: - - '.github/ISSUE_TEMPLATE/**' - - '.github/CODEOWNERS' - - 'sycl/doc/**' - - 'sycl/gdb/**' - - 'clang/docs/**' - - '**.md' - - '**.rst' - - '.github/workflows/sycl-linux-*.yml' - - '.github/workflows/sycl-precommit-aws.yml' - - '.github/workflows/sycl-macos-*.yml' - - '.github/workflows/sycl-nightly.yml' - - 'devops/containers/**' - - 'devops/actions/build_container/**' - -permissions: read-all - -concurrency: - # Cancel a currently running workflow from the same PR, branch or tag. - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - detect_changes: - uses: ./.github/workflows/sycl-detect-changes.yml - - build: - needs: [detect_changes] - if: | - always() && success() - && github.repository == 'intel/llvm' - uses: ./.github/workflows/sycl-windows-build.yml - with: - changes: ${{ needs.detect_changes.outputs.filters }} - - e2e: - needs: build - # Continue if build was successful. - if: | - always() - && !cancelled() - && needs.build.outputs.build_conclusion == 'success' - uses: ./.github/workflows/sycl-windows-run-tests.yml - with: - name: Intel GEN12 Graphics with Level Zero - runner: '["Windows","gen12"]' - sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }} - extra_lit_opts: --param gpu-intel-gen12=True +# name: SYCL Pre Commit on Windows +# +# on: +# pull_request: +# branches: +# - sycl +# - sycl-devops-pr/** +# - llvmspirv_pulldown +# - sycl-rel-** +# # Do not run builds if changes are only in the following locations +# paths-ignore: +# - '.github/ISSUE_TEMPLATE/**' +# - '.github/CODEOWNERS' +# - 'sycl/doc/**' +# - 'sycl/gdb/**' +# - 'clang/docs/**' +# - '**.md' +# - '**.rst' +# - '.github/workflows/sycl-linux-*.yml' +# - '.github/workflows/sycl-precommit-aws.yml' +# - '.github/workflows/sycl-macos-*.yml' +# - '.github/workflows/sycl-nightly.yml' +# - 'devops/containers/**' +# - 'devops/actions/build_container/**' +# +# permissions: read-all +# +# concurrency: +# # Cancel a currently running workflow from the same PR, branch or tag. +# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} +# cancel-in-progress: true +# +# jobs: +# detect_changes: +# uses: ./.github/workflows/sycl-detect-changes.yml +# +# build: +# needs: [detect_changes] +# if: | +# always() && success() +# && github.repository == 'intel/llvm' +# uses: ./.github/workflows/sycl-windows-build.yml +# with: +# changes: ${{ needs.detect_changes.outputs.filters }} +# +# e2e: +# needs: build +# # Continue if build was successful. +# if: | +# always() +# && !cancelled() +# && needs.build.outputs.build_conclusion == 'success' +# uses: ./.github/workflows/sycl-windows-run-tests.yml +# with: +# name: Intel GEN12 Graphics with Level Zero +# runner: '["Windows","gen12"]' +# sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }} +# extra_lit_opts: --param gpu-intel-gen12=True +# \ No newline at end of file From 90575fe2e3730851e3d78316579de39b6bcfcb63 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Fri, 13 Sep 2024 14:00:08 -0700 Subject: [PATCH 24/67] ammend permissions --- .github/workflows/sycl-linux-benchmark.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/sycl-linux-benchmark.yml b/.github/workflows/sycl-linux-benchmark.yml index 55b5927f8e5a6..a25dcfeb4c20b 100644 --- a/.github/workflows/sycl-linux-benchmark.yml +++ b/.github/workflows/sycl-linux-benchmark.yml @@ -123,8 +123,7 @@ on: default: '' required: False -permissions: - contents: read +permissions: read-all jobs: check_build: From 8dfeff646c41c689ae6bf986733e1a7fd16438e6 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Fri, 13 Sep 2024 14:20:29 -0700 Subject: [PATCH 25/67] ammend boolean matching --- .github/workflows/sycl-linux-benchmark.yml | 2 +- .github/workflows/sycl-linux-build.yml | 6 +++--- .github/workflows/sycl-linux-precommit.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/sycl-linux-benchmark.yml b/.github/workflows/sycl-linux-benchmark.yml index a25dcfeb4c20b..121a2c6fba5fe 100644 --- a/.github/workflows/sycl-linux-benchmark.yml +++ b/.github/workflows/sycl-linux-benchmark.yml @@ -253,7 +253,7 @@ jobs: # # retention-days: 7 run_benchmarks: - name: Run benchmarks (${{ inputs.name }}) + name: ${{ inputs.name }} needs: [ check_build ] runs-on: ${{ fromJSON(inputs.runner) }} container: diff --git a/.github/workflows/sycl-linux-build.yml b/.github/workflows/sycl-linux-build.yml index b663976297b87..5ceeaca2c9564 100644 --- a/.github/workflows/sycl-linux-build.yml +++ b/.github/workflows/sycl-linux-build.yml @@ -123,7 +123,7 @@ jobs: build: name: Build + LIT runs-on: [Linux, build] - if: ${{ inputs.skip_build != 'true' }} + if: ${{ inputs.skip_build != true }} container: image: ${{ inputs.build_image }} options: -u 1001:1001 @@ -259,9 +259,9 @@ jobs: retention-days: ${{ inputs.retention-days }} skip_build: - name: Build skipped (${{ inputs.skip_reason }}) + name: Skipping build runs-on: [Linux, aux-tasks] - if: ${{ inputs.skip_build == 'true' }} + if: ${{ inputs.skip_build == true }} outputs: build_conclusion: "skipped" artifact_archive_name: "" diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml index 4e8954a543986..dc2599fe593ee 100644 --- a/.github/workflows/sycl-linux-precommit.yml +++ b/.github/workflows/sycl-linux-precommit.yml @@ -197,7 +197,7 @@ jobs: # image: ghcr.io/intel/llvm/ubuntu2204_build:latest # image_options: -u 1001 --gpus all --cap-add SYS_ADMIN # target_devices: ext_oneapi_cuda:gpu - - name: Intel + - name: Run benchmarks on Intel runner: '["Linux", "gen12"]' image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN From 3346a0939368b3de9009192333316d667e4747ff Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Fri, 13 Sep 2024 15:01:20 -0700 Subject: [PATCH 26/67] change ssh key directory --- .github/workflows/sycl-linux-benchmark.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sycl-linux-benchmark.yml b/.github/workflows/sycl-linux-benchmark.yml index 121a2c6fba5fe..a592d99dab442 100644 --- a/.github/workflows/sycl-linux-benchmark.yml +++ b/.github/workflows/sycl-linux-benchmark.yml @@ -369,14 +369,14 @@ jobs: env: SSH_KEY: ${{secrets.ACTIONS_DEPLOY_KEY}} run: | - mkdir -p ~/.ssh - echo "$SSH_KEY" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa + mkdir -p .ssh + echo "$SSH_KEY" > .ssh/id_rsa + chmod 600 .ssh/id_rsa echo "###" echo "$(ssh-agent -s)" echo "###" eval "$(ssh-agent -s)" - ssh-add -k ~/.ssh/id_rsa + ssh-add -k .ssh/id_rsa cd llvm-ci-perf-results git config --global user.name "iclsrc" git config --global user.email "ia.compiler.tools.git@intel.com" From 3a28ded9a70a1ddfe2d5f3364aab1d8e278e4bea Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Mon, 16 Sep 2024 14:33:32 -0700 Subject: [PATCH 27/67] Added options to disable caching --- .github/workflows/sycl-linux-benchmark.yml | 169 +++------------------ .github/workflows/sycl-linux-precommit.yml | 13 +- devops/scripts/benchmarking/benchmark.sh | 2 +- 3 files changed, 28 insertions(+), 156 deletions(-) diff --git a/.github/workflows/sycl-linux-benchmark.yml b/.github/workflows/sycl-linux-benchmark.yml index a592d99dab442..156b2ad7c91cd 100644 --- a/.github/workflows/sycl-linux-benchmark.yml +++ b/.github/workflows/sycl-linux-benchmark.yml @@ -43,20 +43,16 @@ on: default: '' required: False - reset_gpu: - type: string - required: False - install_drivers: - type: string - required: False - use_dev_igc: - type: string - required: False env: type: string default: '{}' required: False + cache_results: + type: boolean + default: false + required: False + workflow_dispatch: inputs: runner: @@ -89,26 +85,13 @@ on: - 'ext_oneapi_hip:gpu' env: description: | - Suggested variables: for E2E tests - LIT_FILTER, LIT_FILTER_OUT. - LIT_OPTS won't work as we redefine it as part of this workflow. - - For SYCL CTS - CTS_TESTS_TO_BUILD to specify which categories to - build. - Format: '{"VAR1":"VAL1","VAR2":"VAL2",...}' - default: '{}' - install_drivers: - type: choice - options: - - false - - true - - use_dev_igc: - type: choice - options: - - false - - true + Special variables: + * RESET_GPU -- set to "1" to reset Intel GPUs + * INSTALL_DRIVERS -- set to "1" to install drivers + * USE_DEV_IGC -- set to "1" to use dev igc + default: '{}' sycl_toolchain_artifact: type: string @@ -123,6 +106,11 @@ on: default: '' required: False + cache_results: + type: boolean + default: false + required: False + permissions: read-all jobs: @@ -139,119 +127,6 @@ jobs: skip_build: ${{ inputs.sycl_toolchain_archive != '' }} skip_reason: "SYCL toolchain artifact already provided, no build necessary" -# run_archive: -# if: inputs.sycl_toolchain_archive != '' -# runs-on: ${{ fromJSON(inputs.runner) }} -# container: -# image: ${{ inputs.image }} -# options: ${{ inputs.image_options }} -# env: ${{ fromJSON(inputs.env) }} -# steps: -# - name: Reset GPU -# if: inputs.reset_gpu == 'true' -# run: | -# sudo mount -t debugfs none /sys/kernel/debug -# sudo bash -c 'echo 1 > /sys/kernel/debug/dri/0/i915_wedged' -# - uses: actions/checkout@v4 -# with: -# ref: ${{ inputs.ref }} -# sparse-checkout: | -# devops -# - name: Register cleanup after job is finished -# uses: ./devops/actions/cleanup -# - name: Install drivers -# if: inputs.install_drivers == 'true' -# env: -# GITHUB_TOKEN: ${{ github.token }} -# run: | -# if [ "${{ inputs.use_dev_igc }}" = "true" ]; then -# # If libllvm14 is already installed (dev igc docker), still return true. -# sudo apt-get install -yqq libllvm14 || true; -# fi -# sudo -E bash devops/scripts/install_drivers.sh llvm/devops/dependencies.json ${{ inputs.use_dev_igc == 'true' && 'llvm/devops/dependencies-igc-dev.json --use-dev-igc' || '' }} --all -# - name: Source OneAPI TBB vars.sh -# shell: bash -# run: | -# # https://github.com/actions/runner/issues/1964 prevents us from using -# # the ENTRYPOINT in the image. -# env | sort > env_before -# if [ -e /runtimes/oneapi-tbb/env/vars.sh ]; then -# source /runtimes/oneapi-tbb/env/vars.sh; -# elif [ -e /opt/runtimes/oneapi-tbb/env/vars.sh ]; then -# source /opt/runtimes/oneapi-tbb/env/vars.sh; -# else -# echo "no TBB vars in /opt/runtimes or /runtimes"; -# fi -# env | sort > env_after -# comm -13 env_before env_after >> $GITHUB_ENV -# rm env_before env_after -# -# - name: Download SYCL toolchain -# if: inputs.sycl_toolchain_artifact != '' && github.event_name != 'workflow_run' -# uses: actions/download-artifact@v4 -# with: -# name: ${{ inputs.sycl_toolchain_artifact }} -# - name: Download SYCL toolchain [workflow_run] -# if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run' -# uses: actions/github-script@v7 -# with: -# script: | -# const name = '${{ inputs.sycl_toolchain_artifact }}' -# let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ -# owner: context.repo.owner, -# repo: context.repo.repo, -# run_id: context.payload.workflow_run.id, -# }); -# let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { -# return artifact.name == name -# })[0]; -# let download = await github.rest.actions.downloadArtifact({ -# owner: context.repo.owner, -# repo: context.repo.repo, -# artifact_id: matchArtifact.id, -# archive_format: 'zip', -# }); -# let fs = require('fs'); -# fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/` + name + '.zip', Buffer.from(download.data)); -# - name: Unzip artifact [workflow_run] -# if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run' -# run: | -# unzip ${{ inputs.sycl_toolchain_artifact }}.zip -# rm ${{ inputs.sycl_toolchain_artifact }}.zip -# - name: Extract/Setup SYCL toolchain -# if: inputs.sycl_toolchain_artifact != '' -# run: | -# mkdir toolchain -# tar -I '${{ inputs.sycl_toolchain_decompress_command }}' -xf ${{ inputs.sycl_toolchain_archive }} -C toolchain -# rm -f ${{ inputs.sycl_toolchain_archive }} -# echo PATH=$PWD/toolchain/bin/:$PATH >> $GITHUB_ENV -# echo LD_LIBRARY_PATH=$PWD/toolchain/lib/:$LD_LIBRARY_PATH >> $GITHUB_ENV -# - run: which clang++ sycl-ls -# - run: sycl-ls --verbose -# - run: SYCL_UR_TRACE=1 sycl-ls -# - run: | -# if [ -f /usr/local/lib/igc/IGCTAG.txt ]; then -# cat /usr/local/lib/igc/IGCTAG.txt -# fi -# -# - name: Run compute-benchmarks -# id: run_benchmarks -# if: success() -# run: | -# whereis clang++ -# whereis sycl-ls -# clang++ --version -# ls -# export CMPLR_ROOT=$PWD/toolchain -# ./devops/scripts/benchmarking/benchmark.sh -# # - name: Upload sycl-bench microbenchmark results -# # if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success' -# # uses: actions/upload-artifact@v4 -# # with: -# # name: sycl_benchmark_res_${{ steps.run_benchmarks.outputs.TIMESTAMP }} -# # path: ${{ steps.run_benchmarks.outputs.BENCHMARK_RESULTS }} -# # retention-days: 7 - run_benchmarks: name: ${{ inputs.name }} needs: [ check_build ] @@ -262,7 +137,7 @@ jobs: env: ${{ fromJSON(inputs.env) }} steps: - name: Reset GPU - if: inputs.reset_gpu == 'true' + if: env.RESET_GPU == '1' run: | sudo mount -t debugfs none /sys/kernel/debug sudo bash -c 'echo 1 > /sys/kernel/debug/dri/0/i915_wedged' @@ -274,15 +149,15 @@ jobs: - name: Register cleanup after job is finished uses: ./devops/actions/cleanup - name: Install drivers - if: inputs.install_drivers == 'true' + if: env.INSTALL_DRIVERS == '1' env: GITHUB_TOKEN: ${{ github.token }} run: | - if [ "${{ inputs.use_dev_igc }}" = "true" ]; then + if [ "$USE_DEV_IGC" = "1" ]; then # If libllvm14 is already installed (dev igc docker), still return true. sudo apt-get install -yqq libllvm14 || true; fi - sudo -E bash devops/scripts/install_drivers.sh llvm/devops/dependencies.json ${{ inputs.use_dev_igc == 'true' && 'llvm/devops/dependencies-igc-dev.json --use-dev-igc' || '' }} --all + sudo -E bash devops/scripts/install_drivers.sh llvm/devops/dependencies.json ${{ env.USE_DEV_IGC == '1' && 'llvm/devops/dependencies-igc-dev.json --use-dev-igc' || '' }} --all - name: Source OneAPI TBB vars.sh shell: bash run: | @@ -365,16 +240,14 @@ jobs: ./devops/scripts/benchmarking/benchmark.sh exit $? - name: Push compute-benchmarks - if: success() + if: inputs.cache_results == true && success() env: SSH_KEY: ${{secrets.ACTIONS_DEPLOY_KEY}} run: | mkdir -p .ssh + echo "$SSH_KEY" | wc -c echo "$SSH_KEY" > .ssh/id_rsa chmod 600 .ssh/id_rsa - echo "###" - echo "$(ssh-agent -s)" - echo "###" eval "$(ssh-agent -s)" ssh-add -k .ssh/id_rsa cd llvm-ci-perf-results diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml index dc2599fe593ee..ad02a2c17fb19 100644 --- a/.github/workflows/sycl-linux-precommit.yml +++ b/.github/workflows/sycl-linux-precommit.yml @@ -202,16 +202,18 @@ jobs: image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN target_devices: level_zero:gpu;opencl:gpu;opencl:cpu - reset_intel_gpu: true + env: | + { + "RESET_GPU": "1" + } install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }} # - name: E2E tests on Intel Arc A-Series Graphics # runner: '["Linux", "arc"]' # image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest # image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN # target_devices: level_zero:gpu;opencl:gpu - # reset_intel_gpu: true - # install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }} - # env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }' + # + # env: '{"RESET_GPU"="1", "INSTALL_DRIVERS": "${{ contains(needs.detect_changes.outputs.filters, 'drivers') }}", "LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }' # - name: E2E tests with dev igc on Intel Arc A-Series Graphics # runner: '["Linux", "arc"]' # image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:devigc @@ -233,9 +235,6 @@ jobs: image: ${{ matrix.image }} image_options: ${{ matrix.image_options }} target_devices: ${{ matrix.target_devices }} - reset_gpu: ${{ matrix.reset_intel_gpu }} - install_drivers: ${{ matrix.install_drivers }} - use_dev_igc: ${{ matrix.use_dev_igc }} env: ${{ matrix.env || '{}' }} ref: ${{ github.sha }} diff --git a/devops/scripts/benchmarking/benchmark.sh b/devops/scripts/benchmarking/benchmark.sh index 8ff15801ff3af..4b2b5a0a56f7a 100755 --- a/devops/scripts/benchmarking/benchmark.sh +++ b/devops/scripts/benchmarking/benchmark.sh @@ -119,7 +119,7 @@ process_benchmarks() { grep "^[^#]" "$TESTS_CONFIG" | while read -r testcase; do echo "# Running $testcase..." test_csv_output="$OUTPUT_PATH/$testcase-$TIMESTAMP.csv" - $COMPUTE_BENCH_PATH/build/bin/$testcase --csv | tail +8 | tee "$test_csv_output" + $COMPUTE_BENCH_PATH/build/bin/$testcase --csv | tail +8 > "$test_csv_output" # The tail +8 filters out initial debug prints not in csv format if [ "$?" -eq 0 ] && [ -s "$test_csv_output" ]; then check_and_cache $testcase $test_csv_output From e67c29f4b94f655bdeb46a7110622345ef04ab19 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Tue, 17 Sep 2024 12:02:29 -0700 Subject: [PATCH 28/67] attempt to introduce aggregate --- .../workflows/sycl-benchmark-aggregate.yml | 59 +++++++++++++++++++ .github/workflows/sycl-linux-benchmark.yml | 13 +++- 2 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/sycl-benchmark-aggregate.yml diff --git a/.github/workflows/sycl-benchmark-aggregate.yml b/.github/workflows/sycl-benchmark-aggregate.yml new file mode 100644 index 0000000000000..4233344a67c0a --- /dev/null +++ b/.github/workflows/sycl-benchmark-aggregate.yml @@ -0,0 +1,59 @@ +name: Aggregate benchmark averages + +on: + schedule: + - cron: 0 1 * * * + workflow_dispatch: + workflow_call: + +permissions: read-all + +jobs: + aggregate: + name: Aggregate average value for all metrics (median) + permissions: + contents: write # for Git to git push + runs-on: ubuntu-latest + if: github.repository == 'intel/llvm' + steps: + - uses: actions/checkout@v4 + with: + path: llvm + sparse-checkout: | + devops + - name: Load benchmarking configuration + run: + . llvm/devops/scripts/benchmarking/benchmark-ci.conf + echo "PERF_RES_GIT_REPO=$PERF_RES_GIT_REPO" >> $GITHUB_ENV + echo "PERF_RES_BRANCH=$PERF_RES_BRANCH" >> $GITHUB_ENV + echo "PERF_RES_PATH=$PERF_RES_PATH" >> $GITHUB_ENV + - uses: actions/checkout@v4 + with: + # TODO populate default values before loading configuration + path: ${{ env.PERF_RES_PATH }} + repository: ${{ env.PERF_RES_GIT_REPO }} + branch: ${{ env.PERF_RES_BRANCH }} + - run: find . -maxdepth 2 + - name: Run aggregator on cloned data + run: | + for dir in $(find "$PERF_RES_PATH" -mindepth 1 -maxdepth 1 -type d ! -name '.*'); do + python llvm/devops/benchmarking/aggregate.py "$(basename dir)" + done + - name: Debug + run: | + find "$PERF_RES_PATH" -maxdepth 3 ! -name '.*' + - name: Upload average to the repo + env: + SSH_KEY: ${{secrets.ACTIONS_DEPLOY_KEY}} + run: | + mkdir -p ~/.ssh + echo "$SSH_KEY" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + eval "$(ssh-agent -s)" + ssh-add -k ~/.ssh/id_rsa + cd $PERF_RES_PATH + git config --global user.name "iclsrc" + git config --global user.email "ia.compiler.tools.git@intel.com" + git add . + git commit -m "Average aggregate $(date "+%m/%d/%y %H:%M")" -s + git push diff --git a/.github/workflows/sycl-linux-benchmark.yml b/.github/workflows/sycl-linux-benchmark.yml index 156b2ad7c91cd..ff7f8616dff00 100644 --- a/.github/workflows/sycl-linux-benchmark.yml +++ b/.github/workflows/sycl-linux-benchmark.yml @@ -239,13 +239,18 @@ jobs: export CMPLR_ROOT=$PWD/toolchain ./devops/scripts/benchmarking/benchmark.sh exit $? + - name: debug -- delete after + env: + SSH_KEY: ${{secrets.ACTIONS_DEPLOY_KEY}} + run: + # Is this key even a thing? + echo "$SSH_KEY" | wc -c - name: Push compute-benchmarks if: inputs.cache_results == true && success() env: SSH_KEY: ${{secrets.ACTIONS_DEPLOY_KEY}} run: | mkdir -p .ssh - echo "$SSH_KEY" | wc -c echo "$SSH_KEY" > .ssh/id_rsa chmod 600 .ssh/id_rsa eval "$(ssh-agent -s)" @@ -262,4 +267,8 @@ jobs: # with: # name: sycl_benchmark_res_${{ steps.run_benchmarks.outputs.TIMESTAMP }} # path: ${{ steps.run_benchmarks.outputs.BENCHMARK_RESULTS }} - # retention-days: 7 \ No newline at end of file + # retention-days: 7 + + test_aggregate: + name: Testing sycl-benchmark-aggregate.yml (DELETE AFTER) + uses: ./.github/workflows/sycl-benchmark-aggregate.yml \ No newline at end of file From e6cbc2e653d94ab95648199072d228671ad9a73e Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Tue, 17 Sep 2024 13:14:09 -0700 Subject: [PATCH 29/67] changed permissions to allow aggregate --- .github/workflows/sycl-linux-benchmark.yml | 4 +++- .github/workflows/sycl-linux-precommit.yml | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sycl-linux-benchmark.yml b/.github/workflows/sycl-linux-benchmark.yml index ff7f8616dff00..b49fd3f0a35d9 100644 --- a/.github/workflows/sycl-linux-benchmark.yml +++ b/.github/workflows/sycl-linux-benchmark.yml @@ -271,4 +271,6 @@ jobs: test_aggregate: name: Testing sycl-benchmark-aggregate.yml (DELETE AFTER) - uses: ./.github/workflows/sycl-benchmark-aggregate.yml \ No newline at end of file + uses: ./.github/workflows/sycl-benchmark-aggregate.yml + permissions: + contents: write \ No newline at end of file diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml index ac017a0aac1f0..5b75c641063db 100644 --- a/.github/workflows/sycl-linux-precommit.yml +++ b/.github/workflows/sycl-linux-precommit.yml @@ -189,6 +189,8 @@ jobs: benchmark: needs: [build, detect_changes ] + permissions: + contents: write if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }} strategy: fail-fast: false From c6645aa865e513c39a0d8091a93e0987c578fcbb Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Tue, 17 Sep 2024 13:17:18 -0700 Subject: [PATCH 30/67] further write permission correction --- .github/workflows/sycl-benchmark-aggregate.yml | 3 ++- .github/workflows/sycl-linux-benchmark.yml | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sycl-benchmark-aggregate.yml b/.github/workflows/sycl-benchmark-aggregate.yml index 4233344a67c0a..ebf0f6604dd4c 100644 --- a/.github/workflows/sycl-benchmark-aggregate.yml +++ b/.github/workflows/sycl-benchmark-aggregate.yml @@ -6,7 +6,8 @@ on: workflow_dispatch: workflow_call: -permissions: read-all +permissions: + contents: write jobs: aggregate: diff --git a/.github/workflows/sycl-linux-benchmark.yml b/.github/workflows/sycl-linux-benchmark.yml index b49fd3f0a35d9..4b619b040f176 100644 --- a/.github/workflows/sycl-linux-benchmark.yml +++ b/.github/workflows/sycl-linux-benchmark.yml @@ -111,7 +111,8 @@ on: default: false required: False -permissions: read-all +permissions: + contents: write jobs: check_build: @@ -135,6 +136,8 @@ jobs: image: ${{ inputs.image }} options: ${{ inputs.image_options }} env: ${{ fromJSON(inputs.env) }} + permissions: + contents: write steps: - name: Reset GPU if: env.RESET_GPU == '1' From 867e6e6302132a38c795dbc52e38b7a7e7482f43 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Tue, 17 Sep 2024 13:20:58 -0700 Subject: [PATCH 31/67] further read permission correction --- .github/workflows/sycl-linux-benchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sycl-linux-benchmark.yml b/.github/workflows/sycl-linux-benchmark.yml index 4b619b040f176..4a168ba2f7700 100644 --- a/.github/workflows/sycl-linux-benchmark.yml +++ b/.github/workflows/sycl-linux-benchmark.yml @@ -118,6 +118,7 @@ jobs: check_build: name: Build (if no sycl artifact provided) uses: ./.github/workflows/sycl-linux-build.yml + permissions: read-all with: build_ref: ${{ github.sha }} merge_ref: '' From 18ff8b14913b678e1cbfc2e0f54e579d9452b94a Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Tue, 17 Sep 2024 13:24:58 -0700 Subject: [PATCH 32/67] test amendmnet of build permissions --- .github/workflows/sycl-linux-benchmark.yml | 3 ++- .github/workflows/sycl-linux-build.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sycl-linux-benchmark.yml b/.github/workflows/sycl-linux-benchmark.yml index 4a168ba2f7700..3869ad0649e7e 100644 --- a/.github/workflows/sycl-linux-benchmark.yml +++ b/.github/workflows/sycl-linux-benchmark.yml @@ -118,7 +118,8 @@ jobs: check_build: name: Build (if no sycl artifact provided) uses: ./.github/workflows/sycl-linux-build.yml - permissions: read-all + permissions: + contents: read with: build_ref: ${{ github.sha }} merge_ref: '' diff --git a/.github/workflows/sycl-linux-build.yml b/.github/workflows/sycl-linux-build.yml index 5ceeaca2c9564..e982c61e5c1ef 100644 --- a/.github/workflows/sycl-linux-build.yml +++ b/.github/workflows/sycl-linux-build.yml @@ -117,7 +117,8 @@ on: options: - 3 -permissions: read-all +permissions: + contents: read jobs: build: From cc29c231d986837531b8bed698a3e411cdc7c123 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Tue, 17 Sep 2024 14:58:11 -0700 Subject: [PATCH 33/67] Initial support for time ranges --- .../workflows/sycl-benchmark-aggregate.yml | 32 ++++++++++++++++--- .github/workflows/sycl-linux-benchmark.yml | 2 +- devops/scripts/benchmarking/aggregate.py | 19 +++++++---- devops/scripts/benchmarking/benchmark-ci.conf | 7 +++- devops/scripts/benchmarking/benchmark.sh | 6 ++-- devops/scripts/benchmarking/common.py | 10 +++++- 6 files changed, 59 insertions(+), 17 deletions(-) diff --git a/.github/workflows/sycl-benchmark-aggregate.yml b/.github/workflows/sycl-benchmark-aggregate.yml index ebf0f6604dd4c..d85fd78577491 100644 --- a/.github/workflows/sycl-benchmark-aggregate.yml +++ b/.github/workflows/sycl-benchmark-aggregate.yml @@ -4,7 +4,16 @@ on: schedule: - cron: 0 1 * * * workflow_dispatch: + inputs: + cutoff_timestamp: + description: "Timestamp " + type: string + required: false workflow_call: + inputs: + cutoff_timestamp: + type: string + required: false permissions: contents: write @@ -23,22 +32,37 @@ jobs: sparse-checkout: | devops - name: Load benchmarking configuration - run: - . llvm/devops/scripts/benchmarking/benchmark-ci.conf + run: | + . llvm/devops/scripts/benchmarking/benchmark-ci.conf; echo "PERF_RES_GIT_REPO=$PERF_RES_GIT_REPO" >> $GITHUB_ENV echo "PERF_RES_BRANCH=$PERF_RES_BRANCH" >> $GITHUB_ENV echo "PERF_RES_PATH=$PERF_RES_PATH" >> $GITHUB_ENV + + if [ -z '${{ inputs.cutoff_timestamp }}' ]; then + # No time given, use default time period + echo "CUTOFF_TIMESTAMP=$(date --date="$AVERAGE_CUTOFF_RANGE" "$TIMESTAMP_FORMAT")" >> $GITHUB_ENV + else + # If the provided time is a unix `date` timestamp, convert the time to our format + _converted_timestamp="$(date --date '${{ inputs.cutoff_timestamp }}' +"$TIMESTAMP_FORMAT" 2> /dev/null)" + if [ -n "$_converted_timestamp" ]; then + echo "CUTOFF_TIMESTAMP=$_converted_timestamp" >> $GITHUB_ENV + else + # If not a valid unix `date` timestamp, it could be in our timestamp format already. + # aggregate.py will ensure the timestamp is in the proper format, so we can pass the + # time forward regardless: + echo 'CUTOFF_TIMESTAMP=${{ inputs.cutoff_timestamp }}' >> $GITHUB_ENV + fi + fi - uses: actions/checkout@v4 with: # TODO populate default values before loading configuration path: ${{ env.PERF_RES_PATH }} repository: ${{ env.PERF_RES_GIT_REPO }} branch: ${{ env.PERF_RES_BRANCH }} - - run: find . -maxdepth 2 - name: Run aggregator on cloned data run: | for dir in $(find "$PERF_RES_PATH" -mindepth 1 -maxdepth 1 -type d ! -name '.*'); do - python llvm/devops/benchmarking/aggregate.py "$(basename dir)" + python llvm/devops/benchmarking/aggregate.py "$(basename dir)" "$CUTOFF_TIMESTAMP" done - name: Debug run: | diff --git a/.github/workflows/sycl-linux-benchmark.yml b/.github/workflows/sycl-linux-benchmark.yml index 3869ad0649e7e..45c728bb56aa6 100644 --- a/.github/workflows/sycl-linux-benchmark.yml +++ b/.github/workflows/sycl-linux-benchmark.yml @@ -249,7 +249,7 @@ jobs: SSH_KEY: ${{secrets.ACTIONS_DEPLOY_KEY}} run: # Is this key even a thing? - echo "$SSH_KEY" | wc -c + echo "$SSH_KEY" | head -c 4 - name: Push compute-benchmarks if: inputs.cache_results == true && success() env: diff --git a/devops/scripts/benchmarking/aggregate.py b/devops/scripts/benchmarking/aggregate.py index 95fd21964d896..771ae56136f8e 100644 --- a/devops/scripts/benchmarking/aggregate.py +++ b/devops/scripts/benchmarking/aggregate.py @@ -33,14 +33,15 @@ def get_median(self) -> float: return -self.maxheap_smaller[0] -def aggregate_median(benchmark: str): +def aggregate_median(benchmark: str, cutoff: str): def csv_samples() -> list[str]: # TODO check that the path below is valid directory with Path(f"{common.PERF_RES_PATH}/{benchmark}") as cache_dir: # TODO check for time range; What time range do I want? - return filter(lambda f: f.is_file(), - cache_dir.glob(f"{benchmark}-*.csv")) + return filter(lambda f: f.is_file() and + common.valid_timestamp(str(f)[-13:]) and str(f)[-13:] > cutoff, + cache_dir.glob(f"{benchmark}-*_*.csv")) # Calculate median of every desired metric: aggregate_s = dict() @@ -64,7 +65,11 @@ def csv_samples() -> list[str]: if __name__ == "__main__": - if len(sys.argv) < 2: - print(f"Usage: {sys.argv[0]} ") - exit() - aggregate_median(sys.argv[1]) + if len(sys.argv) < 3: + print(f"Usage: {sys.argv[0]} ") + exit(1) + if not common.valid_timestamp(sys.argv[2]): + print(f"Bad cutoff timestamp, please use YYMMDD_HHMMSS.") + exit(1) + + aggregate_median(sys.argv[1], sys.argv[2]) diff --git a/devops/scripts/benchmarking/benchmark-ci.conf b/devops/scripts/benchmarking/benchmark-ci.conf index 36dc6b1b6dbcb..b08d82fcad1a5 100644 --- a/devops/scripts/benchmarking/benchmark-ci.conf +++ b/devops/scripts/benchmarking/benchmark-ci.conf @@ -24,5 +24,10 @@ METRICS_VARIANCE='{"Median": 0.5}' METRICS_RECORDED='["Median", "StdDev"]' # Threshold to store benchmark files before benchmarking +# TODO reconsider this AVERAGE_THRESHOLD=7 -# TODO reconsider this \ No newline at end of file +# Default period of time to aggregate for the average +AVERAGE_CUTOFF_RANGE="7 days ago" + +# Format of timestamps used (linux `date` format string) +TIMESTAMP_FORMAT='%Y%m%d_%H%M%S' \ No newline at end of file diff --git a/devops/scripts/benchmarking/benchmark.sh b/devops/scripts/benchmarking/benchmark.sh index 4b2b5a0a56f7a..5ad82f3c5414a 100755 --- a/devops/scripts/benchmarking/benchmark.sh +++ b/devops/scripts/benchmarking/benchmark.sh @@ -160,11 +160,11 @@ load_configs() { . $BENCHMARK_CI_CONFIG } -COMPUTE_BENCH_COMPILE_FLAGS="" -TIMESTAMP="$(date '+%Y%m%d_%H%M%S')" - load_configs +COMPUTE_BENCH_COMPILE_FLAGS="" +TIMESTAMP="$(date +"$TIMESTAMP_FORMAT")" + # CLI overrides to configuration options while getopts "p:b:r:f:cC" opt; do case $opt in diff --git a/devops/scripts/benchmarking/common.py b/devops/scripts/benchmarking/common.py index 61272db6db618..56e5a2ffa166e 100644 --- a/devops/scripts/benchmarking/common.py +++ b/devops/scripts/benchmarking/common.py @@ -40,4 +40,12 @@ def load_configs(): raise TypeError("Error in benchmark-ci.conf: METRICS_RECORDED is not a python list.") for perf_res in perf_res_re.findall(configs_str): - PERF_RES_PATH = str(perf_res[1:-1]) \ No newline at end of file + PERF_RES_PATH = str(perf_res[1:-1]) + + +def valid_timestamp(timestamp: str) -> bool: + timestamp_re = re.compile( + # YYMMDD_HHMMSS + '^\d{2}(0[1-9]|1[0-2])(0[1-9]|[12][0-9]|3[01])_(0[0-9]|1[0-9]|2[0-3])[0-5][0-9][0-5][0-9]$' + ) + return timestamp_re.match(timestamp) is not None \ No newline at end of file From 3b5454fd7f1a6726b4d2edf75dedccb6410aeaad Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 18 Sep 2024 13:02:16 -0700 Subject: [PATCH 34/67] Fixed aggregate, improved logging --- .../workflows/sycl-benchmark-aggregate.yml | 2 +- .github/workflows/sycl-linux-benchmark.yml | 7 ++-- devops/scripts/benchmarking/aggregate.py | 6 +-- devops/scripts/benchmarking/benchmark-ci.conf | 13 ++++-- devops/scripts/benchmarking/benchmark.sh | 41 +++++++++++++++---- devops/scripts/benchmarking/common.py | 10 +++++ devops/scripts/benchmarking/compare.py | 11 +++-- 7 files changed, 66 insertions(+), 24 deletions(-) diff --git a/.github/workflows/sycl-benchmark-aggregate.yml b/.github/workflows/sycl-benchmark-aggregate.yml index d85fd78577491..da20d0186680e 100644 --- a/.github/workflows/sycl-benchmark-aggregate.yml +++ b/.github/workflows/sycl-benchmark-aggregate.yml @@ -57,7 +57,7 @@ jobs: with: # TODO populate default values before loading configuration path: ${{ env.PERF_RES_PATH }} - repository: ${{ env.PERF_RES_GIT_REPO }} + repository: intel-sandbox/llvm-ci-perf-results #${{ env.PERF_RES_GIT_REPO }} branch: ${{ env.PERF_RES_BRANCH }} - name: Run aggregator on cloned data run: | diff --git a/.github/workflows/sycl-linux-benchmark.yml b/.github/workflows/sycl-linux-benchmark.yml index 45c728bb56aa6..1a4d4680684cb 100644 --- a/.github/workflows/sycl-linux-benchmark.yml +++ b/.github/workflows/sycl-linux-benchmark.yml @@ -51,7 +51,7 @@ on: cache_results: type: boolean default: false - required: False + required: false workflow_dispatch: inputs: @@ -109,7 +109,7 @@ on: cache_results: type: boolean default: false - required: False + required: false permissions: contents: write @@ -242,8 +242,7 @@ jobs: clang++ --version ls export CMPLR_ROOT=$PWD/toolchain - ./devops/scripts/benchmarking/benchmark.sh - exit $? + ./devops/scripts/benchmarking/benchmark.sh ${{ inputs.cache_results == true && '-s' }} - name: debug -- delete after env: SSH_KEY: ${{secrets.ACTIONS_DEPLOY_KEY}} diff --git a/devops/scripts/benchmarking/aggregate.py b/devops/scripts/benchmarking/aggregate.py index 771ae56136f8e..80b5e81f8ea47 100644 --- a/devops/scripts/benchmarking/aggregate.py +++ b/devops/scripts/benchmarking/aggregate.py @@ -40,13 +40,13 @@ def csv_samples() -> list[str]: with Path(f"{common.PERF_RES_PATH}/{benchmark}") as cache_dir: # TODO check for time range; What time range do I want? return filter(lambda f: f.is_file() and - common.valid_timestamp(str(f)[-13:]) and str(f)[-13:] > cutoff, + common.valid_timestamp(str(f)[-17:-4]) and str(f)[-17:-4] > cutoff, cache_dir.glob(f"{benchmark}-*_*.csv")) # Calculate median of every desired metric: aggregate_s = dict() for sample_path in csv_samples(): - with open(sample_path, mode='r') as sample_file: + with open(sample_path, 'r') as sample_file: for s in csv.DictReader(sample_file): if s["TestCase"] not in aggregate_s: aggregate_s[s["TestCase"]] = \ @@ -71,5 +71,5 @@ def csv_samples() -> list[str]: if not common.valid_timestamp(sys.argv[2]): print(f"Bad cutoff timestamp, please use YYMMDD_HHMMSS.") exit(1) - + common.load_configs() aggregate_median(sys.argv[1], sys.argv[2]) diff --git a/devops/scripts/benchmarking/benchmark-ci.conf b/devops/scripts/benchmarking/benchmark-ci.conf index b08d82fcad1a5..adf023d9a1a2a 100644 --- a/devops/scripts/benchmarking/benchmark-ci.conf +++ b/devops/scripts/benchmarking/benchmark-ci.conf @@ -1,5 +1,5 @@ # Git branch settings for llvm-ci-perf-results -PERF_RES_GIT_REPO="https://github.com/ianayl/llvm-ci-perf-results" +PERF_RES_GIT_REPO="https://github.com/intel-sandbox/llvm-ci-perf-results" PERF_RES_BRANCH="test-compute-bench" # Path where llvm-ci-perf-results are cloned PERF_RES_PATH="./llvm-ci-perf-res" @@ -17,7 +17,7 @@ COMPUTE_BENCH_COMPILE_FLAGS="-j2" OUTPUT_PATH="." # Metrics to benchmark, and their allowed variance as a Python dictionary -METRICS_VARIANCE='{"Median": 0.5}' +METRICS_VARIANCE='{"Median": 0.15}' #METRICS_VARIANCE='{"Median": 0.5, "StdDev": 4.0}' # Metrics to record using aggregate.py @@ -25,9 +25,14 @@ METRICS_RECORDED='["Median", "StdDev"]' # Threshold to store benchmark files before benchmarking # TODO reconsider this -AVERAGE_THRESHOLD=7 +AVERAGE_THRESHOLD=3 # Default period of time to aggregate for the average AVERAGE_CUTOFF_RANGE="7 days ago" # Format of timestamps used (linux `date` format string) -TIMESTAMP_FORMAT='%Y%m%d_%H%M%S' \ No newline at end of file +TIMESTAMP_FORMAT='%Y%m%d_%H%M%S' + +# Log file for test cases that perform over the allowed variance +BENCHMARK_SLOW_LOG="./benchmarks-over_tolerance.log" +# Log file for test cases that errored / failed to build +BENCHMARK_ERROR_LOG="./benchmarks-errored.log" \ No newline at end of file diff --git a/devops/scripts/benchmarking/benchmark.sh b/devops/scripts/benchmarking/benchmark.sh index 5ad82f3c5414a..7fbabda307072 100755 --- a/devops/scripts/benchmarking/benchmark.sh +++ b/devops/scripts/benchmarking/benchmark.sh @@ -73,7 +73,7 @@ build_compute_bench() { ### STATUS_SUCCESS=0 -STATUS_FAILED=1 +STATUS_ERROR=1 ### samples_under_threshold () { @@ -89,7 +89,6 @@ check_regression() { fi BENCHMARKING_ROOT="$BENCHMARKING_ROOT" python "$BENCHMARKING_ROOT/compare.py" "$1" "$2" return $? - # return $STATUS_FAILED } cache() { @@ -100,10 +99,14 @@ cache() { check_and_cache() { echo "Checking $testcase..." if check_regression $1 $2; then - echo "Caching $testcase..." - cache $1 $2 + if [ "$CACHE_RESULTS" -eq "1" ]; then + echo "Caching $testcase..." + cache $1 $2 + fi else - echo "Not caching!" + if [ "$CACHE_RESULTS" -eq "1" ]; then + echo "Not caching!" + fi fi } @@ -113,8 +116,9 @@ process_benchmarks() { echo "### Running and processing selected benchmarks ###" if [ -z "$TESTS_CONFIG" ]; then echo "Setting tests to run via cli is not currently supported." - exit $STATUS_FAILED + exit $STATUS_ERROR else + rm "$BENCHMARK_ERROR_LOG" "$BENCHMARK_SLOW_LOG" 2> /dev/null # Ignore lines in the test config starting with #'s grep "^[^#]" "$TESTS_CONFIG" | while read -r testcase; do echo "# Running $testcase..." @@ -124,16 +128,32 @@ process_benchmarks() { if [ "$?" -eq 0 ] && [ -s "$test_csv_output" ]; then check_and_cache $testcase $test_csv_output else + # TODO consider capturing error for logging echo "ERROR @ $test_case" + echo "-- $testcase: error $?" >> "$BENCHMARK_ERROR_LOG" fi done fi } +process_results() { + if [ -s "$BENCHMARK_SLOW_LOG" ]; then + printf "\n### Tests performing over acceptable range of average: ###\n" + cat "$BENCHMARK_SLOW_LOG" + echo "" + fi + if [ -s "$BENCHMARK_ERROR_LOG" ]; then + printf "\n### Tests that failed to run: ###\n" + cat "$BENCHMARK_ERROR_LOG" + echo "" + fi + [ ! -s "$BENCHMARKING_SLOW_LOG" ] && [ ! -s "$BENCHMARK_ERROR_LOG" ] +} + cleanup() { echo "### Cleaning up compute-benchmark builds from prior runs ###" rm -rf $COMPUTE_BENCH_PATH - #rm -rf $PERF_RES_PATH + rm -rf $PERF_RES_PATH [ ! -z "$_exit_after_cleanup" ] && exit } @@ -163,10 +183,11 @@ load_configs() { load_configs COMPUTE_BENCH_COMPILE_FLAGS="" +CACHE_RESULTS="0" TIMESTAMP="$(date +"$TIMESTAMP_FORMAT")" # CLI overrides to configuration options -while getopts "p:b:r:f:cC" opt; do +while getopts "p:b:r:f:cCs" opt; do case $opt in p) COMPUTE_BENCH_PATH=$OPTARG ;; r) COMPUTE_BENCH_GIT_REPO=$OPTARG ;; @@ -176,6 +197,7 @@ while getopts "p:b:r:f:cC" opt; do # performing cleanup c) _cleanup=1 ;; C) _cleanup=1 && _exit_after_cleanup=1 ;; + s) CACHE_RESULTS="1";; \?) usage ;; esac done @@ -189,4 +211,5 @@ fi [ ! -d "$PERF_RES_PATH" ] && clone_perf_res [ ! -d "$COMPUTE_BENCH_PATH" ] && clone_compute_bench [ ! -d "$COMPUTE_BENCH_PATH/build" ] && build_compute_bench -process_benchmarks \ No newline at end of file +process_benchmarks +process_results \ No newline at end of file diff --git a/devops/scripts/benchmarking/common.py b/devops/scripts/benchmarking/common.py index 56e5a2ffa166e..b30bf82301639 100644 --- a/devops/scripts/benchmarking/common.py +++ b/devops/scripts/benchmarking/common.py @@ -3,6 +3,7 @@ import ast PERF_RES_PATH, metrics_variance, metrics_recorded = None, None, None +BENCHMARK_SLOW_LOG, BENCHMARK_ERROR_LOG = None, None def sanitize(stat: str) -> float: # Get rid of % @@ -22,9 +23,12 @@ def load_configs(): raise Exception(f"Please provide path to a valid BENCHMARKING_ROOT.") global PERF_RES_PATH, metrics_variance, metrics_recorded + global BENCHMARK_ERROR_LOG, BENCHMARK_SLOW_LOG perf_res_re = re.compile(r'^PERF_RES_PATH=(.*)$', re.M) m_variance_re = re.compile(r'^METRICS_VARIANCE=(.*)$', re.M) m_recorded_re = re.compile(r'^METRICS_RECORDED=(.*)$', re.M) + b_slow_re = re.compile(r'^BENCHMARK_SLOW_LOG=(.*)$', re.M) + b_error_re = re.compile(r'^BENCHMARK_ERROR_LOG=(.*)$', re.M) with open(benchmarking_ci_conf_path, 'r') as configs_file: configs_str = configs_file.read() @@ -41,6 +45,12 @@ def load_configs(): for perf_res in perf_res_re.findall(configs_str): PERF_RES_PATH = str(perf_res[1:-1]) + + for b_slow_log in b_slow_re.findall(configs_str): + BENCHMARK_SLOW_LOG = str(b_slow_log[1:-1]) + + for b_error_log in b_error_re.findall(configs_str): + BENCHMARK_ERROR_LOG = str(b_error_log[1:-1]) def valid_timestamp(timestamp: str) -> bool: diff --git a/devops/scripts/benchmarking/compare.py b/devops/scripts/benchmarking/compare.py index e2e9b12b0a8e4..c0565cfea1b65 100644 --- a/devops/scripts/benchmarking/compare.py +++ b/devops/scripts/benchmarking/compare.py @@ -18,7 +18,7 @@ def compare_to_median(test_name: str, test_csv_path: str): exit(-1) median = dict() - with open(median_path, mode='r') as median_csv: + with open(median_path, 'r') as median_csv: for stat in csv.DictReader(median_csv): median[stat["TestCase"]] = \ { metric: float(stat[metric]) for metric in common.metrics_variance } @@ -26,7 +26,7 @@ def compare_to_median(test_name: str, test_csv_path: str): # TODO read status codes from a config file status = 0 failure_counts = { metric: 0 for metric in common.metrics_variance } - with open(test_csv_path, mode='r') as sample_csv: + with open(test_csv_path, 'r') as sample_csv: for sample in csv.DictReader(sample_csv): # Ignore test cases we haven't profiled before if sample["TestCase"] not in median: @@ -37,8 +37,13 @@ def compare_to_median(test_name: str, test_csv_path: str): if common.sanitize(sample[metric]) > max_tolerated: print("vvv FAILED vvv") print(sample['TestCase']) - print(f"{metric}: {metric} {common.sanitize(sample[metric])} -- Historic avg. {test_median[metric]} (max tolerance {threshold*100}% -- {max_tolerated})") + print(f"{metric}: {common.sanitize(sample[metric])} -- Historic avg. {test_median[metric]} (max tolerance {threshold*100}%: {max_tolerated})") print("^^^^^^^^^^^^^^") + with open(common.BENCHMARK_SLOW_LOG, 'a') as slow_log: + slow_log.write( + f"-- {test_name}::{sample['TestCase']}\n" + f" {metric}: {common.sanitize(sample[metric])} -- Historic avg. {test_median[metric]} (max tol. {threshold*100}%: {max_tolerated})\n" + ) status = 1 failure_counts[metric] += 1 if status != 0: From 6ed0361cb7d336c61b3fdb1df5d09180510e30f0 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 18 Sep 2024 13:15:48 -0700 Subject: [PATCH 35/67] Change repo used --- .github/workflows/sycl-benchmark-aggregate.yml | 4 ++-- devops/scripts/benchmarking/benchmark-ci.conf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sycl-benchmark-aggregate.yml b/.github/workflows/sycl-benchmark-aggregate.yml index da20d0186680e..ec6893d2088d8 100644 --- a/.github/workflows/sycl-benchmark-aggregate.yml +++ b/.github/workflows/sycl-benchmark-aggregate.yml @@ -57,8 +57,8 @@ jobs: with: # TODO populate default values before loading configuration path: ${{ env.PERF_RES_PATH }} - repository: intel-sandbox/llvm-ci-perf-results #${{ env.PERF_RES_GIT_REPO }} - branch: ${{ env.PERF_RES_BRANCH }} + repository: ianayl/llvm-ci-perf-results #${{ env.PERF_RES_GIT_REPO }} + ref: ${{ env.PERF_RES_BRANCH }} - name: Run aggregator on cloned data run: | for dir in $(find "$PERF_RES_PATH" -mindepth 1 -maxdepth 1 -type d ! -name '.*'); do diff --git a/devops/scripts/benchmarking/benchmark-ci.conf b/devops/scripts/benchmarking/benchmark-ci.conf index adf023d9a1a2a..1730127a7dc5f 100644 --- a/devops/scripts/benchmarking/benchmark-ci.conf +++ b/devops/scripts/benchmarking/benchmark-ci.conf @@ -1,5 +1,5 @@ # Git branch settings for llvm-ci-perf-results -PERF_RES_GIT_REPO="https://github.com/intel-sandbox/llvm-ci-perf-results" +PERF_RES_GIT_REPO="https://github.com/ianayl/llvm-ci-perf-results" PERF_RES_BRANCH="test-compute-bench" # Path where llvm-ci-perf-results are cloned PERF_RES_PATH="./llvm-ci-perf-res" From f54e1d27b5696c3c7ecc75dde98ba29f80af85d9 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 18 Sep 2024 15:33:33 -0700 Subject: [PATCH 36/67] Resolve issue with using two action/checkout's --- .../workflows/sycl-benchmark-aggregate.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/sycl-benchmark-aggregate.yml b/.github/workflows/sycl-benchmark-aggregate.yml index ec6893d2088d8..f904df18da975 100644 --- a/.github/workflows/sycl-benchmark-aggregate.yml +++ b/.github/workflows/sycl-benchmark-aggregate.yml @@ -40,7 +40,7 @@ jobs: if [ -z '${{ inputs.cutoff_timestamp }}' ]; then # No time given, use default time period - echo "CUTOFF_TIMESTAMP=$(date --date="$AVERAGE_CUTOFF_RANGE" "$TIMESTAMP_FORMAT")" >> $GITHUB_ENV + echo "CUTOFF_TIMESTAMP=$(date --date="$AVERAGE_CUTOFF_RANGE" +"$TIMESTAMP_FORMAT")" >> $GITHUB_ENV else # If the provided time is a unix `date` timestamp, convert the time to our format _converted_timestamp="$(date --date '${{ inputs.cutoff_timestamp }}' +"$TIMESTAMP_FORMAT" 2> /dev/null)" @@ -53,20 +53,19 @@ jobs: echo 'CUTOFF_TIMESTAMP=${{ inputs.cutoff_timestamp }}' >> $GITHUB_ENV fi fi - - uses: actions/checkout@v4 - with: - # TODO populate default values before loading configuration - path: ${{ env.PERF_RES_PATH }} - repository: ianayl/llvm-ci-perf-results #${{ env.PERF_RES_GIT_REPO }} - ref: ${{ env.PERF_RES_BRANCH }} + - name: Checkout performance results repository + run: | + git clone -b $PERF_RES_BRANCH $PERF_RES_GIT_REPO $PERF_RES_PATH + - name: Debug + run: | + find "$PERF_RES_PATH" -maxdepth 3 ! -name '.*' + echo "########" + find . -maxdepth 3 ! -name '.*' - name: Run aggregator on cloned data run: | for dir in $(find "$PERF_RES_PATH" -mindepth 1 -maxdepth 1 -type d ! -name '.*'); do python llvm/devops/benchmarking/aggregate.py "$(basename dir)" "$CUTOFF_TIMESTAMP" done - - name: Debug - run: | - find "$PERF_RES_PATH" -maxdepth 3 ! -name '.*' - name: Upload average to the repo env: SSH_KEY: ${{secrets.ACTIONS_DEPLOY_KEY}} From 47693f9c82b98a13440e5927f358a8a142437776 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 18 Sep 2024 15:43:36 -0700 Subject: [PATCH 37/67] amend location of aggregate --- .github/workflows/sycl-benchmark-aggregate.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sycl-benchmark-aggregate.yml b/.github/workflows/sycl-benchmark-aggregate.yml index f904df18da975..5c13c02a6fb4e 100644 --- a/.github/workflows/sycl-benchmark-aggregate.yml +++ b/.github/workflows/sycl-benchmark-aggregate.yml @@ -30,7 +30,7 @@ jobs: with: path: llvm sparse-checkout: | - devops + devops/scripts/benchmarking/** - name: Load benchmarking configuration run: | . llvm/devops/scripts/benchmarking/benchmark-ci.conf; @@ -64,7 +64,7 @@ jobs: - name: Run aggregator on cloned data run: | for dir in $(find "$PERF_RES_PATH" -mindepth 1 -maxdepth 1 -type d ! -name '.*'); do - python llvm/devops/benchmarking/aggregate.py "$(basename dir)" "$CUTOFF_TIMESTAMP" + python llvm/devops/scripts/benchmarking/aggregate.py "$(basename dir)" "$CUTOFF_TIMESTAMP" done - name: Upload average to the repo env: From f1d3a7fac3a8cf330f14f79268301e5acfc816cd Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 18 Sep 2024 16:01:46 -0700 Subject: [PATCH 38/67] amend location of aggregate --- .github/workflows/sycl-benchmark-aggregate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sycl-benchmark-aggregate.yml b/.github/workflows/sycl-benchmark-aggregate.yml index 5c13c02a6fb4e..19acefcd42e79 100644 --- a/.github/workflows/sycl-benchmark-aggregate.yml +++ b/.github/workflows/sycl-benchmark-aggregate.yml @@ -30,7 +30,7 @@ jobs: with: path: llvm sparse-checkout: | - devops/scripts/benchmarking/** + devops/scripts/benchmarking - name: Load benchmarking configuration run: | . llvm/devops/scripts/benchmarking/benchmark-ci.conf; From 652667fd499b8d9c8a98d91e1cdadc1b06a9a446 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 18 Sep 2024 16:16:00 -0700 Subject: [PATCH 39/67] fix timestamp format in common --- devops/scripts/benchmarking/aggregate.py | 2 +- devops/scripts/benchmarking/common.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/devops/scripts/benchmarking/aggregate.py b/devops/scripts/benchmarking/aggregate.py index 80b5e81f8ea47..fc67af884be59 100644 --- a/devops/scripts/benchmarking/aggregate.py +++ b/devops/scripts/benchmarking/aggregate.py @@ -40,7 +40,7 @@ def csv_samples() -> list[str]: with Path(f"{common.PERF_RES_PATH}/{benchmark}") as cache_dir: # TODO check for time range; What time range do I want? return filter(lambda f: f.is_file() and - common.valid_timestamp(str(f)[-17:-4]) and str(f)[-17:-4] > cutoff, + common.valid_timestamp(str(f)[-19:-4]) and str(f)[-19:-4] > cutoff, cache_dir.glob(f"{benchmark}-*_*.csv")) # Calculate median of every desired metric: diff --git a/devops/scripts/benchmarking/common.py b/devops/scripts/benchmarking/common.py index b30bf82301639..f239d0f3b40dc 100644 --- a/devops/scripts/benchmarking/common.py +++ b/devops/scripts/benchmarking/common.py @@ -55,7 +55,7 @@ def load_configs(): def valid_timestamp(timestamp: str) -> bool: timestamp_re = re.compile( - # YYMMDD_HHMMSS - '^\d{2}(0[1-9]|1[0-2])(0[1-9]|[12][0-9]|3[01])_(0[0-9]|1[0-9]|2[0-3])[0-5][0-9][0-5][0-9]$' + # YYYYMMDD_HHMMSS + '^\d{4}(0[1-9]|1[0-2])(0[1-9]|[12][0-9]|3[01])_(0[0-9]|1[0-9]|2[0-3])[0-5][0-9][0-5][0-9]$' ) return timestamp_re.match(timestamp) is not None \ No newline at end of file From be755743c67be4dd9ddf29483895bd2f49d847a6 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 18 Sep 2024 16:24:52 -0700 Subject: [PATCH 40/67] fix missing $ --- .github/workflows/sycl-benchmark-aggregate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sycl-benchmark-aggregate.yml b/.github/workflows/sycl-benchmark-aggregate.yml index 19acefcd42e79..8f8dd1609247c 100644 --- a/.github/workflows/sycl-benchmark-aggregate.yml +++ b/.github/workflows/sycl-benchmark-aggregate.yml @@ -64,7 +64,7 @@ jobs: - name: Run aggregator on cloned data run: | for dir in $(find "$PERF_RES_PATH" -mindepth 1 -maxdepth 1 -type d ! -name '.*'); do - python llvm/devops/scripts/benchmarking/aggregate.py "$(basename dir)" "$CUTOFF_TIMESTAMP" + python llvm/devops/scripts/benchmarking/aggregate.py "$(basename $dir)" "$CUTOFF_TIMESTAMP" done - name: Upload average to the repo env: From 1200217cf93437ee54d822995dc0af9629718169 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 18 Sep 2024 16:57:45 -0700 Subject: [PATCH 41/67] fix missing $ --- .github/workflows/sycl-benchmark-aggregate.yml | 10 +++++----- .github/workflows/sycl-linux-benchmark.yml | 8 +------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/sycl-benchmark-aggregate.yml b/.github/workflows/sycl-benchmark-aggregate.yml index 8f8dd1609247c..ed6ddd503f8d5 100644 --- a/.github/workflows/sycl-benchmark-aggregate.yml +++ b/.github/workflows/sycl-benchmark-aggregate.yml @@ -56,16 +56,16 @@ jobs: - name: Checkout performance results repository run: | git clone -b $PERF_RES_BRANCH $PERF_RES_GIT_REPO $PERF_RES_PATH - - name: Debug - run: | - find "$PERF_RES_PATH" -maxdepth 3 ! -name '.*' - echo "########" - find . -maxdepth 3 ! -name '.*' - name: Run aggregator on cloned data run: | for dir in $(find "$PERF_RES_PATH" -mindepth 1 -maxdepth 1 -type d ! -name '.*'); do python llvm/devops/scripts/benchmarking/aggregate.py "$(basename $dir)" "$CUTOFF_TIMESTAMP" done + - name: Debug + run: | + find "$PERF_RES_PATH" -maxdepth 3 ! -name '.*' + echo "########" + find . -maxdepth 3 ! -name '.*' - name: Upload average to the repo env: SSH_KEY: ${{secrets.ACTIONS_DEPLOY_KEY}} diff --git a/.github/workflows/sycl-linux-benchmark.yml b/.github/workflows/sycl-linux-benchmark.yml index 1a4d4680684cb..428142fd4ce4e 100644 --- a/.github/workflows/sycl-linux-benchmark.yml +++ b/.github/workflows/sycl-linux-benchmark.yml @@ -220,7 +220,6 @@ jobs: rm -f ${{ inputs.sycl_toolchain_archive || needs.check_build.outputs.artifact_archive_name }} echo PATH=$PWD/toolchain/bin/:$PATH >> $GITHUB_ENV echo LD_LIBRARY_PATH=$PWD/toolchain/lib/:$LD_LIBRARY_PATH >> $GITHUB_ENV - - run: ls - run: which clang++ sycl-ls - run: sycl-ls --verbose - run: SYCL_UR_TRACE=1 sycl-ls @@ -243,12 +242,7 @@ jobs: ls export CMPLR_ROOT=$PWD/toolchain ./devops/scripts/benchmarking/benchmark.sh ${{ inputs.cache_results == true && '-s' }} - - name: debug -- delete after - env: - SSH_KEY: ${{secrets.ACTIONS_DEPLOY_KEY}} - run: - # Is this key even a thing? - echo "$SSH_KEY" | head -c 4 + echo $? - name: Push compute-benchmarks if: inputs.cache_results == true && success() env: From 97b2d4f96bb6a1067938eb8077596522641b5bab Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 18 Sep 2024 17:31:39 -0700 Subject: [PATCH 42/67] exit on error --- .github/workflows/sycl-linux-benchmark.yml | 1 - devops/scripts/benchmarking/benchmark.sh | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sycl-linux-benchmark.yml b/.github/workflows/sycl-linux-benchmark.yml index 428142fd4ce4e..040fb31969d4c 100644 --- a/.github/workflows/sycl-linux-benchmark.yml +++ b/.github/workflows/sycl-linux-benchmark.yml @@ -242,7 +242,6 @@ jobs: ls export CMPLR_ROOT=$PWD/toolchain ./devops/scripts/benchmarking/benchmark.sh ${{ inputs.cache_results == true && '-s' }} - echo $? - name: Push compute-benchmarks if: inputs.cache_results == true && success() env: diff --git a/devops/scripts/benchmarking/benchmark.sh b/devops/scripts/benchmarking/benchmark.sh index 7fbabda307072..07d3950dd2b06 100755 --- a/devops/scripts/benchmarking/benchmark.sh +++ b/devops/scripts/benchmarking/benchmark.sh @@ -147,7 +147,9 @@ process_results() { cat "$BENCHMARK_ERROR_LOG" echo "" fi - [ ! -s "$BENCHMARKING_SLOW_LOG" ] && [ ! -s "$BENCHMARK_ERROR_LOG" ] + if [ -s "$BENCHMARKING_SLOW_LOG" ] || [ -s "$BENCHMARK_ERROR_LOG" ]; then + exit 1 + fi } cleanup() { From 08da292290943d4066135fbb54a56cebc24c60e6 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 18 Sep 2024 18:05:31 -0700 Subject: [PATCH 43/67] exit on error --- devops/scripts/benchmarking/benchmark.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/devops/scripts/benchmarking/benchmark.sh b/devops/scripts/benchmarking/benchmark.sh index 07d3950dd2b06..0cc74e64fa455 100755 --- a/devops/scripts/benchmarking/benchmark.sh +++ b/devops/scripts/benchmarking/benchmark.sh @@ -137,19 +137,20 @@ process_benchmarks() { } process_results() { + fail=0 if [ -s "$BENCHMARK_SLOW_LOG" ]; then printf "\n### Tests performing over acceptable range of average: ###\n" cat "$BENCHMARK_SLOW_LOG" echo "" + fail=1 fi if [ -s "$BENCHMARK_ERROR_LOG" ]; then printf "\n### Tests that failed to run: ###\n" cat "$BENCHMARK_ERROR_LOG" echo "" + fail=2 fi - if [ -s "$BENCHMARKING_SLOW_LOG" ] || [ -s "$BENCHMARK_ERROR_LOG" ]; then - exit 1 - fi + exit fail } cleanup() { From 441bc10e31a8d9e70e3c6ce4b4b52926a2179976 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Fri, 20 Sep 2024 07:44:16 -0700 Subject: [PATCH 44/67] update config to trigger new run --- devops/scripts/benchmarking/benchmark-ci.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops/scripts/benchmarking/benchmark-ci.conf b/devops/scripts/benchmarking/benchmark-ci.conf index 1730127a7dc5f..cd81f3630a9ba 100644 --- a/devops/scripts/benchmarking/benchmark-ci.conf +++ b/devops/scripts/benchmarking/benchmark-ci.conf @@ -17,7 +17,7 @@ COMPUTE_BENCH_COMPILE_FLAGS="-j2" OUTPUT_PATH="." # Metrics to benchmark, and their allowed variance as a Python dictionary -METRICS_VARIANCE='{"Median": 0.15}' +METRICS_VARIANCE='{"Median": 0.3}' #METRICS_VARIANCE='{"Median": 0.5, "StdDev": 4.0}' # Metrics to record using aggregate.py From 8979115725171d97e251382b147d9bc4de1953ad Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Fri, 20 Sep 2024 08:27:55 -0700 Subject: [PATCH 45/67] update config to trigger new run --- devops/scripts/benchmarking/benchmark-ci.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops/scripts/benchmarking/benchmark-ci.conf b/devops/scripts/benchmarking/benchmark-ci.conf index cd81f3630a9ba..ad5c3309924af 100644 --- a/devops/scripts/benchmarking/benchmark-ci.conf +++ b/devops/scripts/benchmarking/benchmark-ci.conf @@ -17,7 +17,7 @@ COMPUTE_BENCH_COMPILE_FLAGS="-j2" OUTPUT_PATH="." # Metrics to benchmark, and their allowed variance as a Python dictionary -METRICS_VARIANCE='{"Median": 0.3}' +METRICS_VARIANCE='{"Median": 0.5}' #METRICS_VARIANCE='{"Median": 0.5, "StdDev": 4.0}' # Metrics to record using aggregate.py From 7fa6a2ea774cecbc280178d7ccb53ab9a09eb068 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Fri, 20 Sep 2024 13:47:01 -0700 Subject: [PATCH 46/67] Fix up typo --- devops/scripts/benchmarking/benchmark.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops/scripts/benchmarking/benchmark.sh b/devops/scripts/benchmarking/benchmark.sh index 0cc74e64fa455..cda5788d42dd7 100755 --- a/devops/scripts/benchmarking/benchmark.sh +++ b/devops/scripts/benchmarking/benchmark.sh @@ -150,7 +150,7 @@ process_results() { echo "" fail=2 fi - exit fail + exit $fail } cleanup() { From 21498b4a7bf935da17d7e132cbab44ed837a6b2e Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Tue, 15 Oct 2024 17:25:16 -0700 Subject: [PATCH 47/67] Revert permissions on previous files --- .github/workflows/sycl-linux-build.yml | 3 +-- .github/workflows/sycl-linux-run-tests.yml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sycl-linux-build.yml b/.github/workflows/sycl-linux-build.yml index e982c61e5c1ef..5ceeaca2c9564 100644 --- a/.github/workflows/sycl-linux-build.yml +++ b/.github/workflows/sycl-linux-build.yml @@ -117,8 +117,7 @@ on: options: - 3 -permissions: - contents: read +permissions: read-all jobs: build: diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index 3e4b2678abfef..4128a3d766a6d 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -28,7 +28,7 @@ on: required: False tests_selector: description: | - Two options: "e2e", "cts" + Two possible options: "e2e" and "cts". type: string default: "e2e" From 8506c205e36a69e86de84cf5ff0befefae620a72 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Tue, 15 Oct 2024 17:34:13 -0700 Subject: [PATCH 48/67] Amend permissions to be more strict --- .github/workflows/sycl-benchmark-aggregate.yml | 3 +-- .github/workflows/sycl-linux-benchmark.yml | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/sycl-benchmark-aggregate.yml b/.github/workflows/sycl-benchmark-aggregate.yml index ed6ddd503f8d5..905b126691cf2 100644 --- a/.github/workflows/sycl-benchmark-aggregate.yml +++ b/.github/workflows/sycl-benchmark-aggregate.yml @@ -15,8 +15,7 @@ on: type: string required: false -permissions: - contents: write +permissions: None jobs: aggregate: diff --git a/.github/workflows/sycl-linux-benchmark.yml b/.github/workflows/sycl-linux-benchmark.yml index 040fb31969d4c..42813333e1b2c 100644 --- a/.github/workflows/sycl-linux-benchmark.yml +++ b/.github/workflows/sycl-linux-benchmark.yml @@ -112,14 +112,12 @@ on: required: false permissions: - contents: write + contents: read jobs: check_build: name: Build (if no sycl artifact provided) uses: ./.github/workflows/sycl-linux-build.yml - permissions: - contents: read with: build_ref: ${{ github.sha }} merge_ref: '' From c2835cd00f94341aebf16b738f0ebfb52c3d6706 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Tue, 15 Oct 2024 17:34:55 -0700 Subject: [PATCH 49/67] Amend permissions on aggregate --- .github/workflows/sycl-benchmark-aggregate.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sycl-benchmark-aggregate.yml b/.github/workflows/sycl-benchmark-aggregate.yml index 905b126691cf2..2a0ee74de1675 100644 --- a/.github/workflows/sycl-benchmark-aggregate.yml +++ b/.github/workflows/sycl-benchmark-aggregate.yml @@ -15,7 +15,8 @@ on: type: string required: false -permissions: None +permissions: + contents: read jobs: aggregate: From 51cdaeefe805981e8963a84a3756c19c335853a6 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Tue, 15 Oct 2024 17:39:55 -0700 Subject: [PATCH 50/67] Fix bug with permissions --- .github/workflows/sycl-linux-precommit.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml index 93d8a75c855ec..6ca4f3eac9299 100644 --- a/.github/workflows/sycl-linux-precommit.yml +++ b/.github/workflows/sycl-linux-precommit.yml @@ -189,8 +189,6 @@ jobs: benchmark: needs: [build, detect_changes ] - permissions: - contents: write if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }} strategy: fail-fast: false From ef6a085899578b994da5c972e76711fd8ea6c808 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Tue, 15 Oct 2024 17:41:36 -0700 Subject: [PATCH 51/67] Revert amend --- .github/workflows/sycl-linux-precommit.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml index 6ca4f3eac9299..93d8a75c855ec 100644 --- a/.github/workflows/sycl-linux-precommit.yml +++ b/.github/workflows/sycl-linux-precommit.yml @@ -189,6 +189,8 @@ jobs: benchmark: needs: [build, detect_changes ] + permissions: + contents: write if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }} strategy: fail-fast: false From 2fd6b7bb7f3e33f257b874c490def8483df38b94 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Tue, 15 Oct 2024 17:43:30 -0700 Subject: [PATCH 52/67] Amend benchmarks permissions --- .github/workflows/sycl-linux-benchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sycl-linux-benchmark.yml b/.github/workflows/sycl-linux-benchmark.yml index 42813333e1b2c..5425b9b941c69 100644 --- a/.github/workflows/sycl-linux-benchmark.yml +++ b/.github/workflows/sycl-linux-benchmark.yml @@ -118,6 +118,7 @@ jobs: check_build: name: Build (if no sycl artifact provided) uses: ./.github/workflows/sycl-linux-build.yml + permissions: read-all with: build_ref: ${{ github.sha }} merge_ref: '' From 1d5c676f4c2bd6abc3d2922dc258fcf46eda0b6b Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Tue, 15 Oct 2024 17:46:34 -0700 Subject: [PATCH 53/67] Make build permissions more restrictive --- .github/workflows/sycl-linux-build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sycl-linux-build.yml b/.github/workflows/sycl-linux-build.yml index 5ceeaca2c9564..e982c61e5c1ef 100644 --- a/.github/workflows/sycl-linux-build.yml +++ b/.github/workflows/sycl-linux-build.yml @@ -117,7 +117,8 @@ on: options: - 3 -permissions: read-all +permissions: + contents: read jobs: build: From c20d75ed0838554af060ec0610822f7dc26e1e5a Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Tue, 15 Oct 2024 17:47:23 -0700 Subject: [PATCH 54/67] Amend benchmark permissions --- .github/workflows/sycl-linux-benchmark.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sycl-linux-benchmark.yml b/.github/workflows/sycl-linux-benchmark.yml index 5425b9b941c69..7d4015311df9e 100644 --- a/.github/workflows/sycl-linux-benchmark.yml +++ b/.github/workflows/sycl-linux-benchmark.yml @@ -118,7 +118,8 @@ jobs: check_build: name: Build (if no sycl artifact provided) uses: ./.github/workflows/sycl-linux-build.yml - permissions: read-all + permissions: + contents: read with: build_ref: ${{ github.sha }} merge_ref: '' From a9dad800796febc91a7c3521b17aff9c232fb7f0 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Tue, 22 Oct 2024 11:40:06 -0700 Subject: [PATCH 55/67] Try new config file loading --- devops/scripts/benchmarking/benchmark.sh | 44 +++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/devops/scripts/benchmarking/benchmark.sh b/devops/scripts/benchmarking/benchmark.sh index cda5788d42dd7..ee07cac8f3b05 100755 --- a/devops/scripts/benchmarking/benchmark.sh +++ b/devops/scripts/benchmarking/benchmark.sh @@ -160,6 +160,10 @@ cleanup() { [ ! -z "$_exit_after_cleanup" ] && exit } +_sanitize_configs() { + echo "$1" | sed 's/[^a-zA-Z0-9_-.:/]//g' +} + load_configs() { # This script needs to know where the "BENCHMARKING_ROOT" directory is, # containing all the configuration files and the compare script. @@ -180,7 +184,45 @@ load_configs() { fi done - . $BENCHMARK_CI_CONFIG + # Strict loading of configuration options: + while IFS='=' read -r key value; do + sanitized_value=$(_sanitize_configs "$value") + case "$key" in + 'PERF_RES_GIT_REPO') export PERF_RES_GIT_REPO="$sanitized_value" ;; + 'PERF_RES_BRANCH') export PERF_RES_BRANCH="$sanitized_value" ;; + 'PERF_RES_PATH') export PERF_RES_PATH="$sanitized_value" ;; + 'COMPUTE_BENCH_GIT_REPO') export COMPUTE_BENCH_GIT_REPO="$sanitized_value" ;; + 'COMPUTE_BENCH_BRANCH') export COMPUTE_BENCH_BRANCH="$sanitized_value" ;; + 'COMPUTE_BENCH_PATH') export COMPUTE_BENCH_PATH="$sanitized_value" ;; + 'COMPUTE_BENCH_COMPILE_FLAGS') export COMPUTE_BENCH_COMPILE_FLAGS="$sanitized_value" ;; + 'OUTPUT_PATH') export OUTPUT_PATH="$sanitized_value" ;; + # 'METRICS_VARIANCE') export METRICS_VARIANCE="$sanitized_value" ;; + # 'METRICS_RECORDED') export METRICS_RECORDED="$sanitized_value" ;; + 'AVERAGE_THRESHOLD') export AVERAGE_THRESHOLD="$sanitized_value" ;; + 'AVERAGE_CUTOFF_RANGE') export AVERAGE_CUTOFF_RANGE="$sanitized_value" ;; + 'TIMESTAMP_FORMAT') export TIMESTAMP_FORMAT="$sanitized_value" ;; + 'BENCHMARK_SLOW_LOG') export BENCHMARK_SLOW_LOG="$sanitized_value" ;; + 'BENCHMARK_ERROR_LOG') export BENCHMARK_ERROR_LOG="$sanitized_value" ;; + *) echo "Unknown key: $sanitized_key" ;; + esac + done < "$BENCHMARK_CI_CONFIG" + + # Debug + echo "PERF_RES_GIT_REPO: $PERF_RES_GIT_REPO" + echo "PERF_RES_BRANCH: $PERF_RES_BRANCH" + echo "PERF_RES_PATH: $PERF_RES_PATH" + echo "COMPUTE_BENCH_GIT_REPO: $COMPUTE_BENCH_GIT_REPO" + echo "COMPUTE_BENCH_BRANCH: $COMPUTE_BENCH_BRANCH" + echo "COMPUTE_BENCH_PATH: $COMPUTE_BENCH_PATH" + echo "COMPUTE_BENCH_COMPILE_FLAGS: $COMPUTE_BENCH_COMPILE_FLAGS" + echo "OUTPUT_PATH: $OUTPUT_PATH" + echo "METRICS_VARIANCE: $METRICS_VARIANCE" + echo "METRICS_RECORDED: $METRICS_RECORDED" + echo "AVERAGE_THRESHOLD: $AVERAGE_THRESHOLD" + echo "AVERAGE_CUTOFF_RANGE: $AVERAGE_CUTOFF_RANGE" + echo "TIMESTAMP_FORMAT: $TIMESTAMP_FORMAT" + echo "BENCHMARK_SLOW_LOG: $BENCHMARK_SLOW_LOG" + echo "BENCHMARK_ERROR_LOG: $BENCHMARK_ERROR_LOG" } load_configs From 0c3e901ad6c05218f6b0b4e17e77f1b397f63924 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Tue, 22 Oct 2024 11:56:20 -0700 Subject: [PATCH 56/67] Fixed sed expression --- devops/scripts/benchmarking/benchmark.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops/scripts/benchmarking/benchmark.sh b/devops/scripts/benchmarking/benchmark.sh index ee07cac8f3b05..bc42218193c36 100755 --- a/devops/scripts/benchmarking/benchmark.sh +++ b/devops/scripts/benchmarking/benchmark.sh @@ -161,7 +161,7 @@ cleanup() { } _sanitize_configs() { - echo "$1" | sed 's/[^a-zA-Z0-9_-.:/]//g' + echo "$1" | sed 's/[^a-zA-Z0-9_.:/-]//g' } load_configs() { From 12326ce7b5a73c59ba2a3d6654e4b4cc1fcd6078 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Tue, 22 Oct 2024 12:00:10 -0700 Subject: [PATCH 57/67] trigger workflow --- devops/scripts/benchmarking/benchmark.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/devops/scripts/benchmarking/benchmark.sh b/devops/scripts/benchmarking/benchmark.sh index bc42218193c36..976e25dfbb6d1 100755 --- a/devops/scripts/benchmarking/benchmark.sh +++ b/devops/scripts/benchmarking/benchmark.sh @@ -216,8 +216,8 @@ load_configs() { echo "COMPUTE_BENCH_PATH: $COMPUTE_BENCH_PATH" echo "COMPUTE_BENCH_COMPILE_FLAGS: $COMPUTE_BENCH_COMPILE_FLAGS" echo "OUTPUT_PATH: $OUTPUT_PATH" - echo "METRICS_VARIANCE: $METRICS_VARIANCE" - echo "METRICS_RECORDED: $METRICS_RECORDED" + # echo "METRICS_VARIANCE: $METRICS_VARIANCE" + # echo "METRICS_RECORDED: $METRICS_RECORDED" echo "AVERAGE_THRESHOLD: $AVERAGE_THRESHOLD" echo "AVERAGE_CUTOFF_RANGE: $AVERAGE_CUTOFF_RANGE" echo "TIMESTAMP_FORMAT: $TIMESTAMP_FORMAT" From ba29015b850bf7c1b7c1babe703bde609f7b7410 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 30 Oct 2024 08:24:29 -0700 Subject: [PATCH 58/67] Added iteration options, changed build to each case --- devops/scripts/benchmarking/benchmark-ci.conf | 4 +++- devops/scripts/benchmarking/benchmark.sh | 21 ++++++++++++------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/devops/scripts/benchmarking/benchmark-ci.conf b/devops/scripts/benchmarking/benchmark-ci.conf index ad5c3309924af..8c6ee69c4eb7e 100644 --- a/devops/scripts/benchmarking/benchmark-ci.conf +++ b/devops/scripts/benchmarking/benchmark-ci.conf @@ -12,6 +12,8 @@ COMPUTE_BENCH_BRANCH="update-sycl" COMPUTE_BENCH_PATH="./compute-benchmarks" # Compile flags used to build compute-benchmarks COMPUTE_BENCH_COMPILE_FLAGS="-j2" +# Number of iterations to run tests for +COMPUTE_BENCH_ITERATIONS="50" # Path to temporarily store compute-benchmark results OUTPUT_PATH="." @@ -35,4 +37,4 @@ TIMESTAMP_FORMAT='%Y%m%d_%H%M%S' # Log file for test cases that perform over the allowed variance BENCHMARK_SLOW_LOG="./benchmarks-over_tolerance.log" # Log file for test cases that errored / failed to build -BENCHMARK_ERROR_LOG="./benchmarks-errored.log" \ No newline at end of file +BENCHMARK_ERROR_LOG="./benchmarks-errored.log" diff --git a/devops/scripts/benchmarking/benchmark.sh b/devops/scripts/benchmarking/benchmark.sh index 976e25dfbb6d1..47b0441328023 100755 --- a/devops/scripts/benchmarking/benchmark.sh +++ b/devops/scripts/benchmarking/benchmark.sh @@ -37,7 +37,13 @@ clone_compute_bench() { build_compute_bench() { echo "### Building compute-benchmarks ($COMPUTE_BENCH_GIT_REPO:$COMPUTE_BENCH_BRANCH) ###" mkdir $COMPUTE_BENCH_PATH/build && cd $COMPUTE_BENCH_PATH/build && - cmake .. -DBUILD_SYCL=ON -DCCACHE_ALLOWED=FALSE && cmake --build . $COMPUTE_BENCH_COMPILE_FLAGS + cmake .. -DBUILD_SYCL=ON -DCCACHE_ALLOWED=FALSE # && cmake --build . $COMPUTE_BENCH_COMPILE_FLAGS + + if [ "$?" -eq 0 ]; then + tail -n +2 $TESTS_CONFIG | while IFS= read -r case; do + make -n $COMPUTE_BENCH_COMPILE_FLAGS "$case" + done + fi # No reason to turn on ccache, if this docker image will be disassembled later on #compute_bench_build_stat=$? cd - @@ -123,7 +129,7 @@ process_benchmarks() { grep "^[^#]" "$TESTS_CONFIG" | while read -r testcase; do echo "# Running $testcase..." test_csv_output="$OUTPUT_PATH/$testcase-$TIMESTAMP.csv" - $COMPUTE_BENCH_PATH/build/bin/$testcase --csv | tail +8 > "$test_csv_output" + $COMPUTE_BENCH_PATH/build/bin/$testcase --csv --iterations="$COMPUTE_BENCH_ITERATIONS" | tail +8 > "$test_csv_output" # The tail +8 filters out initial debug prints not in csv format if [ "$?" -eq 0 ] && [ -s "$test_csv_output" ]; then check_and_cache $testcase $test_csv_output @@ -161,7 +167,7 @@ cleanup() { } _sanitize_configs() { - echo "$1" | sed 's/[^a-zA-Z0-9_.:/-]//g' + echo "$1" | sed 's/[^a-zA-Z0-9_.:/%-]//g' } load_configs() { @@ -173,9 +179,9 @@ load_configs() { # configs and scripts are reachable. [ -z "$BENCHMARKING_ROOT" ] && BENCHMARKING_ROOT="$(dirname $0)" - BENCHMARK_CI_CONFIG="$BENCHMARKING_ROOT/benchmark-ci.conf" - TESTS_CONFIG="$BENCHMARKING_ROOT/enabled_tests.conf" - COMPARE_PATH="$BENCHMARKING_ROOT/compare.py" + BENCHMARK_CI_CONFIG="$(realpath $BENCHMARKING_ROOT/benchmark-ci.conf)" + TESTS_CONFIG="$(realpath $BENCHMARKING_ROOT/enabled_tests.conf)" + COMPARE_PATH="$(realpath $BENCHMARKING_ROOT/compare.py)" for file in "$BENCHMARK_CI_CONFIG" "$TESTS_CONFIG" "$COMPARE_PATH"; do if [ ! -f "$file" ]; then @@ -195,6 +201,7 @@ load_configs() { 'COMPUTE_BENCH_BRANCH') export COMPUTE_BENCH_BRANCH="$sanitized_value" ;; 'COMPUTE_BENCH_PATH') export COMPUTE_BENCH_PATH="$sanitized_value" ;; 'COMPUTE_BENCH_COMPILE_FLAGS') export COMPUTE_BENCH_COMPILE_FLAGS="$sanitized_value" ;; + 'COMPUTE_BENCH_ITERATIONS') export COMPUTE_BENCH_ITERATIONS="$sanitized_value" ;; 'OUTPUT_PATH') export OUTPUT_PATH="$sanitized_value" ;; # 'METRICS_VARIANCE') export METRICS_VARIANCE="$sanitized_value" ;; # 'METRICS_RECORDED') export METRICS_RECORDED="$sanitized_value" ;; @@ -203,7 +210,7 @@ load_configs() { 'TIMESTAMP_FORMAT') export TIMESTAMP_FORMAT="$sanitized_value" ;; 'BENCHMARK_SLOW_LOG') export BENCHMARK_SLOW_LOG="$sanitized_value" ;; 'BENCHMARK_ERROR_LOG') export BENCHMARK_ERROR_LOG="$sanitized_value" ;; - *) echo "Unknown key: $sanitized_key" ;; + # *) echo "Unknown key: $sanitized_key" ;; esac done < "$BENCHMARK_CI_CONFIG" From 90fe17fa79d0ff166ebd874feef1e9ac65a0e970 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 30 Oct 2024 09:41:51 -0700 Subject: [PATCH 59/67] Add debug prints --- devops/scripts/benchmarking/benchmark.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/devops/scripts/benchmarking/benchmark.sh b/devops/scripts/benchmarking/benchmark.sh index 47b0441328023..dd9da239e1e74 100755 --- a/devops/scripts/benchmarking/benchmark.sh +++ b/devops/scripts/benchmarking/benchmark.sh @@ -44,6 +44,9 @@ build_compute_bench() { make -n $COMPUTE_BENCH_COMPILE_FLAGS "$case" done fi + echo "###" + ls $COMPUTE_BENCH_PATH/build/bin + echo "###" # No reason to turn on ccache, if this docker image will be disassembled later on #compute_bench_build_stat=$? cd - From 59e38feff032d13bb2726e4ae7b6f95b011b40a2 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 30 Oct 2024 09:44:03 -0700 Subject: [PATCH 60/67] Make changes to the debug prints --- devops/scripts/benchmarking/benchmark.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/devops/scripts/benchmarking/benchmark.sh b/devops/scripts/benchmarking/benchmark.sh index dd9da239e1e74..7b68375cfe9ff 100755 --- a/devops/scripts/benchmarking/benchmark.sh +++ b/devops/scripts/benchmarking/benchmark.sh @@ -41,12 +41,14 @@ build_compute_bench() { if [ "$?" -eq 0 ]; then tail -n +2 $TESTS_CONFIG | while IFS= read -r case; do - make -n $COMPUTE_BENCH_COMPILE_FLAGS "$case" + make $COMPUTE_BENCH_COMPILE_FLAGS "$case" done fi echo "###" ls $COMPUTE_BENCH_PATH/build/bin echo "###" + ls $COMPUTE_BENCH_PATH/build/ + echo "###" # No reason to turn on ccache, if this docker image will be disassembled later on #compute_bench_build_stat=$? cd - From d212adccadf2758f9165468f6d26682fc4defd2f Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Wed, 30 Oct 2024 10:52:47 -0700 Subject: [PATCH 61/67] Upped number of iterations --- devops/scripts/benchmarking/benchmark-ci.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops/scripts/benchmarking/benchmark-ci.conf b/devops/scripts/benchmarking/benchmark-ci.conf index 8c6ee69c4eb7e..c99ed1fb70abc 100644 --- a/devops/scripts/benchmarking/benchmark-ci.conf +++ b/devops/scripts/benchmarking/benchmark-ci.conf @@ -13,7 +13,7 @@ COMPUTE_BENCH_PATH="./compute-benchmarks" # Compile flags used to build compute-benchmarks COMPUTE_BENCH_COMPILE_FLAGS="-j2" # Number of iterations to run tests for -COMPUTE_BENCH_ITERATIONS="50" +COMPUTE_BENCH_ITERATIONS="100" # Path to temporarily store compute-benchmark results OUTPUT_PATH="." From 321d83aacc68533e3c4bc05df1b9e1e7c8e92c55 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Mon, 11 Nov 2024 15:27:07 -0800 Subject: [PATCH 62/67] increase iterations --- devops/scripts/benchmarking/benchmark-ci.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops/scripts/benchmarking/benchmark-ci.conf b/devops/scripts/benchmarking/benchmark-ci.conf index c99ed1fb70abc..76b82c58f5fa5 100644 --- a/devops/scripts/benchmarking/benchmark-ci.conf +++ b/devops/scripts/benchmarking/benchmark-ci.conf @@ -13,7 +13,7 @@ COMPUTE_BENCH_PATH="./compute-benchmarks" # Compile flags used to build compute-benchmarks COMPUTE_BENCH_COMPILE_FLAGS="-j2" # Number of iterations to run tests for -COMPUTE_BENCH_ITERATIONS="100" +COMPUTE_BENCH_ITERATIONS="1000" # Path to temporarily store compute-benchmark results OUTPUT_PATH="." From ba3c45c05366cc34796ee2619c6a3c526ec844e8 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Tue, 26 Nov 2024 12:32:04 -0800 Subject: [PATCH 63/67] Test PVC --- .github/workflows/sycl-linux-precommit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml index 93d8a75c855ec..09cc2f1cd52f0 100644 --- a/.github/workflows/sycl-linux-precommit.yml +++ b/.github/workflows/sycl-linux-precommit.yml @@ -202,7 +202,7 @@ jobs: # image_options: -u 1001 --gpus all --cap-add SYS_ADMIN # target_devices: ext_oneapi_cuda:gpu - name: Run benchmarks on Intel - runner: '["Linux", "gen12"]' + runner: '["Linux", "pvc"]' image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN target_devices: level_zero:gpu;opencl:gpu;opencl:cpu From 410666e45d2fcf1e5612a25cb9b7fb03f16d67cc Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Thu, 2 Jan 2025 14:55:25 -0800 Subject: [PATCH 64/67] disable igc dev for now --- .../workflows/sycl-containers-igc-dev.yaml | 108 +++++++++--------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/.github/workflows/sycl-containers-igc-dev.yaml b/.github/workflows/sycl-containers-igc-dev.yaml index ae2a660a79e93..634998255a5f2 100644 --- a/.github/workflows/sycl-containers-igc-dev.yaml +++ b/.github/workflows/sycl-containers-igc-dev.yaml @@ -1,54 +1,54 @@ -name: IGC DEV CI Containers -on: - workflow_dispatch: - push: - branches: - - sycl - paths: - - 'devops/actions/build_container/**' - - 'devops/scripts/**' - - 'devops/dependencies-igc-dev.json' - - '.github/workflows/sycl-containers-igc-dev.yaml' - pull_request: - paths: - - 'devops/actions/build_container/**' - - 'devops/scripts/**' - - 'devops/dependencies-igc-dev.json' - - '.github/workflows/sycl-containers-igc-dev.yaml' - -permissions: read-all - -jobs: - build_and_push_images: - if: github.repository == 'intel/llvm' - name: Build and Push IGC Dev Docker Images - runs-on: ubuntu-22.04 - permissions: - packages: write - strategy: - matrix: - include: - - name: Intel Drivers Ubuntu 24.04 Docker image with dev IGC - dockerfile: ubuntu2404_intel_drivers_igc_dev - imagefile: ubuntu2404_intel_drivers - tag: devigc - build_args: | - "use_unstable_driver=false" - "use_igc_dev=true" - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - name: Build and Push Container - uses: ./devops/actions/build_container - with: - push: ${{ github.event_name != 'pull_request' }} - file: ${{ matrix.dockerfile }} - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - tags: | - ghcr.io/${{ github.repository }}/${{ matrix.imagefile }}:${{ matrix.tag }}-${{ github.sha }} - ghcr.io/${{ github.repository }}/${{ matrix.imagefile }}:${{ matrix.tag }} - build-args: ${{ matrix.build_args }} - +# name: IGC DEV CI Containers +# on: +# workflow_dispatch: +# push: +# branches: +# - sycl +# paths: +# - 'devops/actions/build_container/**' +# - 'devops/scripts/**' +# - 'devops/dependencies-igc-dev.json' +# - '.github/workflows/sycl-containers-igc-dev.yaml' +# pull_request: +# paths: +# - 'devops/actions/build_container/**' +# - 'devops/scripts/**' +# - 'devops/dependencies-igc-dev.json' +# - '.github/workflows/sycl-containers-igc-dev.yaml' +# +# permissions: read-all +# +# jobs: +# build_and_push_images: +# if: github.repository == 'intel/llvm' +# name: Build and Push IGC Dev Docker Images +# runs-on: ubuntu-22.04 +# permissions: +# packages: write +# strategy: +# matrix: +# include: +# - name: Intel Drivers Ubuntu 24.04 Docker image with dev IGC +# dockerfile: ubuntu2404_intel_drivers_igc_dev +# imagefile: ubuntu2404_intel_drivers +# tag: devigc +# build_args: | +# "use_unstable_driver=false" +# "use_igc_dev=true" +# steps: +# - name: Checkout +# uses: actions/checkout@v4 +# with: +# fetch-depth: 2 +# - name: Build and Push Container +# uses: ./devops/actions/build_container +# with: +# push: ${{ github.event_name != 'pull_request' }} +# file: ${{ matrix.dockerfile }} +# username: ${{ github.repository_owner }} +# password: ${{ secrets.GITHUB_TOKEN }} +# tags: | +# ghcr.io/${{ github.repository }}/${{ matrix.imagefile }}:${{ matrix.tag }}-${{ github.sha }} +# ghcr.io/${{ github.repository }}/${{ matrix.imagefile }}:${{ matrix.tag }} +# build-args: ${{ matrix.build_args }} +# From c32ad362e57b426a5ed4a232322f653227cd0568 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Thu, 2 Jan 2025 15:04:42 -0800 Subject: [PATCH 65/67] fix benchmarking --- .github/workflows/sycl-linux-precommit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml index 220822b911591..3ab4497d6f0b9 100644 --- a/.github/workflows/sycl-linux-precommit.yml +++ b/.github/workflows/sycl-linux-precommit.yml @@ -281,7 +281,7 @@ jobs: # env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }' # use_igc_dev: true - uses: ./.github/workflows/sycl-linux-run-tests.yml + uses: ./.github/workflows/sycl-linux-benchmark.yml with: name: ${{ matrix.name }} runner: ${{ matrix. runner }} From b5fa113a71d5d01990d99fcf30c4bfa395a7360c Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Thu, 2 Jan 2025 15:11:00 -0800 Subject: [PATCH 66/67] fix benchmarking --- .github/workflows/sycl-linux-precommit.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml index 3ab4497d6f0b9..7b745514af380 100644 --- a/.github/workflows/sycl-linux-precommit.yml +++ b/.github/workflows/sycl-linux-precommit.yml @@ -242,6 +242,8 @@ jobs: benchmark: needs: [build, detect_changes] + permissions: + contents: write if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }} strategy: fail-fast: false @@ -264,7 +266,6 @@ jobs: image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN target_devices: level_zero:gpu;opencl:gpu reset_intel_gpu: true - extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True env: | { "LIT_FILTER": ${{ needs.determine_arc_tests.outputs.arc_tests }}, @@ -288,11 +289,10 @@ jobs: image: ${{ matrix.image }} image_options: ${{ matrix.image_options }} target_devices: ${{ matrix.target_devices }} - extra_lit_opts: ${{ matrix.extra_lit_opts }} env: ${{ matrix.env || '{}' }} # TODO integrate these - # # Do not install drivers on AMD and CUDA runners. + # # Do not install drivers on AMD and CUDA runnersu # install_igc_driver: >- # ${{ !contains(matrix.target_devices, 'ext_oneapi_cuda') && # !contains(matrix.target_devices, 'ext_oneapi_hip') && From 0ae396d5d1bd90c7145428822a4954f1f798e872 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Thu, 9 Jan 2025 16:06:34 -0800 Subject: [PATCH 67/67] Introduce separating of results based on runners --- .../workflows/sycl-benchmark-aggregate.yml | 8 +- .github/workflows/sycl-linux-benchmark.yml | 3 +- devops/scripts/benchmarking/aggregate.py | 17 +++-- devops/scripts/benchmarking/benchmark-ci.conf | 8 +- devops/scripts/benchmarking/benchmark.sh | 73 ++++++++++++------- devops/scripts/benchmarking/compare.py | 10 +-- 6 files changed, 73 insertions(+), 46 deletions(-) diff --git a/.github/workflows/sycl-benchmark-aggregate.yml b/.github/workflows/sycl-benchmark-aggregate.yml index 2a0ee74de1675..0c2a963cfa4d0 100644 --- a/.github/workflows/sycl-benchmark-aggregate.yml +++ b/.github/workflows/sycl-benchmark-aggregate.yml @@ -33,6 +33,8 @@ jobs: devops/scripts/benchmarking - name: Load benchmarking configuration run: | + # TODO isolate sanitation from benchmark.sh different script + # use sanitation here . llvm/devops/scripts/benchmarking/benchmark-ci.conf; echo "PERF_RES_GIT_REPO=$PERF_RES_GIT_REPO" >> $GITHUB_ENV echo "PERF_RES_BRANCH=$PERF_RES_BRANCH" >> $GITHUB_ENV @@ -58,8 +60,10 @@ jobs: git clone -b $PERF_RES_BRANCH $PERF_RES_GIT_REPO $PERF_RES_PATH - name: Run aggregator on cloned data run: | - for dir in $(find "$PERF_RES_PATH" -mindepth 1 -maxdepth 1 -type d ! -name '.*'); do - python llvm/devops/scripts/benchmarking/aggregate.py "$(basename $dir)" "$CUTOFF_TIMESTAMP" + for dir in $(find "$PERF_RES_PATH" -mindepth 2 -maxdepth 2 -type d ! -path '*.git*'); do + _runner="$(basename $(dirname $dir))" + _testcase="$(basename $dir)" + python llvm/devops/scripts/benchmarking/aggregate.py "$_runner" "$_testcase" "$CUTOFF_TIMESTAMP" done - name: Debug run: | diff --git a/.github/workflows/sycl-linux-benchmark.yml b/.github/workflows/sycl-linux-benchmark.yml index e9aed200c0569..8f0deaa057035 100644 --- a/.github/workflows/sycl-linux-benchmark.yml +++ b/.github/workflows/sycl-linux-benchmark.yml @@ -240,8 +240,9 @@ jobs: whereis sycl-ls clang++ --version ls + export ONEAPI_DEVICE_SELECTOR="${{ inputs.target_devices }}" export CMPLR_ROOT=$PWD/toolchain - ./devops/scripts/benchmarking/benchmark.sh ${{ inputs.cache_results == true && '-s' }} + ./devops/scripts/benchmarking/benchmark.sh -t '${{ inputs.runner }}' ${{ inputs.cache_results == true && '-s' }} - name: Push compute-benchmarks if: inputs.cache_results == true && success() env: diff --git a/devops/scripts/benchmarking/aggregate.py b/devops/scripts/benchmarking/aggregate.py index fc67af884be59..e6bcdd7783d10 100644 --- a/devops/scripts/benchmarking/aggregate.py +++ b/devops/scripts/benchmarking/aggregate.py @@ -33,11 +33,11 @@ def get_median(self) -> float: return -self.maxheap_smaller[0] -def aggregate_median(benchmark: str, cutoff: str): +def aggregate_median(runner: str, benchmark: str, cutoff: str): def csv_samples() -> list[str]: # TODO check that the path below is valid directory - with Path(f"{common.PERF_RES_PATH}/{benchmark}") as cache_dir: + with Path(f"{common.PERF_RES_PATH}/{runner}/{benchmark}") as cache_dir: # TODO check for time range; What time range do I want? return filter(lambda f: f.is_file() and common.valid_timestamp(str(f)[-19:-4]) and str(f)[-19:-4] > cutoff, @@ -54,7 +54,7 @@ def csv_samples() -> list[str]: for metric in common.metrics_variance: aggregate_s[s["TestCase"]][metric].add(common.sanitize(s[metric])) - with open(f"{common.PERF_RES_PATH}/{benchmark}/{benchmark}-median.csv", 'w') as output_csv: + with open(f"{common.PERF_RES_PATH}/{runner}/{benchmark}/{benchmark}-median.csv", 'w') as output_csv: writer = csv.DictWriter(output_csv, fieldnames=["TestCase", *common.metrics_variance.keys()]) writer.writeheader() @@ -65,11 +65,12 @@ def csv_samples() -> list[str]: if __name__ == "__main__": - if len(sys.argv) < 3: - print(f"Usage: {sys.argv[0]} ") + if len(sys.argv) < 4: + print(f"Usage: {sys.argv[0]} ") exit(1) - if not common.valid_timestamp(sys.argv[2]): - print(f"Bad cutoff timestamp, please use YYMMDD_HHMMSS.") + if not common.valid_timestamp(sys.argv[3]): + print(sys.argv) + print(f"Bad cutoff timestamp, please use YYYYMMDD_HHMMSS.") exit(1) common.load_configs() - aggregate_median(sys.argv[1], sys.argv[2]) + aggregate_median(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/devops/scripts/benchmarking/benchmark-ci.conf b/devops/scripts/benchmarking/benchmark-ci.conf index 76b82c58f5fa5..5fcc7fb3b9d45 100644 --- a/devops/scripts/benchmarking/benchmark-ci.conf +++ b/devops/scripts/benchmarking/benchmark-ci.conf @@ -13,10 +13,10 @@ COMPUTE_BENCH_PATH="./compute-benchmarks" # Compile flags used to build compute-benchmarks COMPUTE_BENCH_COMPILE_FLAGS="-j2" # Number of iterations to run tests for -COMPUTE_BENCH_ITERATIONS="1000" +COMPUTE_BENCH_ITERATIONS="100" # Path to temporarily store compute-benchmark results -OUTPUT_PATH="." +OUTPUT_PATH="./uncached_res" # Metrics to benchmark, and their allowed variance as a Python dictionary METRICS_VARIANCE='{"Median": 0.5}' @@ -38,3 +38,7 @@ TIMESTAMP_FORMAT='%Y%m%d_%H%M%S' BENCHMARK_SLOW_LOG="./benchmarks-over_tolerance.log" # Log file for test cases that errored / failed to build BENCHMARK_ERROR_LOG="./benchmarks-errored.log" + +# Runner types to parse from runner tag +RUNNER_TYPES="arc,pvc,amdgpu,cuda,gen12" +# These type names will be used to separate benchmark results based on hardware diff --git a/devops/scripts/benchmarking/benchmark.sh b/devops/scripts/benchmarking/benchmark.sh index 7b68375cfe9ff..aecfd43898615 100755 --- a/devops/scripts/benchmarking/benchmark.sh +++ b/devops/scripts/benchmarking/benchmark.sh @@ -5,14 +5,16 @@ # usage () { - >&2 echo "Usage: $0 [-B ] + >&2 echo "Usage: $0 -t [-B ] + -t Specify runner type -- Required -B Path to clone and build compute-benchmarks on -p Path to compute-benchmarks (or directory to build compute-benchmarks in) -r Git repo url to use for compute-benchmarks origin -b Git branch to use within compute-benchmarks -f Compile flags passed into building compute-benchmarks - -c _cleanup=1 ;; - -C _cleanup=1 && _exit_after_cleanup=1 ;; + -c Clean up working directory + -C Clean up working directory and exit + -s Cache results This script builds and runs benchmarks from compute-benchmarks." exit 1 @@ -44,11 +46,6 @@ build_compute_bench() { make $COMPUTE_BENCH_COMPILE_FLAGS "$case" done fi - echo "###" - ls $COMPUTE_BENCH_PATH/build/bin - echo "###" - ls $COMPUTE_BENCH_PATH/build/ - echo "###" # No reason to turn on ccache, if this docker image will be disassembled later on #compute_bench_build_stat=$? cd - @@ -94,16 +91,16 @@ samples_under_threshold () { } check_regression() { - if samples_under_threshold "$PERF_RES_PATH/$1"; then + if samples_under_threshold "$PERF_RES_PATH/$RUNNER/$1"; then echo "Not enough samples to construct an average, performance check skipped!" return $STATUS_SUCCESS fi - BENCHMARKING_ROOT="$BENCHMARKING_ROOT" python "$BENCHMARKING_ROOT/compare.py" "$1" "$2" + BENCHMARKING_ROOT="$BENCHMARKING_ROOT" python "$BENCHMARKING_ROOT/compare.py" "$RUNNER" "$1" "$2" return $? } cache() { - mv "$2" "$PERF_RES_PATH/$1/" + mv "$2" "$PERF_RES_PATH/$RUNNER/$1/" } # Check for a regression, and cache if no regression found @@ -133,7 +130,10 @@ process_benchmarks() { # Ignore lines in the test config starting with #'s grep "^[^#]" "$TESTS_CONFIG" | while read -r testcase; do echo "# Running $testcase..." - test_csv_output="$OUTPUT_PATH/$testcase-$TIMESTAMP.csv" + + test_csv_output="$OUTPUT_PATH/$RUNNER/$testcase-$TIMESTAMP.csv" + mkdir -p "$OUTPUT_PATH/$RUNNER/" + $COMPUTE_BENCH_PATH/build/bin/$testcase --csv --iterations="$COMPUTE_BENCH_ITERATIONS" | tail +8 > "$test_csv_output" # The tail +8 filters out initial debug prints not in csv format if [ "$?" -eq 0 ] && [ -s "$test_csv_output" ]; then @@ -172,7 +172,7 @@ cleanup() { } _sanitize_configs() { - echo "$1" | sed 's/[^a-zA-Z0-9_.:/%-]//g' + echo "$1" | sed 's/[^a-zA-Z0-9_.,:/%-]//g' } load_configs() { @@ -215,26 +215,28 @@ load_configs() { 'TIMESTAMP_FORMAT') export TIMESTAMP_FORMAT="$sanitized_value" ;; 'BENCHMARK_SLOW_LOG') export BENCHMARK_SLOW_LOG="$sanitized_value" ;; 'BENCHMARK_ERROR_LOG') export BENCHMARK_ERROR_LOG="$sanitized_value" ;; + 'RUNNER_TYPES') export RUNNER_TYPES="$sanitized_value" ;; # *) echo "Unknown key: $sanitized_key" ;; esac done < "$BENCHMARK_CI_CONFIG" # Debug - echo "PERF_RES_GIT_REPO: $PERF_RES_GIT_REPO" - echo "PERF_RES_BRANCH: $PERF_RES_BRANCH" - echo "PERF_RES_PATH: $PERF_RES_PATH" - echo "COMPUTE_BENCH_GIT_REPO: $COMPUTE_BENCH_GIT_REPO" - echo "COMPUTE_BENCH_BRANCH: $COMPUTE_BENCH_BRANCH" - echo "COMPUTE_BENCH_PATH: $COMPUTE_BENCH_PATH" - echo "COMPUTE_BENCH_COMPILE_FLAGS: $COMPUTE_BENCH_COMPILE_FLAGS" - echo "OUTPUT_PATH: $OUTPUT_PATH" + # echo "PERF_RES_GIT_REPO: $PERF_RES_GIT_REPO" + # echo "PERF_RES_BRANCH: $PERF_RES_BRANCH" + # echo "PERF_RES_PATH: $PERF_RES_PATH" + # echo "COMPUTE_BENCH_GIT_REPO: $COMPUTE_BENCH_GIT_REPO" + # echo "COMPUTE_BENCH_BRANCH: $COMPUTE_BENCH_BRANCH" + # echo "COMPUTE_BENCH_PATH: $COMPUTE_BENCH_PATH" + # echo "COMPUTE_BENCH_COMPILE_FLAGS: $COMPUTE_BENCH_COMPILE_FLAGS" + # echo "OUTPUT_PATH: $OUTPUT_PATH" # echo "METRICS_VARIANCE: $METRICS_VARIANCE" # echo "METRICS_RECORDED: $METRICS_RECORDED" - echo "AVERAGE_THRESHOLD: $AVERAGE_THRESHOLD" - echo "AVERAGE_CUTOFF_RANGE: $AVERAGE_CUTOFF_RANGE" - echo "TIMESTAMP_FORMAT: $TIMESTAMP_FORMAT" - echo "BENCHMARK_SLOW_LOG: $BENCHMARK_SLOW_LOG" - echo "BENCHMARK_ERROR_LOG: $BENCHMARK_ERROR_LOG" + # echo "AVERAGE_THRESHOLD: $AVERAGE_THRESHOLD" + # echo "AVERAGE_CUTOFF_RANGE: $AVERAGE_CUTOFF_RANGE" + # echo "TIMESTAMP_FORMAT: $TIMESTAMP_FORMAT" + # echo "BENCHMARK_SLOW_LOG: $BENCHMARK_SLOW_LOG" + # echo "BENCHMARK_ERROR_LOG: $BENCHMARK_ERROR_LOG" + echo "RUNNER_TYPES: $RUNNER_TYPES" } load_configs @@ -244,12 +246,13 @@ CACHE_RESULTS="0" TIMESTAMP="$(date +"$TIMESTAMP_FORMAT")" # CLI overrides to configuration options -while getopts "p:b:r:f:cCs" opt; do +while getopts "p:b:r:f:t:cCs" opt; do case $opt in p) COMPUTE_BENCH_PATH=$OPTARG ;; r) COMPUTE_BENCH_GIT_REPO=$OPTARG ;; b) COMPUTE_BENCH_BRANCH=$OPTARG ;; f) COMPUTE_BENCH_COMPILE_FLAGS=$OPTARG ;; + t) RUNNER_TYPE=$OPTARG ;; # Cleanup status is saved in a var to ensure all arguments are processed before # performing cleanup c) _cleanup=1 ;; @@ -263,10 +266,24 @@ if [ -z "$CMPLR_ROOT" ]; then echo "Please set \$CMPLR_ROOT first; it is needed by compute-benchmarks to build." exit 1 fi +if [ -z "$RUNNER_TYPE" ]; then + echo "Please specify runner type using -t first; it is needed for comparing benchmark results" + exit 1 +else + # Identify runner being used + runner_regex="$(printf "$RUNNER_TYPES" | sed 's/,/|/g')" + RUNNER="$(printf "$RUNNER_TYPE" | grep -o -E "\b($runner_regex)\b")" + if [ -z "$RUNNER" ]; then + echo "Unknown runner type! Configured runners: $RUNNER_TYPES" + exit 1 + fi + echo "Chosen runner: $RUNNER" +fi + [ ! -z "$_cleanup" ] && cleanup [ ! -d "$PERF_RES_PATH" ] && clone_perf_res [ ! -d "$COMPUTE_BENCH_PATH" ] && clone_compute_bench [ ! -d "$COMPUTE_BENCH_PATH/build" ] && build_compute_bench process_benchmarks -process_results \ No newline at end of file +process_results diff --git a/devops/scripts/benchmarking/compare.py b/devops/scripts/benchmarking/compare.py index c0565cfea1b65..f32c13243e30c 100644 --- a/devops/scripts/benchmarking/compare.py +++ b/devops/scripts/benchmarking/compare.py @@ -6,8 +6,8 @@ import common # TODO compare_to(metric) instead? -def compare_to_median(test_name: str, test_csv_path: str): - median_path = f"{common.PERF_RES_PATH}/{test_name}/{test_name}-median.csv" +def compare_to_median(runner: str, test_name: str, test_csv_path: str): + median_path = f"{common.PERF_RES_PATH}/{runner}/{test_name}/{test_name}-median.csv" if not os.path.isfile(test_csv_path): print("Invalid test file provided: " + test_csv_path) @@ -52,8 +52,8 @@ def compare_to_median(test_name: str, test_csv_path: str): if __name__ == "__main__": - if len(sys.argv) < 3: - print(f"Usage: {sys.argv[0]} ") + if len(sys.argv) < 4: + print(f"Usage: {sys.argv[0]} ") exit(-1) common.load_configs() - exit(compare_to_median(sys.argv[1], sys.argv[2])) + exit(compare_to_median(sys.argv[1], sys.argv[2], sys.argv[3]))