-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: Save CLI benchmark results to files
- Loading branch information
Showing
5 changed files
with
221 additions
and
24 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # This iterates over all combinations of synthetic data+model params and | ||
| # benchmarks each one. | ||
| # NB: This can take a while, a few days at worst | ||
|
|
||
| # Check if at least one argument is provided | ||
| if [ $# -lt 1 ]; then | ||
| echo "Usage: $0 <jar-file> [extra-params...]" | ||
| exit 1 | ||
| fi | ||
|
|
||
| # Extract the JAR file name from the first argument | ||
| jar_file="$1" | ||
| shift # Remove the first argument so that "$@" contains only the extra params | ||
|
|
||
| # Function to handle SIGINT (Ctrl+C) | ||
| cleanup() { | ||
| set +x | ||
| echo "Script interrupted. Exiting..." | ||
| exit 1 | ||
| } | ||
|
|
||
| # Trap SIGINT signal and call cleanup function | ||
| trap cleanup SIGINT | ||
|
|
||
| # Enable command echoing | ||
| set -x | ||
|
|
||
| # Params to iterate over | ||
| num_rows_array=(10 50 200 500 1000 10000) | ||
| num_columns_array=(5 10 20 50 100 500) | ||
| num_views_array=(5 10 20 50 100 200) | ||
| num_clusters_per_view_array=(5 10 20 50 100 200) | ||
|
|
||
| for num_rows in "${num_rows_array[@]}"; do | ||
| for num_columns in "${num_columns_array[@]}"; do | ||
| for num_views in "${num_views_array[@]}"; do | ||
| for num_clusters_per_view in "${num_clusters_per_view_array[@]}"; do | ||
| # Benchmark, but use "|| true" to continue on failure | ||
| java -jar "$jar_file" \ | ||
| --synthetic \ | ||
| --num-rows "$num_rows" \ | ||
| --num-columns "$num_columns" \ | ||
| --num-views "$num_views" \ | ||
| --num-clusters-per-view "$num_clusters_per_view" \ | ||
| --no-overwrite \ | ||
| "$@" || true | ||
| done | ||
| done | ||
| done | ||
| done | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| (ns gensql.query.perf.util) | ||
|
|
||
| ;; time+ adapted from the version at https://clojure-goes-fast.com/kb/benchmarking/time-plus/ | ||
| (let [time* | ||
| (fn time* [^long duration-in-ms f] | ||
| (let [^com.sun.management.ThreadMXBean bean (java.lang.management.ManagementFactory/getThreadMXBean) | ||
| bytes-before (.getCurrentThreadAllocatedBytes bean) | ||
| duration (* duration-in-ms 1000000) | ||
| start (System/nanoTime) | ||
| first-res (f) | ||
| delta (- (System/nanoTime) start) | ||
| deadline (+ start duration) | ||
| tight-iters (max (quot (quot duration delta) 10) 1)] | ||
| (loop [i 1] | ||
| (let [now (System/nanoTime)] | ||
| (if (< now deadline) | ||
| (do (dotimes [_ tight-iters] (f)) | ||
| (recur (+ i tight-iters))) | ||
| (let [i' (double i) | ||
| bytes-after (.getCurrentThreadAllocatedBytes bean) | ||
| total-run-time (- now start) | ||
| t (/ total-run-time i')] | ||
| {:first-result first-res | ||
| :time-per-call (/ t 1e9) | ||
| :total-time (/ total-run-time 1e9) | ||
| :bytes-alloc-per-call (/ (- bytes-after bytes-before) i') | ||
| :num-total-iterations i}))))))] | ||
|
|
||
| (defmacro time+ | ||
| "Like `time`, but runs the supplied body for the duration in ms and returns: | ||
| - the mean time in s | ||
| - the mean bytes allocated | ||
| - the total number of iterations run | ||
| - the result of the first call, if any | ||
| Total time in milliseconds must be provided as the first argument." | ||
| [duration-in-ms & body] | ||
| `(~time* ~duration-in-ms (fn [] ~@body)))) |