From 5b1bd3c0990c4f46d628d8d66faf165fe4eafef1 Mon Sep 17 00:00:00 2001 From: Dongge Liu Date: Fri, 23 Feb 2024 15:33:36 +1100 Subject: [PATCH] Support specifying models in GKE experiment runs (#122) Support specifying models in GKE experiment runs 1. Add `model` argument to `docker_run.sh`, `upload_report.sh`, and `web.py`. 2. Display the model name in experiment reports 3. Some TODOs to rewrite code to adapt to increasing complexity. Preview: image We can beautify the layout later. --- report/docker_run.sh | 13 +++++++++++-- report/templates/base.html | 1 + report/upload_report.sh | 16 +++++++++++++--- report/web.py | 13 ++++++++++--- 4 files changed, 35 insertions(+), 8 deletions(-) diff --git a/report/docker_run.sh b/report/docker_run.sh index e4025b389e..3bd0b3d138 100755 --- a/report/docker_run.sh +++ b/report/docker_run.sh @@ -26,10 +26,13 @@ ## run_timeout: Timeout in seconds for each fuzzing target. ## Default: 300 +# TODO(dongge): Re-write this script in Python as it gets more complex. + BENCHMARK_SET=$1 FREQUENCY_LABEL=$2 RUN_TIMEOUT=$3 SUB_DIR=$4 +MODEL=$5 # Uses python3 by default and /venv/bin/python3 for Docker containers. PYTHON="$( [[ -x "/venv/bin/python3" ]] && echo "/venv/bin/python3" || echo "python3" )" export PYTHON @@ -67,6 +70,12 @@ then echo "Sub-directory was not specified as the fourth argument. Defaulting to ${SUB_DIR:?}. Please consider using sub-directory to classify your experiment." fi +# The LLM used to generate and fix fuzz targets. +if [[ $MODEL = '' ]] +then + MODEL='vertex_ai_code-bison-32k' + echo "LLM was not specified as the fifth argument. Defaulting to ${MODEL:?}." +fi DATE=$(date '+%Y-%m-%d') LOCAL_RESULTS_DIR='results' # Experiment name is used to label the Cloud Builds and as part of the @@ -79,7 +88,7 @@ EXPERIMENT_NAME="${DATE:?}-${FREQUENCY_LABEL:?}-${BENCHMARK_SET:?}" GCS_REPORT_DIR="${SUB_DIR:?}/${EXPERIMENT_NAME:?}" # Generate a report and upload it to GCS -bash report/upload_report.sh "${LOCAL_RESULTS_DIR:?}" "${GCS_REPORT_DIR:?}" "${BENCHMARK_SET:?}" & +bash report/upload_report.sh "${LOCAL_RESULTS_DIR:?}" "${GCS_REPORT_DIR:?}" "${BENCHMARK_SET:?}" "${MODEL:?}" & pid_report=$! # Run the experiment @@ -91,7 +100,7 @@ $PYTHON run_all_experiments.py \ --template-directory 'prompts/template_xml' \ --work-dir ${LOCAL_RESULTS_DIR:?} \ --num-samples 10 \ - --mode 'vertex_ai_code-bison-32k' + --model "$MODEL" export ret_val=$? diff --git a/report/templates/base.html b/report/templates/base.html index 2a46de2709..0e409fb796 100644 --- a/report/templates/base.html +++ b/report/templates/base.html @@ -22,5 +22,6 @@ } +LLM: {{ model }} {% block content %}{% endblock %} diff --git a/report/upload_report.sh b/report/upload_report.sh index e06a2e3d08..ec952d1a20 100644 --- a/report/upload_report.sh +++ b/report/upload_report.sh @@ -21,9 +21,12 @@ ## gcs_dir is the name of the directory for the report in gs://oss-fuzz-gcb-experiment-run-logs/Result-reports/. ## Defaults to '$(whoami)-%YY-%MM-%DD'. +# TODO(dongge): Re-write this script in Python as it gets more complex. + RESULTS_DIR=$1 GCS_DIR=$2 BENCHMARK_SET=$3 +MODEL=$4 WEB_PORT=8080 DATE=$(date '+%Y-%m-%d') @@ -38,15 +41,22 @@ fi if [[ $GCS_DIR = '' ]] then - GCS_DIR="$(whoami)-${DATE:?}" - echo "GCS directory was not specified as the second argument. Defaulting to ${GCS_DIR:?}." + echo "This script needs to take gcloud Bucket directory as the second argument. Consider using $(whoami)-${DATE:?}." + exit 1 +fi + +# The LLM used to generate and fix fuzz targets. +if [[ $MODEL = '' ]] +then + echo "This script needs to take LLM as the third argument." + exit 1 fi mkdir results-report while true; do # Spin up the web server generating the report (and bg the process). - $PYTHON -m report.web "${RESULTS_DIR:?}" "${WEB_PORT:?}" "${BENCHMARK_SET:?}" & + $PYTHON -m report.web "${RESULTS_DIR:?}" "${WEB_PORT:?}" "${BENCHMARK_SET:?}" "$MODEL" & pid_web=$! cd results-report || exit 1 diff --git a/report/web.py b/report/web.py index eca7f5fcd2..0dc31cf18a 100644 --- a/report/web.py +++ b/report/web.py @@ -266,18 +266,23 @@ def get_targets(benchmark: str, sample: str) -> list[Target]: @app.route('/') def index(): - return render_template('index.html', benchmarks=list_benchmarks()) + return render_template('index.html', + benchmarks=list_benchmarks(), + model=model) @app.route('/json') def index_json(): - return render_template('index.json', benchmarks=list_benchmarks()) + return render_template('index.json', + benchmarks=list_benchmarks(), + model=model) @app.route('/sort') def index_sort(): return render_template('index.html', - benchmarks=sort_benchmarks(list_benchmarks())) + benchmarks=sort_benchmarks(list_benchmarks()), + model=model) @app.route('/benchmark/') @@ -334,8 +339,10 @@ def serve(directory: str, port: int, benchmark_set: str): if __name__ == '__main__': + # TODO(Dongge): Use argparser as this script gets more complex. results_dir = sys.argv[1] server_port = int(sys.argv[2]) benchmark_dir = sys.argv[3] if len(sys.argv) > 3 else '' + model = sys.argv[4] if len(sys.argv) > 4 else '' serve(results_dir, server_port, benchmark_dir)