Skip to content

Commit

Permalink
Organize the turnkey benchmark help menu
Browse files Browse the repository at this point in the history 
Signed-off-by: Jeremy <[email protected]>
  • Loading branch information
@jeremyfowers
jeremyfowers committed Dec 15, 2023
1 parent e15f067 commit 1092896
Showing 1 changed file with 105 additions and 78 deletions.
183 changes: 105 additions & 78 deletions src/turnkeyml/cli/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,9 @@ def check_extension(choices, file_name):
return file_name

benchmark_parser = subparsers.add_parser(
"benchmark", help="Benchmark the performance of one or more models"
"benchmark",
help="Benchmark the performance of one or more models",
description="Analyze, build, and then benchmark the model(s) within input file(s).",
)
benchmark_parser.set_defaults(func=benchmark_command)

Expand All @@ -110,68 +112,59 @@ def check_extension(choices, file_name):
type=lambda file: check_extension(("py", "onnx", "txt"), file),
)

slurm_or_processes_group = benchmark_parser.add_mutually_exclusive_group()

slurm_or_processes_group.add_argument(
"--use-slurm",
dest="use_slurm",
help="Execute on Slurm instead of using local compute resources",
action="store_true",
toolchain_select_group = benchmark_parser.add_argument_group(
"Select which phase(s) of the toolchain to run (default is to run analyze, build, and benchmark)"
)

slurm_or_processes_group.add_argument(
"--process-isolation",
dest="process_isolation",
help="Isolate evaluating each input into a separate process",
toolchain_select_group.add_argument(
"-a",
"--analyze-only",
dest="analyze_only",
help="Stop this command after the analyze phase",
action="store_true",
)

benchmark_parser.add_argument(
"--lean-cache",
dest="lean_cache",
help="Delete all build artifacts except for log files when the command completes",
toolchain_select_group.add_argument(
"-b",
"--build-only",
dest="build_only",
help="Stop this command after the analyze and build phases",
action="store_true",
)

benchmark_parser.add_argument(
"-d",
"--cache-dir",
dest="cache_dir",
help="Build cache directory where the resulting build directories will "
f"be stored (defaults to {filesystem.DEFAULT_CACHE_DIR})",
required=False,
default=filesystem.DEFAULT_CACHE_DIR,
analyze_group = benchmark_parser.add_argument_group(
"Options that specifically apply to the `analyze` phase of the toolflow"
)

benchmark_parser.add_argument(
analyze_group.add_argument(
"--labels",
dest="labels",
help="Only benchmark the scripts that have the provided labels",
nargs="*",
default=[],
)

benchmark_parser.add_argument(
"--sequence",
choices=SUPPORTED_SEQUENCES.keys(),
dest="sequence",
help="Name of a build sequence that will define the model-to-model transformations, "
"used to build the models. Each runtime has a default sequence that it uses.",
required=False,
default=None,
analyze_group.add_argument(
"--script-args",
dest="script_args",
type=str,
help="Arguments to pass into the target script(s)",
)

benchmark_parser.add_argument(
"--rebuild",
choices=build.REBUILD_OPTIONS,
dest="rebuild",
help=f"Sets the cache rebuild policy (defaults to {build.DEFAULT_REBUILD_POLICY})",
required=False,
default=build.DEFAULT_REBUILD_POLICY,
analyze_group.add_argument(
"--max-depth",
dest="max_depth",
type=int,
default=0,
help="Maximum depth to analyze within the model structure of the target script(s)",
)

both_build_benchmark_group = benchmark_parser.add_argument_group(
"Options that apply to both the `build` and `benchmark` phases of the toolflow"
)

benchmark_default_device = "x86"
benchmark_parser.add_argument(
both_build_benchmark_group.add_argument(
"--device",
choices=SUPPORTED_DEVICES,
dest="device",
Expand All @@ -181,7 +174,7 @@ def check_extension(choices, file_name):
default=benchmark_default_device,
)

benchmark_parser.add_argument(
both_build_benchmark_group.add_argument(
"--runtime",
choices=SUPPORTED_RUNTIMES.keys(),
dest="runtime",
Expand All @@ -193,46 +186,47 @@ def check_extension(choices, file_name):
default=None,
)

benchmark_parser.add_argument(
"--iterations",
dest="iterations",
type=int,
default=100,
help="Number of execution iterations of the model to capture\
the benchmarking performance (e.g., mean latency)",
both_build_benchmark_group.add_argument(
"-d",
"--cache-dir",
dest="cache_dir",
help="Build cache directory where the resulting build directories will "
f"be stored (defaults to {filesystem.DEFAULT_CACHE_DIR})",
required=False,
default=filesystem.DEFAULT_CACHE_DIR,
)

benchmark_parser.add_argument(
"--analyze-only",
dest="analyze_only",
help="Stop this command after the analysis phase",
both_build_benchmark_group.add_argument(
"--lean-cache",
dest="lean_cache",
help="Delete all build artifacts except for log files when the command completes",
action="store_true",
)

benchmark_parser.add_argument(
"-b",
"--build-only",
dest="build_only",
help="Stop this command after the build phase",
action="store_true",
build_group = benchmark_parser.add_argument_group(
"Options that apply specifically to the `build` phase of the toolflow"
)

benchmark_parser.add_argument(
"--script-args",
dest="script_args",
type=str,
help="Arguments to pass into the target script(s)",
build_group.add_argument(
"--sequence",
choices=SUPPORTED_SEQUENCES.keys(),
dest="sequence",
help="Name of a build sequence that will define the model-to-model transformations, "
"used to build the models. Each runtime has a default sequence that it uses.",
required=False,
default=None,
)

benchmark_parser.add_argument(
"--max-depth",
dest="max_depth",
type=int,
default=0,
help="Maximum depth to analyze within the model structure of the target script(s)",
build_group.add_argument(
"--rebuild",
choices=build.REBUILD_OPTIONS,
dest="rebuild",
help=f"Sets the cache rebuild policy (defaults to {build.DEFAULT_REBUILD_POLICY})",
required=False,
default=build.DEFAULT_REBUILD_POLICY,
)

benchmark_parser.add_argument(
build_group.add_argument(
"--onnx-opset",
dest="onnx_opset",
type=int,
Expand All @@ -241,23 +235,56 @@ def check_extension(choices, file_name):
"Not applicable when input model is already a .onnx file.",
)

benchmark_parser.add_argument(
"--timeout",
benchmark_group = benchmark_parser.add_argument_group(
"Options that apply specifically to the `benchmark` phase of the toolflow"
)

benchmark_group.add_argument(
"--iterations",
dest="iterations",
type=int,
default=None,
help="Build timeout, in seconds, after which a build will be canceled "
f"(default={DEFAULT_TIMEOUT_SECONDS}). Only "
"applies when --process-isolation or --use-slurm is also used.",
default=100,
help="Number of execution iterations of the model to capture\
the benchmarking performance (e.g., mean latency)",
)

benchmark_parser.add_argument(
benchmark_group.add_argument(
"--rt-args",
dest="rt_args",
type=str,
nargs="*",
help="Optional arguments provided to the runtime being used",
)

all_toolflows_group = benchmark_parser.add_argument_group(
"Options that apply to all toolflows"
)

slurm_or_processes_group = all_toolflows_group.add_mutually_exclusive_group()

slurm_or_processes_group.add_argument(
"--use-slurm",
dest="use_slurm",
help="Execute on Slurm instead of using local compute resources",
action="store_true",
)

slurm_or_processes_group.add_argument(
"--process-isolation",
dest="process_isolation",
help="Isolate evaluating each input into a separate process",
action="store_true",
)

all_toolflows_group.add_argument(
"--timeout",
type=int,
default=None,
help="Build timeout, in seconds, after which a build will be canceled "
f"(default={DEFAULT_TIMEOUT_SECONDS}). Only "
"applies when --process-isolation or --use-slurm is also used.",
)

#######################################
# Subparser for the "cache" command
#######################################
Expand Down

0 comments on commit 1092896

Please sign in to comment.