From 94f28b29c535aff69d009d2babd3422d815f39d7 Mon Sep 17 00:00:00 2001 From: Christopher Sidebottom Date: Tue, 19 Apr 2022 11:50:12 +0100 Subject: [PATCH] [TVMC] Add `--config` argument for config files (#11012) * [TVMC] Add `--config` argument for config files Collecting common configurations for users of TVM and exposing them gracefully in tvmc using a `--config` option as defined in https://github.com/apache/tvm-rfcs/blob/main/rfcs/0030-tvmc-comand-line-configuration-files.md Co-authored-by: Shai Maor * Add correct test guards Co-authored-by: Shai Maor --- configs/host/default.json | 7 + configs/test/compile_config_test.json | 9 + configs/test/tune_config_test.json | 6 + python/tvm/driver/tvmc/autotuner.py | 5 +- python/tvm/driver/tvmc/compiler.py | 5 +- python/tvm/driver/tvmc/config_options.py | 159 ++++++++++++++++++ python/tvm/driver/tvmc/main.py | 14 +- python/tvm/driver/tvmc/micro.py | 5 +- python/tvm/driver/tvmc/runner.py | 5 +- python/tvm/driver/tvmc/target.py | 2 +- tests/python/driver/tvmc/test_command_line.py | 36 ++++ .../driver/tvmc/test_parse_config_file.py | 155 +++++++++++++++++ 12 files changed, 401 insertions(+), 7 deletions(-) create mode 100644 configs/host/default.json create mode 100644 configs/test/compile_config_test.json create mode 100644 configs/test/tune_config_test.json create mode 100644 python/tvm/driver/tvmc/config_options.py create mode 100644 tests/python/driver/tvmc/test_parse_config_file.py diff --git a/configs/host/default.json b/configs/host/default.json new file mode 100644 index 0000000000..2c29445501 --- /dev/null +++ b/configs/host/default.json @@ -0,0 +1,7 @@ +{ + "targets": [ + { + "kind": "llvm" + } + ] +} diff --git a/configs/test/compile_config_test.json b/configs/test/compile_config_test.json new file mode 100644 index 0000000000..dcc6dbd27e --- /dev/null +++ b/configs/test/compile_config_test.json @@ -0,0 +1,9 @@ +{ + "targets": [ + {"kind": "cmsis-nn", "from_device": "1"}, + {"kind": "c", "mcpu": "cortex-m55"} + ], + "executor": { "kind": "aot"}, + "runtime": { "kind": "crt"}, + "pass-config": { "tir.disable_vectorize": "1"} +} diff --git a/configs/test/tune_config_test.json b/configs/test/tune_config_test.json new file mode 100644 index 0000000000..69babc753e --- /dev/null +++ b/configs/test/tune_config_test.json @@ -0,0 +1,6 @@ +{ + "targets": [ + { "kind": "llvm" } + ], + "trials": "2" +} diff --git a/python/tvm/driver/tvmc/autotuner.py b/python/tvm/driver/tvmc/autotuner.py index c6c0fda343..97cd3bfbc1 100644 --- a/python/tvm/driver/tvmc/autotuner.py +++ b/python/tvm/driver/tvmc/autotuner.py @@ -47,7 +47,7 @@ @register_parser -def add_tune_parser(subparsers, _): +def add_tune_parser(subparsers, _, json_params): """Include parser for 'tune' subcommand""" parser = subparsers.add_parser("tune", help="auto-tune a model") @@ -224,6 +224,9 @@ def add_tune_parser(subparsers, _): type=parse_shape_string, ) + for one_entry in json_params: + parser.set_defaults(**one_entry) + def drive_tune(args): """Invoke auto-tuning with command line arguments diff --git a/python/tvm/driver/tvmc/compiler.py b/python/tvm/driver/tvmc/compiler.py index b29aede958..a192b93d8c 100644 --- a/python/tvm/driver/tvmc/compiler.py +++ b/python/tvm/driver/tvmc/compiler.py @@ -43,7 +43,7 @@ @register_parser -def add_compile_parser(subparsers, _): +def add_compile_parser(subparsers, _, json_params): """Include parser for 'compile' subcommand""" parser = subparsers.add_parser("compile", help="compile a model.") @@ -143,6 +143,9 @@ def add_compile_parser(subparsers, _): help="The output module name. Defaults to 'default'.", ) + for one_entry in json_params: + parser.set_defaults(**one_entry) + def drive_compile(args): """Invoke tvmc.compiler module with command line arguments diff --git a/python/tvm/driver/tvmc/config_options.py b/python/tvm/driver/tvmc/config_options.py new file mode 100644 index 0000000000..ec5c6f5194 --- /dev/null +++ b/python/tvm/driver/tvmc/config_options.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +manipulate json config file to work with TVMC +""" +import os +import json +from tvm.driver.tvmc import TVMCException + + +def find_json_file(name, path): + """search for json file given file name a path + + Parameters + ---------- + name: string + the file name need to be searched + path: string + path to search at + + Returns + ------- + string + the full path to that file + + """ + match = "" + for root, _dirs, files in os.walk(path): + if name in files: + match = os.path.join(root, name) + break + + return match + + +def read_and_convert_json_into_dict(config_args): + """Read json configuration file and return a dictionary with all parameters + + Parameters + ---------- + args: argparse.Namespace + Arguments from command line parser holding the json file path. + + Returns + ------- + dictionary + dictionary with all the json arguments keys and values + + """ + try: + if ".json" not in config_args.config: + config_args.config = config_args.config.strip() + ".json" + if os.path.isfile(config_args.config): + json_config_file = config_args.config + else: + config_dir = os.path.abspath( + os.path.join(os.path.realpath(__file__), "..", "..", "..", "..", "..", "configs") + ) + json_config_file = find_json_file(config_args.config, config_dir) + return json.load(open(json_config_file, "rb")) + + except FileNotFoundError: + raise TVMCException( + f"File {config_args.config} does not exist at {config_dir} or is wrong format." + ) + + +def parse_target_from_json(one_target, command_line_list): + """parse the targets out of the json file struct + + Parameters + ---------- + one_target: dict + dictionary with all target's details + command_line_list: list + list to update with target parameters + """ + target_kind, *sub_type = [ + one_target[key] if key == "kind" else (key, one_target[key]) for key in one_target + ] + + internal_dict = {} + if sub_type: + sub_target_type = sub_type[0][0] + target_value = sub_type[0][1] + internal_dict[f"target_{target_kind}_{sub_target_type}"] = target_value + command_line_list.append(internal_dict) + + return target_kind + + +def convert_config_json_to_cli(json_params): + """convert all configuration keys & values from dictionary to cli format + + Parameters + ---------- + args: dictionary + dictionary with all configuration keys & values. + + Returns + ------- + int + list of configuration values in cli format + + """ + command_line_list = [] + for param_key in json_params: + if param_key == "targets": + target_list = [ + parse_target_from_json(one_target, command_line_list) + for one_target in json_params[param_key] + ] + + internal_dict = {} + internal_dict["target"] = ", ".join(map(str, target_list)) + command_line_list.append(internal_dict) + + elif param_key in ("executor", "runtime"): + for key, value in json_params[param_key].items(): + if key == "kind": + kind = f"{value}_" + new_dict_key = param_key + else: + new_dict_key = f"{param_key}_{kind}{key}" + + internal_dict = {} + internal_dict[new_dict_key.replace("-", "_")] = value + command_line_list.append(internal_dict) + + elif isinstance(json_params[param_key], dict): + internal_dict = {} + modify_param_key = param_key.replace("-", "_") + internal_dict[modify_param_key] = [] + for key, value in json_params[param_key].items(): + internal_dict[modify_param_key].append(f"{key}={value}") + command_line_list.append(internal_dict) + + else: + internal_dict = {} + internal_dict[param_key.replace("-", "_")] = json_params[param_key] + command_line_list.append(internal_dict) + + return command_line_list diff --git a/python/tvm/driver/tvmc/main.py b/python/tvm/driver/tvmc/main.py index b74cc7d6ee..22a5053aee 100644 --- a/python/tvm/driver/tvmc/main.py +++ b/python/tvm/driver/tvmc/main.py @@ -26,7 +26,10 @@ import tvm from tvm.driver.tvmc import TVMCException, TVMCImportError - +from tvm.driver.tvmc.config_options import ( + read_and_convert_json_into_dict, + convert_config_json_to_cli, +) REGISTERED_PARSER = [] @@ -64,12 +67,19 @@ def _main(argv): # so it doesn't interfere with the creation of the dynamic subparsers. add_help=False, ) + + parser.add_argument("--config", default="default", help="configuration json file") + config_arg, argv = parser.parse_known_args(argv) + + json_param_dict = read_and_convert_json_into_dict(config_arg) + json_config_values = convert_config_json_to_cli(json_param_dict) + parser.add_argument("-v", "--verbose", action="count", default=0, help="increase verbosity") parser.add_argument("--version", action="store_true", help="print the version and exit") subparser = parser.add_subparsers(title="commands") for make_subparser in REGISTERED_PARSER: - make_subparser(subparser, parser) + make_subparser(subparser, parser, json_config_values) # Finally, add help for the main parser. parser.add_argument("-h", "--help", action="help", help="show this help message and exit.") diff --git a/python/tvm/driver/tvmc/micro.py b/python/tvm/driver/tvmc/micro.py index 4f478c7c3a..fdaffac07d 100644 --- a/python/tvm/driver/tvmc/micro.py +++ b/python/tvm/driver/tvmc/micro.py @@ -45,7 +45,7 @@ @register_parser -def add_micro_parser(subparsers, main_parser): +def add_micro_parser(subparsers, main_parser, json_params): """Includes parser for 'micro' context and associated subcommands: create-project (create), build, and flash. """ @@ -231,6 +231,9 @@ def _add_parser(parser, platform): help="show this help message which includes platform-specific options and exit.", ) + for one_entry in json_params: + micro.set_defaults(**one_entry) + def drive_micro(args): # Call proper handler based on subcommand parsed. diff --git a/python/tvm/driver/tvmc/runner.py b/python/tvm/driver/tvmc/runner.py index 1b6d823712..5be588a3ae 100644 --- a/python/tvm/driver/tvmc/runner.py +++ b/python/tvm/driver/tvmc/runner.py @@ -60,7 +60,7 @@ @register_parser -def add_run_parser(subparsers, main_parser): +def add_run_parser(subparsers, main_parser, json_params): """Include parser for 'run' subcommand""" # Use conflict_handler='resolve' to allow '--list-options' option to be properly overriden when @@ -191,6 +191,9 @@ def add_run_parser(subparsers, main_parser): help="show this help message with platform-specific options and exit.", ) + for one_entry in json_params: + parser.set_defaults(**one_entry) + def drive_run(args): """Invoke runner module with command line arguments diff --git a/python/tvm/driver/tvmc/target.py b/python/tvm/driver/tvmc/target.py index 7e1073d9a7..a3602b4eb8 100644 --- a/python/tvm/driver/tvmc/target.py +++ b/python/tvm/driver/tvmc/target.py @@ -81,7 +81,7 @@ def generate_target_args(parser): parser.add_argument( "--target", help="compilation target as plain string, inline JSON or path to a JSON file", - required=True, + required=False, ) for target_kind in _valid_target_kinds(): _generate_target_kind_args(parser, target_kind) diff --git a/tests/python/driver/tvmc/test_command_line.py b/tests/python/driver/tvmc/test_command_line.py index 2e7f8d87c0..bbf608a5f2 100644 --- a/tests/python/driver/tvmc/test_command_line.py +++ b/tests/python/driver/tvmc/test_command_line.py @@ -56,3 +56,39 @@ def test_tvmc_cl_workflow(keras_simple, tmpdir_factory): run_args = run_str.split(" ")[1:] _main(run_args) assert os.path.exists(output_path) + + +@pytest.mark.skipif( + platform.machine() == "aarch64", + reason="Currently failing on AArch64 - see https://github.com/apache/tvm/issues/10673", +) +def test_tvmc_cl_workflow_json_config(keras_simple, tmpdir_factory): + pytest.importorskip("tensorflow") + tune_config_file = "tune_config_test" + tmpdir = tmpdir_factory.mktemp("data") + + # Test model tuning + log_path = os.path.join(tmpdir, "keras-autotuner_records.json") + tuning_str = ( + f"tvmc tune --config {tune_config_file} --output {log_path} " + f"--enable-autoscheduler {keras_simple}" + ) + tuning_args = tuning_str.split(" ")[1:] + _main(tuning_args) + assert os.path.exists(log_path) + + # Test model compilation + package_path = os.path.join(tmpdir, "keras-tvm.tar") + compile_str = ( + f"tvmc compile --tuning-records {log_path} " f"--output {package_path} {keras_simple}" + ) + compile_args = compile_str.split(" ")[1:] + _main(compile_args) + assert os.path.exists(package_path) + + # Test running the model + output_path = os.path.join(tmpdir, "predictions.npz") + run_str = f"tvmc run --outputs {output_path} {package_path}" + run_args = run_str.split(" ")[1:] + _main(run_args) + assert os.path.exists(output_path) diff --git a/tests/python/driver/tvmc/test_parse_config_file.py b/tests/python/driver/tvmc/test_parse_config_file.py new file mode 100644 index 0000000000..a80daba3a4 --- /dev/null +++ b/tests/python/driver/tvmc/test_parse_config_file.py @@ -0,0 +1,155 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import pytest +import os +import shlex + +import tvm +from tvm.driver.tvmc.main import _main +from tvm.driver.tvmc.config_options import convert_config_json_to_cli + + +def test_parse_json_config_file_one_target(): + tokens = convert_config_json_to_cli( + {"targets": [{"kind": "llvm"}], "output": "resnet50-v2-7-autotuner_records.json"} + ) + expected_tokens = [{"target": "llvm"}, {"output": "resnet50-v2-7-autotuner_records.json"}] + + assert len(tokens) == len(expected_tokens) + assert tokens == expected_tokens + + +def test_parse_json_config_file_multipile_targets(): + tokens = convert_config_json_to_cli( + { + "targets": [{"kind": "llvm"}, {"kind": "c", "mcpu": "cortex-m55"}], + "tuning-records": "resnet50-v2-7-autotuner_records.json", + "pass-config": {"tir.disable_vectorizer": "1"}, + } + ) + expected_tokens = [ + {"target_c_mcpu": "cortex-m55"}, + {"target": "llvm, c"}, + {"tuning_records": "resnet50-v2-7-autotuner_records.json"}, + {"pass_config": ["tir.disable_vectorizer=1"]}, + ] + + assert len(tokens) == len(expected_tokens) + assert tokens == expected_tokens + + +def test_parse_json_config_file_executor(): + tokens = convert_config_json_to_cli( + { + "executor": {"kind": "aot", "interface-api": "c"}, + "inputs": "imagenet_cat.npz", + "max-local-memory-per-block": "4", + "repeat": "100", + } + ) + expected_tokens = [ + {"executor": "aot"}, + {"executor_aot_interface_api": "c"}, + {"inputs": "imagenet_cat.npz"}, + {"max_local_memory_per_block": "4"}, + {"repeat": "100"}, + ] + + assert len(tokens) == len(expected_tokens) + assert tokens == expected_tokens + + +def test_parse_json_config_file_target_and_executor(): + tokens = convert_config_json_to_cli( + { + "targets": [ + {"kind": "ethos-u -accelerator_config=ethos-u55-256"}, + {"kind": "c", "mcpu": "cortex-m55"}, + {"kind": "cmsis-nn"}, + ], + "executor": {"kind": "aot", "interface-api": "c", "unpacked-api": "1"}, + "inputs": "imagenet_cat.npz", + "max-local-memory-per-block": "4", + "repeat": "100", + } + ) + expected_tokens = [ + {"target_c_mcpu": "cortex-m55"}, + {"target": "ethos-u -accelerator_config=ethos-u55-256, c, cmsis-nn"}, + {"executor": "aot"}, + {"executor_aot_interface_api": "c"}, + {"executor_aot_unpacked_api": "1"}, + {"inputs": "imagenet_cat.npz"}, + {"max_local_memory_per_block": "4"}, + {"repeat": "100"}, + ] + + assert len(tokens) == len(expected_tokens) + assert tokens == expected_tokens + + +def test_parse_json_config_file_runtime(): + tokens = convert_config_json_to_cli( + { + "targets": [ + {"kind": "cmsis-nn", "from_device": "1"}, + {"kind": "c", "mcpu": "cortex-m55"}, + ], + "runtime": {"kind": "crt"}, + "inputs": "imagenet_cat.npz", + "output": "predictions.npz", + "pass-config": {"tir.disable_vectorize": "1", "relay.backend.use_auto_scheduler": "0"}, + } + ) + expected_tokens = [ + {"target_cmsis-nn_from_device": "1"}, + {"target_c_mcpu": "cortex-m55"}, + {"target": "cmsis-nn, c"}, + {"runtime": "crt"}, + {"inputs": "imagenet_cat.npz"}, + {"output": "predictions.npz"}, + {"pass_config": ["tir.disable_vectorize=1", "relay.backend.use_auto_scheduler=0"]}, + ] + + assert len(tokens) == len(expected_tokens) + assert tokens == expected_tokens + + +@tvm.testing.requires_cmsisnn +def test_tvmc_cl_compile_run_config_file(tflite_mobilenet_v1_1_quant, tmpdir_factory): + compile_config_file = "compile_config_test.json" + pytest.importorskip("tflite") + + output_dir = tmpdir_factory.mktemp("mlf") + input_model = tflite_mobilenet_v1_1_quant + output_file = os.path.join(output_dir, "mock.tar") + + # Compile the input model and generate a Model Library Format (MLF) archive. + tvmc_cmd = ( + f"tvmc compile --config {compile_config_file} {input_model} --output {output_file} " + f"--output-format mlf" + ) + tvmc_args = shlex.split(tvmc_cmd)[1:] + _main(tvmc_args) + assert os.path.exists(output_file), "Could not find the exported MLF archive." + + # Run the MLF archive. It must fail since it's only supported on micro targets. + tvmc_cmd = f"tvmc run {output_file}" + tvmc_args = tvmc_cmd.split(" ")[1:] + exit_code = _main(tvmc_args) + on_error = "Trying to run a MLF archive must fail because it's only supported on micro targets." + assert exit_code != 0, on_error