test.py

import argparse
import os
import os.path as osp

from mmengine.config import Config, DictAction
from mmengine.runner import Runner


def parse_args():
    parser = argparse.ArgumentParser(
        description='STL test (and eval) a model')
    parser.add_argument(
        '--config', 
        help='test config file path, this is not required in the test-best or test-last mode')
    parser.add_argument(
        '--checkpoint', 
        help='checkpoint file, this is not required in the test-best or test-last mode')
    parser.add_argument(
        '--work-dir',
        help='the directory to save the file containing evaluation metrics')
    parser.add_argument(
        '--test-best', action='store_true', default=False,
        help='test the best ckpt saved in the work-dir, cannot meanwhile be True with test-last')
    parser.add_argument(
        '--test-last', action='store_true', default=False,
        help='test the last ckpt saved in the work-dir, cannot meanwhile be True with test-best')
    parser.add_argument(
        '--metric-only', action='store_true', default=False,
        help='only calculate the metrics for the model without calculating complexity info'
    )
    parser.add_argument(
        '--dump',
        type=str,
        help='dump predictions to a pickle file for offline evaluation')
    parser.add_argument(
        '--cfg-options',
        nargs='+',
        action=DictAction,
        help='override some settings in the used config, the key-value pair '
        'in xxx=yyy format will be merged into config file. If the value to '
        'be overwritten is a list, it should be like key="[a,b]" or key=a,b '
        'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" '
        'Note that the quotation marks are necessary and that no white space '
        'is allowed.')
    parser.add_argument(
        '--show-dir',
        help='directory where the visualization images will be saved.')
    parser.add_argument(
        '--show',
        action='store_true',
        help='whether to display the prediction results in a window.')
    parser.add_argument(
        '--interval',
        type=int,
        default=1,
        help='visualize per interval samples.')
    parser.add_argument(
        '--wait-time',
        type=float,
        default=2,
        help='display time of every window. (second)')
    parser.add_argument(
        '--launcher',
        choices=['none', 'pytorch', 'slurm', 'mpi'],
        default='none',
        help='job launcher')
    parser.add_argument('--local_rank', '--local-rank', type=int, default=0)
    args = parser.parse_args()
    if 'LOCAL_RANK' not in os.environ:
        os.environ['LOCAL_RANK'] = str(args.local_rank)
    return args


def merge_args(cfg, args):
    """Merge CLI arguments to config."""
    # -------------------- visualization --------------------
    if args.show or (args.show_dir is not None):
        assert 'visualization' in cfg.default_hooks, \
            'VisualizationHook is not set in the `default_hooks` field of ' \
            'config. Please set `visualization=dict(type="VisualizationHook")`'

        cfg.default_hooks.visualization.enable = True
        cfg.default_hooks.visualization.show = args.show
        cfg.default_hooks.visualization.wait_time = args.wait_time
        cfg.default_hooks.visualization.out_dir = args.show_dir
        cfg.default_hooks.visualization.interval = args.interval

    # -------------------- Dump predictions --------------------
    if args.dump is not None:
        assert args.dump.endswith(('.pkl', '.pickle')), \
            'The dump file must be a pkl file.'
        dump_metric = dict(type='DumpResults', out_file_path=args.dump)
        if isinstance(cfg.test_evaluator, (list, tuple)):
            cfg.test_evaluator = list(cfg.test_evaluator)
            cfg.test_evaluator.append(dump_metric)
        else:
            cfg.test_evaluator = [cfg.test_evaluator, dump_metric]

    return cfg


def main():
    args = parse_args()
    
    config = getattr(args, 'config', None)
    assert not (args.test_best and args.test_last)
    if args.test_best:
        for dir in os.listdir(args.work_dir):
            if dir.startswith('bs') and dir.endswith('.py') and config == None:
                config = osp.join(args.work_dir, dir)
            if dir.endswith('.pth') and dir.startswith('best'):
                checkpoint = osp.join(args.work_dir, dir)
    elif args.test_last:
        for dir in os.listdir(args.work_dir):
            if dir.startswith('bs') and dir.endswith('.py') and config == None:
                config = osp.join(args.work_dir, dir)
        with open(osp.join(args.work_dir, 'last_checkpoint'), 'r') as fp:
            checkpoint = fp.read()
    else:
        checkpoint = args.checkpoint
    
    # load config
    cfg = Config.fromfile(config)
    cfg = merge_args(cfg, args)
    cfg.launcher = args.launcher
    if args.cfg_options is not None:
        cfg.merge_from_dict(args.cfg_options)

    # work_dir is determined in this priority: CLI > segment in file > filename
    if args.work_dir is not None:
        # update configs according to CLI args if args.work_dir is not None
        cfg.work_dir = args.work_dir
    elif cfg.get('work_dir', None) is None:
        # use config filename as default work_dir if cfg.work_dir is None
        cfg.work_dir = osp.join('./work_dirs',
                                osp.splitext(osp.basename(config))[0])

    cfg.load_from = checkpoint
    if hasattr(cfg, 'visualizer'):
        cfg.visualizer = None
    

    # build the runner from config
    runner = Runner.from_cfg(cfg)
    
    # calculate the complexity info for the model
    if not args.metric_only:
        from tools.complexity_analysis import get_model_computational_metrics
        params_and_flops, fps = get_model_computational_metrics(
            runner.model, input_shape=(cfg.input_shape, (cfg.output_len, *cfg.input_shape[1:]))
        )
        runner.logger.info(params_and_flops)
        runner.logger.info(f'fps: {fps}')
    
    # start testing
    runner.test()


if __name__ == '__main__':
    main()