From 34951f3a61d8eb74173ff36cef4d08ea2f3da684 Mon Sep 17 00:00:00 2001 From: YunLiu <55491388+KumoLiu@users.noreply.github.com> Date: Wed, 20 Mar 2024 23:48:50 +0800 Subject: [PATCH 1/4] Update tutorials under reconstruction folder (#1671) Fixes #1670 ### Description Set "image_only=False" in `LoadImage`. ### Checks - [ ] Avoid including large-size files in the PR. - [ ] Clean up long text outputs from code cells in the notebook. - [ ] For security purposes, please check the contents and remove any sensitive info such as user names and private key. - [ ] Ensure (1) hyperlinks and markdown anchors are working (2) use relative paths for tutorial repo files (3) put figure and graphs in the `./figure` folder - [ ] Notebook runs automatically `./runner.sh -t ` --------- Signed-off-by: YunLiu <55491388+KumoLiu@users.noreply.github.com> --- detection/generate_transforms.py | 6 +++--- detection/luna16_prepare_images.py | 1 + detection/luna16_prepare_images_dicom.py | 1 + reconstruction/MRI_reconstruction/unet_demo/inference.ipynb | 2 +- reconstruction/MRI_reconstruction/unet_demo/train.py | 2 +- .../MRI_reconstruction/varnet_demo/inference.ipynb | 2 +- reconstruction/MRI_reconstruction/varnet_demo/train.py | 2 +- 7 files changed, 9 insertions(+), 7 deletions(-) diff --git a/detection/generate_transforms.py b/detection/generate_transforms.py index b255532742..5fc9ac91c2 100644 --- a/detection/generate_transforms.py +++ b/detection/generate_transforms.py @@ -78,7 +78,7 @@ def generate_detection_train_transform( train_transforms = Compose( [ - LoadImaged(keys=[image_key], meta_key_postfix="meta_dict"), + LoadImaged(keys=[image_key], image_only=False, meta_key_postfix="meta_dict"), EnsureChannelFirstd(keys=[image_key]), EnsureTyped(keys=[image_key, box_key], dtype=torch.float32), EnsureTyped(keys=[label_key], dtype=torch.long), @@ -224,7 +224,7 @@ def generate_detection_val_transform( val_transforms = Compose( [ - LoadImaged(keys=[image_key], meta_key_postfix="meta_dict"), + LoadImaged(keys=[image_key], image_only=False, meta_key_postfix="meta_dict"), EnsureChannelFirstd(keys=[image_key]), EnsureTyped(keys=[image_key, box_key], dtype=torch.float32), EnsureTyped(keys=[label_key], dtype=torch.long), @@ -280,7 +280,7 @@ def generate_detection_inference_transform( test_transforms = Compose( [ - LoadImaged(keys=[image_key], meta_key_postfix="meta_dict"), + LoadImaged(keys=[image_key], image_only=False, meta_key_postfix="meta_dict"), EnsureChannelFirstd(keys=[image_key]), EnsureTyped(keys=[image_key], dtype=torch.float32), Orientationd(keys=[image_key], axcodes="RAS"), diff --git a/detection/luna16_prepare_images.py b/detection/luna16_prepare_images.py index 0e9047e5ca..ae0f7dc86e 100644 --- a/detection/luna16_prepare_images.py +++ b/detection/luna16_prepare_images.py @@ -61,6 +61,7 @@ def main(): [ LoadImaged( keys=["image"], + image_only=False, meta_key_postfix="meta_dict", reader="itkreader", affine_lps_to_ras=True, diff --git a/detection/luna16_prepare_images_dicom.py b/detection/luna16_prepare_images_dicom.py index 5e8ed782db..165d158b37 100644 --- a/detection/luna16_prepare_images_dicom.py +++ b/detection/luna16_prepare_images_dicom.py @@ -63,6 +63,7 @@ def main(): [ LoadImaged( keys=["image"], + image_only=False, meta_key_postfix="meta_dict", reader="itkreader", affine_lps_to_ras=True, diff --git a/reconstruction/MRI_reconstruction/unet_demo/inference.ipynb b/reconstruction/MRI_reconstruction/unet_demo/inference.ipynb index 301dd9cb2f..8f8e3a6767 100644 --- a/reconstruction/MRI_reconstruction/unet_demo/inference.ipynb +++ b/reconstruction/MRI_reconstruction/unet_demo/inference.ipynb @@ -236,7 +236,7 @@ "\n", "test_transforms = Compose(\n", " [\n", - " LoadImaged(keys=[\"kspace\"], reader=FastMRIReader, dtype=np.complex64),\n", + " LoadImaged(keys=[\"kspace\"], reader=FastMRIReader, image_only=False, dtype=np.complex64),\n", " # user can also add other random transforms\n", " ExtractDataKeyFromMetaKeyd(keys=[\"reconstruction_rss\", \"mask\"], meta_key=\"kspace_meta_dict\"),\n", " MaskTransform,\n", diff --git a/reconstruction/MRI_reconstruction/unet_demo/train.py b/reconstruction/MRI_reconstruction/unet_demo/train.py index b54bd224bf..4286223eaa 100644 --- a/reconstruction/MRI_reconstruction/unet_demo/train.py +++ b/reconstruction/MRI_reconstruction/unet_demo/train.py @@ -97,7 +97,7 @@ def trainer(args): train_transforms = Compose( [ - LoadImaged(keys=["kspace"], reader=FastMRIReader, dtype=np.complex64), + LoadImaged(keys=["kspace"], reader=FastMRIReader, image_only=False, dtype=np.complex64), # user can also add other random transforms ExtractDataKeyFromMetaKeyd(keys=["reconstruction_rss", "mask"], meta_key="kspace_meta_dict"), MaskTransform, diff --git a/reconstruction/MRI_reconstruction/varnet_demo/inference.ipynb b/reconstruction/MRI_reconstruction/varnet_demo/inference.ipynb index 0533fadef1..d858ddd70a 100644 --- a/reconstruction/MRI_reconstruction/varnet_demo/inference.ipynb +++ b/reconstruction/MRI_reconstruction/varnet_demo/inference.ipynb @@ -251,7 +251,7 @@ "\n", "test_transforms = Compose(\n", " [\n", - " LoadImaged(keys=[\"kspace\"], reader=FastMRIReader, dtype=np.complex64),\n", + " LoadImaged(keys=[\"kspace\"], reader=FastMRIReader, image_only=False, dtype=np.complex64),\n", " # user can also add other random transforms\n", " ExtractDataKeyFromMetaKeyd(keys=[\"reconstruction_rss\", \"mask\"], meta_key=\"kspace_meta_dict\"),\n", " MaskTransform,\n", diff --git a/reconstruction/MRI_reconstruction/varnet_demo/train.py b/reconstruction/MRI_reconstruction/varnet_demo/train.py index 6e8c3d3949..969976bf90 100644 --- a/reconstruction/MRI_reconstruction/varnet_demo/train.py +++ b/reconstruction/MRI_reconstruction/varnet_demo/train.py @@ -106,7 +106,7 @@ def trainer(args): train_transforms = Compose( [ - LoadImaged(keys=["kspace"], reader=FastMRIReader, dtype=np.complex64), + LoadImaged(keys=["kspace"], reader=FastMRIReader, image_only=False, dtype=np.complex64), # user can also add other random transforms but remember to disable randomness for val_transforms ExtractDataKeyFromMetaKeyd(keys=["reconstruction_rss", "mask"], meta_key="kspace_meta_dict"), MaskTransform, From 5d6f600ecd6e71ec8c3981d56b767b192f866479 Mon Sep 17 00:00:00 2001 From: YunLiu <55491388+KumoLiu@users.noreply.github.com> Date: Thu, 21 Mar 2024 10:30:15 +0800 Subject: [PATCH 2/4] Update to use torchrun in Multi-GPU SSL Training (#1663) Fixes #1662 ### Description A few sentences describing the changes proposed in this pull request. ### Checks - [ ] Avoid including large-size files in the PR. - [ ] Clean up long text outputs from code cells in the notebook. - [ ] For security purposes, please check the contents and remove any sensitive info such as user names and private key. - [ ] Ensure (1) hyperlinks and markdown anchors are working (2) use relative paths for tutorial repo files (3) put figure and graphs in the `./figure` folder - [ ] Notebook runs automatically `./runner.sh -t ` --------- Signed-off-by: YunLiu <55491388+KumoLiu@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- self_supervised_pretraining/vit_unetr_ssl/README.md | 2 +- .../vit_unetr_ssl/multi_gpu/mgpu_ssl_train.py | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/self_supervised_pretraining/vit_unetr_ssl/README.md b/self_supervised_pretraining/vit_unetr_ssl/README.md index 6706f00df5..563a1a0bcc 100644 --- a/self_supervised_pretraining/vit_unetr_ssl/README.md +++ b/self_supervised_pretraining/vit_unetr_ssl/README.md @@ -144,7 +144,7 @@ At the time of creation of this tutorial, the below additional dependencies are To begin training with 2 GPU's please see the below example command for execution of the SSL multi-gpu training script: -`CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch --nproc_per_node=2 mgpu_ssl_train.py --batch_size=8 --epochs=500 --base_lr=2e-4 --logdir_path=/to/be/defined --output=/to/be/defined --data_root=/to/be/defined --json_path=/to/be/defined` +`CUDA_VISIBLE_DEVICES=0,1 torchrun --nproc_per_node=2 mgpu_ssl_train.py --batch_size=8 --epochs=500 --base_lr=2e-4 --logdir_path=/to/be/defined --output=/to/be/defined --data_root=/to/be/defined --json_path=/to/be/defined` It can be configured to launch on more GPU's by adding the relevant `CUDA Device` ID in `CUDA_VISIBLE_DEVICES` and increasing the total count of GPU's `--nproc_per_node` diff --git a/self_supervised_pretraining/vit_unetr_ssl/multi_gpu/mgpu_ssl_train.py b/self_supervised_pretraining/vit_unetr_ssl/multi_gpu/mgpu_ssl_train.py index 70ab7debc9..52499756a5 100644 --- a/self_supervised_pretraining/vit_unetr_ssl/multi_gpu/mgpu_ssl_train.py +++ b/self_supervised_pretraining/vit_unetr_ssl/multi_gpu/mgpu_ssl_train.py @@ -67,8 +67,6 @@ def parse_option(): metavar="PATH", help="root of output folder, the full path is // (default: output)", ) - # Distributed Training - parser.add_argument("--local_rank", type=int, help="local rank for DistributedDataParallel") # DL Training Hyper-parameters parser.add_argument("--epochs", default=100, type=int, help="number of epochs") @@ -139,10 +137,6 @@ def main(args): data_list_file_path=json_path, is_segmentation=False, data_list_key="validation", base_dir=data_root ) - # TODO Delete the below print statements - print("List of training samples: {}".format(train_list)) - print("List of validation samples: {}".format(val_list)) - print("Total training data are {} and validation data are {}".format(len(train_list), len(val_list))) train_dataset = CacheDataset(data=train_list, transform=train_transforms, cache_rate=1.0, num_workers=4) @@ -191,7 +185,7 @@ def main(args): optimizer = torch.optim.Adam(model.parameters(), lr=args.base_lr) model = torch.nn.parallel.DistributedDataParallel( - model, device_ids=[args.local_rank], broadcast_buffers=False, find_unused_parameters=True + model, device_ids=[int(os.environ["LOCAL_RANK"])], broadcast_buffers=False, find_unused_parameters=True ) model_without_ddp = model.module @@ -340,7 +334,7 @@ def validate(data_loader, model, loss_functions): else: rank = -1 world_size = -1 - torch.cuda.set_device(args.local_rank) + torch.cuda.set_device(rank) torch.distributed.init_process_group(backend="nccl", init_method="env://", world_size=world_size, rank=rank) torch.distributed.barrier() From 6ca52e4c61ea215c1fb44a07a81302b3fb9ef95e Mon Sep 17 00:00:00 2001 From: YunLiu <55491388+KumoLiu@users.noreply.github.com> Date: Thu, 21 Mar 2024 19:50:24 +0800 Subject: [PATCH 3/4] Update to use torchrun in "brats_training_ddp" (#1666) Fixes #1665 ### Checks - [ ] Avoid including large-size files in the PR. - [ ] Clean up long text outputs from code cells in the notebook. - [ ] For security purposes, please check the contents and remove any sensitive info such as user names and private key. - [ ] Ensure (1) hyperlinks and markdown anchors are working (2) use relative paths for tutorial repo files (3) put figure and graphs in the `./figure` folder - [ ] Notebook runs automatically `./runner.sh -t ` --------- Signed-off-by: YunLiu <55491388+KumoLiu@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- acceleration/README.md | 2 +- .../brats_training_ddp.py | 30 +++++++------------ 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/acceleration/README.md b/acceleration/README.md index 47e2e7fb99..e803b6e445 100644 --- a/acceleration/README.md +++ b/acceleration/README.md @@ -6,7 +6,7 @@ Typically, model training is a time-consuming step during deep learning developm The document introduces details of how to profile the training pipeline, how to analyze the dataset and select suitable algorithms, and how to optimize GPU utilization in single GPU, multi-GPUs or even multi-nodes. #### [distributed_training](./distributed_training) The examples show how to execute distributed training and evaluation based on 3 different frameworks: -- PyTorch native `DistributedDataParallel` module with `torch.distributed.launch`. +- PyTorch native `DistributedDataParallel` module with `torchrun`. - Horovod APIs with `horovodrun`. - PyTorch ignite and MONAI workflows. diff --git a/acceleration/distributed_training/brats_training_ddp.py b/acceleration/distributed_training/brats_training_ddp.py index c8851b5ab3..aadc8a5ba4 100644 --- a/acceleration/distributed_training/brats_training_ddp.py +++ b/acceleration/distributed_training/brats_training_ddp.py @@ -22,31 +22,28 @@ Main steps to set up the distributed training: -- Execute `torch.distributed.launch` to create processes on every node for every GPU. +- Execute `torchrun` to create processes on every node for every GPU. It receives parameters as below: `--nproc_per_node=NUM_GPUS_PER_NODE` `--nnodes=NUM_NODES` - `--node_rank=INDEX_CURRENT_NODE` `--master_addr="localhost"` `--master_port=1234` - For more details, refer to https://github.com/pytorch/pytorch/blob/master/torch/distributed/launch.py. + For more details, refer to https://github.com/pytorch/pytorch/blob/master/torch/distributed/run.py. Alternatively, we can also use `torch.multiprocessing.spawn` to start program, but it that case, need to handle all the above parameters and compute `rank` manually, then set to `init_process_group`, etc. - `torch.distributed.launch` is even more efficient than `torch.multiprocessing.spawn` during training. + `torchrun` is even more efficient than `torch.multiprocessing.spawn` during training. - Use `init_process_group` to initialize every process, every GPU runs in a separate process with unique rank. - Here we use `NVIDIA NCCL` as the backend and must set `init_method="env://"` if use `torch.distributed.launch`. + Here we use `NVIDIA NCCL` as the backend and must set `init_method="env://"` if use `torchrun`. - Wrap the model with `DistributedDataParallel` after moving to expected device. - Partition dataset before training, so every rank process will only handle its own data partition. Note: - `torch.distributed.launch` will launch `nnodes * nproc_per_node = world_size` processes in total. + `torchrun` will launch `nnodes * nproc_per_node = world_size` processes in total. Suggest setting exactly the same software environment for every node, especially `PyTorch`, `nccl`, etc. A good practice is to use the same MONAI docker image for all nodes directly. Example script to execute this program on every node: - python -m torch.distributed.launch --nproc_per_node=NUM_GPUS_PER_NODE - --nnodes=NUM_NODES --node_rank=INDEX_CURRENT_NODE - --master_addr="localhost" --master_port=1234 - brats_training_ddp.py -d DIR_OF_TESTDATA + python -m torchrun --nproc_per_node=NUM_GPUS_PER_NODE --nnodes=NUM_NODES + --master_addr="localhost" --master_port=1234 brats_training_ddp.py -d DIR_OF_TESTDATA This example was tested with [Ubuntu 16.04/20.04], [NCCL 2.6.3]. @@ -162,7 +159,7 @@ def _generate_data_list(self, dataset_dir): def main_worker(args): # disable logging for processes except 0 on every node - if args.local_rank != 0: + if int(os.environ["LOCAL_RANK"]) != 0: f = open(os.devnull, "w") sys.stdout = sys.stderr = f if not os.path.exists(args.dir): @@ -170,7 +167,7 @@ def main_worker(args): # initialize the distributed training process, every GPU runs in a process dist.init_process_group(backend="nccl", init_method="env://") - device = torch.device(f"cuda:{args.local_rank}") + device = torch.device(f"cuda:{os.environ['LOCAL_RANK']}") torch.cuda.set_device(device) # use amp to accelerate training scaler = torch.cuda.amp.GradScaler() @@ -364,8 +361,6 @@ def evaluate(model, val_loader, dice_metric, dice_metric_batch, post_trans): def main(): parser = argparse.ArgumentParser() parser.add_argument("-d", "--dir", default="./testdata", type=str, help="directory of Brain Tumor dataset") - # must parse the command-line argument: ``--local_rank=LOCAL_PROCESS_RANK``, which will be provided by DDP - parser.add_argument("--local_rank", type=int, help="node rank for distributed training") parser.add_argument("--epochs", default=300, type=int, metavar="N", help="number of total epochs to run") parser.add_argument("--lr", default=1e-4, type=float, help="learning rate") parser.add_argument("-b", "--batch_size", default=1, type=int, help="mini-batch size of every GPU") @@ -388,12 +383,9 @@ def main(): main_worker(args=args) -# usage example(refer to https://github.com/pytorch/pytorch/blob/master/torch/distributed/launch.py): +# usage example(refer to https://github.com/pytorch/pytorch/blob/main/torch/distributed/run.py): -# python -m torch.distributed.launch --nproc_per_node=NUM_GPUS_PER_NODE -# --nnodes=NUM_NODES --node_rank=INDEX_CURRENT_NODE -# --master_addr="localhost" --master_port=1234 -# brats_training_ddp.py -d DIR_OF_TESTDATA +# torchrun --nproc_per_node=NUM_GPUS_PER_NODE --nnodes=NUM_NODES brats_training_ddp.py -d DIR_OF_TESTDATA if __name__ == "__main__": main() From 89ca2f101b1be9d99e7c3c4796e45c80c03c2170 Mon Sep 17 00:00:00 2001 From: YunLiu <55491388+KumoLiu@users.noreply.github.com> Date: Fri, 22 Mar 2024 00:28:39 +0800 Subject: [PATCH 4/4] Update "pathology/tumor_detection/ignite/profiling_camelyon_pipeline.ipynb" (#1673) Fixes # . ### Description Update the usage of nvtx in "tumor_detection_camelyon" ### Checks - [ ] Avoid including large-size files in the PR. - [ ] Clean up long text outputs from code cells in the notebook. - [ ] For security purposes, please check the contents and remove any sensitive info such as user names and private key. - [ ] Ensure (1) hyperlinks and markdown anchors are working (2) use relative paths for tutorial repo files (3) put figure and graphs in the `./figure` folder - [ ] Notebook runs automatically `./runner.sh -t ` --------- Signed-off-by: YunLiu <55491388+KumoLiu@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/workflows/test-modified.yml | 2 +- .../ignite/profiling_camelyon_pipeline.ipynb | 67 ++++--------------- runner.sh | 2 +- 3 files changed, 16 insertions(+), 55 deletions(-) diff --git a/.github/workflows/test-modified.yml b/.github/workflows/test-modified.yml index 2cd3857b56..b6dd0223bf 100644 --- a/.github/workflows/test-modified.yml +++ b/.github/workflows/test-modified.yml @@ -16,7 +16,7 @@ jobs: build: if: github.repository == 'Project-MONAI/tutorials' container: - image: nvcr.io/nvidia/pytorch:22.04-py3 + image: nvcr.io/nvidia/pytorch:24.02-py3 options: --gpus all --ipc host runs-on: [self-hosted, linux, x64] steps: diff --git a/pathology/tumor_detection/ignite/profiling_camelyon_pipeline.ipynb b/pathology/tumor_detection/ignite/profiling_camelyon_pipeline.ipynb index 611af00f07..e3261f16ea 100644 --- a/pathology/tumor_detection/ignite/profiling_camelyon_pipeline.ipynb +++ b/pathology/tumor_detection/ignite/profiling_camelyon_pipeline.ipynb @@ -72,7 +72,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -108,12 +108,13 @@ "text": [ "Downloading...\n", "From: https://drive.google.com/uc?id=1uWS4CXKD-NP_6-SgiQbQfhFMzbs0UJIr\n", - "To: /Users/bhashemian/workspace/tutorials/pathology/tumor_detection/ignite/training.csv\n", - "100%|██████████| 153k/153k [00:00<00:00, 1.91MB/s]\n", + "To: /workspace/Code/tutorials/pathology/tumor_detection/ignite/training.csv\n", + "100%|██████████| 153k/153k [00:00<00:00, 1.75MB/s]\n", "Downloading...\n", - "From: https://drive.google.com/uc?id=1OxAeCMVqH9FGpIWpAXSEJe6cLinEGQtF\n", - "To: /Users/bhashemian/workspace/tutorials/pathology/tumor_detection/ignite/training/images/tumor_091.tif\n", - "100%|██████████| 546M/546M [00:22<00:00, 24.1MB/s] \n" + "From (original): https://drive.google.com/uc?id=1OxAeCMVqH9FGpIWpAXSEJe6cLinEGQtF\n", + "From (redirected): https://drive.google.com/uc?id=1OxAeCMVqH9FGpIWpAXSEJe6cLinEGQtF&confirm=t&uuid=cbee2da2-249c-4d81-bc97-6e589a8452ce\n", + "To: /workspace/Code/tutorials/pathology/tumor_detection/ignite/training/images/tumor_091.tif\n", + "100%|██████████| 546M/546M [00:10<00:00, 50.4MB/s] \n" ] }, { @@ -157,8 +158,8 @@ "source": [ "!nsys profile \\\n", " --trace nvtx,osrt,cudnn,cuda, \\\n", - " --delay 15 \\\n", - " --duration 60 \\\n", + " --delay 5 \\\n", + " --duration 10 \\\n", " --show-output true \\\n", " --force-overwrite true \\\n", " --output profile_report.nsys-rep \\\n", @@ -172,52 +173,12 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 1, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Generating SQLite file profile_report.sqlite from profile_report.nsys-rep\n", - "Exporting 265495 events: [=================================================100%]\n", - "Using profile_report.sqlite for SQL queries.\n", - "Running [/usr/local/cuda-11.6/NsightSystems-cli-2021.5.2/target-linux-x64/reports/nvtxppsum.py profile_report.sqlite]... \n", - "\n", - "+----------+-----------------+-----------+--------------+--------------+------------+-------------+--------------+--------------------------+\n", - "| Time (%) | Total Time (ns) | Instances | Avg (ns) | Med (ns) | Min (ns) | Max (ns) | StdDev (ns) | Range |\n", - "+----------+-----------------+-----------+--------------+--------------+------------+-------------+--------------+--------------------------+\n", - "| 28.7 | 33706579200 | 5 | 6741315840.0 | 6324451800.0 | 176995000 | 13682363400 | 4889241407.0 | Iteration |\n", - "| 21.3 | 25011936300 | 5 | 5002387260.0 | 4787481900.0 | 2873517700 | 8072929200 | 2035498721.7 | Batch |\n", - "| 20.3 | 23839370600 | 50 | 476787412.0 | 376589400.0 | 220633100 | 1154097400 | 276441893.0 | Preprocessing |\n", - "| 19.3 | 22742525900 | 450 | 50538946.4 | 36570950.0 | 18874200 | 202062000 | 36166736.1 | TorchVisiond_ColorJitter |\n", - "| 9.4 | 11044461700 | 5 | 2208892340.0 | 1996530400.0 | 148099300 | 4407900900 | 1534918799.1 | ResNet18 |\n", - "| 0.3 | 384269900 | 450 | 853933.1 | 65400.0 | 21000 | 22487800 | 2212634.3 | RandZoomd |\n", - "| 0.2 | 244892100 | 450 | 544204.7 | 441950.0 | 321800 | 8677700 | 541248.4 | ScaleIntensityRanged |\n", - "| 0.1 | 128083500 | 450 | 284630.0 | 243900.0 | 187400 | 4721600 | 230932.6 | Postprocessing |\n", - "| 0.1 | 91848800 | 450 | 204108.4 | 176450.0 | 128700 | 745700 | 87187.5 | ToNumpyd |\n", - "| 0.1 | 65417500 | 450 | 145372.2 | 117000.0 | 90200 | 4613300 | 219185.3 | AsDiscreted |\n", - "| 0.1 | 59017500 | 450 | 131150.0 | 82250.0 | 17400 | 1050600 | 123950.6 | RandRotate90d |\n", - "| 0.0 | 55917100 | 50 | 1118342.0 | 882450.0 | 685900 | 5798000 | 801880.3 | GridSplitd |\n", - "| 0.0 | 54120900 | 450 | 120268.7 | 100700.0 | 68500 | 721200 | 59828.4 | ToTensord |\n", - "| 0.0 | 51677300 | 450 | 114838.4 | 101350.0 | 67500 | 1154000 | 60160.7 | CastToTyped |\n", - "| 0.0 | 50674700 | 450 | 112610.4 | 95650.0 | 71300 | 613700 | 51643.2 | ToTensord_2 |\n", - "| 0.0 | 48966300 | 450 | 108814.0 | 95900.0 | 75200 | 428700 | 44791.2 | Activationsd |\n", - "| 0.0 | 39524100 | 450 | 87831.3 | 45950.0 | 7600 | 2748400 | 153536.8 | RandFlipd |\n", - "| 0.0 | 2460800 | 50 | 49216.0 | 39450.0 | 28700 | 146100 | 23219.6 | Lambdad |\n", - "| 0.0 | 1074200 | 4 | 268550.0 | 194150.0 | 142200 | 543700 | 186523.3 | Loss |\n", - "+----------+-----------------+-----------+--------------+--------------+------------+-------------+--------------+--------------------------+\n", - "\n", - "Running [/usr/local/cuda-11.6/NsightSystems-cli-2021.5.2/target-linux-x64/reports/nvtxppsum.py profile_report.sqlite] to [profile_report_nvtxppsum.csv]... PROCESSED\n", - "\n", - "Running [/usr/local/cuda-11.6/NsightSystems-cli-2021.5.2/target-linux-x64/reports/nvtxpptrace.py profile_report.sqlite] to [profile_report_nvtxpptrace.csv]... PROCESSED\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "!nsys stats \\\n", - " --report nvtxppsum,nvtxppsum,nvtxpptrace \\\n", + " --report nvtx_pushpop_sum,nvtx_pushpop_sum,nvtx_pushpop_trace \\\n", " --format table,csv \\\n", " --output -,. \\\n", " --force-overwrite true \\\n", @@ -576,7 +537,7 @@ ], "source": [ "# Load NVTX Push/Pop Range Summary\n", - "summary = pd.read_csv(\"profile_report_nvtxppsum.csv\")\n", + "summary = pd.read_csv(\"profile_report_nvtx_pushpop_sum.csv\")\n", "# display(summary)\n", "\n", "# Set the Range (which is the name of each range) as the index\n", @@ -704,7 +665,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.13" + "version": "3.8.10" } }, "nbformat": 4, diff --git a/runner.sh b/runner.sh index 553e112d61..81bb4e08fe 100755 --- a/runner.sh +++ b/runner.sh @@ -563,7 +563,7 @@ for file in "${files[@]}"; do unset PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION fi - cmd=$(echo "papermill ${papermill_opt} --progress-bar -k ${kernelspec}") + cmd=$(echo "papermill ${papermill_opt} --progress-bar --log-output -k ${kernelspec}") echo "$cmd" time out=$(echo "$notebook" | eval "$cmd") success=$?