diff --git a/tests/automation/README.md b/tests/automation/README.md new file mode 100644 index 0000000..75ad41b --- /dev/null +++ b/tests/automation/README.md @@ -0,0 +1,150 @@ +# Holoscan CLI Test Automation + +This directory contains scripts to automate the testing of packaging and running applications designed using [Holoscan SDK](https://developer.nvidia.com/holoscan-sdk). + +## Requirements + +To successfully execute the test automation script, the system must meet the requirements specified by both the [Holoscan SDK](https://docs.nvidia.com/holoscan/sdk-user-guide/sdk_installation.html) and the [Holoscan CLI](https://github.com/nvidia-holoscan/holoscan-cli). Additionally, the following tools are required to package and run the application: + +* Holoscan CLI +* jq +* curl +* An [NGC](https://catalog.ngc.nvidia.com/) account, which must be logged in via `docker login nvcr.io` + +## Usage + +To run the test automation script, navigate to the automation directory, then execute the script with the application directory as an argument: + +``` +cd tests/automation +./test.sh [application-directory] +# for example +./tests.sh ./endoscopy_tool_tracking_cpp +``` + +The `test.sh` script invokes `check.sh` to verify that all system requirements are met. It will exit if any of the requirements are missing. + +The automation script performs the following steps: + +1. Clones the specified git repository. +2. Packages the application. +3. Downloads the configured test data. +4. Run the packaged application. +5. Cleans up any downloaded and generated data. + +### Adding New Application + +Automating the testing of a new Holoscan-enabled application is straightforward and can be accomplished with the following steps: + +1. Create a new directory and include a `config.json` file that defines the application and specifies where the script can retrieve the test data. Refer to the sections below for examples and the configuration schema. +2. Optionally, copy the application's configuration YAML file and modify the values to ensure that the application can start and exit on its own. + +This is all that's required. + +### Sample Configuration + +#### Holoscan Application + +Here’s a sample configuration for testing the C++ version of the Video Replayer: + +With this configuration, the test automation script will clone the [Holoscan SDK repository](https://github.com/nvidia-holoscan/holoscan-sdk) from GitHub, download test data from NGC, package the application with `--include holoviz`, and run the application using the supplied `app.yaml` configuration file along with the `--render` option. + +```json +{ + "name": "video-replayer-cpp", + "source": { + "repo": "https://github.com/nvidia-holoscan/holoscan-sdk.git", + "path": "examples/video_replayer/cpp", + "lang": "cpp" + }, + "config": { + "source": "local", + "path": "./app.yaml" + }, + "package": { + "args": "--includes holoviz" + }, + "run": { + "args": "--render" + }, + "data": { + "dirname" : "racerx", + "source" : "https://api.ngc.nvidia.com/v2/resources/org/nvidia/team/clara-holoscan/holoscan_racerx_video/20231009/files" + } +} +``` + +#### Holohub Application + +Another sample configuration is used for testing the Endoscopy Tool Tracking application from Holohub: + +In this case, the test automation script will clone the [Holohub](https://github.com/nvidia-holoscan/holohub) repository, invoke the `devcontainer` script to build the specified application, utilize test data downloaded by the `devcontainer` script, and package the application using the provided `app.yaml` file with additional arguments defined in `package.args`. Finally, it executes the application with the arguments specified in `run.args`. + + +```json +{ + "name": "endoscopy-tool-tracking-cpp", + "source": { + "repo": "https://github.com/nvidia-holoscan/holohub.git", + "app": "endoscopy_tool_tracking", + "lang": "python", + "path": "applications/endoscopy_tool_tracking/python/endoscopy_tool_tracking.py" + }, + "config": { + "source": "local", + "path": "./app.yaml" + }, + "package": { + "args": "--includes onnx holoviz --add /install/lib --add /install/python/lib/" + }, + "run": { + "args": "--render" + }, + "data": { + "source" : "local-holohub", + "dirname" : "endoscopy" + } +} +``` + +> [!NOTE] +> The test automation script supports packaging Holohub applications from the `install/` generated by the devcontainer script. If a desired application does not create artifacts in the `install/` directory, update the associated CMake configuration to [install](https://cmake.org/cmake/help/latest/command/install.html) the application first. + + +### Configuration Schema + +``` +{ + "name": "", # Required + "source": { + "repo": ".git", # Required + "path": "", # Required + "app": "Holohub only: name of application. Same as the application name used by the devcontainer script." + }, + "config": { + "source": "[local]", # optional value + "path": "./app.yaml" # Required + }, + "package": { + "args": "" + }, + "run": { + "args": "" + }, + "data": { + "source" : "", # 'local-holohub' means to use test data downloaded by the devcontainer script + "dirname" : "" + } +} +``` + + +## Links + +- [Holoscan SDK User Guide](https://docs.nvidia.com/holoscan/sdk-user-guide) +- [Holoscan Application Package Specification (HAP)](https://docs.nvidia.com/holoscan/sdk-user-guide/cli/hap.html) +- [Holscan SDK Github Repository](https://github.com/nvidia-holoscan/holoscan-sdk) +- [Holohub Github Repository](https://github.com/nvidia-holoscan/holohub/) +- [Holoscan CLI Github Repository](https://github.com/nvidia-holoscan/holoscan-cli) +- [NGC](https://ngc.nvidia.com/) \ No newline at end of file diff --git a/tests/automation/artifacts.json b/tests/automation/artifacts.json new file mode 100644 index 0000000..c5dad9a --- /dev/null +++ b/tests/automation/artifacts.json @@ -0,0 +1,32 @@ +{ + "2.9.0": { + "holoscan": { + "debian-version": "2.9.0", + "wheel-version": "2.9.0", + "base-images": { + "dgpu": "nvcr.io/nvidia/cuda:12.6.0-runtime-ubuntu22.04", + "igpu": "nvcr.io/nvidia/tensorrt:24.08-py3-igpu" + }, + "build-images": { + "igpu": { + "jetson-agx-orin-devkit": "nvcr.io/nvstaging/holoscan/holoscan:v2.9.0.1-igpu", + "igx-orin-devkit": "nvcr.io/nvstaging/holoscan/holoscan:v2.9.0.1-igpu", + "sbsa": "nvcr.io/nvstaging/holoscan/holoscan:v2.9.0.1-igpu" + }, + "dgpu": { + "x64-workstation": "nvcr.io/nvstaging/holoscan/holoscan:v2.9.0.1-dgpu", + "igx-orin-devkit": "nvcr.io/nvstaging/holoscan/holoscan:v2.9.0.1-dgpu", + "sbsa": "nvcr.io/nvstaging/holoscan/holoscan:v2.9.0.1-dgpu", + "clara-agx-devkit": "nvcr.io/nvstaging/holoscan/holoscan:v2.9.0.1-dgpu" + }, + "cpu": { + "x64-workstation": "nvcr.io/nvstaging/holoscan/holoscan:v2.9.0.1-dgpu" + } + } + }, + "health-probes": { + "linux/amd64": "https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/v0.4.19/grpc_health_probe-linux-amd64", + "linux/arm64": "https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/v0.4.19/grpc_health_probe-linux-arm64" + } + } +} diff --git a/tests/automation/check.sh b/tests/automation/check.sh new file mode 100755 index 0000000..b1096e8 --- /dev/null +++ b/tests/automation/check.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#=============================================================================== + +echo "Checking system requirements..." + +status=0 +if ! command -v curl 2>&1 >/dev/null +then + echo "curl could not be found, use the following command to install curl" + echo "$ sudo apt install curl" + status=-1 +fi + +if ! command -v jq 2>&1 >/dev/null +then + echo "jq could not be found, use the following command to install jq" + echo "$ sudo apt install jq" + status=-1 +fi + +if ! command -v holoscan 2>&1 >/dev/null +then + echo "Holoscan CLI could not be found, use the following command to install holoscan" + echo "$ pip install holoscan-cli" + status=-1 +fi + +if ! docker login nvcr.io < /dev/null >& /dev/null +then + echo "Please login to nvcr.io. For example:" + echo "$ docker login nvcr.io" + status=-1 +fi + +exit $status \ No newline at end of file diff --git a/tests/automation/endoscopy_tool_tracking_cpp/app.yaml b/tests/automation/endoscopy_tool_tracking_cpp/app.yaml new file mode 100644 index 0000000..e2a626b --- /dev/null +++ b/tests/automation/endoscopy_tool_tracking_cpp/app.yaml @@ -0,0 +1,185 @@ +%YAML 1.2 +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# Required fields for packaging the application +# See https://docs.nvidia.com/holoscan/sdk-user-guide/cli/run_config.html# +application: + title: Holohub - Endoscopy Tool Tracking + version: 1.0 + inputFormats: [] + outputFormats: ["screen"] + +resources: + cpu: 2 + gpu: 1 + memory: 1Gi + gpuMemory: 1Gi + +extensions: + - libgxf_lstm_tensor_rt_inference.so + +source: "replayer" # "replayer" or "aja" or "deltacast" or "yuan" +visualizer: "holoviz" # "holoviz" or "vtk" +record_type: "none" # or "input" if you want to record input video stream, or "visualizer" if you want + # to record the visualizer output. + +replayer: + basename: "surgical_video" + frame_rate: 0 # as specified in timestamps + repeat: false # default: false + realtime: false # default: true + count: 0 # default: 0 (no frame count restriction) + +recorder_format_converter: + in_dtype: "rgba8888" + out_dtype: "rgb888" + +recorder: + directory: "/tmp" + basename: "tensor" + +format_converter_replayer: + out_tensor_name: source_video + out_dtype: "float32" + scale_min: 0.0 + scale_max: 255.0 + +format_converter_aja: + in_dtype: "rgba8888" + out_tensor_name: source_video + out_dtype: "float32" + scale_min: 0.0 + scale_max: 255.0 + resize_width: 854 + resize_height: 480 + +format_converter_yuan: + out_tensor_name: source_video + out_dtype: "float32" + scale_min: 0.0 + scale_max: 255.0 + resize_width: 854 + resize_height: 480 + +format_converter_deltacast: + in_dtype: "rgba8888" + out_tensor_name: source_video + out_dtype: "float32" + out_channel_order: [2,1,0] + scale_min: 0.0 + scale_max: 255.0 + resize_width: 854 + resize_height: 480 + +lstm_inference: + input_tensor_names: + - source_video + - cellstate_in + - hiddenstate_in + input_state_tensor_names: + - cellstate_in + - hiddenstate_in + input_binding_names: + - data_ph:0 # (shape=[1, 480, 854, 3], dtype=float32) <==> source_video + - cellstate_ph:0 # (shape=[1, 60, 107, 7], dtype=float32) == internal state + - hiddenstate_ph:0 # (shape=[1, 60, 107, 7], dtype=float32) == internal state + output_tensor_names: + - cellstate_out + - hiddenstate_out + - probs + - scaled_coords + - binary_masks + output_state_tensor_names: + - cellstate_out + - hiddenstate_out + output_binding_names: + - Model/net_states:0 # (shape=[ 1, 60, 107, 7], dtype=float32) + - Model/net_hidden:0 # (shape=[ 1, 60, 107, 7], dtype=float32) + - probs:0 # (shape=[1, 7], dtype=float32) + - Localize/scaled_coords:0 # (shape=[1, 7, 2], dtype=float32) + - Localize_1/binary_masks:0 # (shape=[1, 7, 60, 107], dtype=float32) + force_engine_update: false + verbose: true + max_workspace_size: 2147483648 + enable_fp16_: true + +holoviz: + tensors: + - name: "" + type: color + opacity: 1.0 + priority: 0 + - name: mask + type: color + opacity: 1.0 + priority: 1 + - name: scaled_coords + type: crosses + opacity: 1.0 + line_width: 4 + color: [1.0, 0.0, 0.0, 1.0] + priority: 2 + - name: scaled_coords + type: text + opacity: 1.0 + priority: 3 + color: [1.0, 1.0, 1.0, 0.9] + text: + - Grasper + - Bipolar + - Hook + - Scissors + - Clipper + - Irrigator + - Spec.Bag + headless: false + +holoviz_overlay: + headless: true + tensors: + - name: mask + type: color + opacity: 1.0 + priority: 1 + - name: scaled_coords + type: crosses + opacity: 1.0 + line_width: 4 + color: [1.0, 0.0, 0.0, 1.0] + priority: 2 + - name: scaled_coords + type: text + opacity: 1.0 + priority: 3 + color: [1.0, 1.0, 1.0, 0.9] + text: + - Grasper + - Bipolar + - Hook + - Scissors + - Clipper + - Irrigator + - Spec.Bag + +vtk_op: + labels: + - Grasper + - Bipolar + - Hook + - Scissors + - Clipper + - Irrigator + - Spec.Bag \ No newline at end of file diff --git a/tests/automation/endoscopy_tool_tracking_cpp/config.json b/tests/automation/endoscopy_tool_tracking_cpp/config.json new file mode 100644 index 0000000..19033ea --- /dev/null +++ b/tests/automation/endoscopy_tool_tracking_cpp/config.json @@ -0,0 +1,23 @@ +{ + "name": "endoscopy-tool-tracking-cpp", + "source": { + "repo": "https://github.com/nvidia-holoscan/holohub.git", + "app": "endoscopy_tool_tracking", + "lang": "cpp", + "path": "applications/endoscopy_tool_tracking/cpp" + }, + "config": { + "source": "local", + "path": "./app.yaml" + }, + "package": { + "args": "--includes onnx holoviz --add /install/lib" + }, + "run": { + "args": "--render" + }, + "data": { + "source" : "local-holohub", + "dirname" : "endoscopy" + } +} diff --git a/tests/automation/endoscopy_tool_tracking_python/app.yaml b/tests/automation/endoscopy_tool_tracking_python/app.yaml new file mode 100644 index 0000000..be0b5a7 --- /dev/null +++ b/tests/automation/endoscopy_tool_tracking_python/app.yaml @@ -0,0 +1,186 @@ +%YAML 1.2 +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# Required fields for packaging the application +# See https://docs.nvidia.com/holoscan/sdk-user-guide/cli/run_config.html# +application: + title: Holohub - Endoscopy Tool Tracking + version: 1.0 + inputFormats: [] + outputFormats: ["screen"] + +resources: + cpu: 2 + gpu: 1 + memory: 1Gi + gpuMemory: 1Gi + +extensions: + - libgxf_lstm_tensor_rt_inference.so + - libgxf_qcap_source.so + +source: "replayer" # "replayer" or "aja" or "deltacast" or "yuan" +visualizer: "holoviz" # "holoviz" or "vtk" +record_type: "none" # or "input" if you want to record input video stream, or "visualizer" if you want + # to record the visualizer output. + +replayer: + basename: "surgical_video" + frame_rate: 0 # as specified in timestamps + repeat: false # default: false + realtime: false # default: true + count: 0 # default: 0 (no frame count restriction) + +recorder_format_converter: + in_dtype: "rgba8888" + out_dtype: "rgb888" + +recorder: + directory: "/tmp" + basename: "tensor" + +format_converter_replayer: + out_tensor_name: source_video + out_dtype: "float32" + scale_min: 0.0 + scale_max: 255.0 + +format_converter_aja: + in_dtype: "rgba8888" + out_tensor_name: source_video + out_dtype: "float32" + scale_min: 0.0 + scale_max: 255.0 + resize_width: 854 + resize_height: 480 + +format_converter_yuan: + out_tensor_name: source_video + out_dtype: "float32" + scale_min: 0.0 + scale_max: 255.0 + resize_width: 854 + resize_height: 480 + +format_converter_deltacast: + in_dtype: "rgba8888" + out_tensor_name: source_video + out_dtype: "float32" + out_channel_order: [2,1,0] + scale_min: 0.0 + scale_max: 255.0 + resize_width: 854 + resize_height: 480 + +lstm_inference: + input_tensor_names: + - source_video + - cellstate_in + - hiddenstate_in + input_state_tensor_names: + - cellstate_in + - hiddenstate_in + input_binding_names: + - data_ph:0 # (shape=[1, 480, 854, 3], dtype=float32) <==> source_video + - cellstate_ph:0 # (shape=[1, 60, 107, 7], dtype=float32) == internal state + - hiddenstate_ph:0 # (shape=[1, 60, 107, 7], dtype=float32) == internal state + output_tensor_names: + - cellstate_out + - hiddenstate_out + - probs + - scaled_coords + - binary_masks + output_state_tensor_names: + - cellstate_out + - hiddenstate_out + output_binding_names: + - Model/net_states:0 # (shape=[ 1, 60, 107, 7], dtype=float32) + - Model/net_hidden:0 # (shape=[ 1, 60, 107, 7], dtype=float32) + - probs:0 # (shape=[1, 7], dtype=float32) + - Localize/scaled_coords:0 # (shape=[1, 7, 2], dtype=float32) + - Localize_1/binary_masks:0 # (shape=[1, 7, 60, 107], dtype=float32) + force_engine_update: false + verbose: true + max_workspace_size: 2147483648 + enable_fp16_: true + +holoviz: + tensors: + - name: "" + type: color + opacity: 1.0 + priority: 0 + - name: mask + type: color + opacity: 1.0 + priority: 1 + - name: scaled_coords + type: crosses + opacity: 1.0 + line_width: 4 + color: [1.0, 0.0, 0.0, 1.0] + priority: 2 + - name: scaled_coords + type: text + opacity: 1.0 + priority: 3 + color: [1.0, 1.0, 1.0, 0.9] + text: + - Grasper + - Bipolar + - Hook + - Scissors + - Clipper + - Irrigator + - Spec.Bag + headless: false + +holoviz_overlay: + headless: true + tensors: + - name: mask + type: color + opacity: 1.0 + priority: 1 + - name: scaled_coords + type: crosses + opacity: 1.0 + line_width: 4 + color: [1.0, 0.0, 0.0, 1.0] + priority: 2 + - name: scaled_coords + type: text + opacity: 1.0 + priority: 3 + color: [1.0, 1.0, 1.0, 0.9] + text: + - Grasper + - Bipolar + - Hook + - Scissors + - Clipper + - Irrigator + - Spec.Bag + +vtk_op: + labels: + - Grasper + - Bipolar + - Hook + - Scissors + - Clipper + - Irrigator + - Spec.Bag \ No newline at end of file diff --git a/tests/automation/endoscopy_tool_tracking_python/config.json b/tests/automation/endoscopy_tool_tracking_python/config.json new file mode 100644 index 0000000..156aa02 --- /dev/null +++ b/tests/automation/endoscopy_tool_tracking_python/config.json @@ -0,0 +1,23 @@ +{ + "name": "endoscopy-tool-tracking-cpp", + "source": { + "repo": "https://github.com/nvidia-holoscan/holohub.git", + "app": "endoscopy_tool_tracking", + "lang": "python", + "path": "applications/endoscopy_tool_tracking/python/endoscopy_tool_tracking.py" + }, + "config": { + "source": "local", + "path": "./app.yaml" + }, + "package": { + "args": "--includes onnx holoviz --add /install/lib --add /install/python/lib/" + }, + "run": { + "args": "--render" + }, + "data": { + "source" : "local-holohub", + "dirname" : "endoscopy" + } +} diff --git a/tests/automation/hello-world/app.yaml b/tests/automation/hello-world/app.yaml new file mode 100644 index 0000000..902a9c6 --- /dev/null +++ b/tests/automation/hello-world/app.yaml @@ -0,0 +1,27 @@ +%YAML 1.2 +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +application: + title: Holoscan - Hello World + version: 1.0 + inputFormats: [] + outputFormats: ["screen"] + +resources: + cpu: 1 + gpu: 1 + memory: 1Gi + gpuMemory: 1Gi \ No newline at end of file diff --git a/tests/automation/hello-world/config.json b/tests/automation/hello-world/config.json new file mode 100644 index 0000000..de34ef9 --- /dev/null +++ b/tests/automation/hello-world/config.json @@ -0,0 +1,12 @@ +{ + "name": "hello-world", + "source": { + "repo": "https://github.com/nvidia-holoscan/holoscan-sdk.git", + "path": "examples/hello_world/python/hello_world.py", + "lang": "python" + }, + "config": { + "source": "local", + "path": "./app.yaml" + } +} \ No newline at end of file diff --git a/tests/automation/object_detection_torch/app.yaml b/tests/automation/object_detection_torch/app.yaml new file mode 100644 index 0000000..87b675c --- /dev/null +++ b/tests/automation/object_detection_torch/app.yaml @@ -0,0 +1,80 @@ +%YAML 1.2 +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# Required fields for packaging the application +# See https://docs.nvidia.com/holoscan/sdk-user-guide/cli/run_config.html# +application: + title: Holohub - Object Detection Torch + version: 1.0 + inputFormats: [] + outputFormats: ["screen"] + +resources: + cpu: 2 + gpu: 1 + memory: 1Gi + gpuMemory: 2Gi + +extensions: + +source: "replayer" # or "aja" +record_type: "none" # or "visualizer" if you want to record the visualizer output. + +replayer: + basename: "cars" + frame_rate: 0 # as specified in timestamps + repeat: false # default: false + realtime: false # default: true + count: 10 # default: 0 (no frame count restriction) + +aja: # AJASourceOp + width: 1920 + height: 1080 + rdma: true + enable_overlay: false + +detect_preprocessor: + out_tensor_name: detect_preprocessed + out_dtype: "float32" + scale_min: 0.0 + scale_max: 1.0 + +detect_inference: + backend: "torch" + pre_processor_map: + "detect": ["detect_preprocessed"] + inference_map: + "detect": ["boxes","labels","scores"] + +detect_postprocessor: + process_operations: + "boxes:scores:labels": ["generate_boxes"] + in_tensor_names: ["boxes","scores","labels"] + input_on_cuda: false + output_on_cuda: false + transmit_on_cuda: true + +holoviz: # Holoviz + width: 1920 + height: 1080 + +recorder_format_converter: + in_dtype: "rgba8888" + out_dtype: "rgb888" + +recorder: + directory: "/tmp" + basename: "tensor" diff --git a/tests/automation/object_detection_torch/config.json b/tests/automation/object_detection_torch/config.json new file mode 100644 index 0000000..980016a --- /dev/null +++ b/tests/automation/object_detection_torch/config.json @@ -0,0 +1,23 @@ +{ + "name": "object_detection_torch", + "source": { + "repo": "https://github.com/nvidia-holoscan/holohub.git", + "app": "object_detection_torch", + "lang": "cpp", + "path": "applications/object_detection_torch" + }, + "config": { + "source": "local", + "path": "./app.yaml" + }, + "package": { + "args": "--includes onnx holoviz torch" + }, + "run": { + "args": "--render" + }, + "data": { + "source" : "local-holohub", + "dirname" : "object_detection_torch" + } +} diff --git a/tests/automation/test.sh b/tests/automation/test.sh new file mode 100755 index 0000000..4f75bf3 --- /dev/null +++ b/tests/automation/test.sh @@ -0,0 +1,484 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License.run_command +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#=============================================================================== +set -u +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd) + +if ! ./check.sh +then + exit 1 +fi + +version= +test=$(realpath $1) +repository="" +path="" +language="" + + +#=============================================================================== +# Logging utils +#=============================================================================== + +c_str() { + local old_color=39 + local old_attr=0 + local color=39 + local attr=0 + local text="" + local mode="color" + if [ "${1:-}" = "color" ]; then + mode="color" + shift + elif [ "${1:-}" = "nocolor" ]; then + mode="nocolor" + shift + fi + + for i in "$@"; do + case "$i" in + r|R) + color=31 + ;; + g|G) + color=32 + ;; + y|Y) + color=33 + ;; + b|B) + color=34 + ;; + p|P) + color=35 + ;; + c|C) + color=36 + ;; + w|W) + color=37 + ;; + + z|Z) + color=0 + ;; + esac + case "$i" in + l|L|R|G|Y|B|P|C|W) + attr=1 + ;; + n|N|r|g|y|b|p|c|w) + attr=0 + ;; + z|Z) + attr=0 + ;; + *) + text="${text}$i" + esac + if [ "${mode}" = "color" ]; then + if [ ${old_color} -ne ${color} ] || [ ${old_attr} -ne ${attr} ]; then + text="${text}\033[${attr};${color}m" + old_color=$color + old_attr=$attr + fi + fi + done + /bin/echo -en "$text" +} + +c_echo() { + # Select color/nocolor based on the first argument + local mode="color" + if [ "${1:-}" = "color" ]; then + mode="color" + shift + elif [ "${1:-}" = "nocolor" ]; then + mode="nocolor" + shift + else + if [ ! -t 1 ]; then + mode="nocolor" + fi + fi + + local old_opt="$(shopt -op xtrace)" # save old xtrace option + set +x # unset xtrace + + if [ "${mode}" = "color" ]; then + local text="$(c_str color "$@")" + /bin/echo -e "$text\033[0m" + else + local text="$(c_str nocolor "$@")" + /bin/echo -e "$text" + fi + eval "${old_opt}" # restore old xtrace option +} + + +cecho() { + >&2 c_echo "$@" +} + +info() { + cecho B "$(date -u '+%Y-%m-%d %H:%M:%S') [INFO] " Z "$@" +} + +error() { + cecho R "$(date -u '+%Y-%m-%d %H:%M:%S') [ERROR] " Z "$@" +} + +fatal() { + if [ -n "$*" ]; then + cecho R "$(date -u '+%Y-%m-%d %H:%M:%S') [FATAL] " Z "$@" + echo_err + fi + if [ -z "${CALLER-}" ]; then + return 1 + else + kill -INT $$ # kill the current process instead of exit in shell environment. + fi +} +#=============================================================================== +# Executes a specified command via bash shell +# Parameters +# $@ accept any number of parameters +run_command() { + local status=0 + local cmd="$*" + + info "[command] ${cmd}" + + [ "$(echo -n "$@")" = "" ] && return 1 # return 1 if there is no command available + + "$@" + status=$? + info "[command] exited with status: ${status}" + return $status +} + +# Parse the specified JSON Path from the configuration file +# Returns empty string if JSON path does not exist. +get_config() { + value=$(jq -r "$1" $test) + if [[ "$value" == "null" ]] + then + value="" + fi + echo $value +} + +# Run the HAP container using Holoscan CLI Runner +# Parameters +# $1 Input Data Directory Path +# $2 Container Tag Prefix +run_application() { + info "===== Run Application =====" + local data_dir=$1 + info "Starting application with data directory: $data_dir" + run_args=$(get_config '.run.args') + tag_prefix=$2 + local tag=$(docker images | grep "$tag_prefix" | awk '{print $1":"$2}' | head -n 1) + info "Running application" + info " Tag: $tag" + run_command xhost +local:docker + run_command holoscan run -l DEBUG $run_args $tag --input $data_dir + + if [[ $? -ne 0 ]] + then + error "Failed to run application" + exit 1 + fi +} + +# Download user defined data. +# NOTE: Only NGC is supported at the moment: the script parses the JSON returned by the NGC files API. +# E.g. https://api.ngc.nvidia.com/v2/resources/org/nvidia/team/clara-holoscan/holoscan_racerx_video/20231009/files +# Parameters +# $1 Working directory path +# $2 Path to save the data to +download_data() { + info "===== Download Data =====" + local source=$(get_config '.data.source') + local target=$(get_config '.data.dirname') + + if [[ "$source" != "local-holohub" ]] + then + local tmp_dir=$1 + local data_dir=$2 + if [ -v target ] + then + data_dir="$data_dir/$target" + fi + [[ ! -d "$data_dir" ]] && run_command mkdir -p $data_dir + + for url in $(curl $source | jq -r .urls.[]) + do + info "Downloading $url" + run_command curl -S -# -LO --output-dir "$data_dir" "$url" + if [[ $? -ne 0 ]] + then + error "Failed to download test data" + exit 1 + fi + done + tree $data_dir + fi +} + +# Get type of GPU running on the system +get_host_gpu() { + if ! command -v nvidia-smi >/dev/null; then + error Y "Could not find any GPU drivers on host. Defaulting build to target dGPU/CPU stack." + echo -n "dgpu" + elif nvidia-smi 2>/dev/null | grep nvgpu -q; then + echo -n "igpu" + else + echo -n "dgpu" + fi +} + +# Build Holohub applications using Holohub's devcontainer script. +# Since the devcontainer script embeds the version of the HSDK container to use, we must explicitly set the base image based on the version of the running CLI. +# Parameters: +# $1 Working Directory +# $2 Application Name +# $3 Application Language +build_holohub_app() { + info "===== Build Holohub Application =====" + pushd $1 + + local platform_config=$(get_host_gpu) + local build_image=$(jq -r --arg HSDKVERSION $version --arg PFC $platform_config '.[$HSDKVERSION].holoscan."build-images".[$PFC]."x64-workstation"' "$SCRIPT_DIR/artifacts.json") + info "Using base image $build_image" + run_command ./dev_container build_and_install $2 --base_img $build_image + if [[ $? -ne 0 ]] + then + error "Failed to build Holohub application: $2" + exit 1 + fi + tree + popd +} + +# Package the application using Holoscan CLI Packager +# Parameters: +# $1 Working Directory +# $2 Source Code Directory +# $3 Relative Application Path +# $4 Container Tag Prefix +package() { + info "===== Package Application =====" + local run_args=$(get_config '.package.args') + run_args=$(echo ${run_args///$2}) + local config_source=$(get_config '.config.source') + local config_file_path=$(get_config '.config.path') + local dir= + local app_dir= + + if [[ "$run_args" == "null" ]] + then + run_args="" + fi + + dir=$(dirname $test) + if [ "$config_source" == "local" ] + then + config_file_path="$dir/$config_file_path" + else + config_file_path="$2/$config_file_path" + fi + + app_dir="$2/$3" + pushd $1 + info "Packaging application from $app_dir" + info " App config: $config_file_path" + run_command holoscan package -l DEBUG \ + --config $config_file_path \ + --tag $4 \ + --platform x64-workstation \ + $app_dir \ + $run_args + + if [[ $? -ne 0 ]] + then + error "Failed to package application" + exit 1 + fi + popd +} + +# Clone the user-defined repository. +# For Holohub, clone entire repository. Otherwise, do a sparse clone of the application directory. +# Parameters +# $1 Directory to clone to +# $2 Git Repository +# $3 Relative Application Path +clone() { + info "===== Clone Repository =====" + info "Cloning repository $2 to $1" + info "Dir: $1" + info "Repository: $2" + info "Path: $3" + mkdir -p $1 + pushd $1 + + local path=$3 + if [ -f $3 ] + then + path=$(basename $3) + fi + + if [[ "$2" =~ "holohub" ]] + then + run_command git clone --depth 1 $2 . + else + run_command git clone --filter=blob:none --no-checkout --depth 1 --sparse $2 . + run_command git sparse-checkout add "$path" + run_command git checkout + fi + run_command tree + popd +} + +# For Holoscan SDK example C++ applications, rename CMakeLists.min.txt to CMakeLists.txt if exists. +# This enables the Packager process to build the C++ application. +# Parameters +# $1 Source Code Directory Path +# $2 Relative Path to the Application +prep_cpp_dir() { + info "Preparing C++ source directory for packaging" + info " Source Dir: $1" + info " Path: $2" + pushd $1 + + local path="$1/$2" + info "Searching $path for CMakeLists.min.txt" + if [[ $(find $path -type f -name "CMakeLists.min.txt" | wc -l) -eq 1 ]] + then + info "Overwriting CMakeLists.txt with CMakeLists.min.txt" + run_command mv -f $path/CMakeLists.min.txt $path/CMakeLists.txt + fi + popd +} + +# Parameters +# Value +# Value query path +check_field() { + if [ -z "$1" ] + then + info "$test test configuration is missing the '$2' field" + exit 1 + fi +} + +# Entrypoint of the scriptS +main() { + local repository=$(get_config '.source.repo') + local path=$(get_config '.source.path') + local language=$(get_config '.source.lang') + local tmp_dir=$(mktemp -d) + local source_dir="$tmp_dir/src" + local data_dir="$tmp_dir/data" + local app_name=$(get_config '.source.app') + local package_path=$path + local tag="$(basename $(dirname $test))-$((1 + $RANDOM % 1000))" + + if [[ "$repository" =~ "holohub" ]] + then + path=${path//applications\//} + info "Application: $app_name" + fi + info "Repository: $repository" + info "Path: $path" + info "Language: $language" + info "Working Dir: $tmp_dir" + info "Source Dir: $source_dir" + info "Data Dir: $data_dir" + info "Tag Prefix: $tag" + + check_field $repository ".source.repo" + check_field $path ".source.path" + check_field $language ".source.lang" + + # Clone configured git repository + clone "$source_dir" "$repository" "$path" + + # For non-Holohub repositories and C++ projects + if [[ "$language" == "cpp" && "$repository" != *"holohub"* ]] + then + prep_cpp_dir "$source_dir" "$path" + fi + + # For Holohub C++ applications, append the application binary name to the package path + if [[ "$repository" =~ "holohub" && "$language" == "cpp" ]] + then + package_path="install/bin/$path/$app_name" + fi + + # For Holohub applications, call the build script + if [[ "$repository" =~ "holohub" ]] + then + build_holohub_app "$source_dir" "$app_name" "$language" + fi + + # Call Holoscan CLI Packager + package "$tmp_dir" "$source_dir" "$package_path" "$tag" + + # Download specified data + download_data "$tmp_dir" "$data_dir" + + # Use test data downloaded by Holohub build script + if [[ "$repository" =~ "holohub" && "$(get_config '.data.source')" == "local-holohub" ]] + then + data_dir="$source_dir/data/$(get_config '.data.dirname')" + tree $data_dir + fi + + # Call Holoscan CLI Runner + run_application "$data_dir" "$tag" + + # Clean up the temporary directory + info "Cleaning source code..." + run_command rm -rf $tmp_dir + info "Deleting Containerized Application..." + run_command docker rmi $(docker images --filter=reference="${tag}*:*" -q) +} + +if [ -z "$test" ] +then + info "Please provide the test configuration path" + exit 1 +fi + +if [ -d "$test" ] +then + info "Looking for test configuration in $test" + test="$test/config.json" + if [ ! -f "$test" ] + then + info "No configuration found, expecting $test" + exit 1 + fi +fi + +version=$(holoscan version | tail -n 1 | awk '{print $3}') +info "Using test configuration $test with Holoscan CLI v${version}" + +main + diff --git a/tests/automation/video-replayer-cpp/app.yaml b/tests/automation/video-replayer-cpp/app.yaml new file mode 100644 index 0000000..949eea5 --- /dev/null +++ b/tests/automation/video-replayer-cpp/app.yaml @@ -0,0 +1,60 @@ +%YAML 1.2 +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +application: + title: Video Replayer (C++) + version: 1.0 + inputFormats: [] + outputFormats: ["screen"] + +resources: + cpu: 1 + gpu: 1 + memory: 1Gi + gpuMemory: 1Gi + +dual_window: false + +replayer: + directory: "../data/racerx" + basename: "racerx" + frame_rate: 0 # as specified in timestamps + repeat: false # default: false + realtime: false # default: true + count: 0 # default: 0 (no frame count restriction) + +# Initial size below is set to 8 MB which is sufficient for +# a 1920 * 1080 RGBA image (uint8_t). +rmm_allocator: + device_memory_initial_size: "8 MB" + device_memory_max_size: "8 MB" + host_memory_initial_size: "8 MB" + host_memory_max_size: "8 MB" + dev_id: 0 + +holoviz: + width: 854 + height: 480 + tensors: + - name: "" + type: color + opacity: 1.0 + priority: 0 + +event_based_scheduler: + worker_thread_number: 3 + stop_on_deadlock: true + stop_on_deadlock_timeout: 200 \ No newline at end of file diff --git a/tests/automation/video-replayer-cpp/config.json b/tests/automation/video-replayer-cpp/config.json new file mode 100644 index 0000000..dc5f6a8 --- /dev/null +++ b/tests/automation/video-replayer-cpp/config.json @@ -0,0 +1,22 @@ +{ + "name": "video-replayer-cpp", + "source": { + "repo": "https://github.com/nvidia-holoscan/holoscan-sdk.git", + "path": "examples/video_replayer/cpp", + "lang": "cpp" + }, + "config": { + "source": "local", + "path": "./app.yaml" + }, + "package": { + "args": "--includes holoviz" + }, + "run": { + "args": "--render" + }, + "data": { + "dirname" : "racerx", + "source" : "https://api.ngc.nvidia.com/v2/resources/org/nvidia/team/clara-holoscan/holoscan_racerx_video/20231009/files" + } +} diff --git a/tests/automation/video-replayer-python/app.yaml b/tests/automation/video-replayer-python/app.yaml new file mode 100644 index 0000000..7d7ff7d --- /dev/null +++ b/tests/automation/video-replayer-python/app.yaml @@ -0,0 +1,59 @@ +%YAML 1.2 +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +application: + title: Video Replayer (Python) + version: 1.0 + inputFormats: [] + outputFormats: ["screen"] + +resources: + cpu: 1 + gpu: 1 + memory: 1Gi + gpuMemory: 1Gi + +dual_window: false + +replayer: + basename: "racerx" + frame_rate: 0 # as specified in timestamps + repeat: false # default: false + realtime: false # default: true + count: 0 # default: 0 (no frame count restriction) + +# Initial size below is set to 8 MB which is sufficient for +# a 1920 * 1080 RGBA image (uint8_t). +rmm_allocator: + device_memory_initial_size: "8 MB" + device_memory_max_size: "8 MB" + host_memory_initial_size: "8 MB" + host_memory_max_size: "8 MB" + dev_id: 0 + +holoviz: + width: 854 + height: 480 + tensors: + - name: "" + type: color + opacity: 1.0 + priority: 0 + +event_based_scheduler: + worker_thread_number: 3 + stop_on_deadlock: true + stop_on_deadlock_timeout: 200 \ No newline at end of file diff --git a/tests/automation/video-replayer-python/config.json b/tests/automation/video-replayer-python/config.json new file mode 100644 index 0000000..ff8e8b1 --- /dev/null +++ b/tests/automation/video-replayer-python/config.json @@ -0,0 +1,22 @@ +{ + "name": "video-replayer-python", + "source": { + "repo": "https://github.com/nvidia-holoscan/holoscan-sdk.git", + "path": "examples/video_replayer/python/video_replayer.py", + "lang": "python" + }, + "config": { + "source": "local", + "path": "./app.yaml" + }, + "package": { + "args": "--includes holoviz" + }, + "run": { + "args": "--render" + }, + "data": { + "dirname" : "racerx", + "source" : "https://api.ngc.nvidia.com/v2/resources/org/nvidia/team/clara-holoscan/holoscan_racerx_video/20231009/files" + } +}