From 554c03c2a40355eb8ca483a5de7f90ddd8953de0 Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Thu, 23 Jan 2025 14:44:17 -0800 Subject: [PATCH] Add README and tidy up Signed-off-by: Victor Chang --- tests/automation/README.md | 136 +++++++++++++++++++++++ tests/automation/check.sh | 25 ++--- tests/automation/config.schema.json | 23 ---- tests/automation/{package.sh => test.sh} | 72 +++++++++--- 4 files changed, 205 insertions(+), 51 deletions(-) create mode 100644 tests/automation/README.md delete mode 100644 tests/automation/config.schema.json rename tests/automation/{package.sh => test.sh} (81%) diff --git a/tests/automation/README.md b/tests/automation/README.md new file mode 100644 index 0000000..e2477c6 --- /dev/null +++ b/tests/automation/README.md @@ -0,0 +1,136 @@ +# Holoscan CLI Test Automation + +This directory contains scripts to automate the testing of packaging and running applications designed using [Holoscan SDK](https://developer.nvidia.com/holoscan-sdk). + +## Requirements + +To successfully execute the test automation script, the system must meet the requirements specified by both the [Holoscan SDK](https://docs.nvidia.com/holoscan/sdk-user-guide/sdk_installation.html) and the [Holoscan CLI](https://github.com/nvidia-holoscan/holoscan-cli). Additionally, the following tools are required to package and run the application: + +* Holoscan CLI +* jq +* curl +* An [NGC](https://catalog.ngc.nvidia.com/) account, which must be logged in via `docker login nvcr.io` + +## Usage + +To run the test automation script, navigate to the automation directory, then execute the script with the application directory as an argument: + +``` +cd tests/automation +./test.sh [application-directory] +# for example +./tests.sh ./endoscopy_tool_tracking_cpp +``` + +The `test.sh` script invokes `check.sh` to verify that all system requirements are met. It will exit if any of the requirements are missing. + +The automation script performs the following steps: + +1. Clones the specified git repository. +2. Packages the application. +3. Downloads the configured test data. +4. Run the packaged application. +5. Cleans up any downloaded and generated data. + +### Adding New Application + +Automating the testing of a new Holoscan-enabled application is straightforward and can be accomplished with the following steps: + +1. Create a new directory and include a `config.json` file that defines the application and specifies where the script can retrieve the test data. Refer to the sections below for examples and the configuration schema. +2. Optionally, copy the application's configuration YAML file and modify the values to ensure that the application can start and exit on its own. + +This is all that's required. + +### Sample Configuration + +#### Holoscan Application + +Here’s a sample configuration for testing the C++ version of the Video Replayer: + +With this configuration, the test automation script will clone the [Holoscan SDK repository](https://github.com/nvidia-holoscan/holoscan-sdk) from GitHub, download test data from NGC, package the application with `--include holoviz`, and run the application using the supplied `app.yaml` configuration file along with the `--render` option. + +```json +{ + "name": "video-replayer-cpp", + "source": { + "repo": "https://github.com/nvidia-holoscan/holoscan-sdk.git", + "path": "examples/video_replayer/cpp", + "lang": "cpp" + }, + "config": { + "source": "local", + "path": "./app.yaml" + }, + "package": { + "args": "--includes holoviz" + }, + "run": { + "args": "--render" + }, + "data": { + "dirname" : "racerx", + "source" : "https://api.ngc.nvidia.com/v2/resources/org/nvidia/team/clara-holoscan/holoscan_racerx_video/20231009/files" + } +} +``` + +#### Holohub Application + +Another sample configuration is used for testing the Endoscopy Tool Tracking application from Holohub: + +In this case, the test automation script will clone the [Holohub](https://github.com/nvidia-holoscan/holohub) repository, invoke the `devcontainer` script to build the specified application, utilize test data downloaded by the `devcontainer` script, and package the application using the provided `app.yaml` file with additional arguments defined in `package.args`. Finally, it executes the application with the arguments specified in `run.args`. + + +```json +{ + "name": "endoscopy-tool-tracking-cpp", + "source": { + "repo": "https://github.com/nvidia-holoscan/holohub.git", + "app": "endoscopy_tool_tracking", + "lang": "python", + "path": "applications/endoscopy_tool_tracking/python/endoscopy_tool_tracking.py" + }, + "config": { + "source": "local", + "path": "./app.yaml" + }, + "package": { + "args": "--includes onnx holoviz --add /install/lib --add /install/python/lib/" + }, + "run": { + "args": "--render" + }, + "data": { + "source" : "local-holohub", + "dirname" : "endoscopy" + } +} +``` + +### Configuration Schema + +``` +{ + "name": "", # Required + "source": { + "repo": ".git", # Required + "path": "", # Required + "app": "Holohub only: name of application. Same as the application name used by the devcontainer script." + }, + "config": { + "source": "[local]", # optional value + "path": "./app.yaml" # Required + }, + "package": { + "args": "" + }, + "run": { + "args": "" + }, + "data": { + "source" : "", # 'local-holohub' means to use test data downloaded by the devcontainer script + "dirname" : "" + } +} +``` \ No newline at end of file diff --git a/tests/automation/check.sh b/tests/automation/check.sh index 736d947..75de22d 100755 --- a/tests/automation/check.sh +++ b/tests/automation/check.sh @@ -17,37 +17,34 @@ #=============================================================================== echo "Checking system requirements..." -if ! command -v unzip 2>&1 >/dev/null -then - echo "unzip could not be found, installing unzip" - echo "$ sudo apt install unzip" - exit -1 -fi +status=0 if ! command -v curl 2>&1 >/dev/null then - echo "jq could not be found, installing curl" + echo "jq could not be found, use the following command to install curl" echo "$ sudo apt install curl" - exit -1 + status=-1 fi if ! command -v jq 2>&1 >/dev/null then - echo "jq could not be found, installing jq" + echo "jq could not be found, use the following command to install jq" echo "$ sudo apt install jq" - exit -1 + status=-1 fi if ! command -v holoscan 2>&1 >/dev/null then - echo "Holoscan CLI could not be found, installing holoscan" + echo "Holoscan CLI could not be found, use the following command to install holoscan" echo "$ pip install holoscan-cli" - exit -1 + status=-1 fi if ! docker login nvcr.io < /dev/null >& /dev/null then echo "Please login to nvcr.io. For example:" echo "$ docker login nvcr.io" - exit -1 -fi \ No newline at end of file + status=-1 +fi + +exit $status \ No newline at end of file diff --git a/tests/automation/config.schema.json b/tests/automation/config.schema.json deleted file mode 100644 index 07eae40..0000000 --- a/tests/automation/config.schema.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "name": "", - "source": { - "repo": ".git", - "path": "", - "app": "holohub only: name of application" - }, - "config": { - "source": "[local]", - "path": "./app.yaml" - }, - "package": { - "args": "" - }, - "run": { - "args": "" - }, - "data": { - "source" : "", - "dirname" : "/dev/null; then error Y "Could not find any GPU drivers on host. Defaulting build to target dGPU/CPU stack." @@ -238,10 +252,12 @@ get_host_gpu() { fi } +# Build Holohub applications using Holohub's devcontainer script. +# Since the devcontainer script embeds the version of the HSDK container to use, we must explicitly set the base image based on the version of the running CLI. # Parameters: -# Working Directory -# Application Name -# Application Language +# $1 Working Directory +# $2 Application Name +# $3 Application Language build_holohub_app() { info "===== Build Holohub Application =====" pushd $1 @@ -259,10 +275,12 @@ build_holohub_app() { popd } +# Package the application using Holoscan CLI Packager # Parameters: -# Working Directory -# Source Code Directory -# Relative Application Path +# $1 Working Directory +# $2 Source Code Directory +# $3 Relative Application Path +# $4 Container Tag Prefix package() { info "===== Package Application =====" local run_args=$(get_config '.package.args') @@ -291,7 +309,7 @@ package() { info " App config: $config_file_path" run_command holoscan package -l DEBUG \ --config $config_file_path \ - --tag $(basename $dir) \ + --tag $4 \ --platform x64-workstation \ --source /home/vicchang/sc/github/holoscan-cli/tests/automation/artifacts.json \ --holoscan-sdk-file /home/vicchang/sc/github/holoscan-cli/dist/holoscan-2.9.0.1-cp310-cp310-manylinux_2_35_x86_64.whl \ @@ -308,6 +326,12 @@ package() { popd } +# Clone the user-defined repository. +# For Holohub, clone entire repository. Otherwise, do a sparse clone of the application directory. +# Parameters +# $1 Directory to clone to +# $2 Git Repository +# $3 Relative Application Path clone() { info "===== Clone Repository =====" info "Cloning repository $2 to $1" @@ -335,6 +359,11 @@ clone() { popd } +# For non-Holohub C++ applications, rename CMakeLists.min.txt to CMakeLists.txt if exists. +# This enables the Packager process to build the C++ application. +# Parameters +# $1 Source Code Directory Path +# $2 Relative Path to the Application prep_cpp_dir() { info "Preparing C++ source directory for packaging" info " Source Dir: $1" @@ -362,6 +391,7 @@ check_field() { fi } +# Entrypoint of the scriptS main() { local repository=$(get_config '.source.repo') local path=$(get_config '.source.path') @@ -371,6 +401,7 @@ main() { local data_dir="$tmp_dir/data" local app_name=$(get_config '.source.app') local package_path=$path + local tag="$(basename $(dirname $test))-$((1 + $RANDOM % 1000))" if [[ "$repository" =~ "holohub" ]] then @@ -383,42 +414,54 @@ main() { info "Working Dir: $tmp_dir" info "Source Dir: $source_dir" info "Data Dir: $data_dir" + info "Tag Prefix: $tag" check_field $repository ".source.repo" check_field $path ".source.path" check_field $language ".source.lang" + # Clone configured git repository clone "$source_dir" "$repository" "$path" + # For non-Holohub repositories and C++ projects if [[ "$language" == "cpp" && "$repository" != *"holohub"* ]] then prep_cpp_dir "$source_dir" "$path" fi + # For Holohub C++ applications, append the application binary name to the package path if [[ "$repository" =~ "holohub" && "$language" == "cpp" ]] then package_path="install/bin/$path/$app_name" fi + # For Holohub applications, call the build script if [[ "$repository" =~ "holohub" ]] then build_holohub_app "$source_dir" "$app_name" "$language" fi - package "$tmp_dir" "$source_dir" "$package_path" + # Call Holoscan CLI Packager + package "$tmp_dir" "$source_dir" "$package_path" "$tag" + # Download specified data download_data "$tmp_dir" "$data_dir" + # Use test data downloaded by Holohub build script if [[ "$repository" =~ "holohub" && "$(get_config '.data.source')" == "local-holohub" ]] then data_dir="$source_dir/data/$(get_config '.data.dirname')" tree $data_dir fi - run "$data_dir" + # Call Holoscan CLI Runner + run_application "$data_dir" "$tag" + # Clean up the temporary directory info "Cleaning source code..." run_command rm -rf $tmp_dir + info "Deleting Containerized Application..." + run_command docker rmi $(docker images --filter=reference="${tag}*:*" -q) } if [ -z "$test" ] @@ -442,3 +485,4 @@ version=$(holoscan version | tail -n 1 | awk '{print $3}') info "Using test configuration $test with Holoscan CLI v${version}" main +