From 6454e9b271191c752535e0eca9c4792180810eee Mon Sep 17 00:00:00 2001 From: HailoRT-Automation Date: Wed, 31 Jul 2024 13:29:04 +0300 Subject: [PATCH] v3.29.1 --- README.rst | 18 ++- .../detection/yolo_hailortpp.cpp | 125 +++++++++++++++++- .../detection/yolo_hailortpp.hpp | 22 ++- core/hailo/libs/postprocesses/meson.build | 2 +- .../semantic_segmentation.cpp | 20 ++- core/hailo/meson.build | 2 +- docs/installation/raspberry-pi-install.rst | 2 - install.sh | 15 ++- .../pkg_config/hailo-tappas-core.pc_template | 49 +++++++ .../misc/pkg_config/hailo_tappas.pc_template | 8 +- scripts/misc/pkg_config_setup.sh | 13 +- 11 files changed, 246 insertions(+), 30 deletions(-) create mode 100644 scripts/misc/pkg_config/hailo-tappas-core.pc_template diff --git a/README.rst b/README.rst index e51b4517..a689ad72 100644 --- a/README.rst +++ b/README.rst @@ -92,9 +92,12 @@ Installation * - Yocto installation - `Read more about Yocto installation `_ - Yocto supported BSP's - * - Raspberry pi installation - - `Read more about Raspberry pi installation `_ - - Raspberry Pi OS, Ubuntu 20.04 + * - Raspberry Pi 4 installation + - `Read more about Raspberry Pi 4 installation `_ + - Raspberry Pi OS + * - Raspberry Pi 5 installation + - `Read more about Raspberry Pi 5 installation `_ + - Raspberry Pi OS @@ -351,6 +354,15 @@ The pipeline demonstrates another method for inference based decision making tha Changelog ---------- +**v3.29.1 (Aug 2024)** + +* Hailo-8: + + * Updated infrastructure to better support Raspberry Pi 5 + * Added an option to control Yolo (Detection) Hailort post-process parameters via a JSON configuration + * Semantic segmentation post-process now extracts the argmax tensor using Regular Expressions + + **v3.29.0 (July 2024)** * Hailo-15: diff --git a/core/hailo/libs/postprocesses/detection/yolo_hailortpp.cpp b/core/hailo/libs/postprocesses/detection/yolo_hailortpp.cpp index dc61e6ff..f752a3ea 100644 --- a/core/hailo/libs/postprocesses/detection/yolo_hailortpp.cpp +++ b/core/hailo/libs/postprocesses/detection/yolo_hailortpp.cpp @@ -1,8 +1,17 @@ #include -#include "hailo_nms_decode.hpp" -#include "yolo_hailortpp.hpp" +#include +#include +#include +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/error/en.h" +#include "rapidjson/filereadstream.h" +#include "rapidjson/schema.h" +#include "json_config.hpp" #include "common/labels/coco_eighty.hpp" #include "common/labels/yolo_personface.hpp" +#include "hailo_nms_decode.hpp" +#include "yolo_hailortpp.hpp" static const std::string DEFAULT_YOLOV5S_OUTPUT_LAYER = "yolov5s_nv12/yolov5_nms_postprocess"; static const std::string DEFAULT_YOLOV5M_OUTPUT_LAYER = "yolov5m_wo_spp_60p/yolov5_nms_postprocess"; @@ -10,6 +19,90 @@ static const std::string DEFAULT_YOLOV5M_VEHICLES_OUTPUT_LAYER = "yolov5m_vehicl static const std::string DEFAULT_YOLOV8S_OUTPUT_LAYER = "yolov8s/yolov8_nms_postprocess"; static const std::string DEFAULT_YOLOV8M_OUTPUT_LAYER = "yolov8m/yolov8_nms_postprocess"; +#if __GNUC__ > 8 +#include +namespace fs = std::filesystem; +#else +#include +namespace fs = std::experimental::filesystem; +#endif + +YoloParamsNMS *init(const std::string config_path, const std::string function_name) +{ + YoloParamsNMS *params; + if (!fs::exists(config_path)) + { + params = new YoloParamsNMS(common::coco_eighty); + return params; + } + else + { + params = new YoloParamsNMS(); + char config_buffer[4096]; + const char *json_schema = R""""({ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "detection_threshold": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "max_boxes": { + "type": "integer" + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "labels" + ] + })""""; + + std::FILE *fp = fopen(config_path.c_str(), "r"); + if (fp == nullptr) + { + throw std::runtime_error("JSON config file is not valid"); + } + rapidjson::FileReadStream stream(fp, config_buffer, sizeof(config_buffer)); + bool valid = common::validate_json_with_schema(stream, json_schema); + if (valid) + { + rapidjson::Document doc_config_json; + doc_config_json.ParseStream(stream); + + // parse labels + auto labels = doc_config_json["labels"].GetArray(); + uint i = 0; + for (auto &v : labels) + { + params->labels.insert(std::pair(i, v.GetString())); + i++; + } + + // set the params + if (doc_config_json.HasMember("detection_threshold")) { + params->detection_threshold = doc_config_json["detection_threshold"].GetFloat(); + } + if (doc_config_json.HasMember("max_boxes")) { + params->max_boxes = doc_config_json["max_boxes"].GetInt(); + params->filter_by_score = true; + } + } + fclose(fp); + } + return params; +} +void free_resources(void *params_void_ptr) +{ + YoloParamsNMS *params = reinterpret_cast(params_void_ptr); + delete params; +} + static std::map yolo_vehicles_labels = { {0, "unlabeled"}, {1, "car"}}; @@ -123,22 +216,44 @@ void yolov5_no_persons(HailoROIPtr roi) } hailo_common::add_detections(roi, detections); } - -void filter(HailoROIPtr roi) +void filter(HailoROIPtr roi, void *params_void_ptr) { if (!roi->has_tensors()) { return; } + YoloParamsNMS *params = reinterpret_cast(params_void_ptr); std::vector tensors = roi->get_tensors(); // find the nms tensor for (auto tensor : tensors) { if (std::regex_search(tensor->name(), std::regex("nms_postprocess"))) { - auto post = HailoNMSDecode(tensor, common::coco_eighty); + auto post = HailoNMSDecode(tensor, params->labels, params->detection_threshold, params->max_boxes, params->filter_by_score); auto detections = post.decode(); hailo_common::add_detections(roi, detections); } } } +void filter_letterbox(HailoROIPtr roi, void *params_void_ptr) +{ + filter(roi, params_void_ptr); + // Resize Letterbox + HailoBBox roi_bbox = hailo_common::create_flattened_bbox(roi->get_bbox(), roi->get_scaling_bbox()); + auto detections = hailo_common::get_hailo_detections(roi); + for (auto &detection : detections) + { + auto detection_bbox = detection->get_bbox(); + auto xmin = (detection_bbox.xmin() * roi_bbox.width()) + roi_bbox.xmin(); + auto ymin = (detection_bbox.ymin() * roi_bbox.height()) + roi_bbox.ymin(); + auto xmax = (detection_bbox.xmax() * roi_bbox.width()) + roi_bbox.xmin(); + auto ymax = (detection_bbox.ymax() * roi_bbox.height()) + roi_bbox.ymin(); + + HailoBBox new_bbox(xmin, ymin, xmax - xmin, ymax - ymin); + detection->set_bbox(new_bbox); + } + + // Clear the scaling bbox of main roi because all detections are fixed. + roi->clear_scaling_bbox(); + +} \ No newline at end of file diff --git a/core/hailo/libs/postprocesses/detection/yolo_hailortpp.hpp b/core/hailo/libs/postprocesses/detection/yolo_hailortpp.hpp index b46b992c..bfb7a9a6 100644 --- a/core/hailo/libs/postprocesses/detection/yolo_hailortpp.hpp +++ b/core/hailo/libs/postprocesses/detection/yolo_hailortpp.hpp @@ -6,9 +6,27 @@ #include "hailo_objects.hpp" #include "hailo_common.hpp" - __BEGIN_DECLS -void filter(HailoROIPtr roi); + +class YoloParamsNMS +{ +public: + std::map labels; + float detection_threshold; + uint max_boxes; + bool filter_by_score=false; + YoloParamsNMS(std::map dataset = std::map(), + float detection_threshold = 0.3f, + uint max_boxes = 200) + : labels(dataset), + detection_threshold(detection_threshold), + max_boxes(max_boxes) {} +}; + +YoloParamsNMS *init(const std::string config_path, const std::string function_name); +void free_resources(void *params_void_ptr); +void filter(HailoROIPtr roi, void *params_void_ptr); +void filter_letterbox(HailoROIPtr roi, void *params_void_ptr); void yolov5(HailoROIPtr roi); void yolov5s_nv12(HailoROIPtr roi); void yolov8s(HailoROIPtr roi); diff --git a/core/hailo/libs/postprocesses/meson.build b/core/hailo/libs/postprocesses/meson.build index a265bbea..dddb59e5 100644 --- a/core/hailo/libs/postprocesses/meson.build +++ b/core/hailo/libs/postprocesses/meson.build @@ -27,7 +27,7 @@ yolo_hailortpp_sources = [ shared_library('yolo_hailortpp_post', yolo_hailortpp_sources, cpp_args : hailo_lib_args, - include_directories: [hailo_general_inc, include_directories('./')], + include_directories: [hailo_general_inc, include_directories('./'), rapidjson_inc], dependencies : post_deps, gnu_symbol_visibility : 'default', install: true, diff --git a/core/hailo/libs/postprocesses/semantic_segmentation/semantic_segmentation.cpp b/core/hailo/libs/postprocesses/semantic_segmentation/semantic_segmentation.cpp index bc45a2ab..207133f0 100644 --- a/core/hailo/libs/postprocesses/semantic_segmentation/semantic_segmentation.cpp +++ b/core/hailo/libs/postprocesses/semantic_segmentation/semantic_segmentation.cpp @@ -2,19 +2,31 @@ * Copyright (c) 2021-2022 Hailo Technologies Ltd. All rights reserved. * Distributed under the LGPL license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt) **/ +#include #include "semantic_segmentation.hpp" #include "common/tensors.hpp" -const char *output_layer_name = "argmax1"; - void semantic_segmentation(HailoROIPtr roi) { if (!roi->has_tensors()) { return; } - - HailoTensorPtr tensor_ptr = roi->get_tensor(output_layer_name); + HailoTensorPtr tensor_ptr; + std::vector tensors = roi->get_tensors(); + // find the argmax1 tensor + for (auto tensor : tensors) + { + if (std::regex_search(tensor->name(), std::regex("argmax"))) + { + tensor_ptr = tensor; + } + } + if (!tensor_ptr) + { + std::cerr << "Semantic Segmentation post process: No argmax tensor found" << std::endl; + return; + } xt::xarray tensor_data = common::get_xtensor(tensor_ptr); // allocate and memcpy to a new memory so it points to the right data diff --git a/core/hailo/meson.build b/core/hailo/meson.build index 72bcdcf0..1b15405e 100644 --- a/core/hailo/meson.build +++ b/core/hailo/meson.build @@ -1,7 +1,7 @@ # Project Declaration project('gst-hailo-tools', 'c', 'cpp', - version : '3.29.0', + version : '3.29.1', default_options : [ 'buildtype=release', 'c_std=c11', 'cpp_std=c++17'] ) diff --git a/docs/installation/raspberry-pi-install.rst b/docs/installation/raspberry-pi-install.rst index 3d6d3e29..aea0586d 100644 --- a/docs/installation/raspberry-pi-install.rst +++ b/docs/installation/raspberry-pi-install.rst @@ -4,8 +4,6 @@ Run TAPPAS on Raspberry PI 4 Overview -------- -Hailo supports both Ubuntu 20.04 and Raspberry Pi OS, our recommendation is to install TAPPAS on Raspberry Pi OS. - This guide focuses on the installation of TAPPAS on Raspberry Pi OS. Preparing the Device diff --git a/install.sh b/install.sh index bc867236..97850bcb 100755 --- a/install.sh +++ b/install.sh @@ -2,6 +2,7 @@ set -e skip_hailort=false +core_only=false target_platform="x86" compile_num_cores="" gcc_version=12 @@ -32,6 +33,7 @@ function print_usage() { echo " --compile-num-of-cores Number of cpu cores to compile with (more cores makes the compilation process faster, but may cause 'out of swap memory' issue on weak machines)" echo " --download-apps-data Comma separated list (without spaces) of apps to download data for. Does not work with option '--target-platform'" echo " --list-apps Show the list of available apps" + echo " --core-only Install tappas core only (no apps data)" exit 1 } @@ -41,6 +43,8 @@ function parse_args() { print_usage elif [ "$1" == "--skip-hailort" ]; then skip_hailort=true + elif [ "$1" == "--core-only" ]; then + core_only=true elif [ "$1" == "--target-platform" ]; then target_platform=$2 platform_arg_passed=true @@ -90,10 +94,13 @@ function python_venv_create_and_install() { pip3 install -r $TAPPAS_WORKSPACE/core/requirements/requirements.txt pip3 install -r $TAPPAS_WORKSPACE/core/requirements/gstreamer_requirements.txt pip3 install -r $TAPPAS_WORKSPACE/downloader/requirements.txt - if [[ ${apps_to_set} ]]; then - python3 $TAPPAS_WORKSPACE/downloader/main.py $target_platform --apps-list $apps_to_set - else - python3 $TAPPAS_WORKSPACE/downloader/main.py $target_platform + # if rpi5 (core_only) is set dont download apps data (TAPPAS Core mode) + if [ "$core_only" = false ]; then + if [[ ${apps_to_set} ]]; then + python3 $TAPPAS_WORKSPACE/downloader/main.py $target_platform --apps-list $apps_to_set + else + python3 $TAPPAS_WORKSPACE/downloader/main.py $target_platform + fi fi } diff --git a/scripts/misc/pkg_config/hailo-tappas-core.pc_template b/scripts/misc/pkg_config/hailo-tappas-core.pc_template new file mode 100644 index 00000000..37bc60c3 --- /dev/null +++ b/scripts/misc/pkg_config/hailo-tappas-core.pc_template @@ -0,0 +1,49 @@ +prefix=/usr +includedir=${prefix}/include +arch= +libdir=${prefix}/lib/${arch}-linux-gnu +tappas_libdir=${libdir}/hailo/tappas +tappas_workspace= +tappas_includedir=${includedir}/hailo/tappas +tappas_sources=${tappas_includedir}/sources +tappas_hailometa_includedir=${tappas_includedir}/gsthailometa +tappas_postproc_includedir=${tappas_includedir} +tappas_postproc_lib_dir=${tappas_libdir}/post_processes +tappas_general_includedir=${tappas_includedir}/general +tappas_mat_includedir=${tappas_includedir}/plugins/common +tappas_tracking_includedir=${tappas_includedir}/tracking + + +catch2_includedir=${tappas_sources}/Catch2/include +cxxopts_includedir=${tappas_sources}/cxxopts/include +pybind11_includedir=${tappas_sources}/pybind11/include +rapidjson_includedir=${tappas_sources}/rapidjson/include +xtensor_blas_includedir=${tappas_sources}/xtensor-blas/include +xtensor_includedir=${tappas_sources}/xtensor/include +xtl_includedir=${tappas_sources}/xtl/include + +tappas_cpp_args=-fvisibility=default -fdiagnostics-color=always -D_FILE_OFFSET_BITS=64 -Wall -Winvalid-pch -Wnon-virtual-dtor -std=c++17 -Wpessimizing-move -Wredundant-move -fconcepts -fPIC + +Requires: opencv4 gstreamer-1.0 +Name: hailo_tappas +Version: +Description: Hailo Tappas +Libs: -L${tappas_libdir} -lgsthailometa -lhailo_tracker -lhailo_gst_image -lhailo_cv_singleton + +Cflags: -I${includedir} \ + -I${tappas_includedir} \ + -I${tappas_hailometa_includedir} \ + -I${tappas_postproc_includedir} \ + -I${tappas_general_includedir} \ + -I${tappas_mat_includedir} \ + -I${tappas_tracking_includedir} \ + -I${catch2_includedir} \ + -I${cxxopts_includedir} \ + -I${pybind11_includedir} \ + -I${rapidjson_includedir} \ + -I${xtensor_blas_includedir} \ + -I${xtensor_includedir} \ + -I${xtl_includedir} \ + -I/usr/include/hailo/ \ + -I/usr/include/gstreamer-1.0/gst/hailo/ \ + ${tappas_cpp_args} diff --git a/scripts/misc/pkg_config/hailo_tappas.pc_template b/scripts/misc/pkg_config/hailo_tappas.pc_template index 047f0f7a..710a665a 100644 --- a/scripts/misc/pkg_config/hailo_tappas.pc_template +++ b/scripts/misc/pkg_config/hailo_tappas.pc_template @@ -1,15 +1,15 @@ prefix=/opt/hailo/tappas +includedir=${prefix}/include arch= libdir=${prefix}/lib/${arch}-linux-gnu tappas_installation_dir=/opt/hailo/tappas tappas_libdir=${tappas_installation_dir}/lib/${arch}-linux-gnu tappas_workspace= -tappas_sources=${tappas_workspace}/sources - -includedir=${prefix}/include tappas_includedir=${tappas_installation_dir}/include +tappas_sources=${tappas_workspace}/sources tappas_hailometa_includedir=${tappas_includedir}/gsthailometa tappas_postproc_includedir=${tappas_workspace}/core/hailo/libs/postprocesses +tappas_postproc_lib_dir=${tappas_workspace}/apps/h8/gstreamer/libs/post_processes/ tappas_general_includedir=${tappas_workspace}/core/hailo/general tappas_mat_includedir=${tappas_workspace}/core/hailo/plugins/common tappas_tracking_includedir=${tappas_workspace}/core/hailo/tracking @@ -46,4 +46,4 @@ Cflags: -I${includedir} \ -I${xtl_includedir} \ -I/usr/include/hailo/ \ -I/usr/include/gstreamer-1.0/gst/hailo/ \ - ${tappas_cpp_args} + ${tappas_cpp_args} \ No newline at end of file diff --git a/scripts/misc/pkg_config_setup.sh b/scripts/misc/pkg_config_setup.sh index e9ba3bad..593ed005 100755 --- a/scripts/misc/pkg_config_setup.sh +++ b/scripts/misc/pkg_config_setup.sh @@ -11,21 +11,22 @@ set -e readonly TAPPAS_INSTALLATION_DIR=/opt/hailo/tappas readonly TAPPAS_PKG_CONFIG_DIR=${TAPPAS_INSTALLATION_DIR}/pkgconfig -readonly TAPPAS_PKG_CONFIG_FILE="hailo_tappas.pc" readonly TAPPAS_PKG_CONFIG_FILE_TEMPLATE="hailo_tappas.pc_template" +readonly TAPPAS_CORE_PKG_CONFIG_FILE_TEMPLATE="hailo-tappas-core.pc_template" +TAPPAS_PKG_CONFIG_FILE_BY_MODE=$TAPPAS_PKG_CONFIG_FILE_TEMPLATE readonly ARCH=$(uname -m) function print_usage(){ cat << EOF - Usage: $0 --tappas-workspace TAPPAS_WORKSPACE_PATH --tappas-version VERSION + Usage: $0 --tappas-workspace TAPPAS_WORKSPACE_PATH --tappas-version VERSION [--core-only] EOF return 1 } function parse_args(){ - if [[ $# != 4 ]]; then + if [[ $# -lt 4 || $# -gt 6 ]]; then print_usage fi while test $# -gt 0; do @@ -37,6 +38,9 @@ function parse_args(){ elif [ "$1" == "--tappas-version" ]; then TAPPAS_VERSION=$2 shift 2 + elif [ "$1" == "--core-only" ]; then + TAPPAS_PKG_CONFIG_FILE_BY_MODE=$TAPPAS_CORE_PKG_CONFIG_FILE_TEMPLATE + shift 2 else echo "Unknown parameters, exiting" print_usage @@ -50,10 +54,11 @@ function prepare_pkg_config_dir(){ } function update_pc_file(){ + TAPPAS_PKG_CONFIG_FILE=${TAPPAS_PKG_CONFIG_FILE_BY_MODE%_template} sed "s|tappas_workspace=|tappas_workspace=${TAPPAS_WORKSPACE}|; \ s|arch=|arch=${ARCH}|; \ s|Version:|Version: ${TAPPAS_VERSION}|" \ - pkg_config/${TAPPAS_PKG_CONFIG_FILE_TEMPLATE} > pkg_config/${TAPPAS_PKG_CONFIG_FILE} + pkg_config/${TAPPAS_PKG_CONFIG_FILE_BY_MODE} > pkg_config/${TAPPAS_PKG_CONFIG_FILE} } function copy_pc_file(){