Skip to content

Commit

Permalink
Address PR comments, add perception_utils
Browse files Browse the repository at this point in the history
  • Loading branch information
leungjch committed Mar 18, 2024
1 parent e06f49a commit 9723cc9
Show file tree
Hide file tree
Showing 11 changed files with 468 additions and 345 deletions.
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
ARG BASE_BUILD_IMAGE=ghcr.io/watonomous/wato_monorepo/base:cuda12.0-humble-ubuntu22.04-devel
ARG BASE_PROD_IMAGE=ghcr.io/watonomous/wato_monorepo/base:cuda12.0-humble-ubuntu22.04
ARG BASE_IMAGE=ghcr.io/watonomous/wato_monorepo/base:cuda12.0-humble-ubuntu22.04-devel
ARG RUNTIME_IMAGE=ghcr.io/watonomous/wato_monorepo/base:cuda12.0-humble-ubuntu22.04
ARG PADDLE_INFERENCE_BUILD_URL=ghcr.io/watonomous/perception_paddlepaddle_inference_build_cuda-12.0
################################ Build library ################################
FROM ${PADDLE_INFERENCE_BUILD_URL} as PADDLE_INFERENCE_BUILD

################################ Source ################################
FROM ${BASE_BUILD_IMAGE} as source
FROM ${BASE_IMAGE} as source

WORKDIR ${AMENT_WS}/src

Expand All @@ -17,6 +17,7 @@ RUN rm /paddle/paddle_inference_cuda120_build.tar

# Copy in source code
COPY src/perception/semantic_segmentation semantic_segmentation
COPY src/perception/perception_utils perception_utils

# Scan for rosdeps
RUN apt-get -qq update && rosdep update && \
Expand All @@ -26,7 +27,7 @@ RUN apt-get -qq update && rosdep update && \
| sort > /tmp/colcon_install_list

################################# Dependencies ################################
FROM ${BASE_BUILD_IMAGE} as dependencies
FROM ${BASE_IMAGE} as dependencies

RUN apt update && apt install -y tensorrt ros-humble-cv-bridge libopencv-dev

Expand All @@ -36,8 +37,8 @@ RUN apt-fast install -qq -y --no-install-recommends $(cat /tmp/colcon_install_li

# Copy in source code from source stage
WORKDIR ${AMENT_WS}
COPY --from=source ${AMENT_WS}/src src
COPY --from=source /paddle /paddle
COPY --from=source ${AMENT_WS}/src src

# Dependency Cleanup
WORKDIR /
Expand All @@ -63,7 +64,7 @@ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
ENTRYPOINT ["./wato_ros_entrypoint.sh"]

# ################################ Prod ################################
FROM ${BASE_PROD_IMAGE} as deploy
FROM ${RUNTIME_IMAGE} as deploy


# Install runtime libs
Expand Down
2 changes: 1 addition & 1 deletion modules/docker-compose.perception.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ services:
count: 1
capabilities: [ gpu ]
volumes:
- /mnt/wato-drive2/perception-weights/semantic_segmentation:/perception_models/semantic_segmentation
- /mnt/wato-drive2/perception-weights/semantic_segmentation/pp_liteseg_infer_model:/perception_models/semantic_segmentation/pp_liteseg_infer_model

lane_detection:
build:
Expand Down
49 changes: 49 additions & 0 deletions src/perception/perception_utils/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
cmake_minimum_required(VERSION 3.5)
project(perception_utils)

# Default to C++14
if(NOT CMAKE_CXX_STANDARD)
set(CMAKE_CXX_STANDARD 14)
endif()

if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
add_compile_options(-Wall -Wextra -Wpedantic)
endif()

# Find dependencies
find_package(ament_cmake REQUIRED)
find_package(rclcpp REQUIRED)
find_package(OpenCV REQUIRED)

# Include header files
include_directories(include
${OpenCV_INCLUDE_DIRS}
)

# Declare a C++ library
add_library(perception_utils SHARED
src/camera_utils.cpp
)

# Specify libraries to link a library or executable target against
ament_target_dependencies(perception_utils
rclcpp
OpenCV)

install(TARGETS perception_utils
EXPORT export_perception_utils
ARCHIVE DESTINATION lib
LIBRARY DESTINATION lib
RUNTIME DESTINATION bin
)

ament_export_targets(export_perception_utils HAS_LIBRARY_TARGET)
ament_export_include_directories(include)
ament_export_dependencies(rclcpp)

# Install header files
install(DIRECTORY include/
DESTINATION include
)

ament_package()
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
#ifndef CAMERA_UTILS_HPP_
#define CAMERA_UTILS_HPP_

#include <opencv2/opencv.hpp>

namespace CameraUtils {
// Swaps the channels of an image from HWC to CHW format
void hwc_img_2_chw_data(const cv::Mat& hwc_img, float* data);
cv::Mat resize_image_aspect_ratio(const cv::Mat& original_image, int max_width, int max_height);
cv::Mat resize_with_padding(const cv::Mat& original_image, int target_width, int target_height);
cv::Mat resize_from_center(const cv::Mat& original_image, int target_width, int target_height);
}; // namespace CameraUtils

#endif // CAMERA_UTILS_HPP_
18 changes: 18 additions & 0 deletions src/perception/perception_utils/package.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
<?xml version="1.0"?>
<package format="3">
<name>perception_utils</name>
<version>0.0.1</version>
<description>A utility library for perception tasks in ROS 2.</description>

<maintainer email="[email protected]">Justin Leung</maintainer>
<license>Apache-2.0</license>

<buildtool_depend>ament_cmake</buildtool_depend>
<depend>rclcpp</depend>
<depend>opencv</depend>

<export>
<build_type>ament_cmake</build_type>
</export>

</package>
97 changes: 97 additions & 0 deletions src/perception/perception_utils/src/camera_utils.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
#include "perception_utils/camera_utils.hpp"
#include <opencv2/opencv.hpp>

// Swaps the channels of an image from HWC to CHW format
void CameraUtils::hwc_img_2_chw_data(const cv::Mat& hwc_img, float* data) {
int rows = hwc_img.rows;
int cols = hwc_img.cols;
int chs = hwc_img.channels();
for (int i = 0; i < chs; ++i) {
cv::extractChannel(hwc_img, cv::Mat(rows, cols, CV_32FC1, data + i * rows * cols), i);
}
}

cv::Mat CameraUtils::resize_image_aspect_ratio(const cv::Mat& original_image, int max_width,
int max_height) {
int original_width = original_image.cols;
int original_height = original_image.rows;
double original_aspect_ratio = (double)original_width / original_height;

int new_width, new_height;
double max_aspect_ratio = (double)max_width / max_height;

if (original_aspect_ratio > max_aspect_ratio) {
// Width is the limiting factor
new_width = max_width;
new_height = static_cast<int>(max_width / original_aspect_ratio);
} else {
// Height is the limiting factor
new_height = max_height;
new_width = static_cast<int>(max_height * original_aspect_ratio);
}

cv::Mat resized_image;
cv::resize(original_image, resized_image, cv::Size(new_width, new_height));

return resized_image;
}

cv::Mat CameraUtils::resize_with_padding(const cv::Mat& original_image, int target_width,
int target_height) {
int original_width = original_image.cols;
int original_height = original_image.rows;

double target_ratio = (double)target_width / target_height;
double original_ratio = (double)original_width / original_height;

int new_width, new_height;

if (original_ratio > target_ratio) {
// Original is wider. Fit to width and pad height.
new_width = target_width;
new_height =
static_cast<int>(original_height * (static_cast<double>(target_width) / original_width));
} else {
// Original is taller. Fit to height and pad width.
new_height = target_height;
new_width =
static_cast<int>(original_width * (static_cast<double>(target_height) / original_height));
}

cv::Mat resized_image;
cv::resize(original_image, resized_image, cv::Size(new_width, new_height));

int top = (target_height - new_height) / 2;
int bottom = target_height - new_height - top;
int left = (target_width - new_width) / 2;
int right = target_width - new_width - left;

cv::Mat padded_image;
cv::copyMakeBorder(resized_image, padded_image, top, bottom, left, right, cv::BORDER_CONSTANT,
cv::Scalar(0, 0, 0));

return padded_image;
}

cv::Mat CameraUtils::resize_from_center(const cv::Mat& original_image, int target_width,
int target_height) {
int original_width = original_image.cols;
int original_height = original_image.rows;

// Calculate the new height maintaining the aspect ratio
double target_ratio = (double)target_width / target_height;
int new_height = static_cast<int>(original_width / target_ratio);

// Calculate the cropping area
int startY = (original_height - new_height) / 2;

// Crop the image from the center
cv::Rect roi(0, startY, original_width, new_height);
cv::Mat cropped_image = original_image(roi);

// Resize the cropped image
cv::Mat resized_image;
cv::resize(cropped_image, resized_image, cv::Size(target_width, target_height));

return resized_image;
}
70 changes: 44 additions & 26 deletions src/perception/semantic_segmentation/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
cmake_minimum_required(VERSION 3.8)
include(CMakePrintHelpers)
project(semantic_segmentation LANGUAGES CXX CUDA)
message(STATUS "Perception Utils Include Dirs: ${perception_utils_INCLUDE_DIRS}")
message(STATUS "Perception Utils Include Dirs: ${perception_utils_INCLUDE_DIRS}")

set(PADDLE_LIB "/paddle/paddle_inference_cuda120_build/paddle_inference_install_dir") # Set this path as per your setup

option(WITH_MKL "Compile demo with MKL/OpenBlas support, default use MKL." ON)
option(WITH_GPU "Compile demo with GPU/CPU, default use CPU." ON)
option(WITH_STATIC_LIB "Compile demo with static/shared library, default use static." ON)
option(USE_TENSORRT "Compile demo with TensorRT." ON)
option(WITH_ROCM "Compile demo with rocm." OFF)
cmake_print_variables(PROJECT_SOURCE_DIR)
cmake_print_variables(perception_utils_INCLUDE_DIRS)
set(PADDLE_LIB "/paddle/paddle_inference_cuda120_build/paddle_inference_install_dir")

# Compiler options
if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
Expand All @@ -22,44 +21,63 @@ endif()
# Platform-independent settings
include_directories("${PADDLE_LIB}/")
set(PADDLE_LIB_THIRD_PARTY_PATH "${PADDLE_LIB}/third_party/install/")
include_directories(${PADDLE_LIB_THIRD_PARTY_PATH}protobuf/include ${PADDLE_LIB_THIRD_PARTY_PATH}glog/include ${PADDLE_LIB_THIRD_PARTY_PATH}gflags/include ${PADDLE_LIB_THIRD_PARTY_PATH}xxhash/include ${PADDLE_LIB_THIRD_PARTY_PATH}cryptopp/include ${PADDLE_LIB_THIRD_PARTY_PATH}onnxruntime/include ${PADDLE_LIB_THIRD_PARTY_PATH}paddle2onnx/include)
include_directories(${PADDLE_LIB_THIRD_PARTY_PATH}protobuf/include
${PADDLE_LIB_THIRD_PARTY_PATH}glog/include
${PADDLE_LIB_THIRD_PARTY_PATH}gflags/include
${PADDLE_LIB_THIRD_PARTY_PATH}xxhash/include
${PADDLE_LIB_THIRD_PARTY_PATH}cryptopp/include
${PADDLE_LIB_THIRD_PARTY_PATH}onnxruntime/include
${PADDLE_LIB_THIRD_PARTY_PATH}paddle2onnx/include
)

link_directories(${PADDLE_LIB_THIRD_PARTY_PATH}protobuf/lib ${PADDLE_LIB_THIRD_PARTY_PATH}glog/lib ${PADDLE_LIB_THIRD_PARTY_PATH}gflags/lib ${PADDLE_LIB_THIRD_PARTY_PATH}xxhash/lib ${PADDLE_LIB_THIRD_PARTY_PATH}cryptopp/lib ${PADDLE_LIB_THIRD_PARTY_PATH}onnxruntime/lib ${PADDLE_LIB_THIRD_PARTY_PATH}paddle2onnx/lib ${PADDLE_LIB}/paddle/lib)
link_directories(${PADDLE_LIB_THIRD_PARTY_PATH}protobuf/lib
${PADDLE_LIB_THIRD_PARTY_PATH}glog/lib
${PADDLE_LIB_THIRD_PARTY_PATH}gflags/lib
${PADDLE_LIB_THIRD_PARTY_PATH}xxhash/lib
${PADDLE_LIB_THIRD_PARTY_PATH}cryptopp/lib
${PADDLE_LIB_THIRD_PARTY_PATH}onnxruntime/lib
${PADDLE_LIB_THIRD_PARTY_PATH}paddle2onnx/lib
${PADDLE_LIB}/paddle/lib)

# Find packages
find_package(rclcpp REQUIRED)
find_package(std_msgs REQUIRED)
find_package(sensor_msgs REQUIRED)
find_package(geometry_msgs REQUIRED)
find_package(image_transport REQUIRED)
find_package(OpenCV REQUIRED)
find_package(cv_bridge REQUIRED)
find_package(CUDA REQUIRED)
find_package(ament_cmake REQUIRED)
find_package(yaml-cpp REQUIRED)
message(STATUS "Perception Utils Include Dirs: ${perception_utils_INCLUDE_DIRS}")

set(REQUIRED_PACKAGES
rclcpp
std_msgs
sensor_msgs
image_transport
OpenCV
cv_bridge
CUDA
ament_cmake
yaml-cpp
perception_utils
)

foreach(PACKAGE ${REQUIRED_PACKAGES})
find_package(${PACKAGE} REQUIRED)
endforeach()

# Include directories
include_directories(include
${rclcpp_INCLUDE_DIRS}
${std_msgs_INCLUDE_DIRS}
${sensor_msgs_INCLUDE_DIRS}
${geometry_msgs_INCLUDE_DIRS}
${image_transport_INCLUDE_DIRS}
${CUDA_INCLUDE_DIRS}
${YAML_CPP_INCLUDE_DIRS}
/usr/local/include
${PADDLE_LIB}/paddle/include
${PADDLE_LIB}/third_party/install/mkldnn/include
)

${perception_utils_INCLUDE_DIRS}
)

# Link directories
link_directories(/usr/local/lib ${YAML_CPP_LIBRARIES})


# Set additional dependencies
set(EXTRA_LIBS "-lrt -ldl -lpthread")
set(DEPS ${rclcpp_LIBRARIES} ${std_msgs_LIBRARIES} ${sensor_msgs_LIBRARIES} ${geometry_msgs_LIBRARIES} ${image_transport_LIBRARIES} ${CUDA_LIBRARIES} ${YAML_CPP_LIBRARIES} ${EXTRA_LIBS}
set(DEPS ${rclcpp_LIBRARIES} ${std_msgs_LIBRARIES} ${sensor_msgs_LIBRARIES} ${image_transport_LIBRARIES} ${CUDA_LIBRARIES} ${YAML_CPP_LIBRARIES} ${EXTRA_LIBS}
${PADDLE_LIB}/paddle/lib/libpaddle_inference.so
${PADDLE_LIB}/paddle/lib/libphi.so
${PADDLE_LIB}/paddle/lib/libcommon.so
Expand Down Expand Up @@ -89,8 +107,8 @@ ament_target_dependencies(semantic_segmentation
std_msgs
sensor_msgs
cv_bridge
geometry_msgs
image_transport
OpenCV
perception_utils
)
ament_package()
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ semantic_segmentation_node:
publish_vis_topic: /camera/left/segmentations_viz
model_path: /perception_models/semantic_segmentation/pp_liteseg_infer_model/
save_dir: /segmented
save_images: false
save_images: false
Loading

0 comments on commit 9723cc9

Please sign in to comment.