From 46da6f19bc91789c97ca530bb8d09c24f406d787 Mon Sep 17 00:00:00 2001 From: Jaiveer Singh Date: Tue, 10 Dec 2024 17:48:19 -0800 Subject: [PATCH] Isaac ROS 3.2 --- README.md | 10 +- gxf_isaac_detectnet/CMakeLists.txt | 6 + .../gxf/detectnet/detectnet.cpp | 4 +- .../gxf/detectnet/detectnet_decoder.cpp | 63 +++--- .../gxf/detectnet/detectnet_decoder.hpp | 10 +- gxf_isaac_detectnet/package.xml | 2 +- isaac_ros_detectnet/CMakeLists.txt | 8 +- isaac_ros_detectnet/config/hawk_config.pbtxt | 6 +- .../config/isaac_sim_config.pbtxt | 6 +- .../config/peoplenet_config.pbtxt | 12 +- .../config/quickstart_config.pbtxt | 12 +- .../config/realsense_config.pbtxt | 6 +- isaac_ros_detectnet/config/zed2_config.pbtxt | 6 +- isaac_ros_detectnet/config/zedx_config.pbtxt | 6 +- .../launch/isaac_ros_detectnet.launch.py | 11 +- .../launch/isaac_ros_detectnet_core.launch.py | 11 +- .../isaac_ros_detectnet_isaac_sim.launch.py | 22 +- isaac_ros_detectnet/package.xml | 2 +- isaac_ros_detectnet/scripts/setup_model.sh | 72 +++--- .../test/dummy_model/.gitignore | 3 +- .../test/dummy_model/detectnet/1/.gitkeep | 0 .../detectnet/1/resnet18_detector.etlt | 3 - .../test/dummy_model/detectnet/config.pbtxt | 6 +- .../test/isaac_ros_detectnet_pol_test.py | 83 +++---- .../CMakeLists.txt | 35 +++ .../asset_scripts/install_peoplenet_amr_rs.sh | 65 ++++++ .../package.xml | 40 ++++ isaac_ros_rtdetr/CMakeLists.txt | 6 + .../isaac_ros_rtdetr_isaac_sim.launch.py | 2 +- isaac_ros_rtdetr/package.xml | 2 +- isaac_ros_rtdetr/src/rtdetr_decoder_node.cpp | 2 +- .../src/rtdetr_preprocessor_node.cpp | 9 +- isaac_ros_rtdetr/test/dummy_model/.gitignore | 2 + .../test/dummy_model/rtdetr_dummy_pol.onnx | 3 - .../test/isaac_ros_rtdetr_pol_test.py | 29 ++- isaac_ros_yolov8/CMakeLists.txt | 7 + .../isaac_ros_yolov8/yolov8_decoder_node.hpp | 1 + .../launch/isaac_ros_yolov8_core.launch.py | 24 +- isaac_ros_yolov8/package.xml | 3 +- isaac_ros_yolov8/src/yolov8_decoder_node.cpp | 6 +- .../dummy_model/yolov8/dummy_yolov8s.onnx | 3 + .../test/isaac_ros_yolov8_decoder_node_pol.py | 211 ++++++++++++++++++ .../single_detection/camera_info.json | 55 +++++ .../test_cases/single_detection/image.json | 4 + .../single_detection/people_cycles.jpg | 3 + 45 files changed, 670 insertions(+), 212 deletions(-) create mode 100644 isaac_ros_detectnet/test/dummy_model/detectnet/1/.gitkeep delete mode 100644 isaac_ros_detectnet/test/dummy_model/detectnet/1/resnet18_detector.etlt create mode 100644 isaac_ros_peoplenet_models_install/CMakeLists.txt create mode 100755 isaac_ros_peoplenet_models_install/asset_scripts/install_peoplenet_amr_rs.sh create mode 100644 isaac_ros_peoplenet_models_install/package.xml create mode 100644 isaac_ros_rtdetr/test/dummy_model/.gitignore delete mode 100644 isaac_ros_rtdetr/test/dummy_model/rtdetr_dummy_pol.onnx create mode 100644 isaac_ros_yolov8/test/dummy_model/yolov8/dummy_yolov8s.onnx create mode 100644 isaac_ros_yolov8/test/isaac_ros_yolov8_decoder_node_pol.py create mode 100644 isaac_ros_yolov8/test/test_cases/single_detection/camera_info.json create mode 100644 isaac_ros_yolov8/test/test_cases/single_detection/image.json create mode 100644 isaac_ros_yolov8/test/test_cases/single_detection/people_cycles.jpg diff --git a/README.md b/README.md index 7deab70..0def8fc 100644 --- a/README.md +++ b/README.md @@ -63,10 +63,10 @@ This package is powered by [NVIDIA Isaac Transport for ROS (NITROS)](https://dev ## Performance -| Sample Graph

| Input Size

| AGX Orin

| Orin NX

| Orin Nano 8GB

| x86_64 w/ RTX 4060 Ti

| x86_64 w/ RTX 4090

| -|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [RT-DETR Object Detection Graph](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/benchmarks/isaac_ros_rtdetr_benchmark/scripts/isaac_ros_rtdetr_graph.py)


SyntheticaDETR

| 720p



| [71.9 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_rtdetr_graph-agx_orin.json)


24 ms @ 30Hz

| [30.8 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_rtdetr_graph-orin_nx.json)


41 ms @ 30Hz

| [21.3 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_rtdetr_graph-orin_nano.json)


61 ms @ 30Hz

| [205 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_rtdetr_graph-nuc_4060ti.json)


8.7 ms @ 30Hz

| [400 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_rtdetr_graph-x86_4090.json)


6.3 ms @ 30Hz

| -| [DetectNet Object Detection Graph](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/benchmarks/isaac_ros_detectnet_benchmark/scripts/isaac_ros_detectnet_graph.py)



| 544p



| [165 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_detectnet_graph-agx_orin.json)


20 ms @ 30Hz

| [115 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_detectnet_graph-orin_nx.json)


26 ms @ 30Hz

| [63.2 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_detectnet_graph-orin_nano.json)


36 ms @ 30Hz

| [488 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_detectnet_graph-nuc_4060ti.json)


10 ms @ 30Hz

| [589 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_detectnet_graph-x86_4090.json)


10 ms @ 30Hz

| +| Sample Graph

| Input Size

| AGX Orin

| Orin NX

| Orin Nano 8GB

| x86_64 w/ RTX 4090

| +|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [RT-DETR Object Detection Graph](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/benchmarks/isaac_ros_rtdetr_benchmark/scripts/isaac_ros_rtdetr_graph.py)


SyntheticaDETR

| 720p



| [56.5 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_rtdetr_graph-agx_orin.json)


30 ms @ 30Hz

| [33.8 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_rtdetr_graph-orin_nx.json)


39 ms @ 30Hz

| [24.1 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_rtdetr_graph-orin_nano.json)


53 ms @ 30Hz

| [490 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_rtdetr_graph-x86-4090.json)


7.1 ms @ 30Hz

| +| [DetectNet Object Detection Graph](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/benchmarks/isaac_ros_detectnet_benchmark/scripts/isaac_ros_detectnet_graph.py)



| 544p



| [70.5 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_detectnet_graph-agx_orin.json)


26 ms @ 30Hz

| [30.1 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_detectnet_graph-orin_nx.json)


46 ms @ 30Hz

| [22.9 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_detectnet_graph-orin_nano.json)


57 ms @ 30Hz

| [254 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_detectnet_graph-x86-4090.json)


11 ms @ 30Hz

| --- @@ -97,4 +97,4 @@ Please visit the [Isaac ROS Documentation](https://nvidia-isaac-ros.github.io/re ## Latest -Update 2024-09-26: Update for ZED compatibility +Update 2024-12-10: Update to be compatible with JetPack 6.1 diff --git a/gxf_isaac_detectnet/CMakeLists.txt b/gxf_isaac_detectnet/CMakeLists.txt index bb746aa..329b307 100644 --- a/gxf_isaac_detectnet/CMakeLists.txt +++ b/gxf_isaac_detectnet/CMakeLists.txt @@ -63,4 +63,10 @@ set_target_properties(${PROJECT_NAME} PROPERTIES # Install the binary file install(TARGETS ${PROJECT_NAME} DESTINATION share/${PROJECT_NAME}/gxf/lib) + +# Embed versioning information into installed files +ament_index_get_resource(ISAAC_ROS_COMMON_CMAKE_PATH isaac_ros_common_cmake_path isaac_ros_common) +include("${ISAAC_ROS_COMMON_CMAKE_PATH}/isaac_ros_common-version-info.cmake") +generate_version_info(${PROJECT_NAME}) + ament_auto_package(INSTALL_TO_SHARE) diff --git a/gxf_isaac_detectnet/gxf/detectnet/detectnet.cpp b/gxf_isaac_detectnet/gxf/detectnet/detectnet.cpp index d90fb63..0576641 100644 --- a/gxf_isaac_detectnet/gxf/detectnet/detectnet.cpp +++ b/gxf_isaac_detectnet/gxf/detectnet/detectnet.cpp @@ -14,11 +14,13 @@ // limitations under the License. // // SPDX-License-Identifier: Apache-2.0 + #include +#include +#include "detectnet/detectnet_decoder.hpp" #include "gxf/core/gxf.h" #include "gxf/std/extension_factory_helper.hpp" -#include "detectnet/detectnet_decoder.hpp" extern "C" { diff --git a/gxf_isaac_detectnet/gxf/detectnet/detectnet_decoder.cpp b/gxf_isaac_detectnet/gxf/detectnet/detectnet_decoder.cpp index 8111832..a3e185f 100644 --- a/gxf_isaac_detectnet/gxf/detectnet/detectnet_decoder.cpp +++ b/gxf_isaac_detectnet/gxf/detectnet/detectnet_decoder.cpp @@ -15,18 +15,21 @@ // // SPDX-License-Identifier: Apache-2.0 -#include "detectnet_decoder.hpp" -#include "detection2_d_array_message.hpp" - -#include #include +#include +#include +#include + +#include "./detection2_d_array_message.hpp" +#include "./detectnet_decoder.hpp" + +#include "cuda.h" +#include "cuda_runtime.h" +#include "gxf/core/parameter_parser_std.hpp" #include "gxf/multimedia/camera.hpp" #include "gxf/multimedia/video.hpp" -#include "gxf/core/parameter_parser_std.hpp" #include "gxf/std/timestamp.hpp" -#include "cuda.h" -#include "cuda_runtime.h" namespace nvidia @@ -67,10 +70,10 @@ NvDsInferObjectDetectionInfo GetNewDetectionInfo( } void FillMessage( - Detection2DParts &message_parts, - const std::vector &detection_info_vector, + Detection2DParts& message_parts, + const std::vector& detection_info_vector, gxf::Handle tensorlist_timestamp, - size_t num_detections, const std::vector &label_list) + size_t num_detections, const std::vector& label_list) { for (uint32_t i = 0; i < num_detections; i++) { NvDsInferObjectDetectionInfo detection_info = detection_info_vector[i]; @@ -89,7 +92,7 @@ void FillMessage( } *(message_parts.timestamp) = *tensorlist_timestamp; } -} // anonymous namespace +} // anonymous namespace gxf_result_t DetectnetDecoder::registerInterface(gxf::Registrar * registrar) noexcept @@ -106,8 +109,8 @@ gxf_result_t DetectnetDecoder::registerInterface(gxf::Registrar * registrar) noe result &= registrar->parameter( label_list_, "label_list", "List of network labels", - "List of labels corresponding to the int labels received from the tensors", {"person", "bag", - "face"}); + "List of labels corresponding to the int labels received from the tensors", + {"person", "bag", "face"}); result &= registrar->parameter( enable_confidence_threshold_, "enable_confidence_threshold", "Enable Confidence Threshold", @@ -131,27 +134,27 @@ gxf_result_t DetectnetDecoder::registerInterface(gxf::Registrar * registrar) noe result &= registrar->parameter( dbscan_confidence_threshold_, "dbscan_confidence_threshold", "Dbscan Confidence Threshold", - "Minimum score in a cluster for the cluster to be considered an object \ - during grouping. Different clustering may cause the algorithm \ - to use different scores.", + "Minimum score in a cluster for the cluster to be considered an object " + "during grouping. Different clustering may cause the algorithm " + "to use different scores.", 0.6); result &= registrar->parameter( dbscan_eps_, "dbscan_eps", "Dbscan Epsilon", - "Holds the epsilon to control merging of overlapping boxes. \ - Refer to OpenCV groupRectangles and DBSCAN documentation for more information on epsilon. ", + "Holds the epsilon to control merging of overlapping boxes. " + "Refer to OpenCV groupRectangles and DBSCAN documentation for more information on epsilon. ", 0.01); result &= registrar->parameter( dbscan_min_boxes_, "dbscan_min_boxes", "Dbscan Minimum Boxes", - "Holds the minimum number of boxes in a cluster to be considered \ - an object during grouping using DBSCAN", + "Holds the minimum number of boxes in a cluster to be considered " + "an object during grouping using DBSCAN", 1); result &= registrar->parameter( dbscan_enable_athr_filter_, "dbscan_enable_athr_filter", "Dbscan Enable Athr Filter", - "true enables the area-to-hit ratio (ATHR) filter. \ - The ATHR is calculated as: ATHR = sqrt(clusterArea) / nObjectsInCluster.", + "true enables the area-to-hit ratio (ATHR) filter. " + "The ATHR is calculated as: ATHR = sqrt(clusterArea) / nObjectsInCluster.", 0); result &= registrar->parameter( @@ -196,7 +199,6 @@ gxf_result_t DetectnetDecoder::start() noexcept gxf_result_t DetectnetDecoder::tick() noexcept { - gxf::Expected result; // Receive disparity image and left/right camera info @@ -272,9 +274,10 @@ gxf_result_t DetectnetDecoder::tick() noexcept return GXF_FAILURE; } - float bbox_tensor_arr[bbox_tensor->size() / sizeof(float)]; // since data in tensor is kFloat32 + // data in tensor is kFloat32 + std::vector bbox_tensor_arr(bbox_tensor->size() / sizeof(float)); const cudaError_t cuda_error_bbox_tensor = cudaMemcpy( - &bbox_tensor_arr, bbox_tensor->pointer(), + bbox_tensor_arr.data(), bbox_tensor->pointer(), bbox_tensor->size(), cudaMemcpyDeviceToHost); if (cuda_error_bbox_tensor != cudaSuccess) { GXF_LOG_ERROR("Error while copying kernel: %s", cudaGetErrorString(cuda_error_bbox_tensor)); @@ -315,8 +318,8 @@ gxf_result_t DetectnetDecoder::tick() noexcept float coverage = cov_tensor_arr[cov_pos]; // Center of the grid in pixels - float grid_center_y = (row + bounding_box_offset_ ) * kStride; - float grid_center_x = (col + bounding_box_offset_ ) * kStride; + float grid_center_y = (row + bounding_box_offset_) * kStride; + float grid_center_x = (col + bounding_box_offset_) * kStride; // Get each element of the bounding box float bbox[kBoundingBoxParams]; @@ -342,7 +345,8 @@ gxf_result_t DetectnetDecoder::tick() noexcept // check if object_class is out of range for label_list_ if (static_cast(object_class) >= label_list_.get().size()) { GXF_LOG_ERROR( - "[DetectNet Decoder] object_class %i is out of range for provided label_list_ of size %lu", object_class, + "[DetectNet Decoder] object_class %i is out of range for provided " + "label_list_ of size %lu", object_class, label_list_.get().size()); return GXF_FAILURE; } @@ -360,7 +364,7 @@ gxf_result_t DetectnetDecoder::tick() noexcept size_t num_detections = detection_info_vector.size(); if (enable_dbscan_clustering_) { - NvDsInferObjectDetectionInfo * detection_info_pointer = &detection_info_vector[0]; + NvDsInferObjectDetectionInfo* detection_info_pointer = &detection_info_vector[0]; NvDsInferDBScanHandle dbscan_hdl = NvDsInferDBScanCreate(); if (dbscan_clustering_algorithm_ == kDbscanCluster) { NvDsInferDBScanCluster(dbscan_hdl, ¶ms_, detection_info_pointer, &num_detections); @@ -386,7 +390,6 @@ gxf_result_t DetectnetDecoder::tick() noexcept num_detections, label_list_); return detections_transmitter_->publish(message_parts.message); })); - } } // namespace isaac_ros } // namespace nvidia diff --git a/gxf_isaac_detectnet/gxf/detectnet/detectnet_decoder.hpp b/gxf_isaac_detectnet/gxf/detectnet/detectnet_decoder.hpp index 6bdde4d..09d324c 100644 --- a/gxf_isaac_detectnet/gxf/detectnet/detectnet_decoder.hpp +++ b/gxf_isaac_detectnet/gxf/detectnet/detectnet_decoder.hpp @@ -18,15 +18,19 @@ #ifndef NVIDIA_ISAAC_ROS_EXTENSIONS_DETECTNET_DECODER_HPP_ #define NVIDIA_ISAAC_ROS_EXTENSIONS_DETECTNET_DECODER_HPP_ +#include +#include + +#include "./detection2_d_array_message.hpp" +#include "deepstream_utils/nvdsinferutils/dbscan/nvdsinfer_dbscan.hpp" + #include "gxf/core/entity.hpp" #include "gxf/core/gxf.h" #include "gxf/core/parameter.hpp" -#include "gxf/std/codelet.hpp" #include "gxf/core/parameter_parser_std.hpp" +#include "gxf/std/codelet.hpp" #include "gxf/std/receiver.hpp" #include "gxf/std/transmitter.hpp" -#include "detection2_d_array_message.hpp" -#include "deepstream_utils/nvdsinferutils/dbscan/nvdsinfer_dbscan.hpp" namespace nvidia diff --git a/gxf_isaac_detectnet/package.xml b/gxf_isaac_detectnet/package.xml index f667917..504bd91 100644 --- a/gxf_isaac_detectnet/package.xml +++ b/gxf_isaac_detectnet/package.xml @@ -21,7 +21,7 @@ SPDX-License-Identifier: Apache-2.0 gxf_isaac_detectnet - 3.1.0 + 3.2.0 Detectnet GXF extension. Isaac ROS Maintainers diff --git a/isaac_ros_detectnet/CMakeLists.txt b/isaac_ros_detectnet/CMakeLists.txt index 3342678..1a5f7e7 100644 --- a/isaac_ros_detectnet/CMakeLists.txt +++ b/isaac_ros_detectnet/CMakeLists.txt @@ -47,7 +47,7 @@ if(BUILD_TESTING) endif() find_package(launch_testing_ament_cmake REQUIRED) - add_launch_test(test/isaac_ros_detectnet_pol_test.py TIMEOUT "600") + add_launch_test(test/isaac_ros_detectnet_pol_test.py TIMEOUT "900") endif() # Visualizer python scripts @@ -63,4 +63,10 @@ install(DIRECTORY DESTINATION share/${PROJECT_NAME} ) + +# Embed versioning information into installed files +ament_index_get_resource(ISAAC_ROS_COMMON_CMAKE_PATH isaac_ros_common_cmake_path isaac_ros_common) +include("${ISAAC_ROS_COMMON_CMAKE_PATH}/isaac_ros_common-version-info.cmake") +generate_version_info(${PROJECT_NAME}) + ament_auto_package(INSTALL_TO_SHARE config launch) diff --git a/isaac_ros_detectnet/config/hawk_config.pbtxt b/isaac_ros_detectnet/config/hawk_config.pbtxt index 7e85019..7517670 100644 --- a/isaac_ros_detectnet/config/hawk_config.pbtxt +++ b/isaac_ros_detectnet/config/hawk_config.pbtxt @@ -3,7 +3,7 @@ platform: "tensorrt_plan" max_batch_size: 16 input [ { - name: "input_1" + name: "input_1:0" data_type: TYPE_FP32 format: FORMAT_NCHW dims: [ 3, 1200, 1920 ] @@ -11,12 +11,12 @@ input [ ] output [ { - name: "output_bbox/BiasAdd" + name: "output_bbox/BiasAdd:0" data_type: TYPE_FP32 dims: [ 12, 75, 120] }, { - name: "output_cov/Sigmoid" + name: "output_cov/Sigmoid:0" data_type: TYPE_FP32 dims: [ 3, 75, 120] } diff --git a/isaac_ros_detectnet/config/isaac_sim_config.pbtxt b/isaac_ros_detectnet/config/isaac_sim_config.pbtxt index 118dce6..4fba106 100644 --- a/isaac_ros_detectnet/config/isaac_sim_config.pbtxt +++ b/isaac_ros_detectnet/config/isaac_sim_config.pbtxt @@ -3,7 +3,7 @@ platform: "tensorrt_plan" max_batch_size: 16 input [ { - name: "input_1" + name: "input_1:0" data_type: TYPE_FP32 format: FORMAT_NCHW dims: [ 3, 720, 1280 ] @@ -11,12 +11,12 @@ input [ ] output [ { - name: "output_bbox/BiasAdd" + name: "output_bbox/BiasAdd:0" data_type: TYPE_FP32 dims: [ 12, 45, 80] }, { - name: "output_cov/Sigmoid" + name: "output_cov/Sigmoid:0" data_type: TYPE_FP32 dims: [ 3, 45, 80] } diff --git a/isaac_ros_detectnet/config/peoplenet_config.pbtxt b/isaac_ros_detectnet/config/peoplenet_config.pbtxt index 97f32a4..e731c0a 100644 --- a/isaac_ros_detectnet/config/peoplenet_config.pbtxt +++ b/isaac_ros_detectnet/config/peoplenet_config.pbtxt @@ -3,22 +3,22 @@ platform: "tensorrt_plan" max_batch_size: 16 input [ { - name: "input_1" + name: "input_1:0" data_type: TYPE_FP32 format: FORMAT_NCHW - dims: [ 3, 544, 960 ] + dims: [ 3, 544, 960] } ] output [ { - name: "output_bbox/BiasAdd" + name: "output_bbox/BiasAdd:0" data_type: TYPE_FP32 - dims: [ 12, 34, 60 ] + dims: [ 12, 34, 60] }, { - name: "output_cov/Sigmoid" + name: "output_cov/Sigmoid:0" data_type: TYPE_FP32 - dims: [ 3, 34, 60 ] + dims: [ 3, 34, 60] } ] dynamic_batching { } diff --git a/isaac_ros_detectnet/config/quickstart_config.pbtxt b/isaac_ros_detectnet/config/quickstart_config.pbtxt index e7ebe05..e731c0a 100644 --- a/isaac_ros_detectnet/config/quickstart_config.pbtxt +++ b/isaac_ros_detectnet/config/quickstart_config.pbtxt @@ -3,22 +3,22 @@ platform: "tensorrt_plan" max_batch_size: 16 input [ { - name: "input_1" + name: "input_1:0" data_type: TYPE_FP32 format: FORMAT_NCHW - dims: [ 3, 632, 1200 ] + dims: [ 3, 544, 960] } ] output [ { - name: "output_bbox/BiasAdd" + name: "output_bbox/BiasAdd:0" data_type: TYPE_FP32 - dims: [ 12, 40, 75] + dims: [ 12, 34, 60] }, { - name: "output_cov/Sigmoid" + name: "output_cov/Sigmoid:0" data_type: TYPE_FP32 - dims: [ 3, 40, 75] + dims: [ 3, 34, 60] } ] dynamic_batching { } diff --git a/isaac_ros_detectnet/config/realsense_config.pbtxt b/isaac_ros_detectnet/config/realsense_config.pbtxt index cb837b7..7014d49 100644 --- a/isaac_ros_detectnet/config/realsense_config.pbtxt +++ b/isaac_ros_detectnet/config/realsense_config.pbtxt @@ -3,7 +3,7 @@ platform: "tensorrt_plan" max_batch_size: 16 input [ { - name: "input_1" + name: "input_1:0" data_type: TYPE_FP32 format: FORMAT_NCHW dims: [ 3, 480, 640 ] @@ -11,12 +11,12 @@ input [ ] output [ { - name: "output_bbox/BiasAdd" + name: "output_bbox/BiasAdd:0" data_type: TYPE_FP32 dims: [ 12, 30, 40] }, { - name: "output_cov/Sigmoid" + name: "output_cov/Sigmoid:0" data_type: TYPE_FP32 dims: [ 3, 30, 40] } diff --git a/isaac_ros_detectnet/config/zed2_config.pbtxt b/isaac_ros_detectnet/config/zed2_config.pbtxt index 118dce6..4fba106 100644 --- a/isaac_ros_detectnet/config/zed2_config.pbtxt +++ b/isaac_ros_detectnet/config/zed2_config.pbtxt @@ -3,7 +3,7 @@ platform: "tensorrt_plan" max_batch_size: 16 input [ { - name: "input_1" + name: "input_1:0" data_type: TYPE_FP32 format: FORMAT_NCHW dims: [ 3, 720, 1280 ] @@ -11,12 +11,12 @@ input [ ] output [ { - name: "output_bbox/BiasAdd" + name: "output_bbox/BiasAdd:0" data_type: TYPE_FP32 dims: [ 12, 45, 80] }, { - name: "output_cov/Sigmoid" + name: "output_cov/Sigmoid:0" data_type: TYPE_FP32 dims: [ 3, 45, 80] } diff --git a/isaac_ros_detectnet/config/zedx_config.pbtxt b/isaac_ros_detectnet/config/zedx_config.pbtxt index e5cfae9..4a555bd 100644 --- a/isaac_ros_detectnet/config/zedx_config.pbtxt +++ b/isaac_ros_detectnet/config/zedx_config.pbtxt @@ -3,7 +3,7 @@ platform: "tensorrt_plan" max_batch_size: 16 input [ { - name: "input_1" + name: "input_1:0" data_type: TYPE_FP32 format: FORMAT_NCHW dims: [ 3, 1200, 1920 ] @@ -11,12 +11,12 @@ input [ ] output [ { - name: "output_bbox/BiasAdd" + name: "output_bbox/BiasAdd:0" data_type: TYPE_FP32 dims: [ 12, 75, 120] }, { - name: "output_cov/Sigmoid" + name: "output_cov/Sigmoid:0" data_type: TYPE_FP32 dims: [ 3, 75, 120] } diff --git a/isaac_ros_detectnet/launch/isaac_ros_detectnet.launch.py b/isaac_ros_detectnet/launch/isaac_ros_detectnet.launch.py index dcadfc5..a0d2d02 100644 --- a/isaac_ros_detectnet/launch/isaac_ros_detectnet.launch.py +++ b/isaac_ros_detectnet/launch/isaac_ros_detectnet.launch.py @@ -24,6 +24,9 @@ from launch_ros.actions import ComposableNodeContainer from launch_ros.descriptions import ComposableNode +DETECTNET_DEFAULT_WIDTH = 960 +DETECTNET_DEFAULT_HEIGHT = 544 + def generate_launch_description(): """Generate launch description for testing relevant nodes.""" @@ -47,8 +50,8 @@ def generate_launch_description(): launch_arguments={ 'input_image_width': str(1200), 'input_image_height': str(632), - 'network_image_width': str(1200), - 'network_image_height': str(632), + 'network_image_width': str(DETECTNET_DEFAULT_WIDTH), + 'network_image_height': str(DETECTNET_DEFAULT_HEIGHT), 'image_mean': str([0.0, 0.0, 0.0]), 'image_stddev': str([1.0, 1.0, 1.0]), 'enable_padding': 'False', @@ -69,10 +72,10 @@ def generate_launch_description(): 'model_name': 'detectnet', 'model_repository_paths': [model_dir_path], 'input_tensor_names': ['input_tensor'], - 'input_binding_names': ['input_1'], + 'input_binding_names': ['input_1:0'], 'input_tensor_formats': ['nitros_tensor_list_nchw_rgb_f32'], 'output_tensor_names': ['output_cov', 'output_bbox'], - 'output_binding_names': ['output_cov/Sigmoid', 'output_bbox/BiasAdd'], + 'output_binding_names': ['output_cov/Sigmoid:0', 'output_bbox/BiasAdd:0'], 'output_tensor_formats': ['nitros_tensor_list_nhwc_rgb_f32'], 'log_level': 0 }]) diff --git a/isaac_ros_detectnet/launch/isaac_ros_detectnet_core.launch.py b/isaac_ros_detectnet/launch/isaac_ros_detectnet_core.launch.py index c8efbeb..6e47b73 100644 --- a/isaac_ros_detectnet/launch/isaac_ros_detectnet_core.launch.py +++ b/isaac_ros_detectnet/launch/isaac_ros_detectnet_core.launch.py @@ -26,6 +26,9 @@ from launch.launch_description_sources import PythonLaunchDescriptionSource from launch_ros.descriptions import ComposableNode +DETECTNET_DEFAULT_WIDTH = 960 +DETECTNET_DEFAULT_HEIGHT = 544 + class IsaacROSDetectnetLaunchFragment(IsaacROSLaunchFragment): @@ -61,10 +64,10 @@ def get_composable_nodes(interface_specs: Dict[str, Any]) -> Dict[str, Composabl 'model_name': 'peoplenet', 'model_repository_paths': [model_dir_path], 'input_tensor_names': ['input_tensor'], - 'input_binding_names': ['input_1'], + 'input_binding_names': ['input_1:0'], 'input_tensor_formats': ['nitros_tensor_list_nchw_rgb_f32'], 'output_tensor_names': ['output_cov', 'output_bbox'], - 'output_binding_names': ['output_cov/Sigmoid', 'output_bbox/BiasAdd'], + 'output_binding_names': ['output_cov/Sigmoid:0', 'output_bbox/BiasAdd:0'], 'output_tensor_formats': ['nitros_tensor_list_nhwc_rgb_f32'], 'log_level': 0 }]), @@ -81,8 +84,8 @@ def get_launch_actions(interface_specs: Dict[str, Any]) -> \ launch_arguments={ 'input_image_width': str(interface_specs['camera_resolution']['width']), 'input_image_height': str(interface_specs['camera_resolution']['height']), - 'network_image_width': str(interface_specs['camera_resolution']['width']), - 'network_image_height': str(interface_specs['camera_resolution']['height']), + 'network_image_width': str(DETECTNET_DEFAULT_WIDTH), + 'network_image_height': str(DETECTNET_DEFAULT_HEIGHT), 'image_mean': str([0.0, 0.0, 0.0]), 'image_stddev': str([1.0, 1.0, 1.0]), 'attach_to_shared_component_container': 'True', diff --git a/isaac_ros_detectnet/launch/isaac_ros_detectnet_isaac_sim.launch.py b/isaac_ros_detectnet/launch/isaac_ros_detectnet_isaac_sim.launch.py index 1ce8d7e..d376a7e 100644 --- a/isaac_ros_detectnet/launch/isaac_ros_detectnet_isaac_sim.launch.py +++ b/isaac_ros_detectnet/launch/isaac_ros_detectnet_isaac_sim.launch.py @@ -24,6 +24,9 @@ from launch_ros.actions import ComposableNodeContainer, Node from launch_ros.descriptions import ComposableNode +DETECTNET_DEFAULT_WIDTH = 960 +DETECTNET_DEFAULT_HEIGHT = 544 + def generate_launch_description(): """Generate launch description for testing relevant nodes.""" @@ -43,8 +46,8 @@ def generate_launch_description(): plugin='nvidia::isaac_ros::image_proc::ResizeNode', name='image_resize_node_left', parameters=[{ - 'output_width': 1280, - 'output_height': 720, + 'output_width': DETECTNET_DEFAULT_WIDTH, + 'output_height': DETECTNET_DEFAULT_HEIGHT, 'encoding_desired': 'rgb8', }], remappings=[ @@ -61,10 +64,10 @@ def generate_launch_description(): 'dnn_image_encoder.launch.py')] ), launch_arguments={ - 'input_image_width': str(1280), - 'input_image_height': str(720), - 'network_image_width': str(1280), - 'network_image_height': str(720), + 'input_image_width': str(DETECTNET_DEFAULT_WIDTH), + 'input_image_height': str(DETECTNET_DEFAULT_HEIGHT), + 'network_image_width': str(DETECTNET_DEFAULT_WIDTH), + 'network_image_height': str(DETECTNET_DEFAULT_HEIGHT), 'image_mean': str([0.0, 0.0, 0.0]), 'image_stddev': str([1.0, 1.0, 1.0]), 'enable_padding': 'False', @@ -85,10 +88,10 @@ def generate_launch_description(): 'model_name': 'peoplenet', 'model_repository_paths': [model_dir_path], 'input_tensor_names': ['input_tensor'], - 'input_binding_names': ['input_1'], + 'input_binding_names': ['input_1:0'], 'input_tensor_formats': ['nitros_tensor_list_nchw_rgb_f32'], 'output_tensor_names': ['output_cov', 'output_bbox'], - 'output_binding_names': ['output_cov/Sigmoid', 'output_bbox/BiasAdd'], + 'output_binding_names': ['output_cov/Sigmoid:0', 'output_bbox/BiasAdd:0'], 'output_tensor_formats': ['nitros_tensor_list_nhwc_rgb_f32'], 'log_level': 0 }]) @@ -117,8 +120,7 @@ def generate_launch_description(): package='isaac_ros_detectnet', executable='isaac_ros_detectnet_visualizer.py', name='detectnet_visualizer', - remappings=[('image', 'front_stereo_camera/left/image_resize')] - + remappings=[('image', 'detectnet_encoder/resize/image')] ) rqt_image_view_node = Node( diff --git a/isaac_ros_detectnet/package.xml b/isaac_ros_detectnet/package.xml index 2f483ff..3fbdce2 100644 --- a/isaac_ros_detectnet/package.xml +++ b/isaac_ros_detectnet/package.xml @@ -21,7 +21,7 @@ SPDX-License-Identifier: Apache-2.0 isaac_ros_detectnet - 3.1.0 + 3.2.0 DetectNet model processing Isaac ROS Maintainers diff --git a/isaac_ros_detectnet/scripts/setup_model.sh b/isaac_ros_detectnet/scripts/setup_model.sh index 756f78a..1820055 100755 --- a/isaac_ros_detectnet/scripts/setup_model.sh +++ b/isaac_ros_detectnet/scripts/setup_model.sh @@ -21,19 +21,20 @@ # inside the Docker container # default arguments -MODEL_LINK="https://api.ngc.nvidia.com/v2/models/nvidia/tao/peoplenet/versions/deployable_quantized_v2.5/zip" -MODEL_FILE_NAME="resnet34_peoplenet_int8.etlt" -HEIGHT="632" -WIDTH="1200" +MODEL_LINK="https://api.ngc.nvidia.com/v2/models/nvidia/tao/peoplenet/versions/deployable_quantized_onnx_v2.6.3/zip" +MODEL_FILE_NAME="resnet34_peoplenet.onnx" +HEIGHT="544" +WIDTH="960" CONFIG_FILE="peoplenet_config.pbtxt" PRECISION="int8" -OUTPUT_LAYERS="output_cov/Sigmoid,output_bbox/BiasAdd" +MAX_BATCH_SIZE="16" function print_parameters() { echo echo "***************************" echo using parameters: echo MODEL_LINK : $MODEL_LINK + echo MAX_BATCH_SIZE : $MAX_BATCH_SIZE echo HEIGHT : $HEIGHT echo WIDTH : $WIDTH echo CONFIG_FILE : $CONFIG_FILE @@ -69,39 +70,40 @@ extract_model_name() { } function setup_model() { - # Download pre-trained ETLT model to appropriate directory + # Download pre-traine ONNX model to appropriate directory # Extract model names from URLs model_name_from_model_link=$(extract_model_name "$MODEL_LINK") + OUTPUT_PATH=${ISAAC_ROS_WS}/isaac_ros_assets/models/$model_name_from_model_link echo "Model name from model link: $model_name_from_model_link" - echo Creating Directory : ${ISAAC_ROS_WS}/isaac_ros_assets/models/$model_name_from_model_link/1 - rm -rf ${ISAAC_ROS_WS}/isaac_ros_assets/models/$model_name_from_model_link - mkdir -p ${ISAAC_ROS_WS}/isaac_ros_assets/models/$model_name_from_model_link/1 - cd ${ISAAC_ROS_WS}/isaac_ros_assets/models/$model_name_from_model_link/1 - echo Downloading .etlt file from $MODEL_LINK + echo Creating Directory : "${OUTPUT_PATH}/1" + rm -rf ${OUTPUT_PATH} + mkdir -p ${OUTPUT_PATH}/1 + cd ${OUTPUT_PATH}/1 + echo Downloading .onnx file from $MODEL_LINK echo From $MODEL_LINK wget --content-disposition $MODEL_LINK -O model.zip - echo Unziping network model file .etlt + echo Unziping network model file .onnx unzip -o model.zip - echo Checking if labels.txt exists + echo Checking if labels.txt exists check_labels_files - echo Converting .etlt to a TensorRT Engine Plan - # This is the key for the provided pretrained model - # replace with your own key when using a model trained by any other means - export PRETRAINED_MODEL_ETLT_KEY='tlt_encode' + echo Converting .onnx to a TensorRT Engine Plan + # if model doesnt have labels.txt file, then create one manually # create custom model - /opt/nvidia/tao/tao-converter \ - -k $PRETRAINED_MODEL_ETLT_KEY \ - -d 3,$HEIGHT,$WIDTH \ - -p input_1,1x3x$HEIGHTx$WIDTH,1x3x$HEIGHTx$WIDTH,1x3x$HEIGHTx$WIDTH \ - -t $PRECISION \ - -e model.plan \ - -o $OUTPUT_LAYERS\ - $MODEL_FILE_NAME - echo Copying .pbtxt config file to ${ISAAC_ROS_WS}/isaac_ros_assets/models/$model_name_from_model_link + /usr/src/tensorrt/bin/trtexec \ + --maxShapes="input_1:0":${MAX_BATCH_SIZE}x3x${HEIGHT}x${WIDTH} \ + --minShapes="input_1:0":1x3x${HEIGHT}x${WIDTH} \ + --optShapes="input_1:0":1x3x${HEIGHT}x${WIDTH} \ + --$PRECISION \ + --calib="${OUTPUT_PATH}/1/resnet34_peoplenet_int8.txt" \ + --onnx="${OUTPUT_PATH}/1/${MODEL_FILE_NAME}" \ + --saveEngine="${OUTPUT_PATH}/1/model.plan" \ + --skipInference + + echo Copying .pbtxt config file to ${OUTPUT_PATH} export ISAAC_ROS_DETECTNET_PATH=$(ros2 pkg prefix isaac_ros_detectnet --share) cp $ISAAC_ROS_DETECTNET_PATH/config/$CONFIG_FILE \ - ${ISAAC_ROS_WS}/isaac_ros_assets/models/$model_name_from_model_link/config.pbtxt + ${OUTPUT_PATH}/config.pbtxt echo Completed quickstart setup } @@ -118,8 +120,8 @@ function show_help() { } # Get command line arguments -OPTIONS=m:mfn:hgt:wid:c:p:ol:h -LONGOPTS=model-link:,model-file-name:,height:,width:,config-file:,precision:,output-layers:,help +OPTIONS=m:mfn:b:p:ol:h +LONGOPTS=model-link:,model-file-name:,max-batch-size:,config-file:,precision:,output-layers:,help PARSED=$(getopt --options=$OPTIONS --longoptions=$LONGOPTS --name "$0" -- "$@") eval set -- "$PARSED" @@ -134,14 +136,6 @@ while true; do MODEL_FILE_NAME="$2" shift 2 ;; - --height) - HEIGHT="$2" - shift 2 - ;; - --width) - WIDTH="$2" - shift 2 - ;; -c|--config-file) CONFIG_FILE="$2" shift 2 @@ -150,8 +144,8 @@ while true; do PRECISION="$2" shift 2 ;; - -ol|--output-layers) - OUTPUT_LAYERS="$2" + -b|--max-batch-size) + MAX_BATCH_SIZE="$2" shift 2 ;; -h|--help) diff --git a/isaac_ros_detectnet/test/dummy_model/.gitignore b/isaac_ros_detectnet/test/dummy_model/.gitignore index eb8715e..e8fea18 100644 --- a/isaac_ros_detectnet/test/dummy_model/.gitignore +++ b/isaac_ros_detectnet/test/dummy_model/.gitignore @@ -1 +1,2 @@ -**/model.plan \ No newline at end of file +**/model.plan +**/model.onnx \ No newline at end of file diff --git a/isaac_ros_detectnet/test/dummy_model/detectnet/1/.gitkeep b/isaac_ros_detectnet/test/dummy_model/detectnet/1/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/isaac_ros_detectnet/test/dummy_model/detectnet/1/resnet18_detector.etlt b/isaac_ros_detectnet/test/dummy_model/detectnet/1/resnet18_detector.etlt deleted file mode 100644 index 50b7689..0000000 --- a/isaac_ros_detectnet/test/dummy_model/detectnet/1/resnet18_detector.etlt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:984c3bf856d11b3fae0b0c5f4040fe6d8709799397b52c660ca0216ac09e35ec -size 44874341 diff --git a/isaac_ros_detectnet/test/dummy_model/detectnet/config.pbtxt b/isaac_ros_detectnet/test/dummy_model/detectnet/config.pbtxt index d24c6c0..e91edf9 100644 --- a/isaac_ros_detectnet/test/dummy_model/detectnet/config.pbtxt +++ b/isaac_ros_detectnet/test/dummy_model/detectnet/config.pbtxt @@ -3,7 +3,7 @@ platform: "tensorrt_plan" max_batch_size: 16 input [ { - name: "input_1" + name: "input_1:0" data_type: TYPE_FP32 format: FORMAT_NCHW dims: [ 3, 368, 640 ] @@ -11,12 +11,12 @@ input [ ] output [ { - name: "output_bbox/BiasAdd" + name: "output_bbox/BiasAdd:0" data_type: TYPE_FP32 dims: [ 8, 23, 40 ] }, { - name: "output_cov/Sigmoid" + name: "output_cov/Sigmoid:0" data_type: TYPE_FP32 dims: [ 2, 23, 40 ] } diff --git a/isaac_ros_detectnet/test/isaac_ros_detectnet_pol_test.py b/isaac_ros_detectnet/test/isaac_ros_detectnet_pol_test.py index 47d85aa..6277ba4 100644 --- a/isaac_ros_detectnet/test/isaac_ros_detectnet_pol_test.py +++ b/isaac_ros_detectnet/test/isaac_ros_detectnet_pol_test.py @@ -29,23 +29,22 @@ import os import pathlib -from pprint import pprint import subprocess import time from ament_index_python.packages import get_package_share_directory -from isaac_ros_test import IsaacROSBaseTest, JSONConversion +from isaac_ros_test import IsaacROSBaseTest, JSONConversion, MockModelGenerator from launch.actions import IncludeLaunchDescription from launch.launch_description_sources import PythonLaunchDescriptionSource from launch_ros.actions.composable_node_container import ComposableNodeContainer from launch_ros.descriptions.composable_node import ComposableNode - import pytest import rclpy - from sensor_msgs.msg import CameraInfo, Image +import torch from vision_msgs.msg import Detection2DArray + _TEST_CASE_NAMESPACE = 'detectnet_node_test' @@ -56,30 +55,42 @@ def generate_test_description(): model_dir_path = launch_dir_path + '/dummy_model' model_name = 'detectnet' model_version = 1 + onnx_path = f'{model_dir_path}/{model_name}/{model_version}/model.onnx' engine_file_path = f'{model_dir_path}/{model_name}/{model_version}/model.plan' + # Generate a mock model with DetectNet-like I/O + MockModelGenerator.generate( + input_bindings=[ + MockModelGenerator.Binding('input_1:0', [-1, 3, 368, 640], torch.float32) + ], + output_bindings=[ + MockModelGenerator.Binding('output_bbox/BiasAdd:0', [-1, 8, 23, 40], torch.float32), + MockModelGenerator.Binding('output_cov/Sigmoid:0', [-1, 2, 23, 40], torch.float32) + ], + output_onnx_path=onnx_path + ) + # Read labels from text file labels_file_path = f'{model_dir_path}/{model_name}/labels.txt' with open(labels_file_path, 'r') as fd: label_list = fd.read().strip().splitlines() - # Generate engine file using tao-converter - print('Generating engine file using tao-converter...') - tao_converter_args = [ - '-k', '"object-detection-from-sim-pipeline"', - '-d', '3,368,640', - '-t', 'fp16', - '-p', 'input_1,1x3x368x640,1x3x368x640,1x3x368x640', - '-e', engine_file_path, - '-o', 'output_cov/Sigmoid,output_bbox/BiasAdd', - f'{model_dir_path}/{model_name}/1/resnet18_detector.etlt' + # Generate engine file using trtexec + print('Generating engine file using trtexec...') + trtexec_args = [ + '--maxShapes=input_1:0:1x3x368x640', + '--minShapes=input_1:0:1x3x368x640', + '--optShapes=input_1:0:1x3x368x640', + f'--onnx={onnx_path}', + f'--saveEngine={engine_file_path}', + '--fp16', ] - tao_converter_executable = '/opt/nvidia/tao/tao-converter' + trtexec_executable = '/usr/src/tensorrt/bin/trtexec' print('Running command:\n' + - ' '.join([tao_converter_executable] + tao_converter_args)) + ' '.join([trtexec_executable] + trtexec_args)) start_time = time.time() result = subprocess.run( - [tao_converter_executable] + tao_converter_args, + [trtexec_executable] + trtexec_args, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE @@ -120,10 +131,10 @@ def generate_test_description(): 'model_name': 'detectnet', 'model_repository_paths': [model_dir_path], 'input_tensor_names': ['input_tensor'], - 'input_binding_names': ['input_1'], + 'input_binding_names': ['input_1:0'], 'input_tensor_formats': ['nitros_tensor_list_nchw_rgb_f32'], 'output_tensor_names': ['output_cov', 'output_bbox'], - 'output_binding_names': ['output_cov/Sigmoid', 'output_bbox/BiasAdd'], + 'output_binding_names': ['output_cov/Sigmoid:0', 'output_bbox/BiasAdd:0'], 'output_tensor_formats': ['nitros_tensor_list_nhwc_rgb_f32'], 'log_level': 0 }]) @@ -203,18 +214,6 @@ def test_image_detection(self, test_folder): camera_info = CameraInfo() camera_info.header = image.header camera_info.distortion_model = 'plumb_bob' - ground_truth = open(test_folder.joinpath( - 'expected_detections.txt'), 'r') - expected_detections = [] - - for ground_detection in ground_truth.readlines(): - ground_detection_split = ground_detection.split() - gtd = [float(ground_detection_split[1]), float(ground_detection_split[2]), - float(ground_detection_split[3]), float(ground_detection_split[4])] - expected_detections.append( - {'width': int(gtd[0]), 'height': int(gtd[1]), - 'center': {'x': int(gtd[2]), 'y': int(gtd[3])}} - ) TIMEOUT = 60 end_time = time.time() + TIMEOUT @@ -231,28 +230,6 @@ def test_image_detection(self, test_folder): self.assertTrue( done, "Didn't receive output on detectnet/detections topic!") - detection_list = received_messages['detectnet/detections'].detections - - pprint(detection_list) - - pixel_tolerance = 2.0 - - self.assertGreater(len(detection_list), 0, 'No detections in detection list!') - - self.assertAlmostEqual(detection_list[0].bbox.size_x, - expected_detections[0]['width'], - None, 'Received incorrect width', pixel_tolerance) - self.assertAlmostEqual(detection_list[0].bbox.size_y, - expected_detections[0]['height'], - None, 'Received incorrect height', pixel_tolerance) - self.assertAlmostEqual( - detection_list[0].bbox.center.position.x, - expected_detections[0]['center']['x'], - None, 'Received incorrect center', pixel_tolerance) - self.assertAlmostEqual( - detection_list[0].bbox.center.position.y, - expected_detections[0]['center']['y'], - None, 'Received incorrect center', pixel_tolerance) finally: self.node.destroy_subscription(detectnet_detections) self.node.destroy_publisher(image_pub) diff --git a/isaac_ros_peoplenet_models_install/CMakeLists.txt b/isaac_ros_peoplenet_models_install/CMakeLists.txt new file mode 100644 index 0000000..ef1ed35 --- /dev/null +++ b/isaac_ros_peoplenet_models_install/CMakeLists.txt @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +# Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.22.1) +project(isaac_ros_peoplenet_models_install) + +find_package(ament_cmake_auto REQUIRED) +ament_auto_find_build_dependencies() + +# Download and install models to the asset folder (outside the build) +install_isaac_ros_asset(install_peoplenet_amr_rs) + +# Install the installation scripts such that the models can be installed when not building from source +install(PROGRAMS asset_scripts/install_peoplenet_amr_rs.sh DESTINATION lib/${PROJECT_NAME}) + +# Embed versioning information into installed files +ament_index_get_resource(ISAAC_ROS_COMMON_CMAKE_PATH isaac_ros_common_cmake_path isaac_ros_common) +include("${ISAAC_ROS_COMMON_CMAKE_PATH}/isaac_ros_common-version-info.cmake") +generate_version_info(${PROJECT_NAME}) + +ament_auto_package(INSTALL_TO_SHARE) diff --git a/isaac_ros_peoplenet_models_install/asset_scripts/install_peoplenet_amr_rs.sh b/isaac_ros_peoplenet_models_install/asset_scripts/install_peoplenet_amr_rs.sh new file mode 100755 index 0000000..af0453c --- /dev/null +++ b/isaac_ros_peoplenet_models_install/asset_scripts/install_peoplenet_amr_rs.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# NVIDIA CORPORATION and its licensors retain all intellectual property +# and proprietary rights in and to this software, related documentation +# and any modifications thereto. Any use, reproduction, disclosure or +# distribution of this software and related documentation without an express +# license agreement from NVIDIA CORPORATION is strictly prohibited. + +# Download and TRT-compile PeopleNet models. +# * Models will be stored in the isaac_ros_assets dir +# * The script must be called with the --eula argument prior to downloading. + +set -e + +ASSET_NAME="rsu_rs_480_640_mask" +EULA_URL="https://catalog.ngc.nvidia.com/orgs/nvidia/teams/isaac/models/optimized_peoplenet_amr" +ASSET_DIR="${ISAAC_ROS_WS}/isaac_ros_assets/models/peoplenet/${ASSET_NAME}" +ASSET_INSTALL_PATHS="${ASSET_DIR}/1/model.plan" +MODEL_URL="https://api.ngc.nvidia.com/v2/models/org/nvidia/team/isaac/optimized_peoplenet_amr/v1_1_optimized_mask/files?redirect=true&path=rsu_rs_480_640_mask.onnx" +source "isaac_ros_asset_eula.sh" + +mkdir -p $(dirname "$ASSET_INSTALL_PATHS") + +wget "${MODEL_URL}" -O "${ASSET_DIR}/model.onnx" + +echo "Converting PeopleNet AMR onnx file to plan file." +/usr/src/tensorrt/bin/trtexec \ + --maxShapes="preprocess/input_1:0":1x480x640x3 \ + --minShapes="preprocess/input_1:0":1x480x640x3 \ + --optShapes="preprocess/input_1:0":1x480x640x3 \ + --fp16 \ + --saveEngine="${ASSET_INSTALL_PATHS}" \ + --onnx="${ASSET_DIR}/model.onnx" + +config_file_text=$( + cat <${ASSET_DIR}/config.pbtxt + diff --git a/isaac_ros_peoplenet_models_install/package.xml b/isaac_ros_peoplenet_models_install/package.xml new file mode 100644 index 0000000..fd842d6 --- /dev/null +++ b/isaac_ros_peoplenet_models_install/package.xml @@ -0,0 +1,40 @@ + + + + + + + isaac_ros_peoplenet_models_install + 3.2.0 + Scripts for installing people detection models based on PeopleNet + + Isaac ROS Maintainers + Apache-2.0 + https://developer.nvidia.com/isaac-ros-gems/ + Xinjie Yao + + ament_cmake + + isaac_ros_common + isaac_ros_detectnet + + + ament_cmake + + diff --git a/isaac_ros_rtdetr/CMakeLists.txt b/isaac_ros_rtdetr/CMakeLists.txt index ab91928..04d47f7 100644 --- a/isaac_ros_rtdetr/CMakeLists.txt +++ b/isaac_ros_rtdetr/CMakeLists.txt @@ -56,4 +56,10 @@ install(PROGRAMS DESTINATION lib/${PROJECT_NAME} ) + +# Embed versioning information into installed files +ament_index_get_resource(ISAAC_ROS_COMMON_CMAKE_PATH isaac_ros_common_cmake_path isaac_ros_common) +include("${ISAAC_ROS_COMMON_CMAKE_PATH}/isaac_ros_common-version-info.cmake") +generate_version_info(${PROJECT_NAME}) + ament_auto_package(INSTALL_TO_SHARE launch) diff --git a/isaac_ros_rtdetr/launch/isaac_ros_rtdetr_isaac_sim.launch.py b/isaac_ros_rtdetr/launch/isaac_ros_rtdetr_isaac_sim.launch.py index 28de1df..b0cd33d 100644 --- a/isaac_ros_rtdetr/launch/isaac_ros_rtdetr_isaac_sim.launch.py +++ b/isaac_ros_rtdetr/launch/isaac_ros_rtdetr_isaac_sim.launch.py @@ -51,7 +51,7 @@ def generate_launch_description(): DeclareLaunchArgument( 'ess_depth_threshold', - default_value='0.35', + default_value='0.4', description='Threshold value ranges between 0.0 and 1.0 ' 'for filtering disparity with confidence.'), diff --git a/isaac_ros_rtdetr/package.xml b/isaac_ros_rtdetr/package.xml index ef817bf..3e565c5 100644 --- a/isaac_ros_rtdetr/package.xml +++ b/isaac_ros_rtdetr/package.xml @@ -21,7 +21,7 @@ SPDX-License-Identifier: Apache-2.0 isaac_ros_rtdetr - 3.1.0 + 3.2.0 RT-DETR model processing Isaac ROS Maintainers diff --git a/isaac_ros_rtdetr/src/rtdetr_decoder_node.cpp b/isaac_ros_rtdetr/src/rtdetr_decoder_node.cpp index b1d3f91..32dd399 100644 --- a/isaac_ros_rtdetr/src/rtdetr_decoder_node.cpp +++ b/isaac_ros_rtdetr/src/rtdetr_decoder_node.cpp @@ -75,7 +75,7 @@ void RtDetrDecoderNode::InputCallback( const nvidia::isaac_ros::nitros::NitrosTensorListView & msg) { // Bring labels, boxes, and scores back to CPU - auto labels = TensorToVector(msg, labels_tensor_name_, stream_); + auto labels = TensorToVector(msg, labels_tensor_name_, stream_); auto boxes = TensorToVector(msg, boxes_tensor_name_, stream_); auto scores = TensorToVector(msg, scores_tensor_name_, stream_); cudaStreamSynchronize(stream_); diff --git a/isaac_ros_rtdetr/src/rtdetr_preprocessor_node.cpp b/isaac_ros_rtdetr/src/rtdetr_preprocessor_node.cpp index dcfaced..64e2d93 100644 --- a/isaac_ros_rtdetr/src/rtdetr_preprocessor_node.cpp +++ b/isaac_ros_rtdetr/src/rtdetr_preprocessor_node.cpp @@ -79,10 +79,9 @@ void RtDetrPreprocessorNode::InputCallback( output_image_buffer, input_image_tensor.GetBuffer(), input_image_tensor.GetTensorSize(), cudaMemcpyDefault, stream_); - // The model only requires the larger of the 2 dimensions. - // Narrowing conversion required because model expects int32_t, but ROS 2 only supports int64_t - int32_t image_size = static_cast(std::max(image_height_, image_width_)); - int32_t output_size[2]{image_size, image_size}; + int64_t image_size = std::max(image_height_, image_width_); + + int64_t output_size[2]{image_size, image_size}; void * output_size_buffer; cudaMallocAsync(&output_size_buffer, sizeof(output_size), stream_); cudaMemcpyAsync(output_size_buffer, output_size, sizeof(output_size), cudaMemcpyDefault, stream_); @@ -107,7 +106,7 @@ void RtDetrPreprocessorNode::InputCallback( ( nvidia::isaac_ros::nitros::NitrosTensorBuilder() .WithShape({1, 2}) - .WithDataType(nvidia::isaac_ros::nitros::NitrosDataType::kInt32) + .WithDataType(nvidia::isaac_ros::nitros::NitrosDataType::kInt64) .WithData(output_size_buffer) .Build() ) diff --git a/isaac_ros_rtdetr/test/dummy_model/.gitignore b/isaac_ros_rtdetr/test/dummy_model/.gitignore new file mode 100644 index 0000000..e8fea18 --- /dev/null +++ b/isaac_ros_rtdetr/test/dummy_model/.gitignore @@ -0,0 +1,2 @@ +**/model.plan +**/model.onnx \ No newline at end of file diff --git a/isaac_ros_rtdetr/test/dummy_model/rtdetr_dummy_pol.onnx b/isaac_ros_rtdetr/test/dummy_model/rtdetr_dummy_pol.onnx deleted file mode 100644 index f28919e..0000000 --- a/isaac_ros_rtdetr/test/dummy_model/rtdetr_dummy_pol.onnx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3b09bb405b359b3cf8f6ea718621726cb1c850d572ad2bb82ca8b450c6b1d11f -size 4933199 diff --git a/isaac_ros_rtdetr/test/isaac_ros_rtdetr_pol_test.py b/isaac_ros_rtdetr/test/isaac_ros_rtdetr_pol_test.py index 1aad664..bcd61c0 100644 --- a/isaac_ros_rtdetr/test/isaac_ros_rtdetr_pol_test.py +++ b/isaac_ros_rtdetr/test/isaac_ros_rtdetr_pol_test.py @@ -31,26 +31,39 @@ import pathlib import time -from isaac_ros_test import IsaacROSBaseTest, JSONConversion +from isaac_ros_test import IsaacROSBaseTest, JSONConversion, MockModelGenerator from launch_ros.actions.composable_node_container import ComposableNodeContainer from launch_ros.descriptions.composable_node import ComposableNode - import pytest import rclpy - from sensor_msgs.msg import CameraInfo, Image +import torch from vision_msgs.msg import Detection2DArray -MODEL_FILE_NAME = 'rtdetr_dummy_pol.onnx' +MODEL_ONNX_PATH = '/tmp/model.onnx' MODEL_GENERATION_TIMEOUT_SEC = 300 INIT_WAIT_SEC = 10 -MODEL_PATH = '/tmp/rtdetr_dummy_pol.plan' +MODEL_PLAN_PATH = '/tmp/model.plan' @pytest.mark.rostest def generate_test_description(): """Generate launch description for testing relevant nodes.""" + # Generate a dummy model with RT-DETR-like I/O + MockModelGenerator.generate( + input_bindings=[ + MockModelGenerator.Binding('images', [-1, 3, 640, 640], torch.float32), + MockModelGenerator.Binding('orig_target_sizes', [-1, 2], torch.int64) + ], + output_bindings=[ + MockModelGenerator.Binding('labels', [-1, 300], torch.int64), + MockModelGenerator.Binding('boxes', [-1, 300, 4], torch.float32), + MockModelGenerator.Binding('scores', [-1, 300], torch.float32) + ], + output_onnx_path=MODEL_ONNX_PATH + ) + resize_node = ComposableNode( name='resize_node', package='isaac_ros_image_proc', @@ -156,8 +169,8 @@ def generate_test_description(): plugin='nvidia::isaac_ros::dnn_inference::TensorRTNode', namespace=IsaacROSRtDetrPOLTest.generate_namespace(), parameters=[{ - 'model_file_path': f'{os.path.dirname(__file__)}/dummy_model/{MODEL_FILE_NAME}', - 'engine_file_path': MODEL_PATH, + 'model_file_path': MODEL_ONNX_PATH, + 'engine_file_path': MODEL_PLAN_PATH, 'input_tensor_names': ['images', 'orig_target_sizes'], 'input_binding_names': ['images', 'orig_target_sizes'], 'output_binding_names': ['labels', 'boxes', 'scores'], @@ -203,7 +216,7 @@ def test_object_detection(self, test_folder): self.node._logger.info(f'Generating model (timeout={MODEL_GENERATION_TIMEOUT_SEC}s)') start_time = time.time() wait_cycles = 1 - while not os.path.isfile(MODEL_PATH): + while not os.path.isfile(MODEL_PLAN_PATH): time_diff = time.time() - start_time if time_diff > MODEL_GENERATION_TIMEOUT_SEC: self.fail('Model generation timed out') diff --git a/isaac_ros_yolov8/CMakeLists.txt b/isaac_ros_yolov8/CMakeLists.txt index e48fdc9..eea357d 100644 --- a/isaac_ros_yolov8/CMakeLists.txt +++ b/isaac_ros_yolov8/CMakeLists.txt @@ -56,6 +56,7 @@ if(BUILD_TESTING) endif() find_package(launch_testing_ament_cmake REQUIRED) + add_launch_test(test/isaac_ros_yolov8_decoder_node_pol.py TIMEOUT "600") endif() # Visualizer python scripts @@ -66,4 +67,10 @@ install(PROGRAMS DESTINATION lib/${PROJECT_NAME} ) + +# Embed versioning information into installed files +ament_index_get_resource(ISAAC_ROS_COMMON_CMAKE_PATH isaac_ros_common_cmake_path isaac_ros_common) +include("${ISAAC_ROS_COMMON_CMAKE_PATH}/isaac_ros_common-version-info.cmake") +generate_version_info(${PROJECT_NAME}) + ament_auto_package(INSTALL_TO_SHARE launch) diff --git a/isaac_ros_yolov8/include/isaac_ros_yolov8/yolov8_decoder_node.hpp b/isaac_ros_yolov8/include/isaac_ros_yolov8/yolov8_decoder_node.hpp index d2e59c3..0e5c1c3 100644 --- a/isaac_ros_yolov8/include/isaac_ros_yolov8/yolov8_decoder_node.hpp +++ b/isaac_ros_yolov8/include/isaac_ros_yolov8/yolov8_decoder_node.hpp @@ -59,6 +59,7 @@ class YoloV8DecoderNode : public rclcpp::Node // YOLOv8 Decoder Parameters double confidence_threshold_{}; double nms_threshold_{}; + int64_t num_classes_{}; }; } // namespace yolov8 diff --git a/isaac_ros_yolov8/launch/isaac_ros_yolov8_core.launch.py b/isaac_ros_yolov8/launch/isaac_ros_yolov8_core.launch.py index afafc40..0a96847 100644 --- a/isaac_ros_yolov8/launch/isaac_ros_yolov8_core.launch.py +++ b/isaac_ros_yolov8/launch/isaac_ros_yolov8_core.launch.py @@ -81,8 +81,11 @@ def get_launch_actions(interface_specs: Dict[str, Any]) -> \ network_image_width = LaunchConfiguration('network_image_width') network_image_height = LaunchConfiguration('network_image_height') + input_encoding = LaunchConfiguration('input_encoding') image_mean = LaunchConfiguration('image_mean') image_stddev = LaunchConfiguration('image_stddev') + image_input_topic = LaunchConfiguration('image_input_topic') + camera_info_input_topic = LaunchConfiguration('camera_info_input_topic') encoder_dir = get_package_share_directory('isaac_ros_dnn_image_encoder') @@ -97,6 +100,11 @@ def get_launch_actions(interface_specs: Dict[str, Any]) -> \ default_value='640', description='The input image height that the network expects' ), + 'input_encoding': DeclareLaunchArgument( + 'input_encoding', + default_value='rgb8', + description='The desired image format encoding' + ), 'image_mean': DeclareLaunchArgument( 'image_mean', default_value='[0.0, 0.0, 0.0]', @@ -127,6 +135,16 @@ def get_launch_actions(interface_specs: Dict[str, Any]) -> \ default_value='["images"]', description='A list of input tensor binding names (specified by model)' ), + 'image_input_topic': DeclareLaunchArgument( + 'image_input_topic', + default_value='/image_rect', + description='Input image topic name' + ), + 'camera_info_input_topic': DeclareLaunchArgument( + 'camera_info_input_topic', + default_value='/camera_info_rect', + description='Input camera info topic name' + ), 'output_tensor_names': DeclareLaunchArgument( 'output_tensor_names', default_value='["output_tensor"]', @@ -157,6 +175,7 @@ def get_launch_actions(interface_specs: Dict[str, Any]) -> \ default_value='0.45', description='NMS IOU threshold' ), + 'yolov8_encoder_launch': IncludeLaunchDescription( PythonLaunchDescriptionSource( [os.path.join(encoder_dir, 'launch', 'dnn_image_encoder.launch.py')] @@ -171,9 +190,10 @@ def get_launch_actions(interface_specs: Dict[str, Any]) -> \ 'attach_to_shared_component_container': 'True', 'component_container_name': '/isaac_ros_examples/container', 'dnn_image_encoder_namespace': 'yolov8_encoder', - 'image_input_topic': '/image_rect', - 'camera_info_input_topic': '/camera_info_rect', + 'image_input_topic': image_input_topic, + 'camera_info_input_topic': camera_info_input_topic, 'tensor_output_topic': '/tensor_pub', + 'input_encoding': input_encoding, }.items(), ), } diff --git a/isaac_ros_yolov8/package.xml b/isaac_ros_yolov8/package.xml index e905ddc..3ce0754 100644 --- a/isaac_ros_yolov8/package.xml +++ b/isaac_ros_yolov8/package.xml @@ -21,7 +21,7 @@ isaac_ros_yolov8 - 3.1.0 + 3.2.0 Isaac ROS YOLOv8 decoding Isaac ROS Maintainers @@ -43,7 +43,6 @@ tf2_msgs isaac_ros_nitros isaac_ros_managed_nitros - isaac_ros_nitros_interfaces isaac_ros_tensor_list_interfaces isaac_ros_nitros_tensor_list_type python3-opencv diff --git a/isaac_ros_yolov8/src/yolov8_decoder_node.cpp b/isaac_ros_yolov8/src/yolov8_decoder_node.cpp index b01f71c..9bd85a5 100644 --- a/isaac_ros_yolov8/src/yolov8_decoder_node.cpp +++ b/isaac_ros_yolov8/src/yolov8_decoder_node.cpp @@ -54,7 +54,8 @@ YoloV8DecoderNode::YoloV8DecoderNode(const rclcpp::NodeOptions options) "detections_output", 50)}, tensor_name_{declare_parameter("tensor_name", "output_tensor")}, confidence_threshold_{declare_parameter("confidence_threshold", 0.25)}, - nms_threshold_{declare_parameter("nms_threshold", 0.45)} + nms_threshold_{declare_parameter("nms_threshold", 0.45)}, + num_classes_{declare_parameter("num_classes", 80)} {} YoloV8DecoderNode::~YoloV8DecoderNode() = default; @@ -73,7 +74,6 @@ void YoloV8DecoderNode::InputCallback(const nvidia::isaac_ros::nitros::NitrosTen std::vector classes; // Output dimensions = [1, 84, 8400] - int num_classes = 80; int out_dim = 8400; float * results_data = reinterpret_cast(results_vector.data()); @@ -89,7 +89,7 @@ void YoloV8DecoderNode::InputCallback(const nvidia::isaac_ros::nitros::NitrosTen float height = h; std::vector conf; - for (int j = 0; j < num_classes; j++) { + for (int j = 0; j < num_classes_; j++) { conf.push_back(*(results_data + (out_dim * (4 + j)) + i)); } diff --git a/isaac_ros_yolov8/test/dummy_model/yolov8/dummy_yolov8s.onnx b/isaac_ros_yolov8/test/dummy_model/yolov8/dummy_yolov8s.onnx new file mode 100644 index 0000000..767683b --- /dev/null +++ b/isaac_ros_yolov8/test/dummy_model/yolov8/dummy_yolov8s.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:193aa828d9f71c43768d3a16755c9cb8b04b4dd0d08a9130529a8e17681df748 +size 10561034 diff --git a/isaac_ros_yolov8/test/isaac_ros_yolov8_decoder_node_pol.py b/isaac_ros_yolov8/test/isaac_ros_yolov8_decoder_node_pol.py new file mode 100644 index 0000000..ce9666e --- /dev/null +++ b/isaac_ros_yolov8/test/isaac_ros_yolov8_decoder_node_pol.py @@ -0,0 +1,211 @@ +# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +# Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +""" +Proof-Of-Life test for the Isaac ROS YOLOV8 Decoder Node package. + + 1. Sets up DnnImageEncoderNode, TensorRTNode + 2. Loads a sample image and publishes it + 3. Subscribes to the relevant topics, waiting for an output from YoloV8DecoderNode + 4. Verifies that the output is recieved (based on dummy model) + + Note: the data is not verified because the model is initialized with random weights +""" + + +import os +import pathlib +import time + +from ament_index_python.packages import get_package_share_directory +from isaac_ros_test import IsaacROSBaseTest, JSONConversion +from launch.actions import IncludeLaunchDescription +from launch.launch_description_sources import PythonLaunchDescriptionSource +from launch_ros.actions.composable_node_container import ComposableNodeContainer +from launch_ros.descriptions.composable_node import ComposableNode + +import pytest +import rclpy + +from sensor_msgs.msg import CameraInfo, Image +from vision_msgs.msg import Detection2DArray + +_TEST_CASE_NAMESPACE = 'yolov8_decoder_node_test' + +MODEL_FILE_NAME = 'dummy_yolov8s.onnx' +MODEL_GENERATION_TIMEOUT_SEC = 300 +INIT_WAIT_SEC = 10 +engine_file_path = '/tmp/dummy_yolov8s.plan' +input_tensor_names = ['input_tensor'] +input_binding_names = ['images'] +output_tensor_names = ['output_tensor'] +output_binding_names = ['output0'] + + +@pytest.mark.rostest +def generate_test_description(): + """Generate launch description for testing relevant nodes.""" + launch_dir_path = os.path.dirname(os.path.realpath(__file__)) + model_dir_path = launch_dir_path + '/dummy_model' + model_name = 'yolov8' + model_file_path = f'{model_dir_path}/{model_name}/dummy_yolov8s.onnx' + encoder_dir = get_package_share_directory('isaac_ros_dnn_image_encoder') + yolov8_encoder_launch = IncludeLaunchDescription( + PythonLaunchDescriptionSource( + [os.path.join(encoder_dir, 'launch', 'dnn_image_encoder.launch.py')] + ), + launch_arguments={ + 'input_image_width': '640', + 'input_image_height': '640', + 'network_image_width': '640', + 'network_image_height': '640', + 'image_mean': str([0.5, 0.6, 0.25]), + 'image_stddev': str([0.25, 0.8, 0.5]), + 'attach_to_shared_component_container': 'True', + 'component_container_name': 'tensor_rt_container', + 'dnn_image_encoder_namespace': IsaacROSYoloV8POLTest.generate_namespace( + _TEST_CASE_NAMESPACE), + 'tensor_output_topic': 'tensor_pub', + }.items(), + ) + + reshape_node = ComposableNode( + name='reshape_node', + package='isaac_ros_tensor_proc', + plugin='nvidia::isaac_ros::dnn_inference::ReshapeNode', + namespace=IsaacROSYoloV8POLTest.generate_namespace(_TEST_CASE_NAMESPACE), + parameters=[{ + 'output_tensor_names': output_tensor_names, + 'input_tensor_shape': [3, 640, 640], + 'output_tensor_shape': [1, 3, 640, 640] + }], + remappings=[ + ('tensor_input_topic', 'tensor_sub') + ], + ) + + tensor_rt_node = ComposableNode( + name='tensor_rt', + package='isaac_ros_tensor_rt', + plugin='nvidia::isaac_ros::dnn_inference::TensorRTNode', + namespace=IsaacROSYoloV8POLTest.generate_namespace(_TEST_CASE_NAMESPACE), + parameters=[{ + 'model_file_path': model_file_path, + 'engine_file_path': engine_file_path, + 'output_binding_names': output_binding_names, + 'output_tensor_names': output_tensor_names, + 'input_tensor_names': input_tensor_names, + 'input_binding_names': input_binding_names, + 'verbose': False, + 'force_engine_update': False + }] + ) + + yolov8_decoder_node = ComposableNode( + name='yolov8_decoder_node', + package='isaac_ros_yolov8', + plugin='nvidia::isaac_ros::yolov8::YoloV8DecoderNode', + namespace=IsaacROSYoloV8POLTest.generate_namespace(_TEST_CASE_NAMESPACE), + parameters=[{ + 'confidence_threshold': 0.25, + 'nms_threshold': 0.45, + }] + ) + + tensor_rt_container = ComposableNodeContainer( + name='tensor_rt_container', + package='rclcpp_components', + executable='component_container_mt', + composable_node_descriptions=[ + tensor_rt_node, yolov8_decoder_node, reshape_node], + output='screen', + arguments=['--ros-args', '--log-level', 'INFO'], + namespace='' + ) + + return IsaacROSYoloV8POLTest.generate_test_description( + [tensor_rt_container, yolov8_encoder_launch]) + + +class IsaacROSYoloV8POLTest(IsaacROSBaseTest): + """Validates that the inference pipeline produces outputs using a Yolov8 model.""" + + # filepath is required by IsaacROSBaseTest + filepath = pathlib.Path(os.path.dirname(__file__)) + INIT_WAIT_SEC = 10 + + @IsaacROSBaseTest.for_each_test_case() + def test_yolov8(self, test_folder): + """Expect the node to produce detection array given image.""" + self.node._logger.info(f'Generating model (timeout={MODEL_GENERATION_TIMEOUT_SEC}s)') + start_time = time.time() + wait_cycles = 1 + while not os.path.isfile(engine_file_path): + time_diff = time.time() - start_time + if time_diff > MODEL_GENERATION_TIMEOUT_SEC: + self.fail('Model generation timed out') + if time_diff > wait_cycles*10: + self.node._logger.info( + f'Waiting for model generation to finish... ({time_diff:.0f}s passed)') + wait_cycles += 1 + time.sleep(1) + + self.node._logger.info( + f'Model generation was finished (took {(time.time() - start_time)}s)') + + self.generate_namespace_lookup( + ['image', 'detections_output', 'camera_info'], _TEST_CASE_NAMESPACE) + + image_publisher = self.node.create_publisher( + Image, self.namespaces['image'], self.DEFAULT_QOS) + camera_info_pub = self.node.create_publisher( + CameraInfo, self.namespaces['camera_info'], self.DEFAULT_QOS) + + received_messages = {} + + yolov8_detections = self.create_logging_subscribers( + [('detections_output', Detection2DArray)], + received_messages, accept_multiple_messages=False) + + self.generate_namespace_lookup(['image', 'detections_output'], _TEST_CASE_NAMESPACE) + + try: + image = JSONConversion.load_image_from_json(test_folder / 'image.json') + timestamp = self.node.get_clock().now().to_msg() + image.header.stamp = timestamp + camera_info = CameraInfo() + camera_info.header = image.header + camera_info.distortion_model = 'plumb_bob' + + TIMEOUT = 10 + end_time = time.time() + TIMEOUT + done = False + while time.time() < end_time: + image_publisher.publish(image) + camera_info_pub.publish(camera_info) + rclpy.spin_once(self.node, timeout_sec=1.0) + + if 'detections_output' in received_messages: + done = True + break + + self.assertTrue( + done, "Didn't receive output on detections_output topic!") + + finally: + self.node.destroy_subscription(yolov8_detections) + self.node.destroy_publisher(image_publisher) diff --git a/isaac_ros_yolov8/test/test_cases/single_detection/camera_info.json b/isaac_ros_yolov8/test/test_cases/single_detection/camera_info.json new file mode 100644 index 0000000..85e80ad --- /dev/null +++ b/isaac_ros_yolov8/test/test_cases/single_detection/camera_info.json @@ -0,0 +1,55 @@ +{ + "header": { + "stamp": { + "sec": 1716965234, + "nanosec": 375945635 + }, + "frame_id": "camera" +}, +"height": 640, +"width": 640, +"distortion_model": "plumb_bob", + "D": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "K": [ + 1.0, + 0.0, + 320.0, + 0.0, + 1.0, + 320.0, + 0.0, + 0.0, + 1.0 + ], + "R": [ + 1.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 1.0 + ], + "P": [ + 1.0, + 0.0, + 320.0, + 0.0, + 0.0, + 1.0, + 320.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0 + ] +} \ No newline at end of file diff --git a/isaac_ros_yolov8/test/test_cases/single_detection/image.json b/isaac_ros_yolov8/test/test_cases/single_detection/image.json new file mode 100644 index 0000000..89b7b9f --- /dev/null +++ b/isaac_ros_yolov8/test/test_cases/single_detection/image.json @@ -0,0 +1,4 @@ +{ + "image": "people_cycles.jpg", + "encoding": "bgr8" +} diff --git a/isaac_ros_yolov8/test/test_cases/single_detection/people_cycles.jpg b/isaac_ros_yolov8/test/test_cases/single_detection/people_cycles.jpg new file mode 100644 index 0000000..ae3a584 --- /dev/null +++ b/isaac_ros_yolov8/test/test_cases/single_detection/people_cycles.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2d384fea3638b9c6726255f599148c78f95dc2f788e29e12619d455ee0fe535 +size 78412