rocAL Tensor Retinanet training support (#78)

* tensor changes * Update Doxyfile * test updates * bug fixes * bug fix * adding image augemntation app changes * Minor changes * Add ROI structure Add support to process multidimension ROI * Add support to update multi dimension ROI * Adding mask pipeline support for rocAL * Fix build issues wrt ROI struct Add necessary ROI changes Add Union for ROI Add pybind changes to copy ROI * Fix build issue * Add box IOU matcher changes Add pybind changes for IOU matcher Remove BoundingBoxCordf * Fixing build issues * Minor change * Remove redundant code * Minor change * Minor changes * Modify struct names * Remove strides in ROI * Change shape to end for ROICords * Fix crop dims in ssd random crop * Change ROI type to const in node.h * Resolving review comments * Resolving review comments * Resolving review comments * Formatting changes * Resolving review comments * Adding min_max scaling mode comment * Removing unused vector in coco reader * Improve code readability Add appropriate comments * Remove criteria argument in IOU matcher * Add IOU matcher info struct * Remove unused criteria variable * Change anchors to a ptr in IOU matcher struct * Minor change --------- Co-authored-by: LakshmiKumar23 <[email protected]> Co-authored-by: SundarRajan28 <[email protected]> Co-authored-by: Sundar Rajan Vaithiyanathan <[email protected]>
ROCm · Dec 13, 2023 · 882b732 · 882b732
1 parent a9c18ca
commit 882b732
Show file tree

Hide file tree

Showing 10 changed files with 222 additions and 74 deletions.
diff --git a/rocAL/include/api/rocal_api_meta_data.h b/rocAL/include/api/rocal_api_meta_data.h
@@ -81,9 +81,10 @@ extern "C" RocalMetaData ROCAL_API_CALL rocalCreateTFReaderDetection(RocalContex
  * \param [in] is_box_encoder If set to True, bboxes are returned as encoded bboxes using the anchors
  * \param [in] avoid_class_remapping If set to True, classes are returned directly. Otherwise, classes are mapped to consecutive values
  * \param [in] aspect_ratio_grouping If set to True, images are sorted by their aspect ratio and returned
+ * \param [in] is_box_iou_matcher If set to True, box iou matcher which returns matched indices is enabled in the pipeline
  * \return RocalMetaData object, can be used to inquire about the rocal's output (processed) tensors
  */
-extern "C" RocalMetaData ROCAL_API_CALL rocalCreateCOCOReader(RocalContext rocal_context, const char* source_path, bool is_output, bool mask = false, bool ltrb = true, bool is_box_encoder = false, bool avoid_class_remapping = false, bool aspect_ratio_grouping = false);
+extern "C" RocalMetaData ROCAL_API_CALL rocalCreateCOCOReader(RocalContext rocal_context, const char* source_path, bool is_output, bool mask = false, bool ltrb = true, bool is_box_encoder = false, bool avoid_class_remapping = false, bool aspect_ratio_grouping = false, bool is_box_iou_matcher = false);
 
 /*! \brief create coco reader key points
  * \ingroup group_rocal_meta_data
@@ -296,4 +297,22 @@ extern "C" void ROCAL_API_CALL rocalGetImageId(RocalContext p_context, int* buf)
  */
 extern "C" void ROCAL_API_CALL rocalGetJointsDataPtr(RocalContext p_context, RocalJointsData** joints_data);
 
+/*! \brief API to enable box IOU matcher and pass required params to pipeline
+ * \ingroup group_rocal_meta_data
+ * \param [in] p_context rocAL context
+ * \param [in] anchors The anchors / ground truth bounding box coordinates
+ * \param [in] high_threshold The max threshold for IOU
+ * \param [in] low_threshold The min threshold for IOU
+ * \param [in] allow_low_quality_matches bool value when set to true allows low quality matches
+ */
+extern "C" void ROCAL_API_CALL rocalBoxIouMatcher(RocalContext p_context, std::vector<float>& anchors,
+                                                  float high_threshold, float low_threshold, bool allow_low_quality_matches = true);
+
+/*! \brief API to return the matched indices for the bounding box and anchors
+ * \ingroup group_rocal_meta_data
+ * \param [in] p_context rocAL context
+ * \return RocalTensorList of matched indices
+ */
+extern "C" RocalTensorList ROCAL_API_CALL rocalGetMatchedIndices(RocalContext p_context);
+
 #endif  // MIVISIONX_ROCAL_API_META_DATA_H
diff --git a/rocAL/include/meta_data/bounding_box_graph.h b/rocAL/include/meta_data/bounding_box_graph.h
@@ -26,27 +26,13 @@ THE SOFTWARE.
 #include "meta_data_graph.h"
 #include "meta_node.h"
 
-typedef struct {
-    float xc;
-    float yc;
-    float w;
-    float h;
-} BoundingBoxCord_xcycwh;
-typedef struct {
-    float l;
-    float t;
-    float r;
-    float b;
-} BoundingBoxCord_ltrb;
-typedef union {
-    BoundingBoxCord_xcycwh xcycwh;
-    BoundingBoxCord_ltrb ltrb;
-} BoundingBoxCordf;  // Union comprises of float bbox cords of ltrb/xcycwh type
+typedef  struct { float xc; float yc; float w; float h; } BoundingBoxCord_xcycwh;
 
 class BoundingBoxGraph : public MetaDataGraph {
    public:
     void process(pMetaDataBatch input_meta_data, pMetaDataBatch output_meta_data) override;
     void update_meta_data(pMetaDataBatch meta_data, decoded_image_info decode_image_info) override;
     void update_random_bbox_meta_data(pMetaDataBatch input_meta_data, pMetaDataBatch output_meta_data, decoded_image_info decoded_image_info, crop_image_info crop_image_info) override;
     void update_box_encoder_meta_data(std::vector<float> *anchors, pMetaDataBatch full_batch_meta_data, float criteria, bool offset, float scale, std::vector<float> &means, std::vector<float> &stds, float *encoded_boxes_data, int *encoded_labels_data) override;
+    void update_box_iou_matcher(BoxIouMatcherInfo &iou_matcher_info, int *matches_idx_buffer, pMetaDataBatch full_batch_meta_data) override;
 };
diff --git a/rocAL/include/meta_data/meta_data_graph.h b/rocAL/include/meta_data/meta_data_graph.h
@@ -30,12 +30,20 @@ THE SOFTWARE.
 #include "parameter_factory.h"
 #include "randombboxcrop_meta_data_reader.h"
 
+typedef struct {
+    std::vector<float> *anchors;
+    float high_threshold;
+    float low_threshold;
+    bool allow_low_quality_matches;
+} BoxIouMatcherInfo;
+
 class MetaDataGraph {
    public:
     virtual ~MetaDataGraph() = default;
     virtual void process(pMetaDataBatch input_meta_data, pMetaDataBatch output_meta_data) = 0;
     virtual void update_meta_data(pMetaDataBatch meta_data, decoded_image_info decoded_image_info) = 0;
     virtual void update_random_bbox_meta_data(pMetaDataBatch input_meta_data, pMetaDataBatch output_meta_data, decoded_image_info decoded_image_info, crop_image_info crop_image_info) = 0;
     virtual void update_box_encoder_meta_data(std::vector<float> *anchors, pMetaDataBatch full_batch_meta_data, float criteria, bool offset, float scale, std::vector<float> &means, std::vector<float> &stds, float *encoded_boxes_data, int *encoded_labels_data) = 0;
+    virtual void update_box_iou_matcher(BoxIouMatcherInfo &iou_matcher_info, int *matches_idx_buffer, pMetaDataBatch full_batch_meta_data) = 0;
     std::list<std::shared_ptr<MetaNode>> _meta_nodes;
 };
diff --git a/rocAL/include/pipeline/master_graph.h b/rocAL/include/pipeline/master_graph.h
@@ -46,10 +46,11 @@ THE SOFTWARE.
 #include "randombboxcrop_meta_data_reader.h"
 #include "rocal_api_types.h"
 #define MAX_STRING_LENGTH 100
-#define MAX_OBJECTS 50        // Setting an arbitrary value 50.(Max number of objects/image in COCO dataset is 93)
+#define MAX_OBJECTS 50                // Setting an arbitrary value 50.(Max number of objects/image in COCO dataset is 93)
 #define BBOX_COUNT 4
-#define MAX_NUM_ANCHORS 8732  // Num of bbox achors used in SSD training
+#define MAX_SSD_ANCHORS 8732          // Num of bbox achors used in SSD training
 #define MAX_MASK_BUFFER 10000
+#define MAX_RETINANET_ANCHORS 120087  // Num of bbox achors used in Retinanet training
 
 #if ENABLE_SIMD
 #if _WIN32
@@ -107,18 +108,20 @@ class MasterGraph {
     std::vector<rocalTensorList *> create_label_reader(const char *source_path, MetaDataReaderType reader_type);
     std::vector<rocalTensorList *> create_video_label_reader(const char *source_path, MetaDataReaderType reader_type, unsigned sequence_length, unsigned frame_step, unsigned frame_stride, bool file_list_frame_num = true);
     std::vector<rocalTensorList *> create_coco_meta_data_reader(const char *source_path, bool is_output, MetaDataReaderType reader_type, MetaDataType label_type, bool ltrb_bbox = true, bool is_box_encoder = false,
-                                                                bool avoid_class_remapping = false, bool aspect_ratio_grouping = false, float sigma = 0.0, unsigned pose_output_width = 0, unsigned pose_output_height = 0);
+                                                                bool avoid_class_remapping = false, bool aspect_ratio_grouping = false, bool is_box_iou_matcher = false, float sigma = 0.0, unsigned pose_output_width = 0, unsigned pose_output_height = 0);
     std::vector<rocalTensorList *> create_tf_record_meta_data_reader(const char *source_path, MetaDataReaderType reader_type, MetaDataType label_type, const std::map<std::string, std::string> feature_key_map);
     std::vector<rocalTensorList *> create_caffe_lmdb_record_meta_data_reader(const char *source_path, MetaDataReaderType reader_type, MetaDataType label_type);
     std::vector<rocalTensorList *> create_caffe2_lmdb_record_meta_data_reader(const char *source_path, MetaDataReaderType reader_type, MetaDataType label_type);
     std::vector<rocalTensorList *> create_cifar10_label_reader(const char *source_path, const char *file_prefix);
     std::vector<rocalTensorList *> create_mxnet_label_reader(const char *source_path, bool is_output);
     void box_encoder(std::vector<float> &anchors, float criteria, const std::vector<float> &means, const std::vector<float> &stds, bool offset, float scale);
+    void box_iou_matcher(std::vector<float> &anchors, float high_threshold, float low_threshold, bool allow_low_quality_matches);
     void create_randombboxcrop_reader(RandomBBoxCrop_MetaDataReaderType reader_type, RandomBBoxCrop_MetaDataType label_type, bool all_boxes_overlap, bool no_crop, FloatParam *aspect_ratio, bool has_shape, int crop_width, int crop_height, int num_attempts, FloatParam *scaling, int total_num_attempts, int64_t seed = 0);
     const std::pair<ImageNameBatch, pMetaDataBatch> &meta_data();
     TensorList *labels_meta_data();
     TensorList *bbox_meta_data();
     TensorList *mask_meta_data();
+    TensorList *matched_index_meta_data();
     void set_loop(bool val) { _loop = val; }
     void set_output(Tensor *output_tensor);
     size_t calculate_cpu_num_threads(size_t shard_count);
@@ -164,6 +167,7 @@ class MasterGraph {
     TensorList _labels_tensor_list;
     TensorList _bbox_tensor_list;
     TensorList _mask_tensor_list;
+    TensorList _matches_tensor_list;
     std::vector<size_t> _meta_data_buffer_size;
 #if ENABLE_HIP
     DeviceManagerHip _device;                                                     //!< Keeps the device related constructs needed for running on GPU
@@ -204,6 +208,9 @@ class MasterGraph {
     bool _offset;                                                                 // Returns normalized offsets ((encoded_bboxes*scale - anchors*scale) - mean) / stds in EncodedBBoxes that use std and the mean and scale arguments if offset="True"
     std::vector<float> _means, _stds;                                             //_means:  [x y w h] mean values for normalization _stds: [x y w h] standard deviations for offset normalization.
     bool _augmentation_metanode = false;
+    // box IoU matcher variables
+    bool _is_box_iou_matcher = false;                                             // bool variable to set the box iou matcher
+    BoxIouMatcherInfo _iou_matcher_info;
 #if ENABLE_HIP
     BoxEncoderGpu *_box_encoder_gpu = nullptr;
 #endif

diff --git a/rocAL/source/api/rocal_api_meta_data.cpp b/rocAL/source/api/rocal_api_meta_data.cpp
@@ -71,14 +71,14 @@ RocalMetaData
 
 RocalMetaData
     ROCAL_API_CALL
-    rocalCreateCOCOReader(RocalContext p_context, const char* source_path, bool is_output, bool mask, bool ltrb, bool is_box_encoder, bool avoid_class_remapping, bool aspect_ratio_grouping) {
+    rocalCreateCOCOReader(RocalContext p_context, const char* source_path, bool is_output, bool mask, bool ltrb, bool is_box_encoder, bool avoid_class_remapping, bool aspect_ratio_grouping, bool is_box_iou_matcher) {
     if (!p_context)
         THROW("Invalid rocal context passed to rocalCreateCOCOReader")
     auto context = static_cast<Context*>(p_context);
     if (mask) {
-        return context->master_graph->create_coco_meta_data_reader(source_path, is_output, MetaDataReaderType::COCO_META_DATA_READER, MetaDataType::PolygonMask, ltrb, is_box_encoder, avoid_class_remapping, aspect_ratio_grouping);
+        return context->master_graph->create_coco_meta_data_reader(source_path, is_output, MetaDataReaderType::COCO_META_DATA_READER, MetaDataType::PolygonMask, ltrb, is_box_encoder, avoid_class_remapping, aspect_ratio_grouping, is_box_iou_matcher);
     }
-    return context->master_graph->create_coco_meta_data_reader(source_path, is_output, MetaDataReaderType::COCO_META_DATA_READER, MetaDataType::BoundingBox, ltrb, is_box_encoder, avoid_class_remapping, aspect_ratio_grouping);
+    return context->master_graph->create_coco_meta_data_reader(source_path, is_output, MetaDataReaderType::COCO_META_DATA_READER, MetaDataType::BoundingBox, ltrb, is_box_encoder, avoid_class_remapping, aspect_ratio_grouping, is_box_iou_matcher);
 }
 
 RocalMetaData
@@ -88,7 +88,7 @@ RocalMetaData
         THROW("Invalid rocal context passed to rocalCreateCOCOReaderKeyPoints")
     auto context = static_cast<Context*>(p_context);
 
-    return context->master_graph->create_coco_meta_data_reader(source_path, is_output, MetaDataReaderType::COCO_KEY_POINTS_META_DATA_READER, MetaDataType::KeyPoints, sigma, pose_output_width, pose_output_height);
+    return context->master_graph->create_coco_meta_data_reader(source_path, is_output, MetaDataReaderType::COCO_KEY_POINTS_META_DATA_READER, MetaDataType::KeyPoints, false, false, false, false, sigma, pose_output_width, pose_output_height);
 }
 
 RocalMetaData
@@ -490,3 +490,26 @@ void
 
     *joints_data = (RocalJointsData*)(&(meta_data.second->get_joints_data_batch()));
 }
+
+void
+    ROCAL_API_CALL
+    rocalBoxIouMatcher(RocalContext p_context,
+                       std::vector<float>& anchors,
+                       float high_threshold, float low_threshold,
+                       bool allow_low_quality_matches) {
+    if (!p_context)
+        THROW("Invalid rocal context passed to rocalBoxIouMatcher")
+    auto context = static_cast<Context*>(p_context);
+    context->master_graph->box_iou_matcher(anchors, high_threshold,
+                                           low_threshold,
+                                           allow_low_quality_matches);
+}
+
+RocalTensorList
+    ROCAL_API_CALL
+    rocalGetMatchedIndices(RocalContext p_context) {
+    if (!p_context)
+        THROW("Invalid rocal context passed to rocalGetMatchedIndices")
+    auto context = static_cast<Context*>(p_context);
+    return context->master_graph->matched_index_meta_data();
+}