Merge branch 'developer'

Change-Id: Ibe875ba912e6dd4ff2b0e900ee836e4bf87bd8b1
sophgo · Aug 16, 2024 · 7ad5183 · 7ad5183
2 parents 82dbfda + 44e0f05
commit 7ad5183
Show file tree

Hide file tree

Showing 6 changed files with 52 additions and 18 deletions.
diff --git a/element/multimedia/decode/src/decoder.cc b/element/multimedia/decode/src/decoder.cc
@@ -160,6 +160,7 @@ common::ErrorCode Decoder::process(
     objectMetadata->mFrame = std::make_shared<common::Frame>();
     objectMetadata->mFrame->mHandle = m_handle;
     objectMetadata->mFrame->mFrameId = frame_id;
+    objectMetadata->mFrame->mSubFrameIdVec.push_back(frame_id);
     objectMetadata->mFrame->mSpData = spBmImage;
     objectMetadata->mFrame->mTimestamp = pts;
     objectMetadata->mGraphId = mGraphId;
@@ -185,6 +186,7 @@ common::ErrorCode Decoder::process(
     objectMetadata->mFrame = std::make_shared<common::Frame>();
     objectMetadata->mFrame->mHandle = m_handle;
     objectMetadata->mFrame->mFrameId = frame_id;
+    objectMetadata->mFrame->mSubFrameIdVec.push_back(frame_id);
     objectMetadata->mFrame->mSpData = spBmImage;
     objectMetadata->mFrame->mTimestamp = pts;
     objectMetadata->mGraphId = mGraphId;
@@ -216,6 +218,7 @@ common::ErrorCode Decoder::process(
     objectMetadata->mFrame = std::make_shared<common::Frame>();
     objectMetadata->mFrame->mHandle = m_handle;
     objectMetadata->mFrame->mFrameId = mImgIndex;
+    objectMetadata->mFrame->mSubFrameIdVec.push_back(mImgIndex);
     objectMetadata->mFrame->mSpData = spBmImage;
     objectMetadata->mGraphId = mGraphId;
 
@@ -244,6 +247,7 @@ common::ErrorCode Decoder::process(
     objectMetadata->mFrame = std::make_shared<common::Frame>();
     objectMetadata->mFrame->mHandle = m_handle;
     objectMetadata->mFrame->mFrameId = mImgIndex++;
+    objectMetadata->mFrame->mSubFrameIdVec.push_back(mImgIndex);
     objectMetadata->mFrame->mSpData = spBmImage;
     objectMetadata->mGraphId = mGraphId;
     if (spBmImage != nullptr)
@@ -287,6 +291,7 @@ common::ErrorCode Decoder::process(
     objectMetadata->mFrame = std::make_shared<common::Frame>();
     objectMetadata->mFrame->mHandle = m_handle;
     objectMetadata->mFrame->mFrameId = frame_id;
+    objectMetadata->mFrame->mSubFrameIdVec.push_back(frame_id);
     objectMetadata->mFrame->mSpData = spBmImage;
     objectMetadata->mFrame->mTimestamp = pts;
     objectMetadata->mGraphId = mGraphId;

diff --git a/element/tools/converger/include/converger.h b/element/tools/converger/include/converger.h
@@ -35,10 +35,10 @@ class Converger : public ::sophon_stream::framework::Element {
  private:
   int mDefaultPort;
   /**
-   * @brief key: channel_id, value: map<frame_id, objectMetadata>
+   * @brief key: channel_id, value: map<sub_frame_id, objectMetadata>
    */
   std::unordered_map<int,
-                     std::map<int, std::shared_ptr<common::ObjectMetadata>>>
+                     std::map<std::int64_t, std::shared_ptr<common::ObjectMetadata>>>
       mCandidates;
   /**
    * @brief recored number of branches for every ObjectMetadata in default_port

diff --git a/element/tools/converger/src/converger.cc b/element/tools/converger/src/converger.cc
@@ -55,14 +55,16 @@ common::ErrorCode Converger::doWork(int dataPipeId) {
     auto objectMetadata =
         std::static_pointer_cast<common::ObjectMetadata>(data);
     int channel_id = objectMetadata->mFrame->mChannelIdInternal;
-    int frame_id = objectMetadata->mFrame->mFrameId;
+    int frame_id = objectMetadata->mFrame->mSubFrameIdVec.back();
     // lock
     std::unique_lock<std::mutex> lk(mtx);
     mCandidates[channel_id][frame_id] = objectMetadata;
     lk.unlock();
     IVS_DEBUG(
-        "data recognized, channel_id = {0}, frame_id = {1}, num_branches = {2}",
-        channel_id, frame_id, objectMetadata->numBranches);
+        "data recognized, element_id = {3}, channel_id = {0}, frame_id = {1}, "
+        "num_branches = "
+        "{2}",
+        channel_id, frame_id, objectMetadata->numBranches, getId());
   }
 
   // 非default_port，取出来之后更新分支数的记录
@@ -73,19 +75,23 @@ common::ErrorCode Converger::doWork(int dataPipeId) {
     while (subdata != nullptr) {
       auto subObj = std::static_pointer_cast<common::ObjectMetadata>(subdata);
       int sub_channel_id = subObj->mFrame->mChannelIdInternal;
-      int sub_frame_id = subObj->mFrame->mFrameId;
-      IVS_DEBUG("subData recognized, channel_id = {0}, frame_id = {1}",
-                sub_channel_id, sub_frame_id);
+      auto sub_frame_id_it = subObj->mFrame->mSubFrameIdVec.end();
+      auto sub_frame_id = *(sub_frame_id_it - 2);
+      IVS_DEBUG(
+          "subData recognized, element_id = {2}, channel_id = {0}, frame_id = "
+          "{1}",
+          sub_channel_id, sub_frame_id, getId());
       // lock
       std::unique_lock<std::mutex> lk(mtx);
       mBranches[sub_channel_id][sub_frame_id]++;
       lk.unlock();
       IVS_DEBUG(
-          "data updated, channel_id = {0}, frame_id = {1}, current "
+          "data updated, element_id = {3}, channel_id = {0}, frame_id = {1}, "
+          "current "
           "num_branches "
           "= {2}",
-          sub_channel_id, sub_frame_id,
-          mBranches[sub_channel_id][sub_frame_id]);
+          sub_channel_id, sub_frame_id, mBranches[sub_channel_id][sub_frame_id],
+          getId());
       subdata = popInputData(inputPort, dataPipeId);
     }
     // if (subdata == nullptr) continue;
@@ -95,7 +101,7 @@ common::ErrorCode Converger::doWork(int dataPipeId) {
   for (auto channel_it = mCandidates.begin(); channel_it != mCandidates.end();
        ++channel_it) {
     // 第一层：遍历所有channel
-    // 这里需要判断：如果channelId应该和自己这个datapipeId对上，就操作；否则跳过，给其它线程操作    
+    // 这里需要判断：如果channelId应该和自己这个datapipeId对上，就操作；否则跳过，给其它线程操作
     int channel_id_internal = channel_it->first;
     int dataPipeNums = getThreadNumber();
     if (channel_id_internal % dataPipeNums != dataPipeId) continue;
@@ -107,8 +113,10 @@ common::ErrorCode Converger::doWork(int dataPipeId) {
       // 如果可以弹出，则弹出并循环至下一个
       if (mBranches[channel_id_internal][frame_id] ==
           mCandidates[channel_id_internal][frame_id]->numBranches) {
-        IVS_DEBUG("Data converged! Now pop... channel_id = {0}, frame_id = {1}",
-                  channel_id_internal, frame_id);
+        IVS_DEBUG(
+            "Data converged! Now pop... element_id = {0}, channel_id = {1}, "
+            "frame_id = {2}",
+            getId(), channel_id_internal, frame_id);
         auto obj = mCandidates[channel_id_internal][frame_id];
         int outDataPipeId = getSinkElementFlag()
                                 ? 0

diff --git a/element/tools/distributor/include/distributor.h b/element/tools/distributor/include/distributor.h
@@ -63,10 +63,9 @@ class Distributor : public ::sophon_stream::framework::Element {
       std::shared_ptr<common::DetectedObjectMetadata> detObj,
       std::shared_ptr<common::ObjectMetadata> subObj, int subId);
 
-  bm_image get_rotate_crop_image(bm_handle_t handle, 
-      bm_image input_bmimg_planar, 
-      std::vector<std::vector<int>>);
-
+  bm_image get_rotate_crop_image(bm_handle_t handle,
+                                 bm_image input_bmimg_planar,
+                                 std::vector<std::vector<int>>);
 
   /**
    * @brief 按时间间隔分发的所有规则。key：时间间隔，value：{类名，端口}
@@ -92,6 +91,13 @@ class Distributor : public ::sophon_stream::framework::Element {
    */
   std::unordered_map<int, std::vector<float>> mChannelLastTimes;
 
+  /**
+   * @brief key是ChannelId，value是在distributor内部维护的subFrameId
+   * 这个设计是因为：如果只使用frameId作为判断是否可以弹出数据的map的key，会在多级dist和conv嵌套时出错
+   * 考虑到设计上嵌套等级是无上限的，所以需要在ObjectMetadata里使用一个vector<std::int64_t>来维护最近的frameId
+   */
+  std::unordered_map<int, std::int64_t> mSubFrameIdMap;
+
   sophon_stream::common::Clocker clocker;
 
   bool is_affine = false;

diff --git a/element/tools/distributor/src/distributor.cc b/element/tools/distributor/src/distributor.cc
@@ -181,6 +181,9 @@ void Distributor::makeSubObjectMetadata(
 
   // update frameid, channelid
   subObj->mFrame->mFrameId = obj->mFrame->mFrameId;
+  subObj->mFrame->mSubFrameIdVec = obj->mFrame->mSubFrameIdVec;
+  subObj->mFrame->mSubFrameIdVec.push_back(
+      mSubFrameIdMap[obj->mFrame->mChannelId]);
   subObj->mFrame->mChannelId = obj->mFrame->mChannelId;
   subObj->mFrame->mChannelIdInternal = obj->mFrame->mChannelIdInternal;
   subObj->mSubId = subId;
@@ -419,9 +422,13 @@ void Distributor::makeSubFaceObjectMetadata(
 
   // update frameid, channelid
   subObj->mFrame->mFrameId = obj->mFrame->mFrameId;
+  subObj->mFrame->mSubFrameIdVec = obj->mFrame->mSubFrameIdVec;
+  subObj->mFrame->mSubFrameIdVec.push_back(
+      mSubFrameIdMap[obj->mFrame->mChannelId]);
   subObj->mFrame->mChannelId = obj->mFrame->mChannelId;
   subObj->mFrame->mChannelIdInternal = obj->mFrame->mChannelIdInternal;
   subObj->mSubId = subId;
+  subObj->mFrame->mEndOfStream = obj->mFrame->mEndOfStream;
 }
 
 void Distributor::makeSubOcrObjectMetadata(
@@ -468,9 +475,13 @@ void Distributor::makeSubOcrObjectMetadata(
 
   // update frameid, channelid
   subObj->mFrame->mFrameId = obj->mFrame->mFrameId;
+  subObj->mFrame->mSubFrameIdVec = obj->mFrame->mSubFrameIdVec;
+  subObj->mFrame->mSubFrameIdVec.push_back(
+      mSubFrameIdMap[obj->mFrame->mChannelId]);
   subObj->mFrame->mChannelId = obj->mFrame->mChannelId;
   subObj->mFrame->mChannelIdInternal = obj->mFrame->mChannelIdInternal;
   subObj->mSubId = subId;
+  subObj->mFrame->mEndOfStream = obj->mFrame->mEndOfStream;
 }
 
 bm_image Distributor::get_rotate_crop_image(bm_handle_t handle,
@@ -623,6 +634,7 @@ common::ErrorCode Distributor::doWork(int dataPipeId) {
         }
       }
       ++subId;
+      ++mSubFrameIdMap[objectMetadata->mFrame->mChannelId];
     }
 
     for (auto faceObj : objectMetadata->mFaceObjectMetadatas) {
@@ -657,6 +669,7 @@ common::ErrorCode Distributor::doWork(int dataPipeId) {
         }
       }
       ++subId;
+      ++mSubFrameIdMap[objectMetadata->mFrame->mChannelId];
     }
 
     for (auto detObj : objectMetadata->mDetectedObjectMetadatas) {
@@ -696,6 +709,7 @@ common::ErrorCode Distributor::doWork(int dataPipeId) {
         }
       }
       ++subId;
+      ++mSubFrameIdMap[objectMetadata->mFrame->mChannelId];
     }
 
     if (class2ports.find("full_frame") != class2ports.end()) {

diff --git a/framework/common/frame.h b/framework/common/frame.h
@@ -55,6 +55,7 @@ struct Frame {
   int mChannelId;
   int mChannelIdInternal;
   std::int64_t mFrameId;
+  std::vector<std::int64_t> mSubFrameIdVec;
 
   bm_image_format_ext mFormatType;
   bm_image_data_format_ext mDataType;