Merge branch 'developer'

Change-Id: Ifa9d756c3ece4ecaf091ae75c9e59ac615ac8423

element/multimedia/decode/include/decode.h

@@ -141,6 +141,8 @@ class Decode : public ::sophon_stream::framework::Element {
 
   common::ErrorCode doWork(int dataPipe) override;
 
+  bm_handle_t getHandle() const;
+
   static constexpr const char* JSON_CHANNEL_ID = "channel_id";
   static constexpr const char* JSON_SOURCE_TYPE = "source_type";
   static constexpr const char* JSON_URL = "url";
@@ -159,8 +161,13 @@ class Decode : public ::sophon_stream::framework::Element {
  private:
   std::map<int, std::shared_ptr<ChannelInfo>> mThreadsPool;
   std::mutex mThreadsPoolMtx;
-  std::atomic<int> mChannelCount;
+  // 用于channelIdInternal更新
+  static std::atomic<int> mChannelCount;
+  // channelId -> channelIdInternal 的映射
   std::map<int, int> mChannelIdInternal;
+  // 已经释放出来的channelIdInternal
+  static std::queue<int> mChannelIdInternalReleased;
+
 
   void onStart() override;
   void onStop() override;
@@ -177,6 +184,8 @@ class Decode : public ::sophon_stream::framework::Element {
       std::shared_ptr<ChannelTask>& channelTask);
 
   ::sophon_stream::common::FpsProfiler mFpsProfiler;
+
+  bm_handle_t handle_;
 };
 
 }  // namespace decode

element/multimedia/decode/include/decoder.h

@@ -31,8 +31,8 @@ class Decoder : public ::sophon_stream::common::NoCopyable {
   Decoder();
   ~Decoder();
 
-  common::ErrorCode init(int deviceId, int graphId,
-                         const ChannelOperateRequest& request);
+  common::ErrorCode init(int graphId, const ChannelOperateRequest& request,
+                         bm_handle_t handle);
   common::ErrorCode process(
       std::shared_ptr<common::ObjectMetadata>& objectMetadata);
   void uninit();

element/multimedia/decode/src/decode.cc

@@ -15,12 +15,16 @@ namespace decode {
 
 Decode::Decode() {}
 
+std::atomic<int> Decode::mChannelCount = 0;
+std::queue<int> Decode::mChannelIdInternalReleased;
+
 Decode::~Decode() {
   std::lock_guard<std::mutex> lk(mThreadsPoolMtx);
   for (auto& channelInfo : mThreadsPool) {
     channelInfo.second->mThreadWrapper->stop();
   }
   mThreadsPool.clear();
+  bm_dev_free(handle_);
 }
 
 common::ErrorCode Decode::initInternal(const std::string& json) {
@@ -32,8 +36,9 @@ common::ErrorCode Decode::initInternal(const std::string& json) {
       errorCode = common::ErrorCode::PARSE_CONFIGURE_FAIL;
       break;
     }
-    mChannelCount = 0;
     mFpsProfiler.config("fps_decode", 100);
+    int dev_id = getDeviceId();
+    bm_dev_request(&handle_, dev_id);
   } while (false);
 
   return errorCode;
@@ -317,8 +322,8 @@ common::ErrorCode Decode::startTask(std::shared_ptr<ChannelTask>& channelTask) {
         IVS_INFO("channel info decoder address: {0:p}",
                  static_cast<void*>(channelInfo->mSpDecoder.get()));
 
-        common::ErrorCode ret =
-            channelInfo->mSpDecoder->init(getDeviceId(), getGraphId(), channelTask->request);
+        common::ErrorCode ret = channelInfo->mSpDecoder->init(
+            getGraphId(), channelTask->request, handle_);
         if (ret != common::ErrorCode::SUCCESS) {
           channelTask->response.errorCode = ret;
           std::string error = "Decoder init failed! channel id is " +
@@ -355,11 +360,19 @@ common::ErrorCode Decode::startTask(std::shared_ptr<ChannelTask>& channelTask) {
       std::make_pair(channelTask->request.channelId, channelInfo));
 
   int channel_id = channelTask->request.channelId;
-  if (mChannelIdInternal.find(channel_id) == mChannelIdInternal.end()) {
+  // 这里不需要判断channel_id是否在占用，因为本函数开头就在mThreadsPool里处理了
+  // 更新channel_id。需要判断是否有释放出来的channelIdInternal
+  if (mChannelIdInternalReleased.empty()) {
+    // 没有释放的channelIdInternal，那么只能更新一个。
     mChannelIdInternal[channel_id] = mChannelCount++;
+  } else {
+    // 有可以释放的channelIdInternal
+    int channelIdInternal = mChannelIdInternalReleased.front();
+    mChannelIdInternalReleased.pop();
+    mChannelIdInternal[channel_id] = channelIdInternal;
   }
 
-  IVS_INFO("add one channel task finished!");
+  IVS_INFO("add one channel task finished, channel id = {0}", channel_id);
   return channelTask->response.errorCode;
 }
 
@@ -374,11 +387,21 @@ common::ErrorCode Decode::stopTask(std::shared_ptr<ChannelTask>& channelTask) {
     IVS_ERROR("{0}", error);
     return common::ErrorCode::DECODE_CHANNEL_NOT_FOUND;
   }
+
+  // 停止一路码流，需要记录释放出来的channelIdInternal，然后erase一对kv
+  auto itChannelId = mChannelIdInternal.find(channelTask->request.channelId);
+  int channelIdInternal = itChannelId->second;
+  mChannelIdInternalReleased.push(channelIdInternal);
+  mChannelIdInternal.erase(itChannelId);
+
   common::ErrorCode errorCode = itTask->second->mThreadWrapper->stop();
   itTask->second->mSpDecoder->uninit();
   itTask->second->mThreadWrapper.reset();
   mThreadsPool.erase(itTask);
   channelTask->response.errorCode = errorCode;
+  IVS_INFO("stop one channel task finished, channel id = {0}",
+           channelTask->request.channelId);
+
   return errorCode;
 }
 

element/multimedia/decode/src/decoder.cc

@@ -71,23 +71,26 @@ Decoder::Decoder() {
   numThreadsTotal.fetch_add(1);
 }
 
-Decoder::~Decoder() {}
+Decoder::~Decoder() {
+  // bm_dev_free(m_handle);
+}
 
-common::ErrorCode Decoder::init(int deviceId, int graphId,
-                                const ChannelOperateRequest& request) {
+common::ErrorCode Decoder::init(int graphId,
+                                const ChannelOperateRequest& request,
+                                bm_handle_t handle_) {
   common::ErrorCode errorCode = common::ErrorCode::SUCCESS;
   do {
     mUrl = request.url;
     mLoopNum = request.loopNum;
     mSampleInterval = request.sampleInterval;
     mFps = request.fps;
     mSampleStrategy = request.sampleStrategy;
-    int ret = bm_dev_request(&m_handle, deviceId);
-    mDeviceId = deviceId;
+    // int ret = bm_dev_request(&m_handle, deviceId);
+    m_handle = handle_;
+    mDeviceId = bm_get_devid(m_handle);
     mGraphId = graphId;
     mSourceType = request.sourceType;
     mImgIndex = 0;
-    assert(BM_SUCCESS == ret);
     mRoiPredefined = request.roi_predefined;
     if (mRoiPredefined) {
       mRoi.start_x = request.roi.start_x;

element/multimedia/decode/src/ff_decode.cc

@@ -103,11 +103,6 @@ VideoDecFFM::VideoDecFFM() {
   video_stream_idx = -1;
   refcount = 1;
 
-  pkt = new AVPacket;
-  av_init_packet(pkt);
-  pkt->data = NULL;
-  pkt->size = 0;
-
   avdevice_register_all();
   // frame = av_frame_alloc();
 }
@@ -406,6 +401,10 @@ void VideoDecFFM::mFrameCount(const char* video_file, int& mFrameCount) {
 
 int VideoDecFFM::openDec(bm_handle_t* dec_handle, const char* input) {
   // printf("openDec, tid = %d\n", gettid());
+  pkt = new AVPacket;
+  av_init_packet(pkt);
+  pkt->data = NULL;
+  pkt->size = 0;
   gettimeofday(&last_time, NULL);
   frame = av_frame_alloc();
   frame_id = 0;
@@ -478,17 +477,24 @@ int VideoDecFFM::openDec(bm_handle_t* dec_handle, const char* input) {
 
 void VideoDecFFM::closeDec() {
   if (video_dec_ctx) {
+    avcodec_close(video_dec_ctx);
     avcodec_free_context(&video_dec_ctx);
     video_dec_ctx = NULL;
   }
   if (ifmt_ctx) {
     avformat_close_input(&ifmt_ctx);
+    avformat_free_context(ifmt_ctx);
     ifmt_ctx = NULL;
   }
   if (frame) {
+    av_frame_unref(frame);
     av_frame_free(&frame);
     frame = NULL;
   }
+  if (pkt) {
+    av_packet_unref(pkt);
+    av_packet_free(&pkt);
+  }
   frame_id = 0;
   quit_flag = false;
 }

element/multimedia/osd/CMakeLists.txt

@@ -44,7 +44,13 @@ if (${TARGET_ARCH} STREQUAL "pcie")
     add_library(osd SHARED
         src/osd.cc
     )
+    add_library(cvunitext SHARED src/cvUniText.cc)
+    target_link_libraries(osd cvunitext)
+    add_dependencies(cvunitext freetype)
+    target_link_libraries(cvunitext freetype ${OPENCV_LIBS})
 
+    add_dependencies(osd ivslogger framework cvunitext)
+    add_dependencies(osd ivslogger framework)
     target_link_libraries(osd ${BM_LIBS} ${FFMPEG_LIBS} ${OpenCV_LIBS} ${JPU_LIBS} -lpthread)
 
 elseif (${TARGET_ARCH} STREQUAL "soc")
@@ -76,5 +82,12 @@ elseif (${TARGET_ARCH} STREQUAL "soc")
     add_library(osd SHARED
         src/osd.cc
     )
+    add_library(cvunitext SHARED src/cvUniText.cc)
+    target_link_libraries(${demo_name}  cvunitext)
+    add_dependencies(cvunitext freetype)
+    target_link_libraries(cvunitext freetype ${OPENCV_LIBS})
+
+    add_dependencies(osd ivslogger framework cvunitext)
+    add_dependencies(osd ivslogger framework)
     target_link_libraries(osd ${BM_LIBS} ${OPENCV_LIBS} ${JPU_LIBS} -fprofile-arcs -lgcov -lpthread -lavcodec -lavformat -lavutil)
 endif()

element/multimedia/osd/README.md

@@ -38,17 +38,27 @@ sophon-stream osd插件具有一些可配置的参数，可以根据需求进行
 
 |      参数名      |  类型  |              默认值               |                 说明                  |
 | :--------------: | :----: | :-------------------------------: | :-----------------------------------: |
-|     osd_type     | 字符串 |              "TRACK"              | 画图类型，包括 "DET"、"TRACK"、"POSE" |
+|     osd_type     | 字符串 |              "TRACK"              | 画图类型，包括 "DET"、"TRACK"、"POSE"、"ALGORITHM"、"TEXT" ，其中ALGORITHM代表使用draw_func_name所对应的osd函数，TEXT代表在原图任意位置使用硬件绘制文字|
 | class_names_file | 字符串 |                无                 |         class name文件的路径          |
+| recognice_names_file | 字符串 |                无             |         如果有识别子任务的话，表示识别类别名字文件的路径          |
 |    draw_utils    | 字符串 |             "OPENCV"              |    画图工具，包括 "OPENCV"，"BMCV"    |
 |  draw_interval   | 布尔值 |               false               |          是否画出未采样的帧           |
 |     put_text     | 布尔值 |               false               |             是否输出文本              |
-|  shared_object   | 字符串 | "../../../build/lib/libencode.so" |         libencode 动态库路径          |
+|    draw_func_name    | 字符串 |             "default"              |    对应不同ALGORITHM中的osd方式    |
+|  heatmap_loss  |   字符串   | "MSELoss" | 姿态识别训练所使用的损失函数，暂只支持MSELoss |
+|    tops     |  整数数组  |                 无                 |              在TEXT模式下，texts中每个字符串距离图片顶部的垂直距离               |
+|    lefts     |  整数数组  |                 无                 |              在TEXT模式下，texts中每个字符串距离图片左侧的水平距离               |
+|    texts     |  字符串数组  |                 无                 |              在TEXT模式下，要显示的文本内容组成的数组               |
+|    font_library     |  字符串  |                 无                 |              在TEXT模式下使用的字体库文件的路径               |
+|    r    |  整数  |                 0                 |              TEXT模式中文字颜色的r通道值               |
+|    g    |  整数  |                 0                 |              TEXT模式中文字颜色的g通道值               |
+|    b     |  整数  |                 0                 |              TEXT模式中文字颜色的b通道值               |
+|  shared_object   | 字符串 | "../../../build/lib/libosd.so" |         libosd 动态库路径          |
 |    device_id     |  整数  |                 0                 |              tpu 设备号               |
 |        id        |  整数  |                 0                 |              element id               |
 |       name       | 字符串 |               "osd"               |             element 名称              |
 |       side       | 字符串 |             "sophgo"              |               设备类型                |
 |  thread_number   |  整数  |                 4                 | 启动线程数，需要保证和处理码流数一致  |
 
 > **注意**：
-1. osd_type为"DET"时，需提供class_names_file文件地址
+1. osd_type为"DET"时，需提供class_names_file文件地址

element/multimedia/osd/README_EN.md

@@ -38,17 +38,27 @@ The sophon-stream osd plugin has several configurable parameters that can be adj
 
 | Parameter Name   |  name  |        Default value             |                    Description         |
 | :--------------: | :----: | :-------------------------------: | :-----------------------------------: |
-|     osd_type     | string |              "TRACK"              | drawing type,include "DET","TRACK","POSE" |
+|     osd_type     | string |              "TRACK"              | drawing type,include "DET","TRACK","POSE","ALGORITHM","TEXT" |
 | class_names_file | string |                \                 |        file path of class name        |
+| recognice_names_file | String | None | If there is a recognition subtask, this represents the path to the file containing names to be recognized |     |
 |    draw_utils    | string |             "OPENCV"              |    drawing function，include "OPENCV"，"BMCV"    |
 |  draw_interval   | bool |               false               |         Whether to draw unsampled frames  |
 |     put_text     | bool |               false               |             Whether to output text        |
-|  shared_object   | string | "../../../build/lib/libencode.so" |         libencode dynamic library path  |
+| draw_func_name | string | "default" | Corresponds to the OSD method in different ALGORITHMS |
+| heatmap_loss | string | "MSELoss" | Loss function used in pose recognition training, currently only supports MSELoss |
+| tops | array of integers | None | The vertical distance from each string in the texts array to the top of the image in TEXT mode |
+| lefts | array of integers | None | The horizontal distance from each string in the texts array to the left side of the image in TEXT mode |
+| texts | array of strings | None | An array of text strings to be displayed in TEXT mode |
+| font_library | string | None | The path to the font library file used in TEXT mode |
+| r | int | 0 | The r channel value for text color in TEXT mode |
+| g | int | 0 | The g channel value for text color in TEXT mode |
+| b | int | 0 | The b channel value for text color in TEXT mode |
+|  shared_object   | string | "../../../build/lib/libosd.so" |         libosd dynamic library path  |
 |    device_id     |  int  |                 0                 |              tpu device id               |
 |        id        |  int  |                 0                 |              element id               |
 |       name       | string |               "osd"               |             element name             |
 |       side       | string |             "sophgo"              |               device type                |
 |  thread_number   |  int  |                 4                 | Thread number, it should be consistent with the number of streams being processed.  |
 
 > **notes**：
-1. if osd_type is "DET", the address of the class_names_file should be provided.
+1. if osd_type is "DET", the address of the class_names_file should be provided.

element/multimedia/osd/include/cvUniText.h

@@ -0,0 +1,51 @@
+#ifndef CV_UNI_TEXT_HPP
+#define CV_UNI_TEXT_HPP
+#include <memory>
+#include <opencv2/opencv.hpp>
+#include <common/logger.h>
+#include <common/common_defs.h>
+#include <string>
+
+namespace uni_text {
+class Impl;
+
+class UniText {
+ public:
+  /// Initialization
+  /// \param font_face: the file path of your font
+  /// \param font_size
+  UniText(const std::string& font_face, int font_size);
+  ~UniText();
+
+  /// Detailed parameter of text rendering. Call this before drawing the text
+  /// \param font_size
+  /// \param interval_ratio: Ratio of character interval over character width
+  /// \param whitespace_ratio: Ratio of whitespace width over character width
+  /// \param alpha: Transparency. 1 means totally opaque.
+  void SetParam(int font_size, float interval_ratio = 0.1,
+                float whitespace_ratio = 0.5, float alpha = 1);
+
+  /// Draw text on an Opencv Mat
+  /// \param img
+  /// \param utf8_text: Text encoded in utf8. (if it is hardcoded, make sure
+  /// your source file is saved
+  ///                   with utf8 encoding.
+  /// \param org: The left-bottom point of the text. Notice some letters like
+  /// 'g' may go under this point \param color \param calc_size: true -> return
+  /// rect. False -> return rect and draw text.
+  ///                   Useful e.g. when you want to draw a rectangle under the
+  ///                   text
+  /// \return The precise bounding box of the text in the image
+  cv::Rect PutText(cv::Mat& img, const std::string& utf8_text,
+                   const cv::Point& org, const cv::Scalar& color,
+                   bool calc_size = false);
+  bool genBitMap(bm_handle_t mHandle, const std::string& utf8_text,
+                 bm_image& overlay_image, int r, int g, int b);
+
+ private:
+  /// Hide implementations
+  std::unique_ptr<Impl> pimpl;
+};
+}  // namespace uni_text
+
+#endif