From 923a78f64b51cf000ae411bc2e627e790d2aa08a Mon Sep 17 00:00:00 2001 From: "yizhou.xu" Date: Thu, 8 Aug 2024 18:14:48 +0800 Subject: [PATCH 1/8] fix typo in README Change-Id: I751db9c90cfb1a0f36f0689aa5478e121a45aa69 --- element/multimedia/osd/README.md | 4 ++-- element/multimedia/osd/README_EN.md | 4 ++-- element/tools/filter/README.md | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/element/multimedia/osd/README.md b/element/multimedia/osd/README.md index 4b00b5ea..5067061b 100644 --- a/element/multimedia/osd/README.md +++ b/element/multimedia/osd/README.md @@ -43,7 +43,7 @@ sophon-stream osd插件具有一些可配置的参数，可以根据需求进行 | draw_utils | 字符串 | "OPENCV" | 画图工具，包括 "OPENCV"，"BMCV" | | draw_interval | 布尔值 | false | 是否画出未采样的帧 | | put_text | 布尔值 | false | 是否输出文本 | -| shared_object | 字符串 | "../../../build/lib/libencode.so" | libencode 动态库路径 | +| shared_object | 字符串 | "../../../build/lib/libosd.so" | libosd 动态库路径 | | device_id | 整数 | 0 | tpu 设备号 | | id | 整数 | 0 | element id | | name | 字符串 | "osd" | element 名称 | @@ -51,4 +51,4 @@ sophon-stream osd插件具有一些可配置的参数，可以根据需求进行 | thread_number | 整数 | 4 | 启动线程数，需要保证和处理码流数一致 | > **注意**： -1. osd_type为"DET"时，需提供class_names_file文件地址 \ No newline at end of file +1. osd_type为"DET"时，需提供class_names_file文件地址 diff --git a/element/multimedia/osd/README_EN.md b/element/multimedia/osd/README_EN.md index fa6fbaa6..9b8230d0 100644 --- a/element/multimedia/osd/README_EN.md +++ b/element/multimedia/osd/README_EN.md @@ -43,7 +43,7 @@ The sophon-stream osd plugin has several configurable parameters that can be adj | draw_utils | string | "OPENCV" | drawing function，include "OPENCV"，"BMCV" | | draw_interval | bool | false | Whether to draw unsampled frames | | put_text | bool | false | Whether to output text | -| shared_object | string | "../../../build/lib/libencode.so" | libencode dynamic library path | +| shared_object | string | "../../../build/lib/libosd.so" | libosd dynamic library path | | device_id | int | 0 | tpu device id | | id | int | 0 | element id | | name | string | "osd" | element name | @@ -51,4 +51,4 @@ The sophon-stream osd plugin has several configurable parameters that can be adj | thread_number | int | 4 | Thread number, it should be consistent with the number of streams being processed. | > **notes**： -1. if osd_type is "DET", the address of the class_names_file should be provided. \ No newline at end of file +1. if osd_type is "DET", the address of the class_names_file should be provided. diff --git a/element/tools/filter/README.md b/element/tools/filter/README.md index b5357a51..d67c59e8 100644 --- a/element/tools/filter/README.md +++ b/element/tools/filter/README.md @@ -70,8 +70,8 @@ sophon-stream http_push插件具有一些可配置的参数，可以根据需求 | time_start | string | 无 | 开始时间，格式hh mm ss，为空将会全部筛掉 | | time_end | int | 无 | 结束时间，格式hh mm ss,为空将会全部筛掉 | | type | int | 0 | 筛选类型，recognize:0 track:1 classes:other | -| shared_object | string | "../../../build/lib/libhttp_push.so" | libhttp_push动态库路径 | -| name | string | "http_push" | element名称 | +| shared_object | string | "../../../build/lib/libfilter.so" | libfilter动态库路径 | +| name | string | "filter" | element名称 | | side | string | "sophgo" | 设备类型 | | thread_number | int | 1 | 启动线程数 | From 54706e8c811ac4df0a30139af039079aee372c95 Mon Sep 17 00:00:00 2001 From: "yizhou.xu" Date: Fri, 9 Aug 2024 17:26:03 +0800 Subject: [PATCH 2/8] fix type in README.md of ive element Change-Id: I461fbffd831913ede5e0c0b06e5b67178ff2d282 --- element/tools/ive/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/element/tools/ive/README.md b/element/tools/ive/README.md index 2246fffe..b6804282 100644 --- a/element/tools/ive/README.md +++ b/element/tools/ive/README.md @@ -31,7 +31,7 @@ sophon-stream ive插件具有一些可配置的参数，可以根据需求进行 | ive_mapv | string | "../dwa_dpu_encode/data/maps/mapV.txt" | 给DPU结果进行染色的V通道map文件 | | is_ive | bool | true | 选择是否对DPU结果进行染色 | | shared_object | string | "../../../build/lib/libive.so" | libive动态库路径 | -| name | string | "distributor" | element名称 | +| name | string | "ive" | element名称 | | side | string | "sophgo" | 设备类型 | | thread_number | int | 1 | 启动线程数 | From 531e4331716c19a9eb8feba80c5e38da1c68a9d2 Mon Sep 17 00:00:00 2001 From: "yizhou.xu" Date: Sat, 10 Aug 2024 11:57:50 +0800 Subject: [PATCH 3/8] enable default value of channel_id Change-Id: Ib466426f2731381ca94b26f381012bb0091c861b --- samples/src/main.cc | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/samples/src/main.cc b/samples/src/main.cc index 5bb045e9..84b6f0da 100644 --- a/samples/src/main.cc +++ b/samples/src/main.cc @@ -57,6 +57,8 @@ constexpr const char* JSON_CONFIG_HTTP_CONFIG_IP_FILED = "ip"; constexpr const char* JSON_CONFIG_HTTP_CONFIG_PORT_FILED = "port"; constexpr const char* JSON_CONFIG_HTTP_CONFIG_PATH_FILED = "path"; +static int channel_id_config = 0; + demo_config parse_demo_json(std::string& json_path) { std::ifstream istream; istream.open(json_path); @@ -158,9 +160,14 @@ demo_config parse_demo_json(std::string& json_path) { } else { channel_json["graph_id"] = graph_id_it->get(); } - channel_json["channel_id"] = - channel_it.find(JSON_CONFIG_CHANNEL_CONFIG_CHANNEL_ID_FILED) - ->get(); + auto channel_id_it = + channel_it.find(JSON_CONFIG_CHANNEL_CONFIG_CHANNEL_ID_FILED); + if (channel_id_it == channel_it.end()) { + channel_json["channel_id"] = channel_id_config; + channel_id_config++; + } else { + channel_json["channel_id"] = channel_id_it->get(); + } channel_json["url"] = channel_it.find(JSON_CONFIG_CHANNEL_CONFIG_URL_FILED) ->get(); channel_json["source_type"] = @@ -354,7 +361,7 @@ int main(int argc, char* argv[]) { init_engine(engine, engine_json, sinkHandler, graph_src_id_port_map); for (auto& channel_config : demo_json.channel_configs) { - int graph_id = channel_config["graph_id"]; // 默认是graph0 + int graph_id = channel_config["graph_id"]; // 默认是graph0 auto channelTask = std::make_shared(); channelTask->request.operation = sophon_stream::element::decode:: From f6519dfea53e5f71a2984e2451694b74ffe80166 Mon Sep 17 00:00:00 2001 From: "yizhou.xu" Date: Sat, 10 Aug 2024 12:14:29 +0800 Subject: [PATCH 4/8] mv bind draw_func from main() to another function Change-Id: I1f0dc90fbedd0d631f5988d8960b6804430de698 --- samples/src/main.cc | 105 +++++++++++++++++++++++--------------------- 1 file changed, 56 insertions(+), 49 deletions(-) diff --git a/samples/src/main.cc b/samples/src/main.cc index 84b6f0da..7e37eff4 100644 --- a/samples/src/main.cc +++ b/samples/src/main.cc @@ -235,54 +235,10 @@ demo_config parse_demo_json(std::string& json_path) { return config; } -int main(int argc, char* argv[]) { - const char* keys = - "{demo_config_path | " - "../license_plate_recognition/config/license_plate_recognition_demo.json " - "| demo config path}" - "{help | 0 | print help information.}"; - cv::CommandLineParser parser(argc, argv, keys); - if (parser.get("help")) { - parser.printMessage(); - return 0; - } - std::string demo_config_fpath = parser.get("demo_config_path"); - - ::logInit("debug", ""); - - std::mutex mtx; - std::condition_variable cv; - - sophon_stream::common::Clocker clocker; - std::atomic_uint32_t frameCount(0); - std::atomic_int32_t finishedChannelCount(0); - - auto& engine = sophon_stream::framework::SingletonEngine::getInstance(); - - std::ifstream istream; - nlohmann::json engine_json; - demo_config demo_json = parse_demo_json(demo_config_fpath); - - // 启动每个graph, graph之间没有联系，可以是完全不同的配置 - istream.open(demo_json.engine_config_file); - STREAM_CHECK(istream.is_open(), "Please check if engine_config_file ", - demo_json.engine_config_file, " exists."); - istream >> engine_json; - istream.close(); - - // engine.json里的graph数量 - demo_json.num_graphs = engine_json.size(); - // demo.json里的码流数量，这里每个码流都可以配置graph_id，对应不同的graph - demo_json.num_channels_per_graph = demo_json.channel_configs.size(); - // 总的码流数就是demo_json.num_channels_per_graph，这个命名需要修改 - int num_channels = demo_json.num_channels_per_graph; - - std::vector<::sophon_stream::common::FpsProfiler> fpsProfilers(num_channels); - for (int i = 0; i < num_channels; ++i) { - std::string fpsName = "channel_" + std::to_string(i); - fpsProfilers[i].config(fpsName, 100); - } +using drawFuncType = + std::function)>; +drawFuncType getDrawFunc(demo_config& demo_json) { std::function)> draw_func; std::string out_dir = "./results"; @@ -334,6 +290,59 @@ int main(int argc, char* argv[]) { else IVS_ERROR("No such function! Please check your 'draw_func_name'."); + return draw_func; +} + +int main(int argc, char* argv[]) { + const char* keys = + "{demo_config_path | " + "../license_plate_recognition/config/license_plate_recognition_demo.json " + "| demo config path}" + "{help | 0 | print help information.}"; + cv::CommandLineParser parser(argc, argv, keys); + if (parser.get("help")) { + parser.printMessage(); + return 0; + } + std::string demo_config_fpath = parser.get("demo_config_path"); + + ::logInit("debug", ""); + + std::mutex mtx; + std::condition_variable cv; + + sophon_stream::common::Clocker clocker; + std::atomic_uint32_t frameCount(0); + std::atomic_int32_t finishedChannelCount(0); + + auto& engine = sophon_stream::framework::SingletonEngine::getInstance(); + + std::ifstream istream; + nlohmann::json engine_json; + demo_config demo_json = parse_demo_json(demo_config_fpath); + + // 启动每个graph, graph之间没有联系，可以是完全不同的配置 + istream.open(demo_json.engine_config_file); + STREAM_CHECK(istream.is_open(), "Please check if engine_config_file ", + demo_json.engine_config_file, " exists."); + istream >> engine_json; + istream.close(); + + // engine.json里的graph数量 + demo_json.num_graphs = engine_json.size(); + // demo.json里的码流数量，这里每个码流都可以配置graph_id，对应不同的graph + demo_json.num_channels_per_graph = demo_json.channel_configs.size(); + // 总的码流数就是demo_json.num_channels_per_graph，这个命名需要修改 + int num_channels = demo_json.num_channels_per_graph; + + std::vector<::sophon_stream::common::FpsProfiler> fpsProfilers(num_channels); + for (int i = 0; i < num_channels; ++i) { + std::string fpsName = "channel_" + std::to_string(i); + fpsProfilers[i].config(fpsName, 100); + } + + auto draw_func = getDrawFunc(demo_json); + auto sinkHandler = [&, draw_func](std::shared_ptr data) { // write stop data handler here auto objectMetadata = @@ -369,8 +378,6 @@ int main(int argc, char* argv[]) { channelTask->request.channelId = channel_config["channel_id"]; channelTask->request.json = channel_config.dump(); int decode_id = channel_config["decode_id"]; - // std::pair src_id_port = - // graph_src_id_port_map[graph_id][decode_id]; auto src_id_port_vec = graph_src_id_port_map[graph_id]; for (auto& src_id_port : src_id_port_vec) { From 3491b31e93d90ce7fca8d665b63e552e7707073c Mon Sep 17 00:00:00 2001 From: "yihu.wang" Date: Mon, 12 Aug 2024 10:06:16 +0800 Subject: [PATCH 5/8] fix(decoder): memory leak bug in ZUOAN Change-Id: I832f20a73357935468fa1511452c437b78ff6e16 --- element/multimedia/decode/src/ff_decode.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/element/multimedia/decode/src/ff_decode.cc b/element/multimedia/decode/src/ff_decode.cc index fd6b5a66..744b2f12 100644 --- a/element/multimedia/decode/src/ff_decode.cc +++ b/element/multimedia/decode/src/ff_decode.cc @@ -478,17 +478,24 @@ int VideoDecFFM::openDec(bm_handle_t* dec_handle, const char* input) { void VideoDecFFM::closeDec() { if (video_dec_ctx) { + avcodec_close(video_dec_ctx); avcodec_free_context(&video_dec_ctx); video_dec_ctx = NULL; } if (ifmt_ctx) { avformat_close_input(&ifmt_ctx); + avformat_free_context(ifmt_ctx); ifmt_ctx = NULL; } if (frame) { + av_frame_unref(frame); av_frame_free(&frame); frame = NULL; } + if (pkt) { + av_packet_unref(pkt); + av_packet_free(&pkt); + } frame_id = 0; quit_flag = false; } From 6763ea74157e457da68bf4813866d79087a805e7 Mon Sep 17 00:00:00 2001 From: "yuchuan.he" Date: Mon, 27 May 2024 14:22:15 +0800 Subject: [PATCH 6/8] mv draw_finction to osd Change-Id: I74ddd7c38157b776460deaf51a0f71434640f4f9 --- element/multimedia/osd/CMakeLists.txt | 13 + element/multimedia/osd/README.md | 12 +- element/multimedia/osd/README_EN.md | 12 +- .../multimedia/osd}/include/checked.h | 0 .../multimedia/osd}/include/core.h | 0 element/multimedia/osd/include/cvUniText.h | 51 ++ element/multimedia/osd/include/draw_utils.h | 791 +++++++++++++++++- element/multimedia/osd/include/osd.h | 29 +- .../multimedia/osd}/include/unchecked.h | 0 .../multimedia/osd}/include/utf8.h | 0 element/multimedia/osd/src/cvUniText.cc | 342 ++++++++ element/multimedia/osd/src/osd.cc | 171 +++- samples/CMakeLists.txt | 9 +- samples/include/cvUniText.h | 43 - samples/src/cvUniText.cpp | 235 ------ 15 files changed, 1389 insertions(+), 319 deletions(-) rename {samples => element/multimedia/osd}/include/checked.h (100%) rename {samples => element/multimedia/osd}/include/core.h (100%) create mode 100644 element/multimedia/osd/include/cvUniText.h rename {samples => element/multimedia/osd}/include/unchecked.h (100%) rename {samples => element/multimedia/osd}/include/utf8.h (100%) create mode 100644 element/multimedia/osd/src/cvUniText.cc delete mode 100644 samples/include/cvUniText.h delete mode 100644 samples/src/cvUniText.cpp diff --git a/element/multimedia/osd/CMakeLists.txt b/element/multimedia/osd/CMakeLists.txt index 2a1cb881..58866211 100644 --- a/element/multimedia/osd/CMakeLists.txt +++ b/element/multimedia/osd/CMakeLists.txt @@ -44,7 +44,13 @@ if (${TARGET_ARCH} STREQUAL "pcie") add_library(osd SHARED src/osd.cc ) + add_library(cvunitext SHARED src/cvUniText.cc) + target_link_libraries(osd cvunitext) + add_dependencies(cvunitext freetype) + target_link_libraries(cvunitext freetype ${OPENCV_LIBS}) + add_dependencies(osd ivslogger framework cvunitext) + add_dependencies(osd ivslogger framework) target_link_libraries(osd ${BM_LIBS} ${FFMPEG_LIBS} ${OpenCV_LIBS} ${JPU_LIBS} -lpthread) elseif (${TARGET_ARCH} STREQUAL "soc") @@ -76,5 +82,12 @@ elseif (${TARGET_ARCH} STREQUAL "soc") add_library(osd SHARED src/osd.cc ) + add_library(cvunitext SHARED src/cvUniText.cc) + target_link_libraries(${demo_name} cvunitext) + add_dependencies(cvunitext freetype) + target_link_libraries(cvunitext freetype ${OPENCV_LIBS}) + + add_dependencies(osd ivslogger framework cvunitext) + add_dependencies(osd ivslogger framework) target_link_libraries(osd ${BM_LIBS} ${OPENCV_LIBS} ${JPU_LIBS} -fprofile-arcs -lgcov -lpthread -lavcodec -lavformat -lavutil) endif() diff --git a/element/multimedia/osd/README.md b/element/multimedia/osd/README.md index 5067061b..4760d98c 100644 --- a/element/multimedia/osd/README.md +++ b/element/multimedia/osd/README.md @@ -38,11 +38,21 @@ sophon-stream osd插件具有一些可配置的参数，可以根据需求进行 | 参数名 | 类型 | 默认值 | 说明 | | :--------------: | :----: | :-------------------------------: | :-----------------------------------: | -| osd_type | 字符串 | "TRACK" | 画图类型，包括 "DET"、"TRACK"、"POSE" | +| osd_type | 字符串 | "TRACK" | 画图类型，包括 "DET"、"TRACK"、"POSE"、"ALGORITHM"、"TEXT" ，其中ALGORITHM代表使用draw_func_name所对应的osd函数，TEXT代表在原图任意位置使用硬件绘制文字| | class_names_file | 字符串 | 无 | class name文件的路径 | +| recognice_names_file | 字符串 | 无 | 如果有识别子任务的话，表示识别类别名字文件的路径 | | draw_utils | 字符串 | "OPENCV" | 画图工具，包括 "OPENCV"，"BMCV" | | draw_interval | 布尔值 | false | 是否画出未采样的帧 | | put_text | 布尔值 | false | 是否输出文本 | +| draw_func_name | 字符串 | "default" | 对应不同ALGORITHM中的osd方式 | +| heatmap_loss | 字符串 | "MSELoss" | 姿态识别训练所使用的损失函数，暂只支持MSELoss | +| tops | 整数数组 | 无 | 在TEXT模式下，texts中每个字符串距离图片顶部的垂直距离 | +| lefts | 整数数组 | 无 | 在TEXT模式下，texts中每个字符串距离图片左侧的水平距离 | +| texts | 字符串数组 | 无 | 在TEXT模式下，要显示的文本内容组成的数组 | +| font_library | 字符串 | 无 | 在TEXT模式下使用的字体库文件的路径 | +| r | 整数 | 0 | TEXT模式中文字颜色的r通道值 | +| g | 整数 | 0 | TEXT模式中文字颜色的g通道值 | +| b | 整数 | 0 | TEXT模式中文字颜色的b通道值 | | shared_object | 字符串 | "../../../build/lib/libosd.so" | libosd 动态库路径 | | device_id | 整数 | 0 | tpu 设备号 | | id | 整数 | 0 | element id | diff --git a/element/multimedia/osd/README_EN.md b/element/multimedia/osd/README_EN.md index 9b8230d0..2d76e211 100644 --- a/element/multimedia/osd/README_EN.md +++ b/element/multimedia/osd/README_EN.md @@ -38,11 +38,21 @@ The sophon-stream osd plugin has several configurable parameters that can be adj | Parameter Name | name | Default value | Description | | :--------------: | :----: | :-------------------------------: | :-----------------------------------: | -| osd_type | string | "TRACK" | drawing type,include "DET","TRACK","POSE" | +| osd_type | string | "TRACK" | drawing type,include "DET","TRACK","POSE","ALGORITHM","TEXT" | | class_names_file | string | \ | file path of class name | +| recognice_names_file | String | None | If there is a recognition subtask, this represents the path to the file containing names to be recognized | | | draw_utils | string | "OPENCV" | drawing function，include "OPENCV"，"BMCV" | | draw_interval | bool | false | Whether to draw unsampled frames | | put_text | bool | false | Whether to output text | +| draw_func_name | string | "default" | Corresponds to the OSD method in different ALGORITHMS | +| heatmap_loss | string | "MSELoss" | Loss function used in pose recognition training, currently only supports MSELoss | +| tops | array of integers | None | The vertical distance from each string in the texts array to the top of the image in TEXT mode | +| lefts | array of integers | None | The horizontal distance from each string in the texts array to the left side of the image in TEXT mode | +| texts | array of strings | None | An array of text strings to be displayed in TEXT mode | +| font_library | string | None | The path to the font library file used in TEXT mode | +| r | int | 0 | The r channel value for text color in TEXT mode | +| g | int | 0 | The g channel value for text color in TEXT mode | +| b | int | 0 | The b channel value for text color in TEXT mode | | shared_object | string | "../../../build/lib/libosd.so" | libosd dynamic library path | | device_id | int | 0 | tpu device id | | id | int | 0 | element id | diff --git a/samples/include/checked.h b/element/multimedia/osd/include/checked.h similarity index 100% rename from samples/include/checked.h rename to element/multimedia/osd/include/checked.h diff --git a/samples/include/core.h b/element/multimedia/osd/include/core.h similarity index 100% rename from samples/include/core.h rename to element/multimedia/osd/include/core.h diff --git a/element/multimedia/osd/include/cvUniText.h b/element/multimedia/osd/include/cvUniText.h new file mode 100644 index 00000000..8f05115b --- /dev/null +++ b/element/multimedia/osd/include/cvUniText.h @@ -0,0 +1,51 @@ +#ifndef CV_UNI_TEXT_HPP +#define CV_UNI_TEXT_HPP +#include +#include +#include +#include +#include + +namespace uni_text { +class Impl; + +class UniText { + public: + /// Initialization + /// \param font_face: the file path of your font + /// \param font_size + UniText(const std::string& font_face, int font_size); + ~UniText(); + + /// Detailed parameter of text rendering. Call this before drawing the text + /// \param font_size + /// \param interval_ratio: Ratio of character interval over character width + /// \param whitespace_ratio: Ratio of whitespace width over character width + /// \param alpha: Transparency. 1 means totally opaque. + void SetParam(int font_size, float interval_ratio = 0.1, + float whitespace_ratio = 0.5, float alpha = 1); + + /// Draw text on an Opencv Mat + /// \param img + /// \param utf8_text: Text encoded in utf8. (if it is hardcoded, make sure + /// your source file is saved + /// with utf8 encoding. + /// \param org: The left-bottom point of the text. Notice some letters like + /// 'g' may go under this point \param color \param calc_size: true -> return + /// rect. False -> return rect and draw text. + /// Useful e.g. when you want to draw a rectangle under the + /// text + /// \return The precise bounding box of the text in the image + cv::Rect PutText(cv::Mat& img, const std::string& utf8_text, + const cv::Point& org, const cv::Scalar& color, + bool calc_size = false); + bool genBitMap(bm_handle_t mHandle, const std::string& utf8_text, + bm_image& overlay_image, int r, int g, int b); + + private: + /// Hide implementations + std::unique_ptr pimpl; +}; +} // namespace uni_text + +#endif diff --git a/element/multimedia/osd/include/draw_utils.h b/element/multimedia/osd/include/draw_utils.h index b4baff00..4505a0d3 100644 --- a/element/multimedia/osd/include/draw_utils.h +++ b/element/multimedia/osd/include/draw_utils.h @@ -11,17 +11,34 @@ #define SOPHON_STREAM_ELEMENT_OSD_DRAW_UTILS_H_ #include +#include +#include +#include #include +#include #include #include #include +#include #include +#include +#include "common/clocker.h" +#include "common/common_defs.h" +#include "common/error_code.h" #include "common/logger.h" +#include "common/object_metadata.h" #include "common/posed_object_metadata.h" +#include "cvUniText.h" #include "element_factory.h" - +extern "C" { +extern bm_status_t bmcv_image_overlay(bm_handle_t handle, bm_image image, + int overlay_num, + bmcv_rect_t* overlay_info, + bm_image* overlay_image) + __attribute__((weak)); +} namespace sophon_stream { namespace element { namespace osd { @@ -182,7 +199,8 @@ void draw_bmcv_track_result( void draw_opencv_det_result( std::shared_ptr objectMetadata, - std::vector& class_names, cv::Mat& frame, bool put_text_flag, bool draw_interval) { + std::vector& class_names, cv::Mat& frame, bool put_text_flag, + bool draw_interval) { // Draw a rectangle displaying the bounding box int colors_num = colors.size(); int thickness = 2; @@ -339,8 +357,7 @@ void draw_opencv_pose_result( const auto index2 = (pairs[pair + 1]) * 3; if (poseKeypoints[index1 + 2] > threshold && poseKeypoints[index2 + 2] > threshold) { - const auto colorIndex = - pairs[pair + 1] * 3; + const auto colorIndex = pairs[pair + 1] * 3; const cv::Scalar color{pose_colors[(colorIndex + 2) % numberColors], pose_colors[(colorIndex + 1) % numberColors], pose_colors[(colorIndex + 0) % numberColors]}; @@ -354,27 +371,761 @@ void draw_opencv_pose_result( } } } -void draw_opencv_areas(std::shared_ptr objectMetadata,cv::Mat &frame_to_draw){ - for(int i=0;iareas.size();i++){ - if(objectMetadata->areas[i].size()==2){ - const cv::Point start={objectMetadata->areas[i][0].mY,objectMetadata->areas[i][0].mX}; - const cv::Point end={objectMetadata->areas[i][1].mY,objectMetadata->areas[i][1].mX}; - const cv::Scalar color={255,0,0}; - cv::line(frame_to_draw, start,end, color, 3, 8, 0); +void draw_opencv_areas(std::shared_ptr objectMetadata, + cv::Mat& frame_to_draw) { + for (int i = 0; i < objectMetadata->areas.size(); i++) { + if (objectMetadata->areas[i].size() == 2) { + const cv::Point start = {objectMetadata->areas[i][0].mY, + objectMetadata->areas[i][0].mX}; + const cv::Point end = {objectMetadata->areas[i][1].mY, + objectMetadata->areas[i][1].mX}; + const cv::Scalar color = {255, 0, 0}; + cv::line(frame_to_draw, start, end, color, 3, 8, 0); + } + } +} +void draw_bmcv_areas(std::shared_ptr objectMetadata, + bm_image& frame_to_draw) { + for (int i = 0; i < objectMetadata->areas.size(); i++) { + if (objectMetadata->areas[i].size() == 2) { + bmcv_point_t start = {objectMetadata->areas[i][0].mY, + objectMetadata->areas[i][0].mX}; + bmcv_point_t end = {objectMetadata->areas[i][1].mY, + objectMetadata->areas[i][1].mX}; + bmcv_color_t color = {255, 0, 0}; + bmcv_image_draw_lines(objectMetadata->mFrame->mHandle, frame_to_draw, + &start, &end, 1, color, 3); + } + } +} + +#define POSE_COLORS_RENDER_CPU \ + 255.f, 0.f, 0.f, 255.f, 85.f, 0.f, 255.f, 170.f, 0.f, 255.f, 255.f, 0.f, \ + 170.f, 255.f, 0.f, 85.f, 255.f, 0.f, 0.f, 255.f, 0.f, 0.f, 255.f, 85.f, \ + 0.f, 255.f, 170.f, 0.f, 255.f, 255.f, 0.f, 170.f, 255.f, 0.f, 85.f, \ + 255.f, 0.f, 0.f, 255.f, 85.f, 0.f, 255.f, 170.f, 0.f, 255.f, 255.f, 0.f, \ + 255.f, 255.f, 0.f, 170.f, 255.f, 0.f, 85.f, 255.f, 0.f, 0.f, 255.f, 0.f, \ + 255.f, 255.f, 85.f, 255.f, 255.f, 170.f, 255.f, 255.f, 255.f, 255.f, \ + 170.f, 255.f, 255.f, 85.f, 255.f, 255.f + +const std::vector POSE_COLORS_RENDER{POSE_COLORS_RENDER_CPU}; + +// Max/min functions +template +static T fastMax(const T a, const T b) { + return (a > b ? a : b); +} + +static void _draw_rectangle_and_text_bmcv( + bm_handle_t& handle, std::string& lable, int left, int top, int width, + int height, bm_image& frame, const std::vector& color, + bool put_text_flag) // Draw the predicted bounding box +{ + // Draw a rectangle displaying the bounding box + bmcv_rect_t rect; + rect.start_x = left; + rect.start_y = top; + rect.crop_w = width; + rect.crop_h = height; + std::cout << rect.start_x << "," << rect.start_y << "," << rect.crop_w << "," + << rect.crop_h << std::endl; + int ret = bmcv_image_draw_rectangle(handle, frame, 1, &rect, 3, color[0], + color[1], color[2]); + if (put_text_flag) { + bmcv_point_t org = {left, top - 10}; + bmcv_color_t bmcv_color = {color[0], color[1], color[2]}; + int thickness = 2; + float fontScale = 1.5; + + if (BM_SUCCESS != bmcv_image_put_text(handle, frame, lable.c_str(), org, + bmcv_color, fontScale, thickness)) { + std::cout << "bmcv put text error !!!" << std::endl; + } + } +} + +static void _draw_face_rectangle_bmcv( + bm_handle_t& handle, + std::shared_ptr results, + bm_image& frame) // Draw the predicted bounding box +{ + // Draw a rectangle displaying the bounding box + bmcv_rect_t rect; + for (size_t j = 0; j < results->mFaceObjectMetadatas.size(); j++) { + rect.start_x = results->mFaceObjectMetadatas[j]->left; + rect.start_y = results->mFaceObjectMetadatas[j]->top; + rect.crop_w = results->mFaceObjectMetadatas[j]->right - + results->mFaceObjectMetadatas[j]->left + 1; + rect.crop_h = results->mFaceObjectMetadatas[j]->bottom - + results->mFaceObjectMetadatas[j]->top + 1; + + std::cout << rect.start_x << "," << rect.start_y << "," << rect.crop_w + << "," << rect.crop_h << std::endl; + + bmcv_image_draw_rectangle(handle, frame, 1, &rect, 3, 255, 2, 2); + } +} + +static void _draw_text_bmcv( + bm_handle_t& handle, int left, int top, bm_image& frame, + std::string label) // Draw the predicted bounding box +{ + bmcv_point_t org = {left, top + 40}; + bmcv_color_t bmcv_color = {255, 0, 0}; + int thickness = 2; + float fontScale = 1.5; + if (BM_SUCCESS != bmcv_image_put_text(handle, frame, label.c_str(), org, + bmcv_color, fontScale, thickness)) { + std::cout << "bmcv put text error !!!" << std::endl; + } +} + +static std::vector _get_pose_pairs( + sophon_stream::common::PosedObjectMetadata::EModelType model_type) { + switch (model_type) { + case sophon_stream::common::PosedObjectMetadata::EModelType::BODY_25: + return {1, 8, 1, 2, 1, 5, 2, 3, 3, 4, 5, 6, 6, + 7, 8, 9, 9, 10, 10, 11, 8, 12, 12, 13, 13, 14, + 1, 0, 0, 15, 15, 17, 0, 16, 16, 18, 2, 17, 5, + 18, 14, 19, 19, 20, 14, 21, 11, 22, 22, 23, 11, 24}; + case sophon_stream::common::PosedObjectMetadata::EModelType::COCO_18: + return {1, 2, 1, 5, 2, 3, 3, 4, 5, 6, 6, 7, 1, + 8, 8, 9, 9, 10, 1, 11, 11, 12, 12, 13, 1, 0, + 0, 14, 14, 16, 0, 15, 15, 17, 2, 16, 5, 17}; + default: + // COCO_18 + return {1, 2, 1, 5, 2, 3, 3, 4, 5, 6, 6, 7, 1, + 8, 8, 9, 9, 10, 1, 11, 11, 12, 12, 13, 1, 0, + 0, 14, 14, 16, 0, 15, 15, 17, 2, 16, 5, 17}; + } +} + +static std::vector _get_fastpose_pairs(int kp_num) { + switch (kp_num) { + case 17: + return {0, 1, 0, 2, 1, 3, 2, 4, 5, 6, 5, 7, 7, 9, + 6, 8, 8, 10, 11, 12, 11, 13, 12, 14, 13, 15, 14, 16}; + case 136: + return { + 0, 1, 0, 2, 1, 3, 2, 4, 5, 18, 6, 18, 5, 7, + 7, 9, 6, 8, 8, 10, 17, 18, 18, 19, 19, 11, 19, 12, + 11, 13, 12, 14, 13, 15, 14, 16, 20, 24, 21, 25, 23, 25, + 22, 24, 15, 24, 16, 25, 26, 27, 27, 28, 28, 29, 29, 30, + 30, 31, 31, 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 37, + 37, 38, 38, 39, 39, 40, 40, 41, 41, 42, 43, 44, 44, 45, + 45, 46, 46, 47, 48, 49, 49, 50, 50, 51, 51, 52, 53, 54, + 54, 55, 55, 56, 57, 58, 58, 59, 59, 60, 60, 61, 62, 63, + 63, 64, 64, 65, 65, 66, 66, 67, 68, 69, 69, 70, 70, 71, + 71, 72, 72, 73, 74, 75, 75, 76, 76, 77, 77, 78, 78, 79, + 79, 80, 80, 81, 81, 82, 82, 83, 83, 84, 84, 85, 85, 86, + 86, 87, 87, 88, 88, 89, 89, 90, 90, 91, 91, 92, 92, 93, + 94, 95, 95, 96, 96, 97, 97, 98, 94, 99, 99, 100, 100, 101, + 101, 102, 94, 103, 103, 104, 104, 105, 105, 106, 94, 107, 107, 108, + 108, 109, 109, 110, 94, 111, 111, 112, 112, 113, 113, 114, 115, 116, + 116, 117, 117, 118, 118, 119, 115, 120, 120, 121, 121, 122, 122, 123, + 115, 124, 124, 125, 125, 126, 126, 127, 115, 128, 128, 129, 129, 130, + 130, 131, 115, 132, 132, 133, 133, 134, 134, 135}; + case 133: + return { + 0, 1, 0, 2, 1, 3, 2, 4, 5, 7, 7, 9, 6, 8, + 8, 10, 11, 13, 12, 14, 13, 15, 14, 16, 18, 19, 21, 22, + 20, 22, 17, 19, 15, 19, 16, 22, 23, 24, 24, 25, 25, 26, + 26, 27, 27, 28, 28, 29, 29, 30, 30, 31, 31, 32, 32, 33, + 33, 34, 34, 35, 35, 36, 36, 37, 37, 38, 38, 39, 40, 41, + 41, 42, 42, 43, 43, 44, 45, 46, 46, 47, 47, 48, 48, 49, + 50, 51, 51, 52, 52, 53, 54, 55, 55, 56, 56, 57, 57, 58, + 59, 60, 60, 61, 61, 62, 62, 63, 63, 64, 65, 66, 66, 67, + 67, 68, 68, 69, 69, 70, 71, 72, 72, 73, 73, 74, 74, 75, + 75, 76, 76, 77, 77, 78, 78, 79, 79, 80, 80, 81, 81, 82, + 82, 83, 83, 84, 84, 85, 85, 86, 86, 87, 87, 88, 88, 89, + 89, 90, 91, 92, 92, 93, 93, 94, 94, 95, 91, 96, 96, 97, + 97, 98, 98, 99, 91, 100, 100, 101, 101, 102, 102, 103, 91, 104, + 104, 105, 105, 106, 106, 107, 91, 108, 108, 109, 109, 110, 110, 111, + 112, 113, 113, 114, 114, 115, 115, 116, 112, 117, 117, 118, 118, 119, + 119, 120, 112, 121, 121, 122, 122, 123, 123, 124, 112, 125, 125, 126, + 126, 127, 127, 128, 112, 129, 129, 130, 130, 131, 131, 132}; + case 68: + return {0, 1, 0, 2, 1, 3, 2, 4, 5, 18, 6, 18, 5, 7, 7, 9, + 6, 8, 8, 10, 17, 18, 18, 19, 19, 11, 19, 12, 11, 13, 12, 14, + 13, 15, 14, 16, 20, 24, 21, 25, 23, 25, 22, 24, 15, 24, 16, 25, + 26, 27, 27, 28, 28, 29, 29, 30, 26, 31, 31, 32, 32, 33, 33, 34, + 26, 35, 35, 36, 36, 37, 37, 38, 26, 39, 39, 40, 40, 41, 41, 42, + 26, 43, 43, 44, 44, 45, 45, 46, 47, 48, 48, 49, 49, 50, 50, 51, + 47, 52, 52, 53, 53, 54, 54, 55, 47, 56, 56, 57, 57, 58, 58, 59, + 47, 60, 60, 61, 61, 62, 62, 63, 47, 64, 64, 65, 65, 66, 66, 67}; + case 26: + return { + 0, 1, 0, 2, 1, 3, 2, 4, 5, 18, 6, 18, 5, 7, 7, 9, + 6, 8, 8, 10, 17, 18, 18, 19, 19, 11, 19, 12, 11, 13, 12, 14, + 13, 15, 14, 16, 20, 24, 21, 25, 23, 25, 22, 24, 15, 24, 16, 25, + }; + case 21: + return {0, 1, 1, 2, 2, 3, 3, 4, 0, 5, 5, 6, 6, 7, 7, 8, + 0, 9, 9, 10, 10, 11, 11, 12, 0, 13, 13, 14, 14, 15, 15, 16, + 0, 17, 17, 18, 18, 19, 19, 20, 21, 22, 22, 23, 23, 24, 24, 25, + 21, 26, 26, 27, 27, 28, 28, 29, 21, 30, 30, 31, 31, 32, 32, 33, + 21, 34, 34, 35, 35, 36, 36, 37, 21, 38, 38, 39, 39, 40, 40, 41}; + } +} + +static std::vector _get_fastpose_p_color(int kp_num) { + switch (kp_num) { + case 17: + return {0, 255, 255, 0, 191, 255, 0, 255, 102, 0, 77, + 255, 0, 255, 0, 77, 255, 255, 77, 255, 204, 77, + 204, 255, 191, 255, 77, 77, 191, 255, 191, 255, 77, + 204, 77, 255, 77, 255, 204, 191, 77, 255, 77, 255, + 191, 127, 77, 255, 77, 255, 127, 0, 255, 255}; + case 136: + return {0, 255, 255, 0, 191, 255, 0, 255, 102, 0, 77, 255, 0, + 255, 0, 77, 255, 255, 77, 255, 204, 77, 204, 255, 191, 255, + 77, 77, 191, 255, 191, 255, 77, 204, 77, 255, 77, 255, 204, + 191, 77, 255, 77, 255, 191, 127, 77, 255, 77, 255, 127, 77, + 255, 255, 0, 255, 255, 77, 204, 255, 0, 255, 255, 0, 191, + 255, 0, 255, 102, 0, 77, 255, 0, 255, 0, 77, 255, 255}; + case 133: + return {0, 255, 255, 0, 191, 255, 0, 255, 102, 0, 77, 255, + 0, 255, 0, 77, 255, 255, 77, 255, 204, 77, 204, 255, + 191, 255, 77, 77, 191, 255, 191, 255, 77, 204, 77, 255, + 77, 255, 204, 191, 77, 255, 77, 255, 191, 127, 77, 255, + 77, 255, 127, 0, 255, 255, 0, 191, 255, 0, 255, 102, + 0, 77, 255, 0, 255, 0, 77, 255, 255}; + case 68: + return {0, 255, 255, 0, 191, 255, 0, 255, 102, 0, 77, 255, 0, + 255, 0, 77, 255, 255, 77, 255, 204, 77, 204, 255, 191, 255, + 77, 77, 191, 255, 191, 255, 77, 204, 77, 255, 77, 255, 204, + 191, 77, 255, 77, 255, 191, 127, 77, 255, 77, 255, 127, 77, + 255, 255, 0, 255, 255, 77, 204, 255, 0, 255, 255, 0, 191, + 255, 0, 255, 102, 0, 77, 255, 0, 255, 0, 77, 255, 255}; + case 26: + return {0, 255, 255, 0, 191, 255, 0, 255, 102, 0, 77, 255, 0, + 255, 0, 77, 255, 255, 77, 255, 204, 77, 204, 255, 191, 255, + 77, 77, 191, 255, 191, 255, 77, 204, 77, 255, 77, 255, 204, + 191, 77, 255, 77, 255, 191, 127, 77, 255, 77, 255, 127, 77, + 255, 255, 0, 255, 255, 77, 204, 255, 0, 255, 255, 0, 191, + 255, 0, 255, 102, 0, 77, 255, 0, 255, 0, 77, 255, 255}; + case 21: + return {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}; + } +} + +static std::vector _get_fastpose_line_color(int kp_num) { + switch (kp_num) { + case 17: + return {0, 215, 255, 0, 255, 204, 0, 134, 255, 0, 255, 50, + 77, 255, 222, 77, 196, 255, 77, 135, 255, 191, 255, 77, + 77, 255, 77, 77, 222, 255, 255, 156, 127, 0, 127, 255, + 255, 127, 77, 0, 77, 255, 255, 77, 36}; + case 136: + return {0, 215, 255, 0, 255, 204, 0, 134, 255, 0, 255, 50, + 0, 255, 102, 77, 255, 222, 77, 196, 255, 77, 135, 255, + 191, 255, 77, 77, 255, 77, 77, 191, 255, 204, 77, 255, + 77, 222, 255, 255, 156, 127, 0, 127, 255, 255, 127, 77, + 0, 77, 255, 255, 77, 36, 0, 77, 255, 0, 77, 255, + 0, 77, 255, 0, 77, 255, 255, 156, 127, 255, 156, 127}; + case 133: + return {0, 215, 255, 0, 255, 204, 0, 134, 255, 0, 255, + 50, 0, 255, 102, 77, 255, 222, 77, 196, 255, 77, + 135, 255, 191, 255, 77, 77, 255, 77, 77, 191, 255, + 204, 77, 255, 77, 222, 255, 255, 156, 127, 0, 127, + 255, 255, 127, 77, 0, 77, 255, 255, 77, 36, 0, + 77, 255, 0, 77, 255, 0, 77, 255, 0, 77, 255}; + case 68: + return {0, 215, 255, 0, 255, 204, 0, 134, 255, 0, 255, 50, + 0, 255, 102, 77, 255, 222, 77, 196, 255, 77, 135, 255, + 191, 255, 77, 77, 255, 77, 77, 191, 255, 204, 77, 255, + 77, 222, 255, 255, 156, 127, 0, 127, 255, 255, 127, 77, + 0, 77, 255, 255, 77, 36, 0, 77, 255, 0, 77, 255, + 0, 77, 255, 0, 77, 255, 255, 156, 127, 255, 156, 127}; + case 26: + return {0, 215, 255, 0, 255, 204, 0, 134, 255, 0, 255, 50, + 0, 255, 102, 77, 255, 222, 77, 196, 255, 77, 135, 255, + 191, 255, 77, 77, 255, 77, 77, 191, 255, 204, 77, 255, + 77, 222, 255, 255, 156, 127, 0, 127, 255, 255, 127, 77, + 0, 77, 255, 255, 77, 36, 0, 77, 255, 0, 77, 255, + 0, 77, 255, 0, 77, 255, 255, 156, 127, 255, 156, 127}; + case 21: + return {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}; + } +} + +static void _render_keypoints_bmcv(bm_handle_t& handle, bm_image& frame, + const std::vector& keypoints, + const std::vector& pairs, + const std::vector colors, + const float thicknessCircleRatio, + const float thicknessLineRatioWRTCircle, + const float threshold, float scale) { + // Get frame channels + const auto width = frame.width; + const auto height = frame.height; + const auto area = width * height; + + // Parameters + const auto lineType = 8; + const auto shift = 0; + const auto numberColors = colors.size(); + const auto thresholdRectangle = 0.1f; + + // Keypoints + + const auto ratioAreas = 1; + // Size-dependent variables + const auto thicknessRatio = + fastMax(intRound(std::sqrt(area) * thicknessCircleRatio * ratioAreas), 1); + // Negative thickness in cv::circle means that a filled circle is to + // be drawn. + const auto thicknessCircle = (ratioAreas > 0.05 ? thicknessRatio : -1); + const auto thicknessLine = + 2; // intRound(thicknessRatio * thicknessLineRatioWRTCircle); + const auto radius = thicknessRatio / 2; + + // Draw lines + for (auto pair = 0u; pair < pairs.size(); pair += 2) { + const auto index1 = (pairs[pair]) * 3; + const auto index2 = (pairs[pair + 1]) * 3; + + if (keypoints[index1 + 2] > threshold && + keypoints[index2 + 2] > threshold) { + const auto colorIndex = pairs[pair + 1] * 3; + bmcv_color_t color = {colors[(colorIndex + 2) % numberColors], + colors[(colorIndex + 1) % numberColors], + colors[(colorIndex + 0) % numberColors]}; + bmcv_point_t start = {intRound(keypoints[index1] * scale), + intRound(keypoints[index1 + 1] * scale)}; + bmcv_point_t end = {intRound(keypoints[index2] * scale), + intRound(keypoints[index2 + 1] * scale)}; + + if (BM_SUCCESS != bmcv_image_draw_lines(handle, frame, &start, &end, 1, + color, thicknessLine)) { + std::cout << "bmcv draw lines error !!!" << std::endl; + } + } + } +} + +static void _render_pose_keypoints_bmcv( + bm_handle_t& handle, bm_image& frame, + const std::vector& poseKeypoints, const float renderThreshold, + float scale, + sophon_stream::common::PosedObjectMetadata::EModelType modelType, + const bool blendOriginalFrame) { + // Parameters + const auto thicknessCircleRatio = 1.f / 75.f; + const auto thicknessLineRatioWRTCircle = 0.75f; + const auto& pairs = _get_pose_pairs(modelType); + + // Render keypoints + _render_keypoints_bmcv(handle, frame, poseKeypoints, pairs, + POSE_COLORS_RENDER, thicknessCircleRatio, + thicknessLineRatioWRTCircle, renderThreshold, scale); +} + +static void _render_fastpose_keypoints_bmcv( + bm_handle_t& handle, bm_image& frame, const std::vector& keypoints, + const std::vector& scores, const std::vector& pairs, + const std::vector p_colors, const std::vector line_colors, + const float thicknessCircleRatio, const float thicknessLineRatioWRTCircle, + std::string loss_type, float scale) { + // Get frame channels + const auto width = frame.width; + const auto height = frame.height; + const auto area = width * height; + + // Parameters + const auto lineType = 8; + const auto shift = 0; + const auto pNumberColors = p_colors.size(); + const auto lineNumberColors = line_colors.size(); + const auto thresholdRectangle = 0.1f; + float threshold1 = 0.4, threshold2 = 0.4; + if (loss_type == "L1JointRegression") threshold1 = 0.05, threshold2 = 0.05; + + // Keypoints + + const auto ratioAreas = 1; + // Size-dependent variables + const auto thicknessRatio = + fastMax(intRound(std::sqrt(area) * thicknessCircleRatio * ratioAreas), 1); + // Negative thickness in cv::circle means that a filled circle is to + // be drawn. + const auto thicknessCircle = (ratioAreas > 0.05 ? thicknessRatio : -1); + const auto thicknessLine = + 2; // intRound(thicknessRatio * thicknessLineRatioWRTCircle); + const auto radius = thicknessRatio / 2; + + // Draw lines + for (auto pair = 0u; pair < pairs.size(); pair += 2) { + const auto index1 = pairs[pair]; + const auto index2 = pairs[pair + 1]; + + if (loss_type == "Combined" && keypoints.size() / 2 == 68) { + if (index1 >= 26) + threshold1 = 0.05; + else + threshold1 = 0.4; + if (index2 >= 26) + threshold2 = 0.05; + else + threshold2 = 0.4; + } else if (loss_type == "Combined") { + if (index1 >= keypoints.size() / 2 - 110) + threshold1 = 0.05; + else + threshold1 = 0.4; + if (index2 >= keypoints.size() / 2 - 110) + threshold2 = 0.05; + else + threshold2 = 0.4; } + + if (scores[index1] > threshold1 && scores[index2] > threshold2) { + bmcv_color_t color; + if (pair / 2 < line_colors.size() / 3) + color = {line_colors[pair / 2 * 3 + 2], line_colors[pair / 2 * 3 + 1], + line_colors[pair / 2 * 3 + 0]}; + else + color = {255, 255, 255}; + bmcv_point_t start = {intRound(keypoints[index1 * 2] * scale), + intRound(keypoints[index1 * 2 + 1] * scale)}; + bmcv_point_t end = {intRound(keypoints[index2 * 2] * scale), + intRound(keypoints[index2 * 2 + 1] * scale)}; + + if (BM_SUCCESS != bmcv_image_draw_lines(handle, frame, &start, &end, 1, + color, thicknessLine)) { + std::cout << "bmcv draw lines error !!!" << std::endl; + } + } + } +} + +static void _render_fastpose_keypoints(bm_handle_t& handle, bm_image& frame, + const std::vector& poseKeypoints, + const std::vector& scores, + std::string loss_type, float scale) { + // Parameters + const auto thicknessCircleRatio = 1.f / 75.f; + const auto thicknessLineRatioWRTCircle = 0.75f; + const auto& pairs = _get_fastpose_pairs(poseKeypoints.size() / 2); + const auto& p_color = _get_fastpose_p_color(poseKeypoints.size() / 2); + const auto& line_color = _get_fastpose_line_color(poseKeypoints.size() / 2); + + // Render keypoints + _render_fastpose_keypoints_bmcv( + handle, frame, poseKeypoints, scores, pairs, p_color, line_color, + thicknessCircleRatio, thicknessLineRatioWRTCircle, loss_type, scale); +} + +void draw_bytetrack_results( + std::shared_ptr objectMetadata, + bm_image& frame) { + int colors_num = colors.size(); + for (int i = 0; i < objectMetadata->mTrackedObjectMetadatas.size(); i++) { + // draw image + auto trackObj = objectMetadata->mTrackedObjectMetadatas[i]; + auto detObj = objectMetadata->mDetectedObjectMetadatas[i]; + std::string label = std::to_string(trackObj->mTrackId); + _draw_rectangle_and_text_bmcv( + objectMetadata->mFrame->mHandle, label, detObj->mBox.mX, + detObj->mBox.mY, detObj->mBox.mWidth, detObj->mBox.mHeight, frame, + colors[trackObj->mTrackId % colors_num], true); + } +} + +void draw_license_plate_recognition_results_opencv( + std::shared_ptr objectMetadata, + cv::Mat& img) { + uni_text::UniText uniText( + "../license_plate_recognition/data/wqy-microhei.ttc", 22); + bm_image frame; + cv::bmcv::toBMI(img, &frame); + // get license plate, and draw pics + if (objectMetadata->mSubObjectMetadatas.size() > 0) { + for (auto subObj : objectMetadata->mSubObjectMetadatas) { + IVS_WARN("get recognized license plate datas from yolo and lprnet"); + int subId = subObj->mSubId; + auto reconizedObj = subObj->mRecognizedObjectMetadatas[0]; + auto detObj = objectMetadata->mDetectedObjectMetadatas[subId]; + + // get license plate + std::string oriLabel = reconizedObj->mLabelName; + + // draw pics for wide str + _draw_rectangle_and_text_bmcv(objectMetadata->mFrame->mHandle, oriLabel, + detObj->mBox.mX, detObj->mBox.mY, + detObj->mBox.mWidth, detObj->mBox.mHeight, + frame, colors[0], false); + uniText.PutText(img, oriLabel, + cv::Point(detObj->mBox.mX, detObj->mBox.mY), + cv::Scalar(0, 0, 255), false); + } + } +} +void draw_retinaface_results( + std::shared_ptr objectMetadata, + bm_image& frame) { + _draw_face_rectangle_bmcv(objectMetadata->mFrame->mHandle, objectMetadata, + frame); +} + +void draw_retinaface_distributor_resnet_faiss_converger_results( + std::shared_ptr objectMetadata, + bm_image& frame) { + if (objectMetadata->mSubObjectMetadatas.size() > 0) { + for (auto subObj : objectMetadata->mSubObjectMetadatas) { + int subId = subObj->mSubId; + auto faceObj = objectMetadata->mFaceObjectMetadatas[subId]; // 第一张脸 + auto resnetObj = + subObj->mRecognizedObjectMetadatas[0]; // 第一张脸对应的resnet + int class_id = subObj->mRecognizedObjectMetadatas[0]->mTopKLabels[0]; + auto label = subObj->mRecognizedObjectMetadatas[0]->mLabelName; + + bmcv_rect_t rect; + rect.start_x = std::max(faceObj->left, 0); + rect.start_y = std::max(faceObj->top, 0); + rect.crop_w = std::max(faceObj->right - faceObj->left + 1, 0); + rect.crop_h = std::max(faceObj->bottom - faceObj->top + 1, 0); + + _draw_text_bmcv(objectMetadata->mFrame->mHandle, rect.start_x, + rect.start_y, frame, label); + + std::cout << "label:" << label << std::endl; + } + } +} + +const std::vector> LIMB_COLORS = { + {51, 153, 255}, {51, 153, 255}, {51, 153, 255}, {51, 153, 255}, + {255, 51, 255}, {255, 51, 255}, {255, 51, 255}, {255, 128, 0}, + {255, 128, 0}, {255, 128, 0}, {255, 128, 0}, {255, 128, 0}, + {0, 255, 0}, {0, 255, 0}, {0, 255, 0}, {0, 255, 0}, + {0, 255, 0}, {0, 255, 0}, {0, 255, 0}}; +const std::vector> SKELETON = { + {16, 14}, {14, 12}, {17, 15}, {15, 13}, {12, 13}, {6, 12}, {7, 13}, + {6, 7}, {6, 8}, {7, 9}, {8, 10}, {9, 11}, {2, 3}, {1, 2}, + {1, 3}, {2, 4}, {3, 5}, {4, 6}, {5, 7}}; +const std::vector> KPS_COLORS = { + {0, 255, 0}, {0, 255, 0}, {0, 255, 0}, {0, 255, 0}, + {0, 255, 0}, {255, 128, 0}, {255, 128, 0}, {255, 128, 0}, + {255, 128, 0}, {255, 128, 0}, {255, 128, 0}, {51, 153, 255}, + {51, 153, 255}, {51, 153, 255}, {51, 153, 255}, {51, 153, 255}, + {51, 153, 255}}; + +void draw_yolov8_det_pose( + std::shared_ptr objectMetadata, + cv::Mat& img) { + const int num_point = 17; + int idx = 0; + for (auto& obj : objectMetadata->mDetectedObjectMetadatas) { + cv::rectangle(img, + cv::Rect{obj->mBox.mX, obj->mBox.mY, obj->mBox.mWidth, + obj->mBox.mHeight}, + {0, 0, 255}, 2); + + char text[256]; + sprintf(text, "person %.1f%%", obj->mScores[0] * 100); + + int baseLine = 0; + cv::Size label_size = + cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.4, 1, &baseLine); + + int x = (int)obj->mBox.mX; + int y = (int)obj->mBox.mY + 1; + + if (y > img.rows) y = img.rows; + + cv::rectangle( + img, cv::Rect(x, y, label_size.width, label_size.height + baseLine), + {0, 0, 255}, -1); + + cv::putText(img, text, cv::Point(x, y + label_size.height), + cv::FONT_HERSHEY_SIMPLEX, 0.4, {255, 255, 255}, 1); + + auto& kps = objectMetadata->mPosedObjectMetadatas[idx]->keypoints; + for (int k = 0; k < num_point + 2; k++) { + if (k < num_point) { + int kps_x = std::round(kps[k * 3]); + int kps_y = std::round(kps[k * 3 + 1]); + float kps_s = kps[k * 3 + 2]; + if (kps_s > 0.5f) { + cv::Scalar kps_color = + cv::Scalar(KPS_COLORS[k][0], KPS_COLORS[k][1], KPS_COLORS[k][2]); + cv::circle(img, {kps_x, kps_y}, 5, kps_color, -1); + } + } + auto& ske = SKELETON[k]; + int pos1_x = std::round(kps[(ske[0] - 1) * 3]); + int pos1_y = std::round(kps[(ske[0] - 1) * 3 + 1]); + + int pos2_x = std::round(kps[(ske[1] - 1) * 3]); + int pos2_y = std::round(kps[(ske[1] - 1) * 3 + 1]); + + float pos1_s = kps[(ske[0] - 1) * 3 + 2]; + float pos2_s = kps[(ske[1] - 1) * 3 + 2]; + + if (pos1_s > 0.5f && pos2_s > 0.5f) { + cv::Scalar limb_color = + cv::Scalar(LIMB_COLORS[k][0], LIMB_COLORS[k][1], LIMB_COLORS[k][2]); + cv::line(img, {pos1_x, pos1_y}, {pos2_x, pos2_y}, limb_color, 2); + } + } + ++idx; + } +} + +void draw_yolov5_results( + std::shared_ptr objectMetadata, + bm_image& frame, std::vector& class_names) { + int colors_num = colors.size(); + + for (auto detObj : objectMetadata->mDetectedObjectMetadatas) { + int class_id = detObj->mClassify; + std::string label = + class_names[class_id] + ":" + cv::format("%.2f", detObj->mScores[0]); + _draw_rectangle_and_text_bmcv(objectMetadata->mFrame->mHandle, label, + detObj->mBox.mX, detObj->mBox.mY, + detObj->mBox.mWidth, detObj->mBox.mHeight, + frame, colors[class_id % colors_num], true); + } +} + +void draw_yolov5_bytetrack_distributor_resnet_converger_results( + std::shared_ptr objectMetadata, + bm_image& frame, std::vector& car_attr, + std::vector& person_attr) { + if (objectMetadata->mSubObjectMetadatas.size() > 0) { + int colors_num = colors.size(); + + for (auto subObj : objectMetadata->mSubObjectMetadatas) { + int subId = subObj->mSubId; + auto detObj = objectMetadata->mDetectedObjectMetadatas[subId]; + auto trackObj = objectMetadata->mTrackedObjectMetadatas[subId]; + bmcv_rect_t rect; + rect.start_x = detObj->mBox.mX; + rect.start_y = detObj->mBox.mY; + rect.crop_w = detObj->mBox.mWidth; + rect.crop_h = detObj->mBox.mHeight; + int class_id = subObj->mRecognizedObjectMetadatas[0]->mTopKLabels[0]; + std::string label = std::to_string(trackObj->mTrackId) + "-"; + if (detObj->mClassify == 2) + label += car_attr[class_id]; + else + label += person_attr[class_id]; + _draw_rectangle_and_text_bmcv( + objectMetadata->mFrame->mHandle, label, rect.start_x, rect.start_y, + rect.crop_w, rect.crop_h, frame, + colors[trackObj->mTrackId % colors_num], true); + } + } +} + +void draw_yolox_results( + std::shared_ptr objectMetadata, + bm_image& frame, std::vector& class_names) { + int colors_num = colors.size(); + + for (auto detObj : objectMetadata->mDetectedObjectMetadatas) { + std::string label = class_names[detObj->mClassify] + ":" + + cv::format("%.2f", detObj->mScores[0]); + _draw_rectangle_and_text_bmcv( + objectMetadata->mFrame->mHandle, label, detObj->mBox.mX, + detObj->mBox.mY, detObj->mBox.mWidth, detObj->mBox.mHeight, frame, + colors[detObj->mClassify % colors_num], true); + } +} + +void draw_default( + std::shared_ptr objectMetadata, + bm_image& frame) { + return; +} + +void draw_yolov5_fastpose_posec3d_results( + std::shared_ptr objectMetadata, + bm_image& frame, std::string loss_type) { + if (objectMetadata->mRecognizedObjectMetadatas.size() != 0) + _draw_text_bmcv( + objectMetadata->mFrame->mHandle, 50, 10, frame, + objectMetadata->mRecognizedObjectMetadatas[0]->mLabelName + ":" + + cv::format( + "%.2f", + objectMetadata->mRecognizedObjectMetadatas[0]->mScores[0])); + for (auto subObj : objectMetadata->mPosedObjectMetadatas) { + _render_fastpose_keypoints(objectMetadata->mFrame->mHandle, frame, + subObj->keypoints, subObj->scores, loss_type, + 1.0); + } + for (auto subObj : objectMetadata->mDetectedObjectMetadatas) { + std::string label = "person:" + cv::format("%.2f", subObj->mScores[0]); + _draw_rectangle_and_text_bmcv(objectMetadata->mFrame->mHandle, label, + subObj->mBox.mX, subObj->mBox.mY, + subObj->mBox.mWidth, subObj->mBox.mHeight, + frame, colors[0], true); + } +} + +void draw_ppocr_results( + std::shared_ptr objectMetadata, + cv::Mat& img) { + uni_text::UniText uniText("../ppocr/data/wqy-microhei.ttc", 30); + + // draw words + if (objectMetadata->mSubObjectMetadatas.size() > 0) { + for (auto subObj : objectMetadata->mSubObjectMetadatas) { + IVS_WARN("get recognized words from ppocr"); + int subId = subObj->mSubId; + auto reconizedObj = subObj->mRecognizedObjectMetadatas[0]; + auto detObj = objectMetadata->mDetectedObjectMetadatas[subId]; + + std::string oriLabel = reconizedObj->mLabelName; + + uniText.PutText(img, oriLabel, + cv::Point(detObj->mKeyPoints[0]->mPoint.mX, + detObj->mKeyPoints[0]->mPoint.mY), + cv::Scalar(0, 255, 0), false); + } + } +} + +void draw_text_results_online( + std::shared_ptr objectMetadata, + bm_image& frame, std::string texts_path, std::vector texts, + std::vector top, std::vector left, int r, int g, int b) { + uni_text::UniText uniText(texts_path.c_str(), 30); + for (int i = 0; i < texts.size(); i++) { + bm_image overlay_image; + uniText.genBitMap(objectMetadata->mFrame->mHandle, texts[i], overlay_image, + r, g, b); + bmcv_rect_t overlay_info = {top[i], left[i], overlay_image.height, + overlay_image.width}; + + bmcv_image_overlay(objectMetadata->mFrame->mHandle, frame, 1, &overlay_info, + &overlay_image); + + bm_image_destroy(overlay_image); } } -void draw_bmcv_areas(std::shared_ptr objectMetadata,bm_image &frame_to_draw){ - for(int i=0;iareas.size();i++){ - if(objectMetadata->areas[i].size()==2){ - bmcv_point_t start={objectMetadata->areas[i][0].mY,objectMetadata->areas[i][0].mX}; - bmcv_point_t end={objectMetadata->areas[i][1].mY,objectMetadata->areas[i][1].mX}; - bmcv_color_t color={255,0,0}; - bmcv_image_draw_lines(objectMetadata->mFrame->mHandle,frame_to_draw,&start,&end,1,color,3); +void draw_text_results( + std::shared_ptr objectMetadata, + bm_image& frame, std::vector& overlay_images, + std::vector& top, std::vector& left, bool draw_interval) { + if (!objectMetadata->mFilter || draw_interval) { + for (int i = 0; i < overlay_images.size(); i++) { + bm_image overlay_image = overlay_images[i]; + bmcv_rect_t overlay_info = {top[i], left[i], overlay_image.height, + overlay_image.width}; + bmcv_image_overlay(objectMetadata->mFrame->mHandle, frame, 1, + &overlay_info, &overlay_image); } - } - } } // namespace osd } // namespace element diff --git a/element/multimedia/osd/include/osd.h b/element/multimedia/osd/include/osd.h index 8e37c044..17b234b0 100644 --- a/element/multimedia/osd/include/osd.h +++ b/element/multimedia/osd/include/osd.h @@ -12,6 +12,7 @@ #include #include + #include "common/object_metadata.h" #include "common/profiler.h" #include "element.h" @@ -22,7 +23,7 @@ namespace osd { class Osd : public ::sophon_stream::framework::Element { public: - enum class OsdType { DET, TRACK, REC, POSE, AREA, UNKNOWN }; + enum class OsdType { DET, TRACK, REC, POSE, AREA, ALGORITHM, TEXT, UNKNOWN }; enum class DrawUtils { OPENCV, BMCV, UNKNOWN }; Osd(); ~Osd() override; @@ -33,17 +34,43 @@ class Osd : public ::sophon_stream::framework::Element { static constexpr const char* CONFIG_INTERNAL_OSD_TYPE_FIELD = "osd_type"; static constexpr const char* CONFIG_INTERNAL_CLASS_NAMES_FIELD = "class_names_file"; + static constexpr const char* CONFIG_INTERNAL_RECOGNIZE_NAMES_FIELD = + "recognice_names_file"; static constexpr const char* CONFIG_INTERNAL_DRAW_UTILS_FIELD = "draw_utils"; static constexpr const char* CONFIG_INTERNAL_DRAW_INTERVAL_FIELD = "draw_interval"; static constexpr const char* CONFIG_INTERNAL_PUT_TEXT_FIELD = "put_text"; + static constexpr const char* CONFIG_INTERNAL_DRAW_FUNC_NAME_FIELD = + "draw_func_name"; + static constexpr const char* CONFIG_INTERNAL_HEATMAP_LOSS_FIELD = + "heatmap_loss"; + static constexpr const char* CONFIG_INTERNAL_LEFT_FIELD = "lefts"; + static constexpr const char* CONFIG_INTERNAL_TOP_FIELD = "tops"; + static constexpr const char* CONFIG_INTERNAL_TEXT_FIELD = "texts"; + static constexpr const char* CONFIG_INTERNAL_FONT_LIBRARY_FIELD = "font_library"; + static constexpr const char* CONFIG_INTERNAL_R_FIELD = "r"; + static constexpr const char* CONFIG_INTERNAL_G_FIELD = "g"; + static constexpr const char* CONFIG_INTERNAL_B_FIELD = "b"; private: std::vector mClassNames; + std::vector mRecognizeNames; + std::vector tops, lefts; + std::vector texts; + std::string font_library; OsdType mOsdType; DrawUtils mDrawUtils; bool mDrawInterval; bool mPutText; + std::vector overlay_image_; + int r, g, b; + std::string heatmap_loss; + std::function, + bm_image&)> + draw_func; + std::function, + cv::Mat&)> + draw_func_opencv; ::sophon_stream::common::FpsProfiler mFpsProfiler; void draw(std::shared_ptr objectMetadata); }; diff --git a/samples/include/unchecked.h b/element/multimedia/osd/include/unchecked.h similarity index 100% rename from samples/include/unchecked.h rename to element/multimedia/osd/include/unchecked.h diff --git a/samples/include/utf8.h b/element/multimedia/osd/include/utf8.h similarity index 100% rename from samples/include/utf8.h rename to element/multimedia/osd/include/utf8.h diff --git a/element/multimedia/osd/src/cvUniText.cc b/element/multimedia/osd/src/cvUniText.cc new file mode 100644 index 00000000..71931b4e --- /dev/null +++ b/element/multimedia/osd/src/cvUniText.cc @@ -0,0 +1,342 @@ +#include "cvUniText.h" + +#include +#include + +#include + +#include "utf8.h" + +using namespace uni_text; + +namespace uni_text { +class Impl { + public: + Impl(const std::string& font_face, int font_size); + + ~Impl(); + + void SetParam(int font_size, float interval_ratio = 0.1, + float whitespace_ratio = 0.3, float alpha = 1); + + cv::Rect PutText(cv::Mat& img, const std::string& text, const cv::Point& org, + const cv::Scalar& color, bool calc_size); + bool genBitMap(bm_handle_t mHandle, const std::string& utf8_text, + bm_image& overlay_image, int r, int g, int b); + + private: + cv::Rect _cvPutUniTextUCS2(cv::Mat& img, const std::u16string& text, + const cv::Point& org, const cv::Scalar& color, + bool calc_size); + + double _cvPutUniChar(cv::Mat& img, char16_t wc, const cv::Point& pos, + const cv::Scalar& color, bool calc_size); + + FT_Library m_library; + FT_Face m_face; + int m_fontType; + cv::Scalar m_fontSize; + float m_fontDiaphaneity; +}; +} // namespace uni_text + +UniText::UniText(const std::string& font_face, int font_size) { + pimpl = std::unique_ptr(new Impl(font_face, font_size)); +} + +UniText::~UniText() = default; + +void UniText::SetParam(int font_size, float interval_ratio, + float whitespace_ratio, float alpha) { + pimpl->SetParam(font_size, interval_ratio, whitespace_ratio, alpha); +} + +cv::Rect UniText::PutText(cv::Mat& img, const std::string& text, + const cv::Point& org, const cv::Scalar& color, + bool calc_size) { + return pimpl->PutText(img, text, org, color, calc_size); +} +bool UniText::genBitMap(bm_handle_t mHandle, const std::string& utf8_text, + bm_image& overlay_image, int r, int g, int b) { + return pimpl->genBitMap(mHandle, utf8_text, overlay_image, r, g, b); +} + +Impl::Impl(const std::string& font_face, int font_size) { + if (FT_Init_FreeType(&m_library) != 0) { + fprintf(stderr, "Freetype init failed!\n"); + abort(); + } + + if (FT_New_Face(m_library, font_face.c_str(), 0, &m_face) != 0) { + fprintf(stderr, "Freetype font load failed!\n"); + abort(); + } + + m_fontType = 0; + m_fontSize[0] = font_size; // FontSize + m_fontSize[1] = 0.5; // whitechar ratio, such like ' ' + m_fontSize[2] = 0.1; // inverval ratio, for each char. + m_fontDiaphaneity = 1; // alpha + + FT_Set_Pixel_Sizes(m_face, (int)m_fontSize[0], 0); +} + +Impl::~Impl() { + FT_Done_Face(m_face); + FT_Done_FreeType(m_library); +} + +void Impl::SetParam(int font_size, float interval_ratio, float whitespace_ratio, + float alpha) { + m_fontSize[0] = font_size; // FontSize + m_fontSize[1] = whitespace_ratio; // whitechar ratio, such like ' ' + m_fontSize[2] = interval_ratio; // inverval ratio, for each char. + m_fontDiaphaneity = alpha; // alpha + FT_Set_Pixel_Sizes(m_face, (int)m_fontSize[0], 0); +} + +double Impl::_cvPutUniChar(cv::Mat& img, char16_t wc, const cv::Point& pos, + const cv::Scalar& color, bool calc_size) { + // generate font bitmap from unicode + FT_GlyphSlot ft_slot; + // int ft_ascender; + int ft_bmp_height; + int ft_bmp_width; // 0 when ' ' + int ft_bmp_step; + int ft_bmp_left; + int ft_bmp_top; + unsigned char* ft_bmp_buffer; + + double whitespace_width; + double interval_width; + double horizontal_offset; + + // + // img coordinate + // 0------+ x + // | + // + + // y + // + // freetype bmp coordinate + // y + // + + // | + // 0------+ x + // + // freetype algin + // 'a' + // -+- -+- + // | | + // |height |top + // | | + // -baseline-: -+- -+- + // + // 'g' + // -+- -+- + // | | + // | |top + // | | + // -baseline-: |height -+- + // | + // -+- + // + + // get font_id from database of char + // ft_ascender = m_face->size->metrics.ascender / 64; + + FT_UInt glyph_index = FT_Get_Char_Index(m_face, wc); + // load bitmap font to slot + FT_Load_Glyph(m_face, glyph_index, FT_LOAD_DEFAULT); + // render to 8bits + FT_Render_Glyph(m_face->glyph, FT_RENDER_MODE_NORMAL); + ft_slot = m_face->glyph; + ft_bmp_width = ft_slot->bitmap.width; + ft_bmp_height = ft_slot->bitmap.rows; + ft_bmp_step = ft_slot->bitmap.pitch; + ft_bmp_buffer = ft_slot->bitmap.buffer; + ft_bmp_left = ft_slot->bitmap_left; + ft_bmp_top = ft_slot->bitmap_top; + +#ifdef CVUNITEXT_DEBUG + if (wc < 256) { + printf(" %c: ", ((char*)&wc)[0]); + } else { + printf(" 0x%02x%02x: ", ((char*)&wc)[1] & 0xFF, ((char*)&wc)[0] & 0xFF); + } + printf("width %4d, height %4d, ", ft_bmp_width, ft_bmp_height); + printf("left %4d, ", ft_bmp_left); + printf("top %4d, ", ft_bmp_top); + printf("\n"); +#endif + + // calculate char width + whitespace_width = m_fontSize[0] * m_fontSize[1]; + interval_width = m_fontSize[0] * m_fontSize[2]; + if (ft_bmp_width != 0) { + horizontal_offset = ft_bmp_width + interval_width; + } else { + horizontal_offset = whitespace_width; + } + + int loc_x = pos.x + ft_bmp_left; + int loc_y = pos.y + ft_bmp_height - ft_bmp_top; + + if (calc_size) { + return horizontal_offset; + } + + // draw font bitmap to opencv image + //(bmp_j,bmp_i) is freetype bitmap location + for (int bmp_i = 0; bmp_i < ft_bmp_height; ++bmp_i) { + for (int bmp_j = 0; bmp_j < ft_bmp_width; ++bmp_j) { + int bmp_valoff = bmp_i * ft_bmp_step + bmp_j; + unsigned int bmp_val = ft_bmp_buffer[bmp_valoff]; + float bmp_valf = (float)bmp_val / 255 * m_fontDiaphaneity; + if (bmp_val == 0) { + continue; + } + //(img_x,img_y) is opencv bitmap location + int img_y = loc_y - (ft_bmp_height - 1 - bmp_i); + int img_x = loc_x + bmp_j; + + // update pixel when location is valid + // alpha = font_bitmap_val / 255; + // pixel = alpha * color + (1 - alpha) * pixel; + if ((0 <= img_y) && (img_y < img.rows) && (0 <= img_x) && + (img_x < img.cols)) { + unsigned char* data = img.ptr(img_y); + for (int img_channel = 0; img_channel < img.channels(); img_channel++) { + data[img_x * img.channels() + img_channel] = + (1 - bmp_valf) * data[img_x * img.channels() + img_channel] + + bmp_valf * color[img_channel]; + } + } + } + } + + return horizontal_offset; +} + +cv::Rect Impl::_cvPutUniTextUCS2(cv::Mat& img, const std::u16string& text, + const cv::Point& org, const cv::Scalar& color, + bool calc_size) { + cv::Point pt0 = org; + cv::Point pt1 = org; + double offset; + cv::Rect rect; + int ascender = m_face->size->metrics.ascender / 64; + int descender = m_face->size->metrics.descender / 64; + + for (unsigned int i = 0; i < text.size(); i++) { + offset = _cvPutUniChar(img, text[i], pt1, color, calc_size); + pt1.x += (int)offset; + } + rect.width = pt1.x - pt0.x; + rect.height = ascender - descender; + rect.x = pt0.x; + rect.y = pt0.y - ascender; + return rect; +} + +cv::Rect Impl::PutText(cv::Mat& img, const std::string& text, + const cv::Point& org, const cv::Scalar& color, + bool calc_size) { + // std::wstring_convert, char16_t> + // convert; std::u16string dest = convert.from_bytes(text); + std::u16string dest; + utf8::utf8to32(text.begin(), text.end(), std::back_inserter(dest)); + cv::Rect sz = _cvPutUniTextUCS2(img, dest, org, color, calc_size); + return sz; +} + +bool Impl::genBitMap(bm_handle_t mHandle, const std::string& utf8_text, + bm_image& overlay_image, int r, int g, int b) { + std::u16string dest; + utf8::utf8to32(utf8_text.begin(), utf8_text.end(), std::back_inserter(dest)); + + FT_Pos total_width = 0; + FT_Pos max_height = 0; + FT_Pos base_y_offset = 0; + + // 第一次遍历是为了计算总宽度和最大高度 + for (unsigned int i = 0; i < dest.size(); i++) { + char16_t ch = dest[i]; + // 注意：这里使用FT_Get_Char_Index和FT_Load_Glyph组合来支持多字节字符 + FT_UInt glyph_index = FT_Get_Char_Index(m_face, ch); + if (FT_Load_Glyph(m_face, glyph_index, FT_LOAD_DEFAULT)) { + IVS_ERROR("Could not load glyph"); + continue; + } + total_width += (m_face->glyph->advance.x >> 6); + max_height = + std::max(max_height, static_cast(m_face->glyph->bitmap.rows)); + base_y_offset = + std::max(base_y_offset, static_cast(m_face->glyph->bitmap_top)); + } + + std::vector finalBuffer(max_height * total_width * 4, + 0); // 初始化为0 + FT_Pos x_offset = 0; + // total_width=FFALIGN(total_width,16); + for (unsigned int i = 0; i < dest.size(); i++) { + char16_t ch = dest[i]; + FT_UInt glyph_index = FT_Get_Char_Index(m_face, ch); + if (FT_Load_Glyph(m_face, glyph_index, FT_LOAD_RENDER)) { + IVS_ERROR("Could not load glyph"); + continue; + } + FT_Bitmap* bitmap = &m_face->glyph->bitmap; + std::vector buffer; + buffer.resize(bitmap->rows * bitmap->width * 4); + for (unsigned int i = 0; i < bitmap->rows; ++i) { + for (unsigned int j = 0; j < bitmap->width; ++j) { + // 计算buffer中的索引位置 + size_t index = (i * bitmap->width + j) * 4; + unsigned char alpha = bitmap->buffer[i * bitmap->pitch + j]; + // 设置ARGB8888像素值 + buffer[index] = b; // Blue + buffer[index + 1] = g; // Green + buffer[index + 2] = r; // Red + buffer[index + 3] = alpha; // Alpha + } + } + + FT_Pos y_offset_char = base_y_offset - m_face->glyph->bitmap_top; + for (unsigned int i = 0; i < m_face->glyph->bitmap.rows; ++i) { + for (unsigned int j = 0; j < m_face->glyph->bitmap.width; ++j) { + size_t index = (i * m_face->glyph->bitmap.width + j) * 4; + size_t final_index = + ((y_offset_char + i) * total_width + x_offset + j) * 4; + // 确保我们不会超出最终缓冲区的界限 + if (final_index < finalBuffer.size()) { + std::copy(buffer.begin() + index, buffer.begin() + index + 4, + finalBuffer.begin() + final_index); + } + } + } + + // 更新x_offset，确保下一个字符在正确的位置开始 + x_offset += (m_face->glyph->advance.x >> 6); + } + int overlay_height = max_height; + int overlay_width = total_width; + if (!overlay_height || !overlay_width) return false; + + bm_image overlay_image2; + bm_image_create(mHandle, overlay_height, overlay_width, FORMAT_ARGB_PACKED, + DATA_TYPE_EXT_1N_BYTE, &overlay_image2, NULL); + int ret = bm_image_alloc_dev_mem(overlay_image2, BMCV_HEAP_ANY); + STREAM_CHECK(ret == 0, "Alloc Device Memory Failed! Program Terminated.") + + void* image_data = static_cast(finalBuffer.data()); + bm_image_copy_host_to_device(overlay_image2, &image_data); + + int stride[3]; + stride[0] = FFALIGN(overlay_width * 4, 16); + bm_image_create(mHandle, overlay_height, overlay_width, FORMAT_ARGB_PACKED, + DATA_TYPE_EXT_1N_BYTE, &overlay_image, stride); + bmcv_width_align(mHandle, overlay_image2, overlay_image); + bm_image_destroy(overlay_image2); + return true; +} \ No newline at end of file diff --git a/element/multimedia/osd/src/osd.cc b/element/multimedia/osd/src/osd.cc index 3edb6840..47e9dcbf 100644 --- a/element/multimedia/osd/src/osd.cc +++ b/element/multimedia/osd/src/osd.cc @@ -15,7 +15,6 @@ #include #include - #include "common/common_defs.h" #include "common/logger.h" #include "draw_utils.h" @@ -26,7 +25,10 @@ namespace element { namespace osd { Osd::Osd() {} -Osd::~Osd() {} +Osd::~Osd() { + for (int i = 0; i < overlay_image_.size(); i++) + bm_image_destroy(overlay_image_[i]); +} common::ErrorCode Osd::initInternal(const std::string& json) { common::ErrorCode errorCode = common::ErrorCode::SUCCESS; @@ -45,7 +47,57 @@ common::ErrorCode Osd::initInternal(const std::string& json) { if (osd_type == "TRACK") mOsdType = OsdType::TRACK; if (osd_type == "POSE") mOsdType = OsdType::POSE; if (osd_type == "AREA") mOsdType = OsdType::AREA; - + if (osd_type == "ALGORITHM") mOsdType = OsdType::ALGORITHM; + if (osd_type == "TEXT") mOsdType = OsdType::TEXT; + if (mOsdType == OsdType::TEXT) { + auto leftIt = configure.find(CONFIG_INTERNAL_LEFT_FIELD); + + STREAM_CHECK((leftIt != configure.end() && leftIt->is_array()), + "left must be array, please check your osd element " + "configuration file"); + for (auto& left_obj : *leftIt) { + int left = left_obj.get(); + lefts.push_back(left); + } + auto topIt = configure.find(CONFIG_INTERNAL_TOP_FIELD); + + STREAM_CHECK((topIt != configure.end() && topIt->is_array()), + "top must be array, please check your osd element " + "configuration file"); + for (auto& top_obj : *leftIt) { + int top = top_obj.get(); + tops.push_back(top); + } + auto textIt = configure.find(CONFIG_INTERNAL_TEXT_FIELD); + + STREAM_CHECK((textIt != configure.end() && textIt->is_array()), + "text must be array, please check your osd element " + "configuration file"); + for (auto& text_obj : *textIt) { + std::string text = text_obj.get(); + texts.push_back(text); + } + font_library = configure.find(CONFIG_INTERNAL_FONT_LIBRARY_FIELD) + ->get(); + r = configure.find(CONFIG_INTERNAL_R_FIELD)->get(); + g = configure.find(CONFIG_INTERNAL_G_FIELD)->get(); + b = configure.find(CONFIG_INTERNAL_B_FIELD)->get(); + + uni_text::UniText uniText(font_library.c_str(), 30); + bm_handle_t h; + bm_dev_request(&h, 0); + for (int i = 0; i < texts.size(); i++) { + bm_image overlay_image; + uniText.genBitMap(h, texts[i], overlay_image, r, g, b); + overlay_image_.push_back(overlay_image); + } + if (bmcv_image_overlay == nullptr) { + IVS_ERROR( + "bmcv_image_overlay not support,please check your config file or " + "update SDK version"); + abort(); + } + } if (mOsdType == OsdType::DET) { std::string class_names_file = configure.find(CONFIG_INTERNAL_CLASS_NAMES_FIELD)->get(); @@ -59,7 +111,36 @@ common::ErrorCode Osd::initInternal(const std::string& json) { } istream.close(); } - + if (mOsdType == OsdType::ALGORITHM && + configure.end() != configure.find(CONFIG_INTERNAL_CLASS_NAMES_FIELD)) { + std::string class_names_file = + configure.find(CONFIG_INTERNAL_CLASS_NAMES_FIELD)->get(); + std::ifstream istream; + istream.open(class_names_file); + assert(istream.is_open()); + std::string line; + while (std::getline(istream, line)) { + line = line.substr(0, line.length()); + mClassNames.push_back(line); + } + istream.close(); + } + if (mOsdType == OsdType::ALGORITHM && + configure.end() != + configure.find(CONFIG_INTERNAL_RECOGNIZE_NAMES_FIELD)) { + std::string recognize_names_file = + configure.find(CONFIG_INTERNAL_RECOGNIZE_NAMES_FIELD) + ->get(); + std::ifstream istream; + istream.open(recognize_names_file); + assert(istream.is_open()); + std::string line; + while (std::getline(istream, line)) { + line = line.substr(0, line.length()); + mRecognizeNames.push_back(line); + } + istream.close(); + } mDrawUtils = DrawUtils::OPENCV; auto drawUtilsIt = configure.find(CONFIG_INTERNAL_DRAW_UTILS_FIELD); if (configure.end() != drawUtilsIt) { @@ -97,6 +178,63 @@ common::ErrorCode Osd::initInternal(const std::string& json) { "json:{1}, set default true", CONFIG_INTERNAL_PUT_TEXT_FIELD, json); } + auto heatmaplossIt = configure.find(CONFIG_INTERNAL_HEATMAP_LOSS_FIELD); + if (configure.end() != heatmaplossIt) { + auto heatmaploss = heatmaplossIt->get(); + heatmap_loss = heatmaploss; + } + if (mOsdType == OsdType::ALGORITHM) { + std::string draw_func_name = + configure.find(CONFIG_INTERNAL_DRAW_FUNC_NAME_FIELD) + ->get(); + if (draw_func_name == "draw_bytetrack_results") + draw_func = std::bind(draw_bytetrack_results, std::placeholders::_1, + std::placeholders::_2); + else if (draw_func_name == "draw_license_plate_recognition_results") + draw_func_opencv = + std::bind(draw_license_plate_recognition_results_opencv, + std::placeholders::_1, std::placeholders::_2); + else if (draw_func_name == "draw_retinaface_results") + draw_func = std::bind(draw_retinaface_results, std::placeholders::_1, + std::placeholders::_2); + else if (draw_func_name == + "draw_retinaface_distributor_resnet_faiss_converger_results") + draw_func = std::bind( + draw_retinaface_distributor_resnet_faiss_converger_results, + std::placeholders::_1, std::placeholders::_2); + else if (draw_func_name == "draw_yolov5_results") + draw_func = std::bind(draw_yolov5_results, std::placeholders::_1, + std::placeholders::_2, mClassNames); + else if (draw_func_name == + "draw_yolov5_bytetrack_distributor_resnet_converger_results") + draw_func = std::bind( + draw_yolov5_bytetrack_distributor_resnet_converger_results, + std::placeholders::_1, std::placeholders::_2, mClassNames, + mRecognizeNames); + else if (draw_func_name == "draw_yolox_results") + draw_func = std::bind(draw_yolox_results, std::placeholders::_1, + std::placeholders::_2, mClassNames); + else if (draw_func_name == "draw_yolov7_results") + draw_func = std::bind(draw_yolov5_results, std::placeholders::_1, + std::placeholders::_2, mClassNames); + + else if (draw_func_name == "draw_yolov5_fastpose_posec3d_results") + draw_func = std::bind(draw_yolov5_fastpose_posec3d_results, + std::placeholders::_1, std::placeholders::_2, + heatmap_loss); + else if (draw_func_name == "default") + draw_func = std::function, bm_image&)>( + draw_default); + else if (draw_func_name == "draw_ppocr_results") + draw_func_opencv = std::bind(draw_ppocr_results, std::placeholders::_1, + std::placeholders::_2); + else if (draw_func_name == "draw_yolov8_det_pose") + draw_func_opencv = std::bind( + draw_yolov8_det_pose, std::placeholders::_1, std::placeholders::_2); + else + IVS_ERROR("No such function! Please check your 'draw_func_name'."); + } } while (false); return errorCode; @@ -152,10 +290,12 @@ common::ErrorCode Osd::doWork(int dataPipeId) { } void Osd::draw(std::shared_ptr objectMetadata) { std::shared_ptr imageStorage; - imageStorage.reset(new bm_image, - [&](bm_image* img) { bm_image_destroy(*img); delete img; img = nullptr;}); + imageStorage.reset(new bm_image, [&](bm_image* img) { + bm_image_destroy(*img); + delete img; + img = nullptr; + }); bm_image image = *(objectMetadata->mFrame->mSpData); - if (mDrawUtils == DrawUtils::OPENCV) { cv::Mat frame_to_draw; cv::bmcv::toMAT(&image, frame_to_draw); @@ -176,10 +316,11 @@ void Osd::draw(std::shared_ptr objectMetadata) { break; case OsdType::AREA: - draw_opencv_areas(objectMetadata,frame_to_draw); - + draw_opencv_areas(objectMetadata, frame_to_draw); + break; + case OsdType::ALGORITHM: + draw_func_opencv(objectMetadata, frame_to_draw); break; - default: IVS_WARN("osd_type not support"); } @@ -219,10 +360,16 @@ void Osd::draw(std::shared_ptr objectMetadata) { break; case OsdType::AREA: - draw_bmcv_areas(objectMetadata,*imageStorage); - + draw_bmcv_areas(objectMetadata, *imageStorage); break; + case OsdType::TEXT: + draw_text_results(objectMetadata, *imageStorage, overlay_image_, tops, + lefts, mDrawInterval); + break; + case OsdType::ALGORITHM: + draw_func(objectMetadata, *imageStorage); + break; default: IVS_WARN("osd_type not support"); } diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 8f1d6038..1d29d86b 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -42,6 +42,7 @@ if (${TARGET_ARCH} STREQUAL "pcie") include_directories(../framework) include_directories(../framework/include) + include_directories(../element/multimedia/osd/include) include_directories(./include) set(demo_src @@ -49,10 +50,7 @@ if (${TARGET_ARCH} STREQUAL "pcie") ) get_filename_component(demo_name ${demo_src} NAME_WE) add_executable(${demo_name} ${demo_src}) - add_library(cvunitext SHARED src/cvUniText.cpp) target_link_libraries(${demo_name} cvunitext) - add_dependencies(cvunitext freetype) - target_link_libraries(cvunitext freetype ${OPENCV_LIBS}) add_dependencies(${demo_name} ivslogger framework cvunitext) @@ -96,6 +94,7 @@ elseif(${TARGET_ARCH} STREQUAL "soc") include_directories(../element/multimedia/decode/include) include_directories(../framework) include_directories(../framework/include) + include_directories(../element/multimedia/osd/include) if (DEFINED OPENSSL_PATH) include_directories(${OPENSSL_PATH}/include) @@ -109,10 +108,8 @@ elseif(${TARGET_ARCH} STREQUAL "soc") ) get_filename_component(demo_name ${demo_src} NAME_WE) add_executable(${demo_name} ${demo_src}) - add_library(cvunitext SHARED src/cvUniText.cpp) target_link_libraries(${demo_name} cvunitext) - target_link_libraries(cvunitext freetype ${OPENCV_LIBS}) - add_dependencies(cvunitext freetype) + add_dependencies(${demo_name} ivslogger framework cvunitext) add_dependencies(${demo_name} ivslogger framework) if (DEFINED OPENSSL_PATH) diff --git a/samples/include/cvUniText.h b/samples/include/cvUniText.h deleted file mode 100644 index 7f9b771e..00000000 --- a/samples/include/cvUniText.h +++ /dev/null @@ -1,43 +0,0 @@ -#ifndef CV_UNI_TEXT_HPP -#define CV_UNI_TEXT_HPP -#include -#include -#include - -namespace uni_text { - class Impl; - - class UniText { - public: - /// Initialization - /// \param font_face: the file path of your font - /// \param font_size - UniText(const std::string &font_face, int font_size); - ~UniText(); - - /// Detailed parameter of text rendering. Call this before drawing the text - /// \param font_size - /// \param interval_ratio: Ratio of character interval over character width - /// \param whitespace_ratio: Ratio of whitespace width over character width - /// \param alpha: Transparency. 1 means totally opaque. - void SetParam(int font_size, float interval_ratio = 0.1, float whitespace_ratio = 0.5, float alpha = 1); - - /// Draw text on an Opencv Mat - /// \param img - /// \param utf8_text: Text encoded in utf8. (if it is hardcoded, make sure your source file is saved - /// with utf8 encoding. - /// \param org: The left-bottom point of the text. Notice some letters like 'g' may go under this point - /// \param color - /// \param calc_size: true -> return rect. False -> return rect and draw text. - /// Useful e.g. when you want to draw a rectangle under the text - /// \return The precise bounding box of the text in the image - cv::Rect PutText(cv::Mat &img, const std::string &utf8_text, const cv::Point &org, - const cv::Scalar &color, bool calc_size = false); - - private: - /// Hide implementations - std::unique_ptr pimpl; - }; -} - -#endif diff --git a/samples/src/cvUniText.cpp b/samples/src/cvUniText.cpp deleted file mode 100644 index a7299053..00000000 --- a/samples/src/cvUniText.cpp +++ /dev/null @@ -1,235 +0,0 @@ -#include "cvUniText.h" -#include -#include "utf8.h" -#include -#include - -using namespace uni_text; - -namespace uni_text { - class Impl { - public: - Impl(const std::string &font_face, int font_size); - - ~Impl(); - - void SetParam(int font_size, float interval_ratio = 0.1, float whitespace_ratio = 0.3, float alpha = 1); - - cv::Rect PutText(cv::Mat &img, const std::string &text, const cv::Point &org, - const cv::Scalar &color, bool calc_size); - - private: - cv::Rect _cvPutUniTextUCS2(cv::Mat &img, const std::u16string &text, - const cv::Point &org, const cv::Scalar &color, bool calc_size); - - double _cvPutUniChar(cv::Mat &img, char16_t wc, - const cv::Point &pos, const cv::Scalar &color, bool calc_size); - - FT_Library m_library; - FT_Face m_face; - int m_fontType; - cv::Scalar m_fontSize; - float m_fontDiaphaneity; - }; -} - -UniText::UniText(const std::string &font_face, int font_size) { - pimpl = std::unique_ptr(new Impl(font_face, font_size)); -} - -UniText::~UniText() = default; - -void UniText::SetParam(int font_size, float interval_ratio, float whitespace_ratio, float alpha) { - pimpl->SetParam(font_size, interval_ratio, whitespace_ratio, alpha); -} - -cv::Rect UniText::PutText(cv::Mat& img, const std::string& text, const cv::Point& org, - const cv::Scalar& color, bool calc_size) { - return pimpl->PutText(img, text, org, color, calc_size); -} - -Impl::Impl(const std::string& font_face, int font_size) { - if (FT_Init_FreeType(&m_library) != 0) { - fprintf(stderr, "Freetype init failed!\n"); - abort(); - } - - if (FT_New_Face(m_library, font_face.c_str(), 0, &m_face) != 0) { - fprintf(stderr, "Freetype font load failed!\n"); - abort(); - } - - m_fontType = 0; - m_fontSize[0] = font_size; //FontSize - m_fontSize[1] = 0.5; //whitechar ratio, such like ' ' - m_fontSize[2] = 0.1; //inverval ratio, for each char. - m_fontDiaphaneity = 1;//alpha - - FT_Set_Pixel_Sizes(m_face, (int) m_fontSize[0], 0); -} - -Impl::~Impl() { - FT_Done_Face(m_face); - FT_Done_FreeType(m_library); -} - -void Impl::SetParam(int font_size, float interval_ratio, float whitespace_ratio, float alpha) { - m_fontSize[0] = font_size; //FontSize - m_fontSize[1] = whitespace_ratio; //whitechar ratio, such like ' ' - m_fontSize[2] = interval_ratio; //inverval ratio, for each char. - m_fontDiaphaneity = alpha;//alpha - FT_Set_Pixel_Sizes(m_face, (int) m_fontSize[0], 0); -} - -double Impl::_cvPutUniChar(cv::Mat& img, char16_t wc, - const cv::Point& pos, const cv::Scalar& color, bool calc_size) { - // generate font bitmap from unicode - FT_GlyphSlot ft_slot; -// int ft_ascender; - int ft_bmp_height; - int ft_bmp_width;//0 when ' ' - int ft_bmp_step; - int ft_bmp_left; - int ft_bmp_top; - unsigned char *ft_bmp_buffer; - - - double whitespace_width; - double interval_width; - double horizontal_offset; - - // - // img coordinate - // 0------+ x - // | - // + - // y - // - //freetype bmp coordinate - // y - // + - // | - // 0------+ x - // - //freetype algin - // 'a' - // -+- -+- - // | | - // |height |top - // | | - // -baseline-: -+- -+- - // - // 'g' - // -+- -+- - // | | - // | |top - // | | - // -baseline-: |height -+- - // | - // -+- - // - - //get font_id from database of char -// ft_ascender = m_face->size->metrics.ascender / 64; - - FT_UInt glyph_index = FT_Get_Char_Index(m_face, wc); - //load bitmap font to slot - FT_Load_Glyph(m_face, glyph_index, FT_LOAD_DEFAULT); - //render to 8bits - FT_Render_Glyph(m_face->glyph, FT_RENDER_MODE_NORMAL); - ft_slot = m_face->glyph; - ft_bmp_width = ft_slot->bitmap.width; - ft_bmp_height = ft_slot->bitmap.rows; - ft_bmp_step = ft_slot->bitmap.pitch; - ft_bmp_buffer = ft_slot->bitmap.buffer; - ft_bmp_left = ft_slot->bitmap_left; - ft_bmp_top = ft_slot->bitmap_top; - -#ifdef CVUNITEXT_DEBUG - if (wc < 256) { - printf(" %c: ", ((char*)&wc)[0]); - } else { - printf(" 0x%02x%02x: ", ((char*)&wc)[1] & 0xFF, ((char*)&wc)[0] & 0xFF); - } - printf("width %4d, height %4d, ", ft_bmp_width, ft_bmp_height); - printf("left %4d, ", ft_bmp_left); - printf("top %4d, ", ft_bmp_top); - printf("\n"); -#endif - - //calculate char width - whitespace_width = m_fontSize[0] * m_fontSize[1]; - interval_width = m_fontSize[0] * m_fontSize[2]; - if (ft_bmp_width != 0) { - horizontal_offset = ft_bmp_width + interval_width; - } else { - horizontal_offset = whitespace_width; - } - - int loc_x = pos.x + ft_bmp_left; - int loc_y = pos.y + ft_bmp_height - ft_bmp_top; - - if (calc_size) { - return horizontal_offset; - } - - //draw font bitmap to opencv image - //(bmp_j,bmp_i) is freetype bitmap location - for (int bmp_i = 0; bmp_i < ft_bmp_height; ++bmp_i) { - for (int bmp_j = 0; bmp_j < ft_bmp_width; ++bmp_j) { - int bmp_valoff = bmp_i * ft_bmp_step + bmp_j; - unsigned int bmp_val = ft_bmp_buffer[bmp_valoff]; - float bmp_valf = (float) bmp_val / 255 * m_fontDiaphaneity; - if (bmp_val == 0) { - continue; - } - //(img_x,img_y) is opencv bitmap location - int img_y = loc_y - (ft_bmp_height - 1 - bmp_i); - int img_x = loc_x + bmp_j; - - //update pixel when location is valid - // alpha = font_bitmap_val / 255; - // pixel = alpha * color + (1 - alpha) * pixel; - if ((0 <= img_y) && (img_y < img.rows) && (0 <= img_x) && (img_x < img.cols)) { - unsigned char *data = img.ptr(img_y); - for (int img_channel = 0; img_channel < img.channels(); img_channel++) { - data[img_x * img.channels() + img_channel] = - (1 - bmp_valf) * data[img_x * img.channels() + img_channel] + - bmp_valf * color[img_channel]; - } - } - } - } - - return horizontal_offset; -} - -cv::Rect Impl::_cvPutUniTextUCS2(cv::Mat& img, const std::u16string& text, - const cv::Point& org, const cv::Scalar& color, bool calc_size) { - cv::Point pt0 = org; - cv::Point pt1 = org; - double offset; - cv::Rect rect; - int ascender = m_face->size->metrics.ascender / 64; - int descender = m_face->size->metrics.descender / 64; - - for (unsigned int i = 0; i < text.size(); i++) { - offset = _cvPutUniChar(img, text[i], pt1, color, calc_size); - pt1.x += (int) offset; - } - rect.width = pt1.x - pt0.x; - rect.height = ascender - descender; - rect.x = pt0.x; - rect.y = pt0.y - ascender; - return rect; -} - -cv::Rect Impl::PutText(cv::Mat& img, const std::string& text, const cv::Point &org, - const cv::Scalar& color, bool calc_size) { -// std::wstring_convert, char16_t> convert; -// std::u16string dest = convert.from_bytes(text); - std::u16string dest; - utf8::utf8to32(text.begin(), text.end(), std::back_inserter(dest)); - cv::Rect sz = _cvPutUniTextUCS2(img, dest, org, color, calc_size); - return sz; -} From e00fb419072e0dc162b4cc81caa9c1caf6f01f64 Mon Sep 17 00:00:00 2001 From: "yizhou.xu" Date: Sat, 10 Aug 2024 14:48:14 +0800 Subject: [PATCH 7/8] feat: add or stop a channel by http request Change-Id: Iaedd19b253e32df6a1c20bb663519c645e2179d7 --- element/multimedia/decode/include/decode.h | 11 +- element/multimedia/decode/include/decoder.h | 4 +- element/multimedia/decode/src/decode.cc | 33 ++++- element/multimedia/decode/src/decoder.cc | 15 ++- element/multimedia/decode/src/ff_decode.cc | 13 +- framework/common/error_code.h | 64 ++++----- framework/common/http_defs.cc | 67 ++++++++++ framework/common/http_defs.h | 28 ++++ samples/README.md | 24 +++- samples/src/main.cc | 137 +++++++++++++++++++- 10 files changed, 335 insertions(+), 61 deletions(-) diff --git a/element/multimedia/decode/include/decode.h b/element/multimedia/decode/include/decode.h index 1f1109d0..59052b77 100644 --- a/element/multimedia/decode/include/decode.h +++ b/element/multimedia/decode/include/decode.h @@ -141,6 +141,8 @@ class Decode : public ::sophon_stream::framework::Element { common::ErrorCode doWork(int dataPipe) override; + bm_handle_t getHandle() const; + static constexpr const char* JSON_CHANNEL_ID = "channel_id"; static constexpr const char* JSON_SOURCE_TYPE = "source_type"; static constexpr const char* JSON_URL = "url"; @@ -159,8 +161,13 @@ class Decode : public ::sophon_stream::framework::Element { private: std::map> mThreadsPool; std::mutex mThreadsPoolMtx; - std::atomic mChannelCount; + // 用于channelIdInternal更新 + static std::atomic mChannelCount; + // channelId -> channelIdInternal 的映射 std::map mChannelIdInternal; + // 已经释放出来的channelIdInternal + static std::queue mChannelIdInternalReleased; + void onStart() override; void onStop() override; @@ -177,6 +184,8 @@ class Decode : public ::sophon_stream::framework::Element { std::shared_ptr& channelTask); ::sophon_stream::common::FpsProfiler mFpsProfiler; + + bm_handle_t handle_; }; } // namespace decode diff --git a/element/multimedia/decode/include/decoder.h b/element/multimedia/decode/include/decoder.h index de32fdce..ad240d68 100644 --- a/element/multimedia/decode/include/decoder.h +++ b/element/multimedia/decode/include/decoder.h @@ -31,8 +31,8 @@ class Decoder : public ::sophon_stream::common::NoCopyable { Decoder(); ~Decoder(); - common::ErrorCode init(int deviceId, int graphId, - const ChannelOperateRequest& request); + common::ErrorCode init(int graphId, const ChannelOperateRequest& request, + bm_handle_t handle); common::ErrorCode process( std::shared_ptr& objectMetadata); void uninit(); diff --git a/element/multimedia/decode/src/decode.cc b/element/multimedia/decode/src/decode.cc index afb0b2d1..bcad988d 100644 --- a/element/multimedia/decode/src/decode.cc +++ b/element/multimedia/decode/src/decode.cc @@ -15,12 +15,16 @@ namespace decode { Decode::Decode() {} +std::atomic Decode::mChannelCount = 0; +std::queue Decode::mChannelIdInternalReleased; + Decode::~Decode() { std::lock_guard lk(mThreadsPoolMtx); for (auto& channelInfo : mThreadsPool) { channelInfo.second->mThreadWrapper->stop(); } mThreadsPool.clear(); + bm_dev_free(handle_); } common::ErrorCode Decode::initInternal(const std::string& json) { @@ -32,8 +36,9 @@ common::ErrorCode Decode::initInternal(const std::string& json) { errorCode = common::ErrorCode::PARSE_CONFIGURE_FAIL; break; } - mChannelCount = 0; mFpsProfiler.config("fps_decode", 100); + int dev_id = getDeviceId(); + bm_dev_request(&handle_, dev_id); } while (false); return errorCode; @@ -317,8 +322,8 @@ common::ErrorCode Decode::startTask(std::shared_ptr& channelTask) { IVS_INFO("channel info decoder address: {0:p}", static_cast(channelInfo->mSpDecoder.get())); - common::ErrorCode ret = - channelInfo->mSpDecoder->init(getDeviceId(), getGraphId(), channelTask->request); + common::ErrorCode ret = channelInfo->mSpDecoder->init( + getGraphId(), channelTask->request, handle_); if (ret != common::ErrorCode::SUCCESS) { channelTask->response.errorCode = ret; std::string error = "Decoder init failed! channel id is " + @@ -355,11 +360,19 @@ common::ErrorCode Decode::startTask(std::shared_ptr& channelTask) { std::make_pair(channelTask->request.channelId, channelInfo)); int channel_id = channelTask->request.channelId; - if (mChannelIdInternal.find(channel_id) == mChannelIdInternal.end()) { + // 这里不需要判断channel_id是否在占用，因为本函数开头就在mThreadsPool里处理了 + // 更新channel_id。需要判断是否有释放出来的channelIdInternal + if (mChannelIdInternalReleased.empty()) { + // 没有释放的channelIdInternal，那么只能更新一个。 mChannelIdInternal[channel_id] = mChannelCount++; + } else { + // 有可以释放的channelIdInternal + int channelIdInternal = mChannelIdInternalReleased.front(); + mChannelIdInternalReleased.pop(); + mChannelIdInternal[channel_id] = channelIdInternal; } - IVS_INFO("add one channel task finished!"); + IVS_INFO("add one channel task finished, channel id = {0}", channel_id); return channelTask->response.errorCode; } @@ -374,11 +387,21 @@ common::ErrorCode Decode::stopTask(std::shared_ptr& channelTask) { IVS_ERROR("{0}", error); return common::ErrorCode::DECODE_CHANNEL_NOT_FOUND; } + + // 停止一路码流，需要记录释放出来的channelIdInternal，然后erase一对kv + auto itChannelId = mChannelIdInternal.find(channelTask->request.channelId); + int channelIdInternal = itChannelId->second; + mChannelIdInternalReleased.push(channelIdInternal); + mChannelIdInternal.erase(itChannelId); + common::ErrorCode errorCode = itTask->second->mThreadWrapper->stop(); itTask->second->mSpDecoder->uninit(); itTask->second->mThreadWrapper.reset(); mThreadsPool.erase(itTask); channelTask->response.errorCode = errorCode; + IVS_INFO("stop one channel task finished, channel id = {0}", + channelTask->request.channelId); + return errorCode; } diff --git a/element/multimedia/decode/src/decoder.cc b/element/multimedia/decode/src/decoder.cc index 96e58d59..1678b755 100644 --- a/element/multimedia/decode/src/decoder.cc +++ b/element/multimedia/decode/src/decoder.cc @@ -71,10 +71,13 @@ Decoder::Decoder() { numThreadsTotal.fetch_add(1); } -Decoder::~Decoder() {} +Decoder::~Decoder() { + // bm_dev_free(m_handle); +} -common::ErrorCode Decoder::init(int deviceId, int graphId, - const ChannelOperateRequest& request) { +common::ErrorCode Decoder::init(int graphId, + const ChannelOperateRequest& request, + bm_handle_t handle_) { common::ErrorCode errorCode = common::ErrorCode::SUCCESS; do { mUrl = request.url; @@ -82,12 +85,12 @@ common::ErrorCode Decoder::init(int deviceId, int graphId, mSampleInterval = request.sampleInterval; mFps = request.fps; mSampleStrategy = request.sampleStrategy; - int ret = bm_dev_request(&m_handle, deviceId); - mDeviceId = deviceId; + // int ret = bm_dev_request(&m_handle, deviceId); + m_handle = handle_; + mDeviceId = bm_get_devid(m_handle); mGraphId = graphId; mSourceType = request.sourceType; mImgIndex = 0; - assert(BM_SUCCESS == ret); mRoiPredefined = request.roi_predefined; if (mRoiPredefined) { mRoi.start_x = request.roi.start_x; diff --git a/element/multimedia/decode/src/ff_decode.cc b/element/multimedia/decode/src/ff_decode.cc index 744b2f12..c6a9f406 100644 --- a/element/multimedia/decode/src/ff_decode.cc +++ b/element/multimedia/decode/src/ff_decode.cc @@ -103,11 +103,6 @@ VideoDecFFM::VideoDecFFM() { video_stream_idx = -1; refcount = 1; - pkt = new AVPacket; - av_init_packet(pkt); - pkt->data = NULL; - pkt->size = 0; - avdevice_register_all(); // frame = av_frame_alloc(); } @@ -406,6 +401,10 @@ void VideoDecFFM::mFrameCount(const char* video_file, int& mFrameCount) { int VideoDecFFM::openDec(bm_handle_t* dec_handle, const char* input) { // printf("openDec, tid = %d\n", gettid()); + pkt = new AVPacket; + av_init_packet(pkt); + pkt->data = NULL; + pkt->size = 0; gettimeofday(&last_time, NULL); frame = av_frame_alloc(); frame_id = 0; @@ -484,11 +483,11 @@ void VideoDecFFM::closeDec() { } if (ifmt_ctx) { avformat_close_input(&ifmt_ctx); - avformat_free_context(ifmt_ctx); + avformat_free_context(ifmt_ctx); ifmt_ctx = NULL; } if (frame) { - av_frame_unref(frame); + av_frame_unref(frame); av_frame_free(&frame); frame = NULL; } diff --git a/framework/common/error_code.h b/framework/common/error_code.h index 9e451447..8e4d91b6 100644 --- a/framework/common/error_code.h +++ b/framework/common/error_code.h @@ -87,14 +87,14 @@ enum class ErrorCode { ERR_TRT_ENGINE = 5004, ERR_TRT_CONTEXT = 5005, ERR_TRT_INFERENCE = 5006, - ERR_CUDA_INVALID_DEVICE = 5007, - ERR_CUDA_INVALID_VALUE = 5008, - ERR_CUDA_MEMORY_ALLOCATION = 5009, - ERR_CUDA_INVALID_DEVICE_POINTER = 5010, - ERR_CUDA_INITIALIZATION = 5011, - ERR_CUDA_INVALID_MEMCPY_DIRECTION = 5012, - ERR_CUDA_INVALID_RESOURCE_HANDLE = 5013, - ERR_CUDA_DEVICE_IN_USE = 5014, + ERR_STREAM_INVALID_DEVICE = 5007, + ERR_STREAM_INVALID_VALUE = 5008, + ERR_STREAM_MEMORY_ALLOCATION = 5009, + ERR_STREAM_INVALID_DEVICE_POINTER = 5010, + ERR_STREAM_INITIALIZATION = 5011, + ERR_STREAM_INVALID_MEMCPY_DIRECTION = 5012, + ERR_STREAM_INVALID_RESOURCE_HANDLE = 5013, + ERR_STREAM_DEVICE_IN_USE = 5014, ERR_MXNET_CONTEXT = 5015, ERR_MXNET_INPUT_DATA = 5016, ERR_MXNET_INFERENCE = 5017, @@ -203,14 +203,14 @@ const std::unordered_map ErrorCodeMap = { {ErrorCode::ERR_TRT_ENGINE, "ERR_TRT_ENGINE"}, {ErrorCode::ERR_TRT_CONTEXT, "ERR_TRT_CONTEXT"}, {ErrorCode::ERR_TRT_INFERENCE, "ERR_TRT_INFERENCE"}, - {ErrorCode::ERR_CUDA_INVALID_DEVICE, "ERR_CUDA_INVALID_DEVICE"}, - {ErrorCode::ERR_CUDA_INVALID_VALUE, "ERR_CUDA_INVALID_VALUE"}, - {ErrorCode::ERR_CUDA_MEMORY_ALLOCATION, "ERR_CUDA_MEMORY_ALLOCATION"}, - {ErrorCode::ERR_CUDA_INVALID_DEVICE_POINTER, "ERR_CUDA_INVALID_DEVICE_POINTER"}, - {ErrorCode::ERR_CUDA_INITIALIZATION, "ERR_CUDA_INITIALIZATION"}, - {ErrorCode::ERR_CUDA_INVALID_MEMCPY_DIRECTION, "ERR_CUDA_INVALID_MEMCPY_DIRECTION"}, - {ErrorCode::ERR_CUDA_INVALID_RESOURCE_HANDLE, "ERR_CUDA_INVALID_RESOURCE_HANDLE"}, - {ErrorCode::ERR_CUDA_DEVICE_IN_USE, "ERR_CUDA_DEVICE_IN_USE"}, + {ErrorCode::ERR_STREAM_INVALID_DEVICE, "ERR_STREAM_INVALID_DEVICE"}, + {ErrorCode::ERR_STREAM_INVALID_VALUE, "ERR_STREAM_INVALID_VALUE"}, + {ErrorCode::ERR_STREAM_MEMORY_ALLOCATION, "ERR_STREAM_MEMORY_ALLOCATION"}, + {ErrorCode::ERR_STREAM_INVALID_DEVICE_POINTER, "ERR_STREAM_INVALID_DEVICE_POINTER"}, + {ErrorCode::ERR_STREAM_INITIALIZATION, "ERR_STREAM_INITIALIZATION"}, + {ErrorCode::ERR_STREAM_INVALID_MEMCPY_DIRECTION, "ERR_STREAM_INVALID_MEMCPY_DIRECTION"}, + {ErrorCode::ERR_STREAM_INVALID_RESOURCE_HANDLE, "ERR_STREAM_INVALID_RESOURCE_HANDLE"}, + {ErrorCode::ERR_STREAM_DEVICE_IN_USE, "ERR_STREAM_DEVICE_IN_USE"}, {ErrorCode::ERR_MXNET_CONTEXT, "ERR_MXNET_CONTEXT"}, {ErrorCode::ERR_MXNET_INPUT_DATA, "ERR_MXNET_INPUT_DATA"}, {ErrorCode::ERR_MXNET_INFERENCE, "ERR_MXNET_INFERENCE"}, @@ -392,22 +392,22 @@ static std::string ErrorCodeToString(ErrorCode code) { return "ERR_TRT_CONTEXT"; case ErrorCode::ERR_TRT_INFERENCE: return "ERR_TRT_INFERENCE"; - case ErrorCode::ERR_CUDA_INVALID_DEVICE: - return "ERR_CUDA_INVALID_DEVICE"; - case ErrorCode::ERR_CUDA_INVALID_VALUE: - return "ERR_CUDA_INVALID_VALUE"; - case ErrorCode::ERR_CUDA_MEMORY_ALLOCATION: - return "ERR_CUDA_MEMORY_ALLOCATION"; - case ErrorCode::ERR_CUDA_INVALID_DEVICE_POINTER: - return "ERR_CUDA_INVALID_DEVICE_POINTER"; - case ErrorCode::ERR_CUDA_INITIALIZATION: - return "ERR_CUDA_INITIALIZATION"; - case ErrorCode::ERR_CUDA_INVALID_MEMCPY_DIRECTION: - return "ERR_CUDA_INVALID_MEMCPY_DIRECTION"; - case ErrorCode::ERR_CUDA_INVALID_RESOURCE_HANDLE: - return "ERR_CUDA_INVALID_RESOURCE_HANDLE"; - case ErrorCode::ERR_CUDA_DEVICE_IN_USE: - return "ERR_CUDA_DEVICE_IN_USE"; + case ErrorCode::ERR_STREAM_INVALID_DEVICE: + return "ERR_STREAM_INVALID_DEVICE"; + case ErrorCode::ERR_STREAM_INVALID_VALUE: + return "ERR_STREAM_INVALID_VALUE"; + case ErrorCode::ERR_STREAM_MEMORY_ALLOCATION: + return "ERR_STREAM_MEMORY_ALLOCATION"; + case ErrorCode::ERR_STREAM_INVALID_DEVICE_POINTER: + return "ERR_STREAM_INVALID_DEVICE_POINTER"; + case ErrorCode::ERR_STREAM_INITIALIZATION: + return "ERR_STREAM_INITIALIZATION"; + case ErrorCode::ERR_STREAM_INVALID_MEMCPY_DIRECTION: + return "ERR_STREAM_INVALID_MEMCPY_DIRECTION"; + case ErrorCode::ERR_STREAM_INVALID_RESOURCE_HANDLE: + return "ERR_STREAM_INVALID_RESOURCE_HANDLE"; + case ErrorCode::ERR_STREAM_DEVICE_IN_USE: + return "ERR_STREAM_DEVICE_IN_USE"; case ErrorCode::ERR_MXNET_CONTEXT: return "ERR_MXNET_CONTEXT"; case ErrorCode::ERR_MXNET_INPUT_DATA: diff --git a/framework/common/http_defs.cc b/framework/common/http_defs.cc index 5d1feee8..d884e560 100644 --- a/framework/common/http_defs.cc +++ b/framework/common/http_defs.cc @@ -55,5 +55,72 @@ bool str_to_object(const std::string& strjson, RequestSingleFloat& request) { return true; } +void to_json(nlohmann::json& j, const RequestAddChannel& p) { + j = nlohmann::json{ + {"channel_id", p.channel_id}, {"url", p.url}, + {"source_type", p.source_type}, {"sample_interval", p.sample_interval}, + {"decode_id", p.decode_id}, {"fps", p.fps}, + {"loop_num", p.loop_num}, {"sample_strategy", p.sample_strategy}, + {"graph_id", p.graph_id}}; +} +void from_json(const nlohmann::json& j, RequestAddChannel& p) { + if (j.count("url") == 0 || j.count("source_type") == 0 || + j.count("channel_id") == 0) { + p.errorCode = ErrorCode::ERR_STREAM_INVALID_VALUE; + return; + } + + p.channel_id = j.at("channel_id").get(); + p.url = j.at("url").get(); + p.source_type = j.at("source_type").get(); + if (j.count("sample_interval")) { + p.sample_interval = j.at("sample_interval").get(); + } + if (j.count("decode_id")) { + p.decode_id = j.at("decode_id").get(); + } + if (j.count("fps")) { + p.fps = j.at("fps").get(); + } + if (j.count("loop_num")) { + p.loop_num = j.at("loop_num").get(); + } + if (j.count("sample_strategy")) + p.sample_strategy = j.at("sample_strategy").get(); + if (j.count("graph_id")) { + p.graph_id = j.at("graph_id").get(); + } +} +bool str_to_object(const std::string& strjson, RequestAddChannel& request) { + nlohmann::json json_object = nlohmann::json::parse(strjson); + request = json_object; + return request.errorCode == ErrorCode::SUCCESS ? true : false; +} + +void to_json(nlohmann::json& j, const RequestStopChannel& p) { + j = nlohmann::json{{"channel_id", p.channel_id}, + {"decode_id", p.decode_id}, + {"graph_id", p.graph_id}}; +} +void from_json(const nlohmann::json& j, RequestStopChannel& p) { + if (j.count("channel_id") == 0) { + p.errorCode = ErrorCode::ERR_STREAM_INVALID_VALUE; + return; + } + p.channel_id = j.at("channel_id").get(); + + if (j.count("decode_id")) { + p.decode_id = j.at("decode_id").get(); + } + if (j.count("graph_id")) { + p.graph_id = j.at("graph_id").get(); + } +} +bool str_to_object(const std::string& strjson, RequestStopChannel& request) { + nlohmann::json json_object = nlohmann::json::parse(strjson); + request = json_object; + return request.errorCode == ErrorCode::SUCCESS ? true : false; +} + } // namespace common } // namespace sophon_stream \ No newline at end of file diff --git a/framework/common/http_defs.h b/framework/common/http_defs.h index adecec52..bfc8a001 100644 --- a/framework/common/http_defs.h +++ b/framework/common/http_defs.h @@ -11,6 +11,7 @@ #define SOPHON_STREAM_COMMON_HTTP_DEFS_H_ #include "nlohmann/json.hpp" +#include "error_code.h" namespace sophon_stream { namespace common { @@ -44,6 +45,33 @@ void to_json(nlohmann::json& j, const RequestSingleFloat& p); void from_json(const nlohmann::json& j, RequestSingleFloat& p); bool str_to_object(const std::string& strjson, RequestSingleFloat& request); +struct RequestAddChannel { + int channel_id; + std::string url; + std::string source_type; + int sample_interval = 1; + int decode_id = -1; + float fps = 1; + int loop_num = 1; + std::string sample_strategy = "DROP"; + int graph_id = 0; + ErrorCode errorCode = ErrorCode::SUCCESS; +}; +void to_json(nlohmann::json& j, const RequestAddChannel& p); +void from_json(const nlohmann::json& j, RequestAddChannel& p); +bool str_to_object(const std::string& strjson, RequestAddChannel& request); + +struct RequestStopChannel { + int channel_id; + int decode_id = -1; + int graph_id = 0; + ErrorCode errorCode = ErrorCode::SUCCESS; +}; +void to_json(nlohmann::json& j, const RequestStopChannel& p); +void from_json(const nlohmann::json& j, RequestStopChannel& p); +bool str_to_object(const std::string& strjson, RequestStopChannel& request); + + } // namespace common } // namespace sophon_stream diff --git a/samples/README.md b/samples/README.md index 111c1e02..897db629 100644 --- a/samples/README.md +++ b/samples/README.md @@ -150,4 +150,26 @@ IMG_DIR/ |path | 字符串 | http_listen默认为"/task/test"，http_report默认无 | 上报/监听的路由，report时上报请求到此path，listen时监听此path的post请求。对于监听请求来说，此字段留空即可 | > **注意**： ->1. http_report字段必须完整，否则不会进行上报，默认不上报。 \ No newline at end of file +>1. http_report字段必须完整，否则不会进行上报，默认不上报。 + +## 3. 功能 + +samples支持通过http请求动态增加或停止一路码流的功能。在设置了上面的`http_listen`字段之后，可以使用如下代码增加一路码流： + +```python +# add a channel +url = "http://localhost:8000/stream/addChannel" +payload = {"channel_id": 3,"url": "../yolov5/data/videos/test_car_person_1080P.avi","source_type": "VIDEO","sample_interval": 1,"loop_num": 1,"fps": 1} +headers = {'Content-Type': 'application/json'} +response = requests.request("POST", url, headers=headers, data=json.dumps(payload)) +print(response) + +# stop a channel +url = "http://localhost:8000/stream/stopChannel" +payload = {"channel_id": 3} +headers = {'Content-Type': 'application/json'} +response = requests.request("POST", url, headers=headers, data=json.dumps(payload)) +print(response) +``` + +需要注意，目前bytetrack、osd、encode、http_push等插件的工作示例数量是依照预设的线程数来分配。因此，使用动态增加码流功能时，需要保证前述插件的线程数大于等于增加后的码流总数。 \ No newline at end of file diff --git a/samples/src/main.cc b/samples/src/main.cc index 7e37eff4..0219bb9e 100644 --- a/samples/src/main.cc +++ b/samples/src/main.cc @@ -58,6 +58,11 @@ constexpr const char* JSON_CONFIG_HTTP_CONFIG_PORT_FILED = "port"; constexpr const char* JSON_CONFIG_HTTP_CONFIG_PATH_FILED = "path"; static int channel_id_config = 0; +static int num_channels = 0; +std::vector> fpsProfilers; +std::map>> graph_src_id_port_map; +const std::string addChannelPath = "/stream/addChannel"; +const std::string stopChannelPath = "/stream/stopChannel"; demo_config parse_demo_json(std::string& json_path) { std::ifstream istream; @@ -239,8 +244,7 @@ using drawFuncType = std::function)>; drawFuncType getDrawFunc(demo_config& demo_json) { - std::function)> - draw_func; + drawFuncType draw_func; std::string out_dir = "./results"; if (demo_json.draw_func_name == "draw_bytetrack_results") draw_func = @@ -293,6 +297,118 @@ drawFuncType getDrawFunc(demo_config& demo_json) { return draw_func; } +/** + * @brief 动态增加一路码流的回调 + * + * @param request http request，对格式有要求 + * @param response http response，标识成功或失败 + */ +void addChannel(const httplib::Request& request, httplib::Response& response) { + sophon_stream::common::Response resp; + sophon_stream::common::RequestAddChannel rac; + auto ret = sophon_stream::common::str_to_object(request.body, rac); + if (!ret) { + IVS_ERROR("Invalid Request! Please check it."); + resp.code = 5008; // Invalid value + resp.msg = "Invalid Request"; + nlohmann::json json_res = resp; + response.set_content(json_res.dump(), "application/json"); + return; + } + // update info + num_channels++; + std::string fpsName = "channel_" + std::to_string(num_channels - 1); + fpsProfilers.push_back( + std::make_shared(fpsName, 100)); + + // push START signal + auto& engine = sophon_stream::framework::SingletonEngine::getInstance(); + int graph_id = rac.graph_id; // 默认是graph0 + auto channelTask = + std::make_shared(); + channelTask->request.operation = sophon_stream::element::decode:: + ChannelOperateRequest::ChannelOperate::START; + channelTask->request.channelId = rac.channel_id; + nlohmann::json j; + to_json(j, rac); + channelTask->request.json = j.dump(); + int decode_id = rac.decode_id; + + auto src_id_port_vec = graph_src_id_port_map[graph_id]; + for (auto& src_id_port : src_id_port_vec) { + if ((decode_id == -1 && src_id_port_vec.size() == 1) || + src_id_port.first == decode_id) { + sophon_stream::common::ErrorCode errorCode = + engine.pushSourceData(graph_id, src_id_port.first, src_id_port.second, + std::static_pointer_cast(channelTask)); + IVS_DEBUG( + "Push Source Data, GraphId = {0}, ElementId = {1}, ElementPort = " + "{2}, ChannelId = {3}", + graph_id, src_id_port.first, src_id_port.second, + channelTask->request.channelId); + } + } + resp.code = 0; + resp.msg = "success"; + nlohmann::json json_res = resp; + response.set_content(json_res.dump(), "application/json"); + + return; +} + +/** + * @brief 动态停止一路码流的回调 + * + * @param request http request，对格式有要求 + * @param response http response，标识成功或失败 + */ +void stopChannel(const httplib::Request& request, httplib::Response& response) { + sophon_stream::common::Response resp; + sophon_stream::common::RequestStopChannel rsc; + auto ret = sophon_stream::common::str_to_object(request.body, rsc); + if (!ret) { + IVS_ERROR("Invalid Request! Please check it."); + resp.code = 5008; // Invalid value + resp.msg = "Invalid Request"; + nlohmann::json json_res = resp; + response.set_content(json_res.dump(), "application/json"); + return; + } + num_channels--; + auto& engine = sophon_stream::framework::SingletonEngine::getInstance(); + int graph_id = rsc.graph_id; // 默认是graph0 + auto channelTask = + std::make_shared(); + channelTask->request.operation = sophon_stream::element::decode:: + ChannelOperateRequest::ChannelOperate::STOP; + channelTask->request.channelId = rsc.channel_id; + nlohmann::json j; + to_json(j, rsc); + channelTask->request.json = j.dump(); + int decode_id = rsc.decode_id; + + auto src_id_port_vec = graph_src_id_port_map[graph_id]; + for (auto& src_id_port : src_id_port_vec) { + if ((decode_id == -1 && src_id_port_vec.size() == 1) || + src_id_port.first == decode_id) { + sophon_stream::common::ErrorCode errorCode = + engine.pushSourceData(graph_id, src_id_port.first, src_id_port.second, + std::static_pointer_cast(channelTask)); + IVS_DEBUG( + "Push Source Data, GraphId = {0}, ElementId = {1}, ElementPort = " + "{2}, ChannelId = {3}", + graph_id, src_id_port.first, src_id_port.second, + channelTask->request.channelId); + } + } + resp.code = 0; + resp.msg = "success"; + nlohmann::json json_res = resp; + response.set_content(json_res.dump(), "application/json"); + + return; +} + int main(int argc, char* argv[]) { const char* keys = "{demo_config_path | " @@ -333,12 +449,13 @@ int main(int argc, char* argv[]) { // demo.json里的码流数量，这里每个码流都可以配置graph_id，对应不同的graph demo_json.num_channels_per_graph = demo_json.channel_configs.size(); // 总的码流数就是demo_json.num_channels_per_graph，这个命名需要修改 - int num_channels = demo_json.num_channels_per_graph; + num_channels = demo_json.num_channels_per_graph; - std::vector<::sophon_stream::common::FpsProfiler> fpsProfilers(num_channels); + fpsProfilers.resize(num_channels); for (int i = 0; i < num_channels; ++i) { std::string fpsName = "channel_" + std::to_string(i); - fpsProfilers[i].config(fpsName, 100); + fpsProfilers[i] = + std::make_shared(fpsName, 100); } auto draw_func = getDrawFunc(demo_json); @@ -350,7 +467,7 @@ int main(int argc, char* argv[]) { if (objectMetadata == nullptr) return; if (!objectMetadata->mFilter) { frameCount++; - fpsProfilers[objectMetadata->mFrame->mChannelIdInternal].add(1); + fpsProfilers[objectMetadata->mFrame->mChannelIdInternal]->add(1); } if (objectMetadata->mFrame->mEndOfStream) { printf("meet a eof\n"); @@ -366,7 +483,13 @@ int main(int argc, char* argv[]) { sophon_stream::framework::ListenThread::getInstance(); listenthread->init(demo_json.report_config, demo_json.listen_config); engine.setListener(listenthread); - std::map>> graph_src_id_port_map; + listenthread->setHandler( + addChannelPath, sophon_stream::framework::RequestType::POST, + std::bind(addChannel, std::placeholders::_1, std::placeholders::_2)); + listenthread->setHandler( + stopChannelPath, sophon_stream::framework::RequestType::POST, + std::bind(stopChannel, std::placeholders::_1, std::placeholders::_2)); + init_engine(engine, engine_json, sinkHandler, graph_src_id_port_map); for (auto& channel_config : demo_json.channel_configs) { From 1306d8f07e69d0aaa7ed3ee389a44c2f8d78eb20 Mon Sep 17 00:00:00 2001 From: "yizhou.xu" Date: Wed, 14 Aug 2024 11:19:10 +0800 Subject: [PATCH 8/8] add handle for subObjMetadata Change-Id: Iaf6dca58600d392c0d4253000e729638e333f2f4 --- element/tools/distributor/src/distributor.cc | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/element/tools/distributor/src/distributor.cc b/element/tools/distributor/src/distributor.cc index 5d535821..33321d32 100644 --- a/element/tools/distributor/src/distributor.cc +++ b/element/tools/distributor/src/distributor.cc @@ -185,6 +185,7 @@ void Distributor::makeSubObjectMetadata( subObj->mFrame->mChannelIdInternal = obj->mFrame->mChannelIdInternal; subObj->mSubId = subId; subObj->mFrame->mEndOfStream = obj->mFrame->mEndOfStream; + subObj->mFrame->mHandle = obj->mFrame->mHandle; } cv::Mat Distributor::estimateAffine2D( @@ -503,8 +504,11 @@ bm_image Distributor::get_rotate_crop_image(bm_handle_t handle, bm_image_create(handle, crop_height, crop_width, input_bmimg_planar.image_format, input_bmimg_planar.data_type, &crop_bmimg); - auto ret = bmcv_image_warp_perspective_with_coordinate(handle, 1, &coord, &input_bmimg_planar, &crop_bmimg,0); - STREAM_CHECK(ret == 0,"bmcv_image_warp_perspective_with_coordinate Failed! Program Terminated."); + auto ret = bmcv_image_warp_perspective_with_coordinate( + handle, 1, &coord, &input_bmimg_planar, &crop_bmimg, 0); + STREAM_CHECK(ret == 0, + "bmcv_image_warp_perspective_with_coordinate Failed! Program " + "Terminated."); if ((float)crop_height / crop_width < 1.5) { return crop_bmimg; @@ -530,8 +534,10 @@ bm_image Distributor::get_rotate_crop_image(bm_handle_t handle, matrix_image.matrix->m[5] = rot_mat.at(1, 2) - crop_height / 2.0 + crop_width / 2.0; - auto ret = bmcv_image_warp_affine(handle, 1, &matrix_image,&crop_bmimg, &rot_bmimg,0); - STREAM_CHECK(ret == 0,"bmcv_image_warp_affine Failed. Program Terminated."); + auto ret = bmcv_image_warp_affine(handle, 1, &matrix_image, &crop_bmimg, + &rot_bmimg, 0); + STREAM_CHECK(ret == 0, + "bmcv_image_warp_affine Failed. Program Terminated."); bm_image_destroy(crop_bmimg); return rot_bmimg; } @@ -725,4 +731,4 @@ REGISTER_WORKER("distributor", Distributor) } // namespace distributor } // namespace element -} // namespace sophon_stream \ No newline at end of file +} // namespace sophon_stream