Merge branch 'rtsp_server' into 'dev'

add rtsp server api and demo See merge request maix_sw/k230_canmv!280
kendryte · Jul 15, 2024 · b131b36 · b131b36
2 parents 1b09e6c + 0149e19
commit b131b36
Show file tree

Hide file tree

Showing 17 changed files with 698 additions and 6 deletions.
diff --git a/fs_resource/libs/RtspSever.py b/fs_resource/libs/RtspSever.py
@@ -0,0 +1,101 @@
+
+# Video encode example
+#
+# Note: You will need an SD card to run this example.
+#
+# You can capture videos and encode them into 264 files
+
+from media.vencoder import *
+from media.sensor import *
+from media.media import *
+import time, os
+import _thread
+import multimedia as mm
+from time import *
+
+class RtspServer:
+    def __init__(self,session_name="test",port=8554,video_type = mm.multi_media_type.media_h264,enable_audio=False):
+        self.session_name = session_name
+        self.video_type = video_type
+        self.enable_audio = enable_audio
+        self.port = port
+        self.rtspserver = mm.rtsp_server()
+        self.venc_chn = VENC_CHN_ID_0
+        self.start_stream = False
+        self.runthread_over = False
+
+    def start(self):
+        self._init_stream()
+        self.rtspserver.rtspserver_init(self.port)
+        self.rtspserver.rtspserver_createsession(self.session_name,self.video_type,self.enable_audio)
+        self.rtspserver.rtspserver_start()
+        self._start_stream()
+
+        self.start_stream = True
+        _thread.start_new_thread(self._do_rtsp_stream,())
+
+
+    def stop(self):
+        self.start_stream = False
+        while not self.runthread_over:
+            sleep(0.1)
+        self.runthread_over = False
+
+        self._stop_stream()
+        self.rtspserver.rtspserver_stop()
+        #self.rtspserver.rtspserver_destroysession(self.session_name)
+        self.rtspserver.rtspserver_deinit()
+
+    def _init_stream(self):
+        width = 1280
+        height = 720
+        width = ALIGN_UP(width, 16)
+        self.sensor = Sensor()
+        self.sensor.reset()
+        self.sensor.set_framesize(width = width, height = height, alignment=12)
+        self.sensor.set_pixformat(Sensor.YUV420SP)
+        self.encoder = Encoder()
+        self.encoder.SetOutBufs(self.venc_chn, 15, width, height)
+        self.link = MediaManager.link(self.sensor.bind_info()['src'], (VIDEO_ENCODE_MOD_ID, VENC_DEV_ID, self.venc_chn))
+        MediaManager.init()
+        chnAttr = ChnAttrStr(self.encoder.PAYLOAD_TYPE_H264, self.encoder.H264_PROFILE_MAIN, width, height)
+        self.encoder.Create(self.venc_chn, chnAttr)
+
+    def _start_stream(self):
+        self.encoder.Start(self.venc_chn)
+        self.sensor.run()
+
+    def _stop_stream(self):
+        self.sensor.stop()
+        del self.link
+        self.encoder.Stop(self.venc_chn)
+        self.encoder.Destroy(self.venc_chn)
+        MediaManager.deinit()
+
+    def _do_rtsp_stream(self):
+        try:
+            streamData = StreamData()
+            while self.start_stream:
+                os.exitpoint()
+                self.encoder.GetStream(self.venc_chn, streamData) # 获取一帧码流
+
+                for pack_idx in range(0, streamData.pack_cnt):
+                    stream_data = bytes(uctypes.bytearray_at(streamData.data[pack_idx], streamData.data_size[pack_idx]))
+                    self.rtspserver.rtspserver_sendvideodata(self.session_name,stream_data, streamData.data_size[pack_idx],1000)
+                    #print("stream size: ", streamData.data_size[pack_idx], "stream type: ", streamData.stream_type[pack_idx])
+
+                self.encoder.ReleaseStream(self.venc_chn, streamData) # 释放一帧码流
+
+        except KeyboardInterrupt as e:
+            print("user stop: ", e)
+        except BaseException as e:
+            sys.print_exception(e)
+
+        self.runthread_over = True
+
+if __name__ == "__main__":
+    rtspserver = RtspServer()
+    rtspserver.start()
+    sleep(30)
+    rtspserver.stop()
+    print("done")
diff --git a/fs_resource/tests/media/rtsp_server.py b/fs_resource/tests/media/rtsp_server.py
@@ -0,0 +1,33 @@
+# Description: This example demonstrates how to stream video and audio to the network using the RTSP server.
+#
+# Note: You will need an SD card to run this example.
+#
+# You can run the rtsp server to stream video and audio to the network
+
+from time import *
+from media.rtspserver import *   #导入rtsp server 模块
+import os
+import time
+
+def rtsp_server_test():
+    rtspserver = RtspServer() #创建rtsp server对象
+    rtspserver.start() #启动rtsp server
+    print("rtsp server start:",rtspserver.get_rtsp_url()) #打印rtsp server start
+
+    time_start = time.time() #获取当前时间
+    try:
+        while(time.time() - time_start < 30):
+            time.sleep(0.1)
+            os.exitpoint()
+    except KeyboardInterrupt as e:
+        print("user stop: ", e)
+    except BaseException as e:
+        sys.print_exception(e)
+
+    rtspserver.stop() #停止rtsp server
+    print("rtsp server stop") #打印rtsp server stop
+
+if __name__ == "__main__":
+    os.exitpoint(os.EXITPOINT_ENABLE)
+    rtsp_server_test()
+    print("rtsp server done")
diff --git a/fs_resource/tests/media/video_encoder.py b/fs_resource/tests/media/video_encoder.py
@@ -9,8 +9,6 @@
 from media.media import *
 import time, os
 
-# NOT WORK NOW!!!
-
 def venc_test():
     print("venc_test start")
     width = 1280

diff --git a/...erlay/src/big/mpp/middleware/src/live555/BasicUsageEnvironment/libBasicUsageEnvironment.a b/...erlay/src/big/mpp/middleware/src/live555/BasicUsageEnvironment/libBasicUsageEnvironment.a
diff --git a/k230_sdk_overlay/src/big/mpp/middleware/src/live555/UsageEnvironment/libUsageEnvironment.a b/k230_sdk_overlay/src/big/mpp/middleware/src/live555/UsageEnvironment/libUsageEnvironment.a
diff --git a/k230_sdk_overlay/src/big/mpp/middleware/src/live555/groupsock/libgroupsock.a b/k230_sdk_overlay/src/big/mpp/middleware/src/live555/groupsock/libgroupsock.a
diff --git a/k230_sdk_overlay/src/big/mpp/middleware/src/live555/liveMedia/libliveMedia.a b/k230_sdk_overlay/src/big/mpp/middleware/src/live555/liveMedia/libliveMedia.a
diff --git a/k230_sdk_overlay/src/big/mpp/middleware/src/rtsp_server/include/rtsp_server.h b/k230_sdk_overlay/src/big/mpp/middleware/src/rtsp_server/include/rtsp_server.h
@@ -0,0 +1,54 @@
+#ifndef _KD_RTSP_SERVER_H
+#define _KD_RTSP_SERVER_H
+
+#include <unistd.h>
+#include <string>
+#include <memory>
+
+enum class VideoType {
+    kVideoTypeH264,
+    kVideoTypeH265,
+    kVideoTypeMjpeg,
+    kVideoTypeButt
+};
+
+struct SessionAttr {
+    bool with_video {false};
+    bool with_audio {false};   // G711U
+    bool with_audio_backchannel{false}; // G711U
+    VideoType video_type; // valid when with_video is true
+};
+
+class IOnBackChannel {
+  public:
+    virtual ~IOnBackChannel() {}
+    virtual void OnBackChannelData(std::string &session_name, const uint8_t *data, size_t size, uint64_t timestamp) = 0;
+};
+
+class KdRtspServer {
+  public:
+    KdRtspServer();
+    ~KdRtspServer();
+
+    int Init(int port = 8554, IOnBackChannel *back_channel = nullptr);
+    void DeInit();
+
+    int CreateSession(const std::string &session_name, const SessionAttr &session_attr);
+    int DestroySession(const std::string &session_name);
+    char* GetRtspUrl(const std::string &session_name);
+    void Start();
+    void Stop();
+
+    int SendVideoData(const std::string &session_name, const uint8_t *data, size_t size, uint64_t timestamp);
+    int SendAudioData(const std::string &session_name, const uint8_t *data, size_t size, uint64_t timestamp);
+
+  private:
+    KdRtspServer(const KdRtspServer &) = delete;
+    KdRtspServer& operator=(const KdRtspServer &) = delete;
+
+  private:
+    class Impl;
+    std::unique_ptr<Impl> impl_{nullptr};
+};
+
+#endif // _RTSP_SERVER_H
diff --git a/k230_sdk_overlay/src/big/mpp/middleware/src/rtsp_server/librtsp_server.a b/k230_sdk_overlay/src/big/mpp/middleware/src/rtsp_server/librtsp_server.a
diff --git a/micropython/port/Makefile b/micropython/port/Makefile
@@ -18,7 +18,7 @@ PROG ?= micropython
 USER_C_MODULES = 3d-party/ulab
 
 # include py core make definitions
-include $(TOP)/py/py.mk 
+include $(TOP)/py/py.mk
 include $(TOP)/extmod/extmod.mk
 
 INC += -I.
@@ -38,6 +38,7 @@ INC += -Iinclude/ai_cube
 INC += -Iinclude/ai_demo
 INC += -Iinclude/ai_demo/kws
 INC += -Iinclude/ai_demo/tts_zh
+INC += -Iinclude/multi_media
 INC += -I3d-party
 INC += -I3d-party/ulab
 INC += -I3d-party/freetype/freetype/include
@@ -53,6 +54,13 @@ INC += -I$(K230_CANMV_ROOT)/k230_sdk/src/big/rt-smart/userapps/sdk/rt-thread/inc
 INC += -I$(K230_CANMV_ROOT)/k230_sdk/src/big/rt-smart/userapps/sdk/rt-thread/components/drivers
 INC += -I$(K230_CANMV_ROOT)/k230_sdk/src/big/mpp/middleware/src/mp4_format/include
 
+INC += -I$(K230_CANMV_ROOT)/k230_sdk/src/big/mpp/middleware/src/live555/UsageEnvironment
+INC += -I$(K230_CANMV_ROOT)/k230_sdk/src/big/mpp/middleware/src/live555/BasicUsageEnvironment
+INC += -I$(K230_CANMV_ROOT)/k230_sdk/src/big/mpp/middleware/src/live555/groupsock
+INC += -I$(K230_CANMV_ROOT)/k230_sdk/src/big/mpp/middleware/src/live555/liveMedia
+INC += -I$(K230_CANMV_ROOT)/k230_sdk/src/big/mpp/middleware/src/rtsp_server/include
+
+
 -include $(BUILD)/CANMV_VER
 CANMV_VER = $(shell git describe --tags `git rev-list --tags --max-count=1`)-$(shell git rev-parse --short HEAD)
 ifneq ($(CANMV_VER),$(CANMV_VER_OLD))
@@ -92,7 +100,7 @@ LDFLAGS_ARCH = -Wl,[email protected],--cref -Wl,--gc-sections
 LDFLAGS += -T core/link.lds --static
 LDFLAGS += $(LDFLAGS_MOD) $(LDFLAGS_ARCH) -lm -lpthread $(LDFLAGS_EXTRA)
 
-MPP_LIB_PATH = $(K230_CANMV_ROOT)/k230_sdk/src/big/mpp/userapps/lib  
+MPP_LIB_PATH = $(K230_CANMV_ROOT)/k230_sdk/src/big/mpp/userapps/lib
 CDK_LIB_PATH = $(K230_CANMV_ROOT)/k230_sdk/src/common/cdk/user/component/ipcmsg/host/lib/
 
 MPP_LIBS = $(addprefix -l, $(subst lib, ,$(basename $(notdir $(foreach dir, $(MPP_LIB_PATH) $(CDK_LIB_PATH), $(wildcard $(dir)/*))))))
@@ -111,6 +119,15 @@ OPENCV_DEP_LIB_PATH = $(K230_CANMV_ROOT)/k230_sdk/src/big/utils/lib/opencv/lib/o
 OPENCV_LIBS = -lstdc++ -lopencv_core -lopencv_imgcodecs -lopencv_imgproc -lopencv_highgui -lopencv_videoio -lzlib -llibjpeg-turbo -llibopenjp2 -llibpng -llibtiff -llibwebp -lcsi_cv -latomic
 LDFLAGS += -L$(OPENCV_LIB_PATH) -L$(OPENCV_DEP_LIB_PATH) -Wl,--start-group $(OPENCV_LIBS) -Wl,--end-group
 
+# rtsp server
+LIVE555_USAGEENVIRONMENT = $(K230_CANMV_ROOT)/k230_sdk/src/big/mpp/middleware/src/live555/UsageEnvironment
+LIVE555_BASICUSAGEENVIRONMENT = $(K230_CANMV_ROOT)/k230_sdk/src/big/mpp/middleware/src/live555/BasicUsageEnvironment
+LIVE555_GROUPSOCK = $(K230_CANMV_ROOT)/k230_sdk/src/big/mpp/middleware/src/live555/groupsock
+LIVE555_LIVEMEDIA = $(K230_CANMV_ROOT)/k230_sdk/src/big/mpp/middleware/src/live555/liveMedia
+RTSP_SERVER_LIB_PATH = $(K230_CANMV_ROOT)/k230_sdk/src/big/mpp/middleware/src/rtsp_server
+RTSP_SERVER_LIBS = -lrtsp_server -lliveMedia -lgroupsock -lBasicUsageEnvironment -lUsageEnvironment -lstdc++
+LDFLAGS += -L$(LIVE555_USAGEENVIRONMENT) -L$(LIVE555_BASICUSAGEENVIRONMENT) -L$(LIVE555_GROUPSOCK) -L$(LIVE555_LIVEMEDIA) -L$(RTSP_SERVER_LIB_PATH) -Wl,--start-group $(RTSP_SERVER_LIBS) -Wl,--end-group
+
 # freetype
 FREETYPE_LIB_PATH = $(BUILD)/freetype
 FREETYPE_LIBS = -lfreetype
@@ -122,7 +139,7 @@ endif
 ifeq ($(MICROPY_PY_SOCKET_CANMV),1)
 CFLAGS += -DMICROPY_PY_SOCKET_CANMV=1 -g
 endif
-CFLAGS += -DHAVE_CCONFIG_H 
+CFLAGS += -DHAVE_CCONFIG_H
 # source files
 SRC_C += \
 	core/main.c \
@@ -142,6 +159,7 @@ AIDEMO_SRC_C = $(wildcard ai_demo/*.c)
 SOCKET_SRC_C = $(wildcard socket_network/*.c)
 MODULES_SRC_C = $(wildcard modules/*.c)
 MODULES_SRC_C += $(wildcard modules/*/*.c)
+MULTIMEDIA_SRC_C = $(wildcard multi_media/*.c)
 
 OMV_SRC_C = $(wildcard omv/*.c)
 OMV_SRC_C += $(wildcard omv/*/*.c)
@@ -175,7 +193,8 @@ SRC_C += \
 	${AICUBE_SRC_C} \
 	${AIDEMO_SRC_C} \
 	${LVGL_SRC_C} \
-	${MODULES_SRC_C}
+	${MODULES_SRC_C} \
+	${MULTIMEDIA_SRC_C}
 
 SHARED_SRC_C += $(addprefix shared/,\
 	runtime/gchelper_generic.c \
@@ -197,6 +216,8 @@ SRC_CXX += ${AIDEMO_TTS_ZH_SRC_CXX}
 MODULES_SRC_CXX = $(wildcard modules/*.cpp)
 MODULES_SRC_CXX += $(wildcard modules/*/*.cpp)
 SRC_CXX += ${MODULES_SRC_CXX}
+MULTIMEDIA_SRC_CXX = $(wildcard multi_media/*.cpp)
+SRC_CXX += ${MULTIMEDIA_SRC_CXX}
 
 OBJ = $(PY_O)
 OBJ += $(addprefix $(BUILD)/, $(SRC_C:.c=.o))

diff --git a/micropython/port/builtin_py/media/rtspserver.py b/micropython/port/builtin_py/media/rtspserver.py
@@ -0,0 +1,91 @@
+from media.vencoder import *
+from media.sensor import *
+from media.media import *
+import time, os
+import _thread
+import multimedia as mm
+from time import *
+
+class RtspServer:
+    def __init__(self,session_name="test",port=8554,video_type = mm.multi_media_type.media_h264,enable_audio=False):
+        self.session_name = session_name
+        self.video_type = video_type
+        self.enable_audio = enable_audio
+        self.port = port
+        self.rtspserver = mm.rtsp_server()
+        self.venc_chn = VENC_CHN_ID_0
+        self.start_stream = False
+        self.runthread_over = False
+
+    def start(self):
+        self._init_stream()
+        self.rtspserver.rtspserver_init(self.port)
+        self.rtspserver.rtspserver_createsession(self.session_name,self.video_type,self.enable_audio)
+        self.rtspserver.rtspserver_start()
+        self._start_stream()
+
+        self.start_stream = True
+        _thread.start_new_thread(self._do_rtsp_stream,())
+
+
+    def stop(self):
+        self.start_stream = False
+        while not self.runthread_over:
+            sleep(0.1)
+        self.runthread_over = False
+
+        self._stop_stream()
+        self.rtspserver.rtspserver_stop()
+        #self.rtspserver.rtspserver_destroysession(self.session_name)
+        self.rtspserver.rtspserver_deinit()
+
+    def get_rtsp_url(self):
+        return self.rtspserver.rtspserver_getrtspurl(self.session_name)
+
+    def _init_stream(self):
+        width = 1280
+        height = 720
+        width = ALIGN_UP(width, 16)
+        self.sensor = Sensor()
+        self.sensor.reset()
+        self.sensor.set_framesize(width = width, height = height, alignment=12)
+        self.sensor.set_pixformat(Sensor.YUV420SP)
+        self.encoder = Encoder()
+        self.encoder.SetOutBufs(self.venc_chn, 15, width, height)
+        self.link = MediaManager.link(self.sensor.bind_info()['src'], (VIDEO_ENCODE_MOD_ID, VENC_DEV_ID, self.venc_chn))
+        MediaManager.init()
+        chnAttr = ChnAttrStr(self.encoder.PAYLOAD_TYPE_H264, self.encoder.H264_PROFILE_MAIN, width, height)
+        self.encoder.Create(self.venc_chn, chnAttr)
+
+    def _start_stream(self):
+        self.encoder.Start(self.venc_chn)
+        self.sensor.run()
+
+    def _stop_stream(self):
+        self.sensor.stop()
+        del self.link
+        self.encoder.Stop(self.venc_chn)
+        self.encoder.Destroy(self.venc_chn)
+        MediaManager.deinit()
+
+    def _do_rtsp_stream(self):
+        try:
+            streamData = StreamData()
+            while self.start_stream:
+                os.exitpoint()
+                self.encoder.GetStream(self.venc_chn, streamData) # 获取一帧码流
+
+                for pack_idx in range(0, streamData.pack_cnt):
+                    stream_data = bytes(uctypes.bytearray_at(streamData.data[pack_idx], streamData.data_size[pack_idx]))
+                    self.rtspserver.rtspserver_sendvideodata(self.session_name,stream_data, streamData.data_size[pack_idx],1000)
+                    #print("stream size: ", streamData.data_size[pack_idx], "stream type: ", streamData.stream_type[pack_idx])
+
+                self.encoder.ReleaseStream(self.venc_chn, streamData) # 释放一帧码流
+
+        except KeyboardInterrupt as e:
+            print("user stop: ", e)
+        except BaseException as e:
+            sys.print_exception(e)
+
+        self.runthread_over = True
+        print("_do_rtsp_stream over")