Fix problems with processing images of different sizes (#802)

* #783 do not recreate gst elements on image resolution change * #783 fix adding scale transformation * #783 measure performance * #783 reuse nvjpegenc elements to prevent memory leak * #783 use dealer-router sockets in source_adapter_with_json_metadata sample * #783 make EOS on picture resolution change configurable in savant_rs_video_demux * Revert "#783 use dealer-router sockets in source_adapter_with_json_metadata sample" This reverts commit 1754672. * #783 set EOS_ON_FRAME_PARAMS_CHANGE=true as default * #783 reuse nvjpegenc only on dGPU * #783 update doc * #783 fix default values * #783 measure performance * #783 fix default value for EOS_ON_FRAME_PARAMS_CHANGE * #783 log warning when removing nvjpegenc on jetson
insight-platform · Jul 9, 2024 · 7ed5faa · 7ed5faa
1 parent e1b156e
commit 7ed5faa
Show file tree

Hide file tree

Showing 13 changed files with 229 additions and 54 deletions.
diff --git a/adapters/gst/sources/media_files.sh b/adapters/gst/sources/media_files.sh
@@ -19,6 +19,7 @@ ZMQ_SOCKET_BIND="${ZMQ_BIND:="false"}"
 SYNC_OUTPUT="${SYNC_OUTPUT:="false"}"
 FRAMERATE="${FRAMERATE:="30/1"}"
 FPS_OUTPUT="${FPS_OUTPUT:="stdout"}"
+EOS_ON_FRAME_PARAMS_CHANGE="${EOS_ON_FRAME_PARAMS_CHANGE:="true"}"
 if [[ -n "${FPS_PERIOD_SECONDS}" ]]; then
     FPS_PERIOD="period-seconds=${FPS_PERIOD_SECONDS}"
 elif [[ -n "${FPS_PERIOD_FRAMES}" ]]; then
@@ -41,7 +42,7 @@ SINK_PROPERTIES=(
     source-id="${SOURCE_ID}"
     read-metadata="${READ_METADATA}"
     eos-on-file-end="${EOS_ON_FILE_END}"
-    eos-on-frame-params-change=true
+    eos-on-frame-params-change="${EOS_ON_FRAME_PARAMS_CHANGE}"
     socket="${ZMQ_ENDPOINT}"
     socket-type="${ZMQ_SOCKET_TYPE}"
     bind="${ZMQ_SOCKET_BIND}"

diff --git a/docs/performance.md b/docs/performance.md
@@ -22,6 +22,7 @@
 | [#550](https://github.com/insight-platform/Savant/issues/550)                   | 371.29 |           | 71.79     |
 | [#612](https://github.com/insight-platform/Savant/issues/612)                   | 376.48 |           | 71.95     |
 | [#641](https://github.com/insight-platform/Savant/issues/641)                   | 373.69 |           | 80.4      |
+| [#783](https://github.com/insight-platform/Savant/issues/783)                   | 349.08 |           | 79.80     |
 
 ### conditional_video_processing
 
@@ -52,6 +53,7 @@
 | [#550](https://github.com/insight-platform/Savant/issues/550) | 229.22 |           | 49.77     |
 | [#612](https://github.com/insight-platform/Savant/issues/612) | 229.51 |           | 50.04     |
 | [#641](https://github.com/insight-platform/Savant/issues/641) | 231.26 |           | 54.8      |
+| [#783](https://github.com/insight-platform/Savant/issues/783) | 220.24 |           | 54.48     |
 
 ### fisheye_line_crossing
 
@@ -60,13 +62,15 @@
 | [#193](https://github.com/insight-platform/Savant/issues/193) | 86.6  | 23.9      | 33.7      |
 | [#612](https://github.com/insight-platform/Savant/issues/612) | 86.6  |           | 33.71     |
 | [#641](https://github.com/insight-platform/Savant/issues/641) | 86.64 |           | 35.46     |
+| [#783](https://github.com/insight-platform/Savant/issues/783) | 86.64 |           | 35.48     |
 
 ### keypoint_detection
 
 | Savant ver.                                                   | A4000  | Xavier NX | Orin Nano |
 |---------------------------------------------------------------|--------|-----------|-----------|
 | [#641](https://github.com/insight-platform/Savant/issues/641) | 284.19 | 58.85     | 79.34     |
 | [#692](https://github.com/insight-platform/Savant/issues/692) | 758.62 |           | 202.09    |
+| [#783](https://github.com/insight-platform/Savant/issues/783) | 690.44 |           | 203.47    |
 
 ### intersection_traffic_meter (yolov8m)
 
@@ -78,6 +82,7 @@
 | [#550](https://github.com/insight-platform/Savant/issues/550) | 268.50 |           | 41.11     |
 | [#612](https://github.com/insight-platform/Savant/issues/612) | 271.13 |           | 41.11     |
 | [#641](https://github.com/insight-platform/Savant/issues/641) | 273.91 |           | 43.19     |
+| [#783](https://github.com/insight-platform/Savant/issues/783) | 265.54 |           | 43.10     |
 
 ### license_plate_recognition
 
@@ -90,6 +95,7 @@
 | [#612](https://github.com/insight-platform/Savant/issues/612) | 272.78 |           | 42.47     |
 | [#641](https://github.com/insight-platform/Savant/issues/641) | 276.78 |           |           |
 | [#705](https://github.com/insight-platform/Savant/issues/705) | 309.99 |           | 65.57     |
+| [#783](https://github.com/insight-platform/Savant/issues/783) | 311.85 |           | 65.53     |
 
 ### nvidia_car_classification
 
@@ -109,6 +115,7 @@
 | [#550](https://github.com/insight-platform/Savant/issues/550)                   | 519.29 |           | 144.69    |
 | [#612](https://github.com/insight-platform/Savant/issues/612)                   | 530.02 |           | 142.89    |
 | [#641](https://github.com/insight-platform/Savant/issues/641)                   | 502.43 |           | 141.95    |
+| [#783](https://github.com/insight-platform/Savant/issues/783)                   | 460.95 |           | 138.32    |
 
 ### opencv_cuda_bg_remover_mog2
 
@@ -128,6 +135,7 @@
 | [#550](https://github.com/insight-platform/Savant/issues/550)                   | 749.55 |           | 128.76    |
 | [#612](https://github.com/insight-platform/Savant/issues/612)                   | 748.87 |           | 126.41    |
 | [#641](https://github.com/insight-platform/Savant/issues/641)                   | 750.11 |           | 144.6     |
+| [#783](https://github.com/insight-platform/Savant/issues/783)                   | 708.36 |           | 140       |
 
 ### opencv_cuda_bg_remover_mog2 (multi-stream)
 
@@ -147,6 +155,7 @@
 | [#612](https://github.com/insight-platform/Savant/issues/612) | 62.78 |           | 10.07     |
 | [#641](https://github.com/insight-platform/Savant/issues/641) | 60.64 |           |           |
 | [#705](https://github.com/insight-platform/Savant/issues/705) | 59.66 |           | 10.51     |
+| [#783](https://github.com/insight-platform/Savant/issues/783) | 58.67 |           |           |
 
 ### peoplenet_detector
 
@@ -166,15 +175,17 @@
 | [#550](https://github.com/insight-platform/Savant/issues/550)                   | 416.32 |           | 117.34    |
 | [#612](https://github.com/insight-platform/Savant/issues/612)                   | 430.29 |           | 117.03    |
 | [#641](https://github.com/insight-platform/Savant/issues/641)                   | 430.91 |           | 111.75    |
+| [#783](https://github.com/insight-platform/Savant/issues/783)                   | 389.63 |           | 112.17    |
 
 ### RTDETR R50
 
 | Savant ver.                                                   | A4000  | Xavier NX | Orin Nano |
-|---------------------------------------------------------------|--------|--------|-----------|
-| [#558](https://github.com/insight-platform/Savant/issues/558) | 137.41 |        |           |
-| [#612](https://github.com/insight-platform/Savant/issues/612) | 134.47 |        |           |
-| [#641](https://github.com/insight-platform/Savant/issues/641) | 119.21 |        |           |
-| [#718](https://github.com/insight-platform/Savant/issues/718) | 119.21 |        | 25        |
+|---------------------------------------------------------------|--------|-----------|-----------|
+| [#558](https://github.com/insight-platform/Savant/issues/558) | 137.41 |           |           |
+| [#612](https://github.com/insight-platform/Savant/issues/612) | 134.47 |           |           |
+| [#641](https://github.com/insight-platform/Savant/issues/641) | 119.21 |           |           |
+| [#718](https://github.com/insight-platform/Savant/issues/718) | 119.21 |           | 25        |
+| [#783](https://github.com/insight-platform/Savant/issues/783) | 114.14 |           | 25.41     |
 
 ### traffic_meter (yolov8m)
 
@@ -194,6 +205,7 @@
 | [#550](https://github.com/insight-platform/Savant/issues/550)                   | 255.94 |           | 41.01     |
 | [#612](https://github.com/insight-platform/Savant/issues/612)                   | 261.18 |           | 41.03     |
 | [#641](https://github.com/insight-platform/Savant/issues/641)                   | 260.02 |           | 43.17     |
+| [#783](https://github.com/insight-platform/Savant/issues/783)                   | 255.72 |           | 43.15     |
 
 ### yolov8_seg
 
@@ -212,3 +224,4 @@ Note: `yolov8_seg` always has a queue length of 10.
 | [#550](https://github.com/insight-platform/Savant/issues/550) | 91.04 |           | 36.01     |
 | [#612](https://github.com/insight-platform/Savant/issues/612) | 91.65 |           | 36.11     |
 | [#641](https://github.com/insight-platform/Savant/issues/641) | 92.78 |           | 37.33     |
+| [#783](https://github.com/insight-platform/Savant/issues/783) | 90.71 |           | 37.38     |
diff --git a/docs/source/advanced_topics/0_dead_stream_eviction.rst b/docs/source/advanced_topics/0_dead_stream_eviction.rst
@@ -22,6 +22,6 @@ Take a look at the ``default.yml`` for details:
 
 .. literalinclude:: ../../../savant/config/default.yml
   :language: YAML
-  :lines: 172-200
+  :lines: 172-202
 
 You can override only required parameters in your module YAML configuration file. Also, take a look at corresponding environment variables helping to configure the parameters without specifying them in the module config.
diff --git a/docs/source/savant_101/10_adapters.rst b/docs/source/savant_101/10_adapters.rst
@@ -268,13 +268,20 @@ The images are served from:
 .. note::
     The adapter also can be used to implement asynchronous image processing pipelines. Metadata allows passing per-image identification information over the pipeline to access the results when those images are processed.
 
+.. note::
+    It is advisable to have all pictures in the same resolution to avoid issues with the pipeline. Sending images with different resolutions will degrade the performance of the pipeline. To avoid this there are few options:
+        1. Resize all images to the same resolution.
+        2. Set ``EOS_ON_FRAME_PARAMS_CHANGE=True`` in the adapter and ``pipeline.source.properties.eos_on_frame_resolution_change: true`` in module configuration. This should work on dGPU but won't work on Jetson.
+        3. Sort images by resolution. This will decrease the amount of generated EOS.
+
 **Parameters**:
 
 - ``FILE_TYPE``: a flag specifying that the adapter is used for images; it must always be set to ``image``;
 - ``LOCATION``: a filesystem location (path or directory) or HTTP URL from where images are served;
 - ``FRAMERATE``: a desired framerate for the output video stream generated from image files (the parameter is used only if ``SYNC_OUTPUT=True``);
 - ``SYNC_OUTPUT``: a flag indicating that images are delivered into a module as a video stream; otherwise, the files are sent as fast as the module is capable processing them; default is ``False``;
 - ``EOS_ON_FILE_END``: a flag configuring sending of ``EOS`` message after every image; the ``EOS`` message is important to trackers, helping them to reset tracking when a video stream is no longer continuous; default is ``False``;
+- ``EOS_ON_FRAME_PARAMS_CHANGE``: a flag configuring sending of ``EOS`` message after every change in image resolution; the ``EOS`` message is important to trackers, helping them to reset tracking when a video stream is no longer continuous; default is ``True``;
 - ``SORT_BY_TIME``: a flag specifying sorting by modification time (ascending); by default, it is ``False``, causing the files to be sorted lexicographically;
 - ``READ_METADATA``: a flag specifying the need to augment images with metadata from ``JSON`` files with the corresponding names as the source files; default is ``False``.
 
@@ -315,6 +322,7 @@ The video files are served from:
 - ``FILE_TYPE``: must be set to ``video``;
 - ``LOCATION``: a video file(s) location or URL;
 - ``EOS_ON_FILE_END``: a flag indicating whether to send the ``EOS`` message at the end of each file; default is ``True``; the ``EOS`` message is crucial for trackers to recognize when a video stream is no longer continuous; when sending ordered parts of a single video file without gaps usually must be set to ``False``;
+- ``EOS_ON_FRAME_PARAMS_CHANGE``: a flag indicating whether to send the ``EOS`` message after every change in video parameters (resolution, framerate); default is ``True``; the ``EOS`` message is crucial for trackers to recognize when a video stream is no longer continuous;
 - ``SYNC_OUTPUT``: flag specifying if to send frames synchronously (i.e. at the source file rate); default is ``False``;
 - ``SORT_BY_TIME``: a flag indicating whether files are sorted by modification time (ascending) before sending to a module; by default, it is ``False`` (lexicographical sorting);
 - ``READ_METADATA``: a flag specifying the need to augment video frames with metadata from ``JSON`` files with the corresponding names as the source files; default is ``False``.

diff --git a/gst_plugins/python/savant_rs_video_decode_bin.py b/gst_plugins/python/savant_rs_video_decode_bin.py
@@ -30,6 +30,7 @@
         'source-eviction-interval',
         'max-parallel-streams',
         'zeromq-reader',
+        'eos-on-frame-resolution-change',
     ]
 }
 SAVANT_RS_VIDEO_DECODE_BIN_PROPERTIES = {

diff --git a/gst_plugins/python/savant_rs_video_demux.py b/gst_plugins/python/savant_rs_video_demux.py
@@ -27,6 +27,7 @@
 
 DEFAULT_SOURCE_TIMEOUT = 60
 DEFAULT_SOURCE_EVICTION_INTERVAL = 15
+DEFAULT_EOS_ON_FRAME_RESOLUTION_CHANGE = True
 OUT_CAPS = Gst.Caps.from_string(';'.join(x.value.caps_with_params for x in Codec))
 
 SAVANT_RS_VIDEO_DEMUX_PROPERTIES = {
@@ -56,6 +57,13 @@
         False,
         GObject.ParamFlags.READWRITE,
     ),
+    'eos-on-frame-resolution-change': (
+        bool,
+        'Send EOS when frame resolution changed for JPEG and PNG codecs',
+        'Send EOS when frame resolution changed for JPEG and PNG codecs',
+        DEFAULT_EOS_ON_FRAME_RESOLUTION_CHANGE,
+        GObject.ParamFlags.READWRITE,
+    ),
     'max-parallel-streams': (
         int,
         'Maximum number of parallel streams.',
@@ -158,6 +166,7 @@ def __init__(self):
         super().__init__()
         self.sources: Dict[str, SourceInfo] = {}
         self.eos_on_timestamps_reset = False
+        self.eos_on_frame_resolution_change = DEFAULT_EOS_ON_FRAME_RESOLUTION_CHANGE
         self.source_timeout = DEFAULT_SOURCE_TIMEOUT
         self.source_eviction_interval = DEFAULT_SOURCE_EVICTION_INTERVAL
         self.last_eviction = 0
@@ -220,6 +229,8 @@ def do_get_property(self, prop):
             return self.source_eviction_interval
         if prop.name == 'eos-on-timestamps-reset':
             return self.eos_on_timestamps_reset
+        if prop.name == 'eos-on-frame-resolution-change':
+            return self.eos_on_frame_resolution_change
         if prop.name == 'max-parallel-streams':
             return self.max_parallel_streams
         if prop.name == 'pipeline':
@@ -238,6 +249,8 @@ def do_set_property(self, prop, value):
             self.source_eviction_interval = value
         elif prop.name == 'eos-on-timestamps-reset':
             self.eos_on_timestamps_reset = value
+        elif prop.name == 'eos-on-frame-resolution-change':
+            self.eos_on_frame_resolution_change = value
         elif prop.name == 'max-parallel-streams':
             self.max_parallel_streams = value
         elif prop.name == 'pipeline':
@@ -314,9 +327,8 @@ def handle_buffer(
             source_info.timestamp = time.time()
 
         with source_info.lock():
-            if (
-                source_info.src_pad is not None
-                and source_info.params != frame_info.params
+            if source_info.src_pad is not None and not self.frame_params_equal(
+                source_info.params, frame_info.params
             ):
                 self.update_frame_params(source_info, frame_info.params)
             if source_info.src_pad is not None:
@@ -510,22 +522,15 @@ def add_source(self, source_id: str, source_info: SourceInfo):
     def update_frame_params(self, source_info: SourceInfo, frame_params: FrameParams):
         """Handle changed frame parameters on a source."""
 
-        if source_info.params != frame_params:
-            self.logger.info(
-                'Frame parameters on pad %s was changed from %s to %s',
-                source_info.src_pad.get_name(),
-                source_info.params,
-                frame_params,
-            )
-            source_info.params = frame_params
-            self.remove_source(source_info, send_eos=True)
-            return
-
-        caps = build_caps(frame_params)
-        source_info.src_pad.push_event(Gst.Event.new_caps(caps))
         self.logger.info(
-            'Caps on pad %s changed to %s', source_info.src_pad, caps.to_string()
+            'Frame parameters on pad %s was changed from %s to %s',
+            source_info.src_pad.get_name(),
+            source_info.params,
+            frame_params,
         )
+        source_info.params = frame_params
+        self.remove_source(source_info, send_eos=True)
+        return
 
     def check_timestamps(self, source_info: SourceInfo, buffer: Gst.Buffer):
         """Check frame timestamps (PTS and DTS).
@@ -601,6 +606,15 @@ def eviction_loop(self):
         )
         time.sleep(self.source_eviction_interval)
 
+    def frame_params_equal(self, a: FrameParams, b: FrameParams) -> bool:
+        if self.eos_on_frame_resolution_change:
+            return a == b
+        if a.codec != b.codec:
+            return False
+        if a.codec in [Codec.JPEG, Codec.PNG]:
+            return True
+        return a == b
+
 
 # register plugin
 GObject.type_register(SavantRsVideoDemux)

diff --git a/savant/config/default.yml b/savant/config/default.yml
@@ -197,6 +197,8 @@ pipeline:
       decoder_queue_length: ${oc.decode:${oc.env:DECODER_QUEUE_LENGTH, null}}
       # Size of the queue before decoder in bytes (0 - no limit, default 10485760).
       decoder_queue_byte_size: ${oc.decode:${oc.env:DECODER_QUEUE_BYTE_SIZE, null}}
+      # Send EOS on frame resolution change for JPEG and PNG codecs (default true).
+      eos_on_frame_resolution_change: ${oc.decode:${oc.env:EOS_ON_FRAME_RESOLUTION_CHANGE, null}}
 
 # elements:
 # elements should be defined here

diff --git a/savant/deepstream/buffer_processor.py b/savant/deepstream/buffer_processor.py
@@ -173,11 +173,7 @@ def _prepare_input_frame(
 
         # full frame primary object by default
         source_info = self._sources.get_source(video_frame.source_id)
-        self._add_transformations(
-            frame_idx=frame_idx,
-            source_info=source_info,
-            video_frame=video_frame,
-        )
+        self._add_transformations(frame_idx=frame_idx, video_frame=video_frame)
         scale_factor_x = self._frame_params.width / source_info.src_resolution.width
         scale_factor_y = self._frame_params.height / source_info.src_resolution.height
         primary_bbox = BBox(
@@ -293,12 +289,7 @@ def _prepare_input_frame(
 
         nvds_frame_meta.bInferDone = True  # required for tracker (DS 6.0)
 
-    def _add_transformations(
-        self,
-        frame_idx: Optional[int],
-        source_info: SourceInfo,
-        video_frame: VideoFrame,
-    ):
+    def _add_transformations(self, frame_idx: Optional[int], video_frame: VideoFrame):
         if self._pass_through_mode:
             return
 
@@ -309,7 +300,7 @@ def _add_transformations(
             video_frame.pts,
         )
 
-        if source_info.add_scale_transformation:
+        if self._add_scale_transformation(video_frame):
             self.logger.debug(
                 'Adding scale transformation for frame of source %s '
                 'with IDX %s and PTS %s.',
@@ -329,6 +320,24 @@ def _add_transformations(
             )
             video_frame.add_transformation(self._padding_transformation)
 
+    def _add_scale_transformation(self, video_frame: VideoFrame) -> bool:
+        """Check if scale transformation is needed and add it to the frame."""
+
+        if not video_frame.transformations:
+            return True
+
+        last_transformation = video_frame.transformations[-1]
+        if last_transformation.is_scale:
+            last_transformation_size = last_transformation.as_scale
+        elif last_transformation.is_initial_size:
+            last_transformation_size = last_transformation.as_initial_size
+        elif last_transformation.is_resulting_size:
+            last_transformation_size = last_transformation.as_resulting_size
+        else:
+            last_transformation_size = 0, 0
+
+        return last_transformation_size != self._scale_transformation.as_scale
+
     def prepare_output(
         self,
         buffer: Gst.Buffer,

diff --git a/savant/deepstream/pipeline.py b/savant/deepstream/pipeline.py
@@ -453,10 +453,6 @@ def _on_source_caps(
             with self._source_adding_lock:
                 self._check_pipeline_is_running()
                 source_info.src_resolution = Resolution(width, height)
-                source_info.add_scale_transformation = (
-                    self._frame_params.width != width
-                    or self._frame_params.height != height
-                )
                 self._sources.update_source(source_info)
 
                 if not source_info.after_demuxer:
@@ -702,6 +698,7 @@ def _remove_output_elements(self, source_info: SourceInfo):
         )
 
         try:
+            self._source_output.remove_output(self._pipeline, source_info)
             for elem in source_info.after_demuxer:
                 self._logger.debug('Removing element %s', elem.get_name())
                 elem.set_locked_state(True)