From 8cc84dd8d9ab965c367339bfa25703257fc972e3 Mon Sep 17 00:00:00 2001 From: Stephen von Takach Date: Sun, 2 Jul 2023 22:42:52 +1000 Subject: [PATCH] feat(video): add async frame receive method for as close to real-time frame processing --- shard.yml | 2 +- spec/spec_helper.cr | 4 +++ spec/video_spec.cr | 28 +++++++++++++++ src/ffmpeg/video.cr | 86 +++++++++++++++++++++++++++++++++------------ 4 files changed, 97 insertions(+), 23 deletions(-) diff --git a/shard.yml b/shard.yml index e2eda2e..ffcd549 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: ffmpeg -version: 0.4.4 +version: 0.5.0 dependencies: # we use stumpy to extract each pixels colour and draw bounding boxes diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index 62699c7..b38a3a2 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -2,6 +2,10 @@ require "spec" require "http/client" require "../src/ffmpeg" +Spec.before_suite do + ::Log.setup(:trace) +end + unless File.exists? "./test.mp4" puts "downloading video file..." HTTP::Client.get("https://test-videos.co.uk/vids/bigbuckbunny/mp4/h264/360/Big_Buck_Bunny_360_10s_5MB.mp4") do |response| diff --git a/spec/video_spec.cr b/spec/video_spec.cr index 3f578bf..4a13fd4 100644 --- a/spec/video_spec.cr +++ b/spec/video_spec.cr @@ -49,6 +49,34 @@ module FFmpeg File.exists?("./output2.png").should be_true end + it "skips frames while processing images" do + video = Video.open(Path.new("./test.mp4")) + + ready = Channel(Nil).new(1) + data = Channel(Tuple(StumpyCore::Canvas, Bool)).new(1) + + spawn do + write_frame = 60 + frame_count = 0 + + loop do + ready.send nil + frame, key_frame = data.receive + frame_count += 1 + next if frame_count < write_frame + puts "writing async output" + StumpyPNG.write(frame, "./async_output.png") + break + end + + ready.close + data.close + end + + video.async_frames(ready, data) + File.exists?("./async_output.png").should be_true + end + it "works with streams" do pending!("start a stream to test") video = Video.open URI.parse("udp://239.0.0.2:1234") diff --git a/src/ffmpeg/video.cr b/src/ffmpeg/video.cr index d38521f..818f043 100644 --- a/src/ffmpeg/video.cr +++ b/src/ffmpeg/video.cr @@ -27,13 +27,7 @@ abstract class FFmpeg::Video getter format : Format = Format.new - # Grab each frame and convert it to a StumpyCore::Canvas for simple manipulation - # this can also scale the image to a preferred resolution - def each_frame( - output_width : Int? = nil, - output_height : Int? = nil, - scaling_method : ScalingAlgorithm = ScalingAlgorithm::Bicublin - ) + protected def configure(output_width, output_height, scaling_method) configure_read Log.trace { "opening UDP stream input" } @@ -59,6 +53,7 @@ abstract class FFmpeg::Video rgb_frame = Frame.new(output_width, output_height, 6) scaler = SWScale.new(codec, output_width, output_height, :rgb48Le, scaling_method) canvas = StumpyCore::Canvas.new(output_width, output_height) + cropped = StumpyCore::Canvas.new(desired_width, desired_height) # create a view into the frame buffer for simplified data extraction # works on the assumption that the code is running on a LE system @@ -66,28 +61,75 @@ abstract class FFmpeg::Video pointer = Pointer(UInt16).new(rgb_frame.buffer.to_unsafe.address) frame_buffer = Slice.new(pointer, pixel_components) + {codec, scaler, rgb_frame, frame_buffer, stream_index, canvas, cropped, requires_cropping} + end + + protected def scale_and_extract(scaler, frame, rgb_frame, canvas, frame_buffer, requires_cropping, cropped) + scaler.scale(frame, rgb_frame) + + # copy frame into a stumpy canvas + canvas.pixels.size.times do |index| + idx = index * 3 + r = frame_buffer[idx] + g = frame_buffer[idx + 1] + b = frame_buffer[idx + 2] + canvas.pixels[index] = StumpyCore::RGBA.new(r, g, b) + end + + output = requires_cropping ? Video.crop(canvas, cropped) : canvas + {output, frame.key_frame?} + end + + # Grab each frame and convert it to a StumpyCore::Canvas for simple manipulation + # this can also scale the image to a preferred resolution + def each_frame( + output_width : Int? = nil, + output_height : Int? = nil, + scaling_method : ScalingAlgorithm = ScalingAlgorithm::Bicublin + ) + codec, scaler, rgb_frame, frame_buffer, stream_index, canvas, cropped, requires_cropping = configure(output_width, output_height, scaling_method) + Log.trace { "extracting frames" } while !closed? format.read do |packet| if packet.stream_index == stream_index if frame = codec.decode(packet) - scaler.scale(frame, rgb_frame) - - # copy frame into a stumpy canvas - canvas.pixels.size.times do |index| - idx = index * 3 - r = frame_buffer[idx] - g = frame_buffer[idx + 1] - b = frame_buffer[idx + 2] - canvas.pixels[index] = StumpyCore::RGBA.new(r, g, b) - end + yield *scale_and_extract(scaler, frame, rgb_frame, canvas, frame_buffer, requires_cropping, cropped) + end + end + end + end + ensure + close + @format = Format.new + GC.collect + end - output = requires_cropping ? Video.crop(canvas, desired_width, desired_height) : canvas - yield output, frame.key_frame? + def async_frames( + ready : Channel(Nil), + data : Channel(Tuple(StumpyCore::Canvas, Bool)), + output_width : Int? = nil, + output_height : Int? = nil, + scaling_method : ScalingAlgorithm = ScalingAlgorithm::Bicublin + ) + codec, scaler, rgb_frame, frame_buffer, stream_index, canvas, cropped, requires_cropping = configure(output_width, output_height, scaling_method) + + Log.trace { "extracting frames" } + while !closed? + format.read do |packet| + if packet.stream_index == stream_index + if frame = codec.decode(packet) + select + when ready.receive + data.send scale_and_extract(scaler, frame, rgb_frame, canvas, frame_buffer, requires_cropping, cropped) + else + Log.trace { "skipping frame" } + end end end end end + rescue Channel::ClosedError ensure close @format = Format.new @@ -122,9 +164,9 @@ abstract class FFmpeg::Video # one of desired_width or desired_height will match the canvas width or height # so we only have to crop width or height - def self.crop(canvas, desired_width, desired_height) - cropped = StumpyCore::Canvas.new(desired_width, desired_height) - + def self.crop(canvas : StumpyCore::Canvas, cropped : StumpyCore::Canvas) + desired_width = cropped.width + desired_height = cropped.height x_start = {(canvas.width - desired_width) // 2, 0}.max y_start = {(canvas.height - desired_height) // 2, 0}.max