feat(video): add async frame receive method

for as close to real-time frame processing

shard.yml

@@ -1,5 +1,5 @@
 name: ffmpeg
-version: 0.4.4
+version: 0.5.0
 
 dependencies:
   # we use stumpy to extract each pixels colour and draw bounding boxes

spec/spec_helper.cr

@@ -2,6 +2,10 @@ require "spec"
 require "http/client"
 require "../src/ffmpeg"
 
+Spec.before_suite do
+  ::Log.setup(:trace)
+end
+
 unless File.exists? "./test.mp4"
   puts "downloading video file..."
   HTTP::Client.get("https://test-videos.co.uk/vids/bigbuckbunny/mp4/h264/360/Big_Buck_Bunny_360_10s_5MB.mp4") do |response|

spec/video_spec.cr

@@ -49,6 +49,34 @@ module FFmpeg
       File.exists?("./output2.png").should be_true
     end
 
+    it "skips frames while processing images" do
+      video = Video.open(Path.new("./test.mp4"))
+
+      ready = Channel(Nil).new(1)
+      data = Channel(Tuple(StumpyCore::Canvas, Bool)).new(1)
+
+      spawn do
+        write_frame = 60
+        frame_count = 0
+
+        loop do
+          ready.send nil
+          frame, key_frame = data.receive
+          frame_count += 1
+          next if frame_count < write_frame
+          puts "writing async output"
+          StumpyPNG.write(frame, "./async_output.png")
+          break
+        end
+
+        ready.close
+        data.close
+      end
+
+      video.async_frames(ready, data)
+      File.exists?("./async_output.png").should be_true
+    end
+
     it "works with streams" do
       pending!("start a stream to test")
       video = Video.open URI.parse("udp://239.0.0.2:1234")

src/ffmpeg/video.cr

@@ -27,13 +27,7 @@ abstract class FFmpeg::Video
 
   getter format : Format = Format.new
 
-  # Grab each frame and convert it to a StumpyCore::Canvas for simple manipulation
-  # this can also scale the image to a preferred resolution
-  def each_frame(
-    output_width : Int? = nil,
-    output_height : Int? = nil,
-    scaling_method : ScalingAlgorithm = ScalingAlgorithm::Bicublin
-  )
+  protected def configure(output_width, output_height, scaling_method)
     configure_read
 
     Log.trace { "opening UDP stream input" }
@@ -59,35 +53,83 @@ abstract class FFmpeg::Video
     rgb_frame = Frame.new(output_width, output_height, 6)
     scaler = SWScale.new(codec, output_width, output_height, :rgb48Le, scaling_method)
     canvas = StumpyCore::Canvas.new(output_width, output_height)
+    cropped = StumpyCore::Canvas.new(desired_width, desired_height)
 
     # create a view into the frame buffer for simplified data extraction
     # works on the assumption that the code is running on a LE system
     pixel_components = output_width * output_height * 3
     pointer = Pointer(UInt16).new(rgb_frame.buffer.to_unsafe.address)
     frame_buffer = Slice.new(pointer, pixel_components)
 
+    {codec, scaler, rgb_frame, frame_buffer, stream_index, canvas, cropped, requires_cropping}
+  end
+
+  protected def scale_and_extract(scaler, frame, rgb_frame, canvas, frame_buffer, requires_cropping, cropped)
+    scaler.scale(frame, rgb_frame)
+
+    # copy frame into a stumpy canvas
+    canvas.pixels.size.times do |index|
+      idx = index * 3
+      r = frame_buffer[idx]
+      g = frame_buffer[idx + 1]
+      b = frame_buffer[idx + 2]
+      canvas.pixels[index] = StumpyCore::RGBA.new(r, g, b)
+    end
+
+    output = requires_cropping ? Video.crop(canvas, cropped) : canvas
+    {output, frame.key_frame?}
+  end
+
+  # Grab each frame and convert it to a StumpyCore::Canvas for simple manipulation
+  # this can also scale the image to a preferred resolution
+  def each_frame(
+    output_width : Int? = nil,
+    output_height : Int? = nil,
+    scaling_method : ScalingAlgorithm = ScalingAlgorithm::Bicublin
+  )
+    codec, scaler, rgb_frame, frame_buffer, stream_index, canvas, cropped, requires_cropping = configure(output_width, output_height, scaling_method)
+
     Log.trace { "extracting frames" }
     while !closed?
       format.read do |packet|
         if packet.stream_index == stream_index
           if frame = codec.decode(packet)
-            scaler.scale(frame, rgb_frame)
-
-            # copy frame into a stumpy canvas
-            canvas.pixels.size.times do |index|
-              idx = index * 3
-              r = frame_buffer[idx]
-              g = frame_buffer[idx + 1]
-              b = frame_buffer[idx + 2]
-              canvas.pixels[index] = StumpyCore::RGBA.new(r, g, b)
-            end
+            yield *scale_and_extract(scaler, frame, rgb_frame, canvas, frame_buffer, requires_cropping, cropped)
+          end
+        end
+      end
+    end
+  ensure
+    close
+    @format = Format.new
+    GC.collect
+  end
 
-            output = requires_cropping ? Video.crop(canvas, desired_width, desired_height) : canvas
-            yield output, frame.key_frame?
+  def async_frames(
+    ready : Channel(Nil),
+    data : Channel(Tuple(StumpyCore::Canvas, Bool)),
+    output_width : Int? = nil,
+    output_height : Int? = nil,
+    scaling_method : ScalingAlgorithm = ScalingAlgorithm::Bicublin
+  )
+    codec, scaler, rgb_frame, frame_buffer, stream_index, canvas, cropped, requires_cropping = configure(output_width, output_height, scaling_method)
+
+    Log.trace { "extracting frames" }
+    while !closed?
+      format.read do |packet|
+        if packet.stream_index == stream_index
+          if frame = codec.decode(packet)
+            select
+            when ready.receive
+              data.send scale_and_extract(scaler, frame, rgb_frame, canvas, frame_buffer, requires_cropping, cropped)
+            else
+              Log.trace { "skipping frame" }
+            end
           end
         end
       end
     end
+  rescue Channel::ClosedError
   ensure
     close
     @format = Format.new
@@ -122,9 +164,9 @@ abstract class FFmpeg::Video
 
   # one of desired_width or desired_height will match the canvas width or height
   # so we only have to crop width or height
-  def self.crop(canvas, desired_width, desired_height)
-    cropped = StumpyCore::Canvas.new(desired_width, desired_height)
-
+  def self.crop(canvas : StumpyCore::Canvas, cropped : StumpyCore::Canvas)
+    desired_width = cropped.width
+    desired_height = cropped.height
     x_start = {(canvas.width - desired_width) // 2, 0}.max
     y_start = {(canvas.height - desired_height) // 2, 0}.max