From fcd38a8cdea22405c77f2025f66258e4a755a327 Mon Sep 17 00:00:00 2001 From: Daniel Almeida Date: Wed, 25 Oct 2023 13:51:43 -0300 Subject: [PATCH] video: add suport for VideoSessionParameters Add support for VideoSessionParameters. These are used to deliver stream metadata to the driver. --- examples/src/bin/video.rs | 18 +- vulkano/src/video.rs | 541 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 557 insertions(+), 2 deletions(-) diff --git a/examples/src/bin/video.rs b/examples/src/bin/video.rs index 4f85b7c378..7787cdb597 100644 --- a/examples/src/bin/video.rs +++ b/examples/src/bin/video.rs @@ -6,8 +6,10 @@ use vulkano::{ instance::{Instance, InstanceCreateFlags, InstanceCreateInfo}, video::{ CodecCapabilities, VideoDecodeCapabilityFlags, VideoDecodeH264PictureLayoutFlags, - VideoDecodeH264ProfileInfo, VideoFormatInfo, VideoProfileInfo, VideoProfileListInfo, - VideoSession, VideoSessionCreateInfo, + VideoDecodeH264ProfileInfo, VideoDecodeH264SessionParametersAddInfo, VideoFormatInfo, + VideoProfileInfo, VideoProfileListInfo, VideoSession, VideoSessionCreateInfo, + VideoSessionParameters, VideoSessionParametersCreateFlags, + VideoSessionParametersCreateInfo, }, VulkanLibrary, }; @@ -175,4 +177,16 @@ fn main() { let video_session = VideoSession::new(Arc::clone(&device), video_session_create_info).unwrap(); println!("video session: {:#?}", video_session); + + let video_session_parameters_create_info = VideoSessionParametersCreateInfo::new( + VideoSessionParametersCreateFlags::empty(), None, Arc::clone(&video_session), vulkano::video::VideoSessionParametersCreateInfoNext::VideoDecodeH264SessionParametersCreateInfo { max_std_sps_count: 0, max_std_pps_count: 0, parameter_add_info: Some(VideoDecodeH264SessionParametersAddInfo { + std_sp_ss: vec![], + std_pp_ss: vec![], + }) } + ); + + let empty_session_parameters = + VideoSessionParameters::new(Arc::clone(&device), video_session_parameters_create_info) + .unwrap(); + println!("empty session parameters: {:#?}", empty_session_parameters); } diff --git a/vulkano/src/video.rs b/vulkano/src/video.rs index df0240cf90..92624c3407 100644 --- a/vulkano/src/video.rs +++ b/vulkano/src/video.rs @@ -27,6 +27,8 @@ use crate::{ Version, VulkanError, VulkanObject, }; +pub use ash::vk::native::*; + vulkan_bitflags_enum! { #[non_exhaustive] @@ -781,3 +783,542 @@ impl Default for VideoSessionCreateInfo { } } } + +pub enum VideoSessionParametersCreateInfoNextVk { + VideoDecodeH264(ash::vk::VideoDecodeH264SessionParametersCreateInfoKHR), + VideoDecodeH265(ash::vk::VideoDecodeH265SessionParametersCreateInfoKHR), +} + +pub enum VideoSessionParametersAddInfoVk { + VideoDecodeH264(ash::vk::VideoDecodeH264SessionParametersAddInfoKHR), + VideoDecodeH265(ash::vk::VideoDecodeH265SessionParametersAddInfoKHR), +} + +#[derive(Debug)] +pub struct VideoSessionParameters { + handle: ash::vk::VideoSessionParametersKHR, + device: InstanceOwnedDebugWrapper>, + + pub create_info: VideoSessionParametersCreateInfo, + pub _ne: crate::NonExhaustive, +} + +impl VideoSessionParameters { + pub fn new( + device: Arc, + create_info: VideoSessionParametersCreateInfo, + ) -> Result, Validated> { + Self::validate_new(&device, &create_info)?; + + unsafe { Ok(Self::new_unchecked(device, create_info)?) } + } + + fn validate_new( + device: &Device, + create_info: &VideoSessionParametersCreateInfo, + ) -> Result<(), Box> { + if !device + .physical_device() + .supported_extensions() + .khr_video_queue + || device.physical_device().api_version() < Version::V1_3 + { + return Err(Box::new(ValidationError { + requires_one_of: RequiresOneOf(&[RequiresAllOf(&[ + Requires::DeviceExtension("khr_video_queue"), + // Requires::APIVersion(Version::V1_3), // ? + ])]), + ..Default::default() + })); + } + + create_info + .validate() + .map_err(|err| err.add_context("create_info"))?; + + Ok(()) + } + + unsafe fn new_unchecked( + device: Arc, + create_info: VideoSessionParametersCreateInfo, + ) -> Result, VulkanError> { + let mut video_decode_create_info_next = None; + let mut video_decode_parameter_add_info = None; + + let create_info_vk = create_info.clone().to_vulkan( + &mut video_decode_create_info_next, + &mut video_decode_parameter_add_info, + ); + + let handle = unsafe { + let fns = device.fns(); + let mut output = MaybeUninit::uninit(); + (fns.khr_video_queue.create_video_session_parameters_khr)( + device.handle(), + &create_info_vk, + std::ptr::null(), + output.as_mut_ptr(), + ) + .result() + .map_err(VulkanError::from)?; + output.assume_init() + }; + + Ok(Self::from_handle(handle, create_info, device)) + } + + /// Creates a new `VideoSessionParameters` from a raw object handle. + /// + /// # Safety + /// + /// - `handle` must be a valid Vulkan object handle created from `device`. + /// - `create_info` must match the info used to create the object. + pub unsafe fn from_handle( + handle: ash::vk::VideoSessionParametersKHR, + create_info: VideoSessionParametersCreateInfo, + device: Arc, + ) -> Arc { + Arc::new(VideoSessionParameters { + handle, + device: InstanceOwnedDebugWrapper(device), + create_info, + _ne: crate::NonExhaustive(()), + }) + } +} + +impl Drop for VideoSessionParameters { + fn drop(&mut self) { + unsafe { + let fns = self.device.fns(); + (fns.khr_video_queue.destroy_video_session_parameters_khr)( + self.device.handle(), + self.handle, + std::ptr::null(), + ); + } + } +} + +vulkan_bitflags! { + #[non_exhaustive] + + VideoSessionParametersCreateFlags = VideoSessionParametersCreateFlagsKHR(u32); +} + +#[derive(Clone, Debug)] +pub struct VideoH264SpsFlags(ash::vk::native::StdVideoH264SpsFlags); + +impl VideoH264SpsFlags { + pub fn new( + constraint_set0_flag: u32, + constraint_set1_flag: u32, + constraint_set2_flag: u32, + constraint_set3_flag: u32, + constraint_set4_flag: u32, + constraint_set5_flag: u32, + direct_8x8_inference_flag: u32, + mb_adaptive_frame_field_flag: u32, + frame_mbs_only_flag: u32, + delta_pic_order_always_zero_flag: u32, + separate_colour_plane_flag: u32, + gaps_in_frame_num_value_allowed_flag: u32, + qpprime_y_zero_transform_bypass_flag: u32, + frame_cropping_flag: u32, + seq_scaling_matrix_present_flag: u32, + vui_parameters_present_flag: u32, + ) -> Self { + let _bitfield_1 = ash::vk::native::StdVideoH264SpsFlags::new_bitfield_1( + constraint_set0_flag, + constraint_set1_flag, + constraint_set2_flag, + constraint_set3_flag, + constraint_set4_flag, + constraint_set5_flag, + direct_8x8_inference_flag, + mb_adaptive_frame_field_flag, + frame_mbs_only_flag, + delta_pic_order_always_zero_flag, + separate_colour_plane_flag, + gaps_in_frame_num_value_allowed_flag, + qpprime_y_zero_transform_bypass_flag, + frame_cropping_flag, + seq_scaling_matrix_present_flag, + vui_parameters_present_flag, + ); + + Self(ash::vk::native::StdVideoH264SpsFlags { + _bitfield_align_1: Default::default(), + _bitfield_1, + __bindgen_padding_0: Default::default(), + }) + } +} + +// Using u32 for now. This macro does not work if the path is not reexported +// into ash::vk +// ash::vk_bitflags_wrapped! { +// +// VideoH264ProfileIdc = StdVideoH264ProfileIdc(u32); +// +// VIDEO_H264_PROFILE_IDC_BASELINE = StdVideoH264ProfileIdc_STD_VIDEO_H264_PROFILE_IDC_BASELINE, +// VIDEO_H264_PROFILE_IDC_MAIN = StdVideoH264ProfileIdc_STD_VIDEO_H264_PROFILE_IDC_MAIN, +// VIDEO_H264_PROFILE_IDC_HIGH = StdVideoH264ProfileIdc_STD_VIDEO_H264_PROFILE_IDC_HIGH, +// VIDEO_H264_PROFILE_IDC_HIGH_444_PREDICTIVE = StdVideoH264ProfileIdc_STD_VIDEO_H264_PROFILE_IDC_HIGH_444_PREDICTIVE, +// VIDEO_H264_PROFILE_IDC_INVALID = StdVideoH264ProfileIdc_STD_VIDEO_H264_PROFILE_IDC_INVALID, +// } + +#[derive(Clone, Debug)] +pub struct VideoH264SequenceParameterSet { + pub flags: VideoH264SpsFlags, + pub profile_idc: StdVideoH264ProfileIdc, + pub level_idc: StdVideoH264LevelIdc, + pub chroma_format_idc: StdVideoH264ChromaFormatIdc, + pub seq_parameter_set_id: u8, + pub bit_depth_luma_minus8: u8, + pub bit_depth_chroma_minus8: u8, + pub log2_max_frame_num_minus4: u8, + pub pic_order_cnt_type: StdVideoH264PocType, + pub offset_for_non_ref_pic: i32, + pub offset_for_top_to_bottom_field: i32, + pub log2_max_pic_order_cnt_lsb_minus4: u8, + pub num_ref_frames_in_pic_order_cnt_cycle: u8, + pub max_num_ref_frames: u8, + pub pic_width_in_mbs_minus1: u32, + pub pic_height_in_map_units_minus1: u32, + pub frame_crop_left_offset: u32, + pub frame_crop_right_offset: u32, + pub frame_crop_top_offset: u32, + pub frame_crop_bottom_offset: u32, + pub offset_for_ref_frame: [i32; 255], +} + +#[derive(Clone, Debug)] +pub struct VideoH264PpsFlags(ash::vk::native::StdVideoH264PpsFlags); + +impl VideoH264PpsFlags { + pub fn new( + transform_8x8_mode_flag: u32, + redundant_pic_cnt_present_flag: u32, + constrained_intra_pred_flag: u32, + deblocking_filter_control_present_flag: u32, + weighted_pred_flag: u32, + bottom_field_pic_order_in_frame_present_flag: u32, + entropy_coding_mode_flag: u32, + pic_scaling_matrix_present_flag: u32, + ) -> Self { + let _bitfield_1 = ash::vk::native::StdVideoH264PpsFlags::new_bitfield_1( + transform_8x8_mode_flag, + redundant_pic_cnt_present_flag, + constrained_intra_pred_flag, + deblocking_filter_control_present_flag, + weighted_pred_flag, + bottom_field_pic_order_in_frame_present_flag, + entropy_coding_mode_flag, + pic_scaling_matrix_present_flag, + ); + + Self(ash::vk::native::StdVideoH264PpsFlags { + _bitfield_align_1: Default::default(), + _bitfield_1, + __bindgen_padding_0: Default::default(), + }) + } +} + +#[derive(Clone, Debug)] +pub struct VideoH264ScalingLists { + pub scaling_list_present_mask: u16, + pub use_default_scaling_matrix_mask: u16, + pub scaling_list_4x4: [[u8; 16usize]; 6usize], + pub scaling_list_8x8: [[u8; 64usize]; 6usize], +} + +#[derive(Clone, Debug)] +pub struct VideoH264PictureParameterSet { + pub flags: StdVideoH264PpsFlags, + pub seq_parameter_set_id: u8, + pub pic_parameter_set_id: u8, + pub num_ref_idx_l0_default_active_minus1: u8, + pub num_ref_idx_l1_default_active_minus1: u8, + pub weighted_bipred_idc: StdVideoH264WeightedBipredIdc, + pub pic_init_qp_minus26: i8, + pub pic_init_qs_minus26: i8, + pub chroma_qp_index_offset: i8, + pub second_chroma_qp_index_offset: i8, + pub scaling_lists: VideoH264ScalingLists, +} + +#[derive(Clone, Debug)] +pub struct VideoDecodeH264SessionParametersAddInfo { + pub std_sp_ss: Vec, + pub std_pp_ss: Vec, +} + +#[derive(Clone, Debug)] +pub enum VideoSessionParametersCreateInfoNext { + VideoDecodeH264SessionParametersCreateInfo { + max_std_sps_count: u32, + max_std_pps_count: u32, + parameter_add_info: Option, + }, + VideoDecodeH265SessionParametersCreateInfo {/* TODO */}, +} + +#[derive(Clone, Debug)] +pub struct VideoSessionParametersCreateInfo { + pub flags: VideoSessionParametersCreateFlags, + pub video_session_parameters_template: Option>, + pub video_session: Arc, + pub next: VideoSessionParametersCreateInfoNext, + pub _ne: crate::NonExhaustive, +} + +impl VideoSessionParametersCreateInfo { + #[inline] + pub fn new( + flags: VideoSessionParametersCreateFlags, + video_session_parameters_template: Option>, + video_session: Arc, + next: VideoSessionParametersCreateInfoNext, + ) -> Self { + Self { + flags, + video_session_parameters_template, + video_session: Arc::clone(&video_session), + next, + _ne: crate::NonExhaustive(()), + } + } + + /// Compute "spsAddList" + fn sps_add_list(&self) -> usize { + let mut seq_parameter_ids = vec![]; + match &self.next { + VideoSessionParametersCreateInfoNext::VideoDecodeH264SessionParametersCreateInfo { + parameter_add_info, + .. + } => { + if let Some(parameter_add_info) = parameter_add_info { + seq_parameter_ids.extend( + parameter_add_info + .std_sp_ss + .iter() + .map(|sps| sps.seq_parameter_set_id), + ); + } + } + _ => todo!(), + } + + if let Some(video_session_parameters_template) = &self.video_session_parameters_template { + match &video_session_parameters_template.create_info.next { + VideoSessionParametersCreateInfoNext::VideoDecodeH264SessionParametersCreateInfo { parameter_add_info: Some(template_parameter_add_info), .. } => { + for sps in &template_parameter_add_info.std_sp_ss { + if !seq_parameter_ids.contains(&sps.seq_parameter_set_id) { + seq_parameter_ids.push(sps.seq_parameter_set_id); + } + } + } + _ => todo!(), + } + } + + seq_parameter_ids.len() + } + + fn pps_add_list(&self) -> usize { + let mut seq_parameter_ids = vec![]; + let mut pic_parameter_ids = vec![]; + match &self.next { + VideoSessionParametersCreateInfoNext::VideoDecodeH264SessionParametersCreateInfo { + parameter_add_info, + .. + } => { + if let Some(parameter_add_info) = parameter_add_info { + pic_parameter_ids.extend(parameter_add_info.std_pp_ss.iter().map(|pps| { + seq_parameter_ids.push(pps.seq_parameter_set_id); + pps.pic_parameter_set_id + })); + } + } + _ => todo!(), + } + + if let Some(video_session_parameters_template) = &self.video_session_parameters_template { + match &video_session_parameters_template.create_info.next { + VideoSessionParametersCreateInfoNext::VideoDecodeH264SessionParametersCreateInfo { parameter_add_info: Some(template_parameter_add_info), .. } => { + for pps in &template_parameter_add_info.std_pp_ss { + if !pic_parameter_ids.contains(&pps.pic_parameter_set_id) && !seq_parameter_ids.contains(&pps.seq_parameter_set_id) { + pic_parameter_ids.push(pps.pic_parameter_set_id); + seq_parameter_ids.push(pps.seq_parameter_set_id); + + } + } + } + _ => todo!(), + } + } + + pic_parameter_ids.len() + } + + pub(crate) fn validate(&self) -> Result<(), Box> { + let Self { + video_session_parameters_template, + video_session, + .. + } = self; + + if let Some(video_session_parameters_template) = video_session_parameters_template { + if video_session.handle + != video_session_parameters_template + .create_info + .video_session + .handle + { + return Err(Box::new(ValidationError { + context: "video_session_parameter_templte".into(), + problem: " if videoSessionParametersTemplate represents a valid handle, it must have been created against videoSession" + .into(), + vuids: &[" VUID-VkVideoSessionParametersCreateInfoKHR-videoSessionParametersTemplate-04855"], + ..Default::default() + })); + } + } + + match video_session + .create_info + .video_profile + .video_codec_operation + { + VideoCodecOperation::DecodeH264 => { + if let VideoSessionParametersCreateInfoNext::VideoDecodeH264SessionParametersCreateInfo { max_std_sps_count, max_std_pps_count, ..} = &self.next { + let sps_add_list = self.sps_add_list(); + if sps_add_list as u32 > *max_std_sps_count { + return Err(Box::new(ValidationError { + context: "spsAddList".into(), + problem: "must be less than or equal to the maxStdSPSCount".into(), + vuids: &["VUID-VkVideoSessionParametersCreateInfoKHR-videoSession-07203"], + ..Default::default() + })); + + } + + let pps_add_list = self.pps_add_list(); + if pps_add_list as u32 > *max_std_pps_count { + return Err(Box::new(ValidationError { + context: "ppsAddList".into(), + problem: "must be less than or equal to the maxStdPPSCount".into(), + vuids: &["VUID-VkVideoSessionParametersCreateInfoKHR-videoSession-07205"], + ..Default::default() + })); + } + } else { + return Err(Box::new(ValidationError { + context: "next".into(), + problem: "must be `VideoSessionParametersCreateInfoNext::VideoDecodeH264SessionParametersCreateInfo`".into(), + vuids: &["VUID-VkVideoSessionParametersCreateInfoKHR-videoSession-07203"], + ..Default::default() + })); + } + } + VideoCodecOperation::DecodeH265 => { + if !matches!(self.next, VideoSessionParametersCreateInfoNext::VideoDecodeH265SessionParametersCreateInfo{}) { + return Err(Box::new(ValidationError { + context: "next".into(), + problem: "must be `VideoSessionParametersCreateInfoNext::VideoDecodeH265SessionParametersCreateInfo`".into(), + vuids: &["VUID-VkVideoSessionParametersCreateInfoKHR-videoSession-07203"], + ..Default::default() + })); + } + } + } + + Ok(()) + } + + pub(crate) unsafe fn to_vulkan( + self, + video_decode_create_info_next: &mut Option, + video_session_parameter_add_info: &mut Option, + ) -> ash::vk::VideoSessionParametersCreateInfoKHR { + let mut video_session_parameters_create_info_vk = + ash::vk::VideoSessionParametersCreateInfoKHR { + flags: self.flags.into(), + p_next: std::ptr::null(), + video_session_parameters_template: self + .video_session_parameters_template + .map(|v| v.handle) + .unwrap_or(ash::vk::VideoSessionParametersKHR::null()), + video_session: self.video_session.handle, + ..Default::default() + }; + + match self + .video_session + .create_info + .video_profile + .video_codec_operation + { + VideoCodecOperation::DecodeH264 => { + let (max_sps_count, max_pps_count, parameter_add_info) = match self.next { + VideoSessionParametersCreateInfoNext::VideoDecodeH264SessionParametersCreateInfo { max_std_sps_count, max_std_pps_count, parameter_add_info: Some(parameter_add_info) } => (max_std_sps_count, max_std_pps_count, parameter_add_info), + _ => panic!(), + }; + + let mut video_decode_h264_session_parameters_create_info_vk = + ash::vk::VideoDecodeH264SessionParametersCreateInfoKHR { + max_std_sps_count: max_sps_count, + max_std_pps_count: max_pps_count, + p_next: std::ptr::null(), + ..Default::default() + }; + + let video_decode_h264_session_parameters_add_info_vk = + ash::vk::VideoDecodeH264SessionParametersAddInfoKHR { + std_sps_count: parameter_add_info.std_sp_ss.len() as _, + p_std_sp_ss: parameter_add_info.std_sp_ss.as_ptr() as _, + std_pps_count: parameter_add_info.std_pp_ss.len() as _, + p_std_pp_ss: parameter_add_info.std_pp_ss.as_ptr() as _, + ..Default::default() + }; + + let video_session_parameters_add_info_vk = video_session_parameter_add_info.insert( + VideoSessionParametersAddInfoVk::VideoDecodeH264( + video_decode_h264_session_parameters_add_info_vk, + ), + ); + + let video_session_parameters_add_info_vk = + match video_session_parameters_add_info_vk { + VideoSessionParametersAddInfoVk::VideoDecodeH264(v) => v, + _ => panic!(), + }; + + video_decode_h264_session_parameters_create_info_vk.p_parameters_add_info = + video_session_parameters_add_info_vk as *mut _ as _; + + let video_decode_h264_session_parameters_create_info_vk = + video_decode_create_info_next.insert( + VideoSessionParametersCreateInfoNextVk::VideoDecodeH264( + video_decode_h264_session_parameters_create_info_vk, + ), + ); + + let video_decode_h264_session_parameters_create_info_vk = + match video_decode_h264_session_parameters_create_info_vk { + VideoSessionParametersCreateInfoNextVk::VideoDecodeH264(v) => v, + _ => panic!(), + }; + + video_session_parameters_create_info_vk.p_next = + video_decode_h264_session_parameters_create_info_vk as *mut _ as _; + } + VideoCodecOperation::DecodeH265 => todo!(), + } + + video_session_parameters_create_info_vk + } +}