vulkan: repeatedly remove modifieres until success

src/avhw.rs

@@ -103,33 +103,75 @@ impl AvHwDevCtx {
             hwframe_casted.height = height;
             hwframe_casted.initial_pool_size = 5;
 
+            let mut sts = -1;
             if self.fmt == Pixel::VULKAN {
                 #[cfg(feature = "experimental-vulkan")]
                 {
                     use ash::vk;
-                    use ffmpeg::ffi::AVVulkanFramesContext;
-
-                    let vk_ptr = hwframe_casted.hwctx as *mut AVVulkanFramesContext;
-
-                    (*vk_ptr).tiling = vk::ImageTiling::DRM_FORMAT_MODIFIER_EXT;
-
-                    let mut create_info = vk::ImageDrmFormatModifierListCreateInfoEXT {
-                        drm_format_modifier_count: modifiers.len() as u32,
-                        p_drm_format_modifiers: modifiers.as_ptr() as _,
-                        ..Default::default()
+                    use ffmpeg::ffi::{
+                        av_vkfmt_from_pixfmt, AVHWDeviceContext, AVVulkanDeviceContext,
+                        AVVulkanFramesContext,
                     };
-                    (*vk_ptr).create_pnext = &mut create_info as *mut _ as _;
+
+                    let av_devctx = &(*((*self.as_mut_ptr()).data as *mut AVHWDeviceContext));
+                    let vk_hwctx = &*(av_devctx.hwctx as *mut AVVulkanDeviceContext);
+
+                    let inst = ash::Instance::load(
+                        &ash::StaticFn {
+                            get_instance_proc_addr: vk_hwctx.get_proc_addr,
+                        },
+                        vk_hwctx.inst,
+                    );
+
+                    let mut modifiers_filtered: Vec<DrmModifier> = Vec::new();
+                    for modifier in modifiers {
+                        let mut drm_info = ash::vk::PhysicalDeviceImageDrmFormatModifierInfoEXT {
+                            drm_format_modifier: modifier.0,
+
+                            ..Default::default()
+                        };
+                        let mut image_format_prop = ash::vk::ImageFormatProperties2 {
+                            ..Default::default()
+                        };
+
+                        if let Ok(()) = inst.get_physical_device_image_format_properties2(
+                            vk_hwctx.phys_dev,
+                            &vk::PhysicalDeviceImageFormatInfo2 {
+                                format: *av_vkfmt_from_pixfmt(pixfmt.into()),
+                                p_next: &mut drm_info as *mut _ as _,
+                                ..Default::default()
+                            },
+                            &mut image_format_prop,
+                        ) {
+                            modifiers_filtered.push(*modifier);
+                        }
+                    }
+
+                    // some buffer requirements are complex, just start removing modifiers until it works
+                    while sts != 0 && !modifiers_filtered.is_empty() {
+                        let mut create_info = ash::vk::ImageDrmFormatModifierListCreateInfoEXT {
+                            drm_format_modifier_count: modifiers_filtered.len() as u32,
+                            p_drm_format_modifiers: modifiers_filtered.as_ptr() as _,
+                            ..Default::default()
+                        };
+
+                        let vk_ptr = hwframe_casted.hwctx as *mut AVVulkanFramesContext;
+
+                        (*vk_ptr).tiling = vk::ImageTiling::DRM_FORMAT_MODIFIER_EXT;
+                        (*vk_ptr).create_pnext = &mut create_info as *mut _ as _;
+
+                        sts = av_hwframe_ctx_init(hwframe);
+                        modifiers_filtered.pop();
+                    }
                 }
                 #[cfg(not(feature = "experimental-vulkan"))]
                 panic!("vulkan requested but built without vulkan support")
             } else {
                 if modifiers != &[DrmModifier::LINEAR] {
                     error!("unknown how to request non-linear frames in vaapi");
                 }
+                sts = av_hwframe_ctx_init(hwframe);
             }
-
-
-            let sts = av_hwframe_ctx_init(hwframe);
             if sts != 0 {
                 return Err(ffmpeg::Error::from(sts));
             }