From 7643ce143bbeef2088715e6e7ffe2a4aeeb64b80 Mon Sep 17 00:00:00 2001 From: Hauke Strasdat Date: Wed, 26 Jun 2024 22:54:35 -0700 Subject: [PATCH] feat(simple-viewer): adjust view-port correctly when scaling window --- Cargo.toml | 2 +- crates/sophus/examples/viewer_ex.rs | 59 +++- crates/sophus/src/lib.rs | 2 - crates/sophus_sensor/src/distortion_table.rs | 153 +++++++-- crates/sophus_sensor/src/dyn_camera.rs | 150 +++------ crates/sophus_viewer/src/actor.rs | 2 +- crates/sophus_viewer/src/interactions.rs | 2 +- .../src/interactions/orbit_interaction.rs | 52 ++- crates/sophus_viewer/src/offscreen.rs | 209 +++++++++--- crates/sophus_viewer/src/pixel_renderer.rs | 14 +- crates/sophus_viewer/src/renderables/frame.rs | 2 - crates/sophus_viewer/src/scene_renderer.rs | 38 ++- .../src/scene_renderer/buffers.rs | 55 +-- .../src/scene_renderer/depth_renderer.rs | 60 ---- .../src/scene_renderer/utils.wgsl | 59 ++-- crates/sophus_viewer/src/simple_viewer.rs | 316 +++++++++--------- crates/sophus_viewer/src/views.rs | 14 - .../sophus_viewer/src/views/aspect_ratio.rs | 35 +- crates/sophus_viewer/src/views/view2d.rs | 47 +-- crates/sophus_viewer/src/views/view3d.rs | 45 ++- 20 files changed, 736 insertions(+), 580 deletions(-) delete mode 100644 crates/sophus_viewer/src/scene_renderer/depth_renderer.rs diff --git a/Cargo.toml b/Cargo.toml index 7ad52eb..691aae0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,7 +53,7 @@ image = {version = "0.25", features = [ ]} linked-hash-map = "0.5" log = "0.4" -nalgebra = {version = "0.32", features = ["rand"]} +nalgebra = {version = "0.33", features = ["rand"]} ndarray = {version = "0.15", features = ["approx-0_5"]} num-traits = "0.2" numpy = "0.21" diff --git a/crates/sophus/examples/viewer_ex.rs b/crates/sophus/examples/viewer_ex.rs index 2814449..3f8cd49 100644 --- a/crates/sophus/examples/viewer_ex.rs +++ b/crates/sophus/examples/viewer_ex.rs @@ -8,7 +8,14 @@ use sophus::viewer::actor::ViewerBuilder; use sophus::viewer::actor::ViewerCamera; use sophus::viewer::actor::ViewerConfig; use sophus::viewer::renderables::*; +use sophus_core::linalg::VecF64; +use sophus_image::intensity_image::intensity_arc_image::IsIntensityArcImage; +use sophus_image::mut_image::MutImageF32; +use sophus_image::mut_image_view::IsMutImageView; +use sophus_lie::traits::IsTranslationProductGroup; use sophus_lie::Isometry3; +use sophus_sensor::DynCamera; +use sophus_viewer::interactions::WgpuClippingPlanes; use sophus_viewer::renderables::color::Color; use sophus_viewer::renderables::renderable2d::Points2; use sophus_viewer::renderables::renderable2d::Renderable2d; @@ -96,8 +103,11 @@ pub struct ContentGeneratorOutbound { pub packets: OutboundChannel>, } -fn create_view2_packet(image_size: ImageSize) -> Packet { - let img = make_example_image(image_size); +fn create_view2_packet() -> Packet { + let img = make_example_image(ImageSize { + width: 300, + height: 100, + }); let mut packet_2d = View2dPacket { view_label: "view_2d".to_owned(), @@ -109,18 +119,47 @@ fn create_view2_packet(image_size: ImageSize) -> Packet { let points2 = Points2::make("points2", &[[16.0, 12.0], [32.0, 24.0]], &Color::red(), 5.0); packet_2d.renderables2d.push(Renderable2d::Points2(points2)); + let lines2 = Lines2::make("lines2", &[[[0.0, 0.0], [20.0, 20.0]]], &Color::red(), 5.0); + packet_2d.renderables2d.push(Renderable2d::Lines2(lines2)); + + Packet::View2d(packet_2d) +} + +fn create_tiny_view2_packet() -> Packet { + let mut img = MutImageF32::from_image_size_and_val(ImageSize::new(3, 2), 1.0); + + *img.mut_pixel(0, 0) = 0.0; + *img.mut_pixel(0, 1) = 0.5; + + *img.mut_pixel(1, 1) = 0.0; + + *img.mut_pixel(2, 0) = 0.3; + *img.mut_pixel(2, 1) = 0.6; + + let mut packet_2d = View2dPacket { + view_label: "tiny".to_owned(), + renderables3d: vec![], + renderables2d: vec![], + frame: Some(Frame::from_image(&img.to_shared().to_rgba())), + }; + let lines2 = Lines2::make( "lines2", - &[[[0.0, 0.0], [100.0, 100.0]]], + &[[[-0.5, -0.5], [0.5, 0.5]], [[0.5, -0.5], [-0.5, 0.5]]], &Color::red(), - 5.0, + 0.3, ); packet_2d.renderables2d.push(Renderable2d::Lines2(lines2)); Packet::View2d(packet_2d) } -fn create_view3_packet(initial_camera: ViewerCamera) -> Packet { +fn create_view3_packet() -> Packet { + let initial_camera = ViewerCamera { + intrinsics: DynCamera::default_distorted(ImageSize::new(639, 477)), + clipping_planes: WgpuClippingPlanes::default(), + scene_from_camera: Isometry3::from_t(&VecF64::<3>::new(0.0, 0.0, -5.0)), + }; let mut packet_3d = View3dPacket { view_label: "view_3d".to_owned(), renderables3d: vec![], @@ -164,16 +203,12 @@ impl HasOnMessage for ContentGeneratorMessage { ) { match &self { ContentGeneratorMessage::ClockTick(_time_in_seconds) => { - let initial_camera = ViewerCamera::default(); - let mut packets = Packets { packets: vec![] }; if state.counter == 0 { - packets - .packets - .push(create_view2_packet(initial_camera.intrinsics.image_size())); - - packets.packets.push(create_view3_packet(initial_camera)); + packets.packets.push(create_view2_packet()); + packets.packets.push(create_tiny_view2_packet()); + packets.packets.push(create_view3_packet()); } state.counter += 1; diff --git a/crates/sophus/src/lib.rs b/crates/sophus/src/lib.rs index ccb7f72..61c7c97 100644 --- a/crates/sophus/src/lib.rs +++ b/crates/sophus/src/lib.rs @@ -1,9 +1,7 @@ #![cfg_attr(feature = "simd", feature(portable_simd))] #![allow(clippy::needless_range_loop)] - #![doc = include_str!(concat!("../", std::env!("CARGO_PKG_README")))] - pub mod examples; #[doc(inline)] diff --git a/crates/sophus_sensor/src/distortion_table.rs b/crates/sophus_sensor/src/distortion_table.rs index 074efa6..22cd843 100644 --- a/crates/sophus_sensor/src/distortion_table.rs +++ b/crates/sophus_sensor/src/distortion_table.rs @@ -43,62 +43,48 @@ impl DistortTable { } } -/// Returns the undistortion lookup table -pub fn undistort_table(cam: &DynCamera) -> MutImage2F32 { - let mut table = MutImage2F32::from_image_size(cam.image_size()); - let w = cam.image_size().width; - let h = cam.image_size().height; - for v in 0..h { - for u in 0..w { - let pixel = cam.undistort(&VecF64::<2>::from_f64_array([u as f64, v as f64])); - *table.mut_pixel(u, v) = pixel.cast(); - } - } - table -} - /// Returns the distortion lookup table pub fn distort_table(cam: &DynCamera) -> DistortTable { - // first we find min and max values in the proj plane - // just test the 4 corners might not be enough - // so we will test the image boundary + // First we find min and max values in the proj plane. + // Just test the 4 corners might not be enough, so we will test the image boundary. let mut region = Region::<2>::empty(); - let w = cam.image_size().width; - let h = cam.image_size().height; + let v_top = -0.5; + let v_bottom = cam.image_size().height as f64 - 0.5; + + let u_left = -0.5; + let u_right = cam.image_size().width as f64 - 0.5; - for u in 0..cam.image_size().width { + for i in 0..=cam.image_size().width { + let u = i as f64 - 0.5; // top border - let v = 0; - let point_in_proj = cam.undistort(&VecF64::<2>::from_f64_array([u as f64, v as f64])); - region.extend(&point_in_proj); + region.extend(&cam.undistort(&VecF64::<2>::new(u, v_top))); // bottom border - let v = cam.image_size().height - 1; - let point_in_proj = cam.undistort(&VecF64::<2>::new(u as f64, v as f64)); - region.extend(&point_in_proj); + region.extend(&cam.undistort(&VecF64::<2>::new(u, v_bottom))); } - for v in 0..cam.image_size().height { + for i in 0..=cam.image_size().height { + let v = i as f64 - 0.5; // left border - let u = 0; - let point_in_proj = cam.undistort(&VecF64::<2>::new(u as f64, v as f64)); - region.extend(&point_in_proj); + region.extend(&cam.undistort(&VecF64::<2>::new(u_left, v))); // right border - let u = cam.image_size().width - 1; - let point_in_proj = cam.undistort(&VecF64::<2>::new(u as f64, v as f64)); - region.extend(&point_in_proj); + region.extend(&cam.undistort(&VecF64::<2>::new(u_right, v))); } - let region = Region::<2>::from_min_max(region.min().cast() * 2.0, region.max().cast() * 2.0); + + let mid = region.mid(); + let range = region.range(); + + let larger_region = Region::<2>::from_min_max(mid - range, mid + range); let mut distort_table = DistortTable { table: ArcImage2F32::from_image_size_and_val(cam.image_size(), SVector::::zeros()), - region, + region: larger_region, }; let mut table = MutImage2F32::from_image_size(cam.image_size()); - for v in 0..h { - for u in 0..w { + for v in 0..cam.image_size().height { + for u in 0..cam.image_size().width { let point_proj = VecF64::<2>::new( distort_table.offset().x + (u as f64) * distort_table.incr().x, distort_table.offset().y + (v as f64) * distort_table.incr().y, @@ -110,3 +96,96 @@ pub fn distort_table(cam: &DynCamera) -> DistortTable { distort_table.table = table.into(); distort_table } + +#[test] +fn camera_distortion_table_tests() { + use crate::distortion_table::distort_table; + use crate::distortion_table::DistortTable; + use approx::assert_abs_diff_eq; + use approx::assert_relative_eq; + use sophus_core::calculus::maps::vector_valued_maps::VectorValuedMapFromVector; + use sophus_core::linalg::VecF64; + use sophus_image::ImageSize; + + type DynCameraF64 = DynCamera; + + { + let mut cameras: Vec = vec![]; + cameras.push(DynCameraF64::new_pinhole( + &VecF64::<4>::new(600.0, 600.0, 1.0, 0.5), + ImageSize { + width: 3, + height: 2, + }, + )); + + cameras.push(DynCamera::new_kannala_brandt( + &VecF64::<8>::from_vec(vec![1000.0, 1000.0, 320.0, 280.0, 0.1, 0.01, 0.001, 0.0001]), + ImageSize { + width: 640, + height: 480, + }, + )); + + for camera in cameras { + let pixels_in_image = vec![ + VecF64::<2>::new(0.0, 0.0), + VecF64::<2>::new(2.0, 1.0), + VecF64::<2>::new(2.9, 1.9), + VecF64::<2>::new(2.5, 1.5), + VecF64::<2>::new(3.0, 2.0), + VecF64::<2>::new(-0.5, -0.5), + VecF64::<2>::new(1.0, 400.0), + VecF64::<2>::new(320.0, 240.0), + VecF64::<2>::new(319.5, 239.5), + VecF64::<2>::new(100.0, 40.0), + VecF64::<2>::new(639.0, 479.0), + ]; + + for pixel in pixels_in_image.clone() { + if pixel[0] > camera.image_size().width as f64 + || pixel[1] > camera.image_size().height as f64 + { + continue; + } + + for d in [1.0, 0.1, 0.5, 1.1, 3.0, 15.0] { + let point_in_camera = camera.cam_unproj_with_z(&pixel, d); + assert_relative_eq!(point_in_camera[2], d, epsilon = 1e-6); + + let pixel_in_image2 = camera.cam_proj(&point_in_camera); + assert_relative_eq!(pixel_in_image2, pixel, epsilon = 1e-6); + } + let ab_in_z1plane = camera.undistort(&pixel); + + let pixel_in_image3 = camera.distort(&ab_in_z1plane); + assert_relative_eq!(pixel_in_image3, pixel, epsilon = 1e-6); + + let dx = camera.dx_distort_x(&pixel); + let numeric_dx = VectorValuedMapFromVector::static_sym_diff_quotient( + |x: VecF64<2>| camera.distort(&x), + pixel, + 1e-6, + ); + + assert_relative_eq!(dx, numeric_dx, epsilon = 1e-4); + } + + let table: DistortTable = distort_table(&camera); + + for pixel in pixels_in_image { + if pixel[0] >= camera.image_size().width as f64 + || pixel[1] >= camera.image_size().height as f64 + { + continue; + } + let proj = camera.undistort(&pixel); + let analytic_pixel = camera.distort(&proj); + let lut_pixel = table.lookup(&proj); + + assert_abs_diff_eq!(analytic_pixel, pixel, epsilon = 1e-3); + assert_abs_diff_eq!(analytic_pixel, lut_pixel, epsilon = 1e-3); + } + } + } +} diff --git a/crates/sophus_sensor/src/dyn_camera.rs b/crates/sophus_sensor/src/dyn_camera.rs index d946c11..5864c82 100644 --- a/crates/sophus_sensor/src/dyn_camera.rs +++ b/crates/sophus_sensor/src/dyn_camera.rs @@ -24,7 +24,7 @@ pub type DynCamera = impl, const BATCH: usize, CameraType: IsCameraEnum> DynCameraFacade { - /// Create default pnhole from Image Size + /// Create default pinhole from Image Size pub fn default_pinhole(image_size: ImageSize) -> Self { let w = image_size.width as f64; let h = image_size.height as f64; @@ -44,6 +44,30 @@ impl, const BATCH: usize, CameraType: IsCameraEnum> } } + /// Create default distorted from Image Size + pub fn default_distorted(image_size: ImageSize) -> Self { + let w = image_size.width as f64; + let h = image_size.height as f64; + + let focal_length = (w + h) * 0.5; + Self { + camera_type: CameraType::new_kannala_brandt( + &S::Vector::<8>::from_f64_array([ + focal_length, + focal_length, + 0.5 * w - 0.5, + 0.5 * h - 0.5, + 2.0, + 0.0, + 0.0, + 0.0, + ]), + image_size, + ), + phantom: std::marker::PhantomData, + } + } + /// Create a new dynamic camera facade from a camera model pub fn from_model(camera_type: CameraType) -> Self { Self { @@ -107,16 +131,9 @@ impl, const BATCH: usize> DynCamera { #[test] fn dyn_camera_tests() { - use crate::distortion_table::distort_table; - use crate::distortion_table::undistort_table; - use crate::distortion_table::DistortTable; - use approx::assert_abs_diff_eq; use approx::assert_relative_eq; use sophus_core::calculus::maps::vector_valued_maps::VectorValuedMapFromVector; use sophus_core::linalg::VecF64; - use sophus_image::image_view::IsImageView; - use sophus_image::interpolation::interpolate; - use sophus_image::mut_image::MutImage2F32; use sophus_image::ImageSize; type DynCameraF64 = DynCamera; @@ -124,10 +141,10 @@ fn dyn_camera_tests() { { let mut cameras: Vec = vec![]; cameras.push(DynCameraF64::new_pinhole( - &VecF64::<4>::new(600.0, 600.0, 319.5, 239.5), + &VecF64::<4>::new(600.0, 600.0, 1.0, 0.5), ImageSize { - width: 640, - height: 480, + width: 3, + height: 2, }, )); @@ -142,6 +159,11 @@ fn dyn_camera_tests() { for camera in cameras { let pixels_in_image = vec![ VecF64::<2>::new(0.0, 0.0), + VecF64::<2>::new(2.0, 1.0), + VecF64::<2>::new(2.9, 1.9), + VecF64::<2>::new(2.5, 1.5), + VecF64::<2>::new(3.0, 2.0), + VecF64::<2>::new(-0.5, -0.5), VecF64::<2>::new(1.0, 400.0), VecF64::<2>::new(320.0, 240.0), VecF64::<2>::new(319.5, 239.5), @@ -149,9 +171,15 @@ fn dyn_camera_tests() { VecF64::<2>::new(639.0, 479.0), ]; - let table: MutImage2F32 = undistort_table(&camera); - for pixel in pixels_in_image.clone() { + if pixel[0] > camera.image_size().width as f64 + || pixel[1] > camera.image_size().height as f64 + { + continue; + } + + println!("pixel: {:?}", pixel); + for d in [1.0, 0.1, 0.5, 1.1, 3.0, 15.0] { let point_in_camera = camera.cam_unproj_with_z(&pixel, d); assert_relative_eq!(point_in_camera[2], d, epsilon = 1e-6); @@ -160,9 +188,6 @@ fn dyn_camera_tests() { assert_relative_eq!(pixel_in_image2, pixel, epsilon = 1e-6); } let ab_in_z1plane = camera.undistort(&pixel); - let ab_in_z1plane2_f32 = interpolate(&table.image_view(), pixel.cast()); - let ab_in_z1plane2 = ab_in_z1plane2_f32.cast(); - assert_relative_eq!(ab_in_z1plane, ab_in_z1plane2, epsilon = 0.000001); let pixel_in_image3 = camera.distort(&ab_in_z1plane); assert_relative_eq!(pixel_in_image3, pixel, epsilon = 1e-6); @@ -176,99 +201,6 @@ fn dyn_camera_tests() { assert_relative_eq!(dx, numeric_dx, epsilon = 1e-4); } - - let table: DistortTable = distort_table(&camera); - - for pixel in pixels_in_image { - let proj = camera.undistort(&pixel); - println!("proj: {:?}", proj); - println!("region: {:?}", table.region); - let analytic_pixel = camera.distort(&proj); - let lut_pixel = table.lookup(&proj); - - assert_abs_diff_eq!(analytic_pixel, pixel, epsilon = 1e-3); - assert_abs_diff_eq!(analytic_pixel, lut_pixel, epsilon = 1e-3); - } - } - } - - { - let camera: DynCameraF64 = DynCameraF64::new_pinhole( - &VecF64::<4>::new(600.0, 600.0, 319.5, 239.5), - ImageSize { - width: 640, - height: 480, - }, - ); - - let point_in_z1_plane = [ - VecF64::<2>::new(0.0, 0.0), - VecF64::<2>::new(1.0, 400.0), - VecF64::<2>::new(320.0, 240.0), - VecF64::<2>::new(319.5, 239.5), - VecF64::<2>::new(100.0, 40.0), - VecF64::<2>::new(639.0, 479.0), - ]; - - let expected_distorted_pixels = [ - VecF64::<2>::new(319.5, 239.5), - VecF64::<2>::new(919.5, 240239.5), - VecF64::<2>::new(192319.5, 144239.5), - VecF64::<2>::new(192019.5, 143939.5), - VecF64::<2>::new(60319.5, 24239.5), - VecF64::<2>::new(383719.5, 287639.5), - ]; - - for (pixel, expected_distorted_pixel) in point_in_z1_plane - .iter() - .zip(expected_distorted_pixels.iter()) - { - let distorted_pixel = camera.distort(pixel); - // println!("distorted_pixel: {:?}", distorted_pixel); - assert_relative_eq!(distorted_pixel, *expected_distorted_pixel, epsilon = 1e-6); - - let undistorted_pixel = camera.undistort(&distorted_pixel); - assert_relative_eq!(undistorted_pixel, *pixel, epsilon = 1e-6); - } - } - - { - let camera: DynCameraF64 = DynCameraF64::new_kannala_brandt( - &VecF64::<8>::from_vec(vec![1000.0, 1000.0, 320.0, 280.0, 0.1, 0.01, 0.001, 0.0001]), - ImageSize { - width: 640, - height: 480, - }, - ); - - let point_in_z1_plane = [ - VecF64::<2>::new(0.0, 0.0), - VecF64::<2>::new(1.0, 400.0), - VecF64::<2>::new(320.0, 240.0), - VecF64::<2>::new(319.5, 239.5), - VecF64::<2>::new(100.0, 40.0), - VecF64::<2>::new(639.0, 479.0), - ]; - - let expected_distorted_pixels = [ - VecF64::<2>::new(320.0, 280.0), - VecF64::<2>::new(325.1949172763466, 2357.966910538644), - VecF64::<2>::new(1982.378709731326, 1526.7840322984944), - VecF64::<2>::new(1982.6832644475849, 1526.3619462760455), - VecF64::<2>::new(2235.6822069661744, 1046.2728827864696), - VecF64::<2>::new(1984.8663275417607, 1527.9983895031353), - ]; - - for (pixel, expected_distorted_pixel) in point_in_z1_plane - .iter() - .zip(expected_distorted_pixels.iter()) - { - let distorted_pixel = camera.distort(pixel); - // println!("distorted_pixel: {:?}", distorted_pixel); - assert_relative_eq!(distorted_pixel, *expected_distorted_pixel, epsilon = 1e-6); - - let undistorted_pixel = camera.undistort(&distorted_pixel); - assert_relative_eq!(undistorted_pixel, *pixel, epsilon = 1e-6); } } } diff --git a/crates/sophus_viewer/src/actor.rs b/crates/sophus_viewer/src/actor.rs index 6f305c3..f18cf56 100644 --- a/crates/sophus_viewer/src/actor.rs +++ b/crates/sophus_viewer/src/actor.rs @@ -25,7 +25,7 @@ pub struct ViewerCamera { impl Default for ViewerCamera { fn default() -> Self { - ViewerCamera::default_from(ImageSize::new(640, 480)) + ViewerCamera::default_from(ImageSize::new(639, 479)) } } diff --git a/crates/sophus_viewer/src/interactions.rs b/crates/sophus_viewer/src/interactions.rs index 8294323..8d6fead 100644 --- a/crates/sophus_viewer/src/interactions.rs +++ b/crates/sophus_viewer/src/interactions.rs @@ -54,7 +54,7 @@ pub struct SceneFocus { /// Depth pub depth: f64, /// UV position - pub uv: VecF64<2>, + pub uv_in_virtual_camera: VecF64<2>, } #[derive(Clone, Copy)] diff --git a/crates/sophus_viewer/src/interactions/orbit_interaction.rs b/crates/sophus_viewer/src/interactions/orbit_interaction.rs index 11116d2..de53f31 100644 --- a/crates/sophus_viewer/src/interactions/orbit_interaction.rs +++ b/crates/sophus_viewer/src/interactions/orbit_interaction.rs @@ -71,23 +71,28 @@ impl OrbitalInteraction { let scroll_stopped = self.maybe_scroll_state.is_some() && is_scroll_zero; if scroll_started { - let uv = egui::Vec2::new( - (last_pointer_pos - response.rect.min)[0] * scales[0], - (last_pointer_pos - response.rect.min)[1] * scales[1], + let uv_view_port = egui::Pos2::new( + (last_pointer_pos - response.rect.min)[0], + (last_pointer_pos - response.rect.min)[1], + ); + + let uv_in_virtual_camera = VecF64::<2>::new( + (uv_view_port.x * scales[0]) as f64, + (uv_view_port.y * scales[1]) as f64, ); if self.maybe_scene_focus.is_none() { // Typically, the scene focus shall only be set by the pointer interaction event. But // it was never set, we set it here. - let mut z = self - .clipping_planes - .z_from_ndc(z_buffer.pixel(uv.x as usize, uv.y as usize) as f64); + let mut z = self.clipping_planes.z_from_ndc( + z_buffer.pixel(uv_view_port.x as usize, uv_view_port.y as usize) as f64, + ); if z >= self.clipping_planes.far { z = self.median_scene_depth(z_buffer); } self.maybe_scene_focus = Some(SceneFocus { depth: z, - uv: VecF64::<2>::new(uv.x as f64, uv.y as f64), + uv_in_virtual_camera, }); } self.maybe_scroll_state = Some(ScrollState {}); @@ -109,7 +114,7 @@ impl OrbitalInteraction { let delta_z: f64 = (smooth_scroll_delta.x) as f64; let scene_focus = self.maybe_scene_focus.unwrap(); - let pixel = scene_focus.uv; + let pixel = scene_focus.uv_in_virtual_camera; let depth = scene_focus.depth; let delta = 0.01 * VecF64::<6>::new(0.0, 0.0, 0.0, 0.0, 0.0, delta_z); let camera_from_scene_point = Isometry3::from_t(&cam.cam_unproj_with_z(&pixel, depth)); @@ -141,23 +146,33 @@ impl OrbitalInteraction { let pointer = response.interact_pointer_pos().unwrap(); - let uv = egui::Pos2::new( - (pointer - response.rect.min)[0] * scales[0], - (pointer - response.rect.min)[1] * scales[1], + let uv_view_port = egui::Pos2::new( + (pointer - response.rect.min)[0], + (pointer - response.rect.min)[1], + ); + + let uv_in_virtual_camera = VecF64::<2>::new( + (uv_view_port.x * scales[0]) as f64, + (uv_view_port.y * scales[1]) as f64, ); - let mut z = self - .clipping_planes - .z_from_ndc(z_buffer.pixel(uv.x as usize, uv.y as usize) as f64); + let mut z = self.clipping_planes.z_from_ndc( + z_buffer.pixel(uv_view_port.x as usize, uv_view_port.y as usize) as f64, + ); + println!("z_buffer: {:?}", z_buffer.image_size()); + + println!("uv_view_port: {:?}", uv_view_port); + + println!("z: {}", z); if z >= self.clipping_planes.far { z = self.median_scene_depth(z_buffer); } self.maybe_scene_focus = Some(SceneFocus { depth: z, - uv: VecF64::<2>::new(uv.x as f64, uv.y as f64), + uv_in_virtual_camera, }); self.maybe_pointer_state = Some(InteractionPointerState { - start_uv: VecF64::<2>::new(uv.x as f64, uv.y as f64), + start_uv: uv_in_virtual_camera, }); } else if response.drag_stopped() { // A drag event finished @@ -188,13 +203,14 @@ impl OrbitalInteraction { self.scene_from_camera = scene_from_camera; if let Some(focus) = &mut self.maybe_scene_focus { - focus.uv = VecF64::<2>::new(current_pixel.x as f64, current_pixel.y as f64); + focus.uv_in_virtual_camera = + VecF64::<2>::new(current_pixel.x as f64, current_pixel.y as f64); } } else if response.dragged_by(egui::PointerButton::Primary) { // rotate around scene focus let scene_focus = self.maybe_scene_focus.unwrap(); - let pixel = scene_focus.uv; + let pixel = scene_focus.uv_in_virtual_camera; let depth = scene_focus.depth; let delta = 0.01 * VecF64::<6>::new(0.0, 0.0, 0.0, -delta_y as f64, delta_x as f64, 0.0); diff --git a/crates/sophus_viewer/src/offscreen.rs b/crates/sophus_viewer/src/offscreen.rs index 5392435..860d2db 100644 --- a/crates/sophus_viewer/src/offscreen.rs +++ b/crates/sophus_viewer/src/offscreen.rs @@ -8,18 +8,15 @@ use sophus_image::ImageSize; use wgpu::COPY_BYTES_PER_ROW_ALIGNMENT; #[derive(Debug)] -pub(crate) struct OffscreenTexture { +pub(crate) struct RgbaTexture { pub(crate) rgba_texture_view: wgpu::TextureView, pub(crate) rgba_tex_id: egui::TextureId, - pub(crate) depth_render_target: wgpu::Texture, - pub(crate) depth_output_staging_buffer: wgpu::Buffer, - pub(crate) depth_texture_view: wgpu::TextureView, } -impl OffscreenTexture { - pub(crate) fn new(render_state: &ViewerRenderState, image_size: &ImageSize) -> Self { - let w = image_size.width as u32; - let h = image_size.height as u32; +impl RgbaTexture { + pub(crate) fn new(render_state: &ViewerRenderState, view_port_size: &ImageSize) -> Self { + let w = view_port_size.width as u32; + let h = view_port_size.height as u32; let render_target = render_state .device @@ -47,26 +44,92 @@ impl OffscreenTexture { wgpu::FilterMode::Linear, ); - let depth_render_target = render_state + RgbaTexture { + rgba_texture_view: texture_view, + rgba_tex_id: tex_id, + } + } +} + +#[derive(Debug)] +pub(crate) struct ZBufferTexture { + pub(crate) _depth_texture: wgpu::Texture, + pub(crate) depth_texture_view: wgpu::TextureView, + pub(crate) _depth_texture_sampler: wgpu::Sampler, +} + +impl ZBufferTexture { + pub(crate) fn new(render_state: &ViewerRenderState, view_port_size: &ImageSize) -> Self { + pub const DEPTH_FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Depth32Float; + + let size = wgpu::Extent3d { + width: view_port_size.width as u32, + height: view_port_size.height as u32, + depth_or_array_layers: 1, + }; + let desc = wgpu::TextureDescriptor { + label: Some("depth texture"), + size, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: DEPTH_FORMAT, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT // 3. + | wgpu::TextureUsages::TEXTURE_BINDING|wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }; + let texture = render_state.device.create_texture(&desc); + let view = texture.create_view(&wgpu::TextureViewDescriptor::default()); + let sampler = render_state .device - .create_texture(&wgpu::TextureDescriptor { - label: None, - size: wgpu::Extent3d { - width: w, - height: h, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::R32Float, - usage: wgpu::TextureUsages::RENDER_ATTACHMENT - | wgpu::TextureUsages::COPY_SRC - | wgpu::TextureUsages::TEXTURE_BINDING, - view_formats: &[wgpu::TextureFormat::R32Float], + .create_sampler(&wgpu::SamplerDescriptor { + address_mode_u: wgpu::AddressMode::ClampToEdge, + address_mode_v: wgpu::AddressMode::ClampToEdge, + address_mode_w: wgpu::AddressMode::ClampToEdge, + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + mipmap_filter: wgpu::FilterMode::Nearest, + compare: Some(wgpu::CompareFunction::LessEqual), + lod_min_clamp: 0.0, + lod_max_clamp: 100.0, + ..Default::default() }); - let bytes_per_row = Self::bytes_per_row(w); + ZBufferTexture { + _depth_texture: texture, + depth_texture_view: view, + _depth_texture_sampler: sampler, + } + } +} + +#[derive(Debug)] +pub(crate) struct DepthTexture { + pub(crate) depth_output_staging_buffer: wgpu::Buffer, + pub(crate) depth_render_target_f32: wgpu::Texture, + pub(crate) depth_texture_view_f32: wgpu::TextureView, +} + +impl DepthTexture { + const BYTES_PER_PIXEL: u32 = 4; + fn bytes_per_row(width: u32) -> u32 { + let unaligned_bytes_per_row = width * Self::BYTES_PER_PIXEL; + let align = COPY_BYTES_PER_ROW_ALIGNMENT; + + if unaligned_bytes_per_row % align == 0 { + // already aligned + unaligned_bytes_per_row + } else { + // align to the next multiple of `align` + (unaligned_bytes_per_row / align + 1) * align + } + } + + /// Create a new depth renderer. + pub fn new(render_state: &ViewerRenderState, view_port_size: &ImageSize) -> Self { + let w = view_port_size.width as u32; + let h = view_port_size.height as u32; + let bytes_per_row = DepthTexture::bytes_per_row(w); let required_buffer_size = bytes_per_row * h; // Total bytes needed in the buffer @@ -78,46 +141,62 @@ impl OffscreenTexture { mapped_at_creation: false, }); - let depth_texture_view = depth_render_target.create_view(&wgpu::TextureViewDescriptor { - label: None, - format: Some(wgpu::TextureFormat::R32Float), - dimension: Some(wgpu::TextureViewDimension::D2), - aspect: wgpu::TextureAspect::All, - base_mip_level: 0, - mip_level_count: None, - base_array_layer: 0, - array_layer_count: None, - }); + let depth_render_target_f32 = + render_state + .device + .create_texture(&wgpu::TextureDescriptor { + label: None, + size: wgpu::Extent3d { + width: w, + height: h, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::R32Float, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::COPY_SRC + | wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[wgpu::TextureFormat::R32Float], + }); + let depth_texture_view_f32 = + depth_render_target_f32.create_view(&wgpu::TextureViewDescriptor { + label: None, + format: Some(wgpu::TextureFormat::R32Float), + dimension: Some(wgpu::TextureViewDimension::D2), + aspect: wgpu::TextureAspect::All, + base_mip_level: 0, + mip_level_count: None, + base_array_layer: 0, + array_layer_count: None, + }); Self { - rgba_texture_view: texture_view, - rgba_tex_id: tex_id, - depth_render_target, depth_output_staging_buffer, - depth_texture_view, + depth_texture_view_f32, + depth_render_target_f32, } } - fn bytes_per_row(width: u32) -> u32 { - let bytes_per_pixel = 4; - let unaligned_bytes_per_row = width * bytes_per_pixel; - let align = COPY_BYTES_PER_ROW_ALIGNMENT; // GPU's row alignment requirement - (unaligned_bytes_per_row + align - 1) & !(align - 1) - } - - // download the depth image from the GPU to ArcImageF32 and copy the rgba texture to the egui texture - pub fn download_images( + // download the depth image from the GPU to ArcImageF32 + // + // Note: The depth image return might have a greater width than the requested width, to ensure + // that the bytes per row is a multiple of COPY_BYTES_PER_ROW_ALIGNMENT (i.e. 256). + pub fn download_image( &self, state: &ViewerRenderState, mut command_encoder: wgpu::CommandEncoder, - image_size: &ImageSize, + view_port_size: &ImageSize, ) -> ArcImageF32 { - let w = image_size.width as u32; - let h = image_size.height as u32; + let w = view_port_size.width as u32; + let h = view_port_size.height as u32; + + let bytes_per_row = DepthTexture::bytes_per_row(w); command_encoder.copy_texture_to_buffer( wgpu::ImageCopyTexture { - texture: &self.depth_render_target, + texture: &self.depth_render_target_f32, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All, @@ -126,7 +205,7 @@ impl OffscreenTexture { buffer: &self.depth_output_staging_buffer, layout: wgpu::ImageDataLayout { offset: 0, - bytes_per_row: Some(NonZeroU32::new(Self::bytes_per_row(w)).unwrap().get()), + bytes_per_row: Some(NonZeroU32::new(bytes_per_row).unwrap().get()), rows_per_image: Some(h), }, }, @@ -155,7 +234,7 @@ impl OffscreenTexture { let view = ImageViewF32::from_size_and_slice( ImageSize { - width: w as usize, + width: (bytes_per_row / Self::BYTES_PER_PIXEL) as usize, height: h as usize, }, bytemuck::cast_slice(&data[..]), @@ -168,3 +247,25 @@ impl OffscreenTexture { maybe_depth_image.unwrap() } } + +#[derive(Debug)] +pub(crate) struct OffscreenTextures { + view_port_size: ImageSize, + pub(crate) rgba: RgbaTexture, + pub(crate) z_buffer: ZBufferTexture, + pub(crate) depth: DepthTexture, +} + +impl OffscreenTextures { + pub(crate) fn view_port_size(&self) -> &ImageSize { + &self.view_port_size + } + pub(crate) fn new(render_state: &ViewerRenderState, view_port_size: &ImageSize) -> Self { + Self { + view_port_size: *view_port_size, + rgba: RgbaTexture::new(render_state, view_port_size), + z_buffer: ZBufferTexture::new(render_state, view_port_size), + depth: DepthTexture::new(render_state, view_port_size), + } + } +} diff --git a/crates/sophus_viewer/src/pixel_renderer.rs b/crates/sophus_viewer/src/pixel_renderer.rs index 92a8512..48de566 100644 --- a/crates/sophus_viewer/src/pixel_renderer.rs +++ b/crates/sophus_viewer/src/pixel_renderer.rs @@ -11,9 +11,9 @@ use wgpu::util::DeviceExt; use wgpu::DepthStencilState; use crate::interactions::InteractionEnum; +use crate::offscreen::ZBufferTexture; use crate::pixel_renderer::line::PixelLineRenderer; use crate::pixel_renderer::pixel_point::PixelPointRenderer; -use crate::scene_renderer::depth_renderer::DepthRenderer; use crate::ViewerRenderState; /// Renderer for pixel data @@ -138,7 +138,10 @@ impl PixelRenderer { for _i in 0..6 { vertex_data.push(PointVertex2 { - _pos: [scene_focus.uv[0] as f32, scene_focus.uv[1] as f32], + _pos: [ + scene_focus.uv_in_virtual_camera[0] as f32, + scene_focus.uv_in_virtual_camera[1] as f32, + ], _color: [0.5, 0.5, 0.5, 1.0], _point_size: 5.0, }); @@ -165,7 +168,10 @@ impl PixelRenderer { for _i in 0..6 { vertex_data.push(PointVertex2 { - _pos: [scene_focus.uv[0] as f32, scene_focus.uv[1] as f32], + _pos: [ + scene_focus.uv_in_virtual_camera[0] as f32, + scene_focus.uv_in_virtual_camera[1] as f32, + ], _color: [0.5, 0.5, 0.5, 1.0], _point_size: 5.0, }); @@ -207,7 +213,7 @@ impl PixelRenderer { &'rp self, command_encoder: &'rp mut wgpu::CommandEncoder, texture_view: &'rp wgpu::TextureView, - depth: &DepthRenderer, + depth: &ZBufferTexture, ) { let mut render_pass = command_encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: None, diff --git a/crates/sophus_viewer/src/renderables/frame.rs b/crates/sophus_viewer/src/renderables/frame.rs index 0da43a1..8141690 100644 --- a/crates/sophus_viewer/src/renderables/frame.rs +++ b/crates/sophus_viewer/src/renderables/frame.rs @@ -50,8 +50,6 @@ impl Frame { image.image_size(), ); - println!("!!!!!!!!intrinsics: {:?}", intrinsics); - Frame { image: Some(image.clone()), intrinsics, diff --git a/crates/sophus_viewer/src/scene_renderer.rs b/crates/sophus_viewer/src/scene_renderer.rs index 571469f..f0c99f9 100644 --- a/crates/sophus_viewer/src/scene_renderer.rs +++ b/crates/sophus_viewer/src/scene_renderer.rs @@ -1,9 +1,6 @@ /// buffers for rendering a scene pub mod buffers; -/// depth renderer -pub mod depth_renderer; - /// line renderer pub mod line; @@ -28,8 +25,9 @@ use sophus_sensor::dyn_camera::DynCamera; use wgpu::DepthStencilState; use crate::interactions::InteractionEnum; +use crate::offscreen::ZBufferTexture; +use crate::scene_renderer::buffers::Frustum; use crate::scene_renderer::buffers::SceneRenderBuffers; -use crate::scene_renderer::depth_renderer::DepthRenderer; use crate::scene_renderer::mesh::MeshRenderer; use crate::scene_renderer::point::ScenePointRenderer; use crate::ViewerRenderState; @@ -46,8 +44,6 @@ pub struct SceneRenderer { pub point_renderer: ScenePointRenderer, /// Line renderer pub line_renderer: line::SceneLineRenderer, - /// Depth renderer - pub depth_renderer: DepthRenderer, /// Interaction state pub interaction: InteractionEnum, } @@ -62,12 +58,6 @@ impl SceneRenderer { ) -> Self { let device = &wgpu_render_state.device; - let depth_renderer = DepthRenderer::new( - wgpu_render_state, - &intrinsics.image_size(), - depth_stencil.clone(), - ); - let uniform_bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { label: Some("scene group layout"), @@ -159,7 +149,6 @@ impl SceneRenderer { Self { buffers, - depth_renderer, mesh_renderer: MeshRenderer::new( wgpu_render_state, &pipeline_layout, @@ -199,7 +188,7 @@ impl SceneRenderer { &'rp self, command_encoder: &'rp mut wgpu::CommandEncoder, texture_view: &'rp wgpu::TextureView, - depth: &DepthRenderer, + depth: &ZBufferTexture, ) { let mut render_pass = command_encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: None, @@ -250,7 +239,7 @@ impl SceneRenderer { &'rp self, command_encoder: &'rp mut wgpu::CommandEncoder, depth_texture_view: &'rp wgpu::TextureView, - depth: &DepthRenderer, + depth: &ZBufferTexture, ) { let mut render_pass = command_encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: None, @@ -327,6 +316,23 @@ impl SceneRenderer { bytemuck::cast_slice(self.textured_mesh_renderer.vertices.as_slice()), ); + let frustum_uniforms = Frustum { + camera_image_width: intrinsics.image_size().width as f32, + camera_image_height: intrinsics.image_size().height as f32, + near: 0.1, + far: 1000.0, + fx: intrinsics.pinhole_params()[0] as f32, + fy: intrinsics.pinhole_params()[1] as f32, + px: intrinsics.pinhole_params()[2] as f32, + py: intrinsics.pinhole_params()[3] as f32, + }; + + state.queue.write_buffer( + &self.buffers.frustum_uniform_buffer, + 0, + bytemuck::cast_slice(&[frustum_uniforms]), + ); + let mut scene_from_camera_uniform: [[f32; 4]; 4] = [[0.0; 4]; 4]; for i in 0..4 { for j in 0..4 { @@ -362,7 +368,7 @@ impl SceneRenderer { self.buffers.dist_texture.size(), ); state.queue.write_buffer( - &self.buffers.lut_buffer, + &self.buffers.camara_params_buffer, 0, bytemuck::cast_slice(&[ distort_lut.offset().x as f32, diff --git a/crates/sophus_viewer/src/scene_renderer/buffers.rs b/crates/sophus_viewer/src/scene_renderer/buffers.rs index 3626e0d..2dfef01 100644 --- a/crates/sophus_viewer/src/scene_renderer/buffers.rs +++ b/crates/sophus_viewer/src/scene_renderer/buffers.rs @@ -9,21 +9,22 @@ use crate::ViewerRenderState; #[repr(C)] // This is so we can store this in a buffer #[derive(Debug, Copy, Clone, bytemuck::Pod, bytemuck::Zeroable)] -struct Transforms { - width: f32, - height: f32, - near: f32, - far: f32, - fx: f32, - fy: f32, - px: f32, - py: f32, +pub(crate) struct Frustum { + pub(crate) camera_image_width: f32, // <= this is NOT the view-port width + pub(crate) camera_image_height: f32, // <= this is NOT the view-port height + pub(crate) near: f32, + pub(crate) far: f32, + // pinhole parameters for debugging only + pub(crate) fx: f32, + pub(crate) fy: f32, + pub(crate) px: f32, + pub(crate) py: f32, } #[repr(C)] // This is so we can store this in a buffer #[derive(Debug, Copy, Clone, bytemuck::Pod, bytemuck::Zeroable)] -struct Lut { +struct DistortionLut { lut_offset_x: f32, lut_offset_y: f32, lut_range_x: f32, @@ -39,9 +40,9 @@ struct View { /// Buffers for rendering a scene pub struct SceneRenderBuffers { pub(crate) bind_group: wgpu::BindGroup, - pub(crate) _uniform_buffer: wgpu::Buffer, + pub(crate) frustum_uniform_buffer: wgpu::Buffer, pub(crate) view_uniform_buffer: wgpu::Buffer, - pub(crate) lut_buffer: wgpu::Buffer, + pub(crate) camara_params_buffer: wgpu::Buffer, pub(crate) dist_texture: wgpu::Texture, pub(crate) dist_bind_group: wgpu::BindGroup, pub(crate) distortion_lut: Mutex>, @@ -99,9 +100,9 @@ impl SceneRenderBuffers { [0.0, 0.0, 0.0, 1.0], // 4. ]; - let transform_uniforms = Transforms { - width: intrinsics.image_size().width as f32, - height: intrinsics.image_size().height as f32, + let frustum_uniforms = Frustum { + camera_image_width: intrinsics.image_size().width as f32, + camera_image_height: intrinsics.image_size().height as f32, near: 0.1, far: 1000.0, fx: intrinsics.pinhole_params()[0] as f32, @@ -110,20 +111,22 @@ impl SceneRenderBuffers { py: intrinsics.pinhole_params()[3] as f32, }; - let transform_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + println!("frustum_uniforms: {:?}", frustum_uniforms); + + let frustum_uniform_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("Camera Buffer"), - contents: bytemuck::cast_slice(&[transform_uniforms]), + contents: bytemuck::cast_slice(&[frustum_uniforms]), usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, }); - let lut_uniforms = Lut { + let lut_uniforms = DistortionLut { lut_offset_x: 0.0, lut_offset_y: 0.0, lut_range_x: 0.0, lut_range_y: 0.0, }; - let lut_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + let camara_params_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("Lut Buffer"), contents: bytemuck::cast_slice(&[lut_uniforms]), usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, @@ -145,7 +148,7 @@ impl SceneRenderBuffers { entries: &[ wgpu::BindGroupEntry { binding: 0, - resource: transform_buffer.as_entire_binding(), + resource: frustum_uniform_buffer.as_entire_binding(), }, wgpu::BindGroupEntry { binding: 1, @@ -153,7 +156,7 @@ impl SceneRenderBuffers { }, wgpu::BindGroupEntry { binding: 2, - resource: lut_buffer.as_entire_binding(), + resource: camara_params_buffer.as_entire_binding(), }, ], }); @@ -191,9 +194,9 @@ impl SceneRenderBuffers { address_mode_u: wgpu::AddressMode::ClampToEdge, address_mode_v: wgpu::AddressMode::ClampToEdge, address_mode_w: wgpu::AddressMode::ClampToEdge, - mag_filter: wgpu::FilterMode::Linear, - min_filter: wgpu::FilterMode::Linear, - mipmap_filter: wgpu::FilterMode::Linear, + mag_filter: wgpu::FilterMode::Nearest, + min_filter: wgpu::FilterMode::Nearest, + mipmap_filter: wgpu::FilterMode::Nearest, ..Default::default() }); @@ -270,9 +273,9 @@ impl SceneRenderBuffers { Self { bind_group, - _uniform_buffer: transform_buffer, + frustum_uniform_buffer, view_uniform_buffer: view_buffer, - lut_buffer, + camara_params_buffer, dist_texture, dist_bind_group, distortion_lut: Mutex::new(None), diff --git a/crates/sophus_viewer/src/scene_renderer/depth_renderer.rs b/crates/sophus_viewer/src/scene_renderer/depth_renderer.rs deleted file mode 100644 index 7bc9e09..0000000 --- a/crates/sophus_viewer/src/scene_renderer/depth_renderer.rs +++ /dev/null @@ -1,60 +0,0 @@ -use sophus_image::ImageSize; -use wgpu::DepthStencilState; - -use crate::ViewerRenderState; - -/// Depth renderer. -pub struct DepthRenderer { - pub(crate) _depth_stencil: Option, - pub(crate) _depth_texture: wgpu::Texture, - pub(crate) depth_texture_view: wgpu::TextureView, - pub(crate) _depth_texture_sampler: wgpu::Sampler, -} - -impl DepthRenderer { - /// Create a new depth renderer. - pub fn new( - state: &ViewerRenderState, - image_size: &ImageSize, - depth_stencil: Option, - ) -> Self { - pub const DEPTH_FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Depth32Float; - - let size = wgpu::Extent3d { - width: image_size.width as u32, - height: image_size.height as u32, - depth_or_array_layers: 1, - }; - let desc = wgpu::TextureDescriptor { - label: Some("depth texture"), - size, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: DEPTH_FORMAT, - usage: wgpu::TextureUsages::RENDER_ATTACHMENT // 3. - | wgpu::TextureUsages::TEXTURE_BINDING|wgpu::TextureUsages::COPY_SRC, - view_formats: &[], - }; - let texture = state.device.create_texture(&desc); - let view = texture.create_view(&wgpu::TextureViewDescriptor::default()); - let sampler = state.device.create_sampler(&wgpu::SamplerDescriptor { - address_mode_u: wgpu::AddressMode::ClampToEdge, - address_mode_v: wgpu::AddressMode::ClampToEdge, - address_mode_w: wgpu::AddressMode::ClampToEdge, - mag_filter: wgpu::FilterMode::Linear, - min_filter: wgpu::FilterMode::Linear, - mipmap_filter: wgpu::FilterMode::Nearest, - compare: Some(wgpu::CompareFunction::LessEqual), - lod_min_clamp: 0.0, - lod_max_clamp: 100.0, - ..Default::default() - }); - Self { - _depth_texture: texture, - depth_texture_view: view, - _depth_texture_sampler: sampler, - _depth_stencil: depth_stencil, - } - } -} diff --git a/crates/sophus_viewer/src/scene_renderer/utils.wgsl b/crates/sophus_viewer/src/scene_renderer/utils.wgsl index 6ce9c45..dc7f483 100644 --- a/crates/sophus_viewer/src/scene_renderer/utils.wgsl +++ b/crates/sophus_viewer/src/scene_renderer/utils.wgsl @@ -20,8 +20,8 @@ fn scene_point_to_z1_plane(scene_point: vec3, } struct Frustum { - width: f32, - height: f32, + camera_image_width: f32, // <= NOT the viewport width + camera_image_height: f32, // <= NOT the viewport height near: f32, far: f32, // fx, fy, px, py or just here to debug distortion lut table. @@ -39,35 +39,41 @@ struct DistortionLut { }; fn z1_plane_to_distorted(point_in_proj: vec3, frustum: Frustum, lut: DistortionLut) -> vec3 { - var width = frustum.width; - var height = frustum.height; + var width = frustum.camera_image_width; + var height = frustum.camera_image_height; var lut_offset_x = lut.lut_offset_x; var lut_offset_y = lut.lut_offset_y; var lut_range_x = lut.lut_range_x; var lut_range_y = lut.lut_range_y; - let u = clamp((width - 1.0) * (point_in_proj.x -lut_offset_x) / lut_range_x, 0.0, width - 1.00001); - let v = clamp((height - 1.0) * (point_in_proj.y -lut_offset_y) / lut_range_y, 0.0, height - 1.00001); + let x_lut = clamp((width -1.0 ) * (point_in_proj.x -lut_offset_x) / lut_range_x, 0.0, width - 1.00001); + let y_lut = clamp((height -1.0 ) * (point_in_proj.y -lut_offset_y) / lut_range_y, 0.0, height - 1.00001); // Manual implementation of bilinear interpolation. // This is to workaround apparent limitations of wgpu - such as no/limited support for // sampling of f32 textures and sampling in the vertex shader. // TDDO: Figure out how to use sampling in vertex shader or maybe undistort in fragment shader // (first render pinhole image to texture, then undistort in fragment shader). - let u0 = i32(u); // left nearest coordinate - let v0 = i32(v); // top nearest coordinate - let u1 = u0 + 1; // right nearest coordinate - let v1 = v0 + 1; // bottom nearest coordinate - let frac_u = u - f32(u0); // fractional part of u - let frac_v = v - f32(v0); // fractional part of v - var val00 = textureLoad(distortion_texture, vec2(u0, v0), 0).xy; - var val01 = textureLoad(distortion_texture, vec2(u0, v1), 0).xy; - var val10 = textureLoad(distortion_texture, vec2(u1, v0), 0).xy; - var val11 = textureLoad(distortion_texture, vec2(u1, v1), 0).xy; - var val0 = mix(val00, val01, frac_v); - var val1 = mix(val10, val11, frac_v); - var val = mix(val0, val1, frac_u); - return vec3(val.x, val.y, point_in_proj.z); + let x0 = i32(x_lut); // left nearest coordinate + let y0 = i32(y_lut); // top nearest coordinate + let x1 = x0 + 1; // right nearest coordinate + let y1 = y0 + 1; // bottom nearest coordinate + let frac_x = x_lut - f32(x0); // fractional part of u + let frac_y = y_lut - f32(y0); // fractional part of v + var val00 = textureLoad(distortion_texture, vec2(x0, y0), 0).xy; + var val01 = textureLoad(distortion_texture, vec2(x0, y1), 0).xy; + var val10 = textureLoad(distortion_texture, vec2(x1, y0), 0).xy; + var val11 = textureLoad(distortion_texture, vec2(x1, y1), 0).xy; + var val0 = mix(val00, val01, frac_y); + var val1 = mix(val10, val11, frac_y); + var val = mix(val0, val1, frac_x); + let u = val.x; + let v = val.y; + + // to debug distortion lut table - just using pinhole projection + // let u = point_in_proj.x * frustum.fx + frustum.px; + // let v = point_in_proj.y * frustum.fy + frustum.py; + return vec3(u, v, point_in_proj.z); } fn scene_point_to_distorted(scene_point: vec3, @@ -80,16 +86,17 @@ fn scene_point_to_distorted(scene_point: vec3, // map point from pixel coordinates (Computer Vision convention) to clip space coordinates (WebGPU convention) fn pixel_and_z_to_clip(uv_z: vec2, z: f32, frustum: Frustum) -> vec4 { - var width = frustum.width; - var height = frustum.height; + var width = frustum.camera_image_width; + var height = frustum.camera_image_height; var near = frustum.near; var far = frustum.far; var u = uv_z.x; var v = uv_z.y; - return vec4(2.0 * (u / width - 0.5), - -2.0 * (v / height - 0.5), - // todo: Check whether the z value is correct. - (far * (z - near)) / (z * (far - near)), + let z_clip = (far * (z - near)) / (z * (far - near)); + + return vec4(2.0 * ((u+0.5) / width - 0.5), + -2.0 * ((v+0.5) / height - 0.5), + z_clip, 1.0); } diff --git a/crates/sophus_viewer/src/simple_viewer.rs b/crates/sophus_viewer/src/simple_viewer.rs index c3ee013..60b7efd 100644 --- a/crates/sophus_viewer/src/simple_viewer.rs +++ b/crates/sophus_viewer/src/simple_viewer.rs @@ -8,9 +8,7 @@ use hollywood::compute::pipeline::CancelRequest; use hollywood::RequestWithReplyChannel; use linked_hash_map::LinkedHashMap; use sophus_core::linalg::VecF32; -use sophus_image::arc_image::ArcImageF32; use sophus_lie::Isometry3; -use sophus_sensor::dyn_camera::DynCamera; use crate::actor::ViewerBuilder; use crate::pixel_renderer::LineVertex2; @@ -23,6 +21,7 @@ use crate::scene_renderer::point::PointVertex3; use crate::scene_renderer::textured_mesh::TexturedMeshVertex3; use crate::views::aspect_ratio::get_adjusted_view_size; use crate::views::aspect_ratio::get_max_size; +use crate::views::aspect_ratio::HasAspectRatio; use crate::views::view2d::View2d; use crate::views::view3d::View3d; use crate::views::View; @@ -188,18 +187,12 @@ impl SimpleViewer { } } for (_, lines) in view.pixel.line_renderer.lines_table.iter() { - println!("lines: {}", lines.len()); - for line in lines.iter() { let p0 = line.p0; let p1 = line.p1; let d = (p0 - p1).normalize(); let normal = [d[1], -d[0]]; - println!("p0: {:?}, p1: {:?}", p0, p1); - println!("d: {:?}, normal: {:?}", d, normal); - println!("color: {:?}", line.color); - let v0 = LineVertex2 { _pos: [p0[0], p0[1]], _normal: normal, @@ -306,12 +299,6 @@ impl SimpleViewer { } } -struct Data { - rgba_tex_id: egui::TextureId, - depth_image: ArcImageF32, - intrinsics: DynCamera, -} - impl eframe::App for SimpleViewer { fn on_exit(&mut self, _gl: Option<&eframe::glow::Context>) { self.cancel_request_sender.send(CancelRequest).unwrap(); @@ -335,108 +322,6 @@ impl eframe::App for SimpleViewer { // Add renderables to tables self.add_renderables_to_tables(); - let mut view_data_map = LinkedHashMap::new(); - - // Prepare views and update background textures - for (view_label, view) in self.views.iter_mut() { - match view { - View::View3d(view) => { - view.scene.prepare(&self.state, &view.intrinsics, &None); - - let mut command_encoder = self - .state - .device - .create_command_encoder(&wgpu::CommandEncoderDescriptor::default()); - - view.scene.paint( - &mut command_encoder, - &view.offscreen.rgba_texture_view, - &view.scene.depth_renderer, - ); - - self.state.queue.submit(Some(command_encoder.finish())); - - let mut command_encoder = self - .state - .device - .create_command_encoder(&wgpu::CommandEncoderDescriptor::default()); - view.scene.depth_paint( - &mut command_encoder, - &view.offscreen.depth_texture_view, - &view.scene.depth_renderer, - ); - - let depth_image = view.offscreen.download_images( - &self.state, - command_encoder, - &view.intrinsics.image_size(), - ); - - view_data_map.insert( - view_label.clone(), - Data { - rgba_tex_id: view.offscreen.rgba_tex_id, - depth_image, - intrinsics: view.intrinsics.clone(), - }, - ); - } - View::View2d(view) => { - view.scene - .prepare(&self.state, &view.intrinsics, &view.background_image); - view.pixel.prepare(&self.state); - - view.background_image = None; - - let mut command_encoder = self - .state - .device - .create_command_encoder(&wgpu::CommandEncoderDescriptor::default()); - - view.scene.paint( - &mut command_encoder, - &view.offscreen.rgba_texture_view, - &view.scene.depth_renderer, - ); - view.pixel.paint( - &mut command_encoder, - &view.offscreen.rgba_texture_view, - &view.scene.depth_renderer, - ); - - self.state.queue.submit(Some(command_encoder.finish())); - - view.pixel - .show_interaction_marker(&self.state, &view.scene.interaction); - - let mut command_encoder = self - .state - .device - .create_command_encoder(&wgpu::CommandEncoderDescriptor::default()); - view.scene.depth_paint( - &mut command_encoder, - &view.offscreen.depth_texture_view, - &view.scene.depth_renderer, - ); - - let depth_image = view.offscreen.download_images( - &self.state, - command_encoder, - &view.intrinsics.image_size(), - ); - - view_data_map.insert( - view_label.clone(), - Data { - rgba_tex_id: view.offscreen.rgba_tex_id, - depth_image, - intrinsics: view.intrinsics.clone(), - }, - ); - } - } - } - let mut responses = HashMap::new(); egui::CentralPanel::default().show(ctx, |ui| { @@ -446,42 +331,174 @@ impl eframe::App for SimpleViewer { } let (max_width, max_height) = get_max_size( &self.views, - 0.95 * ui0.available_width(), - 0.95 * ui0.available_height(), + 0.99 * ui0.available_width(), + 0.99 * ui0.available_height(), ); ui0.horizontal_wrapped(|ui| { - for (view_label, view_data) in view_data_map.iter() { + for (view_label, view) in self.views.iter_mut() { + let view_aspect_ratio = view.aspect_ratio(); let adjusted_size = - get_adjusted_view_size(&self.views[view_label], max_width, max_height); - - let response = ui.add( - egui::Image::new(SizedTexture { - size: egui::Vec2::new( - view_data.intrinsics.image_size().width as f32, - view_data.intrinsics.image_size().height as f32, - ), - id: view_data.rgba_tex_id, - }) - .fit_to_exact_size(egui::Vec2 { - x: adjusted_size.0, - y: adjusted_size.1, - }) - .sense(egui::Sense::click_and_drag()), - ); - - responses.insert( - view_label.clone(), - ( - response, - VecF32::<2>::new( - view_data.intrinsics.image_size().width as f32 - / adjusted_size.0, - view_data.intrinsics.image_size().height as f32 - / adjusted_size.1, - ), - ), - ); + get_adjusted_view_size(view_aspect_ratio, max_width, max_height); + match view { + View::View3d(view) => { + view.update_offscreen_texture( + &self.state, + &adjusted_size.image_size(), + ); + let offscreen = view.offscreen.as_ref().unwrap(); + + view.scene.prepare(&self.state, &view.intrinsics, &None); + view.pixel_for_interaction_marker.prepare(&self.state); + + let mut command_encoder = self.state.device.create_command_encoder( + &wgpu::CommandEncoderDescriptor::default(), + ); + + view.scene.paint( + &mut command_encoder, + &offscreen.rgba.rgba_texture_view, + &offscreen.z_buffer, + ); + + view.pixel_for_interaction_marker + .show_interaction_marker(&self.state, &view.scene.interaction); + + view.pixel_for_interaction_marker.paint( + &mut command_encoder, + &offscreen.rgba.rgba_texture_view, + &offscreen.z_buffer, + ); + + self.state.queue.submit(Some(command_encoder.finish())); + + let mut command_encoder = self.state.device.create_command_encoder( + &wgpu::CommandEncoderDescriptor::default(), + ); + view.scene.depth_paint( + &mut command_encoder, + &offscreen.depth.depth_texture_view_f32, + &offscreen.z_buffer, + ); + + let depth_image = offscreen.depth.download_image( + &self.state, + command_encoder, + &adjusted_size.image_size(), + ); + + let response = ui.add( + egui::Image::new(SizedTexture { + size: egui::Vec2::new( + adjusted_size.width, + adjusted_size.height, + ), + id: offscreen.rgba.rgba_tex_id, + }) + .fit_to_exact_size(egui::Vec2 { + x: adjusted_size.width, + y: adjusted_size.height, + }) + .sense(egui::Sense::click_and_drag()), + ); + + // println!("!!!!!{:?}", depth_renderer.view_port_size()); + + responses.insert( + view_label.clone(), + ( + response, + VecF32::<2>::new( + view.intrinsics.image_size().width as f32 + / adjusted_size.width, + view.intrinsics.image_size().height as f32 + / adjusted_size.height, + ), + depth_image, + ), + ); + } + View::View2d(view) => { + view.update_offscreen_texture( + &self.state, + &adjusted_size.image_size(), + ); + let offscreen = view.offscreen.as_ref().unwrap(); + + view.scene.prepare( + &self.state, + &view.intrinsics, + &view.background_image, + ); + view.pixel.prepare(&self.state); + + view.background_image = None; + + let mut command_encoder = self.state.device.create_command_encoder( + &wgpu::CommandEncoderDescriptor::default(), + ); + + view.scene.paint( + &mut command_encoder, + &offscreen.rgba.rgba_texture_view, + &offscreen.z_buffer, + ); + view.pixel.paint( + &mut command_encoder, + &offscreen.rgba.rgba_texture_view, + &offscreen.z_buffer, + ); + + self.state.queue.submit(Some(command_encoder.finish())); + + view.pixel + .show_interaction_marker(&self.state, &view.scene.interaction); + + let mut command_encoder = self.state.device.create_command_encoder( + &wgpu::CommandEncoderDescriptor::default(), + ); + view.scene.depth_paint( + &mut command_encoder, + &offscreen.depth.depth_texture_view_f32, + &offscreen.z_buffer, + ); + + let depth_image = offscreen.depth.download_image( + &self.state, + command_encoder, + &adjusted_size.image_size(), + ); + + let response = ui.add( + egui::Image::new(SizedTexture { + size: egui::Vec2::new( + adjusted_size.width, + adjusted_size.height, + ), + id: offscreen.rgba.rgba_tex_id, + }) + .fit_to_exact_size(egui::Vec2 { + x: adjusted_size.width, + y: adjusted_size.height, + }) + .sense(egui::Sense::click_and_drag()), + ); + + responses.insert( + view_label.clone(), + ( + response, + VecF32::<2>::new( + view.intrinsics.image_size().width as f32 + / adjusted_size.width, + view.intrinsics.image_size().height as f32 + / adjusted_size.height, + ), + depth_image, + ), + ); + } + } } }); }); @@ -491,13 +508,12 @@ impl eframe::App for SimpleViewer { match view { View::View3d(view) => { let response = responses.get(view_label).unwrap(); - let depth_image = view_data_map.get(view_label).unwrap().depth_image.clone(); view.scene.process_event( &view.intrinsics, &response.0, &response.1, - depth_image, + response.2.clone(), ); } View::View2d(_) => {} diff --git a/crates/sophus_viewer/src/views.rs b/crates/sophus_viewer/src/views.rs index a9a0a29..276e1a2 100644 --- a/crates/sophus_viewer/src/views.rs +++ b/crates/sophus_viewer/src/views.rs @@ -5,9 +5,6 @@ pub mod view2d; /// 3d view pub mod view3d; -use sophus_image::ImageSize; -use sophus_sensor::DynCamera; - use crate::views::aspect_ratio::HasAspectRatio; use crate::views::view2d::View2d; use crate::views::view3d::View3d; @@ -27,15 +24,4 @@ impl HasAspectRatio for View { View::View2d(view) => view.aspect_ratio(), } } - - fn intrinsics(&self) -> &DynCamera { - match self { - View::View3d(view) => view.intrinsics(), - View::View2d(view) => view.intrinsics(), - } - } - - fn view_size(&self) -> ImageSize { - self.intrinsics().image_size() - } } diff --git a/crates/sophus_viewer/src/views/aspect_ratio.rs b/crates/sophus_viewer/src/views/aspect_ratio.rs index 312466b..1bff3b7 100644 --- a/crates/sophus_viewer/src/views/aspect_ratio.rs +++ b/crates/sophus_viewer/src/views/aspect_ratio.rs @@ -1,15 +1,10 @@ use linked_hash_map::LinkedHashMap; use sophus_image::ImageSize; -use sophus_sensor::DynCamera; use crate::views::View; pub(crate) trait HasAspectRatio { fn aspect_ratio(&self) -> f32; - - fn view_size(&self) -> ImageSize; - - fn intrinsics(&self) -> &DynCamera; } pub(crate) fn get_median_aspect_ratio(views: &LinkedHashMap) -> f32 { @@ -52,12 +47,26 @@ pub(crate) fn get_max_size( (max_width, max_height) } -pub(crate) fn get_adjusted_view_size(view: &View, max_width: f32, max_height: f32) -> (f32, f32) { - let aspect_ratio = view.aspect_ratio(); - let view_size = view.view_size(); - let view_width = max_width.min(max_height * aspect_ratio); - let view_height = max_height.min(max_width / aspect_ratio); - let view_width = view_width.min(view_size.width as f32); - let view_height = view_height.min(view_size.height as f32); - (view_width, view_height) +pub(crate) struct ViewPortSize { + pub(crate) width: f32, + pub(crate) height: f32, +} + +impl ViewPortSize { + pub(crate) fn image_size(&self) -> ImageSize { + ImageSize { + width: self.width.ceil() as usize, + height: self.height.ceil() as usize, + } + } +} + +pub(crate) fn get_adjusted_view_size( + view_aspect_ratio: f32, + max_width: f32, + max_height: f32, +) -> ViewPortSize { + let width = max_width.min(max_height * view_aspect_ratio); + let height = max_height.min(max_width / view_aspect_ratio); + ViewPortSize { width, height } } diff --git a/crates/sophus_viewer/src/views/view2d.rs b/crates/sophus_viewer/src/views/view2d.rs index 9b2f2d0..25247b6 100644 --- a/crates/sophus_viewer/src/views/view2d.rs +++ b/crates/sophus_viewer/src/views/view2d.rs @@ -9,7 +9,7 @@ use sophus_sensor::DynCamera; use crate::interactions::inplane_interaction::InplaneInteraction; use crate::interactions::InteractionEnum; use crate::interactions::WgpuClippingPlanes; -use crate::offscreen::OffscreenTexture; +use crate::offscreen::OffscreenTextures; use crate::pixel_renderer::PixelRenderer; use crate::renderables::renderable2d::Renderable2d; use crate::renderables::renderable2d::View2dPacket; @@ -24,7 +24,7 @@ pub(crate) struct View2d { pub(crate) scene: SceneRenderer, pub(crate) pixel: PixelRenderer, pub(crate) intrinsics: DynCamera, - pub(crate) offscreen: OffscreenTexture, + pub(crate) offscreen: Option, pub(crate) background_image: Option, } @@ -66,11 +66,11 @@ impl View2d { pixel: PixelRenderer::new( state, &frame.intrinsics().image_size(), - depth_stencil, + depth_stencil.clone(), ), intrinsics: frame.intrinsics().clone(), background_image: frame.maybe_image().cloned(), - offscreen: OffscreenTexture::new(state, &frame.intrinsics().image_size()), + offscreen: None, }), ); return true; @@ -93,7 +93,6 @@ impl View2d { }; if let Some(frame) = packet.frame { - println!("!!!!!!!!!!!Got a new frame"); let depth_stencil = Some(wgpu::DepthStencilState { format: wgpu::TextureFormat::Depth32Float, depth_write_enabled: true, @@ -125,11 +124,11 @@ impl View2d { near: 0.1, far: 1000.0, }, - scene_from_camera: Isometry3::from_t(&VecF64::<3>::new(0.0, 0.0, -5.0)), + scene_from_camera: Isometry3::from_t(&VecF64::<3>::new(0.0, 0.0, 0.0)), }), ); - view.offscreen = OffscreenTexture::new(state, &new_intrinsics.image_size()); + view.offscreen = Some(OffscreenTextures::new(state, &new_intrinsics.image_size())); view.intrinsics = new_intrinsics.clone(); if let Some(background_image) = frame.maybe_image() { @@ -142,19 +141,19 @@ impl View2d { let p0 = view .intrinsics - .cam_unproj_with_z(&VecF64::<2>::new(0.0, 0.0), far) + .cam_unproj_with_z(&VecF64::<2>::new(-0.5, -0.5), far) .cast(); let p1 = view .intrinsics - .cam_unproj_with_z(&VecF64::<2>::new(w as f64, 0.0), far) + .cam_unproj_with_z(&VecF64::<2>::new(w as f64 - 0.5, -0.5), far) .cast(); let p2 = view .intrinsics - .cam_unproj_with_z(&VecF64::<2>::new(0.0, h as f64), far) + .cam_unproj_with_z(&VecF64::<2>::new(-0.5, h as f64 - 0.5), far) .cast(); let p3 = view .intrinsics - .cam_unproj_with_z(&VecF64::<2>::new(w as f64, h as f64), far) + .cam_unproj_with_z(&VecF64::<2>::new(w as f64 - 0.5, h as f64 - 0.5), far) .cast(); let tex_mesh = TexturedMesh3::make(&[ @@ -213,18 +212,28 @@ impl View2d { } } } + + pub fn update_offscreen_texture( + &mut self, + render_state: &ViewerRenderState, + view_port_size: &ImageSize, + ) { + match &mut self.offscreen { + Some(offscreen) => { + let current_view_port_size = offscreen.view_port_size(); + if current_view_port_size != view_port_size { + *offscreen = OffscreenTextures::new(render_state, view_port_size); + } + } + None => { + self.offscreen = Some(OffscreenTextures::new(render_state, view_port_size)); + } + } + } } impl HasAspectRatio for View2d { fn aspect_ratio(&self) -> f32 { self.intrinsics.image_size().aspect_ratio() } - - fn view_size(&self) -> ImageSize { - self.intrinsics.image_size() - } - - fn intrinsics(&self) -> &DynCamera { - &self.intrinsics - } } diff --git a/crates/sophus_viewer/src/views/view3d.rs b/crates/sophus_viewer/src/views/view3d.rs index 562fe13..cb2b81e 100644 --- a/crates/sophus_viewer/src/views/view3d.rs +++ b/crates/sophus_viewer/src/views/view3d.rs @@ -4,7 +4,8 @@ use sophus_sensor::DynCamera; use crate::interactions::orbit_interaction::OrbitalInteraction; use crate::interactions::InteractionEnum; -use crate::offscreen::OffscreenTexture; +use crate::offscreen::OffscreenTextures; +use crate::pixel_renderer::PixelRenderer; use crate::renderables::renderable3d::Renderable3d; use crate::renderables::renderable3d::View3dPacket; use crate::scene_renderer::SceneRenderer; @@ -14,8 +15,10 @@ use crate::ViewerRenderState; pub(crate) struct View3d { pub(crate) scene: SceneRenderer, + // At the moment, the pixel renderer is only used for the interaction marker + pub(crate) pixel_for_interaction_marker: PixelRenderer, pub(crate) intrinsics: DynCamera, - pub(crate) offscreen: OffscreenTexture, + pub(crate) offscreen: Option, } impl View3d { @@ -35,14 +38,11 @@ impl View3d { views.insert( packet.view_label.clone(), View::View3d(View3d { - offscreen: OffscreenTexture::new( - state, - &packet.initial_camera.intrinsics.image_size(), - ), + offscreen: None, scene: SceneRenderer::new( state, &packet.initial_camera.intrinsics, - depth_stencil, + depth_stencil.clone(), InteractionEnum::OrbitalInteraction(OrbitalInteraction { maybe_pointer_state: None, maybe_scroll_state: None, @@ -51,6 +51,11 @@ impl View3d { scene_from_camera: packet.initial_camera.scene_from_camera, }), ), + pixel_for_interaction_marker: PixelRenderer::new( + state, + &packet.initial_camera.intrinsics.image_size(), + depth_stencil.clone(), + ), intrinsics: packet.initial_camera.intrinsics.clone(), }), ); @@ -92,18 +97,28 @@ impl View3d { } } } + + pub fn update_offscreen_texture( + &mut self, + render_state: &ViewerRenderState, + view_port_size: &ImageSize, + ) { + match &mut self.offscreen { + Some(offscreen) => { + let current_view_port_size = offscreen.view_port_size(); + if current_view_port_size != view_port_size { + *offscreen = OffscreenTextures::new(render_state, view_port_size); + } + } + None => { + self.offscreen = Some(OffscreenTextures::new(render_state, view_port_size)); + } + } + } } impl HasAspectRatio for View3d { fn aspect_ratio(&self) -> f32 { self.intrinsics.image_size().aspect_ratio() } - - fn view_size(&self) -> ImageSize { - self.intrinsics.image_size() - } - - fn intrinsics(&self) -> &DynCamera { - &self.intrinsics - } }