From 9d2f1b0ff1bbec2ab5c4c738d3d6908a6f9378a7 Mon Sep 17 00:00:00 2001 From: Hauke Strasdat Date: Sun, 30 Jun 2024 22:47:56 -0700 Subject: [PATCH] render tweaks: proper viewport scaling and in-plane interaction (#24) --- Cargo.toml | 20 +- README.md | 4 +- crates/sophus/examples/pose_graph.rs | 4 +- crates/sophus/examples/viewer_ex.rs | 6 +- crates/sophus_viewer/src/actor.rs | 6 +- crates/sophus_viewer/src/interactions.rs | 98 ---- .../src/interactions/inplane_interaction.rs | 33 -- crates/sophus_viewer/src/lib.rs | 8 +- .../sophus_viewer/src/offscreen_renderer.rs | 11 + .../src/offscreen_renderer/pixel_renderer.rs | 267 +++++++++ .../pixel_renderer/line.rs | 11 +- .../pixel_renderer/line_pixel_shader.wgsl | 15 +- .../pixel_renderer/pixel_point.rs | 11 +- .../pixel_renderer/pixel_utils.wgsl | 30 + .../pixel_renderer/point_pixel_shader.wgsl | 16 +- .../src/offscreen_renderer/renderer.rs | 539 ++++++++++++++++++ .../scene_renderer.rs | 84 +-- .../scene_renderer/buffers.rs | 52 +- .../scene_renderer/line.rs | 2 +- .../scene_renderer/line_scene_shader.wgsl | 12 +- .../scene_renderer/mesh.rs | 2 +- .../scene_renderer/mesh_scene_shader.wgsl | 11 +- .../scene_renderer/point.rs | 2 +- .../scene_renderer/point_scene_shader.wgsl | 12 +- .../scene_renderer/scene_utils.wgsl} | 82 ++- .../scene_renderer/textured_mesh.rs | 2 +- .../textured_mesh_scene_shader.wgsl | 11 +- .../textures.rs} | 5 +- crates/sophus_viewer/src/pixel_renderer.rs | 242 -------- .../depth_triangle_scene_shader.wgsl | 0 crates/sophus_viewer/src/simple_viewer.rs | 433 +++----------- crates/sophus_viewer/src/views.rs | 18 + .../sophus_viewer/src/views/aspect_ratio.rs | 61 +- .../sophus_viewer/src/views/interactions.rs | 114 ++++ .../views/interactions/inplane_interaction.rs | 122 ++++ .../interactions/orbit_interaction.rs | 131 +++-- crates/sophus_viewer/src/views/view2d.rs | 187 +----- crates/sophus_viewer/src/views/view3d.rs | 94 +-- 38 files changed, 1520 insertions(+), 1238 deletions(-) delete mode 100644 crates/sophus_viewer/src/interactions.rs delete mode 100644 crates/sophus_viewer/src/interactions/inplane_interaction.rs create mode 100644 crates/sophus_viewer/src/offscreen_renderer.rs create mode 100644 crates/sophus_viewer/src/offscreen_renderer/pixel_renderer.rs rename crates/sophus_viewer/src/{ => offscreen_renderer}/pixel_renderer/line.rs (92%) rename crates/sophus_viewer/src/{ => offscreen_renderer}/pixel_renderer/line_pixel_shader.wgsl (72%) rename crates/sophus_viewer/src/{ => offscreen_renderer}/pixel_renderer/pixel_point.rs (93%) create mode 100644 crates/sophus_viewer/src/offscreen_renderer/pixel_renderer/pixel_utils.wgsl rename crates/sophus_viewer/src/{ => offscreen_renderer}/pixel_renderer/point_pixel_shader.wgsl (69%) create mode 100644 crates/sophus_viewer/src/offscreen_renderer/renderer.rs rename crates/sophus_viewer/src/{ => offscreen_renderer}/scene_renderer.rs (88%) rename crates/sophus_viewer/src/{ => offscreen_renderer}/scene_renderer/buffers.rs (85%) rename crates/sophus_viewer/src/{ => offscreen_renderer}/scene_renderer/line.rs (98%) rename crates/sophus_viewer/src/{ => offscreen_renderer}/scene_renderer/line_scene_shader.wgsl (87%) rename crates/sophus_viewer/src/{ => offscreen_renderer}/scene_renderer/mesh.rs (99%) rename crates/sophus_viewer/src/{ => offscreen_renderer}/scene_renderer/mesh_scene_shader.wgsl (77%) rename crates/sophus_viewer/src/{ => offscreen_renderer}/scene_renderer/point.rs (98%) rename crates/sophus_viewer/src/{ => offscreen_renderer}/scene_renderer/point_scene_shader.wgsl (86%) rename crates/sophus_viewer/src/{scene_renderer/utils.wgsl => offscreen_renderer/scene_renderer/scene_utils.wgsl} (79%) rename crates/sophus_viewer/src/{ => offscreen_renderer}/scene_renderer/textured_mesh.rs (99%) rename crates/sophus_viewer/src/{ => offscreen_renderer}/scene_renderer/textured_mesh_scene_shader.wgsl (83%) rename crates/sophus_viewer/src/{offscreen.rs => offscreen_renderer/textures.rs} (98%) delete mode 100644 crates/sophus_viewer/src/pixel_renderer.rs delete mode 100644 crates/sophus_viewer/src/scene_renderer/depth_triangle_scene_shader.wgsl create mode 100644 crates/sophus_viewer/src/views/interactions.rs create mode 100644 crates/sophus_viewer/src/views/interactions/inplane_interaction.rs rename crates/sophus_viewer/src/{ => views}/interactions/orbit_interaction.rs (67%) diff --git a/Cargo.toml b/Cargo.toml index 691aae0..1e66a09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,18 +21,18 @@ include = [ ] keywords = ["robotics", "optimization"] license = "MIT OR Apache-2.0" -repository = "https://github.com/farm-ng/sophus-rs/" -version = "0.7.0" +repository = "https://github.com/sophus-vision/sophus-rs/" +version = "0.8.0" [workspace.dependencies] -sophus = {path = "crates/sophus", version = "0.7.0"} -sophus_core = {path = "crates/sophus_core", version = "0.7.0"} -sophus_image = {path = "crates/sophus_image", version = "0.7.0"} -sophus_lie = {path = "crates/sophus_lie", version = "0.7.0"} -sophus_opt = {path = "crates/sophus_opt", version = "0.7.0"} -sophus_pyo3 = {path = "crates/sophus_pyo3", version = "0.7.0"} -sophus_sensor = {path = "crates/sophus_sensor", version = "0.7.0"} -sophus_viewer = {path = "crates/sophus_viewer", version = "0.7.0"} +sophus = {path = "crates/sophus", version = "0.8.0"} +sophus_core = {path = "crates/sophus_core", version = "0.8.0"} +sophus_image = {path = "crates/sophus_image", version = "0.8.0"} +sophus_lie = {path = "crates/sophus_lie", version = "0.8.0"} +sophus_opt = {path = "crates/sophus_opt", version = "0.8.0"} +sophus_pyo3 = {path = "crates/sophus_pyo3", version = "0.8.0"} +sophus_sensor = {path = "crates/sophus_sensor", version = "0.8.0"} +sophus_viewer = {path = "crates/sophus_viewer", version = "0.8.0"} approx = "0.5" as-any = "0.3" diff --git a/README.md b/README.md index cbfaa4c..2a8b1c3 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ sophus-rs builds on stable. ```toml [dependencies] -sophus = "0.7.0" +sophus = "0.8.0" ``` To allow for batch types, such as BatchScalarF64, the 'simd' feature is required. This feature @@ -38,5 +38,5 @@ are no plans to rely on any other nightly features. ```toml [dependencies] -sophus = { version = "0.7.0", features = ["simd"] } +sophus = { version = "0.8.0", features = ["simd"] } ``` diff --git a/crates/sophus/examples/pose_graph.rs b/crates/sophus/examples/pose_graph.rs index eb6209a..97d2f3e 100644 --- a/crates/sophus/examples/pose_graph.rs +++ b/crates/sophus/examples/pose_graph.rs @@ -15,7 +15,7 @@ use sophus_lie::Isometry3; use sophus_sensor::camera_enum::perspective_camera::PerspectiveCameraEnum; use sophus_sensor::dyn_camera::DynCamera; use sophus_sensor::KannalaBrandtCamera; -use sophus_viewer::interactions::WgpuClippingPlanes; +use sophus_viewer::offscreen_renderer::renderer::ClippingPlanes; use sophus_viewer::simple_viewer::SimpleViewer; use crate::color::Color; @@ -120,7 +120,7 @@ impl HasOnMessage for ContentGeneratorMessage { }, ); let scene_from_camera = Isometry3::from_t(&VecF64::<3>::new(0.0, 0.0, -50.0)); - let clipping_planes = WgpuClippingPlanes { + let clipping_planes = ClippingPlanes { near: 0.1, far: 1000.0, }; diff --git a/crates/sophus/examples/viewer_ex.rs b/crates/sophus/examples/viewer_ex.rs index 3f8cd49..48a4e0d 100644 --- a/crates/sophus/examples/viewer_ex.rs +++ b/crates/sophus/examples/viewer_ex.rs @@ -15,7 +15,7 @@ use sophus_image::mut_image_view::IsMutImageView; use sophus_lie::traits::IsTranslationProductGroup; use sophus_lie::Isometry3; use sophus_sensor::DynCamera; -use sophus_viewer::interactions::WgpuClippingPlanes; +use sophus_viewer::offscreen_renderer::renderer::ClippingPlanes; use sophus_viewer::renderables::color::Color; use sophus_viewer::renderables::renderable2d::Points2; use sophus_viewer::renderables::renderable2d::Renderable2d; @@ -147,7 +147,7 @@ fn create_tiny_view2_packet() -> Packet { "lines2", &[[[-0.5, -0.5], [0.5, 0.5]], [[0.5, -0.5], [-0.5, 0.5]]], &Color::red(), - 0.3, + 2.0, ); packet_2d.renderables2d.push(Renderable2d::Lines2(lines2)); @@ -157,7 +157,7 @@ fn create_tiny_view2_packet() -> Packet { fn create_view3_packet() -> Packet { let initial_camera = ViewerCamera { intrinsics: DynCamera::default_distorted(ImageSize::new(639, 477)), - clipping_planes: WgpuClippingPlanes::default(), + clipping_planes: ClippingPlanes::default(), scene_from_camera: Isometry3::from_t(&VecF64::<3>::new(0.0, 0.0, -5.0)), }; let mut packet_3d = View3dPacket { diff --git a/crates/sophus_viewer/src/actor.rs b/crates/sophus_viewer/src/actor.rs index f18cf56..b8710c0 100644 --- a/crates/sophus_viewer/src/actor.rs +++ b/crates/sophus_viewer/src/actor.rs @@ -8,7 +8,7 @@ use sophus_lie::traits::IsTranslationProductGroup; use sophus_lie::Isometry3; use sophus_sensor::dyn_camera::DynCamera; -use crate::interactions::WgpuClippingPlanes; +use crate::offscreen_renderer::renderer::ClippingPlanes; use crate::renderables::Packets; use crate::ViewerRenderState; @@ -18,7 +18,7 @@ pub struct ViewerCamera { /// Camera intrinsics pub intrinsics: DynCamera, /// Clipping planes - pub clipping_planes: WgpuClippingPlanes, + pub clipping_planes: ClippingPlanes, /// Scene from camera pose pub scene_from_camera: Isometry3, } @@ -34,7 +34,7 @@ impl ViewerCamera { pub fn default_from(image_size: ImageSize) -> ViewerCamera { ViewerCamera { intrinsics: DynCamera::default_pinhole(image_size), - clipping_planes: WgpuClippingPlanes::default(), + clipping_planes: ClippingPlanes::default(), scene_from_camera: Isometry3::from_t(&VecF64::<3>::new(0.0, 0.0, -5.0)), } } diff --git a/crates/sophus_viewer/src/interactions.rs b/crates/sophus_viewer/src/interactions.rs deleted file mode 100644 index 8d6fead..0000000 --- a/crates/sophus_viewer/src/interactions.rs +++ /dev/null @@ -1,98 +0,0 @@ -/// in-plane interaction -pub mod inplane_interaction; -/// orbit interaction -pub mod orbit_interaction; - -use eframe::egui; -use sophus_core::linalg::VecF32; -use sophus_core::linalg::VecF64; -use sophus_image::arc_image::ArcImageF32; -use sophus_lie::Isometry3; -use sophus_sensor::dyn_camera::DynCamera; - -use crate::interactions::inplane_interaction::InplaneInteraction; -use crate::interactions::orbit_interaction::OrbitalInteraction; - -/// Clipping planes for the Wgpu renderer -#[derive(Clone, Copy, Debug)] -pub struct WgpuClippingPlanes { - /// Near clipping plane - pub near: f64, - /// Far clipping plane - pub far: f64, -} - -impl Default for WgpuClippingPlanes { - fn default() -> Self { - WgpuClippingPlanes { - near: 0.1, - far: 1000.0, - } - } -} - -impl WgpuClippingPlanes { - fn z_from_ndc(&self, ndc: f64) -> f64 { - -(self.far * self.near) / (-self.far + ndc * self.far - ndc * self.near) - } - - pub(crate) fn _ndc_from_z(&self, z: f64) -> f64 { - (self.far * (z - self.near)) / (z * (self.far - self.near)) - } -} - -/// Interaction state for pointer -#[derive(Clone, Copy)] -pub struct InteractionPointerState { - /// Start uv position - pub start_uv: VecF64<2>, -} - -/// Scene focus -#[derive(Clone, Copy)] -pub struct SceneFocus { - /// Depth - pub depth: f64, - /// UV position - pub uv_in_virtual_camera: VecF64<2>, -} - -#[derive(Clone, Copy)] -/// Scroll state -pub struct ScrollState {} - -/// Interaction state -pub enum InteractionEnum { - /// orbit interaction state - OrbitalInteraction(OrbitalInteraction), - /// in-plane interaction state - InplaneInteraction(InplaneInteraction), -} - -impl InteractionEnum { - /// Get scene_from_camera isometry - pub fn scene_from_camera(&self) -> Isometry3 { - match self { - InteractionEnum::OrbitalInteraction(orbit) => orbit.scene_from_camera, - InteractionEnum::InplaneInteraction(inplane) => inplane.scene_from_camera, - } - } - - /// process event - pub fn process_event( - &mut self, - cam: &DynCamera, - response: &egui::Response, - scales: &VecF32<2>, - z_buffer: &ArcImageF32, - ) { - match self { - InteractionEnum::OrbitalInteraction(orbit) => { - orbit.process_event(cam, response, scales, z_buffer) - } - InteractionEnum::InplaneInteraction(inplane) => { - inplane.process_event(cam, response, scales, z_buffer) - } - } - } -} diff --git a/crates/sophus_viewer/src/interactions/inplane_interaction.rs b/crates/sophus_viewer/src/interactions/inplane_interaction.rs deleted file mode 100644 index 77e7f76..0000000 --- a/crates/sophus_viewer/src/interactions/inplane_interaction.rs +++ /dev/null @@ -1,33 +0,0 @@ -use eframe::egui; -use sophus_core::linalg::VecF32; -use sophus_image::arc_image::ArcImageF32; -use sophus_lie::Isometry3; -use sophus_sensor::DynCamera; - -use crate::interactions::InteractionPointerState; -use crate::interactions::SceneFocus; -use crate::interactions::ScrollState; -use crate::interactions::WgpuClippingPlanes; - -#[derive(Clone, Copy)] -/// Interaction state -pub struct InplaneInteraction { - pub(crate) maybe_pointer_state: Option, - pub(crate) maybe_scroll_state: Option, - pub(crate) maybe_scene_focus: Option, - pub(crate) _clipping_planes: WgpuClippingPlanes, - pub(crate) scene_from_camera: Isometry3, -} - -impl InplaneInteraction { - /// Process event - pub fn process_event( - &mut self, - _cam: &DynCamera, - _response: &egui::Response, - _scales: &VecF32<2>, - _z_buffer: &ArcImageF32, - ) { - todo!() - } -} diff --git a/crates/sophus_viewer/src/lib.rs b/crates/sophus_viewer/src/lib.rs index 9438296..1a4a980 100644 --- a/crates/sophus_viewer/src/lib.rs +++ b/crates/sophus_viewer/src/lib.rs @@ -6,16 +6,10 @@ /// The actor for the viewer. pub mod actor; -/// Interactions -pub mod interactions; /// The offscreen texture for rendering. -pub mod offscreen; -/// The pixel renderer for 2D rendering. -pub mod pixel_renderer; +pub mod offscreen_renderer; /// The renderable structs. pub mod renderables; -/// The scene renderer for 3D rendering. -pub mod scene_renderer; /// The simple viewer. pub mod simple_viewer; /// The view struct. diff --git a/crates/sophus_viewer/src/offscreen_renderer.rs b/crates/sophus_viewer/src/offscreen_renderer.rs new file mode 100644 index 0000000..3e3d56b --- /dev/null +++ b/crates/sophus_viewer/src/offscreen_renderer.rs @@ -0,0 +1,11 @@ +/// offscreen texture for rendering +pub mod textures; + +/// Renderer for offscreen rendering +pub mod renderer; + +/// The pixel renderer for 2D rendering. +pub mod pixel_renderer; + +/// The scene renderer for 3D rendering. +pub mod scene_renderer; diff --git a/crates/sophus_viewer/src/offscreen_renderer/pixel_renderer.rs b/crates/sophus_viewer/src/offscreen_renderer/pixel_renderer.rs new file mode 100644 index 0000000..83ca656 --- /dev/null +++ b/crates/sophus_viewer/src/offscreen_renderer/pixel_renderer.rs @@ -0,0 +1,267 @@ +/// Line renderer +pub mod line; +/// Pixel point renderer +pub mod pixel_point; + +use bytemuck::Pod; +use bytemuck::Zeroable; +use sophus_image::ImageSize; +use sophus_sensor::DynCamera; +use std::num::NonZeroU64; +use wgpu::util::DeviceExt; +use wgpu::DepthStencilState; + +use crate::offscreen_renderer::pixel_renderer::line::PixelLineRenderer; +use crate::offscreen_renderer::pixel_renderer::pixel_point::PixelPointRenderer; +use crate::offscreen_renderer::renderer::TranslationAndScaling; +use crate::offscreen_renderer::renderer::Zoom2d; +use crate::offscreen_renderer::textures::ZBufferTexture; +use crate::views::interactions::InteractionEnum; +use crate::ViewerRenderState; + +/// Renderer for pixel data +pub struct PixelRenderer { + pub(crate) uniform_bind_group: wgpu::BindGroup, + pub(crate) ortho_cam_buffer: wgpu::Buffer, + pub(crate) zoom_buffer: wgpu::Buffer, + pub(crate) line_renderer: PixelLineRenderer, + pub(crate) point_renderer: PixelPointRenderer, +} + +#[repr(C)] +// This is so we can store this in a buffer +#[derive(Debug, Copy, Clone, bytemuck::Pod, bytemuck::Zeroable)] +struct OrthoCam { + virtual_camera_width: f32, + virtual_camera_height: f32, + viewport_scale: f32, // virtual_camera_width / viewport_width + dummy: f32, +} + +/// 2D line vertex +#[repr(C)] +#[derive(Clone, Copy, Pod, Zeroable)] +pub struct LineVertex2 { + pub(crate) _pos: [f32; 2], + pub(crate) _color: [f32; 4], + pub(crate) _normal: [f32; 2], + pub(crate) _line_width: f32, +} + +/// 2D point vertex +#[repr(C)] +#[derive(Clone, Copy, Pod, Zeroable)] +pub struct PointVertex2 { + pub(crate) _pos: [f32; 2], + pub(crate) _point_size: f32, + pub(crate) _color: [f32; 4], +} + +impl PixelRenderer { + /// Create a new pixel renderer + pub fn new( + wgpu_render_state: &ViewerRenderState, + image_size: &ImageSize, + depth_stencil: Option, + ) -> Self { + let device = &wgpu_render_state.device; + + let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("pixel render layout"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::VERTEX, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: NonZeroU64::new(16), + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::VERTEX, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: NonZeroU64::new(16), + }, + count: None, + }, + ], + }); + + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("pixel pipeline"), + bind_group_layouts: &[&bind_group_layout], + push_constant_ranges: &[], + }); + + let camera_uniform = OrthoCam { + virtual_camera_width: image_size.width as f32, + virtual_camera_height: image_size.height as f32, + viewport_scale: 1.0, + dummy: 0.0, + }; + + let ortho_cam_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("pixel uniform buffer"), + contents: bytemuck::cast_slice(&[camera_uniform]), + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + }); + + let t_and_s_uniform = Zoom2d::default(); + + let t_and_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("zoom buffer"), + contents: bytemuck::cast_slice(&[t_and_s_uniform]), + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + }); + + let uniform_bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("pixel uniform bind group"), + layout: &bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: ortho_cam_buffer.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: t_and_buffer.as_entire_binding(), + }, + ], + }); + + Self { + ortho_cam_buffer, + uniform_bind_group, + line_renderer: PixelLineRenderer::new( + wgpu_render_state, + &pipeline_layout, + depth_stencil.clone(), + ), + point_renderer: PixelPointRenderer::new( + wgpu_render_state, + &pipeline_layout, + depth_stencil, + ), + zoom_buffer: t_and_buffer, + } + } + + pub(crate) fn show_interaction_marker( + &self, + state: &ViewerRenderState, + interaction: &InteractionEnum, + ) { + if let Some(scene_focus) = interaction.maybe_scene_focus() { + *self.point_renderer.show_interaction_marker.lock().unwrap() = + if interaction.is_active() { + let mut vertex_data = vec![]; + + for _i in 0..6 { + vertex_data.push(PointVertex2 { + _pos: [ + scene_focus.uv_in_virtual_camera[0] as f32, + scene_focus.uv_in_virtual_camera[1] as f32, + ], + _color: [0.5, 0.5, 0.5, 1.0], + _point_size: 5.0, + }); + } + state.queue.write_buffer( + &self.point_renderer.interaction_vertex_buffer, + 0, + bytemuck::cast_slice(&vertex_data), + ); + + true + } else { + false + }; + } + } + + pub(crate) fn clear_vertex_data(&mut self) { + self.line_renderer.vertex_data.clear(); + self.point_renderer.vertex_data.clear(); + } + + pub(crate) fn prepare( + &self, + state: &ViewerRenderState, + viewport_size: &ImageSize, + intrinsics: &DynCamera, + zoom_2d: TranslationAndScaling, + ) { + state.queue.write_buffer( + &self.point_renderer.vertex_buffer, + 0, + bytemuck::cast_slice(self.point_renderer.vertex_data.as_slice()), + ); + state.queue.write_buffer( + &self.line_renderer.vertex_buffer, + 0, + bytemuck::cast_slice(self.line_renderer.vertex_data.as_slice()), + ); + + let ortho_cam_uniforms = OrthoCam { + virtual_camera_width: intrinsics.image_size().width as f32, + virtual_camera_height: intrinsics.image_size().height as f32, + viewport_scale: intrinsics.image_size().width as f32 / viewport_size.width as f32, + dummy: 0.0, + }; + + state.queue.write_buffer( + &self.ortho_cam_buffer, + 0, + bytemuck::cast_slice(&[ortho_cam_uniforms]), + ); + + let zoom_uniform = Zoom2d { + translation_x: zoom_2d.translation[0] as f32, + translation_y: zoom_2d.translation[1] as f32, + scaling_x: zoom_2d.scaling[0] as f32, + scaling_y: zoom_2d.scaling[1] as f32, + }; + + state + .queue + .write_buffer(&self.zoom_buffer, 0, bytemuck::cast_slice(&[zoom_uniform])); + } + + pub(crate) fn paint<'rp>( + &'rp self, + command_encoder: &'rp mut wgpu::CommandEncoder, + texture_view: &'rp wgpu::TextureView, + depth: &ZBufferTexture, + ) { + let mut render_pass = command_encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: None, + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: texture_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: Some(wgpu::RenderPassDepthStencilAttachment { + view: &depth.depth_texture_view, + depth_ops: Some(wgpu::Operations { + load: wgpu::LoadOp::Clear(1.0), + store: wgpu::StoreOp::Store, + }), + stencil_ops: None, + }), + occlusion_query_set: None, + timestamp_writes: None, + }); + self.line_renderer + .paint(&mut render_pass, &self.uniform_bind_group); + self.point_renderer + .paint(&mut render_pass, &self.uniform_bind_group); + } +} diff --git a/crates/sophus_viewer/src/pixel_renderer/line.rs b/crates/sophus_viewer/src/offscreen_renderer/pixel_renderer/line.rs similarity index 92% rename from crates/sophus_viewer/src/pixel_renderer/line.rs rename to crates/sophus_viewer/src/offscreen_renderer/pixel_renderer/line.rs index 4e18e29..6382e2b 100644 --- a/crates/sophus_viewer/src/pixel_renderer/line.rs +++ b/crates/sophus_viewer/src/offscreen_renderer/pixel_renderer/line.rs @@ -4,7 +4,7 @@ use eframe::egui_wgpu::wgpu::util::DeviceExt; use nalgebra::SVector; use wgpu::DepthStencilState; -use crate::pixel_renderer::LineVertex2; +use crate::offscreen_renderer::pixel_renderer::LineVertex2; use crate::renderables::renderable2d::Line2; use crate::ViewerRenderState; @@ -27,7 +27,14 @@ impl PixelLineRenderer { let line_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { label: Some("pixel line shader"), - source: wgpu::ShaderSource::Wgsl(include_str!("./line_pixel_shader.wgsl").into()), + source: wgpu::ShaderSource::Wgsl( + format!( + "{} {}", + include_str!("./pixel_utils.wgsl"), + include_str!("./line_pixel_shader.wgsl") + ) + .into(), + ), }); // hack: generate a buffer of 1000 lines, because vertex buffer cannot be resized diff --git a/crates/sophus_viewer/src/pixel_renderer/line_pixel_shader.wgsl b/crates/sophus_viewer/src/offscreen_renderer/pixel_renderer/line_pixel_shader.wgsl similarity index 72% rename from crates/sophus_viewer/src/pixel_renderer/line_pixel_shader.wgsl rename to crates/sophus_viewer/src/offscreen_renderer/pixel_renderer/line_pixel_shader.wgsl index 50b8b0b..6dd8134 100644 --- a/crates/sophus_viewer/src/pixel_renderer/line_pixel_shader.wgsl +++ b/crates/sophus_viewer/src/offscreen_renderer/pixel_renderer/line_pixel_shader.wgsl @@ -3,14 +3,6 @@ struct VertexOut { @builtin(position) position: vec4, }; -struct Uniforms { - width_height: vec4 -}; - -@group(0) @binding(0) -var uniforms: Uniforms; - - @vertex fn vs_main( @location(0) position: vec2, @@ -21,7 +13,7 @@ fn vs_main( { var out: VertexOut; - var line_half_width = 0.5 * line_width; + var line_half_width = 0.5 * line_width * ortho_camera.viewport_scale; var p = position; var mod6 = idx % 6u; if mod6 == 0u { @@ -44,10 +36,7 @@ fn vs_main( p -= normal * line_half_width; } - out.position = vec4(2.0 * (p.x+0.5) / uniforms.width_height.x - 1.0, - 2.0 - 2.0*(p.y+0.5) / uniforms.width_height.y - 1.0, - 0.0, - 1.0); + out.position = pixel_and_z_to_clip(p, zoom_2d, ortho_camera); out.color = color; return out; diff --git a/crates/sophus_viewer/src/pixel_renderer/pixel_point.rs b/crates/sophus_viewer/src/offscreen_renderer/pixel_renderer/pixel_point.rs similarity index 93% rename from crates/sophus_viewer/src/pixel_renderer/pixel_point.rs rename to crates/sophus_viewer/src/offscreen_renderer/pixel_renderer/pixel_point.rs index fcb4b58..e14b09c 100644 --- a/crates/sophus_viewer/src/pixel_renderer/pixel_point.rs +++ b/crates/sophus_viewer/src/offscreen_renderer/pixel_renderer/pixel_point.rs @@ -4,7 +4,7 @@ use std::sync::Mutex; use eframe::egui_wgpu::wgpu::util::DeviceExt; use wgpu::DepthStencilState; -use crate::pixel_renderer::PointVertex2; +use crate::offscreen_renderer::pixel_renderer::PointVertex2; use crate::renderables::renderable2d::Point2; use crate::ViewerRenderState; @@ -29,7 +29,14 @@ impl PixelPointRenderer { let point_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { label: Some("pixel point shader"), - source: wgpu::ShaderSource::Wgsl(include_str!("./point_pixel_shader.wgsl").into()), + source: wgpu::ShaderSource::Wgsl( + format!( + "{} {}", + include_str!("./pixel_utils.wgsl"), + include_str!("./point_pixel_shader.wgsl") + ) + .into(), + ), }); // hack: generate a buffer of 1000 points, because vertex buffer cannot be resized diff --git a/crates/sophus_viewer/src/offscreen_renderer/pixel_renderer/pixel_utils.wgsl b/crates/sophus_viewer/src/offscreen_renderer/pixel_renderer/pixel_utils.wgsl new file mode 100644 index 0000000..8966a55 --- /dev/null +++ b/crates/sophus_viewer/src/offscreen_renderer/pixel_renderer/pixel_utils.wgsl @@ -0,0 +1,30 @@ +struct OrthoCamera { + width: f32, + height: f32, + viewport_scale: f32, + dummy: f32, +}; + +struct Zoom2d { + translation_x: f32, + translation_y: f32, + scaling_x: f32, + scaling_y: f32, +}; + +@group(0) @binding(0) +var ortho_camera: OrthoCamera; + +@group(0) @binding(1) +var zoom_2d: Zoom2d; + +// apply zoom and convert from pixel to clip space +fn pixel_and_z_to_clip(uv: vec2, zoom_2d: Zoom2d, ortho_camera: OrthoCamera) -> vec4 { + var p_x = uv.x * zoom_2d.scaling_x + zoom_2d.translation_x; + var p_y = uv.y * zoom_2d.scaling_y + zoom_2d.translation_y; + + return vec4(2.0 * (p_x + 0.5) / ortho_camera.width - 1.0, + 2.0 - 2.0 * (p_y + 0.5) / ortho_camera.height - 1.0, + 0.0, + 1.0); +} diff --git a/crates/sophus_viewer/src/pixel_renderer/point_pixel_shader.wgsl b/crates/sophus_viewer/src/offscreen_renderer/pixel_renderer/point_pixel_shader.wgsl similarity index 69% rename from crates/sophus_viewer/src/pixel_renderer/point_pixel_shader.wgsl rename to crates/sophus_viewer/src/offscreen_renderer/pixel_renderer/point_pixel_shader.wgsl index bd1af7f..53316a1 100644 --- a/crates/sophus_viewer/src/pixel_renderer/point_pixel_shader.wgsl +++ b/crates/sophus_viewer/src/offscreen_renderer/pixel_renderer/point_pixel_shader.wgsl @@ -3,14 +3,6 @@ struct VertexOut { @builtin(position) position: vec4, }; -struct Uniforms { - width_height: vec4 -}; - -@group(0) @binding(0) -var uniforms: Uniforms; - - @vertex fn vs_main( @location(0) position: vec2, @@ -19,8 +11,7 @@ fn vs_main( @builtin(vertex_index) idx: u32)-> VertexOut { var out: VertexOut; - var point_radius = 0.5 * point_size; - + var point_radius = 0.5 * point_size * ortho_camera.viewport_scale; var u = position.x; var v = position.y; @@ -39,10 +30,7 @@ fn vs_main( v += point_radius; } - out.position = vec4(2.0 * (u+0.5) / uniforms.width_height.x - 1.0, - 2.0 - 2.0*(v+0.5) / uniforms.width_height.y - 1.0, - 0.0, - 1.0); + out.position = pixel_and_z_to_clip(vec2(u, v), zoom_2d, ortho_camera); out.color = color; return out; diff --git a/crates/sophus_viewer/src/offscreen_renderer/renderer.rs b/crates/sophus_viewer/src/offscreen_renderer/renderer.rs new file mode 100644 index 0000000..09bfd4a --- /dev/null +++ b/crates/sophus_viewer/src/offscreen_renderer/renderer.rs @@ -0,0 +1,539 @@ +use eframe::egui; +use sophus_core::linalg::VecF64; +use sophus_image::arc_image::ArcImage4U8; +use sophus_image::arc_image::GenArcImage; +use sophus_image::ImageSize; +use sophus_lie::Isometry3; +use sophus_sensor::DynCamera; + +use crate::offscreen_renderer::pixel_renderer::LineVertex2; +use crate::offscreen_renderer::pixel_renderer::PixelRenderer; +use crate::offscreen_renderer::pixel_renderer::PointVertex2; +use crate::offscreen_renderer::scene_renderer::line::LineVertex3; +use crate::offscreen_renderer::scene_renderer::mesh::MeshVertex3; +use crate::offscreen_renderer::scene_renderer::point::PointVertex3; +use crate::offscreen_renderer::scene_renderer::textured_mesh::TexturedMeshVertex3; +use crate::offscreen_renderer::scene_renderer::SceneRenderer; +use crate::offscreen_renderer::textures::OffscreenTextures; +use crate::renderables::renderable2d::Renderable2d; +use crate::renderables::renderable3d::Renderable3d; +use crate::renderables::renderable3d::TexturedMesh3; +use crate::views::aspect_ratio::HasAspectRatio; +use crate::views::interactions::InteractionEnum; +use crate::ViewerRenderState; + +/// Clipping planes for the Wgpu renderer +#[derive(Clone, Copy, Debug)] +pub struct ClippingPlanes { + /// Near clipping plane + pub near: f64, + /// Far clipping plane + pub far: f64, +} + +impl ClippingPlanes { + /// default near clipping plabe + pub const DEFAULT_NEAR: f64 = 0.1; + /// default far clipping plabe + pub const DEFAULT_FAR: f64 = 1000.0; +} + +impl Default for ClippingPlanes { + fn default() -> Self { + ClippingPlanes { + near: ClippingPlanes::DEFAULT_NEAR, + far: ClippingPlanes::DEFAULT_FAR, + } + } +} + +impl ClippingPlanes { + pub(crate) fn z_from_ndc(&self, ndc: f64) -> f64 { + -(self.far * self.near) / (-self.far + ndc * self.far - ndc * self.near) + } + + pub(crate) fn _ndc_from_z(&self, z: f64) -> f64 { + (self.far * (z - self.near)) / (z * (self.far - self.near)) + } +} + +/// Offscreen renderer +pub struct OffscreenRenderer { + intrinsics: DynCamera, + state: ViewerRenderState, + scene: SceneRenderer, + pixel: PixelRenderer, + textures: OffscreenTextures, + maybe_background_image: Option, +} + +/// Render result +pub struct OffscreenRenderResult { + /// depth image - might have a greater width than the requested width + pub depth: GenArcImage<2, 0, f32, f32, 1, 1>, + /// rgba texture id + pub rgba_tex_id: egui::TextureId, +} + +impl HasAspectRatio for OffscreenRenderer { + fn aspect_ratio(&self) -> f32 { + self.intrinsics.image_size().aspect_ratio() + } +} + +#[repr(C)] +#[derive(Debug, Copy, Clone, bytemuck::Pod, bytemuck::Zeroable)] +pub(crate) struct Zoom2d { + pub(crate) translation_x: f32, + pub(crate) translation_y: f32, + pub(crate) scaling_x: f32, + pub(crate) scaling_y: f32, +} + +impl Default for Zoom2d { + fn default() -> Self { + Zoom2d { + translation_x: 0.0, + translation_y: 0.0, + scaling_x: 1.0, + scaling_y: 1.0, + } + } +} + +/// Translation and scaling +/// +/// todo: move to sophus_lie +#[derive(Clone, Copy, Debug)] +pub struct TranslationAndScaling { + /// translation + pub translation: VecF64<2>, + /// scaling + pub scaling: VecF64<2>, +} + +impl TranslationAndScaling { + /// identity + pub fn identity() -> Self { + TranslationAndScaling { + translation: VecF64::<2>::zeros(), + scaling: VecF64::<2>::new(1.0, 1.0), + } + } + + /// apply translation and scaling + pub fn apply(&self, xy: VecF64<2>) -> VecF64<2> { + VecF64::<2>::new( + xy[0] * self.scaling[0] + self.translation[0], + xy[1] * self.scaling[1] + self.translation[1], + ) + } +} + +impl OffscreenRenderer { + /// background image plane + pub const BACKGROUND_IMAGE_PLANE: f64 = 900.0; + + /// create new offscreen renderer + pub fn new(state: &ViewerRenderState, intrinsics: &DynCamera) -> Self { + let depth_stencil = Some(wgpu::DepthStencilState { + format: wgpu::TextureFormat::Depth32Float, + depth_write_enabled: true, + depth_compare: wgpu::CompareFunction::Less, + stencil: wgpu::StencilState::default(), + bias: wgpu::DepthBiasState::default(), + }); + Self { + scene: SceneRenderer::new(state, intrinsics, depth_stencil.clone()), + pixel: PixelRenderer::new(state, &intrinsics.image_size(), depth_stencil), + textures: OffscreenTextures::new(state, &intrinsics.image_size()), + intrinsics: intrinsics.clone(), + state: state.clone(), + maybe_background_image: None, + } + } + + /// get intrinsics + pub fn intrinsics(&self) -> DynCamera { + self.intrinsics.clone() + } + + /// reset 2d frame + pub fn reset_2d_frame( + &mut self, + intrinsics: &DynCamera, + maybe_background_image: Option<&ArcImage4U8>, + ) { + self.intrinsics = intrinsics.clone(); + if let Some(background_image) = maybe_background_image { + let w = self.intrinsics.image_size().width; + let h = self.intrinsics.image_size().height; + + let far = Self::BACKGROUND_IMAGE_PLANE; + + let p0 = self + .intrinsics + .cam_unproj_with_z(&VecF64::<2>::new(-0.5, -0.5), far) + .cast(); + let p1 = self + .intrinsics + .cam_unproj_with_z(&VecF64::<2>::new(w as f64 - 0.5, -0.5), far) + .cast(); + let p2 = self + .intrinsics + .cam_unproj_with_z(&VecF64::<2>::new(-0.5, h as f64 - 0.5), far) + .cast(); + let p3 = self + .intrinsics + .cam_unproj_with_z(&VecF64::<2>::new(w as f64 - 0.5, h as f64 - 0.5), far) + .cast(); + + let tex_mesh = TexturedMesh3::make(&[ + [(p0, [0.0, 0.0]), (p1, [1.0, 0.0]), (p2, [0.0, 1.0])], + [(p1, [1.0, 0.0]), (p2, [0.0, 1.0]), (p3, [1.0, 1.0])], + ]); + self.scene + .textured_mesh_renderer + .mesh_table + .insert("background".to_owned(), tex_mesh.mesh); + self.maybe_background_image = Some(background_image.clone()); + } + } + + /// update 2d renderables + pub fn update_2d_renderables(&mut self, renderables: Vec) { + for m in renderables { + match m { + Renderable2d::Lines2(lines) => { + self.pixel + .line_renderer + .lines_table + .insert(lines.name, lines.lines); + } + Renderable2d::Points2(points) => { + self.pixel + .point_renderer + .points_table + .insert(points.name, points.points); + } + } + } + + self.pixel.clear_vertex_data(); + + for (_, points) in self.pixel.point_renderer.points_table.iter() { + for point in points.iter() { + let v = PointVertex2 { + _pos: [point.p[0], point.p[1]], + _color: [point.color.r, point.color.g, point.color.b, point.color.a], + _point_size: point.point_size, + }; + for _i in 0..6 { + self.pixel.point_renderer.vertex_data.push(v); + } + } + } + for (_, lines) in self.pixel.line_renderer.lines_table.iter() { + for line in lines.iter() { + let p0 = line.p0; + let p1 = line.p1; + let d = (p0 - p1).normalize(); + let normal = [d[1], -d[0]]; + + let v0 = LineVertex2 { + _pos: [p0[0], p0[1]], + _normal: normal, + _color: [line.color.r, line.color.g, line.color.b, line.color.a], + _line_width: line.line_width, + }; + let v1 = LineVertex2 { + _pos: [p1[0], p1[1]], + _normal: normal, + _color: [line.color.r, line.color.g, line.color.b, line.color.a], + _line_width: line.line_width, + }; + self.pixel.line_renderer.vertex_data.push(v0); + self.pixel.line_renderer.vertex_data.push(v0); + self.pixel.line_renderer.vertex_data.push(v1); + self.pixel.line_renderer.vertex_data.push(v0); + self.pixel.line_renderer.vertex_data.push(v1); + self.pixel.line_renderer.vertex_data.push(v1); + } + } + for (_, points) in self.scene.point_renderer.point_table.iter() { + for point in points.iter() { + let v = PointVertex3 { + _pos: [point.p[0], point.p[1], point.p[2]], + _color: [point.color.r, point.color.g, point.color.b, point.color.a], + _point_size: point.point_size, + }; + for _i in 0..6 { + self.scene.point_renderer.vertex_data.push(v); + } + } + } + for (_, lines) in self.scene.line_renderer.line_table.iter() { + for line in lines.iter() { + let p0 = line.p0; + let p1 = line.p1; + + let v0 = LineVertex3 { + _p0: [p0[0], p0[1], p0[2]], + _p1: [p1[0], p1[1], p1[2]], + _color: [line.color.r, line.color.g, line.color.b, line.color.a], + _line_width: line.line_width, + }; + let v1 = LineVertex3 { + _p0: [p0[0], p0[1], p0[2]], + _p1: [p1[0], p1[1], p1[2]], + _color: [line.color.r, line.color.g, line.color.b, line.color.a], + _line_width: line.line_width, + }; + self.scene.line_renderer.vertex_data.push(v0); + self.scene.line_renderer.vertex_data.push(v0); + self.scene.line_renderer.vertex_data.push(v1); + self.scene.line_renderer.vertex_data.push(v0); + self.scene.line_renderer.vertex_data.push(v1); + self.scene.line_renderer.vertex_data.push(v1); + } + } + for (_, mesh) in self.scene.mesh_renderer.mesh_table.iter() { + for trig in mesh.iter() { + let v0 = MeshVertex3 { + _pos: [trig.p0[0], trig.p0[1], trig.p0[2]], + _color: [trig.color.r, trig.color.g, trig.color.b, trig.color.a], + }; + let v1 = MeshVertex3 { + _pos: [trig.p1[0], trig.p1[1], trig.p1[2]], + _color: [trig.color.r, trig.color.g, trig.color.b, trig.color.a], + }; + let v2 = MeshVertex3 { + _pos: [trig.p2[0], trig.p2[1], trig.p2[2]], + _color: [trig.color.r, trig.color.g, trig.color.b, trig.color.a], + }; + self.scene.mesh_renderer.vertices.push(v0); + self.scene.mesh_renderer.vertices.push(v1); + self.scene.mesh_renderer.vertices.push(v2); + } + } + for (_, mesh) in self.scene.textured_mesh_renderer.mesh_table.iter() { + for trig in mesh.iter() { + let v0 = TexturedMeshVertex3 { + _pos: [trig.p0[0], trig.p0[1], trig.p0[2]], + _tex: [trig.tex0[0], trig.tex0[1]], + }; + let v1 = TexturedMeshVertex3 { + _pos: [trig.p1[0], trig.p1[1], trig.p1[2]], + _tex: [trig.tex1[0], trig.tex1[1]], + }; + let v2 = TexturedMeshVertex3 { + _pos: [trig.p2[0], trig.p2[1], trig.p2[2]], + _tex: [trig.tex2[0], trig.tex2[1]], + }; + self.scene.textured_mesh_renderer.vertices.push(v0); + self.scene.textured_mesh_renderer.vertices.push(v1); + self.scene.textured_mesh_renderer.vertices.push(v2); + } + } + } + + /// update 3d renerables + pub fn update_3d_renderables(&mut self, renderables: Vec) { + for m in renderables { + match m { + Renderable3d::Lines3(lines3) => { + self.scene + .line_renderer + .line_table + .insert(lines3.name, lines3.lines); + } + Renderable3d::Points3(points3) => { + self.scene + .point_renderer + .point_table + .insert(points3.name, points3.points); + } + Renderable3d::Mesh3(mesh) => { + self.scene + .mesh_renderer + .mesh_table + .insert(mesh.name, mesh.mesh); + } + } + } + + // maybe separate call needed + + self.scene.clear_vertex_data(); + + for (_, points) in self.scene.point_renderer.point_table.iter() { + for point in points.iter() { + let v = PointVertex3 { + _pos: [point.p[0], point.p[1], point.p[2]], + _color: [point.color.r, point.color.g, point.color.b, point.color.a], + _point_size: point.point_size, + }; + for _i in 0..6 { + self.scene.point_renderer.vertex_data.push(v); + } + } + } + for (_, lines) in self.scene.line_renderer.line_table.iter() { + for line in lines.iter() { + let p0 = line.p0; + let p1 = line.p1; + + let v0 = LineVertex3 { + _p0: [p0[0], p0[1], p0[2]], + _p1: [p1[0], p1[1], p1[2]], + _color: [line.color.r, line.color.g, line.color.b, line.color.a], + _line_width: line.line_width, + }; + let v1 = LineVertex3 { + _p0: [p0[0], p0[1], p0[2]], + _p1: [p1[0], p1[1], p1[2]], + _color: [line.color.r, line.color.g, line.color.b, line.color.a], + _line_width: line.line_width, + }; + self.scene.line_renderer.vertex_data.push(v0); + self.scene.line_renderer.vertex_data.push(v0); + self.scene.line_renderer.vertex_data.push(v1); + self.scene.line_renderer.vertex_data.push(v0); + self.scene.line_renderer.vertex_data.push(v1); + self.scene.line_renderer.vertex_data.push(v1); + } + } + for (_, mesh) in self.scene.mesh_renderer.mesh_table.iter() { + for trig in mesh.iter() { + let v0 = MeshVertex3 { + _pos: [trig.p0[0], trig.p0[1], trig.p0[2]], + _color: [trig.color.r, trig.color.g, trig.color.b, trig.color.a], + }; + let v1 = MeshVertex3 { + _pos: [trig.p1[0], trig.p1[1], trig.p1[2]], + _color: [trig.color.r, trig.color.g, trig.color.b, trig.color.a], + }; + let v2 = MeshVertex3 { + _pos: [trig.p2[0], trig.p2[1], trig.p2[2]], + _color: [trig.color.r, trig.color.g, trig.color.b, trig.color.a], + }; + self.scene.mesh_renderer.vertices.push(v0); + self.scene.mesh_renderer.vertices.push(v1); + self.scene.mesh_renderer.vertices.push(v2); + } + } + for (_, mesh) in self.scene.textured_mesh_renderer.mesh_table.iter() { + for trig in mesh.iter() { + let v0 = TexturedMeshVertex3 { + _pos: [trig.p0[0], trig.p0[1], trig.p0[2]], + _tex: [trig.tex0[0], trig.tex0[1]], + }; + let v1 = TexturedMeshVertex3 { + _pos: [trig.p1[0], trig.p1[1], trig.p1[2]], + _tex: [trig.tex1[0], trig.tex1[1]], + }; + let v2 = TexturedMeshVertex3 { + _pos: [trig.p2[0], trig.p2[1], trig.p2[2]], + _tex: [trig.tex2[0], trig.tex2[1]], + }; + self.scene.textured_mesh_renderer.vertices.push(v0); + self.scene.textured_mesh_renderer.vertices.push(v1); + self.scene.textured_mesh_renderer.vertices.push(v2); + } + } + } + + fn render_impl( + &mut self, + view_port_size: &ImageSize, + zoom: TranslationAndScaling, + scene_from_camera: Isometry3, + maybe_interaction_enum: Option<&InteractionEnum>, + ) -> OffscreenRenderResult { + if self.textures.view_port_size != *view_port_size { + self.textures = OffscreenTextures::new(&self.state, view_port_size); + } + + self.scene.prepare( + &self.state, + zoom, + &self.intrinsics, + &scene_from_camera, + &self.maybe_background_image, + ); + + self.maybe_background_image = None; + self.pixel + .prepare(&self.state, view_port_size, &self.intrinsics, zoom); + + let mut command_encoder = self + .state + .device + .create_command_encoder(&wgpu::CommandEncoderDescriptor::default()); + + self.scene.paint( + &mut command_encoder, + &self.textures.rgba.rgba_texture_view, + &self.textures.z_buffer, + ); + + if let Some(interaction_enum) = maybe_interaction_enum { + self.pixel + .show_interaction_marker(&self.state, interaction_enum); + } + + self.pixel.paint( + &mut command_encoder, + &self.textures.rgba.rgba_texture_view, + &self.textures.z_buffer, + ); + + self.state.queue.submit(Some(command_encoder.finish())); + + let mut command_encoder = self + .state + .device + .create_command_encoder(&wgpu::CommandEncoderDescriptor::default()); + self.scene.depth_paint( + &mut command_encoder, + &self.textures.depth.depth_texture_view_f32, + &self.textures.z_buffer, + ); + + let depth_image = + self.textures + .depth + .download_image(&self.state, command_encoder, view_port_size); + + OffscreenRenderResult { + depth: depth_image, + rgba_tex_id: self.textures.rgba.rgba_tex_id, + } + } + + /// render with interaction marker + pub fn render_with_interaction_marker( + &mut self, + view_port_size: &ImageSize, + zoom: TranslationAndScaling, + scene_from_camera: Isometry3, + interaction_enum: &InteractionEnum, + ) -> OffscreenRenderResult { + self.render_impl( + view_port_size, + zoom, + scene_from_camera, + Some(interaction_enum), + ) + } + + /// render + pub fn render( + &mut self, + view_port_size: &ImageSize, + zoom: TranslationAndScaling, + scene_from_camera: Isometry3, + ) -> OffscreenRenderResult { + self.render_impl(view_port_size, zoom, scene_from_camera, None) + } +} diff --git a/crates/sophus_viewer/src/scene_renderer.rs b/crates/sophus_viewer/src/offscreen_renderer/scene_renderer.rs similarity index 88% rename from crates/sophus_viewer/src/scene_renderer.rs rename to crates/sophus_viewer/src/offscreen_renderer/scene_renderer.rs index f0c99f9..0e3113f 100644 --- a/crates/sophus_viewer/src/scene_renderer.rs +++ b/crates/sophus_viewer/src/offscreen_renderer/scene_renderer.rs @@ -13,23 +13,23 @@ pub mod point; /// textured mesh renderer pub mod textured_mesh; -use eframe::egui; use sophus_core::calculus::region::IsRegion; -use sophus_core::linalg::VecF32; use sophus_core::tensor::tensor_view::IsTensorLike; use sophus_image::arc_image::ArcImage4U8; -use sophus_image::arc_image::ArcImageF32; use sophus_image::image_view::IsImageView; +use sophus_lie::Isometry3; use sophus_sensor::distortion_table::distort_table; use sophus_sensor::dyn_camera::DynCamera; use wgpu::DepthStencilState; -use crate::interactions::InteractionEnum; -use crate::offscreen::ZBufferTexture; -use crate::scene_renderer::buffers::Frustum; -use crate::scene_renderer::buffers::SceneRenderBuffers; -use crate::scene_renderer::mesh::MeshRenderer; -use crate::scene_renderer::point::ScenePointRenderer; +use crate::offscreen_renderer::renderer::ClippingPlanes; +use crate::offscreen_renderer::renderer::TranslationAndScaling; +use crate::offscreen_renderer::renderer::Zoom2d; +use crate::offscreen_renderer::scene_renderer::buffers::Frustum; +use crate::offscreen_renderer::scene_renderer::buffers::SceneRenderBuffers; +use crate::offscreen_renderer::scene_renderer::mesh::MeshRenderer; +use crate::offscreen_renderer::scene_renderer::point::ScenePointRenderer; +use crate::offscreen_renderer::textures::ZBufferTexture; use crate::ViewerRenderState; /// Scene renderer @@ -44,8 +44,6 @@ pub struct SceneRenderer { pub point_renderer: ScenePointRenderer, /// Line renderer pub line_renderer: line::SceneLineRenderer, - /// Interaction state - pub interaction: InteractionEnum, } impl SceneRenderer { @@ -54,7 +52,6 @@ impl SceneRenderer { wgpu_render_state: &ViewerRenderState, intrinsics: &DynCamera, depth_stencil: Option, - interaction: InteractionEnum, ) -> Self { let device = &wgpu_render_state.device; @@ -73,7 +70,7 @@ impl SceneRenderer { }, count: None, }, - // view-transform uniform + // zoom table uniform wgpu::BindGroupLayoutEntry { binding: 1, visibility: wgpu::ShaderStages::VERTEX, @@ -95,6 +92,17 @@ impl SceneRenderer { }, count: None, }, + // view-transform uniform + wgpu::BindGroupLayoutEntry { + binding: 3, + visibility: wgpu::ShaderStages::VERTEX, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, ], }); @@ -169,21 +177,9 @@ impl SceneRenderer { &pipeline_layout, depth_stencil, ), - interaction, } } - pub(crate) fn process_event( - &mut self, - cam: &DynCamera, - response: &egui::Response, - scales: &VecF32<2>, - z_buffer: ArcImageF32, - ) { - self.interaction - .process_event(cam, response, scales, &z_buffer); - } - pub(crate) fn paint<'rp>( &'rp self, command_encoder: &'rp mut wgpu::CommandEncoder, @@ -292,7 +288,9 @@ impl SceneRenderer { pub(crate) fn prepare( &self, state: &ViewerRenderState, + zoom_2d: TranslationAndScaling, intrinsics: &DynCamera, + scene_from_camera: &Isometry3, background_image: &Option, ) { state.queue.write_buffer( @@ -316,11 +314,12 @@ impl SceneRenderer { bytemuck::cast_slice(self.textured_mesh_renderer.vertices.as_slice()), ); + let frustum_uniforms = Frustum { camera_image_width: intrinsics.image_size().width as f32, camera_image_height: intrinsics.image_size().height as f32, - near: 0.1, - far: 1000.0, + near: ClippingPlanes::DEFAULT_NEAR as f32, + far: ClippingPlanes::DEFAULT_FAR as f32, fx: intrinsics.pinhole_params()[0] as f32, fy: intrinsics.pinhole_params()[1] as f32, px: intrinsics.pinhole_params()[2] as f32, @@ -333,17 +332,17 @@ impl SceneRenderer { bytemuck::cast_slice(&[frustum_uniforms]), ); - let mut scene_from_camera_uniform: [[f32; 4]; 4] = [[0.0; 4]; 4]; - for i in 0..4 { - for j in 0..4 { - scene_from_camera_uniform[j][i] = - self.interaction.scene_from_camera().inverse().matrix()[(i, j)] as f32; - } - } + let zoom_uniform = Zoom2d { + translation_x: zoom_2d.translation[0] as f32, + translation_y: zoom_2d.translation[1] as f32, + scaling_x: zoom_2d.scaling[0] as f32, + scaling_y: zoom_2d.scaling[1] as f32, + }; + state.queue.write_buffer( - &self.buffers.view_uniform_buffer, + &self.buffers.zoom_buffer, 0, - bytemuck::cast_slice(&[scene_from_camera_uniform]), + bytemuck::cast_slice(&[zoom_uniform]), ); // distortion table @@ -379,6 +378,19 @@ impl SceneRenderer { ); } + let mut scene_from_camera_uniform: [[f32; 4]; 4] = [[0.0; 4]; 4]; + for i in 0..4 { + for j in 0..4 { + scene_from_camera_uniform[j][i] = + scene_from_camera.inverse().matrix()[(i, j)] as f32; + } + } + state.queue.write_buffer( + &self.buffers.view_uniform_buffer, + 0, + bytemuck::cast_slice(&[scene_from_camera_uniform]), + ); + if let Some(background_image) = background_image { state.queue.write_texture( wgpu::ImageCopyTexture { diff --git a/crates/sophus_viewer/src/scene_renderer/buffers.rs b/crates/sophus_viewer/src/offscreen_renderer/scene_renderer/buffers.rs similarity index 85% rename from crates/sophus_viewer/src/scene_renderer/buffers.rs rename to crates/sophus_viewer/src/offscreen_renderer/scene_renderer/buffers.rs index f5ac268..918c29c 100644 --- a/crates/sophus_viewer/src/scene_renderer/buffers.rs +++ b/crates/sophus_viewer/src/offscreen_renderer/scene_renderer/buffers.rs @@ -4,6 +4,8 @@ use sophus_sensor::distortion_table::DistortTable; use sophus_sensor::DynCamera; use wgpu::util::DeviceExt; +use crate::offscreen_renderer::renderer::ClippingPlanes; +use crate::offscreen_renderer::renderer::Zoom2d; use crate::ViewerRenderState; #[repr(C)] @@ -43,6 +45,7 @@ pub struct SceneRenderBuffers { pub(crate) frustum_uniform_buffer: wgpu::Buffer, pub(crate) view_uniform_buffer: wgpu::Buffer, pub(crate) camara_params_buffer: wgpu::Buffer, + pub(crate) zoom_buffer: wgpu::Buffer, pub(crate) dist_texture: wgpu::Texture, pub(crate) dist_bind_group: wgpu::BindGroup, pub(crate) distortion_lut: Mutex>, @@ -58,7 +61,7 @@ impl SceneRenderBuffers { let device = &wgpu_render_state.device; let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { - label: Some("custom3d"), + label: Some("scene render layout"), entries: &[ wgpu::BindGroupLayoutEntry { binding: 0, @@ -90,6 +93,16 @@ impl SceneRenderBuffers { }, count: None, }, + wgpu::BindGroupLayoutEntry { + binding: 3, + visibility: wgpu::ShaderStages::VERTEX, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, ], }); @@ -103,8 +116,8 @@ impl SceneRenderBuffers { let frustum_uniforms = Frustum { camera_image_width: intrinsics.image_size().width as f32, camera_image_height: intrinsics.image_size().height as f32, - near: 0.1, - far: 1000.0, + near: ClippingPlanes::DEFAULT_NEAR as f32, + far: ClippingPlanes::DEFAULT_FAR as f32, fx: intrinsics.pinhole_params()[0] as f32, fy: intrinsics.pinhole_params()[1] as f32, px: intrinsics.pinhole_params()[2] as f32, @@ -112,21 +125,29 @@ impl SceneRenderBuffers { }; let frustum_uniform_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { - label: Some("Camera Buffer"), + label: Some("frustum buffer"), contents: bytemuck::cast_slice(&[frustum_uniforms]), usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, }); - let lut_uniforms = DistortionLut { + let zoom_uniform = Zoom2d::default(); + + let zoom_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("zoom buffer"), + contents: bytemuck::cast_slice(&[zoom_uniform]), + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + }); + + let distortion_uniforms = DistortionLut { lut_offset_x: 0.0, lut_offset_y: 0.0, lut_range_x: 0.0, lut_range_y: 0.0, }; - let camara_params_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { - label: Some("Lut Buffer"), - contents: bytemuck::cast_slice(&[lut_uniforms]), + let distortion_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("distortion uniform buffer"), + contents: bytemuck::cast_slice(&[distortion_uniforms]), usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, }); @@ -135,13 +156,13 @@ impl SceneRenderBuffers { }; let view_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { - label: Some("Camera Buffer2"), + label: Some("view buffer"), contents: bytemuck::cast_slice(&[view_uniforms]), usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, }); let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("custom3d"), + label: Some("scene render bind group"), layout: &bind_group_layout, entries: &[ wgpu::BindGroupEntry { @@ -150,11 +171,15 @@ impl SceneRenderBuffers { }, wgpu::BindGroupEntry { binding: 1, - resource: view_buffer.as_entire_binding(), + resource: zoom_buffer.as_entire_binding(), }, wgpu::BindGroupEntry { binding: 2, - resource: camara_params_buffer.as_entire_binding(), + resource: distortion_buffer.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 3, + resource: view_buffer.as_entire_binding(), }, ], }); @@ -273,12 +298,13 @@ impl SceneRenderBuffers { bind_group, frustum_uniform_buffer, view_uniform_buffer: view_buffer, - camara_params_buffer, + camara_params_buffer: distortion_buffer, dist_texture, dist_bind_group, distortion_lut: Mutex::new(None), background_bind_group, background_texture, + zoom_buffer, } } } diff --git a/crates/sophus_viewer/src/scene_renderer/line.rs b/crates/sophus_viewer/src/offscreen_renderer/scene_renderer/line.rs similarity index 98% rename from crates/sophus_viewer/src/scene_renderer/line.rs rename to crates/sophus_viewer/src/offscreen_renderer/scene_renderer/line.rs index 2a2eaa6..b45839d 100644 --- a/crates/sophus_viewer/src/scene_renderer/line.rs +++ b/crates/sophus_viewer/src/offscreen_renderer/scene_renderer/line.rs @@ -41,7 +41,7 @@ impl SceneLineRenderer { source: wgpu::ShaderSource::Wgsl( format!( "{} {}", - include_str!("./utils.wgsl"), + include_str!("./scene_utils.wgsl"), include_str!("./line_scene_shader.wgsl") ) .into(), diff --git a/crates/sophus_viewer/src/scene_renderer/line_scene_shader.wgsl b/crates/sophus_viewer/src/offscreen_renderer/scene_renderer/line_scene_shader.wgsl similarity index 87% rename from crates/sophus_viewer/src/scene_renderer/line_scene_shader.wgsl rename to crates/sophus_viewer/src/offscreen_renderer/scene_renderer/line_scene_shader.wgsl index 310096b..eb7057d 100644 --- a/crates/sophus_viewer/src/scene_renderer/line_scene_shader.wgsl +++ b/crates/sophus_viewer/src/offscreen_renderer/scene_renderer/line_scene_shader.wgsl @@ -3,16 +3,6 @@ struct VertexOut { @builtin(position) position: vec4, }; - -@group(0) @binding(0) -var frustum_uniforms: Frustum; - -@group(0) @binding(1) -var view_uniform: ViewTransform; - -@group(0) @binding(2) -var lut_uniform: DistortionLut; - @group(1) @binding(0) var distortion_texture: texture_2d; @@ -66,7 +56,7 @@ fn vs_main( // map point from pixel coordinates (Computer Vision convention) to clip space coordinates (WebGPU convention) - out.position = pixel_and_z_to_clip(uv, z, frustum_uniforms); + out.position = pixel_and_z_to_clip(uv, z, frustum_uniforms, zoom); out.color = color; return out; diff --git a/crates/sophus_viewer/src/scene_renderer/mesh.rs b/crates/sophus_viewer/src/offscreen_renderer/scene_renderer/mesh.rs similarity index 99% rename from crates/sophus_viewer/src/scene_renderer/mesh.rs rename to crates/sophus_viewer/src/offscreen_renderer/scene_renderer/mesh.rs index 9784ac0..7c213b4 100644 --- a/crates/sophus_viewer/src/scene_renderer/mesh.rs +++ b/crates/sophus_viewer/src/offscreen_renderer/scene_renderer/mesh.rs @@ -39,7 +39,7 @@ impl MeshRenderer { source: wgpu::ShaderSource::Wgsl( format!( "{} {}", - include_str!("./utils.wgsl"), + include_str!("./scene_utils.wgsl"), include_str!("./mesh_scene_shader.wgsl") ) .into(), diff --git a/crates/sophus_viewer/src/scene_renderer/mesh_scene_shader.wgsl b/crates/sophus_viewer/src/offscreen_renderer/scene_renderer/mesh_scene_shader.wgsl similarity index 77% rename from crates/sophus_viewer/src/scene_renderer/mesh_scene_shader.wgsl rename to crates/sophus_viewer/src/offscreen_renderer/scene_renderer/mesh_scene_shader.wgsl index b56f3e5..e0d2c69 100644 --- a/crates/sophus_viewer/src/scene_renderer/mesh_scene_shader.wgsl +++ b/crates/sophus_viewer/src/offscreen_renderer/scene_renderer/mesh_scene_shader.wgsl @@ -3,15 +3,6 @@ struct VertexOut { @builtin(position) position: vec4, }; -@group(0) @binding(0) -var frustum_uniforms: Frustum; - -@group(0) @binding(1) -var view_uniform: ViewTransform; - -@group(0) @binding(2) -var lut_uniform: DistortionLut; - @group(1) @binding(0) var distortion_texture: texture_2d; @@ -23,7 +14,7 @@ fn vs_main( { var out: VertexOut; var uv_z = scene_point_to_distorted(position, view_uniform, frustum_uniforms, lut_uniform); - out.position = pixel_and_z_to_clip(uv_z.xy, uv_z.z, frustum_uniforms); + out.position = pixel_and_z_to_clip(uv_z.xy, uv_z.z, frustum_uniforms, zoom); out.color = color; return out; } diff --git a/crates/sophus_viewer/src/scene_renderer/point.rs b/crates/sophus_viewer/src/offscreen_renderer/scene_renderer/point.rs similarity index 98% rename from crates/sophus_viewer/src/scene_renderer/point.rs rename to crates/sophus_viewer/src/offscreen_renderer/scene_renderer/point.rs index 2108135..205aa21 100644 --- a/crates/sophus_viewer/src/scene_renderer/point.rs +++ b/crates/sophus_viewer/src/offscreen_renderer/scene_renderer/point.rs @@ -39,7 +39,7 @@ impl ScenePointRenderer { source: wgpu::ShaderSource::Wgsl( format!( "{} {}", - include_str!("./utils.wgsl"), + include_str!("./scene_utils.wgsl"), include_str!("./point_scene_shader.wgsl") ) .into(), diff --git a/crates/sophus_viewer/src/scene_renderer/point_scene_shader.wgsl b/crates/sophus_viewer/src/offscreen_renderer/scene_renderer/point_scene_shader.wgsl similarity index 86% rename from crates/sophus_viewer/src/scene_renderer/point_scene_shader.wgsl rename to crates/sophus_viewer/src/offscreen_renderer/scene_renderer/point_scene_shader.wgsl index 74dc52b..aa11c66 100644 --- a/crates/sophus_viewer/src/scene_renderer/point_scene_shader.wgsl +++ b/crates/sophus_viewer/src/offscreen_renderer/scene_renderer/point_scene_shader.wgsl @@ -3,16 +3,6 @@ struct VertexOut { @builtin(position) position: vec4, }; - -@group(0) @binding(0) -var frustum_uniforms: Frustum; - -@group(0) @binding(1) -var view_uniform: ViewTransform; - -@group(0) @binding(2) -var lut_uniform: DistortionLut; - @group(1) @binding(0) var distortion_texture: texture_2d; @@ -49,7 +39,7 @@ fn vs_main( } // map point from pixel coordinates (Computer Vision convention) to clip space coordinates (WebGPU convention) - out.position = pixel_and_z_to_clip(vec2(u, v), z, frustum_uniforms); + out.position = pixel_and_z_to_clip(vec2(u, v), z, frustum_uniforms, zoom); out.color = color; return out; diff --git a/crates/sophus_viewer/src/scene_renderer/utils.wgsl b/crates/sophus_viewer/src/offscreen_renderer/scene_renderer/scene_utils.wgsl similarity index 79% rename from crates/sophus_viewer/src/scene_renderer/utils.wgsl rename to crates/sophus_viewer/src/offscreen_renderer/scene_renderer/scene_utils.wgsl index dc7f483..96a994a 100644 --- a/crates/sophus_viewer/src/scene_renderer/utils.wgsl +++ b/crates/sophus_viewer/src/offscreen_renderer/scene_renderer/scene_utils.wgsl @@ -1,8 +1,46 @@ +struct Frustum { + camera_image_width: f32, // <= NOT the viewport width + camera_image_height: f32, // <= NOT the viewport height + near: f32, + far: f32, + // fx, fy, px, py or just here to debug distortion lut table. + fx: f32, + fy: f32, + px: f32, + py: f32, +}; + +struct Zoom2d { + translation_x: f32, + translation_y: f32, + scaling_x: f32, + scaling_y: f32, +}; + +struct DistortionLut { + lut_offset_x: f32, + lut_offset_y: f32, + lut_range_x: f32, + lut_range_y: f32, +}; struct ViewTransform { scene_from_camera: mat4x4, }; +@group(0) @binding(0) +var frustum_uniforms: Frustum; + +@group(0) @binding(1) +var zoom: Zoom2d; + +@group(0) @binding(2) +var lut_uniform: DistortionLut; + +@group(0) @binding(3) +var view_uniform: ViewTransform; + + fn scene_point_to_z1_plane(scene_point: vec3, view: ViewTransform) -> vec3 { var scene_from_camera = view.scene_from_camera; @@ -18,26 +56,6 @@ fn scene_point_to_z1_plane(scene_point: vec3, return vec3(point_in_proj.x, point_in_proj.y, z); } - -struct Frustum { - camera_image_width: f32, // <= NOT the viewport width - camera_image_height: f32, // <= NOT the viewport height - near: f32, - far: f32, - // fx, fy, px, py or just here to debug distortion lut table. - fx: f32, - fy: f32, - px: f32, - py: f32, -}; - -struct DistortionLut { - lut_offset_x: f32, - lut_offset_y: f32, - lut_range_x: f32, - lut_range_y: f32, -}; - fn z1_plane_to_distorted(point_in_proj: vec3, frustum: Frustum, lut: DistortionLut) -> vec3 { var width = frustum.camera_image_width; var height = frustum.camera_image_height; @@ -46,8 +64,18 @@ fn z1_plane_to_distorted(point_in_proj: vec3, frustum: Frustum, lut: Distor var lut_range_x = lut.lut_range_x; var lut_range_y = lut.lut_range_y; - let x_lut = clamp((width -1.0 ) * (point_in_proj.x -lut_offset_x) / lut_range_x, 0.0, width - 1.00001); - let y_lut = clamp((height -1.0 ) * (point_in_proj.y -lut_offset_y) / lut_range_y, 0.0, height - 1.00001); + let x_lut = + clamp( + (width -1.0 ) * (point_in_proj.x -lut_offset_x) / lut_range_x, + 0.0, + width - 1.00001 + ); + let y_lut = + clamp( + (height -1.0 ) * (point_in_proj.y -lut_offset_y) / lut_range_y, + 0.0, + height - 1.00001 + ); // Manual implementation of bilinear interpolation. // This is to workaround apparent limitations of wgpu - such as no/limited support for @@ -85,18 +113,18 @@ fn scene_point_to_distorted(scene_point: vec3, } // map point from pixel coordinates (Computer Vision convention) to clip space coordinates (WebGPU convention) -fn pixel_and_z_to_clip(uv_z: vec2, z: f32, frustum: Frustum) -> vec4 { +fn pixel_and_z_to_clip(uv_z: vec2, z: f32, frustum: Frustum, zoom: Zoom2d) -> vec4 { var width = frustum.camera_image_width; var height = frustum.camera_image_height; var near = frustum.near; var far = frustum.far; - var u = uv_z.x; - var v = uv_z.y; + var u = uv_z.x * zoom.scaling_x + zoom.translation_x; + var v = uv_z.y * zoom.scaling_y + zoom.translation_y; let z_clip = (far * (z - near)) / (z * (far - near)); - return vec4(2.0 * ((u+0.5) / width - 0.5), - -2.0 * ((v+0.5) / height - 0.5), + return vec4(2.0 * ((u + 0.5) / width - 0.5), + -2.0 * ((v + 0.5) / height - 0.5), z_clip, 1.0); } diff --git a/crates/sophus_viewer/src/scene_renderer/textured_mesh.rs b/crates/sophus_viewer/src/offscreen_renderer/scene_renderer/textured_mesh.rs similarity index 99% rename from crates/sophus_viewer/src/scene_renderer/textured_mesh.rs rename to crates/sophus_viewer/src/offscreen_renderer/scene_renderer/textured_mesh.rs index be335cf..bc88ad8 100644 --- a/crates/sophus_viewer/src/scene_renderer/textured_mesh.rs +++ b/crates/sophus_viewer/src/offscreen_renderer/scene_renderer/textured_mesh.rs @@ -39,7 +39,7 @@ impl TexturedMeshRenderer { source: wgpu::ShaderSource::Wgsl( format!( "{} {}", - include_str!("./utils.wgsl"), + include_str!("./scene_utils.wgsl"), include_str!("./textured_mesh_scene_shader.wgsl") ) .into(), diff --git a/crates/sophus_viewer/src/scene_renderer/textured_mesh_scene_shader.wgsl b/crates/sophus_viewer/src/offscreen_renderer/scene_renderer/textured_mesh_scene_shader.wgsl similarity index 83% rename from crates/sophus_viewer/src/scene_renderer/textured_mesh_scene_shader.wgsl rename to crates/sophus_viewer/src/offscreen_renderer/scene_renderer/textured_mesh_scene_shader.wgsl index e3ac696..3f4f879 100644 --- a/crates/sophus_viewer/src/scene_renderer/textured_mesh_scene_shader.wgsl +++ b/crates/sophus_viewer/src/offscreen_renderer/scene_renderer/textured_mesh_scene_shader.wgsl @@ -3,15 +3,6 @@ struct VertexOut { @builtin(position) position: vec4, }; -@group(0) @binding(0) -var frustum_uniforms: Frustum; - -@group(0) @binding(1) -var view_uniform: ViewTransform; - -@group(0) @binding(2) -var lut_uniform: DistortionLut; - @group(1) @binding(0) var distortion_texture: texture_2d; @@ -30,7 +21,7 @@ fn vs_main( ) -> VertexOut { var out: VertexOut; var uv_z = scene_point_to_distorted(position, view_uniform, frustum_uniforms, lut_uniform); - out.position = pixel_and_z_to_clip(uv_z.xy, uv_z.z, frustum_uniforms); + out.position = pixel_and_z_to_clip(uv_z.xy, uv_z.z, frustum_uniforms, zoom); out.texCoords = texCoords; // Pass texture coordinates to the fragment shader return out; } diff --git a/crates/sophus_viewer/src/offscreen.rs b/crates/sophus_viewer/src/offscreen_renderer/textures.rs similarity index 98% rename from crates/sophus_viewer/src/offscreen.rs rename to crates/sophus_viewer/src/offscreen_renderer/textures.rs index 860d2db..bd95104 100644 --- a/crates/sophus_viewer/src/offscreen.rs +++ b/crates/sophus_viewer/src/offscreen_renderer/textures.rs @@ -250,16 +250,13 @@ impl DepthTexture { #[derive(Debug)] pub(crate) struct OffscreenTextures { - view_port_size: ImageSize, + pub(crate) view_port_size: ImageSize, pub(crate) rgba: RgbaTexture, pub(crate) z_buffer: ZBufferTexture, pub(crate) depth: DepthTexture, } impl OffscreenTextures { - pub(crate) fn view_port_size(&self) -> &ImageSize { - &self.view_port_size - } pub(crate) fn new(render_state: &ViewerRenderState, view_port_size: &ImageSize) -> Self { Self { view_port_size: *view_port_size, diff --git a/crates/sophus_viewer/src/pixel_renderer.rs b/crates/sophus_viewer/src/pixel_renderer.rs deleted file mode 100644 index 9c2b6aa..0000000 --- a/crates/sophus_viewer/src/pixel_renderer.rs +++ /dev/null @@ -1,242 +0,0 @@ -/// Line renderer -pub mod line; -/// Pixel point renderer -pub mod pixel_point; - -use bytemuck::Pod; -use bytemuck::Zeroable; -use sophus_image::ImageSize; -use std::num::NonZeroU64; -use wgpu::util::DeviceExt; -use wgpu::DepthStencilState; - -use crate::interactions::InteractionEnum; -use crate::offscreen::ZBufferTexture; -use crate::pixel_renderer::line::PixelLineRenderer; -use crate::pixel_renderer::pixel_point::PixelPointRenderer; -use crate::ViewerRenderState; - -/// Renderer for pixel data -pub struct PixelRenderer { - pub(crate) uniform_bind_group: wgpu::BindGroup, - pub(crate) _uniform_buffer: wgpu::Buffer, - pub(crate) line_renderer: PixelLineRenderer, - pub(crate) point_renderer: PixelPointRenderer, -} - -#[repr(C)] -// This is so we can store this in a buffer -#[derive(Debug, Copy, Clone, bytemuck::Pod, bytemuck::Zeroable)] -struct OrthoCam { - width: f32, - height: f32, - dummy0: f32, - dummy1: f32, -} - -/// 2D line vertex -#[repr(C)] -#[derive(Clone, Copy, Pod, Zeroable)] -pub struct LineVertex2 { - pub(crate) _pos: [f32; 2], - pub(crate) _color: [f32; 4], - pub(crate) _normal: [f32; 2], - pub(crate) _line_width: f32, -} - -/// 2D point vertex -#[repr(C)] -#[derive(Clone, Copy, Pod, Zeroable)] -pub struct PointVertex2 { - pub(crate) _pos: [f32; 2], - pub(crate) _point_size: f32, - pub(crate) _color: [f32; 4], -} - -impl PixelRenderer { - /// Create a new pixel renderer - pub fn new( - wgpu_render_state: &ViewerRenderState, - image_size: &ImageSize, - depth_stencil: Option, - ) -> Self { - let device = &wgpu_render_state.device; - - let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { - label: Some("pixel group layout"), - entries: &[wgpu::BindGroupLayoutEntry { - binding: 0, - visibility: wgpu::ShaderStages::VERTEX, - ty: wgpu::BindingType::Buffer { - ty: wgpu::BufferBindingType::Uniform, - has_dynamic_offset: false, - min_binding_size: NonZeroU64::new(16), - }, - count: None, - }], - }); - - let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { - label: Some("pixel pipeline"), - bind_group_layouts: &[&bind_group_layout], - push_constant_ranges: &[], - }); - - let camera_uniform = OrthoCam { - width: image_size.width as f32, - height: image_size.height as f32, - dummy0: 0.0, - dummy1: 0.0, - }; - - let uniform_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { - label: Some("pixel uniform buffer"), - contents: bytemuck::cast_slice(&[camera_uniform]), - usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, - }); - - let uniform_bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("pixel uniform bind group"), - layout: &bind_group_layout, - entries: &[wgpu::BindGroupEntry { - binding: 0, - resource: uniform_buffer.as_entire_binding(), - }], - }); - - Self { - _uniform_buffer: uniform_buffer, - uniform_bind_group, - line_renderer: PixelLineRenderer::new( - wgpu_render_state, - &pipeline_layout, - depth_stencil.clone(), - ), - point_renderer: PixelPointRenderer::new( - wgpu_render_state, - &pipeline_layout, - depth_stencil, - ), - } - } - - pub(crate) fn show_interaction_marker( - &self, - state: &ViewerRenderState, - interaction_state: &InteractionEnum, - ) { - match interaction_state { - InteractionEnum::OrbitalInteraction(orbital_interaction) => { - if let Some(scene_focus) = orbital_interaction.maybe_scene_focus { - *self.point_renderer.show_interaction_marker.lock().unwrap() = - if orbital_interaction.maybe_pointer_state.is_some() - || orbital_interaction.maybe_scroll_state.is_some() - { - let mut vertex_data = vec![]; - - for _i in 0..6 { - vertex_data.push(PointVertex2 { - _pos: [ - scene_focus.uv_in_virtual_camera[0] as f32, - scene_focus.uv_in_virtual_camera[1] as f32, - ], - _color: [0.5, 0.5, 0.5, 1.0], - _point_size: 5.0, - }); - } - state.queue.write_buffer( - &self.point_renderer.interaction_vertex_buffer, - 0, - bytemuck::cast_slice(&vertex_data), - ); - - true - } else { - false - }; - } - } - InteractionEnum::InplaneInteraction(inplane_interaction) => { - if let Some(scene_focus) = inplane_interaction.maybe_scene_focus { - *self.point_renderer.show_interaction_marker.lock().unwrap() = - if inplane_interaction.maybe_pointer_state.is_some() - || inplane_interaction.maybe_scroll_state.is_some() - { - let mut vertex_data = vec![]; - - for _i in 0..6 { - vertex_data.push(PointVertex2 { - _pos: [ - scene_focus.uv_in_virtual_camera[0] as f32, - scene_focus.uv_in_virtual_camera[1] as f32, - ], - _color: [0.5, 0.5, 0.5, 1.0], - _point_size: 5.0, - }); - } - state.queue.write_buffer( - &self.point_renderer.interaction_vertex_buffer, - 0, - bytemuck::cast_slice(&vertex_data), - ); - - true - } else { - false - }; - } - } - } - } - - pub(crate) fn clear_vertex_data(&mut self) { - self.line_renderer.vertex_data.clear(); - self.point_renderer.vertex_data.clear(); - } - - pub(crate) fn prepare(&self, state: &ViewerRenderState) { - state.queue.write_buffer( - &self.point_renderer.vertex_buffer, - 0, - bytemuck::cast_slice(self.point_renderer.vertex_data.as_slice()), - ); - state.queue.write_buffer( - &self.line_renderer.vertex_buffer, - 0, - bytemuck::cast_slice(self.line_renderer.vertex_data.as_slice()), - ); - } - - pub(crate) fn paint<'rp>( - &'rp self, - command_encoder: &'rp mut wgpu::CommandEncoder, - texture_view: &'rp wgpu::TextureView, - depth: &ZBufferTexture, - ) { - let mut render_pass = command_encoder.begin_render_pass(&wgpu::RenderPassDescriptor { - label: None, - color_attachments: &[Some(wgpu::RenderPassColorAttachment { - view: texture_view, - resolve_target: None, - ops: wgpu::Operations { - load: wgpu::LoadOp::Load, - store: wgpu::StoreOp::Store, - }, - })], - depth_stencil_attachment: Some(wgpu::RenderPassDepthStencilAttachment { - view: &depth.depth_texture_view, - depth_ops: Some(wgpu::Operations { - load: wgpu::LoadOp::Clear(1.0), - store: wgpu::StoreOp::Store, - }), - stencil_ops: None, - }), - occlusion_query_set: None, - timestamp_writes: None, - }); - self.line_renderer - .paint(&mut render_pass, &self.uniform_bind_group); - self.point_renderer - .paint(&mut render_pass, &self.uniform_bind_group); - } -} diff --git a/crates/sophus_viewer/src/scene_renderer/depth_triangle_scene_shader.wgsl b/crates/sophus_viewer/src/scene_renderer/depth_triangle_scene_shader.wgsl deleted file mode 100644 index e69de29..0000000 diff --git a/crates/sophus_viewer/src/simple_viewer.rs b/crates/sophus_viewer/src/simple_viewer.rs index 40a86cc..13ec79a 100644 --- a/crates/sophus_viewer/src/simple_viewer.rs +++ b/crates/sophus_viewer/src/simple_viewer.rs @@ -1,27 +1,22 @@ use std::collections::HashMap; use eframe::egui; -use eframe::egui::load::SizedTexture; use hollywood::actors::egui::EguiAppFromBuilder; use hollywood::actors::egui::Stream; use hollywood::compute::pipeline::CancelRequest; use hollywood::RequestWithReplyChannel; use linked_hash_map::LinkedHashMap; -use sophus_core::linalg::VecF32; +use sophus_image::arc_image::ArcImageF32; +use sophus_image::ImageSize; use sophus_lie::Isometry3; use crate::actor::ViewerBuilder; -use crate::pixel_renderer::LineVertex2; -use crate::pixel_renderer::PointVertex2; use crate::renderables::Packet; use crate::renderables::Packets; -use crate::scene_renderer::line::LineVertex3; -use crate::scene_renderer::mesh::MeshVertex3; -use crate::scene_renderer::point::PointVertex3; -use crate::scene_renderer::textured_mesh::TexturedMeshVertex3; use crate::views::aspect_ratio::get_adjusted_view_size; use crate::views::aspect_ratio::get_max_size; use crate::views::aspect_ratio::HasAspectRatio; +use crate::views::interactions::ViewportScale; use crate::views::view2d::View2d; use crate::views::view3d::View3d; use crate::views::View; @@ -79,226 +74,20 @@ impl SimpleViewer { if self.views.contains_key(&view_label) { let view = self.views.get(&view_label).unwrap(); if let View::View3d(view) = view { - request.reply(view.scene.interaction.scene_from_camera()); - } - } - } - - for (_, view) in self.views.iter_mut() { - match view { - View::View3d(view) => { - for (_, points) in view.scene.point_renderer.point_table.iter() { - for point in points.iter() { - let v = PointVertex3 { - _pos: [point.p[0], point.p[1], point.p[2]], - _color: [ - point.color.r, - point.color.g, - point.color.b, - point.color.a, - ], - _point_size: point.point_size, - }; - for _i in 0..6 { - view.scene.point_renderer.vertex_data.push(v); - } - } - } - for (_, lines) in view.scene.line_renderer.line_table.iter() { - for line in lines.iter() { - let p0 = line.p0; - let p1 = line.p1; - - let v0 = LineVertex3 { - _p0: [p0[0], p0[1], p0[2]], - _p1: [p1[0], p1[1], p1[2]], - _color: [line.color.r, line.color.g, line.color.b, line.color.a], - _line_width: line.line_width, - }; - let v1 = LineVertex3 { - _p0: [p0[0], p0[1], p0[2]], - _p1: [p1[0], p1[1], p1[2]], - _color: [line.color.r, line.color.g, line.color.b, line.color.a], - _line_width: line.line_width, - }; - view.scene.line_renderer.vertex_data.push(v0); - view.scene.line_renderer.vertex_data.push(v0); - view.scene.line_renderer.vertex_data.push(v1); - view.scene.line_renderer.vertex_data.push(v0); - view.scene.line_renderer.vertex_data.push(v1); - view.scene.line_renderer.vertex_data.push(v1); - } - } - for (_, mesh) in view.scene.mesh_renderer.mesh_table.iter() { - for trig in mesh.iter() { - let v0 = MeshVertex3 { - _pos: [trig.p0[0], trig.p0[1], trig.p0[2]], - _color: [trig.color.r, trig.color.g, trig.color.b, trig.color.a], - }; - let v1 = MeshVertex3 { - _pos: [trig.p1[0], trig.p1[1], trig.p1[2]], - _color: [trig.color.r, trig.color.g, trig.color.b, trig.color.a], - }; - let v2 = MeshVertex3 { - _pos: [trig.p2[0], trig.p2[1], trig.p2[2]], - _color: [trig.color.r, trig.color.g, trig.color.b, trig.color.a], - }; - view.scene.mesh_renderer.vertices.push(v0); - view.scene.mesh_renderer.vertices.push(v1); - view.scene.mesh_renderer.vertices.push(v2); - } - } - for (_, mesh) in view.scene.textured_mesh_renderer.mesh_table.iter() { - for trig in mesh.iter() { - let v0 = TexturedMeshVertex3 { - _pos: [trig.p0[0], trig.p0[1], trig.p0[2]], - _tex: [trig.tex0[0], trig.tex0[1]], - }; - let v1 = TexturedMeshVertex3 { - _pos: [trig.p1[0], trig.p1[1], trig.p1[2]], - _tex: [trig.tex1[0], trig.tex1[1]], - }; - let v2 = TexturedMeshVertex3 { - _pos: [trig.p2[0], trig.p2[1], trig.p2[2]], - _tex: [trig.tex2[0], trig.tex2[1]], - }; - view.scene.textured_mesh_renderer.vertices.push(v0); - view.scene.textured_mesh_renderer.vertices.push(v1); - view.scene.textured_mesh_renderer.vertices.push(v2); - } - } - } - View::View2d(view) => { - for (_, points) in view.pixel.point_renderer.points_table.iter() { - for point in points.iter() { - let v = PointVertex2 { - _pos: [point.p[0], point.p[1]], - _color: [ - point.color.r, - point.color.g, - point.color.b, - point.color.a, - ], - _point_size: point.point_size, - }; - for _i in 0..6 { - view.pixel.point_renderer.vertex_data.push(v); - } - } - } - for (_, lines) in view.pixel.line_renderer.lines_table.iter() { - for line in lines.iter() { - let p0 = line.p0; - let p1 = line.p1; - let d = (p0 - p1).normalize(); - let normal = [d[1], -d[0]]; - - let v0 = LineVertex2 { - _pos: [p0[0], p0[1]], - _normal: normal, - _color: [line.color.r, line.color.g, line.color.b, line.color.a], - _line_width: line.line_width, - }; - let v1 = LineVertex2 { - _pos: [p1[0], p1[1]], - _normal: normal, - _color: [line.color.r, line.color.g, line.color.b, line.color.a], - _line_width: line.line_width, - }; - view.pixel.line_renderer.vertex_data.push(v0); - view.pixel.line_renderer.vertex_data.push(v0); - view.pixel.line_renderer.vertex_data.push(v1); - view.pixel.line_renderer.vertex_data.push(v0); - view.pixel.line_renderer.vertex_data.push(v1); - view.pixel.line_renderer.vertex_data.push(v1); - } - } - for (_, points) in view.scene.point_renderer.point_table.iter() { - for point in points.iter() { - let v = PointVertex3 { - _pos: [point.p[0], point.p[1], point.p[2]], - _color: [ - point.color.r, - point.color.g, - point.color.b, - point.color.a, - ], - _point_size: point.point_size, - }; - for _i in 0..6 { - view.scene.point_renderer.vertex_data.push(v); - } - } - } - for (_, lines) in view.scene.line_renderer.line_table.iter() { - for line in lines.iter() { - let p0 = line.p0; - let p1 = line.p1; - - let v0 = LineVertex3 { - _p0: [p0[0], p0[1], p0[2]], - _p1: [p1[0], p1[1], p1[2]], - _color: [line.color.r, line.color.g, line.color.b, line.color.a], - _line_width: line.line_width, - }; - let v1 = LineVertex3 { - _p0: [p0[0], p0[1], p0[2]], - _p1: [p1[0], p1[1], p1[2]], - _color: [line.color.r, line.color.g, line.color.b, line.color.a], - _line_width: line.line_width, - }; - view.scene.line_renderer.vertex_data.push(v0); - view.scene.line_renderer.vertex_data.push(v0); - view.scene.line_renderer.vertex_data.push(v1); - view.scene.line_renderer.vertex_data.push(v0); - view.scene.line_renderer.vertex_data.push(v1); - view.scene.line_renderer.vertex_data.push(v1); - } - } - for (_, mesh) in view.scene.mesh_renderer.mesh_table.iter() { - for trig in mesh.iter() { - let v0 = MeshVertex3 { - _pos: [trig.p0[0], trig.p0[1], trig.p0[2]], - _color: [trig.color.r, trig.color.g, trig.color.b, trig.color.a], - }; - let v1 = MeshVertex3 { - _pos: [trig.p1[0], trig.p1[1], trig.p1[2]], - _color: [trig.color.r, trig.color.g, trig.color.b, trig.color.a], - }; - let v2 = MeshVertex3 { - _pos: [trig.p2[0], trig.p2[1], trig.p2[2]], - _color: [trig.color.r, trig.color.g, trig.color.b, trig.color.a], - }; - view.scene.mesh_renderer.vertices.push(v0); - view.scene.mesh_renderer.vertices.push(v1); - view.scene.mesh_renderer.vertices.push(v2); - } - } - for (_, mesh) in view.scene.textured_mesh_renderer.mesh_table.iter() { - for trig in mesh.iter() { - let v0 = TexturedMeshVertex3 { - _pos: [trig.p0[0], trig.p0[1], trig.p0[2]], - _tex: [trig.tex0[0], trig.tex0[1]], - }; - let v1 = TexturedMeshVertex3 { - _pos: [trig.p1[0], trig.p1[1], trig.p1[2]], - _tex: [trig.tex1[0], trig.tex1[1]], - }; - let v2 = TexturedMeshVertex3 { - _pos: [trig.p2[0], trig.p2[1], trig.p2[2]], - _tex: [trig.tex2[0], trig.tex2[1]], - }; - view.scene.textured_mesh_renderer.vertices.push(v0); - view.scene.textured_mesh_renderer.vertices.push(v1); - view.scene.textured_mesh_renderer.vertices.push(v2); - } - } + request.reply(view.interaction.scene_from_camera()); } } } } } +struct ResponseStruct { + ui_response: egui::Response, + depth_image: ArcImageF32, + scales: ViewportScale, + view_port_size: ImageSize, +} + impl eframe::App for SimpleViewer { fn on_exit(&mut self, _gl: Option<&eframe::glow::Context>) { self.cancel_request_sender.send(CancelRequest).unwrap(); @@ -307,93 +96,57 @@ impl eframe::App for SimpleViewer { fn update(&mut self, ctx: &egui::Context, _frame: &mut eframe::Frame) { egui_extras::install_image_loaders(ctx); - // Clear vertex data - for (_, view) in self.views.iter_mut() { - if let View::View3d(view) = view { - view.scene.clear_vertex_data(); - //view.pixel.clear_vertex_data(); - } - if let View::View2d(view) = view { - view.scene.clear_vertex_data(); - view.pixel.clear_vertex_data(); - } - } - // Add renderables to tables self.add_renderables_to_tables(); let mut responses = HashMap::new(); + egui::SidePanel::left("left").show(ctx, |ui| { + for (view_label, view) in self.views.iter_mut() { + ui.checkbox(view.enabled_mut(), view_label); + } + }); + egui::CentralPanel::default().show(ctx, |ui| { ui.scope(|ui0| { if self.views.is_empty() { return; } - let (max_width, max_height) = get_max_size( + let maybe_max_size = get_max_size( &self.views, 0.99 * ui0.available_width(), 0.99 * ui0.available_height(), ); + if maybe_max_size.is_none() { + return; + } + let (max_width, max_height) = maybe_max_size.unwrap(); ui0.horizontal_wrapped(|ui| { for (view_label, view) in self.views.iter_mut() { + if !view.enabled() { + continue; + } + let view_aspect_ratio = view.aspect_ratio(); let adjusted_size = get_adjusted_view_size(view_aspect_ratio, max_width, max_height); match view { View::View3d(view) => { - view.update_offscreen_texture( - &self.state, + let render_result = view.renderer.render_with_interaction_marker( &adjusted_size.image_size(), + view.interaction.zoom2d(), + view.interaction.scene_from_camera(), + &view.interaction, ); - let offscreen = view.offscreen.as_ref().unwrap(); - - view.scene.prepare(&self.state, &view.intrinsics, &None); - view.pixel_for_interaction_marker.prepare(&self.state); - - let mut command_encoder = self.state.device.create_command_encoder( - &wgpu::CommandEncoderDescriptor::default(), - ); - - view.scene.paint( - &mut command_encoder, - &offscreen.rgba.rgba_texture_view, - &offscreen.z_buffer, - ); - - view.pixel_for_interaction_marker - .show_interaction_marker(&self.state, &view.scene.interaction); - view.pixel_for_interaction_marker.paint( - &mut command_encoder, - &offscreen.rgba.rgba_texture_view, - &offscreen.z_buffer, - ); - - self.state.queue.submit(Some(command_encoder.finish())); - - let mut command_encoder = self.state.device.create_command_encoder( - &wgpu::CommandEncoderDescriptor::default(), - ); - view.scene.depth_paint( - &mut command_encoder, - &offscreen.depth.depth_texture_view_f32, - &offscreen.z_buffer, - ); - - let depth_image = offscreen.depth.download_image( - &self.state, - command_encoder, - &adjusted_size.image_size(), - ); - - let response = ui.add( - egui::Image::new(SizedTexture { + let ui_response = ui.add( + egui::Image::new(egui::load::SizedTexture { size: egui::Vec2::new( adjusted_size.width, adjusted_size.height, ), - id: offscreen.rgba.rgba_tex_id, + id: render_result.rgba_tex_id, }) .fit_to_exact_size(egui::Vec2 { x: adjusted_size.width, @@ -404,76 +157,32 @@ impl eframe::App for SimpleViewer { responses.insert( view_label.clone(), - ( - response, - VecF32::<2>::new( - view.intrinsics.image_size().width as f32 - / adjusted_size.width, - view.intrinsics.image_size().height as f32 - / adjusted_size.height, + ResponseStruct { + ui_response, + scales: ViewportScale::from_image_size_and_viewport_size( + view.intrinsics().image_size(), + adjusted_size, ), - depth_image, - ), + depth_image: render_result.depth, + view_port_size: adjusted_size.image_size(), + }, ); } View::View2d(view) => { - view.update_offscreen_texture( - &self.state, + let render_result = view.renderer.render_with_interaction_marker( &adjusted_size.image_size(), + view.interaction.zoom2d(), + view.interaction.scene_from_camera(), + &view.interaction, ); - let offscreen = view.offscreen.as_ref().unwrap(); - view.scene.prepare( - &self.state, - &view.intrinsics, - &view.background_image, - ); - view.pixel.prepare(&self.state); - - view.background_image = None; - - let mut command_encoder = self.state.device.create_command_encoder( - &wgpu::CommandEncoderDescriptor::default(), - ); - - view.scene.paint( - &mut command_encoder, - &offscreen.rgba.rgba_texture_view, - &offscreen.z_buffer, - ); - view.pixel.paint( - &mut command_encoder, - &offscreen.rgba.rgba_texture_view, - &offscreen.z_buffer, - ); - - self.state.queue.submit(Some(command_encoder.finish())); - - view.pixel - .show_interaction_marker(&self.state, &view.scene.interaction); - - let mut command_encoder = self.state.device.create_command_encoder( - &wgpu::CommandEncoderDescriptor::default(), - ); - view.scene.depth_paint( - &mut command_encoder, - &offscreen.depth.depth_texture_view_f32, - &offscreen.z_buffer, - ); - - let depth_image = offscreen.depth.download_image( - &self.state, - command_encoder, - &adjusted_size.image_size(), - ); - - let response = ui.add( - egui::Image::new(SizedTexture { + let ui_response = ui.add( + egui::Image::new(egui::load::SizedTexture { size: egui::Vec2::new( adjusted_size.width, adjusted_size.height, ), - id: offscreen.rgba.rgba_tex_id, + id: render_result.rgba_tex_id, }) .fit_to_exact_size(egui::Vec2 { x: adjusted_size.width, @@ -484,16 +193,15 @@ impl eframe::App for SimpleViewer { responses.insert( view_label.clone(), - ( - response, - VecF32::<2>::new( - view.intrinsics.image_size().width as f32 - / adjusted_size.width, - view.intrinsics.image_size().height as f32 - / adjusted_size.height, + ResponseStruct { + ui_response, + scales: ViewportScale::from_image_size_and_viewport_size( + view.intrinsics().image_size(), + adjusted_size, ), - depth_image, - ), + depth_image: render_result.depth, + view_port_size: adjusted_size.image_size(), + }, ); } } @@ -505,16 +213,27 @@ impl eframe::App for SimpleViewer { for (view_label, view) in self.views.iter_mut() { match view { View::View3d(view) => { - let response = responses.get(view_label).unwrap(); - - view.scene.process_event( - &view.intrinsics, - &response.0, - &response.1, - response.2.clone(), - ); + if let Some(response) = responses.get(view_label) { + view.interaction.process_event( + &view.intrinsics(), + &response.ui_response, + &response.scales, + response.view_port_size, + &response.depth_image, + ); + } + } + View::View2d(view) => { + if let Some(response) = responses.get(view_label) { + view.interaction.process_event( + &view.intrinsics(), + &response.ui_response, + &response.scales, + response.view_port_size, + &response.depth_image, + ); + } } - View::View2d(_) => {} } } diff --git a/crates/sophus_viewer/src/views.rs b/crates/sophus_viewer/src/views.rs index 276e1a2..cdece8c 100644 --- a/crates/sophus_viewer/src/views.rs +++ b/crates/sophus_viewer/src/views.rs @@ -1,5 +1,7 @@ /// aspect ratio pub mod aspect_ratio; +/// Interactions +pub mod interactions; /// 2d view pub mod view2d; /// 3d view @@ -25,3 +27,19 @@ impl HasAspectRatio for View { } } } + +impl View { + pub(crate) fn enabled_mut(&mut self) -> &mut bool { + match self { + View::View3d(view) => &mut view.enabled, + View::View2d(view) => &mut view.enabled, + } + } + + pub(crate) fn enabled(&self) -> bool { + match self { + View::View3d(view) => view.enabled, + View::View2d(view) => view.enabled, + } + } +} diff --git a/crates/sophus_viewer/src/views/aspect_ratio.rs b/crates/sophus_viewer/src/views/aspect_ratio.rs index 1bff3b7..eaff89e 100644 --- a/crates/sophus_viewer/src/views/aspect_ratio.rs +++ b/crates/sophus_viewer/src/views/aspect_ratio.rs @@ -7,17 +7,27 @@ pub(crate) trait HasAspectRatio { fn aspect_ratio(&self) -> f32; } -pub(crate) fn get_median_aspect_ratio(views: &LinkedHashMap) -> f32 { +pub(crate) fn get_median_aspect_ratio_and_num( + views: &LinkedHashMap, +) -> Option<(f32, usize)> { let mut aspect_ratios = std::vec::Vec::with_capacity(views.len()); - for (_, widget) in views.iter() { - aspect_ratios.push(widget.aspect_ratio()); + for (_label, widget) in views.iter() { + if widget.enabled() { + aspect_ratios.push(widget.aspect_ratio()); + } } aspect_ratios.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); let n = aspect_ratios.len(); + if n == 0 { + return None; + } if n % 2 == 1 { - aspect_ratios[n / 2] + Some((aspect_ratios[n / 2], n)) } else { - 0.5 * aspect_ratios[n / 2] + 0.5 * aspect_ratios[n / 2 - 1] + Some(( + 0.5 * aspect_ratios[n / 2] + 0.5 * aspect_ratios[n / 2 - 1], + n, + )) } } @@ -25,34 +35,35 @@ pub(crate) fn get_max_size( views: &LinkedHashMap, available_width: f32, available_height: f32, -) -> (f32, f32) { - let median_aspect_ratio = get_median_aspect_ratio(views); +) -> Option<(f32, f32)> { + if let Some((median_aspect_ratio, n)) = get_median_aspect_ratio_and_num(views) { + let mut max_width = 0.0; + let mut max_height = 0.0; - let mut max_width = 0.0; - let mut max_height = 0.0; + for num_cols in 1..=n { + let num_rows: f32 = ((n as f32) / (num_cols as f32)).ceil(); - let n = views.len() as u32; - for num_cols in 1..=n { - let num_rows: f32 = ((n as f32) / (num_cols as f32)).ceil(); - - let w: f32 = available_width / (num_cols as f32); - let h = (w / median_aspect_ratio).min(available_height / num_rows); - let w = median_aspect_ratio * h; - if w > max_width { - max_width = w; - max_height = h; + let w: f32 = available_width / (num_cols as f32); + let h = (w / median_aspect_ratio).min(available_height / num_rows); + let w = median_aspect_ratio * h; + if w > max_width { + max_width = w; + max_height = h; + } } - } - (max_width, max_height) + return Some((max_width, max_height)); + } + None } -pub(crate) struct ViewPortSize { +#[derive(Clone, Copy)] +pub(crate) struct ViewportSize { pub(crate) width: f32, pub(crate) height: f32, } -impl ViewPortSize { +impl ViewportSize { pub(crate) fn image_size(&self) -> ImageSize { ImageSize { width: self.width.ceil() as usize, @@ -65,8 +76,8 @@ pub(crate) fn get_adjusted_view_size( view_aspect_ratio: f32, max_width: f32, max_height: f32, -) -> ViewPortSize { +) -> ViewportSize { let width = max_width.min(max_height * view_aspect_ratio); let height = max_height.min(max_width / view_aspect_ratio); - ViewPortSize { width, height } + ViewportSize { width, height } } diff --git a/crates/sophus_viewer/src/views/interactions.rs b/crates/sophus_viewer/src/views/interactions.rs new file mode 100644 index 0000000..b791586 --- /dev/null +++ b/crates/sophus_viewer/src/views/interactions.rs @@ -0,0 +1,114 @@ +/// in-plane interaction +pub mod inplane_interaction; +/// orbit interaction +pub mod orbit_interaction; + +use eframe::egui; +use sophus_core::linalg::VecF64; +use sophus_image::arc_image::ArcImageF32; +use sophus_image::ImageSize; +use sophus_lie::Isometry3; +use sophus_sensor::dyn_camera::DynCamera; + +use crate::offscreen_renderer::renderer::TranslationAndScaling; +use crate::views::aspect_ratio::ViewportSize; +use crate::views::interactions::inplane_interaction::InplaneInteraction; +use crate::views::interactions::orbit_interaction::OrbitalInteraction; + +/// Scene focus +#[derive(Clone, Copy)] +pub struct SceneFocus { + /// Depth + pub depth: f64, + /// UV position + pub uv_in_virtual_camera: VecF64<2>, +} + +/// Viewport scale +pub struct ViewportScale { + /// the scale + pub scale: VecF64<2>, +} + +impl ViewportScale { + pub(crate) fn from_image_size_and_viewport_size( + image_size: ImageSize, + view_port_size: ViewportSize, + ) -> ViewportScale { + let scale = VecF64::<2>::new( + image_size.width as f64 / view_port_size.width as f64, + image_size.height as f64 / view_port_size.height as f64, + ); + ViewportScale { scale } + } + + pub(crate) fn apply(&self, uv_viewport: egui::Pos2) -> VecF64<2> { + VecF64::<2>::new( + (uv_viewport.x as f64 + 0.5) * self.scale[0] - 0.5, + (uv_viewport.y as f64 + 0.5) * self.scale[1] - 0.5, + ) + } +} + +/// Interaction state +pub enum InteractionEnum { + /// orbit interaction state + Orbital(OrbitalInteraction), + /// in-plane interaction state + InPlane(InplaneInteraction), +} + +impl InteractionEnum { + /// Get scene_from_camera isometry + pub fn scene_from_camera(&self) -> Isometry3 { + match self { + InteractionEnum::Orbital(orbit) => orbit.scene_from_camera, + InteractionEnum::InPlane(inplane) => inplane.scene_from_camera(), + } + } + + /// Get zoom + pub fn zoom2d(&self) -> TranslationAndScaling { + match self { + InteractionEnum::Orbital(orbit) => orbit.zoom2d(), + InteractionEnum::InPlane(inplane) => inplane.zoom2d(), + } + } + + /// Get scene focus point + pub fn maybe_scene_focus(&self) -> Option { + match self { + InteractionEnum::Orbital(orbital) => orbital.maybe_scene_focus, + InteractionEnum::InPlane(inplane) => inplane.maybe_scene_focus, + } + } + + /// Is there a current interaction? + pub fn is_active(&self) -> bool { + match self { + InteractionEnum::Orbital(orbital) => { + orbital.maybe_pointer_state.is_some() || orbital.maybe_scroll_state.is_some() + } + InteractionEnum::InPlane(plane) => plane.maybe_scroll_state.is_some(), + } + } + + /// process event + pub fn process_event( + &mut self, + cam: &DynCamera, + response: &egui::Response, + scales: &ViewportScale, + view_port_size: ImageSize, + z_buffer: &ArcImageF32, + ) { + match self { + InteractionEnum::Orbital(orbit) => { + orbit.process_event(cam, response, scales, view_port_size, z_buffer) + } + InteractionEnum::InPlane(inplane) => { + inplane.process_event(cam, response, scales, view_port_size) + } + } + } +} diff --git a/crates/sophus_viewer/src/views/interactions/inplane_interaction.rs b/crates/sophus_viewer/src/views/interactions/inplane_interaction.rs new file mode 100644 index 0000000..cab4034 --- /dev/null +++ b/crates/sophus_viewer/src/views/interactions/inplane_interaction.rs @@ -0,0 +1,122 @@ +use eframe::egui; +use sophus_core::linalg::VecF64; +use sophus_image::ImageSize; +use sophus_lie::traits::IsTranslationProductGroup; +use sophus_lie::Isometry3; +use sophus_sensor::DynCamera; + +use crate::offscreen_renderer::renderer::OffscreenRenderer; +use crate::offscreen_renderer::renderer::TranslationAndScaling; +use crate::views::interactions::SceneFocus; +use crate::views::interactions::ViewportScale; + +#[derive(Clone, Copy)] +pub(crate) struct InplaneScrollState {} + +#[derive(Clone, Copy)] +/// Interaction state +pub struct InplaneInteraction { + pub(crate) maybe_scroll_state: Option, + pub(crate) maybe_scene_focus: Option, + pub(crate) zoom2d: TranslationAndScaling, +} + +impl InplaneInteraction { + pub(crate) fn new() -> Self { + InplaneInteraction { + maybe_scroll_state: None, + maybe_scene_focus: None, + zoom2d: TranslationAndScaling::identity(), + } + } + + /// Process "scroll" events + /// + /// Scroll up/down: zoom in/out + pub fn process_scrolls( + &mut self, + cam: &DynCamera, + response: &egui::Response, + scales: &ViewportScale, + view_port_size: ImageSize, + ) { + let last_pointer_pos = response.ctx.input(|i| i.pointer.latest_pos()); + if last_pointer_pos.is_none() { + return; + } + + let last_pointer_pos = last_pointer_pos.unwrap(); + let uv_view_port = egui::Pos2::new( + (last_pointer_pos - response.rect.min)[0], + (last_pointer_pos - response.rect.min)[1], + ); + + if uv_view_port.x < 0.0 + || uv_view_port.y < 0.0 + || uv_view_port.x >= view_port_size.width as f32 + || uv_view_port.y >= view_port_size.height as f32 + { + return; + } + + let smooth_scroll_delta = response.ctx.input(|i| i.smooth_scroll_delta); + + let is_scroll_zero = smooth_scroll_delta.x == 0.0 && smooth_scroll_delta.y == 0.0; + + let scroll_started = self.maybe_scroll_state.is_none() && !is_scroll_zero; + let scroll_stopped = self.maybe_scroll_state.is_some() && is_scroll_zero; + + if scroll_started { + self.maybe_scroll_state = Some(InplaneScrollState {}); + } else if scroll_stopped { + self.maybe_scroll_state = None; + } + + if smooth_scroll_delta.y != 0.0 { + let zoom2d = self.zoom2d(); + let zoomed_width = cam.image_size().width as f32 / zoom2d.scaling[0] as f32; + + let uv_in_virtual_camera = zoom2d.apply(scales.apply(uv_view_port)); + + self.maybe_scene_focus = Some(SceneFocus { + depth: OffscreenRenderer::BACKGROUND_IMAGE_PLANE, + uv_in_virtual_camera, + }); + + let zoomed_width = (zoomed_width * (smooth_scroll_delta.y * 0.01).exp()) + .clamp(1.0, cam.image_size().width as f32); + + let scale = cam.image_size().width as f32 / zoomed_width; + self.zoom2d = TranslationAndScaling { + translation: VecF64::<2>::new( + (1.0 - scale as f64) * uv_in_virtual_camera[0], + (1.0 - scale as f64) * uv_in_virtual_camera[1], + ), + scaling: VecF64::<2>::new(scale as f64, scale as f64), + }; + } + } + + /// Get scene_from_camera isometry + pub fn scene_from_camera(&self) -> Isometry3 { + Isometry3::from_t(&VecF64::<3>::new(0.0, 0.0, 0.0)) + } + + /// Get zoom + pub fn zoom2d(&self) -> TranslationAndScaling { + self.zoom2d + } +} + +impl InplaneInteraction { + /// Process event + pub fn process_event( + &mut self, + cam: &DynCamera, + response: &egui::Response, + scales: &ViewportScale, + view_port_size: ImageSize, + ) { + self.process_scrolls(cam, response, scales, view_port_size); + } +} diff --git a/crates/sophus_viewer/src/interactions/orbit_interaction.rs b/crates/sophus_viewer/src/views/interactions/orbit_interaction.rs similarity index 67% rename from crates/sophus_viewer/src/interactions/orbit_interaction.rs rename to crates/sophus_viewer/src/views/interactions/orbit_interaction.rs index 3383a5d..59df15e 100644 --- a/crates/sophus_viewer/src/interactions/orbit_interaction.rs +++ b/crates/sophus_viewer/src/views/interactions/orbit_interaction.rs @@ -1,28 +1,51 @@ use eframe::egui; -use sophus_core::linalg::VecF32; use sophus_core::linalg::VecF64; use sophus_core::IsTensorLike; use sophus_image::arc_image::ArcImageF32; use sophus_image::image_view::IsImageView; +use sophus_image::ImageSize; use sophus_lie::traits::IsTranslationProductGroup; use sophus_lie::Isometry3; use sophus_sensor::DynCamera; -use crate::interactions::InteractionPointerState; -use crate::interactions::SceneFocus; -use crate::interactions::ScrollState; -use crate::interactions::WgpuClippingPlanes; +use crate::offscreen_renderer::renderer::ClippingPlanes; +use crate::offscreen_renderer::renderer::TranslationAndScaling; +use crate::views::interactions::SceneFocus; +use crate::views::interactions::ViewportScale; + +#[derive(Clone, Copy)] +pub(crate) struct OrbitalPointerState { + pub(crate) start_uv_virtual_camera: VecF64<2>, +} + +#[derive(Clone, Copy)] +pub(crate) struct OrbitalScrollState {} #[derive(Clone, Copy)] /// Interaction state pub struct OrbitalInteraction { - pub(crate) maybe_pointer_state: Option, - pub(crate) maybe_scroll_state: Option, + pub(crate) maybe_pointer_state: Option, + pub(crate) maybe_scroll_state: Option, pub(crate) maybe_scene_focus: Option, - pub(crate) clipping_planes: WgpuClippingPlanes, + pub(crate) clipping_planes: ClippingPlanes, pub(crate) scene_from_camera: Isometry3, } +impl OrbitalInteraction { + pub(crate) fn new( + scene_from_camera: Isometry3, + clipping_planes: ClippingPlanes, + ) -> OrbitalInteraction { + OrbitalInteraction { + maybe_pointer_state: None, + maybe_scroll_state: None, + maybe_scene_focus: None, + clipping_planes, + scene_from_camera, + } + } +} + impl OrbitalInteraction { fn median_scene_depth(&self, z_buffer: &ArcImageF32) -> f64 { // to median ndc z @@ -45,7 +68,7 @@ impl OrbitalInteraction { self.clipping_planes.z_from_ndc(ndc_z) } - /// Process scroll events + /// Process "scroll" events /// /// Scroll up/down: zoom in/out /// @@ -54,14 +77,28 @@ impl OrbitalInteraction { &mut self, cam: &DynCamera, response: &egui::Response, - scales: &VecF32<2>, + scales: &ViewportScale, + viewport_size: ImageSize, z_buffer: &ArcImageF32, ) { let last_pointer_pos = response.ctx.input(|i| i.pointer.latest_pos()); if last_pointer_pos.is_none() { return; } + let last_pointer_pos = last_pointer_pos.unwrap(); + let uv_viewport = egui::Pos2::new( + (last_pointer_pos - response.rect.min)[0], + (last_pointer_pos - response.rect.min)[1], + ); + + if uv_viewport.x < 0.0 + || uv_viewport.y < 0.0 + || uv_viewport.x >= viewport_size.width as f32 + || uv_viewport.y >= viewport_size.height as f32 + { + return; + } let smooth_scroll_delta = response.ctx.input(|i| i.smooth_scroll_delta); @@ -71,31 +108,18 @@ impl OrbitalInteraction { let scroll_stopped = self.maybe_scroll_state.is_some() && is_scroll_zero; if scroll_started { - let uv_view_port = egui::Pos2::new( - (last_pointer_pos - response.rect.min)[0], - (last_pointer_pos - response.rect.min)[1], - ); - - let uv_in_virtual_camera = VecF64::<2>::new( - (uv_view_port.x * scales[0]) as f64, - (uv_view_port.y * scales[1]) as f64, - ); - - if self.maybe_scene_focus.is_none() { - // Typically, the scene focus shall only be set by the pointer interaction event. But - // it was never set, we set it here. - let mut z = self.clipping_planes.z_from_ndc( - z_buffer.pixel(uv_view_port.x as usize, uv_view_port.y as usize) as f64, - ); - if z >= self.clipping_planes.far { - z = self.median_scene_depth(z_buffer); - } - self.maybe_scene_focus = Some(SceneFocus { - depth: z, - uv_in_virtual_camera, - }); + let uv_in_virtual_camera = scales.apply(uv_viewport); + let mut z = self + .clipping_planes + .z_from_ndc(z_buffer.pixel(uv_viewport.x as usize, uv_viewport.y as usize) as f64); + if z >= self.clipping_planes.far { + z = self.median_scene_depth(z_buffer); } - self.maybe_scroll_state = Some(ScrollState {}); + self.maybe_scene_focus = Some(SceneFocus { + depth: z, + uv_in_virtual_camera, + }); + self.maybe_scroll_state = Some(OrbitalScrollState {}); } else if scroll_stopped { self.maybe_scroll_state = None; } @@ -135,7 +159,7 @@ impl OrbitalInteraction { &mut self, cam: &DynCamera, response: &egui::Response, - scales: &VecF32<2>, + scales: &ViewportScale, z_buffer: &ArcImageF32, ) { let delta_x = response.drag_delta().x; @@ -146,19 +170,16 @@ impl OrbitalInteraction { let pointer = response.interact_pointer_pos().unwrap(); - let uv_view_port = egui::Pos2::new( + let uv_viewport = egui::Pos2::new( (pointer - response.rect.min)[0], (pointer - response.rect.min)[1], ); - let uv_in_virtual_camera = VecF64::<2>::new( - (uv_view_port.x * scales[0]) as f64, - (uv_view_port.y * scales[1]) as f64, - ); + let uv_in_virtual_camera = scales.apply(uv_viewport); - let mut z = self.clipping_planes.z_from_ndc( - z_buffer.pixel(uv_view_port.x as usize, uv_view_port.y as usize) as f64, - ); + let mut z = self + .clipping_planes + .z_from_ndc(z_buffer.pixel(uv_viewport.x as usize, uv_viewport.y as usize) as f64); if z >= self.clipping_planes.far { z = self.median_scene_depth(z_buffer); @@ -167,8 +188,8 @@ impl OrbitalInteraction { depth: z, uv_in_virtual_camera, }); - self.maybe_pointer_state = Some(InteractionPointerState { - start_uv: uv_in_virtual_camera, + self.maybe_pointer_state = Some(OrbitalPointerState { + start_uv_virtual_camera: uv_in_virtual_camera, }); } else if response.drag_stopped() { // A drag event finished @@ -178,16 +199,16 @@ impl OrbitalInteraction { if response.dragged_by(egui::PointerButton::Secondary) { // translate scene - let c = response.interact_pointer_pos().unwrap() - response.rect.min; - let current_pixel = egui::Vec2::new(c[0] * scales[0], c[1] * scales[1]); + let uv_viewport = response.interact_pointer_pos().unwrap() - response.rect.min; + let current_pixel = scales.apply(uv_viewport.to_pos2()).cast::(); let scene_focus = self.maybe_scene_focus.unwrap(); - let start_pixel = self.maybe_pointer_state.unwrap().start_uv; + let start_pixel = self.maybe_pointer_state.unwrap().start_uv_virtual_camera; let depth = scene_focus.depth; let p0 = cam.cam_unproj_with_z(&start_pixel, depth); let p1 = cam.cam_unproj_with_z( &VecF64::<2>::new( - start_pixel.x + delta_x as f64, - start_pixel.y + delta_y as f64, + start_pixel.x + (delta_x as f64 * scales.scale.x), + start_pixel.y + (delta_y as f64 * scales.scale.y), ), depth, ); @@ -224,10 +245,16 @@ impl OrbitalInteraction { &mut self, cam: &DynCamera, response: &egui::Response, - scales: &VecF32<2>, + scales: &ViewportScale, + view_port_size: ImageSize, z_buffer: &ArcImageF32, ) { self.process_pointer(cam, response, scales, z_buffer); - self.process_scrolls(cam, response, scales, z_buffer); + self.process_scrolls(cam, response, scales, view_port_size, z_buffer); + } + + /// Get zoom + pub fn zoom2d(&self) -> TranslationAndScaling { + TranslationAndScaling::identity() } } diff --git a/crates/sophus_viewer/src/views/view2d.rs b/crates/sophus_viewer/src/views/view2d.rs index 25247b6..df46709 100644 --- a/crates/sophus_viewer/src/views/view2d.rs +++ b/crates/sophus_viewer/src/views/view2d.rs @@ -1,31 +1,18 @@ use linked_hash_map::LinkedHashMap; -use sophus_core::linalg::VecF64; -use sophus_image::arc_image::ArcImage4U8; -use sophus_image::ImageSize; -use sophus_lie::traits::IsTranslationProductGroup; -use sophus_lie::Isometry3; use sophus_sensor::DynCamera; -use crate::interactions::inplane_interaction::InplaneInteraction; -use crate::interactions::InteractionEnum; -use crate::interactions::WgpuClippingPlanes; -use crate::offscreen::OffscreenTextures; -use crate::pixel_renderer::PixelRenderer; -use crate::renderables::renderable2d::Renderable2d; +use crate::offscreen_renderer::renderer::OffscreenRenderer; use crate::renderables::renderable2d::View2dPacket; -use crate::renderables::renderable3d::Renderable3d; -use crate::renderables::renderable3d::TexturedMesh3; -use crate::scene_renderer::SceneRenderer; use crate::views::aspect_ratio::HasAspectRatio; +use crate::views::interactions::inplane_interaction::InplaneInteraction; +use crate::views::interactions::InteractionEnum; use crate::views::View; use crate::ViewerRenderState; pub(crate) struct View2d { - pub(crate) scene: SceneRenderer, - pub(crate) pixel: PixelRenderer, - pub(crate) intrinsics: DynCamera, - pub(crate) offscreen: Option, - pub(crate) background_image: Option, + pub(crate) renderer: OffscreenRenderer, + pub(crate) interaction: InteractionEnum, + pub(crate) enabled: bool, } impl View2d { @@ -38,39 +25,13 @@ impl View2d { return false; } if let Some(frame) = &packet.frame { - let depth_stencil = Some(wgpu::DepthStencilState { - format: wgpu::TextureFormat::Depth32Float, - depth_write_enabled: true, - depth_compare: wgpu::CompareFunction::Less, - stencil: wgpu::StencilState::default(), - bias: wgpu::DepthBiasState::default(), - }); views.insert( packet.view_label.clone(), View::View2d(View2d { - scene: SceneRenderer::new( - state, - frame.intrinsics(), - depth_stencil.clone(), - InteractionEnum::InplaneInteraction(InplaneInteraction { - maybe_pointer_state: None, - maybe_scroll_state: None, - maybe_scene_focus: None, - _clipping_planes: WgpuClippingPlanes { - near: 0.1, - far: 1000.0, - }, - scene_from_camera: Isometry3::from_t(&VecF64::<3>::new(0.0, 0.0, -5.0)), - }), - ), - pixel: PixelRenderer::new( - state, - &frame.intrinsics().image_size(), - depth_stencil.clone(), - ), - intrinsics: frame.intrinsics().clone(), - background_image: frame.maybe_image().cloned(), - offscreen: None, + renderer: OffscreenRenderer::new(state, frame.intrinsics()), + interaction: InteractionEnum::InPlane(InplaneInteraction::new( + )), + enabled: true, }), ); return true; @@ -93,78 +54,15 @@ impl View2d { }; if let Some(frame) = packet.frame { - let depth_stencil = Some(wgpu::DepthStencilState { - format: wgpu::TextureFormat::Depth32Float, - depth_write_enabled: true, - depth_compare: wgpu::CompareFunction::Less, - stencil: wgpu::StencilState::default(), - bias: wgpu::DepthBiasState::default(), - }); - let new_intrinsics = frame.intrinsics(); // We got a new frame, hence we need to clear all renderables and then add the // intrinsics and background image if present. The easiest and most error-proof way to // do this is to create a new SceneRenderer and PixelRenderer and replace the old ones. - view.pixel = PixelRenderer::new( - state, - &frame.intrinsics().image_size(), - depth_stencil.clone(), - ); - - view.scene = SceneRenderer::new( - state, - frame.intrinsics(), - depth_stencil, - InteractionEnum::InplaneInteraction(InplaneInteraction { - maybe_pointer_state: None, - maybe_scroll_state: None, - maybe_scene_focus: None, - _clipping_planes: WgpuClippingPlanes { - near: 0.1, - far: 1000.0, - }, - scene_from_camera: Isometry3::from_t(&VecF64::<3>::new(0.0, 0.0, 0.0)), - }), - ); - - view.offscreen = Some(OffscreenTextures::new(state, &new_intrinsics.image_size())); - - view.intrinsics = new_intrinsics.clone(); - if let Some(background_image) = frame.maybe_image() { - view.background_image = Some(background_image.clone()); + view.renderer = OffscreenRenderer::new(state, new_intrinsics); - let w = view.intrinsics.image_size().width; - let h = view.intrinsics.image_size().height; - - let far = 500.0; - - let p0 = view - .intrinsics - .cam_unproj_with_z(&VecF64::<2>::new(-0.5, -0.5), far) - .cast(); - let p1 = view - .intrinsics - .cam_unproj_with_z(&VecF64::<2>::new(w as f64 - 0.5, -0.5), far) - .cast(); - let p2 = view - .intrinsics - .cam_unproj_with_z(&VecF64::<2>::new(-0.5, h as f64 - 0.5), far) - .cast(); - let p3 = view - .intrinsics - .cam_unproj_with_z(&VecF64::<2>::new(w as f64 - 0.5, h as f64 - 0.5), far) - .cast(); - - let tex_mesh = TexturedMesh3::make(&[ - [(p0, [0.0, 0.0]), (p1, [1.0, 0.0]), (p2, [0.0, 1.0])], - [(p1, [1.0, 0.0]), (p2, [0.0, 1.0]), (p3, [1.0, 1.0])], - ]); - view.scene - .textured_mesh_renderer - .mesh_table - .insert("background".to_owned(), tex_mesh.mesh); - } + view.renderer + .reset_2d_frame(new_intrinsics, frame.maybe_image()); } let view = views.get_mut(&packet.view_label).unwrap(); @@ -173,67 +71,16 @@ impl View2d { _ => panic!("View type mismatch"), }; - for m in packet.renderables2d { - match m { - Renderable2d::Lines2(lines) => { - view.pixel - .line_renderer - .lines_table - .insert(lines.name, lines.lines); - } - Renderable2d::Points2(points) => { - view.pixel - .point_renderer - .points_table - .insert(points.name, points.points); - } - } - } - for m in packet.renderables3d { - match m { - Renderable3d::Lines3(lines3) => { - view.scene - .line_renderer - .line_table - .insert(lines3.name, lines3.lines); - } - Renderable3d::Points3(points3) => { - view.scene - .point_renderer - .point_table - .insert(points3.name, points3.points); - } - Renderable3d::Mesh3(mesh) => { - view.scene - .mesh_renderer - .mesh_table - .insert(mesh.name, mesh.mesh); - } - } - } + view.renderer.update_2d_renderables(packet.renderables2d); } - pub fn update_offscreen_texture( - &mut self, - render_state: &ViewerRenderState, - view_port_size: &ImageSize, - ) { - match &mut self.offscreen { - Some(offscreen) => { - let current_view_port_size = offscreen.view_port_size(); - if current_view_port_size != view_port_size { - *offscreen = OffscreenTextures::new(render_state, view_port_size); - } - } - None => { - self.offscreen = Some(OffscreenTextures::new(render_state, view_port_size)); - } - } + pub fn intrinsics(&self) -> DynCamera { + self.renderer.intrinsics() } } impl HasAspectRatio for View2d { fn aspect_ratio(&self) -> f32 { - self.intrinsics.image_size().aspect_ratio() + self.renderer.aspect_ratio() } } diff --git a/crates/sophus_viewer/src/views/view3d.rs b/crates/sophus_viewer/src/views/view3d.rs index cb2b81e..645c8ea 100644 --- a/crates/sophus_viewer/src/views/view3d.rs +++ b/crates/sophus_viewer/src/views/view3d.rs @@ -1,24 +1,18 @@ use linked_hash_map::LinkedHashMap; -use sophus_image::ImageSize; use sophus_sensor::DynCamera; -use crate::interactions::orbit_interaction::OrbitalInteraction; -use crate::interactions::InteractionEnum; -use crate::offscreen::OffscreenTextures; -use crate::pixel_renderer::PixelRenderer; -use crate::renderables::renderable3d::Renderable3d; +use crate::offscreen_renderer::renderer::OffscreenRenderer; use crate::renderables::renderable3d::View3dPacket; -use crate::scene_renderer::SceneRenderer; use crate::views::aspect_ratio::HasAspectRatio; +use crate::views::interactions::orbit_interaction::OrbitalInteraction; +use crate::views::interactions::InteractionEnum; use crate::views::View; use crate::ViewerRenderState; pub(crate) struct View3d { - pub(crate) scene: SceneRenderer, - // At the moment, the pixel renderer is only used for the interaction marker - pub(crate) pixel_for_interaction_marker: PixelRenderer, - pub(crate) intrinsics: DynCamera, - pub(crate) offscreen: Option, + pub(crate) renderer: OffscreenRenderer, + pub(crate) interaction: InteractionEnum, + pub(crate) enabled: bool, } impl View3d { @@ -28,35 +22,15 @@ impl View3d { state: &ViewerRenderState, ) { if !views.contains_key(&packet.view_label) { - let depth_stencil = Some(wgpu::DepthStencilState { - format: wgpu::TextureFormat::Depth32Float, - depth_write_enabled: true, - depth_compare: wgpu::CompareFunction::Less, - stencil: wgpu::StencilState::default(), - bias: wgpu::DepthBiasState::default(), - }); views.insert( packet.view_label.clone(), View::View3d(View3d { - offscreen: None, - scene: SceneRenderer::new( - state, - &packet.initial_camera.intrinsics, - depth_stencil.clone(), - InteractionEnum::OrbitalInteraction(OrbitalInteraction { - maybe_pointer_state: None, - maybe_scroll_state: None, - maybe_scene_focus: None, - clipping_planes: packet.initial_camera.clipping_planes, - scene_from_camera: packet.initial_camera.scene_from_camera, - }), - ), - pixel_for_interaction_marker: PixelRenderer::new( - state, - &packet.initial_camera.intrinsics.image_size(), - depth_stencil.clone(), - ), - intrinsics: packet.initial_camera.intrinsics.clone(), + renderer: OffscreenRenderer::new(state, &packet.initial_camera.intrinsics), + interaction: InteractionEnum::Orbital(OrbitalInteraction::new( + packet.initial_camera.scene_from_camera, + packet.initial_camera.clipping_planes, + )), + enabled: true, }), ); } @@ -74,51 +48,17 @@ impl View3d { View::View3d(view) => view, _ => panic!("View type mismatch"), }; - for m in packet.renderables3d { - match m { - Renderable3d::Lines3(lines3) => { - view.scene - .line_renderer - .line_table - .insert(lines3.name, lines3.lines); - } - Renderable3d::Points3(points3) => { - view.scene - .point_renderer - .point_table - .insert(points3.name, points3.points); - } - Renderable3d::Mesh3(mesh) => { - view.scene - .mesh_renderer - .mesh_table - .insert(mesh.name, mesh.mesh); - } - } - } + + view.renderer.update_3d_renderables(packet.renderables3d); } - pub fn update_offscreen_texture( - &mut self, - render_state: &ViewerRenderState, - view_port_size: &ImageSize, - ) { - match &mut self.offscreen { - Some(offscreen) => { - let current_view_port_size = offscreen.view_port_size(); - if current_view_port_size != view_port_size { - *offscreen = OffscreenTextures::new(render_state, view_port_size); - } - } - None => { - self.offscreen = Some(OffscreenTextures::new(render_state, view_port_size)); - } - } + pub fn intrinsics(&self) -> DynCamera { + self.renderer.intrinsics() } } impl HasAspectRatio for View3d { fn aspect_ratio(&self) -> f32 { - self.intrinsics.image_size().aspect_ratio() + self.renderer.aspect_ratio() } }