From 1bf304e1e8cca8c52e4844f5140aaae93f8ece33 Mon Sep 17 00:00:00 2001 From: Martin de La Gorce Date: Thu, 4 Mar 2021 00:47:01 +0000 Subject: [PATCH 1/8] improve viewer using a class --- deodr/differentiable_renderer.py | 4 +- deodr/examples/mesh_viewer.py | 539 +++++++++++++++++++------------ 2 files changed, 338 insertions(+), 205 deletions(-) diff --git a/deodr/differentiable_renderer.py b/deodr/differentiable_renderer.py index 9aeb789..12f684a 100644 --- a/deodr/differentiable_renderer.py +++ b/deodr/differentiable_renderer.py @@ -562,7 +562,7 @@ def render_error_backward(self, err_buffer_b, make_copies=True): if not self.backface_culling: raise BaseException( - "use backface_culling=True if you use gradient backpropagation to get valid gradient through edge antialiazing." + "use backface_culling=True if you use gradient backpropagation to get valid gradient through edge anti-aliasing." ) sigma, obs, image, z_buffer, err_buffer = self.store_backward antialiase_error = True @@ -598,7 +598,7 @@ def render_backward(self, image_b, make_copies=True): ) if not self.backface_culling: raise BaseException( - "use backface_culling=True if you use gradient backpropagation to get valid gradient through edge antialiazing." + "use backface_culling=True if you use gradient backpropagation to get valid gradient through edge anti-aliasing." ) sigma, image, z_buffer = self.store_backward antialiase_error = False diff --git a/deodr/examples/mesh_viewer.py b/deodr/examples/mesh_viewer.py index c56441e..b0a3df1 100644 --- a/deodr/examples/mesh_viewer.py +++ b/deodr/examples/mesh_viewer.py @@ -1,4 +1,4 @@ -"""Example of interactive 3D mesh visualization using deodr and opencv.""" +"""Example of interactive 3D mesh visualization using DEODR and OpenCV.""" import argparse import os @@ -42,6 +42,13 @@ def __init__( self.xy_translation_speed = xy_translation_speed self.camera = camera + def toggle_mode(self): + if self.mode == "object_centered_trackball": + self.mode = "camera_centered" + else: + self.mode = "object_centered_trackball" + print(f"trackball mode = {self.mode}") + def mouse_callback(self, event, x, y, flags, param): if event == 0 and flags == 0: return @@ -73,23 +80,21 @@ def mouse_callback(self, event, x, y, flags, param): if self.left_is_down and not (self.ctrl_is_down): if self.mode == "camera_centered": - - center_in_camera = self.camera.world_to_camera(self.object_center) rotation = Rotation.from_rotvec( np.array( [ - -self.rotation_speed * (y - self.y_last), - self.rotation_speed * (x - self.x_last), + -0.3 * self.rotation_speed * (y - self.y_last), + 0.3 * self.rotation_speed * (x - self.x_last), 0, ] ) ) - self.camera.extrinsic = rotation.as_dcm().dot(self.camera.extrinsic) + self.camera.extrinsic = rotation.as_matrix().dot(self.camera.extrinsic) # assert np.allclose(center_in_camera, self.camera.world_to_camera(self.object_center)) self.x_last = x self.y_last = y - if self.mode == "object_centered_trackball": + elif self.mode == "object_centered_trackball": rotation = Rotation.from_rotvec( np.array( @@ -100,7 +105,7 @@ def mouse_callback(self, event, x, y, flags, param): ] ) ) - n_rotation = rotation.as_dcm().dot(self.camera.extrinsic[:, :3]) + n_rotation = rotation.as_matrix().dot(self.camera.extrinsic[:, :3]) nt = ( self.camera.extrinsic[:, :3].dot(self.object_center) + self.camera.extrinsic[:, 3] @@ -119,10 +124,29 @@ def mouse_callback(self, event, x, y, flags, param): ) self.x_last = x self.y_last = y - if self.mode == "object_centered_trackball": - self.camera.extrinsic[2, 3] += self.z_translation_speed * ( - self.y_last - y - ) + elif self.mode == "object_centered_trackball": + if np.abs(self.y_last - y) >= np.abs(self.x_last - x): + self.camera.extrinsic[2, 3] += self.z_translation_speed * ( + self.y_last - y + ) + else: + rotation = Rotation.from_rotvec( + np.array( + [ + 0, + 0, + -self.rotation_speed * (self.x_last - x), + ] + ) + ) + + n_rotation = rotation.as_matrix().dot(self.camera.extrinsic[:, :3]) + nt = ( + self.camera.extrinsic[:, :3].dot(self.object_center) + + self.camera.extrinsic[:, 3] + - n_rotation.dot(self.object_center) + ) + self.camera.extrinsic = np.column_stack((n_rotation, nt)) self.x_last = x self.y_last = y else: @@ -144,220 +168,329 @@ def mouse_callback(self, event, x, y, flags, param): ) self.camera.extrinsic[0, 3] += tx self.camera.extrinsic[1, 3] += ty - center_in_camera = self.camera.world_to_camera(self.object_center) self.x_last = x self.y_last = y - assert np.max(np.abs(center_in_camera[:2])) < 1e-3 - - -def mesh_viewer( - file_or_mesh, - display_texture_map=True, - width=640, - height=480, - display_fps=True, - title=None, - use_moderngl=False, - light_directional=(0, 0, -0.5), - light_ambient=0.5, -): - if isinstance(file_or_mesh, str): - if title is None: - title = file_or_mesh - mesh_trimesh = trimesh.load(file_or_mesh) - mesh = ColoredTriMesh.from_trimesh(mesh_trimesh) - elif isinstance(file_or_mesh, trimesh.base.Trimesh): - mesh_trimesh = file_or_mesh - mesh = ColoredTriMesh.from_trimesh(mesh_trimesh) - if title is None: - title = "unknown" - elif isinstance(file_or_mesh, ColoredTriMesh): - mesh = file_or_mesh - if title is None: - title = "unknown" - else: - raise ( - BaseException( - f"unknown type {type(file_or_mesh)}for input obj_file_or_trimesh," - " can be string or trimesh.base.Trimesh" + + def print_help(self): + help_str = "" + if self.mode == "object_centered_trackball": + help_str += "Trackball:\n" + help_str += "mouse right + vertical motion = move in/out\n" + help_str += ( + "mouse right + horizontal motion = rotate object along camera z axis\n" ) - ) + help_str += ( + "mouse left + vertical motion = rotate object along camera x axis\n" + ) + help_str += ( + "mouse left + horizontal motion = rotate object along camera y axis\n" + ) + help_str += "CTRL + mouse left + vertical motion = translate object along camera y axis\n" + help_str += "CTRL + mouse left + horizontal motion = translate object along camera x axis\n" + else: + help_str += "Trackball:\n" + help_str += "mouse right + vertical motion = move in/out\n" + help_str += ( + "mouse right + horizontal motion = rotate camera along camera z axis\n" + ) + help_str += ( + "mouse left + vertical motion = rotate camera along camera x axis\n" + ) + help_str += ( + "mouse left + horizontal motion = rotate camera along camera y axis\n" + ) + help_str += "CTRL + mouse left + vertical motion = translate camera along camera y axis\n" + help_str += "CTRL + mouse left + horizontal motion = translate camera along camera x axis\n" - if display_texture_map: - ax = plt.subplot(111) - if mesh.textured: - mesh.plot_uv_map(ax) + print(help_str) - object_center = 0.5 * (mesh.vertices.max(axis=0) + mesh.vertices.min(axis=0)) - object_radius = np.max(mesh.vertices.max(axis=0) - mesh.vertices.min(axis=0)) - camera_center = object_center + np.array([0, 0, 3]) * object_radius +class Viewer: + def __init__( + self, + file_or_mesh, + display_texture_map=True, + width=640, + height=480, + display_fps=True, + title=None, + use_moderngl=False, + light_directional=(0, 0, -0.5), + light_ambient=0.5, + background_color=(1, 1, 1), + use_antialiasing=True, + use_light=True, + print_help=True, + ): + self.title = title + self.scene = differentiable_renderer.Scene3D(sigma=1) + self.set_mesh(file_or_mesh) + self.windowname = f"DEODR mesh viewer:{self.title}" - rotation = np.array([[1, 0, 0], [0, -1, 0], [0, 0, -1]]) - translation = -rotation.T.dot(camera_center) - extrinsic = np.column_stack((rotation, translation)) + self.width = width + self.height = height + self.display_fps = display_fps + self.use_moderngl = use_moderngl + self.use_antialiasing = use_antialiasing + self.use_light = use_light - focal = 2 * width - intrinsic = np.array([[focal, 0, width / 2], [0, focal, height / 2], [0, 0, 1]]) + if display_texture_map: + self.display_texture_map() - distortion = [0, 0, 0, 0, 0] - camera = differentiable_renderer.Camera( - extrinsic=extrinsic, - intrinsic=intrinsic, - width=width, - height=height, - distortion=distortion, - ) - use_antiliazing = True - use_light = True + self.set_background_color(background_color) + self.set_light(light_directional, light_ambient) + self.recenter_camera() - scene = differentiable_renderer.Scene3D(sigma=1) + if use_moderngl: + self.setup_moderngl() + else: + self.offscreen_renderer = None - scene.set_light( - light_directional=np.array(light_directional), light_ambient=light_ambient - ) - scene.set_mesh(mesh) + if print_help: + self.print_help() - scene.set_background_color([1, 1, 1]) + def set_light(self, light_directional, light_ambient): + self.light_directional = np.array(light_directional) + self.light_ambient = light_ambient + self.scene.set_light( + light_directional=self.light_directional, light_ambient=light_ambient + ) - if mesh.texture is not None: - mesh.texture = mesh.texture[ - :, :, ::-1 - ] # convert texture to GBR to avoid future conversion when ploting in Opencv + def setup_moderngl(self): + import deodr.opengl.moderngl - fps = 0 - fps_decay = 0.1 - windowname = f"DEODR mesh viewer:{title}" + self.offscreen_renderer = deodr.opengl.moderngl.OffscreenRenderer() + self.scene.mesh.compute_vertex_normals() + self.offscreen_renderer.set_scene(self.scene) + + def set_background_color(self, background_color): + self.scene.set_background_color(background_color) + + def display_texture_map(self): + if self.mesh.textured: + ax = plt.subplot(111) + self.mesh.plot_uv_map(ax) + + def set_mesh(self, file_or_mesh): + if isinstance(file_or_mesh, str): + if self.title is None: + self.title = file_or_mesh + mesh_trimesh = trimesh.load(file_or_mesh) + self.mesh = ColoredTriMesh.from_trimesh(mesh_trimesh) + elif isinstance(file_or_mesh, trimesh.base.Trimesh): + mesh_trimesh = file_or_mesh + self.mesh = ColoredTriMesh.from_trimesh(mesh_trimesh) + if self.title is None: + self.title = "unknown" + elif isinstance(file_or_mesh, ColoredTriMesh): + self.mesh = file_or_mesh + if self.title is None: + self.title = "unknown" + else: + raise ( + TypeError( + f"unknown type {type(file_or_mesh)} for input obj_file_or_trimesh," + " can be string or trimesh.base.Trimesh" + ) + ) + self.object_center = 0.5 * ( + self.mesh.vertices.max(axis=0) + self.mesh.vertices.min(axis=0) + ) + self.object_radius = np.max( + self.mesh.vertices.max(axis=0) - self.mesh.vertices.min(axis=0) + ) + self.scene.set_mesh(self.mesh) + + def recenter_camera(self): + camera_center = self.object_center + np.array([0, 0, 3]) * self.object_radius + rotation = np.array([[1, 0, 0], [0, -1, 0], [0, 0, -1]]) + translation = -rotation.T.dot(camera_center) + extrinsic = np.column_stack((rotation, translation)) + focal = 2 * self.width + intrinsic = np.array( + [[focal, 0, self.width / 2], [0, focal, self.height / 2], [0, 0, 1]] + ) - interactor = Interactor( - camera=camera, - object_center=object_center, - z_translation_speed=0.01 * object_radius, - xy_translation_speed=3e-4, - ) + distortion = [0, 0, 0, 0, 0] + self.camera = differentiable_renderer.Camera( + extrinsic=extrinsic, + intrinsic=intrinsic, + width=self.width, + height=self.height, + distortion=distortion, + ) - cv2.namedWindow(windowname, cv2.WINDOW_NORMAL) - cv2.resizeWindow(windowname, width, height) - cv2.setMouseCallback(windowname, interactor.mouse_callback) + self.interactor = Interactor( + camera=self.camera, + object_center=self.object_center, + z_translation_speed=0.01 * self.object_radius, + xy_translation_speed=3e-4, + ) - if use_moderngl: - import deodr.opengl.moderngl + def start(self): + fps = 0 + fps_decay = 0.1 + cv2.namedWindow(self.windowname, cv2.WINDOW_NORMAL) + cv2.resizeWindow(self.windowname, self.width, self.height) + cv2.setMouseCallback(self.windowname, self.interactor.mouse_callback) + while cv2.getWindowProperty(self.windowname, 0) >= 0: + # mesh.set_vertices(mesh.vertices+np.random.randn(*mesh.vertices.shape)*0.001) + self.width, self.height = cv2.getWindowImageRect(self.windowname)[2:] + self.resize_camera() + start = time.perf_counter() + + if self.use_moderngl: + image = self.offscreen_renderer.render(self.camera) + else: + image = self.scene.render(self.interactor.camera).astype(np.float32) + + if self.display_fps: + self.print_fps(image, fps) + + cv2.imshow(self.windowname, cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) + stop = time.perf_counter() + fps = (1 - fps_decay) * fps + fps_decay * (1 / (stop - start)) + key = cv2.waitKey(1) + if key > 0: + self.process_key(key) + + def resize_camera(self): + focal = 2 * self.width + intrinsic = np.array( + [[focal, 0, self.width / 2], [0, focal, self.height / 2], [0, 0, 1]] + ) + self.camera.intrinsic = intrinsic + self.camera.width = self.width + self.camera.height = self.height + + def print_fps(self, image, fps): + font = cv2.FONT_HERSHEY_SIMPLEX + bottom_left_corner_of_text = (20, image.shape[0] - 20) + font_scale = 1 + font_color = (0, 0, 255) + thickness = 2 + cv2.putText( + image, + "fps:%0.1f" % fps, + bottom_left_corner_of_text, + font, + font_scale, + font_color, + thickness, + ) - offscreen_renderer = deodr.opengl.moderngl.OffscreenRenderer() - scene.mesh.compute_vertex_normals() - offscreen_renderer.set_scene(scene) - else: - offscreen_renderer = None - while cv2.getWindowProperty(windowname, 0) >= 0: - - # mesh.set_vertices(mesh.vertices+np.random.randn(*mesh.vertices.shape)*0.001) - width, height = cv2.getWindowImageRect(windowname)[2:] - focal = 2 * width - intrinsic = np.array([[focal, 0, width / 2], [0, focal, height / 2], [0, 0, 1]]) - camera.intrinsic = intrinsic - camera.width = width - camera.height = height - - start = time.clock() - if use_moderngl: - image = offscreen_renderer.render(camera) + def print_help(self): + help_str = "" + help_str += "-----------------\n" + help_str += "DEODR Mesh Viewer\n" + help_str += "-----------------\n" + help_str += "Keys:\n" + help_str += "h: print this help\n" + help_str += "r: toggle renderer between DEODR cpu rendering and moderngl\n" + help_str += "p: toggle between linear texture mapping and perspective correct texture mapping\n" + help_str += "l: toggle between uniform lighting vs directional + ambient\n" + help_str += "a: toggle edge overdraw anti-aliasing (DEODR renderin only)\n" + help_str += "s: save scene and camera in a pickle file\n" + help_str += "t: change camera trackball mode\n" + print(help_str) + self.interactor.print_help() + + def toggle_renderer(self): + # Switch renderer between DEODR cpu rendering and moderngl + self.use_moderngl = not (self.use_moderngl) + print(f"use_moderngl = { self.use_moderngl}") + + if self.use_moderngl and self.offscreen_renderer is None: + self.setup_moderngl() + + def toggle_perspective_texture_mapping(self): + # Switch between linear texture mapping and perspective correct texture mapping + if self.use_moderngl: + print("can only use perspective correct mapping when using moderngl") else: - image = scene.render(interactor.camera) - - if display_fps: - font = cv2.FONT_HERSHEY_SIMPLEX - bottom_left_corner_of_text = (20, height - 20) - font_scale = 1 - font_color = (0, 0, 255) - thickness = 2 - cv2.putText( - image, - "fps:%0.1f" % fps, - bottom_left_corner_of_text, - font, - font_scale, - font_color, - thickness, - ) - cv2.imshow(windowname, image) - stop = time.clock() - fps = (1 - fps_decay) * fps + fps_decay * (1 / (stop - start)) - key = cv2.waitKey(1) - if key >= 0: - if key == ord("r"): - # change renderer between DEODR cpu rendering and moderngl - use_moderngl = not (use_moderngl) - print(f"use_moderngl = {use_moderngl}") - - if offscreen_renderer is None: - offscreen_renderer = deodr.opengl.moderngl.OffscreenRenderer() - scene.mesh.compute_vertex_normals() - offscreen_renderer.set_scene(scene) - - if key == ord("p"): - if use_moderngl: - print( - "can only use perspective corect mapping when using moderngl" - ) - else: - # toggle perspective correct mapping (texture or interpolation) - scene.perspective_correct = not (scene.perspective_correct) - print(f"perspective_correct = {scene.perspective_correct}") - - if key == ord("l"): - # toggle directional light + ambient vs ambient = 1 - use_light = not (use_light) - print(f"use_light = {use_light}") - if use_light: - if use_moderngl: - offscreen_renderer.set_light( - light_directional=np.array(light_directional), - light_ambient=light_ambient, - ) - else: - scene.set_light( - light_directional=np.array(light_directional), - light_ambient=light_ambient, - ) - else: - if use_moderngl: - offscreen_renderer.set_light( - light_directional=(0, 0, 0), light_ambient=1.0, - ) - else: - scene.set_light(light_directional=None, light_ambient=1.0) + # toggle perspective correct mapping (texture or interpolation) + self.scene.perspective_correct = not (self.scene.perspective_correct) + print(f"perspective_correct = {self.scene.perspective_correct}") + + def toggle_lights(self): + # toggle directional light + ambient vs ambient = 1 + self.use_light = not (self.use_light) + print(f"use_light = { self.use_light}") + + if self.use_light: + if self.use_moderngl: + self.offscreen_renderer.set_light( + light_directional=np.array(self.light_directional), + light_ambient=self.light_ambient, + ) + else: + self.scene.set_light( + light_directional=np.array(self.light_directional), + light_ambient=self.light_ambient, + ) + else: + if self.use_moderngl: + self.offscreen_renderer.set_light( + light_directional=(0, 0, 0), + light_ambient=1.0, + ) + else: + self.scene.set_light(light_directional=None, light_ambient=1.0) - if key == ord("a"): - # toggle edge overdraw anti-aliasing - if use_moderngl: - print("no anti-alizaing available when using moderngl") - else: - use_antiliazing = not (use_antiliazing) - print(f"use_antialiazing = {use_antiliazing}") - if use_antiliazing: - scene.sigma = 1.0 - else: - scene.sigma = 0.0 - - if key == ord("s"): - filename = os.path.abspath("scene.pickle") - # save scene and camera in pickle file - with open(filename, "wb") as file: - # dump information to the file - pickle.dump(scene, file) - print(f"saved scene in {filename}") - - filename = os.path.abspath("camera.pickle") - print(f"save scene in {filename}") - with open(filename, "wb") as file: - # dump information to the file - pickle.dump(camera, file) - print(f"saved camera in {filename}") + def toggle_edge_overdraw_antialiasing(self): + # toggle edge overdraw anti-aliasing + if self.use_moderngl: + print("no anti-aliasing available when using moderngl") + else: + self.use_antialiasing = not (self.use_antialiasing) + print(f"use_antialiasing = {self.use_antialiasing}") + if self.use_antialiasing: + self.scene.sigma = 1.0 + else: + self.scene.sigma = 0.0 + + def pickle_scene_and_cameras(self): + filename = os.path.abspath("scene.pickle") + # save scene and camera in pickle file + with open(filename, "wb") as file: + # dump information to the file + pickle.dump(self.scene, file) + print(f"saved scene in {filename}") + + filename = os.path.abspath("camera.pickle") + print(f"save scene in {filename}") + with open(filename, "wb") as file: + # dump information to the file + pickle.dump(self.camera, file) + print(f"saved camera in {filename}") + + def process_key(self, key): + if key == ord("r"): + self.toggle_renderer() + + if key == ord("p"): + self.toggle_perspective_texture_mapping() + + if key == ord("l"): + self.toggle_lights() + + if key == ord("a"): + self.toggle_edge_overdraw_antialiasing() + + if key == ord("s"): + self.pickle_scene_and_cameras() + + if key == ord("h"): + self.print_help() + + if key == ord("t"): + self.interactor.toggle_mode() + self.interactor.print_help() def run(): obj_file = os.path.join(deodr.data_path, "duck.obj") - mesh_viewer(obj_file, use_moderngl=False) + Viewer(obj_file, use_moderngl=False).start() if __name__ == "__main__": @@ -366,4 +499,4 @@ def run(): parser.add_argument("mesh_file", type=str, nargs="?", default=duck_file) args = parser.parse_args() mesh_file = args.mesh_file - mesh_viewer(mesh_file, use_moderngl=True) + Viewer(mesh_file, use_moderngl=True).start() From 926f86060cb1c428c683b6887306b53dc83400c3 Mon Sep 17 00:00:00 2001 From: Martin de La Gorce Date: Thu, 4 Mar 2021 00:49:16 +0000 Subject: [PATCH 2/8] improving viewer code --- deodr/differentiable_renderer.py | 4 +- deodr/examples/mesh_viewer.py | 539 +++++++++++++++++++------------ 2 files changed, 338 insertions(+), 205 deletions(-) diff --git a/deodr/differentiable_renderer.py b/deodr/differentiable_renderer.py index 9aeb789..12f684a 100644 --- a/deodr/differentiable_renderer.py +++ b/deodr/differentiable_renderer.py @@ -562,7 +562,7 @@ def render_error_backward(self, err_buffer_b, make_copies=True): if not self.backface_culling: raise BaseException( - "use backface_culling=True if you use gradient backpropagation to get valid gradient through edge antialiazing." + "use backface_culling=True if you use gradient backpropagation to get valid gradient through edge anti-aliasing." ) sigma, obs, image, z_buffer, err_buffer = self.store_backward antialiase_error = True @@ -598,7 +598,7 @@ def render_backward(self, image_b, make_copies=True): ) if not self.backface_culling: raise BaseException( - "use backface_culling=True if you use gradient backpropagation to get valid gradient through edge antialiazing." + "use backface_culling=True if you use gradient backpropagation to get valid gradient through edge anti-aliasing." ) sigma, image, z_buffer = self.store_backward antialiase_error = False diff --git a/deodr/examples/mesh_viewer.py b/deodr/examples/mesh_viewer.py index c56441e..b0a3df1 100644 --- a/deodr/examples/mesh_viewer.py +++ b/deodr/examples/mesh_viewer.py @@ -1,4 +1,4 @@ -"""Example of interactive 3D mesh visualization using deodr and opencv.""" +"""Example of interactive 3D mesh visualization using DEODR and OpenCV.""" import argparse import os @@ -42,6 +42,13 @@ def __init__( self.xy_translation_speed = xy_translation_speed self.camera = camera + def toggle_mode(self): + if self.mode == "object_centered_trackball": + self.mode = "camera_centered" + else: + self.mode = "object_centered_trackball" + print(f"trackball mode = {self.mode}") + def mouse_callback(self, event, x, y, flags, param): if event == 0 and flags == 0: return @@ -73,23 +80,21 @@ def mouse_callback(self, event, x, y, flags, param): if self.left_is_down and not (self.ctrl_is_down): if self.mode == "camera_centered": - - center_in_camera = self.camera.world_to_camera(self.object_center) rotation = Rotation.from_rotvec( np.array( [ - -self.rotation_speed * (y - self.y_last), - self.rotation_speed * (x - self.x_last), + -0.3 * self.rotation_speed * (y - self.y_last), + 0.3 * self.rotation_speed * (x - self.x_last), 0, ] ) ) - self.camera.extrinsic = rotation.as_dcm().dot(self.camera.extrinsic) + self.camera.extrinsic = rotation.as_matrix().dot(self.camera.extrinsic) # assert np.allclose(center_in_camera, self.camera.world_to_camera(self.object_center)) self.x_last = x self.y_last = y - if self.mode == "object_centered_trackball": + elif self.mode == "object_centered_trackball": rotation = Rotation.from_rotvec( np.array( @@ -100,7 +105,7 @@ def mouse_callback(self, event, x, y, flags, param): ] ) ) - n_rotation = rotation.as_dcm().dot(self.camera.extrinsic[:, :3]) + n_rotation = rotation.as_matrix().dot(self.camera.extrinsic[:, :3]) nt = ( self.camera.extrinsic[:, :3].dot(self.object_center) + self.camera.extrinsic[:, 3] @@ -119,10 +124,29 @@ def mouse_callback(self, event, x, y, flags, param): ) self.x_last = x self.y_last = y - if self.mode == "object_centered_trackball": - self.camera.extrinsic[2, 3] += self.z_translation_speed * ( - self.y_last - y - ) + elif self.mode == "object_centered_trackball": + if np.abs(self.y_last - y) >= np.abs(self.x_last - x): + self.camera.extrinsic[2, 3] += self.z_translation_speed * ( + self.y_last - y + ) + else: + rotation = Rotation.from_rotvec( + np.array( + [ + 0, + 0, + -self.rotation_speed * (self.x_last - x), + ] + ) + ) + + n_rotation = rotation.as_matrix().dot(self.camera.extrinsic[:, :3]) + nt = ( + self.camera.extrinsic[:, :3].dot(self.object_center) + + self.camera.extrinsic[:, 3] + - n_rotation.dot(self.object_center) + ) + self.camera.extrinsic = np.column_stack((n_rotation, nt)) self.x_last = x self.y_last = y else: @@ -144,220 +168,329 @@ def mouse_callback(self, event, x, y, flags, param): ) self.camera.extrinsic[0, 3] += tx self.camera.extrinsic[1, 3] += ty - center_in_camera = self.camera.world_to_camera(self.object_center) self.x_last = x self.y_last = y - assert np.max(np.abs(center_in_camera[:2])) < 1e-3 - - -def mesh_viewer( - file_or_mesh, - display_texture_map=True, - width=640, - height=480, - display_fps=True, - title=None, - use_moderngl=False, - light_directional=(0, 0, -0.5), - light_ambient=0.5, -): - if isinstance(file_or_mesh, str): - if title is None: - title = file_or_mesh - mesh_trimesh = trimesh.load(file_or_mesh) - mesh = ColoredTriMesh.from_trimesh(mesh_trimesh) - elif isinstance(file_or_mesh, trimesh.base.Trimesh): - mesh_trimesh = file_or_mesh - mesh = ColoredTriMesh.from_trimesh(mesh_trimesh) - if title is None: - title = "unknown" - elif isinstance(file_or_mesh, ColoredTriMesh): - mesh = file_or_mesh - if title is None: - title = "unknown" - else: - raise ( - BaseException( - f"unknown type {type(file_or_mesh)}for input obj_file_or_trimesh," - " can be string or trimesh.base.Trimesh" + + def print_help(self): + help_str = "" + if self.mode == "object_centered_trackball": + help_str += "Trackball:\n" + help_str += "mouse right + vertical motion = move in/out\n" + help_str += ( + "mouse right + horizontal motion = rotate object along camera z axis\n" ) - ) + help_str += ( + "mouse left + vertical motion = rotate object along camera x axis\n" + ) + help_str += ( + "mouse left + horizontal motion = rotate object along camera y axis\n" + ) + help_str += "CTRL + mouse left + vertical motion = translate object along camera y axis\n" + help_str += "CTRL + mouse left + horizontal motion = translate object along camera x axis\n" + else: + help_str += "Trackball:\n" + help_str += "mouse right + vertical motion = move in/out\n" + help_str += ( + "mouse right + horizontal motion = rotate camera along camera z axis\n" + ) + help_str += ( + "mouse left + vertical motion = rotate camera along camera x axis\n" + ) + help_str += ( + "mouse left + horizontal motion = rotate camera along camera y axis\n" + ) + help_str += "CTRL + mouse left + vertical motion = translate camera along camera y axis\n" + help_str += "CTRL + mouse left + horizontal motion = translate camera along camera x axis\n" - if display_texture_map: - ax = plt.subplot(111) - if mesh.textured: - mesh.plot_uv_map(ax) + print(help_str) - object_center = 0.5 * (mesh.vertices.max(axis=0) + mesh.vertices.min(axis=0)) - object_radius = np.max(mesh.vertices.max(axis=0) - mesh.vertices.min(axis=0)) - camera_center = object_center + np.array([0, 0, 3]) * object_radius +class Viewer: + def __init__( + self, + file_or_mesh, + display_texture_map=True, + width=640, + height=480, + display_fps=True, + title=None, + use_moderngl=False, + light_directional=(0, 0, -0.5), + light_ambient=0.5, + background_color=(1, 1, 1), + use_antialiasing=True, + use_light=True, + print_help=True, + ): + self.title = title + self.scene = differentiable_renderer.Scene3D(sigma=1) + self.set_mesh(file_or_mesh) + self.windowname = f"DEODR mesh viewer:{self.title}" - rotation = np.array([[1, 0, 0], [0, -1, 0], [0, 0, -1]]) - translation = -rotation.T.dot(camera_center) - extrinsic = np.column_stack((rotation, translation)) + self.width = width + self.height = height + self.display_fps = display_fps + self.use_moderngl = use_moderngl + self.use_antialiasing = use_antialiasing + self.use_light = use_light - focal = 2 * width - intrinsic = np.array([[focal, 0, width / 2], [0, focal, height / 2], [0, 0, 1]]) + if display_texture_map: + self.display_texture_map() - distortion = [0, 0, 0, 0, 0] - camera = differentiable_renderer.Camera( - extrinsic=extrinsic, - intrinsic=intrinsic, - width=width, - height=height, - distortion=distortion, - ) - use_antiliazing = True - use_light = True + self.set_background_color(background_color) + self.set_light(light_directional, light_ambient) + self.recenter_camera() - scene = differentiable_renderer.Scene3D(sigma=1) + if use_moderngl: + self.setup_moderngl() + else: + self.offscreen_renderer = None - scene.set_light( - light_directional=np.array(light_directional), light_ambient=light_ambient - ) - scene.set_mesh(mesh) + if print_help: + self.print_help() - scene.set_background_color([1, 1, 1]) + def set_light(self, light_directional, light_ambient): + self.light_directional = np.array(light_directional) + self.light_ambient = light_ambient + self.scene.set_light( + light_directional=self.light_directional, light_ambient=light_ambient + ) - if mesh.texture is not None: - mesh.texture = mesh.texture[ - :, :, ::-1 - ] # convert texture to GBR to avoid future conversion when ploting in Opencv + def setup_moderngl(self): + import deodr.opengl.moderngl - fps = 0 - fps_decay = 0.1 - windowname = f"DEODR mesh viewer:{title}" + self.offscreen_renderer = deodr.opengl.moderngl.OffscreenRenderer() + self.scene.mesh.compute_vertex_normals() + self.offscreen_renderer.set_scene(self.scene) + + def set_background_color(self, background_color): + self.scene.set_background_color(background_color) + + def display_texture_map(self): + if self.mesh.textured: + ax = plt.subplot(111) + self.mesh.plot_uv_map(ax) + + def set_mesh(self, file_or_mesh): + if isinstance(file_or_mesh, str): + if self.title is None: + self.title = file_or_mesh + mesh_trimesh = trimesh.load(file_or_mesh) + self.mesh = ColoredTriMesh.from_trimesh(mesh_trimesh) + elif isinstance(file_or_mesh, trimesh.base.Trimesh): + mesh_trimesh = file_or_mesh + self.mesh = ColoredTriMesh.from_trimesh(mesh_trimesh) + if self.title is None: + self.title = "unknown" + elif isinstance(file_or_mesh, ColoredTriMesh): + self.mesh = file_or_mesh + if self.title is None: + self.title = "unknown" + else: + raise ( + TypeError( + f"unknown type {type(file_or_mesh)} for input obj_file_or_trimesh," + " can be string or trimesh.base.Trimesh" + ) + ) + self.object_center = 0.5 * ( + self.mesh.vertices.max(axis=0) + self.mesh.vertices.min(axis=0) + ) + self.object_radius = np.max( + self.mesh.vertices.max(axis=0) - self.mesh.vertices.min(axis=0) + ) + self.scene.set_mesh(self.mesh) + + def recenter_camera(self): + camera_center = self.object_center + np.array([0, 0, 3]) * self.object_radius + rotation = np.array([[1, 0, 0], [0, -1, 0], [0, 0, -1]]) + translation = -rotation.T.dot(camera_center) + extrinsic = np.column_stack((rotation, translation)) + focal = 2 * self.width + intrinsic = np.array( + [[focal, 0, self.width / 2], [0, focal, self.height / 2], [0, 0, 1]] + ) - interactor = Interactor( - camera=camera, - object_center=object_center, - z_translation_speed=0.01 * object_radius, - xy_translation_speed=3e-4, - ) + distortion = [0, 0, 0, 0, 0] + self.camera = differentiable_renderer.Camera( + extrinsic=extrinsic, + intrinsic=intrinsic, + width=self.width, + height=self.height, + distortion=distortion, + ) - cv2.namedWindow(windowname, cv2.WINDOW_NORMAL) - cv2.resizeWindow(windowname, width, height) - cv2.setMouseCallback(windowname, interactor.mouse_callback) + self.interactor = Interactor( + camera=self.camera, + object_center=self.object_center, + z_translation_speed=0.01 * self.object_radius, + xy_translation_speed=3e-4, + ) - if use_moderngl: - import deodr.opengl.moderngl + def start(self): + fps = 0 + fps_decay = 0.1 + cv2.namedWindow(self.windowname, cv2.WINDOW_NORMAL) + cv2.resizeWindow(self.windowname, self.width, self.height) + cv2.setMouseCallback(self.windowname, self.interactor.mouse_callback) + while cv2.getWindowProperty(self.windowname, 0) >= 0: + # mesh.set_vertices(mesh.vertices+np.random.randn(*mesh.vertices.shape)*0.001) + self.width, self.height = cv2.getWindowImageRect(self.windowname)[2:] + self.resize_camera() + start = time.perf_counter() + + if self.use_moderngl: + image = self.offscreen_renderer.render(self.camera) + else: + image = self.scene.render(self.interactor.camera).astype(np.float32) + + if self.display_fps: + self.print_fps(image, fps) + + cv2.imshow(self.windowname, cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) + stop = time.perf_counter() + fps = (1 - fps_decay) * fps + fps_decay * (1 / (stop - start)) + key = cv2.waitKey(1) + if key > 0: + self.process_key(key) + + def resize_camera(self): + focal = 2 * self.width + intrinsic = np.array( + [[focal, 0, self.width / 2], [0, focal, self.height / 2], [0, 0, 1]] + ) + self.camera.intrinsic = intrinsic + self.camera.width = self.width + self.camera.height = self.height + + def print_fps(self, image, fps): + font = cv2.FONT_HERSHEY_SIMPLEX + bottom_left_corner_of_text = (20, image.shape[0] - 20) + font_scale = 1 + font_color = (0, 0, 255) + thickness = 2 + cv2.putText( + image, + "fps:%0.1f" % fps, + bottom_left_corner_of_text, + font, + font_scale, + font_color, + thickness, + ) - offscreen_renderer = deodr.opengl.moderngl.OffscreenRenderer() - scene.mesh.compute_vertex_normals() - offscreen_renderer.set_scene(scene) - else: - offscreen_renderer = None - while cv2.getWindowProperty(windowname, 0) >= 0: - - # mesh.set_vertices(mesh.vertices+np.random.randn(*mesh.vertices.shape)*0.001) - width, height = cv2.getWindowImageRect(windowname)[2:] - focal = 2 * width - intrinsic = np.array([[focal, 0, width / 2], [0, focal, height / 2], [0, 0, 1]]) - camera.intrinsic = intrinsic - camera.width = width - camera.height = height - - start = time.clock() - if use_moderngl: - image = offscreen_renderer.render(camera) + def print_help(self): + help_str = "" + help_str += "-----------------\n" + help_str += "DEODR Mesh Viewer\n" + help_str += "-----------------\n" + help_str += "Keys:\n" + help_str += "h: print this help\n" + help_str += "r: toggle renderer between DEODR cpu rendering and moderngl\n" + help_str += "p: toggle between linear texture mapping and perspective correct texture mapping\n" + help_str += "l: toggle between uniform lighting vs directional + ambient\n" + help_str += "a: toggle edge overdraw anti-aliasing (DEODR renderin only)\n" + help_str += "s: save scene and camera in a pickle file\n" + help_str += "t: change camera trackball mode\n" + print(help_str) + self.interactor.print_help() + + def toggle_renderer(self): + # Switch renderer between DEODR cpu rendering and moderngl + self.use_moderngl = not (self.use_moderngl) + print(f"use_moderngl = { self.use_moderngl}") + + if self.use_moderngl and self.offscreen_renderer is None: + self.setup_moderngl() + + def toggle_perspective_texture_mapping(self): + # Switch between linear texture mapping and perspective correct texture mapping + if self.use_moderngl: + print("can only use perspective correct mapping when using moderngl") else: - image = scene.render(interactor.camera) - - if display_fps: - font = cv2.FONT_HERSHEY_SIMPLEX - bottom_left_corner_of_text = (20, height - 20) - font_scale = 1 - font_color = (0, 0, 255) - thickness = 2 - cv2.putText( - image, - "fps:%0.1f" % fps, - bottom_left_corner_of_text, - font, - font_scale, - font_color, - thickness, - ) - cv2.imshow(windowname, image) - stop = time.clock() - fps = (1 - fps_decay) * fps + fps_decay * (1 / (stop - start)) - key = cv2.waitKey(1) - if key >= 0: - if key == ord("r"): - # change renderer between DEODR cpu rendering and moderngl - use_moderngl = not (use_moderngl) - print(f"use_moderngl = {use_moderngl}") - - if offscreen_renderer is None: - offscreen_renderer = deodr.opengl.moderngl.OffscreenRenderer() - scene.mesh.compute_vertex_normals() - offscreen_renderer.set_scene(scene) - - if key == ord("p"): - if use_moderngl: - print( - "can only use perspective corect mapping when using moderngl" - ) - else: - # toggle perspective correct mapping (texture or interpolation) - scene.perspective_correct = not (scene.perspective_correct) - print(f"perspective_correct = {scene.perspective_correct}") - - if key == ord("l"): - # toggle directional light + ambient vs ambient = 1 - use_light = not (use_light) - print(f"use_light = {use_light}") - if use_light: - if use_moderngl: - offscreen_renderer.set_light( - light_directional=np.array(light_directional), - light_ambient=light_ambient, - ) - else: - scene.set_light( - light_directional=np.array(light_directional), - light_ambient=light_ambient, - ) - else: - if use_moderngl: - offscreen_renderer.set_light( - light_directional=(0, 0, 0), light_ambient=1.0, - ) - else: - scene.set_light(light_directional=None, light_ambient=1.0) + # toggle perspective correct mapping (texture or interpolation) + self.scene.perspective_correct = not (self.scene.perspective_correct) + print(f"perspective_correct = {self.scene.perspective_correct}") + + def toggle_lights(self): + # toggle directional light + ambient vs ambient = 1 + self.use_light = not (self.use_light) + print(f"use_light = { self.use_light}") + + if self.use_light: + if self.use_moderngl: + self.offscreen_renderer.set_light( + light_directional=np.array(self.light_directional), + light_ambient=self.light_ambient, + ) + else: + self.scene.set_light( + light_directional=np.array(self.light_directional), + light_ambient=self.light_ambient, + ) + else: + if self.use_moderngl: + self.offscreen_renderer.set_light( + light_directional=(0, 0, 0), + light_ambient=1.0, + ) + else: + self.scene.set_light(light_directional=None, light_ambient=1.0) - if key == ord("a"): - # toggle edge overdraw anti-aliasing - if use_moderngl: - print("no anti-alizaing available when using moderngl") - else: - use_antiliazing = not (use_antiliazing) - print(f"use_antialiazing = {use_antiliazing}") - if use_antiliazing: - scene.sigma = 1.0 - else: - scene.sigma = 0.0 - - if key == ord("s"): - filename = os.path.abspath("scene.pickle") - # save scene and camera in pickle file - with open(filename, "wb") as file: - # dump information to the file - pickle.dump(scene, file) - print(f"saved scene in {filename}") - - filename = os.path.abspath("camera.pickle") - print(f"save scene in {filename}") - with open(filename, "wb") as file: - # dump information to the file - pickle.dump(camera, file) - print(f"saved camera in {filename}") + def toggle_edge_overdraw_antialiasing(self): + # toggle edge overdraw anti-aliasing + if self.use_moderngl: + print("no anti-aliasing available when using moderngl") + else: + self.use_antialiasing = not (self.use_antialiasing) + print(f"use_antialiasing = {self.use_antialiasing}") + if self.use_antialiasing: + self.scene.sigma = 1.0 + else: + self.scene.sigma = 0.0 + + def pickle_scene_and_cameras(self): + filename = os.path.abspath("scene.pickle") + # save scene and camera in pickle file + with open(filename, "wb") as file: + # dump information to the file + pickle.dump(self.scene, file) + print(f"saved scene in {filename}") + + filename = os.path.abspath("camera.pickle") + print(f"save scene in {filename}") + with open(filename, "wb") as file: + # dump information to the file + pickle.dump(self.camera, file) + print(f"saved camera in {filename}") + + def process_key(self, key): + if key == ord("r"): + self.toggle_renderer() + + if key == ord("p"): + self.toggle_perspective_texture_mapping() + + if key == ord("l"): + self.toggle_lights() + + if key == ord("a"): + self.toggle_edge_overdraw_antialiasing() + + if key == ord("s"): + self.pickle_scene_and_cameras() + + if key == ord("h"): + self.print_help() + + if key == ord("t"): + self.interactor.toggle_mode() + self.interactor.print_help() def run(): obj_file = os.path.join(deodr.data_path, "duck.obj") - mesh_viewer(obj_file, use_moderngl=False) + Viewer(obj_file, use_moderngl=False).start() if __name__ == "__main__": @@ -366,4 +499,4 @@ def run(): parser.add_argument("mesh_file", type=str, nargs="?", default=duck_file) args = parser.parse_args() mesh_file = args.mesh_file - mesh_viewer(mesh_file, use_moderngl=True) + Viewer(mesh_file, use_moderngl=True).start() From e630f48516da776dcdf3c99dec6d75600cc88d65 Mon Sep 17 00:00:00 2001 From: Martin de La Gorce Date: Thu, 4 Mar 2021 01:29:10 +0000 Subject: [PATCH 3/8] improve mesh viewer code --- deodr/examples/mesh_viewer.py | 129 +++++++++++++++++++--------------- 1 file changed, 72 insertions(+), 57 deletions(-) diff --git a/deodr/examples/mesh_viewer.py b/deodr/examples/mesh_viewer.py index b0a3df1..3755c74 100644 --- a/deodr/examples/mesh_viewer.py +++ b/deodr/examples/mesh_viewer.py @@ -4,6 +4,7 @@ import os import time import pickle +from typing import OrderedDict import cv2 @@ -173,8 +174,8 @@ def mouse_callback(self, event, x, y, flags, param): def print_help(self): help_str = "" + help_str += "Mouse:\n" if self.mode == "object_centered_trackball": - help_str += "Trackball:\n" help_str += "mouse right + vertical motion = move in/out\n" help_str += ( "mouse right + horizontal motion = rotate object along camera z axis\n" @@ -188,7 +189,6 @@ def print_help(self): help_str += "CTRL + mouse left + vertical motion = translate object along camera y axis\n" help_str += "CTRL + mouse left + horizontal motion = translate object along camera x axis\n" else: - help_str += "Trackball:\n" help_str += "mouse right + vertical motion = move in/out\n" help_str += ( "mouse right + horizontal motion = rotate camera along camera z axis\n" @@ -220,7 +220,7 @@ def __init__( background_color=(1, 1, 1), use_antialiasing=True, use_light=True, - print_help=True, + fps_exp_average_decay=0.1, ): self.title = title self.scene = differentiable_renderer.Scene3D(sigma=1) @@ -233,7 +233,8 @@ def __init__( self.use_moderngl = use_moderngl self.use_antialiasing = use_antialiasing self.use_light = use_light - + self.fps_exp_average_decay = fps_exp_average_decay + self.last_time = None if display_texture_map: self.display_texture_map() @@ -246,8 +247,7 @@ def __init__( else: self.offscreen_renderer = None - if print_help: - self.print_help() + self.register_keys() def set_light(self, light_directional, light_ambient): self.light_directional = np.array(light_directional) @@ -327,32 +327,49 @@ def recenter_camera(self): xy_translation_speed=3e-4, ) - def start(self): - fps = 0 - fps_decay = 0.1 + def start(self, print_help=True, loop=True): + """Open the window and start the loop if loop true.""" + if print_help: + self.print_help() + self.fps = 0 cv2.namedWindow(self.windowname, cv2.WINDOW_NORMAL) cv2.resizeWindow(self.windowname, self.width, self.height) cv2.setMouseCallback(self.windowname, self.interactor.mouse_callback) - while cv2.getWindowProperty(self.windowname, 0) >= 0: - # mesh.set_vertices(mesh.vertices+np.random.randn(*mesh.vertices.shape)*0.001) - self.width, self.height = cv2.getWindowImageRect(self.windowname)[2:] - self.resize_camera() - start = time.perf_counter() + if loop: + while cv2.getWindowProperty(self.windowname, 0) >= 0: + self.refresh() + + def update_fps(self): + new_time = time.perf_counter() + if self.last_time is None: + self.fps = 0 + elif self.fps == 0: + self.fps = 1 / (new_time - self.last_time) + else: + new_fps = 1 / (new_time - self.last_time) + self.fps = ( + 1 - self.fps_exp_average_decay + ) * self.fps + self.fps_exp_average_decay * new_fps + self.last_time = new_time - if self.use_moderngl: - image = self.offscreen_renderer.render(self.camera) - else: - image = self.scene.render(self.interactor.camera).astype(np.float32) + def refresh(self): + self.width, self.height = cv2.getWindowImageRect(self.windowname)[2:] + self.resize_camera() + + if self.use_moderngl: + image = self.offscreen_renderer.render(self.camera) + else: + image = self.scene.render(self.interactor.camera).astype(np.float32) + + self.update_fps() + if self.display_fps: + self.print_fps(image, self.fps) - if self.display_fps: - self.print_fps(image, fps) + cv2.imshow(self.windowname, cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) - cv2.imshow(self.windowname, cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) - stop = time.perf_counter() - fps = (1 - fps_decay) * fps + fps_decay * (1 / (stop - start)) - key = cv2.waitKey(1) - if key > 0: - self.process_key(key) + key = cv2.waitKey(1) + if key > 0: + self.process_key(key) def resize_camera(self): focal = 2 * self.width @@ -380,23 +397,19 @@ def print_fps(self, image, fps): ) def print_help(self): + """Print the help message.""" help_str = "" help_str += "-----------------\n" help_str += "DEODR Mesh Viewer\n" help_str += "-----------------\n" help_str += "Keys:\n" - help_str += "h: print this help\n" - help_str += "r: toggle renderer between DEODR cpu rendering and moderngl\n" - help_str += "p: toggle between linear texture mapping and perspective correct texture mapping\n" - help_str += "l: toggle between uniform lighting vs directional + ambient\n" - help_str += "a: toggle edge overdraw anti-aliasing (DEODR renderin only)\n" - help_str += "s: save scene and camera in a pickle file\n" - help_str += "t: change camera trackball mode\n" + for key, func in self.keys_map.items(): + help_str += f"{key}: {func.__doc__}\n" print(help_str) self.interactor.print_help() def toggle_renderer(self): - # Switch renderer between DEODR cpu rendering and moderngl + """Toggle the renderer between DEODR cpu rendering and moderngl.""" self.use_moderngl = not (self.use_moderngl) print(f"use_moderngl = { self.use_moderngl}") @@ -404,16 +417,15 @@ def toggle_renderer(self): self.setup_moderngl() def toggle_perspective_texture_mapping(self): - # Switch between linear texture mapping and perspective correct texture mapping + """Toggle between linear texture mapping and perspective correct texture mapping.""" if self.use_moderngl: print("can only use perspective correct mapping when using moderngl") else: - # toggle perspective correct mapping (texture or interpolation) self.scene.perspective_correct = not (self.scene.perspective_correct) print(f"perspective_correct = {self.scene.perspective_correct}") def toggle_lights(self): - # toggle directional light + ambient vs ambient = 1 + """Toggle between uniform lighting vs directional + ambient""" self.use_light = not (self.use_light) print(f"use_light = { self.use_light}") @@ -438,7 +450,7 @@ def toggle_lights(self): self.scene.set_light(light_directional=None, light_ambient=1.0) def toggle_edge_overdraw_antialiasing(self): - # toggle edge overdraw anti-aliasing + """Toggle edge overdraw anti-aliasing (DEODR renderin only).""" if self.use_moderngl: print("no anti-aliasing available when using moderngl") else: @@ -450,6 +462,7 @@ def toggle_edge_overdraw_antialiasing(self): self.scene.sigma = 0.0 def pickle_scene_and_cameras(self): + """Save scene and camera in a pickle file.""" filename = os.path.abspath("scene.pickle") # save scene and camera in pickle file with open(filename, "wb") as file: @@ -464,28 +477,30 @@ def pickle_scene_and_cameras(self): pickle.dump(self.camera, file) print(f"saved camera in {filename}") - def process_key(self, key): - if key == ord("r"): - self.toggle_renderer() - - if key == ord("p"): - self.toggle_perspective_texture_mapping() - - if key == ord("l"): - self.toggle_lights() - - if key == ord("a"): - self.toggle_edge_overdraw_antialiasing() + def toggle_interactor_mode(self): + """Change the camera interactor mode.""" + self.interactor.toggle_mode() + self.interactor.print_help() - if key == ord("s"): - self.pickle_scene_and_cameras() + def register_keys(self): + self.keys_map = {} + self.register_key("h", self.print_help) + self.register_key("r", self.toggle_renderer) + self.register_key("p", self.toggle_perspective_texture_mapping) + self.register_key("l", self.toggle_lights) + self.register_key("a", self.toggle_edge_overdraw_antialiasing) + self.register_key("s", self.pickle_scene_and_cameras) + self.register_key("t", self.toggle_interactor_mode) - if key == ord("h"): - self.print_help() + def register_key(self, key, func): + self.keys_map[key] = func - if key == ord("t"): - self.interactor.toggle_mode() - self.interactor.print_help() + def process_key(self, key): + chr_key = chr(key) + if chr_key in self.keys_map: + self.keys_map[chr(key)]() + else: + print(f"no function registered for key {chr_key}") def run(): From 49cffb3d1f829805232ff18b7517b429497322ad Mon Sep 17 00:00:00 2001 From: Martin de La Gorce Date: Thu, 4 Mar 2021 01:39:16 +0000 Subject: [PATCH 4/8] improving interactor mode --- deodr/examples/mesh_viewer.py | 89 +++++++++++++++-------------------- 1 file changed, 39 insertions(+), 50 deletions(-) diff --git a/deodr/examples/mesh_viewer.py b/deodr/examples/mesh_viewer.py index 3755c74..d20833b 100644 --- a/deodr/examples/mesh_viewer.py +++ b/deodr/examples/mesh_viewer.py @@ -4,7 +4,6 @@ import os import time import pickle -from typing import OrderedDict import cv2 @@ -50,6 +49,22 @@ def toggle_mode(self): self.mode = "object_centered_trackball" print(f"trackball mode = {self.mode}") + def rotate( + self, + rot_vec, + ): + rotation = Rotation.from_rotvec(np.array(rot_vec)) + if self.mode == "camera_centered": + self.camera.extrinsic = rotation.as_matrix().dot(self.camera.extrinsic) + else: + n_rotation = rotation.as_matrix().dot(self.camera.extrinsic[:, :3]) + nt = ( + self.camera.extrinsic[:, :3].dot(self.object_center) + + self.camera.extrinsic[:, 3] + - n_rotation.dot(self.object_center) + ) + self.camera.extrinsic = np.column_stack((n_rotation, nt)) + def mouse_callback(self, event, x, y, flags, param): if event == 0 and flags == 0: return @@ -81,75 +96,49 @@ def mouse_callback(self, event, x, y, flags, param): if self.left_is_down and not (self.ctrl_is_down): if self.mode == "camera_centered": - rotation = Rotation.from_rotvec( - np.array( - [ - -0.3 * self.rotation_speed * (y - self.y_last), - 0.3 * self.rotation_speed * (x - self.x_last), - 0, - ] - ) - ) - self.camera.extrinsic = rotation.as_matrix().dot(self.camera.extrinsic) + rot_vec = [ + -0.3 * self.rotation_speed * (y - self.y_last), + 0.3 * self.rotation_speed * (x - self.x_last), + 0, + ] + self.rotate(rot_vec) + # assert np.allclose(center_in_camera, self.camera.world_to_camera(self.object_center)) self.x_last = x self.y_last = y elif self.mode == "object_centered_trackball": - rotation = Rotation.from_rotvec( - np.array( - [ - self.rotation_speed * (y - self.y_last), - -self.rotation_speed * (x - self.x_last), - 0, - ] - ) + self.rotate( + [ + self.rotation_speed * (y - self.y_last), + -self.rotation_speed * (x - self.x_last), + 0, + ] ) - n_rotation = rotation.as_matrix().dot(self.camera.extrinsic[:, :3]) - nt = ( - self.camera.extrinsic[:, :3].dot(self.object_center) - + self.camera.extrinsic[:, 3] - - n_rotation.dot(self.object_center) - ) - self.camera.extrinsic = np.column_stack((n_rotation, nt)) + self.x_last = x self.y_last = y else: raise (BaseException(f"unknown camera mode {self.mode}")) if self.right_is_down and not (self.ctrl_is_down): - if self.mode == "camera_centered": - self.camera.extrinsic[2, 3] += self.z_translation_speed * ( - self.y_last - y - ) - self.x_last = x - self.y_last = y - elif self.mode == "object_centered_trackball": + if self.mode in ["camera_centered", "object_centered_trackball"]: if np.abs(self.y_last - y) >= np.abs(self.x_last - x): self.camera.extrinsic[2, 3] += self.z_translation_speed * ( self.y_last - y ) else: - rotation = Rotation.from_rotvec( - np.array( - [ - 0, - 0, - -self.rotation_speed * (self.x_last - x), - ] - ) - ) - - n_rotation = rotation.as_matrix().dot(self.camera.extrinsic[:, :3]) - nt = ( - self.camera.extrinsic[:, :3].dot(self.object_center) - + self.camera.extrinsic[:, 3] - - n_rotation.dot(self.object_center) + self.rotate( + [ + 0, + 0, + -self.rotation_speed * (self.x_last - x), + ] ) - self.camera.extrinsic = np.column_stack((n_rotation, nt)) self.x_last = x self.y_last = y + else: raise (BaseException(f"unknown camera mode {self.mode}")) @@ -336,7 +325,7 @@ def start(self, print_help=True, loop=True): cv2.resizeWindow(self.windowname, self.width, self.height) cv2.setMouseCallback(self.windowname, self.interactor.mouse_callback) if loop: - while cv2.getWindowProperty(self.windowname, 0) >= 0: + while cv2.getWindowProperty(self.windowname, 0) >= 0: self.refresh() def update_fps(self): From 12d7f876e9a04e25447281c2573dcf9437376b59 Mon Sep 17 00:00:00 2001 From: Martin de La Gorce Date: Thu, 4 Mar 2021 09:47:35 +0000 Subject: [PATCH 5/8] adding control to change the camera field of view in the viewer --- deodr/examples/mesh_viewer.py | 42 +++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/deodr/examples/mesh_viewer.py b/deodr/examples/mesh_viewer.py index d20833b..c03b414 100644 --- a/deodr/examples/mesh_viewer.py +++ b/deodr/examples/mesh_viewer.py @@ -92,6 +92,7 @@ def mouse_callback(self, event, x, y, flags, param): self.middle_is_down = False self.ctrl_is_down = flags & cv2.EVENT_FLAG_CTRLKEY + self.shift_is_down = flags & cv2.EVENT_FLAG_SHIFTKEY if self.left_is_down and not (self.ctrl_is_down): @@ -122,6 +123,14 @@ def mouse_callback(self, event, x, y, flags, param): else: raise (BaseException(f"unknown camera mode {self.mode}")) + if self.right_is_down and self.shift_is_down: + delta_y = self.y_last - y + ratio = np.power(2, delta_y / 20) + self.camera.intrinsic[0, 0] = self.camera.intrinsic[0, 0] * ratio + self.camera.intrinsic[1, 1] = self.camera.intrinsic[1, 1] * ratio + self.x_last = x + self.y_last = y + if self.right_is_down and not (self.ctrl_is_down): if self.mode in ["camera_centered", "object_centered_trackball"]: if np.abs(self.y_last - y) >= np.abs(self.x_last - x): @@ -165,18 +174,25 @@ def print_help(self): help_str = "" help_str += "Mouse:\n" if self.mode == "object_centered_trackball": - help_str += "mouse right + vertical motion = move in/out\n" - help_str += ( - "mouse right + horizontal motion = rotate object along camera z axis\n" - ) + help_str += ( "mouse left + vertical motion = rotate object along camera x axis\n" ) help_str += ( "mouse left + horizontal motion = rotate object along camera y axis\n" ) + help_str += ( + "mouse right + vertical motion = translate object along camera z axis\n" + ) + help_str += ( + "mouse right + horizontal motion = rotate object along camera z axis\n" + ) help_str += "CTRL + mouse left + vertical motion = translate object along camera y axis\n" help_str += "CTRL + mouse left + horizontal motion = translate object along camera x axis\n" + + help_str += ( + "SHIFT + mouse left + vertical motion = changes the field of view\n" + ) else: help_str += "mouse right + vertical motion = move in/out\n" help_str += ( @@ -190,6 +206,10 @@ def print_help(self): ) help_str += "CTRL + mouse left + vertical motion = translate camera along camera y axis\n" help_str += "CTRL + mouse left + horizontal motion = translate camera along camera x axis\n" + help_str += ( + "SHIFT + mouse left + vertical motion = changes the camera field of view\n" + ) + print(help_str) @@ -210,6 +230,7 @@ def __init__( use_antialiasing=True, use_light=True, fps_exp_average_decay=0.1, + horizontal_fov=60, ): self.title = title self.scene = differentiable_renderer.Scene3D(sigma=1) @@ -224,6 +245,8 @@ def __init__( self.use_light = use_light self.fps_exp_average_decay = fps_exp_average_decay self.last_time = None + self.horizontal_fov = horizontal_fov + if display_texture_map: self.display_texture_map() @@ -295,7 +318,7 @@ def recenter_camera(self): rotation = np.array([[1, 0, 0], [0, -1, 0], [0, 0, -1]]) translation = -rotation.T.dot(camera_center) extrinsic = np.column_stack((rotation, translation)) - focal = 2 * self.width + focal = 0.5 * self.width / np.tan(0.5 * self.horizontal_fov * np.pi / 180) intrinsic = np.array( [[focal, 0, self.width / 2], [0, focal, self.height / 2], [0, 0, 1]] ) @@ -361,9 +384,14 @@ def refresh(self): self.process_key(key) def resize_camera(self): - focal = 2 * self.width + ratio = self.width / self.camera.width + intrinsic = np.array( - [[focal, 0, self.width / 2], [0, focal, self.height / 2], [0, 0, 1]] + [ + [self.camera.intrinsic[0, 0] * ratio, 0, self.width / 2], + [0, self.camera.intrinsic[1, 1] * ratio, self.height / 2], + [0, 0, 1], + ] ) self.camera.intrinsic = intrinsic self.camera.width = self.width From eac36a5d19cb8d8290a38dc09dc8e2e7ee66b2ab Mon Sep 17 00:00:00 2001 From: Martin de La Gorce Date: Thu, 4 Mar 2021 09:52:49 +0000 Subject: [PATCH 6/8] cleaning help --- deodr/examples/mesh_viewer.py | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/deodr/examples/mesh_viewer.py b/deodr/examples/mesh_viewer.py index c03b414..fef4c4c 100644 --- a/deodr/examples/mesh_viewer.py +++ b/deodr/examples/mesh_viewer.py @@ -176,40 +176,37 @@ def print_help(self): if self.mode == "object_centered_trackball": help_str += ( - "mouse left + vertical motion = rotate object along camera x axis\n" + "mouse left + vertical motion: rotate object along camera x axis\n" ) help_str += ( - "mouse left + horizontal motion = rotate object along camera y axis\n" + "mouse left + horizontal motion: rotate object along camera y axis\n" ) help_str += ( - "mouse right + vertical motion = translate object along camera z axis\n" + "mouse right + vertical motion: translate object along camera z axis\n" ) help_str += ( - "mouse right + horizontal motion = rotate object along camera z axis\n" + "mouse right + horizontal motion: rotate object along camera z axis\n" ) - help_str += "CTRL + mouse left + vertical motion = translate object along camera y axis\n" - help_str += "CTRL + mouse left + horizontal motion = translate object along camera x axis\n" + help_str += "CTRL + mouse left + vertical motion: translate object along camera y axis\n" + help_str += "CTRL + mouse left + horizontal motion: translate object along camera x axis\n" help_str += ( - "SHIFT + mouse left + vertical motion = changes the field of view\n" + "SHIFT + mouse left + vertical motion: change the camera field of view\n" ) else: - help_str += "mouse right + vertical motion = move in/out\n" + help_str += "mouse right + vertical motion: translate camera along its z axis\n" help_str += ( - "mouse right + horizontal motion = rotate camera along camera z axis\n" + "mouse right + horizontal motion: rotate camera along its z axis\n" ) help_str += ( - "mouse left + vertical motion = rotate camera along camera x axis\n" + "mouse left + vertical motion: rotate camera along its x axis\n" ) help_str += ( - "mouse left + horizontal motion = rotate camera along camera y axis\n" + "mouse left + horizontal motion: rotate camera along its y axis\n" ) - help_str += "CTRL + mouse left + vertical motion = translate camera along camera y axis\n" - help_str += "CTRL + mouse left + horizontal motion = translate camera along camera x axis\n" - help_str += ( - "SHIFT + mouse left + vertical motion = changes the camera field of view\n" - ) - + help_str += "CTRL + mouse left + vertical motion: translate camera along its y axis\n" + help_str += "CTRL + mouse left + horizontal motion: translate camera along its x axis\n" + help_str += "SHIFT + mouse left + vertical motion: change the camera field of view\n" print(help_str) From b913b52bda69505f2bfde4bdae69623f3f64dbd2 Mon Sep 17 00:00:00 2001 From: Martin de La Gorce Date: Thu, 4 Mar 2021 21:22:45 +0000 Subject: [PATCH 7/8] adding a recording button in the viewer --- deodr/examples/mesh_viewer.py | 53 +++++++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/deodr/examples/mesh_viewer.py b/deodr/examples/mesh_viewer.py index fef4c4c..a799384 100644 --- a/deodr/examples/mesh_viewer.py +++ b/deodr/examples/mesh_viewer.py @@ -190,17 +190,15 @@ def print_help(self): help_str += "CTRL + mouse left + vertical motion: translate object along camera y axis\n" help_str += "CTRL + mouse left + horizontal motion: translate object along camera x axis\n" - help_str += ( - "SHIFT + mouse left + vertical motion: change the camera field of view\n" - ) + help_str += "SHIFT + mouse left + vertical motion: change the camera field of view\n" else: - help_str += "mouse right + vertical motion: translate camera along its z axis\n" help_str += ( - "mouse right + horizontal motion: rotate camera along its z axis\n" + "mouse right + vertical motion: translate camera along its z axis\n" ) help_str += ( - "mouse left + vertical motion: rotate camera along its x axis\n" + "mouse right + horizontal motion: rotate camera along its z axis\n" ) + help_str += "mouse left + vertical motion: rotate camera along its x axis\n" help_str += ( "mouse left + horizontal motion: rotate camera along its y axis\n" ) @@ -243,6 +241,8 @@ def __init__( self.fps_exp_average_decay = fps_exp_average_decay self.last_time = None self.horizontal_fov = horizontal_fov + self.video_writer = None + self.recording = False if display_texture_map: self.display_texture_map() @@ -368,13 +368,25 @@ def refresh(self): if self.use_moderngl: image = self.offscreen_renderer.render(self.camera) else: - image = self.scene.render(self.interactor.camera).astype(np.float32) + image = (self.scene.render(self.interactor.camera) * 255).astype(np.uint8) + + bgr_image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + if self.recording: + self.video_writer.write(bgr_image.astype(np.uint8)) self.update_fps() + if self.recording: + cv2.circle( + bgr_image, + (image.shape[1] - 20, image.shape[0] - 20), + 8, + (0, 0, 255), + cv2.FILLED, + ) if self.display_fps: - self.print_fps(image, self.fps) + self.print_fps(bgr_image, self.fps) - cv2.imshow(self.windowname, cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) + cv2.imshow(self.windowname, bgr_image) key = cv2.waitKey(1) if key > 0: @@ -439,7 +451,7 @@ def toggle_perspective_texture_mapping(self): print(f"perspective_correct = {self.scene.perspective_correct}") def toggle_lights(self): - """Toggle between uniform lighting vs directional + ambient""" + """Toggle between uniform lighting vs directional + ambient.""" self.use_light = not (self.use_light) print(f"use_light = { self.use_light}") @@ -464,7 +476,7 @@ def toggle_lights(self): self.scene.set_light(light_directional=None, light_ambient=1.0) def toggle_edge_overdraw_antialiasing(self): - """Toggle edge overdraw anti-aliasing (DEODR renderin only).""" + """Toggle edge overdraw anti-aliasing (DEODR rendering only).""" if self.use_moderngl: print("no anti-aliasing available when using moderngl") else: @@ -496,6 +508,22 @@ def toggle_interactor_mode(self): self.interactor.toggle_mode() self.interactor.print_help() + def toggle_video_recording(self): + """Start and stop video recording.""" + if not self.recording: + id = 0 + while os.path.exists(f"deodr_viewer_recording{id}.avi"): + id += 1 + filename = f"deodr_viewer_recording{id}.avi" + + self.video_writer = cv2.VideoWriter( + filename, cv2.VideoWriter_fourcc(*"MJPG"), 30, (self.width, self.height) + ) + self.recording = True + else: + self.video_writer.release() + self.recording = False + def register_keys(self): self.keys_map = {} self.register_key("h", self.print_help) @@ -503,7 +531,8 @@ def register_keys(self): self.register_key("p", self.toggle_perspective_texture_mapping) self.register_key("l", self.toggle_lights) self.register_key("a", self.toggle_edge_overdraw_antialiasing) - self.register_key("s", self.pickle_scene_and_cameras) + self.register_key("d", self.pickle_scene_and_cameras) + self.register_key("s", self.toggle_video_recording) self.register_key("t", self.toggle_interactor_mode) def register_key(self, key, func): From 29daf207db27902270e2ab8e3d1a0a474225ca88 Mon Sep 17 00:00:00 2001 From: Martin de La Gorce Date: Thu, 4 Mar 2021 21:32:07 +0000 Subject: [PATCH 8/8] adding video file pattern --- deodr/examples/mesh_viewer.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/deodr/examples/mesh_viewer.py b/deodr/examples/mesh_viewer.py index a799384..3e6f5d7 100644 --- a/deodr/examples/mesh_viewer.py +++ b/deodr/examples/mesh_viewer.py @@ -226,6 +226,8 @@ def __init__( use_light=True, fps_exp_average_decay=0.1, horizontal_fov=60, + video_pattern="deodr_viewer_recording{id}.avi", + video_format="MJPG", ): self.title = title self.scene = differentiable_renderer.Scene3D(sigma=1) @@ -243,6 +245,8 @@ def __init__( self.horizontal_fov = horizontal_fov self.video_writer = None self.recording = False + self.video_pattern = video_pattern + self.video_format = video_format if display_texture_map: self.display_texture_map() @@ -371,6 +375,7 @@ def refresh(self): image = (self.scene.render(self.interactor.camera) * 255).astype(np.uint8) bgr_image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + if self.recording: self.video_writer.write(bgr_image.astype(np.uint8)) @@ -512,12 +517,15 @@ def toggle_video_recording(self): """Start and stop video recording.""" if not self.recording: id = 0 - while os.path.exists(f"deodr_viewer_recording{id}.avi"): + while os.path.exists(self.video_pattern.format(**dict(id=id))): id += 1 - filename = f"deodr_viewer_recording{id}.avi" + filename = self.video_pattern.format(**dict(id=id)) self.video_writer = cv2.VideoWriter( - filename, cv2.VideoWriter_fourcc(*"MJPG"), 30, (self.width, self.height) + filename, + cv2.VideoWriter_fourcc(*self.video_format), + 30, + (self.width, self.height), ) self.recording = True else: