From 6a049b2cd626c181e617092466b82a715709a435 Mon Sep 17 00:00:00 2001 From: Gu Wang Date: Sun, 29 Aug 2021 16:16:04 +0800 Subject: [PATCH] add reference code for xyz_crop generation --- lib/meshrenderer/LICENSE | 22 + lib/meshrenderer/__init__.py | 0 lib/meshrenderer/box3d_renderer.py | 265 ++++++ lib/meshrenderer/gl_utils/__init__.py | 23 + lib/meshrenderer/gl_utils/camera.py | 264 ++++++ lib/meshrenderer/gl_utils/ebo.py | 19 + .../gl_utils/egl_offscreen_context.py | 138 +++ lib/meshrenderer/gl_utils/fbo.py | 37 + lib/meshrenderer/gl_utils/geometry.py | 797 ++++++++++++++++++ .../gl_utils/glfw_offscreen_context.py | 49 ++ lib/meshrenderer/gl_utils/ibo.py | 19 + lib/meshrenderer/gl_utils/material.py | 93 ++ lib/meshrenderer/gl_utils/renderbuffer.py | 32 + lib/meshrenderer/gl_utils/shader.py | 118 +++ .../gl_utils/shader_storage_buffer.py | 24 + lib/meshrenderer/gl_utils/texture.py | 178 ++++ lib/meshrenderer/gl_utils/tiles.py | 55 ++ lib/meshrenderer/gl_utils/vao.py | 36 + lib/meshrenderer/gl_utils/vertexbuffer.py | 16 + lib/meshrenderer/gl_utils/window.py | 113 +++ lib/meshrenderer/meshrenderer.py | 241 ++++++ lib/meshrenderer/meshrenderer_phong.py | 278 ++++++ .../meshrenderer_phong_normals.py | 268 ++++++ lib/meshrenderer/scenerenderer.py | 144 ++++ lib/meshrenderer/shader/cad_shader.frag | 46 + lib/meshrenderer/shader/cad_shader.vs | 34 + .../shader/depth_shader_phong.frag | 71 ++ lib/meshrenderer/shader/depth_shader_phong.vs | 32 + lib/meshrenderer/shader/line.frag | 9 + lib/meshrenderer/shader/line.vs | 82 ++ lib/meshrenderer/write_xml.py | 35 + scripts/format_code.sh | 5 +- tools/lm/lm_pbr_1_gen_xyz_crop.py | 232 +++++ 33 files changed, 3773 insertions(+), 2 deletions(-) create mode 100644 lib/meshrenderer/LICENSE create mode 100644 lib/meshrenderer/__init__.py create mode 100644 lib/meshrenderer/box3d_renderer.py create mode 100644 lib/meshrenderer/gl_utils/__init__.py create mode 100644 lib/meshrenderer/gl_utils/camera.py create mode 100644 lib/meshrenderer/gl_utils/ebo.py create mode 100644 lib/meshrenderer/gl_utils/egl_offscreen_context.py create mode 100644 lib/meshrenderer/gl_utils/fbo.py create mode 100644 lib/meshrenderer/gl_utils/geometry.py create mode 100644 lib/meshrenderer/gl_utils/glfw_offscreen_context.py create mode 100644 lib/meshrenderer/gl_utils/ibo.py create mode 100644 lib/meshrenderer/gl_utils/material.py create mode 100644 lib/meshrenderer/gl_utils/renderbuffer.py create mode 100644 lib/meshrenderer/gl_utils/shader.py create mode 100644 lib/meshrenderer/gl_utils/shader_storage_buffer.py create mode 100644 lib/meshrenderer/gl_utils/texture.py create mode 100644 lib/meshrenderer/gl_utils/tiles.py create mode 100644 lib/meshrenderer/gl_utils/vao.py create mode 100644 lib/meshrenderer/gl_utils/vertexbuffer.py create mode 100644 lib/meshrenderer/gl_utils/window.py create mode 100644 lib/meshrenderer/meshrenderer.py create mode 100644 lib/meshrenderer/meshrenderer_phong.py create mode 100644 lib/meshrenderer/meshrenderer_phong_normals.py create mode 100644 lib/meshrenderer/scenerenderer.py create mode 100644 lib/meshrenderer/shader/cad_shader.frag create mode 100644 lib/meshrenderer/shader/cad_shader.vs create mode 100644 lib/meshrenderer/shader/depth_shader_phong.frag create mode 100644 lib/meshrenderer/shader/depth_shader_phong.vs create mode 100644 lib/meshrenderer/shader/line.frag create mode 100644 lib/meshrenderer/shader/line.vs create mode 100644 lib/meshrenderer/write_xml.py create mode 100644 tools/lm/lm_pbr_1_gen_xyz_crop.py diff --git a/lib/meshrenderer/LICENSE b/lib/meshrenderer/LICENSE new file mode 100644 index 00000000..adb95313 --- /dev/null +++ b/lib/meshrenderer/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015 aleju + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/lib/meshrenderer/__init__.py b/lib/meshrenderer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/lib/meshrenderer/box3d_renderer.py b/lib/meshrenderer/box3d_renderer.py new file mode 100644 index 00000000..cf7554de --- /dev/null +++ b/lib/meshrenderer/box3d_renderer.py @@ -0,0 +1,265 @@ +# -*- coding: utf-8 -*- +import os +import numpy as np + +from OpenGL.GL import * + +from . import gl_utils as gu + + +class Renderer(object): + + MAX_FBO_WIDTH = 2000 + MAX_FBO_HEIGHT = 2000 + + def __init__(self, models_cad_files, samples, W, H, vertex_tmp_store_folder=".", debug_mode=False): + self.W, self.H = W, H + self._context = gu.OffscreenContext() + + self._samples = 1 + + self._rgb_tex = gu.Texture(GL_TEXTURE_2D, 1, GL_RGB32F, self.W, self.H) + self._rgb_tex.setFilter(GL_NEAREST, GL_NEAREST) + self._rgb_tex.setWrap(GL_CLAMP_TO_EDGE, GL_CLAMP_TO_EDGE) + + self._edge_tex = gu.Texture(GL_TEXTURE_2D, 1, GL_RGB32F, self.W, self.H) + self._edge_tex.setFilter(GL_NEAREST, GL_NEAREST) + self._edge_tex.setWrap(GL_CLAMP_TO_EDGE, GL_CLAMP_TO_EDGE) + + # TEXTURE BUFFER + handles = [self._rgb_tex.makeResident(), self._edge_tex.makeResident()] + self._material_buffer = gu.ShaderStorage(2, np.array(handles, dtype=np.uint64), False) + self._material_buffer.bind() + + self._fbo = gu.Framebuffer( + { + GL_COLOR_ATTACHMENT0: self._rgb_tex, + GL_DEPTH_ATTACHMENT: gu.Renderbuffer(GL_DEPTH_COMPONENT24, self.W, self.H), + } + ) + + # FOR GRADIENTS + if debug_mode: + self._gradient_fbo = gu.Framebuffer( + { + GL_COLOR_ATTACHMENT0: gu.Texture(GL_TEXTURE_2D, 1, GL_R32F, self.W, self.H), + GL_COLOR_ATTACHMENT1: gu.Texture(GL_TEXTURE_2D, 1, GL_RGB32F, self.W, self.H), + GL_COLOR_ATTACHMENT2: gu.Texture(GL_TEXTURE_2D, 1, GL_RGB32F, self.W, self.H), + } + ) + glNamedFramebufferDrawBuffers( + self._gradient_fbo.id, + 3, + np.array((GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1, GL_COLOR_ATTACHMENT2), dtype=np.uint32), + ) + else: + self._gradient_fbo = gu.Framebuffer( + {GL_COLOR_ATTACHMENT0: gu.Texture(GL_TEXTURE_2D, 1, GL_RGB32F, self.W, self.H)} + ) + # VAO + vert_norms = gu.geo.load_meshes(models_cad_files, vertex_tmp_store_folder, recalculate_normals=True) + + vertices = np.empty(0, dtype=np.float32) + self.min_vert = {} + self.max_vert = {} + for obj_id, vert_norm in enumerate(vert_norms): + vertices = np.hstack((vertices, np.hstack((vert_norm[0], vert_norm[1])).reshape(-1))) + # if obj==obj_id: + self.min_vert[obj_id] = np.min(vert_norm[0], axis=0) + self.max_vert[obj_id] = np.max(vert_norm[0], axis=0) + + print self.min_vert, self.max_vert + + vao = gu.VAO( + { + (gu.Vertexbuffer(vertices), 0, 6 * 4): [ + (0, 3, GL_FLOAT, GL_FALSE, 0 * 4), + (1, 3, GL_FLOAT, GL_FALSE, 3 * 4), + ] + } + ) + vao.bind() + + # IBO + sizes = [vert[0].shape[0] for vert in vert_norms] + offsets = [sum(sizes[:i]) for i in range(len(sizes))] + + ibo = gu.IBO(sizes, np.ones(len(vert_norms)), offsets, np.zeros(len(vert_norms))) + ibo.bind() + + gu.Shader.shader_folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), "shader") + # self.scene_shader = gu.Shader('common_shader.vs', 'common_shader.frag') + # self.scene_shader.compile() + + # self.edge_shader = gu.Shader('screen.vs', 'edge_shader.frag') + # self.edge_shader.compile_and_use() + # glUniform1i(1, 1 if debug_mode else 0) + + # self.outline_shader = gu.Shader('screen.vs', 'edge_shader_lineout.frag') + # self.outline_shader.compile() + + # self.screen_shader = gu.Shader('screen.vs', 'screen.frag') + # self.screen_shader.compile() + + self.line_shader = gu.Shader("line.vs", "line.frag") + self.line_shader.compile() + + self._scene_buffer = gu.ShaderStorage(0, gu.Camera().data, True) + self._scene_buffer.bind() + + glClearColor(0.0, 0.0, 0.0, 1.0) + self.camera = gu.Camera() + self.debug_mode = debug_mode + glLineWidth(3) + + def upload_edges(self, scale, pixels): + H, W = pixels.shape[:2] + pixels = np.ascontiguousarray(np.flipud(pixels)) + self._edge_tex.subImage(0, 0, 0, W, H, GL_RG, GL_FLOAT, pixels) + + def render(self, obj_id, K, R, t, near, far, row=0.0, col=0.0, reconst=False): + W, H = self.W, self.H + + camera = gu.Camera() + camera.realCamera(W, H, K, R, t, near, far) + self._scene_buffer.update(camera.data) + + self._fbo.bind() + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) + glViewport(0, 0, W, H) + + # if reconst: + # self.scene_shader.use() + # glDrawArraysIndirect(GL_TRIANGLES, ctypes.c_void_p(obj_id*16)) + + # self._gradient_fbo.bind() + # self.outline_shader.use() + # glClear(GL_COLOR_BUFFER_BIT) + # glViewport(0, 0, W, H) + # glDrawArrays(GL_TRIANGLE_FAN, 0, 4) + + self.line_shader.use() + # number of lines + glUniform3f(0, self.min_vert[obj_id][0], self.min_vert[obj_id][1], self.min_vert[obj_id][2]) + glUniform3f(1, self.max_vert[obj_id][0], self.max_vert[obj_id][1], self.max_vert[obj_id][2]) + glDrawArraysInstanced(GL_LINES, 0, 2, 15) + + rgb_flipped = np.frombuffer(glReadPixels(0, 0, W, H, GL_RGB, GL_UNSIGNED_BYTE), dtype=np.uint8).reshape(H, W, 3) + return np.flipud(rgb_flipped).copy() + + # def render(self, obj_id, K, R, t, near, far, scale, row, col, outline=False,reconst=False): + + # assert scale <= 1.0 + # W = int(1.*self.W*scale) + # H = int(1.*self.H*scale) + # K[[0,1,2],[0,1,2]] *= scale + + # self._fbo.bind() + # glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) + # glViewport(0, 0, W, H) + + # self.camera.real_camera(W, H, K, R, t, near, far) + # self._scene_buffer.update(self.camera.data) + + # self.scene_shader.use() + + # glEnable(GL_DEPTH_TEST) + # if reconst: + # glDrawArraysIndirect(GL_TRIANGLES, ctypes.c_void_p(obj_id*16)) + # glDisable(GL_DEPTH_TEST) + + # self._gradient_fbo.bind() + # glClear(GL_COLOR_BUFFER_BIT) + # glViewport(0, 0, W, H) + + # if outline: + # self.outline_shader.use() + # else: + # self.edge_shader.use() + # glUniform1f(0, scale) + # glDrawArrays(GL_TRIANGLE_FAN, 0, 4) + + # self.line_shader.use() + # #number of lines + # glUniform3f(0, self.min_vert[0], self.min_vert[1], self.min_vert[2]) + # glUniform3f(1, self.max_vert[0], self.max_vert[1], self.max_vert[2]) + # glDrawArraysInstanced(GL_LINES, 0, 2, 15) + + # #rg_flipped = glReadPixels(x0, self.H-y0, w, h, GL_RGB, GL_FLOAT).reshape(h, w, 3) + # if outline: + # rg_scene_flipped = glReadPixels(0, 0, W, H, GL_RGB, GL_FLOAT).reshape(H, W, 3) + # rg_scene = np.flipud(rg_scene_flipped).copy() + # return (rg_scene,) + # else: + # if not self.debug_mode: + # rg_scene_flipped = glReadPixels(0, 0, W, H, GL_RED, GL_FLOAT).reshape(H, W) + # rg_scene = np.flipud(rg_scene_flipped).copy() + # return (rg_scene,) + # else: + # glNamedFramebufferReadBuffer(self._gradient_fbo.id, GL_COLOR_ATTACHMENT0) + # grad_similarity_flipped = glReadPixels(0, 0, W, H, GL_RED, GL_FLOAT).reshape(H, W, 1) + # grad_similarity = np.flipud(grad_similarity_flipped).copy() + + # glNamedFramebufferReadBuffer(self._gradient_fbo.id, GL_COLOR_ATTACHMENT1) + # canny_flipped = glReadPixels(0, 0, W, H, GL_BGR, GL_FLOAT).reshape(H, W, 3) + # canny = np.flipud(canny_flipped).copy() + + # glNamedFramebufferReadBuffer(self._gradient_fbo.id, GL_COLOR_ATTACHMENT2) + # color_3d_edges_flipped = glReadPixels(0, 0, W, H, GL_BGR, GL_FLOAT).reshape(H, W, 3) + # color_3d_edges = np.flipud(color_3d_edges_flipped).copy() + + # return grad_similarity, canny, color_3d_edges + + def render_color_many(self, obj_ids, W, H, K, Rs, ts, near, far, rows, cols): + assert W <= Renderer.MAX_FBO_WIDTH and H <= Renderer.MAX_FBO_HEIGHT + + if self._samples > 1: + self._render_fbo.bind() + + self._fbo.bind() + self.scene_shader.use() + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) + glViewport(0, 0, W, H) + + for obj_id, R, t, row, col in zip(obj_ids, Rs, ts, rows, cols): + camera = gu.Camera() + camera.realCamera(W, H, K.copy(), R, t, near, far) + # camera.real_camera(W, H, K.copy(), R, t, near, far, r=row, c=col) + self._scene_buffer.update(camera.data) + + # glDrawElementsIndirect(GL_TRIANGLES, GL_UNSIGNED_INT, ctypes.c_void_p(obj_id*4*5)) + glDrawArraysIndirect(GL_TRIANGLES, ctypes.c_void_p(obj_id * 16)) + + self._gradient_fbo.bind() + glClear(GL_COLOR_BUFFER_BIT) + glViewport(0, 0, W, H) + self.outline_shader.use() + + if self._samples > 1: + for i in range(2): + glNamedFramebufferReadBuffer(self._render_fbo.id, GL_COLOR_ATTACHMENT0 + i) + glNamedFramebufferDrawBuffer(self._fbo.id, GL_COLOR_ATTACHMENT0 + i) + glBlitNamedFramebuffer( + self._render_fbo.id, self._fbo.id, 0, 0, W, H, 0, 0, W, H, GL_COLOR_BUFFER_BIT, GL_NEAREST + ) + self._fbo.bind() + + glUniform1f(0, 1.0) + glDrawArrays(GL_TRIANGLE_FAN, 0, 4) + + self.line_shader.use() + for obj_id, R, t, row, col in zip(obj_ids, Rs, ts, rows, cols): + camera = gu.Camera() + camera.realCamera(W, H, K.copy(), R, t, near, far) + # camera.real_camera(W, H, K.copy(), R, t, near, far, r=row, c=col) + self._scene_buffer.update(camera.data) + + glDrawArraysInstanced(GL_LINES, 0, 2, 3) + + rg_scene_flipped = glReadPixels(0, 0, W, H, GL_RGB, GL_FLOAT).reshape(H, W, 3) + rg_scene = np.flipud(rg_scene_flipped).copy() + + return (rg_scene, None) + + def close(self): + self._context.close() diff --git a/lib/meshrenderer/gl_utils/__init__.py b/lib/meshrenderer/gl_utils/__init__.py new file mode 100644 index 00000000..62c19684 --- /dev/null +++ b/lib/meshrenderer/gl_utils/__init__.py @@ -0,0 +1,23 @@ +# from .offscreen_context import OffscreenContext +import os + +if os.environ.get("PYOPENGL_PLATFORM", None) == "egl": + print("using egl") + from .egl_offscreen_context import OffscreenContext +else: + print("using glfw") + from .glfw_offscreen_context import OffscreenContext +from .fbo import Framebuffer +from .renderbuffer import Renderbuffer, RenderbufferMultisample +from .texture import Texture, TextureMultisample, Texture1D, Texture3D +from .shader import Shader +from .shader_storage_buffer import ShaderStorage +from .vertexbuffer import Vertexbuffer +from .vao import VAO +from .ibo import IBO +from .ebo import EBO +from .camera import Camera +from .window import Window +from .material import Material +from . import geometry as geo +from .tiles import tiles, tiles4 diff --git a/lib/meshrenderer/gl_utils/camera.py b/lib/meshrenderer/gl_utils/camera.py new file mode 100644 index 00000000..a1339b68 --- /dev/null +++ b/lib/meshrenderer/gl_utils/camera.py @@ -0,0 +1,264 @@ +# -*- coding: UTF-8 -*- +import logging as log +from OpenGL.GL import * +import numpy as np + + +class Camera(object): + def __init__(self): + self.__T_world_view = np.eye(4, dtype=np.float32) + self.__T_view_world = np.eye(4, dtype=np.float32) + + self.__T_view_proj = np.eye(4, dtype=np.float32) + self.__T_proj_view = np.eye(4, dtype=np.float32) + + self.__T_proj_world = np.eye(4, dtype=np.float32) + + self.__viewport = (0.0, 0.0, 1.0, 1.0) + self.__relative_viewport = True + self._w = 0 + self._h = 0 + + self.dirty = False + + def lookAt(self, pos, target, up): + pos = np.array(pos, dtype=np.float32) + target = np.array(target, dtype=np.float32) + up = np.array(up, dtype=np.float32) + z = pos - target + z *= 1.0 / np.linalg.norm(z) + x = np.cross(up, z) + x *= 1.0 / np.linalg.norm(x) + y = np.cross(z, x) + rot = np.vstack((x, y, z)) + self.__T_view_world[:3, :3] = rot + self.__T_view_world[:3, 3] = -np.dot(rot, pos) + self.__T_world_view[:3, :3] = rot.transpose() + self.__T_world_view[:3, 3] = pos + self.__T_proj_world[:] = np.dot(self.__T_proj_view, self.__T_view_world) + self.dirty = True + + def from_radius_angles(self, radius, theta, phi): + x = radius * np.sin(phi) * np.cos(theta) + y = radius * np.sin(phi) * np.sin(theta) + z = radius * np.cos(phi) + pos = np.array((x, y, z), dtype=np.float32) + target = np.array((0, 0, 0), dtype=np.float32) + _z = pos - target + _z *= 1.0 / np.linalg.norm(_z) + up = (0, 0, 1) + if np.linalg.norm(np.cross(up, _z)) == 0.0: + up = (np.cos(theta), np.sin(theta), 0) + self.lookAt((x, y, z), (0, 0, 0), up) + + def setT_world_view(self, T_world_view): + self.__T_world_view[:] = T_world_view + self.__T_view_world[:] = np.linalg.inv(T_world_view) + self.__T_proj_world[:] = np.dot(self.__T_proj_view, self.__T_view_world) + self.dirty = True + + def setT_view_proj(self, T_view_proj): + self.__T_view_proj[:] = T_view_proj + self.__T_proj_view[:] = np.linalg.inv(T_view_proj) + self.__T_proj_world[:] = np.dot(self.__T_proj_view, self.__T_view_world) + self.dirty = True + + def projection(self, fov, aspect, near, far): + diff = near - far + A = np.tan(fov / 2.0) + self.__T_proj_view[:] = np.array( + [[A / aspect, 0, 0, 0], [0, A, 0, 0], [0, 0, (far + near) / diff, 2 * far * near / diff], [0, 0, -1, 0]], + dtype=np.float32, + ) + self.__T_view_proj[:] = np.linalg.inv(self.__T_proj_view) + self.__T_proj_world[:] = np.dot(self.__T_proj_view, self.__T_view_world) + self.dirty = True + + def ortho(self, left, right, bottom, top, nearVal, farVal): + self.__T_proj_view[:] = Camera.__glOrtho__(left, right, bottom, top, nearVal, farVal) + self.__T_view_proj[:] = np.linalg.inv(self.__T_proj_view) + self.__T_proj_world[:] = np.dot(self.__T_proj_view, self.__T_view_world) + self.dirty = True + + def realCameraIntrinsic(self, fx, fy, x0, y0, W, H, near, far): + I = np.array([[fx, 0.0, x0], [0.0, fy, y0], [0.0, 0, 1.0]]) + self.setIntrinsic(I, W, H, near, far) + + def realCamera(self, W, H, K, R, t, near, far, scale=1.0, originIsInTopLeft=True): + self.setIntrinsic(K, W, H, near, far, scale, originIsInTopLeft) + self.__T_world_view[:3, :3] = R.transpose() + self.__T_world_view[:3, 3] = -np.dot(R.transpose(), t.squeeze()) + z_flip = np.eye(4, dtype=np.float32) + z_flip[2, 2] = -1 + self.__T_world_view[:] = self.__T_world_view.dot(z_flip) + self.__T_view_world[:] = np.linalg.pinv(self.__T_world_view) + # self.__T_view_world[:3,:3] = self.__T_world_view[:3,:3].transpose() + # self.__T_view_world[:3,3] = -np.dot(self.__T_world_view[:3,:3], self.__T_view_world[:3,3].squeeze()) + # self.__T_world_view + + self.__T_proj_world[:] = np.dot(self.__T_proj_view, self.__T_view_world) + + def real_camera(self, W, H, K, R, t, near, far, scale=1.0, originIsInTopLeft=False, r=0.0, c=0.0): + self.setIntrinsic(K, W, H, near, far, scale, originIsInTopLeft) + + Kinv = np.linalg.pinv(K) + + p = np.array([r, c, 1], dtype=np.float64) + d = np.dot(Kinv, p) + + eps = 1e-7 + + alpha = np.arctan(d[1] / (np.sqrt(d[2] ** 2 + d[0] ** 2) + eps)) + beta = np.arctan(d[0] / (d[2] + eps)) + + Rx_alpha = np.array( + [[1, 0, 0, 0], [0, np.cos(alpha), -np.sin(alpha), 0], [0, np.sin(alpha), np.cos(alpha), 0], [0, 0, 0, 1]], + dtype=np.float32, + ) + + Ry_beta = np.array( + [[np.cos(beta), 0, np.sin(beta), 0], [0, 1, 0, 0], [-np.sin(beta), 0, np.cos(beta), 0], [0, 0, 0, 1]], + dtype=np.float32, + ) + + self.__T_view_world[:3, :3] = R + self.__T_view_world[:3, 3] = t.squeeze() + + self.__T_world_view[:3, :3] = R.transpose() + self.__T_world_view[:3, 3] = -np.dot(R.transpose(), t.squeeze()) + + z_flip = np.eye(4, dtype=np.float32) + z_flip[2, 2] = -1 + z_flip[1, 1] = -1 + + self.__T_world_view = z_flip.dot(self.__T_world_view) + self.__T_view_world = z_flip.dot(Ry_beta).dot(Rx_alpha).dot(self.__T_view_world) + + self.__T_proj_world[:] = np.dot(self.__T_proj_view, self.__T_view_world) + + def setIntrinsic(self, I, W, H, near, far, scale=1.0, originIsInTopLeft=True): + """ + Args: + I: 3x3 intrinsic camera matrix from real camera (without any OpenGL stuff) + W: Width of the camera image + H: Height of the camera image + near: Near plane + far: Far plane + originIsInTopLeft: If True then the image origin is in top left + if False the image origin is in image center + + Source: http://ksimek.github.io/2013/06/03/calibrated_cameras_in_opengl/ + """ + Camera.__check_matrix__(I) + + A = near + far + B = near * far + persp = np.array( + [ + [I[0, 0] * scale, I[0, 1] * scale, -I[0, 2] * scale, 0], + [0, I[1, 1] * scale, -I[1, 2] * scale, 0], + [0, 0, A, B], + [0, 0, -1, 0], + ], + dtype=np.float64, + ) + ortho = ( + Camera.__glOrtho__(0, W, H, 0, near, far) + if originIsInTopLeft + else Camera.__glOrtho__(-W / 2.0, W / 2.0, -H / 2.0, H / 2.0, near, far) + ) + + self.__T_proj_view[:] = np.dot(ortho, persp).astype(np.float32) + self.__T_view_proj[:] = np.linalg.inv(self.__T_proj_view) + self.__T_proj_world[:] = np.dot(self.__T_proj_view, self.__T_view_world) + self.dirty = True + + @staticmethod + def __check_matrix__(I): + if len(I.shape) != 2: + log.error("Camera Matrix not 2D but %dD" % len(I.shape)) + exit(-1) + elif I.shape != (3, 3): + log.error("Camera Matrix is not 3x3 but %dx%d" % I.shape) + exit(-1) + elif I[1, 0] != 0.0: + log.error("Camera Matrix Error: Expected Element @ 1,0 to be 0.0 but it's: %.f" % I[1, 0]) + exit(-1) + elif I[2, 0] != 0.0: + log.error("Camera Matrix Error: Expected Element @ 2,0 to be 0.0 but it's: %.f" % I[2, 0]) + exit(-1) + elif I[2, 1] != 0.0: + log.error("Camera Matrix Error: Expected Element @ 2,1 to be 0.0 but it's: %.f" % I[2, 1]) + exit(-1) + else: + pass + # elif I[2,2] != 1.0: + # log.error('Camera Matrix Error: Expected Element @ 2,2 to be 1.0 but it\'s: %.f' % I[2,2]) + # exit(-1) + # log.debug('Camera Matrix valid.') + + @staticmethod + def __glOrtho__(left, right, bottom, top, nearVal, farVal): + """ + Source: https://www.opengl.org/sdk/docs/man2/xhtml/glOrtho.xhtml + """ + tx = -(right + left) / (right - left) + ty = -(top + bottom) / (top - bottom) + tz = -(farVal + nearVal) / (farVal - nearVal) + return np.array( + [ + [2.0 / (right - left), 0.0, 0.0, tx], + [0.0, 2.0 / (top - bottom), 0.0, ty], + [0.0, 0.0, -2.0 / (farVal - nearVal), tz], + [0.0, 0.0, 0.0, 1.0], + ], + dtype=np.float64, + ) + + @property + def data(self): + return np.hstack( + (self.T_view_world.T.reshape(-1), self.T_proj_view.T.reshape(-1), self.T_world_view[:3, 3].reshape(-1)) + ).astype(np.float32) + + def set_window_dimensions(self, w, h): + self._w = w + self._h = h + + def set_viewport(self, x0, y0, w, h): + self.__relative_viewport = all([v >= 0.0 and v <= 1.0 for v in [x0, y0, w, h]]) + self.__viewport = (x0, y0, w, h) + + def split_viewport(self, cols, rows, col, row): + d_r = 1.0 / rows + d_c = 1.0 / cols + viewport = (d_c * col, d_r * row, d_c, d_r) + self.set_viewport(*viewport) + + @property + def T_world_view(self): + return self.__T_world_view + + @property + def T_view_world(self): + return self.__T_view_world + + @property + def T_view_proj(self): + return self.__T_view_proj + + @property + def T_proj_view(self): + return self.__T_proj_view + + @property + def T_proj_world(self): + return self.__T_proj_world + + def get_viewport(self): + v = self.__viewport + if self.__relative_viewport: + W, H = self._w, self._h + return (int(v[0] * W), int(v[1] * H), int(v[2] * W), int(v[3] * H)) + else: + return v diff --git a/lib/meshrenderer/gl_utils/ebo.py b/lib/meshrenderer/gl_utils/ebo.py new file mode 100644 index 00000000..12e22fac --- /dev/null +++ b/lib/meshrenderer/gl_utils/ebo.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +import numpy as np + +from OpenGL.GL import * + + +class EBO(object): + def __init__(self, data, dynamic=False): + self.__id = np.empty(1, dtype=np.uint32) + glCreateBuffers(len(self.__id), self.__id) + code = 0 if not dynamic else GL_DYNAMIC_STORAGE_BIT | GL_MAP_WRITE_BIT | GL_MAP_PERSISTENT_BIT + glNamedBufferStorage(self.__id, data.nbytes, data, code) + + def bind(self): + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, self.__id) + + @property + def id(self): + return self.__id diff --git a/lib/meshrenderer/gl_utils/egl_offscreen_context.py b/lib/meshrenderer/gl_utils/egl_offscreen_context.py new file mode 100644 index 00000000..ccf406a7 --- /dev/null +++ b/lib/meshrenderer/gl_utils/egl_offscreen_context.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +import os +from ctypes import pointer + +if not os.environ.get("PYOPENGL_PLATFORM"): + os.environ["PYOPENGL_PLATFORM"] = "egl" + + +from OpenGL.GL import * +from OpenGL.EGL import ( + EGL_SURFACE_TYPE, + EGL_PBUFFER_BIT, + EGL_BLUE_SIZE, + EGL_RED_SIZE, + EGL_GREEN_SIZE, + EGL_DEPTH_SIZE, + EGL_COLOR_BUFFER_TYPE, + EGL_RGB_BUFFER, + EGL_RENDERABLE_TYPE, + EGL_OPENGL_BIT, + EGL_CONFORMANT, + EGL_NONE, + EGL_DEFAULT_DISPLAY, + EGL_NO_CONTEXT, + EGL_OPENGL_API, + EGL_CONTEXT_MAJOR_VERSION, + EGL_CONTEXT_MINOR_VERSION, + EGL_CONTEXT_OPENGL_PROFILE_MASK, + EGL_CONTEXT_OPENGL_CORE_PROFILE_BIT, + eglGetDisplay, + eglInitialize, + eglChooseConfig, + eglBindAPI, + eglCreateContext, + EGLConfig, +) +from OpenGL import arrays +from OpenGL.GL.NV.bindless_texture import * + + +class OffscreenContext(object): + def __init__(self): + config_attributes = arrays.GLintArray.asArray( + [ + EGL_SURFACE_TYPE, + EGL_PBUFFER_BIT, + EGL_BLUE_SIZE, + 8, + EGL_RED_SIZE, + 8, + EGL_GREEN_SIZE, + 8, + EGL_DEPTH_SIZE, + 24, + EGL_COLOR_BUFFER_TYPE, + EGL_RGB_BUFFER, + EGL_RENDERABLE_TYPE, + EGL_OPENGL_BIT, + EGL_CONFORMANT, + EGL_OPENGL_BIT, + EGL_NONE, + ] + ) + + context_attributes = arrays.GLintArray.asArray( + [ + EGL_CONTEXT_MAJOR_VERSION, + 4, + EGL_CONTEXT_MINOR_VERSION, + 1, + EGL_CONTEXT_OPENGL_PROFILE_MASK, + EGL_CONTEXT_OPENGL_CORE_PROFILE_BIT, + EGL_NONE, + ] + ) + major, minor = ctypes.c_long(), ctypes.c_long() + num_configs = ctypes.c_long() + configs = (EGLConfig * 1)() + + # Cache DISPLAY if necessary and get an off-screen EGL display + orig_dpy = None + if "DISPLAY" in os.environ: + orig_dpy = os.environ["DISPLAY"] + del os.environ["DISPLAY"] + self._egl_display = eglGetDisplay(EGL_DEFAULT_DISPLAY) + if orig_dpy is not None: + os.environ["DISPLAY"] = orig_dpy + + # Initialize EGL + assert eglInitialize(self._egl_display, major, minor) + assert eglChooseConfig(self._egl_display, config_attributes, pointer(configs), 1, pointer(num_configs)) + + # Bind EGL to the OpenGL API + assert eglBindAPI(EGL_OPENGL_API) + + # Create an EGL context + self._egl_context = eglCreateContext(self._egl_display, configs[0], EGL_NO_CONTEXT, context_attributes) + if self._egl_context == EGL_NO_CONTEXT: + raise RuntimeError("Unable to create context") + + # Make it current + self.make_current() + + if not glInitBindlessTextureNV(): + raise RuntimeError("Bindless Textures not supported") + self.__display = self._egl_display + + def make_current(self): + from OpenGL.EGL import eglMakeCurrent, EGL_NO_SURFACE + + assert eglMakeCurrent(self._egl_display, EGL_NO_SURFACE, EGL_NO_SURFACE, self._egl_context) + + def close(self): + self.delete_context() + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() + + def delete_context(self): + from OpenGL.EGL import eglDestroyContext, eglTerminate + + if self._egl_display is not None: + if self._egl_context is not None: + eglDestroyContext(self._egl_display, self._egl_context) + self._egl_context = None + eglTerminate(self._egl_display) + self._egl_display = None + + def supports_framebuffers(self): + return True + + +if __name__ == "__main__": + conntext = OffscreenContext() diff --git a/lib/meshrenderer/gl_utils/fbo.py b/lib/meshrenderer/gl_utils/fbo.py new file mode 100644 index 00000000..accdf7d4 --- /dev/null +++ b/lib/meshrenderer/gl_utils/fbo.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +import numpy as np + +from OpenGL.GL import * + +from .renderbuffer import Renderbuffer, RenderbufferMultisample +from .texture import Texture, TextureMultisample + + +class Framebuffer(object): + def __init__(self, attachements): + self.__id = np.empty(1, dtype=np.uint32) + glCreateFramebuffers(len(self.__id), self.__id) + for k in list(attachements.keys()): + attachement = attachements[k] + if isinstance(attachement, Renderbuffer) or isinstance(attachement, RenderbufferMultisample): + glNamedFramebufferRenderbuffer(self.__id, k, GL_RENDERBUFFER, attachement.id) + elif isinstance(attachement, Texture) or isinstance(attachement, TextureMultisample): + glNamedFramebufferTexture(self.__id, k, attachement.id, 0) + else: + raise ValueError("Unknown frambuffer attachement class: {0}".format(attachement)) + + if glCheckNamedFramebufferStatus(self.__id, GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE: + raise RuntimeError("Framebuffer not complete.") + self.__attachements = attachements + + def bind(self): + glBindFramebuffer(GL_FRAMEBUFFER, self.__id) + + def delete(self): + glDeleteFramebuffers(1, self.__id) + for k in list(self.__attachements.keys()): + self.__attachements[k].delete() + + @property + def id(self): + return self.__id[0] diff --git a/lib/meshrenderer/gl_utils/geometry.py b/lib/meshrenderer/gl_utils/geometry.py new file mode 100644 index 00000000..54ef58a5 --- /dev/null +++ b/lib/meshrenderer/gl_utils/geometry.py @@ -0,0 +1,797 @@ +# -*- coding: utf-8 -*- +import os +import numpy as np +import hashlib + +import pyassimp +import pyassimp.postprocess +from tqdm import tqdm +from lib.pysixd import inout + + +def load(filename): + scene = pyassimp.load( + filename, processing=pyassimp.postprocess.aiProcess_GenUVCoords | pyassimp.postprocess.aiProcess_Triangulate + ) + mesh = scene.meshes[0] + return mesh.vertices, mesh.normals, mesh.texturecoords[0, :, :2] + + +def load_meshes_sixd(obj_files, vertex_tmp_store_folder, recalculate_normals=False): + + hashed_file_name = ( + hashlib.md5(("".join(obj_files) + "load_meshes_sixd" + str(recalculate_normals)).encode("utf-8")).hexdigest() + + ".npy" + ) + + out_file = os.path.join(vertex_tmp_store_folder, hashed_file_name) + if os.path.exists(out_file): + return np.load(out_file, allow_pickle=True) + else: + + attributes = [] + for model_path in tqdm(obj_files): + model = inout.load_ply(model_path) + vertices = np.array(model["pts"]).astype(np.float32) + if recalculate_normals: + normals = calc_normals(vertices) + else: + normals = np.array(model["normals"]).astype(np.float32) + faces = np.array(model["faces"]).astype(np.uint32) + if "colors" in model: + colors = np.array(model["colors"]).astype(np.uint32) + attributes.append((vertices, normals, colors, faces)) + else: + attributes.append((vertices, normals, faces)) + os.makedirs(vertex_tmp_store_folder, exist_ok=True) + np.save(out_file, attributes) + return attributes + + +def load_meshes(obj_files, vertex_tmp_store_folder, recalculate_normals=False): + hashed_file_name = ( + hashlib.md5(("".join(obj_files) + "load_meshes" + str(recalculate_normals)).encode("utf-8")).hexdigest() + + ".npy" + ) + + out_file = os.path.join(vertex_tmp_store_folder, hashed_file_name) + if os.path.exists(out_file): + return np.load(out_file) + else: + attributes = [] + for model_path in tqdm(obj_files): + scene = pyassimp.load(model_path, pyassimp.postprocess.aiProcess_Triangulate) + mesh = scene.meshes[0] + vertices = [] + for face in mesh.faces: + vertices.extend([mesh.vertices[face[0]], mesh.vertices[face[1]], mesh.vertices[face[2]]]) + vertices = np.array(vertices) + # vertices = mesh.vertices + normals = calc_normals(vertices) if recalculate_normals else mesh.normals + attributes.append((vertices, normals)) + pyassimp.release(scene) + np.save(out_file, attributes) + return attributes + + +def calc_normals(vertices): + normals = np.empty_like(vertices) + N = vertices.shape[0] + for i in range(0, N - 1, 3): + v1 = vertices[i] + v2 = vertices[i + 1] + v3 = vertices[i + 2] + normal = np.cross(v2 - v1, v3 - v1) + norm = np.linalg.norm(normal) + normal = np.zeros(3) if norm == 0 else normal / norm + normals[i] = normal + normals[i + 1] = normal + normals[i + 2] = normal + return normals + + +# src: https://github.com/JoeyDeVries/LearnOpenGL/blob/master/src/6.pbr/2.2.2.ibl_specular_textured/ibl_specular_textured.cpp +def sphere(x_segments, y_segments): + + N = (x_segments + 1) * (y_segments + 1) + positions = np.empty((N, 3), dtype=np.float32) + uv = np.empty((N, 2), dtype=np.float32) + normals = np.empty((N, 3), dtype=np.float32) + + i = 0 + for y in range(y_segments + 1): + for x in range(x_segments + 1): + xSegment = float(x) / float(x_segments) + ySegment = float(y) / float(y_segments) + xPos = np.cos(xSegment * 2.0 * np.pi) * np.sin(ySegment * np.pi) + yPos = np.cos(ySegment * np.pi) + zPos = np.sin(xSegment * 2.0 * np.pi) * np.sin(ySegment * np.pi) + + positions[i] = (xPos, yPos, zPos) + uv[i] = (xSegment, ySegment) + normals[i] = (xPos, yPos, zPos) + i += 1 + + indices = [] + oddRow = False + for y in range(y_segments): + if not oddRow: + for x in range(x_segments + 1): + indices.append(y * (x_segments + 1) + x) + indices.append((y + 1) * (x_segments + 1) + x) + else: + for x in reversed(range(x_segments + 1)): + indices.append((y + 1) * (x_segments + 1) + x) + indices.append(y * (x_segments + 1) + x) + oddRow = not oddRow + indices = np.array(indices, dtype=np.uint32) + + return positions, uv, normals, indices + + +def cube(): + positions = np.array( + [ + [-1.0, -1.0, -1.0], + [1.0, 1.0, -1.0], + [1.0, -1.0, -1.0], + [1.0, 1.0, -1.0], + [-1.0, -1.0, -1.0], + [-1.0, 1.0, -1.0], + [-1.0, -1.0, 1.0], + [1.0, -1.0, 1.0], + [1.0, 1.0, 1.0], + [1.0, 1.0, 1.0], + [-1.0, 1.0, 1.0], + [-1.0, -1.0, 1.0], + [-1.0, 1.0, 1.0], + [-1.0, 1.0, -1.0], + [-1.0, -1.0, -1.0], + [-1.0, -1.0, -1.0], + [-1.0, -1.0, 1.0], + [-1.0, 1.0, 1.0], + [1.0, 1.0, 1.0], + [1.0, -1.0, -1.0], + [1.0, 1.0, -1.0], + [1.0, -1.0, -1.0], + [1.0, 1.0, 1.0], + [1.0, -1.0, 1.0], + [-1.0, -1.0, -1.0], + [1.0, -1.0, -1.0], + [1.0, -1.0, 1.0], + [1.0, -1.0, 1.0], + [-1.0, -1.0, 1.0], + [-1.0, -1.0, -1.0], + [-1.0, 1.0, -1.0], + [1.0, 1.0, 1.0], + [1.0, 1.0, -1.0], + [1.0, 1.0, 1.0], + [-1.0, 1.0, -1.0], + [-1.0, 1.0, 1.0], + ], + dtype=np.float32, + ) + + normals = np.array( + [ + [0.0, 0.0, -1.0], + [0.0, 0.0, -1.0], + [0.0, 0.0, -1.0], + [0.0, 0.0, -1.0], + [0.0, 0.0, -1.0], + [0.0, 0.0, -1.0], + [0.0, 0.0, 1.0], + [0.0, 0.0, 1.0], + [0.0, 0.0, 1.0], + [0.0, 0.0, 1.0], + [0.0, 0.0, 1.0], + [0.0, 0.0, 1.0], + [-1.0, 0.0, 0.0], + [-1.0, 0.0, 0.0], + [-1.0, 0.0, 0.0], + [-1.0, 0.0, 0.0], + [-1.0, 0.0, 0.0], + [-1.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + [0.0, -1.0, 0.0], + [0.0, -1.0, 0.0], + [0.0, -1.0, 0.0], + [0.0, -1.0, 0.0], + [0.0, -1.0, 0.0], + [0.0, -1.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 1.0, 0.0], + ], + dtype=np.float32, + ) + + uv = np.array( + [ + [0.0, 0.0], + [1.0, 1.0], + [1.0, 0.0], + [1.0, 1.0], + [0.0, 0.0], + [0.0, 1.0], + [0.0, 0.0], + [1.0, 0.0], + [1.0, 1.0], + [1.0, 1.0], + [0.0, 1.0], + [0.0, 0.0], + [1.0, 0.0], + [1.0, 1.0], + [0.0, 1.0], + [0.0, 1.0], + [0.0, 0.0], + [1.0, 0.0], + [1.0, 0.0], + [0.0, 1.0], + [1.0, 1.0], + [0.0, 1.0], + [1.0, 0.0], + [0.0, 0.0], + [0.0, 1.0], + [1.0, 1.0], + [1.0, 0.0], + [1.0, 0.0], + [0.0, 0.0], + [0.0, 1.0], + [0.0, 1.0], + [1.0, 0.0], + [1.0, 1.0], + [1.0, 0.0], + [0.0, 1.0], + [0.0, 0.0], + ], + dtype=np.float32, + ) + return positions, uv, normals + + +def cube2(min, max): + positions = np.array( + [ + [-1.0, -1.0, -1.0], + [1.0, 1.0, -1.0], + [1.0, -1.0, -1.0], + [1.0, 1.0, -1.0], + [-1.0, -1.0, -1.0], + [-1.0, 1.0, -1.0], + [-1.0, -1.0, 1.0], + [1.0, -1.0, 1.0], + [1.0, 1.0, 1.0], + [1.0, 1.0, 1.0], + [-1.0, 1.0, 1.0], + [-1.0, -1.0, 1.0], + [-1.0, 1.0, 1.0], + [-1.0, 1.0, -1.0], + [-1.0, -1.0, -1.0], + [-1.0, -1.0, -1.0], + [-1.0, -1.0, 1.0], + [-1.0, 1.0, 1.0], + [1.0, 1.0, 1.0], + [1.0, -1.0, -1.0], + [1.0, 1.0, -1.0], + [1.0, -1.0, -1.0], + [1.0, 1.0, 1.0], + [1.0, -1.0, 1.0], + [-1.0, -1.0, -1.0], + [1.0, -1.0, -1.0], + [1.0, -1.0, 1.0], + [1.0, -1.0, 1.0], + [-1.0, -1.0, 1.0], + [-1.0, -1.0, -1.0], + [-1.0, 1.0, -1.0], + [1.0, 1.0, 1.0], + [1.0, 1.0, -1.0], + [1.0, 1.0, 1.0], + [-1.0, 1.0, -1.0], + [-1.0, 1.0, 1.0], + ], + dtype=np.float32, + ) + + normals = np.array( + [ + [0.0, 0.0, -1.0], + [0.0, 0.0, -1.0], + [0.0, 0.0, -1.0], + [0.0, 0.0, -1.0], + [0.0, 0.0, -1.0], + [0.0, 0.0, -1.0], + [0.0, 0.0, 1.0], + [0.0, 0.0, 1.0], + [0.0, 0.0, 1.0], + [0.0, 0.0, 1.0], + [0.0, 0.0, 1.0], + [0.0, 0.0, 1.0], + [-1.0, 0.0, 0.0], + [-1.0, 0.0, 0.0], + [-1.0, 0.0, 0.0], + [-1.0, 0.0, 0.0], + [-1.0, 0.0, 0.0], + [-1.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + [0.0, -1.0, 0.0], + [0.0, -1.0, 0.0], + [0.0, -1.0, 0.0], + [0.0, -1.0, 0.0], + [0.0, -1.0, 0.0], + [0.0, -1.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 1.0, 0.0], + ], + dtype=np.float32, + ) + + uv = np.array( + [ + [min, min], + [max, max], + [max, min], + [max, max], + [min, min], + [min, max], + [min, min], + [max, min], + [max, max], + [max, max], + [min, max], + [min, min], + [max, min], + [max, max], + [min, max], + [min, max], + [min, min], + [max, min], + [max, min], + [min, max], + [max, max], + [min, max], + [max, min], + [min, min], + [min, max], + [max, max], + [max, min], + [max, min], + [min, min], + [min, max], + [min, max], + [max, min], + [max, max], + [max, min], + [min, max], + [min, min], + ], + dtype=np.float32, + ) + return positions, uv, normals + + +def quad(reverse_uv=False): + positions = np.array([[-1.0, 1.0, 0.0], [-1.0, -1.0, 0.0], [1.0, 1.0, 0.0], [1.0, -1.0, 0.0]], dtype=np.float32) + if reverse_uv: + uv = np.array([[0.0, 0.0], [0.0, 1.0], [1.0, 0.0], [1.0, 1.0]], dtype=np.float32) + else: + uv = np.array([[0.0, 1.0], [0.0, 0.0], [1.0, 1.0], [1.0, 0.0]], dtype=np.float32) + return positions, uv + + +cube_vertices_texture = np.array( + [ + -0.5, + -0.5, + -0.5, + 0.0, + 0.0, + 0.5, + -0.5, + -0.5, + 1.0, + 0.0, + 0.5, + 0.5, + -0.5, + 1.0, + 1.0, + 0.5, + 0.5, + -0.5, + 1.0, + 1.0, + -0.5, + 0.5, + -0.5, + 0.0, + 1.0, + -0.5, + -0.5, + -0.5, + 0.0, + 0.0, + -0.5, + -0.5, + 0.5, + 0.0, + 0.0, + 0.5, + -0.5, + 0.5, + 1.0, + 0.0, + 0.5, + 0.5, + 0.5, + 1.0, + 1.0, + 0.5, + 0.5, + 0.5, + 1.0, + 1.0, + -0.5, + 0.5, + 0.5, + 0.0, + 1.0, + -0.5, + -0.5, + 0.5, + 0.0, + 0.0, + -0.5, + 0.5, + 0.5, + 1.0, + 0.0, + -0.5, + 0.5, + -0.5, + 1.0, + 1.0, + -0.5, + -0.5, + -0.5, + 0.0, + 1.0, + -0.5, + -0.5, + -0.5, + 0.0, + 1.0, + -0.5, + -0.5, + 0.5, + 0.0, + 0.0, + -0.5, + 0.5, + 0.5, + 1.0, + 0.0, + 0.5, + 0.5, + 0.5, + 1.0, + 0.0, + 0.5, + 0.5, + -0.5, + 1.0, + 1.0, + 0.5, + -0.5, + -0.5, + 0.0, + 1.0, + 0.5, + -0.5, + -0.5, + 0.0, + 1.0, + 0.5, + -0.5, + 0.5, + 0.0, + 0.0, + 0.5, + 0.5, + 0.5, + 1.0, + 0.0, + -0.5, + -0.5, + -0.5, + 0.0, + 1.0, + 0.5, + -0.5, + -0.5, + 1.0, + 1.0, + 0.5, + -0.5, + 0.5, + 1.0, + 0.0, + 0.5, + -0.5, + 0.5, + 1.0, + 0.0, + -0.5, + -0.5, + 0.5, + 0.0, + 0.0, + -0.5, + -0.5, + -0.5, + 0.0, + 1.0, + -0.5, + 0.5, + -0.5, + 0.0, + 1.0, + 0.5, + 0.5, + -0.5, + 1.0, + 1.0, + 0.5, + 0.5, + 0.5, + 1.0, + 0.0, + 0.5, + 0.5, + 0.5, + 1.0, + 0.0, + -0.5, + 0.5, + 0.5, + 0.0, + 0.0, + -0.5, + 0.5, + -0.5, + 0.0, + 1.0, + ], + dtype=np.float32, +) + + +def quad_bitangent(): + verts = np.array([[-1.0, 1.0, 0.0], [-1.0, -1.0, 0.0], [1.0, -1.0, 0.0], [1.0, 1.0, 0.0]], dtype=np.float32) + uv = np.array([[0.0, 1.0], [0.0, 0.0], [1.0, 0.0], [1.0, 1.0]], dtype=np.float32) + + normal = np.array([0.0, 0.0, 1.0], dtype=np.float32) + + edge1 = verts[1] - verts[0] + edge2 = verts[2] - verts[0] + deltaUV1 = uv[1] - uv[0] + deltaUV2 = uv[2] - uv[0] + + f = 1.0 / (deltaUV1[0] * deltaUV2[1] - deltaUV2[0] * deltaUV1[1]) + + tangent1 = f * np.array( + [ + deltaUV2[1] * edge1[0] - deltaUV1[1] * edge2[0], + deltaUV2[1] * edge1[1] - deltaUV1[1] * edge2[1], + deltaUV2[1] * edge1[2] - deltaUV1[1] * edge2[2], + ], + dtype=np.float32, + ) + tangent1 /= np.linalg.norm(tangent1) + + bitangent1 = f * np.array( + [ + -deltaUV2[0] * edge1[0] + deltaUV1[0] * edge2[0], + -deltaUV2[0] * edge1[1] + deltaUV1[0] * edge2[1], + -deltaUV2[0] * edge1[2] + deltaUV1[0] * edge2[2], + ], + dtype=np.float32, + ) + bitangent1 /= np.linalg.norm(bitangent1) + + edge1 = verts[2] - verts[0] + edge2 = verts[3] - verts[0] + deltaUV1 = uv[2] - uv[0] + deltaUV2 = uv[3] - uv[0] + + f = 1.0 / (deltaUV1[0] * deltaUV2[1] - deltaUV2[0] * deltaUV1[1]) + + tangent2 = f * np.array( + [ + deltaUV2[1] * edge1[0] - deltaUV1[1] * edge2[0], + deltaUV2[1] * edge1[1] - deltaUV1[1] * edge2[1], + deltaUV2[1] * edge1[2] - deltaUV1[1] * edge2[2], + ], + dtype=np.float32, + ) + tangent2 /= np.linalg.norm(tangent2) + + bitangent2 = f * np.array( + [ + -deltaUV2[0] * edge1[0] + deltaUV1[0] * edge2[0], + -deltaUV2[0] * edge1[1] + deltaUV1[0] * edge2[1], + -deltaUV2[0] * edge1[2] + deltaUV1[0] * edge2[2], + ], + dtype=np.float32, + ) + bitangent2 /= np.linalg.norm(bitangent2) + + return np.array( + [ + verts[0][0], + verts[0][1], + verts[0][2], + uv[0][0], + uv[0][1], + normal[0], + normal[1], + normal[2], + tangent1[0], + tangent1[1], + tangent1[2], + bitangent1[0], + bitangent1[1], + bitangent1[2], + verts[1][0], + verts[1][1], + verts[1][2], + uv[1][0], + uv[1][1], + normal[0], + normal[1], + normal[2], + tangent1[0], + tangent1[1], + tangent1[2], + bitangent1[0], + bitangent1[1], + bitangent1[2], + verts[2][0], + verts[2][1], + verts[2][2], + uv[2][0], + uv[2][1], + normal[0], + normal[1], + normal[2], + tangent1[0], + tangent1[1], + tangent1[2], + bitangent1[0], + bitangent1[1], + bitangent1[2], + verts[0][0], + verts[0][1], + verts[0][2], + uv[0][0], + uv[0][1], + normal[0], + normal[1], + normal[2], + tangent2[0], + tangent2[1], + tangent2[2], + bitangent2[0], + bitangent2[1], + bitangent2[2], + verts[2][0], + verts[2][1], + verts[2][2], + uv[2][0], + uv[2][1], + normal[0], + normal[1], + normal[2], + tangent2[0], + tangent2[1], + tangent2[2], + bitangent2[0], + bitangent2[1], + bitangent2[2], + verts[3][0], + verts[3][1], + verts[3][2], + uv[3][0], + uv[3][1], + normal[0], + normal[1], + normal[2], + tangent2[0], + tangent2[1], + tangent2[2], + bitangent2[0], + bitangent2[1], + bitangent2[2], + ], + dtype=np.float32, + ) + + +quad_vert_tex_normal_tangent_bitangent = np.array( + [ + -1, + -1, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 1, + 0, + 1, + -1, + 0, + 1, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 1, + 0, + 1, + 1, + 0, + 1, + 1, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 1, + 0, + -1, + 1, + 0, + 0, + 1, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 1, + 0, + ], + dtype=np.float32, +) diff --git a/lib/meshrenderer/gl_utils/glfw_offscreen_context.py b/lib/meshrenderer/gl_utils/glfw_offscreen_context.py new file mode 100644 index 00000000..573168c6 --- /dev/null +++ b/lib/meshrenderer/gl_utils/glfw_offscreen_context.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +import logging as log +import os + +from OpenGL.GL import * + +import cyglfw3 as glfw + + +class OffscreenContext(object): + def __init__(self): + assert glfw.Init(), "Glfw Init failed!" + glfw.WindowHint(glfw.VISIBLE, False) + self._offscreen_context = glfw.CreateWindow(1, 1, "", None) + assert self._offscreen_context, "Could not create Offscreen Context!" + glfw.MakeContextCurrent(self._offscreen_context) + + self.previous_second = glfw.GetTime() + self.frame_count = 0.0 + self._fps = 0.0 + + def update(self): + self.poll_events() + self.update_fps_counter() + + def poll_events(self): + glfw.PollEvents() + + def update_fps_counter(self): + current_second = glfw.GetTime() + elapsed_seconds = current_second - self.previous_second + if elapsed_seconds > 1.0: + self.previous_second = current_second + self._fps = float(self.frame_count) / float(elapsed_seconds) + self.frame_count = 0.0 + self.frame_count += 1.0 + + @property + def fps(self): + return self._fps + + def close(self): + glfw.Terminate() + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() diff --git a/lib/meshrenderer/gl_utils/ibo.py b/lib/meshrenderer/gl_utils/ibo.py new file mode 100644 index 00000000..c2a734cf --- /dev/null +++ b/lib/meshrenderer/gl_utils/ibo.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +import numpy as np + +from OpenGL.GL import * + + +class IBO(object): + def __init__(self, sizes, instances, offsets, base_instance, first_index=None, dynamic=False): + if not isinstance(first_index, np.ndarray): + indices = np.vstack((sizes, instances, offsets, base_instance)).T.astype(np.uint32).copy() + else: + indices = np.vstack((sizes, instances, offsets, base_instance, first_index)).T.astype(np.uint32).copy() + self.__id = np.empty(1, dtype=np.uint32) + glCreateBuffers(len(self.__id), self.__id) + code = 0 if not dynamic else GL_DYNAMIC_STORAGE_BIT | GL_MAP_WRITE_BIT | GL_MAP_PERSISTENT_BIT + glNamedBufferStorage(self.__id, indices.nbytes, indices, code) + + def bind(self): + glBindBuffer(GL_DRAW_INDIRECT_BUFFER, self.__id) diff --git a/lib/meshrenderer/gl_utils/material.py b/lib/meshrenderer/gl_utils/material.py new file mode 100644 index 00000000..f4513a80 --- /dev/null +++ b/lib/meshrenderer/gl_utils/material.py @@ -0,0 +1,93 @@ +# -*- coding: UTF-8 -*- +from OpenGL.GL import * +import numpy as np + + +class Material(object): + def __init__(self, ambient, diffuse, specular, shininess): + self.__ambient = np.array(ambient, dtype=np.float32) + self.__diffuse = np.array(diffuse, dtype=np.float32) + self.__specular = np.array(specular, dtype=np.float32) + self.__shininess = np.array([shininess], dtype=np.float32) + + def getAmbient(self): + return self.__ambient + + def getDiffuse(self): + return self.__diffuse + + def getSpecular(self): + return self.__specular + + def getShininess(self): + return self.__shininess + + def toOpenGL(self, program): + glUniform3fv(glGetUniformLocation(program, "material.ambient"), 1, self.__ambient) + glUniform3fv(glGetUniformLocation(program, "material.diffuse"), 1, self.__diffuse) + glUniform3fv(glGetUniformLocation(program, "material.specular"), 1, self.__specular) + glUniform1f(glGetUniformLocation(program, "material.shininess"), self.__shininess) + + +Material.Emerald = Material((0.633, 0.727811, 0.633), (0.07568, 0.61424, 0.07568), (0.0215, 0.1745, 0.0215), 0.6) +Material.Jade = Material((0.316228, 0.316228, 0.316228), (0.54, 0.89, 0.63), (0.135, 0.2225, 0.1575), 0.1) +Material.Obsidian = Material((0.332741, 0.328634, 0.346435), (0.18275, 0.17, 0.22525), (0.05375, 0.05, 0.06625), 0.3) +Material.Pearl = Material((0.296648, 0.296648, 0.296648), (1, 0.829, 0.829), (0.25, 0.20725, 0.20725), 0.088) +Material.Ruby = Material((0.727811, 0.626959, 0.626959), (0.61424, 0.04136, 0.04136), (0.1745, 0.01175, 0.01175), 0.6) +Material.Turquoise = Material((0.297254, 0.30829, 0.306678), (0.396, 0.74151, 0.69102), (0.1, 0.18725, 0.1745), 0.1) + +Material.Brass = Material( + (0.992157, 0.941176, 0.807843), (0.780392, 0.568627, 0.113725), (0.329412, 0.223529, 0.027451), 0.21794872 +) +Material.Bonze = Material((0.393548, 0.271906, 0.166721), (0.714, 0.4284, 0.18144), (0.2125, 0.1275, 0.054), 0.2) +Material.Chrome = Material((0.774597, 0.774597, 0.774597), (0.4, 0.4, 0.4), (0.25, 0.25, 0.25), 0.6) +Material.Copper = Material((0.256777, 0.137622, 0.086014), (0.7038, 0.27048, 0.0828), (0.19125, 0.0735, 0.0225), 0.1) +Material.Gold = Material((0.628281, 0.555802, 0.366065), (0.75164, 0.60648, 0.22648), (0.24725, 0.1995, 0.0745), 0.4) +Material.Silver = Material( + (0.508273, 0.508273, 0.508273), (0.50754, 0.50754, 0.50754), (0.19225, 0.19225, 0.19225), 0.4 +) + +Material.BlackPlastic = Material((0.50, 0.50, 0.50), (0.01, 0.01, 0.01), (0.0, 0.0, 0.0), 0.25) +Material.CyanPlastic = Material( + (0.50196078, 0.50196078, 0.50196078), (0.0, 0.50980392, 0.50980392), (0.0, 0.1, 0.06), 0.25 +) +Material.GreenPlastic = Material((0.45, 0.55, 0.45), (0.1, 0.35, 0.1), (0.0, 0.0, 0.0), 0.25) +Material.RedPlastic = Material((0.7, 0.6, 0.6), (0.5, 0.0, 0.0), (0.0, 0.0, 0.0), 0.25) +Material.WhitePlastic = Material((0.70, 0.70, 0.70), (0.55, 0.55, 0.55), (0.0, 0.0, 0.0), 0.25) +Material.YellowPlastic = Material((0.60, 0.60, 0.50), (0.5, 0.5, 0.0), (0.0, 0.0, 0.0), 0.25) + +Material.BlackRubber = Material((0.4, 0.4, 0.4), (0.01, 0.01, 0.01), (0.02, 0.02, 0.02), 0.078125) +Material.CyanRubber = Material((0.04, 0.7, 0.7), (0.4, 0.5, 0.5), (0.0, 0.05, 0.05), 0.078125) +Material.GreenRubber = Material((0.04, 0.7, 0.04), (0.4, 0.5, 0.4), (0.0, 0.05, 0.0), 0.078125) +Material.RedRubber = Material((0.7, 0.04, 0.04), (0.5, 0.4, 0.4), (0.05, 0.0, 0.0), 0.078125) +Material.WhiteRubber = Material((0.7, 0.7, 0.7), (0.5, 0.5, 0.5), (0.05, 0.05, 0.05), 0.078125) +Material.YellowRubber = Material((0.7, 0.7, 0.04), (0.5, 0.5, 0.4), (0.05, 0.05, 0.0), 0.078125) + +Material.ALL = [ + Material.Emerald, + Material.Jade, + Material.Obsidian, + Material.Pearl, + Material.Ruby, + Material.Turquoise, + Material.Brass, + Material.Bonze, + Material.Chrome, + Material.Copper, + Material.Gold, + Material.Silver, + Material.BlackPlastic, + Material.CyanPlastic, + Material.GreenPlastic, + Material.RedPlastic, + Material.WhitePlastic, + Material.YellowPlastic, + Material.BlackRubber, + Material.CyanRubber, + Material.GreenRubber, + Material.RedRubber, + Material.WhiteRubber, + Material.YellowRubber, +] + +# Source: http://devernay.free.fr/cours/opengl/materials.html diff --git a/lib/meshrenderer/gl_utils/renderbuffer.py b/lib/meshrenderer/gl_utils/renderbuffer.py new file mode 100644 index 00000000..391d8812 --- /dev/null +++ b/lib/meshrenderer/gl_utils/renderbuffer.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +import numpy as np + +from OpenGL.GL import * + + +class Renderbuffer(object): + def __init__(self, internalformat, W, H): + self.__id = np.empty(1, dtype=np.uint32) + glCreateRenderbuffers(len(self.__id), self.__id) + glNamedRenderbufferStorage(self.__id[0], internalformat, W, H) + + def delete(self): + glDeleteRenderbuffers(1, self.__id) + + @property + def id(self): + return self.__id[0] + + +class RenderbufferMultisample(object): + def __init__(self, samples, internalformat, W, H): + self.__id = np.empty(1, dtype=np.uint32) + glCreateRenderbuffers(len(self.__id), self.__id) + glNamedRenderbufferStorageMultisample(self.__id[0], samples, internalformat, W, H) + + def delete(self): + glDeleteRenderbuffers(1, self.__id) + + @property + def id(self): + return self.__id[0] diff --git a/lib/meshrenderer/gl_utils/shader.py b/lib/meshrenderer/gl_utils/shader.py new file mode 100644 index 00000000..0a73fafa --- /dev/null +++ b/lib/meshrenderer/gl_utils/shader.py @@ -0,0 +1,118 @@ +# -*- coding: UTF-8 -*- +import logging as log +import os + +from OpenGL.GL import * + + +class Shader(object): + + shader_folder = None + active_shader = None + + def __init__(self, *shaderPaths): + self.__shader = [] + endings = [s[s.rindex(".") + 1 :] for s in shaderPaths] + for end, shType in zip( + ["vs", "tcs", "tes", "gs", "frag", "cs"], + [ + GL_VERTEX_SHADER, + GL_TESS_CONTROL_SHADER, + GL_TESS_EVALUATION_SHADER, + GL_GEOMETRY_SHADER, + GL_FRAGMENT_SHADER, + GL_COMPUTE_SHADER, + ], + ): + try: + shader = shaderPaths[endings.index(end)] + self.__shader.append((shader, shType)) + except ValueError as e: + pass + + def compile(self, varyings=None): + log.debug("Compiling shader.") + shaderIDs = [] + for shader in self.__shader: + path = None + if Shader.shader_folder != None: + path = os.path.join(Shader.shader_folder, shader[0]) + else: + path = shader[0] + code = Shader.__readFile__(path) + shaderID = Shader.__createShader__(path, shader[1], code) + shaderIDs.append(shaderID) + + self.__program = glCreateProgram() + for shaderID in shaderIDs: + glAttachShader(self.__program, shaderID) + + if varyings is not None: + LP_c_char = ctypes.POINTER(ctypes.c_char) + + argv = (LP_c_char * len(varyings))() + for i, arg in enumerate(varyings): + enc_arg = arg.encode("utf-8") + argv[i] = ctypes.create_string_buffer(enc_arg) + + glTransformFeedbackVaryings(self.__program, 2, argv, GL_SEPARATE_ATTRIBS) + + glLinkProgram(self.__program) + if not glGetProgramiv(self.__program, GL_LINK_STATUS): + log.error(glGetProgramInfoLog(self.__program)) + raise RuntimeError("Shader linking failed") + else: + log.debug("Shader linked.") + + for shaderID in shaderIDs: + glDeleteShader(shaderID) + + @staticmethod + def __readFile__(path): + f = None + try: + f = open(path, "r") + data = f.read() + f.close() + return data + except IOError as e: + raise IOError('"{2}": I/O error({0}): {1}'.format(e.errno, e.strerror, path)) + except: + raise RuntimeError("Unexpected error: ", sys.exc_info()[0]) + + @staticmethod + def __createShader__(shaderPath, shaderType, shaderCode): + shader = glCreateShader(shaderType) + glShaderSource(shader, shaderCode) + glCompileShader(shader) + if not glGetShaderiv(shader, GL_COMPILE_STATUS): + log.error(glGetShaderInfoLog(shader)) + raise RuntimeError("[%s]: Shader compilation failed!" % shaderPath) + else: + log.debug("Shader compiled (%s).", shaderPath) + return shader + + def print_info(self): + print(glGetProgramInterfaceiv(self.__program, GL_PROGRAM_OUTPUT, GL_ACTIVE_RESOURCES)) + for i in range(glGetProgramInterfaceiv(self.__program, GL_PROGRAM_OUTPUT, GL_ACTIVE_RESOURCES)): + name = glGetProgramResourceName(self.__program, GL_PROGRAM_OUTPUT, i, 0) + params = glGetProgramResourceiv(self.__program, GL_PROGRAM_OUTPUT, i, 2, [GL_TYPE, GL_LOCATION], 2, 0) + print("Index %d: %s %s @ location %s" % (i, params[0], name, params[1])) + + def delete(self): + glDeleteProgram(self.__program) + + def compile_and_use(self): + self.compile() + self.use() + + def use(self): + glUseProgram(self.__program) + Shader.active_shader = self + + @property + def id(self): + return self.__program + + def get_program(self): + return self.__program diff --git a/lib/meshrenderer/gl_utils/shader_storage_buffer.py b/lib/meshrenderer/gl_utils/shader_storage_buffer.py new file mode 100644 index 00000000..038259d6 --- /dev/null +++ b/lib/meshrenderer/gl_utils/shader_storage_buffer.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +import numpy as np + +from OpenGL.GL import * + + +class ShaderStorage(object): + def __init__(self, biding_point, data, dynamic=True): + self.__dynamic = dynamic + self.__binding_point = biding_point + self.__id = np.empty(1, dtype=np.uint32) + glCreateBuffers(1, self.__id) + code = 0 if not dynamic else GL_DYNAMIC_STORAGE_BIT | GL_MAP_WRITE_BIT | GL_MAP_PERSISTENT_BIT + glNamedBufferStorage(self.__id[0], data.nbytes, data, code) + self.__data_size = data.nbytes + + def bind(self): + glBindBufferRange(GL_SHADER_STORAGE_BUFFER, self.__binding_point, self.__id[0], 0, self.__data_size) + + def update(self, data, offset=0, nbytes=None): + nbytes = data.nbytes if nbytes == None else nbytes + assert self.__dynamic == True, "Updating of a non-updatable buffer." + assert data.nbytes == self.__data_size, "Please put the same amount of data into the buffer as during creation." + glNamedBufferSubData(self.__id[0], offset, nbytes, data) diff --git a/lib/meshrenderer/gl_utils/texture.py b/lib/meshrenderer/gl_utils/texture.py new file mode 100644 index 00000000..65390601 --- /dev/null +++ b/lib/meshrenderer/gl_utils/texture.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- +import numpy as np + +from OpenGL.GL import * +from OpenGL.GL.NV.bindless_texture import * + + +class Texture(object): + def __init__(self, tex_type, levels, internalformat, W, H): + self.__id = np.empty(1, dtype=np.uint32) + glCreateTextures(tex_type, len(self.__id), self.__id) + glTextureStorage2D(self.__id[0], levels, internalformat, W, H) + self.__handle = None + + def setFilter(self, min_filter, max_filter): + glTextureParameteri(self.__id[0], GL_TEXTURE_MIN_FILTER, min_filter) + glTextureParameteri(self.__id[0], GL_TEXTURE_MAG_FILTER, max_filter) + + def setWrap(self, wrap_s, wrap_t, wrap_r=None): + glTextureParameteri(self.__id[0], GL_TEXTURE_WRAP_S, wrap_s) + glTextureParameteri(self.__id[0], GL_TEXTURE_WRAP_T, wrap_t) + if wrap_r != None: + glTextureParameteri(self.__id[0], GL_TEXTURE_WRAP_R, wrap_r) + + def subImage(self, level, xoffset, yoffset, width, height, data_format, data_type, pixels): + glTextureSubImage2D(self.__id[0], level, xoffset, yoffset, width, height, data_format, data_type, pixels) + + def generate_mipmap(self): + glGenerateTextureMipmap(self.__id[0]) + + def makeResident(self): + self.__handle = glGetTextureHandleNV(self.__id[0]) + glMakeTextureHandleResidentNV(self.__handle) + return self.__handle + + def makeNonResident(self): + if self.__handle != None: + glMakeTextureHandleNonResidentNV(self.__handle) + + def delete(self): + glDeleteTextures(1, self.__id) + + @property + def handle(self): + return self.__handle + + @property + def id(self): + return self.__id[0] + + +class Texture1D(object): + def __init__(self, levels, internalformat, W): + self.__id = np.empty(1, dtype=np.uint32) + glCreateTextures(GL_TEXTURE_1D, len(self.__id), self.__id) + glTextureStorage1D(self.__id[0], levels, internalformat, W) + self.__handle = None + + def setFilter(self, min_filter, max_filter): + glTextureParameteri(self.__id[0], GL_TEXTURE_MIN_FILTER, min_filter) + glTextureParameteri(self.__id[0], GL_TEXTURE_MAG_FILTER, max_filter) + + def setWrap(self, wrap_s, wrap_t, wrap_r=None): + glTextureParameteri(self.__id[0], GL_TEXTURE_WRAP_S, wrap_s) + glTextureParameteri(self.__id[0], GL_TEXTURE_WRAP_T, wrap_t) + if wrap_r != None: + glTextureParameteri(self.__id[0], GL_TEXTURE_WRAP_R, wrap_r) + + def subImage(self, level, xoffset, width, data_format, data_type, pixels): + glTextureSubImage1D(self.__id[0], level, xoffset, width, data_format, data_type, pixels) + + def generate_mipmap(self): + glGenerateTextureMipmap(self.__id[0]) + + def makeResident(self): + self.__handle = glGetTextureHandleNV(self.__id[0]) + glMakeTextureHandleResidentNV(self.__handle) + return self.__handle + + def makeNonResident(self): + if self.__handle != None: + glMakeTextureHandleNonResidentNV(self.__handle) + + def delete(self): + glDeleteTextures(1, self.__id) + + @property + def handle(self): + return self.__handle + + @property + def id(self): + return self.__id[0] + + +class Texture3D(object): + def __init__(self, tex_type, levels, internalformat, W, H, C): + self.__id = np.empty(1, dtype=np.uint32) + glCreateTextures(tex_type, len(self.__id), self.__id) + glTextureStorage3D(self.__id[0], levels, internalformat, W, H, C) + self.__handle = None + + def setFilter(self, min_filter, max_filter): + glTextureParameteri(self.__id[0], GL_TEXTURE_MIN_FILTER, min_filter) + glTextureParameteri(self.__id[0], GL_TEXTURE_MAG_FILTER, max_filter) + + def setWrap(self, wrap_s, wrap_t, wrap_r=None): + glTextureParameteri(self.__id[0], GL_TEXTURE_WRAP_S, wrap_s) + glTextureParameteri(self.__id[0], GL_TEXTURE_WRAP_T, wrap_t) + if wrap_r != None: + glTextureParameteri(self.__id[0], GL_TEXTURE_WRAP_R, wrap_r) + + def subImage(self, level, xoffset, yoffset, zoffset, width, height, depth, data_format, data_type, pixels): + glTextureSubImage3D( + self.__id[0], level, xoffset, yoffset, zoffset, width, height, depth, data_format, data_type, pixels + ) + + def generate_mipmap(self): + glGenerateTextureMipmap(self.__id[0]) + + def makeResident(self): + self.__handle = glGetTextureHandleNV(self.__id[0]) + glMakeTextureHandleResidentNV(self.__handle) + return self.__handle + + def makeNonResident(self): + if self.__handle != None: + glMakeTextureHandleNonResidentNV(self.__handle) + + def delete(self): + glDeleteTextures(1, self.__id) + + @property + def handle(self): + return self.__handle + + @property + def id(self): + return self.__id[0] + + +class TextureMultisample(object): + def __init__(self, samples, internalformat, W, H, fixedsamplelocations): + self.__id = np.empty(1, dtype=np.uint32) + glCreateTextures(GL_TEXTURE_2D_MULTISAMPLE, len(self.__id), self.__id) + glTextureStorage2DMultisample(self.__id[0], samples, internalformat, W, H, fixedsamplelocations) + self.__handle = None + + def setFilter(self, min_filter, max_filter): + glTextureParameteri(self.__id[0], GL_TEXTURE_MIN_FILTER, min_filter) + glTextureParameteri(self.__id[0], GL_TEXTURE_MAG_FILTER, max_filter) + + def setWrap(self, wrap_s, wrap_t): + glTextureParameteri(self.__id[0], GL_TEXTURE_WRAP_S, wrap_s) + glTextureParameteri(self.__id[0], GL_TEXTURE_WRAP_T, wrap_t) + + def subImage(self, level, xoffset, yoffset, width, height, data_format, data_type, pixels): + glTextureSubImage2D(self.__id[0], level, xoffset, yoffset, width, height, data_format, data_type, pixels) + + def makeResident(self): + self.__handle = glGetTextureHandleNV(self.__id[0]) + glMakeTextureHandleResidentNV(self.__handle) + return self.__handle + + def makeNonResident(self): + if self.__handle != None: + glMakeTextureHandleNonResidentNV(self.__handle) + + def delete(self): + glDeleteTextures(1, self.__id) + + @property + def handle(self): + return self.__handle + + @property + def id(self): + return self.__id[0] diff --git a/lib/meshrenderer/gl_utils/tiles.py b/lib/meshrenderer/gl_utils/tiles.py new file mode 100644 index 00000000..465ae51a --- /dev/null +++ b/lib/meshrenderer/gl_utils/tiles.py @@ -0,0 +1,55 @@ +import numpy as np +import cv2 + + +def tiles(batch, rows, cols, spacing_x=0, spacing_y=0, scale=1.0): + if batch.ndim == 4: + N, H, W, C = batch.shape + elif batch.ndim == 3: + N, H, W = batch.shape + C = 1 + else: + raise ValueError("Invalid batch shape: {}".format(batch.shape)) + + H = int(H * scale) + W = int(W * scale) + img = np.ones((rows * H + (rows - 1) * spacing_y, cols * W + (cols - 1) * spacing_x, C)) + i = 0 + for row in range(rows): + for col in range(cols): + start_y = row * (H + spacing_y) + end_y = start_y + H + start_x = col * (W + spacing_x) + end_x = start_x + W + if i < N: + if C > 1: + img[start_y:end_y, start_x:end_x, :] = cv2.resize(batch[i], (W, H)) + else: + img[start_y:end_y, start_x:end_x, 0] = cv2.resize(batch[i], (W, H)) + i += 1 + return img + + +def tiles4(batch, rows, cols, spacing_x=0, spacing_y=0, scale=1.0): + if batch.ndim == 4: + N, H, W, C = batch.shape + assert C == 4 + + H = int(H * scale) + W = int(W * scale) + img = np.ones((2 * rows * H + (2 * rows - 1) * spacing_y, cols * W + (cols - 1) * spacing_x, 3)) + i = 0 + for row in range(0, 2 * rows, 2): + for col in range(cols): + start_y = row * (H + spacing_y) + end_y = start_y + H + start_x = col * (W + spacing_x) + end_x = start_x + W + if i < N: + rgb = batch[i, :, :, :3] + depth = batch[i, :, :, 3:4] + depth = np.tile(depth, (1, 1, 3)) + img[start_y:end_y, start_x:end_x, :] = cv2.resize(rgb, (W, H)) + img[end_y : end_y + H, start_x:end_x, :] = cv2.resize(depth, (W, H)) + i += 1 + return img diff --git a/lib/meshrenderer/gl_utils/vao.py b/lib/meshrenderer/gl_utils/vao.py new file mode 100644 index 00000000..e486015e --- /dev/null +++ b/lib/meshrenderer/gl_utils/vao.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +import numpy as np + +from OpenGL.GL import * + +from .ebo import EBO + + +class VAO(object): + def __init__(self, vbo_attrib, ebo=None): + self.__id = np.empty(1, dtype=np.uint32) + glCreateVertexArrays(len(self.__id), self.__id) + i = 0 + for vbo_offset_stride, attribs in vbo_attrib.items(): + vbo = vbo_offset_stride[0] + offset = vbo_offset_stride[1] + stride = vbo_offset_stride[2] + for attrib in attribs: + attribindex = attrib[0] + size = attrib[1] + attribtype = attrib[2] + normalized = attrib[3] + relativeoffset = attrib[4] + glVertexArrayAttribFormat(self.__id, attribindex, size, attribtype, normalized, relativeoffset) + glVertexArrayAttribBinding(self.__id, attribindex, i) + glEnableVertexArrayAttrib(self.__id, attribindex) + glVertexArrayVertexBuffer(self.__id, i, vbo.id, offset, stride) + i += 1 + if ebo != None: + if isinstance(ebo, EBO): + glVertexArrayElementBuffer(self.__id, ebo.id) + else: + ValueError("Invalid EBO type.") + + def bind(self): + glBindVertexArray(self.__id) diff --git a/lib/meshrenderer/gl_utils/vertexbuffer.py b/lib/meshrenderer/gl_utils/vertexbuffer.py new file mode 100644 index 00000000..48739fde --- /dev/null +++ b/lib/meshrenderer/gl_utils/vertexbuffer.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +import numpy as np + +from OpenGL.GL import * + + +class Vertexbuffer(object): + def __init__(self, data, dynamic=False): + self.__id = np.empty(1, dtype=np.uint32) + glCreateBuffers(len(self.__id), self.__id) + code = 0 if not dynamic else GL_DYNAMIC_STORAGE_BIT | GL_MAP_WRITE_BIT | GL_MAP_PERSISTENT_BIT + glNamedBufferStorage(self.__id, data.nbytes, data, code) + + @property + def id(self): + return self.__id diff --git a/lib/meshrenderer/gl_utils/window.py b/lib/meshrenderer/gl_utils/window.py new file mode 100644 index 00000000..7bb1b26b --- /dev/null +++ b/lib/meshrenderer/gl_utils/window.py @@ -0,0 +1,113 @@ +# -*- coding: UTF-8 -*- +import cyglfw3 as glfw +from OpenGL.GL import * +from OpenGL.GL.NV.bindless_texture import * + + +class Window(object): + def __init__( + self, window_width, window_height, samples=1, window_title="", monitor=1, show_at_center=True, offscreen=False + ): + self.window_title = window_title + assert glfw.Init(), "Glfw Init failed!" + glfw.WindowHint(glfw.SAMPLES, samples) + if offscreen: + glfw.WindowHint(glfw.VISIBLE, False) + mon = glfw.GetMonitors()[monitor] if monitor != None else None + self.windowID = glfw.CreateWindow(window_width, window_height, self.window_title, mon) + assert self.windowID, "Could not create Window!" + glfw.MakeContextCurrent(self.windowID) + + if not glInitBindlessTextureNV(): + raise RuntimeError("Bindless Textures not supported") + + self.framebuf_width, self.framebuf_height = glfw.GetFramebufferSize(self.windowID) + self.framebuffer_size_callback = [] + + def framebuffer_size_callback(window, w, h): + self.framebuf_width, self.framebuf_height = w, h + for callback in self.framebuffer_size_callback: + callback(w, h) + + glfw.SetFramebufferSizeCallback(self.windowID, framebuffer_size_callback) + + self.key_callback = [] + + def key_callback(window, key, scancode, action, mode): + if action == glfw.PRESS: + if key == glfw.KEY_ESCAPE: + glfw.SetWindowShouldClose(window, True) + for callback in self.key_callback: + callback(key, scancode, action, mode) + + glfw.SetKeyCallback(self.windowID, key_callback) + + self.mouse_callback = [] + + def mouse_callback(window, xpos, ypos): + for callback in self.mouse_callback: + callback(xpos, ypos) + + glfw.SetCursorPosCallback(self.windowID, mouse_callback) + + self.mouse_button_callback = [] + + def mouse_button_callback(window, button, action, mods): + for callback in self.mouse_button_callback: + callback(button, action, mods) + + glfw.SetMouseButtonCallback(self.windowID, mouse_button_callback) + + self.scroll_callback = [] + + def scroll_callback(window, xoffset, yoffset): + for callback in self.scroll_callback: + callback(xoffset, yoffset) + + glfw.SetScrollCallback(self.windowID, scroll_callback) + + self.previous_second = glfw.GetTime() + self.frame_count = 0.0 + + if show_at_center: + monitors = glfw.GetMonitors() + assert monitor >= 0 and monitor < len(monitors), "Invalid monitor selected." + vidMode = glfw.GetVideoMode(monitors[monitor]) + glfw.SetWindowPos( + self.windowID, + vidMode.width / 2 - self.framebuf_width / 2, + vidMode.height / 2 - self.framebuf_height / 2, + ) + + def update_fps_counter(self): + current_second = glfw.GetTime() + elapsed_seconds = current_second - self.previous_second + if elapsed_seconds > 1.0: + self.previous_second = current_second + fps = float(self.frame_count) / float(elapsed_seconds) + glfw.SetWindowTitle(self.windowID, "%s @ FPS: %.2f" % (self.window_title, fps)) + self.frame_count = 0.0 + self.frame_count += 1.0 + + def is_open(self): + return not glfw.WindowShouldClose(self.windowID) + + def swap_buffers(self): + glfw.SwapBuffers(self.windowID) + + def poll_events(self): + glfw.PollEvents() + + def update(self): + self.swap_buffers() + self.poll_events() + self.update_fps_counter() + + def close(self): + glfw.Terminate() + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() diff --git a/lib/meshrenderer/meshrenderer.py b/lib/meshrenderer/meshrenderer.py new file mode 100644 index 00000000..445c8eec --- /dev/null +++ b/lib/meshrenderer/meshrenderer.py @@ -0,0 +1,241 @@ +# -*- coding: utf-8 -*- +import os +import numpy as np + +from OpenGL.GL import * + +from . import gl_utils as gu + +from lib.pysixd import misc + + +class Renderer(object): + + MAX_FBO_WIDTH = 2000 + MAX_FBO_HEIGHT = 2000 + + def __init__(self, models_cad_files, samples=1, vertex_tmp_store_folder=".", vertex_scale=1.0): + self._samples = samples + self._context = gu.OffscreenContext() + + # FBO + W, H = Renderer.MAX_FBO_WIDTH, Renderer.MAX_FBO_HEIGHT + self._fbo = gu.Framebuffer( + { + GL_COLOR_ATTACHMENT0: gu.Texture(GL_TEXTURE_2D, 1, GL_RGB8, W, H), + GL_COLOR_ATTACHMENT1: gu.Texture(GL_TEXTURE_2D, 1, GL_R32F, W, H), + GL_DEPTH_ATTACHMENT: gu.Renderbuffer(GL_DEPTH_COMPONENT32F, W, H), + } + ) + self._fbo_depth = gu.Framebuffer( + { + GL_COLOR_ATTACHMENT0: gu.Texture(GL_TEXTURE_2D, 1, GL_RGB8, W, H), + GL_COLOR_ATTACHMENT1: gu.Texture(GL_TEXTURE_2D, 1, GL_R32F, W, H), + GL_DEPTH_ATTACHMENT: gu.Renderbuffer(GL_DEPTH_COMPONENT32F, W, H), + } + ) + glNamedFramebufferDrawBuffers( + self._fbo.id, 2, np.array((GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1), dtype=np.uint32) + ) + glNamedFramebufferDrawBuffers( + self._fbo_depth.id, 2, np.array((GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1), dtype=np.uint32) + ) + + if self._samples > 1: + self._render_fbo = gu.Framebuffer( + { + GL_COLOR_ATTACHMENT0: gu.TextureMultisample(self._samples, GL_RGB8, W, H, True), + GL_DEPTH_ATTACHMENT: gu.RenderbufferMultisample(self._samples, GL_DEPTH_COMPONENT32F, W, H), + } + ) + + # VAO + vert_norms = gu.geo.load_meshes(models_cad_files, vertex_tmp_store_folder, recalculate_normals=True) + + vertices = np.empty(0, dtype=np.float32) + for vert_norm in vert_norms: + _verts = vert_norm[0] * vertex_scale + vertices = np.hstack((vertices, np.hstack((_verts, vert_norm[1])).reshape(-1))) + + vao = gu.VAO( + { + (gu.Vertexbuffer(vertices), 0, 6 * 4): [ + (0, 3, GL_FLOAT, GL_FALSE, 0 * 4), + (1, 3, GL_FLOAT, GL_FALSE, 3 * 4), + ] + } + ) + + sizes = [vert[0].shape[0] for vert in vert_norms] + offsets = [sum(sizes[:i]) for i in range(len(sizes))] + + ibo = gu.IBO(sizes, np.ones(len(vert_norms)), offsets, np.zeros(len(vert_norms))) + + gu.Shader.shader_folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), "shader") + shader = gu.Shader("cad_shader.vs", "cad_shader.frag") + shader.compile() + + self._scene_buffer = gu.ShaderStorage(0, gu.Camera().data, True) + + self._fbo.bind() + self._scene_buffer.bind() + vao.bind() + ibo.bind() + shader.use() + + glEnable(GL_DEPTH_TEST) + glClearColor(0.0, 0.0, 0.0, 1.0) + + def set_light_pose(self, direction): + # glUniform3fv(, , ) + glUniform3f(0, direction[0], direction[1], direction[2]) + + def set_ambient_light(self, a): + glUniform1f(2, a) + + def set_diffuse_light(self, a): + glUniform1f(3, a) + + def set_specular_light(self, a): + glUniform1f(4, a) + + def render( + self, + obj_id, + W, + H, + K, + R, + t, + near, + far, + random_light=False, + phong={"ambient": 0.4, "diffuse": 0.8, "specular": 0.3}, + ): + assert W <= Renderer.MAX_FBO_WIDTH and H <= Renderer.MAX_FBO_HEIGHT + W, H = int(W), int(H) + + if self._samples > 1: + self._render_fbo.bind() + + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) + glViewport(0, 0, W, H) + + camera = gu.Camera() + camera.realCamera(W, H, K, R, t, near, far) + + if random_light: + self.set_light_pose(1000.0 * np.random.random(3)) + self.set_ambient_light(phong["ambient"] + 0.1 * (2 * np.random.rand() - 1)) + self.set_diffuse_light(phong["diffuse"] + 0.1 * (2 * np.random.rand() - 1)) + self.set_specular_light(phong["specular"] + 0.1 * (2 * np.random.rand() - 1)) + # self.set_ambient_light(phong['ambient']) + # self.set_diffuse_light(0.7) + # self.set_specular_light(0.3) + else: + self.set_light_pose(np.array([400.0, 400.0, 400])) + self.set_ambient_light(phong["ambient"]) + self.set_diffuse_light(phong["diffuse"]) + self.set_specular_light(phong["specular"]) + + self._scene_buffer.update(camera.data) + + glDrawArraysIndirect(GL_TRIANGLES, ctypes.c_void_p(obj_id * 16)) + + if self._samples > 1: + self._fbo.bind() + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) + + glNamedFramebufferDrawBuffer(self._fbo.id, GL_COLOR_ATTACHMENT1) + glDrawArraysIndirect(GL_TRIANGLES, ctypes.c_void_p(obj_id * 16)) + + glNamedFramebufferReadBuffer(self._render_fbo.id, GL_COLOR_ATTACHMENT0) + glNamedFramebufferDrawBuffer(self._fbo.id, GL_COLOR_ATTACHMENT0) + glBlitNamedFramebuffer( + self._render_fbo.id, self._fbo.id, 0, 0, W, H, 0, 0, W, H, GL_COLOR_BUFFER_BIT, GL_NEAREST + ) + + glNamedFramebufferDrawBuffers( + self._fbo.id, 2, np.array((GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1), dtype=np.uint32) + ) + + glNamedFramebufferReadBuffer(self._fbo.id, GL_COLOR_ATTACHMENT0) + bgr_flipped = np.frombuffer(glReadPixels(0, 0, W, H, GL_BGR, GL_UNSIGNED_BYTE), dtype=np.uint8).reshape(H, W, 3) + bgr = np.flipud(bgr_flipped).copy() + + glNamedFramebufferReadBuffer(self._fbo.id, GL_COLOR_ATTACHMENT1) + depth_flipped = glReadPixels(0, 0, W, H, GL_RED, GL_FLOAT).reshape(H, W) + depth = np.flipud(depth_flipped).copy() + + return bgr, depth + + def render_many( + self, + obj_ids, + W, + H, + K, + Rs, + ts, + near, + far, + random_light=False, + phong={"ambient": 0.4, "diffuse": 0.8, "specular": 0.3}, + ): + assert W <= Renderer.MAX_FBO_WIDTH and H <= Renderer.MAX_FBO_HEIGHT + + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) + glViewport(0, 0, W, H) + + if random_light: + self.set_light_pose(1000.0 * np.random.random(3)) + self.set_ambient_light(phong["ambient"] + 0.1 * (2 * np.random.rand() - 1)) + self.set_diffuse_light(phong["diffuse"] + 0.1 * (2 * np.random.rand() - 1)) + self.set_specular_light(phong["specular"] + 0.1 * (2 * np.random.rand() - 1)) + # self.set_ambient_light(phong['ambient']) + # self.set_diffuse_light(0.7) + # self.set_specular_light(0.3) + else: + self.set_light_pose(np.array([400.0, 400.0, 400])) + self.set_ambient_light(phong["ambient"]) + self.set_diffuse_light(phong["diffuse"]) + self.set_specular_light(phong["specular"]) + + bbs = [] + for i in range(len(obj_ids)): + o = obj_ids[i] + R = Rs[i] + t = ts[i] + camera = gu.Camera() + camera.realCamera(W, H, K, R, t, near, far) + self._scene_buffer.update(camera.data) + + self._fbo.bind() + glDrawArraysIndirect(GL_TRIANGLES, ctypes.c_void_p(o * 16)) + + self._fbo_depth.bind() + glViewport(0, 0, W, H) + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) + glDrawArraysIndirect(GL_TRIANGLES, ctypes.c_void_p(o * 16)) + + glNamedFramebufferReadBuffer(self._fbo_depth.id, GL_COLOR_ATTACHMENT1) + depth_flipped = glReadPixels(0, 0, W, H, GL_RED, GL_FLOAT).reshape(H, W) + depth = np.flipud(depth_flipped).copy() + + ys, xs = np.nonzero(depth > 0) + obj_bb = misc.calc_2d_bbox(xs, ys, (W, H)) + bbs.append(obj_bb) + + glBindFramebuffer(GL_FRAMEBUFFER, self._fbo.id) + glNamedFramebufferReadBuffer(self._fbo.id, GL_COLOR_ATTACHMENT0) + bgr_flipped = np.frombuffer(glReadPixels(0, 0, W, H, GL_BGR, GL_UNSIGNED_BYTE), dtype=np.uint8).reshape(H, W, 3) + bgr = np.flipud(bgr_flipped).copy() + + glNamedFramebufferReadBuffer(self._fbo.id, GL_COLOR_ATTACHMENT1) + depth_flipped = glReadPixels(0, 0, W, H, GL_RED, GL_FLOAT).reshape(H, W) + depth = np.flipud(depth_flipped).copy() + + return bgr, depth, bbs + + def close(self): + self._context.close() diff --git a/lib/meshrenderer/meshrenderer_phong.py b/lib/meshrenderer/meshrenderer_phong.py new file mode 100644 index 00000000..d1e9b355 --- /dev/null +++ b/lib/meshrenderer/meshrenderer_phong.py @@ -0,0 +1,278 @@ +# -*- coding: utf-8 -*- +import os +import numpy as np + +from OpenGL.GL import * + +from . import gl_utils as gu + +from lib.pysixd import misc + + +class Renderer(object): + + MAX_FBO_WIDTH = 2000 + MAX_FBO_HEIGHT = 2000 + + def __init__(self, models_cad_files, samples=1, vertex_tmp_store_folder=".", clamp=False, vertex_scale=1.0): + self._samples = samples + self._context = gu.OffscreenContext() + + # FBO + W, H = Renderer.MAX_FBO_WIDTH, Renderer.MAX_FBO_HEIGHT + self._fbo = gu.Framebuffer( + { + GL_COLOR_ATTACHMENT0: gu.Texture(GL_TEXTURE_2D, 1, GL_RGB8, W, H), + GL_COLOR_ATTACHMENT1: gu.Texture(GL_TEXTURE_2D, 1, GL_R32F, W, H), + GL_DEPTH_ATTACHMENT: gu.Renderbuffer(GL_DEPTH_COMPONENT32F, W, H), + } + ) + + self._fbo_depth = gu.Framebuffer( + { + GL_COLOR_ATTACHMENT0: gu.Texture(GL_TEXTURE_2D, 1, GL_RGB8, W, H), + GL_COLOR_ATTACHMENT1: gu.Texture(GL_TEXTURE_2D, 1, GL_R32F, W, H), + GL_DEPTH_ATTACHMENT: gu.Renderbuffer(GL_DEPTH_COMPONENT32F, W, H), + } + ) + glNamedFramebufferDrawBuffers( + self._fbo.id, 2, np.array((GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1), dtype=np.uint32) + ) + glNamedFramebufferDrawBuffers( + self._fbo_depth.id, 2, np.array((GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1), dtype=np.uint32) + ) + + if self._samples > 1: + self._render_fbo = gu.Framebuffer( + { + GL_COLOR_ATTACHMENT0: gu.TextureMultisample(self._samples, GL_RGB8, W, H, True), + GL_COLOR_ATTACHMENT1: gu.TextureMultisample(self._samples, GL_R32F, W, H, True), + GL_DEPTH_STENCIL_ATTACHMENT: gu.RenderbufferMultisample(self._samples, GL_DEPTH32F_STENCIL8, W, H), + } + ) + glNamedFramebufferDrawBuffers( + self._render_fbo.id, 2, np.array((GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1), dtype=np.uint32) + ) + + self._fbo.bind() + + # VAO + attributes = gu.geo.load_meshes_sixd(models_cad_files, vertex_tmp_store_folder, recalculate_normals=False) + + vertices = [] + indices = [] + for attribute in attributes: + if len(attribute) == 4: + vertex, normal, color, faces = attribute + else: + vertex, normal, faces = attribute + color = np.ones_like(vertex) * 160.0 + indices.append(faces.flatten()) + vertices.append(np.hstack((vertex * vertex_scale, normal, color / 255.0)).flatten()) + + indices = np.hstack(indices).astype(np.uint32) + vertices = np.hstack(vertices).astype(np.float32) + + vao = gu.VAO( + { + (gu.Vertexbuffer(vertices), 0, 9 * 4): [ + (0, 3, GL_FLOAT, GL_FALSE, 0 * 4), + (1, 3, GL_FLOAT, GL_FALSE, 3 * 4), + (2, 3, GL_FLOAT, GL_FALSE, 6 * 4), + ] + }, + gu.EBO(indices), + ) + vao.bind() + + # IBO + vertex_count = [np.prod(vert[-1].shape) for vert in attributes] + instance_count = np.ones(len(attributes)) + first_index = [sum(vertex_count[:i]) for i in range(len(vertex_count))] + + vertex_sizes = [vert[0].shape[0] for vert in attributes] + base_vertex = [sum(vertex_sizes[:i]) for i in range(len(vertex_sizes))] + base_instance = np.zeros(len(attributes)) + + ibo = gu.IBO(vertex_count, instance_count, first_index, base_vertex, base_instance) + ibo.bind() + + gu.Shader.shader_folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), "shader") + # if clamp: + # shader = gu.Shader('depth_shader_phong.vs', 'depth_shader_phong_clamped.frag') + # else: + shader = gu.Shader("depth_shader_phong.vs", "depth_shader_phong.frag") + shader.compile_and_use() + + self._scene_buffer = gu.ShaderStorage(0, gu.Camera().data, True) + self._scene_buffer.bind() + + glEnable(GL_DEPTH_TEST) + glClearColor(0.0, 0.0, 0.0, 1.0) + + def set_light_pose(self, direction): + glUniform3f(1, direction[0], direction[1], direction[2]) + + def set_ambient_light(self, a): + glUniform1f(0, a) + + def set_diffuse_light(self, a): + glUniform1f(2, a) + + def set_specular_light(self, a): + glUniform1f(3, a) + + def render( + self, + obj_id, + W, + H, + K, + R, + t, + near, + far, + random_light=False, + phong={"ambient": 0.4, "diffuse": 0.8, "specular": 0.3}, + ): + assert W <= Renderer.MAX_FBO_WIDTH and H <= Renderer.MAX_FBO_HEIGHT + W = int(W) + H = int(H) + + if self._samples > 1: + self._render_fbo.bind() + + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT) + glViewport(0, 0, int(W), int(H)) + + camera = gu.Camera() + camera.realCamera(W, H, K, R, t, near, far) + + self._scene_buffer.update(camera.data) + # print phong + if random_light: + self.set_light_pose(1000.0 * np.random.random(3)) + self.set_ambient_light(phong["ambient"]) + self.set_diffuse_light(phong["diffuse"] + 0.1 * (2 * np.random.rand() - 1)) + self.set_specular_light(phong["specular"] + 0.1 * (2 * np.random.rand() - 1)) + # self.set_ambient_light(phong['ambient']) + # self.set_diffuse_light(0.7) + # self.set_specular_light(0.3) + else: + self.set_light_pose(np.array([400.0, 400.0, 400])) + self.set_ambient_light(phong["ambient"]) + self.set_diffuse_light(phong["diffuse"]) + self.set_specular_light(phong["specular"]) + + # self.set_ambient_light(0.4) + # self.set_diffuse_light(0.7) + + # self.set_ambient_light(0.5) + # self.set_diffuse_light(0.4) + # self.set_specular_light(0.1) + + glDrawElementsIndirect(GL_TRIANGLES, GL_UNSIGNED_INT, ctypes.c_void_p(obj_id * 4 * 5)) + + # if self._samples > 1: + # for i in range(2): + # glNamedFramebufferReadBuffer(self._render_fbo.id, GL_COLOR_ATTACHMENT0 + i) + # glNamedFramebufferDrawBuffer(self._fbo.id, GL_COLOR_ATTACHMENT0 + i) + # glBlitNamedFramebuffer(self._render_fbo.id, self._fbo.id, 0, 0, W, H, 0, 0, W, H, GL_COLOR_BUFFER_BIT, GL_NEAREST) + # self._fbo.bind() + + if self._samples > 1: + self._fbo.bind() + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) + glNamedFramebufferDrawBuffer(self._fbo.id, GL_COLOR_ATTACHMENT1) + glDrawElementsIndirect(GL_TRIANGLES, GL_UNSIGNED_INT, ctypes.c_void_p(obj_id * 4 * 5)) + + glNamedFramebufferReadBuffer(self._render_fbo.id, GL_COLOR_ATTACHMENT0) + glNamedFramebufferDrawBuffer(self._fbo.id, GL_COLOR_ATTACHMENT0) + glBlitNamedFramebuffer( + self._render_fbo.id, self._fbo.id, 0, 0, W, H, 0, 0, W, H, GL_COLOR_BUFFER_BIT, GL_NEAREST + ) + + glNamedFramebufferDrawBuffers( + self._fbo.id, 2, np.array((GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1), dtype=np.uint32) + ) + + glNamedFramebufferReadBuffer(self._fbo.id, GL_COLOR_ATTACHMENT0) + bgr_flipped = np.frombuffer(glReadPixels(0, 0, W, H, GL_BGR, GL_UNSIGNED_BYTE), dtype=np.uint8).reshape(H, W, 3) + bgr = np.flipud(bgr_flipped).copy() + + glNamedFramebufferReadBuffer(self._fbo.id, GL_COLOR_ATTACHMENT1) + depth_flipped = glReadPixels(0, 0, W, H, GL_RED, GL_FLOAT).reshape(H, W) + depth = np.flipud(depth_flipped).copy() + + return bgr, depth + + def render_many( + self, + obj_ids, + W, + H, + K, + Rs, + ts, + near, + far, + random_light=True, + phong={"ambient": 0.4, "diffuse": 0.8, "specular": 0.3}, + ): + assert W <= Renderer.MAX_FBO_WIDTH and H <= Renderer.MAX_FBO_HEIGHT + + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) + glViewport(0, 0, W, H) + + if random_light: + self.set_light_pose(1000.0 * np.random.random(3)) + self.set_ambient_light(phong["ambient"] + 0.1 * (2 * np.random.rand() - 1)) + self.set_diffuse_light(phong["diffuse"] + 0.1 * (2 * np.random.rand() - 1)) + self.set_specular_light(phong["specular"] + 0.1 * (2 * np.random.rand() - 1)) + # self.set_ambient_light(phong['ambient']) + # self.set_diffuse_light(0.7) + # self.set_specular_light(0.3) + else: + self.set_light_pose(np.array([400.0, 400.0, 400])) + self.set_ambient_light(phong["ambient"]) + self.set_diffuse_light(phong["diffuse"]) + self.set_specular_light(phong["specular"]) + + bbs = [] + for i in range(len(obj_ids)): + o = obj_ids[i] + R = Rs[i] + t = ts[i] + camera = gu.Camera() + camera.realCamera(W, H, K, R, t, near, far) + self._scene_buffer.update(camera.data) + + self._fbo.bind() + glDrawElementsIndirect(GL_TRIANGLES, GL_UNSIGNED_INT, ctypes.c_void_p(o * 4 * 5)) + + self._fbo_depth.bind() + glViewport(0, 0, W, H) + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) + glDrawElementsIndirect(GL_TRIANGLES, GL_UNSIGNED_INT, ctypes.c_void_p(o * 4 * 5)) + + glNamedFramebufferReadBuffer(self._fbo_depth.id, GL_COLOR_ATTACHMENT1) + depth_flipped = glReadPixels(0, 0, W, H, GL_RED, GL_FLOAT).reshape(H, W) + depth = np.flipud(depth_flipped).copy() + + ys, xs = np.nonzero(depth > 0) + obj_bb = misc.calc_2d_bbox(xs, ys, (W, H)) + bbs.append(obj_bb) + + glBindFramebuffer(GL_FRAMEBUFFER, self._fbo.id) + glNamedFramebufferReadBuffer(self._fbo.id, GL_COLOR_ATTACHMENT0) + bgr_flipped = np.frombuffer(glReadPixels(0, 0, W, H, GL_BGR, GL_UNSIGNED_BYTE), dtype=np.uint8).reshape(H, W, 3) + bgr = np.flipud(bgr_flipped).copy() + + glNamedFramebufferReadBuffer(self._fbo.id, GL_COLOR_ATTACHMENT1) + depth_flipped = glReadPixels(0, 0, W, H, GL_RED, GL_FLOAT).reshape(H, W) + depth = np.flipud(depth_flipped).copy() + + return bgr, depth, bbs + + def close(self): + self._context.close() diff --git a/lib/meshrenderer/meshrenderer_phong_normals.py b/lib/meshrenderer/meshrenderer_phong_normals.py new file mode 100644 index 00000000..ba3d57fa --- /dev/null +++ b/lib/meshrenderer/meshrenderer_phong_normals.py @@ -0,0 +1,268 @@ +# -*- coding: utf-8 -*- +import os +import numpy as np + +from OpenGL.GL import * + +from . import gl_utils as gu + +from lib.pysixd import misc + + +class Renderer(object): + + MAX_FBO_WIDTH = 2000 + MAX_FBO_HEIGHT = 2000 + + def __init__(self, models_cad_files, samples=1, vertex_tmp_store_folder=".", clamp=False): + self._samples = samples + self._context = gu.OffscreenContext() + + # FBO + W, H = Renderer.MAX_FBO_WIDTH, Renderer.MAX_FBO_HEIGHT + self._fbo = gu.Framebuffer( + { + GL_COLOR_ATTACHMENT0: gu.Texture(GL_TEXTURE_2D, 1, GL_RGB8, W, H), + GL_COLOR_ATTACHMENT1: gu.Texture(GL_TEXTURE_2D, 1, GL_R32F, W, H), + GL_COLOR_ATTACHMENT2: gu.Texture(GL_TEXTURE_2D, 1, GL_RGB8, W, H), + GL_DEPTH_ATTACHMENT: gu.Renderbuffer(GL_DEPTH_COMPONENT32F, W, H), + } + ) + + self._fbo_depth = gu.Framebuffer( + { + GL_COLOR_ATTACHMENT0: gu.Texture(GL_TEXTURE_2D, 1, GL_RGB8, W, H), + GL_COLOR_ATTACHMENT1: gu.Texture(GL_TEXTURE_2D, 1, GL_R32F, W, H), + GL_DEPTH_ATTACHMENT: gu.Renderbuffer(GL_DEPTH_COMPONENT32F, W, H), + } + ) + glNamedFramebufferDrawBuffers( + self._fbo.id, + 3, + np.array((GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1, GL_COLOR_ATTACHMENT2), dtype=np.uint32), + ) + glNamedFramebufferDrawBuffers( + self._fbo_depth.id, 2, np.array((GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1), dtype=np.uint32) + ) + + # if self._samples > 1: + # self._render_fbo = gu.Framebuffer( { GL_COLOR_ATTACHMENT0: gu.TextureMultisample(self._samples, GL_RGB8, W, H, True), + # GL_COLOR_ATTACHMENT1: gu.TextureMultisample(self._samples, GL_R32F, W, H, True), + # GL_DEPTH_STENCIL_ATTACHMENT: gu.RenderbufferMultisample(self._samples, GL_DEPTH32F_STENCIL8, W, H) } ) + # glNamedFramebufferDrawBuffers(self._render_fbo.id, 2, np.array( (GL_COLOR_ATTACHMENT0,GL_COLOR_ATTACHMENT1),dtype=np.uint32 ) ) + + self._fbo.bind() + + # VAO + attributes = gu.geo.load_meshes_sixd(models_cad_files, vertex_tmp_store_folder, recalculate_normals=False) + + vertices = [] + indices = [] + for vertex, normal, color, faces in attributes: + indices.append(faces.flatten()) + vertices.append(np.hstack((vertex, normal, color / 255.0)).flatten()) + + indices = np.hstack(indices).astype(np.uint32) + vertices = np.hstack(vertices).astype(np.float32) + + vao = gu.VAO( + { + (gu.Vertexbuffer(vertices), 0, 9 * 4): [ + (0, 3, GL_FLOAT, GL_FALSE, 0 * 4), + (1, 3, GL_FLOAT, GL_FALSE, 3 * 4), + (2, 3, GL_FLOAT, GL_FALSE, 6 * 4), + ] + }, + gu.EBO(indices), + ) + vao.bind() + + # IBO + vertex_count = [np.prod(vert[3].shape) for vert in attributes] + instance_count = np.ones(len(attributes)) + first_index = [sum(vertex_count[:i]) for i in range(len(vertex_count))] + + vertex_sizes = [vert[0].shape[0] for vert in attributes] + base_vertex = [sum(vertex_sizes[:i]) for i in range(len(vertex_sizes))] + base_instance = np.zeros(len(attributes)) + + ibo = gu.IBO(vertex_count, instance_count, first_index, base_vertex, base_instance) + ibo.bind() + + gu.Shader.shader_folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), "shader") + # if clamp: + # shader = gu.Shader('depth_shader_phong.vs', 'depth_shader_phong_clamped.frag') + # else: + shader = gu.Shader("depth_shader_phong.vs", "depth_shader_phong.frag") + shader.compile_and_use() + + self._scene_buffer = gu.ShaderStorage(0, gu.Camera().data, True) + self._scene_buffer.bind() + + glEnable(GL_DEPTH_TEST) + glClearColor(0.0, 0.0, 0.0, 1.0) + + def set_light_pose(self, direction): + glUniform3f(1, direction[0], direction[1], direction[2]) + + def set_ambient_light(self, a): + glUniform1f(0, a) + + def set_diffuse_light(self, a): + glUniform1f(2, a) + + def set_specular_light(self, a): + glUniform1f(3, a) + + def render( + self, + obj_id, + W, + H, + K, + R, + t, + near, + far, + random_light=False, + phong={"ambient": 0.4, "diffuse": 0.8, "specular": 0.3}, + ): + assert W <= Renderer.MAX_FBO_WIDTH and H <= Renderer.MAX_FBO_HEIGHT + + # if self._samples > 1: + # self._render_fbo.bind() + + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT) + glViewport(0, 0, W, H) + + camera = gu.Camera() + camera.realCamera(W, H, K, R, t, near, far) + + self._scene_buffer.update(camera.data) + # print phong + if random_light: + self.set_light_pose(1000.0 * np.random.random(3)) + self.set_ambient_light(phong["ambient"]) + self.set_diffuse_light(phong["diffuse"] + 0.1 * (2 * np.random.rand() - 1)) + self.set_specular_light(phong["specular"] + 0.1 * (2 * np.random.rand() - 1)) + # self.set_ambient_light(phong['ambient']) + # self.set_diffuse_light(0.7) + # self.set_specular_light(0.3) + else: + self.set_light_pose(np.array([400.0, 400.0, 400])) + self.set_ambient_light(phong["ambient"]) + self.set_diffuse_light(phong["diffuse"]) + self.set_specular_light(phong["specular"]) + + # self.set_ambient_light(0.4) + # self.set_diffuse_light(0.7) + + # self.set_ambient_light(0.5) + # self.set_diffuse_light(0.4) + # self.set_specular_light(0.1) + + glDrawElementsIndirect(GL_TRIANGLES, GL_UNSIGNED_INT, ctypes.c_void_p(obj_id * 4 * 5)) + + # if self._samples > 1: + # for i in range(2): + # glNamedFramebufferReadBuffer(self._render_fbo.id, GL_COLOR_ATTACHMENT0 + i) + # glNamedFramebufferDrawBuffer(self._fbo.id, GL_COLOR_ATTACHMENT0 + i) + # glBlitNamedFramebuffer(self._render_fbo.id, self._fbo.id, 0, 0, W, H, 0, 0, W, H, GL_COLOR_BUFFER_BIT, GL_NEAREST) + # self._fbo.bind() + + # if self._samples > 1: + # self._fbo.bind() + # glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) + # glNamedFramebufferDrawBuffer(self._fbo.id, GL_COLOR_ATTACHMENT1) + # glDrawElementsIndirect(GL_TRIANGLES, GL_UNSIGNED_INT, ctypes.c_void_p(obj_id*4*5)) + + # glNamedFramebufferReadBuffer(self._render_fbo.id, GL_COLOR_ATTACHMENT0) + # glNamedFramebufferDrawBuffer(self._fbo.id, GL_COLOR_ATTACHMENT0) + # glBlitNamedFramebuffer(self._render_fbo.id, self._fbo.id, 0, 0, W, H, 0, 0, W, H, GL_COLOR_BUFFER_BIT, GL_NEAREST) + + # glNamedFramebufferDrawBuffers(self._fbo.id, 2, np.array( (GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1),dtype=np.uint32 ) ) + + glNamedFramebufferReadBuffer(self._fbo.id, GL_COLOR_ATTACHMENT0) + bgr_flipped = np.frombuffer(glReadPixels(0, 0, W, H, GL_BGR, GL_UNSIGNED_BYTE), dtype=np.uint8).reshape(H, W, 3) + bgr = np.flipud(bgr_flipped).copy() + + glNamedFramebufferReadBuffer(self._fbo.id, GL_COLOR_ATTACHMENT1) + depth_flipped = glReadPixels(0, 0, W, H, GL_RED, GL_FLOAT).reshape(H, W) + depth = np.flipud(depth_flipped).copy() + + glNamedFramebufferReadBuffer(self._fbo.id, GL_COLOR_ATTACHMENT2) + bgr_flipped = np.frombuffer(glReadPixels(0, 0, W, H, GL_BGR, GL_UNSIGNED_BYTE), dtype=np.uint8).reshape(H, W, 3) + bgr_normal = np.flipud(bgr_flipped).copy() + + return bgr, depth, bgr_normal + + def render_many( + self, + obj_ids, + W, + H, + K, + Rs, + ts, + near, + far, + random_light=True, + phong={"ambient": 0.4, "diffuse": 0.8, "specular": 0.3}, + ): + assert W <= Renderer.MAX_FBO_WIDTH and H <= Renderer.MAX_FBO_HEIGHT + + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) + glViewport(0, 0, W, H) + + if random_light: + self.set_light_pose(1000.0 * np.random.random(3)) + self.set_ambient_light(phong["ambient"] + 0.1 * (2 * np.random.rand() - 1)) + self.set_diffuse_light(phong["diffuse"] + 0.1 * (2 * np.random.rand() - 1)) + self.set_specular_light(phong["specular"] + 0.1 * (2 * np.random.rand() - 1)) + # self.set_ambient_light(phong['ambient']) + # self.set_diffuse_light(0.7) + # self.set_specular_light(0.3) + else: + self.set_light_pose(np.array([400.0, 400.0, 400])) + self.set_ambient_light(phong["ambient"]) + self.set_diffuse_light(phong["diffuse"]) + self.set_specular_light(phong["specular"]) + + bbs = [] + for i in range(len(obj_ids)): + o = obj_ids[i] + R = Rs[i] + t = ts[i] + camera = gu.Camera() + camera.realCamera(W, H, K, R, t, near, far) + self._scene_buffer.update(camera.data) + + self._fbo.bind() + glDrawElementsIndirect(GL_TRIANGLES, GL_UNSIGNED_INT, ctypes.c_void_p(o * 4 * 5)) + + self._fbo_depth.bind() + glViewport(0, 0, W, H) + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) + glDrawElementsIndirect(GL_TRIANGLES, GL_UNSIGNED_INT, ctypes.c_void_p(o * 4 * 5)) + + glNamedFramebufferReadBuffer(self._fbo_depth.id, GL_COLOR_ATTACHMENT1) + depth_flipped = glReadPixels(0, 0, W, H, GL_RED, GL_FLOAT).reshape(H, W) + depth = np.flipud(depth_flipped).copy() + + ys, xs = np.nonzero(depth > 0) + obj_bb = misc.calc_2d_bbox(xs, ys, (W, H)) + bbs.append(obj_bb) + + glBindFramebuffer(GL_FRAMEBUFFER, self._fbo.id) + glNamedFramebufferReadBuffer(self._fbo.id, GL_COLOR_ATTACHMENT0) + bgr_flipped = np.frombuffer(glReadPixels(0, 0, W, H, GL_BGR, GL_UNSIGNED_BYTE), dtype=np.uint8).reshape(H, W, 3) + bgr = np.flipud(bgr_flipped).copy() + + glNamedFramebufferReadBuffer(self._fbo.id, GL_COLOR_ATTACHMENT1) + depth_flipped = glReadPixels(0, 0, W, H, GL_RED, GL_FLOAT).reshape(H, W) + depth = np.flipud(depth_flipped).copy() + + return bgr, depth, bbs + + def close(self): + self._context.close() diff --git a/lib/meshrenderer/scenerenderer.py b/lib/meshrenderer/scenerenderer.py new file mode 100644 index 00000000..79932877 --- /dev/null +++ b/lib/meshrenderer/scenerenderer.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- +import os +import glob + +import math +import numpy as np +from .write_xml import * +import lib.meshrenderer.meshrenderer as mr +import lib.meshrenderer.meshrenderer_phong as mr_phong +import cv2 + +from lib.pysixd import view_sampler +from lib.pysixd import transform + + +class SceneRenderer(object): + def __init__( + self, + models_cad_files, + vertex_tmp_store_folder, + vertex_scale, + width, + height, + K, + augmenters, + vocdevkit_path, + min_num_objects_per_scene, + max_num_objects_per_scene, + near_plane=10, + far_plane=2000, + min_n_views=1000, + radius=650, + obj_ids=None, + model_type="reconst", + ): + + self._models_cad_files = models_cad_files + self._width = width + self._height = height + self._radius = radius + self._K = K + self._augmenters = augmenters + self._min_num_objects_per_scene = min_num_objects_per_scene + self._max_num_objects_per_scene = max_num_objects_per_scene + self._near_plane = near_plane + self._far_plane = far_plane + self.obj_ids = np.array(obj_ids) + + # pascal_imgs_path = os.path.join(vocdevkit_path, 'VOC2012/JPEGImages') + self._voc_imgs = glob.glob(os.path.join(vocdevkit_path, "*.jpg")) + glob.glob( + os.path.join(vocdevkit_path, "*.png") + ) + print(len(self._voc_imgs)) + if model_type == "reconst": + self._renderer = mr_phong.Renderer( + self._models_cad_files, 1, vertex_tmp_store_folder=vertex_tmp_store_folder, vertex_scale=vertex_scale + ) + elif model_type == "cad": + self._renderer = mr.Renderer( + self._models_cad_files, 1, vertex_tmp_store_folder=vertex_tmp_store_folder, vertex_scale=vertex_scale + ) + else: + print("unknown model_type, ", model_type) + exit() + + azimuth_range = (0, 2 * math.pi) + elev_range = (-0.5 * math.pi, 0.5 * math.pi) + self.all_views, _ = view_sampler.sample_views(min_n_views, radius, azimuth_range, elev_range) + + def render(self): + if self._min_num_objects_per_scene == self._max_num_objects_per_scene: + N = self._min_num_objects_per_scene + else: + N = np.random.randint(self._min_num_objects_per_scene, self._max_num_objects_per_scene) + views = np.random.choice(self.all_views, N) + obj_is = np.random.choice(len(self._models_cad_files), N) + + ts = [] + ts_norm = [] + Rs = [] + + for v in views: + success = False + while not success: + + tz = np.random.triangular( + self._radius - self._radius / 3, self._radius, self._radius + self._radius / 3 + ) + + tx = np.random.uniform( + -0.35 * tz * self._width / self._K[0, 0], 0.35 * tz * self._width / self._K[0, 0] + ) + ty = np.random.uniform( + -0.35 * tz * self._height / self._K[1, 1], 0.35 * tz * self._height / self._K[1, 1] + ) + + t = np.array([tx, ty, tz]) + R = transform.random_rotation_matrix()[:3, :3] + t_norm = t / np.linalg.norm(t) + + if len(ts_norm) > 0 and np.any(np.dot(np.array(ts_norm), t_norm.reshape(3, 1)) > 0.99): + success = False + print("fail") + else: + ts_norm.append(t_norm) + ts.append(t) + Rs.append(R) + success = True + + bgr, depth, bbs = self._renderer.render_many( + obj_is, + self._width, + self._height, + self._K.copy(), + Rs, + ts, + self._near_plane, + self._far_plane, + random_light=True, + ) + + rand_voc = cv2.imread(self._voc_imgs[np.random.randint(len(self._voc_imgs))]) + rand_voc = cv2.resize(rand_voc, (self._width, self._height)) + rand_voc = rand_voc.astype(np.float32) / 255.0 + # print bgr.max() + bgr = bgr.astype(np.float32) / 255.0 + + depth_three_chan = np.dstack((depth,) * 3) + bgr = rand_voc * (depth_three_chan == 0.0).astype(np.uint8) + bgr * (depth_three_chan > 0).astype(np.uint8) + + obj_info = [] + for (x, y, w, h), obj_id in zip(bbs, self.obj_ids[np.array(obj_is)]): + xmin = np.minimum(x, x + w) + xmax = np.maximum(x, x + w) + ymin = np.minimum(y, y + h) + ymax = np.maximum(y, y + h) + obj_info.append({"id": obj_id, "bb": [int(xmin), int(ymin), int(xmax), int(ymax)]}) + + bgr = (bgr * 255.0).astype(np.uint8) + + if self._augmenters != None: + bgr = self._augmenters.augment_image(bgr) + + return bgr, obj_info diff --git a/lib/meshrenderer/shader/cad_shader.frag b/lib/meshrenderer/shader/cad_shader.frag new file mode 100644 index 00000000..d4ca2564 --- /dev/null +++ b/lib/meshrenderer/shader/cad_shader.frag @@ -0,0 +1,46 @@ +#version 450 core + +in vec3 v_color; +in vec3 v_view; + +in vec3 v_normal; +in vec3 v_light_dir; + +layout (location = 0) out vec3 rgb; +layout (location = 1) out float depth; + +layout (location = 2) uniform float dirlight_ambient; +layout (location = 3) uniform float dirlight_diffuse; +layout (location = 4) uniform float dirlight_specular; + +void main(void) { + vec3 Normal = normalize(v_normal); + vec3 LightDir = normalize(v_light_dir); + vec3 ViewDir = normalize(v_view); + + vec3 material_ambient = vec3(223./255, 214./255, 205./255 ); + vec3 material_diffuse = vec3(223./255, 214./255, 205./255 ); + vec3 material_specular = vec3(223./255, 214./255 , 205./255 ); + + vec3 diffuse = max(dot(Normal, LightDir), 0.0) * material_diffuse; + vec3 R = reflect(-LightDir, Normal); + vec3 specular = max(dot(R, ViewDir), 0.0) * material_specular; + +/* vec3 dirlight_ambient = vec3(0.5); + vec3 dirlight_diffuse = vec3(0.6); + vec3 dirlight_specular = vec3(0.3); */ + + rgb = vec3(dirlight_ambient * material_ambient + + dirlight_diffuse * diffuse + + dirlight_specular * specular); + + if(rgb.x > 1.0) rgb.x = 1.0; + if(rgb.y > 1.0) rgb.y = 1.0; + if(rgb.z > 1.0) rgb.z = 1.0; + + depth = v_view.z; +} + + + + diff --git a/lib/meshrenderer/shader/cad_shader.vs b/lib/meshrenderer/shader/cad_shader.vs new file mode 100644 index 00000000..cdeef570 --- /dev/null +++ b/lib/meshrenderer/shader/cad_shader.vs @@ -0,0 +1,34 @@ +#version 450 core + +layout (location = 0) in vec3 position; +layout (location = 1) in vec3 normal; + +layout (binding=0) readonly buffer SCENE_BUFFER { + mat4 view; + mat4 projection; + vec3 viewPos; +}; + +layout (location = 1) uniform vec3 u_light_eye_pos; +layout (location = 0) uniform vec3 light_pos; + +/* out vec3 ViewDir; */ +out vec3 v_color; +out vec3 v_view; + +out vec3 v_normal; +out vec3 v_light_dir; + +void main(void) { + vec4 P = view * vec4(position, 1.0); + + v_view = -P.xyz; + gl_Position = projection * P; + + mat4 u_nm = transpose(inverse(view)); + + vec3 v_eye_pos = P.xyz; // Vertex position in eye coords. + v_normal = normalize(u_nm * vec4(normal, 1.0)).xyz; // Normal in eye coords. + + v_light_dir = light_pos - P.xyz; +} \ No newline at end of file diff --git a/lib/meshrenderer/shader/depth_shader_phong.frag b/lib/meshrenderer/shader/depth_shader_phong.frag new file mode 100644 index 00000000..ab15e35c --- /dev/null +++ b/lib/meshrenderer/shader/depth_shader_phong.frag @@ -0,0 +1,71 @@ +#version 450 core + +in vec3 v_color; +in vec3 v_view; +in vec3 ViewDir; + +in vec3 v_L; +in vec3 v_normal; + +layout (location = 0) uniform float u_light_ambient_w; +layout (location = 2) uniform float u_light_diffuse_w; +layout (location = 3) uniform float u_light_specular_w; + +layout (location = 0) out vec3 rgb; +layout (location = 1) out float depth; +layout (location = 2) out vec3 rgb_normals; + +void main(void) { + + vec3 Normal = normalize(v_normal); + vec3 LightDir = normalize(v_L); + vec3 ViewDir = normalize(v_view); + + vec3 diffuse = max(dot(Normal, LightDir), 0.0) * v_color; + vec3 R = reflect(-LightDir, Normal); + vec3 specular = max(dot(R, ViewDir), 0.0) * v_color; + + + rgb = vec3(u_light_ambient_w * v_color + + u_light_diffuse_w * diffuse + + u_light_specular_w * specular); + + if(rgb.x > 1.0) rgb.x = 1.0; + if(rgb.y > 1.0) rgb.y = 1.0; + if(rgb.z > 1.0) rgb.z = 1.0; + rgb_normals = vec3(Normal * 0.5 + 0.5); // transforms from [-1,1] to [0,1] + depth = v_view.z; +} + +/* vec3 rgb2hsv(vec3 c) { + vec4 K = vec4(0.0, -1.0 / 3.0, 2.0 / 3.0, -1.0); + vec4 p = mix(vec4(c.bg, K.wz), vec4(c.gb, K.xy), step(c.b, c.g)); + vec4 q = mix(vec4(p.xyw, c.r), vec4(c.r, p.yzx), step(p.x, c.r)); + + float d = q.x - min(q.w, q.y); + float e = 1.0e-10; + return vec3(abs(q.z + (q.w - q.y) / (6.0 * d + e)), d / (q.x + e), q.x); +} + */ +/* vec3 hsv2rgb(vec3 c) { + vec4 K = vec4(1.0, 2.0 / 3.0, 1.0 / 3.0, 3.0); + vec3 p = abs(fract(c.xxx + K.xyz) * 6.0 - K.www); + return c.z * mix(K.xxx, clamp(p - K.xxx, 0.0, 1.0), c.y); +} + */ + + //if(light_w > 1.0) light_w = 1.0; + + //vec3 hsv_color = rgb2hsv(v_color); + //hsv_color.y = 0.5; + //hsv_color.y = 0.0; + //hsv_color.z = clamp(hsv_color.z, 0., 0.62); +/* hsv_color.z = clamp(hsv_color.z, 0., 0.8); + vec3 rgb_color = hsv2rgb(hsv_color); */ + + //rgb = light_w * rgb_color; + + + + + diff --git a/lib/meshrenderer/shader/depth_shader_phong.vs b/lib/meshrenderer/shader/depth_shader_phong.vs new file mode 100644 index 00000000..3d986ae6 --- /dev/null +++ b/lib/meshrenderer/shader/depth_shader_phong.vs @@ -0,0 +1,32 @@ +#version 450 core + +layout (location = 0) in vec3 position; +layout (location = 1) in vec3 normal; +layout (location = 2) in vec3 color; + +layout (binding=0) readonly buffer SCENE_BUFFER { + mat4 view; + mat4 projection; + vec3 viewPos; +}; + +layout (location = 1) uniform vec3 u_light_eye_pos; + +out vec3 v_color; +out vec3 v_view; + +out vec3 v_L; +out vec3 v_normal; + +void main(void) { + vec4 P = view * vec4(position, 1.0); + v_view = -P.xyz; + gl_Position = projection * P; + v_color = color; + + mat4 u_nm = transpose(inverse(view)); + + vec3 v_eye_pos = P.xyz; // Vertex position in eye coords. + v_L = normalize(u_light_eye_pos - v_eye_pos); // Vector to the light + v_normal = normalize(u_nm * vec4(normal, 1.0)).xyz; // Normal in eye coords. +} \ No newline at end of file diff --git a/lib/meshrenderer/shader/line.frag b/lib/meshrenderer/shader/line.frag new file mode 100644 index 00000000..627b9233 --- /dev/null +++ b/lib/meshrenderer/shader/line.frag @@ -0,0 +1,9 @@ +#version 450 core + +layout (location=0) out vec3 color; + +in vec3 v_color; + +void main() { + color = v_color; +} \ No newline at end of file diff --git a/lib/meshrenderer/shader/line.vs b/lib/meshrenderer/shader/line.vs new file mode 100644 index 00000000..cf0397cd --- /dev/null +++ b/lib/meshrenderer/shader/line.vs @@ -0,0 +1,82 @@ +#version 450 core + +layout (location = 0) in vec3 position; +layout (location = 1) in vec3 normal; + +layout (binding=0) readonly buffer SCENE_BUFFER { + mat4 view; + mat4 projection; + vec3 viewPos; +}; + +layout (location = 0) uniform vec3 vert_min; +layout (location = 1) uniform vec3 vert_max; + +out vec3 v_color; + +void main() { + const vec4 vertices[] = vec4[]( /* vec4( 0, 0, 0, 1), + vec4( 30, 0, 0, 1), + vec4( 0, 0, 0, 1), + vec4( 0, 30, 0, 1), + vec4( 0, 0, 0, 1), + vec4( 0, 0, 30, 1), */ + vec4( vert_min[0],vert_min[1],vert_min[2],1), + vec4( vert_min[0],vert_min[1],vert_max[2],1), + vec4( vert_min[0],vert_min[1],vert_min[2],1), + vec4( vert_min[0],vert_max[1],vert_min[2],1), + vec4( vert_min[0],vert_min[1],vert_min[2],1), + vec4( vert_max[0],vert_min[1],vert_min[2],1), + vec4( vert_min[0],vert_min[1],vert_max[2],1), + vec4( vert_min[0],vert_max[1],vert_max[2],1), + vec4( vert_min[0],vert_min[1],vert_max[2],1), + vec4( vert_max[0],vert_min[1],vert_max[2],1), + vec4( vert_min[0],vert_max[1],vert_min[2],1), + vec4( vert_min[0],vert_max[1],vert_max[2],1), + vec4( vert_min[0],vert_max[1],vert_min[2],1), + vec4( vert_max[0],vert_max[1],vert_min[2],1), + vec4( vert_max[0],vert_min[1],vert_min[2],1), + vec4( vert_max[0],vert_max[1],vert_min[2],1), + vec4( vert_max[0],vert_min[1],vert_min[2],1), + vec4( vert_max[0],vert_min[1],vert_max[2],1), + vec4( vert_max[0],vert_max[1],vert_max[2],1), + vec4( vert_min[0],vert_max[1],vert_max[2],1), + vec4( vert_max[0],vert_max[1],vert_max[2],1), + vec4( vert_max[0],vert_min[1],vert_max[2],1), + vec4( vert_max[0],vert_max[1],vert_max[2],1), + vec4( vert_max[0],vert_max[1],vert_min[2],1)); + + const vec3 colors[] = vec3[]( /* vec3(1, 0, 0), + vec3(1, 0, 0), + vec3(0, 1, 0), + vec3(0, 1, 0), + vec3(0, 0, 1), + vec3(0, 0, 1), */ + vec3(0, 1, 0), + vec3(0, 1, 0), + vec3(0, 1, 0), + vec3(0, 1, 0), + vec3(0, 1, 0), + vec3(0, 1, 0), + vec3(0, 1, 0), + vec3(0, 1, 0), + vec3(0, 1, 0), + vec3(0, 1, 0), + vec3(0, 1, 0), + vec3(0, 1, 0), + vec3(0, 1, 0), + vec3(0, 1, 0), + vec3(0, 1, 0), + vec3(0, 1, 0), + vec3(0, 1, 0), + vec3(0, 1, 0), + vec3(0, 1, 0), + vec3(0, 1, 0), + vec3(0, 1, 0), + vec3(0, 1, 0), + vec3(0, 1, 0), + vec3(0, 1, 0)); + + gl_Position = projection * view * vertices[gl_InstanceID*2+gl_VertexID]; + v_color = colors[gl_InstanceID*2+gl_VertexID]; +} \ No newline at end of file diff --git a/lib/meshrenderer/write_xml.py b/lib/meshrenderer/write_xml.py new file mode 100644 index 00000000..6f325a09 --- /dev/null +++ b/lib/meshrenderer/write_xml.py @@ -0,0 +1,35 @@ +import xml.etree.cElementTree as ET +import os + + +def write_xml(obj_infos, width, height, obj_info, classname, outputpath, filename): + annotation = ET.Element("annotation") + + ET.SubElement(annotation, "folder").text = "tobedefined" + ET.SubElement(annotation, "filename").text = filename + ".png" + ET.SubElement(annotation, "path").text = os.path.join(outputpath, filename + ".png") + + source = ET.SubElement(annotation, "source") + ET.SubElement(source, "database").text = "Unknown" + + size = ET.SubElement(annotation, "size") + ET.SubElement(size, "width").text = str(width) + ET.SubElement(size, "height").text = str(height) + ET.SubElement(size, "depth").text = "3" + + ET.SubElement(annotation, "segmented").text = "0" + + for obj_info in obj_infos: + obj = ET.SubElement(annotation, "object") + ET.SubElement(obj, "name").text = str(obj_info["id"]) + ET.SubElement(obj, "pose").text = "Unspecified" + ET.SubElement(obj, "truncated").text = "0" + ET.SubElement(obj, "difficult").text = "0" + bbox = ET.SubElement(obj, "bndbox") + ET.SubElement(bbox, "xmin").text = str(obj_info["bb"][0]) + ET.SubElement(bbox, "ymin").text = str(obj_info["bb"][1]) + ET.SubElement(bbox, "xmax").text = str(obj_info["bb"][2]) + ET.SubElement(bbox, "ymax").text = str(obj_info["bb"][3]) + + tree = ET.ElementTree(annotation) + tree.write(os.path.join(outputpath, filename + ".xml")) diff --git a/scripts/format_code.sh b/scripts/format_code.sh index 5e86446a..c6d69667 100755 --- a/scripts/format_code.sh +++ b/scripts/format_code.sh @@ -7,10 +7,11 @@ # [ "$2" = "$(echo -e "$1\\n$2" | sort -V | head -n1)" ] #} +BLACK_VERSION="21.7b0" { - black --version | grep -E "21.5b0" > /dev/null + black --version | grep -E $BLACK_VERSION > /dev/null } || { - echo "Linter requires 'black==21.5b0' !" + echo "Linter requires 'black==$BLACK_VERSION' !" exit 1 } diff --git a/tools/lm/lm_pbr_1_gen_xyz_crop.py b/tools/lm/lm_pbr_1_gen_xyz_crop.py new file mode 100644 index 00000000..00c10fbf --- /dev/null +++ b/tools/lm/lm_pbr_1_gen_xyz_crop.py @@ -0,0 +1,232 @@ +from __future__ import division, print_function + +import os + +os.environ["PYOPENGL_PLATFORM"] = "egl" +import os.path as osp +import sys + +import mmcv +import numpy as np +from tqdm import tqdm + +cur_dir = osp.abspath(osp.dirname(__file__)) +PROJ_ROOT = osp.join(cur_dir, "../..") +sys.path.insert(0, PROJ_ROOT) +from lib.meshrenderer.meshrenderer_phong import Renderer +from lib.vis_utils.image import grid_show +from lib.pysixd import misc +from lib.utils.mask_utils import mask2bbox_xyxy + + +idx2class = { + 1: "ape", + 2: "benchvise", + 3: "bowl", + 4: "camera", + 5: "can", + 6: "cat", + 7: "cup", + 8: "driller", + 9: "duck", + 10: "eggbox", + 11: "glue", + 12: "holepuncher", + 13: "iron", + 14: "lamp", + 15: "phone", +} + +class2idx = {_name: _id for _id, _name in idx2class.items()} + +classes = idx2class.values() +classes = sorted(classes) + +# DEPTH_FACTOR = 1000. +IM_H = 480 +IM_W = 640 +near = 0.01 +far = 6.5 + +data_dir = osp.normpath(osp.join(PROJ_ROOT, "datasets/BOP_DATASETS/lm/train_pbr")) + +cls_indexes = sorted(idx2class.keys()) +cls_names = [idx2class[cls_idx] for cls_idx in cls_indexes] +lm_model_dir = osp.normpath(osp.join(PROJ_ROOT, "datasets/BOP_DATASETS/lm/models")) +model_paths = [osp.join(lm_model_dir, f"obj_{obj_id:06d}.ply") for obj_id in cls_indexes] +texture_paths = None + +scenes = [i for i in range(0, 49 + 1)] +xyz_root = osp.normpath(osp.join(PROJ_ROOT, "datasets/BOP_DATASETS/lm/train_pbr/xyz_crop")) + +K = np.array([[572.4114, 0, 325.2611], [0, 573.57043, 242.04899], [0, 0, 1]]) + + +def normalize_to_01(img): + if img.max() != img.min(): + return (img - img.min()) / (img.max() - img.min()) + else: + return img + + +def get_emb_show(bbox_emb): + show_emb = bbox_emb.copy() + show_emb = normalize_to_01(show_emb) + return show_emb + + +class XyzGen(object): + def __init__(self, split="train", scene="all"): + if split == "train": + scene_ids = scenes + data_root = data_dir + else: + raise ValueError(f"split {split} error") + + if scene == "all": + sel_scene_ids = scene_ids + else: + assert int(scene) in scene_ids, f"{scene} not in {scene_ids}" + sel_scene_ids = [int(scene)] + print("split: ", split, "selected scene ids: ", sel_scene_ids) + self.split = split + self.scene = scene + self.sel_scene_ids = sel_scene_ids + self.data_root = data_root + self.renderer = None + + def get_renderer(self): + if self.renderer is None: + self.renderer = Renderer( + model_paths, vertex_tmp_store_folder=osp.join(PROJ_ROOT, ".cache"), vertex_scale=0.001 + ) + return self.renderer + + def main(self): + split = self.split + scene = self.scene # "all" or a single scene + sel_scene_ids = self.sel_scene_ids + data_root = self.data_root + + for scene_id in tqdm(sel_scene_ids, postfix=f"{split}_{scene}"): + print("split: {} scene: {}".format(split, scene_id)) + scene_root = osp.join(data_root, f"{scene_id:06d}") + + gt_dict = mmcv.load(osp.join(scene_root, "scene_gt.json")) + # gt_info_dict = mmcv.load(osp.join(scene_root, "scene_gt_info.json")) + # cam_dict = mmcv.load(osp.join(scene_root, "scene_camera.json")) + + for str_im_id in tqdm(gt_dict, postfix=f"{scene_id}"): + int_im_id = int(str_im_id) + + for anno_i, anno in enumerate(gt_dict[str_im_id]): + obj_id = anno["obj_id"] + if obj_id not in idx2class: + continue + + R = np.array(anno["cam_R_m2c"], dtype="float32").reshape(3, 3) + t = np.array(anno["cam_t_m2c"], dtype="float32") / 1000.0 + # pose = np.hstack([R, t.reshape(3, 1)]) + + save_path = osp.join( + xyz_root, + f"{scene_id:06d}/{int_im_id:06d}_{anno_i:06d}-xyz.pkl", + ) + # if osp.exists(save_path) and osp.getsize(save_path) > 0: + # continue + + render_obj_id = cls_indexes.index(obj_id) # 0-based + bgr_gl, depth_gl = self.get_renderer().render(render_obj_id, IM_W, IM_H, K, R, t, near, far) + mask = (depth_gl > 0).astype("uint8") + + if mask.sum() == 0: # NOTE: this should be ignored at training phase + print( + f"not visible, split {split} scene {scene_id}, im {int_im_id} obj {idx2class[obj_id]} {obj_id}" + ) + print(f"{save_path}") + xyz_info = { + "xyz_crop": np.zeros((IM_H, IM_W, 3), dtype=np.float16), + "xyxy": [0, 0, IM_W - 1, IM_H - 1], + } + if VIS: + im_path = osp.join( + data_root, + f"{scene_id:06d}/rgb/{int_im_id:06d}.jpg", + ) + im = mmcv.imread(im_path) + + mask_path = osp.join( + data_root, + f"{scene_id:06d}/mask/{int_im_id:06d}_{anno_i:06d}.png", + ) + mask_visib_path = osp.join( + data_root, + f"{scene_id:06d}/mask_visib/{int_im_id:06d}_{anno_i:06d}.png", + ) + mask_gt = mmcv.imread(mask_path, "unchanged") + mask_visib_gt = mmcv.imread(mask_visib_path, "unchanged") + + show_ims = [ + bgr_gl[:, :, [2, 1, 0]], + im[:, :, [2, 1, 0]], + mask_gt, + mask_visib_gt, + ] + show_titles = [ + "bgr_gl", + "im", + "mask_gt", + "mask_visib_gt", + ] + grid_show(show_ims, show_titles, row=2, col=2) + raise RuntimeError(f"split {split} scene {scene_id}, im {int_im_id}") + else: + x1, y1, x2, y2 = mask2bbox_xyxy(mask) + xyz_np = misc.calc_xyz_bp_fast(depth_gl, R, t, K) + xyz_crop = xyz_np[y1 : y2 + 1, x1 : x2 + 1] + xyz_info = { + "xyz_crop": xyz_crop.astype("float16"), # save disk space w/o performance drop + "xyxy": [x1, y1, x2, y2], + } + + if VIS: + print(f"xyz_crop min {xyz_crop.min()} max {xyz_crop.max()}") + show_ims = [ + bgr_gl[:, :, [2, 1, 0]], + get_emb_show(xyz_np), + get_emb_show(xyz_crop), + ] + show_titles = ["bgr_gl", "xyz", "xyz_crop"] + grid_show(show_ims, show_titles, row=1, col=3) + + if not args.no_save: + mmcv.mkdir_or_exist(osp.dirname(save_path)) + mmcv.dump(xyz_info, save_path) + if self.renderer is not None: + self.renderer.close() + + +if __name__ == "__main__": + import argparse + import time + + import setproctitle + + parser = argparse.ArgumentParser(description="gen lm train_pbr xyz") + parser.add_argument("--split", type=str, default="train", help="split") + parser.add_argument("--scene", type=str, default="all", help="scene id") + parser.add_argument("--vis", default=False, action="store_true", help="vis") + parser.add_argument("--no-save", default=False, action="store_true", help="do not save results") + args = parser.parse_args() + + height = IM_H + width = IM_W + + VIS = args.vis + + T_begin = time.perf_counter() + setproctitle.setproctitle(f"gen_xyz_lm_train_pbr_{args.split}_{args.scene}") + xyz_gen = XyzGen(args.split, args.scene) + xyz_gen.main() + T_end = time.perf_counter() - T_begin + print("split", args.split, "scene", args.scene, "total time: ", T_end)