An Error in imgui_viewer

[wangwang-xyz]

There is a problem when I run ./bin/tum_mono or ./bin/gaussian_viewer. The Error like this

Pass 'Combine redundant instructions' is not initialized.
Verify if there is a pass dependency cycle.
Required Passes:
Segmentation fault

Code related with this problem in imgui_viewer.cpp line 363

glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
        // Draw main window image
        if (show_main_rendered_)
        {
            auto drawlist = ImGui::GetBackgroundDrawList();
            if (pSLAM_ && tracking_vision_)
            {
                drawlist->AddImage((void *)(intptr_t)rendered_img_texture, ImVec2(0, 0),
                                   ImVec2(glfw_window_width_, glfw_window_height_));
            }
            else
            {
                cv::Mat main_img = pGausMapper_->renderFromPose(
                    Tcw_main_, rendered_image_width_main_, rendered_image_height_main_, true);
                cv::Mat main_img_to_show = cv::Mat(rendered_image_height_main_, padded_main_image_width_, CV_32FC3, cv::Vec3f(0.0f, 0.0f, 0.0f));
                main_img.copyTo(main_img_to_show(image_rect_main));
                glBindTexture(GL_TEXTURE_2D, main_img_texture);
                glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, main_img_to_show.cols, main_img_to_show.rows,
                     0, GL_RGB, GL_FLOAT, (float*)main_img_to_show.data);
                drawlist->AddImage((void *)(intptr_t)main_img_texture, ImVec2(0, 0),
                                   ImVec2(glfw_window_width_, glfw_window_height_));
            }
        }

However when I write a test.cpp to test this code, it works well. The test.cpp like this:

/* Copyright 2021 iwatake2222

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
/*** Include ***/
/* for general */
#include <cstdint>
#include <cstdio>
#include <fstream> 
#include <vector>
#include <string>
#include <chrono>

/* for GLFW */
#include <GLFW/glfw3.h>

/* for ImGui */
#include "viewer/imgui/imgui.h"
#include "viewer/imgui/imgui_impl_glfw.h"
#include "viewer/imgui/imgui_impl_opengl3.h"

#include <glm/glm.hpp>
#include <glm/gtc/matrix_transform.hpp>
#include <glm/gtc/type_ptr.hpp>
#include <glm/gtx/string_cast.hpp>

#include <stdio.h>
#include <opencv2/core/core.hpp>
#include <opencv2/opencv.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/types_c.h>

/*** Macro ***/
/* macro functions */
#define RUN_CHECK(x)                                         \
  if (!(x)) {                                                \
    fprintf(stderr, "Error at %s:%d\n", __FILE__, __LINE__); \
    exit(1);                                                 \
  }

/* Setting */
static void glfw_error_callback(int error, const char* description)
{
    fprintf(stderr, "[ImGuiViewer]GLFW Error %d: %s\n", error, description);
}
/*** Global variable ***/


/*** Function ***/
int main(int argc, char *argv[])
{
    // Initialize glfw
    glfwSetErrorCallback(glfw_error_callback);
    if (!glfwInit())
        throw std::runtime_error("[ImGuiViewer]Fails to initialize!");

    const char* glsl_version = "#version 130";
    glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 4);
    glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 5);
    glfwWindowHint(GLFW_RESIZABLE, GL_FALSE);

    // Create window with graphics context
    GLFWwindow* window =
        glfwCreateWindow(1400, 1050,
                         "Photo-SLAM", nullptr, nullptr);

    if (window == nullptr)
        throw std::runtime_error("[ImGuiViewer]Fails to create window!");
    glfwMakeContextCurrent(window);
    glfwSwapInterval(1); // Enable vsync
    glEnable(GL_DEPTH_TEST); // Enable 3D Mouse handler
    
    // Setup Dear ImGui context
    IMGUI_CHECKVERSION();
    ImGui::CreateContext();
    ImGuiIO& io = ImGui::GetIO(); (void)io;
    io.ConfigFlags |= ImGuiConfigFlags_NavEnableKeyboard;  // Enable Keyboard Controls
    io.ConfigFlags |= ImGuiConfigFlags_NavEnableGamepad;   // Enable Gamepad Controls
  

    /* imgui:  Setup Dear ImGui style */
    ImGui::StyleColorsClassic();

    /* imgui:  Setup Platform/Renderer backends */
    ImGui_ImplGlfw_InitForOpenGL(window, true);
    ImGui_ImplOpenGL3_Init(glsl_version);

    GLuint SLAM_img_texture, rendered_img_texture, main_img_texture;

    glGenTextures(1, &SLAM_img_texture);
    glBindTexture(GL_TEXTURE_2D, SLAM_img_texture);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);

    glGenTextures(1, &rendered_img_texture);
    glBindTexture(GL_TEXTURE_2D, rendered_img_texture);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);

    glGenTextures(1, &main_img_texture);
    glBindTexture(GL_TEXTURE_2D, main_img_texture);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);

    glm::mat4 cam_trans_(1.0f);
    /*** Start loop ***/
    while(1) {
        cv::Mat main_img = cv::imread("/home/wang/MonoGS/datasets/tum/rgbd_dataset_freiburg2_xyz/rgb/1311867170.562349.png");
        // Poll and handle events (inputs, window resize, etc.)
        // You can read the io.WantCaptureMouse, io.WantCaptureKeyboard flags to tell if dear imgui wants to use your inputs.
        // - When io.WantCaptureMouse is true, do not dispatch mouse input data to your main application.
        // - When io.WantCaptureKeyboard is true, do not dispatch keyboard input data to your main application.
        // Generally you may always pass all inputs to dear imgui, and hide them from your application based on those two flags.
        glfwPollEvents();

        /* imgui:  Start the Dear ImGui frame */
        ImGui_ImplOpenGL3_NewFrame();
        ImGui_ImplGlfw_NewFrame();
        ImGui::NewFrame();

        int display_w, display_h;
        glfwGetFramebufferSize(window, &display_w, &display_h);
        glViewport(0, 0, display_w, display_h);
        glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);


        cv::Mat SLAM_img_to_show = cv::Mat(480, 640, CV_8UC3, cv::Vec3f(0, 0, 0));
        main_img.copyTo(SLAM_img_to_show);
        // Upload SLAM frame
        glBindTexture(GL_TEXTURE_2D, SLAM_img_texture);
        glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, SLAM_img_to_show.cols, SLAM_img_to_show.rows,
                    0, GL_BGR, GL_UNSIGNED_BYTE, (unsigned char*)SLAM_img_to_show.data);
        // Create an ImGui window to show the SLAM frame
        ImGui::SetNextWindowPos(ImVec2(0, 0), ImGuiCond_Once);
        ImGui::SetNextWindowSize(ImVec2(640 + 12, SLAM_img_to_show.rows + 40), ImGuiCond_Once);
        {
            ImGui::Begin("SLAM Frame");
            ImGui::Image((void *)(intptr_t)SLAM_img_texture,
                        ImVec2(SLAM_img_to_show.cols, SLAM_img_to_show.rows));
            ImGui::End();
        }

        main_img.convertTo(main_img, CV_32FC3, 1/255.0);
        cv::Mat rendered_img_to_show = cv::Mat(480, 640, CV_32FC3, cv::Vec3f(0.0f, 0.0f, 0.0f));
        main_img.copyTo(rendered_img_to_show);
        // Upload rendered frame
        glBindTexture(GL_TEXTURE_2D, rendered_img_texture);
        glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, rendered_img_to_show.cols, rendered_img_to_show.rows,
                    0, GL_RGB, GL_FLOAT, (float*)rendered_img_to_show.data);
        // Create an ImGui window to show the rendered frame
        ImGui::SetNextWindowPos(ImVec2(0, SLAM_img_to_show.rows + 40), ImGuiCond_Once);
        ImGui::SetNextWindowSize(ImVec2(640 + 12, rendered_img_to_show.rows + 40), ImGuiCond_Once);
        {
            ImGui::Begin("Current Rendered Frame");
            ImGui::Image((void *)(intptr_t)rendered_img_texture,
                        ImVec2(rendered_img_to_show.cols, rendered_img_to_show.rows));
            ImGui::End();
        }

        
        cv::Rect image_rect_main(0, 0, 640, 480);
        auto drawlist = ImGui::GetBackgroundDrawList();
        drawlist->AddImage((void *)(intptr_t)rendered_img_texture, ImVec2(0, 0), ImVec2(1400, 1050));
        cv::Mat main_img_to_show = cv::Mat(480, 640, CV_32FC3, cv::Vec3f(0.0f, 0.0f, 0.0f));
        main_img.copyTo(main_img_to_show(image_rect_main));
        glBindTexture(GL_TEXTURE_2D, main_img_texture);
        glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, main_img_to_show.cols, main_img_to_show.rows,
                0, GL_RGB, GL_FLOAT, (float*)main_img_to_show.data);
        drawlist->AddImage((void *)(intptr_t)main_img_texture, ImVec2(0, 0),
                            ImVec2(1400, 1050));


        /* Clear the screen */
        glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
        glClear(GL_COLOR_BUFFER_BIT);

        /* imgui:  Rendering */
        ImGui::Render();
        ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData());

        glPushMatrix();
        glMultMatrixf(&cam_trans_[0][0]);
        glPopMatrix();
        std::cout<<std::endl<<glm::to_string(cam_trans_)<<std::endl;
        /* Swap buffers */
        glfwSwapBuffers(window);
        glfwPollEvents();

        /* Check if the ESC key was pressed or the window was closed */
        if (glfwWindowShouldClose(window) != 0 || glfwGetKey(window, GLFW_KEY_ESCAPE) == GLFW_PRESS) {
            break;
        }
    }

    /*** Finalize ***/
    /* imgui: cleanup */
    ImGui_ImplOpenGL3_Shutdown();
    ImGui_ImplGlfw_Shutdown();
    ImGui::DestroyContext();

    /* Close OpenGL window and terminate GLFW */
    glfwDestroyWindow(window);
    glfwTerminate();

    return 0;
}

The related CMakeLists.txt like this:

add_executable(test examples/test.cpp)
target_link_libraries(test
    # gaussian_viewer
    glm::glm
    glfw
    OpenGL::GL
    imgui
    ${OpenCV_LIBRARIES}
)

[liquorleaf]

Hi!
Which version of CUDA and LibTorch did you use?
If you have integrated graphics along with independent graphics, have you tried setting the system to disable the integrated ones and use only the independent ones?

[wangwang-xyz]

The environment I used is：

• cuda 11.8
• LibTorch 2.0.1
• WSL2

It seems this problem caused by LibTorch which built with llvm.
I flowed this issue to update LibTorch to nightly version(2.4.0.dev20240415+cu118) and solved the problem
pytorch/pytorch#103756

In addition, LibTorch removed c10::guts::to_string in nightly version. Therefore, gaussian_model.cpp need to be modified.
Replace all

key = c10::guts::to_string(param.UnsafeGetTensorImpl());

with

key = param.UnsafeGetTensorImpl();

[Eragonlrc]

Similar segmentation fault error solved thanks to this issue!
BYW in LibTorch nightly version (2.4.0.dev20240425+cu118) replace all

key = c10::guts::to_string(param.UnsafeGetTensorImpl());

with

key = param.unsafeGetTensorImpl();

which uses camel case.

[alexjunholee]

In my case, the error was showing the exact same error message.

it was just a segfault error like below:
104 Segmentation fault (core dumped)

However, changing to nightly build as recommended from @wangwang-xyz and @Eragonlrc solved the issue! Thanks!

*Note that you need to remove the whole build folder when re-building with the new libtorch.

[Jiyu-Yao]

Thank you for your solution!
However,I met some trouble after I update LibTorch to nightly version.The error like this
/usr/bin/ld: /Photo-SLAM/ORB-SLAM3/lib/libORB_SLAM3.so: undefined reference to cv::FileStorage::FileStorage(std::__cxx11::basic_string<char, std::char_traits, std::allocator > const&, int, std::__cxx11::basic_string<char, std::char_traits, std::allocator > const&)' /usr/bin/ld: /Photo-SLAM/ORB-SLAM3/lib/libORB_SLAM3.so: undefined reference to cv::operator<<(cv::FileStorage&, std::__cxx11::basic_string<char, std::char_traits, std::allocator > const&)'
/usr/bin/ld: /Photo-SLAM/ORB-SLAM3/lib/libORB_SLAM3.so: undefined reference to cv::error(int, std::__cxx11::basic_string<char, std::char_traits, std::allocator > const&, char const*, char const*, int)' /usr/bin/ld: /Photo-SLAM/lib/libgaussian_mapper.so: undefined reference to Json::Value::Value(std::string const&)'
/usr/bin/ld: /Photo-SLAM/ORB-SLAM3/lib/libORB_SLAM3.so: undefined reference to cv::getTextSize(std::__cxx11::basic_string<char, std::char_traits, std::allocator > const&, int, double, int, int*)' /usr/bin/ld: /Photo-SLAM/ORB-SLAM3/lib/libORB_SLAM3.so: undefined reference to cv::putText(cv::_InputOutputArray const&, std::cxx11::basic_string<char, std::char_traits, std::allocator > const&, cv::Point, int, double, cv::Scalar, int, int, bool)'
/usr/bin/ld: /Photo-SLAM/ORB-SLAM3/lib/libORB_SLAM3.so: undefined reference to cv::FileStorage::operator[](std::__cxx11::basic_string<char, std::char_traits, std::allocator > const&) const' /usr/bin/ld: /Photo-SLAM/ORB-SLAM3/lib/libORB_SLAM3.so: undefined reference to cv::write(cv::FileStorage&, std::__cxx11::basic_string<char, std::char_traits, std::allocator > const&, int)'
/usr/bin/ld: /Photo-SLAM/ORB-SLAM3/lib/libORB_SLAM3.so: undefined reference to cv::write(cv::FileStorage&, std::__cxx11::basic_string<char, std::char_traits, std::allocator > const&, double)' /usr/bin/ld: /Photo-SLAM/ORB-SLAM3/lib/libORB_SLAM3.so: undefined reference to cv::FileNode::stringabi:cxx11 const'
collect2: error: ld returned 1 exit status
make[2]: *** [CMakeFiles/view_result.dir/build.make:184: /Photo-SLAM/bin/view_result] Error 1
make[1]: *** [CMakeFiles/Makefile2:273: CMakeFiles/view_result.dir/all] Error 2
make[1]: *** Waiting for unfinished jobs....

It seems the opencv mismatch with the libtorch?I have no idea about this. Have you met this trouble?Could you please teach me how to solve it? Thank you!

[HuajianUP]

Hi Jiyu,

I reproduced this error when using the latest version of LibTorch, regardless of the OpenCV version. We will investigate further and try to support the latest version of LibTorch in the future.

Please use the tested LibTorch version first and then you should be able to compile the code successfully.