Support LLM

app/realtime-captioning/README.md

@@ -0,0 +1,30 @@
+1. Install dependencies
+
+```
+python -m pip install -r requirements.txt
+
+2. Deploy latest ion-kit binaries with LLM support
+
+Build llama.cpp following instruction in `ion-kit/src/bb/llm/config.cmake`
+
+Build ion-kit:
+```
+cmake -D CMAKE_BUILD_TYPE=Release -DLlama_DIR=<path-to-llama.cpp-install>/lib/cmake/Llama .. && cmake --build .
+```
+
+Replace binares:
+```
+cp ./install/lib/lib* <path-to-site-packages/ionpy/module/linux/
+```
+
+
+3. Download LLaVA models from Hugging Face
+
+- [mmproj-mistral7b-f16-q6_k.gguf](https://huggingface.co/cmp-nct/llava-1.6-gguf/resolve/main/mmproj-mistral7b-f16-q6_k.gguf?download=true)
+- [ggml-mistral-q_4_k.gguf](https://huggingface.co/cmp-nct/llava-1.6-gguf/resolve/main/ggml-mistral-q_4_k.gguf?download=true)
+
+4. Connect UVC camera and run
+
+```
+python3 main.py
+```

app/realtime-captioning/main.py

@@ -0,0 +1,161 @@
+import sys
+import argparse
+
+from ionpy import Node, Builder, Buffer, Port, Param, Type, TypeCode
+import numpy as np
+
+from tkinter import *
+from tkinter.ttk import *
+import sv_ttk
+from PIL import ImageTk, Image
+
+class App(Frame):
+    def __init__(self, window, args):
+        super().__init__(window, padding=15)
+
+        self.window = window
+
+        # Model
+        self.camera_width = int(args.resolution.split('x')[0])
+        self.camera_height = int(args.resolution.split('x')[1])
+        self.screen_width = self.winfo_screenwidth()
+        self.screen_height = self.winfo_screenheight()
+        self.img = np.zeros((self.camera_height, self.camera_width, 3), dtype=np.uint8)
+        self.prompt = np.zeros(1024, dtype=np.int8)
+        self.response = np.zeros(1024, dtype=np.int8)
+
+        # View model
+        self.prompt_string = StringVar()
+        self.prompt_string.set('Explain the image in a single sentence.')
+
+        # Support variables
+        self.live_mode = True
+        self.advanced_mode = False
+        self.analyze_in_progress = False
+        self.last_response = np.copy(self.response)
+
+        self.init_pipeline()
+        self.init_layout()
+
+        self.window.protocol("WM_DELETE_WINDOW", self.on_closing)
+
+    def init_pipeline(self):
+
+        self.b = Builder()
+        self.b.set_target("host-cuda")
+        self.b.with_bb_module("ion-bb")
+
+
+        # U3V camera
+        # params = [Param("num_devices", 1), Param("realtime_diaplay_mode", True)]
+        # n_img_cwh = self.b.add("image_io_u3v_cameraN_u8x3").set_param(params)
+
+        # UVC camera
+        params = [Param("width", self.camera_width), Param("height", self.camera_height)]
+        n_img_whc = self.b.add("image_io_camera").set_param(params)
+        params = [Param("dim0", 2), Param("dim1", 0), Param("dim2", 1)]
+        n_img_cwh = self.b.add("base_reorder_buffer_3d_uint8").set_iport([n_img_whc.get_port("output")]).set_param(params);
+
+        self.prompt_buf = Buffer(array=self.prompt)
+        prompt_port = Port(name="prompt", type=Type(TypeCode.Int, 8, 1), dim=1)
+        prompt_port.bind(self.prompt_buf)
+
+        params = [Param("width", self.camera_width), Param("height", self.camera_height)]
+        n_txt = self.b.add("llm_llava").set_iport([n_img_cwh.get_port("output")[0], prompt_port]).set_param(params)
+
+        for i in range(self.camera_height):
+            for j in range(self.camera_width):
+                self.img[i][j] = [i%256, i%256, i%256]
+
+        self.img_buf = Buffer(array=self.img)
+        n_img_cwh.get_port("output").bind(self.img_buf)
+
+        self.response_buf = Buffer(array=self.response)
+        n_txt.get_port("output").bind(self.response_buf)
+
+    def init_layout(self):
+        self.img_canvas = Canvas(self, width = self.screen_width, height = self.screen_height)
+        self.img_canvas.pack()
+
+        response_frame = Frame(self, padding=15, height=50)
+        self.response_label = Label(response_frame, font=('Helvetica', 48), wraplength=self.screen_width-30, padding=15, anchor = 'nw', justify='left')
+        self.response_label.pack()
+        self.img_canvas.create_window(40, 40, window = response_frame, anchor = 'nw')
+
+        self.update_prompt()
+        self.update_response()
+        self.update_periodic()
+
+    def update_periodic(self):
+        # Running pipeline
+        self.b.run()
+
+        img = Image.fromarray(self.img)
+        cutoff = self.camera_height - (self.screen_height / self.screen_width) * self.camera_width
+        img = img.crop((0, cutoff/2, self.camera_width, self.camera_height-cutoff/2))
+        img = img.resize((self.screen_width, self.screen_height))
+        self.photo = ImageTk.PhotoImage(image = img)
+        self.img_canvas.create_image(0, 0, image = self.photo, anchor = 'nw')
+
+        if (self.live_mode):
+            self.update_response()
+
+        self.window.after(30, self.update_periodic)
+
+    def update_prompt(self):
+        self.prompt.fill(0)
+        i = 0
+        if not self.advanced_mode:
+            # Append image prompt marker
+            for c in '<image>':
+                self.prompt[i] = ord(c)
+                i += 1
+        offset = i
+        for i, c in enumerate(self.prompt_string.get()):
+            self.prompt[offset+i] = ord(c)
+
+        # Clearing response make look & feel better
+        self.response.fill(0)
+        self.response_label.configure(text='')
+
+    def update_response(self):
+        # question = "Hey buddy, what's on your eyes?\n"
+        response = ''.join([chr(v) for v in self.response])
+        response = response.split('.')[0]
+        self.response_label.configure(text=response)
+
+    def toggle_live(self):
+        self.live_mode = not self.live_mode
+        self.analysis_button.configure(state='disabled' if self.live_mode else 'normal')
+
+    def analyze(self):
+        self.analyze_in_progress = True
+        self.analysis_button.configure(text='Analyzing...')
+        self.last_response = np.copy(self.response)
+        self.window.after(100, self.wait_response)
+
+    def wait_response(self):
+        if np.array_equal(self.last_response, self.response):
+            self.window.after(100, self.wait_response)
+        else:
+            self.update_response()
+            self.analysis_button.configure(text='Analyze')
+            self.analyze_in_progress = False
+
+    def on_closing(self):
+        del self.b
+        self.window.destroy()
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--resolution', default='640x480', help='Camera resolution in "<width>x<height>" format. e.g. 640x480')
+
+    args = parser.parse_args()
+
+    root = Tk()
+    root.wm_attributes('-type', 'splash')
+    root.wm_attributes('-fullscreen', True)
+    sv_ttk.set_theme("dark")
+    App(root, args).pack(expand=True, fill='both')
+    root.mainloop()

app/realtime-captioning/requirements.txt

@@ -0,0 +1,4 @@
+ion-python >= 1.8.2
+numpy
+sv_ttk
+pillow

example/CMakeLists.txt

@@ -6,31 +6,35 @@
 find_package(OpenCV 4 QUIET)
 
 if(${ION_BB_BUILD_base})
-    ion_aot_executable(producerx3 SRCS_COMPILE producerx3_compile.cc SRCS_RUN producerx3_run.cc TARGET_STRING "host-profile")
     if(${CUDA_FOUND})
-        ion_aot_executable(producerx3_gpu SRCS_COMPILE producerx3_compile.cc SRCS_RUN producerx3_run.cc TARGET_STRING "host-cuda-cuda_capability_50-profile")
+        ion_aot_executable(producerx3 SRCS_COMPILE producerx3_compile.cc SRCS_RUN producerx3_run.cc TARGET_STRING "host-cuda-cuda_capability_50-profile")
+    else()
+        ion_aot_executable(producerx3 SRCS_COMPILE producerx3_compile.cc SRCS_RUN producerx3_run.cc TARGET_STRING "host-profile")
     endif()
 endif()
 
 if(${ION_BB_BUILD_image-processing} AND ${ION_BB_BUILD_image-io} AND ${ION_BB_BUILD_sgm} AND OpenCV_FOUND)
-    ion_aot_executable(sgm SRCS_COMPILE sgm_compile.cc SRCS_RUN sgm_run.cc INCS ${OPenCV_INCLUDE_DIR} LIBS ion-bb ${OpenCV_LIBRARIES} TARGET_STRING "host-profile")
     if(${CUDA_FOUND})
-        ion_aot_executable(sgm_gpu SRCS_COMPILE sgm_compile.cc SRCS_RUN sgm_run.cc INCS ${OPenCV_INCLUDE_DIR} LIBS ion-bb ${OpenCV_LIBRARIES} TARGET_STRING "host-cuda-cuda_capability_50-profile")
+        ion_aot_executable(sgm SRCS_COMPILE sgm_compile.cc SRCS_RUN sgm_run.cc INCS ${OPenCV_INCLUDE_DIR} LIBS ion-bb ${OpenCV_LIBRARIES} TARGET_STRING "host-cuda-cuda_capability_50-profile")
+    else()
+        ion_aot_executable(sgm SRCS_COMPILE sgm_compile.cc SRCS_RUN sgm_run.cc INCS ${OPenCV_INCLUDE_DIR} LIBS ion-bb ${OpenCV_LIBRARIES} TARGET_STRING "host-profile")
     endif()
 endif()
 
 if(${ION_BB_BUILD_base} AND ${ION_BB_BUILD_image-io} AND ${ION_BB_BUILD_dnn} AND OpenCV_FOUND)
-    ion_aot_executable(dnn SRCS_COMPILE dnn_compile.cc SRCS_RUN dnn_run.cc INCS ${OpenCV_INCLUDE_DIR} LIBS ion-bb ${OpenCV_LIBRARIES} TARGET_STRING "host-profile")
     if(${CUDA_FOUND})
-        ion_aot_executable(dnn_gpu SRCS_COMPILE dnn_compile.cc SRCS_RUN dnn_run.cc INCS ${OpenCV_INCLUDE_DIR} LIBS ion-bb ${OpenCV_LIBRARIES} TARGET_STRING "host-cuda-cuda_capability_50-profile")
+        ion_aot_executable(dnn SRCS_COMPILE dnn_compile.cc SRCS_RUN dnn_run.cc INCS ${OpenCV_INCLUDE_DIR} LIBS ion-bb ${OpenCV_LIBRARIES} TARGET_STRING "host-cuda-cuda_capability_50-profile")
+    else()
+        ion_aot_executable(dnn SRCS_COMPILE dnn_compile.cc SRCS_RUN dnn_run.cc INCS ${OpenCV_INCLUDE_DIR} LIBS ion-bb ${OpenCV_LIBRARIES} TARGET_STRING "host-profile")
     endif()
     ion_jit_executable(dnn SRCS dnn.cc INCS ${OpenCV_INCLUDE_DIRS} LIBS ${OpenCV_LIBRARIES})
 endif()
 
 if(${ION_BB_BUILD_base} AND ${ION_BB_BUILD_image-processing} AND ${ION_BB_BUILD_image-io} AND OpenCV_FOUND AND UNIX AND NOT APPLE)
-    ion_aot_executable(demo SRCS_COMPILE demo_compile.cc SRCS_RUN demo_run.cc INCS ${OpenCV_INCLUDE_DIR} LIBS ion-bb ${OpenCV_LIBRARIES} TARGET_STRING "host-profile")
     if(${CUDA_FOUND})
-        ion_aot_executable(demo_gpu SRCS_COMPILE demo_compile.cc SRCS_RUN demo_run.cc INCS ${OpenCV_INCLUDE_DIR} LIBS ion-bb ${OpenCV_LIBRARIES} TARGET_STRING "host-cuda-cuda_capability_50-profile")
+        ion_aot_executable(demo SRCS_COMPILE demo_compile.cc SRCS_RUN demo_run.cc INCS ${OpenCV_INCLUDE_DIR} LIBS ion-bb ${OpenCV_LIBRARIES} TARGET_STRING "host-cuda-cuda_capability_50-profile")
+    else()
+        ion_aot_executable(demo SRCS_COMPILE demo_compile.cc SRCS_RUN demo_run.cc INCS ${OpenCV_INCLUDE_DIR} LIBS ion-bb ${OpenCV_LIBRARIES} TARGET_STRING "host-profile")
     endif()
     ion_jit_executable(demo SRCS demo.cc INCS ${OpenCV_INCLUDE_DIR} LIBS ion-bb ${OpenCV_LIBRARIES})
 endif()
@@ -65,6 +69,10 @@ if(${ION_BB_BUILD_fpga})
     ion_aot_executable(fpga_dnn SRCS_COMPILE fpga_dnn_compile.cc SRCS_RUN fpga_dnn_run.cc TARGET_STRING "arm-64-linux-vivado_hls-dpu")
 endif()
 
+if(${ION_BB_BUILD_llm} AND OpenCV_FOUND)
+    ion_jit_executable(llm_llava SRCS llm_llava.cc LIBS opencv_highgui)
+endif()
+
 if(MSVC)
     add_definitions(/bigobj)
     message(STATUS "Allow big object for example/")

example/llm_llava.cc

@@ -0,0 +1,57 @@
+#include <ion/ion.h>
+#include <iostream>
+
+#include <opencv2/highgui.hpp>
+
+using namespace ion;
+
+int main(int argc, char *argv[]) {
+    try {
+        // const int width = 1280;
+        // const int height = 960;
+        const int width = 503;
+        const int height = 337;
+
+        Buffer<int8_t> prompt{1024};
+        prompt.fill(0);
+        std::string prompt_s("<image>Explain the image in one sentence.");
+        for (auto i = 0; i < prompt_s.size(); ++i) {
+            prompt(i) = prompt_s[i];
+        }
+
+        Builder b;
+        b.set_target(Halide::get_target_from_environment());
+        b.with_bb_module("ion-bb");
+
+        auto n_img_cwh = b.add("image_io_color_data_loader").set_param(Param{"url", "http://www.onthejob.education/images/4th_level/Road_Worker/Road_Worker_Darwin.jpg"}, Param{"width", width}, Param{"height", height});
+        auto n_img_whc = b.add("base_reorder_buffer_3d_uint8")(n_img_cwh["output"]).set_param(Param{"dim0", 2}, Param{"dim1", 0}, Param{"dim2", 1});
+        // auto n_img_cwh = b.add("image_io_u3v_cameraN_u8x3").set_param(Param{"num_devices", "1"}, Param{"realtime_diaplay_mode", true});
+        // auto n_img_whc = b.add("base_reorder_buffer_3d_uint8")(n_img_cwh["output"]).set_param(Param{"dim0", 1}, Param{"dim1", 2}, Param{"dim2", 0});
+
+        auto n_disp = b.add("image_io_gui_display")(n_img_whc["output"][0]).set_param(Param{"width", width}, Param{"height", height});
+        auto n_txt = b.add("llm_llava")(n_img_cwh["output"][0], prompt).set_param(Param{"width", width}, Param{"height", height});
+
+        Buffer<int8_t> txt_output{1024};
+        n_txt["output"].bind(txt_output);
+
+        Buffer<int32_t> result = Buffer<int32_t>::make_scalar();
+        n_disp["output"].bind(result);
+
+        // for (int i=0; i<1; ++i) {
+        while (true) {
+            b.run();
+            std::cout << reinterpret_cast<const char *>(txt_output.data()) << std::endl;
+        }
+
+    } catch (const Halide::Error &e) {
+        std::cerr << e.what() << std::endl;
+        return 1;
+    } catch (const std::exception &e) {
+        std::cerr << e.what() << std::endl;
+        return 1;
+    } catch (...) {
+        return 1;
+    }
+
+    return 0;
+}

python/ionpy/module/linux/.gitignore

@@ -0,0 +1 @@
+*.so*

src/bb/CMakeLists.txt

@@ -3,13 +3,15 @@ cmake_policy(SET CMP0057 NEW)
 set(ION_BB_INCLUDE_DIRS)
 set(ION_BB_LINK_DIRS)
 set(ION_BB_LIBRARIES)
+set(ION_BB_SRCS)
 file(GLOB childs ${CMAKE_CURRENT_SOURCE_DIR}/*)
-set(BB_NAMES base dnn fpga image-io image-processing opencv sgm)
+set(BB_NAMES base dnn fpga image-io image-processing opencv sgm llm)
 foreach(BB_NAME IN LISTS BB_NAMES)
     if(IS_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/${BB_NAME})
         set(INCLUDE_DIRS)
         set(LINK_DIRS)
         set(LIBRARIES)
+        set(EXTRA_SRCS)
         include(${CMAKE_CURRENT_SOURCE_DIR}/${BB_NAME}/config.cmake)
         if (${ION_BB_BUILD_${BB_NAME}})
             list(APPEND ION_BB_INCLUDE_DIRS ${INCLUDE_DIRS})
@@ -19,19 +21,24 @@ foreach(BB_NAME IN LISTS BB_NAMES)
             string(REPLACE "-" "_" POSTFIX ${POSTFIX})
             add_definitions("-DION_ENABLE_BB_${POSTFIX}")
             set(ION_BB_BUILD_${BB_NAME} TRUE PARENT_SCOPE)
+
+            foreach(SRC IN LISTS EXTRA_SRCS)
+                list(APPEND ION_BB_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/${BB_NAME}/${SRC})
+            endforeach()
+
         else()
             set(ION_BB_BUILD_${BB_NAME} FALSE PARENT_SCOPE)
             message("Skip building \"${BB_NAME}\"")
         endif()
     endif()
 endforeach()
 
-add_library(ion-bb SHARED bb.cc)
+add_library(ion-bb SHARED bb.cc ${ION_BB_SRCS})
 target_include_directories(ion-bb PUBLIC ${PROJECT_SOURCE_DIR}/include ${PROJECT_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR} ${ION_BB_INCLUDE_DIRS})
-
+target_link_directories(ion-bb PUBLIC ${ION_BB_LINK_DIRS})
 target_link_libraries(ion-bb PUBLIC ion-core ${ION_BB_LIBRARIES})
 if(UNIX)
-    target_compile_options(ion-bb PUBLIC -fno-rtti)  # For Halide::Generator
+    # target_compile_options(ion-bb PUBLIC -fno-rtti)  # For Halide::Generator
     if(NOT APPLE)
         target_link_options(ion-bb PUBLIC -Wl,--export-dynamic) # For JIT compiling
     endif()

src/bb/bb.cc

@@ -34,6 +34,11 @@
 #include "sgm/rt.h"
 #endif
 
+#if defined(ION_ENABLE_BB_LLM)
+#include "llm/bb.h"
+#include "llm/rt.h"
+#endif
+
 extern "C" void register_externs(std::map<std::string, Halide::JITExtern>& externs) {
 #if defined(ION_ENABLE_BB_BASE)
   for (auto kv : ion::bb::base::extern_functions) {
@@ -65,4 +70,9 @@ extern "C" void register_externs(std::map<std::string, Halide::JITExtern>& exter
     externs.insert({kv.first, Halide::JITExtern(kv.second)});
   }
 #endif
+#if defined(ION_ENABLE_BB_LLM)
+  for (auto kv : ion::bb::llm::extern_functions) {
+    externs.insert({kv.first, Halide::JITExtern(kv.second)});
+  }
+#endif
 }