WIP

fixstars · May 1, 2024 · d021033 · d021033
1 parent da42f28
commit d021033
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 15 deletions.
diff --git a/app/demo.py b/app/demo.py
@@ -30,7 +30,7 @@ def __init__(self, window):
         self.prompt_string = StringVar()
         self.prompt_string.set('Explain the image in a single sentence.')
 
-        # Support variables 
+        # Support variables
         self.live_mode = False
         self.advanced_mode = False
         self.analyze_in_progress = False
@@ -42,7 +42,7 @@ def __init__(self, window):
         self.window.protocol("WM_DELETE_WINDOW", self.on_closing)
 
     def init_pipeline(self):
-        
+
         self.b = Builder()
         self.b.set_target("host-cuda")
         self.b.with_bb_module("ion-bb")
@@ -66,11 +66,12 @@ def init_pipeline(self):
 
         self.response_buf = Buffer(array=self.response)
         n_txt.get_port("output").bind(self.response_buf)
-    
+
     def init_layout(self):
         img_frame = Frame(self, style='Card.TFrame', padding=15)
         img_frame.grid(row=0, column=0, sticky='nsew')
-        self.img_canvas = Canvas(img_frame, width = self.width, height = self.height)
+        #self.img_canvas = Canvas(img_frame, width = self.width, height = self.height)
+        self.img_canvas = Canvas(img_frame, width = self.width * 0.85, height = self.height * 0.85)
         self.img_canvas.pack()
 
         control_frame = Frame(self, padding=15)
@@ -81,10 +82,10 @@ def init_layout(self):
 
         self.live_checkbutton = Checkbutton(control_frame, text='Live', style='Switch.TCheckbutton', command=self.toggle_live)
         self.live_checkbutton.grid(row=0, column=11, padx=5)
-        
+
         self.analysis_button = Button(control_frame, text='Analyze', command = self.analyze, width=10, style='Accent.TButton')
         self.analysis_button.grid(row=0, column=12, padx=5)
- 
+
         response_frame = Frame(self, padding=15)
         response_frame.grid(row=2, columnspan=2, sticky='nsew')
         self.response_label = Label(response_frame, font=('Helvetica', 18), wraplength=1200, justify='left')
@@ -93,19 +94,19 @@ def init_layout(self):
         self.update_prompt()
         self.update_response()
         self.update_periodic()
-  
+
     def update_periodic(self):
         # Running pipeline
         self.b.run()
-        
+
         self.photo = ImageTk.PhotoImage(image = Image.fromarray(self.img))
         self.img_canvas.create_image(0, 0, image = self.photo, anchor = NW)
 
         if (self.live_mode):
             self.update_response()
 
         self.window.after(30, self.update_periodic)
- 
+
     def update_prompt(self):
         self.prompt.fill(0)
         i = 0
@@ -117,7 +118,7 @@ def update_prompt(self):
         offset = i
         for i, c in enumerate(self.prompt_string.get()):
             self.prompt[offset+i] = ord(c)
-        
+
         # Clearing response make look & feel better
         self.response.fill(0)
         self.response_label.configure(text='')

diff --git a/example/llm_llava.cc b/example/llm_llava.cc
@@ -7,12 +7,14 @@ using namespace ion;
 
 int main(int argc, char *argv[]) {
     try {
-        const int width = 1280;
-        const int height = 960;
+        // const int width = 1280;
+        // const int height = 960;
+        const int width = 503;
+        const int height = 337;
 
         Buffer<int8_t> prompt{1024};
         prompt.fill(0);
-        std::string prompt_s("Explain the image in a single sentence.");
+        std::string prompt_s("<image>Explain the image in one sentence.");
         for (auto i = 0; i < prompt_s.size(); ++i) {
             prompt(i) = prompt_s[i];
         }
@@ -21,8 +23,11 @@ int main(int argc, char *argv[]) {
         b.set_target(Halide::get_target_from_environment());
         b.with_bb_module("ion-bb");
 
-        auto n_img_cwh = b.add("image_io_u3v_cameraN_u8x3").set_param(Param{"num_devices", "1"}, Param{"realtime_diaplay_mode", true});
-        auto n_img_whc = b.add("base_reorder_buffer_3d_uint8")(n_img_cwh["output"]).set_param(Param{"dim0", 1}, Param{"dim1", 2}, Param{"dim2", 0});
+        auto n_img_cwh = b.add("image_io_color_data_loader").set_param(Param{"url", "http://www.onthejob.education/images/4th_level/Road_Worker/Road_Worker_Darwin.jpg"}, Param{"width", width}, Param{"height", height});
+-       auto n_img_whc = b.add("base_reorder_buffer_3d_uint8")(n_img_cwh["output"]).set_param(Param{"dim0", 2}, Param{"dim1", 0}, Param{"dim2", 1});
+        // auto n_img_cwh = b.add("image_io_u3v_cameraN_u8x3").set_param(Param{"num_devices", "1"}, Param{"realtime_diaplay_mode", true});
+        // auto n_img_whc = b.add("base_reorder_buffer_3d_uint8")(n_img_cwh["output"]).set_param(Param{"dim0", 1}, Param{"dim1", 2}, Param{"dim2", 0});
+
         auto n_disp = b.add("image_io_gui_display")(n_img_whc["output"][0]).set_param(Param{"width", width}, Param{"height", height});
         auto n_txt = b.add("llm_llava")(n_img_cwh["output"][0], prompt).set_param(Param{"width", width}, Param{"height", height});
 
@@ -32,8 +37,10 @@ int main(int argc, char *argv[]) {
         Buffer<int32_t> result = Buffer<int32_t>::make_scalar();
         n_disp["output"].bind(result);
 
+        // for (int i=0; i<1; ++i) {
         while (true) {
             b.run();
+            std::cout << reinterpret_cast<const char *>(txt_output.data()) << std::endl;
         }
 
     } catch (const Halide::Error &e) {