Merge pull request #109 from janhq/pump-fixes

hotfix: update bug fixes for recent batch inference
janhq · Nov 7, 2023 · 3c674cd · 3c674cd
2 parents d358274 + 131359e
commit 3c674cd
Show file tree

Hide file tree

Showing 4 changed files with 1,907 additions and 1,722 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -51,7 +51,7 @@ add_executable(${PROJECT_NAME} main.cc)
 #
 # and comment out the following lines
 find_package(Drogon CONFIG REQUIRED)
-target_link_libraries(${PROJECT_NAME} PRIVATE Drogon::Drogon common llama clip
+target_link_libraries(${PROJECT_NAME} PRIVATE Drogon::Drogon common llama llava
                                               ${CMAKE_THREAD_LIBS_INIT})
 
 # ##############################################################################

diff --git a/controllers/llamaCPP.cc b/controllers/llamaCPP.cc
@@ -110,7 +110,7 @@ void llamaCPP::chatCompletion(
     data["stop"] = stopWords;
   }
 
-  const int task_id = llama.request_completion(data, false);
+  const int task_id = llama.request_completion(data, false,false);
   LOG_INFO << "Resolved request for task_id:" << task_id;
 
   auto state = createState(task_id, this);
@@ -178,7 +178,7 @@ void llamaCPP::embedding(
     prompt = "";
   }
   const int task_id =
-      llama.request_completion({{"prompt", prompt}, {"n_predict", 0}}, false);
+      llama.request_completion({{"prompt", prompt}, {"n_predict", 0}}, false, true);
   task_result result = llama.next_result(task_id);
   std::string embeddingResp = result.result_json.dump();
   auto resp = nitro_utils::nitroHttpResponse();
@@ -226,8 +226,8 @@ void llamaCPP::loadModel(
 
   llama_backend_init(params.numa);
 
-  LOG_INFO_LLAMA("build info",
-                 {{"build", BUILD_NUMBER}, {"commit", BUILD_COMMIT}});
+  // LOG_INFO_LLAMA("build info",
+  //                {{"build", BUILD_NUMBER}, {"commit", BUILD_COMMIT}});
   LOG_INFO_LLAMA("system info",
                  {
                      {"n_threads", params.n_threads},