fix: warm-up (#259)

* fix: warm-up * fix: n_predict 10 * Revert "fix: n_predict 10" This reverts commit d08e952. * revert: CI --------- Co-authored-by: vansangpfiev <[email protected]>
janhq · Oct 22, 2024 · f430ead · f430ead
1 parent 0d268b0
commit f430ead
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/src/llama_engine.cc b/src/llama_engine.cc
@@ -978,6 +978,8 @@ void LlamaEngine::WarmUpModel(const std::string& model_id) {
     pseudo["prompt"] = "Hello";
     pseudo["n_predict"] = 2;
     pseudo["stream"] = false;
+    pseudo["cache_prompt"] = server_map_[model_id].caching_enabled;
+    pseudo["n_keep"] = 0;
     const int task_id =
         si->second.ctx.RequestCompletion(pseudo, false, false, -1);
     TaskResult result = si->second.ctx.NextResult(task_id);