update

engine/cli/commands/chat_completion_cmd.cc

@@ -37,7 +37,7 @@ size_t WriteCallback(char* ptr, size_t size, size_t nmemb, void* userdata) {
     }
 
     try {
-      std::string content =
+      auto content =
           json_helper::ParseJsonString(chunk)["choices"][0]["delta"]["content"]
               .asString();
       std::cout << content << std::flush;
@@ -51,7 +51,6 @@ size_t WriteCallback(char* ptr, size_t size, size_t nmemb, void* userdata) {
 
   return data_length;
 }
-
 }  // namespace
 
 void ChatCompletionCmd::Exec(const std::string& host, int port,

engine/common/model_metadata.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "common/tokenizer.h"
+#include <sstream>
+
+struct ModelMetadata {
+  uint32_t version;
+  uint64_t tensor_count;
+  uint64_t metadata_kv_count;
+  std::unique_ptr<Tokenizer> tokenizer;
+
+  std::string ToString() const {
+    std::ostringstream ss;
+    ss << "ModelMetadata {\n"
+       << "version: " << version << "\n"
+       << "tensor_count: " << tensor_count << "\n"
+       << "metadata_kv_count: " << metadata_kv_count << "\n"
+       << "tokenizer: ";
+
+    if (tokenizer) {
+        ss << "\n" << tokenizer->ToString();
+    } else {
+        ss << "null";
+    }
+
+    ss << "\n}";
+    return ss.str();
+  }
+};

engine/common/tokenizer.h

@@ -0,0 +1,68 @@
+#pragma once
+
+#include <sstream>
+#include <string>
+
+struct Tokenizer {
+  std::string eos_token = "";
+  bool add_eos_token = true;
+
+  std::string bos_token = "";
+  bool add_bos_token = true;
+
+  std::string unknown_token = "";
+  std::string padding_token = "";
+
+  std::string chat_template = "";
+
+  // Helper function for common fields
+  std::string BaseToString() const {
+    std::ostringstream ss;
+    ss << "eos_token: \"" << eos_token << "\"\n"
+       << "add_eos_token: " << (add_eos_token ? "true" : "false") << "\n"
+       << "bos_token: \"" << bos_token << "\"\n"
+       << "add_bos_token: " << (add_bos_token ? "true" : "false") << "\n"
+       << "unknown_token: \"" << unknown_token << "\"\n"
+       << "padding_token: \"" << padding_token << "\"\n"
+       << "chat_template: \"" << chat_template << "\"";
+    return ss.str();
+  }
+
+  virtual ~Tokenizer() = default;
+
+  virtual std::string ToString() = 0;
+};
+
+struct GgufTokenizer : public Tokenizer {
+  std::string pre = "";
+
+  ~GgufTokenizer() override = default;
+
+  std::string ToString() override {
+    std::ostringstream ss;
+    ss << "GgufTokenizer {\n";
+    // Add base class members
+    ss << BaseToString() << "\n";
+    // Add derived class members
+    ss << "pre: \"" << pre << "\"\n";
+    ss << "}";
+    return ss.str();
+  }
+};
+
+struct SafeTensorTokenizer : public Tokenizer {
+  bool add_prefix_space = true;
+
+  ~SafeTensorTokenizer() = default;
+
+  std::string ToString() override {
+    std::ostringstream ss;
+    ss << "SafeTensorTokenizer {\n";
+    // Add base class members
+    ss << BaseToString() << "\n";
+    // Add derived class members
+    ss << "add_prefix_space: " << (add_prefix_space ? "true" : "false") << "\n";
+    ss << "}";
+    return ss.str();
+  }
+};

engine/config/chat_template_renderer.h

@@ -276,15 +276,15 @@ static int32_t llama_chat_apply_template_internal(
     }
   } else if (tmpl == "llama3" || (tmpl_contains("<|start_header_id|>") &&
                                   tmpl_contains("<|end_header_id|>"))) {
-    // Llama 3
-    for (auto message : chat) {
-      std::string role(message->role);
-      ss << "<|start_header_id|>" << role << "<|end_header_id|>\n\n"
-         << trim(message->content) << "<|eot_id|>";
-    }
-    if (add_ass) {
-      ss << "<|start_header_id|>assistant<|end_header_id|>\n\n";
-    }
+    // // Llama 3
+    // for (auto message : chat) {
+    //   std::string role(message->role);
+    //   ss << "<|start_header_id|>" << role << "<|end_header_id|>\n\n"
+    //      << trim(message->content) << "<|eot_id|>";
+    // }
+    // if (add_ass) {
+    //   ss << "<|start_header_id|>assistant<|end_header_id|>\n\n";
+    // }
   } else if (tmpl == "chatglm3" || tmpl_contains("[gMASK]sop")) {
     // chatglm3-6b
     ss << "[gMASK]" << "sop";
@@ -426,4 +426,4 @@ std::string llama_chat_apply_template(const std::string& tmpl,
   std::string formatted_chat(buf.data(), res);
   return formatted_chat;
 }
-}  // namespace config
+}  // namespace config