fix: improve remote engine (#1787)

* fix: improve streaming message for remote engine * feat: improve remote engine * chore: cleanup * fix: correct remote engine check * chore: add unit tests * fix: cleanup * chore: cleanup --------- Co-authored-by: vansangpfiev <[email protected]>
janhq · Dec 17, 2024 · 0255f3d · 0255f3d
1 parent bb97612
commit 0255f3d
Show file tree

Hide file tree

Showing 19 changed files with 332 additions and 346 deletions.
diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt
@@ -144,8 +144,6 @@ add_executable(${TARGET_NAME} main.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/utils/file_logger.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/utils/dylib_path_manager.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/extensions/remote-engine/remote_engine.cc
-    ${CMAKE_CURRENT_SOURCE_DIR}/extensions/remote-engine/openai_engine.cc
-    ${CMAKE_CURRENT_SOURCE_DIR}/extensions/remote-engine/anthropic_engine.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/extensions/remote-engine/template_renderer.cc
 )
 

diff --git a/engine/cli/CMakeLists.txt b/engine/cli/CMakeLists.txt
@@ -84,8 +84,6 @@ add_executable(${TARGET_NAME} main.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/../services/inference_service.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/../services/hardware_service.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/../extensions/remote-engine/remote_engine.cc
-    ${CMAKE_CURRENT_SOURCE_DIR}/../extensions/remote-engine/openai_engine.cc
-    ${CMAKE_CURRENT_SOURCE_DIR}/../extensions/remote-engine/anthropic_engine.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/../extensions/remote-engine/template_renderer.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/utils/easywsclient.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/utils/download_progress.cc

diff --git a/engine/common/engine_servicei.h b/engine/common/engine_servicei.h
@@ -58,4 +58,6 @@ class EngineServiceI {
   GetEngineByNameAndVariant(
       const std::string& engine_name,
       const std::optional<std::string> variant = std::nullopt) = 0;
+
+  virtual bool IsRemoteEngine(const std::string& engine_name) = 0;
 };
diff --git a/engine/config/model_config.h b/engine/config/model_config.h
@@ -8,52 +8,12 @@
 #include <stdexcept>
 #include <string>
 #include <vector>
+#include "config/remote_template.h"
 #include "utils/format_utils.h"
 #include "utils/remote_models_utils.h"
 
 namespace config {
 
-namespace {
-const std::string kOpenAITransformReqTemplate =
-    R"({ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} })";
-const std::string kOpenAITransformRespTemplate =
-    R"({ {%- set first = true -%} {%- for key, value in input_request -%} {%- if key == \"id\" or key == \"choices\" or key == \"created\" or key == \"model\" or key == \"service_tier\" or key == \"system_fingerprint\" or key == \"object\" or key == \"usage\" -%} {%- if not first -%},{%- endif -%} \"{{ key }}\": {{ tojson(value) }} {%- set first = false -%} {%- endif -%} {%- endfor -%} })";
-const std::string kAnthropicTransformReqTemplate =
-    R"({ {% set first = true %} {% for key, value in input_request %} {% if key == \"system\" or key == \"messages\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} })";
-const std::string kAnthropicTransformRespTemplate = R"({
-        "id": "{{ input_request.id }}",
-        "created": null,
-        "object": "chat.completion",
-        "model": "{{ input_request.model }}",
-        "choices": [
-          {
-            "index": 0,
-            "message": {
-              "role": "{{ input_request.role }}",
-              "content": "{% if input_request.content and input_request.content.0.type == "text" %}  {{input_request.content.0.text}} {% endif %}",
-              "refusal": null
-            },
-            "logprobs": null,
-            "finish_reason": "{{ input_request.stop_reason }}"
-          }
-        ],
-        "usage": {
-          "prompt_tokens": {{ input_request.usage.input_tokens }},
-          "completion_tokens": {{ input_request.usage.output_tokens }},
-          "total_tokens": {{ input_request.usage.input_tokens + input_request.usage.output_tokens }},
-          "prompt_tokens_details": {
-            "cached_tokens": 0
-          },
-          "completion_tokens_details": {
-            "reasoning_tokens": 0,
-            "accepted_prediction_tokens": 0,
-            "rejected_prediction_tokens": 0
-          }
-        },
-        "system_fingerprint": "fp_6b68a8204b"
-      })";
-}  // namespace
-
 struct RemoteModelConfig {
   std::string model;
   std::string api_key_template;
@@ -108,6 +68,7 @@ struct RemoteModelConfig {
             kOpenAITransformRespTemplate;
       }
     }
+
     metadata = json.get("metadata", metadata);
   }
 

diff --git a/engine/config/remote_template.h b/engine/config/remote_template.h
@@ -0,0 +1,66 @@
+#include <string>
+
+namespace config {
+const std::string kOpenAITransformReqTemplate =
+    R"({ {% set first = true %} {% for key, value in input_request %} {% if key == "messages" or key == "model" or key == "temperature" or key == "store" or key == "max_tokens" or key == "stream" or key == "presence_penalty" or key == "metadata" or key == "frequency_penalty" or key == "tools" or key == "tool_choice" or key == "logprobs" or key == "top_logprobs" or key == "logit_bias" or key == "n" or key == "modalities" or key == "prediction" or key == "response_format" or key == "service_tier" or key == "seed" or key == "stop" or key == "stream_options" or key == "top_p" or key == "parallel_tool_calls" or key == "user" %} {% if not first %},{% endif %} "{{ key }}": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} })";
+const std::string kOpenAITransformRespTemplate =
+    R"({ {%- set first = true -%} {%- for key, value in input_request -%} {%- if key == "id" or key == "choices" or key == "created" or key == "model" or key == "service_tier" or key == "system_fingerprint" or key == "object" or key == "usage" -%} {%- if not first -%},{%- endif -%} "{{ key }}": {{ tojson(value) }} {%- set first = false -%} {%- endif -%} {%- endfor -%} })";
+const std::string kAnthropicTransformReqTemplate =
+    R"({ 
+      {% for key, value in input_request %}
+        {% if key == "messages" %}  
+          {% if input_request.messages.0.role == "system" %}
+            "system": "{{ input_request.messages.0.content }}",
+            "messages": [
+              {% for message in input_request.messages %}
+                {% if not loop.is_first %}
+                  {"role": "{{ message.role }}", "content": "{{ message.content }}" } {% if not loop.is_last %},{% endif %}
+                {% endif %}
+              {% endfor %}
+            ]
+          {% else %}
+            "messages": [
+              {% for message in input_request.messages %}
+                {"role": " {{ message.role}}", "content": "{{ message.content }}" } {% if not loop.is_last %},{% endif %}
+              {% endfor %}
+            ]
+          {% endif %}
+        {% else if key == "system" or key == "model" or key == "temperature" or key == "store" or key == "max_tokens" or key == "stream" or key == "presence_penalty" or key == "metadata" or key == "frequency_penalty" or key == "tools" or key == "tool_choice" or key == "logprobs" or key == "top_logprobs" or key == "logit_bias" or key == "n" or key == "modalities" or key == "prediction" or key == "response_format" or key == "service_tier" or key == "seed" or key == "stop" or key == "stream_options" or key == "top_p" or key == "parallel_tool_calls" or key == "user" %}
+          "{{ key }}": {{ tojson(value) }}   
+        {% endif %}      
+        {% if not loop.is_last %},{% endif %} 
+      {% endfor %} })";
+const std::string kAnthropicTransformRespTemplate = R"({
+        "id": "{{ input_request.id }}",
+        "created": null,
+        "object": "chat.completion",
+        "model": "{{ input_request.model }}",
+        "choices": [
+          {
+            "index": 0,
+            "message": {
+              "role": "{{ input_request.role }}",
+              "content": "{% if input_request.content and input_request.content.0.type == "text" %}  {{input_request.content.0.text}} {% endif %}",
+              "refusal": null
+            },
+            "logprobs": null,
+            "finish_reason": "{{ input_request.stop_reason }}"
+          }
+        ],
+        "usage": {
+          "prompt_tokens": {{ input_request.usage.input_tokens }},
+          "completion_tokens": {{ input_request.usage.output_tokens }},
+          "total_tokens": {{ input_request.usage.input_tokens + input_request.usage.output_tokens }},
+          "prompt_tokens_details": {
+            "cached_tokens": 0
+          },
+          "completion_tokens_details": {
+            "reasoning_tokens": 0,
+            "accepted_prediction_tokens": 0,
+            "rejected_prediction_tokens": 0
+          }
+        },
+        "system_fingerprint": "fp_6b68a8204b"
+      })";
+
+}  // namespace config
diff --git a/engine/controllers/models.cc b/engine/controllers/models.cc
@@ -200,7 +200,7 @@ void Models::ListModel(
                 .string());
         auto model_config = yaml_handler.GetModelConfig();
 
-        if (!remote_engine::IsRemoteEngine(model_config.engine)) {
+        if (!engine_service_->IsRemoteEngine(model_config.engine)) {
           Json::Value obj = model_config.ToJson();
           obj["id"] = model_entry.model;
           obj["model"] = model_entry.model;
@@ -632,7 +632,7 @@ void Models::GetRemoteModels(
     const HttpRequestPtr& req,
     std::function<void(const HttpResponsePtr&)>&& callback,
     const std::string& engine_id) {
-  if (!remote_engine::IsRemoteEngine(engine_id)) {
+  if (!engine_service_->IsRemoteEngine(engine_id)) {
     Json::Value ret;
     ret["message"] = "Not a remote engine: " + engine_id;
     auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret);
@@ -668,8 +668,7 @@ void Models::AddRemoteModel(
 
   auto model_handle = (*(req->getJsonObject())).get("model", "").asString();
   auto engine_name = (*(req->getJsonObject())).get("engine", "").asString();
-  /* To do: uncomment when remote engine is ready
-  
+
   auto engine_validate = engine_service_->IsEngineReady(engine_name);
   if (engine_validate.has_error()) {
     Json::Value ret;
@@ -679,6 +678,7 @@ void Models::AddRemoteModel(
     callback(resp);
     return;
   }
+
   if (!engine_validate.value()) {
     Json::Value ret;
     ret["message"] = "Engine is not ready! Please install first!";
@@ -687,7 +687,7 @@ void Models::AddRemoteModel(
     callback(resp);
     return;
   }
-  */
+
   config::RemoteModelConfig model_config;
   model_config.LoadFromJson(*(req->getJsonObject()));
   cortex::db::Models modellist_utils_obj;

diff --git a/engine/extensions/remote-engine/anthropic_engine.cc b/engine/extensions/remote-engine/anthropic_engine.cc
diff --git a/engine/extensions/remote-engine/anthropic_engine.h b/engine/extensions/remote-engine/anthropic_engine.h
diff --git a/engine/extensions/remote-engine/openai_engine.cc b/engine/extensions/remote-engine/openai_engine.cc
diff --git a/engine/extensions/remote-engine/openai_engine.h b/engine/extensions/remote-engine/openai_engine.h