diff --git a/engine/controllers/engines.cc b/engine/controllers/engines.cc index 46f35046f..3b3725089 100644 --- a/engine/controllers/engines.cc +++ b/engine/controllers/engines.cc @@ -289,11 +289,6 @@ void Engines::InstallRemoteEngine( resp->setStatusCode(k400BadRequest); callback(resp); } else { - // auto gr = engine_service_->GenerateRemoteModel(engine); - // if (gr.has_error()) { - // CTL_INF("Error: " << gr.error()); - // } - Json::Value res; if (get_models_url.empty()) { res["warning"] = diff --git a/engine/extensions/remote-engine/remote_engine.cc b/engine/extensions/remote-engine/remote_engine.cc index e60c9b947..e944bad6e 100644 --- a/engine/extensions/remote-engine/remote_engine.cc +++ b/engine/extensions/remote-engine/remote_engine.cc @@ -17,7 +17,8 @@ bool is_anthropic(const std::string& model) { } bool is_openai(const std::string& model) { - return model.find("gpt") != std::string::npos; + return model.find("gpt-") != std::string::npos || + model.find("o1-") != std::string::npos; } constexpr const std::array kAnthropicModels = { @@ -181,7 +182,7 @@ static size_t WriteCallback(char* ptr, size_t size, size_t nmemb, } RemoteEngine::RemoteEngine(const std::string& engine_name) - : engine_name_(engine_name) { + : engine_name_(engine_name), q_(1 /*n_parallel*/, engine_name) { curl_global_init(CURL_GLOBAL_ALL); } @@ -552,7 +553,9 @@ void RemoteEngine::HandleChatCompletion( } if (is_stream) { - MakeStreamingChatCompletionRequest(*model_config, result, callback); + q_.runTaskInQueue([this, model_config, result, cb = std::move(callback)] { + MakeStreamingChatCompletionRequest(*model_config, result, cb); + }); } else { auto response = MakeChatCompletionRequest(*model_config, result); diff --git a/engine/extensions/remote-engine/remote_engine.h b/engine/extensions/remote-engine/remote_engine.h index 8478af817..6f5259f39 100644 --- a/engine/extensions/remote-engine/remote_engine.h +++ b/engine/extensions/remote-engine/remote_engine.h @@ -9,6 +9,7 @@ #include #include "cortex-common/remote_enginei.h" #include "extensions/remote-engine/template_renderer.h" +#include "trantor/utils/ConcurrentTaskQueue.h" #include "utils/engine_constants.h" #include "utils/file_logger.h" // Helper for CURL response @@ -52,6 +53,7 @@ class RemoteEngine : public RemoteEngineI { std::string chat_res_template_; std::string api_key_header_; std::string engine_name_; + trantor::ConcurrentTaskQueue q_; // Helper functions CurlResponse MakeChatCompletionRequest(const ModelConfig& config, diff --git a/engine/services/engine_service.cc b/engine/services/engine_service.cc index 3bbb6df82..73212a048 100644 --- a/engine/services/engine_service.cc +++ b/engine/services/engine_service.cc @@ -1140,71 +1140,6 @@ bool EngineService::IsRemoteEngine(const std::string& engine_name) { return true; } -cpp::result EngineService::GenerateRemoteModel( - const std::string& engine_name) { - namespace fmu = file_manager_utils; - namespace fs = std::filesystem; - auto exist_engine = GetEngineByNameAndVariant(engine_name); - if (exist_engine.has_error()) { - return cpp::fail("Remote engine '" + engine_name + "' is not installed"); - } - - if (!IsEngineLoaded(engine_name)) { - engines_[engine_name].engine = new remote_engine::RemoteEngine(engine_name); - CTL_INF("Loaded engine: " << engine_name); - } - - auto remote_engine_json = exist_engine.value().ToJson(); - auto& e = std::get(engines_[engine_name].engine); - auto url = remote_engine_json["metadata"]["get_models_url"].asString(); - auto api_key = remote_engine_json["api_key"].asString(); - auto api_key_template = - remote_engine_json["metadata"]["api_key_template"].asString(); - auto res = e->GetRemoteModels(url, api_key, api_key_template); - if (!res["error"].isNull()) { - return cpp::fail(res["error"].asString()); - } else { - for (auto& d : res["data"]) { - auto model_handle = d["id"].asString(); - config::RemoteModelConfig model_config; - Json::Value body = - json_helper::ParseJsonString(config::kDefaultRemoteModelConfig); - body["model"] = model_handle; - body["engine"] = engine_name; - // CTL_INF(body.toStyledString()); - model_config.LoadFromJson(body); - cortex::db::Models modellist_utils_obj; - - std::string model_yaml_path = - (file_manager_utils::GetModelsContainerPath() / - std::filesystem::path("remote") / - std::filesystem::path(model_handle + ".yml")) - .string(); - try { - auto yaml_rel_path = - fmu::ToRelativeCortexDataPath(fs::path(model_yaml_path)); - cortex::db::ModelEntry model_entry{ - model_handle, "", "", yaml_rel_path.string(), - model_handle, "remote", "imported", cortex::db::ModelStatus::Remote, - engine_name}; - std::filesystem::create_directories( - std::filesystem::path(model_yaml_path).parent_path()); - if (modellist_utils_obj.AddModelEntry(model_entry).value()) { - model_config.SaveToYamlFile(model_yaml_path); - } else { - CTL_INF("Fail to import model, model_id '" + model_handle + - "' already exists!"); - } - } catch (const std::exception& e) { - return cpp::fail("Error while adding Remote model with model_id '" + - model_handle + "': " + e.what()); - } - } - } - - return true; -} - cpp::result, std::string> EngineService::GetSupportedEngineNames() { return file_manager_utils::GetCortexConfig().supportedEngines; diff --git a/engine/services/engine_service.h b/engine/services/engine_service.h index ca652a7bd..8ead4f6d6 100644 --- a/engine/services/engine_service.h +++ b/engine/services/engine_service.h @@ -152,9 +152,6 @@ class EngineService : public EngineServiceI { bool IsRemoteEngine(const std::string& engine_name) override; - cpp::result GenerateRemoteModel( - const std::string& engine_name); - private: bool IsEngineLoaded(const std::string& engine); diff --git a/engine/services/model_service.cc b/engine/services/model_service.cc index 0d909b61f..8c0d2cd66 100644 --- a/engine/services/model_service.cc +++ b/engine/services/model_service.cc @@ -1253,5 +1253,8 @@ ModelService::GetModelMetadata(const std::string& model_id) const { std::shared_ptr ModelService::GetCachedModelMetadata( const std::string& model_id) const { + if (loaded_model_metadata_map_.find(model_id) == + loaded_model_metadata_map_.end()) + return nullptr; return loaded_model_metadata_map_.at(model_id); }