From 4e6ff38495e42eaa3fe8f5e40d53ccd1ac727f9e Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Tue, 21 May 2024 14:19:09 +0700 Subject: [PATCH 1/4] feat: get running models --- cortex-cpp/common/base.h | 3 +++ cortex-cpp/controllers/server.cc | 41 +++++++++++++++++++++++++++--- cortex-cpp/controllers/server.h | 6 +++++ cortex-cpp/cortex-common/EngineI.h | 9 +++++++ 4 files changed, 55 insertions(+), 4 deletions(-) diff --git a/cortex-cpp/common/base.h b/cortex-cpp/common/base.h index 43d612c1b..c6e3aa6a9 100644 --- a/cortex-cpp/common/base.h +++ b/cortex-cpp/common/base.h @@ -16,6 +16,9 @@ class BaseModel { virtual void ModelStatus( const HttpRequestPtr& req, std::function&& callback) = 0; + + virtual void GetModels(const HttpRequestPtr& req, + std::function&& callback) = 0; }; class BaseChatCompletion { diff --git a/cortex-cpp/controllers/server.cc b/cortex-cpp/controllers/server.cc index 143632a3c..1fdb74509 100644 --- a/cortex-cpp/controllers/server.cc +++ b/cortex-cpp/controllers/server.cc @@ -129,6 +129,40 @@ void server::ModelStatus( LOG_TRACE << "Done get model status"; } +void server::GetModels(const HttpRequestPtr& req, + std::function&& callback) { + if (!IsEngineLoaded()) { + Json::Value res; + res["message"] = "Engine is not loaded yet"; + auto resp = cortex_utils::nitroHttpJsonResponse(res); + resp->setStatusCode(k409Conflict); + callback(resp); + LOG_WARN << "Engine is not loaded yet"; + return; + } + + LOG_TRACE << "Start to get models"; + if (engine_->IsSupported("GetModels")) { + engine_->GetModels( + req->getJsonObject(), + [cb = std::move(callback)](Json::Value status, Json::Value res) { + auto resp = cortex_utils::nitroHttpJsonResponse(res); + resp->setStatusCode(static_cast( + status["status_code"].asInt())); + cb(resp); + }); + } else { + Json::Value res; + res["message"] = "Method is not supported yet"; + auto resp = cortex_utils::nitroHttpJsonResponse(res); + resp->setStatusCode(k500InternalServerError); + callback(resp); + LOG_WARN << "Method is not supported yet"; + } + + LOG_TRACE << "Done get models"; +} + void server::LoadModel(const HttpRequestPtr& req, std::function&& callback) { auto engine_type = @@ -144,10 +178,9 @@ void server::LoadModel(const HttpRequestPtr& req, }; try { - std::string abs_path = cortex_utils::GetCurrentPath() + - get_engine_path(cur_engine_name_); - dylib_ = - std::make_unique(abs_path, "engine"); + std::string abs_path = + cortex_utils::GetCurrentPath() + get_engine_path(cur_engine_name_); + dylib_ = std::make_unique(abs_path, "engine"); } catch (const cortex_cpp::dylib::load_error& e) { LOG_ERROR << "Could not load engine: " << e.what(); dylib_.reset(); diff --git a/cortex-cpp/controllers/server.h b/cortex-cpp/controllers/server.h index ba7bbb97b..c6d67ee30 100644 --- a/cortex-cpp/controllers/server.h +++ b/cortex-cpp/controllers/server.h @@ -46,9 +46,12 @@ class server : public drogon::HttpController, METHOD_ADD(server::LoadModel, "loadmodel", Post); METHOD_ADD(server::UnloadModel, "unloadmodel", Post); METHOD_ADD(server::ModelStatus, "modelstatus", Post); + METHOD_ADD(server::GetModels, "models", Get); + // Openai compatible path ADD_METHOD_TO(server::ChatCompletion, "/v1/chat/completions", Post); + ADD_METHOD_TO(server::GetModels, "/v1/models", Get); // ADD_METHOD_TO(server::handlePrelight, "/v1/chat/completions", Options); // NOTE: prelight will be added back when browser support is properly planned @@ -72,6 +75,9 @@ class server : public drogon::HttpController, void ModelStatus( const HttpRequestPtr& req, std::function&& callback) override; + void GetModels( + const HttpRequestPtr& req, + std::function&& callback) override; private: void ProcessStreamRes(std::function cb, diff --git a/cortex-cpp/cortex-common/EngineI.h b/cortex-cpp/cortex-common/EngineI.h index b8770b230..a9216b9c2 100644 --- a/cortex-cpp/cortex-common/EngineI.h +++ b/cortex-cpp/cortex-common/EngineI.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include "json/value.h" @@ -24,4 +25,12 @@ class EngineI { virtual void GetModelStatus( std::shared_ptr jsonBody, std::function&& callback) = 0; + + // Get list of running models + virtual void GetModels( + std::shared_ptr jsonBody, + std::function&& callback) = 0; + + // For backward compatible checking + virtual bool IsSupported(const std::string& f) = 0; }; From 0ccb2b029f6480587d039201cf411a6d33e91e84 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Wed, 22 May 2024 08:34:22 +0700 Subject: [PATCH 2/4] fix: reorder func --- cortex-cpp/cortex-common/EngineI.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cortex-cpp/cortex-common/EngineI.h b/cortex-cpp/cortex-common/EngineI.h index a9216b9c2..3c7ceb68b 100644 --- a/cortex-cpp/cortex-common/EngineI.h +++ b/cortex-cpp/cortex-common/EngineI.h @@ -10,6 +10,9 @@ class EngineI { public: virtual ~EngineI() {} + // For backward compatible checking + virtual bool IsSupported(const std::string& f) = 0; + virtual void HandleChatCompletion( std::shared_ptr jsonBody, std::function&& callback) = 0; @@ -30,7 +33,4 @@ class EngineI { virtual void GetModels( std::shared_ptr jsonBody, std::function&& callback) = 0; - - // For backward compatible checking - virtual bool IsSupported(const std::string& f) = 0; }; From 559809e90dc28b4948516f04615a6a88eab883a9 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Wed, 22 May 2024 08:41:23 +0700 Subject: [PATCH 3/4] fix: reorder func --- cortex-cpp/cortex-common/EngineI.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cortex-cpp/cortex-common/EngineI.h b/cortex-cpp/cortex-common/EngineI.h index 3c7ceb68b..4246c8ade 100644 --- a/cortex-cpp/cortex-common/EngineI.h +++ b/cortex-cpp/cortex-common/EngineI.h @@ -10,9 +10,6 @@ class EngineI { public: virtual ~EngineI() {} - // For backward compatible checking - virtual bool IsSupported(const std::string& f) = 0; - virtual void HandleChatCompletion( std::shared_ptr jsonBody, std::function&& callback) = 0; @@ -29,6 +26,9 @@ class EngineI { std::shared_ptr jsonBody, std::function&& callback) = 0; + // For backward compatible checking + virtual bool IsSupported(const std::string& f) = 0; + // Get list of running models virtual void GetModels( std::shared_ptr jsonBody, From 2a4a4f3421b8b0dd545bd27721ecff5abb53120a Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Thu, 23 May 2024 11:58:56 +0700 Subject: [PATCH 4/4] chore: cortex.llamacpp to 0.1.9 --- cortex-cpp/engines/cortex.llamacpp/engine.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cortex-cpp/engines/cortex.llamacpp/engine.cmake b/cortex-cpp/engines/cortex.llamacpp/engine.cmake index 7bb298904..0ecc3e8f2 100644 --- a/cortex-cpp/engines/cortex.llamacpp/engine.cmake +++ b/cortex-cpp/engines/cortex.llamacpp/engine.cmake @@ -1,5 +1,5 @@ # cortex.llamacpp release version -set(VERSION 0.1.8) +set(VERSION 0.1.9) set(ENGINE_VERSION v${VERSION}) add_compile_definitions(CORTEX_LLAMACPP_VERSION="${VERSION}")