diff --git a/cortex-cpp/common/base.h b/cortex-cpp/common/base.h index 43d612c1b..c6e3aa6a9 100644 --- a/cortex-cpp/common/base.h +++ b/cortex-cpp/common/base.h @@ -16,6 +16,9 @@ class BaseModel { virtual void ModelStatus( const HttpRequestPtr& req, std::function&& callback) = 0; + + virtual void GetModels(const HttpRequestPtr& req, + std::function&& callback) = 0; }; class BaseChatCompletion { diff --git a/cortex-cpp/controllers/server.cc b/cortex-cpp/controllers/server.cc index 143632a3c..1fdb74509 100644 --- a/cortex-cpp/controllers/server.cc +++ b/cortex-cpp/controllers/server.cc @@ -129,6 +129,40 @@ void server::ModelStatus( LOG_TRACE << "Done get model status"; } +void server::GetModels(const HttpRequestPtr& req, + std::function&& callback) { + if (!IsEngineLoaded()) { + Json::Value res; + res["message"] = "Engine is not loaded yet"; + auto resp = cortex_utils::nitroHttpJsonResponse(res); + resp->setStatusCode(k409Conflict); + callback(resp); + LOG_WARN << "Engine is not loaded yet"; + return; + } + + LOG_TRACE << "Start to get models"; + if (engine_->IsSupported("GetModels")) { + engine_->GetModels( + req->getJsonObject(), + [cb = std::move(callback)](Json::Value status, Json::Value res) { + auto resp = cortex_utils::nitroHttpJsonResponse(res); + resp->setStatusCode(static_cast( + status["status_code"].asInt())); + cb(resp); + }); + } else { + Json::Value res; + res["message"] = "Method is not supported yet"; + auto resp = cortex_utils::nitroHttpJsonResponse(res); + resp->setStatusCode(k500InternalServerError); + callback(resp); + LOG_WARN << "Method is not supported yet"; + } + + LOG_TRACE << "Done get models"; +} + void server::LoadModel(const HttpRequestPtr& req, std::function&& callback) { auto engine_type = @@ -144,10 +178,9 @@ void server::LoadModel(const HttpRequestPtr& req, }; try { - std::string abs_path = cortex_utils::GetCurrentPath() + - get_engine_path(cur_engine_name_); - dylib_ = - std::make_unique(abs_path, "engine"); + std::string abs_path = + cortex_utils::GetCurrentPath() + get_engine_path(cur_engine_name_); + dylib_ = std::make_unique(abs_path, "engine"); } catch (const cortex_cpp::dylib::load_error& e) { LOG_ERROR << "Could not load engine: " << e.what(); dylib_.reset(); diff --git a/cortex-cpp/controllers/server.h b/cortex-cpp/controllers/server.h index ba7bbb97b..c6d67ee30 100644 --- a/cortex-cpp/controllers/server.h +++ b/cortex-cpp/controllers/server.h @@ -46,9 +46,12 @@ class server : public drogon::HttpController, METHOD_ADD(server::LoadModel, "loadmodel", Post); METHOD_ADD(server::UnloadModel, "unloadmodel", Post); METHOD_ADD(server::ModelStatus, "modelstatus", Post); + METHOD_ADD(server::GetModels, "models", Get); + // Openai compatible path ADD_METHOD_TO(server::ChatCompletion, "/v1/chat/completions", Post); + ADD_METHOD_TO(server::GetModels, "/v1/models", Get); // ADD_METHOD_TO(server::handlePrelight, "/v1/chat/completions", Options); // NOTE: prelight will be added back when browser support is properly planned @@ -72,6 +75,9 @@ class server : public drogon::HttpController, void ModelStatus( const HttpRequestPtr& req, std::function&& callback) override; + void GetModels( + const HttpRequestPtr& req, + std::function&& callback) override; private: void ProcessStreamRes(std::function cb, diff --git a/cortex-cpp/cortex-common/EngineI.h b/cortex-cpp/cortex-common/EngineI.h index b8770b230..4246c8ade 100644 --- a/cortex-cpp/cortex-common/EngineI.h +++ b/cortex-cpp/cortex-common/EngineI.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include "json/value.h" @@ -24,4 +25,12 @@ class EngineI { virtual void GetModelStatus( std::shared_ptr jsonBody, std::function&& callback) = 0; + + // For backward compatible checking + virtual bool IsSupported(const std::string& f) = 0; + + // Get list of running models + virtual void GetModels( + std::shared_ptr jsonBody, + std::function&& callback) = 0; };