update cortex-cpp run path and engines directory - support vulkan

janhq · Jul 9, 2024 · 7c01b8a · 7c01b8a
1 parent e976635
commit 7c01b8a
Show file tree

Hide file tree

Showing 10 changed files with 110 additions and 183 deletions.
diff --git a/cortex-cpp/addon.cc b/cortex-cpp/addon.cc
@@ -7,6 +7,7 @@
 #include "cortex-common/cortexpythoni.h"
 #include "utils/cortex_utils.h"
 #include "utils/dylib.h"
+#include <stdlib.h>
 
 #if defined(__APPLE__) && defined(__MACH__)
 #include <libgen.h>  // for dirname()
@@ -59,6 +60,19 @@ void stop() {
 Napi::Value Start(const Napi::CallbackInfo &info)
 {
   Napi::Env env = info.Env();
+
+  LOG_INFO << "set env";
+  Napi::Object objs = info[0].As<Napi::Value>().ToObject();
+  Napi::Array props = objs.GetPropertyNames();
+
+  for (unsigned int i = 0; i < props.Length(); i++)
+  {
+      Napi::Value key = props.Get(i);
+      LOG_INFO << key.ToString().Utf8Value();
+      LOG_INFO << objs.Get(key).ToString().Utf8Value();
+      setenv(key.ToString().Utf8Value().c_str(), objs.Get(key).ToString().Utf8Value().c_str(), 1);
+  }
+
   start();
   return Napi::String::New(env, "Server started successfully");
 }

diff --git a/cortex-cpp/binding/index.d.ts b/cortex-cpp/binding/index.d.ts
@@ -2,6 +2,6 @@
 
 /// <reference types="node" />
 declare module "cortex-cpp" {
-  start();
-  stop();
+  export function start();
+  export function stop();
 }
diff --git a/cortex-cpp/controllers/server.cc b/cortex-cpp/controllers/server.cc
@@ -206,7 +206,9 @@ void server::FineTuning(
   if (engines_.find(engine_type) == engines_.end()) {
     try {
       std::string abs_path =
-          cortex_utils::GetCurrentPath() + cortex_utils::kPythonRuntimeLibPath;
+          (getenv("ENGINE_PATH") ? getenv("ENGINE_PATH")
+                                 : cortex_utils::GetCurrentPath()) +
+          cortex_utils::kPythonRuntimeLibPath;
       engines_[engine_type].dl =
           std::make_unique<cortex_cpp::dylib>(abs_path, "engine");
     } catch (const cortex_cpp::dylib::load_error& e) {
@@ -262,9 +264,9 @@ void server::LoadModel(const HttpRequestPtr& req,
     auto get_engine_path = [](std::string_view e) {
       if (e == kLlamaEngine) {
         return cortex_utils::kLlamaLibPath;
-      } else if(e == kOnnxEngine) {
+      } else if (e == kOnnxEngine) {
         return cortex_utils::kOnnxLibPath;
-      } else if(e == kTensorrtLlmEngine) {
+      } else if (e == kTensorrtLlmEngine) {
         return cortex_utils::kTensorrtLlmPath;
       }
       return cortex_utils::kLlamaLibPath;
@@ -277,7 +279,9 @@ void server::LoadModel(const HttpRequestPtr& req,
       }
 
       std::string abs_path =
-          cortex_utils::GetCurrentPath() + get_engine_path(engine_type);
+          (getenv("ENGINE_PATH") ? getenv("ENGINE_PATH")
+                                 : cortex_utils::GetCurrentPath()) +
+          get_engine_path(engine_type);
       engines_[engine_type].dl =
           std::make_unique<cortex_cpp::dylib>(abs_path, "engine");
 

diff --git a/cortex-js/package.json b/cortex-js/package.json
@@ -51,6 +51,7 @@
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.1",
     "cli-progress": "^3.12.0",
+    "cortex-cpp": "^0.4.23",
     "cortexso-node": "^0.0.4",
     "cpu-instructions": "^0.0.11",
     "decompress": "^4.2.1",

diff --git a/cortex-js/src/infrastructure/commanders/models/model-start.command.ts b/cortex-js/src/infrastructure/commanders/models/model-start.command.ts
@@ -75,9 +75,7 @@ export class ModelStartCommand extends CommandRunner {
     if (
       !existsSync(join(await this.fileService.getCortexCppEnginePath(), engine))
     ) {
-      const engineSpinner = ora('Installing engine...').start();
       await this.initUsecases.installEngine(undefined, 'latest', engine);
-      engineSpinner.succeed();
     }
 
     // Attached - stdout logs

diff --git a/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts b/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts
@@ -87,9 +87,7 @@ export class RunCommand extends CommandRunner {
     if (
       !existsSync(join(await this.fileService.getCortexCppEnginePath(), engine))
     ) {
-      const engineSpinner = ora('Installing engine...').start();
       await this.initUsecases.installEngine(undefined, 'latest', engine);
-      engineSpinner.succeed('Engine installed');
     }
 
     return this.cortexUsecases