feat: cortex-cpp node addon (#852)

.github/workflows/cortex-build.yml

@@ -45,25 +45,29 @@ jobs:
             runs-on: "ubuntu-20-04"
             cmake-flags: ""
             build-deps-cmake-flags: ""
-            ccache-dir: ''
+            arch: "x64"
+            platform: "linux"
           - os: "mac"
             name: "amd64"
             runs-on: "macos-13"
             cmake-flags: ""
             build-deps-cmake-flags: ""
-            ccache-dir: ''
+            arch: "x64"
+            platform: "darwin"
           - os: "mac"
             name: "arm64"
             runs-on: "macos-latest"
             cmake-flags: "-DMAC_ARM64=ON"
             build-deps-cmake-flags: ""
-            ccache-dir: ''
+            arch: "arm64"
+            platform: "darwin"
           - os: "windows"
             name: "amd64"
             runs-on: "windows-cuda-12-0"
             cmake-flags: "-DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache -GNinja"
             build-deps-cmake-flags: "-DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache -GNinja"
-            ccache-dir: 'C:\Users\ContainerAdministrator\AppData\Local\ccache'
+            arch: "x64"
+            platform: "win32"
 
     steps:
       - name: Clone
@@ -77,6 +81,11 @@ jobs:
         with:
           dotnet-version: "8.0.x"
 
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "20.x"
+          registry-url: "https://registry.npmjs.org"
+
       - name: Install choco on Windows
         if: runner.os == 'Windows'
         run: |
@@ -116,7 +125,7 @@ jobs:
         run: |
           cd cortex-cpp
           make pre-package
-          
+
       - name: Code Signing macOS
         if: runner.os == 'macOS'
         run: |
@@ -163,6 +172,50 @@ jobs:
           AWS_SECRET_ACCESS_KEY: "${{ secrets.MINIO_SECRET_ACCESS_KEY }}"
           AWS_DEFAULT_REGION: "${{ secrets.MINIO_REGION }}"
 
+      ## cortex-cpp node binding
+
+      # update version in package.json
+      - name: Install jq
+        uses: dcarbone/[email protected]
+
+      - name: "Update version by tag"
+        working-directory: cortex-cpp
+        run: |
+          echo "Version: ${{ needs.create-draft-release.outputs.version }}"
+          # Update the version in package.json
+          jq --arg version "${{ needs.create-draft-release.outputs.version }}" '.version = $version' package.json > package-tmp.json
+          rm package.json
+          mv package-tmp.json package.json
+
+      # build prebuilds
+      - name: Build Prebuilds
+        working-directory: cortex-cpp
+        run: |
+          npm install -g yarn
+          yarn && yarn prebuild
+
+      # upload prebuilds
+      - name: Upload Prebuilds Darwin
+        uses: actions/[email protected]
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ needs.create-draft-release.outputs.upload_url }}
+          asset_path: ./cortex-cpp/prebuilds/cortex-cpp-v${{ needs.create-draft-release.outputs.version }}-napi-v8-${{matrix.platform}}-${{ matrix.arch }}.tar.gz
+          asset_name: cortex-cpp-v${{ needs.create-draft-release.outputs.version }}-napi-v8-${{matrix.platform}}-${{ matrix.arch }}.tar.gz
+          asset_content_type: application/gzip
+
+      # Setup .npmrc file to publish to npm - upload only once
+      - run: npm publish --access public
+        continue-on-error: true
+        if: runner.os == 'linux'
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+        working-directory: ./cortex-cpp
+
+      ## cortex-cpp node binding
+
   build-cortex-single-binary:
     runs-on: ${{ matrix.runs-on }}
     needs: [create-draft-release]
@@ -229,7 +282,7 @@ jobs:
         with:
           python-version: "3.10"
 
-      - run: pip3 install --upgrade setuptools 
+      - run: pip3 install --upgrade setuptools
         if: runner.os == 'macOS'
 
       - run: yarn install && yarn build:binary

cortex-cpp/CMakeLists.txt

@@ -1,6 +1,12 @@
 cmake_minimum_required(VERSION 3.5)
+
 project(cortex-cpp C CXX)
 
+# Build using CMAKE-JS
+if(DEFINED CMAKE_JS_INC)
+  include_directories(${CMAKE_JS_INC})
+endif()
+
 include(CheckIncludeFileCXX)
 
 check_include_file_cxx(any HAS_ANY)
@@ -53,13 +59,29 @@ if(APPLE)
   endif()
 endif()
 
+if(DEFINED CMAKE_JS_INC)
+  # define NPI_VERSION
+  add_compile_definitions(NAPI_VERSION=8)
+endif()
+
 add_compile_definitions(CORTEX_CPP_VERSION="${CORTEX_CPP_VERSION}")
 
 # add_subdirectory(test)
 
-add_executable(${PROJECT_NAME} main.cc
-  ${CMAKE_CURRENT_SOURCE_DIR}/utils/cpuid/cpu_info.cc
-)
+# Build using CMAKE-JS
+if(DEFINED CMAKE_JS_INC)
+  if(("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") OR("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU"))
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14")
+  endif()
+
+  add_library(${PROJECT_NAME} SHARED addon.cc
+    ${CMAKE_CURRENT_SOURCE_DIR}/utils/cpuid/cpu_info.cc
+  )
+else() # Official build
+  add_executable(${PROJECT_NAME} main.cc
+    ${CMAKE_CURRENT_SOURCE_DIR}/utils/cpuid/cpu_info.cc
+  )
+endif()
 
 # ##############################################################################
 # If you include the drogon source code locally in your project, use this method
@@ -69,9 +91,23 @@ add_executable(${PROJECT_NAME} main.cc
 # and comment out the following lines
 
 find_package(Drogon CONFIG REQUIRED)
-target_link_libraries(${PROJECT_NAME} PRIVATE Drogon::Drogon
-                                              ${CMAKE_THREAD_LIBS_INIT})
 
+
+# Build using CMAKE-JS
+if(DEFINED CMAKE_JS_INC)
+  set_target_properties(${PROJECT_NAME} PROPERTIES PREFIX "" SUFFIX ".node")
+
+  target_link_libraries(${PROJECT_NAME} PRIVATE Drogon::Drogon
+  ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_JS_LIB})
+
+  if(MSVC AND CMAKE_JS_NODELIB_DEF AND CMAKE_JS_NODELIB_TARGET)
+    # Generate node.lib
+    execute_process(COMMAND ${CMAKE_AR} /def:${CMAKE_JS_NODELIB_DEF} /out:${CMAKE_JS_NODELIB_TARGET} ${CMAKE_STATIC_LINKER_FLAGS})
+  endif()
+else()
+  target_link_libraries(${PROJECT_NAME} PRIVATE Drogon::Drogon
+  ${CMAKE_THREAD_LIBS_INIT})
+endif()
 # ##############################################################################
 
 if(CMAKE_CXX_STANDARD LESS 17)
@@ -103,4 +139,4 @@ target_sources(${PROJECT_NAME} PRIVATE ${CTL_SRC} ${COMMON_SRC})
 # ${FILTER_SRC} ${PLUGIN_SRC} ${MODEL_SRC})
 # ##############################################################################
 # uncomment the following line for dynamically loading views set_property(TARGET
-# ${PROJECT_NAME} PROPERTY ENABLE_EXPORTS ON)
+# ${PROJECT_NAME} PROPERTY ENABLE_EXPORTS ON)

cortex-cpp/addon.cc

@@ -0,0 +1,88 @@
+#include <napi.h>
+
+#include <drogon/HttpAppFramework.h>
+#include <drogon/drogon.h>
+#include <stdlib.h>
+#include <climits>  // for PATH_MAX
+#include <iostream>
+#include "cortex-common/cortexpythoni.h"
+#include "utils/cortex_utils.h"
+#include "utils/dylib.h"
+
+#if defined(__APPLE__) && defined(__MACH__)
+#include <libgen.h>  // for dirname()
+#include <mach-o/dyld.h>
+#elif defined(__linux__)
+#include <libgen.h>  // for dirname()
+#include <unistd.h>  // for readlink()
+#elif defined(_WIN32)
+#include <windows.h>
+#undef max
+#else
+#error "Unsupported platform!"
+#endif
+
+static Napi::Env* s_env = nullptr;
+
+void start() {
+  int thread_num = 1;
+  std::string host = "127.0.0.1";
+  int port = 3929;
+  std::string uploads_folder_path;
+  int logical_cores = std::thread::hardware_concurrency();
+  int drogon_thread_num = std::max(thread_num, logical_cores);
+#ifdef CORTEX_CPP_VERSION
+  LOG_INFO << "cortex-cpp version: " << CORTEX_CPP_VERSION;
+#else
+  LOG_INFO << "cortex-cpp version: undefined";
+#endif
+#ifdef CORTEX_LLAMACPP_VERSION
+  LOG_INFO << "cortex.llamacpp version: " << CORTEX_LLAMACPP_VERSION;
+#endif
+
+  LOG_INFO << "Server started, listening at: " << host << ":" << port;
+  LOG_INFO << "Please load your model";
+  drogon::app().addListener(host, port);
+  drogon::app().setThreadNum(drogon_thread_num);
+  if (!uploads_folder_path.empty()) {
+    LOG_INFO << "Drogon uploads folder is at: " << uploads_folder_path;
+    drogon::app().setUploadPath(uploads_folder_path);
+  }
+  LOG_INFO << "Number of thread is:" << drogon::app().getThreadNum();
+
+  drogon::app().run();
+}
+
+void stop() {
+  drogon::app().quit();
+}
+
+void exitCallback() {
+  Napi::TypeError::New(*s_env, "Process Exited!").ThrowAsJavaScriptException();
+}
+
+Napi::Value Start(const Napi::CallbackInfo& info) {
+  Napi::Env env = info.Env();
+
+  s_env = &env;
+
+  // Register exitCallback with atexit
+  std::atexit(exitCallback);
+
+  start();
+  return env.Undefined();
+}
+
+Napi::Value Stop(const Napi::CallbackInfo& info) {
+  Napi::Env env = info.Env();
+  stop();
+  return Napi::String::New(env, "Server stopped successfully");
+}
+
+Napi::Object Init(Napi::Env env, Napi::Object exports) {
+  exports.Set(Napi::String::New(env, "start"), Napi::Function::New(env, Start));
+  exports.Set(Napi::String::New(env, "stop"), Napi::Function::New(env, Start));
+  return exports;
+}
+
+NODE_API_MODULE(cortex-cpp, Init)

cortex-cpp/binding/index.d.ts

@@ -0,0 +1,7 @@
+// Type definitions for cortex-cpp node binding
+
+/// <reference types="node" />
+declare module "cortex-cpp" {
+  export function start();
+  export function stop();
+}

cortex-cpp/binding/index.js

@@ -0,0 +1,3 @@
+const addon = require("./../build/Release/cortex-cpp.node");
+
+module.exports = addon;

cortex-cpp/controllers/server.cc

@@ -206,7 +206,9 @@ void server::FineTuning(
   if (engines_.find(engine_type) == engines_.end()) {
     try {
       std::string abs_path =
-          cortex_utils::GetCurrentPath() + cortex_utils::kPythonRuntimeLibPath;
+          (getenv("ENGINE_PATH") ? getenv("ENGINE_PATH")
+                                 : cortex_utils::GetCurrentPath()) +
+          cortex_utils::kPythonRuntimeLibPath;
       engines_[engine_type].dl =
           std::make_unique<cortex_cpp::dylib>(abs_path, "engine");
     } catch (const cortex_cpp::dylib::load_error& e) {
@@ -262,9 +264,9 @@ void server::LoadModel(const HttpRequestPtr& req,
     auto get_engine_path = [](std::string_view e) {
       if (e == kLlamaEngine) {
         return cortex_utils::kLlamaLibPath;
-      } else if(e == kOnnxEngine) {
+      } else if (e == kOnnxEngine) {
         return cortex_utils::kOnnxLibPath;
-      } else if(e == kTensorrtLlmEngine) {
+      } else if (e == kTensorrtLlmEngine) {
         return cortex_utils::kTensorrtLlmPath;
       }
       return cortex_utils::kLlamaLibPath;
@@ -277,7 +279,9 @@ void server::LoadModel(const HttpRequestPtr& req,
       }
 
       std::string abs_path =
-          cortex_utils::GetCurrentPath() + get_engine_path(engine_type);
+          (getenv("ENGINE_PATH") ? getenv("ENGINE_PATH")
+                                 : cortex_utils::GetCurrentPath()) +
+          get_engine_path(engine_type);
       engines_[engine_type].dl =
           std::make_unique<cortex_cpp::dylib>(abs_path, "engine");
 

cortex-cpp/cortex-cpp-deps/CMakeLists.txt

@@ -62,9 +62,11 @@ ExternalProject_Add(
     GIT_TAG cares-1_26_0
     CMAKE_ARGS
     	-DCARES_SHARED=OFF
-	-DCARES_STATIC=ON
+	    -DCARES_STATIC=ON
+        -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
     	-DCMAKE_INSTALL_PREFIX=${THIRD_PARTY_INSTALL_PATH}
         -DCMAKE_BUILD_TYPE=RELEASE
+        -DCARES_STATIC_PIC=ON
 )
 
 ExternalProject_Add(

cortex-cpp/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "cortex-cpp",
+  "version": "0.0.11",
+  "description": "Cortex-cpp is a streamlined, stateless C++ server engineered to be fully compatible with OpenAI's API, particularly its stateless functionalities",
+  "main": "./binding/index.js",
+  "types": "./binding/index.d.ts",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/janhq/cortex.git"
+  },
+  "scripts": {
+    "install": "prebuild-install --runtime napi --backend cmake-js --config Release || cmake-js rebuild --config Release",
+    "build": "cmake-js configure --config Release && cmake-js build --config Release",
+    "rebuild": "cmake-js rebuild --config Release",
+    "prebuild": "prebuild --runtime napi --backend cmake-js --all --strip --verbose --config Release",
+    "upload": "prebuild --runtime napi --backend cmake-js --upload ${GITHUB_TOKEN}"
+  },
+  "author": "Jan <[email protected]>",
+  "license": "Apache-2.0",
+  "gypfile": true,
+  "dependencies": {
+    "bindings": "^1.5.0",
+    "cmake-js": "^7.3.0",
+    "node-addon-api": "^7.0.0",
+    "prebuild": "^13.0.1",
+    "prebuild-install": "^7.1.2"
+  },
+  "devDependencies": {
+    "@types/node": "^20.14.9",
+    "typescript": "^5.5.3"
+  },
+  "binary": {
+    "napi_versions": [
+      8
+    ]
+  },
+  "files": [
+    "binding/*.js",
+    "binding/*.d.ts"
+  ]
+}