Merge branch 'dev' of github.com:janhq/nitro into fix/change-to-abs-p…

…ath-for-engine-loading

.github/workflows/build.yml → .github/workflows/cortex-cpp-build.yml

@@ -1,4 +1,4 @@
-name: CI
+name: CI Cortex CPP
 
 on:
   push:
@@ -25,7 +25,8 @@ jobs:
     steps:
       - name: Extract tag name without v prefix
         id: get_version
-        run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV && echo "::set-output name=version::${GITHUB_REF#refs/tags/v}"
+        run: |
+          echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV && echo "::set-output name=version::${GITHUB_REF#refs/tags/v}"
         env:
           GITHUB_REF: ${{ github.ref }}
       - name: Create Draft Release
@@ -91,7 +92,7 @@ jobs:
 
           - os: "mac"
             name: "arm64"
-            runs-on: "mac-silicon"
+            runs-on: "macos-latest"
             cmake-flags: "-DMAC_ARM64=ON"
             run-e2e: true
 
@@ -161,16 +162,54 @@ jobs:
         with:
           submodules: recursive
 
+      - uses: actions/setup-dotnet@v3
+        if: runner.os == 'Windows'
+        with:
+          dotnet-version: "8.0.x"
+
       - name: Install choco on Windows
         if: runner.os == 'Windows'
         run: |
           choco install make -y
 
+      - name: Get Cer for code signing
+        if: runner.os == 'macOS'
+        run: base64 -d <<< "$CODE_SIGN_P12_BASE64" > /tmp/codesign.p12
+        shell: bash
+        env:
+          CODE_SIGN_P12_BASE64: ${{ secrets.CODE_SIGN_P12_BASE64 }}
+
+      - uses: apple-actions/import-codesign-certs@v2
+        if: runner.os == 'macOS'
+        with:
+          p12-file-base64: ${{ secrets.CODE_SIGN_P12_BASE64 }}
+          p12-password: ${{ secrets.CODE_SIGN_P12_PASSWORD }}
+
       - name: Build
         run: |
           cd cortex-cpp
           make build CMAKE_EXTRA_FLAGS="${{ matrix.cmake-flags }}"
 
+      - name: Pre-package
+        run: |
+          cd cortex-cpp
+          make pre-package
+
+      - name: Code Signing macOS
+        if: runner.os == 'macOS'
+        run: |
+          cd cortex-cpp
+          make codesign CODE_SIGN=true DEVELOPER_ID="${{ secrets.DEVELOPER_ID }}"
+
+      - name: Code Signing Windows
+        if: runner.os == 'Windows'
+        shell: cmd
+        run: |
+          cd cortex-cpp
+          set PATH=%PATH%;%USERPROFILE%\.dotnet\tools
+          make codesign CODE_SIGN=true AZURE_KEY_VAULT_URI="${{ secrets.AZURE_KEY_VAULT_URI }}" AZURE_CLIENT_ID="${{ secrets.AZURE_CLIENT_ID }}" AZURE_TENANT_ID="${{ secrets.AZURE_TENANT_ID }}" AZURE_CLIENT_SECRET="${{ secrets.AZURE_CLIENT_SECRET }}" AZURE_CERT_NAME="${{ secrets.AZURE_CERT_NAME }}"
+  
+
       - name: Package
         run: |
           cd cortex-cpp

.github/workflows/quality-gate.yml → .github/workflows/cortex-cpp-quality-gate.yml

@@ -1,4 +1,4 @@
-name: CI Quality Gate
+name: CI Quality Gate Cortex CPP
 
 on:
   pull_request:
@@ -145,6 +145,11 @@ jobs:
           cd cortex-cpp
           make build CMAKE_EXTRA_FLAGS="${{ matrix.cmake-flags }}"
 
+      - name: Pre-package
+        run: |
+          cd cortex-cpp
+          make pre-package
+ 
       - name: Package
         run: |
           cd cortex-cpp

.github/workflows/cortex-js.yml

@@ -0,0 +1,44 @@
+name: Publish cortex js Package to npmjs
+on:
+  push:
+    tags: ["v[0-9]+.[0-9]+.[0-9]+-cortex-js"]
+    paths:
+      [
+        "cortex-js/**",
+      ]
+jobs:
+  build-and-publish-plugins:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: "0"
+
+      - name: Install jq
+        uses: dcarbone/[email protected]
+
+      - name: "Update version by tag"
+        run: |
+          cd cortex-js
+          # Remove the v prefix
+          tag_version=${GITHUB_REF#refs/tags/v}
+          # Remove the -cortex-js suffix
+          new_version=${tag_version%-cortex-js}
+
+          # Replace the old version with the new version in package.json
+          jq --arg version "$new_version" '.version = $version' ./package.json > /tmp/package.json && mv /tmp/package.json ./package.json
+
+          # Print the new version
+          echo "Updated package.json version to: $new_version"
+
+      # Setup .npmrc file to publish to npm
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "20.x"
+          registry-url: "https://registry.npmjs.org"
+      - run: yarn install && yarn build
+        working-directory: ./cortex-js
+      - run: npm publish --access public
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+        working-directory: ./cortex-js

cortex-cpp/Makefile

@@ -6,6 +6,13 @@ CMAKE_EXTRA_FLAGS ?= ""
 RUN_TESTS ?= false
 LLM_MODEL_URL ?= "https://delta.jan.ai/tinyllama-1.1b-chat-v0.3.Q2_K.gguf"
 EMBEDDING_MODEL_URL ?= "https://catalog.jan.ai/dist/models/embeds/nomic-embed-text-v1.5.f16.gguf"
+CODE_SIGN ?= false
+AZURE_KEY_VAULT_URI ?= xxxx
+AZURE_CLIENT_ID ?= xxxx
+AZURE_TENANT_ID ?= xxxx
+AZURE_CLIENT_SECRET ?= xxxx
+AZURE_CERT_NAME ?= xxxx
+DEVELOPER_ID ?= xxxx
 
 # Default target, does nothing
 all:
@@ -29,24 +36,47 @@ else
 	make -j4;
 endif
 
-package:
+pre-package:
 ifeq ($(OS),Windows_NT)
-	@powershell -Command "mkdir -p cortex-cpp\engines\cortex.llamacpp\; cp build\engines\cortex.llamacpp\engine.dll cortex-cpp\engines\cortex.llamacpp\;"
-	@powershell -Command "cp build\Release\cortex-cpp.exe .\cortex-cpp\;"
-	@powershell -Command "cp build-deps\_install\bin\zlib.dll .\cortex-cpp\;"
-	@powershell -Command "cp ..\.github\patches\windows\msvcp140.dll .\cortex-cpp\;"
-	@powershell -Command "cp ..\.github\patches\windows\vcruntime140_1.dll .\cortex-cpp\;"
-	@powershell -Command "cp ..\.github\patches\windows\vcruntime140.dll .\cortex-cpp\;"
-	@powershell -Command "7z a -ttar temp.tar cortex-cpp\\*; 7z a -tgzip cortex-cpp.tar.gz temp.tar;"
+	@powershell -Command "mkdir -p cortex-cpp\engines\cortex.llamacpp\; cp -r build\engines\cortex.llamacpp\engine.dll cortex-cpp\engines\cortex.llamacpp\;"
+	@powershell -Command "cp -r build\Release\cortex-cpp.exe .\cortex-cpp\;"
+	@powershell -Command "cp -r build-deps\_install\bin\zlib.dll .\cortex-cpp\;"
+	@powershell -Command "cp -r ..\.github\patches\windows\msvcp140.dll .\cortex-cpp\;"
+	@powershell -Command "cp -r ..\.github\patches\windows\vcruntime140_1.dll .\cortex-cpp\;"
+	@powershell -Command "cp -r ..\.github\patches\windows\vcruntime140.dll .\cortex-cpp\;"
 else ifeq ($(shell uname -s),Linux)
 	@mkdir -p cortex-cpp/engines/cortex.llamacpp; \
 	cp build/engines/cortex.llamacpp/libengine.so cortex-cpp/engines/cortex.llamacpp/; \
-	cp build/cortex-cpp cortex-cpp/; \
-	tar -czvf cortex-cpp.tar.gz cortex-cpp;
+	cp build/cortex-cpp cortex-cpp/;
 else
 	@mkdir -p cortex-cpp/engines/cortex.llamacpp; \
 	cp build/engines/cortex.llamacpp/libengine.dylib cortex-cpp/engines/cortex.llamacpp/; \
-	cp build/cortex-cpp cortex-cpp/; \
+	cp build/cortex-cpp cortex-cpp/;
+endif
+
+codesign:
+ifeq ($(CODE_SIGN),false)
+	@echo "Skipping Code Sign"
+	@exit 0
+endif
+
+ifeq ($(OS),Windows_NT)
+	@powershell -Command "dotnet tool install --global AzureSignTool;"
+	@powershell -Command 'azuresigntool.exe sign -kvu "$(AZURE_KEY_VAULT_URI)" -kvi "$(AZURE_CLIENT_ID)" -kvt "$(AZURE_TENANT_ID)" -kvs "$(AZURE_CLIENT_SECRET)" -kvc "$(AZURE_CERT_NAME)" -tr http://timestamp.globalsign.com/tsa/r6advanced1 -v ".\cortex-cpp\cortex-cpp.exe";'
+	@powershell -Command 'azuresigntool.exe sign -kvu "$(AZURE_KEY_VAULT_URI)" -kvi "$(AZURE_CLIENT_ID)" -kvt "$(AZURE_TENANT_ID)" -kvs "$(AZURE_CLIENT_SECRET)" -kvc "$(AZURE_CERT_NAME)" -tr http://timestamp.globalsign.com/tsa/r6advanced1 -v ".\cortex-cpp\engines\cortex.llamacpp\engine.dll";'
+else ifeq ($(shell uname -s),Linux)
+	@echo "Skipping Code Sign for linux"
+	@exit 0
+else
+	find "cortex-cpp" -type f -exec codesign --force -s "$(DEVELOPER_ID)" --options=runtime {} \;
+endif
+
+package:
+ifeq ($(OS),Windows_NT)
+	@powershell -Command "7z a -ttar temp.tar cortex-cpp\*; 7z a -tgzip cortex-cpp.tar.gz temp.tar;"
+else ifeq ($(shell uname -s),Linux)
+	tar -czvf cortex-cpp.tar.gz cortex-cpp;
+else
 	tar -czvf cortex-cpp.tar.gz cortex-cpp;
 endif
 
@@ -65,4 +95,13 @@ else
 	@cd cortex-cpp; \
 	chmod +x ../../.github/scripts/e2e-test-llama-linux-and-mac.sh && ../../.github/scripts/e2e-test-llama-linux-and-mac.sh ./cortex-cpp $(LLM_MODEL_URL) $(EMBEDDING_MODEL_URL); \
 	rm -rf uploads/;
+endif
+
+clean:
+ifeq ($(OS),Windows_NT)
+	@powershell -Command "rm -rf build; rm -rf build-deps; rm -rf cortex-cpp; rm -rf cortex-cpp.tar.gz;"
+else ifeq ($(shell uname -s),Linux)
+	@rm -rf build; rm -rf build-deps; rm -rf cortex-cpp; rm -rf cortex-cpp.tar.gz;
+else
+	@rm -rf build; rm -rf build-deps; rm -rf cortex-cpp; rm -rf cortex-cpp.tar.gz;
 endif

cortex-cpp/engines/cortex.llamacpp/engine.cmake

@@ -1,6 +1,7 @@
 # cortex.llamacpp release version
-set(VERSION 0.1.2)
+set(VERSION 0.1.4)
 set(ENGINE_VERSION v${VERSION})
+add_compile_definitions(CORTEX_LLAMACPP_VERSION="${VERSION}")
 
 # MESSAGE("ENGINE_VERSION=" ${ENGINE_VERSION})
 

cortex-cpp/main.cc

@@ -51,6 +51,10 @@ int main(int argc, char* argv[]) {
 #else
   LOG_INFO << "cortex-cpp version: undefined";
 #endif
+#ifdef CORTEX_LLAMACPP_VERSION
+  LOG_INFO << "cortex.llamacpp version: " << CORTEX_LLAMACPP_VERSION;
+#endif
+
   LOG_INFO << "Server started, listening at: " << host << ":" << port;
   LOG_INFO << "Please load your model";
   drogon::app().addListener(host, port);

cortex-js/.env.example

@@ -1,3 +1,2 @@
 EXTENSIONS_PATH=<EXTENSIONS_PATH>
 CORTEX_MODELS_DIR=<CORTEX_MODELS_DIR>
-CORTEX_BINARY_PATH=<CORTEX_BINARY_PATH>

cortex-js/constant.ts

@@ -4,3 +4,6 @@ export const databaseFile = `${databaseName}.db`;
 
 export const defaultCortexJsHost = 'localhost';
 export const defaultCortexJsPort = 7331;
+
+export const defaultCortexCppHost = '127.0.0.1';
+export const defaultCortexCppPort = 3928;

cortex-js/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "cortex-js",
+  "name": "@janhq/cortex",
   "version": "0.0.1",
   "description": "",
   "author": "",
@@ -25,6 +25,7 @@
     "typeorm": "typeorm-ts-node-esm"
   },
   "dependencies": {
+    "@huggingface/gguf": "^0.1.5",
     "@nestjs/axios": "^3.0.2",
     "@nestjs/common": "^10.0.0",
     "@nestjs/config": "^3.2.2",
@@ -33,10 +34,12 @@
     "@nestjs/mapped-types": "*",
     "@nestjs/platform-express": "^10.0.0",
     "@nestjs/swagger": "^7.3.1",
+    "@terascope/fetch-github-release": "^0.8.8",
     "axios": "^1.6.8",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.1",
     "cli-progress": "^3.12.0",
+    "decompress": "^4.2.1",
     "nest-commander": "^3.13.0",
     "readline": "^1.3.0",
     "reflect-metadata": "^0.2.0",
@@ -52,6 +55,7 @@
     "@nestjs/testing": "^10.0.0",
     "@nestjs/typeorm": "^10.0.2",
     "@types/cli-progress": "^3.11.5",
+    "@types/decompress": "^4.2.7",
     "@types/express": "^4.17.17",
     "@types/jest": "^29.5.2",
     "@types/node": "^20.12.9",

cortex-js/src/app.module.ts

@@ -6,28 +6,26 @@ import { DevtoolsModule } from '@nestjs/devtools-integration';
 import { DatabaseModule } from './infrastructure/database/database.module';
 import { ChatModule } from './usecases/chat/chat.module';
 import { AssistantsModule } from './usecases/assistants/assistants.module';
-import { InferenceSettingsModule } from './usecases/inference-settings/inference-settings.module';
 import { ExtensionModule } from './infrastructure/repositories/extensions/extension.module';
 import { CortexModule } from './usecases/cortex/cortex.module';
 import { ConfigModule } from '@nestjs/config';
+import { env } from 'node:process';
 
 @Module({
   imports: [
     DevtoolsModule.register({
-      http: process.env.NODE_ENV !== 'production',
+      http: env.NODE_ENV !== 'production',
     }),
     ConfigModule.forRoot({
       isGlobal: true,
-      envFilePath:
-        process.env.NODE_ENV === 'production' ? '.env' : '.env.development',
+      envFilePath: env.NODE_ENV !== 'production' ? '.env.development' : '.env',
     }),
     DatabaseModule,
     MessagesModule,
     ThreadsModule,
     ModelsModule,
     ChatModule,
     AssistantsModule,
-    InferenceSettingsModule,
     CortexModule,
     ExtensionModule,
   ],

cortex-js/src/command.module.ts

@@ -1,14 +1,26 @@
 import { Module } from '@nestjs/common';
-import { BasicCommand } from './infrastructure/commanders/basic-command.commander';
 import { ModelsModule } from './usecases/models/models.module';
 import { DatabaseModule } from './infrastructure/database/database.module';
 import { ConfigModule } from '@nestjs/config';
 import { CortexModule } from './usecases/cortex/cortex.module';
 import { ServeCommand } from './infrastructure/commanders/serve.command';
-import { PullCommand } from './infrastructure/commanders/pull.command';
-import { InferenceCommand } from './infrastructure/commanders/inference.command';
 import { ModelsCommand } from './infrastructure/commanders/models.command';
-import { StartCommand } from './infrastructure/commanders/start.command';
+import { ExtensionModule } from './infrastructure/repositories/extensions/extension.module';
+import { ChatModule } from './usecases/chat/chat.module';
+import { InitCommand } from './infrastructure/commanders/init.command';
+import { HttpModule } from '@nestjs/axios';
+import { InitRunModeQuestions } from './infrastructure/commanders/questions/init.questions';
+import { ModelListCommand } from './infrastructure/commanders/models/model-list.command';
+import { ModelPullCommand } from './infrastructure/commanders/models/model-pull.command';
+import { CortexCommand } from './infrastructure/commanders/cortex-command.commander';
+import { ChatCommand } from './infrastructure/commanders/chat.command';
+import { ModelStartCommand } from './infrastructure/commanders/models/model-start.command';
+import { ModelStopCommand } from './infrastructure/commanders/models/model-stop.command';
+import { ModelGetCommand } from './infrastructure/commanders/models/model-get.command';
+import { ModelRemoveCommand } from './infrastructure/commanders/models/model-remove.command';
+import { RunCommand } from './infrastructure/commanders/shortcuts/run.command';
+import { InitCudaQuestions } from './infrastructure/commanders/questions/cuda.questions';
+import { CliUsecasesModule } from './infrastructure/commanders/usecases/cli.usecases.module';
 
 @Module({
   imports: [
@@ -20,14 +32,32 @@ import { StartCommand } from './infrastructure/commanders/start.command';
     DatabaseModule,
     ModelsModule,
     CortexModule,
+    ChatModule,
+    ExtensionModule,
+    HttpModule,
+    CliUsecasesModule,
   ],
   providers: [
-    BasicCommand,
+    CortexCommand,
     ModelsCommand,
-    PullCommand,
     ServeCommand,
-    InferenceCommand,
-    StartCommand,
+    ChatCommand,
+    InitCommand,
+
+    // Questions
+    InitRunModeQuestions,
+    InitCudaQuestions,
+
+    // Model commands
+    ModelStartCommand,
+    ModelStopCommand,
+    ModelListCommand,
+    ModelGetCommand,
+    ModelRemoveCommand,
+    ModelPullCommand,
+
+    // Shortcuts
+    RunCommand,
   ],
 })
 export class CommandModule {}