feat: add engine init endpoint (#888)

cortex-js/src/domain/abstracts/engine.abstract.ts

@@ -10,6 +10,8 @@ export abstract class EngineExtension extends Extension {
 
   transformResponse?: Function;
 
+  initalized: boolean = false;
+
   abstract inference(
     dto: any,
     headers: Record<string, string>,

cortex-js/src/domain/abstracts/extension.abstract.ts

@@ -21,6 +21,9 @@ export abstract class Extension {
   /** @type {string} Extension's version. */
   version?: string;
 
+  /** @type {boolean} Whether the extension is initialized or not. */
+  initalized: boolean;
+
   /**
    * Called when the extension is loaded.
    * Any initialization logic for the extension should be put here.

cortex-js/src/domain/config/config.interface.ts

@@ -1,6 +1,5 @@
 export interface Config {
   dataFolderPath: string;
-  initialized: boolean;
   cortexCppHost: string;
   cortexCppPort: number;
 }

cortex-js/src/domain/models/download.interface.ts

@@ -69,4 +69,5 @@ export interface DownloadStateEvent {
 export enum DownloadType {
   Model = 'model',
   Miscelanous = 'miscelanous',
+  Engine = 'engine'
 }

cortex-js/src/extensions/anthropic.engine.ts

@@ -16,6 +16,7 @@ export default class AnthropicEngineExtension extends OAIEngineExtension {
   productName = 'Anthropic Inference Engine';
   description = 'This extension enables Anthropic chat completion API calls';
   version = '0.0.1';
+  initalized = true;
   apiKey?: string;
 
   constructor(
@@ -39,42 +40,45 @@ export default class AnthropicEngineExtension extends OAIEngineExtension {
     this.apiKey = configs?.apiKey;
   }
 
-  override async inference(dto: any, headers: Record<string, string>): Promise<stream.Readable | any> {
-    headers['x-api-key'] = this.apiKey as string
-    headers['Content-Type'] = 'application/json'
-    headers['anthropic-version'] = '2023-06-01'
-    return super.inference(dto, headers)
+  override async inference(
+    dto: any,
+    headers: Record<string, string>,
+  ): Promise<stream.Readable | any> {
+    headers['x-api-key'] = this.apiKey as string;
+    headers['Content-Type'] = 'application/json';
+    headers['anthropic-version'] = '2023-06-01';
+    return super.inference(dto, headers);
   }
 
   transformPayload = (data: any): any => {
     return _.pick(data, ['messages', 'model', 'stream', 'max_tokens']);
-  }
+  };
 
   transformResponse = (data: any): string => {
     // handling stream response
     if (typeof data === 'string' && data.trim().length === 0) {
-        return '';
+      return '';
     }
     if (typeof data === 'string' && data.startsWith('event: ')) {
-        return ''
+      return '';
     }
     if (typeof data === 'string' && data.startsWith('data: ')) {
       data = data.replace('data: ', '');
       const parsedData = JSON.parse(data);
       if (parsedData.type !== 'content_block_delta') {
-        return ''
+        return '';
       }
       const text = parsedData.delta?.text;
       //convert to have this format data.choices[0]?.delta?.content
       return JSON.stringify({
         choices: [
-            {
-                delta: {
-                content: text
-                }
-            }
-        ]
-      })
+          {
+            delta: {
+              content: text,
+            },
+          },
+        ],
+      });
     }
     // non-stream response
     if (data.content && data.content.length > 0 && data.content[0].text) {
@@ -88,8 +92,8 @@ export default class AnthropicEngineExtension extends OAIEngineExtension {
         ],
       });
     }
-  
+
     console.error('Invalid response format:', data);
     return '';
-  }
+  };
 }

cortex-js/src/extensions/groq.engine.ts

@@ -14,6 +14,7 @@ export default class GroqEngineExtension extends OAIEngineExtension {
   productName = 'Groq Inference Engine';
   description = 'This extension enables fast Groq chat completion API calls';
   version = '0.0.1';
+  initalized = true;
   apiKey?: string;
 
   constructor(

cortex-js/src/extensions/mistral.engine.ts

@@ -14,6 +14,7 @@ export default class MistralEngineExtension extends OAIEngineExtension {
   productName = 'Mistral Inference Engine';
   description = 'This extension enables Mistral chat completion API calls';
   version = '0.0.1';
+  initalized = true;
   apiKey?: string;
 
   constructor(

cortex-js/src/extensions/openai.engine.ts

@@ -14,6 +14,7 @@ export default class OpenAIEngineExtension extends OAIEngineExtension {
   productName = 'OpenAI Inference Engine';
   description = 'This extension enables OpenAI chat completion API calls';
   version = '0.0.1';
+  initalized = true;
   apiKey?: string;
 
   constructor(

cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts

@@ -1,10 +1,10 @@
 import { CommandRunner, Option, SubCommand } from 'nest-commander';
 import { SetCommandContext } from '../decorators/CommandContext';
 import { ContextService } from '@/infrastructure/services/context/context.service';
-import { InitCliUsecases } from '../usecases/init.cli.usecases';
 import { Engines } from '../types/engine.interface';
 import { CortexUsecases } from '@/usecases/cortex/cortex.usecases';
 import { FileManagerService } from '@/infrastructure/services/file-manager/file-manager.service';
+import { EnginesUsecases } from '@/usecases/engines/engines.usecase';
 
 @SubCommand({
   name: '<name> init',
@@ -16,7 +16,7 @@ import { FileManagerService } from '@/infrastructure/services/file-manager/file-
 @SetCommandContext()
 export class EnginesInitCommand extends CommandRunner {
   constructor(
-    private readonly initUsecases: InitCliUsecases,
+    private readonly engineUsecases: EnginesUsecases,
     private readonly cortexUsecases: CortexUsecases,
     private readonly fileManagerService: FileManagerService,
     readonly contextService: ContextService,
@@ -31,7 +31,7 @@ export class EnginesInitCommand extends CommandRunner {
     const engine = passedParams[0];
     const params = passedParams.includes(Engines.llamaCPP)
       ? {
-          ...(await this.initUsecases.defaultInstallationOptions()),
+          ...(await this.engineUsecases.defaultInstallationOptions()),
           ...options,
         }
       : {};
@@ -43,7 +43,8 @@ export class EnginesInitCommand extends CommandRunner {
     if (await this.cortexUsecases.healthCheck(host, port)) {
       await this.cortexUsecases.stopCortex();
     }
-    return this.initUsecases
+    console.log(`Installing engine ${engine}...`);
+    return this.engineUsecases
       .installEngine(
         params,
         engine.includes('@') ? engine.split('@')[1] : 'latest',

cortex-js/src/infrastructure/commanders/models/model-pull.command.ts

@@ -12,9 +12,9 @@ import { ContextService } from '@/infrastructure/services/context/context.servic
 import { existsSync } from 'fs';
 import { join } from 'node:path';
 import { FileManagerService } from '@/infrastructure/services/file-manager/file-manager.service';
-import { InitCliUsecases } from '../usecases/init.cli.usecases';
 import { checkModelCompatibility } from '@/utils/model-check';
 import { Engines } from '../types/engine.interface';
+import { EnginesUsecases } from '@/usecases/engines/engines.usecase';
 
 @SubCommand({
   name: 'pull',
@@ -28,7 +28,7 @@ import { Engines } from '../types/engine.interface';
 export class ModelPullCommand extends CommandRunner {
   constructor(
     private readonly modelsCliUsecases: ModelsCliUsecases,
-    private readonly initUsecases: InitCliUsecases,
+    private readonly engineUsecases: EnginesUsecases,
     private readonly fileService: FileManagerService,
     readonly contextService: ContextService,
     private readonly telemetryUsecases: TelemetryUsecases,
@@ -60,7 +60,7 @@ export class ModelPullCommand extends CommandRunner {
       !existsSync(join(await this.fileService.getCortexCppEnginePath(), engine))
     ) {
       console.log('\n');
-      await this.initUsecases.installEngine(undefined, 'latest', engine);
+      await this.engineUsecases.installEngine(undefined, 'latest', engine);
     }
     this.telemetryUsecases.sendEvent(
       [

cortex-js/src/infrastructure/commanders/models/model-start.command.ts

@@ -10,12 +10,12 @@ import { ModelsCliUsecases } from '@commanders/usecases/models.cli.usecases';
 import { CortexUsecases } from '@/usecases/cortex/cortex.usecases';
 import { SetCommandContext } from '../decorators/CommandContext';
 import { ContextService } from '@/infrastructure/services/context/context.service';
-import { InitCliUsecases } from '../usecases/init.cli.usecases';
 import { createReadStream, existsSync, statSync, watchFile } from 'node:fs';
 import { FileManagerService } from '@/infrastructure/services/file-manager/file-manager.service';
 import { join } from 'node:path';
 import { Engines } from '../types/engine.interface';
 import { checkModelCompatibility } from '@/utils/model-check';
+import { EnginesUsecases } from '@/usecases/engines/engines.usecase';
 
 type ModelStartOptions = {
   attach: boolean;
@@ -36,7 +36,7 @@ export class ModelStartCommand extends CommandRunner {
     private readonly inquirerService: InquirerService,
     private readonly cortexUsecases: CortexUsecases,
     private readonly modelsCliUsecases: ModelsCliUsecases,
-    private readonly initUsecases: InitCliUsecases,
+    private readonly initUsecases: EnginesUsecases,
     private readonly fileService: FileManagerService,
     readonly contextService: ContextService,
   ) {

cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts

@@ -13,9 +13,9 @@ import { ModelNotFoundException } from '@/infrastructure/exception/model-not-fou
 import { existsSync } from 'fs';
 import { join } from 'path';
 import { FileManagerService } from '@/infrastructure/services/file-manager/file-manager.service';
-import { InitCliUsecases } from '../usecases/init.cli.usecases';
 import { Engines } from '../types/engine.interface';
 import { checkModelCompatibility } from '@/utils/model-check';
+import { EnginesUsecases } from '@/usecases/engines/engines.usecase';
 
 type RunOptions = {
   threadId?: string;
@@ -38,7 +38,7 @@ export class RunCommand extends CommandRunner {
     private readonly chatCliUsecases: ChatCliUsecases,
     private readonly inquirerService: InquirerService,
     private readonly fileService: FileManagerService,
-    private readonly initUsecases: InitCliUsecases,
+    private readonly initUsecases: EnginesUsecases,
   ) {
     super();
   }

cortex-js/src/infrastructure/commanders/test/helpers.command.spec.ts

@@ -35,7 +35,6 @@ beforeAll(
       // Attempt to create test folder
       await fileService.writeConfigFile({
         dataFolderPath: join(__dirname, 'test_data'),
-        initialized: false,
         cortexCppHost: 'localhost',
         cortexCppPort: 3929
       });

cortex-js/src/infrastructure/commanders/test/models.command.spec.ts

@@ -23,7 +23,6 @@ beforeAll(
       // Attempt to create test folder
       await fileService.writeConfigFile({
         dataFolderPath: join(__dirname, 'test_data'),
-        initialized: false,
         cortexCppHost: 'localhost',
         cortexCppPort: 3929,
       });

cortex-js/src/infrastructure/commanders/usecases/cli.usecases.module.ts

@@ -1,5 +1,4 @@
 import { Module } from '@nestjs/common';
-import { InitCliUsecases } from './init.cli.usecases';
 import { HttpModule } from '@nestjs/axios';
 import { ModelsCliUsecases } from './models.cli.usecases';
 import { ModelsModule } from '@/usecases/models/models.module';
@@ -13,6 +12,8 @@ import { FileManagerModule } from '@/infrastructure/services/file-manager/file-m
 import { PSCliUsecases } from './ps.cli.usecases';
 import { BenchmarkCliUsecases } from './benchmark.cli.usecases';
 import { TelemetryModule } from '@/usecases/telemetry/telemetry.module';
+import { DownloadManagerModule } from '@/infrastructure/services/download-manager/download-manager.module';
+import { EnginesModule } from '@/usecases/engines/engines.module';
 
 @Module({
   imports: [
@@ -25,16 +26,16 @@ import { TelemetryModule } from '@/usecases/telemetry/telemetry.module';
     MessagesModule,
     FileManagerModule,
     TelemetryModule,
+    DownloadManagerModule,
+    EnginesModule,
   ],
   providers: [
-    InitCliUsecases,
     ModelsCliUsecases,
     ChatCliUsecases,
     PSCliUsecases,
     BenchmarkCliUsecases,
   ],
   exports: [
-    InitCliUsecases,
     ModelsCliUsecases,
     ChatCliUsecases,
     PSCliUsecases,