From f0e9a7a29d5feda3138069eb85011c5ee381a059 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 30 May 2024 12:18:21 +0700 Subject: [PATCH 1/4] chore: cortex chat enhancement (#631) --- .../infrastructure/commanders/chat.command.ts | 52 ++++++++++-- .../commanders/models/model-list.command.ts | 18 +++- .../commanders/models/model-stop.command.ts | 2 +- .../commanders/usecases/chat.cli.usecases.ts | 82 +++++++++---------- 4 files changed, 101 insertions(+), 53 deletions(-) diff --git a/cortex-js/src/infrastructure/commanders/chat.command.ts b/cortex-js/src/infrastructure/commanders/chat.command.ts index 8f3e5c4e7..74d23c7f4 100644 --- a/cortex-js/src/infrastructure/commanders/chat.command.ts +++ b/cortex-js/src/infrastructure/commanders/chat.command.ts @@ -1,6 +1,13 @@ -import { CommandRunner, SubCommand, Option } from 'nest-commander'; +import { + CommandRunner, + SubCommand, + Option, + InquirerService, +} from 'nest-commander'; import { ChatCliUsecases } from './usecases/chat.cli.usecases'; import { exit } from 'node:process'; +import { PSCliUsecases } from './usecases/ps.cli.usecases'; +import { ModelsUsecases } from '@/usecases/models/models.usecases'; type ChatOptions = { threadId?: string; @@ -10,22 +17,52 @@ type ChatOptions = { @SubCommand({ name: 'chat', description: 'Send a chat request to a model' }) export class ChatCommand extends CommandRunner { - constructor(private readonly chatCliUsecases: ChatCliUsecases) { + constructor( + private readonly inquirerService: InquirerService, + private readonly chatCliUsecases: ChatCliUsecases, + private readonly modelsUsecases: ModelsUsecases, + private readonly psCliUsecases: PSCliUsecases, + ) { super(); } async run(_input: string[], options: ChatOptions): Promise { - const modelId = _input[0]; - if (!modelId) { - console.error('Model ID is required'); - exit(1); + let modelId = _input[0]; + // First attempt to get message from input or options + let message = _input[1] ?? options.message; + + // Check for model existing + if (!modelId || !(await this.modelsUsecases.findOne(modelId))) { + // Model ID is not provided + // first input might be message input + message = _input[0] ?? options.message; + // If model ID is not provided, prompt user to select from running models + const models = await this.psCliUsecases.getModels(); + if (models.length === 1) { + modelId = models[0].modelId; + } else if (models.length > 0) { + const { model } = await this.inquirerService.inquirer.prompt({ + type: 'list', + name: 'model', + message: 'Select running model to chat with:', + choices: models.map((e) => ({ + name: e.modelId, + value: e.modelId, + })), + }); + modelId = model; + } else { + console.error('Model ID is required'); + exit(1); + } } return this.chatCliUsecases.chat( modelId, options.threadId, - options.message, + message, // Accept both message from inputs or arguments options.attach, + false, // Do not stop cortex session or loaded model ); } @@ -40,7 +77,6 @@ export class ChatCommand extends CommandRunner { @Option({ flags: '-m, --message ', description: 'Message to send to the model', - required: true, }) parseModelId(value: string) { return value; diff --git a/cortex-js/src/infrastructure/commanders/models/model-list.command.ts b/cortex-js/src/infrastructure/commanders/models/model-list.command.ts index 6e491fc8d..052e8feec 100644 --- a/cortex-js/src/infrastructure/commanders/models/model-list.command.ts +++ b/cortex-js/src/infrastructure/commanders/models/model-list.command.ts @@ -1,14 +1,26 @@ -import { CommandRunner, SubCommand } from 'nest-commander'; +import { CommandRunner, SubCommand, Option } from 'nest-commander'; import { ModelsCliUsecases } from '../usecases/models.cli.usecases'; +interface ModelListOptions { + format: 'table' | 'json'; +} @SubCommand({ name: 'list', description: 'List all models locally.' }) export class ModelListCommand extends CommandRunner { constructor(private readonly modelsCliUsecases: ModelsCliUsecases) { super(); } - async run(): Promise { + async run(_input: string[], option: ModelListOptions): Promise { const models = await this.modelsCliUsecases.listAllModels(); - console.log(models); + option.format === 'table' ? console.table(models) : console.log(models); + } + + @Option({ + flags: '-f, --format ', + defaultValue: 'json', + description: 'Print models list in table or json format', + }) + parseModelId(value: string) { + return value; } } diff --git a/cortex-js/src/infrastructure/commanders/models/model-stop.command.ts b/cortex-js/src/infrastructure/commanders/models/model-stop.command.ts index f13f2021c..3168e9b7a 100644 --- a/cortex-js/src/infrastructure/commanders/models/model-stop.command.ts +++ b/cortex-js/src/infrastructure/commanders/models/model-stop.command.ts @@ -20,7 +20,7 @@ export class ModelStopCommand extends CommandRunner { await this.modelsCliUsecases .stopModel(input[0]) - .then(() => this.cortexUsecases.stopCortex()) + .then(() => this.modelsCliUsecases.stopModel(input[0])) .then(console.log); } } diff --git a/cortex-js/src/infrastructure/commanders/usecases/chat.cli.usecases.ts b/cortex-js/src/infrastructure/commanders/usecases/chat.cli.usecases.ts index 5147b2e1a..2ef890d49 100644 --- a/cortex-js/src/infrastructure/commanders/usecases/chat.cli.usecases.ts +++ b/cortex-js/src/infrastructure/commanders/usecases/chat.cli.usecases.ts @@ -36,47 +36,12 @@ export class ChatCliUsecases { private readonly messagesUsecases: MessagesUsecases, ) {} - private async getOrCreateNewThread( - modelId: string, - threadId?: string, - ): Promise { - if (threadId) { - const thread = await this.threadUsecases.findOne(threadId); - if (!thread) throw new Error(`Cannot find thread with id: ${threadId}`); - return thread; - } - - const model = await this.modelsUsecases.findOne(modelId); - if (!model) throw new Error(`Cannot find model with id: ${modelId}`); - - const assistant = await this.assistantUsecases.findOne('jan'); - if (!assistant) throw new Error('No assistant available'); - - const createThreadModel: CreateThreadModelInfoDto = { - id: modelId, - settings: model.settings, - parameters: model.parameters, - }; - - const assistantDto: CreateThreadAssistantDto = { - assistant_id: assistant.id, - assistant_name: assistant.name, - model: createThreadModel, - }; - - const createThreadDto: CreateThreadDto = { - title: 'New Thread', - assistants: [assistantDto], - }; - - return this.threadUsecases.create(createThreadDto); - } - async chat( modelId: string, threadId?: string, message?: string, attach: boolean = true, + stopModel: boolean = true, ): Promise { if (attach) console.log(`Inorder to exit, type '${this.exitClause}'.`); const thread = await this.getOrCreateNewThread(modelId, threadId); @@ -95,11 +60,10 @@ export class ChatCliUsecases { if (message) sendCompletionMessage.bind(this)(message); if (attach) rl.prompt(); - rl.on('close', () => { - this.cortexUsecases.stopCortex().then(() => { - if (attach) console.log(this.exitMessage); - exit(0); - }); + rl.on('close', async () => { + if (stopModel) await this.modelsUsecases.stopModel(modelId); + if (attach) console.log(this.exitMessage); + exit(0); }); rl.on('line', sendCompletionMessage.bind(this)); @@ -213,4 +177,40 @@ export class ChatCliUsecases { }); } } + + private async getOrCreateNewThread( + modelId: string, + threadId?: string, + ): Promise { + if (threadId) { + const thread = await this.threadUsecases.findOne(threadId); + if (!thread) throw new Error(`Cannot find thread with id: ${threadId}`); + return thread; + } + + const model = await this.modelsUsecases.findOne(modelId); + if (!model) throw new Error(`Cannot find model with id: ${modelId}`); + + const assistant = await this.assistantUsecases.findOne('jan'); + if (!assistant) throw new Error('No assistant available'); + + const createThreadModel: CreateThreadModelInfoDto = { + id: modelId, + settings: model.settings, + parameters: model.parameters, + }; + + const assistantDto: CreateThreadAssistantDto = { + assistant_id: assistant.id, + assistant_name: assistant.name, + model: createThreadModel, + }; + + const createThreadDto: CreateThreadDto = { + title: 'New Thread', + assistants: [assistantDto], + }; + + return this.threadUsecases.create(createThreadDto); + } } From 9b89d4383f00d5ca7da96c25b431bebe72a2feee Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 30 May 2024 12:55:54 +0700 Subject: [PATCH 2/4] fix: bump update-notifier type version (#632) --- cortex-js/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cortex-js/package.json b/cortex-js/package.json index dec85c2ea..ed40df8a8 100644 --- a/cortex-js/package.json +++ b/cortex-js/package.json @@ -71,7 +71,7 @@ "@types/js-yaml": "^4.0.9", "@types/node": "^20.12.9", "@types/supertest": "^6.0.2", - "@types/update-notifier": "^5.0.8", + "@types/update-notifier": "^6.0.8", "@types/uuid": "^9.0.8", "@typescript-eslint/eslint-plugin": "^6.0.0", "@typescript-eslint/parser": "^6.0.0", From 5cb8e7aefa774e03171c60eaf451c3216b25e693 Mon Sep 17 00:00:00 2001 From: irfanpena Date: Thu, 30 May 2024 14:53:22 +0700 Subject: [PATCH 3/4] api: Remove the "/" from the URL --- cortex-js/src/main.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cortex-js/src/main.ts b/cortex-js/src/main.ts index 3a722a5d5..3eaa8fc42 100644 --- a/cortex-js/src/main.ts +++ b/cortex-js/src/main.ts @@ -33,7 +33,7 @@ async function bootstrap() { .addTag('Models', 'These endpoints provide a list and descriptions of all available models within the Cortex framework.') .addTag('Messages', "These endpoints manage the retrieval and storage of conversation content, including responses from LLMs and other metadata related to chat interactions.") .addTag('Threads', 'These endpoints handle the creation, retrieval, updating, and deletion of conversation threads.') - .addServer('http://localhost:1337/') + .addServer('http://localhost:1337') .addServer('http://localhost:1337/v1') .build(); const document = SwaggerModule.createDocument(app, config); From 261ab1137c683c1d1ab43d89719686828378c860 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 30 May 2024 14:54:50 +0700 Subject: [PATCH 4/4] chore: deprecate cortex start stop api (#635) --- .../commanders/models/model-pull.command.ts | 1 + .../commanders/models/model-remove.command.ts | 3 +- .../commanders/models/model-stop.command.ts | 6 +-- .../commanders/usecases/init.cli.usecases.ts | 3 ++ .../controllers/cortex.controller.spec.ts | 18 -------- .../controllers/cortex.controller.ts | 45 ------------------- .../controllers/models.controller.ts | 10 ++++- .../src/usecases/cortex/cortex.module.ts | 2 - 8 files changed, 14 insertions(+), 74 deletions(-) delete mode 100644 cortex-js/src/infrastructure/controllers/cortex.controller.spec.ts delete mode 100644 cortex-js/src/infrastructure/controllers/cortex.controller.ts diff --git a/cortex-js/src/infrastructure/commanders/models/model-pull.command.ts b/cortex-js/src/infrastructure/commanders/models/model-pull.command.ts index 58b4a5d4a..1dade38de 100644 --- a/cortex-js/src/infrastructure/commanders/models/model-pull.command.ts +++ b/cortex-js/src/infrastructure/commanders/models/model-pull.command.ts @@ -50,6 +50,7 @@ export class ModelPullCommand extends CommandRunner { name: sanitizedInput, }; + // eslint-disable-next-line @typescript-eslint/no-unused-vars for await (const _fileInfo of listFiles({ repo })) { break; } diff --git a/cortex-js/src/infrastructure/commanders/models/model-remove.command.ts b/cortex-js/src/infrastructure/commanders/models/model-remove.command.ts index 531f0f893..db31204a1 100644 --- a/cortex-js/src/infrastructure/commanders/models/model-remove.command.ts +++ b/cortex-js/src/infrastructure/commanders/models/model-remove.command.ts @@ -14,7 +14,6 @@ export class ModelRemoveCommand extends CommandRunner { exit(1); } - const result = await this.modelsCliUsecases.removeModel(input[0]); - console.log(result); + await this.modelsCliUsecases.removeModel(input[0]).then(console.log); } } diff --git a/cortex-js/src/infrastructure/commanders/models/model-stop.command.ts b/cortex-js/src/infrastructure/commanders/models/model-stop.command.ts index 3168e9b7a..a72ec0074 100644 --- a/cortex-js/src/infrastructure/commanders/models/model-stop.command.ts +++ b/cortex-js/src/infrastructure/commanders/models/model-stop.command.ts @@ -1,14 +1,10 @@ import { CommandRunner, SubCommand } from 'nest-commander'; import { exit } from 'node:process'; import { ModelsCliUsecases } from '../usecases/models.cli.usecases'; -import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; @SubCommand({ name: 'stop', description: 'Stop a model by ID.' }) export class ModelStopCommand extends CommandRunner { - constructor( - private readonly cortexUsecases: CortexUsecases, - private readonly modelsCliUsecases: ModelsCliUsecases, - ) { + constructor(private readonly modelsCliUsecases: ModelsCliUsecases) { super(); } diff --git a/cortex-js/src/infrastructure/commanders/usecases/init.cli.usecases.ts b/cortex-js/src/infrastructure/commanders/usecases/init.cli.usecases.ts index 443f5cbf3..34527d857 100644 --- a/cortex-js/src/infrastructure/commanders/usecases/init.cli.usecases.ts +++ b/cortex-js/src/infrastructure/commanders/usecases/init.cli.usecases.ts @@ -8,6 +8,7 @@ import { InitOptions } from '../types/init-options.interface'; import { Injectable } from '@nestjs/common'; import { firstValueFrom } from 'rxjs'; import { FileManagerService } from '@/file-manager/file-manager.service'; +import { rm } from 'fs/promises'; @Injectable() export class InitCliUsecases { @@ -105,6 +106,7 @@ export class InitCliUsecases { console.error('Error decompressing file', e); exit(1); } + await rm(destination, { force: true }); }; parseEngineFileName = (options: InitOptions) => { @@ -228,5 +230,6 @@ export class InitCliUsecases { console.log(e); exit(1); } + await rm(destination, { force: true }); }; } diff --git a/cortex-js/src/infrastructure/controllers/cortex.controller.spec.ts b/cortex-js/src/infrastructure/controllers/cortex.controller.spec.ts deleted file mode 100644 index 6d7790c71..000000000 --- a/cortex-js/src/infrastructure/controllers/cortex.controller.spec.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { CortexController } from './cortex.controller'; - -describe('CortexController', () => { - let controller: CortexController; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - controllers: [CortexController], - }).compile(); - - controller = module.get(CortexController); - }); - - it('should be defined', () => { - expect(controller).toBeDefined(); - }); -}); diff --git a/cortex-js/src/infrastructure/controllers/cortex.controller.ts b/cortex-js/src/infrastructure/controllers/cortex.controller.ts deleted file mode 100644 index 907fdba10..000000000 --- a/cortex-js/src/infrastructure/controllers/cortex.controller.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { Body, Controller, HttpCode, Post } from '@nestjs/common'; -import { ApiResponse, ApiTags, ApiOperation } from '@nestjs/swagger'; -import { StartCortexDto } from '@/infrastructure/dtos/cortex/start-cortex.dto'; -import { CortexOperationSuccessfullyDto } from '../dtos/cortex/cortex-operation-successfully.dto'; -import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; - -@ApiTags('Cortex') -@Controller('cortex') -export class CortexController { - constructor(private readonly cortexUsecases: CortexUsecases) {} - - @HttpCode(200) - @ApiResponse({ - status: 200, - description: 'Start Cortex successfully', - type: CortexOperationSuccessfullyDto, - }) - @ApiOperation({ - summary: 'Start cortex', - description: 'Starts the cortex operation.', - }) - @Post('start') - startCortex(@Body() startCortexDto: StartCortexDto) { - return this.cortexUsecases.startCortex( - false, - startCortexDto.host, - startCortexDto.port, - ); - } - - @HttpCode(200) - @ApiResponse({ - status: 200, - description: 'Stop Cortex successfully', - type: CortexOperationSuccessfullyDto, - }) - @ApiOperation({ - summary: 'Stop cortex', - description: 'Stops the cortex operation.', - }) - @Post('stop') - stopCortex() { - return this.cortexUsecases.stopCortex(); - } -} diff --git a/cortex-js/src/infrastructure/controllers/models.controller.ts b/cortex-js/src/infrastructure/controllers/models.controller.ts index 1f8b17e30..bb1d415b1 100644 --- a/cortex-js/src/infrastructure/controllers/models.controller.ts +++ b/cortex-js/src/infrastructure/controllers/models.controller.ts @@ -20,12 +20,16 @@ import { ApiOperation, ApiParam, ApiTags, ApiResponse } from '@nestjs/swagger'; import { StartModelSuccessDto } from '@/infrastructure/dtos/models/start-model-success.dto'; import { ModelSettingParamsDto } from '../dtos/models/model-setting-params.dto'; import { TransformInterceptor } from '../interceptors/transform.interceptor'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; @ApiTags('Models') @Controller('models') @UseInterceptors(TransformInterceptor) export class ModelsController { - constructor(private readonly modelsUsecases: ModelsUsecases) {} + constructor( + private readonly modelsUsecases: ModelsUsecases, + private readonly cortexUsecases: CortexUsecases, + ) {} @HttpCode(201) @ApiResponse({ @@ -62,7 +66,9 @@ export class ModelsController { @Param('modelId') modelId: string, @Body() settings: ModelSettingParamsDto, ) { - return this.modelsUsecases.startModel(modelId, settings); + return this.cortexUsecases + .startCortex() + .then(() => this.modelsUsecases.startModel(modelId, settings)); } @HttpCode(200) diff --git a/cortex-js/src/usecases/cortex/cortex.module.ts b/cortex-js/src/usecases/cortex/cortex.module.ts index 76c757b8c..dcb28a595 100644 --- a/cortex-js/src/usecases/cortex/cortex.module.ts +++ b/cortex-js/src/usecases/cortex/cortex.module.ts @@ -1,13 +1,11 @@ import { Module } from '@nestjs/common'; import { CortexUsecases } from './cortex.usecases'; -import { CortexController } from '@/infrastructure/controllers/cortex.controller'; import { HttpModule } from '@nestjs/axios'; import { FileManagerModule } from '@/file-manager/file-manager.module'; @Module({ imports: [HttpModule, FileManagerModule], providers: [CortexUsecases], - controllers: [CortexController], exports: [CortexUsecases], }) export class CortexModule {}