fix: #644 - model start attach mode can just work once (#851)

janhq · Jul 8, 2024 · 828b679 · 828b679
1 parent 4a59eb2
commit 828b679
Show file tree

Hide file tree

Showing 10 changed files with 77 additions and 42 deletions.
diff --git a/cortex-js/src/infrastructure/commanders/init.command.ts b/cortex-js/src/infrastructure/commanders/init.command.ts
@@ -36,9 +36,7 @@ export class InitCommand extends CommandRunner {
 
   async run(passedParams: string[], options?: InitOptions): Promise<void> {
     if (options?.silent) {
-      const installationOptions =
-        await this.initUsecases.defaultInstallationOptions();
-      await this.initUsecases.installEngine(installationOptions);
+      await this.initUsecases.installEngine(undefined);
     } else {
       options = await this.inquirerService.ask(
         'init-run-mode-questions',

diff --git a/cortex-js/src/infrastructure/commanders/models/model-pull.command.ts b/cortex-js/src/infrastructure/commanders/models/model-pull.command.ts
@@ -60,11 +60,7 @@ export class ModelPullCommand extends CommandRunner {
       !existsSync(join(await this.fileService.getCortexCppEnginePath(), engine))
     ) {
       console.log('\n');
-      await this.initUsecases.installEngine(
-        await this.initUsecases.defaultInstallationOptions(),
-        'latest',
-        engine,
-      );
+      await this.initUsecases.installEngine(undefined, 'latest', engine);
     }
     this.telemetryUsecases.sendEvent(
       [

diff --git a/cortex-js/src/infrastructure/commanders/models/model-start.command.ts b/cortex-js/src/infrastructure/commanders/models/model-start.command.ts
@@ -11,7 +11,7 @@ import { CortexUsecases } from '@/usecases/cortex/cortex.usecases';
 import { SetCommandContext } from '../decorators/CommandContext';
 import { ContextService } from '@/infrastructure/services/context/context.service';
 import { InitCliUsecases } from '../usecases/init.cli.usecases';
-import { existsSync } from 'node:fs';
+import { createReadStream, existsSync, statSync, watchFile } from 'node:fs';
 import { FileManagerService } from '@/infrastructure/services/file-manager/file-manager.service';
 import { join } from 'node:path';
 import { Engines } from '../types/engine.interface';
@@ -61,7 +61,9 @@ export class ModelStartCommand extends CommandRunner {
       !Array.isArray(existingModel.files) ||
       /^(http|https):\/\/[^/]+\/.*/.test(existingModel.files[0])
     ) {
-      checkingSpinner.fail(`Model ${modelId} not found on filesystem.\nPlease try 'cortex pull ${modelId}' first.`);
+      checkingSpinner.fail(
+        `Model ${modelId} not found on filesystem.\nPlease try 'cortex pull ${modelId}' first.`,
+      );
       process.exit(1);
     }
 
@@ -74,18 +76,19 @@ export class ModelStartCommand extends CommandRunner {
       !existsSync(join(await this.fileService.getCortexCppEnginePath(), engine))
     ) {
       const engineSpinner = ora('Installing engine...').start();
-      await this.initUsecases.installEngine(
-        await this.initUsecases.defaultInstallationOptions(),
-        'latest',
-        engine,
-      );
+      await this.initUsecases.installEngine(undefined, 'latest', engine);
       engineSpinner.succeed();
     }
+
+    // Attached - stdout logs
+    if (options.attach) {
+      this.attachLogWatch();
+    }
+
     await this.cortexUsecases
-      .startCortex(options.attach)
+      .startCortex()
       .then(() => this.modelsCliUsecases.startModel(modelId, options.preset))
-      .then(console.log)
-      .then(() => !options.attach && process.exit(0));
+      .then(() => options.attach && ora('Model is running...').start());
   }
 
   modelInquiry = async () => {
@@ -120,4 +123,38 @@ export class ModelStartCommand extends CommandRunner {
   parseTemplate(value: string) {
     return value;
   }
+
+  /**
+   * Attach to the log file and watch for changes
+   */
+  private async attachLogWatch() {
+    const logPath = await this.fileService.getLogPath();
+    const initialSize = statSync(logPath).size;
+    const logStream = createReadStream(logPath, {
+      start: initialSize,
+      encoding: 'utf-8',
+      autoClose: false,
+    });
+    logStream.on('data', (chunk) => {
+      console.log(chunk);
+    });
+    watchFile(logPath, (curr, prev) => {
+      // Check if the file size has increased
+      if (curr.size > prev.size) {
+        // Calculate the position to start reading from
+        const position = prev.size;
+
+        // Create a new read stream from the updated position
+        const updateStream = createReadStream(logPath, {
+          encoding: 'utf8',
+          start: position,
+        });
+
+        // Read the newly written content
+        updateStream.on('data', (chunk) => {
+          console.log(chunk);
+        });
+      }
+    });
+  }
 }
diff --git a/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts b/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts
@@ -88,16 +88,12 @@ export class RunCommand extends CommandRunner {
       !existsSync(join(await this.fileService.getCortexCppEnginePath(), engine))
     ) {
       const engineSpinner = ora('Installing engine...').start();
-      await this.initUsecases.installEngine(
-        await this.initUsecases.defaultInstallationOptions(),
-        'latest',
-        engine,
-      );
+      await this.initUsecases.installEngine(undefined, 'latest', engine);
       engineSpinner.succeed('Engine installed');
     }
 
     return this.cortexUsecases
-      .startCortex(false)
+      .startCortex()
       .then(() => this.modelsCliUsecases.startModel(modelId, options.preset))
       .then(() => this.chatCliUsecases.chat(modelId, options.threadId));
   }

diff --git a/cortex-js/src/infrastructure/commanders/usecases/init.cli.usecases.ts b/cortex-js/src/infrastructure/commanders/usecases/init.cli.usecases.ts
@@ -51,11 +51,15 @@ export class InitCliUsecases {
    * @param version
    */
   installEngine = async (
-    options: InitOptions,
+    options?: InitOptions,
     version: string = 'latest',
     engine: string = 'default',
     force: boolean = true,
   ): Promise<any> => {
+    // Use default option if not defined
+    if (!options) {
+      options = await this.defaultInstallationOptions();
+    }
     const configs = await this.fileManagerService.getConfig();
 
     if (configs.initialized && !force) return;
@@ -271,7 +275,7 @@ export class InitCliUsecases {
   private detectInstructions = (): Promise<
     'AVX' | 'AVX2' | 'AVX512' | undefined
   > => {
-    const cpuInstruction = cpuInfo.cpuInfo()[0]?? 'AVX'
+    const cpuInstruction = cpuInfo.cpuInfo()[0] ?? 'AVX';
     console.log(cpuInstruction, 'CPU instructions detected');
     return Promise.resolve(cpuInstruction);
   };

diff --git a/cortex-js/src/infrastructure/commanders/usecases/models.cli.usecases.ts b/cortex-js/src/infrastructure/commanders/usecases/models.cli.usecases.ts
@@ -41,11 +41,11 @@ export class ModelsCliUsecases {
       .catch(async (e) => {
         console.error('Model start failed with reason:', e.message);
 
-        printLastErrorLines(await this.fileService.getDataFolderPath(), 5);
+        printLastErrorLines(await this.fileService.getLogPath());
 
         console.log(
           'For more information, please check the logs at: %s',
-          join(await this.fileService.getDataFolderPath(), 'cortex.log'),
+          await this.fileService.getLogPath(),
         );
         process.exit(1);
       });

diff --git a/cortex-js/src/infrastructure/services/file-manager/file-manager.service.ts b/cortex-js/src/infrastructure/services/file-manager/file-manager.service.ts
@@ -249,6 +249,14 @@ export class FileManagerService {
     return join(await this.getDataFolderPath(), 'cortex-cpp', 'engines');
   }
 
+  /**
+   * Get log path
+   * @returns the path to the cortex engines folder
+   */
+  async getLogPath(): Promise<string> {
+    return join(await this.getDataFolderPath(), 'cortex.log');
+  }
+
   async createFolderIfNotExistInDataFolder(folderName: string): Promise<void> {
     const dataFolderPath = await this.getDataFolderPath();
     const folderPath = join(dataFolderPath, folderName);

diff --git a/cortex-js/src/usecases/cortex/cortex.usecases.ts b/cortex-js/src/usecases/cortex/cortex.usecases.ts
@@ -29,9 +29,7 @@ export class CortexUsecases {
    * @param attach
    * @returns
    */
-  async startCortex(
-    attach: boolean = false,
-  ): Promise<CortexOperationSuccessfullyDto> {
+  async startCortex(): Promise<CortexOperationSuccessfullyDto> {
     const configs = await this.fileManagerService.getConfig();
     const host = configs.cortexCppHost;
     const port = configs.cortexCppPort;
@@ -56,14 +54,11 @@ export class CortexUsecases {
       'cortex-cpp',
     );
 
-    const writer = openSync(
-      join(await this.fileManagerService.getDataFolderPath(), 'cortex.log'),
-      'a+',
-    );
+    const writer = openSync(await this.fileManagerService.getLogPath(), 'a+');
 
     // go up one level to get the binary folder, have to also work on windows
     this.cortexProcess = spawn(cortexCppPath, args, {
-      detached: !attach,
+      detached: true,
       cwd: cortexCppFolderPath,
       stdio: [0, writer, writer],
       env: {
@@ -143,9 +138,9 @@ export class CortexUsecases {
 
   /**
    * Check whether the Cortex CPP is healthy
-   * @param host 
-   * @param port 
-   * @returns 
+   * @param host
+   * @param port
+   * @returns
    */
   healthCheck(host: string, port: number): Promise<boolean> {
     return fetch(CORTEX_CPP_HEALTH_Z_URL(host, port))

diff --git a/cortex-js/src/usecases/models/models.usecases.ts b/cortex-js/src/usecases/models/models.usecases.ts
@@ -227,6 +227,7 @@ export class ModelsUsecases {
         };
         this.eventEmitter.emit('model.event', modelEvent);
         if (e.code === AxiosError.ERR_BAD_REQUEST) {
+          loadingModelSpinner.succeed('Model loaded');
           return {
             message: 'Model already loaded',
             modelId,

diff --git a/cortex-js/src/utils/logs.ts b/cortex-js/src/utils/logs.ts
@@ -8,12 +8,12 @@ import { createInterface } from 'readline';
  * @param numLines
  */
 export async function printLastErrorLines(
-  dataFolderPath: string,
-  numLines: number = 5,
+  logPath: string,
+  numLines: number = 10,
 ): Promise<void> {
   const errorLines: string[] = [];
 
-  const fileStream = createReadStream(join(dataFolderPath, 'cortex.log'));
+  const fileStream = createReadStream(logPath);
   const rl = createInterface({
     input: fileStream,
     crlfDelay: Infinity,