diff --git a/cortex-js/src/domain/abstracts/oai.abstract.ts b/cortex-js/src/domain/abstracts/oai.abstract.ts index ab7bda32c..7718b8bfd 100644 --- a/cortex-js/src/domain/abstracts/oai.abstract.ts +++ b/cortex-js/src/domain/abstracts/oai.abstract.ts @@ -5,6 +5,7 @@ import { firstValueFrom } from 'rxjs'; export abstract class OAIEngineExtension extends EngineExtension { abstract apiUrl: string; + abstract apiKey?: string; constructor(protected readonly httpService: HttpService) { super(); @@ -21,7 +22,9 @@ export abstract class OAIEngineExtension extends EngineExtension { this.httpService.post(this.apiUrl, createChatDto, { headers: { 'Content-Type': headers['content-type'] ?? 'application/json', - Authorization: headers['authorization'], + Authorization: this.apiKey + ? `Bearer ${this.apiKey}` + : headers['authorization'], }, responseType: stream ? 'stream' : 'json', }), diff --git a/cortex-js/src/extensions/groq.engine.ts b/cortex-js/src/extensions/groq.engine.ts index 046589547..73a376019 100644 --- a/cortex-js/src/extensions/groq.engine.ts +++ b/cortex-js/src/extensions/groq.engine.ts @@ -13,6 +13,7 @@ export default class GroqEngineExtension extends OAIEngineExtension { productName = 'Groq Inference Engine'; description = 'This extension enables fast Groq chat completion API calls'; version = '0.0.1'; + apiKey?: string; constructor( protected readonly httpService: HttpService, @@ -25,6 +26,8 @@ export default class GroqEngineExtension extends OAIEngineExtension { const configs = (await this.configsUsecases.getGroupConfigs( this.name, )) as unknown as { apiKey: string }; + + this.apiKey = configs?.apiKey if (!configs?.apiKey) await this.configsUsecases.saveConfig('apiKey', '', this.name); } diff --git a/cortex-js/src/extensions/mistral.engine.ts b/cortex-js/src/extensions/mistral.engine.ts index 327176f86..19beb6961 100644 --- a/cortex-js/src/extensions/mistral.engine.ts +++ b/cortex-js/src/extensions/mistral.engine.ts @@ -13,6 +13,7 @@ export default class MistralEngineExtension extends OAIEngineExtension { productName = 'Mistral Inference Engine'; description = 'This extension enables Mistral chat completion API calls'; version = '0.0.1'; + apiKey?: string; constructor( protected readonly httpService: HttpService, @@ -25,6 +26,7 @@ export default class MistralEngineExtension extends OAIEngineExtension { const configs = (await this.configsUsecases.getGroupConfigs( this.name, )) as unknown as { apiKey: string }; + this.apiKey = configs?.apiKey; if (!configs?.apiKey) await this.configsUsecases.saveConfig('apiKey', '', this.name); } diff --git a/cortex-js/src/extensions/openai.engine.ts b/cortex-js/src/extensions/openai.engine.ts index 6022fe91f..9f3b9bd01 100644 --- a/cortex-js/src/extensions/openai.engine.ts +++ b/cortex-js/src/extensions/openai.engine.ts @@ -13,6 +13,7 @@ export default class OpenAIEngineExtension extends OAIEngineExtension { productName = 'OpenAI Inference Engine'; description = 'This extension enables OpenAI chat completion API calls'; version = '0.0.1'; + apiKey?: string; constructor( protected readonly httpService: HttpService, @@ -25,6 +26,7 @@ export default class OpenAIEngineExtension extends OAIEngineExtension { const configs = (await this.configsUsecases.getGroupConfigs( this.name, )) as unknown as { apiKey: string }; + this.apiKey = configs?.apiKey; if (!configs?.apiKey) await this.configsUsecases.saveConfig('apiKey', '', this.name); } diff --git a/cortex-js/src/infrastructure/commanders/usecases/chat.cli.usecases.ts b/cortex-js/src/infrastructure/commanders/usecases/chat.cli.usecases.ts index 06254beb4..35a686c28 100644 --- a/cortex-js/src/infrastructure/commanders/usecases/chat.cli.usecases.ts +++ b/cortex-js/src/infrastructure/commanders/usecases/chat.cli.usecases.ts @@ -192,7 +192,7 @@ export class ChatCliUsecases { const toParse = cachedLines + line; if (!line.includes('data: [DONE]')) { const data = JSON.parse(toParse.replace('data: ', '')); - content += data.choices[0]?.delta?.content ?? ''; + content = data.choices[0]?.delta?.content ?? ''; if (content.startsWith('assistant: ')) { content = content.replace('assistant: ', ''); @@ -209,8 +209,10 @@ export class ChatCliUsecases { } }); }) - .catch(() => { - stdout.write('Something went wrong! Please check model status.\n'); + .catch((e: any) => { + stdout.write( + `Something went wrong! Please check model status.\n${e.message}\n`, + ); if (attach) rl.prompt(); else rl.close(); }); diff --git a/cortex-js/src/infrastructure/providers/cortex/cortex.provider.ts b/cortex-js/src/infrastructure/providers/cortex/cortex.provider.ts index 3884eea27..558992f47 100644 --- a/cortex-js/src/infrastructure/providers/cortex/cortex.provider.ts +++ b/cortex-js/src/infrastructure/providers/cortex/cortex.provider.ts @@ -21,6 +21,7 @@ export default class CortexProvider extends OAIEngineExtension { description = 'This extension enables chat completion API calls using the Cortex engine'; version = '0.0.1'; + apiKey?: string | undefined; private loadModelUrl = `http://${defaultCortexCppHost}:${defaultCortexCppPort}/inferences/server/loadmodel`; private unloadModelUrl = `http://${defaultCortexCppHost}:${defaultCortexCppPort}/inferences/server/unloadmodel`;