Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: cortex supports remote engines #786

Merged
merged 1 commit into from
Jun 27, 2024
Merged

feat: cortex supports remote engines #786

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion cortex-js/src/domain/abstracts/oai.abstract.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { firstValueFrom } from 'rxjs';

export abstract class OAIEngineExtension extends EngineExtension {
abstract apiUrl: string;
abstract apiKey?: string;

constructor(protected readonly httpService: HttpService) {
super();
Expand All @@ -21,7 +22,9 @@ export abstract class OAIEngineExtension extends EngineExtension {
this.httpService.post(this.apiUrl, createChatDto, {
headers: {
'Content-Type': headers['content-type'] ?? 'application/json',
Authorization: headers['authorization'],
Authorization: this.apiKey
? `Bearer ${this.apiKey}`
: headers['authorization'],
},
responseType: stream ? 'stream' : 'json',
}),
Expand Down
3 changes: 3 additions & 0 deletions cortex-js/src/extensions/groq.engine.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ export default class GroqEngineExtension extends OAIEngineExtension {
productName = 'Groq Inference Engine';
description = 'This extension enables fast Groq chat completion API calls';
version = '0.0.1';
apiKey?: string;

constructor(
protected readonly httpService: HttpService,
Expand All @@ -25,6 +26,8 @@ export default class GroqEngineExtension extends OAIEngineExtension {
const configs = (await this.configsUsecases.getGroupConfigs(
this.name,
)) as unknown as { apiKey: string };

this.apiKey = configs?.apiKey
if (!configs?.apiKey)
await this.configsUsecases.saveConfig('apiKey', '', this.name);
}
Expand Down
2 changes: 2 additions & 0 deletions cortex-js/src/extensions/mistral.engine.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ export default class MistralEngineExtension extends OAIEngineExtension {
productName = 'Mistral Inference Engine';
description = 'This extension enables Mistral chat completion API calls';
version = '0.0.1';
apiKey?: string;

constructor(
protected readonly httpService: HttpService,
Expand All @@ -25,6 +26,7 @@ export default class MistralEngineExtension extends OAIEngineExtension {
const configs = (await this.configsUsecases.getGroupConfigs(
this.name,
)) as unknown as { apiKey: string };
this.apiKey = configs?.apiKey;
if (!configs?.apiKey)
await this.configsUsecases.saveConfig('apiKey', '', this.name);
}
Expand Down
2 changes: 2 additions & 0 deletions cortex-js/src/extensions/openai.engine.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ export default class OpenAIEngineExtension extends OAIEngineExtension {
productName = 'OpenAI Inference Engine';
description = 'This extension enables OpenAI chat completion API calls';
version = '0.0.1';
apiKey?: string;

constructor(
protected readonly httpService: HttpService,
Expand All @@ -25,6 +26,7 @@ export default class OpenAIEngineExtension extends OAIEngineExtension {
const configs = (await this.configsUsecases.getGroupConfigs(
this.name,
)) as unknown as { apiKey: string };
this.apiKey = configs?.apiKey;
if (!configs?.apiKey)
await this.configsUsecases.saveConfig('apiKey', '', this.name);
}
Expand Down
8 changes: 5 additions & 3 deletions cortex-js/src/infrastructure/commanders/usecases/chat.cli.usecases.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ export class ChatCliUsecases {
const toParse = cachedLines + line;
if (!line.includes('data: [DONE]')) {
const data = JSON.parse(toParse.replace('data: ', ''));
content += data.choices[0]?.delta?.content ?? '';
content = data.choices[0]?.delta?.content ?? '';

if (content.startsWith('assistant: ')) {
content = content.replace('assistant: ', '');
Expand All @@ -209,8 +209,10 @@ export class ChatCliUsecases {
}
});
})
.catch(() => {
stdout.write('Something went wrong! Please check model status.\n');
.catch((e: any) => {
stdout.write(
`Something went wrong! Please check model status.\n${e.message}\n`,
);
if (attach) rl.prompt();
else rl.close();
});
Expand Down
1 change: 1 addition & 0 deletions cortex-js/src/infrastructure/providers/cortex/cortex.provider.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ export default class CortexProvider extends OAIEngineExtension {
description =
'This extension enables chat completion API calls using the Cortex engine';
version = '0.0.1';
apiKey?: string | undefined;

private loadModelUrl = `http://${defaultCortexCppHost}:${defaultCortexCppPort}/inferences/server/loadmodel`;
private unloadModelUrl = `http://${defaultCortexCppHost}:${defaultCortexCppPort}/inferences/server/unloadmodel`;
Expand Down
Loading