From b50d942dc4abc09a447e4f366381c3020a075710 Mon Sep 17 00:00:00 2001 From: axel7083 <42176370+axel7083@users.noreply.github.com> Date: Thu, 18 Jan 2024 18:17:23 +0100 Subject: [PATCH 1/9] feat: adding support for starting / stopping playground Signed-off-by: axel7083 <42176370+axel7083@users.noreply.github.com> --- packages/backend/src/managers/playground.ts | 55 ++++++--- packages/backend/src/studio-api-impl.ts | 15 ++- packages/frontend/src/lib/Card.svelte | 6 +- .../frontend/src/pages/ModelPlayground.svelte | 114 ++++++++++++++---- .../frontend/src/stores/playground-queries.ts | 2 +- .../frontend/src/stores/playground-states.ts | 18 +++ packages/shared/Messages.ts | 1 + packages/shared/src/StudioAPI.ts | 9 +- .../src/models/IPlaygroundQueryState.ts | 2 + .../shared/src/models/IPlaygroundState.ts | 10 ++ 10 files changed, 182 insertions(+), 50 deletions(-) create mode 100644 packages/frontend/src/stores/playground-states.ts create mode 100644 packages/shared/src/models/IPlaygroundState.ts diff --git a/packages/backend/src/managers/playground.ts b/packages/backend/src/managers/playground.ts index 7bf9e8c38..ea60c8dc4 100644 --- a/packages/backend/src/managers/playground.ts +++ b/packages/backend/src/managers/playground.ts @@ -30,7 +30,8 @@ import path from 'node:path'; import * as http from 'node:http'; import { getFreePort } from '../utils/ports'; import type { QueryState } from '@shared/src/models/IPlaygroundQueryState'; -import { MSG_NEW_PLAYGROUND_QUERIES_STATE } from '@shared/Messages'; +import { MSG_NEW_PLAYGROUND_QUERIES_STATE, MSG_PLAYGROUNDS_STATE_UPDATE } from '@shared/Messages'; +import { PlaygroundState } from '@shared/src/models/IPlaygroundState'; // TODO: this should not be hardcoded const LOCALAI_IMAGE = 'quay.io/go-skynet/local-ai:v2.5.1'; @@ -43,14 +44,10 @@ function findFirstProvider(): ProviderContainerConnection | undefined { return engines.length > 0 ? engines[0] : undefined; } -export interface PlaygroundState { - containerId: string; - port: number; -} - export class PlayGroundManager { private queryIdCounter = 0; + // Map private playgrounds: Map; private queries: Map; @@ -64,14 +61,25 @@ export class PlayGroundManager { return images.length > 0 ? images[0] : undefined; } + updatePlaygroundState(modelId: string, state: PlaygroundState) { + this.playgrounds.set(modelId, state); + return this.webview.postMessage({ + id: MSG_PLAYGROUNDS_STATE_UPDATE, + body: this.getPlaygroundsState(), + }); + } + async startPlayground(modelId: string, modelPath: string): Promise { // TODO(feloy) remove previous query from state? - if (this.playgrounds.has(modelId)) { throw new Error('model is already running'); } + + this.updatePlaygroundState(modelId, { status: 'starting', modelId }); + const connection = findFirstProvider(); if (!connection) { + this.updatePlaygroundState(modelId, { status: 'error', modelId }); throw new Error('Unable to find an engine to start playground'); } @@ -80,9 +88,11 @@ export class PlayGroundManager { await containerEngine.pullImage(connection.connection, LOCALAI_IMAGE, () => {}); image = await this.selectImage(connection, LOCALAI_IMAGE); if (!image) { + this.updatePlaygroundState(modelId, { status: 'error', modelId }); throw new Error(`Unable to find ${LOCALAI_IMAGE} image`); } } + const freePort = await getFreePort(); const result = await containerEngine.createContainer(image.engineId, { Image: image.Id, @@ -107,10 +117,16 @@ export class PlayGroundManager { }, Cmd: ['--models-path', '/models', '--context-size', '700', '--threads', '4'], }); - this.playgrounds.set(modelId, { - containerId: result.id, - port: freePort, + + this.updatePlaygroundState(modelId, { + container: { + containerId: result.id, + port: freePort, + }, + status: 'running', + modelId }); + return result.id; } @@ -124,7 +140,7 @@ export class PlayGroundManager { async askPlayground(modelInfo: LocalModelInfo, prompt: string): Promise { const state = this.playgrounds.get(modelInfo.id); - if (!state) { + if (!state || !state.container) { throw new Error('model is not running'); } @@ -142,7 +158,7 @@ export class PlayGroundManager { const post_options: http.RequestOptions = { host: 'localhost', - port: '' + state.port, + port: '' + state.container.port, path: '/v1/completions', method: 'POST', headers: { @@ -164,7 +180,7 @@ export class PlayGroundManager { } q.response = result as ModelResponse; this.queries.set(query.id, q); - this.sendState().catch((err: unknown) => { + this.sendQueriesState().catch((err: unknown) => { console.error('playground: unable to send the response to the frontend', err); }); } @@ -175,20 +191,25 @@ export class PlayGroundManager { post_req.end(); this.queries.set(query.id, query); - await this.sendState(); + await this.sendQueriesState(); return query.id; } getNextQueryId() { return ++this.queryIdCounter; } - getState(): QueryState[] { + getQueriesState(): QueryState[] { return Array.from(this.queries.values()); } - async sendState() { + + getPlaygroundsState(): PlaygroundState[] { + return Array.from(this.playgrounds.values()); + } + + async sendQueriesState() { await this.webview.postMessage({ id: MSG_NEW_PLAYGROUND_QUERIES_STATE, - body: this.getState(), + body: this.getQueriesState(), }); } } diff --git a/packages/backend/src/studio-api-impl.ts b/packages/backend/src/studio-api-impl.ts index 73a73b16b..af0b94f18 100644 --- a/packages/backend/src/studio-api-impl.ts +++ b/packages/backend/src/studio-api-impl.ts @@ -32,6 +32,7 @@ import type { QueryState } from '@shared/src/models/IPlaygroundQueryState'; import * as path from 'node:path'; import type { CatalogManager } from './managers/catalogManager'; import type { Catalog } from '@shared/src/models/ICatalog'; +import { PlaygroundState } from '@shared/src/models/IPlaygroundState'; export const RECENT_CATEGORY_ID = 'recent-category'; @@ -122,16 +123,22 @@ export class StudioApiImpl implements StudioAPI { } async startPlayground(modelId: string): Promise { + // TODO: improve the following const localModelInfo = this.applicationManager.getLocalModels().filter(m => m.id === modelId); if (localModelInfo.length !== 1) { throw new Error('model not found'); } + // TODO: we need to stop doing that. const modelPath = path.resolve(this.applicationManager.appUserDirectory, 'models', modelId, localModelInfo[0].file); await this.playgroundManager.startPlayground(modelId, modelPath); } + async stopPlayground(modelId: string): Promise { + await this.playgroundManager.stopPlayground(modelId); + } + askPlayground(modelId: string, prompt: string): Promise { const localModelInfo = this.applicationManager.getLocalModels().filter(m => m.id === modelId); if (localModelInfo.length !== 1) { @@ -140,8 +147,12 @@ export class StudioApiImpl implements StudioAPI { return this.playgroundManager.askPlayground(localModelInfo[0], prompt); } - async getPlaygroundStates(): Promise { - return this.playgroundManager.getState(); + async getPlaygroundQueriesState(): Promise { + return this.playgroundManager.getQueriesState(); + } + + async getPlaygroundsState(): Promise { + return this.playgroundManager.getPlaygroundsState(); } async getCatalog(): Promise { diff --git a/packages/frontend/src/lib/Card.svelte b/packages/frontend/src/lib/Card.svelte index d0a0b7c7b..11ae9e1aa 100644 --- a/packages/frontend/src/lib/Card.svelte +++ b/packages/frontend/src/lib/Card.svelte @@ -1,6 +1,8 @@ -
-
Prompt
- +{#if playgroundState === undefined} + +{:else} +
+ +
+ {#key playgroundState.status} + Playground {playgroundState.status} + + {/key} +
+ +
Prompt
+ + +
+ +
-
- + {#if result} +
Output
+ + {/if}
+{/if} - {#if result} -
Output
- - {/if} -
diff --git a/packages/frontend/src/stores/playground-queries.ts b/packages/frontend/src/stores/playground-queries.ts index 9196ffd9d..dece000d9 100644 --- a/packages/frontend/src/stores/playground-queries.ts +++ b/packages/frontend/src/stores/playground-queries.ts @@ -9,7 +9,7 @@ export const playgroundQueries: Readable = readable( set(msg); }); // Initialize the store manually - studioClient.getPlaygroundStates().then(state => { + studioClient.getPlaygroundQueriesState().then(state => { set(state); }); return () => { diff --git a/packages/frontend/src/stores/playground-states.ts b/packages/frontend/src/stores/playground-states.ts new file mode 100644 index 000000000..934ef6262 --- /dev/null +++ b/packages/frontend/src/stores/playground-states.ts @@ -0,0 +1,18 @@ +import type { Readable } from 'svelte/store'; +import { readable } from 'svelte/store'; +import { MSG_PLAYGROUNDS_STATE_UPDATE } from '@shared/Messages'; +import { rpcBrowser, studioClient } from '/@/utils/client'; +import type { PlaygroundState } from '@shared/src/models/IPlaygroundState'; + +export const playgroundStates: Readable = readable([], set => { + const sub = rpcBrowser.subscribe(MSG_PLAYGROUNDS_STATE_UPDATE, msg => { + set(msg); + }); + // Initialize the store manually + studioClient.getPlaygroundsState().then(state => { + set(state); + }); + return () => { + sub.unsubscribe(); + }; +}); diff --git a/packages/shared/Messages.ts b/packages/shared/Messages.ts index ede982299..a0a31767d 100644 --- a/packages/shared/Messages.ts +++ b/packages/shared/Messages.ts @@ -1,2 +1,3 @@ +export const MSG_PLAYGROUNDS_STATE_UPDATE = 'playgrounds-state-update'; export const MSG_NEW_PLAYGROUND_QUERIES_STATE = 'new-playground-queries-state'; export const MSG_NEW_CATALOG_STATE = 'new-catalog-state'; diff --git a/packages/shared/src/StudioAPI.ts b/packages/shared/src/StudioAPI.ts index 03edf8e3e..300ea8f23 100644 --- a/packages/shared/src/StudioAPI.ts +++ b/packages/shared/src/StudioAPI.ts @@ -3,6 +3,7 @@ import type { ModelInfo } from './models/IModelInfo'; import type { Task } from './models/ITask'; import type { QueryState } from './models/IPlaygroundQueryState'; import type { Catalog } from './models/ICatalog'; +import type { PlaygroundState } from './models/IPlaygroundState'; export abstract class StudioAPI { abstract ping(): Promise; @@ -16,6 +17,7 @@ export abstract class StudioAPI { abstract getLocalModels(): Promise; abstract startPlayground(modelId: string): Promise; + abstract stopPlayground(modelId: string): Promise; abstract askPlayground(modelId: string, prompt: string): Promise; /** @@ -24,8 +26,7 @@ export abstract class StudioAPI { */ abstract getTasksByLabel(label: string): Promise; - /** - * Ask to send a message MSG_NEW_PLAYGROUND_QUERIES_STATE with the current Playground queries - */ - abstract getPlaygroundStates(): Promise; + abstract getPlaygroundQueriesState(): Promise; + + abstract getPlaygroundsState(): Promise; } diff --git a/packages/shared/src/models/IPlaygroundQueryState.ts b/packages/shared/src/models/IPlaygroundQueryState.ts index 9f8ae9349..3c05295c2 100644 --- a/packages/shared/src/models/IPlaygroundQueryState.ts +++ b/packages/shared/src/models/IPlaygroundQueryState.ts @@ -1,5 +1,7 @@ import type { ModelResponse } from './IModelResponse'; +type PlaygroundStatus = 'none' | 'stopped' | 'starting' | 'stopping' | 'running' | 'error' + export interface QueryState { id: number; modelId: string; diff --git a/packages/shared/src/models/IPlaygroundState.ts b/packages/shared/src/models/IPlaygroundState.ts new file mode 100644 index 000000000..35b46f86b --- /dev/null +++ b/packages/shared/src/models/IPlaygroundState.ts @@ -0,0 +1,10 @@ +export type PlaygroundStatus = 'none' | 'stopped' | 'running' | 'starting' | 'stopping' | 'error'; + +export interface PlaygroundState { + container?: { + containerId: string; + port: number; + }, + modelId: string, + status: PlaygroundStatus +} From ae9bb387f299998fa14c6e28ac0e26d29a1deb54 Mon Sep 17 00:00:00 2001 From: axel7083 <42176370+axel7083@users.noreply.github.com> Date: Fri, 19 Jan 2024 11:34:30 +0100 Subject: [PATCH 2/9] feat: improve playground state management Signed-off-by: axel7083 <42176370+axel7083@users.noreply.github.com> --- packages/backend/src/managers/playground.ts | 34 +++++-- .../src/pages/ModelPlayground.spec.ts | 4 + .../frontend/src/pages/ModelPlayground.svelte | 99 ++++++++++++------- .../shared/src/models/IPlaygroundState.ts | 1 + 4 files changed, 93 insertions(+), 45 deletions(-) diff --git a/packages/backend/src/managers/playground.ts b/packages/backend/src/managers/playground.ts index ea60c8dc4..b3d8629fc 100644 --- a/packages/backend/src/managers/playground.ts +++ b/packages/backend/src/managers/playground.ts @@ -31,7 +31,7 @@ import * as http from 'node:http'; import { getFreePort } from '../utils/ports'; import type { QueryState } from '@shared/src/models/IPlaygroundQueryState'; import { MSG_NEW_PLAYGROUND_QUERIES_STATE, MSG_PLAYGROUNDS_STATE_UPDATE } from '@shared/Messages'; -import { PlaygroundState } from '@shared/src/models/IPlaygroundState'; +import { PlaygroundState, PlaygroundStatus } from '@shared/src/models/IPlaygroundState'; // TODO: this should not be hardcoded const LOCALAI_IMAGE = 'quay.io/go-skynet/local-ai:v2.5.1'; @@ -61,6 +61,14 @@ export class PlayGroundManager { return images.length > 0 ? images[0] : undefined; } + setPlaygroundStatus(modelId: string, status: PlaygroundStatus) { + this.updatePlaygroundState(modelId, { + modelId: modelId, + ...(this.playgrounds.get(modelId) || {}), + status: status + }); + } + updatePlaygroundState(modelId: string, state: PlaygroundState) { this.playgrounds.set(modelId, state); return this.webview.postMessage({ @@ -75,11 +83,11 @@ export class PlayGroundManager { throw new Error('model is already running'); } - this.updatePlaygroundState(modelId, { status: 'starting', modelId }); + this.setPlaygroundStatus(modelId, 'starting'); const connection = findFirstProvider(); if (!connection) { - this.updatePlaygroundState(modelId, { status: 'error', modelId }); + this.setPlaygroundStatus(modelId, 'error'); throw new Error('Unable to find an engine to start playground'); } @@ -88,7 +96,7 @@ export class PlayGroundManager { await containerEngine.pullImage(connection.connection, LOCALAI_IMAGE, () => {}); image = await this.selectImage(connection, LOCALAI_IMAGE); if (!image) { - this.updatePlaygroundState(modelId, { status: 'error', modelId }); + this.setPlaygroundStatus(modelId, 'error'); throw new Error(`Unable to find ${LOCALAI_IMAGE} image`); } } @@ -122,6 +130,7 @@ export class PlayGroundManager { container: { containerId: result.id, port: freePort, + engineId: image.engineId, }, status: 'running', modelId @@ -130,12 +139,19 @@ export class PlayGroundManager { return result.id; } - async stopPlayground(playgroundId: string): Promise { - const connection = findFirstProvider(); - if (!connection) { - throw new Error('Unable to find an engine to start playground'); + async stopPlayground(modelId: string): Promise { + const state = this.playgrounds.get(modelId); + if (!state || !state.container) { + throw new Error('model is not running'); } - return containerEngine.stopContainer(connection.providerId, playgroundId); + this.setPlaygroundStatus(modelId, 'stopping'); + // We do not await since it can take a lot of time + containerEngine.stopContainer(state.container.engineId, state.container.containerId) + .then(() => { + this.setPlaygroundStatus(modelId, 'stopped'); + }).catch(e => { + this.setPlaygroundStatus(modelId, 'error'); + }) } async askPlayground(modelInfo: LocalModelInfo, prompt: string): Promise { diff --git a/packages/frontend/src/pages/ModelPlayground.spec.ts b/packages/frontend/src/pages/ModelPlayground.spec.ts index dd450ca02..07c8d7cbb 100644 --- a/packages/frontend/src/pages/ModelPlayground.spec.ts +++ b/packages/frontend/src/pages/ModelPlayground.spec.ts @@ -144,3 +144,7 @@ test('should display query without response', async () => { expect(response).toBeInTheDocument(); expect(response).toHaveValue('The response is 2'); }); + +test('', async () => { + +}); diff --git a/packages/frontend/src/pages/ModelPlayground.svelte b/packages/frontend/src/pages/ModelPlayground.svelte index 9c78919e8..fed2257b2 100644 --- a/packages/frontend/src/pages/ModelPlayground.svelte +++ b/packages/frontend/src/pages/ModelPlayground.svelte @@ -8,11 +8,10 @@ import type { QueryState } from '@shared/models/IPlaygroundQueryState'; import { playgroundStates } from '/@/stores/playground-states'; import type { PlaygroundState } from '@shared/src/models/IPlaygroundState'; - import LinearProgress from '/@/lib/progress/LinearProgress.svelte'; import Card from '/@/lib/Card.svelte'; export let model: ModelInfo | undefined; import Fa from 'svelte-fa'; - import { faPlay, faPause, faInfo, faWarning } from '@fortawesome/free-solid-svg-icons'; + import { faPlay, faStop, faInfo, faWarning } from '@fortawesome/free-solid-svg-icons'; let prompt = ''; let queryId: number; @@ -88,7 +87,7 @@ case "stopped": return faPlay; case "running": - return faPause + return faStop case "starting": case "stopping": return faInfo; @@ -116,42 +115,70 @@ return faWarning; } } + + const isPromptable = () => { + if(playgroundState === undefined) + return false; + + switch (playgroundState.status) { + case "none": + case "stopped": + case "error": + case "starting": + case "stopping": + return false; + case "running": + return true; + } + } + + const isLoading = () => { + if(playgroundState === undefined) + return true; + + switch (playgroundState.status) { + case "none": + case "stopped": + case "running": + case "error": + return false; + case "starting": + case "stopping": + return true; + } + } -{#if playgroundState === undefined} - -{:else} -
- -
- {#key playgroundState.status} - Playground {playgroundState.status} - - {/key} -
- -
Prompt
- - -
- +
+ +
+ {#key playgroundState?.status} + Playground {playgroundState?.status} +
+ +
Prompt
+ - {#if result} -
Output
- - {/if} +
+
-{/if} + + {#if result} +
Output
+ + {/if} +
diff --git a/packages/shared/src/models/IPlaygroundState.ts b/packages/shared/src/models/IPlaygroundState.ts index 35b46f86b..9c9ae9f40 100644 --- a/packages/shared/src/models/IPlaygroundState.ts +++ b/packages/shared/src/models/IPlaygroundState.ts @@ -4,6 +4,7 @@ export interface PlaygroundState { container?: { containerId: string; port: number; + engineId: string; }, modelId: string, status: PlaygroundStatus From f7a4408f5ad408defeb1f88ea40995c2ff3f196d Mon Sep 17 00:00:00 2001 From: axel7083 <42176370+axel7083@users.noreply.github.com> Date: Fri, 19 Jan 2024 11:35:29 +0100 Subject: [PATCH 3/9] fix: prettier Signed-off-by: axel7083 <42176370+axel7083@users.noreply.github.com> --- packages/backend/src/managers/playground.ts | 14 ++++++++------ .../frontend/src/pages/ModelPlayground.spec.ts | 4 +--- .../shared/src/models/IPlaygroundQueryState.ts | 2 +- packages/shared/src/models/IPlaygroundState.ts | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/backend/src/managers/playground.ts b/packages/backend/src/managers/playground.ts index b3d8629fc..b8eb204cf 100644 --- a/packages/backend/src/managers/playground.ts +++ b/packages/backend/src/managers/playground.ts @@ -65,7 +65,7 @@ export class PlayGroundManager { this.updatePlaygroundState(modelId, { modelId: modelId, ...(this.playgrounds.get(modelId) || {}), - status: status + status: status, }); } @@ -133,7 +133,7 @@ export class PlayGroundManager { engineId: image.engineId, }, status: 'running', - modelId + modelId, }); return result.id; @@ -146,12 +146,14 @@ export class PlayGroundManager { } this.setPlaygroundStatus(modelId, 'stopping'); // We do not await since it can take a lot of time - containerEngine.stopContainer(state.container.engineId, state.container.containerId) + containerEngine + .stopContainer(state.container.engineId, state.container.containerId) .then(() => { this.setPlaygroundStatus(modelId, 'stopped'); - }).catch(e => { - this.setPlaygroundStatus(modelId, 'error'); - }) + }) + .catch(e => { + this.setPlaygroundStatus(modelId, 'error'); + }); } async askPlayground(modelInfo: LocalModelInfo, prompt: string): Promise { diff --git a/packages/frontend/src/pages/ModelPlayground.spec.ts b/packages/frontend/src/pages/ModelPlayground.spec.ts index 07c8d7cbb..21e264289 100644 --- a/packages/frontend/src/pages/ModelPlayground.spec.ts +++ b/packages/frontend/src/pages/ModelPlayground.spec.ts @@ -145,6 +145,4 @@ test('should display query without response', async () => { expect(response).toHaveValue('The response is 2'); }); -test('', async () => { - -}); +test('', async () => {}); diff --git a/packages/shared/src/models/IPlaygroundQueryState.ts b/packages/shared/src/models/IPlaygroundQueryState.ts index 3c05295c2..77269dc91 100644 --- a/packages/shared/src/models/IPlaygroundQueryState.ts +++ b/packages/shared/src/models/IPlaygroundQueryState.ts @@ -1,6 +1,6 @@ import type { ModelResponse } from './IModelResponse'; -type PlaygroundStatus = 'none' | 'stopped' | 'starting' | 'stopping' | 'running' | 'error' +type PlaygroundStatus = 'none' | 'stopped' | 'starting' | 'stopping' | 'running' | 'error'; export interface QueryState { id: number; diff --git a/packages/shared/src/models/IPlaygroundState.ts b/packages/shared/src/models/IPlaygroundState.ts index 9c9ae9f40..9ff588228 100644 --- a/packages/shared/src/models/IPlaygroundState.ts +++ b/packages/shared/src/models/IPlaygroundState.ts @@ -5,7 +5,7 @@ export interface PlaygroundState { containerId: string; port: number; engineId: string; - }, - modelId: string, - status: PlaygroundStatus + }; + modelId: string; + status: PlaygroundStatus; } From 98512a6b6adefd865e32c709036161d2ac6964b0 Mon Sep 17 00:00:00 2001 From: axel7083 <42176370+axel7083@users.noreply.github.com> Date: Fri, 19 Jan 2024 13:58:29 +0100 Subject: [PATCH 4/9] fix: remove useless methods Signed-off-by: axel7083 <42176370+axel7083@users.noreply.github.com> --- packages/backend/src/studio-api-impl.ts | 40 ++----------------------- 1 file changed, 3 insertions(+), 37 deletions(-) diff --git a/packages/backend/src/studio-api-impl.ts b/packages/backend/src/studio-api-impl.ts index af0b94f18..6724da360 100644 --- a/packages/backend/src/studio-api-impl.ts +++ b/packages/backend/src/studio-api-impl.ts @@ -17,8 +17,6 @@ ***********************************************************************/ import type { StudioAPI } from '@shared/src/StudioAPI'; -import type { Category } from '@shared/src/models/ICategory'; -import type { Recipe } from '@shared/src/models/IRecipe'; import type { ApplicationManager } from './managers/applicationManager'; import type { RecipeStatusRegistry } from './registries/RecipeStatusRegistry'; import type { RecipeStatus } from '@shared/src/models/IRecipeStatus'; @@ -34,8 +32,6 @@ import type { CatalogManager } from './managers/catalogManager'; import type { Catalog } from '@shared/src/models/ICatalog'; import { PlaygroundState } from '@shared/src/models/IPlaygroundState'; -export const RECENT_CATEGORY_ID = 'recent-category'; - export class StudioApiImpl implements StudioAPI { constructor( private applicationManager: ApplicationManager, @@ -57,28 +53,6 @@ export class StudioApiImpl implements StudioAPI { return this.recipeStatusRegistry.getStatus(recipeId); } - async getRecentRecipes(): Promise { - return []; // no recent implementation for now - } - - async getCategories(): Promise { - return this.catalogManager.getCategories(); - } - - async getRecipesByCategory(categoryId: string): Promise { - if (categoryId === RECENT_CATEGORY_ID) return this.getRecentRecipes(); - - // TODO: move logic to catalog manager - return this.catalogManager.getRecipes().filter(recipe => recipe.categories.includes(categoryId)); - } - - async getRecipeById(recipeId: string): Promise { - // TODO: move logic to catalog manager - const recipe = this.catalogManager.getRecipes().find(recipe => recipe.id === recipeId); - if (recipe) return recipe; - throw new Error('Not found'); - } - async getModelById(modelId: string): Promise { // TODO: move logic to catalog manager const model = this.catalogManager.getModels().find(m => modelId === m.id); @@ -88,18 +62,10 @@ export class StudioApiImpl implements StudioAPI { return model; } - async getModelsByIds(ids: string[]): Promise { - // TODO: move logic to catalog manager - return this.catalogManager.getModels().filter(m => ids.includes(m.id)) ?? []; - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - async searchRecipes(_query: string): Promise { - return []; // todo: not implemented - } - async pullApplication(recipeId: string): Promise { - const recipe: Recipe = await this.getRecipeById(recipeId); + const recipe = this.catalogManager.getRecipes().find(recipe => recipe.id === recipeId); + if (!recipe) + throw new Error('Not found'); // the user should have selected one model, we use the first one for the moment const modelId = recipe.models[0]; From 57de3e56750736a497a0cc8be37e6e9fee21fa8d Mon Sep 17 00:00:00 2001 From: axel7083 <42176370+axel7083@users.noreply.github.com> Date: Fri, 19 Jan 2024 14:04:49 +0100 Subject: [PATCH 5/9] fix: prettier Signed-off-by: axel7083 <42176370+axel7083@users.noreply.github.com> --- packages/backend/src/managers/playground.ts | 29 ++++++++++--------- packages/backend/src/studio-api-impl.ts | 5 ++-- .../src/models/IPlaygroundQueryState.ts | 2 -- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/packages/backend/src/managers/playground.ts b/packages/backend/src/managers/playground.ts index b8eb204cf..877915f0d 100644 --- a/packages/backend/src/managers/playground.ts +++ b/packages/backend/src/managers/playground.ts @@ -31,7 +31,7 @@ import * as http from 'node:http'; import { getFreePort } from '../utils/ports'; import type { QueryState } from '@shared/src/models/IPlaygroundQueryState'; import { MSG_NEW_PLAYGROUND_QUERIES_STATE, MSG_PLAYGROUNDS_STATE_UPDATE } from '@shared/Messages'; -import { PlaygroundState, PlaygroundStatus } from '@shared/src/models/IPlaygroundState'; +import type { PlaygroundState, PlaygroundStatus } from '@shared/src/models/IPlaygroundState'; // TODO: this should not be hardcoded const LOCALAI_IMAGE = 'quay.io/go-skynet/local-ai:v2.5.1'; @@ -47,7 +47,7 @@ function findFirstProvider(): ProviderContainerConnection | undefined { export class PlayGroundManager { private queryIdCounter = 0; - // Map + // Dict modelId => state private playgrounds: Map; private queries: Map; @@ -62,7 +62,7 @@ export class PlayGroundManager { } setPlaygroundStatus(modelId: string, status: PlaygroundStatus) { - this.updatePlaygroundState(modelId, { + return this.updatePlaygroundState(modelId, { modelId: modelId, ...(this.playgrounds.get(modelId) || {}), status: status, @@ -83,11 +83,11 @@ export class PlayGroundManager { throw new Error('model is already running'); } - this.setPlaygroundStatus(modelId, 'starting'); + await this.setPlaygroundStatus(modelId, 'starting'); const connection = findFirstProvider(); if (!connection) { - this.setPlaygroundStatus(modelId, 'error'); + await this.setPlaygroundStatus(modelId, 'error'); throw new Error('Unable to find an engine to start playground'); } @@ -96,7 +96,7 @@ export class PlayGroundManager { await containerEngine.pullImage(connection.connection, LOCALAI_IMAGE, () => {}); image = await this.selectImage(connection, LOCALAI_IMAGE); if (!image) { - this.setPlaygroundStatus(modelId, 'error'); + await this.setPlaygroundStatus(modelId, 'error'); throw new Error(`Unable to find ${LOCALAI_IMAGE} image`); } } @@ -126,7 +126,7 @@ export class PlayGroundManager { Cmd: ['--models-path', '/models', '--context-size', '700', '--threads', '4'], }); - this.updatePlaygroundState(modelId, { + await this.updatePlaygroundState(modelId, { container: { containerId: result.id, port: freePort, @@ -141,24 +141,25 @@ export class PlayGroundManager { async stopPlayground(modelId: string): Promise { const state = this.playgrounds.get(modelId); - if (!state || !state.container) { + if (state?.container === undefined) { throw new Error('model is not running'); } - this.setPlaygroundStatus(modelId, 'stopping'); + await this.setPlaygroundStatus(modelId, 'stopping'); // We do not await since it can take a lot of time containerEngine .stopContainer(state.container.engineId, state.container.containerId) - .then(() => { - this.setPlaygroundStatus(modelId, 'stopped'); + .then(async () => { + await this.setPlaygroundStatus(modelId, 'stopped'); }) - .catch(e => { - this.setPlaygroundStatus(modelId, 'error'); + .catch(async (error: unknown) => { + console.error(error); + await this.setPlaygroundStatus(modelId, 'error'); }); } async askPlayground(modelInfo: LocalModelInfo, prompt: string): Promise { const state = this.playgrounds.get(modelInfo.id); - if (!state || !state.container) { + if (state?.container === undefined) { throw new Error('model is not running'); } diff --git a/packages/backend/src/studio-api-impl.ts b/packages/backend/src/studio-api-impl.ts index 6724da360..a36f268e5 100644 --- a/packages/backend/src/studio-api-impl.ts +++ b/packages/backend/src/studio-api-impl.ts @@ -30,7 +30,7 @@ import type { QueryState } from '@shared/src/models/IPlaygroundQueryState'; import * as path from 'node:path'; import type { CatalogManager } from './managers/catalogManager'; import type { Catalog } from '@shared/src/models/ICatalog'; -import { PlaygroundState } from '@shared/src/models/IPlaygroundState'; +import type { PlaygroundState } from '@shared/src/models/IPlaygroundState'; export class StudioApiImpl implements StudioAPI { constructor( @@ -64,8 +64,7 @@ export class StudioApiImpl implements StudioAPI { async pullApplication(recipeId: string): Promise { const recipe = this.catalogManager.getRecipes().find(recipe => recipe.id === recipeId); - if (!recipe) - throw new Error('Not found'); + if (!recipe) throw new Error('Not found'); // the user should have selected one model, we use the first one for the moment const modelId = recipe.models[0]; diff --git a/packages/shared/src/models/IPlaygroundQueryState.ts b/packages/shared/src/models/IPlaygroundQueryState.ts index 77269dc91..9f8ae9349 100644 --- a/packages/shared/src/models/IPlaygroundQueryState.ts +++ b/packages/shared/src/models/IPlaygroundQueryState.ts @@ -1,7 +1,5 @@ import type { ModelResponse } from './IModelResponse'; -type PlaygroundStatus = 'none' | 'stopped' | 'starting' | 'stopping' | 'running' | 'error'; - export interface QueryState { id: number; modelId: string; From c5ffa4bcaa6488d76b2ab0a44f2b3cd12ded46a5 Mon Sep 17 00:00:00 2001 From: axel7083 <42176370+axel7083@users.noreply.github.com> Date: Fri, 19 Jan 2024 14:09:39 +0100 Subject: [PATCH 6/9] fix: restart playground Signed-off-by: axel7083 <42176370+axel7083@users.noreply.github.com> --- packages/backend/src/managers/playground.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/managers/playground.ts b/packages/backend/src/managers/playground.ts index 877915f0d..ab4357318 100644 --- a/packages/backend/src/managers/playground.ts +++ b/packages/backend/src/managers/playground.ts @@ -80,7 +80,18 @@ export class PlayGroundManager { async startPlayground(modelId: string, modelPath: string): Promise { // TODO(feloy) remove previous query from state? if (this.playgrounds.has(modelId)) { - throw new Error('model is already running'); + // TODO: check manually if the contains has a matching state + switch (this.playgrounds.get(modelId).status) { + case "running": + throw new Error('playground is already running'); + case "starting": + case "stopping": + throw new Error('playground is transitioning'); + case "error": + case "none": + case "stopped": + break; + } } await this.setPlaygroundStatus(modelId, 'starting'); From 530d54590bfba873bfaeeda59eb9aa48b4a7e2c8 Mon Sep 17 00:00:00 2001 From: axel7083 <42176370+axel7083@users.noreply.github.com> Date: Fri, 19 Jan 2024 14:14:27 +0100 Subject: [PATCH 7/9] fix: restart playground Signed-off-by: axel7083 <42176370+axel7083@users.noreply.github.com> --- packages/frontend/src/pages/ModelPlayground.svelte | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/frontend/src/pages/ModelPlayground.svelte b/packages/frontend/src/pages/ModelPlayground.svelte index fed2257b2..ba9855cbd 100644 --- a/packages/frontend/src/pages/ModelPlayground.svelte +++ b/packages/frontend/src/pages/ModelPlayground.svelte @@ -167,7 +167,9 @@ placeholder="Type your prompt here">
- + {#key playgroundState?.status} + + {/key}
{#if result} From df9a90b5e91b0a1d8a089b82a78e9d2534a42986 Mon Sep 17 00:00:00 2001 From: axel7083 <42176370+axel7083@users.noreply.github.com> Date: Fri, 19 Jan 2024 14:16:09 +0100 Subject: [PATCH 8/9] fix: prettier fix Signed-off-by: axel7083 <42176370+axel7083@users.noreply.github.com> --- packages/backend/src/managers/playground.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/managers/playground.ts b/packages/backend/src/managers/playground.ts index ab4357318..22645f962 100644 --- a/packages/backend/src/managers/playground.ts +++ b/packages/backend/src/managers/playground.ts @@ -82,14 +82,14 @@ export class PlayGroundManager { if (this.playgrounds.has(modelId)) { // TODO: check manually if the contains has a matching state switch (this.playgrounds.get(modelId).status) { - case "running": + case 'running': throw new Error('playground is already running'); - case "starting": - case "stopping": + case 'starting': + case 'stopping': throw new Error('playground is transitioning'); - case "error": - case "none": - case "stopped": + case 'error': + case 'none': + case 'stopped': break; } } From 84160a418ae8db8e7ef0bf08eb34099dd481af62 Mon Sep 17 00:00:00 2001 From: axel7083 <42176370+axel7083@users.noreply.github.com> Date: Fri, 19 Jan 2024 14:47:20 +0100 Subject: [PATCH 9/9] test: fixing model playground tests Signed-off-by: axel7083 <42176370+axel7083@users.noreply.github.com> --- .../src/pages/ModelPlayground.spec.ts | 46 ++++++++----------- .../frontend/src/pages/ModelPlayground.svelte | 2 +- 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/packages/frontend/src/pages/ModelPlayground.spec.ts b/packages/frontend/src/pages/ModelPlayground.spec.ts index 21e264289..14617bd68 100644 --- a/packages/frontend/src/pages/ModelPlayground.spec.ts +++ b/packages/frontend/src/pages/ModelPlayground.spec.ts @@ -1,14 +1,14 @@ import '@testing-library/jest-dom/vitest'; import { vi, test, expect, beforeEach } from 'vitest'; -import { screen, fireEvent, render } from '@testing-library/svelte'; +import { screen, fireEvent, render, waitFor } from '@testing-library/svelte'; import ModelPlayground from './ModelPlayground.svelte'; import type { ModelInfo } from '@shared/src/models/IModelInfo'; -import userEvent from '@testing-library/user-event'; const mocks = vi.hoisted(() => { return { startPlaygroundMock: vi.fn(), askPlaygroundMock: vi.fn(), + getPlaygroundsStateMock: vi.fn().mockImplementation(() => Promise.resolve([])), playgroundQueriesSubscribeMock: vi.fn(), playgroundQueriesMock: { subscribe: (f: (msg: any) => void) => { @@ -22,6 +22,7 @@ const mocks = vi.hoisted(() => { vi.mock('../utils/client', async () => { return { studioClient: { + getPlaygroundsState: mocks.getPlaygroundsStateMock, startPlayground: mocks.startPlaygroundMock, askPlayground: mocks.askPlaygroundMock, askPlaygroundQueries: () => {}, @@ -46,7 +47,7 @@ beforeEach(() => { vi.clearAllMocks(); }); -test('should start playground at init time and call askPlayground when button clicked', async () => { +test('playground should start when clicking on the play button', async () => { mocks.playgroundQueriesSubscribeMock.mockReturnValue([]); render(ModelPlayground, { model: { @@ -60,21 +61,15 @@ test('should start playground at init time and call askPlayground when button cl url: 'https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_S.gguf', } as ModelInfo, }); - await new Promise(resolve => setTimeout(resolve, 200)); - expect(mocks.startPlaygroundMock).toHaveBeenCalledOnce(); + const play = screen.getByTitle('playground-action'); + expect(play).toBeDefined(); - const prompt = screen.getByPlaceholderText('Type your prompt here'); - expect(prompt).toBeInTheDocument(); - const user = userEvent.setup(); - user.type(prompt, 'what is it?'); + await fireEvent.click(play); - const send = screen.getByRole('button', { name: 'Send Request' }); - expect(send).toBeInTheDocument(); - - expect(mocks.askPlaygroundMock).not.toHaveBeenCalled(); - await fireEvent.click(send); - expect(mocks.askPlaygroundMock).toHaveBeenCalledOnce(); + await waitFor(() => { + expect(mocks.startPlaygroundMock).toHaveBeenCalledOnce(); + }); }); test('should display query without response', async () => { @@ -97,11 +92,11 @@ test('should display query without response', async () => { url: 'https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_S.gguf', } as ModelInfo, }); - await new Promise(resolve => setTimeout(resolve, 200)); - - const prompt = screen.getByPlaceholderText('Type your prompt here'); - expect(prompt).toBeInTheDocument(); - expect(prompt).toHaveValue('what is 1+1?'); + await waitFor(() => { + const prompt = screen.getByPlaceholderText('Type your prompt here'); + expect(prompt).toBeInTheDocument(); + expect(prompt).toHaveValue('what is 1+1?'); + }); const response = screen.queryByRole('textbox', { name: 'response' }); expect(response).not.toBeInTheDocument(); @@ -134,15 +129,14 @@ test('should display query without response', async () => { url: 'https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_S.gguf', } as ModelInfo, }); - await new Promise(resolve => setTimeout(resolve, 200)); - const prompt = screen.getByPlaceholderText('Type your prompt here'); - expect(prompt).toBeInTheDocument(); - expect(prompt).toHaveValue('what is 1+1?'); + await waitFor(() => { + const prompt = screen.getByPlaceholderText('Type your prompt here'); + expect(prompt).toBeInTheDocument(); + expect(prompt).toHaveValue('what is 1+1?'); + }); const response = screen.queryByRole('textbox', { name: 'response' }); expect(response).toBeInTheDocument(); expect(response).toHaveValue('The response is 2'); }); - -test('', async () => {}); diff --git a/packages/frontend/src/pages/ModelPlayground.svelte b/packages/frontend/src/pages/ModelPlayground.svelte index ba9855cbd..6331e74aa 100644 --- a/packages/frontend/src/pages/ModelPlayground.svelte +++ b/packages/frontend/src/pages/ModelPlayground.svelte @@ -154,7 +154,7 @@
{#key playgroundState?.status} Playground {playgroundState?.status} -