From e94d91a0c4281e54bd477052caa17941a263d8c1 Mon Sep 17 00:00:00 2001 From: Jeff MAURY Date: Fri, 2 Feb 2024 11:30:19 +0100 Subject: [PATCH] feat: use streaming in playground requests Fixes #23 Signed-off-by: Jeff MAURY --- packages/backend/package.json | 1 + packages/backend/src/managers/playground.ts | 60 +++---- .../frontend/src/pages/ModelPlayground.svelte | 13 +- packages/shared/src/models/IModelResponse.ts | 2 +- yarn.lock | 158 ++++++++++++++++++ 5 files changed, 185 insertions(+), 49 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 1642a89f3..79cb9ba77 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -38,6 +38,7 @@ "watch": "vite --mode development build -w" }, "dependencies": { + "openai": "^4.26.0", "simple-git": "^3.22.0" }, "devDependencies": { diff --git a/packages/backend/src/managers/playground.ts b/packages/backend/src/managers/playground.ts index 6c7de8f8e..ec5d3f109 100644 --- a/packages/backend/src/managers/playground.ts +++ b/packages/backend/src/managers/playground.ts @@ -33,6 +33,9 @@ import type { QueryState } from '@shared/src/models/IPlaygroundQueryState'; import { MSG_NEW_PLAYGROUND_QUERIES_STATE, MSG_PLAYGROUNDS_STATE_UPDATE } from '@shared/Messages'; import type { PlaygroundState, PlaygroundStatus } from '@shared/src/models/IPlaygroundState'; import type { ContainerRegistry } from '../registries/ContainerRegistry'; +import OpenAI from 'openai'; +import { Completion } from 'openai/resources'; +import { Stream } from 'openai/streaming'; // TODO: this should not be hardcoded const PLAYGROUND_IMAGE = 'quay.io/bootsy/playground:v0'; @@ -207,54 +210,31 @@ export class PlayGroundManager { prompt: prompt, }; - const post_data = JSON.stringify({ + const client = new OpenAI({ baseURL: `http://localhost:${state.container.port}/v1` }); + + const response = await client.completions.create({ model: modelInfo.file, - prompt: prompt, + prompt, temperature: 0.7, max_tokens: 1024, + stream: true, }); - const post_options: http.RequestOptions = { - host: 'localhost', - port: '' + state.container.port, - path: '/v1/completions', - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - }; + this.queries.set(query.id, query); + this.sendQueriesState(); - const post_req = http.request(post_options, res => { - res.setEncoding('utf8'); - const chunks = []; - res.on('data', data => chunks.push(data)); - res.on('end', () => { - const resBody = chunks.join(); - if (res.headers['content-type'] === 'application/json') { - const result = JSON.parse(resBody); - const q = this.queries.get(query.id); - if (!q) { - throw new Error('query not found in state'); - } - q.error = undefined; - q.response = result as ModelResponse; - this.queries.set(query.id, q); - this.sendQueriesState(); + (async () => { + for await (const chunk of response) { + query.error = undefined; + if (query.response) { + query.response.choices = query.response.choices.concat(chunk.choices); + } else { + query.response = chunk; } - }); - }); - // post the data - post_req.write(post_data); - post_req.end(); - post_req.on('error', error => { - console.error('connection on error.', error); - const q = this.queries.get(query.id); - q.error = `Something went wrong while trying to request model.${String(error)}`; - this.sendQueriesState(); - }); + this.sendQueriesState(); + } + })().catch((err: unknown) => console.warn(`Error while reading streamed response for model ${modelInfo.id}`, err)); - this.queries.set(query.id, query); - this.sendQueriesState(); return query.id; } diff --git a/packages/frontend/src/pages/ModelPlayground.svelte b/packages/frontend/src/pages/ModelPlayground.svelte index 96b317f3b..7ee772f26 100644 --- a/packages/frontend/src/pages/ModelPlayground.svelte +++ b/packages/frontend/src/pages/ModelPlayground.svelte @@ -1,6 +1,5 @@