From f69245adaa0459d0f73bd22d66326c7fcdef9eab Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Thu, 3 Aug 2023 01:40:50 -0700 Subject: [PATCH 01/17] Partial Azure OpenAI Service support --- .env.example | 4 + next.config.js | 1 + src/common/types/env.d.ts | 5 + src/modules/llms/azure/AzureIcon.tsx | 13 ++ src/modules/llms/azure/AzureSourceSetup.tsx | 87 +++++++++ src/modules/llms/azure/azure.router.ts | 192 ++++++++++++++++++++ src/modules/llms/azure/azure.vendor.ts | 93 ++++++++++ src/modules/llms/llm.types.ts | 2 +- src/modules/llms/vendor.registry.ts | 2 + src/modules/trpc/trpc.router.ts | 2 + 10 files changed, 400 insertions(+), 1 deletion(-) create mode 100644 src/modules/llms/azure/AzureIcon.tsx create mode 100644 src/modules/llms/azure/AzureSourceSetup.tsx create mode 100644 src/modules/llms/azure/azure.router.ts create mode 100644 src/modules/llms/azure/azure.vendor.ts diff --git a/.env.example b/.env.example index 328413b20..89992edab 100644 --- a/.env.example +++ b/.env.example @@ -8,6 +8,10 @@ OPENAI_API_HOST= # [Optional, Helicone] Helicone API key: https://www.helicone.ai/keys HELICONE_API_KEY= +# [Optional] Azure OpenAI Service credentials for the server-side (if set, both must be set) +AZURE_OPENAI_API_KEY= +AZURE_OPENAI_API_HOST= + # [Optional] Anthropic credentials for the server-side ANTHROPIC_API_KEY= ANTHROPIC_API_HOST= diff --git a/next.config.js b/next.config.js index ba8a21f8f..7e79905d0 100644 --- a/next.config.js +++ b/next.config.js @@ -5,6 +5,7 @@ let nextConfig = { // defaults to TRUE, unless API Keys are set at build time; this flag is used by the UI HAS_SERVER_KEYS_GOOGLE_CSE: !!process.env.GOOGLE_CLOUD_API_KEY && !!process.env.GOOGLE_CSE_ID, HAS_SERVER_KEY_ANTHROPIC: !!process.env.ANTHROPIC_API_KEY, + HAS_SERVER_KEY_AZURE_OPENAI: !!process.env.AZURE_OPENAI_API_KEY && !!process.env.AZURE_OPENAI_API_HOST, HAS_SERVER_KEY_ELEVENLABS: !!process.env.ELEVENLABS_API_KEY, HAS_SERVER_KEY_OPENAI: !!process.env.OPENAI_API_KEY, HAS_SERVER_KEY_PRODIA: !!process.env.PRODIA_API_KEY, diff --git a/src/common/types/env.d.ts b/src/common/types/env.d.ts index c7fe31c01..428c69213 100644 --- a/src/common/types/env.d.ts +++ b/src/common/types/env.d.ts @@ -10,6 +10,10 @@ declare namespace NodeJS { OPENAI_API_ORG_ID: string; OPENAI_API_HOST: string; + // LLM: Azure OpenAI + AZURE_OPENAI_API_KEY: string; + AZURE_OPENAI_API_HOST: string; + // LLM: Anthropic ANTHROPIC_API_KEY: string; ANTHROPIC_API_HOST: string; @@ -36,6 +40,7 @@ declare namespace NodeJS { // set in next.config.js and available to the client-side HAS_SERVER_KEYS_GOOGLE_CSE: boolean; HAS_SERVER_KEY_ANTHROPIC?: boolean; + HAS_SERVER_KEY_AZURE_OPENAI?: boolean; HAS_SERVER_KEY_ELEVENLABS: boolean; HAS_SERVER_KEY_OPENAI?: boolean; HAS_SERVER_KEY_PRODIA: boolean; diff --git a/src/modules/llms/azure/AzureIcon.tsx b/src/modules/llms/azure/AzureIcon.tsx new file mode 100644 index 000000000..b4d009fd7 --- /dev/null +++ b/src/modules/llms/azure/AzureIcon.tsx @@ -0,0 +1,13 @@ +import * as React from 'react'; + +import { SvgIcon } from '@mui/joy'; +import { SxProps } from '@mui/joy/styles/types'; + +export function AzureIcon(props: { sx?: SxProps }) { + return + {/**/} + + + + ; +} \ No newline at end of file diff --git a/src/modules/llms/azure/AzureSourceSetup.tsx b/src/modules/llms/azure/AzureSourceSetup.tsx new file mode 100644 index 000000000..6bb16aed0 --- /dev/null +++ b/src/modules/llms/azure/AzureSourceSetup.tsx @@ -0,0 +1,87 @@ +import * as React from 'react'; + +import { Alert, Box, Button, FormControl, FormLabel, Input, Typography } from '@mui/joy'; +import SyncIcon from '@mui/icons-material/Sync'; + +import { apiQuery } from '~/modules/trpc/trpc.client'; +import { modelDescriptionToDLLM } from '~/modules/llms/anthropic/AnthropicSourceSetup'; + +import { FormInputKey } from '~/common/components/FormInputKey'; +import { Link } from '~/common/components/Link'; +import { asValidURL } from '~/common/util/urlUtils'; +import { settingsGap } from '~/common/theme'; + +import { DModelSourceId } from '../llm.types'; +import { useModelsStore, useSourceSetup } from '../store-llms'; + +import { hasServerKeyAzure, isValidAzureApiKey, ModelVendorAzure } from './azure.vendor'; + + +export function AzureSourceSetup(props: { + sourceId: DModelSourceId +}) { + + // external state + const { + source, sourceLLMs, updateSetup, + normSetup: { azureKey, azureHost }, + } = useSourceSetup(props.sourceId, ModelVendorAzure.normalizeSetup); + + const hasModels = !!sourceLLMs.length; + const needsUserKey = !hasServerKeyAzure; + const keyValid = isValidAzureApiKey(azureKey); + const keyError = (/*needsUserKey ||*/ !!azureKey) && !keyValid; + const hostValid = !!asValidURL(azureHost); + const hostError = !!azureHost && !hostValid; + const shallFetchSucceed = azureKey ? keyValid : !needsUserKey; + + // fetch models + const { isFetching, refetch, isError, error } = apiQuery.llmAzure.listModels.useQuery({ + access: { azureHost, azureKey }, + }, { + enabled: !hasModels && shallFetchSucceed, + onSuccess: models => source && useModelsStore.getState().addLLMs(models.models.map(model => modelDescriptionToDLLM(model, source))), + staleTime: Infinity, + }); + + return + + + + Azure Endpoint + + updateSetup({ azureHost: event.target.value })} + placeholder='required: https://...' + error={hostError} + /> + + + {needsUserKey + ? !azureKey && request Key + : '✔️ already set in server'} + } + value={azureKey} onChange={value => updateSetup({ azureKey: value })} + required={needsUserKey} isError={keyError} + placeholder='...' + /> + + + + + + {isError && Issue: {error?.message || error?.toString() || 'unknown'}} + + ; +} diff --git a/src/modules/llms/azure/azure.router.ts b/src/modules/llms/azure/azure.router.ts new file mode 100644 index 000000000..e91fd03b4 --- /dev/null +++ b/src/modules/llms/azure/azure.router.ts @@ -0,0 +1,192 @@ +import { z } from 'zod'; + +import { createTRPCRouter, publicProcedure } from '~/modules/trpc/trpc.server'; +import { historySchema, modelSchema, openAIChatCompletionPayload } from '~/modules/llms/openai/openai.router'; +import { OpenAI } from '~/modules/llms/openai/openai.types'; +import { TRPCError } from '@trpc/server'; +import { listModelsOutputSchema, LLM_IF_OAI_Chat, ModelDescriptionSchema } from '~/modules/llms/llm.router'; +import { fetchJsonOrTRPCError } from '~/modules/trpc/trpc.serverutils'; + + +// input schemas + +const azureAccessSchema = z.object({ + azureKey: z.string().trim(), + azureHost: z.string().trim(), +}); + +const chatGenerateSchema = z.object({ access: azureAccessSchema, model: modelSchema, history: historySchema }); + +const listModelsSchema = z.object({ access: azureAccessSchema }); + + +// Output Schemas + +const chatGenerateOutputSchema = z.object({ + role: z.enum(['assistant', 'system', 'user']), + content: z.string(), + finish_reason: z.union([z.enum(['stop', 'length']), z.null()]), +}); + + +// Wire schemas + +const wireAzureListDeploymentsSchema = z.object({ + data: z.array(z.object({ + // scale_settings: z.object({ scale_type: z.string() }), + model: z.string(), + owner: z.enum(['organization-owner']), + id: z.string(), + status: z.enum(['succeeded']), + created_at: z.number(), + updated_at: z.number(), + object: z.literal('deployment'), + })), + object: z.literal('list'), +}); + + +export const llmAzureRouter = createTRPCRouter({ + + /** + * Chat-based message generation + */ + chatGenerate: publicProcedure + .input(chatGenerateSchema) + .output(chatGenerateOutputSchema) + .mutation(async ({ input }) => { + + const { access, model, history } = input; + + const wireCompletions = await azureOpenAIPOST( + access.azureKey, access.azureHost, + openAIChatCompletionPayload(model, history, null, null, 1, false), + '/v1/chat/completions', + ); + + // expect a single output + if (wireCompletions?.choices?.length !== 1) + throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: `[Azure OpenAI Issue] Expected 1 completion, got ${wireCompletions?.choices?.length}` }); + let { message, finish_reason } = wireCompletions.choices[0]; + + // LocalAI hack/workaround, until https://github.com/go-skynet/LocalAI/issues/788 is fixed + if (finish_reason === undefined) + finish_reason = 'stop'; + + // check for a function output + // return parseChatGenerateOutput(message as OpenAI.Wire.ChatCompletion.ResponseMessage, finish_reason); + return { + role: 'assistant', + content: 'TESTXXX', + finish_reason: finish_reason as 'stop' | 'length', + }; + }), + + + /** + * List the Azure models + */ + listModels: publicProcedure + .input(listModelsSchema) + .output(listModelsOutputSchema) + .query(async ({ input }) => { + + // fetch the Azure OpenAI models + // HACK: this method may stop working soon - see: https://github.com/openai/openai-python/issues/447#issuecomment-1730976835, + const azureModels = await azureOpenaiGET( + input.access.azureKey, input.access.azureHost, + `/openai/deployments?api-version=2023-03-15-preview`, + ); + + // take the GPT models only (e.g. no whisper) + const models = wireAzureListDeploymentsSchema.parse(azureModels).data; + for (const model of models) { + if (model.model.includes('gpt')) + console.log('model', model); + } + + // output + return { + models: [], //models.map(azureModelToModelDescription), + }; + }), + +}); + + +// this will help with adding metadata to the models +const knownAzureModels = [ + { + id: 'gpt-35-turbo', + label: '3.5-Turbo', + context: 4097, + description: 'Fair speed and smarts', + }, + { + id: 'gpt-35-turbo-16k', + label: '3.5-Turbo-16k', + context: 16384, + description: 'Fair speed and smarts, large context', + }, + { + id: 'gpt-4', + label: 'GPT-4', + context: 8192, + description: 'Insightful, big thinker, slower, pricey', + }, + { + id: 'gpt-4-32k', + label: 'GPT-4-32k', + context: 32768, + description: 'Largest context window for big problems', + }, +]; + + +function azureModelToModelDescription(model: { id: string, created_at: number, updated_at: number }): ModelDescriptionSchema { + const knownModel = knownAzureModels.find(m => m.id === model.id); + return { + id: model.id, + label: knownModel?.label || model.id, + created: model.created_at, + updated: model.updated_at || model.created_at, + description: knownModel?.description || 'Unknown model type, please let us know', + contextWindow: knownModel?.context || 2048, + interfaces: [LLM_IF_OAI_Chat], + hidden: !knownModel, + }; +} + +async function azureOpenaiGET(key: string, endpoint: string, apiPath: string /*, signal?: AbortSignal*/): Promise { + const { headers, url } = azureOpenAIAccess(key, endpoint, apiPath); + return await fetchJsonOrTRPCError(url, 'GET', headers, undefined, 'Azure OpenAI'); +} + +async function azureOpenAIPOST(key: string, endpoint: string, body: TPostBody, apiPath: string /*, signal?: AbortSignal*/): Promise { + const { headers, url } = azureOpenAIAccess(key, endpoint, apiPath); + return await fetchJsonOrTRPCError(url, 'POST', headers, body, 'Azure OpenAI'); +} + +function azureOpenAIAccess(key: string, endpoint: string, apiPath: string): { headers: HeadersInit, url: string } { + // API key + const azureKey = key || process.env.AZURE_OPENAI_API_KEY || ''; + + // API endpoint + let azureHost = endpoint || process.env.AZURE_OPENAI_API_HOST || ''; + if (!azureHost.startsWith('http')) + azureHost = `https://${azureHost}`; + if (azureHost.endsWith('/') && apiPath.startsWith('/')) + azureHost = azureHost.slice(0, -1); + + // warn if no key - only for default (non-overridden) hosts + if (!azureKey || !azureHost) + throw new Error('Missing Azure API Key or Host. Add it on the UI (Models Setup) or server side (your deployment).'); + + return { + headers: { + ...(azureKey && { 'api-key': azureKey }), + 'Content-Type': 'application/json', + }, + url: azureHost + apiPath, + }; +} diff --git a/src/modules/llms/azure/azure.vendor.ts b/src/modules/llms/azure/azure.vendor.ts new file mode 100644 index 000000000..8c0f40ab6 --- /dev/null +++ b/src/modules/llms/azure/azure.vendor.ts @@ -0,0 +1,93 @@ +import { DLLM, ModelVendor } from '../llm.types'; + +import { LLMOptionsOpenAI } from '~/modules/llms/openai/openai.vendor'; +import { OpenAILLMOptions } from '~/modules/llms/openai/OpenAILLMOptions'; + +import { AzureIcon } from './AzureIcon'; +import { AzureSourceSetup } from './AzureSourceSetup'; +import { VChatFunctionIn, VChatMessageIn, VChatMessageOrFunctionCallOut, VChatMessageOut } from '~/modules/llms/llm.client'; +import { apiAsync } from '~/modules/trpc/trpc.client'; + + +// special symbols +export const hasServerKeyAzure = !!process.env.HAS_SERVER_KEY_AZURE_OPENAI; +export const isValidAzureApiKey = (apiKey?: string) => !!apiKey && apiKey.length >= 32; + + +export interface SourceSetupAzure { + azureKey: string; + azureHost: string; +} + +/** Implementation Notes for the Azure Vendor + * + * Listing models for Azure OpenAI is complex. The "Azure OpenAI Model List" API lists every model that can + * be deployed, including numerous models that are not accessible by the user. What the users want are the + * "deployed" models. + * + * 1. To list those, there was an API available in the past, but it was removed. It was about hitting the + * "/openai/deployments?api-version=2023-03-15-preview" path on the endpoint. See: + * https://github.com/openai/openai-python/issues/447#issuecomment-1730976835 + * + * 2. Still looking for a solution - in the meantime the way to go seems to be to manyally get the full URL + * of every "Deployment" (Model) and hit the URL directly. However the user will need to fill in the full + * model sheet, as details are not available just from the URL. + * + * Work in progress... + */ +export const ModelVendorAzure: ModelVendor = { + id: 'azure', + name: 'Azure', + rank: 14, + location: 'cloud', + instanceLimit: 1, + + // components + Icon: AzureIcon, + SourceSetupComponent: AzureSourceSetup, + LLMOptionsComponent: OpenAILLMOptions, + + // functions + normalizeSetup: (partialSetup?: Partial): SourceSetupAzure => ({ + azureKey: '', + azureHost: '', + ...partialSetup, + }), + callChat: (llm: DLLM, messages: VChatMessageIn[], maxTokens?: number) => { + return azureCallChatOverloaded(llm, messages, null, maxTokens); + }, + callChatWithFunctions: () => { + throw new Error('Azure does not support functions'); + }, +}; + + +/** + * This function either returns the LLM message, or function calls, or throws a descriptive error string + */ +async function azureCallChatOverloaded( + llm: DLLM, messages: VChatMessageIn[], functions: VChatFunctionIn[] | null, maxTokens?: number, +): Promise { + // access params (source) + const azureSetup = ModelVendorAzure.normalizeSetup(llm._source.setup as Partial); + + // model params (llm) + const { llmRef, llmTemperature = 0.5, llmResponseTokens } = llm.options; + + try { + return await apiAsync.llmAzure.chatGenerate.mutate({ + access: azureSetup, + model: { + id: llmRef!, + temperature: llmTemperature, + maxTokens: maxTokens || llmResponseTokens || 1024, + }, + // functions: functions ?? undefined, + history: messages, + }) as TOut; + } catch (error: any) { + const errorMessage = error?.message || error?.toString() || 'OpenAI Chat Fetch Error'; + console.error(`openAICallChat: ${errorMessage}`); + throw new Error(errorMessage); + } +} \ No newline at end of file diff --git a/src/modules/llms/llm.types.ts b/src/modules/llms/llm.types.ts index 8d7a5d820..0c034e358 100644 --- a/src/modules/llms/llm.types.ts +++ b/src/modules/llms/llm.types.ts @@ -4,7 +4,7 @@ import type { VChatFunctionIn, VChatMessageIn, VChatMessageOrFunctionCallOut, VC export type DLLMId = string; export type DModelSourceId = string; -export type ModelVendorId = 'anthropic' | 'localai' | 'oobabooga' | 'openai' | 'openrouter'; +export type ModelVendorId = 'anthropic' | 'azure' | 'localai' | 'oobabooga' | 'openai' | 'openrouter'; /// Large Language Model - a model that can generate text diff --git a/src/modules/llms/vendor.registry.ts b/src/modules/llms/vendor.registry.ts index bad58024d..31b51c0d6 100644 --- a/src/modules/llms/vendor.registry.ts +++ b/src/modules/llms/vendor.registry.ts @@ -1,5 +1,6 @@ import { DModelSource, DModelSourceId, ModelVendor, ModelVendorId } from './llm.types'; import { ModelVendorAnthropic } from './anthropic/anthropic.vendor'; +import { ModelVendorAzure } from '~/modules/llms/azure/azure.vendor'; import { ModelVendorLocalAI } from './localai/localai.vendor'; import { ModelVendorOoobabooga } from './oobabooga/oobabooga.vendor'; import { ModelVendorOpenAI } from './openai/openai.vendor'; @@ -10,6 +11,7 @@ import { ModelVendorOpenRouter } from './openrouter/openrouter.vendor'; const MODEL_VENDOR_REGISTRY: Record = { anthropic: ModelVendorAnthropic, + azure: ModelVendorAzure, localai: ModelVendorLocalAI, oobabooga: ModelVendorOoobabooga, openai: ModelVendorOpenAI, diff --git a/src/modules/trpc/trpc.router.ts b/src/modules/trpc/trpc.router.ts index af9f411db..393ce0114 100644 --- a/src/modules/trpc/trpc.router.ts +++ b/src/modules/trpc/trpc.router.ts @@ -3,6 +3,7 @@ import { createTRPCRouter } from './trpc.server'; import { elevenlabsRouter } from '~/modules/elevenlabs/elevenlabs.router'; import { googleSearchRouter } from '~/modules/google/search.router'; import { llmAnthropicRouter } from '~/modules/llms/anthropic/anthropic.router'; +import { llmAzureRouter } from '~/modules/llms/azure/azure.router'; import { llmOpenAIRouter } from '~/modules/llms/openai/openai.router'; import { prodiaRouter } from '~/modules/prodia/prodia.router'; import { sharingRouter } from '~/modules/sharing/sharing.router'; @@ -17,6 +18,7 @@ export const appRouter = createTRPCRouter({ elevenlabs: elevenlabsRouter, googleSearch: googleSearchRouter, llmAnthropic: llmAnthropicRouter, + llmAzure: llmAzureRouter, llmOpenAI: llmOpenAIRouter, prodia: prodiaRouter, sharing: sharingRouter, From 01ea8c70915a350ccb089cad5a2952a90da5f1f6 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Fri, 22 Sep 2023 01:48:31 -0700 Subject: [PATCH 02/17] Azure: consistent naming of endpoints --- .env.example | 2 +- next.config.js | 2 +- src/common/types/env.d.ts | 2 +- src/modules/llms/azure/AzureSourceSetup.tsx | 3 +- src/modules/llms/azure/azure.router.ts | 96 ++++++++++----------- 5 files changed, 49 insertions(+), 56 deletions(-) diff --git a/.env.example b/.env.example index 89992edab..a22bae0f9 100644 --- a/.env.example +++ b/.env.example @@ -9,8 +9,8 @@ OPENAI_API_HOST= HELICONE_API_KEY= # [Optional] Azure OpenAI Service credentials for the server-side (if set, both must be set) +AZURE_OPENAI_API_ENDPOINT= AZURE_OPENAI_API_KEY= -AZURE_OPENAI_API_HOST= # [Optional] Anthropic credentials for the server-side ANTHROPIC_API_KEY= diff --git a/next.config.js b/next.config.js index 7e79905d0..0c4d4bfe9 100644 --- a/next.config.js +++ b/next.config.js @@ -5,7 +5,7 @@ let nextConfig = { // defaults to TRUE, unless API Keys are set at build time; this flag is used by the UI HAS_SERVER_KEYS_GOOGLE_CSE: !!process.env.GOOGLE_CLOUD_API_KEY && !!process.env.GOOGLE_CSE_ID, HAS_SERVER_KEY_ANTHROPIC: !!process.env.ANTHROPIC_API_KEY, - HAS_SERVER_KEY_AZURE_OPENAI: !!process.env.AZURE_OPENAI_API_KEY && !!process.env.AZURE_OPENAI_API_HOST, + HAS_SERVER_KEY_AZURE_OPENAI: !!process.env.AZURE_OPENAI_API_KEY && !!process.env.AZURE_OPENAI_API_ENDPOINT, HAS_SERVER_KEY_ELEVENLABS: !!process.env.ELEVENLABS_API_KEY, HAS_SERVER_KEY_OPENAI: !!process.env.OPENAI_API_KEY, HAS_SERVER_KEY_PRODIA: !!process.env.PRODIA_API_KEY, diff --git a/src/common/types/env.d.ts b/src/common/types/env.d.ts index 428c69213..3ed0cffbe 100644 --- a/src/common/types/env.d.ts +++ b/src/common/types/env.d.ts @@ -11,8 +11,8 @@ declare namespace NodeJS { OPENAI_API_HOST: string; // LLM: Azure OpenAI + AZURE_OPENAI_API_ENDPOINT: string; AZURE_OPENAI_API_KEY: string; - AZURE_OPENAI_API_HOST: string; // LLM: Anthropic ANTHROPIC_API_KEY: string; diff --git a/src/modules/llms/azure/AzureSourceSetup.tsx b/src/modules/llms/azure/AzureSourceSetup.tsx index 6bb16aed0..39d2c26b6 100644 --- a/src/modules/llms/azure/AzureSourceSetup.tsx +++ b/src/modules/llms/azure/AzureSourceSetup.tsx @@ -47,8 +47,9 @@ export function AzureSourceSetup(props: { return - + Azure Endpoint + deployments { + + // fetch the Azure OpenAI models + // HACK: this method may stop working soon - see: https://github.com/openai/openai-python/issues/447#issuecomment-1730976835, + const azureModels = await azureOpenaiGET( + input.access.azureKey, input.access.azureHost, + `/openai/deployments?api-version=2023-03-15-preview`, + ); + + // parse and validate output, and take the GPT models only (e.g. no 'whisper') + const models = wireAzureListDeploymentsSchema.parse(azureModels).data; + return { + models: models.filter(m => m.model.includes('gpt')).map(azureModelToModelDescription), + }; + }), + + + /* Chat-based message generation */ chatGenerate: publicProcedure .input(chatGenerateSchema) .output(chatGenerateOutputSchema) @@ -82,81 +104,51 @@ export const llmAzureRouter = createTRPCRouter({ }; }), - - /** - * List the Azure models - */ - listModels: publicProcedure - .input(listModelsSchema) - .output(listModelsOutputSchema) - .query(async ({ input }) => { - - // fetch the Azure OpenAI models - // HACK: this method may stop working soon - see: https://github.com/openai/openai-python/issues/447#issuecomment-1730976835, - const azureModels = await azureOpenaiGET( - input.access.azureKey, input.access.azureHost, - `/openai/deployments?api-version=2023-03-15-preview`, - ); - - // take the GPT models only (e.g. no whisper) - const models = wireAzureListDeploymentsSchema.parse(azureModels).data; - for (const model of models) { - if (model.model.includes('gpt')) - console.log('model', model); - } - - // output - return { - models: [], //models.map(azureModelToModelDescription), - }; - }), - }); -// this will help with adding metadata to the models -const knownAzureModels = [ +function azureModelToModelDescription(model: { id: string, model: string, created_at: number, updated_at: number }): ModelDescriptionSchema { + const knownModel = knownAzureModels.find(m => m.id === model.model); + return { + id: model.id, + label: knownModel?.label || model.id, + created: model.created_at, + updated: model.updated_at || model.created_at, + description: knownModel?.description || 'Unknown model type, please let us know', + contextWindow: knownModel?.contextWindow || 2048, + interfaces: [LLM_IF_OAI_Chat], + hidden: !knownModel, + }; +} + +const knownAzureModels: Partial[] = [ { id: 'gpt-35-turbo', label: '3.5-Turbo', - context: 4097, + contextWindow: 4097, description: 'Fair speed and smarts', }, { id: 'gpt-35-turbo-16k', label: '3.5-Turbo-16k', - context: 16384, + contextWindow: 16384, description: 'Fair speed and smarts, large context', }, { id: 'gpt-4', label: 'GPT-4', - context: 8192, + contextWindow: 8192, description: 'Insightful, big thinker, slower, pricey', }, { id: 'gpt-4-32k', label: 'GPT-4-32k', - context: 32768, + contextWindow: 32768, description: 'Largest context window for big problems', }, ]; -function azureModelToModelDescription(model: { id: string, created_at: number, updated_at: number }): ModelDescriptionSchema { - const knownModel = knownAzureModels.find(m => m.id === model.id); - return { - id: model.id, - label: knownModel?.label || model.id, - created: model.created_at, - updated: model.updated_at || model.created_at, - description: knownModel?.description || 'Unknown model type, please let us know', - contextWindow: knownModel?.context || 2048, - interfaces: [LLM_IF_OAI_Chat], - hidden: !knownModel, - }; -} - async function azureOpenaiGET(key: string, endpoint: string, apiPath: string /*, signal?: AbortSignal*/): Promise { const { headers, url } = azureOpenAIAccess(key, endpoint, apiPath); return await fetchJsonOrTRPCError(url, 'GET', headers, undefined, 'Azure OpenAI'); @@ -172,7 +164,7 @@ function azureOpenAIAccess(key: string, endpoint: string, apiPath: string): { he const azureKey = key || process.env.AZURE_OPENAI_API_KEY || ''; // API endpoint - let azureHost = endpoint || process.env.AZURE_OPENAI_API_HOST || ''; + let azureHost = endpoint || process.env.AZURE_OPENAI_API_ENDPOINT || ''; if (!azureHost.startsWith('http')) azureHost = `https://${azureHost}`; if (azureHost.endsWith('/') && apiPath.startsWith('/')) From cd141048f5903e6979b20e020f138c40739da28f Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Fri, 22 Sep 2023 02:28:38 -0700 Subject: [PATCH 03/17] Azure: immediate chat calls are working - integration still WIP --- .../llms/anthropic/AnthropicSourceSetup.tsx | 28 ++-------- src/modules/llms/azure/AzureSourceSetup.tsx | 20 ++++---- src/modules/llms/azure/azure.router.ts | 51 ++++++++++--------- src/modules/llms/azure/azure.vendor.ts | 16 +++--- src/modules/llms/llm.router.ts | 25 ++++++++- src/modules/llms/llm.types.ts | 4 +- src/modules/llms/openai/openai.router.ts | 2 +- 7 files changed, 75 insertions(+), 71 deletions(-) diff --git a/src/modules/llms/anthropic/AnthropicSourceSetup.tsx b/src/modules/llms/anthropic/AnthropicSourceSetup.tsx index 686e053c0..bff5eb330 100644 --- a/src/modules/llms/anthropic/AnthropicSourceSetup.tsx +++ b/src/modules/llms/anthropic/AnthropicSourceSetup.tsx @@ -10,12 +10,11 @@ import { InlineError } from '~/common/components/InlineError'; import { Link } from '~/common/components/Link'; import { settingsGap } from '~/common/theme'; -import { DLLM, DModelSource, DModelSourceId } from '../llm.types'; -import { LLMOptionsOpenAI } from '~/modules/llms/openai/openai.vendor'; -import { ModelDescriptionSchema } from '../llm.router'; +import { DModelSourceId } from '../llm.types'; +import { modelDescriptionToDLLM } from '../llm.router'; +import { useModelsStore, useSourceSetup } from '../store-llms'; import { hasServerKeyAnthropic, isValidAnthropicApiKey, ModelVendorAnthropic } from './anthropic.vendor'; -import { useModelsStore, useSourceSetup } from '../store-llms'; export function AnthropicSourceSetup(props: { sourceId: DModelSourceId }) { @@ -70,25 +69,4 @@ export function AnthropicSourceSetup(props: { sourceId: DModelSourceId }) { {isError && } ; -} - - -export function modelDescriptionToDLLM(model: ModelDescriptionSchema, source: DModelSource): DLLM { - return { - id: `${source.id}-${model.id}`, - label: model.label, - created: model.created || 0, - updated: model.updated || 0, - description: model.description, - tags: [], // ['stream', 'chat'], - contextTokens: model.contextWindow, - hidden: !!model.hidden, - sId: source.id, - _source: source, - options: { - llmRef: model.id, - llmTemperature: 0.5, - llmResponseTokens: Math.round(model.contextWindow / 8), - }, - }; } \ No newline at end of file diff --git a/src/modules/llms/azure/AzureSourceSetup.tsx b/src/modules/llms/azure/AzureSourceSetup.tsx index 39d2c26b6..d6d5a7742 100644 --- a/src/modules/llms/azure/AzureSourceSetup.tsx +++ b/src/modules/llms/azure/AzureSourceSetup.tsx @@ -4,7 +4,6 @@ import { Alert, Box, Button, FormControl, FormLabel, Input, Typography } from '@ import SyncIcon from '@mui/icons-material/Sync'; import { apiQuery } from '~/modules/trpc/trpc.client'; -import { modelDescriptionToDLLM } from '~/modules/llms/anthropic/AnthropicSourceSetup'; import { FormInputKey } from '~/common/components/FormInputKey'; import { Link } from '~/common/components/Link'; @@ -12,32 +11,31 @@ import { asValidURL } from '~/common/util/urlUtils'; import { settingsGap } from '~/common/theme'; import { DModelSourceId } from '../llm.types'; +import { modelDescriptionToDLLM } from '../llm.router'; import { useModelsStore, useSourceSetup } from '../store-llms'; import { hasServerKeyAzure, isValidAzureApiKey, ModelVendorAzure } from './azure.vendor'; -export function AzureSourceSetup(props: { - sourceId: DModelSourceId -}) { +export function AzureSourceSetup(props: { sourceId: DModelSourceId }) { // external state const { source, sourceLLMs, updateSetup, - normSetup: { azureKey, azureHost }, + normSetup: { azureEndpoint, azureKey }, } = useSourceSetup(props.sourceId, ModelVendorAzure.normalizeSetup); const hasModels = !!sourceLLMs.length; const needsUserKey = !hasServerKeyAzure; const keyValid = isValidAzureApiKey(azureKey); const keyError = (/*needsUserKey ||*/ !!azureKey) && !keyValid; - const hostValid = !!asValidURL(azureHost); - const hostError = !!azureHost && !hostValid; + const hostValid = !!asValidURL(azureEndpoint); + const hostError = !!azureEndpoint && !hostValid; const shallFetchSucceed = azureKey ? keyValid : !needsUserKey; // fetch models const { isFetching, refetch, isError, error } = apiQuery.llmAzure.listModels.useQuery({ - access: { azureHost, azureKey }, + access: { azureEndpoint, azureKey }, }, { enabled: !hasModels && shallFetchSucceed, onSuccess: models => source && useModelsStore.getState().addLLMs(models.models.map(model => modelDescriptionToDLLM(model, source))), @@ -53,8 +51,8 @@ export function AzureSourceSetup(props: { updateSetup({ azureHost: event.target.value })} - placeholder='required: https://...' + value={azureEndpoint} onChange={event => updateSetup({ azureEndpoint: event.target.value })} + placeholder='https://your-resource-name.openai.azure.com/' error={hostError} /> @@ -85,4 +83,4 @@ export function AzureSourceSetup(props: { {isError && Issue: {error?.message || error?.toString() || 'unknown'}} ; -} +} \ No newline at end of file diff --git a/src/modules/llms/azure/azure.router.ts b/src/modules/llms/azure/azure.router.ts index c4d08b671..a532f9c3e 100644 --- a/src/modules/llms/azure/azure.router.ts +++ b/src/modules/llms/azure/azure.router.ts @@ -1,18 +1,19 @@ import { z } from 'zod'; +import { TRPCError } from '@trpc/server'; import { createTRPCRouter, publicProcedure } from '~/modules/trpc/trpc.server'; -import { historySchema, modelSchema, openAIChatCompletionPayload } from '~/modules/llms/openai/openai.router'; -import { OpenAI } from '~/modules/llms/openai/openai.types'; -import { TRPCError } from '@trpc/server'; -import { listModelsOutputSchema, LLM_IF_OAI_Chat, ModelDescriptionSchema } from '~/modules/llms/llm.router'; import { fetchJsonOrTRPCError } from '~/modules/trpc/trpc.serverutils'; +import { OpenAI } from '../openai/openai.types'; +import { historySchema, modelSchema, openAIChatCompletionPayload } from '../openai/openai.router'; +import { listModelsOutputSchema, LLM_IF_OAI_Chat, ModelDescriptionSchema } from '../llm.router'; + // input schemas const azureAccessSchema = z.object({ + azureEndpoint: z.string().trim(), azureKey: z.string().trim(), - azureHost: z.string().trim(), }); const chatGenerateSchema = z.object({ access: azureAccessSchema, model: modelSchema, history: historySchema }); @@ -60,7 +61,7 @@ export const llmAzureRouter = createTRPCRouter({ // fetch the Azure OpenAI models // HACK: this method may stop working soon - see: https://github.com/openai/openai-python/issues/447#issuecomment-1730976835, const azureModels = await azureOpenaiGET( - input.access.azureKey, input.access.azureHost, + input.access.azureEndpoint, input.access.azureKey, `/openai/deployments?api-version=2023-03-15-preview`, ); @@ -80,10 +81,14 @@ export const llmAzureRouter = createTRPCRouter({ const { access, model, history } = input; + // https://eoai1uc1.openai.azure.com/openai/deployments/my-gpt-35-turbo-1/chat/completions?api-version=2023-07-01-preview + // https://eoai1uc1.openai.azure.com/openai/deployments?api-version=2023-03-15-preview + const wireCompletions = await azureOpenAIPOST( - access.azureKey, access.azureHost, + access.azureEndpoint, access.azureKey, openAIChatCompletionPayload(model, history, null, null, 1, false), - '/v1/chat/completions', + // '/v1/chat/completions', + `/openai/deployments/${input.model.id}/chat/completions?api-version=2023-09-01-preview`, ); // expect a single output @@ -99,7 +104,7 @@ export const llmAzureRouter = createTRPCRouter({ // return parseChatGenerateOutput(message as OpenAI.Wire.ChatCompletion.ResponseMessage, finish_reason); return { role: 'assistant', - content: 'TESTXXX', + content: message.content || '', finish_reason: finish_reason as 'stop' | 'length', }; }), @@ -149,29 +154,29 @@ const knownAzureModels: Partial[] = [ ]; -async function azureOpenaiGET(key: string, endpoint: string, apiPath: string /*, signal?: AbortSignal*/): Promise { - const { headers, url } = azureOpenAIAccess(key, endpoint, apiPath); +async function azureOpenaiGET(endpoint: string, key: string, apiPath: string /*, signal?: AbortSignal*/): Promise { + const { headers, url } = azureOpenAIAccess(endpoint, key, apiPath); return await fetchJsonOrTRPCError(url, 'GET', headers, undefined, 'Azure OpenAI'); } -async function azureOpenAIPOST(key: string, endpoint: string, body: TPostBody, apiPath: string /*, signal?: AbortSignal*/): Promise { - const { headers, url } = azureOpenAIAccess(key, endpoint, apiPath); +async function azureOpenAIPOST(endpoint: string, key: string, body: TPostBody, apiPath: string /*, signal?: AbortSignal*/): Promise { + const { headers, url } = azureOpenAIAccess(endpoint, key, apiPath); return await fetchJsonOrTRPCError(url, 'POST', headers, body, 'Azure OpenAI'); } -function azureOpenAIAccess(key: string, endpoint: string, apiPath: string): { headers: HeadersInit, url: string } { +function azureOpenAIAccess(endpoint: string, key: string, apiPath: string): { headers: HeadersInit, url: string } { + // API endpoint + let azureEndpoint = endpoint || process.env.AZURE_OPENAI_API_ENDPOINT || ''; + if (!azureEndpoint.startsWith('http')) + azureEndpoint = `https://${azureEndpoint}`; + if (azureEndpoint.endsWith('/') && apiPath.startsWith('/')) + azureEndpoint = azureEndpoint.slice(0, -1); + // API key const azureKey = key || process.env.AZURE_OPENAI_API_KEY || ''; - // API endpoint - let azureHost = endpoint || process.env.AZURE_OPENAI_API_ENDPOINT || ''; - if (!azureHost.startsWith('http')) - azureHost = `https://${azureHost}`; - if (azureHost.endsWith('/') && apiPath.startsWith('/')) - azureHost = azureHost.slice(0, -1); - // warn if no key - only for default (non-overridden) hosts - if (!azureKey || !azureHost) + if (!azureKey || !azureEndpoint) throw new Error('Missing Azure API Key or Host. Add it on the UI (Models Setup) or server side (your deployment).'); return { @@ -179,6 +184,6 @@ function azureOpenAIAccess(key: string, endpoint: string, apiPath: string): { he ...(azureKey && { 'api-key': azureKey }), 'Content-Type': 'application/json', }, - url: azureHost + apiPath, + url: azureEndpoint + apiPath, }; } diff --git a/src/modules/llms/azure/azure.vendor.ts b/src/modules/llms/azure/azure.vendor.ts index 8c0f40ab6..8262c2e86 100644 --- a/src/modules/llms/azure/azure.vendor.ts +++ b/src/modules/llms/azure/azure.vendor.ts @@ -1,12 +1,12 @@ -import { DLLM, ModelVendor } from '../llm.types'; +import { apiAsync } from '~/modules/trpc/trpc.client'; -import { LLMOptionsOpenAI } from '~/modules/llms/openai/openai.vendor'; -import { OpenAILLMOptions } from '~/modules/llms/openai/OpenAILLMOptions'; +import { DLLM, ModelVendor } from '../llm.types'; +import { LLMOptionsOpenAI } from '../openai/openai.vendor'; +import { OpenAILLMOptions } from '../openai/OpenAILLMOptions'; +import { VChatFunctionIn, VChatMessageIn, VChatMessageOrFunctionCallOut, VChatMessageOut } from '../llm.client'; import { AzureIcon } from './AzureIcon'; import { AzureSourceSetup } from './AzureSourceSetup'; -import { VChatFunctionIn, VChatMessageIn, VChatMessageOrFunctionCallOut, VChatMessageOut } from '~/modules/llms/llm.client'; -import { apiAsync } from '~/modules/trpc/trpc.client'; // special symbols @@ -15,8 +15,8 @@ export const isValidAzureApiKey = (apiKey?: string) => !!apiKey && apiKey.length export interface SourceSetupAzure { + azureEndpoint: string; azureKey: string; - azureHost: string; } /** Implementation Notes for the Azure Vendor @@ -40,7 +40,7 @@ export const ModelVendorAzure: ModelVendor = name: 'Azure', rank: 14, location: 'cloud', - instanceLimit: 1, + instanceLimit: 2, // components Icon: AzureIcon, @@ -49,8 +49,8 @@ export const ModelVendorAzure: ModelVendor = // functions normalizeSetup: (partialSetup?: Partial): SourceSetupAzure => ({ + azureEndpoint: '', azureKey: '', - azureHost: '', ...partialSetup, }), callChat: (llm: DLLM, messages: VChatMessageIn[], maxTokens?: number) => { diff --git a/src/modules/llms/llm.router.ts b/src/modules/llms/llm.router.ts index 32d7e5848..d97155d04 100644 --- a/src/modules/llms/llm.router.ts +++ b/src/modules/llms/llm.router.ts @@ -1,5 +1,8 @@ import { z } from 'zod'; +import { DLLM, DModelSource } from './llm.types'; +import { LLMOptionsOpenAI } from './openai/openai.vendor'; + // these are constants used for model interfaces (chat, and function calls) // they're here as a preview - will be used more broadly in the future export const LLM_IF_OAI_Chat = 'oai-chat'; @@ -21,4 +24,24 @@ export type ModelDescriptionSchema = z.infer; export const listModelsOutputSchema = z.object({ models: z.array(modelDescriptionSchema), -}); \ No newline at end of file +}); + +export function modelDescriptionToDLLM(model: ModelDescriptionSchema, source: DModelSource): DLLM { + return { + id: `${source.id}-${model.id}`, + label: model.label, + created: model.created || 0, + updated: model.updated || 0, + description: model.description, + tags: [], // ['stream', 'chat'], + contextTokens: model.contextWindow, + hidden: !!model.hidden, + sId: source.id, + _source: source, + options: { + llmRef: model.id, + llmTemperature: 0.5, + llmResponseTokens: Math.round(model.contextWindow / 8), + }, + }; +} \ No newline at end of file diff --git a/src/modules/llms/llm.types.ts b/src/modules/llms/llm.types.ts index 0c034e358..6a38d8ae4 100644 --- a/src/modules/llms/llm.types.ts +++ b/src/modules/llms/llm.types.ts @@ -8,7 +8,7 @@ export type ModelVendorId = 'anthropic' | 'azure' | 'localai' | 'oobabooga' | 'o /// Large Language Model - a model that can generate text -export interface DLLM { +export interface DLLM { id: DLLMId; label: string; created: number | 0; @@ -20,7 +20,7 @@ export interface DLLM { // llm -> source sId: DModelSourceId; - _source: DModelSource; + _source: TModelSource; // llm-specific options: Partial<{ llmRef: string } & TLLMOptions>; diff --git a/src/modules/llms/openai/openai.router.ts b/src/modules/llms/openai/openai.router.ts index b540e979b..9823f793b 100644 --- a/src/modules/llms/openai/openai.router.ts +++ b/src/modules/llms/openai/openai.router.ts @@ -291,7 +291,7 @@ function parseChatGenerateFCOutput(isFunctionsCall: boolean, message: OpenAI.Wir }; } -function parseChatGenerateOutput(message: OpenAI.Wire.ChatCompletion.ResponseMessage, finish_reason: 'stop' | 'length' | null) { +export function parseChatGenerateOutput(message: OpenAI.Wire.ChatCompletion.ResponseMessage, finish_reason: 'stop' | 'length' | null) { // validate the message if (message.content === null) throw new TRPCError({ From aee6c853492218284565b04671e25f876d648a7b Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Fri, 22 Sep 2023 02:05:19 -0700 Subject: [PATCH 04/17] Llms: cleanups --- src/modules/llms/llm.client.ts | 2 +- src/modules/llms/llm.types.ts | 2 +- src/modules/llms/localai/localai.vendor.tsx | 2 +- src/modules/llms/oobabooga/oobabooga.vendor.ts | 2 +- src/modules/llms/openrouter/openrouter.vendor.tsx | 2 +- src/modules/llms/vendor.registry.ts | 7 ++----- 6 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/modules/llms/llm.client.ts b/src/modules/llms/llm.client.ts index fa7a469bc..62f4fa841 100644 --- a/src/modules/llms/llm.client.ts +++ b/src/modules/llms/llm.client.ts @@ -5,7 +5,7 @@ import type { DMessage } from '~/common/state/store-chats'; import type { ChatStreamSchema } from './openai/openai.router'; import type { DLLM, DLLMId, ModelVendor } from './llm.types'; import type { OpenAI } from './openai/openai.types'; -import { ModelVendorAnthropic, SourceSetupAnthropic } from '~/modules/llms/anthropic/anthropic.vendor'; +import { ModelVendorAnthropic, SourceSetupAnthropic } from './anthropic/anthropic.vendor'; import { ModelVendorOpenAI, SourceSetupOpenAI } from './openai/openai.vendor'; import { findVendorById } from './vendor.registry'; import { useModelsStore } from './store-llms'; diff --git a/src/modules/llms/llm.types.ts b/src/modules/llms/llm.types.ts index 6a38d8ae4..febb65df7 100644 --- a/src/modules/llms/llm.types.ts +++ b/src/modules/llms/llm.types.ts @@ -54,7 +54,7 @@ export interface ModelVendor { LLMOptionsComponent: React.ComponentType<{ llm: DLLM }>; // functions - initalizeSetup?: () => Partial; + initializeSetup?: () => TSourceSetup; normalizeSetup: (partialSetup?: Partial) => TSourceSetup; callChat: (llm: DLLM, messages: VChatMessageIn[], maxTokens?: number) => Promise; callChatWithFunctions: (llm: DLLM, messages: VChatMessageIn[], functions: VChatFunctionIn[], forceFunctionName?: string, maxTokens?: number) => Promise; diff --git a/src/modules/llms/localai/localai.vendor.tsx b/src/modules/llms/localai/localai.vendor.tsx index 3295e8298..dcb727f9f 100644 --- a/src/modules/llms/localai/localai.vendor.tsx +++ b/src/modules/llms/localai/localai.vendor.tsx @@ -23,7 +23,7 @@ export const ModelVendorLocalAI: ModelVendor ({ + initializeSetup: () => ({ oaiHost: 'http://localhost:8080', }), normalizeSetup: (partialSetup?: Partial) => ({ diff --git a/src/modules/llms/oobabooga/oobabooga.vendor.ts b/src/modules/llms/oobabooga/oobabooga.vendor.ts index 60552638a..94c886a60 100644 --- a/src/modules/llms/oobabooga/oobabooga.vendor.ts +++ b/src/modules/llms/oobabooga/oobabooga.vendor.ts @@ -23,7 +23,7 @@ export const ModelVendorOoobabooga: ModelVendor ({ + initializeSetup: () => ({ oaiHost: 'http://127.0.0.1:5001', }), normalizeSetup: (partialSetup?: Partial) => ({ diff --git a/src/modules/llms/openrouter/openrouter.vendor.tsx b/src/modules/llms/openrouter/openrouter.vendor.tsx index e39ca4a2f..e113bffcd 100644 --- a/src/modules/llms/openrouter/openrouter.vendor.tsx +++ b/src/modules/llms/openrouter/openrouter.vendor.tsx @@ -40,7 +40,7 @@ export const ModelVendorOpenRouter: ModelVendor ({ + initializeSetup: () => ({ oaiHost: 'https://openrouter.ai/api', oaiKey: '', }), diff --git a/src/modules/llms/vendor.registry.ts b/src/modules/llms/vendor.registry.ts index 31b51c0d6..c4da6cee9 100644 --- a/src/modules/llms/vendor.registry.ts +++ b/src/modules/llms/vendor.registry.ts @@ -1,6 +1,6 @@ import { DModelSource, DModelSourceId, ModelVendor, ModelVendorId } from './llm.types'; import { ModelVendorAnthropic } from './anthropic/anthropic.vendor'; -import { ModelVendorAzure } from '~/modules/llms/azure/azure.vendor'; +import { ModelVendorAzure } from './azure/azure.vendor'; import { ModelVendorLocalAI } from './localai/localai.vendor'; import { ModelVendorOoobabooga } from './oobabooga/oobabooga.vendor'; import { ModelVendorOpenAI } from './openai/openai.vendor'; @@ -16,9 +16,6 @@ const MODEL_VENDOR_REGISTRY: Record = { oobabooga: ModelVendorOoobabooga, openai: ModelVendorOpenAI, openrouter: ModelVendorOpenRouter, - // azure_openai: { id: 'azure_openai', name: 'Azure OpenAI', instanceLimit: 1, location: 'cloud', rank: 30 }, - // google_vertex: { id: 'google_vertex', name: 'Google Vertex', instanceLimit: 1, location: 'cloud', rank: 40 }, - // anthropic: { id: 'anthropic', name: 'Anthropic', instanceLimit: 1, location: 'cloud', rank: 50 }, }; const DEFAULT_MODEL_VENDOR: ModelVendorId = 'openai'; @@ -55,6 +52,6 @@ export function createModelSourceForVendor(vendorId: ModelVendorId, otherSources id: sourceId, label: vendor.name + (sourceN > 0 ? ` #${sourceN}` : ''), vId: vendorId, - setup: vendor.initalizeSetup?.() || {}, + setup: vendor.initializeSetup?.() || {}, }; } \ No newline at end of file From 06e866a3e88ed2008840cef2f36fd3ca1af307ae Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Fri, 22 Sep 2023 18:31:02 -0700 Subject: [PATCH 05/17] LLms: unify model priors --- src/modules/llms/azure/azure.router.ts | 64 ++------ src/modules/llms/llm.router.ts | 3 +- src/modules/llms/openai/OpenAISourceSetup.tsx | 62 ++----- src/modules/llms/openai/openai.data.ts | 154 ++++++++++++++++++ 4 files changed, 185 insertions(+), 98 deletions(-) create mode 100644 src/modules/llms/openai/openai.data.ts diff --git a/src/modules/llms/azure/azure.router.ts b/src/modules/llms/azure/azure.router.ts index a532f9c3e..37d74c6bb 100644 --- a/src/modules/llms/azure/azure.router.ts +++ b/src/modules/llms/azure/azure.router.ts @@ -6,7 +6,8 @@ import { fetchJsonOrTRPCError } from '~/modules/trpc/trpc.serverutils'; import { OpenAI } from '../openai/openai.types'; import { historySchema, modelSchema, openAIChatCompletionPayload } from '../openai/openai.router'; -import { listModelsOutputSchema, LLM_IF_OAI_Chat, ModelDescriptionSchema } from '../llm.router'; +import { listModelsOutputSchema, ModelDescriptionSchema } from '../llm.router'; +import { openAIModelToModelDescription } from '../openai/openai.data'; // input schemas @@ -66,10 +67,21 @@ export const llmAzureRouter = createTRPCRouter({ ); // parse and validate output, and take the GPT models only (e.g. no 'whisper') - const models = wireAzureListDeploymentsSchema.parse(azureModels).data; - return { - models: models.filter(m => m.model.includes('gpt')).map(azureModelToModelDescription), - }; + const wireModels = wireAzureListDeploymentsSchema.parse(azureModels).data; + + // map to ModelDescriptions + const models: ModelDescriptionSchema[] = wireModels + .filter(m => m.model.includes('gpt')) + .map((model): ModelDescriptionSchema => { + const { id, label, ...rest } = openAIModelToModelDescription(model.model, model.created_at, model.updated_at); + return { + id: model.id, + label: `${model.id} (${label})`, + ...rest, + }; + }); + + return { models }; }), @@ -112,48 +124,6 @@ export const llmAzureRouter = createTRPCRouter({ }); -function azureModelToModelDescription(model: { id: string, model: string, created_at: number, updated_at: number }): ModelDescriptionSchema { - const knownModel = knownAzureModels.find(m => m.id === model.model); - return { - id: model.id, - label: knownModel?.label || model.id, - created: model.created_at, - updated: model.updated_at || model.created_at, - description: knownModel?.description || 'Unknown model type, please let us know', - contextWindow: knownModel?.contextWindow || 2048, - interfaces: [LLM_IF_OAI_Chat], - hidden: !knownModel, - }; -} - -const knownAzureModels: Partial[] = [ - { - id: 'gpt-35-turbo', - label: '3.5-Turbo', - contextWindow: 4097, - description: 'Fair speed and smarts', - }, - { - id: 'gpt-35-turbo-16k', - label: '3.5-Turbo-16k', - contextWindow: 16384, - description: 'Fair speed and smarts, large context', - }, - { - id: 'gpt-4', - label: 'GPT-4', - contextWindow: 8192, - description: 'Insightful, big thinker, slower, pricey', - }, - { - id: 'gpt-4-32k', - label: 'GPT-4-32k', - contextWindow: 32768, - description: 'Largest context window for big problems', - }, -]; - - async function azureOpenaiGET(endpoint: string, key: string, apiPath: string /*, signal?: AbortSignal*/): Promise { const { headers, url } = azureOpenAIAccess(endpoint, key, apiPath); return await fetchJsonOrTRPCError(url, 'GET', headers, undefined, 'Azure OpenAI'); diff --git a/src/modules/llms/llm.router.ts b/src/modules/llms/llm.router.ts index d97155d04..2780750a6 100644 --- a/src/modules/llms/llm.router.ts +++ b/src/modules/llms/llm.router.ts @@ -7,6 +7,7 @@ import { LLMOptionsOpenAI } from './openai/openai.vendor'; // they're here as a preview - will be used more broadly in the future export const LLM_IF_OAI_Chat = 'oai-chat'; export const LLM_IF_OAI_Fn = 'oai-fn'; +export const LLM_IF_OAI_Complete = 'oai-complete'; const modelDescriptionSchema = z.object({ @@ -16,7 +17,7 @@ const modelDescriptionSchema = z.object({ updated: z.number().optional(), description: z.string(), contextWindow: z.number(), - interfaces: z.array(z.enum([LLM_IF_OAI_Chat, LLM_IF_OAI_Fn])), + interfaces: z.array(z.enum([LLM_IF_OAI_Chat, LLM_IF_OAI_Fn, LLM_IF_OAI_Complete])), hidden: z.boolean().optional(), }); diff --git a/src/modules/llms/openai/OpenAISourceSetup.tsx b/src/modules/llms/openai/OpenAISourceSetup.tsx index cb0b5528c..07984ad26 100644 --- a/src/modules/llms/openai/OpenAISourceSetup.tsx +++ b/src/modules/llms/openai/OpenAISourceSetup.tsx @@ -13,6 +13,7 @@ import { settingsCol1Width, settingsGap } from '~/common/theme'; import { DLLM, DModelSource, DModelSourceId } from '../llm.types'; import { hasServerKeyOpenAI, isValidOpenAIApiKey, LLMOptionsOpenAI, ModelVendorOpenAI } from './openai.vendor'; +import { openAIModelToModelDescription } from './openai.data'; import { useModelsStore, useSourceSetup } from '../store-llms'; @@ -152,65 +153,26 @@ export function OpenAISourceSetup(props: { sourceId: DModelSourceId }) { } -// this will help with adding metadata to the models -const knownBases = [ - { - id: 'gpt-4-32k-0613', - label: 'GPT-4-32k (0613)', - context: 32768, - description: 'Largest context window for big problems', - }, - { - id: 'gpt-4', - label: 'GPT-4', - context: 8192, - description: 'Insightful, big thinker, slower, pricey', - }, - { - id: 'gpt-3.5-turbo-instruct', - label: '3.5-Turbo-Instruct', - context: 4097, - description: 'Not for Chat', - }, - { - id: 'gpt-3.5-turbo-16k', - label: '3.5-Turbo-16k', - context: 16384, - description: 'Fair speed and smarts, large context', - }, - { - id: 'gpt-3.5-turbo', - label: '3.5-Turbo', - context: 4097, - description: 'Fair speed and smarts', - }, - { - id: '', - label: '?:', - context: 4096, - description: 'Unknown, please let us know the ID', - }, -]; - - function openAIModelToDLLM(model: { id: string, created: number }, source: DModelSource): DLLM { - const base = knownBases.find(base => model.id.startsWith(base.id)) || knownBases[knownBases.length - 1]; - const suffix = model.id.slice(base.id.length).trim(); - const hidden = !suffix || suffix.startsWith('-03') || model.id.indexOf('-instruct') !== -1; + const { label, created, updated, description, contextWindow: contextTokens, hidden } = openAIModelToModelDescription(model.id, model.created); return { id: `${source.id}-${model.id}`, - label: base.label + (suffix ? ` (${suffix.replaceAll('-', ' ').trim()})` : ''), - created: model.created, - description: base.description, + + label, + created: created || 0, + updated: updated || 0, + description, tags: [], // ['stream', 'chat'], - contextTokens: base.context, - hidden, + contextTokens, + hidden: hidden || false, + sId: source.id, _source: source, + options: { llmRef: model.id, llmTemperature: 0.5, - llmResponseTokens: Math.round(base.context / 8), + llmResponseTokens: Math.round(contextTokens / 8), }, }; } \ No newline at end of file diff --git a/src/modules/llms/openai/openai.data.ts b/src/modules/llms/openai/openai.data.ts new file mode 100644 index 000000000..d924e3401 --- /dev/null +++ b/src/modules/llms/openai/openai.data.ts @@ -0,0 +1,154 @@ +import { LLM_IF_OAI_Chat, LLM_IF_OAI_Complete, LLM_IF_OAI_Fn, ModelDescriptionSchema } from '~/modules/llms/llm.router'; + +const knownOpenAIChatModels: ({ idPrefix: string } & Omit)[] = [ + // GPT4-32k's + { + idPrefix: 'gpt-4-32k-0314', + label: 'GPT-4-32k (0314)', + description: 'Snapshot of gpt-4-32 from March 14th 2023. Will be deprecated on June 13th 2024 at the earliest.', + contextWindow: 32768, + interfaces: [LLM_IF_OAI_Chat], + hidden: true, + }, + { + idPrefix: 'gpt-4-32k-0613', + label: 'GPT-4-32k (0613)', + description: 'Snapshot of gpt-4-32 from June 13th 2023.', + contextWindow: 32768, + interfaces: [LLM_IF_OAI_Chat], + }, + { + idPrefix: 'gpt-4-32k', + label: 'GPT-4-32k', + description: 'Largest context window for big problems', + contextWindow: 32768, + interfaces: [LLM_IF_OAI_Chat], + hidden: true, + }, + + // GPT4's + { + idPrefix: 'gpt-4-0613', + label: 'GPT-4 (0613)', + description: 'Snapshot of gpt-4 from June 13th 2023 with function calling data.', + contextWindow: 8192, + interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Fn], + }, + { + idPrefix: 'gpt-4-0314', + label: 'GPT-4 (0314)', + description: 'Snapshot of gpt-4 from March 14th 2023 with function calling data.', + contextWindow: 8192, + interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Fn], + hidden: true, + }, + { + idPrefix: 'gpt-4', + label: 'GPT-4', + description: 'Insightful, big thinker, slower, pricey', + contextWindow: 8192, + interfaces: [LLM_IF_OAI_Chat], + hidden: true, + }, + + + // 3.5-Turbo-16k's + { + idPrefix: 'gpt-3.5-turbo-16k-0613', + label: '3.5-Turbo-16k (0613)', + description: 'Snapshot of gpt-3.5-turbo-16k from June 13th 2023.', + contextWindow: 16385, + interfaces: [LLM_IF_OAI_Chat], + }, + { + idPrefix: 'gpt-3.5-turbo-16k', + label: '3.5-Turbo-16k', + description: 'Same capabilities as the standard gpt-3.5-turbo model but with 4 times the context.', + contextWindow: 16385, + interfaces: [LLM_IF_OAI_Chat], + hidden: true, + }, + + // 3.5-Turbo-Instruct + { + idPrefix: 'gpt-3.5-turbo-instruct', + label: '3.5-Turbo-Instruct', + description: 'Not for chat.', + contextWindow: 4097, + interfaces: [LLM_IF_OAI_Complete], + hidden: true, + }, + + // 3.5-Turbo's + { + idPrefix: 'gpt-3.5-turbo-0301', + label: '3.5-Turbo (0301)', + description: 'Snapshot of gpt-3.5-turbo from March 1st 2023. Will be deprecated on June 13th 2024 at the earliest.', + contextWindow: 4097, + hidden: true, + interfaces: [LLM_IF_OAI_Chat], + }, + { + idPrefix: 'gpt-3.5-turbo-0613', + label: '3.5-Turbo (0613)', + description: 'Snapshot of gpt-3.5-turbo from June 13th 2023 with function calling data.', + contextWindow: 4097, + interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Fn], + }, + { + idPrefix: 'gpt-3.5-turbo', + label: '3.5-Turbo', + description: 'Fair speed and smarts.', + contextWindow: 4097, + hidden: true, + interfaces: [LLM_IF_OAI_Chat], + }, + + + // Azure variants - because someone forgot the dot + { + idPrefix: 'gpt-35-turbo-16k', + label: '3.5-Turbo-16k', + description: 'Fair speed and smarts, large context', + contextWindow: 16384, + interfaces: [LLM_IF_OAI_Chat], // as azure doesn't version model id's (in the deployments), let's assume no function calling + }, + { + idPrefix: 'gpt-35-turbo', + label: '3.5-Turbo', + contextWindow: 4097, + description: 'Fair speed and smarts', + interfaces: [LLM_IF_OAI_Chat], // as azure doesn't version model id's (in the deployments), let's assume no function calling + }, + + // Fallback - unknown + { + idPrefix: '', + label: '?:', + description: 'Unknown, please let us know the ID. Assuming a 4097 context window size and Chat capabilities.', + contextWindow: 4097, + interfaces: [LLM_IF_OAI_Chat], + hidden: true, + }, +]; + +export function openAIModelToModelDescription(openAiModelId: string, modelCreated: number, modelUpdated?: number): ModelDescriptionSchema { + // find the closest known model + const known = knownOpenAIChatModels.find(base => openAiModelId.startsWith(base.idPrefix)) + || knownOpenAIChatModels[knownOpenAIChatModels.length - 1]; + + // check whether this is a partial map, which indicates an unknown/new variant + const suffix = openAiModelId.slice(known.idPrefix.length).trim(); + + // return the model description sheet + return { + id: openAiModelId, + label: known.label + (suffix ? ` [${suffix.replaceAll('-', ' ').trim()}]` : ''), + created: modelCreated || 0, + updated: modelUpdated || modelCreated || 0, + description: known.description, + contextWindow: known.contextWindow, + interfaces: known.interfaces, + hidden: known.hidden, + }; +} \ No newline at end of file From 4d2209ca8d2fe5383eb35e4cd1b21c7eb949db16 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Fri, 22 Sep 2023 19:14:41 -0700 Subject: [PATCH 06/17] Anthropic: rename wiretypes --- pages/api/llms/stream.ts | 2 +- src/modules/llms/anthropic/anthropic.router.ts | 2 +- .../anthropic/{anthropic.types.ts => anthropic.wiretypes.ts} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename src/modules/llms/anthropic/{anthropic.types.ts => anthropic.wiretypes.ts} (100%) diff --git a/pages/api/llms/stream.ts b/pages/api/llms/stream.ts index 594c2e021..56bd6b6bd 100644 --- a/pages/api/llms/stream.ts +++ b/pages/api/llms/stream.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { createParser as createEventsourceParser, EventSourceParser, ParsedEvent, ReconnectInterval } from 'eventsource-parser'; -import { AnthropicWire } from '~/modules/llms/anthropic/anthropic.types'; +import { AnthropicWire } from '~/modules/llms/anthropic/anthropic.wiretypes'; import { OpenAI } from '~/modules/llms/openai/openai.types'; import { anthropicAccess, anthropicCompletionRequest } from '~/modules/llms/anthropic/anthropic.router'; import { chatStreamSchema, openAIAccess, openAIChatCompletionPayload } from '~/modules/llms/openai/openai.router'; diff --git a/src/modules/llms/anthropic/anthropic.router.ts b/src/modules/llms/anthropic/anthropic.router.ts index fed8d3e20..ef65d2889 100644 --- a/src/modules/llms/anthropic/anthropic.router.ts +++ b/src/modules/llms/anthropic/anthropic.router.ts @@ -7,7 +7,7 @@ import { fetchJsonOrTRPCError } from '~/modules/trpc/trpc.serverutils'; import { historySchema, modelSchema } from '~/modules/llms/openai/openai.router'; import { listModelsOutputSchema, LLM_IF_OAI_Chat, ModelDescriptionSchema } from '~/modules/llms/llm.router'; -import { AnthropicWire } from './anthropic.types'; +import { AnthropicWire } from './anthropic.wiretypes'; // Input Schemas diff --git a/src/modules/llms/anthropic/anthropic.types.ts b/src/modules/llms/anthropic/anthropic.wiretypes.ts similarity index 100% rename from src/modules/llms/anthropic/anthropic.types.ts rename to src/modules/llms/anthropic/anthropic.wiretypes.ts From 4f3f7963d05dd6a1c4f93cfff89164d93909dd9b Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Fri, 22 Sep 2023 19:15:38 -0700 Subject: [PATCH 07/17] Llms: cleanup routers --- pages/api/llms/stream.ts | 4 +- .../llms/anthropic/anthropic.router.ts | 47 +++---- src/modules/llms/azure/azure.router.ts | 34 ++--- src/modules/llms/openai/openai.router.ts | 120 +++++++++--------- 4 files changed, 88 insertions(+), 117 deletions(-) diff --git a/pages/api/llms/stream.ts b/pages/api/llms/stream.ts index 56bd6b6bd..e5468f7da 100644 --- a/pages/api/llms/stream.ts +++ b/pages/api/llms/stream.ts @@ -4,7 +4,7 @@ import { createParser as createEventsourceParser, EventSourceParser, ParsedEvent import { AnthropicWire } from '~/modules/llms/anthropic/anthropic.wiretypes'; import { OpenAI } from '~/modules/llms/openai/openai.types'; import { anthropicAccess, anthropicCompletionRequest } from '~/modules/llms/anthropic/anthropic.router'; -import { chatStreamSchema, openAIAccess, openAIChatCompletionPayload } from '~/modules/llms/openai/openai.router'; +import { openAIChatStreamSchema, openAIAccess, openAIChatCompletionPayload } from '~/modules/llms/openai/openai.router'; /** @@ -145,7 +145,7 @@ export function createEmptyReadableStream(): ReadableStream { export default async function handler(req: NextRequest): Promise { // inputs - reuse the tRPC schema - const { vendorId, access, model, history } = chatStreamSchema.parse(await req.json()); + const { vendorId, access, model, history } = openAIChatStreamSchema.parse(await req.json()); // begin event streaming from the OpenAI API let upstreamResponse: Response; diff --git a/src/modules/llms/anthropic/anthropic.router.ts b/src/modules/llms/anthropic/anthropic.router.ts index ef65d2889..b637eab6d 100644 --- a/src/modules/llms/anthropic/anthropic.router.ts +++ b/src/modules/llms/anthropic/anthropic.router.ts @@ -4,8 +4,8 @@ import { TRPCError } from '@trpc/server'; import { createTRPCRouter, publicProcedure } from '~/modules/trpc/trpc.server'; import { fetchJsonOrTRPCError } from '~/modules/trpc/trpc.serverutils'; -import { historySchema, modelSchema } from '~/modules/llms/openai/openai.router'; -import { listModelsOutputSchema, LLM_IF_OAI_Chat, ModelDescriptionSchema } from '~/modules/llms/llm.router'; +import { chatGenerateOutputSchema, historySchema, modelSchema } from '../openai/openai.router'; +import { listModelsOutputSchema, LLM_IF_OAI_Chat, ModelDescriptionSchema } from '../llm.router'; import { AnthropicWire } from './anthropic.wiretypes'; @@ -17,27 +17,26 @@ const anthropicAccessSchema = z.object({ anthropicHost: z.string().trim(), }); -const chatGenerateSchema = z.object({ access: anthropicAccessSchema, model: modelSchema, history: historySchema }); +const anthropicChatGenerateSchema = z.object({ access: anthropicAccessSchema, model: modelSchema, history: historySchema }); -const listModelsSchema = z.object({ access: anthropicAccessSchema }); - - -// Output Schemas - -const chatGenerateOutputSchema = z.object({ - role: z.enum(['assistant', 'system', 'user']), - content: z.string(), - finish_reason: z.union([z.enum(['stop', 'length']), z.null()]), -}); +const anthropicListModelsSchema = z.object({ access: anthropicAccessSchema }); export const llmAnthropicRouter = createTRPCRouter({ - /** + /* Anthropic: list models * + * See https://github.com/anthropics/anthropic-sdk-typescript/commit/7c53ded6b7f5f3efec0df295181f18469c37e09d?diff=unified for + * some details on the models, as the API docs are scarce: https://docs.anthropic.com/claude/reference/selecting-a-model */ + listModels: publicProcedure + .input(anthropicListModelsSchema) + .output(listModelsOutputSchema) + .query(() => ({ models: hardcodedAnthropicModels })), + + /* Anthropic: Chat generation */ chatGenerate: publicProcedure - .input(chatGenerateSchema) + .input(anthropicChatGenerateSchema) .output(chatGenerateOutputSchema) .mutation(async ({ input }) => { @@ -67,18 +66,6 @@ export const llmAnthropicRouter = createTRPCRouter({ }; }), - - /** - * List the Models available - * - * See https://github.com/anthropics/anthropic-sdk-typescript/commit/7c53ded6b7f5f3efec0df295181f18469c37e09d?diff=unified for - * some details on the models, as the API docs are scarce: https://docs.anthropic.com/claude/reference/selecting-a-model - */ - listModels: publicProcedure - .input(listModelsSchema) - .output(listModelsOutputSchema) - .query(async () => ({ models: hardcodedAnthropicModels })), - }); const roundTime = (date: string) => Math.round(new Date(date).getTime() / 1000); @@ -138,13 +125,9 @@ async function anthropicPOST(access: AccessSchema, body: TPostB return await fetchJsonOrTRPCError(url, 'POST', headers, body, 'Anthropic'); } - const DEFAULT_ANTHROPIC_HOST = 'api.anthropic.com'; -export function anthropicAccess(access: AccessSchema, apiPath: string): { - headers: HeadersInit, - url: string -} { +export function anthropicAccess(access: AccessSchema, apiPath: string): { headers: HeadersInit, url: string } { // API version const apiVersion = '2023-06-01'; diff --git a/src/modules/llms/azure/azure.router.ts b/src/modules/llms/azure/azure.router.ts index 37d74c6bb..5e7fc7169 100644 --- a/src/modules/llms/azure/azure.router.ts +++ b/src/modules/llms/azure/azure.router.ts @@ -5,7 +5,7 @@ import { createTRPCRouter, publicProcedure } from '~/modules/trpc/trpc.server'; import { fetchJsonOrTRPCError } from '~/modules/trpc/trpc.serverutils'; import { OpenAI } from '../openai/openai.types'; -import { historySchema, modelSchema, openAIChatCompletionPayload } from '../openai/openai.router'; +import { chatGenerateOutputSchema, historySchema, modelSchema, openAIChatCompletionPayload } from '../openai/openai.router'; import { listModelsOutputSchema, ModelDescriptionSchema } from '../llm.router'; import { openAIModelToModelDescription } from '../openai/openai.data'; @@ -17,18 +17,9 @@ const azureAccessSchema = z.object({ azureKey: z.string().trim(), }); -const chatGenerateSchema = z.object({ access: azureAccessSchema, model: modelSchema, history: historySchema }); +const azureChatGenerateSchema = z.object({ access: azureAccessSchema, model: modelSchema, history: historySchema }); -const listModelsSchema = z.object({ access: azureAccessSchema }); - - -// Output Schemas - -const chatGenerateOutputSchema = z.object({ - role: z.enum(['assistant', 'system', 'user']), - content: z.string(), - finish_reason: z.union([z.enum(['stop', 'length']), z.null()]), -}); +const azureListModelsSchema = z.object({ access: azureAccessSchema }); // Wire schemas @@ -50,17 +41,19 @@ const wireAzureListDeploymentsSchema = z.object({ export const llmAzureRouter = createTRPCRouter({ - /* List the Azure models - * Small complexity arises here as the models are called 'deployments' within allocated Azure 'endpoints'. + /* Azure: list models + * + * Some complexity arises here as the models are called 'deployments' within allocated Azure 'endpoints'. * We use an unofficial API to list the deployments, and map them to models descriptions. + * + * See: https://github.com/openai/openai-python/issues/447#issuecomment-1730976835 for our input on the issue. */ listModels: publicProcedure - .input(listModelsSchema) + .input(azureListModelsSchema) .output(listModelsOutputSchema) .query(async ({ input }) => { - // fetch the Azure OpenAI models - // HACK: this method may stop working soon - see: https://github.com/openai/openai-python/issues/447#issuecomment-1730976835, + // fetch the Azure OpenAI 'deployments' const azureModels = await azureOpenaiGET( input.access.azureEndpoint, input.access.azureKey, `/openai/deployments?api-version=2023-03-15-preview`, @@ -84,10 +77,9 @@ export const llmAzureRouter = createTRPCRouter({ return { models }; }), - - /* Chat-based message generation */ + /* Azure: Chat generation */ chatGenerate: publicProcedure - .input(chatGenerateSchema) + .input(azureChatGenerateSchema) .output(chatGenerateOutputSchema) .mutation(async ({ input }) => { @@ -156,4 +148,4 @@ function azureOpenAIAccess(endpoint: string, key: string, apiPath: string): { he }, url: azureEndpoint + apiPath, }; -} +} \ No newline at end of file diff --git a/src/modules/llms/openai/openai.router.ts b/src/modules/llms/openai/openai.router.ts index 9823f793b..1da9c367c 100644 --- a/src/modules/llms/openai/openai.router.ts +++ b/src/modules/llms/openai/openai.router.ts @@ -15,7 +15,7 @@ import { OpenAI } from './openai.types'; // Input Schemas -const accessSchema = z.object({ +const openAIAccessSchema = z.object({ oaiKey: z.string().trim(), oaiOrg: z.string().trim(), oaiHost: z.string().trim(), @@ -48,29 +48,31 @@ export const functionsSchema = z.array(z.object({ }).optional(), })); -export const chatStreamSchema = z.object({ +export const openAIChatStreamSchema = z.object({ vendorId: z.enum(['anthropic', 'openai']), // shall clean this up a bit - access: z.union([accessSchema, z.object({ anthropicKey: z.string().trim(), anthropicHost: z.string().trim() })]), + access: z.union([openAIAccessSchema, z.object({ anthropicKey: z.string().trim(), anthropicHost: z.string().trim() })]), model: modelSchema, history: historySchema, functions: functionsSchema.optional(), }); -export type ChatStreamSchema = z.infer; +export type ChatStreamSchema = z.infer; -const chatGenerateSchema = z.object({ access: accessSchema, model: modelSchema, history: historySchema, functions: functionsSchema.optional(), forceFunctionName: z.string().optional() }); +const openAIChatGenerateSchema = z.object({ access: openAIAccessSchema, model: modelSchema, history: historySchema, functions: functionsSchema.optional(), forceFunctionName: z.string().optional() }); -const chatModerationSchema = z.object({ access: accessSchema, text: z.string() }); +const openAIChatModerationSchema = z.object({ access: openAIAccessSchema, text: z.string() }); -const listModelsSchema = z.object({ access: accessSchema, filterGpt: z.boolean().optional() }); +const openAIListModelsSchema = z.object({ access: openAIAccessSchema, filterGpt: z.boolean().optional() }); // Output Schemas +export const chatGenerateOutputSchema = z.object({ + role: z.enum(['assistant', 'system', 'user']), + content: z.string(), + finish_reason: z.union([z.enum(['stop', 'length']), z.null()]), +}); + const chatGenerateWithFunctionsOutputSchema = z.union([ - z.object({ - role: z.enum(['assistant', 'system', 'user']), - content: z.string(), - finish_reason: z.union([z.enum(['stop', 'length']), z.null()]), - }), + chatGenerateOutputSchema, z.object({ function_name: z.string(), function_arguments: z.record(z.any()), @@ -80,11 +82,48 @@ const chatGenerateWithFunctionsOutputSchema = z.union([ export const llmOpenAIRouter = createTRPCRouter({ - /** - * Chat-based message generation - */ + /* OpenAI: List the Models available */ + listModels: publicProcedure + .input(openAIListModelsSchema) + .query(async ({ input }): Promise => { + + const wireModels: OpenAI.Wire.Models.Response = await openaiGET(input.access, '/v1/models'); + + // filter out the non-gpt models, if requested + let llms = (wireModels.data || []) + .filter(model => !input.filterGpt || model.id.includes('gpt')); + + // remove models with duplicate ids (can happen for local servers) + const preFilterCount = llms.length; + llms = llms.filter((model, index) => llms.findIndex(m => m.id === model.id) === index); + if (preFilterCount !== llms.length) + console.warn(`openai.router.listModels: Duplicate model ids found, removed ${preFilterCount - llms.length} models`); + + // sort by which model has the least number of '-' in the name, and then by id, decreasing + llms.sort((a, b) => { + // model that have '-0' in their name go at the end + // if (a.id.includes('-0') && !b.id.includes('-0')) return 1; + // if (!a.id.includes('-0') && b.id.includes('-0')) return -1; + + // sort by the first 5 chars of id, decreasing, then by the number of '-' in the name + const aId = a.id.slice(0, 5); + const bId = b.id.slice(0, 5); + if (aId === bId) { + const aCount = a.id.split('-').length; + const bCount = b.id.split('-').length; + if (aCount === bCount) + return a.id.localeCompare(b.id); + return aCount - bCount; + } + return bId.localeCompare(aId); + }); + + return llms; + }), + + /* OpenAI: chat generation */ chatGenerateWithFunctions: publicProcedure - .input(chatGenerateSchema) + .input(openAIChatGenerateSchema) .output(chatGenerateWithFunctionsOutputSchema) .mutation(async ({ input }) => { @@ -113,11 +152,9 @@ export const llmOpenAIRouter = createTRPCRouter({ : parseChatGenerateOutput(message as OpenAI.Wire.ChatCompletion.ResponseMessage, finish_reason); }), - /** - * Check for content policy violations - */ + /* OpenAI: check for content policy violations */ moderation: publicProcedure - .input(chatModerationSchema) + .input(openAIChatModerationSchema) .mutation(async ({ input }): Promise => { const { access, text } = input; try { @@ -136,51 +173,10 @@ export const llmOpenAIRouter = createTRPCRouter({ } }), - /** - * List the Models available - */ - listModels: publicProcedure - .input(listModelsSchema) - .query(async ({ input }): Promise => { - - const wireModels: OpenAI.Wire.Models.Response = await openaiGET(input.access, '/v1/models'); - - // filter out the non-gpt models, if requested - let llms = (wireModels.data || []) - .filter(model => !input.filterGpt || model.id.includes('gpt')); - - // remove models with duplicate ids (can happen for local servers) - const preFilterCount = llms.length; - llms = llms.filter((model, index) => llms.findIndex(m => m.id === model.id) === index); - if (preFilterCount !== llms.length) - console.warn(`openai.router.listModels: Duplicate model ids found, removed ${preFilterCount - llms.length} models`); - - // sort by which model has the least number of '-' in the name, and then by id, decreasing - llms.sort((a, b) => { - // model that have '-0' in their name go at the end - // if (a.id.includes('-0') && !b.id.includes('-0')) return 1; - // if (!a.id.includes('-0') && b.id.includes('-0')) return -1; - - // sort by the first 5 chars of id, decreasing, then by the number of '-' in the name - const aId = a.id.slice(0, 5); - const bId = b.id.slice(0, 5); - if (aId === bId) { - const aCount = a.id.split('-').length; - const bCount = b.id.split('-').length; - if (aCount === bCount) - return a.id.localeCompare(b.id); - return aCount - bCount; - } - return bId.localeCompare(aId); - }); - - return llms; - }), - }); -type AccessSchema = z.infer; +type AccessSchema = z.infer; type ModelSchema = z.infer; type HistorySchema = z.infer; type FunctionsSchema = z.infer; From 813d95b8984375140bd1ca49f2be7ebbf4bac36c Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Fri, 22 Sep 2023 19:27:33 -0700 Subject: [PATCH 08/17] Llms: move out icons --- .../chat/components/applayout/ChatDrawerItems.tsx | 4 ++-- src/apps/chat/trade/ImportChats.tsx | 2 +- .../components/icons}/AnthropicIcon.tsx | 0 .../azure => common/components/icons}/AzureIcon.tsx | 0 .../components/icons}/OobaboogaIcon.tsx | 0 .../openai => common/components/icons}/OpenAIIcon.tsx | 0 .../components/icons}/OpenRouterIcon.tsx | 0 src/modules/llms/anthropic/anthropic.vendor.ts | 9 +++++---- src/modules/llms/azure/azure.vendor.ts | 7 ++++--- src/modules/llms/localai/localai.vendor.tsx | 8 +++++--- src/modules/llms/oobabooga/oobabooga.vendor.ts | 8 +++++--- src/modules/llms/openai/openai.vendor.ts | 4 ++-- src/modules/llms/openrouter/openrouter.vendor.tsx | 11 ++++++----- 13 files changed, 30 insertions(+), 23 deletions(-) rename src/{modules/llms/anthropic => common/components/icons}/AnthropicIcon.tsx (100%) rename src/{modules/llms/azure => common/components/icons}/AzureIcon.tsx (100%) rename src/{modules/llms/oobabooga => common/components/icons}/OobaboogaIcon.tsx (100%) rename src/{modules/llms/openai => common/components/icons}/OpenAIIcon.tsx (100%) rename src/{modules/llms/openrouter => common/components/icons}/OpenRouterIcon.tsx (100%) diff --git a/src/apps/chat/components/applayout/ChatDrawerItems.tsx b/src/apps/chat/components/applayout/ChatDrawerItems.tsx index 9621a8731..057e1000c 100644 --- a/src/apps/chat/components/applayout/ChatDrawerItems.tsx +++ b/src/apps/chat/components/applayout/ChatDrawerItems.tsx @@ -6,12 +6,12 @@ import AddIcon from '@mui/icons-material/Add'; import DeleteOutlineIcon from '@mui/icons-material/DeleteOutline'; import FileUploadIcon from '@mui/icons-material/FileUpload'; -import { useChatStore } from '~/common/state/store-chats'; +import { OpenAIIcon } from '~/common/components/icons/OpenAIIcon'; import { setLayoutDrawerAnchor } from '~/common/layout/store-applayout'; +import { useChatStore } from '~/common/state/store-chats'; import { useUIPreferencesStore } from '~/common/state/store-ui'; import { ConversationItem } from './ConversationItem'; -import { OpenAIIcon } from '~/modules/llms/openai/OpenAIIcon'; type ListGrouping = 'off' | 'persona'; diff --git a/src/apps/chat/trade/ImportChats.tsx b/src/apps/chat/trade/ImportChats.tsx index 46dd51c11..18e966d45 100644 --- a/src/apps/chat/trade/ImportChats.tsx +++ b/src/apps/chat/trade/ImportChats.tsx @@ -5,10 +5,10 @@ import { Box, Button, FormControl, FormLabel, Input, Sheet, Typography } from '@ import FileUploadIcon from '@mui/icons-material/FileUpload'; import type { ChatGptSharedChatSchema } from '~/modules/sharing/import.chatgpt'; -import { OpenAIIcon } from '~/modules/llms/openai/OpenAIIcon'; import { apiAsync } from '~/modules/trpc/trpc.client'; import { Brand } from '~/common/brand'; +import { OpenAIIcon } from '~/common/components/icons/OpenAIIcon'; import { createDConversation, createDMessage, DMessage, useChatStore } from '~/common/state/store-chats'; import { ImportedOutcome, ImportOutcomeModal } from './ImportOutcomeModal'; diff --git a/src/modules/llms/anthropic/AnthropicIcon.tsx b/src/common/components/icons/AnthropicIcon.tsx similarity index 100% rename from src/modules/llms/anthropic/AnthropicIcon.tsx rename to src/common/components/icons/AnthropicIcon.tsx diff --git a/src/modules/llms/azure/AzureIcon.tsx b/src/common/components/icons/AzureIcon.tsx similarity index 100% rename from src/modules/llms/azure/AzureIcon.tsx rename to src/common/components/icons/AzureIcon.tsx diff --git a/src/modules/llms/oobabooga/OobaboogaIcon.tsx b/src/common/components/icons/OobaboogaIcon.tsx similarity index 100% rename from src/modules/llms/oobabooga/OobaboogaIcon.tsx rename to src/common/components/icons/OobaboogaIcon.tsx diff --git a/src/modules/llms/openai/OpenAIIcon.tsx b/src/common/components/icons/OpenAIIcon.tsx similarity index 100% rename from src/modules/llms/openai/OpenAIIcon.tsx rename to src/common/components/icons/OpenAIIcon.tsx diff --git a/src/modules/llms/openrouter/OpenRouterIcon.tsx b/src/common/components/icons/OpenRouterIcon.tsx similarity index 100% rename from src/modules/llms/openrouter/OpenRouterIcon.tsx rename to src/common/components/icons/OpenRouterIcon.tsx diff --git a/src/modules/llms/anthropic/anthropic.vendor.ts b/src/modules/llms/anthropic/anthropic.vendor.ts index b0c056583..42ee71d8f 100644 --- a/src/modules/llms/anthropic/anthropic.vendor.ts +++ b/src/modules/llms/anthropic/anthropic.vendor.ts @@ -1,19 +1,20 @@ import { apiAsync } from '~/modules/trpc/trpc.client'; +import { AnthropicIcon } from '~/common/components/icons/AnthropicIcon'; + import { DLLM, ModelVendor } from '../llm.types'; import { VChatMessageIn, VChatMessageOut } from '../llm.client'; -import { LLMOptionsOpenAI } from '~/modules/llms/openai/openai.vendor'; -import { OpenAILLMOptions } from '~/modules/llms/openai/OpenAILLMOptions'; +import { LLMOptionsOpenAI } from '../openai/openai.vendor'; +import { OpenAILLMOptions } from '../openai/OpenAILLMOptions'; -import { AnthropicIcon } from './AnthropicIcon'; import { AnthropicSourceSetup } from './AnthropicSourceSetup'; + // special symbols export const hasServerKeyAnthropic = !!process.env.HAS_SERVER_KEY_ANTHROPIC; export const isValidAnthropicApiKey = (apiKey?: string) => !!apiKey && apiKey.startsWith('sk-') && apiKey.length > 40; - export interface SourceSetupAnthropic { anthropicKey: string; anthropicHost: string; diff --git a/src/modules/llms/azure/azure.vendor.ts b/src/modules/llms/azure/azure.vendor.ts index 8262c2e86..d4492da8c 100644 --- a/src/modules/llms/azure/azure.vendor.ts +++ b/src/modules/llms/azure/azure.vendor.ts @@ -1,11 +1,13 @@ import { apiAsync } from '~/modules/trpc/trpc.client'; +import { AzureIcon } from '~/common/components/icons/AzureIcon'; + import { DLLM, ModelVendor } from '../llm.types'; +import { VChatFunctionIn, VChatMessageIn, VChatMessageOrFunctionCallOut, VChatMessageOut } from '../llm.client'; + import { LLMOptionsOpenAI } from '../openai/openai.vendor'; import { OpenAILLMOptions } from '../openai/OpenAILLMOptions'; -import { VChatFunctionIn, VChatMessageIn, VChatMessageOrFunctionCallOut, VChatMessageOut } from '../llm.client'; -import { AzureIcon } from './AzureIcon'; import { AzureSourceSetup } from './AzureSourceSetup'; @@ -13,7 +15,6 @@ import { AzureSourceSetup } from './AzureSourceSetup'; export const hasServerKeyAzure = !!process.env.HAS_SERVER_KEY_AZURE_OPENAI; export const isValidAzureApiKey = (apiKey?: string) => !!apiKey && apiKey.length >= 32; - export interface SourceSetupAzure { azureEndpoint: string; azureKey: string; diff --git a/src/modules/llms/localai/localai.vendor.tsx b/src/modules/llms/localai/localai.vendor.tsx index dcb727f9f..067eaf8d4 100644 --- a/src/modules/llms/localai/localai.vendor.tsx +++ b/src/modules/llms/localai/localai.vendor.tsx @@ -1,11 +1,13 @@ +import DevicesIcon from '@mui/icons-material/Devices'; + import { ModelVendor } from '../llm.types'; -import { LLMOptionsOpenAI, ModelVendorOpenAI } from '~/modules/llms/openai/openai.vendor'; -import { OpenAILLMOptions } from '~/modules/llms/openai/OpenAILLMOptions'; +import { LLMOptionsOpenAI, ModelVendorOpenAI } from '../openai/openai.vendor'; +import { OpenAILLMOptions } from '../openai/OpenAILLMOptions'; -import DevicesIcon from '@mui/icons-material/Devices'; import { LocalAISourceSetup } from './LocalAISourceSetup'; + export interface SourceSetupLocalAI { oaiHost: string; // use OpenAI-compatible non-default hosts (full origin path) } diff --git a/src/modules/llms/oobabooga/oobabooga.vendor.ts b/src/modules/llms/oobabooga/oobabooga.vendor.ts index 94c886a60..7b53b2f7a 100644 --- a/src/modules/llms/oobabooga/oobabooga.vendor.ts +++ b/src/modules/llms/oobabooga/oobabooga.vendor.ts @@ -1,11 +1,13 @@ +import { OobaboogaIcon } from '~/common/components/icons/OobaboogaIcon'; + import { ModelVendor } from '../llm.types'; -import { LLMOptionsOpenAI, ModelVendorOpenAI } from '~/modules/llms/openai/openai.vendor'; -import { OpenAILLMOptions } from '~/modules/llms/openai/OpenAILLMOptions'; +import { LLMOptionsOpenAI, ModelVendorOpenAI } from '../openai/openai.vendor'; +import { OpenAILLMOptions } from '../openai/OpenAILLMOptions'; -import { OobaboogaIcon } from './OobaboogaIcon'; import { OobaboogaSourceSetup } from './OobaboogaSourceSetup'; + export interface SourceSetupOobabooga { oaiHost: string; // use OpenAI-compatible non-default hosts (full origin path) } diff --git a/src/modules/llms/openai/openai.vendor.ts b/src/modules/llms/openai/openai.vendor.ts index 2c46b8753..5ee7f0763 100644 --- a/src/modules/llms/openai/openai.vendor.ts +++ b/src/modules/llms/openai/openai.vendor.ts @@ -1,9 +1,10 @@ import { apiAsync } from '~/modules/trpc/trpc.client'; +import { OpenAIIcon } from '~/common/components/icons/OpenAIIcon'; + import { DLLM, ModelVendor } from '../llm.types'; import { VChatFunctionIn, VChatMessageIn, VChatMessageOrFunctionCallOut, VChatMessageOut } from '../llm.client'; -import { OpenAIIcon } from './OpenAIIcon'; import { OpenAILLMOptions } from './OpenAILLMOptions'; import { OpenAISourceSetup } from './OpenAISourceSetup'; @@ -12,7 +13,6 @@ import { OpenAISourceSetup } from './OpenAISourceSetup'; export const hasServerKeyOpenAI = !!process.env.HAS_SERVER_KEY_OPENAI; export const isValidOpenAIApiKey = (apiKey?: string) => !!apiKey && apiKey.startsWith('sk-') && apiKey.length > 40; - export interface SourceSetupOpenAI { oaiKey: string; oaiOrg: string; diff --git a/src/modules/llms/openrouter/openrouter.vendor.tsx b/src/modules/llms/openrouter/openrouter.vendor.tsx index e113bffcd..174ef26e7 100644 --- a/src/modules/llms/openrouter/openrouter.vendor.tsx +++ b/src/modules/llms/openrouter/openrouter.vendor.tsx @@ -1,11 +1,13 @@ +import { OpenRouterIcon } from '~/common/components/icons/OpenRouterIcon'; + import { ModelVendor } from '../llm.types'; -import { LLMOptionsOpenAI, ModelVendorOpenAI } from '~/modules/llms/openai/openai.vendor'; -import { OpenAILLMOptions } from '~/modules/llms/openai/OpenAILLMOptions'; +import { LLMOptionsOpenAI, ModelVendorOpenAI } from '../openai/openai.vendor'; +import { OpenAILLMOptions } from '../openai/OpenAILLMOptions'; -import { OpenRouterIcon } from './OpenRouterIcon'; import { OpenRouterSourceSetup } from './OpenRouterSourceSetup'; + // special symbols export const isValidOpenRouterKey = (apiKey?: string) => !!apiKey && apiKey.startsWith('sk-or-') && apiKey.length > 40; @@ -15,7 +17,6 @@ export interface SourceSetupOpenRouter { oaiHost: string; } - /** * NOTE: the support is just started and incomplete - in particular it depends on some code that * hasn't been merged yet. @@ -23,7 +24,7 @@ export interface SourceSetupOpenRouter { * Completion: * [x] raise instanceLimit from 0 to 1 to continue development * [x] add support to the OpenAI Router and Streaming function to add the headers required by OpenRouter (done in the access function) - * [ ] merge the server-side models remapping from Azure OpenAI - not needed, using client-side remapping for now + * [~] merge the server-side models remapping from Azure OpenAI - not needed, using client-side remapping for now * [x] decide whether to do UI work to improve the appearance - prioritized models * [x] works! */ From b26ddc422a5f65a8c5fd8ee043bf271537c0ee38 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Fri, 22 Sep 2023 20:22:54 -0700 Subject: [PATCH 09/17] Llms: remove the 'types' file and extract the vendor description out --- .../components/applayout/useLLMDropdown.tsx | 3 +- src/apps/chat/editors/chat-stream.ts | 2 +- src/apps/chat/editors/editors.ts | 2 +- src/apps/chat/editors/react-tangent.ts | 2 +- src/apps/chat/trade/trade.json.ts | 3 +- src/apps/models-modal/LLMOptionsModal.tsx | 3 +- src/apps/models-modal/ModelsList.tsx | 8 +-- src/apps/models-modal/ModelsModal.tsx | 5 +- .../models-modal/ModelsSourceSelector.tsx | 10 +-- src/apps/models-modal/VendorLLMOptions.tsx | 5 +- src/apps/models-modal/VendorSourceSetup.tsx | 4 +- src/apps/personas/useLLMChain.ts | 3 +- src/common/state/store-chats.ts | 3 +- src/common/state/store-ui.ts | 2 +- src/common/util/token-counter.ts | 3 +- src/modules/aifn/flatten/flatten.ts | 2 +- src/modules/aifn/react/react.ts | 2 +- src/modules/aifn/summarize/ContentReducer.tsx | 3 +- src/modules/aifn/summarize/summerize.ts | 2 +- .../llms/anthropic/AnthropicSourceSetup.tsx | 3 +- .../llms/anthropic/anthropic.vendor.ts | 5 +- src/modules/llms/azure/AzureSourceSetup.tsx | 3 +- src/modules/llms/azure/azure.vendor.ts | 5 +- src/modules/llms/llm.client.ts | 14 ++-- src/modules/llms/llm.router.ts | 2 +- src/modules/llms/llm.types.ts | 61 ---------------- .../llms/localai/LocalAISourceSetup.tsx | 5 +- src/modules/llms/localai/localai.vendor.tsx | 4 +- .../llms/oobabooga/OobaboogaSourceSetup.tsx | 3 +- .../llms/oobabooga/oobabooga.vendor.ts | 4 +- src/modules/llms/openai/OpenAILLMOptions.tsx | 3 +- src/modules/llms/openai/OpenAISourceSetup.tsx | 3 +- src/modules/llms/openai/openai.vendor.ts | 5 +- .../llms/openrouter/OpenRouterSourceSetup.tsx | 3 +- .../llms/openrouter/openrouter.vendor.tsx | 4 +- src/modules/llms/store-llms.ts | 71 ++++++++++++++----- src/modules/llms/vendors/IModelVendor.ts | 27 +++++++ .../llms/{ => vendors}/vendor.registry.ts | 55 +++++++------- 38 files changed, 172 insertions(+), 175 deletions(-) delete mode 100644 src/modules/llms/llm.types.ts create mode 100644 src/modules/llms/vendors/IModelVendor.ts rename src/modules/llms/{ => vendors}/vendor.registry.ts (58%) diff --git a/src/apps/chat/components/applayout/useLLMDropdown.tsx b/src/apps/chat/components/applayout/useLLMDropdown.tsx index cc468d658..4baa07a78 100644 --- a/src/apps/chat/components/applayout/useLLMDropdown.tsx +++ b/src/apps/chat/components/applayout/useLLMDropdown.tsx @@ -5,8 +5,7 @@ import { ListItemButton, ListItemDecorator } from '@mui/joy'; import BuildCircleIcon from '@mui/icons-material/BuildCircle'; import SettingsIcon from '@mui/icons-material/Settings'; -import { DLLM, DLLMId, DModelSourceId } from '~/modules/llms/llm.types'; -import { useModelsStore } from '~/modules/llms/store-llms'; +import { DLLM, DLLMId, DModelSourceId, useModelsStore } from '~/modules/llms/store-llms'; import { AppBarDropdown, DropdownItems } from '~/common/layout/AppBarDropdown'; import { useUIStateStore } from '~/common/state/store-ui'; diff --git a/src/apps/chat/editors/chat-stream.ts b/src/apps/chat/editors/chat-stream.ts index 48f9e35d3..0b3d7bb9c 100644 --- a/src/apps/chat/editors/chat-stream.ts +++ b/src/apps/chat/editors/chat-stream.ts @@ -1,5 +1,5 @@ -import { DLLMId } from '~/modules/llms/llm.types'; import { SystemPurposeId } from '../../../data'; +import { DLLMId } from '~/modules/llms/store-llms'; import { autoSuggestions } from '~/modules/aifn/autosuggestions/autoSuggestions'; import { autoTitle } from '~/modules/aifn/autotitle/autoTitle'; import { speakText } from '~/modules/elevenlabs/elevenlabs.client'; diff --git a/src/apps/chat/editors/editors.ts b/src/apps/chat/editors/editors.ts index 58b8e111f..90190b35a 100644 --- a/src/apps/chat/editors/editors.ts +++ b/src/apps/chat/editors/editors.ts @@ -1,4 +1,4 @@ -import { DLLMId } from '~/modules/llms/llm.types'; +import { DLLMId } from '~/modules/llms/store-llms'; import { SystemPurposeId, SystemPurposes } from '../../../data'; import { createDMessage, DMessage, useChatStore } from '~/common/state/store-chats'; diff --git a/src/apps/chat/editors/react-tangent.ts b/src/apps/chat/editors/react-tangent.ts index 34b6f33db..ad48054f9 100644 --- a/src/apps/chat/editors/react-tangent.ts +++ b/src/apps/chat/editors/react-tangent.ts @@ -1,5 +1,5 @@ import { Agent } from '~/modules/aifn/react/react'; -import { DLLMId } from '~/modules/llms/llm.types'; +import { DLLMId } from '~/modules/llms/store-llms'; import { createDEphemeral, DMessage, useChatStore } from '~/common/state/store-chats'; diff --git a/src/apps/chat/trade/trade.json.ts b/src/apps/chat/trade/trade.json.ts index 54a092ddd..40def02e8 100644 --- a/src/apps/chat/trade/trade.json.ts +++ b/src/apps/chat/trade/trade.json.ts @@ -2,8 +2,7 @@ import { fileSave } from 'browser-fs-access'; import { defaultSystemPurposeId } from '../../../data'; -import { DModelSource } from '~/modules/llms/llm.types'; -import { useModelsStore } from '~/modules/llms/store-llms'; +import { DModelSource, useModelsStore } from '~/modules/llms/store-llms'; import { DConversation, useChatStore } from '~/common/state/store-chats'; import { ImportedOutcome } from './ImportOutcomeModal'; diff --git a/src/apps/models-modal/LLMOptionsModal.tsx b/src/apps/models-modal/LLMOptionsModal.tsx index 5179bdd05..8039d4b45 100644 --- a/src/apps/models-modal/LLMOptionsModal.tsx +++ b/src/apps/models-modal/LLMOptionsModal.tsx @@ -6,8 +6,7 @@ import DeleteOutlineIcon from '@mui/icons-material/DeleteOutline'; import VisibilityIcon from '@mui/icons-material/Visibility'; import VisibilityOffIcon from '@mui/icons-material/VisibilityOff'; -import { DLLMId } from '~/modules/llms/llm.types'; -import { useModelsStore } from '~/modules/llms/store-llms'; +import { DLLMId, useModelsStore } from '~/modules/llms/store-llms'; import { GoodModal } from '~/common/components/GoodModal'; import { useUIStateStore } from '~/common/state/store-ui'; diff --git a/src/apps/models-modal/ModelsList.tsx b/src/apps/models-modal/ModelsList.tsx index 5f354cb27..b49037261 100644 --- a/src/apps/models-modal/ModelsList.tsx +++ b/src/apps/models-modal/ModelsList.tsx @@ -5,14 +5,14 @@ import { Box, Chip, IconButton, List, ListItem, ListItemButton, Tooltip, Typogra import SettingsOutlinedIcon from '@mui/icons-material/SettingsOutlined'; import VisibilityOffOutlinedIcon from '@mui/icons-material/VisibilityOffOutlined'; -import { DLLM, DModelSourceId, ModelVendor } from '~/modules/llms/llm.types'; -import { findVendorById } from '~/modules/llms/vendor.registry'; -import { useModelsStore } from '~/modules/llms/store-llms'; +import { DLLM, DModelSourceId, useModelsStore } from '~/modules/llms/store-llms'; +import { IModelVendor } from '~/modules/llms/vendors/IModelVendor'; +import { findVendorById } from '~/modules/llms/vendors/vendor.registry'; import { useUIStateStore } from '~/common/state/store-ui'; -function ModelItem(props: { llm: DLLM, vendor: ModelVendor, chipChat: boolean, chipFast: boolean, chipFunc: boolean }) { +function ModelItem(props: { llm: DLLM, vendor: IModelVendor, chipChat: boolean, chipFast: boolean, chipFunc: boolean }) { // external state const openLLMOptions = useUIStateStore(state => state.openLLMOptions); diff --git a/src/apps/models-modal/ModelsModal.tsx b/src/apps/models-modal/ModelsModal.tsx index 44d50159a..08c14cdab 100644 --- a/src/apps/models-modal/ModelsModal.tsx +++ b/src/apps/models-modal/ModelsModal.tsx @@ -6,9 +6,8 @@ import { Checkbox, Divider } from '@mui/joy'; import { GoodModal } from '~/common/components/GoodModal'; import { useUIStateStore } from '~/common/state/store-ui'; -import { DModelSourceId } from '~/modules/llms/llm.types'; -import { createModelSourceForDefaultVendor } from '~/modules/llms/vendor.registry'; -import { useModelsStore } from '~/modules/llms/store-llms'; +import { DModelSourceId, useModelsStore } from '~/modules/llms/store-llms'; +import { createModelSourceForDefaultVendor } from '~/modules/llms/vendors/vendor.registry'; import { LLMOptionsModal } from './LLMOptionsModal'; import { ModelsList } from './ModelsList'; diff --git a/src/apps/models-modal/ModelsSourceSelector.tsx b/src/apps/models-modal/ModelsSourceSelector.tsx index 9b9614d54..df7af90f5 100644 --- a/src/apps/models-modal/ModelsSourceSelector.tsx +++ b/src/apps/models-modal/ModelsSourceSelector.tsx @@ -8,23 +8,23 @@ import CloudOutlinedIcon from '@mui/icons-material/CloudOutlined'; import ComputerIcon from '@mui/icons-material/Computer'; import DeleteOutlineIcon from '@mui/icons-material/DeleteOutline'; -import { DModelSourceId, ModelVendor, ModelVendorId } from '~/modules/llms/llm.types'; -import { createModelSourceForVendor, findAllVendors, findVendorById } from '~/modules/llms/vendor.registry'; +import { DModelSourceId, useModelsStore } from '~/modules/llms/store-llms'; +import { IModelVendor, ModelVendorId } from '~/modules/llms/vendors/IModelVendor'; +import { createModelSourceForVendor, findAllVendors, findVendorById } from '~/modules/llms/vendors/vendor.registry'; import { hasServerKeyOpenAI } from '~/modules/llms/openai/openai.vendor'; -import { useModelsStore } from '~/modules/llms/store-llms'; import { CloseableMenu } from '~/common/components/CloseableMenu'; import { ConfirmationModal } from '~/common/components/ConfirmationModal'; import { hideOnDesktop, hideOnMobile } from '~/common/theme'; -function locationIcon(vendor?: ModelVendor | null) { +function locationIcon(vendor?: IModelVendor | null) { if (vendor && vendor.id === 'openai' && hasServerKeyOpenAI) return ; return !vendor ? null : vendor.location === 'local' ? : ; } -function vendorIcon(vendor?: ModelVendor | null) { +function vendorIcon(vendor?: IModelVendor | null) { const Icon = !vendor ? null : vendor.Icon; return Icon ? : null; } diff --git a/src/apps/models-modal/VendorLLMOptions.tsx b/src/apps/models-modal/VendorLLMOptions.tsx index cfbabc101..db4c65832 100644 --- a/src/apps/models-modal/VendorLLMOptions.tsx +++ b/src/apps/models-modal/VendorLLMOptions.tsx @@ -1,9 +1,8 @@ import * as React from 'react'; import { shallow } from 'zustand/shallow'; -import { DLLMId } from '~/modules/llms/llm.types'; -import { findVendorById } from '~/modules/llms/vendor.registry'; -import { useModelsStore } from '~/modules/llms/store-llms'; +import { DLLMId, useModelsStore } from '~/modules/llms/store-llms'; +import { findVendorById } from '~/modules/llms/vendors/vendor.registry'; export function VendorLLMOptions(props: { id: DLLMId }) { diff --git a/src/apps/models-modal/VendorSourceSetup.tsx b/src/apps/models-modal/VendorSourceSetup.tsx index 010224bf0..f4c53097d 100644 --- a/src/apps/models-modal/VendorSourceSetup.tsx +++ b/src/apps/models-modal/VendorSourceSetup.tsx @@ -1,7 +1,7 @@ import * as React from 'react'; -import { DModelSource } from '~/modules/llms/llm.types'; -import { findVendorById } from '~/modules/llms/vendor.registry'; +import { DModelSource } from '~/modules/llms/store-llms'; +import { findVendorById } from '~/modules/llms/vendors/vendor.registry'; export function VendorSourceSetup(props: { source: DModelSource }) { diff --git a/src/apps/personas/useLLMChain.ts b/src/apps/personas/useLLMChain.ts index a3f5d8a74..3da6ab3ff 100644 --- a/src/apps/personas/useLLMChain.ts +++ b/src/apps/personas/useLLMChain.ts @@ -1,8 +1,7 @@ import * as React from 'react'; -import { DLLMId } from '~/modules/llms/llm.types'; +import { DLLMId, useModelsStore } from '~/modules/llms/store-llms'; import { callChatGenerate, VChatMessageIn } from '~/modules/llms/llm.client'; -import { useModelsStore } from '~/modules/llms/store-llms'; export interface LLMChainStep { diff --git a/src/common/state/store-chats.ts b/src/common/state/store-chats.ts index 5a4f07595..1e0424094 100644 --- a/src/common/state/store-chats.ts +++ b/src/common/state/store-chats.ts @@ -2,8 +2,7 @@ import { create } from 'zustand'; import { createJSONStorage, devtools, persist } from 'zustand/middleware'; import { v4 as uuidv4 } from 'uuid'; -import { DLLMId } from '~/modules/llms/llm.types'; -import { useModelsStore } from '~/modules/llms/store-llms'; +import { DLLMId, useModelsStore } from '~/modules/llms/store-llms'; import { countModelTokens } from '../util/token-counter'; import { defaultSystemPurposeId, SystemPurposeId } from '../../data'; diff --git a/src/common/state/store-ui.ts b/src/common/state/store-ui.ts index e2d949634..478b07b7e 100644 --- a/src/common/state/store-ui.ts +++ b/src/common/state/store-ui.ts @@ -1,7 +1,7 @@ import { create } from 'zustand'; import { persist } from 'zustand/middleware'; -import { DLLMId } from '~/modules/llms/llm.types'; +import { DLLMId } from '~/modules/llms/store-llms'; // UI State - not persisted diff --git a/src/common/util/token-counter.ts b/src/common/util/token-counter.ts index 014545cbd..a3ea91901 100644 --- a/src/common/util/token-counter.ts +++ b/src/common/util/token-counter.ts @@ -1,8 +1,7 @@ import { encoding_for_model, get_encoding, Tiktoken, TiktokenModel } from '@dqbd/tiktoken'; -import { DLLMId } from '~/modules/llms/llm.types'; import { findLLMOrThrow } from '~/modules/llms/llm.client'; -import { useModelsStore } from '~/modules/llms/store-llms'; +import { DLLMId, useModelsStore } from '~/modules/llms/store-llms'; // Do not set this to true in production, it's very verbose diff --git a/src/modules/aifn/flatten/flatten.ts b/src/modules/aifn/flatten/flatten.ts index 520a8c6f2..f0af3c21e 100644 --- a/src/modules/aifn/flatten/flatten.ts +++ b/src/modules/aifn/flatten/flatten.ts @@ -1,4 +1,4 @@ -import { DLLMId } from '~/modules/llms/llm.types'; +import { DLLMId } from '~/modules/llms/store-llms'; import { callChatGenerate } from '~/modules/llms/llm.client'; import { DConversation } from '~/common/state/store-chats'; diff --git a/src/modules/aifn/react/react.ts b/src/modules/aifn/react/react.ts index f812791a5..4ba73b75b 100644 --- a/src/modules/aifn/react/react.ts +++ b/src/modules/aifn/react/react.ts @@ -2,7 +2,7 @@ * porting of implementation from here: https://til.simonwillison.net/llms/python-react-pattern */ -import { DLLMId } from '~/modules/llms/llm.types'; +import { DLLMId } from '~/modules/llms/store-llms'; import { callApiSearchGoogle } from '~/modules/google/search.client'; import { callChatGenerate, VChatMessageIn } from '~/modules/llms/llm.client'; diff --git a/src/modules/aifn/summarize/ContentReducer.tsx b/src/modules/aifn/summarize/ContentReducer.tsx index 66360eb35..b7fc16833 100644 --- a/src/modules/aifn/summarize/ContentReducer.tsx +++ b/src/modules/aifn/summarize/ContentReducer.tsx @@ -3,9 +3,8 @@ import { shallow } from 'zustand/shallow'; import { Alert, Box, Button, CircularProgress, Divider, FormControl, FormHelperText, FormLabel, Modal, ModalClose, ModalDialog, Option, Select, Slider, Stack, Textarea, Typography } from '@mui/joy'; -import { DLLM, DLLMId } from '~/modules/llms/llm.types'; +import { DLLM, DLLMId, useModelsStore } from '~/modules/llms/store-llms'; import { summerizeToFitContextBudget } from '~/modules/aifn/summarize/summerize'; -import { useModelsStore } from '~/modules/llms/store-llms'; import { Section } from '~/common/components/Section'; import { countModelTokens } from '~/common/util/token-counter'; diff --git a/src/modules/aifn/summarize/summerize.ts b/src/modules/aifn/summarize/summerize.ts index 185202d7d..2e06950ff 100644 --- a/src/modules/aifn/summarize/summerize.ts +++ b/src/modules/aifn/summarize/summerize.ts @@ -1,4 +1,4 @@ -import { DLLMId } from '~/modules/llms/llm.types'; +import { DLLMId } from '~/modules/llms/store-llms'; import { callChatGenerate, findLLMOrThrow } from '~/modules/llms/llm.client'; diff --git a/src/modules/llms/anthropic/AnthropicSourceSetup.tsx b/src/modules/llms/anthropic/AnthropicSourceSetup.tsx index bff5eb330..3f80aa53e 100644 --- a/src/modules/llms/anthropic/AnthropicSourceSetup.tsx +++ b/src/modules/llms/anthropic/AnthropicSourceSetup.tsx @@ -10,9 +10,8 @@ import { InlineError } from '~/common/components/InlineError'; import { Link } from '~/common/components/Link'; import { settingsGap } from '~/common/theme'; -import { DModelSourceId } from '../llm.types'; +import { DModelSourceId, useModelsStore, useSourceSetup } from '../store-llms'; import { modelDescriptionToDLLM } from '../llm.router'; -import { useModelsStore, useSourceSetup } from '../store-llms'; import { hasServerKeyAnthropic, isValidAnthropicApiKey, ModelVendorAnthropic } from './anthropic.vendor'; diff --git a/src/modules/llms/anthropic/anthropic.vendor.ts b/src/modules/llms/anthropic/anthropic.vendor.ts index 42ee71d8f..201ad3f8e 100644 --- a/src/modules/llms/anthropic/anthropic.vendor.ts +++ b/src/modules/llms/anthropic/anthropic.vendor.ts @@ -2,7 +2,8 @@ import { apiAsync } from '~/modules/trpc/trpc.client'; import { AnthropicIcon } from '~/common/components/icons/AnthropicIcon'; -import { DLLM, ModelVendor } from '../llm.types'; +import { DLLM } from '../store-llms'; +import { IModelVendor } from '../vendors/IModelVendor'; import { VChatMessageIn, VChatMessageOut } from '../llm.client'; import { LLMOptionsOpenAI } from '../openai/openai.vendor'; @@ -20,7 +21,7 @@ export interface SourceSetupAnthropic { anthropicHost: string; } -export const ModelVendorAnthropic: ModelVendor = { +export const ModelVendorAnthropic: IModelVendor = { id: 'anthropic', name: 'Anthropic', rank: 13, diff --git a/src/modules/llms/azure/AzureSourceSetup.tsx b/src/modules/llms/azure/AzureSourceSetup.tsx index d6d5a7742..f37e0365d 100644 --- a/src/modules/llms/azure/AzureSourceSetup.tsx +++ b/src/modules/llms/azure/AzureSourceSetup.tsx @@ -10,9 +10,8 @@ import { Link } from '~/common/components/Link'; import { asValidURL } from '~/common/util/urlUtils'; import { settingsGap } from '~/common/theme'; -import { DModelSourceId } from '../llm.types'; +import { DModelSourceId, useModelsStore, useSourceSetup } from '../store-llms'; import { modelDescriptionToDLLM } from '../llm.router'; -import { useModelsStore, useSourceSetup } from '../store-llms'; import { hasServerKeyAzure, isValidAzureApiKey, ModelVendorAzure } from './azure.vendor'; diff --git a/src/modules/llms/azure/azure.vendor.ts b/src/modules/llms/azure/azure.vendor.ts index d4492da8c..bbb5532a5 100644 --- a/src/modules/llms/azure/azure.vendor.ts +++ b/src/modules/llms/azure/azure.vendor.ts @@ -2,7 +2,8 @@ import { apiAsync } from '~/modules/trpc/trpc.client'; import { AzureIcon } from '~/common/components/icons/AzureIcon'; -import { DLLM, ModelVendor } from '../llm.types'; +import { DLLM } from '../store-llms'; +import { IModelVendor } from '../vendors/IModelVendor'; import { VChatFunctionIn, VChatMessageIn, VChatMessageOrFunctionCallOut, VChatMessageOut } from '../llm.client'; import { LLMOptionsOpenAI } from '../openai/openai.vendor'; @@ -36,7 +37,7 @@ export interface SourceSetupAzure { * * Work in progress... */ -export const ModelVendorAzure: ModelVendor = { +export const ModelVendorAzure: IModelVendor = { id: 'azure', name: 'Azure', rank: 14, diff --git a/src/modules/llms/llm.client.ts b/src/modules/llms/llm.client.ts index 62f4fa841..70089800c 100644 --- a/src/modules/llms/llm.client.ts +++ b/src/modules/llms/llm.client.ts @@ -2,13 +2,15 @@ import { apiAsync } from '~/modules/trpc/trpc.client'; import type { DMessage } from '~/common/state/store-chats'; -import type { ChatStreamSchema } from './openai/openai.router'; -import type { DLLM, DLLMId, ModelVendor } from './llm.types'; -import type { OpenAI } from './openai/openai.types'; +import { ChatStreamSchema } from './openai/openai.router'; import { ModelVendorAnthropic, SourceSetupAnthropic } from './anthropic/anthropic.vendor'; import { ModelVendorOpenAI, SourceSetupOpenAI } from './openai/openai.vendor'; -import { findVendorById } from './vendor.registry'; -import { useModelsStore } from './store-llms'; +import { OpenAI } from './openai/openai.types'; + +import { IModelVendor } from './vendors/IModelVendor'; +import { findVendorById } from './vendors/vendor.registry'; + +import { DLLM, DLLMId, useModelsStore } from './store-llms'; export interface VChatMessageIn { @@ -76,7 +78,7 @@ function getLLMAndVendorOrThrow(llmId: DLLMId) { * @param updateMessage callback when a piece of a message (text, model name, typing..) is received */ async function vendorStreamChat( - vendor: ModelVendor, llm: DLLM, messages: VChatMessageIn[], + vendor: IModelVendor, llm: DLLM, messages: VChatMessageIn[], abortSignal: AbortSignal, updateMessage: (updatedMessage: Partial, done: boolean) => void, ) { diff --git a/src/modules/llms/llm.router.ts b/src/modules/llms/llm.router.ts index 2780750a6..57cef5c77 100644 --- a/src/modules/llms/llm.router.ts +++ b/src/modules/llms/llm.router.ts @@ -1,6 +1,6 @@ import { z } from 'zod'; -import { DLLM, DModelSource } from './llm.types'; +import { DLLM, DModelSource } from './store-llms'; import { LLMOptionsOpenAI } from './openai/openai.vendor'; // these are constants used for model interfaces (chat, and function calls) diff --git a/src/modules/llms/llm.types.ts b/src/modules/llms/llm.types.ts deleted file mode 100644 index febb65df7..000000000 --- a/src/modules/llms/llm.types.ts +++ /dev/null @@ -1,61 +0,0 @@ -import type React from 'react'; -import type { VChatFunctionIn, VChatMessageIn, VChatMessageOrFunctionCallOut, VChatMessageOut } from './llm.client'; - - -export type DLLMId = string; -export type DModelSourceId = string; -export type ModelVendorId = 'anthropic' | 'azure' | 'localai' | 'oobabooga' | 'openai' | 'openrouter'; - - -/// Large Language Model - a model that can generate text -export interface DLLM { - id: DLLMId; - label: string; - created: number | 0; - updated?: number | 0; - description: string; - tags: string[]; // UNUSED for now - contextTokens: number; - hidden: boolean; - - // llm -> source - sId: DModelSourceId; - _source: TModelSource; - - // llm-specific - options: Partial<{ llmRef: string } & TLLMOptions>; -} - - -/// An origin of models - has enough parameters to list models and invoke generation -export interface DModelSource { - id: DModelSourceId; - label: string; - - // source -> vendor - vId: ModelVendorId; - - // source-specific - setup: Partial; -} - - -/// Hardcoded vendors - have factory methods to enable dynamic configuration / access -export interface ModelVendor { - id: ModelVendorId; - name: string; - rank: number; - location: 'local' | 'cloud'; - instanceLimit: number; - - // components - Icon: React.ComponentType; - SourceSetupComponent: React.ComponentType<{ sourceId: DModelSourceId }>; - LLMOptionsComponent: React.ComponentType<{ llm: DLLM }>; - - // functions - initializeSetup?: () => TSourceSetup; - normalizeSetup: (partialSetup?: Partial) => TSourceSetup; - callChat: (llm: DLLM, messages: VChatMessageIn[], maxTokens?: number) => Promise; - callChatWithFunctions: (llm: DLLM, messages: VChatMessageIn[], functions: VChatFunctionIn[], forceFunctionName?: string, maxTokens?: number) => Promise; -} \ No newline at end of file diff --git a/src/modules/llms/localai/LocalAISourceSetup.tsx b/src/modules/llms/localai/LocalAISourceSetup.tsx index 6c49aa419..862dbdd24 100644 --- a/src/modules/llms/localai/LocalAISourceSetup.tsx +++ b/src/modules/llms/localai/LocalAISourceSetup.tsx @@ -12,11 +12,10 @@ import { InlineError } from '~/common/components/InlineError'; import { Link } from '~/common/components/Link'; import { settingsGap } from '~/common/theme'; -import { LLMOptionsOpenAI, ModelVendorOpenAI } from '~/modules/llms/openai/openai.vendor'; +import { LLMOptionsOpenAI, ModelVendorOpenAI } from '../openai/openai.vendor'; -import { DLLM, DModelSource, DModelSourceId } from '../llm.types'; +import { DLLM, DModelSource, DModelSourceId, useModelsStore, useSourceSetup } from '../store-llms'; import { ModelVendorLocalAI } from './localai.vendor'; -import { useModelsStore, useSourceSetup } from '../store-llms'; const urlSchema = z.string().url().startsWith('http'); diff --git a/src/modules/llms/localai/localai.vendor.tsx b/src/modules/llms/localai/localai.vendor.tsx index 067eaf8d4..6c1e90a39 100644 --- a/src/modules/llms/localai/localai.vendor.tsx +++ b/src/modules/llms/localai/localai.vendor.tsx @@ -1,6 +1,6 @@ import DevicesIcon from '@mui/icons-material/Devices'; -import { ModelVendor } from '../llm.types'; +import { IModelVendor } from '../vendors/IModelVendor'; import { LLMOptionsOpenAI, ModelVendorOpenAI } from '../openai/openai.vendor'; import { OpenAILLMOptions } from '../openai/OpenAILLMOptions'; @@ -12,7 +12,7 @@ export interface SourceSetupLocalAI { oaiHost: string; // use OpenAI-compatible non-default hosts (full origin path) } -export const ModelVendorLocalAI: ModelVendor = { +export const ModelVendorLocalAI: IModelVendor = { id: 'localai', name: 'LocalAI', rank: 20, diff --git a/src/modules/llms/oobabooga/OobaboogaSourceSetup.tsx b/src/modules/llms/oobabooga/OobaboogaSourceSetup.tsx index 9853c6bf1..2d78308f8 100644 --- a/src/modules/llms/oobabooga/OobaboogaSourceSetup.tsx +++ b/src/modules/llms/oobabooga/OobaboogaSourceSetup.tsx @@ -12,9 +12,8 @@ import { settingsCol1Width, settingsGap } from '~/common/theme'; import { LLMOptionsOpenAI, ModelVendorOpenAI } from '~/modules/llms/openai/openai.vendor'; import { OpenAI } from '~/modules/llms/openai/openai.types'; -import { DLLM, DModelSource, DModelSourceId } from '../llm.types'; +import { DLLM, DModelSource, DModelSourceId, useModelsStore, useSourceSetup } from '../store-llms'; import { ModelVendorOoobabooga } from './oobabooga.vendor'; -import { useModelsStore, useSourceSetup } from '../store-llms'; export function OobaboogaSourceSetup(props: { sourceId: DModelSourceId }) { diff --git a/src/modules/llms/oobabooga/oobabooga.vendor.ts b/src/modules/llms/oobabooga/oobabooga.vendor.ts index 7b53b2f7a..7e1602e5a 100644 --- a/src/modules/llms/oobabooga/oobabooga.vendor.ts +++ b/src/modules/llms/oobabooga/oobabooga.vendor.ts @@ -1,6 +1,6 @@ import { OobaboogaIcon } from '~/common/components/icons/OobaboogaIcon'; -import { ModelVendor } from '../llm.types'; +import { IModelVendor } from '../vendors/IModelVendor'; import { LLMOptionsOpenAI, ModelVendorOpenAI } from '../openai/openai.vendor'; import { OpenAILLMOptions } from '../openai/OpenAILLMOptions'; @@ -12,7 +12,7 @@ export interface SourceSetupOobabooga { oaiHost: string; // use OpenAI-compatible non-default hosts (full origin path) } -export const ModelVendorOoobabooga: ModelVendor = { +export const ModelVendorOoobabooga: IModelVendor = { id: 'oobabooga', name: 'Oobabooga', rank: 15, diff --git a/src/modules/llms/openai/OpenAILLMOptions.tsx b/src/modules/llms/openai/OpenAILLMOptions.tsx index e410282a4..1f4a3c010 100644 --- a/src/modules/llms/openai/OpenAILLMOptions.tsx +++ b/src/modules/llms/openai/OpenAILLMOptions.tsx @@ -3,9 +3,8 @@ import * as React from 'react'; import { Box, FormControl, FormHelperText, FormLabel, Slider } from '@mui/joy'; import { settingsCol1Width, settingsGap } from '~/common/theme'; -import { DLLM } from '../llm.types'; +import { DLLM, useModelsStore } from '../store-llms'; import { LLMOptionsOpenAI } from './openai.vendor'; -import { useModelsStore } from '../store-llms'; function normalizeOpenAIOptions(partialOptions?: Partial) { return { diff --git a/src/modules/llms/openai/OpenAISourceSetup.tsx b/src/modules/llms/openai/OpenAISourceSetup.tsx index 07984ad26..56d794d58 100644 --- a/src/modules/llms/openai/OpenAISourceSetup.tsx +++ b/src/modules/llms/openai/OpenAISourceSetup.tsx @@ -11,10 +11,9 @@ import { InlineError } from '~/common/components/InlineError'; import { Link } from '~/common/components/Link'; import { settingsCol1Width, settingsGap } from '~/common/theme'; -import { DLLM, DModelSource, DModelSourceId } from '../llm.types'; +import { DLLM, DModelSource, DModelSourceId, useModelsStore, useSourceSetup } from '../store-llms'; import { hasServerKeyOpenAI, isValidOpenAIApiKey, LLMOptionsOpenAI, ModelVendorOpenAI } from './openai.vendor'; import { openAIModelToModelDescription } from './openai.data'; -import { useModelsStore, useSourceSetup } from '../store-llms'; export function OpenAISourceSetup(props: { sourceId: DModelSourceId }) { diff --git a/src/modules/llms/openai/openai.vendor.ts b/src/modules/llms/openai/openai.vendor.ts index 5ee7f0763..84662b0a7 100644 --- a/src/modules/llms/openai/openai.vendor.ts +++ b/src/modules/llms/openai/openai.vendor.ts @@ -2,7 +2,8 @@ import { apiAsync } from '~/modules/trpc/trpc.client'; import { OpenAIIcon } from '~/common/components/icons/OpenAIIcon'; -import { DLLM, ModelVendor } from '../llm.types'; +import { DLLM } from '../store-llms'; +import { IModelVendor } from '../vendors/IModelVendor'; import { VChatFunctionIn, VChatMessageIn, VChatMessageOrFunctionCallOut, VChatMessageOut } from '../llm.client'; import { OpenAILLMOptions } from './OpenAILLMOptions'; @@ -27,7 +28,7 @@ export interface LLMOptionsOpenAI { llmResponseTokens: number; } -export const ModelVendorOpenAI: ModelVendor = { +export const ModelVendorOpenAI: IModelVendor = { id: 'openai', name: 'OpenAI', rank: 10, diff --git a/src/modules/llms/openrouter/OpenRouterSourceSetup.tsx b/src/modules/llms/openrouter/OpenRouterSourceSetup.tsx index 9d55c4447..08db1105e 100644 --- a/src/modules/llms/openrouter/OpenRouterSourceSetup.tsx +++ b/src/modules/llms/openrouter/OpenRouterSourceSetup.tsx @@ -12,8 +12,7 @@ import { InlineError } from '~/common/components/InlineError'; import { Link } from '~/common/components/Link'; import { settingsGap } from '~/common/theme'; -import { DLLM, DModelSource, DModelSourceId } from '../llm.types'; -import { useModelsStore, useSourceSetup } from '../store-llms'; +import { DLLM, DModelSource, DModelSourceId, useModelsStore, useSourceSetup } from '../store-llms'; import { isValidOpenRouterKey, ModelVendorOpenRouter } from './openrouter.vendor'; diff --git a/src/modules/llms/openrouter/openrouter.vendor.tsx b/src/modules/llms/openrouter/openrouter.vendor.tsx index 174ef26e7..b8d7d8a12 100644 --- a/src/modules/llms/openrouter/openrouter.vendor.tsx +++ b/src/modules/llms/openrouter/openrouter.vendor.tsx @@ -1,6 +1,6 @@ import { OpenRouterIcon } from '~/common/components/icons/OpenRouterIcon'; -import { ModelVendor } from '../llm.types'; +import { IModelVendor } from '../vendors/IModelVendor'; import { LLMOptionsOpenAI, ModelVendorOpenAI } from '../openai/openai.vendor'; import { OpenAILLMOptions } from '../openai/OpenAILLMOptions'; @@ -28,7 +28,7 @@ export interface SourceSetupOpenRouter { * [x] decide whether to do UI work to improve the appearance - prioritized models * [x] works! */ -export const ModelVendorOpenRouter: ModelVendor = { +export const ModelVendorOpenRouter: IModelVendor = { id: 'openrouter', name: 'OpenRouter', rank: 25, diff --git a/src/modules/llms/store-llms.ts b/src/modules/llms/store-llms.ts index 4be1e3076..84905078a 100644 --- a/src/modules/llms/store-llms.ts +++ b/src/modules/llms/store-llms.ts @@ -2,22 +2,59 @@ import { create } from 'zustand'; import { shallow } from 'zustand/shallow'; import { persist } from 'zustand/middleware'; -import { DLLM, DLLMId, DModelSource, DModelSourceId } from './llm.types'; +import { ModelVendorId } from './vendors/IModelVendor'; +export type DLLMId = string; +export type DModelSourceId = string; -/// ModelsStore - a store for LLMs and their origins -interface ModelsStore { +/** + * Large Language Model - description and configuration (data object, stored) + */ +export interface DLLM { + + id: DLLMId; + label: string; + created: number | 0; + updated?: number | 0; + description: string; + tags: string[]; // UNUSED for now + contextTokens: number; + hidden: boolean; + + // llm -> source + sId: DModelSourceId; + _source: TModelSource; + + // llm-specific + options: Partial<{ llmRef: string } & TLLMOptions>; +} + +/** + * Model Server - configured to be a unique origin of models (data object, stored) + */ +export interface DModelSource { + id: DModelSourceId; + label: string; + + // source -> vendor + vId: ModelVendorId; + + // source-specific + setup: Partial; +} + + +/// ModelsStore - a store for LLMs and their origins +interface ModelsData { + llms: DLLM[]; + sources: DModelSource[]; chatLLMId: DLLMId | null; fastLLMId: DLLMId | null; funcLLMId: DLLMId | null; - llms: DLLM[]; - sources: DModelSource[]; - - setChatLLMId: (id: DLLMId | null) => void; - setFastLLMId: (id: DLLMId | null) => void; - setFuncLLMId: (id: DLLMId | null) => void; +} +interface ModelsActions { addLLMs: (llms: DLLM[]) => void; removeLLM: (id: DLLMId) => void; updateLLM: (id: DLLMId, partial: Partial) => void; @@ -27,18 +64,20 @@ interface ModelsStore { removeSource: (id: DModelSourceId) => void; updateSourceSetup: (id: DModelSourceId, partialSetup: Partial) => void; + setChatLLMId: (id: DLLMId | null) => void; + setFastLLMId: (id: DLLMId | null) => void; + setFuncLLMId: (id: DLLMId | null) => void; } - -export const useModelsStore = create()( +export const useModelsStore = create()( persist( (set) => ({ + llms: [], + sources: [], chatLLMId: null, fastLLMId: null, funcLLMId: null, - llms: [], - sources: [], setChatLLMId: (id: DLLMId | null) => set(state => updateSelectedIds(state.llms, id, state.fastLLMId, state.funcLLMId)), @@ -118,7 +157,7 @@ export const useModelsStore = create()( { name: 'app-models', - // omit the memory references from the persisted state + // Pre-saving: omit the memory references from the persisted state partialize: (state) => ({ ...state, llms: state.llms.map(llm => { @@ -127,7 +166,7 @@ export const useModelsStore = create()( }), }), - // re-link the memory references on rehydration + // Post-loading: re-link the memory references on rehydration onRehydrateStorage: () => (state) => { if (!state) return; @@ -155,7 +194,7 @@ function findLlmIdBySuffix(llms: DLLM[], suffixes: string[], fallbackToFirst: bo return fallbackToFirst ? llms[0].id : null; } -function updateSelectedIds(allLlms: DLLM[], chatLlmId: DLLMId | null, fastLlmId: DLLMId | null, funcLlmId: DLLMId | null): Partial { +function updateSelectedIds(allLlms: DLLM[], chatLlmId: DLLMId | null, fastLlmId: DLLMId | null, funcLlmId: DLLMId | null): Partial { if (chatLlmId && !allLlms.find(llm => llm.id === chatLlmId)) chatLlmId = null; if (!chatLlmId) chatLlmId = findLlmIdBySuffix(allLlms, defaultChatSuffixPreference, true); diff --git a/src/modules/llms/vendors/IModelVendor.ts b/src/modules/llms/vendors/IModelVendor.ts new file mode 100644 index 000000000..6debccd09 --- /dev/null +++ b/src/modules/llms/vendors/IModelVendor.ts @@ -0,0 +1,27 @@ +import type React from 'react'; + +import type { DLLM, DModelSourceId } from '../store-llms'; +import type { VChatFunctionIn, VChatMessageIn, VChatMessageOrFunctionCallOut, VChatMessageOut } from '../llm.client'; + + +export type ModelVendorId = 'anthropic' | 'azure' | 'localai' | 'oobabooga' | 'openai' | 'openrouter'; + + +export interface IModelVendor { + id: ModelVendorId; + name: string; + rank: number; + location: 'local' | 'cloud'; + instanceLimit: number; + + // components + Icon: React.ComponentType; + SourceSetupComponent: React.ComponentType<{ sourceId: DModelSourceId }>; + LLMOptionsComponent: React.ComponentType<{ llm: DLLM }>; + + // functions + initializeSetup?: () => TSourceSetup; + normalizeSetup: (partialSetup?: Partial) => TSourceSetup; + callChat: (llm: DLLM, messages: VChatMessageIn[], maxTokens?: number) => Promise; + callChatWithFunctions: (llm: DLLM, messages: VChatMessageIn[], functions: VChatFunctionIn[], forceFunctionName?: string, maxTokens?: number) => Promise; +} \ No newline at end of file diff --git a/src/modules/llms/vendor.registry.ts b/src/modules/llms/vendors/vendor.registry.ts similarity index 58% rename from src/modules/llms/vendor.registry.ts rename to src/modules/llms/vendors/vendor.registry.ts index c4da6cee9..23b1eeab3 100644 --- a/src/modules/llms/vendor.registry.ts +++ b/src/modules/llms/vendors/vendor.registry.ts @@ -1,39 +1,24 @@ -import { DModelSource, DModelSourceId, ModelVendor, ModelVendorId } from './llm.types'; -import { ModelVendorAnthropic } from './anthropic/anthropic.vendor'; -import { ModelVendorAzure } from './azure/azure.vendor'; -import { ModelVendorLocalAI } from './localai/localai.vendor'; -import { ModelVendorOoobabooga } from './oobabooga/oobabooga.vendor'; -import { ModelVendorOpenAI } from './openai/openai.vendor'; -import { ModelVendorOpenRouter } from './openrouter/openrouter.vendor'; +import { ModelVendorAnthropic } from '../anthropic/anthropic.vendor'; +import { ModelVendorAzure } from '../azure/azure.vendor'; +import { ModelVendorLocalAI } from '../localai/localai.vendor'; +import { ModelVendorOoobabooga } from '../oobabooga/oobabooga.vendor'; +import { ModelVendorOpenAI } from '../openai/openai.vendor'; +import { ModelVendorOpenRouter } from '../openrouter/openrouter.vendor'; +import { DModelSource, DModelSourceId } from '../store-llms'; +import { IModelVendor, ModelVendorId } from './IModelVendor'; -/// Internal - Main Vendor Registry /// -const MODEL_VENDOR_REGISTRY: Record = { - anthropic: ModelVendorAnthropic, - azure: ModelVendorAzure, - localai: ModelVendorLocalAI, - oobabooga: ModelVendorOoobabooga, - openai: ModelVendorOpenAI, - openrouter: ModelVendorOpenRouter, -}; - -const DEFAULT_MODEL_VENDOR: ModelVendorId = 'openai'; - -export function findAllVendors(): ModelVendor[] { +export function findAllVendors(): IModelVendor[] { const modelVendors = Object.values(MODEL_VENDOR_REGISTRY); modelVendors.sort((a, b) => a.rank - b.rank); return modelVendors; } -export function findVendorById(vendorId?: ModelVendorId): ModelVendor | null { +export function findVendorById(vendorId?: ModelVendorId): IModelVendor | null { return vendorId ? (MODEL_VENDOR_REGISTRY[vendorId] ?? null) : null; } -export function createModelSourceForDefaultVendor(otherSources: DModelSource[]): DModelSource { - return createModelSourceForVendor(DEFAULT_MODEL_VENDOR, otherSources); -} - export function createModelSourceForVendor(vendorId: ModelVendorId, otherSources: DModelSource[]): DModelSource { // get vendor const vendor = findVendorById(vendorId); @@ -54,4 +39,22 @@ export function createModelSourceForVendor(vendorId: ModelVendorId, otherSources vId: vendorId, setup: vendor.initializeSetup?.() || {}, }; -} \ No newline at end of file +} + +export function createModelSourceForDefaultVendor(otherSources: DModelSource[]): DModelSource { + return createModelSourceForVendor(MODEL_VENDOR_DEFAULT, otherSources); +} + + +/// Main Vendor Registry /// + +const MODEL_VENDOR_REGISTRY: Record = { + anthropic: ModelVendorAnthropic, + azure: ModelVendorAzure, + localai: ModelVendorLocalAI, + oobabooga: ModelVendorOoobabooga, + openai: ModelVendorOpenAI, + openrouter: ModelVendorOpenRouter, +}; + +const MODEL_VENDOR_DEFAULT: ModelVendorId = 'openai'; \ No newline at end of file From 1e0f11d06413040baf09615b85497a4821493e91 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Fri, 22 Sep 2023 20:57:32 -0700 Subject: [PATCH 10/17] Llms: move the server side proximally closer --- pages/api/elevenlabs/speech.ts | 4 +- pages/api/llms/stream.ts | 205 +----------------- src/modules/llms/llm.client.ts | 4 +- .../llms/oobabooga/OobaboogaSourceSetup.tsx | 3 +- .../llms/openrouter/OpenRouterSourceSetup.tsx | 5 +- .../server}/anthropic.router.ts | 4 +- .../server}/anthropic.wiretypes.ts | 0 .../server}/azure.router.ts | 8 +- .../server}/openai.router.ts | 2 +- .../transports/server/openai.streaming.ts | 204 +++++++++++++++++ .../server/openai.wiretypes.ts} | 0 src/modules/trpc/trpc.router.ts | 6 +- 12 files changed, 222 insertions(+), 223 deletions(-) rename src/modules/llms/{anthropic => transports/server}/anthropic.router.ts (98%) rename src/modules/llms/{anthropic => transports/server}/anthropic.wiretypes.ts (100%) rename src/modules/llms/{azure => transports/server}/azure.router.ts (96%) rename src/modules/llms/{openai => transports/server}/openai.router.ts (99%) create mode 100644 src/modules/llms/transports/server/openai.streaming.ts rename src/modules/llms/{openai/openai.types.ts => transports/server/openai.wiretypes.ts} (100%) diff --git a/pages/api/elevenlabs/speech.ts b/pages/api/elevenlabs/speech.ts index 87cdb18cf..025a7fe32 100644 --- a/pages/api/elevenlabs/speech.ts +++ b/pages/api/elevenlabs/speech.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; +import { createEmptyReadableStream, throwResponseNotOk } from '~/modules/llms/transports/server/openai.streaming'; import { elevenlabsAccess, elevenlabsVoiceId, ElevenlabsWire, speechInputSchema } from '~/modules/elevenlabs/elevenlabs.router'; -import { createEmptyReadableStream, throwResponseNotOk } from '../llms/stream'; /* NOTE: Why does this file even exist? @@ -10,7 +10,7 @@ This file is a workaround for a limitation in tRPC; it does not support ArrayBuf and that would force us to use base64 encoding for the audio data, which would be a waste of bandwidth. So instead, we use this file to make the request to ElevenLabs, and then return the response as an ArrayBuffer. Unfortunately this means duplicating the code in the server-side -and client-side vs. the TRPC implementation. So at lease we recycle the input structures. +and client-side vs. the tRPC implementation. So at lease we recycle the input structures. */ diff --git a/pages/api/llms/stream.ts b/pages/api/llms/stream.ts index e5468f7da..bb2ff7612 100644 --- a/pages/api/llms/stream.ts +++ b/pages/api/llms/stream.ts @@ -1,207 +1,4 @@ -import { NextRequest, NextResponse } from 'next/server'; -import { createParser as createEventsourceParser, EventSourceParser, ParsedEvent, ReconnectInterval } from 'eventsource-parser'; - -import { AnthropicWire } from '~/modules/llms/anthropic/anthropic.wiretypes'; -import { OpenAI } from '~/modules/llms/openai/openai.types'; -import { anthropicAccess, anthropicCompletionRequest } from '~/modules/llms/anthropic/anthropic.router'; -import { openAIChatStreamSchema, openAIAccess, openAIChatCompletionPayload } from '~/modules/llms/openai/openai.router'; - - -/** - * Vendor stream parsers - * - The vendor can decide to terminate the connection (close: true), transmitting anything in 'text' before doing so - * - The vendor can also throw from this function, which will error and terminate the connection - */ -type AIStreamParser = (data: string) => { text: string, close: boolean }; - - -// The peculiarity of our parser is the injection of a JSON structure at the beginning of the stream, to -// communicate parameters before the text starts flowing to the client. -function parseOpenAIStream(): AIStreamParser { - let hasBegun = false; - let hasWarned = false; - - return data => { - - const json: OpenAI.Wire.ChatCompletion.ResponseStreamingChunk = JSON.parse(data); - - // an upstream error will be handled gracefully and transmitted as text (throw to transmit as 'error') - if (json.error) - return { text: `[OpenAI Issue] ${json.error.message || json.error}`, close: true }; - - if (json.choices.length !== 1) - throw new Error(`[OpenAI Issue] Expected 1 completion, got ${json.choices.length}`); - - const index = json.choices[0].index; - if (index !== 0 && index !== undefined /* LocalAI hack/workaround until https://github.com/go-skynet/LocalAI/issues/788 */) - throw new Error(`[OpenAI Issue] Expected completion index 0, got ${index}`); - let text = json.choices[0].delta?.content /*|| json.choices[0]?.text*/ || ''; - - // hack: prepend the model name to the first packet - if (!hasBegun) { - hasBegun = true; - const firstPacket: OpenAI.API.Chat.StreamingFirstResponse = { - model: json.model, - }; - text = JSON.stringify(firstPacket) + text; - } - - // if there's a warning, log it once - if (json.warning && !hasWarned) { - hasWarned = true; - console.log('/api/llms/stream: OpenAI stream warning:', json.warning); - } - - // workaround: LocalAI doesn't send the [DONE] event, but similarly to OpenAI, it sends a "finish_reason" delta update - const close = !!json.choices[0].finish_reason; - return { text, close }; - }; -} - - -// Anthropic event stream parser -function parseAnthropicStream(): AIStreamParser { - let hasBegun = false; - - return data => { - - const json: AnthropicWire.Complete.Response = JSON.parse(data); - let text = json.completion; - - // hack: prepend the model name to the first packet - if (!hasBegun) { - hasBegun = true; - const firstPacket: OpenAI.API.Chat.StreamingFirstResponse = { - model: json.model, - }; - text = JSON.stringify(firstPacket) + text; - } - - return { text, close: false }; - }; -} - - -/** - * Creates a TransformStream that parses events from an EventSource stream using a custom parser. - * @returns {TransformStream} TransformStream parsing events. - */ -export function createEventStreamTransformer(vendorTextParser: AIStreamParser): TransformStream { - const textDecoder = new TextDecoder(); - const textEncoder = new TextEncoder(); - let eventSourceParser: EventSourceParser; - - return new TransformStream({ - start: async (controller): Promise => { - eventSourceParser = createEventsourceParser( - (event: ParsedEvent | ReconnectInterval) => { - - // ignore 'reconnect-interval' and events with no data - if (event.type !== 'event' || !('data' in event)) - return; - - // event stream termination, close our transformed stream - if (event.data === '[DONE]') { - controller.terminate(); - return; - } - - try { - const { text, close } = vendorTextParser(event.data); - if (text) - controller.enqueue(textEncoder.encode(text)); - if (close) - controller.terminate(); - } catch (error: any) { - // console.log(`/api/llms/stream: parse issue: ${error?.message || error}`); - controller.enqueue(textEncoder.encode(`[Stream Issue] ${error?.message || error}`)); - controller.terminate(); - } - }, - ); - }, - - // stream=true is set because the data is not guaranteed to be final and un-chunked - transform: (chunk: Uint8Array) => { - eventSourceParser.feed(textDecoder.decode(chunk, { stream: true })); - }, - }); -} - -export async function throwResponseNotOk(response: Response) { - if (!response.ok) { - const errorPayload: object | null = await response.json().catch(() => null); - throw new Error(`${response.status} · ${response.statusText}${errorPayload ? ' · ' + JSON.stringify(errorPayload) : ''}`); - } -} - -export function createEmptyReadableStream(): ReadableStream { - return new ReadableStream({ - start: (controller) => controller.close(), - }); -} - - -export default async function handler(req: NextRequest): Promise { - - // inputs - reuse the tRPC schema - const { vendorId, access, model, history } = openAIChatStreamSchema.parse(await req.json()); - - // begin event streaming from the OpenAI API - let upstreamResponse: Response; - let vendorStreamParser: AIStreamParser; - try { - - // prepare the API request data - let headersUrl: { headers: HeadersInit, url: string }; - let body: object; - switch (vendorId) { - case 'anthropic': - headersUrl = anthropicAccess(access as any, '/v1/complete'); - body = anthropicCompletionRequest(model, history, true); - vendorStreamParser = parseAnthropicStream(); - break; - - case 'openai': - headersUrl = openAIAccess(access as any, '/v1/chat/completions'); - body = openAIChatCompletionPayload(model, history, null, null, 1, true); - vendorStreamParser = parseOpenAIStream(); - break; - } - - // POST to our API route - upstreamResponse = await fetch(headersUrl.url, { - method: 'POST', - headers: headersUrl.headers, - body: JSON.stringify(body), - }); - await throwResponseNotOk(upstreamResponse); - - } catch (error: any) { - const fetchOrVendorError = (error?.message || typeof error === 'string' ? error : JSON.stringify(error)) + (error?.cause ? ' · ' + error.cause : ''); - console.log(`/api/llms/stream: fetch issue: ${fetchOrVendorError}`); - return new NextResponse('[OpenAI Issue] ' + fetchOrVendorError, { status: 500 }); - } - - /* The following code is heavily inspired by the Vercel AI SDK, but simplified to our needs and in full control. - * This replaces the former (custom) implementation that used to return a ReadableStream directly, and upon start, - * it was blindly fetching the upstream response and piping it to the client. - * - * We now use backpressure, as explained on: https://sdk.vercel.ai/docs/concepts/backpressure-and-cancellation - * - * NOTE: we have not benchmarked to see if there is performance impact by using this approach - we do want to have - * a 'healthy' level of inventory (i.e., pre-buffering) on the pipe to the client. - */ - const chatResponseStream = (upstreamResponse.body || createEmptyReadableStream()) - .pipeThrough(createEventStreamTransformer(vendorStreamParser)); - - return new NextResponse(chatResponseStream, { - status: 200, - headers: { - 'Content-Type': 'text/event-stream; charset=utf-8', - }, - }); -} +export { openaiStreamingResponse as default } from '~/modules/llms/transports/server/openai.streaming'; // noinspection JSUnusedGlobalSymbols export const runtime = 'edge'; \ No newline at end of file diff --git a/src/modules/llms/llm.client.ts b/src/modules/llms/llm.client.ts index 70089800c..f9e8f1625 100644 --- a/src/modules/llms/llm.client.ts +++ b/src/modules/llms/llm.client.ts @@ -2,10 +2,10 @@ import { apiAsync } from '~/modules/trpc/trpc.client'; import type { DMessage } from '~/common/state/store-chats'; -import { ChatStreamSchema } from './openai/openai.router'; +import type { ChatStreamSchema } from './transports/server/openai.router'; +import type { OpenAI } from './transports/server/openai.wiretypes'; import { ModelVendorAnthropic, SourceSetupAnthropic } from './anthropic/anthropic.vendor'; import { ModelVendorOpenAI, SourceSetupOpenAI } from './openai/openai.vendor'; -import { OpenAI } from './openai/openai.types'; import { IModelVendor } from './vendors/IModelVendor'; import { findVendorById } from './vendors/vendor.registry'; diff --git a/src/modules/llms/oobabooga/OobaboogaSourceSetup.tsx b/src/modules/llms/oobabooga/OobaboogaSourceSetup.tsx index 2d78308f8..67f3638c2 100644 --- a/src/modules/llms/oobabooga/OobaboogaSourceSetup.tsx +++ b/src/modules/llms/oobabooga/OobaboogaSourceSetup.tsx @@ -10,7 +10,6 @@ import { Link } from '~/common/components/Link'; import { settingsCol1Width, settingsGap } from '~/common/theme'; import { LLMOptionsOpenAI, ModelVendorOpenAI } from '~/modules/llms/openai/openai.vendor'; -import { OpenAI } from '~/modules/llms/openai/openai.types'; import { DLLM, DModelSource, DModelSourceId, useModelsStore, useSourceSetup } from '../store-llms'; import { ModelVendorOoobabooga } from './oobabooga.vendor'; @@ -94,7 +93,7 @@ const NotChatModels: string[] = [ ]; -function oobaboogaModelToDLLM(model: OpenAI.Wire.Models.ModelDescription, source: DModelSource): DLLM | null { +function oobaboogaModelToDLLM(model: { id: string, created: number }, source: DModelSource): DLLM | null { // if the model id is one of NotChatModels, we don't want to show it if (NotChatModels.includes(model.id)) return null; diff --git a/src/modules/llms/openrouter/OpenRouterSourceSetup.tsx b/src/modules/llms/openrouter/OpenRouterSourceSetup.tsx index 08db1105e..6b64eb653 100644 --- a/src/modules/llms/openrouter/OpenRouterSourceSetup.tsx +++ b/src/modules/llms/openrouter/OpenRouterSourceSetup.tsx @@ -4,7 +4,6 @@ import { Box, Button, Typography } from '@mui/joy'; import SyncIcon from '@mui/icons-material/Sync'; import { LLMOptionsOpenAI, ModelVendorOpenAI } from '~/modules/llms/openai/openai.vendor'; -import { OpenAI } from '~/modules/llms/openai/openai.types'; import { apiQuery } from '~/modules/trpc/trpc.client'; import { FormInputKey } from '~/common/components/FormInputKey'; @@ -118,7 +117,7 @@ const orModelMap: { [id: string]: { name: string; contextWindowSize: number; isO const orModelFamilyOrder = ['openai/', 'anthropic/', 'google/', 'meta-llama/']; -function orFamilySortFn(a: OpenAI.Wire.Models.ModelDescription, b: OpenAI.Wire.Models.ModelDescription): number { +function orFamilySortFn(a: { id: string }, b: { id: string }): number { const aPrefixIndex = orModelFamilyOrder.findIndex(prefix => a.id.startsWith(prefix)); const bPrefixIndex = orModelFamilyOrder.findIndex(prefix => b.id.startsWith(prefix)); @@ -131,7 +130,7 @@ function orFamilySortFn(a: OpenAI.Wire.Models.ModelDescription, b: OpenAI.Wire.M } -function openRouterModelToDLLM(model: OpenAI.Wire.Models.ModelDescription, source: DModelSource): DLLM { +function openRouterModelToDLLM(model: { id: string, created: number }, source: DModelSource): DLLM { // label: use the known name if available, otherwise format the model id const orModel = orModelMap[model.id] ?? null; const label = orModel?.name || model.id.replace('/', ' · '); diff --git a/src/modules/llms/anthropic/anthropic.router.ts b/src/modules/llms/transports/server/anthropic.router.ts similarity index 98% rename from src/modules/llms/anthropic/anthropic.router.ts rename to src/modules/llms/transports/server/anthropic.router.ts index b637eab6d..06993975e 100644 --- a/src/modules/llms/anthropic/anthropic.router.ts +++ b/src/modules/llms/transports/server/anthropic.router.ts @@ -4,8 +4,8 @@ import { TRPCError } from '@trpc/server'; import { createTRPCRouter, publicProcedure } from '~/modules/trpc/trpc.server'; import { fetchJsonOrTRPCError } from '~/modules/trpc/trpc.serverutils'; -import { chatGenerateOutputSchema, historySchema, modelSchema } from '../openai/openai.router'; -import { listModelsOutputSchema, LLM_IF_OAI_Chat, ModelDescriptionSchema } from '../llm.router'; +import { chatGenerateOutputSchema, historySchema, modelSchema } from './openai.router'; +import { listModelsOutputSchema, LLM_IF_OAI_Chat, ModelDescriptionSchema } from '../../llm.router'; import { AnthropicWire } from './anthropic.wiretypes'; diff --git a/src/modules/llms/anthropic/anthropic.wiretypes.ts b/src/modules/llms/transports/server/anthropic.wiretypes.ts similarity index 100% rename from src/modules/llms/anthropic/anthropic.wiretypes.ts rename to src/modules/llms/transports/server/anthropic.wiretypes.ts diff --git a/src/modules/llms/azure/azure.router.ts b/src/modules/llms/transports/server/azure.router.ts similarity index 96% rename from src/modules/llms/azure/azure.router.ts rename to src/modules/llms/transports/server/azure.router.ts index 5e7fc7169..ff313b258 100644 --- a/src/modules/llms/azure/azure.router.ts +++ b/src/modules/llms/transports/server/azure.router.ts @@ -4,10 +4,10 @@ import { TRPCError } from '@trpc/server'; import { createTRPCRouter, publicProcedure } from '~/modules/trpc/trpc.server'; import { fetchJsonOrTRPCError } from '~/modules/trpc/trpc.serverutils'; -import { OpenAI } from '../openai/openai.types'; -import { chatGenerateOutputSchema, historySchema, modelSchema, openAIChatCompletionPayload } from '../openai/openai.router'; -import { listModelsOutputSchema, ModelDescriptionSchema } from '../llm.router'; -import { openAIModelToModelDescription } from '../openai/openai.data'; +import { OpenAI } from './openai.wiretypes'; +import { chatGenerateOutputSchema, historySchema, modelSchema, openAIChatCompletionPayload } from './openai.router'; +import { listModelsOutputSchema, ModelDescriptionSchema } from '../../llm.router'; +import { openAIModelToModelDescription } from '../../openai/openai.data'; // input schemas diff --git a/src/modules/llms/openai/openai.router.ts b/src/modules/llms/transports/server/openai.router.ts similarity index 99% rename from src/modules/llms/openai/openai.router.ts rename to src/modules/llms/transports/server/openai.router.ts index 1da9c367c..facbfdeef 100644 --- a/src/modules/llms/openai/openai.router.ts +++ b/src/modules/llms/transports/server/openai.router.ts @@ -6,7 +6,7 @@ import { fetchJsonOrTRPCError } from '~/modules/trpc/trpc.serverutils'; import { Brand } from '~/common/brand'; -import { OpenAI } from './openai.types'; +import { OpenAI } from './openai.wiretypes'; // if (!process.env.OPENAI_API_KEY) diff --git a/src/modules/llms/transports/server/openai.streaming.ts b/src/modules/llms/transports/server/openai.streaming.ts new file mode 100644 index 000000000..0991363c8 --- /dev/null +++ b/src/modules/llms/transports/server/openai.streaming.ts @@ -0,0 +1,204 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { createParser as createEventsourceParser, EventSourceParser, ParsedEvent, ReconnectInterval } from 'eventsource-parser'; + +import { AnthropicWire } from './anthropic.wiretypes'; +import { OpenAI } from './openai.wiretypes'; +import { anthropicAccess, anthropicCompletionRequest } from './anthropic.router'; +import { openAIChatStreamSchema, openAIAccess, openAIChatCompletionPayload } from './openai.router'; + + +/** + * Vendor stream parsers + * - The vendor can decide to terminate the connection (close: true), transmitting anything in 'text' before doing so + * - The vendor can also throw from this function, which will error and terminate the connection + */ +type AIStreamParser = (data: string) => { text: string, close: boolean }; + + +// The peculiarity of our parser is the injection of a JSON structure at the beginning of the stream, to +// communicate parameters before the text starts flowing to the client. +function parseOpenAIStream(): AIStreamParser { + let hasBegun = false; + let hasWarned = false; + + return data => { + + const json: OpenAI.Wire.ChatCompletion.ResponseStreamingChunk = JSON.parse(data); + + // an upstream error will be handled gracefully and transmitted as text (throw to transmit as 'error') + if (json.error) + return { text: `[OpenAI Issue] ${json.error.message || json.error}`, close: true }; + + if (json.choices.length !== 1) + throw new Error(`[OpenAI Issue] Expected 1 completion, got ${json.choices.length}`); + + const index = json.choices[0].index; + if (index !== 0 && index !== undefined /* LocalAI hack/workaround until https://github.com/go-skynet/LocalAI/issues/788 */) + throw new Error(`[OpenAI Issue] Expected completion index 0, got ${index}`); + let text = json.choices[0].delta?.content /*|| json.choices[0]?.text*/ || ''; + + // hack: prepend the model name to the first packet + if (!hasBegun) { + hasBegun = true; + const firstPacket: OpenAI.API.Chat.StreamingFirstResponse = { + model: json.model, + }; + text = JSON.stringify(firstPacket) + text; + } + + // if there's a warning, log it once + if (json.warning && !hasWarned) { + hasWarned = true; + console.log('/api/llms/stream: OpenAI stream warning:', json.warning); + } + + // workaround: LocalAI doesn't send the [DONE] event, but similarly to OpenAI, it sends a "finish_reason" delta update + const close = !!json.choices[0].finish_reason; + return { text, close }; + }; +} + + +// Anthropic event stream parser +function parseAnthropicStream(): AIStreamParser { + let hasBegun = false; + + return data => { + + const json: AnthropicWire.Complete.Response = JSON.parse(data); + let text = json.completion; + + // hack: prepend the model name to the first packet + if (!hasBegun) { + hasBegun = true; + const firstPacket: OpenAI.API.Chat.StreamingFirstResponse = { + model: json.model, + }; + text = JSON.stringify(firstPacket) + text; + } + + return { text, close: false }; + }; +} + + +/** + * Creates a TransformStream that parses events from an EventSource stream using a custom parser. + * @returns {TransformStream} TransformStream parsing events. + */ +function createEventStreamTransformer(vendorTextParser: AIStreamParser): TransformStream { + const textDecoder = new TextDecoder(); + const textEncoder = new TextEncoder(); + let eventSourceParser: EventSourceParser; + + return new TransformStream({ + start: async (controller): Promise => { + eventSourceParser = createEventsourceParser( + (event: ParsedEvent | ReconnectInterval) => { + + // ignore 'reconnect-interval' and events with no data + if (event.type !== 'event' || !('data' in event)) + return; + + // event stream termination, close our transformed stream + if (event.data === '[DONE]') { + controller.terminate(); + return; + } + + try { + const { text, close } = vendorTextParser(event.data); + if (text) + controller.enqueue(textEncoder.encode(text)); + if (close) + controller.terminate(); + } catch (error: any) { + // console.log(`/api/llms/stream: parse issue: ${error?.message || error}`); + controller.enqueue(textEncoder.encode(`[Stream Issue] ${error?.message || error}`)); + controller.terminate(); + } + }, + ); + }, + + // stream=true is set because the data is not guaranteed to be final and un-chunked + transform: (chunk: Uint8Array) => { + eventSourceParser.feed(textDecoder.decode(chunk, { stream: true })); + }, + }); +} + +export async function throwResponseNotOk(response: Response) { + if (!response.ok) { + const errorPayload: object | null = await response.json().catch(() => null); + throw new Error(`${response.status} · ${response.statusText}${errorPayload ? ' · ' + JSON.stringify(errorPayload) : ''}`); + } +} + +export function createEmptyReadableStream(): ReadableStream { + return new ReadableStream({ + start: (controller) => controller.close(), + }); +} + + +export async function openaiStreamingResponse(req: NextRequest): Promise { + + // inputs - reuse the tRPC schema + const { vendorId, access, model, history } = openAIChatStreamSchema.parse(await req.json()); + + // begin event streaming from the OpenAI API + let upstreamResponse: Response; + let vendorStreamParser: AIStreamParser; + try { + + // prepare the API request data + let headersUrl: { headers: HeadersInit, url: string }; + let body: object; + switch (vendorId) { + case 'anthropic': + headersUrl = anthropicAccess(access as any, '/v1/complete'); + body = anthropicCompletionRequest(model, history, true); + vendorStreamParser = parseAnthropicStream(); + break; + + case 'openai': + headersUrl = openAIAccess(access as any, '/v1/chat/completions'); + body = openAIChatCompletionPayload(model, history, null, null, 1, true); + vendorStreamParser = parseOpenAIStream(); + break; + } + + // POST to our API route + upstreamResponse = await fetch(headersUrl.url, { + method: 'POST', + headers: headersUrl.headers, + body: JSON.stringify(body), + }); + await throwResponseNotOk(upstreamResponse); + + } catch (error: any) { + const fetchOrVendorError = (error?.message || typeof error === 'string' ? error : JSON.stringify(error)) + (error?.cause ? ' · ' + error.cause : ''); + console.log(`/api/llms/stream: fetch issue: ${fetchOrVendorError}`); + return new NextResponse('[OpenAI Issue] ' + fetchOrVendorError, { status: 500 }); + } + + /* The following code is heavily inspired by the Vercel AI SDK, but simplified to our needs and in full control. + * This replaces the former (custom) implementation that used to return a ReadableStream directly, and upon start, + * it was blindly fetching the upstream response and piping it to the client. + * + * We now use backpressure, as explained on: https://sdk.vercel.ai/docs/concepts/backpressure-and-cancellation + * + * NOTE: we have not benchmarked to see if there is performance impact by using this approach - we do want to have + * a 'healthy' level of inventory (i.e., pre-buffering) on the pipe to the client. + */ + const chatResponseStream = (upstreamResponse.body || createEmptyReadableStream()) + .pipeThrough(createEventStreamTransformer(vendorStreamParser)); + + return new NextResponse(chatResponseStream, { + status: 200, + headers: { + 'Content-Type': 'text/event-stream; charset=utf-8', + }, + }); +} \ No newline at end of file diff --git a/src/modules/llms/openai/openai.types.ts b/src/modules/llms/transports/server/openai.wiretypes.ts similarity index 100% rename from src/modules/llms/openai/openai.types.ts rename to src/modules/llms/transports/server/openai.wiretypes.ts diff --git a/src/modules/trpc/trpc.router.ts b/src/modules/trpc/trpc.router.ts index 393ce0114..82ba735a0 100644 --- a/src/modules/trpc/trpc.router.ts +++ b/src/modules/trpc/trpc.router.ts @@ -2,9 +2,9 @@ import { createTRPCRouter } from './trpc.server'; import { elevenlabsRouter } from '~/modules/elevenlabs/elevenlabs.router'; import { googleSearchRouter } from '~/modules/google/search.router'; -import { llmAnthropicRouter } from '~/modules/llms/anthropic/anthropic.router'; -import { llmAzureRouter } from '~/modules/llms/azure/azure.router'; -import { llmOpenAIRouter } from '~/modules/llms/openai/openai.router'; +import { llmAnthropicRouter } from '~/modules/llms/transports/server/anthropic.router'; +import { llmAzureRouter } from '~/modules/llms/transports/server/azure.router'; +import { llmOpenAIRouter } from '~/modules/llms/transports/server/openai.router'; import { prodiaRouter } from '~/modules/prodia/prodia.router'; import { sharingRouter } from '~/modules/sharing/sharing.router'; import { ytPersonaRouter } from '../../apps/personas/ytpersona.router'; From 2f92c81bee6368ec02d1621e357f9f1cf1b37e3e Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Fri, 22 Sep 2023 21:15:16 -0700 Subject: [PATCH 11/17] Llms: small move --- src/common/util/token-counter.ts | 3 +-- src/modules/aifn/summarize/summerize.ts | 4 ++-- src/modules/llms/llm.client.ts | 9 +-------- src/modules/llms/store-llms.ts | 21 +++++++++++++-------- 4 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/common/util/token-counter.ts b/src/common/util/token-counter.ts index a3ea91901..4a1b25a48 100644 --- a/src/common/util/token-counter.ts +++ b/src/common/util/token-counter.ts @@ -1,7 +1,6 @@ import { encoding_for_model, get_encoding, Tiktoken, TiktokenModel } from '@dqbd/tiktoken'; -import { findLLMOrThrow } from '~/modules/llms/llm.client'; -import { DLLMId, useModelsStore } from '~/modules/llms/store-llms'; +import { DLLMId, findLLMOrThrow, useModelsStore } from '~/modules/llms/store-llms'; // Do not set this to true in production, it's very verbose diff --git a/src/modules/aifn/summarize/summerize.ts b/src/modules/aifn/summarize/summerize.ts index 2e06950ff..cb1f97d38 100644 --- a/src/modules/aifn/summarize/summerize.ts +++ b/src/modules/aifn/summarize/summerize.ts @@ -1,5 +1,5 @@ -import { DLLMId } from '~/modules/llms/store-llms'; -import { callChatGenerate, findLLMOrThrow } from '~/modules/llms/llm.client'; +import { DLLMId, findLLMOrThrow } from '~/modules/llms/store-llms'; +import { callChatGenerate } from '~/modules/llms/llm.client'; // prompt to be tried when doing recursive summerization. diff --git a/src/modules/llms/llm.client.ts b/src/modules/llms/llm.client.ts index f9e8f1625..ed0b3ef2c 100644 --- a/src/modules/llms/llm.client.ts +++ b/src/modules/llms/llm.client.ts @@ -10,7 +10,7 @@ import { ModelVendorOpenAI, SourceSetupOpenAI } from './openai/openai.vendor'; import { IModelVendor } from './vendors/IModelVendor'; import { findVendorById } from './vendors/vendor.registry'; -import { DLLM, DLLMId, useModelsStore } from './store-llms'; +import { DLLM, DLLMId, findLLMOrThrow } from './store-llms'; export interface VChatMessageIn { @@ -49,13 +49,6 @@ export async function callChatGenerateWithFunctions(llmId: DLLMId, messages: VCh } -export function findLLMOrThrow(llmId: DLLMId): DLLM { - const llm = useModelsStore.getState().llms.find(llm => llm.id === llmId); - if (!llm) throw new Error(`LLM ${llmId} not found`); - if (!llm._source) throw new Error(`LLM ${llmId} has no source`); - return llm as DLLM; -} - function getLLMAndVendorOrThrow(llmId: DLLMId) { const llm = findLLMOrThrow(llmId); const vendor = findVendorById(llm?._source.vId); diff --git a/src/modules/llms/store-llms.ts b/src/modules/llms/store-llms.ts index 84905078a..30081ba7e 100644 --- a/src/modules/llms/store-llms.ts +++ b/src/modules/llms/store-llms.ts @@ -184,6 +184,13 @@ const defaultChatSuffixPreference = ['gpt-4-0613', 'gpt-4', 'gpt-4-32k', 'gpt-3. const defaultFastSuffixPreference = ['gpt-3.5-turbo-0613', 'gpt-3.5-turbo-16k-0613', 'gpt-3.5-turbo-16k', 'gpt-3.5-turbo']; const defaultFuncSuffixPreference = ['gpt-3.5-turbo-0613', 'gpt-4-0613']; +export function findLLMOrThrow(llmId: DLLMId): DLLM { + const llm = useModelsStore.getState().llms.find(llm => llm.id === llmId); + if (!llm) throw new Error(`LLM ${llmId} not found`); + if (!llm._source) throw new Error(`LLM ${llmId} has no source`); + return llm as DLLM; +} + function findLlmIdBySuffix(llms: DLLM[], suffixes: string[], fallbackToFirst: boolean): DLLMId | null { if (!llms?.length) return null; for (const suffix of suffixes) @@ -207,21 +214,19 @@ function updateSelectedIds(allLlms: DLLM[], chatLlmId: DLLMId | null, fastLlmId: return { chatLLMId: chatLlmId, fastLLMId: fastLlmId, funcLLMId: funcLlmId }; } - +/** + * Current 'Chat' LLM, or null + */ export function useChatLLM() { return useModelsStore(state => { const { chatLLMId } = state; const chatLLM = chatLLMId ? state.llms.find(llm => llm.id === chatLLMId) ?? null : null; - return { - chatLLMId, - chatLLM, - }; + return { chatLLMId, chatLLM }; }, shallow); } - /** - * Hook used for Source-specific setup + * Source-specific read/write - great time saver */ export function useSourceSetup(sourceId: DModelSourceId, normalizer: (partialSetup?: Partial) => T) { // invalidate when the setup changes @@ -238,4 +243,4 @@ export function useSourceSetup(sourceId: DModelSourceId, normalizer: (partial // convenience function for this source const updateSetup = (partialSetup: Partial) => updateSourceSetup(sourceId, partialSetup); return { ...rest, updateSetup }; -} +} \ No newline at end of file From 1597675f4ec8cce38ce3dc4aeec1e52edc84bd23 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Fri, 22 Sep 2023 21:46:05 -0700 Subject: [PATCH 12/17] Llms: separate client transport functions --- src/apps/chat/editors/chat-stream.ts | 2 +- src/apps/personas/useLLMChain.ts | 2 +- .../aifn/autosuggestions/autoSuggestions.ts | 2 +- src/modules/aifn/autotitle/autoTitle.ts | 2 +- src/modules/aifn/flatten/flatten.ts | 2 +- .../aifn/imagine/imaginePromptFromText.ts | 2 +- src/modules/aifn/react/react.ts | 2 +- src/modules/aifn/summarize/summerize.ts | 2 +- .../llms/anthropic/anthropic.vendor.ts | 2 +- src/modules/llms/azure/azure.vendor.ts | 2 +- src/modules/llms/openai/openai.vendor.ts | 2 +- src/modules/llms/transports/chatGenerate.ts | 35 ++++++ .../streamChat.ts} | 102 ++++++------------ src/modules/llms/vendors/IModelVendor.ts | 2 +- src/modules/llms/vendors/vendor.registry.ts | 9 +- 15 files changed, 89 insertions(+), 81 deletions(-) create mode 100644 src/modules/llms/transports/chatGenerate.ts rename src/modules/llms/{llm.client.ts => transports/streamChat.ts} (59%) diff --git a/src/apps/chat/editors/chat-stream.ts b/src/apps/chat/editors/chat-stream.ts index 0b3d7bb9c..0df36ea0c 100644 --- a/src/apps/chat/editors/chat-stream.ts +++ b/src/apps/chat/editors/chat-stream.ts @@ -3,7 +3,7 @@ import { DLLMId } from '~/modules/llms/store-llms'; import { autoSuggestions } from '~/modules/aifn/autosuggestions/autoSuggestions'; import { autoTitle } from '~/modules/aifn/autotitle/autoTitle'; import { speakText } from '~/modules/elevenlabs/elevenlabs.client'; -import { streamChat } from '~/modules/llms/llm.client'; +import { streamChat } from '~/modules/llms/transports/streamChat'; import { useElevenlabsStore } from '~/modules/elevenlabs/store-elevenlabs'; import { DMessage, useChatStore } from '~/common/state/store-chats'; diff --git a/src/apps/personas/useLLMChain.ts b/src/apps/personas/useLLMChain.ts index 3da6ab3ff..9215cc88c 100644 --- a/src/apps/personas/useLLMChain.ts +++ b/src/apps/personas/useLLMChain.ts @@ -1,7 +1,7 @@ import * as React from 'react'; import { DLLMId, useModelsStore } from '~/modules/llms/store-llms'; -import { callChatGenerate, VChatMessageIn } from '~/modules/llms/llm.client'; +import { callChatGenerate, VChatMessageIn } from '~/modules/llms/transports/chatGenerate'; export interface LLMChainStep { diff --git a/src/modules/aifn/autosuggestions/autoSuggestions.ts b/src/modules/aifn/autosuggestions/autoSuggestions.ts index 8f767d9bd..bbbf76762 100644 --- a/src/modules/aifn/autosuggestions/autoSuggestions.ts +++ b/src/modules/aifn/autosuggestions/autoSuggestions.ts @@ -1,4 +1,4 @@ -import { callChatGenerateWithFunctions, VChatFunctionIn } from '~/modules/llms/llm.client'; +import { callChatGenerateWithFunctions, VChatFunctionIn } from '~/modules/llms/transports/chatGenerate'; import { useModelsStore } from '~/modules/llms/store-llms'; import { useChatStore } from '~/common/state/store-chats'; diff --git a/src/modules/aifn/autotitle/autoTitle.ts b/src/modules/aifn/autotitle/autoTitle.ts index 1fe23db59..ae5de2bbb 100644 --- a/src/modules/aifn/autotitle/autoTitle.ts +++ b/src/modules/aifn/autotitle/autoTitle.ts @@ -1,4 +1,4 @@ -import { callChatGenerate } from '~/modules/llms/llm.client'; +import { callChatGenerate } from '~/modules/llms/transports/chatGenerate'; import { useModelsStore } from '~/modules/llms/store-llms'; import { useChatStore } from '~/common/state/store-chats'; diff --git a/src/modules/aifn/flatten/flatten.ts b/src/modules/aifn/flatten/flatten.ts index f0af3c21e..2bd2c653d 100644 --- a/src/modules/aifn/flatten/flatten.ts +++ b/src/modules/aifn/flatten/flatten.ts @@ -1,5 +1,5 @@ import { DLLMId } from '~/modules/llms/store-llms'; -import { callChatGenerate } from '~/modules/llms/llm.client'; +import { callChatGenerate } from '~/modules/llms/transports/chatGenerate'; import { DConversation } from '~/common/state/store-chats'; diff --git a/src/modules/aifn/imagine/imaginePromptFromText.ts b/src/modules/aifn/imagine/imaginePromptFromText.ts index 26dba2acb..211ac7abd 100644 --- a/src/modules/aifn/imagine/imaginePromptFromText.ts +++ b/src/modules/aifn/imagine/imaginePromptFromText.ts @@ -1,4 +1,4 @@ -import { callChatGenerate } from '~/modules/llms/llm.client'; +import { callChatGenerate } from '~/modules/llms/transports/chatGenerate'; import { useModelsStore } from '~/modules/llms/store-llms'; diff --git a/src/modules/aifn/react/react.ts b/src/modules/aifn/react/react.ts index 4ba73b75b..a86f361dd 100644 --- a/src/modules/aifn/react/react.ts +++ b/src/modules/aifn/react/react.ts @@ -4,7 +4,7 @@ import { DLLMId } from '~/modules/llms/store-llms'; import { callApiSearchGoogle } from '~/modules/google/search.client'; -import { callChatGenerate, VChatMessageIn } from '~/modules/llms/llm.client'; +import { callChatGenerate, VChatMessageIn } from '~/modules/llms/transports/chatGenerate'; // prompt to implement the ReAct paradigm: https://arxiv.org/abs/2210.03629 diff --git a/src/modules/aifn/summarize/summerize.ts b/src/modules/aifn/summarize/summerize.ts index cb1f97d38..8cd3b05c2 100644 --- a/src/modules/aifn/summarize/summerize.ts +++ b/src/modules/aifn/summarize/summerize.ts @@ -1,5 +1,5 @@ import { DLLMId, findLLMOrThrow } from '~/modules/llms/store-llms'; -import { callChatGenerate } from '~/modules/llms/llm.client'; +import { callChatGenerate } from '~/modules/llms/transports/chatGenerate'; // prompt to be tried when doing recursive summerization. diff --git a/src/modules/llms/anthropic/anthropic.vendor.ts b/src/modules/llms/anthropic/anthropic.vendor.ts index 201ad3f8e..9889ab0df 100644 --- a/src/modules/llms/anthropic/anthropic.vendor.ts +++ b/src/modules/llms/anthropic/anthropic.vendor.ts @@ -4,7 +4,7 @@ import { AnthropicIcon } from '~/common/components/icons/AnthropicIcon'; import { DLLM } from '../store-llms'; import { IModelVendor } from '../vendors/IModelVendor'; -import { VChatMessageIn, VChatMessageOut } from '../llm.client'; +import { VChatMessageIn, VChatMessageOut } from '../transports/chatGenerate'; import { LLMOptionsOpenAI } from '../openai/openai.vendor'; import { OpenAILLMOptions } from '../openai/OpenAILLMOptions'; diff --git a/src/modules/llms/azure/azure.vendor.ts b/src/modules/llms/azure/azure.vendor.ts index bbb5532a5..9e6c41c13 100644 --- a/src/modules/llms/azure/azure.vendor.ts +++ b/src/modules/llms/azure/azure.vendor.ts @@ -4,7 +4,7 @@ import { AzureIcon } from '~/common/components/icons/AzureIcon'; import { DLLM } from '../store-llms'; import { IModelVendor } from '../vendors/IModelVendor'; -import { VChatFunctionIn, VChatMessageIn, VChatMessageOrFunctionCallOut, VChatMessageOut } from '../llm.client'; +import { VChatFunctionIn, VChatMessageIn, VChatMessageOrFunctionCallOut, VChatMessageOut } from '../transports/chatGenerate'; import { LLMOptionsOpenAI } from '../openai/openai.vendor'; import { OpenAILLMOptions } from '../openai/OpenAILLMOptions'; diff --git a/src/modules/llms/openai/openai.vendor.ts b/src/modules/llms/openai/openai.vendor.ts index 84662b0a7..b92202afc 100644 --- a/src/modules/llms/openai/openai.vendor.ts +++ b/src/modules/llms/openai/openai.vendor.ts @@ -4,7 +4,7 @@ import { OpenAIIcon } from '~/common/components/icons/OpenAIIcon'; import { DLLM } from '../store-llms'; import { IModelVendor } from '../vendors/IModelVendor'; -import { VChatFunctionIn, VChatMessageIn, VChatMessageOrFunctionCallOut, VChatMessageOut } from '../llm.client'; +import { VChatFunctionIn, VChatMessageIn, VChatMessageOrFunctionCallOut, VChatMessageOut } from '../transports/chatGenerate'; import { OpenAILLMOptions } from './OpenAILLMOptions'; import { OpenAISourceSetup } from './OpenAISourceSetup'; diff --git a/src/modules/llms/transports/chatGenerate.ts b/src/modules/llms/transports/chatGenerate.ts new file mode 100644 index 000000000..3dbffa8e6 --- /dev/null +++ b/src/modules/llms/transports/chatGenerate.ts @@ -0,0 +1,35 @@ +import type { OpenAI } from './server/openai.wiretypes'; +import { DLLMId } from '../store-llms'; + +import { findVendorForLlmOrThrow } from '~/modules/llms/vendors/vendor.registry'; + + +export interface VChatMessageIn { + role: 'assistant' | 'system' | 'user'; // | 'function'; + content: string; + //name?: string; // when role: 'function' +} + +export type VChatFunctionIn = OpenAI.Wire.ChatCompletion.RequestFunctionDef; + +export interface VChatMessageOut { + role: 'assistant' | 'system' | 'user'; + content: string; + finish_reason: 'stop' | 'length' | null; +} + +export interface VChatMessageOrFunctionCallOut extends VChatMessageOut { + function_name: string; + function_arguments: object | null; +} + + +export async function callChatGenerate(llmId: DLLMId, messages: VChatMessageIn[], maxTokens?: number): Promise { + const { llm, vendor } = findVendorForLlmOrThrow(llmId); + return await vendor.callChat(llm, messages, maxTokens); +} + +export async function callChatGenerateWithFunctions(llmId: DLLMId, messages: VChatMessageIn[], functions: VChatFunctionIn[], forceFunctionName?: string, maxTokens?: number): Promise { + const { llm, vendor } = findVendorForLlmOrThrow(llmId); + return await vendor.callChatWithFunctions(llm, messages, functions, forceFunctionName, maxTokens); +} \ No newline at end of file diff --git a/src/modules/llms/llm.client.ts b/src/modules/llms/transports/streamChat.ts similarity index 59% rename from src/modules/llms/llm.client.ts rename to src/modules/llms/transports/streamChat.ts index ed0b3ef2c..f9737fbb2 100644 --- a/src/modules/llms/llm.client.ts +++ b/src/modules/llms/transports/streamChat.ts @@ -1,79 +1,45 @@ import { apiAsync } from '~/modules/trpc/trpc.client'; -import type { DMessage } from '~/common/state/store-chats'; +import type { DLLM, DLLMId } from '../store-llms'; +import type { IModelVendor } from '../vendors/IModelVendor'; +import { findVendorForLlmOrThrow } from '../vendors/vendor.registry'; -import type { ChatStreamSchema } from './transports/server/openai.router'; -import type { OpenAI } from './transports/server/openai.wiretypes'; -import { ModelVendorAnthropic, SourceSetupAnthropic } from './anthropic/anthropic.vendor'; -import { ModelVendorOpenAI, SourceSetupOpenAI } from './openai/openai.vendor'; +import type { ChatStreamSchema } from './server/openai.router'; +import type { OpenAI } from './server/openai.wiretypes'; +import type { VChatMessageIn } from './chatGenerate'; -import { IModelVendor } from './vendors/IModelVendor'; -import { findVendorById } from './vendors/vendor.registry'; - -import { DLLM, DLLMId, findLLMOrThrow } from './store-llms'; - - -export interface VChatMessageIn { - role: 'assistant' | 'system' | 'user'; // | 'function'; - content: string; - //name?: string; // when role: 'function' -} - -export type VChatFunctionIn = OpenAI.Wire.ChatCompletion.RequestFunctionDef; - -export interface VChatMessageOut { - role: 'assistant' | 'system' | 'user'; - content: string; - finish_reason: 'stop' | 'length' | null; -} - -export interface VChatMessageOrFunctionCallOut extends VChatMessageOut { - function_name: string; - function_arguments: object | null; -} - - -export async function streamChat(llmId: DLLMId, messages: VChatMessageIn[], abortSignal: AbortSignal, editMessage: (updatedMessage: Partial, done: boolean) => void): Promise { - const { llm, vendor } = getLLMAndVendorOrThrow(llmId); - return await vendorStreamChat(vendor, llm, messages, abortSignal, editMessage); -} - -export async function callChatGenerate(llmId: DLLMId, messages: VChatMessageIn[], maxTokens?: number): Promise { - const { llm, vendor } = getLLMAndVendorOrThrow(llmId); - return await vendor.callChat(llm, messages, maxTokens); -} - -export async function callChatGenerateWithFunctions(llmId: DLLMId, messages: VChatMessageIn[], functions: VChatFunctionIn[], forceFunctionName?: string, maxTokens?: number): Promise { - const { llm, vendor } = getLLMAndVendorOrThrow(llmId); - return await vendor.callChatWithFunctions(llm, messages, functions, forceFunctionName, maxTokens); -} - - -function getLLMAndVendorOrThrow(llmId: DLLMId) { - const llm = findLLMOrThrow(llmId); - const vendor = findVendorById(llm?._source.vId); - if (!vendor) throw new Error(`callChat: Vendor not found for LLM ${llmId}`); - return { llm, vendor }; -} +import { ModelVendorAnthropic, SourceSetupAnthropic } from '../anthropic/anthropic.vendor'; +import { ModelVendorOpenAI, SourceSetupOpenAI } from '../openai/openai.vendor'; /** * Chat streaming function on the client side. This decodes the (text) streaming response - * from the /api/llms/stream endpoint, and signals updates. + * from the /api/llms/stream endpoint, and signals updates via our callback. * * Vendor-specific implementation is on the backend (API) code. This function tries to be * as generic as possible. * - * @param vendor vendor, mostly for vendor-specific backend - * @param llm the LLM model - * @param messages the history of messages to send to the API endpoint + * @param llmId LLM to use + * @param chatHistory the history of messages to send to the API endpoint * @param abortSignal used to initiate a client-side abort of the fetch request to the API endpoint - * @param updateMessage callback when a piece of a message (text, model name, typing..) is received + * @param onUpdate callback when a piece of a message (text, model name, typing..) is received */ +export async function streamChat( + llmId: DLLMId, + chatHistory: VChatMessageIn[], + abortSignal: AbortSignal, + onUpdate: (update: Partial<{ text: string, typing: boolean, originLLM: string }>, done: boolean) => void, +): Promise { + const { llm, vendor } = findVendorForLlmOrThrow(llmId); + return await vendorStreamChat(vendor, llm, chatHistory, abortSignal, onUpdate); +} + + async function vendorStreamChat( - vendor: IModelVendor, llm: DLLM, messages: VChatMessageIn[], + vendor: IModelVendor, llm: DLLM, + chatHistory: VChatMessageIn[], abortSignal: AbortSignal, - updateMessage: (updatedMessage: Partial, done: boolean) => void, + onUpdate: (update: Partial<{ text: string, typing: boolean, originLLM: string }>, done: boolean) => void, ) { // access params (source) @@ -82,7 +48,7 @@ async function vendorStreamChat( // [OpenAI-only] check for harmful content with the free 'moderation' API if (vendor.id === 'openai') { const openAISourceSetup = sourceSetup as SourceSetupOpenAI; - const lastMessage = messages.at(-1) ?? null; + const lastMessage = chatHistory.at(-1) ?? null; const useModeration = openAISourceSetup.moderationCheck && lastMessage && lastMessage.role === 'user'; if (useModeration) { try { @@ -104,14 +70,14 @@ async function vendorStreamChat( if (issues.size) { const categoriesText = [...issues].map(c => `\`${c}\``).join(', '); // do not proceed with the streaming request - return updateMessage({ + return onUpdate({ text: `[Moderation] I an unable to provide a response to your query as it violated the following categories of the OpenAI usage policies: ${categoriesText}.\nFor further explanation please visit https://platform.openai.com/docs/guides/moderation/moderation`, typing: false, }, true); } } catch (error: any) { // as the moderation check was requested, we cannot proceed in case of error - return updateMessage({ + return onUpdate({ text: `[Issue] There was an error while checking for harmful content. ${error?.toString()}`, typing: false, }, true); @@ -140,14 +106,14 @@ async function vendorStreamChat( maxTokens: llmResponseTokens, }, functions: undefined, - history: messages, + history: chatHistory, } satisfies ChatStreamSchema), signal: abortSignal, }); if (!response.ok || !response.body) { const errorMessage = response.body ? await response.text() : 'No response from server'; - return updateMessage({ text: errorMessage, typing: false }, true); + return onUpdate({ text: errorMessage, typing: false }, true); } const responseReader = response.body.getReader(); @@ -175,7 +141,7 @@ async function vendorStreamChat( parsedFirstPacket = true; try { const parsed: OpenAI.API.Chat.StreamingFirstResponse = JSON.parse(json); - updateMessage({ originLLM: parsed.model }, false); + onUpdate({ originLLM: parsed.model }, false); } catch (e) { // error parsing JSON, ignore console.log('vendorStreamChat: error parsing JSON:', e); @@ -183,6 +149,6 @@ async function vendorStreamChat( } if (incrementalText) - updateMessage({ text: incrementalText }, false); + onUpdate({ text: incrementalText }, false); } -} +} \ No newline at end of file diff --git a/src/modules/llms/vendors/IModelVendor.ts b/src/modules/llms/vendors/IModelVendor.ts index 6debccd09..dfef060e7 100644 --- a/src/modules/llms/vendors/IModelVendor.ts +++ b/src/modules/llms/vendors/IModelVendor.ts @@ -1,7 +1,7 @@ import type React from 'react'; import type { DLLM, DModelSourceId } from '../store-llms'; -import type { VChatFunctionIn, VChatMessageIn, VChatMessageOrFunctionCallOut, VChatMessageOut } from '../llm.client'; +import { VChatFunctionIn, VChatMessageIn, VChatMessageOrFunctionCallOut, VChatMessageOut } from '../transports/chatGenerate'; export type ModelVendorId = 'anthropic' | 'azure' | 'localai' | 'oobabooga' | 'openai' | 'openrouter'; diff --git a/src/modules/llms/vendors/vendor.registry.ts b/src/modules/llms/vendors/vendor.registry.ts index 23b1eeab3..9644f146f 100644 --- a/src/modules/llms/vendors/vendor.registry.ts +++ b/src/modules/llms/vendors/vendor.registry.ts @@ -5,7 +5,7 @@ import { ModelVendorOoobabooga } from '../oobabooga/oobabooga.vendor'; import { ModelVendorOpenAI } from '../openai/openai.vendor'; import { ModelVendorOpenRouter } from '../openrouter/openrouter.vendor'; -import { DModelSource, DModelSourceId } from '../store-llms'; +import { DLLMId, DModelSource, DModelSourceId, findLLMOrThrow } from '../store-llms'; import { IModelVendor, ModelVendorId } from './IModelVendor'; @@ -19,6 +19,13 @@ export function findVendorById(vendorId?: ModelVendorId): IModelVendor | null { return vendorId ? (MODEL_VENDOR_REGISTRY[vendorId] ?? null) : null; } +export function findVendorForLlmOrThrow(llmId: DLLMId) { + const llm = findLLMOrThrow(llmId); + const vendor = findVendorById(llm?._source.vId); + if (!vendor) throw new Error(`callChat: Vendor not found for LLM ${llmId}`); + return { llm, vendor }; +} + export function createModelSourceForVendor(vendorId: ModelVendorId, otherSources: DModelSource[]): DModelSource { // get vendor const vendor = findVendorById(vendorId); From adaff912253c82b01d6b9a3ddda54cb0e1223d0d Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Fri, 22 Sep 2023 22:03:21 -0700 Subject: [PATCH 13/17] Llms: removed and spread out llm.routes --- .../llms/anthropic/AnthropicSourceSetup.tsx | 2 +- src/modules/llms/azure/AzureSourceSetup.tsx | 2 +- src/modules/llms/llm.router.ts | 48 ------------------- src/modules/llms/openai/OpenAISourceSetup.tsx | 21 ++++++++ src/modules/llms/openai/openai.data.ts | 3 +- src/modules/llms/store-llms.ts | 17 +++++-- .../transports/server/anthropic.router.ts | 4 +- .../llms/transports/server/azure.router.ts | 2 +- .../llms/transports/server/server.common.ts | 20 ++++++++ 9 files changed, 61 insertions(+), 58 deletions(-) delete mode 100644 src/modules/llms/llm.router.ts create mode 100644 src/modules/llms/transports/server/server.common.ts diff --git a/src/modules/llms/anthropic/AnthropicSourceSetup.tsx b/src/modules/llms/anthropic/AnthropicSourceSetup.tsx index 3f80aa53e..b8aa76872 100644 --- a/src/modules/llms/anthropic/AnthropicSourceSetup.tsx +++ b/src/modules/llms/anthropic/AnthropicSourceSetup.tsx @@ -11,7 +11,7 @@ import { Link } from '~/common/components/Link'; import { settingsGap } from '~/common/theme'; import { DModelSourceId, useModelsStore, useSourceSetup } from '../store-llms'; -import { modelDescriptionToDLLM } from '../llm.router'; +import { modelDescriptionToDLLM } from '../openai/OpenAISourceSetup'; import { hasServerKeyAnthropic, isValidAnthropicApiKey, ModelVendorAnthropic } from './anthropic.vendor'; diff --git a/src/modules/llms/azure/AzureSourceSetup.tsx b/src/modules/llms/azure/AzureSourceSetup.tsx index f37e0365d..9330deff3 100644 --- a/src/modules/llms/azure/AzureSourceSetup.tsx +++ b/src/modules/llms/azure/AzureSourceSetup.tsx @@ -11,7 +11,7 @@ import { asValidURL } from '~/common/util/urlUtils'; import { settingsGap } from '~/common/theme'; import { DModelSourceId, useModelsStore, useSourceSetup } from '../store-llms'; -import { modelDescriptionToDLLM } from '../llm.router'; +import { modelDescriptionToDLLM } from '../openai/OpenAISourceSetup'; import { hasServerKeyAzure, isValidAzureApiKey, ModelVendorAzure } from './azure.vendor'; diff --git a/src/modules/llms/llm.router.ts b/src/modules/llms/llm.router.ts deleted file mode 100644 index 57cef5c77..000000000 --- a/src/modules/llms/llm.router.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { z } from 'zod'; - -import { DLLM, DModelSource } from './store-llms'; -import { LLMOptionsOpenAI } from './openai/openai.vendor'; - -// these are constants used for model interfaces (chat, and function calls) -// they're here as a preview - will be used more broadly in the future -export const LLM_IF_OAI_Chat = 'oai-chat'; -export const LLM_IF_OAI_Fn = 'oai-fn'; -export const LLM_IF_OAI_Complete = 'oai-complete'; - - -const modelDescriptionSchema = z.object({ - id: z.string(), - label: z.string(), - created: z.number().optional(), - updated: z.number().optional(), - description: z.string(), - contextWindow: z.number(), - interfaces: z.array(z.enum([LLM_IF_OAI_Chat, LLM_IF_OAI_Fn, LLM_IF_OAI_Complete])), - hidden: z.boolean().optional(), -}); - -export type ModelDescriptionSchema = z.infer; - -export const listModelsOutputSchema = z.object({ - models: z.array(modelDescriptionSchema), -}); - -export function modelDescriptionToDLLM(model: ModelDescriptionSchema, source: DModelSource): DLLM { - return { - id: `${source.id}-${model.id}`, - label: model.label, - created: model.created || 0, - updated: model.updated || 0, - description: model.description, - tags: [], // ['stream', 'chat'], - contextTokens: model.contextWindow, - hidden: !!model.hidden, - sId: source.id, - _source: source, - options: { - llmRef: model.id, - llmTemperature: 0.5, - llmResponseTokens: Math.round(model.contextWindow / 8), - }, - }; -} \ No newline at end of file diff --git a/src/modules/llms/openai/OpenAISourceSetup.tsx b/src/modules/llms/openai/OpenAISourceSetup.tsx index 56d794d58..04c9047c8 100644 --- a/src/modules/llms/openai/OpenAISourceSetup.tsx +++ b/src/modules/llms/openai/OpenAISourceSetup.tsx @@ -11,6 +11,7 @@ import { InlineError } from '~/common/components/InlineError'; import { Link } from '~/common/components/Link'; import { settingsCol1Width, settingsGap } from '~/common/theme'; +import type { ModelDescriptionSchema } from '../transports/server/server.common'; import { DLLM, DModelSource, DModelSourceId, useModelsStore, useSourceSetup } from '../store-llms'; import { hasServerKeyOpenAI, isValidOpenAIApiKey, LLMOptionsOpenAI, ModelVendorOpenAI } from './openai.vendor'; import { openAIModelToModelDescription } from './openai.data'; @@ -174,4 +175,24 @@ function openAIModelToDLLM(model: { id: string, created: number }, source: DMode llmResponseTokens: Math.round(contextTokens / 8), }, }; +} + +export function modelDescriptionToDLLM(model: ModelDescriptionSchema, source: DModelSource): DLLM { + return { + id: `${source.id}-${model.id}`, + label: model.label, + created: model.created || 0, + updated: model.updated || 0, + description: model.description, + tags: [], // ['stream', 'chat'], + contextTokens: model.contextWindow, + hidden: !!model.hidden, + sId: source.id, + _source: source, + options: { + llmRef: model.id, + llmTemperature: 0.5, + llmResponseTokens: Math.round(model.contextWindow / 8), + }, + }; } \ No newline at end of file diff --git a/src/modules/llms/openai/openai.data.ts b/src/modules/llms/openai/openai.data.ts index d924e3401..0d7ddb2dc 100644 --- a/src/modules/llms/openai/openai.data.ts +++ b/src/modules/llms/openai/openai.data.ts @@ -1,4 +1,5 @@ -import { LLM_IF_OAI_Chat, LLM_IF_OAI_Complete, LLM_IF_OAI_Fn, ModelDescriptionSchema } from '~/modules/llms/llm.router'; +import type { ModelDescriptionSchema } from '../transports/server/server.common'; +import { LLM_IF_OAI_Chat, LLM_IF_OAI_Complete, LLM_IF_OAI_Fn } from '../store-llms'; const knownOpenAIChatModels: ({ idPrefix: string } & Omit)[] = [ // GPT4-32k's diff --git a/src/modules/llms/store-llms.ts b/src/modules/llms/store-llms.ts index 30081ba7e..eacd217f9 100644 --- a/src/modules/llms/store-llms.ts +++ b/src/modules/llms/store-llms.ts @@ -4,15 +4,11 @@ import { persist } from 'zustand/middleware'; import { ModelVendorId } from './vendors/IModelVendor'; -export type DLLMId = string; -export type DModelSourceId = string; - /** * Large Language Model - description and configuration (data object, stored) */ export interface DLLM { - id: DLLMId; label: string; created: number | 0; @@ -30,6 +26,14 @@ export interface DLLM { options: Partial<{ llmRef: string } & TLLMOptions>; } +export type DLLMId = string; + +// Model interfaces (chat, and function calls) - here as a preview, will be used more broadly in the future +export const LLM_IF_OAI_Chat = 'oai-chat'; +export const LLM_IF_OAI_Fn = 'oai-fn'; +export const LLM_IF_OAI_Complete = 'oai-complete'; + + /** * Model Server - configured to be a unique origin of models (data object, stored) */ @@ -44,8 +48,11 @@ export interface DModelSource { setup: Partial; } +export type DModelSourceId = string; + + +/// ModelsStore - a store for configured LLMs and configured Sources -/// ModelsStore - a store for LLMs and their origins interface ModelsData { llms: DLLM[]; sources: DModelSource[]; diff --git a/src/modules/llms/transports/server/anthropic.router.ts b/src/modules/llms/transports/server/anthropic.router.ts index 06993975e..e4d788eea 100644 --- a/src/modules/llms/transports/server/anthropic.router.ts +++ b/src/modules/llms/transports/server/anthropic.router.ts @@ -4,8 +4,10 @@ import { TRPCError } from '@trpc/server'; import { createTRPCRouter, publicProcedure } from '~/modules/trpc/trpc.server'; import { fetchJsonOrTRPCError } from '~/modules/trpc/trpc.serverutils'; +import { LLM_IF_OAI_Chat } from '../../store-llms'; + import { chatGenerateOutputSchema, historySchema, modelSchema } from './openai.router'; -import { listModelsOutputSchema, LLM_IF_OAI_Chat, ModelDescriptionSchema } from '../../llm.router'; +import { listModelsOutputSchema, ModelDescriptionSchema } from './server.common'; import { AnthropicWire } from './anthropic.wiretypes'; diff --git a/src/modules/llms/transports/server/azure.router.ts b/src/modules/llms/transports/server/azure.router.ts index ff313b258..c6126683e 100644 --- a/src/modules/llms/transports/server/azure.router.ts +++ b/src/modules/llms/transports/server/azure.router.ts @@ -6,7 +6,7 @@ import { fetchJsonOrTRPCError } from '~/modules/trpc/trpc.serverutils'; import { OpenAI } from './openai.wiretypes'; import { chatGenerateOutputSchema, historySchema, modelSchema, openAIChatCompletionPayload } from './openai.router'; -import { listModelsOutputSchema, ModelDescriptionSchema } from '../../llm.router'; +import { listModelsOutputSchema, ModelDescriptionSchema } from './server.common'; import { openAIModelToModelDescription } from '../../openai/openai.data'; diff --git a/src/modules/llms/transports/server/server.common.ts b/src/modules/llms/transports/server/server.common.ts new file mode 100644 index 000000000..95eb65765 --- /dev/null +++ b/src/modules/llms/transports/server/server.common.ts @@ -0,0 +1,20 @@ +import { z } from 'zod'; +import { LLM_IF_OAI_Chat, LLM_IF_OAI_Complete, LLM_IF_OAI_Fn } from '~/modules/llms/store-llms'; + + +const modelDescriptionSchema = z.object({ + id: z.string(), + label: z.string(), + created: z.number().optional(), + updated: z.number().optional(), + description: z.string(), + contextWindow: z.number(), + interfaces: z.array(z.enum([LLM_IF_OAI_Chat, LLM_IF_OAI_Fn, LLM_IF_OAI_Complete])), + hidden: z.boolean().optional(), +}); + +export type ModelDescriptionSchema = z.infer; + +export const listModelsOutputSchema = z.object({ + models: z.array(modelDescriptionSchema), +}); From 617f7676ce887f7a79e6eb13868d7bac1804bc97 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Fri, 22 Sep 2023 22:20:22 -0700 Subject: [PATCH 14/17] Llms: moved (client) vendors inside ../vendors --- .../chat/components/composer/Composer.tsx | 2 +- .../models-modal/ModelsSourceSelector.tsx | 2 +- .../llms/transports/server/azure.router.ts | 3 +- src/modules/llms/transports/streamChat.ts | 4 +- .../anthropic/AnthropicSourceSetup.tsx | 2 +- .../anthropic/anthropic.vendor.ts | 6 +-- .../{ => vendors}/azure/AzureSourceSetup.tsx | 2 +- .../llms/{ => vendors}/azure/azure.vendor.ts | 6 +-- .../localai/LocalAISourceSetup.tsx | 2 +- .../{ => vendors}/localai/localai.vendor.tsx | 2 +- .../oobabooga/OobaboogaSourceSetup.tsx | 4 +- .../oobabooga/oobabooga.vendor.ts | 2 +- .../{ => vendors}/openai/OpenAILLMOptions.tsx | 2 +- .../openai/OpenAISourceSetup.tsx | 4 +- .../llms/{ => vendors}/openai/openai.data.ts | 4 +- .../{ => vendors}/openai/openai.vendor.ts | 6 +-- .../openrouter/OpenRouterSourceSetup.tsx | 4 +- .../openrouter/openrouter.vendor.tsx | 2 +- src/modules/llms/vendors/vendor.registry.ts | 40 +++++++++---------- 19 files changed, 49 insertions(+), 50 deletions(-) rename src/modules/llms/{ => vendors}/anthropic/AnthropicSourceSetup.tsx (99%) rename src/modules/llms/{ => vendors}/anthropic/anthropic.vendor.ts (92%) rename src/modules/llms/{ => vendors}/azure/AzureSourceSetup.tsx (99%) rename src/modules/llms/{ => vendors}/azure/azure.vendor.ts (95%) rename src/modules/llms/{ => vendors}/localai/LocalAISourceSetup.tsx (98%) rename src/modules/llms/{ => vendors}/localai/localai.vendor.tsx (94%) rename src/modules/llms/{ => vendors}/oobabooga/OobaboogaSourceSetup.tsx (97%) rename src/modules/llms/{ => vendors}/oobabooga/oobabooga.vendor.ts (94%) rename src/modules/llms/{ => vendors}/openai/OpenAILLMOptions.tsx (97%) rename src/modules/llms/{ => vendors}/openai/OpenAISourceSetup.tsx (98%) rename src/modules/llms/{ => vendors}/openai/openai.data.ts (97%) rename src/modules/llms/{ => vendors}/openai/openai.vendor.ts (95%) rename src/modules/llms/{ => vendors}/openrouter/OpenRouterSourceSetup.tsx (98%) rename src/modules/llms/{ => vendors}/openrouter/openrouter.vendor.tsx (96%) diff --git a/src/apps/chat/components/composer/Composer.tsx b/src/apps/chat/components/composer/Composer.tsx index f1087cbb2..c46b16ea7 100644 --- a/src/apps/chat/components/composer/Composer.tsx +++ b/src/apps/chat/components/composer/Composer.tsx @@ -19,7 +19,7 @@ import StopOutlinedIcon from '@mui/icons-material/StopOutlined'; import TelegramIcon from '@mui/icons-material/Telegram'; import { ContentReducer } from '~/modules/aifn/summarize/ContentReducer'; -import { LLMOptionsOpenAI } from '~/modules/llms/openai/openai.vendor'; +import { LLMOptionsOpenAI } from '~/modules/llms/vendors/openai/openai.vendor'; import { useChatLLM } from '~/modules/llms/store-llms'; import { CloseableMenu } from '~/common/components/CloseableMenu'; diff --git a/src/apps/models-modal/ModelsSourceSelector.tsx b/src/apps/models-modal/ModelsSourceSelector.tsx index df7af90f5..f7f436adb 100644 --- a/src/apps/models-modal/ModelsSourceSelector.tsx +++ b/src/apps/models-modal/ModelsSourceSelector.tsx @@ -11,7 +11,7 @@ import DeleteOutlineIcon from '@mui/icons-material/DeleteOutline'; import { DModelSourceId, useModelsStore } from '~/modules/llms/store-llms'; import { IModelVendor, ModelVendorId } from '~/modules/llms/vendors/IModelVendor'; import { createModelSourceForVendor, findAllVendors, findVendorById } from '~/modules/llms/vendors/vendor.registry'; -import { hasServerKeyOpenAI } from '~/modules/llms/openai/openai.vendor'; +import { hasServerKeyOpenAI } from '~/modules/llms/vendors/openai/openai.vendor'; import { CloseableMenu } from '~/common/components/CloseableMenu'; import { ConfirmationModal } from '~/common/components/ConfirmationModal'; diff --git a/src/modules/llms/transports/server/azure.router.ts b/src/modules/llms/transports/server/azure.router.ts index c6126683e..af2e12646 100644 --- a/src/modules/llms/transports/server/azure.router.ts +++ b/src/modules/llms/transports/server/azure.router.ts @@ -4,10 +4,11 @@ import { TRPCError } from '@trpc/server'; import { createTRPCRouter, publicProcedure } from '~/modules/trpc/trpc.server'; import { fetchJsonOrTRPCError } from '~/modules/trpc/trpc.serverutils'; +import { openAIModelToModelDescription } from '../../vendors/openai/openai.data'; + import { OpenAI } from './openai.wiretypes'; import { chatGenerateOutputSchema, historySchema, modelSchema, openAIChatCompletionPayload } from './openai.router'; import { listModelsOutputSchema, ModelDescriptionSchema } from './server.common'; -import { openAIModelToModelDescription } from '../../openai/openai.data'; // input schemas diff --git a/src/modules/llms/transports/streamChat.ts b/src/modules/llms/transports/streamChat.ts index f9737fbb2..ed598cc49 100644 --- a/src/modules/llms/transports/streamChat.ts +++ b/src/modules/llms/transports/streamChat.ts @@ -8,8 +8,8 @@ import type { ChatStreamSchema } from './server/openai.router'; import type { OpenAI } from './server/openai.wiretypes'; import type { VChatMessageIn } from './chatGenerate'; -import { ModelVendorAnthropic, SourceSetupAnthropic } from '../anthropic/anthropic.vendor'; -import { ModelVendorOpenAI, SourceSetupOpenAI } from '../openai/openai.vendor'; +import { ModelVendorAnthropic, SourceSetupAnthropic } from '../vendors/anthropic/anthropic.vendor'; +import { ModelVendorOpenAI, SourceSetupOpenAI } from '../vendors/openai/openai.vendor'; /** diff --git a/src/modules/llms/anthropic/AnthropicSourceSetup.tsx b/src/modules/llms/vendors/anthropic/AnthropicSourceSetup.tsx similarity index 99% rename from src/modules/llms/anthropic/AnthropicSourceSetup.tsx rename to src/modules/llms/vendors/anthropic/AnthropicSourceSetup.tsx index b8aa76872..e2e361e60 100644 --- a/src/modules/llms/anthropic/AnthropicSourceSetup.tsx +++ b/src/modules/llms/vendors/anthropic/AnthropicSourceSetup.tsx @@ -10,7 +10,7 @@ import { InlineError } from '~/common/components/InlineError'; import { Link } from '~/common/components/Link'; import { settingsGap } from '~/common/theme'; -import { DModelSourceId, useModelsStore, useSourceSetup } from '../store-llms'; +import { DModelSourceId, useModelsStore, useSourceSetup } from '../../store-llms'; import { modelDescriptionToDLLM } from '../openai/OpenAISourceSetup'; import { hasServerKeyAnthropic, isValidAnthropicApiKey, ModelVendorAnthropic } from './anthropic.vendor'; diff --git a/src/modules/llms/anthropic/anthropic.vendor.ts b/src/modules/llms/vendors/anthropic/anthropic.vendor.ts similarity index 92% rename from src/modules/llms/anthropic/anthropic.vendor.ts rename to src/modules/llms/vendors/anthropic/anthropic.vendor.ts index 9889ab0df..547f4b836 100644 --- a/src/modules/llms/anthropic/anthropic.vendor.ts +++ b/src/modules/llms/vendors/anthropic/anthropic.vendor.ts @@ -2,9 +2,9 @@ import { apiAsync } from '~/modules/trpc/trpc.client'; import { AnthropicIcon } from '~/common/components/icons/AnthropicIcon'; -import { DLLM } from '../store-llms'; -import { IModelVendor } from '../vendors/IModelVendor'; -import { VChatMessageIn, VChatMessageOut } from '../transports/chatGenerate'; +import { DLLM } from '../../store-llms'; +import { IModelVendor } from '../IModelVendor'; +import { VChatMessageIn, VChatMessageOut } from '../../transports/chatGenerate'; import { LLMOptionsOpenAI } from '../openai/openai.vendor'; import { OpenAILLMOptions } from '../openai/OpenAILLMOptions'; diff --git a/src/modules/llms/azure/AzureSourceSetup.tsx b/src/modules/llms/vendors/azure/AzureSourceSetup.tsx similarity index 99% rename from src/modules/llms/azure/AzureSourceSetup.tsx rename to src/modules/llms/vendors/azure/AzureSourceSetup.tsx index 9330deff3..ae55a9fdb 100644 --- a/src/modules/llms/azure/AzureSourceSetup.tsx +++ b/src/modules/llms/vendors/azure/AzureSourceSetup.tsx @@ -10,7 +10,7 @@ import { Link } from '~/common/components/Link'; import { asValidURL } from '~/common/util/urlUtils'; import { settingsGap } from '~/common/theme'; -import { DModelSourceId, useModelsStore, useSourceSetup } from '../store-llms'; +import { DModelSourceId, useModelsStore, useSourceSetup } from '../../store-llms'; import { modelDescriptionToDLLM } from '../openai/OpenAISourceSetup'; import { hasServerKeyAzure, isValidAzureApiKey, ModelVendorAzure } from './azure.vendor'; diff --git a/src/modules/llms/azure/azure.vendor.ts b/src/modules/llms/vendors/azure/azure.vendor.ts similarity index 95% rename from src/modules/llms/azure/azure.vendor.ts rename to src/modules/llms/vendors/azure/azure.vendor.ts index 9e6c41c13..ba068f57a 100644 --- a/src/modules/llms/azure/azure.vendor.ts +++ b/src/modules/llms/vendors/azure/azure.vendor.ts @@ -2,9 +2,9 @@ import { apiAsync } from '~/modules/trpc/trpc.client'; import { AzureIcon } from '~/common/components/icons/AzureIcon'; -import { DLLM } from '../store-llms'; -import { IModelVendor } from '../vendors/IModelVendor'; -import { VChatFunctionIn, VChatMessageIn, VChatMessageOrFunctionCallOut, VChatMessageOut } from '../transports/chatGenerate'; +import { DLLM } from '../../store-llms'; +import { IModelVendor } from '../IModelVendor'; +import { VChatFunctionIn, VChatMessageIn, VChatMessageOrFunctionCallOut, VChatMessageOut } from '../../transports/chatGenerate'; import { LLMOptionsOpenAI } from '../openai/openai.vendor'; import { OpenAILLMOptions } from '../openai/OpenAILLMOptions'; diff --git a/src/modules/llms/localai/LocalAISourceSetup.tsx b/src/modules/llms/vendors/localai/LocalAISourceSetup.tsx similarity index 98% rename from src/modules/llms/localai/LocalAISourceSetup.tsx rename to src/modules/llms/vendors/localai/LocalAISourceSetup.tsx index 862dbdd24..b73306975 100644 --- a/src/modules/llms/localai/LocalAISourceSetup.tsx +++ b/src/modules/llms/vendors/localai/LocalAISourceSetup.tsx @@ -14,7 +14,7 @@ import { settingsGap } from '~/common/theme'; import { LLMOptionsOpenAI, ModelVendorOpenAI } from '../openai/openai.vendor'; -import { DLLM, DModelSource, DModelSourceId, useModelsStore, useSourceSetup } from '../store-llms'; +import { DLLM, DModelSource, DModelSourceId, useModelsStore, useSourceSetup } from '../../store-llms'; import { ModelVendorLocalAI } from './localai.vendor'; diff --git a/src/modules/llms/localai/localai.vendor.tsx b/src/modules/llms/vendors/localai/localai.vendor.tsx similarity index 94% rename from src/modules/llms/localai/localai.vendor.tsx rename to src/modules/llms/vendors/localai/localai.vendor.tsx index 6c1e90a39..71797564f 100644 --- a/src/modules/llms/localai/localai.vendor.tsx +++ b/src/modules/llms/vendors/localai/localai.vendor.tsx @@ -1,6 +1,6 @@ import DevicesIcon from '@mui/icons-material/Devices'; -import { IModelVendor } from '../vendors/IModelVendor'; +import { IModelVendor } from '../IModelVendor'; import { LLMOptionsOpenAI, ModelVendorOpenAI } from '../openai/openai.vendor'; import { OpenAILLMOptions } from '../openai/OpenAILLMOptions'; diff --git a/src/modules/llms/oobabooga/OobaboogaSourceSetup.tsx b/src/modules/llms/vendors/oobabooga/OobaboogaSourceSetup.tsx similarity index 97% rename from src/modules/llms/oobabooga/OobaboogaSourceSetup.tsx rename to src/modules/llms/vendors/oobabooga/OobaboogaSourceSetup.tsx index 67f3638c2..dc8aa6d23 100644 --- a/src/modules/llms/oobabooga/OobaboogaSourceSetup.tsx +++ b/src/modules/llms/vendors/oobabooga/OobaboogaSourceSetup.tsx @@ -9,9 +9,9 @@ import { InlineError } from '~/common/components/InlineError'; import { Link } from '~/common/components/Link'; import { settingsCol1Width, settingsGap } from '~/common/theme'; -import { LLMOptionsOpenAI, ModelVendorOpenAI } from '~/modules/llms/openai/openai.vendor'; +import { LLMOptionsOpenAI, ModelVendorOpenAI } from '../openai/openai.vendor'; -import { DLLM, DModelSource, DModelSourceId, useModelsStore, useSourceSetup } from '../store-llms'; +import { DLLM, DModelSource, DModelSourceId, useModelsStore, useSourceSetup } from '../../store-llms'; import { ModelVendorOoobabooga } from './oobabooga.vendor'; diff --git a/src/modules/llms/oobabooga/oobabooga.vendor.ts b/src/modules/llms/vendors/oobabooga/oobabooga.vendor.ts similarity index 94% rename from src/modules/llms/oobabooga/oobabooga.vendor.ts rename to src/modules/llms/vendors/oobabooga/oobabooga.vendor.ts index 7e1602e5a..b2ccc0c16 100644 --- a/src/modules/llms/oobabooga/oobabooga.vendor.ts +++ b/src/modules/llms/vendors/oobabooga/oobabooga.vendor.ts @@ -1,6 +1,6 @@ import { OobaboogaIcon } from '~/common/components/icons/OobaboogaIcon'; -import { IModelVendor } from '../vendors/IModelVendor'; +import { IModelVendor } from '../IModelVendor'; import { LLMOptionsOpenAI, ModelVendorOpenAI } from '../openai/openai.vendor'; import { OpenAILLMOptions } from '../openai/OpenAILLMOptions'; diff --git a/src/modules/llms/openai/OpenAILLMOptions.tsx b/src/modules/llms/vendors/openai/OpenAILLMOptions.tsx similarity index 97% rename from src/modules/llms/openai/OpenAILLMOptions.tsx rename to src/modules/llms/vendors/openai/OpenAILLMOptions.tsx index 1f4a3c010..535d1dfa8 100644 --- a/src/modules/llms/openai/OpenAILLMOptions.tsx +++ b/src/modules/llms/vendors/openai/OpenAILLMOptions.tsx @@ -3,7 +3,7 @@ import * as React from 'react'; import { Box, FormControl, FormHelperText, FormLabel, Slider } from '@mui/joy'; import { settingsCol1Width, settingsGap } from '~/common/theme'; -import { DLLM, useModelsStore } from '../store-llms'; +import { DLLM, useModelsStore } from '../../store-llms'; import { LLMOptionsOpenAI } from './openai.vendor'; function normalizeOpenAIOptions(partialOptions?: Partial) { diff --git a/src/modules/llms/openai/OpenAISourceSetup.tsx b/src/modules/llms/vendors/openai/OpenAISourceSetup.tsx similarity index 98% rename from src/modules/llms/openai/OpenAISourceSetup.tsx rename to src/modules/llms/vendors/openai/OpenAISourceSetup.tsx index 04c9047c8..222ec6193 100644 --- a/src/modules/llms/openai/OpenAISourceSetup.tsx +++ b/src/modules/llms/vendors/openai/OpenAISourceSetup.tsx @@ -11,8 +11,8 @@ import { InlineError } from '~/common/components/InlineError'; import { Link } from '~/common/components/Link'; import { settingsCol1Width, settingsGap } from '~/common/theme'; -import type { ModelDescriptionSchema } from '../transports/server/server.common'; -import { DLLM, DModelSource, DModelSourceId, useModelsStore, useSourceSetup } from '../store-llms'; +import type { ModelDescriptionSchema } from '../../transports/server/server.common'; +import { DLLM, DModelSource, DModelSourceId, useModelsStore, useSourceSetup } from '../../store-llms'; import { hasServerKeyOpenAI, isValidOpenAIApiKey, LLMOptionsOpenAI, ModelVendorOpenAI } from './openai.vendor'; import { openAIModelToModelDescription } from './openai.data'; diff --git a/src/modules/llms/openai/openai.data.ts b/src/modules/llms/vendors/openai/openai.data.ts similarity index 97% rename from src/modules/llms/openai/openai.data.ts rename to src/modules/llms/vendors/openai/openai.data.ts index 0d7ddb2dc..a98965016 100644 --- a/src/modules/llms/openai/openai.data.ts +++ b/src/modules/llms/vendors/openai/openai.data.ts @@ -1,5 +1,5 @@ -import type { ModelDescriptionSchema } from '../transports/server/server.common'; -import { LLM_IF_OAI_Chat, LLM_IF_OAI_Complete, LLM_IF_OAI_Fn } from '../store-llms'; +import type { ModelDescriptionSchema } from '../../transports/server/server.common'; +import { LLM_IF_OAI_Chat, LLM_IF_OAI_Complete, LLM_IF_OAI_Fn } from '../../store-llms'; const knownOpenAIChatModels: ({ idPrefix: string } & Omit)[] = [ // GPT4-32k's diff --git a/src/modules/llms/openai/openai.vendor.ts b/src/modules/llms/vendors/openai/openai.vendor.ts similarity index 95% rename from src/modules/llms/openai/openai.vendor.ts rename to src/modules/llms/vendors/openai/openai.vendor.ts index b92202afc..29b9b0c8e 100644 --- a/src/modules/llms/openai/openai.vendor.ts +++ b/src/modules/llms/vendors/openai/openai.vendor.ts @@ -2,9 +2,9 @@ import { apiAsync } from '~/modules/trpc/trpc.client'; import { OpenAIIcon } from '~/common/components/icons/OpenAIIcon'; -import { DLLM } from '../store-llms'; -import { IModelVendor } from '../vendors/IModelVendor'; -import { VChatFunctionIn, VChatMessageIn, VChatMessageOrFunctionCallOut, VChatMessageOut } from '../transports/chatGenerate'; +import { DLLM } from '../../store-llms'; +import { IModelVendor } from '../IModelVendor'; +import { VChatFunctionIn, VChatMessageIn, VChatMessageOrFunctionCallOut, VChatMessageOut } from '../../transports/chatGenerate'; import { OpenAILLMOptions } from './OpenAILLMOptions'; import { OpenAISourceSetup } from './OpenAISourceSetup'; diff --git a/src/modules/llms/openrouter/OpenRouterSourceSetup.tsx b/src/modules/llms/vendors/openrouter/OpenRouterSourceSetup.tsx similarity index 98% rename from src/modules/llms/openrouter/OpenRouterSourceSetup.tsx rename to src/modules/llms/vendors/openrouter/OpenRouterSourceSetup.tsx index 6b64eb653..cd11a302d 100644 --- a/src/modules/llms/openrouter/OpenRouterSourceSetup.tsx +++ b/src/modules/llms/vendors/openrouter/OpenRouterSourceSetup.tsx @@ -3,7 +3,7 @@ import * as React from 'react'; import { Box, Button, Typography } from '@mui/joy'; import SyncIcon from '@mui/icons-material/Sync'; -import { LLMOptionsOpenAI, ModelVendorOpenAI } from '~/modules/llms/openai/openai.vendor'; +import { LLMOptionsOpenAI, ModelVendorOpenAI } from '../openai/openai.vendor'; import { apiQuery } from '~/modules/trpc/trpc.client'; import { FormInputKey } from '~/common/components/FormInputKey'; @@ -11,7 +11,7 @@ import { InlineError } from '~/common/components/InlineError'; import { Link } from '~/common/components/Link'; import { settingsGap } from '~/common/theme'; -import { DLLM, DModelSource, DModelSourceId, useModelsStore, useSourceSetup } from '../store-llms'; +import { DLLM, DModelSource, DModelSourceId, useModelsStore, useSourceSetup } from '../../store-llms'; import { isValidOpenRouterKey, ModelVendorOpenRouter } from './openrouter.vendor'; diff --git a/src/modules/llms/openrouter/openrouter.vendor.tsx b/src/modules/llms/vendors/openrouter/openrouter.vendor.tsx similarity index 96% rename from src/modules/llms/openrouter/openrouter.vendor.tsx rename to src/modules/llms/vendors/openrouter/openrouter.vendor.tsx index b8d7d8a12..5cfed952c 100644 --- a/src/modules/llms/openrouter/openrouter.vendor.tsx +++ b/src/modules/llms/vendors/openrouter/openrouter.vendor.tsx @@ -1,6 +1,6 @@ import { OpenRouterIcon } from '~/common/components/icons/OpenRouterIcon'; -import { IModelVendor } from '../vendors/IModelVendor'; +import { IModelVendor } from '../IModelVendor'; import { LLMOptionsOpenAI, ModelVendorOpenAI } from '../openai/openai.vendor'; import { OpenAILLMOptions } from '../openai/OpenAILLMOptions'; diff --git a/src/modules/llms/vendors/vendor.registry.ts b/src/modules/llms/vendors/vendor.registry.ts index 9644f146f..b66cb8652 100644 --- a/src/modules/llms/vendors/vendor.registry.ts +++ b/src/modules/llms/vendors/vendor.registry.ts @@ -1,13 +1,25 @@ -import { ModelVendorAnthropic } from '../anthropic/anthropic.vendor'; -import { ModelVendorAzure } from '../azure/azure.vendor'; -import { ModelVendorLocalAI } from '../localai/localai.vendor'; -import { ModelVendorOoobabooga } from '../oobabooga/oobabooga.vendor'; -import { ModelVendorOpenAI } from '../openai/openai.vendor'; -import { ModelVendorOpenRouter } from '../openrouter/openrouter.vendor'; +import { ModelVendorAnthropic } from './anthropic/anthropic.vendor'; +import { ModelVendorAzure } from './azure/azure.vendor'; +import { ModelVendorLocalAI } from './localai/localai.vendor'; +import { ModelVendorOoobabooga } from './oobabooga/oobabooga.vendor'; +import { ModelVendorOpenAI } from './openai/openai.vendor'; +import { ModelVendorOpenRouter } from './openrouter/openrouter.vendor'; import { DLLMId, DModelSource, DModelSourceId, findLLMOrThrow } from '../store-llms'; import { IModelVendor, ModelVendorId } from './IModelVendor'; +/** Vendor Instances Registry **/ +const MODEL_VENDOR_REGISTRY: Record = { + anthropic: ModelVendorAnthropic, + azure: ModelVendorAzure, + localai: ModelVendorLocalAI, + oobabooga: ModelVendorOoobabooga, + openai: ModelVendorOpenAI, + openrouter: ModelVendorOpenRouter, +}; + +const MODEL_VENDOR_DEFAULT: ModelVendorId = 'openai'; + export function findAllVendors(): IModelVendor[] { const modelVendors = Object.values(MODEL_VENDOR_REGISTRY); @@ -50,18 +62,4 @@ export function createModelSourceForVendor(vendorId: ModelVendorId, otherSources export function createModelSourceForDefaultVendor(otherSources: DModelSource[]): DModelSource { return createModelSourceForVendor(MODEL_VENDOR_DEFAULT, otherSources); -} - - -/// Main Vendor Registry /// - -const MODEL_VENDOR_REGISTRY: Record = { - anthropic: ModelVendorAnthropic, - azure: ModelVendorAzure, - localai: ModelVendorLocalAI, - oobabooga: ModelVendorOoobabooga, - openai: ModelVendorOpenAI, - openrouter: ModelVendorOpenRouter, -}; - -const MODEL_VENDOR_DEFAULT: ModelVendorId = 'openai'; \ No newline at end of file +} \ No newline at end of file From 34c150924e450de4af833248a6dcf9daeb4999d8 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Fri, 22 Sep 2023 22:29:33 -0700 Subject: [PATCH 15/17] Llms: bits --- src/modules/llms/transports/chatGenerate.ts | 3 +-- src/modules/llms/transports/server/server.common.ts | 2 +- src/modules/llms/transports/streamChat.ts | 8 ++++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/modules/llms/transports/chatGenerate.ts b/src/modules/llms/transports/chatGenerate.ts index 3dbffa8e6..9cd5993a4 100644 --- a/src/modules/llms/transports/chatGenerate.ts +++ b/src/modules/llms/transports/chatGenerate.ts @@ -1,7 +1,6 @@ import type { OpenAI } from './server/openai.wiretypes'; import { DLLMId } from '../store-llms'; - -import { findVendorForLlmOrThrow } from '~/modules/llms/vendors/vendor.registry'; +import { findVendorForLlmOrThrow } from '../vendors/vendor.registry'; export interface VChatMessageIn { diff --git a/src/modules/llms/transports/server/server.common.ts b/src/modules/llms/transports/server/server.common.ts index 95eb65765..39e57fcbe 100644 --- a/src/modules/llms/transports/server/server.common.ts +++ b/src/modules/llms/transports/server/server.common.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import { LLM_IF_OAI_Chat, LLM_IF_OAI_Complete, LLM_IF_OAI_Fn } from '~/modules/llms/store-llms'; +import { LLM_IF_OAI_Chat, LLM_IF_OAI_Complete, LLM_IF_OAI_Fn } from '../../store-llms'; const modelDescriptionSchema = z.object({ diff --git a/src/modules/llms/transports/streamChat.ts b/src/modules/llms/transports/streamChat.ts index ed598cc49..0eca80dd4 100644 --- a/src/modules/llms/transports/streamChat.ts +++ b/src/modules/llms/transports/streamChat.ts @@ -13,10 +13,10 @@ import { ModelVendorOpenAI, SourceSetupOpenAI } from '../vendors/openai/openai.v /** - * Chat streaming function on the client side. This decodes the (text) streaming response - * from the /api/llms/stream endpoint, and signals updates via our callback. + * Client side chat generation, with streaming. This decodes the (text) streaming response from + * our server streaming endpoint (plain text, not EventSource), and signals updates via a callback. * - * Vendor-specific implementation is on the backend (API) code. This function tries to be + * Vendor-specific implementation is on our server backend (API) code. This function tries to be * as generic as possible. * * @param llmId LLM to use @@ -90,7 +90,7 @@ async function vendorStreamChat( if (!llmRef || llmTemperature === undefined || llmResponseTokens === undefined) throw new Error(`Error in openAI configuration for model ${llm.id}: ${llm.options}`); - // call /api/llms/stream + // connect to the server-side streaming endpoint const response = await fetch('/api/llms/stream', { method: 'POST', headers: { 'Content-Type': 'application/json' }, From 3448267344a7a22c49a49fbda4cbcea3450dc710 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Fri, 22 Sep 2023 22:47:40 -0700 Subject: [PATCH 16/17] Llms: downgrade tsx -> ts (not required) --- .../vendors/localai/{localai.vendor.tsx => localai.vendor.ts} | 0 .../openrouter/{openrouter.vendor.tsx => openrouter.vendor.ts} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/modules/llms/vendors/localai/{localai.vendor.tsx => localai.vendor.ts} (100%) rename src/modules/llms/vendors/openrouter/{openrouter.vendor.tsx => openrouter.vendor.ts} (100%) diff --git a/src/modules/llms/vendors/localai/localai.vendor.tsx b/src/modules/llms/vendors/localai/localai.vendor.ts similarity index 100% rename from src/modules/llms/vendors/localai/localai.vendor.tsx rename to src/modules/llms/vendors/localai/localai.vendor.ts diff --git a/src/modules/llms/vendors/openrouter/openrouter.vendor.tsx b/src/modules/llms/vendors/openrouter/openrouter.vendor.ts similarity index 100% rename from src/modules/llms/vendors/openrouter/openrouter.vendor.tsx rename to src/modules/llms/vendors/openrouter/openrouter.vendor.ts From 91353ced8a6bb8045a3cb35566f71b8886291487 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Fri, 22 Sep 2023 23:02:47 -0700 Subject: [PATCH 17/17] Azure: land in main, disable instancing as we finish it --- src/modules/llms/transports/server/azure.router.ts | 2 +- src/modules/llms/vendors/azure/azure.vendor.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/modules/llms/transports/server/azure.router.ts b/src/modules/llms/transports/server/azure.router.ts index af2e12646..fef692080 100644 --- a/src/modules/llms/transports/server/azure.router.ts +++ b/src/modules/llms/transports/server/azure.router.ts @@ -70,7 +70,7 @@ export const llmAzureRouter = createTRPCRouter({ const { id, label, ...rest } = openAIModelToModelDescription(model.model, model.created_at, model.updated_at); return { id: model.id, - label: `${model.id} (${label})`, + label: `${label} (${model.id})`, ...rest, }; }); diff --git a/src/modules/llms/vendors/azure/azure.vendor.ts b/src/modules/llms/vendors/azure/azure.vendor.ts index ba068f57a..823345e1a 100644 --- a/src/modules/llms/vendors/azure/azure.vendor.ts +++ b/src/modules/llms/vendors/azure/azure.vendor.ts @@ -31,7 +31,7 @@ export interface SourceSetupAzure { * "/openai/deployments?api-version=2023-03-15-preview" path on the endpoint. See: * https://github.com/openai/openai-python/issues/447#issuecomment-1730976835 * - * 2. Still looking for a solution - in the meantime the way to go seems to be to manyally get the full URL + * 2. Still looking for a solution - in the meantime the way to go seems to be to manually get the full URL * of every "Deployment" (Model) and hit the URL directly. However the user will need to fill in the full * model sheet, as details are not available just from the URL. * @@ -42,7 +42,7 @@ export const ModelVendorAzure: IModelVendor name: 'Azure', rank: 14, location: 'cloud', - instanceLimit: 2, + instanceLimit: 0, // components Icon: AzureIcon,