From a2cca141740989dcc3312cd2002b2c25e30a808b Mon Sep 17 00:00:00 2001 From: eduardruzga Date: Mon, 11 Nov 2024 23:47:12 +0200 Subject: [PATCH 1/9] Refactor/standartize model providers, add "get provider key" for those who have it for first time users --- app/components/chat/APIKeyManager.tsx | 6 +- app/components/chat/BaseChat.tsx | 21 +-- app/utils/constants.ts | 178 ++++++++++++++++++-------- 3 files changed, 134 insertions(+), 71 deletions(-) diff --git a/app/components/chat/APIKeyManager.tsx b/app/components/chat/APIKeyManager.tsx index a35724c8c..1343ece42 100644 --- a/app/components/chat/APIKeyManager.tsx +++ b/app/components/chat/APIKeyManager.tsx @@ -1,8 +1,9 @@ import React, { useState } from 'react'; import { IconButton } from '~/components/ui/IconButton'; +import type { ProviderInfo } from '~/utils/constants'; interface APIKeyManagerProps { - provider: string; + provider: ProviderInfo; apiKey: string; setApiKey: (key: string) => void; } @@ -18,7 +19,7 @@ export const APIKeyManager: React.FC = ({ provider, apiKey, return (
- {provider} API Key: + {provider?.name} API Key: {isEditing ? ( <> = ({ provider, apiKey, setIsEditing(true)} title="Edit API Key">
+ {!!provider?.getApiKeyLink ? {provider?.labelForGetApiKey || "Get API Key"} : "" } )}
diff --git a/app/components/chat/BaseChat.tsx b/app/components/chat/BaseChat.tsx index f33114739..134b4f8f7 100644 --- a/app/components/chat/BaseChat.tsx +++ b/app/components/chat/BaseChat.tsx @@ -7,7 +7,7 @@ import { Menu } from '~/components/sidebar/Menu.client'; import { IconButton } from '~/components/ui/IconButton'; import { Workbench } from '~/components/workbench/Workbench.client'; import { classNames } from '~/utils/classNames'; -import { MODEL_LIST, DEFAULT_PROVIDER } from '~/utils/constants'; +import { MODEL_LIST, DEFAULT_PROVIDER, PROVIDER_LIST } from '~/utils/constants'; import { Messages } from './Messages.client'; import { SendButton } from './SendButton.client'; import { useState } from 'react'; @@ -24,12 +24,12 @@ const EXAMPLE_PROMPTS = [ { text: 'How do I center a div?' }, ]; -const providerList = [...new Set(MODEL_LIST.map((model) => model.provider))] +const providerList = PROVIDER_LIST; const ModelSelector = ({ model, setModel, provider, setProvider, modelList, providerList }) => { return (
- { - setProvider(e.target.value); + setProvider(providerList.find(p => p.name === e.target.value)); const firstModel = [...modelList].find(m => m.provider == e.target.value); setModel(firstModel ? firstModel.name : ''); }} @@ -49,7 +49,7 @@ const ModelSelector = ({ model, setModel, provider, setProvider, modelList, prov onChange={(e) => setModel(e.target.value)} className="flex-1 p-2 rounded-lg border border-bolt-elements-borderColor bg-bolt-elements-prompt-background text-bolt-elements-textPrimary focus:outline-none focus:ring-2 focus:ring-bolt-elements-focus transition-all" > - {[...modelList].filter(e => e.provider == provider && e.name).map((modelOption) => ( + {[...modelList].filter(e => e.provider == provider?.name && e.name).map((modelOption) => ( @@ -74,8 +74,8 @@ interface BaseChatProps { input?: string; model: string; setModel: (model: string) => void; - provider: string; - setProvider: (provider: string) => void; + provider: ProviderInfo; + setProvider: (provider: ProviderInfo) => void; handleStop?: () => void; sendMessage?: (event: React.UIEvent, messageInput?: string) => void; handleInputChange?: (event: React.ChangeEvent) => void; @@ -106,6 +106,7 @@ export const BaseChat = React.forwardRef( }, ref, ) => { + console.log(provider); const TEXTAREA_MAX_HEIGHT = chatStarted ? 400 : 200; const [apiKeys, setApiKeys] = useState>({}); @@ -194,11 +195,12 @@ export const BaseChat = React.forwardRef( setProvider={setProvider} providerList={providerList} /> - updateApiKey(provider, key)} - /> + {provider && + updateApiKey(provider.name, key)} + />}
Promise, getApiKeyLink?: string, labelForGetApiKey?: string, + icon?:string, }; const PROVIDER_LIST: ProviderInfo[] = [ + { + name: 'Anthropic', + staticModels: [ + { name: 'claude-3-5-sonnet-latest', label: 'Claude 3.5 Sonnet (new)', provider: 'Anthropic' }, + { name: 'claude-3-5-sonnet-20240620', label: 'Claude 3.5 Sonnet (old)', provider: 'Anthropic' }, + { name: 'claude-3-5-haiku-latest', label: 'Claude 3.5 Haiku (new)', provider: 'Anthropic' }, + { name: 'claude-3-opus-latest', label: 'Claude 3 Opus', provider: 'Anthropic' }, + { name: 'claude-3-sonnet-20240229', label: 'Claude 3 Sonnet', provider: 'Anthropic' }, + { name: 'claude-3-haiku-20240307', label: 'Claude 3 Haiku', provider: 'Anthropic' } + ], + getApiKeyLink: "https://console.anthropic.com/settings/keys", + }, { name: 'Ollama', staticModels: [], - getDynamicModels: getOllamaModels + getDynamicModels: getOllamaModels, + getApiKeyLink: "https://ollama.com/download", + labelForGetApiKey: "Download Ollama", + icon: "i-ph:cloud-arrow-down", }, { name: 'OpenAILike', staticModels: [], @@ -62,17 +78,6 @@ const PROVIDER_LIST: ProviderInfo[] = [ { name: 'llama-3.2-1b-preview', label: 'Llama 3.2 1b (Groq)', provider: 'Groq' } ], getApiKeyLink: 'https://console.groq.com/keys' - }, { - name: 'Anthropic', - staticModels: [ - { name: 'claude-3-5-sonnet-latest', label: 'Claude 3.5 Sonnet (new)', provider: 'Anthropic' }, - { name: 'claude-3-5-sonnet-20240620', label: 'Claude 3.5 Sonnet (old)', provider: 'Anthropic' }, - { name: 'claude-3-5-haiku-latest', label: 'Claude 3.5 Haiku (new)', provider: 'Anthropic' }, - { name: 'claude-3-opus-latest', label: 'Claude 3 Opus', provider: 'Anthropic' }, - { name: 'claude-3-sonnet-20240229', label: 'Claude 3 Sonnet', provider: 'Anthropic' }, - { name: 'claude-3-haiku-20240307', label: 'Claude 3 Haiku', provider: 'Anthropic' } - ], - getApiKeyLink: "https://console.anthropic.com/settings/keys", }, { name: 'OpenAI', staticModels: [ @@ -114,10 +119,12 @@ const PROVIDER_LIST: ProviderInfo[] = [ staticModels: [], getDynamicModels: getLMStudioModels, getApiKeyLink: 'https://lmstudio.ai/', - labelForGetApiKey: 'Get LMStudio' + labelForGetApiKey: 'Get LMStudio', + icon: "i-ph:cloud-arrow-down", } ]; +export const DEFAULT_PROVIDER = PROVIDER_LIST[0]; const staticModels: ModelInfo[] = PROVIDER_LIST.map(p => p.staticModels).flat(); From c575ee316ba063e7594388567be4daeab2cd76dd Mon Sep 17 00:00:00 2001 From: eduardruzga Date: Wed, 13 Nov 2024 22:20:51 +0200 Subject: [PATCH 3/9] Use cookies instead of request body that is stale sometimes --- app/routes/api.chat.ts | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/app/routes/api.chat.ts b/app/routes/api.chat.ts index 473f8c161..47666c703 100644 --- a/app/routes/api.chat.ts +++ b/app/routes/api.chat.ts @@ -10,12 +10,35 @@ export async function action(args: ActionFunctionArgs) { return chatAction(args); } +function parseCookies(cookieHeader) { + const cookies = {}; + + // Split the cookie string by semicolons and spaces + const items = cookieHeader.split(";").map(cookie => cookie.trim()); + + items.forEach(item => { + const [name, ...rest] = item.split("="); + if (name && rest) { + // Decode the name and value, and join value parts in case it contains '=' + const decodedName = decodeURIComponent(name.trim()); + const decodedValue = decodeURIComponent(rest.join("=").trim()); + cookies[decodedName] = decodedValue; + } + }); + + return cookies; +} + async function chatAction({ context, request }: ActionFunctionArgs) { - const { messages, apiKeys } = await request.json<{ - messages: Messages, - apiKeys: Record + const { messages } = await request.json<{ + messages: Messages }>(); + const cookieHeader = request.headers.get("Cookie"); + + // Parse the cookie's value (returns an object or null if no cookie exists) + const apiKeys = JSON.parse(parseCookies(cookieHeader).apiKeys || "{}"); + const stream = new SwitchableStream(); try { @@ -56,7 +79,7 @@ async function chatAction({ context, request }: ActionFunctionArgs) { }); } catch (error) { console.log(error); - + if (error.message?.includes('API key')) { throw new Response('Invalid or missing API key', { status: 401, From feb19509f6514a55ca3f4090a0492a5b37cceaee Mon Sep 17 00:00:00 2001 From: eduardruzga Date: Wed, 13 Nov 2024 22:21:50 +0200 Subject: [PATCH 4/9] Added dynamic openrouter model list --- app/utils/constants.ts | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/app/utils/constants.ts b/app/utils/constants.ts index ecf476b8a..32bc816e6 100644 --- a/app/utils/constants.ts +++ b/app/utils/constants.ts @@ -60,7 +60,9 @@ const PROVIDER_LIST: ProviderInfo[] = [ { name: 'qwen/qwen-110b-chat', label: 'OpenRouter Qwen 110b Chat (OpenRouter)', provider: 'OpenRouter' }, { name: 'cohere/command', label: 'Cohere Command (OpenRouter)', provider: 'OpenRouter' } ], - getApiKeyLink: 'https://openrouter.ai/settings/keys' + getDynamicModels: getOpenRouterModels, + getApiKeyLink: 'https://openrouter.ai/settings/keys', + }, { name: 'Google', staticModels: [ @@ -183,7 +185,34 @@ async function getOpenAILikeModels(): Promise { } catch (e) { return []; } +} +type OpenRouterModelsResponse = { + data: { + name: string; + id: string; + context_length: number; + pricing: { + prompt: number; + completion: number; + } + }[] +}; + +async function getOpenRouterModels(): Promise { + const data: OpenRouterModelsResponse = await (await fetch('https://openrouter.ai/api/v1/models', { + headers: { + 'Content-Type': 'application/json' + } + })).json(); + + return data.data.sort((a, b) => a.name.localeCompare(b.name)).map(m => ({ + name: m.id, + label: `${m.name} - in:$${(m.pricing.prompt * 1_000_000).toFixed( + 2)} out:$${(m.pricing.completion * 1_000_000).toFixed(2)} - context ${Math.floor( + m.context_length / 1000)}k`, + provider: 'OpenRouter' + })); } async function getLMStudioModels(): Promise { @@ -202,10 +231,11 @@ async function getLMStudioModels(): Promise { } -async function initializeModelList(): Promise { + +async function initializeModelList(): Promise { MODEL_LIST = [...(await Promise.all( PROVIDER_LIST.filter(p => !!p.getDynamicModels).map(p => p.getDynamicModels()))).flat(), ...staticModels]; + return MODEL_LIST; } -initializeModelList().then(); -export { getOllamaModels, getOpenAILikeModels, getLMStudioModels, initializeModelList, PROVIDER_LIST }; +export { getOllamaModels, getOpenAILikeModels, getLMStudioModels, initializeModelList, getOpenRouterModels, PROVIDER_LIST }; From e55fb571385b73a543b00049e8b240eb75fa14f3 Mon Sep 17 00:00:00 2001 From: eduardruzga Date: Wed, 13 Nov 2024 22:22:08 +0200 Subject: [PATCH 5/9] Fix google api key bug --- app/lib/.server/llm/model.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/lib/.server/llm/model.ts b/app/lib/.server/llm/model.ts index 1babe7a2c..13f3555e9 100644 --- a/app/lib/.server/llm/model.ts +++ b/app/lib/.server/llm/model.ts @@ -41,9 +41,9 @@ export function getMistralModel(apiKey: string, model: string) { } export function getGoogleModel(apiKey: string, model: string) { - const google = createGoogleGenerativeAI( + const google = createGoogleGenerativeAI({ apiKey, - ); + }); return google(model); } From 9396734dea1cac65fb111d436cddfe8047e6c37f Mon Sep 17 00:00:00 2001 From: eduardruzga Date: Wed, 13 Nov 2024 22:22:25 +0200 Subject: [PATCH 6/9] Various bug fixes around model/provider selection --- app/components/chat/BaseChat.tsx | 15 ++++++++++++--- app/components/chat/Chat.client.tsx | 12 ++++++------ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/app/components/chat/BaseChat.tsx b/app/components/chat/BaseChat.tsx index cfa2afd45..9a34f5bf2 100644 --- a/app/components/chat/BaseChat.tsx +++ b/app/components/chat/BaseChat.tsx @@ -7,7 +7,7 @@ import { Menu } from '~/components/sidebar/Menu.client'; import { IconButton } from '~/components/ui/IconButton'; import { Workbench } from '~/components/workbench/Workbench.client'; import { classNames } from '~/utils/classNames'; -import { MODEL_LIST, DEFAULT_PROVIDER, PROVIDER_LIST, ProviderInfo } from '~/utils/constants'; +import { MODEL_LIST, DEFAULT_PROVIDER, PROVIDER_LIST, ProviderInfo, initializeModelList } from '~/utils/constants'; import { Messages } from './Messages.client'; import { SendButton } from './SendButton.client'; import { useState } from 'react'; @@ -45,8 +45,10 @@ const ModelSelector = ({ model, setModel, provider, setProvider, modelList, prov ))}