From bfaaf86c69aa558dda35dc638e30108e5d58dc69 Mon Sep 17 00:00:00 2001 From: Aaron Bolton Date: Mon, 18 Nov 2024 20:48:35 +0000 Subject: [PATCH 01/16] Created DEFAULT_NUM_CTX VAR with a deafult of 32768 --- .env.example | 7 +++++++ app/lib/.server/llm/model.ts | 6 +++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 46a21e892..386d40798 100644 --- a/.env.example +++ b/.env.example @@ -56,3 +56,10 @@ XAI_API_KEY= # Include this environment variable if you want more logging for debugging locally VITE_LOG_LEVEL=debug + +# Example Context Values for qwen2.5-coder:32b +# +# DEFAULT_NUM_CTX=32768 # Consumes 36GB of VRAM +# DEFAULT_NUM_CTX=24576 # Consumes 32GB of VRAM +# DEFAULT_NUM_CTX=12288 # Consumes 26GB of VRAM +# DEFAULT_NUM_CTX=6144 # Consumes 24GB of VRAM \ No newline at end of file diff --git a/app/lib/.server/llm/model.ts b/app/lib/.server/llm/model.ts index 6be9d1170..266dd69d8 100644 --- a/app/lib/.server/llm/model.ts +++ b/app/lib/.server/llm/model.ts @@ -8,6 +8,10 @@ import { ollama } from 'ollama-ai-provider'; import { createOpenRouter } from "@openrouter/ai-sdk-provider"; import { createMistral } from '@ai-sdk/mistral'; +export const DEFAULT_NUM_CTX = process.env.DEFAULT_NUM_CTX ? + parseInt(process.env.DEFAULT_NUM_CTX, 10) : + 32768; + export function getAnthropicModel(apiKey: string, model: string) { const anthropic = createAnthropic({ apiKey, @@ -58,7 +62,7 @@ export function getGroqModel(apiKey: string, model: string) { export function getOllamaModel(baseURL: string, model: string) { let Ollama = ollama(model, { - numCtx: 32768, + numCtx: DEFAULT_NUM_CTX, }); Ollama.config.baseURL = `${baseURL}/api`; From 53594234819346a3d8940b9101ebd9ec69b25641 Mon Sep 17 00:00:00 2001 From: Aaron Bolton Date: Tue, 19 Nov 2024 07:46:51 +0000 Subject: [PATCH 02/16] DEFAULT_NUM_CTX additions adding further changes for DEFAULT_NUM_CTX, including docs --- .env.example | 3 ++- CONTRIBUTING.md | 16 ++++++++++++++++ Dockerfile | 8 ++++++-- docker-compose.yaml | 2 ++ 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 386d40798..9710a8eb9 100644 --- a/.env.example +++ b/.env.example @@ -62,4 +62,5 @@ VITE_LOG_LEVEL=debug # DEFAULT_NUM_CTX=32768 # Consumes 36GB of VRAM # DEFAULT_NUM_CTX=24576 # Consumes 32GB of VRAM # DEFAULT_NUM_CTX=12288 # Consumes 26GB of VRAM -# DEFAULT_NUM_CTX=6144 # Consumes 24GB of VRAM \ No newline at end of file +# DEFAULT_NUM_CTX=6144 # Consumes 24GB of VRAM +DEFAULT_NUM_CTX= \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1bf3bfb77..23f2b8d09 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,7 @@ # Contributing to Bolt.new Fork +## DEFAULT_NUM_CTX + +The `DEFAULT_NUM_CTX` environment variable can be used to limit the maximum number of context values used by the qwen2.5-coder model. For example, to limit the context to 24576 values (which uses 32GB of VRAM), set `DEFAULT_NUM_CTX=24576` in your `.env.local` file. First off, thank you for considering contributing to Bolt.new! This fork aims to expand the capabilities of the original project by integrating multiple LLM providers and enhancing functionality. Every contribution helps make Bolt.new a better tool for developers worldwide. @@ -80,6 +83,19 @@ ANTHROPIC_API_KEY=XXX ```bash VITE_LOG_LEVEL=debug ``` + + - Optionally set context size: +```bash +DEFAULT_NUM_CTX=32768 +``` + +Some Example Context Values for the qwen2.5-coder:32b models are. + +* DEFAULT_NUM_CTX=32768 - Consumes 36GB of VRAM +* DEFAULT_NUM_CTX=24576 - Consumes 32GB of VRAM +* DEFAULT_NUM_CTX=12288 - Consumes 26GB of VRAM +* DEFAULT_NUM_CTX=6144 - Consumes 24GB of VRAM + **Important**: Never commit your `.env.local` file to version control. It's already included in .gitignore. ### 🚀 Running the Development Server diff --git a/Dockerfile b/Dockerfile index 3b5a74cde..1d686737b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,7 @@ ARG OPEN_ROUTER_API_KEY ARG GOOGLE_GENERATIVE_AI_API_KEY ARG OLLAMA_API_BASE_URL ARG VITE_LOG_LEVEL=debug +ARG DEFAULT_NUM_CTX ENV WRANGLER_SEND_METRICS=false \ GROQ_API_KEY=${GROQ_API_KEY} \ @@ -33,7 +34,8 @@ ENV WRANGLER_SEND_METRICS=false \ OPEN_ROUTER_API_KEY=${OPEN_ROUTER_API_KEY} \ GOOGLE_GENERATIVE_AI_API_KEY=${GOOGLE_GENERATIVE_AI_API_KEY} \ OLLAMA_API_BASE_URL=${OLLAMA_API_BASE_URL} \ - VITE_LOG_LEVEL=${VITE_LOG_LEVEL} + VITE_LOG_LEVEL=${VITE_LOG_LEVEL} \ + DEFAULT_NUM_CTX=${DEFAULT_NUM_CTX} # Pre-configure wrangler to disable metrics RUN mkdir -p /root/.config/.wrangler && \ @@ -54,6 +56,7 @@ ARG OPEN_ROUTER_API_KEY ARG GOOGLE_GENERATIVE_AI_API_KEY ARG OLLAMA_API_BASE_URL ARG VITE_LOG_LEVEL=debug +ARG DEFAULT_NUM_CTX ENV GROQ_API_KEY=${GROQ_API_KEY} \ OPENAI_API_KEY=${OPENAI_API_KEY} \ @@ -61,7 +64,8 @@ ENV GROQ_API_KEY=${GROQ_API_KEY} \ OPEN_ROUTER_API_KEY=${OPEN_ROUTER_API_KEY} \ GOOGLE_GENERATIVE_AI_API_KEY=${GOOGLE_GENERATIVE_AI_API_KEY} \ OLLAMA_API_BASE_URL=${OLLAMA_API_BASE_URL} \ - VITE_LOG_LEVEL=${VITE_LOG_LEVEL} + VITE_LOG_LEVEL=${VITE_LOG_LEVEL} \ + DEFAULT_NUM_CTX=${DEFAULT_NUM_CTX} RUN mkdir -p ${WORKDIR}/run CMD pnpm run dev --host diff --git a/docker-compose.yaml b/docker-compose.yaml index c391dd732..6fbd704ac 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -20,6 +20,7 @@ services: - GOOGLE_GENERATIVE_AI_API_KEY=${GOOGLE_GENERATIVE_AI_API_KEY} - OLLAMA_API_BASE_URL=${OLLAMA_API_BASE_URL} - VITE_LOG_LEVEL=${VITE_LOG_LEVEL:-debug} + - DEFAULT_NUM_CTX=${DEFAULT_NUM_CTX:-32768} - RUNNING_IN_DOCKER=true extra_hosts: - "host.docker.internal:host-gateway" @@ -46,6 +47,7 @@ services: - GOOGLE_GENERATIVE_AI_API_KEY=${GOOGLE_GENERATIVE_AI_API_KEY} - OLLAMA_API_BASE_URL=${OLLAMA_API_BASE_URL} - VITE_LOG_LEVEL=${VITE_LOG_LEVEL:-debug} + - DEFAULT_NUM_CTX=${DEFAULT_NUM_CTX:-32768} - RUNNING_IN_DOCKER=true extra_hosts: - "host.docker.internal:host-gateway" From d41a0ac2c2b8f4936d168ef003cbf8e1e71fb1f2 Mon Sep 17 00:00:00 2001 From: Raiyan Hasan Date: Wed, 20 Nov 2024 15:08:42 +0000 Subject: [PATCH 03/16] Cohere support added --- .env.example | 4 +++ README.md | 2 +- app/lib/.server/llm/api-key.ts | 2 ++ app/lib/.server/llm/constants.ts | 2 +- app/lib/.server/llm/model.ts | 14 ++++++++++ app/utils/constants.ts | 16 +++++++++++ package.json | 1 + pnpm-lock.yaml | 47 ++++++++++++++++++++++++++++++++ 8 files changed, 86 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 00a0fd94f..473f05ac4 100644 --- a/.env.example +++ b/.env.example @@ -49,6 +49,10 @@ OPENAI_LIKE_API_KEY= # You only need this environment variable set if you want to use Mistral models MISTRAL_API_KEY= +# Get the Cohere Api key by following these instructions - +# https://dashboard.cohere.com/api-keys +# You only need this environment variable set if you want to use Cohere models +COHERE_API_KEY= # Get LMStudio Base URL from LM Studio Developer Console # Make sure to enable CORS diff --git a/README.md b/README.md index 6bcacd4e7..7a3957ac8 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ https://thinktank.ottomator.ai - ⬜ Azure Open AI API Integration - ⬜ Perplexity Integration - ⬜ Vertex AI Integration -- ⬜ Cohere Integration +- ✅ Cohere Integration - ⬜ Deploy directly to Vercel/Netlify/other similar platforms - ⬜ Prompt caching - ⬜ Better prompt enhancing diff --git a/app/lib/.server/llm/api-key.ts b/app/lib/.server/llm/api-key.ts index f282aa80b..7d8d2f937 100644 --- a/app/lib/.server/llm/api-key.ts +++ b/app/lib/.server/llm/api-key.ts @@ -35,6 +35,8 @@ export function getAPIKey(cloudflareEnv: Env, provider: string, userApiKeys?: Re return env.OPENAI_LIKE_API_KEY || cloudflareEnv.OPENAI_LIKE_API_KEY; case "xAI": return env.XAI_API_KEY || cloudflareEnv.XAI_API_KEY; + case "Cohere": + return env.COHERE_API_KEY; default: return ""; } diff --git a/app/lib/.server/llm/constants.ts b/app/lib/.server/llm/constants.ts index 7b3a0f245..1f993c98d 100644 --- a/app/lib/.server/llm/constants.ts +++ b/app/lib/.server/llm/constants.ts @@ -1,5 +1,5 @@ // see https://docs.anthropic.com/en/docs/about-claude/models -export const MAX_TOKENS = 8000; +export const MAX_TOKENS = 4096; // limits the number of model responses that can be returned in a single request export const MAX_RESPONSE_SEGMENTS = 2; diff --git a/app/lib/.server/llm/model.ts b/app/lib/.server/llm/model.ts index e07f2bbca..2e7c56843 100644 --- a/app/lib/.server/llm/model.ts +++ b/app/lib/.server/llm/model.ts @@ -7,6 +7,7 @@ import { createGoogleGenerativeAI } from '@ai-sdk/google'; import { ollama } from 'ollama-ai-provider'; import { createOpenRouter } from "@openrouter/ai-sdk-provider"; import { createMistral } from '@ai-sdk/mistral'; +import { createCohere } from '@ai-sdk/cohere' export function getAnthropicModel(apiKey: string, model: string) { const anthropic = createAnthropic({ @@ -23,6 +24,15 @@ export function getOpenAILikeModel(baseURL:string,apiKey: string, model: string) return openai(model); } + +export function getCohereAIModel(apiKey:string, model: string){ + const cohere = createCohere({ + apiKey, + }); + + return cohere(model); +} + export function getOpenAIModel(apiKey: string, model: string) { const openai = createOpenAI({ apiKey, @@ -108,6 +118,8 @@ export function getXAIModel(apiKey: string, model: string) { return openai(model); } + + export function getModel(provider: string, model: string, env: Env, apiKeys?: Record) { const apiKey = getAPIKey(env, provider, apiKeys); const baseURL = getBaseURL(env, provider); @@ -135,6 +147,8 @@ export function getModel(provider: string, model: string, env: Env, apiKeys?: Re return getLMStudioModel(baseURL, model); case 'xAI': return getXAIModel(apiKey, model); + case 'Cohere': + return getCohereAIModel(apiKey, model); default: return getOllamaModel(baseURL, model); } diff --git a/app/utils/constants.ts b/app/utils/constants.ts index 672c7a81a..fe7693224 100644 --- a/app/utils/constants.ts +++ b/app/utils/constants.ts @@ -33,6 +33,22 @@ const PROVIDER_LIST: ProviderInfo[] = [ staticModels: [], getDynamicModels: getOpenAILikeModels }, + { + name: 'Cohere', + staticModels: [ + { name: 'command-r-plus-08-2024', label: 'Command R plus Latest', provider: 'Cohere' }, + { name: 'command-r-08-2024', label: 'Command R Latest', provider: 'Cohere' }, + { name: 'command-r-plus', label: 'Command R plus', provider: 'Cohere' }, + { name: 'command-r', label: 'Command R', provider: 'Cohere' }, + { name: 'command', label: 'Command', provider: 'Cohere' }, + { name: 'command-nightly', label: 'Command Nightly', provider: 'Cohere' }, + { name: 'command-light', label: 'Command Light', provider: 'Cohere' }, + { name: 'command-light-nightly', label: 'Command Light Nightly', provider: 'Cohere' }, + { name: 'c4ai-aya-expanse-8b', label: 'c4AI Aya Expanse 8b', provider: 'Cohere' }, + { name: 'c4ai-aya-expanse-32b', label: 'c4AI Aya Expanse 32b', provider: 'Cohere' }, + ], + getApiKeyLink: 'https://dashboard.cohere.com/api-keys' + }, { name: 'OpenRouter', staticModels: [ diff --git a/package.json b/package.json index 40ede0f77..cc1c256d0 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ }, "dependencies": { "@ai-sdk/anthropic": "^0.0.39", + "@ai-sdk/cohere": "^1.0.1", "@ai-sdk/google": "^0.0.52", "@ai-sdk/mistral": "^0.0.43", "@ai-sdk/openai": "^0.0.66", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4158d19c2..951d1a4ff 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,6 +14,9 @@ importers: '@ai-sdk/anthropic': specifier: ^0.0.39 version: 0.0.39(zod@3.23.8) + '@ai-sdk/cohere': + specifier: ^1.0.1 + version: 1.0.1(zod@3.23.8) '@ai-sdk/google': specifier: ^0.0.52 version: 0.0.52(zod@3.23.8) @@ -279,6 +282,12 @@ packages: peerDependencies: zod: ^3.0.0 + '@ai-sdk/cohere@1.0.1': + resolution: {integrity: sha512-xLaSYl/hs9EqfpvT9PvqZrDWjJPQPZBd0iT32T6812vN6kwuEQ6sSgQvqHWczIqxeej2GNRgMQwDL6Lh0L5pZw==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.0.0 + '@ai-sdk/google@0.0.52': resolution: {integrity: sha512-bfsA/1Ae0SQ6NfLwWKs5SU4MBwlzJjVhK6bTVBicYFjUxg9liK/W76P1Tq/qK9OlrODACz3i1STOIWsFPpIOuQ==} engines: {node: '>=18'} @@ -324,6 +333,15 @@ packages: zod: optional: true + '@ai-sdk/provider-utils@2.0.1': + resolution: {integrity: sha512-TNg7rPhRtETB2Z9F0JpOvpGii9Fs8EWM8nYy1jEkvSXkrPJ6b/9zVnDdaJsmLFDyrMbOsPJlkblYtmYEQou36w==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.0.0 + peerDependenciesMeta: + zod: + optional: true + '@ai-sdk/provider@0.0.12': resolution: {integrity: sha512-oOwPQD8i2Ynpn22cur4sk26FW3mSy6t6/X/K1Ay2yGBKYiSpRyLfObhOrZEGsXDx+3euKy4nEZ193R36NM+tpQ==} engines: {node: '>=18'} @@ -336,6 +354,10 @@ packages: resolution: {integrity: sha512-XMsNGJdGO+L0cxhhegtqZ8+T6nn4EoShS819OvCgI2kLbYTIvk0GWFGD0AXJmxkxs3DrpsJxKAFukFR7bvTkgQ==} engines: {node: '>=18'} + '@ai-sdk/provider@1.0.0': + resolution: {integrity: sha512-Sj29AzooJ7SYvhPd+AAWt/E7j63E9+AzRnoMHUaJPRYzOd/WDrVNxxv85prF9gDcQ7XPVlSk9j6oAZV9/DXYpA==} + engines: {node: '>=18'} + '@ai-sdk/react@0.0.62': resolution: {integrity: sha512-1asDpxgmeHWL0/EZPCLENxfOHT+0jce0z/zasRhascodm2S6f6/KZn5doLG9jdmarcb+GjMjFmmwyOVXz3W1xg==} engines: {node: '>=18'} @@ -3033,6 +3055,10 @@ packages: resolution: {integrity: sha512-v0eOBUbiaFojBu2s2NPBfYUoRR9GjcDNvCXVaqEf5vVfpIAh9f8RCo4vXTP8c63QRKCFwoLpMpTdPwwhEKVgzA==} engines: {node: '>=14.18'} + eventsource-parser@3.0.0: + resolution: {integrity: sha512-T1C0XCUimhxVQzW4zFipdx0SficT651NnkR0ZSH3yQwh+mFMdLfgjABVi4YtMTtaL4s168593DaoaRLMqryavA==} + engines: {node: '>=18.0.0'} + evp_bytestokey@1.0.3: resolution: {integrity: sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA==} @@ -5687,6 +5713,12 @@ snapshots: '@ai-sdk/provider-utils': 1.0.9(zod@3.23.8) zod: 3.23.8 + '@ai-sdk/cohere@1.0.1(zod@3.23.8)': + dependencies: + '@ai-sdk/provider': 1.0.0 + '@ai-sdk/provider-utils': 2.0.1(zod@3.23.8) + zod: 3.23.8 + '@ai-sdk/google@0.0.52(zod@3.23.8)': dependencies: '@ai-sdk/provider': 0.0.24 @@ -5733,6 +5765,15 @@ snapshots: optionalDependencies: zod: 3.23.8 + '@ai-sdk/provider-utils@2.0.1(zod@3.23.8)': + dependencies: + '@ai-sdk/provider': 1.0.0 + eventsource-parser: 3.0.0 + nanoid: 3.3.7 + secure-json-parse: 2.7.0 + optionalDependencies: + zod: 3.23.8 + '@ai-sdk/provider@0.0.12': dependencies: json-schema: 0.4.0 @@ -5745,6 +5786,10 @@ snapshots: dependencies: json-schema: 0.4.0 + '@ai-sdk/provider@1.0.0': + dependencies: + json-schema: 0.4.0 + '@ai-sdk/react@0.0.62(react@18.3.1)(zod@3.23.8)': dependencies: '@ai-sdk/provider-utils': 1.0.20(zod@3.23.8) @@ -8751,6 +8796,8 @@ snapshots: eventsource-parser@1.1.2: {} + eventsource-parser@3.0.0: {} + evp_bytestokey@1.0.3: dependencies: md5.js: 1.3.5 From eabfbb2220c58b9b54535f6167f3c8a60deaa8b4 Mon Sep 17 00:00:00 2001 From: Raiyan Hasan Date: Wed, 20 Nov 2024 21:41:51 +0530 Subject: [PATCH 04/16] max token is now dynamically handle for each model --- app/lib/.server/llm/constants.ts | 2 +- app/lib/.server/llm/stream-text.ts | 17 ++-- app/utils/constants.ts | 123 +++++++++++++++-------------- app/utils/types.ts | 1 + 4 files changed, 77 insertions(+), 66 deletions(-) diff --git a/app/lib/.server/llm/constants.ts b/app/lib/.server/llm/constants.ts index 1f993c98d..7b3a0f245 100644 --- a/app/lib/.server/llm/constants.ts +++ b/app/lib/.server/llm/constants.ts @@ -1,5 +1,5 @@ // see https://docs.anthropic.com/en/docs/about-claude/models -export const MAX_TOKENS = 4096; +export const MAX_TOKENS = 8000; // limits the number of model responses that can be returned in a single request export const MAX_RESPONSE_SEGMENTS = 2; diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts index 8bfd79786..dba5a610a 100644 --- a/app/lib/.server/llm/stream-text.ts +++ b/app/lib/.server/llm/stream-text.ts @@ -41,10 +41,9 @@ function extractPropertiesFromMessage(message: Message): { model: string; provid return { model, provider, content: cleanedContent }; } - export function streamText( - messages: Messages, - env: Env, + messages: Messages, + env: Env, options?: StreamingOptions, apiKeys?: Record ) { @@ -64,13 +63,21 @@ export function streamText( return { ...message, content }; } - return message; // No changes for non-user messages + return message; }); + const modelDetails = MODEL_LIST.find((m) => m.name === currentModel); + + const dynamicMaxTokens = + modelDetails && modelDetails.maxTokenAllowed + ? Math.min(MAX_TOKENS, modelDetails.maxTokenAllowed) + : MAX_TOKENS; + console.log(dynamicMaxTokens) + return _streamText({ model: getModel(currentProvider, currentModel, env, apiKeys), system: getSystemPrompt(), - maxTokens: MAX_TOKENS, + maxTokens: dynamicMaxTokens, messages: convertToCoreMessages(processedMessages), ...options, }); diff --git a/app/utils/constants.ts b/app/utils/constants.ts index fe7693224..942afc4fb 100644 --- a/app/utils/constants.ts +++ b/app/utils/constants.ts @@ -12,12 +12,12 @@ const PROVIDER_LIST: ProviderInfo[] = [ { name: 'Anthropic', staticModels: [ - { name: 'claude-3-5-sonnet-latest', label: 'Claude 3.5 Sonnet (new)', provider: 'Anthropic' }, - { name: 'claude-3-5-sonnet-20240620', label: 'Claude 3.5 Sonnet (old)', provider: 'Anthropic' }, - { name: 'claude-3-5-haiku-latest', label: 'Claude 3.5 Haiku (new)', provider: 'Anthropic' }, - { name: 'claude-3-opus-latest', label: 'Claude 3 Opus', provider: 'Anthropic' }, - { name: 'claude-3-sonnet-20240229', label: 'Claude 3 Sonnet', provider: 'Anthropic' }, - { name: 'claude-3-haiku-20240307', label: 'Claude 3 Haiku', provider: 'Anthropic' } + { name: 'claude-3-5-sonnet-latest', label: 'Claude 3.5 Sonnet (new)', provider: 'Anthropic', maxTokenAllowed: 8000 }, + { name: 'claude-3-5-sonnet-20240620', label: 'Claude 3.5 Sonnet (old)', provider: 'Anthropic', maxTokenAllowed: 8000 }, + { name: 'claude-3-5-haiku-latest', label: 'Claude 3.5 Haiku (new)', provider: 'Anthropic', maxTokenAllowed: 8000 }, + { name: 'claude-3-opus-latest', label: 'Claude 3 Opus', provider: 'Anthropic', maxTokenAllowed: 8000 }, + { name: 'claude-3-sonnet-20240229', label: 'Claude 3 Sonnet', provider: 'Anthropic', maxTokenAllowed: 8000 }, + { name: 'claude-3-haiku-20240307', label: 'Claude 3 Haiku', provider: 'Anthropic', maxTokenAllowed: 8000 } ], getApiKeyLink: "https://console.anthropic.com/settings/keys", }, @@ -36,36 +36,37 @@ const PROVIDER_LIST: ProviderInfo[] = [ { name: 'Cohere', staticModels: [ - { name: 'command-r-plus-08-2024', label: 'Command R plus Latest', provider: 'Cohere' }, - { name: 'command-r-08-2024', label: 'Command R Latest', provider: 'Cohere' }, - { name: 'command-r-plus', label: 'Command R plus', provider: 'Cohere' }, - { name: 'command-r', label: 'Command R', provider: 'Cohere' }, - { name: 'command', label: 'Command', provider: 'Cohere' }, - { name: 'command-nightly', label: 'Command Nightly', provider: 'Cohere' }, - { name: 'command-light', label: 'Command Light', provider: 'Cohere' }, - { name: 'command-light-nightly', label: 'Command Light Nightly', provider: 'Cohere' }, - { name: 'c4ai-aya-expanse-8b', label: 'c4AI Aya Expanse 8b', provider: 'Cohere' }, - { name: 'c4ai-aya-expanse-32b', label: 'c4AI Aya Expanse 32b', provider: 'Cohere' }, + { name: 'command-r-plus-08-2024', label: 'Command R plus Latest', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-r-08-2024', label: 'Command R Latest', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-r-plus', label: 'Command R plus', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-r', label: 'Command R', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command', label: 'Command', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-nightly', label: 'Command Nightly', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-light', label: 'Command Light', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-light-nightly', label: 'Command Light Nightly', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'c4ai-aya-expanse-8b', label: 'c4AI Aya Expanse 8b', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'c4ai-aya-expanse-32b', label: 'c4AI Aya Expanse 32b', provider: 'Cohere', maxTokenAllowed: 4096 }, ], getApiKeyLink: 'https://dashboard.cohere.com/api-keys' }, { name: 'OpenRouter', staticModels: [ - { name: 'gpt-4o', label: 'GPT-4o', provider: 'OpenAI' }, + { name: 'gpt-4o', label: 'GPT-4o', provider: 'OpenAI', maxTokenAllowed: 8000 }, { name: 'anthropic/claude-3.5-sonnet', label: 'Anthropic: Claude 3.5 Sonnet (OpenRouter)', provider: 'OpenRouter' + , maxTokenAllowed: 8000 }, - { name: 'anthropic/claude-3-haiku', label: 'Anthropic: Claude 3 Haiku (OpenRouter)', provider: 'OpenRouter' }, - { name: 'deepseek/deepseek-coder', label: 'Deepseek-Coder V2 236B (OpenRouter)', provider: 'OpenRouter' }, - { name: 'google/gemini-flash-1.5', label: 'Google Gemini Flash 1.5 (OpenRouter)', provider: 'OpenRouter' }, - { name: 'google/gemini-pro-1.5', label: 'Google Gemini Pro 1.5 (OpenRouter)', provider: 'OpenRouter' }, - { name: 'x-ai/grok-beta', label: 'xAI Grok Beta (OpenRouter)', provider: 'OpenRouter' }, - { name: 'mistralai/mistral-nemo', label: 'OpenRouter Mistral Nemo (OpenRouter)', provider: 'OpenRouter' }, - { name: 'qwen/qwen-110b-chat', label: 'OpenRouter Qwen 110b Chat (OpenRouter)', provider: 'OpenRouter' }, - { name: 'cohere/command', label: 'Cohere Command (OpenRouter)', provider: 'OpenRouter' } + { name: 'anthropic/claude-3-haiku', label: 'Anthropic: Claude 3 Haiku (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, + { name: 'deepseek/deepseek-coder', label: 'Deepseek-Coder V2 236B (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, + { name: 'google/gemini-flash-1.5', label: 'Google Gemini Flash 1.5 (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, + { name: 'google/gemini-pro-1.5', label: 'Google Gemini Pro 1.5 (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, + { name: 'x-ai/grok-beta', label: 'xAI Grok Beta (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, + { name: 'mistralai/mistral-nemo', label: 'OpenRouter Mistral Nemo (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, + { name: 'qwen/qwen-110b-chat', label: 'OpenRouter Qwen 110b Chat (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, + { name: 'cohere/command', label: 'Cohere Command (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 4096 } ], getDynamicModels: getOpenRouterModels, getApiKeyLink: 'https://openrouter.ai/settings/keys', @@ -73,70 +74,70 @@ const PROVIDER_LIST: ProviderInfo[] = [ }, { name: 'Google', staticModels: [ - { name: 'gemini-1.5-flash-latest', label: 'Gemini 1.5 Flash', provider: 'Google' }, - { name: 'gemini-1.5-flash-002', label: 'Gemini 1.5 Flash-002', provider: 'Google' }, - { name: 'gemini-1.5-flash-8b', label: 'Gemini 1.5 Flash-8b', provider: 'Google' }, - { name: 'gemini-1.5-pro-latest', label: 'Gemini 1.5 Pro', provider: 'Google' }, - { name: 'gemini-1.5-pro-002', label: 'Gemini 1.5 Pro-002', provider: 'Google' }, - { name: 'gemini-exp-1114', label: 'Gemini exp-1114', provider: 'Google' } + { name: 'gemini-1.5-flash-latest', label: 'Gemini 1.5 Flash', provider: 'Google', maxTokenAllowed: 8192 }, + { name: 'gemini-1.5-flash-002', label: 'Gemini 1.5 Flash-002', provider: 'Google', maxTokenAllowed: 8192 }, + { name: 'gemini-1.5-flash-8b', label: 'Gemini 1.5 Flash-8b', provider: 'Google', maxTokenAllowed: 8192 }, + { name: 'gemini-1.5-pro-latest', label: 'Gemini 1.5 Pro', provider: 'Google', maxTokenAllowed: 8192 }, + { name: 'gemini-1.5-pro-002', label: 'Gemini 1.5 Pro-002', provider: 'Google', maxTokenAllowed: 8192 }, + { name: 'gemini-exp-1114', label: 'Gemini exp-1114', provider: 'Google', maxTokenAllowed: 8192 } ], getApiKeyLink: 'https://aistudio.google.com/app/apikey' }, { name: 'Groq', staticModels: [ - { name: 'llama-3.1-70b-versatile', label: 'Llama 3.1 70b (Groq)', provider: 'Groq' }, - { name: 'llama-3.1-8b-instant', label: 'Llama 3.1 8b (Groq)', provider: 'Groq' }, - { name: 'llama-3.2-11b-vision-preview', label: 'Llama 3.2 11b (Groq)', provider: 'Groq' }, - { name: 'llama-3.2-3b-preview', label: 'Llama 3.2 3b (Groq)', provider: 'Groq' }, - { name: 'llama-3.2-1b-preview', label: 'Llama 3.2 1b (Groq)', provider: 'Groq' } + { name: 'llama-3.1-70b-versatile', label: 'Llama 3.1 70b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + { name: 'llama-3.1-8b-instant', label: 'Llama 3.1 8b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + { name: 'llama-3.2-11b-vision-preview', label: 'Llama 3.2 11b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + { name: 'llama-3.2-3b-preview', label: 'Llama 3.2 3b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + { name: 'llama-3.2-1b-preview', label: 'Llama 3.2 1b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 } ], getApiKeyLink: 'https://console.groq.com/keys' }, { name: 'HuggingFace', staticModels: [ - { name: 'Qwen/Qwen2.5-Coder-32B-Instruct', label: 'Qwen2.5-Coder-32B-Instruct (HuggingFace)', provider: 'HuggingFace' }, - { name: '01-ai/Yi-1.5-34B-Chat', label: 'Yi-1.5-34B-Chat (HuggingFace)', provider: 'HuggingFace' }, - { name: 'codellama/CodeLlama-34b-Instruct-hf', label: 'CodeLlama-34b-Instruct (HuggingFace)', provider: 'HuggingFace' }, - { name: 'NousResearch/Hermes-3-Llama-3.1-8B', label: 'Hermes-3-Llama-3.1-8B (HuggingFace)', provider: 'HuggingFace' } + { name: 'Qwen/Qwen2.5-Coder-32B-Instruct', label: 'Qwen2.5-Coder-32B-Instruct (HuggingFace)', provider: 'HuggingFace', maxTokenAllowed: 8000 }, + { name: '01-ai/Yi-1.5-34B-Chat', label: 'Yi-1.5-34B-Chat (HuggingFace)', provider: 'HuggingFace', maxTokenAllowed: 8000 }, + { name: 'codellama/CodeLlama-34b-Instruct-hf', label: 'CodeLlama-34b-Instruct (HuggingFace)', provider: 'HuggingFace', maxTokenAllowed: 8000 }, + { name: 'NousResearch/Hermes-3-Llama-3.1-8B', label: 'Hermes-3-Llama-3.1-8B (HuggingFace)', provider: 'HuggingFace', maxTokenAllowed: 8000 } ], getApiKeyLink: 'https://huggingface.co/settings/tokens' }, - + { name: 'OpenAI', staticModels: [ - { name: 'gpt-4o-mini', label: 'GPT-4o Mini', provider: 'OpenAI' }, - { name: 'gpt-4-turbo', label: 'GPT-4 Turbo', provider: 'OpenAI' }, - { name: 'gpt-4', label: 'GPT-4', provider: 'OpenAI' }, - { name: 'gpt-3.5-turbo', label: 'GPT-3.5 Turbo', provider: 'OpenAI' } + { name: 'gpt-4o-mini', label: 'GPT-4o Mini', provider: 'OpenAI', maxTokenAllowed: 8000 }, + { name: 'gpt-4-turbo', label: 'GPT-4 Turbo', provider: 'OpenAI', maxTokenAllowed: 8000 }, + { name: 'gpt-4', label: 'GPT-4', provider: 'OpenAI', maxTokenAllowed: 8000 }, + { name: 'gpt-3.5-turbo', label: 'GPT-3.5 Turbo', provider: 'OpenAI', maxTokenAllowed: 8000 } ], getApiKeyLink: "https://platform.openai.com/api-keys", }, { name: 'xAI', staticModels: [ - { name: 'grok-beta', label: 'xAI Grok Beta', provider: 'xAI' } + { name: 'grok-beta', label: 'xAI Grok Beta', provider: 'xAI', maxTokenAllowed: 8000 } ], getApiKeyLink: 'https://docs.x.ai/docs/quickstart#creating-an-api-key' }, { name: 'Deepseek', staticModels: [ - { name: 'deepseek-coder', label: 'Deepseek-Coder', provider: 'Deepseek' }, - { name: 'deepseek-chat', label: 'Deepseek-Chat', provider: 'Deepseek' } + { name: 'deepseek-coder', label: 'Deepseek-Coder', provider: 'Deepseek', maxTokenAllowed: 8000 }, + { name: 'deepseek-chat', label: 'Deepseek-Chat', provider: 'Deepseek', maxTokenAllowed: 8000 } ], getApiKeyLink: 'https://platform.deepseek.com/api_keys' }, { name: 'Mistral', staticModels: [ - { name: 'open-mistral-7b', label: 'Mistral 7B', provider: 'Mistral' }, - { name: 'open-mixtral-8x7b', label: 'Mistral 8x7B', provider: 'Mistral' }, - { name: 'open-mixtral-8x22b', label: 'Mistral 8x22B', provider: 'Mistral' }, - { name: 'open-codestral-mamba', label: 'Codestral Mamba', provider: 'Mistral' }, - { name: 'open-mistral-nemo', label: 'Mistral Nemo', provider: 'Mistral' }, - { name: 'ministral-8b-latest', label: 'Mistral 8B', provider: 'Mistral' }, - { name: 'mistral-small-latest', label: 'Mistral Small', provider: 'Mistral' }, - { name: 'codestral-latest', label: 'Codestral', provider: 'Mistral' }, - { name: 'mistral-large-latest', label: 'Mistral Large Latest', provider: 'Mistral' } + { name: 'open-mistral-7b', label: 'Mistral 7B', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'open-mixtral-8x7b', label: 'Mistral 8x7B', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'open-mixtral-8x22b', label: 'Mistral 8x22B', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'open-codestral-mamba', label: 'Codestral Mamba', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'open-mistral-nemo', label: 'Mistral Nemo', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'ministral-8b-latest', label: 'Mistral 8B', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'mistral-small-latest', label: 'Mistral Small', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'codestral-latest', label: 'Codestral', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'mistral-large-latest', label: 'Mistral Large Latest', provider: 'Mistral', maxTokenAllowed: 8000 } ], getApiKeyLink: 'https://console.mistral.ai/api-keys/' }, { @@ -180,7 +181,8 @@ async function getOllamaModels(): Promise { return data.models.map((model: OllamaModel) => ({ name: model.name, label: `${model.name} (${model.details.parameter_size})`, - provider: 'Ollama' + provider: 'Ollama', + maxTokenAllowed:8000, })); } catch (e) { return []; @@ -233,8 +235,9 @@ async function getOpenRouterModels(): Promise { name: m.id, label: `${m.name} - in:$${(m.pricing.prompt * 1_000_000).toFixed( 2)} out:$${(m.pricing.completion * 1_000_000).toFixed(2)} - context ${Math.floor( - m.context_length / 1000)}k`, - provider: 'OpenRouter' + m.context_length / 1000)}k`, + provider: 'OpenRouter', + maxTokenAllowed:8000, })); } diff --git a/app/utils/types.ts b/app/utils/types.ts index a5f9fc18b..171edc3d8 100644 --- a/app/utils/types.ts +++ b/app/utils/types.ts @@ -25,6 +25,7 @@ export interface ModelInfo { name: string; label: string; provider: string; + maxTokenAllowed: number; } export interface ProviderInfo { From 934044f73549cbc2a94e7888af172cc6a4ab743c Mon Sep 17 00:00:00 2001 From: Raiyan Hasan Date: Wed, 20 Nov 2024 21:47:43 +0530 Subject: [PATCH 05/16] console message removed --- app/lib/.server/llm/stream-text.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts index dba5a610a..0a9c8e2d9 100644 --- a/app/lib/.server/llm/stream-text.ts +++ b/app/lib/.server/llm/stream-text.ts @@ -72,7 +72,6 @@ export function streamText( modelDetails && modelDetails.maxTokenAllowed ? Math.min(MAX_TOKENS, modelDetails.maxTokenAllowed) : MAX_TOKENS; - console.log(dynamicMaxTokens) return _streamText({ model: getModel(currentProvider, currentModel, env, apiKeys), From 4269d57ea8c1b0f8e3142936af74dbeb6e7b2e6c Mon Sep 17 00:00:00 2001 From: Raiyan Hasan Date: Wed, 20 Nov 2024 21:52:08 +0530 Subject: [PATCH 06/16] README.md updated --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7a3957ac8..f952e0afc 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,8 @@ https://thinktank.ottomator.ai - ⬜ Azure Open AI API Integration - ⬜ Perplexity Integration - ⬜ Vertex AI Integration -- ✅ Cohere Integration +- ✅ Cohere Integration (@hasanraiyan) +- ✅ Dynamic model max token length (@hasanraiyan) - ⬜ Deploy directly to Vercel/Netlify/other similar platforms - ⬜ Prompt caching - ⬜ Better prompt enhancing From e1c3d603dfa9d31d1aa955a86264916a1a56e28a Mon Sep 17 00:00:00 2001 From: Raiyan Hasan Date: Thu, 21 Nov 2024 11:49:09 +0530 Subject: [PATCH 07/16] Update stream-text.ts dynamic model max Token updated --- app/lib/.server/llm/stream-text.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts index 0a9c8e2d9..4fedccb8e 100644 --- a/app/lib/.server/llm/stream-text.ts +++ b/app/lib/.server/llm/stream-text.ts @@ -68,10 +68,12 @@ export function streamText( const modelDetails = MODEL_LIST.find((m) => m.name === currentModel); - const dynamicMaxTokens = - modelDetails && modelDetails.maxTokenAllowed - ? Math.min(MAX_TOKENS, modelDetails.maxTokenAllowed) - : MAX_TOKENS; + + + const dynamicMaxTokens = +modelDetails && modelDetails.maxTokenAllowed + ? modelDetails.maxTokenAllowed + : MAX_TOKENS; return _streamText({ model: getModel(currentProvider, currentModel, env, apiKeys), From f644066189333512eec77b24b9c515f734c6b354 Mon Sep 17 00:00:00 2001 From: Qwikode Date: Thu, 21 Nov 2024 15:12:33 +0200 Subject: [PATCH 08/16] fix(ui): mobile friendly --- app/components/chat/APIKeyManager.tsx | 47 +++++---- app/components/chat/BaseChat.tsx | 36 +++---- app/components/chat/Messages.client.tsx | 97 ++++++++++--------- .../header/HeaderActionButtons.client.tsx | 5 +- app/components/workbench/Workbench.client.tsx | 7 +- app/lib/hooks/index.ts | 1 + app/lib/hooks/useViewport.ts | 18 ++++ 7 files changed, 124 insertions(+), 87 deletions(-) create mode 100644 app/lib/hooks/useViewport.ts diff --git a/app/components/chat/APIKeyManager.tsx b/app/components/chat/APIKeyManager.tsx index 5b2c85e46..c61e466dc 100644 --- a/app/components/chat/APIKeyManager.tsx +++ b/app/components/chat/APIKeyManager.tsx @@ -10,11 +10,7 @@ interface APIKeyManagerProps { labelForGetApiKey?: string; } -export const APIKeyManager: React.FC = ({ - provider, - apiKey, - setApiKey, - }) => { +export const APIKeyManager: React.FC = ({ provider, apiKey, setApiKey }) => { const [isEditing, setIsEditing] = useState(false); const [tempKey, setTempKey] = useState(apiKey); @@ -24,15 +20,29 @@ export const APIKeyManager: React.FC = ({ }; return ( -
- {provider?.name} API Key: +
+
+ {provider?.name} API Key: + {!isEditing && ( +
+ + {apiKey ? '••••••••' : 'Not set (will still work if set in .env file)'} + + setIsEditing(true)} title="Edit API Key"> +
+ +
+ )} +
+ {isEditing ? ( - <> +
setTempKey(e.target.value)} - className="flex-1 p-1 text-sm rounded border border-bolt-elements-borderColor bg-bolt-elements-prompt-background text-bolt-elements-textPrimary focus:outline-none focus:ring-2 focus:ring-bolt-elements-focus" + className="flex-1 px-2 py-1 text-xs lg:text-sm rounded border border-bolt-elements-borderColor bg-bolt-elements-prompt-background text-bolt-elements-textPrimary focus:outline-none focus:ring-2 focus:ring-bolt-elements-focus" />
@@ -40,20 +50,15 @@ export const APIKeyManager: React.FC = ({ setIsEditing(false)} title="Cancel">
- +
) : ( <> - - {apiKey ? '••••••••' : 'Not set (will still work if set in .env file)'} - - setIsEditing(true)} title="Edit API Key"> -
- - - {provider?.getApiKeyLink && window.open(provider?.getApiKeyLink)} title="Edit API Key"> - {provider?.labelForGetApiKey || 'Get API Key'} -
- } + {provider?.getApiKeyLink && ( + window.open(provider?.getApiKeyLink)} title="Edit API Key"> + {provider?.labelForGetApiKey || 'Get API Key'} +
+ + )} )}
diff --git a/app/components/chat/BaseChat.tsx b/app/components/chat/BaseChat.tsx index 902cb232c..396fb012c 100644 --- a/app/components/chat/BaseChat.tsx +++ b/app/components/chat/BaseChat.tsx @@ -27,9 +27,9 @@ const EXAMPLE_PROMPTS = [ const providerList = PROVIDER_LIST; -const ModelSelector = ({ model, setModel, provider, setProvider, modelList, providerList }) => { +const ModelSelector = ({ model, setModel, provider, setProvider, modelList, providerList, apiKeys }) => { return ( -
+