From bfaaf86c69aa558dda35dc638e30108e5d58dc69 Mon Sep 17 00:00:00 2001 From: Aaron Bolton Date: Mon, 18 Nov 2024 20:48:35 +0000 Subject: [PATCH 1/2] Created DEFAULT_NUM_CTX VAR with a deafult of 32768 --- .env.example | 7 +++++++ app/lib/.server/llm/model.ts | 6 +++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 46a21e892..386d40798 100644 --- a/.env.example +++ b/.env.example @@ -56,3 +56,10 @@ XAI_API_KEY= # Include this environment variable if you want more logging for debugging locally VITE_LOG_LEVEL=debug + +# Example Context Values for qwen2.5-coder:32b +# +# DEFAULT_NUM_CTX=32768 # Consumes 36GB of VRAM +# DEFAULT_NUM_CTX=24576 # Consumes 32GB of VRAM +# DEFAULT_NUM_CTX=12288 # Consumes 26GB of VRAM +# DEFAULT_NUM_CTX=6144 # Consumes 24GB of VRAM \ No newline at end of file diff --git a/app/lib/.server/llm/model.ts b/app/lib/.server/llm/model.ts index 6be9d1170..266dd69d8 100644 --- a/app/lib/.server/llm/model.ts +++ b/app/lib/.server/llm/model.ts @@ -8,6 +8,10 @@ import { ollama } from 'ollama-ai-provider'; import { createOpenRouter } from "@openrouter/ai-sdk-provider"; import { createMistral } from '@ai-sdk/mistral'; +export const DEFAULT_NUM_CTX = process.env.DEFAULT_NUM_CTX ? + parseInt(process.env.DEFAULT_NUM_CTX, 10) : + 32768; + export function getAnthropicModel(apiKey: string, model: string) { const anthropic = createAnthropic({ apiKey, @@ -58,7 +62,7 @@ export function getGroqModel(apiKey: string, model: string) { export function getOllamaModel(baseURL: string, model: string) { let Ollama = ollama(model, { - numCtx: 32768, + numCtx: DEFAULT_NUM_CTX, }); Ollama.config.baseURL = `${baseURL}/api`; From 53594234819346a3d8940b9101ebd9ec69b25641 Mon Sep 17 00:00:00 2001 From: Aaron Bolton Date: Tue, 19 Nov 2024 07:46:51 +0000 Subject: [PATCH 2/2] DEFAULT_NUM_CTX additions adding further changes for DEFAULT_NUM_CTX, including docs --- .env.example | 3 ++- CONTRIBUTING.md | 16 ++++++++++++++++ Dockerfile | 8 ++++++-- docker-compose.yaml | 2 ++ 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 386d40798..9710a8eb9 100644 --- a/.env.example +++ b/.env.example @@ -62,4 +62,5 @@ VITE_LOG_LEVEL=debug # DEFAULT_NUM_CTX=32768 # Consumes 36GB of VRAM # DEFAULT_NUM_CTX=24576 # Consumes 32GB of VRAM # DEFAULT_NUM_CTX=12288 # Consumes 26GB of VRAM -# DEFAULT_NUM_CTX=6144 # Consumes 24GB of VRAM \ No newline at end of file +# DEFAULT_NUM_CTX=6144 # Consumes 24GB of VRAM +DEFAULT_NUM_CTX= \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1bf3bfb77..23f2b8d09 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,7 @@ # Contributing to Bolt.new Fork +## DEFAULT_NUM_CTX + +The `DEFAULT_NUM_CTX` environment variable can be used to limit the maximum number of context values used by the qwen2.5-coder model. For example, to limit the context to 24576 values (which uses 32GB of VRAM), set `DEFAULT_NUM_CTX=24576` in your `.env.local` file. First off, thank you for considering contributing to Bolt.new! This fork aims to expand the capabilities of the original project by integrating multiple LLM providers and enhancing functionality. Every contribution helps make Bolt.new a better tool for developers worldwide. @@ -80,6 +83,19 @@ ANTHROPIC_API_KEY=XXX ```bash VITE_LOG_LEVEL=debug ``` + + - Optionally set context size: +```bash +DEFAULT_NUM_CTX=32768 +``` + +Some Example Context Values for the qwen2.5-coder:32b models are. + +* DEFAULT_NUM_CTX=32768 - Consumes 36GB of VRAM +* DEFAULT_NUM_CTX=24576 - Consumes 32GB of VRAM +* DEFAULT_NUM_CTX=12288 - Consumes 26GB of VRAM +* DEFAULT_NUM_CTX=6144 - Consumes 24GB of VRAM + **Important**: Never commit your `.env.local` file to version control. It's already included in .gitignore. ### 🚀 Running the Development Server diff --git a/Dockerfile b/Dockerfile index 3b5a74cde..1d686737b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,7 @@ ARG OPEN_ROUTER_API_KEY ARG GOOGLE_GENERATIVE_AI_API_KEY ARG OLLAMA_API_BASE_URL ARG VITE_LOG_LEVEL=debug +ARG DEFAULT_NUM_CTX ENV WRANGLER_SEND_METRICS=false \ GROQ_API_KEY=${GROQ_API_KEY} \ @@ -33,7 +34,8 @@ ENV WRANGLER_SEND_METRICS=false \ OPEN_ROUTER_API_KEY=${OPEN_ROUTER_API_KEY} \ GOOGLE_GENERATIVE_AI_API_KEY=${GOOGLE_GENERATIVE_AI_API_KEY} \ OLLAMA_API_BASE_URL=${OLLAMA_API_BASE_URL} \ - VITE_LOG_LEVEL=${VITE_LOG_LEVEL} + VITE_LOG_LEVEL=${VITE_LOG_LEVEL} \ + DEFAULT_NUM_CTX=${DEFAULT_NUM_CTX} # Pre-configure wrangler to disable metrics RUN mkdir -p /root/.config/.wrangler && \ @@ -54,6 +56,7 @@ ARG OPEN_ROUTER_API_KEY ARG GOOGLE_GENERATIVE_AI_API_KEY ARG OLLAMA_API_BASE_URL ARG VITE_LOG_LEVEL=debug +ARG DEFAULT_NUM_CTX ENV GROQ_API_KEY=${GROQ_API_KEY} \ OPENAI_API_KEY=${OPENAI_API_KEY} \ @@ -61,7 +64,8 @@ ENV GROQ_API_KEY=${GROQ_API_KEY} \ OPEN_ROUTER_API_KEY=${OPEN_ROUTER_API_KEY} \ GOOGLE_GENERATIVE_AI_API_KEY=${GOOGLE_GENERATIVE_AI_API_KEY} \ OLLAMA_API_BASE_URL=${OLLAMA_API_BASE_URL} \ - VITE_LOG_LEVEL=${VITE_LOG_LEVEL} + VITE_LOG_LEVEL=${VITE_LOG_LEVEL} \ + DEFAULT_NUM_CTX=${DEFAULT_NUM_CTX} RUN mkdir -p ${WORKDIR}/run CMD pnpm run dev --host diff --git a/docker-compose.yaml b/docker-compose.yaml index c391dd732..6fbd704ac 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -20,6 +20,7 @@ services: - GOOGLE_GENERATIVE_AI_API_KEY=${GOOGLE_GENERATIVE_AI_API_KEY} - OLLAMA_API_BASE_URL=${OLLAMA_API_BASE_URL} - VITE_LOG_LEVEL=${VITE_LOG_LEVEL:-debug} + - DEFAULT_NUM_CTX=${DEFAULT_NUM_CTX:-32768} - RUNNING_IN_DOCKER=true extra_hosts: - "host.docker.internal:host-gateway" @@ -46,6 +47,7 @@ services: - GOOGLE_GENERATIVE_AI_API_KEY=${GOOGLE_GENERATIVE_AI_API_KEY} - OLLAMA_API_BASE_URL=${OLLAMA_API_BASE_URL} - VITE_LOG_LEVEL=${VITE_LOG_LEVEL:-debug} + - DEFAULT_NUM_CTX=${DEFAULT_NUM_CTX:-32768} - RUNNING_IN_DOCKER=true extra_hosts: - "host.docker.internal:host-gateway"