diff --git a/.env.example b/.env.example index 08774d2e6..a64ce6a92 100644 --- a/.env.example +++ b/.env.example @@ -64,11 +64,16 @@ LMSTUDIO_API_BASE_URL= # You only need this environment variable set if you want to use xAI models XAI_API_KEY= +# Get your GitHub Models API Token here - +# https://github.com/settings/tokens +# You only need this environment variable set if you want to use GitHub models +GITHUB_API_KEY= + # Include this environment variable if you want more logging for debugging locally VITE_LOG_LEVEL=debug # Example Context Values for qwen2.5-coder:32b -# +# # DEFAULT_NUM_CTX=32768 # Consumes 36GB of VRAM # DEFAULT_NUM_CTX=24576 # Consumes 32GB of VRAM # DEFAULT_NUM_CTX=12288 # Consumes 26GB of VRAM diff --git a/Dockerfile b/Dockerfile index c581f7f1d..5b2d7d3d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,7 @@ ARG ANTHROPIC_API_KEY ARG OPEN_ROUTER_API_KEY ARG GOOGLE_GENERATIVE_AI_API_KEY ARG OLLAMA_API_BASE_URL +ARG GITHUB_API_KEY ARG VITE_LOG_LEVEL=debug ARG DEFAULT_NUM_CTX @@ -36,6 +37,7 @@ ENV WRANGLER_SEND_METRICS=false \ OPEN_ROUTER_API_KEY=${OPEN_ROUTER_API_KEY} \ GOOGLE_GENERATIVE_AI_API_KEY=${GOOGLE_GENERATIVE_AI_API_KEY} \ OLLAMA_API_BASE_URL=${OLLAMA_API_BASE_URL} \ + GITHUB_API_KEY=${GITHUB_API_KEY} \ VITE_LOG_LEVEL=${VITE_LOG_LEVEL} \ DEFAULT_NUM_CTX=${DEFAULT_NUM_CTX} @@ -52,12 +54,13 @@ FROM base AS bolt-ai-development # Define the same environment variables for development ARG GROQ_API_KEY -ARG HuggingFace +ARG HuggingFace_API_KEY ARG OPENAI_API_KEY ARG ANTHROPIC_API_KEY ARG OPEN_ROUTER_API_KEY ARG GOOGLE_GENERATIVE_AI_API_KEY ARG OLLAMA_API_BASE_URL +ARG GITHUB_API_KEY ARG VITE_LOG_LEVEL=debug ARG DEFAULT_NUM_CTX @@ -68,6 +71,7 @@ ENV GROQ_API_KEY=${GROQ_API_KEY} \ OPEN_ROUTER_API_KEY=${OPEN_ROUTER_API_KEY} \ GOOGLE_GENERATIVE_AI_API_KEY=${GOOGLE_GENERATIVE_AI_API_KEY} \ OLLAMA_API_BASE_URL=${OLLAMA_API_BASE_URL} \ + GITHUB_API_KEY=${GITHUB_API_KEY} \ VITE_LOG_LEVEL=${VITE_LOG_LEVEL} \ DEFAULT_NUM_CTX=${DEFAULT_NUM_CTX} diff --git a/README.md b/README.md index 0127dd0fa..5004adb4f 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # Bolt.new Fork by Cole Medin - oTToDev -This fork of Bolt.new (oTToDev) allows you to choose the LLM that you use for each prompt! Currently, you can use OpenAI, Anthropic, Ollama, OpenRouter, Gemini, LMStudio, Mistral, xAI, HuggingFace, DeepSeek, or Groq models - and it is easily extended to use any other model supported by the Vercel AI SDK! See the instructions below for running this locally and extending it to include more models. +This fork of Bolt.new (oTToDev) allows you to choose the LLM that you use for each prompt! Currently, you can use OpenAI, Anthropic, Ollama, OpenRouter, Gemini, LMStudio, Mistral, xAI, HuggingFace, DeepSeek, Groq, or GitHub Models - and it is easily extended to use any other model supported by the Vercel AI SDK! See the instructions below for running this locally and extending it to include more models. Join the community for oTToDev! @@ -31,6 +31,8 @@ https://thinktank.ottomator.ai - ✅ Ability to revert code to earlier version (@wonderwhy-er) - ✅ Cohere Integration (@hasanraiyan) - ✅ Dynamic model max token length (@hasanraiyan) +- ✅ GitHub Models Integration (@ThePsyberSleuth) + - ⬜ **HIGH PRIORITY** - Prevent Bolt from rewriting files as often (file locking and diffs) - ⬜ **HIGH PRIORITY** - Better prompting for smaller LLMs (code window sometimes doesn't start) - ⬜ **HIGH PRIORITY** - Load local projects into the app diff --git a/app/lib/.server/llm/api-key.ts b/app/lib/.server/llm/api-key.ts index 295618191..7aa6bb4a5 100644 --- a/app/lib/.server/llm/api-key.ts +++ b/app/lib/.server/llm/api-key.ts @@ -41,6 +41,8 @@ export function getAPIKey(cloudflareEnv: Env, provider: string, userApiKeys?: Re return env.COHERE_API_KEY; case 'AzureOpenAI': return env.AZURE_OPENAI_API_KEY; + case 'GitHub Models': + return env.GITHUB_API_KEY || cloudflareEnv.GITHUB_API_KEY; default: return ''; } diff --git a/app/lib/.server/llm/model.ts b/app/lib/.server/llm/model.ts index 76a371164..9049d4afb 100644 --- a/app/lib/.server/llm/model.ts +++ b/app/lib/.server/llm/model.ts @@ -18,15 +18,16 @@ type OptionalApiKey = string | undefined; export function getAnthropicModel(apiKey: OptionalApiKey, model: string) { const anthropic = createAnthropic({ - apiKey, + apiKey }); return anthropic(model); } + export function getOpenAILikeModel(baseURL: string, apiKey: OptionalApiKey, model: string) { const openai = createOpenAI({ baseURL, - apiKey, + apiKey }); return openai(model); @@ -34,7 +35,7 @@ export function getOpenAILikeModel(baseURL: string, apiKey: OptionalApiKey, mode export function getCohereAIModel(apiKey: OptionalApiKey, model: string) { const cohere = createCohere({ - apiKey, + apiKey }); return cohere(model); @@ -42,7 +43,7 @@ export function getCohereAIModel(apiKey: OptionalApiKey, model: string) { export function getOpenAIModel(apiKey: OptionalApiKey, model: string) { const openai = createOpenAI({ - apiKey, + apiKey }); return openai(model); @@ -50,7 +51,7 @@ export function getOpenAIModel(apiKey: OptionalApiKey, model: string) { export function getMistralModel(apiKey: OptionalApiKey, model: string) { const mistral = createMistral({ - apiKey, + apiKey }); return mistral(model); @@ -58,7 +59,7 @@ export function getMistralModel(apiKey: OptionalApiKey, model: string) { export function getGoogleModel(apiKey: OptionalApiKey, model: string) { const google = createGoogleGenerativeAI({ - apiKey, + apiKey }); return google(model); @@ -67,7 +68,7 @@ export function getGoogleModel(apiKey: OptionalApiKey, model: string) { export function getGroqModel(apiKey: OptionalApiKey, model: string) { const openai = createOpenAI({ baseURL: 'https://api.groq.com/openai/v1', - apiKey, + apiKey }); return openai(model); @@ -76,7 +77,7 @@ export function getGroqModel(apiKey: OptionalApiKey, model: string) { export function getHuggingFaceModel(apiKey: OptionalApiKey, model: string) { const openai = createOpenAI({ baseURL: 'https://api-inference.huggingface.co/v1/', - apiKey, + apiKey }); return openai(model); @@ -84,7 +85,7 @@ export function getHuggingFaceModel(apiKey: OptionalApiKey, model: string) { export function getOllamaModel(baseURL: string, model: string) { const ollamaInstance = ollama(model, { - numCtx: DEFAULT_NUM_CTX, + numCtx: DEFAULT_NUM_CTX }) as LanguageModelV1 & { config: any }; ollamaInstance.config.baseURL = `${baseURL}/api`; @@ -95,7 +96,7 @@ export function getOllamaModel(baseURL: string, model: string) { export function getDeepseekModel(apiKey: OptionalApiKey, model: string) { const openai = createOpenAI({ baseURL: 'https://api.deepseek.com/beta', - apiKey, + apiKey }); return openai(model); @@ -103,7 +104,7 @@ export function getDeepseekModel(apiKey: OptionalApiKey, model: string) { export function getOpenRouterModel(apiKey: OptionalApiKey, model: string) { const openRouter = createOpenRouter({ - apiKey, + apiKey }); return openRouter.chat(model); @@ -112,7 +113,7 @@ export function getOpenRouterModel(apiKey: OptionalApiKey, model: string) { export function getLMStudioModel(baseURL: string, model: string) { const lmstudio = createOpenAI({ baseUrl: `${baseURL}/v1`, - apiKey: '', + apiKey: '' }); return lmstudio(model); @@ -121,7 +122,16 @@ export function getLMStudioModel(baseURL: string, model: string) { export function getXAIModel(apiKey: OptionalApiKey, model: string) { const openai = createOpenAI({ baseURL: 'https://api.x.ai/v1', - apiKey, + apiKey + }); + + return openai(model); +} + +export function getGitHubModel(apiKey: OptionalApiKey, model: string) { + const openai = createOpenAI({ + baseURL: 'https://models.inference.ai.azure.com', + apiKey }); return openai(model); @@ -156,6 +166,8 @@ export function getModel(provider: string, model: string, env: Env, apiKeys?: Re return getXAIModel(apiKey, model); case 'Cohere': return getCohereAIModel(apiKey, model); + case 'GitHub Models': + return getGitHubModel(apiKey, model); default: return getOllamaModel(baseURL, model); } diff --git a/app/lib/runtime/action-runner.ts b/app/lib/runtime/action-runner.ts index 5fb984fd8..4b32f7ada 100644 --- a/app/lib/runtime/action-runner.ts +++ b/app/lib/runtime/action-runner.ts @@ -100,7 +100,6 @@ export class ActionRunner { .catch((error) => { console.error('Action failed:', error); }); - return this.#currentExecutionPromise; } async #executeAction(actionId: string, isStreaming: boolean = false) { diff --git a/app/utils/constants.ts b/app/utils/constants.ts index fcbb3f724..9f0d7bd5e 100644 --- a/app/utils/constants.ts +++ b/app/utils/constants.ts @@ -259,6 +259,175 @@ const PROVIDER_LIST: ProviderInfo[] = [ labelForGetApiKey: 'Get LMStudio', icon: 'i-ph:cloud-arrow-down', }, + { + name: 'GitHub Models', + staticModels: [ + { + name: 'o1-preview', + label: 'O1 Preview', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'o1-mini', + label: 'O1 Mini', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'gpt-4o', + label: 'GPT-4O', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'gpt-4o-mini', + label: 'GPT-4O Mini', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'meta-llama-3.1-405b-instruct', + label: 'Meta Llama 3.1 405B Instruct', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'meta-llama-3.1-70b-instruct', + label: 'Meta Llama 3.1 70B Instruct', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'meta-llama-3.1-8b-instruct', + label: 'Meta Llama 3.1 8B Instruct', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'llama-3.2-90b-vision-instruct', + label: 'Llama 3.2 90B Vision Instruct', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'llama-3.2-11b-vision-instruct', + label: 'Llama 3.2 11B Vision Instruct', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'mistral-large-2407', + label: 'Mistral Large 2407', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'mistral-large', + label: 'Mistral Large', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'mistral-small', + label: 'Mistral Small', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'mistral-nemo', + label: 'Mistral Nemo', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'ministral-3B', + label: 'Mistral: Ministral 3B', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'cohere-command-r-08-2024', + label: 'Cohere Command R 08-2024', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'cohere-command-r', + label: 'Cohere Command R', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'cohere-command-r-plus-08-2024', + label: 'Cohere Command R+ 08-2024', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'cohere-command-r-plus', + label: 'Cohere Command R+', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'phi-3.5-moe-instruct', + label: 'Phi 3.5 MoE Instruct', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'phi-3.5-vision-instruct', + label: 'Phi 3.5 Vision Instruct', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'phi-3.5-mini-instruct', + label: 'Phi-3.5-mini instruct (128k)', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'phi-3-medium-128k-instruct', + label: 'Phi-3-medium instruct (128k)', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'phi-3-medium-4k-instruct', + label: 'Phi-3-medium instruct (4k)', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'phi-3-mini-128k-instruct', + label: 'Phi-3-mini instruct (128k)', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'phi-3-mini-4k-instruct', + label: 'Phi-3-mini instruct (4k)', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'phi-3-small-128k-instruct', + label: 'Phi-3-small instruct (128k)', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + { + name: 'phi-3-small-8k-instruct', + label: 'Phi-3-small instruct (8k)', + provider: 'GitHub Models', + maxTokenAllowed: 8000, + }, + ], + getApiKeyLink: 'https://github.com/settings/tokens', + labelForGetApiKey: 'Get GitHub API Token', + }, ]; export const DEFAULT_PROVIDER = PROVIDER_LIST[0]; @@ -283,9 +452,9 @@ const getOllamaBaseUrl = () => { }; async function getOllamaModels(): Promise { - //if (typeof window === 'undefined') { - //return []; - //} + if (typeof window === 'undefined') { + return []; + } try { const baseUrl = getOllamaBaseUrl(); diff --git a/docker-compose.yaml b/docker-compose.yaml index 4a3cc0aba..876d60cb1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -14,6 +14,7 @@ services: # No strictly neded but serving as hints for Coolify - PORT=5173 - GROQ_API_KEY=${GROQ_API_KEY} + - GITHUB_API_KEY=${GITHUB_API_KEY} - HuggingFace_API_KEY=${HuggingFace_API_KEY} - OPENAI_API_KEY=${OPENAI_API_KEY} - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} @@ -24,7 +25,7 @@ services: - DEFAULT_NUM_CTX=${DEFAULT_NUM_CTX:-32768} - RUNNING_IN_DOCKER=true extra_hosts: - - "host.docker.internal:host-gateway" + - "host.docker.internal:host-gateway" command: pnpm run dockerstart profiles: - production # This service only runs in the production profile @@ -39,9 +40,10 @@ services: - VITE_HMR_HOST=localhost - VITE_HMR_PORT=5173 - CHOKIDAR_USEPOLLING=true - - WATCHPACK_POLLING=true + - WATCHPACK_POLLING=true - PORT=5173 - GROQ_API_KEY=${GROQ_API_KEY} + - GITHUB_API_KEY=${GITHUB_API_KEY} - HuggingFace_API_KEY=${HuggingFace_API_KEY} - OPENAI_API_KEY=${OPENAI_API_KEY} - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} @@ -52,7 +54,7 @@ services: - DEFAULT_NUM_CTX=${DEFAULT_NUM_CTX:-32768} - RUNNING_IN_DOCKER=true extra_hosts: - - "host.docker.internal:host-gateway" + - "host.docker.internal:host-gateway" volumes: - type: bind source: . diff --git a/docs/docs/CONTRIBUTING.md b/docs/docs/CONTRIBUTING.md index e1edd87da..66b2350d3 100644 --- a/docs/docs/CONTRIBUTING.md +++ b/docs/docs/CONTRIBUTING.md @@ -13,6 +13,7 @@ First off, thank you for considering contributing to Bolt.new! This fork aims to - [Coding Standards](#coding-standards) - [Development Setup](#development-setup) - [Deploymnt with Docker](#docker-deployment-documentation) +- [Model Providers](#model-providers) ## Code of Conduct @@ -124,7 +125,64 @@ pnpm run deploy Make sure you have the necessary permissions and Wrangler is correctly configured for your Cloudflare account. -# Docker Deployment Documentation +## Model Providers + +### GitHub Models Provider by @ThePsyberSleuth + +The GitHub Models provider integrates a comprehensive suite of AI models through Azure ML endpoints, offering access to cutting-edge models from various providers. This integration enables developers to leverage powerful language models while maintaining compatibility with existing OpenAI-style interfaces. + +#### Setup Requirements + +1. Obtain a GitHub API Key from https://github.com/settings/tokens +2. Add the key to your `.env.local`: + ```bash + GITHUB_API_KEY=your_key_here + ``` + +#### Available Models + +1. O1 Series + - O1 Preview (Latest cutting-edge model) + - O1 Mini (Efficient, smaller model) + +2. Meta Llama Series + - Llama 3.1 405B Instruct + - Llama 3.1 70B Instruct + - Llama 3.1 8B Instruct + +3. Mistral Series + - Mistral Large + - Mistral Medium + - Mistral Small + +4. GPT-4O Series + - GPT-4O Advanced + - GPT-4O Standard + +5. Phi Series + - Phi-2 + - Phi-1.5 + +#### Usage Example + +```typescript +// Initialize the model +const model = getModel('GitHub Models', 'mistral-large', env); + +// Use the model +const response = await model.chat([ + { role: 'system', content: 'You are a helpful AI assistant.' }, + { role: 'user', content: 'Hello!' } +]); +``` + +#### Configuration Options + +- Context Window: Configurable through `DEFAULT_NUM_CTX` +- Temperature: Adjustable per request +- Response Format: Supports both streaming and non-streaming responses + +## Docker Deployment Documentation This guide outlines various methods for building and deploying the application using Docker. diff --git a/vite.config.ts b/vite.config.ts index 9c94ceae3..c0c264c46 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -27,7 +27,7 @@ export default defineConfig((config) => { chrome129IssuePlugin(), config.mode === 'production' && optimizeCssModules({ apply: 'build' }), ], - envPrefix:["VITE_","OPENAI_LIKE_API_","OLLAMA_API_BASE_URL","LMSTUDIO_API_BASE_URL"], + envPrefix: ['VITE_', 'OPENAI_LIKE_API_', 'OLLAMA_API_BASE_URL', 'LMSTUDIO_API_BASE_URL'], css: { preprocessorOptions: { scss: { diff --git a/worker-configuration.d.ts b/worker-configuration.d.ts index 9c074b818..0787baaa1 100644 --- a/worker-configuration.d.ts +++ b/worker-configuration.d.ts @@ -12,4 +12,5 @@ interface Env { GOOGLE_GENERATIVE_AI_API_KEY: string; MISTRAL_API_KEY: string; XAI_API_KEY: string; + GITHUB_API_KEY: string; }