UI: added ollama ctx settings

added ollama ctx setting in the SettingWindow save to cookies like the apikey
stackblitz-labs · Dec 9, 2024 · 0642e64 · 0642e64
1 parent 178653d
commit 0642e64
Show file tree

Hide file tree

Showing 2 changed files with 45 additions and 11 deletions.
diff --git a/app/components/settings/SettingsWindow.tsx b/app/components/settings/SettingsWindow.tsx
@@ -1,6 +1,6 @@
 import * as RadixDialog from '@radix-ui/react-dialog';
 import { motion } from 'framer-motion';
-import { useState } from 'react';
+import { useState, useEffect } from 'react';
 import { classNames } from '~/utils/classNames';
 import { DialogTitle, dialogVariants, dialogBackdropVariants } from '~/components/ui/Dialog';
 import { IconButton } from '~/components/ui/IconButton';
@@ -192,6 +192,20 @@ export const SettingsWindow = ({ open, onClose }: SettingsProps) => {
 
   const versionHash = commit.commit; // Get the version hash from commit.json
 
+  // Add a new state for default context
+  const [defaultCtx, setDefaultCtx] = useState<number>(() => {
+    const savedCtx = Cookies.get('ollamaDefaultCtx');
+    return savedCtx ? Number(savedCtx) : 8192; // Default to 8192 if not set
+  });
+
+  const handleDefaultCtxChange = (value: string) => {
+    const numValue = Number(value);
+    if (!isNaN(numValue) && numValue >= 2048 && numValue <= 131072) {
+      setDefaultCtx(numValue);
+      Cookies.set('ollamaDefaultCtx', numValue.toString());
+    }
+  };
+
   return (
     <RadixDialog.Root open={open}>
       <RadixDialog.Portal>
@@ -322,15 +336,31 @@ export const SettingsWindow = ({ open, onClose }: SettingsProps) => {
                           </div>
                           {/* Base URL input for configurable providers */}
                           {URL_CONFIGURABLE_PROVIDERS.includes(provider.name) && provider.isEnabled && (
-                            <div className="mt-2">
-                              <label className="block text-sm text-bolt-elements-textSecondary mb-1">Base URL:</label>
-                              <input
-                                type="text"
-                                value={baseUrls[provider.name]}
-                                onChange={(e) => handleBaseUrlChange(provider.name, e.target.value)}
-                                placeholder={`Enter ${provider.name} base URL`}
-                                className="w-full bg-white dark:bg-bolt-elements-background-depth-4 relative px-2 py-1.5 rounded-md focus:outline-none placeholder-bolt-elements-textTertiary text-bolt-elements-textPrimary dark:text-bolt-elements-textPrimary border border-bolt-elements-borderColor"
-                              />
+                            <div className="mt-2 flex items-center">
+                              <div className="flex-1 mr-2">
+                                <label className="block text-sm text-bolt-elements-textSecondary mb-1">Base URL:</label>
+                                <input
+                                  type="text"
+                                  value={baseUrls[provider.name]}
+                                  onChange={(e) => handleBaseUrlChange(provider.name, e.target.value)}
+                                  placeholder={`Enter ${provider.name} base URL`}
+                                  className="w-full bg-white dark:bg-bolt-elements-background-depth-4 relative px-2 py-1.5 rounded-md focus:outline-none placeholder-bolt-elements-textTertiary text-bolt-elements-textPrimary dark:text-bolt-elements-textPrimary border border-bolt-elements-borderColor"
+                                />
+                              </div>
+                              {provider.name === 'Ollama' && (
+                                <div className="flex-none w-32">
+                                  <label className="block text-sm text-bolt-elements-textSecondary mb-1">Default Context:</label>
+                                  <input
+                                    type="number"
+                                    value={defaultCtx}
+                                    onChange={(e) => handleDefaultCtxChange(e.target.value)}
+                                    min="2048"
+                                    max="131072"
+                                    step="1"
+                                    className="w-full bg-white dark:bg-bolt-elements-background-depth-4 relative px-2 py-1.5 rounded-md focus:outline-none placeholder-bolt-elements-textTertiary text-bolt-elements-textPrimary dark:text-bolt-elements-textPrimary border border-bolt-elements-borderColor"
+                                  />
+                                </div>
+                              )}
                             </div>
                           )}
                         </div>

diff --git a/app/lib/.server/llm/model.ts b/app/lib/.server/llm/model.ts
@@ -11,6 +11,7 @@ import { createOpenRouter } from '@openrouter/ai-sdk-provider';
 import { createMistral } from '@ai-sdk/mistral';
 import { createCohere } from '@ai-sdk/cohere';
 import type { LanguageModelV1 } from 'ai';
+import Cookies from 'js-cookie';
 
 export const DEFAULT_NUM_CTX = process.env.DEFAULT_NUM_CTX ? parseInt(process.env.DEFAULT_NUM_CTX, 10) : 32768;
 
@@ -83,8 +84,11 @@ export function getHuggingFaceModel(apiKey: OptionalApiKey, model: string) {
 }
 
 export function getOllamaModel(baseURL: string, model: string) {
+  const savedCtx = Cookies.get('ollamaDefaultCtx');
+  const numCtx = savedCtx ? Number(savedCtx) : DEFAULT_NUM_CTX;
+
   const ollamaInstance = ollama(model, {
-    numCtx: DEFAULT_NUM_CTX,
+    numCtx,
   }) as LanguageModelV1 & { config: any };
 
   ollamaInstance.config.baseURL = `${baseURL}/api`;