Merge pull request #4213 from janhq/fix/4209-inconsistent-max-tokens-…

…value-persistence fix: 4209 - Inconsistent max tokens value persistence
janhq · Dec 4, 2024 · 6557d64 · 6557d64
2 parents 9e634c7 + fe19065
commit 6557d64
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 13 deletions.
diff --git a/web/containers/ModelDropdown/index.tsx b/web/containers/ModelDropdown/index.tsx
@@ -187,15 +187,19 @@ const ModelDropdown = ({
           ],
         })
 
-        const overriddenSettings =
-          model?.settings.ctx_len && model.settings.ctx_len > 4096
-            ? { ctx_len: 4096 }
-            : {}
+        const defaultContextLength = Math.min(
+          8192,
+          model?.settings.ctx_len ?? 8192
+        )
+        const overriddenParameters = {
+          ctx_len: Math.min(8192, model?.settings.ctx_len ?? 8192),
+          max_tokens: defaultContextLength,
+        }
 
         const modelParams = {
           ...model?.parameters,
           ...model?.settings,
-          ...overriddenSettings,
+          ...overriddenParameters,
         }
 
         // Update model parameter to the thread state

diff --git a/web/hooks/useCreateNewThread.ts b/web/hooks/useCreateNewThread.ts
@@ -105,14 +105,21 @@ export const useCreateNewThread = () => {
       enabled: true,
       settings: assistant.tools && assistant.tools[0].settings,
     }
-    const overriddenSettings =
-      defaultModel?.settings.ctx_len && defaultModel.settings.ctx_len > 2048
-        ? { ctx_len: 4096 }
-        : {}
-
-    const overriddenParameters = defaultModel?.parameters.max_tokens
-      ? { max_tokens: 4096 }
-      : {}
+
+    // Default context length is 8192
+    const defaultContextLength = Math.min(
+      8192,
+      defaultModel?.settings.ctx_len ?? 8192
+    )
+
+    const overriddenSettings = {
+      ctx_len: defaultContextLength,
+    }
+
+    // Use ctx length by default
+    const overriddenParameters = {
+      max_tokens: defaultContextLength,
+    }
 
     const createdAt = Date.now()
     let instructions: string | undefined = assistant.instructions