Merge pull request #4266 from janhq/fix/4238-wrong-max-tokens-remote-models

fix: 4238 - fix default max_tokens set on remote models

extensions/inference-openai-extension/resources/models.json

@@ -99,10 +99,10 @@
     "format": "api",
     "settings": {},
     "parameters": {
+      "max_tokens": 32768,
       "temperature": 1,
       "top_p": 1,
       "stream": true,
-      "max_tokens": 32768,
       "frequency_penalty": 0,
       "presence_penalty": 0
     },
@@ -126,9 +126,9 @@
     "format": "api",
     "settings": {},
     "parameters": {
+      "max_tokens": 65536,
       "temperature": 1,
       "top_p": 1,
-      "max_tokens": 65536,
       "stream": true,
       "frequency_penalty": 0,
       "presence_penalty": 0

web/containers/ModelDropdown/index.tsx

@@ -192,8 +192,12 @@ const ModelDropdown = ({
           model?.settings.ctx_len ?? 8192
         )
         const overriddenParameters = {
-          ctx_len: Math.min(8192, model?.settings.ctx_len ?? 8192),
+          ctx_len: !isLocalEngine(model?.engine)
-          max_tokens: defaultContextLength,
+            ? undefined
+            : defaultContextLength,
+          max_tokens: !isLocalEngine(model?.engine)
+            ? (model?.parameters.max_tokens ?? 8192)
+            : defaultContextLength,
         }
 
         const modelParams = {

web/hooks/useCreateNewThread.ts

@@ -17,6 +17,7 @@ import { fileUploadAtom } from '@/containers/Providers/Jotai'
 
 import { toaster } from '@/containers/Toast'
 
+import { isLocalEngine } from '@/utils/modelEngine'
 import { generateThreadId } from '@/utils/thread'
 
 import { useActiveModel } from './useActiveModel'
@@ -113,12 +114,14 @@ export const useCreateNewThread = () => {
     )
 
     const overriddenSettings = {
-      ctx_len: defaultContextLength,
+      ctx_len: !isLocalEngine(model?.engine) ? undefined : defaultContextLength,
     }
 
     // Use ctx length by default
     const overriddenParameters = {
-      max_tokens: defaultContextLength,
+      max_tokens: !isLocalEngine(model?.engine)
+        ? (model?.parameters.token_limit ?? 8192)
+        : defaultContextLength,
     }
 
     const createdAt = Date.now()

web/utils/modelEngine.ts

@@ -38,7 +38,9 @@ export const getLogoEngine = (engine: InferenceEngine) => {
  * @param engine
  * @returns
  */
-export const isLocalEngine = (engine: string) => {
+export const isLocalEngine = (engine?: string) => {
+  if (!engine) return false
+
   const engineObj = EngineManager.instance().get(engine)
   if (!engineObj) return false
   return (