Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: prompt token

Browse Source
This commit is contained in:
Faisal Amir 2025-10-03 19:01:19 +07:00
parent 404f40cc23
commit 5382e9666e
1 changed files with 36 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
extensions/llamacpp-extension/src/index.ts
View File

@ -332,12 +332,14 @@ export default class llamacpp_extension extends AIEngine {
) )
// Clear the invalid stored preference // Clear the invalid stored preference
this.clearStoredBackendType() this.clearStoredBackendType()
bestAvailableBackendString = bestAvailableBackendString = await this.determineBestBackend(
await this.determineBestBackend(version_backends) version_backends
)
} }
} else { } else {
bestAvailableBackendString = bestAvailableBackendString = await this.determineBestBackend(
await this.determineBestBackend(version_backends) version_backends
)
} }
let settings = structuredClone(SETTINGS) let settings = structuredClone(SETTINGS)
@ -2151,7 +2153,12 @@ export default class llamacpp_extension extends AIEngine {
if (mmprojPath && !this.isAbsolutePath(mmprojPath)) if (mmprojPath && !this.isAbsolutePath(mmprojPath))
mmprojPath = await joinPath([await getJanDataFolderPath(), path]) mmprojPath = await joinPath([await getJanDataFolderPath(), path])
try { try {
const result = await planModelLoadInternal(path, this.memoryMode, mmprojPath, requestedCtx) const result = await planModelLoadInternal(
path,
this.memoryMode,
mmprojPath,
requestedCtx
)
return result return result
} catch (e) { } catch (e) {
throw new Error(String(e)) throw new Error(String(e))
@ -2279,30 +2286,38 @@ export default class llamacpp_extension extends AIEngine {
} }
// Calculate text tokens // Calculate text tokens
const messages = JSON.stringify({ messages: opts.messages }) // Use a direct approach: convert messages to text and tokenize directly
// This avoids issues with enable_thinking and assistant prefills
let textToTokenize = ''
let parseResponse = await fetch(`${baseUrl}/apply-template`, { for (const msg of opts.messages) {
method: 'POST', const rolePrefix =
headers: headers, msg.role === 'user'
body: messages, ? 'User: '
}) : msg.role === 'assistant'
? 'Assistant: '
: msg.role === 'system'
? 'System: '
: ''
if (!parseResponse.ok) { if (typeof msg.content === 'string') {
const errorData = await parseResponse.json().catch(() => null) textToTokenize += `${rolePrefix}${msg.content}\n`
throw new Error( } else if (Array.isArray(msg.content)) {
`API request failed with status ${ for (const part of msg.content) {
parseResponse.status if (part.type === 'text' && part.text) {
}: ${JSON.stringify(errorData)}` textToTokenize += part.text
) }
// Skip image tokens as they're calculated separately
}
textToTokenize += '\n'
}
} }
const parsedPrompt = await parseResponse.json()
const response = await fetch(`${baseUrl}/tokenize`, { const response = await fetch(`${baseUrl}/tokenize`, {
method: 'POST', method: 'POST',
headers: headers, headers: headers,
body: JSON.stringify({ body: JSON.stringify({
content: parsedPrompt.prompt, content: textToTokenize,
}), }),
}) })