Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

🐛fix: llama.cpp default NGL setting does not offload all layers to GPU (#5310)

Browse Source 
* 🐛fix: llama.cpp default NGL setting does not offload all layers to GPU

* chore: cover more cases

* chore: clean up

* fix: should not show GPU section on Mac
This commit is contained in:
Louis 2025-06-17 15:30:07 +07:00 committed by GitHub
parent 50c25b65b6
commit 2899d58ad7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 33 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/src/browser/models/utils.ts
View File

@ -17,7 +17,7 @@ export const validationRules: { [key: string]: (value: any) => boolean } = {
presence_penalty: (value: any) => typeof value === 'number' && value >= 0 && value <= 1, presence_penalty: (value: any) => typeof value === 'number' && value >= 0 && value <= 1,
ctx_len: (value: any) => Number.isInteger(value) && value >= 0, ctx_len: (value: any) => Number.isInteger(value) && value >= 0,
ngl: (value: any) => Number.isInteger(value) && value >= 0, ngl: (value: any) => Number.isInteger(value),
embedding: (value: any) => typeof value === 'boolean', embedding: (value: any) => typeof value === 'boolean',
n_parallel: (value: any) => Number.isInteger(value) && value >= 0, n_parallel: (value: any) => Number.isInteger(value) && value >= 0,
cpu_threads: (value: any) => Number.isInteger(value) && value >= 0, cpu_threads: (value: any) => Number.isInteger(value) && value >= 0,

6
extensions/inference-cortex-extension/src/index.ts
View File

@ -253,11 +253,12 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
} }
} }
} }
const modelSettings = extractModelLoadParams(model.settings)
return await this.apiInstance().then((api) => return await this.apiInstance().then((api) =>
api api
.post('v1/models/start', { .post('v1/models/start', {
json: { json: {
...extractModelLoadParams(model.settings), ...modelSettings,
model: model.id, model: model.id,
engine: engine:
model.engine === 'nitro' // Legacy model cache model.engine === 'nitro' // Legacy model cache
@ -282,6 +283,9 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
...(this.context_shift === false ...(this.context_shift === false
? { 'no-context-shift': true } ? { 'no-context-shift': true }
: {}), : {}),
...(modelSettings.ngl === -1 || modelSettings.ngl === undefined
? { ngl: 100 }
: {}),
}, },
timeout: false, timeout: false,
signal, signal,

4
web-app/src/routes/settings/hardware.tsx
View File

@ -371,6 +371,7 @@ function Hardware() {
)} )}
{/* GPU Information */} {/* GPU Information */}
{!IS_MACOS ? (
<Card title="GPUs"> <Card title="GPUs">
{hardwareData.gpus.length > 0 ? ( {hardwareData.gpus.length > 0 ? (
<DndContext <DndContext
@ -395,6 +396,9 @@ function Hardware() {
<CardItem title="No GPUs detected" actions={<></>} /> <CardItem title="No GPUs detected" actions={<></>} />
)} )}
</Card> </Card>
) : (
<></>
)}
</div> </div>
</div> </div>
</div> </div>