Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: cortex.cpp gpu activation could cause a race condition (#4825)

Browse Source
This commit is contained in:
Louis 2025-03-21 14:55:39 +07:00 committed by GitHub
parent 2271c8d3d6
commit 7e46295af1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 374 additions and 252 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

117
extensions/conversational-extension/src/index.ts
View File

@ -22,22 +22,26 @@ type MessageList = {
export default class CortexConversationalExtension extends ConversationalExtension { export default class CortexConversationalExtension extends ConversationalExtension {
queue = new PQueue({ concurrency: 1 }) queue = new PQueue({ concurrency: 1 })
api?: KyInstance
/** /**
* Extended API instance for making requests to the Cortex API. * Get the API instance
* @returns * @returns
*/ */
api: KyInstance async apiInstance(): Promise<KyInstance> {
if(this.api) return this.api
const apiKey = (await window.core?.api.appToken()) ?? 'cortex.cpp'
this.api = ky.extend({
prefixUrl: API_URL,
headers: {
Authorization: `Bearer ${apiKey}`,
},
})
return this.api
}
/** /**
* Called when the extension is loaded. * Called when the extension is loaded.
*/ */
async onLoad() { async onLoad() {
const apiKey = await window.core?.api.appToken() ?? 'cortex.cpp'
this.api = ky.extend({
prefixUrl: API_URL,
headers: {
Authorization: `Bearer ${apiKey}`,
},
})
this.queue.add(() => this.healthz()) this.queue.add(() => this.healthz())
} }
@ -51,10 +55,12 @@ export default class CortexConversationalExtension extends ConversationalExtensi
*/ */
async listThreads(): Promise<Thread[]> { async listThreads(): Promise<Thread[]> {
return this.queue.add(() => return this.queue.add(() =>
this.api this.apiInstance().then((api) =>
.get('v1/threads?limit=-1') api
.json<ThreadList>() .get('v1/threads?limit=-1')
.then((e) => e.data) .json<ThreadList>()
.then((e) => e.data)
)
) as Promise<Thread[]> ) as Promise<Thread[]>
} }
@ -64,7 +70,9 @@ export default class CortexConversationalExtension extends ConversationalExtensi
*/ */
async createThread(thread: Thread): Promise<Thread> { async createThread(thread: Thread): Promise<Thread> {
return this.queue.add(() => return this.queue.add(() =>
this.api.post('v1/threads', { json: thread }).json<Thread>() this.apiInstance().then((api) =>
api.post('v1/threads', { json: thread }).json<Thread>()
)
) as Promise<Thread> ) as Promise<Thread>
} }
@ -75,7 +83,9 @@ export default class CortexConversationalExtension extends ConversationalExtensi
async modifyThread(thread: Thread): Promise<void> { async modifyThread(thread: Thread): Promise<void> {
return this.queue return this.queue
.add(() => .add(() =>
this.api.patch(`v1/threads/${thread.id}`, { json: thread }) this.apiInstance().then((api) =>
api.patch(`v1/threads/${thread.id}`, { json: thread })
)
) )
.then() .then()
} }
@ -86,7 +96,9 @@ export default class CortexConversationalExtension extends ConversationalExtensi
*/ */
async deleteThread(threadId: string): Promise<void> { async deleteThread(threadId: string): Promise<void> {
return this.queue return this.queue
.add(() => this.api.delete(`v1/threads/${threadId}`)) .add(() =>
this.apiInstance().then((api) => api.delete(`v1/threads/${threadId}`))
)
.then() .then()
} }
@ -97,11 +109,13 @@ export default class CortexConversationalExtension extends ConversationalExtensi
*/ */
async createMessage(message: ThreadMessage): Promise<ThreadMessage> { async createMessage(message: ThreadMessage): Promise<ThreadMessage> {
return this.queue.add(() => return this.queue.add(() =>
this.api this.apiInstance().then((api) =>
.post(`v1/threads/${message.thread_id}/messages`, { api
json: message, .post(`v1/threads/${message.thread_id}/messages`, {
}) json: message,
.json<ThreadMessage>() })
.json<ThreadMessage>()
)
) as Promise<ThreadMessage> ) as Promise<ThreadMessage>
} }
@ -112,14 +126,13 @@ export default class CortexConversationalExtension extends ConversationalExtensi
*/ */
async modifyMessage(message: ThreadMessage): Promise<ThreadMessage> { async modifyMessage(message: ThreadMessage): Promise<ThreadMessage> {
return this.queue.add(() => return this.queue.add(() =>
this.api this.apiInstance().then((api) =>
.patch( api
`v1/threads/${message.thread_id}/messages/${message.id}`, .patch(`v1/threads/${message.thread_id}/messages/${message.id}`, {
{
json: message, json: message,
} })
) .json<ThreadMessage>()
.json<ThreadMessage>() )
) as Promise<ThreadMessage> ) as Promise<ThreadMessage>
} }
@ -132,7 +145,9 @@ export default class CortexConversationalExtension extends ConversationalExtensi
async deleteMessage(threadId: string, messageId: string): Promise<void> { async deleteMessage(threadId: string, messageId: string): Promise<void> {
return this.queue return this.queue
.add(() => .add(() =>
this.api.delete(`v1/threads/${threadId}/messages/${messageId}`) this.apiInstance().then((api) =>
api.delete(`v1/threads/${threadId}/messages/${messageId}`)
)
) )
.then() .then()
} }
@ -144,10 +159,12 @@ export default class CortexConversationalExtension extends ConversationalExtensi
*/ */
async listMessages(threadId: string): Promise<ThreadMessage[]> { async listMessages(threadId: string): Promise<ThreadMessage[]> {
return this.queue.add(() => return this.queue.add(() =>
this.api this.apiInstance().then((api) =>
.get(`v1/threads/${threadId}/messages?order=asc&limit=-1`) api
.json<MessageList>() .get(`v1/threads/${threadId}/messages?order=asc&limit=-1`)
.then((e) => e.data) .json<MessageList>()
.then((e) => e.data)
)
) as Promise<ThreadMessage[]> ) as Promise<ThreadMessage[]>
} }
@ -159,9 +176,11 @@ export default class CortexConversationalExtension extends ConversationalExtensi
*/ */
async getThreadAssistant(threadId: string): Promise<ThreadAssistantInfo> { async getThreadAssistant(threadId: string): Promise<ThreadAssistantInfo> {
return this.queue.add(() => return this.queue.add(() =>
this.api this.apiInstance().then((api) =>
.get(`v1/assistants/${threadId}?limit=-1`) api
.json<ThreadAssistantInfo>() .get(`v1/assistants/${threadId}?limit=-1`)
.json<ThreadAssistantInfo>()
)
) as Promise<ThreadAssistantInfo> ) as Promise<ThreadAssistantInfo>
} }
/** /**
@ -175,9 +194,11 @@ export default class CortexConversationalExtension extends ConversationalExtensi
assistant: ThreadAssistantInfo assistant: ThreadAssistantInfo
): Promise<ThreadAssistantInfo> { ): Promise<ThreadAssistantInfo> {
return this.queue.add(() => return this.queue.add(() =>
this.api this.apiInstance().then((api) =>
.post(`v1/assistants/${threadId}`, { json: assistant }) api
.json<ThreadAssistantInfo>() .post(`v1/assistants/${threadId}`, { json: assistant })
.json<ThreadAssistantInfo>()
)
) as Promise<ThreadAssistantInfo> ) as Promise<ThreadAssistantInfo>
} }
@ -192,9 +213,11 @@ export default class CortexConversationalExtension extends ConversationalExtensi
assistant: ThreadAssistantInfo assistant: ThreadAssistantInfo
): Promise<ThreadAssistantInfo> { ): Promise<ThreadAssistantInfo> {
return this.queue.add(() => return this.queue.add(() =>
this.api this.apiInstance().then((api) =>
.patch(`v1/assistants/${threadId}`, { json: assistant }) api
.json<ThreadAssistantInfo>() .patch(`v1/assistants/${threadId}`, { json: assistant })
.json<ThreadAssistantInfo>()
)
) as Promise<ThreadAssistantInfo> ) as Promise<ThreadAssistantInfo>
} }
@ -203,10 +226,12 @@ export default class CortexConversationalExtension extends ConversationalExtensi
* @returns * @returns
*/ */
async healthz(): Promise<void> { async healthz(): Promise<void> {
return this.api return this.apiInstance()
.get('healthz', { .then((api) =>
retry: { limit: 20, delay: () => 500, methods: ['get'] }, api.get('healthz', {
}) retry: { limit: 20, delay: () => 500, methods: ['get'] },
})
)
.then(() => {}) .then(() => {})
} }
} }

175
extensions/engine-management-extension/src/index.ts
View File

@ -31,15 +31,13 @@ interface ModelList {
export default class JanEngineManagementExtension extends EngineManagementExtension { export default class JanEngineManagementExtension extends EngineManagementExtension {
queue = new PQueue({ concurrency: 1 }) queue = new PQueue({ concurrency: 1 })
api?: KyInstance
/** /**
* Extended API instance for making requests to the Cortex API. * Get the API instance
* @returns * @returns
*/ */
api: KyInstance async apiInstance(): Promise<KyInstance> {
/** if(this.api) return this.api
* Called when the extension is loaded.
*/
async onLoad() {
const apiKey = (await window.core?.api.appToken()) ?? 'cortex.cpp' const apiKey = (await window.core?.api.appToken()) ?? 'cortex.cpp'
this.api = ky.extend({ this.api = ky.extend({
prefixUrl: API_URL, prefixUrl: API_URL,
@ -47,6 +45,12 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
Authorization: `Bearer ${apiKey}`, Authorization: `Bearer ${apiKey}`,
}, },
}) })
return this.api
}
/**
* Called when the extension is loaded.
*/
async onLoad() {
// Symlink Engines Directory // Symlink Engines Directory
await executeOnMain(NODE, 'symlinkEngines') await executeOnMain(NODE, 'symlinkEngines')
// Run Healthcheck // Run Healthcheck
@ -71,10 +75,12 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
*/ */
async getEngines(): Promise<Engines> { async getEngines(): Promise<Engines> {
return this.queue.add(() => return this.queue.add(() =>
this.api this.apiInstance().then((api) =>
.get('v1/engines') api
.json<Engines>() .get('v1/engines')
.then((e) => e) .json<Engines>()
.then((e) => e)
)
) as Promise<Engines> ) as Promise<Engines>
} }
@ -82,12 +88,15 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
* @returns A Promise that resolves to an object of list engines. * @returns A Promise that resolves to an object of list engines.
*/ */
async getRemoteModels(name: string): Promise<any> { async getRemoteModels(name: string): Promise<any> {
return this.api return this.apiInstance().then(
.get(`v1/models/remote/${name}`) (api) =>
.json<ModelList>() api
.catch(() => ({ .get(`v1/models/remote/${name}`)
data: [], .json<ModelList>()
})) as Promise<ModelList> .catch(() => ({
data: [],
})) as Promise<ModelList>
)
} }
/** /**
@ -96,10 +105,12 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
*/ */
async getInstalledEngines(name: InferenceEngine): Promise<EngineVariant[]> { async getInstalledEngines(name: InferenceEngine): Promise<EngineVariant[]> {
return this.queue.add(() => return this.queue.add(() =>
this.api this.apiInstance().then((api) =>
.get(`v1/engines/${name}`) api
.json<EngineVariant[]>() .get(`v1/engines/${name}`)
.then((e) => e) .json<EngineVariant[]>()
.then((e) => e)
)
) as Promise<EngineVariant[]> ) as Promise<EngineVariant[]>
} }
@ -115,12 +126,14 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
platform?: string platform?: string
) { ) {
return this.queue.add(() => return this.queue.add(() =>
this.api this.apiInstance().then((api) =>
.get(`v1/engines/${name}/releases/${version}`) api
.json<EngineReleased[]>() .get(`v1/engines/${name}/releases/${version}`)
.then((e) => .json<EngineReleased[]>()
platform ? e.filter((r) => r.name.includes(platform)) : e .then((e) =>
) platform ? e.filter((r) => r.name.includes(platform)) : e
)
)
) as Promise<EngineReleased[]> ) as Promise<EngineReleased[]>
} }
@ -131,12 +144,14 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
*/ */
async getLatestReleasedEngine(name: InferenceEngine, platform?: string) { async getLatestReleasedEngine(name: InferenceEngine, platform?: string) {
return this.queue.add(() => return this.queue.add(() =>
this.api this.apiInstance().then((api) =>
.get(`v1/engines/${name}/releases/latest`) api
.json<EngineReleased[]>() .get(`v1/engines/${name}/releases/latest`)
.then((e) => .json<EngineReleased[]>()
platform ? e.filter((r) => r.name.includes(platform)) : e .then((e) =>
) platform ? e.filter((r) => r.name.includes(platform)) : e
)
)
) as Promise<EngineReleased[]> ) as Promise<EngineReleased[]>
} }
@ -146,9 +161,11 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
*/ */
async installEngine(name: string, engineConfig: EngineConfig) { async installEngine(name: string, engineConfig: EngineConfig) {
return this.queue.add(() => return this.queue.add(() =>
this.api this.apiInstance().then((api) =>
.post(`v1/engines/${name}/install`, { json: engineConfig }) api
.then((e) => e) .post(`v1/engines/${name}/install`, { json: engineConfig })
.then((e) => e)
)
) as Promise<{ messages: string }> ) as Promise<{ messages: string }>
} }
@ -179,9 +196,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
engineConfig.metadata.header_template = DEFAULT_REQUEST_HEADERS_TRANSFORM engineConfig.metadata.header_template = DEFAULT_REQUEST_HEADERS_TRANSFORM
return this.queue.add(() => return this.queue.add(() =>
this.api this.apiInstance().then((api) =>
.post('v1/engines', { json: engineConfig }) api.post('v1/engines', { json: engineConfig }).then((e) => {
.then((e) => {
if (persistModels && engineConfig.metadata?.get_models_url) { if (persistModels && engineConfig.metadata?.get_models_url) {
// Pull /models from remote models endpoint // Pull /models from remote models endpoint
return this.populateRemoteModels(engineConfig) return this.populateRemoteModels(engineConfig)
@ -190,6 +206,7 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
} }
return e return e
}) })
)
) as Promise<{ messages: string }> ) as Promise<{ messages: string }>
} }
@ -199,9 +216,11 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
*/ */
async uninstallEngine(name: InferenceEngine, engineConfig: EngineConfig) { async uninstallEngine(name: InferenceEngine, engineConfig: EngineConfig) {
return this.queue.add(() => return this.queue.add(() =>
this.api this.apiInstance().then((api) =>
.delete(`v1/engines/${name}/install`, { json: engineConfig }) api
.then((e) => e) .delete(`v1/engines/${name}/install`, { json: engineConfig })
.then((e) => e)
)
) as Promise<{ messages: string }> ) as Promise<{ messages: string }>
} }
@ -210,25 +229,27 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
* @param model - Remote model object. * @param model - Remote model object.
*/ */
async addRemoteModel(model: Model) { async addRemoteModel(model: Model) {
return this.queue return this.queue.add(() =>
.add(() => this.apiInstance()
this.api .then((api) =>
.post('v1/models/add', { api
json: { .post('v1/models/add', {
inference_params: { json: {
max_tokens: 4096, inference_params: {
temperature: 0.7, max_tokens: 4096,
top_p: 0.95, temperature: 0.7,
stream: true, top_p: 0.95,
frequency_penalty: 0, stream: true,
presence_penalty: 0, frequency_penalty: 0,
presence_penalty: 0,
},
...model,
}, },
...model, })
}, .then((e) => e)
}) )
.then((e) => e) .then(() => {})
) )
.then(() => {})
} }
/** /**
@ -237,10 +258,12 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
*/ */
async getDefaultEngineVariant(name: InferenceEngine) { async getDefaultEngineVariant(name: InferenceEngine) {
return this.queue.add(() => return this.queue.add(() =>
this.api this.apiInstance().then((api) =>
.get(`v1/engines/${name}/default`) api
.json<{ messages: string }>() .get(`v1/engines/${name}/default`)
.then((e) => e) .json<{ messages: string }>()
.then((e) => e)
)
) as Promise<DefaultEngineVariant> ) as Promise<DefaultEngineVariant>
} }
@ -254,9 +277,11 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
engineConfig: EngineConfig engineConfig: EngineConfig
) { ) {
return this.queue.add(() => return this.queue.add(() =>
this.api this.apiInstance().then((api) =>
.post(`v1/engines/${name}/default`, { json: engineConfig }) api
.then((e) => e) .post(`v1/engines/${name}/default`, { json: engineConfig })
.then((e) => e)
)
) as Promise<{ messages: string }> ) as Promise<{ messages: string }>
} }
@ -265,9 +290,11 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
*/ */
async updateEngine(name: InferenceEngine, engineConfig?: EngineConfig) { async updateEngine(name: InferenceEngine, engineConfig?: EngineConfig) {
return this.queue.add(() => return this.queue.add(() =>
this.api this.apiInstance().then((api) =>
.post(`v1/engines/${name}/update`, { json: engineConfig }) api
.then((e) => e) .post(`v1/engines/${name}/update`, { json: engineConfig })
.then((e) => e)
)
) as Promise<{ messages: string }> ) as Promise<{ messages: string }>
} }
@ -276,10 +303,12 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
* @returns * @returns
*/ */
async healthz(): Promise<void> { async healthz(): Promise<void> {
return this.api return this.apiInstance()
.get('healthz', { .then((api) =>
retry: { limit: 20, delay: () => 500, methods: ['get'] }, api.get('healthz', {
}) retry: { limit: 20, delay: () => 500, methods: ['get'] },
})
)
.then(() => { .then(() => {
this.queue.concurrency = Infinity this.queue.concurrency = Infinity
}) })

5
extensions/hardware-management-extension/src/index.ts
View File

@ -17,18 +17,21 @@ export default class JSONHardwareManagementExtension extends HardwareManagementE
this.queue.add(() => this.healthz()) this.queue.add(() => this.healthz())
} }
api?: KyInstance
/** /**
* Get the API instance * Get the API instance
* @returns * @returns
*/ */
async apiInstance(): Promise<KyInstance> { async apiInstance(): Promise<KyInstance> {
if(this.api) return this.api
const apiKey = (await window.core?.api.appToken()) ?? 'cortex.cpp' const apiKey = (await window.core?.api.appToken()) ?? 'cortex.cpp'
return ky.extend({ this.api = ky.extend({
prefixUrl: API_URL, prefixUrl: API_URL,
headers: { headers: {
Authorization: `Bearer ${apiKey}`, Authorization: `Bearer ${apiKey}`,
}, },
}) })
return this.api
} }
/** /**

134
extensions/inference-cortex-extension/src/index.ts
View File

@ -75,7 +75,22 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
abortControllers = new Map<string, AbortController>() abortControllers = new Map<string, AbortController>()
api!: KyInstance api?: KyInstance
/**
* Get the API instance
* @returns
*/
async apiInstance(): Promise<KyInstance> {
if(this.api) return this.api
const apiKey = (await window.core?.api.appToken()) ?? 'cortex.cpp'
this.api = ky.extend({
prefixUrl: CORTEX_API_URL,
headers: {
Authorization: `Bearer ${apiKey}`,
},
})
return this.api
}
/** /**
* Authorization headers for the API requests. * Authorization headers for the API requests.
@ -92,13 +107,6 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
*/ */
async onLoad() { async onLoad() {
super.onLoad() super.onLoad()
const apiKey = (await window.core?.api.appToken()) ?? 'cortex.cpp'
this.api = ky.extend({
prefixUrl: CORTEX_API_URL,
headers: {
Authorization: `Bearer ${apiKey}`,
},
})
// Register Settings // Register Settings
this.registerSettings(SETTINGS) this.registerSettings(SETTINGS)
@ -172,45 +180,49 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
this.abortControllers.set(model.id, controller) this.abortControllers.set(model.id, controller)
return await this.queue.add(() => return await this.queue.add(() =>
this.api this.apiInstance().then((api) =>
.post('v1/models/start', { api
json: { .post('v1/models/start', {
...extractModelLoadParams(model.settings), json: {
model: model.id, ...extractModelLoadParams(model.settings),
engine: model: model.id,
model.engine === InferenceEngine.nitro // Legacy model cache engine:
? InferenceEngine.cortex_llamacpp model.engine === InferenceEngine.nitro // Legacy model cache
: model.engine, ? InferenceEngine.cortex_llamacpp
cont_batching: this.cont_batching, : model.engine,
n_parallel: this.n_parallel, cont_batching: this.cont_batching,
caching_enabled: this.caching_enabled, n_parallel: this.n_parallel,
flash_attn: this.flash_attn, caching_enabled: this.caching_enabled,
cache_type: this.cache_type, flash_attn: this.flash_attn,
use_mmap: this.use_mmap, cache_type: this.cache_type,
...(this.cpu_threads ? { cpu_threads: this.cpu_threads } : {}), use_mmap: this.use_mmap,
}, ...(this.cpu_threads ? { cpu_threads: this.cpu_threads } : {}),
timeout: false, },
signal, timeout: false,
}) signal,
.json() })
.catch(async (e) => { .json()
throw (await e.response?.json()) ?? e .catch(async (e) => {
}) throw (await e.response?.json()) ?? e
.finally(() => this.abortControllers.delete(model.id)) })
.then() .finally(() => this.abortControllers.delete(model.id))
.then()
)
) )
} }
override async unloadModel(model: Model): Promise<void> { override async unloadModel(model: Model): Promise<void> {
return this.api return this.apiInstance().then((api) =>
.post('v1/models/stop', { api
json: { model: model.id }, .post('v1/models/stop', {
}) json: { model: model.id },
.json() })
.finally(() => { .json()
this.abortControllers.get(model.id)?.abort() .finally(() => {
}) this.abortControllers.get(model.id)?.abort()
.then() })
.then()
)
} }
/** /**
@ -218,15 +230,17 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
* @returns * @returns
*/ */
private async healthz(): Promise<void> { private async healthz(): Promise<void> {
return this.api return this.apiInstance().then((api) =>
.get('healthz', { api
retry: { .get('healthz', {
limit: 20, retry: {
delay: () => 500, limit: 20,
methods: ['get'], delay: () => 500,
}, methods: ['get'],
}) },
.then(() => {}) })
.then(() => {})
)
} }
/** /**
@ -234,13 +248,15 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
* @returns * @returns
*/ */
private async clean(): Promise<any> { private async clean(): Promise<any> {
return this.api return this.apiInstance()
.delete('processmanager/destroy', { .then((api) =>
timeout: 2000, // maximum 2 seconds api.delete('processmanager/destroy', {
retry: { timeout: 2000, // maximum 2 seconds
limit: 0, retry: {
}, limit: 0,
}) },
})
)
.catch(() => { .catch(() => {
// Do nothing // Do nothing
}) })

185
extensions/model-extension/src/index.ts
View File

@ -33,15 +33,13 @@ type Data<T> = {
export default class JanModelExtension extends ModelExtension { export default class JanModelExtension extends ModelExtension {
queue = new PQueue({ concurrency: 1 }) queue = new PQueue({ concurrency: 1 })
api?: KyInstance
/** /**
* Extended API instance for making requests to the Cortex API. * Get the API instance
* @returns * @returns
*/ */
api: KyInstance async apiInstance(): Promise<KyInstance> {
/** if(this.api) return this.api
* Called when the extension is loaded.
*/
async onLoad() {
const apiKey = (await window.core?.api.appToken()) ?? 'cortex.cpp' const apiKey = (await window.core?.api.appToken()) ?? 'cortex.cpp'
this.api = ky.extend({ this.api = ky.extend({
prefixUrl: CORTEX_API_URL, prefixUrl: CORTEX_API_URL,
@ -49,6 +47,12 @@ export default class JanModelExtension extends ModelExtension {
Authorization: `Bearer ${apiKey}`, Authorization: `Bearer ${apiKey}`,
}, },
}) })
return this.api
}
/**
* Called when the extension is loaded.
*/
async onLoad() {
this.queue.add(() => this.healthz()) this.queue.add(() => this.healthz())
this.registerSettings(SETTINGS) this.registerSettings(SETTINGS)
@ -94,13 +98,15 @@ export default class JanModelExtension extends ModelExtension {
* Sending POST to /models/pull/{id} endpoint to pull the model * Sending POST to /models/pull/{id} endpoint to pull the model
*/ */
return this.queue.add(() => return this.queue.add(() =>
this.api this.apiInstance().then((api) =>
.post('v1/models/pull', { json: { model, id, name }, timeout: false }) api
.json() .post('v1/models/pull', { json: { model, id, name }, timeout: false })
.catch(async (e) => { .json()
throw (await e.response?.json()) ?? e .catch(async (e) => {
}) throw (await e.response?.json()) ?? e
.then() })
.then()
)
) )
} }
@ -115,10 +121,12 @@ export default class JanModelExtension extends ModelExtension {
* Sending DELETE to /models/pull/{id} endpoint to cancel a model pull * Sending DELETE to /models/pull/{id} endpoint to cancel a model pull
*/ */
return this.queue.add(() => return this.queue.add(() =>
this.api this.apiInstance().then((api) =>
.delete('v1/models/pull', { json: { taskId: model } }) api
.json() .delete('v1/models/pull', { json: { taskId: model } })
.then() .json()
.then()
)
) )
} }
@ -129,7 +137,11 @@ export default class JanModelExtension extends ModelExtension {
*/ */
async deleteModel(model: string): Promise<void> { async deleteModel(model: string): Promise<void> {
return this.queue return this.queue
.add(() => this.api.delete(`v1/models/${model}`).json().then()) .add(() =>
this.apiInstance().then((api) =>
api.delete(`v1/models/${model}`).json().then()
)
)
.catch((e) => console.debug(e)) .catch((e) => console.debug(e))
.finally(async () => { .finally(async () => {
// Delete legacy model files // Delete legacy model files
@ -231,13 +243,15 @@ export default class JanModelExtension extends ModelExtension {
async updateModel(model: Partial<Model>): Promise<Model> { async updateModel(model: Partial<Model>): Promise<Model> {
return this.queue return this.queue
.add(() => .add(() =>
this.api this.apiInstance().then((api) =>
.patch(`v1/models/${model.id}`, { api
json: { ...model }, .patch(`v1/models/${model.id}`, {
timeout: false, json: { ...model },
}) timeout: false,
.json() })
.then() .json()
.then()
)
) )
.then(() => this.getModel(model.id)) .then(() => this.getModel(model.id))
} }
@ -248,10 +262,12 @@ export default class JanModelExtension extends ModelExtension {
*/ */
async getModel(model: string): Promise<Model> { async getModel(model: string): Promise<Model> {
return this.queue.add(() => return this.queue.add(() =>
this.api this.apiInstance().then((api) =>
.get(`v1/models/${model}`) api
.json() .get(`v1/models/${model}`)
.then((e) => this.transformModel(e)) .json()
.then((e) => this.transformModel(e))
)
) as Promise<Model> ) as Promise<Model>
} }
@ -267,14 +283,16 @@ export default class JanModelExtension extends ModelExtension {
option?: OptionType option?: OptionType
): Promise<void> { ): Promise<void> {
return this.queue.add(() => return this.queue.add(() =>
this.api this.apiInstance().then((api) =>
.post('v1/models/import', { api
json: { model, modelPath, name, option }, .post('v1/models/import', {
timeout: false, json: { model, modelPath, name, option },
}) timeout: false,
.json() })
.catch((e) => console.debug(e)) // Ignore error .json()
.then() .catch((e) => console.debug(e)) // Ignore error
.then()
)
) )
} }
@ -285,7 +303,11 @@ export default class JanModelExtension extends ModelExtension {
*/ */
async getSources(): Promise<ModelSource[]> { async getSources(): Promise<ModelSource[]> {
const sources = await this.queue const sources = await this.queue
.add(() => this.api.get('v1/models/sources').json<Data<ModelSource>>()) .add(() =>
this.apiInstance().then((api) =>
api.get('v1/models/sources').json<Data<ModelSource>>()
)
)
.then((e) => (typeof e === 'object' ? (e.data as ModelSource[]) : [])) .then((e) => (typeof e === 'object' ? (e.data as ModelSource[]) : []))
.catch(() => []) .catch(() => [])
return sources.concat( return sources.concat(
@ -299,11 +321,13 @@ export default class JanModelExtension extends ModelExtension {
*/ */
async addSource(source: string): Promise<any> { async addSource(source: string): Promise<any> {
return this.queue.add(() => return this.queue.add(() =>
this.api.post('v1/models/sources', { this.apiInstance().then((api) =>
json: { api.post('v1/models/sources', {
source, json: {
}, source,
}) },
})
)
) )
} }
@ -313,12 +337,14 @@ export default class JanModelExtension extends ModelExtension {
*/ */
async deleteSource(source: string): Promise<any> { async deleteSource(source: string): Promise<any> {
return this.queue.add(() => return this.queue.add(() =>
this.api.delete('v1/models/sources', { this.apiInstance().then((api) =>
json: { api.delete('v1/models/sources', {
source, json: {
}, source,
timeout: false, },
}) timeout: false,
})
)
) )
} }
// END - Model Sources // END - Model Sources
@ -329,7 +355,9 @@ export default class JanModelExtension extends ModelExtension {
*/ */
async isModelLoaded(model: string): Promise<boolean> { async isModelLoaded(model: string): Promise<boolean> {
return this.queue return this.queue
.add(() => this.api.get(`v1/models/status/${model}`)) .add(() =>
this.apiInstance().then((api) => api.get(`v1/models/status/${model}`))
)
.then((e) => true) .then((e) => true)
.catch(() => false) .catch(() => false)
} }
@ -348,7 +376,11 @@ export default class JanModelExtension extends ModelExtension {
*/ */
async fetchModels(): Promise<Model[]> { async fetchModels(): Promise<Model[]> {
return this.queue return this.queue
.add(() => this.api.get('v1/models?limit=-1').json<Data<Model>>()) .add(() =>
this.apiInstance().then((api) =>
api.get('v1/models?limit=-1').json<Data<Model>>()
)
)
.then((e) => .then((e) =>
typeof e === 'object' ? e.data.map((e) => this.transformModel(e)) : [] typeof e === 'object' ? e.data.map((e) => this.transformModel(e)) : []
) )
@ -387,7 +419,11 @@ export default class JanModelExtension extends ModelExtension {
[key: string]: any [key: string]: any
}): Promise<void> { }): Promise<void> {
return this.queue return this.queue
.add(() => this.api.patch('v1/configs', { json: body }).then(() => {})) .add(() =>
this.apiInstance().then((api) =>
api.patch('v1/configs', { json: body }).then(() => {})
)
)
.catch((e) => console.debug(e)) .catch((e) => console.debug(e))
} }
@ -396,14 +432,16 @@ export default class JanModelExtension extends ModelExtension {
* @returns * @returns
*/ */
private healthz(): Promise<void> { private healthz(): Promise<void> {
return this.api return this.apiInstance()
.get('healthz', { .then((api) =>
retry: { api.get('healthz', {
limit: 20, retry: {
delay: () => 500, limit: 20,
methods: ['get'], delay: () => 500,
}, methods: ['get'],
}) },
})
)
.then(() => { .then(() => {
this.queue.concurrency = Infinity this.queue.concurrency = Infinity
}) })
@ -416,17 +454,22 @@ export default class JanModelExtension extends ModelExtension {
const models = await this.fetchModels() const models = await this.fetchModels()
return this.queue.add(() => return this.queue.add(() =>
this.api this.apiInstance()
.get('v1/models/hub?author=cortexso&tag=cortex.cpp') .then((api) =>
.json<Data<string>>() api
.then((e) => { .get('v1/models/hub?author=cortexso&tag=cortex.cpp')
e.data?.forEach((model) => { .json<Data<string>>()
if ( .then((e) => {
!models.some((e) => 'modelSource' in e && e.modelSource === model) e.data?.forEach((model) => {
) if (
this.addSource(model).catch((e) => console.debug(e)) !models.some(
}) (e) => 'modelSource' in e && e.modelSource === model
}) )
)
this.addSource(model).catch((e) => console.debug(e))
})
})
)
.catch((e) => console.debug(e)) .catch((e) => console.debug(e))
) )
} }

10
web/screens/Settings/Hardware/index.tsx
View File

@ -6,13 +6,14 @@ import { useState } from 'react'
import { DragDropContext, Draggable, Droppable } from '@hello-pangea/dnd' import { DragDropContext, Draggable, Droppable } from '@hello-pangea/dnd'
import { Progress, ScrollArea, Switch } from '@janhq/joi' import { Progress, ScrollArea, Switch } from '@janhq/joi'
import { useAtom, useAtomValue } from 'jotai' import { useAtom, useAtomValue, useSetAtom } from 'jotai'
import { atomWithStorage } from 'jotai/utils' import { atomWithStorage } from 'jotai/utils'
import { ChevronDownIcon, GripVerticalIcon } from 'lucide-react' import { ChevronDownIcon, GripVerticalIcon } from 'lucide-react'
import { twMerge } from 'tailwind-merge' import { twMerge } from 'tailwind-merge'
import { activeModelAtom } from '@/hooks/useActiveModel'
import { import {
useGetHardwareInfo, useGetHardwareInfo,
setActiveGpus, setActiveGpus,
@ -47,6 +48,7 @@ const Hardware = () => {
const ramUtilitized = useAtomValue(ramUtilitizedAtom) const ramUtilitized = useAtomValue(ramUtilitizedAtom)
const showScrollBar = useAtomValue(showScrollBarAtom) const showScrollBar = useAtomValue(showScrollBarAtom)
const [gpus, setGpus] = useAtom(gpusAtom) const [gpus, setGpus] = useAtom(gpusAtom)
const setActiveModel = useSetAtom(activeModelAtom)
const [orderGpus, setOrderGpus] = useAtom(orderGpusAtom) const [orderGpus, setOrderGpus] = useAtom(orderGpusAtom)
@ -70,11 +72,15 @@ const Hardware = () => {
.filter((gpu: any) => gpu.activated) .filter((gpu: any) => gpu.activated)
.map((gpu: any) => Number(gpu.id)) .map((gpu: any) => Number(gpu.id))
await setActiveGpus({ gpus: activeGpuIds }) await setActiveGpus({ gpus: activeGpuIds })
setActiveModel(undefined)
mutate() mutate()
window.location.reload()
} catch (error) { } catch (error) {
console.error('Failed to update active GPUs:', error) console.error('Failed to update active GPUs:', error)
} }
setIsActivatingGpu((prev) => {
prev.delete(id)
return new Set(prev)
})
} }
const handleDragEnd = (result: any) => { const handleDragEnd = (result: any) => {