Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: frontend uses new engine extension

Browse Source 
# Conflicts:
#	extensions/model-extension/resources/default.json
#	web-app/src/containers/dialogs/DeleteProvider.tsx
#	web-app/src/routes/hub.tsx
This commit is contained in:
Louis 2025-06-23 13:06:29 +07:00
parent ad06b2a903
commit 8bd4a3389f
No known key found for this signature in database
GPG Key ID: 44FA9F4D33C37DE2
84 changed files with 291 additions and 12170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

104
core/src/browser/extensions/engines/AIEngine.ts
View File

@ -4,72 +4,72 @@ import { EngineManager } from './EngineManager'
/* AIEngine class types */ /* AIEngine class types */
export interface chatCompletionRequestMessage { export interface chatCompletionRequestMessage {
role: 'system' | 'user' | 'assistant' | 'tool'; role: 'system' | 'user' | 'assistant' | 'tool'
content: string | null | Content[]; // Content can be a string OR an array of content parts content: string | null | Content[] // Content can be a string OR an array of content parts
name?: string; name?: string
tool_calls?: any[]; // Simplified tool_call_id?: string tool_calls?: any[] // Simplified tool_call_id?: string
} }
export interface Content { export interface Content {
type: 'text' | 'input_image' | 'input_audio'; type: 'text' | 'input_image' | 'input_audio'
text?: string; text?: string
image_url?: string; image_url?: string
input_audio?: InputAudio; input_audio?: InputAudio
} }
export interface InputAudio { export interface InputAudio {
data: string; // Base64 encoded audio data data: string // Base64 encoded audio data
format: 'mp3' | 'wav' | 'ogg' | 'flac'; // Add more formats as needed/llama-server seems to support mp3 format: 'mp3' | 'wav' | 'ogg' | 'flac' // Add more formats as needed/llama-server seems to support mp3
} }
export interface chatCompletionRequest { export interface chatCompletionRequest {
model: string; // Model ID, though for local it might be implicit via sessionInfo model: string // Model ID, though for local it might be implicit via sessionInfo
messages: chatCompletionRequestMessage[]; messages: chatCompletionRequestMessage[]
// Core sampling parameters // Core sampling parameters
temperature?: number | null; temperature?: number | null
dynatemp_range?: number | null; dynatemp_range?: number | null
dynatemp_exponent?: number | null; dynatemp_exponent?: number | null
top_k?: number | null; top_k?: number | null
top_p?: number | null; top_p?: number | null
min_p?: number | null; min_p?: number | null
typical_p?: number | null; typical_p?: number | null
repeat_penalty?: number | null; repeat_penalty?: number | null
repeat_last_n?: number | null; repeat_last_n?: number | null
presence_penalty?: number | null; presence_penalty?: number | null
frequency_penalty?: number | null; frequency_penalty?: number | null
dry_multiplier?: number | null; dry_multiplier?: number | null
dry_base?: number | null; dry_base?: number | null
dry_allowed_length?: number | null; dry_allowed_length?: number | null
dry_penalty_last_n?: number | null; dry_penalty_last_n?: number | null
dry_sequence_breakers?: string[] | null; dry_sequence_breakers?: string[] | null
xtc_probability?: number | null; xtc_probability?: number | null
xtc_threshold?: number | null; xtc_threshold?: number | null
mirostat?: number | null; // 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0 mirostat?: number | null // 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0
mirostat_tau?: number | null; mirostat_tau?: number | null
mirostat_eta?: number | null; mirostat_eta?: number | null
n_predict?: number | null; n_predict?: number | null
n_indent?: number | null; n_indent?: number | null
n_keep?: number | null; n_keep?: number | null
stream?: boolean | null; stream?: boolean | null
stop?: string | string[] | null; stop?: string | string[] | null
seed?: number | null; // RNG seed seed?: number | null // RNG seed
// Advanced sampling // Advanced sampling
logit_bias?: { [key: string]: number } | null; logit_bias?: { [key: string]: number } | null
n_probs?: number | null; n_probs?: number | null
min_keep?: number | null; min_keep?: number | null
t_max_predict_ms?: number | null; t_max_predict_ms?: number | null
image_data?: Array<{ data: string; id: number }> | null; image_data?: Array<{ data: string; id: number }> | null
// Internal/optimization parameters // Internal/optimization parameters
id_slot?: number | null; id_slot?: number | null
cache_prompt?: boolean | null; cache_prompt?: boolean | null
return_tokens?: boolean | null; return_tokens?: boolean | null
samplers?: string[] | null; samplers?: string[] | null
timings_per_token?: boolean | null; timings_per_token?: boolean | null
post_sampling_probs?: boolean | null; post_sampling_probs?: boolean | null
} }
export interface chatCompletionChunkChoiceDelta { export interface chatCompletionChunkChoiceDelta {
@ -208,7 +208,9 @@ export abstract class AIEngine extends BaseExtension {
/** /**
* Sends a chat request to the model * Sends a chat request to the model
*/ */
abstract chat(opts: chatCompletionRequest): Promise<chatCompletion | AsyncIterable<chatCompletionChunk>> abstract chat(
opts: chatCompletionRequest
): Promise<chatCompletion | AsyncIterable<chatCompletionChunk>>
/** /**
* Deletes a model * Deletes a model

566
core/src/browser/extensions/enginesManagement.test.ts
View File

@ -1,566 +0,0 @@
import { EngineManagementExtension } from './enginesManagement'
import { ExtensionTypeEnum } from '../extension'
import {
EngineConfig,
EngineReleased,
EngineVariant,
Engines,
InferenceEngine,
DefaultEngineVariant,
Model
} from '../../types'
// Mock implementation of EngineManagementExtension
class MockEngineManagementExtension extends EngineManagementExtension {
private mockEngines: Engines = {
llama: {
name: 'llama',
variants: [
{
variant: 'cpu',
version: '1.0.0',
path: '/engines/llama/cpu/1.0.0',
installed: true
},
{
variant: 'cuda',
version: '1.0.0',
path: '/engines/llama/cuda/1.0.0',
installed: false
}
],
default: {
variant: 'cpu',
version: '1.0.0'
}
},
gpt4all: {
name: 'gpt4all',
variants: [
{
variant: 'cpu',
version: '2.0.0',
path: '/engines/gpt4all/cpu/2.0.0',
installed: true
}
],
default: {
variant: 'cpu',
version: '2.0.0'
}
}
}
private mockReleases: { [key: string]: EngineReleased[] } = {
'llama-1.0.0': [
{
variant: 'cpu',
version: '1.0.0',
os: ['macos', 'linux', 'windows'],
url: 'https://example.com/llama/1.0.0/cpu'
},
{
variant: 'cuda',
version: '1.0.0',
os: ['linux', 'windows'],
url: 'https://example.com/llama/1.0.0/cuda'
}
],
'llama-1.1.0': [
{
variant: 'cpu',
version: '1.1.0',
os: ['macos', 'linux', 'windows'],
url: 'https://example.com/llama/1.1.0/cpu'
},
{
variant: 'cuda',
version: '1.1.0',
os: ['linux', 'windows'],
url: 'https://example.com/llama/1.1.0/cuda'
}
],
'gpt4all-2.0.0': [
{
variant: 'cpu',
version: '2.0.0',
os: ['macos', 'linux', 'windows'],
url: 'https://example.com/gpt4all/2.0.0/cpu'
}
]
}
private remoteModels: { [engine: string]: Model[] } = {
'llama': [],
'gpt4all': []
}
constructor() {
super('http://mock-url.com', 'mock-engine-extension', 'Mock Engine Extension', true, 'A mock engine extension', '1.0.0')
}
onLoad(): void {
// Mock implementation
}
onUnload(): void {
// Mock implementation
}
async getEngines(): Promise<Engines> {
return JSON.parse(JSON.stringify(this.mockEngines))
}
async getInstalledEngines(name: InferenceEngine): Promise<EngineVariant[]> {
if (!this.mockEngines[name]) {
return []
}
return this.mockEngines[name].variants.filter(variant => variant.installed)
}
async getReleasedEnginesByVersion(
name: InferenceEngine,
version: string,
platform?: string
): Promise<EngineReleased[]> {
const key = `${name}-${version}`
let releases = this.mockReleases[key] || []
if (platform) {
releases = releases.filter(release => release.os.includes(platform))
}
return releases
}
async getLatestReleasedEngine(
name: InferenceEngine,
platform?: string
): Promise<EngineReleased[]> {
// For mock, let's assume latest versions are 1.1.0 for llama and 2.0.0 for gpt4all
const latestVersions = {
'llama': '1.1.0',
'gpt4all': '2.0.0'
}
if (!latestVersions[name]) {
return []
}
return this.getReleasedEnginesByVersion(name, latestVersions[name], platform)
}
async installEngine(
name: string,
engineConfig: EngineConfig
): Promise<{ messages: string }> {
if (!this.mockEngines[name]) {
this.mockEngines[name] = {
name,
variants: [],
default: {
variant: engineConfig.variant,
version: engineConfig.version
}
}
}
// Check if variant already exists
const existingVariantIndex = this.mockEngines[name].variants.findIndex(
v => v.variant === engineConfig.variant && v.version === engineConfig.version
)
if (existingVariantIndex >= 0) {
this.mockEngines[name].variants[existingVariantIndex].installed = true
} else {
this.mockEngines[name].variants.push({
variant: engineConfig.variant,
version: engineConfig.version,
path: `/engines/${name}/${engineConfig.variant}/${engineConfig.version}`,
installed: true
})
}
return { messages: `Successfully installed ${name} ${engineConfig.variant} ${engineConfig.version}` }
}
async addRemoteEngine(
engineConfig: EngineConfig
): Promise<{ messages: string }> {
const name = engineConfig.name || 'remote-engine'
if (!this.mockEngines[name]) {
this.mockEngines[name] = {
name,
variants: [],
default: {
variant: engineConfig.variant,
version: engineConfig.version
}
}
}
this.mockEngines[name].variants.push({
variant: engineConfig.variant,
version: engineConfig.version,
path: engineConfig.path || `/engines/${name}/${engineConfig.variant}/${engineConfig.version}`,
installed: true,
url: engineConfig.url
})
return { messages: `Successfully added remote engine ${name}` }
}
async uninstallEngine(
name: InferenceEngine,
engineConfig: EngineConfig
): Promise<{ messages: string }> {
if (!this.mockEngines[name]) {
return { messages: `Engine ${name} not found` }
}
const variantIndex = this.mockEngines[name].variants.findIndex(
v => v.variant === engineConfig.variant && v.version === engineConfig.version
)
if (variantIndex >= 0) {
this.mockEngines[name].variants[variantIndex].installed = false
// If this was the default variant, reset default
if (
this.mockEngines[name].default.variant === engineConfig.variant &&
this.mockEngines[name].default.version === engineConfig.version
) {
// Find another installed variant to set as default
const installedVariant = this.mockEngines[name].variants.find(v => v.installed)
if (installedVariant) {
this.mockEngines[name].default = {
variant: installedVariant.variant,
version: installedVariant.version
}
} else {
// No installed variants remain, clear default
this.mockEngines[name].default = { variant: '', version: '' }
}
}
return { messages: `Successfully uninstalled ${name} ${engineConfig.variant} ${engineConfig.version}` }
} else {
return { messages: `Variant ${engineConfig.variant} ${engineConfig.version} not found for engine ${name}` }
}
}
async getDefaultEngineVariant(
name: InferenceEngine
): Promise<DefaultEngineVariant> {
if (!this.mockEngines[name]) {
return { variant: '', version: '' }
}
return this.mockEngines[name].default
}
async setDefaultEngineVariant(
name: InferenceEngine,
engineConfig: EngineConfig
): Promise<{ messages: string }> {
if (!this.mockEngines[name]) {
return { messages: `Engine ${name} not found` }
}
const variantExists = this.mockEngines[name].variants.some(
v => v.variant === engineConfig.variant && v.version === engineConfig.version && v.installed
)
if (!variantExists) {
return { messages: `Variant ${engineConfig.variant} ${engineConfig.version} not found or not installed` }
}
this.mockEngines[name].default = {
variant: engineConfig.variant,
version: engineConfig.version
}
return { messages: `Successfully set ${engineConfig.variant} ${engineConfig.version} as default for ${name}` }
}
async updateEngine(
name: InferenceEngine,
engineConfig?: EngineConfig
): Promise<{ messages: string }> {
if (!this.mockEngines[name]) {
return { messages: `Engine ${name} not found` }
}
if (!engineConfig) {
// Assume we're updating to the latest version
return { messages: `Successfully updated ${name} to the latest version` }
}
const variantIndex = this.mockEngines[name].variants.findIndex(
v => v.variant === engineConfig.variant && v.installed
)
if (variantIndex >= 0) {
// Update the version
this.mockEngines[name].variants[variantIndex].version = engineConfig.version
// If this was the default variant, update default version too
if (this.mockEngines[name].default.variant === engineConfig.variant) {
this.mockEngines[name].default.version = engineConfig.version
}
return { messages: `Successfully updated ${name} ${engineConfig.variant} to version ${engineConfig.version}` }
} else {
return { messages: `Installed variant ${engineConfig.variant} not found for engine ${name}` }
}
}
async addRemoteModel(model: Model): Promise<void> {
const engine = model.engine as string
if (!this.remoteModels[engine]) {
this.remoteModels[engine] = []
}
this.remoteModels[engine].push(model)
}
async getRemoteModels(name: InferenceEngine | string): Promise<Model[]> {
return this.remoteModels[name] || []
}
}
describe('EngineManagementExtension', () => {
let extension: MockEngineManagementExtension
beforeEach(() => {
extension = new MockEngineManagementExtension()
})
test('should return the correct extension type', () => {
expect(extension.type()).toBe(ExtensionTypeEnum.Engine)
})
test('should get all engines', async () => {
const engines = await extension.getEngines()
expect(engines).toBeDefined()
expect(engines.llama).toBeDefined()
expect(engines.gpt4all).toBeDefined()
expect(engines.llama.variants).toHaveLength(2)
expect(engines.gpt4all.variants).toHaveLength(1)
})
test('should get installed engines', async () => {
const llamaEngines = await extension.getInstalledEngines('llama')
expect(llamaEngines).toHaveLength(1)
expect(llamaEngines[0].variant).toBe('cpu')
expect(llamaEngines[0].installed).toBe(true)
const gpt4allEngines = await extension.getInstalledEngines('gpt4all')
expect(gpt4allEngines).toHaveLength(1)
expect(gpt4allEngines[0].variant).toBe('cpu')
expect(gpt4allEngines[0].installed).toBe(true)
// Test non-existent engine
const nonExistentEngines = await extension.getInstalledEngines('non-existent' as InferenceEngine)
expect(nonExistentEngines).toHaveLength(0)
})
test('should get released engines by version', async () => {
const llamaReleases = await extension.getReleasedEnginesByVersion('llama', '1.0.0')
expect(llamaReleases).toHaveLength(2)
expect(llamaReleases[0].variant).toBe('cpu')
expect(llamaReleases[1].variant).toBe('cuda')
// Test with platform filter
const llamaLinuxReleases = await extension.getReleasedEnginesByVersion('llama', '1.0.0', 'linux')
expect(llamaLinuxReleases).toHaveLength(2)
const llamaMacReleases = await extension.getReleasedEnginesByVersion('llama', '1.0.0', 'macos')
expect(llamaMacReleases).toHaveLength(1)
expect(llamaMacReleases[0].variant).toBe('cpu')
// Test non-existent version
const nonExistentReleases = await extension.getReleasedEnginesByVersion('llama', '9.9.9')
expect(nonExistentReleases).toHaveLength(0)
})
test('should get latest released engines', async () => {
const latestLlamaReleases = await extension.getLatestReleasedEngine('llama')
expect(latestLlamaReleases).toHaveLength(2)
expect(latestLlamaReleases[0].version).toBe('1.1.0')
// Test with platform filter
const latestLlamaMacReleases = await extension.getLatestReleasedEngine('llama', 'macos')
expect(latestLlamaMacReleases).toHaveLength(1)
expect(latestLlamaMacReleases[0].variant).toBe('cpu')
expect(latestLlamaMacReleases[0].version).toBe('1.1.0')
// Test non-existent engine
const nonExistentReleases = await extension.getLatestReleasedEngine('non-existent' as InferenceEngine)
expect(nonExistentReleases).toHaveLength(0)
})
test('should install engine', async () => {
// Install existing engine variant that is not installed
const result = await extension.installEngine('llama', { variant: 'cuda', version: '1.0.0' })
expect(result.messages).toContain('Successfully installed')
const installedEngines = await extension.getInstalledEngines('llama')
expect(installedEngines).toHaveLength(2)
expect(installedEngines.some(e => e.variant === 'cuda')).toBe(true)
// Install non-existent engine
const newEngineResult = await extension.installEngine('new-engine', { variant: 'cpu', version: '1.0.0' })
expect(newEngineResult.messages).toContain('Successfully installed')
const engines = await extension.getEngines()
expect(engines['new-engine']).toBeDefined()
expect(engines['new-engine'].variants).toHaveLength(1)
expect(engines['new-engine'].variants[0].installed).toBe(true)
})
test('should add remote engine', async () => {
const result = await extension.addRemoteEngine({
name: 'remote-llm',
variant: 'remote',
version: '1.0.0',
url: 'https://example.com/remote-llm-api'
})
expect(result.messages).toContain('Successfully added remote engine')
const engines = await extension.getEngines()
expect(engines['remote-llm']).toBeDefined()
expect(engines['remote-llm'].variants).toHaveLength(1)
expect(engines['remote-llm'].variants[0].url).toBe('https://example.com/remote-llm-api')
})
test('should uninstall engine', async () => {
const result = await extension.uninstallEngine('llama', { variant: 'cpu', version: '1.0.0' })
expect(result.messages).toContain('Successfully uninstalled')
const installedEngines = await extension.getInstalledEngines('llama')
expect(installedEngines).toHaveLength(0)
// Test uninstalling non-existent variant
const nonExistentResult = await extension.uninstallEngine('llama', { variant: 'non-existent', version: '1.0.0' })
expect(nonExistentResult.messages).toContain('not found')
})
test('should handle default variant when uninstalling', async () => {
// First install cuda variant
await extension.installEngine('llama', { variant: 'cuda', version: '1.0.0' })
// Set cuda as default
await extension.setDefaultEngineVariant('llama', { variant: 'cuda', version: '1.0.0' })
// Check that cuda is now default
let defaultVariant = await extension.getDefaultEngineVariant('llama')
expect(defaultVariant.variant).toBe('cuda')
// Uninstall cuda
await extension.uninstallEngine('llama', { variant: 'cuda', version: '1.0.0' })
// Check that default has changed to another installed variant
defaultVariant = await extension.getDefaultEngineVariant('llama')
expect(defaultVariant.variant).toBe('cpu')
// Uninstall all variants
await extension.uninstallEngine('llama', { variant: 'cpu', version: '1.0.0' })
// Check that default is now empty
defaultVariant = await extension.getDefaultEngineVariant('llama')
expect(defaultVariant.variant).toBe('')
expect(defaultVariant.version).toBe('')
})
test('should get default engine variant', async () => {
const llamaDefault = await extension.getDefaultEngineVariant('llama')
expect(llamaDefault.variant).toBe('cpu')
expect(llamaDefault.version).toBe('1.0.0')
// Test non-existent engine
const nonExistentDefault = await extension.getDefaultEngineVariant('non-existent' as InferenceEngine)
expect(nonExistentDefault.variant).toBe('')
expect(nonExistentDefault.version).toBe('')
})
test('should set default engine variant', async () => {
// Install cuda variant
await extension.installEngine('llama', { variant: 'cuda', version: '1.0.0' })
const result = await extension.setDefaultEngineVariant('llama', { variant: 'cuda', version: '1.0.0' })
expect(result.messages).toContain('Successfully set')
const defaultVariant = await extension.getDefaultEngineVariant('llama')
expect(defaultVariant.variant).toBe('cuda')
expect(defaultVariant.version).toBe('1.0.0')
// Test setting non-existent variant as default
const nonExistentResult = await extension.setDefaultEngineVariant('llama', { variant: 'non-existent', version: '1.0.0' })
expect(nonExistentResult.messages).toContain('not found')
})
test('should update engine', async () => {
const result = await extension.updateEngine('llama', { variant: 'cpu', version: '1.1.0' })
expect(result.messages).toContain('Successfully updated')
const engines = await extension.getEngines()
const cpuVariant = engines.llama.variants.find(v => v.variant === 'cpu')
expect(cpuVariant).toBeDefined()
expect(cpuVariant?.version).toBe('1.1.0')
// Default should also be updated since cpu was default
expect(engines.llama.default.version).toBe('1.1.0')
// Test updating non-existent variant
const nonExistentResult = await extension.updateEngine('llama', { variant: 'non-existent', version: '1.1.0' })
expect(nonExistentResult.messages).toContain('not found')
})
test('should add and get remote models', async () => {
const model: Model = {
id: 'remote-model-1',
name: 'Remote Model 1',
path: '/path/to/remote-model',
engine: 'llama',
format: 'gguf',
modelFormat: 'gguf',
source: 'remote',
status: 'ready',
contextLength: 4096,
sizeInGB: 4,
created: new Date().toISOString()
}
await extension.addRemoteModel(model)
const llamaModels = await extension.getRemoteModels('llama')
expect(llamaModels).toHaveLength(1)
expect(llamaModels[0].id).toBe('remote-model-1')
// Test non-existent engine
const nonExistentModels = await extension.getRemoteModels('non-existent')
expect(nonExistentModels).toHaveLength(0)
})
})

115
core/src/browser/extensions/enginesManagement.ts
View File

@ -1,115 +0,0 @@
import {
Engines,
EngineVariant,
EngineReleased,
EngineConfig,
DefaultEngineVariant,
Model,
} from '../../types'
import { BaseExtension, ExtensionTypeEnum } from '../extension'
/**
* Engine management extension. Persists and retrieves engine management.
* @abstract
* @extends BaseExtension
*/
export abstract class EngineManagementExtension extends BaseExtension {
type(): ExtensionTypeEnum | undefined {
return ExtensionTypeEnum.Engine
}
/**
* @returns A Promise that resolves to an object of list engines.
*/
abstract getEngines(): Promise<Engines>
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to an array of installed engine.
*/
abstract getInstalledEngines(name: string): Promise<EngineVariant[]>
/**
* @param name - Inference engine name.
* @param version - Version of the engine.
* @param platform - Optional to sort by operating system. macOS, linux, windows.
* @returns A Promise that resolves to an array of latest released engine by version.
*/
abstract getReleasedEnginesByVersion(
name: string,
version: string,
platform?: string
): Promise<EngineReleased[]>
/**
* @param name - Inference engine name.
* @param platform - Optional to sort by operating system. macOS, linux, windows.
* @returns A Promise that resolves to an array of latest released engine.
*/
abstract getLatestReleasedEngine(
name: string,
platform?: string
): Promise<EngineReleased[]>
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to intall of engine.
*/
abstract installEngine(
name: string,
engineConfig: EngineConfig
): Promise<{ messages: string }>
/**
* Add a new remote engine
* @returns A Promise that resolves to intall of engine.
*/
abstract addRemoteEngine(
engineConfig: EngineConfig
): Promise<{ messages: string }>
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to unintall of engine.
*/
abstract uninstallEngine(
name: string,
engineConfig: EngineConfig
): Promise<{ messages: string }>
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to an object of default engine.
*/
abstract getDefaultEngineVariant(
name: string
): Promise<DefaultEngineVariant>
/**
* @body variant - string
* @body version - string
* @returns A Promise that resolves to set default engine.
*/
abstract setDefaultEngineVariant(
name: string,
engineConfig: EngineConfig
): Promise<{ messages: string }>
/**
* @returns A Promise that resolves to update engine.
*/
abstract updateEngine(
name: string,
engineConfig?: EngineConfig
): Promise<{ messages: string }>
/**
* Add a new remote model for a specific engine
*/
abstract addRemoteModel(model: Model): Promise<void>
/**
* @returns A Promise that resolves to an object of remote models list .
*/
abstract getRemoteModels(name: string): Promise<any>
}

146
core/src/browser/extensions/hardwareManagement.test.ts
View File

@ -1,146 +0,0 @@
import { HardwareManagementExtension } from './hardwareManagement'
import { ExtensionTypeEnum } from '../extension'
import { HardwareInformation } from '../../types'
// Mock implementation of HardwareManagementExtension
class MockHardwareManagementExtension extends HardwareManagementExtension {
private activeGpus: number[] = [0]
private mockHardwareInfo: HardwareInformation = {
cpu: {
manufacturer: 'Mock CPU Manufacturer',
brand: 'Mock CPU',
cores: 8,
physicalCores: 4,
speed: 3.5,
},
memory: {
total: 16 * 1024 * 1024 * 1024, // 16GB in bytes
free: 8 * 1024 * 1024 * 1024, // 8GB in bytes
},
gpus: [
{
id: 0,
vendor: 'Mock GPU Vendor',
model: 'Mock GPU Model 1',
memory: 8 * 1024 * 1024 * 1024, // 8GB in bytes
},
{
id: 1,
vendor: 'Mock GPU Vendor',
model: 'Mock GPU Model 2',
memory: 4 * 1024 * 1024 * 1024, // 4GB in bytes
}
],
active_gpus: [0],
}
constructor() {
super('http://mock-url.com', 'mock-hardware-extension', 'Mock Hardware Extension', true, 'A mock hardware extension', '1.0.0')
}
onLoad(): void {
// Mock implementation
}
onUnload(): void {
// Mock implementation
}
async getHardware(): Promise<HardwareInformation> {
// Return a copy to prevent test side effects
return JSON.parse(JSON.stringify(this.mockHardwareInfo))
}
async setAvtiveGpu(data: { gpus: number[] }): Promise<{
message: string
activated_gpus: number[]
}> {
// Validate GPUs exist
const validGpus = data.gpus.filter(gpuId =>
this.mockHardwareInfo.gpus.some(gpu => gpu.id === gpuId)
)
if (validGpus.length === 0) {
throw new Error('No valid GPUs selected')
}
// Update active GPUs
this.activeGpus = validGpus
this.mockHardwareInfo.active_gpus = validGpus
return {
message: 'GPU activation successful',
activated_gpus: validGpus
}
}
}
describe('HardwareManagementExtension', () => {
let extension: MockHardwareManagementExtension
beforeEach(() => {
extension = new MockHardwareManagementExtension()
})
test('should return the correct extension type', () => {
expect(extension.type()).toBe(ExtensionTypeEnum.Hardware)
})
test('should get hardware information', async () => {
const hardwareInfo = await extension.getHardware()
// Check CPU info
expect(hardwareInfo.cpu).toBeDefined()
expect(hardwareInfo.cpu.manufacturer).toBe('Mock CPU Manufacturer')
expect(hardwareInfo.cpu.cores).toBe(8)
// Check memory info
expect(hardwareInfo.memory).toBeDefined()
expect(hardwareInfo.memory.total).toBe(16 * 1024 * 1024 * 1024)
// Check GPU info
expect(hardwareInfo.gpus).toHaveLength(2)
expect(hardwareInfo.gpus[0].model).toBe('Mock GPU Model 1')
expect(hardwareInfo.gpus[1].model).toBe('Mock GPU Model 2')
// Check active GPUs
expect(hardwareInfo.active_gpus).toEqual([0])
})
test('should set active GPUs', async () => {
const result = await extension.setAvtiveGpu({ gpus: [1] })
expect(result.message).toBe('GPU activation successful')
expect(result.activated_gpus).toEqual([1])
// Verify the change in hardware info
const hardwareInfo = await extension.getHardware()
expect(hardwareInfo.active_gpus).toEqual([1])
})
test('should set multiple active GPUs', async () => {
const result = await extension.setAvtiveGpu({ gpus: [0, 1] })
expect(result.message).toBe('GPU activation successful')
expect(result.activated_gpus).toEqual([0, 1])
// Verify the change in hardware info
const hardwareInfo = await extension.getHardware()
expect(hardwareInfo.active_gpus).toEqual([0, 1])
})
test('should throw error for invalid GPU ids', async () => {
await expect(extension.setAvtiveGpu({ gpus: [999] })).rejects.toThrow('No valid GPUs selected')
})
test('should handle mix of valid and invalid GPU ids', async () => {
const result = await extension.setAvtiveGpu({ gpus: [0, 999] })
// Should only activate valid GPUs
expect(result.activated_gpus).toEqual([0])
// Verify the change in hardware info
const hardwareInfo = await extension.getHardware()
expect(hardwareInfo.active_gpus).toEqual([0])
})
})

26
core/src/browser/extensions/hardwareManagement.ts
View File

@ -1,26 +0,0 @@
import { HardwareInformation } from '../../types'
import { BaseExtension, ExtensionTypeEnum } from '../extension'
/**
* Engine management extension. Persists and retrieves engine management.
* @abstract
* @extends BaseExtension
*/
export abstract class HardwareManagementExtension extends BaseExtension {
type(): ExtensionTypeEnum | undefined {
return ExtensionTypeEnum.Hardware
}
/**
* @returns A Promise that resolves to an object of list hardware.
*/
abstract getHardware(): Promise<HardwareInformation>
/**
* @returns A Promise that resolves to an object of set active gpus.
*/
abstract setActiveGpu(data: { gpus: number[] }): Promise<{
message: string
activated_gpus: number[]
}>
}

17
core/src/browser/extensions/index.ts
View File

@ -9,29 +9,12 @@ export { ConversationalExtension } from './conversational'
*/ */
export { InferenceExtension } from './inference' export { InferenceExtension } from './inference'
/** /**
* Assistant extension for managing assistants. * Assistant extension for managing assistants.
*/ */
export { AssistantExtension } from './assistant' export { AssistantExtension } from './assistant'
/**
* Model extension for managing models.
*/
export { ModelExtension } from './model'
/** /**
* Base AI Engines. * Base AI Engines.
*/ */
export * from './engines' export * from './engines'
/**
* Engines Management
*/
export * from './enginesManagement'
/**
* Hardware Management
*/
export * from './hardwareManagement'

286
core/src/browser/extensions/model.test.ts
View File

@ -1,286 +0,0 @@
import { ModelExtension } from './model'
import { ExtensionTypeEnum } from '../extension'
import { Model, OptionType, ModelSource } from '../../types'
// Mock implementation of ModelExtension
class MockModelExtension extends ModelExtension {
private models: Model[] = []
private sources: ModelSource[] = []
private loadedModels: Set<string> = new Set()
private modelsPulling: Set<string> = new Set()
constructor() {
super('http://mock-url.com', 'mock-model-extension', 'Mock Model Extension', true, 'A mock model extension', '1.0.0')
}
onLoad(): void {
// Mock implementation
}
onUnload(): void {
// Mock implementation
}
async configurePullOptions(configs: { [key: string]: any }): Promise<any> {
return configs
}
async getModels(): Promise<Model[]> {
return this.models
}
async pullModel(model: string, id?: string, name?: string): Promise<void> {
const modelId = id || `model-${Date.now()}`
this.modelsPulling.add(modelId)
// Simulate model pull by adding it to the model list
const newModel: Model = {
id: modelId,
path: `/models/${model}`,
name: name || model,
source: 'mock-source',
modelFormat: 'mock-format',
engine: 'mock-engine',
format: 'mock-format',
status: 'ready',
contextLength: 2048,
sizeInGB: 2,
created: new Date().toISOString(),
pullProgress: {
percent: 100,
transferred: 0,
total: 0
}
}
this.models.push(newModel)
this.loadedModels.add(modelId)
this.modelsPulling.delete(modelId)
}
async cancelModelPull(modelId: string): Promise<void> {
this.modelsPulling.delete(modelId)
// Remove the model if it's in the pulling state
this.models = this.models.filter(m => m.id !== modelId)
}
async importModel(
model: string,
modelPath: string,
name?: string,
optionType?: OptionType
): Promise<void> {
const newModel: Model = {
id: `model-${Date.now()}`,
path: modelPath,
name: name || model,
source: 'local',
modelFormat: optionType?.format || 'mock-format',
engine: optionType?.engine || 'mock-engine',
format: optionType?.format || 'mock-format',
status: 'ready',
contextLength: optionType?.contextLength || 2048,
sizeInGB: 2,
created: new Date().toISOString(),
}
this.models.push(newModel)
this.loadedModels.add(newModel.id)
}
async updateModel(modelInfo: Partial<Model>): Promise<Model> {
if (!modelInfo.id) throw new Error('Model ID is required')
const index = this.models.findIndex(m => m.id === modelInfo.id)
if (index === -1) throw new Error('Model not found')
this.models[index] = { ...this.models[index], ...modelInfo }
return this.models[index]
}
async deleteModel(modelId: string): Promise<void> {
this.models = this.models.filter(m => m.id !== modelId)
this.loadedModels.delete(modelId)
}
async isModelLoaded(modelId: string): Promise<boolean> {
return this.loadedModels.has(modelId)
}
async getSources(): Promise<ModelSource[]> {
return this.sources
}
async addSource(source: string): Promise<void> {
const newSource: ModelSource = {
id: `source-${Date.now()}`,
url: source,
name: `Source ${this.sources.length + 1}`,
type: 'mock-type'
}
this.sources.push(newSource)
}
async deleteSource(sourceId: string): Promise<void> {
this.sources = this.sources.filter(s => s.id !== sourceId)
}
}
describe('ModelExtension', () => {
let extension: MockModelExtension
beforeEach(() => {
extension = new MockModelExtension()
})
test('should return the correct extension type', () => {
expect(extension.type()).toBe(ExtensionTypeEnum.Model)
})
test('should configure pull options', async () => {
const configs = { apiKey: 'test-key', baseUrl: 'https://test-url.com' }
const result = await extension.configurePullOptions(configs)
expect(result).toEqual(configs)
})
test('should add and get models', async () => {
await extension.pullModel('test-model', 'test-id', 'Test Model')
const models = await extension.getModels()
expect(models).toHaveLength(1)
expect(models[0].id).toBe('test-id')
expect(models[0].name).toBe('Test Model')
})
test('should pull model with default id and name', async () => {
await extension.pullModel('test-model')
const models = await extension.getModels()
expect(models).toHaveLength(1)
expect(models[0].name).toBe('test-model')
})
test('should cancel model pull', async () => {
await extension.pullModel('test-model', 'test-id')
// Verify model exists
let models = await extension.getModels()
expect(models).toHaveLength(1)
// Cancel the pull
await extension.cancelModelPull('test-id')
// Verify model was removed
models = await extension.getModels()
expect(models).toHaveLength(0)
})
test('should import model', async () => {
const optionType: OptionType = {
engine: 'test-engine',
format: 'test-format',
contextLength: 4096
}
await extension.importModel('test-model', '/path/to/model', 'Imported Model', optionType)
const models = await extension.getModels()
expect(models).toHaveLength(1)
expect(models[0].name).toBe('Imported Model')
expect(models[0].engine).toBe('test-engine')
expect(models[0].format).toBe('test-format')
expect(models[0].contextLength).toBe(4096)
})
test('should import model with default values', async () => {
await extension.importModel('test-model', '/path/to/model')
const models = await extension.getModels()
expect(models).toHaveLength(1)
expect(models[0].name).toBe('test-model')
expect(models[0].engine).toBe('mock-engine')
expect(models[0].format).toBe('mock-format')
})
test('should update model', async () => {
await extension.pullModel('test-model', 'test-id', 'Test Model')
const updatedModel = await extension.updateModel({
id: 'test-id',
name: 'Updated Model',
contextLength: 8192
})
expect(updatedModel.name).toBe('Updated Model')
expect(updatedModel.contextLength).toBe(8192)
// Verify changes persisted
const models = await extension.getModels()
expect(models[0].name).toBe('Updated Model')
expect(models[0].contextLength).toBe(8192)
})
test('should throw error when updating non-existent model', async () => {
await expect(extension.updateModel({
id: 'non-existent',
name: 'Updated Model'
})).rejects.toThrow('Model not found')
})
test('should throw error when updating model without ID', async () => {
await expect(extension.updateModel({
name: 'Updated Model'
})).rejects.toThrow('Model ID is required')
})
test('should delete model', async () => {
await extension.pullModel('test-model', 'test-id')
// Verify model exists
let models = await extension.getModels()
expect(models).toHaveLength(1)
// Delete the model
await extension.deleteModel('test-id')
// Verify model was removed
models = await extension.getModels()
expect(models).toHaveLength(0)
})
test('should check if model is loaded', async () => {
await extension.pullModel('test-model', 'test-id')
// Check if model is loaded
const isLoaded = await extension.isModelLoaded('test-id')
expect(isLoaded).toBe(true)
// Check if non-existent model is loaded
const nonExistentLoaded = await extension.isModelLoaded('non-existent')
expect(nonExistentLoaded).toBe(false)
})
test('should add and get sources', async () => {
await extension.addSource('https://test-source.com')
const sources = await extension.getSources()
expect(sources).toHaveLength(1)
expect(sources[0].url).toBe('https://test-source.com')
})
test('should delete source', async () => {
await extension.addSource('https://test-source.com')
// Get the source ID
const sources = await extension.getSources()
const sourceId = sources[0].id
// Delete the source
await extension.deleteSource(sourceId)
// Verify source was removed
const updatedSources = await extension.getSources()
expect(updatedSources).toHaveLength(0)
})
})

48
core/src/browser/extensions/model.ts
View File

@ -1,48 +0,0 @@
import { BaseExtension, ExtensionTypeEnum } from '../extension'
import { Model, ModelInterface, ModelSource, OptionType } from '../../types'
/**
* Model extension for managing models.
*/
export abstract class ModelExtension
extends BaseExtension
implements ModelInterface
{
/**
* Model extension type.
*/
type(): ExtensionTypeEnum | undefined {
return ExtensionTypeEnum.Model
}
abstract configurePullOptions(configs: { [key: string]: any }): Promise<any>
abstract getModels(): Promise<Model[]>
abstract pullModel(model: string, id?: string, name?: string): Promise<void>
abstract cancelModelPull(modelId: string): Promise<void>
abstract importModel(
model: string,
modePath: string,
name?: string,
optionType?: OptionType
): Promise<void>
abstract updateModel(modelInfo: Partial<Model>): Promise<Model>
abstract deleteModel(model: string): Promise<void>
abstract isModelLoaded(model: string): Promise<boolean>
/**
* Get model sources
*/
abstract getSources(): Promise<ModelSource[]>
/**
* Add a model source
*/
abstract addSource(source: string): Promise<void>
/**
* Delete a model source
*/
abstract deleteSource(source: string): Promise<void>
/**
* Fetch models hub
*/
abstract fetchModelsHub(): Promise<void>
}

47
extensions/engine-management-extension/engines.mjs
View File

@ -1,47 +0,0 @@
import anthropic from './resources/anthropic.json' with { type: 'json' }
import cohere from './resources/cohere.json' with { type: 'json' }
import openai from './resources/openai.json' with { type: 'json' }
import openrouter from './resources/openrouter.json' with { type: 'json' }
import groq from './resources/groq.json' with { type: 'json' }
import martian from './resources/martian.json' with { type: 'json' }
import mistral from './resources/mistral.json' with { type: 'json' }
import nvidia from './resources/nvidia.json' with { type: 'json' }
import deepseek from './resources/deepseek.json' with { type: 'json' }
import googleGemini from './resources/google_gemini.json' with { type: 'json' }
import anthropicModels from './models/anthropic.json' with { type: 'json' }
import cohereModels from './models/cohere.json' with { type: 'json' }
import openaiModels from './models/openai.json' with { type: 'json' }
import openrouterModels from './models/openrouter.json' with { type: 'json' }
import groqModels from './models/groq.json' with { type: 'json' }
import martianModels from './models/martian.json' with { type: 'json' }
import mistralModels from './models/mistral.json' with { type: 'json' }
import nvidiaModels from './models/nvidia.json' with { type: 'json' }
import deepseekModels from './models/deepseek.json' with { type: 'json' }
import googleGeminiModels from './models/google_gemini.json' with { type: 'json' }
const engines = [
anthropic,
openai,
cohere,
openrouter,
groq,
mistral,
martian,
nvidia,
deepseek,
googleGemini,
]
const models = [
...anthropicModels,
...openaiModels,
...cohereModels,
...openrouterModels,
...groqModels,
...mistralModels,
...martianModels,
...nvidiaModels,
...deepseekModels,
...googleGeminiModels,
]
export { engines, models }

5
extensions/engine-management-extension/jest.config.js
View File

@ -1,5 +0,0 @@
/** @type {import('ts-jest').JestConfigWithTsJest} */
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
}

58
extensions/engine-management-extension/models/anthropic.json
View File

@ -1,58 +0,0 @@
[
{
"model": "claude-3-opus-latest",
"object": "model",
"name": "Claude 3 Opus Latest",
"version": "1.0",
"description": "Claude 3 Opus is a powerful model suitables for highly complex task.",
"inference_params": {
"max_tokens": 4096,
"temperature": 0.7,
"max_temperature": 1.0,
"stream": true
},
"engine": "anthropic"
},
{
"model": "claude-3-5-haiku-latest",
"object": "model",
"name": "Claude 3.5 Haiku Latest",
"version": "1.0",
"description": "Claude 3.5 Haiku is the fastest model provides near-instant responsiveness.",
"inference_params": {
"max_tokens": 8192,
"temperature": 0.7,
"max_temperature": 1.0,
"stream": true
},
"engine": "anthropic"
},
{
"model": "claude-3-5-sonnet-latest",
"object": "model",
"name": "Claude 3.5 Sonnet Latest",
"version": "1.0",
"description": "Claude 3.5 Sonnet raises the industry bar for intelligence, outperforming competitor models and Claude 3 Opus on a wide range of evaluations, with the speed and cost of our mid-tier model, Claude 3 Sonnet.",
"inference_params": {
"max_tokens": 8192,
"temperature": 0.7,
"max_temperature": 1.0,
"stream": true
},
"engine": "anthropic"
},
{
"model": "claude-3-7-sonnet-latest",
"object": "model",
"name": "Claude 3.7 Sonnet Latest",
"version": "1.0",
"description": "Claude 3.7 Sonnet is the first hybrid reasoning model on the market. It is the most intelligent model yet. It is faster, more cost effective, and more capable than any other model in its class.",
"inference_params": {
"max_tokens": 8192,
"temperature": 0.7,
"max_temperature": 1.0,
"stream": true
},
"engine": "anthropic"
}
]

44
extensions/engine-management-extension/models/cohere.json
View File

@ -1,44 +0,0 @@
[
{
"model": "command-r-plus",
"object": "model",
"name": "Command R+",
"version": "1.0",
"description": "Command R+ is an instruction-following conversational model that performs language tasks at a higher quality, more reliably, and with a longer context than previous models. It is best suited for complex RAG workflows and multi-step tool use.",
"inference_params": {
"max_tokens": 4096,
"temperature": 0.7,
"max_temperature": 1.0,
"stream": true
},
"engine": "cohere"
},
{
"model": "command-r",
"object": "model",
"name": "Command R",
"version": "1.0",
"description": "Command R is an instruction-following conversational model that performs language tasks at a higher quality, more reliably, and with a longer context than previous models. It can be used for complex workflows like code generation, retrieval augmented generation (RAG), tool use, and agents.",
"inference_params": {
"max_tokens": 4096,
"temperature": 0.7,
"max_temperature": 1.0,
"stream": true
},
"engine": "cohere"
},
{
"model": "command-a-03-2025",
"object": "model",
"name": "Command A",
"version": "1.0",
"description": "Command A is an instruction-following conversational model that performs language tasks at a higher quality, more reliably, and with a longer context than previous models. It is best suited for complex RAG workflows and multi-step tool use.",
"inference_params": {
"max_tokens": 4096,
"temperature": 0.7,
"max_temperature": 1.0,
"stream": true
},
"engine": "cohere"
}
]

28
extensions/engine-management-extension/models/deepseek.json
View File

@ -1,28 +0,0 @@
[
{
"model": "deepseek-chat",
"object": "model",
"name": "DeepSeek V3",
"version": "1.0",
"description": "The deepseek-chat model has been upgraded to DeepSeek-V3. deepseek-reasoner points to the new model DeepSeek-R1",
"inference_params": {
"max_tokens": 8192,
"temperature": 0.6,
"stream": true
},
"engine": "deepseek"
},
{
"model": "deepseek-reasoner",
"object": "model",
"name": "DeepSeek R1",
"version": "1.0",
"description": "CoT (Chain of Thought) is the reasoning content deepseek-reasoner gives before output the final answer. For details, please refer to Reasoning Model.",
"inference_params": {
"max_tokens": 8192,
"temperature": 0.6,
"stream": true
},
"engine": "deepseek"
}
]

93
extensions/engine-management-extension/models/google_gemini.json
View File

@ -1,93 +0,0 @@
[
{
"model": "gemini-1.5-flash",
"object": "model",
"name": "Gemini 1.5 Flash",
"version": "1.0",
"description": "Gemini 1.5 Flash is a fast and versatile multimodal model for scaling across diverse tasks.",
"inference_params": {
"max_tokens": 8192,
"temperature": 0.6,
"stream": true
},
"engine": "google_gemini"
},
{
"model": "gemini-1.5-flash-8b",
"object": "model",
"name": "Gemini 1.5 Flash-8B",
"version": "1.0",
"description": "Gemini 1.5 Flash-8B is a small model designed for lower intelligence tasks.",
"inference_params": {
"max_tokens": 8192,
"temperature": 0.6,
"stream": true
},
"engine": "google_gemini"
},
{
"model": "gemini-1.5-pro",
"object": "model",
"name": "Gemini 1.5 Pro",
"version": "1.0",
"description": "Gemini 1.5 Pro is a mid-size multimodal model that is optimized for a wide-range of reasoning tasks. 1.5 Pro can process large amounts of data at once, including 2 hours of video, 19 hours of audio, codebases with 60,000 lines of code, or 2,000 pages of text. ",
"inference_params": {
"max_tokens": 8192,
"temperature": 0.6,
"stream": true
},
"engine": "google_gemini"
},
{
"model": "gemini-2.5-pro-preview-05-06",
"object": "model",
"name": "Gemini 2.5 Pro Preview",
"version": "1.0",
"description": "Gemini 2.5 Pro is our state-of-the-art thinking model, capable of reasoning over complex problems in code, math, and STEM, as well as analyzing large datasets, codebases, and documents using long context. Gemini 2.5 Pro rate limits are more restricted since it is an experimental / preview model.",
"inference_params": {
"max_tokens": 65536,
"temperature": 0.6,
"stream": true
},
"engine": "google_gemini"
},
{
"model": "gemini-2.5-flash-preview-04-17",
"object": "model",
"name": "Our best model in terms of price-performance, offering well-rounded capabilities. Gemini 2.5 Flash rate limits are more restricted since it is an experimental / preview model.",
"version": "1.0",
"description": "Gemini 2.5 Flash preview",
"inference_params": {
"max_tokens": 8192,
"temperature": 0.6,
"stream": true
},
"engine": "google_gemini"
},
{
"model": "gemini-2.0-flash",
"object": "model",
"name": "Gemini 2.0 Flash",
"version": "1.0",
"description": "Gemini 2.0 Flash delivers next-gen features and improved capabilities, including superior speed, native tool use, multimodal generation, and a 1M token context window.",
"inference_params": {
"max_tokens": 8192,
"temperature": 0.6,
"stream": true
},
"engine": "google_gemini"
},
{
"model": "gemini-2.0-flash-lite",
"object": "model",
"name": "Gemini 2.0 Flash-Lite",
"version": "1.0",
"description": "A Gemini 2.0 Flash model optimized for cost efficiency and low latency.",
"inference_params": {
"max_tokens": 8192,
"temperature": 0.6,
"stream": true
},
"engine": "google_gemini"
}
]

87
extensions/engine-management-extension/models/groq.json
View File

@ -1,87 +0,0 @@
[
{
"model": "llama3-70b-8192",
"object": "model",
"name": "Groq Llama 3 70b",
"version": "1.1",
"description": "Groq Llama 3 70b with supercharged speed!",
"inference_params": {
"max_tokens": 8192,
"temperature": 0.7,
"top_p": 0.95,
"stream": true,
"stop": [],
"frequency_penalty": 0,
"presence_penalty": 0
},
"engine": "groq"
},
{
"model": "llama3-8b-8192",
"object": "model",
"name": "Groq Llama 3 8b",
"version": "1.1",
"description": "Groq Llama 3 8b with supercharged speed!",
"inference_params": {
"max_tokens": 8192,
"temperature": 0.7,
"top_p": 0.95,
"stream": true,
"stop": [],
"frequency_penalty": 0,
"presence_penalty": 0
},
"engine": "groq"
},
{
"model": "llama-3.1-8b-instant",
"object": "model",
"name": "Groq Llama 3.1 8b Instant",
"version": "1.1",
"description": "Groq Llama 3.1 8b with supercharged speed!",
"inference_params": {
"max_tokens": 8000,
"temperature": 0.7,
"top_p": 0.95,
"stream": true,
"stop": [],
"frequency_penalty": 0,
"presence_penalty": 0
},
"engine": "groq"
},
{
"model": "gemma2-9b-it",
"object": "model",
"name": "Groq Gemma 9B Instruct",
"version": "1.2",
"description": "Groq Gemma 9b Instruct with supercharged speed!",
"parameters": {
"max_tokens": 8192,
"temperature": 0.7,
"top_p": 0.95,
"stream": true,
"stop": [],
"frequency_penalty": 0,
"presence_penalty": 0
},
"engine": "groq"
},
{
"model": "llama-3.3-70b-versatile",
"object": "model",
"name": "Groq Llama 3.3 70b Versatile",
"version": "3.3",
"description": "Groq Llama 3.3 70b Versatile with supercharged speed!",
"parameters": {
"max_tokens": 32768,
"temperature": 0.7,
"top_p": 0.95,
"stream": true,
"stop": [],
"frequency_penalty": 0,
"presence_penalty": 0
},
"engine": "groq"
}
]

19
extensions/engine-management-extension/models/martian.json
View File

@ -1,19 +0,0 @@
[
{
"model": "router",
"object": "model",
"name": "Martian Model Router",
"version": "1.0",
"description": "Martian Model Router dynamically routes requests to the best LLM in real-time",
"inference_params": {
"max_tokens": 4096,
"temperature": 0.7,
"top_p": 0.95,
"stream": true,
"stop": [],
"frequency_penalty": 0,
"presence_penalty": 0
},
"engine": "martian"
}
]

47
extensions/engine-management-extension/models/mistral.json
View File

@ -1,47 +0,0 @@
[
{
"model": "mistral-small-latest",
"object": "model",
"name": "Mistral Small",
"version": "1.1",
"description": "Mistral Small is the ideal choice for simple tasks (Classification, Customer Support, or Text Generation) at an affordable price.",
"inference_params": {
"max_tokens": 32000,
"temperature": 0.7,
"max_temperature": 1.0,
"top_p": 0.95,
"stream": true
},
"engine": "mistral"
},
{
"model": "mistral-large-latest",
"object": "model",
"name": "Mistral Large",
"version": "1.1",
"description": "Mistral Large is ideal for complex tasks (Synthetic Text Generation, Code Generation, RAG, or Agents).",
"inference_params": {
"max_tokens": 32000,
"temperature": 0.7,
"max_temperature": 1.0,
"top_p": 0.95,
"stream": true
},
"engine": "mistral"
},
{
"model": "open-mixtral-8x22b",
"object": "model",
"name": "Mixtral 8x22B",
"version": "1.1",
"description": "Mixtral 8x22B is a high-performance, cost-effective model designed for complex tasks.",
"inference_params": {
"max_tokens": 32000,
"temperature": 0.7,
"max_temperature": 1.0,
"top_p": 0.95,
"stream": true
},
"engine": "mistral"
}
]

21
extensions/engine-management-extension/models/nvidia.json
View File

@ -1,21 +0,0 @@
[
{
"model": "mistralai/mistral-7b-instruct-v0.2",
"object": "model",
"name": "Mistral 7B",
"version": "1.1",
"description": "Mistral 7B with NVIDIA",
"inference_params": {
"max_tokens": 1024,
"temperature": 0.3,
"max_temperature": 1.0,
"top_p": 1,
"stream": false,
"frequency_penalty": 0,
"presence_penalty": 0,
"stop": null,
"seed": null
},
"engine": "nvidia"
}
]

143
extensions/engine-management-extension/models/openai.json
View File

@ -1,143 +0,0 @@
[
{
"model": "gpt-4.5-preview",
"object": "model",
"name": "OpenAI GPT 4.5 Preview",
"version": "1.2",
"description": "OpenAI GPT 4.5 Preview is a research preview of GPT-4.5, our largest and most capable GPT model yet",
"format": "api",
"inference_params": {
"max_tokens": 16384,
"temperature": 0.7,
"top_p": 0.95,
"stream": true,
"stop": [],
"frequency_penalty": 0,
"presence_penalty": 0
},
"engine": "openai"
},
{
"model": "gpt-4-turbo",
"object": "model",
"name": "OpenAI GPT 4 Turbo",
"version": "1.2",
"description": "OpenAI GPT 4 Turbo model is extremely good",
"format": "api",
"inference_params": {
"max_tokens": 4096,
"temperature": 0.7,
"top_p": 0.95,
"stream": true,
"stop": [],
"frequency_penalty": 0,
"presence_penalty": 0
},
"engine": "openai"
},
{
"model": "gpt-3.5-turbo",
"object": "model",
"name": "OpenAI GPT 3.5 Turbo",
"version": "1.1",
"description": "OpenAI GPT 3.5 Turbo model is extremely fast",
"format": "api",
"inference_params": {
"max_tokens": 4096,
"temperature": 0.7,
"top_p": 0.95,
"stream": true,
"stop": [],
"frequency_penalty": 0,
"presence_penalty": 0
},
"engine": "openai"
},
{
"model": "gpt-4o",
"object": "model",
"name": "OpenAI GPT 4o",
"version": "1.1",
"description": "OpenAI GPT 4o is a new flagship model with fast speed and high quality",
"format": "api",
"inference_params": {
"max_tokens": 4096,
"temperature": 0.7,
"top_p": 0.95,
"stream": true,
"stop": [],
"frequency_penalty": 0,
"presence_penalty": 0
},
"engine": "openai"
},
{
"model": "gpt-4o-mini",
"object": "model",
"name": "OpenAI GPT 4o-mini",
"version": "1.1",
"description": "GPT-4o mini (“o” for “omni”) is a fast, affordable small model for focused tasks.",
"format": "api",
"inference_params": {
"max_tokens": 16384,
"temperature": 0.7,
"top_p": 0.95,
"stream": true,
"stop": [],
"frequency_penalty": 0,
"presence_penalty": 0
},
"engine": "openai"
},
{
"model": "o1",
"object": "model",
"name": "OpenAI o1",
"version": "1.0",
"description": "OpenAI o1 is a new model with complex reasoning",
"format": "api",
"inference_params": {
"max_tokens": 100000
},
"engine": "openai"
},
{
"model": "o1-preview",
"object": "model",
"name": "OpenAI o1-preview",
"version": "1.0",
"description": "OpenAI o1-preview is a new model with complex reasoning",
"format": "api",
"inference_params": {
"max_tokens": 32768,
"stream": true
},
"engine": "openai"
},
{
"model": "o1-mini",
"object": "model",
"name": "OpenAI o1-mini",
"version": "1.0",
"description": "OpenAI o1-mini is a lightweight reasoning model",
"format": "api",
"inference_params": {
"max_tokens": 65536,
"stream": true
},
"engine": "openai"
},
{
"model": "o3-mini",
"object": "model",
"name": "OpenAI o3-mini",
"version": "1.0",
"description": "OpenAI most recent reasoning model, providing high intelligence at the same cost and latency targets of o1-mini.",
"format": "api",
"inference_params": {
"max_tokens": 100000,
"stream": true
},
"engine": "openai"
}
]

92
extensions/engine-management-extension/models/openrouter.json
View File

@ -1,92 +0,0 @@
[
{
"model": "deepseek/deepseek-r1:free",
"object": "model",
"name": "DeepSeek: R1",
"version": "1.0",
"description": "OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.",
"inference_params": {
"temperature": 0.7,
"top_p": 0.95,
"frequency_penalty": 0,
"presence_penalty": 0,
"stream": true
},
"engine": "openrouter"
},
{
"model": "deepseek/deepseek-r1-distill-llama-70b:free",
"object": "model",
"name": "DeepSeek: R1 Distill Llama 70B",
"version": "1.0",
"description": " OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.",
"inference_params": {
"temperature": 0.7,
"top_p": 0.95,
"frequency_penalty": 0,
"presence_penalty": 0,
"stream": true
},
"engine": "openrouter"
},
{
"model": "deepseek/deepseek-r1-distill-llama-70b:free",
"object": "model",
"name": "DeepSeek: R1 Distill Llama 70B",
"version": "1.0",
"description": "OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.",
"inference_params": {
"temperature": 0.7,
"top_p": 0.95,
"frequency_penalty": 0,
"presence_penalty": 0,
"stream": true
},
"engine": "openrouter"
},
{
"model": "meta-llama/llama-3.1-405b-instruct:free",
"object": "model",
"name": "Meta: Llama 3.1 405B Instruct",
"version": "1.0",
"description": "OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.",
"inference_params": {
"temperature": 0.7,
"top_p": 0.95,
"frequency_penalty": 0,
"presence_penalty": 0,
"stream": true
},
"engine": "openrouter"
},
{
"model": "qwen/qwen-vl-plus:free",
"object": "model",
"name": "Qwen: Qwen VL Plus",
"version": "1.0",
"description": "OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.",
"inference_params": {
"temperature": 0.7,
"top_p": 0.95,
"frequency_penalty": 0,
"presence_penalty": 0,
"stream": true
},
"engine": "openrouter"
},
{
"model": "qwen/qwen2.5-vl-72b-instruct:free",
"object": "model",
"name": "Qwen: Qwen2.5 VL 72B Instruct",
"version": "1.0",
"description": "OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.",
"inference_params": {
"temperature": 0.7,
"top_p": 0.95,
"frequency_penalty": 0,
"presence_penalty": 0,
"stream": true
},
"engine": "openrouter"
}
]

47
extensions/engine-management-extension/package.json
View File

@ -1,47 +0,0 @@
{
"name": "@janhq/engine-management-extension",
"productName": "Engine Management",
"version": "1.0.3",
"description": "Manages AI engines and their configurations.",
"main": "dist/index.js",
"node": "dist/node/index.cjs.js",
"author": "Jan <service@jan.ai>",
"license": "MIT",
"scripts": {
"test": "vitest run",
"build": "rolldown -c rolldown.config.mjs",
"codesign:darwin": "../../.github/scripts/auto-sign.sh",
"codesign:win32:linux": "echo 'No codesigning required'",
"codesign": "run-script-os",
"build:publish": "rimraf *.tgz --glob || true && yarn build && yarn codesign && npm pack && cpx *.tgz ../../pre-install"
},
"exports": {
".": "./dist/index.js",
"./main": "./dist/module.js"
},
"devDependencies": {
"cpx": "^1.5.0",
"rimraf": "^3.0.2",
"rolldown": "^1.0.0-beta.1",
"run-script-os": "^1.1.6",
"ts-loader": "^9.5.0",
"typescript": "^5.3.3",
"vitest": "^3.0.6"
},
"dependencies": {
"@janhq/core": "../../core/package.tgz",
"ky": "^1.7.2",
"p-queue": "^8.0.1"
},
"bundledDependencies": [
"@janhq/core"
],
"engines": {
"node": ">=18.0.0"
},
"files": [
"dist/*",
"package.json",
"README.md"
]
}

23
extensions/engine-management-extension/resources/anthropic.json
View File

@ -1,23 +0,0 @@
{
"id": "anthropic",
"type": "remote",
"engine": "anthropic",
"url": "https://console.anthropic.com/settings/keys",
"api_key": "",
"metadata": {
"get_models_url": "https://api.anthropic.com/v1/models",
"header_template": "x-api-key: {{api_key}} anthropic-version: 2023-06-01",
"transform_req": {
"chat_completions": {
"url": "https://api.anthropic.com/v1/messages",
"template": "{ {% for key, value in input_request %} {% if key == \"messages\" %} {% if input_request.messages.0.role == \"system\" %} \"system\": {{ tojson(input_request.messages.0.content) }}, \"messages\": [{% for message in input_request.messages %} {% if not loop.is_first %} {\"role\": {{ tojson(message.role) }}, \"content\": {% if not message.content or message.content == \"\" %} \".\" {% else %} {{ tojson(message.content) }} {% endif %} } {% if not loop.is_last %},{% endif %} {% endif %} {% endfor %}] {% else %} \"messages\": [{% for message in input_request.messages %} {\"role\": {{ tojson(message.role) }}, \"content\": {% if not message.content or message.content == \"\" %} \".\" {% else %} {{ tojson(message.content) }} {% endif %} } {% if not loop.is_last %},{% endif %} {% endfor %}] {% endif %} {% if not loop.is_last %},{% endif %} {% else if key == \"system\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"metadata\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %}\"{{ key }}\": {{ tojson(value) }} {% if not loop.is_last %},{% endif %} {% endif %} {% endfor %} }"
}
},
"transform_resp": {
"chat_completions": {
"template": "{% if input_request.stream %} {\"object\": \"chat.completion.chunk\", \"model\": \"{{ input_request.model }}\", \"choices\": [{\"index\": 0, \"delta\": { {% if input_request.type == \"message_start\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"ping\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"content_block_delta\" %} \"role\": \"assistant\", \"content\": {{ tojson(input_request.delta.text) }} {% else if input_request.type == \"content_block_stop\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"content_block_stop\" %} \"role\": \"assistant\", \"content\": null {% endif %} }, {% if input_request.type == \"content_block_stop\" %} \"finish_reason\": \"stop\" {% else %} \"finish_reason\": null {% endif %} }]} {% else %} {{tojson(input_request)}} {% endif %}"
}
},
"explore_models_url": "https://docs.anthropic.com/en/docs/about-claude/models"
}
}

23
extensions/engine-management-extension/resources/cohere.json
View File

@ -1,23 +0,0 @@
{
"id": "cohere",
"type": "remote",
"engine": "cohere",
"url": "https://dashboard.cohere.com/api-keys",
"api_key": "",
"metadata": {
"get_models_url": "https://api.cohere.ai/v1/models",
"header_template": "Authorization: Bearer {{api_key}}",
"transform_req": {
"chat_completions": {
"url": "https://api.cohere.ai/v1/chat",
"template": "{ {% for key, value in input_request %} {% if key == \"messages\" %} {% if input_request.messages.0.role == \"system\" %} \"preamble\": {{ tojson(input_request.messages.0.content) }}, {% if length(input_request.messages) > 2 %} \"chatHistory\": [{% for message in input_request.messages %} {% if not loop.is_first and not loop.is_last %} {\"role\": {% if message.role == \"user\" %} \"USER\" {% else %} \"CHATBOT\" {% endif %}, \"content\": {{ tojson(message.content) }} } {% if loop.index < length(input_request.messages) - 2 %},{% endif %} {% endif %} {% endfor %}], {% endif %} \"message\": {{ tojson(last(input_request.messages).content) }} {% else %} {% if length(input_request.messages) > 2 %} \"chatHistory\": [{% for message in input_request.messages %} {% if not loop.is_last %} { \"role\": {% if message.role == \"user\" %} \"USER\" {% else %} \"CHATBOT\" {% endif %}, \"content\": {{ tojson(message.content) }} } {% if loop.index < length(input_request.messages) - 2 %},{% endif %} {% endif %} {% endfor %}],{% endif %}\"message\": {{ tojson(last(input_request.messages).content) }} {% endif %}{% if not loop.is_last %},{% endif %} {% else if key == \"system\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} \"{{ key }}\": {{ tojson(value) }} {% if not loop.is_last %},{% endif %} {% endif %} {% endfor %} }"
}
},
"transform_resp": {
"chat_completions": {
"template": "{% if input_request.stream %} {\"object\": \"chat.completion.chunk\", \"model\": \"{{ input_request.model }}\", \"choices\": [{\"index\": 0, \"delta\": { {% if input_request.event_type == \"text-generation\" %} \"role\": \"assistant\", \"content\": {{ tojson(input_request.text) }} {% else %} \"role\": \"assistant\", \"content\": null {% endif %} }, {% if input_request.event_type == \"stream-end\" %} \"finish_reason\": \"{{ input_request.finish_reason }}\" {% else %} \"finish_reason\": null {% endif %} }]} {% else %} {\"id\": \"{{ input_request.generation_id }}\", \"created\": null, \"object\": \"chat.completion\", \"model\": {% if input_request.model %} \"{{ input_request.model }}\" {% else %} \"command-r-plus-08-2024\" {% endif %}, \"choices\": [{ \"index\": 0, \"message\": { \"role\": \"assistant\", \"content\": {% if not input_request.text %} null {% else %} {{ tojson(input_request.text) }} {% endif %}, \"refusal\": null }, \"logprobs\": null, \"finish_reason\": \"{{ input_request.finish_reason }}\" } ], \"usage\": { \"prompt_tokens\": {{ input_request.meta.tokens.input_tokens }}, \"completion_tokens\": {{ input_request.meta.tokens.output_tokens }},\"total_tokens\": {{ input_request.meta.tokens.input_tokens + input_request.meta.tokens.output_tokens }}, \"prompt_tokens_details\": { \"cached_tokens\": 0 },\"completion_tokens_details\": { \"reasoning_tokens\": 0, \"accepted_prediction_tokens\": 0, \"rejected_prediction_tokens\": 0 } }, \"system_fingerprint\": \"fp_6b68a8204b\"} {% endif %}"
}
},
"explore_models_url": "https://docs.cohere.com/v2/docs/models"
}
}

23
extensions/engine-management-extension/resources/deepseek.json
View File

@ -1,23 +0,0 @@
{
"id": "deepseek",
"type": "remote",
"engine": "deepseek",
"url": "https://platform.deepseek.com/api_keys",
"api_key": "",
"metadata": {
"get_models_url": "https://api.deepseek.com/models",
"header_template": "Authorization: Bearer {{api_key}}",
"transform_req": {
"chat_completions": {
"url": "https://api.deepseek.com/chat/completions",
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"model\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
}
},
"transform_resp": {
"chat_completions": {
"template": "{{tojson(input_request)}}"
}
},
"explore_models_url": "https://api-docs.deepseek.com/quick_start/pricing"
}
}

23
extensions/engine-management-extension/resources/google_gemini.json
View File

@ -1,23 +0,0 @@
{
"id": "google_gemini",
"type": "remote",
"engine": "google_gemini",
"url": "https://aistudio.google.com/apikey",
"api_key": "",
"metadata": {
"get_models_url": "https://generativelanguage.googleapis.com/openai/v1beta/models",
"header_template": "Authorization: Bearer {{api_key}}",
"transform_req": {
"chat_completions": {
"url": "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions",
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"model\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
}
},
"transform_resp": {
"chat_completions": {
"template": "{{tojson(input_request)}}"
}
},
"explore_models_url": "https://ai.google.dev/gemini-api/docs/models/gemini"
}
}

23
extensions/engine-management-extension/resources/groq.json
View File

@ -1,23 +0,0 @@
{
"id": "groq",
"type": "remote",
"engine": "groq",
"url": "https://console.groq.com/keys",
"api_key": "",
"metadata": {
"get_models_url": "https://api.groq.com/openai/v1/models",
"header_template": "Authorization: Bearer {{api_key}}",
"transform_req": {
"chat_completions": {
"url": "https://api.groq.com/openai/v1/chat/completions",
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
}
},
"transform_resp": {
"chat_completions": {
"template": "{{tojson(input_request)}}"
}
},
"explore_models_url": "https://console.groq.com/docs/models"
}
}

23
extensions/engine-management-extension/resources/martian.json
View File

@ -1,23 +0,0 @@
{
"id": "martian",
"type": "remote",
"engine": "martian",
"url": "https://withmartian.com/dashboard",
"api_key": "",
"metadata": {
"get_models_url": "https://withmartian.com/api/openai/v1/models",
"header_template": "Authorization: Bearer {{api_key}}",
"transform_req": {
"chat_completions": {
"url": "https://withmartian.com/api/openai/v1/chat/completions",
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
}
},
"transform_resp": {
"chat_completions": {
"template": "{{tojson(input_request)}}"
}
},
"explore_models_url": "https://withmartian.github.io/llm-adapters/"
}
}

23
extensions/engine-management-extension/resources/mistral.json
View File

@ -1,23 +0,0 @@
{
"id": "mistral",
"type": "remote",
"engine": "mistral",
"url": "https://console.mistral.ai/api-keys/",
"api_key": "",
"metadata": {
"get_models_url": "https://api.mistral.ai/v1/models",
"header_template": "Authorization: Bearer {{api_key}}",
"transform_req": {
"chat_completions": {
"url": "https://api.mistral.ai/v1/chat/completions",
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
}
},
"transform_resp": {
"chat_completions": {
"template": "{{tojson(input_request)}}"
}
},
"explore_models_url": "https://docs.mistral.ai/getting-started/models/models_overview/"
}
}

23
extensions/engine-management-extension/resources/nvidia.json
View File

@ -1,23 +0,0 @@
{
"id": "nvidia",
"type": "remote",
"engine": "nvidia",
"url": "https://org.ngc.nvidia.com/setup/personal-keys",
"api_key": "",
"metadata": {
"get_models_url": "https://integrate.api.nvidia.com/v1/models",
"header_template": "Authorization: Bearer {{api_key}}",
"transform_req": {
"chat_completions": {
"url": "https://integrate.api.nvidia.com/v1/chat/completions",
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
}
},
"transform_resp": {
"chat_completions": {
"template": "{{tojson(input_request)}}"
}
},
"explore_models_url": "https://build.nvidia.com/models"
}
}

23
extensions/engine-management-extension/resources/openai.json
View File

@ -1,23 +0,0 @@
{
"id": "openai",
"type": "remote",
"engine": "openai",
"url": "https://platform.openai.com/account/api-keys",
"api_key": "",
"metadata": {
"get_models_url": "https://api.openai.com/v1/models",
"header_template": "Authorization: Bearer {{api_key}}",
"transform_req": {
"chat_completions": {
"url": "https://api.openai.com/v1/chat/completions",
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"messages\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" or key == \"max_tokens\" or key == \"stop\" %} {% if not first %}, {% endif %} {% if key == \"messages\" and (input_request.model == \"o1\" or input_request.model == \"o1-preview\" or input_request.model == \"o1-mini\") and input_request.messages.0.role == \"system\" %} \"messages\": [ {% for message in input_request.messages %} {% if not loop.is_first %} { \"role\": \"{{ message.role }}\", \"content\": \"{{ message.content }}\" } {% if not loop.is_last %}, {% endif %} {% endif %} {% endfor %} ] {% else if key == \"stop\" and (input_request.model == \"o1\" or input_request.model == \"o1-preview\" or input_request.model == \"o1-mini\" or input_request.model == \"o3\" or input_request.model == \"o3-mini\") %} {% set first = false %} {% else if key == \"max_tokens\" and (input_request.model == \"o1\" or input_request.model == \"o1-preview\" or input_request.model == \"o1-mini\" or input_request.model == \"o3\" or input_request.model == \"o3-mini\") %} \"max_completion_tokens\": {{ tojson(value) }} {% set first = false %} {% else %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endif %} {% endfor %} }"
}
},
"transform_resp": {
"chat_completions": {
"template": "{{tojson(input_request)}}"
}
},
"explore_models_url": "https://platform.openai.com/docs/models"
}
}

23
extensions/engine-management-extension/resources/openrouter.json
View File

@ -1,23 +0,0 @@
{
"id": "openrouter",
"type": "remote",
"engine": "openrouter",
"url": "https://openrouter.ai/keys",
"api_key": "",
"metadata": {
"get_models_url": "https://openrouter.ai/api/v1/models",
"header_template": "Authorization: Bearer {{api_key}}",
"transform_req": {
"chat_completions": {
"url": "https://openrouter.ai/api/v1/chat/completions",
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"model\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
}
},
"transform_resp": {
"chat_completions": {
"template": "{{tojson(input_request)}}"
}
},
"explore_models_url": "https://openrouter.ai/models"
}
}

44
extensions/engine-management-extension/rolldown.config.mjs
View File

@ -1,44 +0,0 @@
import { defineConfig } from 'rolldown'
import { engines, models } from './engines.mjs'
import pkgJson from './package.json' with { type: 'json' }
export default defineConfig([
{
input: 'src/index.ts',
output: {
format: 'esm',
file: 'dist/index.js',
},
define: {
NODE: JSON.stringify(`${pkgJson.name}/${pkgJson.node}`),
API_URL: JSON.stringify(
`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? '39291'}`
),
PLATFORM: JSON.stringify(process.platform),
CORTEX_ENGINE_VERSION: JSON.stringify('b5509'),
DEFAULT_REMOTE_ENGINES: JSON.stringify(engines),
DEFAULT_REMOTE_MODELS: JSON.stringify(models),
DEFAULT_REQUEST_PAYLOAD_TRANSFORM: JSON.stringify(
`{ {% set first = true %} {% for key, value in input_request %} {% if key == "messages" or key == "model" or key == "temperature" or key == "store" or key == "max_tokens" or key == "stream" or key == "presence_penalty" or key == "metadata" or key == "frequency_penalty" or key == "tools" or key == "tool_choice" or key == "logprobs" or key == "top_logprobs" or key == "logit_bias" or key == "n" or key == "modalities" or key == "prediction" or key == "response_format" or key == "service_tier" or key == "seed" or key == "stop" or key == "stream_options" or key == "top_p" or key == "parallel_tool_calls" or key == "user" %} {% if not first %},{% endif %} "{{ key }}": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }`
),
DEFAULT_RESPONSE_BODY_TRANSFORM: JSON.stringify(
'{{tojson(input_request)}}'
),
DEFAULT_REQUEST_HEADERS_TRANSFORM: JSON.stringify(
'Authorization: Bearer {{api_key}}'
),
VERSION: JSON.stringify(pkgJson.version ?? '0.0.0'),
},
},
{
input: 'src/node/index.ts',
external: ['@janhq/core/node'],
output: {
format: 'cjs',
file: 'dist/node/index.cjs.js',
},
define: {
CORTEX_ENGINE_VERSION: JSON.stringify('b5509'),
},
},
])

23
extensions/engine-management-extension/src/@types/global.d.ts vendored
View File

@ -1,23 +0,0 @@
declare const API_URL: string
declare const CORTEX_ENGINE_VERSION: string
declare const PLATFORM: string
declare const NODE: string
declare const DEFAULT_REQUEST_PAYLOAD_TRANSFORM: string
declare const DEFAULT_RESPONSE_BODY_TRANSFORM: string
declare const DEFAULT_REQUEST_HEADERS_TRANSFORM: string
declare const VERSION: string
declare const DEFAULT_REMOTE_ENGINES: ({
id: string
engine: string
} & EngineConfig)[]
declare const DEFAULT_REMOTE_MODELS: Model[]
interface Core {
api: APIFunctions
events: EventEmitter
}
interface Window {
core?: Core | undefined
electronAPI?: any | undefined
}

199
extensions/engine-management-extension/src/api.test.ts
View File

@ -1,199 +0,0 @@
import { describe, beforeEach, it, expect, vi } from 'vitest'
import JanEngineManagementExtension from './index'
import { InferenceEngine } from '@janhq/core'
describe('API methods', () => {
let extension: JanEngineManagementExtension
beforeEach(() => {
// @ts-ignore
extension = new JanEngineManagementExtension()
vi.resetAllMocks()
})
describe('getReleasedEnginesByVersion', () => {
it('should return engines filtered by platform if provided', async () => {
const mockEngines = [
{
name: 'windows-amd64-avx2',
version: '1.0.0',
},
{
name: 'linux-amd64-avx2',
version: '1.0.0',
},
]
vi.mock('ky', () => ({
default: {
get: () => ({
json: () => Promise.resolve(mockEngines),
}),
},
}))
const mock = vi.spyOn(extension, 'getReleasedEnginesByVersion')
mock.mockImplementation(async (name, version, platform) => {
const result = await Promise.resolve(mockEngines)
return platform ? result.filter(r => r.name.includes(platform)) : result
})
const result = await extension.getReleasedEnginesByVersion(
InferenceEngine.cortex_llamacpp,
'1.0.0',
'windows'
)
expect(result).toHaveLength(1)
expect(result[0].name).toBe('windows-amd64-avx2')
})
it('should return all engines if platform is not provided', async () => {
const mockEngines = [
{
name: 'windows-amd64-avx2',
version: '1.0.0',
},
{
name: 'linux-amd64-avx2',
version: '1.0.0',
},
]
vi.mock('ky', () => ({
default: {
get: () => ({
json: () => Promise.resolve(mockEngines),
}),
},
}))
const mock = vi.spyOn(extension, 'getReleasedEnginesByVersion')
mock.mockImplementation(async (name, version, platform) => {
const result = await Promise.resolve(mockEngines)
return platform ? result.filter(r => r.name.includes(platform)) : result
})
const result = await extension.getReleasedEnginesByVersion(
InferenceEngine.cortex_llamacpp,
'1.0.0'
)
expect(result).toHaveLength(2)
})
})
describe('getLatestReleasedEngine', () => {
it('should return engines filtered by platform if provided', async () => {
const mockEngines = [
{
name: 'windows-amd64-avx2',
version: '1.0.0',
},
{
name: 'linux-amd64-avx2',
version: '1.0.0',
},
]
vi.mock('ky', () => ({
default: {
get: () => ({
json: () => Promise.resolve(mockEngines),
}),
},
}))
const mock = vi.spyOn(extension, 'getLatestReleasedEngine')
mock.mockImplementation(async (name, platform) => {
const result = await Promise.resolve(mockEngines)
return platform ? result.filter(r => r.name.includes(platform)) : result
})
const result = await extension.getLatestReleasedEngine(
InferenceEngine.cortex_llamacpp,
'linux'
)
expect(result).toHaveLength(1)
expect(result[0].name).toBe('linux-amd64-avx2')
})
})
describe('installEngine', () => {
it('should send install request with correct parameters', async () => {
const mockEngineConfig = {
variant: 'windows-amd64-avx2',
version: '1.0.0',
}
vi.mock('ky', () => ({
default: {
post: (url, options) => {
expect(url).toBe(`${API_URL}/v1/engines/${InferenceEngine.cortex_llamacpp}/install`)
expect(options.json).toEqual(mockEngineConfig)
return Promise.resolve({ messages: 'OK' })
},
},
}))
const result = await extension.installEngine(
InferenceEngine.cortex_llamacpp,
mockEngineConfig
)
expect(result).toEqual({ messages: 'OK' })
})
})
describe('uninstallEngine', () => {
it('should send uninstall request with correct parameters', async () => {
const mockEngineConfig = {
variant: 'windows-amd64-avx2',
version: '1.0.0',
}
vi.mock('ky', () => ({
default: {
delete: (url, options) => {
expect(url).toBe(`${API_URL}/v1/engines/${InferenceEngine.cortex_llamacpp}/install`)
expect(options.json).toEqual(mockEngineConfig)
return Promise.resolve({ messages: 'OK' })
},
},
}))
const result = await extension.uninstallEngine(
InferenceEngine.cortex_llamacpp,
mockEngineConfig
)
expect(result).toEqual({ messages: 'OK' })
})
})
describe('addRemoteModel', () => {
it('should send add model request with correct parameters', async () => {
const mockModel = {
id: 'gpt-4',
name: 'GPT-4',
engine: InferenceEngine.openai,
}
vi.mock('ky', () => ({
default: {
post: (url, options) => {
expect(url).toBe(`${API_URL}/v1/models/add`)
expect(options.json).toHaveProperty('id', 'gpt-4')
expect(options.json).toHaveProperty('engine', InferenceEngine.openai)
expect(options.json).toHaveProperty('inference_params')
return Promise.resolve()
},
},
}))
await extension.addRemoteModel(mockModel)
// Success is implied by no thrown exceptions
})
})
})

19
extensions/engine-management-extension/src/error.test.ts
View File

@ -1,19 +0,0 @@
import { describe, it, expect } from 'vitest'
import { EngineError } from './error'
describe('EngineError', () => {
it('should create an error with the correct message', () => {
const errorMessage = 'Test error message'
const error = new EngineError(errorMessage)
expect(error).toBeInstanceOf(Error)
expect(error.message).toBe(errorMessage)
expect(error.name).toBe('EngineError')
})
it('should create an error with default message if none provided', () => {
const error = new EngineError()
expect(error.message).toBe('Engine error occurred')
})
})

10
extensions/engine-management-extension/src/error.ts
View File

@ -1,10 +0,0 @@
/**
* Custom Engine Error
*/
export class EngineError extends Error {
message: string
constructor(message: string) {
super()
this.message = message
}
}

449
extensions/engine-management-extension/src/index.test.ts
View File

@ -1,449 +0,0 @@
import { describe, beforeEach, it, expect, vi } from 'vitest'
import JanEngineManagementExtension from './index'
import { Engines, InferenceEngine } from '@janhq/core'
import { EngineError } from './error'
import { HTTPError } from 'ky'
vi.stubGlobal('API_URL', 'http://localhost:3000')
const mockEngines: Engines = [
{
name: 'variant1',
version: '1.0.0',
type: 'local',
engine: InferenceEngine.cortex_llamacpp,
},
]
const mockRemoteEngines: Engines = [
{
name: 'openai',
version: '1.0.0',
type: 'remote',
engine: InferenceEngine.openai,
},
]
const mockRemoteModels = {
data: [
{
id: 'gpt-4',
name: 'GPT-4',
engine: InferenceEngine.openai,
},
],
}
vi.stubGlobal('DEFAULT_REMOTE_ENGINES', mockEngines)
vi.stubGlobal('DEFAULT_REMOTE_MODELS', mockRemoteModels.data)
describe('migrate engine settings', () => {
let extension: JanEngineManagementExtension
beforeEach(() => {
// @ts-ignore
extension = new JanEngineManagementExtension()
vi.resetAllMocks()
})
it('engines should be migrated', async () => {
vi.stubGlobal('VERSION', '2.0.0')
vi.spyOn(extension, 'getEngines').mockResolvedValue([])
const mockUpdateEngines = vi
.spyOn(extension, 'updateEngine')
.mockReturnThis()
mockUpdateEngines.mockResolvedValue({
messages: 'OK',
})
await extension.migrate()
// Assert that the returned value is equal to the mockEngines object
expect(mockUpdateEngines).toBeCalled()
})
it('should not migrate when extension version is not updated', async () => {
vi.stubGlobal('VERSION', '0.0.0')
vi.spyOn(extension, 'getEngines').mockResolvedValue([])
const mockUpdateEngines = vi
.spyOn(extension, 'updateEngine')
.mockReturnThis()
mockUpdateEngines.mockResolvedValue({
messages: 'OK',
})
await extension.migrate()
// Assert that the returned value is equal to the mockEngines object
expect(mockUpdateEngines).not.toBeCalled()
})
})
describe('getEngines', () => {
let extension: JanEngineManagementExtension
beforeEach(() => {
// @ts-ignore
extension = new JanEngineManagementExtension()
vi.resetAllMocks()
})
it('should return a list of engines', async () => {
const mockKyGet = vi.spyOn(extension, 'getEngines')
mockKyGet.mockResolvedValue(mockEngines)
const engines = await extension.getEngines()
expect(engines).toEqual(mockEngines)
})
})
describe('getRemoteModels', () => {
let extension: JanEngineManagementExtension
beforeEach(() => {
// @ts-ignore
extension = new JanEngineManagementExtension()
vi.resetAllMocks()
})
it('should return a list of remote models', async () => {
vi.mock('ky', () => ({
default: {
get: () => ({
json: () => Promise.resolve(mockRemoteModels),
}),
},
}))
const models = await extension.getRemoteModels('openai')
expect(models).toEqual(mockRemoteModels)
})
it('should return empty data array when request fails', async () => {
vi.mock('ky', () => ({
default: {
get: () => ({
json: () => Promise.reject(new Error('Failed to fetch')),
}),
},
}))
const models = await extension.getRemoteModels('openai')
expect(models).toEqual({ data: [] })
})
})
describe('getInstalledEngines', () => {
let extension: JanEngineManagementExtension
beforeEach(() => {
// @ts-ignore
extension = new JanEngineManagementExtension()
vi.resetAllMocks()
})
it('should return a list of installed engines', async () => {
const mockEngineVariants = [
{
name: 'windows-amd64-noavx',
version: '1.0.0',
},
]
vi.mock('ky', () => ({
default: {
get: () => ({
json: () => Promise.resolve(mockEngineVariants),
}),
},
}))
const mock = vi.spyOn(extension, 'getInstalledEngines')
mock.mockResolvedValue(mockEngineVariants)
const engines = await extension.getInstalledEngines(InferenceEngine.cortex_llamacpp)
expect(engines).toEqual(mockEngineVariants)
})
})
describe('healthz', () => {
let extension: JanEngineManagementExtension
beforeEach(() => {
// @ts-ignore
extension = new JanEngineManagementExtension()
vi.resetAllMocks()
})
it('should perform health check successfully', async () => {
vi.mock('ky', () => ({
default: {
get: () => Promise.resolve(),
},
}))
await extension.healthz()
expect(extension.queue.concurrency).toBe(Infinity)
})
})
describe('updateDefaultEngine', () => {
let extension: JanEngineManagementExtension
beforeEach(() => {
// @ts-ignore
extension = new JanEngineManagementExtension()
vi.resetAllMocks()
})
it('should set default engine variant if not installed', async () => {
vi.stubGlobal('PLATFORM', 'win32')
vi.stubGlobal('CORTEX_ENGINE_VERSION', '1.0.0')
const mockGetDefaultEngineVariant = vi.spyOn(
extension,
'getDefaultEngineVariant'
)
mockGetDefaultEngineVariant.mockResolvedValue({
variant: 'variant1',
version: '1.0.0',
})
const mockGetInstalledEngines = vi.spyOn(extension, 'getInstalledEngines')
mockGetInstalledEngines.mockResolvedValue([])
const mockSetDefaultEngineVariant = vi.spyOn(
extension,
'setDefaultEngineVariant'
)
mockSetDefaultEngineVariant.mockResolvedValue({ messages: 'OK' })
vi.mock('@janhq/core', async (importOriginal) => {
const actual = (await importOriginal()) as any
return {
...actual,
systemInformation: vi.fn().mockResolvedValue({ gpuSetting: 'high' }),
}
})
vi.mock('./utils', async (importOriginal) => {
const actual = (await importOriginal()) as any
return {
...actual,
engineVariant: vi.fn().mockResolvedValue('windows-amd64-noavx'),
}
})
await extension.updateDefaultEngine()
expect(mockSetDefaultEngineVariant).toHaveBeenCalledWith('llama-cpp', {
variant: 'windows-amd64-noavx',
version: '1.0.0',
})
})
it('should not reset default engine variant if installed', async () => {
vi.stubGlobal('PLATFORM', 'win32')
vi.stubGlobal('CORTEX_ENGINE_VERSION', '1.0.0')
const mockGetDefaultEngineVariant = vi.spyOn(
extension,
'getDefaultEngineVariant'
)
mockGetDefaultEngineVariant.mockResolvedValue({
variant: 'windows-amd64-noavx',
version: '1.0.0',
})
const mockGetInstalledEngines = vi.spyOn(extension, 'getInstalledEngines')
mockGetInstalledEngines.mockResolvedValue([
{
name: 'windows-amd64-noavx',
version: '1.0.0',
type: 'local',
engine: InferenceEngine.cortex_llamacpp,
},
])
const mockSetDefaultEngineVariant = vi.spyOn(
extension,
'setDefaultEngineVariant'
)
mockSetDefaultEngineVariant.mockResolvedValue({ messages: 'OK' })
vi.mock('@janhq/core', async (importOriginal) => {
const actual = (await importOriginal()) as any
return {
...actual,
systemInformation: vi.fn().mockResolvedValue({ gpuSetting: 'high' }),
}
})
vi.mock('./utils', async (importOriginal) => {
const actual = (await importOriginal()) as any
return {
...actual,
engineVariant: vi.fn().mockResolvedValue('windows-amd64-noavx'),
}
})
await extension.updateDefaultEngine()
expect(mockSetDefaultEngineVariant).not.toBeCalled()
})
it('should handle HTTPError when getting default engine variant', async () => {
vi.stubGlobal('PLATFORM', 'win32')
vi.stubGlobal('CORTEX_ENGINE_VERSION', '1.0.0')
const httpError = new Error('HTTP Error') as HTTPError
httpError.response = { status: 400 } as Response
const mockGetDefaultEngineVariant = vi.spyOn(
extension,
'getDefaultEngineVariant'
)
mockGetDefaultEngineVariant.mockRejectedValue(httpError)
const mockSetDefaultEngineVariant = vi.spyOn(
extension,
'setDefaultEngineVariant'
)
mockSetDefaultEngineVariant.mockResolvedValue({ messages: 'OK' })
vi.mock('@janhq/core', async (importOriginal) => {
const actual = (await importOriginal()) as any
return {
...actual,
systemInformation: vi.fn().mockResolvedValue({ gpuSetting: 'high' }),
}
})
vi.mock('./utils', async (importOriginal) => {
const actual = (await importOriginal()) as any
return {
...actual,
engineVariant: vi.fn().mockResolvedValue('windows-amd64-noavx'),
}
})
await extension.updateDefaultEngine()
expect(mockSetDefaultEngineVariant).toHaveBeenCalledWith('llama-cpp', {
variant: 'windows-amd64-noavx',
version: '1.0.0',
})
})
it('should handle EngineError when getting default engine variant', async () => {
vi.stubGlobal('PLATFORM', 'win32')
vi.stubGlobal('CORTEX_ENGINE_VERSION', '1.0.0')
const mockGetDefaultEngineVariant = vi.spyOn(
extension,
'getDefaultEngineVariant'
)
mockGetDefaultEngineVariant.mockRejectedValue(new EngineError('Test error'))
const mockSetDefaultEngineVariant = vi.spyOn(
extension,
'setDefaultEngineVariant'
)
mockSetDefaultEngineVariant.mockResolvedValue({ messages: 'OK' })
vi.mock('@janhq/core', async (importOriginal) => {
const actual = (await importOriginal()) as any
return {
...actual,
systemInformation: vi.fn().mockResolvedValue({ gpuSetting: 'high' }),
}
})
vi.mock('./utils', async (importOriginal) => {
const actual = (await importOriginal()) as any
return {
...actual,
engineVariant: vi.fn().mockResolvedValue('windows-amd64-noavx'),
}
})
await extension.updateDefaultEngine()
expect(mockSetDefaultEngineVariant).toHaveBeenCalledWith('llama-cpp', {
variant: 'windows-amd64-noavx',
version: '1.0.0',
})
})
it('should handle unexpected errors gracefully', async () => {
vi.stubGlobal('PLATFORM', 'win32')
const mockGetDefaultEngineVariant = vi.spyOn(
extension,
'getDefaultEngineVariant'
)
mockGetDefaultEngineVariant.mockRejectedValue(new Error('Unexpected error'))
const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
await extension.updateDefaultEngine()
expect(consoleSpy).toHaveBeenCalled()
})
})
describe('populateDefaultRemoteEngines', () => {
let extension: JanEngineManagementExtension
beforeEach(() => {
// @ts-ignore
extension = new JanEngineManagementExtension()
vi.resetAllMocks()
})
it('should not add default remote engines if remote engines already exist', async () => {
const mockGetEngines = vi.spyOn(extension, 'getEngines')
mockGetEngines.mockResolvedValue(mockRemoteEngines)
const mockAddRemoteEngine = vi.spyOn(extension, 'addRemoteEngine')
await extension.populateDefaultRemoteEngines()
expect(mockAddRemoteEngine).not.toBeCalled()
})
it('should add default remote engines if no remote engines exist', async () => {
const mockGetEngines = vi.spyOn(extension, 'getEngines')
mockGetEngines.mockResolvedValue([])
const mockAddRemoteEngine = vi.spyOn(extension, 'addRemoteEngine')
mockAddRemoteEngine.mockResolvedValue({ messages: 'OK' })
const mockAddRemoteModel = vi.spyOn(extension, 'addRemoteModel')
mockAddRemoteModel.mockResolvedValue(undefined)
vi.mock('@janhq/core', async (importOriginal) => {
const actual = (await importOriginal()) as any
return {
...actual,
events: {
emit: vi.fn(),
},
joinPath: vi.fn().mockResolvedValue('/path/to/settings.json'),
getJanDataFolderPath: vi.fn().mockResolvedValue('/path/to/data'),
fs: {
existsSync: vi.fn().mockResolvedValue(false),
},
}
})
await extension.populateDefaultRemoteEngines()
expect(mockAddRemoteEngine).toHaveBeenCalled()
expect(mockAddRemoteModel).toHaveBeenCalled()
})
})

412
extensions/engine-management-extension/src/index.ts
View File

@ -1,412 +0,0 @@
import {
EngineManagementExtension,
DefaultEngineVariant,
Engines,
EngineConfig,
EngineVariant,
EngineReleased,
executeOnMain,
systemInformation,
Model,
fs,
joinPath,
events,
ModelEvent,
EngineEvent,
} from '@janhq/core'
import ky, { HTTPError, KyInstance } from 'ky'
import { EngineError } from './error'
import { getJanDataFolderPath } from '@janhq/core'
import { engineVariant } from './utils'
interface ModelList {
data: Model[]
}
/**
* JanEngineManagementExtension is a EngineManagementExtension implementation that provides
* functionality for managing engines.
*/
export default class JanEngineManagementExtension extends EngineManagementExtension {
api?: KyInstance
/**
* Get the API instance
* @returns
*/
async apiInstance(): Promise<KyInstance> {
if (this.api) return this.api
const apiKey = await window.core?.api.appToken()
this.api = ky.extend({
prefixUrl: API_URL,
headers: apiKey
? {
Authorization: `Bearer ${apiKey}`,
}
: {},
retry: 10,
})
return this.api
}
/**
* Called when the extension is loaded.
*/
async onLoad() {
// Update default local engine
// this.updateDefaultEngine()
// Migrate
this.migrate()
}
/**
* Called when the extension is unloaded.
*/
onUnload() { }
/**
* @returns A Promise that resolves to an object of list engines.
*/
async getEngines(): Promise<Engines> {
return {}
return this.apiInstance().then((api) =>
api
.get('v1/engines')
.json<Engines>()
.then((e) => e)
) as Promise<Engines>
}
/**
* @returns A Promise that resolves to an object of list engines.
*/
async getRemoteModels(name: string): Promise<any> {
return this.apiInstance().then(
(api) =>
api
.get(`v1/models/remote/${name}`)
.json<ModelList>()
.catch(() => ({
data: [],
})) as Promise<ModelList>
)
}
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to an array of installed engine.
*/
async getInstalledEngines(name: string): Promise<EngineVariant[]> {
return []
return this.apiInstance().then((api) =>
api
.get(`v1/engines/${name}`)
.json<EngineVariant[]>()
.then((e) => e)
) as Promise<EngineVariant[]>
}
/**
* @param name - Inference engine name.
* @param version - Version of the engine.
* @param platform - Optional to sort by operating system. macOS, linux, windows.
* @returns A Promise that resolves to an array of latest released engine by version.
*/
async getReleasedEnginesByVersion(
name: string,
version: string,
platform?: string
) {
return this.apiInstance().then((api) =>
api
.get(`v1/engines/${name}/releases/${version}`)
.json<EngineReleased[]>()
.then((e) =>
platform ? e.filter((r) => r.name.includes(platform)) : e
)
) as Promise<EngineReleased[]>
}
/**
* @param name - Inference engine name.
* @param platform - Optional to sort by operating system. macOS, linux, windows.
* @returns A Promise that resolves to an array of latest released engine by version.
*/
async getLatestReleasedEngine(name: string, platform?: string) {
return this.apiInstance().then((api) =>
api
.get(`v1/engines/${name}/releases/latest`)
.json<EngineReleased[]>()
.then((e) =>
platform ? e.filter((r) => r.name.includes(platform)) : e
)
) as Promise<EngineReleased[]>
}
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to intall of engine.
*/
async installEngine(name: string, engineConfig: EngineConfig) {
return this.apiInstance().then((api) =>
api
.post(`v1/engines/${name}/install`, { json: engineConfig })
.then((e) => e)
) as Promise<{ messages: string }>
}
/**
* Add a new remote engine
* @returns A Promise that resolves to intall of engine.
*/
async addRemoteEngine(
engineConfig: EngineConfig,
persistModels: boolean = true
) {
// Populate default settings
if (
engineConfig.metadata?.transform_req?.chat_completions &&
!engineConfig.metadata.transform_req.chat_completions.template
)
engineConfig.metadata.transform_req.chat_completions.template =
DEFAULT_REQUEST_PAYLOAD_TRANSFORM
if (
engineConfig.metadata?.transform_resp?.chat_completions &&
!engineConfig.metadata.transform_resp.chat_completions?.template
)
engineConfig.metadata.transform_resp.chat_completions.template =
DEFAULT_RESPONSE_BODY_TRANSFORM
if (engineConfig.metadata && !engineConfig.metadata?.header_template)
engineConfig.metadata.header_template = DEFAULT_REQUEST_HEADERS_TRANSFORM
return this.apiInstance().then((api) =>
api.post('v1/engines', { json: engineConfig }).then((e) => {
if (persistModels && engineConfig.metadata?.get_models_url) {
// Pull /models from remote models endpoint
return this.populateRemoteModels(engineConfig)
.then(() => e)
.catch(() => e)
}
return e
})
) as Promise<{ messages: string }>
}
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to unintall of engine.
*/
async uninstallEngine(name: string, engineConfig: EngineConfig) {
return this.apiInstance().then((api) =>
api
.delete(`v1/engines/${name}/install`, { json: engineConfig })
.then((e) => e)
) as Promise<{ messages: string }>
}
/**
* Add a new remote model
* @param model - Remote model object.
*/
async addRemoteModel(model: Model) {
return this.apiInstance().then((api) =>
api
.post('v1/models/add', {
json: {
inference_params: {
max_tokens: 4096,
temperature: 0.7,
top_p: 0.95,
stream: true,
frequency_penalty: 0,
presence_penalty: 0,
},
...model,
},
})
.then((e) => e)
.then(() => { })
)
}
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to an object of default engine.
*/
async getDefaultEngineVariant(name: string) {
return this.apiInstance().then((api) =>
api
.get(`v1/engines/${name}/default`)
.json<{ messages: string }>()
.then((e) => e)
) as Promise<DefaultEngineVariant>
}
/**
* @body variant - string
* @body version - string
* @returns A Promise that resolves to set default engine.
*/
async setDefaultEngineVariant(name: string, engineConfig: EngineConfig) {
return this.apiInstance().then((api) =>
api
.post(`v1/engines/${name}/default`, { json: engineConfig })
.then((e) => e)
) as Promise<{ messages: string }>
}
/**
* @returns A Promise that resolves to update engine.
*/
async updateEngine(name: string, engineConfig?: EngineConfig) {
return this.apiInstance().then((api) =>
api
.post(`v1/engines/${name}/update`, { json: engineConfig })
.then((e) => e)
) as Promise<{ messages: string }>
}
/**
* Update default local engine
* This is to use built-in engine variant in case there is no default engine set
*/
async updateDefaultEngine() {
const systemInfo = await systemInformation()
try {
const variant = await this.getDefaultEngineVariant('llama-cpp')
if (
(systemInfo.gpuSetting.vulkan && !variant.variant.includes('vulkan')) ||
(systemInfo.gpuSetting.vulkan === false &&
variant.variant.includes('vulkan'))
) {
throw new EngineError('Switch engine.')
}
const installedEngines = await this.getInstalledEngines('llama-cpp')
if (
!installedEngines.some(
(e) => e.name === variant.variant && e.version === variant.version
) ||
variant.version < CORTEX_ENGINE_VERSION
) {
throw new EngineError(
'Default engine is not available, use bundled version.'
)
}
} catch (error) {
if (
(error instanceof HTTPError && error.response.status === 400) ||
error instanceof EngineError
) {
const variant = await engineVariant(systemInfo.gpuSetting)
// TODO: Use correct provider name when moving to llama.cpp extension
await this.setDefaultEngineVariant('llama-cpp', {
variant: variant,
version: `${CORTEX_ENGINE_VERSION}`,
})
} else {
console.error('An unexpected error occurred:', error)
}
}
}
/**
* This is to populate default remote engines in case there is no customized remote engine setting
*/
async populateDefaultRemoteEngines() {
const engines = await this.getEngines()
if (
!Object.values(engines)
.flat()
.some((e) => e.type === 'remote')
) {
await Promise.all(
DEFAULT_REMOTE_ENGINES.map(async (engine) => {
const { id, ...data } = engine
/// BEGIN - Migrate legacy api key settings
let api_key = undefined
if (id) {
const apiKeyPath = await joinPath([
await getJanDataFolderPath(),
'settings',
id,
'settings.json',
])
if (await fs.existsSync(apiKeyPath)) {
const settings = await fs.readFileSync(apiKeyPath, 'utf-8')
api_key = JSON.parse(settings).find(
(e) => e.key === `${data.engine}-api-key`
)?.controllerProps?.value
}
}
data.api_key = api_key
/// END - Migrate legacy api key settings
await this.addRemoteEngine(data, false).catch(console.error)
})
)
events.emit(EngineEvent.OnEngineUpdate, {})
await Promise.all(
DEFAULT_REMOTE_MODELS.map((data: Model) =>
this.addRemoteModel(data).catch(() => { })
)
)
events.emit(ModelEvent.OnModelsUpdate, { fetch: true })
}
}
/**
* Pulls models list from the remote provider and persist
* @param engineConfig
* @returns
*/
private populateRemoteModels = async (engineConfig: EngineConfig) => {
return this.getRemoteModels(engineConfig.engine)
.then((models: ModelList) => {
if (models?.data)
Promise.all(
models.data.map((model) =>
this.addRemoteModel({
...model,
engine: engineConfig.engine,
model: model.model ?? model.id,
}).catch(console.info)
)
).then(() => {
events.emit(ModelEvent.OnModelsUpdate, { fetch: true })
})
})
.catch(console.info)
}
/**
* Update engine settings to the latest version
*/
migrate = async () => {
// Ensure health check is done
const version = await this.getSetting<string>('version', '0.0.0')
const engines = await this.getEngines()
if (version < VERSION) {
console.log('Migrating engine settings...')
// Migrate engine settings
await Promise.all(
DEFAULT_REMOTE_ENGINES.map((engine) => {
const { id, ...data } = engine
data.api_key = engines[id]?.api_key
return this.updateEngine(id, {
...data,
}).catch(console.error)
})
)
await this.updateSettings([
{
key: 'version',
controllerProps: {
value: VERSION,
},
},
])
}
}
}

69
extensions/engine-management-extension/src/node/index.ts
View File

@ -1,69 +0,0 @@
import * as path from 'path'
import {
appResourcePath,
getJanDataFolderPath,
log,
} from '@janhq/core/node'
import { mkdir, readdir, symlink, cp } from 'fs/promises'
import { existsSync } from 'fs'
/**
* Create symlink to each variant for the default bundled version
* If running in AppImage environment, copy files instead of creating symlinks
*/
const symlinkEngines = async () => {
const sourceEnginePath = path.join(
appResourcePath(),
'shared',
'engines',
'llama.cpp'
)
const symlinkEnginePath = path.join(
getJanDataFolderPath(),
'engines',
'llama.cpp'
)
const variantFolders = await readdir(sourceEnginePath)
const isStandalone = process.platform === 'linux'
for (const variant of variantFolders) {
const targetVariantPath = path.join(
sourceEnginePath,
variant,
CORTEX_ENGINE_VERSION
)
const symlinkVariantPath = path.join(
symlinkEnginePath,
variant,
CORTEX_ENGINE_VERSION
)
await mkdir(path.join(symlinkEnginePath, variant), {
recursive: true,
}).catch((error) => log(JSON.stringify(error)))
// Skip if already exists
if (existsSync(symlinkVariantPath)) {
console.log(`Target already exists: ${symlinkVariantPath}`)
continue
}
if (isStandalone) {
// Copy files for AppImage environments instead of symlinking
await cp(targetVariantPath, symlinkVariantPath, { recursive: true }).catch(
(error) => log(JSON.stringify(error))
)
console.log(`Files copied: ${targetVariantPath} -> ${symlinkVariantPath}`)
} else {
// Create symlink for other environments
await symlink(targetVariantPath, symlinkVariantPath, 'junction').catch(
(error) => log(JSON.stringify(error))
)
console.log(`Symlink created: ${targetVariantPath} -> ${symlinkVariantPath}`)
}
}
}
export default {
symlinkEngines,
}

139
extensions/engine-management-extension/src/populateRemoteModels.test.ts
View File

@ -1,139 +0,0 @@
import { describe, beforeEach, it, expect, vi } from 'vitest'
import JanEngineManagementExtension from './index'
import { InferenceEngine } from '@janhq/core'
describe('populateRemoteModels', () => {
let extension: JanEngineManagementExtension
beforeEach(() => {
// @ts-ignore
extension = new JanEngineManagementExtension()
vi.resetAllMocks()
})
it('should populate remote models successfully', async () => {
const mockEngineConfig = {
engine: InferenceEngine.openai,
}
const mockRemoteModels = {
data: [
{
id: 'gpt-4',
name: 'GPT-4',
},
],
}
const mockGetRemoteModels = vi.spyOn(extension, 'getRemoteModels')
mockGetRemoteModels.mockResolvedValue(mockRemoteModels)
const mockAddRemoteModel = vi.spyOn(extension, 'addRemoteModel')
mockAddRemoteModel.mockResolvedValue(undefined)
vi.mock('@janhq/core', async (importOriginal) => {
const actual = (await importOriginal()) as any
return {
...actual,
events: {
emit: vi.fn(),
},
}
})
// Use the private method through index.ts
// @ts-ignore - Accessing private method for testing
await extension.populateRemoteModels(mockEngineConfig)
expect(mockGetRemoteModels).toHaveBeenCalledWith(mockEngineConfig.engine)
expect(mockAddRemoteModel).toHaveBeenCalledWith({
...mockRemoteModels.data[0],
engine: mockEngineConfig.engine,
model: 'gpt-4',
})
})
it('should handle empty data from remote models', async () => {
const mockEngineConfig = {
engine: InferenceEngine.openai,
}
const mockGetRemoteModels = vi.spyOn(extension, 'getRemoteModels')
mockGetRemoteModels.mockResolvedValue({ data: [] })
const mockAddRemoteModel = vi.spyOn(extension, 'addRemoteModel')
vi.mock('@janhq/core', async (importOriginal) => {
const actual = (await importOriginal()) as any
return {
...actual,
events: {
emit: vi.fn(),
},
}
})
// @ts-ignore - Accessing private method for testing
await extension.populateRemoteModels(mockEngineConfig)
expect(mockGetRemoteModels).toHaveBeenCalledWith(mockEngineConfig.engine)
expect(mockAddRemoteModel).not.toHaveBeenCalled()
})
it('should handle errors when getting remote models', async () => {
const mockEngineConfig = {
engine: InferenceEngine.openai,
}
const mockGetRemoteModels = vi.spyOn(extension, 'getRemoteModels')
mockGetRemoteModels.mockRejectedValue(new Error('Failed to fetch models'))
const consoleSpy = vi.spyOn(console, 'info').mockImplementation(() => {})
// @ts-ignore - Accessing private method for testing
await extension.populateRemoteModels(mockEngineConfig)
expect(mockGetRemoteModels).toHaveBeenCalledWith(mockEngineConfig.engine)
expect(consoleSpy).toHaveBeenCalled()
})
it('should handle errors when adding remote models', async () => {
const mockEngineConfig = {
engine: InferenceEngine.openai,
}
const mockRemoteModels = {
data: [
{
id: 'gpt-4',
name: 'GPT-4',
},
],
}
const mockGetRemoteModels = vi.spyOn(extension, 'getRemoteModels')
mockGetRemoteModels.mockResolvedValue(mockRemoteModels)
const mockAddRemoteModel = vi.spyOn(extension, 'addRemoteModel')
mockAddRemoteModel.mockRejectedValue(new Error('Failed to add model'))
const consoleSpy = vi.spyOn(console, 'info').mockImplementation(() => {})
vi.mock('@janhq/core', async (importOriginal) => {
const actual = (await importOriginal()) as any
return {
...actual,
events: {
emit: vi.fn(),
},
}
})
// @ts-ignore - Accessing private method for testing
await extension.populateRemoteModels(mockEngineConfig)
expect(mockGetRemoteModels).toHaveBeenCalledWith(mockEngineConfig.engine)
expect(mockAddRemoteModel).toHaveBeenCalled()
expect(consoleSpy).toHaveBeenCalled()
})
})

90
extensions/engine-management-extension/src/utils.test.ts
View File

@ -1,90 +0,0 @@
import { describe, it, expect, vi } from 'vitest'
import { engineVariant } from './utils'
vi.mock('@janhq/core', () => {
return {
log: () => {},
}
})
describe('engineVariant', () => {
it('should return mac-arm64 when platform is darwin and arch is arm64', async () => {
vi.stubGlobal('PLATFORM', 'darwin')
const result = await engineVariant({
cpu: { arch: 'arm64', instructions: '' },
gpus: [],
vulkan: false,
})
expect(result).toBe('mac-arm64')
})
it('should return mac-amd64 when platform is darwin and arch is not arm64', async () => {
vi.stubGlobal('PLATFORM', 'darwin')
const result = await engineVariant({
cpu: { arch: 'x64', instructions: [] },
gpus: [],
vulkan: false,
})
expect(result).toBe('mac-amd64')
})
it('should return windows-amd64-noavx-cuda-12-0 when platform is win32, cuda is enabled, and cuda version is 12', async () => {
vi.stubGlobal('PLATFORM', 'win32')
const result = await engineVariant({
cpu: { arch: 'x64', instructions: ['avx2'] },
gpus: [
{
activated: true,
version: '12',
additional_information: { driver_version: '1.0' },
},
],
vulkan: false,
})
expect(result).toBe('windows-amd64-avx2-cuda-12-0')
})
it('should return linux-amd64-noavx-cuda-11-7 when platform is linux, cuda is enabled, and cuda version is 11', async () => {
vi.stubGlobal('PLATFORM', 'linux')
const result = await engineVariant({
cpu: { arch: 'x64', instructions: [] },
gpus: [
{
activated: true,
version: '11',
additional_information: { driver_version: '1.0' },
},
],
vulkan: false,
})
expect(result).toBe('linux-amd64-noavx-cuda-11-7')
})
it('should return windows-amd64-vulkan when platform is win32 and vulkan is enabled', async () => {
vi.stubGlobal('PLATFORM', 'win32')
const result = await engineVariant({
cpu: { arch: 'x64', instructions: [] },
gpus: [{ activated: true, version: '12' }],
vulkan: true,
})
expect(result).toBe('windows-amd64-vulkan')
})
it('should return windows-amd64-avx512 when platform is win32, no gpu detected and avx512 cpu instruction is supported', async () => {
vi.stubGlobal('PLATFORM', 'win32')
const result = await engineVariant({
cpu: { arch: 'x64', instructions: ['avx512'] },
gpus: [{ activated: true, version: '12' }],
})
expect(result).toBe('windows-amd64-avx512')
})
it('should return windows-amd64-avx512 when platform is win32, no gpu detected and no accelerated cpu instructions are supported', async () => {
vi.stubGlobal('PLATFORM', 'win32')
const result = await engineVariant({
cpu: { arch: 'x64', instructions: [''] },
gpus: [{ activated: true, version: '12' }],
})
expect(result).toBe('windows-amd64-noavx')
})
})

105
extensions/engine-management-extension/src/utils.ts
View File

@ -1,105 +0,0 @@
import { GpuSetting, log } from '@janhq/core'
// Supported run modes
enum RunMode {
Cuda = 'cuda',
CPU = 'cpu',
}
// Supported instruction sets
const instructionBinaryNames = ['noavx', 'avx', 'avx2', 'avx512']
/**
* The GPU runMode that will be set - either 'vulkan', 'cuda', or empty for cpu.
* @param settings
* @returns
*/
const gpuRunMode = (settings?: GpuSetting): RunMode => {
return settings.gpus?.some(
(gpu) =>
gpu.activated &&
gpu.additional_information &&
gpu.additional_information.driver_version
)
? RunMode.Cuda
: RunMode.CPU
}
/**
* The OS & architecture that the current process is running on.
* @returns win, mac-x64, mac-arm64, or linux
*/
const os = (settings?: GpuSetting): string => {
return PLATFORM === 'win32'
? 'win'
: PLATFORM === 'darwin'
? settings?.cpu?.arch === 'arm64'
? 'macos-arm64'
: 'macos-x64'
: 'linux'
}
/**
* The CUDA version that will be set - either 'cu12.0' or 'cu11.7'.
* @param settings
* @returns
*/
const cudaVersion = (
settings?: GpuSetting
): 'cu12.0' | 'cu11.7' | undefined => {
return settings.gpus?.some((gpu) => gpu.version.includes('12'))
? 'cu12.0'
: 'cu11.7'
}
/**
* The CPU instructions that will be set - either 'avx512', 'avx2', 'avx', or 'noavx'.
* @returns
*/
/**
* Find which variant to run based on the current platform.
*/
export const engineVariant = async (
gpuSetting?: GpuSetting
): Promise<string> => {
const platform = os(gpuSetting)
// There is no need to append the variant extension for mac
if (platform.startsWith('mac')) return platform
const runMode = gpuRunMode(gpuSetting)
// Only Nvidia GPUs have addition_information set and activated by default
let engineVariant =
!gpuSetting?.vulkan &&
(!gpuSetting.gpus?.length ||
gpuSetting.gpus.some((e) => e.additional_information && e.activated))
? [
platform,
...(runMode === RunMode.Cuda
? // For cuda we only need to check if the cpu supports avx2 or noavx - since other binaries are not shipped with the extension
[
gpuSetting.cpu?.instructions.includes('avx2') ||
gpuSetting.cpu?.instructions.includes('avx512')
? 'avx2'
: 'noavx',
runMode,
cudaVersion(gpuSetting),
'x64',
]
: // For cpu only we need to check all available supported instructions
[
(gpuSetting.cpu?.instructions ?? ['noavx']).find((e) =>
instructionBinaryNames.includes(e.toLowerCase())
) ?? 'noavx',
'x64',
]),
].filter(Boolean)
: [platform, 'vulkan', 'x64']
let engineVariantString = engineVariant.join('-')
log(`[CORTEX]: Engine variant: ${engineVariantString}`)
return engineVariantString
}

16
extensions/engine-management-extension/tsconfig.json
View File

@ -1,16 +0,0 @@
{
"compilerOptions": {
"target": "es2016",
"module": "ES6",
"moduleResolution": "node",
"outDir": "./dist",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"strict": false,
"skipLibCheck": true,
"rootDir": "./src",
"resolveJsonModule": true
},
"include": ["./src"],
"exclude": ["src/**/*.test.ts", "rolldown.config.mjs"]
}

5
extensions/hardware-management-extension/jest.config.js
View File

@ -1,5 +0,0 @@
/** @type {import('ts-jest').JestConfigWithTsJest} */
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
}

46
extensions/hardware-management-extension/package.json
View File

@ -1,46 +0,0 @@
{
"name": "@janhq/hardware-management-extension",
"productName": "Hardware Management",
"version": "1.0.0",
"description": "Manages hardware settings.",
"main": "dist/index.js",
"node": "dist/node/index.cjs.js",
"author": "Jan <service@jan.ai>",
"license": "MIT",
"scripts": {
"test": "jest",
"build": "rolldown -c rolldown.config.mjs",
"codesign:darwin": "../../.github/scripts/auto-sign.sh",
"codesign:win32:linux": "echo 'No codesigning required'",
"codesign": "run-script-os",
"build:publish": "rimraf *.tgz --glob || true && yarn build && yarn codesign && npm pack && cpx *.tgz ../../pre-install"
},
"exports": {
".": "./dist/index.js",
"./main": "./dist/module.js"
},
"devDependencies": {
"cpx": "^1.5.0",
"rimraf": "^3.0.2",
"rolldown": "^1.0.0-beta.1",
"run-script-os": "^1.1.6",
"ts-loader": "^9.5.0",
"typescript": "^5.3.3"
},
"dependencies": {
"@janhq/core": "../../core/package.tgz",
"ky": "^1.7.2",
"p-queue": "^8.0.1"
},
"bundledDependencies": [
"@janhq/core"
],
"hardwares": {
"node": ">=18.0.0"
},
"files": [
"dist/*",
"package.json",
"README.md"
]
}

16
extensions/hardware-management-extension/rolldown.config.mjs
View File

@ -1,16 +0,0 @@
import { defineConfig } from 'rolldown'
import pkgJson from './package.json' with { type: 'json' }
export default defineConfig([
{
input: 'src/index.ts',
output: {
format: 'esm',
file: 'dist/index.js',
},
define: {
NODE: JSON.stringify(`${pkgJson.name}/${pkgJson.node}`),
API_URL: JSON.stringify(`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? "39291"}`),
},
},
])

11
extensions/hardware-management-extension/src/@types/global.d.ts vendored
View File

@ -1,11 +0,0 @@
declare const API_URL: string
declare const NODE: string
interface Core {
api: APIFunctions
events: EventEmitter
}
interface Window {
core?: Core | undefined
electronAPI?: any | undefined
}

65
extensions/hardware-management-extension/src/index.ts
View File

@ -1,65 +0,0 @@
import { HardwareManagementExtension, HardwareInformation } from '@janhq/core'
import ky, { KyInstance } from 'ky'
/**
* JSONHardwareManagementExtension is a HardwareManagementExtension implementation that provides
* functionality for managing engines.
*/
export default class JSONHardwareManagementExtension extends HardwareManagementExtension {
/**
* Called when the extension is loaded.
*/
async onLoad() {}
api?: KyInstance
/**
* Get the API instance
* @returns
*/
async apiInstance(): Promise<KyInstance> {
if (this.api) return this.api
const apiKey = (await window.core?.api.appToken())
this.api = ky.extend({
prefixUrl: API_URL,
headers: apiKey
? {
Authorization: `Bearer ${apiKey}`,
}
: {},
retry: 10,
})
return this.api
}
/**
* Called when the extension is unloaded.
*/
onUnload() {}
/**
* @returns A Promise that resolves to an object of hardware.
*/
async getHardware(): Promise<HardwareInformation> {
return this.apiInstance().then((api) =>
api
.get('v1/hardware')
.json<HardwareInformation>()
.then((e) => e)
) as Promise<HardwareInformation>
}
/**
* @returns A Promise that resolves to an object of set gpu activate.
*/
async setActiveGpu(data: { gpus: number[] }): Promise<{
message: string
activated_gpus: number[]
}> {
return this.apiInstance().then((api) =>
api.post('v1/hardware/activate', { json: data }).then((e) => e)
) as Promise<{
message: string
activated_gpus: number[]
}>
}
}

16
extensions/hardware-management-extension/tsconfig.json
View File

@ -1,16 +0,0 @@
{
"compilerOptions": {
"target": "es2016",
"module": "ES6",
"moduleResolution": "node",
"outDir": "./dist",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"strict": false,
"skipLibCheck": true,
"rootDir": "./src",
"resolveJsonModule": true
},
"include": ["./src"],
"exclude": ["src/**/*.test.ts", "rolldown.config.mjs"]
}

35
extensions/llamacpp-extension/src/index.ts
View File

@ -152,6 +152,16 @@ export default class llamacpp_extension extends AIEngine {
]) ])
} }
async getProviderPath(): Promise<string> {
if (!this.providerPath) {
this.providerPath = await joinPath([
await getJanDataFolderPath(),
this.providerId,
])
}
return this.providerPath
}
override async onUnload(): Promise<void> { override async onUnload(): Promise<void> {
// Terminate all active sessions // Terminate all active sessions
for (const [_, sInfo] of this.activeSessions) { for (const [_, sInfo] of this.activeSessions) {
@ -193,7 +203,7 @@ export default class llamacpp_extension extends AIEngine {
// Implement the required LocalProvider interface methods // Implement the required LocalProvider interface methods
override async list(): Promise<modelInfo[]> { override async list(): Promise<modelInfo[]> {
const modelsDir = await joinPath([this.providerPath, 'models']) const modelsDir = await joinPath([await this.getProviderPath(), 'models'])
if (!(await fs.existsSync(modelsDir))) { if (!(await fs.existsSync(modelsDir))) {
return [] return []
} }
@ -262,7 +272,7 @@ export default class llamacpp_extension extends AIEngine {
) )
const configPath = await joinPath([ const configPath = await joinPath([
this.providerPath, await this.getProviderPath(),
'models', 'models',
modelId, modelId,
'model.yml', 'model.yml',
@ -498,7 +508,7 @@ export default class llamacpp_extension extends AIEngine {
console.log('Calling Tauri command llama_load with args:', args) console.log('Calling Tauri command llama_load with args:', args)
const backendPath = await getBackendExePath(backend, version) const backendPath = await getBackendExePath(backend, version)
const libraryPath = await joinPath([this.providerPath, 'lib']) const libraryPath = await joinPath([await this.getProviderPath(), 'lib'])
try { try {
// TODO: add LIBRARY_PATH // TODO: add LIBRARY_PATH
@ -568,7 +578,9 @@ export default class llamacpp_extension extends AIEngine {
if (!response.ok) { if (!response.ok) {
const errorData = await response.json().catch(() => null) const errorData = await response.json().catch(() => null)
throw new Error( throw new Error(
`API request failed with status ${response.status}: ${JSON.stringify(errorData)}` `API request failed with status ${response.status}: ${JSON.stringify(
errorData
)}`
) )
} }
@ -622,7 +634,8 @@ export default class llamacpp_extension extends AIEngine {
} }
override async chat( override async chat(
opts: chatCompletionRequest opts: chatCompletionRequest,
abortController?: AbortController
): Promise<chatCompletion | AsyncIterable<chatCompletionChunk>> { ): Promise<chatCompletion | AsyncIterable<chatCompletionChunk>> {
const sessionInfo = this.findSessionByModel(opts.model) const sessionInfo = this.findSessionByModel(opts.model)
if (!sessionInfo) { if (!sessionInfo) {
@ -630,6 +643,7 @@ export default class llamacpp_extension extends AIEngine {
} }
const baseUrl = `http://localhost:${sessionInfo.port}/v1` const baseUrl = `http://localhost:${sessionInfo.port}/v1`
const url = `${baseUrl}/chat/completions` const url = `${baseUrl}/chat/completions`
console.log('Session Info:', sessionInfo, sessionInfo.api_key)
const headers = { const headers = {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
'Authorization': `Bearer ${sessionInfo.api_key}`, 'Authorization': `Bearer ${sessionInfo.api_key}`,
@ -644,12 +658,15 @@ export default class llamacpp_extension extends AIEngine {
method: 'POST', method: 'POST',
headers, headers,
body, body,
signal: abortController?.signal,
}) })
if (!response.ok) { if (!response.ok) {
const errorData = await response.json().catch(() => null) const errorData = await response.json().catch(() => null)
throw new Error( throw new Error(
`API request failed with status ${response.status}: ${JSON.stringify(errorData)}` `API request failed with status ${response.status}: ${JSON.stringify(
errorData
)}`
) )
} }
@ -657,7 +674,11 @@ export default class llamacpp_extension extends AIEngine {
} }
override async delete(modelId: string): Promise<void> { override async delete(modelId: string): Promise<void> {
const modelDir = await joinPath([this.providerPath, 'models', modelId]) const modelDir = await joinPath([
await this.getProviderPath(),
'models',
modelId,
])
if (!(await fs.existsSync(await joinPath([modelDir, 'model.yml'])))) { if (!(await fs.existsSync(await joinPath([modelDir, 'model.yml'])))) {
throw new Error(`Model ${modelId} does not exist`) throw new Error(`Model ${modelId} does not exist`)

75
extensions/model-extension/README.md
View File

@ -1,75 +0,0 @@
# Create a Jan Extension using Typescript
Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀
## Create Your Own Extension
To create your own extension, you can use this repository as a template! Just follow the below instructions:
1. Click the Use this template button at the top of the repository
2. Select Create a new repository
3. Select an owner and name for your new repository
4. Click Create repository
5. Clone your new repository
## Initial Setup
After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension.
> [!NOTE]
>
> You'll need to have a reasonably modern version of
> [Node.js](https://nodejs.org) handy. If you are using a version manager like
> [`nodenv`](https://github.com/nodenv/nodenv) or
> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the
> root of your repository to install the version specified in
> [`package.json`](./package.json). Otherwise, 20.x or later should work!
1. :hammer_and_wrench: Install the dependencies
```bash
npm install
```
1. :building_construction: Package the TypeScript for distribution
```bash
npm run bundle
```
1. :white_check_mark: Check your artifact
There will be a tgz file in your extension directory now
## Update the Extension Metadata
The [`package.json`](package.json) file defines metadata about your extension, such as
extension name, main entry, description and version.
When you copy this repository, update `package.json` with the name, description for your extension.
## Update the Extension Code
The [`src/`](./src/) directory is the heart of your extension! This contains the
source code that will be run when your extension functions are invoked. You can replace the
contents of this directory with your own code.
There are a few things to keep in mind when writing your extension code:
- Most Jan Extension functions are processed asynchronously.
In `index.ts`, you will see that the extension function will return a `Promise<any>`.
```typescript
import { events, MessageEvent, MessageRequest } from '@janhq/core'
function onStart(): Promise<any> {
return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) =>
this.inference(data)
)
}
```
For more information about the Jan Extension Core module, see the
[documentation](https://github.com/menloresearch/jan/blob/main/core/README.md).
So, what are you waiting for? Go ahead and start customizing your extension!

37
extensions/model-extension/package.json
View File

@ -1,37 +0,0 @@
{
"name": "@janhq/model-extension",
"productName": "Model Management",
"version": "1.0.36",
"description": "Manages model operations including listing, importing, updating, and deleting.",
"main": "dist/index.js",
"author": "Jan <service@jan.ai>",
"license": "AGPL-3.0",
"scripts": {
"test": "vitest run",
"build": "rolldown -c rolldown.config.mjs",
"build:publish": "rimraf *.tgz --glob || true && yarn build && npm pack && cpx *.tgz ../../pre-install"
},
"devDependencies": {
"cpx": "^1.5.0",
"rimraf": "^3.0.2",
"rolldown": "1.0.0-beta.1",
"run-script-os": "^1.1.6",
"typescript": "5.3.3",
"vitest": "^3.0.6"
},
"files": [
"dist/*",
"package.json",
"README.md"
],
"dependencies": {
"@janhq/core": "../../core/package.tgz",
"ky": "^1.7.2",
"p-queue": "^8.0.1"
},
"bundleDependencies": [],
"installConfig": {
"hoistingLimits": "workspaces"
},
"packageManager": "yarn@4.5.3"
}

6635
extensions/model-extension/resources/default.json
View File

File diff suppressed because it is too large Load Diff

14
extensions/model-extension/resources/settings.json
View File

@ -1,14 +0,0 @@
[
{
"key": "hugging-face-access-token",
"title": "Hugging Face Access Token",
"description": "Access tokens programmatically authenticate your identity to the Hugging Face Hub, allowing applications to perform specific actions specified by the scope of permissions granted.",
"controllerType": "input",
"controllerProps": {
"value": "",
"placeholder": "hf_**********************************",
"type": "password",
"inputActions": ["unobscure", "copy"]
}
}
]

17
extensions/model-extension/rolldown.config.mjs
View File

@ -1,17 +0,0 @@
import { defineConfig } from 'rolldown'
import settingJson from './resources/settings.json' with { type: 'json' }
import modelSources from './resources/default.json' with { type: 'json' }
export default defineConfig({
input: 'src/index.ts',
output: {
format: 'esm',
file: 'dist/index.js',
},
platform: 'browser',
define: {
SETTINGS: JSON.stringify(settingJson),
CORTEX_API_URL: JSON.stringify(`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? "39291"}`),
DEFAULT_MODEL_SOURCES: JSON.stringify(modelSources),
},
})

13
extensions/model-extension/src/@types/global.d.ts vendored
View File

@ -1,13 +0,0 @@
declare const NODE: string
declare const CORTEX_API_URL: string
declare const SETTINGS: SettingComponentProps[]
declare const DEFAULT_MODEL_SOURCES: any
interface Core {
api: APIFunctions
events: EventEmitter
}
interface Window {
core?: Core | undefined
electronAPI?: any | undefined
}

88
extensions/model-extension/src/index.test.ts
View File

@ -1,88 +0,0 @@
import { describe, it, expect, beforeEach, vi } from 'vitest'
import JanModelExtension from './index'
import ky from 'ky'
import { ModelManager } from '@janhq/core'
const API_URL = 'http://localhost:3000'
vi.stubGlobal('API_URL', API_URL)
describe('JanModelExtension', () => {
let extension: JanModelExtension
beforeEach(() => {
extension = new JanModelExtension()
vi.spyOn(ModelManager, 'instance').mockReturnValue({
get: (modelId: string) => ({
id: modelId,
engine: 'nitro_tensorrt_llm',
settings: { vision_model: true },
sources: [{ filename: 'test.bin' }],
}),
} as any)
vi.spyOn(JanModelExtension.prototype, 'cancelModelPull').mockImplementation(
async (model: string) => {
const kyDeleteSpy = vi.spyOn(ky, 'delete').mockResolvedValue({
json: () => Promise.resolve({}),
} as any)
await ky.delete(`${API_URL}/v1/models/pull`, {
json: { taskId: model },
})
expect(kyDeleteSpy).toHaveBeenCalledWith(`${API_URL}/v1/models/pull`, {
json: { taskId: model },
})
kyDeleteSpy.mockRestore() // Restore the original implementation
}
)
})
it('should initialize with an empty queue', () => {
expect(extension.queue.size).toBe(0)
})
describe('pullModel', () => {
it('should call the pull model endpoint with correct parameters', async () => {
const model = 'test-model'
const id = 'test-id'
const name = 'test-name'
const kyPostSpy = vi.spyOn(ky, 'post').mockReturnValue({
json: () => Promise.resolve({}),
} as any)
await extension.pullModel(model, id, name)
expect(kyPostSpy).toHaveBeenCalledWith(`${API_URL}/v1/models/pull`, {
json: { model, id, name },
})
kyPostSpy.mockRestore() // Restore the original implementation
})
})
describe('cancelModelPull', () => {
it('should call the cancel model pull endpoint with the correct model', async () => {
const model = 'test-model'
await extension.cancelModelPull(model)
})
})
describe('deleteModel', () => {
it('should call the delete model endpoint with the correct model', async () => {
const model = 'test-model'
const kyDeleteSpy = vi
.spyOn(ky, 'delete')
.mockResolvedValue({ json: () => Promise.resolve({}) } as any)
await extension.deleteModel(model)
expect(kyDeleteSpy).toHaveBeenCalledWith(`${API_URL}/v1/models/${model}`)
kyDeleteSpy.mockRestore() // Restore the original implementation
})
})
})

436
extensions/model-extension/src/index.ts
View File

@ -1,436 +0,0 @@
import {
ModelExtension,
Model,
joinPath,
dirName,
fs,
OptionType,
ModelSource,
extractInferenceParams,
extractModelLoadParams,
} from '@janhq/core'
import { scanModelsFolder } from './legacy/model-json'
import { deleteModelFiles } from './legacy/delete'
import ky, { KyInstance } from 'ky'
/**
* cortex.cpp setting keys
*/
export enum Settings {
huggingfaceToken = 'hugging-face-access-token',
}
/** Data List Response Type */
type Data<T> = {
data: T[]
}
/**
* Defaul mode sources
*/
const defaultModelSources = ['Menlo/Jan-nano-gguf', 'Menlo/Jan-nano-128k-gguf']
/**
* A extension for models
*/
export default class JanModelExtension extends ModelExtension {
api?: KyInstance
/**
* Get the API instance
* @returns
*/
async apiInstance(): Promise<KyInstance> {
if (this.api) return this.api
const apiKey = await window.core?.api.appToken()
this.api = ky.extend({
prefixUrl: CORTEX_API_URL,
headers: apiKey
? {
Authorization: `Bearer ${apiKey}`,
}
: {},
retry: 10,
})
return this.api
}
/**
* Called when the extension is loaded.
*/
async onLoad() {
this.registerSettings(SETTINGS)
// Configure huggingface token if available
const huggingfaceToken = await this.getSetting<string>(
Settings.huggingfaceToken,
undefined
)
if (huggingfaceToken) {
this.updateCortexConfig({ huggingface_token: huggingfaceToken })
}
// Sync with cortexsohub
this.fetchModelsHub()
}
/**
* Subscribe to settings update and make change accordingly
* @param key
* @param value
*/
onSettingUpdate<T>(key: string, value: T): void {
if (key === Settings.huggingfaceToken) {
this.updateCortexConfig({ huggingface_token: value })
}
}
/**
* Called when the extension is unloaded.
* @override
*/
async onUnload() { }
// BEGIN: - Public API
/**
* Downloads a machine learning model.
* @param model - The model to download.
* @returns A Promise that resolves when the model is downloaded.
*/
async pullModel(model: string, id?: string, name?: string): Promise<void> {
/**
* Sending POST to /models/pull/{id} endpoint to pull the model
*/
return this.apiInstance().then((api) =>
api
.post('v1/models/pull', { json: { model, id, name }, timeout: false })
.json()
.catch(async (e) => {
throw (await e.response?.json()) ?? e
})
.then()
)
}
/**
* Cancels the download of a specific machine learning model.
*
* @param {string} model - The ID of the model whose download is to be cancelled.
* @returns {Promise<void>} A promise that resolves when the download has been cancelled.
*/
async cancelModelPull(model: string): Promise<void> {
/**
* Sending DELETE to /models/pull/{id} endpoint to cancel a model pull
*/
return this.apiInstance().then((api) =>
api
.delete('v1/models/pull', { json: { taskId: model } })
.json()
.then()
)
}
/**
* Deletes a pulled model
* @param model - The model to delete
* @returns A Promise that resolves when the model is deleted.
*/
async deleteModel(model: string): Promise<void> {
return this.apiInstance()
.then((api) => api.delete(`v1/models/${model}`).json().then())
.catch((e) => console.debug(e))
.finally(async () => {
// Delete legacy model files
await deleteModelFiles(model).catch((e) => console.debug(e))
}) as Promise<void>
}
/**
* Gets all pulled models
* @returns A Promise that resolves with an array of all models.
*/
async getModels(): Promise<Model[]> {
/**
* Legacy models should be supported
*/
let legacyModels = await scanModelsFolder()
/**
* Here we are filtering out the models that are not imported
* and are not using llama.cpp engine
*/
var toImportModels = legacyModels.filter((e) => e.engine === 'nitro')
/**
* Fetch models from cortex.cpp
*/
var fetchedModels = await this.fetchModels().catch(() => [])
// Checking if there are models to import
const existingIds = fetchedModels.map((e) => e.id)
toImportModels = toImportModels.filter(
(e: Model) => !existingIds.includes(e.id) && !e.settings?.vision_model
)
/**
* There is no model to import
* just return fetched models
*/
if (!toImportModels.length)
return fetchedModels.concat(
legacyModels.filter((e) => !fetchedModels.some((x) => x.id === e.id))
)
console.log('To import models:', toImportModels.length)
/**
* There are models to import
*/
if (toImportModels.length > 0) {
// Import models
await Promise.all(
toImportModels.map(async (model: Model & { file_path: string }) => {
return this.importModel(
model.id,
model.sources?.[0]?.url.startsWith('http') ||
!(await fs.existsSync(model.sources?.[0]?.url))
? await joinPath([
await dirName(model.file_path),
model.sources?.[0]?.filename ??
model.settings?.llama_model_path ??
model.sources?.[0]?.url.split('/').pop() ??
model.id,
]) // Copied models
: model.sources?.[0]?.url, // Symlink models,
model.name
)
.then((e) => {
this.updateModel({
id: model.id,
...model.settings,
...model.parameters,
} as Partial<Model>)
})
.catch((e) => {
console.debug(e)
})
})
)
}
/**
* Models are imported successfully before
* Now return models from cortex.cpp and merge with legacy models which are not imported
*/
return await this.fetchModels()
.then((models) => {
return models.concat(
legacyModels.filter((e) => !models.some((x) => x.id === e.id))
)
})
.catch(() => Promise.resolve(legacyModels))
}
/**
* Update a pulled model metadata
* @param model - The metadata of the model
*/
async updateModel(model: Partial<Model>): Promise<Model> {
return this.apiInstance()
.then((api) =>
api
.patch(`v1/models/${model.id}`, {
json: { ...model },
timeout: false,
})
.json()
.then()
)
.then(() => this.getModel(model.id))
}
/**
* Get a model by its ID
* @param model - The ID of the model
*/
async getModel(model: string): Promise<Model> {
return this.apiInstance().then((api) =>
api
.get(`v1/models/${model}`)
.json()
.then((e) => this.transformModel(e))
) as Promise<Model>
}
/**
* Import an existing model file
* @param model
* @param optionType
*/
async importModel(
model: string,
modelPath: string,
name?: string,
option?: OptionType
): Promise<void> {
return this.apiInstance().then((api) =>
api
.post('v1/models/import', {
json: { model, modelPath, name, option },
timeout: false,
})
.json()
.catch((e) => console.debug(e)) // Ignore error
.then()
)
}
// BEGIN - Model Sources
/**
* Get model sources
* @param model
*/
async getSources(): Promise<ModelSource[]> {
return []
const sources = await this.apiInstance()
.then((api) => api.get('v1/models/sources').json<Data<ModelSource>>())
.then((e) => (typeof e === 'object' ? (e.data as ModelSource[]) : []))
// Deprecated source - filter out from legacy sources
.then((e) => e.filter((x) => x.id.toLowerCase() !== 'menlo/jan-nano'))
.catch(() => [])
return sources.concat(
DEFAULT_MODEL_SOURCES.filter((e) => !sources.some((x) => x.id === e.id))
)
}
/**
* Add a model source
* @param model
*/
async addSource(source: string): Promise<any> {
return
return this.apiInstance().then((api) =>
api.post('v1/models/sources', {
json: {
source,
},
})
)
}
/**
* Delete a model source
* @param model
*/
async deleteSource(source: string): Promise<any> {
return this.apiInstance().then((api) =>
api.delete('v1/models/sources', {
json: {
source,
},
timeout: false,
})
)
}
// END - Model Sources
/**
* Check model status
* @param model
*/
async isModelLoaded(model: string): Promise<boolean> {
return this.apiInstance()
.then((api) => api.get(`v1/models/status/${model}`))
.then((e) => true)
.catch(() => false)
}
/**
* Configure pull options such as proxy, headers, etc.
*/
async configurePullOptions(options: { [key: string]: any }): Promise<any> {
return this.updateCortexConfig(options).catch((e) => console.debug(e))
}
/**
* Fetches models list from cortex.cpp
* @param model
* @returns
*/
async fetchModels(): Promise<Model[]> {
return []
return this.apiInstance()
.then((api) => api.get('v1/models?limit=-1').json<Data<Model>>())
.then((e) =>
typeof e === 'object' ? e.data.map((e) => this.transformModel(e)) : []
)
}
// END: - Public API
// BEGIN: - Private API
/**
* Transform model to the expected format (e.g. parameters, settings, metadata)
* @param model
* @returns
*/
private transformModel(model: any) {
model.parameters = {
...extractInferenceParams(model),
...model.parameters,
...model.inference_params,
}
model.settings = {
...extractModelLoadParams(model),
...model.settings,
}
model.metadata = model.metadata ?? {
tags: [],
size: model.size ?? model.metadata?.size ?? 0,
}
return model as Model
}
/**
* Update cortex config
* @param body
*/
private async updateCortexConfig(body: {
[key: string]: any
}): Promise<void> {
return this.apiInstance()
.then((api) => api.patch('v1/configs', { json: body }).then(() => { }))
.catch((e) => console.debug(e))
}
/**
* Fetch models from cortex.so
*/
fetchModelsHub = async () => {
return
const models = await this.fetchModels()
defaultModelSources.forEach((model) => {
this.addSource(model).catch((e) => {
console.debug(`Failed to add default model source ${model}:`, e)
})
})
return this.apiInstance()
.then((api) =>
api
.get('v1/models/hub?author=cortexso&tag=cortex.cpp')
.json<Data<string>>()
.then(async (e) => {
await Promise.all(
[...(e.data ?? []), ...defaultModelSources].map((model) => {
if (
!models.some(
(e) => 'modelSource' in e && e.modelSource === model
)
)
return this.addSource(model).catch((e) => console.debug(e))
})
)
})
)
.catch((e) => console.debug(e))
}
// END: - Private API
}

13
extensions/model-extension/src/legacy/delete.ts
View File

@ -1,13 +0,0 @@
import { dirName, fs } from '@janhq/core'
import { scanModelsFolder } from './model-json'
export const deleteModelFiles = async (id: string) => {
try {
const models = await scanModelsFolder()
const dirPath = models.find((e) => e.id === id)?.file_path
// remove model folder directory
if (dirPath) await fs.rm(await dirName(dirPath))
} catch (err) {
console.error(err)
}
}

89
extensions/model-extension/src/legacy/model-json.test.ts
View File

@ -1,89 +0,0 @@
import { describe, it, expect, beforeEach, vi } from 'vitest'
import { scanModelsFolder, getModelJsonPath } from './model-json'
// Mock the @janhq/core module
vi.mock('@janhq/core', () => ({
InferenceEngine: {
nitro: 'nitro',
},
fs: {
existsSync: vi.fn(),
readdirSync: vi.fn(),
fileStat: vi.fn(),
readFileSync: vi.fn(),
},
joinPath: vi.fn((paths) => paths.join('/')),
}))
// Import the mocked fs and joinPath after the mock is set up
import { fs } from '@janhq/core'
describe('model-json', () => {
beforeEach(() => {
vi.clearAllMocks()
})
describe('scanModelsFolder', () => {
it('should return an empty array when models folder does not exist', async () => {
vi.spyOn(fs, 'existsSync').mockReturnValue(false)
const result = await scanModelsFolder()
expect(result).toEqual([])
})
it('should return an array of models when valid model folders exist', async () => {
const mockModelJson = {
id: 'test-model',
sources: [
{
filename: 'test-model',
url: 'file://models/test-model/test-model.gguf',
},
],
}
vi.spyOn(fs, 'existsSync').mockReturnValue(true)
vi.spyOn(fs, 'readdirSync').mockReturnValueOnce(['test-model'])
vi.spyOn(fs, 'fileStat').mockResolvedValue({ isDirectory: () => true })
vi.spyOn(fs, 'readFileSync').mockReturnValue(
JSON.stringify(mockModelJson)
)
vi.spyOn(fs, 'readdirSync').mockReturnValueOnce([
'test-model.gguf',
'model.json',
])
const result = await scanModelsFolder()
expect(result).toHaveLength(1)
expect(result[0]).toMatchObject(mockModelJson)
})
})
describe('getModelJsonPath', () => {
it('should return undefined when folder does not exist', async () => {
vi.spyOn(fs, 'existsSync').mockReturnValue(false)
const result = await getModelJsonPath('non-existent-folder')
expect(result).toBeUndefined()
})
it('should return the path when model.json exists in the root folder', async () => {
vi.spyOn(fs, 'existsSync').mockReturnValue(true)
vi.spyOn(fs, 'readdirSync').mockReturnValue(['model.json'])
const result = await getModelJsonPath('test-folder')
expect(result).toBe('test-folder/model.json')
})
it('should return the path when model.json exists in a subfolder', async () => {
vi.spyOn(fs, 'existsSync').mockReturnValue(true)
vi.spyOn(fs, 'readdirSync')
.mockReturnValueOnce(['subfolder'])
.mockReturnValueOnce(['model.json'])
vi.spyOn(fs, 'fileStat').mockResolvedValue({ isDirectory: () => true })
const result = await getModelJsonPath('test-folder')
expect(result).toBe('test-folder/subfolder/model.json')
})
})
})

141
extensions/model-extension/src/legacy/model-json.ts
View File

@ -1,141 +0,0 @@
import { Model, fs, joinPath } from '@janhq/core'
//// LEGACY MODEL FOLDER ////
/**
* Scan through models folder and return downloaded models
* @returns
*/
export const scanModelsFolder = async (): Promise<
(Model & { file_path?: string })[]
> => {
const _homeDir = 'file://models'
try {
if (!(await fs.existsSync(_homeDir))) {
console.debug('Model folder not found')
return []
}
const files: string[] = await fs.readdirSync(_homeDir)
const allDirectories: string[] = []
for (const modelFolder of files) {
const fullModelFolderPath = await joinPath([_homeDir, modelFolder])
if (!(await fs.fileStat(fullModelFolderPath)).isDirectory) continue
allDirectories.push(modelFolder)
}
const readJsonPromises = allDirectories.map(async (dirName) => {
// filter out directories that don't match the selector
// read model.json
const folderFullPath = await joinPath([_homeDir, dirName])
const jsonPath = await getModelJsonPath(folderFullPath)
if (jsonPath && (await fs.existsSync(jsonPath))) {
// if we have the model.json file, read it
let model = await fs.readFileSync(jsonPath, 'utf-8')
model = typeof model === 'object' ? model : JSON.parse(model)
// This to ensure backward compatibility with `model.json` with `source_url`
if (model['source_url'] != null) {
model['sources'] = [
{
filename: model.id,
url: model['source_url'],
},
]
}
model.file_path = jsonPath
model.file_name = 'model.json'
// Check model file exist
// model binaries (sources) are absolute path & exist (symlinked)
const existFiles = await Promise.all(
model.sources.map(
(source) =>
// Supposed to be a local file url
!source.url.startsWith(`http://`) &&
!source.url.startsWith(`https://`)
)
)
if (
!['cortex', 'llama-cpp', 'nitro'].includes(model.engine) ||
existFiles.every((exist) => exist)
)
return model
const result = await fs
.readdirSync(await joinPath([_homeDir, dirName]))
.then((files: string[]) => {
// Model binary exists in the directory
// Model binary name can match model ID or be a .gguf file and not be an incompleted model file
return (
files.includes(dirName) || // Legacy model GGUF without extension
files.filter((file) => {
return (
file.toLowerCase().endsWith('.gguf') || // GGUF
file.toLowerCase().endsWith('.engine') // Tensort-LLM
)
})?.length >=
(model.engine === 'nitro-tensorrt-llm'
? 1
: model.sources?.length ?? 1)
)
})
if (result) return model
else return undefined
}
})
const results = await Promise.allSettled(readJsonPromises)
const modelData = results
.map((result) => {
if (result.status === 'fulfilled' && result.value) {
try {
const model =
typeof result.value === 'object'
? result.value
: JSON.parse(result.value)
return model as Model
} catch {
console.debug(`Unable to parse model metadata: ${result.value}`)
}
}
return undefined
})
.filter(Boolean)
return modelData
} catch (err) {
console.error(err)
return []
}
}
/**
* Retrieve the model.json path from a folder
* @param folderFullPath
* @returns
*/
export const getModelJsonPath = async (
folderFullPath: string
): Promise<string | undefined> => {
// try to find model.json recursively inside each folder
if (!(await fs.existsSync(folderFullPath))) return undefined
const files: string[] = await fs.readdirSync(folderFullPath)
if (files.length === 0) return undefined
if (files.includes('model.json')) {
return joinPath([folderFullPath, 'model.json'])
}
// continue recursive
for (const file of files) {
const path = await joinPath([folderFullPath, file])
const fileStats = await fs.fileStat(path)
if (fileStats.isDirectory) {
const result = await getModelJsonPath(path)
if (result) return result
}
}
}
//// END LEGACY MODEL FOLDER ////

160
extensions/model-extension/src/migration.test.ts
View File

@ -1,160 +0,0 @@
import { describe, it, expect, beforeEach, vi } from 'vitest'
vi.stubGlobal('API_URL', 'http://localhost:3000')
// Mock the @janhq/core module
vi.mock('@janhq/core', (actual) => ({
...actual,
ModelExtension: class {},
InferenceEngine: {
nitro: 'nitro',
},
joinPath: vi.fn(),
dirName: vi.fn(),
fs: {
existsSync: vi.fn(),
readFileSync: vi.fn(),
writeFileSync: vi.fn(),
mkdirSync: vi.fn(),
},
}))
import { Model, InferenceEngine } from '@janhq/core'
import JanModelExtension from './index'
// Mock the model-json module
vi.mock('./legacy/model-json', () => ({
scanModelsFolder: vi.fn(),
}))
// Import the mocked scanModelsFolder after the mock is set up
import * as legacy from './legacy/model-json'
describe('JanModelExtension', () => {
let extension: JanModelExtension
let mockLocalStorage: { [key: string]: string }
beforeEach(() => {
// @ts-ignore
extension = new JanModelExtension()
mockLocalStorage = {}
// Mock localStorage
Object.defineProperty(global, 'localStorage', {
value: {
getItem: vi.fn((key) => mockLocalStorage[key]),
setItem: vi.fn((key, value) => {
mockLocalStorage[key] = value
}),
},
writable: true,
})
})
describe('getModels', () => {
it('should scan models folder when localStorage is empty', async () => {
const mockModels: Model[] = [
{
id: 'model1',
object: 'model',
version: '1',
format: 'gguf',
engine: InferenceEngine.nitro,
sources: [
{ filename: 'model1.gguf', url: 'file://models/model1.gguf' },
],
file_path: '/path/to/model1',
},
{
id: 'model2',
object: 'model',
version: '1',
format: 'gguf',
engine: InferenceEngine.nitro,
sources: [
{ filename: 'model2.gguf', url: 'file://models/model2.gguf' },
],
file_path: '/path/to/model2',
},
] as any
vi.mocked(legacy.scanModelsFolder).mockResolvedValue(mockModels)
vi.spyOn(extension, 'fetchModels').mockResolvedValue([mockModels[0]])
vi.spyOn(extension, 'updateModel').mockResolvedValue(undefined)
vi.spyOn(extension, 'importModel').mockResolvedValueOnce(mockModels[1])
vi.spyOn(extension, 'fetchModels').mockResolvedValue([mockModels[0], mockModels[1]])
const result = await extension.getModels()
expect(legacy.scanModelsFolder).toHaveBeenCalled()
expect(result).toEqual(mockModels)
})
it('should import models when there are models to import', async () => {
const mockModels: Model[] = [
{
id: 'model1',
object: 'model',
version: '1',
format: 'gguf',
engine: InferenceEngine.nitro,
file_path: '/path/to/model1',
sources: [
{ filename: 'model1.gguf', url: 'file://models/model1.gguf' },
],
},
{
id: 'model2',
object: 'model',
version: '1',
format: 'gguf',
engine: InferenceEngine.nitro,
file_path: '/path/to/model2',
sources: [
{ filename: 'model2.gguf', url: 'file://models/model2.gguf' },
],
},
] as any
mockLocalStorage['downloadedModels'] = JSON.stringify(mockModels)
vi.spyOn(extension, 'updateModel').mockResolvedValue(undefined)
vi.spyOn(extension, 'importModel').mockResolvedValue(undefined)
const result = await extension.getModels()
expect(extension.importModel).toHaveBeenCalledTimes(2)
expect(result).toEqual(mockModels)
})
it('should return models from cortexAPI when all models are already imported', async () => {
const mockModels: Model[] = [
{
id: 'model1',
object: 'model',
version: '1',
format: 'gguf',
engine: InferenceEngine.nitro,
sources: [
{ filename: 'model1.gguf', url: 'file://models/model1.gguf' },
],
},
{
id: 'model2',
object: 'model',
version: '1',
format: 'gguf',
engine: InferenceEngine.nitro,
sources: [
{ filename: 'model2.gguf', url: 'file://models/model2.gguf' },
],
},
] as any
mockLocalStorage['downloadedModels'] = JSON.stringify(mockModels)
vi.spyOn(extension, 'fetchModels').mockResolvedValue(mockModels)
extension.getModels = vi.fn().mockResolvedValue(mockModels)
const result = await extension.getModels()
expect(extension.getModels).toHaveBeenCalled()
expect(result).toEqual(mockModels)
})
})
})

15
extensions/model-extension/tsconfig.json
View File

@ -1,15 +0,0 @@
{
"compilerOptions": {
"target": "es2016",
"module": "esnext",
"moduleResolution": "node",
"outDir": "./dist",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"strict": false,
"skipLibCheck": true,
"rootDir": "./src"
},
"include": ["./src"],
"exclude": ["**/*.test.ts", "vite.config.ts"]
}

8
extensions/model-extension/vite.config.ts
View File

@ -1,8 +0,0 @@
import { defineConfig } from "vite"
export default defineConfig(({ mode }) => ({
define: process.env.VITEST ? {} : { global: 'window' },
test: {
environment: 'jsdom',
},
}))

5
mise.toml
View File

@ -54,9 +54,7 @@ depends = ["build-extensions"]
description = "Start development server (matches Makefile)" description = "Start development server (matches Makefile)"
depends = ["install-and-build"] depends = ["install-and-build"]
run = [ run = [
"yarn install:cortex",
"yarn download:bin", "yarn download:bin",
"yarn copy:lib",
"yarn dev" "yarn dev"
] ]
@ -64,9 +62,7 @@ run = [
description = "Start development server with Tauri (DEPRECATED - matches Makefile)" description = "Start development server with Tauri (DEPRECATED - matches Makefile)"
depends = ["install-and-build"] depends = ["install-and-build"]
run = [ run = [
"yarn install:cortex",
"yarn download:bin", "yarn download:bin",
"yarn copy:lib",
"yarn dev:tauri" "yarn dev:tauri"
] ]
@ -83,7 +79,6 @@ run = "yarn build"
description = "Build Tauri application (DEPRECATED - matches Makefile)" description = "Build Tauri application (DEPRECATED - matches Makefile)"
depends = ["install-and-build"] depends = ["install-and-build"]
run = [ run = [
"yarn copy:lib",
"yarn build" "yarn build"
] ]

2
web-app/src/containers/ChatInput.tsx
View File

@ -404,7 +404,7 @@ const ChatInput = ({ model, className, initialMessage }: ChatInputProps) => {
streamingContent && 'opacity-50 pointer-events-none' streamingContent && 'opacity-50 pointer-events-none'
)} )}
> >
{model?.provider === 'llama.cpp' && loadingModel ? ( {model?.provider === 'llamacpp' && loadingModel ? (
<ModelLoader /> <ModelLoader />
) : ( ) : (
<DropdownModelProvider <DropdownModelProvider

10
web-app/src/containers/DropdownModelProvider.tsx
View File

@ -96,15 +96,15 @@ const DropdownModelProvider = ({
selectModelProvider(lastUsed.provider, lastUsed.model) selectModelProvider(lastUsed.provider, lastUsed.model)
} else { } else {
// Fallback to default model if last used model no longer exists // Fallback to default model if last used model no longer exists
selectModelProvider('llama.cpp', 'llama3.2:3b') selectModelProvider('llamacpp', 'llama3.2:3b')
} }
} else { } else {
// default model, we should add from setting // default model, we should add from setting
selectModelProvider('llama.cpp', 'llama3.2:3b') selectModelProvider('llamacpp', 'llama3.2:3b')
} }
} else { } else {
// default model for non-new-chat contexts // default model for non-new-chat contexts
selectModelProvider('llama.cpp', 'llama3.2:3b') selectModelProvider('llamacpp', 'llama3.2:3b')
} }
}, [ }, [
model, model,
@ -150,8 +150,8 @@ const DropdownModelProvider = ({
if (!provider.active) return if (!provider.active) return
provider.models.forEach((modelItem) => { provider.models.forEach((modelItem) => {
// Skip models that require API key but don't have one (except llama.cpp) // Skip models that require API key but don't have one (except llamacpp)
if (provider.provider !== 'llama.cpp' && !provider.api_key?.length) { if (provider.provider !== 'llamacpp' && !provider.api_key?.length) {
return return
} }

4
web-app/src/containers/SettingsMenu.tsx
View File

@ -18,6 +18,10 @@ import ProvidersAvatar from '@/containers/ProvidersAvatar'
const SettingsMenu = () => { const SettingsMenu = () => {
const { t } = useTranslation() const { t } = useTranslation()
const { experimentalFeatures } = useGeneralSetting()
const { providers } = useModelProvider()
const firstItemProvider =
providers.length > 0 ? providers[0].provider : 'llamacpp'
const [expandedProviders, setExpandedProviders] = useState(false) const [expandedProviders, setExpandedProviders] = useState(false)
const [isMenuOpen, setIsMenuOpen] = useState(false) const [isMenuOpen, setIsMenuOpen] = useState(false)
const matches = useMatches() const matches = useMatches()

3
web-app/src/containers/dialogs/DeleteProvider.tsx
View File

@ -17,7 +17,6 @@ import { EngineManager } from '@janhq/core'
import { useModelProvider } from '@/hooks/useModelProvider' import { useModelProvider } from '@/hooks/useModelProvider'
import { useRouter } from '@tanstack/react-router' import { useRouter } from '@tanstack/react-router'
import { route } from '@/constants/routes' import { route } from '@/constants/routes'
import { normalizeProvider } from '@/lib/models'
import { useTranslation } from '@/i18n/react-i18next-compat' import { useTranslation } from '@/i18n/react-i18next-compat'
type Props = { type Props = {
@ -30,7 +29,7 @@ const DeleteProvider = ({ provider }: Props) => {
if ( if (
!provider || !provider ||
Object.keys(models).includes(provider.provider) || Object.keys(models).includes(provider.provider) ||
EngineManager.instance().get(normalizeProvider(provider.provider)) EngineManager.instance().get(provider.provider)
) )
return null return null

49
web-app/src/hooks/useChat.ts
View File

@ -115,15 +115,11 @@ export const useChat = () => {
]) ])
const restartModel = useCallback( const restartModel = useCallback(
async ( async (provider: ProviderObject, modelId: string) => {
provider: ProviderObject,
modelId: string,
abortController: AbortController
) => {
await stopAllModels() await stopAllModels()
await new Promise((resolve) => setTimeout(resolve, 1000)) await new Promise((resolve) => setTimeout(resolve, 1000))
updateLoadingModel(true) updateLoadingModel(true)
await startModel(provider, modelId, abortController).catch(console.error) await startModel(provider, modelId).catch(console.error)
updateLoadingModel(false) updateLoadingModel(false)
await new Promise((resolve) => setTimeout(resolve, 1000)) await new Promise((resolve) => setTimeout(resolve, 1000))
}, },
@ -131,11 +127,7 @@ export const useChat = () => {
) )
const increaseModelContextSize = useCallback( const increaseModelContextSize = useCallback(
async ( async (modelId: string, provider: ProviderObject) => {
modelId: string,
provider: ProviderObject,
controller: AbortController
) => {
/** /**
* Should increase the context size of the model by 2x * Should increase the context size of the model by 2x
* If the context size is not set or too low, it defaults to 8192. * If the context size is not set or too low, it defaults to 8192.
@ -180,19 +172,14 @@ export const useChat = () => {
}) })
} }
const updatedProvider = getProviderByName(provider.provider) const updatedProvider = getProviderByName(provider.provider)
if (updatedProvider) if (updatedProvider) await restartModel(updatedProvider, model.id)
await restartModel(updatedProvider, model.id, controller)
return updatedProvider return updatedProvider
}, },
[getProviderByName, restartModel, updateProvider] [getProviderByName, restartModel, updateProvider]
) )
const toggleOnContextShifting = useCallback( const toggleOnContextShifting = useCallback(
async ( async (modelId: string, provider: ProviderObject) => {
modelId: string,
provider: ProviderObject,
controller: AbortController
) => {
const providerName = provider.provider const providerName = provider.provider
const newSettings = [...provider.settings] const newSettings = [...provider.settings]
const settingKey = 'context_shift' const settingKey = 'context_shift'
@ -218,8 +205,7 @@ export const useChat = () => {
...updateObj, ...updateObj,
}) })
const updatedProvider = getProviderByName(providerName) const updatedProvider = getProviderByName(providerName)
if (updatedProvider) if (updatedProvider) await restartModel(updatedProvider, modelId)
await restartModel(updatedProvider, modelId, controller)
return updatedProvider return updatedProvider
}, },
[updateProvider, getProviderByName, restartModel] [updateProvider, getProviderByName, restartModel]
@ -246,11 +232,9 @@ export const useChat = () => {
try { try {
if (selectedModel?.id) { if (selectedModel?.id) {
updateLoadingModel(true) updateLoadingModel(true)
await startModel( await startModel(activeProvider, selectedModel.id).catch(
activeProvider, console.error
selectedModel.id, )
abortController
).catch(console.error)
updateLoadingModel(false) updateLoadingModel(false)
} }
@ -286,10 +270,6 @@ export const useChat = () => {
availableTools, availableTools,
currentAssistant.parameters?.stream === false ? false : true, currentAssistant.parameters?.stream === false ? false : true,
currentAssistant.parameters as unknown as Record<string, object> currentAssistant.parameters as unknown as Record<string, object>
// TODO: replace it with according provider setting later on
// selectedProvider === 'llama.cpp' && availableTools.length > 0
// ? false
// : true
) )
if (!completion) throw new Error('No completion received') if (!completion) throw new Error('No completion received')
@ -298,7 +278,8 @@ export const useChat = () => {
const toolCalls: ChatCompletionMessageToolCall[] = [] const toolCalls: ChatCompletionMessageToolCall[] = []
try { try {
if (isCompletionResponse(completion)) { if (isCompletionResponse(completion)) {
accumulatedText = completion.choices[0]?.message?.content || '' accumulatedText =
(completion.choices[0]?.message?.content as string) || ''
if (completion.choices[0]?.message?.tool_calls) { if (completion.choices[0]?.message?.tool_calls) {
toolCalls.push(...completion.choices[0].message.tool_calls) toolCalls.push(...completion.choices[0].message.tool_calls)
} }
@ -365,16 +346,14 @@ export const useChat = () => {
/// Increase context size /// Increase context size
activeProvider = await increaseModelContextSize( activeProvider = await increaseModelContextSize(
selectedModel.id, selectedModel.id,
activeProvider, activeProvider
abortController
) )
continue continue
} else if (method === 'context_shift' && selectedModel?.id) { } else if (method === 'context_shift' && selectedModel?.id) {
/// Enable context_shift /// Enable context_shift
activeProvider = await toggleOnContextShifting( activeProvider = await toggleOnContextShifting(
selectedModel?.id, selectedModel?.id,
activeProvider, activeProvider
abortController
) )
continue continue
} else throw error } else throw error
@ -387,7 +366,7 @@ export const useChat = () => {
accumulatedText.length === 0 && accumulatedText.length === 0 &&
toolCalls.length === 0 && toolCalls.length === 0 &&
activeThread.model?.id && activeThread.model?.id &&
activeProvider.provider === 'llama.cpp' provider?.provider === 'llamacpp'
) { ) {
await stopModel(activeThread.model.id, 'cortex') await stopModel(activeThread.model.id, 'cortex')
throw new Error('No response received from the model') throw new Error('No response received from the model')

2
web-app/src/hooks/useModelProvider.ts
View File

@ -24,7 +24,7 @@ export const useModelProvider = create<ModelProviderState>()(
persist( persist(
(set, get) => ({ (set, get) => ({
providers: [], providers: [],
selectedProvider: 'llama.cpp', selectedProvider: 'llamacpp',
selectedModel: null, selectedModel: null,
deletedModels: [], deletedModels: [],
getModelBy: (modelId: string) => { getModelBy: (modelId: string) => {

39
web-app/src/lib/completion.ts
View File

@ -5,6 +5,9 @@ import {
MessageStatus, MessageStatus,
EngineManager, EngineManager,
ModelManager, ModelManager,
chatCompletionRequestMessage,
chatCompletion,
chatCompletionChunk,
} from '@janhq/core' } from '@janhq/core'
import { invoke } from '@tauri-apps/api/core' import { invoke } from '@tauri-apps/api/core'
import { fetch as fetchTauri } from '@tauri-apps/plugin-http' import { fetch as fetchTauri } from '@tauri-apps/plugin-http'
@ -24,11 +27,17 @@ type ExtendedConfigOptions = ConfigOptions & {
fetch?: typeof fetch fetch?: typeof fetch
} }
import { ulid } from 'ulidx' import { ulid } from 'ulidx'
import { normalizeProvider } from './models'
import { MCPTool } from '@/types/completion' import { MCPTool } from '@/types/completion'
import { CompletionMessagesBuilder } from './messages' import { CompletionMessagesBuilder } from './messages'
import { ChatCompletionMessageToolCall } from 'openai/resources' import { ChatCompletionMessageToolCall } from 'openai/resources'
import { callTool } from '@/services/mcp' import { callTool } from '@/services/mcp'
import { ExtensionManager } from './extension'
export type ChatCompletionResponse =
| chatCompletion
| AsyncIterable<chatCompletionChunk>
| StreamCompletionResponse
| CompletionResponse
/** /**
* @fileoverview Helper functions for creating thread content. * @fileoverview Helper functions for creating thread content.
@ -124,7 +133,7 @@ export const sendCompletion = async (
tools: MCPTool[] = [], tools: MCPTool[] = [],
stream: boolean = true, stream: boolean = true,
params: Record<string, object> = {} params: Record<string, object> = {}
): Promise<StreamCompletionResponse | CompletionResponse | undefined> => { ): Promise<ChatCompletionResponse | undefined> => {
if (!thread?.model?.id || !provider) return undefined if (!thread?.model?.id || !provider) return undefined
let providerName = provider.provider as unknown as keyof typeof models let providerName = provider.provider as unknown as keyof typeof models
@ -144,7 +153,7 @@ export const sendCompletion = async (
!(thread.model.id in Object.values(models).flat()) && !(thread.model.id in Object.values(models).flat()) &&
// eslint-disable-next-line @typescript-eslint/no-explicit-any // eslint-disable-next-line @typescript-eslint/no-explicit-any
!tokenJS.extendedModelExist(providerName as any, thread.model?.id) && !tokenJS.extendedModelExist(providerName as any, thread.model?.id) &&
provider.provider !== 'llama.cpp' provider.provider !== 'llamacpp'
) { ) {
try { try {
tokenJS.extendModelList( tokenJS.extendModelList(
@ -163,8 +172,18 @@ export const sendCompletion = async (
} }
} }
// TODO: Add message history const engine = ExtensionManager.getInstance().getEngine(provider.provider)
const completion = stream
const completion = engine
? await engine.chat({
messages: messages as chatCompletionRequestMessage[],
model: thread.model?.id,
tools: normalizeTools(tools),
tool_choice: tools.length ? 'auto' : undefined,
stream: true,
...params,
})
: stream
? await tokenJS.chat.completions.create( ? await tokenJS.chat.completions.create(
{ {
stream: true, stream: true,
@ -193,8 +212,8 @@ export const sendCompletion = async (
} }
export const isCompletionResponse = ( export const isCompletionResponse = (
response: StreamCompletionResponse | CompletionResponse response: ChatCompletionResponse
): response is CompletionResponse => { ): response is CompletionResponse | chatCompletion => {
return 'choices' in response return 'choices' in response
} }
@ -209,9 +228,9 @@ export const stopModel = async (
provider: string, provider: string,
model: string model: string
): Promise<void> => { ): Promise<void> => {
const providerObj = EngineManager.instance().get(normalizeProvider(provider)) const providerObj = EngineManager.instance().get(provider)
const modelObj = ModelManager.instance().get(model) const modelObj = ModelManager.instance().get(model)
if (providerObj && modelObj) return providerObj?.unload(modelObj) if (providerObj && modelObj) return providerObj?.unload(model).then(() => {})
} }
/** /**
@ -241,7 +260,7 @@ export const normalizeTools = (
* @param calls * @param calls
*/ */
export const extractToolCall = ( export const extractToolCall = (
part: CompletionResponseChunk, part: chatCompletionChunk | CompletionResponseChunk,
currentCall: ChatCompletionMessageToolCall | null, currentCall: ChatCompletionMessageToolCall | null,
calls: ChatCompletionMessageToolCall[] calls: ChatCompletionMessageToolCall[]
) => { ) => {

6
web-app/src/lib/model.spec.ts
View File

@ -1,6 +1,2 @@
import { expect, test } from 'vitest'
import { normalizeProvider } from './models'
test('provider name should be normalized', () => {
expect(normalizeProvider('llama.cpp')).toBe('cortex')
})

9
web-app/src/lib/models.ts
View File

@ -58,12 +58,3 @@ export const extractModelName = (model?: string) => {
export const extractModelRepo = (model?: string) => { export const extractModelRepo = (model?: string) => {
return model?.replace('https://huggingface.co/', '') return model?.replace('https://huggingface.co/', '')
} }
/**
* Normalize the provider name to match the format used in the models object
* @param provider - The provider name to normalize
*/
export const normalizeProvider = (provider: string) => {
// TODO: After migrating to the new provider extension, remove this function
return provider === 'llama.cpp' ? 'cortex' : provider
}

10
web-app/src/lib/utils.ts
View File

@ -1,5 +1,6 @@
import { type ClassValue, clsx } from 'clsx' import { type ClassValue, clsx } from 'clsx'
import { twMerge } from 'tailwind-merge' import { twMerge } from 'tailwind-merge'
import { ExtensionManager } from './extension'
export function cn(...inputs: ClassValue[]) { export function cn(...inputs: ClassValue[]) {
return twMerge(clsx(inputs)) return twMerge(clsx(inputs))
@ -7,7 +8,7 @@ export function cn(...inputs: ClassValue[]) {
export function getProviderLogo(provider: string) { export function getProviderLogo(provider: string) {
switch (provider) { switch (provider) {
case 'llama.cpp': case 'llamacpp':
return '/images/model-provider/llamacpp.svg' return '/images/model-provider/llamacpp.svg'
case 'anthropic': case 'anthropic':
return '/images/model-provider/anthropic.svg' return '/images/model-provider/anthropic.svg'
@ -38,7 +39,7 @@ export function getProviderLogo(provider: string) {
export const getProviderTitle = (provider: string) => { export const getProviderTitle = (provider: string) => {
switch (provider) { switch (provider) {
case 'llama.cpp': case 'llamacpp':
return 'Llama.cpp' return 'Llama.cpp'
case 'openai': case 'openai':
return 'OpenAI' return 'OpenAI'
@ -89,6 +90,11 @@ export function getReadableLanguageName(language: string): string {
) )
} }
export const isLocalProvider = (provider: string) => {
const extension = ExtensionManager.getInstance().getEngine(provider)
return extension && 'load' in extension
}
export function fuzzySearch(needle: string, haystack: string) { export function fuzzySearch(needle: string, haystack: string) {
const hlen = haystack.length const hlen = haystack.length
const nlen = needle.length const nlen = needle.length

6
web-app/src/providers/DataProvider.tsx
View File

@ -3,10 +3,8 @@ import { useModelProvider } from '@/hooks/useModelProvider'
import { useAppUpdater } from '@/hooks/useAppUpdater' import { useAppUpdater } from '@/hooks/useAppUpdater'
import { fetchMessages } from '@/services/messages' import { fetchMessages } from '@/services/messages'
import { fetchModels } from '@/services/models'
import { getProviders } from '@/services/providers' import { getProviders } from '@/services/providers'
import { fetchThreads } from '@/services/threads' import { fetchThreads } from '@/services/threads'
import { ModelManager } from '@janhq/core'
import { useEffect } from 'react' import { useEffect } from 'react'
import { useMCPServers } from '@/hooks/useMCPServers' import { useMCPServers } from '@/hooks/useMCPServers'
import { getMCPConfig } from '@/services/mcp' import { getMCPConfig } from '@/services/mcp'
@ -31,10 +29,8 @@ export function DataProvider() {
const navigate = useNavigate() const navigate = useNavigate()
useEffect(() => { useEffect(() => {
fetchModels().then((models) => { console.log('Initializing DataProvider...')
models?.forEach((model) => ModelManager.instance().register(model))
getProviders().then(setProviders) getProviders().then(setProviders)
})
getMCPConfig().then((data) => setServers(data.mcpServers ?? [])) getMCPConfig().then((data) => setServers(data.mcpServers ?? []))
getAssistants() getAssistants()
.then((data) => { .then((data) => {

10
web-app/src/routes/hub.tsx
View File

@ -31,7 +31,7 @@ import {
DropdownMenuItem, DropdownMenuItem,
DropdownMenuTrigger, DropdownMenuTrigger,
} from '@/components/ui/dropdown-menu' } from '@/components/ui/dropdown-menu'
import { addModelSource, downloadModel, fetchModelHub } from '@/services/models' import { addModelSource, fetchModelHub, pullModel } from '@/services/models'
import { useDownloadStore } from '@/hooks/useDownloadStore' import { useDownloadStore } from '@/hooks/useDownloadStore'
import { Progress } from '@/components/ui/progress' import { Progress } from '@/components/ui/progress'
import HeaderPage from '@/containers/HeaderPage' import HeaderPage from '@/containers/HeaderPage'
@ -83,7 +83,7 @@ function Hub() {
const hasTriggeredDownload = useRef(false) const hasTriggeredDownload = useRef(false)
const { getProviderByName } = useModelProvider() const { getProviderByName } = useModelProvider()
const llamaProvider = getProviderByName('llama.cpp') const llamaProvider = getProviderByName('llamacpp')
const toggleModelExpansion = (modelId: string) => { const toggleModelExpansion = (modelId: string) => {
setExpandedModels((prev) => ({ setExpandedModels((prev) => ({
@ -213,7 +213,7 @@ function Hub() {
search: { search: {
model: { model: {
id: modelId, id: modelId,
provider: 'llama.cpp', provider: 'llamacpp',
}, },
}, },
}) })
@ -240,7 +240,7 @@ function Hub() {
const handleDownload = () => { const handleDownload = () => {
// Immediately set local downloading state // Immediately set local downloading state
addLocalDownloadingModel(modelId) addLocalDownloadingModel(modelId)
downloadModel(modelId) pullModel(modelId, modelId)
} }
return ( return (
@ -650,7 +650,7 @@ function Hub() {
addLocalDownloadingModel( addLocalDownloadingModel(
variant.id variant.id
) )
downloadModel(variant.id) pullModel(variant.id, variant.id)
}} }}
> >
<IconDownload <IconDownload

2
web-app/src/routes/index.tsx
View File

@ -37,7 +37,7 @@ function Index() {
const hasValidProviders = providers.some( const hasValidProviders = providers.some(
(provider) => (provider) =>
provider.api_key?.length || provider.api_key?.length ||
(provider.provider === 'llama.cpp' && provider.models.length) (provider.provider === 'llamacpp' && provider.models.length)
) )
useEffect(() => { useEffect(() => {

41
web-app/src/routes/settings/providers/$providerName.tsx
View File

@ -6,7 +6,7 @@ import { cn, getProviderTitle } from '@/lib/utils'
import { open } from '@tauri-apps/plugin-dialog' import { open } from '@tauri-apps/plugin-dialog'
import { import {
getActiveModels, getActiveModels,
importModel, pullModel,
startModel, startModel,
stopAllModels, stopAllModels,
stopModel, stopModel,
@ -35,7 +35,6 @@ import { Button } from '@/components/ui/button'
import { IconFolderPlus, IconLoader, IconRefresh } from '@tabler/icons-react' import { IconFolderPlus, IconLoader, IconRefresh } from '@tabler/icons-react'
import { getProviders } from '@/services/providers' import { getProviders } from '@/services/providers'
import { toast } from 'sonner' import { toast } from 'sonner'
import { ActiveModel } from '@/types/models'
import { useEffect, useState } from 'react' import { useEffect, useState } from 'react'
import { predefinedProviders } from '@/mock/data' import { predefinedProviders } from '@/mock/data'
@ -73,7 +72,7 @@ function ProviderDetail() {
}, },
] ]
const { step } = useSearch({ from: Route.id }) const { step } = useSearch({ from: Route.id })
const [activeModels, setActiveModels] = useState<ActiveModel[]>([]) const [activeModels, setActiveModels] = useState<string[]>([])
const [loadingModels, setLoadingModels] = useState<string[]>([]) const [loadingModels, setLoadingModels] = useState<string[]>([])
const [refreshingModels, setRefreshingModels] = useState(false) const [refreshingModels, setRefreshingModels] = useState(false)
const { providerName } = useParams({ from: Route.id }) const { providerName } = useParams({ from: Route.id })
@ -171,10 +170,7 @@ function ProviderDetail() {
if (provider) if (provider)
startModel(provider, modelId) startModel(provider, modelId)
.then(() => { .then(() => {
setActiveModels((prevModels) => [ setActiveModels((prevModels) => [...prevModels, modelId])
...prevModels,
{ id: modelId } as ActiveModel,
])
}) })
.catch((error) => { .catch((error) => {
console.error('Error starting model:', error) console.error('Error starting model:', error)
@ -189,7 +185,7 @@ function ProviderDetail() {
stopModel(modelId) stopModel(modelId)
.then(() => { .then(() => {
setActiveModels((prevModels) => setActiveModels((prevModels) =>
prevModels.filter((model) => model.id !== modelId) prevModels.filter((model) => model !== modelId)
) )
}) })
.catch((error) => { .catch((error) => {
@ -240,7 +236,7 @@ function ProviderDetail() {
className={cn( className={cn(
'flex flex-col gap-3', 'flex flex-col gap-3',
provider && provider &&
provider.provider === 'llama.cpp' && provider.provider === 'llamacpp' &&
'flex-col-reverse' 'flex-col-reverse'
)} )}
> >
@ -353,7 +349,7 @@ function ProviderDetail() {
{t('providers:models')} {t('providers:models')}
</h1> </h1>
<div className="flex items-center gap-2"> <div className="flex items-center gap-2">
{provider && provider.provider !== 'llama.cpp' && ( {provider && provider.provider !== 'llamacpp' && (
<> <>
{!predefinedProviders.some( {!predefinedProviders.some(
(p) => p.provider === provider.provider (p) => p.provider === provider.provider
@ -388,7 +384,7 @@ function ProviderDetail() {
<DialogAddModel provider={provider} /> <DialogAddModel provider={provider} />
</> </>
)} )}
{provider && provider.provider === 'llama.cpp' && ( {provider && provider.provider === 'llamacpp' && (
<Button <Button
variant="link" variant="link"
size="sm" size="sm"
@ -404,10 +400,15 @@ function ProviderDetail() {
}, },
], ],
}) })
// If the dialog returns a file path, extract just the file name
const fileName =
typeof selectedFile === 'string'
? selectedFile.split(/[\\/]/).pop()
: undefined
if (selectedFile) { if (selectedFile && fileName) {
try { try {
await importModel(selectedFile) await pullModel(fileName, selectedFile)
} catch (error) { } catch (error) {
console.error( console.error(
t('providers:importModelError'), t('providers:importModelError'),
@ -475,19 +476,15 @@ function ProviderDetail() {
provider={provider} provider={provider}
modelId={model.id} modelId={model.id}
/> />
{provider && {provider && provider.provider === 'llamacpp' && (
provider.provider === 'llama.cpp' && (
<div className="ml-2"> <div className="ml-2">
{activeModels.some( {activeModels.some(
(activeModel) => (activeModel) => activeModel === model.id
activeModel.id === model.id
) ? ( ) ? (
<Button <Button
size="sm" size="sm"
variant="destructive" variant="destructive"
onClick={() => onClick={() => handleStopModel(model.id)}
handleStopModel(model.id)
}
> >
{t('providers:stop')} {t('providers:stop')}
</Button> </Button>
@ -497,9 +494,7 @@ function ProviderDetail() {
disabled={loadingModels.includes( disabled={loadingModels.includes(
model.id model.id
)} )}
onClick={() => onClick={() => handleStartModel(model.id)}
handleStartModel(model.id)
}
> >
{loadingModels.includes(model.id) ? ( {loadingModels.includes(model.id) ? (
<div className="flex items-center gap-2"> <div className="flex items-center gap-2">

17
web-app/src/routes/system-monitor.tsx
View File

@ -5,10 +5,9 @@ import { getHardwareInfo } from '@/services/hardware'
import { Progress } from '@/components/ui/progress' import { Progress } from '@/components/ui/progress'
import type { HardwareData } from '@/hooks/useHardware' import type { HardwareData } from '@/hooks/useHardware'
import { route } from '@/constants/routes' import { route } from '@/constants/routes'
import { formatDuration, formatMegaBytes } from '@/lib/utils' import { formatMegaBytes } from '@/lib/utils'
import { IconDeviceDesktopAnalytics } from '@tabler/icons-react' import { IconDeviceDesktopAnalytics } from '@tabler/icons-react'
import { getActiveModels, stopModel } from '@/services/models' import { getActiveModels, stopModel } from '@/services/models'
import { ActiveModel } from '@/types/models'
import { Button } from '@/components/ui/button' import { Button } from '@/components/ui/button'
import { useTranslation } from '@/i18n/react-i18next-compat' import { useTranslation } from '@/i18n/react-i18next-compat'
@ -21,7 +20,7 @@ function SystemMonitor() {
const { t } = useTranslation() const { t } = useTranslation()
const { hardwareData, setHardwareData, updateCPUUsage, updateRAMAvailable } = const { hardwareData, setHardwareData, updateCPUUsage, updateRAMAvailable } =
useHardware() useHardware()
const [activeModels, setActiveModels] = useState<ActiveModel[]>([]) const [activeModels, setActiveModels] = useState<string[]>([])
useEffect(() => { useEffect(() => {
// Initial data fetch // Initial data fetch
@ -47,7 +46,7 @@ function SystemMonitor() {
stopModel(modelId) stopModel(modelId)
.then(() => { .then(() => {
setActiveModels((prevModels) => setActiveModels((prevModels) =>
prevModels.filter((model) => model.id !== modelId) prevModels.filter((model) => model !== modelId)
) )
}) })
.catch((error) => { .catch((error) => {
@ -173,10 +172,10 @@ function SystemMonitor() {
{activeModels.length > 0 && ( {activeModels.length > 0 && (
<div className="flex flex-col gap-4"> <div className="flex flex-col gap-4">
{activeModels.map((model) => ( {activeModels.map((model) => (
<div className="bg-main-view-fg/3 rounded-lg p-4" key={model.id}> <div className="bg-main-view-fg/3 rounded-lg p-4" key={model}>
<div className="flex justify-between items-center mb-2"> <div className="flex justify-between items-center mb-2">
<span className="font-semibold text-main-view-fg"> <span className="font-semibold text-main-view-fg">
{model.id} {model}
</span> </span>
</div> </div>
<div className="flex flex-col gap-2 mt-3"> <div className="flex flex-col gap-2 mt-3">
@ -190,9 +189,9 @@ function SystemMonitor() {
<span className="text-main-view-fg/70"> <span className="text-main-view-fg/70">
{t('system-monitor:uptime')} {t('system-monitor:uptime')}
</span> </span>
<span className="text-main-view-fg"> {/* <span className="text-main-view-fg">
{model.start_time && formatDuration(model.start_time)} {model.start_time && formatDuration(model.start_time)}
</span> </span> */}
</div> </div>
<div className="flex justify-between items-center"> <div className="flex justify-between items-center">
<span className="text-main-view-fg/70"> <span className="text-main-view-fg/70">
@ -202,7 +201,7 @@ function SystemMonitor() {
<Button <Button
variant="destructive" variant="destructive"
size="sm" size="sm"
onClick={() => stopRunningModel(model.id)} onClick={() => stopRunningModel(model)}
> >
{t('system-monitor:stop')} {t('system-monitor:stop')}
</Button> </Button>

292
web-app/src/services/models.ts
View File

@ -1,56 +1,37 @@
/* eslint-disable @typescript-eslint/no-explicit-any */ /* eslint-disable @typescript-eslint/no-explicit-any */
import { ExtensionManager } from '@/lib/extension' import { AIEngine, EngineManager, SettingComponentProps } from '@janhq/core'
import { normalizeProvider } from '@/lib/models'
import { EngineManager, ExtensionTypeEnum, ModelExtension } from '@janhq/core'
import { Model as CoreModel } from '@janhq/core' import { Model as CoreModel } from '@janhq/core'
// TODO: Replace this with the actual provider later
const defaultProvider = 'llamacpp'
const getEngine = (provider: string = defaultProvider) => {
return EngineManager.instance().get(provider) as AIEngine
}
/** /**
* Fetches all available models. * Fetches all available models.
* @returns A promise that resolves to the models. * @returns A promise that resolves to the models.
*/ */
export const fetchModels = async () => { export const fetchModels = async () => {
return ExtensionManager.getInstance() return getEngine().list()
.get<ModelExtension>(ExtensionTypeEnum.Model)
?.getModels()
} }
/** /**
* Fetches the sources of the models. * Fetches the sources of the models.
* @returns A promise that resolves to the model sources. * @returns A promise that resolves to the model sources.
*/ */
export const fetchModelSources = async (): Promise<any[]> => { export const fetchModelSources = async () => {
const extension = ExtensionManager.getInstance().get<ModelExtension>( // TODO: New Hub
ExtensionTypeEnum.Model
)
if (!extension) return []
try {
const sources = await extension.getSources()
const mappedSources = sources.map((m) => ({
...m,
models: m.models.sort((a, b) => a.size - b.size),
}))
// Prepend the hardcoded model to the sources
return [...mappedSources]
} catch (error) {
console.error('Failed to fetch model sources:', error)
return [] return []
}
} }
/** /**
* Fetches the model hub. * Fetches the model hub.
* @returns A promise that resolves to the model hub. * @returns A promise that resolves to the model hub.
*/ */
export const fetchModelHub = async (): Promise<any[]> => { export const fetchModelHub = async () => {
const hubData = await ExtensionManager.getInstance() // TODO: New Hub
.get<ModelExtension>(ExtensionTypeEnum.Model) return
?.fetchModelsHub()
// Prepend the hardcoded model to the hub data
return hubData ? [...hubData] : []
} }
/** /**
@ -59,18 +40,9 @@ export const fetchModelHub = async (): Promise<any[]> => {
* @returns A promise that resolves when the source is added. * @returns A promise that resolves when the source is added.
*/ */
export const addModelSource = async (source: string) => { export const addModelSource = async (source: string) => {
const extension = ExtensionManager.getInstance().get<ModelExtension>( // TODO: New Hub
ExtensionTypeEnum.Model console.log(source)
) return
if (!extension) throw new Error('Model extension not found')
try {
return await extension.addSource(source)
} catch (error) {
console.error('Failed to add model source:', error)
throw error
}
} }
/** /**
@ -79,18 +51,9 @@ export const addModelSource = async (source: string) => {
* @returns A promise that resolves when the source is deleted. * @returns A promise that resolves when the source is deleted.
*/ */
export const deleteModelSource = async (source: string) => { export const deleteModelSource = async (source: string) => {
const extension = ExtensionManager.getInstance().get<ModelExtension>( // TODO: New Hub
ExtensionTypeEnum.Model console.log(source)
) return
if (!extension) throw new Error('Model extension not found')
try {
return await extension.deleteSource(source)
} catch (error) {
console.error('Failed to delete model source:', error)
throw error
}
} }
/** /**
@ -102,38 +65,19 @@ export const updateModel = async (
model: Partial<CoreModel> model: Partial<CoreModel>
// provider: string, // provider: string,
) => { ) => {
const extension = ExtensionManager.getInstance().get<ModelExtension>( if (model.settings)
ExtensionTypeEnum.Model getEngine().updateSettings(model.settings as SettingComponentProps[])
)
if (!extension) throw new Error('Model extension not found')
try {
return await extension.updateModel(model)
} catch (error) {
console.error('Failed to update model:', error)
throw error
}
} }
/** /**
* Downloads a model. * Pull or import a model.
* @param model The model to download. * @param model The model to pull.
* @returns A promise that resolves when the model download task is created. * @returns A promise that resolves when the model download task is created.
*/ */
export const downloadModel = async (id: string) => { export const pullModel = async (id: string, modelPath: string) => {
const extension = ExtensionManager.getInstance().get<ModelExtension>( return getEngine().import(id, {
ExtensionTypeEnum.Model modelPath,
) })
if (!extension) throw new Error('Model extension not found')
try {
return await extension.pullModel(id)
} catch (error) {
console.error('Failed to download model:', error)
throw error
}
} }
/** /**
@ -142,18 +86,7 @@ export const downloadModel = async (id: string) => {
* @returns * @returns
*/ */
export const abortDownload = async (id: string) => { export const abortDownload = async (id: string) => {
const extension = ExtensionManager.getInstance().get<ModelExtension>( return getEngine().abortImport(id)
ExtensionTypeEnum.Model
)
if (!extension) throw new Error('Model extension not found')
try {
return await extension.cancelModelPull(id)
} catch (error) {
console.error('Failed to abort model download:', error)
throw error
}
} }
/** /**
@ -162,64 +95,7 @@ export const abortDownload = async (id: string) => {
* @returns * @returns
*/ */
export const deleteModel = async (id: string) => { export const deleteModel = async (id: string) => {
const extension = ExtensionManager.getInstance().get<ModelExtension>( return getEngine().delete(id)
ExtensionTypeEnum.Model
)
if (!extension) throw new Error('Model extension not found')
try {
return await extension.deleteModel(id).then(() => {
// TODO: This should be removed when we integrate new llama.cpp extension
if (id.includes(':')) {
extension.addSource(`cortexso/${id.split(':')[0]}`)
}
})
} catch (error) {
console.error('Failed to delete model:', error)
throw error
}
}
/**
* Imports a model from a file path.
* @param filePath The path to the model file or an array of file paths.
* @param modelId Optional model ID. If not provided, it will be derived from the file name.
* @param provider The provider for the model (default: 'llama.cpp').
* @returns A promise that resolves when the model is imported.
*/
export const importModel = async (
filePath: string | string[],
modelId?: string,
provider: string = 'llama.cpp'
) => {
const extension = ExtensionManager.getInstance().get<ModelExtension>(
ExtensionTypeEnum.Model
)
if (!extension) throw new Error('Model extension not found')
try {
// If filePath is an array, use the first element
const path = Array.isArray(filePath) ? filePath[0] : filePath
// If no path was selected, throw an error
if (!path) throw new Error('No file selected')
// Extract filename from path to use as model ID if not provided
const defaultModelId =
path
.split(/[/\\]/)
.pop()
?.replace(/ /g, '-')
.replace(/\.gguf$/i, '') || path
const modelIdToUse = modelId || defaultModelId
return await extension.importModel(modelIdToUse, path, provider)
} catch (error) {
console.error('Failed to import model:', error)
throw error
}
} }
/** /**
@ -228,20 +104,8 @@ export const importModel = async (
* @returns * @returns
*/ */
export const getActiveModels = async (provider?: string) => { export const getActiveModels = async (provider?: string) => {
const providerName = provider || 'cortex' // we will go down to llama.cpp extension later on // getEngine(provider)
const extension = EngineManager.instance().get(providerName) return getEngine(provider).getLoadedModels()
if (!extension) throw new Error('Model extension not found')
try {
return 'activeModels' in extension &&
typeof extension.activeModels === 'function'
? ((await extension.activeModels()) ?? [])
: []
} catch (error) {
console.error('Failed to get active models:', error)
return []
}
} }
/** /**
@ -251,20 +115,7 @@ export const getActiveModels = async (provider?: string) => {
* @returns * @returns
*/ */
export const stopModel = async (model: string, provider?: string) => { export const stopModel = async (model: string, provider?: string) => {
const providerName = provider || 'cortex' // we will go down to llama.cpp extension later on getEngine(provider).unload(model)
const extension = EngineManager.instance().get(providerName)
if (!extension) throw new Error('Model extension not found')
try {
return await extension.unloadModel({
model,
id: model,
})
} catch (error) {
console.error('Failed to stop model:', error)
return []
}
} }
/** /**
@ -273,10 +124,7 @@ export const stopModel = async (model: string, provider?: string) => {
*/ */
export const stopAllModels = async () => { export const stopAllModels = async () => {
const models = await getActiveModels() const models = await getActiveModels()
if (models) if (models) await Promise.all(models.map((model) => stopModel(model)))
await Promise.all(
models.map((model: { id: string }) => stopModel(model.id))
)
} }
/** /**
@ -289,28 +137,17 @@ export const stopAllModels = async () => {
*/ */
export const startModel = async ( export const startModel = async (
provider: ProviderObject, provider: ProviderObject,
model: string, model: string
abortController?: AbortController
): Promise<void> => { ): Promise<void> => {
const providerObj = EngineManager.instance().get( getEngine(provider.provider)
normalizeProvider(provider.provider) .load(model)
.catch((error) => {
console.error(
`Failed to start model ${model} for provider ${provider.provider}:`,
error
) )
const modelObj = provider.models.find((m) => m.id === model) throw error
})
if (providerObj && modelObj) {
return providerObj?.loadModel(
{
id: modelObj.id,
settings: Object.fromEntries(
Object.entries(modelObj.settings ?? {}).map(([key, value]) => [
key,
value.controller_props?.value, // assuming each setting is { value: ... }
])
),
},
abortController
)
}
} }
/** /**
@ -329,37 +166,16 @@ export const configurePullOptions = async ({
verifyHostSSL, verifyHostSSL,
noProxy, noProxy,
}: ProxyOptions) => { }: ProxyOptions) => {
const extension = ExtensionManager.getInstance().get<ModelExtension>( console.log('Configuring proxy options:', {
ExtensionTypeEnum.Model proxyEnabled,
) proxyUrl,
proxyUsername,
if (!extension) throw new Error('Model extension not found') proxyPassword,
try { proxyIgnoreSSL,
await extension.configurePullOptions( verifyProxySSL,
proxyEnabled verifyProxyHostSSL,
? { verifyPeerSSL,
proxy_username: proxyUsername, verifyHostSSL,
proxy_password: proxyPassword, noProxy,
proxy_url: proxyUrl, })
verify_proxy_ssl: proxyIgnoreSSL ? false : verifyProxySSL,
verify_proxy_host_ssl: proxyIgnoreSSL ? false : verifyProxyHostSSL,
verify_peer_ssl: proxyIgnoreSSL ? false : verifyPeerSSL,
verify_host_ssl: proxyIgnoreSSL ? false : verifyHostSSL,
no_proxy: noProxy,
}
: {
proxy_username: '',
proxy_password: '',
proxy_url: '',
verify_proxy_ssl: false,
verify_proxy_host_ssl: false,
verify_peer_ssl: false,
verify_host_ssl: false,
no_proxy: '',
}
)
} catch (error) {
console.error('Failed to configure pull options:', error)
throw error
}
} }

55
web-app/src/services/providers.ts
View File

@ -1,11 +1,6 @@
import { models as providerModels } from 'token.js' import { models as providerModels } from 'token.js'
import { predefinedProviders } from '@/mock/data' import { predefinedProviders } from '@/mock/data'
import { import { EngineManager, SettingComponentProps } from '@janhq/core'
EngineManagementExtension,
EngineManager,
ExtensionTypeEnum,
SettingComponentProps,
} from '@janhq/core'
import { import {
DefaultToolUseSupportedModels, DefaultToolUseSupportedModels,
ModelCapabilities, ModelCapabilities,
@ -17,11 +12,6 @@ import { fetch as fetchTauri } from '@tauri-apps/plugin-http'
export const getProviders = async (): Promise<ModelProvider[]> => { export const getProviders = async (): Promise<ModelProvider[]> => {
const engines = !localStorage.getItem('migration_completed')
? await ExtensionManager.getInstance()
.get<EngineManagementExtension>(ExtensionTypeEnum.Engine)
?.getEngines()
: {}
const builtinProviders = predefinedProviders.map((provider) => { const builtinProviders = predefinedProviders.map((provider) => {
let models = provider.models as Model[] let models = provider.models as Model[]
if (Object.keys(providerModels).includes(provider.provider)) { if (Object.keys(providerModels).includes(provider.provider)) {
@ -29,29 +19,6 @@ export const getProviders = async (): Promise<ModelProvider[]> => {
provider.provider as unknown as keyof typeof providerModels provider.provider as unknown as keyof typeof providerModels
].models as unknown as string[] ].models as unknown as string[]
if (engines && Object.keys(engines).length > 0) {
for (const [key, value] of Object.entries(engines)) {
const providerName = key.replace('google_gemini', 'gemini')
if (provider.provider !== providerName) continue
const engine = value[0] as
| {
api_key?: string
url?: string
engine?: string
}
| undefined
if (engine && 'api_key' in engine) {
const settings = provider?.settings.map((e) => {
if (e.key === 'api-key')
e.controller_props.value = (engine.api_key as string) ?? ''
return e
})
provider.settings = settings
}
}
}
if (Array.isArray(builtInModels)) if (Array.isArray(builtInModels))
models = builtInModels.map((model) => { models = builtInModels.map((model) => {
const modelManifest = models.find((e) => e.id === model) const modelManifest = models.find((e) => e.id === model)
@ -77,24 +44,11 @@ export const getProviders = async (): Promise<ModelProvider[]> => {
models, models,
} }
}) })
if (engines && Object.keys(engines).length > 0) {
localStorage.setItem('migration_completed', 'true')
}
const runtimeProviders: ModelProvider[] = [] const runtimeProviders: ModelProvider[] = []
for (const [providerName, value] of EngineManager.instance().engines) {
const models = (await fetchModels()) ?? []
for (const [key, value] of EngineManager.instance().engines) {
// TODO: Remove this when the cortex extension is removed
const providerName = key === 'cortex' ? 'llama.cpp' : key
const models =
((await fetchModels()) ?? []).filter(
(model) =>
(model.engine === 'llama-cpp' ? 'llama.cpp' : model.engine) ===
providerName &&
'status' in model &&
model.status === 'downloaded'
) ?? []
const provider: ModelProvider = { const provider: ModelProvider = {
active: false, active: false,
persist: true, persist: true,
@ -246,9 +200,8 @@ export const updateSettings = async (
providerName: string, providerName: string,
settings: ProviderSetting[] settings: ProviderSetting[]
): Promise<void> => { ): Promise<void> => {
const provider = providerName === 'llama.cpp' ? 'cortex' : providerName
return ExtensionManager.getInstance() return ExtensionManager.getInstance()
.getEngine(provider) .getEngine(providerName)
?.updateSettings( ?.updateSettings(
settings.map((setting) => ({ settings.map((setting) => ({
...setting, ...setting,

6
web-app/src/services/threads.ts
View File

@ -51,7 +51,7 @@ export const createThread = async (thread: Thread): Promise<Thread> => {
...(thread.assistants?.[0] ?? defaultAssistant), ...(thread.assistants?.[0] ?? defaultAssistant),
model: { model: {
id: thread.model?.id ?? '*', id: thread.model?.id ?? '*',
engine: thread.model?.provider ?? 'llama.cpp', engine: thread.model?.provider ?? 'llamacpp',
}, },
}, },
], ],
@ -88,7 +88,7 @@ export const updateThread = (thread: Thread) => {
return { return {
model: { model: {
id: thread.model?.id ?? '*', id: thread.model?.id ?? '*',
engine: thread.model?.provider ?? 'llama.cpp', engine: thread.model?.provider ?? 'llamacpp',
}, },
id: e.id, id: e.id,
name: e.name, name: e.name,
@ -98,7 +98,7 @@ export const updateThread = (thread: Thread) => {
{ {
model: { model: {
id: thread.model?.id ?? '*', id: thread.model?.id ?? '*',
engine: thread.model?.provider ?? 'llama.cpp', engine: thread.model?.provider ?? 'llamacpp',
}, },
id: 'jan', id: 'jan',
name: 'Jan', name: 'Jan',

10
web-app/src/types/models.ts
View File

@ -20,13 +20,3 @@ export enum DefaultToolUseSupportedModels {
JanNano = 'jan-nano', JanNano = 'jan-nano',
Qwen3 = 'qwen3', Qwen3 = 'qwen3',
} }
export type ActiveModel = {
engine: string
id: string
model_size: number
object: 'model'
ram: number
start_time: number
vram: number
}