refactor: frontend uses new engine extension
# Conflicts: # extensions/model-extension/resources/default.json # web-app/src/containers/dialogs/DeleteProvider.tsx # web-app/src/routes/hub.tsx
This commit is contained in:
Louis
parent
ad06b2a903
commit
8bd4a3389f
@ -4,72 +4,72 @@ import { EngineManager } from './EngineManager'
|
||||
/* AIEngine class types */
|
||||
|
||||
export interface chatCompletionRequestMessage {
|
||||
role: 'system' | 'user' | 'assistant' | 'tool';
|
||||
content: string | null | Content[]; // Content can be a string OR an array of content parts
|
||||
name?: string;
|
||||
tool_calls?: any[]; // Simplified tool_call_id?: string
|
||||
role: 'system' | 'user' | 'assistant' | 'tool'
|
||||
content: string | null | Content[] // Content can be a string OR an array of content parts
|
||||
name?: string
|
||||
tool_calls?: any[] // Simplified tool_call_id?: string
|
||||
}
|
||||
|
||||
export interface Content {
|
||||
type: 'text' | 'input_image' | 'input_audio';
|
||||
text?: string;
|
||||
image_url?: string;
|
||||
input_audio?: InputAudio;
|
||||
type: 'text' | 'input_image' | 'input_audio'
|
||||
text?: string
|
||||
image_url?: string
|
||||
input_audio?: InputAudio
|
||||
}
|
||||
|
||||
export interface InputAudio {
|
||||
data: string; // Base64 encoded audio data
|
||||
format: 'mp3' | 'wav' | 'ogg' | 'flac'; // Add more formats as needed/llama-server seems to support mp3
|
||||
data: string // Base64 encoded audio data
|
||||
format: 'mp3' | 'wav' | 'ogg' | 'flac' // Add more formats as needed/llama-server seems to support mp3
|
||||
}
|
||||
|
||||
export interface chatCompletionRequest {
|
||||
model: string; // Model ID, though for local it might be implicit via sessionInfo
|
||||
messages: chatCompletionRequestMessage[];
|
||||
model: string // Model ID, though for local it might be implicit via sessionInfo
|
||||
messages: chatCompletionRequestMessage[]
|
||||
|
||||
// Core sampling parameters
|
||||
temperature?: number | null;
|
||||
dynatemp_range?: number | null;
|
||||
dynatemp_exponent?: number | null;
|
||||
top_k?: number | null;
|
||||
top_p?: number | null;
|
||||
min_p?: number | null;
|
||||
typical_p?: number | null;
|
||||
repeat_penalty?: number | null;
|
||||
repeat_last_n?: number | null;
|
||||
presence_penalty?: number | null;
|
||||
frequency_penalty?: number | null;
|
||||
dry_multiplier?: number | null;
|
||||
dry_base?: number | null;
|
||||
dry_allowed_length?: number | null;
|
||||
dry_penalty_last_n?: number | null;
|
||||
dry_sequence_breakers?: string[] | null;
|
||||
xtc_probability?: number | null;
|
||||
xtc_threshold?: number | null;
|
||||
mirostat?: number | null; // 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0
|
||||
mirostat_tau?: number | null;
|
||||
mirostat_eta?: number | null;
|
||||
temperature?: number | null
|
||||
dynatemp_range?: number | null
|
||||
dynatemp_exponent?: number | null
|
||||
top_k?: number | null
|
||||
top_p?: number | null
|
||||
min_p?: number | null
|
||||
typical_p?: number | null
|
||||
repeat_penalty?: number | null
|
||||
repeat_last_n?: number | null
|
||||
presence_penalty?: number | null
|
||||
frequency_penalty?: number | null
|
||||
dry_multiplier?: number | null
|
||||
dry_base?: number | null
|
||||
dry_allowed_length?: number | null
|
||||
dry_penalty_last_n?: number | null
|
||||
dry_sequence_breakers?: string[] | null
|
||||
xtc_probability?: number | null
|
||||
xtc_threshold?: number | null
|
||||
mirostat?: number | null // 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0
|
||||
mirostat_tau?: number | null
|
||||
mirostat_eta?: number | null
|
||||
|
||||
n_predict?: number | null;
|
||||
n_indent?: number | null;
|
||||
n_keep?: number | null;
|
||||
stream?: boolean | null;
|
||||
stop?: string | string[] | null;
|
||||
seed?: number | null; // RNG seed
|
||||
n_predict?: number | null
|
||||
n_indent?: number | null
|
||||
n_keep?: number | null
|
||||
stream?: boolean | null
|
||||
stop?: string | string[] | null
|
||||
seed?: number | null // RNG seed
|
||||
|
||||
// Advanced sampling
|
||||
logit_bias?: { [key: string]: number } | null;
|
||||
n_probs?: number | null;
|
||||
min_keep?: number | null;
|
||||
t_max_predict_ms?: number | null;
|
||||
image_data?: Array<{ data: string; id: number }> | null;
|
||||
logit_bias?: { [key: string]: number } | null
|
||||
n_probs?: number | null
|
||||
min_keep?: number | null
|
||||
t_max_predict_ms?: number | null
|
||||
image_data?: Array<{ data: string; id: number }> | null
|
||||
|
||||
// Internal/optimization parameters
|
||||
id_slot?: number | null;
|
||||
cache_prompt?: boolean | null;
|
||||
return_tokens?: boolean | null;
|
||||
samplers?: string[] | null;
|
||||
timings_per_token?: boolean | null;
|
||||
post_sampling_probs?: boolean | null;
|
||||
id_slot?: number | null
|
||||
cache_prompt?: boolean | null
|
||||
return_tokens?: boolean | null
|
||||
samplers?: string[] | null
|
||||
timings_per_token?: boolean | null
|
||||
post_sampling_probs?: boolean | null
|
||||
}
|
||||
|
||||
export interface chatCompletionChunkChoiceDelta {
|
||||
@ -208,7 +208,9 @@ export abstract class AIEngine extends BaseExtension {
|
||||
/**
|
||||
* Sends a chat request to the model
|
||||
*/
|
||||
abstract chat(opts: chatCompletionRequest): Promise<chatCompletion | AsyncIterable<chatCompletionChunk>>
|
||||
abstract chat(
|
||||
opts: chatCompletionRequest
|
||||
): Promise<chatCompletion | AsyncIterable<chatCompletionChunk>>
|
||||
|
||||
/**
|
||||
* Deletes a model
|
||||
|
||||
@ -1,566 +0,0 @@
|
||||
import { EngineManagementExtension } from './enginesManagement'
|
||||
import { ExtensionTypeEnum } from '../extension'
|
||||
import {
|
||||
EngineConfig,
|
||||
EngineReleased,
|
||||
EngineVariant,
|
||||
Engines,
|
||||
InferenceEngine,
|
||||
DefaultEngineVariant,
|
||||
Model
|
||||
} from '../../types'
|
||||
|
||||
// Mock implementation of EngineManagementExtension
|
||||
class MockEngineManagementExtension extends EngineManagementExtension {
|
||||
private mockEngines: Engines = {
|
||||
llama: {
|
||||
name: 'llama',
|
||||
variants: [
|
||||
{
|
||||
variant: 'cpu',
|
||||
version: '1.0.0',
|
||||
path: '/engines/llama/cpu/1.0.0',
|
||||
installed: true
|
||||
},
|
||||
{
|
||||
variant: 'cuda',
|
||||
version: '1.0.0',
|
||||
path: '/engines/llama/cuda/1.0.0',
|
||||
installed: false
|
||||
}
|
||||
],
|
||||
default: {
|
||||
variant: 'cpu',
|
||||
version: '1.0.0'
|
||||
}
|
||||
},
|
||||
gpt4all: {
|
||||
name: 'gpt4all',
|
||||
variants: [
|
||||
{
|
||||
variant: 'cpu',
|
||||
version: '2.0.0',
|
||||
path: '/engines/gpt4all/cpu/2.0.0',
|
||||
installed: true
|
||||
}
|
||||
],
|
||||
default: {
|
||||
variant: 'cpu',
|
||||
version: '2.0.0'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private mockReleases: { [key: string]: EngineReleased[] } = {
|
||||
'llama-1.0.0': [
|
||||
{
|
||||
variant: 'cpu',
|
||||
version: '1.0.0',
|
||||
os: ['macos', 'linux', 'windows'],
|
||||
url: 'https://example.com/llama/1.0.0/cpu'
|
||||
},
|
||||
{
|
||||
variant: 'cuda',
|
||||
version: '1.0.0',
|
||||
os: ['linux', 'windows'],
|
||||
url: 'https://example.com/llama/1.0.0/cuda'
|
||||
}
|
||||
],
|
||||
'llama-1.1.0': [
|
||||
{
|
||||
variant: 'cpu',
|
||||
version: '1.1.0',
|
||||
os: ['macos', 'linux', 'windows'],
|
||||
url: 'https://example.com/llama/1.1.0/cpu'
|
||||
},
|
||||
{
|
||||
variant: 'cuda',
|
||||
version: '1.1.0',
|
||||
os: ['linux', 'windows'],
|
||||
url: 'https://example.com/llama/1.1.0/cuda'
|
||||
}
|
||||
],
|
||||
'gpt4all-2.0.0': [
|
||||
{
|
||||
variant: 'cpu',
|
||||
version: '2.0.0',
|
||||
os: ['macos', 'linux', 'windows'],
|
||||
url: 'https://example.com/gpt4all/2.0.0/cpu'
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
private remoteModels: { [engine: string]: Model[] } = {
|
||||
'llama': [],
|
||||
'gpt4all': []
|
||||
}
|
||||
|
||||
constructor() {
|
||||
super('http://mock-url.com', 'mock-engine-extension', 'Mock Engine Extension', true, 'A mock engine extension', '1.0.0')
|
||||
}
|
||||
|
||||
onLoad(): void {
|
||||
// Mock implementation
|
||||
}
|
||||
|
||||
onUnload(): void {
|
||||
// Mock implementation
|
||||
}
|
||||
|
||||
async getEngines(): Promise<Engines> {
|
||||
return JSON.parse(JSON.stringify(this.mockEngines))
|
||||
}
|
||||
|
||||
async getInstalledEngines(name: InferenceEngine): Promise<EngineVariant[]> {
|
||||
if (!this.mockEngines[name]) {
|
||||
return []
|
||||
}
|
||||
|
||||
return this.mockEngines[name].variants.filter(variant => variant.installed)
|
||||
}
|
||||
|
||||
async getReleasedEnginesByVersion(
|
||||
name: InferenceEngine,
|
||||
version: string,
|
||||
platform?: string
|
||||
): Promise<EngineReleased[]> {
|
||||
const key = `${name}-${version}`
|
||||
let releases = this.mockReleases[key] || []
|
||||
|
||||
if (platform) {
|
||||
releases = releases.filter(release => release.os.includes(platform))
|
||||
}
|
||||
|
||||
return releases
|
||||
}
|
||||
|
||||
async getLatestReleasedEngine(
|
||||
name: InferenceEngine,
|
||||
platform?: string
|
||||
): Promise<EngineReleased[]> {
|
||||
// For mock, let's assume latest versions are 1.1.0 for llama and 2.0.0 for gpt4all
|
||||
const latestVersions = {
|
||||
'llama': '1.1.0',
|
||||
'gpt4all': '2.0.0'
|
||||
}
|
||||
|
||||
if (!latestVersions[name]) {
|
||||
return []
|
||||
}
|
||||
|
||||
return this.getReleasedEnginesByVersion(name, latestVersions[name], platform)
|
||||
}
|
||||
|
||||
async installEngine(
|
||||
name: string,
|
||||
engineConfig: EngineConfig
|
||||
): Promise<{ messages: string }> {
|
||||
if (!this.mockEngines[name]) {
|
||||
this.mockEngines[name] = {
|
||||
name,
|
||||
variants: [],
|
||||
default: {
|
||||
variant: engineConfig.variant,
|
||||
version: engineConfig.version
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if variant already exists
|
||||
const existingVariantIndex = this.mockEngines[name].variants.findIndex(
|
||||
v => v.variant === engineConfig.variant && v.version === engineConfig.version
|
||||
)
|
||||
|
||||
if (existingVariantIndex >= 0) {
|
||||
this.mockEngines[name].variants[existingVariantIndex].installed = true
|
||||
} else {
|
||||
this.mockEngines[name].variants.push({
|
||||
variant: engineConfig.variant,
|
||||
version: engineConfig.version,
|
||||
path: `/engines/${name}/${engineConfig.variant}/${engineConfig.version}`,
|
||||
installed: true
|
||||
})
|
||||
}
|
||||
|
||||
return { messages: `Successfully installed ${name} ${engineConfig.variant} ${engineConfig.version}` }
|
||||
}
|
||||
|
||||
async addRemoteEngine(
|
||||
engineConfig: EngineConfig
|
||||
): Promise<{ messages: string }> {
|
||||
const name = engineConfig.name || 'remote-engine'
|
||||
|
||||
if (!this.mockEngines[name]) {
|
||||
this.mockEngines[name] = {
|
||||
name,
|
||||
variants: [],
|
||||
default: {
|
||||
variant: engineConfig.variant,
|
||||
version: engineConfig.version
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.mockEngines[name].variants.push({
|
||||
variant: engineConfig.variant,
|
||||
version: engineConfig.version,
|
||||
path: engineConfig.path || `/engines/${name}/${engineConfig.variant}/${engineConfig.version}`,
|
||||
installed: true,
|
||||
url: engineConfig.url
|
||||
})
|
||||
|
||||
return { messages: `Successfully added remote engine ${name}` }
|
||||
}
|
||||
|
||||
async uninstallEngine(
|
||||
name: InferenceEngine,
|
||||
engineConfig: EngineConfig
|
||||
): Promise<{ messages: string }> {
|
||||
if (!this.mockEngines[name]) {
|
||||
return { messages: `Engine ${name} not found` }
|
||||
}
|
||||
|
||||
const variantIndex = this.mockEngines[name].variants.findIndex(
|
||||
v => v.variant === engineConfig.variant && v.version === engineConfig.version
|
||||
)
|
||||
|
||||
if (variantIndex >= 0) {
|
||||
this.mockEngines[name].variants[variantIndex].installed = false
|
||||
|
||||
// If this was the default variant, reset default
|
||||
if (
|
||||
this.mockEngines[name].default.variant === engineConfig.variant &&
|
||||
this.mockEngines[name].default.version === engineConfig.version
|
||||
) {
|
||||
// Find another installed variant to set as default
|
||||
const installedVariant = this.mockEngines[name].variants.find(v => v.installed)
|
||||
if (installedVariant) {
|
||||
this.mockEngines[name].default = {
|
||||
variant: installedVariant.variant,
|
||||
version: installedVariant.version
|
||||
}
|
||||
} else {
|
||||
// No installed variants remain, clear default
|
||||
this.mockEngines[name].default = { variant: '', version: '' }
|
||||
}
|
||||
}
|
||||
|
||||
return { messages: `Successfully uninstalled ${name} ${engineConfig.variant} ${engineConfig.version}` }
|
||||
} else {
|
||||
return { messages: `Variant ${engineConfig.variant} ${engineConfig.version} not found for engine ${name}` }
|
||||
}
|
||||
}
|
||||
|
||||
async getDefaultEngineVariant(
|
||||
name: InferenceEngine
|
||||
): Promise<DefaultEngineVariant> {
|
||||
if (!this.mockEngines[name]) {
|
||||
return { variant: '', version: '' }
|
||||
}
|
||||
|
||||
return this.mockEngines[name].default
|
||||
}
|
||||
|
||||
async setDefaultEngineVariant(
|
||||
name: InferenceEngine,
|
||||
engineConfig: EngineConfig
|
||||
): Promise<{ messages: string }> {
|
||||
if (!this.mockEngines[name]) {
|
||||
return { messages: `Engine ${name} not found` }
|
||||
}
|
||||
|
||||
const variantExists = this.mockEngines[name].variants.some(
|
||||
v => v.variant === engineConfig.variant && v.version === engineConfig.version && v.installed
|
||||
)
|
||||
|
||||
if (!variantExists) {
|
||||
return { messages: `Variant ${engineConfig.variant} ${engineConfig.version} not found or not installed` }
|
||||
}
|
||||
|
||||
this.mockEngines[name].default = {
|
||||
variant: engineConfig.variant,
|
||||
version: engineConfig.version
|
||||
}
|
||||
|
||||
return { messages: `Successfully set ${engineConfig.variant} ${engineConfig.version} as default for ${name}` }
|
||||
}
|
||||
|
||||
async updateEngine(
|
||||
name: InferenceEngine,
|
||||
engineConfig?: EngineConfig
|
||||
): Promise<{ messages: string }> {
|
||||
if (!this.mockEngines[name]) {
|
||||
return { messages: `Engine ${name} not found` }
|
||||
}
|
||||
|
||||
if (!engineConfig) {
|
||||
// Assume we're updating to the latest version
|
||||
return { messages: `Successfully updated ${name} to the latest version` }
|
||||
}
|
||||
|
||||
const variantIndex = this.mockEngines[name].variants.findIndex(
|
||||
v => v.variant === engineConfig.variant && v.installed
|
||||
)
|
||||
|
||||
if (variantIndex >= 0) {
|
||||
// Update the version
|
||||
this.mockEngines[name].variants[variantIndex].version = engineConfig.version
|
||||
|
||||
// If this was the default variant, update default version too
|
||||
if (this.mockEngines[name].default.variant === engineConfig.variant) {
|
||||
this.mockEngines[name].default.version = engineConfig.version
|
||||
}
|
||||
|
||||
return { messages: `Successfully updated ${name} ${engineConfig.variant} to version ${engineConfig.version}` }
|
||||
} else {
|
||||
return { messages: `Installed variant ${engineConfig.variant} not found for engine ${name}` }
|
||||
}
|
||||
}
|
||||
|
||||
async addRemoteModel(model: Model): Promise<void> {
|
||||
const engine = model.engine as string
|
||||
|
||||
if (!this.remoteModels[engine]) {
|
||||
this.remoteModels[engine] = []
|
||||
}
|
||||
|
||||
this.remoteModels[engine].push(model)
|
||||
}
|
||||
|
||||
async getRemoteModels(name: InferenceEngine | string): Promise<Model[]> {
|
||||
return this.remoteModels[name] || []
|
||||
}
|
||||
}
|
||||
|
||||
describe('EngineManagementExtension', () => {
|
||||
let extension: MockEngineManagementExtension
|
||||
|
||||
beforeEach(() => {
|
||||
extension = new MockEngineManagementExtension()
|
||||
})
|
||||
|
||||
test('should return the correct extension type', () => {
|
||||
expect(extension.type()).toBe(ExtensionTypeEnum.Engine)
|
||||
})
|
||||
|
||||
test('should get all engines', async () => {
|
||||
const engines = await extension.getEngines()
|
||||
|
||||
expect(engines).toBeDefined()
|
||||
expect(engines.llama).toBeDefined()
|
||||
expect(engines.gpt4all).toBeDefined()
|
||||
expect(engines.llama.variants).toHaveLength(2)
|
||||
expect(engines.gpt4all.variants).toHaveLength(1)
|
||||
})
|
||||
|
||||
test('should get installed engines', async () => {
|
||||
const llamaEngines = await extension.getInstalledEngines('llama')
|
||||
|
||||
expect(llamaEngines).toHaveLength(1)
|
||||
expect(llamaEngines[0].variant).toBe('cpu')
|
||||
expect(llamaEngines[0].installed).toBe(true)
|
||||
|
||||
const gpt4allEngines = await extension.getInstalledEngines('gpt4all')
|
||||
|
||||
expect(gpt4allEngines).toHaveLength(1)
|
||||
expect(gpt4allEngines[0].variant).toBe('cpu')
|
||||
expect(gpt4allEngines[0].installed).toBe(true)
|
||||
|
||||
// Test non-existent engine
|
||||
const nonExistentEngines = await extension.getInstalledEngines('non-existent' as InferenceEngine)
|
||||
expect(nonExistentEngines).toHaveLength(0)
|
||||
})
|
||||
|
||||
test('should get released engines by version', async () => {
|
||||
const llamaReleases = await extension.getReleasedEnginesByVersion('llama', '1.0.0')
|
||||
|
||||
expect(llamaReleases).toHaveLength(2)
|
||||
expect(llamaReleases[0].variant).toBe('cpu')
|
||||
expect(llamaReleases[1].variant).toBe('cuda')
|
||||
|
||||
// Test with platform filter
|
||||
const llamaLinuxReleases = await extension.getReleasedEnginesByVersion('llama', '1.0.0', 'linux')
|
||||
|
||||
expect(llamaLinuxReleases).toHaveLength(2)
|
||||
|
||||
const llamaMacReleases = await extension.getReleasedEnginesByVersion('llama', '1.0.0', 'macos')
|
||||
|
||||
expect(llamaMacReleases).toHaveLength(1)
|
||||
expect(llamaMacReleases[0].variant).toBe('cpu')
|
||||
|
||||
// Test non-existent version
|
||||
const nonExistentReleases = await extension.getReleasedEnginesByVersion('llama', '9.9.9')
|
||||
expect(nonExistentReleases).toHaveLength(0)
|
||||
})
|
||||
|
||||
test('should get latest released engines', async () => {
|
||||
const latestLlamaReleases = await extension.getLatestReleasedEngine('llama')
|
||||
|
||||
expect(latestLlamaReleases).toHaveLength(2)
|
||||
expect(latestLlamaReleases[0].version).toBe('1.1.0')
|
||||
|
||||
// Test with platform filter
|
||||
const latestLlamaMacReleases = await extension.getLatestReleasedEngine('llama', 'macos')
|
||||
|
||||
expect(latestLlamaMacReleases).toHaveLength(1)
|
||||
expect(latestLlamaMacReleases[0].variant).toBe('cpu')
|
||||
expect(latestLlamaMacReleases[0].version).toBe('1.1.0')
|
||||
|
||||
// Test non-existent engine
|
||||
const nonExistentReleases = await extension.getLatestReleasedEngine('non-existent' as InferenceEngine)
|
||||
expect(nonExistentReleases).toHaveLength(0)
|
||||
})
|
||||
|
||||
test('should install engine', async () => {
|
||||
// Install existing engine variant that is not installed
|
||||
const result = await extension.installEngine('llama', { variant: 'cuda', version: '1.0.0' })
|
||||
|
||||
expect(result.messages).toContain('Successfully installed')
|
||||
|
||||
const installedEngines = await extension.getInstalledEngines('llama')
|
||||
expect(installedEngines).toHaveLength(2)
|
||||
expect(installedEngines.some(e => e.variant === 'cuda')).toBe(true)
|
||||
|
||||
// Install non-existent engine
|
||||
const newEngineResult = await extension.installEngine('new-engine', { variant: 'cpu', version: '1.0.0' })
|
||||
|
||||
expect(newEngineResult.messages).toContain('Successfully installed')
|
||||
|
||||
const engines = await extension.getEngines()
|
||||
expect(engines['new-engine']).toBeDefined()
|
||||
expect(engines['new-engine'].variants).toHaveLength(1)
|
||||
expect(engines['new-engine'].variants[0].installed).toBe(true)
|
||||
})
|
||||
|
||||
test('should add remote engine', async () => {
|
||||
const result = await extension.addRemoteEngine({
|
||||
name: 'remote-llm',
|
||||
variant: 'remote',
|
||||
version: '1.0.0',
|
||||
url: 'https://example.com/remote-llm-api'
|
||||
})
|
||||
|
||||
expect(result.messages).toContain('Successfully added remote engine')
|
||||
|
||||
const engines = await extension.getEngines()
|
||||
expect(engines['remote-llm']).toBeDefined()
|
||||
expect(engines['remote-llm'].variants).toHaveLength(1)
|
||||
expect(engines['remote-llm'].variants[0].url).toBe('https://example.com/remote-llm-api')
|
||||
})
|
||||
|
||||
test('should uninstall engine', async () => {
|
||||
const result = await extension.uninstallEngine('llama', { variant: 'cpu', version: '1.0.0' })
|
||||
|
||||
expect(result.messages).toContain('Successfully uninstalled')
|
||||
|
||||
const installedEngines = await extension.getInstalledEngines('llama')
|
||||
expect(installedEngines).toHaveLength(0)
|
||||
|
||||
// Test uninstalling non-existent variant
|
||||
const nonExistentResult = await extension.uninstallEngine('llama', { variant: 'non-existent', version: '1.0.0' })
|
||||
|
||||
expect(nonExistentResult.messages).toContain('not found')
|
||||
})
|
||||
|
||||
test('should handle default variant when uninstalling', async () => {
|
||||
// First install cuda variant
|
||||
await extension.installEngine('llama', { variant: 'cuda', version: '1.0.0' })
|
||||
|
||||
// Set cuda as default
|
||||
await extension.setDefaultEngineVariant('llama', { variant: 'cuda', version: '1.0.0' })
|
||||
|
||||
// Check that cuda is now default
|
||||
let defaultVariant = await extension.getDefaultEngineVariant('llama')
|
||||
expect(defaultVariant.variant).toBe('cuda')
|
||||
|
||||
// Uninstall cuda
|
||||
await extension.uninstallEngine('llama', { variant: 'cuda', version: '1.0.0' })
|
||||
|
||||
// Check that default has changed to another installed variant
|
||||
defaultVariant = await extension.getDefaultEngineVariant('llama')
|
||||
expect(defaultVariant.variant).toBe('cpu')
|
||||
|
||||
// Uninstall all variants
|
||||
await extension.uninstallEngine('llama', { variant: 'cpu', version: '1.0.0' })
|
||||
|
||||
// Check that default is now empty
|
||||
defaultVariant = await extension.getDefaultEngineVariant('llama')
|
||||
expect(defaultVariant.variant).toBe('')
|
||||
expect(defaultVariant.version).toBe('')
|
||||
})
|
||||
|
||||
test('should get default engine variant', async () => {
|
||||
const llamaDefault = await extension.getDefaultEngineVariant('llama')
|
||||
|
||||
expect(llamaDefault.variant).toBe('cpu')
|
||||
expect(llamaDefault.version).toBe('1.0.0')
|
||||
|
||||
// Test non-existent engine
|
||||
const nonExistentDefault = await extension.getDefaultEngineVariant('non-existent' as InferenceEngine)
|
||||
expect(nonExistentDefault.variant).toBe('')
|
||||
expect(nonExistentDefault.version).toBe('')
|
||||
})
|
||||
|
||||
test('should set default engine variant', async () => {
|
||||
// Install cuda variant
|
||||
await extension.installEngine('llama', { variant: 'cuda', version: '1.0.0' })
|
||||
|
||||
const result = await extension.setDefaultEngineVariant('llama', { variant: 'cuda', version: '1.0.0' })
|
||||
|
||||
expect(result.messages).toContain('Successfully set')
|
||||
|
||||
const defaultVariant = await extension.getDefaultEngineVariant('llama')
|
||||
expect(defaultVariant.variant).toBe('cuda')
|
||||
expect(defaultVariant.version).toBe('1.0.0')
|
||||
|
||||
// Test setting non-existent variant as default
|
||||
const nonExistentResult = await extension.setDefaultEngineVariant('llama', { variant: 'non-existent', version: '1.0.0' })
|
||||
|
||||
expect(nonExistentResult.messages).toContain('not found')
|
||||
})
|
||||
|
||||
test('should update engine', async () => {
|
||||
const result = await extension.updateEngine('llama', { variant: 'cpu', version: '1.1.0' })
|
||||
|
||||
expect(result.messages).toContain('Successfully updated')
|
||||
|
||||
const engines = await extension.getEngines()
|
||||
const cpuVariant = engines.llama.variants.find(v => v.variant === 'cpu')
|
||||
expect(cpuVariant).toBeDefined()
|
||||
expect(cpuVariant?.version).toBe('1.1.0')
|
||||
|
||||
// Default should also be updated since cpu was default
|
||||
expect(engines.llama.default.version).toBe('1.1.0')
|
||||
|
||||
// Test updating non-existent variant
|
||||
const nonExistentResult = await extension.updateEngine('llama', { variant: 'non-existent', version: '1.1.0' })
|
||||
|
||||
expect(nonExistentResult.messages).toContain('not found')
|
||||
})
|
||||
|
||||
test('should add and get remote models', async () => {
|
||||
const model: Model = {
|
||||
id: 'remote-model-1',
|
||||
name: 'Remote Model 1',
|
||||
path: '/path/to/remote-model',
|
||||
engine: 'llama',
|
||||
format: 'gguf',
|
||||
modelFormat: 'gguf',
|
||||
source: 'remote',
|
||||
status: 'ready',
|
||||
contextLength: 4096,
|
||||
sizeInGB: 4,
|
||||
created: new Date().toISOString()
|
||||
}
|
||||
|
||||
await extension.addRemoteModel(model)
|
||||
|
||||
const llamaModels = await extension.getRemoteModels('llama')
|
||||
expect(llamaModels).toHaveLength(1)
|
||||
expect(llamaModels[0].id).toBe('remote-model-1')
|
||||
|
||||
// Test non-existent engine
|
||||
const nonExistentModels = await extension.getRemoteModels('non-existent')
|
||||
expect(nonExistentModels).toHaveLength(0)
|
||||
})
|
||||
})
|
||||
@ -1,115 +0,0 @@
|
||||
import {
|
||||
Engines,
|
||||
EngineVariant,
|
||||
EngineReleased,
|
||||
EngineConfig,
|
||||
DefaultEngineVariant,
|
||||
Model,
|
||||
} from '../../types'
|
||||
import { BaseExtension, ExtensionTypeEnum } from '../extension'
|
||||
|
||||
/**
|
||||
* Engine management extension. Persists and retrieves engine management.
|
||||
* @abstract
|
||||
* @extends BaseExtension
|
||||
*/
|
||||
export abstract class EngineManagementExtension extends BaseExtension {
|
||||
type(): ExtensionTypeEnum | undefined {
|
||||
return ExtensionTypeEnum.Engine
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns A Promise that resolves to an object of list engines.
|
||||
*/
|
||||
abstract getEngines(): Promise<Engines>
|
||||
|
||||
/**
|
||||
* @param name - Inference engine name.
|
||||
* @returns A Promise that resolves to an array of installed engine.
|
||||
*/
|
||||
abstract getInstalledEngines(name: string): Promise<EngineVariant[]>
|
||||
|
||||
/**
|
||||
* @param name - Inference engine name.
|
||||
* @param version - Version of the engine.
|
||||
* @param platform - Optional to sort by operating system. macOS, linux, windows.
|
||||
* @returns A Promise that resolves to an array of latest released engine by version.
|
||||
*/
|
||||
abstract getReleasedEnginesByVersion(
|
||||
name: string,
|
||||
version: string,
|
||||
platform?: string
|
||||
): Promise<EngineReleased[]>
|
||||
|
||||
/**
|
||||
* @param name - Inference engine name.
|
||||
* @param platform - Optional to sort by operating system. macOS, linux, windows.
|
||||
* @returns A Promise that resolves to an array of latest released engine.
|
||||
*/
|
||||
abstract getLatestReleasedEngine(
|
||||
name: string,
|
||||
platform?: string
|
||||
): Promise<EngineReleased[]>
|
||||
|
||||
/**
|
||||
* @param name - Inference engine name.
|
||||
* @returns A Promise that resolves to intall of engine.
|
||||
*/
|
||||
abstract installEngine(
|
||||
name: string,
|
||||
engineConfig: EngineConfig
|
||||
): Promise<{ messages: string }>
|
||||
|
||||
/**
|
||||
* Add a new remote engine
|
||||
* @returns A Promise that resolves to intall of engine.
|
||||
*/
|
||||
abstract addRemoteEngine(
|
||||
engineConfig: EngineConfig
|
||||
): Promise<{ messages: string }>
|
||||
|
||||
/**
|
||||
* @param name - Inference engine name.
|
||||
* @returns A Promise that resolves to unintall of engine.
|
||||
*/
|
||||
abstract uninstallEngine(
|
||||
name: string,
|
||||
engineConfig: EngineConfig
|
||||
): Promise<{ messages: string }>
|
||||
|
||||
/**
|
||||
* @param name - Inference engine name.
|
||||
* @returns A Promise that resolves to an object of default engine.
|
||||
*/
|
||||
abstract getDefaultEngineVariant(
|
||||
name: string
|
||||
): Promise<DefaultEngineVariant>
|
||||
|
||||
/**
|
||||
* @body variant - string
|
||||
* @body version - string
|
||||
* @returns A Promise that resolves to set default engine.
|
||||
*/
|
||||
abstract setDefaultEngineVariant(
|
||||
name: string,
|
||||
engineConfig: EngineConfig
|
||||
): Promise<{ messages: string }>
|
||||
|
||||
/**
|
||||
* @returns A Promise that resolves to update engine.
|
||||
*/
|
||||
abstract updateEngine(
|
||||
name: string,
|
||||
engineConfig?: EngineConfig
|
||||
): Promise<{ messages: string }>
|
||||
|
||||
/**
|
||||
* Add a new remote model for a specific engine
|
||||
*/
|
||||
abstract addRemoteModel(model: Model): Promise<void>
|
||||
|
||||
/**
|
||||
* @returns A Promise that resolves to an object of remote models list .
|
||||
*/
|
||||
abstract getRemoteModels(name: string): Promise<any>
|
||||
}
|
||||
@ -1,146 +0,0 @@
|
||||
import { HardwareManagementExtension } from './hardwareManagement'
|
||||
import { ExtensionTypeEnum } from '../extension'
|
||||
import { HardwareInformation } from '../../types'
|
||||
|
||||
// Mock implementation of HardwareManagementExtension
|
||||
class MockHardwareManagementExtension extends HardwareManagementExtension {
|
||||
private activeGpus: number[] = [0]
|
||||
private mockHardwareInfo: HardwareInformation = {
|
||||
cpu: {
|
||||
manufacturer: 'Mock CPU Manufacturer',
|
||||
brand: 'Mock CPU',
|
||||
cores: 8,
|
||||
physicalCores: 4,
|
||||
speed: 3.5,
|
||||
},
|
||||
memory: {
|
||||
total: 16 * 1024 * 1024 * 1024, // 16GB in bytes
|
||||
free: 8 * 1024 * 1024 * 1024, // 8GB in bytes
|
||||
},
|
||||
gpus: [
|
||||
{
|
||||
id: 0,
|
||||
vendor: 'Mock GPU Vendor',
|
||||
model: 'Mock GPU Model 1',
|
||||
memory: 8 * 1024 * 1024 * 1024, // 8GB in bytes
|
||||
},
|
||||
{
|
||||
id: 1,
|
||||
vendor: 'Mock GPU Vendor',
|
||||
model: 'Mock GPU Model 2',
|
||||
memory: 4 * 1024 * 1024 * 1024, // 4GB in bytes
|
||||
}
|
||||
],
|
||||
active_gpus: [0],
|
||||
}
|
||||
|
||||
constructor() {
|
||||
super('http://mock-url.com', 'mock-hardware-extension', 'Mock Hardware Extension', true, 'A mock hardware extension', '1.0.0')
|
||||
}
|
||||
|
||||
onLoad(): void {
|
||||
// Mock implementation
|
||||
}
|
||||
|
||||
onUnload(): void {
|
||||
// Mock implementation
|
||||
}
|
||||
|
||||
async getHardware(): Promise<HardwareInformation> {
|
||||
// Return a copy to prevent test side effects
|
||||
return JSON.parse(JSON.stringify(this.mockHardwareInfo))
|
||||
}
|
||||
|
||||
async setAvtiveGpu(data: { gpus: number[] }): Promise<{
|
||||
message: string
|
||||
activated_gpus: number[]
|
||||
}> {
|
||||
// Validate GPUs exist
|
||||
const validGpus = data.gpus.filter(gpuId =>
|
||||
this.mockHardwareInfo.gpus.some(gpu => gpu.id === gpuId)
|
||||
)
|
||||
|
||||
if (validGpus.length === 0) {
|
||||
throw new Error('No valid GPUs selected')
|
||||
}
|
||||
|
||||
// Update active GPUs
|
||||
this.activeGpus = validGpus
|
||||
this.mockHardwareInfo.active_gpus = validGpus
|
||||
|
||||
return {
|
||||
message: 'GPU activation successful',
|
||||
activated_gpus: validGpus
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
describe('HardwareManagementExtension', () => {
|
||||
let extension: MockHardwareManagementExtension
|
||||
|
||||
beforeEach(() => {
|
||||
extension = new MockHardwareManagementExtension()
|
||||
})
|
||||
|
||||
test('should return the correct extension type', () => {
|
||||
expect(extension.type()).toBe(ExtensionTypeEnum.Hardware)
|
||||
})
|
||||
|
||||
test('should get hardware information', async () => {
|
||||
const hardwareInfo = await extension.getHardware()
|
||||
|
||||
// Check CPU info
|
||||
expect(hardwareInfo.cpu).toBeDefined()
|
||||
expect(hardwareInfo.cpu.manufacturer).toBe('Mock CPU Manufacturer')
|
||||
expect(hardwareInfo.cpu.cores).toBe(8)
|
||||
|
||||
// Check memory info
|
||||
expect(hardwareInfo.memory).toBeDefined()
|
||||
expect(hardwareInfo.memory.total).toBe(16 * 1024 * 1024 * 1024)
|
||||
|
||||
// Check GPU info
|
||||
expect(hardwareInfo.gpus).toHaveLength(2)
|
||||
expect(hardwareInfo.gpus[0].model).toBe('Mock GPU Model 1')
|
||||
expect(hardwareInfo.gpus[1].model).toBe('Mock GPU Model 2')
|
||||
|
||||
// Check active GPUs
|
||||
expect(hardwareInfo.active_gpus).toEqual([0])
|
||||
})
|
||||
|
||||
test('should set active GPUs', async () => {
|
||||
const result = await extension.setAvtiveGpu({ gpus: [1] })
|
||||
|
||||
expect(result.message).toBe('GPU activation successful')
|
||||
expect(result.activated_gpus).toEqual([1])
|
||||
|
||||
// Verify the change in hardware info
|
||||
const hardwareInfo = await extension.getHardware()
|
||||
expect(hardwareInfo.active_gpus).toEqual([1])
|
||||
})
|
||||
|
||||
test('should set multiple active GPUs', async () => {
|
||||
const result = await extension.setAvtiveGpu({ gpus: [0, 1] })
|
||||
|
||||
expect(result.message).toBe('GPU activation successful')
|
||||
expect(result.activated_gpus).toEqual([0, 1])
|
||||
|
||||
// Verify the change in hardware info
|
||||
const hardwareInfo = await extension.getHardware()
|
||||
expect(hardwareInfo.active_gpus).toEqual([0, 1])
|
||||
})
|
||||
|
||||
test('should throw error for invalid GPU ids', async () => {
|
||||
await expect(extension.setAvtiveGpu({ gpus: [999] })).rejects.toThrow('No valid GPUs selected')
|
||||
})
|
||||
|
||||
test('should handle mix of valid and invalid GPU ids', async () => {
|
||||
const result = await extension.setAvtiveGpu({ gpus: [0, 999] })
|
||||
|
||||
// Should only activate valid GPUs
|
||||
expect(result.activated_gpus).toEqual([0])
|
||||
|
||||
// Verify the change in hardware info
|
||||
const hardwareInfo = await extension.getHardware()
|
||||
expect(hardwareInfo.active_gpus).toEqual([0])
|
||||
})
|
||||
})
|
||||
@ -1,26 +0,0 @@
|
||||
import { HardwareInformation } from '../../types'
|
||||
import { BaseExtension, ExtensionTypeEnum } from '../extension'
|
||||
|
||||
/**
|
||||
* Engine management extension. Persists and retrieves engine management.
|
||||
* @abstract
|
||||
* @extends BaseExtension
|
||||
*/
|
||||
export abstract class HardwareManagementExtension extends BaseExtension {
|
||||
type(): ExtensionTypeEnum | undefined {
|
||||
return ExtensionTypeEnum.Hardware
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns A Promise that resolves to an object of list hardware.
|
||||
*/
|
||||
abstract getHardware(): Promise<HardwareInformation>
|
||||
|
||||
/**
|
||||
* @returns A Promise that resolves to an object of set active gpus.
|
||||
*/
|
||||
abstract setActiveGpu(data: { gpus: number[] }): Promise<{
|
||||
message: string
|
||||
activated_gpus: number[]
|
||||
}>
|
||||
}
|
||||
@ -9,29 +9,12 @@ export { ConversationalExtension } from './conversational'
|
||||
*/
|
||||
export { InferenceExtension } from './inference'
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Assistant extension for managing assistants.
|
||||
*/
|
||||
export { AssistantExtension } from './assistant'
|
||||
|
||||
/**
|
||||
* Model extension for managing models.
|
||||
*/
|
||||
export { ModelExtension } from './model'
|
||||
|
||||
/**
|
||||
* Base AI Engines.
|
||||
*/
|
||||
export * from './engines'
|
||||
|
||||
/**
|
||||
* Engines Management
|
||||
*/
|
||||
export * from './enginesManagement'
|
||||
|
||||
/**
|
||||
* Hardware Management
|
||||
*/
|
||||
export * from './hardwareManagement'
|
||||
|
||||
@ -1,286 +0,0 @@
|
||||
import { ModelExtension } from './model'
|
||||
import { ExtensionTypeEnum } from '../extension'
|
||||
import { Model, OptionType, ModelSource } from '../../types'
|
||||
|
||||
// Mock implementation of ModelExtension
|
||||
class MockModelExtension extends ModelExtension {
|
||||
private models: Model[] = []
|
||||
private sources: ModelSource[] = []
|
||||
private loadedModels: Set<string> = new Set()
|
||||
private modelsPulling: Set<string> = new Set()
|
||||
|
||||
constructor() {
|
||||
super('http://mock-url.com', 'mock-model-extension', 'Mock Model Extension', true, 'A mock model extension', '1.0.0')
|
||||
}
|
||||
|
||||
onLoad(): void {
|
||||
// Mock implementation
|
||||
}
|
||||
|
||||
onUnload(): void {
|
||||
// Mock implementation
|
||||
}
|
||||
|
||||
async configurePullOptions(configs: { [key: string]: any }): Promise<any> {
|
||||
return configs
|
||||
}
|
||||
|
||||
async getModels(): Promise<Model[]> {
|
||||
return this.models
|
||||
}
|
||||
|
||||
async pullModel(model: string, id?: string, name?: string): Promise<void> {
|
||||
const modelId = id || `model-${Date.now()}`
|
||||
this.modelsPulling.add(modelId)
|
||||
|
||||
// Simulate model pull by adding it to the model list
|
||||
const newModel: Model = {
|
||||
id: modelId,
|
||||
path: `/models/${model}`,
|
||||
name: name || model,
|
||||
source: 'mock-source',
|
||||
modelFormat: 'mock-format',
|
||||
engine: 'mock-engine',
|
||||
format: 'mock-format',
|
||||
status: 'ready',
|
||||
contextLength: 2048,
|
||||
sizeInGB: 2,
|
||||
created: new Date().toISOString(),
|
||||
pullProgress: {
|
||||
percent: 100,
|
||||
transferred: 0,
|
||||
total: 0
|
||||
}
|
||||
}
|
||||
|
||||
this.models.push(newModel)
|
||||
this.loadedModels.add(modelId)
|
||||
this.modelsPulling.delete(modelId)
|
||||
}
|
||||
|
||||
async cancelModelPull(modelId: string): Promise<void> {
|
||||
this.modelsPulling.delete(modelId)
|
||||
// Remove the model if it's in the pulling state
|
||||
this.models = this.models.filter(m => m.id !== modelId)
|
||||
}
|
||||
|
||||
async importModel(
|
||||
model: string,
|
||||
modelPath: string,
|
||||
name?: string,
|
||||
optionType?: OptionType
|
||||
): Promise<void> {
|
||||
const newModel: Model = {
|
||||
id: `model-${Date.now()}`,
|
||||
path: modelPath,
|
||||
name: name || model,
|
||||
source: 'local',
|
||||
modelFormat: optionType?.format || 'mock-format',
|
||||
engine: optionType?.engine || 'mock-engine',
|
||||
format: optionType?.format || 'mock-format',
|
||||
status: 'ready',
|
||||
contextLength: optionType?.contextLength || 2048,
|
||||
sizeInGB: 2,
|
||||
created: new Date().toISOString(),
|
||||
}
|
||||
|
||||
this.models.push(newModel)
|
||||
this.loadedModels.add(newModel.id)
|
||||
}
|
||||
|
||||
async updateModel(modelInfo: Partial<Model>): Promise<Model> {
|
||||
if (!modelInfo.id) throw new Error('Model ID is required')
|
||||
|
||||
const index = this.models.findIndex(m => m.id === modelInfo.id)
|
||||
if (index === -1) throw new Error('Model not found')
|
||||
|
||||
this.models[index] = { ...this.models[index], ...modelInfo }
|
||||
return this.models[index]
|
||||
}
|
||||
|
||||
async deleteModel(modelId: string): Promise<void> {
|
||||
this.models = this.models.filter(m => m.id !== modelId)
|
||||
this.loadedModels.delete(modelId)
|
||||
}
|
||||
|
||||
async isModelLoaded(modelId: string): Promise<boolean> {
|
||||
return this.loadedModels.has(modelId)
|
||||
}
|
||||
|
||||
async getSources(): Promise<ModelSource[]> {
|
||||
return this.sources
|
||||
}
|
||||
|
||||
async addSource(source: string): Promise<void> {
|
||||
const newSource: ModelSource = {
|
||||
id: `source-${Date.now()}`,
|
||||
url: source,
|
||||
name: `Source ${this.sources.length + 1}`,
|
||||
type: 'mock-type'
|
||||
}
|
||||
|
||||
this.sources.push(newSource)
|
||||
}
|
||||
|
||||
async deleteSource(sourceId: string): Promise<void> {
|
||||
this.sources = this.sources.filter(s => s.id !== sourceId)
|
||||
}
|
||||
}
|
||||
|
||||
describe('ModelExtension', () => {
|
||||
let extension: MockModelExtension
|
||||
|
||||
beforeEach(() => {
|
||||
extension = new MockModelExtension()
|
||||
})
|
||||
|
||||
test('should return the correct extension type', () => {
|
||||
expect(extension.type()).toBe(ExtensionTypeEnum.Model)
|
||||
})
|
||||
|
||||
test('should configure pull options', async () => {
|
||||
const configs = { apiKey: 'test-key', baseUrl: 'https://test-url.com' }
|
||||
const result = await extension.configurePullOptions(configs)
|
||||
expect(result).toEqual(configs)
|
||||
})
|
||||
|
||||
test('should add and get models', async () => {
|
||||
await extension.pullModel('test-model', 'test-id', 'Test Model')
|
||||
|
||||
const models = await extension.getModels()
|
||||
expect(models).toHaveLength(1)
|
||||
expect(models[0].id).toBe('test-id')
|
||||
expect(models[0].name).toBe('Test Model')
|
||||
})
|
||||
|
||||
test('should pull model with default id and name', async () => {
|
||||
await extension.pullModel('test-model')
|
||||
|
||||
const models = await extension.getModels()
|
||||
expect(models).toHaveLength(1)
|
||||
expect(models[0].name).toBe('test-model')
|
||||
})
|
||||
|
||||
test('should cancel model pull', async () => {
|
||||
await extension.pullModel('test-model', 'test-id')
|
||||
|
||||
// Verify model exists
|
||||
let models = await extension.getModels()
|
||||
expect(models).toHaveLength(1)
|
||||
|
||||
// Cancel the pull
|
||||
await extension.cancelModelPull('test-id')
|
||||
|
||||
// Verify model was removed
|
||||
models = await extension.getModels()
|
||||
expect(models).toHaveLength(0)
|
||||
})
|
||||
|
||||
test('should import model', async () => {
|
||||
const optionType: OptionType = {
|
||||
engine: 'test-engine',
|
||||
format: 'test-format',
|
||||
contextLength: 4096
|
||||
}
|
||||
|
||||
await extension.importModel('test-model', '/path/to/model', 'Imported Model', optionType)
|
||||
|
||||
const models = await extension.getModels()
|
||||
expect(models).toHaveLength(1)
|
||||
expect(models[0].name).toBe('Imported Model')
|
||||
expect(models[0].engine).toBe('test-engine')
|
||||
expect(models[0].format).toBe('test-format')
|
||||
expect(models[0].contextLength).toBe(4096)
|
||||
})
|
||||
|
||||
test('should import model with default values', async () => {
|
||||
await extension.importModel('test-model', '/path/to/model')
|
||||
|
||||
const models = await extension.getModels()
|
||||
expect(models).toHaveLength(1)
|
||||
expect(models[0].name).toBe('test-model')
|
||||
expect(models[0].engine).toBe('mock-engine')
|
||||
expect(models[0].format).toBe('mock-format')
|
||||
})
|
||||
|
||||
test('should update model', async () => {
|
||||
await extension.pullModel('test-model', 'test-id', 'Test Model')
|
||||
|
||||
const updatedModel = await extension.updateModel({
|
||||
id: 'test-id',
|
||||
name: 'Updated Model',
|
||||
contextLength: 8192
|
||||
})
|
||||
|
||||
expect(updatedModel.name).toBe('Updated Model')
|
||||
expect(updatedModel.contextLength).toBe(8192)
|
||||
|
||||
// Verify changes persisted
|
||||
const models = await extension.getModels()
|
||||
expect(models[0].name).toBe('Updated Model')
|
||||
expect(models[0].contextLength).toBe(8192)
|
||||
})
|
||||
|
||||
test('should throw error when updating non-existent model', async () => {
|
||||
await expect(extension.updateModel({
|
||||
id: 'non-existent',
|
||||
name: 'Updated Model'
|
||||
})).rejects.toThrow('Model not found')
|
||||
})
|
||||
|
||||
test('should throw error when updating model without ID', async () => {
|
||||
await expect(extension.updateModel({
|
||||
name: 'Updated Model'
|
||||
})).rejects.toThrow('Model ID is required')
|
||||
})
|
||||
|
||||
test('should delete model', async () => {
|
||||
await extension.pullModel('test-model', 'test-id')
|
||||
|
||||
// Verify model exists
|
||||
let models = await extension.getModels()
|
||||
expect(models).toHaveLength(1)
|
||||
|
||||
// Delete the model
|
||||
await extension.deleteModel('test-id')
|
||||
|
||||
// Verify model was removed
|
||||
models = await extension.getModels()
|
||||
expect(models).toHaveLength(0)
|
||||
})
|
||||
|
||||
test('should check if model is loaded', async () => {
|
||||
await extension.pullModel('test-model', 'test-id')
|
||||
|
||||
// Check if model is loaded
|
||||
const isLoaded = await extension.isModelLoaded('test-id')
|
||||
expect(isLoaded).toBe(true)
|
||||
|
||||
// Check if non-existent model is loaded
|
||||
const nonExistentLoaded = await extension.isModelLoaded('non-existent')
|
||||
expect(nonExistentLoaded).toBe(false)
|
||||
})
|
||||
|
||||
test('should add and get sources', async () => {
|
||||
await extension.addSource('https://test-source.com')
|
||||
|
||||
const sources = await extension.getSources()
|
||||
expect(sources).toHaveLength(1)
|
||||
expect(sources[0].url).toBe('https://test-source.com')
|
||||
})
|
||||
|
||||
test('should delete source', async () => {
|
||||
await extension.addSource('https://test-source.com')
|
||||
|
||||
// Get the source ID
|
||||
const sources = await extension.getSources()
|
||||
const sourceId = sources[0].id
|
||||
|
||||
// Delete the source
|
||||
await extension.deleteSource(sourceId)
|
||||
|
||||
// Verify source was removed
|
||||
const updatedSources = await extension.getSources()
|
||||
expect(updatedSources).toHaveLength(0)
|
||||
})
|
||||
})
|
||||
@ -1,48 +0,0 @@
|
||||
import { BaseExtension, ExtensionTypeEnum } from '../extension'
|
||||
import { Model, ModelInterface, ModelSource, OptionType } from '../../types'
|
||||
|
||||
/**
|
||||
* Model extension for managing models.
|
||||
*/
|
||||
export abstract class ModelExtension
|
||||
extends BaseExtension
|
||||
implements ModelInterface
|
||||
{
|
||||
/**
|
||||
* Model extension type.
|
||||
*/
|
||||
type(): ExtensionTypeEnum | undefined {
|
||||
return ExtensionTypeEnum.Model
|
||||
}
|
||||
|
||||
abstract configurePullOptions(configs: { [key: string]: any }): Promise<any>
|
||||
abstract getModels(): Promise<Model[]>
|
||||
abstract pullModel(model: string, id?: string, name?: string): Promise<void>
|
||||
abstract cancelModelPull(modelId: string): Promise<void>
|
||||
abstract importModel(
|
||||
model: string,
|
||||
modePath: string,
|
||||
name?: string,
|
||||
optionType?: OptionType
|
||||
): Promise<void>
|
||||
abstract updateModel(modelInfo: Partial<Model>): Promise<Model>
|
||||
abstract deleteModel(model: string): Promise<void>
|
||||
abstract isModelLoaded(model: string): Promise<boolean>
|
||||
/**
|
||||
* Get model sources
|
||||
*/
|
||||
abstract getSources(): Promise<ModelSource[]>
|
||||
/**
|
||||
* Add a model source
|
||||
*/
|
||||
abstract addSource(source: string): Promise<void>
|
||||
/**
|
||||
* Delete a model source
|
||||
*/
|
||||
abstract deleteSource(source: string): Promise<void>
|
||||
|
||||
/**
|
||||
* Fetch models hub
|
||||
*/
|
||||
abstract fetchModelsHub(): Promise<void>
|
||||
}
|
||||
@ -1,47 +0,0 @@
|
||||
import anthropic from './resources/anthropic.json' with { type: 'json' }
|
||||
import cohere from './resources/cohere.json' with { type: 'json' }
|
||||
import openai from './resources/openai.json' with { type: 'json' }
|
||||
import openrouter from './resources/openrouter.json' with { type: 'json' }
|
||||
import groq from './resources/groq.json' with { type: 'json' }
|
||||
import martian from './resources/martian.json' with { type: 'json' }
|
||||
import mistral from './resources/mistral.json' with { type: 'json' }
|
||||
import nvidia from './resources/nvidia.json' with { type: 'json' }
|
||||
import deepseek from './resources/deepseek.json' with { type: 'json' }
|
||||
import googleGemini from './resources/google_gemini.json' with { type: 'json' }
|
||||
|
||||
import anthropicModels from './models/anthropic.json' with { type: 'json' }
|
||||
import cohereModels from './models/cohere.json' with { type: 'json' }
|
||||
import openaiModels from './models/openai.json' with { type: 'json' }
|
||||
import openrouterModels from './models/openrouter.json' with { type: 'json' }
|
||||
import groqModels from './models/groq.json' with { type: 'json' }
|
||||
import martianModels from './models/martian.json' with { type: 'json' }
|
||||
import mistralModels from './models/mistral.json' with { type: 'json' }
|
||||
import nvidiaModels from './models/nvidia.json' with { type: 'json' }
|
||||
import deepseekModels from './models/deepseek.json' with { type: 'json' }
|
||||
import googleGeminiModels from './models/google_gemini.json' with { type: 'json' }
|
||||
|
||||
const engines = [
|
||||
anthropic,
|
||||
openai,
|
||||
cohere,
|
||||
openrouter,
|
||||
groq,
|
||||
mistral,
|
||||
martian,
|
||||
nvidia,
|
||||
deepseek,
|
||||
googleGemini,
|
||||
]
|
||||
const models = [
|
||||
...anthropicModels,
|
||||
...openaiModels,
|
||||
...cohereModels,
|
||||
...openrouterModels,
|
||||
...groqModels,
|
||||
...mistralModels,
|
||||
...martianModels,
|
||||
...nvidiaModels,
|
||||
...deepseekModels,
|
||||
...googleGeminiModels,
|
||||
]
|
||||
export { engines, models }
|
||||
@ -1,5 +0,0 @@
|
||||
/** @type {import('ts-jest').JestConfigWithTsJest} */
|
||||
module.exports = {
|
||||
preset: 'ts-jest',
|
||||
testEnvironment: 'node',
|
||||
}
|
||||
@ -1,58 +0,0 @@
|
||||
[
|
||||
{
|
||||
"model": "claude-3-opus-latest",
|
||||
"object": "model",
|
||||
"name": "Claude 3 Opus Latest",
|
||||
"version": "1.0",
|
||||
"description": "Claude 3 Opus is a powerful model suitables for highly complex task.",
|
||||
"inference_params": {
|
||||
"max_tokens": 4096,
|
||||
"temperature": 0.7,
|
||||
"max_temperature": 1.0,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "anthropic"
|
||||
},
|
||||
{
|
||||
"model": "claude-3-5-haiku-latest",
|
||||
"object": "model",
|
||||
"name": "Claude 3.5 Haiku Latest",
|
||||
"version": "1.0",
|
||||
"description": "Claude 3.5 Haiku is the fastest model provides near-instant responsiveness.",
|
||||
"inference_params": {
|
||||
"max_tokens": 8192,
|
||||
"temperature": 0.7,
|
||||
"max_temperature": 1.0,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "anthropic"
|
||||
},
|
||||
{
|
||||
"model": "claude-3-5-sonnet-latest",
|
||||
"object": "model",
|
||||
"name": "Claude 3.5 Sonnet Latest",
|
||||
"version": "1.0",
|
||||
"description": "Claude 3.5 Sonnet raises the industry bar for intelligence, outperforming competitor models and Claude 3 Opus on a wide range of evaluations, with the speed and cost of our mid-tier model, Claude 3 Sonnet.",
|
||||
"inference_params": {
|
||||
"max_tokens": 8192,
|
||||
"temperature": 0.7,
|
||||
"max_temperature": 1.0,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "anthropic"
|
||||
},
|
||||
{
|
||||
"model": "claude-3-7-sonnet-latest",
|
||||
"object": "model",
|
||||
"name": "Claude 3.7 Sonnet Latest",
|
||||
"version": "1.0",
|
||||
"description": "Claude 3.7 Sonnet is the first hybrid reasoning model on the market. It is the most intelligent model yet. It is faster, more cost effective, and more capable than any other model in its class.",
|
||||
"inference_params": {
|
||||
"max_tokens": 8192,
|
||||
"temperature": 0.7,
|
||||
"max_temperature": 1.0,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "anthropic"
|
||||
}
|
||||
]
|
||||
@ -1,44 +0,0 @@
|
||||
[
|
||||
{
|
||||
"model": "command-r-plus",
|
||||
"object": "model",
|
||||
"name": "Command R+",
|
||||
"version": "1.0",
|
||||
"description": "Command R+ is an instruction-following conversational model that performs language tasks at a higher quality, more reliably, and with a longer context than previous models. It is best suited for complex RAG workflows and multi-step tool use.",
|
||||
"inference_params": {
|
||||
"max_tokens": 4096,
|
||||
"temperature": 0.7,
|
||||
"max_temperature": 1.0,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "cohere"
|
||||
},
|
||||
{
|
||||
"model": "command-r",
|
||||
"object": "model",
|
||||
"name": "Command R",
|
||||
"version": "1.0",
|
||||
"description": "Command R is an instruction-following conversational model that performs language tasks at a higher quality, more reliably, and with a longer context than previous models. It can be used for complex workflows like code generation, retrieval augmented generation (RAG), tool use, and agents.",
|
||||
"inference_params": {
|
||||
"max_tokens": 4096,
|
||||
"temperature": 0.7,
|
||||
"max_temperature": 1.0,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "cohere"
|
||||
},
|
||||
{
|
||||
"model": "command-a-03-2025",
|
||||
"object": "model",
|
||||
"name": "Command A",
|
||||
"version": "1.0",
|
||||
"description": "Command A is an instruction-following conversational model that performs language tasks at a higher quality, more reliably, and with a longer context than previous models. It is best suited for complex RAG workflows and multi-step tool use.",
|
||||
"inference_params": {
|
||||
"max_tokens": 4096,
|
||||
"temperature": 0.7,
|
||||
"max_temperature": 1.0,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "cohere"
|
||||
}
|
||||
]
|
||||
@ -1,28 +0,0 @@
|
||||
[
|
||||
{
|
||||
"model": "deepseek-chat",
|
||||
"object": "model",
|
||||
"name": "DeepSeek V3",
|
||||
"version": "1.0",
|
||||
"description": "The deepseek-chat model has been upgraded to DeepSeek-V3. deepseek-reasoner points to the new model DeepSeek-R1",
|
||||
"inference_params": {
|
||||
"max_tokens": 8192,
|
||||
"temperature": 0.6,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "deepseek"
|
||||
},
|
||||
{
|
||||
"model": "deepseek-reasoner",
|
||||
"object": "model",
|
||||
"name": "DeepSeek R1",
|
||||
"version": "1.0",
|
||||
"description": "CoT (Chain of Thought) is the reasoning content deepseek-reasoner gives before output the final answer. For details, please refer to Reasoning Model.",
|
||||
"inference_params": {
|
||||
"max_tokens": 8192,
|
||||
"temperature": 0.6,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "deepseek"
|
||||
}
|
||||
]
|
||||
@ -1,93 +0,0 @@
|
||||
[
|
||||
{
|
||||
"model": "gemini-1.5-flash",
|
||||
"object": "model",
|
||||
"name": "Gemini 1.5 Flash",
|
||||
"version": "1.0",
|
||||
"description": "Gemini 1.5 Flash is a fast and versatile multimodal model for scaling across diverse tasks.",
|
||||
"inference_params": {
|
||||
"max_tokens": 8192,
|
||||
"temperature": 0.6,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "google_gemini"
|
||||
},
|
||||
{
|
||||
"model": "gemini-1.5-flash-8b",
|
||||
"object": "model",
|
||||
"name": "Gemini 1.5 Flash-8B",
|
||||
"version": "1.0",
|
||||
"description": "Gemini 1.5 Flash-8B is a small model designed for lower intelligence tasks.",
|
||||
"inference_params": {
|
||||
"max_tokens": 8192,
|
||||
"temperature": 0.6,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "google_gemini"
|
||||
},
|
||||
{
|
||||
"model": "gemini-1.5-pro",
|
||||
"object": "model",
|
||||
"name": "Gemini 1.5 Pro",
|
||||
"version": "1.0",
|
||||
"description": "Gemini 1.5 Pro is a mid-size multimodal model that is optimized for a wide-range of reasoning tasks. 1.5 Pro can process large amounts of data at once, including 2 hours of video, 19 hours of audio, codebases with 60,000 lines of code, or 2,000 pages of text. ",
|
||||
"inference_params": {
|
||||
"max_tokens": 8192,
|
||||
"temperature": 0.6,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "google_gemini"
|
||||
},
|
||||
{
|
||||
"model": "gemini-2.5-pro-preview-05-06",
|
||||
"object": "model",
|
||||
"name": "Gemini 2.5 Pro Preview",
|
||||
"version": "1.0",
|
||||
"description": "Gemini 2.5 Pro is our state-of-the-art thinking model, capable of reasoning over complex problems in code, math, and STEM, as well as analyzing large datasets, codebases, and documents using long context. Gemini 2.5 Pro rate limits are more restricted since it is an experimental / preview model.",
|
||||
"inference_params": {
|
||||
"max_tokens": 65536,
|
||||
"temperature": 0.6,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "google_gemini"
|
||||
},
|
||||
{
|
||||
"model": "gemini-2.5-flash-preview-04-17",
|
||||
"object": "model",
|
||||
"name": "Our best model in terms of price-performance, offering well-rounded capabilities. Gemini 2.5 Flash rate limits are more restricted since it is an experimental / preview model.",
|
||||
"version": "1.0",
|
||||
"description": "Gemini 2.5 Flash preview",
|
||||
"inference_params": {
|
||||
"max_tokens": 8192,
|
||||
"temperature": 0.6,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "google_gemini"
|
||||
},
|
||||
{
|
||||
"model": "gemini-2.0-flash",
|
||||
"object": "model",
|
||||
"name": "Gemini 2.0 Flash",
|
||||
"version": "1.0",
|
||||
"description": "Gemini 2.0 Flash delivers next-gen features and improved capabilities, including superior speed, native tool use, multimodal generation, and a 1M token context window.",
|
||||
"inference_params": {
|
||||
"max_tokens": 8192,
|
||||
"temperature": 0.6,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "google_gemini"
|
||||
},
|
||||
{
|
||||
"model": "gemini-2.0-flash-lite",
|
||||
"object": "model",
|
||||
"name": "Gemini 2.0 Flash-Lite",
|
||||
"version": "1.0",
|
||||
"description": "A Gemini 2.0 Flash model optimized for cost efficiency and low latency.",
|
||||
"inference_params": {
|
||||
"max_tokens": 8192,
|
||||
"temperature": 0.6,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "google_gemini"
|
||||
}
|
||||
]
|
||||
@ -1,87 +0,0 @@
|
||||
[
|
||||
{
|
||||
"model": "llama3-70b-8192",
|
||||
"object": "model",
|
||||
"name": "Groq Llama 3 70b",
|
||||
"version": "1.1",
|
||||
"description": "Groq Llama 3 70b with supercharged speed!",
|
||||
"inference_params": {
|
||||
"max_tokens": 8192,
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.95,
|
||||
"stream": true,
|
||||
"stop": [],
|
||||
"frequency_penalty": 0,
|
||||
"presence_penalty": 0
|
||||
},
|
||||
"engine": "groq"
|
||||
},
|
||||
{
|
||||
"model": "llama3-8b-8192",
|
||||
"object": "model",
|
||||
"name": "Groq Llama 3 8b",
|
||||
"version": "1.1",
|
||||
"description": "Groq Llama 3 8b with supercharged speed!",
|
||||
"inference_params": {
|
||||
"max_tokens": 8192,
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.95,
|
||||
"stream": true,
|
||||
"stop": [],
|
||||
"frequency_penalty": 0,
|
||||
"presence_penalty": 0
|
||||
},
|
||||
"engine": "groq"
|
||||
},
|
||||
{
|
||||
"model": "llama-3.1-8b-instant",
|
||||
"object": "model",
|
||||
"name": "Groq Llama 3.1 8b Instant",
|
||||
"version": "1.1",
|
||||
"description": "Groq Llama 3.1 8b with supercharged speed!",
|
||||
"inference_params": {
|
||||
"max_tokens": 8000,
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.95,
|
||||
"stream": true,
|
||||
"stop": [],
|
||||
"frequency_penalty": 0,
|
||||
"presence_penalty": 0
|
||||
},
|
||||
"engine": "groq"
|
||||
},
|
||||
{
|
||||
"model": "gemma2-9b-it",
|
||||
"object": "model",
|
||||
"name": "Groq Gemma 9B Instruct",
|
||||
"version": "1.2",
|
||||
"description": "Groq Gemma 9b Instruct with supercharged speed!",
|
||||
"parameters": {
|
||||
"max_tokens": 8192,
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.95,
|
||||
"stream": true,
|
||||
"stop": [],
|
||||
"frequency_penalty": 0,
|
||||
"presence_penalty": 0
|
||||
},
|
||||
"engine": "groq"
|
||||
},
|
||||
{
|
||||
"model": "llama-3.3-70b-versatile",
|
||||
"object": "model",
|
||||
"name": "Groq Llama 3.3 70b Versatile",
|
||||
"version": "3.3",
|
||||
"description": "Groq Llama 3.3 70b Versatile with supercharged speed!",
|
||||
"parameters": {
|
||||
"max_tokens": 32768,
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.95,
|
||||
"stream": true,
|
||||
"stop": [],
|
||||
"frequency_penalty": 0,
|
||||
"presence_penalty": 0
|
||||
},
|
||||
"engine": "groq"
|
||||
}
|
||||
]
|
||||
@ -1,19 +0,0 @@
|
||||
[
|
||||
{
|
||||
"model": "router",
|
||||
"object": "model",
|
||||
"name": "Martian Model Router",
|
||||
"version": "1.0",
|
||||
"description": "Martian Model Router dynamically routes requests to the best LLM in real-time",
|
||||
"inference_params": {
|
||||
"max_tokens": 4096,
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.95,
|
||||
"stream": true,
|
||||
"stop": [],
|
||||
"frequency_penalty": 0,
|
||||
"presence_penalty": 0
|
||||
},
|
||||
"engine": "martian"
|
||||
}
|
||||
]
|
||||
@ -1,47 +0,0 @@
|
||||
[
|
||||
{
|
||||
"model": "mistral-small-latest",
|
||||
"object": "model",
|
||||
"name": "Mistral Small",
|
||||
"version": "1.1",
|
||||
"description": "Mistral Small is the ideal choice for simple tasks (Classification, Customer Support, or Text Generation) at an affordable price.",
|
||||
"inference_params": {
|
||||
"max_tokens": 32000,
|
||||
"temperature": 0.7,
|
||||
"max_temperature": 1.0,
|
||||
"top_p": 0.95,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "mistral"
|
||||
},
|
||||
{
|
||||
"model": "mistral-large-latest",
|
||||
"object": "model",
|
||||
"name": "Mistral Large",
|
||||
"version": "1.1",
|
||||
"description": "Mistral Large is ideal for complex tasks (Synthetic Text Generation, Code Generation, RAG, or Agents).",
|
||||
"inference_params": {
|
||||
"max_tokens": 32000,
|
||||
"temperature": 0.7,
|
||||
"max_temperature": 1.0,
|
||||
"top_p": 0.95,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "mistral"
|
||||
},
|
||||
{
|
||||
"model": "open-mixtral-8x22b",
|
||||
"object": "model",
|
||||
"name": "Mixtral 8x22B",
|
||||
"version": "1.1",
|
||||
"description": "Mixtral 8x22B is a high-performance, cost-effective model designed for complex tasks.",
|
||||
"inference_params": {
|
||||
"max_tokens": 32000,
|
||||
"temperature": 0.7,
|
||||
"max_temperature": 1.0,
|
||||
"top_p": 0.95,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "mistral"
|
||||
}
|
||||
]
|
||||
@ -1,21 +0,0 @@
|
||||
[
|
||||
{
|
||||
"model": "mistralai/mistral-7b-instruct-v0.2",
|
||||
"object": "model",
|
||||
"name": "Mistral 7B",
|
||||
"version": "1.1",
|
||||
"description": "Mistral 7B with NVIDIA",
|
||||
"inference_params": {
|
||||
"max_tokens": 1024,
|
||||
"temperature": 0.3,
|
||||
"max_temperature": 1.0,
|
||||
"top_p": 1,
|
||||
"stream": false,
|
||||
"frequency_penalty": 0,
|
||||
"presence_penalty": 0,
|
||||
"stop": null,
|
||||
"seed": null
|
||||
},
|
||||
"engine": "nvidia"
|
||||
}
|
||||
]
|
||||
@ -1,143 +0,0 @@
|
||||
[
|
||||
{
|
||||
"model": "gpt-4.5-preview",
|
||||
"object": "model",
|
||||
"name": "OpenAI GPT 4.5 Preview",
|
||||
"version": "1.2",
|
||||
"description": "OpenAI GPT 4.5 Preview is a research preview of GPT-4.5, our largest and most capable GPT model yet",
|
||||
"format": "api",
|
||||
"inference_params": {
|
||||
"max_tokens": 16384,
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.95,
|
||||
"stream": true,
|
||||
"stop": [],
|
||||
"frequency_penalty": 0,
|
||||
"presence_penalty": 0
|
||||
},
|
||||
"engine": "openai"
|
||||
},
|
||||
{
|
||||
"model": "gpt-4-turbo",
|
||||
"object": "model",
|
||||
"name": "OpenAI GPT 4 Turbo",
|
||||
"version": "1.2",
|
||||
"description": "OpenAI GPT 4 Turbo model is extremely good",
|
||||
"format": "api",
|
||||
"inference_params": {
|
||||
"max_tokens": 4096,
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.95,
|
||||
"stream": true,
|
||||
"stop": [],
|
||||
"frequency_penalty": 0,
|
||||
"presence_penalty": 0
|
||||
},
|
||||
"engine": "openai"
|
||||
},
|
||||
{
|
||||
"model": "gpt-3.5-turbo",
|
||||
"object": "model",
|
||||
"name": "OpenAI GPT 3.5 Turbo",
|
||||
"version": "1.1",
|
||||
"description": "OpenAI GPT 3.5 Turbo model is extremely fast",
|
||||
"format": "api",
|
||||
"inference_params": {
|
||||
"max_tokens": 4096,
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.95,
|
||||
"stream": true,
|
||||
"stop": [],
|
||||
"frequency_penalty": 0,
|
||||
"presence_penalty": 0
|
||||
},
|
||||
"engine": "openai"
|
||||
},
|
||||
{
|
||||
"model": "gpt-4o",
|
||||
"object": "model",
|
||||
"name": "OpenAI GPT 4o",
|
||||
"version": "1.1",
|
||||
"description": "OpenAI GPT 4o is a new flagship model with fast speed and high quality",
|
||||
"format": "api",
|
||||
"inference_params": {
|
||||
"max_tokens": 4096,
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.95,
|
||||
"stream": true,
|
||||
"stop": [],
|
||||
"frequency_penalty": 0,
|
||||
"presence_penalty": 0
|
||||
},
|
||||
"engine": "openai"
|
||||
},
|
||||
{
|
||||
"model": "gpt-4o-mini",
|
||||
"object": "model",
|
||||
"name": "OpenAI GPT 4o-mini",
|
||||
"version": "1.1",
|
||||
"description": "GPT-4o mini (“o” for “omni”) is a fast, affordable small model for focused tasks.",
|
||||
"format": "api",
|
||||
"inference_params": {
|
||||
"max_tokens": 16384,
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.95,
|
||||
"stream": true,
|
||||
"stop": [],
|
||||
"frequency_penalty": 0,
|
||||
"presence_penalty": 0
|
||||
},
|
||||
"engine": "openai"
|
||||
},
|
||||
{
|
||||
"model": "o1",
|
||||
"object": "model",
|
||||
"name": "OpenAI o1",
|
||||
"version": "1.0",
|
||||
"description": "OpenAI o1 is a new model with complex reasoning",
|
||||
"format": "api",
|
||||
"inference_params": {
|
||||
"max_tokens": 100000
|
||||
},
|
||||
"engine": "openai"
|
||||
},
|
||||
{
|
||||
"model": "o1-preview",
|
||||
"object": "model",
|
||||
"name": "OpenAI o1-preview",
|
||||
"version": "1.0",
|
||||
"description": "OpenAI o1-preview is a new model with complex reasoning",
|
||||
"format": "api",
|
||||
"inference_params": {
|
||||
"max_tokens": 32768,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "openai"
|
||||
},
|
||||
{
|
||||
"model": "o1-mini",
|
||||
"object": "model",
|
||||
"name": "OpenAI o1-mini",
|
||||
"version": "1.0",
|
||||
"description": "OpenAI o1-mini is a lightweight reasoning model",
|
||||
"format": "api",
|
||||
"inference_params": {
|
||||
"max_tokens": 65536,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "openai"
|
||||
},
|
||||
{
|
||||
"model": "o3-mini",
|
||||
"object": "model",
|
||||
"name": "OpenAI o3-mini",
|
||||
"version": "1.0",
|
||||
"description": "OpenAI most recent reasoning model, providing high intelligence at the same cost and latency targets of o1-mini.",
|
||||
"format": "api",
|
||||
"inference_params": {
|
||||
"max_tokens": 100000,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "openai"
|
||||
}
|
||||
]
|
||||
@ -1,92 +0,0 @@
|
||||
[
|
||||
{
|
||||
"model": "deepseek/deepseek-r1:free",
|
||||
"object": "model",
|
||||
"name": "DeepSeek: R1",
|
||||
"version": "1.0",
|
||||
"description": "OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.",
|
||||
"inference_params": {
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.95,
|
||||
"frequency_penalty": 0,
|
||||
"presence_penalty": 0,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "openrouter"
|
||||
},
|
||||
{
|
||||
"model": "deepseek/deepseek-r1-distill-llama-70b:free",
|
||||
"object": "model",
|
||||
"name": "DeepSeek: R1 Distill Llama 70B",
|
||||
"version": "1.0",
|
||||
"description": " OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.",
|
||||
"inference_params": {
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.95,
|
||||
"frequency_penalty": 0,
|
||||
"presence_penalty": 0,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "openrouter"
|
||||
},
|
||||
{
|
||||
"model": "deepseek/deepseek-r1-distill-llama-70b:free",
|
||||
"object": "model",
|
||||
"name": "DeepSeek: R1 Distill Llama 70B",
|
||||
"version": "1.0",
|
||||
"description": "OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.",
|
||||
"inference_params": {
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.95,
|
||||
"frequency_penalty": 0,
|
||||
"presence_penalty": 0,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "openrouter"
|
||||
},
|
||||
{
|
||||
"model": "meta-llama/llama-3.1-405b-instruct:free",
|
||||
"object": "model",
|
||||
"name": "Meta: Llama 3.1 405B Instruct",
|
||||
"version": "1.0",
|
||||
"description": "OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.",
|
||||
"inference_params": {
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.95,
|
||||
"frequency_penalty": 0,
|
||||
"presence_penalty": 0,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "openrouter"
|
||||
},
|
||||
{
|
||||
"model": "qwen/qwen-vl-plus:free",
|
||||
"object": "model",
|
||||
"name": "Qwen: Qwen VL Plus",
|
||||
"version": "1.0",
|
||||
"description": "OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.",
|
||||
"inference_params": {
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.95,
|
||||
"frequency_penalty": 0,
|
||||
"presence_penalty": 0,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "openrouter"
|
||||
},
|
||||
{
|
||||
"model": "qwen/qwen2.5-vl-72b-instruct:free",
|
||||
"object": "model",
|
||||
"name": "Qwen: Qwen2.5 VL 72B Instruct",
|
||||
"version": "1.0",
|
||||
"description": "OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.",
|
||||
"inference_params": {
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.95,
|
||||
"frequency_penalty": 0,
|
||||
"presence_penalty": 0,
|
||||
"stream": true
|
||||
},
|
||||
"engine": "openrouter"
|
||||
}
|
||||
]
|
||||
@ -1,47 +0,0 @@
|
||||
{
|
||||
"name": "@janhq/engine-management-extension",
|
||||
"productName": "Engine Management",
|
||||
"version": "1.0.3",
|
||||
"description": "Manages AI engines and their configurations.",
|
||||
"main": "dist/index.js",
|
||||
"node": "dist/node/index.cjs.js",
|
||||
"author": "Jan <service@jan.ai>",
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
"test": "vitest run",
|
||||
"build": "rolldown -c rolldown.config.mjs",
|
||||
"codesign:darwin": "../../.github/scripts/auto-sign.sh",
|
||||
"codesign:win32:linux": "echo 'No codesigning required'",
|
||||
"codesign": "run-script-os",
|
||||
"build:publish": "rimraf *.tgz --glob || true && yarn build && yarn codesign && npm pack && cpx *.tgz ../../pre-install"
|
||||
},
|
||||
"exports": {
|
||||
".": "./dist/index.js",
|
||||
"./main": "./dist/module.js"
|
||||
},
|
||||
"devDependencies": {
|
||||
"cpx": "^1.5.0",
|
||||
"rimraf": "^3.0.2",
|
||||
"rolldown": "^1.0.0-beta.1",
|
||||
"run-script-os": "^1.1.6",
|
||||
"ts-loader": "^9.5.0",
|
||||
"typescript": "^5.3.3",
|
||||
"vitest": "^3.0.6"
|
||||
},
|
||||
"dependencies": {
|
||||
"@janhq/core": "../../core/package.tgz",
|
||||
"ky": "^1.7.2",
|
||||
"p-queue": "^8.0.1"
|
||||
},
|
||||
"bundledDependencies": [
|
||||
"@janhq/core"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
},
|
||||
"files": [
|
||||
"dist/*",
|
||||
"package.json",
|
||||
"README.md"
|
||||
]
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
{
|
||||
"id": "anthropic",
|
||||
"type": "remote",
|
||||
"engine": "anthropic",
|
||||
"url": "https://console.anthropic.com/settings/keys",
|
||||
"api_key": "",
|
||||
"metadata": {
|
||||
"get_models_url": "https://api.anthropic.com/v1/models",
|
||||
"header_template": "x-api-key: {{api_key}} anthropic-version: 2023-06-01",
|
||||
"transform_req": {
|
||||
"chat_completions": {
|
||||
"url": "https://api.anthropic.com/v1/messages",
|
||||
"template": "{ {% for key, value in input_request %} {% if key == \"messages\" %} {% if input_request.messages.0.role == \"system\" %} \"system\": {{ tojson(input_request.messages.0.content) }}, \"messages\": [{% for message in input_request.messages %} {% if not loop.is_first %} {\"role\": {{ tojson(message.role) }}, \"content\": {% if not message.content or message.content == \"\" %} \".\" {% else %} {{ tojson(message.content) }} {% endif %} } {% if not loop.is_last %},{% endif %} {% endif %} {% endfor %}] {% else %} \"messages\": [{% for message in input_request.messages %} {\"role\": {{ tojson(message.role) }}, \"content\": {% if not message.content or message.content == \"\" %} \".\" {% else %} {{ tojson(message.content) }} {% endif %} } {% if not loop.is_last %},{% endif %} {% endfor %}] {% endif %} {% if not loop.is_last %},{% endif %} {% else if key == \"system\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"metadata\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %}\"{{ key }}\": {{ tojson(value) }} {% if not loop.is_last %},{% endif %} {% endif %} {% endfor %} }"
|
||||
}
|
||||
},
|
||||
"transform_resp": {
|
||||
"chat_completions": {
|
||||
"template": "{% if input_request.stream %} {\"object\": \"chat.completion.chunk\", \"model\": \"{{ input_request.model }}\", \"choices\": [{\"index\": 0, \"delta\": { {% if input_request.type == \"message_start\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"ping\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"content_block_delta\" %} \"role\": \"assistant\", \"content\": {{ tojson(input_request.delta.text) }} {% else if input_request.type == \"content_block_stop\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"content_block_stop\" %} \"role\": \"assistant\", \"content\": null {% endif %} }, {% if input_request.type == \"content_block_stop\" %} \"finish_reason\": \"stop\" {% else %} \"finish_reason\": null {% endif %} }]} {% else %} {{tojson(input_request)}} {% endif %}"
|
||||
}
|
||||
},
|
||||
"explore_models_url": "https://docs.anthropic.com/en/docs/about-claude/models"
|
||||
}
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
{
|
||||
"id": "cohere",
|
||||
"type": "remote",
|
||||
"engine": "cohere",
|
||||
"url": "https://dashboard.cohere.com/api-keys",
|
||||
"api_key": "",
|
||||
"metadata": {
|
||||
"get_models_url": "https://api.cohere.ai/v1/models",
|
||||
"header_template": "Authorization: Bearer {{api_key}}",
|
||||
"transform_req": {
|
||||
"chat_completions": {
|
||||
"url": "https://api.cohere.ai/v1/chat",
|
||||
"template": "{ {% for key, value in input_request %} {% if key == \"messages\" %} {% if input_request.messages.0.role == \"system\" %} \"preamble\": {{ tojson(input_request.messages.0.content) }}, {% if length(input_request.messages) > 2 %} \"chatHistory\": [{% for message in input_request.messages %} {% if not loop.is_first and not loop.is_last %} {\"role\": {% if message.role == \"user\" %} \"USER\" {% else %} \"CHATBOT\" {% endif %}, \"content\": {{ tojson(message.content) }} } {% if loop.index < length(input_request.messages) - 2 %},{% endif %} {% endif %} {% endfor %}], {% endif %} \"message\": {{ tojson(last(input_request.messages).content) }} {% else %} {% if length(input_request.messages) > 2 %} \"chatHistory\": [{% for message in input_request.messages %} {% if not loop.is_last %} { \"role\": {% if message.role == \"user\" %} \"USER\" {% else %} \"CHATBOT\" {% endif %}, \"content\": {{ tojson(message.content) }} } {% if loop.index < length(input_request.messages) - 2 %},{% endif %} {% endif %} {% endfor %}],{% endif %}\"message\": {{ tojson(last(input_request.messages).content) }} {% endif %}{% if not loop.is_last %},{% endif %} {% else if key == \"system\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} \"{{ key }}\": {{ tojson(value) }} {% if not loop.is_last %},{% endif %} {% endif %} {% endfor %} }"
|
||||
}
|
||||
},
|
||||
"transform_resp": {
|
||||
"chat_completions": {
|
||||
"template": "{% if input_request.stream %} {\"object\": \"chat.completion.chunk\", \"model\": \"{{ input_request.model }}\", \"choices\": [{\"index\": 0, \"delta\": { {% if input_request.event_type == \"text-generation\" %} \"role\": \"assistant\", \"content\": {{ tojson(input_request.text) }} {% else %} \"role\": \"assistant\", \"content\": null {% endif %} }, {% if input_request.event_type == \"stream-end\" %} \"finish_reason\": \"{{ input_request.finish_reason }}\" {% else %} \"finish_reason\": null {% endif %} }]} {% else %} {\"id\": \"{{ input_request.generation_id }}\", \"created\": null, \"object\": \"chat.completion\", \"model\": {% if input_request.model %} \"{{ input_request.model }}\" {% else %} \"command-r-plus-08-2024\" {% endif %}, \"choices\": [{ \"index\": 0, \"message\": { \"role\": \"assistant\", \"content\": {% if not input_request.text %} null {% else %} {{ tojson(input_request.text) }} {% endif %}, \"refusal\": null }, \"logprobs\": null, \"finish_reason\": \"{{ input_request.finish_reason }}\" } ], \"usage\": { \"prompt_tokens\": {{ input_request.meta.tokens.input_tokens }}, \"completion_tokens\": {{ input_request.meta.tokens.output_tokens }},\"total_tokens\": {{ input_request.meta.tokens.input_tokens + input_request.meta.tokens.output_tokens }}, \"prompt_tokens_details\": { \"cached_tokens\": 0 },\"completion_tokens_details\": { \"reasoning_tokens\": 0, \"accepted_prediction_tokens\": 0, \"rejected_prediction_tokens\": 0 } }, \"system_fingerprint\": \"fp_6b68a8204b\"} {% endif %}"
|
||||
}
|
||||
},
|
||||
"explore_models_url": "https://docs.cohere.com/v2/docs/models"
|
||||
}
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
{
|
||||
"id": "deepseek",
|
||||
"type": "remote",
|
||||
"engine": "deepseek",
|
||||
"url": "https://platform.deepseek.com/api_keys",
|
||||
"api_key": "",
|
||||
"metadata": {
|
||||
"get_models_url": "https://api.deepseek.com/models",
|
||||
"header_template": "Authorization: Bearer {{api_key}}",
|
||||
"transform_req": {
|
||||
"chat_completions": {
|
||||
"url": "https://api.deepseek.com/chat/completions",
|
||||
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"model\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
|
||||
}
|
||||
},
|
||||
"transform_resp": {
|
||||
"chat_completions": {
|
||||
"template": "{{tojson(input_request)}}"
|
||||
}
|
||||
},
|
||||
"explore_models_url": "https://api-docs.deepseek.com/quick_start/pricing"
|
||||
}
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
{
|
||||
"id": "google_gemini",
|
||||
"type": "remote",
|
||||
"engine": "google_gemini",
|
||||
"url": "https://aistudio.google.com/apikey",
|
||||
"api_key": "",
|
||||
"metadata": {
|
||||
"get_models_url": "https://generativelanguage.googleapis.com/openai/v1beta/models",
|
||||
"header_template": "Authorization: Bearer {{api_key}}",
|
||||
"transform_req": {
|
||||
"chat_completions": {
|
||||
"url": "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions",
|
||||
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"model\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
|
||||
}
|
||||
},
|
||||
"transform_resp": {
|
||||
"chat_completions": {
|
||||
"template": "{{tojson(input_request)}}"
|
||||
}
|
||||
},
|
||||
"explore_models_url": "https://ai.google.dev/gemini-api/docs/models/gemini"
|
||||
}
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
{
|
||||
"id": "groq",
|
||||
"type": "remote",
|
||||
"engine": "groq",
|
||||
"url": "https://console.groq.com/keys",
|
||||
"api_key": "",
|
||||
"metadata": {
|
||||
"get_models_url": "https://api.groq.com/openai/v1/models",
|
||||
"header_template": "Authorization: Bearer {{api_key}}",
|
||||
"transform_req": {
|
||||
"chat_completions": {
|
||||
"url": "https://api.groq.com/openai/v1/chat/completions",
|
||||
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
|
||||
}
|
||||
},
|
||||
"transform_resp": {
|
||||
"chat_completions": {
|
||||
"template": "{{tojson(input_request)}}"
|
||||
}
|
||||
},
|
||||
"explore_models_url": "https://console.groq.com/docs/models"
|
||||
}
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
{
|
||||
"id": "martian",
|
||||
"type": "remote",
|
||||
"engine": "martian",
|
||||
"url": "https://withmartian.com/dashboard",
|
||||
"api_key": "",
|
||||
"metadata": {
|
||||
"get_models_url": "https://withmartian.com/api/openai/v1/models",
|
||||
"header_template": "Authorization: Bearer {{api_key}}",
|
||||
"transform_req": {
|
||||
"chat_completions": {
|
||||
"url": "https://withmartian.com/api/openai/v1/chat/completions",
|
||||
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
|
||||
}
|
||||
},
|
||||
"transform_resp": {
|
||||
"chat_completions": {
|
||||
"template": "{{tojson(input_request)}}"
|
||||
}
|
||||
},
|
||||
"explore_models_url": "https://withmartian.github.io/llm-adapters/"
|
||||
}
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
{
|
||||
"id": "mistral",
|
||||
"type": "remote",
|
||||
"engine": "mistral",
|
||||
"url": "https://console.mistral.ai/api-keys/",
|
||||
"api_key": "",
|
||||
"metadata": {
|
||||
"get_models_url": "https://api.mistral.ai/v1/models",
|
||||
"header_template": "Authorization: Bearer {{api_key}}",
|
||||
"transform_req": {
|
||||
"chat_completions": {
|
||||
"url": "https://api.mistral.ai/v1/chat/completions",
|
||||
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
|
||||
}
|
||||
},
|
||||
"transform_resp": {
|
||||
"chat_completions": {
|
||||
"template": "{{tojson(input_request)}}"
|
||||
}
|
||||
},
|
||||
"explore_models_url": "https://docs.mistral.ai/getting-started/models/models_overview/"
|
||||
}
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
{
|
||||
"id": "nvidia",
|
||||
"type": "remote",
|
||||
"engine": "nvidia",
|
||||
"url": "https://org.ngc.nvidia.com/setup/personal-keys",
|
||||
"api_key": "",
|
||||
"metadata": {
|
||||
"get_models_url": "https://integrate.api.nvidia.com/v1/models",
|
||||
"header_template": "Authorization: Bearer {{api_key}}",
|
||||
"transform_req": {
|
||||
"chat_completions": {
|
||||
"url": "https://integrate.api.nvidia.com/v1/chat/completions",
|
||||
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
|
||||
}
|
||||
},
|
||||
"transform_resp": {
|
||||
"chat_completions": {
|
||||
"template": "{{tojson(input_request)}}"
|
||||
}
|
||||
},
|
||||
"explore_models_url": "https://build.nvidia.com/models"
|
||||
}
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
{
|
||||
"id": "openai",
|
||||
"type": "remote",
|
||||
"engine": "openai",
|
||||
"url": "https://platform.openai.com/account/api-keys",
|
||||
"api_key": "",
|
||||
"metadata": {
|
||||
"get_models_url": "https://api.openai.com/v1/models",
|
||||
"header_template": "Authorization: Bearer {{api_key}}",
|
||||
"transform_req": {
|
||||
"chat_completions": {
|
||||
"url": "https://api.openai.com/v1/chat/completions",
|
||||
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"messages\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" or key == \"max_tokens\" or key == \"stop\" %} {% if not first %}, {% endif %} {% if key == \"messages\" and (input_request.model == \"o1\" or input_request.model == \"o1-preview\" or input_request.model == \"o1-mini\") and input_request.messages.0.role == \"system\" %} \"messages\": [ {% for message in input_request.messages %} {% if not loop.is_first %} { \"role\": \"{{ message.role }}\", \"content\": \"{{ message.content }}\" } {% if not loop.is_last %}, {% endif %} {% endif %} {% endfor %} ] {% else if key == \"stop\" and (input_request.model == \"o1\" or input_request.model == \"o1-preview\" or input_request.model == \"o1-mini\" or input_request.model == \"o3\" or input_request.model == \"o3-mini\") %} {% set first = false %} {% else if key == \"max_tokens\" and (input_request.model == \"o1\" or input_request.model == \"o1-preview\" or input_request.model == \"o1-mini\" or input_request.model == \"o3\" or input_request.model == \"o3-mini\") %} \"max_completion_tokens\": {{ tojson(value) }} {% set first = false %} {% else %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endif %} {% endfor %} }"
|
||||
}
|
||||
},
|
||||
"transform_resp": {
|
||||
"chat_completions": {
|
||||
"template": "{{tojson(input_request)}}"
|
||||
}
|
||||
},
|
||||
"explore_models_url": "https://platform.openai.com/docs/models"
|
||||
}
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
{
|
||||
"id": "openrouter",
|
||||
"type": "remote",
|
||||
"engine": "openrouter",
|
||||
"url": "https://openrouter.ai/keys",
|
||||
"api_key": "",
|
||||
"metadata": {
|
||||
"get_models_url": "https://openrouter.ai/api/v1/models",
|
||||
"header_template": "Authorization: Bearer {{api_key}}",
|
||||
"transform_req": {
|
||||
"chat_completions": {
|
||||
"url": "https://openrouter.ai/api/v1/chat/completions",
|
||||
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"model\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
|
||||
}
|
||||
},
|
||||
"transform_resp": {
|
||||
"chat_completions": {
|
||||
"template": "{{tojson(input_request)}}"
|
||||
}
|
||||
},
|
||||
"explore_models_url": "https://openrouter.ai/models"
|
||||
}
|
||||
}
|
||||
@ -1,44 +0,0 @@
|
||||
import { defineConfig } from 'rolldown'
|
||||
import { engines, models } from './engines.mjs'
|
||||
import pkgJson from './package.json' with { type: 'json' }
|
||||
|
||||
export default defineConfig([
|
||||
{
|
||||
input: 'src/index.ts',
|
||||
output: {
|
||||
format: 'esm',
|
||||
file: 'dist/index.js',
|
||||
},
|
||||
define: {
|
||||
NODE: JSON.stringify(`${pkgJson.name}/${pkgJson.node}`),
|
||||
API_URL: JSON.stringify(
|
||||
`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? '39291'}`
|
||||
),
|
||||
PLATFORM: JSON.stringify(process.platform),
|
||||
CORTEX_ENGINE_VERSION: JSON.stringify('b5509'),
|
||||
DEFAULT_REMOTE_ENGINES: JSON.stringify(engines),
|
||||
DEFAULT_REMOTE_MODELS: JSON.stringify(models),
|
||||
DEFAULT_REQUEST_PAYLOAD_TRANSFORM: JSON.stringify(
|
||||
`{ {% set first = true %} {% for key, value in input_request %} {% if key == "messages" or key == "model" or key == "temperature" or key == "store" or key == "max_tokens" or key == "stream" or key == "presence_penalty" or key == "metadata" or key == "frequency_penalty" or key == "tools" or key == "tool_choice" or key == "logprobs" or key == "top_logprobs" or key == "logit_bias" or key == "n" or key == "modalities" or key == "prediction" or key == "response_format" or key == "service_tier" or key == "seed" or key == "stop" or key == "stream_options" or key == "top_p" or key == "parallel_tool_calls" or key == "user" %} {% if not first %},{% endif %} "{{ key }}": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }`
|
||||
),
|
||||
DEFAULT_RESPONSE_BODY_TRANSFORM: JSON.stringify(
|
||||
'{{tojson(input_request)}}'
|
||||
),
|
||||
DEFAULT_REQUEST_HEADERS_TRANSFORM: JSON.stringify(
|
||||
'Authorization: Bearer {{api_key}}'
|
||||
),
|
||||
VERSION: JSON.stringify(pkgJson.version ?? '0.0.0'),
|
||||
},
|
||||
},
|
||||
{
|
||||
input: 'src/node/index.ts',
|
||||
external: ['@janhq/core/node'],
|
||||
output: {
|
||||
format: 'cjs',
|
||||
file: 'dist/node/index.cjs.js',
|
||||
},
|
||||
define: {
|
||||
CORTEX_ENGINE_VERSION: JSON.stringify('b5509'),
|
||||
},
|
||||
},
|
||||
])
|
||||
@ -1,23 +0,0 @@
|
||||
declare const API_URL: string
|
||||
declare const CORTEX_ENGINE_VERSION: string
|
||||
declare const PLATFORM: string
|
||||
declare const NODE: string
|
||||
declare const DEFAULT_REQUEST_PAYLOAD_TRANSFORM: string
|
||||
declare const DEFAULT_RESPONSE_BODY_TRANSFORM: string
|
||||
declare const DEFAULT_REQUEST_HEADERS_TRANSFORM: string
|
||||
declare const VERSION: string
|
||||
|
||||
declare const DEFAULT_REMOTE_ENGINES: ({
|
||||
id: string
|
||||
engine: string
|
||||
} & EngineConfig)[]
|
||||
declare const DEFAULT_REMOTE_MODELS: Model[]
|
||||
|
||||
interface Core {
|
||||
api: APIFunctions
|
||||
events: EventEmitter
|
||||
}
|
||||
interface Window {
|
||||
core?: Core | undefined
|
||||
electronAPI?: any | undefined
|
||||
}
|
||||
@ -1,199 +0,0 @@
|
||||
import { describe, beforeEach, it, expect, vi } from 'vitest'
|
||||
import JanEngineManagementExtension from './index'
|
||||
import { InferenceEngine } from '@janhq/core'
|
||||
|
||||
describe('API methods', () => {
|
||||
let extension: JanEngineManagementExtension
|
||||
|
||||
beforeEach(() => {
|
||||
// @ts-ignore
|
||||
extension = new JanEngineManagementExtension()
|
||||
vi.resetAllMocks()
|
||||
})
|
||||
|
||||
describe('getReleasedEnginesByVersion', () => {
|
||||
it('should return engines filtered by platform if provided', async () => {
|
||||
const mockEngines = [
|
||||
{
|
||||
name: 'windows-amd64-avx2',
|
||||
version: '1.0.0',
|
||||
},
|
||||
{
|
||||
name: 'linux-amd64-avx2',
|
||||
version: '1.0.0',
|
||||
},
|
||||
]
|
||||
|
||||
vi.mock('ky', () => ({
|
||||
default: {
|
||||
get: () => ({
|
||||
json: () => Promise.resolve(mockEngines),
|
||||
}),
|
||||
},
|
||||
}))
|
||||
|
||||
const mock = vi.spyOn(extension, 'getReleasedEnginesByVersion')
|
||||
mock.mockImplementation(async (name, version, platform) => {
|
||||
const result = await Promise.resolve(mockEngines)
|
||||
return platform ? result.filter(r => r.name.includes(platform)) : result
|
||||
})
|
||||
|
||||
const result = await extension.getReleasedEnginesByVersion(
|
||||
InferenceEngine.cortex_llamacpp,
|
||||
'1.0.0',
|
||||
'windows'
|
||||
)
|
||||
|
||||
expect(result).toHaveLength(1)
|
||||
expect(result[0].name).toBe('windows-amd64-avx2')
|
||||
})
|
||||
|
||||
it('should return all engines if platform is not provided', async () => {
|
||||
const mockEngines = [
|
||||
{
|
||||
name: 'windows-amd64-avx2',
|
||||
version: '1.0.0',
|
||||
},
|
||||
{
|
||||
name: 'linux-amd64-avx2',
|
||||
version: '1.0.0',
|
||||
},
|
||||
]
|
||||
|
||||
vi.mock('ky', () => ({
|
||||
default: {
|
||||
get: () => ({
|
||||
json: () => Promise.resolve(mockEngines),
|
||||
}),
|
||||
},
|
||||
}))
|
||||
|
||||
const mock = vi.spyOn(extension, 'getReleasedEnginesByVersion')
|
||||
mock.mockImplementation(async (name, version, platform) => {
|
||||
const result = await Promise.resolve(mockEngines)
|
||||
return platform ? result.filter(r => r.name.includes(platform)) : result
|
||||
})
|
||||
|
||||
const result = await extension.getReleasedEnginesByVersion(
|
||||
InferenceEngine.cortex_llamacpp,
|
||||
'1.0.0'
|
||||
)
|
||||
|
||||
expect(result).toHaveLength(2)
|
||||
})
|
||||
})
|
||||
|
||||
describe('getLatestReleasedEngine', () => {
|
||||
it('should return engines filtered by platform if provided', async () => {
|
||||
const mockEngines = [
|
||||
{
|
||||
name: 'windows-amd64-avx2',
|
||||
version: '1.0.0',
|
||||
},
|
||||
{
|
||||
name: 'linux-amd64-avx2',
|
||||
version: '1.0.0',
|
||||
},
|
||||
]
|
||||
|
||||
vi.mock('ky', () => ({
|
||||
default: {
|
||||
get: () => ({
|
||||
json: () => Promise.resolve(mockEngines),
|
||||
}),
|
||||
},
|
||||
}))
|
||||
|
||||
const mock = vi.spyOn(extension, 'getLatestReleasedEngine')
|
||||
mock.mockImplementation(async (name, platform) => {
|
||||
const result = await Promise.resolve(mockEngines)
|
||||
return platform ? result.filter(r => r.name.includes(platform)) : result
|
||||
})
|
||||
|
||||
const result = await extension.getLatestReleasedEngine(
|
||||
InferenceEngine.cortex_llamacpp,
|
||||
'linux'
|
||||
)
|
||||
|
||||
expect(result).toHaveLength(1)
|
||||
expect(result[0].name).toBe('linux-amd64-avx2')
|
||||
})
|
||||
})
|
||||
|
||||
describe('installEngine', () => {
|
||||
it('should send install request with correct parameters', async () => {
|
||||
const mockEngineConfig = {
|
||||
variant: 'windows-amd64-avx2',
|
||||
version: '1.0.0',
|
||||
}
|
||||
|
||||
vi.mock('ky', () => ({
|
||||
default: {
|
||||
post: (url, options) => {
|
||||
expect(url).toBe(`${API_URL}/v1/engines/${InferenceEngine.cortex_llamacpp}/install`)
|
||||
expect(options.json).toEqual(mockEngineConfig)
|
||||
return Promise.resolve({ messages: 'OK' })
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
const result = await extension.installEngine(
|
||||
InferenceEngine.cortex_llamacpp,
|
||||
mockEngineConfig
|
||||
)
|
||||
|
||||
expect(result).toEqual({ messages: 'OK' })
|
||||
})
|
||||
})
|
||||
|
||||
describe('uninstallEngine', () => {
|
||||
it('should send uninstall request with correct parameters', async () => {
|
||||
const mockEngineConfig = {
|
||||
variant: 'windows-amd64-avx2',
|
||||
version: '1.0.0',
|
||||
}
|
||||
|
||||
vi.mock('ky', () => ({
|
||||
default: {
|
||||
delete: (url, options) => {
|
||||
expect(url).toBe(`${API_URL}/v1/engines/${InferenceEngine.cortex_llamacpp}/install`)
|
||||
expect(options.json).toEqual(mockEngineConfig)
|
||||
return Promise.resolve({ messages: 'OK' })
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
const result = await extension.uninstallEngine(
|
||||
InferenceEngine.cortex_llamacpp,
|
||||
mockEngineConfig
|
||||
)
|
||||
|
||||
expect(result).toEqual({ messages: 'OK' })
|
||||
})
|
||||
})
|
||||
|
||||
describe('addRemoteModel', () => {
|
||||
it('should send add model request with correct parameters', async () => {
|
||||
const mockModel = {
|
||||
id: 'gpt-4',
|
||||
name: 'GPT-4',
|
||||
engine: InferenceEngine.openai,
|
||||
}
|
||||
|
||||
vi.mock('ky', () => ({
|
||||
default: {
|
||||
post: (url, options) => {
|
||||
expect(url).toBe(`${API_URL}/v1/models/add`)
|
||||
expect(options.json).toHaveProperty('id', 'gpt-4')
|
||||
expect(options.json).toHaveProperty('engine', InferenceEngine.openai)
|
||||
expect(options.json).toHaveProperty('inference_params')
|
||||
return Promise.resolve()
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
await extension.addRemoteModel(mockModel)
|
||||
// Success is implied by no thrown exceptions
|
||||
})
|
||||
})
|
||||
})
|
||||
@ -1,19 +0,0 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { EngineError } from './error'
|
||||
|
||||
describe('EngineError', () => {
|
||||
it('should create an error with the correct message', () => {
|
||||
const errorMessage = 'Test error message'
|
||||
const error = new EngineError(errorMessage)
|
||||
|
||||
expect(error).toBeInstanceOf(Error)
|
||||
expect(error.message).toBe(errorMessage)
|
||||
expect(error.name).toBe('EngineError')
|
||||
})
|
||||
|
||||
it('should create an error with default message if none provided', () => {
|
||||
const error = new EngineError()
|
||||
|
||||
expect(error.message).toBe('Engine error occurred')
|
||||
})
|
||||
})
|
||||
@ -1,10 +0,0 @@
|
||||
/**
|
||||
* Custom Engine Error
|
||||
*/
|
||||
export class EngineError extends Error {
|
||||
message: string
|
||||
constructor(message: string) {
|
||||
super()
|
||||
this.message = message
|
||||
}
|
||||
}
|
||||
@ -1,449 +0,0 @@
|
||||
import { describe, beforeEach, it, expect, vi } from 'vitest'
|
||||
import JanEngineManagementExtension from './index'
|
||||
import { Engines, InferenceEngine } from '@janhq/core'
|
||||
import { EngineError } from './error'
|
||||
import { HTTPError } from 'ky'
|
||||
|
||||
vi.stubGlobal('API_URL', 'http://localhost:3000')
|
||||
|
||||
const mockEngines: Engines = [
|
||||
{
|
||||
name: 'variant1',
|
||||
version: '1.0.0',
|
||||
type: 'local',
|
||||
engine: InferenceEngine.cortex_llamacpp,
|
||||
},
|
||||
]
|
||||
|
||||
const mockRemoteEngines: Engines = [
|
||||
{
|
||||
name: 'openai',
|
||||
version: '1.0.0',
|
||||
type: 'remote',
|
||||
engine: InferenceEngine.openai,
|
||||
},
|
||||
]
|
||||
|
||||
const mockRemoteModels = {
|
||||
data: [
|
||||
{
|
||||
id: 'gpt-4',
|
||||
name: 'GPT-4',
|
||||
engine: InferenceEngine.openai,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
vi.stubGlobal('DEFAULT_REMOTE_ENGINES', mockEngines)
|
||||
vi.stubGlobal('DEFAULT_REMOTE_MODELS', mockRemoteModels.data)
|
||||
|
||||
describe('migrate engine settings', () => {
|
||||
let extension: JanEngineManagementExtension
|
||||
|
||||
beforeEach(() => {
|
||||
// @ts-ignore
|
||||
extension = new JanEngineManagementExtension()
|
||||
vi.resetAllMocks()
|
||||
})
|
||||
|
||||
it('engines should be migrated', async () => {
|
||||
vi.stubGlobal('VERSION', '2.0.0')
|
||||
|
||||
vi.spyOn(extension, 'getEngines').mockResolvedValue([])
|
||||
const mockUpdateEngines = vi
|
||||
.spyOn(extension, 'updateEngine')
|
||||
.mockReturnThis()
|
||||
|
||||
mockUpdateEngines.mockResolvedValue({
|
||||
messages: 'OK',
|
||||
})
|
||||
|
||||
await extension.migrate()
|
||||
|
||||
// Assert that the returned value is equal to the mockEngines object
|
||||
expect(mockUpdateEngines).toBeCalled()
|
||||
})
|
||||
|
||||
it('should not migrate when extension version is not updated', async () => {
|
||||
vi.stubGlobal('VERSION', '0.0.0')
|
||||
vi.spyOn(extension, 'getEngines').mockResolvedValue([])
|
||||
const mockUpdateEngines = vi
|
||||
.spyOn(extension, 'updateEngine')
|
||||
.mockReturnThis()
|
||||
|
||||
mockUpdateEngines.mockResolvedValue({
|
||||
messages: 'OK',
|
||||
})
|
||||
|
||||
await extension.migrate()
|
||||
|
||||
// Assert that the returned value is equal to the mockEngines object
|
||||
expect(mockUpdateEngines).not.toBeCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe('getEngines', () => {
|
||||
let extension: JanEngineManagementExtension
|
||||
|
||||
beforeEach(() => {
|
||||
// @ts-ignore
|
||||
extension = new JanEngineManagementExtension()
|
||||
vi.resetAllMocks()
|
||||
})
|
||||
|
||||
it('should return a list of engines', async () => {
|
||||
const mockKyGet = vi.spyOn(extension, 'getEngines')
|
||||
mockKyGet.mockResolvedValue(mockEngines)
|
||||
|
||||
const engines = await extension.getEngines()
|
||||
|
||||
expect(engines).toEqual(mockEngines)
|
||||
})
|
||||
})
|
||||
|
||||
describe('getRemoteModels', () => {
|
||||
let extension: JanEngineManagementExtension
|
||||
|
||||
beforeEach(() => {
|
||||
// @ts-ignore
|
||||
extension = new JanEngineManagementExtension()
|
||||
vi.resetAllMocks()
|
||||
})
|
||||
|
||||
it('should return a list of remote models', async () => {
|
||||
vi.mock('ky', () => ({
|
||||
default: {
|
||||
get: () => ({
|
||||
json: () => Promise.resolve(mockRemoteModels),
|
||||
}),
|
||||
},
|
||||
}))
|
||||
|
||||
const models = await extension.getRemoteModels('openai')
|
||||
expect(models).toEqual(mockRemoteModels)
|
||||
})
|
||||
|
||||
it('should return empty data array when request fails', async () => {
|
||||
vi.mock('ky', () => ({
|
||||
default: {
|
||||
get: () => ({
|
||||
json: () => Promise.reject(new Error('Failed to fetch')),
|
||||
}),
|
||||
},
|
||||
}))
|
||||
|
||||
const models = await extension.getRemoteModels('openai')
|
||||
expect(models).toEqual({ data: [] })
|
||||
})
|
||||
})
|
||||
|
||||
describe('getInstalledEngines', () => {
|
||||
let extension: JanEngineManagementExtension
|
||||
|
||||
beforeEach(() => {
|
||||
// @ts-ignore
|
||||
extension = new JanEngineManagementExtension()
|
||||
vi.resetAllMocks()
|
||||
})
|
||||
|
||||
it('should return a list of installed engines', async () => {
|
||||
const mockEngineVariants = [
|
||||
{
|
||||
name: 'windows-amd64-noavx',
|
||||
version: '1.0.0',
|
||||
},
|
||||
]
|
||||
|
||||
vi.mock('ky', () => ({
|
||||
default: {
|
||||
get: () => ({
|
||||
json: () => Promise.resolve(mockEngineVariants),
|
||||
}),
|
||||
},
|
||||
}))
|
||||
|
||||
const mock = vi.spyOn(extension, 'getInstalledEngines')
|
||||
mock.mockResolvedValue(mockEngineVariants)
|
||||
|
||||
const engines = await extension.getInstalledEngines(InferenceEngine.cortex_llamacpp)
|
||||
expect(engines).toEqual(mockEngineVariants)
|
||||
})
|
||||
})
|
||||
|
||||
describe('healthz', () => {
|
||||
let extension: JanEngineManagementExtension
|
||||
|
||||
beforeEach(() => {
|
||||
// @ts-ignore
|
||||
extension = new JanEngineManagementExtension()
|
||||
vi.resetAllMocks()
|
||||
})
|
||||
|
||||
it('should perform health check successfully', async () => {
|
||||
vi.mock('ky', () => ({
|
||||
default: {
|
||||
get: () => Promise.resolve(),
|
||||
},
|
||||
}))
|
||||
|
||||
await extension.healthz()
|
||||
expect(extension.queue.concurrency).toBe(Infinity)
|
||||
})
|
||||
})
|
||||
|
||||
describe('updateDefaultEngine', () => {
|
||||
let extension: JanEngineManagementExtension
|
||||
|
||||
beforeEach(() => {
|
||||
// @ts-ignore
|
||||
extension = new JanEngineManagementExtension()
|
||||
vi.resetAllMocks()
|
||||
})
|
||||
|
||||
it('should set default engine variant if not installed', async () => {
|
||||
vi.stubGlobal('PLATFORM', 'win32')
|
||||
vi.stubGlobal('CORTEX_ENGINE_VERSION', '1.0.0')
|
||||
|
||||
const mockGetDefaultEngineVariant = vi.spyOn(
|
||||
extension,
|
||||
'getDefaultEngineVariant'
|
||||
)
|
||||
mockGetDefaultEngineVariant.mockResolvedValue({
|
||||
variant: 'variant1',
|
||||
version: '1.0.0',
|
||||
})
|
||||
|
||||
const mockGetInstalledEngines = vi.spyOn(extension, 'getInstalledEngines')
|
||||
mockGetInstalledEngines.mockResolvedValue([])
|
||||
|
||||
const mockSetDefaultEngineVariant = vi.spyOn(
|
||||
extension,
|
||||
'setDefaultEngineVariant'
|
||||
)
|
||||
mockSetDefaultEngineVariant.mockResolvedValue({ messages: 'OK' })
|
||||
|
||||
vi.mock('@janhq/core', async (importOriginal) => {
|
||||
const actual = (await importOriginal()) as any
|
||||
return {
|
||||
...actual,
|
||||
systemInformation: vi.fn().mockResolvedValue({ gpuSetting: 'high' }),
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('./utils', async (importOriginal) => {
|
||||
const actual = (await importOriginal()) as any
|
||||
return {
|
||||
...actual,
|
||||
engineVariant: vi.fn().mockResolvedValue('windows-amd64-noavx'),
|
||||
}
|
||||
})
|
||||
|
||||
await extension.updateDefaultEngine()
|
||||
|
||||
expect(mockSetDefaultEngineVariant).toHaveBeenCalledWith('llama-cpp', {
|
||||
variant: 'windows-amd64-noavx',
|
||||
version: '1.0.0',
|
||||
})
|
||||
})
|
||||
|
||||
it('should not reset default engine variant if installed', async () => {
|
||||
vi.stubGlobal('PLATFORM', 'win32')
|
||||
vi.stubGlobal('CORTEX_ENGINE_VERSION', '1.0.0')
|
||||
|
||||
const mockGetDefaultEngineVariant = vi.spyOn(
|
||||
extension,
|
||||
'getDefaultEngineVariant'
|
||||
)
|
||||
mockGetDefaultEngineVariant.mockResolvedValue({
|
||||
variant: 'windows-amd64-noavx',
|
||||
version: '1.0.0',
|
||||
})
|
||||
|
||||
const mockGetInstalledEngines = vi.spyOn(extension, 'getInstalledEngines')
|
||||
mockGetInstalledEngines.mockResolvedValue([
|
||||
{
|
||||
name: 'windows-amd64-noavx',
|
||||
version: '1.0.0',
|
||||
type: 'local',
|
||||
engine: InferenceEngine.cortex_llamacpp,
|
||||
},
|
||||
])
|
||||
|
||||
const mockSetDefaultEngineVariant = vi.spyOn(
|
||||
extension,
|
||||
'setDefaultEngineVariant'
|
||||
)
|
||||
mockSetDefaultEngineVariant.mockResolvedValue({ messages: 'OK' })
|
||||
|
||||
vi.mock('@janhq/core', async (importOriginal) => {
|
||||
const actual = (await importOriginal()) as any
|
||||
return {
|
||||
...actual,
|
||||
systemInformation: vi.fn().mockResolvedValue({ gpuSetting: 'high' }),
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('./utils', async (importOriginal) => {
|
||||
const actual = (await importOriginal()) as any
|
||||
return {
|
||||
...actual,
|
||||
engineVariant: vi.fn().mockResolvedValue('windows-amd64-noavx'),
|
||||
}
|
||||
})
|
||||
|
||||
await extension.updateDefaultEngine()
|
||||
|
||||
expect(mockSetDefaultEngineVariant).not.toBeCalled()
|
||||
})
|
||||
|
||||
it('should handle HTTPError when getting default engine variant', async () => {
|
||||
vi.stubGlobal('PLATFORM', 'win32')
|
||||
vi.stubGlobal('CORTEX_ENGINE_VERSION', '1.0.0')
|
||||
|
||||
const httpError = new Error('HTTP Error') as HTTPError
|
||||
httpError.response = { status: 400 } as Response
|
||||
|
||||
const mockGetDefaultEngineVariant = vi.spyOn(
|
||||
extension,
|
||||
'getDefaultEngineVariant'
|
||||
)
|
||||
mockGetDefaultEngineVariant.mockRejectedValue(httpError)
|
||||
|
||||
const mockSetDefaultEngineVariant = vi.spyOn(
|
||||
extension,
|
||||
'setDefaultEngineVariant'
|
||||
)
|
||||
mockSetDefaultEngineVariant.mockResolvedValue({ messages: 'OK' })
|
||||
|
||||
vi.mock('@janhq/core', async (importOriginal) => {
|
||||
const actual = (await importOriginal()) as any
|
||||
return {
|
||||
...actual,
|
||||
systemInformation: vi.fn().mockResolvedValue({ gpuSetting: 'high' }),
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('./utils', async (importOriginal) => {
|
||||
const actual = (await importOriginal()) as any
|
||||
return {
|
||||
...actual,
|
||||
engineVariant: vi.fn().mockResolvedValue('windows-amd64-noavx'),
|
||||
}
|
||||
})
|
||||
|
||||
await extension.updateDefaultEngine()
|
||||
|
||||
expect(mockSetDefaultEngineVariant).toHaveBeenCalledWith('llama-cpp', {
|
||||
variant: 'windows-amd64-noavx',
|
||||
version: '1.0.0',
|
||||
})
|
||||
})
|
||||
|
||||
it('should handle EngineError when getting default engine variant', async () => {
|
||||
vi.stubGlobal('PLATFORM', 'win32')
|
||||
vi.stubGlobal('CORTEX_ENGINE_VERSION', '1.0.0')
|
||||
|
||||
const mockGetDefaultEngineVariant = vi.spyOn(
|
||||
extension,
|
||||
'getDefaultEngineVariant'
|
||||
)
|
||||
mockGetDefaultEngineVariant.mockRejectedValue(new EngineError('Test error'))
|
||||
|
||||
const mockSetDefaultEngineVariant = vi.spyOn(
|
||||
extension,
|
||||
'setDefaultEngineVariant'
|
||||
)
|
||||
mockSetDefaultEngineVariant.mockResolvedValue({ messages: 'OK' })
|
||||
|
||||
vi.mock('@janhq/core', async (importOriginal) => {
|
||||
const actual = (await importOriginal()) as any
|
||||
return {
|
||||
...actual,
|
||||
systemInformation: vi.fn().mockResolvedValue({ gpuSetting: 'high' }),
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('./utils', async (importOriginal) => {
|
||||
const actual = (await importOriginal()) as any
|
||||
return {
|
||||
...actual,
|
||||
engineVariant: vi.fn().mockResolvedValue('windows-amd64-noavx'),
|
||||
}
|
||||
})
|
||||
|
||||
await extension.updateDefaultEngine()
|
||||
|
||||
expect(mockSetDefaultEngineVariant).toHaveBeenCalledWith('llama-cpp', {
|
||||
variant: 'windows-amd64-noavx',
|
||||
version: '1.0.0',
|
||||
})
|
||||
})
|
||||
|
||||
it('should handle unexpected errors gracefully', async () => {
|
||||
vi.stubGlobal('PLATFORM', 'win32')
|
||||
|
||||
const mockGetDefaultEngineVariant = vi.spyOn(
|
||||
extension,
|
||||
'getDefaultEngineVariant'
|
||||
)
|
||||
mockGetDefaultEngineVariant.mockRejectedValue(new Error('Unexpected error'))
|
||||
|
||||
const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
|
||||
await extension.updateDefaultEngine()
|
||||
|
||||
expect(consoleSpy).toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe('populateDefaultRemoteEngines', () => {
|
||||
let extension: JanEngineManagementExtension
|
||||
|
||||
beforeEach(() => {
|
||||
// @ts-ignore
|
||||
extension = new JanEngineManagementExtension()
|
||||
vi.resetAllMocks()
|
||||
})
|
||||
|
||||
it('should not add default remote engines if remote engines already exist', async () => {
|
||||
const mockGetEngines = vi.spyOn(extension, 'getEngines')
|
||||
mockGetEngines.mockResolvedValue(mockRemoteEngines)
|
||||
|
||||
const mockAddRemoteEngine = vi.spyOn(extension, 'addRemoteEngine')
|
||||
|
||||
await extension.populateDefaultRemoteEngines()
|
||||
|
||||
expect(mockAddRemoteEngine).not.toBeCalled()
|
||||
})
|
||||
|
||||
it('should add default remote engines if no remote engines exist', async () => {
|
||||
const mockGetEngines = vi.spyOn(extension, 'getEngines')
|
||||
mockGetEngines.mockResolvedValue([])
|
||||
|
||||
const mockAddRemoteEngine = vi.spyOn(extension, 'addRemoteEngine')
|
||||
mockAddRemoteEngine.mockResolvedValue({ messages: 'OK' })
|
||||
|
||||
const mockAddRemoteModel = vi.spyOn(extension, 'addRemoteModel')
|
||||
mockAddRemoteModel.mockResolvedValue(undefined)
|
||||
|
||||
vi.mock('@janhq/core', async (importOriginal) => {
|
||||
const actual = (await importOriginal()) as any
|
||||
return {
|
||||
...actual,
|
||||
events: {
|
||||
emit: vi.fn(),
|
||||
},
|
||||
joinPath: vi.fn().mockResolvedValue('/path/to/settings.json'),
|
||||
getJanDataFolderPath: vi.fn().mockResolvedValue('/path/to/data'),
|
||||
fs: {
|
||||
existsSync: vi.fn().mockResolvedValue(false),
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
await extension.populateDefaultRemoteEngines()
|
||||
|
||||
expect(mockAddRemoteEngine).toHaveBeenCalled()
|
||||
expect(mockAddRemoteModel).toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@ -1,412 +0,0 @@
|
||||
import {
|
||||
EngineManagementExtension,
|
||||
DefaultEngineVariant,
|
||||
Engines,
|
||||
EngineConfig,
|
||||
EngineVariant,
|
||||
EngineReleased,
|
||||
executeOnMain,
|
||||
systemInformation,
|
||||
Model,
|
||||
fs,
|
||||
joinPath,
|
||||
events,
|
||||
ModelEvent,
|
||||
EngineEvent,
|
||||
} from '@janhq/core'
|
||||
import ky, { HTTPError, KyInstance } from 'ky'
|
||||
import { EngineError } from './error'
|
||||
import { getJanDataFolderPath } from '@janhq/core'
|
||||
import { engineVariant } from './utils'
|
||||
|
||||
interface ModelList {
|
||||
data: Model[]
|
||||
}
|
||||
/**
|
||||
* JanEngineManagementExtension is a EngineManagementExtension implementation that provides
|
||||
* functionality for managing engines.
|
||||
*/
|
||||
export default class JanEngineManagementExtension extends EngineManagementExtension {
|
||||
api?: KyInstance
|
||||
/**
|
||||
* Get the API instance
|
||||
* @returns
|
||||
*/
|
||||
async apiInstance(): Promise<KyInstance> {
|
||||
if (this.api) return this.api
|
||||
const apiKey = await window.core?.api.appToken()
|
||||
this.api = ky.extend({
|
||||
prefixUrl: API_URL,
|
||||
headers: apiKey
|
||||
? {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
}
|
||||
: {},
|
||||
retry: 10,
|
||||
})
|
||||
return this.api
|
||||
}
|
||||
/**
|
||||
* Called when the extension is loaded.
|
||||
*/
|
||||
async onLoad() {
|
||||
// Update default local engine
|
||||
// this.updateDefaultEngine()
|
||||
|
||||
// Migrate
|
||||
this.migrate()
|
||||
}
|
||||
|
||||
/**
|
||||
* Called when the extension is unloaded.
|
||||
*/
|
||||
onUnload() { }
|
||||
|
||||
/**
|
||||
* @returns A Promise that resolves to an object of list engines.
|
||||
*/
|
||||
async getEngines(): Promise<Engines> {
|
||||
return {}
|
||||
return this.apiInstance().then((api) =>
|
||||
api
|
||||
.get('v1/engines')
|
||||
.json<Engines>()
|
||||
.then((e) => e)
|
||||
) as Promise<Engines>
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns A Promise that resolves to an object of list engines.
|
||||
*/
|
||||
async getRemoteModels(name: string): Promise<any> {
|
||||
return this.apiInstance().then(
|
||||
(api) =>
|
||||
api
|
||||
.get(`v1/models/remote/${name}`)
|
||||
.json<ModelList>()
|
||||
.catch(() => ({
|
||||
data: [],
|
||||
})) as Promise<ModelList>
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* @param name - Inference engine name.
|
||||
* @returns A Promise that resolves to an array of installed engine.
|
||||
*/
|
||||
async getInstalledEngines(name: string): Promise<EngineVariant[]> {
|
||||
return []
|
||||
return this.apiInstance().then((api) =>
|
||||
api
|
||||
.get(`v1/engines/${name}`)
|
||||
.json<EngineVariant[]>()
|
||||
.then((e) => e)
|
||||
) as Promise<EngineVariant[]>
|
||||
}
|
||||
|
||||
/**
|
||||
* @param name - Inference engine name.
|
||||
* @param version - Version of the engine.
|
||||
* @param platform - Optional to sort by operating system. macOS, linux, windows.
|
||||
* @returns A Promise that resolves to an array of latest released engine by version.
|
||||
*/
|
||||
async getReleasedEnginesByVersion(
|
||||
name: string,
|
||||
version: string,
|
||||
platform?: string
|
||||
) {
|
||||
return this.apiInstance().then((api) =>
|
||||
api
|
||||
.get(`v1/engines/${name}/releases/${version}`)
|
||||
.json<EngineReleased[]>()
|
||||
.then((e) =>
|
||||
platform ? e.filter((r) => r.name.includes(platform)) : e
|
||||
)
|
||||
) as Promise<EngineReleased[]>
|
||||
}
|
||||
|
||||
/**
|
||||
* @param name - Inference engine name.
|
||||
* @param platform - Optional to sort by operating system. macOS, linux, windows.
|
||||
* @returns A Promise that resolves to an array of latest released engine by version.
|
||||
*/
|
||||
async getLatestReleasedEngine(name: string, platform?: string) {
|
||||
return this.apiInstance().then((api) =>
|
||||
api
|
||||
.get(`v1/engines/${name}/releases/latest`)
|
||||
.json<EngineReleased[]>()
|
||||
.then((e) =>
|
||||
platform ? e.filter((r) => r.name.includes(platform)) : e
|
||||
)
|
||||
) as Promise<EngineReleased[]>
|
||||
}
|
||||
|
||||
/**
|
||||
* @param name - Inference engine name.
|
||||
* @returns A Promise that resolves to intall of engine.
|
||||
*/
|
||||
async installEngine(name: string, engineConfig: EngineConfig) {
|
||||
return this.apiInstance().then((api) =>
|
||||
api
|
||||
.post(`v1/engines/${name}/install`, { json: engineConfig })
|
||||
.then((e) => e)
|
||||
) as Promise<{ messages: string }>
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a new remote engine
|
||||
* @returns A Promise that resolves to intall of engine.
|
||||
*/
|
||||
async addRemoteEngine(
|
||||
engineConfig: EngineConfig,
|
||||
persistModels: boolean = true
|
||||
) {
|
||||
// Populate default settings
|
||||
if (
|
||||
engineConfig.metadata?.transform_req?.chat_completions &&
|
||||
!engineConfig.metadata.transform_req.chat_completions.template
|
||||
)
|
||||
engineConfig.metadata.transform_req.chat_completions.template =
|
||||
DEFAULT_REQUEST_PAYLOAD_TRANSFORM
|
||||
|
||||
if (
|
||||
engineConfig.metadata?.transform_resp?.chat_completions &&
|
||||
!engineConfig.metadata.transform_resp.chat_completions?.template
|
||||
)
|
||||
engineConfig.metadata.transform_resp.chat_completions.template =
|
||||
DEFAULT_RESPONSE_BODY_TRANSFORM
|
||||
|
||||
if (engineConfig.metadata && !engineConfig.metadata?.header_template)
|
||||
engineConfig.metadata.header_template = DEFAULT_REQUEST_HEADERS_TRANSFORM
|
||||
|
||||
return this.apiInstance().then((api) =>
|
||||
api.post('v1/engines', { json: engineConfig }).then((e) => {
|
||||
if (persistModels && engineConfig.metadata?.get_models_url) {
|
||||
// Pull /models from remote models endpoint
|
||||
return this.populateRemoteModels(engineConfig)
|
||||
.then(() => e)
|
||||
.catch(() => e)
|
||||
}
|
||||
return e
|
||||
})
|
||||
) as Promise<{ messages: string }>
|
||||
}
|
||||
|
||||
/**
|
||||
* @param name - Inference engine name.
|
||||
* @returns A Promise that resolves to unintall of engine.
|
||||
*/
|
||||
async uninstallEngine(name: string, engineConfig: EngineConfig) {
|
||||
return this.apiInstance().then((api) =>
|
||||
api
|
||||
.delete(`v1/engines/${name}/install`, { json: engineConfig })
|
||||
.then((e) => e)
|
||||
) as Promise<{ messages: string }>
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a new remote model
|
||||
* @param model - Remote model object.
|
||||
*/
|
||||
async addRemoteModel(model: Model) {
|
||||
return this.apiInstance().then((api) =>
|
||||
api
|
||||
.post('v1/models/add', {
|
||||
json: {
|
||||
inference_params: {
|
||||
max_tokens: 4096,
|
||||
temperature: 0.7,
|
||||
top_p: 0.95,
|
||||
stream: true,
|
||||
frequency_penalty: 0,
|
||||
presence_penalty: 0,
|
||||
},
|
||||
...model,
|
||||
},
|
||||
})
|
||||
.then((e) => e)
|
||||
.then(() => { })
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* @param name - Inference engine name.
|
||||
* @returns A Promise that resolves to an object of default engine.
|
||||
*/
|
||||
async getDefaultEngineVariant(name: string) {
|
||||
return this.apiInstance().then((api) =>
|
||||
api
|
||||
.get(`v1/engines/${name}/default`)
|
||||
.json<{ messages: string }>()
|
||||
.then((e) => e)
|
||||
) as Promise<DefaultEngineVariant>
|
||||
}
|
||||
|
||||
/**
|
||||
* @body variant - string
|
||||
* @body version - string
|
||||
* @returns A Promise that resolves to set default engine.
|
||||
*/
|
||||
async setDefaultEngineVariant(name: string, engineConfig: EngineConfig) {
|
||||
return this.apiInstance().then((api) =>
|
||||
api
|
||||
.post(`v1/engines/${name}/default`, { json: engineConfig })
|
||||
.then((e) => e)
|
||||
) as Promise<{ messages: string }>
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns A Promise that resolves to update engine.
|
||||
*/
|
||||
async updateEngine(name: string, engineConfig?: EngineConfig) {
|
||||
return this.apiInstance().then((api) =>
|
||||
api
|
||||
.post(`v1/engines/${name}/update`, { json: engineConfig })
|
||||
.then((e) => e)
|
||||
) as Promise<{ messages: string }>
|
||||
}
|
||||
|
||||
/**
|
||||
* Update default local engine
|
||||
* This is to use built-in engine variant in case there is no default engine set
|
||||
*/
|
||||
async updateDefaultEngine() {
|
||||
const systemInfo = await systemInformation()
|
||||
try {
|
||||
const variant = await this.getDefaultEngineVariant('llama-cpp')
|
||||
if (
|
||||
(systemInfo.gpuSetting.vulkan && !variant.variant.includes('vulkan')) ||
|
||||
(systemInfo.gpuSetting.vulkan === false &&
|
||||
variant.variant.includes('vulkan'))
|
||||
) {
|
||||
throw new EngineError('Switch engine.')
|
||||
}
|
||||
const installedEngines = await this.getInstalledEngines('llama-cpp')
|
||||
if (
|
||||
!installedEngines.some(
|
||||
(e) => e.name === variant.variant && e.version === variant.version
|
||||
) ||
|
||||
variant.version < CORTEX_ENGINE_VERSION
|
||||
) {
|
||||
throw new EngineError(
|
||||
'Default engine is not available, use bundled version.'
|
||||
)
|
||||
}
|
||||
} catch (error) {
|
||||
if (
|
||||
(error instanceof HTTPError && error.response.status === 400) ||
|
||||
error instanceof EngineError
|
||||
) {
|
||||
const variant = await engineVariant(systemInfo.gpuSetting)
|
||||
// TODO: Use correct provider name when moving to llama.cpp extension
|
||||
await this.setDefaultEngineVariant('llama-cpp', {
|
||||
variant: variant,
|
||||
version: `${CORTEX_ENGINE_VERSION}`,
|
||||
})
|
||||
} else {
|
||||
console.error('An unexpected error occurred:', error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This is to populate default remote engines in case there is no customized remote engine setting
|
||||
*/
|
||||
async populateDefaultRemoteEngines() {
|
||||
const engines = await this.getEngines()
|
||||
if (
|
||||
!Object.values(engines)
|
||||
.flat()
|
||||
.some((e) => e.type === 'remote')
|
||||
) {
|
||||
await Promise.all(
|
||||
DEFAULT_REMOTE_ENGINES.map(async (engine) => {
|
||||
const { id, ...data } = engine
|
||||
|
||||
/// BEGIN - Migrate legacy api key settings
|
||||
let api_key = undefined
|
||||
if (id) {
|
||||
const apiKeyPath = await joinPath([
|
||||
await getJanDataFolderPath(),
|
||||
'settings',
|
||||
id,
|
||||
'settings.json',
|
||||
])
|
||||
if (await fs.existsSync(apiKeyPath)) {
|
||||
const settings = await fs.readFileSync(apiKeyPath, 'utf-8')
|
||||
api_key = JSON.parse(settings).find(
|
||||
(e) => e.key === `${data.engine}-api-key`
|
||||
)?.controllerProps?.value
|
||||
}
|
||||
}
|
||||
data.api_key = api_key
|
||||
/// END - Migrate legacy api key settings
|
||||
|
||||
await this.addRemoteEngine(data, false).catch(console.error)
|
||||
})
|
||||
)
|
||||
events.emit(EngineEvent.OnEngineUpdate, {})
|
||||
await Promise.all(
|
||||
DEFAULT_REMOTE_MODELS.map((data: Model) =>
|
||||
this.addRemoteModel(data).catch(() => { })
|
||||
)
|
||||
)
|
||||
events.emit(ModelEvent.OnModelsUpdate, { fetch: true })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pulls models list from the remote provider and persist
|
||||
* @param engineConfig
|
||||
* @returns
|
||||
*/
|
||||
private populateRemoteModels = async (engineConfig: EngineConfig) => {
|
||||
return this.getRemoteModels(engineConfig.engine)
|
||||
.then((models: ModelList) => {
|
||||
if (models?.data)
|
||||
Promise.all(
|
||||
models.data.map((model) =>
|
||||
this.addRemoteModel({
|
||||
...model,
|
||||
engine: engineConfig.engine,
|
||||
model: model.model ?? model.id,
|
||||
}).catch(console.info)
|
||||
)
|
||||
).then(() => {
|
||||
events.emit(ModelEvent.OnModelsUpdate, { fetch: true })
|
||||
})
|
||||
})
|
||||
.catch(console.info)
|
||||
}
|
||||
|
||||
/**
|
||||
* Update engine settings to the latest version
|
||||
*/
|
||||
migrate = async () => {
|
||||
// Ensure health check is done
|
||||
const version = await this.getSetting<string>('version', '0.0.0')
|
||||
const engines = await this.getEngines()
|
||||
if (version < VERSION) {
|
||||
console.log('Migrating engine settings...')
|
||||
// Migrate engine settings
|
||||
await Promise.all(
|
||||
DEFAULT_REMOTE_ENGINES.map((engine) => {
|
||||
const { id, ...data } = engine
|
||||
|
||||
data.api_key = engines[id]?.api_key
|
||||
return this.updateEngine(id, {
|
||||
...data,
|
||||
}).catch(console.error)
|
||||
})
|
||||
)
|
||||
await this.updateSettings([
|
||||
{
|
||||
key: 'version',
|
||||
controllerProps: {
|
||||
value: VERSION,
|
||||
},
|
||||
},
|
||||
])
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,69 +0,0 @@
|
||||
import * as path from 'path'
|
||||
import {
|
||||
appResourcePath,
|
||||
getJanDataFolderPath,
|
||||
log,
|
||||
} from '@janhq/core/node'
|
||||
import { mkdir, readdir, symlink, cp } from 'fs/promises'
|
||||
import { existsSync } from 'fs'
|
||||
|
||||
/**
|
||||
* Create symlink to each variant for the default bundled version
|
||||
* If running in AppImage environment, copy files instead of creating symlinks
|
||||
*/
|
||||
const symlinkEngines = async () => {
|
||||
const sourceEnginePath = path.join(
|
||||
appResourcePath(),
|
||||
'shared',
|
||||
'engines',
|
||||
'llama.cpp'
|
||||
)
|
||||
const symlinkEnginePath = path.join(
|
||||
getJanDataFolderPath(),
|
||||
'engines',
|
||||
'llama.cpp'
|
||||
)
|
||||
const variantFolders = await readdir(sourceEnginePath)
|
||||
const isStandalone = process.platform === 'linux'
|
||||
|
||||
for (const variant of variantFolders) {
|
||||
const targetVariantPath = path.join(
|
||||
sourceEnginePath,
|
||||
variant,
|
||||
CORTEX_ENGINE_VERSION
|
||||
)
|
||||
const symlinkVariantPath = path.join(
|
||||
symlinkEnginePath,
|
||||
variant,
|
||||
CORTEX_ENGINE_VERSION
|
||||
)
|
||||
|
||||
await mkdir(path.join(symlinkEnginePath, variant), {
|
||||
recursive: true,
|
||||
}).catch((error) => log(JSON.stringify(error)))
|
||||
|
||||
// Skip if already exists
|
||||
if (existsSync(symlinkVariantPath)) {
|
||||
console.log(`Target already exists: ${symlinkVariantPath}`)
|
||||
continue
|
||||
}
|
||||
|
||||
if (isStandalone) {
|
||||
// Copy files for AppImage environments instead of symlinking
|
||||
await cp(targetVariantPath, symlinkVariantPath, { recursive: true }).catch(
|
||||
(error) => log(JSON.stringify(error))
|
||||
)
|
||||
console.log(`Files copied: ${targetVariantPath} -> ${symlinkVariantPath}`)
|
||||
} else {
|
||||
// Create symlink for other environments
|
||||
await symlink(targetVariantPath, symlinkVariantPath, 'junction').catch(
|
||||
(error) => log(JSON.stringify(error))
|
||||
)
|
||||
console.log(`Symlink created: ${targetVariantPath} -> ${symlinkVariantPath}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default {
|
||||
symlinkEngines,
|
||||
}
|
||||
@ -1,139 +0,0 @@
|
||||
import { describe, beforeEach, it, expect, vi } from 'vitest'
|
||||
import JanEngineManagementExtension from './index'
|
||||
import { InferenceEngine } from '@janhq/core'
|
||||
|
||||
describe('populateRemoteModels', () => {
|
||||
let extension: JanEngineManagementExtension
|
||||
|
||||
beforeEach(() => {
|
||||
// @ts-ignore
|
||||
extension = new JanEngineManagementExtension()
|
||||
vi.resetAllMocks()
|
||||
})
|
||||
|
||||
it('should populate remote models successfully', async () => {
|
||||
const mockEngineConfig = {
|
||||
engine: InferenceEngine.openai,
|
||||
}
|
||||
|
||||
const mockRemoteModels = {
|
||||
data: [
|
||||
{
|
||||
id: 'gpt-4',
|
||||
name: 'GPT-4',
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
const mockGetRemoteModels = vi.spyOn(extension, 'getRemoteModels')
|
||||
mockGetRemoteModels.mockResolvedValue(mockRemoteModels)
|
||||
|
||||
const mockAddRemoteModel = vi.spyOn(extension, 'addRemoteModel')
|
||||
mockAddRemoteModel.mockResolvedValue(undefined)
|
||||
|
||||
vi.mock('@janhq/core', async (importOriginal) => {
|
||||
const actual = (await importOriginal()) as any
|
||||
return {
|
||||
...actual,
|
||||
events: {
|
||||
emit: vi.fn(),
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
// Use the private method through index.ts
|
||||
// @ts-ignore - Accessing private method for testing
|
||||
await extension.populateRemoteModels(mockEngineConfig)
|
||||
|
||||
expect(mockGetRemoteModels).toHaveBeenCalledWith(mockEngineConfig.engine)
|
||||
expect(mockAddRemoteModel).toHaveBeenCalledWith({
|
||||
...mockRemoteModels.data[0],
|
||||
engine: mockEngineConfig.engine,
|
||||
model: 'gpt-4',
|
||||
})
|
||||
})
|
||||
|
||||
it('should handle empty data from remote models', async () => {
|
||||
const mockEngineConfig = {
|
||||
engine: InferenceEngine.openai,
|
||||
}
|
||||
|
||||
const mockGetRemoteModels = vi.spyOn(extension, 'getRemoteModels')
|
||||
mockGetRemoteModels.mockResolvedValue({ data: [] })
|
||||
|
||||
const mockAddRemoteModel = vi.spyOn(extension, 'addRemoteModel')
|
||||
|
||||
vi.mock('@janhq/core', async (importOriginal) => {
|
||||
const actual = (await importOriginal()) as any
|
||||
return {
|
||||
...actual,
|
||||
events: {
|
||||
emit: vi.fn(),
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
// @ts-ignore - Accessing private method for testing
|
||||
await extension.populateRemoteModels(mockEngineConfig)
|
||||
|
||||
expect(mockGetRemoteModels).toHaveBeenCalledWith(mockEngineConfig.engine)
|
||||
expect(mockAddRemoteModel).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should handle errors when getting remote models', async () => {
|
||||
const mockEngineConfig = {
|
||||
engine: InferenceEngine.openai,
|
||||
}
|
||||
|
||||
const mockGetRemoteModels = vi.spyOn(extension, 'getRemoteModels')
|
||||
mockGetRemoteModels.mockRejectedValue(new Error('Failed to fetch models'))
|
||||
|
||||
const consoleSpy = vi.spyOn(console, 'info').mockImplementation(() => {})
|
||||
|
||||
// @ts-ignore - Accessing private method for testing
|
||||
await extension.populateRemoteModels(mockEngineConfig)
|
||||
|
||||
expect(mockGetRemoteModels).toHaveBeenCalledWith(mockEngineConfig.engine)
|
||||
expect(consoleSpy).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should handle errors when adding remote models', async () => {
|
||||
const mockEngineConfig = {
|
||||
engine: InferenceEngine.openai,
|
||||
}
|
||||
|
||||
const mockRemoteModels = {
|
||||
data: [
|
||||
{
|
||||
id: 'gpt-4',
|
||||
name: 'GPT-4',
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
const mockGetRemoteModels = vi.spyOn(extension, 'getRemoteModels')
|
||||
mockGetRemoteModels.mockResolvedValue(mockRemoteModels)
|
||||
|
||||
const mockAddRemoteModel = vi.spyOn(extension, 'addRemoteModel')
|
||||
mockAddRemoteModel.mockRejectedValue(new Error('Failed to add model'))
|
||||
|
||||
const consoleSpy = vi.spyOn(console, 'info').mockImplementation(() => {})
|
||||
|
||||
vi.mock('@janhq/core', async (importOriginal) => {
|
||||
const actual = (await importOriginal()) as any
|
||||
return {
|
||||
...actual,
|
||||
events: {
|
||||
emit: vi.fn(),
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
// @ts-ignore - Accessing private method for testing
|
||||
await extension.populateRemoteModels(mockEngineConfig)
|
||||
|
||||
expect(mockGetRemoteModels).toHaveBeenCalledWith(mockEngineConfig.engine)
|
||||
expect(mockAddRemoteModel).toHaveBeenCalled()
|
||||
expect(consoleSpy).toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@ -1,90 +0,0 @@
|
||||
import { describe, it, expect, vi } from 'vitest'
|
||||
import { engineVariant } from './utils'
|
||||
|
||||
vi.mock('@janhq/core', () => {
|
||||
return {
|
||||
log: () => {},
|
||||
}
|
||||
})
|
||||
|
||||
describe('engineVariant', () => {
|
||||
it('should return mac-arm64 when platform is darwin and arch is arm64', async () => {
|
||||
vi.stubGlobal('PLATFORM', 'darwin')
|
||||
const result = await engineVariant({
|
||||
cpu: { arch: 'arm64', instructions: '' },
|
||||
gpus: [],
|
||||
vulkan: false,
|
||||
})
|
||||
expect(result).toBe('mac-arm64')
|
||||
})
|
||||
|
||||
it('should return mac-amd64 when platform is darwin and arch is not arm64', async () => {
|
||||
vi.stubGlobal('PLATFORM', 'darwin')
|
||||
const result = await engineVariant({
|
||||
cpu: { arch: 'x64', instructions: [] },
|
||||
gpus: [],
|
||||
vulkan: false,
|
||||
})
|
||||
expect(result).toBe('mac-amd64')
|
||||
})
|
||||
|
||||
it('should return windows-amd64-noavx-cuda-12-0 when platform is win32, cuda is enabled, and cuda version is 12', async () => {
|
||||
vi.stubGlobal('PLATFORM', 'win32')
|
||||
const result = await engineVariant({
|
||||
cpu: { arch: 'x64', instructions: ['avx2'] },
|
||||
gpus: [
|
||||
{
|
||||
activated: true,
|
||||
version: '12',
|
||||
additional_information: { driver_version: '1.0' },
|
||||
},
|
||||
],
|
||||
vulkan: false,
|
||||
})
|
||||
expect(result).toBe('windows-amd64-avx2-cuda-12-0')
|
||||
})
|
||||
|
||||
it('should return linux-amd64-noavx-cuda-11-7 when platform is linux, cuda is enabled, and cuda version is 11', async () => {
|
||||
vi.stubGlobal('PLATFORM', 'linux')
|
||||
const result = await engineVariant({
|
||||
cpu: { arch: 'x64', instructions: [] },
|
||||
gpus: [
|
||||
{
|
||||
activated: true,
|
||||
version: '11',
|
||||
additional_information: { driver_version: '1.0' },
|
||||
},
|
||||
],
|
||||
vulkan: false,
|
||||
})
|
||||
expect(result).toBe('linux-amd64-noavx-cuda-11-7')
|
||||
})
|
||||
|
||||
it('should return windows-amd64-vulkan when platform is win32 and vulkan is enabled', async () => {
|
||||
vi.stubGlobal('PLATFORM', 'win32')
|
||||
const result = await engineVariant({
|
||||
cpu: { arch: 'x64', instructions: [] },
|
||||
gpus: [{ activated: true, version: '12' }],
|
||||
vulkan: true,
|
||||
})
|
||||
expect(result).toBe('windows-amd64-vulkan')
|
||||
})
|
||||
|
||||
it('should return windows-amd64-avx512 when platform is win32, no gpu detected and avx512 cpu instruction is supported', async () => {
|
||||
vi.stubGlobal('PLATFORM', 'win32')
|
||||
const result = await engineVariant({
|
||||
cpu: { arch: 'x64', instructions: ['avx512'] },
|
||||
gpus: [{ activated: true, version: '12' }],
|
||||
})
|
||||
expect(result).toBe('windows-amd64-avx512')
|
||||
})
|
||||
|
||||
it('should return windows-amd64-avx512 when platform is win32, no gpu detected and no accelerated cpu instructions are supported', async () => {
|
||||
vi.stubGlobal('PLATFORM', 'win32')
|
||||
const result = await engineVariant({
|
||||
cpu: { arch: 'x64', instructions: [''] },
|
||||
gpus: [{ activated: true, version: '12' }],
|
||||
})
|
||||
expect(result).toBe('windows-amd64-noavx')
|
||||
})
|
||||
})
|
||||
@ -1,105 +0,0 @@
|
||||
import { GpuSetting, log } from '@janhq/core'
|
||||
|
||||
// Supported run modes
|
||||
enum RunMode {
|
||||
Cuda = 'cuda',
|
||||
CPU = 'cpu',
|
||||
}
|
||||
|
||||
// Supported instruction sets
|
||||
const instructionBinaryNames = ['noavx', 'avx', 'avx2', 'avx512']
|
||||
|
||||
/**
|
||||
* The GPU runMode that will be set - either 'vulkan', 'cuda', or empty for cpu.
|
||||
* @param settings
|
||||
* @returns
|
||||
*/
|
||||
|
||||
const gpuRunMode = (settings?: GpuSetting): RunMode => {
|
||||
return settings.gpus?.some(
|
||||
(gpu) =>
|
||||
gpu.activated &&
|
||||
gpu.additional_information &&
|
||||
gpu.additional_information.driver_version
|
||||
)
|
||||
? RunMode.Cuda
|
||||
: RunMode.CPU
|
||||
}
|
||||
|
||||
/**
|
||||
* The OS & architecture that the current process is running on.
|
||||
* @returns win, mac-x64, mac-arm64, or linux
|
||||
*/
|
||||
const os = (settings?: GpuSetting): string => {
|
||||
return PLATFORM === 'win32'
|
||||
? 'win'
|
||||
: PLATFORM === 'darwin'
|
||||
? settings?.cpu?.arch === 'arm64'
|
||||
? 'macos-arm64'
|
||||
: 'macos-x64'
|
||||
: 'linux'
|
||||
}
|
||||
|
||||
/**
|
||||
* The CUDA version that will be set - either 'cu12.0' or 'cu11.7'.
|
||||
* @param settings
|
||||
* @returns
|
||||
*/
|
||||
const cudaVersion = (
|
||||
settings?: GpuSetting
|
||||
): 'cu12.0' | 'cu11.7' | undefined => {
|
||||
return settings.gpus?.some((gpu) => gpu.version.includes('12'))
|
||||
? 'cu12.0'
|
||||
: 'cu11.7'
|
||||
}
|
||||
|
||||
/**
|
||||
* The CPU instructions that will be set - either 'avx512', 'avx2', 'avx', or 'noavx'.
|
||||
* @returns
|
||||
*/
|
||||
|
||||
/**
|
||||
* Find which variant to run based on the current platform.
|
||||
*/
|
||||
export const engineVariant = async (
|
||||
gpuSetting?: GpuSetting
|
||||
): Promise<string> => {
|
||||
const platform = os(gpuSetting)
|
||||
|
||||
// There is no need to append the variant extension for mac
|
||||
if (platform.startsWith('mac')) return platform
|
||||
|
||||
const runMode = gpuRunMode(gpuSetting)
|
||||
// Only Nvidia GPUs have addition_information set and activated by default
|
||||
let engineVariant =
|
||||
!gpuSetting?.vulkan &&
|
||||
(!gpuSetting.gpus?.length ||
|
||||
gpuSetting.gpus.some((e) => e.additional_information && e.activated))
|
||||
? [
|
||||
platform,
|
||||
...(runMode === RunMode.Cuda
|
||||
? // For cuda we only need to check if the cpu supports avx2 or noavx - since other binaries are not shipped with the extension
|
||||
[
|
||||
gpuSetting.cpu?.instructions.includes('avx2') ||
|
||||
gpuSetting.cpu?.instructions.includes('avx512')
|
||||
? 'avx2'
|
||||
: 'noavx',
|
||||
runMode,
|
||||
cudaVersion(gpuSetting),
|
||||
'x64',
|
||||
]
|
||||
: // For cpu only we need to check all available supported instructions
|
||||
[
|
||||
(gpuSetting.cpu?.instructions ?? ['noavx']).find((e) =>
|
||||
instructionBinaryNames.includes(e.toLowerCase())
|
||||
) ?? 'noavx',
|
||||
'x64',
|
||||
]),
|
||||
].filter(Boolean)
|
||||
: [platform, 'vulkan', 'x64']
|
||||
|
||||
let engineVariantString = engineVariant.join('-')
|
||||
|
||||
log(`[CORTEX]: Engine variant: ${engineVariantString}`)
|
||||
return engineVariantString
|
||||
}
|
||||
@ -1,16 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "es2016",
|
||||
"module": "ES6",
|
||||
"moduleResolution": "node",
|
||||
"outDir": "./dist",
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"strict": false,
|
||||
"skipLibCheck": true,
|
||||
"rootDir": "./src",
|
||||
"resolveJsonModule": true
|
||||
},
|
||||
"include": ["./src"],
|
||||
"exclude": ["src/**/*.test.ts", "rolldown.config.mjs"]
|
||||
}
|
||||
@ -1,5 +0,0 @@
|
||||
/** @type {import('ts-jest').JestConfigWithTsJest} */
|
||||
module.exports = {
|
||||
preset: 'ts-jest',
|
||||
testEnvironment: 'node',
|
||||
}
|
||||
@ -1,46 +0,0 @@
|
||||
{
|
||||
"name": "@janhq/hardware-management-extension",
|
||||
"productName": "Hardware Management",
|
||||
"version": "1.0.0",
|
||||
"description": "Manages hardware settings.",
|
||||
"main": "dist/index.js",
|
||||
"node": "dist/node/index.cjs.js",
|
||||
"author": "Jan <service@jan.ai>",
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
"test": "jest",
|
||||
"build": "rolldown -c rolldown.config.mjs",
|
||||
"codesign:darwin": "../../.github/scripts/auto-sign.sh",
|
||||
"codesign:win32:linux": "echo 'No codesigning required'",
|
||||
"codesign": "run-script-os",
|
||||
"build:publish": "rimraf *.tgz --glob || true && yarn build && yarn codesign && npm pack && cpx *.tgz ../../pre-install"
|
||||
},
|
||||
"exports": {
|
||||
".": "./dist/index.js",
|
||||
"./main": "./dist/module.js"
|
||||
},
|
||||
"devDependencies": {
|
||||
"cpx": "^1.5.0",
|
||||
"rimraf": "^3.0.2",
|
||||
"rolldown": "^1.0.0-beta.1",
|
||||
"run-script-os": "^1.1.6",
|
||||
"ts-loader": "^9.5.0",
|
||||
"typescript": "^5.3.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"@janhq/core": "../../core/package.tgz",
|
||||
"ky": "^1.7.2",
|
||||
"p-queue": "^8.0.1"
|
||||
},
|
||||
"bundledDependencies": [
|
||||
"@janhq/core"
|
||||
],
|
||||
"hardwares": {
|
||||
"node": ">=18.0.0"
|
||||
},
|
||||
"files": [
|
||||
"dist/*",
|
||||
"package.json",
|
||||
"README.md"
|
||||
]
|
||||
}
|
||||
@ -1,16 +0,0 @@
|
||||
import { defineConfig } from 'rolldown'
|
||||
import pkgJson from './package.json' with { type: 'json' }
|
||||
|
||||
export default defineConfig([
|
||||
{
|
||||
input: 'src/index.ts',
|
||||
output: {
|
||||
format: 'esm',
|
||||
file: 'dist/index.js',
|
||||
},
|
||||
define: {
|
||||
NODE: JSON.stringify(`${pkgJson.name}/${pkgJson.node}`),
|
||||
API_URL: JSON.stringify(`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? "39291"}`),
|
||||
},
|
||||
},
|
||||
])
|
||||
@ -1,11 +0,0 @@
|
||||
declare const API_URL: string
|
||||
declare const NODE: string
|
||||
|
||||
interface Core {
|
||||
api: APIFunctions
|
||||
events: EventEmitter
|
||||
}
|
||||
interface Window {
|
||||
core?: Core | undefined
|
||||
electronAPI?: any | undefined
|
||||
}
|
||||
@ -1,65 +0,0 @@
|
||||
import { HardwareManagementExtension, HardwareInformation } from '@janhq/core'
|
||||
import ky, { KyInstance } from 'ky'
|
||||
|
||||
/**
|
||||
* JSONHardwareManagementExtension is a HardwareManagementExtension implementation that provides
|
||||
* functionality for managing engines.
|
||||
*/
|
||||
export default class JSONHardwareManagementExtension extends HardwareManagementExtension {
|
||||
/**
|
||||
* Called when the extension is loaded.
|
||||
*/
|
||||
async onLoad() {}
|
||||
|
||||
api?: KyInstance
|
||||
/**
|
||||
* Get the API instance
|
||||
* @returns
|
||||
*/
|
||||
async apiInstance(): Promise<KyInstance> {
|
||||
if (this.api) return this.api
|
||||
const apiKey = (await window.core?.api.appToken())
|
||||
this.api = ky.extend({
|
||||
prefixUrl: API_URL,
|
||||
headers: apiKey
|
||||
? {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
}
|
||||
: {},
|
||||
retry: 10,
|
||||
})
|
||||
return this.api
|
||||
}
|
||||
|
||||
/**
|
||||
* Called when the extension is unloaded.
|
||||
*/
|
||||
onUnload() {}
|
||||
|
||||
/**
|
||||
* @returns A Promise that resolves to an object of hardware.
|
||||
*/
|
||||
async getHardware(): Promise<HardwareInformation> {
|
||||
return this.apiInstance().then((api) =>
|
||||
api
|
||||
.get('v1/hardware')
|
||||
.json<HardwareInformation>()
|
||||
.then((e) => e)
|
||||
) as Promise<HardwareInformation>
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns A Promise that resolves to an object of set gpu activate.
|
||||
*/
|
||||
async setActiveGpu(data: { gpus: number[] }): Promise<{
|
||||
message: string
|
||||
activated_gpus: number[]
|
||||
}> {
|
||||
return this.apiInstance().then((api) =>
|
||||
api.post('v1/hardware/activate', { json: data }).then((e) => e)
|
||||
) as Promise<{
|
||||
message: string
|
||||
activated_gpus: number[]
|
||||
}>
|
||||
}
|
||||
}
|
||||
@ -1,16 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "es2016",
|
||||
"module": "ES6",
|
||||
"moduleResolution": "node",
|
||||
"outDir": "./dist",
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"strict": false,
|
||||
"skipLibCheck": true,
|
||||
"rootDir": "./src",
|
||||
"resolveJsonModule": true
|
||||
},
|
||||
"include": ["./src"],
|
||||
"exclude": ["src/**/*.test.ts", "rolldown.config.mjs"]
|
||||
}
|
||||
@ -152,6 +152,16 @@ export default class llamacpp_extension extends AIEngine {
|
||||
])
|
||||
}
|
||||
|
||||
async getProviderPath(): Promise<string> {
|
||||
if (!this.providerPath) {
|
||||
this.providerPath = await joinPath([
|
||||
await getJanDataFolderPath(),
|
||||
this.providerId,
|
||||
])
|
||||
}
|
||||
return this.providerPath
|
||||
}
|
||||
|
||||
override async onUnload(): Promise<void> {
|
||||
// Terminate all active sessions
|
||||
for (const [_, sInfo] of this.activeSessions) {
|
||||
@ -193,7 +203,7 @@ export default class llamacpp_extension extends AIEngine {
|
||||
|
||||
// Implement the required LocalProvider interface methods
|
||||
override async list(): Promise<modelInfo[]> {
|
||||
const modelsDir = await joinPath([this.providerPath, 'models'])
|
||||
const modelsDir = await joinPath([await this.getProviderPath(), 'models'])
|
||||
if (!(await fs.existsSync(modelsDir))) {
|
||||
return []
|
||||
}
|
||||
@ -262,7 +272,7 @@ export default class llamacpp_extension extends AIEngine {
|
||||
)
|
||||
|
||||
const configPath = await joinPath([
|
||||
this.providerPath,
|
||||
await this.getProviderPath(),
|
||||
'models',
|
||||
modelId,
|
||||
'model.yml',
|
||||
@ -498,7 +508,7 @@ export default class llamacpp_extension extends AIEngine {
|
||||
|
||||
console.log('Calling Tauri command llama_load with args:', args)
|
||||
const backendPath = await getBackendExePath(backend, version)
|
||||
const libraryPath = await joinPath([this.providerPath, 'lib'])
|
||||
const libraryPath = await joinPath([await this.getProviderPath(), 'lib'])
|
||||
|
||||
try {
|
||||
// TODO: add LIBRARY_PATH
|
||||
@ -568,7 +578,9 @@ export default class llamacpp_extension extends AIEngine {
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => null)
|
||||
throw new Error(
|
||||
`API request failed with status ${response.status}: ${JSON.stringify(errorData)}`
|
||||
`API request failed with status ${response.status}: ${JSON.stringify(
|
||||
errorData
|
||||
)}`
|
||||
)
|
||||
}
|
||||
|
||||
@ -622,7 +634,8 @@ export default class llamacpp_extension extends AIEngine {
|
||||
}
|
||||
|
||||
override async chat(
|
||||
opts: chatCompletionRequest
|
||||
opts: chatCompletionRequest,
|
||||
abortController?: AbortController
|
||||
): Promise<chatCompletion | AsyncIterable<chatCompletionChunk>> {
|
||||
const sessionInfo = this.findSessionByModel(opts.model)
|
||||
if (!sessionInfo) {
|
||||
@ -630,6 +643,7 @@ export default class llamacpp_extension extends AIEngine {
|
||||
}
|
||||
const baseUrl = `http://localhost:${sessionInfo.port}/v1`
|
||||
const url = `${baseUrl}/chat/completions`
|
||||
console.log('Session Info:', sessionInfo, sessionInfo.api_key)
|
||||
const headers = {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${sessionInfo.api_key}`,
|
||||
@ -644,12 +658,15 @@ export default class llamacpp_extension extends AIEngine {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body,
|
||||
signal: abortController?.signal,
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => null)
|
||||
throw new Error(
|
||||
`API request failed with status ${response.status}: ${JSON.stringify(errorData)}`
|
||||
`API request failed with status ${response.status}: ${JSON.stringify(
|
||||
errorData
|
||||
)}`
|
||||
)
|
||||
}
|
||||
|
||||
@ -657,7 +674,11 @@ export default class llamacpp_extension extends AIEngine {
|
||||
}
|
||||
|
||||
override async delete(modelId: string): Promise<void> {
|
||||
const modelDir = await joinPath([this.providerPath, 'models', modelId])
|
||||
const modelDir = await joinPath([
|
||||
await this.getProviderPath(),
|
||||
'models',
|
||||
modelId,
|
||||
])
|
||||
|
||||
if (!(await fs.existsSync(await joinPath([modelDir, 'model.yml'])))) {
|
||||
throw new Error(`Model ${modelId} does not exist`)
|
||||
|
||||
@ -1,75 +0,0 @@
|
||||
# Create a Jan Extension using Typescript
|
||||
|
||||
Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀
|
||||
|
||||
## Create Your Own Extension
|
||||
|
||||
To create your own extension, you can use this repository as a template! Just follow the below instructions:
|
||||
|
||||
1. Click the Use this template button at the top of the repository
|
||||
2. Select Create a new repository
|
||||
3. Select an owner and name for your new repository
|
||||
4. Click Create repository
|
||||
5. Clone your new repository
|
||||
|
||||
## Initial Setup
|
||||
|
||||
After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension.
|
||||
|
||||
> [!NOTE]
|
||||
>
|
||||
> You'll need to have a reasonably modern version of
|
||||
> [Node.js](https://nodejs.org) handy. If you are using a version manager like
|
||||
> [`nodenv`](https://github.com/nodenv/nodenv) or
|
||||
> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the
|
||||
> root of your repository to install the version specified in
|
||||
> [`package.json`](./package.json). Otherwise, 20.x or later should work!
|
||||
|
||||
1. :hammer_and_wrench: Install the dependencies
|
||||
|
||||
```bash
|
||||
npm install
|
||||
```
|
||||
|
||||
1. :building_construction: Package the TypeScript for distribution
|
||||
|
||||
```bash
|
||||
npm run bundle
|
||||
```
|
||||
|
||||
1. :white_check_mark: Check your artifact
|
||||
|
||||
There will be a tgz file in your extension directory now
|
||||
|
||||
## Update the Extension Metadata
|
||||
|
||||
The [`package.json`](package.json) file defines metadata about your extension, such as
|
||||
extension name, main entry, description and version.
|
||||
|
||||
When you copy this repository, update `package.json` with the name, description for your extension.
|
||||
|
||||
## Update the Extension Code
|
||||
|
||||
The [`src/`](./src/) directory is the heart of your extension! This contains the
|
||||
source code that will be run when your extension functions are invoked. You can replace the
|
||||
contents of this directory with your own code.
|
||||
|
||||
There are a few things to keep in mind when writing your extension code:
|
||||
|
||||
- Most Jan Extension functions are processed asynchronously.
|
||||
In `index.ts`, you will see that the extension function will return a `Promise<any>`.
|
||||
|
||||
```typescript
|
||||
import { events, MessageEvent, MessageRequest } from '@janhq/core'
|
||||
|
||||
function onStart(): Promise<any> {
|
||||
return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) =>
|
||||
this.inference(data)
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
For more information about the Jan Extension Core module, see the
|
||||
[documentation](https://github.com/menloresearch/jan/blob/main/core/README.md).
|
||||
|
||||
So, what are you waiting for? Go ahead and start customizing your extension!
|
||||
@ -1,37 +0,0 @@
|
||||
{
|
||||
"name": "@janhq/model-extension",
|
||||
"productName": "Model Management",
|
||||
"version": "1.0.36",
|
||||
"description": "Manages model operations including listing, importing, updating, and deleting.",
|
||||
"main": "dist/index.js",
|
||||
"author": "Jan <service@jan.ai>",
|
||||
"license": "AGPL-3.0",
|
||||
"scripts": {
|
||||
"test": "vitest run",
|
||||
"build": "rolldown -c rolldown.config.mjs",
|
||||
"build:publish": "rimraf *.tgz --glob || true && yarn build && npm pack && cpx *.tgz ../../pre-install"
|
||||
},
|
||||
"devDependencies": {
|
||||
"cpx": "^1.5.0",
|
||||
"rimraf": "^3.0.2",
|
||||
"rolldown": "1.0.0-beta.1",
|
||||
"run-script-os": "^1.1.6",
|
||||
"typescript": "5.3.3",
|
||||
"vitest": "^3.0.6"
|
||||
},
|
||||
"files": [
|
||||
"dist/*",
|
||||
"package.json",
|
||||
"README.md"
|
||||
],
|
||||
"dependencies": {
|
||||
"@janhq/core": "../../core/package.tgz",
|
||||
"ky": "^1.7.2",
|
||||
"p-queue": "^8.0.1"
|
||||
},
|
||||
"bundleDependencies": [],
|
||||
"installConfig": {
|
||||
"hoistingLimits": "workspaces"
|
||||
},
|
||||
"packageManager": "yarn@4.5.3"
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,14 +0,0 @@
|
||||
[
|
||||
{
|
||||
"key": "hugging-face-access-token",
|
||||
"title": "Hugging Face Access Token",
|
||||
"description": "Access tokens programmatically authenticate your identity to the Hugging Face Hub, allowing applications to perform specific actions specified by the scope of permissions granted.",
|
||||
"controllerType": "input",
|
||||
"controllerProps": {
|
||||
"value": "",
|
||||
"placeholder": "hf_**********************************",
|
||||
"type": "password",
|
||||
"inputActions": ["unobscure", "copy"]
|
||||
}
|
||||
}
|
||||
]
|
||||
@ -1,17 +0,0 @@
|
||||
import { defineConfig } from 'rolldown'
|
||||
import settingJson from './resources/settings.json' with { type: 'json' }
|
||||
import modelSources from './resources/default.json' with { type: 'json' }
|
||||
|
||||
export default defineConfig({
|
||||
input: 'src/index.ts',
|
||||
output: {
|
||||
format: 'esm',
|
||||
file: 'dist/index.js',
|
||||
},
|
||||
platform: 'browser',
|
||||
define: {
|
||||
SETTINGS: JSON.stringify(settingJson),
|
||||
CORTEX_API_URL: JSON.stringify(`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? "39291"}`),
|
||||
DEFAULT_MODEL_SOURCES: JSON.stringify(modelSources),
|
||||
},
|
||||
})
|
||||
@ -1,13 +0,0 @@
|
||||
declare const NODE: string
|
||||
declare const CORTEX_API_URL: string
|
||||
declare const SETTINGS: SettingComponentProps[]
|
||||
declare const DEFAULT_MODEL_SOURCES: any
|
||||
|
||||
interface Core {
|
||||
api: APIFunctions
|
||||
events: EventEmitter
|
||||
}
|
||||
interface Window {
|
||||
core?: Core | undefined
|
||||
electronAPI?: any | undefined
|
||||
}
|
||||
@ -1,88 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, vi } from 'vitest'
|
||||
import JanModelExtension from './index'
|
||||
import ky from 'ky'
|
||||
import { ModelManager } from '@janhq/core'
|
||||
|
||||
const API_URL = 'http://localhost:3000'
|
||||
|
||||
vi.stubGlobal('API_URL', API_URL)
|
||||
|
||||
describe('JanModelExtension', () => {
|
||||
let extension: JanModelExtension
|
||||
|
||||
beforeEach(() => {
|
||||
extension = new JanModelExtension()
|
||||
vi.spyOn(ModelManager, 'instance').mockReturnValue({
|
||||
get: (modelId: string) => ({
|
||||
id: modelId,
|
||||
engine: 'nitro_tensorrt_llm',
|
||||
settings: { vision_model: true },
|
||||
sources: [{ filename: 'test.bin' }],
|
||||
}),
|
||||
} as any)
|
||||
vi.spyOn(JanModelExtension.prototype, 'cancelModelPull').mockImplementation(
|
||||
async (model: string) => {
|
||||
const kyDeleteSpy = vi.spyOn(ky, 'delete').mockResolvedValue({
|
||||
json: () => Promise.resolve({}),
|
||||
} as any)
|
||||
|
||||
await ky.delete(`${API_URL}/v1/models/pull`, {
|
||||
json: { taskId: model },
|
||||
})
|
||||
|
||||
expect(kyDeleteSpy).toHaveBeenCalledWith(`${API_URL}/v1/models/pull`, {
|
||||
json: { taskId: model },
|
||||
})
|
||||
|
||||
kyDeleteSpy.mockRestore() // Restore the original implementation
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
it('should initialize with an empty queue', () => {
|
||||
expect(extension.queue.size).toBe(0)
|
||||
})
|
||||
|
||||
describe('pullModel', () => {
|
||||
it('should call the pull model endpoint with correct parameters', async () => {
|
||||
const model = 'test-model'
|
||||
const id = 'test-id'
|
||||
const name = 'test-name'
|
||||
|
||||
const kyPostSpy = vi.spyOn(ky, 'post').mockReturnValue({
|
||||
json: () => Promise.resolve({}),
|
||||
} as any)
|
||||
|
||||
await extension.pullModel(model, id, name)
|
||||
|
||||
expect(kyPostSpy).toHaveBeenCalledWith(`${API_URL}/v1/models/pull`, {
|
||||
json: { model, id, name },
|
||||
})
|
||||
|
||||
kyPostSpy.mockRestore() // Restore the original implementation
|
||||
})
|
||||
})
|
||||
|
||||
describe('cancelModelPull', () => {
|
||||
it('should call the cancel model pull endpoint with the correct model', async () => {
|
||||
const model = 'test-model'
|
||||
|
||||
await extension.cancelModelPull(model)
|
||||
})
|
||||
})
|
||||
|
||||
describe('deleteModel', () => {
|
||||
it('should call the delete model endpoint with the correct model', async () => {
|
||||
const model = 'test-model'
|
||||
const kyDeleteSpy = vi
|
||||
.spyOn(ky, 'delete')
|
||||
.mockResolvedValue({ json: () => Promise.resolve({}) } as any)
|
||||
|
||||
await extension.deleteModel(model)
|
||||
|
||||
expect(kyDeleteSpy).toHaveBeenCalledWith(`${API_URL}/v1/models/${model}`)
|
||||
|
||||
kyDeleteSpy.mockRestore() // Restore the original implementation
|
||||
})
|
||||
})
|
||||
})
|
||||
@ -1,436 +0,0 @@
|
||||
import {
|
||||
ModelExtension,
|
||||
Model,
|
||||
joinPath,
|
||||
dirName,
|
||||
fs,
|
||||
OptionType,
|
||||
ModelSource,
|
||||
extractInferenceParams,
|
||||
extractModelLoadParams,
|
||||
} from '@janhq/core'
|
||||
import { scanModelsFolder } from './legacy/model-json'
|
||||
import { deleteModelFiles } from './legacy/delete'
|
||||
import ky, { KyInstance } from 'ky'
|
||||
|
||||
/**
|
||||
* cortex.cpp setting keys
|
||||
*/
|
||||
export enum Settings {
|
||||
huggingfaceToken = 'hugging-face-access-token',
|
||||
}
|
||||
|
||||
/** Data List Response Type */
|
||||
type Data<T> = {
|
||||
data: T[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Defaul mode sources
|
||||
*/
|
||||
const defaultModelSources = ['Menlo/Jan-nano-gguf', 'Menlo/Jan-nano-128k-gguf']
|
||||
|
||||
/**
|
||||
* A extension for models
|
||||
*/
|
||||
export default class JanModelExtension extends ModelExtension {
|
||||
api?: KyInstance
|
||||
/**
|
||||
* Get the API instance
|
||||
* @returns
|
||||
*/
|
||||
async apiInstance(): Promise<KyInstance> {
|
||||
if (this.api) return this.api
|
||||
const apiKey = await window.core?.api.appToken()
|
||||
this.api = ky.extend({
|
||||
prefixUrl: CORTEX_API_URL,
|
||||
headers: apiKey
|
||||
? {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
}
|
||||
: {},
|
||||
retry: 10,
|
||||
})
|
||||
return this.api
|
||||
}
|
||||
/**
|
||||
* Called when the extension is loaded.
|
||||
*/
|
||||
async onLoad() {
|
||||
this.registerSettings(SETTINGS)
|
||||
|
||||
// Configure huggingface token if available
|
||||
const huggingfaceToken = await this.getSetting<string>(
|
||||
Settings.huggingfaceToken,
|
||||
undefined
|
||||
)
|
||||
if (huggingfaceToken) {
|
||||
this.updateCortexConfig({ huggingface_token: huggingfaceToken })
|
||||
}
|
||||
|
||||
// Sync with cortexsohub
|
||||
this.fetchModelsHub()
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscribe to settings update and make change accordingly
|
||||
* @param key
|
||||
* @param value
|
||||
*/
|
||||
onSettingUpdate<T>(key: string, value: T): void {
|
||||
if (key === Settings.huggingfaceToken) {
|
||||
this.updateCortexConfig({ huggingface_token: value })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Called when the extension is unloaded.
|
||||
* @override
|
||||
*/
|
||||
async onUnload() { }
|
||||
|
||||
// BEGIN: - Public API
|
||||
/**
|
||||
* Downloads a machine learning model.
|
||||
* @param model - The model to download.
|
||||
* @returns A Promise that resolves when the model is downloaded.
|
||||
*/
|
||||
async pullModel(model: string, id?: string, name?: string): Promise<void> {
|
||||
/**
|
||||
* Sending POST to /models/pull/{id} endpoint to pull the model
|
||||
*/
|
||||
return this.apiInstance().then((api) =>
|
||||
api
|
||||
.post('v1/models/pull', { json: { model, id, name }, timeout: false })
|
||||
.json()
|
||||
.catch(async (e) => {
|
||||
throw (await e.response?.json()) ?? e
|
||||
})
|
||||
.then()
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancels the download of a specific machine learning model.
|
||||
*
|
||||
* @param {string} model - The ID of the model whose download is to be cancelled.
|
||||
* @returns {Promise<void>} A promise that resolves when the download has been cancelled.
|
||||
*/
|
||||
async cancelModelPull(model: string): Promise<void> {
|
||||
/**
|
||||
* Sending DELETE to /models/pull/{id} endpoint to cancel a model pull
|
||||
*/
|
||||
return this.apiInstance().then((api) =>
|
||||
api
|
||||
.delete('v1/models/pull', { json: { taskId: model } })
|
||||
.json()
|
||||
.then()
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes a pulled model
|
||||
* @param model - The model to delete
|
||||
* @returns A Promise that resolves when the model is deleted.
|
||||
*/
|
||||
async deleteModel(model: string): Promise<void> {
|
||||
return this.apiInstance()
|
||||
.then((api) => api.delete(`v1/models/${model}`).json().then())
|
||||
.catch((e) => console.debug(e))
|
||||
.finally(async () => {
|
||||
// Delete legacy model files
|
||||
await deleteModelFiles(model).catch((e) => console.debug(e))
|
||||
}) as Promise<void>
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets all pulled models
|
||||
* @returns A Promise that resolves with an array of all models.
|
||||
*/
|
||||
async getModels(): Promise<Model[]> {
|
||||
/**
|
||||
* Legacy models should be supported
|
||||
*/
|
||||
let legacyModels = await scanModelsFolder()
|
||||
|
||||
/**
|
||||
* Here we are filtering out the models that are not imported
|
||||
* and are not using llama.cpp engine
|
||||
*/
|
||||
var toImportModels = legacyModels.filter((e) => e.engine === 'nitro')
|
||||
|
||||
/**
|
||||
* Fetch models from cortex.cpp
|
||||
*/
|
||||
var fetchedModels = await this.fetchModels().catch(() => [])
|
||||
|
||||
// Checking if there are models to import
|
||||
const existingIds = fetchedModels.map((e) => e.id)
|
||||
toImportModels = toImportModels.filter(
|
||||
(e: Model) => !existingIds.includes(e.id) && !e.settings?.vision_model
|
||||
)
|
||||
|
||||
/**
|
||||
* There is no model to import
|
||||
* just return fetched models
|
||||
*/
|
||||
if (!toImportModels.length)
|
||||
return fetchedModels.concat(
|
||||
legacyModels.filter((e) => !fetchedModels.some((x) => x.id === e.id))
|
||||
)
|
||||
|
||||
console.log('To import models:', toImportModels.length)
|
||||
/**
|
||||
* There are models to import
|
||||
*/
|
||||
if (toImportModels.length > 0) {
|
||||
// Import models
|
||||
await Promise.all(
|
||||
toImportModels.map(async (model: Model & { file_path: string }) => {
|
||||
return this.importModel(
|
||||
model.id,
|
||||
model.sources?.[0]?.url.startsWith('http') ||
|
||||
!(await fs.existsSync(model.sources?.[0]?.url))
|
||||
? await joinPath([
|
||||
await dirName(model.file_path),
|
||||
model.sources?.[0]?.filename ??
|
||||
model.settings?.llama_model_path ??
|
||||
model.sources?.[0]?.url.split('/').pop() ??
|
||||
model.id,
|
||||
]) // Copied models
|
||||
: model.sources?.[0]?.url, // Symlink models,
|
||||
model.name
|
||||
)
|
||||
.then((e) => {
|
||||
this.updateModel({
|
||||
id: model.id,
|
||||
...model.settings,
|
||||
...model.parameters,
|
||||
} as Partial<Model>)
|
||||
})
|
||||
.catch((e) => {
|
||||
console.debug(e)
|
||||
})
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Models are imported successfully before
|
||||
* Now return models from cortex.cpp and merge with legacy models which are not imported
|
||||
*/
|
||||
return await this.fetchModels()
|
||||
.then((models) => {
|
||||
return models.concat(
|
||||
legacyModels.filter((e) => !models.some((x) => x.id === e.id))
|
||||
)
|
||||
})
|
||||
.catch(() => Promise.resolve(legacyModels))
|
||||
}
|
||||
|
||||
/**
|
||||
* Update a pulled model metadata
|
||||
* @param model - The metadata of the model
|
||||
*/
|
||||
async updateModel(model: Partial<Model>): Promise<Model> {
|
||||
return this.apiInstance()
|
||||
.then((api) =>
|
||||
api
|
||||
.patch(`v1/models/${model.id}`, {
|
||||
json: { ...model },
|
||||
timeout: false,
|
||||
})
|
||||
.json()
|
||||
.then()
|
||||
)
|
||||
.then(() => this.getModel(model.id))
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a model by its ID
|
||||
* @param model - The ID of the model
|
||||
*/
|
||||
async getModel(model: string): Promise<Model> {
|
||||
return this.apiInstance().then((api) =>
|
||||
api
|
||||
.get(`v1/models/${model}`)
|
||||
.json()
|
||||
.then((e) => this.transformModel(e))
|
||||
) as Promise<Model>
|
||||
}
|
||||
|
||||
/**
|
||||
* Import an existing model file
|
||||
* @param model
|
||||
* @param optionType
|
||||
*/
|
||||
async importModel(
|
||||
model: string,
|
||||
modelPath: string,
|
||||
name?: string,
|
||||
option?: OptionType
|
||||
): Promise<void> {
|
||||
return this.apiInstance().then((api) =>
|
||||
api
|
||||
.post('v1/models/import', {
|
||||
json: { model, modelPath, name, option },
|
||||
timeout: false,
|
||||
})
|
||||
.json()
|
||||
.catch((e) => console.debug(e)) // Ignore error
|
||||
.then()
|
||||
)
|
||||
}
|
||||
|
||||
// BEGIN - Model Sources
|
||||
/**
|
||||
* Get model sources
|
||||
* @param model
|
||||
*/
|
||||
async getSources(): Promise<ModelSource[]> {
|
||||
return []
|
||||
const sources = await this.apiInstance()
|
||||
.then((api) => api.get('v1/models/sources').json<Data<ModelSource>>())
|
||||
.then((e) => (typeof e === 'object' ? (e.data as ModelSource[]) : []))
|
||||
// Deprecated source - filter out from legacy sources
|
||||
.then((e) => e.filter((x) => x.id.toLowerCase() !== 'menlo/jan-nano'))
|
||||
.catch(() => [])
|
||||
return sources.concat(
|
||||
DEFAULT_MODEL_SOURCES.filter((e) => !sources.some((x) => x.id === e.id))
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a model source
|
||||
* @param model
|
||||
*/
|
||||
async addSource(source: string): Promise<any> {
|
||||
return
|
||||
return this.apiInstance().then((api) =>
|
||||
api.post('v1/models/sources', {
|
||||
json: {
|
||||
source,
|
||||
},
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a model source
|
||||
* @param model
|
||||
*/
|
||||
async deleteSource(source: string): Promise<any> {
|
||||
return this.apiInstance().then((api) =>
|
||||
api.delete('v1/models/sources', {
|
||||
json: {
|
||||
source,
|
||||
},
|
||||
timeout: false,
|
||||
})
|
||||
)
|
||||
}
|
||||
// END - Model Sources
|
||||
|
||||
/**
|
||||
* Check model status
|
||||
* @param model
|
||||
*/
|
||||
async isModelLoaded(model: string): Promise<boolean> {
|
||||
return this.apiInstance()
|
||||
.then((api) => api.get(`v1/models/status/${model}`))
|
||||
.then((e) => true)
|
||||
.catch(() => false)
|
||||
}
|
||||
|
||||
/**
|
||||
* Configure pull options such as proxy, headers, etc.
|
||||
*/
|
||||
async configurePullOptions(options: { [key: string]: any }): Promise<any> {
|
||||
return this.updateCortexConfig(options).catch((e) => console.debug(e))
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches models list from cortex.cpp
|
||||
* @param model
|
||||
* @returns
|
||||
*/
|
||||
async fetchModels(): Promise<Model[]> {
|
||||
return []
|
||||
return this.apiInstance()
|
||||
.then((api) => api.get('v1/models?limit=-1').json<Data<Model>>())
|
||||
.then((e) =>
|
||||
typeof e === 'object' ? e.data.map((e) => this.transformModel(e)) : []
|
||||
)
|
||||
}
|
||||
// END: - Public API
|
||||
|
||||
// BEGIN: - Private API
|
||||
|
||||
/**
|
||||
* Transform model to the expected format (e.g. parameters, settings, metadata)
|
||||
* @param model
|
||||
* @returns
|
||||
*/
|
||||
private transformModel(model: any) {
|
||||
model.parameters = {
|
||||
...extractInferenceParams(model),
|
||||
...model.parameters,
|
||||
...model.inference_params,
|
||||
}
|
||||
model.settings = {
|
||||
...extractModelLoadParams(model),
|
||||
...model.settings,
|
||||
}
|
||||
model.metadata = model.metadata ?? {
|
||||
tags: [],
|
||||
size: model.size ?? model.metadata?.size ?? 0,
|
||||
}
|
||||
return model as Model
|
||||
}
|
||||
|
||||
/**
|
||||
* Update cortex config
|
||||
* @param body
|
||||
*/
|
||||
private async updateCortexConfig(body: {
|
||||
[key: string]: any
|
||||
}): Promise<void> {
|
||||
return this.apiInstance()
|
||||
.then((api) => api.patch('v1/configs', { json: body }).then(() => { }))
|
||||
.catch((e) => console.debug(e))
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch models from cortex.so
|
||||
*/
|
||||
fetchModelsHub = async () => {
|
||||
return
|
||||
const models = await this.fetchModels()
|
||||
|
||||
defaultModelSources.forEach((model) => {
|
||||
this.addSource(model).catch((e) => {
|
||||
console.debug(`Failed to add default model source ${model}:`, e)
|
||||
})
|
||||
})
|
||||
return this.apiInstance()
|
||||
.then((api) =>
|
||||
api
|
||||
.get('v1/models/hub?author=cortexso&tag=cortex.cpp')
|
||||
.json<Data<string>>()
|
||||
.then(async (e) => {
|
||||
await Promise.all(
|
||||
[...(e.data ?? []), ...defaultModelSources].map((model) => {
|
||||
if (
|
||||
!models.some(
|
||||
(e) => 'modelSource' in e && e.modelSource === model
|
||||
)
|
||||
)
|
||||
return this.addSource(model).catch((e) => console.debug(e))
|
||||
})
|
||||
)
|
||||
})
|
||||
)
|
||||
.catch((e) => console.debug(e))
|
||||
}
|
||||
// END: - Private API
|
||||
}
|
||||
@ -1,13 +0,0 @@
|
||||
import { dirName, fs } from '@janhq/core'
|
||||
import { scanModelsFolder } from './model-json'
|
||||
|
||||
export const deleteModelFiles = async (id: string) => {
|
||||
try {
|
||||
const models = await scanModelsFolder()
|
||||
const dirPath = models.find((e) => e.id === id)?.file_path
|
||||
// remove model folder directory
|
||||
if (dirPath) await fs.rm(await dirName(dirPath))
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
}
|
||||
}
|
||||
@ -1,89 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, vi } from 'vitest'
|
||||
import { scanModelsFolder, getModelJsonPath } from './model-json'
|
||||
|
||||
// Mock the @janhq/core module
|
||||
vi.mock('@janhq/core', () => ({
|
||||
InferenceEngine: {
|
||||
nitro: 'nitro',
|
||||
},
|
||||
fs: {
|
||||
existsSync: vi.fn(),
|
||||
readdirSync: vi.fn(),
|
||||
fileStat: vi.fn(),
|
||||
readFileSync: vi.fn(),
|
||||
},
|
||||
joinPath: vi.fn((paths) => paths.join('/')),
|
||||
}))
|
||||
|
||||
// Import the mocked fs and joinPath after the mock is set up
|
||||
import { fs } from '@janhq/core'
|
||||
|
||||
describe('model-json', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
describe('scanModelsFolder', () => {
|
||||
it('should return an empty array when models folder does not exist', async () => {
|
||||
vi.spyOn(fs, 'existsSync').mockReturnValue(false)
|
||||
|
||||
const result = await scanModelsFolder()
|
||||
expect(result).toEqual([])
|
||||
})
|
||||
|
||||
it('should return an array of models when valid model folders exist', async () => {
|
||||
const mockModelJson = {
|
||||
id: 'test-model',
|
||||
sources: [
|
||||
{
|
||||
filename: 'test-model',
|
||||
url: 'file://models/test-model/test-model.gguf',
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
vi.spyOn(fs, 'existsSync').mockReturnValue(true)
|
||||
vi.spyOn(fs, 'readdirSync').mockReturnValueOnce(['test-model'])
|
||||
vi.spyOn(fs, 'fileStat').mockResolvedValue({ isDirectory: () => true })
|
||||
vi.spyOn(fs, 'readFileSync').mockReturnValue(
|
||||
JSON.stringify(mockModelJson)
|
||||
)
|
||||
vi.spyOn(fs, 'readdirSync').mockReturnValueOnce([
|
||||
'test-model.gguf',
|
||||
'model.json',
|
||||
])
|
||||
|
||||
const result = await scanModelsFolder()
|
||||
expect(result).toHaveLength(1)
|
||||
expect(result[0]).toMatchObject(mockModelJson)
|
||||
})
|
||||
})
|
||||
|
||||
describe('getModelJsonPath', () => {
|
||||
it('should return undefined when folder does not exist', async () => {
|
||||
vi.spyOn(fs, 'existsSync').mockReturnValue(false)
|
||||
|
||||
const result = await getModelJsonPath('non-existent-folder')
|
||||
expect(result).toBeUndefined()
|
||||
})
|
||||
|
||||
it('should return the path when model.json exists in the root folder', async () => {
|
||||
vi.spyOn(fs, 'existsSync').mockReturnValue(true)
|
||||
vi.spyOn(fs, 'readdirSync').mockReturnValue(['model.json'])
|
||||
|
||||
const result = await getModelJsonPath('test-folder')
|
||||
expect(result).toBe('test-folder/model.json')
|
||||
})
|
||||
|
||||
it('should return the path when model.json exists in a subfolder', async () => {
|
||||
vi.spyOn(fs, 'existsSync').mockReturnValue(true)
|
||||
vi.spyOn(fs, 'readdirSync')
|
||||
.mockReturnValueOnce(['subfolder'])
|
||||
.mockReturnValueOnce(['model.json'])
|
||||
vi.spyOn(fs, 'fileStat').mockResolvedValue({ isDirectory: () => true })
|
||||
|
||||
const result = await getModelJsonPath('test-folder')
|
||||
expect(result).toBe('test-folder/subfolder/model.json')
|
||||
})
|
||||
})
|
||||
})
|
||||
@ -1,141 +0,0 @@
|
||||
import { Model, fs, joinPath } from '@janhq/core'
|
||||
//// LEGACY MODEL FOLDER ////
|
||||
/**
|
||||
* Scan through models folder and return downloaded models
|
||||
* @returns
|
||||
*/
|
||||
export const scanModelsFolder = async (): Promise<
|
||||
(Model & { file_path?: string })[]
|
||||
> => {
|
||||
const _homeDir = 'file://models'
|
||||
try {
|
||||
if (!(await fs.existsSync(_homeDir))) {
|
||||
console.debug('Model folder not found')
|
||||
return []
|
||||
}
|
||||
|
||||
const files: string[] = await fs.readdirSync(_homeDir)
|
||||
|
||||
const allDirectories: string[] = []
|
||||
|
||||
for (const modelFolder of files) {
|
||||
const fullModelFolderPath = await joinPath([_homeDir, modelFolder])
|
||||
if (!(await fs.fileStat(fullModelFolderPath)).isDirectory) continue
|
||||
allDirectories.push(modelFolder)
|
||||
}
|
||||
|
||||
const readJsonPromises = allDirectories.map(async (dirName) => {
|
||||
// filter out directories that don't match the selector
|
||||
// read model.json
|
||||
const folderFullPath = await joinPath([_homeDir, dirName])
|
||||
|
||||
const jsonPath = await getModelJsonPath(folderFullPath)
|
||||
|
||||
if (jsonPath && (await fs.existsSync(jsonPath))) {
|
||||
// if we have the model.json file, read it
|
||||
let model = await fs.readFileSync(jsonPath, 'utf-8')
|
||||
|
||||
model = typeof model === 'object' ? model : JSON.parse(model)
|
||||
|
||||
// This to ensure backward compatibility with `model.json` with `source_url`
|
||||
if (model['source_url'] != null) {
|
||||
model['sources'] = [
|
||||
{
|
||||
filename: model.id,
|
||||
url: model['source_url'],
|
||||
},
|
||||
]
|
||||
}
|
||||
model.file_path = jsonPath
|
||||
model.file_name = 'model.json'
|
||||
|
||||
// Check model file exist
|
||||
// model binaries (sources) are absolute path & exist (symlinked)
|
||||
const existFiles = await Promise.all(
|
||||
model.sources.map(
|
||||
(source) =>
|
||||
// Supposed to be a local file url
|
||||
!source.url.startsWith(`http://`) &&
|
||||
!source.url.startsWith(`https://`)
|
||||
)
|
||||
)
|
||||
if (
|
||||
!['cortex', 'llama-cpp', 'nitro'].includes(model.engine) ||
|
||||
existFiles.every((exist) => exist)
|
||||
)
|
||||
return model
|
||||
|
||||
const result = await fs
|
||||
.readdirSync(await joinPath([_homeDir, dirName]))
|
||||
.then((files: string[]) => {
|
||||
// Model binary exists in the directory
|
||||
// Model binary name can match model ID or be a .gguf file and not be an incompleted model file
|
||||
return (
|
||||
files.includes(dirName) || // Legacy model GGUF without extension
|
||||
files.filter((file) => {
|
||||
return (
|
||||
file.toLowerCase().endsWith('.gguf') || // GGUF
|
||||
file.toLowerCase().endsWith('.engine') // Tensort-LLM
|
||||
)
|
||||
})?.length >=
|
||||
(model.engine === 'nitro-tensorrt-llm'
|
||||
? 1
|
||||
: model.sources?.length ?? 1)
|
||||
)
|
||||
})
|
||||
|
||||
if (result) return model
|
||||
else return undefined
|
||||
}
|
||||
})
|
||||
const results = await Promise.allSettled(readJsonPromises)
|
||||
const modelData = results
|
||||
.map((result) => {
|
||||
if (result.status === 'fulfilled' && result.value) {
|
||||
try {
|
||||
const model =
|
||||
typeof result.value === 'object'
|
||||
? result.value
|
||||
: JSON.parse(result.value)
|
||||
return model as Model
|
||||
} catch {
|
||||
console.debug(`Unable to parse model metadata: ${result.value}`)
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
})
|
||||
.filter(Boolean)
|
||||
|
||||
return modelData
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve the model.json path from a folder
|
||||
* @param folderFullPath
|
||||
* @returns
|
||||
*/
|
||||
export const getModelJsonPath = async (
|
||||
folderFullPath: string
|
||||
): Promise<string | undefined> => {
|
||||
// try to find model.json recursively inside each folder
|
||||
if (!(await fs.existsSync(folderFullPath))) return undefined
|
||||
const files: string[] = await fs.readdirSync(folderFullPath)
|
||||
if (files.length === 0) return undefined
|
||||
if (files.includes('model.json')) {
|
||||
return joinPath([folderFullPath, 'model.json'])
|
||||
}
|
||||
// continue recursive
|
||||
for (const file of files) {
|
||||
const path = await joinPath([folderFullPath, file])
|
||||
const fileStats = await fs.fileStat(path)
|
||||
if (fileStats.isDirectory) {
|
||||
const result = await getModelJsonPath(path)
|
||||
if (result) return result
|
||||
}
|
||||
}
|
||||
}
|
||||
//// END LEGACY MODEL FOLDER ////
|
||||
@ -1,160 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, vi } from 'vitest'
|
||||
|
||||
vi.stubGlobal('API_URL', 'http://localhost:3000')
|
||||
|
||||
|
||||
// Mock the @janhq/core module
|
||||
vi.mock('@janhq/core', (actual) => ({
|
||||
...actual,
|
||||
ModelExtension: class {},
|
||||
InferenceEngine: {
|
||||
nitro: 'nitro',
|
||||
},
|
||||
joinPath: vi.fn(),
|
||||
dirName: vi.fn(),
|
||||
fs: {
|
||||
existsSync: vi.fn(),
|
||||
readFileSync: vi.fn(),
|
||||
writeFileSync: vi.fn(),
|
||||
mkdirSync: vi.fn(),
|
||||
},
|
||||
}))
|
||||
|
||||
import { Model, InferenceEngine } from '@janhq/core'
|
||||
|
||||
import JanModelExtension from './index'
|
||||
|
||||
// Mock the model-json module
|
||||
vi.mock('./legacy/model-json', () => ({
|
||||
scanModelsFolder: vi.fn(),
|
||||
}))
|
||||
|
||||
// Import the mocked scanModelsFolder after the mock is set up
|
||||
import * as legacy from './legacy/model-json'
|
||||
|
||||
describe('JanModelExtension', () => {
|
||||
let extension: JanModelExtension
|
||||
let mockLocalStorage: { [key: string]: string }
|
||||
|
||||
beforeEach(() => {
|
||||
// @ts-ignore
|
||||
extension = new JanModelExtension()
|
||||
mockLocalStorage = {}
|
||||
|
||||
// Mock localStorage
|
||||
Object.defineProperty(global, 'localStorage', {
|
||||
value: {
|
||||
getItem: vi.fn((key) => mockLocalStorage[key]),
|
||||
setItem: vi.fn((key, value) => {
|
||||
mockLocalStorage[key] = value
|
||||
}),
|
||||
},
|
||||
writable: true,
|
||||
})
|
||||
})
|
||||
|
||||
describe('getModels', () => {
|
||||
it('should scan models folder when localStorage is empty', async () => {
|
||||
const mockModels: Model[] = [
|
||||
{
|
||||
id: 'model1',
|
||||
object: 'model',
|
||||
version: '1',
|
||||
format: 'gguf',
|
||||
engine: InferenceEngine.nitro,
|
||||
sources: [
|
||||
{ filename: 'model1.gguf', url: 'file://models/model1.gguf' },
|
||||
],
|
||||
file_path: '/path/to/model1',
|
||||
},
|
||||
{
|
||||
id: 'model2',
|
||||
object: 'model',
|
||||
version: '1',
|
||||
format: 'gguf',
|
||||
engine: InferenceEngine.nitro,
|
||||
sources: [
|
||||
{ filename: 'model2.gguf', url: 'file://models/model2.gguf' },
|
||||
],
|
||||
file_path: '/path/to/model2',
|
||||
},
|
||||
] as any
|
||||
vi.mocked(legacy.scanModelsFolder).mockResolvedValue(mockModels)
|
||||
vi.spyOn(extension, 'fetchModels').mockResolvedValue([mockModels[0]])
|
||||
vi.spyOn(extension, 'updateModel').mockResolvedValue(undefined)
|
||||
vi.spyOn(extension, 'importModel').mockResolvedValueOnce(mockModels[1])
|
||||
vi.spyOn(extension, 'fetchModels').mockResolvedValue([mockModels[0], mockModels[1]])
|
||||
const result = await extension.getModels()
|
||||
expect(legacy.scanModelsFolder).toHaveBeenCalled()
|
||||
expect(result).toEqual(mockModels)
|
||||
})
|
||||
|
||||
it('should import models when there are models to import', async () => {
|
||||
const mockModels: Model[] = [
|
||||
{
|
||||
id: 'model1',
|
||||
object: 'model',
|
||||
version: '1',
|
||||
format: 'gguf',
|
||||
engine: InferenceEngine.nitro,
|
||||
file_path: '/path/to/model1',
|
||||
sources: [
|
||||
{ filename: 'model1.gguf', url: 'file://models/model1.gguf' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'model2',
|
||||
object: 'model',
|
||||
version: '1',
|
||||
format: 'gguf',
|
||||
engine: InferenceEngine.nitro,
|
||||
file_path: '/path/to/model2',
|
||||
sources: [
|
||||
{ filename: 'model2.gguf', url: 'file://models/model2.gguf' },
|
||||
],
|
||||
},
|
||||
] as any
|
||||
mockLocalStorage['downloadedModels'] = JSON.stringify(mockModels)
|
||||
vi.spyOn(extension, 'updateModel').mockResolvedValue(undefined)
|
||||
vi.spyOn(extension, 'importModel').mockResolvedValue(undefined)
|
||||
|
||||
const result = await extension.getModels()
|
||||
|
||||
expect(extension.importModel).toHaveBeenCalledTimes(2)
|
||||
expect(result).toEqual(mockModels)
|
||||
})
|
||||
|
||||
it('should return models from cortexAPI when all models are already imported', async () => {
|
||||
const mockModels: Model[] = [
|
||||
{
|
||||
id: 'model1',
|
||||
object: 'model',
|
||||
version: '1',
|
||||
format: 'gguf',
|
||||
engine: InferenceEngine.nitro,
|
||||
sources: [
|
||||
{ filename: 'model1.gguf', url: 'file://models/model1.gguf' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'model2',
|
||||
object: 'model',
|
||||
version: '1',
|
||||
format: 'gguf',
|
||||
engine: InferenceEngine.nitro,
|
||||
sources: [
|
||||
{ filename: 'model2.gguf', url: 'file://models/model2.gguf' },
|
||||
],
|
||||
},
|
||||
] as any
|
||||
mockLocalStorage['downloadedModels'] = JSON.stringify(mockModels)
|
||||
vi.spyOn(extension, 'fetchModels').mockResolvedValue(mockModels)
|
||||
extension.getModels = vi.fn().mockResolvedValue(mockModels)
|
||||
|
||||
const result = await extension.getModels()
|
||||
|
||||
expect(extension.getModels).toHaveBeenCalled()
|
||||
expect(result).toEqual(mockModels)
|
||||
})
|
||||
})
|
||||
})
|
||||
@ -1,15 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "es2016",
|
||||
"module": "esnext",
|
||||
"moduleResolution": "node",
|
||||
"outDir": "./dist",
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"strict": false,
|
||||
"skipLibCheck": true,
|
||||
"rootDir": "./src"
|
||||
},
|
||||
"include": ["./src"],
|
||||
"exclude": ["**/*.test.ts", "vite.config.ts"]
|
||||
}
|
||||
@ -1,8 +0,0 @@
|
||||
import { defineConfig } from "vite"
|
||||
export default defineConfig(({ mode }) => ({
|
||||
define: process.env.VITEST ? {} : { global: 'window' },
|
||||
test: {
|
||||
environment: 'jsdom',
|
||||
},
|
||||
}))
|
||||
|
||||
@ -54,9 +54,7 @@ depends = ["build-extensions"]
|
||||
description = "Start development server (matches Makefile)"
|
||||
depends = ["install-and-build"]
|
||||
run = [
|
||||
"yarn install:cortex",
|
||||
"yarn download:bin",
|
||||
"yarn copy:lib",
|
||||
"yarn dev"
|
||||
]
|
||||
|
||||
@ -64,9 +62,7 @@ run = [
|
||||
description = "Start development server with Tauri (DEPRECATED - matches Makefile)"
|
||||
depends = ["install-and-build"]
|
||||
run = [
|
||||
"yarn install:cortex",
|
||||
"yarn download:bin",
|
||||
"yarn copy:lib",
|
||||
"yarn dev:tauri"
|
||||
]
|
||||
|
||||
@ -83,7 +79,6 @@ run = "yarn build"
|
||||
description = "Build Tauri application (DEPRECATED - matches Makefile)"
|
||||
depends = ["install-and-build"]
|
||||
run = [
|
||||
"yarn copy:lib",
|
||||
"yarn build"
|
||||
]
|
||||
|
||||
|
||||
@ -404,7 +404,7 @@ const ChatInput = ({ model, className, initialMessage }: ChatInputProps) => {
|
||||
streamingContent && 'opacity-50 pointer-events-none'
|
||||
)}
|
||||
>
|
||||
{model?.provider === 'llama.cpp' && loadingModel ? (
|
||||
{model?.provider === 'llamacpp' && loadingModel ? (
|
||||
<ModelLoader />
|
||||
) : (
|
||||
<DropdownModelProvider
|
||||
|
||||
@ -96,15 +96,15 @@ const DropdownModelProvider = ({
|
||||
selectModelProvider(lastUsed.provider, lastUsed.model)
|
||||
} else {
|
||||
// Fallback to default model if last used model no longer exists
|
||||
selectModelProvider('llama.cpp', 'llama3.2:3b')
|
||||
selectModelProvider('llamacpp', 'llama3.2:3b')
|
||||
}
|
||||
} else {
|
||||
// default model, we should add from setting
|
||||
selectModelProvider('llama.cpp', 'llama3.2:3b')
|
||||
selectModelProvider('llamacpp', 'llama3.2:3b')
|
||||
}
|
||||
} else {
|
||||
// default model for non-new-chat contexts
|
||||
selectModelProvider('llama.cpp', 'llama3.2:3b')
|
||||
selectModelProvider('llamacpp', 'llama3.2:3b')
|
||||
}
|
||||
}, [
|
||||
model,
|
||||
@ -150,8 +150,8 @@ const DropdownModelProvider = ({
|
||||
if (!provider.active) return
|
||||
|
||||
provider.models.forEach((modelItem) => {
|
||||
// Skip models that require API key but don't have one (except llama.cpp)
|
||||
if (provider.provider !== 'llama.cpp' && !provider.api_key?.length) {
|
||||
// Skip models that require API key but don't have one (except llamacpp)
|
||||
if (provider.provider !== 'llamacpp' && !provider.api_key?.length) {
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@ -18,6 +18,10 @@ import ProvidersAvatar from '@/containers/ProvidersAvatar'
|
||||
|
||||
const SettingsMenu = () => {
|
||||
const { t } = useTranslation()
|
||||
const { experimentalFeatures } = useGeneralSetting()
|
||||
const { providers } = useModelProvider()
|
||||
const firstItemProvider =
|
||||
providers.length > 0 ? providers[0].provider : 'llamacpp'
|
||||
const [expandedProviders, setExpandedProviders] = useState(false)
|
||||
const [isMenuOpen, setIsMenuOpen] = useState(false)
|
||||
const matches = useMatches()
|
||||
|
||||
@ -17,7 +17,6 @@ import { EngineManager } from '@janhq/core'
|
||||
import { useModelProvider } from '@/hooks/useModelProvider'
|
||||
import { useRouter } from '@tanstack/react-router'
|
||||
import { route } from '@/constants/routes'
|
||||
import { normalizeProvider } from '@/lib/models'
|
||||
import { useTranslation } from '@/i18n/react-i18next-compat'
|
||||
|
||||
type Props = {
|
||||
@ -30,7 +29,7 @@ const DeleteProvider = ({ provider }: Props) => {
|
||||
if (
|
||||
!provider ||
|
||||
Object.keys(models).includes(provider.provider) ||
|
||||
EngineManager.instance().get(normalizeProvider(provider.provider))
|
||||
EngineManager.instance().get(provider.provider)
|
||||
)
|
||||
return null
|
||||
|
||||
|
||||
@ -115,15 +115,11 @@ export const useChat = () => {
|
||||
])
|
||||
|
||||
const restartModel = useCallback(
|
||||
async (
|
||||
provider: ProviderObject,
|
||||
modelId: string,
|
||||
abortController: AbortController
|
||||
) => {
|
||||
async (provider: ProviderObject, modelId: string) => {
|
||||
await stopAllModels()
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000))
|
||||
updateLoadingModel(true)
|
||||
await startModel(provider, modelId, abortController).catch(console.error)
|
||||
await startModel(provider, modelId).catch(console.error)
|
||||
updateLoadingModel(false)
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000))
|
||||
},
|
||||
@ -131,11 +127,7 @@ export const useChat = () => {
|
||||
)
|
||||
|
||||
const increaseModelContextSize = useCallback(
|
||||
async (
|
||||
modelId: string,
|
||||
provider: ProviderObject,
|
||||
controller: AbortController
|
||||
) => {
|
||||
async (modelId: string, provider: ProviderObject) => {
|
||||
/**
|
||||
* Should increase the context size of the model by 2x
|
||||
* If the context size is not set or too low, it defaults to 8192.
|
||||
@ -180,19 +172,14 @@ export const useChat = () => {
|
||||
})
|
||||
}
|
||||
const updatedProvider = getProviderByName(provider.provider)
|
||||
if (updatedProvider)
|
||||
await restartModel(updatedProvider, model.id, controller)
|
||||
if (updatedProvider) await restartModel(updatedProvider, model.id)
|
||||
|
||||
return updatedProvider
|
||||
},
|
||||
[getProviderByName, restartModel, updateProvider]
|
||||
)
|
||||
const toggleOnContextShifting = useCallback(
|
||||
async (
|
||||
modelId: string,
|
||||
provider: ProviderObject,
|
||||
controller: AbortController
|
||||
) => {
|
||||
async (modelId: string, provider: ProviderObject) => {
|
||||
const providerName = provider.provider
|
||||
const newSettings = [...provider.settings]
|
||||
const settingKey = 'context_shift'
|
||||
@ -218,8 +205,7 @@ export const useChat = () => {
|
||||
...updateObj,
|
||||
})
|
||||
const updatedProvider = getProviderByName(providerName)
|
||||
if (updatedProvider)
|
||||
await restartModel(updatedProvider, modelId, controller)
|
||||
if (updatedProvider) await restartModel(updatedProvider, modelId)
|
||||
return updatedProvider
|
||||
},
|
||||
[updateProvider, getProviderByName, restartModel]
|
||||
@ -246,11 +232,9 @@ export const useChat = () => {
|
||||
try {
|
||||
if (selectedModel?.id) {
|
||||
updateLoadingModel(true)
|
||||
await startModel(
|
||||
activeProvider,
|
||||
selectedModel.id,
|
||||
abortController
|
||||
).catch(console.error)
|
||||
await startModel(activeProvider, selectedModel.id).catch(
|
||||
console.error
|
||||
)
|
||||
updateLoadingModel(false)
|
||||
}
|
||||
|
||||
@ -286,10 +270,6 @@ export const useChat = () => {
|
||||
availableTools,
|
||||
currentAssistant.parameters?.stream === false ? false : true,
|
||||
currentAssistant.parameters as unknown as Record<string, object>
|
||||
// TODO: replace it with according provider setting later on
|
||||
// selectedProvider === 'llama.cpp' && availableTools.length > 0
|
||||
// ? false
|
||||
// : true
|
||||
)
|
||||
|
||||
if (!completion) throw new Error('No completion received')
|
||||
@ -298,7 +278,8 @@ export const useChat = () => {
|
||||
const toolCalls: ChatCompletionMessageToolCall[] = []
|
||||
try {
|
||||
if (isCompletionResponse(completion)) {
|
||||
accumulatedText = completion.choices[0]?.message?.content || ''
|
||||
accumulatedText =
|
||||
(completion.choices[0]?.message?.content as string) || ''
|
||||
if (completion.choices[0]?.message?.tool_calls) {
|
||||
toolCalls.push(...completion.choices[0].message.tool_calls)
|
||||
}
|
||||
@ -365,16 +346,14 @@ export const useChat = () => {
|
||||
/// Increase context size
|
||||
activeProvider = await increaseModelContextSize(
|
||||
selectedModel.id,
|
||||
activeProvider,
|
||||
abortController
|
||||
activeProvider
|
||||
)
|
||||
continue
|
||||
} else if (method === 'context_shift' && selectedModel?.id) {
|
||||
/// Enable context_shift
|
||||
activeProvider = await toggleOnContextShifting(
|
||||
selectedModel?.id,
|
||||
activeProvider,
|
||||
abortController
|
||||
activeProvider
|
||||
)
|
||||
continue
|
||||
} else throw error
|
||||
@ -387,7 +366,7 @@ export const useChat = () => {
|
||||
accumulatedText.length === 0 &&
|
||||
toolCalls.length === 0 &&
|
||||
activeThread.model?.id &&
|
||||
activeProvider.provider === 'llama.cpp'
|
||||
provider?.provider === 'llamacpp'
|
||||
) {
|
||||
await stopModel(activeThread.model.id, 'cortex')
|
||||
throw new Error('No response received from the model')
|
||||
|
||||
@ -24,7 +24,7 @@ export const useModelProvider = create<ModelProviderState>()(
|
||||
persist(
|
||||
(set, get) => ({
|
||||
providers: [],
|
||||
selectedProvider: 'llama.cpp',
|
||||
selectedProvider: 'llamacpp',
|
||||
selectedModel: null,
|
||||
deletedModels: [],
|
||||
getModelBy: (modelId: string) => {
|
||||
|
||||
@ -5,6 +5,9 @@ import {
|
||||
MessageStatus,
|
||||
EngineManager,
|
||||
ModelManager,
|
||||
chatCompletionRequestMessage,
|
||||
chatCompletion,
|
||||
chatCompletionChunk,
|
||||
} from '@janhq/core'
|
||||
import { invoke } from '@tauri-apps/api/core'
|
||||
import { fetch as fetchTauri } from '@tauri-apps/plugin-http'
|
||||
@ -24,11 +27,17 @@ type ExtendedConfigOptions = ConfigOptions & {
|
||||
fetch?: typeof fetch
|
||||
}
|
||||
import { ulid } from 'ulidx'
|
||||
import { normalizeProvider } from './models'
|
||||
import { MCPTool } from '@/types/completion'
|
||||
import { CompletionMessagesBuilder } from './messages'
|
||||
import { ChatCompletionMessageToolCall } from 'openai/resources'
|
||||
import { callTool } from '@/services/mcp'
|
||||
import { ExtensionManager } from './extension'
|
||||
|
||||
export type ChatCompletionResponse =
|
||||
| chatCompletion
|
||||
| AsyncIterable<chatCompletionChunk>
|
||||
| StreamCompletionResponse
|
||||
| CompletionResponse
|
||||
|
||||
/**
|
||||
* @fileoverview Helper functions for creating thread content.
|
||||
@ -124,7 +133,7 @@ export const sendCompletion = async (
|
||||
tools: MCPTool[] = [],
|
||||
stream: boolean = true,
|
||||
params: Record<string, object> = {}
|
||||
): Promise<StreamCompletionResponse | CompletionResponse | undefined> => {
|
||||
): Promise<ChatCompletionResponse | undefined> => {
|
||||
if (!thread?.model?.id || !provider) return undefined
|
||||
|
||||
let providerName = provider.provider as unknown as keyof typeof models
|
||||
@ -144,7 +153,7 @@ export const sendCompletion = async (
|
||||
!(thread.model.id in Object.values(models).flat()) &&
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
!tokenJS.extendedModelExist(providerName as any, thread.model?.id) &&
|
||||
provider.provider !== 'llama.cpp'
|
||||
provider.provider !== 'llamacpp'
|
||||
) {
|
||||
try {
|
||||
tokenJS.extendModelList(
|
||||
@ -163,38 +172,48 @@ export const sendCompletion = async (
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Add message history
|
||||
const completion = stream
|
||||
? await tokenJS.chat.completions.create(
|
||||
{
|
||||
stream: true,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
provider: providerName as any,
|
||||
const engine = ExtensionManager.getInstance().getEngine(provider.provider)
|
||||
|
||||
const completion = engine
|
||||
? await engine.chat({
|
||||
messages: messages as chatCompletionRequestMessage[],
|
||||
model: thread.model?.id,
|
||||
tools: normalizeTools(tools),
|
||||
tool_choice: tools.length ? 'auto' : undefined,
|
||||
stream: true,
|
||||
...params,
|
||||
})
|
||||
: stream
|
||||
? await tokenJS.chat.completions.create(
|
||||
{
|
||||
stream: true,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
provider: providerName as any,
|
||||
model: thread.model?.id,
|
||||
messages,
|
||||
tools: normalizeTools(tools),
|
||||
tool_choice: tools.length ? 'auto' : undefined,
|
||||
...params,
|
||||
},
|
||||
{
|
||||
signal: abortController.signal,
|
||||
}
|
||||
)
|
||||
: await tokenJS.chat.completions.create({
|
||||
stream: false,
|
||||
provider: providerName,
|
||||
model: thread.model?.id,
|
||||
messages,
|
||||
tools: normalizeTools(tools),
|
||||
tool_choice: tools.length ? 'auto' : undefined,
|
||||
...params,
|
||||
},
|
||||
{
|
||||
signal: abortController.signal,
|
||||
}
|
||||
)
|
||||
: await tokenJS.chat.completions.create({
|
||||
stream: false,
|
||||
provider: providerName,
|
||||
model: thread.model?.id,
|
||||
messages,
|
||||
tools: normalizeTools(tools),
|
||||
tool_choice: tools.length ? 'auto' : undefined,
|
||||
...params,
|
||||
})
|
||||
})
|
||||
return completion
|
||||
}
|
||||
|
||||
export const isCompletionResponse = (
|
||||
response: StreamCompletionResponse | CompletionResponse
|
||||
): response is CompletionResponse => {
|
||||
response: ChatCompletionResponse
|
||||
): response is CompletionResponse | chatCompletion => {
|
||||
return 'choices' in response
|
||||
}
|
||||
|
||||
@ -209,9 +228,9 @@ export const stopModel = async (
|
||||
provider: string,
|
||||
model: string
|
||||
): Promise<void> => {
|
||||
const providerObj = EngineManager.instance().get(normalizeProvider(provider))
|
||||
const providerObj = EngineManager.instance().get(provider)
|
||||
const modelObj = ModelManager.instance().get(model)
|
||||
if (providerObj && modelObj) return providerObj?.unload(modelObj)
|
||||
if (providerObj && modelObj) return providerObj?.unload(model).then(() => {})
|
||||
}
|
||||
|
||||
/**
|
||||
@ -241,7 +260,7 @@ export const normalizeTools = (
|
||||
* @param calls
|
||||
*/
|
||||
export const extractToolCall = (
|
||||
part: CompletionResponseChunk,
|
||||
part: chatCompletionChunk | CompletionResponseChunk,
|
||||
currentCall: ChatCompletionMessageToolCall | null,
|
||||
calls: ChatCompletionMessageToolCall[]
|
||||
) => {
|
||||
|
||||
@ -1,6 +1,2 @@
|
||||
import { expect, test } from 'vitest'
|
||||
import { normalizeProvider } from './models'
|
||||
|
||||
test('provider name should be normalized', () => {
|
||||
expect(normalizeProvider('llama.cpp')).toBe('cortex')
|
||||
})
|
||||
|
||||
|
||||
@ -58,12 +58,3 @@ export const extractModelName = (model?: string) => {
|
||||
export const extractModelRepo = (model?: string) => {
|
||||
return model?.replace('https://huggingface.co/', '')
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize the provider name to match the format used in the models object
|
||||
* @param provider - The provider name to normalize
|
||||
*/
|
||||
export const normalizeProvider = (provider: string) => {
|
||||
// TODO: After migrating to the new provider extension, remove this function
|
||||
return provider === 'llama.cpp' ? 'cortex' : provider
|
||||
}
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import { type ClassValue, clsx } from 'clsx'
|
||||
import { twMerge } from 'tailwind-merge'
|
||||
import { ExtensionManager } from './extension'
|
||||
|
||||
export function cn(...inputs: ClassValue[]) {
|
||||
return twMerge(clsx(inputs))
|
||||
@ -7,7 +8,7 @@ export function cn(...inputs: ClassValue[]) {
|
||||
|
||||
export function getProviderLogo(provider: string) {
|
||||
switch (provider) {
|
||||
case 'llama.cpp':
|
||||
case 'llamacpp':
|
||||
return '/images/model-provider/llamacpp.svg'
|
||||
case 'anthropic':
|
||||
return '/images/model-provider/anthropic.svg'
|
||||
@ -38,7 +39,7 @@ export function getProviderLogo(provider: string) {
|
||||
|
||||
export const getProviderTitle = (provider: string) => {
|
||||
switch (provider) {
|
||||
case 'llama.cpp':
|
||||
case 'llamacpp':
|
||||
return 'Llama.cpp'
|
||||
case 'openai':
|
||||
return 'OpenAI'
|
||||
@ -89,6 +90,11 @@ export function getReadableLanguageName(language: string): string {
|
||||
)
|
||||
}
|
||||
|
||||
export const isLocalProvider = (provider: string) => {
|
||||
const extension = ExtensionManager.getInstance().getEngine(provider)
|
||||
return extension && 'load' in extension
|
||||
}
|
||||
|
||||
export function fuzzySearch(needle: string, haystack: string) {
|
||||
const hlen = haystack.length
|
||||
const nlen = needle.length
|
||||
|
||||
@ -3,10 +3,8 @@ import { useModelProvider } from '@/hooks/useModelProvider'
|
||||
|
||||
import { useAppUpdater } from '@/hooks/useAppUpdater'
|
||||
import { fetchMessages } from '@/services/messages'
|
||||
import { fetchModels } from '@/services/models'
|
||||
import { getProviders } from '@/services/providers'
|
||||
import { fetchThreads } from '@/services/threads'
|
||||
import { ModelManager } from '@janhq/core'
|
||||
import { useEffect } from 'react'
|
||||
import { useMCPServers } from '@/hooks/useMCPServers'
|
||||
import { getMCPConfig } from '@/services/mcp'
|
||||
@ -31,10 +29,8 @@ export function DataProvider() {
|
||||
const navigate = useNavigate()
|
||||
|
||||
useEffect(() => {
|
||||
fetchModels().then((models) => {
|
||||
models?.forEach((model) => ModelManager.instance().register(model))
|
||||
getProviders().then(setProviders)
|
||||
})
|
||||
console.log('Initializing DataProvider...')
|
||||
getProviders().then(setProviders)
|
||||
getMCPConfig().then((data) => setServers(data.mcpServers ?? []))
|
||||
getAssistants()
|
||||
.then((data) => {
|
||||
|
||||
@ -31,7 +31,7 @@ import {
|
||||
DropdownMenuItem,
|
||||
DropdownMenuTrigger,
|
||||
} from '@/components/ui/dropdown-menu'
|
||||
import { addModelSource, downloadModel, fetchModelHub } from '@/services/models'
|
||||
import { addModelSource, fetchModelHub, pullModel } from '@/services/models'
|
||||
import { useDownloadStore } from '@/hooks/useDownloadStore'
|
||||
import { Progress } from '@/components/ui/progress'
|
||||
import HeaderPage from '@/containers/HeaderPage'
|
||||
@ -83,7 +83,7 @@ function Hub() {
|
||||
const hasTriggeredDownload = useRef(false)
|
||||
|
||||
const { getProviderByName } = useModelProvider()
|
||||
const llamaProvider = getProviderByName('llama.cpp')
|
||||
const llamaProvider = getProviderByName('llamacpp')
|
||||
|
||||
const toggleModelExpansion = (modelId: string) => {
|
||||
setExpandedModels((prev) => ({
|
||||
@ -213,7 +213,7 @@ function Hub() {
|
||||
search: {
|
||||
model: {
|
||||
id: modelId,
|
||||
provider: 'llama.cpp',
|
||||
provider: 'llamacpp',
|
||||
},
|
||||
},
|
||||
})
|
||||
@ -240,7 +240,7 @@ function Hub() {
|
||||
const handleDownload = () => {
|
||||
// Immediately set local downloading state
|
||||
addLocalDownloadingModel(modelId)
|
||||
downloadModel(modelId)
|
||||
pullModel(modelId, modelId)
|
||||
}
|
||||
|
||||
return (
|
||||
@ -650,7 +650,7 @@ function Hub() {
|
||||
addLocalDownloadingModel(
|
||||
variant.id
|
||||
)
|
||||
downloadModel(variant.id)
|
||||
pullModel(variant.id, variant.id)
|
||||
}}
|
||||
>
|
||||
<IconDownload
|
||||
|
||||
@ -37,7 +37,7 @@ function Index() {
|
||||
const hasValidProviders = providers.some(
|
||||
(provider) =>
|
||||
provider.api_key?.length ||
|
||||
(provider.provider === 'llama.cpp' && provider.models.length)
|
||||
(provider.provider === 'llamacpp' && provider.models.length)
|
||||
)
|
||||
|
||||
useEffect(() => {
|
||||
|
||||
@ -6,7 +6,7 @@ import { cn, getProviderTitle } from '@/lib/utils'
|
||||
import { open } from '@tauri-apps/plugin-dialog'
|
||||
import {
|
||||
getActiveModels,
|
||||
importModel,
|
||||
pullModel,
|
||||
startModel,
|
||||
stopAllModels,
|
||||
stopModel,
|
||||
@ -35,7 +35,6 @@ import { Button } from '@/components/ui/button'
|
||||
import { IconFolderPlus, IconLoader, IconRefresh } from '@tabler/icons-react'
|
||||
import { getProviders } from '@/services/providers'
|
||||
import { toast } from 'sonner'
|
||||
import { ActiveModel } from '@/types/models'
|
||||
import { useEffect, useState } from 'react'
|
||||
import { predefinedProviders } from '@/mock/data'
|
||||
|
||||
@ -73,7 +72,7 @@ function ProviderDetail() {
|
||||
},
|
||||
]
|
||||
const { step } = useSearch({ from: Route.id })
|
||||
const [activeModels, setActiveModels] = useState<ActiveModel[]>([])
|
||||
const [activeModels, setActiveModels] = useState<string[]>([])
|
||||
const [loadingModels, setLoadingModels] = useState<string[]>([])
|
||||
const [refreshingModels, setRefreshingModels] = useState(false)
|
||||
const { providerName } = useParams({ from: Route.id })
|
||||
@ -171,10 +170,7 @@ function ProviderDetail() {
|
||||
if (provider)
|
||||
startModel(provider, modelId)
|
||||
.then(() => {
|
||||
setActiveModels((prevModels) => [
|
||||
...prevModels,
|
||||
{ id: modelId } as ActiveModel,
|
||||
])
|
||||
setActiveModels((prevModels) => [...prevModels, modelId])
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error('Error starting model:', error)
|
||||
@ -189,7 +185,7 @@ function ProviderDetail() {
|
||||
stopModel(modelId)
|
||||
.then(() => {
|
||||
setActiveModels((prevModels) =>
|
||||
prevModels.filter((model) => model.id !== modelId)
|
||||
prevModels.filter((model) => model !== modelId)
|
||||
)
|
||||
})
|
||||
.catch((error) => {
|
||||
@ -240,7 +236,7 @@ function ProviderDetail() {
|
||||
className={cn(
|
||||
'flex flex-col gap-3',
|
||||
provider &&
|
||||
provider.provider === 'llama.cpp' &&
|
||||
provider.provider === 'llamacpp' &&
|
||||
'flex-col-reverse'
|
||||
)}
|
||||
>
|
||||
@ -353,7 +349,7 @@ function ProviderDetail() {
|
||||
{t('providers:models')}
|
||||
</h1>
|
||||
<div className="flex items-center gap-2">
|
||||
{provider && provider.provider !== 'llama.cpp' && (
|
||||
{provider && provider.provider !== 'llamacpp' && (
|
||||
<>
|
||||
{!predefinedProviders.some(
|
||||
(p) => p.provider === provider.provider
|
||||
@ -388,7 +384,7 @@ function ProviderDetail() {
|
||||
<DialogAddModel provider={provider} />
|
||||
</>
|
||||
)}
|
||||
{provider && provider.provider === 'llama.cpp' && (
|
||||
{provider && provider.provider === 'llamacpp' && (
|
||||
<Button
|
||||
variant="link"
|
||||
size="sm"
|
||||
@ -404,10 +400,15 @@ function ProviderDetail() {
|
||||
},
|
||||
],
|
||||
})
|
||||
// If the dialog returns a file path, extract just the file name
|
||||
const fileName =
|
||||
typeof selectedFile === 'string'
|
||||
? selectedFile.split(/[\\/]/).pop()
|
||||
: undefined
|
||||
|
||||
if (selectedFile) {
|
||||
if (selectedFile && fileName) {
|
||||
try {
|
||||
await importModel(selectedFile)
|
||||
await pullModel(fileName, selectedFile)
|
||||
} catch (error) {
|
||||
console.error(
|
||||
t('providers:importModelError'),
|
||||
@ -475,46 +476,40 @@ function ProviderDetail() {
|
||||
provider={provider}
|
||||
modelId={model.id}
|
||||
/>
|
||||
{provider &&
|
||||
provider.provider === 'llama.cpp' && (
|
||||
<div className="ml-2">
|
||||
{activeModels.some(
|
||||
(activeModel) =>
|
||||
activeModel.id === model.id
|
||||
) ? (
|
||||
<Button
|
||||
size="sm"
|
||||
variant="destructive"
|
||||
onClick={() =>
|
||||
handleStopModel(model.id)
|
||||
}
|
||||
>
|
||||
{t('providers:stop')}
|
||||
</Button>
|
||||
) : (
|
||||
<Button
|
||||
size="sm"
|
||||
disabled={loadingModels.includes(
|
||||
model.id
|
||||
)}
|
||||
onClick={() =>
|
||||
handleStartModel(model.id)
|
||||
}
|
||||
>
|
||||
{loadingModels.includes(model.id) ? (
|
||||
<div className="flex items-center gap-2">
|
||||
<IconLoader
|
||||
size={16}
|
||||
className="animate-spin"
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
t('providers:start')
|
||||
)}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{provider && provider.provider === 'llamacpp' && (
|
||||
<div className="ml-2">
|
||||
{activeModels.some(
|
||||
(activeModel) => activeModel === model.id
|
||||
) ? (
|
||||
<Button
|
||||
size="sm"
|
||||
variant="destructive"
|
||||
onClick={() => handleStopModel(model.id)}
|
||||
>
|
||||
{t('providers:stop')}
|
||||
</Button>
|
||||
) : (
|
||||
<Button
|
||||
size="sm"
|
||||
disabled={loadingModels.includes(
|
||||
model.id
|
||||
)}
|
||||
onClick={() => handleStartModel(model.id)}
|
||||
>
|
||||
{loadingModels.includes(model.id) ? (
|
||||
<div className="flex items-center gap-2">
|
||||
<IconLoader
|
||||
size={16}
|
||||
className="animate-spin"
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
t('providers:start')
|
||||
)}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
}
|
||||
/>
|
||||
|
||||
@ -5,10 +5,9 @@ import { getHardwareInfo } from '@/services/hardware'
|
||||
import { Progress } from '@/components/ui/progress'
|
||||
import type { HardwareData } from '@/hooks/useHardware'
|
||||
import { route } from '@/constants/routes'
|
||||
import { formatDuration, formatMegaBytes } from '@/lib/utils'
|
||||
import { formatMegaBytes } from '@/lib/utils'
|
||||
import { IconDeviceDesktopAnalytics } from '@tabler/icons-react'
|
||||
import { getActiveModels, stopModel } from '@/services/models'
|
||||
import { ActiveModel } from '@/types/models'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { useTranslation } from '@/i18n/react-i18next-compat'
|
||||
|
||||
@ -21,7 +20,7 @@ function SystemMonitor() {
|
||||
const { t } = useTranslation()
|
||||
const { hardwareData, setHardwareData, updateCPUUsage, updateRAMAvailable } =
|
||||
useHardware()
|
||||
const [activeModels, setActiveModels] = useState<ActiveModel[]>([])
|
||||
const [activeModels, setActiveModels] = useState<string[]>([])
|
||||
|
||||
useEffect(() => {
|
||||
// Initial data fetch
|
||||
@ -47,7 +46,7 @@ function SystemMonitor() {
|
||||
stopModel(modelId)
|
||||
.then(() => {
|
||||
setActiveModels((prevModels) =>
|
||||
prevModels.filter((model) => model.id !== modelId)
|
||||
prevModels.filter((model) => model !== modelId)
|
||||
)
|
||||
})
|
||||
.catch((error) => {
|
||||
@ -173,10 +172,10 @@ function SystemMonitor() {
|
||||
{activeModels.length > 0 && (
|
||||
<div className="flex flex-col gap-4">
|
||||
{activeModels.map((model) => (
|
||||
<div className="bg-main-view-fg/3 rounded-lg p-4" key={model.id}>
|
||||
<div className="bg-main-view-fg/3 rounded-lg p-4" key={model}>
|
||||
<div className="flex justify-between items-center mb-2">
|
||||
<span className="font-semibold text-main-view-fg">
|
||||
{model.id}
|
||||
{model}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex flex-col gap-2 mt-3">
|
||||
@ -190,9 +189,9 @@ function SystemMonitor() {
|
||||
<span className="text-main-view-fg/70">
|
||||
{t('system-monitor:uptime')}
|
||||
</span>
|
||||
<span className="text-main-view-fg">
|
||||
{/* <span className="text-main-view-fg">
|
||||
{model.start_time && formatDuration(model.start_time)}
|
||||
</span>
|
||||
</span> */}
|
||||
</div>
|
||||
<div className="flex justify-between items-center">
|
||||
<span className="text-main-view-fg/70">
|
||||
@ -202,7 +201,7 @@ function SystemMonitor() {
|
||||
<Button
|
||||
variant="destructive"
|
||||
size="sm"
|
||||
onClick={() => stopRunningModel(model.id)}
|
||||
onClick={() => stopRunningModel(model)}
|
||||
>
|
||||
{t('system-monitor:stop')}
|
||||
</Button>
|
||||
|
||||
@ -1,56 +1,37 @@
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
import { ExtensionManager } from '@/lib/extension'
|
||||
import { normalizeProvider } from '@/lib/models'
|
||||
import { EngineManager, ExtensionTypeEnum, ModelExtension } from '@janhq/core'
|
||||
import { AIEngine, EngineManager, SettingComponentProps } from '@janhq/core'
|
||||
import { Model as CoreModel } from '@janhq/core'
|
||||
|
||||
// TODO: Replace this with the actual provider later
|
||||
const defaultProvider = 'llamacpp'
|
||||
|
||||
const getEngine = (provider: string = defaultProvider) => {
|
||||
return EngineManager.instance().get(provider) as AIEngine
|
||||
}
|
||||
/**
|
||||
* Fetches all available models.
|
||||
* @returns A promise that resolves to the models.
|
||||
*/
|
||||
export const fetchModels = async () => {
|
||||
return ExtensionManager.getInstance()
|
||||
.get<ModelExtension>(ExtensionTypeEnum.Model)
|
||||
?.getModels()
|
||||
return getEngine().list()
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the sources of the models.
|
||||
* @returns A promise that resolves to the model sources.
|
||||
*/
|
||||
export const fetchModelSources = async (): Promise<any[]> => {
|
||||
const extension = ExtensionManager.getInstance().get<ModelExtension>(
|
||||
ExtensionTypeEnum.Model
|
||||
)
|
||||
|
||||
if (!extension) return []
|
||||
|
||||
try {
|
||||
const sources = await extension.getSources()
|
||||
const mappedSources = sources.map((m) => ({
|
||||
...m,
|
||||
models: m.models.sort((a, b) => a.size - b.size),
|
||||
}))
|
||||
|
||||
// Prepend the hardcoded model to the sources
|
||||
return [...mappedSources]
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch model sources:', error)
|
||||
return []
|
||||
}
|
||||
export const fetchModelSources = async () => {
|
||||
// TODO: New Hub
|
||||
return []
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the model hub.
|
||||
* @returns A promise that resolves to the model hub.
|
||||
*/
|
||||
export const fetchModelHub = async (): Promise<any[]> => {
|
||||
const hubData = await ExtensionManager.getInstance()
|
||||
.get<ModelExtension>(ExtensionTypeEnum.Model)
|
||||
?.fetchModelsHub()
|
||||
|
||||
// Prepend the hardcoded model to the hub data
|
||||
return hubData ? [...hubData] : []
|
||||
export const fetchModelHub = async () => {
|
||||
// TODO: New Hub
|
||||
return
|
||||
}
|
||||
|
||||
/**
|
||||
@ -59,18 +40,9 @@ export const fetchModelHub = async (): Promise<any[]> => {
|
||||
* @returns A promise that resolves when the source is added.
|
||||
*/
|
||||
export const addModelSource = async (source: string) => {
|
||||
const extension = ExtensionManager.getInstance().get<ModelExtension>(
|
||||
ExtensionTypeEnum.Model
|
||||
)
|
||||
|
||||
if (!extension) throw new Error('Model extension not found')
|
||||
|
||||
try {
|
||||
return await extension.addSource(source)
|
||||
} catch (error) {
|
||||
console.error('Failed to add model source:', error)
|
||||
throw error
|
||||
}
|
||||
// TODO: New Hub
|
||||
console.log(source)
|
||||
return
|
||||
}
|
||||
|
||||
/**
|
||||
@ -79,18 +51,9 @@ export const addModelSource = async (source: string) => {
|
||||
* @returns A promise that resolves when the source is deleted.
|
||||
*/
|
||||
export const deleteModelSource = async (source: string) => {
|
||||
const extension = ExtensionManager.getInstance().get<ModelExtension>(
|
||||
ExtensionTypeEnum.Model
|
||||
)
|
||||
|
||||
if (!extension) throw new Error('Model extension not found')
|
||||
|
||||
try {
|
||||
return await extension.deleteSource(source)
|
||||
} catch (error) {
|
||||
console.error('Failed to delete model source:', error)
|
||||
throw error
|
||||
}
|
||||
// TODO: New Hub
|
||||
console.log(source)
|
||||
return
|
||||
}
|
||||
|
||||
/**
|
||||
@ -102,38 +65,19 @@ export const updateModel = async (
|
||||
model: Partial<CoreModel>
|
||||
// provider: string,
|
||||
) => {
|
||||
const extension = ExtensionManager.getInstance().get<ModelExtension>(
|
||||
ExtensionTypeEnum.Model
|
||||
)
|
||||
|
||||
if (!extension) throw new Error('Model extension not found')
|
||||
|
||||
try {
|
||||
return await extension.updateModel(model)
|
||||
} catch (error) {
|
||||
console.error('Failed to update model:', error)
|
||||
throw error
|
||||
}
|
||||
if (model.settings)
|
||||
getEngine().updateSettings(model.settings as SettingComponentProps[])
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads a model.
|
||||
* @param model The model to download.
|
||||
* Pull or import a model.
|
||||
* @param model The model to pull.
|
||||
* @returns A promise that resolves when the model download task is created.
|
||||
*/
|
||||
export const downloadModel = async (id: string) => {
|
||||
const extension = ExtensionManager.getInstance().get<ModelExtension>(
|
||||
ExtensionTypeEnum.Model
|
||||
)
|
||||
|
||||
if (!extension) throw new Error('Model extension not found')
|
||||
|
||||
try {
|
||||
return await extension.pullModel(id)
|
||||
} catch (error) {
|
||||
console.error('Failed to download model:', error)
|
||||
throw error
|
||||
}
|
||||
export const pullModel = async (id: string, modelPath: string) => {
|
||||
return getEngine().import(id, {
|
||||
modelPath,
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
@ -142,18 +86,7 @@ export const downloadModel = async (id: string) => {
|
||||
* @returns
|
||||
*/
|
||||
export const abortDownload = async (id: string) => {
|
||||
const extension = ExtensionManager.getInstance().get<ModelExtension>(
|
||||
ExtensionTypeEnum.Model
|
||||
)
|
||||
|
||||
if (!extension) throw new Error('Model extension not found')
|
||||
|
||||
try {
|
||||
return await extension.cancelModelPull(id)
|
||||
} catch (error) {
|
||||
console.error('Failed to abort model download:', error)
|
||||
throw error
|
||||
}
|
||||
return getEngine().abortImport(id)
|
||||
}
|
||||
|
||||
/**
|
||||
@ -162,64 +95,7 @@ export const abortDownload = async (id: string) => {
|
||||
* @returns
|
||||
*/
|
||||
export const deleteModel = async (id: string) => {
|
||||
const extension = ExtensionManager.getInstance().get<ModelExtension>(
|
||||
ExtensionTypeEnum.Model
|
||||
)
|
||||
|
||||
if (!extension) throw new Error('Model extension not found')
|
||||
|
||||
try {
|
||||
return await extension.deleteModel(id).then(() => {
|
||||
// TODO: This should be removed when we integrate new llama.cpp extension
|
||||
if (id.includes(':')) {
|
||||
extension.addSource(`cortexso/${id.split(':')[0]}`)
|
||||
}
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('Failed to delete model:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Imports a model from a file path.
|
||||
* @param filePath The path to the model file or an array of file paths.
|
||||
* @param modelId Optional model ID. If not provided, it will be derived from the file name.
|
||||
* @param provider The provider for the model (default: 'llama.cpp').
|
||||
* @returns A promise that resolves when the model is imported.
|
||||
*/
|
||||
export const importModel = async (
|
||||
filePath: string | string[],
|
||||
modelId?: string,
|
||||
provider: string = 'llama.cpp'
|
||||
) => {
|
||||
const extension = ExtensionManager.getInstance().get<ModelExtension>(
|
||||
ExtensionTypeEnum.Model
|
||||
)
|
||||
|
||||
if (!extension) throw new Error('Model extension not found')
|
||||
|
||||
try {
|
||||
// If filePath is an array, use the first element
|
||||
const path = Array.isArray(filePath) ? filePath[0] : filePath
|
||||
|
||||
// If no path was selected, throw an error
|
||||
if (!path) throw new Error('No file selected')
|
||||
|
||||
// Extract filename from path to use as model ID if not provided
|
||||
const defaultModelId =
|
||||
path
|
||||
.split(/[/\\]/)
|
||||
.pop()
|
||||
?.replace(/ /g, '-')
|
||||
.replace(/\.gguf$/i, '') || path
|
||||
const modelIdToUse = modelId || defaultModelId
|
||||
|
||||
return await extension.importModel(modelIdToUse, path, provider)
|
||||
} catch (error) {
|
||||
console.error('Failed to import model:', error)
|
||||
throw error
|
||||
}
|
||||
return getEngine().delete(id)
|
||||
}
|
||||
|
||||
/**
|
||||
@ -228,20 +104,8 @@ export const importModel = async (
|
||||
* @returns
|
||||
*/
|
||||
export const getActiveModels = async (provider?: string) => {
|
||||
const providerName = provider || 'cortex' // we will go down to llama.cpp extension later on
|
||||
const extension = EngineManager.instance().get(providerName)
|
||||
|
||||
if (!extension) throw new Error('Model extension not found')
|
||||
|
||||
try {
|
||||
return 'activeModels' in extension &&
|
||||
typeof extension.activeModels === 'function'
|
||||
? ((await extension.activeModels()) ?? [])
|
||||
: []
|
||||
} catch (error) {
|
||||
console.error('Failed to get active models:', error)
|
||||
return []
|
||||
}
|
||||
// getEngine(provider)
|
||||
return getEngine(provider).getLoadedModels()
|
||||
}
|
||||
|
||||
/**
|
||||
@ -251,20 +115,7 @@ export const getActiveModels = async (provider?: string) => {
|
||||
* @returns
|
||||
*/
|
||||
export const stopModel = async (model: string, provider?: string) => {
|
||||
const providerName = provider || 'cortex' // we will go down to llama.cpp extension later on
|
||||
const extension = EngineManager.instance().get(providerName)
|
||||
|
||||
if (!extension) throw new Error('Model extension not found')
|
||||
|
||||
try {
|
||||
return await extension.unloadModel({
|
||||
model,
|
||||
id: model,
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('Failed to stop model:', error)
|
||||
return []
|
||||
}
|
||||
getEngine(provider).unload(model)
|
||||
}
|
||||
|
||||
/**
|
||||
@ -273,10 +124,7 @@ export const stopModel = async (model: string, provider?: string) => {
|
||||
*/
|
||||
export const stopAllModels = async () => {
|
||||
const models = await getActiveModels()
|
||||
if (models)
|
||||
await Promise.all(
|
||||
models.map((model: { id: string }) => stopModel(model.id))
|
||||
)
|
||||
if (models) await Promise.all(models.map((model) => stopModel(model)))
|
||||
}
|
||||
|
||||
/**
|
||||
@ -289,28 +137,17 @@ export const stopAllModels = async () => {
|
||||
*/
|
||||
export const startModel = async (
|
||||
provider: ProviderObject,
|
||||
model: string,
|
||||
abortController?: AbortController
|
||||
model: string
|
||||
): Promise<void> => {
|
||||
const providerObj = EngineManager.instance().get(
|
||||
normalizeProvider(provider.provider)
|
||||
)
|
||||
const modelObj = provider.models.find((m) => m.id === model)
|
||||
|
||||
if (providerObj && modelObj) {
|
||||
return providerObj?.loadModel(
|
||||
{
|
||||
id: modelObj.id,
|
||||
settings: Object.fromEntries(
|
||||
Object.entries(modelObj.settings ?? {}).map(([key, value]) => [
|
||||
key,
|
||||
value.controller_props?.value, // assuming each setting is { value: ... }
|
||||
])
|
||||
),
|
||||
},
|
||||
abortController
|
||||
)
|
||||
}
|
||||
getEngine(provider.provider)
|
||||
.load(model)
|
||||
.catch((error) => {
|
||||
console.error(
|
||||
`Failed to start model ${model} for provider ${provider.provider}:`,
|
||||
error
|
||||
)
|
||||
throw error
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
@ -329,37 +166,16 @@ export const configurePullOptions = async ({
|
||||
verifyHostSSL,
|
||||
noProxy,
|
||||
}: ProxyOptions) => {
|
||||
const extension = ExtensionManager.getInstance().get<ModelExtension>(
|
||||
ExtensionTypeEnum.Model
|
||||
)
|
||||
|
||||
if (!extension) throw new Error('Model extension not found')
|
||||
try {
|
||||
await extension.configurePullOptions(
|
||||
proxyEnabled
|
||||
? {
|
||||
proxy_username: proxyUsername,
|
||||
proxy_password: proxyPassword,
|
||||
proxy_url: proxyUrl,
|
||||
verify_proxy_ssl: proxyIgnoreSSL ? false : verifyProxySSL,
|
||||
verify_proxy_host_ssl: proxyIgnoreSSL ? false : verifyProxyHostSSL,
|
||||
verify_peer_ssl: proxyIgnoreSSL ? false : verifyPeerSSL,
|
||||
verify_host_ssl: proxyIgnoreSSL ? false : verifyHostSSL,
|
||||
no_proxy: noProxy,
|
||||
}
|
||||
: {
|
||||
proxy_username: '',
|
||||
proxy_password: '',
|
||||
proxy_url: '',
|
||||
verify_proxy_ssl: false,
|
||||
verify_proxy_host_ssl: false,
|
||||
verify_peer_ssl: false,
|
||||
verify_host_ssl: false,
|
||||
no_proxy: '',
|
||||
}
|
||||
)
|
||||
} catch (error) {
|
||||
console.error('Failed to configure pull options:', error)
|
||||
throw error
|
||||
}
|
||||
console.log('Configuring proxy options:', {
|
||||
proxyEnabled,
|
||||
proxyUrl,
|
||||
proxyUsername,
|
||||
proxyPassword,
|
||||
proxyIgnoreSSL,
|
||||
verifyProxySSL,
|
||||
verifyProxyHostSSL,
|
||||
verifyPeerSSL,
|
||||
verifyHostSSL,
|
||||
noProxy,
|
||||
})
|
||||
}
|
||||
|
||||
@ -1,11 +1,6 @@
|
||||
import { models as providerModels } from 'token.js'
|
||||
import { predefinedProviders } from '@/mock/data'
|
||||
import {
|
||||
EngineManagementExtension,
|
||||
EngineManager,
|
||||
ExtensionTypeEnum,
|
||||
SettingComponentProps,
|
||||
} from '@janhq/core'
|
||||
import { EngineManager, SettingComponentProps } from '@janhq/core'
|
||||
import {
|
||||
DefaultToolUseSupportedModels,
|
||||
ModelCapabilities,
|
||||
@ -17,11 +12,6 @@ import { fetch as fetchTauri } from '@tauri-apps/plugin-http'
|
||||
|
||||
|
||||
export const getProviders = async (): Promise<ModelProvider[]> => {
|
||||
const engines = !localStorage.getItem('migration_completed')
|
||||
? await ExtensionManager.getInstance()
|
||||
.get<EngineManagementExtension>(ExtensionTypeEnum.Engine)
|
||||
?.getEngines()
|
||||
: {}
|
||||
const builtinProviders = predefinedProviders.map((provider) => {
|
||||
let models = provider.models as Model[]
|
||||
if (Object.keys(providerModels).includes(provider.provider)) {
|
||||
@ -29,29 +19,6 @@ export const getProviders = async (): Promise<ModelProvider[]> => {
|
||||
provider.provider as unknown as keyof typeof providerModels
|
||||
].models as unknown as string[]
|
||||
|
||||
if (engines && Object.keys(engines).length > 0) {
|
||||
for (const [key, value] of Object.entries(engines)) {
|
||||
const providerName = key.replace('google_gemini', 'gemini')
|
||||
if (provider.provider !== providerName) continue
|
||||
const engine = value[0] as
|
||||
| {
|
||||
api_key?: string
|
||||
url?: string
|
||||
engine?: string
|
||||
}
|
||||
| undefined
|
||||
if (engine && 'api_key' in engine) {
|
||||
const settings = provider?.settings.map((e) => {
|
||||
if (e.key === 'api-key')
|
||||
e.controller_props.value = (engine.api_key as string) ?? ''
|
||||
return e
|
||||
})
|
||||
|
||||
provider.settings = settings
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (Array.isArray(builtInModels))
|
||||
models = builtInModels.map((model) => {
|
||||
const modelManifest = models.find((e) => e.id === model)
|
||||
@ -77,24 +44,11 @@ export const getProviders = async (): Promise<ModelProvider[]> => {
|
||||
models,
|
||||
}
|
||||
})
|
||||
if (engines && Object.keys(engines).length > 0) {
|
||||
localStorage.setItem('migration_completed', 'true')
|
||||
}
|
||||
|
||||
const runtimeProviders: ModelProvider[] = []
|
||||
for (const [providerName, value] of EngineManager.instance().engines) {
|
||||
const models = (await fetchModels()) ?? []
|
||||
|
||||
for (const [key, value] of EngineManager.instance().engines) {
|
||||
// TODO: Remove this when the cortex extension is removed
|
||||
const providerName = key === 'cortex' ? 'llama.cpp' : key
|
||||
|
||||
const models =
|
||||
((await fetchModels()) ?? []).filter(
|
||||
(model) =>
|
||||
(model.engine === 'llama-cpp' ? 'llama.cpp' : model.engine) ===
|
||||
providerName &&
|
||||
'status' in model &&
|
||||
model.status === 'downloaded'
|
||||
) ?? []
|
||||
const provider: ModelProvider = {
|
||||
active: false,
|
||||
persist: true,
|
||||
@ -246,9 +200,8 @@ export const updateSettings = async (
|
||||
providerName: string,
|
||||
settings: ProviderSetting[]
|
||||
): Promise<void> => {
|
||||
const provider = providerName === 'llama.cpp' ? 'cortex' : providerName
|
||||
return ExtensionManager.getInstance()
|
||||
.getEngine(provider)
|
||||
.getEngine(providerName)
|
||||
?.updateSettings(
|
||||
settings.map((setting) => ({
|
||||
...setting,
|
||||
|
||||
@ -51,7 +51,7 @@ export const createThread = async (thread: Thread): Promise<Thread> => {
|
||||
...(thread.assistants?.[0] ?? defaultAssistant),
|
||||
model: {
|
||||
id: thread.model?.id ?? '*',
|
||||
engine: thread.model?.provider ?? 'llama.cpp',
|
||||
engine: thread.model?.provider ?? 'llamacpp',
|
||||
},
|
||||
},
|
||||
],
|
||||
@ -88,7 +88,7 @@ export const updateThread = (thread: Thread) => {
|
||||
return {
|
||||
model: {
|
||||
id: thread.model?.id ?? '*',
|
||||
engine: thread.model?.provider ?? 'llama.cpp',
|
||||
engine: thread.model?.provider ?? 'llamacpp',
|
||||
},
|
||||
id: e.id,
|
||||
name: e.name,
|
||||
@ -98,7 +98,7 @@ export const updateThread = (thread: Thread) => {
|
||||
{
|
||||
model: {
|
||||
id: thread.model?.id ?? '*',
|
||||
engine: thread.model?.provider ?? 'llama.cpp',
|
||||
engine: thread.model?.provider ?? 'llamacpp',
|
||||
},
|
||||
id: 'jan',
|
||||
name: 'Jan',
|
||||
|
||||
@ -20,13 +20,3 @@ export enum DefaultToolUseSupportedModels {
|
||||
JanNano = 'jan-nano',
|
||||
Qwen3 = 'qwen3',
|
||||
}
|
||||
|
||||
export type ActiveModel = {
|
||||
engine: string
|
||||
id: string
|
||||
model_size: number
|
||||
object: 'model'
|
||||
ram: number
|
||||
start_time: number
|
||||
vram: number
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user