diff --git a/.github/workflows/publish-npm-core.yml b/.github/workflows/publish-npm-core.yml index 49eac8dd2..462dbdc8e 100644 --- a/.github/workflows/publish-npm-core.yml +++ b/.github/workflows/publish-npm-core.yml @@ -2,9 +2,9 @@ name: Publish core Package to npmjs on: push: tags: ["v[0-9]+.[0-9]+.[0-9]+-core"] - paths: ["core/**"] + paths: ["core/**", ".github/workflows/publish-npm-core.yml"] pull_request: - paths: ["core/**"] + paths: ["core/**", ".github/workflows/publish-npm-core.yml"] jobs: build-and-publish-plugins: environment: production @@ -45,7 +45,7 @@ jobs: node-version: "20.x" registry-url: "https://registry.npmjs.org" - - run: cd core && corepack enable && corepack prepare yarn@4.5.3 --activate && yarn --version && yarn install && yarn build + - run: cd core && corepack enable && corepack prepare yarn@4.5.3 --activate && yarn --version && yarn config set -H enableImmutableInstalls false && yarn install && yarn build - run: cd core && yarn publish --access public if: github.event_name == 'push' diff --git a/.github/workflows/publish-npm-joi.yml b/.github/workflows/publish-npm-joi.yml index 8c6ee6f68..867ad80fe 100644 --- a/.github/workflows/publish-npm-joi.yml +++ b/.github/workflows/publish-npm-joi.yml @@ -2,9 +2,9 @@ name: Publish joi Package to npmjs on: push: tags: ["v[0-9]+.[0-9]+.[0-9]+-joi"] - paths: ["joi/**"] + paths: ["joi/**", ".github/workflows/publish-npm-joi.yml"] pull_request: - paths: ["joi/**"] + paths: ["joi/**", ".github/workflows/publish-npm-joi.yml"] jobs: build-and-publish-plugins: environment: production @@ -45,7 +45,7 @@ jobs: node-version: "20.x" registry-url: "https://registry.npmjs.org" - - run: cd joi && corepack enable && corepack prepare yarn@4.5.3 --activate && yarn --version && yarn install && yarn build + - run: cd joi && corepack enable && corepack prepare yarn@4.5.3 --activate && yarn --version && yarn config set -H enableImmutableInstalls false && yarn install && yarn build - run: cd joi && yarn publish --access public if: github.event_name == 'push' diff --git a/core/src/browser/extensions/engines/helpers/sse.ts b/core/src/browser/extensions/engines/helpers/sse.ts index 153d741da..aaafbf7e5 100644 --- a/core/src/browser/extensions/engines/helpers/sse.ts +++ b/core/src/browser/extensions/engines/helpers/sse.ts @@ -22,7 +22,9 @@ export function requestInference( headers: { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*', - 'Accept': model.parameters?.stream ? 'text/event-stream' : 'application/json', + 'Accept': model.parameters?.stream + ? 'text/event-stream' + : 'application/json', ...headers, }, body: JSON.stringify(requestBody), @@ -47,12 +49,24 @@ export function requestInference( } // There could be overriden stream parameter in the model // that is set in request body (transformed payload) - if (requestBody?.stream === false || model.parameters?.stream === false) { + if ( + requestBody?.stream === false || + model.parameters?.stream === false + ) { const data = await response.json() + if (data.error || data.message) { + subscriber.error(data.error ?? data) + subscriber.complete() + return + } if (transformResponse) { subscriber.next(transformResponse(data)) } else { - subscriber.next(data.choices[0]?.message?.content ?? '') + subscriber.next( + data.choices + ? data.choices[0]?.message?.content + : (data.content[0]?.text ?? '') + ) } } else { const stream = response.body diff --git a/core/src/browser/extensions/enginesManagement.ts b/core/src/browser/extensions/enginesManagement.ts index 524546b05..66dff87df 100644 --- a/core/src/browser/extensions/enginesManagement.ts +++ b/core/src/browser/extensions/enginesManagement.ts @@ -3,6 +3,7 @@ import { Engines, EngineVariant, EngineReleased, + EngineConfig, DefaultEngineVariant, } from '../../types' import { BaseExtension, ExtensionTypeEnum } from '../extension' @@ -55,8 +56,16 @@ export abstract class EngineManagementExtension extends BaseExtension { * @returns A Promise that resolves to intall of engine. */ abstract installEngine( - name: InferenceEngine, - engineConfig: { variant: string; version?: string } + name: string, + engineConfig: EngineConfig + ): Promise<{ messages: string }> + + /** + * Add a new remote engine + * @returns A Promise that resolves to intall of engine. + */ + abstract addRemoteEngine( + engineConfig: EngineConfig ): Promise<{ messages: string }> /** @@ -65,14 +74,16 @@ export abstract class EngineManagementExtension extends BaseExtension { */ abstract uninstallEngine( name: InferenceEngine, - engineConfig: { variant: string; version: string } + engineConfig: EngineConfig ): Promise<{ messages: string }> /** * @param name - Inference engine name. * @returns A Promise that resolves to an object of default engine. */ - abstract getDefaultEngineVariant(name: InferenceEngine): Promise + abstract getDefaultEngineVariant( + name: InferenceEngine + ): Promise /** * @body variant - string @@ -81,11 +92,19 @@ export abstract class EngineManagementExtension extends BaseExtension { */ abstract setDefaultEngineVariant( name: InferenceEngine, - engineConfig: { variant: string; version: string } + engineConfig: EngineConfig ): Promise<{ messages: string }> /** * @returns A Promise that resolves to update engine. */ - abstract updateEngine(name: InferenceEngine): Promise<{ messages: string }> + abstract updateEngine( + name: InferenceEngine, + engineConfig?: EngineConfig + ): Promise<{ messages: string }> + + /** + * @returns A Promise that resolves to an object of remote models list . + */ + abstract getRemoteModels(name: InferenceEngine | string): Promise } diff --git a/core/src/node/helper/config.test.ts b/core/src/node/helper/config.test.ts index d46750d5f..617a8f7ef 100644 --- a/core/src/node/helper/config.test.ts +++ b/core/src/node/helper/config.test.ts @@ -1,28 +1,19 @@ -import { getEngineConfiguration } from './config'; -import { getAppConfigurations, defaultAppConfig } from './config'; - -import { getJanExtensionsPath } from './config'; -import { getJanDataFolderPath } from './config'; -it('should return undefined for invalid engine ID', async () => { - const config = await getEngineConfiguration('invalid_engine'); - expect(config).toBeUndefined(); -}); +import { getAppConfigurations, defaultAppConfig } from './config' +import { getJanExtensionsPath, getJanDataFolderPath } from './config' it('should return default config when CI is e2e', () => { - process.env.CI = 'e2e'; - const config = getAppConfigurations(); - expect(config).toEqual(defaultAppConfig()); -}); - + process.env.CI = 'e2e' + const config = getAppConfigurations() + expect(config).toEqual(defaultAppConfig()) +}) it('should return extensions path when retrieved successfully', () => { - const extensionsPath = getJanExtensionsPath(); - expect(extensionsPath).not.toBeUndefined(); -}); - + const extensionsPath = getJanExtensionsPath() + expect(extensionsPath).not.toBeUndefined() +}) it('should return data folder path when retrieved successfully', () => { - const dataFolderPath = getJanDataFolderPath(); - expect(dataFolderPath).not.toBeUndefined(); -}); + const dataFolderPath = getJanDataFolderPath() + expect(dataFolderPath).not.toBeUndefined() +}) diff --git a/core/src/node/helper/config.ts b/core/src/node/helper/config.ts index 8bf48d629..6fb28d01f 100644 --- a/core/src/node/helper/config.ts +++ b/core/src/node/helper/config.ts @@ -1,8 +1,7 @@ -import { AppConfiguration, SettingComponentProps } from '../../types' +import { AppConfiguration } from '../../types' import { join, resolve } from 'path' import fs from 'fs' import os from 'os' -import childProcess from 'child_process' const configurationFileName = 'settings.json' /** @@ -19,7 +18,9 @@ export const getAppConfigurations = (): AppConfiguration => { if (!fs.existsSync(configurationFile)) { // create default app config if we don't have one - console.debug(`App config not found, creating default config at ${configurationFile}`) + console.debug( + `App config not found, creating default config at ${configurationFile}` + ) fs.writeFileSync(configurationFile, JSON.stringify(appDefaultConfiguration)) return appDefaultConfiguration } @@ -30,20 +31,28 @@ export const getAppConfigurations = (): AppConfiguration => { ) return appConfigurations } catch (err) { - console.error(`Failed to read app config, return default config instead! Err: ${err}`) + console.error( + `Failed to read app config, return default config instead! Err: ${err}` + ) return defaultAppConfig() } } const getConfigurationFilePath = () => join( - global.core?.appPath() || process.env[process.platform == 'win32' ? 'USERPROFILE' : 'HOME'], + global.core?.appPath() || + process.env[process.platform == 'win32' ? 'USERPROFILE' : 'HOME'], configurationFileName ) -export const updateAppConfiguration = (configuration: AppConfiguration): Promise => { +export const updateAppConfiguration = ( + configuration: AppConfiguration +): Promise => { const configurationFile = getConfigurationFilePath() - console.debug('updateAppConfiguration, configurationFile: ', configurationFile) + console.debug( + 'updateAppConfiguration, configurationFile: ', + configurationFile + ) fs.writeFileSync(configurationFile, JSON.stringify(configuration)) return Promise.resolve() @@ -69,86 +78,6 @@ export const getJanExtensionsPath = (): string => { return join(appConfigurations.data_folder, 'extensions') } -/** - * Utility function to physical cpu count - * - * @returns {number} The physical cpu count. - */ -export const physicalCpuCount = async (): Promise => { - const platform = os.platform() - try { - if (platform === 'linux') { - const output = await exec('lscpu -p | egrep -v "^#" | sort -u -t, -k 2,4 | wc -l') - return parseInt(output.trim(), 10) - } else if (platform === 'darwin') { - const output = await exec('sysctl -n hw.physicalcpu_max') - return parseInt(output.trim(), 10) - } else if (platform === 'win32') { - const output = await exec('WMIC CPU Get NumberOfCores') - return output - .split(os.EOL) - .map((line: string) => parseInt(line)) - .filter((value: number) => !isNaN(value)) - .reduce((sum: number, number: number) => sum + number, 1) - } else { - const cores = os.cpus().filter((cpu: any, index: number) => { - const hasHyperthreading = cpu.model.includes('Intel') - const isOdd = index % 2 === 1 - return !hasHyperthreading || isOdd - }) - return cores.length - } - } catch (err) { - console.warn('Failed to get physical CPU count', err) - // Divide by 2 to get rid of hyper threading - const coreCount = Math.ceil(os.cpus().length / 2) - console.debug('Using node API to get physical CPU count:', coreCount) - return coreCount - } -} - -const exec = async (command: string): Promise => { - return new Promise((resolve, reject) => { - childProcess.exec(command, { encoding: 'utf8' }, (error, stdout) => { - if (error) { - reject(error) - } else { - resolve(stdout) - } - }) - }) -} - -// a hacky way to get the api key. we should comes up with a better -// way to handle this -export const getEngineConfiguration = async (engineId: string) => { - if (engineId !== 'openai' && engineId !== 'groq') return undefined - - const settingDirectoryPath = join( - getJanDataFolderPath(), - 'settings', - '@janhq', - engineId === 'openai' ? 'inference-openai-extension' : 'inference-groq-extension', - 'settings.json' - ) - - const content = fs.readFileSync(settingDirectoryPath, 'utf-8') - const settings: SettingComponentProps[] = JSON.parse(content) - const apiKeyId = engineId === 'openai' ? 'openai-api-key' : 'groq-api-key' - const keySetting = settings.find((setting) => setting.key === apiKeyId) - let fullUrl = settings.find((setting) => setting.key === 'chat-completions-endpoint') - ?.controllerProps.value - - let apiKey = keySetting?.controllerProps.value - if (typeof apiKey !== 'string') apiKey = '' - if (typeof fullUrl !== 'string') fullUrl = '' - - return { - api_key: apiKey, - full_url: fullUrl, - } -} - /** * Default app configurations * App Data Folder default to Electron's userData @@ -158,7 +87,10 @@ export const getEngineConfiguration = async (engineId: string) => { */ export const defaultAppConfig = (): AppConfiguration => { const { app } = require('electron') - const defaultJanDataFolder = join(app?.getPath('userData') ?? os?.homedir() ?? '', 'data') + const defaultJanDataFolder = join( + app?.getPath('userData') ?? os?.homedir() ?? '', + 'data' + ) return { data_folder: process.env.CI === 'e2e' diff --git a/core/src/node/helper/resource.test.ts b/core/src/node/helper/resource.test.ts index aaeab9d65..c82d481db 100644 --- a/core/src/node/helper/resource.test.ts +++ b/core/src/node/helper/resource.test.ts @@ -1,15 +1,9 @@ -import { getSystemResourceInfo } from './resource'; +import { getSystemResourceInfo } from './resource' it('should return the correct system resource information with a valid CPU count', async () => { - const mockCpuCount = 4; - jest.spyOn(require('./config'), 'physicalCpuCount').mockResolvedValue(mockCpuCount); - const logSpy = jest.spyOn(require('./logger'), 'log').mockImplementation(() => {}); - - const result = await getSystemResourceInfo(); + const result = await getSystemResourceInfo() expect(result).toEqual({ - numCpuPhysicalCore: mockCpuCount, memAvailable: 0, - }); - expect(logSpy).toHaveBeenCalledWith(`[CORTEX]::CPU information - ${mockCpuCount}`); -}); + }) +}) diff --git a/core/src/node/helper/resource.ts b/core/src/node/helper/resource.ts index c7bfbf20c..5d75e54eb 100644 --- a/core/src/node/helper/resource.ts +++ b/core/src/node/helper/resource.ts @@ -1,13 +1,7 @@ import { SystemResourceInfo } from '../../types' -import { physicalCpuCount } from './config' -import { log } from './logger' export const getSystemResourceInfo = async (): Promise => { - const cpu = await physicalCpuCount() - log(`[CORTEX]::CPU information - ${cpu}`) - return { - numCpuPhysicalCore: cpu, memAvailable: 0, // TODO: this should not be 0 } } diff --git a/core/src/types/engine/index.ts b/core/src/types/engine/index.ts index 9e6f5c9c8..7c848a279 100644 --- a/core/src/types/engine/index.ts +++ b/core/src/types/engine/index.ts @@ -1,7 +1,23 @@ import { InferenceEngine } from '../../types' export type Engines = { - [key in InferenceEngine]: EngineVariant[] + [key in InferenceEngine]: (EngineVariant & EngineConfig)[] +} + +export type EngineMetadata = { + get_models_url?: string + header_template?: string + transform_req?: { + chat_completions?: { + url?: string + template?: string + } + } + transform_resp?: { + chat_completions?: { + template?: string + } + } } export type EngineVariant = { @@ -23,6 +39,16 @@ export type EngineReleased = { size: number } +export type EngineConfig = { + engine?: string + version?: string + variant?: string + type?: string + url?: string + api_key?: string + metadata?: EngineMetadata +} + export enum EngineEvent { OnEngineUpdate = 'OnEngineUpdate', } diff --git a/core/src/types/message/messageEntity.ts b/core/src/types/message/messageEntity.ts index 302b824ee..edd253a57 100644 --- a/core/src/types/message/messageEntity.ts +++ b/core/src/types/message/messageEntity.ts @@ -32,9 +32,8 @@ export type ThreadMessage = { completed_at: number /** The additional metadata of this message. **/ metadata?: Record - + /** Type of the message */ type?: string - /** The error code which explain what error type. Used in conjunction with MessageStatus.Error */ error_code?: ErrorCode } @@ -72,6 +71,10 @@ export type MessageRequest = { // TODO: deprecate threadId field thread?: Thread + /** Engine name to process */ + engine?: string + + /** Message type */ type?: string } @@ -147,7 +150,9 @@ export interface Attachment { /** * The tools to add this file to. */ - tools?: Array + tools?: Array< + CodeInterpreterTool | Attachment.AssistantToolsFileSearchTypeOnly + > } export namespace Attachment { @@ -166,5 +171,10 @@ export interface IncompleteDetails { /** * The reason the message is incomplete. */ - reason: 'content_filter' | 'max_tokens' | 'run_cancelled' | 'run_expired' | 'run_failed' + reason: + | 'content_filter' + | 'max_tokens' + | 'run_cancelled' + | 'run_expired' + | 'run_failed' } diff --git a/core/src/types/miscellaneous/systemResourceInfo.ts b/core/src/types/miscellaneous/systemResourceInfo.ts index 6ceea0822..82db5d941 100644 --- a/core/src/types/miscellaneous/systemResourceInfo.ts +++ b/core/src/types/miscellaneous/systemResourceInfo.ts @@ -1,5 +1,4 @@ export type SystemResourceInfo = { - numCpuPhysicalCore: number memAvailable: number } diff --git a/electron/package.json b/electron/package.json index 72163cb42..329f2b4c4 100644 --- a/electron/package.json +++ b/electron/package.json @@ -1,6 +1,6 @@ { "name": "jan", - "version": "0.1.3", + "version": "0.1.1736316956", "main": "./build/main.js", "author": "Jan ", "license": "MIT", diff --git a/extensions/engine-management-extension/engines.mjs b/extensions/engine-management-extension/engines.mjs new file mode 100644 index 000000000..e85035423 --- /dev/null +++ b/extensions/engine-management-extension/engines.mjs @@ -0,0 +1,39 @@ +import anthropic from './resources/anthropic.json' with { type: 'json' } +import cohere from './resources/cohere.json' with { type: 'json' } +import openai from './resources/openai.json' with { type: 'json' } +import openrouter from './resources/openrouter.json' with { type: 'json' } +import groq from './resources/groq.json' with { type: 'json' } +import martian from './resources/martian.json' with { type: 'json' } +import mistral from './resources/mistral.json' with { type: 'json' } +import nvidia from './resources/nvidia.json' with { type: 'json' } + +import anthropicModels from './models/anthropic.json' with { type: 'json' } +import cohereModels from './models/cohere.json' with { type: 'json' } +import openaiModels from './models/openai.json' with { type: 'json' } +import openrouterModels from './models/openrouter.json' with { type: 'json' } +import groqModels from './models/groq.json' with { type: 'json' } +import martianModels from './models/martian.json' with { type: 'json' } +import mistralModels from './models/mistral.json' with { type: 'json' } +import nvidiaModels from './models/nvidia.json' with { type: 'json' } + +const engines = [ + anthropic, + openai, + cohere, + openrouter, + groq, + mistral, + martian, + nvidia, +] +const models = [ + ...anthropicModels, + ...openaiModels, + ...cohereModels, + ...openrouterModels, + ...groqModels, + ...mistralModels, + ...martianModels, + ...nvidiaModels, +] +export { engines, models } diff --git a/extensions/inference-anthropic-extension/resources/models.json b/extensions/engine-management-extension/models/anthropic.json similarity index 54% rename from extensions/inference-anthropic-extension/resources/models.json rename to extensions/engine-management-extension/models/anthropic.json index 59e41245b..d35ba4c22 100644 --- a/extensions/inference-anthropic-extension/resources/models.json +++ b/extensions/engine-management-extension/models/anthropic.json @@ -1,74 +1,41 @@ [ { - "sources": [ - { - "url": "https://www.anthropic.com/" - } - ], - "id": "claude-3-opus-latest", + "model": "claude-3-opus-latest", "object": "model", "name": "Claude 3 Opus Latest", "version": "1.0", "description": "Claude 3 Opus is a powerful model suitables for highly complex task.", - "format": "api", - "settings": {}, - "parameters": { + "inference_params": { "max_tokens": 4096, "temperature": 0.7, "stream": false }, - "metadata": { - "author": "Anthropic", - "tags": ["General", "Big Context Length"] - }, "engine": "anthropic" }, { - "sources": [ - { - "url": "https://www.anthropic.com/" - } - ], - "id": "claude-3-5-haiku-latest", + "model": "claude-3-5-haiku-latest", "object": "model", "name": "Claude 3.5 Haiku Latest", "version": "1.0", "description": "Claude 3.5 Haiku is the fastest model provides near-instant responsiveness.", - "format": "api", - "settings": {}, - "parameters": { + "inference_params": { "max_tokens": 8192, "temperature": 0.7, "stream": false }, - "metadata": { - "author": "Anthropic", - "tags": ["General", "Big Context Length"] - }, "engine": "anthropic" }, { - "sources": [ - { - "url": "https://www.anthropic.com/" - } - ], - "id": "claude-3-5-sonnet-latest", + "model": "claude-3-5-sonnet-latest", "object": "model", "name": "Claude 3.5 Sonnet Latest", "version": "1.0", "description": "Claude 3.5 Sonnet raises the industry bar for intelligence, outperforming competitor models and Claude 3 Opus on a wide range of evaluations, with the speed and cost of our mid-tier model, Claude 3 Sonnet.", - "format": "api", - "settings": {}, - "parameters": { + "inference_params": { "max_tokens": 8192, "temperature": 0.7, "stream": true }, - "metadata": { - "author": "Anthropic", - "tags": ["General", "Big Context Length"] - }, "engine": "anthropic" } ] diff --git a/extensions/inference-cohere-extension/resources/models.json b/extensions/engine-management-extension/models/cohere.json similarity index 58% rename from extensions/inference-cohere-extension/resources/models.json rename to extensions/engine-management-extension/models/cohere.json index 2b4cc3e8e..458e4278b 100644 --- a/extensions/inference-cohere-extension/resources/models.json +++ b/extensions/engine-management-extension/models/cohere.json @@ -1,56 +1,28 @@ [ { - "sources": [ - { - "url": "https://cohere.com" - } - ], - "id": "command-r-plus", + "model": "command-r-plus", "object": "model", "name": "Command R+", "version": "1.0", "description": "Command R+ is an instruction-following conversational model that performs language tasks at a higher quality, more reliably, and with a longer context than previous models. It is best suited for complex RAG workflows and multi-step tool use.", - "format": "api", - "settings": {}, - "parameters": { - "max_tokens": 128000, + "inference_params": { + "max_tokens": 4096, "temperature": 0.7, "stream": false }, - "metadata": { - "author": "Cohere", - "tags": [ - "General", - "Big Context Length" - ] - }, "engine": "cohere" }, { - "sources": [ - { - "url": "https://cohere.com" - } - ], - "id": "command-r", + "model": "command-r", "object": "model", "name": "Command R", "version": "1.0", "description": "Command R is an instruction-following conversational model that performs language tasks at a higher quality, more reliably, and with a longer context than previous models. It can be used for complex workflows like code generation, retrieval augmented generation (RAG), tool use, and agents.", - "format": "api", - "settings": {}, - "parameters": { - "max_tokens": 128000, + "inference_params": { + "max_tokens": 4096, "temperature": 0.7, "stream": false }, - "metadata": { - "author": "Cohere", - "tags": [ - "General", - "Big Context Length" - ] - }, "engine": "cohere" } ] diff --git a/extensions/inference-groq-extension/resources/models.json b/extensions/engine-management-extension/models/groq.json similarity index 53% rename from extensions/inference-groq-extension/resources/models.json rename to extensions/engine-management-extension/models/groq.json index b4b013dad..38a0f3835 100644 --- a/extensions/inference-groq-extension/resources/models.json +++ b/extensions/engine-management-extension/models/groq.json @@ -1,18 +1,11 @@ [ { - "sources": [ - { - "url": "https://groq.com" - } - ], - "id": "llama3-70b-8192", + "model": "llama3-70b-8192", "object": "model", "name": "Groq Llama 3 70b", "version": "1.1", "description": "Groq Llama 3 70b with supercharged speed!", - "format": "api", - "settings": {}, - "parameters": { + "inference_params": { "max_tokens": 8192, "temperature": 0.7, "top_p": 0.95, @@ -21,29 +14,15 @@ "frequency_penalty": 0, "presence_penalty": 0 }, - "metadata": { - "author": "Meta", - "tags": [ - "General", - "Big Context Length" - ] - }, "engine": "groq" }, { - "sources": [ - { - "url": "https://groq.com" - } - ], - "id": "llama3-8b-8192", + "model": "llama3-8b-8192", "object": "model", "name": "Groq Llama 3 8b", "version": "1.1", "description": "Groq Llama 3 8b with supercharged speed!", - "format": "api", - "settings": {}, - "parameters": { + "inference_params": { "max_tokens": 8192, "temperature": 0.7, "top_p": 0.95, @@ -52,29 +31,15 @@ "frequency_penalty": 0, "presence_penalty": 0 }, - "metadata": { - "author": "Meta", - "tags": [ - "General", - "Big Context Length" - ] - }, "engine": "groq" }, { - "sources": [ - { - "url": "https://groq.com" - } - ], - "id": "llama-3.1-8b-instant", + "model": "llama-3.1-8b-instant", "object": "model", "name": "Groq Llama 3.1 8b Instant", "version": "1.1", "description": "Groq Llama 3.1 8b with supercharged speed!", - "format": "api", - "settings": {}, - "parameters": { + "inference_params": { "max_tokens": 8000, "temperature": 0.7, "top_p": 0.95, @@ -83,29 +48,15 @@ "frequency_penalty": 0, "presence_penalty": 0 }, - "metadata": { - "author": "Meta", - "tags": [ - "General", - "Big Context Length" - ] - }, "engine": "groq" }, { - "sources": [ - { - "url": "https://groq.com" - } - ], - "id": "llama-3.2-11b-text-preview", + "model": "llama-3.2-11b-text-preview", "object": "model", "name": "Groq Llama 3.2 11b Text Preview", "version": "1.1", "description": "Groq Llama 3.2 11b Text Preview with supercharged speed!", - "format": "api", - "settings": {}, - "parameters": { + "inference_params": { "max_tokens": 8192, "temperature": 0.7, "top_p": 0.95, @@ -114,29 +65,15 @@ "frequency_penalty": 0, "presence_penalty": 0 }, - "metadata": { - "author": "Meta", - "tags": [ - "General", - "Big Context Length" - ] - }, "engine": "groq" }, { - "sources": [ - { - "url": "https://groq.com" - } - ], - "id": "llama-3.2-11b-vision-preview", + "model": "llama-3.2-11b-vision-preview", "object": "model", "name": "Groq Llama 3.2 11b Vision Preview", "version": "1.1", "description": "Groq Llama 3.2 11b Vision Preview with supercharged speed!", - "format": "api", - "settings": {}, - "parameters": { + "inference_params": { "max_tokens": 8192, "temperature": 0.7, "top_p": 0.95, @@ -145,28 +82,14 @@ "frequency_penalty": 0, "presence_penalty": 0 }, - "metadata": { - "author": "Meta", - "tags": [ - "General", - "Big Context Length" - ] - }, "engine": "groq" }, { - "sources": [ - { - "url": "https://groq.com" - } - ], - "id": "llama-3.2-1b-preview", + "model": "llama-3.2-1b-preview", "object": "model", "name": "Groq Llama 3.2 1b Preview", "version": "1.1", "description": "Groq Llama 3.2 1b Preview with supercharged speed!", - "format": "api", - "settings": {}, "parameters": { "max_tokens": 8192, "temperature": 0.7, @@ -176,28 +99,14 @@ "frequency_penalty": 0, "presence_penalty": 0 }, - "metadata": { - "author": "Meta", - "tags": [ - "General", - "Big Context Length" - ] - }, "engine": "groq" }, { - "sources": [ - { - "url": "https://groq.com" - } - ], - "id": "llama-3.2-3b-preview", + "model": "llama-3.2-3b-preview", "object": "model", "name": "Groq Llama 3.2 3b Preview", "version": "1.1", "description": "Groq Llama 3.2 3b Preview with supercharged speed!", - "format": "api", - "settings": {}, "parameters": { "max_tokens": 8192, "temperature": 0.7, @@ -207,28 +116,14 @@ "frequency_penalty": 0, "presence_penalty": 0 }, - "metadata": { - "author": "Meta", - "tags": [ - "General", - "Big Context Length" - ] - }, "engine": "groq" }, { - "sources": [ - { - "url": "https://groq.com" - } - ], - "id": "llama-3.2-90b-text-preview", + "model": "llama-3.2-90b-text-preview", "object": "model", "name": "Groq Llama 3.2 90b Text Preview", "version": "1.1", "description": "Groq Llama 3.2 90b Text Preview with supercharged speed!", - "format": "api", - "settings": {}, "parameters": { "max_tokens": 8192, "temperature": 0.7, @@ -238,28 +133,14 @@ "frequency_penalty": 0, "presence_penalty": 0 }, - "metadata": { - "author": "Meta", - "tags": [ - "General", - "Big Context Length" - ] - }, "engine": "groq" }, { - "sources": [ - { - "url": "https://groq.com" - } - ], - "id": "llama-3.2-90b-vision-preview", + "model": "llama-3.2-90b-vision-preview", "object": "model", "name": "Groq Llama 3.2 90b Vision Preview", "version": "1.1", "description": "Groq Llama 3.2 90b Vision Preview with supercharged speed!", - "format": "api", - "settings": {}, "parameters": { "max_tokens": 8192, "temperature": 0.7, @@ -269,58 +150,14 @@ "frequency_penalty": 0, "presence_penalty": 0 }, - "metadata": { - "author": "Meta", - "tags": [ - "General", - "Big Context Length" - ] - }, "engine": "groq" }, { - "sources": [ - { - "url": "https://groq.com" - } - ], - "id": "gemma-7b-it", - "object": "model", - "name": "Groq Gemma 7B Instruct", - "version": "1.2", - "description": "Groq Gemma 7B Instruct with supercharged speed!", - "format": "api", - "settings": {}, - "parameters": { - "max_tokens": 8192, - "temperature": 0.7, - "top_p": 0.95, - "stream": true, - "stop": [], - "frequency_penalty": 0, - "presence_penalty": 0 - }, - "metadata": { - "author": "Google", - "tags": [ - "General" - ] - }, - "engine": "groq" - }, - { - "sources": [ - { - "url": "https://groq.com" - } - ], - "id": "gemma2-9b-it", + "model": "gemma2-9b-it", "object": "model", "name": "Groq Gemma 9B Instruct", "version": "1.2", "description": "Groq Gemma 9b Instruct with supercharged speed!", - "format": "api", - "settings": {}, "parameters": { "max_tokens": 8192, "temperature": 0.7, @@ -330,27 +167,14 @@ "frequency_penalty": 0, "presence_penalty": 0 }, - "metadata": { - "author": "Google", - "tags": [ - "General" - ] - }, "engine": "groq" }, { - "sources": [ - { - "url": "https://groq.com" - } - ], - "id": "mixtral-8x7b-32768", + "model": "mixtral-8x7b-32768", "object": "model", "name": "Groq Mixtral 8x7B Instruct", "version": "1.2", "description": "Groq Mixtral 8x7B Instruct is Mixtral with supercharged speed!", - "format": "api", - "settings": {}, "parameters": { "max_tokens": 32768, "temperature": 0.7, @@ -360,13 +184,6 @@ "frequency_penalty": 0, "presence_penalty": 0 }, - "metadata": { - "author": "Mistral", - "tags": [ - "General", - "Big Context Length" - ] - }, "engine": "groq" } -] \ No newline at end of file +] diff --git a/extensions/inference-martian-extension/resources/models.json b/extensions/engine-management-extension/models/martian.json similarity index 63% rename from extensions/inference-martian-extension/resources/models.json rename to extensions/engine-management-extension/models/martian.json index cf59e958e..b935587cc 100644 --- a/extensions/inference-martian-extension/resources/models.json +++ b/extensions/engine-management-extension/models/martian.json @@ -1,17 +1,10 @@ [ { - "sources": [ - { - "url": "https://withmartian.com/" - } - ], - "id": "router", + "model": "router", "object": "model", "name": "Martian Model Router", "version": "1.0", "description": "Martian Model Router dynamically routes requests to the best LLM in real-time", - "format": "api", - "settings": {}, "parameters": { "max_tokens": 4096, "temperature": 0.7, @@ -21,12 +14,6 @@ "frequency_penalty": 0, "presence_penalty": 0 }, - "metadata": { - "author": "Martian", - "tags": [ - "General" - ] - }, "engine": "martian" } -] \ No newline at end of file +] diff --git a/extensions/inference-mistral-extension/resources/models.json b/extensions/engine-management-extension/models/mistral.json similarity index 58% rename from extensions/inference-mistral-extension/resources/models.json rename to extensions/engine-management-extension/models/mistral.json index 23ecd6fdd..47833a31c 100644 --- a/extensions/inference-mistral-extension/resources/models.json +++ b/extensions/engine-management-extension/models/mistral.json @@ -1,83 +1,44 @@ [ { - "sources": [ - { - "url": "https://docs.mistral.ai/api/" - } - ], - "id": "mistral-small-latest", + "model": "mistral-small-latest", "object": "model", "name": "Mistral Small", "version": "1.1", "description": "Mistral Small is the ideal choice for simple tasks (Classification, Customer Support, or Text Generation) at an affordable price.", - "format": "api", - "settings": {}, "parameters": { "max_tokens": 32000, "temperature": 0.7, "top_p": 0.95, "stream": true }, - "metadata": { - "author": "Mistral", - "tags": [ - "General" - ] - }, "engine": "mistral" }, { - "sources": [ - { - "url": "https://docs.mistral.ai/api/" - } - ], - "id": "mistral-large-latest", + "model": "mistral-large-latest", "object": "model", "name": "Mistral Large", "version": "1.1", "description": "Mistral Large is ideal for complex tasks (Synthetic Text Generation, Code Generation, RAG, or Agents).", - "format": "api", - "settings": {}, "parameters": { "max_tokens": 32000, "temperature": 0.7, "top_p": 0.95, "stream": true }, - "metadata": { - "author": "Mistral", - "tags": [ - "General" - ] - }, "engine": "mistral" }, { - "sources": [ - { - "url": "https://docs.mistral.ai/api/" - } - ], - "id": "open-mixtral-8x22b", + "model": "open-mixtral-8x22b", "object": "model", "name": "Mixtral 8x22B", "version": "1.1", "description": "Mixtral 8x22B is a high-performance, cost-effective model designed for complex tasks.", - "format": "api", - "settings": {}, "parameters": { "max_tokens": 32000, "temperature": 0.7, "top_p": 0.95, "stream": true }, - "metadata": { - "author": "Mistral", - "tags": [ - "General" - ] - }, "engine": "mistral" } ] diff --git a/extensions/inference-nvidia-extension/resources/models.json b/extensions/engine-management-extension/models/nvidia.json similarity index 57% rename from extensions/inference-nvidia-extension/resources/models.json rename to extensions/engine-management-extension/models/nvidia.json index b97644fc9..f2adac779 100644 --- a/extensions/inference-nvidia-extension/resources/models.json +++ b/extensions/engine-management-extension/models/nvidia.json @@ -1,17 +1,10 @@ [ { - "sources": [ - { - "url": "https://integrate.api.nvidia.com/v1/chat/completions" - } - ], - "id": "mistralai/mistral-7b-instruct-v0.2", + "model": "mistralai/mistral-7b-instruct-v0.2", "object": "model", "name": "Mistral 7B", "version": "1.1", "description": "Mistral 7B with NVIDIA", - "format": "api", - "settings": {}, "parameters": { "max_tokens": 1024, "temperature": 0.3, @@ -22,10 +15,6 @@ "stop": null, "seed": null }, - "metadata": { - "author": "NVIDIA", - "tags": ["General"] - }, "engine": "nvidia" } ] diff --git a/extensions/inference-openai-extension/resources/models.json b/extensions/engine-management-extension/models/openai.json similarity index 62% rename from extensions/inference-openai-extension/resources/models.json rename to extensions/engine-management-extension/models/openai.json index 0c822fde2..8f59b42ea 100644 --- a/extensions/inference-openai-extension/resources/models.json +++ b/extensions/engine-management-extension/models/openai.json @@ -1,18 +1,12 @@ [ { - "sources": [ - { - "url": "https://openai.com" - } - ], - "id": "gpt-4-turbo", + "model": "gpt-4-turbo", "object": "model", "name": "OpenAI GPT 4 Turbo", "version": "1.2", "description": "OpenAI GPT 4 Turbo model is extremely good", "format": "api", - "settings": {}, - "parameters": { + "inference_params": { "max_tokens": 4096, "temperature": 0.7, "top_p": 0.95, @@ -21,26 +15,16 @@ "frequency_penalty": 0, "presence_penalty": 0 }, - "metadata": { - "author": "OpenAI", - "tags": ["General"] - }, "engine": "openai" }, { - "sources": [ - { - "url": "https://openai.com" - } - ], - "id": "gpt-3.5-turbo", + "model": "gpt-3.5-turbo", "object": "model", "name": "OpenAI GPT 3.5 Turbo", "version": "1.1", "description": "OpenAI GPT 3.5 Turbo model is extremely fast", "format": "api", - "settings": {}, - "parameters": { + "inference_params": { "max_tokens": 4096, "temperature": 0.7, "top_p": 0.95, @@ -49,28 +33,16 @@ "frequency_penalty": 0, "presence_penalty": 0 }, - "metadata": { - "author": "OpenAI", - "tags": ["General"] - }, "engine": "openai" }, { - "sources": [ - { - "url": "https://openai.com" - } - ], - "id": "gpt-4o", + "model": "gpt-4o", "object": "model", "name": "OpenAI GPT 4o", "version": "1.1", "description": "OpenAI GPT 4o is a new flagship model with fast speed and high quality", "format": "api", - "settings": { - "vision_model": true - }, - "parameters": { + "inference_params": { "max_tokens": 4096, "temperature": 0.7, "top_p": 0.95, @@ -79,28 +51,16 @@ "frequency_penalty": 0, "presence_penalty": 0 }, - "metadata": { - "author": "OpenAI", - "tags": ["General"] - }, "engine": "openai" }, { - "sources": [ - { - "url": "https://openai.com" - } - ], - "id": "gpt-4o-mini", + "model": "gpt-4o-mini", "object": "model", "name": "OpenAI GPT 4o-mini", "version": "1.1", "description": "GPT-4o mini (“o” for “omni”) is a fast, affordable small model for focused tasks.", "format": "api", - "settings": { - "vision_model": true - }, - "parameters": { + "inference_params": { "max_tokens": 16384, "temperature": 0.7, "top_p": 0.95, @@ -109,26 +69,16 @@ "frequency_penalty": 0, "presence_penalty": 0 }, - "metadata": { - "author": "OpenAI", - "tags": ["General"] - }, "engine": "openai" }, { - "sources": [ - { - "url": "https://openai.com" - } - ], - "id": "o1", + "model": "o1", "object": "model", "name": "OpenAI o1", "version": "1.0", "description": "OpenAI o1 is a new model with complex reasoning", "format": "api", - "settings": {}, - "parameters": { + "inference_params": { "max_tokens": 100000, "temperature": 1, "top_p": 1, @@ -136,26 +86,16 @@ "frequency_penalty": 0, "presence_penalty": 0 }, - "metadata": { - "author": "OpenAI", - "tags": ["General"] - }, "engine": "openai" }, { - "sources": [ - { - "url": "https://openai.com" - } - ], - "id": "o1-preview", + "model": "o1-preview", "object": "model", "name": "OpenAI o1-preview", "version": "1.0", "description": "OpenAI o1-preview is a new model with complex reasoning", "format": "api", - "settings": {}, - "parameters": { + "inference_params": { "max_tokens": 32768, "temperature": 1, "top_p": 1, @@ -163,26 +103,16 @@ "frequency_penalty": 0, "presence_penalty": 0 }, - "metadata": { - "author": "OpenAI", - "tags": ["General"] - }, "engine": "openai" }, { - "sources": [ - { - "url": "https://openai.com" - } - ], - "id": "o1-mini", + "model": "o1-mini", "object": "model", "name": "OpenAI o1-mini", "version": "1.0", "description": "OpenAI o1-mini is a lightweight reasoning model", "format": "api", - "settings": {}, - "parameters": { + "inference_params": { "max_tokens": 65536, "temperature": 1, "top_p": 1, @@ -190,10 +120,6 @@ "frequency_penalty": 0, "presence_penalty": 0 }, - "metadata": { - "author": "OpenAI", - "tags": ["General"] - }, "engine": "openai" } ] diff --git a/extensions/inference-openrouter-extension/resources/models.json b/extensions/engine-management-extension/models/openrouter.json similarity index 63% rename from extensions/inference-openrouter-extension/resources/models.json rename to extensions/engine-management-extension/models/openrouter.json index 31dea8734..5ac189a81 100644 --- a/extensions/inference-openrouter-extension/resources/models.json +++ b/extensions/engine-management-extension/models/openrouter.json @@ -1,17 +1,10 @@ [ { - "sources": [ - { - "url": "https://openrouter.ai" - } - ], - "id": "open-router-auto", + "model": "open-router-auto", "object": "model", "name": "OpenRouter", "version": "1.0", "description": " OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.", - "format": "api", - "settings": {}, "parameters": { "max_tokens": 128000, "temperature": 0.7, @@ -19,10 +12,6 @@ "frequency_penalty": 0, "presence_penalty": 0 }, - "metadata": { - "author": "OpenRouter", - "tags": ["General", "Big Context Length"] - }, "engine": "openrouter" } ] diff --git a/extensions/engine-management-extension/resources/anthropic.json b/extensions/engine-management-extension/resources/anthropic.json new file mode 100644 index 000000000..a599d7ce3 --- /dev/null +++ b/extensions/engine-management-extension/resources/anthropic.json @@ -0,0 +1,22 @@ +{ + "id": "@janhq/inference-anthropic-extension", + "type": "remote", + "engine": "anthropic", + "url": "https://api.anthropic.com", + "api_key": "", + "metadata": { + "get_models_url": "https://api.anthropic.com/v1/models", + "header_template": "x-api-key: {{api_key}} anthropic-version: 2023-06-01", + "transform_req": { + "chat_completions": { + "url": "https://api.anthropic.com/v1/messages", + "template": "{ {% for key, value in input_request %} {% if key == \"messages\" %} {% if input_request.messages.0.role == \"system\" %} \"system\": \"{{ input_request.messages.0.content }}\", \"messages\": [{% for message in input_request.messages %} {% if not loop.is_first %} {\"role\": \"{{ message.role }}\", \"content\": \"{{ message.content }}\" } {% if not loop.is_last %},{% endif %} {% endif %} {% endfor %}] {% else %} \"messages\": [{% for message in input_request.messages %} {\"role\": \"{{ message.role}}\", \"content\": \"{{ message.content }}\" } {% if not loop.is_last %},{% endif %} {% endfor %}] {% endif %} {% if not loop.is_last %},{% endif %} {% else if key == \"system\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %}\"{{ key }}\": {{ tojson(value) }} {% if not loop.is_last %},{% endif %} {% endif %} {% endfor %} }" + } + }, + "transform_resp": { + "chat_completions": { + "template": "{% if input_request.stream %} {\"object\": \"chat.completion.chunk\", \"model\": \"{{ input_request.model }}\", \"choices\": [{\"index\": 0, \"delta\": { {% if input_request.type == \"message_start\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"ping\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"content_block_delta\" %} \"role\": \"assistant\", \"content\": \"{{ input_request.delta.text }}\" {% else if input_request.type == \"content_block_stop\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"content_block_stop\" %} \"role\": \"assistant\", \"content\": null {% endif %} }, {% if input_request.type == \"content_block_stop\" %} \"finish_reason\": \"stop\" {% else %} \"finish_reason\": null {% endif %} }]} {% else %} {\"id\": \"{{ input_request.id }}\", \"created\": null, \"object\": \"chat.completion\", \"model\": \"{{ input_request.model }}\", \"choices\": [{ \"index\": 0, \"message\": { \"role\": \"{{ input_request.role }}\", \"content\": \"{% if input_request.content and input_request.content.0.type == \"text\" %} \"{{input_request.content.0.text}}\" {% endif %}\", \"refusal\": null }, \"logprobs\": null, \"finish_reason\": \"{{ input_request.stop_reason }}\" } ], \"usage\": { \"prompt_tokens\": {{ input_request.usage.input_tokens }}, \"completion_tokens\": {{ input_request.usage.output_tokens }}, \"total_tokens\": {{ input_request.usage.input_tokens + input_request.usage.output_tokens }}, \"prompt_tokens_details\": { \"cached_tokens\": 0 }, \"completion_tokens_details\": { \"reasoning_tokens\": 0, \"accepted_prediction_tokens\": 0, \"rejected_prediction_tokens\": 0 } }, \"system_fingerprint\": \"fp_6b68a8204b\"} {% endif %}" + } + } + } +} diff --git a/extensions/engine-management-extension/resources/cohere.json b/extensions/engine-management-extension/resources/cohere.json new file mode 100644 index 000000000..6cb51dc04 --- /dev/null +++ b/extensions/engine-management-extension/resources/cohere.json @@ -0,0 +1,22 @@ +{ + "id": "@janhq/inference-cohere-extension", + "type": "remote", + "engine": "cohere", + "url": "https://api.cohere.ai", + "api_key": "", + "metadata": { + "get_models_url": "https://api.cohere.ai/v1/models", + "header_template": "Authorization: Bearer {{api_key}}", + "transform_req": { + "chat_completions": { + "url": "https://api.cohere.ai/v1/chat", + "template": "{ {% for key, value in input_request %} {% if key == \"messages\" %} {% if input_request.messages.0.role == \"system\" %} \"preamble\": \"{{ input_request.messages.0.content }}\", {% if length(input_request.messages) > 2 %} \"chatHistory\": [{% for message in input_request.messages %} {% if not loop.is_first and not loop.is_last %} {\"role\": {% if message.role == \"user\" %} \"USER\" {% else %} \"CHATBOT\" {% endif %}, \"content\": \"{{ message.content }}\" } {% if loop.index < length(input_request.messages) - 2 %},{% endif %} {% endif %} {% endfor %}], {% endif %} \"message\": \"{{ last(input_request.messages).content }}\" {% else %} {% if length(input_request.messages) > 2 %} \"chatHistory\": [{% for message in input_request.messages %} {% if not loop.is_last %} { \"role\": {% if message.role == \"user\" %} \"USER\" {% else %} \"CHATBOT\" {% endif %}, \"content\": \"{{ message.content }}\" } {% if loop.index < length(input_request.messages) - 2 %},{% endif %} {% endif %} {% endfor %}],{% endif %}\"message\": \"{{ last(input_request.messages).content }}\" {% endif %}{% if not loop.is_last %},{% endif %} {% else if key == \"system\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} \"{{ key }}\": {{ tojson(value) }} {% if not loop.is_last %},{% endif %} {% endif %} {% endfor %} }" + } + }, + "transform_resp": { + "chat_completions": { + "template": "{% if input_request.stream %} {\"object\": \"chat.completion.chunk\", \"model\": \"{{ input_request.model }}\", \"choices\": [{\"index\": 0, \"delta\": { {% if input_request.event_type == \"text-generation\" %} \"role\": \"assistant\", \"content\": \"{{ input_request.text }}\" {% else %} \"role\": \"assistant\", \"content\": null {% endif %} }, {% if input_request.event_type == \"stream-end\" %} \"finish_reason\": \"{{ input_request.finish_reason }}\" {% else %} \"finish_reason\": null {% endif %} }]} {% else %} {\"id\": \"{{ input_request.generation_id }}\", \"created\": null, \"object\": \"chat.completion\", \"model\": {% if input_request.model %} \"{{ input_request.model }}\" {% else %} \"command-r-plus-08-2024\" {% endif %}, \"choices\": [{ \"index\": 0, \"message\": { \"role\": \"assistant\", \"content\": {% if not input_request.text %} null {% else %} \"{{ input_request.text }}\" {% endif %}, \"refusal\": null }, \"logprobs\": null, \"finish_reason\": \"{{ input_request.finish_reason }}\" } ], \"usage\": { \"prompt_tokens\": {{ input_request.meta.tokens.input_tokens }}, \"completion_tokens\": {{ input_request.meta.tokens.output_tokens }},\"total_tokens\": {{ input_request.meta.tokens.input_tokens + input_request.meta.tokens.output_tokens }}, \"prompt_tokens_details\": { \"cached_tokens\": 0 },\"completion_tokens_details\": { \"reasoning_tokens\": 0, \"accepted_prediction_tokens\": 0, \"rejected_prediction_tokens\": 0 } }, \"system_fingerprint\": \"fp_6b68a8204b\"} {% endif %}" + } + } + } +} diff --git a/extensions/engine-management-extension/resources/groq.json b/extensions/engine-management-extension/resources/groq.json new file mode 100644 index 000000000..6be1ce6ef --- /dev/null +++ b/extensions/engine-management-extension/resources/groq.json @@ -0,0 +1,22 @@ +{ + "id": "@janhq/inference-groq-extension", + "type": "remote", + "engine": "groq", + "url": "https://api.groq.com", + "api_key": "", + "metadata": { + "get_models_url": "https://api.groq.com/openai/v1/models", + "header_template": "Authorization: Bearer {{api_key}}", + "transform_req": { + "chat_completions": { + "url": "https://api.groq.com/openai/v1/chat/completions", + "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }" + } + }, + "transform_resp": { + "chat_completions": { + "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"choices\" or key == \"created\" or key == \"model\" or key == \"service_tier\" or key == \"system_fingerprint\" or key == \"stream\" or key == \"object\" or key == \"usage\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }" + } + } + } +} diff --git a/extensions/engine-management-extension/resources/martian.json b/extensions/engine-management-extension/resources/martian.json new file mode 100644 index 000000000..4523ab754 --- /dev/null +++ b/extensions/engine-management-extension/resources/martian.json @@ -0,0 +1,22 @@ +{ + "id": "@janhq/inference-martian-extension", + "type": "remote", + "engine": "martian", + "url": "https://withmartian.com", + "api_key": "", + "metadata": { + "get_models_url": "https://withmartian.com/api/openai/v1/models", + "header_template": "Authorization: Bearer {{api_key}}", + "transform_req": { + "chat_completions": { + "url": "https://withmartian.com/api/openai/v1/chat/completions", + "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }" + } + }, + "transform_resp": { + "chat_completions": { + "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"choices\" or key == \"created\" or key == \"model\" or key == \"service_tier\" or key == \"system_fingerprint\" or key == \"stream\" or key == \"object\" or key == \"usage\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }" + } + } + } +} diff --git a/extensions/engine-management-extension/resources/mistral.json b/extensions/engine-management-extension/resources/mistral.json new file mode 100644 index 000000000..4a032c4c8 --- /dev/null +++ b/extensions/engine-management-extension/resources/mistral.json @@ -0,0 +1,22 @@ +{ + "id": "@janhq/inference-mistral-extension", + "type": "remote", + "engine": "mistral", + "url": "https://api.mistral.ai", + "api_key": "", + "metadata": { + "get_models_url": "https://api.mistral.ai/v1/models", + "header_template": "Authorization: Bearer {{api_key}}", + "transform_req": { + "chat_completions": { + "url": "https://api.mistral.ai/v1/chat/completions", + "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }" + } + }, + "transform_resp": { + "chat_completions": { + "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"choices\" or key == \"created\" or key == \"model\" or key == \"service_tier\" or key == \"system_fingerprint\" or key == \"stream\" or key == \"object\" or key == \"usage\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }" + } + } + } +} diff --git a/extensions/engine-management-extension/resources/nvidia.json b/extensions/engine-management-extension/resources/nvidia.json new file mode 100644 index 000000000..fa224c32c --- /dev/null +++ b/extensions/engine-management-extension/resources/nvidia.json @@ -0,0 +1,22 @@ +{ + "id": "@janhq/inference-nvidia-extension", + "type": "remote", + "engine": "nvidia", + "url": "https://integrate.api.nvidia.com", + "api_key": "", + "metadata": { + "get_models_url": "https://integrate.api.nvidia.com/v1/models", + "header_template": "Authorization: Bearer {{api_key}}", + "transform_req": { + "chat_completions": { + "url": "https://integrate.api.nvidia.com/v1/chat/completions", + "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }" + } + }, + "transform_resp": { + "chat_completions": { + "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"choices\" or key == \"created\" or key == \"model\" or key == \"service_tier\" or key == \"system_fingerprint\" or key == \"stream\" or key == \"object\" or key == \"usage\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }" + } + } + } +} diff --git a/extensions/engine-management-extension/resources/openai.json b/extensions/engine-management-extension/resources/openai.json new file mode 100644 index 000000000..8f86a1785 --- /dev/null +++ b/extensions/engine-management-extension/resources/openai.json @@ -0,0 +1,22 @@ +{ + "id": "@janhq/inference-openai-extension", + "type": "remote", + "engine": "openai", + "url": "https://api.openai.com", + "api_key": "", + "metadata": { + "get_models_url": "https://api.openai.com/v1/models", + "header_template": "Authorization: Bearer {{api_key}}", + "transform_req": { + "chat_completions": { + "url": "https://api.openai.com/v1/chat/completions", + "template": "{ {% set first = true %}{% for key, value in input_request %}{% if key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"messages\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" or (not \"o1\" in input_request.model and (key == \"max_tokens\" or key == \"stop\")) %} {% if key == \"max_tokens\" and \"o1\" in input_request.model %} \"max_completion_tokens\": {{ tojson(value) }} {% else %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endif %} {% endfor %} }" + } + }, + "transform_resp": { + "chat_completions": { + "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"choices\" or key == \"created\" or key == \"model\" or key == \"service_tier\" or key == \"stream\" or key == \"object\" or key == \"usage\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }" + } + } + } +} diff --git a/extensions/engine-management-extension/resources/openrouter.json b/extensions/engine-management-extension/resources/openrouter.json new file mode 100644 index 000000000..631b211fa --- /dev/null +++ b/extensions/engine-management-extension/resources/openrouter.json @@ -0,0 +1,22 @@ +{ + "id": "@janhq/inference-openrouter-extension", + "type": "remote", + "engine": "openrouter", + "url": "https://openrouter.ai", + "api_key": "", + "metadata": { + "get_models_url": "https://openrouter.ai/api/v1/models", + "header_template": "Authorization: Bearer {{api_key}}", + "transform_req": { + "chat_completions": { + "url": "https://openrouter.ai/api/v1/chat/completions", + "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }" + } + }, + "transform_resp": { + "chat_completions": { + "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"choices\" or key == \"created\" or key == \"model\" or key == \"service_tier\" or key == \"system_fingerprint\" or key == \"stream\" or key == \"object\" or key == \"usage\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }" + } + } + } +} diff --git a/extensions/engine-management-extension/rolldown.config.mjs b/extensions/engine-management-extension/rolldown.config.mjs index 038f23cc3..b59d395d1 100644 --- a/extensions/engine-management-extension/rolldown.config.mjs +++ b/extensions/engine-management-extension/rolldown.config.mjs @@ -1,4 +1,5 @@ import { defineConfig } from 'rolldown' +import { engines, models } from './engines.mjs' import pkgJson from './package.json' with { type: 'json' } export default defineConfig([ @@ -13,6 +14,8 @@ export default defineConfig([ API_URL: JSON.stringify('http://127.0.0.1:39291'), SOCKET_URL: JSON.stringify('ws://127.0.0.1:39291'), CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.43'), + DEFAULT_REMOTE_ENGINES: JSON.stringify(engines), + DEFAULT_REMOTE_MODELS: JSON.stringify(models), }, }, { diff --git a/extensions/engine-management-extension/src/@types/global.d.ts b/extensions/engine-management-extension/src/@types/global.d.ts index 8d0a94fef..2d520d5f9 100644 --- a/extensions/engine-management-extension/src/@types/global.d.ts +++ b/extensions/engine-management-extension/src/@types/global.d.ts @@ -3,6 +3,12 @@ declare const CORTEX_ENGINE_VERSION: string declare const SOCKET_URL: string declare const NODE: string +declare const DEFAULT_REMOTE_ENGINES: ({ + id: string + engine: string +} & EngineConfig)[] +declare const DEFAULT_REMOTE_MODELS: Model[] + interface Core { api: APIFunctions events: EventEmitter diff --git a/extensions/engine-management-extension/src/index.ts b/extensions/engine-management-extension/src/index.ts index 215ae3bc2..0d30bf4ea 100644 --- a/extensions/engine-management-extension/src/index.ts +++ b/extensions/engine-management-extension/src/index.ts @@ -3,14 +3,22 @@ import { InferenceEngine, DefaultEngineVariant, Engines, + EngineConfig, EngineVariant, EngineReleased, executeOnMain, systemInformation, + Model, + fs, + joinPath, + events, + ModelEvent, + EngineEvent, } from '@janhq/core' import ky, { HTTPError } from 'ky' import PQueue from 'p-queue' import { EngineError } from './error' +import { getJanDataFolderPath } from '@janhq/core' /** * JSONEngineManagementExtension is a EngineManagementExtension implementation that provides @@ -27,41 +35,11 @@ export default class JSONEngineManagementExtension extends EngineManagementExten await executeOnMain(NODE, 'symlinkEngines') // Run Healthcheck this.queue.add(() => this.healthz()) - try { - const variant = await this.getDefaultEngineVariant( - InferenceEngine.cortex_llamacpp - ) - const installedEngines = await this.getInstalledEngines( - InferenceEngine.cortex_llamacpp - ) - if ( - !installedEngines.some( - (e) => e.name === variant.variant && e.version === variant.version - ) - ) { - throw new EngineError( - 'Default engine is not available, use bundled version.' - ) - } - } catch (error) { - if ( - (error instanceof HTTPError && error.response.status === 400) || - error instanceof EngineError - ) { - const systemInfo = await systemInformation() - const variant = await executeOnMain( - NODE, - 'engineVariant', - systemInfo.gpuSetting - ) - await this.setDefaultEngineVariant(InferenceEngine.cortex_llamacpp, { - variant: variant, - version: `${CORTEX_ENGINE_VERSION}`, - }) - } else { - console.error('An unexpected error occurred:', error) - } - } + // Update default local engine + this.updateDefaultEngine() + + // Populate default remote engines + this.populateDefaultRemoteEngines() } /** @@ -81,6 +59,19 @@ export default class JSONEngineManagementExtension extends EngineManagementExten ) as Promise } + /** + * @returns A Promise that resolves to an object of list engines. + */ + async getRemoteModels(name: string): Promise { + return this.queue.add(() => + ky + .get(`${API_URL}/v1/models/remote/${name}`) + .json() + .then((e) => e) + .catch(() => []) + ) as Promise + } + /** * @param name - Inference engine name. * @returns A Promise that resolves to an array of installed engine. @@ -135,10 +126,7 @@ export default class JSONEngineManagementExtension extends EngineManagementExten * @param name - Inference engine name. * @returns A Promise that resolves to intall of engine. */ - async installEngine( - name: InferenceEngine, - engineConfig: { variant: string; version?: string } - ) { + async installEngine(name: string, engineConfig: EngineConfig) { return this.queue.add(() => ky .post(`${API_URL}/v1/engines/${name}/install`, { json: engineConfig }) @@ -146,14 +134,21 @@ export default class JSONEngineManagementExtension extends EngineManagementExten ) as Promise<{ messages: string }> } + /** + * Add a new remote engine + * @returns A Promise that resolves to intall of engine. + */ + async addRemoteEngine(engineConfig: EngineConfig) { + return this.queue.add(() => + ky.post(`${API_URL}/v1/engines`, { json: engineConfig }).then((e) => e) + ) as Promise<{ messages: string }> + } + /** * @param name - Inference engine name. * @returns A Promise that resolves to unintall of engine. */ - async uninstallEngine( - name: InferenceEngine, - engineConfig: { variant: string; version: string } - ) { + async uninstallEngine(name: InferenceEngine, engineConfig: EngineConfig) { return this.queue.add(() => ky .delete(`${API_URL}/v1/engines/${name}/install`, { json: engineConfig }) @@ -161,6 +156,16 @@ export default class JSONEngineManagementExtension extends EngineManagementExten ) as Promise<{ messages: string }> } + /** + * Add a new remote model + * @param model - Remote model object. + */ + async addRemoteModel(model: Model) { + return this.queue.add(() => + ky.post(`${API_URL}/v1/models/add`, { json: model }).then((e) => e) + ) + } + /** * @param name - Inference engine name. * @returns A Promise that resolves to an object of default engine. @@ -181,7 +186,7 @@ export default class JSONEngineManagementExtension extends EngineManagementExten */ async setDefaultEngineVariant( name: InferenceEngine, - engineConfig: { variant: string; version: string } + engineConfig: EngineConfig ) { return this.queue.add(() => ky @@ -193,9 +198,11 @@ export default class JSONEngineManagementExtension extends EngineManagementExten /** * @returns A Promise that resolves to update engine. */ - async updateEngine(name: InferenceEngine) { + async updateEngine(name: InferenceEngine, engineConfig?: EngineConfig) { return this.queue.add(() => - ky.post(`${API_URL}/v1/engines/${name}/update`).then((e) => e) + ky + .post(`${API_URL}/v1/engines/${name}/update`, { json: engineConfig }) + .then((e) => e) ) as Promise<{ messages: string }> } @@ -210,4 +217,90 @@ export default class JSONEngineManagementExtension extends EngineManagementExten }) .then(() => {}) } + + /** + * Update default local engine + * This is to use built-in engine variant in case there is no default engine set + */ + async updateDefaultEngine() { + try { + const variant = await this.getDefaultEngineVariant( + InferenceEngine.cortex_llamacpp + ) + const installedEngines = await this.getInstalledEngines( + InferenceEngine.cortex_llamacpp + ) + if ( + !installedEngines.some( + (e) => e.name === variant.variant && e.version === variant.version + ) + ) { + throw new EngineError( + 'Default engine is not available, use bundled version.' + ) + } + } catch (error) { + if ( + (error instanceof HTTPError && error.response.status === 400) || + error instanceof EngineError + ) { + const systemInfo = await systemInformation() + const variant = await executeOnMain( + NODE, + 'engineVariant', + systemInfo.gpuSetting + ) + await this.setDefaultEngineVariant(InferenceEngine.cortex_llamacpp, { + variant: variant, + version: `${CORTEX_ENGINE_VERSION}`, + }) + } else { + console.error('An unexpected error occurred:', error) + } + } + } + + /** + * This is to populate default remote engines in case there is no customized remote engine setting + */ + async populateDefaultRemoteEngines() { + const engines = await this.getEngines() + if ( + !Object.values(engines) + .flat() + .some((e) => e.type === 'remote') + ) { + await Promise.all( + DEFAULT_REMOTE_ENGINES.map(async (engine) => { + const { id, ...data } = engine + + /// BEGIN - Migrate legacy api key settings + let api_key = undefined + if (id) { + const apiKeyPath = await joinPath([ + await getJanDataFolderPath(), + 'settings', + id, + 'settings.json', + ]) + if (await fs.existsSync(apiKeyPath)) { + const settings = await fs.readFileSync(apiKeyPath, 'utf-8') + api_key = JSON.parse(settings).find( + (e) => e.key === `${data.engine}-api-key` + )?.controllerProps?.value + } + } + data.api_key = api_key + /// END - Migrate legacy api key settings + + await this.addRemoteEngine(data).catch(console.error) + }) + ) + events.emit(EngineEvent.OnEngineUpdate, {}) + DEFAULT_REMOTE_MODELS.forEach(async (data: Model) => { + await this.addRemoteModel(data).catch(() => {}) + }) + events.emit(ModelEvent.OnModelsUpdate, { fetch: true }) + } + } } diff --git a/extensions/engine-management-extension/tsconfig.json b/extensions/engine-management-extension/tsconfig.json index 891d28a60..72e1e1895 100644 --- a/extensions/engine-management-extension/tsconfig.json +++ b/extensions/engine-management-extension/tsconfig.json @@ -8,7 +8,8 @@ "forceConsistentCasingInFileNames": true, "strict": false, "skipLibCheck": true, - "rootDir": "./src" + "rootDir": "./src", + "resolveJsonModule": true }, "include": ["./src"], "exclude": ["src/**/*.test.ts", "rolldown.config.mjs"] diff --git a/extensions/inference-anthropic-extension/README.md b/extensions/inference-anthropic-extension/README.md deleted file mode 100644 index 1c0dcbd3d..000000000 --- a/extensions/inference-anthropic-extension/README.md +++ /dev/null @@ -1,79 +0,0 @@ -# Anthropic Engine Extension - -Created using Jan extension example - -# Create a Jan Extension using Typescript - -Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀 - -## Create Your Own Extension - -To create your own extension, you can use this repository as a template! Just follow the below instructions: - -1. Click the Use this template button at the top of the repository -2. Select Create a new repository -3. Select an owner and name for your new repository -4. Click Create repository -5. Clone your new repository - -## Initial Setup - -After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension. - -> [!NOTE] -> -> You'll need to have a reasonably modern version of -> [Node.js](https://nodejs.org) handy. If you are using a version manager like -> [`nodenv`](https://github.com/nodenv/nodenv) or -> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the -> root of your repository to install the version specified in -> [`package.json`](./package.json). Otherwise, 20.x or later should work! - -1. :hammer_and_wrench: Install the dependencies - - ```bash - npm install - ``` - -1. :building_construction: Package the TypeScript for distribution - - ```bash - npm run bundle - ``` - -1. :white_check_mark: Check your artifact - - There will be a tgz file in your extension directory now - -## Update the Extension Metadata - -The [`package.json`](package.json) file defines metadata about your extension, such as -extension name, main entry, description and version. - -When you copy this repository, update `package.json` with the name, description for your extension. - -## Update the Extension Code - -The [`src/`](./src/) directory is the heart of your extension! This contains the -source code that will be run when your extension functions are invoked. You can replace the -contents of this directory with your own code. - -There are a few things to keep in mind when writing your extension code: - -- Most Jan Extension functions are processed asynchronously. - In `index.ts`, you will see that the extension function will return a `Promise`. - - ```typescript - import { events, MessageEvent, MessageRequest } from '@janhq/core' - - function onStart(): Promise { - return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => - this.inference(data) - ) - } - ``` - - For more information about the Jan Extension Core module, see the - [documentation](https://github.com/janhq/jan/blob/main/core/README.md). - -So, what are you waiting for? Go ahead and start customizing your extension! diff --git a/extensions/inference-anthropic-extension/jest.config.js b/extensions/inference-anthropic-extension/jest.config.js deleted file mode 100644 index 3e32adceb..000000000 --- a/extensions/inference-anthropic-extension/jest.config.js +++ /dev/null @@ -1,9 +0,0 @@ -/** @type {import('ts-jest').JestConfigWithTsJest} */ -module.exports = { - preset: 'ts-jest', - testEnvironment: 'node', - transform: { - 'node_modules/@janhq/core/.+\\.(j|t)s?$': 'ts-jest', - }, - transformIgnorePatterns: ['node_modules/(?!@janhq/core/.*)'], -} diff --git a/extensions/inference-anthropic-extension/package.json b/extensions/inference-anthropic-extension/package.json deleted file mode 100644 index cb064d2aa..000000000 --- a/extensions/inference-anthropic-extension/package.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "name": "@janhq/inference-anthropic-extension", - "productName": "Anthropic Inference Engine", - "version": "1.0.3", - "description": "This extension enables Anthropic chat completion API calls", - "main": "dist/index.js", - "engine": "anthropic", - "author": "Jan ", - "license": "AGPL-3.0", - "scripts": { - "test": "jest test", - "build": "rolldown -c rolldown.config.mjs", - "build:publish": "rimraf *.tgz --glob || true && yarn build && npm pack && cpx *.tgz ../../pre-install" - }, - "devDependencies": { - "cpx": "^1.5.0", - "rimraf": "^3.0.2", - "rolldown": "1.0.0-beta.1", - "ts-loader": "^9.5.0", - "typescript": "^5.7.2" - }, - "dependencies": { - "@janhq/core": "../../core/package.tgz", - "fetch-retry": "^5.0.6", - "ulidx": "^2.3.0" - }, - "engines": { - "node": ">=18.0.0" - }, - "files": [ - "dist/*", - "package.json", - "README.md" - ], - "bundleDependencies": [ - "fetch-retry" - ], - "installConfig": { - "hoistingLimits": "workspaces" - }, - "packageManager": "yarn@4.5.3" -} diff --git a/extensions/inference-anthropic-extension/resources/settings.json b/extensions/inference-anthropic-extension/resources/settings.json deleted file mode 100644 index 9ca4405ac..000000000 --- a/extensions/inference-anthropic-extension/resources/settings.json +++ /dev/null @@ -1,24 +0,0 @@ -[ - { - "key": "anthropic-api-key", - "title": "API Key", - "description": "The Anthropic API uses API keys for authentication. Visit your [API Keys](https://console.anthropic.com/settings/keys) page to retrieve the API key you'll use in your requests.", - "controllerType": "input", - "controllerProps": { - "placeholder": "Insert API Key", - "value": "", - "type": "password", - "inputActions": ["unobscure", "copy"] - } - }, - { - "key": "chat-completions-endpoint", - "title": "Chat Completions Endpoint", - "description": "The endpoint to use for chat completions. See the [Anthropic API documentation](https://docs.anthropic.com/claude/docs/intro-to-claude) for more information.", - "controllerType": "input", - "controllerProps": { - "placeholder": "https://api.anthropic.com/v1/messages", - "value": "https://api.anthropic.com/v1/messages" - } - } -] diff --git a/extensions/inference-anthropic-extension/rolldown.config.mjs b/extensions/inference-anthropic-extension/rolldown.config.mjs deleted file mode 100644 index 9ebaace2e..000000000 --- a/extensions/inference-anthropic-extension/rolldown.config.mjs +++ /dev/null @@ -1,18 +0,0 @@ -import { defineConfig } from 'rolldown' -import pkgJson from './package.json' with { type: 'json' } -import settingJson from './resources/settings.json' with { type: 'json' } -import modelsJson from './resources/models.json' with { type: 'json' } - -export default defineConfig({ - input: 'src/index.ts', - output: { - format: 'esm', - file: 'dist/index.js', - }, - platform: 'browser', - define: { - MODELS: JSON.stringify(modelsJson), - SETTINGS: JSON.stringify(settingJson), - ENGINE: JSON.stringify(pkgJson.engine), - }, -}) diff --git a/extensions/inference-anthropic-extension/src/anthropic.test.ts b/extensions/inference-anthropic-extension/src/anthropic.test.ts deleted file mode 100644 index 703ead0fb..000000000 --- a/extensions/inference-anthropic-extension/src/anthropic.test.ts +++ /dev/null @@ -1,77 +0,0 @@ -// Import necessary modules -import JanInferenceAnthropicExtension, { Settings } from '.' -import { PayloadType, ChatCompletionRole } from '@janhq/core' - -// Mocks -jest.mock('@janhq/core', () => ({ - RemoteOAIEngine: jest.fn().mockImplementation(() => ({ - registerSettings: jest.fn(), - registerModels: jest.fn(), - getSetting: jest.fn(), - onChange: jest.fn(), - onSettingUpdate: jest.fn(), - onLoad: jest.fn(), - headers: jest.fn(), - })), - PayloadType: jest.fn(), - ChatCompletionRole: { - User: 'user' as const, - Assistant: 'assistant' as const, - System: 'system' as const, - }, -})) - -// Helper functions -const createMockPayload = (): PayloadType => ({ - messages: [ - { role: ChatCompletionRole.System, content: 'Meow' }, - { role: ChatCompletionRole.User, content: 'Hello' }, - { role: ChatCompletionRole.Assistant, content: 'Hi there' }, - ], - model: 'claude-v1', - stream: false, -}) - -describe('JanInferenceAnthropicExtension', () => { - let extension: JanInferenceAnthropicExtension - - beforeEach(() => { - extension = new JanInferenceAnthropicExtension('', '') - extension.apiKey = 'mock-api-key' - extension.inferenceUrl = 'mock-endpoint' - jest.clearAllMocks() - }) - - it('should initialize with correct settings', async () => { - await extension.onLoad() - expect(extension.apiKey).toBe('mock-api-key') - expect(extension.inferenceUrl).toBe('mock-endpoint') - }) - - it('should transform payload correctly', () => { - const payload = createMockPayload() - const transformedPayload = extension.transformPayload(payload) - - expect(transformedPayload).toEqual({ - max_tokens: 4096, - model: 'claude-v1', - stream: false, - system: 'Meow', - messages: [ - { role: 'user', content: 'Hello' }, - { role: 'assistant', content: 'Hi there' }, - ], - }) - }) - - it('should transform response correctly', () => { - const nonStreamResponse = { content: [{ text: 'Test response' }] } - const streamResponse = - 'data: {"type":"content_block_delta","delta":{"text":"Hello"}}' - - expect(extension.transformResponse(nonStreamResponse)).toBe('Test response') - expect(extension.transformResponse(streamResponse)).toBe('Hello') - expect(extension.transformResponse('')).toBe('') - expect(extension.transformResponse('event: something')).toBe('') - }) -}) diff --git a/extensions/inference-anthropic-extension/src/env.d.ts b/extensions/inference-anthropic-extension/src/env.d.ts deleted file mode 100644 index 40ca58094..000000000 --- a/extensions/inference-anthropic-extension/src/env.d.ts +++ /dev/null @@ -1,2 +0,0 @@ -declare const SETTINGS: SettingComponentProps[] -declare const MODELS: Model[] diff --git a/extensions/inference-anthropic-extension/src/index.ts b/extensions/inference-anthropic-extension/src/index.ts deleted file mode 100644 index 8c286d7a5..000000000 --- a/extensions/inference-anthropic-extension/src/index.ts +++ /dev/null @@ -1,150 +0,0 @@ -/** - * @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package. - * The class provides methods for initializing and stopping a model, and for making inference requests. - * It also subscribes to events emitted by the @janhq/core package and handles new message requests. - * @version 1.0.0 - * @module inference-anthropic-extension/src/index - */ - -import { RemoteOAIEngine } from '@janhq/core' -import { PayloadType } from '@janhq/core' -import { ChatCompletionRole } from '@janhq/core' - -export enum Settings { - apiKey = 'anthropic-api-key', - chatCompletionsEndPoint = 'chat-completions-endpoint', -} - -type AnthropicPayloadType = { - stream: boolean - model?: string - max_tokens?: number - messages?: Array<{ role: string; content: string }> - system?: string -} - -/** - * A class that implements the InferenceExtension interface from the @janhq/core package. - * The class provides methods for initializing and stopping a model, and for making inference requests. - * It also subscribes to events emitted by the @janhq/core package and handles new message requests. - */ -export default class JanInferenceAnthropicExtension extends RemoteOAIEngine { - inferenceUrl: string = '' - provider: string = 'anthropic' - maxTokens: number = 4096 - - override async onLoad(): Promise { - super.onLoad() - - // Register Settings - this.registerSettings(SETTINGS) - this.registerModels(MODELS) - - this.apiKey = await this.getSetting(Settings.apiKey, '') - this.inferenceUrl = await this.getSetting( - Settings.chatCompletionsEndPoint, - '' - ) - - if (this.inferenceUrl.length === 0) { - SETTINGS.forEach((setting) => { - if (setting.key === Settings.chatCompletionsEndPoint) { - this.inferenceUrl = setting.controllerProps.value as string - } - }) - } - } - - // Override the headers method to include the x-API-key in the request headers - override async headers(): Promise { - return { - 'Content-Type': 'application/json', - 'x-api-key': this.apiKey, - 'anthropic-version': '2023-06-01', - } - } - - onSettingUpdate(key: string, value: T): void { - if (key === Settings.apiKey) { - this.apiKey = value as string - } else if (key === Settings.chatCompletionsEndPoint) { - if (typeof value !== 'string') return - - if (value.trim().length === 0) { - SETTINGS.forEach((setting) => { - if (setting.key === Settings.chatCompletionsEndPoint) { - this.inferenceUrl = setting.controllerProps.value as string - } - }) - } else { - this.inferenceUrl = value - } - } - } - - // Override the transformPayload method to convert the payload to the required format - transformPayload = (payload: PayloadType): AnthropicPayloadType => { - if (!payload.messages || payload.messages.length === 0) { - return { - max_tokens: this.maxTokens, - messages: [], - model: payload.model, - stream: payload.stream, - } - } - - const convertedData: AnthropicPayloadType = { - max_tokens: this.maxTokens, - messages: [], - model: payload.model, - stream: payload.stream, - } - - payload.messages.forEach((item) => { - if (item.role === ChatCompletionRole.User) { - convertedData.messages.push({ - role: 'user', - content: item.content as string, - }) - } else if (item.role === ChatCompletionRole.Assistant) { - convertedData.messages.push({ - role: 'assistant', - content: item.content as string, - }) - } else if (item.role === ChatCompletionRole.System) { - // When using Claude, you can dramatically improve its performance by using the system parameter to give it a role. - // This technique, known as role prompting, is the most powerful way to use system prompts with Claude. - convertedData.system = item.content as string - } - }) - - return convertedData - } - - // Sample returned stream data from anthropic - // {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} } - // {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello"} } - // {"type":"content_block_stop","index":0 } - // {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":12} } - - // Override the transformResponse method to convert the response to the required format - transformResponse = (data: any): string => { - // handling stream response - if (typeof data === 'string' && data.trim().length === 0) return '' - if (typeof data === 'string' && data.startsWith('event: ')) return '' - if (typeof data === 'string' && data.startsWith('data: ')) { - data = data.replace('data: ', '') - const parsedData = JSON.parse(data) - if (parsedData.type !== 'content_block_delta') return '' - return parsedData.delta?.text ?? '' - } - - // non stream response - if (data.content && data.content.length > 0 && data.content[0].text) { - return data.content[0].text - } - - console.error('Invalid response format:', data) - return '' - } -} diff --git a/extensions/inference-anthropic-extension/tsconfig.json b/extensions/inference-anthropic-extension/tsconfig.json deleted file mode 100644 index 6db951c9e..000000000 --- a/extensions/inference-anthropic-extension/tsconfig.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "compilerOptions": { - "target": "es2016", - "module": "ES6", - "moduleResolution": "node", - "outDir": "./dist", - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "strict": false, - "skipLibCheck": true, - "rootDir": "./src" - }, - "include": ["./src"], - "exclude": ["**/*.test.ts"] -} diff --git a/extensions/inference-cohere-extension/README.md b/extensions/inference-cohere-extension/README.md deleted file mode 100644 index 089a096e8..000000000 --- a/extensions/inference-cohere-extension/README.md +++ /dev/null @@ -1,79 +0,0 @@ -# Cohere Engine Extension - -Created using Jan extension example - -# Create a Jan Extension using Typescript - -Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀 - -## Create Your Own Extension - -To create your own extension, you can use this repository as a template! Just follow the below instructions: - -1. Click the Use this template button at the top of the repository -2. Select Create a new repository -3. Select an owner and name for your new repository -4. Click Create repository -5. Clone your new repository - -## Initial Setup - -After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension. - -> [!NOTE] -> -> You'll need to have a reasonably modern version of -> [Node.js](https://nodejs.org) handy. If you are using a version manager like -> [`nodenv`](https://github.com/nodenv/nodenv) or -> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the -> root of your repository to install the version specified in -> [`package.json`](./package.json). Otherwise, 20.x or later should work! - -1. :hammer_and_wrench: Install the dependencies - - ```bash - npm install - ``` - -1. :building_construction: Package the TypeScript for distribution - - ```bash - npm run bundle - ``` - -1. :white_check_mark: Check your artifact - - There will be a tgz file in your extension directory now - -## Update the Extension Metadata - -The [`package.json`](package.json) file defines metadata about your extension, such as -extension name, main entry, description and version. - -When you copy this repository, update `package.json` with the name, description for your extension. - -## Update the Extension Code - -The [`src/`](./src/) directory is the heart of your extension! This contains the -source code that will be run when your extension functions are invoked. You can replace the -contents of this directory with your own code. - -There are a few things to keep in mind when writing your extension code: - -- Most Jan Extension functions are processed asynchronously. - In `index.ts`, you will see that the extension function will return a `Promise`. - - ```typescript - import { events, MessageEvent, MessageRequest } from '@janhq/core' - - function onStart(): Promise { - return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => - this.inference(data) - ) - } - ``` - - For more information about the Jan Extension Core module, see the - [documentation](https://github.com/janhq/jan/blob/main/core/README.md). - -So, what are you waiting for? Go ahead and start customizing your extension! diff --git a/extensions/inference-cohere-extension/package.json b/extensions/inference-cohere-extension/package.json deleted file mode 100644 index 7058c0e53..000000000 --- a/extensions/inference-cohere-extension/package.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "name": "@janhq/inference-cohere-extension", - "productName": "Cohere Inference Engine", - "version": "1.0.0", - "description": "This extension enables Cohere chat completion API calls", - "main": "dist/index.js", - "module": "dist/module.js", - "engine": "cohere", - "author": "Jan ", - "license": "AGPL-3.0", - "scripts": { - "build": "rolldown -c rolldown.config.mjs", - "build:publish": "rimraf *.tgz --glob || true && yarn build && npm pack && cpx *.tgz ../../pre-install" - }, - "exports": { - ".": "./dist/index.js", - "./main": "./dist/module.js" - }, - "devDependencies": { - "cpx": "^1.5.0", - "rimraf": "^3.0.2", - "rolldown": "1.0.0-beta.1", - "ts-loader": "^9.5.0", - "typescript": "^5.7.2" - }, - "dependencies": { - "@janhq/core": "../../core/package.tgz", - "fetch-retry": "^5.0.6", - "ulidx": "^2.3.0" - }, - "engines": { - "node": ">=18.0.0" - }, - "files": [ - "dist/*", - "package.json", - "README.md" - ], - "bundleDependencies": [ - "fetch-retry" - ], - "installConfig": { - "hoistingLimits": "workspaces" - }, - "packageManager": "yarn@4.5.3" -} diff --git a/extensions/inference-cohere-extension/resources/settings.json b/extensions/inference-cohere-extension/resources/settings.json deleted file mode 100644 index 79150d7e5..000000000 --- a/extensions/inference-cohere-extension/resources/settings.json +++ /dev/null @@ -1,24 +0,0 @@ -[ - { - "key": "cohere-api-key", - "title": "API Key", - "description": "The Cohere API uses API keys for authentication. Visit your [API Keys](https://dashboard.cohere.com/api-keys) page to retrieve the API key you'll use in your requests.", - "controllerType": "input", - "controllerProps": { - "placeholder": "Insert API Key", - "value": "", - "type": "password", - "inputActions": ["unobscure", "copy"] - } - }, - { - "key": "chat-completions-endpoint", - "title": "Chat Completions Endpoint", - "description": "The endpoint to use for chat completions. See the [Cohere API documentation](https://docs.cohere.com/reference/chat) for more information.", - "controllerType": "input", - "controllerProps": { - "placeholder": "https://api.cohere.ai/v1/chat", - "value": "https://api.cohere.ai/v1/chat" - } - } -] diff --git a/extensions/inference-cohere-extension/rolldown.config.mjs b/extensions/inference-cohere-extension/rolldown.config.mjs deleted file mode 100644 index 9ebaace2e..000000000 --- a/extensions/inference-cohere-extension/rolldown.config.mjs +++ /dev/null @@ -1,18 +0,0 @@ -import { defineConfig } from 'rolldown' -import pkgJson from './package.json' with { type: 'json' } -import settingJson from './resources/settings.json' with { type: 'json' } -import modelsJson from './resources/models.json' with { type: 'json' } - -export default defineConfig({ - input: 'src/index.ts', - output: { - format: 'esm', - file: 'dist/index.js', - }, - platform: 'browser', - define: { - MODELS: JSON.stringify(modelsJson), - SETTINGS: JSON.stringify(settingJson), - ENGINE: JSON.stringify(pkgJson.engine), - }, -}) diff --git a/extensions/inference-cohere-extension/src/env.d.ts b/extensions/inference-cohere-extension/src/env.d.ts deleted file mode 100644 index 40ca58094..000000000 --- a/extensions/inference-cohere-extension/src/env.d.ts +++ /dev/null @@ -1,2 +0,0 @@ -declare const SETTINGS: SettingComponentProps[] -declare const MODELS: Model[] diff --git a/extensions/inference-cohere-extension/src/index.ts b/extensions/inference-cohere-extension/src/index.ts deleted file mode 100644 index 018df60f9..000000000 --- a/extensions/inference-cohere-extension/src/index.ts +++ /dev/null @@ -1,117 +0,0 @@ -/** - * @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package. - * The class provides methods for initializing and stopping a model, and for making inference requests. - * It also subscribes to events emitted by the @janhq/core package and handles new message requests. - * @version 1.0.0 - * @module inference-cohere-extension/src/index - */ - -import { RemoteOAIEngine } from '@janhq/core' -import { PayloadType } from '@janhq/core' -import { ChatCompletionRole } from '@janhq/core' - -enum Settings { - apiKey = 'cohere-api-key', - chatCompletionsEndPoint = 'chat-completions-endpoint', -} - -enum RoleType { - user = 'USER', - chatbot = 'CHATBOT', - system = 'SYSTEM', -} - -type CoherePayloadType = { - chat_history?: Array<{ role: RoleType; message: string }> - message?: string - preamble?: string -} - -/** - * A class that implements the InferenceExtension interface from the @janhq/core package. - * The class provides methods for initializing and stopping a model, and for making inference requests. - * It also subscribes to events emitted by the @janhq/core package and handles new message requests. - */ -export default class JanInferenceCohereExtension extends RemoteOAIEngine { - inferenceUrl: string = '' - provider: string = 'cohere' - - override async onLoad(): Promise { - super.onLoad() - - // Register Settings - this.registerSettings(SETTINGS) - this.registerModels(MODELS) - - this.apiKey = await this.getSetting(Settings.apiKey, '') - this.inferenceUrl = await this.getSetting( - Settings.chatCompletionsEndPoint, - '' - ) - if (this.inferenceUrl.length === 0) { - SETTINGS.forEach((setting) => { - if (setting.key === Settings.chatCompletionsEndPoint) { - this.inferenceUrl = setting.controllerProps.value as string - } - }) - } - } - - onSettingUpdate(key: string, value: T): void { - if (key === Settings.apiKey) { - this.apiKey = value as string - } else if (key === Settings.chatCompletionsEndPoint) { - if (typeof value !== 'string') return - - if (value.trim().length === 0) { - SETTINGS.forEach((setting) => { - if (setting.key === Settings.chatCompletionsEndPoint) { - this.inferenceUrl = setting.controllerProps.value as string - } - }) - } else { - this.inferenceUrl = value - } - } - } - - transformPayload = (payload: PayloadType): CoherePayloadType => { - if (payload.messages.length === 0) { - return {} - } - - const { messages, ...params } = payload - const convertedData: CoherePayloadType = { - ...params, - chat_history: [], - message: '', - } - messages.forEach((item, index) => { - // Assign the message of the last item to the `message` property - if (index === messages.length - 1) { - convertedData.message = item.content as string - return - } - if (item.role === ChatCompletionRole.User) { - convertedData.chat_history.push({ - role: RoleType.user, - message: item.content as string, - }) - } else if (item.role === ChatCompletionRole.Assistant) { - convertedData.chat_history.push({ - role: RoleType.chatbot, - message: item.content as string, - }) - } else if (item.role === ChatCompletionRole.System) { - convertedData.preamble = item.content as string - } - }) - return convertedData - } - - transformResponse = (data: any) => { - return typeof data === 'object' - ? data.text - : (JSON.parse(data.replace('data: ', '').trim()).text ?? '') - } -} diff --git a/extensions/inference-cohere-extension/tsconfig.json b/extensions/inference-cohere-extension/tsconfig.json deleted file mode 100644 index 2477d58ce..000000000 --- a/extensions/inference-cohere-extension/tsconfig.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "compilerOptions": { - "target": "es2016", - "module": "ES6", - "moduleResolution": "node", - "outDir": "./dist", - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "strict": false, - "skipLibCheck": true, - "rootDir": "./src" - }, - "include": ["./src"] -} diff --git a/extensions/inference-cortex-extension/bin/version.txt b/extensions/inference-cortex-extension/bin/version.txt index b0f3d96f8..9b3178149 100644 --- a/extensions/inference-cortex-extension/bin/version.txt +++ b/extensions/inference-cortex-extension/bin/version.txt @@ -1 +1 @@ -1.0.8 +1.0.9-rc1 diff --git a/extensions/inference-cortex-extension/src/@types/global.d.ts b/extensions/inference-cortex-extension/src/@types/global.d.ts index 907db0df1..2de432c29 100644 --- a/extensions/inference-cortex-extension/src/@types/global.d.ts +++ b/extensions/inference-cortex-extension/src/@types/global.d.ts @@ -2,5 +2,5 @@ declare const NODE: string declare const CORTEX_API_URL: string declare const CORTEX_SOCKET_URL: string declare const CORTEX_ENGINE_VERSION: string -declare const SETTINGS: object[] -declare const MODELS: object[] +declare const SETTINGS: any +declare const MODELS: any diff --git a/extensions/inference-cortex-extension/src/index.ts b/extensions/inference-cortex-extension/src/index.ts index 03c094ec1..84cc49b94 100644 --- a/extensions/inference-cortex-extension/src/index.ts +++ b/extensions/inference-cortex-extension/src/index.ts @@ -111,8 +111,6 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine { ) if (!Number.isNaN(threads_number)) this.cpu_threads = threads_number - this.queue.add(() => this.clean()) - // Run the process watchdog const systemInfo = await systemInformation() this.queue.add(() => executeOnMain(NODE, 'run', systemInfo)) diff --git a/extensions/inference-groq-extension/README.md b/extensions/inference-groq-extension/README.md deleted file mode 100644 index f9690da09..000000000 --- a/extensions/inference-groq-extension/README.md +++ /dev/null @@ -1,75 +0,0 @@ -# Create a Jan Extension using Typescript - -Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀 - -## Create Your Own Extension - -To create your own extension, you can use this repository as a template! Just follow the below instructions: - -1. Click the Use this template button at the top of the repository -2. Select Create a new repository -3. Select an owner and name for your new repository -4. Click Create repository -5. Clone your new repository - -## Initial Setup - -After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension. - -> [!NOTE] -> -> You'll need to have a reasonably modern version of -> [Node.js](https://nodejs.org) handy. If you are using a version manager like -> [`nodenv`](https://github.com/nodenv/nodenv) or -> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the -> root of your repository to install the version specified in -> [`package.json`](./package.json). Otherwise, 20.x or later should work! - -1. :hammer_and_wrench: Install the dependencies - - ```bash - npm install - ``` - -1. :building_construction: Package the TypeScript for distribution - - ```bash - npm run bundle - ``` - -1. :white_check_mark: Check your artifact - - There will be a tgz file in your extension directory now - -## Update the Extension Metadata - -The [`package.json`](package.json) file defines metadata about your extension, such as -extension name, main entry, description and version. - -When you copy this repository, update `package.json` with the name, description for your extension. - -## Update the Extension Code - -The [`src/`](./src/) directory is the heart of your extension! This contains the -source code that will be run when your extension functions are invoked. You can replace the -contents of this directory with your own code. - -There are a few things to keep in mind when writing your extension code: - -- Most Jan Extension functions are processed asynchronously. - In `index.ts`, you will see that the extension function will return a `Promise`. - - ```typescript - import { events, MessageEvent, MessageRequest } from '@janhq/core' - - function onStart(): Promise { - return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => - this.inference(data) - ) - } - ``` - - For more information about the Jan Extension Core module, see the - [documentation](https://github.com/janhq/jan/blob/main/core/README.md). - -So, what are you waiting for? Go ahead and start customizing your extension! diff --git a/extensions/inference-groq-extension/package.json b/extensions/inference-groq-extension/package.json deleted file mode 100644 index e400c517b..000000000 --- a/extensions/inference-groq-extension/package.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "name": "@janhq/inference-groq-extension", - "productName": "Groq Inference Engine", - "version": "1.0.1", - "description": "This extension enables fast Groq chat completion API calls", - "main": "dist/index.js", - "engine": "groq", - "module": "dist/module.js", - "author": "Carsen Klock & Jan", - "license": "AGPL-3.0", - "scripts": { - "build": "rolldown -c rolldown.config.mjs", - "build:publish": "rimraf *.tgz --glob || true && yarn build && npm pack && cpx *.tgz ../../pre-install" - }, - "devDependencies": { - "cpx": "^1.5.0", - "rimraf": "^3.0.2", - "rolldown": "1.0.0-beta.1", - "ts-loader": "^9.5.0", - "typescript": "^5.7.2" - }, - "dependencies": { - "@janhq/core": "../../core/package.tgz", - "fetch-retry": "^5.0.6", - "ulidx": "^2.3.0" - }, - "engines": { - "node": ">=18.0.0" - }, - "files": [ - "dist/*", - "package.json", - "README.md" - ], - "bundleDependencies": [ - "fetch-retry" - ], - "installConfig": { - "hoistingLimits": "workspaces" - }, - "packageManager": "yarn@4.5.3" -} diff --git a/extensions/inference-groq-extension/resources/settings.json b/extensions/inference-groq-extension/resources/settings.json deleted file mode 100644 index 767fec0ba..000000000 --- a/extensions/inference-groq-extension/resources/settings.json +++ /dev/null @@ -1,24 +0,0 @@ -[ - { - "key": "groq-api-key", - "title": "API Key", - "description": "The Groq API uses API keys for authentication. Visit your [API Keys](https://console.groq.com/keys) page to retrieve the API key you'll use in your requests.", - "controllerType": "input", - "controllerProps": { - "placeholder": "Insert API Key", - "value": "", - "type": "password", - "inputActions": ["unobscure", "copy"] - } - }, - { - "key": "chat-completions-endpoint", - "title": "Chat Completions Endpoint", - "description": "The endpoint to use for chat completions. See the [Groq documentation](https://console.groq.com/docs/openai) for more information.", - "controllerType": "input", - "controllerProps": { - "placeholder": "https://api.groq.com/openai/v1/chat/completions", - "value": "https://api.groq.com/openai/v1/chat/completions" - } - } -] diff --git a/extensions/inference-groq-extension/rolldown.config.mjs b/extensions/inference-groq-extension/rolldown.config.mjs deleted file mode 100644 index 9ebaace2e..000000000 --- a/extensions/inference-groq-extension/rolldown.config.mjs +++ /dev/null @@ -1,18 +0,0 @@ -import { defineConfig } from 'rolldown' -import pkgJson from './package.json' with { type: 'json' } -import settingJson from './resources/settings.json' with { type: 'json' } -import modelsJson from './resources/models.json' with { type: 'json' } - -export default defineConfig({ - input: 'src/index.ts', - output: { - format: 'esm', - file: 'dist/index.js', - }, - platform: 'browser', - define: { - MODELS: JSON.stringify(modelsJson), - SETTINGS: JSON.stringify(settingJson), - ENGINE: JSON.stringify(pkgJson.engine), - }, -}) diff --git a/extensions/inference-groq-extension/src/env.d.ts b/extensions/inference-groq-extension/src/env.d.ts deleted file mode 100644 index 40ca58094..000000000 --- a/extensions/inference-groq-extension/src/env.d.ts +++ /dev/null @@ -1,2 +0,0 @@ -declare const SETTINGS: SettingComponentProps[] -declare const MODELS: Model[] diff --git a/extensions/inference-groq-extension/src/index.ts b/extensions/inference-groq-extension/src/index.ts deleted file mode 100644 index d6969b48f..000000000 --- a/extensions/inference-groq-extension/src/index.ts +++ /dev/null @@ -1,64 +0,0 @@ -/** - * @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package. - * The class provides methods for initializing and stopping a model, and for making inference requests. - * It also subscribes to events emitted by the @janhq/core package and handles new message requests. - * @version 1.0.0 - * @module inference-groq-extension/src/index - */ - -import { RemoteOAIEngine } from '@janhq/core' - -enum Settings { - apiKey = 'groq-api-key', - chatCompletionsEndPoint = 'chat-completions-endpoint', -} -/** - * A class that implements the InferenceExtension interface from the @janhq/core package. - * The class provides methods for initializing and stopping a model, and for making inference requests. - * It also subscribes to events emitted by the @janhq/core package and handles new message requests. - */ -export default class JanInferenceGroqExtension extends RemoteOAIEngine { - inferenceUrl: string = '' - provider = 'groq' - - override async onLoad(): Promise { - super.onLoad() - - // Register Settings - this.registerSettings(SETTINGS) - this.registerModels(MODELS) - - // Retrieve API Key Setting - this.apiKey = await this.getSetting(Settings.apiKey, '') - this.inferenceUrl = await this.getSetting( - Settings.chatCompletionsEndPoint, - '' - ) - - if (this.inferenceUrl.length === 0) { - SETTINGS.forEach((setting) => { - if (setting.key === Settings.chatCompletionsEndPoint) { - this.inferenceUrl = setting.controllerProps.value as string - } - }) - } - } - - onSettingUpdate(key: string, value: T): void { - if (key === Settings.apiKey) { - this.apiKey = value as string - } else if (key === Settings.chatCompletionsEndPoint) { - if (typeof value !== 'string') return - - if (value.trim().length === 0) { - SETTINGS.forEach((setting) => { - if (setting.key === Settings.chatCompletionsEndPoint) { - this.inferenceUrl = setting.controllerProps.value as string - } - }) - } else { - this.inferenceUrl = value - } - } - } -} diff --git a/extensions/inference-groq-extension/tsconfig.json b/extensions/inference-groq-extension/tsconfig.json deleted file mode 100644 index 2477d58ce..000000000 --- a/extensions/inference-groq-extension/tsconfig.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "compilerOptions": { - "target": "es2016", - "module": "ES6", - "moduleResolution": "node", - "outDir": "./dist", - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "strict": false, - "skipLibCheck": true, - "rootDir": "./src" - }, - "include": ["./src"] -} diff --git a/extensions/inference-martian-extension/README.md b/extensions/inference-martian-extension/README.md deleted file mode 100644 index 5b8e898d7..000000000 --- a/extensions/inference-martian-extension/README.md +++ /dev/null @@ -1,79 +0,0 @@ -# Martian Engine Extension - -Created using Jan extension example - -# Create a Jan Extension using Typescript - -Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀 - -## Create Your Own Extension - -To create your own extension, you can use this repository as a template! Just follow the below instructions: - -1. Click the Use this template button at the top of the repository -2. Select Create a new repository -3. Select an owner and name for your new repository -4. Click Create repository -5. Clone your new repository - -## Initial Setup - -After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension. - -> [!NOTE] -> -> You'll need to have a reasonably modern version of -> [Node.js](https://nodejs.org) handy. If you are using a version manager like -> [`nodenv`](https://github.com/nodenv/nodenv) or -> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the -> root of your repository to install the version specified in -> [`package.json`](./package.json). Otherwise, 20.x or later should work! - -1. :hammer_and_wrench: Install the dependencies - - ```bash - npm install - ``` - -1. :building_construction: Package the TypeScript for distribution - - ```bash - npm run bundle - ``` - -1. :white_check_mark: Check your artifact - - There will be a tgz file in your extension directory now - -## Update the Extension Metadata - -The [`package.json`](package.json) file defines metadata about your extension, such as -extension name, main entry, description and version. - -When you copy this repository, update `package.json` with the name, description for your extension. - -## Update the Extension Code - -The [`src/`](./src/) directory is the heart of your extension! This contains the -source code that will be run when your extension functions are invoked. You can replace the -contents of this directory with your own code. - -There are a few things to keep in mind when writing your extension code: - -- Most Jan Extension functions are processed asynchronously. - In `index.ts`, you will see that the extension function will return a `Promise`. - - ```typescript - import { events, MessageEvent, MessageRequest } from '@janhq/core' - - function onStart(): Promise { - return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => - this.inference(data) - ) - } - ``` - - For more information about the Jan Extension Core module, see the - [documentation](https://github.com/janhq/jan/blob/main/core/README.md). - -So, what are you waiting for? Go ahead and start customizing your extension! diff --git a/extensions/inference-martian-extension/package.json b/extensions/inference-martian-extension/package.json deleted file mode 100644 index c1371917c..000000000 --- a/extensions/inference-martian-extension/package.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "name": "@janhq/inference-martian-extension", - "productName": "Martian Inference Engine", - "version": "1.0.1", - "description": "This extension enables Martian chat completion API calls", - "main": "dist/index.js", - "module": "dist/module.js", - "engine": "martian", - "author": "Jan ", - "license": "AGPL-3.0", - "scripts": { - "build": "rolldown -c rolldown.config.mjs", - "build:publish": "rimraf *.tgz --glob || true && yarn build && npm pack && cpx *.tgz ../../pre-install" - }, - "devDependencies": { - "cpx": "^1.5.0", - "rimraf": "^3.0.2", - "rolldown": "1.0.0-beta.1", - "ts-loader": "^9.5.0", - "typescript": "^5.7.2" - }, - "dependencies": { - "@janhq/core": "../../core/package.tgz", - "fetch-retry": "^5.0.6", - "ulidx": "^2.3.0" - }, - "engines": { - "node": ">=18.0.0" - }, - "files": [ - "dist/*", - "package.json", - "README.md" - ], - "bundleDependencies": [ - "fetch-retry" - ], - "installConfig": { - "hoistingLimits": "workspaces" - }, - "packageManager": "yarn@4.5.3" -} diff --git a/extensions/inference-martian-extension/resources/settings.json b/extensions/inference-martian-extension/resources/settings.json deleted file mode 100644 index 6825099f5..000000000 --- a/extensions/inference-martian-extension/resources/settings.json +++ /dev/null @@ -1,24 +0,0 @@ -[ - { - "key": "martian-api-key", - "title": "API Key", - "description": "The Martian API uses API keys for authentication. Visit your [API Keys](https://withmartian.com/dashboard) page to retrieve the API key you'll use in your requests.", - "controllerType": "input", - "controllerProps": { - "placeholder": "Insert API Key", - "value": "", - "type": "password", - "inputActions": ["unobscure", "copy"] - } - }, - { - "key": "chat-completions-endpoint", - "title": "Chat Completions Endpoint", - "description": "The endpoint to use for chat completions. See the [Martian API documentation](https://docs.withmartian.com/martian-model-router/getting-started/quickstart-integrating-martian-into-your-codebase) for more information.", - "controllerType": "input", - "controllerProps": { - "placeholder": "https://withmartian.com/api/openai/v1/chat/completions", - "value": "https://withmartian.com/api/openai/v1/chat/completions" - } - } -] diff --git a/extensions/inference-martian-extension/rolldown.config.mjs b/extensions/inference-martian-extension/rolldown.config.mjs deleted file mode 100644 index 9ebaace2e..000000000 --- a/extensions/inference-martian-extension/rolldown.config.mjs +++ /dev/null @@ -1,18 +0,0 @@ -import { defineConfig } from 'rolldown' -import pkgJson from './package.json' with { type: 'json' } -import settingJson from './resources/settings.json' with { type: 'json' } -import modelsJson from './resources/models.json' with { type: 'json' } - -export default defineConfig({ - input: 'src/index.ts', - output: { - format: 'esm', - file: 'dist/index.js', - }, - platform: 'browser', - define: { - MODELS: JSON.stringify(modelsJson), - SETTINGS: JSON.stringify(settingJson), - ENGINE: JSON.stringify(pkgJson.engine), - }, -}) diff --git a/extensions/inference-martian-extension/src/env.d.ts b/extensions/inference-martian-extension/src/env.d.ts deleted file mode 100644 index 40ca58094..000000000 --- a/extensions/inference-martian-extension/src/env.d.ts +++ /dev/null @@ -1,2 +0,0 @@ -declare const SETTINGS: SettingComponentProps[] -declare const MODELS: Model[] diff --git a/extensions/inference-martian-extension/src/index.ts b/extensions/inference-martian-extension/src/index.ts deleted file mode 100644 index 8cbe4e52d..000000000 --- a/extensions/inference-martian-extension/src/index.ts +++ /dev/null @@ -1,63 +0,0 @@ -/** - * @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package. - * The class provides methods for initializing and stopping a model, and for making inference requests. - * It also subscribes to events emitted by the @janhq/core package and handles new message requests. - * @version 1.0.0 - * @module inference-martian-extension/src/index - */ - -import { RemoteOAIEngine } from '@janhq/core' - -enum Settings { - apiKey = 'martian-api-key', - chatCompletionsEndPoint = 'chat-completions-endpoint', -} - -/** - * A class that implements the InferenceExtension interface from the @janhq/core package. - * The class provides methods for initializing and stopping a model, and for making inference requests. - * It also subscribes to events emitted by the @janhq/core package and handles new message requests. - */ -export default class JanInferenceMartianExtension extends RemoteOAIEngine { - inferenceUrl: string = '' - provider: string = 'martian' - - override async onLoad(): Promise { - super.onLoad() - - // Register Settings - this.registerSettings(SETTINGS) - this.registerModels(MODELS) - - this.apiKey = await this.getSetting(Settings.apiKey, '') - this.inferenceUrl = await this.getSetting( - Settings.chatCompletionsEndPoint, - '' - ) - if (this.inferenceUrl.length === 0) { - SETTINGS.forEach((setting) => { - if (setting.key === Settings.chatCompletionsEndPoint) { - this.inferenceUrl = setting.controllerProps.value as string - } - }) - } - } - - onSettingUpdate(key: string, value: T): void { - if (key === Settings.apiKey) { - this.apiKey = value as string - } else if (key === Settings.chatCompletionsEndPoint) { - if (typeof value !== 'string') return - - if (value.trim().length === 0) { - SETTINGS.forEach((setting) => { - if (setting.key === Settings.chatCompletionsEndPoint) { - this.inferenceUrl = setting.controllerProps.value as string - } - }) - } else { - this.inferenceUrl = value - } - } - } -} diff --git a/extensions/inference-martian-extension/tsconfig.json b/extensions/inference-martian-extension/tsconfig.json deleted file mode 100644 index 2477d58ce..000000000 --- a/extensions/inference-martian-extension/tsconfig.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "compilerOptions": { - "target": "es2016", - "module": "ES6", - "moduleResolution": "node", - "outDir": "./dist", - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "strict": false, - "skipLibCheck": true, - "rootDir": "./src" - }, - "include": ["./src"] -} diff --git a/extensions/inference-mistral-extension/README.md b/extensions/inference-mistral-extension/README.md deleted file mode 100644 index adb36558c..000000000 --- a/extensions/inference-mistral-extension/README.md +++ /dev/null @@ -1,79 +0,0 @@ -# Mistral Engine Extension - -Created using Jan extension example - -# Create a Jan Extension using Typescript - -Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀 - -## Create Your Own Extension - -To create your own extension, you can use this repository as a template! Just follow the below instructions: - -1. Click the Use this template button at the top of the repository -2. Select Create a new repository -3. Select an owner and name for your new repository -4. Click Create repository -5. Clone your new repository - -## Initial Setup - -After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension. - -> [!NOTE] -> -> You'll need to have a reasonably modern version of -> [Node.js](https://nodejs.org) handy. If you are using a version manager like -> [`nodenv`](https://github.com/nodenv/nodenv) or -> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the -> root of your repository to install the version specified in -> [`package.json`](./package.json). Otherwise, 20.x or later should work! - -1. :hammer_and_wrench: Install the dependencies - - ```bash - npm install - ``` - -1. :building_construction: Package the TypeScript for distribution - - ```bash - npm run bundle - ``` - -1. :white_check_mark: Check your artifact - - There will be a tgz file in your extension directory now - -## Update the Extension Metadata - -The [`package.json`](package.json) file defines metadata about your extension, such as -extension name, main entry, description and version. - -When you copy this repository, update `package.json` with the name, description for your extension. - -## Update the Extension Code - -The [`src/`](./src/) directory is the heart of your extension! This contains the -source code that will be run when your extension functions are invoked. You can replace the -contents of this directory with your own code. - -There are a few things to keep in mind when writing your extension code: - -- Most Jan Extension functions are processed asynchronously. - In `index.ts`, you will see that the extension function will return a `Promise`. - - ```typescript - import { events, MessageEvent, MessageRequest } from '@janhq/core' - - function onStart(): Promise { - return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => - this.inference(data) - ) - } - ``` - - For more information about the Jan Extension Core module, see the - [documentation](https://github.com/janhq/jan/blob/main/core/README.md). - -So, what are you waiting for? Go ahead and start customizing your extension! diff --git a/extensions/inference-mistral-extension/package.json b/extensions/inference-mistral-extension/package.json deleted file mode 100644 index 504a3d613..000000000 --- a/extensions/inference-mistral-extension/package.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "name": "@janhq/inference-mistral-extension", - "productName": "MistralAI Inference Engine", - "version": "1.0.1", - "description": "This extension enables Mistral chat completion API calls", - "main": "dist/index.js", - "module": "dist/module.js", - "engine": "mistral", - "author": "Jan ", - "license": "AGPL-3.0", - "scripts": { - "build": "rolldown -c rolldown.config.mjs", - "build:publish": "rimraf *.tgz --glob || true && yarn build && npm pack && cpx *.tgz ../../pre-install" - }, - "devDependencies": { - "cpx": "^1.5.0", - "rimraf": "^3.0.2", - "rolldown": "1.0.0-beta.1", - "ts-loader": "^9.5.0", - "typescript": "^5.7.2" - }, - "dependencies": { - "@janhq/core": "../../core/package.tgz", - "fetch-retry": "^5.0.6", - "ulidx": "^2.3.0" - }, - "engines": { - "node": ">=18.0.0" - }, - "files": [ - "dist/*", - "package.json", - "README.md" - ], - "bundleDependencies": [ - "fetch-retry" - ], - "installConfig": { - "hoistingLimits": "workspaces" - }, - "packageManager": "yarn@4.5.3" -} diff --git a/extensions/inference-mistral-extension/resources/settings.json b/extensions/inference-mistral-extension/resources/settings.json deleted file mode 100644 index 963674b02..000000000 --- a/extensions/inference-mistral-extension/resources/settings.json +++ /dev/null @@ -1,24 +0,0 @@ -[ - { - "key": "mistral-api-key", - "title": "API Key", - "description": "The Mistral API uses API keys for authentication. Visit your [API Keys](https://console.mistral.ai/api-keys/) page to retrieve the API key you'll use in your requests.", - "controllerType": "input", - "controllerProps": { - "placeholder": "Insert API Key", - "value": "", - "type": "password", - "inputActions": ["unobscure", "copy"] - } - }, - { - "key": "chat-completions-endpoint", - "title": "Chat Completions Endpoint", - "description": "The endpoint to use for chat completions. See the [Mistral API documentation](https://docs.mistral.ai/api/#operation/createChatCompletion) for more information.", - "controllerType": "input", - "controllerProps": { - "placeholder": "https://api.mistral.ai/v1/chat/completions", - "value": "https://api.mistral.ai/v1/chat/completions" - } - } -] diff --git a/extensions/inference-mistral-extension/rolldown.config.mjs b/extensions/inference-mistral-extension/rolldown.config.mjs deleted file mode 100644 index 9ebaace2e..000000000 --- a/extensions/inference-mistral-extension/rolldown.config.mjs +++ /dev/null @@ -1,18 +0,0 @@ -import { defineConfig } from 'rolldown' -import pkgJson from './package.json' with { type: 'json' } -import settingJson from './resources/settings.json' with { type: 'json' } -import modelsJson from './resources/models.json' with { type: 'json' } - -export default defineConfig({ - input: 'src/index.ts', - output: { - format: 'esm', - file: 'dist/index.js', - }, - platform: 'browser', - define: { - MODELS: JSON.stringify(modelsJson), - SETTINGS: JSON.stringify(settingJson), - ENGINE: JSON.stringify(pkgJson.engine), - }, -}) diff --git a/extensions/inference-mistral-extension/src/env.d.ts b/extensions/inference-mistral-extension/src/env.d.ts deleted file mode 100644 index 40ca58094..000000000 --- a/extensions/inference-mistral-extension/src/env.d.ts +++ /dev/null @@ -1,2 +0,0 @@ -declare const SETTINGS: SettingComponentProps[] -declare const MODELS: Model[] diff --git a/extensions/inference-mistral-extension/src/index.ts b/extensions/inference-mistral-extension/src/index.ts deleted file mode 100644 index 22b977c5a..000000000 --- a/extensions/inference-mistral-extension/src/index.ts +++ /dev/null @@ -1,63 +0,0 @@ -/** - * @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package. - * The class provides methods for initializing and stopping a model, and for making inference requests. - * It also subscribes to events emitted by the @janhq/core package and handles new message requests. - * @version 1.0.0 - * @module inference-mistral-extension/src/index - */ - -import { RemoteOAIEngine } from '@janhq/core' - -enum Settings { - apiKey = 'mistral-api-key', - chatCompletionsEndPoint = 'chat-completions-endpoint', -} -/** - * A class that implements the InferenceExtension interface from the @janhq/core package. - * The class provides methods for initializing and stopping a model, and for making inference requests. - * It also subscribes to events emitted by the @janhq/core package and handles new message requests. - */ -export default class JanInferenceMistralExtension extends RemoteOAIEngine { - inferenceUrl: string = '' - provider: string = 'mistral' - - override async onLoad(): Promise { - super.onLoad() - - // Register Settings - this.registerSettings(SETTINGS) - this.registerModels(MODELS) - - this.apiKey = await this.getSetting(Settings.apiKey, '') - this.inferenceUrl = await this.getSetting( - Settings.chatCompletionsEndPoint, - '' - ) - - if (this.inferenceUrl.length === 0) { - SETTINGS.forEach((setting) => { - if (setting.key === Settings.chatCompletionsEndPoint) { - this.inferenceUrl = setting.controllerProps.value as string - } - }) - } - } - - onSettingUpdate(key: string, value: T): void { - if (key === Settings.apiKey) { - this.apiKey = value as string - } else if (key === Settings.chatCompletionsEndPoint) { - if (typeof value !== 'string') return - - if (value.trim().length === 0) { - SETTINGS.forEach((setting) => { - if (setting.key === Settings.chatCompletionsEndPoint) { - this.inferenceUrl = setting.controllerProps.value as string - } - }) - } else { - this.inferenceUrl = value - } - } - } -} diff --git a/extensions/inference-mistral-extension/tsconfig.json b/extensions/inference-mistral-extension/tsconfig.json deleted file mode 100644 index 2477d58ce..000000000 --- a/extensions/inference-mistral-extension/tsconfig.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "compilerOptions": { - "target": "es2016", - "module": "ES6", - "moduleResolution": "node", - "outDir": "./dist", - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "strict": false, - "skipLibCheck": true, - "rootDir": "./src" - }, - "include": ["./src"] -} diff --git a/extensions/inference-nvidia-extension/README.md b/extensions/inference-nvidia-extension/README.md deleted file mode 100644 index 65a1b2b59..000000000 --- a/extensions/inference-nvidia-extension/README.md +++ /dev/null @@ -1,79 +0,0 @@ -# Nvidia Engine Extension - -Created using Jan extension example - -# Create a Jan Extension using Typescript - -Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀 - -## Create Your Own Extension - -To create your own extension, you can use this repository as a template! Just follow the below instructions: - -1. Click the Use this template button at the top of the repository -2. Select Create a new repository -3. Select an owner and name for your new repository -4. Click Create repository -5. Clone your new repository - -## Initial Setup - -After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension. - -> [!NOTE] -> -> You'll need to have a reasonably modern version of -> [Node.js](https://nodejs.org) handy. If you are using a version manager like -> [`nodenv`](https://github.com/nodenv/nodenv) or -> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the -> root of your repository to install the version specified in -> [`package.json`](./package.json). Otherwise, 20.x or later should work! - -1. :hammer_and_wrench: Install the dependencies - - ```bash - npm install - ``` - -1. :building_construction: Package the TypeScript for distribution - - ```bash - npm run bundle - ``` - -1. :white_check_mark: Check your artifact - - There will be a tgz file in your extension directory now - -## Update the Extension Metadata - -The [`package.json`](package.json) file defines metadata about your extension, such as -extension name, main entry, description and version. - -When you copy this repository, update `package.json` with the name, description for your extension. - -## Update the Extension Code - -The [`src/`](./src/) directory is the heart of your extension! This contains the -source code that will be run when your extension functions are invoked. You can replace the -contents of this directory with your own code. - -There are a few things to keep in mind when writing your extension code: - -- Most Jan Extension functions are processed asynchronously. - In `index.ts`, you will see that the extension function will return a `Promise`. - - ```typescript - import { events, MessageEvent, MessageRequest } from '@janhq/core' - - function onStart(): Promise { - return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => - this.inference(data) - ) - } - ``` - - For more information about the Jan Extension Core module, see the - [documentation](https://github.com/janhq/jan/blob/main/core/README.md). - -So, what are you waiting for? Go ahead and start customizing your extension! diff --git a/extensions/inference-nvidia-extension/package.json b/extensions/inference-nvidia-extension/package.json deleted file mode 100644 index 771ccec4f..000000000 --- a/extensions/inference-nvidia-extension/package.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "name": "@janhq/inference-nvidia-extension", - "productName": "NVIDIA NIM Inference Engine", - "version": "1.0.1", - "description": "This extension enables NVIDIA chat completion API calls", - "main": "dist/index.js", - "module": "dist/module.js", - "engine": "nvidia", - "author": "Jan ", - "license": "AGPL-3.0", - "scripts": { - "build": "rolldown -c rolldown.config.mjs", - "build:publish": "rimraf *.tgz --glob || true && yarn build && npm pack && cpx *.tgz ../../pre-install" - }, - "devDependencies": { - "cpx": "^1.5.0", - "rimraf": "^3.0.2", - "rolldown": "1.0.0-beta.1", - "ts-loader": "^9.5.0", - "typescript": "^5.7.2" - }, - "dependencies": { - "@janhq/core": "../../core/package.tgz", - "fetch-retry": "^5.0.6", - "ulidx": "^2.3.0" - }, - "engines": { - "node": ">=18.0.0" - }, - "files": [ - "dist/*", - "package.json", - "README.md" - ], - "bundleDependencies": [ - "fetch-retry" - ], - "installConfig": { - "hoistingLimits": "workspaces" - }, - "packageManager": "yarn@4.5.3" -} diff --git a/extensions/inference-nvidia-extension/resources/settings.json b/extensions/inference-nvidia-extension/resources/settings.json deleted file mode 100644 index 6b2652653..000000000 --- a/extensions/inference-nvidia-extension/resources/settings.json +++ /dev/null @@ -1,24 +0,0 @@ -[ - { - "key": "nvidia-api-key", - "title": "API Key", - "description": "The NVIDIA API uses API keys for authentication. Visit your [API Keys](https://org.ngc.nvidia.com/setup/personal-keys) page to retrieve the API key you'll use in your requests..", - "controllerType": "input", - "controllerProps": { - "placeholder": "Insert API Key", - "value": "", - "type": "password", - "inputActions": ["unobscure", "copy"] - } - }, - { - "key": "chat-completions-endpoint", - "title": "Chat Completions Endpoint", - "description": "The endpoint to use for chat completions. See the [NVIDIA API documentation](https://www.nvidia.com/en-us/ai/) for more information.", - "controllerType": "input", - "controllerProps": { - "placeholder": "https://integrate.api.nvidia.com/v1/chat/completions", - "value": "https://integrate.api.nvidia.com/v1/chat/completions" - } - } -] diff --git a/extensions/inference-nvidia-extension/rolldown.config.mjs b/extensions/inference-nvidia-extension/rolldown.config.mjs deleted file mode 100644 index 9ebaace2e..000000000 --- a/extensions/inference-nvidia-extension/rolldown.config.mjs +++ /dev/null @@ -1,18 +0,0 @@ -import { defineConfig } from 'rolldown' -import pkgJson from './package.json' with { type: 'json' } -import settingJson from './resources/settings.json' with { type: 'json' } -import modelsJson from './resources/models.json' with { type: 'json' } - -export default defineConfig({ - input: 'src/index.ts', - output: { - format: 'esm', - file: 'dist/index.js', - }, - platform: 'browser', - define: { - MODELS: JSON.stringify(modelsJson), - SETTINGS: JSON.stringify(settingJson), - ENGINE: JSON.stringify(pkgJson.engine), - }, -}) diff --git a/extensions/inference-nvidia-extension/src/env.d.ts b/extensions/inference-nvidia-extension/src/env.d.ts deleted file mode 100644 index 40ca58094..000000000 --- a/extensions/inference-nvidia-extension/src/env.d.ts +++ /dev/null @@ -1,2 +0,0 @@ -declare const SETTINGS: SettingComponentProps[] -declare const MODELS: Model[] diff --git a/extensions/inference-nvidia-extension/src/index.ts b/extensions/inference-nvidia-extension/src/index.ts deleted file mode 100644 index 0e5bb8135..000000000 --- a/extensions/inference-nvidia-extension/src/index.ts +++ /dev/null @@ -1,63 +0,0 @@ -/** - * @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package. - * The class provides methods for initializing and stopping a model, and for making inference requests. - * It also subscribes to events emitted by the @janhq/core package and handles new message requests. - * @version 1.0.0 - * @module inference-mistral-extension/src/index - */ - -import { RemoteOAIEngine } from '@janhq/core' - -enum Settings { - apiKey = 'nvidia-api-key', - chatCompletionsEndPoint = 'chat-completions-endpoint', -} -/** - * A class that implements the InferenceExtension interface from the @janhq/core package. - * The class provides methods for initializing and stopping a model, and for making inference requests. - * It also subscribes to events emitted by the @janhq/core package and handles new message requests. - */ -export default class JanNVIDIANIMInferenceEngine extends RemoteOAIEngine { - inferenceUrl: string = '' - provider: string = 'nvidia' - - override async onLoad(): Promise { - super.onLoad() - - // Register Settings - this.registerSettings(SETTINGS) - this.registerModels(MODELS) - - this.apiKey = await this.getSetting(Settings.apiKey, '') - this.inferenceUrl = await this.getSetting( - Settings.chatCompletionsEndPoint, - '' - ) - - if (this.inferenceUrl.length === 0) { - SETTINGS.forEach((setting) => { - if (setting.key === Settings.chatCompletionsEndPoint) { - this.inferenceUrl = setting.controllerProps.value as string - } - }) - } - } - - onSettingUpdate(key: string, value: T): void { - if (key === Settings.apiKey) { - this.apiKey = value as string - } else if (key === Settings.chatCompletionsEndPoint) { - if (typeof value !== 'string') return - - if (value.trim().length === 0) { - SETTINGS.forEach((setting) => { - if (setting.key === Settings.chatCompletionsEndPoint) { - this.inferenceUrl = setting.controllerProps.value as string - } - }) - } else { - this.inferenceUrl = value - } - } - } -} diff --git a/extensions/inference-nvidia-extension/tsconfig.json b/extensions/inference-nvidia-extension/tsconfig.json deleted file mode 100644 index 2477d58ce..000000000 --- a/extensions/inference-nvidia-extension/tsconfig.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "compilerOptions": { - "target": "es2016", - "module": "ES6", - "moduleResolution": "node", - "outDir": "./dist", - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "strict": false, - "skipLibCheck": true, - "rootDir": "./src" - }, - "include": ["./src"] -} diff --git a/extensions/inference-openai-extension/README.md b/extensions/inference-openai-extension/README.md deleted file mode 100644 index c716c725c..000000000 --- a/extensions/inference-openai-extension/README.md +++ /dev/null @@ -1,79 +0,0 @@ -# OpenAI Engine Extension - -Created using Jan extension example - -# Create a Jan Extension using Typescript - -Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀 - -## Create Your Own Extension - -To create your own extension, you can use this repository as a template! Just follow the below instructions: - -1. Click the Use this template button at the top of the repository -2. Select Create a new repository -3. Select an owner and name for your new repository -4. Click Create repository -5. Clone your new repository - -## Initial Setup - -After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension. - -> [!NOTE] -> -> You'll need to have a reasonably modern version of -> [Node.js](https://nodejs.org) handy. If you are using a version manager like -> [`nodenv`](https://github.com/nodenv/nodenv) or -> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the -> root of your repository to install the version specified in -> [`package.json`](./package.json). Otherwise, 20.x or later should work! - -1. :hammer_and_wrench: Install the dependencies - - ```bash - npm install - ``` - -1. :building_construction: Package the TypeScript for distribution - - ```bash - npm run bundle - ``` - -1. :white_check_mark: Check your artifact - - There will be a tgz file in your extension directory now - -## Update the Extension Metadata - -The [`package.json`](package.json) file defines metadata about your extension, such as -extension name, main entry, description and version. - -When you copy this repository, update `package.json` with the name, description for your extension. - -## Update the Extension Code - -The [`src/`](./src/) directory is the heart of your extension! This contains the -source code that will be run when your extension functions are invoked. You can replace the -contents of this directory with your own code. - -There are a few things to keep in mind when writing your extension code: - -- Most Jan Extension functions are processed asynchronously. - In `index.ts`, you will see that the extension function will return a `Promise`. - - ```typescript - import { events, MessageEvent, MessageRequest } from '@janhq/core' - - function onStart(): Promise { - return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => - this.inference(data) - ) - } - ``` - - For more information about the Jan Extension Core module, see the - [documentation](https://github.com/janhq/jan/blob/main/core/README.md). - -So, what are you waiting for? Go ahead and start customizing your extension! diff --git a/extensions/inference-openai-extension/jest.config.js b/extensions/inference-openai-extension/jest.config.js deleted file mode 100644 index 3e32adceb..000000000 --- a/extensions/inference-openai-extension/jest.config.js +++ /dev/null @@ -1,9 +0,0 @@ -/** @type {import('ts-jest').JestConfigWithTsJest} */ -module.exports = { - preset: 'ts-jest', - testEnvironment: 'node', - transform: { - 'node_modules/@janhq/core/.+\\.(j|t)s?$': 'ts-jest', - }, - transformIgnorePatterns: ['node_modules/(?!@janhq/core/.*)'], -} diff --git a/extensions/inference-openai-extension/package.json b/extensions/inference-openai-extension/package.json deleted file mode 100644 index f790b60ca..000000000 --- a/extensions/inference-openai-extension/package.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "name": "@janhq/inference-openai-extension", - "productName": "OpenAI Inference Engine", - "version": "1.0.5", - "description": "This extension enables OpenAI chat completion API calls", - "main": "dist/index.js", - "module": "dist/module.js", - "engine": "openai", - "author": "Jan ", - "license": "AGPL-3.0", - "scripts": { - "build": "rolldown -c rolldown.config.mjs", - "build:publish": "rimraf *.tgz --glob || true && yarn build && npm pack && cpx *.tgz ../../pre-install" - }, - "devDependencies": { - "cpx": "^1.5.0", - "rimraf": "^3.0.2", - "rolldown": "1.0.0-beta.1", - "ts-loader": "^9.5.0", - "typescript": "^5.7.2" - }, - "dependencies": { - "@janhq/core": "../../core/package.tgz", - "fetch-retry": "^5.0.6", - "ulidx": "^2.3.0" - }, - "engines": { - "node": ">=18.0.0" - }, - "files": [ - "dist/*", - "package.json", - "README.md" - ], - "bundleDependencies": [ - "fetch-retry" - ], - "installConfig": { - "hoistingLimits": "workspaces" - }, - "packageManager": "yarn@4.5.3" -} diff --git a/extensions/inference-openai-extension/resources/settings.json b/extensions/inference-openai-extension/resources/settings.json deleted file mode 100644 index db2e80c9b..000000000 --- a/extensions/inference-openai-extension/resources/settings.json +++ /dev/null @@ -1,24 +0,0 @@ -[ - { - "key": "openai-api-key", - "title": "API Key", - "description": "The OpenAI API uses API keys for authentication. Visit your [API Keys](https://platform.openai.com/account/api-keys) page to retrieve the API key you'll use in your requests.", - "controllerType": "input", - "controllerProps": { - "placeholder": "Insert API Key", - "value": "", - "type": "password", - "inputActions": ["unobscure", "copy"] - } - }, - { - "key": "chat-completions-endpoint", - "title": "Chat Completions Endpoint", - "description": "The endpoint to use for chat completions. See the [OpenAI API documentation](https://platform.openai.com/docs/api-reference/chat/create) for more information.", - "controllerType": "input", - "controllerProps": { - "placeholder": "https://api.openai.com/v1/chat/completions", - "value": "https://api.openai.com/v1/chat/completions" - } - } -] diff --git a/extensions/inference-openai-extension/rolldown.config.mjs b/extensions/inference-openai-extension/rolldown.config.mjs deleted file mode 100644 index 9ebaace2e..000000000 --- a/extensions/inference-openai-extension/rolldown.config.mjs +++ /dev/null @@ -1,18 +0,0 @@ -import { defineConfig } from 'rolldown' -import pkgJson from './package.json' with { type: 'json' } -import settingJson from './resources/settings.json' with { type: 'json' } -import modelsJson from './resources/models.json' with { type: 'json' } - -export default defineConfig({ - input: 'src/index.ts', - output: { - format: 'esm', - file: 'dist/index.js', - }, - platform: 'browser', - define: { - MODELS: JSON.stringify(modelsJson), - SETTINGS: JSON.stringify(settingJson), - ENGINE: JSON.stringify(pkgJson.engine), - }, -}) diff --git a/extensions/inference-openai-extension/src/OpenAIExtension.test.ts b/extensions/inference-openai-extension/src/OpenAIExtension.test.ts deleted file mode 100644 index 4d46bc007..000000000 --- a/extensions/inference-openai-extension/src/OpenAIExtension.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -/** - * @jest-environment jsdom - */ -jest.mock('@janhq/core', () => ({ - ...jest.requireActual('@janhq/core/node'), - RemoteOAIEngine: jest.fn().mockImplementation(() => ({ - onLoad: jest.fn(), - registerSettings: jest.fn(), - registerModels: jest.fn(), - getSetting: jest.fn(), - onSettingUpdate: jest.fn(), - })), -})) -import JanInferenceOpenAIExtension, { Settings } from '.' - -describe('JanInferenceOpenAIExtension', () => { - let extension: JanInferenceOpenAIExtension - - beforeEach(() => { - // @ts-ignore - extension = new JanInferenceOpenAIExtension() - }) - - it('should initialize with settings and models', async () => { - await extension.onLoad() - // Assuming there are some default SETTINGS and MODELS being registered - expect(extension.apiKey).toBe(undefined) - expect(extension.inferenceUrl).toBe('') - }) - - it('should transform the payload for preview models', () => { - const payload: any = { - max_tokens: 100, - model: 'o1-mini', - // Add other required properties... - } - - const transformedPayload = extension.transformPayload(payload) - expect(transformedPayload.max_completion_tokens).toBe(payload.max_tokens) - expect(transformedPayload).not.toHaveProperty('max_tokens') - expect(transformedPayload).toHaveProperty('max_completion_tokens') - }) - - it('should not transform the payload for non-preview models', () => { - const payload: any = { - max_tokens: 100, - model: 'non-preview-model', - // Add other required properties... - } - - const transformedPayload = extension.transformPayload(payload) - expect(transformedPayload).toEqual(payload) - }) -}) diff --git a/extensions/inference-openai-extension/src/env.d.ts b/extensions/inference-openai-extension/src/env.d.ts deleted file mode 100644 index 40ca58094..000000000 --- a/extensions/inference-openai-extension/src/env.d.ts +++ /dev/null @@ -1,2 +0,0 @@ -declare const SETTINGS: SettingComponentProps[] -declare const MODELS: Model[] diff --git a/extensions/inference-openai-extension/src/index.ts b/extensions/inference-openai-extension/src/index.ts deleted file mode 100644 index 0996c6bef..000000000 --- a/extensions/inference-openai-extension/src/index.ts +++ /dev/null @@ -1,90 +0,0 @@ -/** - * @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package. - * The class provides methods for initializing and stopping a model, and for making inference requests. - * It also subscribes to events emitted by the @janhq/core package and handles new message requests. - * @version 1.0.0 - * @module inference-openai-extension/src/index - */ - -import { ModelRuntimeParams, PayloadType, RemoteOAIEngine } from '@janhq/core' - -export enum Settings { - apiKey = 'openai-api-key', - chatCompletionsEndPoint = 'chat-completions-endpoint', -} -type OpenAIPayloadType = PayloadType & - ModelRuntimeParams & { max_completion_tokens: number } -/** - * A class that implements the InferenceExtension interface from the @janhq/core package. - * The class provides methods for initializing and stopping a model, and for making inference requests. - * It also subscribes to events emitted by the @janhq/core package and handles new message requests. - */ -export default class JanInferenceOpenAIExtension extends RemoteOAIEngine { - inferenceUrl: string = '' - provider: string = 'openai' - previewModels = ['o1-mini', 'o1-preview'] - - override async onLoad(): Promise { - super.onLoad() - - // Register Settings - this.registerSettings(SETTINGS) - this.registerModels(MODELS) - - this.apiKey = await this.getSetting(Settings.apiKey, '') - this.inferenceUrl = await this.getSetting( - Settings.chatCompletionsEndPoint, - '' - ) - if (this.inferenceUrl.length === 0) { - SETTINGS.forEach((setting) => { - if (setting.key === Settings.chatCompletionsEndPoint) { - this.inferenceUrl = setting.controllerProps.value as string - } - }) - } - } - - onSettingUpdate(key: string, value: T): void { - if (key === Settings.apiKey) { - this.apiKey = value as string - } else if (key === Settings.chatCompletionsEndPoint) { - if (typeof value !== 'string') return - - if (value.trim().length === 0) { - SETTINGS.forEach((setting) => { - if (setting.key === Settings.chatCompletionsEndPoint) { - this.inferenceUrl = setting.controllerProps.value as string - } - }) - } else { - this.inferenceUrl = value - } - } - } - - /** - * Tranform the payload before sending it to the inference endpoint. - * The new preview models such as o1-mini and o1-preview replaced max_tokens by max_completion_tokens parameter. - * Others do not. - * @param payload - * @returns - */ - transformPayload = (payload: OpenAIPayloadType): OpenAIPayloadType => { - // Remove empty stop words - if (payload.stop?.length === 0) { - const { stop, ...params } = payload - payload = params - } - // Transform the payload for preview models - if (this.previewModels.includes(payload.model)) { - const { max_tokens, stop, ...params } = payload - return { - ...params, - max_completion_tokens: max_tokens, - } - } - // Pass through for non-preview models - return payload - } -} diff --git a/extensions/inference-openai-extension/tsconfig.json b/extensions/inference-openai-extension/tsconfig.json deleted file mode 100644 index 6db951c9e..000000000 --- a/extensions/inference-openai-extension/tsconfig.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "compilerOptions": { - "target": "es2016", - "module": "ES6", - "moduleResolution": "node", - "outDir": "./dist", - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "strict": false, - "skipLibCheck": true, - "rootDir": "./src" - }, - "include": ["./src"], - "exclude": ["**/*.test.ts"] -} diff --git a/extensions/inference-openrouter-extension/README.md b/extensions/inference-openrouter-extension/README.md deleted file mode 100644 index aab10755d..000000000 --- a/extensions/inference-openrouter-extension/README.md +++ /dev/null @@ -1,79 +0,0 @@ -# Open Router Engine Extension - -Created using Jan extension example - -# Create a Jan Extension using Typescript - -Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀 - -## Create Your Own Extension - -To create your own extension, you can use this repository as a template! Just follow the below instructions: - -1. Click the Use this template button at the top of the repository -2. Select Create a new repository -3. Select an owner and name for your new repository -4. Click Create repository -5. Clone your new repository - -## Initial Setup - -After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension. - -> [!NOTE] -> -> You'll need to have a reasonably modern version of -> [Node.js](https://nodejs.org) handy. If you are using a version manager like -> [`nodenv`](https://github.com/nodenv/nodenv) or -> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the -> root of your repository to install the version specified in -> [`package.json`](./package.json). Otherwise, 20.x or later should work! - -1. :hammer_and_wrench: Install the dependencies - - ```bash - npm install - ``` - -1. :building_construction: Package the TypeScript for distribution - - ```bash - npm run bundle - ``` - -1. :white_check_mark: Check your artifact - - There will be a tgz file in your extension directory now - -## Update the Extension Metadata - -The [`package.json`](package.json) file defines metadata about your extension, such as -extension name, main entry, description and version. - -When you copy this repository, update `package.json` with the name, description for your extension. - -## Update the Extension Code - -The [`src/`](./src/) directory is the heart of your extension! This contains the -source code that will be run when your extension functions are invoked. You can replace the -contents of this directory with your own code. - -There are a few things to keep in mind when writing your extension code: - -- Most Jan Extension functions are processed asynchronously. - In `index.ts`, you will see that the extension function will return a `Promise`. - - ```typescript - import { events, MessageEvent, MessageRequest } from '@janhq/core' - - function onStart(): Promise { - return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => - this.inference(data) - ) - } - ``` - - For more information about the Jan Extension Core module, see the - [documentation](https://github.com/janhq/jan/blob/main/core/README.md). - -So, what are you waiting for? Go ahead and start customizing your extension! diff --git a/extensions/inference-openrouter-extension/package.json b/extensions/inference-openrouter-extension/package.json deleted file mode 100644 index fd53ad0f5..000000000 --- a/extensions/inference-openrouter-extension/package.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "name": "@janhq/inference-openrouter-extension", - "productName": "OpenRouter Inference Engine", - "version": "1.0.0", - "description": "This extension enables Open Router chat completion API calls", - "main": "dist/index.js", - "module": "dist/module.js", - "engine": "openrouter", - "author": "Jan ", - "license": "AGPL-3.0", - "scripts": { - "build": "rolldown -c rolldown.config.mjs", - "build:publish": "rimraf *.tgz --glob || true && yarn build && npm pack && cpx *.tgz ../../pre-install" - }, - "devDependencies": { - "cpx": "^1.5.0", - "rimraf": "^3.0.2", - "rolldown": "1.0.0-beta.1", - "ts-loader": "^9.5.0", - "typescript": "^5.7.2" - }, - "dependencies": { - "@janhq/core": "../../core/package.tgz", - "fetch-retry": "^5.0.6", - "ulidx": "^2.3.0" - }, - "engines": { - "node": ">=18.0.0" - }, - "files": [ - "dist/*", - "package.json", - "README.md" - ], - "bundleDependencies": [ - "fetch-retry" - ], - "installConfig": { - "hoistingLimits": "workspaces" - }, - "packageManager": "yarn@4.5.3" -} diff --git a/extensions/inference-openrouter-extension/resources/settings.json b/extensions/inference-openrouter-extension/resources/settings.json deleted file mode 100644 index 189aee0a0..000000000 --- a/extensions/inference-openrouter-extension/resources/settings.json +++ /dev/null @@ -1,34 +0,0 @@ -[ - { - "key": "openrouter-api-key", - "title": "API Key", - "description": "The OpenRouter API uses API keys for authentication. Visit your [API Keys](https://openrouter.ai/keys) page to retrieve the API key you'll use in your requests.", - "controllerType": "input", - "controllerProps": { - "placeholder": "Insert API Key", - "value": "", - "type": "password", - "inputActions": ["unobscure", "copy"] - } - }, - { - "key": "chat-completions-endpoint", - "title": "Chat Completions Endpoint", - "description": "The endpoint to use for chat completions. See the [OpenRouter API documentation](https://openrouter.ai/docs/requests) for more information.", - "controllerType": "input", - "controllerProps": { - "placeholder": "https://openrouter.ai/api/v1/chat/completions", - "value": "https://openrouter.ai/api/v1/chat/completions" - } - }, - { - "key": "openrouter-model", - "title": "Model", - "description": "If the model parameter is omitted, the user or payer's default is used. Otherwise, remember to select a value for model from the [supported models](https://openrouter.ai/docs/models) or API, and include the organization prefix.", - "controllerType": "input", - "controllerProps": { - "placeholder": "Leave empty for default model", - "value": "" - } - } -] diff --git a/extensions/inference-openrouter-extension/rolldown.config.mjs b/extensions/inference-openrouter-extension/rolldown.config.mjs deleted file mode 100644 index 9ebaace2e..000000000 --- a/extensions/inference-openrouter-extension/rolldown.config.mjs +++ /dev/null @@ -1,18 +0,0 @@ -import { defineConfig } from 'rolldown' -import pkgJson from './package.json' with { type: 'json' } -import settingJson from './resources/settings.json' with { type: 'json' } -import modelsJson from './resources/models.json' with { type: 'json' } - -export default defineConfig({ - input: 'src/index.ts', - output: { - format: 'esm', - file: 'dist/index.js', - }, - platform: 'browser', - define: { - MODELS: JSON.stringify(modelsJson), - SETTINGS: JSON.stringify(settingJson), - ENGINE: JSON.stringify(pkgJson.engine), - }, -}) diff --git a/extensions/inference-openrouter-extension/src/env.d.ts b/extensions/inference-openrouter-extension/src/env.d.ts deleted file mode 100644 index 40ca58094..000000000 --- a/extensions/inference-openrouter-extension/src/env.d.ts +++ /dev/null @@ -1,2 +0,0 @@ -declare const SETTINGS: SettingComponentProps[] -declare const MODELS: Model[] diff --git a/extensions/inference-openrouter-extension/src/index.ts b/extensions/inference-openrouter-extension/src/index.ts deleted file mode 100644 index a34c4c38b..000000000 --- a/extensions/inference-openrouter-extension/src/index.ts +++ /dev/null @@ -1,85 +0,0 @@ -/** - * @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package. - * The class provides methods for initializing and stopping a model, and for making inference requests. - * It also subscribes to events emitted by the @janhq/core package and handles new message requests. - * @version 1.0.0 - * @module inference-openai-extension/src/index - */ - -import { RemoteOAIEngine } from '@janhq/core' -import { PayloadType } from '@janhq/core' - -enum Settings { - apiKey = 'openrouter-api-key', - model = 'openrouter-model', - chatCompletionsEndPoint = 'chat-completions-endpoint', -} - -/** - * A class that implements the InferenceExtension interface from the @janhq/core package. - * The class provides methods for initializing and stopping a model, and for making inference requests. - * It also subscribes to events emitted by the @janhq/core package and handles new message requests. - */ -export default class JanInferenceOpenRouterExtension extends RemoteOAIEngine { - inferenceUrl: string = '' - provider: string = 'openrouter' - model?: string | undefined - - override async onLoad(): Promise { - super.onLoad() - - // Register Settings - this.registerSettings(SETTINGS) - this.registerModels(MODELS) - - this.apiKey = await this.getSetting(Settings.apiKey, '') - this.inferenceUrl = await this.getSetting( - Settings.chatCompletionsEndPoint, - '' - ) - this.model = await this.getSetting(Settings.model, '') - // Openrouter uses default model on no model param set - if (!this.model?.length) this.model = undefined - if (this.inferenceUrl.length === 0) { - SETTINGS.forEach((setting) => { - if (setting.key === Settings.chatCompletionsEndPoint) { - this.inferenceUrl = setting.controllerProps.value as string - } - }) - } - } - - override async headers(): Promise { - return { - 'Content-Type': 'application/json', - 'HTTP-Referer': 'https://jan.ai', - 'Authorization': `Bearer ${this.apiKey}`, - } - } - - onSettingUpdate(key: string, value: T): void { - if (key === Settings.apiKey) { - this.apiKey = value as string - } else if (key === Settings.chatCompletionsEndPoint) { - if (typeof value !== 'string') return - - if (value.trim().length === 0) { - SETTINGS.forEach((setting) => { - if (setting.key === Settings.chatCompletionsEndPoint) { - this.inferenceUrl = setting.controllerProps.value as string - } - }) - } else { - this.inferenceUrl = value - } - } else if (key === Settings.model) { - this.model = - typeof value === 'string' && value.length > 0 ? value : undefined - } - } - - transformPayload = (payload: PayloadType) => ({ - ...payload, - model: payload.model !== 'open-router-auto' ? payload.model : this.model, - }) -} diff --git a/extensions/inference-openrouter-extension/tsconfig.json b/extensions/inference-openrouter-extension/tsconfig.json deleted file mode 100644 index 2477d58ce..000000000 --- a/extensions/inference-openrouter-extension/tsconfig.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "compilerOptions": { - "target": "es2016", - "module": "ES6", - "moduleResolution": "node", - "outDir": "./dist", - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "strict": false, - "skipLibCheck": true, - "rootDir": "./src" - }, - "include": ["./src"] -} diff --git a/extensions/inference-triton-trtllm-extension/README.md b/extensions/inference-triton-trtllm-extension/README.md deleted file mode 100644 index f9690da09..000000000 --- a/extensions/inference-triton-trtllm-extension/README.md +++ /dev/null @@ -1,75 +0,0 @@ -# Create a Jan Extension using Typescript - -Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀 - -## Create Your Own Extension - -To create your own extension, you can use this repository as a template! Just follow the below instructions: - -1. Click the Use this template button at the top of the repository -2. Select Create a new repository -3. Select an owner and name for your new repository -4. Click Create repository -5. Clone your new repository - -## Initial Setup - -After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension. - -> [!NOTE] -> -> You'll need to have a reasonably modern version of -> [Node.js](https://nodejs.org) handy. If you are using a version manager like -> [`nodenv`](https://github.com/nodenv/nodenv) or -> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the -> root of your repository to install the version specified in -> [`package.json`](./package.json). Otherwise, 20.x or later should work! - -1. :hammer_and_wrench: Install the dependencies - - ```bash - npm install - ``` - -1. :building_construction: Package the TypeScript for distribution - - ```bash - npm run bundle - ``` - -1. :white_check_mark: Check your artifact - - There will be a tgz file in your extension directory now - -## Update the Extension Metadata - -The [`package.json`](package.json) file defines metadata about your extension, such as -extension name, main entry, description and version. - -When you copy this repository, update `package.json` with the name, description for your extension. - -## Update the Extension Code - -The [`src/`](./src/) directory is the heart of your extension! This contains the -source code that will be run when your extension functions are invoked. You can replace the -contents of this directory with your own code. - -There are a few things to keep in mind when writing your extension code: - -- Most Jan Extension functions are processed asynchronously. - In `index.ts`, you will see that the extension function will return a `Promise`. - - ```typescript - import { events, MessageEvent, MessageRequest } from '@janhq/core' - - function onStart(): Promise { - return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => - this.inference(data) - ) - } - ``` - - For more information about the Jan Extension Core module, see the - [documentation](https://github.com/janhq/jan/blob/main/core/README.md). - -So, what are you waiting for? Go ahead and start customizing your extension! diff --git a/extensions/inference-triton-trtllm-extension/package.json b/extensions/inference-triton-trtllm-extension/package.json deleted file mode 100644 index 9c1f5e05f..000000000 --- a/extensions/inference-triton-trtllm-extension/package.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "name": "@janhq/inference-triton-trt-llm-extension", - "productName": "Triton-TRT-LLM Inference Engine", - "version": "1.0.0", - "description": "This extension enables Nvidia's TensorRT-LLM as an inference engine option", - "main": "dist/index.js", - "engine": "triton_trtllm", - "author": "Jan ", - "license": "AGPL-3.0", - "scripts": { - "build": "rolldown -c rolldown.config.mjs", - "build:publish": "rimraf *.tgz --glob || true && yarn build && npm pack && cpx *.tgz ../../pre-install" - }, - "devDependencies": { - "cpx": "^1.5.0", - "rimraf": "^3.0.2", - "rolldown": "1.0.0-beta.1", - "ts-loader": "^9.5.0", - "typescript": "^5.7.2" - }, - "dependencies": { - "@janhq/core": "../../core/package.tgz", - "fetch-retry": "^5.0.6", - "rxjs": "^7.8.1", - "ulidx": "^2.3.0" - }, - "engines": { - "node": ">=18.0.0" - }, - "files": [ - "dist/*", - "package.json", - "README.md" - ], - "bundleDependencies": [ - "fetch-retry" - ], - "installConfig": { - "hoistingLimits": "workspaces" - }, - "packageManager": "yarn@4.5.3" -} diff --git a/extensions/inference-triton-trtllm-extension/resources/settings.json b/extensions/inference-triton-trtllm-extension/resources/settings.json deleted file mode 100644 index 26b80a686..000000000 --- a/extensions/inference-triton-trtllm-extension/resources/settings.json +++ /dev/null @@ -1,24 +0,0 @@ -[ - { - "key": "tritonllm-api-key", - "title": "API Key", - "description": "The Triton LLM API uses API keys for authentication.", - "controllerType": "input", - "controllerProps": { - "placeholder": "Insert API Key", - "value": "", - "type": "password", - "inputActions": ["unobscure", "copy"] - } - }, - { - "key": "chat-completions-endpoint", - "title": "Chat Completions Endpoint", - "description": "The endpoint to use for chat completions.", - "controllerType": "input", - "controllerProps": { - "placeholder": "http://localhost:8000/v2/models/tensorrt_llm_bls/generate", - "value": "http://localhost:8000/v2/models/tensorrt_llm_bls/generate" - } - } -] diff --git a/extensions/inference-triton-trtllm-extension/rolldown.config.mjs b/extensions/inference-triton-trtllm-extension/rolldown.config.mjs deleted file mode 100644 index e0659a485..000000000 --- a/extensions/inference-triton-trtllm-extension/rolldown.config.mjs +++ /dev/null @@ -1,16 +0,0 @@ -import { defineConfig } from 'rolldown' -import pkgJson from './package.json' with { type: 'json' } -import settingJson from './resources/settings.json' with { type: 'json' } - -export default defineConfig({ - input: 'src/index.ts', - output: { - format: 'esm', - file: 'dist/index.js', - }, - platform: 'browser', - define: { - SETTINGS: JSON.stringify(settingJson), - ENGINE: JSON.stringify(pkgJson.engine), - }, -}) diff --git a/extensions/inference-triton-trtllm-extension/src/env.d.ts b/extensions/inference-triton-trtllm-extension/src/env.d.ts deleted file mode 100644 index 4ff21449c..000000000 --- a/extensions/inference-triton-trtllm-extension/src/env.d.ts +++ /dev/null @@ -1 +0,0 @@ -declare const SETTINGS: SettingComponentProps[] diff --git a/extensions/inference-triton-trtllm-extension/src/index.ts b/extensions/inference-triton-trtllm-extension/src/index.ts deleted file mode 100644 index 2f351defe..000000000 --- a/extensions/inference-triton-trtllm-extension/src/index.ts +++ /dev/null @@ -1,66 +0,0 @@ -/** - * @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package. - * The class provides methods for initializing and stopping a model, and for making inference requests. - * It also subscribes to events emitted by the @janhq/core package and handles new message requests. - * @version 1.0.0 - * @module inference-nvidia-triton-trt-llm-extension/src/index - */ - -import { RemoteOAIEngine } from '@janhq/core' - -enum Settings { - apiKey = 'tritonllm-api-key', - chatCompletionsEndPoint = 'chat-completions-endpoint', -} -/** - * A class that implements the InferenceExtension interface from the @janhq/core package. - * The class provides methods for initializing and stopping a model, and for making inference requests. - * It also subscribes to events emitted by the @janhq/core package and handles new message requests. - */ -export default class JanInferenceTritonTrtLLMExtension extends RemoteOAIEngine { - inferenceUrl: string = '' - provider: string = 'triton_trtllm' - - /** - * Subscribes to events emitted by the @janhq/core package. - */ - async onLoad() { - super.onLoad() - - // Register Settings - this.registerSettings(SETTINGS) - - // Retrieve API Key Setting - this.apiKey = await this.getSetting(Settings.apiKey, '') - this.inferenceUrl = await this.getSetting( - Settings.chatCompletionsEndPoint, - '' - ) - - if (this.inferenceUrl.length === 0) { - SETTINGS.forEach((setting) => { - if (setting.key === Settings.chatCompletionsEndPoint) { - this.inferenceUrl = setting.controllerProps.value as string - } - }) - } - } - - onSettingUpdate(key: string, value: T): void { - if (key === Settings.apiKey) { - this.apiKey = value as string - } else if (key === Settings.chatCompletionsEndPoint) { - if (typeof value !== 'string') return - - if (value.trim().length === 0) { - SETTINGS.forEach((setting) => { - if (setting.key === Settings.chatCompletionsEndPoint) { - this.inferenceUrl = setting.controllerProps.value as string - } - }) - } else { - this.inferenceUrl = value - } - } - } -} diff --git a/extensions/inference-triton-trtllm-extension/tsconfig.json b/extensions/inference-triton-trtllm-extension/tsconfig.json deleted file mode 100644 index 2477d58ce..000000000 --- a/extensions/inference-triton-trtllm-extension/tsconfig.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "compilerOptions": { - "target": "es2016", - "module": "ES6", - "moduleResolution": "node", - "outDir": "./dist", - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "strict": false, - "skipLibCheck": true, - "rootDir": "./src" - }, - "include": ["./src"] -} diff --git a/extensions/model-extension/src/cortex.ts b/extensions/model-extension/src/cortex.ts index ca5763962..7618e8170 100644 --- a/extensions/model-extension/src/cortex.ts +++ b/extensions/model-extension/src/cortex.ts @@ -183,6 +183,7 @@ export class CortexAPI implements ICortexAPI { model.parameters = { ...extractInferenceParams(model), ...model.parameters, + ...model.inference_params, } model.settings = { ...extractModelLoadParams(model), diff --git a/extensions/yarn.lock b/extensions/yarn.lock index d139b917c..d7ef3c4cb 100644 --- a/extensions/yarn.lock +++ b/extensions/yarn.lock @@ -509,161 +509,71 @@ __metadata: "@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Fassistant-extension%40workspace%3Aassistant-extension": version: 0.1.10 - resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=91cd98&locator=%40janhq%2Fassistant-extension%40workspace%3Aassistant-extension" + resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=e8dac7&locator=%40janhq%2Fassistant-extension%40workspace%3Aassistant-extension" dependencies: rxjs: "npm:^7.8.1" ulidx: "npm:^2.3.0" - checksum: 10c0/af79c509b1ff8a2893f5fd6545cfa8b3bb6a2e2bc13acdd5963766a1caac635b8b69ab627bfb356e052f16542f2b7187b607bdaed6acec24cd7c9a6087e4abc2 + checksum: 10c0/394734b0cc26f051a9ad138f8ae642d066acff07de26ab7c4b944d190c030cb422ac044da6352461589e13a480425200cbd8e5a549cf0181cd5c9af4b1d7eb2c languageName: node linkType: hard "@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Fconversational-extension%40workspace%3Aconversational-extension": version: 0.1.10 - resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=91cd98&locator=%40janhq%2Fconversational-extension%40workspace%3Aconversational-extension" + resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=e8dac7&locator=%40janhq%2Fconversational-extension%40workspace%3Aconversational-extension" dependencies: rxjs: "npm:^7.8.1" ulidx: "npm:^2.3.0" - checksum: 10c0/af79c509b1ff8a2893f5fd6545cfa8b3bb6a2e2bc13acdd5963766a1caac635b8b69ab627bfb356e052f16542f2b7187b607bdaed6acec24cd7c9a6087e4abc2 + checksum: 10c0/394734b0cc26f051a9ad138f8ae642d066acff07de26ab7c4b944d190c030cb422ac044da6352461589e13a480425200cbd8e5a549cf0181cd5c9af4b1d7eb2c languageName: node linkType: hard "@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Fengine-management-extension%40workspace%3Aengine-management-extension": version: 0.1.10 - resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=91cd98&locator=%40janhq%2Fengine-management-extension%40workspace%3Aengine-management-extension" + resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=e8dac7&locator=%40janhq%2Fengine-management-extension%40workspace%3Aengine-management-extension" dependencies: rxjs: "npm:^7.8.1" ulidx: "npm:^2.3.0" - checksum: 10c0/af79c509b1ff8a2893f5fd6545cfa8b3bb6a2e2bc13acdd5963766a1caac635b8b69ab627bfb356e052f16542f2b7187b607bdaed6acec24cd7c9a6087e4abc2 - languageName: node - linkType: hard - -"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Finference-anthropic-extension%40workspace%3Ainference-anthropic-extension": - version: 0.1.10 - resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=91cd98&locator=%40janhq%2Finference-anthropic-extension%40workspace%3Ainference-anthropic-extension" - dependencies: - rxjs: "npm:^7.8.1" - ulidx: "npm:^2.3.0" - checksum: 10c0/af79c509b1ff8a2893f5fd6545cfa8b3bb6a2e2bc13acdd5963766a1caac635b8b69ab627bfb356e052f16542f2b7187b607bdaed6acec24cd7c9a6087e4abc2 - languageName: node - linkType: hard - -"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Finference-cohere-extension%40workspace%3Ainference-cohere-extension": - version: 0.1.10 - resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=91cd98&locator=%40janhq%2Finference-cohere-extension%40workspace%3Ainference-cohere-extension" - dependencies: - rxjs: "npm:^7.8.1" - ulidx: "npm:^2.3.0" - checksum: 10c0/af79c509b1ff8a2893f5fd6545cfa8b3bb6a2e2bc13acdd5963766a1caac635b8b69ab627bfb356e052f16542f2b7187b607bdaed6acec24cd7c9a6087e4abc2 + checksum: 10c0/394734b0cc26f051a9ad138f8ae642d066acff07de26ab7c4b944d190c030cb422ac044da6352461589e13a480425200cbd8e5a549cf0181cd5c9af4b1d7eb2c languageName: node linkType: hard "@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Finference-cortex-extension%40workspace%3Ainference-cortex-extension": version: 0.1.10 - resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=91cd98&locator=%40janhq%2Finference-cortex-extension%40workspace%3Ainference-cortex-extension" + resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=e8dac7&locator=%40janhq%2Finference-cortex-extension%40workspace%3Ainference-cortex-extension" dependencies: rxjs: "npm:^7.8.1" ulidx: "npm:^2.3.0" - checksum: 10c0/af79c509b1ff8a2893f5fd6545cfa8b3bb6a2e2bc13acdd5963766a1caac635b8b69ab627bfb356e052f16542f2b7187b607bdaed6acec24cd7c9a6087e4abc2 - languageName: node - linkType: hard - -"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Finference-groq-extension%40workspace%3Ainference-groq-extension": - version: 0.1.10 - resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=91cd98&locator=%40janhq%2Finference-groq-extension%40workspace%3Ainference-groq-extension" - dependencies: - rxjs: "npm:^7.8.1" - ulidx: "npm:^2.3.0" - checksum: 10c0/af79c509b1ff8a2893f5fd6545cfa8b3bb6a2e2bc13acdd5963766a1caac635b8b69ab627bfb356e052f16542f2b7187b607bdaed6acec24cd7c9a6087e4abc2 - languageName: node - linkType: hard - -"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Finference-martian-extension%40workspace%3Ainference-martian-extension": - version: 0.1.10 - resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=91cd98&locator=%40janhq%2Finference-martian-extension%40workspace%3Ainference-martian-extension" - dependencies: - rxjs: "npm:^7.8.1" - ulidx: "npm:^2.3.0" - checksum: 10c0/af79c509b1ff8a2893f5fd6545cfa8b3bb6a2e2bc13acdd5963766a1caac635b8b69ab627bfb356e052f16542f2b7187b607bdaed6acec24cd7c9a6087e4abc2 - languageName: node - linkType: hard - -"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Finference-mistral-extension%40workspace%3Ainference-mistral-extension": - version: 0.1.10 - resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=91cd98&locator=%40janhq%2Finference-mistral-extension%40workspace%3Ainference-mistral-extension" - dependencies: - rxjs: "npm:^7.8.1" - ulidx: "npm:^2.3.0" - checksum: 10c0/af79c509b1ff8a2893f5fd6545cfa8b3bb6a2e2bc13acdd5963766a1caac635b8b69ab627bfb356e052f16542f2b7187b607bdaed6acec24cd7c9a6087e4abc2 - languageName: node - linkType: hard - -"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Finference-nvidia-extension%40workspace%3Ainference-nvidia-extension": - version: 0.1.10 - resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=91cd98&locator=%40janhq%2Finference-nvidia-extension%40workspace%3Ainference-nvidia-extension" - dependencies: - rxjs: "npm:^7.8.1" - ulidx: "npm:^2.3.0" - checksum: 10c0/af79c509b1ff8a2893f5fd6545cfa8b3bb6a2e2bc13acdd5963766a1caac635b8b69ab627bfb356e052f16542f2b7187b607bdaed6acec24cd7c9a6087e4abc2 - languageName: node - linkType: hard - -"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Finference-openai-extension%40workspace%3Ainference-openai-extension": - version: 0.1.10 - resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=91cd98&locator=%40janhq%2Finference-openai-extension%40workspace%3Ainference-openai-extension" - dependencies: - rxjs: "npm:^7.8.1" - ulidx: "npm:^2.3.0" - checksum: 10c0/af79c509b1ff8a2893f5fd6545cfa8b3bb6a2e2bc13acdd5963766a1caac635b8b69ab627bfb356e052f16542f2b7187b607bdaed6acec24cd7c9a6087e4abc2 - languageName: node - linkType: hard - -"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Finference-openrouter-extension%40workspace%3Ainference-openrouter-extension": - version: 0.1.10 - resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=91cd98&locator=%40janhq%2Finference-openrouter-extension%40workspace%3Ainference-openrouter-extension" - dependencies: - rxjs: "npm:^7.8.1" - ulidx: "npm:^2.3.0" - checksum: 10c0/af79c509b1ff8a2893f5fd6545cfa8b3bb6a2e2bc13acdd5963766a1caac635b8b69ab627bfb356e052f16542f2b7187b607bdaed6acec24cd7c9a6087e4abc2 - languageName: node - linkType: hard - -"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Finference-triton-trt-llm-extension%40workspace%3Ainference-triton-trtllm-extension": - version: 0.1.10 - resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=91cd98&locator=%40janhq%2Finference-triton-trt-llm-extension%40workspace%3Ainference-triton-trtllm-extension" - dependencies: - rxjs: "npm:^7.8.1" - ulidx: "npm:^2.3.0" - checksum: 10c0/af79c509b1ff8a2893f5fd6545cfa8b3bb6a2e2bc13acdd5963766a1caac635b8b69ab627bfb356e052f16542f2b7187b607bdaed6acec24cd7c9a6087e4abc2 + checksum: 10c0/394734b0cc26f051a9ad138f8ae642d066acff07de26ab7c4b944d190c030cb422ac044da6352461589e13a480425200cbd8e5a549cf0181cd5c9af4b1d7eb2c languageName: node linkType: hard "@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Fmodel-extension%40workspace%3Amodel-extension": version: 0.1.10 - resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=91cd98&locator=%40janhq%2Fmodel-extension%40workspace%3Amodel-extension" + resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=e8dac7&locator=%40janhq%2Fmodel-extension%40workspace%3Amodel-extension" dependencies: rxjs: "npm:^7.8.1" ulidx: "npm:^2.3.0" - checksum: 10c0/af79c509b1ff8a2893f5fd6545cfa8b3bb6a2e2bc13acdd5963766a1caac635b8b69ab627bfb356e052f16542f2b7187b607bdaed6acec24cd7c9a6087e4abc2 + checksum: 10c0/394734b0cc26f051a9ad138f8ae642d066acff07de26ab7c4b944d190c030cb422ac044da6352461589e13a480425200cbd8e5a549cf0181cd5c9af4b1d7eb2c languageName: node linkType: hard "@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Fmonitoring-extension%40workspace%3Amonitoring-extension": version: 0.1.10 - resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=91cd98&locator=%40janhq%2Fmonitoring-extension%40workspace%3Amonitoring-extension" + resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=e8dac7&locator=%40janhq%2Fmonitoring-extension%40workspace%3Amonitoring-extension" dependencies: rxjs: "npm:^7.8.1" ulidx: "npm:^2.3.0" - checksum: 10c0/af79c509b1ff8a2893f5fd6545cfa8b3bb6a2e2bc13acdd5963766a1caac635b8b69ab627bfb356e052f16542f2b7187b607bdaed6acec24cd7c9a6087e4abc2 + checksum: 10c0/394734b0cc26f051a9ad138f8ae642d066acff07de26ab7c4b944d190c030cb422ac044da6352461589e13a480425200cbd8e5a549cf0181cd5c9af4b1d7eb2c languageName: node linkType: hard "@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Ftensorrt-llm-extension%40workspace%3Atensorrt-llm-extension": version: 0.1.10 - resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=91cd98&locator=%40janhq%2Ftensorrt-llm-extension%40workspace%3Atensorrt-llm-extension" + resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=e8dac7&locator=%40janhq%2Ftensorrt-llm-extension%40workspace%3Atensorrt-llm-extension" dependencies: rxjs: "npm:^7.8.1" ulidx: "npm:^2.3.0" - checksum: 10c0/af79c509b1ff8a2893f5fd6545cfa8b3bb6a2e2bc13acdd5963766a1caac635b8b69ab627bfb356e052f16542f2b7187b607bdaed6acec24cd7c9a6087e4abc2 + checksum: 10c0/394734b0cc26f051a9ad138f8ae642d066acff07de26ab7c4b944d190c030cb422ac044da6352461589e13a480425200cbd8e5a549cf0181cd5c9af4b1d7eb2c languageName: node linkType: hard @@ -683,36 +593,6 @@ __metadata: languageName: unknown linkType: soft -"@janhq/inference-anthropic-extension@workspace:inference-anthropic-extension": - version: 0.0.0-use.local - resolution: "@janhq/inference-anthropic-extension@workspace:inference-anthropic-extension" - dependencies: - "@janhq/core": ../../core/package.tgz - cpx: "npm:^1.5.0" - fetch-retry: "npm:^5.0.6" - rimraf: "npm:^3.0.2" - rolldown: "npm:1.0.0-beta.1" - ts-loader: "npm:^9.5.0" - typescript: "npm:^5.7.2" - ulidx: "npm:^2.3.0" - languageName: unknown - linkType: soft - -"@janhq/inference-cohere-extension@workspace:inference-cohere-extension": - version: 0.0.0-use.local - resolution: "@janhq/inference-cohere-extension@workspace:inference-cohere-extension" - dependencies: - "@janhq/core": ../../core/package.tgz - cpx: "npm:^1.5.0" - fetch-retry: "npm:^5.0.6" - rimraf: "npm:^3.0.2" - rolldown: "npm:1.0.0-beta.1" - ts-loader: "npm:^9.5.0" - typescript: "npm:^5.7.2" - ulidx: "npm:^2.3.0" - languageName: unknown - linkType: soft - "@janhq/inference-cortex-extension@workspace:inference-cortex-extension": version: 0.0.0-use.local resolution: "@janhq/inference-cortex-extension@workspace:inference-cortex-extension" @@ -743,112 +623,6 @@ __metadata: languageName: unknown linkType: soft -"@janhq/inference-groq-extension@workspace:inference-groq-extension": - version: 0.0.0-use.local - resolution: "@janhq/inference-groq-extension@workspace:inference-groq-extension" - dependencies: - "@janhq/core": ../../core/package.tgz - cpx: "npm:^1.5.0" - fetch-retry: "npm:^5.0.6" - rimraf: "npm:^3.0.2" - rolldown: "npm:1.0.0-beta.1" - ts-loader: "npm:^9.5.0" - typescript: "npm:^5.7.2" - ulidx: "npm:^2.3.0" - languageName: unknown - linkType: soft - -"@janhq/inference-martian-extension@workspace:inference-martian-extension": - version: 0.0.0-use.local - resolution: "@janhq/inference-martian-extension@workspace:inference-martian-extension" - dependencies: - "@janhq/core": ../../core/package.tgz - cpx: "npm:^1.5.0" - fetch-retry: "npm:^5.0.6" - rimraf: "npm:^3.0.2" - rolldown: "npm:1.0.0-beta.1" - ts-loader: "npm:^9.5.0" - typescript: "npm:^5.7.2" - ulidx: "npm:^2.3.0" - languageName: unknown - linkType: soft - -"@janhq/inference-mistral-extension@workspace:inference-mistral-extension": - version: 0.0.0-use.local - resolution: "@janhq/inference-mistral-extension@workspace:inference-mistral-extension" - dependencies: - "@janhq/core": ../../core/package.tgz - cpx: "npm:^1.5.0" - fetch-retry: "npm:^5.0.6" - rimraf: "npm:^3.0.2" - rolldown: "npm:1.0.0-beta.1" - ts-loader: "npm:^9.5.0" - typescript: "npm:^5.7.2" - ulidx: "npm:^2.3.0" - languageName: unknown - linkType: soft - -"@janhq/inference-nvidia-extension@workspace:inference-nvidia-extension": - version: 0.0.0-use.local - resolution: "@janhq/inference-nvidia-extension@workspace:inference-nvidia-extension" - dependencies: - "@janhq/core": ../../core/package.tgz - cpx: "npm:^1.5.0" - fetch-retry: "npm:^5.0.6" - rimraf: "npm:^3.0.2" - rolldown: "npm:1.0.0-beta.1" - ts-loader: "npm:^9.5.0" - typescript: "npm:^5.7.2" - ulidx: "npm:^2.3.0" - languageName: unknown - linkType: soft - -"@janhq/inference-openai-extension@workspace:inference-openai-extension": - version: 0.0.0-use.local - resolution: "@janhq/inference-openai-extension@workspace:inference-openai-extension" - dependencies: - "@janhq/core": ../../core/package.tgz - cpx: "npm:^1.5.0" - fetch-retry: "npm:^5.0.6" - rimraf: "npm:^3.0.2" - rolldown: "npm:1.0.0-beta.1" - ts-loader: "npm:^9.5.0" - typescript: "npm:^5.7.2" - ulidx: "npm:^2.3.0" - languageName: unknown - linkType: soft - -"@janhq/inference-openrouter-extension@workspace:inference-openrouter-extension": - version: 0.0.0-use.local - resolution: "@janhq/inference-openrouter-extension@workspace:inference-openrouter-extension" - dependencies: - "@janhq/core": ../../core/package.tgz - cpx: "npm:^1.5.0" - fetch-retry: "npm:^5.0.6" - rimraf: "npm:^3.0.2" - rolldown: "npm:1.0.0-beta.1" - ts-loader: "npm:^9.5.0" - typescript: "npm:^5.7.2" - ulidx: "npm:^2.3.0" - languageName: unknown - linkType: soft - -"@janhq/inference-triton-trt-llm-extension@workspace:inference-triton-trtllm-extension": - version: 0.0.0-use.local - resolution: "@janhq/inference-triton-trt-llm-extension@workspace:inference-triton-trtllm-extension" - dependencies: - "@janhq/core": ../../core/package.tgz - cpx: "npm:^1.5.0" - fetch-retry: "npm:^5.0.6" - rimraf: "npm:^3.0.2" - rolldown: "npm:1.0.0-beta.1" - rxjs: "npm:^7.8.1" - ts-loader: "npm:^9.5.0" - typescript: "npm:^5.7.2" - ulidx: "npm:^2.3.0" - languageName: unknown - linkType: soft - "@janhq/model-extension@workspace:model-extension": version: 0.0.0-use.local resolution: "@janhq/model-extension@workspace:model-extension" diff --git a/web/containers/ErrorMessage/index.tsx b/web/containers/ErrorMessage/index.tsx index e0705e6b6..71fd56c5d 100644 --- a/web/containers/ErrorMessage/index.tsx +++ b/web/containers/ErrorMessage/index.tsx @@ -1,7 +1,7 @@ import { EngineManager, ErrorCode, - MessageStatus, + InferenceEngine, ThreadMessage, } from '@janhq/core' @@ -14,8 +14,6 @@ import ModalTroubleShooting, { import { MainViewState } from '@/constants/screens' -import { isLocalEngine } from '@/utils/modelEngine' - import { mainViewStateAtom } from '@/helpers/atoms/App.atom' import { activeAssistantAtom } from '@/helpers/atoms/Assistant.atom' @@ -82,7 +80,7 @@ const ErrorMessage = ({ message }: { message: ThreadMessage }) => { > {message.content[0]?.text?.value === 'Failed to fetch' && engine && - !isLocalEngine(String(engine?.name)) ? ( + engine?.name !== InferenceEngine.cortex_llamacpp ? ( No internet connection.
Switch to an on-device model or check connection. diff --git a/web/containers/Layout/BottomPanel/SystemMonitor/TableActiveModel/index.tsx b/web/containers/Layout/BottomPanel/SystemMonitor/TableActiveModel/index.tsx index 8ad16eeba..06eebea92 100644 --- a/web/containers/Layout/BottomPanel/SystemMonitor/TableActiveModel/index.tsx +++ b/web/containers/Layout/BottomPanel/SystemMonitor/TableActiveModel/index.tsx @@ -1,6 +1,6 @@ import { Tooltip, Button, Badge } from '@janhq/joi' -import { useAtom } from 'jotai' +import { useAtom, useAtomValue } from 'jotai' import { useActiveModel } from '@/hooks/useActiveModel' @@ -8,10 +8,12 @@ import { toGibibytes } from '@/utils/converter' import { isLocalEngine } from '@/utils/modelEngine' +import { installedEnginesAtom } from '@/helpers/atoms/Engines.atom' import { serverEnabledAtom } from '@/helpers/atoms/LocalServer.atom' const TableActiveModel = () => { const { activeModel, stateModel, stopModel } = useActiveModel() + const engines = useAtomValue(installedEnginesAtom) const [serverEnabled, setServerEnabled] = useAtom(serverEnabledAtom) @@ -19,7 +21,9 @@ const TableActiveModel = () => {
- {activeModel && isLocalEngine(activeModel.engine) ? ( + {activeModel && + engines && + isLocalEngine(engines, activeModel.engine) ? (
(null) const configuredModels = useAtomValue(configuredModelsAtom) - const featuredModel = configuredModels.filter( + const featuredModels = configuredModels.filter( (x) => manualRecommendationModel.includes(x.id) && x.metadata?.tags?.includes('Featured') && @@ -105,6 +102,16 @@ const ModelDropdown = ({ ) const { updateThreadMetadata } = useCreateNewThread() + const engineList = useMemo( + () => + Object.entries(engines ?? {}).flatMap((e) => ({ + name: e[0], + type: e[1][0]?.type === 'remote' ? 'remote' : 'local', + engine: e[1][0], + })), + [engines] + ) + useClickOutside(() => handleChangeStateOpen(false), null, [ dropdownOptions, toggle, @@ -122,13 +129,6 @@ const ModelDropdown = ({ [setModelDropdownState] ) - const isModelSupportRagAndTools = useCallback((model: Model) => { - return ( - model?.engine === InferenceEngine.openai || - isLocalEngine(model?.engine as InferenceEngine) - ) - }, []) - const filteredDownloadedModels = useMemo( () => configuredModels @@ -142,11 +142,12 @@ const ModelDropdown = ({ ) .filter((e) => { if (searchFilter === 'local') { - return isLocalEngine(e.engine) - } - if (searchFilter === 'remote') { - return !isLocalEngine(e.engine) + return ( + engineList.find((t) => t.engine?.engine === e.engine)?.type === + 'local' + ) } + return true }) .sort((a, b) => a.name.localeCompare(b.name)) .sort((a, b) => { @@ -164,7 +165,7 @@ const ModelDropdown = ({ return 0 } }), - [configuredModels, searchText, searchFilter, downloadedModels] + [configuredModels, searchText, searchFilter, downloadedModels, engineList] ) useEffect(() => { @@ -179,6 +180,15 @@ const ModelDropdown = ({ } }, [open]) + useEffect(() => { + setShowEngineListModel((prev) => [ + ...prev, + ...engineList + .filter((x) => (x.engine?.api_key?.length ?? 0) > 0) + .map((e) => e.name), + ]) + }, [setShowEngineListModel, engineList]) + useEffect(() => { if (!activeThread) return const modelId = activeAssistant?.model?.id @@ -193,6 +203,14 @@ const ModelDropdown = ({ activeAssistant?.model?.id, ]) + const isLocalEngine = useCallback( + (engine?: string) => { + if (!engine) return false + return engineList.some((t) => t.name === engine && t.type === 'local') + }, + [engineList] + ) + const onClickModelItem = useCallback( async (modelId: string) => { if (!activeAssistant) return @@ -210,7 +228,7 @@ const ModelDropdown = ({ tools: [ { type: 'retrieval', - enabled: isModelSupportRagAndTools(model as Model), + enabled: model?.engine === InferenceEngine.cortex, settings: { ...(activeAssistant.tools && activeAssistant.tools[0]?.settings), @@ -225,13 +243,15 @@ const ModelDropdown = ({ 8192, model?.settings.ctx_len ?? 8192 ) + const overriddenParameters = { - ctx_len: !isLocalEngine(model?.engine) - ? undefined - : defaultContextLength, - max_tokens: !isLocalEngine(model?.engine) - ? (model?.parameters.max_tokens ?? 8192) - : defaultContextLength, + ctx_len: model?.settings.ctx_len ? defaultContextLength : undefined, + max_tokens: defaultContextLength + ? Math.min( + model?.parameters.token_limit ?? 8192, + defaultContextLength + ) + : model?.parameters.token_limit, } const modelParams = { @@ -258,95 +278,17 @@ const ModelDropdown = ({ setSelectedModel, activeThread, updateThreadMetadata, - isModelSupportRagAndTools, setThreadModelParams, updateModelParameter, ] ) - const [extensionHasSettings, setExtensionHasSettings] = useState< - { name?: string; setting: string; apiKey: string; provider: string }[] - >([]) - - const inActiveEngineProvider = useAtomValue(inActiveEngineProviderAtom) - - useEffect(() => { - const getAllSettings = async () => { - const extensionsMenu: { - name?: string - setting: string - apiKey: string - provider: string - }[] = [] - const extensions = extensionManager.getAll() - - for (const extension of extensions) { - if (typeof extension.getSettings === 'function') { - const settings = await extension.getSettings() - if ( - (settings && settings.length > 0) || - (await extension.installationState()) !== 'NotRequired' - ) { - extensionsMenu.push({ - name: extension.productName, - setting: extension.name, - apiKey: - 'apiKey' in extension && typeof extension.apiKey === 'string' - ? extension.apiKey - : '', - provider: - 'provider' in extension && - typeof extension.provider === 'string' - ? extension.provider - : '', - }) - } - } - } - setExtensionHasSettings(extensionsMenu) - } - getAllSettings() - }, []) - - const findByEngine = filteredDownloadedModels - .map((x) => { - // Legacy engine support - they will be grouped under Cortex LlamaCPP - if (x.engine === InferenceEngine.nitro) - return InferenceEngine.cortex_llamacpp - return x.engine - }) - .filter((x) => !inActiveEngineProvider.includes(x)) - - const groupByEngine = findByEngine - .filter(function (item, index) { - if (findByEngine.indexOf(item) === index) return item - }) - .sort((a, b) => { - if (priorityEngine.includes(a) && priorityEngine.includes(b)) { - return priorityEngine.indexOf(a) - priorityEngine.indexOf(b) - } else if (priorityEngine.includes(a)) { - return -1 - } else if (priorityEngine.includes(b)) { - return 1 - } else { - return 0 // Leave the rest in their original order - } - }) - - const getEngineStatusReady: InferenceEngine[] = extensionHasSettings - ?.filter((e) => e.apiKey.length > 0) - .map((x) => x.provider as InferenceEngine) - - useEffect(() => { - setShowEngineListModel((prev) => [ - ...prev, - ...(getEngineStatusReady as InferenceEngine[]), - ]) - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [setShowEngineListModel, extensionHasSettings]) - - const isDownloadALocalModel = downloadedModels.some((x) => - isLocalEngine(x.engine) + const isDownloadALocalModel = useMemo( + () => + downloadedModels.some((x) => + engineList.some((t) => t.name === x.engine && t.type === 'local') + ), + [downloadedModels, engineList] ) if (strictedThread && !activeThread) { @@ -434,85 +376,193 @@ const ModelDropdown = ({ /> - {groupByEngine.map((engine, i) => { - const apiKey = !isLocalEngine(engine) - ? extensionHasSettings.filter((x) => x.provider === engine)[0] - ?.apiKey.length > 1 - : true - const engineLogo = getLogoEngine(engine as InferenceEngine) - const showModel = showEngineListModel.includes(engine) - const onClickChevron = () => { - if (showModel) { - setShowEngineListModel((prev) => - prev.filter((item) => item !== engine) - ) - } else { - setShowEngineListModel((prev) => [...prev, engine]) + {engineList + .filter((e) => e.type === searchFilter) + .filter( + (e) => + e.type === 'remote' || + e.name === InferenceEngine.cortex_llamacpp || + filteredDownloadedModels.some((e) => e.engine === e.name) + ) + .map((engine, i) => { + const isConfigured = + engine.type === 'local' || + ((engine.engine as EngineConfig).api_key?.length ?? 0) > 1 + const engineLogo = getLogoEngine(engine.name as InferenceEngine) + const showModel = showEngineListModel.includes(engine.name) + const onClickChevron = () => { + if (showModel) { + setShowEngineListModel((prev) => + prev.filter((item) => item !== engine.name) + ) + } else { + setShowEngineListModel((prev) => [...prev, engine.name]) + } } - } - return ( -
-
-
-
- {engineLogo && ( - logo - )} -
- {getTitleByEngine(engine)} -
-
-
- {!isLocalEngine(engine) && ( - - )} - {!showModel ? ( - - ) : ( -
+
+ {engine.type === 'remote' && ( + 0 + } /> - - )} + )} + {!showModel ? ( + + ) : ( + + )} +
-
- {isLocalEngine(engine) && - !isDownloadALocalModel && - showModel && - !searchText.length && ( -
    - {featuredModel.map((model) => { + {engine.type === 'local' && + !isDownloadALocalModel && + showModel && + !searchText.length && ( +
      + {featuredModels.map((model) => { + const isDownloading = downloadingModels.some( + (md) => md === model.id + ) + return ( +
    • +
      +

      + {model.name} +

      + +
      +
      + + {toGibibytes(model.metadata?.size)} + + {!isDownloading ? ( + + downloadModel( + model.sources[0].url, + model.id + ) + } + /> + ) : ( + Object.values(downloadStates) + .filter((x) => x.modelId === model.id) + .map((item) => ( + + )) + )} +
      +
      • + ) + })} +
    + )} + +
      + {filteredDownloadedModels + .filter( + (x) => + x.engine === engine.name || + (x.engine === InferenceEngine.nitro && + engine.name === InferenceEngine.cortex_llamacpp) + ) + .filter((y) => { + if (isLocalEngine(y.engine) && !searchText.length) { + return downloadedModels.find((c) => c.id === y.id) + } else { + return y + } + }) + .map((model) => { + if (!showModel) return null const isDownloading = downloadingModels.some( (md) => md === model.id ) + const isDownloaded = downloadedModels.some( + (c) => c.id === model.id + ) return (
    • { + if (!isConfigured && engine.type === 'remote') + return null + if (isDownloaded) { + onClickModelItem(model.id) + } + }} > -
      +

      {model.name} @@ -523,10 +573,12 @@ const ModelDropdown = ({ />

      - - {toGibibytes(model.metadata?.size)} - - {!isDownloading ? ( + {!isDownloaded && ( + + {toGibibytes(model.metadata?.size)} + + )} + {!isDownloading && !isDownloaded ? ( ) })} -
    - )} - -
      - {filteredDownloadedModels - .filter( - (x) => - x.engine === engine || - (x.engine === InferenceEngine.nitro && - engine === InferenceEngine.cortex_llamacpp) - ) - .filter((y) => { - if (isLocalEngine(y.engine) && !searchText.length) { - return downloadedModels.find((c) => c.id === y.id) - } else { - return y - } - }) - .map((model) => { - if (!showModel) return null - const isDownloading = downloadingModels.some( - (md) => md === model.id - ) - const isDownloaded = downloadedModels.some( - (c) => c.id === model.id - ) - return ( -
    • { - if (!apiKey && !isLocalEngine(model.engine)) - return null - if (isDownloaded) { - onClickModelItem(model.id) - } - }} - > -
      -

      - {model.name} -

      - -
      -
      - {!isDownloaded && ( - - {toGibibytes(model.metadata?.size)} - - )} - {!isDownloading && !isDownloaded ? ( - - downloadModel( - model.sources[0].url, - model.id - ) - } - /> - ) : ( - Object.values(downloadStates) - .filter((x) => x.modelId === model.id) - .map((item) => ( - - )) - )} -
      -
      • - ) - })} -
    +
+
- - ) - })} + ) + })} diff --git a/web/containers/Providers/DataLoader.tsx b/web/containers/Providers/DataLoader.tsx index 470294996..01093e4b2 100644 --- a/web/containers/Providers/DataLoader.tsx +++ b/web/containers/Providers/DataLoader.tsx @@ -6,6 +6,7 @@ import { AppConfiguration, getUserHomePath } from '@janhq/core' import { useSetAtom } from 'jotai' import useAssistants from '@/hooks/useAssistants' +import useEngines from '@/hooks/useEngines' import useGetSystemResources from '@/hooks/useGetSystemResources' import useModels from '@/hooks/useModels' import useThreads from '@/hooks/useThreads' @@ -25,6 +26,7 @@ const DataLoader: React.FC = () => { const setJanDefaultDataFolder = useSetAtom(defaultJanDataFolderAtom) const setJanSettingScreen = useSetAtom(janSettingScreenAtom) const { getData: loadModels } = useModels() + const { getData: loadEngines } = useEngines() useThreads() useAssistants() @@ -33,6 +35,7 @@ const DataLoader: React.FC = () => { useEffect(() => { // Load data once loadModels() + loadEngines() // eslint-disable-next-line react-hooks/exhaustive-deps }, []) diff --git a/web/containers/Providers/ModelHandler.tsx b/web/containers/Providers/ModelHandler.tsx index 42376c081..32182804c 100644 --- a/web/containers/Providers/ModelHandler.tsx +++ b/web/containers/Providers/ModelHandler.tsx @@ -18,7 +18,7 @@ import { extractInferenceParams, ModelExtension, } from '@janhq/core' -import { useAtom, useAtomValue, useSetAtom } from 'jotai' +import { useAtomValue, useSetAtom } from 'jotai' import { ulid } from 'ulidx' import { activeModelAtom, stateModelAtom } from '@/hooks/useActiveModel' @@ -34,6 +34,7 @@ import { deleteMessageAtom, subscribedGeneratingMessageAtom, } from '@/helpers/atoms/ChatMessage.atom' +import { installedEnginesAtom } from '@/helpers/atoms/Engines.atom' import { downloadedModelsAtom } from '@/helpers/atoms/Model.atom' import { updateThreadWaitingForResponseAtom, @@ -74,6 +75,7 @@ export default function ModelHandler() { const activeModelParams = useAtomValue(getActiveThreadModelParamsAtom) const activeModelParamsRef = useRef(activeModelParams) const setTokenSpeed = useSetAtom(tokenSpeedAtom) + const engines = useAtomValue(installedEnginesAtom) useEffect(() => { activeThreadRef.current = activeThread @@ -241,7 +243,8 @@ export default function ModelHandler() { } else if ( message.status === MessageStatus.Error && activeModelRef.current?.engine && - isLocalEngine(activeModelRef.current.engine) + engines && + isLocalEngine(engines, activeModelRef.current.engine) ) { ;(async () => { if ( @@ -332,7 +335,9 @@ export default function ModelHandler() { if (!activeModelRef.current) return // Check model engine; we don't want to generate a title when it's not a local engine. remote model using first promp - if (!isLocalEngine(activeModelRef.current?.engine as InferenceEngine)) { + if ( + !isLocalEngine(engines, activeModelRef.current?.engine as InferenceEngine) + ) { const updatedThread: Thread = { ...thread, title: (thread.metadata?.lastMessage as string) || defaultThreadTitle, diff --git a/web/containers/SetupRemoteModel/index.tsx b/web/containers/SetupRemoteModel/index.tsx index 1f5478d73..b4b2cfa42 100644 --- a/web/containers/SetupRemoteModel/index.tsx +++ b/web/containers/SetupRemoteModel/index.tsx @@ -1,5 +1,3 @@ -import { useState, useEffect } from 'react' - import { InferenceEngine } from '@janhq/core' import { Button } from '@janhq/joi' @@ -8,77 +6,23 @@ import { SettingsIcon, PlusIcon } from 'lucide-react' import { MainViewState } from '@/constants/screens' -import { isLocalEngine } from '@/utils/modelEngine' - -import { extensionManager } from '@/extension' import { mainViewStateAtom } from '@/helpers/atoms/App.atom' import { selectedSettingAtom } from '@/helpers/atoms/Setting.atom' type Props = { engine: InferenceEngine + isConfigured: boolean } -const SetupRemoteModel = ({ engine }: Props) => { +const SetupRemoteModel = ({ engine, isConfigured }: Props) => { const setSelectedSetting = useSetAtom(selectedSettingAtom) const setMainViewState = useSetAtom(mainViewStateAtom) - const [extensionHasSettings, setExtensionHasSettings] = useState< - { name?: string; setting: string; apiKey: string; provider: string }[] - >([]) - - useEffect(() => { - const getAllSettings = async () => { - const extensionsMenu: { - name?: string - setting: string - apiKey: string - provider: string - }[] = [] - const extensions = extensionManager.getAll() - - for (const extension of extensions) { - if (typeof extension.getSettings === 'function') { - const settings = await extension.getSettings() - - if ( - (settings && settings.length > 0) || - (await extension.installationState()) !== 'NotRequired' - ) { - extensionsMenu.push({ - name: extension.productName, - setting: extension.name, - apiKey: - 'apiKey' in extension && typeof extension.apiKey === 'string' - ? extension.apiKey - : '', - provider: - 'provider' in extension && - typeof extension.provider === 'string' - ? extension.provider - : '', - }) - } - } - } - setExtensionHasSettings(extensionsMenu) - } - getAllSettings() - }, []) - const onSetupItemClick = (setting: InferenceEngine) => { + setSelectedSetting(setting) setMainViewState(MainViewState.Settings) - setSelectedSetting( - extensionHasSettings.filter((x) => - x.provider.toLowerCase().includes(setting) - )[0]?.setting - ) } - const apiKey = !isLocalEngine(engine) - ? extensionHasSettings.filter((x) => x.provider === engine)[0]?.apiKey - .length > 1 - : true - return ( + + + + + ) +} + +export default LocalEngineItems diff --git a/web/screens/Settings/Engines/Settings.tsx b/web/screens/Settings/Engines/LocalEngineSettings.tsx similarity index 99% rename from web/screens/Settings/Engines/Settings.tsx rename to web/screens/Settings/Engines/LocalEngineSettings.tsx index 8cc8501ef..41ca622da 100644 --- a/web/screens/Settings/Engines/Settings.tsx +++ b/web/screens/Settings/Engines/LocalEngineSettings.tsx @@ -36,7 +36,7 @@ const os = () => { } } -const EngineSettings = ({ engine }: { engine: InferenceEngine }) => { +const LocalEngineSettings = ({ engine }: { engine: InferenceEngine }) => { const { installedEngines, mutate: mutateInstalledEngines } = useGetInstalledEngines(engine) const { defaultEngineVariant, mutate: mutateDefaultEngineVariant } = @@ -343,4 +343,4 @@ const EngineSettings = ({ engine }: { engine: InferenceEngine }) => { ) } -export default EngineSettings +export default LocalEngineSettings diff --git a/web/screens/Settings/Engines/ModalAddRemoteEngine.tsx b/web/screens/Settings/Engines/ModalAddRemoteEngine.tsx new file mode 100644 index 000000000..7ea4877cc --- /dev/null +++ b/web/screens/Settings/Engines/ModalAddRemoteEngine.tsx @@ -0,0 +1,213 @@ +import { memo, useState } from 'react' +import { useForm } from 'react-hook-form' + +import { zodResolver } from '@hookform/resolvers/zod' + +import { Button, Input, Modal, TextArea } from '@janhq/joi' +import { PlusIcon } from 'lucide-react' +import { z } from 'zod' + +import { addRemoteEngine, useGetEngines } from '@/hooks/useEngineManagement' + +const engineSchema = z.object({ + engineName: z.string().min(1, 'Engine name is required'), + modelListUrl: z.string().url('Enter a valid Model List URL'), + headerTemplate: z.string().optional(), + apiKey: z.string().optional(), + requestFormat: z.string().optional(), + responseFormat: z.string().optional(), +}) + +const ModalAddRemoteEngine = () => { + const [open, setOpen] = useState(false) + const { mutate: mutateListEngines } = useGetEngines() + const { + register, + handleSubmit, + formState: { errors }, + } = useForm({ + resolver: zodResolver(engineSchema), + defaultValues: { + engineName: '', + apiUrl: '', + modelListUrl: '', + headerTemplate: '', + apiKey: '', + requestFormat: '', + responseFormat: '', + }, + }) + + const onSubmit = async (data: z.infer) => { + await addRemoteEngine({ + type: 'remote', + engine: data.engineName, + api_key: data.apiKey, + metadata: { + header_template: data.headerTemplate, + get_models_url: data.modelListUrl, + transform_req: { + chat_completions: { + template: data.requestFormat, + }, + }, + transform_resp: { + chat_completions: { + template: data.requestFormat, + }, + }, + }, + }) + mutateListEngines() + + setOpen(false) + } + + // Helper to render labels with asterisks for required fields + const renderLabel = (label: string, isRequired: boolean, desc?: string) => ( + <> + + {label} {isRequired && *} + +

+ {desc} +

+ + ) + + return ( + +

Install Remote Engine

+

+ Only OpenAI API-compatible engines are supported +

+ + } + fullPage + open={open} + onOpenChange={() => setOpen(!open)} + trigger={ + + } + content={ +
+
+
+ + + {errors.engineName && ( +

+ {errors.engineName.message} +

+ )} +
+ +
+ + + {errors.modelListUrl && ( +

+ {errors.modelListUrl.message} +

+ )} +
+ +
+ + +
+ +
+ +