Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: remote engine management (#4364)

Browse Source 
* feat: remote engine management

* chore: fix linter issue

* chore: remove unused imports

* fix: populate engines, models and legacy settings (#4403)

* fix: populate engines, models and legacy settings

* chore: legacy logics update configured remote engine

* fix: check exist path before reading

* fix: engines and models persist - race condition

* chore: update issue state

* test: update test cases

* chore: bring back Cortex extension settings

* chore: setup button gear / plus based apikey

* chore: fix remote engine from welcome screen

* chore: resolve linter issue

* chore: support request headers template

* chore: update engines using header_template instead of api_key_template

* chore: update models on changes

* fix: anthropic response template

* chore: fix welcome screen and debounce update value input

* chore: update engines list on changes

* chore: update engines list on change

* chore: update desc form add modal remote engines

* chore: bump cortex version to latest RC

* chore: fix linter

* fix: transform payload of Anthropic and OpenAI

* fix: typo

* fix: openrouter model id for auto routing

* chore: remove remote engine URL setting

* chore: add cohere engine and model support

* fix: should not clean on app launch - models list display issue

* fix: local engine check logic

* chore: bump app version to latest release 0.5.13

* test: fix failed tests

---------

Co-authored-by: Louis <louis@jan.ai>
This commit is contained in:
Faisal Amir 2025-01-14 17:29:56 +07:00 committed by GitHub
parent 6b98f459d4
commit 2a0601f75a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
138 changed files with 1949 additions and 4204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
core/src/browser/extensions/engines/helpers/sse.ts
View File

@ -22,7 +22,9 @@ export function requestInference(
headers: {
'Content-Type': 'application/json',
'Access-Control-Allow-Origin': '*',
'Accept': model.parameters?.stream ? 'text/event-stream' : 'application/json',
'Accept': model.parameters?.stream
? 'text/event-stream'
: 'application/json',
...headers,
},
body: JSON.stringify(requestBody),
@ -47,12 +49,24 @@ export function requestInference(
}
// There could be overriden stream parameter in the model
// that is set in request body (transformed payload)
if (requestBody?.stream === false || model.parameters?.stream === false) {
if (
requestBody?.stream === false ||
model.parameters?.stream === false
) {
const data = await response.json()
if (data.error || data.message) {
subscriber.error(data.error ?? data)
subscriber.complete()
return
}
if (transformResponse) {
subscriber.next(transformResponse(data))
} else {
subscriber.next(data.choices[0]?.message?.content ?? '')
subscriber.next(
data.choices
? data.choices[0]?.message?.content
: (data.content[0]?.text ?? '')
)
}
} else {
const stream = response.body

31
core/src/browser/extensions/enginesManagement.ts
View File

@ -3,6 +3,7 @@ import {
Engines,
EngineVariant,
EngineReleased,
EngineConfig,
DefaultEngineVariant,
} from '../../types'
import { BaseExtension, ExtensionTypeEnum } from '../extension'
@ -55,8 +56,16 @@ export abstract class EngineManagementExtension extends BaseExtension {
* @returns A Promise that resolves to intall of engine.
*/
abstract installEngine(
name: InferenceEngine,
engineConfig: { variant: string; version?: string }
name: string,
engineConfig: EngineConfig
): Promise<{ messages: string }>
/**
* Add a new remote engine
* @returns A Promise that resolves to intall of engine.
*/
abstract addRemoteEngine(
engineConfig: EngineConfig
): Promise<{ messages: string }>
/**
@ -65,14 +74,16 @@ export abstract class EngineManagementExtension extends BaseExtension {
*/
abstract uninstallEngine(
name: InferenceEngine,
engineConfig: { variant: string; version: string }
engineConfig: EngineConfig
): Promise<{ messages: string }>
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to an object of default engine.
*/
abstract getDefaultEngineVariant(name: InferenceEngine): Promise<DefaultEngineVariant>
abstract getDefaultEngineVariant(
name: InferenceEngine
): Promise<DefaultEngineVariant>
/**
* @body variant - string
@ -81,11 +92,19 @@ export abstract class EngineManagementExtension extends BaseExtension {
*/
abstract setDefaultEngineVariant(
name: InferenceEngine,
engineConfig: { variant: string; version: string }
engineConfig: EngineConfig
): Promise<{ messages: string }>
/**
* @returns A Promise that resolves to update engine.
*/
abstract updateEngine(name: InferenceEngine): Promise<{ messages: string }>
abstract updateEngine(
name: InferenceEngine,
engineConfig?: EngineConfig
): Promise<{ messages: string }>
/**
* @returns A Promise that resolves to an object of remote models list .
*/
abstract getRemoteModels(name: InferenceEngine | string): Promise<any>
}

33
core/src/node/helper/config.test.ts
View File

@ -1,28 +1,19 @@
import { getEngineConfiguration } from './config';
import { getAppConfigurations, defaultAppConfig } from './config';
import { getJanExtensionsPath } from './config';
import { getJanDataFolderPath } from './config';
it('should return undefined for invalid engine ID', async () => {
const config = await getEngineConfiguration('invalid_engine');
expect(config).toBeUndefined();
});
import { getAppConfigurations, defaultAppConfig } from './config'
import { getJanExtensionsPath, getJanDataFolderPath } from './config'
it('should return default config when CI is e2e', () => {
process.env.CI = 'e2e';
const config = getAppConfigurations();
expect(config).toEqual(defaultAppConfig());
});
process.env.CI = 'e2e'
const config = getAppConfigurations()
expect(config).toEqual(defaultAppConfig())
})
it('should return extensions path when retrieved successfully', () => {
const extensionsPath = getJanExtensionsPath();
expect(extensionsPath).not.toBeUndefined();
});
const extensionsPath = getJanExtensionsPath()
expect(extensionsPath).not.toBeUndefined()
})
it('should return data folder path when retrieved successfully', () => {
const dataFolderPath = getJanDataFolderPath();
expect(dataFolderPath).not.toBeUndefined();
});
const dataFolderPath = getJanDataFolderPath()
expect(dataFolderPath).not.toBeUndefined()
})

108
core/src/node/helper/config.ts
View File

@ -1,8 +1,7 @@
import { AppConfiguration, SettingComponentProps } from '../../types'
import { AppConfiguration } from '../../types'
import { join, resolve } from 'path'
import fs from 'fs'
import os from 'os'
import childProcess from 'child_process'
const configurationFileName = 'settings.json'
/**
@ -19,7 +18,9 @@ export const getAppConfigurations = (): AppConfiguration => {
if (!fs.existsSync(configurationFile)) {
// create default app config if we don't have one
console.debug(`App config not found, creating default config at ${configurationFile}`)
console.debug(
`App config not found, creating default config at ${configurationFile}`
)
fs.writeFileSync(configurationFile, JSON.stringify(appDefaultConfiguration))
return appDefaultConfiguration
}
@ -30,20 +31,28 @@ export const getAppConfigurations = (): AppConfiguration => {
)
return appConfigurations
} catch (err) {
console.error(`Failed to read app config, return default config instead! Err: ${err}`)
console.error(
`Failed to read app config, return default config instead! Err: ${err}`
)
return defaultAppConfig()
}
}
const getConfigurationFilePath = () =>
join(
global.core?.appPath() || process.env[process.platform == 'win32' ? 'USERPROFILE' : 'HOME'],
global.core?.appPath() ||
process.env[process.platform == 'win32' ? 'USERPROFILE' : 'HOME'],
configurationFileName
)
export const updateAppConfiguration = (configuration: AppConfiguration): Promise<void> => {
export const updateAppConfiguration = (
configuration: AppConfiguration
): Promise<void> => {
const configurationFile = getConfigurationFilePath()
console.debug('updateAppConfiguration, configurationFile: ', configurationFile)
console.debug(
'updateAppConfiguration, configurationFile: ',
configurationFile
)
fs.writeFileSync(configurationFile, JSON.stringify(configuration))
return Promise.resolve()
@ -69,86 +78,6 @@ export const getJanExtensionsPath = (): string => {
return join(appConfigurations.data_folder, 'extensions')
}
/**
* Utility function to physical cpu count
*
* @returns {number} The physical cpu count.
*/
export const physicalCpuCount = async (): Promise<number> => {
const platform = os.platform()
try {
if (platform === 'linux') {
const output = await exec('lscpu -p | egrep -v "^#" | sort -u -t, -k 2,4 | wc -l')
return parseInt(output.trim(), 10)
} else if (platform === 'darwin') {
const output = await exec('sysctl -n hw.physicalcpu_max')
return parseInt(output.trim(), 10)
} else if (platform === 'win32') {
const output = await exec('WMIC CPU Get NumberOfCores')
return output
.split(os.EOL)
.map((line: string) => parseInt(line))
.filter((value: number) => !isNaN(value))
.reduce((sum: number, number: number) => sum + number, 1)
} else {
const cores = os.cpus().filter((cpu: any, index: number) => {
const hasHyperthreading = cpu.model.includes('Intel')
const isOdd = index % 2 === 1
return !hasHyperthreading || isOdd
})
return cores.length
}
} catch (err) {
console.warn('Failed to get physical CPU count', err)
// Divide by 2 to get rid of hyper threading
const coreCount = Math.ceil(os.cpus().length / 2)
console.debug('Using node API to get physical CPU count:', coreCount)
return coreCount
}
}
const exec = async (command: string): Promise<string> => {
return new Promise((resolve, reject) => {
childProcess.exec(command, { encoding: 'utf8' }, (error, stdout) => {
if (error) {
reject(error)
} else {
resolve(stdout)
}
})
})
}
// a hacky way to get the api key. we should comes up with a better
// way to handle this
export const getEngineConfiguration = async (engineId: string) => {
if (engineId !== 'openai' && engineId !== 'groq') return undefined
const settingDirectoryPath = join(
getJanDataFolderPath(),
'settings',
'@janhq',
engineId === 'openai' ? 'inference-openai-extension' : 'inference-groq-extension',
'settings.json'
)
const content = fs.readFileSync(settingDirectoryPath, 'utf-8')
const settings: SettingComponentProps[] = JSON.parse(content)
const apiKeyId = engineId === 'openai' ? 'openai-api-key' : 'groq-api-key'
const keySetting = settings.find((setting) => setting.key === apiKeyId)
let fullUrl = settings.find((setting) => setting.key === 'chat-completions-endpoint')
?.controllerProps.value
let apiKey = keySetting?.controllerProps.value
if (typeof apiKey !== 'string') apiKey = ''
if (typeof fullUrl !== 'string') fullUrl = ''
return {
api_key: apiKey,
full_url: fullUrl,
}
}
/**
* Default app configurations
* App Data Folder default to Electron's userData
@ -158,7 +87,10 @@ export const getEngineConfiguration = async (engineId: string) => {
*/
export const defaultAppConfig = (): AppConfiguration => {
const { app } = require('electron')
const defaultJanDataFolder = join(app?.getPath('userData') ?? os?.homedir() ?? '', 'data')
const defaultJanDataFolder = join(
app?.getPath('userData') ?? os?.homedir() ?? '',
'data'
)
return {
data_folder:
process.env.CI === 'e2e'

14
core/src/node/helper/resource.test.ts
View File

@ -1,15 +1,9 @@
import { getSystemResourceInfo } from './resource';
import { getSystemResourceInfo } from './resource'
it('should return the correct system resource information with a valid CPU count', async () => {
const mockCpuCount = 4;
jest.spyOn(require('./config'), 'physicalCpuCount').mockResolvedValue(mockCpuCount);
const logSpy = jest.spyOn(require('./logger'), 'log').mockImplementation(() => {});
const result = await getSystemResourceInfo();
const result = await getSystemResourceInfo()
expect(result).toEqual({
numCpuPhysicalCore: mockCpuCount,
memAvailable: 0,
});
expect(logSpy).toHaveBeenCalledWith(`[CORTEX]::CPU information - ${mockCpuCount}`);
});
})
})

6
core/src/node/helper/resource.ts
View File

@ -1,13 +1,7 @@
import { SystemResourceInfo } from '../../types'
import { physicalCpuCount } from './config'
import { log } from './logger'
export const getSystemResourceInfo = async (): Promise<SystemResourceInfo> => {
const cpu = await physicalCpuCount()
log(`[CORTEX]::CPU information - ${cpu}`)
return {
numCpuPhysicalCore: cpu,
memAvailable: 0, // TODO: this should not be 0
}
}

28
core/src/types/engine/index.ts
View File

@ -1,7 +1,23 @@
import { InferenceEngine } from '../../types'
export type Engines = {
[key in InferenceEngine]: EngineVariant[]
[key in InferenceEngine]: (EngineVariant & EngineConfig)[]
}
export type EngineMetadata = {
get_models_url?: string
header_template?: string
transform_req?: {
chat_completions?: {
url?: string
template?: string
}
}
transform_resp?: {
chat_completions?: {
template?: string
}
}
}
export type EngineVariant = {
@ -23,6 +39,16 @@ export type EngineReleased = {
size: number
}
export type EngineConfig = {
engine?: string
version?: string
variant?: string
type?: string
url?: string
api_key?: string
metadata?: EngineMetadata
}
export enum EngineEvent {
OnEngineUpdate = 'OnEngineUpdate',
}

18
core/src/types/message/messageEntity.ts
View File

@ -32,9 +32,8 @@ export type ThreadMessage = {
completed_at: number
/** The additional metadata of this message. **/
metadata?: Record<string, unknown>
/** Type of the message */
type?: string
/** The error code which explain what error type. Used in conjunction with MessageStatus.Error */
error_code?: ErrorCode
}
@ -72,6 +71,10 @@ export type MessageRequest = {
// TODO: deprecate threadId field
thread?: Thread
/** Engine name to process */
engine?: string
/** Message type */
type?: string
}
@ -147,7 +150,9 @@ export interface Attachment {
/**
* The tools to add this file to.
*/
tools?: Array<CodeInterpreterTool | Attachment.AssistantToolsFileSearchTypeOnly>
tools?: Array<
CodeInterpreterTool | Attachment.AssistantToolsFileSearchTypeOnly
>
}
export namespace Attachment {
@ -166,5 +171,10 @@ export interface IncompleteDetails {
/**
* The reason the message is incomplete.
*/
reason: 'content_filter' | 'max_tokens' | 'run_cancelled' | 'run_expired' | 'run_failed'
reason:
| 'content_filter'
| 'max_tokens'
| 'run_cancelled'
| 'run_expired'
| 'run_failed'
}

1
core/src/types/miscellaneous/systemResourceInfo.ts
View File

@ -1,5 +1,4 @@
export type SystemResourceInfo = {
numCpuPhysicalCore: number
memAvailable: number
}

2
electron/package.json
View File

@ -1,6 +1,6 @@
{
"name": "jan",
"version": "0.1.3",
"version": "0.1.1736316956",
"main": "./build/main.js",
"author": "Jan <service@jan.ai>",
"license": "MIT",

39
extensions/engine-management-extension/engines.mjs Normal file
View File

@ -0,0 +1,39 @@
import anthropic from './resources/anthropic.json' with { type: 'json' }
import cohere from './resources/cohere.json' with { type: 'json' }
import openai from './resources/openai.json' with { type: 'json' }
import openrouter from './resources/openrouter.json' with { type: 'json' }
import groq from './resources/groq.json' with { type: 'json' }
import martian from './resources/martian.json' with { type: 'json' }
import mistral from './resources/mistral.json' with { type: 'json' }
import nvidia from './resources/nvidia.json' with { type: 'json' }
import anthropicModels from './models/anthropic.json' with { type: 'json' }
import cohereModels from './models/cohere.json' with { type: 'json' }
import openaiModels from './models/openai.json' with { type: 'json' }
import openrouterModels from './models/openrouter.json' with { type: 'json' }
import groqModels from './models/groq.json' with { type: 'json' }
import martianModels from './models/martian.json' with { type: 'json' }
import mistralModels from './models/mistral.json' with { type: 'json' }
import nvidiaModels from './models/nvidia.json' with { type: 'json' }
const engines = [
anthropic,
openai,
cohere,
openrouter,
groq,
mistral,
martian,
nvidia,
]
const models = [
...anthropicModels,
...openaiModels,
...cohereModels,
...openrouterModels,
...groqModels,
...mistralModels,
...martianModels,
...nvidiaModels,
]
export { engines, models }

45
extensions/inference-anthropic-extension/resources/models.json → extensions/engine-management-extension/models/anthropic.json
View File

@ -1,74 +1,41 @@
[
{
"sources": [
{
"url": "https://www.anthropic.com/"
}
],
"id": "claude-3-opus-latest",
"model": "claude-3-opus-latest",
"object": "model",
"name": "Claude 3 Opus Latest",
"version": "1.0",
"description": "Claude 3 Opus is a powerful model suitables for highly complex task.",
"format": "api",
"settings": {},
"parameters": {
"inference_params": {
"max_tokens": 4096,
"temperature": 0.7,
"stream": false
},
"metadata": {
"author": "Anthropic",
"tags": ["General", "Big Context Length"]
},
"engine": "anthropic"
},
{
"sources": [
{
"url": "https://www.anthropic.com/"
}
],
"id": "claude-3-5-haiku-latest",
"model": "claude-3-5-haiku-latest",
"object": "model",
"name": "Claude 3.5 Haiku Latest",
"version": "1.0",
"description": "Claude 3.5 Haiku is the fastest model provides near-instant responsiveness.",
"format": "api",
"settings": {},
"parameters": {
"inference_params": {
"max_tokens": 8192,
"temperature": 0.7,
"stream": false
},
"metadata": {
"author": "Anthropic",
"tags": ["General", "Big Context Length"]
},
"engine": "anthropic"
},
{
"sources": [
{
"url": "https://www.anthropic.com/"
}
],
"id": "claude-3-5-sonnet-latest",
"model": "claude-3-5-sonnet-latest",
"object": "model",
"name": "Claude 3.5 Sonnet Latest",
"version": "1.0",
"description": "Claude 3.5 Sonnet raises the industry bar for intelligence, outperforming competitor models and Claude 3 Opus on a wide range of evaluations, with the speed and cost of our mid-tier model, Claude 3 Sonnet.",
"format": "api",
"settings": {},
"parameters": {
"inference_params": {
"max_tokens": 8192,
"temperature": 0.7,
"stream": true
},
"metadata": {
"author": "Anthropic",
"tags": ["General", "Big Context Length"]
},
"engine": "anthropic"
}
]

40
extensions/inference-cohere-extension/resources/models.json → extensions/engine-management-extension/models/cohere.json
View File

@ -1,56 +1,28 @@
[
{
"sources": [
{
"url": "https://cohere.com"
}
],
"id": "command-r-plus",
"model": "command-r-plus",
"object": "model",
"name": "Command R+",
"version": "1.0",
"description": "Command R+ is an instruction-following conversational model that performs language tasks at a higher quality, more reliably, and with a longer context than previous models. It is best suited for complex RAG workflows and multi-step tool use.",
"format": "api",
"settings": {},
"parameters": {
"max_tokens": 128000,
"inference_params": {
"max_tokens": 4096,
"temperature": 0.7,
"stream": false
},
"metadata": {
"author": "Cohere",
"tags": [
"General",
"Big Context Length"
]
},
"engine": "cohere"
},
{
"sources": [
{
"url": "https://cohere.com"
}
],
"id": "command-r",
"model": "command-r",
"object": "model",
"name": "Command R",
"version": "1.0",
"description": "Command R is an instruction-following conversational model that performs language tasks at a higher quality, more reliably, and with a longer context than previous models. It can be used for complex workflows like code generation, retrieval augmented generation (RAG), tool use, and agents.",
"format": "api",
"settings": {},
"parameters": {
"max_tokens": 128000,
"inference_params": {
"max_tokens": 4096,
"temperature": 0.7,
"stream": false
},
"metadata": {
"author": "Cohere",
"tags": [
"General",
"Big Context Length"
]
},
"engine": "cohere"
}
]

217
extensions/inference-groq-extension/resources/models.json → extensions/engine-management-extension/models/groq.json
View File

@ -1,18 +1,11 @@
[
{
"sources": [
{
"url": "https://groq.com"
}
],
"id": "llama3-70b-8192",
"model": "llama3-70b-8192",
"object": "model",
"name": "Groq Llama 3 70b",
"version": "1.1",
"description": "Groq Llama 3 70b with supercharged speed!",
"format": "api",
"settings": {},
"parameters": {
"inference_params": {
"max_tokens": 8192,
"temperature": 0.7,
"top_p": 0.95,
@ -21,29 +14,15 @@
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "Meta",
"tags": [
"General",
"Big Context Length"
]
},
"engine": "groq"
},
{
"sources": [
{
"url": "https://groq.com"
}
],
"id": "llama3-8b-8192",
"model": "llama3-8b-8192",
"object": "model",
"name": "Groq Llama 3 8b",
"version": "1.1",
"description": "Groq Llama 3 8b with supercharged speed!",
"format": "api",
"settings": {},
"parameters": {
"inference_params": {
"max_tokens": 8192,
"temperature": 0.7,
"top_p": 0.95,
@ -52,29 +31,15 @@
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "Meta",
"tags": [
"General",
"Big Context Length"
]
},
"engine": "groq"
},
{
"sources": [
{
"url": "https://groq.com"
}
],
"id": "llama-3.1-8b-instant",
"model": "llama-3.1-8b-instant",
"object": "model",
"name": "Groq Llama 3.1 8b Instant",
"version": "1.1",
"description": "Groq Llama 3.1 8b with supercharged speed!",
"format": "api",
"settings": {},
"parameters": {
"inference_params": {
"max_tokens": 8000,
"temperature": 0.7,
"top_p": 0.95,
@ -83,29 +48,15 @@
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "Meta",
"tags": [
"General",
"Big Context Length"
]
},
"engine": "groq"
},
{
"sources": [
{
"url": "https://groq.com"
}
],
"id": "llama-3.2-11b-text-preview",
"model": "llama-3.2-11b-text-preview",
"object": "model",
"name": "Groq Llama 3.2 11b Text Preview",
"version": "1.1",
"description": "Groq Llama 3.2 11b Text Preview with supercharged speed!",
"format": "api",
"settings": {},
"parameters": {
"inference_params": {
"max_tokens": 8192,
"temperature": 0.7,
"top_p": 0.95,
@ -114,29 +65,15 @@
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "Meta",
"tags": [
"General",
"Big Context Length"
]
},
"engine": "groq"
},
{
"sources": [
{
"url": "https://groq.com"
}
],
"id": "llama-3.2-11b-vision-preview",
"model": "llama-3.2-11b-vision-preview",
"object": "model",
"name": "Groq Llama 3.2 11b Vision Preview",
"version": "1.1",
"description": "Groq Llama 3.2 11b Vision Preview with supercharged speed!",
"format": "api",
"settings": {},
"parameters": {
"inference_params": {
"max_tokens": 8192,
"temperature": 0.7,
"top_p": 0.95,
@ -145,28 +82,14 @@
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "Meta",
"tags": [
"General",
"Big Context Length"
]
},
"engine": "groq"
},
{
"sources": [
{
"url": "https://groq.com"
}
],
"id": "llama-3.2-1b-preview",
"model": "llama-3.2-1b-preview",
"object": "model",
"name": "Groq Llama 3.2 1b Preview",
"version": "1.1",
"description": "Groq Llama 3.2 1b Preview with supercharged speed!",
"format": "api",
"settings": {},
"parameters": {
"max_tokens": 8192,
"temperature": 0.7,
@ -176,28 +99,14 @@
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "Meta",
"tags": [
"General",
"Big Context Length"
]
},
"engine": "groq"
},
{
"sources": [
{
"url": "https://groq.com"
}
],
"id": "llama-3.2-3b-preview",
"model": "llama-3.2-3b-preview",
"object": "model",
"name": "Groq Llama 3.2 3b Preview",
"version": "1.1",
"description": "Groq Llama 3.2 3b Preview with supercharged speed!",
"format": "api",
"settings": {},
"parameters": {
"max_tokens": 8192,
"temperature": 0.7,
@ -207,28 +116,14 @@
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "Meta",
"tags": [
"General",
"Big Context Length"
]
},
"engine": "groq"
},
{
"sources": [
{
"url": "https://groq.com"
}
],
"id": "llama-3.2-90b-text-preview",
"model": "llama-3.2-90b-text-preview",
"object": "model",
"name": "Groq Llama 3.2 90b Text Preview",
"version": "1.1",
"description": "Groq Llama 3.2 90b Text Preview with supercharged speed!",
"format": "api",
"settings": {},
"parameters": {
"max_tokens": 8192,
"temperature": 0.7,
@ -238,28 +133,14 @@
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "Meta",
"tags": [
"General",
"Big Context Length"
]
},
"engine": "groq"
},
{
"sources": [
{
"url": "https://groq.com"
}
],
"id": "llama-3.2-90b-vision-preview",
"model": "llama-3.2-90b-vision-preview",
"object": "model",
"name": "Groq Llama 3.2 90b Vision Preview",
"version": "1.1",
"description": "Groq Llama 3.2 90b Vision Preview with supercharged speed!",
"format": "api",
"settings": {},
"parameters": {
"max_tokens": 8192,
"temperature": 0.7,
@ -269,58 +150,14 @@
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "Meta",
"tags": [
"General",
"Big Context Length"
]
},
"engine": "groq"
},
{
"sources": [
{
"url": "https://groq.com"
}
],
"id": "gemma-7b-it",
"object": "model",
"name": "Groq Gemma 7B Instruct",
"version": "1.2",
"description": "Groq Gemma 7B Instruct with supercharged speed!",
"format": "api",
"settings": {},
"parameters": {
"max_tokens": 8192,
"temperature": 0.7,
"top_p": 0.95,
"stream": true,
"stop": [],
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "Google",
"tags": [
"General"
]
},
"engine": "groq"
},
{
"sources": [
{
"url": "https://groq.com"
}
],
"id": "gemma2-9b-it",
"model": "gemma2-9b-it",
"object": "model",
"name": "Groq Gemma 9B Instruct",
"version": "1.2",
"description": "Groq Gemma 9b Instruct with supercharged speed!",
"format": "api",
"settings": {},
"parameters": {
"max_tokens": 8192,
"temperature": 0.7,
@ -330,27 +167,14 @@
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "Google",
"tags": [
"General"
]
},
"engine": "groq"
},
{
"sources": [
{
"url": "https://groq.com"
}
],
"id": "mixtral-8x7b-32768",
"model": "mixtral-8x7b-32768",
"object": "model",
"name": "Groq Mixtral 8x7B Instruct",
"version": "1.2",
"description": "Groq Mixtral 8x7B Instruct is Mixtral with supercharged speed!",
"format": "api",
"settings": {},
"parameters": {
"max_tokens": 32768,
"temperature": 0.7,
@ -360,13 +184,6 @@
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "Mistral",
"tags": [
"General",
"Big Context Length"
]
},
"engine": "groq"
}
]
]

17
extensions/inference-martian-extension/resources/models.json → extensions/engine-management-extension/models/martian.json
View File

@ -1,17 +1,10 @@
[
{
"sources": [
{
"url": "https://withmartian.com/"
}
],
"id": "router",
"model": "router",
"object": "model",
"name": "Martian Model Router",
"version": "1.0",
"description": "Martian Model Router dynamically routes requests to the best LLM in real-time",
"format": "api",
"settings": {},
"parameters": {
"max_tokens": 4096,
"temperature": 0.7,
@ -21,12 +14,6 @@
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "Martian",
"tags": [
"General"
]
},
"engine": "martian"
}
]
]

45
extensions/inference-mistral-extension/resources/models.json → extensions/engine-management-extension/models/mistral.json
View File

@ -1,83 +1,44 @@
[
{
"sources": [
{
"url": "https://docs.mistral.ai/api/"
}
],
"id": "mistral-small-latest",
"model": "mistral-small-latest",
"object": "model",
"name": "Mistral Small",
"version": "1.1",
"description": "Mistral Small is the ideal choice for simple tasks (Classification, Customer Support, or Text Generation) at an affordable price.",
"format": "api",
"settings": {},
"parameters": {
"max_tokens": 32000,
"temperature": 0.7,
"top_p": 0.95,
"stream": true
},
"metadata": {
"author": "Mistral",
"tags": [
"General"
]
},
"engine": "mistral"
},
{
"sources": [
{
"url": "https://docs.mistral.ai/api/"
}
],
"id": "mistral-large-latest",
"model": "mistral-large-latest",
"object": "model",
"name": "Mistral Large",
"version": "1.1",
"description": "Mistral Large is ideal for complex tasks (Synthetic Text Generation, Code Generation, RAG, or Agents).",
"format": "api",
"settings": {},
"parameters": {
"max_tokens": 32000,
"temperature": 0.7,
"top_p": 0.95,
"stream": true
},
"metadata": {
"author": "Mistral",
"tags": [
"General"
]
},
"engine": "mistral"
},
{
"sources": [
{
"url": "https://docs.mistral.ai/api/"
}
],
"id": "open-mixtral-8x22b",
"model": "open-mixtral-8x22b",
"object": "model",
"name": "Mixtral 8x22B",
"version": "1.1",
"description": "Mixtral 8x22B is a high-performance, cost-effective model designed for complex tasks.",
"format": "api",
"settings": {},
"parameters": {
"max_tokens": 32000,
"temperature": 0.7,
"top_p": 0.95,
"stream": true
},
"metadata": {
"author": "Mistral",
"tags": [
"General"
]
},
"engine": "mistral"
}
]

13
extensions/inference-nvidia-extension/resources/models.json → extensions/engine-management-extension/models/nvidia.json
View File

@ -1,17 +1,10 @@
[
{
"sources": [
{
"url": "https://integrate.api.nvidia.com/v1/chat/completions"
}
],
"id": "mistralai/mistral-7b-instruct-v0.2",
"model": "mistralai/mistral-7b-instruct-v0.2",
"object": "model",
"name": "Mistral 7B",
"version": "1.1",
"description": "Mistral 7B with NVIDIA",
"format": "api",
"settings": {},
"parameters": {
"max_tokens": 1024,
"temperature": 0.3,
@ -22,10 +15,6 @@
"stop": null,
"seed": null
},
"metadata": {
"author": "NVIDIA",
"tags": ["General"]
},
"engine": "nvidia"
}
]

102
extensions/inference-openai-extension/resources/models.json → extensions/engine-management-extension/models/openai.json
View File

@ -1,18 +1,12 @@
[
{
"sources": [
{
"url": "https://openai.com"
}
],
"id": "gpt-4-turbo",
"model": "gpt-4-turbo",
"object": "model",
"name": "OpenAI GPT 4 Turbo",
"version": "1.2",
"description": "OpenAI GPT 4 Turbo model is extremely good",
"format": "api",
"settings": {},
"parameters": {
"inference_params": {
"max_tokens": 4096,
"temperature": 0.7,
"top_p": 0.95,
@ -21,26 +15,16 @@
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "OpenAI",
"tags": ["General"]
},
"engine": "openai"
},
{
"sources": [
{
"url": "https://openai.com"
}
],
"id": "gpt-3.5-turbo",
"model": "gpt-3.5-turbo",
"object": "model",
"name": "OpenAI GPT 3.5 Turbo",
"version": "1.1",
"description": "OpenAI GPT 3.5 Turbo model is extremely fast",
"format": "api",
"settings": {},
"parameters": {
"inference_params": {
"max_tokens": 4096,
"temperature": 0.7,
"top_p": 0.95,
@ -49,28 +33,16 @@
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "OpenAI",
"tags": ["General"]
},
"engine": "openai"
},
{
"sources": [
{
"url": "https://openai.com"
}
],
"id": "gpt-4o",
"model": "gpt-4o",
"object": "model",
"name": "OpenAI GPT 4o",
"version": "1.1",
"description": "OpenAI GPT 4o is a new flagship model with fast speed and high quality",
"format": "api",
"settings": {
"vision_model": true
},
"parameters": {
"inference_params": {
"max_tokens": 4096,
"temperature": 0.7,
"top_p": 0.95,
@ -79,28 +51,16 @@
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "OpenAI",
"tags": ["General"]
},
"engine": "openai"
},
{
"sources": [
{
"url": "https://openai.com"
}
],
"id": "gpt-4o-mini",
"model": "gpt-4o-mini",
"object": "model",
"name": "OpenAI GPT 4o-mini",
"version": "1.1",
"description": "GPT-4o mini (“o” for “omni”) is a fast, affordable small model for focused tasks.",
"format": "api",
"settings": {
"vision_model": true
},
"parameters": {
"inference_params": {
"max_tokens": 16384,
"temperature": 0.7,
"top_p": 0.95,
@ -109,26 +69,16 @@
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "OpenAI",
"tags": ["General"]
},
"engine": "openai"
},
{
"sources": [
{
"url": "https://openai.com"
}
],
"id": "o1",
"model": "o1",
"object": "model",
"name": "OpenAI o1",
"version": "1.0",
"description": "OpenAI o1 is a new model with complex reasoning",
"format": "api",
"settings": {},
"parameters": {
"inference_params": {
"max_tokens": 100000,
"temperature": 1,
"top_p": 1,
@ -136,26 +86,16 @@
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "OpenAI",
"tags": ["General"]
},
"engine": "openai"
},
{
"sources": [
{
"url": "https://openai.com"
}
],
"id": "o1-preview",
"model": "o1-preview",
"object": "model",
"name": "OpenAI o1-preview",
"version": "1.0",
"description": "OpenAI o1-preview is a new model with complex reasoning",
"format": "api",
"settings": {},
"parameters": {
"inference_params": {
"max_tokens": 32768,
"temperature": 1,
"top_p": 1,
@ -163,26 +103,16 @@
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "OpenAI",
"tags": ["General"]
},
"engine": "openai"
},
{
"sources": [
{
"url": "https://openai.com"
}
],
"id": "o1-mini",
"model": "o1-mini",
"object": "model",
"name": "OpenAI o1-mini",
"version": "1.0",
"description": "OpenAI o1-mini is a lightweight reasoning model",
"format": "api",
"settings": {},
"parameters": {
"inference_params": {
"max_tokens": 65536,
"temperature": 1,
"top_p": 1,
@ -190,10 +120,6 @@
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "OpenAI",
"tags": ["General"]
},
"engine": "openai"
}
]

13
extensions/inference-openrouter-extension/resources/models.json → extensions/engine-management-extension/models/openrouter.json
View File

@ -1,17 +1,10 @@
[
{
"sources": [
{
"url": "https://openrouter.ai"
}
],
"id": "open-router-auto",
"model": "open-router-auto",
"object": "model",
"name": "OpenRouter",
"version": "1.0",
"description": " OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.",
"format": "api",
"settings": {},
"parameters": {
"max_tokens": 128000,
"temperature": 0.7,
@ -19,10 +12,6 @@
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "OpenRouter",
"tags": ["General", "Big Context Length"]
},
"engine": "openrouter"
}
]

22
extensions/engine-management-extension/resources/anthropic.json Normal file
View File

@ -0,0 +1,22 @@
{
"id": "@janhq/inference-anthropic-extension",
"type": "remote",
"engine": "anthropic",
"url": "https://api.anthropic.com",
"api_key": "",
"metadata": {
"get_models_url": "https://api.anthropic.com/v1/models",
"header_template": "x-api-key: {{api_key}} anthropic-version: 2023-06-01",
"transform_req": {
"chat_completions": {
"url": "https://api.anthropic.com/v1/messages",
"template": "{ {% for key, value in input_request %} {% if key == \"messages\" %} {% if input_request.messages.0.role == \"system\" %} \"system\": \"{{ input_request.messages.0.content }}\", \"messages\": [{% for message in input_request.messages %} {% if not loop.is_first %} {\"role\": \"{{ message.role }}\", \"content\": \"{{ message.content }}\" } {% if not loop.is_last %},{% endif %} {% endif %} {% endfor %}] {% else %} \"messages\": [{% for message in input_request.messages %} {\"role\": \"{{ message.role}}\", \"content\": \"{{ message.content }}\" } {% if not loop.is_last %},{% endif %} {% endfor %}] {% endif %} {% if not loop.is_last %},{% endif %} {% else if key == \"system\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %}\"{{ key }}\": {{ tojson(value) }} {% if not loop.is_last %},{% endif %} {% endif %} {% endfor %} }"
}
},
"transform_resp": {
"chat_completions": {
"template": "{% if input_request.stream %} {\"object\": \"chat.completion.chunk\", \"model\": \"{{ input_request.model }}\", \"choices\": [{\"index\": 0, \"delta\": { {% if input_request.type == \"message_start\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"ping\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"content_block_delta\" %} \"role\": \"assistant\", \"content\": \"{{ input_request.delta.text }}\" {% else if input_request.type == \"content_block_stop\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"content_block_stop\" %} \"role\": \"assistant\", \"content\": null {% endif %} }, {% if input_request.type == \"content_block_stop\" %} \"finish_reason\": \"stop\" {% else %} \"finish_reason\": null {% endif %} }]} {% else %} {\"id\": \"{{ input_request.id }}\", \"created\": null, \"object\": \"chat.completion\", \"model\": \"{{ input_request.model }}\", \"choices\": [{ \"index\": 0, \"message\": { \"role\": \"{{ input_request.role }}\", \"content\": \"{% if input_request.content and input_request.content.0.type == \"text\" %} \"{{input_request.content.0.text}}\" {% endif %}\", \"refusal\": null }, \"logprobs\": null, \"finish_reason\": \"{{ input_request.stop_reason }}\" } ], \"usage\": { \"prompt_tokens\": {{ input_request.usage.input_tokens }}, \"completion_tokens\": {{ input_request.usage.output_tokens }}, \"total_tokens\": {{ input_request.usage.input_tokens + input_request.usage.output_tokens }}, \"prompt_tokens_details\": { \"cached_tokens\": 0 }, \"completion_tokens_details\": { \"reasoning_tokens\": 0, \"accepted_prediction_tokens\": 0, \"rejected_prediction_tokens\": 0 } }, \"system_fingerprint\": \"fp_6b68a8204b\"} {% endif %}"
}
}
}
}

22
extensions/engine-management-extension/resources/cohere.json Normal file
View File

@ -0,0 +1,22 @@
{
"id": "@janhq/inference-cohere-extension",
"type": "remote",
"engine": "cohere",
"url": "https://api.cohere.ai",
"api_key": "",
"metadata": {
"get_models_url": "https://api.cohere.ai/v1/models",
"header_template": "Authorization: Bearer {{api_key}}",
"transform_req": {
"chat_completions": {
"url": "https://api.cohere.ai/v1/chat",
"template": "{ {% for key, value in input_request %} {% if key == \"messages\" %} {% if input_request.messages.0.role == \"system\" %} \"preamble\": \"{{ input_request.messages.0.content }}\", {% if length(input_request.messages) > 2 %} \"chatHistory\": [{% for message in input_request.messages %} {% if not loop.is_first and not loop.is_last %} {\"role\": {% if message.role == \"user\" %} \"USER\" {% else %} \"CHATBOT\" {% endif %}, \"content\": \"{{ message.content }}\" } {% if loop.index < length(input_request.messages) - 2 %},{% endif %} {% endif %} {% endfor %}], {% endif %} \"message\": \"{{ last(input_request.messages).content }}\" {% else %} {% if length(input_request.messages) > 2 %} \"chatHistory\": [{% for message in input_request.messages %} {% if not loop.is_last %} { \"role\": {% if message.role == \"user\" %} \"USER\" {% else %} \"CHATBOT\" {% endif %}, \"content\": \"{{ message.content }}\" } {% if loop.index < length(input_request.messages) - 2 %},{% endif %} {% endif %} {% endfor %}],{% endif %}\"message\": \"{{ last(input_request.messages).content }}\" {% endif %}{% if not loop.is_last %},{% endif %} {% else if key == \"system\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} \"{{ key }}\": {{ tojson(value) }} {% if not loop.is_last %},{% endif %} {% endif %} {% endfor %} }"
}
},
"transform_resp": {
"chat_completions": {
"template": "{% if input_request.stream %} {\"object\": \"chat.completion.chunk\", \"model\": \"{{ input_request.model }}\", \"choices\": [{\"index\": 0, \"delta\": { {% if input_request.event_type == \"text-generation\" %} \"role\": \"assistant\", \"content\": \"{{ input_request.text }}\" {% else %} \"role\": \"assistant\", \"content\": null {% endif %} }, {% if input_request.event_type == \"stream-end\" %} \"finish_reason\": \"{{ input_request.finish_reason }}\" {% else %} \"finish_reason\": null {% endif %} }]} {% else %} {\"id\": \"{{ input_request.generation_id }}\", \"created\": null, \"object\": \"chat.completion\", \"model\": {% if input_request.model %} \"{{ input_request.model }}\" {% else %} \"command-r-plus-08-2024\" {% endif %}, \"choices\": [{ \"index\": 0, \"message\": { \"role\": \"assistant\", \"content\": {% if not input_request.text %} null {% else %} \"{{ input_request.text }}\" {% endif %}, \"refusal\": null }, \"logprobs\": null, \"finish_reason\": \"{{ input_request.finish_reason }}\" } ], \"usage\": { \"prompt_tokens\": {{ input_request.meta.tokens.input_tokens }}, \"completion_tokens\": {{ input_request.meta.tokens.output_tokens }},\"total_tokens\": {{ input_request.meta.tokens.input_tokens + input_request.meta.tokens.output_tokens }}, \"prompt_tokens_details\": { \"cached_tokens\": 0 },\"completion_tokens_details\": { \"reasoning_tokens\": 0, \"accepted_prediction_tokens\": 0, \"rejected_prediction_tokens\": 0 } }, \"system_fingerprint\": \"fp_6b68a8204b\"} {% endif %}"
}
}
}
}

22
extensions/engine-management-extension/resources/groq.json Normal file
View File

@ -0,0 +1,22 @@
{
"id": "@janhq/inference-groq-extension",
"type": "remote",
"engine": "groq",
"url": "https://api.groq.com",
"api_key": "",
"metadata": {
"get_models_url": "https://api.groq.com/openai/v1/models",
"header_template": "Authorization: Bearer {{api_key}}",
"transform_req": {
"chat_completions": {
"url": "https://api.groq.com/openai/v1/chat/completions",
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
}
},
"transform_resp": {
"chat_completions": {
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"choices\" or key == \"created\" or key == \"model\" or key == \"service_tier\" or key == \"system_fingerprint\" or key == \"stream\" or key == \"object\" or key == \"usage\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
}
}
}
}

22
extensions/engine-management-extension/resources/martian.json Normal file
View File

@ -0,0 +1,22 @@
{
"id": "@janhq/inference-martian-extension",
"type": "remote",
"engine": "martian",
"url": "https://withmartian.com",
"api_key": "",
"metadata": {
"get_models_url": "https://withmartian.com/api/openai/v1/models",
"header_template": "Authorization: Bearer {{api_key}}",
"transform_req": {
"chat_completions": {
"url": "https://withmartian.com/api/openai/v1/chat/completions",
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
}
},
"transform_resp": {
"chat_completions": {
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"choices\" or key == \"created\" or key == \"model\" or key == \"service_tier\" or key == \"system_fingerprint\" or key == \"stream\" or key == \"object\" or key == \"usage\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
}
}
}
}

22
extensions/engine-management-extension/resources/mistral.json Normal file
View File

@ -0,0 +1,22 @@
{
"id": "@janhq/inference-mistral-extension",
"type": "remote",
"engine": "mistral",
"url": "https://api.mistral.ai",
"api_key": "",
"metadata": {
"get_models_url": "https://api.mistral.ai/v1/models",
"header_template": "Authorization: Bearer {{api_key}}",
"transform_req": {
"chat_completions": {
"url": "https://api.mistral.ai/v1/chat/completions",
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
}
},
"transform_resp": {
"chat_completions": {
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"choices\" or key == \"created\" or key == \"model\" or key == \"service_tier\" or key == \"system_fingerprint\" or key == \"stream\" or key == \"object\" or key == \"usage\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
}
}
}
}

22
extensions/engine-management-extension/resources/nvidia.json Normal file
View File

@ -0,0 +1,22 @@
{
"id": "@janhq/inference-nvidia-extension",
"type": "remote",
"engine": "nvidia",
"url": "https://integrate.api.nvidia.com",
"api_key": "",
"metadata": {
"get_models_url": "https://integrate.api.nvidia.com/v1/models",
"header_template": "Authorization: Bearer {{api_key}}",
"transform_req": {
"chat_completions": {
"url": "https://integrate.api.nvidia.com/v1/chat/completions",
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
}
},
"transform_resp": {
"chat_completions": {
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"choices\" or key == \"created\" or key == \"model\" or key == \"service_tier\" or key == \"system_fingerprint\" or key == \"stream\" or key == \"object\" or key == \"usage\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
}
}
}
}

22
extensions/engine-management-extension/resources/openai.json Normal file
View File

@ -0,0 +1,22 @@
{
"id": "@janhq/inference-openai-extension",
"type": "remote",
"engine": "openai",
"url": "https://api.openai.com",
"api_key": "",
"metadata": {
"get_models_url": "https://api.openai.com/v1/models",
"header_template": "Authorization: Bearer {{api_key}}",
"transform_req": {
"chat_completions": {
"url": "https://api.openai.com/v1/chat/completions",
"template": "{ {% set first = true %}{% for key, value in input_request %}{% if key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"messages\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" or (not \"o1\" in input_request.model and (key == \"max_tokens\" or key == \"stop\")) %} {% if key == \"max_tokens\" and \"o1\" in input_request.model %} \"max_completion_tokens\": {{ tojson(value) }} {% else %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endif %} {% endfor %} }"
}
},
"transform_resp": {
"chat_completions": {
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"choices\" or key == \"created\" or key == \"model\" or key == \"service_tier\" or key == \"stream\" or key == \"object\" or key == \"usage\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
}
}
}
}

22
extensions/engine-management-extension/resources/openrouter.json Normal file
View File

@ -0,0 +1,22 @@
{
"id": "@janhq/inference-openrouter-extension",
"type": "remote",
"engine": "openrouter",
"url": "https://openrouter.ai",
"api_key": "",
"metadata": {
"get_models_url": "https://openrouter.ai/api/v1/models",
"header_template": "Authorization: Bearer {{api_key}}",
"transform_req": {
"chat_completions": {
"url": "https://openrouter.ai/api/v1/chat/completions",
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
}
},
"transform_resp": {
"chat_completions": {
"template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"choices\" or key == \"created\" or key == \"model\" or key == \"service_tier\" or key == \"system_fingerprint\" or key == \"stream\" or key == \"object\" or key == \"usage\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }"
}
}
}
}

3
extensions/engine-management-extension/rolldown.config.mjs
View File

@ -1,4 +1,5 @@
import { defineConfig } from 'rolldown'
import { engines, models } from './engines.mjs'
import pkgJson from './package.json' with { type: 'json' }
export default defineConfig([
@ -13,6 +14,8 @@ export default defineConfig([
API_URL: JSON.stringify('http://127.0.0.1:39291'),
SOCKET_URL: JSON.stringify('ws://127.0.0.1:39291'),
CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.43'),
DEFAULT_REMOTE_ENGINES: JSON.stringify(engines),
DEFAULT_REMOTE_MODELS: JSON.stringify(models),
},
},
{

6
extensions/engine-management-extension/src/@types/global.d.ts vendored
View File

@ -3,6 +3,12 @@ declare const CORTEX_ENGINE_VERSION: string
declare const SOCKET_URL: string
declare const NODE: string
declare const DEFAULT_REMOTE_ENGINES: ({
id: string
engine: string
} & EngineConfig)[]
declare const DEFAULT_REMOTE_MODELS: Model[]
interface Core {
api: APIFunctions
events: EventEmitter

185
extensions/engine-management-extension/src/index.ts
View File

@ -3,14 +3,22 @@ import {
InferenceEngine,
DefaultEngineVariant,
Engines,
EngineConfig,
EngineVariant,
EngineReleased,
executeOnMain,
systemInformation,
Model,
fs,
joinPath,
events,
ModelEvent,
EngineEvent,
} from '@janhq/core'
import ky, { HTTPError } from 'ky'
import PQueue from 'p-queue'
import { EngineError } from './error'
import { getJanDataFolderPath } from '@janhq/core'
/**
* JSONEngineManagementExtension is a EngineManagementExtension implementation that provides
@ -27,41 +35,11 @@ export default class JSONEngineManagementExtension extends EngineManagementExten
await executeOnMain(NODE, 'symlinkEngines')
// Run Healthcheck
this.queue.add(() => this.healthz())
try {
const variant = await this.getDefaultEngineVariant(
InferenceEngine.cortex_llamacpp
)
const installedEngines = await this.getInstalledEngines(
InferenceEngine.cortex_llamacpp
)
if (
!installedEngines.some(
(e) => e.name === variant.variant && e.version === variant.version
)
) {
throw new EngineError(
'Default engine is not available, use bundled version.'
)
}
} catch (error) {
if (
(error instanceof HTTPError && error.response.status === 400) ||
error instanceof EngineError
) {
const systemInfo = await systemInformation()
const variant = await executeOnMain(
NODE,
'engineVariant',
systemInfo.gpuSetting
)
await this.setDefaultEngineVariant(InferenceEngine.cortex_llamacpp, {
variant: variant,
version: `${CORTEX_ENGINE_VERSION}`,
})
} else {
console.error('An unexpected error occurred:', error)
}
}
// Update default local engine
this.updateDefaultEngine()
// Populate default remote engines
this.populateDefaultRemoteEngines()
}
/**
@ -81,6 +59,19 @@ export default class JSONEngineManagementExtension extends EngineManagementExten
) as Promise<Engines>
}
/**
* @returns A Promise that resolves to an object of list engines.
*/
async getRemoteModels(name: string): Promise<any> {
return this.queue.add(() =>
ky
.get(`${API_URL}/v1/models/remote/${name}`)
.json<Model[]>()
.then((e) => e)
.catch(() => [])
) as Promise<Model[]>
}
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to an array of installed engine.
@ -135,10 +126,7 @@ export default class JSONEngineManagementExtension extends EngineManagementExten
* @param name - Inference engine name.
* @returns A Promise that resolves to intall of engine.
*/
async installEngine(
name: InferenceEngine,
engineConfig: { variant: string; version?: string }
) {
async installEngine(name: string, engineConfig: EngineConfig) {
return this.queue.add(() =>
ky
.post(`${API_URL}/v1/engines/${name}/install`, { json: engineConfig })
@ -146,14 +134,21 @@ export default class JSONEngineManagementExtension extends EngineManagementExten
) as Promise<{ messages: string }>
}
/**
* Add a new remote engine
* @returns A Promise that resolves to intall of engine.
*/
async addRemoteEngine(engineConfig: EngineConfig) {
return this.queue.add(() =>
ky.post(`${API_URL}/v1/engines`, { json: engineConfig }).then((e) => e)
) as Promise<{ messages: string }>
}
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to unintall of engine.
*/
async uninstallEngine(
name: InferenceEngine,
engineConfig: { variant: string; version: string }
) {
async uninstallEngine(name: InferenceEngine, engineConfig: EngineConfig) {
return this.queue.add(() =>
ky
.delete(`${API_URL}/v1/engines/${name}/install`, { json: engineConfig })
@ -161,6 +156,16 @@ export default class JSONEngineManagementExtension extends EngineManagementExten
) as Promise<{ messages: string }>
}
/**
* Add a new remote model
* @param model - Remote model object.
*/
async addRemoteModel(model: Model) {
return this.queue.add(() =>
ky.post(`${API_URL}/v1/models/add`, { json: model }).then((e) => e)
)
}
/**
* @param name - Inference engine name.
* @returns A Promise that resolves to an object of default engine.
@ -181,7 +186,7 @@ export default class JSONEngineManagementExtension extends EngineManagementExten
*/
async setDefaultEngineVariant(
name: InferenceEngine,
engineConfig: { variant: string; version: string }
engineConfig: EngineConfig
) {
return this.queue.add(() =>
ky
@ -193,9 +198,11 @@ export default class JSONEngineManagementExtension extends EngineManagementExten
/**
* @returns A Promise that resolves to update engine.
*/
async updateEngine(name: InferenceEngine) {
async updateEngine(name: InferenceEngine, engineConfig?: EngineConfig) {
return this.queue.add(() =>
ky.post(`${API_URL}/v1/engines/${name}/update`).then((e) => e)
ky
.post(`${API_URL}/v1/engines/${name}/update`, { json: engineConfig })
.then((e) => e)
) as Promise<{ messages: string }>
}
@ -210,4 +217,90 @@ export default class JSONEngineManagementExtension extends EngineManagementExten
})
.then(() => {})
}
/**
* Update default local engine
* This is to use built-in engine variant in case there is no default engine set
*/
async updateDefaultEngine() {
try {
const variant = await this.getDefaultEngineVariant(
InferenceEngine.cortex_llamacpp
)
const installedEngines = await this.getInstalledEngines(
InferenceEngine.cortex_llamacpp
)
if (
!installedEngines.some(
(e) => e.name === variant.variant && e.version === variant.version
)
) {
throw new EngineError(
'Default engine is not available, use bundled version.'
)
}
} catch (error) {
if (
(error instanceof HTTPError && error.response.status === 400) ||
error instanceof EngineError
) {
const systemInfo = await systemInformation()
const variant = await executeOnMain(
NODE,
'engineVariant',
systemInfo.gpuSetting
)
await this.setDefaultEngineVariant(InferenceEngine.cortex_llamacpp, {
variant: variant,
version: `${CORTEX_ENGINE_VERSION}`,
})
} else {
console.error('An unexpected error occurred:', error)
}
}
}
/**
* This is to populate default remote engines in case there is no customized remote engine setting
*/
async populateDefaultRemoteEngines() {
const engines = await this.getEngines()
if (
!Object.values(engines)
.flat()
.some((e) => e.type === 'remote')
) {
await Promise.all(
DEFAULT_REMOTE_ENGINES.map(async (engine) => {
const { id, ...data } = engine
/// BEGIN - Migrate legacy api key settings
let api_key = undefined
if (id) {
const apiKeyPath = await joinPath([
await getJanDataFolderPath(),
'settings',
id,
'settings.json',
])
if (await fs.existsSync(apiKeyPath)) {
const settings = await fs.readFileSync(apiKeyPath, 'utf-8')
api_key = JSON.parse(settings).find(
(e) => e.key === `${data.engine}-api-key`
)?.controllerProps?.value
}
}
data.api_key = api_key
/// END - Migrate legacy api key settings
await this.addRemoteEngine(data).catch(console.error)
})
)
events.emit(EngineEvent.OnEngineUpdate, {})
DEFAULT_REMOTE_MODELS.forEach(async (data: Model) => {
await this.addRemoteModel(data).catch(() => {})
})
events.emit(ModelEvent.OnModelsUpdate, { fetch: true })
}
}
}

3
extensions/engine-management-extension/tsconfig.json
View File

@ -8,7 +8,8 @@
"forceConsistentCasingInFileNames": true,
"strict": false,
"skipLibCheck": true,
"rootDir": "./src"
"rootDir": "./src",
"resolveJsonModule": true
},
"include": ["./src"],
"exclude": ["src/**/*.test.ts", "rolldown.config.mjs"]

79
extensions/inference-anthropic-extension/README.md
View File

@ -1,79 +0,0 @@
# Anthropic Engine Extension
Created using Jan extension example
# Create a Jan Extension using Typescript
Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀
## Create Your Own Extension
To create your own extension, you can use this repository as a template! Just follow the below instructions:
1. Click the Use this template button at the top of the repository
2. Select Create a new repository
3. Select an owner and name for your new repository
4. Click Create repository
5. Clone your new repository
## Initial Setup
After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension.
> [!NOTE]
>
> You'll need to have a reasonably modern version of
> [Node.js](https://nodejs.org) handy. If you are using a version manager like
> [`nodenv`](https://github.com/nodenv/nodenv) or
> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the
> root of your repository to install the version specified in
> [`package.json`](./package.json). Otherwise, 20.x or later should work!
1. :hammer_and_wrench: Install the dependencies
```bash
npm install
```
1. :building_construction: Package the TypeScript for distribution
```bash
npm run bundle
```
1. :white_check_mark: Check your artifact
There will be a tgz file in your extension directory now
## Update the Extension Metadata
The [`package.json`](package.json) file defines metadata about your extension, such as
extension name, main entry, description and version.
When you copy this repository, update `package.json` with the name, description for your extension.
## Update the Extension Code
The [`src/`](./src/) directory is the heart of your extension! This contains the
source code that will be run when your extension functions are invoked. You can replace the
contents of this directory with your own code.
There are a few things to keep in mind when writing your extension code:
- Most Jan Extension functions are processed asynchronously.
In `index.ts`, you will see that the extension function will return a `Promise<any>`.
```typescript
import { events, MessageEvent, MessageRequest } from '@janhq/core'
function onStart(): Promise<any> {
return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) =>
this.inference(data)
)
}
```
For more information about the Jan Extension Core module, see the
[documentation](https://github.com/janhq/jan/blob/main/core/README.md).
So, what are you waiting for? Go ahead and start customizing your extension!

9
extensions/inference-anthropic-extension/jest.config.js
View File

@ -1,9 +0,0 @@
/** @type {import('ts-jest').JestConfigWithTsJest} */
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
transform: {
'node_modules/@janhq/core/.+\\.(j|t)s?$': 'ts-jest',
},
transformIgnorePatterns: ['node_modules/(?!@janhq/core/.*)'],
}

42
extensions/inference-anthropic-extension/package.json
View File

@ -1,42 +0,0 @@
{
"name": "@janhq/inference-anthropic-extension",
"productName": "Anthropic Inference Engine",
"version": "1.0.3",
"description": "This extension enables Anthropic chat completion API calls",
"main": "dist/index.js",
"engine": "anthropic",
"author": "Jan <service@jan.ai>",
"license": "AGPL-3.0",
"scripts": {
"test": "jest test",
"build": "rolldown -c rolldown.config.mjs",
"build:publish": "rimraf *.tgz --glob || true && yarn build && npm pack && cpx *.tgz ../../pre-install"
},
"devDependencies": {
"cpx": "^1.5.0",
"rimraf": "^3.0.2",
"rolldown": "1.0.0-beta.1",
"ts-loader": "^9.5.0",
"typescript": "^5.7.2"
},
"dependencies": {
"@janhq/core": "../../core/package.tgz",
"fetch-retry": "^5.0.6",
"ulidx": "^2.3.0"
},
"engines": {
"node": ">=18.0.0"
},
"files": [
"dist/*",
"package.json",
"README.md"
],
"bundleDependencies": [
"fetch-retry"
],
"installConfig": {
"hoistingLimits": "workspaces"
},
"packageManager": "yarn@4.5.3"
}

24
extensions/inference-anthropic-extension/resources/settings.json
View File

@ -1,24 +0,0 @@
[
{
"key": "anthropic-api-key",
"title": "API Key",
"description": "The Anthropic API uses API keys for authentication. Visit your [API Keys](https://console.anthropic.com/settings/keys) page to retrieve the API key you'll use in your requests.",
"controllerType": "input",
"controllerProps": {
"placeholder": "Insert API Key",
"value": "",
"type": "password",
"inputActions": ["unobscure", "copy"]
}
},
{
"key": "chat-completions-endpoint",
"title": "Chat Completions Endpoint",
"description": "The endpoint to use for chat completions. See the [Anthropic API documentation](https://docs.anthropic.com/claude/docs/intro-to-claude) for more information.",
"controllerType": "input",
"controllerProps": {
"placeholder": "https://api.anthropic.com/v1/messages",
"value": "https://api.anthropic.com/v1/messages"
}
}
]

18
extensions/inference-anthropic-extension/rolldown.config.mjs
View File

@ -1,18 +0,0 @@
import { defineConfig } from 'rolldown'
import pkgJson from './package.json' with { type: 'json' }
import settingJson from './resources/settings.json' with { type: 'json' }
import modelsJson from './resources/models.json' with { type: 'json' }
export default defineConfig({
input: 'src/index.ts',
output: {
format: 'esm',
file: 'dist/index.js',
},
platform: 'browser',
define: {
MODELS: JSON.stringify(modelsJson),
SETTINGS: JSON.stringify(settingJson),
ENGINE: JSON.stringify(pkgJson.engine),
},
})

77
extensions/inference-anthropic-extension/src/anthropic.test.ts
View File

@ -1,77 +0,0 @@
// Import necessary modules
import JanInferenceAnthropicExtension, { Settings } from '.'
import { PayloadType, ChatCompletionRole } from '@janhq/core'
// Mocks
jest.mock('@janhq/core', () => ({
RemoteOAIEngine: jest.fn().mockImplementation(() => ({
registerSettings: jest.fn(),
registerModels: jest.fn(),
getSetting: jest.fn(),
onChange: jest.fn(),
onSettingUpdate: jest.fn(),
onLoad: jest.fn(),
headers: jest.fn(),
})),
PayloadType: jest.fn(),
ChatCompletionRole: {
User: 'user' as const,
Assistant: 'assistant' as const,
System: 'system' as const,
},
}))
// Helper functions
const createMockPayload = (): PayloadType => ({
messages: [
{ role: ChatCompletionRole.System, content: 'Meow' },
{ role: ChatCompletionRole.User, content: 'Hello' },
{ role: ChatCompletionRole.Assistant, content: 'Hi there' },
],
model: 'claude-v1',
stream: false,
})
describe('JanInferenceAnthropicExtension', () => {
let extension: JanInferenceAnthropicExtension
beforeEach(() => {
extension = new JanInferenceAnthropicExtension('', '')
extension.apiKey = 'mock-api-key'
extension.inferenceUrl = 'mock-endpoint'
jest.clearAllMocks()
})
it('should initialize with correct settings', async () => {
await extension.onLoad()
expect(extension.apiKey).toBe('mock-api-key')
expect(extension.inferenceUrl).toBe('mock-endpoint')
})
it('should transform payload correctly', () => {
const payload = createMockPayload()
const transformedPayload = extension.transformPayload(payload)
expect(transformedPayload).toEqual({
max_tokens: 4096,
model: 'claude-v1',
stream: false,
system: 'Meow',
messages: [
{ role: 'user', content: 'Hello' },
{ role: 'assistant', content: 'Hi there' },
],
})
})
it('should transform response correctly', () => {
const nonStreamResponse = { content: [{ text: 'Test response' }] }
const streamResponse =
'data: {"type":"content_block_delta","delta":{"text":"Hello"}}'
expect(extension.transformResponse(nonStreamResponse)).toBe('Test response')
expect(extension.transformResponse(streamResponse)).toBe('Hello')
expect(extension.transformResponse('')).toBe('')
expect(extension.transformResponse('event: something')).toBe('')
})
})

2
extensions/inference-anthropic-extension/src/env.d.ts vendored
View File

@ -1,2 +0,0 @@
declare const SETTINGS: SettingComponentProps[]
declare const MODELS: Model[]

150
extensions/inference-anthropic-extension/src/index.ts
View File

@ -1,150 +0,0 @@
/**
* @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package.
* The class provides methods for initializing and stopping a model, and for making inference requests.
* It also subscribes to events emitted by the @janhq/core package and handles new message requests.
* @version 1.0.0
* @module inference-anthropic-extension/src/index
*/
import { RemoteOAIEngine } from '@janhq/core'
import { PayloadType } from '@janhq/core'
import { ChatCompletionRole } from '@janhq/core'
export enum Settings {
apiKey = 'anthropic-api-key',
chatCompletionsEndPoint = 'chat-completions-endpoint',
}
type AnthropicPayloadType = {
stream: boolean
model?: string
max_tokens?: number
messages?: Array<{ role: string; content: string }>
system?: string
}
/**
* A class that implements the InferenceExtension interface from the @janhq/core package.
* The class provides methods for initializing and stopping a model, and for making inference requests.
* It also subscribes to events emitted by the @janhq/core package and handles new message requests.
*/
export default class JanInferenceAnthropicExtension extends RemoteOAIEngine {
inferenceUrl: string = ''
provider: string = 'anthropic'
maxTokens: number = 4096
override async onLoad(): Promise<void> {
super.onLoad()
// Register Settings
this.registerSettings(SETTINGS)
this.registerModels(MODELS)
this.apiKey = await this.getSetting<string>(Settings.apiKey, '')
this.inferenceUrl = await this.getSetting<string>(
Settings.chatCompletionsEndPoint,
''
)
if (this.inferenceUrl.length === 0) {
SETTINGS.forEach((setting) => {
if (setting.key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = setting.controllerProps.value as string
}
})
}
}
// Override the headers method to include the x-API-key in the request headers
override async headers(): Promise<HeadersInit> {
return {
'Content-Type': 'application/json',
'x-api-key': this.apiKey,
'anthropic-version': '2023-06-01',
}
}
onSettingUpdate<T>(key: string, value: T): void {
if (key === Settings.apiKey) {
this.apiKey = value as string
} else if (key === Settings.chatCompletionsEndPoint) {
if (typeof value !== 'string') return
if (value.trim().length === 0) {
SETTINGS.forEach((setting) => {
if (setting.key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = setting.controllerProps.value as string
}
})
} else {
this.inferenceUrl = value
}
}
}
// Override the transformPayload method to convert the payload to the required format
transformPayload = (payload: PayloadType): AnthropicPayloadType => {
if (!payload.messages || payload.messages.length === 0) {
return {
max_tokens: this.maxTokens,
messages: [],
model: payload.model,
stream: payload.stream,
}
}
const convertedData: AnthropicPayloadType = {
max_tokens: this.maxTokens,
messages: [],
model: payload.model,
stream: payload.stream,
}
payload.messages.forEach((item) => {
if (item.role === ChatCompletionRole.User) {
convertedData.messages.push({
role: 'user',
content: item.content as string,
})
} else if (item.role === ChatCompletionRole.Assistant) {
convertedData.messages.push({
role: 'assistant',
content: item.content as string,
})
} else if (item.role === ChatCompletionRole.System) {
// When using Claude, you can dramatically improve its performance by using the system parameter to give it a role.
// This technique, known as role prompting, is the most powerful way to use system prompts with Claude.
convertedData.system = item.content as string
}
})
return convertedData
}
// Sample returned stream data from anthropic
// {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} }
// {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello"} }
// {"type":"content_block_stop","index":0 }
// {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":12} }
// Override the transformResponse method to convert the response to the required format
transformResponse = (data: any): string => {
// handling stream response
if (typeof data === 'string' && data.trim().length === 0) return ''
if (typeof data === 'string' && data.startsWith('event: ')) return ''
if (typeof data === 'string' && data.startsWith('data: ')) {
data = data.replace('data: ', '')
const parsedData = JSON.parse(data)
if (parsedData.type !== 'content_block_delta') return ''
return parsedData.delta?.text ?? ''
}
// non stream response
if (data.content && data.content.length > 0 && data.content[0].text) {
return data.content[0].text
}
console.error('Invalid response format:', data)
return ''
}
}

15
extensions/inference-anthropic-extension/tsconfig.json
View File

@ -1,15 +0,0 @@
{
"compilerOptions": {
"target": "es2016",
"module": "ES6",
"moduleResolution": "node",
"outDir": "./dist",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"strict": false,
"skipLibCheck": true,
"rootDir": "./src"
},
"include": ["./src"],
"exclude": ["**/*.test.ts"]
}

79
extensions/inference-cohere-extension/README.md
View File

@ -1,79 +0,0 @@
# Cohere Engine Extension
Created using Jan extension example
# Create a Jan Extension using Typescript
Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀
## Create Your Own Extension
To create your own extension, you can use this repository as a template! Just follow the below instructions:
1. Click the Use this template button at the top of the repository
2. Select Create a new repository
3. Select an owner and name for your new repository
4. Click Create repository
5. Clone your new repository
## Initial Setup
After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension.
> [!NOTE]
>
> You'll need to have a reasonably modern version of
> [Node.js](https://nodejs.org) handy. If you are using a version manager like
> [`nodenv`](https://github.com/nodenv/nodenv) or
> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the
> root of your repository to install the version specified in
> [`package.json`](./package.json). Otherwise, 20.x or later should work!
1. :hammer_and_wrench: Install the dependencies
```bash
npm install
```
1. :building_construction: Package the TypeScript for distribution
```bash
npm run bundle
```
1. :white_check_mark: Check your artifact
There will be a tgz file in your extension directory now
## Update the Extension Metadata
The [`package.json`](package.json) file defines metadata about your extension, such as
extension name, main entry, description and version.
When you copy this repository, update `package.json` with the name, description for your extension.
## Update the Extension Code
The [`src/`](./src/) directory is the heart of your extension! This contains the
source code that will be run when your extension functions are invoked. You can replace the
contents of this directory with your own code.
There are a few things to keep in mind when writing your extension code:
- Most Jan Extension functions are processed asynchronously.
In `index.ts`, you will see that the extension function will return a `Promise<any>`.
```typescript
import { events, MessageEvent, MessageRequest } from '@janhq/core'
function onStart(): Promise<any> {
return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) =>
this.inference(data)
)
}
```
For more information about the Jan Extension Core module, see the
[documentation](https://github.com/janhq/jan/blob/main/core/README.md).
So, what are you waiting for? Go ahead and start customizing your extension!

46
extensions/inference-cohere-extension/package.json
View File

@ -1,46 +0,0 @@
{
"name": "@janhq/inference-cohere-extension",
"productName": "Cohere Inference Engine",
"version": "1.0.0",
"description": "This extension enables Cohere chat completion API calls",
"main": "dist/index.js",
"module": "dist/module.js",
"engine": "cohere",
"author": "Jan <service@jan.ai>",
"license": "AGPL-3.0",
"scripts": {
"build": "rolldown -c rolldown.config.mjs",
"build:publish": "rimraf *.tgz --glob || true && yarn build && npm pack && cpx *.tgz ../../pre-install"
},
"exports": {
".": "./dist/index.js",
"./main": "./dist/module.js"
},
"devDependencies": {
"cpx": "^1.5.0",
"rimraf": "^3.0.2",
"rolldown": "1.0.0-beta.1",
"ts-loader": "^9.5.0",
"typescript": "^5.7.2"
},
"dependencies": {
"@janhq/core": "../../core/package.tgz",
"fetch-retry": "^5.0.6",
"ulidx": "^2.3.0"
},
"engines": {
"node": ">=18.0.0"
},
"files": [
"dist/*",
"package.json",
"README.md"
],
"bundleDependencies": [
"fetch-retry"
],
"installConfig": {
"hoistingLimits": "workspaces"
},
"packageManager": "yarn@4.5.3"
}

24
extensions/inference-cohere-extension/resources/settings.json
View File

@ -1,24 +0,0 @@
[
{
"key": "cohere-api-key",
"title": "API Key",
"description": "The Cohere API uses API keys for authentication. Visit your [API Keys](https://dashboard.cohere.com/api-keys) page to retrieve the API key you'll use in your requests.",
"controllerType": "input",
"controllerProps": {
"placeholder": "Insert API Key",
"value": "",
"type": "password",
"inputActions": ["unobscure", "copy"]
}
},
{
"key": "chat-completions-endpoint",
"title": "Chat Completions Endpoint",
"description": "The endpoint to use for chat completions. See the [Cohere API documentation](https://docs.cohere.com/reference/chat) for more information.",
"controllerType": "input",
"controllerProps": {
"placeholder": "https://api.cohere.ai/v1/chat",
"value": "https://api.cohere.ai/v1/chat"
}
}
]

18
extensions/inference-cohere-extension/rolldown.config.mjs
View File

@ -1,18 +0,0 @@
import { defineConfig } from 'rolldown'
import pkgJson from './package.json' with { type: 'json' }
import settingJson from './resources/settings.json' with { type: 'json' }
import modelsJson from './resources/models.json' with { type: 'json' }
export default defineConfig({
input: 'src/index.ts',
output: {
format: 'esm',
file: 'dist/index.js',
},
platform: 'browser',
define: {
MODELS: JSON.stringify(modelsJson),
SETTINGS: JSON.stringify(settingJson),
ENGINE: JSON.stringify(pkgJson.engine),
},
})

2
extensions/inference-cohere-extension/src/env.d.ts vendored
View File

@ -1,2 +0,0 @@
declare const SETTINGS: SettingComponentProps[]
declare const MODELS: Model[]

117
extensions/inference-cohere-extension/src/index.ts
View File

@ -1,117 +0,0 @@
/**
* @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package.
* The class provides methods for initializing and stopping a model, and for making inference requests.
* It also subscribes to events emitted by the @janhq/core package and handles new message requests.
* @version 1.0.0
* @module inference-cohere-extension/src/index
*/
import { RemoteOAIEngine } from '@janhq/core'
import { PayloadType } from '@janhq/core'
import { ChatCompletionRole } from '@janhq/core'
enum Settings {
apiKey = 'cohere-api-key',
chatCompletionsEndPoint = 'chat-completions-endpoint',
}
enum RoleType {
user = 'USER',
chatbot = 'CHATBOT',
system = 'SYSTEM',
}
type CoherePayloadType = {
chat_history?: Array<{ role: RoleType; message: string }>
message?: string
preamble?: string
}
/**
* A class that implements the InferenceExtension interface from the @janhq/core package.
* The class provides methods for initializing and stopping a model, and for making inference requests.
* It also subscribes to events emitted by the @janhq/core package and handles new message requests.
*/
export default class JanInferenceCohereExtension extends RemoteOAIEngine {
inferenceUrl: string = ''
provider: string = 'cohere'
override async onLoad(): Promise<void> {
super.onLoad()
// Register Settings
this.registerSettings(SETTINGS)
this.registerModels(MODELS)
this.apiKey = await this.getSetting<string>(Settings.apiKey, '')
this.inferenceUrl = await this.getSetting<string>(
Settings.chatCompletionsEndPoint,
''
)
if (this.inferenceUrl.length === 0) {
SETTINGS.forEach((setting) => {
if (setting.key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = setting.controllerProps.value as string
}
})
}
}
onSettingUpdate<T>(key: string, value: T): void {
if (key === Settings.apiKey) {
this.apiKey = value as string
} else if (key === Settings.chatCompletionsEndPoint) {
if (typeof value !== 'string') return
if (value.trim().length === 0) {
SETTINGS.forEach((setting) => {
if (setting.key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = setting.controllerProps.value as string
}
})
} else {
this.inferenceUrl = value
}
}
}
transformPayload = (payload: PayloadType): CoherePayloadType => {
if (payload.messages.length === 0) {
return {}
}
const { messages, ...params } = payload
const convertedData: CoherePayloadType = {
...params,
chat_history: [],
message: '',
}
messages.forEach((item, index) => {
// Assign the message of the last item to the `message` property
if (index === messages.length - 1) {
convertedData.message = item.content as string
return
}
if (item.role === ChatCompletionRole.User) {
convertedData.chat_history.push({
role: RoleType.user,
message: item.content as string,
})
} else if (item.role === ChatCompletionRole.Assistant) {
convertedData.chat_history.push({
role: RoleType.chatbot,
message: item.content as string,
})
} else if (item.role === ChatCompletionRole.System) {
convertedData.preamble = item.content as string
}
})
return convertedData
}
transformResponse = (data: any) => {
return typeof data === 'object'
? data.text
: (JSON.parse(data.replace('data: ', '').trim()).text ?? '')
}
}

14
extensions/inference-cohere-extension/tsconfig.json
View File

@ -1,14 +0,0 @@
{
"compilerOptions": {
"target": "es2016",
"module": "ES6",
"moduleResolution": "node",
"outDir": "./dist",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"strict": false,
"skipLibCheck": true,
"rootDir": "./src"
},
"include": ["./src"]
}

2
extensions/inference-cortex-extension/bin/version.txt
View File

@ -1 +1 @@
1.0.8
1.0.9-rc1

4
extensions/inference-cortex-extension/src/@types/global.d.ts vendored
View File

@ -2,5 +2,5 @@ declare const NODE: string
declare const CORTEX_API_URL: string
declare const CORTEX_SOCKET_URL: string
declare const CORTEX_ENGINE_VERSION: string
declare const SETTINGS: object[]
declare const MODELS: object[]
declare const SETTINGS: any
declare const MODELS: any

2
extensions/inference-cortex-extension/src/index.ts
View File

@ -111,8 +111,6 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
)
if (!Number.isNaN(threads_number)) this.cpu_threads = threads_number
this.queue.add(() => this.clean())
// Run the process watchdog
const systemInfo = await systemInformation()
this.queue.add(() => executeOnMain(NODE, 'run', systemInfo))

75
extensions/inference-groq-extension/README.md
View File

@ -1,75 +0,0 @@
# Create a Jan Extension using Typescript
Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀
## Create Your Own Extension
To create your own extension, you can use this repository as a template! Just follow the below instructions:
1. Click the Use this template button at the top of the repository
2. Select Create a new repository
3. Select an owner and name for your new repository
4. Click Create repository
5. Clone your new repository
## Initial Setup
After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension.
> [!NOTE]
>
> You'll need to have a reasonably modern version of
> [Node.js](https://nodejs.org) handy. If you are using a version manager like
> [`nodenv`](https://github.com/nodenv/nodenv) or
> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the
> root of your repository to install the version specified in
> [`package.json`](./package.json). Otherwise, 20.x or later should work!
1. :hammer_and_wrench: Install the dependencies
```bash
npm install
```
1. :building_construction: Package the TypeScript for distribution
```bash
npm run bundle
```
1. :white_check_mark: Check your artifact
There will be a tgz file in your extension directory now
## Update the Extension Metadata
The [`package.json`](package.json) file defines metadata about your extension, such as
extension name, main entry, description and version.
When you copy this repository, update `package.json` with the name, description for your extension.
## Update the Extension Code
The [`src/`](./src/) directory is the heart of your extension! This contains the
source code that will be run when your extension functions are invoked. You can replace the
contents of this directory with your own code.
There are a few things to keep in mind when writing your extension code:
- Most Jan Extension functions are processed asynchronously.
In `index.ts`, you will see that the extension function will return a `Promise<any>`.
```typescript
import { events, MessageEvent, MessageRequest } from '@janhq/core'
function onStart(): Promise<any> {
return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) =>
this.inference(data)
)
}
```
For more information about the Jan Extension Core module, see the
[documentation](https://github.com/janhq/jan/blob/main/core/README.md).
So, what are you waiting for? Go ahead and start customizing your extension!

42
extensions/inference-groq-extension/package.json
View File

@ -1,42 +0,0 @@
{
"name": "@janhq/inference-groq-extension",
"productName": "Groq Inference Engine",
"version": "1.0.1",
"description": "This extension enables fast Groq chat completion API calls",
"main": "dist/index.js",
"engine": "groq",
"module": "dist/module.js",
"author": "Carsen Klock & Jan",
"license": "AGPL-3.0",
"scripts": {
"build": "rolldown -c rolldown.config.mjs",
"build:publish": "rimraf *.tgz --glob || true && yarn build && npm pack && cpx *.tgz ../../pre-install"
},
"devDependencies": {
"cpx": "^1.5.0",
"rimraf": "^3.0.2",
"rolldown": "1.0.0-beta.1",
"ts-loader": "^9.5.0",
"typescript": "^5.7.2"
},
"dependencies": {
"@janhq/core": "../../core/package.tgz",
"fetch-retry": "^5.0.6",
"ulidx": "^2.3.0"
},
"engines": {
"node": ">=18.0.0"
},
"files": [
"dist/*",
"package.json",
"README.md"
],
"bundleDependencies": [
"fetch-retry"
],
"installConfig": {
"hoistingLimits": "workspaces"
},
"packageManager": "yarn@4.5.3"
}

24
extensions/inference-groq-extension/resources/settings.json
View File

@ -1,24 +0,0 @@
[
{
"key": "groq-api-key",
"title": "API Key",
"description": "The Groq API uses API keys for authentication. Visit your [API Keys](https://console.groq.com/keys) page to retrieve the API key you'll use in your requests.",
"controllerType": "input",
"controllerProps": {
"placeholder": "Insert API Key",
"value": "",
"type": "password",
"inputActions": ["unobscure", "copy"]
}
},
{
"key": "chat-completions-endpoint",
"title": "Chat Completions Endpoint",
"description": "The endpoint to use for chat completions. See the [Groq documentation](https://console.groq.com/docs/openai) for more information.",
"controllerType": "input",
"controllerProps": {
"placeholder": "https://api.groq.com/openai/v1/chat/completions",
"value": "https://api.groq.com/openai/v1/chat/completions"
}
}
]

18
extensions/inference-groq-extension/rolldown.config.mjs
View File

@ -1,18 +0,0 @@
import { defineConfig } from 'rolldown'
import pkgJson from './package.json' with { type: 'json' }
import settingJson from './resources/settings.json' with { type: 'json' }
import modelsJson from './resources/models.json' with { type: 'json' }
export default defineConfig({
input: 'src/index.ts',
output: {
format: 'esm',
file: 'dist/index.js',
},
platform: 'browser',
define: {
MODELS: JSON.stringify(modelsJson),
SETTINGS: JSON.stringify(settingJson),
ENGINE: JSON.stringify(pkgJson.engine),
},
})

2
extensions/inference-groq-extension/src/env.d.ts vendored
View File

@ -1,2 +0,0 @@
declare const SETTINGS: SettingComponentProps[]
declare const MODELS: Model[]

64
extensions/inference-groq-extension/src/index.ts
View File

@ -1,64 +0,0 @@
/**
* @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package.
* The class provides methods for initializing and stopping a model, and for making inference requests.
* It also subscribes to events emitted by the @janhq/core package and handles new message requests.
* @version 1.0.0
* @module inference-groq-extension/src/index
*/
import { RemoteOAIEngine } from '@janhq/core'
enum Settings {
apiKey = 'groq-api-key',
chatCompletionsEndPoint = 'chat-completions-endpoint',
}
/**
* A class that implements the InferenceExtension interface from the @janhq/core package.
* The class provides methods for initializing and stopping a model, and for making inference requests.
* It also subscribes to events emitted by the @janhq/core package and handles new message requests.
*/
export default class JanInferenceGroqExtension extends RemoteOAIEngine {
inferenceUrl: string = ''
provider = 'groq'
override async onLoad(): Promise<void> {
super.onLoad()
// Register Settings
this.registerSettings(SETTINGS)
this.registerModels(MODELS)
// Retrieve API Key Setting
this.apiKey = await this.getSetting<string>(Settings.apiKey, '')
this.inferenceUrl = await this.getSetting<string>(
Settings.chatCompletionsEndPoint,
''
)
if (this.inferenceUrl.length === 0) {
SETTINGS.forEach((setting) => {
if (setting.key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = setting.controllerProps.value as string
}
})
}
}
onSettingUpdate<T>(key: string, value: T): void {
if (key === Settings.apiKey) {
this.apiKey = value as string
} else if (key === Settings.chatCompletionsEndPoint) {
if (typeof value !== 'string') return
if (value.trim().length === 0) {
SETTINGS.forEach((setting) => {
if (setting.key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = setting.controllerProps.value as string
}
})
} else {
this.inferenceUrl = value
}
}
}
}

14
extensions/inference-groq-extension/tsconfig.json
View File

@ -1,14 +0,0 @@
{
"compilerOptions": {
"target": "es2016",
"module": "ES6",
"moduleResolution": "node",
"outDir": "./dist",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"strict": false,
"skipLibCheck": true,
"rootDir": "./src"
},
"include": ["./src"]
}

79
extensions/inference-martian-extension/README.md
View File

@ -1,79 +0,0 @@
# Martian Engine Extension
Created using Jan extension example
# Create a Jan Extension using Typescript
Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀
## Create Your Own Extension
To create your own extension, you can use this repository as a template! Just follow the below instructions:
1. Click the Use this template button at the top of the repository
2. Select Create a new repository
3. Select an owner and name for your new repository
4. Click Create repository
5. Clone your new repository
## Initial Setup
After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension.
> [!NOTE]
>
> You'll need to have a reasonably modern version of
> [Node.js](https://nodejs.org) handy. If you are using a version manager like
> [`nodenv`](https://github.com/nodenv/nodenv) or
> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the
> root of your repository to install the version specified in
> [`package.json`](./package.json). Otherwise, 20.x or later should work!
1. :hammer_and_wrench: Install the dependencies
```bash
npm install
```
1. :building_construction: Package the TypeScript for distribution
```bash
npm run bundle
```
1. :white_check_mark: Check your artifact
There will be a tgz file in your extension directory now
## Update the Extension Metadata
The [`package.json`](package.json) file defines metadata about your extension, such as
extension name, main entry, description and version.
When you copy this repository, update `package.json` with the name, description for your extension.
## Update the Extension Code
The [`src/`](./src/) directory is the heart of your extension! This contains the
source code that will be run when your extension functions are invoked. You can replace the
contents of this directory with your own code.
There are a few things to keep in mind when writing your extension code:
- Most Jan Extension functions are processed asynchronously.
In `index.ts`, you will see that the extension function will return a `Promise<any>`.
```typescript
import { events, MessageEvent, MessageRequest } from '@janhq/core'
function onStart(): Promise<any> {
return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) =>
this.inference(data)
)
}
```
For more information about the Jan Extension Core module, see the
[documentation](https://github.com/janhq/jan/blob/main/core/README.md).
So, what are you waiting for? Go ahead and start customizing your extension!

42
extensions/inference-martian-extension/package.json
View File

@ -1,42 +0,0 @@
{
"name": "@janhq/inference-martian-extension",
"productName": "Martian Inference Engine",
"version": "1.0.1",
"description": "This extension enables Martian chat completion API calls",
"main": "dist/index.js",
"module": "dist/module.js",
"engine": "martian",
"author": "Jan <service@jan.ai>",
"license": "AGPL-3.0",
"scripts": {
"build": "rolldown -c rolldown.config.mjs",
"build:publish": "rimraf *.tgz --glob || true && yarn build && npm pack && cpx *.tgz ../../pre-install"
},
"devDependencies": {
"cpx": "^1.5.0",
"rimraf": "^3.0.2",
"rolldown": "1.0.0-beta.1",
"ts-loader": "^9.5.0",
"typescript": "^5.7.2"
},
"dependencies": {
"@janhq/core": "../../core/package.tgz",
"fetch-retry": "^5.0.6",
"ulidx": "^2.3.0"
},
"engines": {
"node": ">=18.0.0"
},
"files": [
"dist/*",
"package.json",
"README.md"
],
"bundleDependencies": [
"fetch-retry"
],
"installConfig": {
"hoistingLimits": "workspaces"
},
"packageManager": "yarn@4.5.3"
}

24
extensions/inference-martian-extension/resources/settings.json
View File

@ -1,24 +0,0 @@
[
{
"key": "martian-api-key",
"title": "API Key",
"description": "The Martian API uses API keys for authentication. Visit your [API Keys](https://withmartian.com/dashboard) page to retrieve the API key you'll use in your requests.",
"controllerType": "input",
"controllerProps": {
"placeholder": "Insert API Key",
"value": "",
"type": "password",
"inputActions": ["unobscure", "copy"]
}
},
{
"key": "chat-completions-endpoint",
"title": "Chat Completions Endpoint",
"description": "The endpoint to use for chat completions. See the [Martian API documentation](https://docs.withmartian.com/martian-model-router/getting-started/quickstart-integrating-martian-into-your-codebase) for more information.",
"controllerType": "input",
"controllerProps": {
"placeholder": "https://withmartian.com/api/openai/v1/chat/completions",
"value": "https://withmartian.com/api/openai/v1/chat/completions"
}
}
]

18
extensions/inference-martian-extension/rolldown.config.mjs
View File

@ -1,18 +0,0 @@
import { defineConfig } from 'rolldown'
import pkgJson from './package.json' with { type: 'json' }
import settingJson from './resources/settings.json' with { type: 'json' }
import modelsJson from './resources/models.json' with { type: 'json' }
export default defineConfig({
input: 'src/index.ts',
output: {
format: 'esm',
file: 'dist/index.js',
},
platform: 'browser',
define: {
MODELS: JSON.stringify(modelsJson),
SETTINGS: JSON.stringify(settingJson),
ENGINE: JSON.stringify(pkgJson.engine),
},
})

2
extensions/inference-martian-extension/src/env.d.ts vendored
View File

@ -1,2 +0,0 @@
declare const SETTINGS: SettingComponentProps[]
declare const MODELS: Model[]

63
extensions/inference-martian-extension/src/index.ts
View File

@ -1,63 +0,0 @@
/**
* @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package.
* The class provides methods for initializing and stopping a model, and for making inference requests.
* It also subscribes to events emitted by the @janhq/core package and handles new message requests.
* @version 1.0.0
* @module inference-martian-extension/src/index
*/
import { RemoteOAIEngine } from '@janhq/core'
enum Settings {
apiKey = 'martian-api-key',
chatCompletionsEndPoint = 'chat-completions-endpoint',
}
/**
* A class that implements the InferenceExtension interface from the @janhq/core package.
* The class provides methods for initializing and stopping a model, and for making inference requests.
* It also subscribes to events emitted by the @janhq/core package and handles new message requests.
*/
export default class JanInferenceMartianExtension extends RemoteOAIEngine {
inferenceUrl: string = ''
provider: string = 'martian'
override async onLoad(): Promise<void> {
super.onLoad()
// Register Settings
this.registerSettings(SETTINGS)
this.registerModels(MODELS)
this.apiKey = await this.getSetting<string>(Settings.apiKey, '')
this.inferenceUrl = await this.getSetting<string>(
Settings.chatCompletionsEndPoint,
''
)
if (this.inferenceUrl.length === 0) {
SETTINGS.forEach((setting) => {
if (setting.key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = setting.controllerProps.value as string
}
})
}
}
onSettingUpdate<T>(key: string, value: T): void {
if (key === Settings.apiKey) {
this.apiKey = value as string
} else if (key === Settings.chatCompletionsEndPoint) {
if (typeof value !== 'string') return
if (value.trim().length === 0) {
SETTINGS.forEach((setting) => {
if (setting.key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = setting.controllerProps.value as string
}
})
} else {
this.inferenceUrl = value
}
}
}
}

14
extensions/inference-martian-extension/tsconfig.json
View File

@ -1,14 +0,0 @@
{
"compilerOptions": {
"target": "es2016",
"module": "ES6",
"moduleResolution": "node",
"outDir": "./dist",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"strict": false,
"skipLibCheck": true,
"rootDir": "./src"
},
"include": ["./src"]
}

79
extensions/inference-mistral-extension/README.md
View File

@ -1,79 +0,0 @@
# Mistral Engine Extension
Created using Jan extension example
# Create a Jan Extension using Typescript
Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀
## Create Your Own Extension
To create your own extension, you can use this repository as a template! Just follow the below instructions:
1. Click the Use this template button at the top of the repository
2. Select Create a new repository
3. Select an owner and name for your new repository
4. Click Create repository
5. Clone your new repository
## Initial Setup
After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension.
> [!NOTE]
>
> You'll need to have a reasonably modern version of
> [Node.js](https://nodejs.org) handy. If you are using a version manager like
> [`nodenv`](https://github.com/nodenv/nodenv) or
> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the
> root of your repository to install the version specified in
> [`package.json`](./package.json). Otherwise, 20.x or later should work!
1. :hammer_and_wrench: Install the dependencies
```bash
npm install
```
1. :building_construction: Package the TypeScript for distribution
```bash
npm run bundle
```
1. :white_check_mark: Check your artifact
There will be a tgz file in your extension directory now
## Update the Extension Metadata
The [`package.json`](package.json) file defines metadata about your extension, such as
extension name, main entry, description and version.
When you copy this repository, update `package.json` with the name, description for your extension.
## Update the Extension Code
The [`src/`](./src/) directory is the heart of your extension! This contains the
source code that will be run when your extension functions are invoked. You can replace the
contents of this directory with your own code.
There are a few things to keep in mind when writing your extension code:
- Most Jan Extension functions are processed asynchronously.
In `index.ts`, you will see that the extension function will return a `Promise<any>`.
```typescript
import { events, MessageEvent, MessageRequest } from '@janhq/core'
function onStart(): Promise<any> {
return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) =>
this.inference(data)
)
}
```
For more information about the Jan Extension Core module, see the
[documentation](https://github.com/janhq/jan/blob/main/core/README.md).
So, what are you waiting for? Go ahead and start customizing your extension!

42
extensions/inference-mistral-extension/package.json
View File

@ -1,42 +0,0 @@
{
"name": "@janhq/inference-mistral-extension",
"productName": "MistralAI Inference Engine",
"version": "1.0.1",
"description": "This extension enables Mistral chat completion API calls",
"main": "dist/index.js",
"module": "dist/module.js",
"engine": "mistral",
"author": "Jan <service@jan.ai>",
"license": "AGPL-3.0",
"scripts": {
"build": "rolldown -c rolldown.config.mjs",
"build:publish": "rimraf *.tgz --glob || true && yarn build && npm pack && cpx *.tgz ../../pre-install"
},
"devDependencies": {
"cpx": "^1.5.0",
"rimraf": "^3.0.2",
"rolldown": "1.0.0-beta.1",
"ts-loader": "^9.5.0",
"typescript": "^5.7.2"
},
"dependencies": {
"@janhq/core": "../../core/package.tgz",
"fetch-retry": "^5.0.6",
"ulidx": "^2.3.0"
},
"engines": {
"node": ">=18.0.0"
},
"files": [
"dist/*",
"package.json",
"README.md"
],
"bundleDependencies": [
"fetch-retry"
],
"installConfig": {
"hoistingLimits": "workspaces"
},
"packageManager": "yarn@4.5.3"
}

24
extensions/inference-mistral-extension/resources/settings.json
View File

@ -1,24 +0,0 @@
[
{
"key": "mistral-api-key",
"title": "API Key",
"description": "The Mistral API uses API keys for authentication. Visit your [API Keys](https://console.mistral.ai/api-keys/) page to retrieve the API key you'll use in your requests.",
"controllerType": "input",
"controllerProps": {
"placeholder": "Insert API Key",
"value": "",
"type": "password",
"inputActions": ["unobscure", "copy"]
}
},
{
"key": "chat-completions-endpoint",
"title": "Chat Completions Endpoint",
"description": "The endpoint to use for chat completions. See the [Mistral API documentation](https://docs.mistral.ai/api/#operation/createChatCompletion) for more information.",
"controllerType": "input",
"controllerProps": {
"placeholder": "https://api.mistral.ai/v1/chat/completions",
"value": "https://api.mistral.ai/v1/chat/completions"
}
}
]

18
extensions/inference-mistral-extension/rolldown.config.mjs
View File

@ -1,18 +0,0 @@
import { defineConfig } from 'rolldown'
import pkgJson from './package.json' with { type: 'json' }
import settingJson from './resources/settings.json' with { type: 'json' }
import modelsJson from './resources/models.json' with { type: 'json' }
export default defineConfig({
input: 'src/index.ts',
output: {
format: 'esm',
file: 'dist/index.js',
},
platform: 'browser',
define: {
MODELS: JSON.stringify(modelsJson),
SETTINGS: JSON.stringify(settingJson),
ENGINE: JSON.stringify(pkgJson.engine),
},
})

2
extensions/inference-mistral-extension/src/env.d.ts vendored
View File

@ -1,2 +0,0 @@
declare const SETTINGS: SettingComponentProps[]
declare const MODELS: Model[]

63
extensions/inference-mistral-extension/src/index.ts
View File

@ -1,63 +0,0 @@
/**
* @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package.
* The class provides methods for initializing and stopping a model, and for making inference requests.
* It also subscribes to events emitted by the @janhq/core package and handles new message requests.
* @version 1.0.0
* @module inference-mistral-extension/src/index
*/
import { RemoteOAIEngine } from '@janhq/core'
enum Settings {
apiKey = 'mistral-api-key',
chatCompletionsEndPoint = 'chat-completions-endpoint',
}
/**
* A class that implements the InferenceExtension interface from the @janhq/core package.
* The class provides methods for initializing and stopping a model, and for making inference requests.
* It also subscribes to events emitted by the @janhq/core package and handles new message requests.
*/
export default class JanInferenceMistralExtension extends RemoteOAIEngine {
inferenceUrl: string = ''
provider: string = 'mistral'
override async onLoad(): Promise<void> {
super.onLoad()
// Register Settings
this.registerSettings(SETTINGS)
this.registerModels(MODELS)
this.apiKey = await this.getSetting<string>(Settings.apiKey, '')
this.inferenceUrl = await this.getSetting<string>(
Settings.chatCompletionsEndPoint,
''
)
if (this.inferenceUrl.length === 0) {
SETTINGS.forEach((setting) => {
if (setting.key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = setting.controllerProps.value as string
}
})
}
}
onSettingUpdate<T>(key: string, value: T): void {
if (key === Settings.apiKey) {
this.apiKey = value as string
} else if (key === Settings.chatCompletionsEndPoint) {
if (typeof value !== 'string') return
if (value.trim().length === 0) {
SETTINGS.forEach((setting) => {
if (setting.key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = setting.controllerProps.value as string
}
})
} else {
this.inferenceUrl = value
}
}
}
}

14
extensions/inference-mistral-extension/tsconfig.json
View File

@ -1,14 +0,0 @@
{
"compilerOptions": {
"target": "es2016",
"module": "ES6",
"moduleResolution": "node",
"outDir": "./dist",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"strict": false,
"skipLibCheck": true,
"rootDir": "./src"
},
"include": ["./src"]
}

79
extensions/inference-nvidia-extension/README.md
View File

@ -1,79 +0,0 @@
# Nvidia Engine Extension
Created using Jan extension example
# Create a Jan Extension using Typescript
Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀
## Create Your Own Extension
To create your own extension, you can use this repository as a template! Just follow the below instructions:
1. Click the Use this template button at the top of the repository
2. Select Create a new repository
3. Select an owner and name for your new repository
4. Click Create repository
5. Clone your new repository
## Initial Setup
After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension.
> [!NOTE]
>
> You'll need to have a reasonably modern version of
> [Node.js](https://nodejs.org) handy. If you are using a version manager like
> [`nodenv`](https://github.com/nodenv/nodenv) or
> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the
> root of your repository to install the version specified in
> [`package.json`](./package.json). Otherwise, 20.x or later should work!
1. :hammer_and_wrench: Install the dependencies
```bash
npm install
```
1. :building_construction: Package the TypeScript for distribution
```bash
npm run bundle
```
1. :white_check_mark: Check your artifact
There will be a tgz file in your extension directory now
## Update the Extension Metadata
The [`package.json`](package.json) file defines metadata about your extension, such as
extension name, main entry, description and version.
When you copy this repository, update `package.json` with the name, description for your extension.
## Update the Extension Code
The [`src/`](./src/) directory is the heart of your extension! This contains the
source code that will be run when your extension functions are invoked. You can replace the
contents of this directory with your own code.
There are a few things to keep in mind when writing your extension code:
- Most Jan Extension functions are processed asynchronously.
In `index.ts`, you will see that the extension function will return a `Promise<any>`.
```typescript
import { events, MessageEvent, MessageRequest } from '@janhq/core'
function onStart(): Promise<any> {
return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) =>
this.inference(data)
)
}
```
For more information about the Jan Extension Core module, see the
[documentation](https://github.com/janhq/jan/blob/main/core/README.md).
So, what are you waiting for? Go ahead and start customizing your extension!

42
extensions/inference-nvidia-extension/package.json
View File

@ -1,42 +0,0 @@
{
"name": "@janhq/inference-nvidia-extension",
"productName": "NVIDIA NIM Inference Engine",
"version": "1.0.1",
"description": "This extension enables NVIDIA chat completion API calls",
"main": "dist/index.js",
"module": "dist/module.js",
"engine": "nvidia",
"author": "Jan <service@jan.ai>",
"license": "AGPL-3.0",
"scripts": {
"build": "rolldown -c rolldown.config.mjs",
"build:publish": "rimraf *.tgz --glob || true && yarn build && npm pack && cpx *.tgz ../../pre-install"
},
"devDependencies": {
"cpx": "^1.5.0",
"rimraf": "^3.0.2",
"rolldown": "1.0.0-beta.1",
"ts-loader": "^9.5.0",
"typescript": "^5.7.2"
},
"dependencies": {
"@janhq/core": "../../core/package.tgz",
"fetch-retry": "^5.0.6",
"ulidx": "^2.3.0"
},
"engines": {
"node": ">=18.0.0"
},
"files": [
"dist/*",
"package.json",
"README.md"
],
"bundleDependencies": [
"fetch-retry"
],
"installConfig": {
"hoistingLimits": "workspaces"
},
"packageManager": "yarn@4.5.3"
}

24
extensions/inference-nvidia-extension/resources/settings.json
View File

@ -1,24 +0,0 @@
[
{
"key": "nvidia-api-key",
"title": "API Key",
"description": "The NVIDIA API uses API keys for authentication. Visit your [API Keys](https://org.ngc.nvidia.com/setup/personal-keys) page to retrieve the API key you'll use in your requests..",
"controllerType": "input",
"controllerProps": {
"placeholder": "Insert API Key",
"value": "",
"type": "password",
"inputActions": ["unobscure", "copy"]
}
},
{
"key": "chat-completions-endpoint",
"title": "Chat Completions Endpoint",
"description": "The endpoint to use for chat completions. See the [NVIDIA API documentation](https://www.nvidia.com/en-us/ai/) for more information.",
"controllerType": "input",
"controllerProps": {
"placeholder": "https://integrate.api.nvidia.com/v1/chat/completions",
"value": "https://integrate.api.nvidia.com/v1/chat/completions"
}
}
]

18
extensions/inference-nvidia-extension/rolldown.config.mjs
View File

@ -1,18 +0,0 @@
import { defineConfig } from 'rolldown'
import pkgJson from './package.json' with { type: 'json' }
import settingJson from './resources/settings.json' with { type: 'json' }
import modelsJson from './resources/models.json' with { type: 'json' }
export default defineConfig({
input: 'src/index.ts',
output: {
format: 'esm',
file: 'dist/index.js',
},
platform: 'browser',
define: {
MODELS: JSON.stringify(modelsJson),
SETTINGS: JSON.stringify(settingJson),
ENGINE: JSON.stringify(pkgJson.engine),
},
})

2
extensions/inference-nvidia-extension/src/env.d.ts vendored
View File

@ -1,2 +0,0 @@
declare const SETTINGS: SettingComponentProps[]
declare const MODELS: Model[]

63
extensions/inference-nvidia-extension/src/index.ts
View File

@ -1,63 +0,0 @@
/**
* @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package.
* The class provides methods for initializing and stopping a model, and for making inference requests.
* It also subscribes to events emitted by the @janhq/core package and handles new message requests.
* @version 1.0.0
* @module inference-mistral-extension/src/index
*/
import { RemoteOAIEngine } from '@janhq/core'
enum Settings {
apiKey = 'nvidia-api-key',
chatCompletionsEndPoint = 'chat-completions-endpoint',
}
/**
* A class that implements the InferenceExtension interface from the @janhq/core package.
* The class provides methods for initializing and stopping a model, and for making inference requests.
* It also subscribes to events emitted by the @janhq/core package and handles new message requests.
*/
export default class JanNVIDIANIMInferenceEngine extends RemoteOAIEngine {
inferenceUrl: string = ''
provider: string = 'nvidia'
override async onLoad(): Promise<void> {
super.onLoad()
// Register Settings
this.registerSettings(SETTINGS)
this.registerModels(MODELS)
this.apiKey = await this.getSetting<string>(Settings.apiKey, '')
this.inferenceUrl = await this.getSetting<string>(
Settings.chatCompletionsEndPoint,
''
)
if (this.inferenceUrl.length === 0) {
SETTINGS.forEach((setting) => {
if (setting.key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = setting.controllerProps.value as string
}
})
}
}
onSettingUpdate<T>(key: string, value: T): void {
if (key === Settings.apiKey) {
this.apiKey = value as string
} else if (key === Settings.chatCompletionsEndPoint) {
if (typeof value !== 'string') return
if (value.trim().length === 0) {
SETTINGS.forEach((setting) => {
if (setting.key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = setting.controllerProps.value as string
}
})
} else {
this.inferenceUrl = value
}
}
}
}

14
extensions/inference-nvidia-extension/tsconfig.json
View File

@ -1,14 +0,0 @@
{
"compilerOptions": {
"target": "es2016",
"module": "ES6",
"moduleResolution": "node",
"outDir": "./dist",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"strict": false,
"skipLibCheck": true,
"rootDir": "./src"
},
"include": ["./src"]
}

79
extensions/inference-openai-extension/README.md
View File

@ -1,79 +0,0 @@
# OpenAI Engine Extension
Created using Jan extension example
# Create a Jan Extension using Typescript
Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀
## Create Your Own Extension
To create your own extension, you can use this repository as a template! Just follow the below instructions:
1. Click the Use this template button at the top of the repository
2. Select Create a new repository
3. Select an owner and name for your new repository
4. Click Create repository
5. Clone your new repository
## Initial Setup
After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension.
> [!NOTE]
>
> You'll need to have a reasonably modern version of
> [Node.js](https://nodejs.org) handy. If you are using a version manager like
> [`nodenv`](https://github.com/nodenv/nodenv) or
> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the
> root of your repository to install the version specified in
> [`package.json`](./package.json). Otherwise, 20.x or later should work!
1. :hammer_and_wrench: Install the dependencies
```bash
npm install
```
1. :building_construction: Package the TypeScript for distribution
```bash
npm run bundle
```
1. :white_check_mark: Check your artifact
There will be a tgz file in your extension directory now
## Update the Extension Metadata
The [`package.json`](package.json) file defines metadata about your extension, such as
extension name, main entry, description and version.
When you copy this repository, update `package.json` with the name, description for your extension.
## Update the Extension Code
The [`src/`](./src/) directory is the heart of your extension! This contains the
source code that will be run when your extension functions are invoked. You can replace the
contents of this directory with your own code.
There are a few things to keep in mind when writing your extension code:
- Most Jan Extension functions are processed asynchronously.
In `index.ts`, you will see that the extension function will return a `Promise<any>`.
```typescript
import { events, MessageEvent, MessageRequest } from '@janhq/core'
function onStart(): Promise<any> {
return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) =>
this.inference(data)
)
}
```
For more information about the Jan Extension Core module, see the
[documentation](https://github.com/janhq/jan/blob/main/core/README.md).
So, what are you waiting for? Go ahead and start customizing your extension!

9
extensions/inference-openai-extension/jest.config.js
View File

@ -1,9 +0,0 @@
/** @type {import('ts-jest').JestConfigWithTsJest} */
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
transform: {
'node_modules/@janhq/core/.+\\.(j|t)s?$': 'ts-jest',
},
transformIgnorePatterns: ['node_modules/(?!@janhq/core/.*)'],
}

42
extensions/inference-openai-extension/package.json
View File

@ -1,42 +0,0 @@
{
"name": "@janhq/inference-openai-extension",
"productName": "OpenAI Inference Engine",
"version": "1.0.5",
"description": "This extension enables OpenAI chat completion API calls",
"main": "dist/index.js",
"module": "dist/module.js",
"engine": "openai",
"author": "Jan <service@jan.ai>",
"license": "AGPL-3.0",
"scripts": {
"build": "rolldown -c rolldown.config.mjs",
"build:publish": "rimraf *.tgz --glob || true && yarn build && npm pack && cpx *.tgz ../../pre-install"
},
"devDependencies": {
"cpx": "^1.5.0",
"rimraf": "^3.0.2",
"rolldown": "1.0.0-beta.1",
"ts-loader": "^9.5.0",
"typescript": "^5.7.2"
},
"dependencies": {
"@janhq/core": "../../core/package.tgz",
"fetch-retry": "^5.0.6",
"ulidx": "^2.3.0"
},
"engines": {
"node": ">=18.0.0"
},
"files": [
"dist/*",
"package.json",
"README.md"
],
"bundleDependencies": [
"fetch-retry"
],
"installConfig": {
"hoistingLimits": "workspaces"
},
"packageManager": "yarn@4.5.3"
}

24
extensions/inference-openai-extension/resources/settings.json
View File

@ -1,24 +0,0 @@
[
{
"key": "openai-api-key",
"title": "API Key",
"description": "The OpenAI API uses API keys for authentication. Visit your [API Keys](https://platform.openai.com/account/api-keys) page to retrieve the API key you'll use in your requests.",
"controllerType": "input",
"controllerProps": {
"placeholder": "Insert API Key",
"value": "",
"type": "password",
"inputActions": ["unobscure", "copy"]
}
},
{
"key": "chat-completions-endpoint",
"title": "Chat Completions Endpoint",
"description": "The endpoint to use for chat completions. See the [OpenAI API documentation](https://platform.openai.com/docs/api-reference/chat/create) for more information.",
"controllerType": "input",
"controllerProps": {
"placeholder": "https://api.openai.com/v1/chat/completions",
"value": "https://api.openai.com/v1/chat/completions"
}
}
]

18
extensions/inference-openai-extension/rolldown.config.mjs
View File

@ -1,18 +0,0 @@
import { defineConfig } from 'rolldown'
import pkgJson from './package.json' with { type: 'json' }
import settingJson from './resources/settings.json' with { type: 'json' }
import modelsJson from './resources/models.json' with { type: 'json' }
export default defineConfig({
input: 'src/index.ts',
output: {
format: 'esm',
file: 'dist/index.js',
},
platform: 'browser',
define: {
MODELS: JSON.stringify(modelsJson),
SETTINGS: JSON.stringify(settingJson),
ENGINE: JSON.stringify(pkgJson.engine),
},
})

54
extensions/inference-openai-extension/src/OpenAIExtension.test.ts
View File

@ -1,54 +0,0 @@
/**
* @jest-environment jsdom
*/
jest.mock('@janhq/core', () => ({
...jest.requireActual('@janhq/core/node'),
RemoteOAIEngine: jest.fn().mockImplementation(() => ({
onLoad: jest.fn(),
registerSettings: jest.fn(),
registerModels: jest.fn(),
getSetting: jest.fn(),
onSettingUpdate: jest.fn(),
})),
}))
import JanInferenceOpenAIExtension, { Settings } from '.'
describe('JanInferenceOpenAIExtension', () => {
let extension: JanInferenceOpenAIExtension
beforeEach(() => {
// @ts-ignore
extension = new JanInferenceOpenAIExtension()
})
it('should initialize with settings and models', async () => {
await extension.onLoad()
// Assuming there are some default SETTINGS and MODELS being registered
expect(extension.apiKey).toBe(undefined)
expect(extension.inferenceUrl).toBe('')
})
it('should transform the payload for preview models', () => {
const payload: any = {
max_tokens: 100,
model: 'o1-mini',
// Add other required properties...
}
const transformedPayload = extension.transformPayload(payload)
expect(transformedPayload.max_completion_tokens).toBe(payload.max_tokens)
expect(transformedPayload).not.toHaveProperty('max_tokens')
expect(transformedPayload).toHaveProperty('max_completion_tokens')
})
it('should not transform the payload for non-preview models', () => {
const payload: any = {
max_tokens: 100,
model: 'non-preview-model',
// Add other required properties...
}
const transformedPayload = extension.transformPayload(payload)
expect(transformedPayload).toEqual(payload)
})
})

2
extensions/inference-openai-extension/src/env.d.ts vendored
View File

@ -1,2 +0,0 @@
declare const SETTINGS: SettingComponentProps[]
declare const MODELS: Model[]

90
extensions/inference-openai-extension/src/index.ts
View File

@ -1,90 +0,0 @@
/**
* @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package.
* The class provides methods for initializing and stopping a model, and for making inference requests.
* It also subscribes to events emitted by the @janhq/core package and handles new message requests.
* @version 1.0.0
* @module inference-openai-extension/src/index
*/
import { ModelRuntimeParams, PayloadType, RemoteOAIEngine } from '@janhq/core'
export enum Settings {
apiKey = 'openai-api-key',
chatCompletionsEndPoint = 'chat-completions-endpoint',
}
type OpenAIPayloadType = PayloadType &
ModelRuntimeParams & { max_completion_tokens: number }
/**
* A class that implements the InferenceExtension interface from the @janhq/core package.
* The class provides methods for initializing and stopping a model, and for making inference requests.
* It also subscribes to events emitted by the @janhq/core package and handles new message requests.
*/
export default class JanInferenceOpenAIExtension extends RemoteOAIEngine {
inferenceUrl: string = ''
provider: string = 'openai'
previewModels = ['o1-mini', 'o1-preview']
override async onLoad(): Promise<void> {
super.onLoad()
// Register Settings
this.registerSettings(SETTINGS)
this.registerModels(MODELS)
this.apiKey = await this.getSetting<string>(Settings.apiKey, '')
this.inferenceUrl = await this.getSetting<string>(
Settings.chatCompletionsEndPoint,
''
)
if (this.inferenceUrl.length === 0) {
SETTINGS.forEach((setting) => {
if (setting.key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = setting.controllerProps.value as string
}
})
}
}
onSettingUpdate<T>(key: string, value: T): void {
if (key === Settings.apiKey) {
this.apiKey = value as string
} else if (key === Settings.chatCompletionsEndPoint) {
if (typeof value !== 'string') return
if (value.trim().length === 0) {
SETTINGS.forEach((setting) => {
if (setting.key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = setting.controllerProps.value as string
}
})
} else {
this.inferenceUrl = value
}
}
}
/**
* Tranform the payload before sending it to the inference endpoint.
* The new preview models such as o1-mini and o1-preview replaced max_tokens by max_completion_tokens parameter.
* Others do not.
* @param payload
* @returns
*/
transformPayload = (payload: OpenAIPayloadType): OpenAIPayloadType => {
// Remove empty stop words
if (payload.stop?.length === 0) {
const { stop, ...params } = payload
payload = params
}
// Transform the payload for preview models
if (this.previewModels.includes(payload.model)) {
const { max_tokens, stop, ...params } = payload
return {
...params,
max_completion_tokens: max_tokens,
}
}
// Pass through for non-preview models
return payload
}
}

15
extensions/inference-openai-extension/tsconfig.json
View File

@ -1,15 +0,0 @@
{
"compilerOptions": {
"target": "es2016",
"module": "ES6",
"moduleResolution": "node",
"outDir": "./dist",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"strict": false,
"skipLibCheck": true,
"rootDir": "./src"
},
"include": ["./src"],
"exclude": ["**/*.test.ts"]
}

79
extensions/inference-openrouter-extension/README.md
View File

@ -1,79 +0,0 @@
# Open Router Engine Extension
Created using Jan extension example
# Create a Jan Extension using Typescript
Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀
## Create Your Own Extension
To create your own extension, you can use this repository as a template! Just follow the below instructions:
1. Click the Use this template button at the top of the repository
2. Select Create a new repository
3. Select an owner and name for your new repository
4. Click Create repository
5. Clone your new repository
## Initial Setup
After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension.
> [!NOTE]
>
> You'll need to have a reasonably modern version of
> [Node.js](https://nodejs.org) handy. If you are using a version manager like
> [`nodenv`](https://github.com/nodenv/nodenv) or
> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the
> root of your repository to install the version specified in
> [`package.json`](./package.json). Otherwise, 20.x or later should work!
1. :hammer_and_wrench: Install the dependencies
```bash
npm install
```
1. :building_construction: Package the TypeScript for distribution
```bash
npm run bundle
```
1. :white_check_mark: Check your artifact
There will be a tgz file in your extension directory now
## Update the Extension Metadata
The [`package.json`](package.json) file defines metadata about your extension, such as
extension name, main entry, description and version.
When you copy this repository, update `package.json` with the name, description for your extension.
## Update the Extension Code
The [`src/`](./src/) directory is the heart of your extension! This contains the
source code that will be run when your extension functions are invoked. You can replace the
contents of this directory with your own code.
There are a few things to keep in mind when writing your extension code:
- Most Jan Extension functions are processed asynchronously.
In `index.ts`, you will see that the extension function will return a `Promise<any>`.
```typescript
import { events, MessageEvent, MessageRequest } from '@janhq/core'
function onStart(): Promise<any> {
return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) =>
this.inference(data)
)
}
```
For more information about the Jan Extension Core module, see the
[documentation](https://github.com/janhq/jan/blob/main/core/README.md).
So, what are you waiting for? Go ahead and start customizing your extension!

42
extensions/inference-openrouter-extension/package.json
View File

@ -1,42 +0,0 @@
{
"name": "@janhq/inference-openrouter-extension",
"productName": "OpenRouter Inference Engine",
"version": "1.0.0",
"description": "This extension enables Open Router chat completion API calls",
"main": "dist/index.js",
"module": "dist/module.js",
"engine": "openrouter",
"author": "Jan <service@jan.ai>",
"license": "AGPL-3.0",
"scripts": {
"build": "rolldown -c rolldown.config.mjs",
"build:publish": "rimraf *.tgz --glob || true && yarn build && npm pack && cpx *.tgz ../../pre-install"
},
"devDependencies": {
"cpx": "^1.5.0",
"rimraf": "^3.0.2",
"rolldown": "1.0.0-beta.1",
"ts-loader": "^9.5.0",
"typescript": "^5.7.2"
},
"dependencies": {
"@janhq/core": "../../core/package.tgz",
"fetch-retry": "^5.0.6",
"ulidx": "^2.3.0"
},
"engines": {
"node": ">=18.0.0"
},
"files": [
"dist/*",
"package.json",
"README.md"
],
"bundleDependencies": [
"fetch-retry"
],
"installConfig": {
"hoistingLimits": "workspaces"
},
"packageManager": "yarn@4.5.3"
}

34
extensions/inference-openrouter-extension/resources/settings.json
View File

@ -1,34 +0,0 @@
[
{
"key": "openrouter-api-key",
"title": "API Key",
"description": "The OpenRouter API uses API keys for authentication. Visit your [API Keys](https://openrouter.ai/keys) page to retrieve the API key you'll use in your requests.",
"controllerType": "input",
"controllerProps": {
"placeholder": "Insert API Key",
"value": "",
"type": "password",
"inputActions": ["unobscure", "copy"]
}
},
{
"key": "chat-completions-endpoint",
"title": "Chat Completions Endpoint",
"description": "The endpoint to use for chat completions. See the [OpenRouter API documentation](https://openrouter.ai/docs/requests) for more information.",
"controllerType": "input",
"controllerProps": {
"placeholder": "https://openrouter.ai/api/v1/chat/completions",
"value": "https://openrouter.ai/api/v1/chat/completions"
}
},
{
"key": "openrouter-model",
"title": "Model",
"description": "If the model parameter is omitted, the user or payer's default is used. Otherwise, remember to select a value for model from the [supported models](https://openrouter.ai/docs/models) or API, and include the organization prefix.",
"controllerType": "input",
"controllerProps": {
"placeholder": "Leave empty for default model",
"value": ""
}
}
]

18
extensions/inference-openrouter-extension/rolldown.config.mjs
View File

@ -1,18 +0,0 @@
import { defineConfig } from 'rolldown'
import pkgJson from './package.json' with { type: 'json' }
import settingJson from './resources/settings.json' with { type: 'json' }
import modelsJson from './resources/models.json' with { type: 'json' }
export default defineConfig({
input: 'src/index.ts',
output: {
format: 'esm',
file: 'dist/index.js',
},
platform: 'browser',
define: {
MODELS: JSON.stringify(modelsJson),
SETTINGS: JSON.stringify(settingJson),
ENGINE: JSON.stringify(pkgJson.engine),
},
})

2
extensions/inference-openrouter-extension/src/env.d.ts vendored
View File

@ -1,2 +0,0 @@
declare const SETTINGS: SettingComponentProps[]
declare const MODELS: Model[]

85
extensions/inference-openrouter-extension/src/index.ts
View File

@ -1,85 +0,0 @@
/**
* @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package.
* The class provides methods for initializing and stopping a model, and for making inference requests.
* It also subscribes to events emitted by the @janhq/core package and handles new message requests.
* @version 1.0.0
* @module inference-openai-extension/src/index
*/
import { RemoteOAIEngine } from '@janhq/core'
import { PayloadType } from '@janhq/core'
enum Settings {
apiKey = 'openrouter-api-key',
model = 'openrouter-model',
chatCompletionsEndPoint = 'chat-completions-endpoint',
}
/**
* A class that implements the InferenceExtension interface from the @janhq/core package.
* The class provides methods for initializing and stopping a model, and for making inference requests.
* It also subscribes to events emitted by the @janhq/core package and handles new message requests.
*/
export default class JanInferenceOpenRouterExtension extends RemoteOAIEngine {
inferenceUrl: string = ''
provider: string = 'openrouter'
model?: string | undefined
override async onLoad(): Promise<void> {
super.onLoad()
// Register Settings
this.registerSettings(SETTINGS)
this.registerModels(MODELS)
this.apiKey = await this.getSetting<string>(Settings.apiKey, '')
this.inferenceUrl = await this.getSetting<string>(
Settings.chatCompletionsEndPoint,
''
)
this.model = await this.getSetting<string>(Settings.model, '')
// Openrouter uses default model on no model param set
if (!this.model?.length) this.model = undefined
if (this.inferenceUrl.length === 0) {
SETTINGS.forEach((setting) => {
if (setting.key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = setting.controllerProps.value as string
}
})
}
}
override async headers(): Promise<HeadersInit> {
return {
'Content-Type': 'application/json',
'HTTP-Referer': 'https://jan.ai',
'Authorization': `Bearer ${this.apiKey}`,
}
}
onSettingUpdate<T>(key: string, value: T): void {
if (key === Settings.apiKey) {
this.apiKey = value as string
} else if (key === Settings.chatCompletionsEndPoint) {
if (typeof value !== 'string') return
if (value.trim().length === 0) {
SETTINGS.forEach((setting) => {
if (setting.key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = setting.controllerProps.value as string
}
})
} else {
this.inferenceUrl = value
}
} else if (key === Settings.model) {
this.model =
typeof value === 'string' && value.length > 0 ? value : undefined
}
}
transformPayload = (payload: PayloadType) => ({
...payload,
model: payload.model !== 'open-router-auto' ? payload.model : this.model,
})
}

14
extensions/inference-openrouter-extension/tsconfig.json
View File

@ -1,14 +0,0 @@
{
"compilerOptions": {
"target": "es2016",
"module": "ES6",
"moduleResolution": "node",
"outDir": "./dist",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"strict": false,
"skipLibCheck": true,
"rootDir": "./src"
},
"include": ["./src"]
}

75
extensions/inference-triton-trtllm-extension/README.md
View File

@ -1,75 +0,0 @@
# Create a Jan Extension using Typescript
Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀
## Create Your Own Extension
To create your own extension, you can use this repository as a template! Just follow the below instructions:
1. Click the Use this template button at the top of the repository
2. Select Create a new repository
3. Select an owner and name for your new repository
4. Click Create repository
5. Clone your new repository
## Initial Setup
After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension.
> [!NOTE]
>
> You'll need to have a reasonably modern version of
> [Node.js](https://nodejs.org) handy. If you are using a version manager like
> [`nodenv`](https://github.com/nodenv/nodenv) or
> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the
> root of your repository to install the version specified in
> [`package.json`](./package.json). Otherwise, 20.x or later should work!
1. :hammer_and_wrench: Install the dependencies
```bash
npm install
```
1. :building_construction: Package the TypeScript for distribution
```bash
npm run bundle
```
1. :white_check_mark: Check your artifact
There will be a tgz file in your extension directory now
## Update the Extension Metadata
The [`package.json`](package.json) file defines metadata about your extension, such as
extension name, main entry, description and version.
When you copy this repository, update `package.json` with the name, description for your extension.
## Update the Extension Code
The [`src/`](./src/) directory is the heart of your extension! This contains the
source code that will be run when your extension functions are invoked. You can replace the
contents of this directory with your own code.
There are a few things to keep in mind when writing your extension code:
- Most Jan Extension functions are processed asynchronously.
In `index.ts`, you will see that the extension function will return a `Promise<any>`.
```typescript
import { events, MessageEvent, MessageRequest } from '@janhq/core'
function onStart(): Promise<any> {
return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) =>
this.inference(data)
)
}
```
For more information about the Jan Extension Core module, see the
[documentation](https://github.com/janhq/jan/blob/main/core/README.md).
So, what are you waiting for? Go ahead and start customizing your extension!

42
extensions/inference-triton-trtllm-extension/package.json
View File

@ -1,42 +0,0 @@
{
"name": "@janhq/inference-triton-trt-llm-extension",
"productName": "Triton-TRT-LLM Inference Engine",
"version": "1.0.0",
"description": "This extension enables Nvidia's TensorRT-LLM as an inference engine option",
"main": "dist/index.js",
"engine": "triton_trtllm",
"author": "Jan <service@jan.ai>",
"license": "AGPL-3.0",
"scripts": {
"build": "rolldown -c rolldown.config.mjs",
"build:publish": "rimraf *.tgz --glob || true && yarn build && npm pack && cpx *.tgz ../../pre-install"
},
"devDependencies": {
"cpx": "^1.5.0",
"rimraf": "^3.0.2",
"rolldown": "1.0.0-beta.1",
"ts-loader": "^9.5.0",
"typescript": "^5.7.2"
},
"dependencies": {
"@janhq/core": "../../core/package.tgz",
"fetch-retry": "^5.0.6",
"rxjs": "^7.8.1",
"ulidx": "^2.3.0"
},
"engines": {
"node": ">=18.0.0"
},
"files": [
"dist/*",
"package.json",
"README.md"
],
"bundleDependencies": [
"fetch-retry"
],
"installConfig": {
"hoistingLimits": "workspaces"
},
"packageManager": "yarn@4.5.3"
}

24
extensions/inference-triton-trtllm-extension/resources/settings.json
View File

@ -1,24 +0,0 @@
[
{
"key": "tritonllm-api-key",
"title": "API Key",
"description": "The Triton LLM API uses API keys for authentication.",
"controllerType": "input",
"controllerProps": {
"placeholder": "Insert API Key",
"value": "",
"type": "password",
"inputActions": ["unobscure", "copy"]
}
},
{
"key": "chat-completions-endpoint",
"title": "Chat Completions Endpoint",
"description": "The endpoint to use for chat completions.",
"controllerType": "input",
"controllerProps": {
"placeholder": "http://localhost:8000/v2/models/tensorrt_llm_bls/generate",
"value": "http://localhost:8000/v2/models/tensorrt_llm_bls/generate"
}
}
]

16
extensions/inference-triton-trtllm-extension/rolldown.config.mjs
View File

@ -1,16 +0,0 @@
import { defineConfig } from 'rolldown'
import pkgJson from './package.json' with { type: 'json' }
import settingJson from './resources/settings.json' with { type: 'json' }
export default defineConfig({
input: 'src/index.ts',
output: {
format: 'esm',
file: 'dist/index.js',
},
platform: 'browser',
define: {
SETTINGS: JSON.stringify(settingJson),
ENGINE: JSON.stringify(pkgJson.engine),
},
})

1
extensions/inference-triton-trtllm-extension/src/env.d.ts vendored
View File

@ -1 +0,0 @@
declare const SETTINGS: SettingComponentProps[]

66
extensions/inference-triton-trtllm-extension/src/index.ts
View File

@ -1,66 +0,0 @@
/**
* @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package.
* The class provides methods for initializing and stopping a model, and for making inference requests.
* It also subscribes to events emitted by the @janhq/core package and handles new message requests.
* @version 1.0.0
* @module inference-nvidia-triton-trt-llm-extension/src/index
*/
import { RemoteOAIEngine } from '@janhq/core'
enum Settings {
apiKey = 'tritonllm-api-key',
chatCompletionsEndPoint = 'chat-completions-endpoint',
}
/**
* A class that implements the InferenceExtension interface from the @janhq/core package.
* The class provides methods for initializing and stopping a model, and for making inference requests.
* It also subscribes to events emitted by the @janhq/core package and handles new message requests.
*/
export default class JanInferenceTritonTrtLLMExtension extends RemoteOAIEngine {
inferenceUrl: string = ''
provider: string = 'triton_trtllm'
/**
* Subscribes to events emitted by the @janhq/core package.
*/
async onLoad() {
super.onLoad()
// Register Settings
this.registerSettings(SETTINGS)
// Retrieve API Key Setting
this.apiKey = await this.getSetting<string>(Settings.apiKey, '')
this.inferenceUrl = await this.getSetting<string>(
Settings.chatCompletionsEndPoint,
''
)
if (this.inferenceUrl.length === 0) {
SETTINGS.forEach((setting) => {
if (setting.key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = setting.controllerProps.value as string
}
})
}
}
onSettingUpdate<T>(key: string, value: T): void {
if (key === Settings.apiKey) {
this.apiKey = value as string
} else if (key === Settings.chatCompletionsEndPoint) {
if (typeof value !== 'string') return
if (value.trim().length === 0) {
SETTINGS.forEach((setting) => {
if (setting.key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = setting.controllerProps.value as string
}
})
} else {
this.inferenceUrl = value
}
}
}
}

Some files were not shown because too many files have changed in this diff Show More