Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: deprecate Jan core's REST module - all API requests go to cortex.cpp (#4297)

Browse Source
This commit is contained in:
Louis 2024-12-19 21:37:37 +07:00 committed by GitHub
parent 0271774773
commit 4489af6ad9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
41 changed files with 6338 additions and 5801 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
core/src/node/api/HttpServer.ts
View File

@ -1,8 +0,0 @@
export interface HttpServer {
post: (route: string, handler: (req: any, res: any) => Promise<any>) => void
get: (route: string, handler: (req: any, res: any) => Promise<any>) => void
patch: (route: string, handler: (req: any, res: any) => Promise<any>) => void
put: (route: string, handler: (req: any, res: any) => Promise<any>) => void
delete: (route: string, handler: (req: any, res: any) => Promise<any>) => void
register: (router: any, opts?: any) => void
}

7
core/src/node/api/index.test.ts
View File

@ -1,7 +0,0 @@
import * as restfulV1 from './restful/v1';
it('should re-export from restful/v1', () => {
const restfulV1Exports = require('./restful/v1');
expect(restfulV1Exports).toBeDefined();
})

2
core/src/node/api/index.ts
View File

@ -1,3 +1 @@
export * from './HttpServer'
export * from './restful/v1'
export * from './common/handler' export * from './common/handler'

1
core/src/node/api/processors/app.test.ts
View File

@ -2,7 +2,6 @@ jest.mock('../../helper', () => ({
...jest.requireActual('../../helper'), ...jest.requireActual('../../helper'),
getJanDataFolderPath: () => './app', getJanDataFolderPath: () => './app',
})) }))
import { dirname } from 'path'
import { App } from './app' import { App } from './app'
it('should call stopServer', () => { it('should call stopServer', () => {

3
core/src/node/api/processors/app.ts
View File

@ -3,7 +3,6 @@ import { basename, dirname, isAbsolute, join, relative } from 'path'
import { Processor } from './Processor' import { Processor } from './Processor'
import { import {
log as writeLog, log as writeLog,
appResourcePath,
getAppConfigurations as appConfiguration, getAppConfigurations as appConfiguration,
updateAppConfiguration, updateAppConfiguration,
normalizeFilePath, normalizeFilePath,
@ -91,8 +90,6 @@ export class App implements Processor {
port: args?.port, port: args?.port,
isCorsEnabled: args?.isCorsEnabled, isCorsEnabled: args?.isCorsEnabled,
isVerboseEnabled: args?.isVerboseEnabled, isVerboseEnabled: args?.isVerboseEnabled,
schemaPath: join(appResourcePath(), 'docs', 'openapi', 'jan.yaml'),
baseDir: join(appResourcePath(), 'docs', 'openapi'),
prefix: args?.prefix, prefix: args?.prefix,
}) })
} }

62
core/src/node/api/restful/app/download.test.ts
View File

@ -1,62 +0,0 @@
import { HttpServer } from '../../HttpServer'
import { DownloadManager } from '../../../helper/download'
describe('downloadRouter', () => {
let app: HttpServer
beforeEach(() => {
app = {
register: jest.fn(),
post: jest.fn(),
get: jest.fn(),
patch: jest.fn(),
put: jest.fn(),
delete: jest.fn(),
}
})
it('should return download progress for a given modelId', async () => {
const modelId = '123'
const downloadProgress = { progress: 50 }
DownloadManager.instance.downloadProgressMap[modelId] = downloadProgress as any
const req = { params: { modelId } }
const res = {
status: jest.fn(),
send: jest.fn(),
}
jest.spyOn(app, 'get').mockImplementation((path, handler) => {
if (path === `/download/getDownloadProgress/${modelId}`) {
res.status(200)
res.send(downloadProgress)
}
})
app.get(`/download/getDownloadProgress/${modelId}`, req as any)
expect(res.status).toHaveBeenCalledWith(200)
expect(res.send).toHaveBeenCalledWith(downloadProgress)
})
it('should return 404 if download progress is not found', async () => {
const modelId = '123'
const req = { params: { modelId } }
const res = {
status: jest.fn(),
send: jest.fn(),
}
jest.spyOn(app, 'get').mockImplementation((path, handler) => {
if (path === `/download/getDownloadProgress/${modelId}`) {
res.status(404)
res.send({ message: 'Download progress not found' })
}
})
app.get(`/download/getDownloadProgress/${modelId}`, req as any)
expect(res.status).toHaveBeenCalledWith(404)
expect(res.send).toHaveBeenCalledWith({ message: 'Download progress not found' })
})
})

23
core/src/node/api/restful/app/download.ts
View File

@ -1,23 +0,0 @@
import { DownloadRoute } from '../../../../types/api'
import { DownloadManager } from '../../../helper/download'
import { HttpServer } from '../../HttpServer'
export const downloadRouter = async (app: HttpServer) => {
app.get(`/download/${DownloadRoute.getDownloadProgress}/:modelId`, async (req, res) => {
const modelId = req.params.modelId
console.debug(`Getting download progress for model ${modelId}`)
console.debug(
`All Download progress: ${JSON.stringify(DownloadManager.instance.downloadProgressMap)}`
)
// check if null DownloadManager.instance.downloadProgressMap
if (!DownloadManager.instance.downloadProgressMap[modelId]) {
return res.status(404).send({
message: 'Download progress not found',
})
} else {
return res.status(200).send(DownloadManager.instance.downloadProgressMap[modelId])
}
})
}

16
core/src/node/api/restful/app/handlers.test.ts
View File

@ -1,16 +0,0 @@
//
import { jest } from '@jest/globals';
import { HttpServer } from '../../HttpServer';
import { handleRequests } from './handlers';
import { Handler, RequestHandler } from '../../common/handler';
it('should initialize RequestHandler and call handle', () => {
const mockHandle = jest.fn();
jest.spyOn(RequestHandler.prototype, 'handle').mockImplementation(mockHandle);
const mockApp = { post: jest.fn() };
handleRequests(mockApp as unknown as HttpServer);
expect(mockHandle).toHaveBeenCalled();
});

13
core/src/node/api/restful/app/handlers.ts
View File

@ -1,13 +0,0 @@
import { HttpServer } from '../../HttpServer'
import { Handler, RequestHandler } from '../../common/handler'
export function handleRequests(app: HttpServer) {
const restWrapper: Handler = (route: string, listener: (...args: any[]) => any) => {
app.post(`/app/${route}`, async (request: any, reply: any) => {
const args = JSON.parse(request.body) as any[]
reply.send(JSON.stringify(await listener(...args)))
})
}
const handler = new RequestHandler(restWrapper)
handler.handle()
}

21
core/src/node/api/restful/common.test.ts
View File

@ -1,21 +0,0 @@
import { commonRouter } from './common';
import { JanApiRouteConfiguration } from './helper/configuration';
test('commonRouter sets up routes for each key in JanApiRouteConfiguration', async () => {
const mockHttpServer = {
get: jest.fn(),
post: jest.fn(),
patch: jest.fn(),
put: jest.fn(),
delete: jest.fn(),
};
await commonRouter(mockHttpServer as any);
const expectedRoutes = Object.keys(JanApiRouteConfiguration);
expectedRoutes.forEach((key) => {
expect(mockHttpServer.get).toHaveBeenCalledWith(`/${key}`, expect.any(Function));
expect(mockHttpServer.get).toHaveBeenCalledWith(`/${key}/:id`, expect.any(Function));
expect(mockHttpServer.delete).toHaveBeenCalledWith(`/${key}/:id`, expect.any(Function));
});
});

82
core/src/node/api/restful/common.ts
View File

@ -1,82 +0,0 @@
import { HttpServer } from '../HttpServer'
import {
chatCompletions,
downloadModel,
getBuilder,
retrieveBuilder,
createMessage,
createThread,
getMessages,
retrieveMessage,
updateThread,
models,
} from './helper/builder'
import { JanApiRouteConfiguration } from './helper/configuration'
export const commonRouter = async (app: HttpServer) => {
const normalizeData = (data: any) => {
return {
object: 'list',
data,
}
}
// Common Routes
// Read & Delete :: Threads | Models | Assistants
Object.keys(JanApiRouteConfiguration).forEach((key) => {
app.get(`/${key}`, async (_req, _res) => {
if (key.includes('models')) {
return models(_req, _res)
}
return getBuilder(JanApiRouteConfiguration[key]).then(normalizeData)
})
app.get(`/${key}/:id`, async (_req: any, _res: any) => {
if (key.includes('models')) {
return models(_req, _res)
}
return retrieveBuilder(JanApiRouteConfiguration[key], _req.params.id)
})
app.delete(`/${key}/:id`, async (_req: any, _res: any) => {
if (key.includes('models')) {
return models(_req, _res)
}
return retrieveBuilder(JanApiRouteConfiguration[key], _req.params.id)
})
})
// Threads
app.post(`/threads`, async (req, res) => createThread(req.body))
app.get(`/threads/:threadId/messages`, async (req, res) =>
getMessages(req.params.threadId).then(normalizeData)
)
app.get(`/threads/:threadId/messages/:messageId`, async (req, res) =>
retrieveMessage(req.params.threadId, req.params.messageId)
)
app.post(`/threads/:threadId/messages`, async (req, res) =>
createMessage(req.params.threadId as any, req.body as any)
)
app.patch(`/threads/:threadId`, async (request: any) =>
updateThread(request.params.threadId, request.body)
)
// Models
app.get(`/models/download/:modelId`, async (request: any) =>
downloadModel(request.params.modelId, {
ignoreSSL: request.query.ignoreSSL === 'true',
proxy: request.query.proxy,
})
)
app.post(`/models/start`, async (request: any, reply: any) => models(request, reply))
app.post(`/models/stop`, async (request: any, reply: any) => models(request, reply))
// Chat Completion
app.post(`/chat/completions`, async (request: any, reply: any) => chatCompletions(request, reply))
}

251
core/src/node/api/restful/helper/builder.test.ts
View File

@ -1,251 +0,0 @@
import { existsSync, readdirSync, readFileSync, writeFileSync, mkdirSync, appendFileSync } from 'fs'
import {
getBuilder,
retrieveBuilder,
getMessages,
retrieveMessage,
createThread,
updateThread,
createMessage,
downloadModel,
chatCompletions,
} from './builder'
import { RouteConfiguration } from './configuration'
jest.mock('fs')
jest.mock('path')
jest.mock('../../../helper', () => ({
getEngineConfiguration: jest.fn(),
getJanDataFolderPath: jest.fn().mockReturnValue('/mock/path'),
}))
jest.mock('request')
jest.mock('request-progress')
jest.mock('node-fetch')
describe('builder helper functions', () => {
const mockConfiguration: RouteConfiguration = {
dirName: 'mockDir',
metadataFileName: 'metadata.json',
delete: {
object: 'mockObject',
},
}
beforeEach(() => {
jest.clearAllMocks()
})
describe('getBuilder', () => {
it('should return an empty array if directory does not exist', async () => {
;(existsSync as jest.Mock).mockReturnValue(false)
const result = await getBuilder(mockConfiguration)
expect(result).toEqual([])
})
it('should return model data if directory exists', async () => {
;(existsSync as jest.Mock).mockReturnValue(true)
;(readdirSync as jest.Mock).mockReturnValue(['file1'])
;(readFileSync as jest.Mock).mockReturnValue(JSON.stringify({ id: 'model1' }))
const result = await getBuilder(mockConfiguration)
expect(result).toEqual([{ id: 'model1' }])
})
})
describe('retrieveBuilder', () => {
it('should return undefined if no data matches the id', async () => {
;(existsSync as jest.Mock).mockReturnValue(true)
;(readdirSync as jest.Mock).mockReturnValue(['file1'])
;(readFileSync as jest.Mock).mockReturnValue(JSON.stringify({ id: 'model1' }))
const result = await retrieveBuilder(mockConfiguration, 'nonexistentId')
expect(result).toBeUndefined()
})
it('should return the matching data', async () => {
;(existsSync as jest.Mock).mockReturnValue(true)
;(readdirSync as jest.Mock).mockReturnValue(['file1'])
;(readFileSync as jest.Mock).mockReturnValue(JSON.stringify({ id: 'model1' }))
const result = await retrieveBuilder(mockConfiguration, 'model1')
expect(result).toEqual({ id: 'model1' })
})
})
describe('getMessages', () => {
it('should return an empty array if message file does not exist', async () => {
;(existsSync as jest.Mock).mockReturnValue(false)
const result = await getMessages('thread1')
expect(result).toEqual([])
})
it('should return messages if message file exists', async () => {
;(existsSync as jest.Mock).mockReturnValue(true)
;(readdirSync as jest.Mock).mockReturnValue(['messages.jsonl'])
;(readFileSync as jest.Mock).mockReturnValue('{"id":"msg1"}\n{"id":"msg2"}\n')
const result = await getMessages('thread1')
expect(result).toEqual([{ id: 'msg1' }, { id: 'msg2' }])
})
})
describe('retrieveMessage', () => {
it('should return a message if no messages match the id', async () => {
;(existsSync as jest.Mock).mockReturnValue(true)
;(readdirSync as jest.Mock).mockReturnValue(['messages.jsonl'])
;(readFileSync as jest.Mock).mockReturnValue('{"id":"msg1"}\n')
const result = await retrieveMessage('thread1', 'nonexistentId')
expect(result).toEqual({ message: 'Not found' })
})
it('should return the matching message', async () => {
;(existsSync as jest.Mock).mockReturnValue(true)
;(readdirSync as jest.Mock).mockReturnValue(['messages.jsonl'])
;(readFileSync as jest.Mock).mockReturnValue('{"id":"msg1"}\n')
const result = await retrieveMessage('thread1', 'msg1')
expect(result).toEqual({ id: 'msg1' })
})
})
describe('createThread', () => {
it('should return a message if thread has no assistants', async () => {
const result = await createThread({})
expect(result).toEqual({ message: 'Thread must have at least one assistant' })
})
it('should create a thread and return the updated thread', async () => {
;(existsSync as jest.Mock).mockReturnValue(false)
const thread = { assistants: [{ assistant_id: 'assistant1' }] }
const result = await createThread(thread)
expect(mkdirSync).toHaveBeenCalled()
expect(writeFileSync).toHaveBeenCalled()
expect(result.id).toBeDefined()
})
})
describe('updateThread', () => {
it('should return a message if thread is not found', async () => {
;(existsSync as jest.Mock).mockReturnValue(true)
;(readdirSync as jest.Mock).mockReturnValue(['file1'])
;(readFileSync as jest.Mock).mockReturnValue(JSON.stringify({ id: 'model1' }))
const result = await updateThread('nonexistentId', {})
expect(result).toEqual({ message: 'Thread not found' })
})
it('should update the thread and return the updated thread', async () => {
;(existsSync as jest.Mock).mockReturnValue(true)
;(readdirSync as jest.Mock).mockReturnValue(['file1'])
;(readFileSync as jest.Mock).mockReturnValue(JSON.stringify({ id: 'model1' }))
const result = await updateThread('model1', { name: 'updatedName' })
expect(writeFileSync).toHaveBeenCalled()
expect(result.name).toEqual('updatedName')
})
})
describe('createMessage', () => {
it('should create a message and return the created message', async () => {
;(existsSync as jest.Mock).mockReturnValue(false)
const message = { role: 'user', content: 'Hello' }
const result = (await createMessage('thread1', message)) as any
expect(mkdirSync).toHaveBeenCalled()
expect(appendFileSync).toHaveBeenCalled()
expect(result.id).toBeDefined()
})
})
describe('downloadModel', () => {
it('should return a message if model is not found', async () => {
;(existsSync as jest.Mock).mockReturnValue(true)
;(readdirSync as jest.Mock).mockReturnValue(['file1'])
;(readFileSync as jest.Mock).mockReturnValue(JSON.stringify({ id: 'model1' }))
const result = await downloadModel('nonexistentId')
expect(result).toEqual({ message: 'Model not found' })
})
it('should start downloading the model', async () => {
;(existsSync as jest.Mock).mockReturnValue(true)
;(readdirSync as jest.Mock).mockReturnValue(['file1'])
;(readFileSync as jest.Mock).mockReturnValue(
JSON.stringify({ id: 'model1', object: 'model', sources: ['http://example.com'] })
)
const result = await downloadModel('model1')
expect(result).toEqual({ message: 'Starting download model1' })
})
})
describe('chatCompletions', () => {
it('should return the error on status not ok', async () => {
const request = { body: { model: 'model1' } }
const mockSend = jest.fn()
const reply = {
code: jest.fn().mockReturnThis(),
send: jest.fn(),
headers: jest.fn().mockReturnValue({
send: mockSend,
}),
raw: {
writeHead: jest.fn(),
pipe: jest.fn(),
},
}
;(existsSync as jest.Mock).mockReturnValue(true)
;(readdirSync as jest.Mock).mockReturnValue(['file1'])
;(readFileSync as jest.Mock).mockReturnValue(
JSON.stringify({ id: 'model1', engine: 'openai' })
)
// Mock fetch
const fetch = require('node-fetch')
fetch.mockResolvedValue({
status: 400,
headers: new Map([
['content-type', 'application/json'],
['x-request-id', '123456'],
]),
body: { pipe: jest.fn() },
text: jest.fn().mockResolvedValue({ error: 'Mock error response' }),
})
await chatCompletions(request, reply)
expect(reply.code).toHaveBeenCalledWith(400)
expect(mockSend).toHaveBeenCalledWith(
expect.objectContaining({
error: 'Mock error response',
})
)
})
it('should return the chat completions', async () => {
const request = { body: { model: 'model1' } }
const reply = {
code: jest.fn().mockReturnThis(),
send: jest.fn(),
raw: { writeHead: jest.fn(), pipe: jest.fn() },
}
;(existsSync as jest.Mock).mockReturnValue(true)
;(readdirSync as jest.Mock).mockReturnValue(['file1'])
;(readFileSync as jest.Mock).mockReturnValue(
JSON.stringify({ id: 'model1', engine: 'openai' })
)
// Mock fetch
const fetch = require('node-fetch')
fetch.mockResolvedValue({
status: 200,
body: { pipe: jest.fn() },
json: jest.fn().mockResolvedValue({ completions: ['completion1'] }),
})
await chatCompletions(request, reply)
expect(reply.raw.writeHead).toHaveBeenCalledWith(200, expect.any(Object))
})
})
})

339
core/src/node/api/restful/helper/builder.ts
View File

@ -1,339 +0,0 @@
import {
existsSync,
readdirSync,
readFileSync,
writeFileSync,
mkdirSync,
appendFileSync,
createWriteStream,
} from 'fs'
import { JanApiRouteConfiguration, RouteConfiguration } from './configuration'
import { join } from 'path'
import { ContentType, InferenceEngine, MessageStatus, ThreadMessage } from '../../../../types'
import { getJanDataFolderPath } from '../../../helper'
import { CORTEX_API_URL } from './consts'
// TODO: Refactor these
export const getBuilder = async (configuration: RouteConfiguration) => {
const directoryPath = join(getJanDataFolderPath(), configuration.dirName)
try {
if (!existsSync(directoryPath)) {
console.debug('model folder not found')
return []
}
const files: string[] = readdirSync(directoryPath)
const allDirectories: string[] = []
for (const file of files) {
if (file === '.DS_Store') continue
allDirectories.push(file)
}
const results = allDirectories
.map((dirName) => {
const jsonPath = join(directoryPath, dirName, configuration.metadataFileName)
return readModelMetadata(jsonPath)
})
.filter((data) => !!data)
const modelData = results
.map((result: any) => {
try {
return JSON.parse(result)
} catch (err) {
console.error(err)
}
})
.filter((e: any) => !!e)
return modelData
} catch (err) {
console.error(err)
return []
}
}
const readModelMetadata = (path: string): string | undefined => {
if (existsSync(path)) {
return readFileSync(path, 'utf-8')
} else {
return undefined
}
}
export const retrieveBuilder = async (configuration: RouteConfiguration, id: string) => {
const data = await getBuilder(configuration)
const filteredData = data.filter((d: any) => d.id === id)[0]
if (!filteredData) {
return undefined
}
return filteredData
}
export const getMessages = async (threadId: string): Promise<ThreadMessage[]> => {
const threadDirPath = join(getJanDataFolderPath(), 'threads', threadId)
const messageFile = 'messages.jsonl'
try {
const files: string[] = readdirSync(threadDirPath)
if (!files.includes(messageFile)) {
console.error(`${threadDirPath} not contains message file`)
return []
}
const messageFilePath = join(threadDirPath, messageFile)
if (!existsSync(messageFilePath)) {
console.debug('message file not found')
return []
}
const lines = readFileSync(messageFilePath, 'utf-8')
.toString()
.split('\n')
.filter((line: any) => line !== '')
const messages: ThreadMessage[] = []
lines.forEach((line: string) => {
messages.push(JSON.parse(line) as ThreadMessage)
})
return messages
} catch (err) {
console.error(err)
return []
}
}
export const retrieveMessage = async (threadId: string, messageId: string) => {
const messages = await getMessages(threadId)
const filteredMessages = messages.filter((m) => m.id === messageId)
if (!filteredMessages || filteredMessages.length === 0) {
return {
message: 'Not found',
}
}
return filteredMessages[0]
}
export const createThread = async (thread: any) => {
const threadMetadataFileName = 'thread.json'
// TODO: add validation
if (!thread.assistants || thread.assistants.length === 0) {
return {
message: 'Thread must have at least one assistant',
}
}
const threadId = generateThreadId(thread.assistants[0]?.assistant_id)
try {
const updatedThread = {
...thread,
id: threadId,
created: Date.now(),
updated: Date.now(),
}
const threadDirPath = join(getJanDataFolderPath(), 'threads', updatedThread.id)
const threadJsonPath = join(threadDirPath, threadMetadataFileName)
if (!existsSync(threadDirPath)) {
mkdirSync(threadDirPath)
}
await writeFileSync(threadJsonPath, JSON.stringify(updatedThread, null, 2))
return updatedThread
} catch (err) {
return {
error: err,
}
}
}
export const updateThread = async (threadId: string, thread: any) => {
const threadMetadataFileName = 'thread.json'
const currentThreadData = await retrieveBuilder(JanApiRouteConfiguration.threads, threadId)
if (!currentThreadData) {
return {
message: 'Thread not found',
}
}
// we don't want to update the id and object
delete thread.id
delete thread.object
const updatedThread = {
...currentThreadData,
...thread,
updated: Date.now(),
}
try {
const threadDirPath = join(getJanDataFolderPath(), 'threads', updatedThread.id)
const threadJsonPath = join(threadDirPath, threadMetadataFileName)
await writeFileSync(threadJsonPath, JSON.stringify(updatedThread, null, 2))
return updatedThread
} catch (err) {
return {
message: err,
}
}
}
const generateThreadId = (assistantId: string) => {
return `${assistantId}_${(Date.now() / 1000).toFixed(0)}`
}
export const createMessage = async (threadId: string, message: any) => {
const threadMessagesFileName = 'messages.jsonl'
try {
const { ulid } = require('ulidx')
const msgId = ulid()
const createdAt = Date.now()
const threadMessage: ThreadMessage = {
id: msgId,
thread_id: threadId,
status: MessageStatus.Ready,
created_at: createdAt,
completed_at: createdAt,
object: 'thread.message',
role: message.role,
content: [
{
type: ContentType.Text,
text: {
value: message.content,
annotations: [],
},
},
],
}
const threadDirPath = join(getJanDataFolderPath(), 'threads', threadId)
const threadMessagePath = join(threadDirPath, threadMessagesFileName)
if (!existsSync(threadDirPath)) {
mkdirSync(threadDirPath)
}
appendFileSync(threadMessagePath, JSON.stringify(threadMessage) + '\n')
return threadMessage
} catch (err) {
return {
message: err,
}
}
}
export const downloadModel = async (
modelId: string,
network?: { proxy?: string; ignoreSSL?: boolean }
) => {
const strictSSL = !network?.ignoreSSL
const proxy = network?.proxy?.startsWith('http') ? network.proxy : undefined
const model = await retrieveBuilder(JanApiRouteConfiguration.models, modelId)
if (!model || model.object !== 'model') {
return {
message: 'Model not found',
}
}
const directoryPath = join(getJanDataFolderPath(), 'models', modelId)
if (!existsSync(directoryPath)) {
mkdirSync(directoryPath)
}
// path to model binary
const modelBinaryPath = join(directoryPath, modelId)
const request = require('request')
const progress = require('request-progress')
for (const source of model.sources) {
const rq = request({ url: source, strictSSL, proxy })
progress(rq, {})
?.on('progress', function (state: any) {
console.debug('progress', JSON.stringify(state, null, 2))
})
?.on('error', function (err: Error) {
console.error('error', err)
})
?.on('end', function () {
console.debug('end')
})
.pipe(createWriteStream(modelBinaryPath))
}
return {
message: `Starting download ${modelId}`,
}
}
/**
* Proxy /models to cortex
* @param request
* @param reply
*/
export const models = async (request: any, reply: any) => {
const fetch = require('node-fetch')
const headers: Record<string, any> = {
'Content-Type': 'application/json',
}
const response = await fetch(`${CORTEX_API_URL}/models${request.url.split('/models')[1] ?? ''}`, {
method: request.method,
headers: headers,
body: JSON.stringify(request.body),
})
if (response.status !== 200) {
// Forward the error response to client via reply
const responseBody = await response.text()
const responseHeaders = Object.fromEntries(response.headers)
reply.code(response.status).headers(responseHeaders).send(responseBody)
} else {
reply.raw.writeHead(200, {
'Content-Type': 'application/json',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
'Access-Control-Allow-Origin': '*',
})
response.body.pipe(reply.raw)
}
}
/**
* Proxy chat completions
* @param request
* @param reply
*/
export const chatCompletions = async (request: any, reply: any) => {
const headers: Record<string, any> = {
'Content-Type': 'application/json',
}
// add engine for new cortex cpp engine
if (request.body.engine === InferenceEngine.nitro) {
request.body.engine = InferenceEngine.cortex_llamacpp
}
const fetch = require('node-fetch')
const response = await fetch(`${CORTEX_API_URL}/chat/completions`, {
method: 'POST',
headers: headers,
body: JSON.stringify(request.body),
})
if (response.status !== 200) {
// Forward the error response to client via reply
const responseBody = await response.text()
const responseHeaders = Object.fromEntries(response.headers)
reply.code(response.status).headers(responseHeaders).send(responseBody)
} else {
reply.raw.writeHead(200, {
'Content-Type': request.body.stream === true ? 'text/event-stream' : 'application/json',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
'Access-Control-Allow-Origin': '*',
})
response.body.pipe(reply.raw)
}
}

24
core/src/node/api/restful/helper/configuration.test.ts
View File

@ -1,24 +0,0 @@
import { JanApiRouteConfiguration } from './configuration'
describe('JanApiRouteConfiguration', () => {
it('should have the correct models configuration', () => {
const modelsConfig = JanApiRouteConfiguration.models;
expect(modelsConfig.dirName).toBe('models');
expect(modelsConfig.metadataFileName).toBe('model.json');
expect(modelsConfig.delete.object).toBe('model');
});
it('should have the correct assistants configuration', () => {
const assistantsConfig = JanApiRouteConfiguration.assistants;
expect(assistantsConfig.dirName).toBe('assistants');
expect(assistantsConfig.metadataFileName).toBe('assistant.json');
expect(assistantsConfig.delete.object).toBe('assistant');
});
it('should have the correct threads configuration', () => {
const threadsConfig = JanApiRouteConfiguration.threads;
expect(threadsConfig.dirName).toBe('threads');
expect(threadsConfig.metadataFileName).toBe('thread.json');
expect(threadsConfig.delete.object).toBe('thread');
});
});

31
core/src/node/api/restful/helper/configuration.ts
View File

@ -1,31 +0,0 @@
export const JanApiRouteConfiguration: Record<string, RouteConfiguration> = {
models: {
dirName: 'models',
metadataFileName: 'model.json',
delete: {
object: 'model',
},
},
assistants: {
dirName: 'assistants',
metadataFileName: 'assistant.json',
delete: {
object: 'assistant',
},
},
threads: {
dirName: 'threads',
metadataFileName: 'thread.json',
delete: {
object: 'thread',
},
},
}
export type RouteConfiguration = {
dirName: string
metadataFileName: string
delete: {
object: string
}
}

5
core/src/node/api/restful/helper/consts.test.ts
View File

@ -1,5 +0,0 @@
import { CORTEX_DEFAULT_PORT } from './consts'
it('should test CORTEX_DEFAULT_PORT', () => {
expect(CORTEX_DEFAULT_PORT).toBe(39291)
})

7
core/src/node/api/restful/helper/consts.ts
View File

@ -1,7 +0,0 @@
export const CORTEX_DEFAULT_PORT = 39291
export const LOCAL_HOST = '127.0.0.1'
export const SUPPORTED_MODEL_FORMAT = '.gguf'
export const CORTEX_API_URL = `http://${LOCAL_HOST}:${CORTEX_DEFAULT_PORT}/v1`

16
core/src/node/api/restful/v1.test.ts
View File

@ -1,16 +0,0 @@
import { v1Router } from './v1';
import { commonRouter } from './common';
test('should define v1Router function', () => {
expect(v1Router).toBeDefined();
});
test('should register commonRouter', () => {
const mockApp = {
register: jest.fn(),
};
v1Router(mockApp as any);
expect(mockApp.register).toHaveBeenCalledWith(commonRouter);
});

16
core/src/node/api/restful/v1.ts
View File

@ -1,16 +0,0 @@
import { HttpServer } from '../HttpServer'
import { commonRouter } from './common'
export const v1Router = async (app: HttpServer) => {
// MARK: Public API Routes
app.register(commonRouter)
// MARK: Internal Application Routes
// DEPRECATED: Vulnerability possible issues
// handleRequests(app)
// Expanded route for tracking download progress
// TODO: Replace by Observer Wrapper (ZeroMQ / Vanilla Websocket)
// DEPRECATED: Jan FE Docker deploy is deprecated
// app.register(downloadRouter)
}

4
core/src/node/helper/path.ts
View File

@ -1,5 +1,4 @@
import { join, resolve } from 'path' import { join } from 'path'
import { getJanDataFolderPath } from './config'
/** /**
* Normalize file path * Normalize file path
@ -35,3 +34,4 @@ export function appResourcePath() {
// server // server
return join(global.core.appPath(), '../../..') return join(global.core.appPath(), '../../..')
} }

3
core/tsconfig.json
View File

@ -13,7 +13,8 @@
"declarationDir": "dist/types", "declarationDir": "dist/types",
"outDir": "dist/lib", "outDir": "dist/lib",
"importHelpers": true, "importHelpers": true,
"types": ["@types/jest"] "types": ["@types/jest"],
"resolveJsonModule": true
}, },
"include": ["src"], "include": ["src"],
"exclude": ["**/*.test.ts"] "exclude": ["**/*.test.ts"]

0
docs/openapi/.gitkeep
View File

2397
docs/openapi/jan.json
View File

File diff suppressed because it is too large Load Diff

1043
docs/openapi/jan.yaml
View File

File diff suppressed because it is too large Load Diff

319
docs/openapi/specs/assistants.yaml
View File

@ -1,319 +0,0 @@
---
components:
schemas:
AssistantObject:
type: object
properties:
id:
type: string
description: The identifier of the assistant.
example: asst_abc123
object:
type: string
description: Type of the object, indicating it's an assistant.
default: assistant
version:
type: integer
description: Version number of the assistant.
example: 1
created_at:
type: integer
format: int64
description: Unix timestamp representing the creation time of the assistant.
example: 1698984975
name:
type: string
description: Name of the assistant.
example: Math Tutor
description:
type: string
description: Description of the assistant. Can be null.
example: null
avatar:
type: string
description: URL of the assistant's avatar. Jan-specific property.
example: https://pic.png
models:
type: array
description: List of models associated with the assistant. Jan-specific property.
items:
type: object
properties:
model_id:
type: string
example: model_0
instructions:
type: string
description: A system prompt for the assistant.
example: Be concise
events:
type: object
description: Event subscription settings for the assistant.
properties:
in:
type: array
items:
type: string
out:
type: array
items:
type: string
metadata:
type: object
description: Metadata associated with the assistant.
ListAssistantsResponse: null
CreateAssistantResponse:
type: object
properties:
id:
type: string
description: The identifier of the assistant.
example: asst_abc123
object:
type: string
description: Type of the object, indicating it's an assistant.
default: assistant
version:
type: integer
description: Version number of the assistant.
example: 1
created_at:
type: integer
format: int64
description: Unix timestamp representing the creation time of the assistant.
example: 1698984975
name:
type: string
description: Name of the assistant.
example: Math Tutor
description:
type: string
description: Description of the assistant. Can be null.
example: null
avatar:
type: string
description: URL of the assistant's avatar. Jan-specific property.
example: https://pic.png
models:
type: array
description: List of models associated with the assistant. Jan-specific property.
items:
type: object
properties:
model_id:
type: string
example: model_0
instructions:
type: string
description: A system prompt for the assistant.
example: Be concise
events:
type: object
description: Event subscription settings for the assistant.
properties:
in:
type: array
items:
type: string
out:
type: array
items:
type: string
metadata:
type: object
description: Metadata associated with the assistant.
RetrieveAssistantResponse:
type: object
properties:
id:
type: string
description: The identifier of the assistant.
example: asst_abc123
object:
type: string
description: Type of the object, indicating it's an assistant.
default: assistant
version:
type: integer
description: Version number of the assistant.
example: 1
created_at:
type: integer
format: int64
description: Unix timestamp representing the creation time of the assistant.
example: 1698984975
name:
type: string
description: Name of the assistant.
example: Math Tutor
description:
type: string
description: Description of the assistant. Can be null.
example: null
avatar:
type: string
description: URL of the assistant's avatar. Jan-specific property.
example: https://pic.png
models:
type: array
description: List of models associated with the assistant. Jan-specific property.
items:
type: object
properties:
model_id:
type: string
example: model_0
instructions:
type: string
description: A system prompt for the assistant.
example: Be concise
events:
type: object
description: Event subscription settings for the assistant.
properties:
in:
type: array
items:
type: string
out:
type: array
items:
type: string
metadata:
type: object
description: Metadata associated with the assistant.
ModifyAssistantObject:
type: object
properties:
id:
type: string
description: The identifier of the assistant.
example: asst_abc123
object:
type: string
description: Type of the object, indicating it's an assistant.
default: assistant
version:
type: integer
description: Version number of the assistant.
example: 1
created_at:
type: integer
format: int64
description: Unix timestamp representing the creation time of the assistant.
example: 1698984975
name:
type: string
description: Name of the assistant.
example: Math Tutor
description:
type: string
description: Description of the assistant. Can be null.
example: null
avatar:
type: string
description: URL of the assistant's avatar. Jan-specific property.
example: https://pic.png
models:
type: array
description: List of models associated with the assistant. Jan-specific property.
items:
type: object
properties:
model_id:
type: string
example: model_0
instructions:
type: string
description: A system prompt for the assistant.
example: Be concise
events:
type: object
description: Event subscription settings for the assistant.
properties:
in:
type: array
items:
type: string
out:
type: array
items:
type: string
metadata:
type: object
description: Metadata associated with the assistant.
ModifyAssistantResponse:
type: object
properties:
id:
type: string
description: The identifier of the assistant.
example: asst_abc123
object:
type: string
description: Type of the object, indicating it's an assistant.
default: assistant
version:
type: integer
description: Version number of the assistant.
example: 1
created_at:
type: integer
format: int64
description: Unix timestamp representing the creation time of the assistant.
example: 1698984975
name:
type: string
description: Name of the assistant.
example: Physics Tutor
description:
type: string
description: Description of the assistant. Can be null.
example: null
avatar:
type: string
description: URL of the assistant's avatar. Jan-specific property.
example: https://pic.png
models:
type: array
description: List of models associated with the assistant. Jan-specific property.
items:
type: object
properties:
model_id:
type: string
example: model_0
instructions:
type: string
description: A system prompt for the assistant.
example: Be concise!
events:
type: object
description: Event subscription settings for the assistant.
properties:
in:
type: array
items:
type: string
out:
type: array
items:
type: string
metadata:
type: object
description: Metadata associated with the assistant.
DeleteAssistantResponse:
type: object
properties:
id:
type: string
description: The identifier of the deleted assistant.
example: asst_abc123
object:
type: string
description: Type of the object, indicating the assistant has been deleted.
example: assistant.deleted
deleted:
type: boolean
description: Indicates whether the assistant was successfully deleted.
example: true

196
docs/openapi/specs/chat.yaml
View File

@ -1,196 +0,0 @@
---
components:
schemas:
ChatObject:
type: object
properties:
messages:
type: arrays
description: |
Contains input data or prompts for the model to process.
example:
- content: 'Hello there :wave:'
role: assistant
- content: Can you write a long story
role: user
stream:
type: boolean
default: true
description:
Enables continuous output generation, allowing for streaming of
model responses.
model:
type: string
example: gpt-3.5-turbo
description: Specifies the model being used for inference or processing tasks.
max_tokens:
type: number
default: 2048
description:
The maximum number of tokens the model will generate in a single
response.
stop:
type: arrays
example:
- hello
description:
Defines specific tokens or phrases at which the model will stop
generating further output/
frequency_penalty:
type: number
default: 0
description:
Adjusts the likelihood of the model repeating words or phrases in
its output.
presence_penalty:
type: number
default: 0
description:
Influences the generation of new and varied concepts in the model's
output.
temperature:
type: number
default: 0.7
min: 0
max: 1
description: Controls the randomness of the model's output.
top_p:
type: number
default: 0.95
min: 0
max: 1
description: Set probability threshold for more relevant outputs.
cache_prompt:
type: boolean
default: true
description: Optimize performance in repeated or similar requests.
ChatCompletionRequest:
type: object
properties:
messages:
type: arrays
description: |
Contains input data or prompts for the model to process.
example:
- content: You are a helpful assistant.
role: system
- content: Hello!
role: user
model:
type: string
example: tinyllama-1.1b
description: |
Specifies the model being used for inference or processing tasks.
stream:
type: boolean
default: true
description: >
Enables continuous output generation, allowing for streaming of
model responses.
max_tokens:
type: number
default: 2048
description: >
The maximum number of tokens the model will generate in a single
response.
stop:
type: arrays
example:
- hello
description: >
Defines specific tokens or phrases at which the model will stop
generating further output.
frequency_penalty:
type: number
default: 0
description: >
Adjusts the likelihood of the model repeating words or phrases in
its output.
presence_penalty:
type: number
default: 0
description: >
Influences the generation of new and varied concepts in the model's
output.
temperature:
type: number
default: 0.7
min: 0
max: 1
description: |
Controls the randomness of the model's output.
top_p:
type: number
default: 0.95
min: 0
max: 1
description: |
Set probability threshold for more relevant outputs.
ChatCompletionResponse:
type: object
description: Description of the response structure
properties:
choices:
type: array
description: Array of choice objects
items:
type: object
properties:
finish_reason:
type: string
nullable: true
example: null
description: Reason for finishing the response, if applicable
index:
type: integer
example: 0
description: Index of the choice
message:
type: object
properties:
content:
type: string
example: Hello user. What can I help you with?
description: Content of the message
role:
type: string
example: assistant
description: Role of the sender
created:
type: integer
example: 1700193928
description: Timestamp of when the response was created
id:
type: string
example: ebwd2niJvJB1Q2Whyvkz
description: Unique identifier of the response
model:
type: string
nullable: true
example: _
description: Model used for generating the response
object:
type: string
example: chat.completion
description: Type of the response object
system_fingerprint:
type: string
nullable: true
example: _
description: System fingerprint
usage:
type: object
description: Information about the usage of tokens
properties:
completion_tokens:
type: integer
example: 500
description: Number of tokens used for completion
prompt_tokens:
type: integer
example: 33
description: Number of tokens used in the prompt
total_tokens:
type: integer
example: 533
description: Total number of tokens used

313
docs/openapi/specs/messages.yaml
View File

@ -1,313 +0,0 @@
---
components:
schemas:
MessageObject:
type: object
properties:
id:
type: string
description: |
Sequential or UUID identifier of the message.
example: 0
object:
type: string
description: |
Type of the object, defaults to 'thread.message'.
example: thread.message
created_at:
type: integer
format: int64
description: |
Unix timestamp representing the creation time of the message.
thread_id:
type: string
description: >
Identifier of the thread to which this message belongs. Defaults to
parent thread.
example: thread_asdf
assistant_id:
type: string
description: >
Identifier of the assistant involved in the message. Defaults to
parent thread.
example: jan
role:
type: string
enum:
- user
- assistant
description: |
Role of the sender, either 'user' or 'assistant'.
content:
type: array
items:
type: object
properties:
type:
type: string
description: |
Type of content, e.g., 'text'.
text:
type: object
properties:
value:
type: string
description: |
Text content of the message.
example: Hi!?
annotations:
type: array
items:
type: string
description: |
Annotations for the text content, if any.
example: []
metadata:
type: object
description: |
Metadata associated with the message, defaults to an empty object.
example: {}
GetMessageResponse:
type: object
properties:
id:
type: string
description: The identifier of the message.
example: msg_abc123
object:
type: string
description: Type of the object, indicating it's a thread message.
default: thread.message
created_at:
type: integer
format: int64
description: Unix timestamp representing the creation time of the message.
example: 1699017614
thread_id:
type: string
description: Identifier of the thread to which this message belongs.
example: thread_abc123
role:
type: string
description: Role of the sender, either 'user' or 'assistant'.
example: user
content:
type: array
items:
type: object
properties:
type:
type: string
description: Type of content, e.g., 'text'.
example: text
text:
type: object
properties:
value:
type: string
description: Text content of the message.
example: How does AI work? Explain it in simple terms.
annotations:
type: array
items:
type: string
description: Annotations for the text content, if any.
example: []
file_ids:
type: array
items:
type: string
description: Array of file IDs associated with the message, if any.
example: []
assistant_id:
type: string
description: Identifier of the assistant involved in the message, if applicable.
example: null
run_id:
type: string
description: Run ID associated with the message, if applicable.
example: null
metadata:
type: object
description: Metadata associated with the message.
example: {}
CreateMessageResponse:
type: object
properties:
id:
type: string
description: The identifier of the created message.
example: msg_abc123
object:
type: string
description: Type of the object, indicating it's a thread message.
example: thread.message
created_at:
type: integer
format: int64
description: Unix timestamp representing the creation time of the message.
example: 1699017614
thread_id:
type: string
description: Identifier of the thread to which this message belongs.
example: thread_abc123
role:
type: string
description: Role of the sender, either 'user' or 'assistant'.
example: user
content:
type: array
items:
type: object
properties:
type:
type: string
description: Type of content, e.g., 'text'.
example: text
text:
type: object
properties:
value:
type: string
description: Text content of the message.
example: How does AI work? Explain it in simple terms.
annotations:
type: array
items:
type: string
description: Annotations for the text content, if any.
example: []
file_ids:
type: array
items:
type: string
description: Array of file IDs associated with the message, if any.
example: []
assistant_id:
type: string
description: Identifier of the assistant involved in the message, if applicable.
example: null
run_id:
type: string
description: Run ID associated with the message, if applicable.
example: null
metadata:
type: object
description: Metadata associated with the message.
example: {}
ListMessagesResponse:
type: object
properties:
object:
type: string
description: Type of the object, indicating it's a list.
default: list
data:
type: array
items:
$ref: '#/components/schemas/ListMessageObject'
first_id:
type: string
description: Identifier of the first message in the list.
example: msg_abc123
last_id:
type: string
description: Identifier of the last message in the list.
example: msg_abc456
has_more:
type: boolean
description: Indicates whether there are more messages to retrieve.
example: false
ListMessageObject:
type: object
properties:
id:
type: string
description: The identifier of the message.
example: msg_abc123
object:
type: string
description: Type of the object, indicating it's a thread message.
example: thread.message
created_at:
type: integer
format: int64
description: Unix timestamp representing the creation time of the message.
example: 1699017614
thread_id:
type: string
description: Identifier of the thread to which this message belongs.
example: thread_abc123
role:
type: string
description: Role of the sender, either 'user' or 'assistant'.
example: user
content:
type: array
items:
type: object
properties:
type:
type: string
description: Type of content, e.g., 'text'.
text:
type: object
properties:
value:
type: string
description: Text content of the message.
example: How does AI work? Explain it in simple terms.
annotations:
type: array
items:
type: string
description: Annotations for the text content, if any.
file_ids:
type: array
items:
type: string
description: Array of file IDs associated with the message, if any.
example: []
assistant_id:
type: string
description: Identifier of the assistant involved in the message, if applicable.
example: null
run_id:
type: string
description: Run ID associated with the message, if applicable.
example: null
metadata:
type: object
description: Metadata associated with the message.
example: {}
MessageFileObject:
type: object
properties:
id:
type: string
description: The identifier of the file.
example: file-abc123
object:
type: string
description: Type of the object, indicating it's a thread message file.
example: thread.message.file
created_at:
type: integer
format: int64
description: Unix timestamp representing the creation time of the file.
example: 1699061776
message_id:
type: string
description: Identifier of the message to which this file is associated.
example: msg_abc123
ListMessageFilesResponse:
type: object
properties:
object:
type: string
description: Type of the object, indicating it's a list.
default: list
data:
type: array
items:
$ref: '#/components/schemas/MessageFileObject'

259
docs/openapi/specs/models.yaml
View File

@ -1,259 +0,0 @@
---
components:
schemas:
ListModelsResponse:
type: object
properties:
object:
type: string
enum:
- list
data:
type: array
items:
$ref: '#/components/schemas/Model'
required:
- object
- data
Model:
type: object
properties:
source_url:
type: string
format: uri
description: URL to the source of the model.
example: https://huggingface.co/janhq/trinity-v1.2-GGUF/resolve/main/trinity-v1.2.Q4_K_M.gguf
id:
type: string
description:
Unique identifier used in chat-completions model_name, matches
folder name.
example: trinity-v1.2-7b
object:
type: string
example: model
name:
type: string
description: Name of the model.
example: Trinity-v1.2 7B Q4
version:
type: string
default: '1.0'
description: The version number of the model.
description:
type: string
description: Description of the model.
example:
Trinity is an experimental model merge using the Slerp method.
Recommended for daily assistance purposes.
format:
type: string
description: State format of the model, distinct from the engine.
example: gguf
settings:
type: object
properties:
ctx_len:
type: integer
description: Context length.
example: 4096
prompt_template:
type: string
example: "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant"
additionalProperties: false
parameters:
type: object
properties:
temperature:
example: 0.7
top_p:
example: 0.95
stream:
example: true
max_tokens:
example: 4096
stop:
example: []
frequency_penalty:
example: 0
presence_penalty:
example: 0
additionalProperties: false
metadata:
author:
type: string
example: Jan
tags:
example:
- 7B
- Merged
- Featured
size:
example: 4370000000,
cover:
example: https://raw.githubusercontent.com/janhq/jan/main/models/trinity-v1.2-7b/cover.png
engine:
example: nitro
ModelObject:
type: object
properties:
id:
type: string
description: |
The identifier of the model.
example: trinity-v1.2-7b
object:
type: string
description: |
The type of the object, indicating it's a model.
default: model
created:
type: integer
format: int64
description: |
Unix timestamp representing the creation time of the model.
example: 1253935178
owned_by:
type: string
description: |
The entity that owns the model.
example: _
GetModelResponse:
type: object
properties:
source_url:
type: string
format: uri
description: URL to the source of the model.
example: https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q4_K_M.gguf
id:
type: string
description:
Unique identifier used in chat-completions model_name, matches
folder name.
example: mistral-ins-7b-q4
object:
type: string
example: model
name:
type: string
description: Name of the model.
example: Mistral Instruct 7B Q4
version:
type: string
default: '1.0'
description: The version number of the model.
description:
type: string
description: Description of the model.
example:
Trinity is an experimental model merge using the Slerp method.
Recommended for daily assistance purposes.
format:
type: string
description: State format of the model, distinct from the engine.
example: gguf
settings:
type: object
properties:
ctx_len:
type: integer
description: Context length.
example: 4096
prompt_template:
type: string
example: '[INST] {prompt} [/INST]'
additionalProperties: false
parameters:
type: object
properties:
temperature:
example: 0.7
top_p:
example: 0.95
stream:
example: true
max_tokens:
example: 4096
stop:
example: []
frequency_penalty:
example: 0
presence_penalty:
example: 0
additionalProperties: false
metadata:
author:
type: string
example: MistralAI
tags:
example:
- 7B
- Featured
- Foundation Model
size:
example: 4370000000,
cover:
example: https://raw.githubusercontent.com/janhq/jan/main/models/mistral-ins-7b-q4/cover.png
engine:
example: nitro
DeleteModelResponse:
type: object
properties:
id:
type: string
description: The identifier of the model that was deleted.
example: mistral-ins-7b-q4
object:
type: string
description: Type of the object, indicating it's a model.
default: model
deleted:
type: boolean
description: Indicates whether the model was successfully deleted.
example: true
StartModelResponse:
type: object
properties:
id:
type: string
description: The identifier of the model that was started.
example: model-zephyr-7B
object:
type: string
description: Type of the object, indicating it's a model.
default: model
state:
type: string
description: The current state of the model after the start operation.
example: running
required:
- id
- object
- state
StopModelResponse:
type: object
properties:
id:
type: string
description: The identifier of the model that was started.
example: model-zephyr-7B
object:
type: string
description: Type of the object, indicating it's a model.
default: model
state:
type: string
description: The current state of the model after the start operation.
example: stopped
required:
- id
- object
- state
DownloadModelResponse:
type: object
properties:
message:
type: string
description: Message indicates Jan starting download corresponding model.
example: Starting download mistral-ins-7b-q4

227
docs/openapi/specs/threads.yaml
View File

@ -1,227 +0,0 @@
---
components:
schemas:
ThreadObject:
type: object
properties:
id:
type: string
description: |
The identifier of the thread, defaults to foldername.
example: thread_....
object:
type: string
description: |
Type of the object, defaults to thread.
example: thread
title:
type: string
description: >
A brief summary or description of the thread, defaults to an empty
string.
example: funny physics joke
assistants:
type: array
description: ''
items:
properties:
assistant_id:
type: string
description: |
The identifier of assistant, defaults to "jan"
example: jan
model:
type: object
properties:
id:
type: string
description: ''
example: ...
settings:
type: object
description: >
Defaults to and overrides assistant.json's "settings" (and if none,
then model.json "settings")
parameters:
type: object
description: >
Defaults to and overrides assistant.json's "parameters" (and if
none, then model.json "parameters")
created:
type: integer
format: int64
description: >
Unix timestamp representing the creation time of the thread,
defaults to file creation time.
example: 1231231
metadata:
type: object
description: |
Metadata associated with the thread, defaults to an empty object.
example: {}
GetThreadResponse:
type: object
properties:
id:
type: string
description: The identifier of the thread.
example: thread_abc123
object:
type: string
description: Type of the object
example: thread
created_at:
type: integer
format: int64
description: Unix timestamp representing the creation time of the thread.
example: 1699014083
assistants:
type: array
items:
type: string
description: List of assistants involved in the thread.
example:
- assistant-001
metadata:
type: object
description: Metadata associated with the thread.
example: {}
messages:
type: array
items:
type: string
description: List of messages within the thread.
example: []
CreateThreadResponse:
type: object
properties:
id:
type: string
description: The identifier of the newly created thread.
example: thread_abc123
object:
type: string
description: Type of the object, indicating it's a thread.
example: thread
created_at:
type: integer
format: int64
description: Unix timestamp representing the creation time of the thread.
example: 1699014083
metadata:
type: object
description: Metadata associated with the newly created thread.
example: {}
CreateThreadObject:
type: object
properties:
object:
type: string
description: Type of the object, indicating it's a thread.
example: thread
title:
type: string
description: >
A brief summary or description of the thread, defaults to an empty
string.
example: funny physics joke
assistants:
type: array
description: assistant involved in the thread
items:
properties:
assistant_id:
type: string
description: |
The identifier of assistant, defaults to "jan"
example: jan
assistant_name:
type: string
description: |
The name of assistant, defaults to "Jan"
example: Jan
instructions:
type: string
description: >
The instruction of assistant, defaults to "Be my grammar corrector"
model:
type: object
properties:
id:
type: string
description: Model id
example: mistral-ins-7b-q4
settings:
type: object
description: >
Defaults to and overrides assistant.json's "settings" (and if none,
then model.json "settings")
parameters:
type: object
description: >
Defaults to and overrides assistant.json's "parameters" (and if
none, then model.json "parameters")
engine:
type: string
description: Engine id
example: nitro
metadata:
type: object
description: |
Metadata associated with the thread, defaults to an empty object.
ThreadMessageObject:
type: object
properties:
role:
type: string
description: |
"Role of the sender, either 'user' or 'assistant'."
enum:
- user
- assistant
content:
type: string
description: |
"Text content of the message."
file_ids:
type: array
items:
type: string
description: |
"Array of file IDs associated with the message, if any."
ModifyThreadResponse:
type: object
properties:
id:
type: string
description: |
"The identifier of the modified thread."
example: thread_abc123
object:
type: string
description: Type of the object, indicating it's a thread.
example: thread
created_at:
type: integer
format: int64
description: Unix timestamp representing the creation time of the thread.
example: 1699014083
metadata:
type: object
description: Metadata associated with the modified thread.
example: {}
DeleteThreadResponse:
type: object
properties:
id:
type: string
description: The identifier of the deleted thread.
example: thread_abc123
object:
type: string
description: Type of the object, indicating the thread has been deleted.
example: thread.deleted
deleted:
type: boolean
description: Indicates whether the thread was successfully deleted.
example: true

1
docs/openapi/version.txt
View File

@ -1 +0,0 @@
v1.23.2

0
electron/docs/openapi/.gitkeep
View File

1
electron/package.json
View File

@ -15,7 +15,6 @@
"build/**/*.{js,map}", "build/**/*.{js,map}",
"pre-install", "pre-install",
"themes", "themes",
"docs/**/*",
"scripts/**/*", "scripts/**/*",
"icons/**/*", "icons/**/*",
"themes", "themes",

2
extensions/conversational-extension/src/index.ts
View File

@ -19,7 +19,7 @@ type MessageList = {
* JSONConversationalExtension is a ConversationalExtension implementation that provides * JSONConversationalExtension is a ConversationalExtension implementation that provides
* functionality for managing threads. * functionality for managing threads.
*/ */
export default class JSONConversationalExtension extends ConversationalExtension { export default class CortexConversationalExtension extends ConversationalExtension {
queue = new PQueue({ concurrency: 1 }) queue = new PQueue({ concurrency: 1 })
/** /**

6
package.json
View File

@ -27,12 +27,12 @@
"pre-install:linux": "find extensions -type f -path \"**/*.tgz\" -exec cp {} pre-install \\;", "pre-install:linux": "find extensions -type f -path \"**/*.tgz\" -exec cp {} pre-install \\;",
"pre-install:win32": "powershell -Command \"Get-ChildItem -Path \"extensions\" -Recurse -File -Filter \"*.tgz\" | ForEach-Object { Copy-Item -Path $_.FullName -Destination \"pre-install\" }\"", "pre-install:win32": "powershell -Command \"Get-ChildItem -Path \"extensions\" -Recurse -File -Filter \"*.tgz\" | ForEach-Object { Copy-Item -Path $_.FullName -Destination \"pre-install\" }\"",
"pre-install": "run-script-os", "pre-install": "run-script-os",
"copy:assets": "cpx \"pre-install/*.tgz\" \"electron/pre-install/\" && cpx \"themes/**\" \"electron/themes\" && cpx \"docs/openapi/**\" \"electron/docs/openapi\"", "copy:assets": "cpx \"pre-install/*.tgz\" \"electron/pre-install/\" && cpx \"themes/**\" \"electron/themes\"",
"dev:electron": "yarn copy:assets && yarn workspace jan dev", "dev:electron": "yarn copy:assets && yarn workspace jan dev",
"dev:web": "yarn workspace @janhq/web dev", "dev:web": "yarn workspace @janhq/web dev",
"dev:server": "yarn copy:assets && yarn workspace @janhq/server dev", "dev:server": "yarn workspace @janhq/server dev",
"dev": "turbo run dev --parallel --filter=!@janhq/server", "dev": "turbo run dev --parallel --filter=!@janhq/server",
"build:server": "yarn copy:assets && cd server && yarn install && yarn run build", "build:server": "cd server && yarn install && yarn run build",
"build:core": "cd core && yarn install && yarn run build", "build:core": "cd core && yarn install && yarn run build",
"build:web": "yarn workspace @janhq/web build && cpx \"web/out/**\" \"electron/renderer/\"", "build:web": "yarn workspace @janhq/web build && cpx \"web/out/**\" \"electron/renderer/\"",
"build:electron": "yarn copy:assets && yarn workspace jan build", "build:electron": "yarn copy:assets && yarn workspace jan build",

6308
server/cortex.json Normal file
View File

File diff suppressed because it is too large Load Diff

39
server/index.ts
View File

@ -1,9 +1,9 @@
import fastify from 'fastify' import fastify from 'fastify'
import dotenv from 'dotenv' import dotenv from 'dotenv'
import { v1Router, log, getJanExtensionsPath } from '@janhq/core/node' import { log } from '@janhq/core/node'
import { join } from 'path'
import tcpPortUsed from 'tcp-port-used' import tcpPortUsed from 'tcp-port-used'
import { Logger } from './helpers/logger' import { Logger } from './helpers/logger'
import CORTEX_SCHEMA from './cortex.json'
// Load environment variables // Load environment variables
dotenv.config() dotenv.config()
@ -66,34 +66,29 @@ export const startServer = async (configs?: ServerConfig): Promise<boolean> => {
// Initialize Fastify server with logging // Initialize Fastify server with logging
server = fastify({ server = fastify({
logger: new Logger(), loggerInstance: new Logger(),
// Set body limit to 100MB - Default is 1MB // Set body limit to 100MB - Default is 1MB
// According to OpenAI - a batch input file can be up to 100 MB in size // According to OpenAI - a batch input file can be up to 100 MB in size
// Whisper endpoints accept up to 25MB // Whisper endpoints accept up to 25MB
// Vision endpoints accept up to 4MB // Vision endpoints accept up to 4MB
bodyLimit: 104_857_600 bodyLimit: 104_857_600,
}) })
// Register CORS if enabled // Register CORS if enabled
if (corsEnabled) await server.register(require('@fastify/cors'), {}) if (corsEnabled) await server.register(require('@fastify/cors'), {})
CORTEX_SCHEMA.servers[0].url = configs?.prefix ?? '/v1'
// Register Swagger for API documentation // Register Swagger for API documentation
await server.register(require('@fastify/swagger'), { await server.register(require('@fastify/swagger'), {
mode: 'static', mode: 'static',
specification: { specification: {
path: configs?.schemaPath ?? './../docs/openapi/jan.yaml', document: CORTEX_SCHEMA,
baseDir: configs?.baseDir ?? './../docs/openapi',
postProcessor: function (swaggerObject: any) {
swaggerObject.servers[0].url = configs?.prefix ?? '/v1'
return swaggerObject
},
}, },
}) })
// Register Swagger UI // Register Swagger UI
await server.register(require('@fastify/swagger-ui'), { await server.register(require('@fastify/swagger-ui'), {
routePrefix: '/', routePrefix: '/',
baseDir: configs?.baseDir ?? join(__dirname, '../..', './docs/openapi'),
uiConfig: { uiConfig: {
docExpansion: 'full', docExpansion: 'full',
deepLinking: false, deepLinking: false,
@ -102,26 +97,12 @@ export const startServer = async (configs?: ServerConfig): Promise<boolean> => {
transformSpecificationClone: true, transformSpecificationClone: true,
}) })
// Register static file serving for extensions server.register(require('@fastify/http-proxy'), {
// TODO: Watch extension files changes and reload upstream: 'http://127.0.0.1:39291/v1',
await server.register( prefix: configs?.prefix ?? '/v1',
(childContext: any, _: any, done: any) => { http2: false,
childContext.register(require('@fastify/static'), {
root: getJanExtensionsPath(),
wildcard: false,
}) })
done()
},
{ prefix: 'extensions' }
)
// Register proxy middleware
if (configs?.storageAdataper)
server.addHook('preHandler', configs.storageAdataper)
// Register API routes
await server.register(v1Router, { prefix: configs?.prefix ?? '/v1' })
// Start listening for requests // Start listening for requests
await server await server
.listen({ .listen({

3
server/main.ts
View File

@ -1,7 +1,6 @@
import { s3 } from './middleware/s3'
import { setup } from './helpers/setup' import { setup } from './helpers/setup'
import { startServer as start } from './index' import { startServer as start } from './index'
/** /**
* Setup extensions and start the server * Setup extensions and start the server
*/ */
setup().then(() => start({ storageAdataper: s3 })) setup().then(() => start())

70
server/middleware/s3.ts
View File

@ -1,70 +0,0 @@
import { join } from 'path'
// Middleware to intercept requests and proxy if certain conditions are met
const config = {
endpoint: process.env.AWS_ENDPOINT,
region: process.env.AWS_REGION,
credentials: {
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
},
}
const S3_BUCKET_NAME = process.env.S3_BUCKET_NAME
const fs = require('@cyclic.sh/s3fs')(S3_BUCKET_NAME, config)
const PROXY_PREFIX = '/v1/fs'
const PROXY_ROUTES = ['/threads', '/messages']
export const s3 = (req: any, reply: any, done: any) => {
// Proxy FS requests to S3 using S3FS
if (req.url.startsWith(PROXY_PREFIX)) {
const route = req.url.split('/').pop()
const args = parseRequestArgs(req)
// Proxy matched requests to the s3fs module
if (args.length && PROXY_ROUTES.some((route) => args[0].includes(route))) {
try {
// Handle customized route
// S3FS does not handle appendFileSync
if (route === 'appendFileSync') {
let result = handAppendFileSync(args)
reply.status(200).send(result)
return
}
// Reroute the other requests to the s3fs module
const result = fs[route](...args)
reply.status(200).send(result)
return
} catch (ex) {
console.error(ex)
}
}
}
// Let other requests go through
done()
}
const parseRequestArgs = (req: Request) => {
const {
getJanDataFolderPath,
normalizeFilePath,
} = require('@janhq/core/node')
return JSON.parse(req.body as any).map((arg: any) =>
typeof arg === 'string' &&
(arg.startsWith(`file:/`) || arg.startsWith(`file:\\`))
? join(getJanDataFolderPath(), normalizeFilePath(arg))
: arg
)
}
const handAppendFileSync = (args: any[]) => {
if (fs.existsSync(args[0])) {
const data = fs.readFileSync(args[0], 'utf-8')
return fs.writeFileSync(args[0], data + args[1])
} else {
return fs.writeFileSync(args[0], args[1])
}
}

12
server/package.json
View File

@ -8,7 +8,8 @@
"homepage": "https://jan.ai", "homepage": "https://jan.ai",
"description": "Use offline LLMs with your own data. Run open source models like Llama2 or Falcon on your internal computers/servers.", "description": "Use offline LLMs with your own data. Run open source models like Llama2 or Falcon on your internal computers/servers.",
"files": [ "files": [
"build/**" "build/**",
"cortex.json"
], ],
"scripts": { "scripts": {
"lint": "eslint . --ext \".js,.jsx,.ts,.tsx\"", "lint": "eslint . --ext \".js,.jsx,.ts,.tsx\"",
@ -19,14 +20,15 @@
"dependencies": { "dependencies": {
"@alumna/reflect": "^1.1.3", "@alumna/reflect": "^1.1.3",
"@cyclic.sh/s3fs": "^1.2.9", "@cyclic.sh/s3fs": "^1.2.9",
"@fastify/cors": "^8.4.2", "@fastify/cors": "^10.0.1",
"@fastify/http-proxy": "^10.0.0",
"@fastify/static": "^6.12.0", "@fastify/static": "^6.12.0",
"@fastify/swagger": "^8.13.0", "@fastify/swagger": "^9.4.0",
"@fastify/swagger-ui": "2.0.1", "@fastify/swagger-ui": "5.2.0",
"@janhq/core": "link:./core", "@janhq/core": "link:./core",
"@npmcli/arborist": "^7.3.1", "@npmcli/arborist": "^7.3.1",
"dotenv": "^16.3.1", "dotenv": "^16.3.1",
"fastify": "^4.24.3", "fastify": "^5.2.0",
"fetch-retry": "^5.0.6", "fetch-retry": "^5.0.6",
"node-fetch": "2", "node-fetch": "2",
"request": "^2.88.2", "request": "^2.88.2",

3
server/tsconfig.json
View File

@ -15,7 +15,8 @@
"paths": { "*": ["node_modules/*"] }, "paths": { "*": ["node_modules/*"] },
"typeRoots": ["node_modules/@types"], "typeRoots": ["node_modules/@types"],
"ignoreDeprecations": "5.0", "ignoreDeprecations": "5.0",
"declaration": true "declaration": true,
"resolveJsonModule": true
}, },
// "sourceMap": true, // "sourceMap": true,

2
web/hooks/useActiveModel.ts
View File

@ -10,7 +10,6 @@ import { LAST_USED_MODEL_ID } from './useRecommendedModel'
import { vulkanEnabledAtom } from '@/helpers/atoms/AppConfig.atom' import { vulkanEnabledAtom } from '@/helpers/atoms/AppConfig.atom'
import { activeAssistantAtom } from '@/helpers/atoms/Assistant.atom' import { activeAssistantAtom } from '@/helpers/atoms/Assistant.atom'
import { downloadedModelsAtom } from '@/helpers/atoms/Model.atom' import { downloadedModelsAtom } from '@/helpers/atoms/Model.atom'
import { activeThreadAtom } from '@/helpers/atoms/Thread.atom'
export const activeModelAtom = atom<Model | undefined>(undefined) export const activeModelAtom = atom<Model | undefined>(undefined)
export const loadModelErrorAtom = atom<string | undefined>(undefined) export const loadModelErrorAtom = atom<string | undefined>(undefined)
@ -29,7 +28,6 @@ export const stateModelAtom = atom<ModelState>({
export function useActiveModel() { export function useActiveModel() {
const [activeModel, setActiveModel] = useAtom(activeModelAtom) const [activeModel, setActiveModel] = useAtom(activeModelAtom)
const activeThread = useAtomValue(activeThreadAtom)
const [stateModel, setStateModel] = useAtom(stateModelAtom) const [stateModel, setStateModel] = useAtom(stateModelAtom)
const downloadedModels = useAtomValue(downloadedModelsAtom) const downloadedModels = useAtomValue(downloadedModelsAtom)
const setLoadModelError = useSetAtom(loadModelErrorAtom) const setLoadModelError = useSetAtom(loadModelErrorAtom)