Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: merge release/v0.5.16 into dev (#4833)

Browse Source 
* chore: bump cortex 1.0.11-rc10

* chore: bump to latest cortex release

* feat: Cortex API Authorization

* chore: correct CI CD repo name

* chore: correct new menloresearch repo name

* feat: rotate api token for each run (#4820)

* feat: rotate api token for each run

* chore: correct github repo url

* chore: correct github api url

* chore: should not filter out models first launch

* chore: bump cortex release

* chore: should get hardware information on launch (#4821)

* chore: should have an option to not revalidate hardware information

* chore: cortex.cpp gpu activation could cause a race condition (#4825)

* fix: jan beta logo displayed in jan release (#4828)

---------

Co-authored-by: David <davidpt.janai@gmail.com>
Co-authored-by: Nguyen Ngoc Minh <91668012+Minh141120@users.noreply.github.com>
This commit is contained in:
Louis 2025-03-24 11:53:30 +07:00 committed by GitHub
parent eacae5f17a
commit 9d64e9bef4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
26 changed files with 562 additions and 551 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/jan-electron-build.yml vendored
View File

@ -88,4 +88,4 @@ jobs:
# config-name: my-config.yml # config-name: my-config.yml
# disable-autolabeler: true # disable-autolabeler: true
env: env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

98
.github/workflows/nightly-integrate-cortex-cpp.yml vendored
View File

@ -18,67 +18,67 @@ jobs:
pr_created: ${{ steps.check-update.outputs.pr_created }} pr_created: ${{ steps.check-update.outputs.pr_created }}
steps: steps:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@v3 uses: actions/checkout@v3
with: with:
submodules: recursive submodules: recursive
ref: dev ref: dev
fetch-depth: 0 fetch-depth: 0
token: ${{ secrets.PAT_SERVICE_ACCOUNT }} token: ${{ secrets.PAT_SERVICE_ACCOUNT }}
- name: Configure Git - name: Configure Git
run: | run: |
git config --global user.name 'github-actions[bot]' git config --global user.name 'github-actions[bot]'
git config --global user.email 'github-actions[bot]@users.noreply.github.com' git config --global user.email 'github-actions[bot]@users.noreply.github.com'
- name: Update submodule to latest release - name: Update submodule to latest release
id: check-update id: check-update
env: env:
GITHUB_TOKEN: ${{ secrets.PAT_SERVICE_ACCOUNT }} GITHUB_TOKEN: ${{ secrets.PAT_SERVICE_ACCOUNT }}
run: | run: |
curl -s https://api.github.com/repos/menloresearch/cortex/releases > /tmp/github_api_releases.json curl -s https://api.github.com/repos/menloresearch/cortex/releases > /tmp/github_api_releases.json
latest_prerelease_name=$(cat /tmp/github_api_releases.json | jq -r '.[] | select(.prerelease) | .name' | head -n 1) latest_prerelease_name=$(cat /tmp/github_api_releases.json | jq -r '.[] | select(.prerelease) | .name' | head -n 1)
get_asset_count() { get_asset_count() {
local version_name=$1 local version_name=$1
cat /tmp/github_api_releases.json | jq -r --arg version_name "$version_name" '.[] | select(.name == $version_name) | .assets | length' cat /tmp/github_api_releases.json | jq -r --arg version_name "$version_name" '.[] | select(.name == $version_name) | .assets | length'
} }
cortex_cpp_version_file_path="extensions/inference-nitro-extension/bin/version.txt" cortex_cpp_version_file_path="extensions/inference-nitro-extension/bin/version.txt"
current_version_name=$(cat "$cortex_cpp_version_file_path" | head -n 1) current_version_name=$(cat "$cortex_cpp_version_file_path" | head -n 1)
current_version_asset_count=$(get_asset_count "$current_version_name") current_version_asset_count=$(get_asset_count "$current_version_name")
latest_prerelease_asset_count=$(get_asset_count "$latest_prerelease_name") latest_prerelease_asset_count=$(get_asset_count "$latest_prerelease_name")
if [ "$current_version_name" = "$latest_prerelease_name" ]; then if [ "$current_version_name" = "$latest_prerelease_name" ]; then
echo "cortex cpp remote repo doesn't have update today, skip update cortex.cpp for today nightly build" echo "cortex cpp remote repo doesn't have update today, skip update cortex.cpp for today nightly build"
echo "::set-output name=pr_created::false" echo "::set-output name=pr_created::false"
exit 0 exit 0
fi fi
if [ "$current_version_asset_count" != "$latest_prerelease_asset_count" ]; then if [ "$current_version_asset_count" != "$latest_prerelease_asset_count" ]; then
echo "Latest prerelease version has different number of assets, somethink went wrong, skip update cortex.cpp for today nightly build" echo "Latest prerelease version has different number of assets, somethink went wrong, skip update cortex.cpp for today nightly build"
echo "::set-output name=pr_created::false" echo "::set-output name=pr_created::false"
exit 1 exit 1
fi fi
echo $latest_prerelease_name > $cortex_cpp_version_file_path echo $latest_prerelease_name > $cortex_cpp_version_file_path
echo "Updated version from $current_version_name to $latest_prerelease_name." echo "Updated version from $current_version_name to $latest_prerelease_name."
echo "::set-output name=pr_created::true" echo "::set-output name=pr_created::true"
git add -f $cortex_cpp_version_file_path git add -f $cortex_cpp_version_file_path
git commit -m "Update cortex cpp nightly to version $latest_prerelease_name" git commit -m "Update cortex cpp nightly to version $latest_prerelease_name"
branch_name="update-nightly-$(date +'%Y-%m-%d-%H-%M')" branch_name="update-nightly-$(date +'%Y-%m-%d-%H-%M')"
git checkout -b $branch_name git checkout -b $branch_name
git push origin $branch_name git push origin $branch_name
pr_title="Update cortex cpp nightly to version $latest_prerelease_name" pr_title="Update cortex cpp nightly to version $latest_prerelease_name"
pr_body="This PR updates the Update cortex cpp nightly to version $latest_prerelease_name" pr_body="This PR updates the Update cortex cpp nightly to version $latest_prerelease_name"
gh pr create --title "$pr_title" --body "$pr_body" --head $branch_name --base dev --reviewer Van-QA gh pr create --title "$pr_title" --body "$pr_body" --head $branch_name --base dev --reviewer Van-QA
pr_number=$(gh pr list --head $branch_name --json number --jq '.[0].number') pr_number=$(gh pr list --head $branch_name --json number --jq '.[0].number')
echo "::set-output name=pr_number::$pr_number" echo "::set-output name=pr_number::$pr_number"
check-and-merge-pr: check-and-merge-pr:
needs: update-submodule needs: update-submodule

2
.github/workflows/template-build-linux-x64.yml vendored
View File

@ -184,4 +184,4 @@ jobs:
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: jan-linux-amd64-${{ inputs.new_version }}-AppImage name: jan-linux-amd64-${{ inputs.new_version }}-AppImage
path: ./electron/dist/*.AppImage path: ./electron/dist/*.AppImage

2
.github/workflows/template-build-macos.yml vendored
View File

@ -230,4 +230,4 @@ jobs:
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: jan-mac-universal-${{ inputs.new_version }} name: jan-mac-universal-${{ inputs.new_version }}
path: ./electron/dist/*.dmg path: ./electron/dist/*.dmg

2
.github/workflows/template-build-windows-x64.yml vendored
View File

@ -226,4 +226,4 @@ jobs:
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: jan-win-x64-${{ inputs.new_version }} name: jan-win-x64-${{ inputs.new_version }}
path: ./electron/dist/*.exe path: ./electron/dist/*.exe

2
core/src/types/api/index.ts
View File

@ -33,6 +33,8 @@ export enum NativeRoute {
stopServer = 'stopServer', stopServer = 'stopServer',
appUpdateDownload = 'appUpdateDownload', appUpdateDownload = 'appUpdateDownload',
appToken = 'appToken',
} }
/** /**

7
electron/handlers/native.ts
View File

@ -317,4 +317,11 @@ export function handleAppIPCs() {
const { stopServer } = require('@janhq/server') const { stopServer } = require('@janhq/server')
return stopServer() return stopServer()
}) })
/**
* Handles the "appToken" IPC message to generate a random app ID.
*/
ipcMain.handle(NativeRoute.appToken, async (_event): Promise<string> => {
return process.env.appToken ?? 'cortex.cpp'
})
} }

5
electron/main.ts
View File

@ -29,6 +29,7 @@ import { trayManager } from './managers/tray'
import { logSystemInfo } from './utils/system' import { logSystemInfo } from './utils/system'
import { registerGlobalShortcuts } from './utils/shortcut' import { registerGlobalShortcuts } from './utils/shortcut'
import { registerLogger } from './utils/logger' import { registerLogger } from './utils/logger'
import { randomBytes } from 'crypto'
const preloadPath = join(__dirname, 'preload.js') const preloadPath = join(__dirname, 'preload.js')
const preloadQuickAskPath = join(__dirname, 'preload.quickask.js') const preloadQuickAskPath = join(__dirname, 'preload.quickask.js')
@ -56,6 +57,10 @@ const createMainWindow = () => {
windowManager.createMainWindow(preloadPath, startUrl) windowManager.createMainWindow(preloadPath, startUrl)
} }
// Generate a random token for the app
// This token is used for authentication when making request to cortex.cpp server
process.env.appToken = randomBytes(16).toString('hex')
app app
.whenReady() .whenReady()
.then(() => { .then(() => {

12
extensions/assistant-extension/src/node/retrieval.ts
View File

@ -23,11 +23,16 @@ export class Retrieval {
constructor(chunkSize: number = 4000, chunkOverlap: number = 200) { constructor(chunkSize: number = 4000, chunkOverlap: number = 200) {
this.updateTextSplitter(chunkSize, chunkOverlap) this.updateTextSplitter(chunkSize, chunkOverlap)
this.initialize()
}
private async initialize() {
const apiKey = await window.core?.api.appToken() ?? 'cortex.cpp'
// declare time-weighted retriever and storage // declare time-weighted retriever and storage
this.timeWeightedVectorStore = new MemoryVectorStore( this.timeWeightedVectorStore = new MemoryVectorStore(
new OpenAIEmbeddings( new OpenAIEmbeddings(
{ openAIApiKey: 'cortex-embedding' }, { openAIApiKey: apiKey },
{ basePath: `${CORTEX_API_URL}/v1` } { basePath: `${CORTEX_API_URL}/v1` }
) )
) )
@ -47,9 +52,10 @@ export class Retrieval {
}) })
} }
public updateEmbeddingEngine(model: string, engine: string): void { public async updateEmbeddingEngine(model: string, engine: string) {
const apiKey = await window.core?.api.appToken() ?? 'cortex.cpp'
this.embeddingModel = new OpenAIEmbeddings( this.embeddingModel = new OpenAIEmbeddings(
{ openAIApiKey: 'cortex-embedding', model }, { openAIApiKey: apiKey, model },
// TODO: Raw settings // TODO: Raw settings
{ basePath: `${CORTEX_API_URL}/v1` } { basePath: `${CORTEX_API_URL}/v1` }
) )

113
extensions/conversational-extension/src/index.ts
View File

@ -4,7 +4,7 @@ import {
ThreadAssistantInfo, ThreadAssistantInfo,
ThreadMessage, ThreadMessage,
} from '@janhq/core' } from '@janhq/core'
import ky from 'ky' import ky, { KyInstance } from 'ky'
import PQueue from 'p-queue' import PQueue from 'p-queue'
type ThreadList = { type ThreadList = {
@ -22,6 +22,22 @@ type MessageList = {
export default class CortexConversationalExtension extends ConversationalExtension { export default class CortexConversationalExtension extends ConversationalExtension {
queue = new PQueue({ concurrency: 1 }) queue = new PQueue({ concurrency: 1 })
api?: KyInstance
/**
* Get the API instance
* @returns
*/
async apiInstance(): Promise<KyInstance> {
if(this.api) return this.api
const apiKey = (await window.core?.api.appToken()) ?? 'cortex.cpp'
this.api = ky.extend({
prefixUrl: API_URL,
headers: {
Authorization: `Bearer ${apiKey}`,
},
})
return this.api
}
/** /**
* Called when the extension is loaded. * Called when the extension is loaded.
*/ */
@ -39,10 +55,12 @@ export default class CortexConversationalExtension extends ConversationalExtensi
*/ */
async listThreads(): Promise<Thread[]> { async listThreads(): Promise<Thread[]> {
return this.queue.add(() => return this.queue.add(() =>
ky this.apiInstance().then((api) =>
.get(`${API_URL}/v1/threads?limit=-1`) api
.json<ThreadList>() .get('v1/threads?limit=-1')
.then((e) => e.data) .json<ThreadList>()
.then((e) => e.data)
)
) as Promise<Thread[]> ) as Promise<Thread[]>
} }
@ -52,7 +70,9 @@ export default class CortexConversationalExtension extends ConversationalExtensi
*/ */
async createThread(thread: Thread): Promise<Thread> { async createThread(thread: Thread): Promise<Thread> {
return this.queue.add(() => return this.queue.add(() =>
ky.post(`${API_URL}/v1/threads`, { json: thread }).json<Thread>() this.apiInstance().then((api) =>
api.post('v1/threads', { json: thread }).json<Thread>()
)
) as Promise<Thread> ) as Promise<Thread>
} }
@ -63,7 +83,9 @@ export default class CortexConversationalExtension extends ConversationalExtensi
async modifyThread(thread: Thread): Promise<void> { async modifyThread(thread: Thread): Promise<void> {
return this.queue return this.queue
.add(() => .add(() =>
ky.patch(`${API_URL}/v1/threads/${thread.id}`, { json: thread }) this.apiInstance().then((api) =>
api.patch(`v1/threads/${thread.id}`, { json: thread })
)
) )
.then() .then()
} }
@ -74,7 +96,9 @@ export default class CortexConversationalExtension extends ConversationalExtensi
*/ */
async deleteThread(threadId: string): Promise<void> { async deleteThread(threadId: string): Promise<void> {
return this.queue return this.queue
.add(() => ky.delete(`${API_URL}/v1/threads/${threadId}`)) .add(() =>
this.apiInstance().then((api) => api.delete(`v1/threads/${threadId}`))
)
.then() .then()
} }
@ -85,11 +109,13 @@ export default class CortexConversationalExtension extends ConversationalExtensi
*/ */
async createMessage(message: ThreadMessage): Promise<ThreadMessage> { async createMessage(message: ThreadMessage): Promise<ThreadMessage> {
return this.queue.add(() => return this.queue.add(() =>
ky this.apiInstance().then((api) =>
.post(`${API_URL}/v1/threads/${message.thread_id}/messages`, { api
json: message, .post(`v1/threads/${message.thread_id}/messages`, {
}) json: message,
.json<ThreadMessage>() })
.json<ThreadMessage>()
)
) as Promise<ThreadMessage> ) as Promise<ThreadMessage>
} }
@ -100,14 +126,13 @@ export default class CortexConversationalExtension extends ConversationalExtensi
*/ */
async modifyMessage(message: ThreadMessage): Promise<ThreadMessage> { async modifyMessage(message: ThreadMessage): Promise<ThreadMessage> {
return this.queue.add(() => return this.queue.add(() =>
ky this.apiInstance().then((api) =>
.patch( api
`${API_URL}/v1/threads/${message.thread_id}/messages/${message.id}`, .patch(`v1/threads/${message.thread_id}/messages/${message.id}`, {
{
json: message, json: message,
} })
) .json<ThreadMessage>()
.json<ThreadMessage>() )
) as Promise<ThreadMessage> ) as Promise<ThreadMessage>
} }
@ -120,7 +145,9 @@ export default class CortexConversationalExtension extends ConversationalExtensi
async deleteMessage(threadId: string, messageId: string): Promise<void> { async deleteMessage(threadId: string, messageId: string): Promise<void> {
return this.queue return this.queue
.add(() => .add(() =>
ky.delete(`${API_URL}/v1/threads/${threadId}/messages/${messageId}`) this.apiInstance().then((api) =>
api.delete(`v1/threads/${threadId}/messages/${messageId}`)
)
) )
.then() .then()
} }
@ -132,10 +159,12 @@ export default class CortexConversationalExtension extends ConversationalExtensi
*/ */
async listMessages(threadId: string): Promise<ThreadMessage[]> { async listMessages(threadId: string): Promise<ThreadMessage[]> {
return this.queue.add(() => return this.queue.add(() =>
ky this.apiInstance().then((api) =>
.get(`${API_URL}/v1/threads/${threadId}/messages?order=asc&limit=-1`) api
.json<MessageList>() .get(`v1/threads/${threadId}/messages?order=asc&limit=-1`)
.then((e) => e.data) .json<MessageList>()
.then((e) => e.data)
)
) as Promise<ThreadMessage[]> ) as Promise<ThreadMessage[]>
} }
@ -147,9 +176,11 @@ export default class CortexConversationalExtension extends ConversationalExtensi
*/ */
async getThreadAssistant(threadId: string): Promise<ThreadAssistantInfo> { async getThreadAssistant(threadId: string): Promise<ThreadAssistantInfo> {
return this.queue.add(() => return this.queue.add(() =>
ky this.apiInstance().then((api) =>
.get(`${API_URL}/v1/assistants/${threadId}?limit=-1`) api
.json<ThreadAssistantInfo>() .get(`v1/assistants/${threadId}?limit=-1`)
.json<ThreadAssistantInfo>()
)
) as Promise<ThreadAssistantInfo> ) as Promise<ThreadAssistantInfo>
} }
/** /**
@ -163,9 +194,11 @@ export default class CortexConversationalExtension extends ConversationalExtensi
assistant: ThreadAssistantInfo assistant: ThreadAssistantInfo
): Promise<ThreadAssistantInfo> { ): Promise<ThreadAssistantInfo> {
return this.queue.add(() => return this.queue.add(() =>
ky this.apiInstance().then((api) =>
.post(`${API_URL}/v1/assistants/${threadId}`, { json: assistant }) api
.json<ThreadAssistantInfo>() .post(`v1/assistants/${threadId}`, { json: assistant })
.json<ThreadAssistantInfo>()
)
) as Promise<ThreadAssistantInfo> ) as Promise<ThreadAssistantInfo>
} }
@ -180,9 +213,11 @@ export default class CortexConversationalExtension extends ConversationalExtensi
assistant: ThreadAssistantInfo assistant: ThreadAssistantInfo
): Promise<ThreadAssistantInfo> { ): Promise<ThreadAssistantInfo> {
return this.queue.add(() => return this.queue.add(() =>
ky this.apiInstance().then((api) =>
.patch(`${API_URL}/v1/assistants/${threadId}`, { json: assistant }) api
.json<ThreadAssistantInfo>() .patch(`v1/assistants/${threadId}`, { json: assistant })
.json<ThreadAssistantInfo>()
)
) as Promise<ThreadAssistantInfo> ) as Promise<ThreadAssistantInfo>
} }
@ -191,10 +226,12 @@ export default class CortexConversationalExtension extends ConversationalExtensi
* @returns * @returns
*/ */
async healthz(): Promise<void> { async healthz(): Promise<void> {
return ky return this.apiInstance()
.get(`${API_URL}/healthz`, { .then((api) =>
retry: { limit: 20, delay: () => 500, methods: ['get'] }, api.get('healthz', {
}) retry: { limit: 20, delay: () => 500, methods: ['get'] },
})
)
.then(() => {}) .then(() => {})
} }
} }

194
extensions/engine-management-extension/src/index.ts
View File

@ -15,7 +15,7 @@ import {
ModelEvent, ModelEvent,
EngineEvent, EngineEvent,
} from '@janhq/core' } from '@janhq/core'
import ky, { HTTPError } from 'ky' import ky, { HTTPError, KyInstance } from 'ky'
import PQueue from 'p-queue' import PQueue from 'p-queue'
import { EngineError } from './error' import { EngineError } from './error'
import { getJanDataFolderPath } from '@janhq/core' import { getJanDataFolderPath } from '@janhq/core'
@ -31,6 +31,22 @@ interface ModelList {
export default class JanEngineManagementExtension extends EngineManagementExtension { export default class JanEngineManagementExtension extends EngineManagementExtension {
queue = new PQueue({ concurrency: 1 }) queue = new PQueue({ concurrency: 1 })
api?: KyInstance
/**
* Get the API instance
* @returns
*/
async apiInstance(): Promise<KyInstance> {
if(this.api) return this.api
const apiKey = (await window.core?.api.appToken()) ?? 'cortex.cpp'
this.api = ky.extend({
prefixUrl: API_URL,
headers: {
Authorization: `Bearer ${apiKey}`,
},
})
return this.api
}
/** /**
* Called when the extension is loaded. * Called when the extension is loaded.
*/ */
@ -59,10 +75,12 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
*/ */
async getEngines(): Promise<Engines> { async getEngines(): Promise<Engines> {
return this.queue.add(() => return this.queue.add(() =>
ky this.apiInstance().then((api) =>
.get(`${API_URL}/v1/engines`) api
.json<Engines>() .get('v1/engines')
.then((e) => e) .json<Engines>()
.then((e) => e)
)
) as Promise<Engines> ) as Promise<Engines>
} }
@ -70,12 +88,15 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
* @returns A Promise that resolves to an object of list engines. * @returns A Promise that resolves to an object of list engines.
*/ */
async getRemoteModels(name: string): Promise<any> { async getRemoteModels(name: string): Promise<any> {
return ky return this.apiInstance().then(
.get(`${API_URL}/v1/models/remote/${name}`) (api) =>
.json<ModelList>() api
.catch(() => ({ .get(`v1/models/remote/${name}`)
data: [], .json<ModelList>()
})) as Promise<ModelList> .catch(() => ({
data: [],
})) as Promise<ModelList>
)
} }
/** /**
@ -84,10 +105,12 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
*/ */
async getInstalledEngines(name: InferenceEngine): Promise<EngineVariant[]> { async getInstalledEngines(name: InferenceEngine): Promise<EngineVariant[]> {
return this.queue.add(() => return this.queue.add(() =>
ky this.apiInstance().then((api) =>
.get(`${API_URL}/v1/engines/${name}`) api
.json<EngineVariant[]>() .get(`v1/engines/${name}`)
.then((e) => e) .json<EngineVariant[]>()
.then((e) => e)
)
) as Promise<EngineVariant[]> ) as Promise<EngineVariant[]>
} }
@ -103,12 +126,14 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
platform?: string platform?: string
) { ) {
return this.queue.add(() => return this.queue.add(() =>
ky this.apiInstance().then((api) =>
.get(`${API_URL}/v1/engines/${name}/releases/${version}`) api
.json<EngineReleased[]>() .get(`v1/engines/${name}/releases/${version}`)
.then((e) => .json<EngineReleased[]>()
platform ? e.filter((r) => r.name.includes(platform)) : e .then((e) =>
) platform ? e.filter((r) => r.name.includes(platform)) : e
)
)
) as Promise<EngineReleased[]> ) as Promise<EngineReleased[]>
} }
@ -119,12 +144,14 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
*/ */
async getLatestReleasedEngine(name: InferenceEngine, platform?: string) { async getLatestReleasedEngine(name: InferenceEngine, platform?: string) {
return this.queue.add(() => return this.queue.add(() =>
ky this.apiInstance().then((api) =>
.get(`${API_URL}/v1/engines/${name}/releases/latest`) api
.json<EngineReleased[]>() .get(`v1/engines/${name}/releases/latest`)
.then((e) => .json<EngineReleased[]>()
platform ? e.filter((r) => r.name.includes(platform)) : e .then((e) =>
) platform ? e.filter((r) => r.name.includes(platform)) : e
)
)
) as Promise<EngineReleased[]> ) as Promise<EngineReleased[]>
} }
@ -134,9 +161,11 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
*/ */
async installEngine(name: string, engineConfig: EngineConfig) { async installEngine(name: string, engineConfig: EngineConfig) {
return this.queue.add(() => return this.queue.add(() =>
ky this.apiInstance().then((api) =>
.post(`${API_URL}/v1/engines/${name}/install`, { json: engineConfig }) api
.then((e) => e) .post(`v1/engines/${name}/install`, { json: engineConfig })
.then((e) => e)
)
) as Promise<{ messages: string }> ) as Promise<{ messages: string }>
} }
@ -167,15 +196,17 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
engineConfig.metadata.header_template = DEFAULT_REQUEST_HEADERS_TRANSFORM engineConfig.metadata.header_template = DEFAULT_REQUEST_HEADERS_TRANSFORM
return this.queue.add(() => return this.queue.add(() =>
ky.post(`${API_URL}/v1/engines`, { json: engineConfig }).then((e) => { this.apiInstance().then((api) =>
if (persistModels && engineConfig.metadata?.get_models_url) { api.post('v1/engines', { json: engineConfig }).then((e) => {
// Pull /models from remote models endpoint if (persistModels && engineConfig.metadata?.get_models_url) {
return this.populateRemoteModels(engineConfig) // Pull /models from remote models endpoint
.then(() => e) return this.populateRemoteModels(engineConfig)
.catch(() => e) .then(() => e)
} .catch(() => e)
return e }
}) return e
})
)
) as Promise<{ messages: string }> ) as Promise<{ messages: string }>
} }
@ -185,9 +216,11 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
*/ */
async uninstallEngine(name: InferenceEngine, engineConfig: EngineConfig) { async uninstallEngine(name: InferenceEngine, engineConfig: EngineConfig) {
return this.queue.add(() => return this.queue.add(() =>
ky this.apiInstance().then((api) =>
.delete(`${API_URL}/v1/engines/${name}/install`, { json: engineConfig }) api
.then((e) => e) .delete(`v1/engines/${name}/install`, { json: engineConfig })
.then((e) => e)
)
) as Promise<{ messages: string }> ) as Promise<{ messages: string }>
} }
@ -196,25 +229,27 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
* @param model - Remote model object. * @param model - Remote model object.
*/ */
async addRemoteModel(model: Model) { async addRemoteModel(model: Model) {
return this.queue return this.queue.add(() =>
.add(() => this.apiInstance()
ky .then((api) =>
.post(`${API_URL}/v1/models/add`, { api
json: { .post('v1/models/add', {
inference_params: { json: {
max_tokens: 4096, inference_params: {
temperature: 0.7, max_tokens: 4096,
top_p: 0.95, temperature: 0.7,
stream: true, top_p: 0.95,
frequency_penalty: 0, stream: true,
presence_penalty: 0, frequency_penalty: 0,
presence_penalty: 0,
},
...model,
}, },
...model, })
}, .then((e) => e)
}) )
.then((e) => e) .then(() => {})
) )
.then(() => {})
} }
/** /**
@ -223,10 +258,12 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
*/ */
async getDefaultEngineVariant(name: InferenceEngine) { async getDefaultEngineVariant(name: InferenceEngine) {
return this.queue.add(() => return this.queue.add(() =>
ky this.apiInstance().then((api) =>
.get(`${API_URL}/v1/engines/${name}/default`) api
.json<{ messages: string }>() .get(`v1/engines/${name}/default`)
.then((e) => e) .json<{ messages: string }>()
.then((e) => e)
)
) as Promise<DefaultEngineVariant> ) as Promise<DefaultEngineVariant>
} }
@ -240,9 +277,11 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
engineConfig: EngineConfig engineConfig: EngineConfig
) { ) {
return this.queue.add(() => return this.queue.add(() =>
ky this.apiInstance().then((api) =>
.post(`${API_URL}/v1/engines/${name}/default`, { json: engineConfig }) api
.then((e) => e) .post(`v1/engines/${name}/default`, { json: engineConfig })
.then((e) => e)
)
) as Promise<{ messages: string }> ) as Promise<{ messages: string }>
} }
@ -251,9 +290,11 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
*/ */
async updateEngine(name: InferenceEngine, engineConfig?: EngineConfig) { async updateEngine(name: InferenceEngine, engineConfig?: EngineConfig) {
return this.queue.add(() => return this.queue.add(() =>
ky this.apiInstance().then((api) =>
.post(`${API_URL}/v1/engines/${name}/update`, { json: engineConfig }) api
.then((e) => e) .post(`v1/engines/${name}/update`, { json: engineConfig })
.then((e) => e)
)
) as Promise<{ messages: string }> ) as Promise<{ messages: string }>
} }
@ -262,10 +303,12 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
* @returns * @returns
*/ */
async healthz(): Promise<void> { async healthz(): Promise<void> {
return ky return this.apiInstance()
.get(`${API_URL}/healthz`, { .then((api) =>
retry: { limit: 20, delay: () => 500, methods: ['get'] }, api.get('healthz', {
}) retry: { limit: 20, delay: () => 500, methods: ['get'] },
})
)
.then(() => { .then(() => {
this.queue.concurrency = Infinity this.queue.concurrency = Infinity
}) })
@ -390,7 +433,6 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
const version = await this.getSetting<string>('version', '0.0.0') const version = await this.getSetting<string>('version', '0.0.0')
const engines = await this.getEngines() const engines = await this.getEngines()
if (version < VERSION) { if (version < VERSION) {
console.log('Migrating engine settings...') console.log('Migrating engine settings...')
// Migrate engine settings // Migrate engine settings
await Promise.all( await Promise.all(
@ -398,7 +440,7 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
const { id, ...data } = engine const { id, ...data } = engine
data.api_key = engines[id]?.api_key data.api_key = engines[id]?.api_key
return this.updateEngine(id,{ return this.updateEngine(id, {
...data, ...data,
}).catch(console.error) }).catch(console.error)
}) })

2
extensions/hardware-management-extension/package.json
View File

@ -29,12 +29,10 @@
}, },
"dependencies": { "dependencies": {
"@janhq/core": "../../core/package.tgz", "@janhq/core": "../../core/package.tgz",
"cpu-instructions": "^0.0.13",
"ky": "^1.7.2", "ky": "^1.7.2",
"p-queue": "^8.0.1" "p-queue": "^8.0.1"
}, },
"bundledDependencies": [ "bundledDependencies": [
"cpu-instructions",
"@janhq/core" "@janhq/core"
], ],
"hardwares": { "hardwares": {

45
extensions/hardware-management-extension/src/index.ts
View File

@ -1,5 +1,5 @@
import { HardwareManagementExtension, HardwareInformation } from '@janhq/core' import { HardwareManagementExtension, HardwareInformation } from '@janhq/core'
import ky from 'ky' import ky, { KyInstance } from 'ky'
import PQueue from 'p-queue' import PQueue from 'p-queue'
/** /**
@ -17,6 +17,23 @@ export default class JSONHardwareManagementExtension extends HardwareManagementE
this.queue.add(() => this.healthz()) this.queue.add(() => this.healthz())
} }
api?: KyInstance
/**
* Get the API instance
* @returns
*/
async apiInstance(): Promise<KyInstance> {
if(this.api) return this.api
const apiKey = (await window.core?.api.appToken()) ?? 'cortex.cpp'
this.api = ky.extend({
prefixUrl: API_URL,
headers: {
Authorization: `Bearer ${apiKey}`,
},
})
return this.api
}
/** /**
* Called when the extension is unloaded. * Called when the extension is unloaded.
*/ */
@ -27,11 +44,13 @@ export default class JSONHardwareManagementExtension extends HardwareManagementE
* @returns * @returns
*/ */
async healthz(): Promise<void> { async healthz(): Promise<void> {
return ky return this.apiInstance().then((api) =>
.get(`${API_URL}/healthz`, { api
retry: { limit: 20, delay: () => 500, methods: ['get'] }, .get('healthz', {
}) retry: { limit: 20, delay: () => 500, methods: ['get'] },
.then(() => {}) })
.then(() => {})
)
} }
/** /**
@ -39,10 +58,12 @@ export default class JSONHardwareManagementExtension extends HardwareManagementE
*/ */
async getHardware(): Promise<HardwareInformation> { async getHardware(): Promise<HardwareInformation> {
return this.queue.add(() => return this.queue.add(() =>
ky this.apiInstance().then((api) =>
.get(`${API_URL}/v1/hardware`) api
.json<HardwareInformation>() .get('v1/hardware')
.then((e) => e) .json<HardwareInformation>()
.then((e) => e)
)
) as Promise<HardwareInformation> ) as Promise<HardwareInformation>
} }
@ -54,7 +75,9 @@ export default class JSONHardwareManagementExtension extends HardwareManagementE
activated_gpus: number[] activated_gpus: number[]
}> { }> {
return this.queue.add(() => return this.queue.add(() =>
ky.post(`${API_URL}/v1/hardware/activate`, { json: data }).then((e) => e) this.apiInstance().then((api) =>
api.post('v1/hardware/activate', { json: data }).then((e) => e)
)
) as Promise<{ ) as Promise<{
message: string message: string
activated_gpus: number[] activated_gpus: number[]

2
extensions/inference-cortex-extension/bin/version.txt
View File

@ -1 +1 @@
1.0.11-rc9 1.0.12

2
extensions/inference-cortex-extension/rolldown.config.mjs
View File

@ -24,7 +24,7 @@ export default defineConfig([
}, },
{ {
input: 'src/node/index.ts', input: 'src/node/index.ts',
external: ['@janhq/core/node', 'cpu-instructions'], external: ['@janhq/core/node'],
output: { output: {
format: 'cjs', format: 'cjs',
file: 'dist/node/index.cjs.js', file: 'dist/node/index.cjs.js',

141
extensions/inference-cortex-extension/src/index.ts
View File

@ -17,7 +17,7 @@ import {
ModelEvent, ModelEvent,
} from '@janhq/core' } from '@janhq/core'
import PQueue from 'p-queue' import PQueue from 'p-queue'
import ky from 'ky' import ky, { KyInstance } from 'ky'
/** /**
* Event subscription types of Downloader * Event subscription types of Downloader
@ -75,8 +75,35 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
abortControllers = new Map<string, AbortController>() abortControllers = new Map<string, AbortController>()
api?: KyInstance
/** /**
* Subscribes to events emitted by the @janhq/core package. * Get the API instance
* @returns
*/
async apiInstance(): Promise<KyInstance> {
if(this.api) return this.api
const apiKey = (await window.core?.api.appToken()) ?? 'cortex.cpp'
this.api = ky.extend({
prefixUrl: CORTEX_API_URL,
headers: {
Authorization: `Bearer ${apiKey}`,
},
})
return this.api
}
/**
* Authorization headers for the API requests.
* @returns
*/
headers(): Promise<HeadersInit> {
return window.core?.api.appToken().then((token: string) => ({
Authorization: `Bearer ${token}`,
}))
}
/**
* Called when the extension is loaded.
*/ */
async onLoad() { async onLoad() {
super.onLoad() super.onLoad()
@ -153,45 +180,49 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
this.abortControllers.set(model.id, controller) this.abortControllers.set(model.id, controller)
return await this.queue.add(() => return await this.queue.add(() =>
ky this.apiInstance().then((api) =>
.post(`${CORTEX_API_URL}/v1/models/start`, { api
json: { .post('v1/models/start', {
...extractModelLoadParams(model.settings), json: {
model: model.id, ...extractModelLoadParams(model.settings),
engine: model: model.id,
model.engine === InferenceEngine.nitro // Legacy model cache engine:
? InferenceEngine.cortex_llamacpp model.engine === InferenceEngine.nitro // Legacy model cache
: model.engine, ? InferenceEngine.cortex_llamacpp
cont_batching: this.cont_batching, : model.engine,
n_parallel: this.n_parallel, cont_batching: this.cont_batching,
caching_enabled: this.caching_enabled, n_parallel: this.n_parallel,
flash_attn: this.flash_attn, caching_enabled: this.caching_enabled,
cache_type: this.cache_type, flash_attn: this.flash_attn,
use_mmap: this.use_mmap, cache_type: this.cache_type,
...(this.cpu_threads ? { cpu_threads: this.cpu_threads } : {}), use_mmap: this.use_mmap,
}, ...(this.cpu_threads ? { cpu_threads: this.cpu_threads } : {}),
timeout: false, },
signal, timeout: false,
}) signal,
.json() })
.catch(async (e) => { .json()
throw (await e.response?.json()) ?? e .catch(async (e) => {
}) throw (await e.response?.json()) ?? e
.finally(() => this.abortControllers.delete(model.id)) })
.then() .finally(() => this.abortControllers.delete(model.id))
.then()
)
) )
} }
override async unloadModel(model: Model): Promise<void> { override async unloadModel(model: Model): Promise<void> {
return ky return this.apiInstance().then((api) =>
.post(`${CORTEX_API_URL}/v1/models/stop`, { api
json: { model: model.id }, .post('v1/models/stop', {
}) json: { model: model.id },
.json() })
.finally(() => { .json()
this.abortControllers.get(model.id)?.abort() .finally(() => {
}) this.abortControllers.get(model.id)?.abort()
.then() })
.then()
)
} }
/** /**
@ -199,15 +230,17 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
* @returns * @returns
*/ */
private async healthz(): Promise<void> { private async healthz(): Promise<void> {
return ky return this.apiInstance().then((api) =>
.get(`${CORTEX_API_URL}/healthz`, { api
retry: { .get('healthz', {
limit: 20, retry: {
delay: () => 500, limit: 20,
methods: ['get'], delay: () => 500,
}, methods: ['get'],
}) },
.then(() => {}) })
.then(() => {})
)
} }
/** /**
@ -215,13 +248,15 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
* @returns * @returns
*/ */
private async clean(): Promise<any> { private async clean(): Promise<any> {
return ky return this.apiInstance()
.delete(`${CORTEX_API_URL}/processmanager/destroy`, { .then((api) =>
timeout: 2000, // maximum 2 seconds api.delete('processmanager/destroy', {
retry: { timeout: 2000, // maximum 2 seconds
limit: 0, retry: {
}, limit: 0,
}) },
})
)
.catch(() => { .catch(() => {
// Do nothing // Do nothing
}) })

5
extensions/inference-cortex-extension/src/node/index.ts
View File

@ -44,8 +44,9 @@ function run(): Promise<any> {
`${path.join(dataFolderPath, '.janrc')}`, `${path.join(dataFolderPath, '.janrc')}`,
'--data_folder_path', '--data_folder_path',
dataFolderPath, dataFolderPath,
'--loglevel', 'config',
'INFO', '--api_keys',
process.env.appToken ?? 'cortex.cpp',
], ],
{ {
env: { env: {

2
extensions/model-extension/rolldown.config.mjs
View File

@ -11,7 +11,7 @@ export default defineConfig({
platform: 'browser', platform: 'browser',
define: { define: {
SETTINGS: JSON.stringify(settingJson), SETTINGS: JSON.stringify(settingJson),
API_URL: JSON.stringify(`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? "39291"}`), CORTEX_API_URL: JSON.stringify(`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? "39291"}`),
DEFAULT_MODEL_SOURCES: JSON.stringify(modelSources), DEFAULT_MODEL_SOURCES: JSON.stringify(modelSources),
}, },
}) })

2
extensions/model-extension/src/@types/global.d.ts vendored
View File

@ -1,5 +1,5 @@
declare const NODE: string declare const NODE: string
declare const API_URL: string declare const CORTEX_API_URL: string
declare const SETTINGS: SettingComponentProps[] declare const SETTINGS: SettingComponentProps[]
declare const DEFAULT_MODEL_SOURCES: any declare const DEFAULT_MODEL_SOURCES: any

186
extensions/model-extension/src/index.ts
View File

@ -13,7 +13,7 @@ import {
import { scanModelsFolder } from './legacy/model-json' import { scanModelsFolder } from './legacy/model-json'
import { deleteModelFiles } from './legacy/delete' import { deleteModelFiles } from './legacy/delete'
import PQueue from 'p-queue' import PQueue from 'p-queue'
import ky from 'ky' import ky, { KyInstance } from 'ky'
/** /**
* cortex.cpp setting keys * cortex.cpp setting keys
@ -32,9 +32,25 @@ type Data<T> = {
*/ */
export default class JanModelExtension extends ModelExtension { export default class JanModelExtension extends ModelExtension {
queue = new PQueue({ concurrency: 1 }) queue = new PQueue({ concurrency: 1 })
api?: KyInstance
/**
* Get the API instance
* @returns
*/
async apiInstance(): Promise<KyInstance> {
if(this.api) return this.api
const apiKey = (await window.core?.api.appToken()) ?? 'cortex.cpp'
this.api = ky.extend({
prefixUrl: CORTEX_API_URL,
headers: {
Authorization: `Bearer ${apiKey}`,
},
})
return this.api
}
/** /**
* Called when the extension is loaded. * Called when the extension is loaded.
* @override
*/ */
async onLoad() { async onLoad() {
this.queue.add(() => this.healthz()) this.queue.add(() => this.healthz())
@ -82,13 +98,15 @@ export default class JanModelExtension extends ModelExtension {
* Sending POST to /models/pull/{id} endpoint to pull the model * Sending POST to /models/pull/{id} endpoint to pull the model
*/ */
return this.queue.add(() => return this.queue.add(() =>
ky this.apiInstance().then((api) =>
.post(`${API_URL}/v1/models/pull`, { json: { model, id, name } }) api
.json() .post('v1/models/pull', { json: { model, id, name }, timeout: false })
.catch(async (e) => { .json()
throw (await e.response?.json()) ?? e .catch(async (e) => {
}) throw (await e.response?.json()) ?? e
.then() })
.then()
)
) )
} }
@ -103,10 +121,12 @@ export default class JanModelExtension extends ModelExtension {
* Sending DELETE to /models/pull/{id} endpoint to cancel a model pull * Sending DELETE to /models/pull/{id} endpoint to cancel a model pull
*/ */
return this.queue.add(() => return this.queue.add(() =>
ky this.apiInstance().then((api) =>
.delete(`${API_URL}/v1/models/pull`, { json: { taskId: model } }) api
.json() .delete('v1/models/pull', { json: { taskId: model } })
.then() .json()
.then()
)
) )
} }
@ -117,7 +137,11 @@ export default class JanModelExtension extends ModelExtension {
*/ */
async deleteModel(model: string): Promise<void> { async deleteModel(model: string): Promise<void> {
return this.queue return this.queue
.add(() => ky.delete(`${API_URL}/v1/models/${model}`).json().then()) .add(() =>
this.apiInstance().then((api) =>
api.delete(`v1/models/${model}`).json().then()
)
)
.catch((e) => console.debug(e)) .catch((e) => console.debug(e))
.finally(async () => { .finally(async () => {
// Delete legacy model files // Delete legacy model files
@ -219,10 +243,15 @@ export default class JanModelExtension extends ModelExtension {
async updateModel(model: Partial<Model>): Promise<Model> { async updateModel(model: Partial<Model>): Promise<Model> {
return this.queue return this.queue
.add(() => .add(() =>
ky this.apiInstance().then((api) =>
.patch(`${API_URL}/v1/models/${model.id}`, { json: { ...model } }) api
.json() .patch(`v1/models/${model.id}`, {
.then() json: { ...model },
timeout: false,
})
.json()
.then()
)
) )
.then(() => this.getModel(model.id)) .then(() => this.getModel(model.id))
} }
@ -233,10 +262,12 @@ export default class JanModelExtension extends ModelExtension {
*/ */
async getModel(model: string): Promise<Model> { async getModel(model: string): Promise<Model> {
return this.queue.add(() => return this.queue.add(() =>
ky this.apiInstance().then((api) =>
.get(`${API_URL}/v1/models/${model}`) api
.json() .get(`v1/models/${model}`)
.then((e) => this.transformModel(e)) .json()
.then((e) => this.transformModel(e))
)
) as Promise<Model> ) as Promise<Model>
} }
@ -252,13 +283,16 @@ export default class JanModelExtension extends ModelExtension {
option?: OptionType option?: OptionType
): Promise<void> { ): Promise<void> {
return this.queue.add(() => return this.queue.add(() =>
ky this.apiInstance().then((api) =>
.post(`${API_URL}/v1/models/import`, { api
json: { model, modelPath, name, option }, .post('v1/models/import', {
}) json: { model, modelPath, name, option },
.json() timeout: false,
.catch((e) => console.debug(e)) // Ignore error })
.then() .json()
.catch((e) => console.debug(e)) // Ignore error
.then()
)
) )
} }
@ -269,7 +303,11 @@ export default class JanModelExtension extends ModelExtension {
*/ */
async getSources(): Promise<ModelSource[]> { async getSources(): Promise<ModelSource[]> {
const sources = await this.queue const sources = await this.queue
.add(() => ky.get(`${API_URL}/v1/models/sources`).json<Data<ModelSource>>()) .add(() =>
this.apiInstance().then((api) =>
api.get('v1/models/sources').json<Data<ModelSource>>()
)
)
.then((e) => (typeof e === 'object' ? (e.data as ModelSource[]) : [])) .then((e) => (typeof e === 'object' ? (e.data as ModelSource[]) : []))
.catch(() => []) .catch(() => [])
return sources.concat( return sources.concat(
@ -283,11 +321,13 @@ export default class JanModelExtension extends ModelExtension {
*/ */
async addSource(source: string): Promise<any> { async addSource(source: string): Promise<any> {
return this.queue.add(() => return this.queue.add(() =>
ky.post(`${API_URL}/v1/models/sources`, { this.apiInstance().then((api) =>
json: { api.post('v1/models/sources', {
source, json: {
}, source,
}) },
})
)
) )
} }
@ -297,11 +337,14 @@ export default class JanModelExtension extends ModelExtension {
*/ */
async deleteSource(source: string): Promise<any> { async deleteSource(source: string): Promise<any> {
return this.queue.add(() => return this.queue.add(() =>
ky.delete(`${API_URL}/v1/models/sources`, { this.apiInstance().then((api) =>
json: { api.delete('v1/models/sources', {
source, json: {
}, source,
}) },
timeout: false,
})
)
) )
} }
// END - Model Sources // END - Model Sources
@ -312,7 +355,9 @@ export default class JanModelExtension extends ModelExtension {
*/ */
async isModelLoaded(model: string): Promise<boolean> { async isModelLoaded(model: string): Promise<boolean> {
return this.queue return this.queue
.add(() => ky.get(`${API_URL}/v1/models/status/${model}`)) .add(() =>
this.apiInstance().then((api) => api.get(`v1/models/status/${model}`))
)
.then((e) => true) .then((e) => true)
.catch(() => false) .catch(() => false)
} }
@ -324,14 +369,18 @@ export default class JanModelExtension extends ModelExtension {
return this.updateCortexConfig(options).catch((e) => console.debug(e)) return this.updateCortexConfig(options).catch((e) => console.debug(e))
} }
/** /**
* Fetches models list from cortex.cpp * Fetches models list from cortex.cpp
* @param model * @param model
* @returns * @returns
*/ */
async fetchModels(): Promise<Model[]> { async fetchModels(): Promise<Model[]> {
return this.queue return this.queue
.add(() => ky.get(`${API_URL}/v1/models?limit=-1`).json<Data<Model>>()) .add(() =>
this.apiInstance().then((api) =>
api.get('v1/models?limit=-1').json<Data<Model>>()
)
)
.then((e) => .then((e) =>
typeof e === 'object' ? e.data.map((e) => this.transformModel(e)) : [] typeof e === 'object' ? e.data.map((e) => this.transformModel(e)) : []
) )
@ -371,7 +420,9 @@ export default class JanModelExtension extends ModelExtension {
}): Promise<void> { }): Promise<void> {
return this.queue return this.queue
.add(() => .add(() =>
ky.patch(`${API_URL}/v1/configs`, { json: body }).then(() => {}) this.apiInstance().then((api) =>
api.patch('v1/configs', { json: body }).then(() => {})
)
) )
.catch((e) => console.debug(e)) .catch((e) => console.debug(e))
} }
@ -381,14 +432,16 @@ export default class JanModelExtension extends ModelExtension {
* @returns * @returns
*/ */
private healthz(): Promise<void> { private healthz(): Promise<void> {
return ky return this.apiInstance()
.get(`${API_URL}/healthz`, { .then((api) =>
retry: { api.get('healthz', {
limit: 20, retry: {
delay: () => 500, limit: 20,
methods: ['get'], delay: () => 500,
}, methods: ['get'],
}) },
})
)
.then(() => { .then(() => {
this.queue.concurrency = Infinity this.queue.concurrency = Infinity
}) })
@ -401,17 +454,22 @@ export default class JanModelExtension extends ModelExtension {
const models = await this.fetchModels() const models = await this.fetchModels()
return this.queue.add(() => return this.queue.add(() =>
ky this.apiInstance()
.get(`${API_URL}/v1/models/hub?author=cortexso&tag=cortex.cpp`) .then((api) =>
.json<Data<string>>() api
.then((e) => { .get('v1/models/hub?author=cortexso&tag=cortex.cpp')
e.data?.forEach((model) => { .json<Data<string>>()
if ( .then((e) => {
!models.some((e) => 'modelSource' in e && e.modelSource === model) e.data?.forEach((model) => {
) if (
this.addSource(model).catch((e) => console.debug(e)) !models.some(
}) (e) => 'modelSource' in e && e.modelSource === model
}) )
)
this.addSource(model).catch((e) => console.debug(e))
})
})
)
.catch((e) => console.debug(e)) .catch((e) => console.debug(e))
) )
} }

232
server/cortex.json
View File

@ -3633,238 +3633,6 @@
}, },
"tags": ["Files"] "tags": ["Files"]
} }
},
"/configs": {
"get": {
"summary": "Get Configurations",
"description": "Retrieves the current configuration settings of the Cortex server.",
"responses": {
"200": {
"description": "Successful response",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"allowed_origins": {
"type": "array",
"items": {
"type": "string"
},
"example": ["http://127.0.0.1:39281", "https://cortex.so"]
},
"cors": {
"type": "boolean",
"example": false
},
"proxy_username": {
"type": "string",
"example": "username"
},
"proxy_password": {
"type": "string",
"example": "password"
},
"proxy_url": {
"type": "string",
"example": "http://proxy.example.com:8080"
},
"verify_proxy_ssl": {
"type": "boolean",
"description": "test",
"example": false
},
"verify_proxy_host_ssl": {
"type": "boolean",
"example": false
},
"verify_peer_ssl": {
"type": "boolean",
"example": false
},
"verify_host_ssl": {
"type": "boolean",
"example": false
},
"no_proxy": {
"type": "string",
"example": "localhost"
},
"huggingface_token": {
"type": "string",
"example": "your_token"
}
}
},
"example": {
"allowed_origins": [
"http://127.0.0.1:39281",
"https://cortex.so"
],
"cors": false,
"proxy_username": "username",
"proxy_password": "password",
"proxy_url": "http://proxy.example.com:8080",
"verify_proxy_ssl": false,
"verify_proxy_host_ssl": false,
"verify_peer_ssl": false,
"verify_host_ssl": false,
"no_proxy": "localhost",
"huggingface_token": "your_token"
}
}
}
}
},
"tags": ["Configurations"]
},
"patch": {
"tags": ["Configurations"],
"summary": "Update configuration settings",
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"cors": {
"type": "boolean",
"description": "Indicates whether CORS is enabled.",
"example": false
},
"allowed_origins": {
"type": "array",
"items": {
"type": "string"
},
"description": "List of allowed origins.",
"example": ["http://127.0.0.1:39281", "https://cortex.so"]
},
"proxy_username": {
"type": "string",
"description": "Username for the proxy server.",
"example": "username"
},
"proxy_password": {
"type": "string",
"description": "Password for the proxy server.",
"example": "password"
},
"proxy_url": {
"type": "string",
"description": "URL for the proxy server.",
"example": "http://proxy.example.com:8080"
},
"verify_proxy_ssl": {
"type": "boolean",
"description": "Indicates whether to verify the SSL certificate of the proxy server.",
"example": false
},
"verify_proxy_host_ssl": {
"type": "boolean",
"description": "Indicates whether to verify the SSL certificate of the proxy server host.",
"example": false
},
"verify_peer_ssl": {
"type": "boolean",
"description": "Indicates whether to verify the SSL certificate of the peer.",
"example": false
},
"verify_host_ssl": {
"type": "boolean",
"description": "Indicates whether to verify the SSL certificate of the host.",
"example": false
},
"no_proxy": {
"type": "string",
"description": "List of hosts that should not be proxied.",
"example": "localhost"
},
"huggingface_token": {
"type": "string",
"description": "HuggingFace token to pull models.",
"example": "your_token"
}
}
}
}
}
},
"responses": {
"200": {
"description": "Configuration updated successfully",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"config": {
"type": "object",
"properties": {
"allowed_origins": {
"type": "array",
"items": {
"type": "string"
},
"example": [
"http://127.0.0.1:39281",
"https://cortex.so"
]
},
"cors": {
"type": "boolean",
"example": false
},
"proxy_username": {
"type": "string",
"example": "username"
},
"proxy_password": {
"type": "string",
"example": "password"
},
"proxy_url": {
"type": "string",
"example": "http://proxy.example.com:8080"
},
"verify_proxy_ssl": {
"type": "boolean",
"example": false
},
"verify_proxy_host_ssl": {
"type": "boolean",
"example": false
},
"verify_peer_ssl": {
"type": "boolean",
"example": false
},
"verify_host_ssl": {
"type": "boolean",
"example": false
},
"no_proxy": {
"type": "string",
"example": "localhost"
},
"huggingface_token": {
"type": "string",
"example": "your_token"
}
}
},
"message": {
"type": "string",
"example": "Configuration updated successfully"
}
}
}
}
}
}
}
}
} }
}, },
"info": { "info": {

36
server/index.ts
View File

@ -86,6 +86,14 @@ export const startServer = async (configs?: ServerConfig): Promise<boolean> => {
}, },
}) })
const rewriteRequestHeaders = (req: any, headers: any) => {
if (req.url.includes('/configs')) return headers
return {
...headers,
authorization: `Bearer ${process.env.appToken}`, // Add or modify Authorization header
}
}
// Register Swagger UI // Register Swagger UI
await server.register(require('@fastify/swagger-ui'), { await server.register(require('@fastify/swagger-ui'), {
routePrefix: '/', routePrefix: '/',
@ -102,24 +110,36 @@ export const startServer = async (configs?: ServerConfig): Promise<boolean> => {
upstream: `${CORTEX_API_URL}/v1`, upstream: `${CORTEX_API_URL}/v1`,
prefix: configs?.prefix ?? '/v1', prefix: configs?.prefix ?? '/v1',
http2: false, http2: false,
}) replyOptions: {
rewriteRequestHeaders,
server.register(proxy, { },
upstream: `${CORTEX_API_URL}/system`,
prefix:'/system',
http2: false,
}) })
server.register(proxy, { server.register(proxy, {
upstream: `${CORTEX_API_URL}/processManager`, upstream: `${CORTEX_API_URL}/processManager`,
prefix:'/processManager', prefix: '/processManager',
http2: false, http2: false,
replyOptions: {
rewriteRequestHeaders,
},
})
server.register(proxy, {
upstream: `${CORTEX_API_URL}/system`,
prefix: '/system',
http2: false,
replyOptions: {
rewriteRequestHeaders,
},
}) })
server.register(proxy, { server.register(proxy, {
upstream: `${CORTEX_API_URL}/healthz`, upstream: `${CORTEX_API_URL}/healthz`,
prefix:'/healthz', prefix: '/healthz',
http2: false, http2: false,
replyOptions: {
rewriteRequestHeaders,
},
}) })
// Start listening for requests // Start listening for requests

3
web/containers/Providers/DataLoader.tsx
View File

@ -15,6 +15,7 @@ import { useDebouncedCallback } from 'use-debounce'
import useAssistants from '@/hooks/useAssistants' import useAssistants from '@/hooks/useAssistants'
import { useGetEngines } from '@/hooks/useEngineManagement' import { useGetEngines } from '@/hooks/useEngineManagement'
import useGetSystemResources from '@/hooks/useGetSystemResources' import useGetSystemResources from '@/hooks/useGetSystemResources'
import { useGetHardwareInfo } from '@/hooks/useHardwareManagement'
import useModels from '@/hooks/useModels' import useModels from '@/hooks/useModels'
import useThreads from '@/hooks/useThreads' import useThreads from '@/hooks/useThreads'
@ -34,6 +35,7 @@ const DataLoader: React.FC = () => {
const setJanSettingScreen = useSetAtom(janSettingScreenAtom) const setJanSettingScreen = useSetAtom(janSettingScreenAtom)
const { getData: loadModels } = useModels() const { getData: loadModels } = useModels()
const { mutate } = useGetEngines() const { mutate } = useGetEngines()
const { mutate: getHardwareInfo } = useGetHardwareInfo(false)
useThreads() useThreads()
useAssistants() useAssistants()
@ -42,6 +44,7 @@ const DataLoader: React.FC = () => {
useEffect(() => { useEffect(() => {
// Load data once // Load data once
loadModels() loadModels()
getHardwareInfo()
// eslint-disable-next-line react-hooks/exhaustive-deps // eslint-disable-next-line react-hooks/exhaustive-deps
}, []) }, [])
const reloadData = useDebouncedCallback(() => { const reloadData = useDebouncedCallback(() => {

4
web/hooks/useHardwareManagement.ts
View File

@ -32,7 +32,7 @@ const getExtension = () =>
/** /**
* @returns A Promise that resolves to an object of list engines. * @returns A Promise that resolves to an object of list engines.
*/ */
export function useGetHardwareInfo() { export function useGetHardwareInfo(updatePeriodically: boolean = true) {
const setCpuUsage = useSetAtom(cpuUsageAtom) const setCpuUsage = useSetAtom(cpuUsageAtom)
const setUsedRam = useSetAtom(usedRamAtom) const setUsedRam = useSetAtom(usedRamAtom)
const setTotalRam = useSetAtom(totalRamAtom) const setTotalRam = useSetAtom(totalRamAtom)
@ -56,7 +56,7 @@ export function useGetHardwareInfo() {
{ {
revalidateOnFocus: false, revalidateOnFocus: false,
revalidateOnReconnect: false, revalidateOnReconnect: false,
refreshInterval: 2000, refreshInterval: updatePeriodically ? 2000 : undefined,
} }
) )

2
web/screens/Hub/index.tsx
View File

@ -80,7 +80,7 @@ const filterOptions = [
}, },
] ]
const hubCompatibleAtom = atom(true) const hubCompatibleAtom = atom(false)
const HubScreen = () => { const HubScreen = () => {
const { sources } = useGetModelSources() const { sources } = useGetModelSources()

10
web/screens/Settings/Hardware/index.tsx
View File

@ -6,13 +6,14 @@ import { useState } from 'react'
import { DragDropContext, Draggable, Droppable } from '@hello-pangea/dnd' import { DragDropContext, Draggable, Droppable } from '@hello-pangea/dnd'
import { Progress, ScrollArea, Switch } from '@janhq/joi' import { Progress, ScrollArea, Switch } from '@janhq/joi'
import { useAtom, useAtomValue } from 'jotai' import { useAtom, useAtomValue, useSetAtom } from 'jotai'
import { atomWithStorage } from 'jotai/utils' import { atomWithStorage } from 'jotai/utils'
import { ChevronDownIcon, GripVerticalIcon } from 'lucide-react' import { ChevronDownIcon, GripVerticalIcon } from 'lucide-react'
import { twMerge } from 'tailwind-merge' import { twMerge } from 'tailwind-merge'
import { activeModelAtom } from '@/hooks/useActiveModel'
import { import {
useGetHardwareInfo, useGetHardwareInfo,
setActiveGpus, setActiveGpus,
@ -47,6 +48,7 @@ const Hardware = () => {
const ramUtilitized = useAtomValue(ramUtilitizedAtom) const ramUtilitized = useAtomValue(ramUtilitizedAtom)
const showScrollBar = useAtomValue(showScrollBarAtom) const showScrollBar = useAtomValue(showScrollBarAtom)
const [gpus, setGpus] = useAtom(gpusAtom) const [gpus, setGpus] = useAtom(gpusAtom)
const setActiveModel = useSetAtom(activeModelAtom)
const [orderGpus, setOrderGpus] = useAtom(orderGpusAtom) const [orderGpus, setOrderGpus] = useAtom(orderGpusAtom)
@ -70,11 +72,15 @@ const Hardware = () => {
.filter((gpu: any) => gpu.activated) .filter((gpu: any) => gpu.activated)
.map((gpu: any) => Number(gpu.id)) .map((gpu: any) => Number(gpu.id))
await setActiveGpus({ gpus: activeGpuIds }) await setActiveGpus({ gpus: activeGpuIds })
setActiveModel(undefined)
mutate() mutate()
window.location.reload()
} catch (error) { } catch (error) {
console.error('Failed to update active GPUs:', error) console.error('Failed to update active GPUs:', error)
} }
setIsActivatingGpu((prev) => {
prev.delete(id)
return new Set(prev)
})
} }
const handleDragEnd = (result: any) => { const handleDragEnd = (result: any) => {