Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'dev' into patch-1

Browse Source
This commit is contained in:
Ashu 2024-11-13 23:03:32 +05:30 committed by GitHub
commit 44b7198bdb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
29 changed files with 258 additions and 191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.github/workflows/jan-electron-build-nightly.yml vendored
View File

@ -114,15 +114,14 @@ jobs:
- name: Upload latest-mac.yml
if: ${{ needs.set-public-provider.outputs.public_provider == 'aws-s3' }}
run: |
aws s3 cp ./latest-mac.yml "s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/temp-latest/latest-mac.yml"
aws s3 sync s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/temp-latest/ s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/latest/
aws s3 cp ./latest-mac.yml "s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/temp-nightly/latest-mac.yml"
aws s3 sync s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/temp-nightly/ s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/nightly/
env:
AWS_ACCESS_KEY_ID: ${{ secrets.DELTA_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.DELTA_AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.DELTA_AWS_REGION }}
AWS_EC2_METADATA_DISABLED: "true"
noti-discord-nightly-and-update-url-readme:
needs: [build-macos-x64, build-macos-arm64, build-windows-x64, build-linux-x64, get-update-version, set-public-provider, combine-latest-mac-yml]
secrets: inherit

16
.github/workflows/template-build-linux-x64.yml vendored
View File

@ -60,16 +60,16 @@ jobs:
mv /tmp/package.json electron/package.json
jq --arg version "${{ inputs.new_version }}" '.version = $version' web/package.json > /tmp/package.json
mv /tmp/package.json web/package.json
jq '.build.publish = [{"provider": "generic", "url": "https://delta.jan.ai/latest", "channel": "latest"}, {"provider": "s3", "acl": null, "bucket": "${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}", "region": "${{ secrets.DELTA_AWS_REGION}}", "path": "temp-latest", "channel": "latest"}]' electron/package.json > /tmp/package.json
jq '.build.publish = [{"provider": "generic", "url": "https://delta.jan.ai/nightly", "channel": "latest"}, {"provider": "s3", "acl": null, "bucket": "${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}", "region": "${{ secrets.DELTA_AWS_REGION}}", "path": "temp-nightly", "channel": "latest"}]' electron/package.json > /tmp/package.json
mv /tmp/package.json electron/package.json
cat electron/package.json
# chmod +x .github/scripts/rename-app.sh
# .github/scripts/rename-app.sh ./electron/package.json nightly
# chmod +x .github/scripts/rename-workspace.sh
# .github/scripts/rename-workspace.sh ./package.json nightly
# echo "------------------------"
# cat ./electron/package.json
# echo "------------------------"
chmod +x .github/scripts/rename-app.sh
.github/scripts/rename-app.sh ./electron/package.json nightly
chmod +x .github/scripts/rename-workspace.sh
.github/scripts/rename-workspace.sh ./package.json nightly
echo "------------------------"
cat ./electron/package.json
echo "------------------------"
- name: Change App Name for beta version
if: inputs.beta == true

16
.github/workflows/template-build-macos-arm64.yml vendored
View File

@ -72,20 +72,20 @@ jobs:
jq --arg version "${{ inputs.new_version }}" '.version = $version' web/package.json > /tmp/package.json
mv /tmp/package.json web/package.json
jq '.build.publish = [{"provider": "generic", "url": "https://delta.jan.ai/latest", "channel": "latest"}, {"provider": "s3", "acl": null, "bucket": "${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}", "region": "${{ secrets.DELTA_AWS_REGION}}", "path": "temp-latest", "channel": "latest"}]' electron/package.json > /tmp/package.json
jq '.build.publish = [{"provider": "generic", "url": "https://delta.jan.ai/nightly", "channel": "latest"}, {"provider": "s3", "acl": null, "bucket": "${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}", "region": "${{ secrets.DELTA_AWS_REGION}}", "path": "temp-nightly", "channel": "latest"}]' electron/package.json > /tmp/package.json
mv /tmp/package.json electron/package.json
jq --arg teamid "${{ secrets.APPLE_TEAM_ID }}" '.build.mac.notarize.teamId = $teamid' electron/package.json > /tmp/package.json
mv /tmp/package.json electron/package.json
# cat electron/package.json
# chmod +x .github/scripts/rename-app.sh
# .github/scripts/rename-app.sh ./electron/package.json nightly
# chmod +x .github/scripts/rename-workspace.sh
# .github/scripts/rename-workspace.sh ./package.json nightly
# echo "------------------------"
# cat ./electron/package.json
# echo "------------------------"
chmod +x .github/scripts/rename-app.sh
.github/scripts/rename-app.sh ./electron/package.json nightly
chmod +x .github/scripts/rename-workspace.sh
.github/scripts/rename-workspace.sh ./package.json nightly
echo "------------------------"
cat ./electron/package.json
echo "------------------------"
- name: Change App Name for beta version
if: inputs.beta == true

18
.github/workflows/template-build-macos-x64.yml vendored
View File

@ -72,20 +72,20 @@ jobs:
jq --arg version "${{ inputs.new_version }}" '.version = $version' web/package.json > /tmp/package.json
mv /tmp/package.json web/package.json
jq '.build.publish = [{"provider": "generic", "url": "https://delta.jan.ai/latest", "channel": "latest"}, {"provider": "s3", "acl": null, "bucket": "${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}", "region": "${{ secrets.DELTA_AWS_REGION}}", "path": "temp-latest", "channel": "latest"}]' electron/package.json > /tmp/package.json
jq '.build.publish = [{"provider": "generic", "url": "https://delta.jan.ai/nightly", "channel": "latest"}, {"provider": "s3", "acl": null, "bucket": "${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}", "region": "${{ secrets.DELTA_AWS_REGION}}", "path": "temp-nightly", "channel": "latest"}]' electron/package.json > /tmp/package.json
mv /tmp/package.json electron/package.json
jq --arg teamid "${{ secrets.APPLE_TEAM_ID }}" '.build.mac.notarize.teamId = $teamid' electron/package.json > /tmp/package.json
mv /tmp/package.json electron/package.json
# cat electron/package.json
# chmod +x .github/scripts/rename-app.sh
# .github/scripts/rename-app.sh ./electron/package.json nightly
# chmod +x .github/scripts/rename-workspace.sh
# .github/scripts/rename-workspace.sh ./package.json nightly
# echo "------------------------"
# cat ./electron/package.json
# echo "------------------------"
cat electron/package.json
chmod +x .github/scripts/rename-app.sh
.github/scripts/rename-app.sh ./electron/package.json nightly
chmod +x .github/scripts/rename-workspace.sh
.github/scripts/rename-workspace.sh ./package.json nightly
echo "------------------------"
cat ./electron/package.json
echo "------------------------"
- name: Change App Name for beta version
if: inputs.beta == true

24
.github/workflows/template-build-windows-x64.yml vendored
View File

@ -73,24 +73,24 @@ jobs:
jq --arg version "${{ inputs.new_version }}" '.version = $version' web/package.json > /tmp/package.json
mv /tmp/package.json web/package.json
jq '.build.publish = [{"provider": "generic", "url": "https://delta.jan.ai/latest", "channel": "latest"}, {"provider": "s3", "acl": null, "bucket": "${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}", "region": "${{ secrets.DELTA_AWS_REGION}}", "path": "temp-latest", "channel": "latest"}]' electron/package.json > /tmp/package.json
jq '.build.publish = [{"provider": "generic", "url": "https://delta.jan.ai/nightly", "channel": "latest"}, {"provider": "s3", "acl": null, "bucket": "${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}", "region": "${{ secrets.DELTA_AWS_REGION}}", "path": "temp-nightly", "channel": "latest"}]' electron/package.json > /tmp/package.json
mv /tmp/package.json electron/package.json
jq '.build.win.sign = "./sign.js"' electron/package.json > /tmp/package.json
mv /tmp/package.json electron/package.json
cat electron/package.json
# chmod +x .github/scripts/rename-app.sh
# .github/scripts/rename-app.sh ./electron/package.json nightly
# chmod +x .github/scripts/rename-workspace.sh
# .github/scripts/rename-workspace.sh ./package.json nightly
# chmod +x .github/scripts/rename-uninstaller.sh
# .github/scripts/rename-uninstaller.sh nightly
# echo "------------------------"
# cat ./electron/package.json
# echo "------------------------"
# cat ./package.json
# echo "------------------------"
chmod +x .github/scripts/rename-app.sh
.github/scripts/rename-app.sh ./electron/package.json nightly
chmod +x .github/scripts/rename-workspace.sh
.github/scripts/rename-workspace.sh ./package.json nightly
chmod +x .github/scripts/rename-uninstaller.sh
.github/scripts/rename-uninstaller.sh nightly
echo "------------------------"
cat ./electron/package.json
echo "------------------------"
cat ./package.json
echo "------------------------"
- name: Change App Name for beta version
if: inputs.beta == true

10
.github/workflows/template-noti-discord-and-update-url-readme.yml vendored
View File

@ -47,11 +47,11 @@ jobs:
with:
args: |
Jan App ${{ inputs.build_reason }} build artifact version {{ VERSION }}:
- Windows: https://delta.jan.ai/latest/jan-win-x64-{{ VERSION }}.exe
- macOS Intel: https://delta.jan.ai/latest/jan-mac-x64-{{ VERSION }}.dmg
- macOS Apple Silicon: https://delta.jan.ai/latest/jan-mac-arm64-{{ VERSION }}.dmg
- Linux Deb: https://delta.jan.ai/latest/jan-linux-amd64-{{ VERSION }}.deb
- Linux AppImage: https://delta.jan.ai/latest/jan-linux-x86_64-{{ VERSION }}.AppImage
- Windows: https://delta.jan.ai/nightly/jan-win-x64-{{ VERSION }}.exe
- macOS Intel: https://delta.jan.ai/nightly/jan-mac-x64-{{ VERSION }}.dmg
- macOS Apple Silicon: https://delta.jan.ai/nightly/jan-mac-arm64-{{ VERSION }}.dmg
- Linux Deb: https://delta.jan.ai/nightly/jan-linux-amd64-{{ VERSION }}.deb
- Linux AppImage: https://delta.jan.ai/nightly/jan-linux-x86_64-{{ VERSION }}.AppImage
- Github action run: https://github.com/janhq/jan/actions/runs/{{ GITHUB_RUN_ID }}
env:
DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}

3
core/src/node/api/processors/download.ts
View File

@ -1,6 +1,6 @@
import { resolve, sep } from 'path'
import { DownloadEvent } from '../../../types/api'
import { normalizeFilePath, validatePath } from '../../helper/path'
import { normalizeFilePath } from '../../helper/path'
import { getJanDataFolderPath } from '../../helper'
import { DownloadManager } from '../../helper/download'
import { createWriteStream, renameSync } from 'fs'
@ -37,7 +37,6 @@ export class Downloader implements Processor {
const modelId = downloadRequest.modelId ?? array.pop() ?? ''
const destination = resolve(getJanDataFolderPath(), normalizedPath)
validatePath(destination)
const rq = request({ url, strictSSL, proxy })
// Put request to download manager instance

5
core/src/node/api/processors/fs.ts
View File

@ -1,5 +1,5 @@
import { join, resolve } from 'path'
import { normalizeFilePath, validatePath } from '../../helper/path'
import { normalizeFilePath } from '../../helper/path'
import { getJanDataFolderPath } from '../../helper'
import { Processor } from './Processor'
import fs from 'fs'
@ -36,7 +36,6 @@ export class FileSystem implements Processor {
return path
}
const absolutePath = resolve(path)
validatePath(absolutePath)
return absolutePath
})
)
@ -55,7 +54,6 @@ export class FileSystem implements Processor {
}
const absolutePath = resolve(path)
validatePath(absolutePath)
return new Promise((resolve, reject) => {
fs.rm(absolutePath, { recursive: true, force: true }, (err) => {
@ -79,7 +77,6 @@ export class FileSystem implements Processor {
}
const absolutePath = resolve(path)
validatePath(absolutePath)
return new Promise((resolve, reject) => {
fs.mkdir(absolutePath, { recursive: true }, (err) => {

4
core/src/node/api/processors/fsExt.ts
View File

@ -1,6 +1,6 @@
import { basename, join } from 'path'
import fs, { readdirSync } from 'fs'
import { appResourcePath, normalizeFilePath, validatePath } from '../../helper/path'
import { appResourcePath, normalizeFilePath } from '../../helper/path'
import { defaultAppConfig, getJanDataFolderPath, getJanDataFolderPath as getPath } from '../../helper'
import { Processor } from './Processor'
import { FileStat } from '../../../types'
@ -61,7 +61,6 @@ export class FSExt implements Processor {
const dataBuffer = Buffer.from(data, 'base64')
const writePath = join(getJanDataFolderPath(), normalizedPath)
validatePath(writePath)
fs.writeFileSync(writePath, dataBuffer)
} catch (err) {
console.error(`writeFile ${path} result: ${err}`)
@ -69,7 +68,6 @@ export class FSExt implements Processor {
}
copyFile(src: string, dest: string): Promise<void> {
validatePath(dest)
return new Promise((resolve, reject) => {
fs.copyFile(src, dest, (err) => {
if (err) {

14
core/src/node/helper/path.ts
View File

@ -35,17 +35,3 @@ export function appResourcePath() {
// server
return join(global.core.appPath(), '../../..')
}
export function validatePath(path: string) {
const appDataFolderPath = getJanDataFolderPath()
const resourcePath = appResourcePath()
const applicationSupportPath = global.core?.appPath() ?? resourcePath
const absolutePath = resolve(__dirname, path)
if (
![appDataFolderPath, resourcePath, applicationSupportPath].some((whiteListedPath) =>
absolutePath.startsWith(whiteListedPath)
)
) {
throw new Error(`Invalid path: ${absolutePath}`)
}
}

3
electron/utils/migration.ts
View File

@ -61,6 +61,9 @@ async function checkAndMigrateTheme(
)
if (existingTheme) {
const desTheme = join(janDataThemesFolder, existingTheme)
if (!lstatSync(desTheme).isDirectory()) {
return
}
console.debug('Updating theme', existingTheme)
rmdirSync(desTheme, { recursive: true })
cpSync(sourceThemePath, join(janDataThemesFolder, sourceThemeName), {

2
extensions/inference-cortex-extension/package.json
View File

@ -1,7 +1,7 @@
{
"name": "@janhq/inference-cortex-extension",
"productName": "Cortex Inference Engine",
"version": "1.0.20",
"version": "1.0.21",
"description": "This extension embeds cortex.cpp, a lightweight inference engine written in C++. See https://jan.ai.\nAdditional dependencies could be installed to run without Cuda Toolkit installation.",
"main": "dist/index.js",
"node": "dist/node/index.cjs.js",

36
extensions/inference-cortex-extension/resources/models/qwen2.5-coder-14b-instruct/model.json Normal file
View File

@ -0,0 +1,36 @@
{
"sources": [
{
"filename": "Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf",
"url": "https://huggingface.co/bartowski/Qwen2.5-Coder-14B-Instruct-GGUF/resolve/main/Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf"
}
],
"id": "qwen2.5-coder-14b-instruct",
"object": "model",
"name": "Qwen2.5 Coder 14B Instruct Q4",
"version": "1.0",
"description": "Qwen2.5-Coder is the latest series of Code-Specific Qwen large language models. Significantly improvements in code generation, code reasoning and code fixing.",
"format": "gguf",
"settings": {
"ctx_len": 32768,
"prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
"llama_model_path": "Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf",
"ngl": 29
},
"parameters": {
"temperature": 0.7,
"top_p": 0.95,
"stream": true,
"max_tokens": 32768,
"stop": ["<|endoftext|>", "<|im_end|>"],
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "QwenLM",
"tags": ["14B", "Featured"],
"size": 8990000000
},
"engine": "llama-cpp"
}

36
extensions/inference-cortex-extension/resources/models/qwen2.5-coder-32b-instruct/model.json Normal file
View File

@ -0,0 +1,36 @@
{
"sources": [
{
"filename": "Qwen2.5-Coder-32B-Instruct-Q4_K_M.gguf",
"url": "https://huggingface.co/bartowski/Qwen2.5-Coder-32B-Instruct-GGUF/resolve/main/Qwen2.5-Coder-32B-Instruct-Q4_K_M.gguf"
}
],
"id": "qwen2.5-coder-32b-instruct",
"object": "model",
"name": "Qwen2.5 Coder 32B Instruct Q4",
"version": "1.0",
"description": "Qwen2.5-Coder is the latest series of Code-Specific Qwen large language models. Significantly improvements in code generation, code reasoning and code fixing.",
"format": "gguf",
"settings": {
"ctx_len": 32768,
"prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
"llama_model_path": "Qwen2.5-Coder-32B-Instruct-Q4_K_M.gguf",
"ngl": 29
},
"parameters": {
"temperature": 0.7,
"top_p": 0.95,
"stream": true,
"max_tokens": 32768,
"stop": ["<|endoftext|>", "<|im_end|>"],
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "QwenLM",
"tags": ["32B", "Featured"],
"size": 19900000000
},
"engine": "llama-cpp"
}

5
extensions/inference-cortex-extension/rollup.config.ts
View File

@ -49,6 +49,8 @@ const llama321bJson = require('./resources/models/llama3.2-1b-instruct/model.jso
const llama323bJson = require('./resources/models/llama3.2-3b-instruct/model.json')
const qwen257bJson = require('./resources/models/qwen2.5-7b-instruct/model.json')
const qwen25coder7bJson = require('./resources/models/qwen2.5-coder-7b-instruct/model.json')
const qwen25coder14bJson = require('./resources/models/qwen2.5-coder-14b-instruct/model.json')
const qwen25coder32bJson = require('./resources/models/qwen2.5-coder-32b-instruct/model.json')
const qwen2514bJson = require('./resources/models/qwen2.5-14b-instruct/model.json')
const qwen2532bJson = require('./resources/models/qwen2.5-32b-instruct/model.json')
const qwen2572bJson = require('./resources/models/qwen2.5-72b-instruct/model.json')
@ -108,6 +110,8 @@ export default [
llama323bJson,
qwen257bJson,
qwen25coder7bJson,
qwen25coder14bJson,
qwen25coder32bJson,
qwen2514bJson,
qwen2532bJson,
qwen2572bJson,
@ -115,6 +119,7 @@ export default [
NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`),
DEFAULT_SETTINGS: JSON.stringify(defaultSettingJson),
CORTEX_API_URL: JSON.stringify('http://127.0.0.1:39291'),
CORTEX_SOCKET_URL: JSON.stringify('ws://127.0.0.1:39291'),
}),
// Allow json resolution
json(),

1
extensions/inference-cortex-extension/src/@types/global.d.ts vendored
View File

@ -1,5 +1,6 @@
declare const NODE: string
declare const CORTEX_API_URL: string
declare const CORTEX_SOCKET_URL: string
declare const DEFAULT_SETTINGS: Array<any>
declare const MODELS: Array<any>

67
extensions/inference-cortex-extension/src/index.ts
View File

@ -16,17 +16,29 @@ import {
getJanDataFolderPath,
extractModelLoadParams,
fs,
events,
ModelEvent
} from '@janhq/core'
import PQueue from 'p-queue'
import ky from 'ky'
/**
* Event subscription types of Downloader
*/
enum DownloadTypes {
DownloadUpdated = 'onFileDownloadUpdate',
DownloadError = 'onFileDownloadError',
DownloadSuccess = 'onFileDownloadSuccess',
DownloadStopped = 'onFileDownloadStopped',
DownloadStarted = 'onFileDownloadStarted',
}
/**
* A class that implements the InferenceExtension interface from the @janhq/core package.
* The class provides methods for initializing and stopping a model, and for making inference requests.
* It also subscribes to events emitted by the @janhq/core package and handles new message requests.
*/
export default class JanInferenceCortexExtension extends LocalOAIEngine {
// DEPRECATED
nodeModule: string = 'node'
queue = new PQueue({ concurrency: 1 })
@ -38,6 +50,11 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
*/
inferenceUrl = `${CORTEX_API_URL}/v1/chat/completions`
/**
* Socket instance of events subscription
*/
socket?: WebSocket = undefined
/**
* Subscribes to events emitted by the @janhq/core package.
*/
@ -55,6 +72,8 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
this.queue.add(() => this.healthz())
this.subscribeToEvents()
window.addEventListener('beforeunload', () => {
this.clean()
})
@ -138,7 +157,7 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
methods: ['get'],
},
})
.then(() => {})
.then(() => { })
}
/**
@ -154,6 +173,50 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
// Do nothing
})
}
/**
* Subscribe to cortex.cpp websocket events
*/
subscribeToEvents() {
this.queue.add(
() =>
new Promise<void>((resolve) => {
this.socket = new WebSocket(`${CORTEX_SOCKET_URL}/events`)
this.socket.addEventListener('message', (event) => {
const data = JSON.parse(event.data)
const transferred = data.task.items.reduce(
(acc: number, cur: any) => acc + cur.downloadedBytes,
0
)
const total = data.task.items.reduce(
(acc: number, cur: any) => acc + cur.bytes,
0
)
const percent = total > 0 ? transferred / total : 0
events.emit(DownloadTypes[data.type as keyof typeof DownloadTypes], {
modelId: data.task.id,
percent: percent,
size: {
transferred: transferred,
total: total,
},
})
// Update models list from Hub
if (data.type === DownloadTypes.DownloadSuccess) {
// Delay for the state update from cortex.cpp
// Just to be sure
setTimeout(() => {
events.emit(ModelEvent.OnModelsUpdate, {})
}, 500)
}
})
resolve()
})
)
}
}
/// Legacy

3
extensions/inference-openai-extension/src/index.ts
View File

@ -76,10 +76,11 @@ export default class JanInferenceOpenAIExtension extends RemoteOAIEngine {
transformPayload = (payload: OpenAIPayloadType): OpenAIPayloadType => {
// Transform the payload for preview models
if (this.previewModels.includes(payload.model)) {
const { max_tokens, ...params } = payload
const { max_tokens, temperature, top_p, stop, ...params } = payload
return {
...params,
max_completion_tokens: max_tokens,
stream: false // o1 only support stream = false
}
}
// Pass through for non-preview models

55
extensions/model-extension/src/cortex.ts
View File

@ -1,6 +1,6 @@
import PQueue from 'p-queue'
import ky from 'ky'
import { events, extractModelLoadParams, Model, ModelEvent } from '@janhq/core'
import { extractModelLoadParams, Model } from '@janhq/core'
import { extractInferenceParams } from '@janhq/core'
/**
* cortex.cpp Model APIs interface
@ -24,21 +24,11 @@ type ModelList = {
data: any[]
}
enum DownloadTypes {
DownloadUpdated = 'onFileDownloadUpdate',
DownloadError = 'onFileDownloadError',
DownloadSuccess = 'onFileDownloadSuccess',
DownloadStopped = 'onFileDownloadStopped',
DownloadStarted = 'onFileDownloadStarted',
}
export class CortexAPI implements ICortexAPI {
queue = new PQueue({ concurrency: 1 })
socket?: WebSocket = undefined
constructor() {
this.queue.add(() => this.healthz())
this.subscribeToEvents()
}
/**
@ -172,49 +162,6 @@ export class CortexAPI implements ICortexAPI {
.then(() => {})
}
/**
* Subscribe to cortex.cpp websocket events
*/
subscribeToEvents() {
this.queue.add(
() =>
new Promise<void>((resolve) => {
this.socket = new WebSocket(`${SOCKET_URL}/events`)
this.socket.addEventListener('message', (event) => {
const data = JSON.parse(event.data)
const transferred = data.task.items.reduce(
(acc, cur) => acc + cur.downloadedBytes,
0
)
const total = data.task.items.reduce(
(acc, cur) => acc + cur.bytes,
0
)
const percent = total > 0 ? transferred / total : 0
events.emit(DownloadTypes[data.type], {
modelId: data.task.id,
percent: percent,
size: {
transferred: transferred,
total: total,
},
})
// Update models list from Hub
if (data.type === DownloadTypes.DownloadSuccess) {
// Delay for the state update from cortex.cpp
// Just to be sure
setTimeout(() => {
events.emit(ModelEvent.OnModelsUpdate, {})
}, 500)
}
})
resolve()
})
)
}
/**
* TRansform model to the expected format (e.g. parameters, settings, metadata)
* @param model

6
extensions/model-extension/src/index.ts
View File

@ -179,8 +179,8 @@ export default class JanModelExtension extends ModelExtension {
if (toImportModels.length > 0) {
// Import models
await Promise.all(
toImportModels.map(async (model: Model & { file_path: string }) =>
this.importModel(
toImportModels.map(async (model: Model & { file_path: string }) => {
return this.importModel(
model.id,
model.sources[0].url.startsWith('http') ||
!(await fs.existsSync(model.sources[0].url))
@ -200,7 +200,7 @@ export default class JanModelExtension extends ModelExtension {
...model.parameters,
} as Partial<Model>)
})
)
})
)
return currentModels

3
web/containers/Layout/BottomPanel/SystemMonitor/index.tsx
View File

@ -79,7 +79,7 @@ const SystemMonitor = () => {
{showSystemMonitorPanel && (
<div
className={twMerge(
'fixed bottom-9 left-[49px] z-50 flex h-[200px] w-[calc(100%-48px)] flex-shrink-0 flex-col border-t border-[hsla(var(--app-border))] bg-[hsla(var(--app-bg))]',
'system-monitor-panel fixed bottom-9 left-[49px] z-50 flex w-[calc(100%-48px)] flex-shrink-0 flex-col border-t border-[hsla(var(--app-border))] bg-[hsla(var(--app-bg))]',
showFullScreen && 'h-[calc(100%-63px)]',
reduceTransparent && 'w-[calc(100%-48px)] rounded-none'
)}
@ -147,7 +147,6 @@ const SystemMonitor = () => {
<span className="flex-shrink-0 ">{ramUtilitized}%</span>
</div>
</div>
{gpus.length > 0 && (
<div className="mb-4 border-b border-[hsla(var(--app-border))] pb-4 last:border-none">
{gpus.map((gpu, index) => {

5
web/containers/ModelDropdown/index.tsx
View File

@ -108,6 +108,11 @@ const ModelDropdown = ({
const filteredDownloadedModels = useMemo(
() =>
configuredModels
.concat(
downloadedModels.filter(
(e) => !configuredModels.some((x) => x.id === e.id)
)
)
.filter((e) =>
e.name.toLowerCase().includes(searchText.toLowerCase().trim())
)

2
web/hooks/useDropModelBinaries.ts
View File

@ -29,7 +29,7 @@ export default function useDropModelBinaries() {
const importingModels: ImportingModel[] = supportedFiles.map((file) => ({
importId: uuidv4(),
modelId: undefined,
name: file.name.replace('.gguf', ''),
name: file.name.replace(/ /g, '').replace('.gguf', ''),
description: '',
path: file.path,
tags: [],

26
web/hooks/useImportModel.ts
View File

@ -5,14 +5,14 @@ import {
ImportingModel,
LocalImportModelEvent,
Model,
ModelEvent,
ModelExtension,
OptionType,
events,
fs,
baseName,
} from '@janhq/core'
import { atom, useSetAtom } from 'jotai'
import { atom, useAtomValue, useSetAtom } from 'jotai'
import { v4 as uuidv4 } from 'uuid'
@ -23,6 +23,7 @@ import { FilePathWithSize } from '@/utils/file'
import { extensionManager } from '@/extension'
import {
addDownloadingModelAtom,
downloadedModelsAtom,
importingModelsAtom,
removeDownloadingModelAtom,
} from '@/helpers/atoms/Model.atom'
@ -58,11 +59,24 @@ const useImportModel = () => {
const setImportingModels = useSetAtom(importingModelsAtom)
const addDownloadingModel = useSetAtom(addDownloadingModelAtom)
const removeDownloadingModel = useSetAtom(removeDownloadingModelAtom)
const downloadedModels = useAtomValue(downloadedModelsAtom)
const incrementalModelName = useCallback(
(name: string, startIndex: number = 0): string => {
const newModelName = startIndex ? `${name}-${startIndex}` : name
if (downloadedModels.some((model) => model.id === newModelName)) {
return incrementalModelName(name, startIndex + 1)
} else {
return newModelName
}
},
[downloadedModels]
)
const importModels = useCallback(
(models: ImportingModel[], optionType: OptionType) => {
models.map((model) => {
const modelId = model.modelId ?? model.path.split('/').pop()
models.map(async (model) => {
const modelId = model.modelId ?? incrementalModelName(model.name)
if (modelId) {
addDownloadingModel(modelId)
extensionManager
@ -78,7 +92,7 @@ const useImportModel = () => {
}
})
},
[addDownloadingModel, removeDownloadingModel]
[addDownloadingModel, incrementalModelName, removeDownloadingModel]
)
const updateModelInfo = useCallback(
@ -100,7 +114,7 @@ const useImportModel = () => {
({ path, name, size }: FilePathWithSize) => ({
importId: uuidv4(),
modelId: undefined,
name: name.replace('.gguf', ''),
name: name.replace(/ /g, '').replace('.gguf', ''),
description: '',
path: path,
tags: [],

2
web/package.json
View File

@ -38,7 +38,7 @@
"react": "18.2.0",
"react-circular-progressbar": "^2.1.0",
"react-dom": "18.2.0",
"react-dropzone": "^14.2.3",
"react-dropzone": "14.2.3",
"react-hook-form": "^7.47.0",
"react-hot-toast": "^2.4.1",
"react-icons": "^4.12.0",

12
web/screens/Settings/Advanced/index.tsx
View File

@ -189,7 +189,7 @@ const Advanced = () => {
* @param gpuId
* @returns
*/
const handleGPUChange = (gpuId: string) => {
const handleGPUChange = async (gpuId: string) => {
let updatedGpusInUse = [...gpusInUse]
if (updatedGpusInUse.includes(gpuId)) {
updatedGpusInUse = updatedGpusInUse.filter((id) => id !== gpuId)
@ -208,7 +208,7 @@ const Advanced = () => {
updatedGpusInUse.push(gpuId)
}
setGpusInUse(updatedGpusInUse)
saveSettings({ gpusInUse: updatedGpusInUse })
await saveSettings({ gpusInUse: updatedGpusInUse })
window.core?.api?.relaunch()
}
@ -306,7 +306,13 @@ const Advanced = () => {
})
}
// Stop any running model to apply the changes
if (e.target.checked !== gpuEnabled) stopModel()
if (e.target.checked !== gpuEnabled) {
stopModel().finally(() => {
setTimeout(() => {
window.location.reload()
}, 300)
})
}
}}
/>
}

2
web/screens/Settings/SelectingModelModal/index.tsx
View File

@ -21,7 +21,7 @@ const SelectingModelModal = () => {
const onSelectFileClick = useCallback(async () => {
const platform = (await systemInformation()).osInfo?.platform
if (platform === 'win32') {
if (platform !== 'darwin') {
setImportModelStage('CHOOSE_WHAT_TO_IMPORT')
return
}

43
web/screens/Thread/ThreadCenterPanel/LoadModelError/index.tsx
View File

@ -64,49 +64,6 @@ const LoadModelError = () => {
to continue using it.
</p>
)
} else if (
settings &&
settings.run_mode === 'gpu' &&
!settings.vulkan &&
(!settings.nvidia_driver?.exist || !settings.cuda?.exist)
) {
return (
<>
{!settings?.cuda.exist ? (
<p>
The CUDA toolkit may be unavailable. Please use the{' '}
<span
className="cursor-pointer font-medium text-[hsla(var(--app-link))]"
onClick={() => {
setMainState(MainViewState.Settings)
if (activeThread?.assistants[0]?.model.engine) {
const engine = EngineManager.instance().get(
activeThread.assistants[0].model.engine
)
engine?.name && setSelectedSettingScreen(engine.name)
}
}}
>
Install Additional Dependencies
</span>{' '}
setting to proceed with the download / installation process.
</p>
) : (
<div>
Problem with Nvidia drivers. Please follow the{' '}
<a
className="font-medium text-[hsla(var(--app-link))]"
href="https://www.nvidia.com/Download/index.aspx"
target="_blank"
>
Nvidia Drivers guideline
</a>{' '}
to access installation instructions and ensure proper functioning
of the application.
</div>
)}
</>
)
} else {
return (
<div>

19
web/screens/Thread/ThreadCenterPanel/index.tsx
View File

@ -147,6 +147,20 @@ const ThreadCenterPanel = () => {
const showSystemMonitorPanel = useAtomValue(showSystemMonitorPanelAtom)
const [height, setHeight] = useState<number>(0)
useEffect(() => {
if (showSystemMonitorPanel) {
const element = document.querySelector('.system-monitor-panel')
if (element) {
setHeight(element.clientHeight) // You can also use offsetHeight if needed
}
} else {
setHeight(0)
}
}, [showSystemMonitorPanel])
return (
<CenterPanelContainer>
<div
@ -193,9 +207,10 @@ const ThreadCenterPanel = () => {
)}
<div
className={twMerge(
'flex h-full w-full flex-col justify-between',
showSystemMonitorPanel && 'h-[calc(100%-200px)]'
'flex h-full w-full flex-col justify-between'
// showSystemMonitorPanel && `h-[calc(100%-${height}px)]`
)}
style={{ height: `calc(100% - ${height}px)` }}
>
{activeThread ? (
<div className="flex h-full w-full overflow-x-hidden">