From 9d57ecd6f318e0a9c8524e32a1afcab555110d57 Mon Sep 17 00:00:00 2001 From: Louis Date: Fri, 8 Nov 2024 13:38:34 +0700 Subject: [PATCH 01/13] chore: retrieves the exact model running status upon message error --- core/src/browser/extensions/model.ts | 8 +++++++- extensions/model-extension/src/cortex.ts | 20 ++++++++++++++++++-- extensions/model-extension/src/index.ts | 8 ++++++++ web/containers/Providers/EventHandler.tsx | 13 +++++++++++-- 4 files changed, 44 insertions(+), 5 deletions(-) diff --git a/core/src/browser/extensions/model.ts b/core/src/browser/extensions/model.ts index 1fb94fba3..e224ec5cc 100644 --- a/core/src/browser/extensions/model.ts +++ b/core/src/browser/extensions/model.ts @@ -15,7 +15,13 @@ export abstract class ModelExtension extends BaseExtension implements ModelInter abstract getModels(): Promise abstract pullModel(model: string, id?: string, name?: string): Promise abstract cancelModelPull(modelId: string): Promise - abstract importModel(model: string, modePath: string, name?: string, optionType?: OptionType): Promise + abstract importModel( + model: string, + modePath: string, + name?: string, + optionType?: OptionType + ): Promise abstract updateModel(modelInfo: Partial): Promise abstract deleteModel(model: string): Promise + abstract isModelLoaded(model: string): Promise } diff --git a/extensions/model-extension/src/cortex.ts b/extensions/model-extension/src/cortex.ts index 024aa2223..5b7d1e36b 100644 --- a/extensions/model-extension/src/cortex.ts +++ b/extensions/model-extension/src/cortex.ts @@ -9,7 +9,12 @@ interface ICortexAPI { getModel(model: string): Promise getModels(): Promise pullModel(model: string, id?: string, name?: string): Promise - importModel(path: string, modelPath: string, name?: string, option?: string): Promise + importModel( + path: string, + modelPath: string, + name?: string, + option?: string + ): Promise deleteModel(model: string): Promise updateModel(model: object): Promise cancelModelPull(model: string): Promise @@ -141,6 +146,17 @@ export class CortexAPI implements ICortexAPI { ) } + /** + * Check model status + * @param model + */ + async getModelStatus(model: string): Promise { + return this.queue + .add(() => ky.get(`${API_URL}/models/status/${model}`)) + .then((e) => true) + .catch(() => false) + } + /** * Do health check on cortex.cpp * @returns @@ -215,7 +231,7 @@ export class CortexAPI implements ICortexAPI { } model.metadata = model.metadata ?? { tags: [], - size: model.size ?? model.metadata?.size ?? 0 + size: model.size ?? model.metadata?.size ?? 0, } return model as Model } diff --git a/extensions/model-extension/src/index.ts b/extensions/model-extension/src/index.ts index 7d7514f3b..8f50bd5d0 100644 --- a/extensions/model-extension/src/index.ts +++ b/extensions/model-extension/src/index.ts @@ -238,6 +238,14 @@ export default class JanModelExtension extends ModelExtension { return this.cortexAPI.importModel(model, modelPath, name, option) } + /** + * Check model status + * @param model + */ + async isModelLoaded(model: string): Promise { + return this.cortexAPI.getModelStatus(model) + } + /** * Handle download state from main app */ diff --git a/web/containers/Providers/EventHandler.tsx b/web/containers/Providers/EventHandler.tsx index 0f5cf389d..6cad910f7 100644 --- a/web/containers/Providers/EventHandler.tsx +++ b/web/containers/Providers/EventHandler.tsx @@ -16,6 +16,7 @@ import { EngineManager, InferenceEngine, extractInferenceParams, + ModelExtension, } from '@janhq/core' import { useAtomValue, useSetAtom } from 'jotai' import { ulid } from 'ulidx' @@ -180,8 +181,16 @@ export default function EventHandler({ children }: { children: ReactNode }) { } return } else if (message.status === MessageStatus.Error) { - setActiveModel(undefined) - setStateModel({ state: 'start', loading: false, model: undefined }) + ;(async () => { + if ( + !(await extensionManager + .get(ExtensionTypeEnum.Model) + ?.isModelLoaded(activeModelRef.current?.id as string)) + ) { + setActiveModel(undefined) + setStateModel({ state: 'start', loading: false, model: undefined }) + } + })() } // Mark the thread as not waiting for response updateThreadWaiting(message.thread_id, false) From 102d8b08c15b4ef2349ee38018bcb7c863be82e3 Mon Sep 17 00:00:00 2001 From: hiento09 <136591877+hiento09@users.noreply.github.com> Date: Fri, 8 Nov 2024 14:24:02 +0700 Subject: [PATCH 02/13] chore: enable notification nightly (#3963) Co-authored-by: Hien To --- .../workflows/jan-electron-build-nightly.yml | 64 +++++++++---------- .../workflows/template-build-linux-x64.yml | 2 +- .../workflows/template-build-macos-arm64.yml | 2 +- .../workflows/template-build-macos-x64.yml | 2 +- .../workflows/template-build-windows-x64.yml | 2 +- 5 files changed, 36 insertions(+), 36 deletions(-) diff --git a/.github/workflows/jan-electron-build-nightly.yml b/.github/workflows/jan-electron-build-nightly.yml index 1b29b84af..d79080990 100644 --- a/.github/workflows/jan-electron-build-nightly.yml +++ b/.github/workflows/jan-electron-build-nightly.yml @@ -114,8 +114,8 @@ jobs: - name: Upload latest-mac.yml if: ${{ needs.set-public-provider.outputs.public_provider == 'aws-s3' }} run: | - aws s3 cp ./latest-mac.yml "s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/temp-nightly/latest-mac.yml" - aws s3 sync s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/temp-nightly/ s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/nightly/ + aws s3 cp ./latest-mac.yml "s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/temp-latest/latest-mac.yml" + aws s3 sync s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/temp-latest/ s3://${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}/latest/ env: AWS_ACCESS_KEY_ID: ${{ secrets.DELTA_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DELTA_AWS_SECRET_ACCESS_KEY }} @@ -123,35 +123,35 @@ jobs: AWS_EC2_METADATA_DISABLED: "true" - # noti-discord-nightly-and-update-url-readme: - # needs: [build-macos-x64, build-macos-arm64, build-windows-x64, build-linux-x64, get-update-version, set-public-provider, combine-latest-mac-yml] - # secrets: inherit - # if: github.event_name == 'schedule' - # uses: ./.github/workflows/template-noti-discord-and-update-url-readme.yml - # with: - # ref: refs/heads/dev - # build_reason: Nightly - # push_to_branch: dev - # new_version: ${{ needs.get-update-version.outputs.new_version }} + noti-discord-nightly-and-update-url-readme: + needs: [build-macos-x64, build-macos-arm64, build-windows-x64, build-linux-x64, get-update-version, set-public-provider, combine-latest-mac-yml] + secrets: inherit + if: github.event_name == 'schedule' + uses: ./.github/workflows/template-noti-discord-and-update-url-readme.yml + with: + ref: refs/heads/dev + build_reason: Nightly + push_to_branch: dev + new_version: ${{ needs.get-update-version.outputs.new_version }} - # noti-discord-pre-release-and-update-url-readme: - # needs: [build-macos-x64, build-macos-arm64, build-windows-x64, build-linux-x64, get-update-version, set-public-provider, combine-latest-mac-yml] - # secrets: inherit - # if: github.event_name == 'push' - # uses: ./.github/workflows/template-noti-discord-and-update-url-readme.yml - # with: - # ref: refs/heads/dev - # build_reason: Pre-release - # push_to_branch: dev - # new_version: ${{ needs.get-update-version.outputs.new_version }} + noti-discord-pre-release-and-update-url-readme: + needs: [build-macos-x64, build-macos-arm64, build-windows-x64, build-linux-x64, get-update-version, set-public-provider, combine-latest-mac-yml] + secrets: inherit + if: github.event_name == 'push' + uses: ./.github/workflows/template-noti-discord-and-update-url-readme.yml + with: + ref: refs/heads/dev + build_reason: Pre-release + push_to_branch: dev + new_version: ${{ needs.get-update-version.outputs.new_version }} - # noti-discord-manual-and-update-url-readme: - # needs: [build-macos-x64, build-macos-arm64, build-windows-x64, build-linux-x64, get-update-version, set-public-provider, combine-latest-mac-yml] - # secrets: inherit - # if: github.event_name == 'workflow_dispatch' && github.event.inputs.public_provider == 'aws-s3' - # uses: ./.github/workflows/template-noti-discord-and-update-url-readme.yml - # with: - # ref: refs/heads/dev - # build_reason: Manual - # push_to_branch: dev - # new_version: ${{ needs.get-update-version.outputs.new_version }} + noti-discord-manual-and-update-url-readme: + needs: [build-macos-x64, build-macos-arm64, build-windows-x64, build-linux-x64, get-update-version, set-public-provider, combine-latest-mac-yml] + secrets: inherit + if: github.event_name == 'workflow_dispatch' && github.event.inputs.public_provider == 'aws-s3' + uses: ./.github/workflows/template-noti-discord-and-update-url-readme.yml + with: + ref: refs/heads/dev + build_reason: Manual + push_to_branch: dev + new_version: ${{ needs.get-update-version.outputs.new_version }} diff --git a/.github/workflows/template-build-linux-x64.yml b/.github/workflows/template-build-linux-x64.yml index 0280b1014..92188c364 100644 --- a/.github/workflows/template-build-linux-x64.yml +++ b/.github/workflows/template-build-linux-x64.yml @@ -60,7 +60,7 @@ jobs: mv /tmp/package.json electron/package.json jq --arg version "${{ inputs.new_version }}" '.version = $version' web/package.json > /tmp/package.json mv /tmp/package.json web/package.json - jq '.build.publish = [{"provider": "generic", "url": "https://delta.jan.ai/nightly", "channel": "latest"}, {"provider": "s3", "acl": null, "bucket": "${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}", "region": "${{ secrets.DELTA_AWS_REGION}}", "path": "temp-nightly", "channel": "latest"}]' electron/package.json > /tmp/package.json + jq '.build.publish = [{"provider": "generic", "url": "https://delta.jan.ai/latest", "channel": "latest"}, {"provider": "s3", "acl": null, "bucket": "${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}", "region": "${{ secrets.DELTA_AWS_REGION}}", "path": "temp-latest", "channel": "latest"}]' electron/package.json > /tmp/package.json mv /tmp/package.json electron/package.json cat electron/package.json # chmod +x .github/scripts/rename-app.sh diff --git a/.github/workflows/template-build-macos-arm64.yml b/.github/workflows/template-build-macos-arm64.yml index e23ee5ed5..a23e34cf9 100644 --- a/.github/workflows/template-build-macos-arm64.yml +++ b/.github/workflows/template-build-macos-arm64.yml @@ -72,7 +72,7 @@ jobs: jq --arg version "${{ inputs.new_version }}" '.version = $version' web/package.json > /tmp/package.json mv /tmp/package.json web/package.json - jq '.build.publish = [{"provider": "generic", "url": "https://delta.jan.ai/nightly", "channel": "latest"}, {"provider": "s3", "acl": null, "bucket": "${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}", "region": "${{ secrets.DELTA_AWS_REGION}}", "path": "temp-nightly", "channel": "latest"}]' electron/package.json > /tmp/package.json + jq '.build.publish = [{"provider": "generic", "url": "https://delta.jan.ai/latest", "channel": "latest"}, {"provider": "s3", "acl": null, "bucket": "${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}", "region": "${{ secrets.DELTA_AWS_REGION}}", "path": "temp-latest", "channel": "latest"}]' electron/package.json > /tmp/package.json mv /tmp/package.json electron/package.json jq --arg teamid "${{ secrets.APPLE_TEAM_ID }}" '.build.mac.notarize.teamId = $teamid' electron/package.json > /tmp/package.json diff --git a/.github/workflows/template-build-macos-x64.yml b/.github/workflows/template-build-macos-x64.yml index 06a9baaa1..18309fca0 100644 --- a/.github/workflows/template-build-macos-x64.yml +++ b/.github/workflows/template-build-macos-x64.yml @@ -72,7 +72,7 @@ jobs: jq --arg version "${{ inputs.new_version }}" '.version = $version' web/package.json > /tmp/package.json mv /tmp/package.json web/package.json - jq '.build.publish = [{"provider": "generic", "url": "https://delta.jan.ai/nightly", "channel": "latest"}, {"provider": "s3", "acl": null, "bucket": "${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}", "region": "${{ secrets.DELTA_AWS_REGION}}", "path": "temp-nightly", "channel": "latest"}]' electron/package.json > /tmp/package.json + jq '.build.publish = [{"provider": "generic", "url": "https://delta.jan.ai/latest", "channel": "latest"}, {"provider": "s3", "acl": null, "bucket": "${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}", "region": "${{ secrets.DELTA_AWS_REGION}}", "path": "temp-latest", "channel": "latest"}]' electron/package.json > /tmp/package.json mv /tmp/package.json electron/package.json jq --arg teamid "${{ secrets.APPLE_TEAM_ID }}" '.build.mac.notarize.teamId = $teamid' electron/package.json > /tmp/package.json diff --git a/.github/workflows/template-build-windows-x64.yml b/.github/workflows/template-build-windows-x64.yml index c683392f5..2a1d3f15b 100644 --- a/.github/workflows/template-build-windows-x64.yml +++ b/.github/workflows/template-build-windows-x64.yml @@ -73,7 +73,7 @@ jobs: jq --arg version "${{ inputs.new_version }}" '.version = $version' web/package.json > /tmp/package.json mv /tmp/package.json web/package.json - jq '.build.publish = [{"provider": "generic", "url": "https://delta.jan.ai/nightly", "channel": "latest"}, {"provider": "s3", "acl": null, "bucket": "${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}", "region": "${{ secrets.DELTA_AWS_REGION}}", "path": "temp-nightly", "channel": "latest"}]' electron/package.json > /tmp/package.json + jq '.build.publish = [{"provider": "generic", "url": "https://delta.jan.ai/latest", "channel": "latest"}, {"provider": "s3", "acl": null, "bucket": "${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}", "region": "${{ secrets.DELTA_AWS_REGION}}", "path": "temp-latest", "channel": "latest"}]' electron/package.json > /tmp/package.json mv /tmp/package.json electron/package.json jq '.build.win.sign = "./sign.js"' electron/package.json > /tmp/package.json From 9ddd26d952ae4e5685ab2bd15ba62c33b742264c Mon Sep 17 00:00:00 2001 From: Louis Date: Fri, 8 Nov 2024 15:50:15 +0700 Subject: [PATCH 03/13] chore: maintain model.json settings - cortex.cpp might extract a wrong prompt template --- web/hooks/useModels.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/web/hooks/useModels.ts b/web/hooks/useModels.ts index 400e02793..97c398b43 100644 --- a/web/hooks/useModels.ts +++ b/web/hooks/useModels.ts @@ -35,6 +35,8 @@ const useModels = () => { const localModels = (await getModels()).map((e) => ({ ...e, name: ModelManager.instance().models.get(e.id)?.name ?? e.id, + settings: + ModelManager.instance().models.get(e.id)?.settings ?? e.settings, metadata: ModelManager.instance().models.get(e.id)?.metadata ?? e.metadata, })) From 43d0e272ed2111e9a1853c046f4ea2ecff0b976d Mon Sep 17 00:00:00 2001 From: Louis Date: Fri, 8 Nov 2024 16:19:04 +0700 Subject: [PATCH 04/13] fix: should maintain legacy jan model parameters --- web/hooks/useModels.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/web/hooks/useModels.ts b/web/hooks/useModels.ts index 97c398b43..b8b680715 100644 --- a/web/hooks/useModels.ts +++ b/web/hooks/useModels.ts @@ -37,6 +37,8 @@ const useModels = () => { name: ModelManager.instance().models.get(e.id)?.name ?? e.id, settings: ModelManager.instance().models.get(e.id)?.settings ?? e.settings, + parameters: + ModelManager.instance().models.get(e.id)?.parameters ?? e.parameters, metadata: ModelManager.instance().models.get(e.id)?.metadata ?? e.metadata, })) From 387f140ba85aae79504e34e6bebbb97b91d3dbaa Mon Sep 17 00:00:00 2001 From: Faisal Amir Date: Fri, 8 Nov 2024 16:32:27 +0700 Subject: [PATCH 05/13] chore: add cortex log into modal troubleshoot (#3978) --- .../ModalTroubleShoot/CortexLogs.tsx | 226 ++++++++++++++++++ web/containers/ModalTroubleShoot/index.tsx | 10 +- 2 files changed, 234 insertions(+), 2 deletions(-) create mode 100644 web/containers/ModalTroubleShoot/CortexLogs.tsx diff --git a/web/containers/ModalTroubleShoot/CortexLogs.tsx b/web/containers/ModalTroubleShoot/CortexLogs.tsx new file mode 100644 index 000000000..3323a1694 --- /dev/null +++ b/web/containers/ModalTroubleShoot/CortexLogs.tsx @@ -0,0 +1,226 @@ +import React, { useEffect, useState, memo } from 'react' + +import { Button } from '@janhq/joi' + +import { CopyIcon, CheckIcon, FolderIcon } from 'lucide-react' + +import { twMerge } from 'tailwind-merge' + +import { useClipboard } from '@/hooks/useClipboard' +import { useLogs } from '@/hooks/useLogs' +import { usePath } from '@/hooks/usePath' + +const CortexLogs = () => { + const { getLogs } = useLogs() + const [logs, setLogs] = useState([]) + const { onRevealInFinder } = usePath() + + useEffect(() => { + getLogs('cortex').then((log) => { + if (typeof log?.split === 'function') { + if (log.length > 0) { + setLogs(log.split(/\r?\n|\r|\n/g)) + } + } + }) + + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) + + const clipboard = useClipboard({ timeout: 1000 }) + + return ( +
+
+
+ + +
+
+
+ {logs.length > 0 ? ( + + {logs.slice(-100).map((log, i) => { + return ( +

+ {log} +

+ ) + })} + + ) : ( +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Empty logs

+
+ )} +
+
+ ) +} + +export default memo(CortexLogs) diff --git a/web/containers/ModalTroubleShoot/index.tsx b/web/containers/ModalTroubleShoot/index.tsx index 67ccbe22f..77ee51034 100644 --- a/web/containers/ModalTroubleShoot/index.tsx +++ b/web/containers/ModalTroubleShoot/index.tsx @@ -8,10 +8,11 @@ import { twMerge } from 'tailwind-merge' import ServerLogs from '@/containers/ServerLogs' import AppLogs from './AppLogs' +import CortexLogs from './CortexLogs' import DeviceSpecs from './DeviceSpecs' export const modalTroubleShootingAtom = atom(false) -const logOption = ['App Logs', 'Server Logs', 'Device Specs'] +const logOption = ['App Logs', 'Cortex Logs', 'Server Logs', 'Device Specs'] const ModalTroubleShooting = () => { const [modalTroubleShooting, setModalTroubleShooting] = useAtom( @@ -144,10 +145,15 @@ const ModalTroubleShooting = () => {
- +
+ +
+
From ebad6c3e937257ec79f838b421db37382aeff485 Mon Sep 17 00:00:00 2001 From: Faisal Amir Date: Fri, 8 Nov 2024 16:32:42 +0700 Subject: [PATCH 06/13] fix: correct chat input cursor behavior when content is scrollable (#3979) --- .../Thread/ThreadCenterPanel/ChatInput/RichTextEditor.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/web/screens/Thread/ThreadCenterPanel/ChatInput/RichTextEditor.tsx b/web/screens/Thread/ThreadCenterPanel/ChatInput/RichTextEditor.tsx index 096ef51e0..bca808e28 100644 --- a/web/screens/Thread/ThreadCenterPanel/ChatInput/RichTextEditor.tsx +++ b/web/screens/Thread/ThreadCenterPanel/ChatInput/RichTextEditor.tsx @@ -269,6 +269,10 @@ const RichTextEditor = ({ ? '100px' : '40px' textareaRef.current.style.height = textareaRef.current.scrollHeight + 'px' + textareaRef.current?.scrollTo({ + top: textareaRef.current.scrollHeight, + behavior: 'instant', + }) textareaRef.current.style.overflow = textareaRef.current.clientHeight >= 390 ? 'auto' : 'hidden' } From 5bac278be4aadc8fe9d4d55683814afc4ba2274c Mon Sep 17 00:00:00 2001 From: Faisal Amir Date: Fri, 8 Nov 2024 16:42:54 +0700 Subject: [PATCH 07/13] fix: update copy error message (#3977) * fix: update copy error message * chore: copy nits --- web/containers/ErrorMessage/index.tsx | 2 +- web/screens/Thread/ThreadCenterPanel/LoadModelError/index.tsx | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/web/containers/ErrorMessage/index.tsx b/web/containers/ErrorMessage/index.tsx index add2bd89b..18558c1d8 100644 --- a/web/containers/ErrorMessage/index.tsx +++ b/web/containers/ErrorMessage/index.tsx @@ -73,7 +73,7 @@ const ErrorMessage = ({ message }: { message: ThreadMessage }) => { > {getErrorTitle()}

- Jan’s in beta. Access  + {`Something's wrong.`} Access  setModalTroubleShooting(true)} diff --git a/web/screens/Thread/ThreadCenterPanel/LoadModelError/index.tsx b/web/screens/Thread/ThreadCenterPanel/LoadModelError/index.tsx index f5f74f9c9..19a1f628c 100644 --- a/web/screens/Thread/ThreadCenterPanel/LoadModelError/index.tsx +++ b/web/screens/Thread/ThreadCenterPanel/LoadModelError/index.tsx @@ -110,9 +110,9 @@ const LoadModelError = () => { } else { return (

- Apologies, something’s amiss! + Apologies, {`Something's wrong.`}. 

- Jan’s in beta. Access  + Access  setModalTroubleShooting(true)} From f3802bff048f449715992a2812dbdf6c769c56ac Mon Sep 17 00:00:00 2001 From: Louis Date: Fri, 8 Nov 2024 17:00:33 +0700 Subject: [PATCH 08/13] chore: bump to latest release --- extensions/inference-cortex-extension/bin/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/inference-cortex-extension/bin/version.txt b/extensions/inference-cortex-extension/bin/version.txt index c89636bcf..e6d5cb833 100644 --- a/extensions/inference-cortex-extension/bin/version.txt +++ b/extensions/inference-cortex-extension/bin/version.txt @@ -1 +1 @@ -1.0.2-rc4 \ No newline at end of file +1.0.2 \ No newline at end of file From 731d42863c71402aa8954c551bc4da1925489437 Mon Sep 17 00:00:00 2001 From: Faisal Amir Date: Fri, 8 Nov 2024 17:50:56 +0700 Subject: [PATCH 09/13] fix: aligment stop model btn system monitor (#3981) --- .../Layout/BottomPanel/SystemMonitor/TableActiveModel/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/containers/Layout/BottomPanel/SystemMonitor/TableActiveModel/index.tsx b/web/containers/Layout/BottomPanel/SystemMonitor/TableActiveModel/index.tsx index 00d528f99..8ad16eeba 100644 --- a/web/containers/Layout/BottomPanel/SystemMonitor/TableActiveModel/index.tsx +++ b/web/containers/Layout/BottomPanel/SystemMonitor/TableActiveModel/index.tsx @@ -35,7 +35,7 @@ const TableActiveModel = () => { : '-'} - + Date: Fri, 8 Nov 2024 17:59:09 +0700 Subject: [PATCH 10/13] chore: proxies Jan APIs to cortex.cpp --- core/src/node/api/restful/common.ts | 10 ++- core/src/node/api/restful/helper/builder.ts | 85 ++++++++++----------- core/src/node/api/restful/helper/consts.ts | 2 +- 3 files changed, 50 insertions(+), 47 deletions(-) diff --git a/core/src/node/api/restful/common.ts b/core/src/node/api/restful/common.ts index c8061c34a..4705bea2f 100644 --- a/core/src/node/api/restful/common.ts +++ b/core/src/node/api/restful/common.ts @@ -10,6 +10,7 @@ import { getMessages, retrieveMessage, updateThread, + getModels, } from './helper/builder' import { JanApiRouteConfiguration } from './helper/configuration' @@ -26,9 +27,12 @@ export const commonRouter = async (app: HttpServer) => { // Common Routes // Read & Delete :: Threads | Models | Assistants Object.keys(JanApiRouteConfiguration).forEach((key) => { - app.get(`/${key}`, async (_request) => - getBuilder(JanApiRouteConfiguration[key]).then(normalizeData) - ) + app.get(`/${key}`, async (_req, _res) => { + if (key === 'models') { + return getModels(_req, _res) + } + return getBuilder(JanApiRouteConfiguration[key]).then(normalizeData) + }) app.get(`/${key}/:id`, async (request: any) => retrieveBuilder(JanApiRouteConfiguration[key], request.params.id) diff --git a/core/src/node/api/restful/helper/builder.ts b/core/src/node/api/restful/helper/builder.ts index da33808dc..455b489d2 100644 --- a/core/src/node/api/restful/helper/builder.ts +++ b/core/src/node/api/restful/helper/builder.ts @@ -10,9 +10,9 @@ import { } from 'fs' import { JanApiRouteConfiguration, RouteConfiguration } from './configuration' import { join } from 'path' -import { ContentType, MessageStatus, Model, ThreadMessage } from '../../../../types' -import { getEngineConfiguration, getJanDataFolderPath } from '../../../helper' -import { DEFAULT_CHAT_COMPLETION_URL } from './consts' +import { ContentType, InferenceEngine, MessageStatus, ThreadMessage } from '../../../../types' +import { getJanDataFolderPath } from '../../../helper' +import { CORTEX_API_URL } from './consts' // TODO: Refactor these export const getBuilder = async (configuration: RouteConfiguration) => { @@ -297,57 +297,56 @@ export const downloadModel = async ( } } -export const chatCompletions = async (request: any, reply: any) => { - const modelList = await getBuilder(JanApiRouteConfiguration.models) - const modelId = request.body.model - - const matchedModels = modelList.filter((model: Model) => model.id === modelId) - if (matchedModels.length === 0) { - const error = { - error: { - message: `The model ${request.body.model} does not exist`, - type: 'invalid_request_error', - param: null, - code: 'model_not_found', - }, - } - reply.code(404).send(error) - return - } - - const requestedModel = matchedModels[0] - - const engineConfiguration = await getEngineConfiguration(requestedModel.engine) - - let apiKey: string | undefined = undefined - let apiUrl: string = DEFAULT_CHAT_COMPLETION_URL - - if (engineConfiguration) { - apiKey = engineConfiguration.api_key - apiUrl = engineConfiguration.full_url ?? DEFAULT_CHAT_COMPLETION_URL - } - +/** + * Proxy /models to cortex + * @param request + * @param reply + */ +export const getModels = async (request: any, reply: any) => { + const fetch = require('node-fetch') const headers: Record = { 'Content-Type': 'application/json', } - if (apiKey) { - headers['Authorization'] = `Bearer ${apiKey}` - headers['api-key'] = apiKey - } + const response = await fetch(`${CORTEX_API_URL}/models`, { + method: 'GET', + headers: headers, + body: JSON.stringify(request.body), + }) - if (requestedModel.engine === 'openai' && request.body.stop) { - // openai only allows max 4 stop words - request.body.stop = request.body.stop.slice(0, 4) + if (response.status !== 200) { + // Forward the error response to client via reply + const responseBody = await response.text() + const responseHeaders = Object.fromEntries(response.headers) + reply.code(response.status).headers(responseHeaders).send(responseBody) + } else { + reply.raw.writeHead(200, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-cache', + 'Connection': 'keep-alive', + 'Access-Control-Allow-Origin': '*', + }) + response.body.pipe(reply.raw) + } +} + +/** + * Proxy chat completions + * @param request + * @param reply + */ +export const chatCompletions = async (request: any, reply: any) => { + const headers: Record = { + 'Content-Type': 'application/json', } // add engine for new cortex cpp engine - if (requestedModel.engine === 'nitro') { - request.body.engine = 'llama-cpp' + if (request.body.engine === InferenceEngine.nitro) { + request.body.engine = InferenceEngine.cortex_llamacpp } const fetch = require('node-fetch') - const response = await fetch(apiUrl, { + const response = await fetch(`${CORTEX_API_URL}/chat/completions`, { method: 'POST', headers: headers, body: JSON.stringify(request.body), diff --git a/core/src/node/api/restful/helper/consts.ts b/core/src/node/api/restful/helper/consts.ts index 0f57bb5ff..e81b8f8d8 100644 --- a/core/src/node/api/restful/helper/consts.ts +++ b/core/src/node/api/restful/helper/consts.ts @@ -6,4 +6,4 @@ export const LOCAL_HOST = '127.0.0.1' export const SUPPORTED_MODEL_FORMAT = '.gguf' -export const DEFAULT_CHAT_COMPLETION_URL = `http://${LOCAL_HOST}:${CORTEX_DEFAULT_PORT}/v1/chat/completions` // default nitro url +export const CORTEX_API_URL = `http://${LOCAL_HOST}:${CORTEX_DEFAULT_PORT}/v1` // default nitro url From 81015bab3c042d5bef10381e5f40cd67e8d69f3d Mon Sep 17 00:00:00 2001 From: Louis Date: Fri, 8 Nov 2024 18:03:37 +0700 Subject: [PATCH 11/13] chore: clean nitro stuffs --- core/src/node/api/restful/helper/consts.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/src/node/api/restful/helper/consts.ts b/core/src/node/api/restful/helper/consts.ts index e81b8f8d8..412d304ee 100644 --- a/core/src/node/api/restful/helper/consts.ts +++ b/core/src/node/api/restful/helper/consts.ts @@ -1,9 +1,7 @@ -// The PORT to use for the Nitro subprocess export const CORTEX_DEFAULT_PORT = 39291 -// The HOST address to use for the Nitro subprocess export const LOCAL_HOST = '127.0.0.1' export const SUPPORTED_MODEL_FORMAT = '.gguf' -export const CORTEX_API_URL = `http://${LOCAL_HOST}:${CORTEX_DEFAULT_PORT}/v1` // default nitro url +export const CORTEX_API_URL = `http://${LOCAL_HOST}:${CORTEX_DEFAULT_PORT}/v1` From 4a414bf5e9db9412e98035872af106ba72b7f82c Mon Sep 17 00:00:00 2001 From: Louis Date: Fri, 8 Nov 2024 18:16:09 +0700 Subject: [PATCH 12/13] chore: proxy all /models methods to cortex.cpp --- core/src/node/api/restful/common.ts | 4 ++-- core/src/node/api/restful/helper/builder.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/node/api/restful/common.ts b/core/src/node/api/restful/common.ts index 4705bea2f..39f7b8d8b 100644 --- a/core/src/node/api/restful/common.ts +++ b/core/src/node/api/restful/common.ts @@ -10,7 +10,7 @@ import { getMessages, retrieveMessage, updateThread, - getModels, + models, } from './helper/builder' import { JanApiRouteConfiguration } from './helper/configuration' @@ -29,7 +29,7 @@ export const commonRouter = async (app: HttpServer) => { Object.keys(JanApiRouteConfiguration).forEach((key) => { app.get(`/${key}`, async (_req, _res) => { if (key === 'models') { - return getModels(_req, _res) + return models(_req, _res) } return getBuilder(JanApiRouteConfiguration[key]).then(normalizeData) }) diff --git a/core/src/node/api/restful/helper/builder.ts b/core/src/node/api/restful/helper/builder.ts index 455b489d2..c3493a8be 100644 --- a/core/src/node/api/restful/helper/builder.ts +++ b/core/src/node/api/restful/helper/builder.ts @@ -302,14 +302,14 @@ export const downloadModel = async ( * @param request * @param reply */ -export const getModels = async (request: any, reply: any) => { +export const models = async (request: any, reply: any) => { const fetch = require('node-fetch') const headers: Record = { 'Content-Type': 'application/json', } const response = await fetch(`${CORTEX_API_URL}/models`, { - method: 'GET', + method: request.method, headers: headers, body: JSON.stringify(request.body), }) From 487fd279121a1eb1a96d35e201f6a1bea8c3e9c6 Mon Sep 17 00:00:00 2001 From: Louis Date: Sat, 9 Nov 2024 12:21:51 +0700 Subject: [PATCH 13/13] test: remove outdated test --- core/src/node/api/restful/helper/builder.test.ts | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/core/src/node/api/restful/helper/builder.test.ts b/core/src/node/api/restful/helper/builder.test.ts index eb21e9401..f21257098 100644 --- a/core/src/node/api/restful/helper/builder.test.ts +++ b/core/src/node/api/restful/helper/builder.test.ts @@ -220,22 +220,6 @@ describe('builder helper functions', () => { }) describe('chatCompletions', () => { - it('should return an error if model is not found', async () => { - const request = { body: { model: 'nonexistentModel' } } - const reply = { code: jest.fn().mockReturnThis(), send: jest.fn() } - - await chatCompletions(request, reply) - expect(reply.code).toHaveBeenCalledWith(404) - expect(reply.send).toHaveBeenCalledWith({ - error: { - message: 'The model nonexistentModel does not exist', - type: 'invalid_request_error', - param: null, - code: 'model_not_found', - }, - }) - }) - it('should return the error on status not ok', async () => { const request = { body: { model: 'model1' } } const mockSend = jest.fn()