diff --git a/core/src/browser/extensions/engines/helpers/sse.ts b/core/src/browser/extensions/engines/helpers/sse.ts index aaafbf7e5..55cde56b4 100644 --- a/core/src/browser/extensions/engines/helpers/sse.ts +++ b/core/src/browser/extensions/engines/helpers/sse.ts @@ -91,8 +91,12 @@ export function requestInference( const toParse = cachedLines + line if (!line.includes('data: [DONE]')) { const data = JSON.parse(toParse.replace('data: ', '')) - if ('error' in data) { - subscriber.error(data.error) + if ( + 'error' in data || + 'message' in data || + 'detail' in data + ) { + subscriber.error(data.error ?? data) subscriber.complete() return } diff --git a/extensions/engine-management-extension/models/anthropic.json b/extensions/engine-management-extension/models/anthropic.json index d35ba4c22..46b5893d1 100644 --- a/extensions/engine-management-extension/models/anthropic.json +++ b/extensions/engine-management-extension/models/anthropic.json @@ -8,7 +8,7 @@ "inference_params": { "max_tokens": 4096, "temperature": 0.7, - "stream": false + "stream": true }, "engine": "anthropic" }, @@ -21,7 +21,7 @@ "inference_params": { "max_tokens": 8192, "temperature": 0.7, - "stream": false + "stream": true }, "engine": "anthropic" }, diff --git a/extensions/engine-management-extension/models/martian.json b/extensions/engine-management-extension/models/martian.json index b935587cc..9ce7b69ba 100644 --- a/extensions/engine-management-extension/models/martian.json +++ b/extensions/engine-management-extension/models/martian.json @@ -5,7 +5,7 @@ "name": "Martian Model Router", "version": "1.0", "description": "Martian Model Router dynamically routes requests to the best LLM in real-time", - "parameters": { + "inference_params": { "max_tokens": 4096, "temperature": 0.7, "top_p": 0.95, diff --git a/extensions/engine-management-extension/models/mistral.json b/extensions/engine-management-extension/models/mistral.json index 47833a31c..12fcf938d 100644 --- a/extensions/engine-management-extension/models/mistral.json +++ b/extensions/engine-management-extension/models/mistral.json @@ -5,7 +5,7 @@ "name": "Mistral Small", "version": "1.1", "description": "Mistral Small is the ideal choice for simple tasks (Classification, Customer Support, or Text Generation) at an affordable price.", - "parameters": { + "inference_params": { "max_tokens": 32000, "temperature": 0.7, "top_p": 0.95, @@ -19,7 +19,7 @@ "name": "Mistral Large", "version": "1.1", "description": "Mistral Large is ideal for complex tasks (Synthetic Text Generation, Code Generation, RAG, or Agents).", - "parameters": { + "inference_params": { "max_tokens": 32000, "temperature": 0.7, "top_p": 0.95, @@ -33,7 +33,7 @@ "name": "Mixtral 8x22B", "version": "1.1", "description": "Mixtral 8x22B is a high-performance, cost-effective model designed for complex tasks.", - "parameters": { + "inference_params": { "max_tokens": 32000, "temperature": 0.7, "top_p": 0.95, diff --git a/extensions/engine-management-extension/models/nvidia.json b/extensions/engine-management-extension/models/nvidia.json index f2adac779..dfce9f8bc 100644 --- a/extensions/engine-management-extension/models/nvidia.json +++ b/extensions/engine-management-extension/models/nvidia.json @@ -5,7 +5,7 @@ "name": "Mistral 7B", "version": "1.1", "description": "Mistral 7B with NVIDIA", - "parameters": { + "inference_params": { "max_tokens": 1024, "temperature": 0.3, "top_p": 1, diff --git a/extensions/engine-management-extension/models/openrouter.json b/extensions/engine-management-extension/models/openrouter.json index 5ac189a81..b9714bb57 100644 --- a/extensions/engine-management-extension/models/openrouter.json +++ b/extensions/engine-management-extension/models/openrouter.json @@ -5,7 +5,7 @@ "name": "OpenRouter", "version": "1.0", "description": " OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.", - "parameters": { + "inference_params": { "max_tokens": 128000, "temperature": 0.7, "top_p": 0.95, diff --git a/extensions/engine-management-extension/resources/anthropic.json b/extensions/engine-management-extension/resources/anthropic.json index 81d6c99c0..12a3f08b8 100644 --- a/extensions/engine-management-extension/resources/anthropic.json +++ b/extensions/engine-management-extension/resources/anthropic.json @@ -15,7 +15,7 @@ }, "transform_resp": { "chat_completions": { - "template": "{% if input_request.stream %} {\"object\": \"chat.completion.chunk\", \"model\": \"{{ input_request.model }}\", \"choices\": [{\"index\": 0, \"delta\": { {% if input_request.type == \"message_start\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"ping\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"content_block_delta\" %} \"role\": \"assistant\", \"content\": \"{{ input_request.delta.text }}\" {% else if input_request.type == \"content_block_stop\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"content_block_stop\" %} \"role\": \"assistant\", \"content\": null {% endif %} }, {% if input_request.type == \"content_block_stop\" %} \"finish_reason\": \"stop\" {% else %} \"finish_reason\": null {% endif %} }]} {% else %} {\"id\": \"{{ input_request.id }}\", \"created\": null, \"object\": \"chat.completion\", \"model\": \"{{ input_request.model }}\", \"choices\": [{ \"index\": 0, \"message\": { \"role\": \"{{ input_request.role }}\", \"content\": \"{% if input_request.content and input_request.content.0.type == \"text\" %} \"{{input_request.content.0.text}}\" {% endif %}\", \"refusal\": null }, \"logprobs\": null, \"finish_reason\": \"{{ input_request.stop_reason }}\" } ], \"usage\": { \"prompt_tokens\": {{ input_request.usage.input_tokens }}, \"completion_tokens\": {{ input_request.usage.output_tokens }}, \"total_tokens\": {{ input_request.usage.input_tokens + input_request.usage.output_tokens }}, \"prompt_tokens_details\": { \"cached_tokens\": 0 }, \"completion_tokens_details\": { \"reasoning_tokens\": 0, \"accepted_prediction_tokens\": 0, \"rejected_prediction_tokens\": 0 } }, \"system_fingerprint\": \"fp_6b68a8204b\"} {% endif %}" + "template": "{% if input_request.stream %} {\"object\": \"chat.completion.chunk\", \"model\": \"{{ input_request.model }}\", \"choices\": [{\"index\": 0, \"delta\": { {% if input_request.type == \"message_start\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"ping\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"content_block_delta\" %} \"role\": \"assistant\", \"content\": \"{{ input_request.delta.text }}\" {% else if input_request.type == \"content_block_stop\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"content_block_stop\" %} \"role\": \"assistant\", \"content\": null {% endif %} }, {% if input_request.type == \"content_block_stop\" %} \"finish_reason\": \"stop\" {% else %} \"finish_reason\": null {% endif %} }]} {% else %} {\"id\": \"{{ input_request.id }}\", \"created\": null, \"object\": \"chat.completion\", \"model\": \"{{ input_request.model }}\", \"choices\": [{ \"index\": 0, \"message\": { \"role\": \"{{ input_request.role }}\", \"content\": {% if input_request.content and input_request.content.0.type == \"text\" %} \"{{input_request.content.0.text}}\" {% else %} null {% endif %}, \"refusal\": null }, \"logprobs\": null, \"finish_reason\": \"{{ input_request.stop_reason }}\" } ], \"usage\": { \"prompt_tokens\": {{ input_request.usage.input_tokens }}, \"completion_tokens\": {{ input_request.usage.output_tokens }}, \"total_tokens\": {{ input_request.usage.input_tokens + input_request.usage.output_tokens }}, \"prompt_tokens_details\": { \"cached_tokens\": 0 }, \"completion_tokens_details\": { \"reasoning_tokens\": 0, \"accepted_prediction_tokens\": 0, \"rejected_prediction_tokens\": 0 } }, \"system_fingerprint\": \"fp_6b68a8204b\"} {% endif %}" } } } diff --git a/extensions/engine-management-extension/resources/mistral.json b/extensions/engine-management-extension/resources/mistral.json index 40a77cbf1..3f447dc4c 100644 --- a/extensions/engine-management-extension/resources/mistral.json +++ b/extensions/engine-management-extension/resources/mistral.json @@ -15,7 +15,7 @@ }, "transform_resp": { "chat_completions": { - "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"choices\" or key == \"created\" or key == \"model\" or key == \"service_tier\" or key == \"system_fingerprint\" or key == \"stream\" or key == \"object\" or key == \"usage\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }" + "template": "{{tojson(input_request)}}" } } } diff --git a/extensions/inference-cortex-extension/bin/version.txt b/extensions/inference-cortex-extension/bin/version.txt index fad0ae17f..d3b342f1a 100644 --- a/extensions/inference-cortex-extension/bin/version.txt +++ b/extensions/inference-cortex-extension/bin/version.txt @@ -1 +1 @@ -1.0.9-rc3 +1.0.9-rc4 diff --git a/web/containers/Layout/BottomPanel/SystemMonitor/TableActiveModel/index.tsx b/web/containers/Layout/BottomPanel/SystemMonitor/TableActiveModel/index.tsx index 06eebea92..7a7dd1e24 100644 --- a/web/containers/Layout/BottomPanel/SystemMonitor/TableActiveModel/index.tsx +++ b/web/containers/Layout/BottomPanel/SystemMonitor/TableActiveModel/index.tsx @@ -1,19 +1,20 @@ import { Tooltip, Button, Badge } from '@janhq/joi' -import { useAtom, useAtomValue } from 'jotai' +import { useAtom } from 'jotai' import { useActiveModel } from '@/hooks/useActiveModel' +import { useGetEngines } from '@/hooks/useEngineManagement' + import { toGibibytes } from '@/utils/converter' import { isLocalEngine } from '@/utils/modelEngine' -import { installedEnginesAtom } from '@/helpers/atoms/Engines.atom' import { serverEnabledAtom } from '@/helpers/atoms/LocalServer.atom' const TableActiveModel = () => { const { activeModel, stateModel, stopModel } = useActiveModel() - const engines = useAtomValue(installedEnginesAtom) + const { engines } = useGetEngines() const [serverEnabled, setServerEnabled] = useAtom(serverEnabledAtom) @@ -21,9 +22,7 @@ const TableActiveModel = () => {
- {activeModel && - engines && - isLocalEngine(engines, activeModel.engine) ? ( + {activeModel && isLocalEngine(engines, activeModel.engine) ? (
model.id === modelId) - setSelectedModel(model) + if (model) { + if ( + engines?.[model.engine]?.[0]?.type === 'local' || + (engines?.[model.engine]?.[0]?.api_key?.length ?? 0) > 0 + ) + setSelectedModel(model) + } else { + setSelectedModel(undefined) + } }, [ recommendedModel, activeThread, downloadedModels, setSelectedModel, activeAssistant?.model?.id, + engines, ]) const isLocalEngine = useCallback( diff --git a/web/containers/Providers/DataLoader.tsx b/web/containers/Providers/DataLoader.tsx index 01093e4b2..832e47d1a 100644 --- a/web/containers/Providers/DataLoader.tsx +++ b/web/containers/Providers/DataLoader.tsx @@ -2,11 +2,18 @@ import { Fragment, useEffect } from 'react' -import { AppConfiguration, getUserHomePath } from '@janhq/core' +import { + AppConfiguration, + EngineEvent, + events, + getUserHomePath, +} from '@janhq/core' import { useSetAtom } from 'jotai' +import { useDebouncedCallback } from 'use-debounce' + import useAssistants from '@/hooks/useAssistants' -import useEngines from '@/hooks/useEngines' +import { useGetEngines } from '@/hooks/useEngineManagement' import useGetSystemResources from '@/hooks/useGetSystemResources' import useModels from '@/hooks/useModels' import useThreads from '@/hooks/useThreads' @@ -26,7 +33,7 @@ const DataLoader: React.FC = () => { const setJanDefaultDataFolder = useSetAtom(defaultJanDataFolderAtom) const setJanSettingScreen = useSetAtom(janSettingScreenAtom) const { getData: loadModels } = useModels() - const { getData: loadEngines } = useEngines() + const { mutate } = useGetEngines() useThreads() useAssistants() @@ -35,9 +42,19 @@ const DataLoader: React.FC = () => { useEffect(() => { // Load data once loadModels() - loadEngines() // eslint-disable-next-line react-hooks/exhaustive-deps }, []) + const reloadData = useDebouncedCallback(() => { + mutate() + }, 300) + + useEffect(() => { + events.on(EngineEvent.OnEngineUpdate, reloadData) + return () => { + // Remove listener on unmount + events.off(EngineEvent.OnEngineUpdate, reloadData) + } + }, [reloadData]) useEffect(() => { window.core?.api diff --git a/web/containers/Providers/ModelHandler.tsx b/web/containers/Providers/ModelHandler.tsx index 32182804c..cc7a0da80 100644 --- a/web/containers/Providers/ModelHandler.tsx +++ b/web/containers/Providers/ModelHandler.tsx @@ -23,6 +23,8 @@ import { ulid } from 'ulidx' import { activeModelAtom, stateModelAtom } from '@/hooks/useActiveModel' +import { useGetEngines } from '@/hooks/useEngineManagement' + import { isLocalEngine } from '@/utils/modelEngine' import { extensionManager } from '@/extension' @@ -34,7 +36,6 @@ import { deleteMessageAtom, subscribedGeneratingMessageAtom, } from '@/helpers/atoms/ChatMessage.atom' -import { installedEnginesAtom } from '@/helpers/atoms/Engines.atom' import { downloadedModelsAtom } from '@/helpers/atoms/Model.atom' import { updateThreadWaitingForResponseAtom, @@ -75,7 +76,7 @@ export default function ModelHandler() { const activeModelParams = useAtomValue(getActiveThreadModelParamsAtom) const activeModelParamsRef = useRef(activeModelParams) const setTokenSpeed = useSetAtom(tokenSpeedAtom) - const engines = useAtomValue(installedEnginesAtom) + const { engines } = useGetEngines() useEffect(() => { activeThreadRef.current = activeThread @@ -336,7 +337,8 @@ export default function ModelHandler() { // Check model engine; we don't want to generate a title when it's not a local engine. remote model using first promp if ( - !isLocalEngine(engines, activeModelRef.current?.engine as InferenceEngine) + activeModelRef.current?.engine !== InferenceEngine.cortex && + activeModelRef.current?.engine !== InferenceEngine.cortex_llamacpp ) { const updatedThread: Thread = { ...thread, @@ -396,9 +398,7 @@ export default function ModelHandler() { // 2. Update the title with the result of the inference setTimeout(() => { - const engine = EngineManager.instance().get( - messageRequest.model?.engine ?? activeModelRef.current?.engine ?? '' - ) + const engine = EngineManager.instance().get(InferenceEngine.cortex) engine?.inference(messageRequest) }, 1000) } diff --git a/web/containers/Providers/SWRConfigProvider.tsx b/web/containers/Providers/SWRConfigProvider.tsx new file mode 100644 index 000000000..346385343 --- /dev/null +++ b/web/containers/Providers/SWRConfigProvider.tsx @@ -0,0 +1,30 @@ +'use client' + +import * as React from 'react' + +import { SWRConfig } from 'swr' + +function SWRConfigProvider({ children }: { children: React.ReactNode }) { + // https://swr.vercel.app/docs/advanced/cache#localstorage-based-persistent-cache + // When initializing, we restore the data from `localStorage` into a map. + + const map = React.useMemo(() => new Map(), []) + React.useEffect(() => { + const savedCache = JSON.parse( + window.localStorage.getItem('app-cache') || '[]' + ) + savedCache.forEach(([key, value]: [string, object]) => { + map.set(key, value) + }) + + // Before unloading the app, we write back all the data into `localStorage`. + window.addEventListener('beforeunload', () => { + const appCache = JSON.stringify(Array.from(map.entries())) + window.localStorage.setItem('app-cache', appCache) + }) + }, [map]) + + return map }}>{children} +} + +export default SWRConfigProvider diff --git a/web/containers/Providers/index.tsx b/web/containers/Providers/index.tsx index 67778e30c..bed0f07ec 100644 --- a/web/containers/Providers/index.tsx +++ b/web/containers/Providers/index.tsx @@ -4,6 +4,8 @@ import { PropsWithChildren } from 'react' import { Toaster } from 'react-hot-toast' +import { SWRConfig } from 'swr' + import EventListener from '@/containers/Providers/EventListener' import JotaiWrapper from '@/containers/Providers/Jotai' @@ -18,27 +20,30 @@ import DeepLinkListener from './DeepLinkListener' import KeyListener from './KeyListener' import Responsive from './Responsive' +import SWRConfigProvider from './SWRConfigProvider' import SettingsHandler from './SettingsHandler' const Providers = ({ children }: PropsWithChildren) => { return ( - - - - - <> - - - - - - - - {children} - - - - + + + + + + <> + + + + + + + + {children} + + + + + ) } diff --git a/web/helpers/atoms/Engines.atom.ts b/web/helpers/atoms/Engines.atom.ts deleted file mode 100644 index b1b09426a..000000000 --- a/web/helpers/atoms/Engines.atom.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { Engines } from '@janhq/core' -import { atom } from 'jotai' - -/** - * Store all of the installed engines including local and remote engines - */ -export const installedEnginesAtom = atom() diff --git a/web/hooks/useActiveModel.ts b/web/hooks/useActiveModel.ts index 25cc7fcc8..b4b8a5033 100644 --- a/web/hooks/useActiveModel.ts +++ b/web/hooks/useActiveModel.ts @@ -159,7 +159,7 @@ export function useActiveModel() { } if (!activeModel) return - const engine = EngineManager.instance().get(activeModel.engine) + const engine = EngineManager.instance().get(InferenceEngine.cortex) engine?.stopInference() }, [activeModel, stateModel, stopModel]) diff --git a/web/hooks/useConfigurations.ts b/web/hooks/useConfigurations.ts index 9d4999b2f..681239033 100644 --- a/web/hooks/useConfigurations.ts +++ b/web/hooks/useConfigurations.ts @@ -33,6 +33,7 @@ export const useConfigurations = () => { useEffect(() => { configurePullOptions() + // eslint-disable-next-line react-hooks/exhaustive-deps }, []) return { diff --git a/web/hooks/useEngines.ts b/web/hooks/useEngines.ts deleted file mode 100644 index f910394e3..000000000 --- a/web/hooks/useEngines.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { useCallback, useEffect } from 'react' - -import { - ExtensionTypeEnum, - events, - EngineEvent, - EngineManagementExtension, - Engines, -} from '@janhq/core' - -import { useSetAtom } from 'jotai' - -import { useDebouncedCallback } from 'use-debounce' - -import { extensionManager } from '@/extension' - -import { installedEnginesAtom } from '@/helpers/atoms/Engines.atom' - -/** - * useModels hook - Handles the state of models - * It fetches the downloaded models, configured models and default model from Model Extension - * and updates the atoms accordingly. - */ -const useEngines = () => { - const setInstalledEngines = useSetAtom(installedEnginesAtom) - - const getData = useCallback(() => { - getEngines().then(setInstalledEngines) - }, [setInstalledEngines]) - - const reloadData = useDebouncedCallback(() => getData(), 300) - - const getEngines = async (): Promise => - extensionManager - .get(ExtensionTypeEnum.Engine) - ?.getEngines() - .catch(() => ({}) as Engines) ?? ({} as Engines) - - useEffect(() => { - // Listen for engine updates - events.on(EngineEvent.OnEngineUpdate, reloadData) - return () => { - // Remove listener on unmount - events.off(EngineEvent.OnEngineUpdate, reloadData) - } - }, [reloadData]) - - return { - getData, - } -} - -export default useEngines diff --git a/web/hooks/useRecommendedModel.ts b/web/hooks/useRecommendedModel.ts index 03bcc4a30..4107087f8 100644 --- a/web/hooks/useRecommendedModel.ts +++ b/web/hooks/useRecommendedModel.ts @@ -6,6 +6,8 @@ import { atom, useAtomValue } from 'jotai' import { activeModelAtom } from './useActiveModel' +import { useGetEngines } from './useEngineManagement' + import { activeAssistantAtom } from '@/helpers/atoms/Assistant.atom' import { downloadedModelsAtom } from '@/helpers/atoms/Model.atom' import { activeThreadAtom } from '@/helpers/atoms/Thread.atom' @@ -30,6 +32,7 @@ export default function useRecommendedModel() { const activeThread = useAtomValue(activeThreadAtom) const downloadedModels = useAtomValue(downloadedModelsAtom) const activeAssistant = useAtomValue(activeAssistantAtom) + const { engines } = useGetEngines() const getAndSortDownloadedModels = useCallback(async (): Promise => { const models = downloadedModels.sort((a, b) => @@ -45,7 +48,12 @@ export default function useRecommendedModel() { const getRecommendedModel = useCallback(async (): Promise< Model | undefined > => { - const models = await getAndSortDownloadedModels() + const models = (await getAndSortDownloadedModels()).filter((e: Model) => + engines?.[e.engine]?.[0].type === 'local' || + (engines?.[e.engine]?.[0].api_key?.length ?? 0) > 0 + ? true + : false + ) if (!activeThread || !activeAssistant) return const modelId = activeAssistant.model.id @@ -63,10 +71,8 @@ export default function useRecommendedModel() { } // sort the model, for display purpose - if (models.length === 0) { // if we have no downloaded models, then can't recommend anything - console.debug("No downloaded models, can't recommend anything") setRecommendedModel(undefined) return } @@ -94,7 +100,7 @@ export default function useRecommendedModel() { setRecommendedModel(lastUsedModel) // eslint-disable-next-line react-hooks/exhaustive-deps - }, [getAndSortDownloadedModels, activeThread]) + }, [getAndSortDownloadedModels, activeThread, engines]) useEffect(() => { getRecommendedModel() diff --git a/web/hooks/useStarterScreen.ts b/web/hooks/useStarterScreen.ts index 23a659a04..f0bc7eeda 100644 --- a/web/hooks/useStarterScreen.ts +++ b/web/hooks/useStarterScreen.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ import { useMemo } from 'react' import { InferenceEngine, EngineConfig } from '@janhq/core' @@ -5,7 +6,8 @@ import { useAtomValue } from 'jotai' import { isLocalEngine } from '@/utils/modelEngine' -import { installedEnginesAtom } from '@/helpers/atoms/Engines.atom' +import { useGetEngines } from './useEngineManagement' + import { downloadedModelsAtom } from '@/helpers/atoms/Model.atom' import { threadsAtom } from '@/helpers/atoms/Thread.atom' @@ -13,7 +15,7 @@ export function useStarterScreen() { const downloadedModels = useAtomValue(downloadedModelsAtom) const threads = useAtomValue(threadsAtom) - const engines = useAtomValue(installedEnginesAtom) + const { engines } = useGetEngines() const remoteEngines = engines && diff --git a/web/screens/Hub/ModelList/index.tsx b/web/screens/Hub/ModelList/index.tsx index 0d7865a81..4c6c7d993 100644 --- a/web/screens/Hub/ModelList/index.tsx +++ b/web/screens/Hub/ModelList/index.tsx @@ -4,6 +4,8 @@ import { Model } from '@janhq/core' import { useAtomValue } from 'jotai' +import { useGetEngines } from '@/hooks/useEngineManagement' + import ModelItem from '@/screens/Hub/ModelList/ModelItem' import { downloadedModelsAtom } from '@/helpers/atoms/Model.atom' @@ -14,6 +16,7 @@ type Props = { const ModelList = ({ models }: Props) => { const downloadedModels = useAtomValue(downloadedModelsAtom) + const { engines } = useGetEngines() const sortedModels: Model[] = useMemo(() => { const featuredModels: Model[] = [] const remoteModels: Model[] = [] @@ -22,7 +25,7 @@ const ModelList = ({ models }: Props) => { models.forEach((m) => { if (m.metadata?.tags?.includes('Featured')) { featuredModels.push(m) - } else if (m.format === 'api') { + } else if (engines?.[m.engine]?.[0]?.type === 'remote') { remoteModels.push(m) } else if (downloadedModels.map((m) => m.id).includes(m.id)) { localModels.push(m) @@ -40,7 +43,7 @@ const ModelList = ({ models }: Props) => { ...remainingModels, ...remoteModels, ] - }, [models, downloadedModels]) + }, [models, downloadedModels, engines]) return (
diff --git a/web/screens/LocalServer/LocalServerLeftPanel/index.tsx b/web/screens/LocalServer/LocalServerLeftPanel/index.tsx index a5250b013..99c2d7488 100644 --- a/web/screens/LocalServer/LocalServerLeftPanel/index.tsx +++ b/web/screens/LocalServer/LocalServerLeftPanel/index.tsx @@ -1,6 +1,11 @@ import { Fragment, useCallback, useState } from 'react' -import { EngineManager, Model, ModelSettingParams } from '@janhq/core' +import { + EngineManager, + InferenceEngine, + Model, + ModelSettingParams, +} from '@janhq/core' import { Button, Tooltip, Select, Input, Checkbox } from '@janhq/joi' import { useAtom, useAtomValue, useSetAtom } from 'jotai' @@ -94,7 +99,7 @@ const LocalServerLeftPanel = () => { localStorage.setItem(FIRST_TIME_VISIT_API_SERVER, 'false') setFirstTimeVisitAPIServer(false) } - const engine = EngineManager.instance().get((model as Model).engine) + const engine = EngineManager.instance().get(InferenceEngine.cortex) engine?.loadModel(model as Model) // startModel(selectedModel.id, false).catch((e) => console.error(e)) setIsLoading(false) diff --git a/web/screens/Settings/Engines/RemoteEngineSettings.tsx b/web/screens/Settings/Engines/RemoteEngineSettings.tsx index 5396b86e0..cb05f7f2b 100644 --- a/web/screens/Settings/Engines/RemoteEngineSettings.tsx +++ b/web/screens/Settings/Engines/RemoteEngineSettings.tsx @@ -1,11 +1,14 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ /* eslint-disable react/no-unescaped-entities */ +/* eslint-disable @typescript-eslint/no-unused-vars */ import React, { useCallback, useRef, useState, useEffect } from 'react' import { EngineConfig as OriginalEngineConfig, InferenceEngine, + events, + EngineEvent, } from '@janhq/core' interface EngineConfig extends OriginalEngineConfig { @@ -64,6 +67,7 @@ const RemoteEngineSettings = ({ set(updatedEngine, field, value) await updateEngine(name, updatedEngine) mutate() + events.emit(EngineEvent.OnEngineUpdate, {}) }, 300) }, [engine, name, mutate] @@ -115,6 +119,8 @@ const RemoteEngineSettings = ({ } }, [engine]) + if (!engine) return null + return (
diff --git a/web/screens/Settings/Engines/index.tsx b/web/screens/Settings/Engines/index.tsx index a79ba5be6..4ad155939 100644 --- a/web/screens/Settings/Engines/index.tsx +++ b/web/screens/Settings/Engines/index.tsx @@ -4,16 +4,16 @@ import { InferenceEngine } from '@janhq/core' import { ScrollArea } from '@janhq/joi' import { useAtomValue } from 'jotai' +import { useGetEngines } from '@/hooks/useEngineManagement' + import { isLocalEngine } from '@/utils/modelEngine' import LocalEngineItems from './LocalEngineItem' import ModalAddRemoteEngine from './ModalAddRemoteEngine' import RemoteEngineItems from './RemoteEngineItem' -import { installedEnginesAtom } from '@/helpers/atoms/Engines.atom' - const Engines = () => { - const engines = useAtomValue(installedEnginesAtom) + const { engines } = useGetEngines() return ( diff --git a/web/screens/Settings/MyModels/MyModelList/index.tsx b/web/screens/Settings/MyModels/MyModelList/index.tsx index 61f22238e..6824d1471 100644 --- a/web/screens/Settings/MyModels/MyModelList/index.tsx +++ b/web/screens/Settings/MyModels/MyModelList/index.tsx @@ -2,7 +2,7 @@ import { memo, useState } from 'react' import { Model } from '@janhq/core' import { Badge, Button, Tooltip, useClickOutside } from '@janhq/joi' -import { useAtom, useAtomValue } from 'jotai' +import { useAtom } from 'jotai' import { MoreVerticalIcon, PlayIcon, @@ -14,11 +14,12 @@ import { twMerge } from 'tailwind-merge' import { useActiveModel } from '@/hooks/useActiveModel' import useDeleteModel from '@/hooks/useDeleteModel' +import { useGetEngines } from '@/hooks/useEngineManagement' + import { toGibibytes } from '@/utils/converter' import { isLocalEngine } from '@/utils/modelEngine' -import { installedEnginesAtom } from '@/helpers/atoms/Engines.atom' import { serverEnabledAtom } from '@/helpers/atoms/LocalServer.atom' type Props = { @@ -32,7 +33,7 @@ const MyModelList = ({ model }: Props) => { const { deleteModel } = useDeleteModel() const [more, setMore] = useState(false) const [serverEnabled, setServerEnabled] = useAtom(serverEnabledAtom) - const engines = useAtomValue(installedEnginesAtom) + const { engines } = useGetEngines() const [menu, setMenu] = useState(null) const [toggle, setToggle] = useState(null) diff --git a/web/screens/Settings/MyModels/index.tsx b/web/screens/Settings/MyModels/index.tsx index 7128d7016..e4d1b9c47 100644 --- a/web/screens/Settings/MyModels/index.tsx +++ b/web/screens/Settings/MyModels/index.tsx @@ -36,8 +36,6 @@ import { import MyModelList from './MyModelList' -import { extensionManager } from '@/extension' - import { downloadedModelsAtom, showEngineListModelAtom, @@ -52,9 +50,6 @@ const MyModels = () => { showEngineListModelAtom ) - const [extensionHasSettings, setExtensionHasSettings] = useState< - { name?: string; setting: string; apiKey: string; provider: string }[] - >([]) const { engines } = useGetEngines() const isLocalEngine = useCallback( @@ -97,45 +92,6 @@ const MyModels = () => { setSearchText(input) }, []) - useEffect(() => { - const getAllSettings = async () => { - const extensionsMenu: { - name?: string - setting: string - apiKey: string - provider: string - }[] = [] - const extensions = extensionManager.getAll() - - for (const extension of extensions) { - if (typeof extension.getSettings === 'function') { - const settings = await extension.getSettings() - - if ( - (settings && settings.length > 0) || - (await extension.installationState()) !== 'NotRequired' - ) { - extensionsMenu.push({ - name: extension.productName, - setting: extension.name, - apiKey: - 'apiKey' in extension && typeof extension.apiKey === 'string' - ? extension.apiKey - : '', - provider: - 'provider' in extension && - typeof extension.provider === 'string' - ? extension.provider - : '', - }) - } - } - } - setExtensionHasSettings(extensionsMenu) - } - getAllSettings() - }, []) - const findByEngine = filteredDownloadedModels.map((x) => { // Legacy engine support - they will be grouped under Cortex LlamaCPP if (x.engine === InferenceEngine.nitro) @@ -158,9 +114,11 @@ const MyModels = () => { } }) - const getEngineStatusReady: InferenceEngine[] = extensionHasSettings - ?.filter((e) => e.apiKey.length > 0) - .map((x) => x.provider as InferenceEngine) + const getEngineStatusReady: InferenceEngine[] = Object.entries(engines ?? {}) + // eslint-disable-next-line @typescript-eslint/no-unused-vars + ?.filter(([_, value]) => (value?.[0]?.api_key?.length ?? 0) > 0) + // eslint-disable-next-line @typescript-eslint/no-unused-vars + .map(([key, _]) => key as InferenceEngine) useEffect(() => { setShowEngineListModel((prev) => [ @@ -168,7 +126,7 @@ const MyModels = () => { ...(getEngineStatusReady as InferenceEngine[]), ]) // eslint-disable-next-line react-hooks/exhaustive-deps - }, [setShowEngineListModel, extensionHasSettings]) + }, [setShowEngineListModel, engines]) return (
diff --git a/web/screens/Settings/SettingDetail/index.tsx b/web/screens/Settings/SettingDetail/index.tsx index 1e4b79282..d4a2c4d82 100644 --- a/web/screens/Settings/SettingDetail/index.tsx +++ b/web/screens/Settings/SettingDetail/index.tsx @@ -1,6 +1,8 @@ import { InferenceEngine } from '@janhq/core' import { useAtomValue } from 'jotai' +import { useGetEngines } from '@/hooks/useEngineManagement' + import Advanced from '@/screens/Settings/Advanced' import AppearanceOptions from '@/screens/Settings/Appearance' import ExtensionCatalog from '@/screens/Settings/CoreExtensions' @@ -14,12 +16,11 @@ import Privacy from '@/screens/Settings/Privacy' import { isLocalEngine } from '@/utils/modelEngine' -import { installedEnginesAtom } from '@/helpers/atoms/Engines.atom' import { selectedSettingAtom } from '@/helpers/atoms/Setting.atom' const SettingDetail = () => { const selectedSetting = useAtomValue(selectedSettingAtom) - const engines = useAtomValue(installedEnginesAtom) + const { engines } = useGetEngines() switch (selectedSetting) { case 'Engines': diff --git a/web/screens/Settings/SettingLeftPanel/index.tsx b/web/screens/Settings/SettingLeftPanel/index.tsx index 564ffc6d2..db4c9108c 100644 --- a/web/screens/Settings/SettingLeftPanel/index.tsx +++ b/web/screens/Settings/SettingLeftPanel/index.tsx @@ -6,12 +6,13 @@ import { useAtomValue } from 'jotai' import LeftPanelContainer from '@/containers/LeftPanelContainer' +import { useGetEngines } from '@/hooks/useEngineManagement' + import { getTitleByEngine, isLocalEngine } from '@/utils/modelEngine' import SettingItem from './SettingItem' import { extensionManager } from '@/extension' -import { installedEnginesAtom } from '@/helpers/atoms/Engines.atom' import { showSettingActiveLocalEngineAtom, @@ -20,7 +21,7 @@ import { import { janSettingScreenAtom } from '@/helpers/atoms/Setting.atom' const SettingLeftPanel = () => { - const engines = useAtomValue(installedEnginesAtom) + const { engines } = useGetEngines() const settingScreens = useAtomValue(janSettingScreenAtom) const showSettingActiveLocalEngine = useAtomValue( diff --git a/web/screens/Thread/ThreadCenterPanel/AssistantSetting/index.tsx b/web/screens/Thread/ThreadCenterPanel/AssistantSetting/index.tsx index d04f9b233..1ecb6f7fd 100644 --- a/web/screens/Thread/ThreadCenterPanel/AssistantSetting/index.tsx +++ b/web/screens/Thread/ThreadCenterPanel/AssistantSetting/index.tsx @@ -88,6 +88,7 @@ const AssistantSetting: React.FC = ({ componentData }) => { setEngineParamsUpdate, stopModel, updateThreadMetadata, + resetGenerating, ] ) diff --git a/web/screens/Thread/ThreadCenterPanel/ChatBody/EmptyThread/index.tsx b/web/screens/Thread/ThreadCenterPanel/ChatBody/EmptyThread/index.tsx index ae377eb24..e69d22b48 100644 --- a/web/screens/Thread/ThreadCenterPanel/ChatBody/EmptyThread/index.tsx +++ b/web/screens/Thread/ThreadCenterPanel/ChatBody/EmptyThread/index.tsx @@ -7,16 +7,17 @@ import LogoMark from '@/containers/Brand/Logo/Mark' import { MainViewState } from '@/constants/screens' +import { useGetEngines } from '@/hooks/useEngineManagement' + import { isLocalEngine } from '@/utils/modelEngine' import { mainViewStateAtom } from '@/helpers/atoms/App.atom' -import { installedEnginesAtom } from '@/helpers/atoms/Engines.atom' import { downloadedModelsAtom } from '@/helpers/atoms/Model.atom' const EmptyThread = () => { const downloadedModels = useAtomValue(downloadedModelsAtom) const setMainViewState = useSetAtom(mainViewStateAtom) - const engines = useAtomValue(installedEnginesAtom) + const { engines } = useGetEngines() const showOnboardingStep = useMemo( () => !downloadedModels.some( diff --git a/web/screens/Thread/ThreadCenterPanel/ChatBody/OnDeviceStarterScreen/index.tsx b/web/screens/Thread/ThreadCenterPanel/ChatBody/OnDeviceStarterScreen/index.tsx index e05e793b5..70f7ef476 100644 --- a/web/screens/Thread/ThreadCenterPanel/ChatBody/OnDeviceStarterScreen/index.tsx +++ b/web/screens/Thread/ThreadCenterPanel/ChatBody/OnDeviceStarterScreen/index.tsx @@ -24,6 +24,8 @@ import useDownloadModel from '@/hooks/useDownloadModel' import { modelDownloadStateAtom } from '@/hooks/useDownloadState' +import { useGetEngines } from '@/hooks/useEngineManagement' + import { formatDownloadPercentage, toGibibytes } from '@/utils/converter' import { manualRecommendationModel } from '@/utils/model' import { @@ -33,7 +35,6 @@ import { } from '@/utils/modelEngine' import { mainViewStateAtom } from '@/helpers/atoms/App.atom' -import { installedEnginesAtom } from '@/helpers/atoms/Engines.atom' import { configuredModelsAtom, getDownloadingModelAtom, @@ -51,7 +52,7 @@ const OnDeviceStarterScreen = ({ isShowStarterScreen }: Props) => { const { downloadModel } = useDownloadModel() const downloadStates = useAtomValue(modelDownloadStateAtom) const setSelectedSetting = useSetAtom(selectedSettingAtom) - const engines = useAtomValue(installedEnginesAtom) + const { engines } = useGetEngines() const configuredModels = useAtomValue(configuredModelsAtom) const setMainViewState = useSetAtom(mainViewStateAtom) @@ -298,40 +299,46 @@ const OnDeviceStarterScreen = ({ isShowStarterScreen }: Props) => { key={rowIndex} className="my-2 flex items-center gap-4 md:gap-10" > - {row.map((remoteEngine) => { - const engineLogo = getLogoEngine( - remoteEngine as InferenceEngine + {row + .filter( + (e) => + engines?.[e as InferenceEngine]?.[0]?.type === + 'remote' ) + .map((remoteEngine) => { + const engineLogo = getLogoEngine( + remoteEngine as InferenceEngine + ) - return ( -
{ - setMainViewState(MainViewState.Settings) - setSelectedSetting( - remoteEngine as InferenceEngine - ) - }} - > - {engineLogo && ( - Engine logo - )} - -

- {getTitleByEngine( - remoteEngine as InferenceEngine + return ( +

{ + setMainViewState(MainViewState.Settings) + setSelectedSetting( + remoteEngine as InferenceEngine + ) + }} + > + {engineLogo && ( + Engine logo )} -

-
- ) - })} + +

+ {getTitleByEngine( + remoteEngine as InferenceEngine + )} +

+
+ ) + })}
) })} diff --git a/web/screens/Thread/ThreadCenterPanel/ChatInput/index.tsx b/web/screens/Thread/ThreadCenterPanel/ChatInput/index.tsx index e70047d05..457758749 100644 --- a/web/screens/Thread/ThreadCenterPanel/ChatInput/index.tsx +++ b/web/screens/Thread/ThreadCenterPanel/ChatInput/index.tsx @@ -22,6 +22,7 @@ import { currentPromptAtom, fileUploadAtom } from '@/containers/Providers/Jotai' import { useActiveModel } from '@/hooks/useActiveModel' +import { useGetEngines } from '@/hooks/useEngineManagement' import useSendChatMessage from '@/hooks/useSendChatMessage' import { uploader } from '@/utils/file' @@ -35,7 +36,6 @@ import RichTextEditor from './RichTextEditor' import { showRightPanelAtom } from '@/helpers/atoms/App.atom' import { experimentalFeatureEnabledAtom } from '@/helpers/atoms/AppConfig.atom' import { activeAssistantAtom } from '@/helpers/atoms/Assistant.atom' -import { installedEnginesAtom } from '@/helpers/atoms/Engines.atom' import { selectedModelAtom } from '@/helpers/atoms/Model.atom' import { spellCheckAtom } from '@/helpers/atoms/Setting.atom' import { @@ -64,7 +64,7 @@ const ChatInput = () => { const textareaRef = useRef(null) const fileInputRef = useRef(null) const imageInputRef = useRef(null) - const engines = useAtomValue(installedEnginesAtom) + const { engines } = useGetEngines() const experimentalFeature = useAtomValue(experimentalFeatureEnabledAtom) const isBlockingSend = useAtomValue(isBlockingSendAtom) const activeAssistant = useAtomValue(activeAssistantAtom) diff --git a/web/screens/Thread/ThreadCenterPanel/LoadModelError/index.tsx b/web/screens/Thread/ThreadCenterPanel/LoadModelError/index.tsx index 204ec40fb..3a887e8ea 100644 --- a/web/screens/Thread/ThreadCenterPanel/LoadModelError/index.tsx +++ b/web/screens/Thread/ThreadCenterPanel/LoadModelError/index.tsx @@ -1,4 +1,4 @@ -import { EngineManager } from '@janhq/core' +import { EngineManager, InferenceEngine } from '@janhq/core' import { useAtomValue, useSetAtom } from 'jotai' import ModalTroubleShooting, { @@ -35,7 +35,7 @@ const LoadModelError = () => { setMainState(MainViewState.Settings) if (activeAssistant?.model.engine) { const engine = EngineManager.instance().get( - activeAssistant.model.engine + InferenceEngine.cortex ) engine?.name && setSelectedSettingScreen(engine.name) } diff --git a/web/screens/Thread/ThreadRightPanel/index.tsx b/web/screens/Thread/ThreadRightPanel/index.tsx index 20f365a88..ba801fd0b 100644 --- a/web/screens/Thread/ThreadRightPanel/index.tsx +++ b/web/screens/Thread/ThreadRightPanel/index.tsx @@ -29,6 +29,7 @@ import RightPanelContainer from '@/containers/RightPanelContainer' import { useActiveModel } from '@/hooks/useActiveModel' import { useCreateNewThread } from '@/hooks/useCreateNewThread' +import { useGetEngines } from '@/hooks/useEngineManagement' import useUpdateModelParameters from '@/hooks/useUpdateModelParameters' import { getConfigurationsData } from '@/utils/componentSettings' @@ -39,7 +40,7 @@ import Tools from './Tools' import { experimentalFeatureEnabledAtom } from '@/helpers/atoms/AppConfig.atom' import { activeAssistantAtom } from '@/helpers/atoms/Assistant.atom' -import { installedEnginesAtom } from '@/helpers/atoms/Engines.atom' + import { selectedModelAtom } from '@/helpers/atoms/Model.atom' import { activeThreadAtom, @@ -61,7 +62,7 @@ const ThreadRightPanel = () => { const [activeTabThreadRightPanel, setActiveTabThreadRightPanel] = useAtom( activeTabThreadRightPanelAtom ) - const engines = useAtomValue(installedEnginesAtom) + const { engines } = useGetEngines() const { updateThreadMetadata } = useCreateNewThread() const experimentalFeature = useAtomValue(experimentalFeatureEnabledAtom) diff --git a/web/utils/messageRequestBuilder.ts b/web/utils/messageRequestBuilder.ts index b75dbecb0..c3da9cbd8 100644 --- a/web/utils/messageRequestBuilder.ts +++ b/web/utils/messageRequestBuilder.ts @@ -167,7 +167,6 @@ export class MessageRequestBuilder { messages: this.normalizeMessages(this.messages), model: this.model, thread: this.thread, - engine: this.model.engine, } } }