From bd850fb3571fd3c45320254a84737852d839c5cc Mon Sep 17 00:00:00 2001 From: Louis Date: Tue, 19 Nov 2024 12:37:41 +0700 Subject: [PATCH 1/4] chore: reduce destroy attempts --- extensions/inference-cortex-extension/bin/version.txt | 2 +- extensions/inference-cortex-extension/src/index.ts | 8 ++++++-- extensions/model-extension/src/cortex.ts | 5 +++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/extensions/inference-cortex-extension/bin/version.txt b/extensions/inference-cortex-extension/bin/version.txt index 89f843d1d..b26ebdeac 100644 --- a/extensions/inference-cortex-extension/bin/version.txt +++ b/extensions/inference-cortex-extension/bin/version.txt @@ -1 +1 @@ -1.0.3-rc1 \ No newline at end of file +1.0.3-rc2 \ No newline at end of file diff --git a/extensions/inference-cortex-extension/src/index.ts b/extensions/inference-cortex-extension/src/index.ts index 1fb78c13e..8236d7de4 100644 --- a/extensions/inference-cortex-extension/src/index.ts +++ b/extensions/inference-cortex-extension/src/index.ts @@ -69,11 +69,11 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine { super.onLoad() + await this.queue.add(() => this.clean()) this.queue.add(() => this.healthz()) this.queue.add(() => this.setDefaultEngine(systemInfo)) // Run the process watchdog const systemInfo = await systemInformation() - await this.clean() await executeOnMain(NODE, 'run', systemInfo) this.subscribeToEvents() @@ -160,7 +160,8 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine { return ky .get(`${CORTEX_API_URL}/healthz`, { retry: { - limit: 10, + limit: 20, + delay: () => 500, methods: ['get'], }, }) @@ -192,6 +193,9 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine { return ky .delete(`${CORTEX_API_URL}/processmanager/destroy`, { timeout: 2000, // maximum 2 seconds + retry: { + limit: 0, + }, }) .catch(() => { // Do nothing diff --git a/extensions/model-extension/src/cortex.ts b/extensions/model-extension/src/cortex.ts index 7a65e8e3f..f81cda553 100644 --- a/extensions/model-extension/src/cortex.ts +++ b/extensions/model-extension/src/cortex.ts @@ -1,6 +1,6 @@ import PQueue from 'p-queue' import ky from 'ky' -import { extractModelLoadParams, Model } from '@janhq/core' +import { extractModelLoadParams, Model } from '@janhq/core' import { extractInferenceParams } from '@janhq/core' /** * cortex.cpp Model APIs interface @@ -155,7 +155,8 @@ export class CortexAPI implements ICortexAPI { return ky .get(`${API_URL}/healthz`, { retry: { - limit: 10, + limit: 20, + delay: () => 500, methods: ['get'], }, }) From 28add39a5184ed503aedd723671723cc205fd0af Mon Sep 17 00:00:00 2001 From: Louis Date: Tue, 19 Nov 2024 13:28:07 +0700 Subject: [PATCH 2/4] chore: add model pull options - correct url path --- core/src/browser/extensions/model.ts | 1 + extensions/model-extension/src/cortex.ts | 19 ++++++++++--- extensions/model-extension/src/index.ts | 7 +++++ .../monitoring-extension/src/node/index.ts | 4 +-- web/containers/Providers/DataLoader.tsx | 4 ++- web/hooks/useModels.ts | 27 ++++++++++++++++++- web/screens/Settings/Advanced/index.tsx | 26 +++++++++++++++--- 7 files changed, 77 insertions(+), 11 deletions(-) diff --git a/core/src/browser/extensions/model.ts b/core/src/browser/extensions/model.ts index e224ec5cc..9a3428988 100644 --- a/core/src/browser/extensions/model.ts +++ b/core/src/browser/extensions/model.ts @@ -12,6 +12,7 @@ export abstract class ModelExtension extends BaseExtension implements ModelInter return ExtensionTypeEnum.Model } + abstract configurePullOptions(configs: { [key: string]: any }): Promise abstract getModels(): Promise abstract pullModel(model: string, id?: string, name?: string): Promise abstract cancelModelPull(modelId: string): Promise diff --git a/extensions/model-extension/src/cortex.ts b/extensions/model-extension/src/cortex.ts index f81cda553..26316fbbc 100644 --- a/extensions/model-extension/src/cortex.ts +++ b/extensions/model-extension/src/cortex.ts @@ -18,6 +18,7 @@ interface ICortexAPI { deleteModel(model: string): Promise updateModel(model: object): Promise cancelModelPull(model: string): Promise + configs(body: { [key: string]: any }): Promise } type ModelList = { @@ -52,7 +53,7 @@ export class CortexAPI implements ICortexAPI { */ getModels(): Promise { return this.queue - .add(() => ky.get(`${API_URL}/models`).json()) + .add(() => ky.get(`${API_URL}/v1/models`).json()) .then((e) => typeof e === 'object' ? e.data.map((e) => this.transformModel(e)) : [] ) @@ -104,7 +105,7 @@ export class CortexAPI implements ICortexAPI { */ deleteModel(model: string): Promise { return this.queue.add(() => - ky.delete(`${API_URL}/models/${model}`).json().then() + ky.delete(`${API_URL}/v1/models/${model}`).json().then() ) } @@ -130,7 +131,7 @@ export class CortexAPI implements ICortexAPI { cancelModelPull(model: string): Promise { return this.queue.add(() => ky - .delete(`${API_URL}/models/pull`, { json: { taskId: model } }) + .delete(`${API_URL}/v1/models/pull`, { json: { taskId: model } }) .json() .then() ) @@ -142,7 +143,7 @@ export class CortexAPI implements ICortexAPI { */ async getModelStatus(model: string): Promise { return this.queue - .add(() => ky.get(`${API_URL}/models/status/${model}`)) + .add(() => ky.get(`${API_URL}/v1/models/status/${model}`)) .then((e) => true) .catch(() => false) } @@ -163,6 +164,16 @@ export class CortexAPI implements ICortexAPI { .then(() => {}) } + /** + * Configure model pull options + * @param body + */ + configs(body: { [key: string]: any }): Promise { + return this.queue.add(() => + ky.patch(`${API_URL}/v1/configs`, { json: body }).then(() => {}) + ) + } + /** * TRansform model to the expected format (e.g. parameters, settings, metadata) * @param model diff --git a/extensions/model-extension/src/index.ts b/extensions/model-extension/src/index.ts index 38c57e916..f1ce069f6 100644 --- a/extensions/model-extension/src/index.ts +++ b/extensions/model-extension/src/index.ts @@ -227,6 +227,13 @@ export default class JanModelExtension extends ModelExtension { return this.cortexAPI.getModelStatus(model) } + /** + * Configure pull options such as proxy, headers, etc. + */ + async configurePullOptions(options: { [key: string]: any }): Promise { + return this.cortexAPI.configs(options).catch((e) => console.debug(e)) + } + /** * Handle download state from main app */ diff --git a/extensions/monitoring-extension/src/node/index.ts b/extensions/monitoring-extension/src/node/index.ts index 980ee75d1..a900490f3 100644 --- a/extensions/monitoring-extension/src/node/index.ts +++ b/extensions/monitoring-extension/src/node/index.ts @@ -267,7 +267,7 @@ const updateGpuInfo = async () => } data = await updateCudaExistence(data) - console.log(data) + console.log('[MONITORING]::Cuda info: ', data) writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) log(`[APP]::${JSON.stringify(data)}`) resolve({}) @@ -344,7 +344,7 @@ const updateCudaExistence = async ( data.cuda.version = match[1] } } - console.log(data) + console.log('[MONITORING]::Finalized cuda info update: ', data) resolve() }) }) diff --git a/web/containers/Providers/DataLoader.tsx b/web/containers/Providers/DataLoader.tsx index d3d747d02..245c254ac 100644 --- a/web/containers/Providers/DataLoader.tsx +++ b/web/containers/Providers/DataLoader.tsx @@ -29,7 +29,7 @@ const DataLoader: React.FC = ({ children }) => { const setQuickAskEnabled = useSetAtom(quickAskEnabledAtom) const setJanDefaultDataFolder = useSetAtom(defaultJanDataFolderAtom) const setJanSettingScreen = useSetAtom(janSettingScreenAtom) - const { loadDataModel } = useModels() + const { loadDataModel, configurePullOptions } = useModels() useThreads() useAssistants() @@ -39,6 +39,8 @@ const DataLoader: React.FC = ({ children }) => { useEffect(() => { // Load data once loadDataModel() + // Configure pull options once + configurePullOptions() // eslint-disable-next-line react-hooks/exhaustive-deps }, []) diff --git a/web/hooks/useModels.ts b/web/hooks/useModels.ts index 88ec3afb5..75c86035a 100644 --- a/web/hooks/useModels.ts +++ b/web/hooks/useModels.ts @@ -9,13 +9,18 @@ import { ModelManager, } from '@janhq/core' -import { useSetAtom, useAtom } from 'jotai' +import { useSetAtom, useAtom, useAtomValue } from 'jotai' import { useDebouncedCallback } from 'use-debounce' import { isLocalEngine } from '@/utils/modelEngine' import { extensionManager } from '@/extension' +import { + ignoreSslAtom, + proxyAtom, + proxyEnabledAtom, +} from '@/helpers/atoms/AppConfig.atom' import { configuredModelsAtom, downloadedModelsAtom, @@ -29,6 +34,9 @@ import { const useModels = () => { const [downloadedModels, setDownloadedModels] = useAtom(downloadedModelsAtom) const setExtensionModels = useSetAtom(configuredModelsAtom) + const proxyEnabled = useAtomValue(proxyEnabledAtom) + const proxyUrl = useAtomValue(proxyAtom) + const proxyIgnoreSSL = useAtomValue(ignoreSslAtom) const getData = useCallback(() => { const getDownloadedModels = async () => { @@ -107,8 +115,25 @@ const useModels = () => { } }, [reloadData, updateStates]) + const configurePullOptions = useCallback(() => { + extensionManager + .get(ExtensionTypeEnum.Model) + ?.configurePullOptions( + proxyEnabled + ? { + proxy_url: proxyUrl, + verify_peer_ssl: !proxyIgnoreSSL, + } + : { + proxy_url: '', + verify_peer_ssl: false, + } + ) + }, [proxyEnabled, proxyUrl, proxyIgnoreSSL]) + return { loadDataModel: getData, + configurePullOptions, } } diff --git a/web/screens/Settings/Advanced/index.tsx b/web/screens/Settings/Advanced/index.tsx index 150f70398..a5956fc33 100644 --- a/web/screens/Settings/Advanced/index.tsx +++ b/web/screens/Settings/Advanced/index.tsx @@ -20,9 +20,12 @@ import { AlertTriangleIcon, AlertCircleIcon } from 'lucide-react' import { twMerge } from 'tailwind-merge' +import { useDebouncedCallback } from 'use-debounce' + import { snackbar, toaster } from '@/containers/Toast' import { useActiveModel } from '@/hooks/useActiveModel' +import useModels from '@/hooks/useModels' import { useSettings } from '@/hooks/useSettings' import DataFolder from './DataFolder' @@ -65,6 +68,7 @@ const Advanced = () => { const [dropdownOptions, setDropdownOptions] = useState( null ) + const { configurePullOptions } = useModels() const [toggle, setToggle] = useState(null) @@ -78,6 +82,15 @@ const Advanced = () => { return y['name'] }) + /** + * There could be a case where the state update is not synced + * so that retrieving state value from other hooks would not be accurate + * there is also a case where state update persist everytime user type in the input + */ + const updatePullOptions = useDebouncedCallback( + () => configurePullOptions(), + 300 + ) /** * Handle proxy change */ @@ -90,8 +103,9 @@ const Advanced = () => { } else { setProxy('') } + updatePullOptions() }, - [setPartialProxy, setProxy] + [setPartialProxy, setProxy, updatePullOptions] ) /** @@ -452,7 +466,10 @@ const Advanced = () => { setProxyEnabled(!proxyEnabled)} + onChange={() => { + setProxyEnabled(!proxyEnabled) + updatePullOptions() + }} />
{ setIgnoreSSL(e.target.checked)} + onChange={(e) => { + setIgnoreSSL(e.target.checked) + updatePullOptions() + }} />
From 52c520d2c389f73e5bbc82d478db924ea19144ef Mon Sep 17 00:00:00 2001 From: Louis Date: Tue, 19 Nov 2024 19:43:12 +0700 Subject: [PATCH 3/4] fix: app does not relaunch on Linux - add tests --- web/hooks/useModels.test.ts | 82 ++++++++++++++++++++++++- web/screens/Settings/Advanced/index.tsx | 4 +- 2 files changed, 83 insertions(+), 3 deletions(-) diff --git a/web/hooks/useModels.test.ts b/web/hooks/useModels.test.ts index f9c3b04b4..e848c455c 100644 --- a/web/hooks/useModels.test.ts +++ b/web/hooks/useModels.test.ts @@ -1,5 +1,5 @@ // useModels.test.ts -import { renderHook, act } from '@testing-library/react' +import { renderHook, act, waitFor } from '@testing-library/react' import { events, ModelEvent, ModelManager } from '@janhq/core' import { extensionManager } from '@/extension' @@ -36,7 +36,6 @@ describe('useModels', () => { }), get: () => undefined, has: () => true, - // set: () => {} }, }) @@ -50,6 +49,85 @@ describe('useModels', () => { expect(mockModelExtension.getModels).toHaveBeenCalled() }) + it('should return empty on error', async () => { + const mockModelExtension = { + getModels: jest.fn().mockRejectedValue(new Error('Error')), + } as any + ;(ModelManager.instance as jest.Mock).mockReturnValue({ + models: { + values: () => ({ + toArray: () => ({ + filter: () => models, + }), + }), + get: () => undefined, + has: () => true, + }, + }) + + jest.spyOn(extensionManager, 'get').mockReturnValue(mockModelExtension) + + const { result } = renderHook(() => useModels()) + + await act(() => { + result.current?.loadDataModel() + }) + + expect(mockModelExtension.getModels()).rejects.toThrow() + }) + + it('should update states on models update', async () => { + const mockModelExtension = { + getModels: jest.fn().mockResolvedValue(models), + } as any + + ;(ModelManager.instance as jest.Mock).mockReturnValue({ + models: { + values: () => ({ + toArray: () => ({ + filter: () => models, + }), + }), + get: () => undefined, + has: () => true, + }, + }) + + jest.spyOn(extensionManager, 'get').mockReturnValue(mockModelExtension) + jest.spyOn(events, 'on').mockImplementationOnce((event, cb) => { + cb({ fetch: false }) + }) + renderHook(() => useModels()) + + expect(mockModelExtension.getModels).not.toHaveBeenCalled() + }) + + it('should update states on models update', async () => { + const mockModelExtension = { + getModels: jest.fn().mockResolvedValue(models), + } as any + + ;(ModelManager.instance as jest.Mock).mockReturnValue({ + models: { + values: () => ({ + toArray: () => ({ + filter: () => models, + }), + }), + get: () => undefined, + has: () => true, + }, + }) + + jest.spyOn(extensionManager, 'get').mockReturnValue(mockModelExtension) + jest.spyOn(events, 'on').mockImplementationOnce((event, cb) => { + cb({ fetch: true }) + }) + renderHook(() => useModels()) + + expect(mockModelExtension.getModels).toHaveBeenCalled() + }) + it('should remove event listener on unmount', async () => { const removeListenerSpy = jest.spyOn(events, 'off') diff --git a/web/screens/Settings/Advanced/index.tsx b/web/screens/Settings/Advanced/index.tsx index a5956fc33..62a2aded0 100644 --- a/web/screens/Settings/Advanced/index.tsx +++ b/web/screens/Settings/Advanced/index.tsx @@ -223,7 +223,9 @@ const Advanced = () => { } setGpusInUse(updatedGpusInUse) await saveSettings({ gpusInUse: updatedGpusInUse }) - window.core?.api?.relaunch() + // Reload window to apply changes + // This will trigger engine servers to restart + window.location.reload() } const gpuSelectionPlaceHolder = From 0f05910f632e1c00013a18cc8d58af4de1383655 Mon Sep 17 00:00:00 2001 From: Louis Date: Tue, 19 Nov 2024 19:48:50 +0700 Subject: [PATCH 4/4] chore: bump cortex.cpp 1.0.3-rc4 --- extensions/inference-cortex-extension/bin/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/inference-cortex-extension/bin/version.txt b/extensions/inference-cortex-extension/bin/version.txt index b26ebdeac..32ee492e0 100644 --- a/extensions/inference-cortex-extension/bin/version.txt +++ b/extensions/inference-cortex-extension/bin/version.txt @@ -1 +1 @@ -1.0.3-rc2 \ No newline at end of file +1.0.3-rc4 \ No newline at end of file