diff --git a/extensions/inference-cortex-extension/src/index.ts b/extensions/inference-cortex-extension/src/index.ts index 79c6c8bfc..77f177316 100644 --- a/extensions/inference-cortex-extension/src/index.ts +++ b/extensions/inference-cortex-extension/src/index.ts @@ -180,12 +180,20 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine { 'engineVariant', systemInfo.gpuSetting ) - return ky - .post( - `${CORTEX_API_URL}/v1/engines/${InferenceEngine.cortex_llamacpp}/default?version=${CORTEX_ENGINE_VERSION}&variant=${variant}`, - { json: {} } - ) - .then(() => {}) + return ( + ky + // Fallback support for legacy API + .post( + `${CORTEX_API_URL}/v1/engines/${InferenceEngine.cortex_llamacpp}/default?version=${CORTEX_ENGINE_VERSION}&variant=${variant}`, + { + json: { + version: CORTEX_ENGINE_VERSION, + variant, + }, + } + ) + .then(() => {}) + ) } /** diff --git a/extensions/inference-cortex-extension/src/node/execute.test.ts b/extensions/inference-cortex-extension/src/node/execute.test.ts index 73f114ce1..1bcefce9d 100644 --- a/extensions/inference-cortex-extension/src/node/execute.test.ts +++ b/extensions/inference-cortex-extension/src/node/execute.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from '@jest/globals' import { engineVariant, executableCortexFile } from './execute' import { GpuSetting } from '@janhq/core/node' import { cpuInfo } from 'cpu-instructions' +import { fork } from 'child_process' let testSettings: GpuSetting = { run_mode: 'cpu', @@ -31,9 +32,13 @@ let mockCpuInfo = cpuInfo.cpuInfo as jest.Mock mockCpuInfo.mockReturnValue([]) jest.mock('@janhq/core/node', () => ({ - appResourcePath: () => ".", - log: jest.fn() + appResourcePath: () => '.', + log: jest.fn(), })) +jest.mock('child_process', () => ({ + fork: jest.fn(), +})) +const mockFork = fork as jest.Mock describe('test executable cortex file', () => { afterAll(function () { @@ -43,6 +48,14 @@ describe('test executable cortex file', () => { }) it('executes on MacOS', () => { + const mockProcess = { + on: jest.fn((event, callback) => { + if (event === 'message') { + callback('noavx') + } + }), + send: jest.fn(), + } Object.defineProperty(process, 'platform', { value: 'darwin', }) @@ -51,7 +64,7 @@ describe('test executable cortex file', () => { }) expect(executableCortexFile(testSettings)).toEqual( expect.objectContaining({ - enginePath: expect.stringContaining("shared"), + enginePath: expect.stringContaining('shared'), executablePath: originalPlatform === 'darwin' ? expect.stringContaining(`cortex-server`) @@ -60,13 +73,35 @@ describe('test executable cortex file', () => { vkVisibleDevices: '', }) ) - expect(engineVariant(testSettings)).toEqual('mac-arm64') + + mockFork.mockReturnValue(mockProcess) + expect(engineVariant(testSettings)).resolves.toEqual('mac-arm64') + }) + + it('executes on MacOS', () => { + Object.defineProperty(process, 'platform', { + value: 'darwin', + }) + Object.defineProperty(process, 'arch', { + value: 'arm64', + }) + + const mockProcess = { + on: jest.fn((event, callback) => { + if (event === 'message') { + callback('noavx') + } + }), + send: jest.fn(), + } + mockFork.mockReturnValue(mockProcess) Object.defineProperty(process, 'arch', { value: 'x64', }) + expect(executableCortexFile(testSettings)).toEqual( expect.objectContaining({ - enginePath: expect.stringContaining("shared"), + enginePath: expect.stringContaining('shared'), executablePath: originalPlatform === 'darwin' ? expect.stringContaining(`cortex-server`) @@ -75,7 +110,7 @@ describe('test executable cortex file', () => { vkVisibleDevices: '', }) ) - expect(engineVariant(testSettings)).toEqual('mac-amd64') + expect(engineVariant(testSettings)).resolves.toEqual('mac-amd64') }) it('executes on Windows CPU', () => { @@ -86,16 +121,25 @@ describe('test executable cortex file', () => { ...testSettings, run_mode: 'cpu', } - mockCpuInfo.mockReturnValue(['avx']) + const mockProcess = { + on: jest.fn((event, callback) => { + if (event === 'message') { + callback('avx') + } + }), + send: jest.fn(), + } + mockFork.mockReturnValue(mockProcess) + expect(executableCortexFile(settings)).toEqual( expect.objectContaining({ - enginePath: expect.stringContaining("shared"), + enginePath: expect.stringContaining('shared'), executablePath: expect.stringContaining(`cortex-server.exe`), cudaVisibleDevices: '', vkVisibleDevices: '', }) ) - expect(engineVariant()).toEqual('windows-amd64-avx') + expect(engineVariant()).resolves.toEqual('windows-amd64-avx') }) it('executes on Windows Cuda 11', () => { @@ -122,16 +166,27 @@ describe('test executable cortex file', () => { }, ], } - mockCpuInfo.mockReturnValue(['avx2']) + + const mockProcess = { + on: jest.fn((event, callback) => { + if (event === 'message') { + callback('avx2') + } + }), + send: jest.fn(), + } + mockFork.mockReturnValue(mockProcess) expect(executableCortexFile(settings)).toEqual( expect.objectContaining({ - enginePath: expect.stringContaining("shared"), + enginePath: expect.stringContaining('shared'), executablePath: expect.stringContaining(`cortex-server.exe`), cudaVisibleDevices: '0', vkVisibleDevices: '0', }) ) - expect(engineVariant(settings)).toEqual('windows-amd64-avx2-cuda-11-7') + expect(engineVariant(settings)).resolves.toEqual( + 'windows-amd64-avx2-cuda-11-7' + ) }) it('executes on Windows Cuda 12', () => { @@ -158,18 +213,36 @@ describe('test executable cortex file', () => { }, ], } - mockCpuInfo.mockReturnValue(['noavx']) + mockFork.mockReturnValue({ + on: jest.fn((event, callback) => { + if (event === 'message') { + callback('noavx') + } + }), + send: jest.fn(), + }) expect(executableCortexFile(settings)).toEqual( expect.objectContaining({ - enginePath: expect.stringContaining("shared"), + enginePath: expect.stringContaining('shared'), executablePath: expect.stringContaining(`cortex-server.exe`), cudaVisibleDevices: '0', vkVisibleDevices: '0', }) ) - expect(engineVariant(settings)).toEqual('windows-amd64-noavx-cuda-12-0') - mockCpuInfo.mockReturnValue(['avx512']) - expect(engineVariant(settings)).toEqual('windows-amd64-avx2-cuda-12-0') + expect(engineVariant(settings)).resolves.toEqual( + 'windows-amd64-noavx-cuda-12-0' + ) + mockFork.mockReturnValue({ + on: jest.fn((event, callback) => { + if (event === 'message') { + callback('avx512') + } + }), + send: jest.fn(), + }) + expect(engineVariant(settings)).resolves.toEqual( + 'windows-amd64-avx2-cuda-12-0' + ) }) it('executes on Linux CPU', () => { @@ -180,16 +253,23 @@ describe('test executable cortex file', () => { ...testSettings, run_mode: 'cpu', } - mockCpuInfo.mockReturnValue(['noavx']) + mockFork.mockReturnValue({ + on: jest.fn((event, callback) => { + if (event === 'message') { + callback('noavx') + } + }), + send: jest.fn(), + }) expect(executableCortexFile(settings)).toEqual( expect.objectContaining({ - enginePath: expect.stringContaining("shared"), + enginePath: expect.stringContaining('shared'), executablePath: expect.stringContaining(`cortex-server`), cudaVisibleDevices: '', vkVisibleDevices: '', }) ) - expect(engineVariant()).toEqual('linux-amd64-noavx') + expect(engineVariant()).resolves.toEqual('linux-amd64-noavx') }) it('executes on Linux Cuda 11', () => { @@ -216,16 +296,25 @@ describe('test executable cortex file', () => { }, ], } - mockCpuInfo.mockReturnValue(['avx512']) + + mockFork.mockReturnValue({ + on: jest.fn((event, callback) => { + if (event === 'message') { + callback('avx512') + } + }), + send: jest.fn(), + }) + expect(executableCortexFile(settings)).toEqual( expect.objectContaining({ - enginePath: expect.stringContaining("shared"), + enginePath: expect.stringContaining('shared'), executablePath: expect.stringContaining(`cortex-server`), cudaVisibleDevices: '0', vkVisibleDevices: '0', }) ) - expect(engineVariant(settings)).toEqual('linux-amd64-avx2-cuda-11-7') + expect(engineVariant(settings)).resolves.toBe('linux-amd64-avx2-cuda-11-7') }) it('executes on Linux Cuda 12', () => { @@ -252,15 +341,25 @@ describe('test executable cortex file', () => { }, ], } + mockFork.mockReturnValue({ + on: jest.fn((event, callback) => { + if (event === 'message') { + callback('avx2') + } + }), + send: jest.fn(), + }) expect(executableCortexFile(settings)).toEqual( expect.objectContaining({ - enginePath: expect.stringContaining("shared"), + enginePath: expect.stringContaining('shared'), executablePath: expect.stringContaining(`cortex-server`), cudaVisibleDevices: '0', vkVisibleDevices: '0', }) ) - expect(engineVariant(settings)).toEqual('linux-amd64-avx2-cuda-12-0') + expect(engineVariant(settings)).resolves.toEqual( + 'linux-amd64-avx2-cuda-12-0' + ) }) // Generate test for different cpu instructions on Linux @@ -275,7 +374,14 @@ describe('test executable cortex file', () => { const cpuInstructions = ['avx512', 'avx2', 'avx', 'noavx'] cpuInstructions.forEach((instruction) => { - mockCpuInfo.mockReturnValue([instruction]) + mockFork.mockReturnValue({ + on: jest.fn((event, callback) => { + if (event === 'message') { + callback(instruction) + } + }), + send: jest.fn(), + }) expect(executableCortexFile(settings)).toEqual( expect.objectContaining({ @@ -286,7 +392,9 @@ describe('test executable cortex file', () => { vkVisibleDevices: '', }) ) - expect(engineVariant(settings)).toEqual(`linux-amd64-${instruction}`) + expect(engineVariant(settings)).resolves.toEqual( + `linux-amd64-${instruction}` + ) }) }) // Generate test for different cpu instructions on Windows @@ -300,7 +408,14 @@ describe('test executable cortex file', () => { } const cpuInstructions = ['avx512', 'avx2', 'avx', 'noavx'] cpuInstructions.forEach((instruction) => { - mockCpuInfo.mockReturnValue([instruction]) + mockFork.mockReturnValue({ + on: jest.fn((event, callback) => { + if (event === 'message') { + callback(instruction) + } + }), + send: jest.fn(), + }) expect(executableCortexFile(settings)).toEqual( expect.objectContaining({ enginePath: expect.stringContaining('shared'), @@ -309,7 +424,9 @@ describe('test executable cortex file', () => { vkVisibleDevices: '', }) ) - expect(engineVariant(settings)).toEqual(`windows-amd64-${instruction}`) + expect(engineVariant(settings)).resolves.toEqual( + `windows-amd64-${instruction}` + ) }) }) @@ -340,16 +457,23 @@ describe('test executable cortex file', () => { } const cpuInstructions = ['avx512', 'avx2', 'avx', 'noavx'] cpuInstructions.forEach((instruction) => { - mockCpuInfo.mockReturnValue([instruction]) + mockFork.mockReturnValue({ + on: jest.fn((event, callback) => { + if (event === 'message') { + callback(instruction) + } + }), + send: jest.fn(), + }) expect(executableCortexFile(settings)).toEqual( expect.objectContaining({ - enginePath: expect.stringContaining("shared"), + enginePath: expect.stringContaining('shared'), executablePath: expect.stringContaining(`cortex-server.exe`), cudaVisibleDevices: '0', vkVisibleDevices: '0', }) ) - expect(engineVariant(settings)).toEqual( + expect(engineVariant(settings)).resolves.toEqual( `windows-amd64-${instruction === 'avx512' || instruction === 'avx2' ? 'avx2' : 'noavx'}-cuda-12-0` ) }) @@ -382,16 +506,23 @@ describe('test executable cortex file', () => { ], } cpuInstructions.forEach((instruction) => { - mockCpuInfo.mockReturnValue([instruction]) + mockFork.mockReturnValue({ + on: jest.fn((event, callback) => { + if (event === 'message') { + callback(instruction) + } + }), + send: jest.fn(), + }) expect(executableCortexFile(settings)).toEqual( expect.objectContaining({ - enginePath: expect.stringContaining("shared"), + enginePath: expect.stringContaining('shared'), executablePath: expect.stringContaining(`cortex-server`), cudaVisibleDevices: '0', vkVisibleDevices: '0', }) ) - expect(engineVariant(settings)).toEqual( + expect(engineVariant(settings)).resolves.toEqual( `linux-amd64-${instruction === 'avx512' || instruction === 'avx2' ? 'avx2' : 'noavx'}-cuda-12-0` ) }) @@ -425,16 +556,23 @@ describe('test executable cortex file', () => { ], } cpuInstructions.forEach((instruction) => { - mockCpuInfo.mockReturnValue([instruction]) + mockFork.mockReturnValue({ + on: jest.fn((event, callback) => { + if (event === 'message') { + callback(instruction) + } + }), + send: jest.fn(), + }) expect(executableCortexFile(settings)).toEqual( expect.objectContaining({ - enginePath: expect.stringContaining("shared"), + enginePath: expect.stringContaining('shared'), executablePath: expect.stringContaining(`cortex-server`), cudaVisibleDevices: '0', vkVisibleDevices: '0', }) ) - expect(engineVariant(settings)).toEqual(`linux-amd64-vulkan`) + expect(engineVariant(settings)).resolves.toEqual(`linux-amd64-vulkan`) }) }) @@ -452,10 +590,17 @@ describe('test executable cortex file', () => { ...testSettings, run_mode: 'cpu', } - mockCpuInfo.mockReturnValue([]) + mockFork.mockReturnValue({ + on: jest.fn((event, callback) => { + if (event === 'message') { + callback('noavx') + } + }), + send: jest.fn(), + }) expect(executableCortexFile(settings)).toEqual( expect.objectContaining({ - enginePath: expect.stringContaining("shared"), + enginePath: expect.stringContaining('shared'), executablePath: originalPlatform === 'darwin' ? expect.stringContaining(`cortex-server`)