Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #4130 from janhq/chore/4123-update-default-engine-request-body

Browse Source 
feat: update Jan’s client request to adapt API changes from Cortex
This commit is contained in:
Louis 2024-11-26 21:38:14 +07:00 committed by GitHub
commit 2478cf7bc3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 199 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
extensions/inference-cortex-extension/src/index.ts
View File

@ -180,12 +180,20 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
'engineVariant',
systemInfo.gpuSetting
)
return ky
return (
ky
// Fallback support for legacy API
.post(
`${CORTEX_API_URL}/v1/engines/${InferenceEngine.cortex_llamacpp}/default?version=${CORTEX_ENGINE_VERSION}&variant=${variant}`,
{ json: {} }
{
json: {
version: CORTEX_ENGINE_VERSION,
variant,
},
}
)
.then(() => {})
)
}
/**

225
extensions/inference-cortex-extension/src/node/execute.test.ts
View File

@ -2,6 +2,7 @@ import { describe, expect, it } from '@jest/globals'
import { engineVariant, executableCortexFile } from './execute'
import { GpuSetting } from '@janhq/core/node'
import { cpuInfo } from 'cpu-instructions'
import { fork } from 'child_process'
let testSettings: GpuSetting = {
run_mode: 'cpu',
@ -31,9 +32,13 @@ let mockCpuInfo = cpuInfo.cpuInfo as jest.Mock
mockCpuInfo.mockReturnValue([])
jest.mock('@janhq/core/node', () => ({
appResourcePath: () => ".",
log: jest.fn()
appResourcePath: () => '.',
log: jest.fn(),
}))
jest.mock('child_process', () => ({
fork: jest.fn(),
}))
const mockFork = fork as jest.Mock
describe('test executable cortex file', () => {
afterAll(function () {
@ -43,6 +48,14 @@ describe('test executable cortex file', () => {
})
it('executes on MacOS', () => {
const mockProcess = {
on: jest.fn((event, callback) => {
if (event === 'message') {
callback('noavx')
}
}),
send: jest.fn(),
}
Object.defineProperty(process, 'platform', {
value: 'darwin',
})
@ -51,7 +64,7 @@ describe('test executable cortex file', () => {
})
expect(executableCortexFile(testSettings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining("shared"),
enginePath: expect.stringContaining('shared'),
executablePath:
originalPlatform === 'darwin'
? expect.stringContaining(`cortex-server`)
@ -60,13 +73,35 @@ describe('test executable cortex file', () => {
vkVisibleDevices: '',
})
)
expect(engineVariant(testSettings)).toEqual('mac-arm64')
mockFork.mockReturnValue(mockProcess)
expect(engineVariant(testSettings)).resolves.toEqual('mac-arm64')
})
it('executes on MacOS', () => {
Object.defineProperty(process, 'platform', {
value: 'darwin',
})
Object.defineProperty(process, 'arch', {
value: 'arm64',
})
const mockProcess = {
on: jest.fn((event, callback) => {
if (event === 'message') {
callback('noavx')
}
}),
send: jest.fn(),
}
mockFork.mockReturnValue(mockProcess)
Object.defineProperty(process, 'arch', {
value: 'x64',
})
expect(executableCortexFile(testSettings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining("shared"),
enginePath: expect.stringContaining('shared'),
executablePath:
originalPlatform === 'darwin'
? expect.stringContaining(`cortex-server`)
@ -75,7 +110,7 @@ describe('test executable cortex file', () => {
vkVisibleDevices: '',
})
)
expect(engineVariant(testSettings)).toEqual('mac-amd64')
expect(engineVariant(testSettings)).resolves.toEqual('mac-amd64')
})
it('executes on Windows CPU', () => {
@ -86,16 +121,25 @@ describe('test executable cortex file', () => {
...testSettings,
run_mode: 'cpu',
}
mockCpuInfo.mockReturnValue(['avx'])
const mockProcess = {
on: jest.fn((event, callback) => {
if (event === 'message') {
callback('avx')
}
}),
send: jest.fn(),
}
mockFork.mockReturnValue(mockProcess)
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining("shared"),
enginePath: expect.stringContaining('shared'),
executablePath: expect.stringContaining(`cortex-server.exe`),
cudaVisibleDevices: '',
vkVisibleDevices: '',
})
)
expect(engineVariant()).toEqual('windows-amd64-avx')
expect(engineVariant()).resolves.toEqual('windows-amd64-avx')
})
it('executes on Windows Cuda 11', () => {
@ -122,16 +166,27 @@ describe('test executable cortex file', () => {
},
],
}
mockCpuInfo.mockReturnValue(['avx2'])
const mockProcess = {
on: jest.fn((event, callback) => {
if (event === 'message') {
callback('avx2')
}
}),
send: jest.fn(),
}
mockFork.mockReturnValue(mockProcess)
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining("shared"),
enginePath: expect.stringContaining('shared'),
executablePath: expect.stringContaining(`cortex-server.exe`),
cudaVisibleDevices: '0',
vkVisibleDevices: '0',
})
)
expect(engineVariant(settings)).toEqual('windows-amd64-avx2-cuda-11-7')
expect(engineVariant(settings)).resolves.toEqual(
'windows-amd64-avx2-cuda-11-7'
)
})
it('executes on Windows Cuda 12', () => {
@ -158,18 +213,36 @@ describe('test executable cortex file', () => {
},
],
}
mockCpuInfo.mockReturnValue(['noavx'])
mockFork.mockReturnValue({
on: jest.fn((event, callback) => {
if (event === 'message') {
callback('noavx')
}
}),
send: jest.fn(),
})
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining("shared"),
enginePath: expect.stringContaining('shared'),
executablePath: expect.stringContaining(`cortex-server.exe`),
cudaVisibleDevices: '0',
vkVisibleDevices: '0',
})
)
expect(engineVariant(settings)).toEqual('windows-amd64-noavx-cuda-12-0')
mockCpuInfo.mockReturnValue(['avx512'])
expect(engineVariant(settings)).toEqual('windows-amd64-avx2-cuda-12-0')
expect(engineVariant(settings)).resolves.toEqual(
'windows-amd64-noavx-cuda-12-0'
)
mockFork.mockReturnValue({
on: jest.fn((event, callback) => {
if (event === 'message') {
callback('avx512')
}
}),
send: jest.fn(),
})
expect(engineVariant(settings)).resolves.toEqual(
'windows-amd64-avx2-cuda-12-0'
)
})
it('executes on Linux CPU', () => {
@ -180,16 +253,23 @@ describe('test executable cortex file', () => {
...testSettings,
run_mode: 'cpu',
}
mockCpuInfo.mockReturnValue(['noavx'])
mockFork.mockReturnValue({
on: jest.fn((event, callback) => {
if (event === 'message') {
callback('noavx')
}
}),
send: jest.fn(),
})
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining("shared"),
enginePath: expect.stringContaining('shared'),
executablePath: expect.stringContaining(`cortex-server`),
cudaVisibleDevices: '',
vkVisibleDevices: '',
})
)
expect(engineVariant()).toEqual('linux-amd64-noavx')
expect(engineVariant()).resolves.toEqual('linux-amd64-noavx')
})
it('executes on Linux Cuda 11', () => {
@ -216,16 +296,25 @@ describe('test executable cortex file', () => {
},
],
}
mockCpuInfo.mockReturnValue(['avx512'])
mockFork.mockReturnValue({
on: jest.fn((event, callback) => {
if (event === 'message') {
callback('avx512')
}
}),
send: jest.fn(),
})
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining("shared"),
enginePath: expect.stringContaining('shared'),
executablePath: expect.stringContaining(`cortex-server`),
cudaVisibleDevices: '0',
vkVisibleDevices: '0',
})
)
expect(engineVariant(settings)).toEqual('linux-amd64-avx2-cuda-11-7')
expect(engineVariant(settings)).resolves.toBe('linux-amd64-avx2-cuda-11-7')
})
it('executes on Linux Cuda 12', () => {
@ -252,15 +341,25 @@ describe('test executable cortex file', () => {
},
],
}
mockFork.mockReturnValue({
on: jest.fn((event, callback) => {
if (event === 'message') {
callback('avx2')
}
}),
send: jest.fn(),
})
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining("shared"),
enginePath: expect.stringContaining('shared'),
executablePath: expect.stringContaining(`cortex-server`),
cudaVisibleDevices: '0',
vkVisibleDevices: '0',
})
)
expect(engineVariant(settings)).toEqual('linux-amd64-avx2-cuda-12-0')
expect(engineVariant(settings)).resolves.toEqual(
'linux-amd64-avx2-cuda-12-0'
)
})
// Generate test for different cpu instructions on Linux
@ -275,7 +374,14 @@ describe('test executable cortex file', () => {
const cpuInstructions = ['avx512', 'avx2', 'avx', 'noavx']
cpuInstructions.forEach((instruction) => {
mockCpuInfo.mockReturnValue([instruction])
mockFork.mockReturnValue({
on: jest.fn((event, callback) => {
if (event === 'message') {
callback(instruction)
}
}),
send: jest.fn(),
})
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
@ -286,7 +392,9 @@ describe('test executable cortex file', () => {
vkVisibleDevices: '',
})
)
expect(engineVariant(settings)).toEqual(`linux-amd64-${instruction}`)
expect(engineVariant(settings)).resolves.toEqual(
`linux-amd64-${instruction}`
)
})
})
// Generate test for different cpu instructions on Windows
@ -300,7 +408,14 @@ describe('test executable cortex file', () => {
}
const cpuInstructions = ['avx512', 'avx2', 'avx', 'noavx']
cpuInstructions.forEach((instruction) => {
mockCpuInfo.mockReturnValue([instruction])
mockFork.mockReturnValue({
on: jest.fn((event, callback) => {
if (event === 'message') {
callback(instruction)
}
}),
send: jest.fn(),
})
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining('shared'),
@ -309,7 +424,9 @@ describe('test executable cortex file', () => {
vkVisibleDevices: '',
})
)
expect(engineVariant(settings)).toEqual(`windows-amd64-${instruction}`)
expect(engineVariant(settings)).resolves.toEqual(
`windows-amd64-${instruction}`
)
})
})
@ -340,16 +457,23 @@ describe('test executable cortex file', () => {
}
const cpuInstructions = ['avx512', 'avx2', 'avx', 'noavx']
cpuInstructions.forEach((instruction) => {
mockCpuInfo.mockReturnValue([instruction])
mockFork.mockReturnValue({
on: jest.fn((event, callback) => {
if (event === 'message') {
callback(instruction)
}
}),
send: jest.fn(),
})
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining("shared"),
enginePath: expect.stringContaining('shared'),
executablePath: expect.stringContaining(`cortex-server.exe`),
cudaVisibleDevices: '0',
vkVisibleDevices: '0',
})
)
expect(engineVariant(settings)).toEqual(
expect(engineVariant(settings)).resolves.toEqual(
`windows-amd64-${instruction === 'avx512' || instruction === 'avx2' ? 'avx2' : 'noavx'}-cuda-12-0`
)
})
@ -382,16 +506,23 @@ describe('test executable cortex file', () => {
],
}
cpuInstructions.forEach((instruction) => {
mockCpuInfo.mockReturnValue([instruction])
mockFork.mockReturnValue({
on: jest.fn((event, callback) => {
if (event === 'message') {
callback(instruction)
}
}),
send: jest.fn(),
})
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining("shared"),
enginePath: expect.stringContaining('shared'),
executablePath: expect.stringContaining(`cortex-server`),
cudaVisibleDevices: '0',
vkVisibleDevices: '0',
})
)
expect(engineVariant(settings)).toEqual(
expect(engineVariant(settings)).resolves.toEqual(
`linux-amd64-${instruction === 'avx512' || instruction === 'avx2' ? 'avx2' : 'noavx'}-cuda-12-0`
)
})
@ -425,16 +556,23 @@ describe('test executable cortex file', () => {
],
}
cpuInstructions.forEach((instruction) => {
mockCpuInfo.mockReturnValue([instruction])
mockFork.mockReturnValue({
on: jest.fn((event, callback) => {
if (event === 'message') {
callback(instruction)
}
}),
send: jest.fn(),
})
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining("shared"),
enginePath: expect.stringContaining('shared'),
executablePath: expect.stringContaining(`cortex-server`),
cudaVisibleDevices: '0',
vkVisibleDevices: '0',
})
)
expect(engineVariant(settings)).toEqual(`linux-amd64-vulkan`)
expect(engineVariant(settings)).resolves.toEqual(`linux-amd64-vulkan`)
})
})
@ -452,10 +590,17 @@ describe('test executable cortex file', () => {
...testSettings,
run_mode: 'cpu',
}
mockCpuInfo.mockReturnValue([])
mockFork.mockReturnValue({
on: jest.fn((event, callback) => {
if (event === 'message') {
callback('noavx')
}
}),
send: jest.fn(),
})
expect(executableCortexFile(settings)).toEqual(
expect.objectContaining({
enginePath: expect.stringContaining("shared"),
enginePath: expect.stringContaining('shared'),
executablePath:
originalPlatform === 'darwin'
? expect.stringContaining(`cortex-server`)