Merge pull request #4130 from janhq/chore/4123-update-default-engine-request-body

feat: update Jan’s client request to adapt API changes from Cortex

extensions/inference-cortex-extension/src/index.ts

@@ -180,12 +180,20 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
       'engineVariant',
       systemInfo.gpuSetting
     )
-    return ky
+    return (
+      ky
+        // Fallback support for legacy API
         .post(
           `${CORTEX_API_URL}/v1/engines/${InferenceEngine.cortex_llamacpp}/default?version=${CORTEX_ENGINE_VERSION}&variant=${variant}`,
-        { json: {} }
+          {
+            json: {
+              version: CORTEX_ENGINE_VERSION,
+              variant,
+            },
+          }
         )
         .then(() => {})
+    )
   }
 
   /**

extensions/inference-cortex-extension/src/node/execute.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, it } from '@jest/globals'
 import { engineVariant, executableCortexFile } from './execute'
 import { GpuSetting } from '@janhq/core/node'
 import { cpuInfo } from 'cpu-instructions'
+import { fork } from 'child_process'
 
 let testSettings: GpuSetting = {
   run_mode: 'cpu',
@@ -31,9 +32,13 @@ let mockCpuInfo = cpuInfo.cpuInfo as jest.Mock
 mockCpuInfo.mockReturnValue([])
 
 jest.mock('@janhq/core/node', () => ({
-  appResourcePath: () => ".",
+  appResourcePath: () => '.',
-  log: jest.fn()
+  log: jest.fn(),
 }))
+jest.mock('child_process', () => ({
+  fork: jest.fn(),
+}))
+const mockFork = fork as jest.Mock
 
 describe('test executable cortex file', () => {
   afterAll(function () {
@@ -43,6 +48,14 @@ describe('test executable cortex file', () => {
   })
 
   it('executes on MacOS', () => {
+    const mockProcess = {
+      on: jest.fn((event, callback) => {
+        if (event === 'message') {
+          callback('noavx')
+        }
+      }),
+      send: jest.fn(),
+    }
     Object.defineProperty(process, 'platform', {
       value: 'darwin',
     })
@@ -51,7 +64,7 @@ describe('test executable cortex file', () => {
     })
     expect(executableCortexFile(testSettings)).toEqual(
       expect.objectContaining({
-        enginePath: expect.stringContaining("shared"),
+        enginePath: expect.stringContaining('shared'),
         executablePath:
           originalPlatform === 'darwin'
             ? expect.stringContaining(`cortex-server`)
@@ -60,13 +73,35 @@ describe('test executable cortex file', () => {
         vkVisibleDevices: '',
       })
     )
-    expect(engineVariant(testSettings)).toEqual('mac-arm64')
+
+    mockFork.mockReturnValue(mockProcess)
+    expect(engineVariant(testSettings)).resolves.toEqual('mac-arm64')
+  })
+
+  it('executes on MacOS', () => {
+    Object.defineProperty(process, 'platform', {
+      value: 'darwin',
+    })
+    Object.defineProperty(process, 'arch', {
+      value: 'arm64',
+    })
+
+    const mockProcess = {
+      on: jest.fn((event, callback) => {
+        if (event === 'message') {
+          callback('noavx')
+        }
+      }),
+      send: jest.fn(),
+    }
+    mockFork.mockReturnValue(mockProcess)
     Object.defineProperty(process, 'arch', {
       value: 'x64',
     })
+
     expect(executableCortexFile(testSettings)).toEqual(
       expect.objectContaining({
-        enginePath: expect.stringContaining("shared"),
+        enginePath: expect.stringContaining('shared'),
         executablePath:
           originalPlatform === 'darwin'
             ? expect.stringContaining(`cortex-server`)
@@ -75,7 +110,7 @@ describe('test executable cortex file', () => {
         vkVisibleDevices: '',
       })
     )
-    expect(engineVariant(testSettings)).toEqual('mac-amd64')
+    expect(engineVariant(testSettings)).resolves.toEqual('mac-amd64')
   })
 
   it('executes on Windows CPU', () => {
@@ -86,16 +121,25 @@ describe('test executable cortex file', () => {
       ...testSettings,
       run_mode: 'cpu',
     }
-    mockCpuInfo.mockReturnValue(['avx'])
+    const mockProcess = {
+      on: jest.fn((event, callback) => {
+        if (event === 'message') {
+          callback('avx')
+        }
+      }),
+      send: jest.fn(),
+    }
+    mockFork.mockReturnValue(mockProcess)
+
     expect(executableCortexFile(settings)).toEqual(
       expect.objectContaining({
-        enginePath: expect.stringContaining("shared"),
+        enginePath: expect.stringContaining('shared'),
         executablePath: expect.stringContaining(`cortex-server.exe`),
         cudaVisibleDevices: '',
         vkVisibleDevices: '',
       })
     )
-    expect(engineVariant()).toEqual('windows-amd64-avx')
+    expect(engineVariant()).resolves.toEqual('windows-amd64-avx')
   })
 
   it('executes on Windows Cuda 11', () => {
@@ -122,16 +166,27 @@ describe('test executable cortex file', () => {
         },
       ],
     }
-    mockCpuInfo.mockReturnValue(['avx2'])
+
+    const mockProcess = {
+      on: jest.fn((event, callback) => {
+        if (event === 'message') {
+          callback('avx2')
+        }
+      }),
+      send: jest.fn(),
+    }
+    mockFork.mockReturnValue(mockProcess)
     expect(executableCortexFile(settings)).toEqual(
       expect.objectContaining({
-        enginePath: expect.stringContaining("shared"),
+        enginePath: expect.stringContaining('shared'),
         executablePath: expect.stringContaining(`cortex-server.exe`),
         cudaVisibleDevices: '0',
         vkVisibleDevices: '0',
       })
     )
-    expect(engineVariant(settings)).toEqual('windows-amd64-avx2-cuda-11-7')
+    expect(engineVariant(settings)).resolves.toEqual(
+      'windows-amd64-avx2-cuda-11-7'
+    )
   })
 
   it('executes on Windows Cuda 12', () => {
@@ -158,18 +213,36 @@ describe('test executable cortex file', () => {
         },
       ],
     }
-    mockCpuInfo.mockReturnValue(['noavx'])
+    mockFork.mockReturnValue({
+      on: jest.fn((event, callback) => {
+        if (event === 'message') {
+          callback('noavx')
+        }
+      }),
+      send: jest.fn(),
+    })
     expect(executableCortexFile(settings)).toEqual(
       expect.objectContaining({
-        enginePath: expect.stringContaining("shared"),
+        enginePath: expect.stringContaining('shared'),
         executablePath: expect.stringContaining(`cortex-server.exe`),
         cudaVisibleDevices: '0',
         vkVisibleDevices: '0',
       })
     )
-    expect(engineVariant(settings)).toEqual('windows-amd64-noavx-cuda-12-0')
+    expect(engineVariant(settings)).resolves.toEqual(
-    mockCpuInfo.mockReturnValue(['avx512'])
+      'windows-amd64-noavx-cuda-12-0'
-    expect(engineVariant(settings)).toEqual('windows-amd64-avx2-cuda-12-0')
+    )
+    mockFork.mockReturnValue({
+      on: jest.fn((event, callback) => {
+        if (event === 'message') {
+          callback('avx512')
+        }
+      }),
+      send: jest.fn(),
+    })
+    expect(engineVariant(settings)).resolves.toEqual(
+      'windows-amd64-avx2-cuda-12-0'
+    )
   })
 
   it('executes on Linux CPU', () => {
@@ -180,16 +253,23 @@ describe('test executable cortex file', () => {
       ...testSettings,
       run_mode: 'cpu',
     }
-    mockCpuInfo.mockReturnValue(['noavx'])
+    mockFork.mockReturnValue({
+      on: jest.fn((event, callback) => {
+        if (event === 'message') {
+          callback('noavx')
+        }
+      }),
+      send: jest.fn(),
+    })
     expect(executableCortexFile(settings)).toEqual(
       expect.objectContaining({
-        enginePath: expect.stringContaining("shared"),
+        enginePath: expect.stringContaining('shared'),
         executablePath: expect.stringContaining(`cortex-server`),
         cudaVisibleDevices: '',
         vkVisibleDevices: '',
       })
     )
-    expect(engineVariant()).toEqual('linux-amd64-noavx')
+    expect(engineVariant()).resolves.toEqual('linux-amd64-noavx')
   })
 
   it('executes on Linux Cuda 11', () => {
@@ -216,16 +296,25 @@ describe('test executable cortex file', () => {
         },
       ],
     }
-    mockCpuInfo.mockReturnValue(['avx512'])
+
+    mockFork.mockReturnValue({
+      on: jest.fn((event, callback) => {
+        if (event === 'message') {
+          callback('avx512')
+        }
+      }),
+      send: jest.fn(),
+    })
+
     expect(executableCortexFile(settings)).toEqual(
       expect.objectContaining({
-        enginePath: expect.stringContaining("shared"),
+        enginePath: expect.stringContaining('shared'),
         executablePath: expect.stringContaining(`cortex-server`),
         cudaVisibleDevices: '0',
         vkVisibleDevices: '0',
       })
     )
-    expect(engineVariant(settings)).toEqual('linux-amd64-avx2-cuda-11-7')
+    expect(engineVariant(settings)).resolves.toBe('linux-amd64-avx2-cuda-11-7')
   })
 
   it('executes on Linux Cuda 12', () => {
@@ -252,15 +341,25 @@ describe('test executable cortex file', () => {
         },
       ],
     }
+    mockFork.mockReturnValue({
+      on: jest.fn((event, callback) => {
+        if (event === 'message') {
+          callback('avx2')
+        }
+      }),
+      send: jest.fn(),
+    })
     expect(executableCortexFile(settings)).toEqual(
       expect.objectContaining({
-        enginePath: expect.stringContaining("shared"),
+        enginePath: expect.stringContaining('shared'),
         executablePath: expect.stringContaining(`cortex-server`),
         cudaVisibleDevices: '0',
         vkVisibleDevices: '0',
       })
     )
-    expect(engineVariant(settings)).toEqual('linux-amd64-avx2-cuda-12-0')
+    expect(engineVariant(settings)).resolves.toEqual(
+      'linux-amd64-avx2-cuda-12-0'
+    )
   })
 
   // Generate test for different cpu instructions on Linux
@@ -275,7 +374,14 @@ describe('test executable cortex file', () => {
 
     const cpuInstructions = ['avx512', 'avx2', 'avx', 'noavx']
     cpuInstructions.forEach((instruction) => {
-      mockCpuInfo.mockReturnValue([instruction])
+      mockFork.mockReturnValue({
+        on: jest.fn((event, callback) => {
+          if (event === 'message') {
+            callback(instruction)
+          }
+        }),
+        send: jest.fn(),
+      })
 
       expect(executableCortexFile(settings)).toEqual(
         expect.objectContaining({
@@ -286,7 +392,9 @@ describe('test executable cortex file', () => {
           vkVisibleDevices: '',
         })
       )
-      expect(engineVariant(settings)).toEqual(`linux-amd64-${instruction}`)
+      expect(engineVariant(settings)).resolves.toEqual(
+        `linux-amd64-${instruction}`
+      )
     })
   })
   // Generate test for different cpu instructions on Windows
@@ -300,7 +408,14 @@ describe('test executable cortex file', () => {
     }
     const cpuInstructions = ['avx512', 'avx2', 'avx', 'noavx']
     cpuInstructions.forEach((instruction) => {
-      mockCpuInfo.mockReturnValue([instruction])
+      mockFork.mockReturnValue({
+        on: jest.fn((event, callback) => {
+          if (event === 'message') {
+            callback(instruction)
+          }
+        }),
+        send: jest.fn(),
+      })
       expect(executableCortexFile(settings)).toEqual(
         expect.objectContaining({
           enginePath: expect.stringContaining('shared'),
@@ -309,7 +424,9 @@ describe('test executable cortex file', () => {
           vkVisibleDevices: '',
         })
       )
-      expect(engineVariant(settings)).toEqual(`windows-amd64-${instruction}`)
+      expect(engineVariant(settings)).resolves.toEqual(
+        `windows-amd64-${instruction}`
+      )
     })
   })
 
@@ -340,16 +457,23 @@ describe('test executable cortex file', () => {
     }
     const cpuInstructions = ['avx512', 'avx2', 'avx', 'noavx']
     cpuInstructions.forEach((instruction) => {
-      mockCpuInfo.mockReturnValue([instruction])
+      mockFork.mockReturnValue({
+        on: jest.fn((event, callback) => {
+          if (event === 'message') {
+            callback(instruction)
+          }
+        }),
+        send: jest.fn(),
+      })
       expect(executableCortexFile(settings)).toEqual(
         expect.objectContaining({
-          enginePath: expect.stringContaining("shared"),
+          enginePath: expect.stringContaining('shared'),
           executablePath: expect.stringContaining(`cortex-server.exe`),
           cudaVisibleDevices: '0',
           vkVisibleDevices: '0',
         })
       )
-      expect(engineVariant(settings)).toEqual(
+      expect(engineVariant(settings)).resolves.toEqual(
         `windows-amd64-${instruction === 'avx512' || instruction === 'avx2' ? 'avx2' : 'noavx'}-cuda-12-0`
       )
     })
@@ -382,16 +506,23 @@ describe('test executable cortex file', () => {
       ],
     }
     cpuInstructions.forEach((instruction) => {
-      mockCpuInfo.mockReturnValue([instruction])
+      mockFork.mockReturnValue({
+        on: jest.fn((event, callback) => {
+          if (event === 'message') {
+            callback(instruction)
+          }
+        }),
+        send: jest.fn(),
+      })
       expect(executableCortexFile(settings)).toEqual(
         expect.objectContaining({
-          enginePath: expect.stringContaining("shared"),
+          enginePath: expect.stringContaining('shared'),
           executablePath: expect.stringContaining(`cortex-server`),
           cudaVisibleDevices: '0',
           vkVisibleDevices: '0',
         })
       )
-      expect(engineVariant(settings)).toEqual(
+      expect(engineVariant(settings)).resolves.toEqual(
         `linux-amd64-${instruction === 'avx512' || instruction === 'avx2' ? 'avx2' : 'noavx'}-cuda-12-0`
       )
     })
@@ -425,16 +556,23 @@ describe('test executable cortex file', () => {
       ],
     }
     cpuInstructions.forEach((instruction) => {
-      mockCpuInfo.mockReturnValue([instruction])
+      mockFork.mockReturnValue({
+        on: jest.fn((event, callback) => {
+          if (event === 'message') {
+            callback(instruction)
+          }
+        }),
+        send: jest.fn(),
+      })
       expect(executableCortexFile(settings)).toEqual(
         expect.objectContaining({
-          enginePath: expect.stringContaining("shared"),
+          enginePath: expect.stringContaining('shared'),
           executablePath: expect.stringContaining(`cortex-server`),
           cudaVisibleDevices: '0',
           vkVisibleDevices: '0',
         })
       )
-      expect(engineVariant(settings)).toEqual(`linux-amd64-vulkan`)
+      expect(engineVariant(settings)).resolves.toEqual(`linux-amd64-vulkan`)
     })
   })
 
@@ -452,10 +590,17 @@ describe('test executable cortex file', () => {
         ...testSettings,
         run_mode: 'cpu',
       }
-      mockCpuInfo.mockReturnValue([])
+      mockFork.mockReturnValue({
+        on: jest.fn((event, callback) => {
+          if (event === 'message') {
+            callback('noavx')
+          }
+        }),
+        send: jest.fn(),
+      })
       expect(executableCortexFile(settings)).toEqual(
         expect.objectContaining({
-          enginePath: expect.stringContaining("shared"),
+          enginePath: expect.stringContaining('shared'),
           executablePath:
             originalPlatform === 'darwin'
               ? expect.stringContaining(`cortex-server`)