Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: Jan supports cortex with llama-server (#4904)

Browse Source 
* fix: update logic to pull llama-cpp engine

* chore: bump cortex version

* fix: chat macos

* chore: bump llama.cpp

* chore: download windows

* chore: download linux

* chore: bump cortex to 1.0.13-rc6

* fix: stop cortex before exit

* chore: bum llama.cpp engine to b5350

---------

Co-authored-by: vansangpfiev <sang@jan.ai>
This commit is contained in:
vansangpfiev 2025-05-11 22:52:51 -07:00 committed by Louis
parent f3a808cb89
commit dea29ff8c4
No known key found for this signature in database
GPG Key ID: 44FA9F4D33C37DE2
12 changed files with 108 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
extensions/engine-management-extension/rolldown.config.mjs
View File

@ -15,7 +15,7 @@ export default defineConfig([
`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? '39291'}` `http://127.0.0.1:${process.env.CORTEX_API_PORT ?? '39291'}`
), ),
PLATFORM: JSON.stringify(process.platform), PLATFORM: JSON.stringify(process.platform),
CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.56'), CORTEX_ENGINE_VERSION: JSON.stringify('b5350'),
DEFAULT_REMOTE_ENGINES: JSON.stringify(engines), DEFAULT_REMOTE_ENGINES: JSON.stringify(engines),
DEFAULT_REMOTE_MODELS: JSON.stringify(models), DEFAULT_REMOTE_MODELS: JSON.stringify(models),
DEFAULT_REQUEST_PAYLOAD_TRANSFORM: JSON.stringify( DEFAULT_REQUEST_PAYLOAD_TRANSFORM: JSON.stringify(
@ -38,7 +38,7 @@ export default defineConfig([
file: 'dist/node/index.cjs.js', file: 'dist/node/index.cjs.js',
}, },
define: { define: {
CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.56'), CORTEX_ENGINE_VERSION: JSON.stringify('b5350'),
}, },
}, },
]) ])

4
extensions/engine-management-extension/src/node/index.ts
View File

@ -16,12 +16,12 @@ const symlinkEngines = async () => {
appResourcePath(), appResourcePath(),
'shared', 'shared',
'engines', 'engines',
'cortex.llamacpp' 'llama.cpp'
) )
const symlinkEnginePath = path.join( const symlinkEnginePath = path.join(
getJanDataFolderPath(), getJanDataFolderPath(),
'engines', 'engines',
'cortex.llamacpp' 'llama.cpp'
) )
const variantFolders = await readdir(sourceEnginePath) const variantFolders = await readdir(sourceEnginePath)
const isStandalone = process.platform === 'linux' const isStandalone = process.platform === 'linux'

20
extensions/engine-management-extension/src/utils.ts
View File

@ -32,23 +32,23 @@ const gpuRunMode = (settings?: GpuSetting): RunMode => {
*/ */
const os = (settings?: GpuSetting): string => { const os = (settings?: GpuSetting): string => {
return PLATFORM === 'win32' return PLATFORM === 'win32'
? 'windows-amd64' ? 'win'
: PLATFORM === 'darwin' : PLATFORM === 'darwin'
? settings?.cpu?.arch === 'arm64' ? settings?.cpu?.arch === 'arm64'
? 'mac-arm64' ? 'macos-arm64'
: 'mac-amd64' : 'macos-x64'
: 'linux-amd64' : 'linux'
} }
/** /**
* The CUDA version that will be set - either '11-7' or '12-0'. * The CUDA version that will be set - either 'cu12.0' or 'cu11.7'.
* @param settings * @param settings
* @returns * @returns
*/ */
const cudaVersion = (settings?: GpuSetting): '12-0' | '11-7' | undefined => { const cudaVersion = (settings?: GpuSetting): 'cu12.0' | 'cu11.7' | undefined => {
return settings.gpus?.some((gpu) => gpu.version.includes('12')) return settings.gpus?.some((gpu) => gpu.version.includes('12'))
? '12-0' ? 'cu12.0'
: '11-7' : 'cu11.7'
} }
/** /**
@ -84,15 +84,17 @@ export const engineVariant = async (
: 'noavx', : 'noavx',
runMode, runMode,
cudaVersion(gpuSetting), cudaVersion(gpuSetting),
'x64',
] ]
: // For cpu only we need to check all available supported instructions : // For cpu only we need to check all available supported instructions
[ [
(gpuSetting.cpu?.instructions ?? ['noavx']).find((e) => (gpuSetting.cpu?.instructions ?? ['noavx']).find((e) =>
instructionBinaryNames.includes(e.toLowerCase()) instructionBinaryNames.includes(e.toLowerCase())
) ?? 'noavx', ) ?? 'noavx',
'x64',
]), ]),
].filter(Boolean) ].filter(Boolean)
: [platform, 'vulkan'] : [platform, 'vulkan', 'x64']
let engineVariantString = engineVariant.join('-') let engineVariantString = engineVariant.join('-')

2
extensions/inference-cortex-extension/bin/version.txt
View File

@ -1 +1 @@
1.0.12 1.0.13-rc6

45
extensions/inference-cortex-extension/download.bat
View File

@ -2,39 +2,38 @@
set BIN_PATH=./bin set BIN_PATH=./bin
set SHARED_PATH=./../../electron/shared set SHARED_PATH=./../../electron/shared
set /p CORTEX_VERSION=<./bin/version.txt set /p CORTEX_VERSION=<./bin/version.txt
set ENGINE_VERSION=0.1.56 set ENGINE_VERSION=b5350
@REM Download cortex.llamacpp binaries @REM Download llama.cpp binaries
set DOWNLOAD_URL=https://github.com/menloresearch/cortex.llamacpp/releases/download/v%ENGINE_VERSION%/cortex.llamacpp-%ENGINE_VERSION%-windows-amd64 set DOWNLOAD_URL=https://github.com/menloresearch/llama.cpp/releases/download/%ENGINE_VERSION%/llama-%ENGINE_VERSION%-bin-win
set CUDA_DOWNLOAD_URL=https://github.com/menloresearch/cortex.llamacpp/releases/download/v%ENGINE_VERSION% set DOWNLOAD_GGML_URL=https://github.com/ggml-org/llama.cpp/releases/download/%ENGINE_VERSION%/llama-%ENGINE_VERSION%-bin-win
set SUBFOLDERS=windows-amd64-noavx-cuda-12-0 windows-amd64-noavx-cuda-11-7 windows-amd64-avx2-cuda-12-0 windows-amd64-avx2-cuda-11-7 windows-amd64-noavx windows-amd64-avx windows-amd64-avx2 windows-amd64-avx512 windows-amd64-vulkan set CUDA_DOWNLOAD_URL=https://github.com/menloresearch/llama.cpp/releases/download/%ENGINE_VERSION%
set SUBFOLDERS=win-noavx-cuda-cu12.0-x64 win-noavx-cuda-cu11.7-x64 win-avx2-cuda-cu12.0-x64 win-avx2-cuda-cu11.7-x64 win-noavx-x64 win-avx-x64 win-avx2-x64 win-avx512-x64 win-vulkan-x64
call .\node_modules\.bin\download -e --strip 1 -o %BIN_PATH% https://github.com/menloresearch/cortex.cpp/releases/download/v%CORTEX_VERSION%/cortex-%CORTEX_VERSION%-windows-amd64.tar.gz call .\node_modules\.bin\download -e --strip 1 -o %BIN_PATH% https://github.com/menloresearch/cortex.cpp/releases/download/v%CORTEX_VERSION%/cortex-%CORTEX_VERSION%-windows-amd64.tar.gz
call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2-cuda-12-0.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-avx2-cuda-12-0/v%ENGINE_VERSION% call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2-cuda-cu12.0-x64.tar.gz -e --strip 2 -o %SHARED_PATH%/engines/llama.cpp/win-avx2-cuda-cu12.0-x64/%ENGINE_VERSION%
call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2-cuda-11-7.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-avx2-cuda-11-7/v%ENGINE_VERSION% call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2-cuda-cu11.7-x64.tar.gz -e --strip 2 -o %SHARED_PATH%/engines/llama.cpp/win-avx2-cuda-cu11.7-x64/%ENGINE_VERSION%
call .\node_modules\.bin\download %DOWNLOAD_URL%-noavx-cuda-12-0.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-noavx-cuda-12-0/v%ENGINE_VERSION% call .\node_modules\.bin\download %DOWNLOAD_URL%-noavx-cuda-cu12.0-x64.tar.gz -e --strip 2 -o %SHARED_PATH%/engines/llama.cpp/win-noavx-cuda-cu12.0-x64/%ENGINE_VERSION%
call .\node_modules\.bin\download %DOWNLOAD_URL%-noavx-cuda-11-7.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-noavx-cuda-11-7/v%ENGINE_VERSION% call .\node_modules\.bin\download %DOWNLOAD_URL%-noavx-cuda-cu11.7-x64.tar.gz -e --strip 2 -o %SHARED_PATH%/engines/llama.cpp/win-noavx-cuda-cu11.7-x64/%ENGINE_VERSION%
call .\node_modules\.bin\download %DOWNLOAD_URL%-noavx.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-noavx/v%ENGINE_VERSION% @REM call .\node_modules\.bin\download %DOWNLOAD_GGML_URL%-noavx-x64.zip -e --strip 1 -o %SHARED_PATH%/engines/llama.cpp/win-noavx-x64/%ENGINE_VERSION%
call .\node_modules\.bin\download %DOWNLOAD_URL%-avx.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-avx/v%ENGINE_VERSION% @REM call .\node_modules\.bin\download %DOWNLOAD_GGML_URL%-avx-x64.zip -e --strip 1 -o %SHARED_PATH%/engines/llama.cpp/win-avx-x64/%ENGINE_VERSION%
call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-avx2/v%ENGINE_VERSION% @REM call .\node_modules\.bin\download %DOWNLOAD_GGML_URL%-avx2-x64.zip -e --strip 1 -o %SHARED_PATH%/engines/llama.cpp/win-avx2-x64/%ENGINE_VERSION%
call .\node_modules\.bin\download %DOWNLOAD_URL%-avx512.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-avx512/v%ENGINE_VERSION% @REM call .\node_modules\.bin\download %DOWNLOAD_GGML_URL%-avx512-x64.zip -e --strip 1 -o %SHARED_PATH%/engines/llama.cpp/win-avx512-x64/%ENGINE_VERSION%
call .\node_modules\.bin\download %DOWNLOAD_URL%-vulkan.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-vulkan/v%ENGINE_VERSION% @REM call .\node_modules\.bin\download %DOWNLOAD_GGML_URL%-vulkan-x64.zip -e --strip 1 -o %SHARED_PATH%/engines/llama.cpp/win-vulkan-x64/%ENGINE_VERSION%
call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cuda-12-0-windows-amd64.tar.gz -e --strip 1 -o %BIN_PATH% call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cudart-llama-bin-win-cu12.0-x64.tar.gz -e --strip 1 -o %BIN_PATH%
call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cuda-11-7-windows-amd64.tar.gz -e --strip 1 -o %BIN_PATH% call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cudart-llama-bin-win-cu11.7-x64.tar.gz -e --strip 1 -o %BIN_PATH%
move %BIN_PATH%\cortex-server-beta.exe %BIN_PATH%\cortex-server.exe move %BIN_PATH%\cortex-server-beta.exe %BIN_PATH%\cortex-server.exe
del %BIN_PATH%\cortex-beta.exe del %BIN_PATH%\cortex-beta.exe
del %BIN_PATH%\cortex.exe del %BIN_PATH%\cortex.exe
@REM Loop through each folder and move DLLs (excluding engine.dll) @REM Loop through each folder and move DLLs
for %%F in (%SUBFOLDERS%) do ( for %%F in (%SUBFOLDERS%) do (
echo Processing folder: %SHARED_PATH%\engines\cortex.llamacpp\%%F\v%ENGINE_VERSION% echo Processing folder: %SHARED_PATH%\engines\llama.cpp\%%F\%ENGINE_VERSION%
@REM Move all .dll files except engine.dll @REM Move cu*.dll files
for %%D in (%SHARED_PATH%\engines\cortex.llamacpp\%%F\v%ENGINE_VERSION%\*.dll) do ( for %%D in (%SHARED_PATH%\engines\llama.cpp\%%F\%ENGINE_VERSION%\cu*.dll) do (
if /I not "%%~nxD"=="engine.dll" ( move "%%D" "%BIN_PATH%"
move "%%D" "%BIN_PATH%"
)
) )
) )

32
extensions/inference-cortex-extension/download.sh
View File

@ -2,10 +2,10 @@
# Read CORTEX_VERSION # Read CORTEX_VERSION
CORTEX_VERSION=$(cat ./bin/version.txt) CORTEX_VERSION=$(cat ./bin/version.txt)
ENGINE_VERSION=0.1.56 ENGINE_VERSION=b5350
CORTEX_RELEASE_URL="https://github.com/menloresearch/cortex.cpp/releases/download" CORTEX_RELEASE_URL="https://github.com/menloresearch/cortex.cpp/releases/download"
ENGINE_DOWNLOAD_URL="https://github.com/menloresearch/cortex.llamacpp/releases/download/v${ENGINE_VERSION}/cortex.llamacpp-${ENGINE_VERSION}" ENGINE_DOWNLOAD_URL=https://github.com/menloresearch/llama.cpp/releases/download/${ENGINE_VERSION}/llama-${ENGINE_VERSION}-bin
CUDA_DOWNLOAD_URL="https://github.com/menloresearch/cortex.llamacpp/releases/download/v${ENGINE_VERSION}" CUDA_DOWNLOAD_URL=https://github.com/menloresearch/llama.cpp/releases/download/${ENGINE_VERSION}
BIN_PATH=./bin BIN_PATH=./bin
SHARED_PATH="../../electron/shared" SHARED_PATH="../../electron/shared"
# Detect platform # Detect platform
@ -20,17 +20,17 @@ if [ "$OS_TYPE" == "Linux" ]; then
chmod +x "./bin/cortex-server" chmod +x "./bin/cortex-server"
# Download engines for Linux # Download engines for Linux
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-noavx.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-noavx/v${ENGINE_VERSION}" 1 download "${ENGINE_DOWNLOAD_URL}-linux-noavx-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-noavx-x64/${ENGINE_VERSION}" 1
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx/v${ENGINE_VERSION}" 1 download "${ENGINE_DOWNLOAD_URL}-linux-avx-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-avx-x64/${ENGINE_VERSION}" 1
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx2.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx2/v${ENGINE_VERSION}" 1 download "${ENGINE_DOWNLOAD_URL}-linux-avx2-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-avx2-x64/${ENGINE_VERSION}" 1
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx512.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx512/v${ENGINE_VERSION}" 1 download "${ENGINE_DOWNLOAD_URL}-linux-avx512-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-avx512-x64/${ENGINE_VERSION}" 1
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx2-cuda-12-0.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx2-cuda-12-0/v${ENGINE_VERSION}" 1 download "${ENGINE_DOWNLOAD_URL}-linux-avx2-cuda-cu12.0-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-avx2-cuda-cu12.0-x64/${ENGINE_VERSION}" 1
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx2-cuda-11-7.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx2-cuda-11-7/v${ENGINE_VERSION}" 1 download "${ENGINE_DOWNLOAD_URL}-linux-avx2-cuda-cu11.7-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-avx2-cuda-cu11.7-x64/${ENGINE_VERSION}" 1
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-noavx-cuda-12-0.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-noavx-cuda-12-0/v${ENGINE_VERSION}" 1 download "${ENGINE_DOWNLOAD_URL}-linux-noavx-cuda-cu12.0-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-noavx-cuda-cu12.0-x64/${ENGINE_VERSION}" 1
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-noavx-cuda-11-7.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-noavx-cuda-11-7/v${ENGINE_VERSION}" 1 download "${ENGINE_DOWNLOAD_URL}-linux-noavx-cuda-cu11.7-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-noavx-cuda-cu11.7-x64/${ENGINE_VERSION}" 1
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-vulkan.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-vulkan/v${ENGINE_VERSION}" 1 download "${ENGINE_DOWNLOAD_URL}-linux-vulkan-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-vulkan-x64/${ENGINE_VERSION}" 1
download "${CUDA_DOWNLOAD_URL}/cuda-12-0-linux-amd64.tar.gz" -e --strip 1 -o "${BIN_PATH}" 1 download "${CUDA_DOWNLOAD_URL}/cudart-llama-bin-linux-cu12.0-x64.tar.gz" -e --strip 1 -o "${BIN_PATH}" 1
download "${CUDA_DOWNLOAD_URL}/cuda-11-7-linux-amd64.tar.gz" -e --strip 1 -o "${BIN_PATH}" 1 download "${CUDA_DOWNLOAD_URL}/cudart-llama-bin-linux-cu11.7-x64.tar.gz" -e --strip 1 -o "${BIN_PATH}" 1
elif [ "$OS_TYPE" == "Darwin" ]; then elif [ "$OS_TYPE" == "Darwin" ]; then
# macOS downloads # macOS downloads
@ -41,8 +41,8 @@ elif [ "$OS_TYPE" == "Darwin" ]; then
chmod +x "./bin/cortex-server" chmod +x "./bin/cortex-server"
# Download engines for macOS # Download engines for macOS
download "${ENGINE_DOWNLOAD_URL}-mac-arm64.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/mac-arm64/v${ENGINE_VERSION}" download "${ENGINE_DOWNLOAD_URL}-macos-arm64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/macos-arm64/${ENGINE_VERSION}"
download "${ENGINE_DOWNLOAD_URL}-mac-amd64.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/mac-amd64/v${ENGINE_VERSION}" download "${ENGINE_DOWNLOAD_URL}-macos-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/macos-x64/${ENGINE_VERSION}"
else else
echo "Unsupported operating system: $OS_TYPE" echo "Unsupported operating system: $OS_TYPE"

2
extensions/inference-cortex-extension/rolldown.config.mjs
View File

@ -19,7 +19,7 @@ export default defineConfig([
CORTEX_SOCKET_URL: JSON.stringify( CORTEX_SOCKET_URL: JSON.stringify(
`ws://127.0.0.1:${process.env.CORTEX_API_PORT ?? '39291'}` `ws://127.0.0.1:${process.env.CORTEX_API_PORT ?? '39291'}`
), ),
CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.56'), CORTEX_ENGINE_VERSION: JSON.stringify('b5350'),
}, },
}, },
{ {

2
src-tauri/Cargo.toml
View File

@ -32,7 +32,7 @@ rand = "0.8"
tauri-plugin-http = { version = "2", features = ["unsafe-headers"] } tauri-plugin-http = { version = "2", features = ["unsafe-headers"] }
tauri-plugin-store = "2" tauri-plugin-store = "2"
hyper = { version = "0.14", features = ["server"] } hyper = { version = "0.14", features = ["server"] }
reqwest = { version = "0.11", features = ["json"] } reqwest = { version = "0.11", features = ["json", "blocking"] }
tokio = { version = "1", features = ["full"] } tokio = { version = "1", features = ["full"] }
rmcp = { git = "https://github.com/modelcontextprotocol/rust-sdk", branch = "main", features = [ rmcp = { git = "https://github.com/modelcontextprotocol/rust-sdk", branch = "main", features = [
"client", "client",

41
src-tauri/binaries/download.bat
View File

@ -1,8 +1,9 @@
@echo off @echo off
set CORTEX_VERSION=1.0.13-rc1 set CORTEX_VERSION=1.0.13-rc6
set ENGINE_VERSION=0.1.55 set ENGINE_VERSION=b5350
set ENGINE_DOWNLOAD_URL=https://github.com/menloresearch/cortex.llamacpp/releases/download/v%ENGINE_VERSION%/cortex.llamacpp-%ENGINE_VERSION%-windows-amd64 set ENGINE_DOWNLOAD_URL=https://github.com/menloresearch/llama.cpp/releases/download/%ENGINE_VERSION%/llama-%ENGINE_VERSION%-bin-win
set ENGINE_DOWNLOAD_GGML_URL=https://github.com/ggml-org/llama.cpp/releases/download/%ENGINE_VERSION%/llama-%ENGINE_VERSION%-bin-win
set CUDA_DOWNLOAD_URL=https://github.com/menloresearch/cortex.llamacpp/releases/download/v%ENGINE_VERSION% set CUDA_DOWNLOAD_URL=https://github.com/menloresearch/cortex.llamacpp/releases/download/v%ENGINE_VERSION%
set SUBFOLDERS=windows-amd64-noavx-cuda-12-0 windows-amd64-noavx-cuda-11-7 windows-amd64-avx2-cuda-12-0 windows-amd64-avx2-cuda-11-7 windows-amd64-noavx windows-amd64-avx windows-amd64-avx2 windows-amd64-avx512 windows-amd64-vulkan set SUBFOLDERS=windows-amd64-noavx-cuda-12-0 windows-amd64-noavx-cuda-11-7 windows-amd64-avx2-cuda-12-0 windows-amd64-avx2-cuda-11-7 windows-amd64-noavx windows-amd64-avx windows-amd64-avx2 windows-amd64-avx512 windows-amd64-vulkan
set BIN_PATH="./" set BIN_PATH="./"
@ -11,32 +12,30 @@ set DOWNLOAD_TOOL=..\..\extensions\inference-cortex-extension\node_modules\.bin\
@REM Download cortex.llamacpp binaries @REM Download cortex.llamacpp binaries
call %DOWNLOAD_TOOL% -e --strip 1 -o %BIN_PATH% https://github.com/menloresearch/cortex.cpp/releases/download/v%CORTEX_VERSION%/cortex-%CORTEX_VERSION%-windows-amd64.tar.gz call %DOWNLOAD_TOOL% -e --strip 1 -o %BIN_PATH% https://github.com/menloresearch/cortex.cpp/releases/download/v%CORTEX_VERSION%/cortex-%CORTEX_VERSION%-windows-amd64.tar.gz
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-avx2-cuda-12-0.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-avx2-cuda-12-0/v%ENGINE_VERSION% call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-avx2-cuda-cu12.0-x64.tar.gz -e --strip 2 -o./engines/llama.cpp/win-avx2-cuda-cu12.0-x64/%ENGINE_VERSION%
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-avx2-cuda-11-7.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-avx2-cuda-11-7/v%ENGINE_VERSION% call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-avx2-cuda-cu11.7-x64.tar.gz -e --strip 2 -o./engines/llama.cpp/win-avx2-cuda-cu11.7-x64/%ENGINE_VERSION%
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-noavx-cuda-12-0.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-noavx-cuda-12-0/v%ENGINE_VERSION% call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-noavx-cuda-cu12.0-x64.tar.gz -e --strip 2 -o./engines/llama.cpp/win-noavx-cuda-cu12.0-x64/%ENGINE_VERSION%
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-noavx-cuda-11-7.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-noavx-cuda-11-7/v%ENGINE_VERSION% call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-noavx-cuda-cu11.7-x64.tar.gz -e --strip 2 -o./engines/llama.cpp/win-noavx-cuda-cu11.7-x64/%ENGINE_VERSION%
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-noavx.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-noavx/v%ENGINE_VERSION% call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_GGML_URL%-noavx-x64.zip -e --strip 1 -o./engines/llama.cpp/win-noavx-x64/%ENGINE_VERSION%
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-avx.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-avx/v%ENGINE_VERSION% call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_GGML_URL%-avx-x64.zip -e --strip 1 -o./engines/llama.cpp/win-avx-x64/%ENGINE_VERSION%
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-avx2.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-avx2/v%ENGINE_VERSION% call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_GGML_URL%-avx2-x64.zip -e --strip 1 -o./engines/llama.cpp/win-avx2-x64/%ENGINE_VERSION%
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-avx512.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-avx512/v%ENGINE_VERSION% call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_GGML_URL%-avx512-x64.zip -e --strip 1 -o./engines/llama.cpp/win-avx512-x64/%ENGINE_VERSION%
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-vulkan.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-vulkan/v%ENGINE_VERSION% call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_GGML_URL%-vulkan-x64.zip -e --strip 1 -o./engines/llama.cpp/win-vulkan-x64/%ENGINE_VERSION%
call %DOWNLOAD_TOOL% %CUDA_DOWNLOAD_URL%/cuda-12-0-windows-amd64.tar.gz -e --strip 1 -o %BIN_PATH% call %DOWNLOAD_TOOL% %CUDA_DOWNLOAD_URL%/cudart-llama-bin-win-cu12.0-x64.tar.gz -e --strip 1 -o %BIN_PATH%
call %DOWNLOAD_TOOL% %CUDA_DOWNLOAD_URL%/cuda-11-7-windows-amd64.tar.gz -e --strip 1 -o %BIN_PATH% call %DOWNLOAD_TOOL% %CUDA_DOWNLOAD_URL%/cudart-llama-bin-win-cu11.7-x64.tar.gz -e --strip 1 -o %BIN_PATH%
move %BIN_PATH%cortex-server-beta.exe %BIN_PATH%cortex-server.exe move %BIN_PATH%cortex-server-beta.exe %BIN_PATH%cortex-server.exe
copy %BIN_PATH%cortex-server.exe %BIN_PATH%cortex-server-x86_64-pc-windows-msvc.exe copy %BIN_PATH%cortex-server.exe %BIN_PATH%cortex-server-x86_64-pc-windows-msvc.exe
del %BIN_PATH%cortex-beta.exe del %BIN_PATH%cortex-beta.exe
del %BIN_PATH%cortex.exe del %BIN_PATH%cortex.exe
@REM Loop through each folder and move DLLs (excluding engine.dll) @REM Loop through each folder and move DLLs
for %%F in (%SUBFOLDERS%) do ( for %%F in (%SUBFOLDERS%) do (
echo Processing folder: .\engines\cortex.llamacpp\%%F\v%ENGINE_VERSION% echo Processing folder: .\engines\llama.cpp\%%F\%ENGINE_VERSION%
@REM Move all .dll files except engine.dll @REM Move cu*.dll files
for %%D in (.\engines\cortex.llamacpp\%%F\v%ENGINE_VERSION%\*.dll) do ( for %%D in (.\engines\engines\llama.cpp\%%F\%ENGINE_VERSION%\cu*.dll) do (
if /I not "%%~nxD"=="engine.dll" ( move "%%D" "%BIN_PATH%"
move "%%D" "%BIN_PATH%"
)
) )
) )

47
src-tauri/binaries/download.sh
View File

@ -2,23 +2,23 @@
download() { download() {
URL="$1" URL="$1"
EXTRA_ARGS="${@:2}" EXTRA_ARGS="${@:3}"
OUTPUT_DIR="${EXTRA_ARGS[${#EXTRA_ARGS[@]} -1]}" OUTPUT_DIR="${EXTRA_ARGS[${#EXTRA_ARGS[@]} -1]}"
mkdir -p "$OUTPUT_DIR" mkdir -p "$OUTPUT_DIR"
echo "Downloading $URL to $OUTPUT_DIR using curl..." echo "Downloading $URL to $OUTPUT_DIR using curl..."
curl -L "$URL" -o "$OUTPUT_DIR/$(basename "$URL")" curl -L "$URL" -o "$OUTPUT_DIR/$(basename "$URL")"
tar -xzf "$OUTPUT_DIR/$(basename "$URL")" -C "$OUTPUT_DIR" --strip-components 1 tar -xzf "$OUTPUT_DIR/$(basename "$URL")" -C "$OUTPUT_DIR" --strip-components $2
rm "$OUTPUT_DIR/$(basename "$URL")" rm "$OUTPUT_DIR/$(basename "$URL")"
} }
# Read CORTEX_VERSION # Read CORTEX_VERSION
CORTEX_VERSION=1.0.13-rc1 CORTEX_VERSION=1.0.13-rc6
ENGINE_VERSION=0.1.55 ENGINE_VERSION=b5350
CORTEX_RELEASE_URL="https://github.com/menloresearch/cortex.cpp/releases/download" CORTEX_RELEASE_URL="https://github.com/menloresearch/cortex.cpp/releases/download"
ENGINE_DOWNLOAD_URL="https://github.com/menloresearch/cortex.llamacpp/releases/download/v${ENGINE_VERSION}/cortex.llamacpp-${ENGINE_VERSION}" ENGINE_DOWNLOAD_URL=https://github.com/menloresearch/llama.cpp/releases/download/${ENGINE_VERSION}/llama-${ENGINE_VERSION}-bin
CUDA_DOWNLOAD_URL="https://github.com/menloresearch/cortex.llamacpp/releases/download/v${ENGINE_VERSION}" CUDA_DOWNLOAD_URL=https://github.com/menloresearch/llama.cpp/releases/download/${ENGINE_VERSION}
BIN_PATH=./ BIN_PATH=./
SHARED_PATH="." SHARED_PATH="."
# Detect platform # Detect platform
@ -31,38 +31,41 @@ fi
if [ "$OS_TYPE" == "Linux" ]; then if [ "$OS_TYPE" == "Linux" ]; then
# Linux downloads # Linux downloads
download "${CORTEX_RELEASE_URL}/v${CORTEX_VERSION}/cortex-${CORTEX_VERSION}-linux-amd64.tar.gz" "${BIN_PATH}" download "${CORTEX_RELEASE_URL}/v${CORTEX_VERSION}/cortex-${CORTEX_VERSION}-linux-amd64.tar.gz" 1 "${BIN_PATH}"
mv .cortex-server-beta ./cortex-server mv ./cortex-server-beta ./cortex-server
rm -rf ./cortex rm -rf ./cortex
rm -rf ./cortex-beta rm -rf ./cortex-beta
chmod +x "./cortex-server" chmod +x "./cortex-server"
cp ./cortex-server ./cortex-server-x86_64-unknown-linux-gnu
# Download engines for Linux # Download engines for Linux
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-noavx.tar.gz" "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-noavx/v${ENGINE_VERSION}" download "${ENGINE_DOWNLOAD_URL}-linux-noavx-x64.tar.gz" 2 "${SHARED_PATH}/engines/llama.cpp/linux-noavx-x64/${ENGINE_VERSION}"
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx.tar.gz" "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx/v${ENGINE_VERSION}" download "${ENGINE_DOWNLOAD_URL}-linux-avx-x64.tar.gz" 2 "${SHARED_PATH}/engines/llama.cpp/linux-avx-x64/${ENGINE_VERSION}"
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx2.tar.gz" "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx2/v${ENGINE_VERSION}" download "${ENGINE_DOWNLOAD_URL}-linux-avx2-x64.tar.gz" 2 "${SHARED_PATH}/engines/llama.cpp/linux-avx2-x64/${ENGINE_VERSION}"
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx512.tar.gz" "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx512/v${ENGINE_VERSION}" download "${ENGINE_DOWNLOAD_URL}-linux-avx512-x64.tar.gz" 2 "${SHARED_PATH}/engines/llama.cpp/linux-avx512-x64/${ENGINE_VERSION}"
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx2-cuda-12-0.tar.gz" "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx2-cuda-12-0/v${ENGINE_VERSION}" download "${ENGINE_DOWNLOAD_URL}-linux-avx2-cuda-cu12.0-x64.tar.gz" 2 "${SHARED_PATH}/engines/llama.cpp/linux-avx2-cuda-cu12.0-x64/${ENGINE_VERSION}"
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx2-cuda-11-7.tar.gz" "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx2-cuda-11-7/v${ENGINE_VERSION}" download "${ENGINE_DOWNLOAD_URL}-linux-avx2-cuda-cu11.7-x64.tar.gz" 2 "${SHARED_PATH}/engines/llama.cpp/linux-avx2-cuda-cu11.7-x64/${ENGINE_VERSION}"
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-noavx-cuda-12-0.tar.gz" "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-noavx-cuda-12-0/v${ENGINE_VERSION}" download "${ENGINE_DOWNLOAD_URL}-linux-noavx-cuda-cu12.0-x64.tar.gz" 2 "${SHARED_PATH}/engines/llama.cpp/linux-noavx-cuda-cu12.0-x64/${ENGINE_VERSION}"
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-noavx-cuda-11-7.tar.gz" "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-noavx-cuda-11-7/v${ENGINE_VERSION}" download "${ENGINE_DOWNLOAD_URL}-linux-noavx-cuda-cu11.7-x64.tar.gz" 2 "${SHARED_PATH}/engines/llama.cpp/linux-noavx-cuda-cu11.7-x64/${ENGINE_VERSION}"
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-vulkan.tar.gz" "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-vulkan/v${ENGINE_VERSION}" download "${ENGINE_DOWNLOAD_URL}-linux-vulkan-x64.tar.gz" 2 "${SHARED_PATH}/engines/llama.cpp/linux-vulkan-x64/${ENGINE_VERSION}"
download "${CUDA_DOWNLOAD_URL}/cuda-12-0-linux-amd64.tar.gz" "${BIN_PATH}" download "${CUDA_DOWNLOAD_URL}/cudart-llama-bin-linux-cu12.0-x64.tar.gz" 0 "${BIN_PATH}"
download "${CUDA_DOWNLOAD_URL}/cuda-11-7-linux-amd64.tar.gz" "${BIN_PATH}" download "${CUDA_DOWNLOAD_URL}/cudart-llama-bin-linux-cu11.7-x64.tar.gz" 0 "${BIN_PATH}"
elif [ "$OS_TYPE" == "Darwin" ]; then elif [ "$OS_TYPE" == "Darwin" ]; then
# macOS downloads # macOS downloads
download "${CORTEX_RELEASE_URL}/v${CORTEX_VERSION}/cortex-${CORTEX_VERSION}-mac-universal.tar.gz" "${BIN_PATH}" download "${CORTEX_RELEASE_URL}/v${CORTEX_VERSION}/cortex-${CORTEX_VERSION}-mac-universal.tar.gz" 1 "${BIN_PATH}"
mv ./cortex-server-beta ./cortex-server mv ./cortex-server-beta ./cortex-server
rm -rf ./cortex rm -rf ./cortex
rm -rf ./cortex-beta rm -rf ./cortex-beta
chmod +x "./cortex-server" chmod +x "./cortex-server"
mv ./cortex-server ./cortex-server-universal-apple-darwin mv ./cortex-server ./cortex-server-universal-apple-darwin
cp ./cortex-server-universal-apple-darwin ./cortex-server-aarch64-apple-darwin cp ./cortex-server-universal-apple-darwin ./cortex-server-aarch64-apple-darwin
cp ./cortex-server-universal-apple-darwin ./cortex-server-x86_64-apple-darwin
# Download engines for macOS # Download engines for macOS
download "${ENGINE_DOWNLOAD_URL}-mac-arm64.tar.gz" "${SHARED_PATH}/engines/cortex.llamacpp/mac-arm64/v${ENGINE_VERSION}" download "${ENGINE_DOWNLOAD_URL}-macos-arm64.tar.gz" 2 "${SHARED_PATH}/engines/llama.cpp/macos-arm64/${ENGINE_VERSION}"
download "${ENGINE_DOWNLOAD_URL}-mac-amd64.tar.gz" "${SHARED_PATH}/engines/cortex.llamacpp/mac-amd64/v${ENGINE_VERSION}" download "${ENGINE_DOWNLOAD_URL}-macos-x64.tar.gz" 2 "${SHARED_PATH}/engines/llama.cpp/macos-x64/${ENGINE_VERSION}"
else else
echo "Unsupported operating system: $OS_TYPE" echo "Unsupported operating system: $OS_TYPE"

6
src-tauri/src/lib.rs
View File

@ -9,6 +9,8 @@ use std::{collections::HashMap, sync::Arc};
use tauri::Emitter; use tauri::Emitter;
use tokio::sync::Mutex; use tokio::sync::Mutex;
use reqwest::blocking::Client;
#[cfg_attr(mobile, tauri::mobile_entry_point)] #[cfg_attr(mobile, tauri::mobile_entry_point)]
pub fn run() { pub fn run() {
tauri::Builder::default() tauri::Builder::default()
@ -86,6 +88,10 @@ pub fn run() {
}) })
.on_window_event(|window, event| match event { .on_window_event(|window, event| match event {
tauri::WindowEvent::CloseRequested { .. } => { tauri::WindowEvent::CloseRequested { .. } => {
let client = Client::new();
let url = "http://127.0.0.1:39291/processManager/destroy";
let _ = client.delete(url).send();
window.emit("kill-sidecar", ()).unwrap(); window.emit("kill-sidecar", ()).unwrap();
} }
_ => {} _ => {}