fix: Jan supports cortex with llama-server (#4904)
* fix: update logic to pull llama-cpp engine * chore: bump cortex version * fix: chat macos * chore: bump llama.cpp * chore: download windows * chore: download linux * chore: bump cortex to 1.0.13-rc6 * fix: stop cortex before exit * chore: bum llama.cpp engine to b5350 --------- Co-authored-by: vansangpfiev <sang@jan.ai>
This commit is contained in:
vansangpfiev
Louis
parent
f3a808cb89
commit
dea29ff8c4
@ -15,7 +15,7 @@ export default defineConfig([
|
||||
`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? '39291'}`
|
||||
),
|
||||
PLATFORM: JSON.stringify(process.platform),
|
||||
CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.56'),
|
||||
CORTEX_ENGINE_VERSION: JSON.stringify('b5350'),
|
||||
DEFAULT_REMOTE_ENGINES: JSON.stringify(engines),
|
||||
DEFAULT_REMOTE_MODELS: JSON.stringify(models),
|
||||
DEFAULT_REQUEST_PAYLOAD_TRANSFORM: JSON.stringify(
|
||||
@ -38,7 +38,7 @@ export default defineConfig([
|
||||
file: 'dist/node/index.cjs.js',
|
||||
},
|
||||
define: {
|
||||
CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.56'),
|
||||
CORTEX_ENGINE_VERSION: JSON.stringify('b5350'),
|
||||
},
|
||||
},
|
||||
])
|
||||
|
||||
@ -16,12 +16,12 @@ const symlinkEngines = async () => {
|
||||
appResourcePath(),
|
||||
'shared',
|
||||
'engines',
|
||||
'cortex.llamacpp'
|
||||
'llama.cpp'
|
||||
)
|
||||
const symlinkEnginePath = path.join(
|
||||
getJanDataFolderPath(),
|
||||
'engines',
|
||||
'cortex.llamacpp'
|
||||
'llama.cpp'
|
||||
)
|
||||
const variantFolders = await readdir(sourceEnginePath)
|
||||
const isStandalone = process.platform === 'linux'
|
||||
|
||||
@ -32,23 +32,23 @@ const gpuRunMode = (settings?: GpuSetting): RunMode => {
|
||||
*/
|
||||
const os = (settings?: GpuSetting): string => {
|
||||
return PLATFORM === 'win32'
|
||||
? 'windows-amd64'
|
||||
? 'win'
|
||||
: PLATFORM === 'darwin'
|
||||
? settings?.cpu?.arch === 'arm64'
|
||||
? 'mac-arm64'
|
||||
: 'mac-amd64'
|
||||
: 'linux-amd64'
|
||||
? 'macos-arm64'
|
||||
: 'macos-x64'
|
||||
: 'linux'
|
||||
}
|
||||
|
||||
/**
|
||||
* The CUDA version that will be set - either '11-7' or '12-0'.
|
||||
* The CUDA version that will be set - either 'cu12.0' or 'cu11.7'.
|
||||
* @param settings
|
||||
* @returns
|
||||
*/
|
||||
const cudaVersion = (settings?: GpuSetting): '12-0' | '11-7' | undefined => {
|
||||
const cudaVersion = (settings?: GpuSetting): 'cu12.0' | 'cu11.7' | undefined => {
|
||||
return settings.gpus?.some((gpu) => gpu.version.includes('12'))
|
||||
? '12-0'
|
||||
: '11-7'
|
||||
? 'cu12.0'
|
||||
: 'cu11.7'
|
||||
}
|
||||
|
||||
/**
|
||||
@ -84,15 +84,17 @@ export const engineVariant = async (
|
||||
: 'noavx',
|
||||
runMode,
|
||||
cudaVersion(gpuSetting),
|
||||
'x64',
|
||||
]
|
||||
: // For cpu only we need to check all available supported instructions
|
||||
[
|
||||
(gpuSetting.cpu?.instructions ?? ['noavx']).find((e) =>
|
||||
instructionBinaryNames.includes(e.toLowerCase())
|
||||
) ?? 'noavx',
|
||||
'x64',
|
||||
]),
|
||||
].filter(Boolean)
|
||||
: [platform, 'vulkan']
|
||||
: [platform, 'vulkan', 'x64']
|
||||
|
||||
let engineVariantString = engineVariant.join('-')
|
||||
|
||||
|
||||
@ -1 +1 @@
|
||||
1.0.12
|
||||
1.0.13-rc6
|
||||
@ -2,39 +2,38 @@
|
||||
set BIN_PATH=./bin
|
||||
set SHARED_PATH=./../../electron/shared
|
||||
set /p CORTEX_VERSION=<./bin/version.txt
|
||||
set ENGINE_VERSION=0.1.56
|
||||
set ENGINE_VERSION=b5350
|
||||
|
||||
@REM Download cortex.llamacpp binaries
|
||||
set DOWNLOAD_URL=https://github.com/menloresearch/cortex.llamacpp/releases/download/v%ENGINE_VERSION%/cortex.llamacpp-%ENGINE_VERSION%-windows-amd64
|
||||
set CUDA_DOWNLOAD_URL=https://github.com/menloresearch/cortex.llamacpp/releases/download/v%ENGINE_VERSION%
|
||||
set SUBFOLDERS=windows-amd64-noavx-cuda-12-0 windows-amd64-noavx-cuda-11-7 windows-amd64-avx2-cuda-12-0 windows-amd64-avx2-cuda-11-7 windows-amd64-noavx windows-amd64-avx windows-amd64-avx2 windows-amd64-avx512 windows-amd64-vulkan
|
||||
@REM Download llama.cpp binaries
|
||||
set DOWNLOAD_URL=https://github.com/menloresearch/llama.cpp/releases/download/%ENGINE_VERSION%/llama-%ENGINE_VERSION%-bin-win
|
||||
set DOWNLOAD_GGML_URL=https://github.com/ggml-org/llama.cpp/releases/download/%ENGINE_VERSION%/llama-%ENGINE_VERSION%-bin-win
|
||||
set CUDA_DOWNLOAD_URL=https://github.com/menloresearch/llama.cpp/releases/download/%ENGINE_VERSION%
|
||||
set SUBFOLDERS=win-noavx-cuda-cu12.0-x64 win-noavx-cuda-cu11.7-x64 win-avx2-cuda-cu12.0-x64 win-avx2-cuda-cu11.7-x64 win-noavx-x64 win-avx-x64 win-avx2-x64 win-avx512-x64 win-vulkan-x64
|
||||
|
||||
call .\node_modules\.bin\download -e --strip 1 -o %BIN_PATH% https://github.com/menloresearch/cortex.cpp/releases/download/v%CORTEX_VERSION%/cortex-%CORTEX_VERSION%-windows-amd64.tar.gz
|
||||
call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2-cuda-12-0.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-avx2-cuda-12-0/v%ENGINE_VERSION%
|
||||
call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2-cuda-11-7.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-avx2-cuda-11-7/v%ENGINE_VERSION%
|
||||
call .\node_modules\.bin\download %DOWNLOAD_URL%-noavx-cuda-12-0.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-noavx-cuda-12-0/v%ENGINE_VERSION%
|
||||
call .\node_modules\.bin\download %DOWNLOAD_URL%-noavx-cuda-11-7.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-noavx-cuda-11-7/v%ENGINE_VERSION%
|
||||
call .\node_modules\.bin\download %DOWNLOAD_URL%-noavx.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-noavx/v%ENGINE_VERSION%
|
||||
call .\node_modules\.bin\download %DOWNLOAD_URL%-avx.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-avx/v%ENGINE_VERSION%
|
||||
call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-avx2/v%ENGINE_VERSION%
|
||||
call .\node_modules\.bin\download %DOWNLOAD_URL%-avx512.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-avx512/v%ENGINE_VERSION%
|
||||
call .\node_modules\.bin\download %DOWNLOAD_URL%-vulkan.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-vulkan/v%ENGINE_VERSION%
|
||||
call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cuda-12-0-windows-amd64.tar.gz -e --strip 1 -o %BIN_PATH%
|
||||
call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cuda-11-7-windows-amd64.tar.gz -e --strip 1 -o %BIN_PATH%
|
||||
call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2-cuda-cu12.0-x64.tar.gz -e --strip 2 -o %SHARED_PATH%/engines/llama.cpp/win-avx2-cuda-cu12.0-x64/%ENGINE_VERSION%
|
||||
call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2-cuda-cu11.7-x64.tar.gz -e --strip 2 -o %SHARED_PATH%/engines/llama.cpp/win-avx2-cuda-cu11.7-x64/%ENGINE_VERSION%
|
||||
call .\node_modules\.bin\download %DOWNLOAD_URL%-noavx-cuda-cu12.0-x64.tar.gz -e --strip 2 -o %SHARED_PATH%/engines/llama.cpp/win-noavx-cuda-cu12.0-x64/%ENGINE_VERSION%
|
||||
call .\node_modules\.bin\download %DOWNLOAD_URL%-noavx-cuda-cu11.7-x64.tar.gz -e --strip 2 -o %SHARED_PATH%/engines/llama.cpp/win-noavx-cuda-cu11.7-x64/%ENGINE_VERSION%
|
||||
@REM call .\node_modules\.bin\download %DOWNLOAD_GGML_URL%-noavx-x64.zip -e --strip 1 -o %SHARED_PATH%/engines/llama.cpp/win-noavx-x64/%ENGINE_VERSION%
|
||||
@REM call .\node_modules\.bin\download %DOWNLOAD_GGML_URL%-avx-x64.zip -e --strip 1 -o %SHARED_PATH%/engines/llama.cpp/win-avx-x64/%ENGINE_VERSION%
|
||||
@REM call .\node_modules\.bin\download %DOWNLOAD_GGML_URL%-avx2-x64.zip -e --strip 1 -o %SHARED_PATH%/engines/llama.cpp/win-avx2-x64/%ENGINE_VERSION%
|
||||
@REM call .\node_modules\.bin\download %DOWNLOAD_GGML_URL%-avx512-x64.zip -e --strip 1 -o %SHARED_PATH%/engines/llama.cpp/win-avx512-x64/%ENGINE_VERSION%
|
||||
@REM call .\node_modules\.bin\download %DOWNLOAD_GGML_URL%-vulkan-x64.zip -e --strip 1 -o %SHARED_PATH%/engines/llama.cpp/win-vulkan-x64/%ENGINE_VERSION%
|
||||
call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cudart-llama-bin-win-cu12.0-x64.tar.gz -e --strip 1 -o %BIN_PATH%
|
||||
call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cudart-llama-bin-win-cu11.7-x64.tar.gz -e --strip 1 -o %BIN_PATH%
|
||||
|
||||
move %BIN_PATH%\cortex-server-beta.exe %BIN_PATH%\cortex-server.exe
|
||||
del %BIN_PATH%\cortex-beta.exe
|
||||
del %BIN_PATH%\cortex.exe
|
||||
|
||||
@REM Loop through each folder and move DLLs (excluding engine.dll)
|
||||
@REM Loop through each folder and move DLLs
|
||||
for %%F in (%SUBFOLDERS%) do (
|
||||
echo Processing folder: %SHARED_PATH%\engines\cortex.llamacpp\%%F\v%ENGINE_VERSION%
|
||||
echo Processing folder: %SHARED_PATH%\engines\llama.cpp\%%F\%ENGINE_VERSION%
|
||||
|
||||
@REM Move all .dll files except engine.dll
|
||||
for %%D in (%SHARED_PATH%\engines\cortex.llamacpp\%%F\v%ENGINE_VERSION%\*.dll) do (
|
||||
if /I not "%%~nxD"=="engine.dll" (
|
||||
move "%%D" "%BIN_PATH%"
|
||||
)
|
||||
@REM Move cu*.dll files
|
||||
for %%D in (%SHARED_PATH%\engines\llama.cpp\%%F\%ENGINE_VERSION%\cu*.dll) do (
|
||||
move "%%D" "%BIN_PATH%"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@ -2,10 +2,10 @@
|
||||
|
||||
# Read CORTEX_VERSION
|
||||
CORTEX_VERSION=$(cat ./bin/version.txt)
|
||||
ENGINE_VERSION=0.1.56
|
||||
ENGINE_VERSION=b5350
|
||||
CORTEX_RELEASE_URL="https://github.com/menloresearch/cortex.cpp/releases/download"
|
||||
ENGINE_DOWNLOAD_URL="https://github.com/menloresearch/cortex.llamacpp/releases/download/v${ENGINE_VERSION}/cortex.llamacpp-${ENGINE_VERSION}"
|
||||
CUDA_DOWNLOAD_URL="https://github.com/menloresearch/cortex.llamacpp/releases/download/v${ENGINE_VERSION}"
|
||||
ENGINE_DOWNLOAD_URL=https://github.com/menloresearch/llama.cpp/releases/download/${ENGINE_VERSION}/llama-${ENGINE_VERSION}-bin
|
||||
CUDA_DOWNLOAD_URL=https://github.com/menloresearch/llama.cpp/releases/download/${ENGINE_VERSION}
|
||||
BIN_PATH=./bin
|
||||
SHARED_PATH="../../electron/shared"
|
||||
# Detect platform
|
||||
@ -20,17 +20,17 @@ if [ "$OS_TYPE" == "Linux" ]; then
|
||||
chmod +x "./bin/cortex-server"
|
||||
|
||||
# Download engines for Linux
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-noavx.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-noavx/v${ENGINE_VERSION}" 1
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx/v${ENGINE_VERSION}" 1
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx2.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx2/v${ENGINE_VERSION}" 1
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx512.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx512/v${ENGINE_VERSION}" 1
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx2-cuda-12-0.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx2-cuda-12-0/v${ENGINE_VERSION}" 1
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx2-cuda-11-7.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx2-cuda-11-7/v${ENGINE_VERSION}" 1
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-noavx-cuda-12-0.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-noavx-cuda-12-0/v${ENGINE_VERSION}" 1
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-noavx-cuda-11-7.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-noavx-cuda-11-7/v${ENGINE_VERSION}" 1
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-vulkan.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-vulkan/v${ENGINE_VERSION}" 1
|
||||
download "${CUDA_DOWNLOAD_URL}/cuda-12-0-linux-amd64.tar.gz" -e --strip 1 -o "${BIN_PATH}" 1
|
||||
download "${CUDA_DOWNLOAD_URL}/cuda-11-7-linux-amd64.tar.gz" -e --strip 1 -o "${BIN_PATH}" 1
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-noavx-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-noavx-x64/${ENGINE_VERSION}" 1
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-avx-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-avx-x64/${ENGINE_VERSION}" 1
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-avx2-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-avx2-x64/${ENGINE_VERSION}" 1
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-avx512-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-avx512-x64/${ENGINE_VERSION}" 1
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-avx2-cuda-cu12.0-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-avx2-cuda-cu12.0-x64/${ENGINE_VERSION}" 1
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-avx2-cuda-cu11.7-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-avx2-cuda-cu11.7-x64/${ENGINE_VERSION}" 1
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-noavx-cuda-cu12.0-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-noavx-cuda-cu12.0-x64/${ENGINE_VERSION}" 1
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-noavx-cuda-cu11.7-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-noavx-cuda-cu11.7-x64/${ENGINE_VERSION}" 1
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-vulkan-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/linux-vulkan-x64/${ENGINE_VERSION}" 1
|
||||
download "${CUDA_DOWNLOAD_URL}/cudart-llama-bin-linux-cu12.0-x64.tar.gz" -e --strip 1 -o "${BIN_PATH}" 1
|
||||
download "${CUDA_DOWNLOAD_URL}/cudart-llama-bin-linux-cu11.7-x64.tar.gz" -e --strip 1 -o "${BIN_PATH}" 1
|
||||
|
||||
elif [ "$OS_TYPE" == "Darwin" ]; then
|
||||
# macOS downloads
|
||||
@ -41,8 +41,8 @@ elif [ "$OS_TYPE" == "Darwin" ]; then
|
||||
chmod +x "./bin/cortex-server"
|
||||
|
||||
# Download engines for macOS
|
||||
download "${ENGINE_DOWNLOAD_URL}-mac-arm64.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/mac-arm64/v${ENGINE_VERSION}"
|
||||
download "${ENGINE_DOWNLOAD_URL}-mac-amd64.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/mac-amd64/v${ENGINE_VERSION}"
|
||||
download "${ENGINE_DOWNLOAD_URL}-macos-arm64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/macos-arm64/${ENGINE_VERSION}"
|
||||
download "${ENGINE_DOWNLOAD_URL}-macos-x64.tar.gz" -e --strip 2 -o "${SHARED_PATH}/engines/llama.cpp/macos-x64/${ENGINE_VERSION}"
|
||||
|
||||
else
|
||||
echo "Unsupported operating system: $OS_TYPE"
|
||||
|
||||
@ -19,7 +19,7 @@ export default defineConfig([
|
||||
CORTEX_SOCKET_URL: JSON.stringify(
|
||||
`ws://127.0.0.1:${process.env.CORTEX_API_PORT ?? '39291'}`
|
||||
),
|
||||
CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.56'),
|
||||
CORTEX_ENGINE_VERSION: JSON.stringify('b5350'),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
@ -32,7 +32,7 @@ rand = "0.8"
|
||||
tauri-plugin-http = { version = "2", features = ["unsafe-headers"] }
|
||||
tauri-plugin-store = "2"
|
||||
hyper = { version = "0.14", features = ["server"] }
|
||||
reqwest = { version = "0.11", features = ["json"] }
|
||||
reqwest = { version = "0.11", features = ["json", "blocking"] }
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
rmcp = { git = "https://github.com/modelcontextprotocol/rust-sdk", branch = "main", features = [
|
||||
"client",
|
||||
|
||||
@ -1,8 +1,9 @@
|
||||
@echo off
|
||||
|
||||
set CORTEX_VERSION=1.0.13-rc1
|
||||
set ENGINE_VERSION=0.1.55
|
||||
set ENGINE_DOWNLOAD_URL=https://github.com/menloresearch/cortex.llamacpp/releases/download/v%ENGINE_VERSION%/cortex.llamacpp-%ENGINE_VERSION%-windows-amd64
|
||||
set CORTEX_VERSION=1.0.13-rc6
|
||||
set ENGINE_VERSION=b5350
|
||||
set ENGINE_DOWNLOAD_URL=https://github.com/menloresearch/llama.cpp/releases/download/%ENGINE_VERSION%/llama-%ENGINE_VERSION%-bin-win
|
||||
set ENGINE_DOWNLOAD_GGML_URL=https://github.com/ggml-org/llama.cpp/releases/download/%ENGINE_VERSION%/llama-%ENGINE_VERSION%-bin-win
|
||||
set CUDA_DOWNLOAD_URL=https://github.com/menloresearch/cortex.llamacpp/releases/download/v%ENGINE_VERSION%
|
||||
set SUBFOLDERS=windows-amd64-noavx-cuda-12-0 windows-amd64-noavx-cuda-11-7 windows-amd64-avx2-cuda-12-0 windows-amd64-avx2-cuda-11-7 windows-amd64-noavx windows-amd64-avx windows-amd64-avx2 windows-amd64-avx512 windows-amd64-vulkan
|
||||
set BIN_PATH="./"
|
||||
@ -11,32 +12,30 @@ set DOWNLOAD_TOOL=..\..\extensions\inference-cortex-extension\node_modules\.bin\
|
||||
@REM Download cortex.llamacpp binaries
|
||||
|
||||
call %DOWNLOAD_TOOL% -e --strip 1 -o %BIN_PATH% https://github.com/menloresearch/cortex.cpp/releases/download/v%CORTEX_VERSION%/cortex-%CORTEX_VERSION%-windows-amd64.tar.gz
|
||||
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-avx2-cuda-12-0.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-avx2-cuda-12-0/v%ENGINE_VERSION%
|
||||
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-avx2-cuda-11-7.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-avx2-cuda-11-7/v%ENGINE_VERSION%
|
||||
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-noavx-cuda-12-0.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-noavx-cuda-12-0/v%ENGINE_VERSION%
|
||||
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-noavx-cuda-11-7.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-noavx-cuda-11-7/v%ENGINE_VERSION%
|
||||
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-noavx.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-noavx/v%ENGINE_VERSION%
|
||||
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-avx.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-avx/v%ENGINE_VERSION%
|
||||
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-avx2.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-avx2/v%ENGINE_VERSION%
|
||||
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-avx512.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-avx512/v%ENGINE_VERSION%
|
||||
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-vulkan.tar.gz -e --strip 1 -o ./engines/cortex.llamacpp/windows-amd64-vulkan/v%ENGINE_VERSION%
|
||||
call %DOWNLOAD_TOOL% %CUDA_DOWNLOAD_URL%/cuda-12-0-windows-amd64.tar.gz -e --strip 1 -o %BIN_PATH%
|
||||
call %DOWNLOAD_TOOL% %CUDA_DOWNLOAD_URL%/cuda-11-7-windows-amd64.tar.gz -e --strip 1 -o %BIN_PATH%
|
||||
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-avx2-cuda-cu12.0-x64.tar.gz -e --strip 2 -o./engines/llama.cpp/win-avx2-cuda-cu12.0-x64/%ENGINE_VERSION%
|
||||
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-avx2-cuda-cu11.7-x64.tar.gz -e --strip 2 -o./engines/llama.cpp/win-avx2-cuda-cu11.7-x64/%ENGINE_VERSION%
|
||||
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-noavx-cuda-cu12.0-x64.tar.gz -e --strip 2 -o./engines/llama.cpp/win-noavx-cuda-cu12.0-x64/%ENGINE_VERSION%
|
||||
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_URL%-noavx-cuda-cu11.7-x64.tar.gz -e --strip 2 -o./engines/llama.cpp/win-noavx-cuda-cu11.7-x64/%ENGINE_VERSION%
|
||||
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_GGML_URL%-noavx-x64.zip -e --strip 1 -o./engines/llama.cpp/win-noavx-x64/%ENGINE_VERSION%
|
||||
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_GGML_URL%-avx-x64.zip -e --strip 1 -o./engines/llama.cpp/win-avx-x64/%ENGINE_VERSION%
|
||||
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_GGML_URL%-avx2-x64.zip -e --strip 1 -o./engines/llama.cpp/win-avx2-x64/%ENGINE_VERSION%
|
||||
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_GGML_URL%-avx512-x64.zip -e --strip 1 -o./engines/llama.cpp/win-avx512-x64/%ENGINE_VERSION%
|
||||
call %DOWNLOAD_TOOL% %ENGINE_DOWNLOAD_GGML_URL%-vulkan-x64.zip -e --strip 1 -o./engines/llama.cpp/win-vulkan-x64/%ENGINE_VERSION%
|
||||
call %DOWNLOAD_TOOL% %CUDA_DOWNLOAD_URL%/cudart-llama-bin-win-cu12.0-x64.tar.gz -e --strip 1 -o %BIN_PATH%
|
||||
call %DOWNLOAD_TOOL% %CUDA_DOWNLOAD_URL%/cudart-llama-bin-win-cu11.7-x64.tar.gz -e --strip 1 -o %BIN_PATH%
|
||||
|
||||
move %BIN_PATH%cortex-server-beta.exe %BIN_PATH%cortex-server.exe
|
||||
copy %BIN_PATH%cortex-server.exe %BIN_PATH%cortex-server-x86_64-pc-windows-msvc.exe
|
||||
del %BIN_PATH%cortex-beta.exe
|
||||
del %BIN_PATH%cortex.exe
|
||||
|
||||
@REM Loop through each folder and move DLLs (excluding engine.dll)
|
||||
@REM Loop through each folder and move DLLs
|
||||
for %%F in (%SUBFOLDERS%) do (
|
||||
echo Processing folder: .\engines\cortex.llamacpp\%%F\v%ENGINE_VERSION%
|
||||
echo Processing folder: .\engines\llama.cpp\%%F\%ENGINE_VERSION%
|
||||
|
||||
@REM Move all .dll files except engine.dll
|
||||
for %%D in (.\engines\cortex.llamacpp\%%F\v%ENGINE_VERSION%\*.dll) do (
|
||||
if /I not "%%~nxD"=="engine.dll" (
|
||||
move "%%D" "%BIN_PATH%"
|
||||
)
|
||||
@REM Move cu*.dll files
|
||||
for %%D in (.\engines\engines\llama.cpp\%%F\%ENGINE_VERSION%\cu*.dll) do (
|
||||
move "%%D" "%BIN_PATH%"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@ -2,23 +2,23 @@
|
||||
|
||||
download() {
|
||||
URL="$1"
|
||||
EXTRA_ARGS="${@:2}"
|
||||
EXTRA_ARGS="${@:3}"
|
||||
OUTPUT_DIR="${EXTRA_ARGS[${#EXTRA_ARGS[@]} -1]}"
|
||||
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
echo "Downloading $URL to $OUTPUT_DIR using curl..."
|
||||
curl -L "$URL" -o "$OUTPUT_DIR/$(basename "$URL")"
|
||||
tar -xzf "$OUTPUT_DIR/$(basename "$URL")" -C "$OUTPUT_DIR" --strip-components 1
|
||||
tar -xzf "$OUTPUT_DIR/$(basename "$URL")" -C "$OUTPUT_DIR" --strip-components $2
|
||||
rm "$OUTPUT_DIR/$(basename "$URL")"
|
||||
}
|
||||
|
||||
# Read CORTEX_VERSION
|
||||
CORTEX_VERSION=1.0.13-rc1
|
||||
ENGINE_VERSION=0.1.55
|
||||
CORTEX_VERSION=1.0.13-rc6
|
||||
ENGINE_VERSION=b5350
|
||||
CORTEX_RELEASE_URL="https://github.com/menloresearch/cortex.cpp/releases/download"
|
||||
ENGINE_DOWNLOAD_URL="https://github.com/menloresearch/cortex.llamacpp/releases/download/v${ENGINE_VERSION}/cortex.llamacpp-${ENGINE_VERSION}"
|
||||
CUDA_DOWNLOAD_URL="https://github.com/menloresearch/cortex.llamacpp/releases/download/v${ENGINE_VERSION}"
|
||||
ENGINE_DOWNLOAD_URL=https://github.com/menloresearch/llama.cpp/releases/download/${ENGINE_VERSION}/llama-${ENGINE_VERSION}-bin
|
||||
CUDA_DOWNLOAD_URL=https://github.com/menloresearch/llama.cpp/releases/download/${ENGINE_VERSION}
|
||||
BIN_PATH=./
|
||||
SHARED_PATH="."
|
||||
# Detect platform
|
||||
@ -31,38 +31,41 @@ fi
|
||||
|
||||
if [ "$OS_TYPE" == "Linux" ]; then
|
||||
# Linux downloads
|
||||
download "${CORTEX_RELEASE_URL}/v${CORTEX_VERSION}/cortex-${CORTEX_VERSION}-linux-amd64.tar.gz" "${BIN_PATH}"
|
||||
mv .cortex-server-beta ./cortex-server
|
||||
download "${CORTEX_RELEASE_URL}/v${CORTEX_VERSION}/cortex-${CORTEX_VERSION}-linux-amd64.tar.gz" 1 "${BIN_PATH}"
|
||||
mv ./cortex-server-beta ./cortex-server
|
||||
rm -rf ./cortex
|
||||
rm -rf ./cortex-beta
|
||||
chmod +x "./cortex-server"
|
||||
cp ./cortex-server ./cortex-server-x86_64-unknown-linux-gnu
|
||||
|
||||
# Download engines for Linux
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-noavx.tar.gz" "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-noavx/v${ENGINE_VERSION}"
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx.tar.gz" "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx/v${ENGINE_VERSION}"
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx2.tar.gz" "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx2/v${ENGINE_VERSION}"
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx512.tar.gz" "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx512/v${ENGINE_VERSION}"
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx2-cuda-12-0.tar.gz" "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx2-cuda-12-0/v${ENGINE_VERSION}"
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-avx2-cuda-11-7.tar.gz" "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-avx2-cuda-11-7/v${ENGINE_VERSION}"
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-noavx-cuda-12-0.tar.gz" "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-noavx-cuda-12-0/v${ENGINE_VERSION}"
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-noavx-cuda-11-7.tar.gz" "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-noavx-cuda-11-7/v${ENGINE_VERSION}"
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-amd64-vulkan.tar.gz" "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-vulkan/v${ENGINE_VERSION}"
|
||||
download "${CUDA_DOWNLOAD_URL}/cuda-12-0-linux-amd64.tar.gz" "${BIN_PATH}"
|
||||
download "${CUDA_DOWNLOAD_URL}/cuda-11-7-linux-amd64.tar.gz" "${BIN_PATH}"
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-noavx-x64.tar.gz" 2 "${SHARED_PATH}/engines/llama.cpp/linux-noavx-x64/${ENGINE_VERSION}"
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-avx-x64.tar.gz" 2 "${SHARED_PATH}/engines/llama.cpp/linux-avx-x64/${ENGINE_VERSION}"
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-avx2-x64.tar.gz" 2 "${SHARED_PATH}/engines/llama.cpp/linux-avx2-x64/${ENGINE_VERSION}"
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-avx512-x64.tar.gz" 2 "${SHARED_PATH}/engines/llama.cpp/linux-avx512-x64/${ENGINE_VERSION}"
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-avx2-cuda-cu12.0-x64.tar.gz" 2 "${SHARED_PATH}/engines/llama.cpp/linux-avx2-cuda-cu12.0-x64/${ENGINE_VERSION}"
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-avx2-cuda-cu11.7-x64.tar.gz" 2 "${SHARED_PATH}/engines/llama.cpp/linux-avx2-cuda-cu11.7-x64/${ENGINE_VERSION}"
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-noavx-cuda-cu12.0-x64.tar.gz" 2 "${SHARED_PATH}/engines/llama.cpp/linux-noavx-cuda-cu12.0-x64/${ENGINE_VERSION}"
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-noavx-cuda-cu11.7-x64.tar.gz" 2 "${SHARED_PATH}/engines/llama.cpp/linux-noavx-cuda-cu11.7-x64/${ENGINE_VERSION}"
|
||||
download "${ENGINE_DOWNLOAD_URL}-linux-vulkan-x64.tar.gz" 2 "${SHARED_PATH}/engines/llama.cpp/linux-vulkan-x64/${ENGINE_VERSION}"
|
||||
download "${CUDA_DOWNLOAD_URL}/cudart-llama-bin-linux-cu12.0-x64.tar.gz" 0 "${BIN_PATH}"
|
||||
download "${CUDA_DOWNLOAD_URL}/cudart-llama-bin-linux-cu11.7-x64.tar.gz" 0 "${BIN_PATH}"
|
||||
|
||||
elif [ "$OS_TYPE" == "Darwin" ]; then
|
||||
# macOS downloads
|
||||
download "${CORTEX_RELEASE_URL}/v${CORTEX_VERSION}/cortex-${CORTEX_VERSION}-mac-universal.tar.gz" "${BIN_PATH}"
|
||||
download "${CORTEX_RELEASE_URL}/v${CORTEX_VERSION}/cortex-${CORTEX_VERSION}-mac-universal.tar.gz" 1 "${BIN_PATH}"
|
||||
mv ./cortex-server-beta ./cortex-server
|
||||
rm -rf ./cortex
|
||||
rm -rf ./cortex-beta
|
||||
chmod +x "./cortex-server"
|
||||
mv ./cortex-server ./cortex-server-universal-apple-darwin
|
||||
cp ./cortex-server-universal-apple-darwin ./cortex-server-aarch64-apple-darwin
|
||||
cp ./cortex-server-universal-apple-darwin ./cortex-server-x86_64-apple-darwin
|
||||
|
||||
# Download engines for macOS
|
||||
download "${ENGINE_DOWNLOAD_URL}-mac-arm64.tar.gz" "${SHARED_PATH}/engines/cortex.llamacpp/mac-arm64/v${ENGINE_VERSION}"
|
||||
download "${ENGINE_DOWNLOAD_URL}-mac-amd64.tar.gz" "${SHARED_PATH}/engines/cortex.llamacpp/mac-amd64/v${ENGINE_VERSION}"
|
||||
download "${ENGINE_DOWNLOAD_URL}-macos-arm64.tar.gz" 2 "${SHARED_PATH}/engines/llama.cpp/macos-arm64/${ENGINE_VERSION}"
|
||||
download "${ENGINE_DOWNLOAD_URL}-macos-x64.tar.gz" 2 "${SHARED_PATH}/engines/llama.cpp/macos-x64/${ENGINE_VERSION}"
|
||||
|
||||
|
||||
else
|
||||
echo "Unsupported operating system: $OS_TYPE"
|
||||
|
||||
@ -9,6 +9,8 @@ use std::{collections::HashMap, sync::Arc};
|
||||
use tauri::Emitter;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use reqwest::blocking::Client;
|
||||
|
||||
#[cfg_attr(mobile, tauri::mobile_entry_point)]
|
||||
pub fn run() {
|
||||
tauri::Builder::default()
|
||||
@ -86,6 +88,10 @@ pub fn run() {
|
||||
})
|
||||
.on_window_event(|window, event| match event {
|
||||
tauri::WindowEvent::CloseRequested { .. } => {
|
||||
let client = Client::new();
|
||||
let url = "http://127.0.0.1:39291/processManager/destroy";
|
||||
let _ = client.delete(url).send();
|
||||
|
||||
window.emit("kill-sidecar", ()).unwrap();
|
||||
}
|
||||
_ => {}
|
||||
|
||||
@ -571,4 +571,4 @@ export default function useSendChatMessage(
|
||||
sendChatMessage,
|
||||
resendChatMessage,
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user