From 821036945fcd5857dc222f22462745318a944889 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 20 Mar 2025 19:00:29 +0700 Subject: [PATCH] feat: Cortex API Authorization --- .../assistant-extension/rolldown.config.mjs | 1 + .../src/@types/global.d.ts | 1 + .../assistant-extension/src/node/retrieval.ts | 4 +- .../rolldown.config.mjs | 1 + .../src/@types/global.d.ts | 1 + .../conversational-extension/src/index.ts | 51 ++-- .../rolldown.config.mjs | 1 + .../src/@types/global.d.ts | 1 + .../engine-management-extension/src/index.ts | 82 ++++--- .../package.json | 2 - .../rolldown.config.mjs | 1 + .../src/@types/global.d.ts | 1 + .../src/index.ts | 21 +- .../bin/version.txt | 2 +- .../inference-cortex-extension/download.bat | 6 +- .../inference-cortex-extension/download.sh | 6 +- .../rolldown.config.mjs | 4 +- .../src/@types/global.d.ts | 1 + .../inference-cortex-extension/src/index.ts | 38 ++- .../src/node/index.ts | 5 +- .../model-extension/rolldown.config.mjs | 3 +- .../model-extension/src/@types/global.d.ts | 3 +- extensions/model-extension/src/index.ts | 61 +++-- server/cortex.json | 232 ------------------ server/global.d.ts | 3 +- server/index.ts | 36 ++- server/rolldown.config.mjs | 1 + 27 files changed, 221 insertions(+), 348 deletions(-) diff --git a/extensions/assistant-extension/rolldown.config.mjs b/extensions/assistant-extension/rolldown.config.mjs index e549ea7d9..b38e07ffb 100644 --- a/extensions/assistant-extension/rolldown.config.mjs +++ b/extensions/assistant-extension/rolldown.config.mjs @@ -28,6 +28,7 @@ export default defineConfig([ }, define: { CORTEX_API_URL: JSON.stringify(`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? "39291"}`), + CORTEX_API_KEY: JSON.stringify(process.env.CORTEX_API_KEY ?? 'cortexserver'), }, platform: 'node', }, diff --git a/extensions/assistant-extension/src/@types/global.d.ts b/extensions/assistant-extension/src/@types/global.d.ts index b724db8d0..697fee034 100644 --- a/extensions/assistant-extension/src/@types/global.d.ts +++ b/extensions/assistant-extension/src/@types/global.d.ts @@ -1,3 +1,4 @@ declare const NODE: string declare const VERSION: string declare const CORTEX_API_URL: string +declare const CORTEX_API_KEY: string diff --git a/extensions/assistant-extension/src/node/retrieval.ts b/extensions/assistant-extension/src/node/retrieval.ts index 2db2bb4fb..6e2951b00 100644 --- a/extensions/assistant-extension/src/node/retrieval.ts +++ b/extensions/assistant-extension/src/node/retrieval.ts @@ -27,7 +27,7 @@ export class Retrieval { // declare time-weighted retriever and storage this.timeWeightedVectorStore = new MemoryVectorStore( new OpenAIEmbeddings( - { openAIApiKey: 'cortex-embedding' }, + { openAIApiKey: CORTEX_API_KEY }, { basePath: `${CORTEX_API_URL}/v1` } ) ) @@ -49,7 +49,7 @@ export class Retrieval { public updateEmbeddingEngine(model: string, engine: string): void { this.embeddingModel = new OpenAIEmbeddings( - { openAIApiKey: 'cortex-embedding', model }, + { openAIApiKey: CORTEX_API_KEY, model }, // TODO: Raw settings { basePath: `${CORTEX_API_URL}/v1` } ) diff --git a/extensions/conversational-extension/rolldown.config.mjs b/extensions/conversational-extension/rolldown.config.mjs index 6d396f611..b2b97e7e4 100644 --- a/extensions/conversational-extension/rolldown.config.mjs +++ b/extensions/conversational-extension/rolldown.config.mjs @@ -9,5 +9,6 @@ export default defineConfig({ platform: 'browser', define: { API_URL: JSON.stringify(`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? "39291"}`), + CORTEX_API_KEY: JSON.stringify(process.env.CORTEX_API_KEY ?? 'cortexserver'), }, }) diff --git a/extensions/conversational-extension/src/@types/global.d.ts b/extensions/conversational-extension/src/@types/global.d.ts index abe60d318..f4d5e3eb7 100644 --- a/extensions/conversational-extension/src/@types/global.d.ts +++ b/extensions/conversational-extension/src/@types/global.d.ts @@ -1,4 +1,5 @@ declare const API_URL: string +declare const CORTEX_API_KEY: string interface Core { api: APIFunctions diff --git a/extensions/conversational-extension/src/index.ts b/extensions/conversational-extension/src/index.ts index eeb4fcf38..de4e6b180 100644 --- a/extensions/conversational-extension/src/index.ts +++ b/extensions/conversational-extension/src/index.ts @@ -22,6 +22,17 @@ type MessageList = { export default class CortexConversationalExtension extends ConversationalExtension { queue = new PQueue({ concurrency: 1 }) + /** + * Extended API instance for making requests to the Cortex API. + * @returns + */ + api = () => + ky.extend({ + prefixUrl: API_URL, + headers: { + Authorization: `Bearer ${CORTEX_API_KEY}`, + }, + }) /** * Called when the extension is loaded. */ @@ -39,8 +50,8 @@ export default class CortexConversationalExtension extends ConversationalExtensi */ async listThreads(): Promise { return this.queue.add(() => - ky - .get(`${API_URL}/v1/threads?limit=-1`) + this.api() + .get('v1/threads?limit=-1') .json() .then((e) => e.data) ) as Promise @@ -52,7 +63,7 @@ export default class CortexConversationalExtension extends ConversationalExtensi */ async createThread(thread: Thread): Promise { return this.queue.add(() => - ky.post(`${API_URL}/v1/threads`, { json: thread }).json() + this.api().post('v1/threads', { json: thread }).json() ) as Promise } @@ -63,7 +74,7 @@ export default class CortexConversationalExtension extends ConversationalExtensi async modifyThread(thread: Thread): Promise { return this.queue .add(() => - ky.patch(`${API_URL}/v1/threads/${thread.id}`, { json: thread }) + this.api().patch(`v1/threads/${thread.id}`, { json: thread }) ) .then() } @@ -74,7 +85,7 @@ export default class CortexConversationalExtension extends ConversationalExtensi */ async deleteThread(threadId: string): Promise { return this.queue - .add(() => ky.delete(`${API_URL}/v1/threads/${threadId}`)) + .add(() => this.api().delete(`v1/threads/${threadId}`)) .then() } @@ -85,8 +96,8 @@ export default class CortexConversationalExtension extends ConversationalExtensi */ async createMessage(message: ThreadMessage): Promise { return this.queue.add(() => - ky - .post(`${API_URL}/v1/threads/${message.thread_id}/messages`, { + this.api() + .post(`v1/threads/${message.thread_id}/messages`, { json: message, }) .json() @@ -100,9 +111,9 @@ export default class CortexConversationalExtension extends ConversationalExtensi */ async modifyMessage(message: ThreadMessage): Promise { return this.queue.add(() => - ky + this.api() .patch( - `${API_URL}/v1/threads/${message.thread_id}/messages/${message.id}`, + `v1/threads/${message.thread_id}/messages/${message.id}`, { json: message, } @@ -120,7 +131,7 @@ export default class CortexConversationalExtension extends ConversationalExtensi async deleteMessage(threadId: string, messageId: string): Promise { return this.queue .add(() => - ky.delete(`${API_URL}/v1/threads/${threadId}/messages/${messageId}`) + this.api().delete(`v1/threads/${threadId}/messages/${messageId}`) ) .then() } @@ -132,8 +143,8 @@ export default class CortexConversationalExtension extends ConversationalExtensi */ async listMessages(threadId: string): Promise { return this.queue.add(() => - ky - .get(`${API_URL}/v1/threads/${threadId}/messages?order=asc&limit=-1`) + this.api() + .get(`v1/threads/${threadId}/messages?order=asc&limit=-1`) .json() .then((e) => e.data) ) as Promise @@ -147,8 +158,8 @@ export default class CortexConversationalExtension extends ConversationalExtensi */ async getThreadAssistant(threadId: string): Promise { return this.queue.add(() => - ky - .get(`${API_URL}/v1/assistants/${threadId}?limit=-1`) + this.api() + .get(`v1/assistants/${threadId}?limit=-1`) .json() ) as Promise } @@ -163,8 +174,8 @@ export default class CortexConversationalExtension extends ConversationalExtensi assistant: ThreadAssistantInfo ): Promise { return this.queue.add(() => - ky - .post(`${API_URL}/v1/assistants/${threadId}`, { json: assistant }) + this.api() + .post(`v1/assistants/${threadId}`, { json: assistant }) .json() ) as Promise } @@ -180,8 +191,8 @@ export default class CortexConversationalExtension extends ConversationalExtensi assistant: ThreadAssistantInfo ): Promise { return this.queue.add(() => - ky - .patch(`${API_URL}/v1/assistants/${threadId}`, { json: assistant }) + this.api() + .patch(`v1/assistants/${threadId}`, { json: assistant }) .json() ) as Promise } @@ -191,8 +202,8 @@ export default class CortexConversationalExtension extends ConversationalExtensi * @returns */ async healthz(): Promise { - return ky - .get(`${API_URL}/healthz`, { + return this.api() + .get('healthz', { retry: { limit: 20, delay: () => 500, methods: ['get'] }, }) .then(() => {}) diff --git a/extensions/engine-management-extension/rolldown.config.mjs b/extensions/engine-management-extension/rolldown.config.mjs index 02b84b363..a7e529215 100644 --- a/extensions/engine-management-extension/rolldown.config.mjs +++ b/extensions/engine-management-extension/rolldown.config.mjs @@ -28,6 +28,7 @@ export default defineConfig([ 'Authorization: Bearer {{api_key}}' ), VERSION: JSON.stringify(pkgJson.version ?? '0.0.0'), + CORTEX_API_KEY: JSON.stringify(process.env.CORTEX_API_KEY ?? 'cortexserver'), }, }, { diff --git a/extensions/engine-management-extension/src/@types/global.d.ts b/extensions/engine-management-extension/src/@types/global.d.ts index 0dbed3806..ef66c328e 100644 --- a/extensions/engine-management-extension/src/@types/global.d.ts +++ b/extensions/engine-management-extension/src/@types/global.d.ts @@ -6,6 +6,7 @@ declare const DEFAULT_REQUEST_PAYLOAD_TRANSFORM: string declare const DEFAULT_RESPONSE_BODY_TRANSFORM: string declare const DEFAULT_REQUEST_HEADERS_TRANSFORM: string declare const VERSION: string +declare const CORTEX_API_KEY: string declare const DEFAULT_REMOTE_ENGINES: ({ id: string diff --git a/extensions/engine-management-extension/src/index.ts b/extensions/engine-management-extension/src/index.ts index 1a5b004f7..b617673c8 100644 --- a/extensions/engine-management-extension/src/index.ts +++ b/extensions/engine-management-extension/src/index.ts @@ -31,6 +31,17 @@ interface ModelList { export default class JanEngineManagementExtension extends EngineManagementExtension { queue = new PQueue({ concurrency: 1 }) + /** + * Extended API instance for making requests to the Cortex API. + * @returns + */ + api = () => + ky.extend({ + prefixUrl: API_URL, + headers: { + Authorization: `Bearer ${CORTEX_API_KEY}`, + }, + }) /** * Called when the extension is loaded. */ @@ -59,8 +70,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens */ async getEngines(): Promise { return this.queue.add(() => - ky - .get(`${API_URL}/v1/engines`) + this.api() + .get('v1/engines') .json() .then((e) => e) ) as Promise @@ -70,8 +81,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens * @returns A Promise that resolves to an object of list engines. */ async getRemoteModels(name: string): Promise { - return ky - .get(`${API_URL}/v1/models/remote/${name}`) + return this.api() + .get(`v1/models/remote/${name}`) .json() .catch(() => ({ data: [], @@ -84,8 +95,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens */ async getInstalledEngines(name: InferenceEngine): Promise { return this.queue.add(() => - ky - .get(`${API_URL}/v1/engines/${name}`) + this.api() + .get(`v1/engines/${name}`) .json() .then((e) => e) ) as Promise @@ -103,8 +114,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens platform?: string ) { return this.queue.add(() => - ky - .get(`${API_URL}/v1/engines/${name}/releases/${version}`) + this.api() + .get(`v1/engines/${name}/releases/${version}`) .json() .then((e) => platform ? e.filter((r) => r.name.includes(platform)) : e @@ -119,8 +130,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens */ async getLatestReleasedEngine(name: InferenceEngine, platform?: string) { return this.queue.add(() => - ky - .get(`${API_URL}/v1/engines/${name}/releases/latest`) + this.api() + .get(`v1/engines/${name}/releases/latest`) .json() .then((e) => platform ? e.filter((r) => r.name.includes(platform)) : e @@ -134,8 +145,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens */ async installEngine(name: string, engineConfig: EngineConfig) { return this.queue.add(() => - ky - .post(`${API_URL}/v1/engines/${name}/install`, { json: engineConfig }) + this.api() + .post(`v1/engines/${name}/install`, { json: engineConfig }) .then((e) => e) ) as Promise<{ messages: string }> } @@ -167,15 +178,17 @@ export default class JanEngineManagementExtension extends EngineManagementExtens engineConfig.metadata.header_template = DEFAULT_REQUEST_HEADERS_TRANSFORM return this.queue.add(() => - ky.post(`${API_URL}/v1/engines`, { json: engineConfig }).then((e) => { - if (persistModels && engineConfig.metadata?.get_models_url) { - // Pull /models from remote models endpoint - return this.populateRemoteModels(engineConfig) - .then(() => e) - .catch(() => e) - } - return e - }) + this.api() + .post('v1/engines', { json: engineConfig }) + .then((e) => { + if (persistModels && engineConfig.metadata?.get_models_url) { + // Pull /models from remote models endpoint + return this.populateRemoteModels(engineConfig) + .then(() => e) + .catch(() => e) + } + return e + }) ) as Promise<{ messages: string }> } @@ -185,8 +198,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens */ async uninstallEngine(name: InferenceEngine, engineConfig: EngineConfig) { return this.queue.add(() => - ky - .delete(`${API_URL}/v1/engines/${name}/install`, { json: engineConfig }) + this.api() + .delete(`v1/engines/${name}/install`, { json: engineConfig }) .then((e) => e) ) as Promise<{ messages: string }> } @@ -198,8 +211,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens async addRemoteModel(model: Model) { return this.queue .add(() => - ky - .post(`${API_URL}/v1/models/add`, { + this.api() + .post('v1/models/add', { json: { inference_params: { max_tokens: 4096, @@ -223,8 +236,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens */ async getDefaultEngineVariant(name: InferenceEngine) { return this.queue.add(() => - ky - .get(`${API_URL}/v1/engines/${name}/default`) + this.api() + .get(`v1/engines/${name}/default`) .json<{ messages: string }>() .then((e) => e) ) as Promise @@ -240,8 +253,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens engineConfig: EngineConfig ) { return this.queue.add(() => - ky - .post(`${API_URL}/v1/engines/${name}/default`, { json: engineConfig }) + this.api() + .post(`v1/engines/${name}/default`, { json: engineConfig }) .then((e) => e) ) as Promise<{ messages: string }> } @@ -251,8 +264,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens */ async updateEngine(name: InferenceEngine, engineConfig?: EngineConfig) { return this.queue.add(() => - ky - .post(`${API_URL}/v1/engines/${name}/update`, { json: engineConfig }) + this.api() + .post(`v1/engines/${name}/update`, { json: engineConfig }) .then((e) => e) ) as Promise<{ messages: string }> } @@ -262,8 +275,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens * @returns */ async healthz(): Promise { - return ky - .get(`${API_URL}/healthz`, { + return this.api() + .get('healthz', { retry: { limit: 20, delay: () => 500, methods: ['get'] }, }) .then(() => { @@ -390,7 +403,6 @@ export default class JanEngineManagementExtension extends EngineManagementExtens const version = await this.getSetting('version', '0.0.0') const engines = await this.getEngines() if (version < VERSION) { - console.log('Migrating engine settings...') // Migrate engine settings await Promise.all( @@ -398,7 +410,7 @@ export default class JanEngineManagementExtension extends EngineManagementExtens const { id, ...data } = engine data.api_key = engines[id]?.api_key - return this.updateEngine(id,{ + return this.updateEngine(id, { ...data, }).catch(console.error) }) diff --git a/extensions/hardware-management-extension/package.json b/extensions/hardware-management-extension/package.json index ec98c7440..396404df9 100644 --- a/extensions/hardware-management-extension/package.json +++ b/extensions/hardware-management-extension/package.json @@ -29,12 +29,10 @@ }, "dependencies": { "@janhq/core": "../../core/package.tgz", - "cpu-instructions": "^0.0.13", "ky": "^1.7.2", "p-queue": "^8.0.1" }, "bundledDependencies": [ - "cpu-instructions", "@janhq/core" ], "hardwares": { diff --git a/extensions/hardware-management-extension/rolldown.config.mjs b/extensions/hardware-management-extension/rolldown.config.mjs index 1a9c34ba0..d5d8a60d9 100644 --- a/extensions/hardware-management-extension/rolldown.config.mjs +++ b/extensions/hardware-management-extension/rolldown.config.mjs @@ -11,6 +11,7 @@ export default defineConfig([ define: { NODE: JSON.stringify(`${pkgJson.name}/${pkgJson.node}`), API_URL: JSON.stringify(`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? "39291"}`), + CORTEX_API_KEY: JSON.stringify(process.env.CORTEX_API_KEY ?? 'cortexserver'), }, }, ]) diff --git a/extensions/hardware-management-extension/src/@types/global.d.ts b/extensions/hardware-management-extension/src/@types/global.d.ts index a412681e8..1ee055750 100644 --- a/extensions/hardware-management-extension/src/@types/global.d.ts +++ b/extensions/hardware-management-extension/src/@types/global.d.ts @@ -1,5 +1,6 @@ declare const API_URL: string declare const NODE: string +declare const CORTEX_API_KEY: string interface Core { api: APIFunctions diff --git a/extensions/hardware-management-extension/src/index.ts b/extensions/hardware-management-extension/src/index.ts index f64c2eea8..b81cef876 100644 --- a/extensions/hardware-management-extension/src/index.ts +++ b/extensions/hardware-management-extension/src/index.ts @@ -9,6 +9,17 @@ import PQueue from 'p-queue' export default class JSONHardwareManagementExtension extends HardwareManagementExtension { queue = new PQueue({ concurrency: 1 }) + /** + * Extended API instance for making requests to the Cortex API. + * @returns + */ + api = () => + ky.extend({ + prefixUrl: API_URL, + headers: { + 'Authorization': `Bearer ${CORTEX_API_KEY}`, + }, + }); /** * Called when the extension is loaded. */ @@ -27,8 +38,8 @@ export default class JSONHardwareManagementExtension extends HardwareManagementE * @returns */ async healthz(): Promise { - return ky - .get(`${API_URL}/healthz`, { + return this.api() + .get('healthz', { retry: { limit: 20, delay: () => 500, methods: ['get'] }, }) .then(() => {}) @@ -39,8 +50,8 @@ export default class JSONHardwareManagementExtension extends HardwareManagementE */ async getHardware(): Promise { return this.queue.add(() => - ky - .get(`${API_URL}/v1/hardware`) + this.api() + .get('v1/hardware') .json() .then((e) => e) ) as Promise @@ -54,7 +65,7 @@ export default class JSONHardwareManagementExtension extends HardwareManagementE activated_gpus: number[] }> { return this.queue.add(() => - ky.post(`${API_URL}/v1/hardware/activate`, { json: data }).then((e) => e) + this.api().post('v1/hardware/activate', { json: data }).then((e) => e) ) as Promise<{ message: string activated_gpus: number[] diff --git a/extensions/inference-cortex-extension/bin/version.txt b/extensions/inference-cortex-extension/bin/version.txt index 59e9e6049..275aae17b 100644 --- a/extensions/inference-cortex-extension/bin/version.txt +++ b/extensions/inference-cortex-extension/bin/version.txt @@ -1 +1 @@ -1.0.11 +1.0.12-rc3 \ No newline at end of file diff --git a/extensions/inference-cortex-extension/download.bat b/extensions/inference-cortex-extension/download.bat index 045bb3cf3..ca2930bdd 100644 --- a/extensions/inference-cortex-extension/download.bat +++ b/extensions/inference-cortex-extension/download.bat @@ -5,11 +5,11 @@ set /p CORTEX_VERSION=<./bin/version.txt set ENGINE_VERSION=0.1.55 @REM Download cortex.llamacpp binaries -set DOWNLOAD_URL=https://github.com/janhq/cortex.llamacpp/releases/download/v%ENGINE_VERSION%/cortex.llamacpp-%ENGINE_VERSION%-windows-amd64 -set CUDA_DOWNLOAD_URL=https://github.com/janhq/cortex.llamacpp/releases/download/v%ENGINE_VERSION% +set DOWNLOAD_URL=https://github.com/menloresearch/cortex.llamacpp/releases/download/v%ENGINE_VERSION%/cortex.llamacpp-%ENGINE_VERSION%-windows-amd64 +set CUDA_DOWNLOAD_URL=https://github.com/menloresearch/cortex.llamacpp/releases/download/v%ENGINE_VERSION% set SUBFOLDERS=windows-amd64-noavx-cuda-12-0 windows-amd64-noavx-cuda-11-7 windows-amd64-avx2-cuda-12-0 windows-amd64-avx2-cuda-11-7 windows-amd64-noavx windows-amd64-avx windows-amd64-avx2 windows-amd64-avx512 windows-amd64-vulkan -call .\node_modules\.bin\download -e --strip 1 -o %BIN_PATH% https://github.com/janhq/cortex.cpp/releases/download/v%CORTEX_VERSION%/cortex-%CORTEX_VERSION%-windows-amd64.tar.gz +call .\node_modules\.bin\download -e --strip 1 -o %BIN_PATH% https://github.com/menloresearch/cortex.cpp/releases/download/v%CORTEX_VERSION%/cortex-%CORTEX_VERSION%-windows-amd64.tar.gz call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2-cuda-12-0.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-avx2-cuda-12-0/v%ENGINE_VERSION% call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2-cuda-11-7.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-avx2-cuda-11-7/v%ENGINE_VERSION% call .\node_modules\.bin\download %DOWNLOAD_URL%-noavx-cuda-12-0.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-noavx-cuda-12-0/v%ENGINE_VERSION% diff --git a/extensions/inference-cortex-extension/download.sh b/extensions/inference-cortex-extension/download.sh index c88dffc72..3476708bb 100755 --- a/extensions/inference-cortex-extension/download.sh +++ b/extensions/inference-cortex-extension/download.sh @@ -3,9 +3,9 @@ # Read CORTEX_VERSION CORTEX_VERSION=$(cat ./bin/version.txt) ENGINE_VERSION=0.1.55 -CORTEX_RELEASE_URL="https://github.com/janhq/cortex.cpp/releases/download" -ENGINE_DOWNLOAD_URL="https://github.com/janhq/cortex.llamacpp/releases/download/v${ENGINE_VERSION}/cortex.llamacpp-${ENGINE_VERSION}" -CUDA_DOWNLOAD_URL="https://github.com/janhq/cortex.llamacpp/releases/download/v${ENGINE_VERSION}" +CORTEX_RELEASE_URL="https://github.com/menloresearch/cortex.cpp/releases/download" +ENGINE_DOWNLOAD_URL="https://github.com/menloresearch/cortex.llamacpp/releases/download/v${ENGINE_VERSION}/cortex.llamacpp-${ENGINE_VERSION}" +CUDA_DOWNLOAD_URL="https://github.com/menloresearch/cortex.llamacpp/releases/download/v${ENGINE_VERSION}" BIN_PATH=./bin SHARED_PATH="../../electron/shared" # Detect platform diff --git a/extensions/inference-cortex-extension/rolldown.config.mjs b/extensions/inference-cortex-extension/rolldown.config.mjs index 3bad30b58..0be60c9a7 100644 --- a/extensions/inference-cortex-extension/rolldown.config.mjs +++ b/extensions/inference-cortex-extension/rolldown.config.mjs @@ -19,12 +19,13 @@ export default defineConfig([ CORTEX_SOCKET_URL: JSON.stringify( `ws://127.0.0.1:${process.env.CORTEX_API_PORT ?? '39291'}` ), + CORTEX_API_KEY: JSON.stringify(process.env.CORTEX_API_KEY ?? 'cortexserver'), CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.55'), }, }, { input: 'src/node/index.ts', - external: ['@janhq/core/node', 'cpu-instructions'], + external: ['@janhq/core/node'], output: { format: 'cjs', file: 'dist/node/index.cjs.js', @@ -35,6 +36,7 @@ export default defineConfig([ extensions: ['.js', '.ts', '.json'], }, define: { + CORTEX_API_KEY: JSON.stringify(process.env.CORTEX_API_KEY ?? 'cortexserver'), CORTEX_API_URL: JSON.stringify( `http://127.0.0.1:${process.env.CORTEX_API_PORT ?? '39291'}` ), diff --git a/extensions/inference-cortex-extension/src/@types/global.d.ts b/extensions/inference-cortex-extension/src/@types/global.d.ts index 52f97b9ab..2f942495f 100644 --- a/extensions/inference-cortex-extension/src/@types/global.d.ts +++ b/extensions/inference-cortex-extension/src/@types/global.d.ts @@ -3,3 +3,4 @@ declare const CORTEX_API_URL: string declare const CORTEX_SOCKET_URL: string declare const CORTEX_ENGINE_VERSION: string declare const SETTINGS: any +declare const CORTEX_API_KEY: string diff --git a/extensions/inference-cortex-extension/src/index.ts b/extensions/inference-cortex-extension/src/index.ts index 5e1adf322..568ab6c96 100644 --- a/extensions/inference-cortex-extension/src/index.ts +++ b/extensions/inference-cortex-extension/src/index.ts @@ -75,6 +75,28 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine { abortControllers = new Map() + /** + * Extended API instance for making requests to the Cortex API. + * @returns + */ + api = () => + ky.extend({ + prefixUrl: CORTEX_API_URL, + headers: { + Authorization: `Bearer ${CORTEX_API_KEY}`, + }, + }) + + /** + * Authorization headers for the API requests. + * @returns + */ + headers(): Promise { + return Promise.resolve({ + Authorization: `Bearer ${CORTEX_API_KEY}`, + }) + } + /** * Subscribes to events emitted by the @janhq/core package. */ @@ -153,8 +175,8 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine { this.abortControllers.set(model.id, controller) return await this.queue.add(() => - ky - .post(`${CORTEX_API_URL}/v1/models/start`, { + this.api() + .post('v1/models/start', { json: { ...extractModelLoadParams(model.settings), model: model.id, @@ -183,8 +205,8 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine { } override async unloadModel(model: Model): Promise { - return ky - .post(`${CORTEX_API_URL}/v1/models/stop`, { + return this.api() + .post('v1/models/stop', { json: { model: model.id }, }) .json() @@ -199,8 +221,8 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine { * @returns */ private async healthz(): Promise { - return ky - .get(`${CORTEX_API_URL}/healthz`, { + return this.api() + .get('healthz', { retry: { limit: 20, delay: () => 500, @@ -215,8 +237,8 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine { * @returns */ private async clean(): Promise { - return ky - .delete(`${CORTEX_API_URL}/processmanager/destroy`, { + return this.api() + .delete('processmanager/destroy', { timeout: 2000, // maximum 2 seconds retry: { limit: 0, diff --git a/extensions/inference-cortex-extension/src/node/index.ts b/extensions/inference-cortex-extension/src/node/index.ts index 87a1c7096..b243b83c2 100644 --- a/extensions/inference-cortex-extension/src/node/index.ts +++ b/extensions/inference-cortex-extension/src/node/index.ts @@ -44,8 +44,9 @@ function run(): Promise { `${path.join(dataFolderPath, '.janrc')}`, '--data_folder_path', dataFolderPath, - '--loglevel', - 'INFO', + 'config', + '--api_keys', + CORTEX_API_KEY ?? 'cortexserver', ], { env: { diff --git a/extensions/model-extension/rolldown.config.mjs b/extensions/model-extension/rolldown.config.mjs index e6a0e8add..55aae387d 100644 --- a/extensions/model-extension/rolldown.config.mjs +++ b/extensions/model-extension/rolldown.config.mjs @@ -11,7 +11,8 @@ export default defineConfig({ platform: 'browser', define: { SETTINGS: JSON.stringify(settingJson), - API_URL: JSON.stringify(`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? "39291"}`), + CORTEX_API_URL: JSON.stringify(`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? "39291"}`), DEFAULT_MODEL_SOURCES: JSON.stringify(modelSources), + CORTEX_API_KEY: JSON.stringify(process.env.CORTEX_API_KEY ?? 'cortexserver'), }, }) diff --git a/extensions/model-extension/src/@types/global.d.ts b/extensions/model-extension/src/@types/global.d.ts index 27b23fa30..a623300e2 100644 --- a/extensions/model-extension/src/@types/global.d.ts +++ b/extensions/model-extension/src/@types/global.d.ts @@ -1,7 +1,8 @@ declare const NODE: string -declare const API_URL: string +declare const CORTEX_API_URL: string declare const SETTINGS: SettingComponentProps[] declare const DEFAULT_MODEL_SOURCES: any +declare const CORTEX_API_KEY: string interface Core { api: APIFunctions diff --git a/extensions/model-extension/src/index.ts b/extensions/model-extension/src/index.ts index 4add0b6fa..de543c150 100644 --- a/extensions/model-extension/src/index.ts +++ b/extensions/model-extension/src/index.ts @@ -32,6 +32,19 @@ type Data = { */ export default class JanModelExtension extends ModelExtension { queue = new PQueue({ concurrency: 1 }) + + /** + * Extended API instance for making requests to the Cortex API. + * @returns + */ + api = () => + ky.extend({ + prefixUrl: CORTEX_API_URL, + headers: { + Authorization: `Bearer ${CORTEX_API_KEY}`, + }, + }) + /** * Called when the extension is loaded. * @override @@ -82,8 +95,8 @@ export default class JanModelExtension extends ModelExtension { * Sending POST to /models/pull/{id} endpoint to pull the model */ return this.queue.add(() => - ky - .post(`${API_URL}/v1/models/pull`, { json: { model, id, name } }) + this.api() + .post('v1/models/pull', { json: { model, id, name } }) .json() .catch(async (e) => { throw (await e.response?.json()) ?? e @@ -103,8 +116,8 @@ export default class JanModelExtension extends ModelExtension { * Sending DELETE to /models/pull/{id} endpoint to cancel a model pull */ return this.queue.add(() => - ky - .delete(`${API_URL}/v1/models/pull`, { json: { taskId: model } }) + this.api() + .delete('v1/models/pull', { json: { taskId: model } }) .json() .then() ) @@ -117,7 +130,7 @@ export default class JanModelExtension extends ModelExtension { */ async deleteModel(model: string): Promise { return this.queue - .add(() => ky.delete(`${API_URL}/v1/models/${model}`).json().then()) + .add(() => this.api().delete(`v1/models/${model}`).json().then()) .catch((e) => console.debug(e)) .finally(async () => { // Delete legacy model files @@ -219,8 +232,8 @@ export default class JanModelExtension extends ModelExtension { async updateModel(model: Partial): Promise { return this.queue .add(() => - ky - .patch(`${API_URL}/v1/models/${model.id}`, { json: { ...model } }) + this.api() + .patch(`v1/models/${model.id}`, { json: { ...model } }) .json() .then() ) @@ -233,8 +246,8 @@ export default class JanModelExtension extends ModelExtension { */ async getModel(model: string): Promise { return this.queue.add(() => - ky - .get(`${API_URL}/v1/models/${model}`) + this.api() + .get(`v1/models/${model}`) .json() .then((e) => this.transformModel(e)) ) as Promise @@ -252,8 +265,8 @@ export default class JanModelExtension extends ModelExtension { option?: OptionType ): Promise { return this.queue.add(() => - ky - .post(`${API_URL}/v1/models/import`, { + this.api() + .post('v1/models/import', { json: { model, modelPath, name, option }, }) .json() @@ -269,7 +282,7 @@ export default class JanModelExtension extends ModelExtension { */ async getSources(): Promise { const sources = await this.queue - .add(() => ky.get(`${API_URL}/v1/models/sources`).json>()) + .add(() => this.api().get('v1/models/sources').json>()) .then((e) => (typeof e === 'object' ? (e.data as ModelSource[]) : [])) .catch(() => []) return sources.concat( @@ -283,7 +296,7 @@ export default class JanModelExtension extends ModelExtension { */ async addSource(source: string): Promise { return this.queue.add(() => - ky.post(`${API_URL}/v1/models/sources`, { + this.api().post('v1/models/sources', { json: { source, }, @@ -297,7 +310,7 @@ export default class JanModelExtension extends ModelExtension { */ async deleteSource(source: string): Promise { return this.queue.add(() => - ky.delete(`${API_URL}/v1/models/sources`, { + this.api().delete('v1/models/sources', { json: { source, }, @@ -312,7 +325,7 @@ export default class JanModelExtension extends ModelExtension { */ async isModelLoaded(model: string): Promise { return this.queue - .add(() => ky.get(`${API_URL}/v1/models/status/${model}`)) + .add(() => this.api().get(`v1/models/status/${model}`)) .then((e) => true) .catch(() => false) } @@ -324,14 +337,14 @@ export default class JanModelExtension extends ModelExtension { return this.updateCortexConfig(options).catch((e) => console.debug(e)) } - /** + /** * Fetches models list from cortex.cpp * @param model * @returns */ - async fetchModels(): Promise { + async fetchModels(): Promise { return this.queue - .add(() => ky.get(`${API_URL}/v1/models?limit=-1`).json>()) + .add(() => this.api().get('v1/models?limit=-1').json>()) .then((e) => typeof e === 'object' ? e.data.map((e) => this.transformModel(e)) : [] ) @@ -371,7 +384,9 @@ export default class JanModelExtension extends ModelExtension { }): Promise { return this.queue .add(() => - ky.patch(`${API_URL}/v1/configs`, { json: body }).then(() => {}) + this.api() + .patch('v1/configs', { json: body }) + .then(() => {}) ) .catch((e) => console.debug(e)) } @@ -381,8 +396,8 @@ export default class JanModelExtension extends ModelExtension { * @returns */ private healthz(): Promise { - return ky - .get(`${API_URL}/healthz`, { + return this.api() + .get('healthz', { retry: { limit: 20, delay: () => 500, @@ -401,8 +416,8 @@ export default class JanModelExtension extends ModelExtension { const models = await this.fetchModels() return this.queue.add(() => - ky - .get(`${API_URL}/v1/models/hub?author=cortexso&tag=cortex.cpp`) + this.api() + .get('v1/models/hub?author=cortexso&tag=cortex.cpp') .json>() .then((e) => { e.data?.forEach((model) => { diff --git a/server/cortex.json b/server/cortex.json index 9d62a8b03..215e44dc6 100644 --- a/server/cortex.json +++ b/server/cortex.json @@ -3633,238 +3633,6 @@ }, "tags": ["Files"] } - }, - "/configs": { - "get": { - "summary": "Get Configurations", - "description": "Retrieves the current configuration settings of the Cortex server.", - "responses": { - "200": { - "description": "Successful response", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "allowed_origins": { - "type": "array", - "items": { - "type": "string" - }, - "example": ["http://127.0.0.1:39281", "https://cortex.so"] - }, - "cors": { - "type": "boolean", - "example": false - }, - "proxy_username": { - "type": "string", - "example": "username" - }, - "proxy_password": { - "type": "string", - "example": "password" - }, - "proxy_url": { - "type": "string", - "example": "http://proxy.example.com:8080" - }, - "verify_proxy_ssl": { - "type": "boolean", - "description": "test", - "example": false - }, - "verify_proxy_host_ssl": { - "type": "boolean", - "example": false - }, - "verify_peer_ssl": { - "type": "boolean", - "example": false - }, - "verify_host_ssl": { - "type": "boolean", - "example": false - }, - "no_proxy": { - "type": "string", - "example": "localhost" - }, - "huggingface_token": { - "type": "string", - "example": "your_token" - } - } - }, - "example": { - "allowed_origins": [ - "http://127.0.0.1:39281", - "https://cortex.so" - ], - "cors": false, - "proxy_username": "username", - "proxy_password": "password", - "proxy_url": "http://proxy.example.com:8080", - "verify_proxy_ssl": false, - "verify_proxy_host_ssl": false, - "verify_peer_ssl": false, - "verify_host_ssl": false, - "no_proxy": "localhost", - "huggingface_token": "your_token" - } - } - } - } - }, - "tags": ["Configurations"] - }, - "patch": { - "tags": ["Configurations"], - "summary": "Update configuration settings", - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "cors": { - "type": "boolean", - "description": "Indicates whether CORS is enabled.", - "example": false - }, - "allowed_origins": { - "type": "array", - "items": { - "type": "string" - }, - "description": "List of allowed origins.", - "example": ["http://127.0.0.1:39281", "https://cortex.so"] - }, - "proxy_username": { - "type": "string", - "description": "Username for the proxy server.", - "example": "username" - }, - "proxy_password": { - "type": "string", - "description": "Password for the proxy server.", - "example": "password" - }, - "proxy_url": { - "type": "string", - "description": "URL for the proxy server.", - "example": "http://proxy.example.com:8080" - }, - "verify_proxy_ssl": { - "type": "boolean", - "description": "Indicates whether to verify the SSL certificate of the proxy server.", - "example": false - }, - "verify_proxy_host_ssl": { - "type": "boolean", - "description": "Indicates whether to verify the SSL certificate of the proxy server host.", - "example": false - }, - "verify_peer_ssl": { - "type": "boolean", - "description": "Indicates whether to verify the SSL certificate of the peer.", - "example": false - }, - "verify_host_ssl": { - "type": "boolean", - "description": "Indicates whether to verify the SSL certificate of the host.", - "example": false - }, - "no_proxy": { - "type": "string", - "description": "List of hosts that should not be proxied.", - "example": "localhost" - }, - "huggingface_token": { - "type": "string", - "description": "HuggingFace token to pull models.", - "example": "your_token" - } - } - } - } - } - }, - "responses": { - "200": { - "description": "Configuration updated successfully", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "config": { - "type": "object", - "properties": { - "allowed_origins": { - "type": "array", - "items": { - "type": "string" - }, - "example": [ - "http://127.0.0.1:39281", - "https://cortex.so" - ] - }, - "cors": { - "type": "boolean", - "example": false - }, - "proxy_username": { - "type": "string", - "example": "username" - }, - "proxy_password": { - "type": "string", - "example": "password" - }, - "proxy_url": { - "type": "string", - "example": "http://proxy.example.com:8080" - }, - "verify_proxy_ssl": { - "type": "boolean", - "example": false - }, - "verify_proxy_host_ssl": { - "type": "boolean", - "example": false - }, - "verify_peer_ssl": { - "type": "boolean", - "example": false - }, - "verify_host_ssl": { - "type": "boolean", - "example": false - }, - "no_proxy": { - "type": "string", - "example": "localhost" - }, - "huggingface_token": { - "type": "string", - "example": "your_token" - } - } - }, - "message": { - "type": "string", - "example": "Configuration updated successfully" - } - } - } - } - } - } - } - } } }, "info": { diff --git a/server/global.d.ts b/server/global.d.ts index 1fb95b9a0..3b58bab56 100644 --- a/server/global.d.ts +++ b/server/global.d.ts @@ -1 +1,2 @@ -declare const CORTEX_API_URL: string \ No newline at end of file +declare const CORTEX_API_URL: string +declare const CORTEX_API_KEY: string \ No newline at end of file diff --git a/server/index.ts b/server/index.ts index cee3e4df2..4c14eb4c1 100644 --- a/server/index.ts +++ b/server/index.ts @@ -86,6 +86,14 @@ export const startServer = async (configs?: ServerConfig): Promise => { }, }) + const rewriteRequestHeaders = (req: any, headers: any) => { + if (req.url.includes('/configs')) return headers + return { + ...headers, + authorization: `Bearer ${CORTEX_API_KEY}`, // Add or modify Authorization header + } + } + // Register Swagger UI await server.register(require('@fastify/swagger-ui'), { routePrefix: '/', @@ -102,24 +110,36 @@ export const startServer = async (configs?: ServerConfig): Promise => { upstream: `${CORTEX_API_URL}/v1`, prefix: configs?.prefix ?? '/v1', http2: false, - }) - - server.register(proxy, { - upstream: `${CORTEX_API_URL}/system`, - prefix:'/system', - http2: false, + replyOptions: { + rewriteRequestHeaders, + }, }) server.register(proxy, { upstream: `${CORTEX_API_URL}/processManager`, - prefix:'/processManager', + prefix: '/processManager', http2: false, + replyOptions: { + rewriteRequestHeaders, + }, + }) + + server.register(proxy, { + upstream: `${CORTEX_API_URL}/system`, + prefix: '/system', + http2: false, + replyOptions: { + rewriteRequestHeaders, + }, }) server.register(proxy, { upstream: `${CORTEX_API_URL}/healthz`, - prefix:'/healthz', + prefix: '/healthz', http2: false, + replyOptions: { + rewriteRequestHeaders, + }, }) // Start listening for requests diff --git a/server/rolldown.config.mjs b/server/rolldown.config.mjs index 5f094a1af..cfb6c714f 100644 --- a/server/rolldown.config.mjs +++ b/server/rolldown.config.mjs @@ -16,6 +16,7 @@ export default defineConfig([ platform: 'node', define: { CORTEX_API_URL: JSON.stringify(`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? "39291"}`), + CORTEX_API_KEY: JSON.stringify(process.env.CORTEX_API_KEY ?? 'cortexserver'), } }, ])