Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Cortex API Authorization

Browse Source
This commit is contained in:
Louis 2025-03-20 19:00:29 +07:00
parent 09877c94a2
commit 821036945f
No known key found for this signature in database
GPG Key ID: 44FA9F4D33C37DE2
27 changed files with 221 additions and 348 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
extensions/assistant-extension/rolldown.config.mjs
View File

@ -28,6 +28,7 @@ export default defineConfig([
},
define: {
CORTEX_API_URL: JSON.stringify(`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? "39291"}`),
CORTEX_API_KEY: JSON.stringify(process.env.CORTEX_API_KEY ?? 'cortexserver'),
},
platform: 'node',
},

1
extensions/assistant-extension/src/@types/global.d.ts vendored
View File

@ -1,3 +1,4 @@
declare const NODE: string
declare const VERSION: string
declare const CORTEX_API_URL: string
declare const CORTEX_API_KEY: string

4
extensions/assistant-extension/src/node/retrieval.ts
View File

@ -27,7 +27,7 @@ export class Retrieval {
// declare time-weighted retriever and storage
this.timeWeightedVectorStore = new MemoryVectorStore(
new OpenAIEmbeddings(
{ openAIApiKey: 'cortex-embedding' },
{ openAIApiKey: CORTEX_API_KEY },
{ basePath: `${CORTEX_API_URL}/v1` }
)
)
@ -49,7 +49,7 @@ export class Retrieval {
public updateEmbeddingEngine(model: string, engine: string): void {
this.embeddingModel = new OpenAIEmbeddings(
{ openAIApiKey: 'cortex-embedding', model },
{ openAIApiKey: CORTEX_API_KEY, model },
// TODO: Raw settings
{ basePath: `${CORTEX_API_URL}/v1` }
)

1
extensions/conversational-extension/rolldown.config.mjs
View File

@ -9,5 +9,6 @@ export default defineConfig({
platform: 'browser',
define: {
API_URL: JSON.stringify(`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? "39291"}`),
CORTEX_API_KEY: JSON.stringify(process.env.CORTEX_API_KEY ?? 'cortexserver'),
},
})

1
extensions/conversational-extension/src/@types/global.d.ts vendored
View File

@ -1,4 +1,5 @@
declare const API_URL: string
declare const CORTEX_API_KEY: string
interface Core {
api: APIFunctions

51
extensions/conversational-extension/src/index.ts
View File

@ -22,6 +22,17 @@ type MessageList = {
export default class CortexConversationalExtension extends ConversationalExtension {
queue = new PQueue({ concurrency: 1 })
/**
* Extended API instance for making requests to the Cortex API.
* @returns
*/
api = () =>
ky.extend({
prefixUrl: API_URL,
headers: {
Authorization: `Bearer ${CORTEX_API_KEY}`,
},
})
/**
* Called when the extension is loaded.
*/
@ -39,8 +50,8 @@ export default class CortexConversationalExtension extends ConversationalExtensi
*/
async listThreads(): Promise<Thread[]> {
return this.queue.add(() =>
ky
.get(`${API_URL}/v1/threads?limit=-1`)
this.api()
.get('v1/threads?limit=-1')
.json<ThreadList>()
.then((e) => e.data)
) as Promise<Thread[]>
@ -52,7 +63,7 @@ export default class CortexConversationalExtension extends ConversationalExtensi
*/
async createThread(thread: Thread): Promise<Thread> {
return this.queue.add(() =>
ky.post(`${API_URL}/v1/threads`, { json: thread }).json<Thread>()
this.api().post('v1/threads', { json: thread }).json<Thread>()
) as Promise<Thread>
}
@ -63,7 +74,7 @@ export default class CortexConversationalExtension extends ConversationalExtensi
async modifyThread(thread: Thread): Promise<void> {
return this.queue
.add(() =>
ky.patch(`${API_URL}/v1/threads/${thread.id}`, { json: thread })
this.api().patch(`v1/threads/${thread.id}`, { json: thread })
)
.then()
}
@ -74,7 +85,7 @@ export default class CortexConversationalExtension extends ConversationalExtensi
*/
async deleteThread(threadId: string): Promise<void> {
return this.queue
.add(() => ky.delete(`${API_URL}/v1/threads/${threadId}`))
.add(() => this.api().delete(`v1/threads/${threadId}`))
.then()
}
@ -85,8 +96,8 @@ export default class CortexConversationalExtension extends ConversationalExtensi
*/
async createMessage(message: ThreadMessage): Promise<ThreadMessage> {
return this.queue.add(() =>
ky
.post(`${API_URL}/v1/threads/${message.thread_id}/messages`, {
this.api()
.post(`v1/threads/${message.thread_id}/messages`, {
json: message,
})
.json<ThreadMessage>()
@ -100,9 +111,9 @@ export default class CortexConversationalExtension extends ConversationalExtensi
*/
async modifyMessage(message: ThreadMessage): Promise<ThreadMessage> {
return this.queue.add(() =>
ky
this.api()
.patch(
`${API_URL}/v1/threads/${message.thread_id}/messages/${message.id}`,
`v1/threads/${message.thread_id}/messages/${message.id}`,
{
json: message,
}
@ -120,7 +131,7 @@ export default class CortexConversationalExtension extends ConversationalExtensi
async deleteMessage(threadId: string, messageId: string): Promise<void> {
return this.queue
.add(() =>
ky.delete(`${API_URL}/v1/threads/${threadId}/messages/${messageId}`)
this.api().delete(`v1/threads/${threadId}/messages/${messageId}`)
)
.then()
}
@ -132,8 +143,8 @@ export default class CortexConversationalExtension extends ConversationalExtensi
*/
async listMessages(threadId: string): Promise<ThreadMessage[]> {
return this.queue.add(() =>
ky
.get(`${API_URL}/v1/threads/${threadId}/messages?order=asc&limit=-1`)
this.api()
.get(`v1/threads/${threadId}/messages?order=asc&limit=-1`)
.json<MessageList>()
.then((e) => e.data)
) as Promise<ThreadMessage[]>
@ -147,8 +158,8 @@ export default class CortexConversationalExtension extends ConversationalExtensi
*/
async getThreadAssistant(threadId: string): Promise<ThreadAssistantInfo> {
return this.queue.add(() =>
ky
.get(`${API_URL}/v1/assistants/${threadId}?limit=-1`)
this.api()
.get(`v1/assistants/${threadId}?limit=-1`)
.json<ThreadAssistantInfo>()
) as Promise<ThreadAssistantInfo>
}
@ -163,8 +174,8 @@ export default class CortexConversationalExtension extends ConversationalExtensi
assistant: ThreadAssistantInfo
): Promise<ThreadAssistantInfo> {
return this.queue.add(() =>
ky
.post(`${API_URL}/v1/assistants/${threadId}`, { json: assistant })
this.api()
.post(`v1/assistants/${threadId}`, { json: assistant })
.json<ThreadAssistantInfo>()
) as Promise<ThreadAssistantInfo>
}
@ -180,8 +191,8 @@ export default class CortexConversationalExtension extends ConversationalExtensi
assistant: ThreadAssistantInfo
): Promise<ThreadAssistantInfo> {
return this.queue.add(() =>
ky
.patch(`${API_URL}/v1/assistants/${threadId}`, { json: assistant })
this.api()
.patch(`v1/assistants/${threadId}`, { json: assistant })
.json<ThreadAssistantInfo>()
) as Promise<ThreadAssistantInfo>
}
@ -191,8 +202,8 @@ export default class CortexConversationalExtension extends ConversationalExtensi
* @returns
*/
async healthz(): Promise<void> {
return ky
.get(`${API_URL}/healthz`, {
return this.api()
.get('healthz', {
retry: { limit: 20, delay: () => 500, methods: ['get'] },
})
.then(() => {})

1
extensions/engine-management-extension/rolldown.config.mjs
View File

@ -28,6 +28,7 @@ export default defineConfig([
'Authorization: Bearer {{api_key}}'
),
VERSION: JSON.stringify(pkgJson.version ?? '0.0.0'),
CORTEX_API_KEY: JSON.stringify(process.env.CORTEX_API_KEY ?? 'cortexserver'),
},
},
{

1
extensions/engine-management-extension/src/@types/global.d.ts vendored
View File

@ -6,6 +6,7 @@ declare const DEFAULT_REQUEST_PAYLOAD_TRANSFORM: string
declare const DEFAULT_RESPONSE_BODY_TRANSFORM: string
declare const DEFAULT_REQUEST_HEADERS_TRANSFORM: string
declare const VERSION: string
declare const CORTEX_API_KEY: string
declare const DEFAULT_REMOTE_ENGINES: ({
id: string

82
extensions/engine-management-extension/src/index.ts
View File

@ -31,6 +31,17 @@ interface ModelList {
export default class JanEngineManagementExtension extends EngineManagementExtension {
queue = new PQueue({ concurrency: 1 })
/**
* Extended API instance for making requests to the Cortex API.
* @returns
*/
api = () =>
ky.extend({
prefixUrl: API_URL,
headers: {
Authorization: `Bearer ${CORTEX_API_KEY}`,
},
})
/**
* Called when the extension is loaded.
*/
@ -59,8 +70,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
*/
async getEngines(): Promise<Engines> {
return this.queue.add(() =>
ky
.get(`${API_URL}/v1/engines`)
this.api()
.get('v1/engines')
.json<Engines>()
.then((e) => e)
) as Promise<Engines>
@ -70,8 +81,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
* @returns A Promise that resolves to an object of list engines.
*/
async getRemoteModels(name: string): Promise<any> {
return ky
.get(`${API_URL}/v1/models/remote/${name}`)
return this.api()
.get(`v1/models/remote/${name}`)
.json<ModelList>()
.catch(() => ({
data: [],
@ -84,8 +95,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
*/
async getInstalledEngines(name: InferenceEngine): Promise<EngineVariant[]> {
return this.queue.add(() =>
ky
.get(`${API_URL}/v1/engines/${name}`)
this.api()
.get(`v1/engines/${name}`)
.json<EngineVariant[]>()
.then((e) => e)
) as Promise<EngineVariant[]>
@ -103,8 +114,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
platform?: string
) {
return this.queue.add(() =>
ky
.get(`${API_URL}/v1/engines/${name}/releases/${version}`)
this.api()
.get(`v1/engines/${name}/releases/${version}`)
.json<EngineReleased[]>()
.then((e) =>
platform ? e.filter((r) => r.name.includes(platform)) : e
@ -119,8 +130,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
*/
async getLatestReleasedEngine(name: InferenceEngine, platform?: string) {
return this.queue.add(() =>
ky
.get(`${API_URL}/v1/engines/${name}/releases/latest`)
this.api()
.get(`v1/engines/${name}/releases/latest`)
.json<EngineReleased[]>()
.then((e) =>
platform ? e.filter((r) => r.name.includes(platform)) : e
@ -134,8 +145,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
*/
async installEngine(name: string, engineConfig: EngineConfig) {
return this.queue.add(() =>
ky
.post(`${API_URL}/v1/engines/${name}/install`, { json: engineConfig })
this.api()
.post(`v1/engines/${name}/install`, { json: engineConfig })
.then((e) => e)
) as Promise<{ messages: string }>
}
@ -167,15 +178,17 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
engineConfig.metadata.header_template = DEFAULT_REQUEST_HEADERS_TRANSFORM
return this.queue.add(() =>
ky.post(`${API_URL}/v1/engines`, { json: engineConfig }).then((e) => {
if (persistModels && engineConfig.metadata?.get_models_url) {
// Pull /models from remote models endpoint
return this.populateRemoteModels(engineConfig)
.then(() => e)
.catch(() => e)
}
return e
})
this.api()
.post('v1/engines', { json: engineConfig })
.then((e) => {
if (persistModels && engineConfig.metadata?.get_models_url) {
// Pull /models from remote models endpoint
return this.populateRemoteModels(engineConfig)
.then(() => e)
.catch(() => e)
}
return e
})
) as Promise<{ messages: string }>
}
@ -185,8 +198,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
*/
async uninstallEngine(name: InferenceEngine, engineConfig: EngineConfig) {
return this.queue.add(() =>
ky
.delete(`${API_URL}/v1/engines/${name}/install`, { json: engineConfig })
this.api()
.delete(`v1/engines/${name}/install`, { json: engineConfig })
.then((e) => e)
) as Promise<{ messages: string }>
}
@ -198,8 +211,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
async addRemoteModel(model: Model) {
return this.queue
.add(() =>
ky
.post(`${API_URL}/v1/models/add`, {
this.api()
.post('v1/models/add', {
json: {
inference_params: {
max_tokens: 4096,
@ -223,8 +236,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
*/
async getDefaultEngineVariant(name: InferenceEngine) {
return this.queue.add(() =>
ky
.get(`${API_URL}/v1/engines/${name}/default`)
this.api()
.get(`v1/engines/${name}/default`)
.json<{ messages: string }>()
.then((e) => e)
) as Promise<DefaultEngineVariant>
@ -240,8 +253,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
engineConfig: EngineConfig
) {
return this.queue.add(() =>
ky
.post(`${API_URL}/v1/engines/${name}/default`, { json: engineConfig })
this.api()
.post(`v1/engines/${name}/default`, { json: engineConfig })
.then((e) => e)
) as Promise<{ messages: string }>
}
@ -251,8 +264,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
*/
async updateEngine(name: InferenceEngine, engineConfig?: EngineConfig) {
return this.queue.add(() =>
ky
.post(`${API_URL}/v1/engines/${name}/update`, { json: engineConfig })
this.api()
.post(`v1/engines/${name}/update`, { json: engineConfig })
.then((e) => e)
) as Promise<{ messages: string }>
}
@ -262,8 +275,8 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
* @returns
*/
async healthz(): Promise<void> {
return ky
.get(`${API_URL}/healthz`, {
return this.api()
.get('healthz', {
retry: { limit: 20, delay: () => 500, methods: ['get'] },
})
.then(() => {
@ -390,7 +403,6 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
const version = await this.getSetting<string>('version', '0.0.0')
const engines = await this.getEngines()
if (version < VERSION) {
console.log('Migrating engine settings...')
// Migrate engine settings
await Promise.all(
@ -398,7 +410,7 @@ export default class JanEngineManagementExtension extends EngineManagementExtens
const { id, ...data } = engine
data.api_key = engines[id]?.api_key
return this.updateEngine(id,{
return this.updateEngine(id, {
...data,
}).catch(console.error)
})

2
extensions/hardware-management-extension/package.json
View File

@ -29,12 +29,10 @@
},
"dependencies": {
"@janhq/core": "../../core/package.tgz",
"cpu-instructions": "^0.0.13",
"ky": "^1.7.2",
"p-queue": "^8.0.1"
},
"bundledDependencies": [
"cpu-instructions",
"@janhq/core"
],
"hardwares": {

1
extensions/hardware-management-extension/rolldown.config.mjs
View File

@ -11,6 +11,7 @@ export default defineConfig([
define: {
NODE: JSON.stringify(`${pkgJson.name}/${pkgJson.node}`),
API_URL: JSON.stringify(`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? "39291"}`),
CORTEX_API_KEY: JSON.stringify(process.env.CORTEX_API_KEY ?? 'cortexserver'),
},
},
])

1
extensions/hardware-management-extension/src/@types/global.d.ts vendored
View File

@ -1,5 +1,6 @@
declare const API_URL: string
declare const NODE: string
declare const CORTEX_API_KEY: string
interface Core {
api: APIFunctions

21
extensions/hardware-management-extension/src/index.ts
View File

@ -9,6 +9,17 @@ import PQueue from 'p-queue'
export default class JSONHardwareManagementExtension extends HardwareManagementExtension {
queue = new PQueue({ concurrency: 1 })
/**
* Extended API instance for making requests to the Cortex API.
* @returns
*/
api = () =>
ky.extend({
prefixUrl: API_URL,
headers: {
'Authorization': `Bearer ${CORTEX_API_KEY}`,
},
});
/**
* Called when the extension is loaded.
*/
@ -27,8 +38,8 @@ export default class JSONHardwareManagementExtension extends HardwareManagementE
* @returns
*/
async healthz(): Promise<void> {
return ky
.get(`${API_URL}/healthz`, {
return this.api()
.get('healthz', {
retry: { limit: 20, delay: () => 500, methods: ['get'] },
})
.then(() => {})
@ -39,8 +50,8 @@ export default class JSONHardwareManagementExtension extends HardwareManagementE
*/
async getHardware(): Promise<HardwareInformation> {
return this.queue.add(() =>
ky
.get(`${API_URL}/v1/hardware`)
this.api()
.get('v1/hardware')
.json<HardwareInformation>()
.then((e) => e)
) as Promise<HardwareInformation>
@ -54,7 +65,7 @@ export default class JSONHardwareManagementExtension extends HardwareManagementE
activated_gpus: number[]
}> {
return this.queue.add(() =>
ky.post(`${API_URL}/v1/hardware/activate`, { json: data }).then((e) => e)
this.api().post('v1/hardware/activate', { json: data }).then((e) => e)
) as Promise<{
message: string
activated_gpus: number[]

2
extensions/inference-cortex-extension/bin/version.txt
View File

@ -1 +1 @@
1.0.11
1.0.12-rc3

6
extensions/inference-cortex-extension/download.bat
View File

@ -5,11 +5,11 @@ set /p CORTEX_VERSION=<./bin/version.txt
set ENGINE_VERSION=0.1.55
@REM Download cortex.llamacpp binaries
set DOWNLOAD_URL=https://github.com/janhq/cortex.llamacpp/releases/download/v%ENGINE_VERSION%/cortex.llamacpp-%ENGINE_VERSION%-windows-amd64
set CUDA_DOWNLOAD_URL=https://github.com/janhq/cortex.llamacpp/releases/download/v%ENGINE_VERSION%
set DOWNLOAD_URL=https://github.com/menloresearch/cortex.llamacpp/releases/download/v%ENGINE_VERSION%/cortex.llamacpp-%ENGINE_VERSION%-windows-amd64
set CUDA_DOWNLOAD_URL=https://github.com/menloresearch/cortex.llamacpp/releases/download/v%ENGINE_VERSION%
set SUBFOLDERS=windows-amd64-noavx-cuda-12-0 windows-amd64-noavx-cuda-11-7 windows-amd64-avx2-cuda-12-0 windows-amd64-avx2-cuda-11-7 windows-amd64-noavx windows-amd64-avx windows-amd64-avx2 windows-amd64-avx512 windows-amd64-vulkan
call .\node_modules\.bin\download -e --strip 1 -o %BIN_PATH% https://github.com/janhq/cortex.cpp/releases/download/v%CORTEX_VERSION%/cortex-%CORTEX_VERSION%-windows-amd64.tar.gz
call .\node_modules\.bin\download -e --strip 1 -o %BIN_PATH% https://github.com/menloresearch/cortex.cpp/releases/download/v%CORTEX_VERSION%/cortex-%CORTEX_VERSION%-windows-amd64.tar.gz
call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2-cuda-12-0.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-avx2-cuda-12-0/v%ENGINE_VERSION%
call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2-cuda-11-7.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-avx2-cuda-11-7/v%ENGINE_VERSION%
call .\node_modules\.bin\download %DOWNLOAD_URL%-noavx-cuda-12-0.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-noavx-cuda-12-0/v%ENGINE_VERSION%

6
extensions/inference-cortex-extension/download.sh
View File

@ -3,9 +3,9 @@
# Read CORTEX_VERSION
CORTEX_VERSION=$(cat ./bin/version.txt)
ENGINE_VERSION=0.1.55
CORTEX_RELEASE_URL="https://github.com/janhq/cortex.cpp/releases/download"
ENGINE_DOWNLOAD_URL="https://github.com/janhq/cortex.llamacpp/releases/download/v${ENGINE_VERSION}/cortex.llamacpp-${ENGINE_VERSION}"
CUDA_DOWNLOAD_URL="https://github.com/janhq/cortex.llamacpp/releases/download/v${ENGINE_VERSION}"
CORTEX_RELEASE_URL="https://github.com/menloresearch/cortex.cpp/releases/download"
ENGINE_DOWNLOAD_URL="https://github.com/menloresearch/cortex.llamacpp/releases/download/v${ENGINE_VERSION}/cortex.llamacpp-${ENGINE_VERSION}"
CUDA_DOWNLOAD_URL="https://github.com/menloresearch/cortex.llamacpp/releases/download/v${ENGINE_VERSION}"
BIN_PATH=./bin
SHARED_PATH="../../electron/shared"
# Detect platform

4
extensions/inference-cortex-extension/rolldown.config.mjs
View File

@ -19,12 +19,13 @@ export default defineConfig([
CORTEX_SOCKET_URL: JSON.stringify(
`ws://127.0.0.1:${process.env.CORTEX_API_PORT ?? '39291'}`
),
CORTEX_API_KEY: JSON.stringify(process.env.CORTEX_API_KEY ?? 'cortexserver'),
CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.55'),
},
},
{
input: 'src/node/index.ts',
external: ['@janhq/core/node', 'cpu-instructions'],
external: ['@janhq/core/node'],
output: {
format: 'cjs',
file: 'dist/node/index.cjs.js',
@ -35,6 +36,7 @@ export default defineConfig([
extensions: ['.js', '.ts', '.json'],
},
define: {
CORTEX_API_KEY: JSON.stringify(process.env.CORTEX_API_KEY ?? 'cortexserver'),
CORTEX_API_URL: JSON.stringify(
`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? '39291'}`
),

1
extensions/inference-cortex-extension/src/@types/global.d.ts vendored
View File

@ -3,3 +3,4 @@ declare const CORTEX_API_URL: string
declare const CORTEX_SOCKET_URL: string
declare const CORTEX_ENGINE_VERSION: string
declare const SETTINGS: any
declare const CORTEX_API_KEY: string

38
extensions/inference-cortex-extension/src/index.ts
View File

@ -75,6 +75,28 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
abortControllers = new Map<string, AbortController>()
/**
* Extended API instance for making requests to the Cortex API.
* @returns
*/
api = () =>
ky.extend({
prefixUrl: CORTEX_API_URL,
headers: {
Authorization: `Bearer ${CORTEX_API_KEY}`,
},
})
/**
* Authorization headers for the API requests.
* @returns
*/
headers(): Promise<HeadersInit> {
return Promise.resolve({
Authorization: `Bearer ${CORTEX_API_KEY}`,
})
}
/**
* Subscribes to events emitted by the @janhq/core package.
*/
@ -153,8 +175,8 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
this.abortControllers.set(model.id, controller)
return await this.queue.add(() =>
ky
.post(`${CORTEX_API_URL}/v1/models/start`, {
this.api()
.post('v1/models/start', {
json: {
...extractModelLoadParams(model.settings),
model: model.id,
@ -183,8 +205,8 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
}
override async unloadModel(model: Model): Promise<void> {
return ky
.post(`${CORTEX_API_URL}/v1/models/stop`, {
return this.api()
.post('v1/models/stop', {
json: { model: model.id },
})
.json()
@ -199,8 +221,8 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
* @returns
*/
private async healthz(): Promise<void> {
return ky
.get(`${CORTEX_API_URL}/healthz`, {
return this.api()
.get('healthz', {
retry: {
limit: 20,
delay: () => 500,
@ -215,8 +237,8 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
* @returns
*/
private async clean(): Promise<any> {
return ky
.delete(`${CORTEX_API_URL}/processmanager/destroy`, {
return this.api()
.delete('processmanager/destroy', {
timeout: 2000, // maximum 2 seconds
retry: {
limit: 0,

5
extensions/inference-cortex-extension/src/node/index.ts
View File

@ -44,8 +44,9 @@ function run(): Promise<any> {
`${path.join(dataFolderPath, '.janrc')}`,
'--data_folder_path',
dataFolderPath,
'--loglevel',
'INFO',
'config',
'--api_keys',
CORTEX_API_KEY ?? 'cortexserver',
],
{
env: {

3
extensions/model-extension/rolldown.config.mjs
View File

@ -11,7 +11,8 @@ export default defineConfig({
platform: 'browser',
define: {
SETTINGS: JSON.stringify(settingJson),
API_URL: JSON.stringify(`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? "39291"}`),
CORTEX_API_URL: JSON.stringify(`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? "39291"}`),
DEFAULT_MODEL_SOURCES: JSON.stringify(modelSources),
CORTEX_API_KEY: JSON.stringify(process.env.CORTEX_API_KEY ?? 'cortexserver'),
},
})

3
extensions/model-extension/src/@types/global.d.ts vendored
View File

@ -1,7 +1,8 @@
declare const NODE: string
declare const API_URL: string
declare const CORTEX_API_URL: string
declare const SETTINGS: SettingComponentProps[]
declare const DEFAULT_MODEL_SOURCES: any
declare const CORTEX_API_KEY: string
interface Core {
api: APIFunctions

61
extensions/model-extension/src/index.ts
View File

@ -32,6 +32,19 @@ type Data<T> = {
*/
export default class JanModelExtension extends ModelExtension {
queue = new PQueue({ concurrency: 1 })
/**
* Extended API instance for making requests to the Cortex API.
* @returns
*/
api = () =>
ky.extend({
prefixUrl: CORTEX_API_URL,
headers: {
Authorization: `Bearer ${CORTEX_API_KEY}`,
},
})
/**
* Called when the extension is loaded.
* @override
@ -82,8 +95,8 @@ export default class JanModelExtension extends ModelExtension {
* Sending POST to /models/pull/{id} endpoint to pull the model
*/
return this.queue.add(() =>
ky
.post(`${API_URL}/v1/models/pull`, { json: { model, id, name } })
this.api()
.post('v1/models/pull', { json: { model, id, name } })
.json()
.catch(async (e) => {
throw (await e.response?.json()) ?? e
@ -103,8 +116,8 @@ export default class JanModelExtension extends ModelExtension {
* Sending DELETE to /models/pull/{id} endpoint to cancel a model pull
*/
return this.queue.add(() =>
ky
.delete(`${API_URL}/v1/models/pull`, { json: { taskId: model } })
this.api()
.delete('v1/models/pull', { json: { taskId: model } })
.json()
.then()
)
@ -117,7 +130,7 @@ export default class JanModelExtension extends ModelExtension {
*/
async deleteModel(model: string): Promise<void> {
return this.queue
.add(() => ky.delete(`${API_URL}/v1/models/${model}`).json().then())
.add(() => this.api().delete(`v1/models/${model}`).json().then())
.catch((e) => console.debug(e))
.finally(async () => {
// Delete legacy model files
@ -219,8 +232,8 @@ export default class JanModelExtension extends ModelExtension {
async updateModel(model: Partial<Model>): Promise<Model> {
return this.queue
.add(() =>
ky
.patch(`${API_URL}/v1/models/${model.id}`, { json: { ...model } })
this.api()
.patch(`v1/models/${model.id}`, { json: { ...model } })
.json()
.then()
)
@ -233,8 +246,8 @@ export default class JanModelExtension extends ModelExtension {
*/
async getModel(model: string): Promise<Model> {
return this.queue.add(() =>
ky
.get(`${API_URL}/v1/models/${model}`)
this.api()
.get(`v1/models/${model}`)
.json()
.then((e) => this.transformModel(e))
) as Promise<Model>
@ -252,8 +265,8 @@ export default class JanModelExtension extends ModelExtension {
option?: OptionType
): Promise<void> {
return this.queue.add(() =>
ky
.post(`${API_URL}/v1/models/import`, {
this.api()
.post('v1/models/import', {
json: { model, modelPath, name, option },
})
.json()
@ -269,7 +282,7 @@ export default class JanModelExtension extends ModelExtension {
*/
async getSources(): Promise<ModelSource[]> {
const sources = await this.queue
.add(() => ky.get(`${API_URL}/v1/models/sources`).json<Data<ModelSource>>())
.add(() => this.api().get('v1/models/sources').json<Data<ModelSource>>())
.then((e) => (typeof e === 'object' ? (e.data as ModelSource[]) : []))
.catch(() => [])
return sources.concat(
@ -283,7 +296,7 @@ export default class JanModelExtension extends ModelExtension {
*/
async addSource(source: string): Promise<any> {
return this.queue.add(() =>
ky.post(`${API_URL}/v1/models/sources`, {
this.api().post('v1/models/sources', {
json: {
source,
},
@ -297,7 +310,7 @@ export default class JanModelExtension extends ModelExtension {
*/
async deleteSource(source: string): Promise<any> {
return this.queue.add(() =>
ky.delete(`${API_URL}/v1/models/sources`, {
this.api().delete('v1/models/sources', {
json: {
source,
},
@ -312,7 +325,7 @@ export default class JanModelExtension extends ModelExtension {
*/
async isModelLoaded(model: string): Promise<boolean> {
return this.queue
.add(() => ky.get(`${API_URL}/v1/models/status/${model}`))
.add(() => this.api().get(`v1/models/status/${model}`))
.then((e) => true)
.catch(() => false)
}
@ -324,14 +337,14 @@ export default class JanModelExtension extends ModelExtension {
return this.updateCortexConfig(options).catch((e) => console.debug(e))
}
/**
/**
* Fetches models list from cortex.cpp
* @param model
* @returns
*/
async fetchModels(): Promise<Model[]> {
async fetchModels(): Promise<Model[]> {
return this.queue
.add(() => ky.get(`${API_URL}/v1/models?limit=-1`).json<Data<Model>>())
.add(() => this.api().get('v1/models?limit=-1').json<Data<Model>>())
.then((e) =>
typeof e === 'object' ? e.data.map((e) => this.transformModel(e)) : []
)
@ -371,7 +384,9 @@ export default class JanModelExtension extends ModelExtension {
}): Promise<void> {
return this.queue
.add(() =>
ky.patch(`${API_URL}/v1/configs`, { json: body }).then(() => {})
this.api()
.patch('v1/configs', { json: body })
.then(() => {})
)
.catch((e) => console.debug(e))
}
@ -381,8 +396,8 @@ export default class JanModelExtension extends ModelExtension {
* @returns
*/
private healthz(): Promise<void> {
return ky
.get(`${API_URL}/healthz`, {
return this.api()
.get('healthz', {
retry: {
limit: 20,
delay: () => 500,
@ -401,8 +416,8 @@ export default class JanModelExtension extends ModelExtension {
const models = await this.fetchModels()
return this.queue.add(() =>
ky
.get(`${API_URL}/v1/models/hub?author=cortexso&tag=cortex.cpp`)
this.api()
.get('v1/models/hub?author=cortexso&tag=cortex.cpp')
.json<Data<string>>()
.then((e) => {
e.data?.forEach((model) => {

232
server/cortex.json
View File

@ -3633,238 +3633,6 @@
},
"tags": ["Files"]
}
},
"/configs": {
"get": {
"summary": "Get Configurations",
"description": "Retrieves the current configuration settings of the Cortex server.",
"responses": {
"200": {
"description": "Successful response",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"allowed_origins": {
"type": "array",
"items": {
"type": "string"
},
"example": ["http://127.0.0.1:39281", "https://cortex.so"]
},
"cors": {
"type": "boolean",
"example": false
},
"proxy_username": {
"type": "string",
"example": "username"
},
"proxy_password": {
"type": "string",
"example": "password"
},
"proxy_url": {
"type": "string",
"example": "http://proxy.example.com:8080"
},
"verify_proxy_ssl": {
"type": "boolean",
"description": "test",
"example": false
},
"verify_proxy_host_ssl": {
"type": "boolean",
"example": false
},
"verify_peer_ssl": {
"type": "boolean",
"example": false
},
"verify_host_ssl": {
"type": "boolean",
"example": false
},
"no_proxy": {
"type": "string",
"example": "localhost"
},
"huggingface_token": {
"type": "string",
"example": "your_token"
}
}
},
"example": {
"allowed_origins": [
"http://127.0.0.1:39281",
"https://cortex.so"
],
"cors": false,
"proxy_username": "username",
"proxy_password": "password",
"proxy_url": "http://proxy.example.com:8080",
"verify_proxy_ssl": false,
"verify_proxy_host_ssl": false,
"verify_peer_ssl": false,
"verify_host_ssl": false,
"no_proxy": "localhost",
"huggingface_token": "your_token"
}
}
}
}
},
"tags": ["Configurations"]
},
"patch": {
"tags": ["Configurations"],
"summary": "Update configuration settings",
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"cors": {
"type": "boolean",
"description": "Indicates whether CORS is enabled.",
"example": false
},
"allowed_origins": {
"type": "array",
"items": {
"type": "string"
},
"description": "List of allowed origins.",
"example": ["http://127.0.0.1:39281", "https://cortex.so"]
},
"proxy_username": {
"type": "string",
"description": "Username for the proxy server.",
"example": "username"
},
"proxy_password": {
"type": "string",
"description": "Password for the proxy server.",
"example": "password"
},
"proxy_url": {
"type": "string",
"description": "URL for the proxy server.",
"example": "http://proxy.example.com:8080"
},
"verify_proxy_ssl": {
"type": "boolean",
"description": "Indicates whether to verify the SSL certificate of the proxy server.",
"example": false
},
"verify_proxy_host_ssl": {
"type": "boolean",
"description": "Indicates whether to verify the SSL certificate of the proxy server host.",
"example": false
},
"verify_peer_ssl": {
"type": "boolean",
"description": "Indicates whether to verify the SSL certificate of the peer.",
"example": false
},
"verify_host_ssl": {
"type": "boolean",
"description": "Indicates whether to verify the SSL certificate of the host.",
"example": false
},
"no_proxy": {
"type": "string",
"description": "List of hosts that should not be proxied.",
"example": "localhost"
},
"huggingface_token": {
"type": "string",
"description": "HuggingFace token to pull models.",
"example": "your_token"
}
}
}
}
}
},
"responses": {
"200": {
"description": "Configuration updated successfully",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"config": {
"type": "object",
"properties": {
"allowed_origins": {
"type": "array",
"items": {
"type": "string"
},
"example": [
"http://127.0.0.1:39281",
"https://cortex.so"
]
},
"cors": {
"type": "boolean",
"example": false
},
"proxy_username": {
"type": "string",
"example": "username"
},
"proxy_password": {
"type": "string",
"example": "password"
},
"proxy_url": {
"type": "string",
"example": "http://proxy.example.com:8080"
},
"verify_proxy_ssl": {
"type": "boolean",
"example": false
},
"verify_proxy_host_ssl": {
"type": "boolean",
"example": false
},
"verify_peer_ssl": {
"type": "boolean",
"example": false
},
"verify_host_ssl": {
"type": "boolean",
"example": false
},
"no_proxy": {
"type": "string",
"example": "localhost"
},
"huggingface_token": {
"type": "string",
"example": "your_token"
}
}
},
"message": {
"type": "string",
"example": "Configuration updated successfully"
}
}
}
}
}
}
}
}
}
},
"info": {

3
server/global.d.ts vendored
View File

@ -1 +1,2 @@
declare const CORTEX_API_URL: string
declare const CORTEX_API_URL: string
declare const CORTEX_API_KEY: string

36
server/index.ts
View File

@ -86,6 +86,14 @@ export const startServer = async (configs?: ServerConfig): Promise<boolean> => {
},
})
const rewriteRequestHeaders = (req: any, headers: any) => {
if (req.url.includes('/configs')) return headers
return {
...headers,
authorization: `Bearer ${CORTEX_API_KEY}`, // Add or modify Authorization header
}
}
// Register Swagger UI
await server.register(require('@fastify/swagger-ui'), {
routePrefix: '/',
@ -102,24 +110,36 @@ export const startServer = async (configs?: ServerConfig): Promise<boolean> => {
upstream: `${CORTEX_API_URL}/v1`,
prefix: configs?.prefix ?? '/v1',
http2: false,
})
server.register(proxy, {
upstream: `${CORTEX_API_URL}/system`,
prefix:'/system',
http2: false,
replyOptions: {
rewriteRequestHeaders,
},
})
server.register(proxy, {
upstream: `${CORTEX_API_URL}/processManager`,
prefix:'/processManager',
prefix: '/processManager',
http2: false,
replyOptions: {
rewriteRequestHeaders,
},
})
server.register(proxy, {
upstream: `${CORTEX_API_URL}/system`,
prefix: '/system',
http2: false,
replyOptions: {
rewriteRequestHeaders,
},
})
server.register(proxy, {
upstream: `${CORTEX_API_URL}/healthz`,
prefix:'/healthz',
prefix: '/healthz',
http2: false,
replyOptions: {
rewriteRequestHeaders,
},
})
// Start listening for requests

1
server/rolldown.config.mjs
View File

@ -16,6 +16,7 @@ export default defineConfig([
platform: 'node',
define: {
CORTEX_API_URL: JSON.stringify(`http://127.0.0.1:${process.env.CORTEX_API_PORT ?? "39291"}`),
CORTEX_API_KEY: JSON.stringify(process.env.CORTEX_API_KEY ?? 'cortexserver'),
}
},
])