diff --git a/.github/workflows/auto-assign-author.yml b/.github/workflows/auto-assign-author.yml index 0e861df00..13c93c717 100644 --- a/.github/workflows/auto-assign-author.yml +++ b/.github/workflows/auto-assign-author.yml @@ -6,6 +6,7 @@ on: jobs: assign-author: runs-on: ubuntu-latest + if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} permissions: pull-requests: write steps: diff --git a/.github/workflows/jan-electron-build-beta.yml b/.github/workflows/jan-electron-build-beta.yml index 9cae31d67..4f9886010 100644 --- a/.github/workflows/jan-electron-build-beta.yml +++ b/.github/workflows/jan-electron-build-beta.yml @@ -9,31 +9,6 @@ jobs: get-update-version: uses: ./.github/workflows/template-get-update-version.yml - create-draft-release: - runs-on: ubuntu-latest - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') - outputs: - upload_url: ${{ steps.create_release.outputs.upload_url }} - version: ${{ steps.get_version.outputs.version }} - permissions: - contents: write - steps: - - name: Extract tag name without v prefix - id: get_version - run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV && echo "::set-output name=version::${GITHUB_REF#refs/tags/v}" - env: - GITHUB_REF: ${{ github.ref }} - - name: Create Draft Release - id: create_release - uses: softprops/action-gh-release@v2 - with: - tag_name: ${{ github.ref_name }} - token: ${{ secrets.GITHUB_TOKEN }} - name: "${{ env.VERSION }}" - draft: true - prerelease: false - generate_release_notes: true - build-macos: uses: ./.github/workflows/template-build-macos.yml secrets: inherit @@ -65,7 +40,7 @@ jobs: beta: true sync-temp-to-latest: - needs: [build-macos, create-draft-release, build-windows-x64, build-linux-x64] + needs: [build-macos, build-windows-x64, build-linux-x64] runs-on: ubuntu-latest permissions: contents: write @@ -82,19 +57,15 @@ jobs: AWS_DEFAULT_REGION: ${{ secrets.DELTA_AWS_REGION }} AWS_EC2_METADATA_DISABLED: "true" - - name: set release to prerelease - run: | - gh release edit v${{ needs.create-draft-release.outputs.version }} --draft=false --prerelease - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - noti-discord-and-update-url-readme: - needs: [build-macos, create-draft-release, build-windows-x64, build-linux-x64, sync-temp-to-latest] + needs: [build-macos, get-update-version, build-windows-x64, build-linux-x64, sync-temp-to-latest] runs-on: ubuntu-latest steps: - name: Set version to environment variable run: | - echo "VERSION=${{ needs.create-draft-release.outputs.version }}" >> $GITHUB_ENV + VERSION=${{ needs.get-update-version.outputs.new_version }} + VERSION="${VERSION#v}" + echo "VERSION=$VERSION" >> $GITHUB_ENV - name: Notify Discord uses: Ilshidur/action-discord@master @@ -105,6 +76,5 @@ jobs: - macOS Universal: https://delta.jan.ai/beta/jan-beta-mac-universal-{{ VERSION }}.dmg - Linux Deb: https://delta.jan.ai/beta/jan-beta-linux-amd64-{{ VERSION }}.deb - Linux AppImage: https://delta.jan.ai/beta/jan-beta-linux-x86_64-{{ VERSION }}.AppImage - - Github Release URL: https://github.com/janhq/jan/releases/tag/v{{ VERSION }} env: DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK_JAN_BETA }} \ No newline at end of file diff --git a/.github/workflows/template-build-jan-server.yml b/.github/workflows/template-build-jan-server.yml deleted file mode 100644 index 9bb772605..000000000 --- a/.github/workflows/template-build-jan-server.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: build-jan-server -on: - workflow_call: - inputs: - dockerfile_path: - required: false - type: string - default: './Dockerfile' - docker_image_tag: - required: true - type: string - default: 'ghcr.io/janhq/jan-server:dev-latest' - -jobs: - build: - runs-on: ubuntu-latest - env: - REGISTRY: ghcr.io - IMAGE_NAME: janhq/jan-server - permissions: - packages: write - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push Docker image - uses: docker/build-push-action@v3 - with: - context: . - file: ${{ inputs.dockerfile_path }} - push: true - tags: ${{ inputs.docker_image_tag }} \ No newline at end of file diff --git a/.github/workflows/template-build-linux-x64.yml b/.github/workflows/template-build-linux-x64.yml index 9d12c4394..68079d0dc 100644 --- a/.github/workflows/template-build-linux-x64.yml +++ b/.github/workflows/template-build-linux-x64.yml @@ -83,7 +83,7 @@ jobs: cat ./electron/package.json echo "------------------------" cat ./package.json - jq '.build.publish = [{"provider": "generic", "url": "https://delta.jan.ai/beta", "channel": "beta"}, {"provider": "github", "owner": "janhq", "repo": "jan", "channel": "beta"}, {"provider": "s3", "acl": null, "bucket": "${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}", "region": "${{ secrets.DELTA_AWS_REGION}}", "path": "temp-beta", "channel": "beta"}]' electron/package.json > /tmp/package.json + jq '.build.publish = [{"provider": "generic", "url": "https://delta.jan.ai/beta", "channel": "beta"}, {"provider": "s3", "acl": null, "bucket": "${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}", "region": "${{ secrets.DELTA_AWS_REGION}}", "path": "temp-beta", "channel": "beta"}]' electron/package.json > /tmp/package.json mv /tmp/package.json electron/package.json cat electron/package.json @@ -103,7 +103,7 @@ jobs: # check public_provider is true or not echo "public_provider is ${{ inputs.public_provider }}" if [ "${{ inputs.public_provider }}" == "none" ]; then - make build + make build else make build-and-publish fi @@ -122,8 +122,6 @@ jobs: make build-and-publish env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ANALYTICS_ID: ${{ secrets.JAN_APP_UMAMI_PROJECT_API_KEY }} - ANALYTICS_HOST: ${{ secrets.JAN_APP_UMAMI_URL }} POSTHOG_KEY: ${{ secrets.POSTHOG_KEY }} POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }} diff --git a/.github/workflows/template-build-macos.yml b/.github/workflows/template-build-macos.yml index b415d665d..6bc38f786 100644 --- a/.github/workflows/template-build-macos.yml +++ b/.github/workflows/template-build-macos.yml @@ -99,7 +99,7 @@ jobs: cat ./electron/package.json echo "------------------------" cat ./package.json - jq '.build.publish = [{"provider": "generic", "url": "https://delta.jan.ai/beta", "channel": "beta"}, {"provider": "github", "owner": "janhq", "repo": "jan", "channel": "beta"}, {"provider": "s3", "acl": null, "bucket": "${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}", "region": "${{ secrets.DELTA_AWS_REGION}}", "path": "temp-beta", "channel": "beta"}]' electron/package.json > /tmp/package.json + jq '.build.publish = [{"provider": "generic", "url": "https://delta.jan.ai/beta", "channel": "beta"}, {"provider": "s3", "acl": null, "bucket": "${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}", "region": "${{ secrets.DELTA_AWS_REGION}}", "path": "temp-beta", "channel": "beta"}]' electron/package.json > /tmp/package.json mv /tmp/package.json electron/package.json cat electron/package.json @@ -134,7 +134,7 @@ jobs: # check public_provider is true or not echo "public_provider is ${{ inputs.public_provider }}" if [ "${{ inputs.public_provider }}" == "none" ]; then - make build + make build else make build-and-publish fi @@ -168,8 +168,6 @@ jobs: APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }} APP_PATH: '.' DEVELOPER_ID: ${{ secrets.DEVELOPER_ID }} - ANALYTICS_ID: ${{ secrets.JAN_APP_UMAMI_PROJECT_API_KEY }} - ANALYTICS_HOST: ${{ secrets.JAN_APP_UMAMI_URL }} POSTHOG_KEY: ${{ secrets.POSTHOG_KEY }} POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }} diff --git a/.github/workflows/template-build-windows-x64.yml b/.github/workflows/template-build-windows-x64.yml index 52ff22ce3..9e3171e03 100644 --- a/.github/workflows/template-build-windows-x64.yml +++ b/.github/workflows/template-build-windows-x64.yml @@ -108,7 +108,7 @@ jobs: cat ./package.json echo "------------------------" cat ./electron/scripts/uninstaller.nsh - jq '.build.publish = [{"provider": "generic", "url": "https://delta.jan.ai/beta", "channel": "beta"}, {"provider": "github", "owner": "janhq", "repo": "jan", "channel": "beta"}, {"provider": "s3", "acl": null, "bucket": "${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}", "region": "${{ secrets.DELTA_AWS_REGION}}", "path": "temp-beta", "channel": "beta"}]' electron/package.json > /tmp/package.json + jq '.build.publish = [{"provider": "generic", "url": "https://delta.jan.ai/beta", "channel": "beta"}, {"provider": "s3", "acl": null, "bucket": "${{ secrets.DELTA_AWS_S3_BUCKET_NAME }}", "region": "${{ secrets.DELTA_AWS_REGION}}", "path": "temp-beta", "channel": "beta"}]' electron/package.json > /tmp/package.json mv /tmp/package.json electron/package.json cat electron/package.json @@ -136,7 +136,7 @@ jobs: # check public_provider is true or not echo "public_provider is ${{ inputs.public_provider }}" if [ "${{ inputs.public_provider }}" == "none" ]; then - make build + make build else make build-and-publish fi @@ -160,8 +160,6 @@ jobs: make build-and-publish env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ANALYTICS_ID: ${{ secrets.JAN_APP_UMAMI_PROJECT_API_KEY }} - ANALYTICS_HOST: ${{ secrets.JAN_APP_UMAMI_URL }} AZURE_KEY_VAULT_URI: ${{ secrets.AZURE_KEY_VAULT_URI }} AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} diff --git a/core/src/browser/extension.ts b/core/src/browser/extension.ts index 1d641980b..d768473c9 100644 --- a/core/src/browser/extension.ts +++ b/core/src/browser/extension.ts @@ -12,6 +12,7 @@ export enum ExtensionTypeEnum { SystemMonitoring = 'systemMonitoring', HuggingFace = 'huggingFace', Engine = 'engine', + Hardware = 'hardware', } export interface ExtensionType { diff --git a/core/src/browser/extensions/engines/OAIEngine.test.ts b/core/src/browser/extensions/engines/OAIEngine.test.ts index 81348786c..66537d0be 100644 --- a/core/src/browser/extensions/engines/OAIEngine.test.ts +++ b/core/src/browser/extensions/engines/OAIEngine.test.ts @@ -38,8 +38,14 @@ describe('OAIEngine', () => { it('should subscribe to events on load', () => { engine.onLoad() - expect(events.on).toHaveBeenCalledWith(MessageEvent.OnMessageSent, expect.any(Function)) - expect(events.on).toHaveBeenCalledWith(InferenceEvent.OnInferenceStopped, expect.any(Function)) + expect(events.on).toHaveBeenCalledWith( + MessageEvent.OnMessageSent, + expect.any(Function) + ) + expect(events.on).toHaveBeenCalledWith( + InferenceEvent.OnInferenceStopped, + expect.any(Function) + ) }) it('should handle inference request', async () => { @@ -77,7 +83,12 @@ describe('OAIEngine', () => { expect(events.emit).toHaveBeenCalledWith( MessageEvent.OnMessageUpdate, expect.objectContaining({ - content: [{ type: ContentType.Text, text: { value: 'test response', annotations: [] } }], + content: [ + { + type: ContentType.Text, + text: { value: 'test response', annotations: [] }, + }, + ], status: MessageStatus.Ready, }) ) @@ -101,11 +112,10 @@ describe('OAIEngine', () => { await engine.inference(data) - expect(events.emit).toHaveBeenCalledWith( + expect(events.emit).toHaveBeenLastCalledWith( MessageEvent.OnMessageUpdate, expect.objectContaining({ - content: [{ type: ContentType.Text, text: { value: 'test error', annotations: [] } }], - status: MessageStatus.Error, + status: 'error', error_code: 500, }) ) diff --git a/core/src/browser/extensions/engines/OAIEngine.ts b/core/src/browser/extensions/engines/OAIEngine.ts index 6b4c20a19..61032357c 100644 --- a/core/src/browser/extensions/engines/OAIEngine.ts +++ b/core/src/browser/extensions/engines/OAIEngine.ts @@ -42,7 +42,9 @@ export abstract class OAIEngine extends AIEngine { */ override onLoad() { super.onLoad() - events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => this.inference(data)) + events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => + this.inference(data) + ) events.on(InferenceEvent.OnInferenceStopped, () => this.stopInference()) } @@ -128,7 +130,9 @@ export abstract class OAIEngine extends AIEngine { events.emit(MessageEvent.OnMessageUpdate, message) }, complete: async () => { - message.status = message.content.length ? MessageStatus.Ready : MessageStatus.Error + message.status = message.content.length + ? MessageStatus.Ready + : MessageStatus.Error events.emit(MessageEvent.OnMessageUpdate, message) }, error: async (err: any) => { @@ -141,7 +145,10 @@ export abstract class OAIEngine extends AIEngine { message.content[0] = { type: ContentType.Text, text: { - value: err.message, + value: + typeof message === 'string' + ? err.message + : (JSON.stringify(err.message) ?? err.detail), annotations: [], }, } diff --git a/core/src/browser/extensions/engines/helpers/sse.test.ts b/core/src/browser/extensions/engines/helpers/sse.test.ts index 0b78aa9b5..f8c2ac6b4 100644 --- a/core/src/browser/extensions/engines/helpers/sse.test.ts +++ b/core/src/browser/extensions/engines/helpers/sse.test.ts @@ -1,14 +1,17 @@ import { lastValueFrom, Observable } from 'rxjs' import { requestInference } from './sse' -import { ReadableStream } from 'stream/web'; +import { ReadableStream } from 'stream/web' describe('requestInference', () => { it('should send a request to the inference server and return an Observable', () => { // Mock the fetch function const mockFetch: any = jest.fn(() => Promise.resolve({ ok: true, - json: () => Promise.resolve({ choices: [{ message: { content: 'Generated response' } }] }), + json: () => + Promise.resolve({ + choices: [{ message: { content: 'Generated response' } }], + }), headers: new Headers(), redirected: false, status: 200, @@ -36,7 +39,10 @@ describe('requestInference', () => { const mockFetch: any = jest.fn(() => Promise.resolve({ ok: false, - json: () => Promise.resolve({ error: { message: 'Wrong API Key', code: 'invalid_api_key' } }), + json: () => + Promise.resolve({ + error: { message: 'Invalid API Key.', code: 'invalid_api_key' }, + }), headers: new Headers(), redirected: false, status: 401, @@ -56,69 +62,85 @@ describe('requestInference', () => { // Assert the expected behavior expect(result).toBeInstanceOf(Observable) - expect(lastValueFrom(result)).rejects.toEqual({ message: 'Wrong API Key', code: 'invalid_api_key' }) + expect(lastValueFrom(result)).rejects.toEqual({ + message: 'Invalid API Key.', + code: 'invalid_api_key', + }) }) }) - it('should handle a successful response with a transformResponse function', () => { - // Mock the fetch function - const mockFetch: any = jest.fn(() => - Promise.resolve({ - ok: true, - json: () => Promise.resolve({ choices: [{ message: { content: 'Generated response' } }] }), - headers: new Headers(), - redirected: false, - status: 200, - statusText: 'OK', - }) - ) - jest.spyOn(global, 'fetch').mockImplementation(mockFetch) - - // Define the test inputs - const inferenceUrl = 'https://inference-server.com' - const requestBody = { message: 'Hello' } - const model = { id: 'model-id', parameters: { stream: false } } - const transformResponse = (data: any) => data.choices[0].message.content.toUpperCase() - - // Call the function - const result = requestInference(inferenceUrl, requestBody, model, undefined, undefined, transformResponse) - - // Assert the expected behavior - expect(result).toBeInstanceOf(Observable) - expect(lastValueFrom(result)).resolves.toEqual('GENERATED RESPONSE') - }) - - - it('should handle a successful response with streaming enabled', () => { - // Mock the fetch function - const mockFetch: any = jest.fn(() => - Promise.resolve({ - ok: true, - body: new ReadableStream({ - start(controller) { - controller.enqueue(new TextEncoder().encode('data: {"choices": [{"delta": {"content": "Streamed"}}]}')); - controller.enqueue(new TextEncoder().encode('data: [DONE]')); - controller.close(); - } +it('should handle a successful response with a transformResponse function', () => { + // Mock the fetch function + const mockFetch: any = jest.fn(() => + Promise.resolve({ + ok: true, + json: () => + Promise.resolve({ + choices: [{ message: { content: 'Generated response' } }], }), - headers: new Headers(), - redirected: false, - status: 200, - statusText: 'OK', - }) - ); - jest.spyOn(global, 'fetch').mockImplementation(mockFetch); - - // Define the test inputs - const inferenceUrl = 'https://inference-server.com'; - const requestBody = { message: 'Hello' }; - const model = { id: 'model-id', parameters: { stream: true } }; - - // Call the function - const result = requestInference(inferenceUrl, requestBody, model); - - // Assert the expected behavior - expect(result).toBeInstanceOf(Observable); - expect(lastValueFrom(result)).resolves.toEqual('Streamed'); - }); + headers: new Headers(), + redirected: false, + status: 200, + statusText: 'OK', + }) + ) + jest.spyOn(global, 'fetch').mockImplementation(mockFetch) + // Define the test inputs + const inferenceUrl = 'https://inference-server.com' + const requestBody = { message: 'Hello' } + const model = { id: 'model-id', parameters: { stream: false } } + const transformResponse = (data: any) => + data.choices[0].message.content.toUpperCase() + + // Call the function + const result = requestInference( + inferenceUrl, + requestBody, + model, + undefined, + undefined, + transformResponse + ) + + // Assert the expected behavior + expect(result).toBeInstanceOf(Observable) + expect(lastValueFrom(result)).resolves.toEqual('GENERATED RESPONSE') +}) + +it('should handle a successful response with streaming enabled', () => { + // Mock the fetch function + const mockFetch: any = jest.fn(() => + Promise.resolve({ + ok: true, + body: new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode( + 'data: {"choices": [{"delta": {"content": "Streamed"}}]}' + ) + ) + controller.enqueue(new TextEncoder().encode('data: [DONE]')) + controller.close() + }, + }), + headers: new Headers(), + redirected: false, + status: 200, + statusText: 'OK', + }) + ) + jest.spyOn(global, 'fetch').mockImplementation(mockFetch) + + // Define the test inputs + const inferenceUrl = 'https://inference-server.com' + const requestBody = { message: 'Hello' } + const model = { id: 'model-id', parameters: { stream: true } } + + // Call the function + const result = requestInference(inferenceUrl, requestBody, model) + + // Assert the expected behavior + expect(result).toBeInstanceOf(Observable) + expect(lastValueFrom(result)).resolves.toEqual('Streamed') +}) diff --git a/core/src/browser/extensions/engines/helpers/sse.ts b/core/src/browser/extensions/engines/helpers/sse.ts index 55cde56b4..5c63008ff 100644 --- a/core/src/browser/extensions/engines/helpers/sse.ts +++ b/core/src/browser/extensions/engines/helpers/sse.ts @@ -32,20 +32,19 @@ export function requestInference( }) .then(async (response) => { if (!response.ok) { - const data = await response.json() - let errorCode = ErrorCode.Unknown - if (data.error) { - errorCode = data.error.code ?? data.error.type ?? ErrorCode.Unknown - } else if (response.status === 401) { - errorCode = ErrorCode.InvalidApiKey + if (response.status === 401) { + throw { + code: ErrorCode.InvalidApiKey, + message: 'Invalid API Key.', + } } - const error = { - message: data.error?.message ?? data.message ?? 'Error occurred.', - code: errorCode, + let data = await response.json() + try { + handleError(data) + } catch (err) { + subscriber.error(err) + return } - subscriber.error(error) - subscriber.complete() - return } // There could be overriden stream parameter in the model // that is set in request body (transformed payload) @@ -54,9 +53,10 @@ export function requestInference( model.parameters?.stream === false ) { const data = await response.json() - if (data.error || data.message) { - subscriber.error(data.error ?? data) - subscriber.complete() + try { + handleError(data) + } catch (err) { + subscriber.error(err) return } if (transformResponse) { @@ -91,13 +91,10 @@ export function requestInference( const toParse = cachedLines + line if (!line.includes('data: [DONE]')) { const data = JSON.parse(toParse.replace('data: ', '')) - if ( - 'error' in data || - 'message' in data || - 'detail' in data - ) { - subscriber.error(data.error ?? data) - subscriber.complete() + try { + handleError(data) + } catch (err) { + subscriber.error(err) return } content += data.choices[0]?.delta?.content ?? '' @@ -118,3 +115,18 @@ export function requestInference( .catch((err) => subscriber.error(err)) }) } + +/** + * Handle error and normalize it to a common format. + * @param data + */ +const handleError = (data: any) => { + if ( + data.error || + data.message || + data.detail || + (Array.isArray(data) && data.length && data[0].error) + ) { + throw data.error ?? data[0]?.error ?? data + } +} diff --git a/core/src/browser/extensions/enginesManagement.ts b/core/src/browser/extensions/enginesManagement.ts index 66dff87df..6986e8f48 100644 --- a/core/src/browser/extensions/enginesManagement.ts +++ b/core/src/browser/extensions/enginesManagement.ts @@ -5,6 +5,7 @@ import { EngineReleased, EngineConfig, DefaultEngineVariant, + Model, } from '../../types' import { BaseExtension, ExtensionTypeEnum } from '../extension' @@ -103,6 +104,11 @@ export abstract class EngineManagementExtension extends BaseExtension { engineConfig?: EngineConfig ): Promise<{ messages: string }> + /** + * Add a new remote model for a specific engine + */ + abstract addRemoteModel(model: Model): Promise + /** * @returns A Promise that resolves to an object of remote models list . */ diff --git a/core/src/browser/extensions/hardwareManagement.ts b/core/src/browser/extensions/hardwareManagement.ts new file mode 100644 index 000000000..1f7c36287 --- /dev/null +++ b/core/src/browser/extensions/hardwareManagement.ts @@ -0,0 +1,26 @@ +import { HardwareInformation } from '../../types' +import { BaseExtension, ExtensionTypeEnum } from '../extension' + +/** + * Engine management extension. Persists and retrieves engine management. + * @abstract + * @extends BaseExtension + */ +export abstract class HardwareManagementExtension extends BaseExtension { + type(): ExtensionTypeEnum | undefined { + return ExtensionTypeEnum.Hardware + } + + /** + * @returns A Promise that resolves to an object of list hardware. + */ + abstract getHardware(): Promise + + /** + * @returns A Promise that resolves to an object of set active gpus. + */ + abstract setAvtiveGpu(data: { gpus: number[] }): Promise<{ + message: string + activated_gpus: number[] + }> +} diff --git a/core/src/browser/extensions/index.test.ts b/core/src/browser/extensions/index.test.ts index 26cbda8c5..bc5a7c358 100644 --- a/core/src/browser/extensions/index.test.ts +++ b/core/src/browser/extensions/index.test.ts @@ -1,6 +1,5 @@ import { ConversationalExtension } from './index'; import { InferenceExtension } from './index'; -import { MonitoringExtension } from './index'; import { AssistantExtension } from './index'; import { ModelExtension } from './index'; import * as Engines from './index'; @@ -14,10 +13,6 @@ describe('index.ts exports', () => { expect(InferenceExtension).toBeDefined(); }); - test('should export MonitoringExtension', () => { - expect(MonitoringExtension).toBeDefined(); - }); - test('should export AssistantExtension', () => { expect(AssistantExtension).toBeDefined(); }); @@ -29,4 +24,4 @@ describe('index.ts exports', () => { test('should export Engines', () => { expect(Engines).toBeDefined(); }); -}); \ No newline at end of file +}); diff --git a/core/src/browser/extensions/index.ts b/core/src/browser/extensions/index.ts index 9dbfe1afe..f11c7b09f 100644 --- a/core/src/browser/extensions/index.ts +++ b/core/src/browser/extensions/index.ts @@ -9,10 +9,7 @@ export { ConversationalExtension } from './conversational' */ export { InferenceExtension } from './inference' -/** - * Monitoring extension for system monitoring. - */ -export { MonitoringExtension } from './monitoring' + /** * Assistant extension for managing assistants. @@ -33,3 +30,8 @@ export * from './engines' * Engines Management */ export * from './enginesManagement' + +/** + * Hardware Management + */ +export * from './hardwareManagement' diff --git a/core/src/browser/extensions/monitoring.test.ts b/core/src/browser/extensions/monitoring.test.ts deleted file mode 100644 index 9bba89a8c..000000000 --- a/core/src/browser/extensions/monitoring.test.ts +++ /dev/null @@ -1,42 +0,0 @@ - -import { ExtensionTypeEnum } from '../extension'; -import { MonitoringExtension } from './monitoring'; - -it('should have the correct type', () => { - class TestMonitoringExtension extends MonitoringExtension { - getGpuSetting(): Promise { - throw new Error('Method not implemented.'); - } - getResourcesInfo(): Promise { - throw new Error('Method not implemented.'); - } - getCurrentLoad(): Promise { - throw new Error('Method not implemented.'); - } - getOsInfo(): Promise { - throw new Error('Method not implemented.'); - } - } - const monitoringExtension = new TestMonitoringExtension(); - expect(monitoringExtension.type()).toBe(ExtensionTypeEnum.SystemMonitoring); -}); - - -it('should create an instance of MonitoringExtension', () => { - class TestMonitoringExtension extends MonitoringExtension { - getGpuSetting(): Promise { - throw new Error('Method not implemented.'); - } - getResourcesInfo(): Promise { - throw new Error('Method not implemented.'); - } - getCurrentLoad(): Promise { - throw new Error('Method not implemented.'); - } - getOsInfo(): Promise { - throw new Error('Method not implemented.'); - } - } - const monitoringExtension = new TestMonitoringExtension(); - expect(monitoringExtension).toBeInstanceOf(MonitoringExtension); -}); diff --git a/core/src/browser/extensions/monitoring.ts b/core/src/browser/extensions/monitoring.ts deleted file mode 100644 index cb544b6b7..000000000 --- a/core/src/browser/extensions/monitoring.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { BaseExtension, ExtensionTypeEnum } from '../extension' -import { GpuSetting, MonitoringInterface, OperatingSystemInfo } from '../../types' - -/** - * Monitoring extension for system monitoring. - * @extends BaseExtension - */ -export abstract class MonitoringExtension extends BaseExtension implements MonitoringInterface { - /** - * Monitoring extension type. - */ - type(): ExtensionTypeEnum | undefined { - return ExtensionTypeEnum.SystemMonitoring - } - - abstract getGpuSetting(): Promise - abstract getResourcesInfo(): Promise - abstract getCurrentLoad(): Promise - abstract getOsInfo(): Promise -} diff --git a/core/src/types/config/appConfigEntity.ts b/core/src/types/config/appConfigEntity.ts index 1402aeca1..bd352d22f 100644 --- a/core/src/types/config/appConfigEntity.ts +++ b/core/src/types/config/appConfigEntity.ts @@ -1,4 +1,5 @@ export type AppConfiguration = { data_folder: string quick_ask: boolean + distinct_id?: string } diff --git a/core/src/types/engine/index.ts b/core/src/types/engine/index.ts index 7c848a279..9a6beeeff 100644 --- a/core/src/types/engine/index.ts +++ b/core/src/types/engine/index.ts @@ -18,6 +18,7 @@ export type EngineMetadata = { template?: string } } + explore_models_url?: string } export type EngineVariant = { diff --git a/core/src/types/hardware/index.ts b/core/src/types/hardware/index.ts new file mode 100644 index 000000000..d154a4417 --- /dev/null +++ b/core/src/types/hardware/index.ts @@ -0,0 +1,55 @@ +export type Cpu = { + arch: string + cores: number + instructions: string[] + model: string + usage: number +} + +export type GpuAdditionalInformation = { + compute_cap: string + driver_version: string +} + +export type Gpu = { + activated: boolean + additional_information?: GpuAdditionalInformation + free_vram: number + id: string + name: string + total_vram: number + uuid: string + version: string +} + +export type Os = { + name: string + version: string +} + +export type Power = { + battery_life: number + charging_status: string + is_power_saving: boolean +} + +export type Ram = { + available: number + total: number + type: string +} + +export type Storage = { + available: number + total: number + type: string +} + +export type HardwareInformation = { + cpu: Cpu + gpus: Gpu[] + os: Os + power: Power + ram: Ram + storage: Storage +} diff --git a/core/src/types/index.test.ts b/core/src/types/index.test.ts index 9dc001c4d..d938feee9 100644 --- a/core/src/types/index.test.ts +++ b/core/src/types/index.test.ts @@ -4,7 +4,6 @@ import * as model from './model'; import * as thread from './thread'; import * as message from './message'; import * as inference from './inference'; -import * as monitoring from './monitoring'; import * as file from './file'; import * as config from './config'; import * as huggingface from './huggingface'; @@ -18,7 +17,6 @@ import * as setting from './setting'; expect(thread).toBeDefined(); expect(message).toBeDefined(); expect(inference).toBeDefined(); - expect(monitoring).toBeDefined(); expect(file).toBeDefined(); expect(config).toBeDefined(); expect(huggingface).toBeDefined(); diff --git a/core/src/types/index.ts b/core/src/types/index.ts index e30dd18c3..3d262a6b7 100644 --- a/core/src/types/index.ts +++ b/core/src/types/index.ts @@ -3,7 +3,6 @@ export * from './model' export * from './thread' export * from './message' export * from './inference' -export * from './monitoring' export * from './file' export * from './config' export * from './huggingface' @@ -11,3 +10,4 @@ export * from './miscellaneous' export * from './api' export * from './setting' export * from './engine' +export * from './hardware' diff --git a/core/src/types/miscellaneous/systemResourceInfo.ts b/core/src/types/miscellaneous/systemResourceInfo.ts index 82db5d941..9361b79b6 100644 --- a/core/src/types/miscellaneous/systemResourceInfo.ts +++ b/core/src/types/miscellaneous/systemResourceInfo.ts @@ -1,33 +1,25 @@ +import { GpuAdditionalInformation } from '../hardware' + export type SystemResourceInfo = { memAvailable: number } -export type RunMode = 'cpu' | 'gpu' - export type GpuSetting = { - notify: boolean - run_mode: RunMode - nvidia_driver: { - exist: boolean - version: string - } - cuda: { - exist: boolean - version: string - } gpus: GpuSettingInfo[] - gpu_highest_vram: string - gpus_in_use: string[] - is_initial: boolean // TODO: This needs to be set based on user toggle in settings vulkan: boolean + cpu?: any } export type GpuSettingInfo = { + activated: boolean + free_vram: number id: string - vram: string name: string - arch?: string + total_vram: number + uuid: string + version: string + additional_information?: GpuAdditionalInformation } export type SystemInformation = { @@ -42,9 +34,6 @@ export type SupportedPlatform = SupportedPlatformTuple[number] export type OperatingSystemInfo = { platform: SupportedPlatform | 'unknown' arch: string - release: string - machine: string - version: string totalMem: number freeMem: number } diff --git a/core/src/types/model/modelEntity.ts b/core/src/types/model/modelEntity.ts index 7b67a8e94..6e47c9ae4 100644 --- a/core/src/types/model/modelEntity.ts +++ b/core/src/types/model/modelEntity.ts @@ -1,5 +1,3 @@ -import { FileMetadata } from '../file' - /** * Represents the information about a model. * @stored @@ -70,6 +68,11 @@ export type Model = { */ id: string + /** + * The model identifier, modern version of id. + */ + model?: string + /** * Human-readable name that is used for UI. */ @@ -147,6 +150,7 @@ export type ModelSettingParams = { */ export type ModelRuntimeParams = { temperature?: number + max_temperature?: number token_limit?: number top_k?: number top_p?: number diff --git a/core/src/types/monitoring/index.test.ts b/core/src/types/monitoring/index.test.ts deleted file mode 100644 index 56c5879e4..000000000 --- a/core/src/types/monitoring/index.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import * as monitoringInterface from './monitoringInterface' -import * as resourceInfo from './resourceInfo' - -import * as index from './index' - -it('should re-export all symbols from monitoringInterface and resourceInfo', () => { - for (const key in monitoringInterface) { - expect(index[key]).toBe(monitoringInterface[key]) - } - for (const key in resourceInfo) { - expect(index[key]).toBe(resourceInfo[key]) - } -}) diff --git a/core/src/types/monitoring/index.ts b/core/src/types/monitoring/index.ts deleted file mode 100644 index b96c518fd..000000000 --- a/core/src/types/monitoring/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './monitoringInterface' -export * from './resourceInfo' diff --git a/core/src/types/monitoring/monitoringInterface.ts b/core/src/types/monitoring/monitoringInterface.ts deleted file mode 100644 index 5ab1394a1..000000000 --- a/core/src/types/monitoring/monitoringInterface.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { GpuSetting, OperatingSystemInfo } from '../miscellaneous' - -/** - * Monitoring extension for system monitoring. - * @extends BaseExtension - */ -export interface MonitoringInterface { - /** - * Returns information about the system resources. - * @returns {Promise} A promise that resolves with the system resources information. - */ - getResourcesInfo(): Promise - - /** - * Returns the current system load. - * @returns {Promise} A promise that resolves with the current system load. - */ - getCurrentLoad(): Promise - - /** - * Returns the GPU configuration. - */ - getGpuSetting(): Promise - - /** - * Returns information about the operating system. - */ - getOsInfo(): Promise -} diff --git a/core/src/types/monitoring/resourceInfo.ts b/core/src/types/monitoring/resourceInfo.ts deleted file mode 100644 index b19da5462..000000000 --- a/core/src/types/monitoring/resourceInfo.ts +++ /dev/null @@ -1,6 +0,0 @@ -export type ResourceInfo = { - mem: { - totalMemory: number - usedMemory: number - } -} diff --git a/docs/src/components/DropdownDownload/index.tsx b/docs/src/components/DropdownDownload/index.tsx index 26e0f49d6..c0cdfb73b 100644 --- a/docs/src/components/DropdownDownload/index.tsx +++ b/docs/src/components/DropdownDownload/index.tsx @@ -65,7 +65,7 @@ const DropdownDownload = ({ lastRelease }: Props) => { const userAgent = navigator.userAgent if (userAgent.includes('Windows')) { // windows user - setDefaultSystem(systems[2]) + setDefaultSystem(systems[1]) } else if (userAgent.includes('Linux')) { // linux user setDefaultSystem(systems[3]) diff --git a/docs/src/pages/about/handbook/analytics.mdx b/docs/src/pages/about/handbook/analytics.mdx index 9e7833e32..5cc34209d 100644 --- a/docs/src/pages/about/handbook/analytics.mdx +++ b/docs/src/pages/about/handbook/analytics.mdx @@ -23,6 +23,4 @@ Adhering to Jan's privacy preserving philosophy, our analytics philosophy is to ## What is tracked 1. By default, Github tracks downloads and device metadata for all public GitHub repositories. This helps us troubleshoot & ensure cross-platform support. -2. We use [Umami](https://umami.is/) to collect, analyze, and understand application data while maintaining visitor privacy and data ownership. We are using the Umami Cloud in Europe to ensure GDPR compliance. Please see [Umami Privacy Policy](https://umami.is/privacy) for more details. -3. We use Umami to track a single `app.opened` event without additional user metadata, in order to understand retention. In addition, we track `app.version` to understand app version usage. -4. Additionally, we plan to enable a `Settings` feature for users to turn off all tracking. +2. Additionally, we plan to enable a `Settings` feature for users to turn off all tracking. diff --git a/docs/src/pages/docs/_meta.json b/docs/src/pages/docs/_meta.json index b395ff7af..4f7852626 100644 --- a/docs/src/pages/docs/_meta.json +++ b/docs/src/pages/docs/_meta.json @@ -14,6 +14,11 @@ "desktop": "Installation", "data-folder": "Jan Data Folder", "privacy": "Privacy", + "privacy-policy": { + "type": "page", + "display": "hidden", + "title": "Privacy Policy" + }, "user-guides": { "title": "BASIC USAGE", "type": "separator" diff --git a/docs/src/pages/docs/configure-extensions.mdx b/docs/src/pages/docs/configure-extensions.mdx index c72d8c6a5..71d226554 100644 --- a/docs/src/pages/docs/configure-extensions.mdx +++ b/docs/src/pages/docs/configure-extensions.mdx @@ -55,7 +55,7 @@ Extensions in Jan are configured through a JSON file that defines their behavior "productName": "Conversational", "version": "1.0.0", "main": "dist/index.js", - "description": "This extension enables conversations and state persistence via your filesystem", + "description": "This extension enables conversations and state persistence via your filesystem.", "url": "extension://@janhq/conversational-extension/dist/index.js" }, "@janhq/inference-anthropic-extension": { @@ -70,7 +70,7 @@ Extensions in Jan are configured through a JSON file that defines their behavior "productName": "Anthropic Inference Engine", "version": "1.0.2", "main": "dist/index.js", - "description": "This extension enables Anthropic chat completion API calls", + "description": "This extension enables Anthropic chat completion API calls.", "url": "extension://@janhq/inference-anthropic-extension/dist/index.js" }, "@janhq/inference-triton-trt-llm-extension": { @@ -85,7 +85,7 @@ Extensions in Jan are configured through a JSON file that defines their behavior "productName": "Triton-TRT-LLM Inference Engine", "version": "1.0.0", "main": "dist/index.js", - "description": "This extension enables Nvidia's TensorRT-LLM as an inference engine option", + "description": "This extension enables Nvidia's TensorRT-LLM as an inference engine option.", "url": "extension://@janhq/inference-triton-trt-llm-extension/dist/index.js" }, "@janhq/inference-mistral-extension": { @@ -100,7 +100,7 @@ Extensions in Jan are configured through a JSON file that defines their behavior "productName": "MistralAI Inference Engine", "version": "1.0.1", "main": "dist/index.js", - "description": "This extension enables Mistral chat completion API calls", + "description": "This extension enables Mistral chat completion API calls.", "url": "extension://@janhq/inference-mistral-extension/dist/index.js" }, "@janhq/inference-martian-extension": { @@ -115,7 +115,7 @@ Extensions in Jan are configured through a JSON file that defines their behavior "productName": "Martian Inference Engine", "version": "1.0.1", "main": "dist/index.js", - "description": "This extension enables Martian chat completion API calls", + "description": "This extension enables Martian chat completion API calls.", "url": "extension://@janhq/inference-martian-extension/dist/index.js" }, "@janhq/inference-openrouter-extension": { @@ -130,7 +130,7 @@ Extensions in Jan are configured through a JSON file that defines their behavior "productName": "OpenRouter Inference Engine", "version": "1.0.0", "main": "dist/index.js", - "description": "This extension enables Open Router chat completion API calls", + "description": "This extension enables Open Router chat completion API calls.", "url": "extension://@janhq/inference-openrouter-extension/dist/index.js" }, "@janhq/inference-nvidia-extension": { @@ -145,7 +145,7 @@ Extensions in Jan are configured through a JSON file that defines their behavior "productName": "NVIDIA NIM Inference Engine", "version": "1.0.1", "main": "dist/index.js", - "description": "This extension enables NVIDIA chat completion API calls", + "description": "This extension enables NVIDIA chat completion API calls.", "url": "extension://@janhq/inference-nvidia-extension/dist/index.js" }, "@janhq/inference-groq-extension": { @@ -160,7 +160,7 @@ Extensions in Jan are configured through a JSON file that defines their behavior "productName": "Groq Inference Engine", "version": "1.0.1", "main": "dist/index.js", - "description": "This extension enables fast Groq chat completion API calls", + "description": "This extension enables fast Groq chat completion API calls.", "url": "extension://@janhq/inference-groq-extension/dist/index.js" }, "@janhq/inference-openai-extension": { @@ -175,7 +175,7 @@ Extensions in Jan are configured through a JSON file that defines their behavior "productName": "OpenAI Inference Engine", "version": "1.0.2", "main": "dist/index.js", - "description": "This extension enables OpenAI chat completion API calls", + "description": "This extension enables OpenAI chat completion API calls.", "url": "extension://@janhq/inference-openai-extension/dist/index.js" }, "@janhq/inference-cohere-extension": { @@ -190,7 +190,7 @@ Extensions in Jan are configured through a JSON file that defines their behavior "productName": "Cohere Inference Engine", "version": "1.0.0", "main": "dist/index.js", - "description": "This extension enables Cohere chat completion API calls", + "description": "This extension enables Cohere chat completion API calls.", "url": "extension://@janhq/inference-cohere-extension/dist/index.js" }, "@janhq/model-extension": { @@ -205,7 +205,7 @@ Extensions in Jan are configured through a JSON file that defines their behavior "productName": "Model Management", "version": "1.0.33", "main": "dist/index.js", - "description": "Model Management Extension provides model exploration and seamless downloads", + "description": "Model Management Extension provides model exploration and seamless downloads.", "url": "extension://@janhq/model-extension/dist/index.js" }, "@janhq/monitoring-extension": { @@ -220,7 +220,7 @@ Extensions in Jan are configured through a JSON file that defines their behavior "productName": "System Monitoring", "version": "1.0.10", "main": "dist/index.js", - "description": "This extension provides system health and OS level data", + "description": "This extension provides system health and OS level data.", "url": "extension://@janhq/monitoring-extension/dist/index.js" }, "@janhq/assistant-extension": { @@ -235,7 +235,7 @@ Extensions in Jan are configured through a JSON file that defines their behavior "productName": "Jan Assistant", "version": "1.0.1", "main": "dist/index.js", - "description": "This extension enables assistants, including Jan, a default assistant that can call all downloaded models", + "description": "This extension enables assistants, including Jan, a default assistant that can call all downloaded models.", "url": "extension://@janhq/assistant-extension/dist/index.js" }, "@janhq/tensorrt-llm-extension": { diff --git a/docs/src/pages/docs/desktop/mac.mdx b/docs/src/pages/docs/desktop/mac.mdx index 6dac155dc..c8da743a6 100644 --- a/docs/src/pages/docs/desktop/mac.mdx +++ b/docs/src/pages/docs/desktop/mac.mdx @@ -41,7 +41,7 @@ Ensure that your system meets the following requirements to use Jan effectively: ### Mac Performance Guide -**Apple Silicon Macs** leverage Metal for GPU acceleration, providing faster performance than **Appple Intel Macs**, which rely solely on CPU processing. +**Apple Silicon Macs** leverage Metal for GPU acceleration, providing faster performance than **Apple Intel Macs**, which rely solely on CPU processing. **Apple Silicon (M1, M2, M3)** - Metal acceleration enabled by default, no configuration required diff --git a/docs/src/pages/docs/install-engines.mdx b/docs/src/pages/docs/install-engines.mdx index 4e99d337d..cded71c83 100644 --- a/docs/src/pages/docs/install-engines.mdx +++ b/docs/src/pages/docs/install-engines.mdx @@ -47,8 +47,8 @@ To add a new remote engine: |-------|-------------|----------| | Engine Name | Name for your engine (e.g., "OpenAI", "Claude") | ✓ | | API URL | The base URL of the provider's API | ✓ | -| API Key | Your authentication key from the provider | ✓ | -| Model List URL | URL for fetching available models | | +| API Key | Your authentication key to activate this engine | ✓ | +| Model List URL | The endpoint URL to fetch available models | | API Key Template | Custom authorization header format | | | Request Format Conversion | Function to convert Jan's request format to provider's format | | | Response Format Conversion | Function to convert provider's response format to Jan's format | | diff --git a/docs/src/pages/docs/privacy-policy.mdx b/docs/src/pages/docs/privacy-policy.mdx new file mode 100644 index 000000000..525e10091 --- /dev/null +++ b/docs/src/pages/docs/privacy-policy.mdx @@ -0,0 +1,125 @@ +--- +title: Jan Privacy Policy +description: Jan's data collection practices, privacy measures, and your rights. Learn how we protect your data and maintain transparency. +--- + +# Privacy Policy + +
+ Last Updated: January 16, 2025 +
+ +## Introduction + +We are committed to protecting your privacy and ensuring you have control over your data. This Privacy Policy outlines what information Menlo Research Pte Ltd (the "Company") collects from users of the Jan desktop app and website (the "Services"), how the Company uses that information, and the measures the Company takes to safeguard that information. + +## 1. Data Collection and Consent + +### Explicit Consent + +The Company does not collect any data until you explicitly allow tracking. + +### Tracking Preferences + +Upon first launching the Jan desktop app or visiting the website, you will be prompted to set your tracking preferences. These preferences can be modified at any time via the app's Settings menu or the website's Privacy Settings. + +### Legal Basis + +Pursuant to the European Union's General Data Protection Regulation (EU) 2016/679 (the "GDPR"), the Company processes data based on your explicit consent (GDPR Article 6(1)(a)). This means: + +- The Company only processes your data after receiving clear, affirmative consent from you. +- You may withdraw your consent at any time through the app's Settings menu or the website's Privacy Settings. +- If you withdraw your consent, the Company will stop optional data collection from the effective date of withdrawal. +- Your withdrawal of consent will not affect the lawfulness of processing before its withdrawal. + +## 2. Data We Do Not Collect + +Regardless of your analytics permissions, the Company does not collect the following: + +- Chat History: Your conversations with the Jan app are private and inaccessible to the Company. +- Chat Settings: Your personalized settings remain solely with you. +- Language Models: The specific language models you use are not tracked. + +## 3. Uses of Information + +To build a reliable and user-friendly product offering, understanding how the Jan app is used is essential. If you permit tracking, the Company collects product analytics data to: + +- Improve User Experience: Enhance app functionality based on usage patterns; and +- Measure Engagement: Assess active users and retention rates to ensure ongoing value. + +## 4. Product Analytics + +### Data Collected + +When you opt-in to tracking, we collect the following anonymous data: + +- Active Users: Number of daily active users to gauge engagement. +- Retention Rates: Track if users continue to find value in the Jan app over time. + +### Data Anonymity + +- User ID: Analytics data is tied to a randomly generated user ID, ensuring no link to your personal identity. +- Privacy Assurance: Your chat history and personal data are not tracked or linked to your usage data. + +## 5. What We Do Not Track + +Even with analytics permissions granted, the Company does not track the following: + +- Conversations: Your interactions with the Jan app remain private. +- Files: The Company does not scan, upload, or view your files. +- Personal Identity: The Company does not collect personally identifiable information about users. +- Prompts: Your prompts and prompt templates are not monitored. +- Conversation Metrics: The Company does not track context length or conversation length. +- Model Usage: The specific models you use or their types are not tracked. +- Storage: You retain full control over storing your files and logs, and your privacy is prioritized. + +## 6. Using Cloud Models + +The Jan app allows you to connect to cloud-based model APIs (e.g. GPT, Claude models). + +- Data Handling: The API provider processes your messages directly; the Jan app does not access or store these messages. +- Local Models: Choosing local models ensures all data remains on your device, with no external access. + +## 7. Data Storage and Processing + +### Analytics Provider + +The Company uses PostHog EU for analytics, which ensures all data is processed within the European Union. + +### Data Security + +- Encryption: All data transfers are encrypted using Transport Layer Security (TLS) to ensure secure transmission. +- Storage: PostHog securely manages the data the Company collects. For more information, please refer to PostHog's GDPR documentation. + +## 8. Data Retention + +- Retention Period: The Company retains analytics data for up to 12 months unless otherwise required to comply with any applicable legal requirements. +- Deletion Requests: If you wish to request the deletion of your analytics data, you may do so by sending a written request to hello@jan.ai. + +## 9. Your Rights and Choices + +- Access and Control: You may access, modify, or delete your tracking preferences at any time through the Jan app or website settings. +- Data Requests: If you have any requests related to your data, please address them to hello@jan.ai. + +## 10. Children's Privacy + +Our Services are not targeted at children under the age of 13. The Company does not knowingly collect data from children under the age of 13. If the Company becomes aware that data of persons under the age of 13 has been collected without verifiable parental consent, the Company will take appropriate actions to delete this information. + +## 11. Changes to the Privacy Policy + +The Company reserves the right, at its sole discretion, to update this Privacy Policy at any time to reflect changes in the practices or legal requirements of the Company. The Company will use reasonable efforts to notify you of any significant changes via app notifications, the website, or email. Your continued use of the Services following such updates means you accept those changes. + +## 12. Cookies and Tracking Technologies + +Our website utilizes cookies to: + +- Enhance user experience; and +- Measure website traffic and usage patterns. + +Most browsers allow you to remove or manage cookie functions and adjust your privacy and security preferences. + +For more details, please refer to our Cookie Policy. + +## 13. Contact Us + +For any questions or concerns about this Privacy Policy or our data practices, please contact hello@jan.ai. \ No newline at end of file diff --git a/docs/src/pages/docs/privacy.mdx b/docs/src/pages/docs/privacy.mdx index d3be5b6de..84d9ba61c 100644 --- a/docs/src/pages/docs/privacy.mdx +++ b/docs/src/pages/docs/privacy.mdx @@ -1,5 +1,5 @@ --- -title: Jan Privacy +title: Jan's Privacy Approach description: Jan is an app that allows you to own your AI. We prioritize your control over your data and explain what data we collect and why. keywords: [ @@ -19,45 +19,57 @@ keywords: ] --- +--- + +import { Callout } from 'nextra/components' + # Privacy -Jan is an app that allows you to own your AI. We prioritize your control over your data and explain what data we collect and why. +Jan is an app that allows you to own your AI. We prioritize local AI models and your control over your data. This page explains what data we collect and why. No tricks. -- Jan can't see your chats with AI -- You're free to opt out + +For a comprehensive overview of our privacy practices, you can read our full [Privacy Policy](/docs/privacy-policy). + + + +We don't collect any data until you explicitly allow tracking. + + +You'll be asked about your tracking preferences when you first launch the app, and you can change them at any time in Settings. + +Regardless of your analytics permissions, Jan will **never** access your chat history, chat settings, or the language models you have used. ## Why and what we track -To build a reliable, user-friendly AI that you own, we need to understand how Jan is used. We collect two types of data: performance data and usage data. +To build a reliable, user-friendly AI that you own, we need to understand how Jan is used. If users allowed us to track, we collect product analytics data. -### Performance data -We track app crashes and collect technical details about what went wrong, along with basic information about the hardware you’re using. - -When Jan crashes, we collect technical details about what went wrong. - -- Specific AI model in use during the crash -- Hardware: `CPU`, `GPU`, `RAM` -- Logs: `Date/Time`, `OS & version`, `app version`, `error codes & messages`. - -### Usage data +### Product Analytics We track data like how often the app is opened to check: - **Active Users**: How many people use Jan daily to measure engagement - **Retention Rates**: To understand if users are finding value in Jan over time -Usage data is tied to a randomly generated telemetry ID. None of our usage data can be linked to your personal identity. +Product analytics data is tied to a randomly generated user ID. None of our usage data can be linked to your personal identity. Your chat history and personal data are never tracked. -## What we **don’t** track: -- Your conversations with Jan. Those stay on your device. -- Your files. We don’t scan, upload, or even look at them. -- Anything tied to your identity. +## What we **don't** track + + +Even if you grant analytics permissions, Jan doesn't track many of your private activities. + + +- We don't track your conversations with Jan. +- We don't scan, upload, or look at your files. +- We don't collect anything tied to your identity. +- We don't track your prompts and prompt templates. +- We don't monitor context length or conversation length. +- We don't track the models you have used or their types. + +You store the files and logs that are a priority for your privacy yourself. ## Using Cloud Models Jan allows you to connect cloud model APIs. If you choose to use cloud-based models (e.g. GPT, Claude models), the API provider handling the model will have access to your messages as part of processing the request. Again, Jan doesn't see or store these messages - they go directly to the provider. Remember: With local models, everything stays on your device, so no one - not even us- can see your messages. ## Where we store & process data -We use [PostHog](https://posthog.com/eu) EU for analytics, ensuring all data is processed within the European Union. This setup complies with GDPR and other strict privacy regulations. PostHog lets us self-host and securely manage the data we collect. Read more [on PostHog's GDPR doc](https://posthog.com/docs/privacy/gdpr-compliance). - -For a detailed breakdown of the analytics data we collect, you can check out our analytics repo. If you have any questions or concerns, feel free to reach out to us at hi@jan.ai. \ No newline at end of file +We use [PostHog](https://posthog.com/eu) EU for analytics, ensuring all data is processed within the European Union. This setup complies with GDPR and other strict privacy regulations. PostHog lets us securely manage the data we collect. Read more [on PostHog's GDPR doc](https://posthog.com/docs/privacy/gdpr-compliance). \ No newline at end of file diff --git a/docs/src/pages/index.mdx b/docs/src/pages/index.mdx index 4a22a7174..ef3484297 100644 --- a/docs/src/pages/index.mdx +++ b/docs/src/pages/index.mdx @@ -1,6 +1,6 @@ --- title: "Jan: Open source ChatGPT-alternative that runs 100% offline" -description: "Chat with AI without privact concerns. Jan is an open-source alternative to ChatGPT, running AI models locally on your device." +description: "Chat with AI without privacy concerns. Jan is an open-source alternative to ChatGPT, running AI models locally on your device." keywords: [ Jan, diff --git a/docs/src/pages/integrations/workflow-automation/raycast.mdx b/docs/src/pages/integrations/workflow-automation/raycast.mdx deleted file mode 100644 index 01c5a4866..000000000 --- a/docs/src/pages/integrations/workflow-automation/raycast.mdx +++ /dev/null @@ -1,47 +0,0 @@ ---- -title: Raycast -keywords: - [ - Jan, - Customizable Intelligence, LLM, - local AI, - privacy focus, - free and open source, - private and offline, - conversational AI, - no-subscription fee, - large language models, - raycast integration, - Raycast, - ] -description: A step-by-step guide on integrating Jan with Raycast. ---- - -import { Steps } from 'nextra/components' - -# Raycast - -## Integrate Raycast with Jan - -[Raycast](https://www.raycast.com/) is a productivity tool designed for macOS that enhances workflow efficiency by providing quick access to various tasks and functionalities through a keyboard-driven interface. To integrate Raycast with Jan, follow the steps below: - - -### Step 1: Download the TinyLlama Model - -1. Open Jan app. -2. Go to the **Hub** and download the TinyLlama model. -3. The model will be available at `~jan/models/tinyllama-1.1b`. - -### Step 2: Clone and Run the Program - -1. Clone this [GitHub repository](https://github.com/InNoobWeTrust/nitro-raycast). -2. Execute the project using the following command: - -```bash title="Node.js" -npm i && npm run dev -``` - -### Step 3: Search for Nitro and Run the Model - -Search for `Nitro` using the program, and you can use the models from Jan in RayCast. - \ No newline at end of file diff --git a/docs/src/pages/privacy.mdx b/docs/src/pages/privacy.mdx index bb77327e6..44127a37a 100644 --- a/docs/src/pages/privacy.mdx +++ b/docs/src/pages/privacy.mdx @@ -10,27 +10,20 @@ Homebrew Computer Company is committed to protecting your privacy and ensuring t ## Data Collection -Jan, Cortex, and all Homebrew Computer Company products do not collect personally identifying information. You can read about [our philosophy](/about#philosophy) here and audit our open-source codebases. +Jan, Cortex, and all Homebrew Computer Company products do not collect personally identifying information. You can read about [our philosophy](/about#philosophy) here and audit our open-source codebases. ### When you voluntarily provide data -We -do- collect personal information you voluntarily provide us, e.g., when you sign up for our newsletter, join our Discord, or contact us via email. +We -do- collect personal information you voluntarily provide us, e.g., when you sign up for our newsletter, join our Discord, or contact us via email. -### Jan +### Jan -Jan runs with privacy by default and is used 100% offline on your own computer. Your data (e.g., conversation history, usage logs) are stored locally and never leave your computer. +Jan runs with privacy by default and is used 100% offline on your own computer. Your data (e.g., conversation history, usage logs) are stored locally and never leave your computer. If you use a Remote AI API (e.g., OpenAI API, Groq API), your data will naturally travel to their servers. They will be subject to the privacy policy of the respective API provider. -Jan uses [Umami](https://umami.is/) for analytics, which is a privacy-focused, GDPR-compliant analytics tool that does not track personal information. We use this to get aggregate reports on OS and hardware types and prioritize our engineering roadmap. As per [Umami's Privacy Policy](https://umami.is/privacy), Umami uses the following data points to generate its reports: - -- OS and device characteristics -- IP address - -Jan does not get any of this data, and we do not track IP addresses or other identifying information. We are actively looking into more privacy-respecting ways to handle analytics, crash reports, and telemetry and would love to work with the community on this. - ### Cortex Cortex is a library that runs large language models (LLMs) locally on your computer. Cortex does not collect any personal information. diff --git a/electron/main.ts b/electron/main.ts index 6ce7f476a..42d16bb74 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -28,6 +28,7 @@ import { setupReactDevTool } from './utils/dev' import { trayManager } from './managers/tray' import { logSystemInfo } from './utils/system' import { registerGlobalShortcuts } from './utils/shortcut' +import { registerLogger } from './utils/logger' const preloadPath = join(__dirname, 'preload.js') const rendererPath = join(__dirname, '..', 'renderer') @@ -79,6 +80,7 @@ app }) .then(setupCore) .then(createUserSpace) + .then(registerLogger) .then(migrate) .then(setupExtensions) .then(setupMenu) diff --git a/electron/package.json b/electron/package.json index 700f009a5..59761623c 100644 --- a/electron/package.json +++ b/electron/package.json @@ -1,6 +1,6 @@ { "name": "jan", - "version": "0.1.4", + "version": "0.1.1737985524", "main": "./build/main.js", "author": "Jan ", "license": "MIT", diff --git a/extensions/monitoring-extension/src/node/logger.ts b/electron/utils/logger.ts similarity index 77% rename from extensions/monitoring-extension/src/node/logger.ts rename to electron/utils/logger.ts index ca64ea2d9..48af0b93a 100644 --- a/extensions/monitoring-extension/src/node/logger.ts +++ b/electron/utils/logger.ts @@ -1,16 +1,28 @@ -import fs from 'fs' +import { + createWriteStream, + existsSync, + mkdirSync, + readdir, + stat, + unlink, + writeFileSync, +} from 'fs' import util from 'util' import { getAppConfigurations, getJanDataFolderPath, Logger, + LoggerManager, } from '@janhq/core/node' import path, { join } from 'path' -export class FileLogger extends Logger { +/** + * File Logger + */ +export class FileLogger implements Logger { name = 'file' logCleaningInterval: number = 120000 - timeout: NodeJS.Timeout | null = null + timeout: NodeJS.Timeout | undefined appLogPath: string = './' logEnabled: boolean = true @@ -18,14 +30,13 @@ export class FileLogger extends Logger { logEnabled: boolean = true, logCleaningInterval: number = 120000 ) { - super() this.logEnabled = logEnabled if (logCleaningInterval) this.logCleaningInterval = logCleaningInterval const appConfigurations = getAppConfigurations() const logFolderPath = join(appConfigurations.data_folder, 'logs') - if (!fs.existsSync(logFolderPath)) { - fs.mkdirSync(logFolderPath, { recursive: true }) + if (!existsSync(logFolderPath)) { + mkdirSync(logFolderPath, { recursive: true }) } this.appLogPath = join(logFolderPath, 'app.log') @@ -69,8 +80,8 @@ export class FileLogger extends Logger { const logDirectory = path.join(getJanDataFolderPath(), 'logs') // Perform log cleaning const currentDate = new Date() - if (fs.existsSync(logDirectory)) - fs.readdir(logDirectory, (err, files) => { + if (existsSync(logDirectory)) + readdir(logDirectory, (err, files) => { if (err) { console.error('Error reading log directory:', err) return @@ -78,7 +89,7 @@ export class FileLogger extends Logger { files.forEach((file) => { const filePath = path.join(logDirectory, file) - fs.stat(filePath, (err, stats) => { + stat(filePath, (err, stats) => { if (err) { console.error('Error getting file stats:', err) return @@ -86,7 +97,7 @@ export class FileLogger extends Logger { // Check size if (stats.size > size) { - fs.unlink(filePath, (err) => { + unlink(filePath, (err) => { if (err) { console.error('Error deleting log file:', err) return @@ -103,7 +114,7 @@ export class FileLogger extends Logger { (1000 * 3600 * 24) ) if (daysDifference > days) { - fs.unlink(filePath, (err) => { + unlink(filePath, (err) => { if (err) { console.error('Error deleting log file:', err) return @@ -124,15 +135,20 @@ export class FileLogger extends Logger { } } +/** + * Write log function implementation + * @param message + * @param logPath + */ const writeLog = (message: string, logPath: string) => { - if (!fs.existsSync(logPath)) { + if (!existsSync(logPath)) { const logDirectory = path.join(getJanDataFolderPath(), 'logs') - if (!fs.existsSync(logDirectory)) { - fs.mkdirSync(logDirectory) + if (!existsSync(logDirectory)) { + mkdirSync(logDirectory) } - fs.writeFileSync(logPath, message) + writeFileSync(logPath, message) } else { - const logFile = fs.createWriteStream(logPath, { + const logFile = createWriteStream(logPath, { flags: 'a', }) logFile.write(util.format(message) + '\n') @@ -140,3 +156,12 @@ const writeLog = (message: string, logPath: string) => { console.debug(message) } } + +/** + * Register logger for global application logging + */ +export const registerLogger = () => { + const logger = new FileLogger() + LoggerManager.instance().register(logger) + logger.cleanLogs() +} diff --git a/extensions/conversational-extension/src/index.ts b/extensions/conversational-extension/src/index.ts index 30766be45..eeb4fcf38 100644 --- a/extensions/conversational-extension/src/index.ts +++ b/extensions/conversational-extension/src/index.ts @@ -63,7 +63,7 @@ export default class CortexConversationalExtension extends ConversationalExtensi async modifyThread(thread: Thread): Promise { return this.queue .add(() => - ky.post(`${API_URL}/v1/threads/${thread.id}`, { json: thread }) + ky.patch(`${API_URL}/v1/threads/${thread.id}`, { json: thread }) ) .then() } @@ -101,7 +101,7 @@ export default class CortexConversationalExtension extends ConversationalExtensi async modifyMessage(message: ThreadMessage): Promise { return this.queue.add(() => ky - .post( + .patch( `${API_URL}/v1/threads/${message.thread_id}/messages/${message.id}`, { json: message, diff --git a/extensions/engine-management-extension/engines.mjs b/extensions/engine-management-extension/engines.mjs index e85035423..eafe8a09c 100644 --- a/extensions/engine-management-extension/engines.mjs +++ b/extensions/engine-management-extension/engines.mjs @@ -6,6 +6,8 @@ import groq from './resources/groq.json' with { type: 'json' } import martian from './resources/martian.json' with { type: 'json' } import mistral from './resources/mistral.json' with { type: 'json' } import nvidia from './resources/nvidia.json' with { type: 'json' } +import deepseek from './resources/deepseek.json' with { type: 'json' } +import googleGemini from './resources/google_gemini.json' with { type: 'json' } import anthropicModels from './models/anthropic.json' with { type: 'json' } import cohereModels from './models/cohere.json' with { type: 'json' } @@ -15,6 +17,8 @@ import groqModels from './models/groq.json' with { type: 'json' } import martianModels from './models/martian.json' with { type: 'json' } import mistralModels from './models/mistral.json' with { type: 'json' } import nvidiaModels from './models/nvidia.json' with { type: 'json' } +import deepseekModels from './models/deepseek.json' with { type: 'json' } +import googleGeminiModels from './models/google_gemini.json' with { type: 'json' } const engines = [ anthropic, @@ -25,6 +29,8 @@ const engines = [ mistral, martian, nvidia, + deepseek, + googleGemini, ] const models = [ ...anthropicModels, @@ -35,5 +41,7 @@ const models = [ ...mistralModels, ...martianModels, ...nvidiaModels, + ...deepseekModels, + ...googleGeminiModels, ] export { engines, models } diff --git a/extensions/engine-management-extension/models/anthropic.json b/extensions/engine-management-extension/models/anthropic.json index 46b5893d1..0212ce1da 100644 --- a/extensions/engine-management-extension/models/anthropic.json +++ b/extensions/engine-management-extension/models/anthropic.json @@ -8,6 +8,7 @@ "inference_params": { "max_tokens": 4096, "temperature": 0.7, + "max_temperature": 1.0, "stream": true }, "engine": "anthropic" @@ -21,6 +22,7 @@ "inference_params": { "max_tokens": 8192, "temperature": 0.7, + "max_temperature": 1.0, "stream": true }, "engine": "anthropic" @@ -34,6 +36,7 @@ "inference_params": { "max_tokens": 8192, "temperature": 0.7, + "max_temperature": 1.0, "stream": true }, "engine": "anthropic" diff --git a/extensions/engine-management-extension/models/cohere.json b/extensions/engine-management-extension/models/cohere.json index 458e4278b..96a830637 100644 --- a/extensions/engine-management-extension/models/cohere.json +++ b/extensions/engine-management-extension/models/cohere.json @@ -8,6 +8,7 @@ "inference_params": { "max_tokens": 4096, "temperature": 0.7, + "max_temperature": 1.0, "stream": false }, "engine": "cohere" @@ -21,6 +22,7 @@ "inference_params": { "max_tokens": 4096, "temperature": 0.7, + "max_temperature": 1.0, "stream": false }, "engine": "cohere" diff --git a/extensions/engine-management-extension/models/deepseek.json b/extensions/engine-management-extension/models/deepseek.json new file mode 100644 index 000000000..29d5406bf --- /dev/null +++ b/extensions/engine-management-extension/models/deepseek.json @@ -0,0 +1,28 @@ +[ + { + "model": "deepseek-chat", + "object": "model", + "name": "DeepSeek Chat", + "version": "1.0", + "description": "The deepseek-chat model has been upgraded to DeepSeek-V3. deepseek-reasoner points to the new model DeepSeek-R1", + "inference_params": { + "max_tokens": 8192, + "temperature": 0.6, + "stream": true + }, + "engine": "deepseek" + }, + { + "model": "deepseek-reasoner", + "object": "model", + "name": "DeepSeek R1", + "version": "1.0", + "description": "CoT (Chain of Thought) is the reasoning content deepseek-reasoner gives before output the final answer. For details, please refer to Reasoning Model.", + "inference_params": { + "max_tokens": 8192, + "temperature": 0.6, + "stream": true + }, + "engine": "deepseek" + } +] diff --git a/extensions/engine-management-extension/models/google_gemini.json b/extensions/engine-management-extension/models/google_gemini.json new file mode 100644 index 000000000..392754ee6 --- /dev/null +++ b/extensions/engine-management-extension/models/google_gemini.json @@ -0,0 +1,67 @@ +[ + { + "model": "gemini-2.0-flash", + "object": "model", + "name": "Gemini 2.0 Flash", + "version": "1.0", + "description": "Gemini 2.0 Flash delivers next-gen features and improved capabilities, including superior speed, native tool use, multimodal generation, and a 1M token context window.", + "inference_params": { + "max_tokens": 8192, + "temperature": 0.6, + "stream": true + }, + "engine": "google_gemini" + }, + { + "model": "gemini-2.0-flash-lite-preview", + "object": "model", + "name": "Gemini 2.0 Flash-Lite Preview", + "version": "1.0", + "description": "A Gemini 2.0 Flash model optimized for cost efficiency and low latency.", + "inference_params": { + "max_tokens": 8192, + "temperature": 0.6, + "stream": true + }, + "engine": "google_gemini" + }, + { + "model": "gemini-1.5-flash", + "object": "model", + "name": "Gemini 1.5 Flash", + "version": "1.0", + "description": "Gemini 1.5 Flash is a fast and versatile multimodal model for scaling across diverse tasks.", + "inference_params": { + "max_tokens": 8192, + "temperature": 0.6, + "stream": true + }, + "engine": "google_gemini" + }, + { + "model": "gemini-1.5-flash-8b", + "object": "model", + "name": "Gemini 1.5 Flash-8B", + "version": "1.0", + "description": "Gemini 1.5 Flash-8B is a small model designed for lower intelligence tasks.", + "inference_params": { + "max_tokens": 8192, + "temperature": 0.6, + "stream": true + }, + "engine": "google_gemini" + }, + { + "model": "gemini-1.5-pro", + "object": "model", + "name": "Gemini 1.5 Pro", + "version": "1.0", + "description": "Gemini 1.5 Pro is a mid-size multimodal model that is optimized for a wide-range of reasoning tasks. 1.5 Pro can process large amounts of data at once, including 2 hours of video, 19 hours of audio, codebases with 60,000 lines of code, or 2,000 pages of text. ", + "inference_params": { + "max_tokens": 8192, + "temperature": 0.6, + "stream": true + }, + "engine": "google_gemini" + } +] diff --git a/extensions/engine-management-extension/models/mistral.json b/extensions/engine-management-extension/models/mistral.json index 12fcf938d..47df5d506 100644 --- a/extensions/engine-management-extension/models/mistral.json +++ b/extensions/engine-management-extension/models/mistral.json @@ -8,6 +8,7 @@ "inference_params": { "max_tokens": 32000, "temperature": 0.7, + "max_temperature": 1.0, "top_p": 0.95, "stream": true }, @@ -22,6 +23,7 @@ "inference_params": { "max_tokens": 32000, "temperature": 0.7, + "max_temperature": 1.0, "top_p": 0.95, "stream": true }, @@ -36,6 +38,7 @@ "inference_params": { "max_tokens": 32000, "temperature": 0.7, + "max_temperature": 1.0, "top_p": 0.95, "stream": true }, diff --git a/extensions/engine-management-extension/models/nvidia.json b/extensions/engine-management-extension/models/nvidia.json index dfce9f8bc..cb6f9dec1 100644 --- a/extensions/engine-management-extension/models/nvidia.json +++ b/extensions/engine-management-extension/models/nvidia.json @@ -8,6 +8,7 @@ "inference_params": { "max_tokens": 1024, "temperature": 0.3, + "max_temperature": 1.0, "top_p": 1, "stream": false, "frequency_penalty": 0, diff --git a/extensions/engine-management-extension/models/openai.json b/extensions/engine-management-extension/models/openai.json index 8f59b42ea..5c1e70b5a 100644 --- a/extensions/engine-management-extension/models/openai.json +++ b/extensions/engine-management-extension/models/openai.json @@ -79,12 +79,7 @@ "description": "OpenAI o1 is a new model with complex reasoning", "format": "api", "inference_params": { - "max_tokens": 100000, - "temperature": 1, - "top_p": 1, - "stream": true, - "frequency_penalty": 0, - "presence_penalty": 0 + "max_tokens": 100000 }, "engine": "openai" }, @@ -97,11 +92,7 @@ "format": "api", "inference_params": { "max_tokens": 32768, - "temperature": 1, - "top_p": 1, - "stream": true, - "frequency_penalty": 0, - "presence_penalty": 0 + "stream": true }, "engine": "openai" }, @@ -114,11 +105,20 @@ "format": "api", "inference_params": { "max_tokens": 65536, - "temperature": 1, - "top_p": 1, - "stream": true, - "frequency_penalty": 0, - "presence_penalty": 0 + "stream": true + }, + "engine": "openai" + }, + { + "model": "o3-mini", + "object": "model", + "name": "OpenAI o3-mini", + "version": "1.0", + "description": "OpenAI most recent reasoning model, providing high intelligence at the same cost and latency targets of o1-mini.", + "format": "api", + "inference_params": { + "max_tokens": 100000, + "stream": true }, "engine": "openai" } diff --git a/extensions/engine-management-extension/models/openrouter.json b/extensions/engine-management-extension/models/openrouter.json index b9714bb57..bf132533c 100644 --- a/extensions/engine-management-extension/models/openrouter.json +++ b/extensions/engine-management-extension/models/openrouter.json @@ -1,16 +1,91 @@ [ { - "model": "open-router-auto", + "model": "deepseek/deepseek-r1:free", "object": "model", - "name": "OpenRouter", + "name": "DeepSeek: R1", "version": "1.0", - "description": " OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.", + "description": "OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.", "inference_params": { - "max_tokens": 128000, "temperature": 0.7, "top_p": 0.95, "frequency_penalty": 0, - "presence_penalty": 0 + "presence_penalty": 0, + "stream": true + }, + "engine": "openrouter" + }, + { + "model": "deepseek/deepseek-r1-distill-llama-70b:free", + "object": "model", + "name": "DeepSeek: R1 Distill Llama 70B", + "version": "1.0", + "description": " OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.", + "inference_params": { + "temperature": 0.7, + "top_p": 0.95, + "frequency_penalty": 0, + "presence_penalty": 0, + "stream": true + }, + "engine": "openrouter" + }, + { + "model": "deepseek/deepseek-r1-distill-llama-70b:free", + "object": "model", + "name": "DeepSeek: R1 Distill Llama 70B", + "version": "1.0", + "description": "OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.", + "inference_params": { + "temperature": 0.7, + "top_p": 0.95, + "frequency_penalty": 0, + "presence_penalty": 0, + "stream": true + }, + "engine": "openrouter" + }, + { + "model": "meta-llama/llama-3.1-405b-instruct:free", + "object": "model", + "name": "Meta: Llama 3.1 405B Instruct", + "version": "1.0", + "description": "OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.", + "inference_params": { + "temperature": 0.7, + "top_p": 0.95, + "frequency_penalty": 0, + "presence_penalty": 0, + "stream": true + }, + "engine": "openrouter" + }, + { + "model": "qwen/qwen-vl-plus:free", + "object": "model", + "name": "Qwen: Qwen VL Plus", + "version": "1.0", + "description": "OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.", + "inference_params": { + "temperature": 0.7, + "top_p": 0.95, + "frequency_penalty": 0, + "presence_penalty": 0, + "stream": true + }, + "engine": "openrouter" + }, + { + "model": "qwen/qwen2.5-vl-72b-instruct:free", + "object": "model", + "name": "Qwen: Qwen2.5 VL 72B Instruct", + "version": "1.0", + "description": "OpenRouter scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to prioritize them.", + "inference_params": { + "temperature": 0.7, + "top_p": 0.95, + "frequency_penalty": 0, + "presence_penalty": 0, + "stream": true }, "engine": "openrouter" } diff --git a/extensions/engine-management-extension/package.json b/extensions/engine-management-extension/package.json index 96f962ffd..571a3852b 100644 --- a/extensions/engine-management-extension/package.json +++ b/extensions/engine-management-extension/package.json @@ -29,12 +29,10 @@ }, "dependencies": { "@janhq/core": "../../core/package.tgz", - "cpu-instructions": "^0.0.13", "ky": "^1.7.2", "p-queue": "^8.0.1" }, "bundledDependencies": [ - "cpu-instructions", "@janhq/core" ], "engines": { diff --git a/extensions/engine-management-extension/resources/anthropic.json b/extensions/engine-management-extension/resources/anthropic.json index 12a3f08b8..4172bcd0b 100644 --- a/extensions/engine-management-extension/resources/anthropic.json +++ b/extensions/engine-management-extension/resources/anthropic.json @@ -1,5 +1,5 @@ { - "id": "@janhq/inference-anthropic-extension", + "id": "anthropic", "type": "remote", "engine": "anthropic", "url": "https://console.anthropic.com/settings/keys", @@ -10,13 +10,14 @@ "transform_req": { "chat_completions": { "url": "https://api.anthropic.com/v1/messages", - "template": "{ {% for key, value in input_request %} {% if key == \"messages\" %} {% if input_request.messages.0.role == \"system\" %} \"system\": \"{{ input_request.messages.0.content }}\", \"messages\": [{% for message in input_request.messages %} {% if not loop.is_first %} {\"role\": \"{{ message.role }}\", \"content\": \"{{ message.content }}\" } {% if not loop.is_last %},{% endif %} {% endif %} {% endfor %}] {% else %} \"messages\": [{% for message in input_request.messages %} {\"role\": \"{{ message.role}}\", \"content\": \"{{ message.content }}\" } {% if not loop.is_last %},{% endif %} {% endfor %}] {% endif %} {% if not loop.is_last %},{% endif %} {% else if key == \"system\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %}\"{{ key }}\": {{ tojson(value) }} {% if not loop.is_last %},{% endif %} {% endif %} {% endfor %} }" + "template": "{ {% for key, value in input_request %} {% if key == \"messages\" %} {% if input_request.messages.0.role == \"system\" %} \"system\": {{ tojson(input_request.messages.0.content) }}, \"messages\": [{% for message in input_request.messages %} {% if not loop.is_first %} {\"role\": {{ tojson(message.role) }}, \"content\": {% if not message.content or message.content == \"\" %} \".\" {% else %} {{ tojson(message.content) }} {% endif %} } {% if not loop.is_last %},{% endif %} {% endif %} {% endfor %}] {% else %} \"messages\": [{% for message in input_request.messages %} {\"role\": {{ tojson(message.role) }}, \"content\": {% if not message.content or message.content == \"\" %} \".\" {% else %} {{ tojson(message.content) }} {% endif %} } {% if not loop.is_last %},{% endif %} {% endfor %}] {% endif %} {% if not loop.is_last %},{% endif %} {% else if key == \"system\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"metadata\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %}\"{{ key }}\": {{ tojson(value) }} {% if not loop.is_last %},{% endif %} {% endif %} {% endfor %} }" } }, "transform_resp": { "chat_completions": { - "template": "{% if input_request.stream %} {\"object\": \"chat.completion.chunk\", \"model\": \"{{ input_request.model }}\", \"choices\": [{\"index\": 0, \"delta\": { {% if input_request.type == \"message_start\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"ping\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"content_block_delta\" %} \"role\": \"assistant\", \"content\": \"{{ input_request.delta.text }}\" {% else if input_request.type == \"content_block_stop\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"content_block_stop\" %} \"role\": \"assistant\", \"content\": null {% endif %} }, {% if input_request.type == \"content_block_stop\" %} \"finish_reason\": \"stop\" {% else %} \"finish_reason\": null {% endif %} }]} {% else %} {\"id\": \"{{ input_request.id }}\", \"created\": null, \"object\": \"chat.completion\", \"model\": \"{{ input_request.model }}\", \"choices\": [{ \"index\": 0, \"message\": { \"role\": \"{{ input_request.role }}\", \"content\": {% if input_request.content and input_request.content.0.type == \"text\" %} \"{{input_request.content.0.text}}\" {% else %} null {% endif %}, \"refusal\": null }, \"logprobs\": null, \"finish_reason\": \"{{ input_request.stop_reason }}\" } ], \"usage\": { \"prompt_tokens\": {{ input_request.usage.input_tokens }}, \"completion_tokens\": {{ input_request.usage.output_tokens }}, \"total_tokens\": {{ input_request.usage.input_tokens + input_request.usage.output_tokens }}, \"prompt_tokens_details\": { \"cached_tokens\": 0 }, \"completion_tokens_details\": { \"reasoning_tokens\": 0, \"accepted_prediction_tokens\": 0, \"rejected_prediction_tokens\": 0 } }, \"system_fingerprint\": \"fp_6b68a8204b\"} {% endif %}" + "template": "{% if input_request.stream %} {\"object\": \"chat.completion.chunk\", \"model\": \"{{ input_request.model }}\", \"choices\": [{\"index\": 0, \"delta\": { {% if input_request.type == \"message_start\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"ping\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"content_block_delta\" %} \"role\": \"assistant\", \"content\": \"{{ input_request.delta.text }}\" {% else if input_request.type == \"content_block_stop\" %} \"role\": \"assistant\", \"content\": null {% else if input_request.type == \"content_block_stop\" %} \"role\": \"assistant\", \"content\": null {% endif %} }, {% if input_request.type == \"content_block_stop\" %} \"finish_reason\": \"stop\" {% else %} \"finish_reason\": null {% endif %} }]} {% else %} {{tojson(input_request)}} {% endif %}" } - } + }, + "explore_models_url": "https://docs.anthropic.com/en/docs/about-claude/models" } } diff --git a/extensions/engine-management-extension/resources/cohere.json b/extensions/engine-management-extension/resources/cohere.json index b10e00e5b..43cd0da5b 100644 --- a/extensions/engine-management-extension/resources/cohere.json +++ b/extensions/engine-management-extension/resources/cohere.json @@ -1,5 +1,5 @@ { - "id": "@janhq/inference-cohere-extension", + "id": "cohere", "type": "remote", "engine": "cohere", "url": "https://dashboard.cohere.com/api-keys", @@ -10,13 +10,14 @@ "transform_req": { "chat_completions": { "url": "https://api.cohere.ai/v1/chat", - "template": "{ {% for key, value in input_request %} {% if key == \"messages\" %} {% if input_request.messages.0.role == \"system\" %} \"preamble\": \"{{ input_request.messages.0.content }}\", {% if length(input_request.messages) > 2 %} \"chatHistory\": [{% for message in input_request.messages %} {% if not loop.is_first and not loop.is_last %} {\"role\": {% if message.role == \"user\" %} \"USER\" {% else %} \"CHATBOT\" {% endif %}, \"content\": \"{{ message.content }}\" } {% if loop.index < length(input_request.messages) - 2 %},{% endif %} {% endif %} {% endfor %}], {% endif %} \"message\": \"{{ last(input_request.messages).content }}\" {% else %} {% if length(input_request.messages) > 2 %} \"chatHistory\": [{% for message in input_request.messages %} {% if not loop.is_last %} { \"role\": {% if message.role == \"user\" %} \"USER\" {% else %} \"CHATBOT\" {% endif %}, \"content\": \"{{ message.content }}\" } {% if loop.index < length(input_request.messages) - 2 %},{% endif %} {% endif %} {% endfor %}],{% endif %}\"message\": \"{{ last(input_request.messages).content }}\" {% endif %}{% if not loop.is_last %},{% endif %} {% else if key == \"system\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} \"{{ key }}\": {{ tojson(value) }} {% if not loop.is_last %},{% endif %} {% endif %} {% endfor %} }" + "template": "{ {% for key, value in input_request %} {% if key == \"messages\" %} {% if input_request.messages.0.role == \"system\" %} \"preamble\": {{ tojson(input_request.messages.0.content) }}, {% if length(input_request.messages) > 2 %} \"chatHistory\": [{% for message in input_request.messages %} {% if not loop.is_first and not loop.is_last %} {\"role\": {% if message.role == \"user\" %} \"USER\" {% else %} \"CHATBOT\" {% endif %}, \"content\": \"{{ message.content }}\" } {% if loop.index < length(input_request.messages) - 2 %},{% endif %} {% endif %} {% endfor %}], {% endif %} \"message\": \"{{ last(input_request.messages).content }}\" {% else %} {% if length(input_request.messages) > 2 %} \"chatHistory\": [{% for message in input_request.messages %} {% if not loop.is_last %} { \"role\": {% if message.role == \"user\" %} \"USER\" {% else %} \"CHATBOT\" {% endif %}, \"content\": \"{{ message.content }}\" } {% if loop.index < length(input_request.messages) - 2 %},{% endif %} {% endif %} {% endfor %}],{% endif %}\"message\": \"{{ last(input_request.messages).content }}\" {% endif %}{% if not loop.is_last %},{% endif %} {% else if key == \"system\" or key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} \"{{ key }}\": {{ tojson(value) }} {% if not loop.is_last %},{% endif %} {% endif %} {% endfor %} }" } }, "transform_resp": { "chat_completions": { "template": "{% if input_request.stream %} {\"object\": \"chat.completion.chunk\", \"model\": \"{{ input_request.model }}\", \"choices\": [{\"index\": 0, \"delta\": { {% if input_request.event_type == \"text-generation\" %} \"role\": \"assistant\", \"content\": \"{{ input_request.text }}\" {% else %} \"role\": \"assistant\", \"content\": null {% endif %} }, {% if input_request.event_type == \"stream-end\" %} \"finish_reason\": \"{{ input_request.finish_reason }}\" {% else %} \"finish_reason\": null {% endif %} }]} {% else %} {\"id\": \"{{ input_request.generation_id }}\", \"created\": null, \"object\": \"chat.completion\", \"model\": {% if input_request.model %} \"{{ input_request.model }}\" {% else %} \"command-r-plus-08-2024\" {% endif %}, \"choices\": [{ \"index\": 0, \"message\": { \"role\": \"assistant\", \"content\": {% if not input_request.text %} null {% else %} \"{{ input_request.text }}\" {% endif %}, \"refusal\": null }, \"logprobs\": null, \"finish_reason\": \"{{ input_request.finish_reason }}\" } ], \"usage\": { \"prompt_tokens\": {{ input_request.meta.tokens.input_tokens }}, \"completion_tokens\": {{ input_request.meta.tokens.output_tokens }},\"total_tokens\": {{ input_request.meta.tokens.input_tokens + input_request.meta.tokens.output_tokens }}, \"prompt_tokens_details\": { \"cached_tokens\": 0 },\"completion_tokens_details\": { \"reasoning_tokens\": 0, \"accepted_prediction_tokens\": 0, \"rejected_prediction_tokens\": 0 } }, \"system_fingerprint\": \"fp_6b68a8204b\"} {% endif %}" } - } + }, + "explore_models_url": "https://docs.cohere.com/v2/docs/models" } } diff --git a/extensions/engine-management-extension/resources/deepseek.json b/extensions/engine-management-extension/resources/deepseek.json new file mode 100644 index 000000000..214ec3b23 --- /dev/null +++ b/extensions/engine-management-extension/resources/deepseek.json @@ -0,0 +1,23 @@ +{ + "id": "deepseek", + "type": "remote", + "engine": "deepseek", + "url": "https://platform.deepseek.com/api_keys", + "api_key": "", + "metadata": { + "get_models_url": "https://api.deepseek.com/models", + "header_template": "Authorization: Bearer {{api_key}}", + "transform_req": { + "chat_completions": { + "url": "https://api.deepseek.com/chat/completions", + "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"model\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }" + } + }, + "transform_resp": { + "chat_completions": { + "template": "{{tojson(input_request)}}" + } + }, + "explore_models_url": "https://api-docs.deepseek.com/quick_start/pricing" + } +} diff --git a/extensions/engine-management-extension/resources/google_gemini.json b/extensions/engine-management-extension/resources/google_gemini.json new file mode 100644 index 000000000..e0fa809a5 --- /dev/null +++ b/extensions/engine-management-extension/resources/google_gemini.json @@ -0,0 +1,23 @@ +{ + "id": "google_gemini", + "type": "remote", + "engine": "google_gemini", + "url": "https://aistudio.google.com/apikey", + "api_key": "", + "metadata": { + "get_models_url": "https://generativelanguage.googleapis.com/v1beta/models", + "header_template": "Authorization: Bearer {{api_key}}", + "transform_req": { + "chat_completions": { + "url": "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions", + "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"model\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }" + } + }, + "transform_resp": { + "chat_completions": { + "template": "{{tojson(input_request)}}" + } + }, + "explore_models_url": "https://ai.google.dev/gemini-api/docs/models/gemini" + } +} diff --git a/extensions/engine-management-extension/resources/groq.json b/extensions/engine-management-extension/resources/groq.json index 60d553a92..87d215ab2 100644 --- a/extensions/engine-management-extension/resources/groq.json +++ b/extensions/engine-management-extension/resources/groq.json @@ -1,5 +1,5 @@ { - "id": "@janhq/inference-groq-extension", + "id": "groq", "type": "remote", "engine": "groq", "url": "https://console.groq.com/keys", @@ -15,8 +15,9 @@ }, "transform_resp": { "chat_completions": { - "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"choices\" or key == \"created\" or key == \"model\" or key == \"service_tier\" or key == \"system_fingerprint\" or key == \"stream\" or key == \"object\" or key == \"usage\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }" + "template": "{{tojson(input_request)}}" } - } + }, + "explore_models_url": "https://console.groq.com/docs/models" } } diff --git a/extensions/engine-management-extension/resources/martian.json b/extensions/engine-management-extension/resources/martian.json index 3a65f3981..3fd458660 100644 --- a/extensions/engine-management-extension/resources/martian.json +++ b/extensions/engine-management-extension/resources/martian.json @@ -1,5 +1,5 @@ { - "id": "@janhq/inference-martian-extension", + "id": "martian", "type": "remote", "engine": "martian", "url": "https://withmartian.com/dashboard", @@ -15,8 +15,9 @@ }, "transform_resp": { "chat_completions": { - "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"choices\" or key == \"created\" or key == \"model\" or key == \"service_tier\" or key == \"system_fingerprint\" or key == \"stream\" or key == \"object\" or key == \"usage\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }" + "template": "{{tojson(input_request)}}" } - } + }, + "explore_models_url": "https://withmartian.github.io/llm-adapters/" } } diff --git a/extensions/engine-management-extension/resources/mistral.json b/extensions/engine-management-extension/resources/mistral.json index 3f447dc4c..4a24471a2 100644 --- a/extensions/engine-management-extension/resources/mistral.json +++ b/extensions/engine-management-extension/resources/mistral.json @@ -1,5 +1,5 @@ { - "id": "@janhq/inference-mistral-extension", + "id": "mistral", "type": "remote", "engine": "mistral", "url": "https://console.mistral.ai/api-keys/", @@ -17,6 +17,7 @@ "chat_completions": { "template": "{{tojson(input_request)}}" } - } + }, + "explore_models_url": "https://docs.mistral.ai/getting-started/models/models_overview/" } } diff --git a/extensions/engine-management-extension/resources/nvidia.json b/extensions/engine-management-extension/resources/nvidia.json index 240130090..573bad4f6 100644 --- a/extensions/engine-management-extension/resources/nvidia.json +++ b/extensions/engine-management-extension/resources/nvidia.json @@ -1,5 +1,5 @@ { - "id": "@janhq/inference-nvidia-extension", + "id": "nvidia", "type": "remote", "engine": "nvidia", "url": "https://org.ngc.nvidia.com/setup/personal-keys", @@ -15,8 +15,9 @@ }, "transform_resp": { "chat_completions": { - "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"choices\" or key == \"created\" or key == \"model\" or key == \"service_tier\" or key == \"system_fingerprint\" or key == \"stream\" or key == \"object\" or key == \"usage\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }" + "template": "{{tojson(input_request)}}" } - } + }, + "explore_models_url": "https://build.nvidia.com/models" } } diff --git a/extensions/engine-management-extension/resources/openai.json b/extensions/engine-management-extension/resources/openai.json index 97effd42a..42f421072 100644 --- a/extensions/engine-management-extension/resources/openai.json +++ b/extensions/engine-management-extension/resources/openai.json @@ -1,5 +1,5 @@ { - "id": "@janhq/inference-openai-extension", + "id": "openai", "type": "remote", "engine": "openai", "url": "https://platform.openai.com/account/api-keys", @@ -10,13 +10,14 @@ "transform_req": { "chat_completions": { "url": "https://api.openai.com/v1/chat/completions", - "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"messages\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" or key == \"max_tokens\" or ((input_request.model == \"o1\" or input_request.model == \"o1-preview\" or input_request.model == \"o1-mini\") and (key == \"stop\")) %} {% if not first %} , {% endif %} {% if key == \"messages\" and (input_request.model == \"o1\" or input_request.model == \"o1-preview\" or input_request.model == \"o1-mini\") and input_request.messages.0.role == \"system\" %} \"messages\": [{% for message in input_request.messages %} {% if not loop.is_first %} { \"role\": \"{{ message.role }}\", \"content\": \"{{ message.content }}\" } {% if not loop.is_last %} , {% endif %} {% endif %} {% endfor %}] {% else if key == \"max_tokens\" and (input_request.model == \"o1\" or input_request.model == \"o1-preview\" or input_request.model == \"o1-mini\") %} \"max_completion_tokens\": {{ tojson(value) }} {% else %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endif %} {% endfor %} }" + "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"model\" or key == \"temperature\" or key == \"store\" or key == \"messages\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" or key == \"max_tokens\" or ((input_request.model == \"o1\" or input_request.model == \"o1-preview\" or input_request.model == \"o1-mini\" or input_request.model == \"o3\" or input_request.model == \"o3-mini\") and (key == \"stop\")) %} {% if not first %} , {% endif %} {% if key == \"messages\" and (input_request.model == \"o1\" or input_request.model == \"o1-preview\" or input_request.model == \"o1-mini\") and input_request.messages.0.role == \"system\" %} \"messages\": [{% for message in input_request.messages %} {% if not loop.is_first %} { \"role\": \"{{ message.role }}\", \"content\": \"{{ message.content }}\" } {% if not loop.is_last %} , {% endif %} {% endif %} {% endfor %}] {% else if key == \"max_tokens\" and (input_request.model == \"o1\" or input_request.model == \"o1-preview\" or input_request.model == \"o1-mini\" or input_request.model == \"o3\" or input_request.model == \"o3-mini\") %} \"max_completion_tokens\": {{ tojson(value) }} {% set first = false %} {% else %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endif %} {% endfor %} }" } }, "transform_resp": { "chat_completions": { - "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"choices\" or key == \"created\" or key == \"model\" or key == \"service_tier\" or key == \"stream\" or key == \"object\" or key == \"usage\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }" + "template": "{{tojson(input_request)}}" } - } + }, + "explore_models_url": "https://platform.openai.com/docs/models" } } diff --git a/extensions/engine-management-extension/resources/openrouter.json b/extensions/engine-management-extension/resources/openrouter.json index 45dc48414..798199708 100644 --- a/extensions/engine-management-extension/resources/openrouter.json +++ b/extensions/engine-management-extension/resources/openrouter.json @@ -1,5 +1,5 @@ { - "id": "@janhq/inference-openrouter-extension", + "id": "openrouter", "type": "remote", "engine": "openrouter", "url": "https://openrouter.ai/keys", @@ -10,13 +10,14 @@ "transform_req": { "chat_completions": { "url": "https://openrouter.ai/api/v1/chat/completions", - "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }" + "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"messages\" or key == \"temperature\" or key == \"store\" or key == \"max_tokens\" or key == \"stream\" or key == \"presence_penalty\" or key == \"metadata\" or key == \"frequency_penalty\" or key == \"tools\" or key == \"tool_choice\" or key == \"logprobs\" or key == \"top_logprobs\" or key == \"logit_bias\" or key == \"n\" or key == \"modalities\" or key == \"prediction\" or key == \"response_format\" or key == \"service_tier\" or key == \"model\" or key == \"seed\" or key == \"stop\" or key == \"stream_options\" or key == \"top_p\" or key == \"parallel_tool_calls\" or key == \"user\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }" } }, "transform_resp": { "chat_completions": { - "template": "{ {% set first = true %} {% for key, value in input_request %} {% if key == \"choices\" or key == \"created\" or key == \"model\" or key == \"service_tier\" or key == \"system_fingerprint\" or key == \"stream\" or key == \"object\" or key == \"usage\" %} {% if not first %},{% endif %} \"{{ key }}\": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }" + "template": "{{tojson(input_request)}}" } - } + }, + "explore_models_url": "https://openrouter.ai/models" } } diff --git a/extensions/engine-management-extension/rolldown.config.mjs b/extensions/engine-management-extension/rolldown.config.mjs index 7e9a2d156..d89fd9bfa 100644 --- a/extensions/engine-management-extension/rolldown.config.mjs +++ b/extensions/engine-management-extension/rolldown.config.mjs @@ -13,9 +13,19 @@ export default defineConfig([ NODE: JSON.stringify(`${pkgJson.name}/${pkgJson.node}`), API_URL: JSON.stringify('http://127.0.0.1:39291'), SOCKET_URL: JSON.stringify('ws://127.0.0.1:39291'), - CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.46'), + PLATFORM: JSON.stringify(process.platform), + CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.49'), DEFAULT_REMOTE_ENGINES: JSON.stringify(engines), DEFAULT_REMOTE_MODELS: JSON.stringify(models), + DEFAULT_REQUEST_PAYLOAD_TRANSFORM: JSON.stringify( + `{ {% set first = true %} {% for key, value in input_request %} {% if key == "messages" or key == "model" or key == "temperature" or key == "store" or key == "max_tokens" or key == "stream" or key == "presence_penalty" or key == "metadata" or key == "frequency_penalty" or key == "tools" or key == "tool_choice" or key == "logprobs" or key == "top_logprobs" or key == "logit_bias" or key == "n" or key == "modalities" or key == "prediction" or key == "response_format" or key == "service_tier" or key == "seed" or key == "stop" or key == "stream_options" or key == "top_p" or key == "parallel_tool_calls" or key == "user" %} {% if not first %},{% endif %} "{{ key }}": {{ tojson(value) }} {% set first = false %} {% endif %} {% endfor %} }` + ), + DEFAULT_RESPONSE_BODY_TRANSFORM: JSON.stringify( + '{{tojson(input_request)}}' + ), + DEFAULT_REQUEST_HEADERS_TRANSFORM: JSON.stringify( + 'Authorization: Bearer {{api_key}}' + ), }, }, { @@ -26,18 +36,7 @@ export default defineConfig([ file: 'dist/node/index.cjs.js', }, define: { - CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.46'), - }, - }, - { - input: 'src/node/cpuInfo.ts', - output: { - format: 'cjs', - file: 'dist/node/cpuInfo.js', - }, - external: ['cpu-instructions'], - resolve: { - extensions: ['.ts', '.js', '.svg'], + CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.49'), }, }, ]) diff --git a/extensions/engine-management-extension/src/@types/global.d.ts b/extensions/engine-management-extension/src/@types/global.d.ts index 2d520d5f9..55874ea9a 100644 --- a/extensions/engine-management-extension/src/@types/global.d.ts +++ b/extensions/engine-management-extension/src/@types/global.d.ts @@ -1,7 +1,11 @@ declare const API_URL: string declare const CORTEX_ENGINE_VERSION: string +declare const PLATFORM: string declare const SOCKET_URL: string declare const NODE: string +declare const DEFAULT_REQUEST_PAYLOAD_TRANSFORM: string +declare const DEFAULT_RESPONSE_BODY_TRANSFORM: string +declare const DEFAULT_REQUEST_HEADERS_TRANSFORM: string declare const DEFAULT_REMOTE_ENGINES: ({ id: string diff --git a/extensions/engine-management-extension/src/index.ts b/extensions/engine-management-extension/src/index.ts index 0d30bf4ea..e2730cc71 100644 --- a/extensions/engine-management-extension/src/index.ts +++ b/extensions/engine-management-extension/src/index.ts @@ -19,12 +19,16 @@ import ky, { HTTPError } from 'ky' import PQueue from 'p-queue' import { EngineError } from './error' import { getJanDataFolderPath } from '@janhq/core' +import { engineVariant } from './utils' +interface ModelList { + data: Model[] +} /** - * JSONEngineManagementExtension is a EngineManagementExtension implementation that provides + * JanEngineManagementExtension is a EngineManagementExtension implementation that provides * functionality for managing engines. */ -export default class JSONEngineManagementExtension extends EngineManagementExtension { +export default class JanEngineManagementExtension extends EngineManagementExtension { queue = new PQueue({ concurrency: 1 }) /** @@ -63,13 +67,12 @@ export default class JSONEngineManagementExtension extends EngineManagementExten * @returns A Promise that resolves to an object of list engines. */ async getRemoteModels(name: string): Promise { - return this.queue.add(() => - ky - .get(`${API_URL}/v1/models/remote/${name}`) - .json() - .then((e) => e) - .catch(() => []) - ) as Promise + return ky + .get(`${API_URL}/v1/models/remote/${name}`) + .json() + .catch(() => ({ + data: [], + })) as Promise } /** @@ -138,9 +141,38 @@ export default class JSONEngineManagementExtension extends EngineManagementExten * Add a new remote engine * @returns A Promise that resolves to intall of engine. */ - async addRemoteEngine(engineConfig: EngineConfig) { + async addRemoteEngine( + engineConfig: EngineConfig, + persistModels: boolean = true + ) { + // Populate default settings + if ( + engineConfig.metadata?.transform_req?.chat_completions && + !engineConfig.metadata.transform_req.chat_completions.template + ) + engineConfig.metadata.transform_req.chat_completions.template = + DEFAULT_REQUEST_PAYLOAD_TRANSFORM + + if ( + engineConfig.metadata?.transform_resp?.chat_completions && + !engineConfig.metadata.transform_resp.chat_completions?.template + ) + engineConfig.metadata.transform_resp.chat_completions.template = + DEFAULT_RESPONSE_BODY_TRANSFORM + + if (engineConfig.metadata && !engineConfig.metadata?.header_template) + engineConfig.metadata.header_template = DEFAULT_REQUEST_HEADERS_TRANSFORM + return this.queue.add(() => - ky.post(`${API_URL}/v1/engines`, { json: engineConfig }).then((e) => e) + ky.post(`${API_URL}/v1/engines`, { json: engineConfig }).then((e) => { + if (persistModels && engineConfig.metadata?.get_models_url) { + // Pull /models from remote models endpoint + return this.populateRemoteModels(engineConfig) + .then(() => e) + .catch(() => e) + } + return e + }) ) as Promise<{ messages: string }> } @@ -161,9 +193,25 @@ export default class JSONEngineManagementExtension extends EngineManagementExten * @param model - Remote model object. */ async addRemoteModel(model: Model) { - return this.queue.add(() => - ky.post(`${API_URL}/v1/models/add`, { json: model }).then((e) => e) - ) + return this.queue + .add(() => + ky + .post(`${API_URL}/v1/models/add`, { + json: { + inference_params: { + max_tokens: 4096, + temperature: 0.7, + top_p: 0.95, + stream: true, + frequency_penalty: 0, + presence_penalty: 0, + }, + ...model, + }, + }) + .then((e) => e) + ) + .then(() => {}) } /** @@ -245,11 +293,7 @@ export default class JSONEngineManagementExtension extends EngineManagementExten error instanceof EngineError ) { const systemInfo = await systemInformation() - const variant = await executeOnMain( - NODE, - 'engineVariant', - systemInfo.gpuSetting - ) + const variant = await engineVariant(systemInfo.gpuSetting) await this.setDefaultEngineVariant(InferenceEngine.cortex_llamacpp, { variant: variant, version: `${CORTEX_ENGINE_VERSION}`, @@ -293,14 +337,40 @@ export default class JSONEngineManagementExtension extends EngineManagementExten data.api_key = api_key /// END - Migrate legacy api key settings - await this.addRemoteEngine(data).catch(console.error) + await this.addRemoteEngine(data, false).catch(console.error) }) ) events.emit(EngineEvent.OnEngineUpdate, {}) - DEFAULT_REMOTE_MODELS.forEach(async (data: Model) => { - await this.addRemoteModel(data).catch(() => {}) - }) + await Promise.all( + DEFAULT_REMOTE_MODELS.map((data: Model) => + this.addRemoteModel(data).catch(() => {}) + ) + ) events.emit(ModelEvent.OnModelsUpdate, { fetch: true }) } } + + /** + * Pulls models list from the remote provider and persist + * @param engineConfig + * @returns + */ + private populateRemoteModels = async (engineConfig: EngineConfig) => { + return this.getRemoteModels(engineConfig.engine) + .then((models: ModelList) => { + if (models?.data) + Promise.all( + models.data.map((model) => + this.addRemoteModel({ + ...model, + engine: engineConfig.engine as InferenceEngine, + model: model.model ?? model.id, + }).catch(console.info) + ) + ).then(() => { + events.emit(ModelEvent.OnModelsUpdate, { fetch: true }) + }) + }) + .catch(console.info) + } } diff --git a/extensions/engine-management-extension/src/node/cpuInfo.ts b/extensions/engine-management-extension/src/node/cpuInfo.ts deleted file mode 100644 index 4366a995b..000000000 --- a/extensions/engine-management-extension/src/node/cpuInfo.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { cpuInfo } from 'cpu-instructions' - -// Check the CPU info and determine the supported instruction set -const info = cpuInfo.cpuInfo().some((e) => e.toUpperCase() === 'AVX512') - ? 'avx512' - : cpuInfo.cpuInfo().some((e) => e.toUpperCase() === 'AVX2') - ? 'avx2' - : cpuInfo.cpuInfo().some((e) => e.toUpperCase() === 'AVX') - ? 'avx' - : 'noavx' - -// Send the result and wait for confirmation before exiting -new Promise((resolve, reject) => { - // @ts-ignore - process.send(info, (error: Error | null) => { - if (error) { - reject(error) - } else { - resolve() - } - }) -}) - .then(() => process.exit(0)) - .catch((error) => { - console.error('Failed to send info:', error) - process.exit(1) - }) diff --git a/extensions/engine-management-extension/src/node/index.test.ts b/extensions/engine-management-extension/src/node/index.test.ts index c73feb9c6..aa2ac8be8 100644 --- a/extensions/engine-management-extension/src/node/index.test.ts +++ b/extensions/engine-management-extension/src/node/index.test.ts @@ -1,7 +1,6 @@ import { describe, expect, it } from '@jest/globals' import engine from './index' -import { GpuSetting } from '@janhq/core/node' -import { cpuInfo } from 'cpu-instructions' +import { GpuSetting } from '@janhq/core' import { fork } from 'child_process' let testSettings: GpuSetting = { @@ -23,22 +22,12 @@ let testSettings: GpuSetting = { } const originalPlatform = process.platform -jest.mock('cpu-instructions', () => ({ - cpuInfo: { - cpuInfo: jest.fn(), - }, -})) -let mockCpuInfo = cpuInfo.cpuInfo as jest.Mock -mockCpuInfo.mockReturnValue([]) -jest.mock('@janhq/core/node', () => ({ + +jest.mock('@janhq/core', () => ({ appResourcePath: () => '.', log: jest.fn(), })) -jest.mock('child_process', () => ({ - fork: jest.fn(), -})) -const mockFork = fork as jest.Mock describe('test executable cortex file', () => { afterAll(function () { @@ -48,14 +37,7 @@ describe('test executable cortex file', () => { }) it('executes on MacOS', () => { - const mockProcess = { - on: jest.fn((event, callback) => { - if (event === 'message') { - callback('noavx') - } - }), - send: jest.fn(), - } + Object.defineProperty(process, 'platform', { value: 'darwin', }) @@ -63,7 +45,7 @@ describe('test executable cortex file', () => { value: 'arm64', }) - mockFork.mockReturnValue(mockProcess) + expect(engine.engineVariant(testSettings)).resolves.toEqual('mac-arm64') }) @@ -83,7 +65,7 @@ describe('test executable cortex file', () => { }), send: jest.fn(), } - mockFork.mockReturnValue(mockProcess) + Object.defineProperty(process, 'arch', { value: 'x64', }) @@ -107,7 +89,6 @@ describe('test executable cortex file', () => { }), send: jest.fn(), } - mockFork.mockReturnValue(mockProcess) expect(engine.engineVariant()).resolves.toEqual('windows-amd64-avx') }) @@ -145,7 +126,6 @@ describe('test executable cortex file', () => { }), send: jest.fn(), } - mockFork.mockReturnValue(mockProcess) expect(engine.engineVariant(settings)).resolves.toEqual( 'windows-amd64-avx2-cuda-11-7' @@ -176,26 +156,11 @@ describe('test executable cortex file', () => { }, ], } - mockFork.mockReturnValue({ - on: jest.fn((event, callback) => { - if (event === 'message') { - callback('noavx') - } - }), - send: jest.fn(), - }) expect(engine.engineVariant(settings)).resolves.toEqual( 'windows-amd64-noavx-cuda-12-0' ) - mockFork.mockReturnValue({ - on: jest.fn((event, callback) => { - if (event === 'message') { - callback('avx512') - } - }), - send: jest.fn(), - }) + expect(engine.engineVariant(settings)).resolves.toEqual( 'windows-amd64-avx2-cuda-12-0' ) @@ -209,14 +174,6 @@ describe('test executable cortex file', () => { ...testSettings, run_mode: 'cpu', } - mockFork.mockReturnValue({ - on: jest.fn((event, callback) => { - if (event === 'message') { - callback('noavx') - } - }), - send: jest.fn(), - }) expect(engine.engineVariant()).resolves.toEqual('linux-amd64-noavx') }) @@ -245,16 +202,6 @@ describe('test executable cortex file', () => { }, ], } - - mockFork.mockReturnValue({ - on: jest.fn((event, callback) => { - if (event === 'message') { - callback('avx512') - } - }), - send: jest.fn(), - }) - expect(engine.engineVariant(settings)).resolves.toBe( 'linux-amd64-avx2-cuda-11-7' ) @@ -284,14 +231,7 @@ describe('test executable cortex file', () => { }, ], } - mockFork.mockReturnValue({ - on: jest.fn((event, callback) => { - if (event === 'message') { - callback('avx2') - } - }), - send: jest.fn(), - }) + expect(engine.engineVariant(settings)).resolves.toEqual( 'linux-amd64-avx2-cuda-12-0' @@ -310,15 +250,6 @@ describe('test executable cortex file', () => { const cpuInstructions = ['avx512', 'avx2', 'avx', 'noavx'] cpuInstructions.forEach((instruction) => { - mockFork.mockReturnValue({ - on: jest.fn((event, callback) => { - if (event === 'message') { - callback(instruction) - } - }), - send: jest.fn(), - }) - expect(engine.engineVariant(settings)).resolves.toEqual( `linux-amd64-${instruction}` ) @@ -335,14 +266,7 @@ describe('test executable cortex file', () => { } const cpuInstructions = ['avx512', 'avx2', 'avx', 'noavx'] cpuInstructions.forEach((instruction) => { - mockFork.mockReturnValue({ - on: jest.fn((event, callback) => { - if (event === 'message') { - callback(instruction) - } - }), - send: jest.fn(), - }) + expect(engine.engineVariant(settings)).resolves.toEqual( `windows-amd64-${instruction}` ) @@ -376,14 +300,7 @@ describe('test executable cortex file', () => { } const cpuInstructions = ['avx512', 'avx2', 'avx', 'noavx'] cpuInstructions.forEach((instruction) => { - mockFork.mockReturnValue({ - on: jest.fn((event, callback) => { - if (event === 'message') { - callback(instruction) - } - }), - send: jest.fn(), - }) + expect(engine.engineVariant(settings)).resolves.toEqual( `windows-amd64-${instruction === 'avx512' || instruction === 'avx2' ? 'avx2' : 'noavx'}-cuda-12-0` ) @@ -417,14 +334,7 @@ describe('test executable cortex file', () => { ], } cpuInstructions.forEach((instruction) => { - mockFork.mockReturnValue({ - on: jest.fn((event, callback) => { - if (event === 'message') { - callback(instruction) - } - }), - send: jest.fn(), - }) + expect(engine.engineVariant(settings)).resolves.toEqual( `linux-amd64-${instruction === 'avx512' || instruction === 'avx2' ? 'avx2' : 'noavx'}-cuda-12-0` ) @@ -459,14 +369,7 @@ describe('test executable cortex file', () => { ], } cpuInstructions.forEach((instruction) => { - mockFork.mockReturnValue({ - on: jest.fn((event, callback) => { - if (event === 'message') { - callback(instruction) - } - }), - send: jest.fn(), - }) + expect(engine.engineVariant(settings)).resolves.toEqual( `linux-amd64-vulkan` ) diff --git a/extensions/engine-management-extension/src/node/index.ts b/extensions/engine-management-extension/src/node/index.ts index 31ad90ed2..4c1daf998 100644 --- a/extensions/engine-management-extension/src/node/index.ts +++ b/extensions/engine-management-extension/src/node/index.ts @@ -2,111 +2,10 @@ import * as path from 'path' import { appResourcePath, getJanDataFolderPath, - GpuSetting, log, } from '@janhq/core/node' -import { fork } from 'child_process' import { mkdir, readdir, symlink } from 'fs/promises' -/** - * The GPU runMode that will be set - either 'vulkan', 'cuda', or empty for cpu. - * @param settings - * @returns - */ -const gpuRunMode = (settings?: GpuSetting): string => { - if (process.platform === 'darwin') - // MacOS now has universal binaries - return '' - - if (!settings) return '' - - return settings.vulkan === true || settings.run_mode === 'cpu' ? '' : 'cuda' -} - -/** - * The OS & architecture that the current process is running on. - * @returns win, mac-x64, mac-arm64, or linux - */ -const os = (): string => { - return process.platform === 'win32' - ? 'windows-amd64' - : process.platform === 'darwin' - ? process.arch === 'arm64' - ? 'mac-arm64' - : 'mac-amd64' - : 'linux-amd64' -} - -/** - * The CUDA version that will be set - either '11-7' or '12-0'. - * @param settings - * @returns - */ -const cudaVersion = (settings?: GpuSetting): '11-7' | '12-0' | undefined => { - const isUsingCuda = - settings?.vulkan !== true && - settings?.run_mode === 'gpu' && - !os().includes('mac') - - if (!isUsingCuda) return undefined - return settings?.cuda?.version === '11' ? '11-7' : '12-0' -} - -/** - * The CPU instructions that will be set - either 'avx512', 'avx2', 'avx', or 'noavx'. - * @returns - */ -const cpuInstructions = async (): Promise => { - if (process.platform === 'darwin') return '' - - const child = fork(path.join(__dirname, './cpuInfo.js')) // Path to the child process file - - return new Promise((resolve, reject) => { - child.on('message', (cpuInfo?: string) => { - resolve(cpuInfo ?? 'noavx') - child.kill() // Kill the child process after receiving the result - }) - - child.on('error', (err) => { - resolve('noavx') - child.kill() - }) - - child.on('exit', (code) => { - if (code !== 0) { - resolve('noavx') - child.kill() - } - }) - }) -} - -/** - * Find which variant to run based on the current platform. - */ -const engineVariant = async (gpuSetting?: GpuSetting): Promise => { - const cpuInstruction = await cpuInstructions() - log(`[CORTEX]: CPU instruction: ${cpuInstruction}`) - let engineVariant = [ - os(), - gpuSetting?.vulkan - ? 'vulkan' - : gpuRunMode(gpuSetting) !== 'cuda' - ? // CPU mode - support all variants - cpuInstruction - : // GPU mode - packaged CUDA variants of avx2 and noavx - cpuInstruction === 'avx2' || cpuInstruction === 'avx512' - ? 'avx2' - : 'noavx', - gpuRunMode(gpuSetting), - cudaVersion(gpuSetting), - ] - .filter((e) => !!e) - .join('-') - - log(`[CORTEX]: Engine variant: ${engineVariant}`) - return engineVariant -} /** * Create symlink to each variant for the default bundled version @@ -148,6 +47,5 @@ const symlinkEngines = async () => { } export default { - engineVariant, symlinkEngines, } diff --git a/extensions/engine-management-extension/src/utils.ts b/extensions/engine-management-extension/src/utils.ts new file mode 100644 index 000000000..30d482313 --- /dev/null +++ b/extensions/engine-management-extension/src/utils.ts @@ -0,0 +1,86 @@ +import { GpuSetting, log } from '@janhq/core' + +/** + * The GPU runMode that will be set - either 'vulkan', 'cuda', or empty for cpu. + * @param settings + * @returns + */ + +const gpuRunMode = (settings?: GpuSetting): string => { + return settings.gpus?.some( + (gpu) => + gpu.activated === true && + gpu.additional_information && + gpu.additional_information.driver_version + ) + ? 'cuda' + : '' +} + +/** + * The OS & architecture that the current process is running on. + * @returns win, mac-x64, mac-arm64, or linux + */ +const os = (settings?: GpuSetting): string => { + return PLATFORM === 'win32' + ? 'windows-amd64' + : PLATFORM === 'darwin' + ? settings?.cpu?.arch === 'arm64' + ? 'mac-arm64' + : 'mac-amd64' + : 'linux-amd64' +} + +/** + * The CUDA version that will be set - either '11-7' or '12-0'. + * @param settings + * @returns + */ +const cudaVersion = (settings?: GpuSetting): '12-0' | '11-7' | undefined => { + const isUsingCuda = + settings?.vulkan !== true && + settings?.gpus?.some((gpu) => (gpu.activated === true ? 'gpu' : 'cpu')) && + !os().includes('mac') + + if (!isUsingCuda) return undefined + // return settings?.cuda?.version === '11' ? '11-7' : '12-0' + return settings.gpus?.some((gpu) => gpu.version.includes('12')) + ? '12-0' + : '11-7' +} + +/** + * The CPU instructions that will be set - either 'avx512', 'avx2', 'avx', or 'noavx'. + * @returns + */ + +/** + * Find which variant to run based on the current platform. + */ +export const engineVariant = async ( + gpuSetting?: GpuSetting +): Promise => { + const platform = os(gpuSetting) + + // There is no need to append the variant extension for mac + if (platform.startsWith('mac')) return platform + + let engineVariant = + gpuSetting?.vulkan || gpuSetting.gpus.some((e) => !e.additional_information) + ? [platform, 'vulkan'] + : [ + platform, + gpuRunMode(gpuSetting) === 'cuda' && + (gpuSetting.cpu.instructions.includes('avx2') || + gpuSetting.cpu.instructions.includes('avx512')) + ? 'avx2' + : 'noavx', + gpuRunMode(gpuSetting), + cudaVersion(gpuSetting), + ].filter(Boolean) // Remove any falsy values + + let engineVariantString = engineVariant.join('-') + + log(`[CORTEX]: Engine variant: ${engineVariantString}`) + return engineVariantString +} diff --git a/extensions/hardware-management-extension/jest.config.js b/extensions/hardware-management-extension/jest.config.js new file mode 100644 index 000000000..8bb37208d --- /dev/null +++ b/extensions/hardware-management-extension/jest.config.js @@ -0,0 +1,5 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', +} diff --git a/extensions/hardware-management-extension/package.json b/extensions/hardware-management-extension/package.json new file mode 100644 index 000000000..ec98c7440 --- /dev/null +++ b/extensions/hardware-management-extension/package.json @@ -0,0 +1,48 @@ +{ + "name": "@janhq/hardware-management-extension", + "productName": "Hardware Management", + "version": "1.0.0", + "description": "Manages Better Hardware settings.", + "main": "dist/index.js", + "node": "dist/node/index.cjs.js", + "author": "Jan ", + "license": "MIT", + "scripts": { + "test": "jest", + "build": "rolldown -c rolldown.config.mjs", + "codesign:darwin": "../../.github/scripts/auto-sign.sh", + "codesign:win32:linux": "echo 'No codesigning required'", + "codesign": "run-script-os", + "build:publish": "rimraf *.tgz --glob || true && yarn build && yarn codesign && npm pack && cpx *.tgz ../../pre-install" + }, + "exports": { + ".": "./dist/index.js", + "./main": "./dist/module.js" + }, + "devDependencies": { + "cpx": "^1.5.0", + "rimraf": "^3.0.2", + "rolldown": "^1.0.0-beta.1", + "run-script-os": "^1.1.6", + "ts-loader": "^9.5.0", + "typescript": "^5.3.3" + }, + "dependencies": { + "@janhq/core": "../../core/package.tgz", + "cpu-instructions": "^0.0.13", + "ky": "^1.7.2", + "p-queue": "^8.0.1" + }, + "bundledDependencies": [ + "cpu-instructions", + "@janhq/core" + ], + "hardwares": { + "node": ">=18.0.0" + }, + "files": [ + "dist/*", + "package.json", + "README.md" + ] +} diff --git a/extensions/hardware-management-extension/rolldown.config.mjs b/extensions/hardware-management-extension/rolldown.config.mjs new file mode 100644 index 000000000..7982ca555 --- /dev/null +++ b/extensions/hardware-management-extension/rolldown.config.mjs @@ -0,0 +1,17 @@ +import { defineConfig } from 'rolldown' +import pkgJson from './package.json' with { type: 'json' } + +export default defineConfig([ + { + input: 'src/index.ts', + output: { + format: 'esm', + file: 'dist/index.js', + }, + define: { + NODE: JSON.stringify(`${pkgJson.name}/${pkgJson.node}`), + API_URL: JSON.stringify('http://127.0.0.1:39291'), + SOCKET_URL: JSON.stringify('ws://127.0.0.1:39291'), + }, + }, +]) diff --git a/extensions/hardware-management-extension/src/@types/global.d.ts b/extensions/hardware-management-extension/src/@types/global.d.ts new file mode 100644 index 000000000..6639b9cbb --- /dev/null +++ b/extensions/hardware-management-extension/src/@types/global.d.ts @@ -0,0 +1,12 @@ +declare const API_URL: string +declare const SOCKET_URL: string +declare const NODE: string + +interface Core { + api: APIFunctions + events: EventEmitter +} +interface Window { + core?: Core | undefined + electronAPI?: any | undefined +} diff --git a/extensions/hardware-management-extension/src/index.ts b/extensions/hardware-management-extension/src/index.ts new file mode 100644 index 000000000..c2edc6159 --- /dev/null +++ b/extensions/hardware-management-extension/src/index.ts @@ -0,0 +1,67 @@ +import { + executeOnMain, + HardwareManagementExtension, + HardwareInformation, +} from '@janhq/core' +import ky from 'ky' +import PQueue from 'p-queue' + +/** + * JSONHardwareManagementExtension is a HardwareManagementExtension implementation that provides + * functionality for managing engines. + */ +export default class JSONHardwareManagementExtension extends HardwareManagementExtension { + queue = new PQueue({ concurrency: 1 }) + + /** + * Called when the extension is loaded. + */ + async onLoad() { + // Run Healthcheck + this.queue.add(() => this.healthz()) + } + + /** + * Called when the extension is unloaded. + */ + onUnload() {} + + /** + * Do health check on cortex.cpp + * @returns + */ + async healthz(): Promise { + return ky + .get(`${API_URL}/healthz`, { + retry: { limit: 20, delay: () => 500, methods: ['get'] }, + }) + .then(() => {}) + } + + /** + * @returns A Promise that resolves to an object of hardware. + */ + async getHardware(): Promise { + return this.queue.add(() => + ky + .get(`${API_URL}/v1/hardware`) + .json() + .then((e) => e) + ) as Promise + } + + /** + * @returns A Promise that resolves to an object of set gpu activate. + */ + async setAvtiveGpu(data: { gpus: number[] }): Promise<{ + message: string + activated_gpus: number[] + }> { + return this.queue.add(() => + ky.post(`${API_URL}/v1/hardware/activate`, { json: data }).then((e) => e) + ) as Promise<{ + message: string + activated_gpus: number[] + }> + } +} diff --git a/extensions/monitoring-extension/tsconfig.json b/extensions/hardware-management-extension/tsconfig.json similarity index 65% rename from extensions/monitoring-extension/tsconfig.json rename to extensions/hardware-management-extension/tsconfig.json index 2477d58ce..72e1e1895 100644 --- a/extensions/monitoring-extension/tsconfig.json +++ b/extensions/hardware-management-extension/tsconfig.json @@ -8,7 +8,9 @@ "forceConsistentCasingInFileNames": true, "strict": false, "skipLibCheck": true, - "rootDir": "./src" + "rootDir": "./src", + "resolveJsonModule": true }, - "include": ["./src"] + "include": ["./src"], + "exclude": ["src/**/*.test.ts", "rolldown.config.mjs"] } diff --git a/extensions/inference-cortex-extension/bin/version.txt b/extensions/inference-cortex-extension/bin/version.txt index d3b342f1a..7ee7020b3 100644 --- a/extensions/inference-cortex-extension/bin/version.txt +++ b/extensions/inference-cortex-extension/bin/version.txt @@ -1 +1 @@ -1.0.9-rc4 +1.0.10 diff --git a/extensions/inference-cortex-extension/package.json b/extensions/inference-cortex-extension/package.json index f191f3071..a4558dc8f 100644 --- a/extensions/inference-cortex-extension/package.json +++ b/extensions/inference-cortex-extension/package.json @@ -1,7 +1,7 @@ { "name": "@janhq/inference-cortex-extension", "productName": "Cortex Inference Engine", - "version": "1.0.24", + "version": "1.0.25", "description": "This extension embeds cortex.cpp, a lightweight inference engine written in C++. See https://jan.ai.\nAdditional dependencies could be installed to run without Cuda Toolkit installation.", "main": "dist/index.js", "node": "dist/node/index.cjs.js", diff --git a/extensions/inference-cortex-extension/resources/default_settings.json b/extensions/inference-cortex-extension/resources/default_settings.json index a3a93f305..945f32729 100644 --- a/extensions/inference-cortex-extension/resources/default_settings.json +++ b/extensions/inference-cortex-extension/resources/default_settings.json @@ -76,7 +76,7 @@ }, { "key": "use_mmap", - "title": "MMAP", + "title": "mmap", "description": "Loads model files more efficiently by mapping them to memory, reducing RAM usage.", "controllerType": "checkbox", "controllerProps": { diff --git a/extensions/inference-cortex-extension/resources/models/deepseek-r1-distill-llama-70b/model.json b/extensions/inference-cortex-extension/resources/models/deepseek-r1-distill-llama-70b/model.json new file mode 100644 index 000000000..7f98b07a1 --- /dev/null +++ b/extensions/inference-cortex-extension/resources/models/deepseek-r1-distill-llama-70b/model.json @@ -0,0 +1,35 @@ +{ + "sources": [ + { + "filename": "DeepSeek-R1-Distill-Llama-70B-Q4_K_M.gguf", + "url": "https://huggingface.co/unsloth/DeepSeek-R1-Distill-Llama-70B-GGUF/resolve/main/DeepSeek-R1-Distill-Llama-70B-Q4_K_M.gguf" + } + ], + "id": "deepseek-r1-distill-llama-70b", + "object": "model", + "name": "DeepSeek R1 Distill Llama 70B Q4", + "version": "1.0", + "description": "DeepSeek-R1 is a cheaper and open-source model that excels at agentic reasoning, superior multilingual capabilities, large context windows, and generalization across domains.", + "format": "gguf", + "settings": { + "ctx_len": 131072, + "prompt_template": "<｜User｜> {prompt} <｜Assistant｜>", + "llama_model_path": "DeepSeek-R1-Distill-Llama-70B-Q4_K_M.gguf", + "ngl": 81 + }, + "parameters": { + "temperature": 0.6, + "top_p": 0.95, + "stream": true, + "max_tokens": 131072, + "stop": [], + "frequency_penalty": 0, + "presence_penalty": 0 + }, + "metadata": { + "author": "DeepSeek", + "tags": ["70B", "Featured"], + "size": 42500000000 + }, + "engine": "llama-cpp" +} diff --git a/extensions/inference-cortex-extension/resources/models/deepseek-r1-distill-llama-8b/model.json b/extensions/inference-cortex-extension/resources/models/deepseek-r1-distill-llama-8b/model.json new file mode 100644 index 000000000..a3a075888 --- /dev/null +++ b/extensions/inference-cortex-extension/resources/models/deepseek-r1-distill-llama-8b/model.json @@ -0,0 +1,35 @@ +{ + "sources": [ + { + "filename": "DeepSeek-R1-Distill-Llama-8B-Q5_K_M.gguf", + "url": "https://huggingface.co/unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF/resolve/main/DeepSeek-R1-Distill-Llama-8B-Q5_K_M.gguf" + } + ], + "id": "deepseek-r1-distill-llama-8b", + "object": "model", + "name": "DeepSeek R1 Distill Llama 8B Q5", + "version": "1.0", + "description": "DeepSeek-R1 is a cheaper and open-source model that excels at agentic reasoning, superior multilingual capabilities, large context windows, and generalization across domains.", + "format": "gguf", + "settings": { + "ctx_len": 131072, + "prompt_template": "<｜User｜> {prompt} <｜Assistant｜>", + "llama_model_path": "DeepSeek-R1-Distill-Llama-8B-Q5_K_M.gguf", + "ngl": 33 + }, + "parameters": { + "temperature": 0.6, + "top_p": 0.95, + "stream": true, + "max_tokens": 131072, + "stop": [], + "frequency_penalty": 0, + "presence_penalty": 0 + }, + "metadata": { + "author": "DeepSeek", + "tags": ["8B", "Featured"], + "size": 5730000000 + }, + "engine": "llama-cpp" +} diff --git a/extensions/inference-cortex-extension/resources/models/deepseek-r1-distill-qwen-1.5b/model.json b/extensions/inference-cortex-extension/resources/models/deepseek-r1-distill-qwen-1.5b/model.json new file mode 100644 index 000000000..74b3dfc54 --- /dev/null +++ b/extensions/inference-cortex-extension/resources/models/deepseek-r1-distill-qwen-1.5b/model.json @@ -0,0 +1,35 @@ +{ + "sources": [ + { + "filename": "DeepSeek-R1-Distill-Qwen-1.5B-Q5_K_M.gguf", + "url": "https://huggingface.co/unsloth/DeepSeek-R1-Distill-Qwen-1.5B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-1.5B-Q5_K_M.gguf" + } + ], + "id": "deepseek-r1-distill-qwen-1.5b", + "object": "model", + "name": "DeepSeek R1 Distill Qwen 1.5B Q5", + "version": "1.0", + "description": "DeepSeek-R1 is a cheaper and open-source model that excels at agentic reasoning, superior multilingual capabilities, large context windows, and generalization across domains.", + "format": "gguf", + "settings": { + "ctx_len": 131072, + "prompt_template": "<｜User｜> {prompt} <｜Assistant｜>", + "llama_model_path": "DeepSeek-R1-Distill-Qwen-1.5B-Q5_K_M.gguf", + "ngl": 29 + }, + "parameters": { + "temperature": 0.6, + "top_p": 0.95, + "stream": true, + "max_tokens": 131072, + "stop": [], + "frequency_penalty": 0, + "presence_penalty": 0 + }, + "metadata": { + "author": "DeepSeek", + "tags": ["1.5B", "Featured"], + "size": 1290000000 + }, + "engine": "llama-cpp" +} diff --git a/extensions/inference-cortex-extension/resources/models/deepseek-r1-distill-qwen-14b/model.json b/extensions/inference-cortex-extension/resources/models/deepseek-r1-distill-qwen-14b/model.json new file mode 100644 index 000000000..594ba6e41 --- /dev/null +++ b/extensions/inference-cortex-extension/resources/models/deepseek-r1-distill-qwen-14b/model.json @@ -0,0 +1,35 @@ +{ + "sources": [ + { + "filename": "DeepSeek-R1-Distill-Qwen-14B-Q4_K_M.gguf", + "url": "https://huggingface.co/unsloth/DeepSeek-R1-Distill-Qwen-14B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-14B-Q4_K_M.gguf" + } + ], + "id": "deepseek-r1-distill-qwen-14b", + "object": "model", + "name": "DeepSeek R1 Distill Qwen 14B Q4", + "version": "1.0", + "description": "DeepSeek-R1 is a cheaper and open-source model that excels at agentic reasoning, superior multilingual capabilities, large context windows, and generalization across domains.", + "format": "gguf", + "settings": { + "ctx_len": 131072, + "prompt_template": "<｜User｜> {prompt} <｜Assistant｜>", + "llama_model_path": "DeepSeek-R1-Distill-Qwen-14B-Q4_K_M.gguf", + "ngl": 49 + }, + "parameters": { + "temperature": 0.6, + "top_p": 0.95, + "stream": true, + "max_tokens": 131072, + "stop": [], + "frequency_penalty": 0, + "presence_penalty": 0 + }, + "metadata": { + "author": "DeepSeek", + "tags": ["14B", "Featured"], + "size": 8990000000 + }, + "engine": "llama-cpp" +} diff --git a/extensions/inference-cortex-extension/resources/models/deepseek-r1-distill-qwen-32b/model.json b/extensions/inference-cortex-extension/resources/models/deepseek-r1-distill-qwen-32b/model.json new file mode 100644 index 000000000..6d7f5accf --- /dev/null +++ b/extensions/inference-cortex-extension/resources/models/deepseek-r1-distill-qwen-32b/model.json @@ -0,0 +1,35 @@ +{ + "sources": [ + { + "filename": "DeepSeek-R1-Distill-Qwen-32B-Q4_K_M.gguf", + "url": "https://huggingface.co/unsloth/DeepSeek-R1-Distill-Qwen-32B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-32B-Q4_K_M.gguf" + } + ], + "id": "deepseek-r1-distill-qwen-32b", + "object": "model", + "name": "DeepSeek R1 Distill Qwen 32B Q4", + "version": "1.0", + "description": "DeepSeek-R1 is a cheaper and open-source model that excels at agentic reasoning, superior multilingual capabilities, large context windows, and generalization across domains.", + "format": "gguf", + "settings": { + "ctx_len": 131072, + "prompt_template": "<｜User｜> {prompt} <｜Assistant｜>", + "llama_model_path": "DeepSeek-R1-Distill-Qwen-32B-Q4_K_M.gguf", + "ngl": 65 + }, + "parameters": { + "temperature": 0.6, + "top_p": 0.95, + "stream": true, + "max_tokens": 131072, + "stop": [], + "frequency_penalty": 0, + "presence_penalty": 0 + }, + "metadata": { + "author": "DeepSeek", + "tags": ["32B", "Featured"], + "size": 19900000000 + }, + "engine": "llama-cpp" +} diff --git a/extensions/inference-cortex-extension/resources/models/deepseek-r1-distill-qwen-7b/model.json b/extensions/inference-cortex-extension/resources/models/deepseek-r1-distill-qwen-7b/model.json new file mode 100644 index 000000000..eae53cf0e --- /dev/null +++ b/extensions/inference-cortex-extension/resources/models/deepseek-r1-distill-qwen-7b/model.json @@ -0,0 +1,35 @@ +{ + "sources": [ + { + "filename": "DeepSeek-R1-Distill-Qwen-7B-Q5_K_M.gguf", + "url": "https://huggingface.co/unsloth/DeepSeek-R1-Distill-Qwen-7B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-7B-Q5_K_M.gguf" + } + ], + "id": "deepseek-r1-distill-qwen-7b", + "object": "model", + "name": "DeepSeek R1 Distill Qwen 7B Q5", + "version": "1.0", + "description": "DeepSeek-R1 is a cheaper and open-source model that excels at agentic reasoning, superior multilingual capabilities, large context windows, and generalization across domains.", + "format": "gguf", + "settings": { + "ctx_len": 131072, + "prompt_template": "<｜User｜> {prompt} <｜Assistant｜>", + "llama_model_path": "DeepSeek-R1-Distill-Qwen-7B-Q5_K_M.gguf", + "ngl": 29 + }, + "parameters": { + "temperature": 0.6, + "top_p": 0.95, + "stream": true, + "max_tokens": 131072, + "stop": [], + "frequency_penalty": 0, + "presence_penalty": 0 + }, + "metadata": { + "author": "DeepSeek", + "tags": ["7B", "Featured"], + "size": 5440000000 + }, + "engine": "llama-cpp" +} diff --git a/extensions/inference-cortex-extension/resources/models/llama3.1-8b-instruct/model.json b/extensions/inference-cortex-extension/resources/models/llama3.1-8b-instruct/model.json index ec9a0284b..1aeb80450 100644 --- a/extensions/inference-cortex-extension/resources/models/llama3.1-8b-instruct/model.json +++ b/extensions/inference-cortex-extension/resources/models/llama3.1-8b-instruct/model.json @@ -22,19 +22,13 @@ "top_p": 0.95, "stream": true, "max_tokens": 8192, - "stop": [ - "<|end_of_text|>", - "<|eot_id|>", - "<|eom_id|>" - ], + "stop": ["<|end_of_text|>", "<|eot_id|>", "<|eom_id|>"], "frequency_penalty": 0, "presence_penalty": 0 }, "metadata": { "author": "MetaAI", - "tags": [ - "8B", "Featured" - ], + "tags": ["8B", "Featured"], "size": 4920000000 }, "engine": "llama-cpp" diff --git a/extensions/inference-cortex-extension/rolldown.config.mjs b/extensions/inference-cortex-extension/rolldown.config.mjs index cebfa7190..aebd8ac38 100644 --- a/extensions/inference-cortex-extension/rolldown.config.mjs +++ b/extensions/inference-cortex-extension/rolldown.config.mjs @@ -49,6 +49,13 @@ import qwen2514bJson from './resources/models/qwen2.5-14b-instruct/model.json' w import qwen2532bJson from './resources/models/qwen2.5-32b-instruct/model.json' with { type: 'json' } import qwen2572bJson from './resources/models/qwen2.5-72b-instruct/model.json' with { type: 'json' } +import deepseekR1DistillQwen_1_5b from './resources/models/deepseek-r1-distill-qwen-1.5b/model.json' with { type: 'json' } +import deepseekR1DistillQwen_7b from './resources/models/deepseek-r1-distill-qwen-7b/model.json' with { type: 'json' } +import deepseekR1DistillQwen_14b from './resources/models/deepseek-r1-distill-qwen-14b/model.json' with { type: 'json' } +import deepseekR1DistillQwen_32b from './resources/models/deepseek-r1-distill-qwen-32b/model.json' with { type: 'json' } +import deepseekR1DistillLlama_8b from './resources/models/deepseek-r1-distill-llama-8b/model.json' with { type: 'json' } +import deepseekR1DistillLlama_70b from './resources/models/deepseek-r1-distill-llama-70b/model.json' with { type: 'json' } + export default defineConfig([ { input: 'src/index.ts', @@ -106,12 +113,18 @@ export default defineConfig([ qwen2514bJson, qwen2532bJson, qwen2572bJson, + deepseekR1DistillQwen_1_5b, + deepseekR1DistillQwen_7b, + deepseekR1DistillQwen_14b, + deepseekR1DistillQwen_32b, + deepseekR1DistillLlama_8b, + deepseekR1DistillLlama_70b, ]), NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`), SETTINGS: JSON.stringify(defaultSettingJson), CORTEX_API_URL: JSON.stringify('http://127.0.0.1:39291'), CORTEX_SOCKET_URL: JSON.stringify('ws://127.0.0.1:39291'), - CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.46'), + CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.49'), }, }, { diff --git a/extensions/inference-cortex-extension/src/index.ts b/extensions/inference-cortex-extension/src/index.ts index 84cc49b94..05efaf735 100644 --- a/extensions/inference-cortex-extension/src/index.ts +++ b/extensions/inference-cortex-extension/src/index.ts @@ -112,8 +112,8 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine { if (!Number.isNaN(threads_number)) this.cpu_threads = threads_number // Run the process watchdog - const systemInfo = await systemInformation() - this.queue.add(() => executeOnMain(NODE, 'run', systemInfo)) + // const systemInfo = await systemInformation() + this.queue.add(() => executeOnMain(NODE, 'run')) this.queue.add(() => this.healthz()) this.subscribeToEvents() diff --git a/extensions/inference-cortex-extension/src/node/index.ts b/extensions/inference-cortex-extension/src/node/index.ts index 420c84b6e..5222bb156 100644 --- a/extensions/inference-cortex-extension/src/node/index.ts +++ b/extensions/inference-cortex-extension/src/node/index.ts @@ -16,15 +16,20 @@ let watchdog: ProcessWatchdog | undefined = undefined * Spawns a Nitro subprocess. * @returns A promise that resolves when the Nitro subprocess is started. */ -function run(systemInfo?: SystemInformation): Promise { +function run(): Promise { log(`[CORTEX]:: Spawning cortex subprocess...`) return new Promise(async (resolve, reject) => { - let gpuVisibleDevices = systemInfo?.gpuSetting?.gpus_in_use.join(',') ?? '' - let binaryName = `cortex-server${process.platform === 'win32' ? '.exe' : ''}` + // let gpuVisibleDevices = systemInfo?.gpuSetting?.gpus_in_use.join(',') ?? '' + let binaryName = `cortex-server${ + process.platform === 'win32' ? '.exe' : '' + }` const binPath = path.join(__dirname, '..', 'bin') const executablePath = path.join(binPath, binaryName) + + addEnvPaths(binPath) + const sharedPath = path.join(appResourcePath(), 'shared') // Execute the binary log(`[CORTEX]:: Spawn cortex at path: ${executablePath}`) @@ -44,15 +49,17 @@ function run(systemInfo?: SystemInformation): Promise { `${path.join(dataFolderPath, '.janrc')}`, '--data_folder_path', dataFolderPath, + '--loglevel', + 'INFO', ], { env: { ...process.env, - CUDA_VISIBLE_DEVICES: gpuVisibleDevices, - // Vulkan - Support 1 device at a time for now - ...(gpuVisibleDevices?.length > 0 && { - GGML_VK_VISIBLE_DEVICES: gpuVisibleDevices, - }), + // CUDA_VISIBLE_DEVICES: gpuVisibleDevices, + // // Vulkan - Support 1 device at a time for now + // ...(gpuVisibleDevices?.length > 0 && { + // GGML_VK_VISIBLE_DEVICES: gpuVisibleDevices, + // }), }, cwd: sharedPath, } @@ -71,6 +78,22 @@ function dispose() { watchdog?.terminate() } +/** + * Set the environment paths for the cortex subprocess + * @param dest + */ +function addEnvPaths(dest: string) { + // Add engine path to the PATH and LD_LIBRARY_PATH + if (process.platform === 'win32') { + process.env.PATH = (process.env.PATH || '').concat(path.delimiter, dest) + } else { + process.env.LD_LIBRARY_PATH = (process.env.LD_LIBRARY_PATH || '').concat( + path.delimiter, + dest + ) + } +} + /** * Cortex process info */ diff --git a/extensions/model-extension/src/index.ts b/extensions/model-extension/src/index.ts index 832f279f9..41081d99a 100644 --- a/extensions/model-extension/src/index.ts +++ b/extensions/model-extension/src/index.ts @@ -14,8 +14,6 @@ import { } from '@janhq/core' import { CortexAPI } from './cortex' import { scanModelsFolder } from './legacy/model-json' -import { downloadModel } from './legacy/download' -import { systemInformation } from '@janhq/core' import { deleteModelFiles } from './legacy/delete' export enum Settings { @@ -70,18 +68,6 @@ export default class JanModelExtension extends ModelExtension { * @returns A Promise that resolves when the model is downloaded. */ async pullModel(model: string, id?: string, name?: string): Promise { - if (id) { - const model: Model = ModelManager.instance().get(id) - // Clip vision model - should not be handled by cortex.cpp - // TensorRT model - should not be handled by cortex.cpp - if ( - model && - (model.engine === InferenceEngine.nitro_tensorrt_llm || - model.settings.vision_model) - ) { - return downloadModel(model, (await systemInformation()).gpuSetting) - } - } /** * Sending POST to /models/pull/{id} endpoint to pull the model */ diff --git a/extensions/model-extension/src/legacy/download.ts b/extensions/model-extension/src/legacy/download.ts index d4d6c62d9..570d0cd13 100644 --- a/extensions/model-extension/src/legacy/download.ts +++ b/extensions/model-extension/src/legacy/download.ts @@ -2,15 +2,12 @@ import { downloadFile, DownloadRequest, fs, - GpuSetting, - InferenceEngine, joinPath, Model, } from '@janhq/core' export const downloadModel = async ( model: Model, - gpuSettings?: GpuSetting, network?: { ignoreSSL?: boolean; proxy?: string } ): Promise => { const homedir = 'file://models' @@ -27,41 +24,6 @@ export const downloadModel = async ( JSON.stringify(model, null, 2) ) - if (model.engine === InferenceEngine.nitro_tensorrt_llm) { - if (!gpuSettings || gpuSettings.gpus.length === 0) { - console.error('No GPU found. Please check your GPU setting.') - return - } - const firstGpu = gpuSettings.gpus[0] - if (!firstGpu.name.toLowerCase().includes('nvidia')) { - console.error('No Nvidia GPU found. Please check your GPU setting.') - return - } - const gpuArch = firstGpu.arch - if (gpuArch === undefined) { - console.error('No GPU architecture found. Please check your GPU setting.') - return - } - - if (!supportedGpuArch.includes(gpuArch)) { - console.debug( - `Your GPU: ${JSON.stringify(firstGpu)} is not supported. Only 30xx, 40xx series are supported.` - ) - return - } - - const os = 'windows' // TODO: remove this hard coded value - - const newSources = model.sources.map((source) => { - const newSource = { ...source } - newSource.url = newSource.url - .replace(//g, os) - .replace(//g, gpuArch) - return newSource - }) - model.sources = newSources - } - console.debug(`Download sources: ${JSON.stringify(model.sources)}`) if (model.sources.length > 1) { diff --git a/extensions/monitoring-extension/README.md b/extensions/monitoring-extension/README.md deleted file mode 100644 index f9690da09..000000000 --- a/extensions/monitoring-extension/README.md +++ /dev/null @@ -1,75 +0,0 @@ -# Create a Jan Extension using Typescript - -Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀 - -## Create Your Own Extension - -To create your own extension, you can use this repository as a template! Just follow the below instructions: - -1. Click the Use this template button at the top of the repository -2. Select Create a new repository -3. Select an owner and name for your new repository -4. Click Create repository -5. Clone your new repository - -## Initial Setup - -After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension. - -> [!NOTE] -> -> You'll need to have a reasonably modern version of -> [Node.js](https://nodejs.org) handy. If you are using a version manager like -> [`nodenv`](https://github.com/nodenv/nodenv) or -> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the -> root of your repository to install the version specified in -> [`package.json`](./package.json). Otherwise, 20.x or later should work! - -1. :hammer_and_wrench: Install the dependencies - - ```bash - npm install - ``` - -1. :building_construction: Package the TypeScript for distribution - - ```bash - npm run bundle - ``` - -1. :white_check_mark: Check your artifact - - There will be a tgz file in your extension directory now - -## Update the Extension Metadata - -The [`package.json`](package.json) file defines metadata about your extension, such as -extension name, main entry, description and version. - -When you copy this repository, update `package.json` with the name, description for your extension. - -## Update the Extension Code - -The [`src/`](./src/) directory is the heart of your extension! This contains the -source code that will be run when your extension functions are invoked. You can replace the -contents of this directory with your own code. - -There are a few things to keep in mind when writing your extension code: - -- Most Jan Extension functions are processed asynchronously. - In `index.ts`, you will see that the extension function will return a `Promise`. - - ```typescript - import { events, MessageEvent, MessageRequest } from '@janhq/core' - - function onStart(): Promise { - return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => - this.inference(data) - ) - } - ``` - - For more information about the Jan Extension Core module, see the - [documentation](https://github.com/janhq/jan/blob/main/core/README.md). - -So, what are you waiting for? Go ahead and start customizing your extension! diff --git a/extensions/monitoring-extension/bin/.gitkeep b/extensions/monitoring-extension/bin/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/extensions/monitoring-extension/download.bat b/extensions/monitoring-extension/download.bat deleted file mode 100644 index 14e0aadd9..000000000 --- a/extensions/monitoring-extension/download.bat +++ /dev/null @@ -1,2 +0,0 @@ -@echo off -.\node_modules\.bin\download https://catalog.jan.ai/vulkaninfoSDK.exe -o ./bin \ No newline at end of file diff --git a/extensions/monitoring-extension/package.json b/extensions/monitoring-extension/package.json deleted file mode 100644 index 2f827b41b..000000000 --- a/extensions/monitoring-extension/package.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "name": "@janhq/monitoring-extension", - "productName": "System Monitoring", - "version": "1.0.10", - "description": "Provides system health and OS level data.", - "main": "dist/index.js", - "node": "dist/node/index.cjs.js", - "author": "Jan ", - "license": "AGPL-3.0", - "scripts": { - "build": "rolldown -c rolldown.config.mjs && yarn download-artifacts", - "download-artifacts": "run-script-os && cpx \"bin/**\" \"dist/bin\"", - "download-artifacts:darwin": "echo 'No artifacts to download for darwin'", - "download-artifacts:win32": "download.bat", - "download-artifacts:linux": "download https://catalog.jan.ai/vulkaninfo -o ./bin && chmod +x ./bin/vulkaninfo", - "build:publish": "rimraf *.tgz --glob || true && yarn build && npm pack && cpx *.tgz ../../pre-install" - }, - "exports": { - ".": "./dist/index.js", - "./main": "./dist/node/index.cjs.js" - }, - "devDependencies": { - "@types/node": "^20.11.4", - "@types/node-os-utils": "^1.3.4", - "cpx": "^1.5.0", - "download-cli": "^1.1.1", - "rimraf": "^3.0.2", - "rolldown": "1.0.0-beta.1", - "run-script-os": "^1.1.6", - "typescript": "^5.3.3" - }, - "dependencies": { - "@janhq/core": "../../core/package.tgz", - "node-os-utils": "^1.3.7" - }, - "files": [ - "dist/*", - "package.json", - "README.md" - ], - "bundleDependencies": [ - "node-os-utils", - "@janhq/core" - ], - "installConfig": { - "hoistingLimits": "workspaces" - }, - "packageManager": "yarn@4.5.3" -} diff --git a/extensions/monitoring-extension/resources/settings.json b/extensions/monitoring-extension/resources/settings.json deleted file mode 100644 index 40b0b97f9..000000000 --- a/extensions/monitoring-extension/resources/settings.json +++ /dev/null @@ -1,22 +0,0 @@ -[ - { - "key": "log-enabled", - "title": "Enable App Logs", - "description": "Saves app logs locally on your computer. This enables you to send us crash reports.", - "controllerType": "checkbox", - "controllerProps": { - "value": true - } - }, - { - "key": "log-cleaning-interval", - "title": "Log Cleaning Interval", - "description": "Automatically delete local logs after a certain time interval (in milliseconds).", - "controllerType": "input", - "controllerProps": { - "value": "120000", - "placeholder": "Interval in milliseconds. E.g. 120000", - "textAlign": "right" - } - } -] \ No newline at end of file diff --git a/extensions/monitoring-extension/rolldown.config.mjs b/extensions/monitoring-extension/rolldown.config.mjs deleted file mode 100644 index 3533e052b..000000000 --- a/extensions/monitoring-extension/rolldown.config.mjs +++ /dev/null @@ -1,32 +0,0 @@ -import { defineConfig } from 'rolldown' -import packageJson from './package.json' with { type: 'json' } -import settingJson from './resources/settings.json' with { type: 'json' } - -export default defineConfig([ - { - input: 'src/index.ts', - output: { - format: 'esm', - file: 'dist/index.js', - }, - platform: 'browser', - define: { - NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`), - SETTINGS: JSON.stringify(settingJson), - }, - }, - { - input: 'src/node/index.ts', - external: ['@janhq/core/node'], - output: { - format: 'cjs', - file: 'dist/node/index.cjs.js', - sourcemap: false, - inlineDynamicImports: true, - }, - resolve: { - extensions: ['.js', '.ts', '.json'], - }, - platform: 'node', - }, -]) diff --git a/extensions/monitoring-extension/src/@types/global.d.ts b/extensions/monitoring-extension/src/@types/global.d.ts deleted file mode 100644 index 7536fabd8..000000000 --- a/extensions/monitoring-extension/src/@types/global.d.ts +++ /dev/null @@ -1,19 +0,0 @@ -declare const NODE: string -declare const SETTINGS: SettingComponentProps[] - -type CpuGpuInfo = { - cpu: { - usage: number - } - gpu: GpuInfo[] -} - -type GpuInfo = { - id: string - name: string - temperature: string - utilization: string - memoryTotal: string - memoryFree: string - memoryUtilization: string -} diff --git a/extensions/monitoring-extension/src/index.ts b/extensions/monitoring-extension/src/index.ts deleted file mode 100644 index 5616c70a8..000000000 --- a/extensions/monitoring-extension/src/index.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { - AppConfigurationEventName, - GpuSetting, - MonitoringExtension, - OperatingSystemInfo, - events, - executeOnMain, -} from '@janhq/core' - -enum Settings { - logEnabled = 'log-enabled', - logCleaningInterval = 'log-cleaning-interval', -} -/** - * JanMonitoringExtension is a extension that provides system monitoring functionality. - * It implements the MonitoringExtension interface from the @janhq/core package. - */ -export default class JanMonitoringExtension extends MonitoringExtension { - /** - * Called when the extension is loaded. - */ - async onLoad() { - // Register extension settings - this.registerSettings(SETTINGS) - - const logEnabled = await this.getSetting(Settings.logEnabled, true) - const logCleaningInterval = parseInt( - await this.getSetting(Settings.logCleaningInterval, '120000') - ) - // Register File Logger provided by this extension - await executeOnMain(NODE, 'registerLogger', { - logEnabled, - logCleaningInterval: isNaN(logCleaningInterval) - ? 120000 - : logCleaningInterval, - }) - - // Attempt to fetch nvidia info - await executeOnMain(NODE, 'updateNvidiaInfo') - events.emit(AppConfigurationEventName.OnConfigurationUpdate, {}) - } - - onSettingUpdate(key: string, value: T): void { - if (key === Settings.logEnabled) { - executeOnMain(NODE, 'updateLogger', { logEnabled: value }) - } else if (key === Settings.logCleaningInterval) { - executeOnMain(NODE, 'updateLogger', { logCleaningInterval: value }) - } - } - - /** - * Called when the extension is unloaded. - */ - onUnload(): void { - // Register File Logger provided by this extension - executeOnMain(NODE, 'unregisterLogger') - } - - /** - * Returns the GPU configuration. - * @returns A Promise that resolves to an object containing the GPU configuration. - */ - async getGpuSetting(): Promise { - return executeOnMain(NODE, 'getGpuConfig') - } - - /** - * Returns information about the system resources. - * @returns A Promise that resolves to an object containing information about the system resources. - */ - getResourcesInfo(): Promise { - return executeOnMain(NODE, 'getResourcesInfo') - } - - /** - * Returns information about the current system load. - * @returns A Promise that resolves to an object containing information about the current system load. - */ - getCurrentLoad(): Promise { - return executeOnMain(NODE, 'getCurrentLoad') - } - - /** - * Returns information about the OS - * @returns - */ - getOsInfo(): Promise { - return executeOnMain(NODE, 'getOsInfo') - } -} diff --git a/extensions/monitoring-extension/src/node/index.ts b/extensions/monitoring-extension/src/node/index.ts deleted file mode 100644 index e32f85082..000000000 --- a/extensions/monitoring-extension/src/node/index.ts +++ /dev/null @@ -1,389 +0,0 @@ -import { - GpuSetting, - GpuSettingInfo, - LoggerManager, - OperatingSystemInfo, - ResourceInfo, - SupportedPlatforms, - getJanDataFolderPath, - log, -} from '@janhq/core/node' -import { mem, cpu } from 'node-os-utils' -import { exec } from 'child_process' -import { writeFileSync, existsSync, readFileSync, mkdirSync } from 'fs' -import path from 'path' -import os from 'os' -import { FileLogger } from './logger' - -/** - * Path to the settings directory - **/ -export const SETTINGS_DIR = path.join(getJanDataFolderPath(), 'settings') -/** - * Path to the settings file - **/ -export const GPU_INFO_FILE = path.join(SETTINGS_DIR, 'settings.json') - -/** - * Default GPU settings - * TODO: This needs to be refactored to support multiple accelerators - **/ -const DEFAULT_SETTINGS: GpuSetting = { - notify: true, - run_mode: 'cpu', - nvidia_driver: { - exist: false, - version: '', - }, - cuda: { - exist: false, - version: '', - }, - gpus: [], - gpu_highest_vram: '', - gpus_in_use: [], - is_initial: true, - // TODO: This needs to be set based on user toggle in settings - vulkan: false, -} - -export const getGpuConfig = async (): Promise => { - if (process.platform === 'darwin') return undefined - if (existsSync(GPU_INFO_FILE)) - return JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) - return DEFAULT_SETTINGS -} - -export const getResourcesInfo = async (): Promise => { - const ramUsedInfo = await mem.used() - const totalMemory = ramUsedInfo.totalMemMb * 1024 * 1024 - const usedMemory = ramUsedInfo.usedMemMb * 1024 * 1024 - - const resourceInfo: ResourceInfo = { - mem: { - totalMemory, - usedMemory, - }, - } - - return resourceInfo -} - -export const getCurrentLoad = () => - new Promise(async (resolve, reject) => { - const cpuPercentage = await cpu.usage() - let data = { - run_mode: 'cpu', - gpus_in_use: [], - } - - if (process.platform !== 'darwin') { - data = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) - } - - if (data.run_mode === 'gpu' && data.gpus_in_use.length > 0) { - const gpuIds = data.gpus_in_use.join(',') - if (gpuIds !== '' && data['vulkan'] !== true) { - exec( - `nvidia-smi --query-gpu=index,name,temperature.gpu,utilization.gpu,memory.total,memory.free,utilization.memory --format=csv,noheader,nounits --id=${gpuIds}`, - (error, stdout, _) => { - if (error) { - console.error(`exec error: ${error}`) - throw new Error(error.message) - } - const gpuInfo: GpuInfo[] = stdout - .trim() - .split('\n') - .map((line) => { - const [ - id, - name, - temperature, - utilization, - memoryTotal, - memoryFree, - memoryUtilization, - ] = line.split(', ').map((item) => item.replace(/\r/g, '')) - return { - id, - name, - temperature, - utilization, - memoryTotal, - memoryFree, - memoryUtilization, - } - }) - - resolve({ - cpu: { usage: cpuPercentage }, - gpu: gpuInfo, - }) - } - ) - } else { - // Handle the case where gpuIds is empty - resolve({ - cpu: { usage: cpuPercentage }, - gpu: [], - }) - } - } else { - // Handle the case where run_mode is not 'gpu' or no GPUs are in use - resolve({ - cpu: { usage: cpuPercentage }, - gpu: [], - }) - } - }) - -/** - * This will retrieve GPU information and persist settings.json - * Will be called when the extension is loaded to turn on GPU acceleration if supported - */ -export const updateNvidiaInfo = async () => { - // ignore if macos - if (process.platform === 'darwin') return - - try { - JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) - } catch (error) { - if (!existsSync(SETTINGS_DIR)) { - mkdirSync(SETTINGS_DIR, { - recursive: true, - }) - } - writeFileSync(GPU_INFO_FILE, JSON.stringify(DEFAULT_SETTINGS, null, 2)) - } - - await updateNvidiaDriverInfo() - await updateGpuInfo() -} - -const updateNvidiaDriverInfo = async () => - new Promise((resolve, reject) => { - exec( - 'nvidia-smi --query-gpu=driver_version --format=csv,noheader', - (error, stdout) => { - const data: GpuSetting = JSON.parse( - readFileSync(GPU_INFO_FILE, 'utf-8') - ) - - if (!error) { - const firstLine = stdout.split('\n')[0].trim() - data.nvidia_driver.exist = true - data.nvidia_driver.version = firstLine - } else { - data.nvidia_driver.exist = false - } - - writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) - resolve({}) - } - ) - }) - -const getGpuArch = (gpuName: string): string => { - if (!gpuName.toLowerCase().includes('nvidia')) return 'unknown' - - if (gpuName.includes('30')) return 'ampere' - else if (gpuName.includes('40')) return 'ada' - else return 'unknown' -} - -const updateGpuInfo = async () => - new Promise((resolve, reject) => { - let data: GpuSetting = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) - - // Cuda - if (data.vulkan === true) { - // Vulkan - exec( - process.platform === 'win32' - ? `${__dirname}\\..\\bin\\vulkaninfoSDK.exe --summary` - : `${__dirname}/../bin/vulkaninfo --summary`, - async (error, stdout) => { - if (!error) { - const output = stdout.toString() - - log(output) - const gpuRegex = /GPU(\d+):(?:[\s\S]*?)deviceName\s*=\s*(.*)/g - - const gpus: GpuSettingInfo[] = [] - let match - while ((match = gpuRegex.exec(output)) !== null) { - const id = match[1] - const name = match[2] - const arch = getGpuArch(name) - gpus.push({ id, vram: '0', name, arch }) - } - data.gpus = gpus - - if (!data.gpus_in_use || data.gpus_in_use.length === 0) { - data.gpus_in_use = [data.gpus.length > 1 ? '1' : '0'] - } - - data = await updateCudaExistence(data) - writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) - log(`[APP]::${JSON.stringify(data)}`) - resolve({}) - } else { - reject(error) - } - } - ) - } else { - exec( - 'nvidia-smi --query-gpu=index,memory.total,name --format=csv,noheader,nounits', - async (error, stdout) => { - if (!error) { - log(`[SPECS]::${stdout}`) - // Get GPU info and gpu has higher memory first - let highestVram = 0 - let highestVramId = '0' - const gpus: GpuSettingInfo[] = stdout - .trim() - .split('\n') - .map((line) => { - let [id, vram, name] = line.split(', ') - const arch = getGpuArch(name) - vram = vram.replace(/\r/g, '') - if (parseFloat(vram) > highestVram) { - highestVram = parseFloat(vram) - highestVramId = id - } - return { id, vram, name, arch } - }) - - data.gpus = gpus - data.gpu_highest_vram = highestVramId - } else { - data.gpus = [] - data.gpu_highest_vram = undefined - } - - if (!data.gpus_in_use || data.gpus_in_use.length === 0) { - data.gpus_in_use = data.gpu_highest_vram ? [data.gpu_highest_vram].filter(e => !!e) : [] - } - - data = await updateCudaExistence(data) - console.log('[MONITORING]::Cuda info: ', data) - writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) - log(`[APP]::${JSON.stringify(data)}`) - resolve({}) - } - ) - } - }) - -/** - * Check if file exists in paths - */ -const checkFileExistenceInPaths = (file: string, paths: string[]): boolean => { - return paths.some((p) => existsSync(path.join(p, file))) -} - -/** - * Validate cuda for linux and windows - */ -const updateCudaExistence = async ( - data: GpuSetting = DEFAULT_SETTINGS -): Promise => { - let filesCuda12: string[] - let filesCuda11: string[] - let paths: string[] - let cudaVersion: string = '' - - if (process.platform === 'win32') { - filesCuda12 = ['cublas64_12.dll', 'cudart64_12.dll', 'cublasLt64_12.dll'] - filesCuda11 = ['cublas64_11.dll', 'cudart64_110.dll', 'cublasLt64_11.dll'] - paths = process.env.PATH ? process.env.PATH.split(path.delimiter) : [] - } else { - filesCuda12 = ['libcudart.so.12', 'libcublas.so.12', 'libcublasLt.so.12'] - filesCuda11 = ['libcudart.so.11.0', 'libcublas.so.11', 'libcublasLt.so.11'] - paths = process.env.LD_LIBRARY_PATH - ? process.env.LD_LIBRARY_PATH.split(path.delimiter) - : [] - paths.push('/usr/lib/x86_64-linux-gnu/') - } - - let cudaExists = filesCuda12.every( - (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) - ) - - if (!cudaExists) { - cudaExists = filesCuda11.every( - (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) - ) - if (cudaExists) { - cudaVersion = '11' - } - } else { - cudaVersion = '12' - } - - data.cuda.exist = cudaExists - data.cuda.version = cudaVersion - - console.debug(data.is_initial, data.gpus_in_use) - - if (cudaExists && data.is_initial && data.gpus_in_use.length > 0) { - data.run_mode = 'gpu' - } - - data.is_initial = false - - // Attempt to query CUDA using NVIDIA SMI - if (!cudaExists) { - await new Promise((resolve) => { - exec('nvidia-smi', (error, stdout) => { - if (!error) { - const regex = /CUDA\s*Version:\s*(\d+\.\d+)/g - const match = regex.exec(stdout) - if (match && match[1]) { - data.cuda.version = match[1] - } - } - console.log('[MONITORING]::Finalized cuda info update: ', data) - resolve() - }) - }) - } - return data -} - -export const getOsInfo = (): OperatingSystemInfo => { - const platform = - SupportedPlatforms.find((p) => p === process.platform) || 'unknown' - - const osInfo: OperatingSystemInfo = { - platform: platform, - arch: process.arch, - release: os.release(), - machine: os.machine(), - version: os.version(), - totalMem: os.totalmem(), - freeMem: os.freemem(), - } - - return osInfo -} - -export const registerLogger = ({ logEnabled, logCleaningInterval }) => { - const logger = new FileLogger(logEnabled, logCleaningInterval) - LoggerManager.instance().register(logger) - logger.cleanLogs() -} - -export const unregisterLogger = () => { - LoggerManager.instance().unregister('file') -} - -export const updateLogger = ({ logEnabled, logCleaningInterval }) => { - const logger = LoggerManager.instance().loggers.get('file') as FileLogger - if (logger && logEnabled !== undefined) logger.logEnabled = logEnabled - if (logger && logCleaningInterval) - logger.logCleaningInterval = logCleaningInterval - // Rerun - logger && logger.cleanLogs() -} diff --git a/extensions/yarn.lock b/extensions/yarn.lock index acf2965ca..547178451 100644 --- a/extensions/yarn.lock +++ b/extensions/yarn.lock @@ -509,67 +509,83 @@ __metadata: "@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Fassistant-extension%40workspace%3Aassistant-extension": version: 0.1.10 - resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=7dd866&locator=%40janhq%2Fassistant-extension%40workspace%3Aassistant-extension" + resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=4260f5&locator=%40janhq%2Fassistant-extension%40workspace%3Aassistant-extension" dependencies: rxjs: "npm:^7.8.1" ulidx: "npm:^2.3.0" - checksum: 10c0/da0eed6e552ce2ff6f52a087e6e221101c3d0c03d92820840ee80c3ca1a17317a66525cb5bf59b6c1e8bd2e36e54763008f97e13000ae339dac49f5682fcfa65 + checksum: 10c0/e5f93764b39cd7b6d01c9c3e4c6bc913d2755dfef28c5aab6102a1b2f3fecd83ed8a4bb4bb89f2c31ebc06284345b7c265a49add72b859c0d40e50802d052072 languageName: node linkType: hard "@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Fconversational-extension%40workspace%3Aconversational-extension": version: 0.1.10 - resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=7dd866&locator=%40janhq%2Fconversational-extension%40workspace%3Aconversational-extension" + resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=4260f5&locator=%40janhq%2Fconversational-extension%40workspace%3Aconversational-extension" dependencies: rxjs: "npm:^7.8.1" ulidx: "npm:^2.3.0" - checksum: 10c0/da0eed6e552ce2ff6f52a087e6e221101c3d0c03d92820840ee80c3ca1a17317a66525cb5bf59b6c1e8bd2e36e54763008f97e13000ae339dac49f5682fcfa65 + checksum: 10c0/e5f93764b39cd7b6d01c9c3e4c6bc913d2755dfef28c5aab6102a1b2f3fecd83ed8a4bb4bb89f2c31ebc06284345b7c265a49add72b859c0d40e50802d052072 languageName: node linkType: hard "@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Fengine-management-extension%40workspace%3Aengine-management-extension": version: 0.1.10 - resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=7dd866&locator=%40janhq%2Fengine-management-extension%40workspace%3Aengine-management-extension" + resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=4260f5&locator=%40janhq%2Fengine-management-extension%40workspace%3Aengine-management-extension" dependencies: rxjs: "npm:^7.8.1" ulidx: "npm:^2.3.0" - checksum: 10c0/da0eed6e552ce2ff6f52a087e6e221101c3d0c03d92820840ee80c3ca1a17317a66525cb5bf59b6c1e8bd2e36e54763008f97e13000ae339dac49f5682fcfa65 + checksum: 10c0/e5f93764b39cd7b6d01c9c3e4c6bc913d2755dfef28c5aab6102a1b2f3fecd83ed8a4bb4bb89f2c31ebc06284345b7c265a49add72b859c0d40e50802d052072 + languageName: node + linkType: hard + +"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Fhardware-management-extension%40workspace%3Ahardware-management-extension": + version: 0.1.10 + resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=4260f5&locator=%40janhq%2Fhardware-management-extension%40workspace%3Ahardware-management-extension" + dependencies: + rxjs: "npm:^7.8.1" + ulidx: "npm:^2.3.0" + checksum: 10c0/e5f93764b39cd7b6d01c9c3e4c6bc913d2755dfef28c5aab6102a1b2f3fecd83ed8a4bb4bb89f2c31ebc06284345b7c265a49add72b859c0d40e50802d052072 languageName: node linkType: hard "@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Finference-cortex-extension%40workspace%3Ainference-cortex-extension": version: 0.1.10 - resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=7dd866&locator=%40janhq%2Finference-cortex-extension%40workspace%3Ainference-cortex-extension" + resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=4260f5&locator=%40janhq%2Finference-cortex-extension%40workspace%3Ainference-cortex-extension" dependencies: rxjs: "npm:^7.8.1" ulidx: "npm:^2.3.0" - checksum: 10c0/da0eed6e552ce2ff6f52a087e6e221101c3d0c03d92820840ee80c3ca1a17317a66525cb5bf59b6c1e8bd2e36e54763008f97e13000ae339dac49f5682fcfa65 + checksum: 10c0/e5f93764b39cd7b6d01c9c3e4c6bc913d2755dfef28c5aab6102a1b2f3fecd83ed8a4bb4bb89f2c31ebc06284345b7c265a49add72b859c0d40e50802d052072 languageName: node linkType: hard "@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Fmodel-extension%40workspace%3Amodel-extension": version: 0.1.10 - resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=7dd866&locator=%40janhq%2Fmodel-extension%40workspace%3Amodel-extension" + resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=4260f5&locator=%40janhq%2Fmodel-extension%40workspace%3Amodel-extension" dependencies: rxjs: "npm:^7.8.1" ulidx: "npm:^2.3.0" - checksum: 10c0/da0eed6e552ce2ff6f52a087e6e221101c3d0c03d92820840ee80c3ca1a17317a66525cb5bf59b6c1e8bd2e36e54763008f97e13000ae339dac49f5682fcfa65 - languageName: node - linkType: hard - -"@janhq/core@file:../../core/package.tgz::locator=%40janhq%2Fmonitoring-extension%40workspace%3Amonitoring-extension": - version: 0.1.10 - resolution: "@janhq/core@file:../../core/package.tgz#../../core/package.tgz::hash=7dd866&locator=%40janhq%2Fmonitoring-extension%40workspace%3Amonitoring-extension" - dependencies: - rxjs: "npm:^7.8.1" - ulidx: "npm:^2.3.0" - checksum: 10c0/da0eed6e552ce2ff6f52a087e6e221101c3d0c03d92820840ee80c3ca1a17317a66525cb5bf59b6c1e8bd2e36e54763008f97e13000ae339dac49f5682fcfa65 + checksum: 10c0/e5f93764b39cd7b6d01c9c3e4c6bc913d2755dfef28c5aab6102a1b2f3fecd83ed8a4bb4bb89f2c31ebc06284345b7c265a49add72b859c0d40e50802d052072 languageName: node linkType: hard "@janhq/engine-management-extension@workspace:engine-management-extension": version: 0.0.0-use.local resolution: "@janhq/engine-management-extension@workspace:engine-management-extension" + dependencies: + "@janhq/core": ../../core/package.tgz + cpx: "npm:^1.5.0" + ky: "npm:^1.7.2" + p-queue: "npm:^8.0.1" + rimraf: "npm:^3.0.2" + rolldown: "npm:^1.0.0-beta.1" + run-script-os: "npm:^1.1.6" + ts-loader: "npm:^9.5.0" + typescript: "npm:^5.3.3" + languageName: unknown + linkType: soft + +"@janhq/hardware-management-extension@workspace:hardware-management-extension": + version: 0.0.0-use.local + resolution: "@janhq/hardware-management-extension@workspace:hardware-management-extension" dependencies: "@janhq/core": ../../core/package.tgz cpu-instructions: "npm:^0.0.13" @@ -630,23 +646,6 @@ __metadata: languageName: unknown linkType: soft -"@janhq/monitoring-extension@workspace:monitoring-extension": - version: 0.0.0-use.local - resolution: "@janhq/monitoring-extension@workspace:monitoring-extension" - dependencies: - "@janhq/core": ../../core/package.tgz - "@types/node": "npm:^20.11.4" - "@types/node-os-utils": "npm:^1.3.4" - cpx: "npm:^1.5.0" - download-cli: "npm:^1.1.1" - node-os-utils: "npm:^1.3.7" - rimraf: "npm:^3.0.2" - rolldown: "npm:1.0.0-beta.1" - run-script-os: "npm:^1.1.6" - typescript: "npm:^5.3.3" - languageName: unknown - linkType: soft - "@jest/console@npm:^29.7.0": version: 29.7.0 resolution: "@jest/console@npm:29.7.0" @@ -1877,13 +1876,6 @@ __metadata: languageName: node linkType: hard -"@types/node-os-utils@npm:^1.3.4": - version: 1.3.4 - resolution: "@types/node-os-utils@npm:1.3.4" - checksum: 10c0/d57bfa84862ee388f538e2bf38b5a6e6a555aebf6e50573ad5700f5858f657ee72388833aa7ed6c9d0b68ce0a6763802366326617b0d5f4d56cc3fe61dd617e1 - languageName: node - linkType: hard - "@types/node@npm:*": version: 22.10.2 resolution: "@types/node@npm:22.10.2" @@ -5999,13 +5991,6 @@ __metadata: languageName: node linkType: hard -"node-os-utils@npm:^1.3.7": - version: 1.3.7 - resolution: "node-os-utils@npm:1.3.7" - checksum: 10c0/88b8a4c7ed99ca0ca8f077f4f4672026e732605d5afb125e856de9ba1880b842facefa4c38f732f5cce20a34f9f471ce18a20c677dcdb702b4b68c17bacf9584 - languageName: node - linkType: hard - "node-releases@npm:^2.0.19": version: 2.0.19 resolution: "node-releases@npm:2.0.19" diff --git a/server/cortex.json b/server/cortex.json index 917cff354..0a9b83efd 100644 --- a/server/cortex.json +++ b/server/cortex.json @@ -5,77 +5,470 @@ "post": { "operationId": "AssistantsController_create", "summary": "Create assistant", - "description": "Creates a new assistant.", - "parameters": [], + "description": "Creates a new assistant with the specified configuration.", "requestBody": { "required": true, "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CreateAssistantDto" + "type": "object", + "properties": { + "model": { + "type": "string", + "description": "The model identifier to use for the assistant." + }, + "name": { + "type": "string", + "description": "The name of the assistant." + }, + "description": { + "type": "string", + "description": "The description of the assistant." + }, + "instructions": { + "type": "string", + "description": "Instructions for the assistant's behavior." + }, + "tools": { + "type": "array", + "description": "A list of tools enabled on the assistant. Maximum of 128 tools.", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "code_interpreter", + "file_search", + "function" + ] + } + } + } + }, + "tool_resources": { + "type": "object", + "description": "Resources used by the assistant's tools.", + "properties": { + "code_interpreter": { + "type": "object" + }, + "file_search": { + "type": "object" + } + } + }, + "metadata": { + "type": "object", + "description": "Set of key-value pairs for the assistant.", + "additionalProperties": true + }, + "temperature": { + "type": "number", + "format": "float", + "description": "Temperature parameter for response generation." + }, + "top_p": { + "type": "number", + "format": "float", + "description": "Top p parameter for response generation." + }, + "response_format": { + "oneOf": [ + { + "type": "string", + "enum": ["auto"] + }, + { + "type": "object" + } + ] + } + }, + "required": ["model"] } } } }, - "responses": { - "201": { - "description": "The assistant has been successfully created." - } - }, - "tags": ["Assistants"] - }, - "get": { - "operationId": "AssistantsController_findAll", - "summary": "List assistants", - "description": "Returns a list of assistants.", - "parameters": [ - { - "name": "limit", - "required": false, - "in": "query", - "description": "A limit on the number of objects to be returned. Limit can range between 1 and 100, and the default is 20.", - "schema": { - "type": "number" - } - }, - { - "name": "order", - "required": false, - "in": "query", - "description": "Sort order by the created_at timestamp of the objects. asc for ascending order and desc for descending order.", - "schema": { - "type": "string" - } - }, - { - "name": "after", - "required": false, - "in": "query", - "description": "A cursor for use in pagination. after is an object ID that defines your place in the list. For instance, if you make a list request and receive 100 objects, ending with obj_foo, your subsequent call can include after=obj_foo in order to fetch the next page of the list.", - "schema": { - "type": "string" - } - }, - { - "name": "before", - "required": false, - "in": "query", - "description": "A cursor for use in pagination. before is an object ID that defines your place in the list. For instance, if you make a list request and receive 100 objects, ending with obj_foo, your subsequent call can include before=obj_foo in order to fetch the previous page of the list.", - "schema": { - "type": "string" - } - } - ], "responses": { "200": { "description": "Ok", "content": { "application/json": { "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/AssistantEntity" + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The unique identifier of the assistant." + }, + "object": { + "type": "string", + "enum": ["assistant"], + "description": "The object type, which is always 'assistant'." + }, + "created_at": { + "type": "integer", + "description": "Unix timestamp (in seconds) of when the assistant was created." + }, + "model": { + "type": "string", + "description": "The model identifier used by the assistant." + }, + "name": { + "type": "string", + "description": "The name of the assistant." + }, + "description": { + "type": "string", + "description": "The description of the assistant." + }, + "instructions": { + "type": "string", + "description": "Instructions for the assistant's behavior." + }, + "tools": { + "type": "array", + "description": "A list of tools enabled on the assistant.", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "code_interpreter", + "file_search", + "function" + ] + } + } + } + }, + "tool_resources": { + "type": "object", + "description": "Resources used by the assistant's tools.", + "properties": { + "code_interpreter": { + "type": "object" + }, + "file_search": { + "type": "object" + } + } + }, + "metadata": { + "type": "object", + "description": "Set of key-value pairs that can be attached to the assistant.", + "additionalProperties": true + }, + "temperature": { + "type": "number", + "format": "float", + "description": "Temperature parameter for response generation." + }, + "top_p": { + "type": "number", + "format": "float", + "description": "Top p parameter for response generation." + }, + "response_format": { + "oneOf": [ + { + "type": "string", + "enum": ["auto"] + }, + { + "type": "object" + } + ] + } + }, + "required": [ + "id", + "object", + "created_at", + "model", + "metadata" + ] + } + } + } + } + }, + "tags": ["Assistants"] + }, + "patch": { + "operationId": "AssistantsController_update", + "summary": "Update assistant", + "description": "Updates an assistant. Requires at least one modifiable field.", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "description": "The unique identifier of the assistant.", + "schema": { + "type": "string" + } + }, + { + "name": "OpenAI-Beta", + "required": true, + "in": "header", + "description": "Beta feature header.", + "schema": { + "type": "string", + "enum": ["assistants=v2"] + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "model": { + "type": "string", + "description": "The model identifier to use for the assistant." + }, + "name": { + "type": "string", + "description": "The name of the assistant." + }, + "description": { + "type": "string", + "description": "The description of the assistant." + }, + "instructions": { + "type": "string", + "description": "Instructions for the assistant's behavior." + }, + "tools": { + "type": "array", + "description": "A list of tools enabled on the assistant. Maximum of 128 tools.", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "code_interpreter", + "file_search", + "function" + ] + } + } + } + }, + "tool_resources": { + "type": "object", + "description": "Resources used by the assistant's tools.", + "properties": { + "code_interpreter": { + "type": "object" + }, + "file_search": { + "type": "object" + } + } + }, + "metadata": { + "type": "object", + "description": "Set of key-value pairs for the assistant.", + "additionalProperties": true + }, + "temperature": { + "type": "number", + "format": "float", + "description": "Temperature parameter for response generation." + }, + "top_p": { + "type": "number", + "format": "float", + "description": "Top p parameter for response generation." + }, + "response_format": { + "oneOf": [ + { + "type": "string", + "enum": ["auto"] + }, + { + "type": "object" + } + ] } + }, + "minProperties": 1 + } + } + } + }, + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The unique identifier of the assistant." + }, + "object": { + "type": "string", + "enum": ["assistant"], + "description": "The object type, which is always 'assistant'." + }, + "created_at": { + "type": "integer", + "description": "Unix timestamp (in seconds) of when the assistant was created." + }, + "model": { + "type": "string", + "description": "The model identifier used by the assistant." + }, + "name": { + "type": "string", + "description": "The name of the assistant." + }, + "description": { + "type": "string", + "description": "The description of the assistant." + }, + "instructions": { + "type": "string", + "description": "Instructions for the assistant's behavior." + }, + "tools": { + "type": "array", + "description": "A list of tools enabled on the assistant.", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "code_interpreter", + "file_search", + "function" + ] + } + } + } + }, + "tool_resources": { + "type": "object", + "description": "Resources used by the assistant's tools.", + "properties": { + "code_interpreter": { + "type": "object" + }, + "file_search": { + "type": "object" + } + } + }, + "metadata": { + "type": "object", + "description": "Set of key-value pairs that can be attached to the assistant.", + "additionalProperties": true + }, + "temperature": { + "type": "number", + "format": "float", + "description": "Temperature parameter for response generation." + }, + "top_p": { + "type": "number", + "format": "float", + "description": "Top p parameter for response generation." + }, + "response_format": { + "oneOf": [ + { + "type": "string", + "enum": ["auto"] + }, + { + "type": "object" + } + ] + } + }, + "required": [ + "id", + "object", + "created_at", + "model", + "metadata" + ] + } + } + } + } + }, + "tags": ["Assistants"] + }, + "get": { + "operationId": "AssistantsController_list", + "summary": "List assistants", + "description": "Returns a list of assistants.", + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "object": { + "type": "string", + "enum": ["list"], + "description": "The object type, which is always 'list' for a list response." + }, + "data": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The unique identifier of the assistant." + }, + "object": { + "type": "string", + "enum": ["assistant"], + "description": "The object type, which is always 'assistant'." + }, + "created_at": { + "type": "integer", + "description": "Unix timestamp (in seconds) of when the assistant was created." + }, + "model": { + "type": "string", + "description": "The model identifier used by the assistant." + }, + "metadata": { + "type": "object", + "description": "Set of key-value pairs that can be attached to the assistant.", + "additionalProperties": true + } + }, + "required": [ + "id", + "object", + "created_at", + "model", + "metadata" + ] + } + } + }, + "required": ["object", "data"] } } } @@ -88,7 +481,77 @@ "get": { "operationId": "AssistantsController_findOne", "summary": "Get assistant", - "description": "Retrieves a specific assistant defined by an assistant's `id`.", + "description": "Retrieves a specific assistant by ID.", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "description": "The unique identifier of the assistant.", + "schema": { + "type": "string" + } + }, + { + "name": "OpenAI-Beta", + "required": true, + "in": "header", + "description": "Beta feature header.", + "schema": { + "type": "string", + "enum": ["assistants=v2"] + } + } + ], + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The unique identifier of the assistant." + }, + "object": { + "type": "string", + "enum": ["assistant"], + "description": "The object type, which is always 'assistant'." + }, + "created_at": { + "type": "integer", + "description": "Unix timestamp (in seconds) of when the assistant was created." + }, + "model": { + "type": "string", + "description": "The model identifier used by the assistant." + }, + "metadata": { + "type": "object", + "description": "Set of key-value pairs attached to the assistant.", + "additionalProperties": true + } + }, + "required": [ + "id", + "object", + "created_at", + "model", + "metadata" + ] + } + } + } + } + }, + "tags": ["Assistants"] + }, + "delete": { + "operationId": "AssistantsController_remove", + "summary": "Delete assistant", + "description": "Deletes a specific assistant by ID.", "parameters": [ { "name": "id", @@ -106,36 +569,24 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AssistantEntity" - } - } - } - } - }, - "tags": ["Assistants"] - }, - "delete": { - "operationId": "AssistantsController_remove", - "summary": "Delete assistant", - "description": "Deletes a specific assistant defined by an assistant's `id`.", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "description": "The unique identifier of the assistant.", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "The assistant has been successfully deleted.", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DeleteAssistantResponseDto" + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The unique identifier of the deleted assistant." + }, + "object": { + "type": "string", + "enum": ["assistant.deleted"], + "description": "The object type for a deleted assistant." + }, + "deleted": { + "type": "boolean", + "enum": [true], + "description": "Indicates the assistant was successfully deleted." + } + }, + "required": ["id", "object", "deleted"] } } } @@ -2199,6 +2650,84 @@ "tags": ["Engines"] } }, + "/engines/{name}/releases/{version}": { + "get": { + "summary": "List variants for a specific engine version", + "description": "Lists all available variants (builds) for a specific version of an engine. Variants can include different CPU architectures (AVX, AVX2, AVX512), GPU support (CUDA, Vulkan), and operating systems (Windows, Linux, macOS).", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string", + "enum": ["llama-cpp", "onnxruntime", "tensorrt-llm"], + "default": "llama-cpp" + }, + "description": "The type of engine" + }, + { + "name": "version", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "The version of the engine" + }, + { + "name": "show", + "in": "query", + "required": false, + "schema": { + "type": "string", + "enum": ["all", "compatible"], + "default": "all" + }, + "description": "Filter the variants list. Use 'compatible' to show only variants compatible with the current system, or 'all' to show all available variants." + } + ], + "responses": { + "200": { + "description": "Successfully retrieved variants list", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The name of the variant, including OS, architecture, and capabilities", + "example": "linux-amd64-avx-cuda-11-7" + }, + "created_at": { + "type": "string", + "format": "date-time", + "description": "Creation timestamp of the variant", + "example": "2024-11-13T04:51:16Z" + }, + "size": { + "type": "integer", + "description": "Size of the variant in bytes", + "example": 151224604 + }, + "download_count": { + "type": "integer", + "description": "Number of times this variant has been downloaded", + "example": 0 + } + } + } + } + } + } + } + }, + "tags": ["Engines"] + } + }, "/engines/{name}/releases/latest": { "get": { "summary": "Get latest release", @@ -2314,7 +2843,7 @@ "get_models_url": { "type": "string", "description": "The URL to get models", - "example": "https://api.openai.com/v1/models" + "example": "https://api.openai.com/models" } } } @@ -3378,6 +3907,7 @@ "Files", "Hardware", "Events", + "Assistants", "Threads", "Messages", "Pulling Models", @@ -4858,8 +5388,8 @@ "engine", "version", "inference_params", - "TransformReq", - "TransformResp", + "transform_req", + "transform_resp", "metadata" ], "properties": { @@ -4867,9 +5397,9 @@ "type": "string", "description": "The identifier of the model." }, - "api_key_template": { + "header_template": { "type": "string", - "description": "Template for the API key header." + "description": "Template for the header." }, "engine": { "type": "string", @@ -4902,7 +5432,7 @@ } } }, - "TransformReq": { + "transform_req": { "type": "object", "properties": { "get_models": { @@ -4924,7 +5454,7 @@ } } }, - "TransformResp": { + "transform_resp": { "type": "object", "properties": { "chat_completions": { @@ -5632,9 +6162,9 @@ "description": "Number of GPU layers.", "example": 33 }, - "api_key_template": { + "header_template": { "type": "string", - "description": "Template for the API key header." + "description": "Template for the header." }, "version": { "type": "string", diff --git a/web/containers/AutoLink/index.tsx b/web/containers/AutoLink/index.tsx index 66c84f7f7..0f10f478a 100644 --- a/web/containers/AutoLink/index.tsx +++ b/web/containers/AutoLink/index.tsx @@ -10,23 +10,25 @@ const AutoLink = ({ text }: Props) => { return ( <> - {text.split(delimiter).map((word) => { - const match = word.match(delimiter) - if (match) { - const url = match[0] - return ( - - {url} - - ) - } - return word - })} + {text && + typeof text === 'string' && + text.split(delimiter).map((word) => { + const match = word.match(delimiter) + if (match) { + const url = match[0] + return ( + + {url} + + ) + } + return word + })} ) } diff --git a/web/containers/CenterPanelContainer/index.tsx b/web/containers/CenterPanelContainer/index.tsx index b3df8face..2be54ac09 100644 --- a/web/containers/CenterPanelContainer/index.tsx +++ b/web/containers/CenterPanelContainer/index.tsx @@ -7,9 +7,9 @@ import { twMerge } from 'tailwind-merge' import { MainViewState } from '@/constants/screens' -import { LEFT_PANEL_WIDTH } from '../LeftPanelContainer' +import { leftPanelWidthAtom } from '../LeftPanelContainer' -import { RIGHT_PANEL_WIDTH } from '../RightPanelContainer' +import { rightPanelWidthAtom } from '../RightPanelContainer' import { mainViewStateAtom, @@ -28,6 +28,8 @@ const CenterPanelContainer = ({ children, isShowStarterScreen }: Props) => { const showLeftPanel = useAtomValue(showLeftPanelAtom) const showRightPanel = useAtomValue(showRightPanelAtom) const mainViewState = useAtomValue(mainViewStateAtom) + const rightPanelWidth = useAtomValue(rightPanelWidthAtom) + const leftPanelWidth = useAtomValue(leftPanelWidthAtom) return (
{ maxWidth: matches ? '100%' : mainViewState === MainViewState.Thread && !isShowStarterScreen - ? `calc(100% - (${showRightPanel ? Number(localStorage.getItem(RIGHT_PANEL_WIDTH)) : 0}px + ${showLeftPanel ? Number(localStorage.getItem(LEFT_PANEL_WIDTH)) : 0}px))` + ? `calc(100% - (${showRightPanel ? rightPanelWidth : 0}px + ${showLeftPanel ? leftPanelWidth : 0}px))` : '100%', }} > diff --git a/web/containers/ErrorMessage/index.test.tsx b/web/containers/ErrorMessage/index.test.tsx index d9866d3c0..cb0cc3cf6 100644 --- a/web/containers/ErrorMessage/index.test.tsx +++ b/web/containers/ErrorMessage/index.test.tsx @@ -69,7 +69,7 @@ describe('ErrorMessage Component', () => { render() - expect(screen.getByText('troubleshooting assistance')).toBeInTheDocument() + expect(screen.getByText('Troubleshooting')).toBeInTheDocument() }) it('opens troubleshooting modal when link is clicked', () => { @@ -84,7 +84,7 @@ describe('ErrorMessage Component', () => { render() - fireEvent.click(screen.getByText('troubleshooting assistance')) + fireEvent.click(screen.getByText('Troubleshooting')) expect(mockSetModalTroubleShooting).toHaveBeenCalledWith(true) }) }) diff --git a/web/containers/ErrorMessage/index.tsx b/web/containers/ErrorMessage/index.tsx index 71fd56c5d..ab5a35d32 100644 --- a/web/containers/ErrorMessage/index.tsx +++ b/web/containers/ErrorMessage/index.tsx @@ -1,3 +1,5 @@ +import { useRef, useState } from 'react' + import { EngineManager, ErrorCode, @@ -7,6 +9,8 @@ import { import { useAtomValue, useSetAtom } from 'jotai' +import { CheckIcon, ClipboardIcon, SearchCodeIcon } from 'lucide-react' + import AutoLink from '@/containers/AutoLink' import ModalTroubleShooting, { modalTroubleShootingAtom, @@ -19,39 +23,40 @@ import { mainViewStateAtom } from '@/helpers/atoms/App.atom' import { activeAssistantAtom } from '@/helpers/atoms/Assistant.atom' import { selectedSettingAtom } from '@/helpers/atoms/Setting.atom' -const ErrorMessage = ({ message }: { message: ThreadMessage }) => { +const ErrorMessage = ({ + message, + errorComponent, +}: { + message?: ThreadMessage + errorComponent?: React.ReactNode +}) => { const setModalTroubleShooting = useSetAtom(modalTroubleShootingAtom) const setMainState = useSetAtom(mainViewStateAtom) const setSelectedSettingScreen = useSetAtom(selectedSettingAtom) const activeAssistant = useAtomValue(activeAssistantAtom) - - const defaultDesc = () => { - return ( - <> -

- {`Something's wrong.`} Access  - setModalTroubleShooting(true)} - > - troubleshooting assistance - -  now. -

- - - ) - } + const errorDivRef = useRef(null) + const [copied, setCopied] = useState(false) const getEngine = () => { const engineName = activeAssistant?.model?.engine return engineName ? EngineManager.instance().get(engineName) : null } + const handleCopy = () => { + if (errorDivRef.current) { + const errorText = errorDivRef.current.innerText + if (errorText) { + navigator.clipboard.writeText(errorText) + setCopied(true) + setTimeout(() => setCopied(false), 2000) + } + } + } + const getErrorTitle = () => { const engine = getEngine() - switch (message.metadata?.error_code) { + switch (message?.metadata?.error_code) { case ErrorCode.InvalidApiKey: case ErrorCode.AuthenticationError: return ( @@ -62,23 +67,23 @@ const ErrorMessage = ({ message }: { message: ThreadMessage }) => { className="font-medium text-[hsla(var(--app-link))] underline" onClick={() => { setMainState(MainViewState.Settings) - engine?.name && setSelectedSettingScreen(engine.name) + setSelectedSettingScreen(activeAssistant?.model?.engine ?? '') }} > Settings {' '} and try again. - {defaultDesc()} ) + default: return (

- {message.content[0]?.text?.value === 'Failed to fetch' && + {message?.content[0]?.text?.value === 'Failed to fetch' && engine && engine?.name !== InferenceEngine.cortex_llamacpp ? ( @@ -90,7 +95,9 @@ const ErrorMessage = ({ message }: { message: ThreadMessage }) => { {message?.content[0]?.text?.value && ( )} - {defaultDesc()} + {!message?.content[0]?.text?.value && ( + Something went wrong. Please try again. + )} )}

@@ -99,15 +106,57 @@ const ErrorMessage = ({ message }: { message: ThreadMessage }) => { } return ( -
- {!!message.metadata?.error && ( -
- {getErrorTitle()} +
+
+
+
+ + Error +
+
+
+ setModalTroubleShooting(true)} + > + + Troubleshooting + + +
+
+ {copied ? ( + <> + + Copied + + ) : ( + <> + + Copy + + )} +
+
- )} +
+
+ {errorComponent ? errorComponent : getErrorTitle()} +
+
+
) } diff --git a/web/containers/Layout/BottomPanel/SystemMonitor/SystemMonitor.test.tsx b/web/containers/Layout/BottomPanel/SystemMonitor/SystemMonitor.test.tsx index 2eba5edbb..ca336b0e5 100644 --- a/web/containers/Layout/BottomPanel/SystemMonitor/SystemMonitor.test.tsx +++ b/web/containers/Layout/BottomPanel/SystemMonitor/SystemMonitor.test.tsx @@ -87,7 +87,7 @@ describe('SystemMonitor', () => { expect(screen.getByText('Running Models')).toBeInTheDocument() expect(screen.getByText('App Log')).toBeInTheDocument() - expect(screen.getByText('7.45/14.90 GB')).toBeInTheDocument() + expect(screen.getByText('7.45GB / 14.90GB')).toBeInTheDocument() expect(screen.getByText('30%')).toBeInTheDocument() }) diff --git a/web/containers/Layout/BottomPanel/SystemMonitor/index.tsx b/web/containers/Layout/BottomPanel/SystemMonitor/index.tsx index 3dfdff2f9..bbced52f7 100644 --- a/web/containers/Layout/BottomPanel/SystemMonitor/index.tsx +++ b/web/containers/Layout/BottomPanel/SystemMonitor/index.tsx @@ -134,8 +134,8 @@ const SystemMonitor = () => {
Memory
- {toGibibytes(usedRam, { hideUnit: true })}/ - {toGibibytes(totalRam, { hideUnit: true })} GB + {toGibibytes(usedRam, { hideUnit: true })}GB /{' '} + {toGibibytes(totalRam, { hideUnit: true })}GB
@@ -149,41 +149,43 @@ const SystemMonitor = () => {
{gpus.length > 0 && (
- {gpus.map((gpu, index) => { - const gpuUtilization = utilizedMemory( - gpu.memoryFree, - gpu.memoryTotal - ) - return ( -
-
- - {gpu.name} - -
-
- - {gpu.memoryTotal - gpu.memoryFree}/ - {gpu.memoryTotal} - - MB + {gpus + .filter((gpu) => gpu.activated === true) + .map((gpu, index) => { + const gpuUtilization = utilizedMemory( + gpu.free_vram, + gpu.total_vram + ) + return ( +
+
+ + {gpu.name} + +
+
+ + {gpu.total_vram - gpu.free_vram}/ + {gpu.total_vram} + + MB +
-
-
- - - {gpuUtilization}% - +
+ + + {gpuUtilization}% + +
-
- ) - })} + ) + })}
)}
diff --git a/web/containers/Layout/TopPanel/index.tsx b/web/containers/Layout/TopPanel/index.tsx index 9ba393e09..a00c2d22c 100644 --- a/web/containers/Layout/TopPanel/index.tsx +++ b/web/containers/Layout/TopPanel/index.tsx @@ -1,11 +1,9 @@ import { Fragment } from 'react' -import { Button } from '@janhq/joi' +import { Button, Tooltip } from '@janhq/joi' import { useAtom, useAtomValue, useSetAtom } from 'jotai' import { PanelLeftCloseIcon, - PanelLeftOpenIcon, - PanelRightOpenIcon, PanelRightCloseIcon, MinusIcon, MenuIcon, @@ -13,6 +11,9 @@ import { PaletteIcon, XIcon, PenSquareIcon, + Settings2, + History, + PanelLeftOpenIcon, } from 'lucide-react' import { twMerge } from 'tailwind-merge' @@ -91,7 +92,14 @@ const TopPanel = () => { ) : ( )} @@ -135,7 +143,10 @@ const TopPanel = () => { } }} > - + } + content="Thread Settings" + /> )} diff --git a/web/containers/Layout/index.tsx b/web/containers/Layout/index.tsx index 29fda70de..7d5f61ed3 100644 --- a/web/containers/Layout/index.tsx +++ b/web/containers/Layout/index.tsx @@ -26,6 +26,8 @@ import ImportModelOptionModal from '@/screens/Settings/ImportModelOptionModal' import ImportingModelModal from '@/screens/Settings/ImportingModelModal' import SelectingModelModal from '@/screens/Settings/SelectingModelModal' +import { getAppDistinctId, updateDistinctId } from '@/utils/settings' + import LoadingModal from '../LoadingModal' import MainViewContainer from '../MainViewContainer' @@ -93,8 +95,16 @@ const BaseLayout = () => { return properties }, }) - posthog.opt_in_capturing() - posthog.register({ app_version: VERSION }) + // Attempt to restore distinct Id from app global settings + getAppDistinctId() + .then((id) => { + if (id) posthog.identify(id) + }) + .finally(() => { + posthog.opt_in_capturing() + posthog.register({ app_version: VERSION }) + updateDistinctId(posthog.get_distinct_id()) + }) } else { posthog.opt_out_capturing() } diff --git a/web/containers/LeftPanelContainer/index.tsx b/web/containers/LeftPanelContainer/index.tsx index c6665a037..523af5ddb 100644 --- a/web/containers/LeftPanelContainer/index.tsx +++ b/web/containers/LeftPanelContainer/index.tsx @@ -7,7 +7,7 @@ import { } from 'react' import { ScrollArea, useClickOutside, useMediaQuery } from '@janhq/joi' -import { useAtom, useAtomValue } from 'jotai' +import { atom, useAtom, useAtomValue } from 'jotai' import { twMerge } from 'tailwind-merge' @@ -18,13 +18,12 @@ type Props = PropsWithChildren const DEFAULT_LEFT_PANEL_WIDTH = 200 export const LEFT_PANEL_WIDTH = 'leftPanelWidth' +export const leftPanelWidthAtom = atom(DEFAULT_LEFT_PANEL_WIDTH) const LeftPanelContainer = ({ children }: Props) => { const [leftPanelRef, setLeftPanelRef] = useState(null) const [isResizing, setIsResizing] = useState(false) - const [threadLeftPanelWidth, setLeftPanelWidth] = useState( - Number(localStorage.getItem(LEFT_PANEL_WIDTH)) || DEFAULT_LEFT_PANEL_WIDTH - ) + const [leftPanelWidth, setLeftPanelWidth] = useAtom(leftPanelWidthAtom) const [showLeftPanel, setShowLeftPanel] = useAtom(showLeftPanelAtom) const matches = useMediaQuery('(max-width: 880px)') const reduceTransparent = useAtomValue(reduceTransparentAtom) @@ -37,10 +36,12 @@ const LeftPanelContainer = ({ children }: Props) => { const startResizing = useCallback(() => { setIsResizing(true) + document.body.classList.add('select-none') }, []) const stopResizing = useCallback(() => { setIsResizing(false) + document.body.classList.remove('select-none') }, []) const resize = useCallback( @@ -69,7 +70,7 @@ const LeftPanelContainer = ({ children }: Props) => { } } }, - [isResizing, leftPanelRef, setShowLeftPanel] + [isResizing, leftPanelRef, setLeftPanelWidth, setShowLeftPanel] ) useEffect(() => { @@ -83,7 +84,7 @@ const LeftPanelContainer = ({ children }: Props) => { window.removeEventListener('mousemove', resize) window.removeEventListener('mouseup', stopResizing) } - }, [resize, stopResizing]) + }, [resize, setLeftPanelWidth, stopResizing]) return (
{ reduceTransparent && 'left-0 border-r border-[hsla(var(--app-border))] bg-[hsla(var(--left-panel-bg))]' )} - style={{ width: showLeftPanel ? threadLeftPanelWidth : 0 }} + style={{ width: showLeftPanel ? leftPanelWidth : 0 }} onMouseDown={(e) => isResizing && e.stopPropagation()} > diff --git a/web/containers/Loader/ModelReload.tsx b/web/containers/Loader/ModelReload.tsx index 29709c0da..fbe673788 100644 --- a/web/containers/Loader/ModelReload.tsx +++ b/web/containers/Loader/ModelReload.tsx @@ -44,11 +44,6 @@ export default function ModelReload() { Reloading model {stateModel.model?.id}
-
- - Model is reloading to apply new changes. - -
) } diff --git a/web/containers/MainViewContainer/index.tsx b/web/containers/MainViewContainer/index.tsx index 811f19c6e..6417ec63a 100644 --- a/web/containers/MainViewContainer/index.tsx +++ b/web/containers/MainViewContainer/index.tsx @@ -1,4 +1,4 @@ -import { memo } from 'react' +import { memo, useEffect, useState } from 'react' import { motion as m } from 'framer-motion' import { useAtomValue } from 'jotai' @@ -12,10 +12,27 @@ import LocalServerScreen from '@/screens/LocalServer' import SettingsScreen from '@/screens/Settings' import ThreadScreen from '@/screens/Thread' -import { mainViewStateAtom } from '@/helpers/atoms/App.atom' +import { + mainViewStateAtom, + showSystemMonitorPanelAtom, +} from '@/helpers/atoms/App.atom' const MainViewContainer = () => { const mainViewState = useAtomValue(mainViewStateAtom) + const showSystemMonitorPanel = useAtomValue(showSystemMonitorPanelAtom) + const [height, setHeight] = useState(0) + + useEffect(() => { + if (showSystemMonitorPanel) { + const element = document.querySelector('.system-monitor-panel') + + if (element) { + setHeight(element.clientHeight) // You can also use offsetHeight if needed + } + } else { + setHeight(0) + } + }, [showSystemMonitorPanel]) let children = null switch (mainViewState) { @@ -37,7 +54,10 @@ const MainViewContainer = () => { } return ( -
+
(null) const configuredModels = useAtomValue(configuredModelsAtom) + const { stopModel } = useActiveModel() const featuredModels = configuredModels.filter( (x) => @@ -226,6 +229,7 @@ const ModelDropdown = ({ const model = downloadedModels.find((m) => m.id === modelId) setSelectedModel(model) setOpen(false) + stopModel() if (activeThread) { // Change assistand tools based on model support RAG @@ -248,18 +252,13 @@ const ModelDropdown = ({ ], }) - const defaultContextLength = Math.min( - 8192, - model?.settings.ctx_len ?? 8192 - ) - + const contextLength = model?.settings.ctx_len + ? Math.min(8192, model?.settings.ctx_len ?? 8192) + : undefined const overriddenParameters = { - ctx_len: model?.settings.ctx_len ? defaultContextLength : undefined, - max_tokens: defaultContextLength - ? Math.min( - model?.parameters.max_tokens ?? 8192, - defaultContextLength - ) + ctx_len: contextLength, + max_tokens: contextLength + ? Math.min(model?.parameters.max_tokens ?? 8192, contextLength) : model?.parameters.max_tokens, } @@ -289,6 +288,7 @@ const ModelDropdown = ({ updateThreadMetadata, setThreadModelParams, updateModelParameter, + stopModel, ] ) @@ -429,7 +429,7 @@ const ModelDropdown = ({ /> )}
- {engine.name} + {getTitleByEngine(engine.name)}
@@ -475,7 +475,7 @@ const ModelDropdown = ({ >

{model.name} @@ -549,75 +549,82 @@ const ModelDropdown = ({ (c) => c.id === model.id ) return ( -

  • { - if (!isConfigured && engine.type === 'remote') - return null - if (isDownloaded) { - onClickModelItem(model.id) - } - }} - > -
    -

    + {isDownloaded && ( +

  • - {model.name} -

    - -
    • -
    - {!isDownloaded && ( - - {toGibibytes(model.metadata?.size)} - - )} - {!isDownloading && !isDownloaded ? ( - - downloadModel( - model.sources[0].url, - model.id - ) + onClick={() => { + if ( + !isConfigured && + engine.type === 'remote' + ) + return null + if (isDownloaded) { + onClickModelItem(model.id) } - /> - ) : ( - Object.values(downloadStates) - .filter((x) => x.modelId === model.id) - .map((item) => ( - +
    +

    + {model.name} +

    + +
    +
    + {!isDownloaded && ( + + {toGibibytes(model.metadata?.size)} + + )} + {!isDownloading && !isDownloaded ? ( + + downloadModel( + model.sources[0].url, + model.id + ) } - size={100} /> - )) - )} -
    - + ) : ( + Object.values(downloadStates) + .filter((x) => x.modelId === model.id) + .map((item) => ( + + )) + )} +
    + + )} + ) })} diff --git a/web/containers/ModelLabel/ModelLabel.test.tsx b/web/containers/ModelLabel/ModelLabel.test.tsx index 48504ff6a..a4d8e82a7 100644 --- a/web/containers/ModelLabel/ModelLabel.test.tsx +++ b/web/containers/ModelLabel/ModelLabel.test.tsx @@ -54,7 +54,7 @@ describe('ModelLabel', () => { it('renders SlowOnYourDeviceLabel when minimumRamModel is less than totalRam but greater than availableRam', async () => { mockUseAtomValue - .mockReturnValueOnce(100) + .mockReturnValueOnce(50) .mockReturnValueOnce(50) .mockReturnValueOnce(10) mockUseActiveModel.mockReturnValue({ diff --git a/web/containers/ModelLabel/index.tsx b/web/containers/ModelLabel/index.tsx index a6237ada6..711913616 100644 --- a/web/containers/ModelLabel/index.tsx +++ b/web/containers/ModelLabel/index.tsx @@ -38,15 +38,20 @@ const ModelLabel = ({ metadata, compact }: Props) => { const { settings } = useSettings() const getLabel = (size: number) => { - const minimumRamModel = size * 1.25 - const availableRam = - settings?.run_mode === 'gpu' - ? availableVram * 1000000 // MB to bytes - : totalRam - usedRam + (activeModel?.metadata?.size ?? 0) + const minimumRamModel = (size * 1.25) / (1024 * 1024) + + const availableRam = settings?.gpus?.some((gpu) => gpu.activated) + ? availableVram * 1000000 // MB to bytes + : totalRam - + (usedRam + + (activeModel?.metadata?.size + ? (activeModel.metadata.size * 1.25) / (1024 * 1024) + : 0)) + if (minimumRamModel > totalRam) { return ( gpu.activated) ? 'VRAM' : 'RAM'} compact={compact} /> ) diff --git a/web/containers/Providers/ModelHandler.tsx b/web/containers/Providers/ModelHandler.tsx index 2c027539e..cceb88a4c 100644 --- a/web/containers/Providers/ModelHandler.tsx +++ b/web/containers/Providers/ModelHandler.tsx @@ -143,8 +143,7 @@ export default function ModelHandler() { return } - // The thread title should not be updated if the message is less than 10 words - // And no new line character is present + // No new line character is presented in the title // And non-alphanumeric characters should be removed if (messageContent.includes('\n')) { messageContent = messageContent.replace(/\n/g, ' ') diff --git a/web/containers/Providers/index.tsx b/web/containers/Providers/index.tsx index bed0f07ec..5d14ea95a 100644 --- a/web/containers/Providers/index.tsx +++ b/web/containers/Providers/index.tsx @@ -4,15 +4,11 @@ import { PropsWithChildren } from 'react' import { Toaster } from 'react-hot-toast' -import { SWRConfig } from 'swr' - import EventListener from '@/containers/Providers/EventListener' import JotaiWrapper from '@/containers/Providers/Jotai' import ThemeWrapper from '@/containers/Providers/Theme' -import Umami from '@/utils/umami' - import { CoreConfigurator } from './CoreConfigurator' import DataLoader from './DataLoader' @@ -28,7 +24,6 @@ const Providers = ({ children }: PropsWithChildren) => { - <> diff --git a/web/containers/RightPanelContainer/index.test.tsx b/web/containers/RightPanelContainer/index.test.tsx index 4bb08913f..b8549254b 100644 --- a/web/containers/RightPanelContainer/index.test.tsx +++ b/web/containers/RightPanelContainer/index.test.tsx @@ -1,9 +1,11 @@ import '@testing-library/jest-dom' +import { waitFor } from '@testing-library/react' import React from 'react' import { render, fireEvent } from '@testing-library/react' -import RightPanelContainer from './index' -import { useAtom } from 'jotai' +import RightPanelContainer, { rightPanelWidthAtom } from './index' +import { showRightPanelAtom } from '@/helpers/atoms/App.atom' +import { reduceTransparentAtom } from '@/helpers/atoms/Setting.atom' // Mocking ResizeObserver class ResizeObserver { @@ -34,24 +36,24 @@ jest.mock('jotai', () => { const originalModule = jest.requireActual('jotai') return { ...originalModule, - useAtom: jest.fn(), - useAtomValue: jest.fn(), + useAtomValue: jest.fn((atom) => { + if (atom === reduceTransparentAtom) return false + if (atom === showRightPanelAtom) return true + }), + useAtom: jest.fn((atom) => { + if (atom === rightPanelWidthAtom) return [280, jest.fn()] + if (atom === showRightPanelAtom) return [true, mockSetShowRightPanel] + return [null, jest.fn()] + }), } }) const mockSetShowRightPanel = jest.fn() -const mockShowRightPanel = true // Change this to test the panel visibility beforeEach(() => { // Setting up the localStorage mock localStorage.clear() localStorage.setItem('rightPanelWidth', '280') // Setting a default width - - // Mocking the atom behavior - ;(useAtom as jest.Mock).mockImplementation(() => [ - mockShowRightPanel, - mockSetShowRightPanel, - ]) }) describe('RightPanelContainer', () => { @@ -66,12 +68,15 @@ describe('RightPanelContainer', () => { expect(getByText('Child Content')).toBeInTheDocument() }) - it('initializes width from localStorage', () => { + it('initializes width from localStorage', async () => { const { container } = render() - // Check the width from localStorage is applied const rightPanel = container.firstChild as HTMLDivElement - expect(rightPanel.style.width).toBe('280px') // Width from localStorage + + // Wait for the width to be applied + await waitFor(() => { + expect(rightPanel.style.width).toBe('280px') // Correct width from localStorage + }) }) it('changes width on resizing', () => { diff --git a/web/containers/RightPanelContainer/index.tsx b/web/containers/RightPanelContainer/index.tsx index 27d339bb7..7443ab61a 100644 --- a/web/containers/RightPanelContainer/index.tsx +++ b/web/containers/RightPanelContainer/index.tsx @@ -7,7 +7,7 @@ import { } from 'react' import { ScrollArea, useClickOutside, useMediaQuery } from '@janhq/joi' -import { useAtom, useAtomValue } from 'jotai' +import { atom, useAtom, useAtomValue } from 'jotai' import { twMerge } from 'tailwind-merge' @@ -19,11 +19,11 @@ type Props = PropsWithChildren const DEFAULT_RIGHT_PANEL_WIDTH = 280 export const RIGHT_PANEL_WIDTH = 'rightPanelWidth' +export const rightPanelWidthAtom = atom(DEFAULT_RIGHT_PANEL_WIDTH) + const RightPanelContainer = ({ children }: Props) => { const [isResizing, setIsResizing] = useState(false) - const [threadRightPanelWidth, setRightPanelWidth] = useState( - Number(localStorage.getItem(RIGHT_PANEL_WIDTH)) || DEFAULT_RIGHT_PANEL_WIDTH - ) + const [rightPanelWidth, setRightPanelWidth] = useAtom(rightPanelWidthAtom) const [rightPanelRef, setRightPanelRef] = useState( null ) @@ -40,10 +40,12 @@ const RightPanelContainer = ({ children }: Props) => { const startResizing = useCallback(() => { setIsResizing(true) + document.body.classList.add('select-none') }, []) const stopResizing = useCallback(() => { setIsResizing(false) + document.body.classList.remove('select-none') }, []) const resize = useCallback( @@ -72,7 +74,7 @@ const RightPanelContainer = ({ children }: Props) => { } } }, - [isResizing, rightPanelRef, setShowRightPanel] + [isResizing, rightPanelRef, setRightPanelWidth, setShowRightPanel] ) useEffect(() => { @@ -86,7 +88,7 @@ const RightPanelContainer = ({ children }: Props) => { window.removeEventListener('mousemove', resize) window.removeEventListener('mouseup', stopResizing) } - }, [resize, stopResizing]) + }, [resize, setRightPanelWidth, stopResizing]) return (
    { reduceTransparent && 'border-l border-[hsla(var(--app-border))] bg-[hsla(var(--right-panel-bg))]' )} - style={{ width: showRightPanel ? threadRightPanelWidth : 0 }} + style={{ width: showRightPanel ? rightPanelWidth : 0 }} onMouseDown={(e) => isResizing && e.preventDefault()} > diff --git a/web/containers/ServerLogs/index.tsx b/web/containers/ServerLogs/index.tsx index 2e978bd23..b89a4c237 100644 --- a/web/containers/ServerLogs/index.tsx +++ b/web/containers/ServerLogs/index.tsx @@ -93,205 +93,211 @@ const ServerLogs = (props: ServerLogsProps) => { }, [listRef.current?.scrollHeight, isUserManuallyScrollingUp, logs]) return ( - - {withCopy && ( -
    -
    - - -
    -
    - )} -
    - {logs.length > 0 ? ( - - {logs.slice(-limit).map((log, i) => { - return ( -

    - {log} -

    - ) - })} -     - ) : ( -
    - - - - +
    + {withCopy && ( +
    +
    + + +
    )}
    - + +
    + {logs.length > 0 ? ( + + {logs.slice(-limit).map((log, i) => { + return ( +

    + {log} +

    + ) + })} +     + ) : ( +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

    + Empty logs +

    +
    + )} +
    +     + ) } diff --git a/web/containers/SliderRightPanel/index.tsx b/web/containers/SliderRightPanel/index.tsx index 3fad10212..5022845c9 100644 --- a/web/containers/SliderRightPanel/index.tsx +++ b/web/containers/SliderRightPanel/index.tsx @@ -73,7 +73,7 @@ const SliderRightPanel = ({ trigger={ (MainViewState.Thread) export const defaultJanDataFolderAtom = atom('') +export const LocalEngineDefaultVariantAtom = atom('') + const SHOW_RIGHT_PANEL = 'showRightPanel' // Store panel atom diff --git a/web/helpers/atoms/AppConfig.atom.ts b/web/helpers/atoms/AppConfig.atom.ts index 68a375f3b..db4157422 100644 --- a/web/helpers/atoms/AppConfig.atom.ts +++ b/web/helpers/atoms/AppConfig.atom.ts @@ -5,8 +5,15 @@ const EXPERIMENTAL_FEATURE = 'experimentalFeature' const PROXY_FEATURE_ENABLED = 'proxyFeatureEnabled' const VULKAN_ENABLED = 'vulkanEnabled' const IGNORE_SSL = 'ignoreSSLFeature' +const VERIFY_PROXY_SSL = 'verifyProxySSL' +const VERIFY_PROXY_HOST_SSL = 'verifyProxyHostSSL' +const VERIFY_PEER_SSL = 'verifyPeerSSL' +const VERIFY_HOST_SSL = 'verifyHostSSL' const HTTPS_PROXY_FEATURE = 'httpsProxyFeature' +const PROXY_USERNAME = 'proxyUsername' +const PROXY_PASSWORD = 'proxyPassword' const QUICK_ASK_ENABLED = 'quickAskEnabled' +const NO_PROXY = 'noProxy' export const janDataFolderPathAtom = atom('') @@ -27,9 +34,56 @@ export const proxyAtom = atomWithStorage(HTTPS_PROXY_FEATURE, '', undefined, { getOnInit: true, }) +export const proxyUsernameAtom = atomWithStorage( + PROXY_USERNAME, + '', + undefined, + { getOnInit: true } +) + +export const proxyPasswordAtom = atomWithStorage( + PROXY_PASSWORD, + '', + undefined, + { getOnInit: true } +) + export const ignoreSslAtom = atomWithStorage(IGNORE_SSL, false, undefined, { getOnInit: true, }) + +export const noProxyAtom = atomWithStorage(NO_PROXY, '', undefined, { + getOnInit: false, +}) + +export const verifyProxySslAtom = atomWithStorage( + VERIFY_PROXY_SSL, + false, + undefined, + { getOnInit: true } +) + +export const verifyProxyHostSslAtom = atomWithStorage( + VERIFY_PROXY_HOST_SSL, + false, + undefined, + { getOnInit: true } +) + +export const verifyPeerSslAtom = atomWithStorage( + VERIFY_PEER_SSL, + false, + undefined, + { getOnInit: true } +) + +export const verifyHostSslAtom = atomWithStorage( + VERIFY_HOST_SSL, + false, + undefined, + { getOnInit: true } +) + export const vulkanEnabledAtom = atomWithStorage( VULKAN_ENABLED, false, diff --git a/web/hooks/useConfigurations.test.ts b/web/hooks/useConfigurations.test.ts new file mode 100644 index 000000000..1fab89f59 --- /dev/null +++ b/web/hooks/useConfigurations.test.ts @@ -0,0 +1,137 @@ +import { renderHook, act } from '@testing-library/react' +import { useConfigurations } from './useConfigurations' +import { useAtomValue } from 'jotai' +import { extensionManager } from '@/extension' + +// Mock dependencies +jest.mock('jotai', () => { + const originalModule = jest.requireActual('jotai') + return { + ...originalModule, + useAtomValue: jest.fn(), + } +}) + +jest.mock('@/extension', () => ({ + extensionManager: { + get: jest.fn(), + }, +})) + +describe('useConfigurations', () => { + beforeEach(() => { + jest.clearAllMocks() + }) + + it('should call configurePullOptions with correct proxy settings when proxy is enabled', () => { + // Explicitly set mock return values for each call + (useAtomValue as jest.Mock) + .mockReturnValueOnce(true) // proxyEnabled + .mockReturnValueOnce('http://proxy.example.com') // proxyUrl + .mockReturnValueOnce('') // proxyIgnoreSSL + .mockReturnValueOnce(true) // verifyProxySSL + .mockReturnValueOnce(true) // verifyProxyHostSSL + .mockReturnValueOnce(true) // verifyPeerSSL + .mockReturnValueOnce(true) // verifyHostSSL + .mockReturnValueOnce('') // noProxy + .mockReturnValueOnce('username') // proxyUsername + .mockReturnValueOnce('password') // proxyPassword + + + const mockConfigurePullOptions = jest.fn() + ;(extensionManager.get as jest.Mock).mockReturnValue({ + configurePullOptions: mockConfigurePullOptions, + }) + + const { result } = renderHook(() => useConfigurations()) + + act(() => { + result.current.configurePullOptions() + }) + + expect(mockConfigurePullOptions).toHaveBeenCalledWith({ + proxy_username: 'username', + proxy_password: 'password', + proxy_url: 'http://proxy.example.com', + verify_proxy_ssl: true, + verify_proxy_host_ssl: true, + verify_peer_ssl: true, + verify_host_ssl: true, + no_proxy: '', + }) + }) + + it('should call configurePullOptions with empty proxy settings when proxy is disabled', () => { + // Mock atom values + ;(useAtomValue as jest.Mock) + .mockReturnValueOnce(false) // proxyEnabled + .mockReturnValueOnce('') // proxyUrl + .mockReturnValueOnce(false) // proxyIgnoreSSL + .mockReturnValueOnce('') // noProxy + .mockReturnValueOnce('') // proxyUsername + .mockReturnValueOnce('') // proxyPassword + .mockReturnValueOnce(false) // verifyProxySSL + .mockReturnValueOnce(false) // verifyProxyHostSSL + .mockReturnValueOnce(false) // verifyPeerSSL + .mockReturnValueOnce(false) // verifyHostSSL + + const mockConfigurePullOptions = jest.fn() + ;(extensionManager.get as jest.Mock).mockReturnValue({ + configurePullOptions: mockConfigurePullOptions, + }) + + const { result } = renderHook(() => useConfigurations()) + + act(() => { + result.current.configurePullOptions() + }) + + expect(mockConfigurePullOptions).toHaveBeenCalledWith({ + proxy_username: '', + proxy_password: '', + proxy_url: '', + verify_proxy_ssl: false, + verify_proxy_host_ssl: false, + verify_peer_ssl: false, + verify_host_ssl: false, + no_proxy: '', + }) + }) + + it('should set all verify SSL to false when proxyIgnoreSSL is true', () => { + // Mock atom values + ;(useAtomValue as jest.Mock) + .mockReturnValueOnce(true) // proxyEnabled + .mockReturnValueOnce('http://proxy.example.com') // proxyUrl + .mockReturnValueOnce(true) // proxyIgnoreSSL + .mockReturnValueOnce(true) // verifyProxySSL + .mockReturnValueOnce(true) // verifyProxyHostSSL + .mockReturnValueOnce(true) // verifyPeerSSL + .mockReturnValueOnce(true) // verifyHostSSL + .mockReturnValueOnce('') // noProxy + .mockReturnValueOnce('username') // proxyUsername + .mockReturnValueOnce('password') // proxyPassword + + const mockConfigurePullOptions = jest.fn() + ;(extensionManager.get as jest.Mock).mockReturnValue({ + configurePullOptions: mockConfigurePullOptions, + }) + + const { result } = renderHook(() => useConfigurations()) + + act(() => { + result.current.configurePullOptions() + }) + + expect(mockConfigurePullOptions).toHaveBeenCalledWith({ + proxy_username: 'username', + proxy_password: 'password', + proxy_url: 'http://proxy.example.com', + verify_proxy_ssl: false, + verify_proxy_host_ssl: false, + verify_peer_ssl: false, + verify_host_ssl: false, + no_proxy: '', + }) + }) +}) \ No newline at end of file diff --git a/web/hooks/useConfigurations.ts b/web/hooks/useConfigurations.ts index 681239033..0bd96e760 100644 --- a/web/hooks/useConfigurations.ts +++ b/web/hooks/useConfigurations.ts @@ -6,14 +6,28 @@ import { useAtomValue } from 'jotai' import { extensionManager } from '@/extension' import { ignoreSslAtom, + noProxyAtom, proxyAtom, proxyEnabledAtom, + proxyPasswordAtom, + proxyUsernameAtom, + verifyHostSslAtom, + verifyPeerSslAtom, + verifyProxyHostSslAtom, + verifyProxySslAtom, } from '@/helpers/atoms/AppConfig.atom' export const useConfigurations = () => { const proxyEnabled = useAtomValue(proxyEnabledAtom) const proxyUrl = useAtomValue(proxyAtom) const proxyIgnoreSSL = useAtomValue(ignoreSslAtom) + const verifyProxySSL = useAtomValue(verifyProxySslAtom) + const verifyProxyHostSSL = useAtomValue(verifyProxyHostSslAtom) + const verifyPeerSSL = useAtomValue(verifyPeerSslAtom) + const verifyHostSSL = useAtomValue(verifyHostSslAtom) + const noProxy = useAtomValue(noProxyAtom) + const proxyUsername = useAtomValue(proxyUsernameAtom) + const proxyPassword = useAtomValue(proxyPasswordAtom) const configurePullOptions = useCallback(() => { extensionManager @@ -21,20 +35,45 @@ export const useConfigurations = () => { ?.configurePullOptions( proxyEnabled ? { + proxy_username: proxyUsername, + proxy_password: proxyPassword, proxy_url: proxyUrl, - verify_peer_ssl: !proxyIgnoreSSL, + verify_proxy_ssl: proxyIgnoreSSL ? false : verifyProxySSL, + verify_proxy_host_ssl: proxyIgnoreSSL + ? false + : verifyProxyHostSSL, + verify_peer_ssl: proxyIgnoreSSL ? false : verifyPeerSSL, + verify_host_ssl: proxyIgnoreSSL ? false : verifyHostSSL, + no_proxy: noProxy, } : { + proxy_username: '', + proxy_password: '', proxy_url: '', + verify_proxy_ssl: false, + verify_proxy_host_ssl: false, verify_peer_ssl: false, + verify_host_ssl: false, + no_proxy: '', } ) - }, [proxyEnabled, proxyUrl, proxyIgnoreSSL]) + }, [ + proxyEnabled, + proxyUrl, + proxyIgnoreSSL, + noProxy, + proxyUsername, + proxyPassword, + verifyProxySSL, + verifyProxyHostSSL, + verifyPeerSSL, + verifyHostSSL, + ]) useEffect(() => { configurePullOptions() // eslint-disable-next-line react-hooks/exhaustive-deps - }, []) + }, [configurePullOptions]) return { configurePullOptions, diff --git a/web/hooks/useCreateNewThread.ts b/web/hooks/useCreateNewThread.ts index aeef2754f..57ceeb385 100644 --- a/web/hooks/useCreateNewThread.ts +++ b/web/hooks/useCreateNewThread.ts @@ -51,11 +51,13 @@ export const useCreateNewThread = () => { const { recommendedModel } = useRecommendedModel() + const selectedModel = useAtomValue(selectedModelAtom) + const requestCreateNewThread = async ( assistant: (ThreadAssistantInfo & { id: string; name: string }) | Assistant, model?: Model | undefined ) => { - const defaultModel = model || recommendedModel + const defaultModel = model || selectedModel || recommendedModel if (!model) { // if we have model, which means user wants to create new thread from Model hub. Allow them. @@ -80,25 +82,19 @@ export const useCreateNewThread = () => { } // Default context length is 8192 - const defaultContextLength = Math.min( - 8192, - defaultModel?.settings.ctx_len ?? 8192 - ) + const contextLength = defaultModel?.settings?.ctx_len + ? Math.min(8192, defaultModel?.settings?.ctx_len) + : undefined const overriddenSettings = { - ctx_len: defaultModel?.settings.ctx_len - ? Math.min(8192, defaultModel.settings.ctx_len) - : undefined, + ctx_len: contextLength, } // Use ctx length by default const overriddenParameters = { - max_tokens: defaultContextLength - ? Math.min( - defaultModel?.parameters.max_tokens ?? 8192, - defaultContextLength - ) - : defaultModel?.parameters.max_tokens, + max_tokens: contextLength + ? Math.min(defaultModel?.parameters?.max_tokens ?? 8192, contextLength) + : defaultModel?.parameters?.max_tokens, } const createdAt = Date.now() diff --git a/web/hooks/useEngineManagement.ts b/web/hooks/useEngineManagement.ts index 8367ecd20..fec5eb0ed 100644 --- a/web/hooks/useEngineManagement.ts +++ b/web/hooks/useEngineManagement.ts @@ -8,6 +8,8 @@ import { EngineConfig, events, EngineEvent, + Model, + ModelEvent, } from '@janhq/core' import { useAtom } from 'jotai' import { atomWithStorage } from 'jotai/utils' @@ -385,3 +387,31 @@ export const uninstallEngine = async ( throw error } } + +/** + * Add a new remote engine model + * @param name + * @param engine + * @returns + */ +export const addRemoteEngineModel = async (name: string, engine: string) => { + const extension = getExtension() + + if (!extension) { + throw new Error('Extension is not available') + } + + try { + // Call the extension's method + const response = await extension.addRemoteModel({ + id: name, + model: name, + engine: engine as InferenceEngine, + } as unknown as Model) + events.emit(ModelEvent.OnModelsUpdate, { fetch: true }) + return response + } catch (error) { + console.error('Failed to install engine variant:', error) + throw error + } +} diff --git a/web/hooks/useFactoryReset.ts b/web/hooks/useFactoryReset.ts index da2e15b03..7344b2eb1 100644 --- a/web/hooks/useFactoryReset.ts +++ b/web/hooks/useFactoryReset.ts @@ -58,6 +58,7 @@ export default function useFactoryReset() { const configuration: AppConfiguration = { data_folder: defaultJanDataFolder, quick_ask: appConfiguration?.quick_ask ?? false, + distinct_id: appConfiguration?.distinct_id, } await window.core?.api?.updateAppConfiguration(configuration) } diff --git a/web/hooks/useGetSystemResources.test.ts b/web/hooks/useGetSystemResources.test.ts index 10e539e07..78392b612 100644 --- a/web/hooks/useGetSystemResources.test.ts +++ b/web/hooks/useGetSystemResources.test.ts @@ -21,7 +21,7 @@ jest.mock('jotai', () => ({ describe('useGetSystemResources', () => { const mockMonitoringExtension = { - getResourcesInfo: jest.fn(), + getHardware: jest.fn(), getCurrentLoad: jest.fn(), } @@ -38,17 +38,17 @@ describe('useGetSystemResources', () => { }) it('should fetch system resources on initial render', async () => { - mockMonitoringExtension.getResourcesInfo.mockResolvedValue({ - mem: { usedMemory: 4000, totalMemory: 8000 }, + mockMonitoringExtension.getHardware.mockResolvedValue({ + cpu: { usage: 50 }, + ram: { available: 4000, total: 8000 }, }) mockMonitoringExtension.getCurrentLoad.mockResolvedValue({ - cpu: { usage: 50 }, gpu: [], }) const { result } = renderHook(() => useGetSystemResources()) - expect(mockMonitoringExtension.getResourcesInfo).toHaveBeenCalledTimes(1) + expect(mockMonitoringExtension.getHardware).toHaveBeenCalledTimes(1) }) it('should start watching system resources when watch is called', () => { @@ -58,14 +58,14 @@ describe('useGetSystemResources', () => { result.current.watch() }) - expect(mockMonitoringExtension.getResourcesInfo).toHaveBeenCalled() + expect(mockMonitoringExtension.getHardware).toHaveBeenCalled() // Fast-forward time by 2 seconds act(() => { jest.advanceTimersByTime(2000) }) - expect(mockMonitoringExtension.getResourcesInfo).toHaveBeenCalled() + expect(mockMonitoringExtension.getHardware).toHaveBeenCalled() }) it('should stop watching when stopWatching is called', () => { @@ -85,7 +85,7 @@ describe('useGetSystemResources', () => { }) // Expect no additional calls after stopping - expect(mockMonitoringExtension.getResourcesInfo).toHaveBeenCalled() + expect(mockMonitoringExtension.getHardware).toHaveBeenCalled() }) it('should not fetch resources if monitoring extension is not available', async () => { @@ -97,7 +97,7 @@ describe('useGetSystemResources', () => { result.current.getSystemResources() }) - expect(mockMonitoringExtension.getResourcesInfo).not.toHaveBeenCalled() + expect(mockMonitoringExtension.getHardware).not.toHaveBeenCalled() expect(mockMonitoringExtension.getCurrentLoad).not.toHaveBeenCalled() }) }) diff --git a/web/hooks/useGetSystemResources.ts b/web/hooks/useGetSystemResources.ts index a05a6a710..e40100a55 100644 --- a/web/hooks/useGetSystemResources.ts +++ b/web/hooks/useGetSystemResources.ts @@ -1,6 +1,7 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ import { useCallback, useEffect, useState } from 'react' -import { ExtensionTypeEnum, MonitoringExtension } from '@janhq/core' +import { ExtensionTypeEnum, HardwareManagementExtension } from '@janhq/core' import { useSetAtom } from 'jotai' @@ -20,58 +21,62 @@ export default function useGetSystemResources() { NodeJS.Timeout | number | undefined >(undefined) - const setTotalRam = useSetAtom(totalRamAtom) const setGpus = useSetAtom(gpusAtom) - const setUsedRam = useSetAtom(usedRamAtom) const setCpuUsage = useSetAtom(cpuUsageAtom) const setTotalNvidiaVram = useSetAtom(nvidiaTotalVramAtom) const setAvailableVram = useSetAtom(availableVramAtom) + const setUsedRam = useSetAtom(usedRamAtom) + const setTotalRam = useSetAtom(totalRamAtom) const setRamUtilitized = useSetAtom(ramUtilitizedAtom) const getSystemResources = useCallback(async () => { if ( - !extensionManager.get( - ExtensionTypeEnum.SystemMonitoring + !extensionManager.get( + ExtensionTypeEnum.Hardware ) ) { return } - const monitoring = extensionManager.get( - ExtensionTypeEnum.SystemMonitoring - ) - const resourceInfor = await monitoring?.getResourcesInfo() - const currentLoadInfor = await monitoring?.getCurrentLoad() - if (resourceInfor?.mem?.usedMemory) setUsedRam(resourceInfor.mem.usedMemory) - if (resourceInfor?.mem?.totalMemory) - setTotalRam(resourceInfor.mem.totalMemory) + const hardwareExtension = extensionManager.get( + ExtensionTypeEnum.Hardware + ) + + const hardwareInfo = await hardwareExtension?.getHardware() + + const usedMemory = + Number(hardwareInfo?.ram.total) - Number(hardwareInfo?.ram.available) + + if (hardwareInfo?.ram?.total && hardwareInfo?.ram?.available) + setUsedRam(Number(usedMemory)) + + if (hardwareInfo?.ram?.total) setTotalRam(hardwareInfo.ram.total) const ramUtilitized = - ((resourceInfor?.mem?.usedMemory ?? 0) / - (resourceInfor?.mem?.totalMemory ?? 1)) * - 100 + ((Number(usedMemory) ?? 0) / (hardwareInfo?.ram.total ?? 1)) * 100 + setRamUtilitized(Math.round(ramUtilitized)) - setCpuUsage(Math.round(currentLoadInfor?.cpu?.usage ?? 0)) + setCpuUsage(Math.round(hardwareInfo?.cpu.usage ?? 0)) - const gpus = currentLoadInfor?.gpu ?? [] - setGpus(gpus) + const gpus = hardwareInfo?.gpus ?? [] + setGpus(gpus as any) let totalNvidiaVram = 0 if (gpus.length > 0) { totalNvidiaVram = gpus.reduce( - (total: number, gpu: { memoryTotal: string }) => - total + Number(gpu.memoryTotal), + (total: number, gpu: { total_vram: number }) => + total + Number(gpu.total_vram), 0 ) } + setTotalNvidiaVram(totalNvidiaVram) + setAvailableVram( - gpus.reduce( - (total: number, gpu: { memoryFree: string }) => - total + Number(gpu.memoryFree), - 0 - ) + gpus.reduce((total, gpu) => { + return total + Number(gpu.free_vram || 0) + }, 0) ) }, [ setUsedRam, diff --git a/web/hooks/useGpuSetting.test.ts b/web/hooks/useGpuSetting.test.ts deleted file mode 100644 index f52f07af8..000000000 --- a/web/hooks/useGpuSetting.test.ts +++ /dev/null @@ -1,87 +0,0 @@ -// useGpuSetting.test.ts - -import { renderHook, act } from '@testing-library/react' -import { ExtensionTypeEnum, MonitoringExtension } from '@janhq/core' - -// Mock dependencies -jest.mock('@/extension') - -import useGpuSetting from './useGpuSetting' -import { extensionManager } from '@/extension' - -describe('useGpuSetting', () => { - beforeEach(() => { - jest.clearAllMocks() - }) - - it('should return GPU settings when available', async () => { - const mockGpuSettings = { - gpuCount: 2, - gpuNames: ['NVIDIA GeForce RTX 3080', 'NVIDIA GeForce RTX 3070'], - totalMemory: 20000, - freeMemory: 15000, - } - - const mockMonitoringExtension: Partial = { - getGpuSetting: jest.fn().mockResolvedValue(mockGpuSettings), - } - - jest - .spyOn(extensionManager, 'get') - .mockReturnValue(mockMonitoringExtension as MonitoringExtension) - - const { result } = renderHook(() => useGpuSetting()) - - let gpuSettings - await act(async () => { - gpuSettings = await result.current.getGpuSettings() - }) - - expect(gpuSettings).toEqual(mockGpuSettings) - expect(extensionManager.get).toHaveBeenCalledWith( - ExtensionTypeEnum.SystemMonitoring - ) - expect(mockMonitoringExtension.getGpuSetting).toHaveBeenCalled() - }) - - it('should return undefined when no GPU settings are found', async () => { - const mockMonitoringExtension: Partial = { - getGpuSetting: jest.fn().mockResolvedValue(undefined), - } - - jest - .spyOn(extensionManager, 'get') - .mockReturnValue(mockMonitoringExtension as MonitoringExtension) - - const { result } = renderHook(() => useGpuSetting()) - - let gpuSettings - await act(async () => { - gpuSettings = await result.current.getGpuSettings() - }) - - expect(gpuSettings).toBeUndefined() - expect(extensionManager.get).toHaveBeenCalledWith( - ExtensionTypeEnum.SystemMonitoring - ) - expect(mockMonitoringExtension.getGpuSetting).toHaveBeenCalled() - }) - - it('should handle missing MonitoringExtension', async () => { - jest.spyOn(extensionManager, 'get').mockReturnValue(undefined) - jest.spyOn(console, 'debug').mockImplementation(() => {}) - - const { result } = renderHook(() => useGpuSetting()) - - let gpuSettings - await act(async () => { - gpuSettings = await result.current.getGpuSettings() - }) - - expect(gpuSettings).toBeUndefined() - expect(extensionManager.get).toHaveBeenCalledWith( - ExtensionTypeEnum.SystemMonitoring - ) - expect(console.debug).toHaveBeenCalledWith('No GPU setting found') - }) -}) diff --git a/web/hooks/useGpuSetting.ts b/web/hooks/useGpuSetting.ts deleted file mode 100644 index 36f51ed57..000000000 --- a/web/hooks/useGpuSetting.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { useCallback } from 'react' - -import { ExtensionTypeEnum, MonitoringExtension } from '@janhq/core' - -import { extensionManager } from '@/extension' - -export default function useGpuSetting() { - const getGpuSettings = useCallback(async () => { - const gpuSetting = await extensionManager - ?.get(ExtensionTypeEnum.SystemMonitoring) - ?.getGpuSetting() - - if (!gpuSetting) { - console.debug('No GPU setting found') - return undefined - } - return gpuSetting - }, []) - - return { getGpuSettings } -} diff --git a/web/hooks/useHardwareManagement.ts b/web/hooks/useHardwareManagement.ts new file mode 100644 index 000000000..d39b3c1fc --- /dev/null +++ b/web/hooks/useHardwareManagement.ts @@ -0,0 +1,99 @@ +import { useMemo } from 'react' + +import { ExtensionTypeEnum, HardwareManagementExtension } from '@janhq/core' + +import { useSetAtom } from 'jotai' +import useSWR from 'swr' + +import { extensionManager } from '@/extension/ExtensionManager' +import { + cpuUsageAtom, + ramUtilitizedAtom, + totalRamAtom, + usedRamAtom, +} from '@/helpers/atoms/SystemBar.atom' + +// fetcher function +async function fetchExtensionData( + extension: HardwareManagementExtension | null, + method: (extension: HardwareManagementExtension) => Promise +): Promise { + if (!extension) { + throw new Error('Extension not found') + } + return method(extension) +} + +const getExtension = () => + extensionManager.get( + ExtensionTypeEnum.Hardware + ) ?? null + +/** + * @returns A Promise that resolves to an object of list engines. + */ +export function useGetHardwareInfo() { + const setCpuUsage = useSetAtom(cpuUsageAtom) + const setUsedRam = useSetAtom(usedRamAtom) + const setTotalRam = useSetAtom(totalRamAtom) + const setRamUtilitized = useSetAtom(ramUtilitizedAtom) + + const extension = useMemo( + () => + extensionManager.get( + ExtensionTypeEnum.Hardware + ) ?? null, + [] + ) + + const { + data: hardware, + error, + mutate, + } = useSWR( + extension ? 'hardware' : null, + () => fetchExtensionData(extension, (ext) => ext.getHardware()), + { + revalidateOnFocus: false, + revalidateOnReconnect: false, + refreshInterval: 2000, + } + ) + + const usedMemory = + Number(hardware?.ram.total) - Number(hardware?.ram.available) + + if (hardware?.ram?.total && hardware?.ram?.available) + setUsedRam(Number(usedMemory)) + + if (hardware?.ram?.total) setTotalRam(hardware.ram.total) + + const ramUtilitized = + ((Number(usedMemory) ?? 0) / (hardware?.ram.total ?? 1)) * 100 + + setRamUtilitized(Math.round(ramUtilitized)) + + setCpuUsage(Math.round(hardware?.cpu.usage ?? 0)) + + return { hardware, error, mutate } +} + +/** + * set gpus activate + * @returns A Promise that resolves set gpus activate. + */ +export const setActiveGpus = async (data: { gpus: number[] }) => { + const extension = getExtension() + + if (!extension) { + throw new Error('Extension is not available') + } + + try { + const response = await extension.setAvtiveGpu(data) + return response + } catch (error) { + console.error('Failed to install engine variant:', error) + throw error + } +} diff --git a/web/hooks/useSendChatMessage.ts b/web/hooks/useSendChatMessage.ts index d3c8ff142..65124fcab 100644 --- a/web/hooks/useSendChatMessage.ts +++ b/web/hooks/useSendChatMessage.ts @@ -196,7 +196,10 @@ export default function useSendChatMessage() { } updateThread(updatedThread) - if (!isResend) { + if ( + !isResend && + (newMessage.content.length || newMessage.attachments?.length) + ) { // Add message const createdMessage = await extensionManager .get(ExtensionTypeEnum.Conversational) diff --git a/web/hooks/useSettings.ts b/web/hooks/useSettings.ts index 0f02d41af..a9635aa93 100644 --- a/web/hooks/useSettings.ts +++ b/web/hooks/useSettings.ts @@ -1,20 +1,10 @@ import { useCallback, useEffect, useState } from 'react' -import { fs, joinPath } from '@janhq/core' - -type NvidiaDriver = { - exist: boolean - version: string -} +import { fs, GpuSettingInfo, joinPath } from '@janhq/core' export type AppSettings = { - run_mode: 'cpu' | 'gpu' | undefined - notify: boolean - gpus_in_use: string[] vulkan: boolean - gpus: string[] - nvidia_driver: NvidiaDriver - cuda: NvidiaDriver + gpus: GpuSettingInfo[] } export const useSettings = () => { @@ -38,29 +28,16 @@ export const useSettings = () => { return {} }, []) - const saveSettings = async ({ - runMode, - notify, - gpusInUse, - vulkan, - }: { - runMode?: string | undefined - notify?: boolean | undefined - gpusInUse?: string[] | undefined - vulkan?: boolean | undefined - }) => { + const saveSettings = async ({ vulkan }: { vulkan?: boolean | undefined }) => { const settingsFile = await joinPath(['file://settings', 'settings.json']) const settings = await readSettings() - if (runMode != null) settings.run_mode = runMode - if (notify != null) settings.notify = notify - if (gpusInUse != null) settings.gpus_in_use = gpusInUse.filter((e) => !!e) if (vulkan != null) { settings.vulkan = vulkan // GPU enabled, set run_mode to 'gpu' if (settings.vulkan === true) { - settings.run_mode = 'gpu' - } else { - settings.run_mode = 'cpu' + settings?.gpus?.some((gpu: { activated: boolean }) => + gpu.activated === true ? 'gpu' : 'cpu' + ) } } await fs.writeFileSync(settingsFile, JSON.stringify(settings)) diff --git a/web/package.json b/web/package.json index c562b6aa6..41386c587 100644 --- a/web/package.json +++ b/web/package.json @@ -14,6 +14,7 @@ "test": "jest" }, "dependencies": { + "@hello-pangea/dnd": "17.0.0", "@hookform/resolvers": "^3.9.1", "@janhq/core": "link:../core", "@janhq/joi": "link:../joi", @@ -29,7 +30,7 @@ "jotai": "^2.6.0", "katex": "^0.16.10", "lodash": "^4.17.21", - "lucide-react": "^0.291.0", + "lucide-react": "^0.311.0", "marked": "^9.1.2", "next": "14.2.3", "next-themes": "^0.2.1", @@ -56,7 +57,7 @@ "slate-react": "0.110.3", "swr": "^2.2.5", "tailwind-merge": "^2.0.0", - "tailwindcss": "3.3.5", + "tailwindcss": "3.4.17", "ulidx": "^2.3.0", "use-debounce": "^10.0.0", "uuid": "^9.0.1", diff --git a/web/public/images/ModelProvider/deepseek.svg b/web/public/images/ModelProvider/deepseek.svg new file mode 100644 index 000000000..6f4b775d3 --- /dev/null +++ b/web/public/images/ModelProvider/deepseek.svg @@ -0,0 +1,25 @@ + + + + +Created with Pixso. + + diff --git a/web/public/images/ModelProvider/google-gemini.svg b/web/public/images/ModelProvider/google-gemini.svg new file mode 100644 index 000000000..787c83710 --- /dev/null +++ b/web/public/images/ModelProvider/google-gemini.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/public/umami_script.js b/web/public/umami_script.js deleted file mode 100644 index b9db0b024..000000000 --- a/web/public/umami_script.js +++ /dev/null @@ -1,210 +0,0 @@ -!(function () { - 'use strict' - !(function (t) { - var e = t.screen, - n = e.width, - r = e.height, - a = t.navigator.language, - i = t.location, - o = t.localStorage, - u = t.document, - c = t.history, - f = 'jan.ai', - s = 'mainpage', - l = i.search, - d = u.currentScript - if (d) { - var m = 'data-', - h = d.getAttribute.bind(d), - v = h(m + 'website-id'), - p = h(m + 'host-url'), - g = 'false' !== h(m + 'auto-track'), - y = h(m + 'do-not-track'), - b = h(m + 'domains') || '', - S = b.split(',').map(function (t) { - return t.trim() - }), - k = - (p ? p.replace(/\/$/, '') : d.src.split('/').slice(0, -1).join('/')) + - '/api/send', - w = n + 'x' + r, - N = /data-umami-event-([\w-_]+)/, - T = m + 'umami-event', - j = 300, - A = function (t, e, n) { - var r = t[e] - return function () { - for (var e = [], a = arguments.length; a--; ) e[a] = arguments[a] - return n.apply(null, e), r.apply(t, e) - } - }, - x = function () { - return { - website: v, - hostname: f, - screen: w, - language: a, - title: M, - url: I, - referrer: J, - } - }, - E = function () { - return ( - (o && o.getItem('umami.disabled')) || - (y && - (function () { - var e = t.doNotTrack, - n = t.navigator, - r = t.external, - a = 'msTrackingProtectionEnabled', - i = - e || - n.doNotTrack || - n.msDoNotTrack || - (r && a in r && r[a]()) - return '1' == i || 'yes' === i - })()) || - (b && !S.includes(f)) - ) - }, - O = function (t, e, n) { - n && - ((J = I), - (I = (function (t) { - try { - return new URL(t).pathname - } catch (e) { - return t - } - })(n.toString())) !== J && setTimeout(D, j)) - }, - L = function (t, e) { - if ((void 0 === e && (e = 'event'), !E())) { - var n = { - // eslint-disable-next-line @typescript-eslint/naming-convention - 'Content-Type': 'application/json', - } - return ( - void 0 !== K && (n['x-umami-cache'] = K), - fetch(k, { - method: 'POST', - body: JSON.stringify({ - type: e, - payload: t, - }), - headers: n, - }) - .then(function (t) { - return t.text() - }) - .then(function (t) { - return (K = t) - }) - .catch(function () {}) - ) - } - }, - D = function (t, e) { - return L( - 'string' == typeof t - ? Object.assign({}, x(), { - name: t, - data: 'object' == typeof e ? e : void 0, - }) - : 'object' == typeof t - ? t - : 'function' == typeof t - ? t(x()) - : x() - ) - } - t.umami || - (t.umami = { - track: D, - identify: function (t) { - return L( - Object.assign({}, x(), { - data: t, - }), - 'identify' - ) - }, - }) - var K, - P, - _, - q, - C, - I = '' + s + l, - J = u.referrer, - M = u.title - if (g && !E()) { - ;(c.pushState = A(c, 'pushState', O)), - (c.replaceState = A(c, 'replaceState', O)), - (C = function (t) { - var e = t.getAttribute.bind(t), - n = e(T) - if (n) { - var r = {} - return ( - t.getAttributeNames().forEach(function (t) { - var n = t.match(N) - n && (r[n[1]] = e(t)) - }), - D(n, r) - ) - } - return Promise.resolve() - }), - u.addEventListener( - 'click', - function (t) { - var e = t.target, - n = - 'A' === e.tagName - ? e - : (function (t, e) { - for (var n = t, r = 0; r < e; r++) { - if ('A' === n.tagName) return n - if (!(n = n.parentElement)) return null - } - return null - })(e, 10) - if (n) { - var r = n.href, - a = - '_blank' === n.target || - t.ctrlKey || - t.shiftKey || - t.metaKey || - (t.button && 1 === t.button) - if (n.getAttribute(T) && r) - return ( - a || t.preventDefault(), - C(n).then(function () { - a || (i.href = r) - }) - ) - } else C(e) - }, - !0 - ), - (_ = new MutationObserver(function (t) { - var e = t[0] - M = e && e.target ? e.target.text : void 0 - })), - (q = u.querySelector('head > title')) && - _.observe(q, { - subtree: !0, - characterData: !0, - childList: !0, - }) - var R = function () { - 'complete' !== u.readyState || P || (D(), (P = !0)) - } - u.addEventListener('readystatechange', R, !0), R() - } - } - })(window) -})() diff --git a/web/screens/Hub/ModelList/ModelHeader/index.tsx b/web/screens/Hub/ModelList/ModelHeader/index.tsx index 9a939aa44..254ce0682 100644 --- a/web/screens/Hub/ModelList/ModelHeader/index.tsx +++ b/web/screens/Hub/ModelList/ModelHeader/index.tsx @@ -57,7 +57,7 @@ const ModelItemHeader = ({ model, onClick, open }: Props) => { // Default nvidia returns vram in MB, need to convert to bytes to match the unit of totalRamW let ram = nvidiaTotalVram * 1024 * 1024 - if (ram === 0 || settings?.run_mode === 'cpu') { + if (ram === 0 || settings?.gpus?.some((gpu) => gpu.activated !== true)) { ram = totalRam } const serverEnabled = useAtomValue(serverEnabledAtom) diff --git a/web/screens/LocalServer/LocalServerRightPanel/index.tsx b/web/screens/LocalServer/LocalServerRightPanel/index.tsx index 900a8128e..02e51868d 100644 --- a/web/screens/LocalServer/LocalServerRightPanel/index.tsx +++ b/web/screens/LocalServer/LocalServerRightPanel/index.tsx @@ -124,16 +124,20 @@ const LocalServerRightPanel = () => { clipboard.copy(selectedModel?.id) }} suffixIcon={ - clipboard.copied ? ( - + selectedModel ? ( + clipboard.copied ? ( + + ) : ( + + ) ) : ( - + <> ) } /> diff --git a/web/screens/Settings/Advanced/FactoryReset/ModalConfirmReset.tsx b/web/screens/Settings/Advanced/FactoryReset/ModalConfirmReset.tsx index c59163735..207a11ee6 100644 --- a/web/screens/Settings/Advanced/FactoryReset/ModalConfirmReset.tsx +++ b/web/screens/Settings/Advanced/FactoryReset/ModalConfirmReset.tsx @@ -30,7 +30,7 @@ const ModalConfirmReset = () => { content={

    - Restore appplication to its initial state, erasing all models and + Restore application to its initial state, erasing all models and chat history. This action is irreversible and recommended only if the application is corrupted.

    diff --git a/web/screens/Settings/Advanced/FactoryReset/index.tsx b/web/screens/Settings/Advanced/FactoryReset/index.tsx index f688e0a61..72642fafe 100644 --- a/web/screens/Settings/Advanced/FactoryReset/index.tsx +++ b/web/screens/Settings/Advanced/FactoryReset/index.tsx @@ -17,7 +17,7 @@ const FactoryReset = () => {

    - Restore appplication to its initial state, erasing all models and chat + Restore application to its initial state, erasing all models and chat history. This action is irreversible and recommended only if the application is corrupted.

    diff --git a/web/screens/Settings/Advanced/ProxySettings/index.test.tsx b/web/screens/Settings/Advanced/ProxySettings/index.test.tsx new file mode 100644 index 000000000..1cd4d645f --- /dev/null +++ b/web/screens/Settings/Advanced/ProxySettings/index.test.tsx @@ -0,0 +1,147 @@ +/** + * @jest-environment jsdom + */ + +import React from 'react' +import { render, screen, fireEvent, waitFor } from '@testing-library/react' +import '@testing-library/jest-dom' +import ProxySettings from '.' + +// Mock ResizeObserver +class ResizeObserverMock { + observe() {} + unobserve() {} + disconnect() {} +} + +global.ResizeObserver = ResizeObserverMock as any + +// Mock global window.core +global.window.core = { + api: { + getAppConfigurations: () => jest.fn(), + updateAppConfiguration: () => jest.fn(), + relaunch: () => jest.fn(), + }, +} + +// Mock dependencies +jest.mock('@/hooks/useConfigurations', () => ({ + useConfigurations: () => ({ + configurePullOptions: jest.fn(), + }), +})) + +jest.mock('jotai', () => { + const originalModule = jest.requireActual('jotai') + return { + ...originalModule, + useAtom: jest.fn().mockImplementation((atom) => { + switch (atom) { + case 'proxyEnabledAtom': + return [true, jest.fn()] + case 'proxyAtom': + return ['', jest.fn()] + case 'proxyUsernameAtom': + return ['', jest.fn()] + case 'proxyPasswordAtom': + return ['', jest.fn()] + case 'ignoreSslAtom': + return [false, jest.fn()] + case 'verifyProxySslAtom': + return [true, jest.fn()] + case 'verifyProxyHostSslAtom': + return [true, jest.fn()] + case 'verifyPeerSslAtom': + return [true, jest.fn()] + case 'verifyHostSslAtom': + return [true, jest.fn()] + case 'noProxyAtom': + return ['localhost', jest.fn()] + default: + return [null, jest.fn()] + } + }), + } +}) + +describe('ProxySettings', () => { + const mockOnBack = jest.fn() + + beforeEach(() => { + jest.clearAllMocks() + }) + + it('renders the component', async () => { + render() + + await waitFor(() => { + expect(screen.getByText('HTTPS Proxy')).toBeInTheDocument() + expect(screen.getByText('Proxy URL')).toBeInTheDocument() + expect(screen.getByText('Authentication')).toBeInTheDocument() + expect(screen.getByText('No Proxy')).toBeInTheDocument() + expect(screen.getByText('SSL Verification')).toBeInTheDocument() + }) + }) + + it('handles back navigation', async () => { + render() + + const backButton = screen.getByText('Advanced Settings') + fireEvent.click(backButton) + + expect(mockOnBack).toHaveBeenCalled() + }) + + it('toggles password visibility', () => { + render() + + const passwordVisibilityToggle = screen.getByTestId('password-visibility-toggle') + const passwordInput = screen.getByTestId('proxy-password') + + expect(passwordInput).toHaveAttribute('type', 'password') + + fireEvent.click(passwordVisibilityToggle) + expect(passwordInput).toHaveAttribute('type', 'text') + + fireEvent.click(passwordVisibilityToggle) + expect(passwordInput).toHaveAttribute('type', 'password') + }) + + it('allows clearing input fields', async () => { + render() + + // Test clearing proxy URL + const proxyInput = screen.getByTestId('proxy-input') + fireEvent.change(proxyInput, { target: { value: 'http://test.proxy' } }) + + const clearProxyButton = screen.getByTestId('clear-proxy-button') + fireEvent.click(clearProxyButton) + expect(proxyInput).toHaveValue('') + + // Test clearing username + const usernameInput = screen.getByTestId('proxy-username') + fireEvent.change(usernameInput, { target: { value: 'testuser' } }) + + // Test clearing password + const passwordInput = screen.getByTestId('proxy-password') + fireEvent.change(passwordInput, { target: { value: 'testpassword' } }) + }) + + it('renders SSL verification switches', async () => { + render() + + const sslSwitches = [ + 'Ignore SSL certificates', + 'Verify Proxy SSL', + 'Verify Proxy Host SSL', + 'Verify Peer SSL', + 'Verify Host SSL' + ] + + sslSwitches.forEach(switchText => { + const switchElement = screen.getByText(switchText) + expect(switchElement).toBeInTheDocument() + }) + }) +}) \ No newline at end of file diff --git a/web/screens/Settings/Advanced/ProxySettings/index.tsx b/web/screens/Settings/Advanced/ProxySettings/index.tsx new file mode 100644 index 000000000..446738ea2 --- /dev/null +++ b/web/screens/Settings/Advanced/ProxySettings/index.tsx @@ -0,0 +1,392 @@ +import { useCallback, useState } from 'react' + +import { Input, ScrollArea, Switch } from '@janhq/joi' +import { useAtom } from 'jotai' +import { EyeIcon, EyeOffIcon, XIcon, ArrowLeftIcon } from 'lucide-react' +import { useDebouncedCallback } from 'use-debounce' + +import { useConfigurations } from '@/hooks/useConfigurations' + +import { + ignoreSslAtom, + proxyAtom, + verifyProxySslAtom, + verifyProxyHostSslAtom, + verifyPeerSslAtom, + verifyHostSslAtom, + noProxyAtom, + proxyUsernameAtom, + proxyPasswordAtom, +} from '@/helpers/atoms/AppConfig.atom' + +const ProxySettings = ({ onBack }: { onBack: () => void }) => { + const [proxy, setProxy] = useAtom(proxyAtom) + const [noProxy, setNoProxy] = useAtom(noProxyAtom) + const [partialProxy, setPartialProxy] = useState(proxy) + const [proxyUsername, setProxyUsername] = useAtom(proxyUsernameAtom) + const [proxyPassword, setProxyPassword] = useAtom(proxyPasswordAtom) + const [proxyPartialPassword, setProxyPartialPassword] = + useState(proxyPassword) + const [proxyPartialUsername, setProxyPartialUsername] = + useState(proxyUsername) + const { configurePullOptions } = useConfigurations() + const [ignoreSSL, setIgnoreSSL] = useAtom(ignoreSslAtom) + const [verifyProxySSL, setVerifyProxySSL] = useAtom(verifyProxySslAtom) + const [verifyProxyHostSSL, setVerifyProxyHostSSL] = useAtom( + verifyProxyHostSslAtom + ) + const [verifyPeerSSL, setVerifyPeerSSL] = useAtom(verifyPeerSslAtom) + const [verifyHostSSL, setVerifyHostSSL] = useAtom(verifyHostSslAtom) + const [showPassword, setShowPassword] = useState(false) + + const updatePullOptions = useDebouncedCallback( + () => configurePullOptions(), + 1000 + ) + + const onProxyChange = useDebouncedCallback((value: string) => { + if (value.trim().startsWith('http')) { + setProxy(value.trim()) + updatePullOptions() + } else { + setProxy('') + updatePullOptions() + } + }, 1000) + + const onProxyUsernameChange = useDebouncedCallback((value: string) => { + setProxyUsername(value) + updatePullOptions() + }, 1000) + + const onProxyPasswordChange = useDebouncedCallback((value: string) => { + setProxyPassword(value) + updatePullOptions() + }, 1000) + + const handleProxyInputChange = useCallback( + (e: React.ChangeEvent) => { + const value = e.target.value || '' + setPartialProxy(value) + onProxyChange(value) + }, + [setPartialProxy, onProxyChange] + ) + + const handleProxyUsernameInputChange = useCallback( + (e: React.ChangeEvent) => { + const value = e.target.value || '' + setProxyPartialUsername(value) + onProxyUsernameChange(value) + }, + [setProxyPartialUsername, onProxyUsernameChange] + ) + + const handleProxyPasswordInputChange = useCallback( + (e: React.ChangeEvent) => { + const value = e.target.value || '' + setProxyPartialPassword(value) + onProxyPasswordChange(value) + }, + [setProxyPartialPassword, onProxyPasswordChange] + ) + + const onNoProxyChange = useCallback( + (e: React.ChangeEvent) => { + const listNoProxy = e.target.value || '' + const listNoProxyTrim = listNoProxy.split(',').map((item) => item.trim()) + setNoProxy(listNoProxyTrim.join(',')) + updatePullOptions() + }, + [setNoProxy, updatePullOptions] + ) + + return ( + + {/* Header */} +
    +
    + + / + + HTTPS Proxy + +
    +
    + + {/* Content */} +
    +
    +
    +

    Proxy Configuration

    +
    + +
    +
    +
    +
    + +

    + URL and port of your proxy server. +

    +
    + +
    +
    + + {partialProxy && ( + + )} +
    + } + /> +
    +
    +
    +
    +
    + +
    +
    +
    +
    + +

    + Credentials for your proxy server (if required). +

    +
    +
    + + {proxyUsername && ( + + )} +
    + } + /> + + {proxyPassword && ( + + )} + +
    + } + /> +
    +
    +
    +
    + + {/* No Proxy */} +
    +
    +
    +
    + +

    + List of hosts that should bypass the proxy. +

    +
    +
    + + {noProxy && ( + + )} +
    + } + /> +
    +
    +
    +
    + +
    +

    SSL Verification

    +
    + + {/* Ignore SSL certificates */} +
    +
    +
    +
    + Ignore SSL certificates +
    +
    +

    + Allow self-signed or unverified certificates (may be required + for certain proxies). Enable this reduces security. Only use + this if you trust your proxy server. +

    +
    + { + setIgnoreSSL(e.target.checked) + updatePullOptions() + }} + /> +
    + + {/* Verify Proxy SSL */} +
    +
    +
    +
    Verify Proxy SSL
    +
    +

    + Validate SSL certificate when connecting to the proxy server. +

    +
    + { + setVerifyProxySSL(e.target.checked) + updatePullOptions() + }} + /> +
    + + {/* Verify Proxy Host SSL */} +
    +
    +
    +
    + Verify Proxy Host SSL +
    +
    +

    + Validate SSL certificate of the proxy server host. +

    +
    + { + setVerifyProxyHostSSL(e.target.checked) + updatePullOptions() + }} + /> +
    + + {/* Verify Peer SSL */} +
    +
    +
    +
    Verify Peer SSL
    +
    +

    + Validate SSL certificate of the peer connections. +

    +
    + { + setVerifyPeerSSL(e.target.checked) + updatePullOptions() + }} + /> +
    + + {/* Verify Host SSL */} +
    +
    +
    +
    Verify Host SSL
    +
    +

    + Validate SSL certificate of destination hosts. +

    +
    + { + setVerifyHostSSL(e.target.checked) + updatePullOptions() + }} + /> +
    +
    +
    + + ) +} + +export default ProxySettings diff --git a/web/screens/Settings/Advanced/index.test.tsx b/web/screens/Settings/Advanced/index.test.tsx index 7a0f3ade5..b43b57beb 100644 --- a/web/screens/Settings/Advanced/index.test.tsx +++ b/web/screens/Settings/Advanced/index.test.tsx @@ -64,7 +64,6 @@ describe('Advanced', () => { await waitFor(() => { expect(screen.getByText('Experimental Mode')).toBeInTheDocument() expect(screen.getByText('HTTPS Proxy')).toBeInTheDocument() - expect(screen.getByText('Ignore SSL certificates')).toBeInTheDocument() expect(screen.getByText('Jan Data Folder')).toBeInTheDocument() expect(screen.getByText('Reset to Factory Settings')).toBeInTheDocument() }) @@ -102,28 +101,6 @@ describe('Advanced', () => { expect(proxyToggle).toBeChecked() }) - it('updates proxy settings', async () => { - render() - let proxyInput - await waitFor(() => { - const proxyToggle = screen.getByTestId(/proxy-switch/i) - fireEvent.click(proxyToggle) - proxyInput = screen.getByTestId(/proxy-input/i) - fireEvent.change(proxyInput, { target: { value: 'http://proxy.com' } }) - }) - expect(proxyInput).toHaveValue('http://proxy.com') - }) - - it('toggles ignore SSL certificates', async () => { - render() - let ignoreSslToggle - await waitFor(() => { - expect(screen.getByText('Ignore SSL certificates')).toBeInTheDocument() - ignoreSslToggle = screen.getByTestId(/ignore-ssl-switch/i) - fireEvent.click(ignoreSslToggle) - }) - expect(ignoreSslToggle).toBeChecked() - }) it('renders DataFolder component', async () => { render() diff --git a/web/screens/Settings/Advanced/index.tsx b/web/screens/Settings/Advanced/index.tsx index 16f1fd0b6..cb10a1778 100644 --- a/web/screens/Settings/Advanced/index.tsx +++ b/web/screens/Settings/Advanced/index.tsx @@ -1,32 +1,19 @@ 'use client' -import { useEffect, useState, useCallback, ChangeEvent } from 'react' +import { ChangeEvent } from 'react' -import { openExternalUrl, AppConfiguration } from '@janhq/core' +import { AppConfiguration } from '@janhq/core' -import { - ScrollArea, - Switch, - Input, - Tooltip, - Checkbox, - useClickOutside, - Button, -} from '@janhq/joi' +import { ScrollArea, Switch, Button } from '@janhq/joi' import { useAtom, useAtomValue, useSetAtom } from 'jotai' -import { ChevronDownIcon } from 'lucide-react' -import { AlertTriangleIcon, AlertCircleIcon } from 'lucide-react' - -import { twMerge } from 'tailwind-merge' +import { ArrowRightIcon } from 'lucide-react' import { useDebouncedCallback } from 'use-debounce' -import { snackbar, toaster } from '@/containers/Toast' +import { toaster } from '@/containers/Toast' -import { useActiveModel } from '@/hooks/useActiveModel' import { useConfigurations } from '@/hooks/useConfigurations' -import { useSettings } from '@/hooks/useSettings' import ModalDeleteAllThreads from '@/screens/Thread/ThreadLeftPanel/ModalDeleteAllThreads' @@ -35,10 +22,7 @@ import FactoryReset from './FactoryReset' import { experimentalFeatureEnabledAtom, - ignoreSslAtom, - proxyAtom, proxyEnabledAtom, - vulkanEnabledAtom, quickAskEnabledAtom, } from '@/helpers/atoms/AppConfig.atom' @@ -46,49 +30,22 @@ import { ThreadModalAction } from '@/helpers/atoms/Thread.atom' import { modalActionThreadAtom } from '@/helpers/atoms/Thread.atom' -type GPU = { - id: string - vram: number | null - name: string -} - /** * Advanced Settings Screen * @returns */ -const Advanced = () => { +const Advanced = ({ setSubdir }: { setSubdir: (subdir: string) => void }) => { const [experimentalEnabled, setExperimentalEnabled] = useAtom( experimentalFeatureEnabledAtom ) - const [vulkanEnabled, setVulkanEnabled] = useAtom(vulkanEnabledAtom) + const [proxyEnabled, setProxyEnabled] = useAtom(proxyEnabledAtom) const quickAskEnabled = useAtomValue(quickAskEnabledAtom) - const [proxy, setProxy] = useAtom(proxyAtom) - const [ignoreSSL, setIgnoreSSL] = useAtom(ignoreSslAtom) - - const [partialProxy, setPartialProxy] = useState(proxy) - const [gpuEnabled, setGpuEnabled] = useState(false) - const [gpuList, setGpuList] = useState([]) - const [gpusInUse, setGpusInUse] = useState([]) - const [dropdownOptions, setDropdownOptions] = useState( - null - ) const { configurePullOptions } = useConfigurations() - const [toggle, setToggle] = useState(null) - - const { readSettings, saveSettings } = useSettings() - const { stopModel } = useActiveModel() - const [open, setOpen] = useState(false) const setModalActionThread = useSetAtom(modalActionThreadAtom) - const selectedGpu = gpuList - .filter((x) => gpusInUse.includes(x.id)) - .map((y) => { - return y['name'] - }) - /** * There could be a case where the state update is not synced * so that retrieving state value from other hooks would not be accurate @@ -98,22 +55,6 @@ const Advanced = () => { () => configurePullOptions(), 300 ) - /** - * Handle proxy change - */ - const onProxyChange = useCallback( - (event: ChangeEvent) => { - const value = event.target.value || '' - setPartialProxy(value) - if (value.trim().startsWith('http')) { - setProxy(value.trim()) - } else { - setProxy('') - } - updatePullOptions() - }, - [setPartialProxy, setProxy, updatePullOptions] - ) /** * Update Quick Ask Enabled @@ -132,24 +73,6 @@ const Advanced = () => { if (relaunch) window.core?.api?.relaunch() } - /** - * Update Vulkan Enabled - * @param e - * @param relaunch - * @returns void - */ - const updateVulkanEnabled = async (e: boolean, relaunch: boolean = true) => { - toaster({ - title: 'Reload', - description: 'Vulkan settings updated. Reload now to apply the changes.', - }) - stopModel() - setVulkanEnabled(e) - await saveSettings({ vulkan: e, gpusInUse: [] }) - // Relaunch to apply settings - if (relaunch) window.location.reload() - } - /** * Update Experimental Enabled * @param e @@ -165,77 +88,17 @@ const Advanced = () => { if (e.target.checked) return // It affects other settings, so we need to reset them - const isRelaunch = quickAskEnabled || vulkanEnabled + const isRelaunch = quickAskEnabled if (quickAskEnabled) await updateQuickAskEnabled(false, false) - if (vulkanEnabled) await updateVulkanEnabled(false, false) if (isRelaunch) window.core?.api?.relaunch() } - /** - * useEffect to set GPU enabled if possible - */ - useEffect(() => { - const setUseGpuIfPossible = async () => { - const settings = await readSettings() - setGpuEnabled(settings.run_mode === 'gpu' && settings.gpus?.length > 0) - setGpusInUse(settings.gpus_in_use || []) - setVulkanEnabled(settings.vulkan || false) - if (settings.gpus) { - setGpuList(settings.gpus) - } - } - setUseGpuIfPossible() - }, [readSettings, setGpuList, setGpuEnabled, setGpusInUse, setVulkanEnabled]) - - /** - * Handle GPU Change - * @param gpuId - * @returns - */ - const handleGPUChange = async (gpuId: string) => { - let updatedGpusInUse = [...gpusInUse] - if (updatedGpusInUse.includes(gpuId)) { - updatedGpusInUse = updatedGpusInUse.filter((id) => id !== gpuId) - if ( - gpuEnabled && - updatedGpusInUse.length === 0 && - gpuId && - gpuId.trim() - ) { - // Vulkan support only allow 1 active device at a time - if (vulkanEnabled) { - updatedGpusInUse = [] - } - updatedGpusInUse.push(gpuId) - } - } else { - // Vulkan support only allow 1 active device at a time - if (vulkanEnabled) { - updatedGpusInUse = [] - } - if (gpuId && gpuId.trim()) updatedGpusInUse.push(gpuId) - } - setGpusInUse(updatedGpusInUse) - await saveSettings({ gpusInUse: updatedGpusInUse.filter((e) => !!e) }) - // Reload window to apply changes - // This will trigger engine servers to restart - window.location.reload() - } - - const gpuSelectionPlaceHolder = - gpuList.length > 0 ? 'Select GPU' : "You don't have any compatible GPU" - - /** - * Handle click outside - */ - useClickOutside(() => setOpen(false), null, [dropdownOptions, toggle]) - return (
    {/* Experimental */} -
    -
    +
    +
    Experimental Mode
    @@ -244,265 +107,43 @@ const Advanced = () => { performance. Enable with caution.

    - +
    + +
    - {/* CPU / GPU switching */} - {!isMac && ( -
    -
    -
    -
    -
    GPU Acceleration
    -
    -

    - Enable to enhance model performance by utilizing your GPU - devices for acceleration. Read{' '} - - {' '} - - openExternalUrl( - 'https://jan.ai/guides/troubleshooting/gpu-not-used/' - ) - } - > - troubleshooting guide - {' '} - {' '} - for further assistance. -

    -
    - -
    - {gpuList.length > 0 && !gpuEnabled && ( - - } - content="Disabling NVIDIA GPU Acceleration may result in reduced - performance. It is recommended to keep this enabled for - optimal user experience." - /> - )} - { - if (e.target.checked === true) { - saveSettings({ runMode: 'gpu' }) - setGpuEnabled(true) - snackbar({ - description: - 'Successfully turned on GPU Acceleration', - type: 'success', - }) - } else { - saveSettings({ runMode: 'cpu' }) - setGpuEnabled(false) - snackbar({ - description: - 'Successfully turned off GPU Acceleration', - type: 'success', - }) - } - // Stop any running model to apply the changes - if (e.target.checked !== gpuEnabled) { - stopModel().finally(() => { - setTimeout(() => { - window.location.reload() - }, 300) - }) - } - }} - /> - } - content="Your current device does not have a compatible GPU for - monitoring. To enable GPU monitoring, please ensure your - device has a supported Nvidia or AMD GPU with updated - drivers." - disabled={gpuList.length > 0} - /> -
    -
    - -
    - -
    - - } - onClick={() => setOpen(!open)} - /> - {gpuList.length > 0 && ( -
    -
    -

    - {vulkanEnabled ? 'Vulkan Supported GPUs' : 'Nvidia'} -

    -
    -
    - {gpuList - .filter((gpu) => - vulkanEnabled - ? gpu.name - : gpu.name?.toLowerCase().includes('nvidia') - ) - .map((gpu) => ( -
    - handleGPUChange(gpu.id)} - label={ - - {gpu.name} - {!vulkanEnabled && ( - {gpu.vram}MB VRAM - )} - - } - /> -
    - ))} -
    - {gpuEnabled && gpusInUse.length > 1 && ( -
    - -

    - If multi-GPU is enabled with different GPU models - or without NVLink, it could impact token speed. -

    -
    - )} -
    -
    -
    - )} -
    -
    -
    - )} - - {/* Vulkan for AMD GPU/ APU and Intel Arc GPU */} - {!isMac && experimentalEnabled && ( -
    -
    -
    -
    Vulkan Support
    -
    -

    - Enable Vulkan with AMD GPU/APU and Intel Arc GPU for better - model performance (reload needed). -

    -
    -
    - updateVulkanEnabled(e.target.checked)} - /> -
    -
    - )} - - {/* Proxy */} + {/* Proxy Settings Link */}
    -
    -
    -
    HTTPS Proxy
    +
    +
    +
    +
    HTTPS Proxy
    +
    +

    + Optional proxy server for internet connections. +

    -

    - Optional proxy server for internet connections. Only HTTPS proxies - supported. -

    -
    - -
    - { - setProxyEnabled(!proxyEnabled) - updatePullOptions() - }} - /> -
    - :@:'} - value={partialProxy} - onChange={onProxyChange} +
    + { + e.stopPropagation() + setProxyEnabled(!proxyEnabled) + updatePullOptions() + }} /> + setSubdir('proxy')} />
    - {/* Ignore SSL certificates */} -
    -
    -
    -
    - Ignore SSL certificates -
    -
    -

    - Allow self-signed or unverified certificates - may be required for - certain proxies. -

    -
    - { - setIgnoreSSL(e.target.checked) - updatePullOptions() - }} - /> -
    - {experimentalEnabled && (
    @@ -510,11 +151,11 @@ const Advanced = () => {
    Jan Quick Ask

    - Enable Quick Ask to be triggered via the default hotkey . + Enable Quick Ask to be triggered via the default hotkey {isMac ? '⌘' : 'Ctrl'} + J {' '} - (reload needed). + .

    {
    - {coreActiveExtensions.length > 0 && ( -
    -
    - Core Extension -
    -
    - )} {coreActiveExtensions .filter((x) => x.name.includes(searchText.toLowerCase().trim())) .sort((a, b) => a.name.localeCompare(b.name)) diff --git a/web/screens/Settings/Engines/DeleteEngineVariant.tsx b/web/screens/Settings/Engines/DeleteEngineVariant.tsx index 1033164e6..d21dac3d8 100644 --- a/web/screens/Settings/Engines/DeleteEngineVariant.tsx +++ b/web/screens/Settings/Engines/DeleteEngineVariant.tsx @@ -25,7 +25,7 @@ const DeleteEngineVariant = ({ return ( Delete {variant.name}} + title={Delete Variant} open={open} onOpenChange={() => setOpen(!open)} trigger={ @@ -39,7 +39,8 @@ const DeleteEngineVariant = ({ content={

    - Are you sure you want to delete this variant? + Are you sure you want to delete {variant.name}? This action cannot + be undone.

    {
    - onSwitchChange(engine)} - /> + {engine !== InferenceEngine.cortex_llamacpp && ( + onSwitchChange(engine)} + /> + )} - ) : ( - - )} - - )} + ) ?? 0 + )} + +
    + + ) : ( + + )} + + )} +
    -
    - ) - })} + ) + })}
    diff --git a/web/screens/Settings/Engines/ModalAddModel.tsx b/web/screens/Settings/Engines/ModalAddModel.tsx new file mode 100644 index 000000000..1fbdabb6a --- /dev/null +++ b/web/screens/Settings/Engines/ModalAddModel.tsx @@ -0,0 +1,160 @@ +import { memo, ReactNode, useState } from 'react' + +import { useForm } from 'react-hook-form' + +import Image from 'next/image' + +import { zodResolver } from '@hookform/resolvers/zod' + +import { InferenceEngine, Model } from '@janhq/core' + +import { Button, Input, Modal } from '@janhq/joi' +import { useAtomValue } from 'jotai' +import { PlusIcon, ArrowUpRightFromSquare } from 'lucide-react' + +import { z } from 'zod' + +import { + addRemoteEngineModel, + useGetEngines, + useGetRemoteModels, +} from '@/hooks/useEngineManagement' + +import { getLogoEngine, getTitleByEngine } from '@/utils/modelEngine' + +import { downloadedModelsAtom } from '@/helpers/atoms/Model.atom' + +const modelSchema = z.object({ + modelName: z.string().min(1, 'Model name is required'), +}) + +const ModelAddModel = ({ engine }: { engine: string }) => { + const [open, setOpen] = useState(false) + const { mutate: mutateListEngines } = useGetRemoteModels(engine) + const { engines } = useGetEngines() + const models = useAtomValue(downloadedModelsAtom) + const { + register, + handleSubmit, + formState: { errors }, + setError, + } = useForm({ + resolver: zodResolver(modelSchema), + defaultValues: { + modelName: '', + }, + }) + + const onSubmit = async (data: z.infer) => { + if (models.some((e: Model) => e.id === data.modelName)) { + setError('modelName', { + type: 'manual', + message: 'Model already exists', + }) + return + } + await addRemoteEngineModel(data.modelName, engine) + mutateListEngines() + + setOpen(false) + } + + // Helper to render labels with asterisks for required fields + const renderLabel = ( + prefix: ReactNode, + label: string, + isRequired: boolean, + desc?: string + ) => ( + <> + + {prefix} + {label} + +

    + {desc} + {isRequired && *} +

    + + ) + + return ( + +

    Add Model

    +
    + } + fullPage + open={open} + onOpenChange={() => setOpen(!open)} + trigger={ + + } + className="w-[500px]" + content={ +
    +
    +
    + + + {errors.modelName && ( +

    + {errors.modelName.message} +

    + )} + +
    + +
    + + +
    +
    +
    + } + /> + ) +} + +export default memo(ModelAddModel) diff --git a/web/screens/Settings/Engines/ModalAddRemoteEngine.tsx b/web/screens/Settings/Engines/ModalAddRemoteEngine.tsx index d500616ae..7b68a5747 100644 --- a/web/screens/Settings/Engines/ModalAddRemoteEngine.tsx +++ b/web/screens/Settings/Engines/ModalAddRemoteEngine.tsx @@ -90,7 +90,7 @@ const ModalAddRemoteEngine = () => {

    Install Remote Engine

    - Only OpenAI API-compatible engines are supported + Only OpenAI API-compatible engines are supported.

    } @@ -123,10 +123,14 @@ const ModalAddRemoteEngine = () => {
    {errors.chatCmpletionsUrl && ( @@ -141,7 +145,7 @@ const ModalAddRemoteEngine = () => { {renderLabel( 'Model List URL', false, - `URL for fetching available models` + `The endpoint URL to fetch available models.` )} { {renderLabel( 'API Key', false, - `Your authentication key from the provider` + `Your authentication key to activate this engine.` )} { {renderLabel( 'Request Headers Template', false, - `Template for request headers format.` + `HTTP headers template required for API authentication and version specification.` )}