diff --git a/.github/workflows/clean-cloudflare-r2.yml b/.github/workflows/clean-cloudflare-r2.yml deleted file mode 100644 index fcab30fc4..000000000 --- a/.github/workflows/clean-cloudflare-r2.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: "Clean Cloudflare R2 nightly build artifacts older than 10 days" -on: - schedule: - - cron: "0 0 * * *" # every day at 00:00 - workflow_dispatch: - -jobs: - clean-cloudflare-r2: - runs-on: ubuntu-latest - environment: production - steps: - - name: install-aws-cli-action - uses: unfor19/install-aws-cli-action@v1 - - name: Delete object older than 10 days - run: | - # Get the list of objects in the 'latest' folder - OBJECTS=$(aws s3api list-objects --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --prefix "latest/" --query 'Contents[?LastModified<`'$(date -d "$current_date -10 days" -u +"%Y-%m-%dT%H:%M:%SZ")'`].{Key: Key}' --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com | jq -c .) - - # Create a JSON file for the delete operation - echo "{\"Objects\": $OBJECTS, \"Quiet\": false}" > delete.json - - # Delete the objects - echo q | aws s3api delete-objects --bucket ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }} --delete file://delete.json --endpoint-url https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com - - # Remove the JSON file - rm delete.json - env: - AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: auto - AWS_EC2_METADATA_DISABLED: "true" diff --git a/.github/workflows/jan-electron-linter-and-test.yml b/.github/workflows/jan-electron-linter-and-test.yml index 3c0e22de4..9dd8cca8d 100644 --- a/.github/workflows/jan-electron-linter-and-test.yml +++ b/.github/workflows/jan-electron-linter-and-test.yml @@ -38,17 +38,57 @@ on: jobs: test-on-macos: + if: (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || github.event_name == 'push' || github.event_name == 'workflow_dispatch' runs-on: [self-hosted, macOS, macos-desktop] steps: - - name: "Cleanup build folder" - run: | - ls -la ./ - rm -rf ./* || true - rm -rf ./.??* || true - ls -la ./ - rm -rf ~/Library/Application\ Support/jan - name: Getting the repo uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Installing node + uses: actions/setup-node@v3 + with: + node-version: 20 + + - name: "Cleanup cache" + continue-on-error: true + run: | + make clean + + - name: Get Commit Message for PR + if : github.event_name == 'pull_request' + run: | + echo "REPORT_PORTAL_DESCRIPTION=${{github.event.after}})" >> $GITHUB_ENV + + - name: Get Commit Message for push event + if : github.event_name == 'push' + run: | + echo "REPORT_PORTAL_DESCRIPTION=${{github.sha}})" >> $GITHUB_ENV + + - name: "Config report portal" + run: | + make update-playwright-config REPORT_PORTAL_URL=${{ secrets.REPORT_PORTAL_URL }} REPORT_PORTAL_API_KEY=${{ secrets.REPORT_PORTAL_API_KEY }} REPORT_PORTAL_PROJECT_NAME=${{ secrets.REPORT_PORTAL_PROJECT_NAME }} REPORT_PORTAL_LAUNCH_NAME="Jan App macos" REPORT_PORTAL_DESCRIPTION="${{env.REPORT_PORTAL_DESCRIPTION}}" + + - name: Linter and test + run: | + npm config set registry ${{ secrets.NPM_PROXY }} --global + yarn config set registry ${{ secrets.NPM_PROXY }} --global + make test + env: + CSC_IDENTITY_AUTO_DISCOVERY: "false" + TURBO_API: "${{ secrets.TURBO_API }}" + TURBO_TEAM: "macos" + TURBO_TOKEN: "${{ secrets.TURBO_TOKEN }}" + + test-on-macos-pr-target: + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository + runs-on: [self-hosted, macOS, macos-desktop] + steps: + - name: Getting the repo + uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Installing node uses: actions/setup-node@v3 @@ -62,29 +102,24 @@ jobs: - name: Linter and test run: | + npm config set registry https://registry.npmjs.org --global + yarn config set registry https://registry.npmjs.org --global make test env: CSC_IDENTITY_AUTO_DISCOVERY: "false" test-on-windows: - if: github.event_name == 'push' + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' strategy: fail-fast: false matrix: antivirus-tools: ['mcafee', 'default-windows-security','bit-defender'] runs-on: windows-desktop-${{ matrix.antivirus-tools }} steps: - - name: Clean workspace - run: | - Remove-Item -Path "\\?\$(Get-Location)\*" -Force -Recurse - $path = "$Env:APPDATA\jan" - if (Test-Path $path) { - Remove-Item "\\?\$path" -Recurse -Force - } else { - Write-Output "Folder does not exist." - } - name: Getting the repo uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Installing node uses: actions/setup-node@v3 @@ -97,26 +132,79 @@ jobs: continue-on-error: true run: | make clean + + - name: Get Commit Message for push event + if : github.event_name == 'push' + shell: bash + run: | + echo "REPORT_PORTAL_DESCRIPTION=${{github.sha}}" >> $GITHUB_ENV + + - name: "Config report portal" + shell: bash + run: | + make update-playwright-config REPORT_PORTAL_URL=${{ secrets.REPORT_PORTAL_URL }} REPORT_PORTAL_API_KEY=${{ secrets.REPORT_PORTAL_API_KEY }} REPORT_PORTAL_PROJECT_NAME=${{ secrets.REPORT_PORTAL_PROJECT_NAME }} REPORT_PORTAL_LAUNCH_NAME="Jan App Windows ${{ matrix.antivirus-tools }}" REPORT_PORTAL_DESCRIPTION="${{env.REPORT_PORTAL_DESCRIPTION}}" - name: Linter and test shell: powershell run: | + npm config set registry ${{ secrets.NPM_PROXY }} --global + yarn config set registry ${{ secrets.NPM_PROXY }} --global make test + env: + TURBO_API: "${{ secrets.TURBO_API }}" + TURBO_TEAM: "windows" + TURBO_TOKEN: "${{ secrets.TURBO_TOKEN }}" test-on-windows-pr: - if: github.event_name == 'pull_request' + if: (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) runs-on: windows-desktop-default-windows-security steps: - - name: Clean workspace - run: | - Remove-Item -Path "\\?\$(Get-Location)\*" -Force -Recurse - $path = "$Env:APPDATA\jan" - if (Test-Path $path) { - Remove-Item "\\?\$path" -Recurse -Force - } else { - Write-Output "Folder does not exist." - } - name: Getting the repo uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Installing node + uses: actions/setup-node@v1 + with: + node-version: 20 + + # Clean cache, continue on error + - name: "Cleanup cache" + shell: powershell + continue-on-error: true + run: | + make clean + + - name: Get Commit Message for PR + if : github.event_name == 'pull_request' + shell: bash + run: | + echo "REPORT_PORTAL_DESCRIPTION=${{github.event.after}}" >> $GITHUB_ENV + + - name: "Config report portal" + shell: bash + run: | + make update-playwright-config REPORT_PORTAL_URL=${{ secrets.REPORT_PORTAL_URL }} REPORT_PORTAL_API_KEY=${{ secrets.REPORT_PORTAL_API_KEY }} REPORT_PORTAL_PROJECT_NAME=${{ secrets.REPORT_PORTAL_PROJECT_NAME }} REPORT_PORTAL_LAUNCH_NAME="Jan App Windows" REPORT_PORTAL_DESCRIPTION="${{env.REPORT_PORTAL_DESCRIPTION}}" + + - name: Linter and test + shell: powershell + run: | + npm config set registry ${{ secrets.NPM_PROXY }} --global + yarn config set registry ${{ secrets.NPM_PROXY }} --global + make test + env: + TURBO_API: "${{ secrets.TURBO_API }}" + TURBO_TEAM: "windows" + TURBO_TOKEN: "${{ secrets.TURBO_TOKEN }}" + + test-on-windows-pr-target: + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository + runs-on: windows-desktop-default-windows-security + steps: + - name: Getting the repo + uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Installing node uses: actions/setup-node@v1 @@ -133,20 +221,65 @@ jobs: - name: Linter and test shell: powershell run: | + npm config set registry https://registry.npmjs.org --global + yarn config set registry https://registry.npmjs.org --global make test + test-on-ubuntu: runs-on: [self-hosted, Linux, ubuntu-desktop] + if: (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || github.event_name == 'push' || github.event_name == 'workflow_dispatch' steps: - - name: "Cleanup build folder" - run: | - ls -la ./ - rm -rf ./* || true - rm -rf ./.??* || true - ls -la ./ - rm -rf ~/.config/jan - name: Getting the repo uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Installing node + uses: actions/setup-node@v3 + with: + node-version: 20 + + - name: "Cleanup cache" + continue-on-error: true + run: | + make clean + + - name: Get Commit Message for PR + if : github.event_name == 'pull_request' + run: | + echo "REPORT_PORTAL_DESCRIPTION=${{github.event.after}}" >> $GITHUB_ENV + + - name: Get Commit Message for push event + if : github.event_name == 'push' + run: | + echo "REPORT_PORTAL_DESCRIPTION=${{github.sha}}" >> $GITHUB_ENV + + - name: "Config report portal" + shell: bash + run: | + make update-playwright-config REPORT_PORTAL_URL=${{ secrets.REPORT_PORTAL_URL }} REPORT_PORTAL_API_KEY=${{ secrets.REPORT_PORTAL_API_KEY }} REPORT_PORTAL_PROJECT_NAME=${{ secrets.REPORT_PORTAL_PROJECT_NAME }} REPORT_PORTAL_LAUNCH_NAME="Jan App Linux" REPORT_PORTAL_DESCRIPTION="${{env.REPORT_PORTAL_DESCRIPTION}}" + + - name: Linter and test + run: | + export DISPLAY=$(w -h | awk 'NR==1 {print $2}') + echo -e "Display ID: $DISPLAY" + npm config set registry ${{ secrets.NPM_PROXY }} --global + yarn config set registry ${{ secrets.NPM_PROXY }} --global + make test + env: + TURBO_API: "${{ secrets.TURBO_API }}" + TURBO_TEAM: "linux" + TURBO_TOKEN: "${{ secrets.TURBO_TOKEN }}" + + test-on-ubuntu-pr-target: + runs-on: [self-hosted, Linux, ubuntu-desktop] + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository + steps: + - name: Getting the repo + uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Installing node uses: actions/setup-node@v3 @@ -162,4 +295,6 @@ jobs: run: | export DISPLAY=$(w -h | awk 'NR==1 {print $2}') echo -e "Display ID: $DISPLAY" - make test + npm config set registry https://registry.npmjs.org --global + yarn config set registry https://registry.npmjs.org --global + make test \ No newline at end of file diff --git a/.github/workflows/template-noti-discord-and-update-url-readme.yml b/.github/workflows/template-noti-discord-and-update-url-readme.yml index c13e79dd7..faf22bac4 100644 --- a/.github/workflows/template-noti-discord-and-update-url-readme.yml +++ b/.github/workflows/template-noti-discord-and-update-url-readme.yml @@ -47,27 +47,11 @@ jobs: with: args: | Jan App ${{ inputs.build_reason }} build artifact version {{ VERSION }}: - - Windows: https://delta.jan.ai/latest/jan-win-x64-{{ VERSION }}.exe - - macOS Intel: https://delta.jan.ai/latest/jan-mac-x64-{{ VERSION }}.dmg - - macOS Apple Silicon: https://delta.jan.ai/latest/jan-mac-arm64-{{ VERSION }}.dmg - - Linux Deb: https://delta.jan.ai/latest/jan-linux-amd64-{{ VERSION }}.deb - - Linux AppImage: https://delta.jan.ai/latest/jan-linux-x86_64-{{ VERSION }}.AppImage + - Windows: https://app.jan.ai/download/nightly/win-x64 + - macOS Intel: https://app.jan.ai/download/nightly/mac-x64 + - macOS Apple Silicon: https://app.jan.ai/download/nightly/mac-arm64 + - Linux Deb: https://app.jan.ai/download/nightly/linux-amd64-deb + - Linux AppImage: https://app.jan.ai/download/nightly/linux-amd64-appimage - Github action run: https://github.com/janhq/jan/actions/runs/{{ GITHUB_RUN_ID }} env: - DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} - - - name: Update README.md with artifact URL - run: | - sed -i "s|||" README.md - sed -i "s|||" README.md - sed -i "s|||" README.md - sed -i "s|||" README.md - sed -i "s|||" README.md - cat README.md - git config --global user.email "service@jan.ai" - git config --global user.name "Service Account" - git add README.md - git commit -m "${GITHUB_REPOSITORY}: Update README.md with nightly build artifact URL" - git -c http.extraheader="AUTHORIZATION: bearer ${{ secrets.PAT_SERVICE_ACCOUNT }}" push origin HEAD:${{ inputs.push_to_branch }} - env: - GITHUB_RUN_ID: ${{ github.run_id }} + DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} \ No newline at end of file diff --git a/.github/workflows/update-release-url.yml b/.github/workflows/update-release-url.yml deleted file mode 100644 index 99a3db0e0..000000000 --- a/.github/workflows/update-release-url.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Update Download URLs - -on: - release: - types: - - published - - workflow_dispatch: - -jobs: - update-readme: - runs-on: ubuntu-latest - environment: production - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: "0" - token: ${{ secrets.PAT_SERVICE_ACCOUNT }} - ref: dev - - - name: Get Latest Release - uses: pozetroninc/github-action-get-latest-release@v0.7.0 - id: get-latest-release - with: - repository: ${{ github.repository }} - - - name: Update Download URLs in README.md - run: | - echo "Latest Release: ${{ steps.get-latest-release.outputs.release }}" - tag=$(/bin/echo -n "${{ steps.get-latest-release.outputs.release }}") - echo "Tag: $tag" - # Remove the v prefix - release=${tag:1} - echo "Release: $release" - sed -i "s|||" README.md - sed -i "s|||" README.md - sed -i "s|||" README.md - sed -i "s|||" README.md - sed -i "s|||" README.md - - - name: Commit and Push changes - if: github.event_name == 'release' - run: | - git config --global user.email "service@jan.ai" - git config --global user.name "Service Account" - git add README.md - git commit -m "Update README.md with Stable Download URLs" - git -c http.extraheader="AUTHORIZATION: bearer ${{ secrets.PAT_SERVICE_ACCOUNT }}" push origin HEAD:dev diff --git a/.gitignore b/.gitignore index d9787d87b..1a7be6867 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ .vscode +.idea .env +.idea # Jan inference error.log @@ -35,4 +37,4 @@ extensions/*-extension/bin/vulkaninfo # Turborepo -.turbo \ No newline at end of file +.turbo diff --git a/Dockerfile b/Dockerfile index e205a3f4f..dee423170 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,7 +41,6 @@ COPY --from=builder /app/pre-install ./pre-install/ # Copy the package.json, yarn.lock, and output of web yarn space to leverage Docker cache COPY --from=builder /app/uikit ./uikit/ COPY --from=builder /app/web ./web/ -COPY --from=builder /app/models ./models/ RUN yarn workspace @janhq/uikit install && yarn workspace @janhq/uikit build RUN yarn workspace @janhq/web install diff --git a/Dockerfile.gpu b/Dockerfile.gpu index d703b8b43..7adc1e02a 100644 --- a/Dockerfile.gpu +++ b/Dockerfile.gpu @@ -65,7 +65,6 @@ COPY --from=builder /app/pre-install ./pre-install/ # Copy the package.json, yarn.lock, and output of web yarn space to leverage Docker cache COPY --from=builder /app/uikit ./uikit/ COPY --from=builder /app/web ./web/ -COPY --from=builder /app/models ./models/ RUN yarn workspace @janhq/uikit install && yarn workspace @janhq/uikit build RUN yarn workspace @janhq/web install diff --git a/Makefile b/Makefile index 77d7e9059..a05f14c51 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,11 @@ # Makefile for Jan Electron App - Build, Lint, Test, and Clean +REPORT_PORTAL_URL ?= "" +REPORT_PORTAL_API_KEY ?= "" +REPORT_PORTAL_PROJECT_NAME ?= "" +REPORT_PORTAL_LAUNCH_NAME ?= "Jan App" +REPORT_PORTAL_DESCRIPTION ?= "Jan App report" + # Default target, does nothing all: @echo "Specify a target to run" @@ -37,6 +43,64 @@ dev: check-file-counts lint: check-file-counts yarn lint +update-playwright-config: +ifeq ($(OS),Windows_NT) + echo -e "const RPconfig = {\n\ + apiKey: '$(REPORT_PORTAL_API_KEY)',\n\ + endpoint: '$(REPORT_PORTAL_URL)',\n\ + project: '$(REPORT_PORTAL_PROJECT_NAME)',\n\ + launch: '$(REPORT_PORTAL_LAUNCH_NAME)',\n\ + attributes: [\n\ + {\n\ + key: 'key',\n\ + value: 'value',\n\ + },\n\ + {\n\ + value: 'value',\n\ + },\n\ + ],\n\ + description: '$(REPORT_PORTAL_DESCRIPTION)',\n\ + }\n$$(cat electron/playwright.config.ts)" > electron/playwright.config.ts; + sed -i "s/^ reporter: .*/ reporter: [['@reportportal\/agent-js-playwright', RPconfig]],/" electron/playwright.config.ts + +else ifeq ($(shell uname -s),Linux) + echo "const RPconfig = {\n\ + apiKey: '$(REPORT_PORTAL_API_KEY)',\n\ + endpoint: '$(REPORT_PORTAL_URL)',\n\ + project: '$(REPORT_PORTAL_PROJECT_NAME)',\n\ + launch: '$(REPORT_PORTAL_LAUNCH_NAME)',\n\ + attributes: [\n\ + {\n\ + key: 'key',\n\ + value: 'value',\n\ + },\n\ + {\n\ + value: 'value',\n\ + },\n\ + ],\n\ + description: '$(REPORT_PORTAL_DESCRIPTION)',\n\ + }\n$$(cat electron/playwright.config.ts)" > electron/playwright.config.ts; + sed -i "s/^ reporter: .*/ reporter: [['@reportportal\/agent-js-playwright', RPconfig]],/" electron/playwright.config.ts +else + echo "const RPconfig = {\n\ + apiKey: '$(REPORT_PORTAL_API_KEY)',\n\ + endpoint: '$(REPORT_PORTAL_URL)',\n\ + project: '$(REPORT_PORTAL_PROJECT_NAME)',\n\ + launch: '$(REPORT_PORTAL_LAUNCH_NAME)',\n\ + attributes: [\n\ + {\n\ + key: 'key',\n\ + value: 'value',\n\ + },\n\ + {\n\ + value: 'value',\n\ + },\n\ + ],\n\ + description: '$(REPORT_PORTAL_DESCRIPTION)',\n\ + }\n$$(cat electron/playwright.config.ts)" > electron/playwright.config.ts; + sed -i '' "s|^ reporter: .*| reporter: [['@reportportal\/agent-js-playwright', RPconfig]],|" electron/playwright.config.ts +endif + # Testing test: lint yarn build:test @@ -53,19 +117,24 @@ build: check-file-counts clean: ifeq ($(OS),Windows_NT) - powershell -Command "Get-ChildItem -Path . -Include node_modules, .next, dist, build, out -Recurse -Directory | Remove-Item -Recurse -Force" - powershell -Command "Get-ChildItem -Path . -Include package-lock.json -Recurse -File | Remove-Item -Recurse -Force" - powershell -Command "Remove-Item -Recurse -Force ./pre-install/*.tgz" - powershell -Command "Remove-Item -Recurse -Force ./electron/pre-install/*.tgz" - powershell -Command "if (Test-Path \"$($env:USERPROFILE)\jan\extensions\") { Remove-Item -Path \"$($env:USERPROFILE)\jan\extensions\" -Recurse -Force }" + -powershell -Command "Get-ChildItem -Path . -Include node_modules, .next, dist, build, out, .turbo -Recurse -Directory | Remove-Item -Recurse -Force" + -powershell -Command "Get-ChildItem -Path . -Include package-lock.json -Recurse -File | Remove-Item -Recurse -Force" + -powershell -Command "Get-ChildItem -Path . -Include yarn.lock -Recurse -File | Remove-Item -Recurse -Force" + -powershell -Command "Remove-Item -Recurse -Force ./pre-install/*.tgz" + -powershell -Command "Remove-Item -Recurse -Force ./extensions/*/*.tgz" + -powershell -Command "Remove-Item -Recurse -Force ./electron/pre-install/*.tgz" + -powershell -Command "if (Test-Path \"$($env:USERPROFILE)\jan\extensions\") { Remove-Item -Path \"$($env:USERPROFILE)\jan\extensions\" -Recurse -Force }" else ifeq ($(shell uname -s),Linux) find . -name "node_modules" -type d -prune -exec rm -rf '{}' + find . -name ".next" -type d -exec rm -rf '{}' + find . -name "dist" -type d -exec rm -rf '{}' + find . -name "build" -type d -exec rm -rf '{}' + find . -name "out" -type d -exec rm -rf '{}' + + find . -name ".turbo" -type d -exec rm -rf '{}' + find . -name "packake-lock.json" -type f -exec rm -rf '{}' + + find . -name "yarn.lock" -type f -exec rm -rf '{}' + rm -rf ./pre-install/*.tgz + rm -rf ./extensions/*/*.tgz rm -rf ./electron/pre-install/*.tgz rm -rf "~/jan/extensions" rm -rf "~/.cache/jan*" @@ -75,8 +144,11 @@ else find . -name "dist" -type d -exec rm -rf '{}' + find . -name "build" -type d -exec rm -rf '{}' + find . -name "out" -type d -exec rm -rf '{}' + + find . -name ".turbo" -type d -exec rm -rf '{}' + find . -name "packake-lock.json" -type f -exec rm -rf '{}' + + find . -name "yarn.lock" -type f -exec rm -rf '{}' + rm -rf ./pre-install/*.tgz + rm -rf ./extensions/*/*.tgz rm -rf ./electron/pre-install/*.tgz rm -rf ~/jan/extensions rm -rf ~/Library/Caches/jan* diff --git a/README.md b/README.md index 3ad55c542..5c4da2985 100644 --- a/README.md +++ b/README.md @@ -43,32 +43,32 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Stable (Recommended) - - + + jan.exe - - + + Intel - - + + M1/M2 - - + + jan.deb - - + + jan.AppImage @@ -76,32 +76,32 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - - + + jan.exe - - + + Intel - - + + M1/M2 - - + + jan.deb - - + + jan.AppImage @@ -240,6 +240,7 @@ This will build the app MacOS m1/m2 for production (with code signing already do - If you intend to run Jan in GPU mode, you need to install `nvidia-driver` and `nvidia-docker2`. Follow the instruction [here](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) for installation. - Run Jan in Docker mode + > User can choose between `docker-compose.yml` with latest prebuilt docker image or `docker-compose-dev.yml` with local docker build | Docker compose Profile | Description | | ---------------------- | -------------------------------------------- | @@ -336,6 +337,15 @@ Jan builds on top of other open-source projects: - For business inquiries: email hello@jan.ai - For jobs: please email hr@jan.ai +## Trust & Safety + +Beware of scams. + +- We will never ask you for personal info +- We are a free product; there's no paid version +- We don't have a token or ICO +- We are not actively fundraising or seeking donations + ## License Jan is free and open source, under the AGPLv3 license. diff --git a/core/rollup.config.ts b/core/rollup.config.ts index 865e86d5c..e3336bfad 100644 --- a/core/rollup.config.ts +++ b/core/rollup.config.ts @@ -30,6 +30,7 @@ export default [ // which external modules to include in the bundle // https://github.com/rollup/rollup-plugin-node-resolve#usage replace({ + 'preventAssignment': true, 'node:crypto': 'crypto', 'delimiters': ['"', '"'], }), diff --git a/core/src/browser/extension.ts b/core/src/browser/extension.ts index 973d4778a..18a6e4491 100644 --- a/core/src/browser/extension.ts +++ b/core/src/browser/extension.ts @@ -1,3 +1,7 @@ +import { SettingComponentProps } from '../types' +import { getJanDataFolderPath, joinPath } from './core' +import { fs } from './fs' + export enum ExtensionTypeEnum { Assistant = 'assistant', Conversational = 'conversational', @@ -19,9 +23,9 @@ export interface Compatibility { const ALL_INSTALLATION_STATE = [ 'NotRequired', // not required. 'Installed', // require and installed. Good to go. - 'Updatable', // require and installed but need to be updated. 'NotInstalled', // require to be installed. 'Corrupted', // require but corrupted. Need to redownload. + 'NotCompatible', // require but not compatible. ] as const export type InstallationStateTuple = typeof ALL_INSTALLATION_STATE @@ -32,6 +36,43 @@ export type InstallationState = InstallationStateTuple[number] * This class should be extended by any class that represents an extension. */ export abstract class BaseExtension implements ExtensionType { + protected settingFolderName = 'settings' + protected settingFileName = 'settings.json' + + /** @type {string} Name of the extension. */ + name: string + + /** @type {string} Product Name of the extension. */ + productName?: string + + /** @type {string} The URL of the extension to load. */ + url: string + + /** @type {boolean} Whether the extension is activated or not. */ + active + + /** @type {string} Extension's description. */ + description + + /** @type {string} Extension's version. */ + version + + constructor( + url: string, + name: string, + productName?: string, + active?: boolean, + description?: string, + version?: string + ) { + this.name = name + this.productName = productName + this.url = url + this.active = active + this.description = description + this.version = version + } + /** * Returns the type of the extension. * @returns {ExtensionType} The type of the extension @@ -40,11 +81,13 @@ export abstract class BaseExtension implements ExtensionType { type(): ExtensionTypeEnum | undefined { return undefined } + /** * Called when the extension is loaded. * Any initialization logic for the extension should be put here. */ abstract onLoad(): void + /** * Called when the extension is unloaded. * Any cleanup logic for the extension should be put here. @@ -60,11 +103,40 @@ export abstract class BaseExtension implements ExtensionType { return undefined } - /** - * Determine if the extension is updatable. - */ - updatable(): boolean { - return false + async registerSettings(settings: SettingComponentProps[]): Promise { + if (!this.name) { + console.error('Extension name is not defined') + return + } + + const extensionSettingFolderPath = await joinPath([ + await getJanDataFolderPath(), + 'settings', + this.name, + ]) + settings.forEach((setting) => { + setting.extensionName = this.name + }) + try { + await fs.mkdir(extensionSettingFolderPath) + const settingFilePath = await joinPath([extensionSettingFolderPath, this.settingFileName]) + + if (await fs.existsSync(settingFilePath)) return + await fs.writeFileSync(settingFilePath, JSON.stringify(settings, null, 2)) + } catch (err) { + console.error(err) + } + } + + async getSetting(key: string, defaultValue: T) { + const keySetting = (await this.getSettings()).find((setting) => setting.key === key) + + const value = keySetting?.controllerProps.value + return (value as T) ?? defaultValue + } + + onSettingUpdate(key: string, value: T) { + return } /** @@ -81,8 +153,59 @@ export abstract class BaseExtension implements ExtensionType { * * @returns {Promise} */ - // @ts-ignore - async install(...args): Promise { + async install(): Promise { return } + + async getSettings(): Promise { + if (!this.name) return [] + + const settingPath = await joinPath([ + await getJanDataFolderPath(), + this.settingFolderName, + this.name, + this.settingFileName, + ]) + + try { + const content = await fs.readFileSync(settingPath, 'utf-8') + const settings: SettingComponentProps[] = JSON.parse(content) + return settings + } catch (err) { + console.warn(err) + return [] + } + } + + async updateSettings(componentProps: Partial[]): Promise { + if (!this.name) return + + const settings = await this.getSettings() + + const updatedSettings = settings.map((setting) => { + const updatedSetting = componentProps.find( + (componentProp) => componentProp.key === setting.key + ) + if (updatedSetting && updatedSetting.controllerProps) { + setting.controllerProps.value = updatedSetting.controllerProps.value + } + return setting + }) + + const settingPath = await joinPath([ + await getJanDataFolderPath(), + this.settingFolderName, + this.name, + this.settingFileName, + ]) + + await fs.writeFileSync(settingPath, JSON.stringify(updatedSettings, null, 2)) + + updatedSettings.forEach((setting) => { + this.onSettingUpdate( + setting.key, + setting.controllerProps.value + ) + }) + } } diff --git a/core/src/browser/extensions/engines/AIEngine.ts b/core/src/browser/extensions/engines/AIEngine.ts index c4f816829..7cd9f513e 100644 --- a/core/src/browser/extensions/engines/AIEngine.ts +++ b/core/src/browser/extensions/engines/AIEngine.ts @@ -10,6 +10,8 @@ import { EngineManager } from './EngineManager' * Applicable to all AI Engines */ export abstract class AIEngine extends BaseExtension { + private static modelsFolder = 'models' + // The inference engine abstract provider: string @@ -21,15 +23,6 @@ export abstract class AIEngine extends BaseExtension { events.on(ModelEvent.OnModelInit, (model: Model) => this.loadModel(model)) events.on(ModelEvent.OnModelStop, (model: Model) => this.unloadModel(model)) - - this.prePopulateModels() - } - - /** - * Defines models - */ - models(): Promise { - return Promise.resolve([]) } /** @@ -39,6 +32,49 @@ export abstract class AIEngine extends BaseExtension { EngineManager.instance().register(this) } + async registerModels(models: Model[]): Promise { + const modelFolderPath = await joinPath([await getJanDataFolderPath(), AIEngine.modelsFolder]) + + let shouldNotifyModelUpdate = false + for (const model of models) { + const modelPath = await joinPath([modelFolderPath, model.id]) + const isExist = await fs.existsSync(modelPath) + + if (isExist) { + await this.migrateModelIfNeeded(model, modelPath) + continue + } + + await fs.mkdir(modelPath) + await fs.writeFileSync( + await joinPath([modelPath, 'model.json']), + JSON.stringify(model, null, 2) + ) + shouldNotifyModelUpdate = true + } + + if (shouldNotifyModelUpdate) { + events.emit(ModelEvent.OnModelsUpdate, {}) + } + } + + async migrateModelIfNeeded(model: Model, modelPath: string): Promise { + try { + const modelJson = await fs.readFileSync(await joinPath([modelPath, 'model.json']), 'utf-8') + const currentModel: Model = JSON.parse(modelJson) + if (currentModel.version !== model.version) { + await fs.writeFileSync( + await joinPath([modelPath, 'model.json']), + JSON.stringify(model, null, 2) + ) + + events.emit(ModelEvent.OnModelsUpdate, {}) + } + } catch (error) { + console.warn('Error while try to migrating model', error) + } + } + /** * Loads the model. */ @@ -65,40 +101,4 @@ export abstract class AIEngine extends BaseExtension { * Stop inference */ stopInference() {} - - /** - * Pre-populate models to App Data Folder - */ - prePopulateModels(): Promise { - const modelFolder = 'models' - return this.models().then((models) => { - const prePoluateOperations = models.map((model) => - getJanDataFolderPath() - .then((janDataFolder) => - // Attempt to create the model folder - joinPath([janDataFolder, modelFolder, model.id]).then((path) => - fs - .mkdir(path) - .catch() - .then(() => path) - ) - ) - .then((path) => joinPath([path, 'model.json'])) - .then((path) => { - // Do not overwite existing model.json - return fs.existsSync(path).then((exist: any) => { - if (!exist) return fs.writeFileSync(path, JSON.stringify(model, null, 2)) - }) - }) - .catch((e: Error) => { - console.error('Error', e) - }) - ) - Promise.all(prePoluateOperations).then(() => - // Emit event to update models - // So the UI can update the models list - events.emit(ModelEvent.OnModelsUpdate, {}) - ) - }) - } } diff --git a/core/src/browser/extensions/engines/OAIEngine.ts b/core/src/browser/extensions/engines/OAIEngine.ts index 52acdba20..6340333e9 100644 --- a/core/src/browser/extensions/engines/OAIEngine.ts +++ b/core/src/browser/extensions/engines/OAIEngine.ts @@ -48,7 +48,7 @@ export abstract class OAIEngine extends AIEngine { /* * Inference request */ - override inference(data: MessageRequest) { + override async inference(data: MessageRequest) { if (data.model?.engine?.toString() !== this.provider) return const timestamp = Date.now() @@ -77,12 +77,14 @@ export abstract class OAIEngine extends AIEngine { ...data.model, } + const header = await this.headers() + requestInference( this.inferenceUrl, data.messages ?? [], model, this.controller, - this.headers() + header ).subscribe({ next: (content: any) => { const messageContent: ThreadContent = { @@ -100,7 +102,9 @@ export abstract class OAIEngine extends AIEngine { events.emit(MessageEvent.OnMessageUpdate, message) }, error: async (err: any) => { - console.error(`Inference error: ${JSON.stringify(err, null, 2)}`) + console.debug('inference url: ', this.inferenceUrl) + console.debug('header: ', header) + console.error(`Inference error:`, JSON.stringify(err)) if (this.isCancelled || message.content.length) { message.status = MessageStatus.Stopped events.emit(MessageEvent.OnMessageUpdate, message) @@ -131,7 +135,7 @@ export abstract class OAIEngine extends AIEngine { /** * Headers for the inference request */ - headers(): HeadersInit { + async headers(): Promise { return {} } } diff --git a/core/src/browser/extensions/engines/RemoteOAIEngine.ts b/core/src/browser/extensions/engines/RemoteOAIEngine.ts index 2d5126c6b..b11235370 100644 --- a/core/src/browser/extensions/engines/RemoteOAIEngine.ts +++ b/core/src/browser/extensions/engines/RemoteOAIEngine.ts @@ -5,8 +5,7 @@ import { OAIEngine } from './OAIEngine' * Added the implementation of loading and unloading model (applicable to local inference providers) */ export abstract class RemoteOAIEngine extends OAIEngine { - // The inference engine - abstract apiKey: string + apiKey?: string /** * On extension load, subscribe to events. */ @@ -17,10 +16,12 @@ export abstract class RemoteOAIEngine extends OAIEngine { /** * Headers for the inference request */ - override headers(): HeadersInit { + override async headers(): Promise { return { - 'Authorization': `Bearer ${this.apiKey}`, - 'api-key': `${this.apiKey}`, + ...(this.apiKey && { + 'Authorization': `Bearer ${this.apiKey}`, + 'api-key': `${this.apiKey}`, + }), } } } diff --git a/core/src/browser/extensions/engines/helpers/sse.ts b/core/src/browser/extensions/engines/helpers/sse.ts index def017ebc..dbc58501a 100644 --- a/core/src/browser/extensions/engines/helpers/sse.ts +++ b/core/src/browser/extensions/engines/helpers/sse.ts @@ -36,9 +36,15 @@ export function requestInference( .then(async (response) => { if (!response.ok) { const data = await response.json() + let errorCode = ErrorCode.Unknown; + if (data.error) { + errorCode = data.error.code ?? data.error.type ?? ErrorCode.Unknown + } else if (response.status === 401) { + errorCode = ErrorCode.InvalidApiKey; + } const error = { message: data.error?.message ?? 'Error occurred.', - code: data.error?.code ?? ErrorCode.Unknown, + code: errorCode, } subscriber.error(error) subscriber.complete() @@ -60,14 +66,20 @@ export function requestInference( } const text = decoder.decode(value) const lines = text.trim().split('\n') + let cachedLines = '' for (const line of lines) { - if (line.startsWith('data: ') && !line.includes('data: [DONE]')) { - const data = JSON.parse(line.replace('data: ', '')) - content += data.choices[0]?.delta?.content ?? '' - if (content.startsWith('assistant: ')) { - content = content.replace('assistant: ', '') + try { + const toParse = cachedLines + line + if (!line.includes('data: [DONE]')) { + const data = JSON.parse(toParse.replace('data: ', '')) + content += data.choices[0]?.delta?.content ?? '' + if (content.startsWith('assistant: ')) { + content = content.replace('assistant: ', '') + } + if (content !== '') subscriber.next(content) } - subscriber.next(content) + } catch { + cachedLines = line } } } diff --git a/core/src/browser/extensions/monitoring.ts b/core/src/browser/extensions/monitoring.ts index c30766f6e..cb544b6b7 100644 --- a/core/src/browser/extensions/monitoring.ts +++ b/core/src/browser/extensions/monitoring.ts @@ -13,7 +13,7 @@ export abstract class MonitoringExtension extends BaseExtension implements Monit return ExtensionTypeEnum.SystemMonitoring } - abstract getGpuSetting(): Promise + abstract getGpuSetting(): Promise abstract getResourcesInfo(): Promise abstract getCurrentLoad(): Promise abstract getOsInfo(): Promise diff --git a/core/src/node/api/common/handler.ts b/core/src/node/api/common/handler.ts index fb958dbd1..5cf232d8a 100644 --- a/core/src/node/api/common/handler.ts +++ b/core/src/node/api/common/handler.ts @@ -5,19 +5,16 @@ export type Handler = (route: string, args: any) => any export class RequestHandler { handler: Handler - adataper: RequestAdapter + adapter: RequestAdapter constructor(handler: Handler, observer?: Function) { this.handler = handler - this.adataper = new RequestAdapter(observer) + this.adapter = new RequestAdapter(observer) } handle() { CoreRoutes.map((route) => { - this.handler(route, async (...args: any[]) => { - const values = await this.adataper.process(route, ...args) - return values - }) + this.handler(route, async (...args: any[]) => this.adapter.process(route, ...args)) }) } } diff --git a/core/src/node/api/processors/app.ts b/core/src/node/api/processors/app.ts index c62b5011d..c98060da4 100644 --- a/core/src/node/api/processors/app.ts +++ b/core/src/node/api/processors/app.ts @@ -1,9 +1,12 @@ import { basename, isAbsolute, join, relative } from 'path' import { Processor } from './Processor' -import { getAppConfigurations as appConfiguration, updateAppConfiguration } from '../../helper' -import { log as writeLog, logServer as writeServerLog } from '../../helper/log' -import { appResourcePath } from '../../helper/path' +import { + log as writeLog, + appResourcePath, + getAppConfigurations as appConfiguration, + updateAppConfiguration, +} from '../../helper' export class App implements Processor { observer?: Function @@ -56,13 +59,6 @@ export class App implements Processor { writeLog(args) } - /** - * Log message to log file. - */ - logServer(args: any) { - writeServerLog(args) - } - getAppConfigurations() { return appConfiguration() } @@ -83,6 +79,7 @@ export class App implements Processor { isVerboseEnabled: args?.isVerboseEnabled, schemaPath: join(await appResourcePath(), 'docs', 'openapi', 'jan.yaml'), baseDir: join(await appResourcePath(), 'docs', 'openapi'), + prefix: args?.prefix, }) } diff --git a/core/src/node/api/restful/helper/builder.ts b/core/src/node/api/restful/helper/builder.ts index e34fb606b..cd121cdb7 100644 --- a/core/src/node/api/restful/helper/builder.ts +++ b/core/src/node/api/restful/helper/builder.ts @@ -316,6 +316,7 @@ export const chatCompletions = async (request: any, reply: any) => { } const requestedModel = matchedModels[0] + const engineConfiguration = await getEngineConfiguration(requestedModel.engine) let apiKey: string | undefined = undefined @@ -323,7 +324,7 @@ export const chatCompletions = async (request: any, reply: any) => { if (engineConfiguration) { apiKey = engineConfiguration.api_key - apiUrl = engineConfiguration.full_url + apiUrl = engineConfiguration.full_url ?? DEFAULT_CHAT_COMPLETION_URL } const headers: Record = { @@ -334,7 +335,6 @@ export const chatCompletions = async (request: any, reply: any) => { headers['Authorization'] = `Bearer ${apiKey}` headers['api-key'] = apiKey } - console.debug(apiUrl) if (requestedModel.engine === 'openai' && request.body.stop) { // openai only allows max 4 stop words @@ -352,7 +352,7 @@ export const chatCompletions = async (request: any, reply: any) => { reply.code(400).send(response) } else { reply.raw.writeHead(200, { - 'Content-Type': 'text/event-stream', + 'Content-Type': request.body.stream === true ? 'text/event-stream' : 'application/json', 'Cache-Control': 'no-cache', 'Connection': 'keep-alive', 'Access-Control-Allow-Origin': '*', diff --git a/core/src/node/api/restful/helper/startStopModel.ts b/core/src/node/api/restful/helper/startStopModel.ts index 4627b4120..3af0404e3 100644 --- a/core/src/node/api/restful/helper/startStopModel.ts +++ b/core/src/node/api/restful/helper/startStopModel.ts @@ -1,7 +1,11 @@ import fs from 'fs' import { join } from 'path' -import { getJanDataFolderPath, getJanExtensionsPath, getSystemResourceInfo } from '../../../helper' -import { logServer } from '../../../helper/log' +import { + getJanDataFolderPath, + getJanExtensionsPath, + getSystemResourceInfo, + log, +} from '../../../helper' import { ChildProcessWithoutNullStreams, spawn } from 'child_process' import { Model, ModelSettingParams, PromptTemplate } from '../../../../types' import { @@ -69,7 +73,7 @@ const runModel = async (modelId: string, settingParams?: ModelSettingParams): Pr }), } - logServer(`[NITRO]::Debug: Nitro model settings: ${JSON.stringify(nitroModelSettings)}`) + log(`[SERVER]::Debug: Nitro model settings: ${JSON.stringify(nitroModelSettings)}`) // Convert settings.prompt_template to system_prompt, user_prompt, ai_prompt if (modelMetadata.settings.prompt_template) { @@ -140,7 +144,7 @@ const runNitroAndLoadModel = async (modelId: string, modelSettings: NitroModelSe } const spawnNitroProcess = async (): Promise => { - logServer(`[NITRO]::Debug: Spawning Nitro subprocess...`) + log(`[SERVER]::Debug: Spawning Nitro subprocess...`) let binaryFolder = join( getJanExtensionsPath(), @@ -155,8 +159,8 @@ const spawnNitroProcess = async (): Promise => { const args: string[] = ['1', LOCAL_HOST, NITRO_DEFAULT_PORT.toString()] // Execute the binary - logServer( - `[NITRO]::Debug: Spawn nitro at path: ${executableOptions.executablePath}, and args: ${args}` + log( + `[SERVER]::Debug: Spawn nitro at path: ${executableOptions.executablePath}, and args: ${args}` ) subprocess = spawn( executableOptions.executablePath, @@ -172,20 +176,20 @@ const spawnNitroProcess = async (): Promise => { // Handle subprocess output subprocess.stdout.on('data', (data: any) => { - logServer(`[NITRO]::Debug: ${data}`) + log(`[SERVER]::Debug: ${data}`) }) subprocess.stderr.on('data', (data: any) => { - logServer(`[NITRO]::Error: ${data}`) + log(`[SERVER]::Error: ${data}`) }) subprocess.on('close', (code: any) => { - logServer(`[NITRO]::Debug: Nitro exited with code: ${code}`) + log(`[SERVER]::Debug: Nitro exited with code: ${code}`) subprocess = undefined }) tcpPortUsed.waitUntilUsed(NITRO_DEFAULT_PORT, 300, 30000).then(() => { - logServer(`[NITRO]::Debug: Nitro is ready`) + log(`[SERVER]::Debug: Nitro is ready`) }) } @@ -227,13 +231,9 @@ const executableNitroFile = (): NitroExecutableOptions => { binaryName = 'nitro.exe' } else if (process.platform === 'darwin') { /** - * For MacOS: mac-arm64 (Silicon), mac-x64 (InteL) + * For MacOS: mac-universal both Silicon and InteL */ - if (process.arch === 'arm64') { - binaryFolder = join(binaryFolder, 'mac-arm64') - } else { - binaryFolder = join(binaryFolder, 'mac-x64') - } + binaryFolder = join(binaryFolder, 'mac-universal') } else { /** * For Linux: linux-cpu, linux-cuda-11-7, linux-cuda-12-0 @@ -271,7 +271,7 @@ const validateModelStatus = async (): Promise => { retries: 5, retryDelay: 500, }).then(async (res: Response) => { - logServer(`[NITRO]::Debug: Validate model state success with response ${JSON.stringify(res)}`) + log(`[SERVER]::Debug: Validate model state success with response ${JSON.stringify(res)}`) // If the response is OK, check model_loaded status. if (res.ok) { const body = await res.json() @@ -286,7 +286,7 @@ const validateModelStatus = async (): Promise => { } const loadLLMModel = async (settings: NitroModelSettings): Promise => { - logServer(`[NITRO]::Debug: Loading model with params ${JSON.stringify(settings)}`) + log(`[SERVER]::Debug: Loading model with params ${JSON.stringify(settings)}`) const fetchRT = require('fetch-retry') const fetchRetry = fetchRT(fetch) @@ -300,11 +300,11 @@ const loadLLMModel = async (settings: NitroModelSettings): Promise => retryDelay: 500, }) .then((res: any) => { - logServer(`[NITRO]::Debug: Load model success with response ${JSON.stringify(res)}`) + log(`[SERVER]::Debug: Load model success with response ${JSON.stringify(res)}`) return Promise.resolve(res) }) .catch((err: any) => { - logServer(`[NITRO]::Error: Load model failed with error ${err}`) + log(`[SERVER]::Error: Load model failed with error ${err}`) return Promise.reject(err) }) } @@ -327,7 +327,7 @@ export const stopModel = async (_modelId: string) => { }) }, 5000) const tcpPortUsed = require('tcp-port-used') - logServer(`[NITRO]::Debug: Request to kill Nitro`) + log(`[SERVER]::Debug: Request to kill Nitro`) fetch(NITRO_HTTP_KILL_URL, { method: 'DELETE', @@ -341,7 +341,7 @@ export const stopModel = async (_modelId: string) => { // don't need to do anything, we still kill the subprocess }) .then(() => tcpPortUsed.waitUntilFree(NITRO_DEFAULT_PORT, 300, 5000)) - .then(() => logServer(`[NITRO]::Debug: Nitro process is terminated`)) + .then(() => log(`[SERVER]::Debug: Nitro process is terminated`)) .then(() => resolve({ message: 'Model stopped', diff --git a/core/src/node/extension/extension.ts b/core/src/node/extension/extension.ts index 8780b3ffe..849f2d5f2 100644 --- a/core/src/node/extension/extension.ts +++ b/core/src/node/extension/extension.ts @@ -11,6 +11,7 @@ export default class Extension { * @property {string} origin Original specification provided to fetch the package. * @property {Object} installOptions Options provided to pacote when fetching the manifest. * @property {name} name The name of the extension as defined in the manifest. + * @property {name} productName The display name of the extension as defined in the manifest. * @property {string} url Electron URL where the package can be accessed. * @property {string} version Version of the package as defined in the manifest. * @property {string} main The entry point as defined in the main entry of the manifest. @@ -19,6 +20,7 @@ export default class Extension { origin?: string installOptions: any name?: string + productName?: string url?: string version?: string main?: string @@ -42,7 +44,7 @@ export default class Extension { const Arborist = require('@npmcli/arborist') const defaultOpts = { version: false, - fullMetadata: false, + fullMetadata: true, Arborist, } @@ -77,6 +79,7 @@ export default class Extension { return pacote.manifest(this.specifier, this.installOptions).then((mnf) => { // set the Package properties based on the it's manifest this.name = mnf.name + this.productName = mnf.productName as string | undefined this.version = mnf.version this.main = mnf.main this.description = mnf.description diff --git a/core/src/node/helper/config.ts b/core/src/node/helper/config.ts index b5ec2e029..ee9a1f856 100644 --- a/core/src/node/helper/config.ts +++ b/core/src/node/helper/config.ts @@ -1,4 +1,4 @@ -import { AppConfiguration } from '../../types' +import { AppConfiguration, SettingComponentProps } from '../../types' import { join } from 'path' import fs from 'fs' import os from 'os' @@ -125,40 +125,32 @@ const exec = async (command: string): Promise => { }) } +// a hacky way to get the api key. we should comes up with a better +// way to handle this export const getEngineConfiguration = async (engineId: string) => { - if (engineId !== 'openai' && engineId !== 'groq') { - return undefined - } - const directoryPath = join(getJanDataFolderPath(), 'engines') - const filePath = join(directoryPath, `${engineId}.json`) - const data = fs.readFileSync(filePath, 'utf-8') - return JSON.parse(data) -} + if (engineId !== 'openai' && engineId !== 'groq') return undefined -/** - * Utility function to get server log path - * - * @returns {string} The log path. - */ -export const getServerLogPath = (): string => { - const appConfigurations = getAppConfigurations() - const logFolderPath = join(appConfigurations.data_folder, 'logs') - if (!fs.existsSync(logFolderPath)) { - fs.mkdirSync(logFolderPath, { recursive: true }) - } - return join(logFolderPath, 'server.log') -} + const settingDirectoryPath = join( + getJanDataFolderPath(), + 'settings', + '@janhq', + engineId === 'openai' ? 'inference-openai-extension' : 'inference-groq-extension', + 'settings.json' + ) -/** - * Utility function to get app log path - * - * @returns {string} The log path. - */ -export const getAppLogPath = (): string => { - const appConfigurations = getAppConfigurations() - const logFolderPath = join(appConfigurations.data_folder, 'logs') - if (!fs.existsSync(logFolderPath)) { - fs.mkdirSync(logFolderPath, { recursive: true }) + const content = fs.readFileSync(settingDirectoryPath, 'utf-8') + const settings: SettingComponentProps[] = JSON.parse(content) + const apiKeyId = engineId === 'openai' ? 'openai-api-key' : 'groq-api-key' + const keySetting = settings.find((setting) => setting.key === apiKeyId) + let fullUrl = settings.find((setting) => setting.key === 'chat-completions-endpoint') + ?.controllerProps.value + + let apiKey = keySetting?.controllerProps.value + if (typeof apiKey !== 'string') apiKey = '' + if (typeof fullUrl !== 'string') fullUrl = '' + + return { + api_key: apiKey, + full_url: fullUrl, } - return join(logFolderPath, 'app.log') } diff --git a/core/src/node/helper/index.ts b/core/src/node/helper/index.ts index 6fc54fc6b..51030023f 100644 --- a/core/src/node/helper/index.ts +++ b/core/src/node/helper/index.ts @@ -1,6 +1,6 @@ export * from './config' export * from './download' -export * from './log' +export * from './logger' export * from './module' export * from './path' export * from './resource' diff --git a/core/src/node/helper/log.ts b/core/src/node/helper/log.ts deleted file mode 100644 index 8ff196943..000000000 --- a/core/src/node/helper/log.ts +++ /dev/null @@ -1,37 +0,0 @@ -import fs from 'fs' -import util from 'util' -import { getAppLogPath, getServerLogPath } from './config' - -export const log = (message: string) => { - const path = getAppLogPath() - if (!message.startsWith('[')) { - message = `[APP]::${message}` - } - - message = `${new Date().toISOString()} ${message}` - - writeLog(message, path) -} - -export const logServer = (message: string) => { - const path = getServerLogPath() - if (!message.startsWith('[')) { - message = `[SERVER]::${message}` - } - - message = `${new Date().toISOString()} ${message}` - writeLog(message, path) -} - -const writeLog = (message: string, logPath: string) => { - if (!fs.existsSync(logPath)) { - fs.writeFileSync(logPath, message) - } else { - const logFile = fs.createWriteStream(logPath, { - flags: 'a', - }) - logFile.write(util.format(message) + '\n') - logFile.close() - console.debug(message) - } -} diff --git a/core/src/node/helper/logger.ts b/core/src/node/helper/logger.ts new file mode 100644 index 000000000..a6b3c8bef --- /dev/null +++ b/core/src/node/helper/logger.ts @@ -0,0 +1,81 @@ +// Abstract Logger class that all loggers should extend. +export abstract class Logger { + // Each logger must have a unique name. + abstract name: string + + /** + * Log message to log file. + * This method should be overridden by subclasses to provide specific logging behavior. + */ + abstract log(args: any): void +} + +// LoggerManager is a singleton class that manages all registered loggers. +export class LoggerManager { + // Map of registered loggers, keyed by their names. + public loggers = new Map() + + // Array to store logs that are queued before the loggers are registered. + queuedLogs: any[] = [] + + // Flag to indicate whether flushLogs is currently running. + private isFlushing = false + + // Register a new logger. If a logger with the same name already exists, it will be replaced. + register(logger: Logger) { + this.loggers.set(logger.name, logger) + } + // Unregister a logger by its name. + unregister(name: string) { + this.loggers.delete(name) + } + + get(name: string) { + return this.loggers.get(name) + } + + // Flush queued logs to all registered loggers. + flushLogs() { + // If flushLogs is already running, do nothing. + if (this.isFlushing) { + return + } + + this.isFlushing = true + + while (this.queuedLogs.length > 0 && this.loggers.size > 0) { + const log = this.queuedLogs.shift() + this.loggers.forEach((logger) => { + logger.log(log) + }) + } + + this.isFlushing = false + } + + // Log message using all registered loggers. + log(args: any) { + this.queuedLogs.push(args) + + this.flushLogs() + } + + /** + * The instance of the logger. + * If an instance doesn't exist, it creates a new one. + * This ensures that there is only one LoggerManager instance at any time. + */ + static instance(): LoggerManager { + let instance: LoggerManager | undefined = global.core?.logger + if (!instance) { + instance = new LoggerManager() + if (!global.core) global.core = {} + global.core.logger = instance + } + return instance + } +} + +export const log = (...args: any) => { + LoggerManager.instance().log(args) +} diff --git a/core/src/node/helper/resource.ts b/core/src/node/helper/resource.ts index faaaace05..27e86c650 100644 --- a/core/src/node/helper/resource.ts +++ b/core/src/node/helper/resource.ts @@ -1,11 +1,10 @@ import { SystemResourceInfo } from '../../types' import { physicalCpuCount } from './config' -import { log } from './log' +import { log } from './logger' export const getSystemResourceInfo = async (): Promise => { const cpu = await physicalCpuCount() - const message = `[NITRO]::CPU informations - ${cpu}` - log(message) + log(`[NITRO]::CPU informations - ${cpu}`) return { numCpuPhysicalCore: cpu, diff --git a/core/src/types/api/index.ts b/core/src/types/api/index.ts index 91d6ae755..d95d0474e 100644 --- a/core/src/types/api/index.ts +++ b/core/src/types/api/index.ts @@ -7,7 +7,7 @@ export enum NativeRoute { openAppDirectory = 'openAppDirectory', openFileExplore = 'openFileExplorer', selectDirectory = 'selectDirectory', - selectModelFiles = 'selectModelFiles', + selectFiles = 'selectFiles', relaunch = 'relaunch', hideQuickAskWindow = 'hideQuickAskWindow', diff --git a/core/src/types/index.ts b/core/src/types/index.ts index 291c73524..6627ebff9 100644 --- a/core/src/types/index.ts +++ b/core/src/types/index.ts @@ -9,3 +9,4 @@ export * from './config' export * from './huggingface' export * from './miscellaneous' export * from './api' +export * from './setting' diff --git a/core/src/types/message/messageEntity.ts b/core/src/types/message/messageEntity.ts index 5444558aa..f96919a8b 100644 --- a/core/src/types/message/messageEntity.ts +++ b/core/src/types/message/messageEntity.ts @@ -85,6 +85,8 @@ export enum ErrorCode { InsufficientQuota = 'insufficient_quota', + InvalidRequestError = 'invalid_request_error', + Unknown = 'unknown', } diff --git a/core/src/types/miscellaneous/index.ts b/core/src/types/miscellaneous/index.ts index b4ef68ab6..2693ffd8b 100644 --- a/core/src/types/miscellaneous/index.ts +++ b/core/src/types/miscellaneous/index.ts @@ -2,4 +2,5 @@ export * from './systemResourceInfo' export * from './promptTemplate' export * from './appUpdate' export * from './fileDownloadRequest' -export * from './networkConfig' \ No newline at end of file +export * from './networkConfig' +export * from './selectFiles' diff --git a/core/src/types/miscellaneous/selectFiles.ts b/core/src/types/miscellaneous/selectFiles.ts new file mode 100644 index 000000000..5e4a95906 --- /dev/null +++ b/core/src/types/miscellaneous/selectFiles.ts @@ -0,0 +1,37 @@ +export type SelectFileOption = { + /** + * The title of the dialog. + */ + title?: string + /** + * Whether the dialog allows multiple selection. + */ + allowMultiple?: boolean + + buttonLabel?: string + + selectDirectory?: boolean + + props?: SelectFileProp[] + + filters?: FilterOption[] +} + +export type FilterOption = { + name: string + extensions: string[] +} + +export const SelectFilePropTuple = [ + 'openFile', + 'openDirectory', + 'multiSelections', + 'showHiddenFiles', + 'createDirectory', + 'promptToCreate', + 'noResolveAliases', + 'treatPackageAsDirectory', + 'dontAddToRecent', +] as const + +export type SelectFileProp = (typeof SelectFilePropTuple)[number] diff --git a/core/src/types/miscellaneous/systemResourceInfo.ts b/core/src/types/miscellaneous/systemResourceInfo.ts index fb059b1ba..6ceea0822 100644 --- a/core/src/types/miscellaneous/systemResourceInfo.ts +++ b/core/src/types/miscellaneous/systemResourceInfo.ts @@ -32,7 +32,7 @@ export type GpuSettingInfo = { } export type SystemInformation = { - gpuSetting: GpuSetting + gpuSetting?: GpuSetting osInfo?: OperatingSystemInfo } diff --git a/core/src/types/model/modelEntity.ts b/core/src/types/model/modelEntity.ts index a313847b6..7b2828b46 100644 --- a/core/src/types/model/modelEntity.ts +++ b/core/src/types/model/modelEntity.ts @@ -41,7 +41,7 @@ export type Model = { /** * The version of the model. */ - version: number + version: string /** * The format of the model. diff --git a/core/src/types/monitoring/monitoringInterface.ts b/core/src/types/monitoring/monitoringInterface.ts index ffdbebcc1..5ab1394a1 100644 --- a/core/src/types/monitoring/monitoringInterface.ts +++ b/core/src/types/monitoring/monitoringInterface.ts @@ -1,3 +1,5 @@ +import { GpuSetting, OperatingSystemInfo } from '../miscellaneous' + /** * Monitoring extension for system monitoring. * @extends BaseExtension @@ -14,4 +16,14 @@ export interface MonitoringInterface { * @returns {Promise} A promise that resolves with the current system load. */ getCurrentLoad(): Promise + + /** + * Returns the GPU configuration. + */ + getGpuSetting(): Promise + + /** + * Returns information about the operating system. + */ + getOsInfo(): Promise } diff --git a/core/src/types/setting/index.ts b/core/src/types/setting/index.ts new file mode 100644 index 000000000..b3407460c --- /dev/null +++ b/core/src/types/setting/index.ts @@ -0,0 +1 @@ +export * from './settingComponent' diff --git a/core/src/types/setting/settingComponent.ts b/core/src/types/setting/settingComponent.ts new file mode 100644 index 000000000..4d9526505 --- /dev/null +++ b/core/src/types/setting/settingComponent.ts @@ -0,0 +1,34 @@ +export type SettingComponentProps = { + key: string + title: string + description: string + controllerType: ControllerType + controllerProps: SliderComponentProps | CheckboxComponentProps | InputComponentProps + + extensionName?: string + requireModelReload?: boolean + configType?: ConfigType +} + +export type ConfigType = 'runtime' | 'setting' + +export type ControllerType = 'slider' | 'checkbox' | 'input' + +export type InputType = 'password' | 'text' | 'email' | 'number' | 'tel' | 'url' + +export type InputComponentProps = { + placeholder: string + value: string + type?: InputType +} + +export type SliderComponentProps = { + min: number + max: number + step: number + value: number +} + +export type CheckboxComponentProps = { + value: boolean +} diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml new file mode 100644 index 000000000..2e09d641b --- /dev/null +++ b/docker-compose-dev.yml @@ -0,0 +1,171 @@ +# Docker Compose file for setting up Minio, createbuckets, app_cpu, and app_gpu services + +version: '3.7' + +services: + # Minio service for object storage + minio: + image: minio/minio + volumes: + - minio_data:/data + ports: + - '9000:9000' + - '9001:9001' + environment: + # Set the root user and password for Minio + MINIO_ROOT_USER: minioadmin # This acts as AWS_ACCESS_KEY + MINIO_ROOT_PASSWORD: minioadmin # This acts as AWS_SECRET_ACCESS_KEY + command: server --console-address ":9001" /data + restart: always + healthcheck: + test: ['CMD', 'curl', '-f', 'http://localhost:9000/minio/health/live'] + interval: 30s + timeout: 20s + retries: 3 + networks: + vpcbr: + ipv4_address: 10.5.0.2 + + # createbuckets service to create a bucket and set its policy + createbuckets: + image: minio/mc + depends_on: + - minio + entrypoint: > + /bin/sh -c " + /usr/bin/mc alias set myminio http://minio:9000 minioadmin minioadmin; + /usr/bin/mc mb myminio/mybucket; + /usr/bin/mc policy set public myminio/mybucket; + exit 0; + " + networks: + vpcbr: + + # app_cpu service for running the CPU version of the application + app_cpu_s3fs: + image: jan:latest + volumes: + - app_data_cpu_s3fs:/app/server/build/jan + build: + context: . + dockerfile: Dockerfile + environment: + # Set the AWS access key, secret access key, bucket name, endpoint, and region for app_cpu + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + S3_BUCKET_NAME: mybucket + AWS_ENDPOINT: http://10.5.0.2:9000 + AWS_REGION: us-east-1 + API_BASE_URL: http://localhost:1337 + restart: always + profiles: + - cpu-s3fs + ports: + - '3000:3000' + - '1337:1337' + - '3928:3928' + networks: + vpcbr: + ipv4_address: 10.5.0.3 + + # app_gpu service for running the GPU version of the application + app_gpu_s3fs: + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + image: jan-gpu:latest + volumes: + - app_data_gpu_s3fs:/app/server/build/jan + build: + context: . + dockerfile: Dockerfile.gpu + restart: always + environment: + # Set the AWS access key, secret access key, bucket name, endpoint, and region for app_gpu + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + S3_BUCKET_NAME: mybucket + AWS_ENDPOINT: http://10.5.0.2:9000 + AWS_REGION: us-east-1 + API_BASE_URL: http://localhost:1337 + profiles: + - gpu-s3fs + ports: + - '3000:3000' + - '1337:1337' + - '3928:3928' + networks: + vpcbr: + ipv4_address: 10.5.0.4 + + app_cpu_fs: + image: jan:latest + volumes: + - app_data_cpu_fs:/app/server/build/jan + build: + context: . + dockerfile: Dockerfile + environment: + API_BASE_URL: http://localhost:1337 + restart: always + profiles: + - cpu-fs + ports: + - '3000:3000' + - '1337:1337' + - '3928:3928' + networks: + vpcbr: + ipv4_address: 10.5.0.5 + + # app_gpu service for running the GPU version of the application + app_gpu_fs: + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + image: jan-gpu:latest + volumes: + - app_data_gpu_fs:/app/server/build/jan + build: + context: . + dockerfile: Dockerfile.gpu + restart: always + environment: + API_BASE_URL: http://localhost:1337 + profiles: + - gpu-fs + ports: + - '3000:3000' + - '1337:1337' + - '3928:3928' + networks: + vpcbr: + ipv4_address: 10.5.0.6 + +volumes: + minio_data: + app_data_cpu_s3fs: + app_data_gpu_s3fs: + app_data_cpu_fs: + app_data_gpu_fs: + +networks: + vpcbr: + driver: bridge + ipam: + config: + - subnet: 10.5.0.0/16 + gateway: 10.5.0.1 +# Usage: +# - Run 'docker compose -f docker-compose-dev.yml --profile cpu-s3fs up -d' to start the app_cpu service +# - Run 'docker compose -f docker-compose-dev.yml --profile gpu-s3fs up -d' to start the app_gpu service +# - Run 'docker compose -f docker-compose-dev.yml --profile cpu-fs up -d' to start the app_cpu service +# - Run 'docker compose -f docker-compose-dev.yml --profile gpu-fs up -d' to start the app_gpu service diff --git a/docker-compose.yml b/docker-compose.yml index 1691a841a..1e5660c12 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,8 +9,8 @@ services: volumes: - minio_data:/data ports: - - "9000:9000" - - "9001:9001" + - '9000:9000' + - '9001:9001' environment: # Set the root user and password for Minio MINIO_ROOT_USER: minioadmin # This acts as AWS_ACCESS_KEY @@ -18,7 +18,7 @@ services: command: server --console-address ":9001" /data restart: always healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + test: ['CMD', 'curl', '-f', 'http://localhost:9000/minio/health/live'] interval: 30s timeout: 20s retries: 3 @@ -43,12 +43,9 @@ services: # app_cpu service for running the CPU version of the application app_cpu_s3fs: - image: jan:latest volumes: - app_data_cpu_s3fs:/app/server/build/jan - build: - context: . - dockerfile: Dockerfile + image: ghcr.io/janhq/jan-server:dev-cpu-latest environment: # Set the AWS access key, secret access key, bucket name, endpoint, and region for app_cpu AWS_ACCESS_KEY_ID: minioadmin @@ -61,9 +58,9 @@ services: profiles: - cpu-s3fs ports: - - "3000:3000" - - "1337:1337" - - "3928:3928" + - '3000:3000' + - '1337:1337' + - '3928:3928' networks: vpcbr: ipv4_address: 10.5.0.3 @@ -74,15 +71,12 @@ services: resources: reservations: devices: - - driver: nvidia - count: all - capabilities: [gpu] - image: jan-gpu:latest + - driver: nvidia + count: all + capabilities: [gpu] + image: ghcr.io/janhq/jan-server:dev-cuda-12.2-latest volumes: - app_data_gpu_s3fs:/app/server/build/jan - build: - context: . - dockerfile: Dockerfile.gpu restart: always environment: # Set the AWS access key, secret access key, bucket name, endpoint, and region for app_gpu @@ -95,29 +89,26 @@ services: profiles: - gpu-s3fs ports: - - "3000:3000" - - "1337:1337" - - "3928:3928" + - '3000:3000' + - '1337:1337' + - '3928:3928' networks: vpcbr: ipv4_address: 10.5.0.4 app_cpu_fs: - image: jan:latest + image: ghcr.io/janhq/jan-server:dev-cpu-latest volumes: - app_data_cpu_fs:/app/server/build/jan - build: - context: . - dockerfile: Dockerfile environment: API_BASE_URL: http://localhost:1337 restart: always profiles: - cpu-fs ports: - - "3000:3000" - - "1337:1337" - - "3928:3928" + - '3000:3000' + - '1337:1337' + - '3928:3928' networks: vpcbr: ipv4_address: 10.5.0.5 @@ -128,24 +119,21 @@ services: resources: reservations: devices: - - driver: nvidia - count: all - capabilities: [gpu] - image: jan-gpu:latest + - driver: nvidia + count: all + capabilities: [gpu] + image: ghcr.io/janhq/jan-server:dev-cuda-12.2-latest volumes: - app_data_gpu_fs:/app/server/build/jan - build: - context: . - dockerfile: Dockerfile.gpu restart: always environment: API_BASE_URL: http://localhost:1337 profiles: - gpu-fs ports: - - "3000:3000" - - "1337:1337" - - "3928:3928" + - '3000:3000' + - '1337:1337' + - '3928:3928' networks: vpcbr: ipv4_address: 10.5.0.6 @@ -161,10 +149,9 @@ networks: vpcbr: driver: bridge ipam: - config: - - subnet: 10.5.0.0/16 - gateway: 10.5.0.1 - + config: + - subnet: 10.5.0.0/16 + gateway: 10.5.0.1 # Usage: # - Run 'docker compose --profile cpu-s3fs up -d' to start the app_cpu service # - Run 'docker compose --profile gpu-s3fs up -d' to start the app_gpu service diff --git a/electron/handlers/native.ts b/electron/handlers/native.ts index 06d9d2a6a..556b66e66 100644 --- a/electron/handlers/native.ts +++ b/electron/handlers/native.ts @@ -6,8 +6,11 @@ import { getJanDataFolderPath, getJanExtensionsPath, init, - AppEvent, NativeRoute, + AppEvent, + NativeRoute, + SelectFileProp, } from '@janhq/core/node' +import { SelectFileOption } from '@janhq/core/.' export function handleAppIPCs() { /** @@ -84,23 +87,39 @@ export function handleAppIPCs() { } }) - ipcMain.handle(NativeRoute.selectModelFiles, async () => { - const mainWindow = windowManager.mainWindow - if (!mainWindow) { - console.error('No main window found') - return - } - const { canceled, filePaths } = await dialog.showOpenDialog(mainWindow, { - title: 'Select model files', - buttonLabel: 'Select', - properties: ['openFile', 'openDirectory', 'multiSelections'], - }) - if (canceled) { - return - } + ipcMain.handle( + NativeRoute.selectFiles, + async (_event, option?: SelectFileOption) => { + const mainWindow = windowManager.mainWindow + if (!mainWindow) { + console.error('No main window found') + return + } - return filePaths - }) + const title = option?.title ?? 'Select files' + const buttonLabel = option?.buttonLabel ?? 'Select' + const props: SelectFileProp[] = ['openFile'] + + if (option?.allowMultiple) { + props.push('multiSelections') + } + + if (option?.selectDirectory) { + props.push('openDirectory') + } + console.debug(`Select files with props: ${props}`) + const { canceled, filePaths } = await dialog.showOpenDialog(mainWindow, { + title, + buttonLabel, + properties: props, + filters: option?.filters, + }) + + if (canceled) return + + return filePaths + } + ) ipcMain.handle( NativeRoute.hideQuickAskWindow, diff --git a/electron/handlers/update.ts b/electron/handlers/update.ts index 690c36d51..5e2200e51 100644 --- a/electron/handlers/update.ts +++ b/electron/handlers/update.ts @@ -39,6 +39,7 @@ export function handleAppUpdates() { }) if (action.response === 0) { trayManager.destroyCurrentTray() + windowManager.closeQuickAskWindow() waitingToInstallVersion = _info?.version autoUpdater.quitAndInstall() } diff --git a/electron/main.ts b/electron/main.ts index cdc001af4..1f4719e8d 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -1,4 +1,4 @@ -import { app, BrowserWindow, Tray } from 'electron' +import { app, BrowserWindow } from 'electron' import { join } from 'path' /** @@ -11,7 +11,7 @@ import { getAppConfigurations, log } from '@janhq/core/node' * IPC Handlers **/ import { injectHandler } from './handlers/common' -import { handleAppUpdates, waitingToInstallVersion } from './handlers/update' +import { handleAppUpdates } from './handlers/update' import { handleAppIPCs } from './handlers/native' /** @@ -24,11 +24,10 @@ import { cleanUpAndQuit } from './utils/clean' import { setupExtensions } from './utils/extension' import { setupCore } from './utils/setup' import { setupReactDevTool } from './utils/dev' -import { cleanLogs } from './utils/log' -import { registerShortcut } from './utils/selectedText' import { trayManager } from './managers/tray' import { logSystemInfo } from './utils/system' +import { registerGlobalShortcuts } from './utils/shortcut' const preloadPath = join(__dirname, 'preload.js') const rendererPath = join(__dirname, '..', 'renderer') @@ -38,8 +37,6 @@ const mainPath = join(rendererPath, 'index.html') const mainUrl = 'http://localhost:3000' const quickAskUrl = `${mainUrl}/search` -const quickAskHotKey = 'CommandOrControl+J' - const gotTheLock = app.requestSingleInstanceLock() app @@ -60,6 +57,7 @@ app .then(handleAppUpdates) .then(() => process.env.CI !== 'e2e' && createQuickAskWindow()) .then(createMainWindow) + .then(registerGlobalShortcuts) .then(() => { if (!app.isPackaged) { windowManager.mainWindow?.webContents.openDevTools() @@ -76,16 +74,11 @@ app } }) }) - .then(() => cleanLogs()) app.on('second-instance', (_event, _commandLine, _workingDirectory) => { windowManager.showMainWindow() }) -app.on('ready', () => { - registerGlobalShortcuts() -}) - app.on('before-quit', function (evt) { trayManager.destroyCurrentTray() }) @@ -96,7 +89,11 @@ app.once('quit', () => { app.once('window-all-closed', () => { // Feature Toggle for Quick Ask - if (getAppConfigurations().quick_ask && !waitingToInstallVersion) return + if ( + getAppConfigurations().quick_ask && + !windowManager.isQuickAskWindowDestroyed() + ) + return cleanUpAndQuit() }) @@ -112,26 +109,6 @@ function createMainWindow() { windowManager.createMainWindow(preloadPath, startUrl) } -function registerGlobalShortcuts() { - const ret = registerShortcut(quickAskHotKey, (selectedText: string) => { - // Feature Toggle for Quick Ask - if (!getAppConfigurations().quick_ask) return - - if (!windowManager.isQuickAskWindowVisible()) { - windowManager.showQuickAskWindow() - windowManager.sendQuickAskSelectedText(selectedText) - } else { - windowManager.hideQuickAskWindow() - } - }) - - if (!ret) { - console.error('Global shortcut registration failed') - } else { - console.log('Global shortcut registered successfully') - } -} - /** * Handles various IPC messages from the renderer process. */ diff --git a/electron/managers/window.ts b/electron/managers/window.ts index f24d25c61..8c7348651 100644 --- a/electron/managers/window.ts +++ b/electron/managers/window.ts @@ -45,7 +45,7 @@ class WindowManager { windowManager.mainWindow?.on('close', function (evt) { // Feature Toggle for Quick Ask if (!getAppConfigurations().quick_ask) return - + if (!isAppQuitting) { evt.preventDefault() windowManager.hideMainWindow() @@ -93,10 +93,22 @@ class WindowManager { this._quickAskWindowVisible = true } + closeQuickAskWindow(): void { + if (this._quickAskWindow?.isDestroyed()) return + this._quickAskWindow?.close() + this._quickAskWindow?.destroy() + this._quickAskWindow = undefined + this._quickAskWindowVisible = false + } + isQuickAskWindowVisible(): boolean { return this._quickAskWindowVisible } + isQuickAskWindowDestroyed(): boolean { + return this._quickAskWindow?.isDestroyed() ?? true + } + expandQuickAskWindow(heightOffset: number): void { const width = quickAskWindowConfig.width! const height = quickAskWindowConfig.height! + heightOffset @@ -112,10 +124,18 @@ class WindowManager { } cleanUp(): void { - this.mainWindow?.destroy() - this._quickAskWindow?.destroy() - this._quickAskWindowVisible = false - this._mainWindowVisible = false + if (!this.mainWindow?.isDestroyed()) { + this.mainWindow?.close() + this.mainWindow?.destroy() + this.mainWindow = undefined + this._mainWindowVisible = false + } + if (!this._quickAskWindow?.isDestroyed()) { + this._quickAskWindow?.close() + this._quickAskWindow?.destroy() + this._quickAskWindow = undefined + this._quickAskWindowVisible = false + } } } diff --git a/electron/package.json b/electron/package.json index 49cbadf7a..e76b9172a 100644 --- a/electron/package.json +++ b/electron/package.json @@ -14,14 +14,12 @@ "renderer/**/*", "build/**/*.{js,map}", "pre-install", - "models/**/*", "docs/**/*", "scripts/**/*", "icons/**/*" ], "asarUnpack": [ "pre-install", - "models", "docs", "scripts", "icons" @@ -110,7 +108,8 @@ "eslint-plugin-react": "^7.34.0", "rimraf": "^5.0.5", "run-script-os": "^1.1.6", - "typescript": "^5.3.3" + "typescript": "^5.3.3", + "@reportportal/agent-js-playwright": "^5.1.7" }, "installConfig": { "hoistingLimits": "workspaces" diff --git a/electron/utils/log.ts b/electron/utils/log.ts deleted file mode 100644 index 9dcd4563b..000000000 --- a/electron/utils/log.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { getJanDataFolderPath } from '@janhq/core/node' -import * as fs from 'fs' -import * as path from 'path' - -export function cleanLogs( - maxFileSizeBytes?: number | undefined, - daysToKeep?: number | undefined, - delayMs?: number | undefined -): void { - const size = maxFileSizeBytes ?? 1 * 1024 * 1024 // 1 MB - const days = daysToKeep ?? 7 // 7 days - const delays = delayMs ?? 10000 // 10 seconds - const logDirectory = path.join(getJanDataFolderPath(), 'logs') - - // Perform log cleaning - const currentDate = new Date() - fs.readdir(logDirectory, (err, files) => { - if (err) { - console.error('Error reading log directory:', err) - return - } - - files.forEach((file) => { - const filePath = path.join(logDirectory, file) - fs.stat(filePath, (err, stats) => { - if (err) { - console.error('Error getting file stats:', err) - return - } - - // Check size - if (stats.size > size) { - fs.unlink(filePath, (err) => { - if (err) { - console.error('Error deleting log file:', err) - return - } - console.debug( - `Deleted log file due to exceeding size limit: ${filePath}` - ) - }) - } else { - // Check age - const creationDate = new Date(stats.ctime) - const daysDifference = Math.floor( - (currentDate.getTime() - creationDate.getTime()) / - (1000 * 3600 * 24) - ) - if (daysDifference > days) { - fs.unlink(filePath, (err) => { - if (err) { - console.error('Error deleting log file:', err) - return - } - console.debug(`Deleted old log file: ${filePath}`) - }) - } - } - }) - }) - }) - - // Schedule the next execution with doubled delays - setTimeout(() => { - cleanLogs(maxFileSizeBytes, daysToKeep, delays * 2) - }, delays) -} diff --git a/electron/utils/shortcut.ts b/electron/utils/shortcut.ts new file mode 100644 index 000000000..aa4607d9a --- /dev/null +++ b/electron/utils/shortcut.ts @@ -0,0 +1,24 @@ +import { getAppConfigurations } from '@janhq/core/node' +import { registerShortcut } from './selectedText' +import { windowManager } from '../managers/window' +// TODO: Retrieve from config later +const quickAskHotKey = 'CommandOrControl+J' + +export function registerGlobalShortcuts() { + if (!getAppConfigurations().quick_ask) return + const ret = registerShortcut(quickAskHotKey, (selectedText: string) => { + // Feature Toggle for Quick Ask + if (!windowManager.isQuickAskWindowVisible()) { + windowManager.showQuickAskWindow() + windowManager.sendQuickAskSelectedText(selectedText) + } else { + windowManager.hideQuickAskWindow() + } + }) + + if (!ret) { + console.error('Global shortcut registration failed') + } else { + console.log('Global shortcut registered successfully') + } +} diff --git a/extensions/assistant-extension/README.md b/extensions/assistant-extension/README.md index 16cde1392..f9690da09 100644 --- a/extensions/assistant-extension/README.md +++ b/extensions/assistant-extension/README.md @@ -1,14 +1,10 @@ -# Jan Assistant plugin +# Create a Jan Extension using Typescript -Created using Jan app example +Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀 -# Create a Jan Plugin using Typescript +## Create Your Own Extension -Use this template to bootstrap the creation of a TypeScript Jan plugin. 🚀 - -## Create Your Own Plugin - -To create your own plugin, you can use this repository as a template! Just follow the below instructions: +To create your own extension, you can use this repository as a template! Just follow the below instructions: 1. Click the Use this template button at the top of the repository 2. Select Create a new repository @@ -18,7 +14,7 @@ To create your own plugin, you can use this repository as a template! Just follo ## Initial Setup -After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your plugin. +After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension. > [!NOTE] > @@ -43,35 +39,37 @@ After you've cloned the repository to your local machine or codespace, you'll ne 1. :white_check_mark: Check your artifact - There will be a tgz file in your plugin directory now + There will be a tgz file in your extension directory now -## Update the Plugin Metadata +## Update the Extension Metadata -The [`package.json`](package.json) file defines metadata about your plugin, such as -plugin name, main entry, description and version. +The [`package.json`](package.json) file defines metadata about your extension, such as +extension name, main entry, description and version. -When you copy this repository, update `package.json` with the name, description for your plugin. +When you copy this repository, update `package.json` with the name, description for your extension. -## Update the Plugin Code +## Update the Extension Code -The [`src/`](./src/) directory is the heart of your plugin! This contains the -source code that will be run when your plugin extension functions are invoked. You can replace the +The [`src/`](./src/) directory is the heart of your extension! This contains the +source code that will be run when your extension functions are invoked. You can replace the contents of this directory with your own code. -There are a few things to keep in mind when writing your plugin code: +There are a few things to keep in mind when writing your extension code: -- Most Jan Plugin Extension functions are processed asynchronously. +- Most Jan Extension functions are processed asynchronously. In `index.ts`, you will see that the extension function will return a `Promise`. ```typescript - import { core } from "@janhq/core"; + import { events, MessageEvent, MessageRequest } from '@janhq/core' function onStart(): Promise { - return core.invokePluginFunc(MODULE_PATH, "run", 0); + return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => + this.inference(data) + ) } ``` - For more information about the Jan Plugin Core module, see the + For more information about the Jan Extension Core module, see the [documentation](https://github.com/janhq/jan/blob/main/core/README.md). -So, what are you waiting for? Go ahead and start customizing your plugin! +So, what are you waiting for? Go ahead and start customizing your extension! diff --git a/extensions/assistant-extension/package.json b/extensions/assistant-extension/package.json index f7fde7f78..a76763244 100644 --- a/extensions/assistant-extension/package.json +++ b/extensions/assistant-extension/package.json @@ -1,5 +1,6 @@ { "name": "@janhq/assistant-extension", + "productName": "Jan Assistant Extension", "version": "1.0.1", "description": "This extension enables assistants, including Jan, a default assistant that can call all downloaded models", "main": "dist/index.js", diff --git a/extensions/assistant-extension/rollup.config.ts b/extensions/assistant-extension/rollup.config.ts index 0d1e4832c..263f6cc60 100644 --- a/extensions/assistant-extension/rollup.config.ts +++ b/extensions/assistant-extension/rollup.config.ts @@ -7,12 +7,10 @@ import replace from '@rollup/plugin-replace' const packageJson = require('./package.json') -const pkg = require('./package.json') - export default [ { input: `src/index.ts`, - output: [{ file: pkg.main, format: 'es', sourcemap: true }], + output: [{ file: packageJson.main, format: 'es', sourcemap: true }], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') external: [], watch: { @@ -20,8 +18,8 @@ export default [ }, plugins: [ replace({ + preventAssignment: true, NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`), - EXTENSION_NAME: JSON.stringify(packageJson.name), VERSION: JSON.stringify(packageJson.version), }), // Allow json resolution @@ -36,7 +34,7 @@ export default [ // https://github.com/rollup/rollup-plugin-node-resolve#usage resolve({ extensions: ['.js', '.ts', '.svelte'], - browser: true + browser: true, }), // Resolve source maps to the original source diff --git a/extensions/assistant-extension/src/@types/global.d.ts b/extensions/assistant-extension/src/@types/global.d.ts index bc97157cd..2ca4a4080 100644 --- a/extensions/assistant-extension/src/@types/global.d.ts +++ b/extensions/assistant-extension/src/@types/global.d.ts @@ -1,3 +1,2 @@ declare const NODE: string -declare const EXTENSION_NAME: string declare const VERSION: string diff --git a/extensions/assistant-extension/src/index.ts b/extensions/assistant-extension/src/index.ts index f4e098947..53d3ed0d5 100644 --- a/extensions/assistant-extension/src/index.ts +++ b/extensions/assistant-extension/src/index.ts @@ -21,7 +21,7 @@ export default class JanAssistantExtension extends AssistantExtension { JanAssistantExtension._homeDir ) if ( - localStorage.getItem(`${EXTENSION_NAME}-version`) !== VERSION || + localStorage.getItem(`${this.name}-version`) !== VERSION || !assistantDirExist ) { if (!assistantDirExist) await fs.mkdir(JanAssistantExtension._homeDir) @@ -29,7 +29,7 @@ export default class JanAssistantExtension extends AssistantExtension { // Write assistant metadata await this.createJanAssistant() // Finished migration - localStorage.setItem(`${EXTENSION_NAME}-version`, VERSION) + localStorage.setItem(`${this.name}-version`, VERSION) // Update the assistant list events.emit(AssistantEvent.OnAssistantsUpdate, {}) } diff --git a/extensions/assistant-extension/src/node/engine.ts b/extensions/assistant-extension/src/node/engine.ts index 70d02af1f..17094ffbc 100644 --- a/extensions/assistant-extension/src/node/engine.ts +++ b/extensions/assistant-extension/src/node/engine.ts @@ -1,13 +1,36 @@ import fs from 'fs' import path from 'path' -import { getJanDataFolderPath } from '@janhq/core/node' +import { SettingComponentProps, getJanDataFolderPath } from '@janhq/core/node' // Sec: Do not send engine settings over requests // Read it manually instead export const readEmbeddingEngine = (engineName: string) => { - const engineSettings = fs.readFileSync( - path.join(getJanDataFolderPath(), 'engines', `${engineName}.json`), - 'utf-8' - ) - return JSON.parse(engineSettings) + if (engineName !== 'openai' && engineName !== 'groq') { + const engineSettings = fs.readFileSync( + path.join(getJanDataFolderPath(), 'engines', `${engineName}.json`), + 'utf-8' + ) + return JSON.parse(engineSettings) + } else { + const settingDirectoryPath = path.join( + getJanDataFolderPath(), + 'settings', + engineName === 'openai' + ? 'inference-openai-extension' + : 'inference-groq-extension', + 'settings.json' + ) + + const content = fs.readFileSync(settingDirectoryPath, 'utf-8') + const settings: SettingComponentProps[] = JSON.parse(content) + const apiKeyId = engineName === 'openai' ? 'openai-api-key' : 'groq-api-key' + const keySetting = settings.find((setting) => setting.key === apiKeyId) + + let apiKey = keySetting?.controllerProps.value + if (typeof apiKey !== 'string') apiKey = '' + + return { + api_key: apiKey, + } + } } diff --git a/extensions/conversational-extension/package.json b/extensions/conversational-extension/package.json index a803320f5..712a9883c 100644 --- a/extensions/conversational-extension/package.json +++ b/extensions/conversational-extension/package.json @@ -1,5 +1,6 @@ { "name": "@janhq/conversational-extension", + "productName": "Conversational Extension", "version": "1.0.0", "description": "This extension enables conversations and state persistence via your filesystem", "main": "dist/index.js", diff --git a/extensions/huggingface-extension/README.md b/extensions/huggingface-extension/README.md index ae70eb4ec..f9690da09 100644 --- a/extensions/huggingface-extension/README.md +++ b/extensions/huggingface-extension/README.md @@ -1,10 +1,10 @@ -# Create a Jan Plugin using Typescript +# Create a Jan Extension using Typescript -Use this template to bootstrap the creation of a TypeScript Jan plugin. 🚀 +Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀 -## Create Your Own Plugin +## Create Your Own Extension -To create your own plugin, you can use this repository as a template! Just follow the below instructions: +To create your own extension, you can use this repository as a template! Just follow the below instructions: 1. Click the Use this template button at the top of the repository 2. Select Create a new repository @@ -14,7 +14,7 @@ To create your own plugin, you can use this repository as a template! Just follo ## Initial Setup -After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your plugin. +After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension. > [!NOTE] > @@ -39,35 +39,37 @@ After you've cloned the repository to your local machine or codespace, you'll ne 1. :white_check_mark: Check your artifact - There will be a tgz file in your plugin directory now + There will be a tgz file in your extension directory now -## Update the Plugin Metadata +## Update the Extension Metadata -The [`package.json`](package.json) file defines metadata about your plugin, such as -plugin name, main entry, description and version. +The [`package.json`](package.json) file defines metadata about your extension, such as +extension name, main entry, description and version. -When you copy this repository, update `package.json` with the name, description for your plugin. +When you copy this repository, update `package.json` with the name, description for your extension. -## Update the Plugin Code +## Update the Extension Code -The [`src/`](./src/) directory is the heart of your plugin! This contains the -source code that will be run when your plugin extension functions are invoked. You can replace the +The [`src/`](./src/) directory is the heart of your extension! This contains the +source code that will be run when your extension functions are invoked. You can replace the contents of this directory with your own code. -There are a few things to keep in mind when writing your plugin code: +There are a few things to keep in mind when writing your extension code: -- Most Jan Plugin Extension functions are processed asynchronously. +- Most Jan Extension functions are processed asynchronously. In `index.ts`, you will see that the extension function will return a `Promise`. ```typescript - import { core } from "@janhq/core"; + import { events, MessageEvent, MessageRequest } from '@janhq/core' function onStart(): Promise { - return core.invokePluginFunc(MODULE_PATH, "run", 0); + return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => + this.inference(data) + ) } ``` - For more information about the Jan Plugin Core module, see the + For more information about the Jan Extension Core module, see the [documentation](https://github.com/janhq/jan/blob/main/core/README.md). -So, what are you waiting for? Go ahead and start customizing your plugin! +So, what are you waiting for? Go ahead and start customizing your extension! diff --git a/extensions/huggingface-extension/package.json b/extensions/huggingface-extension/package.json index e71dc7406..234b806d8 100644 --- a/extensions/huggingface-extension/package.json +++ b/extensions/huggingface-extension/package.json @@ -1,5 +1,6 @@ { "name": "@janhq/huggingface-extension", + "productName": "HuggingFace Extension", "version": "1.0.0", "description": "Hugging Face extension for converting HF models to GGUF", "main": "dist/index.js", diff --git a/extensions/huggingface-extension/rollup.config.ts b/extensions/huggingface-extension/rollup.config.ts index 7ae2c5781..16cf3c46d 100644 --- a/extensions/huggingface-extension/rollup.config.ts +++ b/extensions/huggingface-extension/rollup.config.ts @@ -18,7 +18,7 @@ export default [ }, plugins: [ replace({ - EXTENSION_NAME: JSON.stringify(packageJson.name), + preventAssignment: true, NODE_MODULE_PATH: JSON.stringify( `${packageJson.name}/${packageJson.node}` ), diff --git a/extensions/huggingface-extension/src/@types/global.d.ts b/extensions/huggingface-extension/src/@types/global.d.ts index 495ecf00e..b30fe9d69 100644 --- a/extensions/huggingface-extension/src/@types/global.d.ts +++ b/extensions/huggingface-extension/src/@types/global.d.ts @@ -1,2 +1 @@ -declare const EXTENSION_NAME: string declare const NODE_MODULE_PATH: string diff --git a/extensions/huggingface-extension/src/index.ts b/extensions/huggingface-extension/src/index.ts index b703a3e33..0425b9f88 100644 --- a/extensions/huggingface-extension/src/index.ts +++ b/extensions/huggingface-extension/src/index.ts @@ -338,7 +338,7 @@ export default class JanHuggingFaceExtension extends HuggingFaceExtension { const metadata: Model = { object: 'model', - version: 1, + version: '1.0', format: 'gguf', sources: [ { diff --git a/extensions/huggingface-extension/src/node/index.ts b/extensions/huggingface-extension/src/node/index.ts index d21255509..991548e00 100644 --- a/extensions/huggingface-extension/src/node/index.ts +++ b/extensions/huggingface-extension/src/node/index.ts @@ -32,13 +32,9 @@ export const getQuantizeExecutable = (): string => { binaryName = 'quantize.exe' } else if (process.platform === 'darwin') { /** - * For MacOS: mac-arm64 (Silicon), mac-x64 (InteL) + * For MacOS: mac-universal both Silicon and InteL */ - if (process.arch === 'arm64') { - binaryFolder = pjoin(binaryFolder, 'mac-arm64') - } else { - binaryFolder = pjoin(binaryFolder, 'mac-x64') - } + binaryFolder = pjoin(binaryFolder, 'mac-universal') } else { binaryFolder = pjoin(binaryFolder, 'linux-cpu') } diff --git a/extensions/inference-groq-extension/README.md b/extensions/inference-groq-extension/README.md index 455783efb..f9690da09 100644 --- a/extensions/inference-groq-extension/README.md +++ b/extensions/inference-groq-extension/README.md @@ -1,14 +1,10 @@ -# Jan inference plugin +# Create a Jan Extension using Typescript -Created using Jan app example +Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀 -# Create a Jan Plugin using Typescript +## Create Your Own Extension -Use this template to bootstrap the creation of a TypeScript Jan plugin. 🚀 - -## Create Your Own Plugin - -To create your own plugin, you can use this repository as a template! Just follow the below instructions: +To create your own extension, you can use this repository as a template! Just follow the below instructions: 1. Click the Use this template button at the top of the repository 2. Select Create a new repository @@ -18,7 +14,7 @@ To create your own plugin, you can use this repository as a template! Just follo ## Initial Setup -After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your plugin. +After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension. > [!NOTE] > @@ -43,36 +39,37 @@ After you've cloned the repository to your local machine or codespace, you'll ne 1. :white_check_mark: Check your artifact - There will be a tgz file in your plugin directory now + There will be a tgz file in your extension directory now -## Update the Plugin Metadata +## Update the Extension Metadata -The [`package.json`](package.json) file defines metadata about your plugin, such as -plugin name, main entry, description and version. +The [`package.json`](package.json) file defines metadata about your extension, such as +extension name, main entry, description and version. -When you copy this repository, update `package.json` with the name, description for your plugin. +When you copy this repository, update `package.json` with the name, description for your extension. -## Update the Plugin Code +## Update the Extension Code -The [`src/`](./src/) directory is the heart of your plugin! This contains the -source code that will be run when your plugin extension functions are invoked. You can replace the +The [`src/`](./src/) directory is the heart of your extension! This contains the +source code that will be run when your extension functions are invoked. You can replace the contents of this directory with your own code. -There are a few things to keep in mind when writing your plugin code: +There are a few things to keep in mind when writing your extension code: -- Most Jan Plugin Extension functions are processed asynchronously. +- Most Jan Extension functions are processed asynchronously. In `index.ts`, you will see that the extension function will return a `Promise`. ```typescript - import { core } from "@janhq/core"; + import { events, MessageEvent, MessageRequest } from '@janhq/core' function onStart(): Promise { - return core.invokePluginFunc(MODULE_PATH, "run", 0); + return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => + this.inference(data) + ) } ``` - For more information about the Jan Plugin Core module, see the + For more information about the Jan Extension Core module, see the [documentation](https://github.com/janhq/jan/blob/main/core/README.md). -So, what are you waiting for? Go ahead and start customizing your plugin! - +So, what are you waiting for? Go ahead and start customizing your extension! diff --git a/extensions/inference-groq-extension/package.json b/extensions/inference-groq-extension/package.json index 78efd3552..8d70d1d9f 100644 --- a/extensions/inference-groq-extension/package.json +++ b/extensions/inference-groq-extension/package.json @@ -1,5 +1,6 @@ { "name": "@janhq/inference-groq-extension", + "productName": "Groq Inference Engine Extension", "version": "1.0.0", "description": "This extension enables fast Groq chat completion API calls", "main": "dist/index.js", diff --git a/extensions/inference-groq-extension/resources/models.json b/extensions/inference-groq-extension/resources/models.json new file mode 100644 index 000000000..b2775e2be --- /dev/null +++ b/extensions/inference-groq-extension/resources/models.json @@ -0,0 +1,58 @@ +[ + { + "sources": [ + { + "url": "https://groq.com" + } + ], + "id": "llama2-70b-4096", + "object": "model", + "name": "Groq Llama 2 70b", + "version": "1.0", + "description": "Groq Llama 2 70b with supercharged speed!", + "format": "api", + "settings": { + "text_model": false + }, + "parameters": { + "max_tokens": 4096, + "temperature": 0.7, + "top_p": 1, + "stop": null, + "stream": true + }, + "metadata": { + "author": "Meta", + "tags": ["General", "Big Context Length"] + }, + "engine": "groq" + }, + { + "sources": [ + { + "url": "https://groq.com" + } + ], + "id": "mixtral-8x7b-32768", + "object": "model", + "name": "Groq Mixtral 8x7b Instruct", + "version": "1.0", + "description": "Groq Mixtral 8x7b Instruct is Mixtral with supercharged speed!", + "format": "api", + "settings": { + "text_model": false + }, + "parameters": { + "max_tokens": 4096, + "temperature": 0.7, + "top_p": 1, + "stop": null, + "stream": true + }, + "metadata": { + "author": "Mistral", + "tags": ["General", "Big Context Length"] + }, + "engine": "groq" + } +] diff --git a/extensions/inference-groq-extension/resources/settings.json b/extensions/inference-groq-extension/resources/settings.json new file mode 100644 index 000000000..493b602cd --- /dev/null +++ b/extensions/inference-groq-extension/resources/settings.json @@ -0,0 +1,23 @@ +[ + { + "key": "chat-completions-endpoint", + "title": "Chat Completions Endpoint", + "description": "The endpoint to use for chat completions. See the [Groq documentation](https://console.groq.com/docs/openai) for more information.", + "controllerType": "input", + "controllerProps": { + "placeholder": "https://api.groq.com/openai/v1/chat/completions", + "value": "https://api.groq.com/openai/v1/chat/completions" + } + }, + { + "key": "groq-api-key", + "title": "API Key", + "description": "The Groq API uses API keys for authentication. Visit your [API Keys](https://console.groq.com/keys) page to retrieve the API key you'll use in your requests.", + "controllerType": "input", + "controllerProps": { + "placeholder": "gsk_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "value": "", + "type": "password" + } + } +] diff --git a/extensions/inference-groq-extension/src/index.ts b/extensions/inference-groq-extension/src/index.ts index cd22c62f9..eafb7fe8a 100644 --- a/extensions/inference-groq-extension/src/index.ts +++ b/extensions/inference-groq-extension/src/index.ts @@ -6,78 +6,62 @@ * @module inference-groq-extension/src/index */ -import { - events, - fs, - AppConfigurationEventName, - joinPath, - RemoteOAIEngine, -} from '@janhq/core' -import { join } from 'path' +import { RemoteOAIEngine, SettingComponentProps } from '@janhq/core' -declare const COMPLETION_URL: string +declare const SETTINGS: Array +declare const MODELS: Array + +enum Settings { + apiKey = 'groq-api-key', + chatCompletionsEndPoint = 'chat-completions-endpoint', +} /** * A class that implements the InferenceExtension interface from the @janhq/core package. * The class provides methods for initializing and stopping a model, and for making inference requests. * It also subscribes to events emitted by the @janhq/core package and handles new message requests. */ export default class JanInferenceGroqExtension extends RemoteOAIEngine { - private readonly _engineDir = 'file://engines' - private readonly _engineMetadataFileName = 'groq.json' - - inferenceUrl: string = COMPLETION_URL + inferenceUrl: string = '' provider = 'groq' - apiKey = '' - private _engineSettings = { - full_url: COMPLETION_URL, - api_key: 'gsk-', - } - - /** - * Subscribes to events emitted by the @janhq/core package. - */ - async onLoad() { + override async onLoad(): Promise { super.onLoad() - if (!(await fs.existsSync(this._engineDir))) { - await fs.mkdir(this._engineDir) - } + // Register Settings + this.registerSettings(SETTINGS) + this.registerModels(MODELS) - this.writeDefaultEngineSettings() - - const settingsFilePath = await joinPath([ - this._engineDir, - this._engineMetadataFileName, - ]) - - // Events subscription - events.on( - AppConfigurationEventName.OnConfigurationUpdate, - (settingsKey: string) => { - // Update settings on changes - if (settingsKey === settingsFilePath) this.writeDefaultEngineSettings() - } + // Retrieve API Key Setting + this.apiKey = await this.getSetting(Settings.apiKey, '') + this.inferenceUrl = await this.getSetting( + Settings.chatCompletionsEndPoint, + '' ) + + if (this.inferenceUrl.length === 0) { + SETTINGS.forEach((setting) => { + if (setting.key === Settings.chatCompletionsEndPoint) { + this.inferenceUrl = setting.controllerProps.value as string + } + }) + } } - async writeDefaultEngineSettings() { - try { - const engineFile = join(this._engineDir, this._engineMetadataFileName) - if (await fs.existsSync(engineFile)) { - const engine = await fs.readFileSync(engineFile, 'utf-8') - this._engineSettings = - typeof engine === 'object' ? engine : JSON.parse(engine) - this.inferenceUrl = this._engineSettings.full_url - this.apiKey = this._engineSettings.api_key + onSettingUpdate(key: string, value: T): void { + if (key === Settings.apiKey) { + this.apiKey = value as string + } else if (key === Settings.chatCompletionsEndPoint) { + if (typeof value !== 'string') return + + if (value.trim().length === 0) { + SETTINGS.forEach((setting) => { + if (setting.key === Settings.chatCompletionsEndPoint) { + this.inferenceUrl = setting.controllerProps.value as string + } + }) } else { - await fs.writeFileSync( - engineFile, - JSON.stringify(this._engineSettings, null, 2) - ) + this.inferenceUrl = value } - } catch (err) { - console.error(err) } } } diff --git a/extensions/inference-groq-extension/webpack.config.js b/extensions/inference-groq-extension/webpack.config.js index 5352b56b7..58ade764b 100644 --- a/extensions/inference-groq-extension/webpack.config.js +++ b/extensions/inference-groq-extension/webpack.config.js @@ -1,6 +1,8 @@ const path = require('path') const webpack = require('webpack') const packageJson = require('./package.json') +const settingJson = require('./resources/settings.json') +const modelsJson = require('./resources/models.json') module.exports = { experiments: { outputModule: true }, @@ -17,8 +19,9 @@ module.exports = { }, plugins: [ new webpack.DefinePlugin({ + MODELS: JSON.stringify(modelsJson), + SETTINGS: JSON.stringify(settingJson), MODULE: JSON.stringify(`${packageJson.name}/${packageJson.module}`), - COMPLETION_URL: JSON.stringify('https://api.groq.com/openai/v1/chat/completions'), }), ], output: { diff --git a/extensions/inference-mistral-extension/README.md b/extensions/inference-mistral-extension/README.md new file mode 100644 index 000000000..adb36558c --- /dev/null +++ b/extensions/inference-mistral-extension/README.md @@ -0,0 +1,79 @@ +# Mistral Engine Extension + +Created using Jan extension example + +# Create a Jan Extension using Typescript + +Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀 + +## Create Your Own Extension + +To create your own extension, you can use this repository as a template! Just follow the below instructions: + +1. Click the Use this template button at the top of the repository +2. Select Create a new repository +3. Select an owner and name for your new repository +4. Click Create repository +5. Clone your new repository + +## Initial Setup + +After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension. + +> [!NOTE] +> +> You'll need to have a reasonably modern version of +> [Node.js](https://nodejs.org) handy. If you are using a version manager like +> [`nodenv`](https://github.com/nodenv/nodenv) or +> [`nvm`](https://github.com/nvm-sh/nvm), you can run `nodenv install` in the +> root of your repository to install the version specified in +> [`package.json`](./package.json). Otherwise, 20.x or later should work! + +1. :hammer_and_wrench: Install the dependencies + + ```bash + npm install + ``` + +1. :building_construction: Package the TypeScript for distribution + + ```bash + npm run bundle + ``` + +1. :white_check_mark: Check your artifact + + There will be a tgz file in your extension directory now + +## Update the Extension Metadata + +The [`package.json`](package.json) file defines metadata about your extension, such as +extension name, main entry, description and version. + +When you copy this repository, update `package.json` with the name, description for your extension. + +## Update the Extension Code + +The [`src/`](./src/) directory is the heart of your extension! This contains the +source code that will be run when your extension functions are invoked. You can replace the +contents of this directory with your own code. + +There are a few things to keep in mind when writing your extension code: + +- Most Jan Extension functions are processed asynchronously. + In `index.ts`, you will see that the extension function will return a `Promise`. + + ```typescript + import { events, MessageEvent, MessageRequest } from '@janhq/core' + + function onStart(): Promise { + return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => + this.inference(data) + ) + } + ``` + + For more information about the Jan Extension Core module, see the + [documentation](https://github.com/janhq/jan/blob/main/core/README.md). + +So, what are you waiting for? Go ahead and start customizing your extension! diff --git a/extensions/inference-mistral-extension/package.json b/extensions/inference-mistral-extension/package.json new file mode 100644 index 000000000..c1de1f959 --- /dev/null +++ b/extensions/inference-mistral-extension/package.json @@ -0,0 +1,43 @@ +{ + "name": "@janhq/inference-mistral-extension", + "productName": "Mistral AI Inference Engine Extension", + "version": "1.0.0", + "description": "This extension enables Mistral chat completion API calls", + "main": "dist/index.js", + "module": "dist/module.js", + "engine": "mistral", + "author": "Jan ", + "license": "AGPL-3.0", + "scripts": { + "build": "tsc -b . && webpack --config webpack.config.js", + "build:publish": "rimraf *.tgz --glob && yarn build && npm pack && cpx *.tgz ../../pre-install" + }, + "exports": { + ".": "./dist/index.js", + "./main": "./dist/module.js" + }, + "devDependencies": { + "cpx": "^1.5.0", + "rimraf": "^3.0.2", + "webpack": "^5.88.2", + "webpack-cli": "^5.1.4", + "ts-loader": "^9.5.0" + }, + "dependencies": { + "@janhq/core": "file:../../core", + "fetch-retry": "^5.0.6", + "path-browserify": "^1.0.1", + "ulidx": "^2.3.0" + }, + "engines": { + "node": ">=18.0.0" + }, + "files": [ + "dist/*", + "package.json", + "README.md" + ], + "bundleDependencies": [ + "fetch-retry" + ] +} diff --git a/extensions/inference-mistral-extension/resources/models.json b/extensions/inference-mistral-extension/resources/models.json new file mode 100644 index 000000000..a5189311b --- /dev/null +++ b/extensions/inference-mistral-extension/resources/models.json @@ -0,0 +1,85 @@ +[ + { + "sources": [ + { + "url": "https://docs.mistral.ai/api/" + } + ], + "id": "mistral-small-latest", + "object": "model", + "name": "Mistral Small", + "version": "1.0", + "description": "Mistral Small is the ideal choice for simpe tasks that one can do in builk - like Classification, Customer Support, or Text Generation. It offers excellent performance at an affordable price point.", + "format": "api", + "settings": {}, + "parameters": { + "max_tokens": 4096, + "temperature": 0.7 + }, + "metadata": { + "author": "Mistral", + "tags": [ + "Classification", + "Customer Support", + "Text Generation" + ] + }, + "engine": "mistral" + }, + { + "sources": [ + { + "url": "https://docs.mistral.ai/api/" + } + ], + "id": "mistral-medium-latest", + "object": "model", + "name": "Mistral Medium", + "version": "1.0", + "description": "Mistral Medium is the ideal for intermediate tasks that require moderate reasoning - like Data extraction, Summarizing a Document, Writing a Job Description, or Writing Product Descriptions. Mistral Medium strikes a balance between performance and capability, making it suitable for a wide range of tasks that only require language transformaion", + "format": "api", + "settings": {}, + "parameters": { + "max_tokens": 4096, + "temperature": 0.7 + }, + "metadata": { + "author": "Mistral", + "tags": [ + "Data extraction", + "Summarizing a Document", + "Writing a Job Description", + "Writing Product Descriptions" + ] + }, + "engine": "mistral" + }, + { + "sources": [ + { + "url": "https://docs.mistral.ai/api/" + } + ], + "id": "mistral-large-latest", + "object": "model", + "name": "Mistral Large", + "version": "1.0", + "description": "Mistral Large is ideal for complex tasks that require large reasoning capabilities or are highly specialized - like Synthetic Text Generation, Code Generation, RAG, or Agents.", + "format": "api", + "settings": {}, + "parameters": { + "max_tokens": 4096, + "temperature": 0.7 + }, + "metadata": { + "author": "Mistral", + "tags": [ + "Text Generation", + "Code Generation", + "RAG", + "Agents" + ] + }, + "engine": "mistral" + } +] diff --git a/extensions/inference-mistral-extension/resources/settings.json b/extensions/inference-mistral-extension/resources/settings.json new file mode 100644 index 000000000..2ca8ec7e5 --- /dev/null +++ b/extensions/inference-mistral-extension/resources/settings.json @@ -0,0 +1,23 @@ +[ + { + "key": "chat-completions-endpoint", + "title": "Chat Completions Endpoint", + "description": "The endpoint to use for chat completions. See the [Mistral API documentation](https://docs.mistral.ai/api/#operation/createChatCompletion) for more information.", + "controllerType": "input", + "controllerProps": { + "placeholder": "https://api.mistral.ai/v1/chat/completions", + "value": "https://api.mistral.ai/v1/chat/completions" + } + }, + { + "key": "mistral-api-key", + "title": "API Key", + "description": "The Mistral API uses API keys for authentication. Visit your [API Keys](https://console.mistral.ai/api-keys/) page to retrieve the API key you'll use in your requests.", + "controllerType": "input", + "controllerProps": { + "placeholder": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "value": "", + "type": "password" + } + } +] diff --git a/extensions/inference-mistral-extension/src/index.ts b/extensions/inference-mistral-extension/src/index.ts new file mode 100644 index 000000000..461fc326e --- /dev/null +++ b/extensions/inference-mistral-extension/src/index.ts @@ -0,0 +1,66 @@ +/** + * @file This file exports a class that implements the InferenceExtension interface from the @janhq/core package. + * The class provides methods for initializing and stopping a model, and for making inference requests. + * It also subscribes to events emitted by the @janhq/core package and handles new message requests. + * @version 1.0.0 + * @module inference-mistral-extension/src/index + */ + +import { RemoteOAIEngine } from '@janhq/core' + +declare const SETTINGS: Array +declare const MODELS: Array + +enum Settings { + apiKey = 'mistral-api-key', + chatCompletionsEndPoint = 'chat-completions-endpoint', +} +/** + * A class that implements the InferenceExtension interface from the @janhq/core package. + * The class provides methods for initializing and stopping a model, and for making inference requests. + * It also subscribes to events emitted by the @janhq/core package and handles new message requests. + */ +export default class JanInferenceMistralExtension extends RemoteOAIEngine { + inferenceUrl: string = '' + provider: string = 'mistral' + + override async onLoad(): Promise { + super.onLoad() + + // Register Settings + this.registerSettings(SETTINGS) + this.registerModels(MODELS) + + this.apiKey = await this.getSetting(Settings.apiKey, '') + this.inferenceUrl = await this.getSetting( + Settings.chatCompletionsEndPoint, + '' + ) + + if (this.inferenceUrl.length === 0) { + SETTINGS.forEach((setting) => { + if (setting.key === Settings.chatCompletionsEndPoint) { + this.inferenceUrl = setting.controllerProps.value as string + } + }) + } + } + + onSettingUpdate(key: string, value: T): void { + if (key === Settings.apiKey) { + this.apiKey = value as string + } else if (key === Settings.chatCompletionsEndPoint) { + if (typeof value !== 'string') return + + if (value.trim().length === 0) { + SETTINGS.forEach((setting) => { + if (setting.key === Settings.chatCompletionsEndPoint) { + this.inferenceUrl = setting.controllerProps.value as string + } + }) + } else { + this.inferenceUrl = value + } + } + } +} diff --git a/extensions/inference-mistral-extension/tsconfig.json b/extensions/inference-mistral-extension/tsconfig.json new file mode 100644 index 000000000..2477d58ce --- /dev/null +++ b/extensions/inference-mistral-extension/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "es2016", + "module": "ES6", + "moduleResolution": "node", + "outDir": "./dist", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": false, + "skipLibCheck": true, + "rootDir": "./src" + }, + "include": ["./src"] +} diff --git a/extensions/inference-mistral-extension/webpack.config.js b/extensions/inference-mistral-extension/webpack.config.js new file mode 100644 index 000000000..0e35fc227 --- /dev/null +++ b/extensions/inference-mistral-extension/webpack.config.js @@ -0,0 +1,42 @@ +const path = require('path') +const webpack = require('webpack') +const packageJson = require('./package.json') +const settingJson = require('./resources/settings.json') +const modelsJson = require('./resources/models.json') + +module.exports = { + experiments: { outputModule: true }, + entry: './src/index.ts', // Adjust the entry point to match your project's main file + mode: 'production', + module: { + rules: [ + { + test: /\.tsx?$/, + use: 'ts-loader', + exclude: /node_modules/, + }, + ], + }, + plugins: [ + new webpack.DefinePlugin({ + SETTINGS: JSON.stringify(settingJson), + ENGINE: JSON.stringify(packageJson.engine), + MODELS: JSON.stringify(modelsJson), + }), + ], + output: { + filename: 'index.js', // Adjust the output file name as needed + path: path.resolve(__dirname, 'dist'), + library: { type: 'module' }, // Specify ESM output format + }, + resolve: { + extensions: ['.ts', '.js'], + fallback: { + path: require.resolve('path-browserify'), + }, + }, + optimization: { + minimize: false, + }, + // Add loaders and other configuration as needed for your project +} diff --git a/extensions/inference-nitro-extension/README.md b/extensions/inference-nitro-extension/README.md index f499e0b9c..f9690da09 100644 --- a/extensions/inference-nitro-extension/README.md +++ b/extensions/inference-nitro-extension/README.md @@ -1,14 +1,10 @@ -# Jan inference plugin +# Create a Jan Extension using Typescript -Created using Jan app example +Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀 -# Create a Jan Plugin using Typescript +## Create Your Own Extension -Use this template to bootstrap the creation of a TypeScript Jan plugin. 🚀 - -## Create Your Own Plugin - -To create your own plugin, you can use this repository as a template! Just follow the below instructions: +To create your own extension, you can use this repository as a template! Just follow the below instructions: 1. Click the Use this template button at the top of the repository 2. Select Create a new repository @@ -18,7 +14,7 @@ To create your own plugin, you can use this repository as a template! Just follo ## Initial Setup -After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your plugin. +After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension. > [!NOTE] > @@ -43,35 +39,37 @@ After you've cloned the repository to your local machine or codespace, you'll ne 1. :white_check_mark: Check your artifact - There will be a tgz file in your plugin directory now + There will be a tgz file in your extension directory now -## Update the Plugin Metadata +## Update the Extension Metadata -The [`package.json`](package.json) file defines metadata about your plugin, such as -plugin name, main entry, description and version. +The [`package.json`](package.json) file defines metadata about your extension, such as +extension name, main entry, description and version. -When you copy this repository, update `package.json` with the name, description for your plugin. +When you copy this repository, update `package.json` with the name, description for your extension. -## Update the Plugin Code +## Update the Extension Code -The [`src/`](./src/) directory is the heart of your plugin! This contains the -source code that will be run when your plugin extension functions are invoked. You can replace the +The [`src/`](./src/) directory is the heart of your extension! This contains the +source code that will be run when your extension functions are invoked. You can replace the contents of this directory with your own code. -There are a few things to keep in mind when writing your plugin code: +There are a few things to keep in mind when writing your extension code: -- Most Jan Plugin Extension functions are processed asynchronously. +- Most Jan Extension functions are processed asynchronously. In `index.ts`, you will see that the extension function will return a `Promise`. ```typescript - import { core } from '@janhq/core' + import { events, MessageEvent, MessageRequest } from '@janhq/core' function onStart(): Promise { - return core.invokePluginFunc(MODULE_PATH, 'run', 0) + return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => + this.inference(data) + ) } ``` - For more information about the Jan Plugin Core module, see the + For more information about the Jan Extension Core module, see the [documentation](https://github.com/janhq/jan/blob/main/core/README.md). -So, what are you waiting for? Go ahead and start customizing your plugin! +So, what are you waiting for? Go ahead and start customizing your extension! diff --git a/extensions/inference-nitro-extension/bin/version.txt b/extensions/inference-nitro-extension/bin/version.txt index 0b69c00c5..dfdc36886 100644 --- a/extensions/inference-nitro-extension/bin/version.txt +++ b/extensions/inference-nitro-extension/bin/version.txt @@ -1 +1 @@ -0.3.14 +0.3.21 diff --git a/extensions/inference-nitro-extension/download.bat b/extensions/inference-nitro-extension/download.bat index bb8c4ffdc..c99162eba 100644 --- a/extensions/inference-nitro-extension/download.bat +++ b/extensions/inference-nitro-extension/download.bat @@ -1,3 +1,3 @@ @echo off set /p NITRO_VERSION=<./bin/version.txt -.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/win-cuda-11-7 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/win-vulkan +.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-avx2-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-avx2-cuda-11-7.tar.gz -e --strip 1 -o ./bin/win-cuda-11-7 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-avx2.tar.gz -e --strip 1 -o ./bin/win-cpu && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/win-vulkan diff --git a/extensions/inference-nitro-extension/package.json b/extensions/inference-nitro-extension/package.json index 25abaf049..78558b1c3 100644 --- a/extensions/inference-nitro-extension/package.json +++ b/extensions/inference-nitro-extension/package.json @@ -1,7 +1,8 @@ { "name": "@janhq/inference-nitro-extension", + "productName": "Nitro Inference Engine Extension", "version": "1.0.0", - "description": "This extension embeds Nitro, a lightweight (3mb) inference engine written in C++. See nitro.jan.ai", + "description": "This extension embeds Nitro, a lightweight (3mb) inference engine written in C++. See https://nitro.jan.ai.\nUse this setting if you encounter errors related to **CUDA toolkit** during application execution.", "main": "dist/index.js", "node": "dist/node/index.cjs.js", "author": "Jan ", @@ -9,8 +10,8 @@ "scripts": { "test": "jest", "build": "tsc --module commonjs && rollup -c rollup.config.ts", - "downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-7 && chmod +x ./bin/linux-cuda-11-7/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/linux-vulkan && chmod +x ./bin/linux-vulkan/nitro", - "downloadnitro:darwin": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-arm64.tar.gz -e --strip 1 -o ./bin/mac-arm64 && chmod +x ./bin/mac-arm64/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-amd64.tar.gz -e --strip 1 -o ./bin/mac-x64 && chmod +x ./bin/mac-x64/nitro", + "downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-avx2.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-7 && chmod +x ./bin/linux-cuda-11-7/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/linux-vulkan && chmod +x ./bin/linux-vulkan/nitro", + "downloadnitro:darwin": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-universal.tar.gz -o ./bin/ && mkdir -p ./bin/mac-universal && tar -zxvf ./bin/nitro-${NITRO_VERSION}-mac-universal.tar.gz --strip-components=1 -C ./bin/mac-universal && rm -rf ./bin/nitro-${NITRO_VERSION}-mac-universal.tar.gz && chmod +x ./bin/mac-universal/nitro", "downloadnitro:win32": "download.bat", "downloadnitro": "run-script-os", "build:publish:darwin": "rimraf *.tgz --glob && yarn build && npm run downloadnitro && ../../.github/scripts/auto-sign.sh && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install", @@ -29,6 +30,7 @@ "@rollup/plugin-json": "^6.1.0", "@rollup/plugin-node-resolve": "^15.2.3", "@rollup/plugin-replace": "^5.0.5", + "@types/decompress": "^4.2.7", "@types/jest": "^29.5.12", "@types/node": "^20.11.4", "@types/os-utils": "^0.0.4", @@ -47,10 +49,12 @@ }, "dependencies": { "@janhq/core": "file:../../core", + "decompress": "^4.2.1", "fetch-retry": "^5.0.6", "path-browserify": "^1.0.1", "rxjs": "^7.8.1", "tcp-port-used": "^1.0.2", + "terminate": "^2.6.1", "ulidx": "^2.3.0" }, "engines": { @@ -64,6 +68,7 @@ "bundleDependencies": [ "tcp-port-used", "fetch-retry", - "@janhq/core" + "@janhq/core", + "decompress" ] } diff --git a/extensions/inference-nitro-extension/resources/default_settings.json b/extensions/inference-nitro-extension/resources/default_settings.json new file mode 100644 index 000000000..39f0880b0 --- /dev/null +++ b/extensions/inference-nitro-extension/resources/default_settings.json @@ -0,0 +1,33 @@ +[ + { + "key": "test", + "title": "Test", + "description": "Test", + "controllerType": "input", + "controllerProps": { + "placeholder": "Test", + "value": "" + } + }, + { + "key": "embedding", + "title": "Embedding", + "description": "Whether to enable embedding.", + "controllerType": "checkbox", + "controllerProps": { + "value": true + } + }, + { + "key": "ctx_len", + "title": "Context Length", + "description": "The context length for model operations varies; the maximum depends on the specific model used.", + "controllerType": "slider", + "controllerProps": { + "min": 0, + "max": 4096, + "step": 128, + "value": 4096 + } + } +] diff --git a/models/bakllava-1/model.json b/extensions/inference-nitro-extension/resources/models/bakllava-1/model.json similarity index 100% rename from models/bakllava-1/model.json rename to extensions/inference-nitro-extension/resources/models/bakllava-1/model.json diff --git a/models/codeninja-1.0-7b/model.json b/extensions/inference-nitro-extension/resources/models/codeninja-1.0-7b/model.json similarity index 100% rename from models/codeninja-1.0-7b/model.json rename to extensions/inference-nitro-extension/resources/models/codeninja-1.0-7b/model.json diff --git a/models/command-r-34b/model.json b/extensions/inference-nitro-extension/resources/models/command-r-34b/model.json similarity index 100% rename from models/command-r-34b/model.json rename to extensions/inference-nitro-extension/resources/models/command-r-34b/model.json diff --git a/models/deepseek-coder-1.3b/model.json b/extensions/inference-nitro-extension/resources/models/deepseek-coder-1.3b/model.json similarity index 100% rename from models/deepseek-coder-1.3b/model.json rename to extensions/inference-nitro-extension/resources/models/deepseek-coder-1.3b/model.json diff --git a/models/deepseek-coder-34b/model.json b/extensions/inference-nitro-extension/resources/models/deepseek-coder-34b/model.json similarity index 100% rename from models/deepseek-coder-34b/model.json rename to extensions/inference-nitro-extension/resources/models/deepseek-coder-34b/model.json diff --git a/models/dolphin-phi-2/model.json b/extensions/inference-nitro-extension/resources/models/dolphin-phi-2/model.json similarity index 100% rename from models/dolphin-phi-2/model.json rename to extensions/inference-nitro-extension/resources/models/dolphin-phi-2/model.json diff --git a/models/gemma-2b/model.json b/extensions/inference-nitro-extension/resources/models/gemma-2b/model.json similarity index 100% rename from models/gemma-2b/model.json rename to extensions/inference-nitro-extension/resources/models/gemma-2b/model.json diff --git a/models/gemma-7b/model.json b/extensions/inference-nitro-extension/resources/models/gemma-7b/model.json similarity index 100% rename from models/gemma-7b/model.json rename to extensions/inference-nitro-extension/resources/models/gemma-7b/model.json diff --git a/models/hermes-pro-7b/model.json b/extensions/inference-nitro-extension/resources/models/hermes-pro-7b/model.json similarity index 100% rename from models/hermes-pro-7b/model.json rename to extensions/inference-nitro-extension/resources/models/hermes-pro-7b/model.json diff --git a/models/llama2-chat-70b/model.json b/extensions/inference-nitro-extension/resources/models/llama2-chat-70b/model.json similarity index 100% rename from models/llama2-chat-70b/model.json rename to extensions/inference-nitro-extension/resources/models/llama2-chat-70b/model.json diff --git a/models/llama2-chat-7b/model.json b/extensions/inference-nitro-extension/resources/models/llama2-chat-7b/model.json similarity index 100% rename from models/llama2-chat-7b/model.json rename to extensions/inference-nitro-extension/resources/models/llama2-chat-7b/model.json diff --git a/models/llamacorn-1.1b/model.json b/extensions/inference-nitro-extension/resources/models/llamacorn-1.1b/model.json similarity index 100% rename from models/llamacorn-1.1b/model.json rename to extensions/inference-nitro-extension/resources/models/llamacorn-1.1b/model.json diff --git a/models/llava-13b/model.json b/extensions/inference-nitro-extension/resources/models/llava-13b/model.json similarity index 100% rename from models/llava-13b/model.json rename to extensions/inference-nitro-extension/resources/models/llava-13b/model.json diff --git a/models/llava-7b/model.json b/extensions/inference-nitro-extension/resources/models/llava-7b/model.json similarity index 100% rename from models/llava-7b/model.json rename to extensions/inference-nitro-extension/resources/models/llava-7b/model.json diff --git a/models/miqu-70b/model.json b/extensions/inference-nitro-extension/resources/models/miqu-70b/model.json similarity index 100% rename from models/miqu-70b/model.json rename to extensions/inference-nitro-extension/resources/models/miqu-70b/model.json diff --git a/models/mistral-ins-7b-q4/model.json b/extensions/inference-nitro-extension/resources/models/mistral-ins-7b-q4/model.json similarity index 100% rename from models/mistral-ins-7b-q4/model.json rename to extensions/inference-nitro-extension/resources/models/mistral-ins-7b-q4/model.json diff --git a/models/mixtral-8x7b-instruct/model.json b/extensions/inference-nitro-extension/resources/models/mixtral-8x7b-instruct/model.json similarity index 100% rename from models/mixtral-8x7b-instruct/model.json rename to extensions/inference-nitro-extension/resources/models/mixtral-8x7b-instruct/model.json diff --git a/models/noromaid-7b/model.json b/extensions/inference-nitro-extension/resources/models/noromaid-7b/model.json similarity index 100% rename from models/noromaid-7b/model.json rename to extensions/inference-nitro-extension/resources/models/noromaid-7b/model.json diff --git a/models/openchat-3.5-7b/model.json b/extensions/inference-nitro-extension/resources/models/openchat-3.5-7b/model.json similarity index 100% rename from models/openchat-3.5-7b/model.json rename to extensions/inference-nitro-extension/resources/models/openchat-3.5-7b/model.json diff --git a/models/openhermes-neural-7b/model.json b/extensions/inference-nitro-extension/resources/models/openhermes-neural-7b/model.json similarity index 100% rename from models/openhermes-neural-7b/model.json rename to extensions/inference-nitro-extension/resources/models/openhermes-neural-7b/model.json diff --git a/models/phind-34b/model.json b/extensions/inference-nitro-extension/resources/models/phind-34b/model.json similarity index 100% rename from models/phind-34b/model.json rename to extensions/inference-nitro-extension/resources/models/phind-34b/model.json diff --git a/models/qwen-7b/model.json b/extensions/inference-nitro-extension/resources/models/qwen-7b/model.json similarity index 100% rename from models/qwen-7b/model.json rename to extensions/inference-nitro-extension/resources/models/qwen-7b/model.json diff --git a/models/stable-zephyr-3b/model.json b/extensions/inference-nitro-extension/resources/models/stable-zephyr-3b/model.json similarity index 100% rename from models/stable-zephyr-3b/model.json rename to extensions/inference-nitro-extension/resources/models/stable-zephyr-3b/model.json diff --git a/models/stealth-v1.2-7b/model.json b/extensions/inference-nitro-extension/resources/models/stealth-v1.2-7b/model.json similarity index 100% rename from models/stealth-v1.2-7b/model.json rename to extensions/inference-nitro-extension/resources/models/stealth-v1.2-7b/model.json diff --git a/models/tinyllama-1.1b/model.json b/extensions/inference-nitro-extension/resources/models/tinyllama-1.1b/model.json similarity index 100% rename from models/tinyllama-1.1b/model.json rename to extensions/inference-nitro-extension/resources/models/tinyllama-1.1b/model.json diff --git a/models/trinity-v1.2-7b/model.json b/extensions/inference-nitro-extension/resources/models/trinity-v1.2-7b/model.json similarity index 100% rename from models/trinity-v1.2-7b/model.json rename to extensions/inference-nitro-extension/resources/models/trinity-v1.2-7b/model.json diff --git a/models/vistral-7b/model.json b/extensions/inference-nitro-extension/resources/models/vistral-7b/model.json similarity index 100% rename from models/vistral-7b/model.json rename to extensions/inference-nitro-extension/resources/models/vistral-7b/model.json diff --git a/models/wizardcoder-13b/model.json b/extensions/inference-nitro-extension/resources/models/wizardcoder-13b/model.json similarity index 100% rename from models/wizardcoder-13b/model.json rename to extensions/inference-nitro-extension/resources/models/wizardcoder-13b/model.json diff --git a/models/yi-34b/model.json b/extensions/inference-nitro-extension/resources/models/yi-34b/model.json similarity index 100% rename from models/yi-34b/model.json rename to extensions/inference-nitro-extension/resources/models/yi-34b/model.json diff --git a/extensions/inference-nitro-extension/rollup.config.ts b/extensions/inference-nitro-extension/rollup.config.ts index 396c40d08..7b2758881 100644 --- a/extensions/inference-nitro-extension/rollup.config.ts +++ b/extensions/inference-nitro-extension/rollup.config.ts @@ -5,13 +5,42 @@ import typescript from 'rollup-plugin-typescript2' import json from '@rollup/plugin-json' import replace from '@rollup/plugin-replace' const packageJson = require('./package.json') +const defaultSettingJson = require('./resources/default_settings.json') -const pkg = require('./package.json') +const bakllavaJson = require('./resources/models/bakllava-1/model.json') +const codeninja7bJson = require('./resources/models/codeninja-1.0-7b/model.json') +const commandr34bJson = require('./resources/models/command-r-34b/model.json') +const deepseekCoder13bJson = require('./resources/models/deepseek-coder-1.3b/model.json') +const deepseekCoder34bJson = require('./resources/models/deepseek-coder-34b/model.json') +const dolphinPhi2Json = require('./resources/models/dolphin-phi-2/model.json') +const gemma2bJson = require('./resources/models/gemma-2b/model.json') +const gemma7bJson = require('./resources/models/gemma-7b/model.json') +const hermesPro7bJson = require('./resources/models/hermes-pro-7b/model.json') +const llama2Chat70bJson = require('./resources/models/llama2-chat-70b/model.json') +const llama2Chat7bJson = require('./resources/models/llama2-chat-7b/model.json') +const llamacorn1bJson = require('./resources/models/llamacorn-1.1b/model.json') +const llava13bJson = require('./resources/models/llava-13b/model.json') +const llava7bJson = require('./resources/models/llava-7b/model.json') +const miqu70bJson = require('./resources/models/miqu-70b/model.json') +const mistralIns7bq4Json = require('./resources/models/mistral-ins-7b-q4/model.json') +const mixtral8x7bInstructJson = require('./resources/models/mixtral-8x7b-instruct/model.json') +const noromaid7bJson = require('./resources/models/noromaid-7b/model.json') +const openchat357bJson = require('./resources/models/openchat-3.5-7b/model.json') +const openhermesNeural7bJson = require('./resources/models/openhermes-neural-7b/model.json') +const phind34bJson = require('./resources/models/phind-34b/model.json') +const qwen7bJson = require('./resources/models/qwen-7b/model.json') +const stableZephyr3bJson = require('./resources/models/stable-zephyr-3b/model.json') +const stealthv127bJson = require('./resources/models/stealth-v1.2-7b/model.json') +const tinyllama11bJson = require('./resources/models/tinyllama-1.1b/model.json') +const trinityv127bJson = require('./resources/models/trinity-v1.2-7b/model.json') +const vistral7bJson = require('./resources/models/vistral-7b/model.json') +const wizardcoder13bJson = require('./resources/models/wizardcoder-13b/model.json') +const yi34bJson = require('./resources/models/yi-34b/model.json') export default [ { input: `src/index.ts`, - output: [{ file: pkg.main, format: 'es', sourcemap: true }], + output: [{ file: packageJson.main, format: 'es', sourcemap: true }], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') external: [], watch: { @@ -19,7 +48,40 @@ export default [ }, plugins: [ replace({ + preventAssignment: true, + MODELS: JSON.stringify([ + bakllavaJson, + codeninja7bJson, + commandr34bJson, + deepseekCoder13bJson, + deepseekCoder34bJson, + dolphinPhi2Json, + gemma2bJson, + gemma7bJson, + hermesPro7bJson, + llama2Chat70bJson, + llama2Chat7bJson, + llamacorn1bJson, + llava13bJson, + llava7bJson, + miqu70bJson, + mistralIns7bq4Json, + mixtral8x7bInstructJson, + noromaid7bJson, + openchat357bJson, + openhermesNeural7bJson, + phind34bJson, + qwen7bJson, + stableZephyr3bJson, + stealthv127bJson, + tinyllama11bJson, + trinityv127bJson, + vistral7bJson, + wizardcoder13bJson, + yi34bJson, + ]), NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`), + DEFAULT_SETTINGS: JSON.stringify(defaultSettingJson), INFERENCE_URL: JSON.stringify( process.env.INFERENCE_URL || 'http://127.0.0.1:3928/inferences/llamacpp/chat_completion' @@ -30,6 +92,9 @@ export default [ JAN_SERVER_INFERENCE_URL: JSON.stringify( 'http://localhost:1337/v1/chat/completions' ), + CUDA_DOWNLOAD_URL: JSON.stringify( + 'https://catalog.jan.ai/dist/cuda-dependencies///cuda.tar.gz' + ), }), // Allow json resolution json(), diff --git a/extensions/inference-nitro-extension/src/@types/global.d.ts b/extensions/inference-nitro-extension/src/@types/global.d.ts index 3a3d2aa32..85c9b939f 100644 --- a/extensions/inference-nitro-extension/src/@types/global.d.ts +++ b/extensions/inference-nitro-extension/src/@types/global.d.ts @@ -2,6 +2,8 @@ declare const NODE: string declare const INFERENCE_URL: string declare const TROUBLESHOOTING_URL: string declare const JAN_SERVER_INFERENCE_URL: string +declare const DEFAULT_SETTINGS: Array +declare const MODELS: Array /** * The response from the initModel function. diff --git a/extensions/inference-nitro-extension/src/index.ts b/extensions/inference-nitro-extension/src/index.ts index 313b67365..119d7762f 100644 --- a/extensions/inference-nitro-extension/src/index.ts +++ b/extensions/inference-nitro-extension/src/index.ts @@ -12,8 +12,19 @@ import { Model, ModelEvent, LocalOAIEngine, + InstallationState, + systemInformation, + fs, + getJanDataFolderPath, + joinPath, + DownloadRequest, + baseName, + downloadFile, + DownloadState, + DownloadEvent, } from '@janhq/core' +declare const CUDA_DOWNLOAD_URL: string /** * A class that implements the InferenceExtension interface from the @janhq/core package. * The class provides methods for initializing and stopping a model, and for making inference requests. @@ -23,10 +34,6 @@ export default class JanInferenceNitroExtension extends LocalOAIEngine { nodeModule: string = NODE provider: string = 'nitro' - models(): Promise { - return Promise.resolve([]) - } - /** * Checking the health for Nitro's process each 5 secs. */ @@ -58,14 +65,18 @@ export default class JanInferenceNitroExtension extends LocalOAIEngine { this.inferenceUrl = `${window.core?.api?.baseApiUrl}/v1/chat/completions` } - console.debug('Inference url: ', this.inferenceUrl) - this.getNitroProcesHealthIntervalId = setInterval( () => this.periodicallyGetNitroHealth(), JanInferenceNitroExtension._intervalHealthCheck ) - + const models = MODELS as unknown as Model[] + this.registerModels(models) super.onLoad() + + executeOnMain(NODE, 'addAdditionalDependencies', { + name: this.name, + version: this.version, + }) } /** @@ -101,4 +112,80 @@ export default class JanInferenceNitroExtension extends LocalOAIEngine { } return super.unloadModel(model) } + + override async install(): Promise { + const info = await systemInformation() + + const platform = info.osInfo?.platform === 'win32' ? 'windows' : 'linux' + const downloadUrl = CUDA_DOWNLOAD_URL + + const url = downloadUrl + .replace('', info.gpuSetting?.cuda?.version ?? '12.4') + .replace('', platform) + + console.debug('Downloading Cuda Toolkit Dependency: ', url) + + const janDataFolderPath = await getJanDataFolderPath() + + const executableFolderPath = await joinPath([ + janDataFolderPath, + 'engines', + this.name ?? 'nitro', + this.version ?? '1.0.0', + ]) + + if (!(await fs.existsSync(executableFolderPath))) { + await fs.mkdir(executableFolderPath) + } + + const tarball = await baseName(url) + const tarballFullPath = await joinPath([executableFolderPath, tarball]) + + const downloadRequest: DownloadRequest = { + url, + localPath: tarballFullPath, + extensionId: this.name, + downloadType: 'extension', + } + downloadFile(downloadRequest) + + const onFileDownloadSuccess = async (state: DownloadState) => { + console.log(state) + // if other download, ignore + if (state.fileName !== tarball) return + events.off(DownloadEvent.onFileDownloadSuccess, onFileDownloadSuccess) + await executeOnMain( + NODE, + 'decompressRunner', + tarballFullPath, + executableFolderPath + ) + events.emit(DownloadEvent.onFileUnzipSuccess, state) + } + events.on(DownloadEvent.onFileDownloadSuccess, onFileDownloadSuccess) + } + + override async installationState(): Promise { + const info = await systemInformation() + if ( + info.gpuSetting?.run_mode === 'gpu' && + !info.gpuSetting?.vulkan && + info.osInfo && + info.osInfo.platform !== 'darwin' && + !info.gpuSetting?.cuda?.exist + ) { + const janDataFolderPath = await getJanDataFolderPath() + + const executableFolderPath = await joinPath([ + janDataFolderPath, + 'engines', + this.name ?? 'nitro', + this.version ?? '1.0.0', + ]) + + if (!(await fs.existsSync(executableFolderPath))) return 'NotInstalled' + return 'Installed' + } + return 'NotRequired' + } } diff --git a/extensions/inference-nitro-extension/src/node/execute.test.ts b/extensions/inference-nitro-extension/src/node/execute.test.ts index 62ffdc707..dfd26deb8 100644 --- a/extensions/inference-nitro-extension/src/node/execute.test.ts +++ b/extensions/inference-nitro-extension/src/node/execute.test.ts @@ -29,32 +29,13 @@ describe('test executable nitro file', () => { }) }) - it('executes on MacOS ARM', () => { + it('executes on MacOS', () => { Object.defineProperty(process, 'platform', { value: 'darwin', }) - Object.defineProperty(process, 'arch', { - value: 'arm64', - }) expect(executableNitroFile(testSettings)).toEqual( expect.objectContaining({ - executablePath: expect.stringContaining(`mac-arm64${sep}nitro`), - cudaVisibleDevices: '', - vkVisibleDevices: '', - }) - ) - }) - - it('executes on MacOS Intel', () => { - Object.defineProperty(process, 'platform', { - value: 'darwin', - }) - Object.defineProperty(process, 'arch', { - value: 'x64', - }) - expect(executableNitroFile(testSettings)).toEqual( - expect.objectContaining({ - executablePath: expect.stringContaining(`mac-x64${sep}nitro`), + executablePath: expect.stringContaining(`mac-universal${sep}nitro`), cudaVisibleDevices: '', vkVisibleDevices: '', }) diff --git a/extensions/inference-nitro-extension/src/node/execute.ts b/extensions/inference-nitro-extension/src/node/execute.ts index c9d541654..2cfcfe4f3 100644 --- a/extensions/inference-nitro-extension/src/node/execute.ts +++ b/extensions/inference-nitro-extension/src/node/execute.ts @@ -8,8 +8,8 @@ export interface NitroExecutableOptions { } const runMode = (settings?: GpuSetting): string => { if (process.platform === 'darwin') - // MacOS use arch instead of cpu / cuda - return process.arch === 'arm64' ? 'arm64' : 'x64' + // MacOS now has universal binaries + return '' if (!settings) return 'cpu' @@ -24,7 +24,7 @@ const os = (): string => { return process.platform === 'win32' ? 'win' : process.platform === 'darwin' - ? 'mac' + ? 'mac-universal' : 'linux' } diff --git a/extensions/inference-nitro-extension/src/node/index.ts b/extensions/inference-nitro-extension/src/node/index.ts index 71adac72d..3d742721e 100644 --- a/extensions/inference-nitro-extension/src/node/index.ts +++ b/extensions/inference-nitro-extension/src/node/index.ts @@ -11,8 +11,11 @@ import { ModelSettingParams, PromptTemplate, SystemInformation, + getJanDataFolderPath, } from '@janhq/core/node' import { executableNitroFile } from './execute' +import terminate from 'terminate' +import decompress from 'decompress' // Polyfill fetch with retry const fetchRetry = fetchRT(fetch) @@ -37,6 +40,8 @@ const NITRO_HTTP_VALIDATE_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llama // The URL for the Nitro subprocess to kill itself const NITRO_HTTP_KILL_URL = `${NITRO_HTTP_SERVER_URL}/processmanager/destroy` +const NITRO_PORT_FREE_CHECK_INTERVAL = 100 + // The supported model format // TODO: Should be an array to support more models const SUPPORTED_MODEL_FORMAT = '.gguf' @@ -149,19 +154,9 @@ async function loadModel( async function runNitroAndLoadModel(systemInfo?: SystemInformation) { // Gather system information for CPU physical cores and memory return killSubprocess() - .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) - .then(() => { - /** - * There is a problem with Windows process manager - * Should wait for awhile to make sure the port is free and subprocess is killed - * The tested threshold is 500ms - **/ - if (process.platform === 'win32') { - return new Promise((resolve) => setTimeout(resolve, 500)) - } else { - return Promise.resolve() - } - }) + .then(() => + tcpPortUsed.waitUntilFree(PORT, NITRO_PORT_FREE_CHECK_INTERVAL, 5000) + ) .then(() => spawnNitroProcess(systemInfo)) .then(() => loadLLMModel(currentSettings)) .then(validateModelStatus) @@ -234,7 +229,7 @@ function loadLLMModel(settings: any): Promise { }, body: JSON.stringify(settings), retries: 3, - retryDelay: 500, + retryDelay: 300, }) .then((res) => { log( @@ -265,7 +260,7 @@ async function validateModelStatus(): Promise { 'Content-Type': 'application/json', }, retries: 5, - retryDelay: 500, + retryDelay: 300, }).then(async (res: Response) => { log( `[NITRO]::Debug: Validate model state with response ${JSON.stringify( @@ -304,23 +299,45 @@ async function killSubprocess(): Promise { setTimeout(() => controller.abort(), 5000) log(`[NITRO]::Debug: Request to kill Nitro`) - return fetch(NITRO_HTTP_KILL_URL, { - method: 'DELETE', - signal: controller.signal, - }) - .then(() => { - subprocess?.kill() - subprocess = undefined + const killRequest = () => { + return fetch(NITRO_HTTP_KILL_URL, { + method: 'DELETE', + signal: controller.signal, }) - .catch(() => {}) // Do nothing with this attempt - .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) - .then(() => log(`[NITRO]::Debug: Nitro process is terminated`)) - .catch((err) => { - log( - `[NITRO]::Debug: Could not kill running process on port ${PORT}. Might be another process running on the same port? ${err}` + .catch(() => {}) // Do nothing with this attempt + .then(() => + tcpPortUsed.waitUntilFree(PORT, NITRO_PORT_FREE_CHECK_INTERVAL, 5000) ) - throw 'PORT_NOT_AVAILABLE' + .then(() => log(`[NITRO]::Debug: Nitro process is terminated`)) + .catch((err) => { + log( + `[NITRO]::Debug: Could not kill running process on port ${PORT}. Might be another process running on the same port? ${err}` + ) + throw 'PORT_NOT_AVAILABLE' + }) + } + + if (subprocess?.pid) { + log(`[NITRO]::Debug: Killing PID ${subprocess.pid}`) + const pid = subprocess.pid + return new Promise((resolve, reject) => { + terminate(pid, function (err) { + if (err) { + return killRequest() + } else { + return tcpPortUsed + .waitUntilFree(PORT, NITRO_PORT_FREE_CHECK_INTERVAL, 5000) + .then(() => resolve()) + .then(() => log(`[NITRO]::Debug: Nitro process is terminated`)) + .catch(() => { + killRequest() + }) + } + }) }) + } else { + return killRequest() + } } /** @@ -370,10 +387,12 @@ function spawnNitroProcess(systemInfo?: SystemInformation): Promise { reject(`child process exited with code ${code}`) }) - tcpPortUsed.waitUntilUsed(PORT, 300, 30000).then(() => { - log(`[NITRO]::Debug: Nitro is ready`) - resolve() - }) + tcpPortUsed + .waitUntilUsed(PORT, NITRO_PORT_FREE_CHECK_INTERVAL, 30000) + .then(() => { + log(`[NITRO]::Debug: Nitro is ready`) + resolve() + }) }) } @@ -403,9 +422,32 @@ const getCurrentNitroProcessInfo = (): NitroProcessInfo => { } } +const addAdditionalDependencies = (data: { name: string; version: string }) => { + const additionalPath = path.delimiter.concat( + path.join(getJanDataFolderPath(), 'engines', data.name, data.version) + ) + // Set the updated PATH + process.env.PATH = (process.env.PATH || '').concat(additionalPath) + process.env.LD_LIBRARY_PATH = (process.env.LD_LIBRARY_PATH || '').concat( + additionalPath + ) +} + +const decompressRunner = async (zipPath: string, output: string) => { + console.debug(`Decompressing ${zipPath} to ${output}...`) + try { + const files = await decompress(zipPath, output) + console.debug('Decompress finished!', files) + } catch (err) { + console.error(`Decompress ${zipPath} failed: ${err}`) + } +} + export default { loadModel, unloadModel, dispose, getCurrentNitroProcessInfo, + addAdditionalDependencies, + decompressRunner, } diff --git a/extensions/inference-openai-extension/package.json b/extensions/inference-openai-extension/package.json index 7b112a8fb..3435a5346 100644 --- a/extensions/inference-openai-extension/package.json +++ b/extensions/inference-openai-extension/package.json @@ -1,5 +1,6 @@ { "name": "@janhq/inference-openai-extension", + "productName": "OpenAI Inference Engine Extension", "version": "1.0.0", "description": "This extension enables OpenAI chat completion API calls", "main": "dist/index.js", diff --git a/extensions/inference-openai-extension/resources/models.json b/extensions/inference-openai-extension/resources/models.json new file mode 100644 index 000000000..b7c5885c6 --- /dev/null +++ b/extensions/inference-openai-extension/resources/models.json @@ -0,0 +1,97 @@ +[ + { + "sources": [ + { + "url": "https://openai.com" + } + ], + "id": "gpt-4", + "object": "model", + "name": "OpenAI GPT 4", + "version": "1.0", + "description": "OpenAI GPT 4 model is extremely good", + "format": "api", + "settings": {}, + "parameters": { + "max_tokens": 4096, + "temperature": 0.7 + }, + "metadata": { + "author": "OpenAI", + "tags": ["General", "Big Context Length"] + }, + "engine": "openai" + }, + { + "sources": [ + { + "url": "https://openai.com" + } + ], + "id": "gpt-4-vision-preview", + "object": "model", + "name": "OpenAI GPT 4 with Vision (Preview)", + "version": "1.0", + "description": "OpenAI GPT 4 with Vision model is extremely good in preview", + "format": "api", + "settings": { + "vision_model": true, + "textModel": false + }, + "parameters": { + "max_tokens": 4096, + "temperature": 0.7 + }, + "metadata": { + "author": "OpenAI", + "tags": ["General", "Big Context Length", "Vision"] + }, + "engine": "openai" + }, + { + "sources": [ + { + "url": "https://openai.com" + } + ], + "id": "gpt-3.5-turbo-16k-0613", + "object": "model", + "name": "OpenAI GPT 3.5 Turbo 16k 0613", + "version": "1.0", + "description": "OpenAI GPT 3.5 Turbo 16k 0613 model is extremely good", + "format": "api", + "settings": {}, + "parameters": { + "max_tokens": 4096, + "temperature": 0.7 + }, + "metadata": { + "author": "OpenAI", + "tags": ["General", "Big Context Length"] + }, + "engine": "openai" + }, + { + "sources": [ + { + "url": "https://openai.com" + } + ], + "id": "gpt-3.5-turbo", + "object": "model", + "name": "OpenAI GPT 3.5 Turbo", + "version": "1.0", + "description": "OpenAI GPT 3.5 Turbo model is extremely good", + "format": "api", + "settings": {}, + "parameters": { + "max_tokens": 4096, + "temperature": 0.7 + }, + "metadata": { + "author": "OpenAI", + "tags": ["General", "Big Context Length"] + }, + "engine": "openai" + } +] diff --git a/extensions/inference-openai-extension/resources/settings.json b/extensions/inference-openai-extension/resources/settings.json new file mode 100644 index 000000000..ccd7dd545 --- /dev/null +++ b/extensions/inference-openai-extension/resources/settings.json @@ -0,0 +1,23 @@ +[ + { + "key": "chat-completions-endpoint", + "title": "Chat Completions Endpoint", + "description": "The endpoint to use for chat completions. See the [OpenAI API documentation](https://platform.openai.com/docs/api-reference/chat/create) for more information.", + "controllerType": "input", + "controllerProps": { + "placeholder": "https://api.openai.com/v1/chat/completions", + "value": "https://api.openai.com/v1/chat/completions" + } + }, + { + "key": "openai-api-key", + "title": "API Key", + "description": "The OpenAI API uses API keys for authentication. Visit your [API Keys](https://platform.openai.com/account/api-keys) page to retrieve the API key you'll use in your requests.", + "controllerType": "input", + "controllerProps": { + "placeholder": "sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "value": "", + "type": "password" + } + } +] diff --git a/extensions/inference-openai-extension/src/index.ts b/extensions/inference-openai-extension/src/index.ts index eb60540fa..60446ccce 100644 --- a/extensions/inference-openai-extension/src/index.ts +++ b/extensions/inference-openai-extension/src/index.ts @@ -5,18 +5,16 @@ * @version 1.0.0 * @module inference-openai-extension/src/index */ -declare const ENGINE: string -import { - events, - fs, - AppConfigurationEventName, - joinPath, - RemoteOAIEngine, -} from '@janhq/core' -import { join } from 'path' +import { RemoteOAIEngine, SettingComponentProps } from '@janhq/core' -declare const COMPLETION_URL: string +declare const SETTINGS: Array +declare const MODELS: Array + +enum Settings { + apiKey = 'openai-api-key', + chatCompletionsEndPoint = 'chat-completions-endpoint', +} /** * A class that implements the InferenceExtension interface from the @janhq/core package. @@ -24,66 +22,45 @@ declare const COMPLETION_URL: string * It also subscribes to events emitted by the @janhq/core package and handles new message requests. */ export default class JanInferenceOpenAIExtension extends RemoteOAIEngine { - private static readonly _engineDir = 'file://engines' - private static readonly _engineMetadataFileName = `${ENGINE}.json` - - private _engineSettings = { - full_url: COMPLETION_URL, - api_key: 'sk-', - } - - inferenceUrl: string = COMPLETION_URL + inferenceUrl: string = '' provider: string = 'openai' - apiKey: string = '' - // TODO: Just use registerSettings from BaseExtension - // Remove these methods - /** - * Subscribes to events emitted by the @janhq/core package. - */ - async onLoad() { + override async onLoad(): Promise { super.onLoad() - if (!(await fs.existsSync(JanInferenceOpenAIExtension._engineDir))) { - await fs.mkdir(JanInferenceOpenAIExtension._engineDir) - } + // Register Settings + this.registerSettings(SETTINGS) + this.registerModels(MODELS) - this.writeDefaultEngineSettings() - - const settingsFilePath = await joinPath([ - JanInferenceOpenAIExtension._engineDir, - JanInferenceOpenAIExtension._engineMetadataFileName, - ]) - - events.on( - AppConfigurationEventName.OnConfigurationUpdate, - (settingsKey: string) => { - // Update settings on changes - if (settingsKey === settingsFilePath) this.writeDefaultEngineSettings() - } + this.apiKey = await this.getSetting(Settings.apiKey, '') + this.inferenceUrl = await this.getSetting( + Settings.chatCompletionsEndPoint, + '' ) + if (this.inferenceUrl.length === 0) { + SETTINGS.forEach((setting) => { + if (setting.key === Settings.chatCompletionsEndPoint) { + this.inferenceUrl = setting.controllerProps.value as string + } + }) + } } - async writeDefaultEngineSettings() { - try { - const engineFile = join( - JanInferenceOpenAIExtension._engineDir, - JanInferenceOpenAIExtension._engineMetadataFileName - ) - if (await fs.existsSync(engineFile)) { - const engine = await fs.readFileSync(engineFile, 'utf-8') - this._engineSettings = - typeof engine === 'object' ? engine : JSON.parse(engine) - this.inferenceUrl = this._engineSettings.full_url - this.apiKey = this._engineSettings.api_key + onSettingUpdate(key: string, value: T): void { + if (key === Settings.apiKey) { + this.apiKey = value as string + } else if (key === Settings.chatCompletionsEndPoint) { + if (typeof value !== 'string') return + + if (value.trim().length === 0) { + SETTINGS.forEach((setting) => { + if (setting.key === Settings.chatCompletionsEndPoint) { + this.inferenceUrl = setting.controllerProps.value as string + } + }) } else { - await fs.writeFileSync( - engineFile, - JSON.stringify(this._engineSettings, null, 2) - ) + this.inferenceUrl = value } - } catch (err) { - console.error(err) } } } diff --git a/extensions/inference-openai-extension/webpack.config.js b/extensions/inference-openai-extension/webpack.config.js index ee18035f2..f3d0db183 100644 --- a/extensions/inference-openai-extension/webpack.config.js +++ b/extensions/inference-openai-extension/webpack.config.js @@ -1,6 +1,8 @@ const path = require('path') const webpack = require('webpack') const packageJson = require('./package.json') +const settingJson = require('./resources/settings.json') +const modelsJson = require('./resources/models.json') module.exports = { experiments: { outputModule: true }, @@ -17,8 +19,9 @@ module.exports = { }, plugins: [ new webpack.DefinePlugin({ + MODELS: JSON.stringify(modelsJson), + SETTINGS: JSON.stringify(settingJson), ENGINE: JSON.stringify(packageJson.engine), - COMPLETION_URL: JSON.stringify('https://api.openai.com/v1/chat/completions'), }), ], output: { diff --git a/extensions/inference-triton-trtllm-extension/README.md b/extensions/inference-triton-trtllm-extension/README.md index 455783efb..f9690da09 100644 --- a/extensions/inference-triton-trtllm-extension/README.md +++ b/extensions/inference-triton-trtllm-extension/README.md @@ -1,14 +1,10 @@ -# Jan inference plugin +# Create a Jan Extension using Typescript -Created using Jan app example +Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀 -# Create a Jan Plugin using Typescript +## Create Your Own Extension -Use this template to bootstrap the creation of a TypeScript Jan plugin. 🚀 - -## Create Your Own Plugin - -To create your own plugin, you can use this repository as a template! Just follow the below instructions: +To create your own extension, you can use this repository as a template! Just follow the below instructions: 1. Click the Use this template button at the top of the repository 2. Select Create a new repository @@ -18,7 +14,7 @@ To create your own plugin, you can use this repository as a template! Just follo ## Initial Setup -After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your plugin. +After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension. > [!NOTE] > @@ -43,36 +39,37 @@ After you've cloned the repository to your local machine or codespace, you'll ne 1. :white_check_mark: Check your artifact - There will be a tgz file in your plugin directory now + There will be a tgz file in your extension directory now -## Update the Plugin Metadata +## Update the Extension Metadata -The [`package.json`](package.json) file defines metadata about your plugin, such as -plugin name, main entry, description and version. +The [`package.json`](package.json) file defines metadata about your extension, such as +extension name, main entry, description and version. -When you copy this repository, update `package.json` with the name, description for your plugin. +When you copy this repository, update `package.json` with the name, description for your extension. -## Update the Plugin Code +## Update the Extension Code -The [`src/`](./src/) directory is the heart of your plugin! This contains the -source code that will be run when your plugin extension functions are invoked. You can replace the +The [`src/`](./src/) directory is the heart of your extension! This contains the +source code that will be run when your extension functions are invoked. You can replace the contents of this directory with your own code. -There are a few things to keep in mind when writing your plugin code: +There are a few things to keep in mind when writing your extension code: -- Most Jan Plugin Extension functions are processed asynchronously. +- Most Jan Extension functions are processed asynchronously. In `index.ts`, you will see that the extension function will return a `Promise`. ```typescript - import { core } from "@janhq/core"; + import { events, MessageEvent, MessageRequest } from '@janhq/core' function onStart(): Promise { - return core.invokePluginFunc(MODULE_PATH, "run", 0); + return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => + this.inference(data) + ) } ``` - For more information about the Jan Plugin Core module, see the + For more information about the Jan Extension Core module, see the [documentation](https://github.com/janhq/jan/blob/main/core/README.md). -So, what are you waiting for? Go ahead and start customizing your plugin! - +So, what are you waiting for? Go ahead and start customizing your extension! diff --git a/extensions/inference-triton-trtllm-extension/package.json b/extensions/inference-triton-trtllm-extension/package.json index bb97f15c5..9ce8f11a9 100644 --- a/extensions/inference-triton-trtllm-extension/package.json +++ b/extensions/inference-triton-trtllm-extension/package.json @@ -1,5 +1,6 @@ { "name": "@janhq/inference-triton-trt-llm-extension", + "productName": "Triton-TRT-LLM Inference Engine Extension", "version": "1.0.0", "description": "This extension enables Nvidia's TensorRT-LLM as an inference engine option", "main": "dist/index.js", diff --git a/extensions/inference-triton-trtllm-extension/resources/settings.json b/extensions/inference-triton-trtllm-extension/resources/settings.json new file mode 100644 index 000000000..9c220eed7 --- /dev/null +++ b/extensions/inference-triton-trtllm-extension/resources/settings.json @@ -0,0 +1,23 @@ +[ + { + "key": "chat-completions-endpoint", + "title": "Chat Completions Endpoint", + "description": "The endpoint to use for chat completions.", + "controllerType": "input", + "controllerProps": { + "placeholder": "http://localhost:8000/v2/models/tensorrt_llm_bls/generate", + "value": "http://localhost:8000/v2/models/tensorrt_llm_bls/generate" + } + }, + { + "key": "tritonllm-api-key", + "title": "Triton LLM API Key", + "description": "The Triton LLM API uses API keys for authentication.", + "controllerType": "input", + "controllerProps": { + "placeholder": "xxxxxxxxxxxxxxxxxxxx", + "value": "", + "type": "password" + } + } +] diff --git a/extensions/inference-triton-trtllm-extension/src/index.ts b/extensions/inference-triton-trtllm-extension/src/index.ts index 6f9a01d9b..be34837ac 100644 --- a/extensions/inference-triton-trtllm-extension/src/index.ts +++ b/extensions/inference-triton-trtllm-extension/src/index.ts @@ -6,77 +6,62 @@ * @module inference-nvidia-triton-trt-llm-extension/src/index */ -import { - AppConfigurationEventName, - events, - fs, - joinPath, - Model, - RemoteOAIEngine, -} from '@janhq/core' -import { join } from 'path' +import { RemoteOAIEngine, SettingComponentProps } from '@janhq/core' +declare const SETTINGS: Array +enum Settings { + apiKey = 'tritonllm-api-key', + chatCompletionsEndPoint = 'chat-completions-endpoint', +} /** * A class that implements the InferenceExtension interface from the @janhq/core package. * The class provides methods for initializing and stopping a model, and for making inference requests. * It also subscribes to events emitted by the @janhq/core package and handles new message requests. */ export default class JanInferenceTritonTrtLLMExtension extends RemoteOAIEngine { - private readonly _engineDir = 'file://engines' - private readonly _engineMetadataFileName = 'triton_trtllm.json' - inferenceUrl: string = '' provider: string = 'triton_trtllm' - apiKey: string = '' - - _engineSettings: { - base_url: '' - api_key: '' - } /** * Subscribes to events emitted by the @janhq/core package. */ async onLoad() { super.onLoad() - if (!(await fs.existsSync(this._engineDir))) { - await fs.mkdir(this._engineDir) - } - this.writeDefaultEngineSettings() + // Register Settings + this.registerSettings(SETTINGS) - const settingsFilePath = await joinPath([ - this._engineDir, - this._engineMetadataFileName, - ]) - - // Events subscription - events.on( - AppConfigurationEventName.OnConfigurationUpdate, - (settingsKey: string) => { - // Update settings on changes - if (settingsKey === settingsFilePath) this.writeDefaultEngineSettings() - } + // Retrieve API Key Setting + this.apiKey = await this.getSetting(Settings.apiKey, '') + this.inferenceUrl = await this.getSetting( + Settings.chatCompletionsEndPoint, + '' ) + + if (this.inferenceUrl.length === 0) { + SETTINGS.forEach((setting) => { + if (setting.key === Settings.chatCompletionsEndPoint) { + this.inferenceUrl = setting.controllerProps.value as string + } + }) + } } - async writeDefaultEngineSettings() { - try { - const engine_json = join(this._engineDir, this._engineMetadataFileName) - if (await fs.existsSync(engine_json)) { - const engine = await fs.readFileSync(engine_json, 'utf-8') - this._engineSettings = - typeof engine === 'object' ? engine : JSON.parse(engine) - this.inferenceUrl = this._engineSettings.base_url - this.apiKey = this._engineSettings.api_key + onSettingUpdate(key: string, value: T): void { + if (key === Settings.apiKey) { + this.apiKey = value as string + } else if (key === Settings.chatCompletionsEndPoint) { + if (typeof value !== 'string') return + + if (value.trim().length === 0) { + SETTINGS.forEach((setting) => { + if (setting.key === Settings.chatCompletionsEndPoint) { + this.inferenceUrl = setting.controllerProps.value as string + } + }) } else { - await fs.writeFileSync( - engine_json, - JSON.stringify(this._engineSettings, null, 2) - ) + this.inferenceUrl = value } - } catch (err) { - console.error(err) } } } diff --git a/extensions/inference-triton-trtllm-extension/webpack.config.js b/extensions/inference-triton-trtllm-extension/webpack.config.js index e83370a1a..13d32c52d 100644 --- a/extensions/inference-triton-trtllm-extension/webpack.config.js +++ b/extensions/inference-triton-trtllm-extension/webpack.config.js @@ -1,6 +1,7 @@ const path = require('path') const webpack = require('webpack') const packageJson = require('./package.json') +const settingJson = require('./resources/settings.json') module.exports = { experiments: { outputModule: true }, @@ -17,6 +18,7 @@ module.exports = { }, plugins: [ new webpack.DefinePlugin({ + SETTINGS: JSON.stringify(settingJson), MODULE: JSON.stringify(`${packageJson.name}/${packageJson.module}`), }), ], diff --git a/extensions/model-extension/README.md b/extensions/model-extension/README.md index 516bbec8b..f9690da09 100644 --- a/extensions/model-extension/README.md +++ b/extensions/model-extension/README.md @@ -1,14 +1,10 @@ -# Jan Model Management plugin +# Create a Jan Extension using Typescript -Created using Jan app example +Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀 -# Create a Jan Plugin using Typescript +## Create Your Own Extension -Use this template to bootstrap the creation of a TypeScript Jan plugin. 🚀 - -## Create Your Own Plugin - -To create your own plugin, you can use this repository as a template! Just follow the below instructions: +To create your own extension, you can use this repository as a template! Just follow the below instructions: 1. Click the Use this template button at the top of the repository 2. Select Create a new repository @@ -18,7 +14,7 @@ To create your own plugin, you can use this repository as a template! Just follo ## Initial Setup -After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your plugin. +After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension. > [!NOTE] > @@ -43,36 +39,37 @@ After you've cloned the repository to your local machine or codespace, you'll ne 1. :white_check_mark: Check your artifact - There will be a tgz file in your plugin directory now + There will be a tgz file in your extension directory now -## Update the Plugin Metadata +## Update the Extension Metadata -The [`package.json`](package.json) file defines metadata about your plugin, such as -plugin name, main entry, description and version. +The [`package.json`](package.json) file defines metadata about your extension, such as +extension name, main entry, description and version. -When you copy this repository, update `package.json` with the name, description for your plugin. +When you copy this repository, update `package.json` with the name, description for your extension. -## Update the Plugin Code +## Update the Extension Code -The [`src/`](./src/) directory is the heart of your plugin! This contains the -source code that will be run when your plugin extension functions are invoked. You can replace the +The [`src/`](./src/) directory is the heart of your extension! This contains the +source code that will be run when your extension functions are invoked. You can replace the contents of this directory with your own code. -There are a few things to keep in mind when writing your plugin code: +There are a few things to keep in mind when writing your extension code: -- Most Jan Plugin Extension functions are processed asynchronously. +- Most Jan Extension functions are processed asynchronously. In `index.ts`, you will see that the extension function will return a `Promise`. ```typescript - import { core } from "@janhq/core"; + import { events, MessageEvent, MessageRequest } from '@janhq/core' function onStart(): Promise { - return core.invokePluginFunc(MODULE_PATH, "run", 0); + return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => + this.inference(data) + ) } ``` - For more information about the Jan Plugin Core module, see the + For more information about the Jan Extension Core module, see the [documentation](https://github.com/janhq/jan/blob/main/core/README.md). -So, what are you waiting for? Go ahead and start customizing your plugin! - +So, what are you waiting for? Go ahead and start customizing your extension! diff --git a/extensions/model-extension/package.json b/extensions/model-extension/package.json index 5f53ff513..c6b659322 100644 --- a/extensions/model-extension/package.json +++ b/extensions/model-extension/package.json @@ -1,5 +1,6 @@ { "name": "@janhq/model-extension", + "productName": "Model Management Extension", "version": "1.0.30", "description": "Model Management Extension provides model exploration and seamless downloads", "main": "dist/index.js", diff --git a/models/config/default-model.json b/extensions/model-extension/resources/default-model.json similarity index 97% rename from models/config/default-model.json rename to extensions/model-extension/resources/default-model.json index 2fa2df2ee..f887a2b7a 100644 --- a/models/config/default-model.json +++ b/extensions/model-extension/resources/default-model.json @@ -1,6 +1,6 @@ { "object": "model", - "version": 1, + "version": "1.0", "format": "gguf", "sources": [ { diff --git a/extensions/model-extension/rollup.config.ts b/extensions/model-extension/rollup.config.ts index 722785aa3..256b33add 100644 --- a/extensions/model-extension/rollup.config.ts +++ b/extensions/model-extension/rollup.config.ts @@ -1,18 +1,16 @@ import resolve from '@rollup/plugin-node-resolve' -import commonjs from '@rollup/plugin-commonjs' import sourceMaps from 'rollup-plugin-sourcemaps' import typescript from 'rollup-plugin-typescript2' import json from '@rollup/plugin-json' import replace from '@rollup/plugin-replace' const packageJson = require('./package.json') - -const pkg = require('./package.json') +const defaultModelJson = require('./resources/default-model.json') export default [ { input: `src/index.ts`, - output: [{ file: pkg.main, format: 'es', sourcemap: true }], + output: [{ file: packageJson.main, format: 'es', sourcemap: true }], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') external: [], watch: { @@ -20,7 +18,8 @@ export default [ }, plugins: [ replace({ - EXTENSION_NAME: JSON.stringify(packageJson.name), + preventAssignment: true, + DEFAULT_MODEL: JSON.stringify(defaultModelJson), MODULE_PATH: JSON.stringify( `${packageJson.name}/${packageJson.module}` ), diff --git a/extensions/model-extension/src/@types/global.d.ts b/extensions/model-extension/src/@types/global.d.ts index 7a9202a62..a72b5188e 100644 --- a/extensions/model-extension/src/@types/global.d.ts +++ b/extensions/model-extension/src/@types/global.d.ts @@ -1,6 +1,6 @@ export {} declare global { - declare const EXTENSION_NAME: string + declare const DEFAULT_MODEL: object declare const MODULE_PATH: string declare const VERSION: string diff --git a/extensions/model-extension/src/index.ts b/extensions/model-extension/src/index.ts index d56e2e824..9dd106868 100644 --- a/extensions/model-extension/src/index.ts +++ b/extensions/model-extension/src/index.ts @@ -2,7 +2,6 @@ import { fs, downloadFile, abortDownload, - getResourcePath, InferenceEngine, joinPath, ModelExtension, @@ -11,7 +10,6 @@ import { events, DownloadEvent, DownloadRoute, - ModelEvent, DownloadState, OptionType, ImportingModel, @@ -36,8 +34,6 @@ export default class JanModelExtension extends ModelExtension { InferenceEngine.nitro_tensorrt_llm, ] private static readonly _tensorRtEngineFormat = '.engine' - private static readonly _configDirName = 'config' - private static readonly _defaultModelFileName = 'default-model.json' private static readonly _supportedGpuArch = ['ampere', 'ada'] /** @@ -45,7 +41,6 @@ export default class JanModelExtension extends ModelExtension { * @override */ async onLoad() { - this.copyModelsToHomeDir() // Handle Desktop Events this.handleDesktopEvents() } @@ -56,37 +51,6 @@ export default class JanModelExtension extends ModelExtension { */ onUnload(): void {} - private async copyModelsToHomeDir() { - try { - // Check for migration conditions - if ( - localStorage.getItem(`${EXTENSION_NAME}-version`) === VERSION && - (await fs.existsSync(JanModelExtension._homeDir)) - ) { - // ignore if the there is no need to migrate - console.debug('Models already persisted.') - return - } - // copy models folder from resources to home directory - const resourePath = await getResourcePath() - const srcPath = await joinPath([resourePath, 'models']) - - const janDataFolderPath = await getJanDataFolderPath() - const destPath = await joinPath([janDataFolderPath, 'models']) - - await fs.syncFile(srcPath, destPath) - - console.debug('Finished syncing models') - - // Finished migration - localStorage.setItem(`${EXTENSION_NAME}-version`, VERSION) - - events.emit(ModelEvent.OnModelsUpdate, {}) - } catch (err) { - console.error(err) - } - } - /** * Downloads a machine learning model. * @param model - The model to download. @@ -121,8 +85,8 @@ export default class JanModelExtension extends ModelExtension { } if (!JanModelExtension._supportedGpuArch.includes(gpuArch)) { - console.error( - `Your GPU: ${firstGpu} is not supported. Only 20xx, 30xx, 40xx series are supported.` + console.debug( + `Your GPU: ${JSON.stringify(firstGpu)} is not supported. Only 30xx, 40xx series are supported.` ) return } @@ -489,20 +453,9 @@ export default class JanModelExtension extends ModelExtension { return model } - private async getDefaultModel(): Promise { - const defaultModelPath = await joinPath([ - JanModelExtension._homeDir, - JanModelExtension._configDirName, - JanModelExtension._defaultModelFileName, - ]) - - if (!(await fs.existsSync(defaultModelPath))) { - return undefined - } - - const model = await this.readModelMetadata(defaultModelPath) - - return typeof model === 'object' ? model : JSON.parse(model) + private async getDefaultModel(): Promise { + const defaultModel = DEFAULT_MODEL as Model + return defaultModel } /** diff --git a/extensions/monitoring-extension/README.md b/extensions/monitoring-extension/README.md index 1617b9b13..f9690da09 100644 --- a/extensions/monitoring-extension/README.md +++ b/extensions/monitoring-extension/README.md @@ -1,14 +1,10 @@ -# Jan Monitoring plugin +# Create a Jan Extension using Typescript -Created using Jan app example +Use this template to bootstrap the creation of a TypeScript Jan extension. 🚀 -# Create a Jan Plugin using Typescript +## Create Your Own Extension -Use this template to bootstrap the creation of a TypeScript Jan plugin. 🚀 - -## Create Your Own Plugin - -To create your own plugin, you can use this repository as a template! Just follow the below instructions: +To create your own extension, you can use this repository as a template! Just follow the below instructions: 1. Click the Use this template button at the top of the repository 2. Select Create a new repository @@ -18,7 +14,7 @@ To create your own plugin, you can use this repository as a template! Just follo ## Initial Setup -After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your plugin. +After you've cloned the repository to your local machine or codespace, you'll need to perform some initial setup steps before you can develop your extension. > [!NOTE] > @@ -43,36 +39,37 @@ After you've cloned the repository to your local machine or codespace, you'll ne 1. :white_check_mark: Check your artifact - There will be a tgz file in your plugin directory now + There will be a tgz file in your extension directory now -## Update the Plugin Metadata +## Update the Extension Metadata -The [`package.json`](package.json) file defines metadata about your plugin, such as -plugin name, main entry, description and version. +The [`package.json`](package.json) file defines metadata about your extension, such as +extension name, main entry, description and version. -When you copy this repository, update `package.json` with the name, description for your plugin. +When you copy this repository, update `package.json` with the name, description for your extension. -## Update the Plugin Code +## Update the Extension Code -The [`src/`](./src/) directory is the heart of your plugin! This contains the -source code that will be run when your plugin extension functions are invoked. You can replace the +The [`src/`](./src/) directory is the heart of your extension! This contains the +source code that will be run when your extension functions are invoked. You can replace the contents of this directory with your own code. -There are a few things to keep in mind when writing your plugin code: +There are a few things to keep in mind when writing your extension code: -- Most Jan Plugin Extension functions are processed asynchronously. +- Most Jan Extension functions are processed asynchronously. In `index.ts`, you will see that the extension function will return a `Promise`. ```typescript - import { core } from "@janhq/core"; + import { events, MessageEvent, MessageRequest } from '@janhq/core' function onStart(): Promise { - return core.invokePluginFunc(MODULE_PATH, "run", 0); + return events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => + this.inference(data) + ) } ``` - For more information about the Jan Plugin Core module, see the + For more information about the Jan Extension Core module, see the [documentation](https://github.com/janhq/jan/blob/main/core/README.md). -So, what are you waiting for? Go ahead and start customizing your plugin! - +So, what are you waiting for? Go ahead and start customizing your extension! diff --git a/extensions/monitoring-extension/download.bat b/extensions/monitoring-extension/download.bat index f1cf8b7ea..14e0aadd9 100644 --- a/extensions/monitoring-extension/download.bat +++ b/extensions/monitoring-extension/download.bat @@ -1,2 +1,2 @@ @echo off -.\node_modules\.bin\download https://delta.jan.ai/vulkaninfoSDK.exe -o ./bin \ No newline at end of file +.\node_modules\.bin\download https://catalog.jan.ai/vulkaninfoSDK.exe -o ./bin \ No newline at end of file diff --git a/extensions/monitoring-extension/package.json b/extensions/monitoring-extension/package.json index 0268fb11c..c320db2ba 100644 --- a/extensions/monitoring-extension/package.json +++ b/extensions/monitoring-extension/package.json @@ -1,5 +1,6 @@ { "name": "@janhq/monitoring-extension", + "productName": "System Monitoring Extension", "version": "1.0.10", "description": "This extension provides system health and OS level data", "main": "dist/index.js", @@ -11,7 +12,7 @@ "download-artifacts": "run-script-os && cpx \"bin/**\" \"dist/bin\"", "download-artifacts:darwin": "echo 'No artifacts to download for darwin'", "download-artifacts:win32": "download.bat", - "download-artifacts:linux": "download https://delta.jan.ai/vulkaninfo -o ./bin && chmod +x ./bin/vulkaninfo", + "download-artifacts:linux": "download https://catalog.jan.ai/vulkaninfo -o ./bin && chmod +x ./bin/vulkaninfo", "build:publish": "rimraf *.tgz --glob && yarn build && npm pack && cpx *.tgz ../../pre-install" }, "exports": { diff --git a/extensions/monitoring-extension/resources/settings.json b/extensions/monitoring-extension/resources/settings.json new file mode 100644 index 000000000..fbdaf309a --- /dev/null +++ b/extensions/monitoring-extension/resources/settings.json @@ -0,0 +1,21 @@ +[ + { + "key": "log-enabled", + "title": "App Logging Enabled", + "description": "We recommend enabling this setting to help us improve the app. Your data will be kept private on your computer, and you can opt out at any time.", + "controllerType": "checkbox", + "controllerProps": { + "value": true + } + }, + { + "key": "log-cleaning-interval", + "title": "Log Cleaning Interval", + "description": "Log cleaning interval in milliseconds.", + "controllerType": "input", + "controllerProps": { + "value": "120000", + "placeholder": "Interval in milliseconds. E.g. 120000" + } + } +] diff --git a/extensions/monitoring-extension/rollup.config.ts b/extensions/monitoring-extension/rollup.config.ts index 1b7a40bad..b054d6291 100644 --- a/extensions/monitoring-extension/rollup.config.ts +++ b/extensions/monitoring-extension/rollup.config.ts @@ -4,6 +4,7 @@ import sourceMaps from 'rollup-plugin-sourcemaps' import typescript from 'rollup-plugin-typescript2' import json from '@rollup/plugin-json' import replace from '@rollup/plugin-replace' +const settingJson = require('./resources/settings.json') const packageJson = require('./package.json') export default [ @@ -17,7 +18,9 @@ export default [ }, plugins: [ replace({ + preventAssignment: true, NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`), + SETTINGS: JSON.stringify(settingJson), }), // Allow json resolution json(), diff --git a/extensions/monitoring-extension/src/index.ts b/extensions/monitoring-extension/src/index.ts index 7ef40e7be..166627fd3 100644 --- a/extensions/monitoring-extension/src/index.ts +++ b/extensions/monitoring-extension/src/index.ts @@ -1,10 +1,17 @@ import { GpuSetting, MonitoringExtension, + MonitoringInterface, OperatingSystemInfo, executeOnMain, } from '@janhq/core' +declare const SETTINGS: Array + +enum Settings { + logEnabled = 'log-enabled', + logCleaningInterval = 'log-cleaning-interval', +} /** * JanMonitoringExtension is a extension that provides system monitoring functionality. * It implements the MonitoringExtension interface from the @janhq/core package. @@ -14,14 +21,40 @@ export default class JanMonitoringExtension extends MonitoringExtension { * Called when the extension is loaded. */ async onLoad() { + // Register extension settings + this.registerSettings(SETTINGS) + + const logEnabled = await this.getSetting(Settings.logEnabled, true) + const logCleaningInterval = parseInt( + await this.getSetting(Settings.logCleaningInterval, '120000') + ) + // Register File Logger provided by this extension + await executeOnMain(NODE, 'registerLogger', { + logEnabled, + logCleaningInterval: isNaN(logCleaningInterval) + ? 120000 + : logCleaningInterval, + }) + // Attempt to fetch nvidia info await executeOnMain(NODE, 'updateNvidiaInfo') } + onSettingUpdate(key: string, value: T): void { + if (key === Settings.logEnabled) { + executeOnMain(NODE, 'updateLogger', { logEnabled: value }) + } else if (key === Settings.logCleaningInterval) { + executeOnMain(NODE, 'updateLogger', { logCleaningInterval: value }) + } + } + /** * Called when the extension is unloaded. */ - onUnload(): void {} + onUnload(): void { + // Register File Logger provided by this extension + executeOnMain(NODE, 'unregisterLogger') + } /** * Returns the GPU configuration. @@ -47,6 +80,10 @@ export default class JanMonitoringExtension extends MonitoringExtension { return executeOnMain(NODE, 'getCurrentLoad') } + /** + * Returns information about the OS + * @returns + */ getOsInfo(): Promise { return executeOnMain(NODE, 'getOsInfo') } diff --git a/extensions/monitoring-extension/src/node/index.ts b/extensions/monitoring-extension/src/node/index.ts index 17b56dbfd..bb0c4ac18 100644 --- a/extensions/monitoring-extension/src/node/index.ts +++ b/extensions/monitoring-extension/src/node/index.ts @@ -1,6 +1,7 @@ import { GpuSetting, GpuSettingInfo, + LoggerManager, OperatingSystemInfo, ResourceInfo, SupportedPlatforms, @@ -12,6 +13,7 @@ import { exec } from 'child_process' import { writeFileSync, existsSync, readFileSync, mkdirSync } from 'fs' import path from 'path' import os from 'os' +import { FileLogger } from './logger' /** * Path to the settings directory @@ -198,7 +200,7 @@ const updateGpuInfo = async () => process.platform === 'win32' ? `${__dirname}\\..\\bin\\vulkaninfoSDK.exe --summary` : `${__dirname}/../bin/vulkaninfo --summary`, - (error, stdout) => { + async (error, stdout) => { if (!error) { const output = stdout.toString() @@ -219,8 +221,9 @@ const updateGpuInfo = async () => data.gpus_in_use = [data.gpus.length > 1 ? '1' : '0'] } - data = updateCudaExistence(data) + data = await updateCudaExistence(data) writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) + log(`[APP]::${JSON.stringify(data)}`) resolve({}) } else { reject(error) @@ -230,7 +233,7 @@ const updateGpuInfo = async () => } else { exec( 'nvidia-smi --query-gpu=index,memory.total,name --format=csv,noheader,nounits', - (error, stdout) => { + async (error, stdout) => { if (!error) { log(`[SPECS]::${stdout}`) // Get GPU info and gpu has higher memory first @@ -261,8 +264,10 @@ const updateGpuInfo = async () => data.gpus_in_use = [data.gpu_highest_vram] } - data = updateCudaExistence(data) + data = await updateCudaExistence(data) + console.log(data) writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) + log(`[APP]::${JSON.stringify(data)}`) resolve({}) } ) @@ -279,9 +284,9 @@ const checkFileExistenceInPaths = (file: string, paths: string[]): boolean => { /** * Validate cuda for linux and windows */ -const updateCudaExistence = ( +const updateCudaExistence = async ( data: GpuSetting = DEFAULT_SETTINGS -): GpuSetting => { +): Promise => { let filesCuda12: string[] let filesCuda11: string[] let paths: string[] @@ -289,7 +294,7 @@ const updateCudaExistence = ( if (process.platform === 'win32') { filesCuda12 = ['cublas64_12.dll', 'cudart64_12.dll', 'cublasLt64_12.dll'] - filesCuda11 = ['cublas64_11.dll', 'cudart64_11.dll', 'cublasLt64_11.dll'] + filesCuda11 = ['cublas64_11.dll', 'cudart64_110.dll', 'cublasLt64_11.dll'] paths = process.env.PATH ? process.env.PATH.split(path.delimiter) : [] } else { filesCuda12 = ['libcudart.so.12', 'libcublas.so.12', 'libcublasLt.so.12'] @@ -325,6 +330,23 @@ const updateCudaExistence = ( } data.is_initial = false + + // Attempt to query CUDA using NVIDIA SMI + if (!cudaExists) { + await new Promise((resolve, reject) => { + exec('nvidia-smi', (error, stdout) => { + if (!error) { + const regex = /CUDA\s*Version:\s*(\d+\.\d+)/g + const match = regex.exec(stdout) + if (match && match[1]) { + data.cuda.version = match[1] + } + } + console.log(data) + resolve() + }) + }) + } return data } @@ -344,3 +366,22 @@ export const getOsInfo = (): OperatingSystemInfo => { return osInfo } + +export const registerLogger = ({ logEnabled, logCleaningInterval }) => { + const logger = new FileLogger(logEnabled, logCleaningInterval) + LoggerManager.instance().register(logger) + logger.cleanLogs() +} + +export const unregisterLogger = () => { + LoggerManager.instance().unregister('file') +} + +export const updateLogger = ({ logEnabled, logCleaningInterval }) => { + const logger = LoggerManager.instance().loggers.get('file') as FileLogger + if (logger && logEnabled !== undefined) logger.logEnabled = logEnabled + if (logger && logCleaningInterval) + logger.logCleaningInterval = logCleaningInterval + // Rerun + logger && logger.cleanLogs() +} diff --git a/extensions/monitoring-extension/src/node/logger.ts b/extensions/monitoring-extension/src/node/logger.ts new file mode 100644 index 000000000..3d53e5ed9 --- /dev/null +++ b/extensions/monitoring-extension/src/node/logger.ts @@ -0,0 +1,138 @@ +import fs from 'fs' +import util from 'util' +import { + getAppConfigurations, + getJanDataFolderPath, + Logger, +} from '@janhq/core/node' +import path, { join } from 'path' + +export class FileLogger extends Logger { + name = 'file' + logCleaningInterval: number = 120000 + timeout: NodeJS.Timeout | null = null + appLogPath: string = './' + logEnabled: boolean = true + + constructor( + logEnabled: boolean = true, + logCleaningInterval: number = 120000 + ) { + super() + this.logEnabled = logEnabled + if (logCleaningInterval) this.logCleaningInterval = logCleaningInterval + + const appConfigurations = getAppConfigurations() + const logFolderPath = join(appConfigurations.data_folder, 'logs') + if (!fs.existsSync(logFolderPath)) { + fs.mkdirSync(logFolderPath, { recursive: true }) + } + + this.appLogPath = join(logFolderPath, 'app.log') + } + + log(args: any) { + if (!this.logEnabled) return + let message = args[0] + const scope = args[1] + if (!message) return + const path = this.appLogPath + if (!scope && !message.startsWith('[')) { + message = `[APP]::${message}` + } else if (scope) { + message = `${scope}::${message}` + } + + message = `${new Date().toISOString()} ${message}` + + writeLog(message, path) + } + + cleanLogs( + maxFileSizeBytes?: number | undefined, + daysToKeep?: number | undefined + ): void { + // clear existing timeout + // incase we rerun it with different values + if (this.timeout) clearTimeout(this.timeout) + this.timeout = undefined + + if (!this.logEnabled) return + + console.log( + 'Validating app logs. Next attempt in ', + this.logCleaningInterval + ) + + const size = maxFileSizeBytes ?? 1 * 1024 * 1024 // 1 MB + const days = daysToKeep ?? 7 // 7 days + const logDirectory = path.join(getJanDataFolderPath(), 'logs') + + // Perform log cleaning + const currentDate = new Date() + fs.readdir(logDirectory, (err, files) => { + if (err) { + console.error('Error reading log directory:', err) + return + } + + files.forEach((file) => { + const filePath = path.join(logDirectory, file) + fs.stat(filePath, (err, stats) => { + if (err) { + console.error('Error getting file stats:', err) + return + } + + // Check size + if (stats.size > size) { + fs.unlink(filePath, (err) => { + if (err) { + console.error('Error deleting log file:', err) + return + } + console.debug( + `Deleted log file due to exceeding size limit: ${filePath}` + ) + }) + } else { + // Check age + const creationDate = new Date(stats.ctime) + const daysDifference = Math.floor( + (currentDate.getTime() - creationDate.getTime()) / + (1000 * 3600 * 24) + ) + if (daysDifference > days) { + fs.unlink(filePath, (err) => { + if (err) { + console.error('Error deleting log file:', err) + return + } + console.debug(`Deleted old log file: ${filePath}`) + }) + } + } + }) + }) + }) + + // Schedule the next execution with doubled delays + this.timeout = setTimeout( + () => this.cleanLogs(maxFileSizeBytes, daysToKeep), + this.logCleaningInterval + ) + } +} + +const writeLog = (message: string, logPath: string) => { + if (!fs.existsSync(logPath)) { + fs.writeFileSync(logPath, message) + } else { + const logFile = fs.createWriteStream(logPath, { + flags: 'a', + }) + logFile.write(util.format(message) + '\n') + logFile.close() + console.debug(message) + } +} diff --git a/extensions/tensorrt-llm-extension/package.json b/extensions/tensorrt-llm-extension/package.json index d1cb93f58..02b0b4e8c 100644 --- a/extensions/tensorrt-llm-extension/package.json +++ b/extensions/tensorrt-llm-extension/package.json @@ -1,5 +1,6 @@ { "name": "@janhq/tensorrt-llm-extension", + "productName": "TensorRT-LLM Inference Engine Extension", "version": "0.0.3", "description": "This extension enables Nvidia's TensorRT-LLM for the fastest GPU acceleration. See the [setup guide](https://jan.ai/guides/providers/tensorrt-llm/) for next steps.", "main": "dist/index.js", @@ -36,10 +37,10 @@ "@rollup/plugin-json": "^6.1.0", "@rollup/plugin-node-resolve": "^15.2.3", "@rollup/plugin-replace": "^5.0.5", + "@types/decompress": "4.2.7", "@types/node": "^20.11.4", "@types/os-utils": "^0.0.4", "@types/tcp-port-used": "^1.0.4", - "@types/decompress": "4.2.7", "cpx": "^1.5.0", "download-cli": "^1.1.1", "rimraf": "^3.0.2", @@ -57,6 +58,7 @@ "path-browserify": "^1.0.1", "rxjs": "^7.8.1", "tcp-port-used": "^1.0.2", + "terminate": "^2.6.1", "ulidx": "^2.3.0" }, "engines": { @@ -71,6 +73,7 @@ "tcp-port-used", "fetch-retry", "decompress", - "@janhq/core" + "@janhq/core", + "terminate" ] } diff --git a/extensions/tensorrt-llm-extension/models.json b/extensions/tensorrt-llm-extension/resources/models.json similarity index 52% rename from extensions/tensorrt-llm-extension/models.json rename to extensions/tensorrt-llm-extension/resources/models.json index a27cf059d..387b71104 100644 --- a/extensions/tensorrt-llm-extension/models.json +++ b/extensions/tensorrt-llm-extension/resources/models.json @@ -3,31 +3,31 @@ "sources": [ { "filename": "config.json", - "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/LlamaCorn-1.1B-Chat-fp16/config.json" + "url": "https://catalog.jan.ai/dist/models///tensorrt-llm-v0.7.1/LlamaCorn-1.1B-Chat-fp16/config.json" }, { "filename": "mistral_float16_tp1_rank0.engine", - "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/LlamaCorn-1.1B-Chat-fp16/mistral_float16_tp1_rank0.engine" + "url": "https://catalog.jan.ai/dist/models///tensorrt-llm-v0.7.1/LlamaCorn-1.1B-Chat-fp16/mistral_float16_tp1_rank0.engine" }, { "filename": "tokenizer.model", - "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/LlamaCorn-1.1B-Chat-fp16/tokenizer.model" + "url": "https://catalog.jan.ai/dist/models///tensorrt-llm-v0.7.1/LlamaCorn-1.1B-Chat-fp16/tokenizer.model" }, { "filename": "special_tokens_map.json", - "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/LlamaCorn-1.1B-Chat-fp16/special_tokens_map.json" + "url": "https://catalog.jan.ai/dist/models///tensorrt-llm-v0.7.1/LlamaCorn-1.1B-Chat-fp16/special_tokens_map.json" }, { "filename": "tokenizer.json", - "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/LlamaCorn-1.1B-Chat-fp16/tokenizer.json" + "url": "https://catalog.jan.ai/dist/models///tensorrt-llm-v0.7.1/LlamaCorn-1.1B-Chat-fp16/tokenizer.json" }, { "filename": "tokenizer_config.json", - "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/LlamaCorn-1.1B-Chat-fp16/tokenizer_config.json" + "url": "https://catalog.jan.ai/dist/models///tensorrt-llm-v0.7.1/LlamaCorn-1.1B-Chat-fp16/tokenizer_config.json" }, { "filename": "model.cache", - "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/LlamaCorn-1.1B-Chat-fp16/model.cache" + "url": "https://catalog.jan.ai/dist/models///tensorrt-llm-v0.7.1/LlamaCorn-1.1B-Chat-fp16/model.cache" } ], "id": "llamacorn-1.1b-chat-fp16", @@ -54,31 +54,31 @@ "sources": [ { "filename": "config.json", - "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/TinyJensen-1.1B-Chat-fp16/config.json" + "url": "https://catalog.jan.ai/dist/models///tensorrt-llm-v0.7.1/TinyJensen-1.1B-Chat-fp16/config.json" }, { "filename": "mistral_float16_tp1_rank0.engine", - "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/TinyJensen-1.1B-Chat-fp16/mistral_float16_tp1_rank0.engine" + "url": "https://catalog.jan.ai/dist/models///tensorrt-llm-v0.7.1/TinyJensen-1.1B-Chat-fp16/mistral_float16_tp1_rank0.engine" }, { "filename": "tokenizer.model", - "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/TinyJensen-1.1B-Chat-fp16/tokenizer.model" + "url": "https://catalog.jan.ai/dist/models///tensorrt-llm-v0.7.1/TinyJensen-1.1B-Chat-fp16/tokenizer.model" }, { "filename": "special_tokens_map.json", - "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/TinyJensen-1.1B-Chat-fp16/special_tokens_map.json" + "url": "https://catalog.jan.ai/dist/models///tensorrt-llm-v0.7.1/TinyJensen-1.1B-Chat-fp16/special_tokens_map.json" }, { "filename": "tokenizer.json", - "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/TinyJensen-1.1B-Chat-fp16/tokenizer.json" + "url": "https://catalog.jan.ai/dist/models///tensorrt-llm-v0.7.1/TinyJensen-1.1B-Chat-fp16/tokenizer.json" }, { "filename": "tokenizer_config.json", - "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/TinyJensen-1.1B-Chat-fp16/tokenizer_config.json" + "url": "https://catalog.jan.ai/dist/models///tensorrt-llm-v0.7.1/TinyJensen-1.1B-Chat-fp16/tokenizer_config.json" }, { "filename": "model.cache", - "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/TinyJensen-1.1B-Chat-fp16/model.cache" + "url": "https://catalog.jan.ai/dist/models///tensorrt-llm-v0.7.1/TinyJensen-1.1B-Chat-fp16/model.cache" } ], "id": "tinyjensen-1.1b-chat-fp16", @@ -105,31 +105,31 @@ "sources": [ { "filename": "config.json", - "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/Mistral-7B-Instruct-v0.1-int4/config.json" + "url": "https://catalog.jan.ai/dist/models///tensorrt-llm-v0.7.1/Mistral-7B-Instruct-v0.1-int4/config.json" }, { "filename": "mistral_float16_tp1_rank0.engine", - "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/Mistral-7B-Instruct-v0.1-int4/mistral_float16_tp1_rank0.engine" + "url": "https://catalog.jan.ai/dist/models///tensorrt-llm-v0.7.1/Mistral-7B-Instruct-v0.1-int4/mistral_float16_tp1_rank0.engine" }, { "filename": "tokenizer.model", - "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/Mistral-7B-Instruct-v0.1-int4/tokenizer.model" + "url": "https://catalog.jan.ai/dist/models///tensorrt-llm-v0.7.1/Mistral-7B-Instruct-v0.1-int4/tokenizer.model" }, { "filename": "special_tokens_map.json", - "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/Mistral-7B-Instruct-v0.1-int4/special_tokens_map.json" + "url": "https://catalog.jan.ai/dist/models///tensorrt-llm-v0.7.1/Mistral-7B-Instruct-v0.1-int4/special_tokens_map.json" }, { "filename": "tokenizer.json", - "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/Mistral-7B-Instruct-v0.1-int4/tokenizer.json" + "url": "https://catalog.jan.ai/dist/models///tensorrt-llm-v0.7.1/Mistral-7B-Instruct-v0.1-int4/tokenizer.json" }, { "filename": "tokenizer_config.json", - "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/Mistral-7B-Instruct-v0.1-int4/tokenizer_config.json" + "url": "https://catalog.jan.ai/dist/models///tensorrt-llm-v0.7.1/Mistral-7B-Instruct-v0.1-int4/tokenizer_config.json" }, { "filename": "model.cache", - "url": "https://delta.jan.ai/dist/models///tensorrt-llm-v0.7.1/Mistral-7B-Instruct-v0.1-int4/model.cache" + "url": "https://catalog.jan.ai/dist/models///tensorrt-llm-v0.7.1/Mistral-7B-Instruct-v0.1-int4/model.cache" } ], "id": "mistral-7b-instruct-int4", diff --git a/extensions/tensorrt-llm-extension/rollup.config.ts b/extensions/tensorrt-llm-extension/rollup.config.ts index e602bc720..1fad0e711 100644 --- a/extensions/tensorrt-llm-extension/rollup.config.ts +++ b/extensions/tensorrt-llm-extension/rollup.config.ts @@ -5,6 +5,7 @@ import typescript from 'rollup-plugin-typescript2' import json from '@rollup/plugin-json' import replace from '@rollup/plugin-replace' const packageJson = require('./package.json') +const modelsJson = require('./resources/models.json') export default [ { @@ -15,7 +16,8 @@ export default [ }, plugins: [ replace({ - EXTENSION_NAME: JSON.stringify(packageJson.name), + preventAssignment: true, + MODELS: JSON.stringify(modelsJson), TENSORRT_VERSION: JSON.stringify(packageJson.tensorrtVersion), PROVIDER: JSON.stringify(packageJson.provider), DOWNLOAD_RUNNER_URL: @@ -53,7 +55,7 @@ export default [ }, plugins: [ replace({ - EXTENSION_NAME: JSON.stringify(packageJson.name), + preventAssignment: true, TENSORRT_VERSION: JSON.stringify(packageJson.tensorrtVersion), PROVIDER: JSON.stringify(packageJson.provider), LOAD_MODEL_URL: JSON.stringify( diff --git a/extensions/tensorrt-llm-extension/src/@types/global.d.ts b/extensions/tensorrt-llm-extension/src/@types/global.d.ts index 9cf5b6090..b550080f7 100644 --- a/extensions/tensorrt-llm-extension/src/@types/global.d.ts +++ b/extensions/tensorrt-llm-extension/src/@types/global.d.ts @@ -7,5 +7,5 @@ declare const ENGINE_PORT: string declare const DOWNLOAD_RUNNER_URL: string declare const TENSORRT_VERSION: string declare const COMPATIBILITY: object -declare const EXTENSION_NAME: string declare const PROVIDER: string +declare const MODELS: Array diff --git a/extensions/tensorrt-llm-extension/src/index.ts b/extensions/tensorrt-llm-extension/src/index.ts index ce89beca2..189abc706 100644 --- a/extensions/tensorrt-llm-extension/src/index.ts +++ b/extensions/tensorrt-llm-extension/src/index.ts @@ -22,8 +22,8 @@ import { MessageRequest, ModelEvent, getJanDataFolderPath, + SystemInformation, } from '@janhq/core' -import models from '../models.json' /** * TensorRTLLMExtension - Implementation of LocalOAIEngine @@ -41,52 +41,26 @@ export default class TensorRTLLMExtension extends LocalOAIEngine { private supportedGpuArch = ['ampere', 'ada'] private supportedPlatform = ['win32', 'linux'] - private isUpdateAvailable = false override compatibility() { return COMPATIBILITY as unknown as Compatibility } - /** - * models implemented by the extension - * define pre-populated models - */ - override async models(): Promise { - if ((await this.installationState()) === 'Installed') - return models as unknown as Model[] - return [] + + override async onLoad(): Promise { + super.onLoad() + + if ((await this.installationState()) === 'Installed') { + const models = MODELS as unknown as Model[] + this.registerModels(models) + } } override async install(): Promise { await this.removePopulatedModels() const info = await systemInformation() - console.debug( - `TensorRTLLMExtension installing pre-requisites... ${JSON.stringify(info)}` - ) - const gpuSetting: GpuSetting | undefined = info.gpuSetting - if (gpuSetting === undefined || gpuSetting.gpus.length === 0) { - console.error('No GPU setting found. Please check your GPU setting.') - return - } - // TODO: we only check for the first graphics card. Need to refactor this later. - const firstGpu = gpuSetting.gpus[0] - if (!firstGpu.name.toLowerCase().includes('nvidia')) { - console.error('No Nvidia GPU found. Please check your GPU setting.') - return - } - - if (firstGpu.arch === undefined) { - console.error('No GPU architecture found. Please check your GPU setting.') - return - } - - if (!this.supportedGpuArch.includes(firstGpu.arch)) { - console.error( - `Your GPU: ${firstGpu} is not supported. Only 20xx, 30xx, 40xx series are supported.` - ) - return - } + if (!this.isCompatible(info)) return const janDataFolderPath = await getJanDataFolderPath() const engineVersion = TENSORRT_VERSION @@ -96,7 +70,7 @@ export default class TensorRTLLMExtension extends LocalOAIEngine { 'engines', this.provider, engineVersion, - firstGpu.arch, + info.gpuSetting?.gpus[0].arch, ]) if (!(await fs.existsSync(executableFolderPath))) { @@ -108,7 +82,7 @@ export default class TensorRTLLMExtension extends LocalOAIEngine { const url = placeholderUrl .replace(//g, tensorrtVersion) - .replace(//g, firstGpu.arch) + .replace(//g, info.gpuSetting!.gpus[0]!.arch!) const tarball = await baseName(url) @@ -116,7 +90,7 @@ export default class TensorRTLLMExtension extends LocalOAIEngine { const downloadRequest: DownloadRequest = { url, localPath: tarballFullPath, - extensionId: EXTENSION_NAME, + extensionId: this.name, downloadType: 'extension', } downloadFile(downloadRequest) @@ -134,7 +108,8 @@ export default class TensorRTLLMExtension extends LocalOAIEngine { events.emit(DownloadEvent.onFileUnzipSuccess, state) // Prepopulate models as soon as it's ready - this.prePopulateModels().then(() => { + const models = MODELS as unknown as Model[] + this.registerModels(models).then(() => { showToast( 'Extension installed successfully.', 'New models are added to Model Hub.' @@ -144,7 +119,8 @@ export default class TensorRTLLMExtension extends LocalOAIEngine { events.on(DownloadEvent.onFileDownloadSuccess, onFileDownloadSuccess) } - async removePopulatedModels(): Promise { + private async removePopulatedModels(): Promise { + const models = MODELS as unknown as Model[] console.debug(`removePopulatedModels`, JSON.stringify(models)) const janDataFolderPath = await getJanDataFolderPath() const modelFolderPath = await joinPath([janDataFolderPath, 'models']) @@ -162,70 +138,17 @@ export default class TensorRTLLMExtension extends LocalOAIEngine { } override async loadModel(model: Model): Promise { - if (model.engine !== this.provider) return - if ((await this.installationState()) === 'Installed') return super.loadModel(model) - else { - events.emit(ModelEvent.OnModelFail, { - ...model, - error: { - message: 'EXTENSION_IS_NOT_INSTALLED::TensorRT-LLM extension', - }, - }) - } - } - override updatable() { - return this.isUpdateAvailable + throw new Error('EXTENSION_IS_NOT_INSTALLED::TensorRT-LLM extension') } override async installationState(): Promise { const info = await systemInformation() - const gpuSetting: GpuSetting | undefined = info.gpuSetting - if (gpuSetting === undefined) { - console.warn( - 'No GPU setting found. TensorRT-LLM extension is not installed' - ) - return 'NotInstalled' // TODO: maybe disabled / incompatible is more appropriate - } - - if (gpuSetting.gpus.length === 0) { - console.warn('No GPU found. TensorRT-LLM extension is not installed') - return 'NotInstalled' - } - - const firstGpu = gpuSetting.gpus[0] - if (!firstGpu.name.toLowerCase().includes('nvidia')) { - console.error('No Nvidia GPU found. Please check your GPU setting.') - return 'NotInstalled' - } - - if (firstGpu.arch === undefined) { - console.error('No GPU architecture found. Please check your GPU setting.') - return 'NotInstalled' - } - - if (!this.supportedGpuArch.includes(firstGpu.arch)) { - console.error( - `Your GPU: ${firstGpu} is not supported. Only 20xx, 30xx, 40xx series are supported.` - ) - return 'NotInstalled' - } - - const osInfo = info.osInfo - if (!osInfo) { - console.error('No OS information found. Please check your OS setting.') - return 'NotInstalled' - } - - if (!this.supportedPlatform.includes(osInfo.platform)) { - console.error( - `Your OS: ${osInfo.platform} is not supported. Only Windows and Linux are supported.` - ) - return 'NotInstalled' - } + if (!this.isCompatible(info)) return 'NotCompatible' + const firstGpu = info.gpuSetting?.gpus[0] const janDataFolderPath = await getJanDataFolderPath() const engineVersion = TENSORRT_VERSION @@ -235,7 +158,7 @@ export default class TensorRTLLMExtension extends LocalOAIEngine { this.provider, engineVersion, firstGpu.arch, - osInfo.platform === 'win32' ? 'nitro.exe' : 'nitro', + info.osInfo.platform === 'win32' ? 'nitro.exe' : 'nitro', ]) // For now, we just check the executable of nitro x tensor rt @@ -251,10 +174,26 @@ export default class TensorRTLLMExtension extends LocalOAIEngine { return Promise.resolve() } - override inference(data: MessageRequest): void { + override async inference(data: MessageRequest) { if (!this.loadedModel) return // TensorRT LLM Extension supports streaming only if (data.model) data.model.parameters.stream = true super.inference(data) } + + isCompatible(info: SystemInformation): info is Required & { + gpuSetting: { gpus: { arch: string }[] } + } { + const firstGpu = info.gpuSetting?.gpus[0] + return ( + !!info.osInfo && + !!info.gpuSetting && + !!firstGpu && + info.gpuSetting.gpus.length > 0 && + this.supportedPlatform.includes(info.osInfo.platform) && + !!firstGpu.arch && + firstGpu.name.toLowerCase().includes('nvidia') && + this.supportedGpuArch.includes(firstGpu.arch) + ) + } } diff --git a/extensions/tensorrt-llm-extension/src/node/index.ts b/extensions/tensorrt-llm-extension/src/node/index.ts index 4dad464da..eb92c98af 100644 --- a/extensions/tensorrt-llm-extension/src/node/index.ts +++ b/extensions/tensorrt-llm-extension/src/node/index.ts @@ -9,12 +9,14 @@ import { PromptTemplate, } from '@janhq/core/node' import decompress from 'decompress' +import terminate from 'terminate' // Polyfill fetch with retry const fetchRetry = fetchRT(fetch) const supportedPlatform = (): string[] => ['win32', 'linux'] const supportedGpuArch = (): string[] => ['ampere', 'ada'] +const PORT_CHECK_INTERVAL = 100 /** * The response object for model init operation. @@ -64,28 +66,57 @@ async function loadModel( /** * Stops a Engine subprocess. */ -function unloadModel(): Promise { +function unloadModel(): Promise { const controller = new AbortController() setTimeout(() => controller.abort(), 5000) debugLog(`Request to kill engine`) - subprocess?.kill() - return fetch(TERMINATE_ENGINE_URL, { - method: 'DELETE', - signal: controller.signal, - }) - .then(() => { - subprocess = undefined + const killRequest = () => { + return fetch(TERMINATE_ENGINE_URL, { + method: 'DELETE', + signal: controller.signal, }) - .catch(() => {}) // Do nothing with this attempt - .then(() => tcpPortUsed.waitUntilFree(parseInt(ENGINE_PORT), 300, 5000)) // Wait for port available - .then(() => debugLog(`Engine process is terminated`)) - .catch((err) => { - debugLog( - `Could not kill running process on port ${ENGINE_PORT}. Might be another process running on the same port? ${err}` - ) - return { err: 'PORT_NOT_AVAILABLE' } + .then(() => { + subprocess = undefined + }) + .catch(() => {}) // Do nothing with this attempt + .then(() => + tcpPortUsed.waitUntilFree( + parseInt(ENGINE_PORT), + PORT_CHECK_INTERVAL, + 5000 + ) + ) // Wait for port available + .then(() => debugLog(`Engine process is terminated`)) + .catch((err) => { + debugLog( + `Could not kill running process on port ${ENGINE_PORT}. Might be another process running on the same port? ${err}` + ) + throw 'PORT_NOT_AVAILABLE' + }) + } + + if (subprocess?.pid) { + log(`[NITRO]::Debug: Killing PID ${subprocess.pid}`) + const pid = subprocess.pid + return new Promise((resolve, reject) => { + terminate(pid, function (err) { + if (err) { + return killRequest() + } else { + return tcpPortUsed + .waitUntilFree(parseInt(ENGINE_PORT), PORT_CHECK_INTERVAL, 5000) + .then(() => resolve()) + .then(() => log(`[NITRO]::Debug: Nitro process is terminated`)) + .catch(() => { + killRequest() + }) + } + }) }) + } else { + return killRequest() + } } /** * 1. Spawn engine process @@ -97,11 +128,6 @@ async function runEngineAndLoadModel( systemInfo: SystemInformation ) { return unloadModel() - .then((res) => { - if (res?.error) { - throw new Error(res.error) - } - }) .then(() => runEngine(systemInfo)) .then(() => loadModelRequest(settings)) .catch((err) => { @@ -148,7 +174,7 @@ async function runEngine(systemInfo: SystemInformation): Promise { ) } - if (systemInfo.gpuSetting.gpus.length === 0) { + if (systemInfo.gpuSetting?.gpus.length === 0) { return Promise.reject('No GPU found. Please check your GPU setting.') } @@ -164,7 +190,7 @@ async function runEngine(systemInfo: SystemInformation): Promise { ) } - const gpu = systemInfo.gpuSetting.gpus[0] + const gpu = systemInfo.gpuSetting?.gpus[0] if (gpu.name.toLowerCase().includes('nvidia') === false) { return Promise.reject('No Nvidia GPU found. Please check your GPU setting.') } @@ -220,10 +246,12 @@ async function runEngine(systemInfo: SystemInformation): Promise { reject(`child process exited with code ${code}`) }) - tcpPortUsed.waitUntilUsed(parseInt(ENGINE_PORT), 300, 30000).then(() => { - debugLog(`Engine is ready`) - resolve() - }) + tcpPortUsed + .waitUntilUsed(parseInt(ENGINE_PORT), PORT_CHECK_INTERVAL, 30000) + .then(() => { + debugLog(`Engine is ready`) + resolve() + }) }) } diff --git a/extensions/turbo.json b/extensions/turbo.json index ec87739bd..b2e876d58 100644 --- a/extensions/turbo.json +++ b/extensions/turbo.json @@ -2,10 +2,12 @@ "$schema": "https://turbo.build/schema.json", "pipeline": { "build": { - "outputs": [".dist/**", "*.tgz"] + "dependsOn": ["^build"], + "outputs": ["dist/**"] }, "build:publish": { - "dependsOn": ["^build"] + "dependsOn": ["build"], + "outputs": ["**.tgz"] }, "dev": { "cache": false diff --git a/models/gpt-3.5-turbo-16k-0613/model.json b/models/gpt-3.5-turbo-16k-0613/model.json deleted file mode 100644 index ba4aed14d..000000000 --- a/models/gpt-3.5-turbo-16k-0613/model.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "sources": [ - { - "url": "https://openai.com" - } - ], - "id": "gpt-3.5-turbo-16k-0613", - "object": "model", - "name": "OpenAI GPT 3.5 Turbo 16k 0613", - "version": "1.0", - "description": "OpenAI GPT 3.5 Turbo 16k 0613 model is extremely good", - "format": "api", - "settings": {}, - "parameters": { - "max_tokens": 4096, - "temperature": 0.7 - }, - "metadata": { - "author": "OpenAI", - "tags": ["General", "Big Context Length"] - }, - "engine": "openai" -} diff --git a/models/gpt-3.5-turbo/model.json b/models/gpt-3.5-turbo/model.json deleted file mode 100644 index 0359f1950..000000000 --- a/models/gpt-3.5-turbo/model.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "sources": [ - { - "url": "https://openai.com" - } - ], - "id": "gpt-3.5-turbo", - "object": "model", - "name": "OpenAI GPT 3.5 Turbo", - "version": "1.0", - "description": "OpenAI GPT 3.5 Turbo model is extremely good", - "format": "api", - "settings": {}, - "parameters": { - "max_tokens": 4096, - "temperature": 0.7 - }, - "metadata": { - "author": "OpenAI", - "tags": ["General", "Big Context Length"] - }, - "engine": "openai" -} diff --git a/models/gpt-4-vision-preview/model.json b/models/gpt-4-vision-preview/model.json deleted file mode 100644 index a5febf207..000000000 --- a/models/gpt-4-vision-preview/model.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "sources": [ - { - "url": "https://openai.com" - } - ], - "id": "gpt-4-vision-preview", - "object": "model", - "name": "OpenAI GPT 4 with Vision (Preview)", - "version": "1.0", - "description": "OpenAI GPT 4 with Vision model is extremely good in preview", - "format": "api", - "settings": { - "vision_model": true, - "textModel": false - }, - "parameters": { - "max_tokens": 4096, - "temperature": 0.7 - }, - "metadata": { - "author": "OpenAI", - "tags": ["General", "Big Context Length", "Vision"] - }, - "engine": "openai" -} diff --git a/models/gpt-4/model.json b/models/gpt-4/model.json deleted file mode 100644 index 1d99374ef..000000000 --- a/models/gpt-4/model.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "sources": [ - { - "url": "https://openai.com" - } - ], - "id": "gpt-4", - "object": "model", - "name": "OpenAI GPT 4", - "version": "1.0", - "description": "OpenAI GPT 4 model is extremely good", - "format": "api", - "settings": {}, - "parameters": { - "max_tokens": 4096, - "temperature": 0.7 - }, - "metadata": { - "author": "OpenAI", - "tags": ["General", "Big Context Length"] - }, - "engine": "openai" -} diff --git a/models/groq-llama2-70b/model.json b/models/groq-llama2-70b/model.json deleted file mode 100644 index 3f62aa4da..000000000 --- a/models/groq-llama2-70b/model.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "sources": [ - { - "url": "https://groq.com" - } - ], - "id": "llama2-70b-4096", - "object": "model", - "name": "Groq Llama 2 70b", - "version": "1.0", - "description": "Groq Llama 2 70b with supercharged speed!", - "format": "api", - "settings": { - "text_model": false - }, - "parameters": { - "max_tokens": 4096, - "temperature": 0.7, - "top_p": 1, - "stop": null, - "stream": true - }, - "metadata": { - "author": "Meta", - "tags": ["General", "Big Context Length"] - }, - "engine": "groq" -} diff --git a/models/groq-mixtral-8x7b-instruct/model.json b/models/groq-mixtral-8x7b-instruct/model.json deleted file mode 100644 index bd43fde07..000000000 --- a/models/groq-mixtral-8x7b-instruct/model.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "sources": [ - { - "url": "https://groq.com" - } - ], - "id": "mixtral-8x7b-32768", - "object": "model", - "name": "Groq Mixtral 8x7b Instruct", - "version": "1.0", - "description": "Groq Mixtral 8x7b Instruct is Mixtral with supercharged speed!", - "format": "api", - "settings": { - "text_model": false - }, - "parameters": { - "max_tokens": 4096, - "temperature": 0.7, - "top_p": 1, - "stop": null, - "stream": true - }, - "metadata": { - "author": "Mistral", - "tags": ["General", "Big Context Length"] - }, - "engine": "groq" -} diff --git a/models/mistral-ins-7b-q4/cover.png b/models/mistral-ins-7b-q4/cover.png deleted file mode 100644 index 73b82e599..000000000 Binary files a/models/mistral-ins-7b-q4/cover.png and /dev/null differ diff --git a/models/openhermes-neural-7b/cover.png b/models/openhermes-neural-7b/cover.png deleted file mode 100644 index 8976d8449..000000000 Binary files a/models/openhermes-neural-7b/cover.png and /dev/null differ diff --git a/models/trinity-v1.2-7b/cover.png b/models/trinity-v1.2-7b/cover.png deleted file mode 100644 index fbef0bb56..000000000 Binary files a/models/trinity-v1.2-7b/cover.png and /dev/null differ diff --git a/package.json b/package.json index 8f85b7eeb..9e4e7d259 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "pre-install:linux": "find extensions -type f -path \"**/*.tgz\" -exec cp {} pre-install \\;", "pre-install:win32": "powershell -Command \"Get-ChildItem -Path \"extensions\" -Recurse -File -Filter \"*.tgz\" | ForEach-Object { Copy-Item -Path $_.FullName -Destination \"pre-install\" }\"", "pre-install": "run-script-os", - "copy:assets": "cpx \"models/**\" \"electron/models/\" && cpx \"pre-install/*.tgz\" \"electron/pre-install/\" && cpx \"docs/openapi/**\" \"electron/docs/openapi\"", + "copy:assets": "cpx \"pre-install/*.tgz\" \"electron/pre-install/\" && cpx \"docs/openapi/**\" \"electron/docs/openapi\"", "dev:electron": "yarn copy:assets && yarn workspace jan dev", "dev:web": "yarn workspace @janhq/web dev", "dev:server": "yarn copy:assets && yarn workspace @janhq/server dev", diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts new file mode 100644 index 000000000..c8d4af428 --- /dev/null +++ b/server/helpers/logger.ts @@ -0,0 +1,35 @@ +import { log } from '@janhq/core/node' +import { FastifyBaseLogger } from 'fastify' +import { ChildLoggerOptions } from 'fastify/types/logger' +import pino from 'pino' + +export class Logger implements FastifyBaseLogger { + child( + bindings: pino.Bindings, + options?: ChildLoggerOptions | undefined + ): FastifyBaseLogger { + return new Logger() + } + level = 'info' + + silent = () => {} + + info = function (msg: any) { + log(msg) + } + error = function (msg: any) { + log(msg) + } + debug = function (msg: any) { + log(msg) + } + fatal = function (msg: any) { + log(msg) + } + warn = function (msg: any) { + log(msg) + } + trace = function (msg: any) { + log(msg) + } +} diff --git a/server/index.ts b/server/index.ts index 98cc8385d..f82c4f5bc 100644 --- a/server/index.ts +++ b/server/index.ts @@ -1,13 +1,9 @@ import fastify from 'fastify' import dotenv from 'dotenv' -import { - getServerLogPath, - v1Router, - logServer, - getJanExtensionsPath, -} from '@janhq/core/node' +import { v1Router, log, getJanExtensionsPath } from '@janhq/core/node' import { join } from 'path' import tcpPortUsed from 'tcp-port-used' +import { Logger } from './helpers/logger' // Load environment variables dotenv.config() @@ -39,6 +35,7 @@ export interface ServerConfig { isVerboseEnabled?: boolean schemaPath?: string baseDir?: string + prefix?: string storageAdataper?: any } @@ -51,7 +48,7 @@ export const startServer = async (configs?: ServerConfig): Promise => { const inUse = await tcpPortUsed.check(Number(configs.port), configs.host) if (inUse) { const errorMessage = `Port ${configs.port} is already in use.` - logServer(errorMessage) + log(errorMessage, '[SERVER]') throw new Error(errorMessage) } } @@ -61,19 +58,15 @@ export const startServer = async (configs?: ServerConfig): Promise => { hostSetting = configs?.host ?? JAN_API_HOST portSetting = configs?.port ?? JAN_API_PORT corsEnabled = configs?.isCorsEnabled ?? true - const serverLogPath = getServerLogPath() // Start the server try { // Log server start - if (isVerbose) logServer(`Debug: Starting JAN API server...`) + if (isVerbose) log(`Debug: Starting JAN API server...`, '[SERVER]') // Initialize Fastify server with logging server = fastify({ - logger: { - level: 'info', - file: serverLogPath, - }, + logger: new Logger(), }) // Register CORS if enabled @@ -85,6 +78,10 @@ export const startServer = async (configs?: ServerConfig): Promise => { specification: { path: configs?.schemaPath ?? './../docs/openapi/jan.yaml', baseDir: configs?.baseDir ?? './../docs/openapi', + postProcessor: function (swaggerObject: any) { + swaggerObject.servers[0].url = configs?.prefix ?? '/v1' + return swaggerObject + }, }, }) @@ -119,7 +116,7 @@ export const startServer = async (configs?: ServerConfig): Promise => { server.addHook('preHandler', configs.storageAdataper) // Register API routes - await server.register(v1Router, { prefix: '/v1' }) + await server.register(v1Router, { prefix: configs?.prefix ?? '/v1' }) // Start listening for requests await server .listen({ @@ -129,14 +126,15 @@ export const startServer = async (configs?: ServerConfig): Promise => { .then(() => { // Log server listening if (isVerbose) - logServer( - `Debug: JAN API listening at: http://${hostSetting}:${portSetting}` + log( + `Debug: JAN API listening at: http://${hostSetting}:${portSetting}`, + '[SERVER]' ) }) return true } catch (e) { // Log any errors - if (isVerbose) logServer(`Error: ${e}`) + if (isVerbose) log(`Error: ${e}`, '[SERVER]') } return false } @@ -147,11 +145,11 @@ export const startServer = async (configs?: ServerConfig): Promise => { export const stopServer = async () => { try { // Log server stop - if (isVerbose) logServer(`Debug: Server stopped`) + if (isVerbose) log(`Debug: Server stopped`, '[SERVER]') // Stop the server - await server.close() + await server?.close() } catch (e) { // Log any errors - if (isVerbose) logServer(`Error: ${e}`) + if (isVerbose) log(`Error: ${e}`, '[SERVER]') } } diff --git a/web/app/layout.tsx b/web/app/layout.tsx index 6c6fc65ab..a5b227445 100644 --- a/web/app/layout.tsx +++ b/web/app/layout.tsx @@ -2,8 +2,6 @@ import { PropsWithChildren } from 'react' import { Metadata } from 'next' -import Providers from '@/containers/Providers' - import '@/styles/main.scss' export const metadata: Metadata = { @@ -16,8 +14,7 @@ export default function RootLayout({ children }: PropsWithChildren) { return ( -
- {children} + {children} ) diff --git a/web/app/page.tsx b/web/app/page.tsx index ab619f061..363ca2de4 100644 --- a/web/app/page.tsx +++ b/web/app/page.tsx @@ -1,40 +1,11 @@ -'use client' - -import { useAtomValue } from 'jotai' - import BaseLayout from '@/containers/Layout' -import { MainViewState } from '@/constants/screens' - -import ChatScreen from '@/screens/Chat' -import ExploreModelsScreen from '@/screens/ExploreModels' - -import LocalServerScreen from '@/screens/LocalServer' -import SettingsScreen from '@/screens/Settings' - -import { mainViewStateAtom } from '@/helpers/atoms/App.atom' +import Providers from '@/containers/Providers' export default function Page() { - const mainViewState = useAtomValue(mainViewStateAtom) - - let children = null - switch (mainViewState) { - case MainViewState.Hub: - children = - break - - case MainViewState.Settings: - children = - break - - case MainViewState.LocalServer: - children = - break - - default: - children = - break - } - - return {children} + return ( + + + + ) } diff --git a/web/app/search/SelectedText.tsx b/web/app/search/SelectedText.tsx index 2bb90775e..db2209a21 100644 --- a/web/app/search/SelectedText.tsx +++ b/web/app/search/SelectedText.tsx @@ -10,6 +10,7 @@ const SelectedText = ({ onCleared }: { onCleared?: () => void }) => { const containerRef = useRef(null) useEffect(() => { + if (window.core?.api?.quickAskSizeUpdated !== 'function') return if (text.trim().length === 0) { window.core?.api?.quickAskSizeUpdated(0) } else { diff --git a/web/app/search/layout.tsx b/web/app/search/layout.tsx new file mode 100644 index 000000000..f6125f8d2 --- /dev/null +++ b/web/app/search/layout.tsx @@ -0,0 +1,32 @@ +'use client' + +import { useEffect } from 'react' + +import ClipboardListener from '@/containers/Providers/ClipboardListener' + +import JotaiWrapper from '@/containers/Providers/Jotai' +import ThemeWrapper from '@/containers/Providers/Theme' + +import { setupCoreServices } from '@/services/coreService' + +import Search from './page' + +export default function RootLayout() { + useEffect(() => { + setupCoreServices() + }, []) + + return ( + + + + + + + + + + + + ) +} diff --git a/web/containers/DropdownListSidebar/index.tsx b/web/containers/DropdownListSidebar/index.tsx index b0953cdea..bf4d873cc 100644 --- a/web/containers/DropdownListSidebar/index.tsx +++ b/web/containers/DropdownListSidebar/index.tsx @@ -38,7 +38,6 @@ import useUpdateModelParameters from '@/hooks/useUpdateModelParameters' import { toGibibytes } from '@/utils/converter' import ModelLabel from '../ModelLabel' -import OpenAiKeyInput from '../OpenAiKeyInput' import { mainViewStateAtom } from '@/helpers/atoms/App.atom' import { serverEnabledAtom } from '@/helpers/atoms/LocalServer.atom' @@ -144,7 +143,7 @@ const DropdownListSidebar = ({ // Update model parameter to the thread file if (model) - updateModelParameter(activeThread.id, { + updateModelParameter(activeThread, { params: modelParams, modelId: model.id, engine: model.engine, @@ -170,172 +169,165 @@ const DropdownListSidebar = ({ stateModel.model === selectedModel?.id && stateModel.loading return ( - <> -
+ - - - {selectedModelLoading && ( -
- )} - - {selectedModel?.name} - - - - - + + {selectedModelLoading && ( +
+ )} + -
-
    - {engineOptions.map((name, i) => { - return ( -
  • + + + + +
    +
      + {engineOptions.map((name, i) => { + return ( +
    • setIsTabActive(i)} + > + {i === 0 ? ( + + ) : ( + + )} + setIsTabActive(i)} > - {i === 0 ? ( - - ) : ( - - )} - - {name} - -
      • - ) - })} -
    -
    + {name} + +
    • + ) + })} +
+
-
- {downloadedModels.length === 0 ? ( -
-

{`Oops, you don't have a model yet.`}

-
- ) : ( - - <> - {modelOptions.map((x, i) => ( -
+ + + <> + {modelOptions.length === 0 ? ( +
+

{`Oops, you don't have a model yet.`}

+
+ ) : ( + modelOptions.map((x, i) => ( +
+ - -
-
- - {x.name} - -
- - {toGibibytes(x.metadata.size)} - - {x.metadata.size && ( - - )} -
-
+
+ {x.name} +
+ + {toGibibytes(x.metadata.size)} + + {x.metadata.size && ( + + )}
- -
- {x.id} - {clipboard.copied && copyId === x.id ? ( - - ) : ( - { - clipboard.copy(x.id) - setCopyId(x.id) - }} - /> - )}
+ +
+ {x.id} + {clipboard.copied && copyId === x.id ? ( + + ) : ( + { + clipboard.copy(x.id) + setCopyId(x.id) + }} + /> + )}
- ))} - - - )} -
-
- - -
- - - -
- - - +
+ )) + )} + + +
+
+ + +
+ + + +
) } diff --git a/web/containers/GPUDriverPromptModal/index.tsx b/web/containers/GPUDriverPromptModal/index.tsx deleted file mode 100644 index bdcf1b2f8..000000000 --- a/web/containers/GPUDriverPromptModal/index.tsx +++ /dev/null @@ -1,79 +0,0 @@ -import React from 'react' - -import { openExternalUrl } from '@janhq/core' - -import { - ModalClose, - ModalFooter, - ModalContent, - Modal, - ModalTitle, - ModalHeader, - Button, -} from '@janhq/uikit' - -import { useAtom } from 'jotai' - -import { isShowNotificationAtom, useSettings } from '@/hooks/useSettings' - -const GPUDriverPrompt: React.FC = () => { - const [showNotification, setShowNotification] = useAtom( - isShowNotificationAtom - ) - - const { saveSettings } = useSettings() - const onDoNotShowAgainChange = (e: React.ChangeEvent) => { - const isChecked = !e.target.checked - saveSettings({ notify: isChecked }) - } - - const openChanged = () => { - setShowNotification(false) - } - - return ( -
- - - - - Checking for machine that does not meet the requirements. - - -

- It appears that you are missing some dependencies required to run in - GPU mode. Please follow the instructions below for more details{' '} - - openExternalUrl( - 'https://jan.ai/guides/troubleshooting/gpu-not-used/' - ) - } - > - Jan is Not Using GPU - {' '} - . -

-
- - Don't show again -
- -
- - - -
- - - -
- ) -} -export default GPUDriverPrompt diff --git a/web/containers/Layout/BottomBar/InstallingExtension/index.tsx b/web/containers/Layout/BottomBar/InstallingExtension/index.tsx index 05e803881..0d5460955 100644 --- a/web/containers/Layout/BottomBar/InstallingExtension/index.tsx +++ b/web/containers/Layout/BottomBar/InstallingExtension/index.tsx @@ -34,7 +34,7 @@ const InstallingExtension: React.FC = () => { onClick={onClick} >

- Installing Extension + Installing Additional Dependencies

diff --git a/web/containers/Layout/TopBar/index.tsx b/web/containers/Layout/TopBar/index.tsx index 605d8e44d..7f108f15a 100644 --- a/web/containers/Layout/TopBar/index.tsx +++ b/web/containers/Layout/TopBar/index.tsx @@ -68,7 +68,7 @@ const TopBar = () => { } return ( -
+
{mainViewState !== MainViewState.Thread && mainViewState !== MainViewState.LocalServer ? (
diff --git a/web/containers/Layout/index.tsx b/web/containers/Layout/index.tsx index fb08bc6ac..ed4a30477 100644 --- a/web/containers/Layout/index.tsx +++ b/web/containers/Layout/index.tsx @@ -1,4 +1,5 @@ -import React, { PropsWithChildren, useEffect } from 'react' +'use client' +import React, { useEffect } from 'react' import { useTheme } from 'next-themes' @@ -17,17 +18,19 @@ import { getImportModelStageAtom } from '@/hooks/useImportModel' import { SUCCESS_SET_NEW_DESTINATION } from '@/screens/Settings/Advanced/DataFolder' import CancelModelImportModal from '@/screens/Settings/CancelModelImportModal' +import ChooseWhatToImportModal from '@/screens/Settings/ChooseWhatToImportModal' import EditModelInfoModal from '@/screens/Settings/EditModelInfoModal' import ImportModelOptionModal from '@/screens/Settings/ImportModelOptionModal' import ImportingModelModal from '@/screens/Settings/ImportingModelModal' import SelectingModelModal from '@/screens/Settings/SelectingModelModal' +import MainViewContainer from '../MainViewContainer' + import InstallingExtensionModal from './BottomBar/InstallingExtension/InstallingExtensionModal' import { mainViewStateAtom } from '@/helpers/atoms/App.atom' -const BaseLayout = (props: PropsWithChildren) => { - const { children } = props +const BaseLayout = () => { const [mainViewState, setMainViewState] = useAtom(mainViewStateAtom) const importModelStage = useAtomValue(getImportModelStageAtom) const { theme, setTheme } = useTheme() @@ -60,7 +63,7 @@ const BaseLayout = (props: PropsWithChildren) => { }, }} > - {children} +
@@ -70,6 +73,7 @@ const BaseLayout = (props: PropsWithChildren) => { {importModelStage === 'IMPORTING_MODEL' && } {importModelStage === 'EDIT_MODEL_INFO' && } {importModelStage === 'CONFIRM_CANCEL' && } +
) diff --git a/web/containers/Loader/ModelReload.tsx b/web/containers/Loader/ModelReload.tsx index a432927aa..44fbb9ab9 100644 --- a/web/containers/Loader/ModelReload.tsx +++ b/web/containers/Loader/ModelReload.tsx @@ -41,7 +41,7 @@ export default function ModelReload() { style={{ width: `${loader}%` }} /> - Reloading model {stateModel.model} + Reloading model {stateModel.model?.id}
diff --git a/web/containers/Loader/ModelStart.tsx b/web/containers/Loader/ModelStart.tsx index 7002c7b40..f7bc04481 100644 --- a/web/containers/Loader/ModelStart.tsx +++ b/web/containers/Loader/ModelStart.tsx @@ -44,7 +44,7 @@ export default function ModelStart() { {stateModel.state === 'start' ? 'Starting' : 'Stopping'}  model  - {stateModel.model} + {stateModel.model?.id}
diff --git a/web/containers/MainViewContainer/index.tsx b/web/containers/MainViewContainer/index.tsx new file mode 100644 index 000000000..1de48642a --- /dev/null +++ b/web/containers/MainViewContainer/index.tsx @@ -0,0 +1,37 @@ +import { useAtomValue } from 'jotai' + +import { MainViewState } from '@/constants/screens' + +import ChatScreen from '@/screens/Chat' +import ExploreModelsScreen from '@/screens/ExploreModels' +import LocalServerScreen from '@/screens/LocalServer' +import SettingsScreen from '@/screens/Settings' + +import { mainViewStateAtom } from '@/helpers/atoms/App.atom' + +const MainViewContainer: React.FC = () => { + const mainViewState = useAtomValue(mainViewStateAtom) + + let children = null + switch (mainViewState) { + case MainViewState.Hub: + children = + break + + case MainViewState.Settings: + children = + break + + case MainViewState.LocalServer: + children = + break + + default: + children = + break + } + + return children +} + +export default MainViewContainer diff --git a/web/containers/ModalTroubleShoot/AppLogs.tsx b/web/containers/ModalTroubleShoot/AppLogs.tsx index bd2dd2ceb..60151cc9d 100644 --- a/web/containers/ModalTroubleShoot/AppLogs.tsx +++ b/web/containers/ModalTroubleShoot/AppLogs.tsx @@ -2,14 +2,16 @@ import React, { useEffect, useState } from 'react' import { Button } from '@janhq/uikit' -import { CopyIcon, CheckIcon } from 'lucide-react' +import { CopyIcon, CheckIcon, FolderIcon } from 'lucide-react' import { useClipboard } from '@/hooks/useClipboard' import { useLogs } from '@/hooks/useLogs' +import { usePath } from '@/hooks/usePath' const AppLogs = () => { const { getLogs } = useLogs() const [logs, setLogs] = useState([]) + const { onRevealInFinder } = usePath() useEffect(() => { getLogs('app').then((log) => { @@ -26,27 +28,41 @@ const AppLogs = () => { return ( <>
- +
+ + +
{logs.length > 1 ? ( diff --git a/web/containers/ModalTroubleShoot/DeviceSpecs.tsx b/web/containers/ModalTroubleShoot/DeviceSpecs.tsx index a657ffd5d..792466df3 100644 --- a/web/containers/ModalTroubleShoot/DeviceSpecs.tsx +++ b/web/containers/ModalTroubleShoot/DeviceSpecs.tsx @@ -32,7 +32,7 @@ const DeviceSpecs = () => { themes="outline" className="bg-white dark:bg-secondary/50" onClick={() => { - clipboard.copy(logs ?? '') + clipboard.copy(logs.join('\n') ?? '') }} >
diff --git a/web/containers/ModelConfigInput/index.tsx b/web/containers/ModelConfigInput/index.tsx index d573a0bf9..0c16c916c 100644 --- a/web/containers/ModelConfigInput/index.tsx +++ b/web/containers/ModelConfigInput/index.tsx @@ -26,33 +26,31 @@ const ModelConfigInput: React.FC = ({ description, placeholder, onValueChanged, -}) => { - return ( -
-
-

- {title} -

- - - - - - - {description} - - - - -
-